|
|
@@ -1,5 +1,5 @@
|
|
|
/*
|
|
|
- * Copyright (c) 2006 - 2009 Intel-NE, Inc. All rights reserved.
|
|
|
+ * Copyright (c) 2006 - 2009 Intel Corporation. All rights reserved.
|
|
|
*
|
|
|
* This software is available to you under a choice of one of two
|
|
|
* licenses. You may choose to be licensed under the terms of the GNU
|
|
|
@@ -275,342 +275,236 @@ static int nes_bind_mw(struct ib_qp *ibqp, struct ib_mw *ibmw,
|
|
|
}
|
|
|
|
|
|
|
|
|
-/**
|
|
|
- * nes_alloc_fmr
|
|
|
+/*
|
|
|
+ * nes_alloc_fast_mr
|
|
|
*/
|
|
|
-static struct ib_fmr *nes_alloc_fmr(struct ib_pd *ibpd,
|
|
|
- int ibmr_access_flags,
|
|
|
- struct ib_fmr_attr *ibfmr_attr)
|
|
|
+static int alloc_fast_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd,
|
|
|
+ u32 stag, u32 page_count)
|
|
|
{
|
|
|
- unsigned long flags;
|
|
|
- struct nes_pd *nespd = to_nespd(ibpd);
|
|
|
- struct nes_vnic *nesvnic = to_nesvnic(ibpd->device);
|
|
|
- struct nes_device *nesdev = nesvnic->nesdev;
|
|
|
- struct nes_adapter *nesadapter = nesdev->nesadapter;
|
|
|
- struct nes_fmr *nesfmr;
|
|
|
- struct nes_cqp_request *cqp_request;
|
|
|
struct nes_hw_cqp_wqe *cqp_wqe;
|
|
|
+ struct nes_cqp_request *cqp_request;
|
|
|
+ unsigned long flags;
|
|
|
int ret;
|
|
|
- u32 stag;
|
|
|
- u32 stag_index = 0;
|
|
|
- u32 next_stag_index = 0;
|
|
|
- u32 driver_key = 0;
|
|
|
+ struct nes_adapter *nesadapter = nesdev->nesadapter;
|
|
|
u32 opcode = 0;
|
|
|
- u8 stag_key = 0;
|
|
|
- int i=0;
|
|
|
- struct nes_vpbl vpbl;
|
|
|
-
|
|
|
- get_random_bytes(&next_stag_index, sizeof(next_stag_index));
|
|
|
- stag_key = (u8)next_stag_index;
|
|
|
-
|
|
|
- driver_key = 0;
|
|
|
-
|
|
|
- next_stag_index >>= 8;
|
|
|
- next_stag_index %= nesadapter->max_mr;
|
|
|
-
|
|
|
- ret = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs,
|
|
|
- nesadapter->max_mr, &stag_index, &next_stag_index);
|
|
|
- if (ret) {
|
|
|
- goto failed_resource_alloc;
|
|
|
- }
|
|
|
-
|
|
|
- nesfmr = kzalloc(sizeof(*nesfmr), GFP_KERNEL);
|
|
|
- if (!nesfmr) {
|
|
|
- ret = -ENOMEM;
|
|
|
- goto failed_fmr_alloc;
|
|
|
- }
|
|
|
-
|
|
|
- nesfmr->nesmr.mode = IWNES_MEMREG_TYPE_FMR;
|
|
|
- if (ibfmr_attr->max_pages == 1) {
|
|
|
- /* use zero length PBL */
|
|
|
- nesfmr->nesmr.pbl_4k = 0;
|
|
|
- nesfmr->nesmr.pbls_used = 0;
|
|
|
- } else if (ibfmr_attr->max_pages <= 32) {
|
|
|
- /* use PBL 256 */
|
|
|
- nesfmr->nesmr.pbl_4k = 0;
|
|
|
- nesfmr->nesmr.pbls_used = 1;
|
|
|
- } else if (ibfmr_attr->max_pages <= 512) {
|
|
|
- /* use 4K PBLs */
|
|
|
- nesfmr->nesmr.pbl_4k = 1;
|
|
|
- nesfmr->nesmr.pbls_used = 1;
|
|
|
- } else {
|
|
|
- /* use two level 4K PBLs */
|
|
|
- /* add support for two level 256B PBLs */
|
|
|
- nesfmr->nesmr.pbl_4k = 1;
|
|
|
- nesfmr->nesmr.pbls_used = 1 + (ibfmr_attr->max_pages >> 9) +
|
|
|
- ((ibfmr_attr->max_pages & 511) ? 1 : 0);
|
|
|
- }
|
|
|
- /* Register the region with the adapter */
|
|
|
- spin_lock_irqsave(&nesadapter->pbl_lock, flags);
|
|
|
-
|
|
|
- /* track PBL resources */
|
|
|
- if (nesfmr->nesmr.pbls_used != 0) {
|
|
|
- if (nesfmr->nesmr.pbl_4k) {
|
|
|
- if (nesfmr->nesmr.pbls_used > nesadapter->free_4kpbl) {
|
|
|
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
|
|
|
- ret = -ENOMEM;
|
|
|
- goto failed_vpbl_avail;
|
|
|
- } else {
|
|
|
- nesadapter->free_4kpbl -= nesfmr->nesmr.pbls_used;
|
|
|
- }
|
|
|
- } else {
|
|
|
- if (nesfmr->nesmr.pbls_used > nesadapter->free_256pbl) {
|
|
|
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
|
|
|
- ret = -ENOMEM;
|
|
|
- goto failed_vpbl_avail;
|
|
|
- } else {
|
|
|
- nesadapter->free_256pbl -= nesfmr->nesmr.pbls_used;
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /* one level pbl */
|
|
|
- if (nesfmr->nesmr.pbls_used == 0) {
|
|
|
- nesfmr->root_vpbl.pbl_vbase = NULL;
|
|
|
- nes_debug(NES_DBG_MR, "zero level pbl \n");
|
|
|
- } else if (nesfmr->nesmr.pbls_used == 1) {
|
|
|
- /* can change it to kmalloc & dma_map_single */
|
|
|
- nesfmr->root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096,
|
|
|
- &nesfmr->root_vpbl.pbl_pbase);
|
|
|
- if (!nesfmr->root_vpbl.pbl_vbase) {
|
|
|
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
|
|
|
- ret = -ENOMEM;
|
|
|
- goto failed_vpbl_alloc;
|
|
|
- }
|
|
|
- nesfmr->leaf_pbl_cnt = 0;
|
|
|
- nes_debug(NES_DBG_MR, "one level pbl, root_vpbl.pbl_vbase=%p \n",
|
|
|
- nesfmr->root_vpbl.pbl_vbase);
|
|
|
- }
|
|
|
- /* two level pbl */
|
|
|
- else {
|
|
|
- nesfmr->root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 8192,
|
|
|
- &nesfmr->root_vpbl.pbl_pbase);
|
|
|
- if (!nesfmr->root_vpbl.pbl_vbase) {
|
|
|
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
|
|
|
- ret = -ENOMEM;
|
|
|
- goto failed_vpbl_alloc;
|
|
|
- }
|
|
|
-
|
|
|
- nesfmr->leaf_pbl_cnt = nesfmr->nesmr.pbls_used-1;
|
|
|
- nesfmr->root_vpbl.leaf_vpbl = kzalloc(sizeof(*nesfmr->root_vpbl.leaf_vpbl)*1024, GFP_ATOMIC);
|
|
|
- if (!nesfmr->root_vpbl.leaf_vpbl) {
|
|
|
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
|
|
|
- ret = -ENOMEM;
|
|
|
- goto failed_leaf_vpbl_alloc;
|
|
|
- }
|
|
|
-
|
|
|
- nes_debug(NES_DBG_MR, "two level pbl, root_vpbl.pbl_vbase=%p"
|
|
|
- " leaf_pbl_cnt=%d root_vpbl.leaf_vpbl=%p\n",
|
|
|
- nesfmr->root_vpbl.pbl_vbase, nesfmr->leaf_pbl_cnt, nesfmr->root_vpbl.leaf_vpbl);
|
|
|
-
|
|
|
- for (i=0; i<nesfmr->leaf_pbl_cnt; i++)
|
|
|
- nesfmr->root_vpbl.leaf_vpbl[i].pbl_vbase = NULL;
|
|
|
-
|
|
|
- for (i=0; i<nesfmr->leaf_pbl_cnt; i++) {
|
|
|
- vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096,
|
|
|
- &vpbl.pbl_pbase);
|
|
|
-
|
|
|
- if (!vpbl.pbl_vbase) {
|
|
|
- ret = -ENOMEM;
|
|
|
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
|
|
|
- goto failed_leaf_vpbl_pages_alloc;
|
|
|
- }
|
|
|
-
|
|
|
- nesfmr->root_vpbl.pbl_vbase[i].pa_low = cpu_to_le32((u32)vpbl.pbl_pbase);
|
|
|
- nesfmr->root_vpbl.pbl_vbase[i].pa_high = cpu_to_le32((u32)((((u64)vpbl.pbl_pbase)>>32)));
|
|
|
- nesfmr->root_vpbl.leaf_vpbl[i] = vpbl;
|
|
|
-
|
|
|
- nes_debug(NES_DBG_MR, "pbase_low=0x%x, pbase_high=0x%x, vpbl=%p\n",
|
|
|
- nesfmr->root_vpbl.pbl_vbase[i].pa_low,
|
|
|
- nesfmr->root_vpbl.pbl_vbase[i].pa_high,
|
|
|
- &nesfmr->root_vpbl.leaf_vpbl[i]);
|
|
|
- }
|
|
|
- }
|
|
|
- nesfmr->ib_qp = NULL;
|
|
|
- nesfmr->access_rights =0;
|
|
|
+ u16 major_code;
|
|
|
+ u64 region_length = page_count * PAGE_SIZE;
|
|
|
|
|
|
- stag = stag_index << 8;
|
|
|
- stag |= driver_key;
|
|
|
- stag += (u32)stag_key;
|
|
|
|
|
|
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
|
|
|
cqp_request = nes_get_cqp_request(nesdev);
|
|
|
if (cqp_request == NULL) {
|
|
|
nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n");
|
|
|
- ret = -ENOMEM;
|
|
|
- goto failed_leaf_vpbl_pages_alloc;
|
|
|
+ return -ENOMEM;
|
|
|
}
|
|
|
+ nes_debug(NES_DBG_MR, "alloc_fast_reg_mr: page_count = %d, "
|
|
|
+ "region_length = %llu\n",
|
|
|
+ page_count, region_length);
|
|
|
cqp_request->waiting = 1;
|
|
|
cqp_wqe = &cqp_request->cqp_wqe;
|
|
|
|
|
|
- nes_debug(NES_DBG_MR, "Registering STag 0x%08X, index = 0x%08X\n",
|
|
|
- stag, stag_index);
|
|
|
-
|
|
|
- opcode = NES_CQP_ALLOCATE_STAG | NES_CQP_STAG_VA_TO | NES_CQP_STAG_MR;
|
|
|
-
|
|
|
- if (nesfmr->nesmr.pbl_4k == 1)
|
|
|
- opcode |= NES_CQP_STAG_PBL_BLK_SIZE;
|
|
|
-
|
|
|
- if (ibmr_access_flags & IB_ACCESS_REMOTE_WRITE) {
|
|
|
- opcode |= NES_CQP_STAG_RIGHTS_REMOTE_WRITE |
|
|
|
- NES_CQP_STAG_RIGHTS_LOCAL_WRITE | NES_CQP_STAG_REM_ACC_EN;
|
|
|
- nesfmr->access_rights |=
|
|
|
- NES_CQP_STAG_RIGHTS_REMOTE_WRITE | NES_CQP_STAG_RIGHTS_LOCAL_WRITE |
|
|
|
- NES_CQP_STAG_REM_ACC_EN;
|
|
|
+ spin_lock_irqsave(&nesadapter->pbl_lock, flags);
|
|
|
+ if (nesadapter->free_4kpbl > 0) {
|
|
|
+ nesadapter->free_4kpbl--;
|
|
|
+ spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
|
|
|
+ } else {
|
|
|
+ /* No 4kpbl's available: */
|
|
|
+ spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
|
|
|
+ nes_debug(NES_DBG_MR, "Out of Pbls\n");
|
|
|
+ nes_free_cqp_request(nesdev, cqp_request);
|
|
|
+ return -ENOMEM;
|
|
|
}
|
|
|
|
|
|
- if (ibmr_access_flags & IB_ACCESS_REMOTE_READ) {
|
|
|
- opcode |= NES_CQP_STAG_RIGHTS_REMOTE_READ |
|
|
|
- NES_CQP_STAG_RIGHTS_LOCAL_READ | NES_CQP_STAG_REM_ACC_EN;
|
|
|
- nesfmr->access_rights |=
|
|
|
- NES_CQP_STAG_RIGHTS_REMOTE_READ | NES_CQP_STAG_RIGHTS_LOCAL_READ |
|
|
|
- NES_CQP_STAG_REM_ACC_EN;
|
|
|
- }
|
|
|
+ opcode = NES_CQP_ALLOCATE_STAG | NES_CQP_STAG_MR |
|
|
|
+ NES_CQP_STAG_PBL_BLK_SIZE | NES_CQP_STAG_VA_TO |
|
|
|
+ NES_CQP_STAG_REM_ACC_EN;
|
|
|
+ /*
|
|
|
+ * The current OFED API does not support the zero based TO option.
|
|
|
+ * If added then need to changed the NES_CQP_STAG_VA* option. Also,
|
|
|
+ * the API does not support that ability to have the MR set for local
|
|
|
+ * access only when created and not allow the SQ op to override. Given
|
|
|
+ * this the remote enable must be set here.
|
|
|
+ */
|
|
|
|
|
|
nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
|
|
|
set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode);
|
|
|
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX, (nespd->pd_id & 0x00007fff));
|
|
|
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag);
|
|
|
+ set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX, 1);
|
|
|
|
|
|
- cqp_wqe->wqe_words[NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX] =
|
|
|
- cpu_to_le32((nesfmr->nesmr.pbls_used>1) ?
|
|
|
- (nesfmr->nesmr.pbls_used-1) : nesfmr->nesmr.pbls_used);
|
|
|
+ cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] =
|
|
|
+ cpu_to_le32((u32)(region_length >> 8) & 0xff000000);
|
|
|
+ cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] |=
|
|
|
+ cpu_to_le32(nespd->pd_id & 0x00007fff);
|
|
|
+
|
|
|
+ set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag);
|
|
|
+ set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_VA_LOW_IDX, 0);
|
|
|
+ set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_LOW_IDX, 0);
|
|
|
+ set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, 0);
|
|
|
+ set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_LEN_IDX, (page_count * 8));
|
|
|
+ cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE);
|
|
|
+ barrier();
|
|
|
|
|
|
atomic_set(&cqp_request->refcount, 2);
|
|
|
nes_post_cqp_request(nesdev, cqp_request);
|
|
|
|
|
|
/* Wait for CQP */
|
|
|
- ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
|
|
|
- NES_EVENT_TIMEOUT);
|
|
|
- nes_debug(NES_DBG_MR, "Register STag 0x%08X completed, wait_event_timeout ret = %u,"
|
|
|
- " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
|
|
|
- stag, ret, cqp_request->major_code, cqp_request->minor_code);
|
|
|
-
|
|
|
- if ((!ret) || (cqp_request->major_code)) {
|
|
|
- nes_put_cqp_request(nesdev, cqp_request);
|
|
|
- ret = (!ret) ? -ETIME : -EIO;
|
|
|
- goto failed_leaf_vpbl_pages_alloc;
|
|
|
- }
|
|
|
+ ret = wait_event_timeout(cqp_request->waitq,
|
|
|
+ (0 != cqp_request->request_done),
|
|
|
+ NES_EVENT_TIMEOUT);
|
|
|
+
|
|
|
+ nes_debug(NES_DBG_MR, "Allocate STag 0x%08X completed, "
|
|
|
+ "wait_event_timeout ret = %u, CQP Major:Minor codes = "
|
|
|
+ "0x%04X:0x%04X.\n", stag, ret, cqp_request->major_code,
|
|
|
+ cqp_request->minor_code);
|
|
|
+ major_code = cqp_request->major_code;
|
|
|
nes_put_cqp_request(nesdev, cqp_request);
|
|
|
- nesfmr->nesmr.ibfmr.lkey = stag;
|
|
|
- nesfmr->nesmr.ibfmr.rkey = stag;
|
|
|
- nesfmr->attr = *ibfmr_attr;
|
|
|
-
|
|
|
- return &nesfmr->nesmr.ibfmr;
|
|
|
-
|
|
|
- failed_leaf_vpbl_pages_alloc:
|
|
|
- /* unroll all allocated pages */
|
|
|
- for (i=0; i<nesfmr->leaf_pbl_cnt; i++) {
|
|
|
- if (nesfmr->root_vpbl.leaf_vpbl[i].pbl_vbase) {
|
|
|
- pci_free_consistent(nesdev->pcidev, 4096, nesfmr->root_vpbl.leaf_vpbl[i].pbl_vbase,
|
|
|
- nesfmr->root_vpbl.leaf_vpbl[i].pbl_pbase);
|
|
|
- }
|
|
|
- }
|
|
|
- if (nesfmr->root_vpbl.leaf_vpbl)
|
|
|
- kfree(nesfmr->root_vpbl.leaf_vpbl);
|
|
|
|
|
|
- failed_leaf_vpbl_alloc:
|
|
|
- if (nesfmr->leaf_pbl_cnt == 0) {
|
|
|
- if (nesfmr->root_vpbl.pbl_vbase)
|
|
|
- pci_free_consistent(nesdev->pcidev, 4096, nesfmr->root_vpbl.pbl_vbase,
|
|
|
- nesfmr->root_vpbl.pbl_pbase);
|
|
|
- } else
|
|
|
- pci_free_consistent(nesdev->pcidev, 8192, nesfmr->root_vpbl.pbl_vbase,
|
|
|
- nesfmr->root_vpbl.pbl_pbase);
|
|
|
-
|
|
|
- failed_vpbl_alloc:
|
|
|
- if (nesfmr->nesmr.pbls_used != 0) {
|
|
|
+ if (!ret || major_code) {
|
|
|
spin_lock_irqsave(&nesadapter->pbl_lock, flags);
|
|
|
- if (nesfmr->nesmr.pbl_4k)
|
|
|
- nesadapter->free_4kpbl += nesfmr->nesmr.pbls_used;
|
|
|
- else
|
|
|
- nesadapter->free_256pbl += nesfmr->nesmr.pbls_used;
|
|
|
+ nesadapter->free_4kpbl++;
|
|
|
spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
|
|
|
}
|
|
|
|
|
|
-failed_vpbl_avail:
|
|
|
- kfree(nesfmr);
|
|
|
-
|
|
|
- failed_fmr_alloc:
|
|
|
- nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
|
|
|
-
|
|
|
- failed_resource_alloc:
|
|
|
- return ERR_PTR(ret);
|
|
|
+ if (!ret)
|
|
|
+ return -ETIME;
|
|
|
+ else if (major_code)
|
|
|
+ return -EIO;
|
|
|
+ return 0;
|
|
|
}
|
|
|
|
|
|
-
|
|
|
-/**
|
|
|
- * nes_dealloc_fmr
|
|
|
+/*
|
|
|
+ * nes_alloc_fast_reg_mr
|
|
|
*/
|
|
|
-static int nes_dealloc_fmr(struct ib_fmr *ibfmr)
|
|
|
+struct ib_mr *nes_alloc_fast_reg_mr(struct ib_pd *ibpd, int max_page_list_len)
|
|
|
{
|
|
|
- unsigned long flags;
|
|
|
- struct nes_mr *nesmr = to_nesmr_from_ibfmr(ibfmr);
|
|
|
- struct nes_fmr *nesfmr = to_nesfmr(nesmr);
|
|
|
- struct nes_vnic *nesvnic = to_nesvnic(ibfmr->device);
|
|
|
+ struct nes_pd *nespd = to_nespd(ibpd);
|
|
|
+ struct nes_vnic *nesvnic = to_nesvnic(ibpd->device);
|
|
|
struct nes_device *nesdev = nesvnic->nesdev;
|
|
|
struct nes_adapter *nesadapter = nesdev->nesadapter;
|
|
|
- int i = 0;
|
|
|
- int rc;
|
|
|
|
|
|
- /* free the resources */
|
|
|
- if (nesfmr->leaf_pbl_cnt == 0) {
|
|
|
- /* single PBL case */
|
|
|
- if (nesfmr->root_vpbl.pbl_vbase)
|
|
|
- pci_free_consistent(nesdev->pcidev, 4096, nesfmr->root_vpbl.pbl_vbase,
|
|
|
- nesfmr->root_vpbl.pbl_pbase);
|
|
|
- } else {
|
|
|
- for (i = 0; i < nesfmr->leaf_pbl_cnt; i++) {
|
|
|
- pci_free_consistent(nesdev->pcidev, 4096, nesfmr->root_vpbl.leaf_vpbl[i].pbl_vbase,
|
|
|
- nesfmr->root_vpbl.leaf_vpbl[i].pbl_pbase);
|
|
|
- }
|
|
|
- kfree(nesfmr->root_vpbl.leaf_vpbl);
|
|
|
- pci_free_consistent(nesdev->pcidev, 8192, nesfmr->root_vpbl.pbl_vbase,
|
|
|
- nesfmr->root_vpbl.pbl_pbase);
|
|
|
- }
|
|
|
- nesmr->ibmw.device = ibfmr->device;
|
|
|
- nesmr->ibmw.pd = ibfmr->pd;
|
|
|
- nesmr->ibmw.rkey = ibfmr->rkey;
|
|
|
- nesmr->ibmw.uobject = NULL;
|
|
|
+ u32 next_stag_index;
|
|
|
+ u8 stag_key = 0;
|
|
|
+ u32 driver_key = 0;
|
|
|
+ int err = 0;
|
|
|
+ u32 stag_index = 0;
|
|
|
+ struct nes_mr *nesmr;
|
|
|
+ u32 stag;
|
|
|
+ int ret;
|
|
|
+ struct ib_mr *ibmr;
|
|
|
+/*
|
|
|
+ * Note: Set to always use a fixed length single page entry PBL. This is to allow
|
|
|
+ * for the fast_reg_mr operation to always know the size of the PBL.
|
|
|
+ */
|
|
|
+ if (max_page_list_len > (NES_4K_PBL_CHUNK_SIZE / sizeof(u64)))
|
|
|
+ return ERR_PTR(-E2BIG);
|
|
|
|
|
|
- rc = nes_dealloc_mw(&nesmr->ibmw);
|
|
|
+ get_random_bytes(&next_stag_index, sizeof(next_stag_index));
|
|
|
+ stag_key = (u8)next_stag_index;
|
|
|
+ next_stag_index >>= 8;
|
|
|
+ next_stag_index %= nesadapter->max_mr;
|
|
|
|
|
|
- if ((rc == 0) && (nesfmr->nesmr.pbls_used != 0)) {
|
|
|
- spin_lock_irqsave(&nesadapter->pbl_lock, flags);
|
|
|
- if (nesfmr->nesmr.pbl_4k) {
|
|
|
- nesadapter->free_4kpbl += nesfmr->nesmr.pbls_used;
|
|
|
- WARN_ON(nesadapter->free_4kpbl > nesadapter->max_4kpbl);
|
|
|
- } else {
|
|
|
- nesadapter->free_256pbl += nesfmr->nesmr.pbls_used;
|
|
|
- WARN_ON(nesadapter->free_256pbl > nesadapter->max_256pbl);
|
|
|
- }
|
|
|
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
|
|
|
+ err = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs,
|
|
|
+ nesadapter->max_mr, &stag_index,
|
|
|
+ &next_stag_index);
|
|
|
+ if (err)
|
|
|
+ return ERR_PTR(err);
|
|
|
+
|
|
|
+ nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL);
|
|
|
+ if (!nesmr) {
|
|
|
+ nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
|
|
|
+ return ERR_PTR(-ENOMEM);
|
|
|
}
|
|
|
|
|
|
- return rc;
|
|
|
-}
|
|
|
+ stag = stag_index << 8;
|
|
|
+ stag |= driver_key;
|
|
|
+ stag += (u32)stag_key;
|
|
|
|
|
|
+ nes_debug(NES_DBG_MR, "Allocating STag 0x%08X index = 0x%08X\n",
|
|
|
+ stag, stag_index);
|
|
|
|
|
|
-/**
|
|
|
- * nes_map_phys_fmr
|
|
|
+ ret = alloc_fast_reg_mr(nesdev, nespd, stag, max_page_list_len);
|
|
|
+
|
|
|
+ if (ret == 0) {
|
|
|
+ nesmr->ibmr.rkey = stag;
|
|
|
+ nesmr->ibmr.lkey = stag;
|
|
|
+ nesmr->mode = IWNES_MEMREG_TYPE_FMEM;
|
|
|
+ ibmr = &nesmr->ibmr;
|
|
|
+ } else {
|
|
|
+ kfree(nesmr);
|
|
|
+ nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
|
|
|
+ ibmr = ERR_PTR(-ENOMEM);
|
|
|
+ }
|
|
|
+ return ibmr;
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * nes_alloc_fast_reg_page_list
|
|
|
*/
|
|
|
-static int nes_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
|
|
|
- int list_len, u64 iova)
|
|
|
+static struct ib_fast_reg_page_list *nes_alloc_fast_reg_page_list(
|
|
|
+ struct ib_device *ibdev,
|
|
|
+ int page_list_len)
|
|
|
{
|
|
|
- return 0;
|
|
|
-}
|
|
|
+ struct nes_vnic *nesvnic = to_nesvnic(ibdev);
|
|
|
+ struct nes_device *nesdev = nesvnic->nesdev;
|
|
|
+ struct ib_fast_reg_page_list *pifrpl;
|
|
|
+ struct nes_ib_fast_reg_page_list *pnesfrpl;
|
|
|
|
|
|
+ if (page_list_len > (NES_4K_PBL_CHUNK_SIZE / sizeof(u64)))
|
|
|
+ return ERR_PTR(-E2BIG);
|
|
|
+ /*
|
|
|
+ * Allocate the ib_fast_reg_page_list structure, the
|
|
|
+ * nes_fast_bpl structure, and the PLB table.
|
|
|
+ */
|
|
|
+ pnesfrpl = kmalloc(sizeof(struct nes_ib_fast_reg_page_list) +
|
|
|
+ page_list_len * sizeof(u64), GFP_KERNEL);
|
|
|
+
|
|
|
+ if (!pnesfrpl)
|
|
|
+ return ERR_PTR(-ENOMEM);
|
|
|
|
|
|
-/**
|
|
|
- * nes_unmap_frm
|
|
|
+ pifrpl = &pnesfrpl->ibfrpl;
|
|
|
+ pifrpl->page_list = &pnesfrpl->pbl;
|
|
|
+ pifrpl->max_page_list_len = page_list_len;
|
|
|
+ /*
|
|
|
+ * Allocate the WQE PBL
|
|
|
+ */
|
|
|
+ pnesfrpl->nes_wqe_pbl.kva = pci_alloc_consistent(nesdev->pcidev,
|
|
|
+ page_list_len * sizeof(u64),
|
|
|
+ &pnesfrpl->nes_wqe_pbl.paddr);
|
|
|
+
|
|
|
+ if (!pnesfrpl->nes_wqe_pbl.kva) {
|
|
|
+ kfree(pnesfrpl);
|
|
|
+ return ERR_PTR(-ENOMEM);
|
|
|
+ }
|
|
|
+ nes_debug(NES_DBG_MR, "nes_alloc_fast_reg_pbl: nes_frpl = %p, "
|
|
|
+ "ibfrpl = %p, ibfrpl.page_list = %p, pbl.kva = %p, "
|
|
|
+ "pbl.paddr= %p\n", pnesfrpl, &pnesfrpl->ibfrpl,
|
|
|
+ pnesfrpl->ibfrpl.page_list, pnesfrpl->nes_wqe_pbl.kva,
|
|
|
+ (void *)pnesfrpl->nes_wqe_pbl.paddr);
|
|
|
+
|
|
|
+ return pifrpl;
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * nes_free_fast_reg_page_list
|
|
|
*/
|
|
|
-static int nes_unmap_fmr(struct list_head *ibfmr_list)
|
|
|
+static void nes_free_fast_reg_page_list(struct ib_fast_reg_page_list *pifrpl)
|
|
|
{
|
|
|
- return 0;
|
|
|
+ struct nes_vnic *nesvnic = to_nesvnic(pifrpl->device);
|
|
|
+ struct nes_device *nesdev = nesvnic->nesdev;
|
|
|
+ struct nes_ib_fast_reg_page_list *pnesfrpl;
|
|
|
+
|
|
|
+ pnesfrpl = container_of(pifrpl, struct nes_ib_fast_reg_page_list, ibfrpl);
|
|
|
+ /*
|
|
|
+ * Free the WQE PBL.
|
|
|
+ */
|
|
|
+ pci_free_consistent(nesdev->pcidev,
|
|
|
+ pifrpl->max_page_list_len * sizeof(u64),
|
|
|
+ pnesfrpl->nes_wqe_pbl.kva,
|
|
|
+ pnesfrpl->nes_wqe_pbl.paddr);
|
|
|
+ /*
|
|
|
+ * Free the PBL structure
|
|
|
+ */
|
|
|
+ kfree(pnesfrpl);
|
|
|
}
|
|
|
|
|
|
-
|
|
|
-
|
|
|
/**
|
|
|
* nes_query_device
|
|
|
*/
|
|
|
@@ -633,23 +527,23 @@ static int nes_query_device(struct ib_device *ibdev, struct ib_device_attr *prop
|
|
|
props->max_qp_wr = nesdev->nesadapter->max_qp_wr - 2;
|
|
|
props->max_sge = nesdev->nesadapter->max_sge;
|
|
|
props->max_cq = nesibdev->max_cq;
|
|
|
- props->max_cqe = nesdev->nesadapter->max_cqe - 1;
|
|
|
+ props->max_cqe = nesdev->nesadapter->max_cqe;
|
|
|
props->max_mr = nesibdev->max_mr;
|
|
|
props->max_mw = nesibdev->max_mr;
|
|
|
props->max_pd = nesibdev->max_pd;
|
|
|
props->max_sge_rd = 1;
|
|
|
switch (nesdev->nesadapter->max_irrq_wr) {
|
|
|
case 0:
|
|
|
- props->max_qp_rd_atom = 1;
|
|
|
+ props->max_qp_rd_atom = 2;
|
|
|
break;
|
|
|
case 1:
|
|
|
- props->max_qp_rd_atom = 4;
|
|
|
+ props->max_qp_rd_atom = 8;
|
|
|
break;
|
|
|
case 2:
|
|
|
- props->max_qp_rd_atom = 16;
|
|
|
+ props->max_qp_rd_atom = 32;
|
|
|
break;
|
|
|
case 3:
|
|
|
- props->max_qp_rd_atom = 32;
|
|
|
+ props->max_qp_rd_atom = 64;
|
|
|
break;
|
|
|
default:
|
|
|
props->max_qp_rd_atom = 0;
|
|
|
@@ -1121,6 +1015,7 @@ static int nes_setup_virt_qp(struct nes_qp *nesqp, struct nes_pbl *nespbl,
|
|
|
kunmap(nesqp->page);
|
|
|
return -ENOMEM;
|
|
|
}
|
|
|
+ nesqp->sq_kmapped = 1;
|
|
|
nesqp->hwqp.q2_vbase = mem;
|
|
|
mem += 256;
|
|
|
memset(nesqp->hwqp.q2_vbase, 0, 256);
|
|
|
@@ -1198,7 +1093,10 @@ static inline void nes_free_qp_mem(struct nes_device *nesdev,
|
|
|
pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size, nesqp->hwqp.q2_vbase, nesqp->hwqp.q2_pbase);
|
|
|
pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase );
|
|
|
nesqp->pbl_vbase = NULL;
|
|
|
- kunmap(nesqp->page);
|
|
|
+ if (nesqp->sq_kmapped) {
|
|
|
+ nesqp->sq_kmapped = 0;
|
|
|
+ kunmap(nesqp->page);
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
|
|
|
@@ -1504,8 +1402,6 @@ static struct ib_qp *nes_create_qp(struct ib_pd *ibpd,
|
|
|
nes_debug(NES_DBG_QP, "QP%u structure located @%p.Size = %u.\n",
|
|
|
nesqp->hwqp.qp_id, nesqp, (u32)sizeof(*nesqp));
|
|
|
spin_lock_init(&nesqp->lock);
|
|
|
- init_waitqueue_head(&nesqp->state_waitq);
|
|
|
- init_waitqueue_head(&nesqp->kick_waitq);
|
|
|
nes_add_ref(&nesqp->ibqp);
|
|
|
break;
|
|
|
default:
|
|
|
@@ -1513,6 +1409,8 @@ static struct ib_qp *nes_create_qp(struct ib_pd *ibpd,
|
|
|
return ERR_PTR(-EINVAL);
|
|
|
}
|
|
|
|
|
|
+ nesqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR);
|
|
|
+
|
|
|
/* update the QP table */
|
|
|
nesdev->nesadapter->qp_table[nesqp->hwqp.qp_id-NES_FIRST_QPN] = nesqp;
|
|
|
nes_debug(NES_DBG_QP, "netdev refcnt=%u\n",
|
|
|
@@ -1607,8 +1505,10 @@ static int nes_destroy_qp(struct ib_qp *ibqp)
|
|
|
nes_ucontext->first_free_wq = nesqp->mmap_sq_db_index;
|
|
|
}
|
|
|
}
|
|
|
- if (nesqp->pbl_pbase)
|
|
|
+ if (nesqp->pbl_pbase && nesqp->sq_kmapped) {
|
|
|
+ nesqp->sq_kmapped = 0;
|
|
|
kunmap(nesqp->page);
|
|
|
+ }
|
|
|
} else {
|
|
|
/* Clean any pending completions from the cq(s) */
|
|
|
if (nesqp->nesscq)
|
|
|
@@ -1649,6 +1549,9 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev, int entries,
|
|
|
unsigned long flags;
|
|
|
int ret;
|
|
|
|
|
|
+ if (entries > nesadapter->max_cqe)
|
|
|
+ return ERR_PTR(-EINVAL);
|
|
|
+
|
|
|
err = nes_alloc_resource(nesadapter, nesadapter->allocated_cqs,
|
|
|
nesadapter->max_cq, &cq_num, &nesadapter->next_cq);
|
|
|
if (err) {
|
|
|
@@ -2606,9 +2509,6 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
|
|
|
stag = stag_index << 8;
|
|
|
stag |= driver_key;
|
|
|
stag += (u32)stag_key;
|
|
|
- if (stag == 0) {
|
|
|
- stag = 1;
|
|
|
- }
|
|
|
|
|
|
iova_start = virt;
|
|
|
/* Make the leaf PBL the root if only one PBL */
|
|
|
@@ -3109,7 +3009,6 @@ int nes_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
|
|
|
" already done based on hw state.\n",
|
|
|
nesqp->hwqp.qp_id);
|
|
|
issue_modify_qp = 0;
|
|
|
- nesqp->in_disconnect = 0;
|
|
|
}
|
|
|
switch (nesqp->hw_iwarp_state) {
|
|
|
case NES_AEQE_IWARP_STATE_CLOSING:
|
|
|
@@ -3122,7 +3021,6 @@ int nes_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
|
|
|
break;
|
|
|
default:
|
|
|
next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
|
|
|
- nesqp->in_disconnect = 1;
|
|
|
nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
|
|
|
break;
|
|
|
}
|
|
|
@@ -3139,7 +3037,6 @@ int nes_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
|
|
|
next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
|
|
|
nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;
|
|
|
issue_modify_qp = 1;
|
|
|
- nesqp->in_disconnect = 1;
|
|
|
break;
|
|
|
case IB_QPS_ERR:
|
|
|
case IB_QPS_RESET:
|
|
|
@@ -3162,7 +3059,6 @@ int nes_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
|
|
|
if ((nesqp->hw_tcp_state > NES_AEQE_TCP_STATE_CLOSED) &&
|
|
|
(nesqp->hw_tcp_state != NES_AEQE_TCP_STATE_TIME_WAIT)) {
|
|
|
next_iwarp_state |= NES_CQP_QP_RESET;
|
|
|
- nesqp->in_disconnect = 1;
|
|
|
} else {
|
|
|
nes_debug(NES_DBG_MOD_QP, "QP%u NOT setting NES_CQP_QP_RESET since TCP state = %u\n",
|
|
|
nesqp->hwqp.qp_id, nesqp->hw_tcp_state);
|
|
|
@@ -3373,21 +3269,17 @@ static int nes_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr,
|
|
|
struct nes_device *nesdev = nesvnic->nesdev;
|
|
|
struct nes_qp *nesqp = to_nesqp(ibqp);
|
|
|
struct nes_hw_qp_wqe *wqe;
|
|
|
- int err;
|
|
|
+ int err = 0;
|
|
|
u32 qsize = nesqp->hwqp.sq_size;
|
|
|
u32 head;
|
|
|
- u32 wqe_misc;
|
|
|
- u32 wqe_count;
|
|
|
+ u32 wqe_misc = 0;
|
|
|
+ u32 wqe_count = 0;
|
|
|
u32 counter;
|
|
|
- u32 total_payload_length;
|
|
|
-
|
|
|
- err = 0;
|
|
|
- wqe_misc = 0;
|
|
|
- wqe_count = 0;
|
|
|
- total_payload_length = 0;
|
|
|
|
|
|
- if (nesqp->ibqp_state > IB_QPS_RTS)
|
|
|
- return -EINVAL;
|
|
|
+ if (nesqp->ibqp_state > IB_QPS_RTS) {
|
|
|
+ err = -EINVAL;
|
|
|
+ goto out;
|
|
|
+ }
|
|
|
|
|
|
spin_lock_irqsave(&nesqp->lock, flags);
|
|
|
|
|
|
@@ -3413,94 +3305,208 @@ static int nes_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr,
|
|
|
u64temp = (u64)(ib_wr->wr_id);
|
|
|
set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX,
|
|
|
u64temp);
|
|
|
- switch (ib_wr->opcode) {
|
|
|
- case IB_WR_SEND:
|
|
|
- if (ib_wr->send_flags & IB_SEND_SOLICITED) {
|
|
|
- wqe_misc = NES_IWARP_SQ_OP_SENDSE;
|
|
|
- } else {
|
|
|
- wqe_misc = NES_IWARP_SQ_OP_SEND;
|
|
|
- }
|
|
|
- if (ib_wr->num_sge > nesdev->nesadapter->max_sge) {
|
|
|
- err = -EINVAL;
|
|
|
- break;
|
|
|
- }
|
|
|
- if (ib_wr->send_flags & IB_SEND_FENCE) {
|
|
|
- wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE;
|
|
|
- }
|
|
|
- if ((ib_wr->send_flags & IB_SEND_INLINE) &&
|
|
|
- ((nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) &&
|
|
|
- (ib_wr->sg_list[0].length <= 64)) {
|
|
|
- memcpy(&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX],
|
|
|
- (void *)(unsigned long)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length);
|
|
|
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX,
|
|
|
- ib_wr->sg_list[0].length);
|
|
|
- wqe_misc |= NES_IWARP_SQ_WQE_IMM_DATA;
|
|
|
- } else {
|
|
|
- fill_wqe_sg_send(wqe, ib_wr, 1);
|
|
|
- }
|
|
|
+ switch (ib_wr->opcode) {
|
|
|
+ case IB_WR_SEND:
|
|
|
+ case IB_WR_SEND_WITH_INV:
|
|
|
+ if (IB_WR_SEND == ib_wr->opcode) {
|
|
|
+ if (ib_wr->send_flags & IB_SEND_SOLICITED)
|
|
|
+ wqe_misc = NES_IWARP_SQ_OP_SENDSE;
|
|
|
+ else
|
|
|
+ wqe_misc = NES_IWARP_SQ_OP_SEND;
|
|
|
+ } else {
|
|
|
+ if (ib_wr->send_flags & IB_SEND_SOLICITED)
|
|
|
+ wqe_misc = NES_IWARP_SQ_OP_SENDSEINV;
|
|
|
+ else
|
|
|
+ wqe_misc = NES_IWARP_SQ_OP_SENDINV;
|
|
|
|
|
|
- break;
|
|
|
- case IB_WR_RDMA_WRITE:
|
|
|
- wqe_misc = NES_IWARP_SQ_OP_RDMAW;
|
|
|
- if (ib_wr->num_sge > nesdev->nesadapter->max_sge) {
|
|
|
- nes_debug(NES_DBG_IW_TX, "Exceeded max sge, ib_wr=%u, max=%u\n",
|
|
|
- ib_wr->num_sge,
|
|
|
- nesdev->nesadapter->max_sge);
|
|
|
- err = -EINVAL;
|
|
|
- break;
|
|
|
- }
|
|
|
- if (ib_wr->send_flags & IB_SEND_FENCE) {
|
|
|
- wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE;
|
|
|
- }
|
|
|
+ set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX,
|
|
|
+ ib_wr->ex.invalidate_rkey);
|
|
|
+ }
|
|
|
|
|
|
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_STAG_IDX,
|
|
|
- ib_wr->wr.rdma.rkey);
|
|
|
- set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX,
|
|
|
- ib_wr->wr.rdma.remote_addr);
|
|
|
-
|
|
|
- if ((ib_wr->send_flags & IB_SEND_INLINE) &&
|
|
|
- ((nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) &&
|
|
|
- (ib_wr->sg_list[0].length <= 64)) {
|
|
|
- memcpy(&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX],
|
|
|
- (void *)(unsigned long)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length);
|
|
|
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX,
|
|
|
- ib_wr->sg_list[0].length);
|
|
|
- wqe_misc |= NES_IWARP_SQ_WQE_IMM_DATA;
|
|
|
- } else {
|
|
|
- fill_wqe_sg_send(wqe, ib_wr, 1);
|
|
|
- }
|
|
|
- wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] =
|
|
|
- wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX];
|
|
|
- break;
|
|
|
- case IB_WR_RDMA_READ:
|
|
|
- /* iWARP only supports 1 sge for RDMA reads */
|
|
|
- if (ib_wr->num_sge > 1) {
|
|
|
- nes_debug(NES_DBG_IW_TX, "Exceeded max sge, ib_wr=%u, max=1\n",
|
|
|
- ib_wr->num_sge);
|
|
|
- err = -EINVAL;
|
|
|
- break;
|
|
|
- }
|
|
|
- wqe_misc = NES_IWARP_SQ_OP_RDMAR;
|
|
|
- set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX,
|
|
|
- ib_wr->wr.rdma.remote_addr);
|
|
|
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_STAG_IDX,
|
|
|
- ib_wr->wr.rdma.rkey);
|
|
|
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX,
|
|
|
- ib_wr->sg_list->length);
|
|
|
- set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX,
|
|
|
- ib_wr->sg_list->addr);
|
|
|
- set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX,
|
|
|
- ib_wr->sg_list->lkey);
|
|
|
- break;
|
|
|
- default:
|
|
|
- /* error */
|
|
|
- err = -EINVAL;
|
|
|
- break;
|
|
|
+ if (ib_wr->num_sge > nesdev->nesadapter->max_sge) {
|
|
|
+ err = -EINVAL;
|
|
|
+ break;
|
|
|
}
|
|
|
|
|
|
- if (ib_wr->send_flags & IB_SEND_SIGNALED) {
|
|
|
- wqe_misc |= NES_IWARP_SQ_WQE_SIGNALED_COMPL;
|
|
|
+ if (ib_wr->send_flags & IB_SEND_FENCE)
|
|
|
+ wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE;
|
|
|
+
|
|
|
+ if ((ib_wr->send_flags & IB_SEND_INLINE) &&
|
|
|
+ ((nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) &&
|
|
|
+ (ib_wr->sg_list[0].length <= 64)) {
|
|
|
+ memcpy(&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX],
|
|
|
+ (void *)(unsigned long)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length);
|
|
|
+ set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX,
|
|
|
+ ib_wr->sg_list[0].length);
|
|
|
+ wqe_misc |= NES_IWARP_SQ_WQE_IMM_DATA;
|
|
|
+ } else {
|
|
|
+ fill_wqe_sg_send(wqe, ib_wr, 1);
|
|
|
+ }
|
|
|
+
|
|
|
+ break;
|
|
|
+ case IB_WR_RDMA_WRITE:
|
|
|
+ wqe_misc = NES_IWARP_SQ_OP_RDMAW;
|
|
|
+ if (ib_wr->num_sge > nesdev->nesadapter->max_sge) {
|
|
|
+ nes_debug(NES_DBG_IW_TX, "Exceeded max sge, ib_wr=%u, max=%u\n",
|
|
|
+ ib_wr->num_sge, nesdev->nesadapter->max_sge);
|
|
|
+ err = -EINVAL;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (ib_wr->send_flags & IB_SEND_FENCE)
|
|
|
+ wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE;
|
|
|
+
|
|
|
+ set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_STAG_IDX,
|
|
|
+ ib_wr->wr.rdma.rkey);
|
|
|
+ set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX,
|
|
|
+ ib_wr->wr.rdma.remote_addr);
|
|
|
+
|
|
|
+ if ((ib_wr->send_flags & IB_SEND_INLINE) &&
|
|
|
+ ((nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) &&
|
|
|
+ (ib_wr->sg_list[0].length <= 64)) {
|
|
|
+ memcpy(&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX],
|
|
|
+ (void *)(unsigned long)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length);
|
|
|
+ set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX,
|
|
|
+ ib_wr->sg_list[0].length);
|
|
|
+ wqe_misc |= NES_IWARP_SQ_WQE_IMM_DATA;
|
|
|
+ } else {
|
|
|
+ fill_wqe_sg_send(wqe, ib_wr, 1);
|
|
|
+ }
|
|
|
+
|
|
|
+ wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] =
|
|
|
+ wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX];
|
|
|
+ break;
|
|
|
+ case IB_WR_RDMA_READ:
|
|
|
+ case IB_WR_RDMA_READ_WITH_INV:
|
|
|
+ /* iWARP only supports 1 sge for RDMA reads */
|
|
|
+ if (ib_wr->num_sge > 1) {
|
|
|
+ nes_debug(NES_DBG_IW_TX, "Exceeded max sge, ib_wr=%u, max=1\n",
|
|
|
+ ib_wr->num_sge);
|
|
|
+ err = -EINVAL;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ if (ib_wr->opcode == IB_WR_RDMA_READ) {
|
|
|
+ wqe_misc = NES_IWARP_SQ_OP_RDMAR;
|
|
|
+ } else {
|
|
|
+ wqe_misc = NES_IWARP_SQ_OP_RDMAR_LOCINV;
|
|
|
+ set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX,
|
|
|
+ ib_wr->ex.invalidate_rkey);
|
|
|
+ }
|
|
|
+
|
|
|
+ set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX,
|
|
|
+ ib_wr->wr.rdma.remote_addr);
|
|
|
+ set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_STAG_IDX,
|
|
|
+ ib_wr->wr.rdma.rkey);
|
|
|
+ set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX,
|
|
|
+ ib_wr->sg_list->length);
|
|
|
+ set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX,
|
|
|
+ ib_wr->sg_list->addr);
|
|
|
+ set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX,
|
|
|
+ ib_wr->sg_list->lkey);
|
|
|
+ break;
|
|
|
+ case IB_WR_LOCAL_INV:
|
|
|
+ wqe_misc = NES_IWARP_SQ_OP_LOCINV;
|
|
|
+ set_wqe_32bit_value(wqe->wqe_words,
|
|
|
+ NES_IWARP_SQ_LOCINV_WQE_INV_STAG_IDX,
|
|
|
+ ib_wr->ex.invalidate_rkey);
|
|
|
+ break;
|
|
|
+ case IB_WR_FAST_REG_MR:
|
|
|
+ {
|
|
|
+ int i;
|
|
|
+ int flags = ib_wr->wr.fast_reg.access_flags;
|
|
|
+ struct nes_ib_fast_reg_page_list *pnesfrpl =
|
|
|
+ container_of(ib_wr->wr.fast_reg.page_list,
|
|
|
+ struct nes_ib_fast_reg_page_list,
|
|
|
+ ibfrpl);
|
|
|
+ u64 *src_page_list = pnesfrpl->ibfrpl.page_list;
|
|
|
+ u64 *dst_page_list = pnesfrpl->nes_wqe_pbl.kva;
|
|
|
+
|
|
|
+ if (ib_wr->wr.fast_reg.page_list_len >
|
|
|
+ (NES_4K_PBL_CHUNK_SIZE / sizeof(u64))) {
|
|
|
+ nes_debug(NES_DBG_IW_TX, "SQ_FMR: bad page_list_len\n");
|
|
|
+ err = -EINVAL;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ wqe_misc = NES_IWARP_SQ_OP_FAST_REG;
|
|
|
+ set_wqe_64bit_value(wqe->wqe_words,
|
|
|
+ NES_IWARP_SQ_FMR_WQE_VA_FBO_LOW_IDX,
|
|
|
+ ib_wr->wr.fast_reg.iova_start);
|
|
|
+ set_wqe_32bit_value(wqe->wqe_words,
|
|
|
+ NES_IWARP_SQ_FMR_WQE_LENGTH_LOW_IDX,
|
|
|
+ ib_wr->wr.fast_reg.length);
|
|
|
+ set_wqe_32bit_value(wqe->wqe_words,
|
|
|
+ NES_IWARP_SQ_FMR_WQE_MR_STAG_IDX,
|
|
|
+ ib_wr->wr.fast_reg.rkey);
|
|
|
+ /* Set page size: */
|
|
|
+ if (ib_wr->wr.fast_reg.page_shift == 12) {
|
|
|
+ wqe_misc |= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_4K;
|
|
|
+ } else if (ib_wr->wr.fast_reg.page_shift == 21) {
|
|
|
+ wqe_misc |= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_2M;
|
|
|
+ } else {
|
|
|
+ nes_debug(NES_DBG_IW_TX, "Invalid page shift,"
|
|
|
+ " ib_wr=%u, max=1\n", ib_wr->num_sge);
|
|
|
+ err = -EINVAL;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ /* Set access_flags */
|
|
|
+ wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_READ;
|
|
|
+ if (flags & IB_ACCESS_LOCAL_WRITE)
|
|
|
+ wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_WRITE;
|
|
|
+
|
|
|
+ if (flags & IB_ACCESS_REMOTE_WRITE)
|
|
|
+ wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_WRITE;
|
|
|
+
|
|
|
+ if (flags & IB_ACCESS_REMOTE_READ)
|
|
|
+ wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_READ;
|
|
|
+
|
|
|
+ if (flags & IB_ACCESS_MW_BIND)
|
|
|
+ wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_WINDOW_BIND;
|
|
|
+
|
|
|
+ /* Fill in PBL info: */
|
|
|
+ if (ib_wr->wr.fast_reg.page_list_len >
|
|
|
+ pnesfrpl->ibfrpl.max_page_list_len) {
|
|
|
+ nes_debug(NES_DBG_IW_TX, "Invalid page list length,"
|
|
|
+ " ib_wr=%p, value=%u, max=%u\n",
|
|
|
+ ib_wr, ib_wr->wr.fast_reg.page_list_len,
|
|
|
+ pnesfrpl->ibfrpl.max_page_list_len);
|
|
|
+ err = -EINVAL;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+
|
|
|
+ set_wqe_64bit_value(wqe->wqe_words,
|
|
|
+ NES_IWARP_SQ_FMR_WQE_PBL_ADDR_LOW_IDX,
|
|
|
+ pnesfrpl->nes_wqe_pbl.paddr);
|
|
|
+
|
|
|
+ set_wqe_32bit_value(wqe->wqe_words,
|
|
|
+ NES_IWARP_SQ_FMR_WQE_PBL_LENGTH_IDX,
|
|
|
+ ib_wr->wr.fast_reg.page_list_len * 8);
|
|
|
+
|
|
|
+ for (i = 0; i < ib_wr->wr.fast_reg.page_list_len; i++)
|
|
|
+ dst_page_list[i] = cpu_to_le64(src_page_list[i]);
|
|
|
+
|
|
|
+ nes_debug(NES_DBG_IW_TX, "SQ_FMR: iova_start: %p, "
|
|
|
+ "length: %d, rkey: %0x, pgl_paddr: %p, "
|
|
|
+ "page_list_len: %u, wqe_misc: %x\n",
|
|
|
+ (void *)ib_wr->wr.fast_reg.iova_start,
|
|
|
+ ib_wr->wr.fast_reg.length,
|
|
|
+ ib_wr->wr.fast_reg.rkey,
|
|
|
+ (void *)pnesfrpl->nes_wqe_pbl.paddr,
|
|
|
+ ib_wr->wr.fast_reg.page_list_len,
|
|
|
+ wqe_misc);
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ default:
|
|
|
+ /* error */
|
|
|
+ err = -EINVAL;
|
|
|
+ break;
|
|
|
}
|
|
|
+
|
|
|
+ if (err)
|
|
|
+ break;
|
|
|
+
|
|
|
+ if ((ib_wr->send_flags & IB_SEND_SIGNALED) || nesqp->sig_all)
|
|
|
+ wqe_misc |= NES_IWARP_SQ_WQE_SIGNALED_COMPL;
|
|
|
+
|
|
|
wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = cpu_to_le32(wqe_misc);
|
|
|
|
|
|
ib_wr = ib_wr->next;
|
|
|
@@ -3522,6 +3528,7 @@ static int nes_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr,
|
|
|
|
|
|
spin_unlock_irqrestore(&nesqp->lock, flags);
|
|
|
|
|
|
+out:
|
|
|
if (err)
|
|
|
*bad_wr = ib_wr;
|
|
|
return err;
|
|
|
@@ -3548,8 +3555,10 @@ static int nes_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr,
|
|
|
u32 counter;
|
|
|
u32 total_payload_length;
|
|
|
|
|
|
- if (nesqp->ibqp_state > IB_QPS_RTS)
|
|
|
- return -EINVAL;
|
|
|
+ if (nesqp->ibqp_state > IB_QPS_RTS) {
|
|
|
+ err = -EINVAL;
|
|
|
+ goto out;
|
|
|
+ }
|
|
|
|
|
|
spin_lock_irqsave(&nesqp->lock, flags);
|
|
|
|
|
|
@@ -3612,6 +3621,7 @@ static int nes_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr,
|
|
|
|
|
|
spin_unlock_irqrestore(&nesqp->lock, flags);
|
|
|
|
|
|
+out:
|
|
|
if (err)
|
|
|
*bad_wr = ib_wr;
|
|
|
return err;
|
|
|
@@ -3720,6 +3730,12 @@ static int nes_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry)
|
|
|
nes_debug(NES_DBG_CQ, "Operation = Send.\n");
|
|
|
entry->opcode = IB_WC_SEND;
|
|
|
break;
|
|
|
+ case NES_IWARP_SQ_OP_LOCINV:
|
|
|
+ entry->opcode = IB_WR_LOCAL_INV;
|
|
|
+ break;
|
|
|
+ case NES_IWARP_SQ_OP_FAST_REG:
|
|
|
+ entry->opcode = IB_WC_FAST_REG_MR;
|
|
|
+ break;
|
|
|
}
|
|
|
|
|
|
nesqp->hwqp.sq_tail = (wqe_index+1)&(nesqp->hwqp.sq_size - 1);
|
|
|
@@ -3890,10 +3906,9 @@ struct nes_ib_device *nes_init_ofa_device(struct net_device *netdev)
|
|
|
nesibdev->ibdev.dealloc_mw = nes_dealloc_mw;
|
|
|
nesibdev->ibdev.bind_mw = nes_bind_mw;
|
|
|
|
|
|
- nesibdev->ibdev.alloc_fmr = nes_alloc_fmr;
|
|
|
- nesibdev->ibdev.unmap_fmr = nes_unmap_fmr;
|
|
|
- nesibdev->ibdev.dealloc_fmr = nes_dealloc_fmr;
|
|
|
- nesibdev->ibdev.map_phys_fmr = nes_map_phys_fmr;
|
|
|
+ nesibdev->ibdev.alloc_fast_reg_mr = nes_alloc_fast_reg_mr;
|
|
|
+ nesibdev->ibdev.alloc_fast_reg_page_list = nes_alloc_fast_reg_page_list;
|
|
|
+ nesibdev->ibdev.free_fast_reg_page_list = nes_free_fast_reg_page_list;
|
|
|
|
|
|
nesibdev->ibdev.attach_mcast = nes_multicast_attach;
|
|
|
nesibdev->ibdev.detach_mcast = nes_multicast_detach;
|
Linus Torvalds