|
|
@@ -113,12 +113,6 @@ static inline void cxgbi_device_destroy(struct cxgbi_device *cdev)
|
|
|
"cdev 0x%p, p# %u.\n", cdev, cdev->nports);
|
|
|
cxgbi_hbas_remove(cdev);
|
|
|
cxgbi_device_portmap_cleanup(cdev);
|
|
|
- if (cdev->dev_ddp_cleanup)
|
|
|
- cdev->dev_ddp_cleanup(cdev);
|
|
|
- else
|
|
|
- cxgbi_ddp_cleanup(cdev);
|
|
|
- if (cdev->ddp)
|
|
|
- cxgbi_ddp_cleanup(cdev);
|
|
|
if (cdev->pmap.max_connect)
|
|
|
cxgbi_free_big_mem(cdev->pmap.port_csk);
|
|
|
kfree(cdev);
|
|
|
@@ -1182,504 +1176,6 @@ out_err:
|
|
|
goto done;
|
|
|
}
|
|
|
|
|
|
-/*
|
|
|
- * Direct Data Placement -
|
|
|
- * Directly place the iSCSI Data-In or Data-Out PDU's payload into pre-posted
|
|
|
- * final destination host-memory buffers based on the Initiator Task Tag (ITT)
|
|
|
- * in Data-In or Target Task Tag (TTT) in Data-Out PDUs.
|
|
|
- * The host memory address is programmed into h/w in the format of pagepod
|
|
|
- * entries.
|
|
|
- * The location of the pagepod entry is encoded into ddp tag which is used as
|
|
|
- * the base for ITT/TTT.
|
|
|
- */
|
|
|
-
|
|
|
-static unsigned char ddp_page_order[DDP_PGIDX_MAX] = {0, 1, 2, 4};
|
|
|
-static unsigned char ddp_page_shift[DDP_PGIDX_MAX] = {12, 13, 14, 16};
|
|
|
-static unsigned char page_idx = DDP_PGIDX_MAX;
|
|
|
-
|
|
|
-static unsigned char sw_tag_idx_bits;
|
|
|
-static unsigned char sw_tag_age_bits;
|
|
|
-
|
|
|
-/*
|
|
|
- * Direct-Data Placement page size adjustment
|
|
|
- */
|
|
|
-static int ddp_adjust_page_table(void)
|
|
|
-{
|
|
|
- int i;
|
|
|
- unsigned int base_order, order;
|
|
|
-
|
|
|
- if (PAGE_SIZE < (1UL << ddp_page_shift[0])) {
|
|
|
- pr_info("PAGE_SIZE 0x%lx too small, min 0x%lx\n",
|
|
|
- PAGE_SIZE, 1UL << ddp_page_shift[0]);
|
|
|
- return -EINVAL;
|
|
|
- }
|
|
|
-
|
|
|
- base_order = get_order(1UL << ddp_page_shift[0]);
|
|
|
- order = get_order(1UL << PAGE_SHIFT);
|
|
|
-
|
|
|
- for (i = 0; i < DDP_PGIDX_MAX; i++) {
|
|
|
- /* first is the kernel page size, then just doubling */
|
|
|
- ddp_page_order[i] = order - base_order + i;
|
|
|
- ddp_page_shift[i] = PAGE_SHIFT + i;
|
|
|
- }
|
|
|
- return 0;
|
|
|
-}
|
|
|
-
|
|
|
-static int ddp_find_page_index(unsigned long pgsz)
|
|
|
-{
|
|
|
- int i;
|
|
|
-
|
|
|
- for (i = 0; i < DDP_PGIDX_MAX; i++) {
|
|
|
- if (pgsz == (1UL << ddp_page_shift[i]))
|
|
|
- return i;
|
|
|
- }
|
|
|
- pr_info("ddp page size %lu not supported.\n", pgsz);
|
|
|
- return DDP_PGIDX_MAX;
|
|
|
-}
|
|
|
-
|
|
|
-static void ddp_setup_host_page_size(void)
|
|
|
-{
|
|
|
- if (page_idx == DDP_PGIDX_MAX) {
|
|
|
- page_idx = ddp_find_page_index(PAGE_SIZE);
|
|
|
-
|
|
|
- if (page_idx == DDP_PGIDX_MAX) {
|
|
|
- pr_info("system PAGE %lu, update hw.\n", PAGE_SIZE);
|
|
|
- if (ddp_adjust_page_table() < 0) {
|
|
|
- pr_info("PAGE %lu, disable ddp.\n", PAGE_SIZE);
|
|
|
- return;
|
|
|
- }
|
|
|
- page_idx = ddp_find_page_index(PAGE_SIZE);
|
|
|
- }
|
|
|
- pr_info("system PAGE %lu, ddp idx %u.\n", PAGE_SIZE, page_idx);
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-void cxgbi_ddp_page_size_factor(int *pgsz_factor)
|
|
|
-{
|
|
|
- int i;
|
|
|
-
|
|
|
- for (i = 0; i < DDP_PGIDX_MAX; i++)
|
|
|
- pgsz_factor[i] = ddp_page_order[i];
|
|
|
-}
|
|
|
-EXPORT_SYMBOL_GPL(cxgbi_ddp_page_size_factor);
|
|
|
-
|
|
|
-/*
|
|
|
- * DDP setup & teardown
|
|
|
- */
|
|
|
-
|
|
|
-void cxgbi_ddp_ppod_set(struct cxgbi_pagepod *ppod,
|
|
|
- struct cxgbi_pagepod_hdr *hdr,
|
|
|
- struct cxgbi_gather_list *gl, unsigned int gidx)
|
|
|
-{
|
|
|
- int i;
|
|
|
-
|
|
|
- memcpy(ppod, hdr, sizeof(*hdr));
|
|
|
- for (i = 0; i < (PPOD_PAGES_MAX + 1); i++, gidx++) {
|
|
|
- ppod->addr[i] = gidx < gl->nelem ?
|
|
|
- cpu_to_be64(gl->phys_addr[gidx]) : 0ULL;
|
|
|
- }
|
|
|
-}
|
|
|
-EXPORT_SYMBOL_GPL(cxgbi_ddp_ppod_set);
|
|
|
-
|
|
|
-void cxgbi_ddp_ppod_clear(struct cxgbi_pagepod *ppod)
|
|
|
-{
|
|
|
- memset(ppod, 0, sizeof(*ppod));
|
|
|
-}
|
|
|
-EXPORT_SYMBOL_GPL(cxgbi_ddp_ppod_clear);
|
|
|
-
|
|
|
-static inline int ddp_find_unused_entries(struct cxgbi_ddp_info *ddp,
|
|
|
- unsigned int start, unsigned int max,
|
|
|
- unsigned int count,
|
|
|
- struct cxgbi_gather_list *gl)
|
|
|
-{
|
|
|
- unsigned int i, j, k;
|
|
|
-
|
|
|
- /* not enough entries */
|
|
|
- if ((max - start) < count) {
|
|
|
- log_debug(1 << CXGBI_DBG_DDP,
|
|
|
- "NOT enough entries %u+%u < %u.\n", start, count, max);
|
|
|
- return -EBUSY;
|
|
|
- }
|
|
|
-
|
|
|
- max -= count;
|
|
|
- spin_lock(&ddp->map_lock);
|
|
|
- for (i = start; i < max;) {
|
|
|
- for (j = 0, k = i; j < count; j++, k++) {
|
|
|
- if (ddp->gl_map[k])
|
|
|
- break;
|
|
|
- }
|
|
|
- if (j == count) {
|
|
|
- for (j = 0, k = i; j < count; j++, k++)
|
|
|
- ddp->gl_map[k] = gl;
|
|
|
- spin_unlock(&ddp->map_lock);
|
|
|
- return i;
|
|
|
- }
|
|
|
- i += j + 1;
|
|
|
- }
|
|
|
- spin_unlock(&ddp->map_lock);
|
|
|
- log_debug(1 << CXGBI_DBG_DDP,
|
|
|
- "NO suitable entries %u available.\n", count);
|
|
|
- return -EBUSY;
|
|
|
-}
|
|
|
-
|
|
|
-static inline void ddp_unmark_entries(struct cxgbi_ddp_info *ddp,
|
|
|
- int start, int count)
|
|
|
-{
|
|
|
- spin_lock(&ddp->map_lock);
|
|
|
- memset(&ddp->gl_map[start], 0,
|
|
|
- count * sizeof(struct cxgbi_gather_list *));
|
|
|
- spin_unlock(&ddp->map_lock);
|
|
|
-}
|
|
|
-
|
|
|
-static inline void ddp_gl_unmap(struct pci_dev *pdev,
|
|
|
- struct cxgbi_gather_list *gl)
|
|
|
-{
|
|
|
- int i;
|
|
|
-
|
|
|
- for (i = 0; i < gl->nelem; i++)
|
|
|
- dma_unmap_page(&pdev->dev, gl->phys_addr[i], PAGE_SIZE,
|
|
|
- PCI_DMA_FROMDEVICE);
|
|
|
-}
|
|
|
-
|
|
|
-static inline int ddp_gl_map(struct pci_dev *pdev,
|
|
|
- struct cxgbi_gather_list *gl)
|
|
|
-{
|
|
|
- int i;
|
|
|
-
|
|
|
- for (i = 0; i < gl->nelem; i++) {
|
|
|
- gl->phys_addr[i] = dma_map_page(&pdev->dev, gl->pages[i], 0,
|
|
|
- PAGE_SIZE,
|
|
|
- PCI_DMA_FROMDEVICE);
|
|
|
- if (unlikely(dma_mapping_error(&pdev->dev, gl->phys_addr[i]))) {
|
|
|
- log_debug(1 << CXGBI_DBG_DDP,
|
|
|
- "page %d 0x%p, 0x%p dma mapping err.\n",
|
|
|
- i, gl->pages[i], pdev);
|
|
|
- goto unmap;
|
|
|
- }
|
|
|
- }
|
|
|
- return i;
|
|
|
-unmap:
|
|
|
- if (i) {
|
|
|
- unsigned int nelem = gl->nelem;
|
|
|
-
|
|
|
- gl->nelem = i;
|
|
|
- ddp_gl_unmap(pdev, gl);
|
|
|
- gl->nelem = nelem;
|
|
|
- }
|
|
|
- return -EINVAL;
|
|
|
-}
|
|
|
-
|
|
|
-static void ddp_release_gl(struct cxgbi_gather_list *gl,
|
|
|
- struct pci_dev *pdev)
|
|
|
-{
|
|
|
- ddp_gl_unmap(pdev, gl);
|
|
|
- kfree(gl);
|
|
|
-}
|
|
|
-
|
|
|
-static struct cxgbi_gather_list *ddp_make_gl(unsigned int xferlen,
|
|
|
- struct scatterlist *sgl,
|
|
|
- unsigned int sgcnt,
|
|
|
- struct pci_dev *pdev,
|
|
|
- gfp_t gfp)
|
|
|
-{
|
|
|
- struct cxgbi_gather_list *gl;
|
|
|
- struct scatterlist *sg = sgl;
|
|
|
- struct page *sgpage = sg_page(sg);
|
|
|
- unsigned int sglen = sg->length;
|
|
|
- unsigned int sgoffset = sg->offset;
|
|
|
- unsigned int npages = (xferlen + sgoffset + PAGE_SIZE - 1) >>
|
|
|
- PAGE_SHIFT;
|
|
|
- int i = 1, j = 0;
|
|
|
-
|
|
|
- if (xferlen < DDP_THRESHOLD) {
|
|
|
- log_debug(1 << CXGBI_DBG_DDP,
|
|
|
- "xfer %u < threshold %u, no ddp.\n",
|
|
|
- xferlen, DDP_THRESHOLD);
|
|
|
- return NULL;
|
|
|
- }
|
|
|
-
|
|
|
- gl = kzalloc(sizeof(struct cxgbi_gather_list) +
|
|
|
- npages * (sizeof(dma_addr_t) +
|
|
|
- sizeof(struct page *)), gfp);
|
|
|
- if (!gl) {
|
|
|
- log_debug(1 << CXGBI_DBG_DDP,
|
|
|
- "xfer %u, %u pages, OOM.\n", xferlen, npages);
|
|
|
- return NULL;
|
|
|
- }
|
|
|
-
|
|
|
- log_debug(1 << CXGBI_DBG_DDP,
|
|
|
- "xfer %u, sgl %u, gl max %u.\n", xferlen, sgcnt, npages);
|
|
|
-
|
|
|
- gl->pages = (struct page **)&gl->phys_addr[npages];
|
|
|
- gl->nelem = npages;
|
|
|
- gl->length = xferlen;
|
|
|
- gl->offset = sgoffset;
|
|
|
- gl->pages[0] = sgpage;
|
|
|
-
|
|
|
- for (i = 1, sg = sg_next(sgl), j = 0; i < sgcnt;
|
|
|
- i++, sg = sg_next(sg)) {
|
|
|
- struct page *page = sg_page(sg);
|
|
|
-
|
|
|
- if (sgpage == page && sg->offset == sgoffset + sglen)
|
|
|
- sglen += sg->length;
|
|
|
- else {
|
|
|
- /* make sure the sgl is fit for ddp:
|
|
|
- * each has the same page size, and
|
|
|
- * all of the middle pages are used completely
|
|
|
- */
|
|
|
- if ((j && sgoffset) || ((i != sgcnt - 1) &&
|
|
|
- ((sglen + sgoffset) & ~PAGE_MASK))) {
|
|
|
- log_debug(1 << CXGBI_DBG_DDP,
|
|
|
- "page %d/%u, %u + %u.\n",
|
|
|
- i, sgcnt, sgoffset, sglen);
|
|
|
- goto error_out;
|
|
|
- }
|
|
|
-
|
|
|
- j++;
|
|
|
- if (j == gl->nelem || sg->offset) {
|
|
|
- log_debug(1 << CXGBI_DBG_DDP,
|
|
|
- "page %d/%u, offset %u.\n",
|
|
|
- j, gl->nelem, sg->offset);
|
|
|
- goto error_out;
|
|
|
- }
|
|
|
- gl->pages[j] = page;
|
|
|
- sglen = sg->length;
|
|
|
- sgoffset = sg->offset;
|
|
|
- sgpage = page;
|
|
|
- }
|
|
|
- }
|
|
|
- gl->nelem = ++j;
|
|
|
-
|
|
|
- if (ddp_gl_map(pdev, gl) < 0)
|
|
|
- goto error_out;
|
|
|
-
|
|
|
- return gl;
|
|
|
-
|
|
|
-error_out:
|
|
|
- kfree(gl);
|
|
|
- return NULL;
|
|
|
-}
|
|
|
-
|
|
|
-static void ddp_tag_release(struct cxgbi_hba *chba, u32 tag)
|
|
|
-{
|
|
|
- struct cxgbi_device *cdev = chba->cdev;
|
|
|
- struct cxgbi_ddp_info *ddp = cdev->ddp;
|
|
|
- u32 idx;
|
|
|
-
|
|
|
- idx = (tag >> PPOD_IDX_SHIFT) & ddp->idx_mask;
|
|
|
- if (idx < ddp->nppods) {
|
|
|
- struct cxgbi_gather_list *gl = ddp->gl_map[idx];
|
|
|
- unsigned int npods;
|
|
|
-
|
|
|
- if (!gl || !gl->nelem) {
|
|
|
- pr_warn("tag 0x%x, idx %u, gl 0x%p, %u.\n",
|
|
|
- tag, idx, gl, gl ? gl->nelem : 0);
|
|
|
- return;
|
|
|
- }
|
|
|
- npods = (gl->nelem + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
|
|
|
- log_debug(1 << CXGBI_DBG_DDP,
|
|
|
- "tag 0x%x, release idx %u, npods %u.\n",
|
|
|
- tag, idx, npods);
|
|
|
- cdev->csk_ddp_clear(chba, tag, idx, npods);
|
|
|
- ddp_unmark_entries(ddp, idx, npods);
|
|
|
- ddp_release_gl(gl, ddp->pdev);
|
|
|
- } else
|
|
|
- pr_warn("tag 0x%x, idx %u > max %u.\n", tag, idx, ddp->nppods);
|
|
|
-}
|
|
|
-
|
|
|
-static int ddp_tag_reserve(struct cxgbi_sock *csk, unsigned int tid,
|
|
|
- u32 sw_tag, u32 *tagp, struct cxgbi_gather_list *gl,
|
|
|
- gfp_t gfp)
|
|
|
-{
|
|
|
- struct cxgbi_device *cdev = csk->cdev;
|
|
|
- struct cxgbi_ddp_info *ddp = cdev->ddp;
|
|
|
- struct cxgbi_tag_format *tformat = &cdev->tag_format;
|
|
|
- struct cxgbi_pagepod_hdr hdr;
|
|
|
- unsigned int npods;
|
|
|
- int idx = -1;
|
|
|
- int err = -ENOMEM;
|
|
|
- u32 tag;
|
|
|
-
|
|
|
- npods = (gl->nelem + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
|
|
|
- if (ddp->idx_last == ddp->nppods)
|
|
|
- idx = ddp_find_unused_entries(ddp, 0, ddp->nppods,
|
|
|
- npods, gl);
|
|
|
- else {
|
|
|
- idx = ddp_find_unused_entries(ddp, ddp->idx_last + 1,
|
|
|
- ddp->nppods, npods,
|
|
|
- gl);
|
|
|
- if (idx < 0 && ddp->idx_last >= npods) {
|
|
|
- idx = ddp_find_unused_entries(ddp, 0,
|
|
|
- min(ddp->idx_last + npods, ddp->nppods),
|
|
|
- npods, gl);
|
|
|
- }
|
|
|
- }
|
|
|
- if (idx < 0) {
|
|
|
- log_debug(1 << CXGBI_DBG_DDP,
|
|
|
- "xferlen %u, gl %u, npods %u NO DDP.\n",
|
|
|
- gl->length, gl->nelem, npods);
|
|
|
- return idx;
|
|
|
- }
|
|
|
-
|
|
|
- tag = cxgbi_ddp_tag_base(tformat, sw_tag);
|
|
|
- tag |= idx << PPOD_IDX_SHIFT;
|
|
|
-
|
|
|
- hdr.rsvd = 0;
|
|
|
- hdr.vld_tid = htonl(PPOD_VALID_FLAG | PPOD_TID(tid));
|
|
|
- hdr.pgsz_tag_clr = htonl(tag & ddp->rsvd_tag_mask);
|
|
|
- hdr.max_offset = htonl(gl->length);
|
|
|
- hdr.page_offset = htonl(gl->offset);
|
|
|
-
|
|
|
- err = cdev->csk_ddp_set(csk, &hdr, idx, npods, gl);
|
|
|
- if (err < 0)
|
|
|
- goto unmark_entries;
|
|
|
-
|
|
|
- ddp->idx_last = idx;
|
|
|
- log_debug(1 << CXGBI_DBG_DDP,
|
|
|
- "xfer %u, gl %u,%u, tid 0x%x, tag 0x%x->0x%x(%u,%u).\n",
|
|
|
- gl->length, gl->nelem, gl->offset, tid, sw_tag, tag, idx,
|
|
|
- npods);
|
|
|
- *tagp = tag;
|
|
|
- return 0;
|
|
|
-
|
|
|
-unmark_entries:
|
|
|
- ddp_unmark_entries(ddp, idx, npods);
|
|
|
- return err;
|
|
|
-}
|
|
|
-
|
|
|
-int cxgbi_ddp_reserve(struct cxgbi_sock *csk, unsigned int *tagp,
|
|
|
- unsigned int sw_tag, unsigned int xferlen,
|
|
|
- struct scatterlist *sgl, unsigned int sgcnt, gfp_t gfp)
|
|
|
-{
|
|
|
- struct cxgbi_device *cdev = csk->cdev;
|
|
|
- struct cxgbi_tag_format *tformat = &cdev->tag_format;
|
|
|
- struct cxgbi_gather_list *gl;
|
|
|
- int err;
|
|
|
-
|
|
|
- if (page_idx >= DDP_PGIDX_MAX || !cdev->ddp ||
|
|
|
- xferlen < DDP_THRESHOLD) {
|
|
|
- log_debug(1 << CXGBI_DBG_DDP,
|
|
|
- "pgidx %u, xfer %u, NO ddp.\n", page_idx, xferlen);
|
|
|
- return -EINVAL;
|
|
|
- }
|
|
|
-
|
|
|
- if (!cxgbi_sw_tag_usable(tformat, sw_tag)) {
|
|
|
- log_debug(1 << CXGBI_DBG_DDP,
|
|
|
- "sw_tag 0x%x NOT usable.\n", sw_tag);
|
|
|
- return -EINVAL;
|
|
|
- }
|
|
|
-
|
|
|
- gl = ddp_make_gl(xferlen, sgl, sgcnt, cdev->pdev, gfp);
|
|
|
- if (!gl)
|
|
|
- return -ENOMEM;
|
|
|
-
|
|
|
- err = ddp_tag_reserve(csk, csk->tid, sw_tag, tagp, gl, gfp);
|
|
|
- if (err < 0)
|
|
|
- ddp_release_gl(gl, cdev->pdev);
|
|
|
-
|
|
|
- return err;
|
|
|
-}
|
|
|
-
|
|
|
-static void ddp_destroy(struct kref *kref)
|
|
|
-{
|
|
|
- struct cxgbi_ddp_info *ddp = container_of(kref,
|
|
|
- struct cxgbi_ddp_info,
|
|
|
- refcnt);
|
|
|
- struct cxgbi_device *cdev = ddp->cdev;
|
|
|
- int i = 0;
|
|
|
-
|
|
|
- pr_info("kref 0, destroy ddp 0x%p, cdev 0x%p.\n", ddp, cdev);
|
|
|
-
|
|
|
- while (i < ddp->nppods) {
|
|
|
- struct cxgbi_gather_list *gl = ddp->gl_map[i];
|
|
|
-
|
|
|
- if (gl) {
|
|
|
- int npods = (gl->nelem + PPOD_PAGES_MAX - 1)
|
|
|
- >> PPOD_PAGES_SHIFT;
|
|
|
- pr_info("cdev 0x%p, ddp %d + %d.\n", cdev, i, npods);
|
|
|
- kfree(gl);
|
|
|
- i += npods;
|
|
|
- } else
|
|
|
- i++;
|
|
|
- }
|
|
|
- cxgbi_free_big_mem(ddp);
|
|
|
-}
|
|
|
-
|
|
|
-int cxgbi_ddp_cleanup(struct cxgbi_device *cdev)
|
|
|
-{
|
|
|
- struct cxgbi_ddp_info *ddp = cdev->ddp;
|
|
|
-
|
|
|
- log_debug(1 << CXGBI_DBG_DDP,
|
|
|
- "cdev 0x%p, release ddp 0x%p.\n", cdev, ddp);
|
|
|
- cdev->ddp = NULL;
|
|
|
- if (ddp)
|
|
|
- return kref_put(&ddp->refcnt, ddp_destroy);
|
|
|
- return 0;
|
|
|
-}
|
|
|
-EXPORT_SYMBOL_GPL(cxgbi_ddp_cleanup);
|
|
|
-
|
|
|
-int cxgbi_ddp_init(struct cxgbi_device *cdev,
|
|
|
- unsigned int llimit, unsigned int ulimit,
|
|
|
- unsigned int max_txsz, unsigned int max_rxsz)
|
|
|
-{
|
|
|
- struct cxgbi_ddp_info *ddp;
|
|
|
- unsigned int ppmax, bits;
|
|
|
-
|
|
|
- ppmax = (ulimit - llimit + 1) >> PPOD_SIZE_SHIFT;
|
|
|
- bits = __ilog2_u32(ppmax) + 1;
|
|
|
- if (bits > PPOD_IDX_MAX_SIZE)
|
|
|
- bits = PPOD_IDX_MAX_SIZE;
|
|
|
- ppmax = (1 << (bits - 1)) - 1;
|
|
|
-
|
|
|
- ddp = cxgbi_alloc_big_mem(sizeof(struct cxgbi_ddp_info) +
|
|
|
- ppmax * (sizeof(struct cxgbi_gather_list *) +
|
|
|
- sizeof(struct sk_buff *)),
|
|
|
- GFP_KERNEL);
|
|
|
- if (!ddp) {
|
|
|
- pr_warn("cdev 0x%p, ddp ppmax %u OOM.\n", cdev, ppmax);
|
|
|
- return -ENOMEM;
|
|
|
- }
|
|
|
- ddp->gl_map = (struct cxgbi_gather_list **)(ddp + 1);
|
|
|
- cdev->ddp = ddp;
|
|
|
-
|
|
|
- spin_lock_init(&ddp->map_lock);
|
|
|
- kref_init(&ddp->refcnt);
|
|
|
-
|
|
|
- ddp->cdev = cdev;
|
|
|
- ddp->pdev = cdev->pdev;
|
|
|
- ddp->llimit = llimit;
|
|
|
- ddp->ulimit = ulimit;
|
|
|
- ddp->max_txsz = min_t(unsigned int, max_txsz, ULP2_MAX_PKT_SIZE);
|
|
|
- ddp->max_rxsz = min_t(unsigned int, max_rxsz, ULP2_MAX_PKT_SIZE);
|
|
|
- ddp->nppods = ppmax;
|
|
|
- ddp->idx_last = ppmax;
|
|
|
- ddp->idx_bits = bits;
|
|
|
- ddp->idx_mask = (1 << bits) - 1;
|
|
|
- ddp->rsvd_tag_mask = (1 << (bits + PPOD_IDX_SHIFT)) - 1;
|
|
|
-
|
|
|
- cdev->tag_format.sw_bits = sw_tag_idx_bits + sw_tag_age_bits;
|
|
|
- cdev->tag_format.rsvd_bits = ddp->idx_bits;
|
|
|
- cdev->tag_format.rsvd_shift = PPOD_IDX_SHIFT;
|
|
|
- cdev->tag_format.rsvd_mask = (1 << cdev->tag_format.rsvd_bits) - 1;
|
|
|
-
|
|
|
- pr_info("%s tag format, sw %u, rsvd %u,%u, mask 0x%x.\n",
|
|
|
- cdev->ports[0]->name, cdev->tag_format.sw_bits,
|
|
|
- cdev->tag_format.rsvd_bits, cdev->tag_format.rsvd_shift,
|
|
|
- cdev->tag_format.rsvd_mask);
|
|
|
-
|
|
|
- cdev->tx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
|
|
|
- ddp->max_txsz - ISCSI_PDU_NONPAYLOAD_LEN);
|
|
|
- cdev->rx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
|
|
|
- ddp->max_rxsz - ISCSI_PDU_NONPAYLOAD_LEN);
|
|
|
-
|
|
|
- log_debug(1 << CXGBI_DBG_DDP,
|
|
|
- "%s max payload size: %u/%u, %u/%u.\n",
|
|
|
- cdev->ports[0]->name, cdev->tx_max_size, ddp->max_txsz,
|
|
|
- cdev->rx_max_size, ddp->max_rxsz);
|
|
|
- return 0;
|
|
|
-}
|
|
|
-EXPORT_SYMBOL_GPL(cxgbi_ddp_init);
|
|
|
-
|
|
|
/*
|
|
|
* APIs interacting with open-iscsi libraries
|
|
|
*/
|
|
|
@@ -1688,77 +1184,15 @@ static unsigned char padding[4];
|
|
|
|
|
|
static void task_release_itt(struct iscsi_task *task, itt_t hdr_itt)
|
|
|
{
|
|
|
- struct scsi_cmnd *sc = task->sc;
|
|
|
- struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data;
|
|
|
- struct cxgbi_conn *cconn = tcp_conn->dd_data;
|
|
|
- struct cxgbi_hba *chba = cconn->chba;
|
|
|
- struct cxgbi_tag_format *tformat = &chba->cdev->tag_format;
|
|
|
- u32 tag = ntohl((__force u32)hdr_itt);
|
|
|
-
|
|
|
- log_debug(1 << CXGBI_DBG_DDP,
|
|
|
- "cdev 0x%p, release tag 0x%x.\n", chba->cdev, tag);
|
|
|
- if (sc &&
|
|
|
- (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_FROM_DEVICE) &&
|
|
|
- cxgbi_is_ddp_tag(tformat, tag))
|
|
|
- ddp_tag_release(chba, tag);
|
|
|
}
|
|
|
|
|
|
static int task_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt)
|
|
|
{
|
|
|
- struct scsi_cmnd *sc = task->sc;
|
|
|
- struct iscsi_conn *conn = task->conn;
|
|
|
- struct iscsi_session *sess = conn->session;
|
|
|
- struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
|
|
|
- struct cxgbi_conn *cconn = tcp_conn->dd_data;
|
|
|
- struct cxgbi_hba *chba = cconn->chba;
|
|
|
- struct cxgbi_tag_format *tformat = &chba->cdev->tag_format;
|
|
|
- u32 sw_tag = (sess->age << cconn->task_idx_bits) | task->itt;
|
|
|
- u32 tag = 0;
|
|
|
- int err = -EINVAL;
|
|
|
-
|
|
|
- if (sc &&
|
|
|
- (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_FROM_DEVICE)) {
|
|
|
- err = cxgbi_ddp_reserve(cconn->cep->csk, &tag, sw_tag,
|
|
|
- scsi_in(sc)->length,
|
|
|
- scsi_in(sc)->table.sgl,
|
|
|
- scsi_in(sc)->table.nents,
|
|
|
- GFP_ATOMIC);
|
|
|
- if (err < 0)
|
|
|
- log_debug(1 << CXGBI_DBG_DDP,
|
|
|
- "csk 0x%p, R task 0x%p, %u,%u, no ddp.\n",
|
|
|
- cconn->cep->csk, task, scsi_in(sc)->length,
|
|
|
- scsi_in(sc)->table.nents);
|
|
|
- }
|
|
|
-
|
|
|
- if (err < 0)
|
|
|
- tag = cxgbi_set_non_ddp_tag(tformat, sw_tag);
|
|
|
- /* the itt need to sent in big-endian order */
|
|
|
- *hdr_itt = (__force itt_t)htonl(tag);
|
|
|
-
|
|
|
- log_debug(1 << CXGBI_DBG_DDP,
|
|
|
- "cdev 0x%p, task 0x%p, 0x%x(0x%x,0x%x)->0x%x/0x%x.\n",
|
|
|
- chba->cdev, task, sw_tag, task->itt, sess->age, tag, *hdr_itt);
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
void cxgbi_parse_pdu_itt(struct iscsi_conn *conn, itt_t itt, int *idx, int *age)
|
|
|
{
|
|
|
- struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
|
|
|
- struct cxgbi_conn *cconn = tcp_conn->dd_data;
|
|
|
- struct cxgbi_device *cdev = cconn->chba->cdev;
|
|
|
- u32 tag = ntohl((__force u32) itt);
|
|
|
- u32 sw_bits;
|
|
|
-
|
|
|
- sw_bits = cxgbi_tag_nonrsvd_bits(&cdev->tag_format, tag);
|
|
|
- if (idx)
|
|
|
- *idx = sw_bits & ((1 << cconn->task_idx_bits) - 1);
|
|
|
- if (age)
|
|
|
- *age = (sw_bits >> cconn->task_idx_bits) & ISCSI_AGE_MASK;
|
|
|
-
|
|
|
- log_debug(1 << CXGBI_DBG_DDP,
|
|
|
- "cdev 0x%p, tag 0x%x/0x%x, -> 0x%x(0x%x,0x%x).\n",
|
|
|
- cdev, tag, itt, sw_bits, idx ? *idx : 0xFFFFF,
|
|
|
- age ? *age : 0xFF);
|
|
|
}
|
|
|
EXPORT_SYMBOL_GPL(cxgbi_parse_pdu_itt);
|
|
|
|
|
|
@@ -2540,9 +1974,6 @@ int cxgbi_bind_conn(struct iscsi_cls_session *cls_session,
|
|
|
/* setup ddp pagesize */
|
|
|
cep = ep->dd_data;
|
|
|
csk = cep->csk;
|
|
|
- err = csk->cdev->csk_ddp_setup_pgidx(csk, csk->tid, page_idx, 0);
|
|
|
- if (err < 0)
|
|
|
- return err;
|
|
|
|
|
|
err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
|
|
|
if (err)
|
|
|
@@ -2915,16 +2346,7 @@ EXPORT_SYMBOL_GPL(cxgbi_attr_is_visible);
|
|
|
|
|
|
static int __init libcxgbi_init_module(void)
|
|
|
{
|
|
|
- sw_tag_idx_bits = (__ilog2_u32(ISCSI_ITT_MASK)) + 1;
|
|
|
- sw_tag_age_bits = (__ilog2_u32(ISCSI_AGE_MASK)) + 1;
|
|
|
-
|
|
|
pr_info("%s", version);
|
|
|
-
|
|
|
- pr_info("tag itt 0x%x, %u bits, age 0x%x, %u bits.\n",
|
|
|
- ISCSI_ITT_MASK, sw_tag_idx_bits,
|
|
|
- ISCSI_AGE_MASK, sw_tag_age_bits);
|
|
|
-
|
|
|
- ddp_setup_host_page_size();
|
|
|
return 0;
|
|
|
}
|
|
|
|
Varun Prakash