|
|
@@ -113,26 +113,6 @@ static unsigned int ib_qib_disable_sma;
|
|
|
module_param_named(disable_sma, ib_qib_disable_sma, uint, S_IWUSR | S_IRUGO);
|
|
|
MODULE_PARM_DESC(disable_sma, "Disable the SMA");
|
|
|
|
|
|
-/*
|
|
|
- * Note that it is OK to post send work requests in the SQE and ERR
|
|
|
- * states; qib_do_send() will process them and generate error
|
|
|
- * completions as per IB 1.2 C10-96.
|
|
|
- */
|
|
|
-const int ib_qib_state_ops[IB_QPS_ERR + 1] = {
|
|
|
- [IB_QPS_RESET] = 0,
|
|
|
- [IB_QPS_INIT] = QIB_POST_RECV_OK,
|
|
|
- [IB_QPS_RTR] = QIB_POST_RECV_OK | QIB_PROCESS_RECV_OK,
|
|
|
- [IB_QPS_RTS] = QIB_POST_RECV_OK | QIB_PROCESS_RECV_OK |
|
|
|
- QIB_POST_SEND_OK | QIB_PROCESS_SEND_OK |
|
|
|
- QIB_PROCESS_NEXT_SEND_OK,
|
|
|
- [IB_QPS_SQD] = QIB_POST_RECV_OK | QIB_PROCESS_RECV_OK |
|
|
|
- QIB_POST_SEND_OK | QIB_PROCESS_SEND_OK,
|
|
|
- [IB_QPS_SQE] = QIB_POST_RECV_OK | QIB_PROCESS_RECV_OK |
|
|
|
- QIB_POST_SEND_OK | QIB_FLUSH_SEND,
|
|
|
- [IB_QPS_ERR] = QIB_POST_RECV_OK | QIB_FLUSH_RECV |
|
|
|
- QIB_POST_SEND_OK | QIB_FLUSH_SEND,
|
|
|
-};
|
|
|
-
|
|
|
/*
|
|
|
* Translate ib_wr_opcode into ib_wc_opcode.
|
|
|
*/
|
|
|
@@ -321,179 +301,7 @@ static void qib_copy_from_sge(void *data, struct rvt_sge_state *ss, u32 length)
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
- * qib_post_one_send - post one RC, UC, or UD send work request
|
|
|
- * @qp: the QP to post on
|
|
|
- * @wr: the work request to send
|
|
|
- */
|
|
|
-static int qib_post_one_send(struct rvt_qp *qp, struct ib_send_wr *wr,
|
|
|
- int *scheduled)
|
|
|
-{
|
|
|
- struct rvt_swqe *wqe;
|
|
|
- u32 next;
|
|
|
- int i;
|
|
|
- int j;
|
|
|
- int acc;
|
|
|
- int ret;
|
|
|
- unsigned long flags;
|
|
|
- struct rvt_lkey_table *rkt;
|
|
|
- struct rvt_pd *pd;
|
|
|
- int avoid_schedule = 0;
|
|
|
-
|
|
|
- spin_lock_irqsave(&qp->s_lock, flags);
|
|
|
-
|
|
|
- /* Check that state is OK to post send. */
|
|
|
- if (unlikely(!(ib_qib_state_ops[qp->state] & QIB_POST_SEND_OK)))
|
|
|
- goto bail_inval;
|
|
|
-
|
|
|
- /* IB spec says that num_sge == 0 is OK. */
|
|
|
- if (wr->num_sge > qp->s_max_sge)
|
|
|
- goto bail_inval;
|
|
|
-
|
|
|
- /*
|
|
|
- * Don't allow RDMA reads or atomic operations on UC or
|
|
|
- * undefined operations.
|
|
|
- * Make sure buffer is large enough to hold the result for atomics.
|
|
|
- */
|
|
|
- if (qp->ibqp.qp_type == IB_QPT_UC) {
|
|
|
- if ((unsigned) wr->opcode >= IB_WR_RDMA_READ)
|
|
|
- goto bail_inval;
|
|
|
- } else if (qp->ibqp.qp_type != IB_QPT_RC) {
|
|
|
- /* Check IB_QPT_SMI, IB_QPT_GSI, IB_QPT_UD opcode */
|
|
|
- if (wr->opcode != IB_WR_SEND &&
|
|
|
- wr->opcode != IB_WR_SEND_WITH_IMM)
|
|
|
- goto bail_inval;
|
|
|
- /* Check UD destination address PD */
|
|
|
- if (qp->ibqp.pd != ud_wr(wr)->ah->pd)
|
|
|
- goto bail_inval;
|
|
|
- } else if ((unsigned) wr->opcode > IB_WR_ATOMIC_FETCH_AND_ADD)
|
|
|
- goto bail_inval;
|
|
|
- else if (wr->opcode >= IB_WR_ATOMIC_CMP_AND_SWP &&
|
|
|
- (wr->num_sge == 0 ||
|
|
|
- wr->sg_list[0].length < sizeof(u64) ||
|
|
|
- wr->sg_list[0].addr & (sizeof(u64) - 1)))
|
|
|
- goto bail_inval;
|
|
|
- else if (wr->opcode >= IB_WR_RDMA_READ && !qp->s_max_rd_atomic)
|
|
|
- goto bail_inval;
|
|
|
-
|
|
|
- next = qp->s_head + 1;
|
|
|
- if (next >= qp->s_size)
|
|
|
- next = 0;
|
|
|
- if (next == qp->s_last) {
|
|
|
- ret = -ENOMEM;
|
|
|
- goto bail;
|
|
|
- }
|
|
|
-
|
|
|
- rkt = &to_idev(qp->ibqp.device)->rdi.lkey_table;
|
|
|
- pd = ibpd_to_rvtpd(qp->ibqp.pd);
|
|
|
- wqe = get_swqe_ptr(qp, qp->s_head);
|
|
|
-
|
|
|
- if (qp->ibqp.qp_type != IB_QPT_UC &&
|
|
|
- qp->ibqp.qp_type != IB_QPT_RC)
|
|
|
- memcpy(&wqe->ud_wr, ud_wr(wr), sizeof(wqe->ud_wr));
|
|
|
- else if (wr->opcode == IB_WR_REG_MR)
|
|
|
- memcpy(&wqe->reg_wr, reg_wr(wr),
|
|
|
- sizeof(wqe->reg_wr));
|
|
|
- else if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
|
|
|
- wr->opcode == IB_WR_RDMA_WRITE ||
|
|
|
- wr->opcode == IB_WR_RDMA_READ)
|
|
|
- memcpy(&wqe->rdma_wr, rdma_wr(wr), sizeof(wqe->rdma_wr));
|
|
|
- else if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
|
|
|
- wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD)
|
|
|
- memcpy(&wqe->atomic_wr, atomic_wr(wr), sizeof(wqe->atomic_wr));
|
|
|
- else
|
|
|
- memcpy(&wqe->wr, wr, sizeof(wqe->wr));
|
|
|
-
|
|
|
- wqe->length = 0;
|
|
|
- j = 0;
|
|
|
- if (wr->num_sge) {
|
|
|
- acc = wr->opcode >= IB_WR_RDMA_READ ?
|
|
|
- IB_ACCESS_LOCAL_WRITE : 0;
|
|
|
- for (i = 0; i < wr->num_sge; i++) {
|
|
|
- u32 length = wr->sg_list[i].length;
|
|
|
- int ok;
|
|
|
-
|
|
|
- if (length == 0)
|
|
|
- continue;
|
|
|
- ok = rvt_lkey_ok(rkt, pd, &wqe->sg_list[j],
|
|
|
- &wr->sg_list[i], acc);
|
|
|
- if (!ok)
|
|
|
- goto bail_inval_free;
|
|
|
- wqe->length += length;
|
|
|
- j++;
|
|
|
- }
|
|
|
- wqe->wr.num_sge = j;
|
|
|
- }
|
|
|
- if (qp->ibqp.qp_type == IB_QPT_UC ||
|
|
|
- qp->ibqp.qp_type == IB_QPT_RC) {
|
|
|
- if (wqe->length > 0x80000000U)
|
|
|
- goto bail_inval_free;
|
|
|
- if (wqe->length <= qp->pmtu)
|
|
|
- avoid_schedule = 1;
|
|
|
- } else if (wqe->length > (dd_from_ibdev(qp->ibqp.device)->pport +
|
|
|
- qp->port_num - 1)->ibmtu) {
|
|
|
- goto bail_inval_free;
|
|
|
- } else {
|
|
|
- atomic_inc(&ibah_to_rvtah(ud_wr(wr)->ah)->refcount);
|
|
|
- avoid_schedule = 1;
|
|
|
- }
|
|
|
- wqe->ssn = qp->s_ssn++;
|
|
|
- qp->s_head = next;
|
|
|
-
|
|
|
- ret = 0;
|
|
|
- goto bail;
|
|
|
-
|
|
|
-bail_inval_free:
|
|
|
- while (j) {
|
|
|
- struct rvt_sge *sge = &wqe->sg_list[--j];
|
|
|
|
|
|
- rvt_put_mr(sge->mr);
|
|
|
- }
|
|
|
-bail_inval:
|
|
|
- ret = -EINVAL;
|
|
|
-bail:
|
|
|
- if (!ret && !wr->next && !avoid_schedule &&
|
|
|
- !qib_sdma_empty(
|
|
|
- dd_from_ibdev(qp->ibqp.device)->pport + qp->port_num - 1)) {
|
|
|
- qib_schedule_send(qp);
|
|
|
- *scheduled = 1;
|
|
|
- }
|
|
|
- spin_unlock_irqrestore(&qp->s_lock, flags);
|
|
|
- return ret;
|
|
|
-}
|
|
|
-
|
|
|
-/**
|
|
|
- * qib_post_send - post a send on a QP
|
|
|
- * @ibqp: the QP to post the send on
|
|
|
- * @wr: the list of work requests to post
|
|
|
- * @bad_wr: the first bad WR is put here
|
|
|
- *
|
|
|
- * This may be called from interrupt context.
|
|
|
- */
|
|
|
-static int qib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
- struct ib_send_wr **bad_wr)
|
|
|
-{
|
|
|
- struct rvt_qp *qp = to_iqp(ibqp);
|
|
|
- struct qib_qp_priv *priv = qp->priv;
|
|
|
- int err = 0;
|
|
|
- int scheduled = 0;
|
|
|
-
|
|
|
- for (; wr; wr = wr->next) {
|
|
|
- err = qib_post_one_send(qp, wr, &scheduled);
|
|
|
- if (err) {
|
|
|
- *bad_wr = wr;
|
|
|
- goto bail;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /* Try to do the send work in the caller's context. */
|
|
|
- if (!scheduled)
|
|
|
- qib_do_send(&priv->s_work);
|
|
|
-
|
|
|
-bail:
|
|
|
- return err;
|
|
|
-}
|
|
|
-
|
|
|
-/**
|
|
|
* qib_post_receive - post a receive on a QP
|
|
|
* @ibqp: the QP to post the receive on
|
|
|
* @wr: the WR to post
|
|
|
@@ -504,13 +312,13 @@ bail:
|
|
|
static int qib_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
|
|
|
struct ib_recv_wr **bad_wr)
|
|
|
{
|
|
|
- struct rvt_qp *qp = to_iqp(ibqp);
|
|
|
+ struct rvt_qp *qp = ibqp_to_rvtqp(ibqp);
|
|
|
struct rvt_rwq *wq = qp->r_rq.wq;
|
|
|
unsigned long flags;
|
|
|
int ret;
|
|
|
|
|
|
/* Check that state is OK to post receive. */
|
|
|
- if (!(ib_qib_state_ops[qp->state] & QIB_POST_RECV_OK) || !wq) {
|
|
|
+ if (!(ib_rvt_state_ops[qp->state] & RVT_POST_RECV_OK) || !wq) {
|
|
|
*bad_wr = wr;
|
|
|
ret = -EINVAL;
|
|
|
goto bail;
|
|
|
@@ -575,7 +383,7 @@ static void qib_qp_rcv(struct qib_ctxtdata *rcd, struct qib_ib_header *hdr,
|
|
|
spin_lock(&qp->r_lock);
|
|
|
|
|
|
/* Check for valid receive state. */
|
|
|
- if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) {
|
|
|
+ if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) {
|
|
|
ibp->rvp.n_pkt_drops++;
|
|
|
goto unlock;
|
|
|
}
|
|
|
@@ -955,7 +763,7 @@ static noinline struct qib_verbs_txreq *__get_txreq(struct qib_ibdev *dev,
|
|
|
spin_unlock_irqrestore(&qp->s_lock, flags);
|
|
|
tx = list_entry(l, struct qib_verbs_txreq, txreq.list);
|
|
|
} else {
|
|
|
- if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK &&
|
|
|
+ if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK &&
|
|
|
list_empty(&priv->iowait)) {
|
|
|
dev->n_txwait++;
|
|
|
qp->s_flags |= RVT_S_WAIT_TX;
|
|
|
@@ -1136,7 +944,7 @@ static int wait_kmem(struct qib_ibdev *dev, struct rvt_qp *qp)
|
|
|
int ret = 0;
|
|
|
|
|
|
spin_lock_irqsave(&qp->s_lock, flags);
|
|
|
- if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK) {
|
|
|
+ if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) {
|
|
|
spin_lock(&dev->rdi.pending_lock);
|
|
|
if (list_empty(&priv->iowait)) {
|
|
|
if (list_empty(&dev->memwait))
|
|
|
@@ -1273,7 +1081,7 @@ static int no_bufs_available(struct rvt_qp *qp)
|
|
|
* enabling the PIO avail interrupt.
|
|
|
*/
|
|
|
spin_lock_irqsave(&qp->s_lock, flags);
|
|
|
- if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK) {
|
|
|
+ if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) {
|
|
|
spin_lock(&dev->rdi.pending_lock);
|
|
|
if (list_empty(&priv->iowait)) {
|
|
|
dev->n_piowait++;
|
|
|
@@ -2017,7 +1825,7 @@ int qib_register_ib_device(struct qib_devdata *dd)
|
|
|
ibdev->modify_qp = qib_modify_qp;
|
|
|
ibdev->query_qp = qib_query_qp;
|
|
|
ibdev->destroy_qp = qib_destroy_qp;
|
|
|
- ibdev->post_send = qib_post_send;
|
|
|
+ ibdev->post_send = NULL;
|
|
|
ibdev->post_recv = qib_post_receive;
|
|
|
ibdev->post_srq_recv = qib_post_srq_receive;
|
|
|
ibdev->create_cq = NULL;
|
|
|
@@ -2057,6 +1865,8 @@ int qib_register_ib_device(struct qib_devdata *dd)
|
|
|
dd->verbs_dev.rdi.driver_f.qp_priv_free = qp_priv_free;
|
|
|
dd->verbs_dev.rdi.driver_f.free_all_qps = qib_free_all_qps;
|
|
|
dd->verbs_dev.rdi.driver_f.notify_qp_reset = notify_qp_reset;
|
|
|
+ dd->verbs_dev.rdi.driver_f.do_send = qib_do_send;
|
|
|
+ dd->verbs_dev.rdi.driver_f.schedule_send = qib_schedule_send;
|
|
|
|
|
|
dd->verbs_dev.rdi.flags = 0;
|
|
|
|
Harish Chegondi