|
@@ -3738,24 +3738,6 @@ static void dump_wqe(struct mlx5_ib_qp *qp, int idx, int size_16)
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-static u8 get_fence(u8 fence, struct ib_send_wr *wr)
|
|
|
-{
|
|
|
- if (unlikely(wr->opcode == IB_WR_LOCAL_INV &&
|
|
|
- wr->send_flags & IB_SEND_FENCE))
|
|
|
- return MLX5_FENCE_MODE_STRONG_ORDERING;
|
|
|
-
|
|
|
- if (unlikely(fence)) {
|
|
|
- if (wr->send_flags & IB_SEND_FENCE)
|
|
|
- return MLX5_FENCE_MODE_SMALL_AND_FENCE;
|
|
|
- else
|
|
|
- return fence;
|
|
|
- } else if (unlikely(wr->send_flags & IB_SEND_FENCE)) {
|
|
|
- return MLX5_FENCE_MODE_FENCE;
|
|
|
- }
|
|
|
-
|
|
|
- return 0;
|
|
|
-}
|
|
|
-
|
|
|
static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
|
|
|
struct mlx5_wqe_ctrl_seg **ctrl,
|
|
|
struct ib_send_wr *wr, unsigned *idx,
|
|
@@ -3784,8 +3766,7 @@ static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
|
|
|
static void finish_wqe(struct mlx5_ib_qp *qp,
|
|
|
struct mlx5_wqe_ctrl_seg *ctrl,
|
|
|
u8 size, unsigned idx, u64 wr_id,
|
|
|
- int nreq, u8 fence, u8 next_fence,
|
|
|
- u32 mlx5_opcode)
|
|
|
+ int nreq, u8 fence, u32 mlx5_opcode)
|
|
|
{
|
|
|
u8 opmod = 0;
|
|
|
|
|
@@ -3793,7 +3774,6 @@ static void finish_wqe(struct mlx5_ib_qp *qp,
|
|
|
mlx5_opcode | ((u32)opmod << 24));
|
|
|
ctrl->qpn_ds = cpu_to_be32(size | (qp->trans_qp.base.mqp.qpn << 8));
|
|
|
ctrl->fm_ce_se |= fence;
|
|
|
- qp->fm_cache = next_fence;
|
|
|
if (unlikely(qp->wq_sig))
|
|
|
ctrl->signature = wq_sig(ctrl);
|
|
|
|
|
@@ -3853,7 +3833,6 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
goto out;
|
|
|
}
|
|
|
|
|
|
- fence = qp->fm_cache;
|
|
|
num_sge = wr->num_sge;
|
|
|
if (unlikely(num_sge > qp->sq.max_gs)) {
|
|
|
mlx5_ib_warn(dev, "\n");
|
|
@@ -3870,6 +3849,19 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
goto out;
|
|
|
}
|
|
|
|
|
|
+ if (wr->opcode == IB_WR_LOCAL_INV ||
|
|
|
+ wr->opcode == IB_WR_REG_MR) {
|
|
|
+ fence = dev->umr_fence;
|
|
|
+ next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
|
|
|
+ } else if (wr->send_flags & IB_SEND_FENCE) {
|
|
|
+ if (qp->next_fence)
|
|
|
+ fence = MLX5_FENCE_MODE_SMALL_AND_FENCE;
|
|
|
+ else
|
|
|
+ fence = MLX5_FENCE_MODE_FENCE;
|
|
|
+ } else {
|
|
|
+ fence = qp->next_fence;
|
|
|
+ }
|
|
|
+
|
|
|
switch (ibqp->qp_type) {
|
|
|
case IB_QPT_XRC_INI:
|
|
|
xrc = seg;
|
|
@@ -3896,7 +3888,6 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
goto out;
|
|
|
|
|
|
case IB_WR_LOCAL_INV:
|
|
|
- next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
|
|
|
qp->sq.wr_data[idx] = IB_WR_LOCAL_INV;
|
|
|
ctrl->imm = cpu_to_be32(wr->ex.invalidate_rkey);
|
|
|
set_linv_wr(qp, &seg, &size);
|
|
@@ -3904,7 +3895,6 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
break;
|
|
|
|
|
|
case IB_WR_REG_MR:
|
|
|
- next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
|
|
|
qp->sq.wr_data[idx] = IB_WR_REG_MR;
|
|
|
ctrl->imm = cpu_to_be32(reg_wr(wr)->key);
|
|
|
err = set_reg_wr(qp, reg_wr(wr), &seg, &size);
|
|
@@ -3927,9 +3917,8 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
goto out;
|
|
|
}
|
|
|
|
|
|
- finish_wqe(qp, ctrl, size, idx, wr->wr_id,
|
|
|
- nreq, get_fence(fence, wr),
|
|
|
- next_fence, MLX5_OPCODE_UMR);
|
|
|
+ finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,
|
|
|
+ fence, MLX5_OPCODE_UMR);
|
|
|
/*
|
|
|
* SET_PSV WQEs are not signaled and solicited
|
|
|
* on error
|
|
@@ -3954,9 +3943,8 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
goto out;
|
|
|
}
|
|
|
|
|
|
- finish_wqe(qp, ctrl, size, idx, wr->wr_id,
|
|
|
- nreq, get_fence(fence, wr),
|
|
|
- next_fence, MLX5_OPCODE_SET_PSV);
|
|
|
+ finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,
|
|
|
+ fence, MLX5_OPCODE_SET_PSV);
|
|
|
err = begin_wqe(qp, &seg, &ctrl, wr,
|
|
|
&idx, &size, nreq);
|
|
|
if (err) {
|
|
@@ -3966,7 +3954,6 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
goto out;
|
|
|
}
|
|
|
|
|
|
- next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
|
|
|
err = set_psv_wr(&sig_handover_wr(wr)->sig_attrs->wire,
|
|
|
mr->sig->psv_wire.psv_idx, &seg,
|
|
|
&size);
|
|
@@ -3976,9 +3963,9 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
goto out;
|
|
|
}
|
|
|
|
|
|
- finish_wqe(qp, ctrl, size, idx, wr->wr_id,
|
|
|
- nreq, get_fence(fence, wr),
|
|
|
- next_fence, MLX5_OPCODE_SET_PSV);
|
|
|
+ finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,
|
|
|
+ fence, MLX5_OPCODE_SET_PSV);
|
|
|
+ qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
|
|
|
num_sge = 0;
|
|
|
goto skip_psv;
|
|
|
|
|
@@ -4089,8 +4076,8 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,
|
|
|
- get_fence(fence, wr), next_fence,
|
|
|
+ qp->next_fence = next_fence;
|
|
|
+ finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq, fence,
|
|
|
mlx5_ib_opcode[wr->opcode]);
|
|
|
skip_psv:
|
|
|
if (0)
|