|
@@ -730,6 +730,17 @@ static inline bool nvme_cqe_valid(struct nvme_queue *nvmeq, u16 head,
|
|
return (le16_to_cpu(nvmeq->cqes[head].status) & 1) == phase;
|
|
return (le16_to_cpu(nvmeq->cqes[head].status) & 1) == phase;
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq)
|
|
|
|
+{
|
|
|
|
+ u16 head = nvmeq->cq_head;
|
|
|
|
+
|
|
|
|
+ if (likely(nvmeq->cq_vector >= 0)) {
|
|
|
|
+ if (nvme_dbbuf_update_and_check_event(head, nvmeq->dbbuf_cq_db,
|
|
|
|
+ nvmeq->dbbuf_cq_ei))
|
|
|
|
+ writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
|
|
|
|
+ }
|
|
|
|
+}
|
|
|
|
+
|
|
static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
|
|
static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
|
|
{
|
|
{
|
|
u16 head, phase;
|
|
u16 head, phase;
|
|
@@ -776,13 +787,11 @@ static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
|
|
if (head == nvmeq->cq_head && phase == nvmeq->cq_phase)
|
|
if (head == nvmeq->cq_head && phase == nvmeq->cq_phase)
|
|
return;
|
|
return;
|
|
|
|
|
|
- if (likely(nvmeq->cq_vector >= 0))
|
|
|
|
- if (nvme_dbbuf_update_and_check_event(head, nvmeq->dbbuf_cq_db,
|
|
|
|
- nvmeq->dbbuf_cq_ei))
|
|
|
|
- writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
|
|
|
|
nvmeq->cq_head = head;
|
|
nvmeq->cq_head = head;
|
|
nvmeq->cq_phase = phase;
|
|
nvmeq->cq_phase = phase;
|
|
|
|
|
|
|
|
+ nvme_ring_cq_doorbell(nvmeq);
|
|
|
|
+
|
|
nvmeq->cqe_seen = 1;
|
|
nvmeq->cqe_seen = 1;
|
|
}
|
|
}
|
|
|
|
|