|
@@ -406,12 +406,19 @@ static void __blk_mq_complete_request_remote(void *data)
|
|
rq->q->softirq_done_fn(rq);
|
|
rq->q->softirq_done_fn(rq);
|
|
}
|
|
}
|
|
|
|
|
|
-static void blk_mq_ipi_complete_request(struct request *rq)
|
|
|
|
|
|
+static void __blk_mq_complete_request(struct request *rq)
|
|
{
|
|
{
|
|
struct blk_mq_ctx *ctx = rq->mq_ctx;
|
|
struct blk_mq_ctx *ctx = rq->mq_ctx;
|
|
bool shared = false;
|
|
bool shared = false;
|
|
int cpu;
|
|
int cpu;
|
|
|
|
|
|
|
|
+ if (rq->internal_tag != -1)
|
|
|
|
+ blk_mq_sched_completed_request(rq);
|
|
|
|
+ if (rq->rq_flags & RQF_STATS) {
|
|
|
|
+ blk_mq_poll_stats_start(rq->q);
|
|
|
|
+ blk_stat_add(rq);
|
|
|
|
+ }
|
|
|
|
+
|
|
if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
|
|
if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
|
|
rq->q->softirq_done_fn(rq);
|
|
rq->q->softirq_done_fn(rq);
|
|
return;
|
|
return;
|
|
@@ -432,22 +439,6 @@ static void blk_mq_ipi_complete_request(struct request *rq)
|
|
put_cpu();
|
|
put_cpu();
|
|
}
|
|
}
|
|
|
|
|
|
-static void blk_mq_stat_add(struct request *rq)
|
|
|
|
-{
|
|
|
|
- if (rq->rq_flags & RQF_STATS) {
|
|
|
|
- blk_mq_poll_stats_start(rq->q);
|
|
|
|
- blk_stat_add(rq);
|
|
|
|
- }
|
|
|
|
-}
|
|
|
|
-
|
|
|
|
-static void __blk_mq_complete_request(struct request *rq)
|
|
|
|
-{
|
|
|
|
- if (rq->internal_tag != -1)
|
|
|
|
- blk_mq_sched_completed_request(rq);
|
|
|
|
- blk_mq_stat_add(rq);
|
|
|
|
- blk_mq_ipi_complete_request(rq);
|
|
|
|
-}
|
|
|
|
-
|
|
|
|
/**
|
|
/**
|
|
* blk_mq_complete_request - end I/O on a request
|
|
* blk_mq_complete_request - end I/O on a request
|
|
* @rq: the request being processed
|
|
* @rq: the request being processed
|