|
@@ -232,13 +232,26 @@ static bool bt_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data)
|
|
|
|
|
|
/*
|
|
|
* We can hit rq == NULL here, because the tagging functions
|
|
|
- * test and set the bit before assining ->rqs[].
|
|
|
+ * test and set the bit before assigning ->rqs[].
|
|
|
*/
|
|
|
if (rq && rq->q == hctx->queue)
|
|
|
iter_data->fn(hctx, rq, iter_data->data, reserved);
|
|
|
return true;
|
|
|
}
|
|
|
|
|
|
+/**
|
|
|
+ * bt_for_each - iterate over the requests associated with a hardware queue
|
|
|
+ * @hctx: Hardware queue to examine.
|
|
|
+ * @bt: sbitmap to examine. This is either the breserved_tags member
|
|
|
+ * or the bitmap_tags member of struct blk_mq_tags.
|
|
|
+ * @fn: Pointer to the function that will be called for each request
|
|
|
+ * associated with @hctx that has been assigned a driver tag.
|
|
|
+ * @fn will be called as follows: @fn(@hctx, rq, @data, @reserved)
|
|
|
+ * where rq is a pointer to a request.
|
|
|
+ * @data: Will be passed as third argument to @fn.
|
|
|
+ * @reserved: Indicates whether @bt is the breserved_tags member or the
|
|
|
+ * bitmap_tags member of struct blk_mq_tags.
|
|
|
+ */
|
|
|
static void bt_for_each(struct blk_mq_hw_ctx *hctx, struct sbitmap_queue *bt,
|
|
|
busy_iter_fn *fn, void *data, bool reserved)
|
|
|
{
|
|
@@ -280,6 +293,18 @@ static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data)
|
|
|
return true;
|
|
|
}
|
|
|
|
|
|
+/**
|
|
|
+ * bt_tags_for_each - iterate over the requests in a tag map
|
|
|
+ * @tags: Tag map to iterate over.
|
|
|
+ * @bt: sbitmap to examine. This is either the breserved_tags member
|
|
|
+ * or the bitmap_tags member of struct blk_mq_tags.
|
|
|
+ * @fn: Pointer to the function that will be called for each started
|
|
|
+ * request. @fn will be called as follows: @fn(rq, @data,
|
|
|
+ * @reserved) where rq is a pointer to a request.
|
|
|
+ * @data: Will be passed as second argument to @fn.
|
|
|
+ * @reserved: Indicates whether @bt is the breserved_tags member or the
|
|
|
+ * bitmap_tags member of struct blk_mq_tags.
|
|
|
+ */
|
|
|
static void bt_tags_for_each(struct blk_mq_tags *tags, struct sbitmap_queue *bt,
|
|
|
busy_tag_iter_fn *fn, void *data, bool reserved)
|
|
|
{
|
|
@@ -294,6 +319,15 @@ static void bt_tags_for_each(struct blk_mq_tags *tags, struct sbitmap_queue *bt,
|
|
|
sbitmap_for_each_set(&bt->sb, bt_tags_iter, &iter_data);
|
|
|
}
|
|
|
|
|
|
+/**
|
|
|
+ * blk_mq_all_tag_busy_iter - iterate over all started requests in a tag map
|
|
|
+ * @tags: Tag map to iterate over.
|
|
|
+ * @fn: Pointer to the function that will be called for each started
|
|
|
+ * request. @fn will be called as follows: @fn(rq, @priv,
|
|
|
+ * reserved) where rq is a pointer to a request. 'reserved'
|
|
|
+ * indicates whether or not @rq is a reserved request.
|
|
|
+ * @priv: Will be passed as second argument to @fn.
|
|
|
+ */
|
|
|
static void blk_mq_all_tag_busy_iter(struct blk_mq_tags *tags,
|
|
|
busy_tag_iter_fn *fn, void *priv)
|
|
|
{
|
|
@@ -302,6 +336,15 @@ static void blk_mq_all_tag_busy_iter(struct blk_mq_tags *tags,
|
|
|
bt_tags_for_each(tags, &tags->bitmap_tags, fn, priv, false);
|
|
|
}
|
|
|
|
|
|
+/**
|
|
|
+ * blk_mq_tagset_busy_iter - iterate over all started requests in a tag set
|
|
|
+ * @tagset: Tag set to iterate over.
|
|
|
+ * @fn: Pointer to the function that will be called for each started
|
|
|
+ * request. @fn will be called as follows: @fn(rq, @priv,
|
|
|
+ * reserved) where rq is a pointer to a request. 'reserved'
|
|
|
+ * indicates whether or not @rq is a reserved request.
|
|
|
+ * @priv: Will be passed as second argument to @fn.
|
|
|
+ */
|
|
|
void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset,
|
|
|
busy_tag_iter_fn *fn, void *priv)
|
|
|
{
|
|
@@ -314,6 +357,20 @@ void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset,
|
|
|
}
|
|
|
EXPORT_SYMBOL(blk_mq_tagset_busy_iter);
|
|
|
|
|
|
+/**
|
|
|
+ * blk_mq_queue_tag_busy_iter - iterate over all requests with a driver tag
|
|
|
+ * @q: Request queue to examine.
|
|
|
+ * @fn: Pointer to the function that will be called for each request
|
|
|
+ * on @q. @fn will be called as follows: @fn(hctx, rq, @priv,
|
|
|
+ * reserved) where rq is a pointer to a request and hctx points
|
|
|
+ * to the hardware queue associated with the request. 'reserved'
|
|
|
+ * indicates whether or not @rq is a reserved request.
|
|
|
+ * @priv: Will be passed as third argument to @fn.
|
|
|
+ *
|
|
|
+ * Note: if @q->tag_set is shared with other request queues then @fn will be
|
|
|
+ * called for all requests on all queues that share that tag set and not only
|
|
|
+ * for requests associated with @q.
|
|
|
+ */
|
|
|
void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn,
|
|
|
void *priv)
|
|
|
{
|
|
@@ -321,11 +378,11 @@ void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn,
|
|
|
int i;
|
|
|
|
|
|
/*
|
|
|
- * __blk_mq_update_nr_hw_queues will update the nr_hw_queues and
|
|
|
- * queue_hw_ctx after freeze the queue. So we could use q_usage_counter
|
|
|
- * to avoid race with it. __blk_mq_update_nr_hw_queues will users
|
|
|
- * synchronize_rcu to ensure all of the users go out of the critical
|
|
|
- * section below and see zeroed q_usage_counter.
|
|
|
+ * __blk_mq_update_nr_hw_queues() updates nr_hw_queues and queue_hw_ctx
|
|
|
+ * while the queue is frozen. So we can use q_usage_counter to avoid
|
|
|
+ * racing with it. __blk_mq_update_nr_hw_queues() uses
|
|
|
+ * synchronize_rcu() to ensure this function left the critical section
|
|
|
+ * below.
|
|
|
*/
|
|
|
rcu_read_lock();
|
|
|
if (percpu_ref_is_zero(&q->q_usage_counter)) {
|
|
@@ -337,7 +394,7 @@ void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn,
|
|
|
struct blk_mq_tags *tags = hctx->tags;
|
|
|
|
|
|
/*
|
|
|
- * If not software queues are currently mapped to this
|
|
|
+ * If no software queues are currently mapped to this
|
|
|
* hardware queue, there's nothing to check
|
|
|
*/
|
|
|
if (!blk_mq_hw_queue_mapped(hctx))
|