|
|
@@ -9,12 +9,158 @@
|
|
|
|
|
|
#include "blk.h"
|
|
|
|
|
|
+static struct bio *blk_bio_discard_split(struct request_queue *q,
|
|
|
+ struct bio *bio,
|
|
|
+ struct bio_set *bs)
|
|
|
+{
|
|
|
+ unsigned int max_discard_sectors, granularity;
|
|
|
+ int alignment;
|
|
|
+ sector_t tmp;
|
|
|
+ unsigned split_sectors;
|
|
|
+
|
|
|
+ /* Zero-sector (unknown) and one-sector granularities are the same. */
|
|
|
+ granularity = max(q->limits.discard_granularity >> 9, 1U);
|
|
|
+
|
|
|
+ max_discard_sectors = min(q->limits.max_discard_sectors, UINT_MAX >> 9);
|
|
|
+ max_discard_sectors -= max_discard_sectors % granularity;
|
|
|
+
|
|
|
+ if (unlikely(!max_discard_sectors)) {
|
|
|
+ /* XXX: warn */
|
|
|
+ return NULL;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (bio_sectors(bio) <= max_discard_sectors)
|
|
|
+ return NULL;
|
|
|
+
|
|
|
+ split_sectors = max_discard_sectors;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * If the next starting sector would be misaligned, stop the discard at
|
|
|
+ * the previous aligned sector.
|
|
|
+ */
|
|
|
+ alignment = (q->limits.discard_alignment >> 9) % granularity;
|
|
|
+
|
|
|
+ tmp = bio->bi_iter.bi_sector + split_sectors - alignment;
|
|
|
+ tmp = sector_div(tmp, granularity);
|
|
|
+
|
|
|
+ if (split_sectors > tmp)
|
|
|
+ split_sectors -= tmp;
|
|
|
+
|
|
|
+ return bio_split(bio, split_sectors, GFP_NOIO, bs);
|
|
|
+}
|
|
|
+
|
|
|
+static struct bio *blk_bio_write_same_split(struct request_queue *q,
|
|
|
+ struct bio *bio,
|
|
|
+ struct bio_set *bs)
|
|
|
+{
|
|
|
+ if (!q->limits.max_write_same_sectors)
|
|
|
+ return NULL;
|
|
|
+
|
|
|
+ if (bio_sectors(bio) <= q->limits.max_write_same_sectors)
|
|
|
+ return NULL;
|
|
|
+
|
|
|
+ return bio_split(bio, q->limits.max_write_same_sectors, GFP_NOIO, bs);
|
|
|
+}
|
|
|
+
|
|
|
+static struct bio *blk_bio_segment_split(struct request_queue *q,
|
|
|
+ struct bio *bio,
|
|
|
+ struct bio_set *bs)
|
|
|
+{
|
|
|
+ struct bio *split;
|
|
|
+ struct bio_vec bv, bvprv;
|
|
|
+ struct bvec_iter iter;
|
|
|
+ unsigned seg_size = 0, nsegs = 0;
|
|
|
+ int prev = 0;
|
|
|
+
|
|
|
+ struct bvec_merge_data bvm = {
|
|
|
+ .bi_bdev = bio->bi_bdev,
|
|
|
+ .bi_sector = bio->bi_iter.bi_sector,
|
|
|
+ .bi_size = 0,
|
|
|
+ .bi_rw = bio->bi_rw,
|
|
|
+ };
|
|
|
+
|
|
|
+ bio_for_each_segment(bv, bio, iter) {
|
|
|
+ if (q->merge_bvec_fn &&
|
|
|
+ q->merge_bvec_fn(q, &bvm, &bv) < (int) bv.bv_len)
|
|
|
+ goto split;
|
|
|
+
|
|
|
+ bvm.bi_size += bv.bv_len;
|
|
|
+
|
|
|
+ if (bvm.bi_size >> 9 > queue_max_sectors(q))
|
|
|
+ goto split;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * If the queue doesn't support SG gaps and adding this
|
|
|
+ * offset would create a gap, disallow it.
|
|
|
+ */
|
|
|
+ if (q->queue_flags & (1 << QUEUE_FLAG_SG_GAPS) &&
|
|
|
+ prev && bvec_gap_to_prev(&bvprv, bv.bv_offset))
|
|
|
+ goto split;
|
|
|
+
|
|
|
+ if (prev && blk_queue_cluster(q)) {
|
|
|
+ if (seg_size + bv.bv_len > queue_max_segment_size(q))
|
|
|
+ goto new_segment;
|
|
|
+ if (!BIOVEC_PHYS_MERGEABLE(&bvprv, &bv))
|
|
|
+ goto new_segment;
|
|
|
+ if (!BIOVEC_SEG_BOUNDARY(q, &bvprv, &bv))
|
|
|
+ goto new_segment;
|
|
|
+
|
|
|
+ seg_size += bv.bv_len;
|
|
|
+ bvprv = bv;
|
|
|
+ prev = 1;
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+new_segment:
|
|
|
+ if (nsegs == queue_max_segments(q))
|
|
|
+ goto split;
|
|
|
+
|
|
|
+ nsegs++;
|
|
|
+ bvprv = bv;
|
|
|
+ prev = 1;
|
|
|
+ seg_size = bv.bv_len;
|
|
|
+ }
|
|
|
+
|
|
|
+ return NULL;
|
|
|
+split:
|
|
|
+ split = bio_clone_bioset(bio, GFP_NOIO, bs);
|
|
|
+
|
|
|
+ split->bi_iter.bi_size -= iter.bi_size;
|
|
|
+ bio->bi_iter = iter;
|
|
|
+
|
|
|
+ if (bio_integrity(bio)) {
|
|
|
+ bio_integrity_advance(bio, split->bi_iter.bi_size);
|
|
|
+ bio_integrity_trim(split, 0, bio_sectors(split));
|
|
|
+ }
|
|
|
+
|
|
|
+ return split;
|
|
|
+}
|
|
|
+
|
|
|
+void blk_queue_split(struct request_queue *q, struct bio **bio,
|
|
|
+ struct bio_set *bs)
|
|
|
+{
|
|
|
+ struct bio *split;
|
|
|
+
|
|
|
+ if ((*bio)->bi_rw & REQ_DISCARD)
|
|
|
+ split = blk_bio_discard_split(q, *bio, bs);
|
|
|
+ else if ((*bio)->bi_rw & REQ_WRITE_SAME)
|
|
|
+ split = blk_bio_write_same_split(q, *bio, bs);
|
|
|
+ else
|
|
|
+ split = blk_bio_segment_split(q, *bio, q->bio_split);
|
|
|
+
|
|
|
+ if (split) {
|
|
|
+ bio_chain(split, *bio);
|
|
|
+ generic_make_request(*bio);
|
|
|
+ *bio = split;
|
|
|
+ }
|
|
|
+}
|
|
|
+EXPORT_SYMBOL(blk_queue_split);
|
|
|
+
|
|
|
static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
|
|
|
struct bio *bio,
|
|
|
bool no_sg_merge)
|
|
|
{
|
|
|
struct bio_vec bv, bvprv = { NULL };
|
|
|
- int cluster, high, highprv = 1;
|
|
|
+ int cluster, prev = 0;
|
|
|
unsigned int seg_size, nr_phys_segs;
|
|
|
struct bio *fbio, *bbio;
|
|
|
struct bvec_iter iter;
|
|
|
@@ -36,7 +182,6 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
|
|
|
cluster = blk_queue_cluster(q);
|
|
|
seg_size = 0;
|
|
|
nr_phys_segs = 0;
|
|
|
- high = 0;
|
|
|
for_each_bio(bio) {
|
|
|
bio_for_each_segment(bv, bio, iter) {
|
|
|
/*
|
|
|
@@ -46,13 +191,7 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
|
|
|
if (no_sg_merge)
|
|
|
goto new_segment;
|
|
|
|
|
|
- /*
|
|
|
- * the trick here is making sure that a high page is
|
|
|
- * never considered part of another segment, since
|
|
|
- * that might change with the bounce page.
|
|
|
- */
|
|
|
- high = page_to_pfn(bv.bv_page) > queue_bounce_pfn(q);
|
|
|
- if (!high && !highprv && cluster) {
|
|
|
+ if (prev && cluster) {
|
|
|
if (seg_size + bv.bv_len
|
|
|
> queue_max_segment_size(q))
|
|
|
goto new_segment;
|
|
|
@@ -72,8 +211,8 @@ new_segment:
|
|
|
|
|
|
nr_phys_segs++;
|
|
|
bvprv = bv;
|
|
|
+ prev = 1;
|
|
|
seg_size = bv.bv_len;
|
|
|
- highprv = high;
|
|
|
}
|
|
|
bbio = bio;
|
|
|
}
|
Kent Overstreet