|
@@ -84,23 +84,71 @@ xfs_find_bdev_for_inode(
|
|
|
}
|
|
|
|
|
|
/*
|
|
|
- * We're now finished for good with this ioend structure.
|
|
|
- * Update the page state via the associated buffer_heads,
|
|
|
- * release holds on the inode and bio, and finally free
|
|
|
- * up memory. Do not use the ioend after this.
|
|
|
+ * We're now finished for good with this page. Update the page state via the
|
|
|
+ * associated buffer_heads, paying attention to the start and end offsets that
|
|
|
+ * we need to process on the page.
|
|
|
+ */
|
|
|
+static void
|
|
|
+xfs_finish_page_writeback(
|
|
|
+ struct inode *inode,
|
|
|
+ struct bio_vec *bvec,
|
|
|
+ int error)
|
|
|
+{
|
|
|
+ unsigned int end = bvec->bv_offset + bvec->bv_len - 1;
|
|
|
+ struct buffer_head *head, *bh;
|
|
|
+ unsigned int off = 0;
|
|
|
+
|
|
|
+ ASSERT(bvec->bv_offset < PAGE_SIZE);
|
|
|
+ ASSERT((bvec->bv_offset & ((1 << inode->i_blkbits) - 1)) == 0);
|
|
|
+ ASSERT(end < PAGE_SIZE);
|
|
|
+ ASSERT((bvec->bv_len & ((1 << inode->i_blkbits) - 1)) == 0);
|
|
|
+
|
|
|
+ bh = head = page_buffers(bvec->bv_page);
|
|
|
+
|
|
|
+ do {
|
|
|
+ if (off < bvec->bv_offset)
|
|
|
+ goto next_bh;
|
|
|
+ if (off > end)
|
|
|
+ break;
|
|
|
+ bh->b_end_io(bh, !error);
|
|
|
+next_bh:
|
|
|
+ off += bh->b_size;
|
|
|
+ } while ((bh = bh->b_this_page) != head);
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * We're now finished for good with this ioend structure. Update the page
|
|
|
+ * state, release holds on bios, and finally free up memory. Do not use the
|
|
|
+ * ioend after this.
|
|
|
*/
|
|
|
STATIC void
|
|
|
xfs_destroy_ioend(
|
|
|
- xfs_ioend_t *ioend)
|
|
|
+ struct xfs_ioend *ioend,
|
|
|
+ int error)
|
|
|
{
|
|
|
- struct buffer_head *bh, *next;
|
|
|
+ struct inode *inode = ioend->io_inode;
|
|
|
+ struct bio *last = ioend->io_bio;
|
|
|
+ struct bio *bio, *next;
|
|
|
|
|
|
- for (bh = ioend->io_buffer_head; bh; bh = next) {
|
|
|
- next = bh->b_private;
|
|
|
- bh->b_end_io(bh, !ioend->io_error);
|
|
|
- }
|
|
|
+ for (bio = &ioend->io_inline_bio; bio; bio = next) {
|
|
|
+ struct bio_vec *bvec;
|
|
|
+ int i;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * For the last bio, bi_private points to the ioend, so we
|
|
|
+ * need to explicitly end the iteration here.
|
|
|
+ */
|
|
|
+ if (bio == last)
|
|
|
+ next = NULL;
|
|
|
+ else
|
|
|
+ next = bio->bi_private;
|
|
|
|
|
|
- mempool_free(ioend, xfs_ioend_pool);
|
|
|
+ /* walk each page on bio, ending page IO on them */
|
|
|
+ bio_for_each_segment_all(bvec, bio, i)
|
|
|
+ xfs_finish_page_writeback(inode, bvec, error);
|
|
|
+
|
|
|
+ bio_put(bio);
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -120,13 +168,9 @@ xfs_setfilesize_trans_alloc(
|
|
|
struct xfs_trans *tp;
|
|
|
int error;
|
|
|
|
|
|
- tp = xfs_trans_alloc(mp, XFS_TRANS_FSYNC_TS);
|
|
|
-
|
|
|
- error = xfs_trans_reserve(tp, &M_RES(mp)->tr_fsyncts, 0, 0);
|
|
|
- if (error) {
|
|
|
- xfs_trans_cancel(tp);
|
|
|
+ error = xfs_trans_alloc(mp, &M_RES(mp)->tr_fsyncts, 0, 0, 0, &tp);
|
|
|
+ if (error)
|
|
|
return error;
|
|
|
- }
|
|
|
|
|
|
ioend->io_append_trans = tp;
|
|
|
|
|
@@ -174,7 +218,8 @@ xfs_setfilesize(
|
|
|
|
|
|
STATIC int
|
|
|
xfs_setfilesize_ioend(
|
|
|
- struct xfs_ioend *ioend)
|
|
|
+ struct xfs_ioend *ioend,
|
|
|
+ int error)
|
|
|
{
|
|
|
struct xfs_inode *ip = XFS_I(ioend->io_inode);
|
|
|
struct xfs_trans *tp = ioend->io_append_trans;
|
|
@@ -188,36 +233,14 @@ xfs_setfilesize_ioend(
|
|
|
__sb_writers_acquired(VFS_I(ip)->i_sb, SB_FREEZE_FS);
|
|
|
|
|
|
/* we abort the update if there was an IO error */
|
|
|
- if (ioend->io_error) {
|
|
|
+ if (error) {
|
|
|
xfs_trans_cancel(tp);
|
|
|
- return ioend->io_error;
|
|
|
+ return error;
|
|
|
}
|
|
|
|
|
|
return xfs_setfilesize(ip, tp, ioend->io_offset, ioend->io_size);
|
|
|
}
|
|
|
|
|
|
-/*
|
|
|
- * Schedule IO completion handling on the final put of an ioend.
|
|
|
- *
|
|
|
- * If there is no work to do we might as well call it a day and free the
|
|
|
- * ioend right now.
|
|
|
- */
|
|
|
-STATIC void
|
|
|
-xfs_finish_ioend(
|
|
|
- struct xfs_ioend *ioend)
|
|
|
-{
|
|
|
- if (atomic_dec_and_test(&ioend->io_remaining)) {
|
|
|
- struct xfs_mount *mp = XFS_I(ioend->io_inode)->i_mount;
|
|
|
-
|
|
|
- if (ioend->io_type == XFS_IO_UNWRITTEN)
|
|
|
- queue_work(mp->m_unwritten_workqueue, &ioend->io_work);
|
|
|
- else if (ioend->io_append_trans)
|
|
|
- queue_work(mp->m_data_workqueue, &ioend->io_work);
|
|
|
- else
|
|
|
- xfs_destroy_ioend(ioend);
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
/*
|
|
|
* IO write completion.
|
|
|
*/
|
|
@@ -225,16 +248,17 @@ STATIC void
|
|
|
xfs_end_io(
|
|
|
struct work_struct *work)
|
|
|
{
|
|
|
- xfs_ioend_t *ioend = container_of(work, xfs_ioend_t, io_work);
|
|
|
- struct xfs_inode *ip = XFS_I(ioend->io_inode);
|
|
|
- int error = 0;
|
|
|
+ struct xfs_ioend *ioend =
|
|
|
+ container_of(work, struct xfs_ioend, io_work);
|
|
|
+ struct xfs_inode *ip = XFS_I(ioend->io_inode);
|
|
|
+ int error = ioend->io_bio->bi_error;
|
|
|
|
|
|
/*
|
|
|
* Set an error if the mount has shut down and proceed with end I/O
|
|
|
* processing so it can perform whatever cleanups are necessary.
|
|
|
*/
|
|
|
if (XFS_FORCED_SHUTDOWN(ip->i_mount))
|
|
|
- ioend->io_error = -EIO;
|
|
|
+ error = -EIO;
|
|
|
|
|
|
/*
|
|
|
* For unwritten extents we need to issue transactions to convert a
|
|
@@ -244,55 +268,33 @@ xfs_end_io(
|
|
|
* on error.
|
|
|
*/
|
|
|
if (ioend->io_type == XFS_IO_UNWRITTEN) {
|
|
|
- if (ioend->io_error)
|
|
|
+ if (error)
|
|
|
goto done;
|
|
|
error = xfs_iomap_write_unwritten(ip, ioend->io_offset,
|
|
|
ioend->io_size);
|
|
|
} else if (ioend->io_append_trans) {
|
|
|
- error = xfs_setfilesize_ioend(ioend);
|
|
|
+ error = xfs_setfilesize_ioend(ioend, error);
|
|
|
} else {
|
|
|
ASSERT(!xfs_ioend_is_append(ioend));
|
|
|
}
|
|
|
|
|
|
done:
|
|
|
- if (error)
|
|
|
- ioend->io_error = error;
|
|
|
- xfs_destroy_ioend(ioend);
|
|
|
+ xfs_destroy_ioend(ioend, error);
|
|
|
}
|
|
|
|
|
|
-/*
|
|
|
- * Allocate and initialise an IO completion structure.
|
|
|
- * We need to track unwritten extent write completion here initially.
|
|
|
- * We'll need to extend this for updating the ondisk inode size later
|
|
|
- * (vs. incore size).
|
|
|
- */
|
|
|
-STATIC xfs_ioend_t *
|
|
|
-xfs_alloc_ioend(
|
|
|
- struct inode *inode,
|
|
|
- unsigned int type)
|
|
|
+STATIC void
|
|
|
+xfs_end_bio(
|
|
|
+ struct bio *bio)
|
|
|
{
|
|
|
- xfs_ioend_t *ioend;
|
|
|
-
|
|
|
- ioend = mempool_alloc(xfs_ioend_pool, GFP_NOFS);
|
|
|
-
|
|
|
- /*
|
|
|
- * Set the count to 1 initially, which will prevent an I/O
|
|
|
- * completion callback from happening before we have started
|
|
|
- * all the I/O from calling the completion routine too early.
|
|
|
- */
|
|
|
- atomic_set(&ioend->io_remaining, 1);
|
|
|
- ioend->io_error = 0;
|
|
|
- INIT_LIST_HEAD(&ioend->io_list);
|
|
|
- ioend->io_type = type;
|
|
|
- ioend->io_inode = inode;
|
|
|
- ioend->io_buffer_head = NULL;
|
|
|
- ioend->io_buffer_tail = NULL;
|
|
|
- ioend->io_offset = 0;
|
|
|
- ioend->io_size = 0;
|
|
|
- ioend->io_append_trans = NULL;
|
|
|
+ struct xfs_ioend *ioend = bio->bi_private;
|
|
|
+ struct xfs_mount *mp = XFS_I(ioend->io_inode)->i_mount;
|
|
|
|
|
|
- INIT_WORK(&ioend->io_work, xfs_end_io);
|
|
|
- return ioend;
|
|
|
+ if (ioend->io_type == XFS_IO_UNWRITTEN)
|
|
|
+ queue_work(mp->m_unwritten_workqueue, &ioend->io_work);
|
|
|
+ else if (ioend->io_append_trans)
|
|
|
+ queue_work(mp->m_data_workqueue, &ioend->io_work);
|
|
|
+ else
|
|
|
+ xfs_destroy_ioend(ioend, bio->bi_error);
|
|
|
}
|
|
|
|
|
|
STATIC int
|
|
@@ -364,50 +366,6 @@ xfs_imap_valid(
|
|
|
offset < imap->br_startoff + imap->br_blockcount;
|
|
|
}
|
|
|
|
|
|
-/*
|
|
|
- * BIO completion handler for buffered IO.
|
|
|
- */
|
|
|
-STATIC void
|
|
|
-xfs_end_bio(
|
|
|
- struct bio *bio)
|
|
|
-{
|
|
|
- xfs_ioend_t *ioend = bio->bi_private;
|
|
|
-
|
|
|
- if (!ioend->io_error)
|
|
|
- ioend->io_error = bio->bi_error;
|
|
|
-
|
|
|
- /* Toss bio and pass work off to an xfsdatad thread */
|
|
|
- bio->bi_private = NULL;
|
|
|
- bio->bi_end_io = NULL;
|
|
|
- bio_put(bio);
|
|
|
-
|
|
|
- xfs_finish_ioend(ioend);
|
|
|
-}
|
|
|
-
|
|
|
-STATIC void
|
|
|
-xfs_submit_ioend_bio(
|
|
|
- struct writeback_control *wbc,
|
|
|
- xfs_ioend_t *ioend,
|
|
|
- struct bio *bio)
|
|
|
-{
|
|
|
- atomic_inc(&ioend->io_remaining);
|
|
|
- bio->bi_private = ioend;
|
|
|
- bio->bi_end_io = xfs_end_bio;
|
|
|
- submit_bio(wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE, bio);
|
|
|
-}
|
|
|
-
|
|
|
-STATIC struct bio *
|
|
|
-xfs_alloc_ioend_bio(
|
|
|
- struct buffer_head *bh)
|
|
|
-{
|
|
|
- struct bio *bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES);
|
|
|
-
|
|
|
- ASSERT(bio->bi_private == NULL);
|
|
|
- bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
|
|
|
- bio->bi_bdev = bh->b_bdev;
|
|
|
- return bio;
|
|
|
-}
|
|
|
-
|
|
|
STATIC void
|
|
|
xfs_start_buffer_writeback(
|
|
|
struct buffer_head *bh)
|
|
@@ -452,28 +410,35 @@ static inline int xfs_bio_add_buffer(struct bio *bio, struct buffer_head *bh)
|
|
|
}
|
|
|
|
|
|
/*
|
|
|
- * Submit all of the bios for an ioend. We are only passed a single ioend at a
|
|
|
- * time; the caller is responsible for chaining prior to submission.
|
|
|
+ * Submit the bio for an ioend. We are passed an ioend with a bio attached to
|
|
|
+ * it, and we submit that bio. The ioend may be used for multiple bio
|
|
|
+ * submissions, so we only want to allocate an append transaction for the ioend
|
|
|
+ * once. In the case of multiple bio submission, each bio will take an IO
|
|
|
+ * reference to the ioend to ensure that the ioend completion is only done once
|
|
|
+ * all bios have been submitted and the ioend is really done.
|
|
|
*
|
|
|
* If @fail is non-zero, it means that we have a situation where some part of
|
|
|
* the submission process has failed after we have marked paged for writeback
|
|
|
- * and unlocked them. In this situation, we need to fail the ioend chain rather
|
|
|
- * than submit it to IO. This typically only happens on a filesystem shutdown.
|
|
|
+ * and unlocked them. In this situation, we need to fail the bio and ioend
|
|
|
+ * rather than submit it to IO. This typically only happens on a filesystem
|
|
|
+ * shutdown.
|
|
|
*/
|
|
|
STATIC int
|
|
|
xfs_submit_ioend(
|
|
|
struct writeback_control *wbc,
|
|
|
- xfs_ioend_t *ioend,
|
|
|
+ struct xfs_ioend *ioend,
|
|
|
int status)
|
|
|
{
|
|
|
- struct buffer_head *bh;
|
|
|
- struct bio *bio;
|
|
|
- sector_t lastblock = 0;
|
|
|
-
|
|
|
/* Reserve log space if we might write beyond the on-disk inode size. */
|
|
|
if (!status &&
|
|
|
- ioend->io_type != XFS_IO_UNWRITTEN && xfs_ioend_is_append(ioend))
|
|
|
+ ioend->io_type != XFS_IO_UNWRITTEN &&
|
|
|
+ xfs_ioend_is_append(ioend) &&
|
|
|
+ !ioend->io_append_trans)
|
|
|
status = xfs_setfilesize_trans_alloc(ioend);
|
|
|
+
|
|
|
+ ioend->io_bio->bi_private = ioend;
|
|
|
+ ioend->io_bio->bi_end_io = xfs_end_bio;
|
|
|
+
|
|
|
/*
|
|
|
* If we are failing the IO now, just mark the ioend with an
|
|
|
* error and finish it. This will run IO completion immediately
|
|
@@ -481,33 +446,73 @@ xfs_submit_ioend(
|
|
|
* time.
|
|
|
*/
|
|
|
if (status) {
|
|
|
- ioend->io_error = status;
|
|
|
- xfs_finish_ioend(ioend);
|
|
|
+ ioend->io_bio->bi_error = status;
|
|
|
+ bio_endio(ioend->io_bio);
|
|
|
return status;
|
|
|
}
|
|
|
|
|
|
- bio = NULL;
|
|
|
- for (bh = ioend->io_buffer_head; bh; bh = bh->b_private) {
|
|
|
+ submit_bio(wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE,
|
|
|
+ ioend->io_bio);
|
|
|
+ return 0;
|
|
|
+}
|
|
|
|
|
|
- if (!bio) {
|
|
|
-retry:
|
|
|
- bio = xfs_alloc_ioend_bio(bh);
|
|
|
- } else if (bh->b_blocknr != lastblock + 1) {
|
|
|
- xfs_submit_ioend_bio(wbc, ioend, bio);
|
|
|
- goto retry;
|
|
|
- }
|
|
|
+static void
|
|
|
+xfs_init_bio_from_bh(
|
|
|
+ struct bio *bio,
|
|
|
+ struct buffer_head *bh)
|
|
|
+{
|
|
|
+ bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
|
|
|
+ bio->bi_bdev = bh->b_bdev;
|
|
|
+}
|
|
|
|
|
|
- if (xfs_bio_add_buffer(bio, bh) != bh->b_size) {
|
|
|
- xfs_submit_ioend_bio(wbc, ioend, bio);
|
|
|
- goto retry;
|
|
|
- }
|
|
|
+static struct xfs_ioend *
|
|
|
+xfs_alloc_ioend(
|
|
|
+ struct inode *inode,
|
|
|
+ unsigned int type,
|
|
|
+ xfs_off_t offset,
|
|
|
+ struct buffer_head *bh)
|
|
|
+{
|
|
|
+ struct xfs_ioend *ioend;
|
|
|
+ struct bio *bio;
|
|
|
|
|
|
- lastblock = bh->b_blocknr;
|
|
|
- }
|
|
|
- if (bio)
|
|
|
- xfs_submit_ioend_bio(wbc, ioend, bio);
|
|
|
- xfs_finish_ioend(ioend);
|
|
|
- return 0;
|
|
|
+ bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, xfs_ioend_bioset);
|
|
|
+ xfs_init_bio_from_bh(bio, bh);
|
|
|
+
|
|
|
+ ioend = container_of(bio, struct xfs_ioend, io_inline_bio);
|
|
|
+ INIT_LIST_HEAD(&ioend->io_list);
|
|
|
+ ioend->io_type = type;
|
|
|
+ ioend->io_inode = inode;
|
|
|
+ ioend->io_size = 0;
|
|
|
+ ioend->io_offset = offset;
|
|
|
+ INIT_WORK(&ioend->io_work, xfs_end_io);
|
|
|
+ ioend->io_append_trans = NULL;
|
|
|
+ ioend->io_bio = bio;
|
|
|
+ return ioend;
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * Allocate a new bio, and chain the old bio to the new one.
|
|
|
+ *
|
|
|
+ * Note that we have to do perform the chaining in this unintuitive order
|
|
|
+ * so that the bi_private linkage is set up in the right direction for the
|
|
|
+ * traversal in xfs_destroy_ioend().
|
|
|
+ */
|
|
|
+static void
|
|
|
+xfs_chain_bio(
|
|
|
+ struct xfs_ioend *ioend,
|
|
|
+ struct writeback_control *wbc,
|
|
|
+ struct buffer_head *bh)
|
|
|
+{
|
|
|
+ struct bio *new;
|
|
|
+
|
|
|
+ new = bio_alloc(GFP_NOFS, BIO_MAX_PAGES);
|
|
|
+ xfs_init_bio_from_bh(new, bh);
|
|
|
+
|
|
|
+ bio_chain(ioend->io_bio, new);
|
|
|
+ bio_get(ioend->io_bio); /* for xfs_destroy_ioend */
|
|
|
+ submit_bio(wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE,
|
|
|
+ ioend->io_bio);
|
|
|
+ ioend->io_bio = new;
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -523,27 +528,24 @@ xfs_add_to_ioend(
|
|
|
struct buffer_head *bh,
|
|
|
xfs_off_t offset,
|
|
|
struct xfs_writepage_ctx *wpc,
|
|
|
+ struct writeback_control *wbc,
|
|
|
struct list_head *iolist)
|
|
|
{
|
|
|
if (!wpc->ioend || wpc->io_type != wpc->ioend->io_type ||
|
|
|
bh->b_blocknr != wpc->last_block + 1 ||
|
|
|
offset != wpc->ioend->io_offset + wpc->ioend->io_size) {
|
|
|
- struct xfs_ioend *new;
|
|
|
-
|
|
|
if (wpc->ioend)
|
|
|
list_add(&wpc->ioend->io_list, iolist);
|
|
|
-
|
|
|
- new = xfs_alloc_ioend(inode, wpc->io_type);
|
|
|
- new->io_offset = offset;
|
|
|
- new->io_buffer_head = bh;
|
|
|
- new->io_buffer_tail = bh;
|
|
|
- wpc->ioend = new;
|
|
|
- } else {
|
|
|
- wpc->ioend->io_buffer_tail->b_private = bh;
|
|
|
- wpc->ioend->io_buffer_tail = bh;
|
|
|
+ wpc->ioend = xfs_alloc_ioend(inode, wpc->io_type, offset, bh);
|
|
|
}
|
|
|
|
|
|
- bh->b_private = NULL;
|
|
|
+ /*
|
|
|
+ * If the buffer doesn't fit into the bio we need to allocate a new
|
|
|
+ * one. This shouldn't happen more than once for a given buffer.
|
|
|
+ */
|
|
|
+ while (xfs_bio_add_buffer(wpc->ioend->io_bio, bh) != bh->b_size)
|
|
|
+ xfs_chain_bio(wpc->ioend, wbc, bh);
|
|
|
+
|
|
|
wpc->ioend->io_size += bh->b_size;
|
|
|
wpc->last_block = bh->b_blocknr;
|
|
|
xfs_start_buffer_writeback(bh);
|
|
@@ -803,7 +805,7 @@ xfs_writepage_map(
|
|
|
lock_buffer(bh);
|
|
|
if (wpc->io_type != XFS_IO_OVERWRITE)
|
|
|
xfs_map_at_offset(inode, bh, &wpc->imap, offset);
|
|
|
- xfs_add_to_ioend(inode, bh, offset, wpc, &submit_list);
|
|
|
+ xfs_add_to_ioend(inode, bh, offset, wpc, wbc, &submit_list);
|
|
|
count++;
|
|
|
}
|
|
|
|
|
@@ -1391,13 +1393,10 @@ xfs_end_io_direct_write(
|
|
|
|
|
|
trace_xfs_end_io_direct_write_append(ip, offset, size);
|
|
|
|
|
|
- tp = xfs_trans_alloc(mp, XFS_TRANS_FSYNC_TS);
|
|
|
- error = xfs_trans_reserve(tp, &M_RES(mp)->tr_fsyncts, 0, 0);
|
|
|
- if (error) {
|
|
|
- xfs_trans_cancel(tp);
|
|
|
- return error;
|
|
|
- }
|
|
|
- error = xfs_setfilesize(ip, tp, offset, size);
|
|
|
+ error = xfs_trans_alloc(mp, &M_RES(mp)->tr_fsyncts, 0, 0, 0,
|
|
|
+ &tp);
|
|
|
+ if (!error)
|
|
|
+ error = xfs_setfilesize(ip, tp, offset, size);
|
|
|
}
|
|
|
|
|
|
return error;
|