diff --git a/block/fops.c b/block/fops.c index 15783a6180de..4cff76e9eb71 100644 --- a/block/fops.c +++ b/block/fops.c @@ -453,8 +453,10 @@ static int blkdev_iomap_begin(struct inode *inode, loff_t offset, loff_t length, return 0; } +static DEFINE_IOMAP_ITER_NEXT(blkdev_iomap_next, blkdev_iomap_begin); + static const struct iomap_ops blkdev_iomap_ops = { - .iomap_begin = blkdev_iomap_begin, + .iomap_next = blkdev_iomap_next, }; #ifdef CONFIG_BUFFER_HEAD diff --git a/fs/btrfs/direct-io.c b/fs/btrfs/direct-io.c index 460326d34143..d5439b06cdc9 100644 --- a/fs/btrfs/direct-io.c +++ b/fs/btrfs/direct-io.c @@ -798,9 +798,11 @@ static void btrfs_dio_submit_io(const struct iomap_iter *iter, struct bio *bio, btrfs_submit_bbio(bbio, 0); } +static DEFINE_IOMAP_ITER_NEXT_END(btrfs_dio_iomap_next, btrfs_dio_iomap_begin, + btrfs_dio_iomap_end); + static const struct iomap_ops btrfs_dio_iomap_ops = { - .iomap_begin = btrfs_dio_iomap_begin, - .iomap_end = btrfs_dio_iomap_end, + .iomap_next = btrfs_dio_iomap_next, }; static const struct iomap_dio_ops btrfs_dio_ops = { diff --git a/fs/erofs/data.c b/fs/erofs/data.c index 9aa48c8d67d1..d2f01245ee79 100644 --- a/fs/erofs/data.c +++ b/fs/erofs/data.c @@ -380,9 +380,11 @@ static int erofs_iomap_end(struct inode *inode, loff_t pos, loff_t length, return written; } +static DEFINE_IOMAP_ITER_NEXT_END(erofs_iomap_next, erofs_iomap_begin, + erofs_iomap_end); + static const struct iomap_ops erofs_iomap_ops = { - .iomap_begin = erofs_iomap_begin, - .iomap_end = erofs_iomap_end, + .iomap_next = erofs_iomap_next, }; int erofs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c index bab521613552..3966b62a7051 100644 --- a/fs/erofs/zmap.c +++ b/fs/erofs/zmap.c @@ -821,6 +821,9 @@ static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset, return 0; } +static DEFINE_IOMAP_ITER_NEXT(z_erofs_iomap_next_report, + z_erofs_iomap_begin_report); + const struct iomap_ops z_erofs_iomap_report_ops = { - .iomap_begin = z_erofs_iomap_begin_report, + .iomap_next = z_erofs_iomap_next_report, }; diff --git a/fs/exfat/iomap.c b/fs/exfat/iomap.c index 1aac38e63fe6..b6dd396aa60a 100644 --- a/fs/exfat/iomap.c +++ b/fs/exfat/iomap.c @@ -151,8 +151,10 @@ static int exfat_write_iomap_begin(struct inode *inode, loff_t offset, loff_t le return __exfat_iomap_begin(inode, offset, length, flags, iomap, true); } +static DEFINE_IOMAP_ITER_NEXT(exfat_iomap_next, exfat_iomap_begin); + const struct iomap_ops exfat_iomap_ops = { - .iomap_begin = exfat_iomap_begin, + .iomap_next = exfat_iomap_next, }; /* @@ -186,9 +188,11 @@ static int exfat_write_iomap_end(struct inode *inode, loff_t pos, loff_t length, return written; } +static DEFINE_IOMAP_ITER_NEXT_END(exfat_write_iomap_next, + exfat_write_iomap_begin, exfat_write_iomap_end); + const struct iomap_ops exfat_write_iomap_ops = { - .iomap_begin = exfat_write_iomap_begin, - .iomap_end = exfat_write_iomap_end, + .iomap_next = exfat_write_iomap_next, }; /* diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c index 29808629cce5..7e0fa9c454e1 100644 --- a/fs/ext2/inode.c +++ b/fs/ext2/inode.c @@ -860,9 +860,11 @@ ext2_iomap_end(struct inode *inode, loff_t offset, loff_t length, return 0; } +static DEFINE_IOMAP_ITER_NEXT_END(ext2_iomap_next, ext2_iomap_begin, + ext2_iomap_end); + const struct iomap_ops ext2_iomap_ops = { - .iomap_begin = ext2_iomap_begin, - .iomap_end = ext2_iomap_end, + .iomap_next = ext2_iomap_next, }; int ext2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index b37c136ea3ab..e134c0193e2b 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -4007,6 +4007,9 @@ static inline void ext4_clear_io_unwritten_flag(ext4_io_end_t *io_end) extern const struct iomap_ops ext4_iomap_ops; extern const struct iomap_ops ext4_iomap_report_ops; +int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned flags, struct iomap *iomap, struct iomap *srcmap); + static inline int ext4_buffer_uptodate(struct buffer_head *bh) { /* diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index 91c97af64b31..15972410d460 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -5171,8 +5171,10 @@ static int ext4_iomap_xattr_begin(struct inode *inode, loff_t offset, return error; } +static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_xattr_next, ext4_iomap_xattr_begin); + static const struct iomap_ops ext4_iomap_xattr_ops = { - .iomap_begin = ext4_iomap_xattr_begin, + .iomap_next = ext4_iomap_xattr_next, }; static int ext4_fiemap_check_ranges(struct inode *inode, u64 start, u64 *len) diff --git a/fs/ext4/file.c b/fs/ext4/file.c index eb1a323962b1..f20d92255546 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -91,7 +91,9 @@ static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to) return generic_file_read_iter(iocb, to); } - ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0, NULL, 0); + ret = iomap_dio_read_simple(iocb, to, ext4_iomap_begin); + if (ret == -ENOTBLK) + ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0, NULL, 0); inode_unlock_shared(inode); file_accessed(iocb->ki_filp); diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index ce99807c5f5b..bf9755b541be 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -3771,7 +3771,7 @@ static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map, } -static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length, +int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length, unsigned flags, struct iomap *iomap, struct iomap *srcmap) { int ret; @@ -3850,8 +3850,10 @@ static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length, return 0; } +static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next, ext4_iomap_begin); + const struct iomap_ops ext4_iomap_ops = { - .iomap_begin = ext4_iomap_begin, + .iomap_next = ext4_iomap_next, }; static int ext4_iomap_begin_report(struct inode *inode, loff_t offset, @@ -3905,8 +3907,10 @@ static int ext4_iomap_begin_report(struct inode *inode, loff_t offset, return 0; } +static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next_report, ext4_iomap_begin_report); + const struct iomap_ops ext4_iomap_report_ops = { - .iomap_begin = ext4_iomap_begin_report, + .iomap_next = ext4_iomap_next_report, }; /* diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index a765fda71536..8977ad379f50 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -4653,6 +4653,8 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, return 0; } +static DEFINE_IOMAP_ITER_NEXT(f2fs_iomap_next, f2fs_iomap_begin); + const struct iomap_ops f2fs_iomap_ops = { - .iomap_begin = f2fs_iomap_begin, + .iomap_next = f2fs_iomap_next, }; diff --git a/fs/fuse/dax.c b/fs/fuse/dax.c index 8b53625ac7ab..85cdf0199bc0 100644 --- a/fs/fuse/dax.c +++ b/fs/fuse/dax.c @@ -653,9 +653,11 @@ static int fuse_iomap_end(struct inode *inode, loff_t pos, loff_t length, return 0; } +static DEFINE_IOMAP_ITER_NEXT_END(fuse_iomap_next, fuse_iomap_begin, + fuse_iomap_end); + static const struct iomap_ops fuse_iomap_ops = { - .iomap_begin = fuse_iomap_begin, - .iomap_end = fuse_iomap_end, + .iomap_next = fuse_iomap_next, }; static void fuse_wait_dax_page(struct inode *inode) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index ea4a15a7635a..1c346802f877 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -890,8 +890,10 @@ static int fuse_iomap_begin(struct inode *inode, loff_t offset, loff_t length, return 0; } +static DEFINE_IOMAP_ITER_NEXT(fuse_iomap_next, fuse_iomap_begin); + static const struct iomap_ops fuse_iomap_ops = { - .iomap_begin = fuse_iomap_begin, + .iomap_next = fuse_iomap_next, }; struct fuse_fill_read_data { diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c index df25d4faca41..f15e516ebcb5 100644 --- a/fs/fuse/virtio_fs.c +++ b/fs/fuse/virtio_fs.c @@ -1024,8 +1024,7 @@ static void virtio_fs_cleanup_vqs(struct virtio_device *vdev) } /* Map a window offset to a page frame number. The window offset will have - * been produced by .iomap_begin(), which maps a file offset to a window - * offset. + * been produced by .iomap_next(), which maps a file offset to a window offset. */ static long virtio_fs_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages, enum dax_access_mode mode, diff --git a/fs/gfs2/bmap.c b/fs/gfs2/bmap.c index 51ac1fd44f78..73c626971163 100644 --- a/fs/gfs2/bmap.c +++ b/fs/gfs2/bmap.c @@ -1200,9 +1200,11 @@ static int gfs2_iomap_end(struct inode *inode, loff_t pos, loff_t length, return 0; } +static DEFINE_IOMAP_ITER_NEXT_END(gfs2_iomap_next, gfs2_iomap_begin, + gfs2_iomap_end); + const struct iomap_ops gfs2_iomap_ops = { - .iomap_begin = gfs2_iomap_begin, - .iomap_end = gfs2_iomap_end, + .iomap_next = gfs2_iomap_next, }; /** diff --git a/fs/hpfs/file.c b/fs/hpfs/file.c index 29e876705369..6a629ab956fc 100644 --- a/fs/hpfs/file.c +++ b/fs/hpfs/file.c @@ -156,8 +156,10 @@ static int hpfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, return 0; } +static DEFINE_IOMAP_ITER_NEXT(hpfs_iomap_next, hpfs_iomap_begin); + static const struct iomap_ops hpfs_iomap_ops = { - .iomap_begin = hpfs_iomap_begin, + .iomap_next = hpfs_iomap_next, }; static int hpfs_read_folio(struct file *file, struct folio *folio) diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index ca790239e5eb..5c28124b9f02 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -894,6 +894,21 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, } EXPORT_SYMBOL_GPL(__iomap_dio_rw); +ssize_t +iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, + const struct iomap_ops *ops, const struct iomap_dio_ops *dops, + unsigned int dio_flags, void *private, size_t done_before) +{ + struct iomap_dio *dio; + + dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags, private, + done_before); + if (IS_ERR_OR_NULL(dio)) + return PTR_ERR_OR_ZERO(dio); + return iomap_dio_complete(dio); +} +EXPORT_SYMBOL_GPL(iomap_dio_rw); + struct iomap_dio_simple { struct kiocb *iocb; size_t size; @@ -968,211 +983,93 @@ static void iomap_dio_simple_end_io(struct bio *bio) iocb->ki_complete(iocb, iomap_dio_simple_complete(sr)); } -static inline bool -iomap_dio_simple_supported(struct kiocb *iocb, struct iov_iter *iter, - const struct iomap_dio_ops *dops, - unsigned int dio_flags, size_t done_before) +ssize_t __iomap_dio_read_simple(struct kiocb *iocb, struct iov_iter *iter, + struct iomap_iter *iomi) { - struct inode *inode = file_inode(iocb->ki_filp); - size_t count = iov_iter_count(iter); - - if (dops || done_before) - return false; - if (iov_iter_rw(iter) != READ) - return false; - if (!count) - return false; - /* - * Simple dio is an optimization for small IO. Filter out large IO - * early as it's the most common case to fail for typical direct IO - * workloads. - */ - if (count > inode->i_sb->s_blocksize) - return false; - if (dio_flags & (IOMAP_DIO_FORCE_WAIT | IOMAP_DIO_PARTIAL | - IOMAP_DIO_BOUNCE)) - return false; - if (iocb->ki_pos + count > i_size_read(inode)) - return false; - if (IS_ENCRYPTED(inode)) - return false; - - return true; -} - -/* - * Fast path for small, block-aligned direct I/Os that map to a single - * contiguous on-disk extent. - * - * iomap_dio_simple_supported() enforces the cheap up-front constraints before - * entering this path. - * - * @dops must be NULL: a non-NULL @dops means the caller wants its - * ->end_io / ->submit_io hooks invoked, and in particular wants its bios to be - * allocated from the filesystem-private @dops->bio_set (whose front_pad sizes a - * filesystem-private wrapper around the bio). The fast path instead allocates - * from the shared iomap_dio_simple_pool, whose front_pad matches struct - * iomap_dio_simple; the two wrappers are not interchangeable, so we must fall - * back to __iomap_dio_rw() in that case. - * - * @done_before must be zero: a non-zero caller-accumulated residual cannot be - * carried through a single-bio inline completion. - * - * @iter must describe a non-empty READ no larger than the inode block size: - * writes, zero-length I/O, and larger requests need the generic iomap direct - * I/O path. - * - * @dio_flags must not request IOMAP_DIO_FORCE_WAIT, IOMAP_DIO_PARTIAL, or - * IOMAP_DIO_BOUNCE: this path does not support forced waiting, partial direct - * I/O, or bouncing. The range must also stay within i_size and encrypted - * inodes must use the generic iomap direct I/O path. - * - * -ENOTBLK is the private sentinel returned by iomap_dio_simple() when it - * decides the request does not fit the fast path. In that case we proceed to - * the generic __iomap_dio_rw() slow path. Any other errno is a real result and - * is propagated as-is, in particular -EAGAIN for IOCB_NOWAIT must reach the - * caller. - */ -static ssize_t -iomap_dio_simple(struct kiocb *iocb, struct iov_iter *iter, - const struct iomap_ops *ops, void *private, - unsigned int dio_flags) -{ - struct inode *inode = file_inode(iocb->ki_filp); - size_t count = iov_iter_count(iter); - bool wait_for_completion = is_sync_kiocb(iocb); - struct iomap_iter iomi = { - .inode = inode, - .pos = iocb->ki_pos, - .len = count, - .flags = IOMAP_DIRECT, - .private = private, - }; + gfp_t gfp = (iomi->flags & IOMAP_NOWAIT) ? GFP_NOWAIT : GFP_KERNEL; struct iomap_dio_simple *sr; unsigned int alignment; struct bio *bio; ssize_t ret; - if (iocb->ki_flags & IOCB_NOWAIT) - iomi.flags |= IOMAP_NOWAIT; - - ret = kiocb_write_and_wait(iocb, count); - if (ret) - return ret; - - inode_dio_begin(inode); - - ret = ops->iomap_begin(inode, iomi.pos, count, iomi.flags, - &iomi.iomap, &iomi.srcmap); - if (ret) { - inode_dio_end(inode); - return ret; - } - - if (iomi.iomap.type != IOMAP_MAPPED || - iomi.iomap.offset + iomi.iomap.length < iomi.pos + count || - (iomi.iomap.flags & IOMAP_F_INTEGRITY)) { + if (iomi->iomap.type != IOMAP_MAPPED || + iomi->iomap.offset + iomi->iomap.length < iomi->pos + iomi->len || + (iomi->iomap.flags & IOMAP_F_INTEGRITY)) { ret = -ENOTBLK; - goto out_iomap_end; + goto out_dio_end; } - alignment = iomap_dio_alignment(inode, iomi.iomap.bdev, dio_flags); - if ((iomi.pos | count) & (alignment - 1)) { + alignment = iomap_dio_alignment(iomi->inode, iomi->iomap.bdev, 0); + if ((iomi->pos | iomi->len) & (alignment - 1)) { ret = -EINVAL; - goto out_iomap_end; + goto out_dio_end; } - if (!wait_for_completion && unlikely(!inode->i_sb->s_dio_done_wq)) { - ret = sb_init_dio_done_wq(inode->i_sb); + if (unlikely(!iomi->inode->i_sb->s_dio_done_wq && + !is_sync_kiocb(iocb))) { + ret = sb_init_dio_done_wq(iomi->inode->i_sb); if (ret < 0) - goto out_iomap_end; + goto out_dio_end; } - trace_iomap_dio_rw_begin(iocb, iter, dio_flags, 0); + trace_iomap_dio_rw_begin(iocb, iter, 0, 0); - if (user_backed_iter(iter)) - dio_flags |= IOMAP_DIO_USER_BACKED; - - bio = bio_alloc_bioset(iomi.iomap.bdev, + bio = bio_alloc_bioset(iomi->iomap.bdev, bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS), - REQ_OP_READ, GFP_KERNEL, &iomap_dio_simple_pool); + REQ_OP_READ, gfp, &iomap_dio_simple_pool); + if (!bio) { + ret = -EAGAIN; + goto out_dio_end; + } sr = container_of(bio, struct iomap_dio_simple, bio); sr->iocb = iocb; - sr->dio_flags = dio_flags; + sr->dio_flags = 0; - bio->bi_iter.bi_sector = iomap_sector(&iomi.iomap, iomi.pos); + bio->bi_iter.bi_sector = iomap_sector(&iomi->iomap, iomi->pos); bio->bi_ioprio = iocb->ki_ioprio; ret = bio_iov_iter_get_pages(bio, iter, alignment - 1); if (unlikely(ret)) goto out_bio_put; - if (bio->bi_iter.bi_size != count) { + if (bio->bi_iter.bi_size != iomi->len) { iov_iter_revert(iter, bio->bi_iter.bi_size); ret = -ENOTBLK; goto out_bio_release_pages; } sr->size = bio->bi_iter.bi_size; - - if (dio_flags & IOMAP_DIO_USER_BACKED) + if (user_backed_iter(iter)) { bio_set_pages_dirty(bio); + sr->dio_flags |= IOMAP_DIO_USER_BACKED; + } if (iocb->ki_flags & IOCB_NOWAIT) bio->bi_opf |= REQ_NOWAIT; - if ((iocb->ki_flags & IOCB_HIPRI) && !wait_for_completion) { + + if (is_sync_kiocb(iocb)) { + submit_bio_wait(bio); + return iomap_dio_simple_complete(sr); + } + + if ((iocb->ki_flags & IOCB_HIPRI)) { bio->bi_opf |= REQ_POLLED; WRITE_ONCE(iocb->private, bio); } - - if (ops->iomap_end) - ops->iomap_end(inode, iomi.pos, count, count, iomi.flags, - &iomi.iomap); - - if (!wait_for_completion) { - bio->bi_end_io = iomap_dio_simple_end_io; - submit_bio(bio); - trace_iomap_dio_rw_queued(inode, iomi.pos, count); - return -EIOCBQUEUED; - } - - submit_bio_wait(bio); - return iomap_dio_simple_complete(sr); + bio->bi_end_io = iomap_dio_simple_end_io; + submit_bio(bio); + trace_iomap_dio_rw_queued(iomi->inode, iocb->ki_pos, iomi->len); + return -EIOCBQUEUED; out_bio_release_pages: bio_release_pages(bio, false); out_bio_put: bio_put(bio); -out_iomap_end: - if (ops->iomap_end) - ops->iomap_end(inode, iomi.pos, count, 0, iomi.flags, - &iomi.iomap); - inode_dio_end(inode); +out_dio_end: + inode_dio_end(iomi->inode); return ret; } - -ssize_t -iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, - const struct iomap_ops *ops, const struct iomap_dio_ops *dops, - unsigned int dio_flags, void *private, size_t done_before) -{ - struct iomap_dio *dio; - ssize_t ret; - - if (iomap_dio_simple_supported(iocb, iter, dops, dio_flags, - done_before)) { - ret = iomap_dio_simple(iocb, iter, ops, private, dio_flags); - if (ret != -ENOTBLK) - return ret; - } - - dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags, private, - done_before); - if (IS_ERR_OR_NULL(dio)) - return PTR_ERR_OR_ZERO(dio); - return iomap_dio_complete(dio); -} -EXPORT_SYMBOL_GPL(iomap_dio_rw); +EXPORT_SYMBOL_GPL(__iomap_dio_read_simple); static int __init iomap_dio_init(void) { diff --git a/fs/iomap/iter.c b/fs/iomap/iter.c index e4a29829591a..c445a38b6285 100644 --- a/fs/iomap/iter.c +++ b/fs/iomap/iter.c @@ -6,12 +6,19 @@ #include #include "trace.h" -static inline void iomap_iter_clean_fbatch(struct iomap_iter *iter) +/* + * Release the iter folio batch. Note that the iomap flag is meant to control + * the I/O path for the mapping and may not be set in error situations. + */ +static inline void iomap_iter_clean_fbatch(const struct iomap_iter *iter, + struct iomap *iomap) { - if (iter->iomap.flags & IOMAP_F_FOLIO_BATCH) { + if (!iter->fbatch) + return; + iomap->flags &= ~IOMAP_F_FOLIO_BATCH; + if (folio_batch_count(iter->fbatch)) { folio_batch_release(iter->fbatch); folio_batch_reinit(iter->fbatch); - iter->iomap.flags &= ~IOMAP_F_FOLIO_BATCH; } } @@ -40,9 +47,60 @@ static inline void iomap_iter_done(struct iomap_iter *iter) } /** - * iomap_iter - iterate over a ranges in a file - * @iter: iteration structue - * @ops: iomap ops provided by the file system + * iomap_iter_continue - decide whether iteration should continue + * @iter: iteration structure + * @iomap: the mapping that was just processed + * @srcmap: the source mapping that was just processed + * + * Helper normally called via iomap_iter_next(). Called after the previous + * mapping has been finished to determine whether there is more of the file + * range left to process. + * + * Returns 1 if there is more work to do, in which case @iomap and @srcmap are + * cleared so the caller can produce the next mapping; zero if the range is + * fully consumed; or a negative errno on error. + */ +int iomap_iter_continue(const struct iomap_iter *iter, struct iomap *iomap, + struct iomap *srcmap, int ret) +{ + const bool stale = iomap->flags & IOMAP_F_STALE; + const ssize_t advanced = iter->pos - iter->iter_start_pos; + + if (ret < 0 && !advanced) + return ret; + + /* + * Use iter->len to determine whether to continue onto the next mapping. + * Explicitly terminate on error status or if the current iter has not + * advanced at all (i.e. no work was done for some reason) unless the + * mapping has been marked stale and needs to be reprocessed. + */ + if (WARN_ON_ONCE(iter->status > 0)) + /* detect old return semantics where this would advance */ + ret = -EIO; + else if (iter->status < 0) + ret = iter->status; + else if (iter->len == 0 || (!advanced && !stale)) + ret = 0; + else + ret = 1; + + iomap_iter_clean_fbatch(iter, iomap); + + if (ret <= 0) + return ret; + + memset(iomap, 0, sizeof(*iomap)); + memset(srcmap, 0, sizeof(*srcmap)); + + return ret; +} +EXPORT_SYMBOL_GPL(iomap_iter_continue); + +/** + * iomap_iter - iterate over ranges in a file + * @iter: iteration structure + * @ops: iomap ops provided by the filesystem * * Iterate over filesystem-provided space mappings for the provided file range. * @@ -56,61 +114,21 @@ static inline void iomap_iter_done(struct iomap_iter *iter) */ int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops) { - bool stale = iter->iomap.flags & IOMAP_F_STALE; - ssize_t advanced; - u64 olen; int ret; trace_iomap_iter(iter, ops, _RET_IP_); - if (!iter->iomap.length) - goto begin; - - /* - * Calculate how far the iter was advanced and the original length bytes - * for ->iomap_end(). - */ - advanced = iter->pos - iter->iter_start_pos; - olen = iter->len + advanced; - - if (ops->iomap_end) { - ret = ops->iomap_end(iter->inode, iter->iter_start_pos, - iomap_length_trim(iter, iter->iter_start_pos, - olen), - advanced, iter->flags, &iter->iomap); - if (ret < 0 && !advanced) - return ret; - } - - /* detect old return semantics where this would advance */ - if (WARN_ON_ONCE(iter->status > 0)) - iter->status = -EIO; - - /* - * Use iter->len to determine whether to continue onto the next mapping. - * Explicitly terminate on error status or if the current iter has not - * advanced at all (i.e. no work was done for some reason) unless the - * mapping has been marked stale and needs to be reprocessed. - */ - if (iter->status < 0) - ret = iter->status; - else if (iter->len == 0 || (!advanced && !stale)) - ret = 0; + if (ops->iomap_next) + ret = ops->iomap_next(iter, &iter->iomap, &iter->srcmap); else - ret = 1; - iomap_iter_clean_fbatch(iter); + ret = iomap_iter_next(iter, &iter->iomap, &iter->srcmap, + ops->iomap_begin, ops->iomap_end); + iter->status = 0; - if (ret <= 0) - return ret; + if (ret > 0) + iomap_iter_done(iter); + else if (ret < 0) + iomap_iter_clean_fbatch(iter, &iter->iomap); - memset(&iter->iomap, 0, sizeof(iter->iomap)); - memset(&iter->srcmap, 0, sizeof(iter->srcmap)); - -begin: - ret = ops->iomap_begin(iter->inode, iter->pos, iter->len, iter->flags, - &iter->iomap, &iter->srcmap); - if (ret < 0) - return ret; - iomap_iter_done(iter); - return 1; + return ret; } diff --git a/fs/ntfs/iomap.c b/fs/ntfs/iomap.c index 52eecf5cb256..d0964ac840d9 100644 --- a/fs/ntfs/iomap.c +++ b/fs/ntfs/iomap.c @@ -277,8 +277,10 @@ static int ntfs_read_iomap_begin(struct inode *inode, loff_t offset, loff_t leng srcmap, true); } +static DEFINE_IOMAP_ITER_NEXT(ntfs_read_iomap_next, ntfs_read_iomap_begin); + const struct iomap_ops ntfs_read_iomap_ops = { - .iomap_begin = ntfs_read_iomap_begin, + .iomap_next = ntfs_read_iomap_next, }; /* @@ -329,13 +331,17 @@ static int ntfs_zero_read_iomap_end(struct inode *inode, loff_t pos, loff_t leng return written; } +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_zero_read_iomap_next, + ntfs_seek_iomap_begin, ntfs_zero_read_iomap_end); + static const struct iomap_ops ntfs_zero_read_iomap_ops = { - .iomap_begin = ntfs_seek_iomap_begin, - .iomap_end = ntfs_zero_read_iomap_end, + .iomap_next = ntfs_zero_read_iomap_next, }; +static DEFINE_IOMAP_ITER_NEXT(ntfs_seek_iomap_next, ntfs_seek_iomap_begin); + const struct iomap_ops ntfs_seek_iomap_ops = { - .iomap_begin = ntfs_seek_iomap_begin, + .iomap_next = ntfs_seek_iomap_next, }; int ntfs_dio_zero_range(struct inode *inode, loff_t offset, loff_t length) @@ -764,9 +770,11 @@ static int ntfs_write_iomap_end(struct inode *inode, loff_t pos, loff_t length, return written; } +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_write_iomap_next, + ntfs_write_iomap_begin, ntfs_write_iomap_end); + const struct iomap_ops ntfs_write_iomap_ops = { - .iomap_begin = ntfs_write_iomap_begin, - .iomap_end = ntfs_write_iomap_end, + .iomap_next = ntfs_write_iomap_next, }; static int ntfs_page_mkwrite_iomap_begin(struct inode *inode, loff_t offset, @@ -777,9 +785,11 @@ static int ntfs_page_mkwrite_iomap_begin(struct inode *inode, loff_t offset, NTFS_IOMAP_FLAGS_MKWRITE); } +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_page_mkwrite_iomap_next, + ntfs_page_mkwrite_iomap_begin, ntfs_write_iomap_end); + const struct iomap_ops ntfs_page_mkwrite_iomap_ops = { - .iomap_begin = ntfs_page_mkwrite_iomap_begin, - .iomap_end = ntfs_write_iomap_end, + .iomap_next = ntfs_page_mkwrite_iomap_next, }; static int ntfs_dio_iomap_begin(struct inode *inode, loff_t offset, @@ -790,9 +800,11 @@ static int ntfs_dio_iomap_begin(struct inode *inode, loff_t offset, NTFS_IOMAP_FLAGS_DIO); } +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_dio_iomap_next, + ntfs_dio_iomap_begin, ntfs_write_iomap_end); + const struct iomap_ops ntfs_dio_iomap_ops = { - .iomap_begin = ntfs_dio_iomap_begin, - .iomap_end = ntfs_write_iomap_end, + .iomap_next = ntfs_dio_iomap_next, }; static ssize_t ntfs_writeback_range(struct iomap_writepage_ctx *wpc, diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c index c43101cc064d..53031e71c8fc 100644 --- a/fs/ntfs3/inode.c +++ b/fs/ntfs3/inode.c @@ -2101,9 +2101,11 @@ const struct address_space_operations ntfs_aops_cmpr = { .invalidate_folio = iomap_invalidate_folio, }; +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_iomap_next, ntfs_iomap_begin, + ntfs_iomap_end); + const struct iomap_ops ntfs_iomap_ops = { - .iomap_begin = ntfs_iomap_begin, - .iomap_end = ntfs_iomap_end, + .iomap_next = ntfs_iomap_next, }; const struct iomap_write_ops ntfs_iomap_folio_ops = { diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 845a97c9b063..768cabf6250b 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -251,8 +251,6 @@ xfs_file_dio_read( struct iov_iter *to) { struct xfs_inode *ip = XFS_I(file_inode(iocb->ki_filp)); - unsigned int dio_flags = 0; - const struct iomap_dio_ops *dio_ops = NULL; ssize_t ret; trace_xfs_file_direct_read(iocb, to); @@ -266,11 +264,15 @@ xfs_file_dio_read( if (ret) return ret; if (mapping_stable_writes(iocb->ki_filp->f_mapping)) { - dio_ops = &xfs_dio_read_bounce_ops; - dio_flags |= IOMAP_DIO_BOUNCE; + ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, + &xfs_dio_read_bounce_ops, IOMAP_DIO_BOUNCE, + NULL, 0); + } else { + ret = iomap_dio_read_simple(iocb, to, xfs_read_iomap_begin); + if (ret == -ENOTBLK) + ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, NULL, + 0, NULL, 0); } - ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, dio_ops, dio_flags, - NULL, 0); xfs_iunlock(ip, XFS_IOLOCK_SHARED); return ret; @@ -857,9 +859,9 @@ xfs_file_dio_write_atomic( NULL, 0); /* - * The retry mechanism is based on the ->iomap_begin method returning + * The retry mechanism is based on the ->iomap_next method returning * -ENOPROTOOPT, which would be when the REQ_ATOMIC-based write is not - * possible. The REQ_ATOMIC-based method typically not be possible if + * possible. The REQ_ATOMIC-based method is typically not possible if * the write spans multiple extents or the disk blocks are misaligned. */ if (ret == -ENOPROTOOPT && dops == &xfs_direct_write_iomap_ops) { diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index 225c3de88d03..71c45be8c652 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -1037,8 +1037,11 @@ xfs_direct_write_iomap_begin( return error; } +static DEFINE_IOMAP_ITER_NEXT(xfs_direct_write_iomap_next, + xfs_direct_write_iomap_begin); + const struct iomap_ops xfs_direct_write_iomap_ops = { - .iomap_begin = xfs_direct_write_iomap_begin, + .iomap_next = xfs_direct_write_iomap_next, }; #ifdef CONFIG_XFS_RT @@ -1089,8 +1092,11 @@ xfs_zoned_direct_write_iomap_begin( return 0; } +static DEFINE_IOMAP_ITER_NEXT(xfs_zoned_direct_write_iomap_next, + xfs_zoned_direct_write_iomap_begin); + const struct iomap_ops xfs_zoned_direct_write_iomap_ops = { - .iomap_begin = xfs_zoned_direct_write_iomap_begin, + .iomap_next = xfs_zoned_direct_write_iomap_next, }; #endif /* CONFIG_XFS_RT */ @@ -1274,8 +1280,11 @@ xfs_atomic_write_cow_iomap_begin( return error; } +static DEFINE_IOMAP_ITER_NEXT(xfs_atomic_write_cow_iomap_next, + xfs_atomic_write_cow_iomap_begin); + const struct iomap_ops xfs_atomic_write_cow_iomap_ops = { - .iomap_begin = xfs_atomic_write_cow_iomap_begin, + .iomap_next = xfs_atomic_write_cow_iomap_next, }; static int @@ -1298,9 +1307,11 @@ xfs_dax_write_iomap_end( return xfs_reflink_end_cow(ip, pos, written); } +static DEFINE_IOMAP_ITER_NEXT_END(xfs_dax_write_iomap_next, + xfs_direct_write_iomap_begin, xfs_dax_write_iomap_end); + const struct iomap_ops xfs_dax_write_iomap_ops = { - .iomap_begin = xfs_direct_write_iomap_begin, - .iomap_end = xfs_dax_write_iomap_end, + .iomap_next = xfs_dax_write_iomap_next, }; /* @@ -2168,12 +2179,14 @@ xfs_buffered_write_iomap_end( return 0; } +static DEFINE_IOMAP_ITER_NEXT_END(xfs_buffered_write_iomap_next, + xfs_buffered_write_iomap_begin, xfs_buffered_write_iomap_end); + const struct iomap_ops xfs_buffered_write_iomap_ops = { - .iomap_begin = xfs_buffered_write_iomap_begin, - .iomap_end = xfs_buffered_write_iomap_end, + .iomap_next = xfs_buffered_write_iomap_next, }; -static int +int xfs_read_iomap_begin( struct inode *inode, loff_t offset, @@ -2214,8 +2227,10 @@ xfs_read_iomap_begin( shared ? IOMAP_F_SHARED : 0, seq); } +static DEFINE_IOMAP_ITER_NEXT(xfs_read_iomap_next, xfs_read_iomap_begin); + const struct iomap_ops xfs_read_iomap_ops = { - .iomap_begin = xfs_read_iomap_begin, + .iomap_next = xfs_read_iomap_next, }; static int @@ -2302,8 +2317,10 @@ xfs_seek_iomap_begin( return error; } +static DEFINE_IOMAP_ITER_NEXT(xfs_seek_iomap_next, xfs_seek_iomap_begin); + const struct iomap_ops xfs_seek_iomap_ops = { - .iomap_begin = xfs_seek_iomap_begin, + .iomap_next = xfs_seek_iomap_next, }; static int @@ -2349,8 +2366,10 @@ xfs_xattr_iomap_begin( return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, IOMAP_F_XATTR, seq); } +static DEFINE_IOMAP_ITER_NEXT(xfs_xattr_iomap_next, xfs_xattr_iomap_begin); + const struct iomap_ops xfs_xattr_iomap_ops = { - .iomap_begin = xfs_xattr_iomap_begin, + .iomap_next = xfs_xattr_iomap_next, }; int diff --git a/fs/xfs/xfs_iomap.h b/fs/xfs/xfs_iomap.h index ebcce7d49446..cffcec532ea6 100644 --- a/fs/xfs/xfs_iomap.h +++ b/fs/xfs/xfs_iomap.h @@ -49,6 +49,10 @@ xfs_aligned_fsb_count( return count_fsb; } +int xfs_read_iomap_begin(struct inode *inode, loff_t offset, + loff_t length, unsigned flags, struct iomap *iomap, + struct iomap *srcmap); + extern const struct iomap_ops xfs_buffered_write_iomap_ops; extern const struct iomap_ops xfs_direct_write_iomap_ops; extern const struct iomap_ops xfs_zoned_direct_write_iomap_ops; diff --git a/fs/zonefs/file.c b/fs/zonefs/file.c index 5ada33f70bb4..5b34849be7a2 100644 --- a/fs/zonefs/file.c +++ b/fs/zonefs/file.c @@ -57,8 +57,10 @@ static int zonefs_read_iomap_begin(struct inode *inode, loff_t offset, return 0; } +static DEFINE_IOMAP_ITER_NEXT(zonefs_read_iomap_next, zonefs_read_iomap_begin); + static const struct iomap_ops zonefs_read_iomap_ops = { - .iomap_begin = zonefs_read_iomap_begin, + .iomap_next = zonefs_read_iomap_next, }; static int zonefs_write_iomap_begin(struct inode *inode, loff_t offset, @@ -106,8 +108,11 @@ static int zonefs_write_iomap_begin(struct inode *inode, loff_t offset, return 0; } +static DEFINE_IOMAP_ITER_NEXT(zonefs_write_iomap_next, + zonefs_write_iomap_begin); + static const struct iomap_ops zonefs_write_iomap_ops = { - .iomap_begin = zonefs_write_iomap_begin, + .iomap_next = zonefs_write_iomap_next, }; static int zonefs_read_folio(struct file *unused, struct folio *folio) diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 21e73cb9c51e..d203d9fe0f89 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -10,6 +10,7 @@ #include #include #include +#include struct address_space; struct fiemap_extent_info; @@ -212,24 +213,36 @@ struct iomap_write_ops { #define IOMAP_ATOMIC (1 << 9) /* torn-write protection */ #define IOMAP_DONTCACHE (1 << 10) -struct iomap_ops { - /* - * Return the existing mapping at pos, or reserve space starting at - * pos for up to length, as long as we can do it as a single mapping. - * The actual length is returned in iomap->length. - */ - int (*iomap_begin)(struct inode *inode, loff_t pos, loff_t length, - unsigned flags, struct iomap *iomap, - struct iomap *srcmap); +/* + * Return the existing mapping at pos, or reserve space starting at pos for up + * to length, as long as we can do it as a single mapping. + * The actual length is returned in iomap->length. + */ +typedef int (*iomap_iter_begin_fn)(struct inode *inode, loff_t pos, + loff_t length, unsigned flags, struct iomap *iomap, + struct iomap *srcmap); - /* - * Commit and/or unreserve space previous allocated using iomap_begin. - * Written indicates the length of the successful write operation which - * needs to be commited, while the rest needs to be unreserved. - * Written might be zero if no data was written. - */ - int (*iomap_end)(struct inode *inode, loff_t pos, loff_t length, - ssize_t written, unsigned flags, struct iomap *iomap); +/* + * Commit and/or unreserve space previously allocated by iomap_iter_begin_fn. + * Written indicates the length of the successful write operation which needs + * to be committed, while the rest needs to be unreserved. + * Written might be zero if no data was written. + */ +typedef int (*iomap_iter_end_fn)(struct inode *inode, loff_t pos, loff_t length, + ssize_t written, unsigned flags, struct iomap *iomap); + +/* + * Produce the next mapping (finishing the previous one if needed). + * Return 1 to continue iterating, 0 if the range is fully consumed, or a + * negative error on failure. + */ +typedef int (*iomap_iter_next_fn)(const struct iomap_iter *iter, + struct iomap *iomap, struct iomap *srcmap); + +struct iomap_ops { + iomap_iter_begin_fn iomap_begin; + iomap_iter_end_fn iomap_end; + iomap_iter_next_fn iomap_next; }; /** @@ -317,6 +330,71 @@ static inline const struct iomap *iomap_iter_srcmap(const struct iomap_iter *i) return &i->iomap; } +int iomap_iter_continue(const struct iomap_iter *iter, struct iomap *iomap, + struct iomap *srcmap, int ret); + +/** + * iomap_iter_next - finish the previous mapping and produce the next one + * @iter: iteration structure + * @iomap: mapping to finish and then repopulate + * @srcmap: source mapping to finish and then repopulate + * @begin: callback that produces a mapping for the current position + * @end: optional callback that finishes the previous mapping, or NULL + * + * Inline helper that implements the common body of an ->iomap_next() + * callback: it finishes the previous mapping via @end (if present), decides + * via iomap_iter_continue() whether to keep going, and obtains the next + * mapping via @begin. + * + * This helper is marked __always_inline so that when a caller passes + * compile-time-constant @begin and @end callbacks, the compiler can call them + * directly, avoiding the indirect-call overhead. + * + * Returns 1 to continue iterating, 0 once the range is fully consumed, or a + * negative errno on error. + */ +static __always_inline int iomap_iter_next(const struct iomap_iter *iter, + struct iomap *iomap, struct iomap *srcmap, + iomap_iter_begin_fn begin, iomap_iter_end_fn end) +{ + int ret = 0; + + if (iomap->length) { + if (end) { + /* + * Calculate how far the iter was advanced and the + * original length bytes for end(). + */ + ssize_t advanced = iter->pos - iter->iter_start_pos; + loff_t len; + + len = iomap_length_trim(iter, iter->iter_start_pos, + iter->len + advanced); + + ret = end(iter->inode, iter->iter_start_pos, len, + advanced, iter->flags, iomap); + } + ret = iomap_iter_continue(iter, iomap, srcmap, ret); + if (ret <= 0) + return ret; + } + + ret = begin(iter->inode, iter->pos, iter->len, iter->flags, iomap, + srcmap); + + return ret < 0 ? ret : 1; +} + +#define DEFINE_IOMAP_ITER_NEXT_END(name, begin_fn, end_fn) \ +int name(const struct iomap_iter *iter, struct iomap *iomap, \ + struct iomap *srcmap) \ +{ \ + return iomap_iter_next(iter, iomap, srcmap, begin_fn, end_fn); \ +} + +#define DEFINE_IOMAP_ITER_NEXT(name, begin_fn) \ + DEFINE_IOMAP_ITER_NEXT_END(name, begin_fn, NULL) + /* * Return the file offset for the first unchanged block after a short write. * @@ -607,6 +685,71 @@ struct iomap_dio *__iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, ssize_t iomap_dio_complete(struct iomap_dio *dio); void iomap_dio_bio_end_io(struct bio *bio); +/* + * Fast path for small, block-aligned direct I/Os that map to a single + * contiguous on-disk extent. + * + * @iter must describe a non-empty READ no larger than the inode block size: + * writes, zero-length I/O, and larger requests need the generic iomap direct + * I/O path. + * + * Does not support iomap_dio_ops, dio_flags, done_before or private data. + * The range must also stay within i_size and encrypted inodes must use the + * generic iomap direct I/O path. + * + * -ENOTBLK indicates the generic path must be used by the caller instead. + * Any other errno is a real result and is propagated as-is, in particular + * -EAGAIN for IOCB_NOWAIT must reach the caller. + * + * The caller can only provide an iomap begin handler, and the iterator + * is never advanced. + */ +ssize_t __iomap_dio_read_simple(struct kiocb *iocb, struct iov_iter *iter, + struct iomap_iter *iomi); +static __always_inline ssize_t iomap_dio_read_simple(struct kiocb *iocb, + struct iov_iter *iter, iomap_iter_begin_fn begin) +{ + struct iomap_iter iomi = { + .inode = file_inode(iocb->ki_filp), + .pos = iocb->ki_pos, + .len = iov_iter_count(iter), + .flags = IOMAP_DIRECT, + }; + ssize_t ret; + + if (!iomi.len) + return 0; + + /* + * Simple dio is an optimization for small IO. Filter out large IO + * early as it's the most common case to fail for typical direct IO + * workloads. + */ + if (iomi.len > iomi.inode->i_sb->s_blocksize) + return -ENOTBLK; + if (iocb->ki_pos + iomi.len > i_size_read(iomi.inode)) + return -ENOTBLK; + if (IS_ENCRYPTED(iomi.inode)) + return -ENOTBLK; + + ret = kiocb_write_and_wait(iocb, iomi.len); + if (ret) + return ret; + + if (iocb->ki_flags & IOCB_NOWAIT) + iomi.flags |= IOMAP_NOWAIT; + + inode_dio_begin(iomi.inode); + ret = begin(iomi.inode, iomi.pos, iomi.len, iomi.flags, &iomi.iomap, + &iomi.srcmap); + if (ret) { + inode_dio_end(iomi.inode); + return ret; + } + + return __iomap_dio_read_simple(iocb, iter, &iomi); +} + #ifdef CONFIG_SWAP struct file; struct swap_info_struct;