From 27b7efca5bb4e0e9ed5c6434e03828234c08f72c Mon Sep 17 00:00:00 2001 From: Fengnan Chang Date: Wed, 1 Jul 2026 11:32:51 +0800 Subject: [PATCH 01/29] iomap: factor out iomap_dio_alignment helper Extract the alignment computation from iomap_dio_bio_iter() into a standalone helper so the upcoming simple direct I/O path can reuse it without requiring a struct iomap_dio. No functional change. Signed-off-by: Fengnan Chang Link: https://patch.msgid.link/20260701033253.46420-2-changfengnan@bytedance.com Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner (Amutable) --- fs/iomap/direct-io.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index b485e3b191da..487c4763f3fd 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -398,6 +398,14 @@ static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter, return ret; } +static inline unsigned int iomap_dio_alignment(struct inode *inode, + struct block_device *bdev, unsigned int dio_flags) +{ + if (dio_flags & IOMAP_DIO_FSBLOCK_ALIGNED) + return i_blocksize(inode); + return bdev_logical_block_size(bdev); +} + static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio) { const struct iomap *iomap = &iter->iomap; @@ -416,10 +424,7 @@ static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio) * File systems that write out of place and always allocate new blocks * need each bio to be block aligned as that's the unit of allocation. */ - if (dio->flags & IOMAP_DIO_FSBLOCK_ALIGNED) - alignment = fs_block_size; - else - alignment = bdev_logical_block_size(iomap->bdev); + alignment = iomap_dio_alignment(inode, iomap->bdev, dio->flags); if ((pos | length) & (alignment - 1)) return -EINVAL; From e08fd6119126689a50003b3a07ed139461ccdfb5 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Jun 2026 18:42:26 +0100 Subject: [PATCH 02/29] iomap: Remove FGP_NOFS from iomap_get_folio() FGP_NOFS is legacy; filesystems should be using memalloc_nofs_save/restore instead. We have it here in iomap because it was buried in grab_cache_page_write_begin() and we didn't want to change this behaviour as part of the folio transition. I have tested this with XFS and see no issues. Other filesystems (cc'd) may need to make adjustments. Please test with lockdep enabled. Cc: Darrick J. Wong Cc: Jens Axboe Cc: Namjae Jeon Cc: Sungjong Seo Cc: Yuezhang Mo Cc: Miklos Szeredi Cc: Andreas Gruenbacher Cc: Hyunchul Lee Cc: Konstantin Komarov Cc: Carlos Maiolino Cc: Damien Le Moal Cc: Naohiro Aota Cc: Johannes Thumshirn Cc: linux-xfs@vger.kernel.org Cc: linux-fsdevel@vger.kernel.org Cc: linux-block@vger.kernel.org Cc: fuse-devel@lists.linux.dev Cc: gfs2@lists.linux.dev Cc: ntfs3@lists.linux.dev Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260624174228.2015893-1-willy@infradead.org Signed-off-by: Christian Brauner (Amutable) --- fs/iomap/buffered-io.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index 8d4806dc46d4..27bc2455a98d 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -768,7 +768,7 @@ EXPORT_SYMBOL_GPL(iomap_is_partially_uptodate); */ struct folio *iomap_get_folio(struct iomap_iter *iter, loff_t pos, size_t len) { - fgf_t fgp = FGP_WRITEBEGIN | FGP_NOFS; + fgf_t fgp = FGP_WRITEBEGIN; if (iter->flags & IOMAP_NOWAIT) fgp |= FGP_NOWAIT; From ef793297cd085fac0bca4813a399e0016db94773 Mon Sep 17 00:00:00 2001 From: Fengnan Chang Date: Wed, 1 Jul 2026 11:32:52 +0800 Subject: [PATCH 03/29] iomap: pass error code to should_report_dio_fserror directly Change should_report_dio_fserror() to take an error code instead of the full struct iomap_dio, decoupling it for reuse by the upcoming simple direct I/O path. No functional change. Signed-off-by: Fengnan Chang Link: https://patch.msgid.link/20260701033253.46420-3-changfengnan@bytedance.com Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner (Amutable) --- fs/iomap/direct-io.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 487c4763f3fd..1b9abdd831d0 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -88,9 +88,9 @@ static inline enum fserror_type iomap_dio_err_type(const struct iomap_dio *dio) return FSERR_DIRECTIO_READ; } -static inline bool should_report_dio_fserror(const struct iomap_dio *dio) +static inline bool should_report_dio_fserror(int error) { - switch (dio->error) { + switch (error) { case 0: case -EAGAIN: case -ENOTBLK: @@ -110,7 +110,7 @@ ssize_t iomap_dio_complete(struct iomap_dio *dio) if (dops && dops->end_io) ret = dops->end_io(iocb, dio->size, ret, dio->flags); - if (should_report_dio_fserror(dio)) + if (should_report_dio_fserror(dio->error)) fserror_report_io(file_inode(iocb->ki_filp), iomap_dio_err_type(dio), offset, dio->size, dio->error, GFP_NOFS); From 36f199c8d0ee4b8dab3559c8bc23dd5e7c366972 Mon Sep 17 00:00:00 2001 From: Fengnan Chang Date: Wed, 1 Jul 2026 11:32:53 +0800 Subject: [PATCH 04/29] iomap: add simple dio path for small direct I/O When running 4K random read workloads on high-performance Gen5 NVMe SSDs, the software overhead in the iomap direct I/O path (__iomap_dio_rw) becomes a significant bottleneck. Using io_uring with poll mode for a 4K randread test on a raw block device: taskset -c 30 ./t/io_uring -p1 -d512 -b4096 -s32 -c32 -F1 -B1 -R1 -X1 -n1 -P1 /dev/nvme10n1 Result: ~3.2M IOPS Running the exact same workload on ext4 and XFS: taskset -c 30 ./t/io_uring -p1 -d512 -b4096 -s32 -c32 -F1 -B1 -R1 -X1 -n1 -P1 /mnt/testfile Result: ~1.92M IOPS Profiling the ext4 workload reveals that a significant portion of CPU time is spent on memory allocation and the iomap state machine iteration: 5.33% [kernel] [k] __iomap_dio_rw 3.26% [kernel] [k] iomap_iter 2.37% [kernel] [k] iomap_dio_bio_iter 2.35% [kernel] [k] kfree 1.33% [kernel] [k] iomap_dio_complete Introduce a simple dio path to reduce the overhead of iomap. It is triggered when the request satisfies all of: - a READ request whose I/O size is <= inode blocksize (fits in a single block, no splits); - no custom iomap_dio_ops (dops) registered by the filesystem; - no caller-accumulated residual (done_before == 0); - none of IOMAP_DIO_FORCE_WAIT / IOMAP_DIO_PARTIAL / IOMAP_DIO_BOUNCE set, the range is within i_size, and the inode is not encrypted. The bio is allocated from a dedicated bioset whose front_pad embeds struct iomap_dio_simple, so the whole request lives in a single cacheline-aligned allocation and no separate struct iomap_dio is needed. Completion is handled inline from ->bi_end_io for the common success case, and only punted to the s_dio_done_wq workqueue on error. After this optimization, the heavy generic functions disappear from the profile, replaced by a single streamlined execution path: 4.83% [kernel] [k] iomap_dio_simple With this patch, 4K random read IOPS on ext4 increases from 1.92M to 2.19M in the original single-core io_uring poll-mode workload. Below are the test results using fio: fs workload qd simple=0 simple=1 gain ext4 libaio 1 18,740 18,761 +0.11% ext4 libaio 64 462,850 480,587 +3.83% ext4 libaio 128 459,498 478,824 +4.21% ext4 libaio 256 459,938 480,156 +4.40% ext4 io_uring 1 18,836 18,880 +0.24% ext4 io_uring 64 568,193 600,625 +5.71% ext4 io_uring 128 570,998 602,148 +5.46% ext4 io_uring 256 572,052 602,536 +5.33% ext4 io_uring_poll 1 19,283 19,272 -0.06% ext4 io_uring_poll 64 989,735 1,013,342 +2.39% ext4 io_uring_poll 128 1,467,336 1,538,444 +4.85% ext4 io_uring_poll 256 1,663,498 1,830,842 +10.06% xfs libaio 1 18,764 18,776 +0.06% xfs libaio 64 462,408 480,860 +3.99% xfs libaio 128 461,280 480,819 +4.24% xfs libaio 256 461,626 480,190 +4.02% xfs io_uring 1 18,871 18,903 +0.17% xfs io_uring 64 570,383 597,399 +4.74% xfs io_uring 128 568,290 597,370 +5.12% xfs io_uring 256 570,616 598,775 +4.93% xfs io_uring_poll 1 19,211 19,315 +0.54% xfs io_uring_poll 64 989,726 1,008,455 +1.89% xfs io_uring_poll 128 1,430,426 1,513,064 +5.78% xfs io_uring_poll 256 1,587,339 1,742,220 +9.76% Signed-off-by: Fengnan Chang Link: https://patch.msgid.link/20260701033253.46420-4-changfengnan@bytedance.com Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner (Amutable) --- fs/iomap/direct-io.c | 274 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 274 insertions(+) diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 1b9abdd831d0..ca790239e5eb 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -10,6 +10,7 @@ #include #include #include +#include #include "internal.h" #include "trace.h" @@ -893,12 +894,277 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, } EXPORT_SYMBOL_GPL(__iomap_dio_rw); +struct iomap_dio_simple { + struct kiocb *iocb; + size_t size; + unsigned int dio_flags; + struct work_struct work; + /* + * Align @bio to a cacheline boundary so that, combined with the + * front_pad passed to bioset_init(), the bio sits at the start of + * a cacheline in memory returned by the (HWCACHE-aligned) bio + * slab. This keeps the hot fields block layer touches on submit + * and completion (bi_iter, bi_status, ...) within a single line. + */ + struct bio bio ____cacheline_aligned_in_smp; +}; + +static struct bio_set iomap_dio_simple_pool; + +static ssize_t iomap_dio_simple_complete(struct iomap_dio_simple *sr) +{ + struct bio *bio = &sr->bio; + struct kiocb *iocb = sr->iocb; + struct inode *inode = file_inode(iocb->ki_filp); + ssize_t ret; + + if (unlikely(bio->bi_status)) { + ret = blk_status_to_errno(bio->bi_status); + if (should_report_dio_fserror(ret)) + fserror_report_io(inode, FSERR_DIRECTIO_READ, + iocb->ki_pos, sr->size, ret, + GFP_NOFS); + } else { + ret = sr->size; + iocb->ki_pos += ret; + } + + if (sr->dio_flags & IOMAP_DIO_USER_BACKED) { + bio_check_pages_dirty(bio); + } else { + bio_release_pages(bio, false); + bio_put(bio); + } + inode_dio_end(inode); + trace_iomap_dio_complete(iocb, ret < 0 ? ret : 0, ret); + return ret; +} + +static void iomap_dio_simple_complete_work(struct work_struct *work) +{ + struct iomap_dio_simple *sr = + container_of(work, struct iomap_dio_simple, work); + struct kiocb *iocb = sr->iocb; + + WRITE_ONCE(iocb->private, NULL); + iocb->ki_complete(iocb, iomap_dio_simple_complete(sr)); +} + +static void iomap_dio_simple_end_io(struct bio *bio) +{ + struct iomap_dio_simple *sr = + container_of(bio, struct iomap_dio_simple, bio); + struct kiocb *iocb = sr->iocb; + + if (unlikely(sr->bio.bi_status)) { + struct inode *inode = file_inode(iocb->ki_filp); + + INIT_WORK(&sr->work, iomap_dio_simple_complete_work); + queue_work(inode->i_sb->s_dio_done_wq, &sr->work); + return; + } + + WRITE_ONCE(iocb->private, NULL); + iocb->ki_complete(iocb, iomap_dio_simple_complete(sr)); +} + +static inline bool +iomap_dio_simple_supported(struct kiocb *iocb, struct iov_iter *iter, + const struct iomap_dio_ops *dops, + unsigned int dio_flags, size_t done_before) +{ + struct inode *inode = file_inode(iocb->ki_filp); + size_t count = iov_iter_count(iter); + + if (dops || done_before) + return false; + if (iov_iter_rw(iter) != READ) + return false; + if (!count) + return false; + /* + * Simple dio is an optimization for small IO. Filter out large IO + * early as it's the most common case to fail for typical direct IO + * workloads. + */ + if (count > inode->i_sb->s_blocksize) + return false; + if (dio_flags & (IOMAP_DIO_FORCE_WAIT | IOMAP_DIO_PARTIAL | + IOMAP_DIO_BOUNCE)) + return false; + if (iocb->ki_pos + count > i_size_read(inode)) + return false; + if (IS_ENCRYPTED(inode)) + return false; + + return true; +} + +/* + * Fast path for small, block-aligned direct I/Os that map to a single + * contiguous on-disk extent. + * + * iomap_dio_simple_supported() enforces the cheap up-front constraints before + * entering this path. + * + * @dops must be NULL: a non-NULL @dops means the caller wants its + * ->end_io / ->submit_io hooks invoked, and in particular wants its bios to be + * allocated from the filesystem-private @dops->bio_set (whose front_pad sizes a + * filesystem-private wrapper around the bio). The fast path instead allocates + * from the shared iomap_dio_simple_pool, whose front_pad matches struct + * iomap_dio_simple; the two wrappers are not interchangeable, so we must fall + * back to __iomap_dio_rw() in that case. + * + * @done_before must be zero: a non-zero caller-accumulated residual cannot be + * carried through a single-bio inline completion. + * + * @iter must describe a non-empty READ no larger than the inode block size: + * writes, zero-length I/O, and larger requests need the generic iomap direct + * I/O path. + * + * @dio_flags must not request IOMAP_DIO_FORCE_WAIT, IOMAP_DIO_PARTIAL, or + * IOMAP_DIO_BOUNCE: this path does not support forced waiting, partial direct + * I/O, or bouncing. The range must also stay within i_size and encrypted + * inodes must use the generic iomap direct I/O path. + * + * -ENOTBLK is the private sentinel returned by iomap_dio_simple() when it + * decides the request does not fit the fast path. In that case we proceed to + * the generic __iomap_dio_rw() slow path. Any other errno is a real result and + * is propagated as-is, in particular -EAGAIN for IOCB_NOWAIT must reach the + * caller. + */ +static ssize_t +iomap_dio_simple(struct kiocb *iocb, struct iov_iter *iter, + const struct iomap_ops *ops, void *private, + unsigned int dio_flags) +{ + struct inode *inode = file_inode(iocb->ki_filp); + size_t count = iov_iter_count(iter); + bool wait_for_completion = is_sync_kiocb(iocb); + struct iomap_iter iomi = { + .inode = inode, + .pos = iocb->ki_pos, + .len = count, + .flags = IOMAP_DIRECT, + .private = private, + }; + struct iomap_dio_simple *sr; + unsigned int alignment; + struct bio *bio; + ssize_t ret; + + if (iocb->ki_flags & IOCB_NOWAIT) + iomi.flags |= IOMAP_NOWAIT; + + ret = kiocb_write_and_wait(iocb, count); + if (ret) + return ret; + + inode_dio_begin(inode); + + ret = ops->iomap_begin(inode, iomi.pos, count, iomi.flags, + &iomi.iomap, &iomi.srcmap); + if (ret) { + inode_dio_end(inode); + return ret; + } + + if (iomi.iomap.type != IOMAP_MAPPED || + iomi.iomap.offset + iomi.iomap.length < iomi.pos + count || + (iomi.iomap.flags & IOMAP_F_INTEGRITY)) { + ret = -ENOTBLK; + goto out_iomap_end; + } + + alignment = iomap_dio_alignment(inode, iomi.iomap.bdev, dio_flags); + if ((iomi.pos | count) & (alignment - 1)) { + ret = -EINVAL; + goto out_iomap_end; + } + + if (!wait_for_completion && unlikely(!inode->i_sb->s_dio_done_wq)) { + ret = sb_init_dio_done_wq(inode->i_sb); + if (ret < 0) + goto out_iomap_end; + } + + trace_iomap_dio_rw_begin(iocb, iter, dio_flags, 0); + + if (user_backed_iter(iter)) + dio_flags |= IOMAP_DIO_USER_BACKED; + + bio = bio_alloc_bioset(iomi.iomap.bdev, + bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS), + REQ_OP_READ, GFP_KERNEL, &iomap_dio_simple_pool); + sr = container_of(bio, struct iomap_dio_simple, bio); + sr->iocb = iocb; + sr->dio_flags = dio_flags; + + bio->bi_iter.bi_sector = iomap_sector(&iomi.iomap, iomi.pos); + bio->bi_ioprio = iocb->ki_ioprio; + + ret = bio_iov_iter_get_pages(bio, iter, alignment - 1); + if (unlikely(ret)) + goto out_bio_put; + + if (bio->bi_iter.bi_size != count) { + iov_iter_revert(iter, bio->bi_iter.bi_size); + ret = -ENOTBLK; + goto out_bio_release_pages; + } + + sr->size = bio->bi_iter.bi_size; + + if (dio_flags & IOMAP_DIO_USER_BACKED) + bio_set_pages_dirty(bio); + + if (iocb->ki_flags & IOCB_NOWAIT) + bio->bi_opf |= REQ_NOWAIT; + if ((iocb->ki_flags & IOCB_HIPRI) && !wait_for_completion) { + bio->bi_opf |= REQ_POLLED; + WRITE_ONCE(iocb->private, bio); + } + + if (ops->iomap_end) + ops->iomap_end(inode, iomi.pos, count, count, iomi.flags, + &iomi.iomap); + + if (!wait_for_completion) { + bio->bi_end_io = iomap_dio_simple_end_io; + submit_bio(bio); + trace_iomap_dio_rw_queued(inode, iomi.pos, count); + return -EIOCBQUEUED; + } + + submit_bio_wait(bio); + return iomap_dio_simple_complete(sr); + +out_bio_release_pages: + bio_release_pages(bio, false); +out_bio_put: + bio_put(bio); +out_iomap_end: + if (ops->iomap_end) + ops->iomap_end(inode, iomi.pos, count, 0, iomi.flags, + &iomi.iomap); + inode_dio_end(inode); + return ret; +} + ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, const struct iomap_ops *ops, const struct iomap_dio_ops *dops, unsigned int dio_flags, void *private, size_t done_before) { struct iomap_dio *dio; + ssize_t ret; + + if (iomap_dio_simple_supported(iocb, iter, dops, dio_flags, + done_before)) { + ret = iomap_dio_simple(iocb, iter, ops, private, dio_flags); + if (ret != -ENOTBLK) + return ret; + } dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags, private, done_before); @@ -907,3 +1173,11 @@ iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, return iomap_dio_complete(dio); } EXPORT_SYMBOL_GPL(iomap_dio_rw); + +static int __init iomap_dio_init(void) +{ + return bioset_init(&iomap_dio_simple_pool, 4, + offsetof(struct iomap_dio_simple, bio), + BIOSET_NEED_BVECS | BIOSET_PERCPU_CACHE); +} +fs_initcall(iomap_dio_init); From 1a061c5542533515886d5bb5ee0f1676e83048ba Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Tue, 7 Jul 2026 15:04:48 -0700 Subject: [PATCH 05/29] fuse: don't clear folio uptodate on writethrough errors In the writethrough path (fuse_send_write_pages()), if the write to the server failed or was a short write, the uptodate flag on the folios are cleared. As explained by Matthew in [1], this is dangerous because the folio may be mapped into userspace. The mm code has the invariant that a non-uptodate folio must never be visible to userspace (to avoid potentially leaking confidental information to userspace) and has checks in place for this that if violated can bring down the whole machine. Practically speaking, the effect of this change for the fuse writethrough error path is that if an application does a write and then the server fails to persist the data or only services a short write, the page cache folio keeps the data the application wrote instead of being reverted to the server's contents on the next read. The failure is still reported to the application synchronously through the short count / error return of the write() syscall. Folios that were only partially written are unaffected since they were never marked uptodate in the first place (fuse_fill_write_page() only marks a folio as uptodate if the whole folio was written to). [1] https://lore.kernel.org/linux-fsdevel/ajtPMgO65FA1TXhi@casper.infradead.org/ Suggested-by: Matthew Wilcox Reviewed-by: Darrick J. Wong Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260707220450.1200943-2-joannelkoong@gmail.com Acked-by: Miklos Szeredi Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner (Amutable) --- fs/fuse/file.c | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index e052a0d44dee..a72959dbcf12 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1227,8 +1227,7 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, struct file *file = iocb->ki_filp; struct fuse_file *ff = file->private_data; struct fuse_mount *fm = ff->fm; - unsigned int offset, i; - bool short_write; + unsigned int i; int err; for (i = 0; i < ap->num_folios; i++) @@ -1243,24 +1242,9 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, if (!err && ia->write.out.size > count) err = -EIO; - short_write = ia->write.out.size < count; - offset = ap->descs[0].offset; - count = ia->write.out.size; for (i = 0; i < ap->num_folios; i++) { struct folio *folio = ap->folios[i]; - if (err) { - folio_clear_uptodate(folio); - } else { - if (count >= folio_size(folio) - offset) - count -= folio_size(folio) - offset; - else { - if (short_write) - folio_clear_uptodate(folio); - count = 0; - } - offset = 0; - } if (ia->write.folio_locked && (i == ap->num_folios - 1)) folio_unlock(folio); folio_put(folio); From 456b873e63c7c0b298e04d85a79f5b1673b6f1ad Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Tue, 7 Jul 2026 15:04:49 -0700 Subject: [PATCH 06/29] iomap: add helper to mark folio uptodate Add an exported helper iomap_folio_mark_uptodate() to mark a folio as uptodate and update its uptodate bitmap if the folio has iomap state data attached. This is needed because there are some filesystems (eg fuse) that have paths outside of conventional iomap calls that need to mark a folio as uptodate (eg writing server-pushed data directly into the page cache) and need the iomap-internal uptodate bitmap to be in sync with the uptodate state of the folio. Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260707220450.1200943-3-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- fs/iomap/buffered-io.c | 6 ++++++ include/linux/iomap.h | 1 + 2 files changed, 7 insertions(+) diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index 27bc2455a98d..f6040199d114 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -105,6 +105,12 @@ static void iomap_set_range_uptodate(struct folio *folio, size_t off, folio_mark_uptodate(folio); } +void iomap_folio_mark_uptodate(struct folio *folio) +{ + iomap_set_range_uptodate(folio, 0, folio_size(folio)); +} +EXPORT_SYMBOL_GPL(iomap_folio_mark_uptodate); + /* * Find the next dirty block in the folio. end_blk is inclusive. * If no dirty block is found, this will return end_blk + 1. diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 3582ed1fe236..21e73cb9c51e 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -365,6 +365,7 @@ struct folio *iomap_get_folio(struct iomap_iter *iter, loff_t pos, size_t len); bool iomap_release_folio(struct folio *folio, gfp_t gfp_flags); void iomap_invalidate_folio(struct folio *folio, size_t offset, size_t len); bool iomap_dirty_folio(struct address_space *mapping, struct folio *folio); +void iomap_folio_mark_uptodate(struct folio *folio); int iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len, const struct iomap_ops *ops, const struct iomap_write_ops *write_ops); From 881a27082e4d7faf21ad6960a3c6a0b6c5abc78d Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Tue, 7 Jul 2026 15:04:50 -0700 Subject: [PATCH 07/29] fuse: use iomap helper to mark folio uptodate When fuse enables large folios, a large folio will be backed by iomap_folio_state that keeps track of uptodate and dirty state in an internal bitmap. Fuse writethrough and notify store paths currently set folio uptodate state with folio_mark_uptodate(), which touches only the folio-level flag, but on an iomap-backed folio, that leaves the uptodate bitmap out of sync. Use the iomap_folio_mark_uptodate() helper to update both the folio uptodate state and the iomap uptodate bitmap. Reviewed-by: Darrick J. Wong Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260707220450.1200943-4-joannelkoong@gmail.com Acked-by: Miklos Szeredi Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner (Amutable) --- fs/fuse/file.c | 2 +- fs/fuse/notify.c | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index a72959dbcf12..ea4a15a7635a 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1319,7 +1319,7 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, /* If we copied full folio, mark it uptodate */ if (tmp == folio_size(folio)) - folio_mark_uptodate(folio); + iomap_folio_mark_uptodate(folio); if (folio_test_uptodate(folio)) { folio_unlock(folio); diff --git a/fs/fuse/notify.c b/fs/fuse/notify.c index 29578104ae6c..1ba763705d91 100644 --- a/fs/fuse/notify.c +++ b/fs/fuse/notify.c @@ -2,6 +2,8 @@ #include "dev.h" #include "fuse_i.h" + +#include #include static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size, @@ -192,7 +194,7 @@ static int fuse_notify_store(struct fuse_conn *fc, unsigned int size, if (!folio_test_uptodate(folio) && !err && folio_offset == 0 && (nr_bytes == folio_size(folio) || file_size == end)) { folio_zero_segment(folio, nr_bytes, folio_size(folio)); - folio_mark_uptodate(folio); + iomap_folio_mark_uptodate(folio); } folio_unlock(folio); folio_put(folio); From 31e3d833d522746a93d135e8b465d16f8ad33453 Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Wed, 29 Jul 2026 12:27:16 -0700 Subject: [PATCH 08/29] iomap: release the folio batch on iomap callback failures A sashiko review of an unrelated patch points out that the folio batch mechanism used for iomap zero range fails to release the batch in a couple error scenarios. If either calls to ->iomap_end() or ->iomap_begin() fail, the direct return paths bypass the batch cleanup. The ->iomap_end() case is not a practical issue at the moment because there is no user of the mechanism that returns an error from this path. The ->iomap_begin() case is theoretically possible because XFS can invoke the fill helper and error out at various points thereafter. This subtly complicates things because XFS does not transfer iomap_flags to the iomap data structure in the error path. To deal with both of these issues, first make sure to invoke the cleanup helper in the error path for either fs callback. Second, update the helper to clear the flag unconditionally and release the batch so long as it is populated. This more clearly delineates the purpose of the flag to control the I/O path and not necessarily the status of the fbatch, so add a comment around this as well. Reported-by: Sashiko Fixes: 395ed1ef0012 ("iomap: optional zero range dirty folio processing") Signed-off-by: Brian Foster Link: https://patch.msgid.link/20260729192737.3190206-2-joannelkoong@gmail.com Reviewed-by: "Darrick J. Wong" Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner (Amutable) --- fs/iomap/iter.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/fs/iomap/iter.c b/fs/iomap/iter.c index e4a29829591a..63617ec48250 100644 --- a/fs/iomap/iter.c +++ b/fs/iomap/iter.c @@ -6,12 +6,18 @@ #include #include "trace.h" +/* + * Release the iter folio batch. Note that the iomap flag is meant to control + * the I/O path for the mapping and may not be set in error situations. + */ static inline void iomap_iter_clean_fbatch(struct iomap_iter *iter) { - if (iter->iomap.flags & IOMAP_F_FOLIO_BATCH) { + if (!iter->fbatch) + return; + iter->iomap.flags &= ~IOMAP_F_FOLIO_BATCH; + if (folio_batch_count(iter->fbatch)) { folio_batch_release(iter->fbatch); folio_batch_reinit(iter->fbatch); - iter->iomap.flags &= ~IOMAP_F_FOLIO_BATCH; } } @@ -79,7 +85,7 @@ int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops) olen), advanced, iter->flags, &iter->iomap); if (ret < 0 && !advanced) - return ret; + goto error; } /* detect old return semantics where this would advance */ @@ -110,7 +116,11 @@ int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops) ret = ops->iomap_begin(iter->inode, iter->pos, iter->len, iter->flags, &iter->iomap, &iter->srcmap); if (ret < 0) - return ret; + goto error; iomap_iter_done(iter); return 1; + +error: + iomap_iter_clean_fbatch(iter); + return ret; } From 19eb9f6ab5ce1d15c7f5e48ca16804a6d7740084 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Wed, 29 Jul 2026 12:27:17 -0700 Subject: [PATCH 09/29] iomap: split iomap_iter() logic into iomap_iter_next() In preparation for changing iomap to use an in-iter (->iomap_next()) model, move the iomap_iter() logic out into the new iomap_iter_next() helper function. iomap_iter_next() is added as an inlined helper so it can be called directly by ->iomap_next() implementations where the begin()/end() callbacks can be direct calls. The DEFINE_IOMAP_ITER_NEXT() and DEFINE_IOMAP_ITER_NEXT_END() macros are also provided to generate the boilerplate ->iomap_next() wrapper functions that simply forward to iomap_iter_next() with the appropriate begin/end callbacks. DEFINE_IOMAP_ITER_NEXT() is for the common case where there is no end() callback. DEFINE_IOMAP_ITER_NEXT_END() is for the case where there is an explicit end() callback. No functional change intended. The one would-be behavioral difference is that on the iomap_end() error path (ret < 0 && !advanced), the old code returned with iter.status left as the caller's last value whereas the new code zeroes it, but this is not observable in practice as there are no in-tree callers that read iter.status after the iteration loop. Reviewed-by: Darrick J. Wong Reviewed-by: Fengnan Chang Reviewed-by: Christoph Hellwig Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260729192737.3190206-3-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- fs/iomap/iter.c | 122 ++++++++++++++++++++++-------------------- include/linux/iomap.h | 102 +++++++++++++++++++++++++++++------ 2 files changed, 148 insertions(+), 76 deletions(-) diff --git a/fs/iomap/iter.c b/fs/iomap/iter.c index 63617ec48250..bf7d4cccc1a7 100644 --- a/fs/iomap/iter.c +++ b/fs/iomap/iter.c @@ -10,11 +10,12 @@ * Release the iter folio batch. Note that the iomap flag is meant to control * the I/O path for the mapping and may not be set in error situations. */ -static inline void iomap_iter_clean_fbatch(struct iomap_iter *iter) +static inline void iomap_iter_clean_fbatch(const struct iomap_iter *iter, + struct iomap *iomap) { if (!iter->fbatch) return; - iter->iomap.flags &= ~IOMAP_F_FOLIO_BATCH; + iomap->flags &= ~IOMAP_F_FOLIO_BATCH; if (folio_batch_count(iter->fbatch)) { folio_batch_release(iter->fbatch); folio_batch_reinit(iter->fbatch); @@ -46,9 +47,60 @@ static inline void iomap_iter_done(struct iomap_iter *iter) } /** - * iomap_iter - iterate over a ranges in a file - * @iter: iteration structue - * @ops: iomap ops provided by the file system + * iomap_iter_continue - decide whether iteration should continue + * @iter: iteration structure + * @iomap: the mapping that was just processed + * @srcmap: the source mapping that was just processed + * + * Helper normally called via iomap_iter_next(). Called after the previous + * mapping has been finished to determine whether there is more of the file + * range left to process. + * + * Returns 1 if there is more work to do, in which case @iomap and @srcmap are + * cleared so the caller can produce the next mapping; zero if the range is + * fully consumed; or a negative errno on error. + */ +int iomap_iter_continue(const struct iomap_iter *iter, struct iomap *iomap, + struct iomap *srcmap, int ret) +{ + const bool stale = iomap->flags & IOMAP_F_STALE; + const ssize_t advanced = iter->pos - iter->iter_start_pos; + + if (ret < 0 && !advanced) + return ret; + + /* + * Use iter->len to determine whether to continue onto the next mapping. + * Explicitly terminate on error status or if the current iter has not + * advanced at all (i.e. no work was done for some reason) unless the + * mapping has been marked stale and needs to be reprocessed. + */ + if (WARN_ON_ONCE(iter->status > 0)) + /* detect old return semantics where this would advance */ + ret = -EIO; + else if (iter->status < 0) + ret = iter->status; + else if (iter->len == 0 || (!advanced && !stale)) + ret = 0; + else + ret = 1; + + iomap_iter_clean_fbatch(iter, iomap); + + if (ret <= 0) + return ret; + + memset(iomap, 0, sizeof(*iomap)); + memset(srcmap, 0, sizeof(*srcmap)); + + return ret; +} +EXPORT_SYMBOL_GPL(iomap_iter_continue); + +/** + * iomap_iter - iterate over ranges in a file + * @iter: iteration structure + * @ops: iomap ops provided by the filesystem * * Iterate over filesystem-provided space mappings for the provided file range. * @@ -62,65 +114,17 @@ static inline void iomap_iter_done(struct iomap_iter *iter) */ int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops) { - bool stale = iter->iomap.flags & IOMAP_F_STALE; - ssize_t advanced; - u64 olen; int ret; trace_iomap_iter(iter, ops, _RET_IP_); - if (!iter->iomap.length) - goto begin; - - /* - * Calculate how far the iter was advanced and the original length bytes - * for ->iomap_end(). - */ - advanced = iter->pos - iter->iter_start_pos; - olen = iter->len + advanced; - - if (ops->iomap_end) { - ret = ops->iomap_end(iter->inode, iter->iter_start_pos, - iomap_length_trim(iter, iter->iter_start_pos, - olen), - advanced, iter->flags, &iter->iomap); - if (ret < 0 && !advanced) - goto error; - } - - /* detect old return semantics where this would advance */ - if (WARN_ON_ONCE(iter->status > 0)) - iter->status = -EIO; - - /* - * Use iter->len to determine whether to continue onto the next mapping. - * Explicitly terminate on error status or if the current iter has not - * advanced at all (i.e. no work was done for some reason) unless the - * mapping has been marked stale and needs to be reprocessed. - */ - if (iter->status < 0) - ret = iter->status; - else if (iter->len == 0 || (!advanced && !stale)) - ret = 0; - else - ret = 1; - iomap_iter_clean_fbatch(iter); + ret = iomap_iter_next(iter, &iter->iomap, &iter->srcmap, + ops->iomap_begin, ops->iomap_end); iter->status = 0; - if (ret <= 0) - return ret; + if (ret > 0) + iomap_iter_done(iter); + else if (ret < 0) + iomap_iter_clean_fbatch(iter, &iter->iomap); - memset(&iter->iomap, 0, sizeof(iter->iomap)); - memset(&iter->srcmap, 0, sizeof(iter->srcmap)); - -begin: - ret = ops->iomap_begin(iter->inode, iter->pos, iter->len, iter->flags, - &iter->iomap, &iter->srcmap); - if (ret < 0) - goto error; - iomap_iter_done(iter); - return 1; - -error: - iomap_iter_clean_fbatch(iter); return ret; } diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 21e73cb9c51e..36490c08d6e9 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -212,24 +212,27 @@ struct iomap_write_ops { #define IOMAP_ATOMIC (1 << 9) /* torn-write protection */ #define IOMAP_DONTCACHE (1 << 10) -struct iomap_ops { - /* - * Return the existing mapping at pos, or reserve space starting at - * pos for up to length, as long as we can do it as a single mapping. - * The actual length is returned in iomap->length. - */ - int (*iomap_begin)(struct inode *inode, loff_t pos, loff_t length, - unsigned flags, struct iomap *iomap, - struct iomap *srcmap); +/* + * Return the existing mapping at pos, or reserve space starting at pos for up + * to length, as long as we can do it as a single mapping. + * The actual length is returned in iomap->length. + */ +typedef int (*iomap_iter_begin_fn)(struct inode *inode, loff_t pos, + loff_t length, unsigned flags, struct iomap *iomap, + struct iomap *srcmap); - /* - * Commit and/or unreserve space previous allocated using iomap_begin. - * Written indicates the length of the successful write operation which - * needs to be commited, while the rest needs to be unreserved. - * Written might be zero if no data was written. - */ - int (*iomap_end)(struct inode *inode, loff_t pos, loff_t length, - ssize_t written, unsigned flags, struct iomap *iomap); +/* + * Commit and/or unreserve space previously allocated by iomap_iter_begin_fn. + * Written indicates the length of the successful write operation which needs + * to be committed, while the rest needs to be unreserved. + * Written might be zero if no data was written. + */ +typedef int (*iomap_iter_end_fn)(struct inode *inode, loff_t pos, loff_t length, + ssize_t written, unsigned flags, struct iomap *iomap); + +struct iomap_ops { + iomap_iter_begin_fn iomap_begin; + iomap_iter_end_fn iomap_end; }; /** @@ -317,6 +320,71 @@ static inline const struct iomap *iomap_iter_srcmap(const struct iomap_iter *i) return &i->iomap; } +int iomap_iter_continue(const struct iomap_iter *iter, struct iomap *iomap, + struct iomap *srcmap, int ret); + +/** + * iomap_iter_next - finish the previous mapping and produce the next one + * @iter: iteration structure + * @iomap: mapping to finish and then repopulate + * @srcmap: source mapping to finish and then repopulate + * @begin: callback that produces a mapping for the current position + * @end: optional callback that finishes the previous mapping, or NULL + * + * Inline helper that implements the common body of an ->iomap_next() + * callback: it finishes the previous mapping via @end (if present), decides + * via iomap_iter_continue() whether to keep going, and obtains the next + * mapping via @begin. + * + * This helper is marked __always_inline so that when a caller passes + * compile-time-constant @begin and @end callbacks, the compiler can call them + * directly, avoiding the indirect-call overhead. + * + * Returns 1 to continue iterating, 0 once the range is fully consumed, or a + * negative errno on error. + */ +static __always_inline int iomap_iter_next(const struct iomap_iter *iter, + struct iomap *iomap, struct iomap *srcmap, + iomap_iter_begin_fn begin, iomap_iter_end_fn end) +{ + int ret = 0; + + if (iomap->length) { + if (end) { + /* + * Calculate how far the iter was advanced and the + * original length bytes for end(). + */ + ssize_t advanced = iter->pos - iter->iter_start_pos; + loff_t len; + + len = iomap_length_trim(iter, iter->iter_start_pos, + iter->len + advanced); + + ret = end(iter->inode, iter->iter_start_pos, len, + advanced, iter->flags, iomap); + } + ret = iomap_iter_continue(iter, iomap, srcmap, ret); + if (ret <= 0) + return ret; + } + + ret = begin(iter->inode, iter->pos, iter->len, iter->flags, iomap, + srcmap); + + return ret < 0 ? ret : 1; +} + +#define DEFINE_IOMAP_ITER_NEXT_END(name, begin_fn, end_fn) \ +int name(const struct iomap_iter *iter, struct iomap *iomap, \ + struct iomap *srcmap) \ +{ \ + return iomap_iter_next(iter, iomap, srcmap, begin_fn, end_fn); \ +} + +#define DEFINE_IOMAP_ITER_NEXT(name, begin_fn) \ + DEFINE_IOMAP_ITER_NEXT_END(name, begin_fn, NULL) + /* * Return the file offset for the first unchanged block after a short write. * From 335d4b6201ac317d906e6a694f07de0792325fee Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 29 Jul 2026 12:27:18 -0700 Subject: [PATCH 10/29] iomap: decouple simple direct I/O reads from iomap_dio_rw The pending iomap_iter_next conversion creates performance issues for the new simple direct I/O read fast path, because it assumes a model where the iterator must be advanced at the end, which the direct I/O read fast path tries to avoid. Side step this by splitting the simple path from iomap_dio_rw, and require the file systems to call into it explicitly, and pass only a ->begin callback. This allows to drop various checks for incompatible features while creating a requirement for the file system to only call the simple path for cases that it can handle. As a side-benefit we can now inline the initial part of the simple direct I/O read fast path and let the compiler convert the indirect call to ->begin into a direct call. Reviewed-by: "Darrick J. Wong" Reviewed-by: Fengnan Chang Reviewed-by: Joanne Koong Signed-off-by: Joanne Koong Signed-off-by: Christoph Hellwig Link: https://patch.msgid.link/20260729192737.3190206-4-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- fs/ext4/ext4.h | 3 + fs/ext4/file.c | 4 +- fs/ext4/inode.c | 2 +- fs/iomap/direct-io.c | 208 ++++++++++-------------------------------- fs/xfs/xfs_file.c | 14 +-- fs/xfs/xfs_iomap.c | 2 +- fs/xfs/xfs_iomap.h | 4 + include/linux/iomap.h | 66 ++++++++++++++ 8 files changed, 136 insertions(+), 167 deletions(-) diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index b37c136ea3ab..e134c0193e2b 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -4007,6 +4007,9 @@ static inline void ext4_clear_io_unwritten_flag(ext4_io_end_t *io_end) extern const struct iomap_ops ext4_iomap_ops; extern const struct iomap_ops ext4_iomap_report_ops; +int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned flags, struct iomap *iomap, struct iomap *srcmap); + static inline int ext4_buffer_uptodate(struct buffer_head *bh) { /* diff --git a/fs/ext4/file.c b/fs/ext4/file.c index eb1a323962b1..f20d92255546 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -91,7 +91,9 @@ static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to) return generic_file_read_iter(iocb, to); } - ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0, NULL, 0); + ret = iomap_dio_read_simple(iocb, to, ext4_iomap_begin); + if (ret == -ENOTBLK) + ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0, NULL, 0); inode_unlock_shared(inode); file_accessed(iocb->ki_filp); diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index ce99807c5f5b..b48c54f3312b 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -3771,7 +3771,7 @@ static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map, } -static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length, +int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length, unsigned flags, struct iomap *iomap, struct iomap *srcmap) { int ret; diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index ca790239e5eb..36c976cf0848 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -894,6 +894,21 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, } EXPORT_SYMBOL_GPL(__iomap_dio_rw); +ssize_t +iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, + const struct iomap_ops *ops, const struct iomap_dio_ops *dops, + unsigned int dio_flags, void *private, size_t done_before) +{ + struct iomap_dio *dio; + + dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags, private, + done_before); + if (IS_ERR_OR_NULL(dio)) + return PTR_ERR_OR_ZERO(dio); + return iomap_dio_complete(dio); +} +EXPORT_SYMBOL_GPL(iomap_dio_rw); + struct iomap_dio_simple { struct kiocb *iocb; size_t size; @@ -968,211 +983,88 @@ static void iomap_dio_simple_end_io(struct bio *bio) iocb->ki_complete(iocb, iomap_dio_simple_complete(sr)); } -static inline bool -iomap_dio_simple_supported(struct kiocb *iocb, struct iov_iter *iter, - const struct iomap_dio_ops *dops, - unsigned int dio_flags, size_t done_before) +ssize_t __iomap_dio_read_simple(struct kiocb *iocb, struct iov_iter *iter, + struct iomap_iter *iomi) { - struct inode *inode = file_inode(iocb->ki_filp); - size_t count = iov_iter_count(iter); - - if (dops || done_before) - return false; - if (iov_iter_rw(iter) != READ) - return false; - if (!count) - return false; - /* - * Simple dio is an optimization for small IO. Filter out large IO - * early as it's the most common case to fail for typical direct IO - * workloads. - */ - if (count > inode->i_sb->s_blocksize) - return false; - if (dio_flags & (IOMAP_DIO_FORCE_WAIT | IOMAP_DIO_PARTIAL | - IOMAP_DIO_BOUNCE)) - return false; - if (iocb->ki_pos + count > i_size_read(inode)) - return false; - if (IS_ENCRYPTED(inode)) - return false; - - return true; -} - -/* - * Fast path for small, block-aligned direct I/Os that map to a single - * contiguous on-disk extent. - * - * iomap_dio_simple_supported() enforces the cheap up-front constraints before - * entering this path. - * - * @dops must be NULL: a non-NULL @dops means the caller wants its - * ->end_io / ->submit_io hooks invoked, and in particular wants its bios to be - * allocated from the filesystem-private @dops->bio_set (whose front_pad sizes a - * filesystem-private wrapper around the bio). The fast path instead allocates - * from the shared iomap_dio_simple_pool, whose front_pad matches struct - * iomap_dio_simple; the two wrappers are not interchangeable, so we must fall - * back to __iomap_dio_rw() in that case. - * - * @done_before must be zero: a non-zero caller-accumulated residual cannot be - * carried through a single-bio inline completion. - * - * @iter must describe a non-empty READ no larger than the inode block size: - * writes, zero-length I/O, and larger requests need the generic iomap direct - * I/O path. - * - * @dio_flags must not request IOMAP_DIO_FORCE_WAIT, IOMAP_DIO_PARTIAL, or - * IOMAP_DIO_BOUNCE: this path does not support forced waiting, partial direct - * I/O, or bouncing. The range must also stay within i_size and encrypted - * inodes must use the generic iomap direct I/O path. - * - * -ENOTBLK is the private sentinel returned by iomap_dio_simple() when it - * decides the request does not fit the fast path. In that case we proceed to - * the generic __iomap_dio_rw() slow path. Any other errno is a real result and - * is propagated as-is, in particular -EAGAIN for IOCB_NOWAIT must reach the - * caller. - */ -static ssize_t -iomap_dio_simple(struct kiocb *iocb, struct iov_iter *iter, - const struct iomap_ops *ops, void *private, - unsigned int dio_flags) -{ - struct inode *inode = file_inode(iocb->ki_filp); - size_t count = iov_iter_count(iter); - bool wait_for_completion = is_sync_kiocb(iocb); - struct iomap_iter iomi = { - .inode = inode, - .pos = iocb->ki_pos, - .len = count, - .flags = IOMAP_DIRECT, - .private = private, - }; struct iomap_dio_simple *sr; unsigned int alignment; struct bio *bio; ssize_t ret; - if (iocb->ki_flags & IOCB_NOWAIT) - iomi.flags |= IOMAP_NOWAIT; - - ret = kiocb_write_and_wait(iocb, count); - if (ret) - return ret; - - inode_dio_begin(inode); - - ret = ops->iomap_begin(inode, iomi.pos, count, iomi.flags, - &iomi.iomap, &iomi.srcmap); - if (ret) { - inode_dio_end(inode); - return ret; - } - - if (iomi.iomap.type != IOMAP_MAPPED || - iomi.iomap.offset + iomi.iomap.length < iomi.pos + count || - (iomi.iomap.flags & IOMAP_F_INTEGRITY)) { + if (iomi->iomap.type != IOMAP_MAPPED || + iomi->iomap.offset + iomi->iomap.length < iomi->pos + iomi->len || + (iomi->iomap.flags & IOMAP_F_INTEGRITY)) { ret = -ENOTBLK; - goto out_iomap_end; + goto out_dio_end; } - alignment = iomap_dio_alignment(inode, iomi.iomap.bdev, dio_flags); - if ((iomi.pos | count) & (alignment - 1)) { + alignment = iomap_dio_alignment(iomi->inode, iomi->iomap.bdev, 0); + if ((iomi->pos | iomi->len) & (alignment - 1)) { ret = -EINVAL; - goto out_iomap_end; + goto out_dio_end; } - if (!wait_for_completion && unlikely(!inode->i_sb->s_dio_done_wq)) { - ret = sb_init_dio_done_wq(inode->i_sb); + if (unlikely(!iomi->inode->i_sb->s_dio_done_wq && + !is_sync_kiocb(iocb))) { + ret = sb_init_dio_done_wq(iomi->inode->i_sb); if (ret < 0) - goto out_iomap_end; + goto out_dio_end; } - trace_iomap_dio_rw_begin(iocb, iter, dio_flags, 0); + trace_iomap_dio_rw_begin(iocb, iter, 0, 0); - if (user_backed_iter(iter)) - dio_flags |= IOMAP_DIO_USER_BACKED; - - bio = bio_alloc_bioset(iomi.iomap.bdev, + bio = bio_alloc_bioset(iomi->iomap.bdev, bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS), REQ_OP_READ, GFP_KERNEL, &iomap_dio_simple_pool); sr = container_of(bio, struct iomap_dio_simple, bio); sr->iocb = iocb; - sr->dio_flags = dio_flags; + sr->dio_flags = 0; - bio->bi_iter.bi_sector = iomap_sector(&iomi.iomap, iomi.pos); + bio->bi_iter.bi_sector = iomap_sector(&iomi->iomap, iomi->pos); bio->bi_ioprio = iocb->ki_ioprio; ret = bio_iov_iter_get_pages(bio, iter, alignment - 1); if (unlikely(ret)) goto out_bio_put; - if (bio->bi_iter.bi_size != count) { + if (bio->bi_iter.bi_size != iomi->len) { iov_iter_revert(iter, bio->bi_iter.bi_size); ret = -ENOTBLK; goto out_bio_release_pages; } sr->size = bio->bi_iter.bi_size; - - if (dio_flags & IOMAP_DIO_USER_BACKED) + if (user_backed_iter(iter)) { bio_set_pages_dirty(bio); + sr->dio_flags |= IOMAP_DIO_USER_BACKED; + } if (iocb->ki_flags & IOCB_NOWAIT) bio->bi_opf |= REQ_NOWAIT; - if ((iocb->ki_flags & IOCB_HIPRI) && !wait_for_completion) { + + if (is_sync_kiocb(iocb)) { + submit_bio_wait(bio); + return iomap_dio_simple_complete(sr); + } + + if ((iocb->ki_flags & IOCB_HIPRI)) { bio->bi_opf |= REQ_POLLED; WRITE_ONCE(iocb->private, bio); } - - if (ops->iomap_end) - ops->iomap_end(inode, iomi.pos, count, count, iomi.flags, - &iomi.iomap); - - if (!wait_for_completion) { - bio->bi_end_io = iomap_dio_simple_end_io; - submit_bio(bio); - trace_iomap_dio_rw_queued(inode, iomi.pos, count); - return -EIOCBQUEUED; - } - - submit_bio_wait(bio); - return iomap_dio_simple_complete(sr); + bio->bi_end_io = iomap_dio_simple_end_io; + submit_bio(bio); + trace_iomap_dio_rw_queued(iomi->inode, iocb->ki_pos, iomi->len); + return -EIOCBQUEUED; out_bio_release_pages: bio_release_pages(bio, false); out_bio_put: bio_put(bio); -out_iomap_end: - if (ops->iomap_end) - ops->iomap_end(inode, iomi.pos, count, 0, iomi.flags, - &iomi.iomap); - inode_dio_end(inode); +out_dio_end: + inode_dio_end(iomi->inode); return ret; } - -ssize_t -iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, - const struct iomap_ops *ops, const struct iomap_dio_ops *dops, - unsigned int dio_flags, void *private, size_t done_before) -{ - struct iomap_dio *dio; - ssize_t ret; - - if (iomap_dio_simple_supported(iocb, iter, dops, dio_flags, - done_before)) { - ret = iomap_dio_simple(iocb, iter, ops, private, dio_flags); - if (ret != -ENOTBLK) - return ret; - } - - dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags, private, - done_before); - if (IS_ERR_OR_NULL(dio)) - return PTR_ERR_OR_ZERO(dio); - return iomap_dio_complete(dio); -} -EXPORT_SYMBOL_GPL(iomap_dio_rw); +EXPORT_SYMBOL_GPL(__iomap_dio_read_simple); static int __init iomap_dio_init(void) { diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 845a97c9b063..b733225d2864 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -251,8 +251,6 @@ xfs_file_dio_read( struct iov_iter *to) { struct xfs_inode *ip = XFS_I(file_inode(iocb->ki_filp)); - unsigned int dio_flags = 0; - const struct iomap_dio_ops *dio_ops = NULL; ssize_t ret; trace_xfs_file_direct_read(iocb, to); @@ -266,11 +264,15 @@ xfs_file_dio_read( if (ret) return ret; if (mapping_stable_writes(iocb->ki_filp->f_mapping)) { - dio_ops = &xfs_dio_read_bounce_ops; - dio_flags |= IOMAP_DIO_BOUNCE; + ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, + &xfs_dio_read_bounce_ops, IOMAP_DIO_BOUNCE, + NULL, 0); + } else { + ret = iomap_dio_read_simple(iocb, to, xfs_read_iomap_begin); + if (ret == -ENOTBLK) + ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, NULL, + 0, NULL, 0); } - ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, dio_ops, dio_flags, - NULL, 0); xfs_iunlock(ip, XFS_IOLOCK_SHARED); return ret; diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index 225c3de88d03..0536e2aeddcc 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -2173,7 +2173,7 @@ const struct iomap_ops xfs_buffered_write_iomap_ops = { .iomap_end = xfs_buffered_write_iomap_end, }; -static int +int xfs_read_iomap_begin( struct inode *inode, loff_t offset, diff --git a/fs/xfs/xfs_iomap.h b/fs/xfs/xfs_iomap.h index ebcce7d49446..cffcec532ea6 100644 --- a/fs/xfs/xfs_iomap.h +++ b/fs/xfs/xfs_iomap.h @@ -49,6 +49,10 @@ xfs_aligned_fsb_count( return count_fsb; } +int xfs_read_iomap_begin(struct inode *inode, loff_t offset, + loff_t length, unsigned flags, struct iomap *iomap, + struct iomap *srcmap); + extern const struct iomap_ops xfs_buffered_write_iomap_ops; extern const struct iomap_ops xfs_direct_write_iomap_ops; extern const struct iomap_ops xfs_zoned_direct_write_iomap_ops; diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 36490c08d6e9..80832edc7ec2 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -10,6 +10,7 @@ #include #include #include +#include struct address_space; struct fiemap_extent_info; @@ -675,6 +676,71 @@ struct iomap_dio *__iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, ssize_t iomap_dio_complete(struct iomap_dio *dio); void iomap_dio_bio_end_io(struct bio *bio); +/* + * Fast path for small, block-aligned direct I/Os that map to a single + * contiguous on-disk extent. + * + * @iter must describe a non-empty READ no larger than the inode block size: + * writes, zero-length I/O, and larger requests need the generic iomap direct + * I/O path. + * + * Does not support iomap_dio_ops, dio_flags, done_before or private data. + * The range must also stay within i_size and encrypted inodes must use the + * generic iomap direct I/O path. + * + * -ENOTBLK indicates the generic path must be used by the caller instead. + * Any other errno is a real result and is propagated as-is, in particular + * -EAGAIN for IOCB_NOWAIT must reach the caller. + * + * The caller can only provide an iomap begin handler, and the iterator + * is never advanced. + */ +ssize_t __iomap_dio_read_simple(struct kiocb *iocb, struct iov_iter *iter, + struct iomap_iter *iomi); +static __always_inline ssize_t iomap_dio_read_simple(struct kiocb *iocb, + struct iov_iter *iter, iomap_iter_begin_fn begin) +{ + struct iomap_iter iomi = { + .inode = file_inode(iocb->ki_filp), + .pos = iocb->ki_pos, + .len = iov_iter_count(iter), + .flags = IOMAP_DIRECT, + }; + ssize_t ret; + + if (!iomi.len) + return 0; + + /* + * Simple dio is an optimization for small IO. Filter out large IO + * early as it's the most common case to fail for typical direct IO + * workloads. + */ + if (iomi.len > iomi.inode->i_sb->s_blocksize) + return -ENOTBLK; + if (iocb->ki_pos + iomi.len > i_size_read(iomi.inode)) + return -ENOTBLK; + if (IS_ENCRYPTED(iomi.inode)) + return -ENOTBLK; + + ret = kiocb_write_and_wait(iocb, iomi.len); + if (ret) + return ret; + + if (iocb->ki_flags & IOCB_NOWAIT) + iomi.flags |= IOMAP_NOWAIT; + + inode_dio_begin(iomi.inode); + ret = begin(iomi.inode, iomi.pos, iomi.len, iomi.flags, &iomi.iomap, + &iomi.srcmap); + if (ret) { + inode_dio_end(iomi.inode); + return ret; + } + + return __iomap_dio_read_simple(iocb, iter, &iomi); +} + #ifdef CONFIG_SWAP struct file; struct swap_info_struct; From eecfab484dc7b4156d4297b723da20bbc46e92b9 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 29 Jul 2026 12:27:19 -0700 Subject: [PATCH 11/29] iomap: use GFP_NOWAIT when application for iomap_dio_simple allocations For non-blocking iocbs we should avoid blocking allocation where possible, so switch to a GFP_NOWAIT allocation here. Reviewed-by: Darrick J. Wong Reviewed-by: Fengnan Chang Reviewed-by: Joanne Koong Signed-off-by: Joanne Koong Signed-off-by: Christoph Hellwig Link: https://patch.msgid.link/20260729192737.3190206-5-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- fs/iomap/direct-io.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 36c976cf0848..5c28124b9f02 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -986,6 +986,7 @@ static void iomap_dio_simple_end_io(struct bio *bio) ssize_t __iomap_dio_read_simple(struct kiocb *iocb, struct iov_iter *iter, struct iomap_iter *iomi) { + gfp_t gfp = (iomi->flags & IOMAP_NOWAIT) ? GFP_NOWAIT : GFP_KERNEL; struct iomap_dio_simple *sr; unsigned int alignment; struct bio *bio; @@ -1015,7 +1016,11 @@ ssize_t __iomap_dio_read_simple(struct kiocb *iocb, struct iov_iter *iter, bio = bio_alloc_bioset(iomi->iomap.bdev, bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS), - REQ_OP_READ, GFP_KERNEL, &iomap_dio_simple_pool); + REQ_OP_READ, gfp, &iomap_dio_simple_pool); + if (!bio) { + ret = -EAGAIN; + goto out_dio_end; + } sr = container_of(bio, struct iomap_dio_simple, bio); sr->iocb = iocb; sr->dio_flags = 0; From 26dfed9c7a289089b76cff9ab9e4c66b6f90e872 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Wed, 29 Jul 2026 12:27:20 -0700 Subject: [PATCH 12/29] iomap: add ->iomap_next() Have one ->iomap_next() callback instead of ->iomap_begin() and ->iomap_end(). ->iomap_next() finishes the previous mapping if needed, and produces the next mapping. Collapsing to a single callback lets a performance-critical caller inline its iteration loop and pass its ->iomap_next() function as a compile-time constant, so the compiler can devirtualize that callback into a direct call instead of an indirect call through a function pointer. iomap_iter() uses ->iomap_next() when the filesystem provides that callback and otherwise falls back to the ->iomap_begin()/->iomap_end() path, so filesystems can be converted one at a time. Suggested-by: Christoph Hellwig Suggested-by: Matthew Wilcox (Oracle) Reviewed-by: "Darrick J. Wong" Reviewed-by: Christoph Hellwig Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260729192737.3190206-6-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- fs/iomap/iter.c | 8 ++++++-- include/linux/iomap.h | 9 +++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/fs/iomap/iter.c b/fs/iomap/iter.c index bf7d4cccc1a7..c445a38b6285 100644 --- a/fs/iomap/iter.c +++ b/fs/iomap/iter.c @@ -118,8 +118,12 @@ int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops) trace_iomap_iter(iter, ops, _RET_IP_); - ret = iomap_iter_next(iter, &iter->iomap, &iter->srcmap, - ops->iomap_begin, ops->iomap_end); + if (ops->iomap_next) + ret = ops->iomap_next(iter, &iter->iomap, &iter->srcmap); + else + ret = iomap_iter_next(iter, &iter->iomap, &iter->srcmap, + ops->iomap_begin, ops->iomap_end); + iter->status = 0; if (ret > 0) iomap_iter_done(iter); diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 80832edc7ec2..d203d9fe0f89 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -231,9 +231,18 @@ typedef int (*iomap_iter_begin_fn)(struct inode *inode, loff_t pos, typedef int (*iomap_iter_end_fn)(struct inode *inode, loff_t pos, loff_t length, ssize_t written, unsigned flags, struct iomap *iomap); +/* + * Produce the next mapping (finishing the previous one if needed). + * Return 1 to continue iterating, 0 if the range is fully consumed, or a + * negative error on failure. + */ +typedef int (*iomap_iter_next_fn)(const struct iomap_iter *iter, + struct iomap *iomap, struct iomap *srcmap); + struct iomap_ops { iomap_iter_begin_fn iomap_begin; iomap_iter_end_fn iomap_end; + iomap_iter_next_fn iomap_next; }; /** From 9418f36456f6488265ac14417b252cd2da8d694e Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Wed, 29 Jul 2026 12:27:21 -0700 Subject: [PATCH 13/29] xfs: convert iomap ops to ->iomap_next() Convert xfs iomap_ops to the new ->iomap_next() callback. Each callback is generated with the DEFINE_IOMAP_ITER_NEXT()/DEFINE_IOMAP_ITER_NEXT_END() macros, which wrap the iomap_iter_next() helper to finish the previous mapping if needed and produce the next one. No functional changes are intended. Reviewed-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260729192737.3190206-7-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- fs/xfs/xfs_file.c | 4 ++-- fs/xfs/xfs_iomap.c | 39 +++++++++++++++++++++++++++++---------- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index b733225d2864..768cabf6250b 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -859,9 +859,9 @@ xfs_file_dio_write_atomic( NULL, 0); /* - * The retry mechanism is based on the ->iomap_begin method returning + * The retry mechanism is based on the ->iomap_next method returning * -ENOPROTOOPT, which would be when the REQ_ATOMIC-based write is not - * possible. The REQ_ATOMIC-based method typically not be possible if + * possible. The REQ_ATOMIC-based method is typically not possible if * the write spans multiple extents or the disk blocks are misaligned. */ if (ret == -ENOPROTOOPT && dops == &xfs_direct_write_iomap_ops) { diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index 0536e2aeddcc..71c45be8c652 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -1037,8 +1037,11 @@ xfs_direct_write_iomap_begin( return error; } +static DEFINE_IOMAP_ITER_NEXT(xfs_direct_write_iomap_next, + xfs_direct_write_iomap_begin); + const struct iomap_ops xfs_direct_write_iomap_ops = { - .iomap_begin = xfs_direct_write_iomap_begin, + .iomap_next = xfs_direct_write_iomap_next, }; #ifdef CONFIG_XFS_RT @@ -1089,8 +1092,11 @@ xfs_zoned_direct_write_iomap_begin( return 0; } +static DEFINE_IOMAP_ITER_NEXT(xfs_zoned_direct_write_iomap_next, + xfs_zoned_direct_write_iomap_begin); + const struct iomap_ops xfs_zoned_direct_write_iomap_ops = { - .iomap_begin = xfs_zoned_direct_write_iomap_begin, + .iomap_next = xfs_zoned_direct_write_iomap_next, }; #endif /* CONFIG_XFS_RT */ @@ -1274,8 +1280,11 @@ xfs_atomic_write_cow_iomap_begin( return error; } +static DEFINE_IOMAP_ITER_NEXT(xfs_atomic_write_cow_iomap_next, + xfs_atomic_write_cow_iomap_begin); + const struct iomap_ops xfs_atomic_write_cow_iomap_ops = { - .iomap_begin = xfs_atomic_write_cow_iomap_begin, + .iomap_next = xfs_atomic_write_cow_iomap_next, }; static int @@ -1298,9 +1307,11 @@ xfs_dax_write_iomap_end( return xfs_reflink_end_cow(ip, pos, written); } +static DEFINE_IOMAP_ITER_NEXT_END(xfs_dax_write_iomap_next, + xfs_direct_write_iomap_begin, xfs_dax_write_iomap_end); + const struct iomap_ops xfs_dax_write_iomap_ops = { - .iomap_begin = xfs_direct_write_iomap_begin, - .iomap_end = xfs_dax_write_iomap_end, + .iomap_next = xfs_dax_write_iomap_next, }; /* @@ -2168,9 +2179,11 @@ xfs_buffered_write_iomap_end( return 0; } +static DEFINE_IOMAP_ITER_NEXT_END(xfs_buffered_write_iomap_next, + xfs_buffered_write_iomap_begin, xfs_buffered_write_iomap_end); + const struct iomap_ops xfs_buffered_write_iomap_ops = { - .iomap_begin = xfs_buffered_write_iomap_begin, - .iomap_end = xfs_buffered_write_iomap_end, + .iomap_next = xfs_buffered_write_iomap_next, }; int @@ -2214,8 +2227,10 @@ xfs_read_iomap_begin( shared ? IOMAP_F_SHARED : 0, seq); } +static DEFINE_IOMAP_ITER_NEXT(xfs_read_iomap_next, xfs_read_iomap_begin); + const struct iomap_ops xfs_read_iomap_ops = { - .iomap_begin = xfs_read_iomap_begin, + .iomap_next = xfs_read_iomap_next, }; static int @@ -2302,8 +2317,10 @@ xfs_seek_iomap_begin( return error; } +static DEFINE_IOMAP_ITER_NEXT(xfs_seek_iomap_next, xfs_seek_iomap_begin); + const struct iomap_ops xfs_seek_iomap_ops = { - .iomap_begin = xfs_seek_iomap_begin, + .iomap_next = xfs_seek_iomap_next, }; static int @@ -2349,8 +2366,10 @@ xfs_xattr_iomap_begin( return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, IOMAP_F_XATTR, seq); } +static DEFINE_IOMAP_ITER_NEXT(xfs_xattr_iomap_next, xfs_xattr_iomap_begin); + const struct iomap_ops xfs_xattr_iomap_ops = { - .iomap_begin = xfs_xattr_iomap_begin, + .iomap_next = xfs_xattr_iomap_next, }; int From ff2ab3146e7c23969bf0c75bde180f737d7e4b51 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Wed, 29 Jul 2026 12:27:22 -0700 Subject: [PATCH 14/29] btrfs: convert iomap ops to ->iomap_next() Convert btrfs iomap_ops to the new ->iomap_next() callback. The callback is generated with the DEFINE_IOMAP_ITER_NEXT_END() macro, which wraps the iomap_iter_next() helper to finish the previous mapping if needed and produce the next one. No functional changes are intended. Acked-by: David Sterba Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260729192737.3190206-8-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- fs/btrfs/direct-io.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/btrfs/direct-io.c b/fs/btrfs/direct-io.c index 460326d34143..d5439b06cdc9 100644 --- a/fs/btrfs/direct-io.c +++ b/fs/btrfs/direct-io.c @@ -798,9 +798,11 @@ static void btrfs_dio_submit_io(const struct iomap_iter *iter, struct bio *bio, btrfs_submit_bbio(bbio, 0); } +static DEFINE_IOMAP_ITER_NEXT_END(btrfs_dio_iomap_next, btrfs_dio_iomap_begin, + btrfs_dio_iomap_end); + static const struct iomap_ops btrfs_dio_iomap_ops = { - .iomap_begin = btrfs_dio_iomap_begin, - .iomap_end = btrfs_dio_iomap_end, + .iomap_next = btrfs_dio_iomap_next, }; static const struct iomap_dio_ops btrfs_dio_ops = { From 9aff0a5221e836e457de2996d4b0fcc850094588 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Wed, 29 Jul 2026 12:27:23 -0700 Subject: [PATCH 15/29] ntfs3: convert iomap ops to ->iomap_next() Convert ntfs3 iomap_ops to the new ->iomap_next() callback. The callback is generated with the DEFINE_IOMAP_ITER_NEXT_END() macro, which wraps the iomap_iter_next() helper to finish the previous mapping if needed and produce the next one. No functional changes are intended. Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260729192737.3190206-9-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- fs/ntfs3/inode.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c index c43101cc064d..53031e71c8fc 100644 --- a/fs/ntfs3/inode.c +++ b/fs/ntfs3/inode.c @@ -2101,9 +2101,11 @@ const struct address_space_operations ntfs_aops_cmpr = { .invalidate_folio = iomap_invalidate_folio, }; +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_iomap_next, ntfs_iomap_begin, + ntfs_iomap_end); + const struct iomap_ops ntfs_iomap_ops = { - .iomap_begin = ntfs_iomap_begin, - .iomap_end = ntfs_iomap_end, + .iomap_next = ntfs_iomap_next, }; const struct iomap_write_ops ntfs_iomap_folio_ops = { From 7a7bf7551624f8e7e1877f1b6ca088499b5ad25e Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Wed, 29 Jul 2026 12:27:24 -0700 Subject: [PATCH 16/29] ntfs: convert iomap ops to ->iomap_next() Convert ntfs iomap_ops to the new ->iomap_next() callback. Each callback is generated with the DEFINE_IOMAP_ITER_NEXT()/DEFINE_IOMAP_ITER_NEXT_END() macros, which wrap the iomap_iter_next() helper to finish the previous mapping if needed and produce the next one. No functional changes are intended. Acked-by: Namjae Jeon Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260729192737.3190206-10-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- fs/ntfs/iomap.c | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/fs/ntfs/iomap.c b/fs/ntfs/iomap.c index 52eecf5cb256..d0964ac840d9 100644 --- a/fs/ntfs/iomap.c +++ b/fs/ntfs/iomap.c @@ -277,8 +277,10 @@ static int ntfs_read_iomap_begin(struct inode *inode, loff_t offset, loff_t leng srcmap, true); } +static DEFINE_IOMAP_ITER_NEXT(ntfs_read_iomap_next, ntfs_read_iomap_begin); + const struct iomap_ops ntfs_read_iomap_ops = { - .iomap_begin = ntfs_read_iomap_begin, + .iomap_next = ntfs_read_iomap_next, }; /* @@ -329,13 +331,17 @@ static int ntfs_zero_read_iomap_end(struct inode *inode, loff_t pos, loff_t leng return written; } +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_zero_read_iomap_next, + ntfs_seek_iomap_begin, ntfs_zero_read_iomap_end); + static const struct iomap_ops ntfs_zero_read_iomap_ops = { - .iomap_begin = ntfs_seek_iomap_begin, - .iomap_end = ntfs_zero_read_iomap_end, + .iomap_next = ntfs_zero_read_iomap_next, }; +static DEFINE_IOMAP_ITER_NEXT(ntfs_seek_iomap_next, ntfs_seek_iomap_begin); + const struct iomap_ops ntfs_seek_iomap_ops = { - .iomap_begin = ntfs_seek_iomap_begin, + .iomap_next = ntfs_seek_iomap_next, }; int ntfs_dio_zero_range(struct inode *inode, loff_t offset, loff_t length) @@ -764,9 +770,11 @@ static int ntfs_write_iomap_end(struct inode *inode, loff_t pos, loff_t length, return written; } +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_write_iomap_next, + ntfs_write_iomap_begin, ntfs_write_iomap_end); + const struct iomap_ops ntfs_write_iomap_ops = { - .iomap_begin = ntfs_write_iomap_begin, - .iomap_end = ntfs_write_iomap_end, + .iomap_next = ntfs_write_iomap_next, }; static int ntfs_page_mkwrite_iomap_begin(struct inode *inode, loff_t offset, @@ -777,9 +785,11 @@ static int ntfs_page_mkwrite_iomap_begin(struct inode *inode, loff_t offset, NTFS_IOMAP_FLAGS_MKWRITE); } +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_page_mkwrite_iomap_next, + ntfs_page_mkwrite_iomap_begin, ntfs_write_iomap_end); + const struct iomap_ops ntfs_page_mkwrite_iomap_ops = { - .iomap_begin = ntfs_page_mkwrite_iomap_begin, - .iomap_end = ntfs_write_iomap_end, + .iomap_next = ntfs_page_mkwrite_iomap_next, }; static int ntfs_dio_iomap_begin(struct inode *inode, loff_t offset, @@ -790,9 +800,11 @@ static int ntfs_dio_iomap_begin(struct inode *inode, loff_t offset, NTFS_IOMAP_FLAGS_DIO); } +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_dio_iomap_next, + ntfs_dio_iomap_begin, ntfs_write_iomap_end); + const struct iomap_ops ntfs_dio_iomap_ops = { - .iomap_begin = ntfs_dio_iomap_begin, - .iomap_end = ntfs_write_iomap_end, + .iomap_next = ntfs_dio_iomap_next, }; static ssize_t ntfs_writeback_range(struct iomap_writepage_ctx *wpc, From aa35a8a03acd8f207531d7e677f39cec2f8b19b4 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Wed, 29 Jul 2026 12:27:25 -0700 Subject: [PATCH 17/29] ext4: convert iomap ops to ->iomap_next() Convert ext4 iomap_ops to the new ->iomap_next() callback. Each callback is generated with the DEFINE_IOMAP_ITER_NEXT() macro, which wraps the iomap_iter_next() helper to finish the previous mapping if needed and produce the next one. No functional changes are intended. Reviewed-by: Jan Kara Reviewed-by: Baokun Li Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260729192737.3190206-11-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- fs/ext4/extents.c | 4 +++- fs/ext4/inode.c | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index 91c97af64b31..15972410d460 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -5171,8 +5171,10 @@ static int ext4_iomap_xattr_begin(struct inode *inode, loff_t offset, return error; } +static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_xattr_next, ext4_iomap_xattr_begin); + static const struct iomap_ops ext4_iomap_xattr_ops = { - .iomap_begin = ext4_iomap_xattr_begin, + .iomap_next = ext4_iomap_xattr_next, }; static int ext4_fiemap_check_ranges(struct inode *inode, u64 start, u64 *len) diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index b48c54f3312b..bf9755b541be 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -3850,8 +3850,10 @@ int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length, return 0; } +static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next, ext4_iomap_begin); + const struct iomap_ops ext4_iomap_ops = { - .iomap_begin = ext4_iomap_begin, + .iomap_next = ext4_iomap_next, }; static int ext4_iomap_begin_report(struct inode *inode, loff_t offset, @@ -3905,8 +3907,10 @@ static int ext4_iomap_begin_report(struct inode *inode, loff_t offset, return 0; } +static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next_report, ext4_iomap_begin_report); + const struct iomap_ops ext4_iomap_report_ops = { - .iomap_begin = ext4_iomap_begin_report, + .iomap_next = ext4_iomap_next_report, }; /* From 32055631cc5e3a8e86b27e0738c62aa6385321b8 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Wed, 29 Jul 2026 12:27:26 -0700 Subject: [PATCH 18/29] erofs: convert iomap ops to ->iomap_next() Convert erofs iomap_ops to the new ->iomap_next() callback. Each callback is generated with the DEFINE_IOMAP_ITER_NEXT() and DEFINE_IOMAP_ITER_NEXT_END() macros, which wrap the iomap_iter_next() helper to finish the previous mapping if needed and produce the next one. No functional changes are intended. Reviewed-by: Gao Xiang Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260729192737.3190206-12-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- fs/erofs/data.c | 6 ++++-- fs/erofs/zmap.c | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/fs/erofs/data.c b/fs/erofs/data.c index 9aa48c8d67d1..d2f01245ee79 100644 --- a/fs/erofs/data.c +++ b/fs/erofs/data.c @@ -380,9 +380,11 @@ static int erofs_iomap_end(struct inode *inode, loff_t pos, loff_t length, return written; } +static DEFINE_IOMAP_ITER_NEXT_END(erofs_iomap_next, erofs_iomap_begin, + erofs_iomap_end); + static const struct iomap_ops erofs_iomap_ops = { - .iomap_begin = erofs_iomap_begin, - .iomap_end = erofs_iomap_end, + .iomap_next = erofs_iomap_next, }; int erofs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c index bab521613552..3966b62a7051 100644 --- a/fs/erofs/zmap.c +++ b/fs/erofs/zmap.c @@ -821,6 +821,9 @@ static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset, return 0; } +static DEFINE_IOMAP_ITER_NEXT(z_erofs_iomap_next_report, + z_erofs_iomap_begin_report); + const struct iomap_ops z_erofs_iomap_report_ops = { - .iomap_begin = z_erofs_iomap_begin_report, + .iomap_next = z_erofs_iomap_next_report, }; From ef56723067bc17bdcdfb13d5f6fa6ca9f06e0d9c Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Wed, 29 Jul 2026 12:27:27 -0700 Subject: [PATCH 19/29] zonefs: convert iomap ops to ->iomap_next() Convert zonefs iomap_ops to the new ->iomap_next() callback. Each callback is generated with the DEFINE_IOMAP_ITER_NEXT() macro, which wraps the iomap_iter_next() helper to finish the previous mapping if needed and produce the next one. No functional changes are intended. Acked-by: Damien Le Moal Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260729192737.3190206-13-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- fs/zonefs/file.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/fs/zonefs/file.c b/fs/zonefs/file.c index 5ada33f70bb4..5b34849be7a2 100644 --- a/fs/zonefs/file.c +++ b/fs/zonefs/file.c @@ -57,8 +57,10 @@ static int zonefs_read_iomap_begin(struct inode *inode, loff_t offset, return 0; } +static DEFINE_IOMAP_ITER_NEXT(zonefs_read_iomap_next, zonefs_read_iomap_begin); + static const struct iomap_ops zonefs_read_iomap_ops = { - .iomap_begin = zonefs_read_iomap_begin, + .iomap_next = zonefs_read_iomap_next, }; static int zonefs_write_iomap_begin(struct inode *inode, loff_t offset, @@ -106,8 +108,11 @@ static int zonefs_write_iomap_begin(struct inode *inode, loff_t offset, return 0; } +static DEFINE_IOMAP_ITER_NEXT(zonefs_write_iomap_next, + zonefs_write_iomap_begin); + static const struct iomap_ops zonefs_write_iomap_ops = { - .iomap_begin = zonefs_write_iomap_begin, + .iomap_next = zonefs_write_iomap_next, }; static int zonefs_read_folio(struct file *unused, struct folio *folio) From e23b789f9faaf1ff2e988c5c9520b91998f752c7 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Wed, 29 Jul 2026 12:27:28 -0700 Subject: [PATCH 20/29] ext2: convert iomap ops to ->iomap_next() Convert ext2 iomap_ops to the new ->iomap_next() callback. The callback is generated with the DEFINE_IOMAP_ITER_NEXT_END() macro, which wraps the iomap_iter_next() helper to finish the previous mapping if needed and produce the next one. No functional changes are intended. Reviewed-by: Jan Kara Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260729192737.3190206-14-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- fs/ext2/inode.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c index 29808629cce5..7e0fa9c454e1 100644 --- a/fs/ext2/inode.c +++ b/fs/ext2/inode.c @@ -860,9 +860,11 @@ ext2_iomap_end(struct inode *inode, loff_t offset, loff_t length, return 0; } +static DEFINE_IOMAP_ITER_NEXT_END(ext2_iomap_next, ext2_iomap_begin, + ext2_iomap_end); + const struct iomap_ops ext2_iomap_ops = { - .iomap_begin = ext2_iomap_begin, - .iomap_end = ext2_iomap_end, + .iomap_next = ext2_iomap_next, }; int ext2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, From d3f0fcc22e93f3e3756f98d07aef37aed64d4940 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Wed, 29 Jul 2026 12:27:29 -0700 Subject: [PATCH 21/29] block: convert iomap ops to ->iomap_next() Convert block iomap_ops to the new ->iomap_next() callback. The callback is generated with the DEFINE_IOMAP_ITER_NEXT() macro, which wraps the iomap_iter_next() helper to finish the previous mapping if needed and produce the next one. No functional changes are intended. Reviewed-by: Christoph Hellwig Reviewed-by: Keith Busch Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260729192737.3190206-15-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- block/fops.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/block/fops.c b/block/fops.c index 15783a6180de..4cff76e9eb71 100644 --- a/block/fops.c +++ b/block/fops.c @@ -453,8 +453,10 @@ static int blkdev_iomap_begin(struct inode *inode, loff_t offset, loff_t length, return 0; } +static DEFINE_IOMAP_ITER_NEXT(blkdev_iomap_next, blkdev_iomap_begin); + static const struct iomap_ops blkdev_iomap_ops = { - .iomap_begin = blkdev_iomap_begin, + .iomap_next = blkdev_iomap_next, }; #ifdef CONFIG_BUFFER_HEAD From 8808f09e95c1184df5502b09e761c135accd3921 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Wed, 29 Jul 2026 12:27:30 -0700 Subject: [PATCH 22/29] f2fs: convert iomap ops to ->iomap_next() Convert f2fs iomap_ops to the new ->iomap_next() callback. The callback is generated with the DEFINE_IOMAP_ITER_NEXT() macro, which wraps the iomap_iter_next() helper to finish the previous mapping if needed and produce the next one. No functional changes are intended. Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260729192737.3190206-16-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- fs/f2fs/data.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index a765fda71536..8977ad379f50 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -4653,6 +4653,8 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, return 0; } +static DEFINE_IOMAP_ITER_NEXT(f2fs_iomap_next, f2fs_iomap_begin); + const struct iomap_ops f2fs_iomap_ops = { - .iomap_begin = f2fs_iomap_begin, + .iomap_next = f2fs_iomap_next, }; From f3f2f10d8cc5d0a3630020641b79f47ba44463c8 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Wed, 29 Jul 2026 12:27:31 -0700 Subject: [PATCH 23/29] gfs2: convert iomap ops to ->iomap_next() Convert gfs2 iomap_ops to the new ->iomap_next() callback. The callback is generated with the DEFINE_IOMAP_ITER_NEXT_END() macro, which wraps the iomap_iter_next() helper to finish the previous mapping if needed and produce the next one. No functional changes are intended. Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260729192737.3190206-17-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- fs/gfs2/bmap.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/gfs2/bmap.c b/fs/gfs2/bmap.c index 51ac1fd44f78..73c626971163 100644 --- a/fs/gfs2/bmap.c +++ b/fs/gfs2/bmap.c @@ -1200,9 +1200,11 @@ static int gfs2_iomap_end(struct inode *inode, loff_t pos, loff_t length, return 0; } +static DEFINE_IOMAP_ITER_NEXT_END(gfs2_iomap_next, gfs2_iomap_begin, + gfs2_iomap_end); + const struct iomap_ops gfs2_iomap_ops = { - .iomap_begin = gfs2_iomap_begin, - .iomap_end = gfs2_iomap_end, + .iomap_next = gfs2_iomap_next, }; /** From 20adeec6e3816beb7a25dccfeaa41acfbf67ce85 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Wed, 29 Jul 2026 12:27:32 -0700 Subject: [PATCH 24/29] hpfs: convert iomap ops to ->iomap_next() Convert hpfs iomap_ops to the new ->iomap_next() callback. The callback is generated with the DEFINE_IOMAP_ITER_NEXT() macro, which wraps the iomap_iter_next() helper to finish the previous mapping if needed and produce the next one. No functional changes are intended. Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260729192737.3190206-18-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- fs/hpfs/file.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/hpfs/file.c b/fs/hpfs/file.c index 29e876705369..6a629ab956fc 100644 --- a/fs/hpfs/file.c +++ b/fs/hpfs/file.c @@ -156,8 +156,10 @@ static int hpfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, return 0; } +static DEFINE_IOMAP_ITER_NEXT(hpfs_iomap_next, hpfs_iomap_begin); + static const struct iomap_ops hpfs_iomap_ops = { - .iomap_begin = hpfs_iomap_begin, + .iomap_next = hpfs_iomap_next, }; static int hpfs_read_folio(struct file *file, struct folio *folio) From 2fe3d401066ae3c22eadba38db1e2a99bd3bb4af Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Wed, 29 Jul 2026 12:27:33 -0700 Subject: [PATCH 25/29] fuse: convert iomap ops to ->iomap_next() Convert fuse iomap_ops to the new ->iomap_next() callback. Each callback is generated with the DEFINE_IOMAP_ITER_NEXT()/DEFINE_IOMAP_ITER_NEXT_END() macros, which wrap the iomap_iter_next() helper to finish the previous mapping if needed and produce the next one. No functional changes are intended. Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260729192737.3190206-19-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- fs/fuse/dax.c | 6 ++++-- fs/fuse/file.c | 4 +++- fs/fuse/virtio_fs.c | 3 +-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/fs/fuse/dax.c b/fs/fuse/dax.c index 8b53625ac7ab..85cdf0199bc0 100644 --- a/fs/fuse/dax.c +++ b/fs/fuse/dax.c @@ -653,9 +653,11 @@ static int fuse_iomap_end(struct inode *inode, loff_t pos, loff_t length, return 0; } +static DEFINE_IOMAP_ITER_NEXT_END(fuse_iomap_next, fuse_iomap_begin, + fuse_iomap_end); + static const struct iomap_ops fuse_iomap_ops = { - .iomap_begin = fuse_iomap_begin, - .iomap_end = fuse_iomap_end, + .iomap_next = fuse_iomap_next, }; static void fuse_wait_dax_page(struct inode *inode) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index ea4a15a7635a..1c346802f877 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -890,8 +890,10 @@ static int fuse_iomap_begin(struct inode *inode, loff_t offset, loff_t length, return 0; } +static DEFINE_IOMAP_ITER_NEXT(fuse_iomap_next, fuse_iomap_begin); + static const struct iomap_ops fuse_iomap_ops = { - .iomap_begin = fuse_iomap_begin, + .iomap_next = fuse_iomap_next, }; struct fuse_fill_read_data { diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c index df25d4faca41..f15e516ebcb5 100644 --- a/fs/fuse/virtio_fs.c +++ b/fs/fuse/virtio_fs.c @@ -1024,8 +1024,7 @@ static void virtio_fs_cleanup_vqs(struct virtio_device *vdev) } /* Map a window offset to a page frame number. The window offset will have - * been produced by .iomap_begin(), which maps a file offset to a window - * offset. + * been produced by .iomap_next(), which maps a file offset to a window offset. */ static long virtio_fs_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages, enum dax_access_mode mode, From d14541b3d8acbf0587e502c9dbad649f84598094 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Wed, 29 Jul 2026 12:27:34 -0700 Subject: [PATCH 26/29] exfat: convert iomap ops to ->iomap_next() Convert exfat iomap_ops to the new ->iomap_next() callback. Each callback is generated with the DEFINE_IOMAP_ITER_NEXT() and DEFINE_IOMAP_ITER_NEXT_END() macros, which wrap the iomap_iter_next() helper to finish the previous mapping if needed and produce the next one. No functional changes are intended. Acked-by: Namjae Jeon Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260729192737.3190206-20-joannelkoong@gmail.com Signed-off-by: Christian Brauner (Amutable) --- fs/exfat/iomap.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/fs/exfat/iomap.c b/fs/exfat/iomap.c index 1aac38e63fe6..b6dd396aa60a 100644 --- a/fs/exfat/iomap.c +++ b/fs/exfat/iomap.c @@ -151,8 +151,10 @@ static int exfat_write_iomap_begin(struct inode *inode, loff_t offset, loff_t le return __exfat_iomap_begin(inode, offset, length, flags, iomap, true); } +static DEFINE_IOMAP_ITER_NEXT(exfat_iomap_next, exfat_iomap_begin); + const struct iomap_ops exfat_iomap_ops = { - .iomap_begin = exfat_iomap_begin, + .iomap_next = exfat_iomap_next, }; /* @@ -186,9 +188,11 @@ static int exfat_write_iomap_end(struct inode *inode, loff_t pos, loff_t length, return written; } +static DEFINE_IOMAP_ITER_NEXT_END(exfat_write_iomap_next, + exfat_write_iomap_begin, exfat_write_iomap_end); + const struct iomap_ops exfat_write_iomap_ops = { - .iomap_begin = exfat_write_iomap_begin, - .iomap_end = exfat_write_iomap_end, + .iomap_next = exfat_write_iomap_next, }; /* From 7c7fe554f40adac2a7f8d9b6d5886ca332edd496 Mon Sep 17 00:00:00 2001 From: Benjamin Wu Date: Sun, 9 Aug 2026 23:36:31 -0700 Subject: [PATCH 27/29] docs: fix grammatical error in iomap docs Change "This origins" -> "The origins" Signed-off-by: Benjamin Wu Link: https://patch.msgid.link/20260810063704.355933-1-benjamin.wu37@gmail.com Signed-off-by: Christian Brauner (Amutable) --- Documentation/filesystems/iomap/design.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/filesystems/iomap/design.rst b/Documentation/filesystems/iomap/design.rst index 0f7672676c0b..681e0cbb6944 100644 --- a/Documentation/filesystems/iomap/design.rst +++ b/Documentation/filesystems/iomap/design.rst @@ -43,7 +43,7 @@ as: * lseek ``SEEK_DATA`` and ``SEEK_HOLE`` * swapfile activation -This origins of this library is the file I/O path that XFS once used; it +The origins of this library is the file I/O path that XFS once used; it has now been extended to cover several other operations. Who Should Read This? From 8a8685b32c0718cc7b2cb4d6202e5a5b8e0a8e2d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 4 Aug 2026 05:43:57 -0700 Subject: [PATCH 28/29] iomap: don't free integrity payload that doesn't exist fs_bio_integrity_alloc might not allocate a bio integrity payload if PI verification is disabled on the block device. Check for that case before calling fs_bio_integrity_free in iomap_bio_read_folio_range_sync to avoid a NULL pointer dereferences. Make the branch cover the PI verification as well - while fs_bio_integrity_verify works without an integrity payload, it requires one to actually do useful work. Fixes: 0b10a370529c ("iomap: support T10 protection information") Cc: stable@vger.kernel.org # v7.1 Signed-off-by: Christoph Hellwig Reviewed-by: Anuj Gupta Reviewed-by: Kanchan Joshi Reviewed-by: "Darrick J. Wong" Link: https://patch.msgid.link/20260804124404.737145-2-hch@lst.de Signed-off-by: Christian Brauner (Amutable) --- fs/iomap/bio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/iomap/bio.c b/fs/iomap/bio.c index 4504f4633f17..311199344b28 100644 --- a/fs/iomap/bio.c +++ b/fs/iomap/bio.c @@ -170,7 +170,7 @@ int iomap_bio_read_folio_range_sync(const struct iomap_iter *iter, if (srcmap->flags & IOMAP_F_INTEGRITY) fs_bio_integrity_alloc(&bio); error = submit_bio_wait(&bio); - if (srcmap->flags & IOMAP_F_INTEGRITY) { + if (bio_integrity(&bio)) { if (!error) error = fs_bio_integrity_verify(&bio, sector, len); fs_bio_integrity_free(&bio); From accb6624e383872e7703974a5d5826755fb27032 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 4 Aug 2026 05:43:58 -0700 Subject: [PATCH 29/29] iomap: iomap_bio_read_folio_range_sync is missing a call to bio_uninit Which could leak blkg references. Fixes: c03cea42149d ("iomap: add initial support for writes without buffer heads") Signed-off-by: Christoph Hellwig Link: https://patch.msgid.link/20260804124404.737145-3-hch@lst.de Reviewed-by: "Darrick J. Wong" Reviewed-by: Anuj Gupta Signed-off-by: Christian Brauner (Amutable) --- fs/iomap/bio.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/iomap/bio.c b/fs/iomap/bio.c index 311199344b28..24969b71a965 100644 --- a/fs/iomap/bio.c +++ b/fs/iomap/bio.c @@ -175,5 +175,6 @@ int iomap_bio_read_folio_range_sync(const struct iomap_iter *iter, error = fs_bio_integrity_verify(&bio, sector, len); fs_bio_integrity_free(&bio); } + bio_uninit(&bio); return error; }