From 6648f54f3459c5d069b7ce294170d721ce9bac2f Mon Sep 17 00:00:00 2001 From: Miklos Szeredi Date: Mon, 6 Jul 2026 15:44:58 +0200 Subject: [PATCH 01/35] fuse: move "epoch" from dentry.d_time to fuse_dentry.epoch ...in hope of removing d_time one day. Fixes: 2396356a945b ("fuse: add more control over cache invalidation behaviour") Signed-off-by: Miklos Szeredi --- fs/fuse/dir.c | 23 ++++++++++++++++------- fs/fuse/fuse_i.h | 2 ++ fs/fuse/readdir.c | 2 +- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 0e2a1039fa43..b689503bc880 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -96,6 +96,7 @@ static void fuse_advise_use_readdirplus(struct inode *dir) struct fuse_dentry { u64 time; + u64 epoch; union { struct rcu_head rcu; struct rb_node node; @@ -236,6 +237,13 @@ void fuse_dentry_tree_cleanup(void) WARN_ON_ONCE(!RB_EMPTY_ROOT(&dentry_hash[i].tree)); } +void fuse_dentry_set_epoch(struct dentry *dentry, u64 epoch) +{ + struct fuse_dentry *fd = dentry->d_fsdata; + + fd->epoch = epoch; +} + static inline void __fuse_dentry_settime(struct dentry *dentry, u64 time) { ((struct fuse_dentry *) dentry->d_fsdata)->time = time; @@ -387,10 +395,11 @@ static int fuse_dentry_revalidate(struct inode *dir, const struct qstr *name, struct fuse_mount *fm; struct fuse_conn *fc; struct fuse_inode *fi; + struct fuse_dentry *fd = entry->d_fsdata; int ret; fc = get_fuse_conn_super(dir->i_sb); - if (entry->d_time < atomic_read(&fc->epoch)) + if (fd->epoch < atomic_read(&fc->epoch)) goto invalid; inode = d_inode_rcu(entry); @@ -480,10 +489,10 @@ static int fuse_dentry_init(struct dentry *dentry) RB_CLEAR_NODE(&fd->node); dentry->d_fsdata = fd; /* - * Initialising d_time (epoch) to '0' ensures the dentry is invalid + * Initialising epoch to '0' ensures the dentry is invalid * if compared to fc->epoch, which is initialized to '1'. */ - dentry->d_time = 0; + fuse_dentry_set_epoch(dentry, 0); return 0; } @@ -641,7 +650,7 @@ static struct dentry *fuse_lookup(struct inode *dir, struct dentry *entry, goto out_err; entry = newent ? newent : entry; - entry->d_time = epoch; + fuse_dentry_set_epoch(entry, epoch); if (outarg_valid) fuse_change_entry_timeout(entry, &outarg); else @@ -898,7 +907,7 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, } kfree(forget); d_instantiate(entry, inode); - entry->d_time = epoch; + fuse_dentry_set_epoch(entry, epoch); fuse_change_entry_timeout(entry, &outentry); fuse_dir_changed(dir); err = generic_file_open(inode, file); @@ -1028,10 +1037,10 @@ static struct dentry *create_new_entry(struct mnt_idmap *idmap, struct fuse_moun return d; if (d) { - d->d_time = epoch; + fuse_dentry_set_epoch(d, epoch); fuse_change_entry_timeout(d, &outarg); } else { - entry->d_time = epoch; + fuse_dentry_set_epoch(entry, epoch); fuse_change_entry_timeout(entry, &outarg); } fuse_dir_changed(dir); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 85f738c53122..c8d4c5f3af7e 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -1054,6 +1054,8 @@ u64 fuse_time_to_jiffies(u64 sec, u32 nsec); void fuse_change_entry_timeout(struct dentry *entry, struct fuse_entry_out *o); +void fuse_dentry_set_epoch(struct dentry *dentry, u64 epoch); + /* * Initialize fuse_conn */ diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index 0e1321491747..5ca87151d70d 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -260,7 +260,7 @@ static int fuse_direntplus_link(struct file *file, } if (fc->readdirplus_auto) set_bit(FUSE_I_INIT_RDPLUS, &get_fuse_inode(inode)->state); - dentry->d_time = epoch; + fuse_dentry_set_epoch(dentry, epoch); fuse_change_entry_timeout(dentry, o); dput(dentry); From cc6f804e785f9a7c04333cb55372723da482ae3f Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Tue, 7 Jul 2026 15:04:48 -0700 Subject: [PATCH 02/35] fuse: don't clear folio uptodate on writethrough errors In the writethrough path (fuse_send_write_pages()), if the write to the server failed or was a short write, the uptodate flag on the folios are cleared. As explained by Matthew in [1], this is dangerous because the folio may be mapped into userspace. The mm code has the invariant that a non-uptodate folio must never be visible to userspace (to avoid potentially leaking confidental information to userspace) and has checks in place for this that if violated can bring down the whole machine. Practically speaking, the effect of this change for the fuse writethrough error path is that if an application does a write and then the server fails to persist the data or only services a short write, the page cache folio keeps the data the application wrote instead of being reverted to the server's contents on the next read. The failure is still reported to the application synchronously through the short count / error return of the write() syscall. Folios that were only partially written are unaffected since they were never marked uptodate in the first place (fuse_fill_write_page() only marks a folio as uptodate if the whole folio was written to). [1] https://lore.kernel.org/linux-fsdevel/ajtPMgO65FA1TXhi@casper.infradead.org/ Suggested-by: Matthew Wilcox Reviewed-by: Darrick J. Wong Signed-off-by: Joanne Koong Reviewed-by: Christoph Hellwig Signed-off-by: Miklos Szeredi --- fs/fuse/file.c | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index ceada75310b8..f5bcbfa8b6ae 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1219,8 +1219,7 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, struct file *file = iocb->ki_filp; struct fuse_file *ff = file->private_data; struct fuse_mount *fm = ff->fm; - unsigned int offset, i; - bool short_write; + unsigned int i; int err; for (i = 0; i < ap->num_folios; i++) @@ -1235,24 +1234,9 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, if (!err && ia->write.out.size > count) err = -EIO; - short_write = ia->write.out.size < count; - offset = ap->descs[0].offset; - count = ia->write.out.size; for (i = 0; i < ap->num_folios; i++) { struct folio *folio = ap->folios[i]; - if (err) { - folio_clear_uptodate(folio); - } else { - if (count >= folio_size(folio) - offset) - count -= folio_size(folio) - offset; - else { - if (short_write) - folio_clear_uptodate(folio); - count = 0; - } - offset = 0; - } if (ia->write.folio_locked && (i == ap->num_folios - 1)) folio_unlock(folio); folio_put(folio); From 03e1dd35c206f24b6bc987198ac58e5138176cb7 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Tue, 7 Jul 2026 15:04:49 -0700 Subject: [PATCH 03/35] iomap: add helper to mark folio uptodate Add an exported helper iomap_folio_mark_uptodate() to mark a folio as uptodate and update its uptodate bitmap if the folio has iomap state data attached. This is needed because there are some filesystems (eg fuse) that have paths outside of conventional iomap calls that need to mark a folio as uptodate (eg writing server-pushed data directly into the page cache) and need the iomap-internal uptodate bitmap to be in sync with the uptodate state of the folio. Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/iomap/buffered-io.c | 6 ++++++ include/linux/iomap.h | 1 + 2 files changed, 7 insertions(+) diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index 276720bc18dc..af415293e265 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -105,6 +105,12 @@ static void iomap_set_range_uptodate(struct folio *folio, size_t off, folio_mark_uptodate(folio); } +void iomap_folio_mark_uptodate(struct folio *folio) +{ + iomap_set_range_uptodate(folio, 0, folio_size(folio)); +} +EXPORT_SYMBOL_GPL(iomap_folio_mark_uptodate); + /* * Find the next dirty block in the folio. end_blk is inclusive. * If no dirty block is found, this will return end_blk + 1. diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 56b43d594e6e..40aa3476a351 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -365,6 +365,7 @@ struct folio *iomap_get_folio(struct iomap_iter *iter, loff_t pos, size_t len); bool iomap_release_folio(struct folio *folio, gfp_t gfp_flags); void iomap_invalidate_folio(struct folio *folio, size_t offset, size_t len); bool iomap_dirty_folio(struct address_space *mapping, struct folio *folio); +void iomap_folio_mark_uptodate(struct folio *folio); int iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len, const struct iomap_ops *ops, const struct iomap_write_ops *write_ops); From 16f4be93c65ab3fc9c4c831849722581f8f3ca14 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Tue, 7 Jul 2026 15:04:50 -0700 Subject: [PATCH 04/35] fuse: use iomap helper to mark folio uptodate When fuse enables large folios, a large folio will be backed by iomap_folio_state that keeps track of uptodate and dirty state in an internal bitmap. Fuse writethrough and notify store paths currently set folio uptodate state with folio_mark_uptodate(), which touches only the folio-level flag, but on an iomap-backed folio, that leaves the uptodate bitmap out of sync. Use the iomap_folio_mark_uptodate() helper to update both the folio uptodate state and the iomap uptodate bitmap. Reviewed-by: Darrick J. Wong Signed-off-by: Joanne Koong Reviewed-by: Christoph Hellwig Signed-off-by: Miklos Szeredi --- fs/fuse/file.c | 2 +- fs/fuse/notify.c | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index f5bcbfa8b6ae..da5859e8159d 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1311,7 +1311,7 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, /* If we copied full folio, mark it uptodate */ if (tmp == folio_size(folio)) - folio_mark_uptodate(folio); + iomap_folio_mark_uptodate(folio); if (folio_test_uptodate(folio)) { folio_unlock(folio); diff --git a/fs/fuse/notify.c b/fs/fuse/notify.c index 29578104ae6c..1ba763705d91 100644 --- a/fs/fuse/notify.c +++ b/fs/fuse/notify.c @@ -2,6 +2,8 @@ #include "dev.h" #include "fuse_i.h" + +#include #include static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size, @@ -192,7 +194,7 @@ static int fuse_notify_store(struct fuse_conn *fc, unsigned int size, if (!folio_test_uptodate(folio) && !err && folio_offset == 0 && (nr_bytes == folio_size(folio) || file_size == end)) { folio_zero_segment(folio, nr_bytes, folio_size(folio)); - folio_mark_uptodate(folio); + iomap_folio_mark_uptodate(folio); } folio_unlock(folio); folio_put(folio); From ed9c881f3b498383f73c42712b359419da42a7b0 Mon Sep 17 00:00:00 2001 From: Miklos Szeredi Date: Thu, 9 Jul 2026 08:37:05 +0200 Subject: [PATCH 05/35] fuse: fix race between interrupt and resend After commit f8fce75fedf7 ("fuse: clear intr_entry in fuse_resend and fuse_remove_pending_req") the WARN_ON(!list_empty(&req->intr_entry)) in fuse_request_free() still triggers due to the following race: In request_wait_answer() if (test_bit(FR_SENT, &req->flags)) -> returns true In fuse_chan_resend() clear_bit(FR_SENT, &req->flags) In request_wait_answer() queue_interrupt(req) Fix by: - move clearing FR_SENT inside fpq->lock - move setting FR_PENDING inside fiq->lock - recheck FR_SENT after acquiring fiq->lock in fuse_dev_queue_interrupt() Reported-by: zdi-disclosures@trendmicro.com Fixes: f8fce75fedf7 ("fuse: clear intr_entry in fuse_resend and fuse_remove_pending_req") Cc: stable@vger.kernel.org # 6.9 Signed-off-by: Miklos Szeredi --- fs/fuse/dev.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 5763a7cd3b37..7e81dea4f1c7 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -240,7 +240,8 @@ void fuse_dev_queue_forget(struct fuse_iqueue *fiq, void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req) { spin_lock(&fiq->lock); - if (list_empty(&req->intr_entry)) { + /* Repeat FR_SENT test after obtaining the lock to prevent race with fuse_resend() */ + if (list_empty(&req->intr_entry) && test_bit(FR_SENT, &req->flags)) { list_add_tail(&req->intr_entry, &fiq->interrupts); /* * Pairs with smp_mb() implied by test_and_set_bit() @@ -1760,7 +1761,7 @@ static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos, void fuse_chan_resend(struct fuse_chan *fch) { struct fuse_dev *fud; - struct fuse_req *req, *next; + struct fuse_req *req; struct fuse_iqueue *fiq = &fch->iq; LIST_HEAD(to_queue); unsigned int i; @@ -1775,24 +1776,20 @@ void fuse_chan_resend(struct fuse_chan *fch) struct fuse_pqueue *fpq = &fud->pq; spin_lock(&fpq->lock); - for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) - list_splice_tail_init(&fpq->processing[i], &to_queue); + for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) { + struct list_head *this_queue = &fpq->processing[i]; + + list_for_each_entry(req, this_queue, list) + clear_bit(FR_SENT, &req->flags); + list_splice_tail_init(this_queue, &to_queue); + } spin_unlock(&fpq->lock); } spin_unlock(&fch->lock); - list_for_each_entry_safe(req, next, &to_queue, list) { - set_bit(FR_PENDING, &req->flags); - clear_bit(FR_SENT, &req->flags); - /* mark the request as resend request */ - req->in.h.unique |= FUSE_UNIQUE_RESEND; - } - spin_lock(&fiq->lock); if (!fiq->connected) { spin_unlock(&fiq->lock); - list_for_each_entry(req, &to_queue, list) - clear_bit(FR_PENDING, &req->flags); fuse_dev_end_requests(&to_queue); return; } @@ -1801,6 +1798,10 @@ void fuse_chan_resend(struct fuse_chan *fch) * intr_entry on fiq->interrupts after the request is re-queued. */ list_for_each_entry(req, &to_queue, list) { + set_bit(FR_PENDING, &req->flags); + /* mark the request as resend request */ + req->in.h.unique |= FUSE_UNIQUE_RESEND; + if (test_bit(FR_INTERRUPTED, &req->flags)) list_del_init(&req->intr_entry); } From edb310bc27f0ad83e7fd558a3caf1a94ca511654 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Thu, 16 Jul 2026 11:31:42 -0700 Subject: [PATCH 06/35] fuse: fix missing barrier when checking io-uring readiness fuse_block_alloc() reads fch->initialized and then fch->io_uring. fch->io_uring is set before fch->initialized, ordered by the smp_wmb() in fuse_chan_set_intialized(), but fuse_block_alloc() has no matching read barrier between the two loads. This may lead a CPU to observe fch->initialized=1 but fch->io_uring=0, and skip the check that blocks request allocation until the io-uring queues are ready. This can reintroduce the lock-order inversion deadlock that commit 3393ff964e0f prevents. Add an smp_rmb() barrier to pair with the smp_wmb() in fuse_chan_set_initialized() to prevent this. Fixes: 3393ff964e0f ("fuse: block request allocation until io-uring init is complete") Cc: stable@vger.kernel.org Reviewed-by: Bernd Schubert Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/dev.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 7e81dea4f1c7..a27ea64d763a 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -85,7 +85,13 @@ void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *pa static bool fuse_block_alloc(struct fuse_chan *fch, bool for_background) { - return !fch->initialized || (for_background && fch->blocked) || + if (!fch->initialized) + return true; + + /* Pairs with smp_wmb() in fuse_chan_set_initialized() */ + smp_rmb(); + + return (for_background && fch->blocked) || (fch->io_uring && fch->connected && !fuse_uring_ready(fch)); } @@ -120,9 +126,6 @@ static struct fuse_req *fuse_get_req(struct fuse_chan *fch, bool for_background) goto out; } - /* Matches smp_wmb() in fuse_chan_set_initialized() */ - smp_rmb(); - err = -ENOTCONN; if (!fch->connected) goto out; From 4ef7c8cc9894fccc7aa5fdaf6b39faa45c58c23e Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Thu, 16 Jul 2026 11:31:43 -0700 Subject: [PATCH 07/35] fuse: use release/acquire for fch->initialized fuse_chan_set_initialized() sets values for the connection state and then sets fch->initialized to true, but lockless readers read fch->initialized and if true, go to read the connection state values, without using any barriers. There are a few instances where this happens (fuse_uring_cmd() before dispatching register / commit-and-fetch cmds, fuse_dev_do_wriite() for handling notify retrieves, etc). To make this as simple as possible, use release/acquire semantics for writing/reading fch->initialized. Add the missing read barriers. This is not marked for stable as these are not realistically reachable on a well-behaved server, and buggy/malicious servers who trigger this path fail benignly rather than crash or deadlock the kernel. Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/cuse.c | 3 ++- fs/fuse/dev.c | 14 ++++++-------- fs/fuse/dev_uring.c | 4 +++- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/fs/fuse/cuse.c b/fs/fuse/cuse.c index 3c15b5ba16d7..96d57735a79f 100644 --- a/fs/fuse/cuse.c +++ b/fs/fuse/cuse.c @@ -530,7 +530,8 @@ static int cuse_channel_open(struct inode *inode, struct file *file) INIT_LIST_HEAD(&cc->list); - cc->fc.chan->initialized = 1; + /* Pairs with smp_load_acquire() readers of fch->initialized */ + smp_store_release(&cc->fc.chan->initialized, 1); rc = cuse_send_init(cc); if (rc) { fuse_dev_put(fud); diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index a27ea64d763a..27dafda2a841 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -77,20 +77,17 @@ void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *pa fch->max_pages = param->max_pages; } - /* Make sure stores before this are seen on another CPU */ - smp_wmb(); - fch->initialized = 1; + /* Pairs with smp_load_acquire() readers of fch->initialized */ + smp_store_release(&fch->initialized, 1); wake_up_all(&fch->blocked_waitq); } static bool fuse_block_alloc(struct fuse_chan *fch, bool for_background) { - if (!fch->initialized) + /* Pairs with smp_store_release() in fuse_chan_set_initialized() */ + if (!smp_load_acquire(&fch->initialized)) return true; - /* Pairs with smp_wmb() in fuse_chan_set_initialized() */ - smp_rmb(); - return (for_background && fch->blocked) || (fch->io_uring && fch->connected && !fuse_uring_ready(fch)); } @@ -1892,7 +1889,8 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud, * initialized and connected state */ err = -EINVAL; - if (!fch->initialized || !fch->connected) + /* Pairs with smp_store_release() in fuse_chan_set_initialized() */ + if (!smp_load_acquire(&fch->initialized) || !fch->connected) goto copy_finish; /* Don't try to move folios (yet) */ diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 77c8cec43d9c..51f985154aa1 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -1251,8 +1251,10 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) /* * fuse_uring_register() needs the ring to be initialized, * we need to know the max payload size + * + * Pairs with smp_store_release() in fuse_chan_set_initialized() */ - if (!fch->initialized) + if (!smp_load_acquire(&fch->initialized)) return -EAGAIN; switch (cmd_op) { From 42df916e5a5f8fb4b60c8cefb54318d1ec02c580 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Thu, 16 Jul 2026 11:31:44 -0700 Subject: [PATCH 08/35] fuse: publish io-uring queues with release semantics fuse_uring_create_queue() initializes a fuse_ring_queue and then publishes the pointer into ring->queues[qid] with WRITE_ONCE() under the fch->lock. There are several readers that may concurrently be fetching that pointer locklessly and then deferencing it. WRITE_ONCE() doesn't ensure ordering of the queue's field initialization before the ring->queues[qid] pointer assignment. The queue must be published with smp_store_release() so the field initialization is guaranteed to happen before. Readers in paths where the read may happen concurrently with the store need to use READ_ONCE() because any race involving a plain access is undefined. Fixes: 24fe962c86f5 ("fuse: {io-uring} Handle SQEs - register commands") Cc: stable@vger.kernel.org Reviewed-by: Bernd Schubert Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/dev_uring.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 51f985154aa1..c8488ebc1d1f 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -204,7 +204,7 @@ void fuse_uring_destruct(struct fuse_chan *fch) return; for (qid = 0; qid < ring->nr_queues; qid++) { - struct fuse_ring_queue *queue = ring->queues[qid]; + struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]); struct fuse_ring_ent *ent, *next; if (!queue) @@ -223,7 +223,7 @@ void fuse_uring_destruct(struct fuse_chan *fch) kfree(queue->fpq.processing); kfree(queue); - ring->queues[qid] = NULL; + WRITE_ONCE(ring->queues[qid], NULL); } kfree(ring->queues); @@ -321,9 +321,11 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, } /* - * write_once and lock as the caller mostly doesn't take the lock at all + * fch->lock serializes concurrent creators for this qid. + * smp_store_release() are for the lockless readers who must see a + * fully initialized queue after &ring->queues[qid] is set */ - WRITE_ONCE(ring->queues[qid], queue); + smp_store_release(&ring->queues[qid], queue); spin_unlock(&fch->lock); return queue; @@ -434,7 +436,7 @@ static void fuse_uring_log_ent_state(struct fuse_ring *ring) struct fuse_ring_ent *ent; for (qid = 0; qid < ring->nr_queues; qid++) { - struct fuse_ring_queue *queue = ring->queues[qid]; + struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]); if (!queue) continue; @@ -967,7 +969,7 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, if (qid >= ring->nr_queues) return -EINVAL; - queue = ring->queues[qid]; + queue = READ_ONCE(ring->queues[qid]); if (!queue) return err; fpq = &queue->fpq; @@ -1035,7 +1037,7 @@ static bool is_ring_ready(struct fuse_ring *ring, int current_qid) if (current_qid == qid) continue; - queue = ring->queues[qid]; + queue = READ_ONCE(ring->queues[qid]); if (!queue) { ready = false; break; @@ -1191,7 +1193,7 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, return -EINVAL; } - queue = ring->queues[qid]; + queue = READ_ONCE(ring->queues[qid]); if (!queue) { queue = fuse_uring_create_queue(ring, qid); if (!queue) @@ -1332,7 +1334,7 @@ static struct fuse_ring_queue *fuse_uring_task_to_queue(struct fuse_ring *ring) ring->nr_queues)) qid = 0; - queue = ring->queues[qid]; + queue = READ_ONCE(ring->queues[qid]); WARN_ONCE(!queue, "Missing queue for qid %d\n", qid); return queue; From 98b4ca2378e1f6b6c06a74f699623ebecfb3549d Mon Sep 17 00:00:00 2001 From: Jim Harris Date: Mon, 22 Jun 2026 17:42:27 -0700 Subject: [PATCH 09/35] fuse: allow larger read requests by setting bdi->io_pages A FUSE server that advertises a large max_pages and max_write (e.g. max_pages=256, max_write=1MB) cannot currently obtain matching FUSE_READ request sizes from the kernel. Buffered sequential writes arrive at the server at the negotiated max_write size, but a large buffered read() is split into several smaller FUSE_READ requests. For a buffered read, filemap_get_pages() -> page_cache_sync_ra() sizes the read against ractl_max_pages(): max_pages = ractl->ra->ra_pages; if (req_size > max_pages && bdi->io_pages > max_pages) max_pages = min(req_size, bdi->io_pages); fuse leaves bdi->io_pages at the default VM_READAHEAD_PAGES (128KB), so a 1MB read() (req_size = 256 pages) is clamped to the readahead window (128KB, or 256KB for POSIX_FADV_SEQUENTIAL), producing four 256KB FUSE_READ round-trips instead of one. Set bdi->io_pages to fc->max_pages after feature negotiation. As the code above shows, io_pages only raises the limit when the request size already exceeds the readahead window, so it enlarges explicitly requested reads without enlarging the speculative readahead window. This avoids increasing speculative page-cache readahead on behalf of an unprivileged server. NFS does the same, setting io_pages from rpages while leaving ra_pages at the default. fc->max_pages is already bounded by fc->max_pages_limit (and, for virtio-fs, by the virtqueue descriptor count), so io_pages inherits the same bound. Suggested-by: Joanne Koong Signed-off-by: Jim Harris Assisted-by: Cursor:claude-opus-4.8 Reviewed-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/inode.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index d975073c6029..f7a0a0860a04 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1416,6 +1416,7 @@ static void process_init_reply(struct fuse_args *args, int error) fm->sb->s_bdi->ra_pages = min(fm->sb->s_bdi->ra_pages, ra_pages); + fm->sb->s_bdi->io_pages = fc->max_pages; fc->minor = arg->minor; fc->max_write = arg->minor < 5 ? 4096 : arg->max_write; fc->max_write = max_t(unsigned, 4096, fc->max_write); From 51e08eaf954de51b991889a5f5b5b5edcc712c6c Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 12 Jun 2026 11:48:37 -0700 Subject: [PATCH 10/35] io_uring/rsrc: rename io_buffer_register_bvec()/io_buffer_unregister_bvec() Currently, io_buffer_register_bvec() takes in a request. In preparation for supporting kernel-populated buffers in fuse io-uring (which will need to register bvecs directly, not through a struct request), rename this to io_buffer_register_request(). A subsequent patch will commandeer the "io_buffer_register_bvec()" function name to support registering bvecs directly. Rename io_buffer_unregister_bvec() to a more generic name, io_buffer_unregister(), as both io_buffer_register_request() and io_buffer_register_bvec() callers will use it for unregistration. Signed-off-by: Joanne Koong Reviewed-by: Caleb Sander Mateos Link: https://patch.msgid.link/20260612184840.4058966-2-joannelkoong@gmail.com Signed-off-by: Jens Axboe Signed-off-by: Miklos Szeredi --- Documentation/block/ublk.rst | 14 +++++++------- drivers/block/ublk_drv.c | 22 +++++++++++----------- include/linux/io_uring/cmd.h | 25 +++++++++++++++++++------ io_uring/rsrc.c | 14 +++++++------- 4 files changed, 44 insertions(+), 31 deletions(-) diff --git a/Documentation/block/ublk.rst b/Documentation/block/ublk.rst index 0413dcd9ef69..28300fee22bf 100644 --- a/Documentation/block/ublk.rst +++ b/Documentation/block/ublk.rst @@ -382,17 +382,17 @@ Zero copy --------- ublk zero copy relies on io_uring's fixed kernel buffer, which provides -two APIs: `io_buffer_register_bvec()` and `io_buffer_unregister_bvec`. +two APIs: `io_buffer_register_request()` and `io_buffer_unregister`. ublk adds IO command of `UBLK_IO_REGISTER_IO_BUF` to call -`io_buffer_register_bvec()` for ublk server to register client request +`io_buffer_register_request()` for ublk server to register client request buffer into io_uring buffer table, then ublk server can submit io_uring IOs with the registered buffer index. IO command of `UBLK_IO_UNREGISTER_IO_BUF` -calls `io_buffer_unregister_bvec()` to unregister the buffer, which is -guaranteed to be live between calling `io_buffer_register_bvec()` and -`io_buffer_unregister_bvec()`. Any io_uring operation which supports this -kind of kernel buffer will grab one reference of the buffer until the -operation is completed. +calls `io_buffer_unregister()` to unregister the buffer, which is guaranteed +to be live between calling `io_buffer_register_request()` and +`io_buffer_unregister()`. Any io_uring operation which supports this kind of +kernel buffer will grab one reference of the buffer until the operation is +completed. ublk server implementing zero copy or user copy has to be CAP_SYS_ADMIN and be trusted, because it is ublk server's responsibility to make sure IO buffer diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 4f6d9e652187..4036eb6be056 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -1699,8 +1699,8 @@ ublk_auto_buf_register(const struct ublk_queue *ubq, struct request *req, { int ret; - ret = io_buffer_register_bvec(cmd, req, ublk_io_release, - io->buf.auto_reg.index, issue_flags); + ret = io_buffer_register_request(cmd, req, ublk_io_release, + io->buf.auto_reg.index, issue_flags); if (ret) { if (io->buf.auto_reg.flags & UBLK_AUTO_BUF_REG_FALLBACK) { ublk_auto_buf_reg_fallback(ubq, req->tag); @@ -1906,7 +1906,7 @@ static noinline void ublk_batch_dispatch_fail(struct ublk_queue *ubq, ublk_io_unlock(io); if (index != -1) - io_buffer_unregister_bvec(data->cmd, index, + io_buffer_unregister(data->cmd, index, data->issue_flags); } @@ -3194,8 +3194,8 @@ static int ublk_register_io_buf(struct io_uring_cmd *cmd, if (!req) return -EINVAL; - ret = io_buffer_register_bvec(cmd, req, ublk_io_release, index, - issue_flags); + ret = io_buffer_register_request(cmd, req, ublk_io_release, index, + issue_flags); if (ret) { ublk_put_req_ref(io, req); return ret; @@ -3226,8 +3226,8 @@ ublk_daemon_register_io_buf(struct io_uring_cmd *cmd, if (!ublk_dev_support_zero_copy(ub) || !blk_rq_has_data(req)) return -EINVAL; - ret = io_buffer_register_bvec(cmd, req, ublk_io_release, index, - issue_flags); + ret = io_buffer_register_request(cmd, req, ublk_io_release, index, + issue_flags); if (ret) return ret; @@ -3242,7 +3242,7 @@ static int ublk_unregister_io_buf(struct io_uring_cmd *cmd, if (!(ub->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY)) return -EINVAL; - return io_buffer_unregister_bvec(cmd, index, issue_flags); + return io_buffer_unregister(cmd, index, issue_flags); } static int ublk_check_fetch_buf(const struct ublk_device *ub, __u64 buf_addr) @@ -3383,7 +3383,7 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, goto out; /* - * io_buffer_unregister_bvec() doesn't access the ubq or io, + * io_buffer_unregister() doesn't access the ubq or io, * so no need to validate the q_id, tag, or task */ if (_IOC_NR(cmd_op) == UBLK_IO_UNREGISTER_IO_BUF) @@ -3450,7 +3450,7 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, req = ublk_fill_io_cmd(io, cmd); ret = ublk_config_io_buf(ub, io, cmd, addr, &buf_idx); if (buf_idx != UBLK_INVALID_BUF_IDX) - io_buffer_unregister_bvec(cmd, buf_idx, issue_flags); + io_buffer_unregister(cmd, buf_idx, issue_flags); compl = ublk_need_complete_req(ub, io); if (req_op(req) == REQ_OP_ZONE_APPEND) @@ -3787,7 +3787,7 @@ static int ublk_batch_commit_io(struct ublk_queue *ubq, } if (buf_idx != UBLK_INVALID_BUF_IDX) - io_buffer_unregister_bvec(data->cmd, buf_idx, data->issue_flags); + io_buffer_unregister(data->cmd, buf_idx, data->issue_flags); if (req_op(req) == REQ_OP_ZONE_APPEND) req->__sector = ublk_batch_zone_lba(uc, elem); if (compl) diff --git a/include/linux/io_uring/cmd.h b/include/linux/io_uring/cmd.h index 331dcbefe72f..bbf57da1e4c8 100644 --- a/include/linux/io_uring/cmd.h +++ b/include/linux/io_uring/cmd.h @@ -91,6 +91,11 @@ struct io_br_sel io_uring_cmd_buffer_select(struct io_uring_cmd *ioucmd, bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd, struct io_br_sel *sel, unsigned int issue_flags); +int io_buffer_register_request(struct io_uring_cmd *cmd, struct request *rq, + void (*release)(void *), unsigned int index, + unsigned int issue_flags); +int io_buffer_unregister(struct io_uring_cmd *cmd, unsigned int index, + unsigned int issue_flags); #else static inline int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, @@ -133,6 +138,20 @@ static inline bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd, { return true; } +static inline int io_buffer_register_request(struct io_uring_cmd *cmd, + struct request *rq, + void (*release)(void *), + unsigned int index, + unsigned int issue_flags) +{ + return -EOPNOTSUPP; +} +static inline int io_buffer_unregister(struct io_uring_cmd *cmd, + unsigned int index, + unsigned int issue_flags) +{ + return -EOPNOTSUPP; +} #endif static inline struct io_uring_cmd *io_uring_cmd_from_tw(struct io_tw_req tw_req) @@ -182,10 +201,4 @@ static inline void io_uring_cmd_done32(struct io_uring_cmd *ioucmd, s32 ret, return __io_uring_cmd_done(ioucmd, ret, res2, issue_flags, true); } -int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq, - void (*release)(void *), unsigned int index, - unsigned int issue_flags); -int io_buffer_unregister_bvec(struct io_uring_cmd *cmd, unsigned int index, - unsigned int issue_flags); - #endif /* _LINUX_IO_URING_CMD_H */ diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 8d0f2ee24e0c..40807994a8f4 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -1015,9 +1015,9 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, return ret; } -int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq, - void (*release)(void *), unsigned int index, - unsigned int issue_flags) +int io_buffer_register_request(struct io_uring_cmd *cmd, struct request *rq, + void (*release)(void *), unsigned int index, + unsigned int issue_flags) { struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx; struct io_rsrc_data *data = &ctx->buf_table; @@ -1076,10 +1076,10 @@ int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq, io_ring_submit_unlock(ctx, issue_flags); return ret; } -EXPORT_SYMBOL_GPL(io_buffer_register_bvec); +EXPORT_SYMBOL_GPL(io_buffer_register_request); -int io_buffer_unregister_bvec(struct io_uring_cmd *cmd, unsigned int index, - unsigned int issue_flags) +int io_buffer_unregister(struct io_uring_cmd *cmd, unsigned int index, + unsigned int issue_flags) { struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx; struct io_rsrc_data *data = &ctx->buf_table; @@ -1109,7 +1109,7 @@ int io_buffer_unregister_bvec(struct io_uring_cmd *cmd, unsigned int index, io_ring_submit_unlock(ctx, issue_flags); return ret; } -EXPORT_SYMBOL_GPL(io_buffer_unregister_bvec); +EXPORT_SYMBOL_GPL(io_buffer_unregister); static int validate_fixed_range(u64 buf_addr, size_t len, const struct io_mapped_ubuf *imu) From fbc32d5f44d3e2134443909e816c5f9c657e81ad Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 12 Jun 2026 11:48:38 -0700 Subject: [PATCH 11/35] io_uring/rsrc: split io_buffer_register_request() logic Split the main initialization logic in io_buffer_register_request() into a helper function. This is a preparatory patch for supporting kernel-populated buffers in fuse io-uring, which will be reusing this logic. Signed-off-by: Joanne Koong Reviewed-by: Caleb Sander Mateos Link: https://patch.msgid.link/20260612184840.4058966-3-joannelkoong@gmail.com Signed-off-by: Jens Axboe Signed-off-by: Miklos Szeredi --- io_uring/rsrc.c | 94 +++++++++++++++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 38 deletions(-) diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 40807994a8f4..5d50b967645b 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -1015,63 +1015,81 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, return ret; } +static struct io_mapped_ubuf *io_kernel_buffer_init(struct io_ring_ctx *ctx, + unsigned int nr_bvecs, + unsigned int total_bytes, + u8 dir, + void (*release)(void *), + void *priv, + unsigned int index) +{ + struct io_rsrc_data *data = &ctx->buf_table; + struct io_mapped_ubuf *imu; + struct io_rsrc_node *node; + + if (index >= data->nr) + return ERR_PTR(-EINVAL); + index = array_index_nospec(index, data->nr); + + if (data->nodes[index]) + return ERR_PTR(-EBUSY); + + node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); + if (!node) + return ERR_PTR(-ENOMEM); + + imu = io_alloc_imu(ctx, nr_bvecs); + if (!imu) { + io_cache_free(&ctx->node_cache, node); + return ERR_PTR(-ENOMEM); + } + + imu->ubuf = 0; + imu->len = total_bytes; + imu->folio_shift = PAGE_SHIFT; + imu->nr_bvecs = nr_bvecs; + refcount_set(&imu->refs, 1); + imu->release = release; + imu->priv = priv; + imu->dir = dir; + imu->flags = IO_REGBUF_F_KBUF; + + node->buf = imu; + data->nodes[index] = node; + + return imu; +} + int io_buffer_register_request(struct io_uring_cmd *cmd, struct request *rq, void (*release)(void *), unsigned int index, unsigned int issue_flags) { struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx; - struct io_rsrc_data *data = &ctx->buf_table; struct req_iterator rq_iter; struct io_mapped_ubuf *imu; - struct io_rsrc_node *node; struct bio_vec bv; - unsigned int nr_bvecs = 0; - int ret = 0; - - io_ring_submit_lock(ctx, issue_flags); - if (index >= data->nr) { - ret = -EINVAL; - goto unlock; - } - index = array_index_nospec(index, data->nr); - - if (data->nodes[index]) { - ret = -EBUSY; - goto unlock; - } - - node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); - if (!node) { - ret = -ENOMEM; - goto unlock; - } - /* * blk_rq_nr_phys_segments() may overestimate the number of bvecs * but avoids needing to iterate over the bvecs */ - imu = io_alloc_imu(ctx, blk_rq_nr_phys_segments(rq)); - if (!imu) { - io_cache_free(&ctx->node_cache, node); - ret = -ENOMEM; + unsigned int nr_bvecs = blk_rq_nr_phys_segments(rq); + unsigned int total_bytes = blk_rq_bytes(rq); + int ret = 0; + + io_ring_submit_lock(ctx, issue_flags); + + imu = io_kernel_buffer_init(ctx, nr_bvecs, total_bytes, + 1 << rq_data_dir(rq), release, rq, index); + if (IS_ERR(imu)) { + ret = PTR_ERR(imu); goto unlock; } - imu->ubuf = 0; - imu->len = blk_rq_bytes(rq); - imu->folio_shift = PAGE_SHIFT; - refcount_set(&imu->refs, 1); - imu->release = release; - imu->priv = rq; - imu->flags = IO_REGBUF_F_KBUF; - imu->dir = 1 << rq_data_dir(rq); - + nr_bvecs = 0; rq_for_each_bvec(bv, rq, rq_iter) imu->bvec[nr_bvecs++] = bv; imu->nr_bvecs = nr_bvecs; - node->buf = imu; - data->nodes[index] = node; unlock: io_ring_submit_unlock(ctx, issue_flags); return ret; From bd62a2cfff9f37d85927050f082ffc3a423e6b53 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 12 Jun 2026 11:48:39 -0700 Subject: [PATCH 12/35] io_uring/rsrc: add io_buffer_register_bvec() Add io_buffer_register_bvec() for registering a bvec array. This is a preparatory patch for fuse-over-io-uring zero-copy. Signed-off-by: Joanne Koong Reviewed-by: Caleb Sander Mateos Link: https://patch.msgid.link/20260612184840.4058966-4-joannelkoong@gmail.com Signed-off-by: Jens Axboe Signed-off-by: Miklos Szeredi --- include/linux/io_uring/cmd.h | 13 +++++++++++++ io_uring/rsrc.c | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/include/linux/io_uring/cmd.h b/include/linux/io_uring/cmd.h index bbf57da1e4c8..42801f0b6456 100644 --- a/include/linux/io_uring/cmd.h +++ b/include/linux/io_uring/cmd.h @@ -94,6 +94,10 @@ bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd, int io_buffer_register_request(struct io_uring_cmd *cmd, struct request *rq, void (*release)(void *), unsigned int index, unsigned int issue_flags); +int io_buffer_register_bvec(struct io_uring_cmd *cmd, const struct bio_vec *bvs, + unsigned int nr_bvecs, void (*release)(void *), + void *priv, u8 dir, unsigned int index, + unsigned int issue_flags); int io_buffer_unregister(struct io_uring_cmd *cmd, unsigned int index, unsigned int issue_flags); #else @@ -146,6 +150,15 @@ static inline int io_buffer_register_request(struct io_uring_cmd *cmd, { return -EOPNOTSUPP; } +static inline int io_buffer_register_bvec(struct io_uring_cmd *cmd, + const struct bio_vec *bvs, + unsigned int nr_bvecs, + void (*release)(void *), void *priv, + u8 dir, unsigned int index, + unsigned int issue_flags) +{ + return -EOPNOTSUPP; +} static inline int io_buffer_unregister(struct io_uring_cmd *cmd, unsigned int index, unsigned int issue_flags) diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 5d50b967645b..819c5087d8d3 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -1096,6 +1096,41 @@ int io_buffer_register_request(struct io_uring_cmd *cmd, struct request *rq, } EXPORT_SYMBOL_GPL(io_buffer_register_request); +/* + * bvs is copied internally. caller may free it on return. + */ +int io_buffer_register_bvec(struct io_uring_cmd *cmd, const struct bio_vec *bvs, + unsigned int nr_bvecs, void (*release)(void *), + void *priv, u8 dir, unsigned int index, + unsigned int issue_flags) +{ + struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx; + struct io_mapped_ubuf *imu; + struct bio_vec *bvec; + unsigned int i, total_bytes = 0; + int ret = 0; + + for (i = 0; i < nr_bvecs; i++) + total_bytes += bvs[i].bv_len; + + io_ring_submit_lock(ctx, issue_flags); + imu = io_kernel_buffer_init(ctx, nr_bvecs, total_bytes, dir, release, + priv, index); + if (IS_ERR(imu)) { + ret = PTR_ERR(imu); + goto unlock; + } + + bvec = imu->bvec; + for (i = 0; i < nr_bvecs; i++) + bvec[i] = bvs[i]; + +unlock: + io_ring_submit_unlock(ctx, issue_flags); + return ret; +} +EXPORT_SYMBOL_GPL(io_buffer_register_bvec); + int io_buffer_unregister(struct io_uring_cmd *cmd, unsigned int index, unsigned int issue_flags) { From 95961b72c57b29a96c14f86d16f1d32787f2e009 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 12 Jun 2026 11:48:40 -0700 Subject: [PATCH 13/35] io_uring/rsrc: rename and export IO_IMU_DEST / IO_IMU_SOURCE Rename IO_IMU_DEST and IO_IMU_SOURCE to IO_BUF_DEST and IO_BUF_SOURCE and export it so subsystems may use it. This is needed by the io_buffer_register_bvec() path for callers who may need the buffer to be both readable and writable. Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260612184840.4058966-5-joannelkoong@gmail.com Signed-off-by: Jens Axboe Signed-off-by: Miklos Szeredi --- include/linux/io_uring_types.h | 5 +++++ io_uring/io_uring.c | 2 +- io_uring/rsrc.c | 2 +- io_uring/rsrc.h | 5 ----- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 87151a5b62c1..db42a548c7a5 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -44,6 +44,11 @@ enum io_uring_cmd_flags { IO_URING_F_COMPAT = (1 << 12), }; +enum { + IO_BUF_DEST = 1 << ITER_DEST, + IO_BUF_SOURCE = 1 << ITER_SOURCE, +}; + struct iou_loop_params; struct io_wq_work_node { diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 1ea2fca34a36..ba685b6052ed 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -3245,7 +3245,7 @@ static int __init io_uring_init(void) io_uring_optable_init(); /* imu->dir is u8 */ - BUILD_BUG_ON((IO_IMU_DEST | IO_IMU_SOURCE) > U8_MAX); + BUILD_BUG_ON((IO_BUF_DEST | IO_BUF_SOURCE) > U8_MAX); /* * Allow user copy in the per-command field, which starts after the diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 819c5087d8d3..f3f01e0c8102 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -912,7 +912,7 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, imu->release = io_release_ubuf; imu->priv = imu; imu->flags = 0; - imu->dir = IO_IMU_DEST | IO_IMU_SOURCE; + imu->dir = IO_BUF_DEST | IO_BUF_SOURCE; if (coalesced) imu->folio_shift = data.folio_shift; refcount_set(&imu->refs, 1); diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index 98ae8ef51009..e503b02aa61a 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -23,11 +23,6 @@ struct io_rsrc_node { }; }; -enum { - IO_IMU_DEST = 1 << ITER_DEST, - IO_IMU_SOURCE = 1 << ITER_SOURCE, -}; - enum { IO_REGBUF_F_KBUF = 1, }; From 6330b1f61ed1d17850fc61bdb8920ca1056e2cf9 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 14 Aug 2026 11:59:41 -0700 Subject: [PATCH 14/35] fuse: decouple fuse_ring creation from ent registration Currently, the connection's fuse_ring is created lazily on the first FUSE_IO_URING_CMD_REGISTER command. A server registers entries from one thread per queue (one per CPU) and those threads issue their first REGISTER command concurrently. They then race to create the single per-connection fuse_ring, which required open-coded handling in fuse_uring_create() to detect and protect against concurrent creations. Decouple fuse_ring creation from ent registration and move it to FUSE_INIT reply processing after a server has negotiated and set FUSE_OVER_IO_URING. The ring is published before the connection is marked initialized. fuse_uring_register() no longer creates the ring and it instead uses the ring set up at init time. Reviewed-by: Bernd Schubert Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/dev.c | 8 +++----- fs/fuse/dev.h | 2 +- fs/fuse/dev_uring.c | 26 ++++++++++---------------- fs/fuse/dev_uring_i.h | 5 +++++ fs/fuse/inode.c | 4 +++- 5 files changed, 22 insertions(+), 23 deletions(-) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 27dafda2a841..d8f97943e973 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -75,6 +75,9 @@ void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *pa fch->minor = param->minor; fch->max_write = param->max_write; fch->max_pages = param->max_pages; + + if (param->io_uring_enabled) + fuse_uring_conn_init(fch); } /* Pairs with smp_load_acquire() readers of fch->initialized */ @@ -412,11 +415,6 @@ void fuse_chan_set_fc(struct fuse_chan *fch, struct fuse_conn *fc) fch->conn = fc; } -void fuse_chan_io_uring_enable(struct fuse_chan *fch) -{ - fch->io_uring = 1; -} - void fuse_pqueue_init(struct fuse_pqueue *fpq) { spin_lock_init(&fpq->lock); diff --git a/fs/fuse/dev.h b/fs/fuse/dev.h index aed69fd14c41..8d25378c0918 100644 --- a/fs/fuse/dev.h +++ b/fs/fuse/dev.h @@ -22,6 +22,7 @@ struct fuse_chan_param { unsigned int minor; unsigned int max_write; unsigned int max_pages; + bool io_uring_enabled; }; struct fuse_chan *fuse_chan_new(void); @@ -34,7 +35,6 @@ void fuse_chan_max_background_set(struct fuse_chan *fch, unsigned int val); unsigned int fuse_chan_num_waiting(struct fuse_chan *fch); void fuse_chan_set_fc(struct fuse_chan *fch, struct fuse_conn *fc); void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *param); -void fuse_chan_io_uring_enable(struct fuse_chan *fch); ssize_t fuse_chan_send(struct fuse_chan *fch, struct fuse_args *args); int fuse_chan_send_bg(struct fuse_chan *fch, struct fuse_args *args, gfp_t gfp_flags); int fuse_chan_send_notify_reply(struct fuse_chan *fch, struct fuse_args *args, u64 unique); diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index c8488ebc1d1f..9616778505ba 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -238,7 +238,6 @@ static struct fuse_ring *fuse_uring_create(struct fuse_chan *fch) { struct fuse_ring *ring; size_t nr_queues = num_possible_cpus(); - struct fuse_ring *res = NULL; size_t max_payload_size; ring = kzalloc_obj(*ring, GFP_KERNEL_ACCOUNT); @@ -258,12 +257,6 @@ static struct fuse_ring *fuse_uring_create(struct fuse_chan *fch) spin_unlock(&fch->lock); goto out_err; } - if (fch->ring) { - /* race, another thread created the ring in the meantime */ - spin_unlock(&fch->lock); - res = fch->ring; - goto out_err; - } init_waitqueue_head(&ring->stop_waitq); @@ -278,7 +271,13 @@ static struct fuse_ring *fuse_uring_create(struct fuse_chan *fch) out_err: kfree(ring->queues); kfree(ring); - return res; + return NULL; +} + +void fuse_uring_conn_init(struct fuse_chan *fch) +{ + if (fuse_uring_create(fch)) + fch->io_uring = 1; } static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, @@ -1178,15 +1177,10 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, struct fuse_ring *ring = smp_load_acquire(&fch->ring); struct fuse_ring_queue *queue; struct fuse_ring_ent *ent; - int err; unsigned int qid = READ_ONCE(cmd_req->qid); - err = -ENOMEM; - if (!ring) { - ring = fuse_uring_create(fch); - if (!ring) - return err; - } + if (!ring) + return -EINVAL; if (qid >= ring->nr_queues) { pr_info_ratelimited("fuse: Invalid ring qid %u\n", qid); @@ -1197,7 +1191,7 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, if (!queue) { queue = fuse_uring_create_queue(ring, qid); if (!queue) - return err; + return -ENOMEM; } /* diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 55f8d04e4b0b..d721a4fc0215 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -135,6 +135,7 @@ struct fuse_ring { bool ready; }; +void fuse_uring_conn_init(struct fuse_chan *fch); void fuse_uring_stop_queues(struct fuse_ring *ring); void fuse_uring_abort_end_requests(struct fuse_ring *ring); int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags); @@ -174,6 +175,10 @@ static inline bool fuse_uring_ready(struct fuse_chan *fch) #else /* CONFIG_FUSE_IO_URING */ +static inline void fuse_uring_conn_init(struct fuse_chan *fch) +{ +} + static inline void fuse_uring_abort(struct fuse_chan *fch) { } diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index f7a0a0860a04..33773c7d129a 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1272,6 +1272,7 @@ static void process_init_reply(struct fuse_args *args, int error) struct fuse_mount *fm = ia->fm; struct fuse_conn *fc = fm->fc; struct fuse_init_out *arg = &ia->out; + bool io_uring_enabled = false; bool ok = true; if (error || arg->major != FUSE_KERNEL_VERSION) @@ -1402,7 +1403,7 @@ static void process_init_reply(struct fuse_args *args, int error) ok = false; } if (flags & FUSE_OVER_IO_URING && fuse_uring_enabled()) - fuse_chan_io_uring_enable(fc->chan); + io_uring_enabled = true; if (flags & FUSE_REQUEST_TIMEOUT) timeout = arg->request_timeout; @@ -1433,6 +1434,7 @@ static void process_init_reply(struct fuse_args *args, int error) .minor = fc->minor, .max_write = fc->max_write, .max_pages = fc->max_pages, + .io_uring_enabled = io_uring_enabled, }; fuse_chan_set_initialized(fc->chan, &cp); } From ebed9ea5b469588c6074f3ed5b8d8ec63c4ccf48 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 14 Aug 2026 11:59:42 -0700 Subject: [PATCH 15/35] fuse: add FUSE_IO_URING_CMD_ADD_QUEUE fuse-over-io-uring queues are currently created lazily, as a side effect of the first FUSE_IO_URING_CMD_REGISTER command for a given qid. This ties queue creation to entry registration. Add a FUSE_IO_URING_CMD_ADD_QUEUE command so a server can create a queue explicitly, decoupling queue setup from entry registration. This is additionally a prerequisite for FUSE_IO_URING_CMD_ADD_BUFPOOL, which attaches a buffer pool to an existing queue and therefore needs the queue to have been created first. Reviewed-by: Bernd Schubert Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/dev_uring.c | 45 +++++++++++++++++++++++++++++++++------ include/uapi/linux/fuse.h | 8 ++++++- 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 9616778505ba..3b9fd0daef66 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -281,7 +281,8 @@ void fuse_uring_conn_init(struct fuse_chan *fch) } static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, - int qid) + int qid, + bool fail_if_exists) { struct fuse_chan *fch = ring->chan; struct fuse_ring_queue *queue; @@ -289,11 +290,11 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, queue = kzalloc_obj(*queue, GFP_KERNEL_ACCOUNT); if (!queue) - return NULL; + return ERR_PTR(-ENOMEM); pq = fuse_pqueue_alloc(); if (!pq) { kfree(queue); - return NULL; + return ERR_PTR(-ENOMEM); } queue->qid = qid; @@ -316,7 +317,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, spin_unlock(&fch->lock); kfree(queue->fpq.processing); kfree(queue); - return ring->queues[qid]; + return fail_if_exists ? ERR_PTR(-EEXIST) : ring->queues[qid]; } /* @@ -1189,9 +1190,9 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, queue = READ_ONCE(ring->queues[qid]); if (!queue) { - queue = fuse_uring_create_queue(ring, qid); - if (!queue) - return -ENOMEM; + queue = fuse_uring_create_queue(ring, qid, false); + if (IS_ERR(queue)) + return PTR_ERR(queue); } /* @@ -1206,6 +1207,30 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, return fuse_uring_do_register(ent, cmd, issue_flags); } +static int fuse_uring_add_queue(struct io_uring_cmd *cmd, struct fuse_chan *fch) +{ + const struct fuse_uring_cmd_req *cmd_req = + io_uring_sqe128_cmd(cmd->sqe, struct fuse_uring_cmd_req); + struct fuse_ring *ring = smp_load_acquire(&fch->ring); + unsigned int qid = READ_ONCE(cmd_req->qid); + uint64_t flags = READ_ONCE(cmd_req->flags); + struct fuse_ring_queue *queue; + + if (!ring || flags) + return -EINVAL; + + if (qid >= ring->nr_queues) { + pr_info_ratelimited("fuse: Invalid ring qid %u\n", qid); + return -EINVAL; + } + + queue = fuse_uring_create_queue(ring, qid, true); + if (IS_ERR(queue)) + return PTR_ERR(queue); + + return 0; +} + /* * Entry function from io_uring to handle the given passthrough command * (op code IORING_OP_URING_CMD) @@ -1272,6 +1297,12 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) return err; } break; + case FUSE_IO_URING_CMD_ADD_QUEUE: + err = fuse_uring_add_queue(cmd, fch); + if (err) + pr_info_once("FUSE_IO_URING_CMD_ADD_QUEUE failed err=%d\n", + err); + return err; default: return -EINVAL; } diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index c13e1f9a2f12..cfb055c0c764 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -240,6 +240,9 @@ * - add FUSE_COPY_FILE_RANGE_64 * - add struct fuse_copy_file_range_out * - add FUSE_NOTIFY_PRUNE + * + * 7.46 + * - add FUSE_IO_URING_CMD_ADD_QUEUE */ #ifndef _LINUX_FUSE_H @@ -275,7 +278,7 @@ #define FUSE_KERNEL_VERSION 7 /** Minor version number of this interface */ -#define FUSE_KERNEL_MINOR_VERSION 45 +#define FUSE_KERNEL_MINOR_VERSION 46 /** The node ID of the root inode */ #define FUSE_ROOT_ID 1 @@ -1292,6 +1295,9 @@ enum fuse_uring_cmd { /* commit fuse request result and fetch next request */ FUSE_IO_URING_CMD_COMMIT_AND_FETCH = 2, + + /* add a queue */ + FUSE_IO_URING_CMD_ADD_QUEUE = 3, }; /** From b45aaabc628bc7356e21bd2eb0c2ae9bdfa13894 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 14 Aug 2026 11:59:43 -0700 Subject: [PATCH 16/35] fuse: add io-uring buffer pools Right now, ents and buffers are tightly coupled in fuse io-uring where each entry has its own dedicated payload buffer, requiring N buffers for N entries where each buffer must be large enough to accomodate the maximum payload size. This is suboptimal as most request types (lookup, open, release, getattr, etc) require vastly less bytes than the maximum payload size and some requests (unlink, rmdir, fsync, flush, etc) do not require payload buffers at all. Instead of requiring a 1:1 coupling between ents and payload buffers, allow the server to pass in a buffer pool (a contiguous chunk of memory) that the kernel will use as it wishes for servicing ents/requests. Entries only reserve a "buffer" from the pool while actively processing a request that requires a payload buffer. This decoupling and letting the kernel delegate memory from the pool for requests allows the kernel to optimize memory usage and reduces the memory usage requirements needed to use fuse-over-io-uring. A pool is registered per queue with the new FUSE_IO_URING_CMD_ADD_BUFPOOL command. The server passes the pool's base address and length in fuse_uring_cmd_req.bufpool.{uaddr,len}. Internally, the kernel splits the region into buffers of ring->max_payload_sz bytes each (nr_bufs = pool len / max_payload_sz). A queue commits to a payload mode on first use: registering an entry that carries its own payload selects the legacy per-entry mode, while ADD_BUFPOOL selects pool mode. The two are mutually exclusive, so ADD_BUFPOOL must be issued before any payload-carrying entries are registered on that queue. The queue must have been created before the bufpool is added, through the FUSE_IO_URING_CMD_ADD_QUEUE command. The kernel tracks free buffers with a bitmap (a set bit marks a free buffer). On dispatch, a request that needs a payload claims a free buffer (find_first_bit + clear). A request that needs none claims nothing. The buffer's byte offset within the pool is reported to the server in the new fuse_uring_ent_in_out.offset field so that the server can locate the payload. On completion the buffer is returned to the pool or reused directly if the next request on that entry also has a payload. The FUSE_HAS_IO_URING_BUFPOOL flag advertises kernel support to the server for bufpools. Buffer pool request flow ~~~~~~~~~~~~~~~~~~~~~~~~ | Kernel | FUSE daemon | | | [request arrives] | | [claim a free pool buffer] | | >fuse_uring_select_buffer() | | [copy headers to ring] | | [copy payload to buffer] | | [report buffer offset in ent_in_out] | | >io_uring_cmd_done() | | | [read headers] | | [read/write payload at offset] | | [process request] | | >io_uring_submit() | | COMMIT_AND_FETCH | >fuse_uring_commit_fetch() | | [copy reply from ring] | | [return buffer to the pool] | | >fuse_uring_recycle_buffer() | Reviewed-by: Bernd Schubert Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/dev_uring.c | 239 +++++++++++++++++++++++++++++++++----- fs/fuse/dev_uring_i.h | 37 +++++- fs/fuse/inode.c | 2 +- include/uapi/linux/fuse.h | 21 +++- 4 files changed, 269 insertions(+), 30 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 3b9fd0daef66..d300c7f441c4 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -9,6 +9,7 @@ #include "dev_uring_i.h" #include "fuse_trace.h" +#include #include #include @@ -41,6 +42,11 @@ enum fuse_uring_header_type { FUSE_URING_HEADER_RING_ENT, }; +static inline bool bufpool_enabled(struct fuse_ring_queue *queue) +{ + return queue->payload_mode == FUSE_PAYLOAD_BUFPOOL; +} + static void uring_cmd_set_ring_ent(struct io_uring_cmd *cmd, struct fuse_ring_ent *ring_ent) { @@ -222,6 +228,7 @@ void fuse_uring_destruct(struct fuse_chan *fch) } kfree(queue->fpq.processing); + kfree(queue->bufpool); kfree(queue); WRITE_ONCE(ring->queues[qid], NULL); } @@ -316,6 +323,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, if (ring->queues[qid]) { spin_unlock(&fch->lock); kfree(queue->fpq.processing); + kfree(queue->bufpool); kfree(queue); return fail_if_exists ? ERR_PTR(-EEXIST) : ring->queues[qid]; } @@ -646,13 +654,14 @@ static int copy_header_from_ring(struct fuse_ring_ent *ent, } static int setup_fuse_copy_state(struct fuse_copy_state *cs, - struct fuse_ring *ring, struct fuse_req *req, + struct fuse_req *req, struct fuse_ring_ent *ent, int dir, struct iov_iter *iter) { int err; - err = import_ubuf(dir, ent->payload, ring->max_payload_sz, iter); + err = import_ubuf(dir, ent->payload.iov_base, ent->payload.iov_len, + iter); if (err) { pr_info_ratelimited("fuse: Import of user buffer failed\n"); return err; @@ -666,8 +675,7 @@ static int setup_fuse_copy_state(struct fuse_copy_state *cs, return 0; } -static int fuse_uring_copy_from_ring(struct fuse_ring *ring, - struct fuse_req *req, +static int fuse_uring_copy_from_ring(struct fuse_req *req, struct fuse_ring_ent *ent) { struct fuse_copy_state cs; @@ -681,7 +689,7 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring, if (err) return err; - err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_SOURCE, &iter); + err = setup_fuse_copy_state(&cs, req, ent, ITER_SOURCE, &iter); if (err) return err; @@ -693,7 +701,7 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring, /* * Copy data from the req to the ring buffer */ -static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, +static int fuse_uring_args_to_ring(struct fuse_req *req, struct fuse_ring_ent *ent) { struct fuse_copy_state cs; @@ -707,7 +715,7 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, .commit_id = req->in.h.unique, }; - err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_DEST, &iter); + err = setup_fuse_copy_state(&cs, req, ent, ITER_DEST, &iter); if (err) return err; @@ -737,6 +745,10 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, } ent_in_out.payload_sz = cs.ring.copied_sz; + if (bufpool_enabled(ent->queue) && ent->payload.iov_base) + ent_in_out.offset = + (uintptr_t)ent->payload.iov_base - ent->queue->bufpool->base_uaddr; + return copy_header_to_ring(ent, FUSE_URING_HEADER_RING_ENT, &ent_in_out, sizeof(ent_in_out)); } @@ -745,7 +757,6 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, struct fuse_req *req) { struct fuse_ring_queue *queue = ent->queue; - struct fuse_ring *ring = queue->ring; int err; err = -EIO; @@ -760,7 +771,7 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, return err; /* copy the request */ - err = fuse_uring_args_to_ring(ring, req, ent); + err = fuse_uring_args_to_ring(req, ent); if (unlikely(err)) { pr_info_ratelimited("Copy to ring failed: %d\n", err); return err; @@ -771,6 +782,91 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, sizeof(req->in.h)); } +static bool fuse_uring_req_has_payload(struct fuse_req *req) +{ + struct fuse_args *args = req->args; + + return args->in_numargs > 1 || args->out_numargs; +} + +static int fuse_uring_select_buffer(struct fuse_ring_ent *ent) +{ + struct fuse_ring_queue *queue = ent->queue; + struct fuse_bufpool *pool = queue->bufpool; + unsigned int id; + + lockdep_assert_held(&queue->lock); + + id = find_first_bit(pool->free_map, pool->nr_bufs); + if (id >= pool->nr_bufs) + return -ENOBUFS; + + WARN_ON_ONCE(ent->payload.iov_base); + __clear_bit(id, pool->free_map); + + ent->buf_id = id; + ent->payload.iov_base = + (void __user *)(pool->base_uaddr + id * pool->buf_size); + ent->payload.iov_len = pool->buf_size; + + return 0; +} + +static void fuse_uring_recycle_buffer(struct fuse_ring_ent *ent) +{ + struct iovec *ent_payload = &ent->payload; + struct fuse_ring_queue *queue = ent->queue; + struct fuse_bufpool *pool; + + lockdep_assert_held(&queue->lock); + + if (!bufpool_enabled(queue) || !ent_payload->iov_base) + return; + + pool = queue->bufpool; + + /* a buffer should never be recycled twice */ + WARN_ON_ONCE(test_bit(ent->buf_id, pool->free_map)); + __set_bit(ent->buf_id, pool->free_map); + + memset(ent_payload, 0, sizeof(*ent_payload)); + ent->buf_id = 0; +} + +static int fuse_uring_next_req_update_buffer(struct fuse_ring_ent *ent, + struct fuse_req *req) +{ + bool buffer_selected; + bool has_payload; + + if (!bufpool_enabled(ent->queue)) + return 0; + + buffer_selected = !!ent->payload.iov_base; + has_payload = fuse_uring_req_has_payload(req); + + if (has_payload && !buffer_selected) + return fuse_uring_select_buffer(ent); + + if (!has_payload && buffer_selected) + fuse_uring_recycle_buffer(ent); + + return 0; +} + +static int fuse_uring_prep_buffer(struct fuse_ring_ent *ent, + struct fuse_req *req) +{ + if (!bufpool_enabled(ent->queue)) + return 0; + + /* no payload to copy, can skip selecting a buffer */ + if (!fuse_uring_req_has_payload(req)) + return 0; + + return fuse_uring_select_buffer(ent); +} + static int fuse_uring_prepare_send(struct fuse_ring_ent *ent, struct fuse_req *req) { @@ -858,9 +954,12 @@ static struct fuse_req *fuse_uring_ent_assign_req(struct fuse_ring_ent *ent) /* get and assign the next entry while it is still holding the lock */ req = list_first_entry_or_null(req_queue, struct fuse_req, list); - if (req) - fuse_uring_add_req_to_ring_ent(ent, req); + if (!req || fuse_uring_next_req_update_buffer(ent, req)) { + fuse_uring_recycle_buffer(ent); + return NULL; + } + fuse_uring_add_req_to_ring_ent(ent, req); return req; } @@ -872,7 +971,6 @@ static struct fuse_req *fuse_uring_ent_assign_req(struct fuse_ring_ent *ent) static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, unsigned int issue_flags) { - struct fuse_ring *ring = ent->queue->ring; ssize_t err = -EFAULT; if (copy_header_from_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->out.h, @@ -885,7 +983,7 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, goto out; } - err = fuse_uring_copy_from_ring(ring, req, ent); + err = fuse_uring_copy_from_ring(req, ent); out: fuse_uring_req_end(ent, req, err); } @@ -1004,6 +1102,7 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, if (err != 0) { pr_info_ratelimited("qid=%d commit_id %llu state %d", queue->qid, commit_id, ent->state); + fuse_uring_recycle_buffer(ent); spin_unlock(&queue->lock); fuse_uring_req_end(ent, req, err); return err; @@ -1021,6 +1120,11 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, * fuse requests would otherwise not get processed - committing * and fetching is done in one step vs legacy fuse, which has separated * read (fetch request) and write (commit result). + * + * If there is no next request or if all buffers are busy (if using a + * bufpool), the cmd is not returned to userspace. The entry is left + * available and the cmd only returns to userspace when there's a + * next request and an available buffer. */ if (fuse_uring_get_next_fuse_req(ent, queue)) fuse_uring_send(ent, cmd, 0, issue_flags); @@ -1145,11 +1249,23 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, } payload = &iov[FUSE_URING_IOV_PAYLOAD]; - if (payload->iov_len < ring->max_payload_sz) { - pr_info_ratelimited("Invalid req payload len %zu\n", - payload->iov_len); - return ERR_PTR(err); + + spin_lock(&queue->lock); + if (bufpool_enabled(queue)) { + if (payload->iov_base || payload->iov_len) { + spin_unlock(&queue->lock); + return ERR_PTR(err); + } + } else { + if (payload->iov_len < ring->max_payload_sz) { + pr_info_ratelimited("Invalid req payload len %zu\n", + payload->iov_len); + spin_unlock(&queue->lock); + return ERR_PTR(err); + } + queue->payload_mode = FUSE_PAYLOAD_PER_ENT; } + spin_unlock(&queue->lock); err = -ENOMEM; ent = kzalloc_obj(*ent, GFP_KERNEL_ACCOUNT); @@ -1160,7 +1276,8 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, ent->queue = queue; ent->headers = headers->iov_base; - ent->payload = payload->iov_base; + if (queue->payload_mode == FUSE_PAYLOAD_PER_ENT) + ent->payload = *payload; atomic_inc(&ring->queue_refs); return ent; @@ -1231,6 +1348,67 @@ static int fuse_uring_add_queue(struct io_uring_cmd *cmd, struct fuse_chan *fch) return 0; } +static int fuse_uring_add_bufpool(struct io_uring_cmd *cmd, + struct fuse_chan *fch) +{ + const struct fuse_uring_cmd_req *cmd_req = + io_uring_sqe128_cmd(cmd->sqe, struct fuse_uring_cmd_req); + unsigned int qid = READ_ONCE(cmd_req->qid); + uint64_t flags = READ_ONCE(cmd_req->flags); + /* paired with the smp_store_release() in fuse_uring_create */ + struct fuse_ring *ring = smp_load_acquire(&fch->ring); + struct fuse_ring_queue *queue; + struct fuse_bufpool *pool; + uintptr_t pool_uaddr; + unsigned int pool_len, nr_bufs; + size_t pool_size, buf_size; + + if (!ring || qid >= ring->nr_queues || flags) + return -EINVAL; + + /* reserved for future use, must be zero */ + if (READ_ONCE(cmd_req->bufpool.reserved)) + return -EINVAL; + + /* Pairs with smp_store_release() in fuse_uring_create_queue() */ + queue = smp_load_acquire(&ring->queues[qid]); + if (!queue) + return -EINVAL; + + pool_uaddr = READ_ONCE(cmd_req->bufpool.uaddr); + pool_len = READ_ONCE(cmd_req->bufpool.len); + + /* each buffer holds the max payload size */ + buf_size = queue->ring->max_payload_sz; + + nr_bufs = pool_len / buf_size; + if (!nr_bufs) + return -EINVAL; + + pool_size = struct_size(pool, free_map, BITS_TO_LONGS(nr_bufs)); + pool = kzalloc(pool_size, GFP_KERNEL_ACCOUNT); + if (!pool) + return -ENOMEM; + + pool->base_uaddr = pool_uaddr; + pool->buf_size = buf_size; + pool->nr_bufs = nr_bufs; + /* all buffers are free */ + bitmap_set(pool->free_map, 0, nr_bufs); + + spin_lock(&queue->lock); + if (queue->payload_mode != FUSE_PAYLOAD_UNSET) { + spin_unlock(&queue->lock); + kfree(pool); + return -EINVAL; + } + queue->bufpool = pool; + queue->payload_mode = FUSE_PAYLOAD_BUFPOOL; + spin_unlock(&queue->lock); + + return 0; +} + /* * Entry function from io_uring to handle the given passthrough command * (op code IORING_OP_URING_CMD) @@ -1303,6 +1481,12 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) pr_info_once("FUSE_IO_URING_CMD_ADD_QUEUE failed err=%d\n", err); return err; + case FUSE_IO_URING_CMD_ADD_BUFPOOL: + err = fuse_uring_add_bufpool(cmd, fch); + if (err) + pr_info_once("FUSE_IO_URING_ADD_BUFPOOL failed err=%d\n", + err); + return err; default: return -EINVAL; } @@ -1336,6 +1520,7 @@ static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw) spin_lock(&queue->lock); list_del_init(&ent->list); + fuse_uring_recycle_buffer(ent); spin_unlock(&queue->lock); io_uring_cmd_done(cmd, err, issue_flags); @@ -1397,15 +1582,16 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) req->ring_queue = queue; ent = list_first_entry_or_null(&queue->ent_avail_queue, struct fuse_ring_ent, list); - if (ent) - fuse_uring_add_req_to_ring_ent(ent, req); - else + + if (!ent || fuse_uring_prep_buffer(ent, req)) { list_add_tail(&req->list, &queue->fuse_req_queue); + spin_unlock(&queue->lock); + return; + } + + fuse_uring_add_req_to_ring_ent(ent, req); spin_unlock(&queue->lock); - - if (ent) - fuse_uring_dispatch_ent(ent); - + fuse_uring_dispatch_ent(ent); return; err_unlock: @@ -1453,10 +1639,9 @@ bool fuse_uring_queue_bq_req(struct fuse_req *req) */ req = list_first_entry_or_null(&queue->fuse_req_queue, struct fuse_req, list); - if (ent && req) { + if (ent && req && !fuse_uring_prep_buffer(ent, req)) { fuse_uring_add_req_to_ring_ent(ent, req); spin_unlock(&queue->lock); - fuse_uring_dispatch_ent(ent); } else { spin_unlock(&queue->lock); diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index d721a4fc0215..cdf56f8b38b5 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -7,6 +7,8 @@ #ifndef _FS_FUSE_DEV_URING_I_H #define _FS_FUSE_DEV_URING_I_H +#include + #include "fuse_dev_i.h" #ifdef CONFIG_FUSE_IO_URING @@ -36,11 +38,38 @@ enum fuse_ring_req_state { FRRS_RELEASED, }; +/* how a queue's payload buffers are provided */ +enum fuse_queue_payload_mode { + /* not yet committed (a bufpool may still be added) */ + FUSE_PAYLOAD_UNSET = 0, + /* each entry registers its own payload buffer */ + FUSE_PAYLOAD_PER_ENT, + /* each entry's payload buffer is assigned from a bufpool */ + FUSE_PAYLOAD_BUFPOOL, +}; + +struct fuse_bufpool { + /* starting uaddr of the bufpool */ + uintptr_t base_uaddr; + + /* size of each buffer in the pool */ + size_t buf_size; + + /* total number of buffers in the pool */ + unsigned int nr_bufs; + + /* bitmap tracking which buffers are free */ + unsigned long free_map[]; +}; + /** A fuse ring entry, part of the ring queue */ struct fuse_ring_ent { /* userspace buffer */ struct fuse_uring_req_header __user *headers; - void __user *payload; + struct iovec payload; + + /* buffer id in the pool, if bufpools are used. ignored otherwise */ + unsigned int buf_id; /* the ring queue that owns the request */ struct fuse_ring_queue *queue; @@ -99,6 +128,12 @@ struct fuse_ring_queue { unsigned int active_background; bool stopped; + + /* how this queue's payload buffers are provided */ + enum fuse_queue_payload_mode payload_mode; + + /* only allocated when payload_mode == FUSE_PAYLOAD_BUFPOOL */ + struct fuse_bufpool *bufpool; }; /* diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 33773c7d129a..9779adc98593 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1482,7 +1482,7 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm) * the reply - server is either sending IORING_OP_URING_CMD or not. */ if (fuse_uring_enabled()) - flags |= FUSE_OVER_IO_URING; + flags |= FUSE_OVER_IO_URING | FUSE_HAS_IO_URING_BUFPOOL; ia->in.flags = flags; ia->in.flags2 = flags >> 32; diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index cfb055c0c764..538d844da099 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -243,6 +243,9 @@ * * 7.46 * - add FUSE_IO_URING_CMD_ADD_QUEUE + * - add FUSE_HAS_IO_URING_BUFPOOL + * - add fuse_uring_cmd_req bufpool struct + * - add bufpool offset field to fuse_uring_ent_in_out struct */ #ifndef _LINUX_FUSE_H @@ -451,6 +454,7 @@ struct fuse_file_lock { * FUSE_OVER_IO_URING: Indicate that client supports io-uring * FUSE_REQUEST_TIMEOUT: kernel supports timing out requests. * init_out.request_timeout contains the timeout (in secs) + * FUSE_HAS_IO_URING_BUFPOOL: kernel supports io-uring buffer pools */ #define FUSE_ASYNC_READ (1 << 0) #define FUSE_POSIX_LOCKS (1 << 1) @@ -498,6 +502,7 @@ struct fuse_file_lock { #define FUSE_ALLOW_IDMAP (1ULL << 40) #define FUSE_OVER_IO_URING (1ULL << 41) #define FUSE_REQUEST_TIMEOUT (1ULL << 42) +#define FUSE_HAS_IO_URING_BUFPOOL (1ULL << 43) /** * CUSE INIT request/reply flags @@ -1266,7 +1271,9 @@ struct fuse_uring_ent_in_out { /* size of user payload buffer */ uint32_t payload_sz; - uint32_t padding; + + /* Offset into the bufpool, if bufpools are used */ + uint32_t offset; uint64_t reserved; }; @@ -1298,6 +1305,9 @@ enum fuse_uring_cmd { /* add a queue */ FUSE_IO_URING_CMD_ADD_QUEUE = 3, + + /* add a bufpool to a queue */ + FUSE_IO_URING_CMD_ADD_BUFPOOL = 4, }; /** @@ -1312,6 +1322,15 @@ struct fuse_uring_cmd_req { /* queue the command is for (queue index) */ uint16_t qid; uint8_t padding[6]; + + union { + struct { + /* base address of bufpool */ + uint64_t uaddr; + uint32_t len; + uint32_t reserved; + } bufpool; + }; }; #endif /* _LINUX_FUSE_H */ From 96caf2496e15b3b12e1e4f3ac592648291333ecb Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 14 Aug 2026 11:59:44 -0700 Subject: [PATCH 17/35] fuse: support registered buffer pools in io-uring Allow servers to use a buffer pool that is also registered through io-uring. When the server registers a buffer pool with io-uring, the pages backing the pool are pinned upfront. This eliminates the overhead of pinning/unpinning user pages and translating virtual addresses per i/o request. This also allows servers to use the same registered memory for subsequent backing store I/O (eg read_fixed/write_fixed), keeping data in the same pinned pages without additional pinning or mapping overhead required. To use this, the server needs to set the FUSE_URING_REGISTERED_BUFPOOL flag when adding a bufpool through the FUSE_IO_URING_CMD_ADD_BUFPOOL cmd. For every sqe submitted (including the one for adding the bufpool), it should set sqe->uring_cmd_flags to include IORING_URING_CMD_FIXED, and pass in the index where the registered bufpool resides to sqe->buf_index. Benchmarked with passthrough_hp (--nopassthrough, q_depth=8) on a 2-socket Intel Xeon Gold 6138 (40 cores / 80 threads), using fio (sync engine, bs=1M, O_DIRECT, numjobs=2, 30s run + 10s ramp, 3 runs) where direct-I/O throughput is against a RAM-backed (tmpfs) source (backing I/O is not the bottleneck): baseline registered buffers direct read ~5.1 GB/s ~5.4 GB/s (+~5%) direct write ~3.4 GB/s ~4.8 GB/s (+~45%) Registered buffers bring up the write path speed up closer to speed of reads. There isn't much improvement for reads because it is already fast enough where it's at the copy-bound ceiling (surpassing that requires doing zero-copy). On a device-bound NVMe though, the differences are within noise, as backing I/O dominates per-request latency. Signed-off-by: Joanne Koong Reviewed-by: Bernd Schubert Signed-off-by: Miklos Szeredi --- fs/fuse/dev_uring.c | 112 ++++++++++++++++++++++++++++++++++-------- fs/fuse/dev_uring_i.h | 8 +++ 2 files changed, 99 insertions(+), 21 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index d300c7f441c4..17806da93039 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -47,6 +47,27 @@ static inline bool bufpool_enabled(struct fuse_ring_queue *queue) return queue->payload_mode == FUSE_PAYLOAD_BUFPOOL; } +static inline bool bufpool_registered(struct fuse_ring_queue *queue) +{ + return queue->bufpool && queue->bufpool->registered; +} + +/* + * For a registered bufpool, every sqe that drives a payload import (REGISTER, + * COMMIT_AND_FETCH) must carry the registered buffer index of the pool. + * This also must be called from the command's issue handler, where cmd->sqe is + * still valid + */ +static inline bool fuse_uring_cmd_index_ok(struct io_uring_cmd *cmd, + struct fuse_ring_queue *queue) +{ + if (!bufpool_registered(queue)) + return true; + + return (cmd->flags & IORING_URING_CMD_FIXED) && + READ_ONCE(cmd->sqe->buf_index) == queue->bufpool->registered_index; +} + static void uring_cmd_set_ring_ent(struct io_uring_cmd *cmd, struct fuse_ring_ent *ring_ent) { @@ -653,19 +674,42 @@ static int copy_header_from_ring(struct fuse_ring_ent *ent, return 0; } +static int fuse_uring_import_payload(struct fuse_ring_ent *ent, int dir, + struct iov_iter *iter, + unsigned int issue_flags) +{ + void __user *base = ent->payload.iov_base; + size_t len = ent->payload.iov_len; + int err = 0; + + if (!base) { + memset(iter, 0, sizeof(*iter)); + return 0; + } + + if (bufpool_registered(ent->queue)) + err = io_uring_cmd_import_fixed((u64)(uintptr_t)base, len, dir, + iter, ent->cmd, issue_flags); + else + err = import_ubuf(dir, base, len, iter); + + if (err) + pr_info_ratelimited("fuse: Import of user buffer failed\n"); + + return err; +} + static int setup_fuse_copy_state(struct fuse_copy_state *cs, struct fuse_req *req, struct fuse_ring_ent *ent, int dir, - struct iov_iter *iter) + struct iov_iter *iter, + unsigned int issue_flags) { int err; - err = import_ubuf(dir, ent->payload.iov_base, ent->payload.iov_len, - iter); - if (err) { - pr_info_ratelimited("fuse: Import of user buffer failed\n"); + err = fuse_uring_import_payload(ent, dir, iter, issue_flags); + if (err) return err; - } fuse_copy_init(cs, dir == ITER_DEST, iter); @@ -676,7 +720,8 @@ static int setup_fuse_copy_state(struct fuse_copy_state *cs, } static int fuse_uring_copy_from_ring(struct fuse_req *req, - struct fuse_ring_ent *ent) + struct fuse_ring_ent *ent, + unsigned int issue_flags) { struct fuse_copy_state cs; struct fuse_args *args = req->args; @@ -689,7 +734,8 @@ static int fuse_uring_copy_from_ring(struct fuse_req *req, if (err) return err; - err = setup_fuse_copy_state(&cs, req, ent, ITER_SOURCE, &iter); + err = setup_fuse_copy_state(&cs, req, ent, ITER_SOURCE, &iter, + issue_flags); if (err) return err; @@ -702,7 +748,8 @@ static int fuse_uring_copy_from_ring(struct fuse_req *req, * Copy data from the req to the ring buffer */ static int fuse_uring_args_to_ring(struct fuse_req *req, - struct fuse_ring_ent *ent) + struct fuse_ring_ent *ent, + unsigned int issue_flags) { struct fuse_copy_state cs; struct fuse_args *args = req->args; @@ -715,7 +762,8 @@ static int fuse_uring_args_to_ring(struct fuse_req *req, .commit_id = req->in.h.unique, }; - err = setup_fuse_copy_state(&cs, req, ent, ITER_DEST, &iter); + err = setup_fuse_copy_state(&cs, req, ent, ITER_DEST, &iter, + issue_flags); if (err) return err; @@ -754,7 +802,8 @@ static int fuse_uring_args_to_ring(struct fuse_req *req, } static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, - struct fuse_req *req) + struct fuse_req *req, + unsigned int issue_flags) { struct fuse_ring_queue *queue = ent->queue; int err; @@ -771,7 +820,7 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, return err; /* copy the request */ - err = fuse_uring_args_to_ring(req, ent); + err = fuse_uring_args_to_ring(req, ent, issue_flags); if (unlikely(err)) { pr_info_ratelimited("Copy to ring failed: %d\n", err); return err; @@ -868,11 +917,12 @@ static int fuse_uring_prep_buffer(struct fuse_ring_ent *ent, } static int fuse_uring_prepare_send(struct fuse_ring_ent *ent, - struct fuse_req *req) + struct fuse_req *req, + unsigned int issue_flags) { int err; - err = fuse_uring_copy_to_ring(ent, req); + err = fuse_uring_copy_to_ring(ent, req, issue_flags); if (!err) { set_bit(FR_SENT, &req->flags); trace_fuse_request_sent(req); @@ -983,7 +1033,7 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, goto out; } - err = fuse_uring_copy_from_ring(req, ent); + err = fuse_uring_copy_from_ring(req, ent, issue_flags); out: fuse_uring_req_end(ent, req, err); } @@ -995,7 +1045,8 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, * Else, there is no next fuse request and this returns false. */ static bool fuse_uring_get_next_fuse_req(struct fuse_ring_ent *ent, - struct fuse_ring_queue *queue) + struct fuse_ring_queue *queue, + unsigned int issue_flags) { int err; struct fuse_req *req; @@ -1007,7 +1058,7 @@ static bool fuse_uring_get_next_fuse_req(struct fuse_ring_ent *ent, spin_unlock(&queue->lock); if (req) { - err = fuse_uring_prepare_send(ent, req); + err = fuse_uring_prepare_send(ent, req, issue_flags); if (err) goto retry; } @@ -1081,6 +1132,11 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, return err; } + if (!fuse_uring_cmd_index_ok(cmd, queue)) { + spin_unlock(&queue->lock); + return -EINVAL; + } + /* Find a request based on the unique ID of the fuse request * This should get revised, as it needs a hash calculation and list * search. And full struct fuse_pqueue is needed (memory overhead). @@ -1126,7 +1182,7 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, * available and the cmd only returns to userspace when there's a * next request and an available buffer. */ - if (fuse_uring_get_next_fuse_req(ent, queue)) + if (fuse_uring_get_next_fuse_req(ent, queue, issue_flags)) fuse_uring_send(ent, cmd, 0, issue_flags); return 0; } @@ -1252,7 +1308,8 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, spin_lock(&queue->lock); if (bufpool_enabled(queue)) { - if (payload->iov_base || payload->iov_len) { + if (payload->iov_base || payload->iov_len || + !fuse_uring_cmd_index_ok(cmd, queue)) { spin_unlock(&queue->lock); return ERR_PTR(err); } @@ -1362,6 +1419,7 @@ static int fuse_uring_add_bufpool(struct io_uring_cmd *cmd, uintptr_t pool_uaddr; unsigned int pool_len, nr_bufs; size_t pool_size, buf_size; + bool registered = cmd->flags & IORING_URING_CMD_FIXED; if (!ring || qid >= ring->nr_queues || flags) return -EINVAL; @@ -1396,6 +1454,17 @@ static int fuse_uring_add_bufpool(struct io_uring_cmd *cmd, /* all buffers are free */ bitmap_set(pool->free_map, 0, nr_bufs); + /* + * A registered bufpool is reached through an io_uring fixed buffer, so + * the pool is registered iff this command was submitted with + * IORING_URING_CMD_FIXED. The registered buffer index is taken from + * sqe->buf_index. + */ + if (registered) { + pool->registered = true; + pool->registered_index = READ_ONCE(cmd->sqe->buf_index); + } + spin_lock(&queue->lock); if (queue->payload_mode != FUSE_PAYLOAD_UNSET) { spin_unlock(&queue->lock); @@ -1508,9 +1577,10 @@ static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw) int err; if (!tw.cancel) { - err = fuse_uring_prepare_send(ent, ent->fuse_req); + err = fuse_uring_prepare_send(ent, ent->fuse_req, issue_flags); if (err) { - if (!fuse_uring_get_next_fuse_req(ent, queue)) + if (!fuse_uring_get_next_fuse_req(ent, queue, + issue_flags)) return; err = 0; } diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index cdf56f8b38b5..e142cae43022 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -49,6 +49,14 @@ enum fuse_queue_payload_mode { }; struct fuse_bufpool { + bool registered; + + /* + * io_uring registered buffer table index for this pool, bound at + * ADD_BUFPOOL time. Only valid if the bufpool is registered + */ + u16 registered_index; + /* starting uaddr of the bufpool */ uintptr_t base_uaddr; From 43f8343858eb942d7f7c49964b31c54dcc314890 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 14 Aug 2026 11:59:45 -0700 Subject: [PATCH 18/35] fuse: add zero-copy over io-uring Implement zero-copy in fuse io-uring to eliminate memory copies between the application, kernel, and server for read/write operations. The server can directly access client pages or page cache folios without copying data through an intermediary buffer. When a fuse request arrives, the kernel registers the relevant pages into a sparse slot in the server's io_uring registered buffer table. The server can then operate on these pages directly using io-uring fixed buffer operations (eg read_fixed/write_fixed) and the kernel unregisters these pages when the request completes. Non-page-backed args (eg op out headers) will go through the payload buffer as normal. The server can specify which open files should have their reads/writes go through zero-copy, by setting the FOPEN_IO_URING_ZERO_COPY flag when servicing opens. This requires CAP_SYS_ADMIN and bufpools. This is gated behind CAP_SYS_ADMIN because zero-copy allows the server direct access to the client's underlying pages, rather than operating on an intermediary buffer that the contents of the client's pages were copied into or on page cache folios. The request flow for the zero-copy direct-io write path (client writes data, server reads it) is as follows: ======================================================================= | Kernel | FUSE server | | | "write(fd, buf, 1MB)" | | | | >sys_write() | | >fuse_file_write_iter() | | >fuse_send_one() | | [req->args->in_pages = true] | | [folios hold client write data] | | | | >fuse_uring_copy_to_ring() | | >copy_header_to_ring(IN_OUT) | | [memcpy fuse_in_header] | | >copy_header_to_ring(OP) | | [memcpy write_in header] | | | | >fuse_uring_args_to_ring() | | >setup_fuse_copy_state() | | [skip_folio_copy = true] | | | | >fuse_uring_set_up_zero_copy() | | [folio_get for each client folio] | | [build bio_vec array from folios] | | >io_buffer_register_bvec() | | [register pages at ent->zero_copy_index] | | [ent->zero_copied = true] | | | | >fuse_copy_args() | | [skip_folio_copy => return 0 | | for page arg, skip data copy] | | | | >copy_header_to_ring(RING_ENT) | | [memcpy ent_in_out] | | >io_uring_cmd_done() | | | | | [CQE received] | | | | [issue io_uring READ at | | ent->zero_copy_index] | | [reads directly from | |client's pages (ZERO_COPY)] | | | | [write data to backing | | store] | | [submit COMMIT AND FETCH] | | | >fuse_uring_commit_fetch() | | >fuse_uring_commit() | | >fuse_uring_copy_from_ring() | | >fuse_uring_req_end() | | >io_buffer_unregister(ent->zero_copy_index) | | [unregister pages from index] | | >fuse_zero_copy_release() | | [folio_put for each folio] | | [ent->zero_copied = false] | | >fuse_request_end() | | [wake up client] | The zero-copy read path is analogous. Some requests may have both page-backed args and non-page-backed args. For these requests, the page-backed args are zero-copied while the non-page-backed args are copied to the buffer selected from the buffer pool: zero-copy: pages registered via io_buffer_register_bvec() non-page-backed: copied to payload buffer via fuse_copy_args() For a request whose payload is zero-copied, the registration/unregistration path looks like: register: fuse_uring_set_up_zero_copy() folio_get() for each folio io_buffer_register_bvec(ent->zero_copy_index) unregister: fuse_uring_req_end() io_buffer_unregister(ent->zero_copy_index) -> fuse_zero_copy_release() callback folio_put() for each folio Please note that on abort for in-flight zero-copied requests that have been sent to userspace, the registered bvec slot remains occupied and its folios remain pinned until the io-uring ring is destroyed, at which point io-uring unregisters all buffers and the fuse_zero_copy_release() callback drops the folio references. Unregistering at teardown would require operating on the ring context directly, whose validity is hard to ascertain; this is deemed not worth the complexity for the abort race, since everything is freed when the ring is torn down. The throughput improvement from zero-copy depends on how much of the per-request latency is spent on data copying vs backing I/O. The gain comes from eliminating the payload-buffer memcpy, but accessing the zero-copied pages requires the server to issue the read/write as an IORING_OP_READ/WRITE_FIXED operation. The benefit is largest when the mempcy is a meaningful fraction of per-request latency while backing i/o is still noticable enough that the extra io-uring op's overhead doesn't dominate. Benchmarked with passthrough_hp (--nopassthrough, q_depth=8) on a 2-socket Intel Xeon Gold 6138 (40 cores / 80 threads), using fio (sync engine, bs=1M, O_DIRECT, numjobs=2, 30s run + 10s ramp, 3 runs) where direct-I/O throughput is against a RAM-backed (tmpfs) source (backing I/O is not the bottleneck): baseline registered-buf zero-copy (zc vs base) direct read ~5.1 GB/s ~5.4 GB/s ~8.9 GB/s (+75%) direct write ~3.4 GB/s ~4.8 GB/s ~5.1 GB/s (+50%) Reads end up higher than writes because the backing store reads faster than it writes (the baseline shows the same read>write gap, and the raw device does too). On a device-bound NVMe (~2 GB/s reads) the read gain shrinks to ~10-16% (and no measurable gains for writes), as backing I/O rather than the eliminated copy dominates latency. The benefit overall scales with how much of the per-request latency is the data copy versus backing I/O. Signed-off-by: Joanne Koong Reviewed-by: Bernd Schubert Signed-off-by: Miklos Szeredi --- fs/fuse/args.h | 2 + fs/fuse/dev.c | 24 ++++- fs/fuse/dev_uring.c | 182 +++++++++++++++++++++++++++++++++++--- fs/fuse/dev_uring_i.h | 6 ++ fs/fuse/file.c | 2 + fs/fuse/fuse_dev_i.h | 2 + include/uapi/linux/fuse.h | 34 +++++++ 7 files changed, 236 insertions(+), 16 deletions(-) diff --git a/fs/fuse/args.h b/fs/fuse/args.h index ecfe51a192af..5173264a1261 100644 --- a/fs/fuse/args.h +++ b/fs/fuse/args.h @@ -42,6 +42,8 @@ struct fuse_args { bool is_pinned:1; bool invalidate_vmap:1; bool abort_on_kill:1; + /* server requested io-uring zero-copy for this op */ + bool zero_copy:1; struct fuse_in_arg in_args[4]; struct fuse_arg out_args[2]; void (*end)(struct fuse_args *args, int error); diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index d8f97943e973..90dceb7da571 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -1248,11 +1248,25 @@ int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop, if (folio) { size = folio_size(folio); - if (zeroing && count < size) - folio_zero_range(folio, 0, size); + if (zeroing && count < size) { + /* + * When the copy is skipped the folio already holds the + * payload, so only the bytes outside [offset, offset + + * count) may be zeroed. + * + * Otherwise, the whole folio is cleared first so that a + * failed copy leaves zeros rather than stale folio + * contents. + */ + if (cs->skip_folio_copy) + folio_zero_segments(folio, 0, offset, + offset + count, size); + else + folio_zero_range(folio, 0, size); + } } - while (count) { + while (!cs->skip_folio_copy && count) { if (cs->write && cs->pipebufs && folio) { /* * Can't control lifetime of pipe buffers, so always @@ -1345,6 +1359,10 @@ int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs, for (i = 0; !err && i < numargs; i++) { struct fuse_arg *arg = &args[i]; if (i == numargs - 1 && argpages) + /* + * if cs->skip_folio_copy is set, this just does any + * needed zeroing. No copying is involved. + */ err = fuse_copy_folios(cs, arg->size, zeroing); else err = fuse_copy_one(cs, arg->value, arg->size); diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 17806da93039..1547a4f0d9b9 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -22,6 +22,8 @@ MODULE_PARM_DESC(enable_uring, #define FUSE_URING_IOV_HEADERS 0 #define FUSE_URING_IOV_PAYLOAD 1 +#define FUSE_URING_ADD_QUEUE_FLAGS (FUSE_URING_ZERO_COPY) + bool fuse_uring_enabled(void) { return enable_uring; @@ -31,6 +33,11 @@ struct fuse_uring_pdu { struct fuse_ring_ent *ent; }; +struct fuse_zero_copy_bvs { + unsigned int nr_bvs; + struct bio_vec bvs[]; +}; + static const struct fuse_iqueue_ops fuse_io_uring_ops; enum fuse_uring_header_type { @@ -113,8 +120,36 @@ static void fuse_uring_flush_bg(struct fuse_ring_queue *queue) } } +static bool can_zero_copy_req(struct fuse_ring_ent *ent, struct fuse_req *req) +{ + struct fuse_args *args = req->args; + + if (!ent->queue->zero_copy || !args->zero_copy) + return false; + + if (args->opcode != FUSE_READ && args->opcode != FUSE_WRITE) + return false; + + return args->in_pages || args->out_pages; +} + +static void zero_copy_unregister(struct io_uring_cmd *cmd, + struct fuse_ring_ent *ent, + unsigned int issue_flags) +{ + if (ent->zero_copied) { + int err = io_buffer_unregister(cmd, ent->zero_copy_index, + issue_flags); + + if (err) + pr_warn_ratelimited("qid=%d zero-copy unregister failed: %d\n", + ent->queue->qid, err); + ent->zero_copied = false; + } +} + static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, - int error) + int error, unsigned int issue_flags) { struct fuse_ring_queue *queue = ent->queue; struct fuse_ring *ring = queue->ring; @@ -134,6 +169,8 @@ static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, spin_unlock(&queue->lock); + zero_copy_unregister(ent->cmd, ent, issue_flags); + if (error) req->out.h.error = error; @@ -309,7 +346,7 @@ void fuse_uring_conn_init(struct fuse_chan *fch) } static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, - int qid, + int qid, bool zero_copy, bool fail_if_exists) { struct fuse_chan *fch = ring->chan; @@ -328,6 +365,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, queue->qid = qid; queue->ring = ring; spin_lock_init(&queue->lock); + queue->zero_copy = zero_copy; INIT_LIST_HEAD(&queue->ent_avail_queue); INIT_LIST_HEAD(&queue->ent_commit_queue); @@ -713,6 +751,9 @@ static int setup_fuse_copy_state(struct fuse_copy_state *cs, fuse_copy_init(cs, dir == ITER_DEST, iter); + if (ent->zero_copied) + cs->skip_folio_copy = true; + cs->is_uring = true; cs->req = req; @@ -744,6 +785,62 @@ static int fuse_uring_copy_from_ring(struct fuse_req *req, return err; } +static void fuse_zero_copy_release(void *priv) +{ + struct fuse_zero_copy_bvs *zc_bvs = priv; + unsigned int i; + + for (i = 0; i < zc_bvs->nr_bvs; i++) + folio_put(page_folio(zc_bvs->bvs[i].bv_page)); + + kvfree(zc_bvs); +} + +static int fuse_uring_set_up_zero_copy(struct fuse_ring_ent *ent, + struct fuse_req *req, + unsigned int issue_flags) +{ + struct fuse_args_pages *ap; + int err, i, ddir = 0; + struct fuse_zero_copy_bvs *zc_bvs; + struct bio_vec *bvs; + + /* out_pages indicates a read, in_pages indicates a write */ + if (req->args->out_pages) + ddir |= IO_BUF_DEST; + if (req->args->in_pages) + ddir |= IO_BUF_SOURCE; + + ap = container_of(req->args, typeof(*ap), args); + + zc_bvs = kvmalloc_flex(*zc_bvs, bvs, ap->num_folios, + GFP_KERNEL_ACCOUNT); + if (!zc_bvs) + return -ENOMEM; + + zc_bvs->nr_bvs = ap->num_folios; + bvs = zc_bvs->bvs; + for (i = 0; i < ap->num_folios; i++) { + bvs[i].bv_page = folio_page(ap->folios[i], 0); + bvs[i].bv_offset = ap->descs[i].offset; + bvs[i].bv_len = ap->descs[i].length; + folio_get(ap->folios[i]); + } + + err = io_buffer_register_bvec(ent->cmd, bvs, ap->num_folios, + fuse_zero_copy_release, zc_bvs, + ddir, ent->zero_copy_index, + issue_flags); + if (err) { + fuse_zero_copy_release(zc_bvs); + return err; + } + + ent->zero_copied = true; + + return 0; +} + /* * Copy data from the req to the ring buffer */ @@ -762,6 +859,13 @@ static int fuse_uring_args_to_ring(struct fuse_req *req, .commit_id = req->in.h.unique, }; + if (can_zero_copy_req(ent, req)) { + ent_in_out.flags |= FUSE_URING_ENT_ZERO_COPY; + err = fuse_uring_set_up_zero_copy(ent, req, issue_flags); + if (err) + return err; + } + err = setup_fuse_copy_state(&cs, req, ent, ITER_DEST, &iter, issue_flags); if (err) @@ -793,6 +897,18 @@ static int fuse_uring_args_to_ring(struct fuse_req *req, } ent_in_out.payload_sz = cs.ring.copied_sz; + /* + * on a zero-copied write the pages are registered for the server to + * read via a fixed-buffer op rather than copied into the payload + * buffer, so copied_sz does not account for it. The server still needs + * the total inbound size to know how many bytes to read from the + * registered buffer, so add the page arg (always the last in-arg) back + * in + */ + if (cs.skip_folio_copy && args->in_pages) + ent_in_out.payload_sz += + args->in_args[args->in_numargs - 1].size; + if (bufpool_enabled(ent->queue) && ent->payload.iov_base) ent_in_out.offset = (uintptr_t)ent->payload.iov_base - ent->queue->bufpool->base_uaddr; @@ -831,11 +947,25 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, sizeof(req->in.h)); } -static bool fuse_uring_req_has_payload(struct fuse_req *req) +static bool fuse_uring_req_has_copyable_payload(struct fuse_ring_ent *ent, + struct fuse_req *req) { struct fuse_args *args = req->args; - return args->in_numargs > 1 || args->out_numargs; + if (!can_zero_copy_req(ent, req)) + return args->in_numargs > 1 || args->out_numargs; + + /* + * the asymmetry between in_numargs > 2 and out_numargs > 1 is because + * the per-op header is extracted before fuse_copy_args() for inargs but + * not for outargs + */ + if ((args->in_numargs > 1) && (!args->in_pages || args->in_numargs > 2)) + return true; + if (args->out_numargs && (!args->out_pages || args->out_numargs > 1)) + return true; + + return false; } static int fuse_uring_select_buffer(struct fuse_ring_ent *ent) @@ -892,7 +1022,7 @@ static int fuse_uring_next_req_update_buffer(struct fuse_ring_ent *ent, return 0; buffer_selected = !!ent->payload.iov_base; - has_payload = fuse_uring_req_has_payload(req); + has_payload = fuse_uring_req_has_copyable_payload(ent, req); if (has_payload && !buffer_selected) return fuse_uring_select_buffer(ent); @@ -910,7 +1040,7 @@ static int fuse_uring_prep_buffer(struct fuse_ring_ent *ent, return 0; /* no payload to copy, can skip selecting a buffer */ - if (!fuse_uring_req_has_payload(req)) + if (!fuse_uring_req_has_copyable_payload(ent, req)) return 0; return fuse_uring_select_buffer(ent); @@ -936,7 +1066,7 @@ static int fuse_uring_prepare_send(struct fuse_ring_ent *ent, ent->state = FRRS_INVALID; spin_unlock(&ent->queue->lock); - fuse_uring_req_end(ent, req, err); + fuse_uring_req_end(ent, req, err, issue_flags); } return err; @@ -1035,7 +1165,7 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, err = fuse_uring_copy_from_ring(req, ent, issue_flags); out: - fuse_uring_req_end(ent, req, err); + fuse_uring_req_end(ent, req, err, issue_flags); } /* @@ -1160,7 +1290,12 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, queue->qid, commit_id, ent->state); fuse_uring_recycle_buffer(ent); spin_unlock(&queue->lock); - fuse_uring_req_end(ent, req, err); + /* + * Unregister any zero copyable pages since ent->cmd is null + * when it hits fuse_uring_req_end() in this path + */ + zero_copy_unregister(cmd, ent, issue_flags); + fuse_uring_req_end(ent, req, err, issue_flags); return err; } @@ -1284,10 +1419,14 @@ static struct fuse_ring_ent * fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, struct fuse_ring_queue *queue) { + const struct fuse_uring_cmd_req *cmd_req = + io_uring_sqe128_cmd(cmd->sqe, struct fuse_uring_cmd_req); struct fuse_ring *ring = queue->ring; struct fuse_ring_ent *ent; struct iovec iov[FUSE_URING_IOV_SEGS]; struct iovec *headers, *payload; + unsigned int zero_copy_index; + int err; err = fuse_uring_get_iovec_from_sqe(cmd->sqe, iov); @@ -1297,6 +1436,10 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, return ERR_PTR(err); } + zero_copy_index = READ_ONCE(cmd_req->ent_zero_copy_buf_index); + if (zero_copy_index && !queue->zero_copy) + return ERR_PTR(-EINVAL); + err = -EINVAL; headers = &iov[FUSE_URING_IOV_HEADERS]; if (headers->iov_len < sizeof(struct fuse_uring_req_header)) { @@ -1315,9 +1458,14 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, } } else { if (payload->iov_len < ring->max_payload_sz) { + spin_unlock(&queue->lock); pr_info_ratelimited("Invalid req payload len %zu\n", payload->iov_len); + return ERR_PTR(err); + } + if (queue->zero_copy) { spin_unlock(&queue->lock); + pr_info_ratelimited("Can only use zero copy with bufpools\n"); return ERR_PTR(err); } queue->payload_mode = FUSE_PAYLOAD_PER_ENT; @@ -1335,6 +1483,7 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, ent->headers = headers->iov_base; if (queue->payload_mode == FUSE_PAYLOAD_PER_ENT) ent->payload = *payload; + ent->zero_copy_index = zero_copy_index; atomic_inc(&ring->queue_refs); return ent; @@ -1364,7 +1513,7 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, queue = READ_ONCE(ring->queues[qid]); if (!queue) { - queue = fuse_uring_create_queue(ring, qid, false); + queue = fuse_uring_create_queue(ring, qid, false, false); if (IS_ERR(queue)) return PTR_ERR(queue); } @@ -1389,8 +1538,9 @@ static int fuse_uring_add_queue(struct io_uring_cmd *cmd, struct fuse_chan *fch) unsigned int qid = READ_ONCE(cmd_req->qid); uint64_t flags = READ_ONCE(cmd_req->flags); struct fuse_ring_queue *queue; + bool zero_copy = flags & FUSE_URING_ZERO_COPY; - if (!ring || flags) + if (!ring) return -EINVAL; if (qid >= ring->nr_queues) { @@ -1398,7 +1548,13 @@ static int fuse_uring_add_queue(struct io_uring_cmd *cmd, struct fuse_chan *fch) return -EINVAL; } - queue = fuse_uring_create_queue(ring, qid, true); + if (flags & ~FUSE_URING_ADD_QUEUE_FLAGS) + return -EINVAL; + + if (zero_copy && !capable(CAP_SYS_ADMIN)) + return -EPERM; + + queue = fuse_uring_create_queue(ring, qid, zero_copy, true); if (IS_ERR(queue)) return PTR_ERR(queue); @@ -1595,7 +1751,7 @@ static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw) io_uring_cmd_done(cmd, err, issue_flags); - fuse_uring_req_end(ent, ent->fuse_req, err); + fuse_uring_req_end(ent, ent->fuse_req, err, issue_flags); kfree(ent); if (atomic_dec_and_test(&queue->ring->queue_refs)) wake_up_all(&queue->ring->stop_waitq); diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index e142cae43022..263d0f8b9714 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -79,6 +79,10 @@ struct fuse_ring_ent { /* buffer id in the pool, if bufpools are used. ignored otherwise */ unsigned int buf_id; + /* true if the request's pages are being zero-copied */ + bool zero_copied; + unsigned int zero_copy_index; + /* the ring queue that owns the request */ struct fuse_ring_queue *queue; @@ -142,6 +146,8 @@ struct fuse_ring_queue { /* only allocated when payload_mode == FUSE_PAYLOAD_BUFPOOL */ struct fuse_bufpool *bufpool; + + bool zero_copy; }; /* diff --git a/fs/fuse/file.c b/fs/fuse/file.c index da5859e8159d..7b883cf170ac 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -605,6 +605,7 @@ void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, args->out_argvar = true; args->out_numargs = 1; args->out_args[0].size = count; + args->zero_copy = ff->open_flags & FOPEN_IO_URING_ZERO_COPY; } static void fuse_release_user_pages(struct fuse_args_pages *ap, ssize_t nres, @@ -1151,6 +1152,7 @@ static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff, args->out_numargs = 1; args->out_args[0].size = sizeof(ia->write.out); args->out_args[0].value = &ia->write.out; + args->zero_copy = ff->open_flags & FOPEN_IO_URING_ZERO_COPY; } static unsigned int fuse_write_flags(struct kiocb *iocb) diff --git a/fs/fuse/fuse_dev_i.h b/fs/fuse/fuse_dev_i.h index 668c8391d61c..f41749f484df 100644 --- a/fs/fuse/fuse_dev_i.h +++ b/fs/fuse/fuse_dev_i.h @@ -325,6 +325,8 @@ struct fuse_copy_state { bool write:1; bool move_folios:1; bool is_uring:1; + /* set when the payload is zero-copied. folios are filled in place */ + bool skip_folio_copy:1; struct { unsigned int copied_sz; /* copied size into the user buffer */ } ring; diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index 538d844da099..7435e09c87fe 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -246,6 +246,8 @@ * - add FUSE_HAS_IO_URING_BUFPOOL * - add fuse_uring_cmd_req bufpool struct * - add bufpool offset field to fuse_uring_ent_in_out struct + * - add FUSE_URING_ZERO_COPY, FUSE_URING_ENT_ZERO_COPY, and + * FOPEN_IO_URING_ZERO_COPY flag */ #ifndef _LINUX_FUSE_H @@ -389,6 +391,12 @@ struct fuse_file_lock { * FOPEN_NOFLUSH: don't flush data cache on close (unless FUSE_WRITEBACK_CACHE) * FOPEN_PARALLEL_DIRECT_WRITES: Allow concurrent direct writes on the same inode * FOPEN_PASSTHROUGH: passthrough read/write io for this open file + * FOPEN_IO_URING_ZERO_COPY: use io-uring zero-copy for reads/writes on this + * open file. Honored only when the serving io-uring + * queue was set up for zero-copy + * (FUSE_URING_ZERO_COPY) and the request carries page + * payload. Otherwise reads/writes fall back to + * copying. */ #define FOPEN_DIRECT_IO (1 << 0) #define FOPEN_KEEP_CACHE (1 << 1) @@ -398,6 +406,7 @@ struct fuse_file_lock { #define FOPEN_NOFLUSH (1 << 5) #define FOPEN_PARALLEL_DIRECT_WRITES (1 << 6) #define FOPEN_PASSTHROUGH (1 << 7) +#define FOPEN_IO_URING_ZERO_COPY (1 << 8) /** * INIT request/reply flags @@ -1259,6 +1268,13 @@ struct fuse_supp_groups { #define FUSE_URING_IN_OUT_HEADER_SZ 128 #define FUSE_URING_OP_IN_OUT_SZ 128 +/** + * fuse_uring_ent_in_out flags + * + * FUSE_URING_ENT_ZERO_COPY: Set if the ent's payload is zero-copied + */ +#define FUSE_URING_ENT_ZERO_COPY (1 << 0) + /* Used as part of the fuse_uring_req_header */ struct fuse_uring_ent_in_out { uint64_t flags; @@ -1310,6 +1326,14 @@ enum fuse_uring_cmd { FUSE_IO_URING_CMD_ADD_BUFPOOL = 4, }; +/* + * fuse_uring_cmd_req flags for FUSE_IO_URING_CMD_ADD_QUEUE + * + * FUSE_URING_ZERO_COPY is only supported for queues with bufpools on privileged + * servers + */ +#define FUSE_URING_ZERO_COPY (1 << 0) + /** * In the 80B command area of the SQE. */ @@ -1330,6 +1354,16 @@ struct fuse_uring_cmd_req { uint32_t len; uint32_t reserved; } bufpool; + + /* + * Index of this entry's slot in the server's io_uring + * registered buffer table, where the kernel registers the + * request's pages for zero-copy. Set for + * FUSE_IO_URING_CMD_REGISTER cmds only, and only on queues + * created with FUSE_URING_ZERO_COPY. On a non-zero-copy queue + * this must be 0 + */ + uint16_t ent_zero_copy_buf_index; }; }; From 767094250c6c87cc5d1a9951bbfb80409759a956 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 14 Aug 2026 11:59:46 -0700 Subject: [PATCH 19/35] docs: fuse: document io-uring buffer pool and zero-copy uapi Add documentation for fuse over io-uring usage of buffer pools and zero-copy. Reviewed-by: Bernd Schubert Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- .../filesystems/fuse/fuse-io-uring.rst | 32 +++++ Documentation/filesystems/fuse/index.rst | 1 + .../fuse/uapi/fuse-uapi-io-uring.rst | 126 ++++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 Documentation/filesystems/fuse/uapi/fuse-uapi-io-uring.rst diff --git a/Documentation/filesystems/fuse/fuse-io-uring.rst b/Documentation/filesystems/fuse/fuse-io-uring.rst index d73dd0dbd238..29f98057500d 100644 --- a/Documentation/filesystems/fuse/fuse-io-uring.rst +++ b/Documentation/filesystems/fuse/fuse-io-uring.rst @@ -11,6 +11,9 @@ and works. For generic details about FUSE see fuse.rst. This document also covers the current interface, which is still in development and might change. +For the userspace protocol, see +Documentation/filesystems/fuse/uapi/fuse-uapi-io-uring.rst. + Limitations =========== As of now not all requests types are supported through io-uring, userspace @@ -95,5 +98,34 @@ Sending requests with CQEs | uring_cmd_flags`` and the index of the registered bufpool in +``sqe->buf_index``. Every SQE the server submits afterwards must follow the +same fixed-buffer protocol, carrying ``IORING_URING_CMD_FIXED`` and that same +``sqe->buf_index``. The same registered buffer can be reused for the server's +backing-store I/O as well (e.g. ``IORING_OP_READ_FIXED`` / +``IORING_OP_WRITE_FIXED``). + +Zero-copy +========= +Requirements: + +* The server must be privileged (``CAP_SYS_ADMIN``). +* A zero-copy queue: ``ADD_QUEUE`` with the ``FUSE_URING_ZERO_COPY`` flag set. +* A buffer pool: ``ADD_BUFPOOL``. +* For each entry, ``REGISTER`` with ``ent_zero_copy_buf_index`` set to the + index this entry uses in the server's io_uring registered-buffer table. + This is where the kernel registers the request's pages for the server to + access (it is separate from the payload pool). On a non-zero-copy queue this + field must be 0. + +Zero-copy is selected per open file. The server sets the open-file flag in +the ``FUSE_OPEN`` / ``FUSE_CREATE`` reply: + +``FOPEN_IO_URING_ZERO_COPY`` + Reads/writes on this open file should use zero-copy. + +For a request that is zero-copied, the kernel sets ``FUSE_URING_ENT_ZERO_COPY`` +in ``fuse_uring_ent_in_out.flags`` and places the request's pages at the +entry's ``ent_zero_copy_buf_index``. The server then issues +``IORING_OP_READ_FIXED`` / ``IORING_OP_WRITE_FIXED`` against that index to +transfer the data directly to/from the client's pages. + +For such a request, ``payload_sz`` includes the zero-copied page bytes +(transferred via the registered buffer at ``ent_zero_copy_buf_index``). Any +non-page-backed args (e.g. op headers) are still copied through the pool +payload buffer at ``offset``. From 6e64df0f73f1c070db816ef71ada96c847bca1c9 Mon Sep 17 00:00:00 2001 From: Ben Dooks Date: Tue, 23 Jun 2026 10:42:06 +0100 Subject: [PATCH 20/35] fuse: make dentry_tree_work static The dentry_tree_work is not exported, so make it static to remove the followign sparse warning: fs/fuse/dir.c:37:21: warning: symbol 'dentry_tree_work' was not declared. Should it be static? Signed-off-by: Ben Dooks Signed-off-by: Miklos Szeredi --- fs/fuse/dir.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index b689503bc880..d73e2bf65634 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -34,7 +34,7 @@ struct dentry_bucket { #define FUSE_HASH_BITS 5 #define FUSE_HASH_SIZE (1 << FUSE_HASH_BITS) static struct dentry_bucket dentry_hash[FUSE_HASH_SIZE]; -struct delayed_work dentry_tree_work; +static struct delayed_work dentry_tree_work; /* Minimum invalidation work queue frequency */ #define FUSE_DENTRY_INVAL_FREQ_MIN 5 From 64b0b5cacbd2fea88001464cb712c9dfc795b26e Mon Sep 17 00:00:00 2001 From: Rochan Avlur Date: Wed, 12 Aug 2026 20:58:36 -0700 Subject: [PATCH 21/35] fuse: wait for FR_FINISHED on abort_on_kill to prevent use-after-free The abort_on_kill path in request_wait_answer() calls fuse_abort_conn() and returns without waiting for FR_FINISHED. If fuse_dev_do_write() is concurrently processing the same request (FR_LOCKED set), the caller frees req->args while it is still being accessed, causing a use-after-free. Fix this by jumping to the existing wait_event(FR_FINISHED) instead of returning early. The wait will not hang because fuse_abort_conn() ensures all requests are ended. Reported-by: syzbot+d6540a3fa1626e11360d@syzkaller.appspotmail.com Fixes: 204aa22a686b ("fuse: abort on fatal signal during sync init") Cc: stable@vger.kernel.org Assisted-by: Claude:claude-opus-4-6 Signed-off-by: Rochan Avlur Signed-off-by: Miklos Szeredi --- fs/fuse/dev.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 90dceb7da571..34106f6e66a0 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -724,7 +724,7 @@ static void request_wait_answer(struct fuse_req *req) if (req->args->abort_on_kill) { fuse_chan_abort(fch, false); - return; + goto wait_for_finish; } if (test_bit(FR_URING, &req->flags)) @@ -735,6 +735,7 @@ static void request_wait_answer(struct fuse_req *req) return; } +wait_for_finish: /* * Either request is already in userspace, or it was forced. * Wait it out. From 9afeca0d569c9fc89d758fe7a9339d1e8afb1546 Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Mon, 17 Aug 2026 23:18:00 +0800 Subject: [PATCH 22/35] fuse: fix invalidate lock leak on setattr writeback failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fuse_do_setattr() takes filemap_invalidate_lock() for a DAX truncate (fault_blocked = true) and releases it at the out:/error: labels. But when a writeback flush is also needed, a write_inode_now() failure returns directly and leaks the lock, so any later fault or truncate on the file stalls on the stale rwsem. For example, truncate(2) on a setuid file reaches fuse_do_setattr() with both ATTR_SIZE and ATTR_MODE set: truncate(2) └─ do_truncate() ├─ dentry_needs_remove_privs() # S_ISUID └─ notify_change() # KILL_SUID -> ATTR_MODE └─ fuse_setattr() # no killpriv: │ # ia_valid |= ATTR_MODE └─ fuse_do_setattr() ├─ filemap_invalidate_lock() # IS_DAX && is_truncate └─ write_inode_now() # is_wb && ATTR_MODE └─ if (err) # e.g. daemon -> -EIO return err # <- lock leaked Fix this by adding an unlock label that releases the lock before returning the error, and use it for the fuse_dax_break_layouts() failure path as well. Fixes: 6ae330cad6ef ("virtiofs: serialize truncate/punch_hole and dax fault path") Cc: stable@vger.kernel.org # v5.10+ Signed-off-by: Baokun Li Signed-off-by: Miklos Szeredi --- fs/fuse/dir.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index d73e2bf65634..172261bad71e 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2170,10 +2170,8 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, filemap_invalidate_lock(mapping); fault_blocked = true; err = fuse_dax_break_layouts(inode, 0, -1); - if (err) { - filemap_invalidate_unlock(mapping); - return err; - } + if (err) + goto unlock; } if (attr->ia_valid & ATTR_OPEN) { @@ -2200,7 +2198,7 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, ATTR_TIMES_SET)) { err = write_inode_now(inode, true); if (err) - return err; + goto unlock; fuse_set_nowrite(inode); fuse_release_nowrite(inode); @@ -2308,6 +2306,7 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); +unlock: if (fault_blocked) filemap_invalidate_unlock(mapping); return err; From a927f1867e61b78f39f9da0bbba3c98c2ca151fe Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Mon, 17 Aug 2026 23:18:01 +0800 Subject: [PATCH 23/35] fuse: fix invalidate lock leak on open O_TRUNC DAX failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fuse_open() takes filemap_invalidate_lock() for a DAX truncate (dax_truncate = true) and releases it before the out_inode_unlock label. But when fuse_dax_break_layouts() fails, the goto out_inode_unlock skips the unlock and leaks the rwsem, so any later fault or truncate on the file stalls on the stale lock. fuse_dax_break_layouts() can fail with -ERESTARTSYS when a signal interrupts the wait for busy DAX pages to drain: open("file", O_RDWR | O_TRUNC) └─ fuse_open() ├─ filemap_invalidate_lock() # dax_truncate └─ fuse_dax_break_layouts() └─ dax_break_layout() └─ wait_page_idle() # TASK_INTERRUPTIBLE └─ fuse_wait_dax_page() # unlock, schedule, re-lock └─ signal → -ERESTARTSYS goto out_inode_unlock # <- lock leaked Fix this by moving filemap_invalidate_unlock() below the label so that all error paths release the lock, and rename the label to out_unlock as it now covers more than just the inode lock. Fixes: 2fdbb8dd0155 ("fuse: fix deadlock between atomic O_TRUNC and page invalidation") Cc: stable@vger.kernel.org # v6.0+ Signed-off-by: Baokun Li Signed-off-by: Miklos Szeredi --- fs/fuse/file.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 7b883cf170ac..3f67348d0f7e 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -272,7 +272,7 @@ static int fuse_open(struct inode *inode, struct file *file) filemap_invalidate_lock(inode->i_mapping); err = fuse_dax_break_layouts(inode, 0, -1); if (err) - goto out_inode_unlock; + goto out_unlock; } if (is_wb_truncate || dax_truncate) @@ -296,9 +296,9 @@ static int fuse_open(struct inode *inode, struct file *file) else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) invalidate_inode_pages2(inode->i_mapping); } +out_unlock: if (dax_truncate) filemap_invalidate_unlock(inode->i_mapping); -out_inode_unlock: if (is_wb_truncate || dax_truncate) inode_unlock(inode); From 4deb3edead0c0e172cc7349e8855d741d3c5e162 Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Fri, 14 Aug 2026 21:40:15 +0800 Subject: [PATCH 24/35] cuse: wait for pending RCU callbacks on module exit Since commit 053fc4f755ad ("fuse: fix UAF in rcu pathwalks"), fuse_conn_put() frees the fuse_conn through call_rcu() rather than synchronously. For cuse, fc->release is cuse_fc_release(), which lives in the cuse module. If the module is removed before the RCU grace period ends, the callback jumps into freed module memory: userspace / module unload | RCU softirq ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ close(/dev/cuse) | cuse_channel_release() | fuse_dev_release() | fuse_conn_put(fch->conn) | call_rcu(delayed_release) ------+---> callback queued | rmmod cuse | cuse_exit() | cuse_channel_destroy() | ... | return | | | | rcu_do_batch() | delayed_release() | fc->release() | -> cuse_fc_release() | ^^^ freed text! The freed module text is unmapped by vfree(), so the jump into the stale callback triggers a page-fault Oops. If the virtual address is subsequently reused, the callback could execute unrelated code (undefined behaviour). Fix this by calling rcu_barrier() in cuse_exit() so that any pending fuse_conn release callback completes before the module is removed. Fixes: 053fc4f755ad ("fuse: fix UAF in rcu pathwalks") Signed-off-by: Baokun Li Signed-off-by: Miklos Szeredi --- fs/fuse/cuse.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fs/fuse/cuse.c b/fs/fuse/cuse.c index 96d57735a79f..4079cf8e5974 100644 --- a/fs/fuse/cuse.c +++ b/fs/fuse/cuse.c @@ -654,6 +654,11 @@ static void __exit cuse_exit(void) { misc_deregister(&cuse_miscdev); class_destroy(cuse_class); + /* + * Wait for pending call_rcu() callbacks that call back into + * this module via fc->release (cuse_fc_release). + */ + rcu_barrier(); } module_init(cuse_init); From ed976994939f05ffbb6e9e5482adc788e679c0ea Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Tue, 11 Aug 2026 12:53:50 +0800 Subject: [PATCH 25/35] fuse: reject a duplicate fd= mount option fuse_opt_fd() stored the fuse device in ctx->fud and bumped its refcount unconditionally: ctx->fud = fuse_dev_grab(file); If fd= is given twice (two fsconfig FSCONFIG_SET_FD calls), the second call overwrites ctx->fud and grabs the new device, while the reference taken on the first device is never released - a permanent refcount leak that pins the first fuse_dev until reboot. Reject a second fd= outright. ctx is zeroed on allocation, so a non-NULL ctx->fud reliably means the option was already processed. Fixes: d42eb23b2ef9 ("fuse: don't require /dev/fuse fd to be kept open during mount") Signed-off-by: Baokun Li Reviewed-by: Jingbo Xu Signed-off-by: Miklos Szeredi --- fs/fuse/inode.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 9779adc98593..9d877b2e5c12 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -791,6 +791,9 @@ static int fuse_opt_fd(struct fs_context *fsc, struct file *file) { struct fuse_fs_context *ctx = fsc->fs_private; + if (ctx->fud) + return invalfc(fsc, "Multiple fd specified"); + if (file->f_op != &fuse_dev_operations) return invalfc(fsc, "fd is not a fuse device"); /* From 928f659a3e3650978a5b4829cc982324f72b474b Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Sat, 8 Aug 2026 12:13:39 +0800 Subject: [PATCH 26/35] fuse: check for NULL root inode in fuse_fill_super_submount fuse_iget() can return NULL when its inode allocation fails, but fuse_fill_super_submount() passed the result straight to get_fuse_inode() and decremented fi->nlookup without checking it: root = fuse_iget(sb, parent_fi->nodeid, ...); fi = get_fuse_inode(root); fi->nlookup--; Inside fuse_iget() the inode allocation can fail and return NULL. The submount root takes the iget5_locked() path, whose alloc_inode() can fail under memory pressure (the auto-submount branch can fail the same way in new_inode() or fuse_alloc_submount_lookup()): inode = iget5_locked(sb, nodeid, fuse_inode_eq, fuse_inode_set, &nodeid); if (!inode) return NULL; A NULL root makes get_fuse_inode() a container_of() on NULL and the nlookup decrement a write to a bogus address, oopsing the mount. With CONFIG_KASAN the following null pointer dereference is reported when the root inode allocation of an auto-submount fails (e.g. under memory pressure): ================================================================== BUG: KASAN: null-ptr-deref in fuse_get_tree_submount+0x656/0x8b0 Read of size 8 at addr 00000000000002b0 by task ls/942 CPU: 0 PID: 942 Comm: ls Tainted: G W 6.6 #15 Call Trace: fuse_get_tree_submount+0x656/0x8b0 vfs_get_tree+0x48/0x140 fc_mount+0x13/0x50 fuse_dentry_automount+0x7a/0xb0 __traverse_mounts+0xca/0x330 step_into+0x339/0xac0 path_lookupat+0xc5/0x2f0 filename_lookup+0x163/0x2a0 vfs_statx+0xd5/0x200 do_statx+0x83/0xd0 __x64_sys_statx+0xa0/0xc0 do_syscall_64+0x37/0x90 entry_SYSCALL_64_after_hwframe+0x78/0xe2 ================================================================== Return -ENOMEM instead; the caller tears down the partially built superblock on error, matching the other error returns in this function. Fixes: 1866d779d5d2 ("fuse: Allow fuse_fill_super_common() for submounts") Signed-off-by: Baokun Li Reviewed-by: Jingbo Xu Signed-off-by: Miklos Szeredi --- fs/fuse/inode.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 9d877b2e5c12..1c6ee01c6796 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1645,6 +1645,8 @@ static int fuse_fill_super_submount(struct super_block *sb, fuse_fill_attr_from_inode(&root_attr, parent_fi); root = fuse_iget(sb, parent_fi->nodeid, 0, &root_attr, 0, 0, fuse_get_evict_ctr(fm->fc)); + if (!root) + return -ENOMEM; /* * This inode is just a duplicate, so it is not looked up and * its nlookup should not be incremented. fuse_iget() does From 4332cf75e4dafbe0981356d1b898d23104acd5db Mon Sep 17 00:00:00 2001 From: Xuewen Yan Date: Fri, 31 Jul 2026 15:01:02 +0800 Subject: [PATCH 27/35] fuse: give wakeup hints to the scheduler for synchronous requests When a synchronous FUSE request is sent, the in-kernel client queues it on fiq->pending and wakes the userspace daemon sleeping in fuse_dev_do_read()->wait_event_interruptible_exclusive(fiq->waitq, ...). The client then blocks in request_wait_answer() waiting for the reply, so the waker is about to go to sleep: this is exactly the pattern that WF_SYNC is meant to optimise. As Peter Zijlstra explained in the earlier discussion [1], WF_SYNC is a hint that the waker is about to sleep and the waker and wakee share data, so stacking the woken thread on the current CPU is beneficial for cache locality instead of searching for an idle one. Add a wake_up_sync() wrapper for task on the synchronous request path. Performance: On an Android big.LITTLE device where the FUSE daemon (MediaProvider) runs as a background service on the little cores while foreground applications run on the big cores, the synchronous wakeup hint lets the scheduler pull the daemon thread onto the big core that is issuing the request, where the request data is cache-hot. Measured by qixiaoyu [2] on a 2000-picture zip decompression to /sdcard: ------------------------------------------ | Default | patched | Improvement | ------------------------------------------ | 13.0 s | 7.0 s | 46% | ------------------------------------------ Server thread wall duration: 3583 ms -> 1276 ms Server runs on big core: 5% -> 79% The original 4K-file copy/compress/decompress workload [1] on the same kind of device showed a ~28% improvement (13.8s -> 9.9s). Note: Miklos reported [2] that on his test box he could not observe an actual migration from wake_up_interruptible_sync(); the benefit appears to be most visible on asymmetric topologies (big.LITTLE, where the daemon normally lives on a little core) and on workloads dominated by small synchronous requests. No regression was reported on the symmetric- SMP test setups tried. The earlier version of this change [1] added a `bool sync` argument to all three hooks of `struct fuse_iqueue_ops` and threaded it through virtio_fs as well. Miklos questioned the interface churn, and the patch has been stalled since. Re-work it so the exported interface is left alone. The hint is carried in a new FR_SYNC_WAKEUP bit of the existing `fuse_req->flags` bitfield (an `unsigned long`, so no layout change): - __fuse_request_send() sets the flag before fuse_send_one(). - fuse_dev_queue_req() consumes it with test_and_clear_bit() and forwards the result to fuse_dev_wake_and_unlock(), which then picks wake_up_sync() or wake_up(). - The forget, interrupt and resend paths pass `false` explicitly, preserving their original wake_up() behaviour. Only /dev/fuse ever wakes fiq->waitq; virtio_fs and fuse_uring dispatch through their own transport and never call wake_up(), so threading `sync` through their ops would just add an unused argument. test_and_clear_bit() makes the flag a one-shot hint that cannot leak into a future requeue, and no extra cleanup is needed in fuse_request_end()/fuse_put_request(). [1] https://lore.kernel.org/lkml/1638780405-38026-1-git-send-email-quic_pragalla@quicinc.com/ [2] https://lore.kernel.org/lkml/20221222093407.GA1141@mi-HP-ProDesk-680-G4-MT/ This work is based on "Pradeep P V K " and "Pavankumar Kondeti " Assisted-by: TRAE:GLM-5.2 Signed-off-by: Xuewen Yan Signed-off-by: Miklos Szeredi --- fs/fuse/dev.c | 22 ++++++++++++++++------ fs/fuse/fuse_dev_i.h | 3 +++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 34106f6e66a0..3f498630f421 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -213,10 +213,13 @@ EXPORT_SYMBOL_GPL(fuse_req_hash); /* * A new request is available, wake fiq->waitq */ -static void fuse_dev_wake_and_unlock(struct fuse_iqueue *fiq) +static void fuse_dev_wake_and_unlock(struct fuse_iqueue *fiq, bool sync) __releases(fiq->lock) { - wake_up(&fiq->waitq); + if (sync) + wake_up_sync(&fiq->waitq); + else + wake_up(&fiq->waitq); kill_fasync(&fiq->fasync, SIGIO, POLL_IN); spin_unlock(&fiq->lock); } @@ -233,7 +236,7 @@ void fuse_dev_queue_forget(struct fuse_iqueue *fiq, if (fiq->connected) { fiq->forget_list_tail->next = forget; fiq->forget_list_tail = forget; - fuse_dev_wake_and_unlock(fiq); + fuse_dev_wake_and_unlock(fiq, false); } else { kfree(forget); spin_unlock(&fiq->lock); @@ -255,7 +258,7 @@ void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req) list_del_init(&req->intr_entry); spin_unlock(&fiq->lock); } else { - fuse_dev_wake_and_unlock(fiq); + fuse_dev_wake_and_unlock(fiq, false); } } else { spin_unlock(&fiq->lock); @@ -285,11 +288,13 @@ EXPORT_SYMBOL_GPL(fuse_request_assign_unique); static void fuse_dev_queue_req(struct fuse_iqueue *fiq, struct fuse_req *req) { + bool sync = test_and_clear_bit(FR_SYNC_WAKEUP, &req->flags); + spin_lock(&fiq->lock); if (fiq->connected) { fuse_request_assign_unique_locked(fiq, req); list_add_tail(&req->list, &fiq->pending); - fuse_dev_wake_and_unlock(fiq); + fuse_dev_wake_and_unlock(fiq, sync); } else { spin_unlock(&fiq->lock); req->out.h.error = -ENOTCONN; @@ -752,6 +757,11 @@ static void __fuse_request_send(struct fuse_req *req) /* acquire extra reference, since request is still needed after fuse_request_end() */ __fuse_get_request(req); + /* + * This is a synchronous request: the caller will block waiting for + * the answer. Hint the scheduler via wake_up_sync(). + */ + set_bit(FR_SYNC_WAKEUP, &req->flags); fuse_send_one(fiq, req); request_wait_answer(req); @@ -1824,7 +1834,7 @@ void fuse_chan_resend(struct fuse_chan *fch) } /* iq and pq requests are both oldest to newest */ list_splice(&to_queue, &fiq->pending); - fuse_dev_wake_and_unlock(fiq); + fuse_dev_wake_and_unlock(fiq, false); } /* Look up request on processing list by unique ID */ diff --git a/fs/fuse/fuse_dev_i.h b/fs/fuse/fuse_dev_i.h index f41749f484df..4b412a76225f 100644 --- a/fs/fuse/fuse_dev_i.h +++ b/fs/fuse/fuse_dev_i.h @@ -38,6 +38,8 @@ struct fuse_iqueue; * @FR_PRIVATE: request is on private list * @FR_ASYNC: request is asynchronous * @FR_URING: request is handled through fuse-io-uring + * @FR_SYNC_WAKEUP: use synchronous wakeup when queueing this request to + * give the scheduler a hint about the waker task */ enum fuse_req_flag { FR_ISREPLY, @@ -53,6 +55,7 @@ enum fuse_req_flag { FR_PRIVATE, FR_ASYNC, FR_URING, + FR_SYNC_WAKEUP, }; /** From fd10f40af314f07b6d6e028b1ca25c8b49903aab Mon Sep 17 00:00:00 2001 From: Xiang Mei Date: Mon, 27 Jul 2026 16:37:04 -0700 Subject: [PATCH 28/35] fuse: copy request headers via a stack buffer for io-uring The fuse-io-uring transport copies req->in.h out to the ring in fuse_uring_copy_to_ring() and req->out.h back in fuse_uring_commit(). Both headers live inside the fuse_request slab object, whose cache (fuse_req_cachep) is created without a usercopy whitelist, so copying them directly to/from userspace trips CONFIG_HARDENED_USERCOPY and panics: usercopy: Kernel memory exposure attempt detected from SLUB object 'fuse_request' (offset 56, size 40)! kernel BUG at mm/usercopy.c:102! Oops: invalid opcode: 0000 [#1] SMP KASAN NOPTI RIP: 0010:usercopy_abort (mm/usercopy.c:90) Call Trace: __check_heap_object (mm/slub.c:8268) __check_object_size (mm/usercopy.c:197 mm/usercopy.c:258 mm/usercopy.c:223) copy_header_to_ring (fs/fuse/dev_uring.c:618) fuse_uring_prepare_send (fs/fuse/dev_uring.c:776 fs/fuse/dev_uring.c:785) fuse_uring_send_in_task (fs/fuse/dev_uring.c:1306) tctx_task_work_run (io_uring/tw.c:96) task_work_run (kernel/task_work.c:233) io_run_task_work (io_uring/tw.h:84) io_cqring_wait (io_uring/wait.c:278) __do_sys_io_uring_enter (io_uring/io_uring.c:2685) entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:121) Bounce both headers through an on-stack copy so the usercopy touches stack memory, not the slab object. Fixes: c090c8abae4b ("fuse: Add io-uring sqe commit and fetch support") Cc: stable@vger.kernel.org Reported-by: Weiming Shi Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Xiang Mei Reviewed-by: Bernd Schubert Reviewed-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/dev_uring.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 1547a4f0d9b9..e22a48c9a678 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -922,6 +922,7 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, unsigned int issue_flags) { struct fuse_ring_queue *queue = ent->queue; + struct fuse_in_header in_header; int err; err = -EIO; @@ -943,8 +944,9 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, } /* copy fuse_in_header */ - return copy_header_to_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->in.h, - sizeof(req->in.h)); + in_header = req->in.h; + return copy_header_to_ring(ent, FUSE_URING_HEADER_IN_OUT, &in_header, + sizeof(in_header)); } static bool fuse_uring_req_has_copyable_payload(struct fuse_ring_ent *ent, @@ -1151,11 +1153,13 @@ static struct fuse_req *fuse_uring_ent_assign_req(struct fuse_ring_ent *ent) static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, unsigned int issue_flags) { + struct fuse_out_header out_header; ssize_t err = -EFAULT; - if (copy_header_from_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->out.h, - sizeof(req->out.h))) + if (copy_header_from_ring(ent, FUSE_URING_HEADER_IN_OUT, &out_header, + sizeof(out_header))) goto out; + req->out.h = out_header; err = fuse_uring_out_header_has_err(&req->out.h, req); if (err) { From b77334654027ef56583e257e26e6d8a6ed8f9830 Mon Sep 17 00:00:00 2001 From: Sang-Heon Jeon Date: Sat, 1 Aug 2026 00:44:36 +0900 Subject: [PATCH 29/35] fuse: use min_not_zero() in fuse_init_server_timeout() fuse_init_server_timeout() limits timeout to fuse_max_req_timeout with the same logic as min_not_zero(), and returns early exactly when the computed timeout would be zero. So use min_not_zero() instead and return when the computed timeout is zero. No functional change. Signed-off-by: Sang-Heon Jeon Reviewed-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/req_timeout.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/fs/fuse/req_timeout.c b/fs/fuse/req_timeout.c index 6cc6fc491343..95a1acd7bc08 100644 --- a/fs/fuse/req_timeout.c +++ b/fs/fuse/req_timeout.c @@ -128,18 +128,12 @@ static void set_request_timeout(struct fuse_chan *fch, unsigned int timeout) void fuse_init_server_timeout(struct fuse_chan *fch, unsigned int timeout) { - if (!timeout && !fuse_max_req_timeout && !fuse_default_req_timeout) - return; - if (!timeout) timeout = fuse_default_req_timeout; - if (fuse_max_req_timeout) { - if (timeout) - timeout = min(fuse_max_req_timeout, timeout); - else - timeout = fuse_max_req_timeout; - } + timeout = min_not_zero(timeout, fuse_max_req_timeout); + if (!timeout) + return; timeout = max(FUSE_TIMEOUT_TIMER_FREQ, timeout); From d1dbc59200b54944f00251ca4dfbb2b318beca13 Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Sat, 1 Aug 2026 16:24:51 +0800 Subject: [PATCH 30/35] fuse: wake one waiter per freed slot when raising max_background MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fuse_get_req() parks background allocations on fch->blocked_waitq via wait_event_state_exclusive(), so each wakeup releases exactly one waiter. fuse_chan_max_background_set() clears fch->blocked when the new limit exceeds num_background, but the accompanying wake_up() releases a single waiter regardless of how many slots just became available. Raising max_background from 10 to 100 therefore admits one request instead of ninety. The remaining waiters are not permanently stranded — the "else if (!fch->blocked)" branch in fuse_request_end() wakes one more per completion — but that only helps while requests keep completing. Consider a fixed pool of threads doing readahead or async direct I/O with the quota exhausted: every thread is either in flight or parked, and each completion wakes one waiter while freeing one slot, a net change of zero. num_background oscillates around the old limit and the added quota is never taken up. Waking one waiter per freed slot also preserves submission order: once fch->blocked is clear, new callers of fuse_get_req() skip the waitqueue entirely, overtaking waiters that parked before the limit was raised. Use wake_up_nr() with the number of slots that just became available. Since the wakeup is guarded by !fch->blocked, num_background is strictly below max_background, so the count is at least 1 and never degenerates into wake_up_all(). Signed-off-by: Baokun Li Reviewed-By: Horst Birthelmer Signed-off-by: Miklos Szeredi --- fs/fuse/dev.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 3f498630f421..4fec31fc0b84 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -406,7 +406,8 @@ void fuse_chan_max_background_set(struct fuse_chan *fch, unsigned int val) fch->max_background = val; fch->blocked = fch->num_background >= fch->max_background; if (!fch->blocked) - wake_up(&fch->blocked_waitq); + wake_up_nr(&fch->blocked_waitq, + fch->max_background - fch->num_background); spin_unlock(&fch->bg_lock); } From 60dbcce1567f6502215b36ba4066e8fd1f86d706 Mon Sep 17 00:00:00 2001 From: Jimmy Zuber Date: Fri, 31 Jul 2026 20:38:42 +0000 Subject: [PATCH 31/35] selftests/fuse: test post-EOF page zeroing when a file is extended Add a regression test for the bug where extending a file left the tail of the old partial EOF page exposing stale mmap-dirtied data instead of zeros. The test is a self-contained raw /dev/fuse server (no libfuse dependency) that runs without writeback_cache and returns FOPEN_KEEP_CACHE, the configuration in which the bug is visible. Its backing data is always zero in the hole, so any non-zero byte a read sees is stale page-cache data. All offsets are relative to the runtime page size. Four cases: - write_extend: pollute the post-EOF tail, extend past it by writing into a later page, and verify the tail reads back as zero; - ftruncate_extend: same, but extend via ftruncate(); - fallocate_extend: same, but extend via fallocate() at the old EOF; - extend_into_eof_page_preserves_data: an extending write landing inside the old EOF page must not be clobbered by the zeroing. Each case fails without the fix and passes with it. Signed-off-by: Jimmy Zuber Signed-off-by: Miklos Szeredi --- .../selftests/filesystems/fuse/.gitignore | 1 + .../selftests/filesystems/fuse/Makefile | 3 + .../filesystems/fuse/write_extend_eof_test.c | 368 ++++++++++++++++++ 3 files changed, 372 insertions(+) create mode 100644 tools/testing/selftests/filesystems/fuse/write_extend_eof_test.c diff --git a/tools/testing/selftests/filesystems/fuse/.gitignore b/tools/testing/selftests/filesystems/fuse/.gitignore index 3e72e742d08e..fb51603fe419 100644 --- a/tools/testing/selftests/filesystems/fuse/.gitignore +++ b/tools/testing/selftests/filesystems/fuse/.gitignore @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only fuse_mnt fusectl_test +write_extend_eof_test diff --git a/tools/testing/selftests/filesystems/fuse/Makefile b/tools/testing/selftests/filesystems/fuse/Makefile index 612aad69a93a..0c2c613af0ac 100644 --- a/tools/testing/selftests/filesystems/fuse/Makefile +++ b/tools/testing/selftests/filesystems/fuse/Makefile @@ -3,10 +3,13 @@ CFLAGS += -Wall -O2 -g $(KHDR_INCLUDES) TEST_GEN_PROGS := fusectl_test +TEST_GEN_PROGS += write_extend_eof_test TEST_GEN_FILES := fuse_mnt include ../../lib.mk +$(OUTPUT)/write_extend_eof_test: LDLIBS += -lpthread + VAR_CFLAGS := $(shell pkg-config fuse --cflags 2>/dev/null) ifeq ($(VAR_CFLAGS),) VAR_CFLAGS := -D_FILE_OFFSET_BITS=64 -I/usr/include/fuse diff --git a/tools/testing/selftests/filesystems/fuse/write_extend_eof_test.c b/tools/testing/selftests/filesystems/fuse/write_extend_eof_test.c new file mode 100644 index 000000000000..ca6ce6eca382 --- /dev/null +++ b/tools/testing/selftests/filesystems/fuse/write_extend_eof_test.c @@ -0,0 +1,368 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Regression test for the fuse write-extend partial-EOF-page zeroing bug. + * + * A buffered write that extends i_size past a non-page-aligned EOF must zero + * the tail of the old last page. If an application has mmap'd that page and + * stored into the post-EOF region (undefined until the file grows), the + * now-in-bounds tail must read back as zero, not as the stale stored bytes. + * + * The bug is exposed on a non-writeback_cache server that keeps the page cache + * across the write (FOPEN_KEEP_CACHE without FOPEN_DIRECT_IO). This test is a + * raw /dev/fuse server in that mode; the backing data is always zero in the + * hole, so any non-zero byte a read sees is stale page-cache data. + * + * Requires root to mount fuse. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest_harness.h" + +#define FUSE_ROOT_ID 1 +#define FILE_INO 2 +#define MAX_WRITE (128 * 1024) +#define BACKING_SIZE (4 * 1024 * 1024) +#define POLLUTE 0xee + +/* Server-side state, shared with the responder thread. */ +struct server { + int fd; + unsigned char backing[BACKING_SIZE]; /* authoritative bytes */ + uint64_t size; +}; + +static void reply(int fd, uint64_t unique, int error, void *data, size_t len) +{ + struct fuse_out_header oh = { + .len = sizeof(oh) + (data ? len : 0), + .error = error, + .unique = unique, + }; + struct iovec iov[2] = { { &oh, sizeof(oh) }, { data, len } }; + + /* Errors here are teardown races (device closed on unmount); ignore. */ + if (writev(fd, iov, data ? 2 : 1) < 0) + return; +} + +static void fill_attr(struct fuse_attr *a, uint64_t ino, uint32_t mode, + uint64_t size) +{ + memset(a, 0, sizeof(*a)); + a->ino = ino; + a->mode = mode; + a->nlink = 1; + a->size = size; + a->blksize = sysconf(_SC_PAGESIZE); +} + +static void *server_thread(void *arg) +{ + struct server *s = arg; + static char buf[MAX_WRITE + 4096]; + + for (;;) { + ssize_t n = read(s->fd, buf, sizeof(buf)); + struct fuse_in_header *ih = (void *)buf; + + if (n < 0) { + if (errno == EINTR || errno == EAGAIN) + continue; + return NULL; /* device closed on unmount */ + } + if (n < (ssize_t)sizeof(*ih)) + continue; + + switch (ih->opcode) { + case FUSE_INIT: { + struct fuse_init_in *in = (void *)(ih + 1); + struct fuse_init_out out = {0}; + + /* No FUSE_WRITEBACK_CACHE: the exposed configuration. */ + out.major = FUSE_KERNEL_VERSION; + out.minor = FUSE_KERNEL_MINOR_VERSION; + out.max_readahead = in->max_readahead; + out.max_write = MAX_WRITE; + out.max_background = 16; + out.congestion_threshold = 12; + out.flags = FUSE_MAX_PAGES; + out.max_pages = MAX_WRITE / sysconf(_SC_PAGESIZE); + reply(s->fd, ih->unique, 0, &out, sizeof(out)); + break; + } + case FUSE_GETATTR: { + struct fuse_attr_out out = {0}; + int root = ih->nodeid == FUSE_ROOT_ID; + + out.attr_valid = 3600; + fill_attr(&out.attr, ih->nodeid, + root ? (S_IFDIR | 0755) : (S_IFREG | 0644), + root ? 0 : s->size); + reply(s->fd, ih->unique, 0, &out, sizeof(out)); + break; + } + case FUSE_LOOKUP: { + struct fuse_entry_out out = {0}; + + out.nodeid = FILE_INO; + out.attr_valid = 3600; + out.entry_valid = 3600; + fill_attr(&out.attr, FILE_INO, S_IFREG | 0644, s->size); + reply(s->fd, ih->unique, 0, &out, sizeof(out)); + break; + } + case FUSE_OPEN: + case FUSE_OPENDIR: { + struct fuse_open_out out = {0}; + + /* Keep the cache across the write, but not direct I/O. */ + out.open_flags = FOPEN_KEEP_CACHE; + reply(s->fd, ih->unique, 0, &out, sizeof(out)); + break; + } + case FUSE_READ: { + struct fuse_read_in *in = (void *)(ih + 1); + uint64_t off = in->offset; + uint32_t size = in->size; + + if (off >= BACKING_SIZE) + size = 0; + else if (off + size > BACKING_SIZE) + size = BACKING_SIZE - off; + reply(s->fd, ih->unique, 0, s->backing + off, size); + break; + } + case FUSE_WRITE: { + struct fuse_write_in *in = (void *)(ih + 1); + struct fuse_write_out out = {0}; + uint64_t off = in->offset; + uint32_t size = in->size; + + if (off < BACKING_SIZE) { + uint32_t c = size; + + if (off + c > BACKING_SIZE) + c = BACKING_SIZE - off; + memcpy(s->backing + off, in + 1, c); + if (off + c > s->size) + s->size = off + c; + } + out.size = size; + reply(s->fd, ih->unique, 0, &out, sizeof(out)); + break; + } + case FUSE_SETATTR: { + struct fuse_setattr_in *in = (void *)(ih + 1); + struct fuse_attr_out out = {0}; + + if ((in->valid & FATTR_SIZE) && in->size <= BACKING_SIZE) { + if (in->size > s->size) + memset(s->backing + s->size, 0, + in->size - s->size); + s->size = in->size; + } + out.attr_valid = 3600; + fill_attr(&out.attr, ih->nodeid, S_IFREG | 0644, s->size); + reply(s->fd, ih->unique, 0, &out, sizeof(out)); + break; + } + case FUSE_FALLOCATE: { + struct fuse_fallocate_in *in = (void *)(ih + 1); + uint64_t end = in->offset + in->length; + + /* Only plain (size-extending) fallocate is used here. */ + if (!(in->mode & FALLOC_FL_KEEP_SIZE) && + end <= BACKING_SIZE && end > s->size) { + memset(s->backing + s->size, 0, end - s->size); + s->size = end; + } + reply(s->fd, ih->unique, 0, NULL, 0); + break; + } + case FUSE_FLUSH: + case FUSE_RELEASE: + case FUSE_RELEASEDIR: + case FUSE_FSYNC: + case FUSE_ACCESS: + reply(s->fd, ih->unique, 0, NULL, 0); + break; + case FUSE_FORGET: + break; + default: + reply(s->fd, ih->unique, -EOPNOTSUPP, NULL, 0); + break; + } + } +} + +FIXTURE(fuse) +{ + struct server *srv; + pthread_t thread; + char dir[64]; + long page; /* runtime page size */ + off_t eof; /* mid-page EOF, page-relative */ + int fd; /* open test file */ + char *map; /* mmap of the EOF page */ + int mounted; +}; + +FIXTURE_SETUP(fuse) +{ + char opts[128]; + pthread_t t; + + if (geteuid() != 0) + SKIP(return, "need root to mount fuse"); + + self->page = sysconf(_SC_PAGESIZE); + self->fd = -1; + self->map = MAP_FAILED; + + self->srv = mmap(NULL, sizeof(*self->srv), PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0); + ASSERT_NE(MAP_FAILED, self->srv); + + self->srv->fd = open("/dev/fuse", O_RDWR); + ASSERT_GE(self->srv->fd, 0); + + strcpy(self->dir, "/tmp/fuse_weof_XXXXXX"); + ASSERT_NE(NULL, mkdtemp(self->dir)); + + snprintf(opts, sizeof(opts), + "fd=%d,rootmode=40000,user_id=0,group_id=0", + self->srv->fd); + ASSERT_EQ(0, mount("fuse", self->dir, "fuse", 0, opts)); + self->mounted = 1; + + ASSERT_EQ(0, pthread_create(&t, NULL, server_thread, self->srv)); + self->thread = t; +} + +FIXTURE_TEARDOWN(fuse) +{ + if (self->map != MAP_FAILED) + munmap(self->map, self->page); + if (self->fd >= 0) + close(self->fd); + if (self->mounted) + umount2(self->dir, MNT_DETACH); + if (self->srv && self->srv != MAP_FAILED) { + if (self->srv->fd > 0) + close(self->srv->fd); + munmap(self->srv, sizeof(*self->srv)); + } + if (self->dir[0]) + rmdir(self->dir); +} + +/* + * Create the test file with a mid-page EOF and mmap-store POLLUTE into its + * post-EOF tail (a legal store, undefined until the file grows). Leaves the + * file open and the EOF page mapped in the fixture for the caller to extend. + */ +static void pollute_eof_tail(struct __test_metadata *_metadata, + FIXTURE_DATA(fuse) * self) +{ + off_t eof = 2 * self->page + self->page / 4; + char path[128]; + char *buf; + + snprintf(path, sizeof(path), "%s/file", self->dir); + self->fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0644); + ASSERT_GE(self->fd, 0); + self->eof = eof; + + buf = malloc(eof); + ASSERT_NE(NULL, buf); + memset(buf, 'A', eof); + ASSERT_EQ(eof, pwrite(self->fd, buf, eof, 0)); + free(buf); + + self->map = mmap(NULL, self->page, PROT_READ | PROT_WRITE, MAP_SHARED, + self->fd, eof & ~(self->page - 1)); + ASSERT_NE(MAP_FAILED, self->map); + memset(self->map + (eof & (self->page - 1)), POLLUTE, + self->page - (eof & (self->page - 1))); +} + +/* Assert the old post-EOF tail [eof, end of its page) now reads back as zero. */ +static void assert_tail_zeroed(struct __test_metadata *_metadata, + FIXTURE_DATA(fuse) * self) +{ + off_t base = self->eof & ~(self->page - 1); + char *tail = malloc(self->page); + int i; + + ASSERT_NE(NULL, tail); + ASSERT_EQ(self->page, pread(self->fd, tail, self->page, base)); + for (i = self->eof & (self->page - 1); i < self->page; i++) + ASSERT_EQ(0, tail[i]); + free(tail); +} + +/* Basic: pollute the post-EOF tail, extend past it by a later write. */ +TEST_F(fuse, write_extend) +{ + pollute_eof_tail(_metadata, self); + ASSERT_EQ(4, pwrite(self->fd, "data", 4, 5 * self->page + self->page / 3)); + assert_tail_zeroed(_metadata, self); +} + +/* Extend via ftruncate() rather than a write. */ +TEST_F(fuse, ftruncate_extend) +{ + pollute_eof_tail(_metadata, self); + ASSERT_EQ(0, ftruncate(self->fd, 8 * self->page)); + assert_tail_zeroed(_metadata, self); +} + +/* Extend via fallocate() starting at the old EOF. */ +TEST_F(fuse, fallocate_extend) +{ + pollute_eof_tail(_metadata, self); + ASSERT_EQ(0, fallocate(self->fd, 0, self->eof, 4 * self->page)); + assert_tail_zeroed(_metadata, self); +} + +/* A write landing inside the old EOF page must not clobber its own data. */ +TEST_F(fuse, extend_into_eof_page_preserves_data) +{ + off_t base, wr; + char *buf, *rd; + int i; + + pollute_eof_tail(_metadata, self); + base = self->eof & ~(self->page - 1); + wr = base + 3 * self->page / 4; /* starts in the EOF page */ + + buf = malloc(2 * self->page); + ASSERT_NE(NULL, buf); + memset(buf, 'B', 2 * self->page); + ASSERT_EQ(2 * self->page, pwrite(self->fd, buf, 2 * self->page, wr)); + free(buf); + + rd = malloc(self->page); + ASSERT_NE(NULL, rd); + ASSERT_EQ(self->page, pread(self->fd, rd, self->page, base)); + /* [eof, wr) is hole -> zero; [wr, page) is written data -> 'B'. */ + for (i = self->eof & (self->page - 1); i < wr - base; i++) + ASSERT_EQ(0, rd[i]); + for (i = wr - base; i < self->page; i++) + ASSERT_EQ('B', rd[i]); + free(rd); +} + +TEST_HARNESS_MAIN From 26d7e1f5c407b5859122b5cd47d7ebbf4b4c1cd2 Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Wed, 19 Aug 2026 17:07:28 +0800 Subject: [PATCH 32/35] fuse: invalidate the correct range after O_APPEND direct write fuse_direct_write_iter() captures pos before generic_write_checks(), which moves ki_pos to EOF for O_APPEND writes: fuse_direct_write_iter() { pos = iocb->ki_pos; /* 0 (user-supplied) */ generic_write_checks(); /* ki_pos -> EOF */ fuse_direct_io(); /* writes at EOF, correct */ invalidate(pos, pos + res); /* [0, res) -- wrong */ } The post-write invalidation targets a stale range instead of the actual written range at EOF. This can cause data inconsistency when the file size is not page-aligned. The tail page straddling EOF has a valid portion before EOF that concurrent readers can fault back in during the DIO write window: Tail page (file size X not page-aligned): page_start X (EOF) page_end |--- valid data ----|-- stale --| CPU0 (O_APPEND DIO writer) CPU1 (buffered reader) -------------------------- ---------------------- invalidate [X, X+len) tail page evicted FUSE_WRITE in flight ... read [page_start, X) tail page re-faulted [X, page_end) = stale FUSE_WRITE completes i_size = X + len invalidate [0, len) <- WRONG tail page still cached read [X, X+len) hits stale tail page returns old data Fix by reading pos back from iocb->ki_pos after generic_write_checks(), as generic_file_direct_write() does. Also fix a typo in the comment ("may have" -> "may have competed"). Fixes: 2b0408d0284f ("fuse: invalidate page cache after DIO and async DIO writes") Signed-off-by: Baokun Li Reviewed-by: Bernd Schubert Signed-off-by: Miklos Szeredi --- fs/fuse/file.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 3f67348d0f7e..5b421d619e16 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1787,13 +1787,14 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) { struct inode *inode = file_inode(iocb->ki_filp); struct address_space *mapping = inode->i_mapping; - loff_t pos = iocb->ki_pos; ssize_t res; bool exclusive; fuse_dio_lock(iocb, from, &exclusive); res = generic_write_checks(iocb, from); if (res > 0) { + loff_t pos = iocb->ki_pos; + task_io_account_write(res); if (!is_sync_kiocb(iocb)) { res = fuse_direct_IO(iocb, from); @@ -1808,7 +1809,7 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) /* * As in generic_file_direct_write(), invalidate after * write, to invalidate read-ahead cache that may have - * with the write. + * competed with the write. */ invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT, From 1f59015e958174e89be58cc8db16d70a60d17255 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Fri, 21 Aug 2026 18:19:09 +0200 Subject: [PATCH 33/35] fuse: Fix the condition to enable over-io-uring The existing condition in fuse_uring_cmd() is there only to avoid disabling io-uring for connections that already run with it, missing was a condition to refuse any IORING_OP_URING_CMD if the connection/channel didn't get enabled because of missing FUSE_INIT reply flag FUSE_OVER_IO_URING. Without the reply flag the barrier in fuse_uring_ready() doesn't work and IO could already be going on and cause deadlock states (at a minimum one between fch->bg_lock and queue->lock). The change itself is trivial, but brings behavior change, FUSE_OVER_IO_URING has to be set in the FUSE_INIT_REPLY by fuse servers to accept any IORING_OP_URING_CMD. Libfuse does that and the only non-libfuse implementation I found (fractal-fuse) also does it. Qemu patches for fuse-io-uring are not merged yet, as far as I know. Moved up is the smp_load_acquire(&fch->initialized) check, as a fuse-server implementation might try to setup io-uring before FUSE_INIT is processed and might have gotten -EOPNOTSUPP instead of -EAGAIN. Also fixed is a stale comment that explains the handling of the FUSE_OVER_IO_URING flag in early RFC versions. If there should be a report from any library or application we probably need to revert this commit. Fixes: 3393ff964e0f ("fuse: block request allocation until io-uring init is complete") Signed-off-by: Bernd Schubert Signed-off-by: Miklos Szeredi --- fs/fuse/dev_uring.c | 31 ++++++++++++++++++------------- fs/fuse/inode.c | 4 ---- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index e22a48c9a678..c6dd420c4034 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -1665,25 +1665,30 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) } fch = fud->chan; - /* Once a connection has io-uring enabled on it, it can't be disabled */ - if (!enable_uring && !fch->io_uring) { - pr_info_ratelimited("fuse-io-uring is disabled\n"); - return -EOPNOTSUPP; - } + /* + * The ring is sized from values negotiated by FUSE_INIT + * + * Pairs with smp_store_release() in fuse_chan_set_initialized() + */ + if (!smp_load_acquire(&fch->initialized)) + return -EAGAIN; if (fch->abort_with_err) return -ECONNABORTED; if (!fch->connected) return -ENOTCONN; - /* - * fuse_uring_register() needs the ring to be initialized, - * we need to know the max payload size - * - * Pairs with smp_store_release() in fuse_chan_set_initialized() - */ - if (!smp_load_acquire(&fch->initialized)) - return -EAGAIN; + /* Once a connection has io-uring enabled on it, it can't be disabled */ + if (!enable_uring && !fch->io_uring) { + pr_info_ratelimited("fuse-io-uring is disabled by module parameter\n"); + return -EOPNOTSUPP; + } + + if (!fch->io_uring) { + pr_info_ratelimited( + "fuse-io-uring not enabled on this connection\n"); + return -EOPNOTSUPP; + } switch (cmd_op) { case FUSE_IO_URING_CMD_REGISTER: diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 1c6ee01c6796..e9552be3637b 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1480,10 +1480,6 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm) if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) flags |= FUSE_PASSTHROUGH; - /* - * This is just an information flag for fuse server. No need to check - * the reply - server is either sending IORING_OP_URING_CMD or not. - */ if (fuse_uring_enabled()) flags |= FUSE_OVER_IO_URING | FUSE_HAS_IO_URING_BUFPOOL; From 1b04ca2aca2d4015e7e1e17e7911f9484fe16fd9 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Tue, 18 Aug 2026 19:42:09 +0100 Subject: [PATCH 34/35] io_uring: Add missing include for ITER_SOURCE and ITER_DEST Fix IWYU issues: /tmp/next/build/include/linux/io_uring_types.h:56:32: error: 'ITER_DEST' undeclared here (not in a function) 56 | IO_BUF_DEST = 1 << ITER_DEST, | ^~~~~~~~~ /tmp/next/build/include/linux/io_uring_types.h:57:32: error: 'ITER_SOURCE' undeclared here (not in a function) 57 | IO_BUF_SOURCE = 1 << ITER_SOURCE, | ^~~~~~~~~~~ Fixes: 95961b72c57b2 ("io_uring/rsrc: rename and export IO_IMU_DEST / IO_IMU_SOURCE") Signed-off-by: Mark Brown Signed-off-by: Miklos Szeredi --- include/linux/io_uring_types.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index db42a548c7a5..731a484d5a95 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -6,6 +6,7 @@ #include #include #include +#include #include struct iou_loop_params; From 34b5c4a6e4fb9dbb3f9d87f3b0fb0372105c8302 Mon Sep 17 00:00:00 2001 From: Jimmy Zuber Date: Mon, 24 Aug 2026 14:30:51 +0000 Subject: [PATCH 35/35] fuse: zero the partial EOF page when extending a file Extending a fuse file past a non-page-aligned EOF does not zero the tail of the old last page. When that page is cached and has been mmap-dirtied beyond the old EOF, the now in-bounds tail is served to later reads as stale data rather than zeros, which violates POSIX file-extension semantics. Some file systems get this zeroing automatically at writeback time (block_write_full_folio() / iomap_writeback_handle_eof() zero the tail of the folio straddling i_size). A non-writeback caching fuse file system uses neither path, so it has to zero the tail itself from the size-extending paths, like XFS (xfs_file_write_zero_eof()) and ext4 (ext4_block_zero_eof()) do. Call truncate_pagecache_range() over the newly-exposed range up front from the three paths that extend a file, before the new size is published: - a buffered write whose position is past the old EOF (fuse_perform_write()); - a size-extending setattr/truncate (fuse_do_setattr()); - a size-extending fallocate (fuse_file_fallocate()). This unmaps the stale mappings and zeroes the partial tail of the old EOF folio, so a later read returns zeros. Truncating [old EOF, write start) before a buffered write keeps the dropped range disjoint from the written data, so a write that lands inside the old EOF folio is preserved. writeback_cache connections are unaffected, as their writes go through iomap_file_buffered_write(), which zeroes post-EOF folios. The bug is observable on a non-writeback_cache server that returns FOPEN_KEEP_CACHE on writable files (without FOPEN_DIRECT_IO), and is caught by the new write_extend_eof fuse selftest. Signed-off-by: Jimmy Zuber Signed-off-by: Miklos Szeredi --- fs/fuse/dir.c | 3 +++ fs/fuse/file.c | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 172261bad71e..d3907aa3642b 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2289,6 +2289,9 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, */ if ((is_truncate || !is_wb) && S_ISREG(inode->i_mode) && oldsize != outarg.attr.size) { + if (outarg.attr.size > oldsize) + truncate_pagecache_range(inode, oldsize, + outarg.attr.size - 1); truncate_pagecache(inode, outarg.attr.size); invalidate_inode_pages2(mapping); } diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 5b421d619e16..e9b46c8e75b3 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1346,9 +1346,13 @@ static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii) struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); loff_t pos = iocb->ki_pos; + loff_t old_size = i_size_read(inode); int err = 0; ssize_t res = 0; + if (pos > old_size) + truncate_pagecache_range(inode, old_size, pos - 1); + if (inode->i_size < pos + iov_iter_count(ii)) set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); @@ -2895,6 +2899,11 @@ static long fuse_file_fallocate(struct file *file, int mode, loff_t offset, /* we could have extended the file */ if (!(mode & FALLOC_FL_KEEP_SIZE)) { + loff_t oldsize = i_size_read(inode); + + if (offset + length > oldsize) + truncate_pagecache_range(inode, oldsize, + offset + length - 1); if (fuse_write_update_attr(inode, offset + length, length)) file_update_time(file); }