From c905736a46892e4776efc7f50888d67715d6ec08 Mon Sep 17 00:00:00 2001 From: Robert Femmer Date: Wed, 24 Jun 2026 11:01:46 +0200 Subject: [PATCH 01/30] io_uring: annotate remote tasks for kcoverage Fuzzers use coverage information to guide generation of test cases towards new or interesting code paths. Syzkaller, specifically, makes use kcoverage (CONFIG_KCOV). Coverage information is not collected for kernel tasks unless annotated by kcov_remote_start and kcov_remote_stop. This patch annotates io-uring's work queue and sqpoll tasks. Depends-On: 20260430-kcov-refactor-common-handle-v1-1-23a0c7a0ba38@google.com Signed-off-by: Robert Femmer Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 2 ++ io_uring/io-wq.c | 5 +++++ io_uring/io_uring.c | 2 ++ io_uring/sqpoll.c | 7 ++++++- 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 87151a5b62c1..a2c623a67a25 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -534,6 +534,8 @@ struct io_ring_ctx { struct io_mapped_region ring_region; /* used for optimised request parameter and wait argument passing */ struct io_mapped_region param_region; + + struct kcov_common_handle_id kcov_handle; }; /* diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c index 2e14880eef92..be8d75731d24 100644 --- a/io_uring/io-wq.c +++ b/io_uring/io-wq.c @@ -19,6 +19,7 @@ #include #include #include +#include #include "io-wq.h" #include "slist.h" @@ -643,13 +644,17 @@ static void io_worker_handle_work(struct io_wq_acct *acct, unsigned int hash = __io_wq_is_hashed(work_flags) ? __io_get_work_hash(work_flags) : -1U; + struct io_kiocb *req; next_hashed = wq_next_work(work); if (do_kill && (work_flags & IO_WQ_WORK_UNBOUND)) atomic_or(IO_WQ_WORK_CANCEL, &work->flags); + req = container_of(work, struct io_kiocb, work); + kcov_remote_start_common(req->ctx->kcov_handle); io_wq_submit_work(work); + kcov_remote_stop(); io_assign_current_work(worker, NULL); linked = io_wq_free_work(work); diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 1ea2fca34a36..1279e27c2c6d 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -59,6 +59,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -293,6 +294,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_HLIST_HEAD(&ctx->cancelable_uring_cmd); io_napi_init(ctx); mutex_init(&ctx->mmap_lock); + ctx->kcov_handle = kcov_common_handle(); return ctx; diff --git a/io_uring/sqpoll.c b/io_uring/sqpoll.c index 2460bd605266..ad42e8eb1002 100644 --- a/io_uring/sqpoll.c +++ b/io_uring/sqpoll.c @@ -13,6 +13,7 @@ #include #include #include +#include #include @@ -332,10 +333,14 @@ static int io_sq_thread(void *data) cap_entries = !list_is_singular(&sqd->ctx_list); list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { - int ret = __io_sq_thread(ctx, sqd, cap_entries, &ist); + int ret; + + kcov_remote_start_common(ctx->kcov_handle); + ret = __io_sq_thread(ctx, sqd, cap_entries, &ist); if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list))) sqt_spin = true; + kcov_remote_stop(); } if (io_sq_tw(IORING_TW_CAP_ENTRIES_VALUE)) sqt_spin = true; From 73e7019097473fc9f83a334ef2c6ab3343709fef Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 30 Jul 2026 06:50:13 -0600 Subject: [PATCH 02/30] io_uring/futex: don't mark futex wake requests as inflight Commit 079afb081c42 ("io_uring/futex: mark wait requests as inflight") added inflight tracking to ensure that do_exit() -> io_uring_files_cancel() finds and cancels pending futex waits before the mm goes away, as a private futex wait depends on the mm private futex hash staying alive for the duration of the request. However, as io_futex_prep() is shared between FUTEX_WAIT and FUTEX_WAKE, wake requests got marked as inflight as well. A futex wake executes fully inline at issue time and never depends on the mm staying alive after completion, hence there's no need to track it. Kill it. Cc: stable@vger.kernel.org Fixes: 079afb081c42 ("io_uring/futex: mark wait requests as inflight") Reported-by: Chengfeng Lin Link: https://lore.kernel.org/io-uring/CANGjgdn=R_qyUdE=j9za+vkmqcxacbP-84OHXF4nZ4ho9qRyVg@mail.gmail.com/ Signed-off-by: Jens Axboe --- io_uring/futex.c | 11 +++++++++++ io_uring/futex.h | 1 + io_uring/opdef.c | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/io_uring/futex.c b/io_uring/futex.c index 906701b3c5c6..eea0425f2bcb 100644 --- a/io_uring/futex.c +++ b/io_uring/futex.c @@ -149,6 +149,17 @@ int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) !futex_validate_input(iof->futex_flags, iof->futex_mask)) return -EINVAL; + return 0; +} + +int io_futex_wait_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +{ + int ret; + + ret = io_futex_prep(req, sqe); + if (unlikely(ret)) + return ret; + /* Mark as inflight, so file exit cancelation will find it */ io_req_track_inflight(req); return 0; diff --git a/io_uring/futex.h b/io_uring/futex.h index d789fcf715e3..987db3f2c6d9 100644 --- a/io_uring/futex.h +++ b/io_uring/futex.h @@ -3,6 +3,7 @@ #include "cancel.h" int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); +int io_futex_wait_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags); int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags); diff --git a/io_uring/opdef.c b/io_uring/opdef.c index 4e58eb1344ea..cf3aa2242cd7 100644 --- a/io_uring/opdef.c +++ b/io_uring/opdef.c @@ -467,7 +467,7 @@ const struct io_issue_def io_issue_defs[] = { }, [IORING_OP_FUTEX_WAIT] = { #if defined(CONFIG_FUTEX) - .prep = io_futex_prep, + .prep = io_futex_wait_prep, .issue = io_futex_wait, #else .prep = io_eopnotsupp_prep, From 4d327bbd1cd2485783553b6b2b401aa475f90dd0 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 30 Jul 2026 06:50:47 -0600 Subject: [PATCH 03/30] io_uring/futex: only mark private futex waits as inflight Inflight tracking of futex wait requests exists to ensure that do_exit() -> io_uring_files_cancel() cancels them before the mm goes away, as a private futex wait depends on the mm private futex hash staying alive for the duration of the request. Shared futexes have no such dependency. A FLAGS_SHARED request always resolves to either an inode based key or an mm-shared key, both of which fail futex_key_is_private() and hence always hash into the global futex hash, whose lifetime isn't tied to the mm. Only mark vectored futex waits as inflight if the futex is private. Cc: stable@vger.kernel.org Fixes: 079afb081c42 ("io_uring/futex: mark wait requests as inflight") Link: https://lore.kernel.org/io-uring/CANGjgdn=R_qyUdE=j9za+vkmqcxacbP-84OHXF4nZ4ho9qRyVg@mail.gmail.com/ Signed-off-by: Jens Axboe --- io_uring/futex.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/io_uring/futex.c b/io_uring/futex.c index eea0425f2bcb..f0d80a444f45 100644 --- a/io_uring/futex.c +++ b/io_uring/futex.c @@ -154,14 +154,16 @@ int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_futex_wait_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { + struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); int ret; ret = io_futex_prep(req, sqe); if (unlikely(ret)) return ret; - /* Mark as inflight, so file exit cancelation will find it */ - io_req_track_inflight(req); + /* inflight tracking only needed for mm private hash */ + if (!(iof->futex_flags & FLAGS_SHARED)) + io_req_track_inflight(req); return 0; } @@ -186,6 +188,7 @@ int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); struct io_futexv_data *ifd; + unsigned int i; int ret; /* No flags or mask supported for waitv */ @@ -210,8 +213,14 @@ int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return ret; } - /* Mark as inflight, so file exit cancelation will find it */ - io_req_track_inflight(req); + /* inflight tracking only needed for mm private hash */ + for (i = 0; i < iof->futex_nr; i++) { + if (!(ifd->futexv[i].w.flags & FLAGS_SHARED)) { + io_req_track_inflight(req); + break; + } + } + iof->futexv_unqueued = 0; req->flags |= REQ_F_ASYNC_DATA; req->async_data = ifd; From 13b0742fd05864384724710c1bb12643b5448015 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Thu, 2 Jul 2026 12:48:45 -0600 Subject: [PATCH 04/30] io_uring/uring_cmd: skip io_uring_cmd_issue_blocking() task work io_uring_cmd_issue_blocking() is only called from blk_cmd_complete(), which is already a task work callback. However, it queues another task work item, to call io_queue_iowq(). Just call io_queue_iowq() directly to skip the CPU cost and latency of the redundant task work proxying. Signed-off-by: Caleb Sander Mateos Link: https://patch.msgid.link/20260702184847.1709378-1-csander@purestorage.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 13 +------------ io_uring/io_uring.h | 2 +- io_uring/uring_cmd.c | 2 +- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 1279e27c2c6d..4c83a94b4bdc 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -407,7 +407,7 @@ static void io_prep_async_link(struct io_kiocb *req) } } -static void io_queue_iowq(struct io_kiocb *req) +void io_queue_iowq(struct io_kiocb *req) { struct io_uring_task *tctx = req->tctx; @@ -435,17 +435,6 @@ static void io_queue_iowq(struct io_kiocb *req) io_wq_enqueue(tctx->io_wq, &req->work); } -static void io_req_queue_iowq_tw(struct io_tw_req tw_req, io_tw_token_t tw) -{ - io_queue_iowq(tw_req.req); -} - -void io_req_queue_iowq(struct io_kiocb *req) -{ - req->io_task_work.func = io_req_queue_iowq_tw; - io_req_task_work_add(req); -} - unsigned io_linked_nr(struct io_kiocb *req) { struct io_kiocb *tmp; diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index cb736b815422..dfe26a9c21bf 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -195,7 +195,7 @@ __cold void io_uring_drop_tctx_refs(struct task_struct *task); int io_ring_add_registered_file(struct io_uring_task *tctx, struct file *file, int start, int end); -void io_req_queue_iowq(struct io_kiocb *req); +void io_queue_iowq(struct io_kiocb *req); int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw); int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr); diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index 7b25dcd9d05f..5525267d2e03 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -326,7 +326,7 @@ void io_uring_cmd_issue_blocking(struct io_uring_cmd *ioucmd) { struct io_kiocb *req = cmd_to_io_kiocb(ioucmd); - io_req_queue_iowq(req); + io_queue_iowq(req); } int io_cmd_poll_multishot(struct io_uring_cmd *cmd, From bb34ae5da3365699d53a756f4c96b6ea9f8ba0c1 Mon Sep 17 00:00:00 2001 From: Woraphat Khiaodaeng Date: Sun, 2 Aug 2026 14:35:18 +0700 Subject: [PATCH 05/30] io_uring/cmd: fix iovec leak when the async cmd is not recycled An io_async_cmd carries an iovec array in ->vec.iovec, allocated when the vec has to grow and kept across recycling through ctx->cmd_cache. On two paths nothing frees it and io_clean_op()'s kfree(req->async_data) drops the io_async_cmd without it. io_req_uring_cleanup() clears the async data flags only when io_alloc_cache_put() succeeds, and the cache holds IO_ALLOC_CACHE_MAX == 128 entries, so once it is full the put fails and the vec is left behind. An NVMe passthrough workload gets there without doing anything unusual: nvme_uring_cmd_io() returns -EIOCBQUEUED, so the io_async_cmd stays attached for the lifetime of the command and the live object count tracks the queue depth. Above 128 the puts start failing. ->cleanup is the last chance to free an inherited vec, since io_req_uring_cleanup() returns early for an io-wq issued command and is not called at all for one completed without ever being issued. But io_clean_op() calls ->cleanup only if REQ_F_NEED_CLEANUP is set, and for uring_cmd that happens only where the vec has to grow, so a command reusing a large enough cached vec never sets it. io_rw_alloc_async() and io_msg_alloc_async() flag an inherited vec for exactly this reason; io_uring_cmd_prep() does not. Flag an inherited vec in io_uring_cmd_prep(), and free the vec when the cache put fails, as io_req_rw_cleanup() does. The leak is invisible under KASAN, where io_alloc_cache_vec_kasan() frees the vec unconditionally. Fixes: 3a4689ac109f ("io_uring/cmd: add iovec cache for commands") Cc: stable@vger.kernel.org Signed-off-by: Woraphat Khiaodaeng Link: https://patch.msgid.link/20260802073518.419-1-worapat.kd2@gmail.com Signed-off-by: Jens Axboe --- io_uring/uring_cmd.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index 5525267d2e03..ea1432251ccf 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -38,6 +38,8 @@ static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags) if (io_alloc_cache_put(&req->ctx->cmd_cache, ac)) { ioucmd->sqe = NULL; io_req_async_data_clear(req, REQ_F_NEED_CLEANUP); + } else { + io_vec_free(&ac->vec); } } @@ -208,6 +210,8 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ac = io_uring_alloc_async_data(&req->ctx->cmd_cache, req); if (!ac) return -ENOMEM; + if (ac->vec.iovec) + req->flags |= REQ_F_NEED_CLEANUP; ioucmd->sqe = sqe; return 0; } From 3f3a6a16bbe8bde76532d9415438f8cdef439e5d Mon Sep 17 00:00:00 2001 From: Ali Ahmet Memis Date: Sun, 2 Aug 2026 16:30:30 +0000 Subject: [PATCH 06/30] io_uring/rsrc: fix folio size overflow in io_vec_fill_bvec() io_vec_fill_bvec() computes the folio size with a plain int 1: unsigned long folio_size = 1 << imu->folio_shift; imu->folio_shift is unsigned int and comes from folio_shift() of the folio backing the registered buffer, so it can be 32 or more on a 64 bit kernel. Shifting int 1 that far is undefined, and on x86 and arm64 the count is taken modulo 32, so a shift of 34 yields 4 rather than 16G. Every other folio_shift shift in this file already uses 1UL. The result is that the segment estimate and the fill loop disagree. io_estimate_bvec_size() sizes the bvec array with the real shift: max_segs += (iov[i].iov_len >> shift) + 2; so a 1M iovec on a 16G folio is charged 2 segments, while io_vec_fill_bvec() then walks the same iovec in folio_size chunks of 4 bytes and writes res_bvec[bvec_idx] a quarter of a million times, past the end of the array it was given. src_bvec is advanced once per iteration as well, so imu->bvec is read past its end at the same time. validate_fixed_range() only checks that the range is inside the registered buffer and does not bound the segment count. Reaching it needs a folio with a shift of at least 32, which means a gigantic hugetlb page: 16G on arm64 with 64K pages, where CONT_PMD_SHIFT is 34 and hugetlb_add_hstate(CONT_PMD_SHIFT - PAGE_SHIFT) registers that size, and likewise on powerpc. x86_64 tops out at 1G, so a shift of 30, which still fits in int and is unaffected. Use 1UL, as the rest of the file does. Fixes: 9ef4cbbcb4ac ("io_uring: add infra for importing vectored reg buffers") Cc: stable@vger.kernel.org Signed-off-by: Ali Ahmet Memis Link: https://patch.msgid.link/20260802163030.51005-1-ali@iusegentoo.com Signed-off-by: Jens Axboe --- io_uring/rsrc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 8d0f2ee24e0c..deb2a844f568 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -1477,7 +1477,7 @@ static int io_vec_fill_bvec(int ddir, struct iov_iter *iter, struct iovec *iovec, unsigned nr_iovs, struct iou_vec *vec) { - unsigned long folio_size = 1 << imu->folio_shift; + unsigned long folio_size = 1UL << imu->folio_shift; unsigned long folio_mask = folio_size - 1; struct bio_vec *res_bvec = vec->bvec; size_t total_len = 0; From cd305ee3633a45fcf5f3a5d83f99f3cb77d87b6e Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 14 Aug 2026 06:58:43 -0600 Subject: [PATCH 07/30] io_uring: defer eventfd signaling when queued from a wakeup handler io_req_local_work_add() signals the CQ ring eventfd inline when it is the one to push the first entry onto ->work_list. For DEFER_TASKRUN rings that add is frequently done from a waitqueue wakeup handler, where an arbitrary waitqueue lock is held. eventfd_signal_mask() only refuses to recurse when current->in_eventfd is set, but that bit is set by eventfd_signal_mask() itself. If the wake chain starts somewhere else, signal goes out inline and can feed back into epoll. Add IOU_F_TWQ_IN_WAKE, set it on the task_work add done from the three waitqueue callbacks, and use it to force io_eventfd_signal() down the existing call_rcu_hurry() deferral instead of signaling inline. Fixes: 21a091b970cd ("io_uring: signal registered eventfd to process deferred task work") Cc: stable@vger.kernel.org Link: https://lore.kernel.org/all/20260813133843.2933127-1-4ncienth@gmail.com/ Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 8 ++++++++ io_uring/eventfd.c | 8 ++++---- io_uring/eventfd.h | 2 +- io_uring/futex.c | 4 ++-- io_uring/io_uring.c | 2 +- io_uring/poll.c | 23 ++++++++++++----------- io_uring/tw.c | 2 +- io_uring/waitid.c | 2 +- 8 files changed, 30 insertions(+), 21 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index a2c623a67a25..f6e90cc64a1f 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -20,6 +20,14 @@ enum { * It's also ignored unless IORING_SETUP_DEFER_TASKRUN is set. */ IOU_F_TWQ_LAZY_WAKE = 1, + + /* + * Set when task_work is queued from a waitqueue wakeup handler, where + * an arbitrary provider waitqueue lock is held. Signaling the CQ ring + * eventfd inline from there can recurse back into that lock through + * epoll, so the eventfd signal must be deferred. + */ + IOU_F_TWQ_IN_WAKE = 2, }; enum io_uring_cmd_flags { diff --git a/io_uring/eventfd.c b/io_uring/eventfd.c index d656cc2a0b9b..63fe6e5d79ba 100644 --- a/io_uring/eventfd.c +++ b/io_uring/eventfd.c @@ -51,9 +51,9 @@ static void io_eventfd_do_signal(struct rcu_head *rcu) /* * Returns true if the caller should put the ev_fd reference, false if not. */ -static bool __io_eventfd_signal(struct io_ev_fd *ev_fd) +static bool __io_eventfd_signal(struct io_ev_fd *ev_fd, bool defer) { - if (eventfd_signal_allowed()) { + if (!defer && eventfd_signal_allowed()) { eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE); return true; } @@ -73,7 +73,7 @@ static bool io_eventfd_trigger(struct io_ev_fd *ev_fd) return !ev_fd->eventfd_async || io_wq_current_is_worker(); } -void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event) +void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event, bool defer) { bool skip = false; struct io_ev_fd *ev_fd; @@ -113,7 +113,7 @@ void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event) spin_unlock(&ctx->completion_lock); } - if (skip || __io_eventfd_signal(ev_fd)) + if (skip || __io_eventfd_signal(ev_fd, defer)) io_eventfd_put(ev_fd); } diff --git a/io_uring/eventfd.h b/io_uring/eventfd.h index 400eda4a4165..e965d80d9fdc 100644 --- a/io_uring/eventfd.h +++ b/io_uring/eventfd.h @@ -5,4 +5,4 @@ int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, unsigned int eventfd_async); int io_eventfd_unregister(struct io_ring_ctx *ctx); -void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event); +void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event, bool defer); diff --git a/io_uring/futex.c b/io_uring/futex.c index f0d80a444f45..eaee14242a3a 100644 --- a/io_uring/futex.c +++ b/io_uring/futex.c @@ -181,7 +181,7 @@ static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q) io_req_set_res(req, 0, 0); req->io_task_work.func = io_futexv_complete; - io_req_task_work_add(req); + __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE); } int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) @@ -237,7 +237,7 @@ static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q) io_req_set_res(req, 0, 0); req->io_task_work.func = io_futex_complete; - io_req_task_work_add(req); + __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE); } int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 4c83a94b4bdc..76f049e29aa2 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -475,7 +475,7 @@ void __io_commit_cqring_flush(struct io_ring_ctx *ctx) if (ctx->int_flags & IO_RING_F_OFF_TIMEOUT_USED) io_flush_timeouts(ctx); if (ctx->int_flags & IO_RING_F_HAS_EVFD) - io_eventfd_signal(ctx, true); + io_eventfd_signal(ctx, true, false); } static inline void __io_cq_lock(struct io_ring_ctx *ctx) diff --git a/io_uring/poll.c b/io_uring/poll.c index 0204affdc308..5447a7c24dce 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -208,9 +208,9 @@ enum { IOU_POLL_REQUEUE = 4, }; -static void __io_poll_execute(struct io_kiocb *req, int mask) +static void __io_poll_execute(struct io_kiocb *req, int mask, unsigned tw_flags) { - unsigned flags = 0; + unsigned flags = tw_flags; io_req_set_res(req, mask, 0); req->io_task_work.func = io_poll_task_func; @@ -218,14 +218,15 @@ static void __io_poll_execute(struct io_kiocb *req, int mask) trace_io_uring_task_add(req, mask); if (!(req->flags & REQ_F_POLL_NO_LAZY)) - flags = IOU_F_TWQ_LAZY_WAKE; + flags |= IOU_F_TWQ_LAZY_WAKE; __io_req_task_work_add(req, flags); } -static inline void io_poll_execute(struct io_kiocb *req, int res) +static inline void io_poll_execute(struct io_kiocb *req, int res, + unsigned tw_flags) { if (io_poll_get_ownership(req)) - __io_poll_execute(req, res); + __io_poll_execute(req, res, tw_flags); } /* @@ -344,7 +345,7 @@ void io_poll_task_func(struct io_tw_req tw_req, io_tw_token_t tw) if (ret == IOU_POLL_NO_ACTION) { return; } else if (ret == IOU_POLL_REQUEUE) { - __io_poll_execute(req, 0); + __io_poll_execute(req, 0, 0); return; } io_poll_remove_entries(req); @@ -383,7 +384,7 @@ static void io_poll_cancel_req(struct io_kiocb *req) { io_poll_mark_cancelled(req); /* kick tw, which should complete the request */ - io_poll_execute(req, 0); + io_poll_execute(req, 0, 0); } #define IO_ASYNC_POLL_COMMON (EPOLLONESHOT | EPOLLPRI) @@ -392,7 +393,7 @@ static __cold int io_pollfree_wake(struct io_kiocb *req, struct io_poll *poll) { io_poll_mark_cancelled(req); /* we have to kick tw in case it's not already */ - io_poll_execute(req, 0); + io_poll_execute(req, 0, IOU_F_TWQ_IN_WAKE); io_poll_remove_waitq(poll); return 1; } @@ -430,7 +431,7 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, else req->flags &= ~REQ_F_SINGLE_POLL; } - __io_poll_execute(req, mask); + __io_poll_execute(req, mask, IOU_F_TWQ_IN_WAKE); } return 1; } @@ -618,7 +619,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, if (mask && (poll->events & EPOLLET) && io_poll_can_finish_inline(req, ipt)) { - __io_poll_execute(req, mask); + __io_poll_execute(req, mask, 0); return 0; } io_napi_add(req); @@ -629,7 +630,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, * poll was waken up, queue up a tw, it'll deal with it. */ if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1) - __io_poll_execute(req, 0); + __io_poll_execute(req, 0, 0); } return 0; } diff --git a/io_uring/tw.c b/io_uring/tw.c index a4c872870d81..bf4e5aa5c2e7 100644 --- a/io_uring/tw.c +++ b/io_uring/tw.c @@ -170,7 +170,7 @@ void io_req_local_work_add(struct io_kiocb *req, unsigned flags) if (mpscq_push(&ctx->work_list, &req->io_task_work.node)) { io_ctx_mark_taskrun(ctx); if (data_race(ctx->int_flags) & IO_RING_F_HAS_EVFD) - io_eventfd_signal(ctx, false); + io_eventfd_signal(ctx, false, flags & IOU_F_TWQ_IN_WAKE); } /* diff --git a/io_uring/waitid.c b/io_uring/waitid.c index 32f68fd7fcdd..76af129ba8ca 100644 --- a/io_uring/waitid.c +++ b/io_uring/waitid.c @@ -253,7 +253,7 @@ static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode, return 1; req->io_task_work.func = io_waitid_cb; - io_req_task_work_add(req); + __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE); return 1; } From 0ca89205857c43e02cea3ee2a342e7d3c0c8a421 Mon Sep 17 00:00:00 2001 From: Gabriel Krisman Bertazi Date: Wed, 12 Aug 2026 20:40:20 -0400 Subject: [PATCH 08/30] io_uring/rw: Drop custom iov copy in io_iov_buffer_select_prep Similar to commit f4eaf8eda89e ("io_uring/rsrc: Drop io_copy_iov in favor of iovec API"), avoid custom copy and just rely on the iovec api. Signed-off-by: Gabriel Krisman Bertazi Link: https://patch.msgid.link/20260813004022.3514537-2-krisman@suse.de Signed-off-by: Jens Axboe --- io_uring/rw.c | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/io_uring/rw.c b/io_uring/rw.c index 63b6519e498c..b0ddc9fa36af 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include @@ -50,33 +49,20 @@ static bool io_file_supports_nowait(struct io_kiocb *req, __poll_t mask) return false; } -static int io_iov_compat_buffer_select_prep(struct io_rw *rw) -{ - struct compat_iovec __user *uiov = u64_to_user_ptr(rw->addr); - struct compat_iovec iov; - - if (copy_from_user(&iov, uiov, sizeof(iov))) - return -EFAULT; - rw->len = iov.iov_len; - return 0; -} - static int io_iov_buffer_select_prep(struct io_kiocb *req) { struct iovec __user *uiov; - struct iovec iov; + struct iovec fast_iov, *iov; struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); if (rw->len != 1) return -EINVAL; - if (io_is_compat(req->ctx)) - return io_iov_compat_buffer_select_prep(rw); - uiov = u64_to_user_ptr(rw->addr); - if (copy_from_user(&iov, uiov, sizeof(*uiov))) - return -EFAULT; - rw->len = iov.iov_len; + iov = iovec_from_user(uiov, 1, 1, &fast_iov, io_is_compat(req->ctx)); + if (IS_ERR(iov)) + return PTR_ERR(iov); + rw->len = iov->iov_len; return 0; } From 5ec4f12a0b01597f9a55e01618849e3a8ac46cc2 Mon Sep 17 00:00:00 2001 From: Gabriel Krisman Bertazi Date: Wed, 12 Aug 2026 20:40:21 -0400 Subject: [PATCH 09/30] io_uring/net: Drop custom iov copy in io_msg_copy_hdr Similar to commit f4eaf8eda89e ("io_uring/rsrc: Drop io_copy_iov in favor of iovec API"), avoid the custom copy of a single iovec and just rely on the iovec api. This lets the compat and native paths share the buffer-select length lookup Signed-off-by: Gabriel Krisman Bertazi Link: https://patch.msgid.link/20260813004022.3514537-3-krisman@suse.de Signed-off-by: Jens Axboe --- io_uring/net.c | 71 ++++++++++++++------------------------------------ 1 file changed, 20 insertions(+), 51 deletions(-) diff --git a/io_uring/net.c b/io_uring/net.c index 00a7df803b99..37a93c0a88a8 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -236,39 +236,6 @@ static int io_net_import_vec(struct io_kiocb *req, struct io_async_msghdr *iomsg return 0; } -static int io_compat_msg_copy_hdr(struct io_kiocb *req, - struct io_async_msghdr *iomsg, - struct compat_msghdr *msg, int ddir, - struct sockaddr __user **save_addr) -{ - struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); - struct compat_iovec __user *uiov; - int ret; - - if (copy_from_user(msg, sr->umsg_compat, sizeof(*msg))) - return -EFAULT; - - ret = __get_compat_msghdr(&iomsg->msg, msg, save_addr); - if (ret) - return ret; - - uiov = compat_ptr(msg->msg_iov); - if (req->flags & REQ_F_BUFFER_SELECT) { - if (msg->msg_iovlen == 0) { - sr->len = 0; - } else if (msg->msg_iovlen > 1) { - return -EINVAL; - } else { - struct compat_iovec tmp_iov; - - if (copy_from_user(&tmp_iov, uiov, sizeof(tmp_iov))) - return -EFAULT; - sr->len = tmp_iov.iov_len; - } - } - return 0; -} - static int io_copy_msghdr_from_user(struct user_msghdr *msg, struct user_msghdr __user *umsg) { @@ -292,7 +259,6 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg, struct sockaddr __user **save_addr) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); - struct user_msghdr __user *umsg = sr->umsg; int ret; iomsg->msg.msg_name = &iomsg->addr; @@ -301,7 +267,10 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg, if (io_is_compat(req->ctx)) { struct compat_msghdr cmsg; - ret = io_compat_msg_copy_hdr(req, iomsg, &cmsg, ddir, save_addr); + if (copy_from_user(&cmsg, sr->umsg_compat, sizeof(cmsg))) + return -EFAULT; + + ret = __get_compat_msghdr(&iomsg->msg, &cmsg, save_addr); if (ret) return ret; @@ -310,31 +279,31 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg, msg->msg_controllen = cmsg.msg_controllen; msg->msg_iov = compat_ptr(cmsg.msg_iov); msg->msg_iovlen = cmsg.msg_iovlen; - return 0; + } else { + ret = io_copy_msghdr_from_user(msg, sr->umsg); + if (unlikely(ret)) + return ret; + + msg->msg_flags = 0; + + ret = __copy_msghdr(&iomsg->msg, msg, save_addr); + if (ret) + return ret; } - ret = io_copy_msghdr_from_user(msg, umsg); - if (unlikely(ret)) - return ret; - - msg->msg_flags = 0; - - ret = __copy_msghdr(&iomsg->msg, msg, save_addr); - if (ret) - return ret; - if (req->flags & REQ_F_BUFFER_SELECT) { if (msg->msg_iovlen == 0) { sr->len = 0; } else if (msg->msg_iovlen > 1) { return -EINVAL; } else { - struct iovec __user *uiov = msg->msg_iov; - struct iovec tmp_iov; + struct iovec fast_iov, *iov; - if (copy_from_user(&tmp_iov, uiov, sizeof(tmp_iov))) - return -EFAULT; - sr->len = tmp_iov.iov_len; + iov = iovec_from_user(msg->msg_iov, 1, 1, &fast_iov, + io_is_compat(req->ctx)); + if (IS_ERR(iov)) + return PTR_ERR(iov); + sr->len = iov->iov_len; } } return 0; From b76d6b068e693a1bea1df9cc5af16893c39b5bd5 Mon Sep 17 00:00:00 2001 From: Gabriel Krisman Bertazi Date: Wed, 12 Aug 2026 20:40:22 -0400 Subject: [PATCH 10/30] io_uring/net: Drop ddir argument from io_msg_copy_hdr ddir was only ever forwarded to io_compat_msg_copy_hdr, which never looked at it. Drop it. Signed-off-by: Gabriel Krisman Bertazi Link: https://patch.msgid.link/20260813004022.3514537-4-krisman@suse.de Signed-off-by: Jens Axboe --- io_uring/net.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/io_uring/net.c b/io_uring/net.c index 37a93c0a88a8..f8637f8bc3e8 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -255,7 +255,7 @@ static int io_copy_msghdr_from_user(struct user_msghdr *msg, } static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg, - struct user_msghdr *msg, int ddir, + struct user_msghdr *msg, struct sockaddr __user **save_addr) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); @@ -371,7 +371,7 @@ static int io_sendmsg_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe sr->flags |= IORING_SEND_VECTORIZED; sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); - ret = io_msg_copy_hdr(req, kmsg, &msg, ITER_SOURCE, NULL); + ret = io_msg_copy_hdr(req, kmsg, &msg, NULL); if (unlikely(ret)) return ret; /* save msg_control as sys_sendmsg() overwrites it */ @@ -729,7 +729,7 @@ static int io_recvmsg_copy_hdr(struct io_kiocb *req, struct user_msghdr msg; int ret; - ret = io_msg_copy_hdr(req, iomsg, &msg, ITER_DEST, &iomsg->uaddr); + ret = io_msg_copy_hdr(req, iomsg, &msg, &iomsg->uaddr); if (unlikely(ret)) return ret; From 297b5ccea4acacaa47c150f043bce695202afbf1 Mon Sep 17 00:00:00 2001 From: Vishnu Razdan Date: Tue, 11 Aug 2026 00:01:27 -0700 Subject: [PATCH 11/30] io_uring/io-wq: fix worker accounting when canceling creation callbacks create_worker_cb() reserves an io-wq worker slot only after its task-work callback runs. If the callback is canceled before then, io_worker_cancel_cb() still decrements acct->nr_workers. When an existing worker retires with its creation callback pending, that worker has already decremented the same account's worker count. The resulting undercount permits worker creation beyond the account's configured limit. On an AST2600 OpenBMC system, an unchanged sensor daemon reached 4,291 threads with the original kernel. With an equivalent downstream fix, 25 passive samples under its normal workload showed 6-9 threads. Decrement nr_workers only when the canceled callback is not create_worker_cb(). Continuation callbacks still release their reserved slot, and both callback types retain the existing running-count, reference-count, and create-state cleanup. Fixes: 1d5f5ea7cb7d ("io-wq: remove worker to owner tw dependency") Cc: stable@vger.kernel.org Assisted-by: Codex:gpt-5.6-sol Signed-off-by: Vishnu Razdan Reviewed-by: Gabriel Krisman Bertazi Link: https://patch.msgid.link/20260811-vrazdan-io-wq-b4-submit-v1-1-719ced16c921@openai.com Signed-off-by: Jens Axboe --- io_uring/io-wq.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c index be8d75731d24..2ca223e47d41 100644 --- a/io_uring/io-wq.c +++ b/io_uring/io-wq.c @@ -212,9 +212,12 @@ static void io_worker_cancel_cb(struct io_worker *worker) struct io_wq *wq = worker->wq; atomic_dec(&acct->nr_running); - raw_spin_lock(&acct->workers_lock); - acct->nr_workers--; - raw_spin_unlock(&acct->workers_lock); + /* create_worker_cb() has not reserved a worker slot yet. */ + if (worker->create_work.func != create_worker_cb) { + raw_spin_lock(&acct->workers_lock); + acct->nr_workers--; + raw_spin_unlock(&acct->workers_lock); + } io_worker_ref_put(wq); clear_bit_unlock(0, &worker->create_state); io_worker_release(worker); From 1afcef59fea02c271f5bcf3496402c34b3508a0f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 7 Aug 2026 14:19:19 +0100 Subject: [PATCH 12/30] io_uring/zcrx: scale refilling with large pages io_zcrx_ring_refill() caps the loop by mixing the max number of allocated netmems and the number of available RQEs together, which caps the number of entries to process the pp cache size. As a result, when niovs are heavily fragmented, the refilling logic allocates only a small number of niovs per call on average and sometimes even none. Keep a separate counter for the number of processed RQ entries, which is capped by a roughly calculated from the page size value to keep the cache full. And separately break if it allocates enough niovs. Signed-off-by: Pavel Begunkov Link: https://patch.msgid.link/143cf439299728759eb0a840c866363fe99293f0.1786108672.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 7ad52f499f87..19a432df0ebe 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -28,6 +28,13 @@ #include "zcrx.h" #include "rsrc.h" +#define ZCRX_MAX_FRAGS_PER_PAGE MAX(PAGE_SIZE / 1024, 1) +/* + * We need a reasonable limit to be able to fill in 64 entries on average + * for 1500 byte MTU. Over-estimate it to keep it pow2. + */ +#define ZCRX_REFILL_CAP MIN(64 * ZCRX_MAX_FRAGS_PER_PAGE, 1024) + #define IO_ZCRX_AREA_SUPPORTED_FLAGS (IORING_ZCRX_AREA_DMABUF) #define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) @@ -1124,17 +1131,15 @@ static unsigned io_zcrx_ring_refill(struct page_pool *pp, { struct zcrx_rq *rq = &ifq->rq; unsigned int mask = rq->nr_entries - 1; - unsigned int entries; + unsigned int rqes_left; unsigned allocated = 0; guard(spinlock_bh)(&rq->lock); - entries = zcrx_rq_entries(rq); - entries = min_t(unsigned, entries, to_alloc); - if (unlikely(!entries)) - return 0; + rqes_left = zcrx_rq_entries(rq); + rqes_left = min_t(unsigned, rqes_left, ZCRX_REFILL_CAP); - do { + for (; rqes_left; rqes_left--) { struct io_uring_zcrx_rqe *rqe = zcrx_next_rqe(rq, mask); struct net_iov *niov; netmem_ref netmem; @@ -1155,7 +1160,9 @@ static unsigned io_zcrx_ring_refill(struct page_pool *pp, netmems[allocated] = netmem; allocated++; - } while (--entries); + if (allocated >= to_alloc) + break; + } smp_store_release(&rq->ring->head, rq->cached_head); return allocated; From 433c57e9b175ed82c0f68a58200d2f10a6ea5580 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 7 Aug 2026 14:19:20 +0100 Subject: [PATCH 13/30] io_uring/zcrx: move RQ head/tail to separate cache lines RQ head and tail are currently put into the same cache line, which can cause false sharing problems when refill is run on another CPU. Put them into separate cache lines. Signed-off-by: Pavel Begunkov Reviewed-by: Mina Almasry Link: https://patch.msgid.link/f769116a3f5267f06cb9a9a819d482ddea2e0852.1786108672.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/query.c | 2 +- io_uring/zcrx.c | 8 ++++---- io_uring/zcrx.h | 7 ++++++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/io_uring/query.c b/io_uring/query.c index d529d94aa8f4..2e7b893cc8f0 100644 --- a/io_uring/query.c +++ b/io_uring/query.c @@ -38,7 +38,7 @@ static ssize_t io_query_zcrx(union io_query_data *data) e->register_flags = ZCRX_SUPPORTED_REG_FLAGS; e->area_flags = IORING_ZCRX_AREA_DMABUF; e->nr_ctrl_opcodes = __ZCRX_CTRL_LAST; - e->rq_hdr_size = sizeof(struct io_uring); + e->rq_hdr_size = sizeof(struct zcrx_rq_hdr); e->rq_hdr_alignment = L1_CACHE_BYTES; e->features = ZCRX_FEATURES; e->__resv2 = 0; diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 19a432df0ebe..c81d52b29f30 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -379,9 +379,9 @@ static void io_zcrx_get_niov_uref(struct net_iov *niov) static void io_fill_zcrx_offsets(struct io_uring_zcrx_offsets *offsets) { - offsets->head = offsetof(struct io_uring, head); - offsets->tail = offsetof(struct io_uring, tail); - offsets->rqes = ALIGN(sizeof(struct io_uring), L1_CACHE_BYTES); + offsets->head = offsetof(struct zcrx_rq_hdr, head); + offsets->tail = offsetof(struct zcrx_rq_hdr, tail); + offsets->rqes = ALIGN(sizeof(struct zcrx_rq_hdr), L1_CACHE_BYTES); } static int io_allocate_rbuf_ring(struct io_ring_ctx *ctx, @@ -409,7 +409,7 @@ static int io_allocate_rbuf_ring(struct io_ring_ctx *ctx, return ret; ptr = io_region_get_ptr(&ifq->rq_region); - ifq->rq.ring = (struct io_uring *)ptr; + ifq->rq.ring = (struct zcrx_rq_hdr *)ptr; ifq->rq.rqes = (struct io_uring_zcrx_rqe *)(ptr + off); memset(ifq->rq.ring, 0, sizeof(*ifq->rq.ring)); diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h index fa00900e479e..3cdfa4415d62 100644 --- a/io_uring/zcrx.h +++ b/io_uring/zcrx.h @@ -43,9 +43,14 @@ struct io_zcrx_area { struct io_zcrx_mem mem; }; +struct zcrx_rq_hdr { + u32 head ____cacheline_aligned_in_smp; + u32 tail ____cacheline_aligned_in_smp; +}; + struct zcrx_rq { spinlock_t lock; - struct io_uring *ring; + struct zcrx_rq_hdr *ring; struct io_uring_zcrx_rqe *rqes; u32 cached_head; u32 nr_entries; From 2292f4fbc54ed15b6ad07bf788a17903a3337730 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 7 Aug 2026 14:19:21 +0100 Subject: [PATCH 14/30] io_uring/zcrx: add RQ iterator Add a iterator structure and helper functions for the refill queue processing to avoid polluting io_zcrx_ring_refill() with extra state and logic once it's extended in following patches. Signed-off-by: Pavel Begunkov Link: https://patch.msgid.link/5d2933ea7bcf9c19c44e43afb3638d7afb362b3c.1786108672.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index c81d52b29f30..d1b82daf18f0 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -1087,6 +1087,10 @@ void io_unregister_zcrx(struct io_ring_ctx *ctx) xa_destroy(&ctx->zcrx_ctxs); } +struct zcrx_rq_iter { + int rqes_left; +}; + static inline u32 zcrx_rq_entries(struct zcrx_rq *rq) { u32 entries; @@ -1102,6 +1106,24 @@ static struct io_uring_zcrx_rqe *zcrx_next_rqe(struct zcrx_rq *rq, unsigned mask return &rq->rqes[idx]; } +static inline void zcrx_rq_iter_init(struct zcrx_rq_iter *it, + struct zcrx_rq *rq) +{ + it->rqes_left = min_t(unsigned, zcrx_rq_entries(rq), ZCRX_REFILL_CAP); +} + +static inline bool zcrx_rq_iter_next(struct zcrx_rq_iter *it, + struct zcrx_rq *rq, + struct io_uring_zcrx_rqe **rqe) +{ + it->rqes_left--; + if (unlikely(it->rqes_left < 0)) + return false; + + *rqe = zcrx_next_rqe(rq, rq->nr_entries - 1); + return true; +} + static inline bool io_parse_rqe(struct io_uring_zcrx_rqe *rqe, struct io_zcrx_ifq *ifq, struct net_iov **ret_niov) @@ -1130,17 +1152,15 @@ static unsigned io_zcrx_ring_refill(struct page_pool *pp, netmem_ref *netmems, unsigned to_alloc) { struct zcrx_rq *rq = &ifq->rq; - unsigned int mask = rq->nr_entries - 1; - unsigned int rqes_left; + struct io_uring_zcrx_rqe *rqe; + struct zcrx_rq_iter it; unsigned allocated = 0; guard(spinlock_bh)(&rq->lock); - rqes_left = zcrx_rq_entries(rq); - rqes_left = min_t(unsigned, rqes_left, ZCRX_REFILL_CAP); + zcrx_rq_iter_init(&it, rq); - for (; rqes_left; rqes_left--) { - struct io_uring_zcrx_rqe *rqe = zcrx_next_rqe(rq, mask); + while (zcrx_rq_iter_next(&it, rq, &rqe)) { struct net_iov *niov; netmem_ref netmem; From 888f7ab164508b4c7246dcb4475294d61b2db055 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 7 Aug 2026 14:19:22 +0100 Subject: [PATCH 15/30] io_uring/zcrx: cache RQ tail The RQ tail is updated by the user space. Cache it to reduce cache line bouncing. Refilling now tries to exhaust the previous batch of rqes, but since it could be too low, the iterator is allowed to recalculate the rqes to process once after synching the tail value. Signed-off-by: Pavel Begunkov Link: https://patch.msgid.link/b42d2ed8b0e697110b646573b5da2a5f8e85f92e.1786108672.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 29 ++++++++++++++++++++++------- io_uring/zcrx.h | 1 + 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index d1b82daf18f0..ef17c0e16bd4 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -1089,14 +1089,20 @@ void io_unregister_zcrx(struct io_ring_ctx *ctx) struct zcrx_rq_iter { int rqes_left; + bool flushed; }; +static inline u32 __zcrx_rq_entries(struct zcrx_rq *rq) +{ + u32 entries = rq->cached_tail - rq->cached_head; + + return min(entries, rq->nr_entries); +} + static inline u32 zcrx_rq_entries(struct zcrx_rq *rq) { - u32 entries; - - entries = smp_load_acquire(&rq->ring->tail) - rq->cached_head; - return min(entries, rq->nr_entries); + rq->cached_tail = smp_load_acquire(&rq->ring->tail); + return __zcrx_rq_entries(rq); } static struct io_uring_zcrx_rqe *zcrx_next_rqe(struct zcrx_rq *rq, unsigned mask) @@ -1109,7 +1115,8 @@ static struct io_uring_zcrx_rqe *zcrx_next_rqe(struct zcrx_rq *rq, unsigned mask static inline void zcrx_rq_iter_init(struct zcrx_rq_iter *it, struct zcrx_rq *rq) { - it->rqes_left = min_t(unsigned, zcrx_rq_entries(rq), ZCRX_REFILL_CAP); + it->rqes_left = min_t(unsigned, __zcrx_rq_entries(rq), ZCRX_REFILL_CAP); + it->flushed = false; } static inline bool zcrx_rq_iter_next(struct zcrx_rq_iter *it, @@ -1117,8 +1124,16 @@ static inline bool zcrx_rq_iter_next(struct zcrx_rq_iter *it, struct io_uring_zcrx_rqe **rqe) { it->rqes_left--; - if (unlikely(it->rqes_left < 0)) - return false; + if (unlikely(it->rqes_left < 0)) { + if (it->flushed) + return false; + rq->cached_tail = smp_load_acquire(&rq->ring->tail); + it->rqes_left = min_t(unsigned, __zcrx_rq_entries(rq), + ZCRX_REFILL_CAP); + it->flushed = true; + if (--it->rqes_left < 0) + return false; + } *rqe = zcrx_next_rqe(rq, rq->nr_entries - 1); return true; diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h index 3cdfa4415d62..0eb7ea35a9ff 100644 --- a/io_uring/zcrx.h +++ b/io_uring/zcrx.h @@ -53,6 +53,7 @@ struct zcrx_rq { struct zcrx_rq_hdr *ring; struct io_uring_zcrx_rqe *rqes; u32 cached_head; + u32 cached_tail; u32 nr_entries; }; From 7b188cf990abe5e1f7bbc9d79db22ce1b414d7c3 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 7 Aug 2026 14:19:23 +0100 Subject: [PATCH 16/30] io_uring/zcrx: coalesce same-niov RQEs on refill With large rx piages I often see >10 sequential RQEs referring to the same niov. Instead of putting them one by one, count such RQEs during parsing and batch refcounting for the niov. Signed-off-by: Pavel Begunkov Link: https://patch.msgid.link/71f11d5de47553f187737e0e892ab783b03f53b1.1786108672.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 56 +++++++++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index ef17c0e16bd4..6d196858f5c7 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -358,16 +358,16 @@ static inline atomic_t *io_get_user_counter(struct net_iov *niov) return &area->user_refs[net_iov_idx(niov)]; } -static bool io_zcrx_put_niov_uref(struct net_iov *niov) +static bool io_zcrx_put_niov_uref(struct net_iov *niov, unsigned refs) { atomic_t *uref = io_get_user_counter(niov); int old; old = atomic_read(uref); do { - if (unlikely(old == 0)) + if (unlikely(old < refs)) return false; - } while (!atomic_try_cmpxchg(uref, &old, old - 1)); + } while (!atomic_try_cmpxchg(uref, &old, old - refs)); return true; } @@ -1162,6 +1162,22 @@ static inline bool io_parse_rqe(struct io_uring_zcrx_rqe *rqe, return true; } +static bool zcrx_put_refill_niov(struct net_iov *niov, struct page_pool *pp, + unsigned refs) +{ + netmem_ref netmem = net_iov_to_netmem(niov); + + if (!io_zcrx_put_niov_uref(niov, refs)) + return false; + if (page_pool_unref_netmem(netmem, refs) != 0) + return false; + if (unlikely(niov->desc.pp != pp)) { + io_zcrx_return_niov(niov); + return false; + } + return true; +} + static unsigned io_zcrx_ring_refill(struct page_pool *pp, struct io_zcrx_ifq *ifq, netmem_ref *netmems, unsigned to_alloc) @@ -1169,34 +1185,34 @@ static unsigned io_zcrx_ring_refill(struct page_pool *pp, struct zcrx_rq *rq = &ifq->rq; struct io_uring_zcrx_rqe *rqe; struct zcrx_rq_iter it; + struct net_iov *niov = NULL; + unsigned niov_refs = 0; unsigned allocated = 0; guard(spinlock_bh)(&rq->lock); zcrx_rq_iter_init(&it, rq); - while (zcrx_rq_iter_next(&it, rq, &rqe)) { - struct net_iov *niov; - netmem_ref netmem; + while (allocated < to_alloc - 1 && zcrx_rq_iter_next(&it, rq, &rqe)) { + struct net_iov *next_niov; - if (!io_parse_rqe(rqe, ifq, &niov)) + if (!io_parse_rqe(rqe, ifq, &next_niov)) continue; - if (!io_zcrx_put_niov_uref(niov)) - continue; - - netmem = net_iov_to_netmem(niov); - if (!page_pool_unref_and_test(netmem)) - continue; - - if (unlikely(niov->desc.pp != pp)) { - io_zcrx_return_niov(niov); + if (niov == next_niov) { + niov_refs++; continue; } + if (niov && zcrx_put_refill_niov(niov, pp, niov_refs)) { + netmems[allocated] = net_iov_to_netmem(niov); + allocated++; + } + niov = next_niov; + niov_refs = 1; + } - netmems[allocated] = netmem; + if (niov && zcrx_put_refill_niov(niov, pp, niov_refs)) { + netmems[allocated] = net_iov_to_netmem(niov); allocated++; - if (allocated >= to_alloc) - break; } smp_store_release(&rq->ring->head, rq->cached_head); @@ -1408,7 +1424,7 @@ static void zcrx_return_buffers(netmem_ref *netmems, unsigned nr) netmem_ref netmem = netmems[i]; struct net_iov *niov = netmem_to_net_iov(netmem); - if (!io_zcrx_put_niov_uref(niov)) + if (!io_zcrx_put_niov_uref(niov, 1)) continue; if (!page_pool_unref_and_test(netmem)) continue; From ebb2b81a2d063675a09186a9dffd627ce50e5b9b Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 7 Aug 2026 14:19:24 +0100 Subject: [PATCH 17/30] io_uring/zcrx: constify area_reg on import io_import_area() doesn't modify its struct io_uring_zcrx_area_reg argument, add const to enforce that, it'll make later modifications easier. Signed-off-by: Pavel Begunkov Link: https://patch.msgid.link/2cd4a3e43875576587c900ebdf8f1fb4af266e4a.1786108672.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 6d196858f5c7..07f40267571a 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -136,7 +136,7 @@ static void io_release_dmabuf(struct io_zcrx_mem *mem) static int io_import_dmabuf(struct io_zcrx_ifq *ifq, struct io_zcrx_mem *mem, - struct io_uring_zcrx_area_reg *area_reg) + const struct io_uring_zcrx_area_reg *area_reg) { unsigned long off = (unsigned long)area_reg->addr; unsigned long len = (unsigned long)area_reg->len; @@ -208,7 +208,7 @@ static unsigned long io_count_account_pages(struct page **pages, unsigned nr_pag static int io_import_umem(struct io_zcrx_ifq *ifq, struct io_zcrx_mem *mem, - struct io_uring_zcrx_area_reg *area_reg) + const struct io_uring_zcrx_area_reg *area_reg) { struct page **pages; int nr_pages, ret; @@ -274,7 +274,7 @@ static void io_release_area_mem(struct io_zcrx_mem *mem) static int io_import_area(struct io_zcrx_ifq *ifq, struct io_zcrx_mem *mem, - struct io_uring_zcrx_area_reg *area_reg) + const struct io_uring_zcrx_area_reg *area_reg) { int ret; From 7f32f17ffbcd2bd8157b9602e1e50b0129ef4522 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 7 Aug 2026 14:19:25 +0100 Subject: [PATCH 18/30] io_urint/zcrx: narrow var scope in io_zcrx_recv_skb() A preparation patch that limits scopes of a couple variables in io_zcrx_recv_skb() and rename them, it makes it easier to reason about the code. Signed-off-by: Pavel Begunkov Link: https://patch.msgid.link/182c140502e7be534caf10360714a4d78f2ce5f4.1786108672.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 07f40267571a..0ddb5e46eda9 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -1692,8 +1692,7 @@ io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, struct io_kiocb *req = args->req; struct sk_buff *frag_iter; unsigned start, start_off = offset; - int i, copy, end, off; - int ret = 0; + int i, ret = 0; len = min_t(size_t, len, desc->count); /* @@ -1731,20 +1730,19 @@ io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { const skb_frag_t *frag; + unsigned frag_end; if (WARN_ON(start > offset + len)) return -EFAULT; frag = &skb_shinfo(skb)->frags[i]; - end = start + skb_frag_size(frag); + frag_end = start + skb_frag_size(frag); - if (offset < end) { - copy = end - offset; - if (copy > len) - copy = len; + if (offset < frag_end) { + unsigned copy = min(frag_end - offset, len); + unsigned frag_off = offset - start; - off = offset - start; - ret = io_zcrx_recv_frag(req, ifq, frag, off, copy); + ret = io_zcrx_recv_frag(req, ifq, frag, frag_off, copy); if (ret < 0) goto out; @@ -1753,24 +1751,23 @@ io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, if (len == 0 || ret != copy) goto out; } - start = end; + start = frag_end; } skb_walk_frags(skb, frag_iter) { + unsigned frag_end; + if (WARN_ON(start > offset + len)) return -EFAULT; - end = start + frag_iter->len; - if (offset < end) { + frag_end = start + frag_iter->len; + if (offset < frag_end) { + unsigned copy = min(frag_end - offset, len); + unsigned frag_off = offset - start; size_t count; - copy = end - offset; - if (copy > len) - copy = len; - - off = offset - start; count = desc->count; - ret = io_zcrx_recv_skb(desc, frag_iter, off, copy); + ret = io_zcrx_recv_skb(desc, frag_iter, frag_off, copy); desc->count = count; if (ret < 0) goto out; @@ -1780,7 +1777,7 @@ io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, if (len == 0 || ret != copy) goto out; } - start = end; + start = frag_end; } out: From 2d837b9961ec7ce46d1f1241a88d2b1f57e87f5a Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 7 Aug 2026 14:19:26 +0100 Subject: [PATCH 19/30] io_uring/zcrx: don't reload skb_shinfo Keep skb_shinfo in a local variable so that it doesn't reload it on every iteration of the loop. Signed-off-by: Pavel Begunkov Link: https://patch.msgid.link/1e4c864a5ea639803c866aea70b8d5bf558f189f.1786108672.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 0ddb5e46eda9..e41f3a4e2c0d 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -1692,6 +1692,7 @@ io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, struct io_kiocb *req = args->req; struct sk_buff *frag_iter; unsigned start, start_off = offset; + struct skb_shared_info *shi; int i, ret = 0; len = min_t(size_t, len, desc->count); @@ -1727,17 +1728,15 @@ io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, } start = skb_headlen(skb); + shi = skb_shinfo(skb); - for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { - const skb_frag_t *frag; - unsigned frag_end; + for (i = 0; i < shi->nr_frags; i++) { + const skb_frag_t *frag = &shi->frags[i]; + unsigned frag_end = start + skb_frag_size(frag); if (WARN_ON(start > offset + len)) return -EFAULT; - frag = &skb_shinfo(skb)->frags[i]; - frag_end = start + skb_frag_size(frag); - if (offset < frag_end) { unsigned copy = min(frag_end - offset, len); unsigned frag_off = offset - start; From 84a7fc9b1862dfe9e2e743ca6627d311a5e03545 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 7 Aug 2026 14:19:27 +0100 Subject: [PATCH 20/30] io_uring/zcrx: add helper for deriving area token Add zcrx_area_id_to_token() to deduplicate the way the area token is calculated out of the area index. Signed-off-by: Pavel Begunkov Link: https://patch.msgid.link/c7752161065a1f7273ce3de9683804dd105c6d53.1786108672.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index e41f3a4e2c0d..db529736bdab 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -39,6 +39,11 @@ #define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) +static inline u64 zcrx_area_id_to_token(u32 area_id) +{ + return (u64)area_id << IORING_ZCRX_AREA_SHIFT; +} + static inline struct io_zcrx_ifq *io_pp_to_ifq(struct page_pool *pp) { return pp->mp_priv; @@ -526,7 +531,7 @@ static int io_zcrx_create_area(struct io_zcrx_ifq *ifq, area->free_count = nr_iovs; /* we're only supporting one area per ifq for now */ area->area_id = 0; - area_reg->rq_area_token = (u64)area->area_id << IORING_ZCRX_AREA_SHIFT; + area_reg->rq_area_token = zcrx_area_id_to_token(area->area_id); spin_lock_init(&area->freelist_lock); ret = io_zcrx_append_area(ifq, area); @@ -1532,7 +1537,7 @@ static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov, area = io_zcrx_iov_to_area(niov); offset = off + (net_iov_idx(niov) << ifq->niov_shift); rcqe = (struct io_uring_zcrx_cqe *)(cqe + 1); - rcqe->off = offset + ((u64)area->area_id << IORING_ZCRX_AREA_SHIFT); + rcqe->off = offset + zcrx_area_id_to_token(area->area_id); rcqe->__pad = 0; return true; } From eb840beb51bb17da87968f3b8359183bde11be22 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 7 Aug 2026 14:19:28 +0100 Subject: [PATCH 21/30] io_uring/zcrx: don't pass ifq_reg to area creation We might want to create an area without having an instance of struct io_uring_zcrx_ifq_reg. Extract a helper that doesn't have the ifq registration structure as an argument but takes the buf length explicitly. Signed-off-by: Pavel Begunkov Link: https://patch.msgid.link/76751ee4747abb4893058fcaaa36895dfe24a297.1786108672.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index db529736bdab..8914869ed29a 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -459,21 +459,22 @@ static int io_zcrx_append_area(struct io_zcrx_ifq *ifq, return 0; } -static int io_zcrx_create_area(struct io_zcrx_ifq *ifq, +static int __zcrx_create_area(struct io_zcrx_ifq *ifq, struct io_uring_zcrx_area_reg *area_reg, - struct io_uring_zcrx_ifq_reg *reg) + u32 rx_buf_len) { int buf_size_shift = PAGE_SHIFT; struct io_zcrx_area *area; unsigned nr_iovs; int i, ret; - if (reg->rx_buf_len) { - if (!is_power_of_2(reg->rx_buf_len) || - reg->rx_buf_len < PAGE_SIZE) + if (rx_buf_len) { + if (!is_power_of_2(rx_buf_len) || rx_buf_len < PAGE_SIZE) return -EINVAL; - buf_size_shift = ilog2(reg->rx_buf_len); + buf_size_shift = ilog2(rx_buf_len); } + if (WARN_ON_ONCE(ifq->niov_shift)) + return -EINVAL; if (!ifq->dev && buf_size_shift != PAGE_SHIFT) return -EOPNOTSUPP; @@ -543,6 +544,13 @@ static int io_zcrx_create_area(struct io_zcrx_ifq *ifq, return ret; } +static int io_zcrx_create_area(struct io_zcrx_ifq *ifq, + struct io_uring_zcrx_area_reg *area_reg, + struct io_uring_zcrx_ifq_reg *reg) +{ + return __zcrx_create_area(ifq, area_reg, reg->rx_buf_len); +} + static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx) { struct io_zcrx_ifq *ifq; From be94c24ce377c528f10176906f2b939d90b53716 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 7 Aug 2026 14:19:29 +0100 Subject: [PATCH 22/30] io_uring/zcrx: split dmabuf unmap and release Until now unmapping and destroying dmabuf were the same thing. To keep it consistent with non-dmabuf, split it into two separate helpers. Unmap destroys mappings and attachements as it should, and release only putting down the dmabuf fd reference. Signed-off-by: Pavel Begunkov Link: https://patch.msgid.link/bd8c1fd8dac85ce0cf839b389ecd6050761965ca.1786108672.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 8914869ed29a..ceca13de070f 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -121,21 +121,25 @@ static int io_populate_area_dma(struct io_zcrx_ifq *ifq, return 0; } -static void io_release_dmabuf(struct io_zcrx_mem *mem) +static void io_unmap_dmabuf(struct io_zcrx_mem *mem) { if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER)) return; - if (mem->sgt) dma_buf_unmap_attachment_unlocked(mem->attach, mem->sgt, DMA_FROM_DEVICE); if (mem->attach) dma_buf_detach(mem->dmabuf, mem->attach); - if (mem->dmabuf) - dma_buf_put(mem->dmabuf); - mem->sgt = NULL; mem->attach = NULL; +} + +static void io_release_dmabuf(struct io_zcrx_mem *mem) +{ + if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER)) + return; + if (mem->dmabuf) + dma_buf_put(mem->dmabuf); mem->dmabuf = NULL; } @@ -190,6 +194,7 @@ static int io_import_dmabuf(struct io_zcrx_ifq *ifq, mem->size = len; return 0; err: + io_unmap_dmabuf(mem); io_release_dmabuf(mem); return ret; } @@ -317,7 +322,7 @@ static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq, } if (area->mem.is_dmabuf) { - io_release_dmabuf(&area->mem); + io_unmap_dmabuf(&area->mem); } else { dma_unmap_sgtable(ifq->dev, &area->mem.page_sg_table, DMA_FROM_DEVICE, IO_DMA_ATTR); From 6bc4d9dec9059f1373eb140fc80b6f64fd153ecd Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 7 Aug 2026 14:19:30 +0100 Subject: [PATCH 23/30] io_uring/zcrx: unmap under netdev lock Make sure we unmap areas while closing a queue. Signed-off-by: Pavel Begunkov Link: https://patch.msgid.link/1c41f349f8bcbcaafb17a9c81d4157aa9d30de93.1786108672.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index ceca13de070f..8f9654d4a6de 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -311,6 +311,9 @@ static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq, { int i; + if (!area) + return; + guard(mutex)(&ifq->pp_lock); if (!area->is_mapped) return; @@ -437,7 +440,8 @@ static void io_free_rbuf_ring(struct io_zcrx_ifq *ifq) static void io_zcrx_free_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) { - io_zcrx_unmap_area(ifq, area); + if (WARN_ON_ONCE(area->is_mapped)) + return; io_release_area_mem(&area->mem); if (area->mem.account_pages) @@ -544,8 +548,10 @@ static int __zcrx_create_area(struct io_zcrx_ifq *ifq, if (!ret) return 0; err: - if (area) + if (area) { + io_zcrx_unmap_area(ifq, area); io_zcrx_free_area(ifq, area); + } return ret; } @@ -599,11 +605,12 @@ static void io_close_queue(struct io_zcrx_ifq *ifq) } if (netdev) { - if (ifq->if_rxq != -1) { - netdev_lock(netdev); + netdev_lock(netdev); + if (ifq->if_rxq != -1) netif_mp_close_rxq(netdev, ifq->if_rxq, &p); - netdev_unlock(netdev); - } + + io_zcrx_unmap_area(ifq, ifq->area); + netdev_unlock(netdev); netdev_put(netdev, &netdev_tracker); } ifq->if_rxq = -1; @@ -1396,8 +1403,7 @@ static void io_pp_uninstall(void *mp_priv, struct netdev_rx_queue *rxq) struct io_zcrx_ifq *ifq = mp_priv; io_zcrx_drop_netdev(ifq); - if (ifq->area) - io_zcrx_unmap_area(ifq, ifq->area); + io_zcrx_unmap_area(ifq, ifq->area); p->mp_ops = NULL; p->mp_priv = NULL; From 7d5d94fe9af1b402271236ff8536252789f3eb17 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 7 Aug 2026 14:19:31 +0100 Subject: [PATCH 24/30] io_uring/zcrx: move freelist lock to struct zcrx freelist_lock, which protects slow path allocations, is currently stored in struct io_zcrx_area. Once we add support for multiple queues, we'll need a lock in the zcrx ctx, move it there. Signed-off-by: Pavel Begunkov Link: https://patch.msgid.link/be7feadff1e210861433f99fe44170aa27ec5b33.1786108672.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 14 +++++++------- io_uring/zcrx.h | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 8f9654d4a6de..7fda74475c46 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -542,7 +542,6 @@ static int __zcrx_create_area(struct io_zcrx_ifq *ifq, /* we're only supporting one area per ifq for now */ area->area_id = 0; area_reg->rq_area_token = zcrx_area_id_to_token(area->area_id); - spin_lock_init(&area->freelist_lock); ret = io_zcrx_append_area(ifq, area); if (!ret) @@ -573,6 +572,7 @@ static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx) ifq->if_rxq = -1; spin_lock_init(&ifq->ctx_lock); spin_lock_init(&ifq->rq.lock); + spin_lock_init(&ifq->alloc_lock); mutex_init(&ifq->pp_lock); refcount_set(&ifq->refs, 1); refcount_set(&ifq->user_refs, 1); @@ -647,8 +647,9 @@ static void io_put_zcrx_ifq(struct io_zcrx_ifq *ifq) static void io_zcrx_return_niov_freelist(struct net_iov *niov) { struct io_zcrx_area *area = io_zcrx_iov_to_area(niov); + struct io_zcrx_ifq *ifq = area->ifq; - guard(spinlock_bh)(&area->freelist_lock); + guard(spinlock_bh)(&ifq->alloc_lock); if (WARN_ON_ONCE(area->free_count >= area->nia.num_niovs)) return; area->freelist[area->free_count++] = net_iov_idx(niov); @@ -658,7 +659,7 @@ static struct net_iov *zcrx_get_free_niov(struct io_zcrx_area *area) { unsigned niov_idx; - lockdep_assert_held(&area->freelist_lock); + lockdep_assert_held(&area->ifq->alloc_lock); if (unlikely(!area->free_count)) return NULL; @@ -1250,7 +1251,7 @@ static unsigned io_zcrx_refill_slow(struct page_pool *pp, struct io_zcrx_ifq *if struct io_zcrx_area *area = ifq->area; unsigned allocated = 0; - guard(spinlock_bh)(&area->freelist_lock); + guard(spinlock_bh)(&ifq->alloc_lock); for (allocated = 0; allocated < to_alloc; allocated++) { struct net_iov *niov = zcrx_get_free_niov(area); @@ -1563,14 +1564,13 @@ static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov, static struct net_iov *io_alloc_fallback_niov(struct io_zcrx_ifq *ifq) { - struct io_zcrx_area *area = ifq->area; struct net_iov *niov = NULL; if (!ifq->kern_readable) return NULL; - scoped_guard(spinlock_bh, &area->freelist_lock) - niov = zcrx_get_free_niov(area); + scoped_guard(spinlock_bh, &ifq->alloc_lock) + niov = zcrx_get_free_niov(ifq->area); if (niov) page_pool_fragment_netmem(net_iov_to_netmem(niov), 1); diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h index 0eb7ea35a9ff..302659669ba4 100644 --- a/io_uring/zcrx.h +++ b/io_uring/zcrx.h @@ -36,7 +36,6 @@ struct io_zcrx_area { u16 area_id; /* freelist */ - spinlock_t freelist_lock ____cacheline_aligned_in_smp; u32 free_count; u32 *freelist; @@ -65,6 +64,7 @@ struct io_zcrx_ifq { bool kern_readable; struct zcrx_rq rq ____cacheline_aligned_in_smp; + spinlock_t alloc_lock ____cacheline_aligned_in_smp; u32 if_rxq; struct device *dev; From ef0f0e10ee66d9c96298b5331515f46ab074957e Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 7 Aug 2026 14:19:32 +0100 Subject: [PATCH 25/30] io_uring/zcrx: keep array of areas Currently, we have only a one area per zcrx instance, and struct io_zcrx_ifq stores a single pointer. To prepare for adding more areas, replace it with an array of areas. We'll be creating them at runtime, and the array is protected by 3 locks: ->pp_lock, ->alloc_lock and ->rq.lock. It takes all of them when switching arrays, and readers should hold either of them. Signed-off-by: Pavel Begunkov Link: https://patch.msgid.link/d186adfa0ea26f651804cffef77d477f257eb404.1786108672.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 112 +++++++++++++++++++++++++++++++++++------------- io_uring/zcrx.h | 5 ++- 2 files changed, 87 insertions(+), 30 deletions(-) diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 7fda74475c46..8a9109cb7231 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -306,16 +306,14 @@ static int io_import_area(struct io_zcrx_ifq *ifq, return io_import_umem(ifq, mem, area_reg); } -static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq, +static void __io_zcrx_unmap_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) { int i; - if (!area) - return; + lockdep_assert_held(&ifq->pp_lock); - guard(mutex)(&ifq->pp_lock); - if (!area->is_mapped) + if (!area || !area->is_mapped) return; area->is_mapped = false; @@ -332,6 +330,23 @@ static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq, } } +static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq, + struct io_zcrx_area *area) +{ + guard(mutex)(&ifq->pp_lock); + __io_zcrx_unmap_area(ifq, area); +} + +static void io_zcrx_unmap_areas(struct io_zcrx_ifq *ifq) +{ + unsigned area_idx; + + guard(mutex)(&ifq->pp_lock); + + for (area_idx = 0; area_idx < ifq->nr_areas; area_idx++) + __io_zcrx_unmap_area(ifq, ifq->areas[area_idx]); +} + static void zcrx_sync_for_device(struct page_pool *pp, struct io_zcrx_ifq *zcrx, netmem_ref *netmems, unsigned nr) { @@ -458,13 +473,29 @@ static int io_zcrx_append_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) { bool kern_readable = !area->mem.is_dmabuf; + struct io_zcrx_area **areas, **old_areas; + unsigned old_nr; - if (WARN_ON_ONCE(ifq->area)) - return -EINVAL; if (WARN_ON_ONCE(ifq->kern_readable != kern_readable)) return -EINVAL; - ifq->area = area; + old_areas = ifq->areas; + old_nr = ifq->nr_areas; + + areas = kmalloc_array(old_nr + 1, sizeof(areas[0]), + GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!areas) + return -ENOMEM; + if (old_areas) + memcpy(areas, old_areas, old_nr * sizeof(areas[0])); + areas[old_nr] = area; + + scoped_guard(spinlock_bh, &ifq->rq.lock) { + guard(spinlock_bh)(&ifq->alloc_lock); + ifq->areas = areas; + ifq->nr_areas = old_nr + 1; + } + kfree(old_areas); return 0; } @@ -609,7 +640,7 @@ static void io_close_queue(struct io_zcrx_ifq *ifq) if (ifq->if_rxq != -1) netif_mp_close_rxq(netdev, ifq->if_rxq, &p); - io_zcrx_unmap_area(ifq, ifq->area); + io_zcrx_unmap_areas(ifq); netdev_unlock(netdev); netdev_put(netdev, &netdev_tracker); } @@ -618,6 +649,8 @@ static void io_close_queue(struct io_zcrx_ifq *ifq) static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq) { + int i; + if (WARN_ON_ONCE(ifq->if_rxq != -1)) return; if (WARN_ON_ONCE(ifq->netdev != NULL)) @@ -625,8 +658,8 @@ static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq) if (WARN_ON_ONCE(ifq->master_ctx)) return; - if (ifq->area) - io_zcrx_free_area(ifq, ifq->area); + for (i = 0; i < ifq->nr_areas; i++) + io_zcrx_free_area(ifq, ifq->areas[i]); if (ifq->mm_account) mmdrop(ifq->mm_account); if (ifq->dev) @@ -635,6 +668,7 @@ static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq) io_free_rbuf_ring(ifq); free_uid(ifq->user); mutex_destroy(&ifq->pp_lock); + kfree(ifq->areas); kfree(ifq); } @@ -680,14 +714,10 @@ static void io_zcrx_return_niov(struct net_iov *niov) page_pool_put_unrefed_netmem(niov->desc.pp, netmem, -1, false); } -static void io_zcrx_scrub(struct io_zcrx_ifq *ifq) +static void io_zcrx_scrub_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) { - struct io_zcrx_area *area = ifq->area; int i; - if (!area) - return; - /* Reclaim back all buffers given to the user space. */ for (i = 0; i < area->nia.num_niovs; i++) { struct net_iov *niov = &area->nia.niovs[i]; @@ -701,6 +731,15 @@ static void io_zcrx_scrub(struct io_zcrx_ifq *ifq) } } +static void io_zcrx_scrub(struct io_zcrx_ifq *ifq) +{ + int i; + + guard(mutex)(&ifq->pp_lock); + for (i = 0; i < ifq->nr_areas; i++) + io_zcrx_scrub_area(ifq, ifq->areas[i]); +} + static void zcrx_unregister_user(struct io_zcrx_ifq *ifq, struct io_ring_ctx *ctx) { scoped_guard(spinlock_bh, &ifq->ctx_lock) { @@ -1173,12 +1212,15 @@ static inline bool io_parse_rqe(struct io_uring_zcrx_rqe *rqe, unsigned niov_idx, area_idx; struct io_zcrx_area *area; + lockdep_assert_held(&ifq->rq.lock); + area_idx = off >> IORING_ZCRX_AREA_SHIFT; niov_idx = (off & ~IORING_ZCRX_AREA_MASK) >> ifq->niov_shift; - if (unlikely(rqe->__pad || area_idx)) + if (unlikely(rqe->__pad || area_idx >= ifq->nr_areas)) return false; - area = ifq->area; + area_idx = array_index_nospec(area_idx, ifq->nr_areas); + area = ifq->areas[area_idx]; if (unlikely(niov_idx >= area->nia.num_niovs)) return false; @@ -1248,18 +1290,24 @@ static unsigned io_zcrx_ring_refill(struct page_pool *pp, static unsigned io_zcrx_refill_slow(struct page_pool *pp, struct io_zcrx_ifq *ifq, netmem_ref *netmems, unsigned to_alloc) { - struct io_zcrx_area *area = ifq->area; + unsigned area_idx = 0; unsigned allocated = 0; guard(spinlock_bh)(&ifq->alloc_lock); - for (allocated = 0; allocated < to_alloc; allocated++) { - struct net_iov *niov = zcrx_get_free_niov(area); + while (allocated < to_alloc) { + struct net_iov *niov = zcrx_get_free_niov(ifq->areas[area_idx]); + + if (!niov) { + area_idx++; + if (area_idx >= ifq->nr_areas) + break; + continue; + } - if (!niov) - break; net_mp_niov_set_page_pool(pp, niov); netmems[allocated] = net_iov_to_netmem(niov); + allocated++; } return allocated; } @@ -1403,8 +1451,8 @@ static void io_pp_uninstall(void *mp_priv, struct netdev_rx_queue *rxq) struct pp_memory_provider_params *p = &rxq->mp_params; struct io_zcrx_ifq *ifq = mp_priv; + io_zcrx_unmap_areas(ifq); io_zcrx_drop_netdev(ifq); - io_zcrx_unmap_area(ifq, ifq->area); p->mp_ops = NULL; p->mp_priv = NULL; @@ -1565,16 +1613,22 @@ static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov, static struct net_iov *io_alloc_fallback_niov(struct io_zcrx_ifq *ifq) { struct net_iov *niov = NULL; + unsigned area_idx; if (!ifq->kern_readable) return NULL; - scoped_guard(spinlock_bh, &ifq->alloc_lock) - niov = zcrx_get_free_niov(ifq->area); + guard(spinlock_bh)(&ifq->alloc_lock); - if (niov) - page_pool_fragment_netmem(net_iov_to_netmem(niov), 1); - return niov; + for (area_idx = 0; area_idx < ifq->nr_areas; area_idx++) { + niov = zcrx_get_free_niov(ifq->areas[area_idx]); + if (niov) { + page_pool_fragment_netmem(net_iov_to_netmem(niov), 1); + return niov; + } + } + + return NULL; } struct io_copy_cache { diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h index 302659669ba4..05598f08eda0 100644 --- a/io_uring/zcrx.h +++ b/io_uring/zcrx.h @@ -57,7 +57,10 @@ struct zcrx_rq { }; struct io_zcrx_ifq { - struct io_zcrx_area *area; + /* read-protected by any of: ->pp_lock, ->alloc_lock, ->rq.lock */ + struct io_zcrx_area **areas; + unsigned nr_areas; + unsigned niov_shift; struct user_struct *user; struct mm_struct *mm_account; From 5a22bfdd615bf47a0ea5e11f5090b1011764b3a4 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 7 Aug 2026 14:19:33 +0100 Subject: [PATCH 26/30] io_uring/zcrx: lock area creation with pp_lock Protect __zcrx_create_area() with pp_lock. It's not needed for now, nobody can take the lock in parallel, but we'll need it for dynamic area creation for avoiding races with the device dying. Signed-off-by: Pavel Begunkov Link: https://patch.msgid.link/a667399971120227a28d54ebabc50e707f717bf3.1786108672.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/zcrx.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 8a9109cb7231..22f04a9f2410 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -330,13 +330,6 @@ static void __io_zcrx_unmap_area(struct io_zcrx_ifq *ifq, } } -static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq, - struct io_zcrx_area *area) -{ - guard(mutex)(&ifq->pp_lock); - __io_zcrx_unmap_area(ifq, area); -} - static void io_zcrx_unmap_areas(struct io_zcrx_ifq *ifq) { unsigned area_idx; @@ -508,6 +501,8 @@ static int __zcrx_create_area(struct io_zcrx_ifq *ifq, unsigned nr_iovs; int i, ret; + lockdep_assert_held(&ifq->pp_lock); + if (rx_buf_len) { if (!is_power_of_2(rx_buf_len) || rx_buf_len < PAGE_SIZE) return -EINVAL; @@ -579,7 +574,7 @@ static int __zcrx_create_area(struct io_zcrx_ifq *ifq, return 0; err: if (area) { - io_zcrx_unmap_area(ifq, area); + __io_zcrx_unmap_area(ifq, area); io_zcrx_free_area(ifq, area); } return ret; @@ -589,6 +584,7 @@ static int io_zcrx_create_area(struct io_zcrx_ifq *ifq, struct io_uring_zcrx_area_reg *area_reg, struct io_uring_zcrx_ifq_reg *reg) { + guard(mutex)(&ifq->pp_lock); return __zcrx_create_area(ifq, area_reg, reg->rx_buf_len); } From 3c8a5e271594f6ba6d2af40f229cbcd0bcb08c06 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 7 Aug 2026 14:19:34 +0100 Subject: [PATCH 27/30] io_uring/zcrx: add dynamic area provisioning It's not always possible for the user to predict during registration how much memory zcrx will need to sustain the traffic. Allow to dynamically add more areas with a new ctrl code ZCRX_CTRL_ADD_AREA. Signed-off-by: Pavel Begunkov Link: https://patch.msgid.link/43495ca686713b3f0014c9f47c4c0276d704da4d.1786108672.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- include/uapi/linux/io_uring/zcrx.h | 7 ++++ io_uring/zcrx.c | 56 ++++++++++++++++++++++++++---- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/include/uapi/linux/io_uring/zcrx.h b/include/uapi/linux/io_uring/zcrx.h index 15c05c45ce36..08cdb173b04b 100644 --- a/include/uapi/linux/io_uring/zcrx.h +++ b/include/uapi/linux/io_uring/zcrx.h @@ -116,6 +116,7 @@ enum zcrx_ctrl_op { ZCRX_CTRL_FLUSH_RQ, ZCRX_CTRL_EXPORT, ZCRX_CTRL_ARM_NOTIFICATION, + ZCRX_CTRL_ADD_AREA, __ZCRX_CTRL_LAST, }; @@ -134,6 +135,11 @@ struct zcrx_ctrl_arm_notif { __u32 __resv[11]; }; +struct zcrx_ctrl_add_area { + __u64 area_ptr; /* pointer to struct io_uring_zcrx_area_reg */ + __u64 __resv[5]; +}; + struct zcrx_ctrl { __u32 zcrx_id; __u32 op; /* see enum zcrx_ctrl_op */ @@ -143,6 +149,7 @@ struct zcrx_ctrl { struct zcrx_ctrl_export zc_export; struct zcrx_ctrl_flush_rq zc_flush; struct zcrx_ctrl_arm_notif zc_arm_notif; + struct zcrx_ctrl_add_area zc_area; }; }; diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 22f04a9f2410..194418833b9c 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -36,9 +36,15 @@ #define ZCRX_REFILL_CAP MIN(64 * ZCRX_MAX_FRAGS_PER_PAGE, 1024) #define IO_ZCRX_AREA_SUPPORTED_FLAGS (IORING_ZCRX_AREA_DMABUF) +#define ZCRX_MAX_AREAS 1024 #define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) +static inline u32 zcrx_next_area_id(struct io_zcrx_ifq *zcrx) +{ + return zcrx->nr_areas; +} + static inline u64 zcrx_area_id_to_token(u32 area_id) { return (u64)area_id << IORING_ZCRX_AREA_SHIFT; @@ -334,7 +340,7 @@ static void io_zcrx_unmap_areas(struct io_zcrx_ifq *ifq) { unsigned area_idx; - guard(mutex)(&ifq->pp_lock); + lockdep_assert_held(&ifq->pp_lock); for (area_idx = 0; area_idx < ifq->nr_areas; area_idx++) __io_zcrx_unmap_area(ifq, ifq->areas[area_idx]); @@ -469,7 +475,9 @@ static int io_zcrx_append_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area **areas, **old_areas; unsigned old_nr; - if (WARN_ON_ONCE(ifq->kern_readable != kern_readable)) + if (ifq->kern_readable != kern_readable) + return -EINVAL; + if (ifq->nr_areas + 1 > ZCRX_MAX_AREAS) return -EINVAL; old_areas = ifq->areas; @@ -508,7 +516,7 @@ static int __zcrx_create_area(struct io_zcrx_ifq *ifq, return -EINVAL; buf_size_shift = ilog2(rx_buf_len); } - if (WARN_ON_ONCE(ifq->niov_shift)) + if (ifq->niov_shift && ifq->niov_shift != buf_size_shift) return -EINVAL; if (!ifq->dev && buf_size_shift != PAGE_SHIFT) return -EOPNOTSUPP; @@ -566,7 +574,7 @@ static int __zcrx_create_area(struct io_zcrx_ifq *ifq, area->free_count = nr_iovs; /* we're only supporting one area per ifq for now */ - area->area_id = 0; + area->area_id = zcrx_next_area_id(ifq); area_reg->rq_area_token = zcrx_area_id_to_token(area->area_id); ret = io_zcrx_append_area(ifq, area); @@ -608,7 +616,7 @@ static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx) static void io_zcrx_drop_netdev(struct io_zcrx_ifq *ifq) { - guard(mutex)(&ifq->pp_lock); + lockdep_assert_held(&ifq->pp_lock); if (!ifq->netdev) return; @@ -636,7 +644,8 @@ static void io_close_queue(struct io_zcrx_ifq *ifq) if (ifq->if_rxq != -1) netif_mp_close_rxq(netdev, ifq->if_rxq, &p); - io_zcrx_unmap_areas(ifq); + scoped_guard(mutex, &ifq->pp_lock) + io_zcrx_unmap_areas(ifq); netdev_unlock(netdev); netdev_put(netdev, &netdev_tracker); } @@ -994,6 +1003,8 @@ int io_register_zcrx(struct io_ring_ctx *ctx, if (copy_from_user(&area, u64_to_user_ptr(reg.area_ptr), sizeof(area))) return -EFAULT; + if (area.rq_area_token) + return -EINVAL; memset(¬if, 0, sizeof(notif)); if (reg.notif_desc && copy_from_user(¬if, u64_to_user_ptr(reg.notif_desc), @@ -1056,6 +1067,8 @@ int io_register_zcrx(struct io_ring_ctx *ctx, goto err; } + WARN_ON_ONCE(!ifq->niov_shift); + reg.zcrx_id = id; scoped_guard(mutex, &ctx->mmap_lock) { @@ -1447,6 +1460,7 @@ static void io_pp_uninstall(void *mp_priv, struct netdev_rx_queue *rxq) struct pp_memory_provider_params *p = &rxq->mp_params; struct io_zcrx_ifq *ifq = mp_priv; + guard(mutex)(&ifq->pp_lock); io_zcrx_unmap_areas(ifq); io_zcrx_drop_netdev(ifq); @@ -1549,6 +1563,34 @@ static int zcrx_arm_notif(struct io_ring_ctx *ctx, struct io_zcrx_ifq *zcrx, return 0; } +static int zcrx_ctrl_add_area(struct io_ring_ctx *ctx, struct io_zcrx_ifq *ifq, + struct zcrx_ctrl *ctrl) +{ + struct zcrx_ctrl_add_area *ctrl_add = &ctrl->zc_area; + struct io_uring_zcrx_area_reg __user *area_uptr; + struct io_uring_zcrx_area_reg area_reg; + + area_uptr = u64_to_user_ptr(ctrl_add->area_ptr); + if (copy_from_user(&area_reg, area_uptr, sizeof(area_reg))) + return -EFAULT; + if (!mem_is_zero(&ctrl_add->__resv, sizeof(ctrl_add->__resv))) + return -EINVAL; + if (area_reg.rq_area_token) + return -EINVAL; + + guard(mutex)(&ifq->pp_lock); + if (ifq->dev && !ifq->netdev) + return -EFAULT; + + /* we can't safely roll back area append, copy it out first */ + area_reg.rq_area_token = zcrx_area_id_to_token(zcrx_next_area_id(ifq)); + if (copy_to_user(area_uptr, &area_reg, sizeof(area_reg))) + return -EFAULT; + area_reg.rq_area_token = 0; + + return __zcrx_create_area(ifq, &area_reg, 1U << ifq->niov_shift); +} + int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) { struct zcrx_ctrl ctrl; @@ -1575,6 +1617,8 @@ int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) return zcrx_export(ctx, zcrx, &ctrl, arg); case ZCRX_CTRL_ARM_NOTIFICATION: return zcrx_arm_notif(ctx, zcrx, &ctrl); + case ZCRX_CTRL_ADD_AREA: + return zcrx_ctrl_add_area(ctx, zcrx, &ctrl); } return -EOPNOTSUPP; From f12f0234cc14886bcfd53ffb7c8df4216dad51f1 Mon Sep 17 00:00:00 2001 From: Ali Ahmet Memis Date: Thu, 6 Aug 2026 18:00:44 +0000 Subject: [PATCH 28/30] io_uring/memmap: account the pages a compound region really uses io_mem_alloc_compound() allocates get_order(size) pages, which rounds a region size that is not a power of two up to the next order. The pages past the region are part of the same allocation and cannot be used for anything else, but io_create_region() accounts reg->size >> PAGE_SHIFT, so they are never charged against RLIMIT_MEMLOCK. For a ring with 4096 SQ entries and the default CQ size the region is 37 pages while the allocation is 64. Account the tail pages together with the region, and fall back to the exact sized bulk allocation when they do not fit the limit, so a user close to their limit still gets the region rather than an error. io_free_region() gives the same amount back, the compound case being the one that set IO_REGION_F_SINGLE_REF. Counting how many 4096 entry rings an unprivileged user can create under a given RLIMIT_MEMLOCK, before and after: limit (pages) before after 256 2 2 300 2 2 350 3 2 400 3 3 512 5 4 Five rings under a 512 page limit really pin 640 pages. Fixes: dfbbfbf19187 ("io_uring: introduce concept of memory regions") Link: https://lore.kernel.org/all/87ik5ncj8d.fsf@mailhost.krisman.be/ Signed-off-by: Ali Ahmet Memis Link: https://patch.msgid.link/20260806180044.275543-1-ali@iusegentoo.com Signed-off-by: Jens Axboe --- io_uring/memmap.c | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/io_uring/memmap.c b/io_uring/memmap.c index da1f6c5d07f8..71ee6c8728f7 100644 --- a/io_uring/memmap.c +++ b/io_uring/memmap.c @@ -16,8 +16,10 @@ #include "zcrx.h" static bool io_mem_alloc_compound(struct page **pages, int nr_pages, - size_t size, gfp_t gfp) + size_t size, gfp_t gfp, + struct user_struct *user) { + unsigned long nr_compound, extra; struct page *page; int i, order; @@ -27,10 +29,23 @@ static bool io_mem_alloc_compound(struct page **pages, int nr_pages, else if (order) gfp |= __GFP_COMP; - page = alloc_pages(gfp, order); - if (!page) + /* + * get_order() rounds a non power of two size up, so the allocation + * can hold more pages than the region exposes. Account those too, + * and leave the compound allocation alone if they do not fit. + */ + nr_compound = 1UL << order; + extra = nr_compound - nr_pages; + if (extra && user && __io_account_mem(user, extra)) return false; + page = alloc_pages(gfp, order); + if (!page) { + if (extra && user) + __io_unaccount_mem(user, extra); + return false; + } + for (i = 0; i < nr_pages; i++) pages[i] = page + i; @@ -105,8 +120,15 @@ void io_free_region(struct user_struct *user, struct io_mapped_region *mr) } if ((mr->flags & IO_REGION_F_VMAP) && mr->ptr) vunmap(mr->ptr); - if (mr->nr_pages && user) - __io_unaccount_mem(user, mr->nr_pages); + if (mr->nr_pages && user) { + unsigned long nr_accounted = mr->nr_pages; + + /* a compound region was accounted for the whole allocation */ + if (mr->flags & IO_REGION_F_SINGLE_REF) + nr_accounted = 1UL << get_order(io_region_size(mr)); + + __io_unaccount_mem(user, nr_accounted); + } memset(mr, 0, sizeof(*mr)); } @@ -151,7 +173,8 @@ static int io_region_pin_pages(struct io_mapped_region *mr, static int io_region_allocate_pages(struct io_mapped_region *mr, struct io_uring_region_desc *reg, - unsigned long mmap_offset) + unsigned long mmap_offset, + struct user_struct *user) { gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN; size_t size = io_region_size(mr); @@ -162,7 +185,7 @@ static int io_region_allocate_pages(struct io_mapped_region *mr, if (!pages) return -ENOMEM; - if (io_mem_alloc_compound(pages, mr->nr_pages, size, gfp)) { + if (io_mem_alloc_compound(pages, mr->nr_pages, size, gfp, user)) { mr->flags |= IO_REGION_F_SINGLE_REF; goto done; } @@ -217,7 +240,7 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr, if (reg->flags & IORING_MEM_REGION_TYPE_USER) ret = io_region_pin_pages(mr, reg); else - ret = io_region_allocate_pages(mr, reg, mmap_offset); + ret = io_region_allocate_pages(mr, reg, mmap_offset, ctx->user); if (ret) goto out_free; From 360941242f09437a1e07cbed9b5a96663ffeabb6 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 15 Aug 2026 17:53:44 -0600 Subject: [PATCH 29/30] io_uring/uring_cmd: don't skip completion for a synchronous multishot cmd If IORING_URING_CMD_MULTISHOT is set, io_uring_cmd() treats any non-negative return from ->uring_cmd() as the driver having taken ownership of the request and returns IOU_ISSUE_SKIP_COMPLETE. But nothing guarantees that the driver did so, and any handler that just completes the command inline and returns 0 or a positive result then leaves the request orphaned, leaking the io_kiocb, the async data, and the file reference. The special case isn't needed. ublk returns -EIOCBQUEUED for the multishot fetch command, which is passed through as-is, and the poll driven socket timestamp command returns -EAGAIN. Kill it, a multishot handler that wants to hang on to the request must return -EIOCBQUEUED or -EAGAIN like any other command. Fixes: 620a50c92700 ("io_uring: uring_cmd: add multishot support") Cc: stable@vger.kernel.org Reported-by: syzbot+a4ccdd7ebf452e4d4701@syzkaller.appspotmail.com Closes: https://lore.kernel.org/all/6a7a0194.b50370da.49fe0.0056.GAE@google.com/ Link: https://lore.kernel.org/all/20260811115125.1831170-1-vasilisalmpanis@gmail.com/ Signed-off-by: Jens Axboe --- io_uring/uring_cmd.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index ea1432251ccf..0642995ac3f6 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -273,10 +273,6 @@ int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags) } ret = file->f_op->uring_cmd(ioucmd, issue_flags); - if (ioucmd->flags & IORING_URING_CMD_MULTISHOT) { - if (ret >= 0) - return IOU_ISSUE_SKIP_COMPLETE; - } if (ret == -EAGAIN) { ioucmd->flags |= IORING_URING_CMD_REISSUE; return ret; From 881dc9dd66d2367e7fe1be0f953a8837cb3b26a9 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Tue, 18 Aug 2026 19:41:34 +0100 Subject: [PATCH 30/30] io_uring: Add missing include for ITER_SOURCE and ITER_DEST Fix IWYU issues: /tmp/next/build/include/linux/io_uring_types.h:56:32: error: 'ITER_DEST' undeclared here (not in a function) 56 | IO_BUF_DEST = 1 << ITER_DEST, | ^~~~~~~~~ /tmp/next/build/include/linux/io_uring_types.h:57:32: error: 'ITER_SOURCE' undeclared here (not in a function) 57 | IO_BUF_SOURCE = 1 << ITER_SOURCE, | ^~~~~~~~~~~ Fixes: 95961b72c57b2 ("io_uring/rsrc: rename and export IO_IMU_DEST / IO_IMU_SOURCE") Signed-off-by: Mark Brown Link: https://patch.msgid.link/20260818184134.384991-1-broonie@kernel.org Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index f6e90cc64a1f..94936ee297ef 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -6,6 +6,7 @@ #include #include #include +#include #include struct iou_loop_params;