From 4d9be388910e5fbdb3f2794ed20737515ca6b96d Mon Sep 17 00:00:00 2001 From: Yousef Alhouseen Date: Wed, 24 Jun 2026 14:33:36 +0200 Subject: [PATCH 01/23] tools/workqueue: parse help before importing drgn wq_monitor.py and wq_dump.py import drgn before argparse can handle "-h". That makes help fail on systems where drgn is not installed, even though the scripts do not need drgn to print usage text. Parse arguments before importing drgn so the help path works without the runtime debugging dependency. Normal execution still imports drgn before reading kernel state. Signed-off-by: Yousef Alhouseen Signed-off-by: Tejun Heo --- tools/workqueue/wq_dump.py | 10 +++++----- tools/workqueue/wq_monitor.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/workqueue/wq_dump.py b/tools/workqueue/wq_dump.py index ce4161f52f2f..a0c72237531f 100644 --- a/tools/workqueue/wq_dump.py +++ b/tools/workqueue/wq_dump.py @@ -46,6 +46,11 @@ each workqueue: import sys +import argparse +parser = argparse.ArgumentParser(description=desc, + formatter_class=argparse.RawTextHelpFormatter) +args = parser.parse_args() + import drgn from drgn.helpers.linux.list import list_for_each_entry,list_empty from drgn.helpers.linux.percpu import per_cpu_ptr @@ -53,11 +58,6 @@ from drgn.helpers.linux.cpumask import for_each_cpu,for_each_possible_cpu from drgn.helpers.linux.nodemask import for_each_node from drgn.helpers.linux.idr import idr_for_each -import argparse -parser = argparse.ArgumentParser(description=desc, - formatter_class=argparse.RawTextHelpFormatter) -args = parser.parse_args() - def err(s): print(s, file=sys.stderr, flush=True) sys.exit(1) diff --git a/tools/workqueue/wq_monitor.py b/tools/workqueue/wq_monitor.py index 9e964c5be40c..7f47fa398e3c 100644 --- a/tools/workqueue/wq_monitor.py +++ b/tools/workqueue/wq_monitor.py @@ -37,9 +37,6 @@ import re import time import json -import drgn -from drgn.helpers.linux.list import list_for_each_entry - import argparse parser = argparse.ArgumentParser(description=desc, formatter_class=argparse.RawTextHelpFormatter) @@ -51,6 +48,9 @@ parser.add_argument('-j', '--json', action='store_true', help='Output in json') args = parser.parse_args() +import drgn +from drgn.helpers.linux.list import list_for_each_entry + workqueues = prog['workqueues'] WQ_UNBOUND = prog['WQ_UNBOUND'] From 1ad5dcee7c819031cf02eaf5e1e03728d0ffeb09 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Fri, 26 Jun 2026 02:57:53 -0700 Subject: [PATCH 02/23] workqueue: split kick_pool() into kick_pool_pick() Factor the worker selection out of kick_pool() into kick_pool_pick(), which picks and claims the worker under pool->lock but, instead of waking it, returns the worker's task via an out-param so the caller can issue the wakeup after dropping pool->lock. BH kicks and wake_cpu setup still happen under the lock. kick_pool() becomes a thin wrapper that wakes the returned task, so all existing callers keep waking under pool->lock. Pure refactor, no functional change. Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 78068ae8f28a..49770093e785 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -1258,19 +1258,27 @@ static void kick_bh_pool(struct worker_pool *pool) } /** - * kick_pool - wake up an idle worker if necessary + * kick_pool_pick - select an idle worker to kick, deferring the wakeup * @pool: pool to kick + * @wakep: out-param, set to the task to wake after pool->lock is dropped * - * @pool may have pending work items. Wake up worker if necessary. Returns - * whether a worker was woken up. + * Like kick_pool() but, for a regular (non-BH) pool, returns the picked + * worker's task via @wakep instead of waking it, so the caller can issue the + * wakeup after dropping pool->lock (the wakeup takes rq->lock). Worker + * selection, wake_cpu setup and the BH kick still happen under the lock. + * Returns whether a worker was selected or kicked. + * + * Must be called with @pool->lock held. */ -static bool kick_pool(struct worker_pool *pool) +static bool kick_pool_pick(struct worker_pool *pool, struct task_struct **wakep) { struct worker *worker = first_idle_worker(pool); struct task_struct *p; lockdep_assert_held(&pool->lock); + *wakep = NULL; + if (!need_more_worker(pool) || !worker) return false; @@ -1310,10 +1318,27 @@ static bool kick_pool(struct worker_pool *pool) } } #endif - wake_up_process(p); + *wakep = p; return true; } +/** + * kick_pool - wake up an idle worker if necessary + * @pool: pool to kick + * + * @pool may have pending work items. Wake up worker if necessary. Returns + * whether a worker was woken up. + */ +static bool kick_pool(struct worker_pool *pool) +{ + struct task_struct *p; + bool kicked = kick_pool_pick(pool, &p); + + if (p) + wake_up_process(p); + return kicked; +} + #ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT /* From d070f2916ae918c4dddadce6160576c070efcd3e Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Fri, 26 Jun 2026 02:57:54 -0700 Subject: [PATCH 03/23] workqueue: defer the worker wakeup outside pool->lock in __queue_work() __queue_work() is the enqueue hot path: it inserts the work item and calls kick_pool() while holding pool->lock. kick_pool() ends in a wakeup, which takes the target task's rq->lock, so rq->lock nests under pool->lock on every enqueue that wakes a worker on a contended unbound pool. Use kick_pool_pick() to select and claim the worker under pool->lock and issue the wakeup with wake_up_process() right after dropping the lock. Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 49770093e785..2d41000c918f 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -2302,6 +2302,7 @@ static void __queue_work(int cpu, struct workqueue_struct *wq, { struct pool_workqueue *pwq; struct worker_pool *last_pool, *pool; + struct task_struct *wake_task = NULL; unsigned int work_flags; unsigned int req_cpu = cpu; @@ -2424,7 +2425,7 @@ static void __queue_work(int cpu, struct workqueue_struct *wq, trace_workqueue_activate_work(work); insert_work(pwq, work, &pool->worklist, work_flags); - kick_pool(pool); + kick_pool_pick(pool, &wake_task); } else { work_flags |= WORK_STRUCT_INACTIVE; insert_work(pwq, work, &pwq->inactive_works, work_flags); @@ -2432,6 +2433,8 @@ static void __queue_work(int cpu, struct workqueue_struct *wq, out: raw_spin_unlock(&pool->lock); + if (wake_task) + wake_up_process(wake_task); rcu_read_unlock(); } From f504706a25eb3462cc00c76340feb6751f9e37f9 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Fri, 26 Jun 2026 02:57:55 -0700 Subject: [PATCH 04/23] workqueue: defer the worker wakeup outside pool->lock in process_one_work() Use kick_pool_pick() to select and claim the worker under pool->lock and issue the wakeup with wake_up_process() after the lock is dropped. Unlike __queue_work(), this path has no surrounding RCU section, so take rcu_read_lock() before dropping pool->lock to keep the picked worker's task_struct valid across the wakeup. Signed-off-by: Breno Leitao Tested-by: Krishna Magar Signed-off-by: Tejun Heo --- kernel/workqueue.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 2d41000c918f..c647031d5bd7 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -3251,6 +3251,7 @@ __acquires(&pool->lock) { struct pool_workqueue *pwq = get_work_pwq(work); struct worker_pool *pool = worker->pool; + struct task_struct *wake_task = NULL; unsigned long work_data; int lockdep_start_depth, rcu_start_depth; bool bh_draining = pool->flags & POOL_BH_DRAINING; @@ -3304,8 +3305,11 @@ __acquires(&pool->lock) * since nr_running would always be >= 1 at this point. This is used to * chain execution of the pending work items for WORKER_NOT_RUNNING * workers such as the UNBOUND and CPU_INTENSIVE ones. + * + * Select the worker under pool->lock; the wakeup is deferred until + * after the lock is dropped, guarded by the rcu_read_lock() below. */ - kick_pool(pool); + kick_pool_pick(pool, &wake_task); /* * Record the last pool and clear PENDING which should be the last @@ -3316,7 +3320,12 @@ __acquires(&pool->lock) set_work_pool_and_clear_pending(work, pool->id, pool_offq_flags(pool)); pwq->stats[PWQ_STAT_STARTED]++; + + rcu_read_lock(); raw_spin_unlock_irq(&pool->lock); + if (wake_task) + wake_up_process(wake_task); + rcu_read_unlock(); rcu_start_depth = rcu_preempt_depth(); lockdep_start_depth = lockdep_depth(current); From 7ddfa24d3f12ab9f3ac0e0b4e8e573ff45574d86 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Tue, 30 Jun 2026 09:15:20 -0700 Subject: [PATCH 05/23] workqueue: only show running workers in stall diagnostics show_cpu_pool_busy_workers() dumps every in-flight worker in the pool's busy_hash, including workers that are not currently running on the CPU. Restore the task_is_running() filter so only running workers are dumped. When no running worker is found the pool may be stuck, unable to wake an idle worker to process pending work, and the watchdog would otherwise give no feedback. Add show_pool_no_running_worker() to report the pool id, CPU, idle state, and worker counts in that case. The pool info message is printed inside pool->lock using printk_deferred_enter/exit, the same pattern used by the existing busy-worker loop, to avoid deadlocks with console drivers that queue work while holding locks also taken in their write paths. This has been running on the Meta fleet for a while and caught some real issues, for instance EFI stalls stalling the workqueue [1]. Link: https://lore.kernel.org/all/20260616-efi_timeout-v3-0-76dd1d26657b@debian.org/ [1] Suggested-by: Petr Mladek Fixes: 8823eaef45da7 ("workqueue: Show all busy workers in stall diagnostics") Reviewed-by: Petr Mladek Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index c647031d5bd7..56edead95aab 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -7726,13 +7726,31 @@ module_param_named(panic_on_stall_time, wq_panic_on_stall_time, uint, 0644); MODULE_PARM_DESC(panic_on_stall_time, "Panic if stall exceeds this many seconds (0=disabled)"); /* - * Show workers that might prevent the processing of pending work items. - * A busy worker that is not running on the CPU (e.g. sleeping in - * wait_event_idle() with PF_WQ_WORKER cleared) can stall the pool just as - * effectively as a CPU-bound one, so dump every in-flight worker. + * Report that a pool has no worker in running state, which is a sign that the + * pool may be stuck. Print pool info. Must be called with pool->lock held and + * inside a printk_deferred_enter/exit region. + */ +static void show_pool_no_running_worker(struct worker_pool *pool) +{ + lockdep_assert_held(&pool->lock); + + printk_deferred_enter(); + pr_info("pool %d: no worker in running state, cpu=%d is %s (nr_workers=%d nr_idle=%d)\n", + pool->id, pool->cpu, + idle_cpu(pool->cpu) ? "idle" : "busy", + pool->nr_workers, pool->nr_idle); + pr_info("The pool might have trouble waking an idle worker.\n"); + printk_deferred_exit(); +} + +/* + * Show running workers that might prevent the processing of pending work items. + * If no running worker is found, the pool may be stuck waiting for an idle + * worker to be woken, so report the pool state. */ static void show_cpu_pool_busy_workers(struct worker_pool *pool) { + bool found_running = false; struct worker *worker; unsigned long irq_flags; int bkt; @@ -7740,6 +7758,11 @@ static void show_cpu_pool_busy_workers(struct worker_pool *pool) raw_spin_lock_irqsave(&pool->lock, irq_flags); hash_for_each(pool->busy_hash, bkt, worker, hentry) { + /* Skip workers that are not actively running on the CPU. */ + if (!task_is_running(worker->task)) + continue; + + found_running = true; /* * Defer printing to avoid deadlocks in console * drivers that queue work while holding locks @@ -7753,6 +7776,13 @@ static void show_cpu_pool_busy_workers(struct worker_pool *pool) printk_deferred_exit(); } + /* + * If no running worker was found, the pool is likely stuck. Print pool + * state. + */ + if (!found_running) + show_pool_no_running_worker(pool); + raw_spin_unlock_irqrestore(&pool->lock, irq_flags); } From f7dc93388946dacae5ddf6bdf55822f066798a40 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Tue, 30 Jun 2026 09:15:21 -0700 Subject: [PATCH 06/23] workqueue: trigger a single-CPU backtrace for stalled pools When a CPU pool is stalled with no running worker, the task occupying the CPU may not be a workqueue worker at all. Trigger a single-CPU backtrace for the stalled CPU to capture what it is currently executing. The CPU is snapshotted under pool->lock and the backtrace is triggered after releasing the lock to avoid any potential issues with NMI delivery. Skip the backtrace when the CPU is offline. A pool disassociated by CPU hotplug keeps its pool->cpu, and an NMI to an offline CPU is never acked, so nmi_trigger_cpumask_backtrace() would busy-wait for its full timeout in the watchdog's timer context. Suggested-by: Petr Mladek Reviewed-by: Petr Mladek Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 56edead95aab..a3aea405d773 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -7753,10 +7753,13 @@ static void show_cpu_pool_busy_workers(struct worker_pool *pool) bool found_running = false; struct worker *worker; unsigned long irq_flags; - int bkt; + int cpu, bkt; raw_spin_lock_irqsave(&pool->lock, irq_flags); + /* Snapshot cpu inside the lock to safely use it after unlock. */ + cpu = pool->cpu; + hash_for_each(pool->busy_hash, bkt, worker, hentry) { /* Skip workers that are not actively running on the CPU. */ if (!task_is_running(worker->task)) @@ -7784,6 +7787,15 @@ static void show_cpu_pool_busy_workers(struct worker_pool *pool) show_pool_no_running_worker(pool); raw_spin_unlock_irqrestore(&pool->lock, irq_flags); + + /* + * Trigger a backtrace on the stalled CPU to capture what it is + * currently executing. Skip an offline CPU, whose NMI is never acked + * and would make the backtrace busy-wait until it times out. Done + * after releasing the lock to avoid issues with NMI delivery. + */ + if (!found_running && cpu_online(cpu)) + trigger_single_cpu_backtrace(cpu); } static void show_cpu_pools_busy_workers(void) From e73c290bd75338ab514b0c0f0e1431005a8467d7 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 1 Jul 2026 04:05:00 -0700 Subject: [PATCH 07/23] workqueue: dump the last woken worker for stalled pools To identify the task most likely responsible for a stall, add last_woken_worker (L: pool->lock) to worker_pool and record it in kick_pool() just before wake_up_process(). This captures the idle worker that was kicked to take over when the last running worker went to sleep; if the pool is now stuck with no running worker, that task is the prime suspect and its backtrace is dumped by show_pool_no_running_worker(). Using struct worker * rather than struct task_struct * avoids any lifetime concern: workers are only destroyed via set_worker_dying() which requires pool->lock, and set_worker_dying() clears last_woken_worker when the dying worker matches. show_cpu_pool_busy_workers() holds pool->lock while calling sched_show_task(), so last_woken_worker is either NULL or points to a live worker with a valid task. More precisely, set_worker_dying() clears last_woken_worker before setting WORKER_DIE, so a non-NULL last_woken_worker means the kthread has not yet exited and worker->task is still alive. Suggested-by: Petr Mladek Reviewed-by: Petr Mladek Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index a3aea405d773..86b6e43d41b5 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -226,6 +226,8 @@ struct worker_pool { /* L: hash of busy workers */ struct worker *manager; /* L: purely informational */ + /* L: last worker woken by kick_pool() */ + struct worker *last_woken_worker; struct list_head workers; /* A: attached workers */ struct ida worker_ida; /* worker IDs for task name */ @@ -1318,6 +1320,9 @@ static bool kick_pool_pick(struct worker_pool *pool, struct task_struct **wakep) } } #endif + /* Track the last idle worker woken, used for stall diagnostics. */ + pool->last_woken_worker = worker; + *wakep = p; return true; } @@ -2976,6 +2981,13 @@ static void set_worker_dying(struct worker *worker, struct list_head *list) pool->nr_workers--; pool->nr_idle--; + /* + * Clear last_woken_worker if it points to this worker, so that + * show_cpu_pool_busy_workers() cannot dereference a freed worker. + */ + if (pool->last_woken_worker == worker) + pool->last_woken_worker = NULL; + worker->flags |= WORKER_DIE; list_move(&worker->entry, list); @@ -7740,13 +7752,25 @@ static void show_pool_no_running_worker(struct worker_pool *pool) idle_cpu(pool->cpu) ? "idle" : "busy", pool->nr_workers, pool->nr_idle); pr_info("The pool might have trouble waking an idle worker.\n"); + /* + * last_woken_worker and its task are valid here: set_worker_dying() + * clears it under pool->lock before setting WORKER_DIE, so if + * last_woken_worker is non-NULL the kthread has not yet exited and + * worker->task is still alive. + */ + if (pool->last_woken_worker) { + pr_info("Backtrace of last woken worker:\n"); + sched_show_task(pool->last_woken_worker->task); + } else { + pr_info("Last woken worker empty\n"); + } printk_deferred_exit(); } /* * Show running workers that might prevent the processing of pending work items. * If no running worker is found, the pool may be stuck waiting for an idle - * worker to be woken, so report the pool state. + * worker to be woken, so report the pool state and the last woken worker. */ static void show_cpu_pool_busy_workers(struct worker_pool *pool) { @@ -7781,7 +7805,8 @@ static void show_cpu_pool_busy_workers(struct worker_pool *pool) /* * If no running worker was found, the pool is likely stuck. Print pool - * state. + * state and the backtrace of the last woken worker, which is the prime + * suspect for the stall. */ if (!found_running) show_pool_no_running_worker(pool); From ecf5aad9a4417fece80890f27a9899db90c9c457 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Thu, 2 Jul 2026 09:28:00 -0700 Subject: [PATCH 08/23] workqueue: annotate racy PWQ_STAT_CPU_TIME update in wq_worker_tick() wq_worker_tick() bumps pwq->stats[PWQ_STAT_CPU_TIME] on every scheduler tick before pool->lock is taken. For unbound workqueues the pool_workqueue is shared by all workers of the pool across CPUs, so concurrent ticks on different CPUs perform an unsynchronized 64-bit read-modify-write on the same counter. KCSAN reports this as a data-race: BUG: KCSAN: data-race in wq_worker_tick / wq_worker_tick read-write to 0xffff0004d6989500 of 8 bytes by interrupt on cpu 29: wq_worker_tick+0x70/0x418 sched_tick+0x248/0x3a0 update_process_times+0x200/0x260 tick_nohz_handler+0x230/0x2f8 __hrtimer_run_queues+0x1ec/0x6c8 hrtimer_interrupt+0x174/0x4b8 ... read-write to 0xffff0004d6989500 of 8 bytes by interrupt on cpu 24: wq_worker_tick+0x70/0x418 sched_tick+0x248/0x3a0 ... value changed: 0x000000000010a1d0 -> 0x000000000010a9a0 The counter is purely advisory, so an occasional lost update is harmless, and every other stats[] update already runs under pool->lock. Annotate the update with data_race(). Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 86b6e43d41b5..f8b5598f7272 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -1535,7 +1535,11 @@ void wq_worker_tick(struct task_struct *task) if (!pwq) return; - pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC; + /* + * @pwq is shared across CPUs for unbound wqs and this advisory stat is + * bumped outside pool->lock, so the update is intentionally racy. + */ + data_race(pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC); if (!wq_cpu_intensive_thresh_us) return; From 5eaadebf10e77b190f01ecb211c110c69275b7bd Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Fri, 3 Jul 2026 08:52:50 -0700 Subject: [PATCH 09/23] workqueue: annotate racy sum_exec_runtime reads for CPU-intensive detection The automatic CPU-intensive work item detection reads the worker task's se.sum_exec_runtime without a lock in wq_worker_running(), wq_worker_tick() and process_one_work(). The scheduler updates that field under the rq lock (from the tick via update_curr(), or cross-CPU via task_sched_runtime()), raising: BUG: KCSAN: data-race in wq_worker_running+0xa8/0xe8 race at unknown origin, with read to 0xffff0009a11d1df8 of 8 bytes by task 238535 on cpu 68: wq_worker_running schedule schedule_preempt_disabled __mutex_lock mutex_lock_nested cgroup_bpf_release process_one_work worker_thread kthread ret_from_fork value changed: 0x0000000088482ba0 -> 0x00000000884893c0 The value only feeds a heuristic, so the race is benign-ish. Unlike commit ecf5aad9a441 ("workqueue: annotate racy PWQ_STAT_CPU_TIME update in wq_worker_tick()") that only needs data_race(), these are plain reads whose result drives a subtraction and comparison, so use READ_ONCE() for a single, non-torn load, which also silences KCSAN. Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index f8b5598f7272..4ec3db31493d 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -1468,7 +1468,7 @@ void wq_worker_running(struct task_struct *task) * CPU intensive auto-detection cares about how long a work item hogged * CPU without sleeping. Reset the starting timestamp on wakeup. */ - worker->current_at = worker->task->se.sum_exec_runtime; + worker->current_at = READ_ONCE(worker->task->se.sum_exec_runtime); WRITE_ONCE(worker->sleeping, 0); } @@ -1557,7 +1557,7 @@ void wq_worker_tick(struct task_struct *task) * We probably want to make this prettier in the future. */ if ((worker->flags & WORKER_NOT_RUNNING) || READ_ONCE(worker->sleeping) || - worker->task->se.sum_exec_runtime - worker->current_at < + READ_ONCE(worker->task->se.sum_exec_runtime) - worker->current_at < wq_cpu_intensive_thresh_us * NSEC_PER_USEC) return; @@ -3294,7 +3294,7 @@ __acquires(&pool->lock) worker->current_func = work->func; worker->current_pwq = pwq; if (worker->task) - worker->current_at = worker->task->se.sum_exec_runtime; + worker->current_at = READ_ONCE(worker->task->se.sum_exec_runtime); worker->current_start = jiffies; work_data = *work_data_bits(work); worker->current_color = get_work_color(work_data); From dbff1ec23f68640a743512e8ada125795d1b72c7 Mon Sep 17 00:00:00 2001 From: Manuel Ebner Date: Thu, 9 Jul 2026 16:29:39 +0200 Subject: [PATCH 10/23] docs: workqueue: Fix bracket Add missing ')'. Signed-off-by: Manuel Ebner Acked-by: Randy Dunlap Signed-off-by: Tejun Heo --- Documentation/core-api/workqueue.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/core-api/workqueue.rst b/Documentation/core-api/workqueue.rst index 411e1b28b8de..bb770f556568 100644 --- a/Documentation/core-api/workqueue.rst +++ b/Documentation/core-api/workqueue.rst @@ -356,7 +356,7 @@ Guidelines well under the default limit. * A wq serves as a domain for forward progress guarantee - (``WQ_MEM_RECLAIM``, flush and work item attributes. Work items + (``WQ_MEM_RECLAIM``), flush and work item attributes. Work items which are not involved in memory reclaim and don't need to be flushed as a part of a group of work items, and don't require any special attribute, can use one of the system wq. There is no From 79f23600bc7b13a35fd148131245c283d14604a8 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 5 Aug 2026 07:52:28 -0700 Subject: [PATCH 11/23] workqueue: factor out get_percpu_pool() Move the static per-cpu worker_pool lookup in alloc_and_link_pwqs() into a helper, get_percpu_pool(), so the lookup can be shared by other pool-selection paths. No functional change. Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 4ec3db31493d..63a39bb3f5e4 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -5350,6 +5350,20 @@ static void link_pwq(struct pool_workqueue *pwq) list_add_tail_rcu(&pwq->pwqs_node, &wq->pwqs); } +/* Return the static per-cpu worker_pool that backs @wq on @cpu. */ +static struct worker_pool *get_percpu_pool(struct workqueue_struct *wq, int cpu) +{ + struct worker_pool __percpu *pools; + bool highpri = wq->flags & WQ_HIGHPRI; + + if (wq->flags & WQ_BH) + pools = bh_worker_pools; + else + pools = cpu_worker_pools; + + return &per_cpu_ptr(pools, cpu)[highpri]; +} + /* obtain a pool matching @attr and create a pwq associating the pool and @wq */ static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq, const struct workqueue_attrs *attrs) @@ -5664,19 +5678,9 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq) goto enomem; if (!(wq->flags & WQ_UNBOUND)) { - struct worker_pool __percpu *pools; - - if (wq->flags & WQ_BH) - pools = bh_worker_pools; - else - pools = cpu_worker_pools; - for_each_possible_cpu(cpu) { - struct pool_workqueue **pwq_p; - struct worker_pool *pool; - - pool = &(per_cpu_ptr(pools, cpu)[highpri]); - pwq_p = per_cpu_ptr(wq->cpu_pwq, cpu); + struct pool_workqueue **pwq_p = per_cpu_ptr(wq->cpu_pwq, cpu); + struct worker_pool *pool = get_percpu_pool(wq, cpu); *pwq_p = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node); From a6a80c1cc6883e44876dad85dfec79b414eec0b5 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 5 Aug 2026 07:52:29 -0700 Subject: [PATCH 12/23] workqueue: factor out alloc_and_link_percpu_pwqs() Move the per-cpu pwq allocation loop out of alloc_and_link_pwqs() into a helper. The inner allocation-failure path now returns -ENOMEM and the caller jumps to the existing enomem cleanup, equivalent to the previous goto. No functional change. Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 63a39bb3f5e4..b386a457c038 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -5666,6 +5666,28 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu) put_pwq_unlocked(old_pwq); } +static int alloc_and_link_percpu_pwqs(struct workqueue_struct *wq) +{ + int cpu; + + for_each_possible_cpu(cpu) { + struct pool_workqueue **pwq_p = per_cpu_ptr(wq->cpu_pwq, cpu); + struct worker_pool *pool = get_percpu_pool(wq, cpu); + + *pwq_p = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node); + if (!*pwq_p) + return -ENOMEM; + + init_pwq(*pwq_p, wq, pool); + + mutex_lock(&wq->mutex); + link_pwq(*pwq_p); + mutex_unlock(&wq->mutex); + } + + return 0; +} + static int alloc_and_link_pwqs(struct workqueue_struct *wq) { bool highpri = wq->flags & WQ_HIGHPRI; @@ -5678,25 +5700,8 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq) goto enomem; if (!(wq->flags & WQ_UNBOUND)) { - for_each_possible_cpu(cpu) { - struct pool_workqueue **pwq_p = per_cpu_ptr(wq->cpu_pwq, cpu); - struct worker_pool *pool = get_percpu_pool(wq, cpu); - - *pwq_p = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, - pool->node); - if (!*pwq_p) - goto enomem; - - init_pwq(*pwq_p, wq, pool); - - mutex_lock(&wq->mutex); - link_pwq(*pwq_p); - mutex_unlock(&wq->mutex); - } - return 0; - } - - if (wq->flags & __WQ_ORDERED) { + ret = alloc_and_link_percpu_pwqs(wq); + } else if (wq->flags & __WQ_ORDERED) { struct pool_workqueue *dfl_pwq; ret = apply_workqueue_attrs_locked(wq, ordered_wq_attrs[highpri]); From 3180ee71b676845603274c66f6bb5528939cf470 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 5 Aug 2026 07:52:30 -0700 Subject: [PATCH 13/23] workqueue: release pwq pools by pool type Add is_percpu_pool() and test the pool directly for per cpu. Convert the other open-coded pool->cpu checks -- in put_unbound_pool(), pool_allowed_cpus() and the workqueue watchdog -- to the same helper. No functional change. Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index b386a457c038..b96090c85bca 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -1604,6 +1604,12 @@ work_func_t wq_worker_last_func(struct task_struct *task) return worker->last_func; } +/* True if @pool is a static per-cpu pool rather than an unbound one. */ +static bool is_percpu_pool(struct worker_pool *pool) +{ + return pool->cpu >= 0; +} + /** * wq_node_nr_active - Determine wq_node_nr_active to use * @wq: workqueue of interest @@ -2753,7 +2759,7 @@ static struct worker *alloc_worker(int node) static cpumask_t *pool_allowed_cpus(struct worker_pool *pool) { - if (pool->cpu < 0 && pool->attrs->affn_strict) + if (!is_percpu_pool(pool) && pool->attrs->affn_strict) return pool->attrs->__pod_cpumask; else return pool->attrs->cpumask; @@ -5121,7 +5127,7 @@ static void put_unbound_pool(struct worker_pool *pool) return; /* sanity checks */ - if (WARN_ON(!(pool->cpu < 0)) || + if (WARN_ON(is_percpu_pool(pool)) || WARN_ON(!list_empty(&pool->worklist))) return; @@ -5273,7 +5279,7 @@ static void pwq_release_workfn(struct kthread_work *work) mutex_unlock(&wq->mutex); } - if (wq->flags & WQ_UNBOUND) { + if (!is_percpu_pool(pool)) { mutex_lock(&wq_pool_mutex); put_unbound_pool(pool); mutex_unlock(&wq_pool_mutex); @@ -7949,7 +7955,7 @@ static void wq_watchdog_timer_fn(struct timer_list *unused) lockup_detected = true; stall_time = jiffies_to_msecs(now - pool_ts) / 1000; max_stall_time = max(max_stall_time, stall_time); - if (pool->cpu >= 0 && !(pool->flags & POOL_BH)) { + if (is_percpu_pool(pool) && !(pool->flags & POOL_BH)) { pool->cpu_stall = true; cpu_pool_stall = true; } From b72fdc651056cf66714e841bd6cc59907ab1858c Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 5 Aug 2026 07:52:31 -0700 Subject: [PATCH 14/23] workqueue: account nr_active by the backing pool pwq_tryinc_nr_active() and pwq_dec_nr_active() choose between the shared per-node nr_active and the plain per-pwq one by testing wq_node_nr_active() for NULL. Test the backing pool with is_percpu_pool() instead, so the accounting follows the pool that runs the work rather than the workqueue type. No functional change. Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index b96090c85bca..7b20d459d644 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -1615,9 +1615,8 @@ static bool is_percpu_pool(struct worker_pool *pool) * @wq: workqueue of interest * @node: NUMA node, can be %NUMA_NO_NODE * - * Determine wq_node_nr_active to use for @wq on @node. Returns: - * - * - %NULL for per-cpu workqueues as they don't need to use shared nr_active. + * Determine wq_node_nr_active to use for @wq on @node. @wq must be unbound. + * Returns: * * - node_nr_active[nr_node_ids] if @node is %NUMA_NO_NODE. * @@ -1626,7 +1625,7 @@ static bool is_percpu_pool(struct worker_pool *pool) static struct wq_node_nr_active *wq_node_nr_active(struct workqueue_struct *wq, int node) { - if (!(wq->flags & WQ_UNBOUND)) + if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND))) return NULL; if (node == NUMA_NO_NODE) @@ -1782,13 +1781,16 @@ static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill) { struct workqueue_struct *wq = pwq->wq; struct worker_pool *pool = pwq->pool; - struct wq_node_nr_active *nna = wq_node_nr_active(wq, pool->node); + struct wq_node_nr_active *nna; bool obtained = false; lockdep_assert_held(&pool->lock); - if (!nna) { - /* BH or per-cpu workqueue, pwq->nr_active is sufficient */ + /* + * A concurrency-managed per-cpu pool accounts nr_active per pwq, so + * pwq->nr_active against wq->max_active is sufficient. + */ + if (is_percpu_pool(pool)) { obtained = pwq->nr_active < READ_ONCE(wq->max_active); goto out; } @@ -1796,6 +1798,8 @@ static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill) if (unlikely(pwq->plugged)) return false; + nna = wq_node_nr_active(wq, pool->node); + /* * Unbound workqueue uses per-node shared nr_active $nna. If @pwq is * already waiting on $nna, pwq_dec_nr_active() will maintain the @@ -2013,7 +2017,7 @@ static void node_activate_pending_pwq(struct wq_node_nr_active *nna, static void pwq_dec_nr_active(struct pool_workqueue *pwq) { struct worker_pool *pool = pwq->pool; - struct wq_node_nr_active *nna = wq_node_nr_active(pwq->wq, pool->node); + struct wq_node_nr_active *nna; lockdep_assert_held(&pool->lock); @@ -2024,14 +2028,16 @@ static void pwq_dec_nr_active(struct pool_workqueue *pwq) pwq->nr_active--; /* - * For a percpu workqueue, it's simple. Just need to kick the first + * A concurrency-managed per-cpu pool only needs to kick the first * inactive work item on @pwq itself. */ - if (!nna) { + if (is_percpu_pool(pool)) { pwq_activate_first_inactive(pwq, false); return; } + nna = wq_node_nr_active(pwq->wq, pool->node); + /* * If @pwq is for an unbound workqueue, it's more complicated because * multiple pwqs and pools may be sharing the nr_active count. When a From dd55381120e2032f60806e2595ee70142fe22533 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 5 Aug 2026 07:52:32 -0700 Subject: [PATCH 15/23] workqueue: test WQ_UNBOUND explicitly in the hotplug loops workqueue_online_cpu() and workqueue_offline_cpu() decide whether a workqueue needs a pod affinity update by testing wq->unbound_attrs for NULL, which is only meaningful because the attrs are allocated for unbound workqueues alone. Test the flag instead, so the attrs can later be allocated for every workqueue. No functional change. Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 7b20d459d644..b6458ee53852 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -6945,7 +6945,7 @@ int workqueue_online_cpu(unsigned int cpu) list_for_each_entry(wq, &workqueues, list) { struct workqueue_attrs *attrs = wq->unbound_attrs; - if (attrs) { + if (wq->flags & WQ_UNBOUND) { const struct wq_pod_type *pt = wqattrs_pod_type(attrs); int tcpu; @@ -6980,7 +6980,7 @@ int workqueue_offline_cpu(unsigned int cpu) list_for_each_entry(wq, &workqueues, list) { struct workqueue_attrs *attrs = wq->unbound_attrs; - if (attrs) { + if (wq->flags & WQ_UNBOUND) { const struct wq_pod_type *pt = wqattrs_pod_type(attrs); int tcpu; From 464e454e1cb4c22836fd7d1a17b3c3b11f47d989 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 5 Aug 2026 07:52:33 -0700 Subject: [PATCH 16/23] workqueue: rename wq->unbound_attrs to wq->attrs The unbound prefix says which workqueues currently have the field rather than what it holds, and the next patch allocates it for every workqueue. Rename it first so that change stays a single line. tools/workqueue/wq_dump.py reads the field by name, so rename it there too. wq_sysfs_unbound_attrs[] keeps its name: it is the set of sysfs files that only unbound workqueues expose. No functional change. Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 38 +++++++++++++++++++------------------- tools/workqueue/wq_dump.py | 6 +++--- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index b6458ee53852..fbe13c9be4c2 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -371,7 +371,7 @@ struct workqueue_struct { int saved_max_active; /* WQ: saved max_active */ int saved_min_active; /* WQ: saved min_active */ - struct workqueue_attrs *unbound_attrs; /* PW: only for unbound wqs */ + struct workqueue_attrs *attrs; /* PW: workqueue attributes */ struct pool_workqueue __rcu *dfl_pwq; /* PW: only for unbound wqs */ #ifdef CONFIG_SYSFS @@ -759,7 +759,7 @@ static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu) * unbound_effective_cpumask - effective cpumask of an unbound workqueue * @wq: workqueue of interest * - * @wq->unbound_attrs->cpumask contains the cpumask requested by the user which + * @wq->attrs->cpumask contains the cpumask requested by the user which * is masked with wq_unbound_cpumask to determine the effective cpumask. The * default pwq is always mapped to the pool with the current effective cpumask. */ @@ -5098,7 +5098,7 @@ static void rcu_free_wq(struct rcu_head *rcu) wq_free_lockdep(wq); free_percpu(wq->cpu_pwq); - free_workqueue_attrs(wq->unbound_attrs); + free_workqueue_attrs(wq->attrs); kfree(wq); } @@ -5548,7 +5548,7 @@ static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx) /* all pwqs have been created successfully, let's install'em */ mutex_lock(&ctx->wq->mutex); - copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs); + copy_workqueue_attrs(ctx->wq->attrs, ctx->attrs); /* save the previous pwqs and install the new ones */ for_each_possible_cpu(cpu) @@ -5635,7 +5635,7 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu) lockdep_assert_held(&wq_pool_mutex); - if (!(wq->flags & WQ_UNBOUND) || wq->unbound_attrs->ordered) + if (!(wq->flags & WQ_UNBOUND) || wq->attrs->ordered) return; /* @@ -5645,7 +5645,7 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu) */ target_attrs = unbound_wq_update_pwq_attrs_buf; - copy_workqueue_attrs(target_attrs, wq->unbound_attrs); + copy_workqueue_attrs(target_attrs, wq->attrs); wqattrs_actualize_cpumask(target_attrs, wq_unbound_cpumask); /* nothing to do if the target cpumask matches the current pwq */ @@ -5903,8 +5903,8 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt, return NULL; if (flags & WQ_UNBOUND) { - wq->unbound_attrs = alloc_workqueue_attrs_noprof(); - if (!wq->unbound_attrs) + wq->attrs = alloc_workqueue_attrs_noprof(); + if (!wq->attrs) goto err_free_wq; } @@ -5999,7 +5999,7 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt, free_node_nr_active(wq->node_nr_active); } err_free_wq: - free_workqueue_attrs(wq->unbound_attrs); + free_workqueue_attrs(wq->attrs); kfree(wq); return NULL; err_unlock_destroy: @@ -6943,7 +6943,7 @@ int workqueue_online_cpu(unsigned int cpu) /* update pod affinity of unbound workqueues */ list_for_each_entry(wq, &workqueues, list) { - struct workqueue_attrs *attrs = wq->unbound_attrs; + struct workqueue_attrs *attrs = wq->attrs; if (wq->flags & WQ_UNBOUND) { const struct wq_pod_type *pt = wqattrs_pod_type(attrs); @@ -6978,7 +6978,7 @@ int workqueue_offline_cpu(unsigned int cpu) cpumask_clear_cpu(cpu, wq_online_cpumask); list_for_each_entry(wq, &workqueues, list) { - struct workqueue_attrs *attrs = wq->unbound_attrs; + struct workqueue_attrs *attrs = wq->attrs; if (wq->flags & WQ_UNBOUND) { const struct wq_pod_type *pt = wqattrs_pod_type(attrs); @@ -7158,7 +7158,7 @@ static int workqueue_apply_unbound_cpumask(const cpumask_var_t unbound_cpumask) if (!(wq->flags & WQ_UNBOUND) || (wq->flags & __WQ_DESTROYING)) continue; - ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs, unbound_cpumask); + ctx = apply_wqattrs_prepare(wq, wq->attrs, unbound_cpumask); if (IS_ERR(ctx)) { ret = PTR_ERR(ctx); break; @@ -7376,7 +7376,7 @@ static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr, int written; mutex_lock(&wq->mutex); - written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice); + written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->attrs->nice); mutex_unlock(&wq->mutex); return written; @@ -7393,7 +7393,7 @@ static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq) if (!attrs) return NULL; - copy_workqueue_attrs(attrs, wq->unbound_attrs); + copy_workqueue_attrs(attrs, wq->attrs); return attrs; } @@ -7430,7 +7430,7 @@ static ssize_t wq_cpumask_show(struct device *dev, mutex_lock(&wq->mutex); written = scnprintf(buf, PAGE_SIZE, "%*pb\n", - cpumask_pr_args(wq->unbound_attrs->cpumask)); + cpumask_pr_args(wq->attrs->cpumask)); mutex_unlock(&wq->mutex); return written; } @@ -7466,13 +7466,13 @@ static ssize_t wq_affn_scope_show(struct device *dev, int written; mutex_lock(&wq->mutex); - if (wq->unbound_attrs->affn_scope == WQ_AFFN_DFL) + if (wq->attrs->affn_scope == WQ_AFFN_DFL) written = scnprintf(buf, PAGE_SIZE, "%s (%s)\n", wq_affn_names[WQ_AFFN_DFL], wq_affn_names[wq_affn_dfl]); else written = scnprintf(buf, PAGE_SIZE, "%s\n", - wq_affn_names[wq->unbound_attrs->affn_scope]); + wq_affn_names[wq->attrs->affn_scope]); mutex_unlock(&wq->mutex); return written; @@ -7507,7 +7507,7 @@ static ssize_t wq_affinity_strict_show(struct device *dev, struct workqueue_struct *wq = dev_to_wq(dev); return scnprintf(buf, PAGE_SIZE, "%d\n", - wq->unbound_attrs->affn_strict); + wq->attrs->affn_strict); } static ssize_t wq_affinity_strict_store(struct device *dev, @@ -7680,7 +7680,7 @@ int workqueue_sysfs_register(struct workqueue_struct *wq) dev_set_name(&wq_dev->dev, "%s", wq->name); /* - * unbound_attrs are created separately. Suppress uevent until + * attrs are created separately. Suppress uevent until * everything is ready. */ dev_set_uevent_suppress(&wq_dev->dev, true); diff --git a/tools/workqueue/wq_dump.py b/tools/workqueue/wq_dump.py index a0c72237531f..e0a6936a2a37 100644 --- a/tools/workqueue/wq_dump.py +++ b/tools/workqueue/wq_dump.py @@ -85,7 +85,7 @@ def wq_type_str(wq): if wq.flags & WQ_ORDERED: return f'{"ordered":{wq_type_len}}' else: - if wq.unbound_attrs.affn_strict: + if wq.attrs.affn_strict: return f'{"unbound,S":{wq_type_len}}' else: return f'{"unbound":{wq_type_len}}' @@ -205,8 +205,8 @@ for wq in list_for_each_entry('struct workqueue_struct', workqueues.address_of_( continue print(f'{wq.name.string_().decode():{WQ_NAME_LEN}}', end='') - if wq.unbound_attrs.value_() != 0: - print(f' {cpumask_str(wq.unbound_attrs.cpumask):{ucpus_len}}', end='') + if wq.attrs.value_() != 0: + print(f' {cpumask_str(wq.attrs.cpumask):{ucpus_len}}', end='') else: print(f' {"":{ucpus_len}}', end='') From f784d9ce8d0202805c0bee7dfb820d404779ad63 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 5 Aug 2026 07:52:34 -0700 Subject: [PATCH 17/23] workqueue: allocate attrs for all workqueues The attrs are where the affinity scope lives, and a per-cpu workqueue will need one once per-cpu becomes a scope rather than a separate backend. Allocate them unconditionally. wq_dump.py used a non-NULL wq->attrs as its test for an unbound workqueue, which no longer holds; test WQ_UNBOUND there instead. Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 8 +++----- tools/workqueue/wq_dump.py | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index fbe13c9be4c2..e13f223e8502 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -5902,11 +5902,9 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt, if (!wq) return NULL; - if (flags & WQ_UNBOUND) { - wq->attrs = alloc_workqueue_attrs_noprof(); - if (!wq->attrs) - goto err_free_wq; - } + wq->attrs = alloc_workqueue_attrs_noprof(); + if (!wq->attrs) + goto err_free_wq; name_len = vsnprintf(wq->name, sizeof(wq->name), fmt, args); diff --git a/tools/workqueue/wq_dump.py b/tools/workqueue/wq_dump.py index e0a6936a2a37..31afc24ef17b 100644 --- a/tools/workqueue/wq_dump.py +++ b/tools/workqueue/wq_dump.py @@ -205,7 +205,7 @@ for wq in list_for_each_entry('struct workqueue_struct', workqueues.address_of_( continue print(f'{wq.name.string_().decode():{WQ_NAME_LEN}}', end='') - if wq.attrs.value_() != 0: + if wq.flags & WQ_UNBOUND: print(f' {cpumask_str(wq.attrs.cpumask):{ucpus_len}}', end='') else: print(f' {"":{ucpus_len}}', end='') From 7cc62d8cd3c5fdef475ba22a5668dcb13453d926 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 5 Aug 2026 07:52:35 -0700 Subject: [PATCH 18/23] workqueue: rename alloc_unbound_pwq() to alloc_pwq() This allocates a pwq and binds it to the pool @attrs asks for. Which pool that is becomes a property of the attrs (once per-cpu becomes an affinity scope). Remove the 'unbound" from the function name, given it will be bigger than unbound. No functional change. Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index e13f223e8502..65ac75431c3b 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -5377,7 +5377,7 @@ static struct worker_pool *get_percpu_pool(struct workqueue_struct *wq, int cpu) } /* obtain a pool matching @attr and create a pwq associating the pool and @wq */ -static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq, +static struct pool_workqueue *alloc_pwq(struct workqueue_struct *wq, const struct workqueue_attrs *attrs) { struct worker_pool *pool; @@ -5500,7 +5500,7 @@ apply_wqattrs_prepare(struct workqueue_struct *wq, copy_workqueue_attrs(new_attrs, attrs); wqattrs_actualize_cpumask(new_attrs, unbound_cpumask); cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask); - ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs); + ctx->dfl_pwq = alloc_pwq(wq, new_attrs); if (!ctx->dfl_pwq) goto out_free; @@ -5510,7 +5510,7 @@ apply_wqattrs_prepare(struct workqueue_struct *wq, ctx->pwq_tbl[cpu] = ctx->dfl_pwq; } else { wq_calc_pod_cpumask(new_attrs, cpu); - ctx->pwq_tbl[cpu] = alloc_unbound_pwq(wq, new_attrs); + ctx->pwq_tbl[cpu] = alloc_pwq(wq, new_attrs); if (!ctx->pwq_tbl[cpu]) goto out_free; } @@ -5654,7 +5654,7 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu) return; /* create a new pwq */ - pwq = alloc_unbound_pwq(wq, target_attrs); + pwq = alloc_pwq(wq, target_attrs); if (!pwq) { pr_warn("workqueue: allocation failed while updating CPU pod affinity of \"%s\"\n", wq->name); From a5bde5d8fde8a8cb28e59a672d5ddc5b9c1e7656 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 5 Aug 2026 07:52:36 -0700 Subject: [PATCH 19/23] workqueue: skip the node_nr_active update for non-unbound workqueues apply_wqattrs_commit() updates node_nr_active->max unconditionally. wq->node_nr_active[] is only allocated for unbound workqueues, so guard the call before per-cpu workqueues start using this path. No functional change: only unbound workqueues reach apply_wqattrs_*() today. Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 65ac75431c3b..8fd6af72ffd8 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -5556,8 +5556,9 @@ static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx) ctx->pwq_tbl[cpu]); ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq); - /* update node_nr_active->max */ - wq_update_node_max_active(ctx->wq, -1); + /* update node_nr_active->max, which only unbound workqueues have */ + if (ctx->wq->flags & WQ_UNBOUND) + wq_update_node_max_active(ctx->wq, -1); mutex_unlock(&ctx->wq->mutex); } From 7aef540078adc7cdfa5ee2c9784269b49f51b539 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 12 Aug 2026 09:03:18 -0700 Subject: [PATCH 20/23] workqueue: use rcu_dereference_sched() in workqueue_congested() workqueue_congested() fetches the pwq out of wq->cpu_pwq with a plain load, so sparse complains about the dropped __rcu: kernel/workqueue.c:6304:13: sparse: incorrect type in assignment (different address spaces) @@ expected struct pool_workqueue *pwq @@ got struct pool_workqueue [noderef] __rcu * @@ A pwq is released with kfree_rcu() and the read is protected by the surrounding preempt_disable(), which is what commit fd5081f4ef33 ("workqueue: Remove redundant rcu_read_lock/unlock() in workqueue_congested()") relied on when it dropped the rcu_read_lock() here. Use the rcu_dereference_sched() helper to make that explicit. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202608120931.tvTzq1gD-lkp@intel.com/ Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 8fd6af72ffd8..e66ee1016568 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -6317,7 +6317,7 @@ bool workqueue_congested(int cpu, struct workqueue_struct *wq) if (cpu == WORK_CPU_UNBOUND) cpu = smp_processor_id(); - pwq = *per_cpu_ptr(wq->cpu_pwq, cpu); + pwq = rcu_dereference_sched(*per_cpu_ptr(wq->cpu_pwq, cpu)); ret = !list_empty(&pwq->inactive_works); preempt_enable(); From 1d125f0e6cbd34f6260affac85987201b6899ed0 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 12 Aug 2026 09:03:19 -0700 Subject: [PATCH 21/23] workqueue: use RCU accessors when populating wq->cpu_pwq wq->cpu_pwq holds RCU-protected pwq pointers, but the percpu allocation path fills it in with plain loads and stores, which sparse flags: kernel/workqueue.c:5682:57: sparse: incorrect type in initializer (different address spaces) @@ expected struct pool_workqueue **pwq_p @@ got struct pool_workqueue [noderef] __rcu ** @@ Allocate the array as __rcu pointers and publish each pwq with rcu_assign_pointer() once it is initialized and linked, the order install_unbound_pwq() uses. The warnings are not new: commit 79f23600bc7b ("workqueue: factor out get_percpu_pool()") only turned the flagged assignment into an initializer. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202608120931.tvTzq1gD-lkp@intel.com/ Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index e66ee1016568..e602ab2049a7 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -5681,21 +5681,23 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu) static int alloc_and_link_percpu_pwqs(struct workqueue_struct *wq) { + struct pool_workqueue *pwq; int cpu; for_each_possible_cpu(cpu) { - struct pool_workqueue **pwq_p = per_cpu_ptr(wq->cpu_pwq, cpu); struct worker_pool *pool = get_percpu_pool(wq, cpu); - *pwq_p = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node); - if (!*pwq_p) + pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node); + if (!pwq) return -ENOMEM; - init_pwq(*pwq_p, wq, pool); + init_pwq(pwq, wq, pool); mutex_lock(&wq->mutex); - link_pwq(*pwq_p); + link_pwq(pwq); mutex_unlock(&wq->mutex); + + rcu_assign_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu), pwq); } return 0; @@ -5708,7 +5710,7 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq) lockdep_assert_held(&wq_pool_mutex); - wq->cpu_pwq = alloc_percpu(struct pool_workqueue *); + wq->cpu_pwq = alloc_percpu(struct pool_workqueue __rcu *); if (!wq->cpu_pwq) goto enomem; @@ -5734,8 +5736,11 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq) enomem: if (wq->cpu_pwq) { for_each_possible_cpu(cpu) { - struct pool_workqueue *pwq = *per_cpu_ptr(wq->cpu_pwq, cpu); + struct pool_workqueue __rcu **slot; + struct pool_workqueue *pwq; + slot = per_cpu_ptr(wq->cpu_pwq, cpu); + pwq = rcu_access_pointer(*slot); if (pwq) { /* * Unlink pwq from wq->pwqs since link_pwq() From 4e0ee51cc2b7a542e5679edaa14aaa82be3b4abb Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 12 Aug 2026 09:03:20 -0700 Subject: [PATCH 22/23] workqueue: BUG_ON() instead of returning NULL in wq_node_nr_active() wq_node_nr_active() warns and returns NULL when @wq is not unbound, but every caller dereferences the result right away, so the WARN_ON_ONCE() only moves the oops one frame up, as raised by Tejun. Fix it by BUGing_ON() instead of this silly WARN_ON_ONCE(); Fixes: b72fdc651056 ("workqueue: account nr_active by the backing pool") Suggested-by: Tejun Heo Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index e602ab2049a7..d4ad5d93e1a7 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -1625,8 +1625,7 @@ static bool is_percpu_pool(struct worker_pool *pool) static struct wq_node_nr_active *wq_node_nr_active(struct workqueue_struct *wq, int node) { - if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND))) - return NULL; + BUG_ON(!(wq->flags & WQ_UNBOUND)); if (node == NUMA_NO_NODE) node = nr_node_ids; From 20a80e7f6b71bd664c98e95589f0cbc68804d200 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Thu, 13 Aug 2026 06:12:12 -0700 Subject: [PATCH 23/23] workqueue: annotate racy p->wake_cpu accesses in kick_pool_pick() kick_pool_pick() reads and writes p->wake_cpu while the scheduler can update it concurrently. KCSAN reports: BUG: KCSAN: data-race in kick_pool_pick+0xf8/0x2d8 race at unknown origin, with read to 0xffff000663229da4 of 4 bytes by task 1817002 on cpu 40: kick_pool_pick+0xf8/0x2d8 process_scheduled_works+0x2bc/0x888 worker_thread+0x394/0x548 kthread+0x1b8/0x1f0 ret_from_fork+0x10/0x20 value changed: 0x0000002b -> 0x0000002f The race is harmless. wake_cpu is a best-effort placement hint: every writer stores a valid CPU id and the wakeup path validates it through select_task_rq(), so a stale value only affects which CPU the worker wakes up on. Mark both accesses with READ_ONCE() and WRITE_ONCE() to document that they are intentionally racy and to stop the compiler from reloading or tearing them. Signed-off-by: Breno Leitao Reviewed-by: Bradley Morgan Signed-off-by: Tejun Heo --- kernel/workqueue.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index d4ad5d93e1a7..bfeef512f6dd 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -1309,13 +1309,14 @@ static bool kick_pool_pick(struct worker_pool *pool, struct task_struct **wakep) * its affinity scope. Repatriate. */ if (!pool->attrs->affn_strict && - !cpumask_test_cpu(p->wake_cpu, pool->attrs->__pod_cpumask)) { + !cpumask_test_cpu(READ_ONCE(p->wake_cpu), + pool->attrs->__pod_cpumask)) { struct work_struct *work = list_first_entry(&pool->worklist, struct work_struct, entry); int wake_cpu = cpumask_any_and_distribute(pool->attrs->__pod_cpumask, cpu_online_mask); if (wake_cpu < nr_cpu_ids) { - p->wake_cpu = wake_cpu; + WRITE_ONCE(p->wake_cpu, wake_cpu); get_work_pwq(work)->stats[PWQ_STAT_REPATRIATED]++; } }