mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-30 23:59:33 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'wq-for-7.3' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq

Browse Source 
Pull workqueue updates from Tejun Heo:

 - Worker wakeups moved out of pool->lock on the hot paths, shortening
   lock hold times. The wakeup can be expensive on arm64 due to the idle
   wakeup IPI, and the change improves the workqueue microbenchmark
   there by up to 10%

 - Stall diagnostics now report pools stuck with no running worker, with
   a backtrace of what the CPU is executing and the likely culprit
   worker, instead of dumping every in-flight worker

 - Preparation for turning per-cpu workqueues into an affinity scope of
   unbound workqueues instead of a separate backend

 - Race annotations for KCSAN and sparse warnings, and doc and
   monitoring script fixes

* tag 'wq-for-7.3' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq: (23 commits)
  workqueue: annotate racy p->wake_cpu accesses in kick_pool_pick()
  workqueue: BUG_ON() instead of returning NULL in wq_node_nr_active()
  workqueue: use RCU accessors when populating wq->cpu_pwq
  workqueue: use rcu_dereference_sched() in workqueue_congested()
  workqueue: skip the node_nr_active update for non-unbound workqueues
  workqueue: rename alloc_unbound_pwq() to alloc_pwq()
  workqueue: allocate attrs for all workqueues
  workqueue: rename wq->unbound_attrs to wq->attrs
  workqueue: test WQ_UNBOUND explicitly in the hotplug loops
  workqueue: account nr_active by the backing pool
  workqueue: release pwq pools by pool type
  workqueue: factor out alloc_and_link_percpu_pwqs()
  workqueue: factor out get_percpu_pool()
  docs: workqueue: Fix bracket
  workqueue: annotate racy sum_exec_runtime reads for CPU-intensive detection
  workqueue: annotate racy PWQ_STAT_CPU_TIME update in wq_worker_tick()
  workqueue: dump the last woken worker for stalled pools
  workqueue: trigger a single-CPU backtrace for stalled pools
  workqueue: only show running workers in stall diagnostics
  workqueue: defer the worker wakeup outside pool->lock in process_one_work()
  ...
This commit is contained in:
Linus Torvalds
2026-08-20 10:34:25 -07:00
parent 72fdff1416 20a80e7f6b
commit 39e34e88ec
4 changed files with 240 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Documentation/core-api/workqueue.rst
View File

@@ -356,7 +356,7 @@ Guidelines
well under the default limit.
* A wq serves as a domain for forward progress guarantee
(``WQ_MEM_RECLAIM``, flush and work item attributes. Work items
(``WQ_MEM_RECLAIM``), flush and work item attributes. Work items
which are not involved in memory reclaim and don't need to be
flushed as a part of a group of work items, and don't require any
special attribute, can use one of the system wq. There is no

323
kernel/workqueue.c
View File

@@ -226,6 +226,8 @@ struct worker_pool {
/* L: hash of busy workers */
struct worker *manager; /* L: purely informational */
/* L: last worker woken by kick_pool() */
struct worker *last_woken_worker;
struct list_head workers; /* A: attached workers */
struct ida worker_ida; /* worker IDs for task name */
@@ -369,7 +371,7 @@ struct workqueue_struct {
int saved_max_active; /* WQ: saved max_active */
int saved_min_active; /* WQ: saved min_active */
struct workqueue_attrs *unbound_attrs; /* PW: only for unbound wqs */
struct workqueue_attrs *attrs; /* PW: workqueue attributes */
struct pool_workqueue __rcu *dfl_pwq; /* PW: only for unbound wqs */
#ifdef CONFIG_SYSFS
@@ -757,7 +759,7 @@ static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu)
* unbound_effective_cpumask - effective cpumask of an unbound workqueue
* @wq: workqueue of interest
*
* @wq->unbound_attrs->cpumask contains the cpumask requested by the user which
* @wq->attrs->cpumask contains the cpumask requested by the user which
* is masked with wq_unbound_cpumask to determine the effective cpumask. The
* default pwq is always mapped to the pool with the current effective cpumask.
*/
@@ -1258,19 +1260,27 @@ static void kick_bh_pool(struct worker_pool *pool)
}
/**
* kick_pool - wake up an idle worker if necessary
* kick_pool_pick - select an idle worker to kick, deferring the wakeup
* @pool: pool to kick
* @wakep: out-param, set to the task to wake after pool->lock is dropped
*
* @pool may have pending work items. Wake up worker if necessary. Returns
* whether a worker was woken up.
* Like kick_pool() but, for a regular (non-BH) pool, returns the picked
* worker's task via @wakep instead of waking it, so the caller can issue the
* wakeup after dropping pool->lock (the wakeup takes rq->lock). Worker
* selection, wake_cpu setup and the BH kick still happen under the lock.
* Returns whether a worker was selected or kicked.
*
* Must be called with @pool->lock held.
*/
static bool kick_pool(struct worker_pool *pool)
static bool kick_pool_pick(struct worker_pool *pool, struct task_struct **wakep)
{
struct worker *worker = first_idle_worker(pool);
struct task_struct *p;
lockdep_assert_held(&pool->lock);
*wakep = NULL;
if (!need_more_worker(pool) || !worker)
return false;
@@ -1299,21 +1309,42 @@ static bool kick_pool(struct worker_pool *pool)
* its affinity scope. Repatriate.
*/
if (!pool->attrs->affn_strict &&
!cpumask_test_cpu(p->wake_cpu, pool->attrs->__pod_cpumask)) {
!cpumask_test_cpu(READ_ONCE(p->wake_cpu),
pool->attrs->__pod_cpumask)) {
struct work_struct *work = list_first_entry(&pool->worklist,
struct work_struct, entry);
int wake_cpu = cpumask_any_and_distribute(pool->attrs->__pod_cpumask,
cpu_online_mask);
if (wake_cpu < nr_cpu_ids) {
p->wake_cpu = wake_cpu;
WRITE_ONCE(p->wake_cpu, wake_cpu);
get_work_pwq(work)->stats[PWQ_STAT_REPATRIATED]++;
}
}
#endif
wake_up_process(p);
/* Track the last idle worker woken, used for stall diagnostics. */
pool->last_woken_worker = worker;
*wakep = p;
return true;
}
/**
* kick_pool - wake up an idle worker if necessary
* @pool: pool to kick
*
* @pool may have pending work items. Wake up worker if necessary. Returns
* whether a worker was woken up.
*/
static bool kick_pool(struct worker_pool *pool)
{
struct task_struct *p;
bool kicked = kick_pool_pick(pool, &p);
if (p)
wake_up_process(p);
return kicked;
}
#ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT
/*
@@ -1438,7 +1469,7 @@ void wq_worker_running(struct task_struct *task)
* CPU intensive auto-detection cares about how long a work item hogged
* CPU without sleeping. Reset the starting timestamp on wakeup.
*/
worker->current_at = worker->task->se.sum_exec_runtime;
worker->current_at = READ_ONCE(worker->task->se.sum_exec_runtime);
WRITE_ONCE(worker->sleeping, 0);
}
@@ -1505,7 +1536,11 @@ void wq_worker_tick(struct task_struct *task)
if (!pwq)
return;
pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC;
/*
* @pwq is shared across CPUs for unbound wqs and this advisory stat is
* bumped outside pool->lock, so the update is intentionally racy.
*/
data_race(pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC);
if (!wq_cpu_intensive_thresh_us)
return;
@@ -1523,7 +1558,7 @@ void wq_worker_tick(struct task_struct *task)
* We probably want to make this prettier in the future.
*/
if ((worker->flags & WORKER_NOT_RUNNING) || READ_ONCE(worker->sleeping) ||
worker->task->se.sum_exec_runtime - worker->current_at <
READ_ONCE(worker->task->se.sum_exec_runtime) - worker->current_at <
wq_cpu_intensive_thresh_us * NSEC_PER_USEC)
return;
@@ -1570,14 +1605,19 @@ work_func_t wq_worker_last_func(struct task_struct *task)
return worker->last_func;
}
/* True if @pool is a static per-cpu pool rather than an unbound one. */
static bool is_percpu_pool(struct worker_pool *pool)
{
return pool->cpu >= 0;
}
/**
* wq_node_nr_active - Determine wq_node_nr_active to use
* @wq: workqueue of interest
* @node: NUMA node, can be %NUMA_NO_NODE
*
* Determine wq_node_nr_active to use for @wq on @node. Returns:
*
* - %NULL for per-cpu workqueues as they don't need to use shared nr_active.
* Determine wq_node_nr_active to use for @wq on @node. @wq must be unbound.
* Returns:
*
* - node_nr_active[nr_node_ids] if @node is %NUMA_NO_NODE.
*
@@ -1586,8 +1626,7 @@ work_func_t wq_worker_last_func(struct task_struct *task)
static struct wq_node_nr_active *wq_node_nr_active(struct workqueue_struct *wq,
int node)
{
if (!(wq->flags & WQ_UNBOUND))
return NULL;
BUG_ON(!(wq->flags & WQ_UNBOUND));
if (node == NUMA_NO_NODE)
node = nr_node_ids;
@@ -1742,13 +1781,16 @@ static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill)
{
struct workqueue_struct *wq = pwq->wq;
struct worker_pool *pool = pwq->pool;
struct wq_node_nr_active *nna = wq_node_nr_active(wq, pool->node);
struct wq_node_nr_active *nna;
bool obtained = false;
lockdep_assert_held(&pool->lock);
if (!nna) {
/* BH or per-cpu workqueue, pwq->nr_active is sufficient */
/*
* A concurrency-managed per-cpu pool accounts nr_active per pwq, so
* pwq->nr_active against wq->max_active is sufficient.
*/
if (is_percpu_pool(pool)) {
obtained = pwq->nr_active < READ_ONCE(wq->max_active);
goto out;
}
@@ -1756,6 +1798,8 @@ static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill)
if (unlikely(pwq->plugged))
return false;
nna = wq_node_nr_active(wq, pool->node);
/*
* Unbound workqueue uses per-node shared nr_active $nna. If @pwq is
* already waiting on $nna, pwq_dec_nr_active() will maintain the
@@ -1973,7 +2017,7 @@ static void node_activate_pending_pwq(struct wq_node_nr_active *nna,
static void pwq_dec_nr_active(struct pool_workqueue *pwq)
{
struct worker_pool *pool = pwq->pool;
struct wq_node_nr_active *nna = wq_node_nr_active(pwq->wq, pool->node);
struct wq_node_nr_active *nna;
lockdep_assert_held(&pool->lock);
@@ -1984,14 +2028,16 @@ static void pwq_dec_nr_active(struct pool_workqueue *pwq)
pwq->nr_active--;
/*
* For a percpu workqueue, it's simple. Just need to kick the first
* A concurrency-managed per-cpu pool only needs to kick the first
* inactive work item on @pwq itself.
*/
if (!nna) {
if (is_percpu_pool(pool)) {
pwq_activate_first_inactive(pwq, false);
return;
}
nna = wq_node_nr_active(pwq->wq, pool->node);
/*
* If @pwq is for an unbound workqueue, it's more complicated because
* multiple pwqs and pools may be sharing the nr_active count. When a
@@ -2277,6 +2323,7 @@ static void __queue_work(int cpu, struct workqueue_struct *wq,
{
struct pool_workqueue *pwq;
struct worker_pool *last_pool, *pool;
struct task_struct *wake_task = NULL;
unsigned int work_flags;
unsigned int req_cpu = cpu;
@@ -2399,7 +2446,7 @@ static void __queue_work(int cpu, struct workqueue_struct *wq,
trace_workqueue_activate_work(work);
insert_work(pwq, work, &pool->worklist, work_flags);
kick_pool(pool);
kick_pool_pick(pool, &wake_task);
} else {
work_flags |= WORK_STRUCT_INACTIVE;
insert_work(pwq, work, &pwq->inactive_works, work_flags);
@@ -2407,6 +2454,8 @@ static void __queue_work(int cpu, struct workqueue_struct *wq,
out:
raw_spin_unlock(&pool->lock);
if (wake_task)
wake_up_process(wake_task);
rcu_read_unlock();
}
@@ -2716,7 +2765,7 @@ static struct worker *alloc_worker(int node)
static cpumask_t *pool_allowed_cpus(struct worker_pool *pool)
{
if (pool->cpu < 0 && pool->attrs->affn_strict)
if (!is_percpu_pool(pool) && pool->attrs->affn_strict)
return pool->attrs->__pod_cpumask;
else
return pool->attrs->cpumask;
@@ -2948,6 +2997,13 @@ static void set_worker_dying(struct worker *worker, struct list_head *list)
pool->nr_workers--;
pool->nr_idle--;
/*
* Clear last_woken_worker if it points to this worker, so that
* show_cpu_pool_busy_workers() cannot dereference a freed worker.
*/
if (pool->last_woken_worker == worker)
pool->last_woken_worker = NULL;
worker->flags |= WORKER_DIE;
list_move(&worker->entry, list);
@@ -3223,6 +3279,7 @@ __acquires(&pool->lock)
{
struct pool_workqueue *pwq = get_work_pwq(work);
struct worker_pool *pool = worker->pool;
struct task_struct *wake_task = NULL;
unsigned long work_data;
int lockdep_start_depth, rcu_start_depth;
bool bh_draining = pool->flags & POOL_BH_DRAINING;
@@ -3249,7 +3306,7 @@ __acquires(&pool->lock)
worker->current_func = work->func;
worker->current_pwq = pwq;
if (worker->task)
worker->current_at = worker->task->se.sum_exec_runtime;
worker->current_at = READ_ONCE(worker->task->se.sum_exec_runtime);
worker->current_start = jiffies;
work_data = *work_data_bits(work);
worker->current_color = get_work_color(work_data);
@@ -3276,8 +3333,11 @@ __acquires(&pool->lock)
* since nr_running would always be >= 1 at this point. This is used to
* chain execution of the pending work items for WORKER_NOT_RUNNING
* workers such as the UNBOUND and CPU_INTENSIVE ones.
*
* Select the worker under pool->lock; the wakeup is deferred until
* after the lock is dropped, guarded by the rcu_read_lock() below.
*/
kick_pool(pool);
kick_pool_pick(pool, &wake_task);
/*
* Record the last pool and clear PENDING which should be the last
@@ -3288,7 +3348,12 @@ __acquires(&pool->lock)
set_work_pool_and_clear_pending(work, pool->id, pool_offq_flags(pool));
pwq->stats[PWQ_STAT_STARTED]++;
rcu_read_lock();
raw_spin_unlock_irq(&pool->lock);
if (wake_task)
wake_up_process(wake_task);
rcu_read_unlock();
rcu_start_depth = rcu_preempt_depth();
lockdep_start_depth = lockdep_depth(current);
@@ -5033,7 +5098,7 @@ static void rcu_free_wq(struct rcu_head *rcu)
wq_free_lockdep(wq);
free_percpu(wq->cpu_pwq);
free_workqueue_attrs(wq->unbound_attrs);
free_workqueue_attrs(wq->attrs);
kfree(wq);
}
@@ -5068,7 +5133,7 @@ static void put_unbound_pool(struct worker_pool *pool)
return;
/* sanity checks */
if (WARN_ON(!(pool->cpu < 0)) ||
if (WARN_ON(is_percpu_pool(pool)) ||
WARN_ON(!list_empty(&pool->worklist)))
return;
@@ -5220,7 +5285,7 @@ static void pwq_release_workfn(struct kthread_work *work)
mutex_unlock(&wq->mutex);
}
if (wq->flags & WQ_UNBOUND) {
if (!is_percpu_pool(pool)) {
mutex_lock(&wq_pool_mutex);
put_unbound_pool(pool);
mutex_unlock(&wq_pool_mutex);
@@ -5297,8 +5362,22 @@ static void link_pwq(struct pool_workqueue *pwq)
list_add_tail_rcu(&pwq->pwqs_node, &wq->pwqs);
}
/* Return the static per-cpu worker_pool that backs @wq on @cpu. */
static struct worker_pool *get_percpu_pool(struct workqueue_struct *wq, int cpu)
{
struct worker_pool __percpu *pools;
bool highpri = wq->flags & WQ_HIGHPRI;
if (wq->flags & WQ_BH)
pools = bh_worker_pools;
else
pools = cpu_worker_pools;
return &per_cpu_ptr(pools, cpu)[highpri];
}
/* obtain a pool matching @attr and create a pwq associating the pool and @wq */
static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
static struct pool_workqueue *alloc_pwq(struct workqueue_struct *wq,
const struct workqueue_attrs *attrs)
{
struct worker_pool *pool;
@@ -5421,7 +5500,7 @@ apply_wqattrs_prepare(struct workqueue_struct *wq,
copy_workqueue_attrs(new_attrs, attrs);
wqattrs_actualize_cpumask(new_attrs, unbound_cpumask);
cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask);
ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
ctx->dfl_pwq = alloc_pwq(wq, new_attrs);
if (!ctx->dfl_pwq)
goto out_free;
@@ -5431,7 +5510,7 @@ apply_wqattrs_prepare(struct workqueue_struct *wq,
ctx->pwq_tbl[cpu] = ctx->dfl_pwq;
} else {
wq_calc_pod_cpumask(new_attrs, cpu);
ctx->pwq_tbl[cpu] = alloc_unbound_pwq(wq, new_attrs);
ctx->pwq_tbl[cpu] = alloc_pwq(wq, new_attrs);
if (!ctx->pwq_tbl[cpu])
goto out_free;
}
@@ -5469,7 +5548,7 @@ static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx)
/* all pwqs have been created successfully, let's install'em */
mutex_lock(&ctx->wq->mutex);
copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);
copy_workqueue_attrs(ctx->wq->attrs, ctx->attrs);
/* save the previous pwqs and install the new ones */
for_each_possible_cpu(cpu)
@@ -5477,8 +5556,9 @@ static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx)
ctx->pwq_tbl[cpu]);
ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq);
/* update node_nr_active->max */
wq_update_node_max_active(ctx->wq, -1);
/* update node_nr_active->max, which only unbound workqueues have */
if (ctx->wq->flags & WQ_UNBOUND)
wq_update_node_max_active(ctx->wq, -1);
mutex_unlock(&ctx->wq->mutex);
}
@@ -5556,7 +5636,7 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu)
lockdep_assert_held(&wq_pool_mutex);
if (!(wq->flags & WQ_UNBOUND) || wq->unbound_attrs->ordered)
if (!(wq->flags & WQ_UNBOUND) || wq->attrs->ordered)
return;
/*
@@ -5566,7 +5646,7 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu)
*/
target_attrs = unbound_wq_update_pwq_attrs_buf;
copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
copy_workqueue_attrs(target_attrs, wq->attrs);
wqattrs_actualize_cpumask(target_attrs, wq_unbound_cpumask);
/* nothing to do if the target cpumask matches the current pwq */
@@ -5575,7 +5655,7 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu)
return;
/* create a new pwq */
pwq = alloc_unbound_pwq(wq, target_attrs);
pwq = alloc_pwq(wq, target_attrs);
if (!pwq) {
pr_warn("workqueue: allocation failed while updating CPU pod affinity of \"%s\"\n",
wq->name);
@@ -5599,6 +5679,30 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu)
put_pwq_unlocked(old_pwq);
}
static int alloc_and_link_percpu_pwqs(struct workqueue_struct *wq)
{
struct pool_workqueue *pwq;
int cpu;
for_each_possible_cpu(cpu) {
struct worker_pool *pool = get_percpu_pool(wq, cpu);
pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
if (!pwq)
return -ENOMEM;
init_pwq(pwq, wq, pool);
mutex_lock(&wq->mutex);
link_pwq(pwq);
mutex_unlock(&wq->mutex);
rcu_assign_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu), pwq);
}
return 0;
}
static int alloc_and_link_pwqs(struct workqueue_struct *wq)
{
bool highpri = wq->flags & WQ_HIGHPRI;
@@ -5606,40 +5710,13 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq)
lockdep_assert_held(&wq_pool_mutex);
wq->cpu_pwq = alloc_percpu(struct pool_workqueue *);
wq->cpu_pwq = alloc_percpu(struct pool_workqueue __rcu *);
if (!wq->cpu_pwq)
goto enomem;
if (!(wq->flags & WQ_UNBOUND)) {
struct worker_pool __percpu *pools;
if (wq->flags & WQ_BH)
pools = bh_worker_pools;
else
pools = cpu_worker_pools;
for_each_possible_cpu(cpu) {
struct pool_workqueue **pwq_p;
struct worker_pool *pool;
pool = &(per_cpu_ptr(pools, cpu)[highpri]);
pwq_p = per_cpu_ptr(wq->cpu_pwq, cpu);
*pwq_p = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL,
pool->node);
if (!*pwq_p)
goto enomem;
init_pwq(*pwq_p, wq, pool);
mutex_lock(&wq->mutex);
link_pwq(*pwq_p);
mutex_unlock(&wq->mutex);
}
return 0;
}
if (wq->flags & __WQ_ORDERED) {
ret = alloc_and_link_percpu_pwqs(wq);
} else if (wq->flags & __WQ_ORDERED) {
struct pool_workqueue *dfl_pwq;
ret = apply_workqueue_attrs_locked(wq, ordered_wq_attrs[highpri]);
@@ -5659,8 +5736,11 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq)
enomem:
if (wq->cpu_pwq) {
for_each_possible_cpu(cpu) {
struct pool_workqueue *pwq = *per_cpu_ptr(wq->cpu_pwq, cpu);
struct pool_workqueue __rcu **slot;
struct pool_workqueue *pwq;
slot = per_cpu_ptr(wq->cpu_pwq, cpu);
pwq = rcu_access_pointer(*slot);
if (pwq) {
/*
* Unlink pwq from wq->pwqs since link_pwq()
@@ -5828,11 +5908,9 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt,
if (!wq)
return NULL;
if (flags & WQ_UNBOUND) {
wq->unbound_attrs = alloc_workqueue_attrs_noprof();
if (!wq->unbound_attrs)
goto err_free_wq;
}
wq->attrs = alloc_workqueue_attrs_noprof();
if (!wq->attrs)
goto err_free_wq;
name_len = vsnprintf(wq->name, sizeof(wq->name), fmt, args);
@@ -5925,7 +6003,7 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt,
free_node_nr_active(wq->node_nr_active);
}
err_free_wq:
free_workqueue_attrs(wq->unbound_attrs);
free_workqueue_attrs(wq->attrs);
kfree(wq);
return NULL;
err_unlock_destroy:
@@ -6244,7 +6322,7 @@ bool workqueue_congested(int cpu, struct workqueue_struct *wq)
if (cpu == WORK_CPU_UNBOUND)
cpu = smp_processor_id();
pwq = *per_cpu_ptr(wq->cpu_pwq, cpu);
pwq = rcu_dereference_sched(*per_cpu_ptr(wq->cpu_pwq, cpu));
ret = !list_empty(&pwq->inactive_works);
preempt_enable();
@@ -6869,9 +6947,9 @@ int workqueue_online_cpu(unsigned int cpu)
/* update pod affinity of unbound workqueues */
list_for_each_entry(wq, &workqueues, list) {
struct workqueue_attrs *attrs = wq->unbound_attrs;
struct workqueue_attrs *attrs = wq->attrs;
if (attrs) {
if (wq->flags & WQ_UNBOUND) {
const struct wq_pod_type *pt = wqattrs_pod_type(attrs);
int tcpu;
@@ -6904,9 +6982,9 @@ int workqueue_offline_cpu(unsigned int cpu)
cpumask_clear_cpu(cpu, wq_online_cpumask);
list_for_each_entry(wq, &workqueues, list) {
struct workqueue_attrs *attrs = wq->unbound_attrs;
struct workqueue_attrs *attrs = wq->attrs;
if (attrs) {
if (wq->flags & WQ_UNBOUND) {
const struct wq_pod_type *pt = wqattrs_pod_type(attrs);
int tcpu;
@@ -7084,7 +7162,7 @@ static int workqueue_apply_unbound_cpumask(const cpumask_var_t unbound_cpumask)
if (!(wq->flags & WQ_UNBOUND) || (wq->flags & __WQ_DESTROYING))
continue;
ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs, unbound_cpumask);
ctx = apply_wqattrs_prepare(wq, wq->attrs, unbound_cpumask);
if (IS_ERR(ctx)) {
ret = PTR_ERR(ctx);
break;
@@ -7302,7 +7380,7 @@ static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr,
int written;
mutex_lock(&wq->mutex);
written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->attrs->nice);
mutex_unlock(&wq->mutex);
return written;
@@ -7319,7 +7397,7 @@ static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq)
if (!attrs)
return NULL;
copy_workqueue_attrs(attrs, wq->unbound_attrs);
copy_workqueue_attrs(attrs, wq->attrs);
return attrs;
}
@@ -7356,7 +7434,7 @@ static ssize_t wq_cpumask_show(struct device *dev,
mutex_lock(&wq->mutex);
written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
cpumask_pr_args(wq->unbound_attrs->cpumask));
cpumask_pr_args(wq->attrs->cpumask));
mutex_unlock(&wq->mutex);
return written;
}
@@ -7392,13 +7470,13 @@ static ssize_t wq_affn_scope_show(struct device *dev,
int written;
mutex_lock(&wq->mutex);
if (wq->unbound_attrs->affn_scope == WQ_AFFN_DFL)
if (wq->attrs->affn_scope == WQ_AFFN_DFL)
written = scnprintf(buf, PAGE_SIZE, "%s (%s)\n",
wq_affn_names[WQ_AFFN_DFL],
wq_affn_names[wq_affn_dfl]);
else
written = scnprintf(buf, PAGE_SIZE, "%s\n",
wq_affn_names[wq->unbound_attrs->affn_scope]);
wq_affn_names[wq->attrs->affn_scope]);
mutex_unlock(&wq->mutex);
return written;
@@ -7433,7 +7511,7 @@ static ssize_t wq_affinity_strict_show(struct device *dev,
struct workqueue_struct *wq = dev_to_wq(dev);
return scnprintf(buf, PAGE_SIZE, "%d\n",
wq->unbound_attrs->affn_strict);
wq->attrs->affn_strict);
}
static ssize_t wq_affinity_strict_store(struct device *dev,
@@ -7606,7 +7684,7 @@ int workqueue_sysfs_register(struct workqueue_struct *wq)
dev_set_name(&wq_dev->dev, "%s", wq->name);
/*
* unbound_attrs are created separately. Suppress uevent until
* attrs are created separately. Suppress uevent until
* everything is ready.
*/
dev_set_uevent_suppress(&wq_dev->dev, true);
@@ -7689,20 +7767,58 @@ module_param_named(panic_on_stall_time, wq_panic_on_stall_time, uint, 0644);
MODULE_PARM_DESC(panic_on_stall_time, "Panic if stall exceeds this many seconds (0=disabled)");
/*
* Show workers that might prevent the processing of pending work items.
* A busy worker that is not running on the CPU (e.g. sleeping in
* wait_event_idle() with PF_WQ_WORKER cleared) can stall the pool just as
* effectively as a CPU-bound one, so dump every in-flight worker.
* Report that a pool has no worker in running state, which is a sign that the
* pool may be stuck. Print pool info. Must be called with pool->lock held and
* inside a printk_deferred_enter/exit region.
*/
static void show_pool_no_running_worker(struct worker_pool *pool)
{
lockdep_assert_held(&pool->lock);
printk_deferred_enter();
pr_info("pool %d: no worker in running state, cpu=%d is %s (nr_workers=%d nr_idle=%d)\n",
pool->id, pool->cpu,
idle_cpu(pool->cpu) ? "idle" : "busy",
pool->nr_workers, pool->nr_idle);
pr_info("The pool might have trouble waking an idle worker.\n");
/*
* last_woken_worker and its task are valid here: set_worker_dying()
* clears it under pool->lock before setting WORKER_DIE, so if
* last_woken_worker is non-NULL the kthread has not yet exited and
* worker->task is still alive.
*/
if (pool->last_woken_worker) {
pr_info("Backtrace of last woken worker:\n");
sched_show_task(pool->last_woken_worker->task);
} else {
pr_info("Last woken worker empty\n");
}
printk_deferred_exit();
}
/*
* Show running workers that might prevent the processing of pending work items.
* If no running worker is found, the pool may be stuck waiting for an idle
* worker to be woken, so report the pool state and the last woken worker.
*/
static void show_cpu_pool_busy_workers(struct worker_pool *pool)
{
bool found_running = false;
struct worker *worker;
unsigned long irq_flags;
int bkt;
int cpu, bkt;
raw_spin_lock_irqsave(&pool->lock, irq_flags);
/* Snapshot cpu inside the lock to safely use it after unlock. */
cpu = pool->cpu;
hash_for_each(pool->busy_hash, bkt, worker, hentry) {
/* Skip workers that are not actively running on the CPU. */
if (!task_is_running(worker->task))
continue;
found_running = true;
/*
* Defer printing to avoid deadlocks in console
* drivers that queue work while holding locks
@@ -7716,7 +7832,24 @@ static void show_cpu_pool_busy_workers(struct worker_pool *pool)
printk_deferred_exit();
}
/*
* If no running worker was found, the pool is likely stuck. Print pool
* state and the backtrace of the last woken worker, which is the prime
* suspect for the stall.
*/
if (!found_running)
show_pool_no_running_worker(pool);
raw_spin_unlock_irqrestore(&pool->lock, irq_flags);
/*
* Trigger a backtrace on the stalled CPU to capture what it is
* currently executing. Skip an offline CPU, whose NMI is never acked
* and would make the backtrace busy-wait until it times out. Done
* after releasing the lock to avoid issues with NMI delivery.
*/
if (!found_running && cpu_online(cpu))
trigger_single_cpu_backtrace(cpu);
}
static void show_cpu_pools_busy_workers(void)
@@ -7832,7 +7965,7 @@ static void wq_watchdog_timer_fn(struct timer_list *unused)
lockup_detected = true;
stall_time = jiffies_to_msecs(now - pool_ts) / 1000;
max_stall_time = max(max_stall_time, stall_time);
if (pool->cpu >= 0 && !(pool->flags & POOL_BH)) {
if (is_percpu_pool(pool) && !(pool->flags & POOL_BH)) {
pool->cpu_stall = true;
cpu_pool_stall = true;
}

16
tools/workqueue/wq_dump.py
View File

@@ -46,6 +46,11 @@ each workqueue:
import sys
import argparse
parser = argparse.ArgumentParser(description=desc,
formatter_class=argparse.RawTextHelpFormatter)
args = parser.parse_args()
import drgn
from drgn.helpers.linux.list import list_for_each_entry,list_empty
from drgn.helpers.linux.percpu import per_cpu_ptr
@@ -53,11 +58,6 @@ from drgn.helpers.linux.cpumask import for_each_cpu,for_each_possible_cpu
from drgn.helpers.linux.nodemask import for_each_node
from drgn.helpers.linux.idr import idr_for_each
import argparse
parser = argparse.ArgumentParser(description=desc,
formatter_class=argparse.RawTextHelpFormatter)
args = parser.parse_args()
def err(s):
print(s, file=sys.stderr, flush=True)
sys.exit(1)
@@ -85,7 +85,7 @@ def wq_type_str(wq):
if wq.flags & WQ_ORDERED:
return f'{"ordered":{wq_type_len}}'
else:
if wq.unbound_attrs.affn_strict:
if wq.attrs.affn_strict:
return f'{"unbound,S":{wq_type_len}}'
else:
return f'{"unbound":{wq_type_len}}'
@@ -205,8 +205,8 @@ for wq in list_for_each_entry('struct workqueue_struct', workqueues.address_of_(
continue
print(f'{wq.name.string_().decode():{WQ_NAME_LEN}}', end='')
if wq.unbound_attrs.value_() != 0:
print(f' {cpumask_str(wq.unbound_attrs.cpumask):{ucpus_len}}', end='')
if wq.flags & WQ_UNBOUND:
print(f' {cpumask_str(wq.attrs.cpumask):{ucpus_len}}', end='')
else:
print(f' {"":{ucpus_len}}', end='')

6
tools/workqueue/wq_monitor.py
View File

@@ -37,9 +37,6 @@ import re
import time
import json
import drgn
from drgn.helpers.linux.list import list_for_each_entry
import argparse
parser = argparse.ArgumentParser(description=desc,
formatter_class=argparse.RawTextHelpFormatter)
@@ -51,6 +48,9 @@ parser.add_argument('-j', '--json', action='store_true',
help='Output in json')
args = parser.parse_args()
import drgn
from drgn.helpers.linux.list import list_for_each_entry
workqueues = prog['workqueues']
WQ_UNBOUND = prog['WQ_UNBOUND']