From fadeedd7cfc5d73d33fa3d7ac54b9b27aabd09d2 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sun, 12 Jul 2026 07:23:55 -1000 Subject: [PATCH 1/3] sched/psi: Create the psimon kthread outside of cgroup_mutex a5b98009f16d ("sched/psi: fix race between file release and pressure write") made pressure_write() hold cgroup_mutex across psi_trigger_create(), which forks the psimon kthread for the first rtpoll trigger. As kthread creation depends on the whole fork path, the commit inadvertently created a lot of unwanted locking dependencies from cgroup_mutex. sched_ext got hit by one: its enable path blocks forks and then grabs cgroup_mutex, so a pressure write racing a scheduler enable deadlocks, with every other fork piling up behind. Fix it by splitting trigger creation so that the worker is forked with cgroup_mutex dropped and the kernfs active reference left broken. The latter matters because rmdir and cgroup.pressure writes drain active references under cgroup_mutex. Publishing the trigger last keeps error reporting synchronous and preserves the of->priv lifetime rules. The trigger registered in the first stage pins the group's rtpoll machinery across the unlocked window, leaving only creation races to resolve. The catch-up poll on installation covers scheduling attempts dropped while there was no worker. v2: Retagged sched/psi (was cgroup). Fixes: a5b98009f16d ("sched/psi: fix race between file release and pressure write") Cc: stable@vger.kernel.org Cc: Edward Adam Davis Cc: Chen Ridong Reported-by: Matt Fleming Closes: https://lore.kernel.org/all/20260710100441.2653477-1-matt@readmodwrite.com/ Signed-off-by: Tejun Heo Acked-by: Johannes Weiner Tested-by: Matt Fleming Acked-by: Suren Baghdasaryan --- include/linux/psi.h | 4 ++- kernel/cgroup/cgroup.c | 23 +++++++++++++- kernel/sched/psi.c | 69 ++++++++++++++++++++++++++++++++---------- 3 files changed, 78 insertions(+), 18 deletions(-) diff --git a/include/linux/psi.h b/include/linux/psi.h index e0745873e3f2..7966e3ac03b9 100644 --- a/include/linux/psi.h +++ b/include/linux/psi.h @@ -25,7 +25,9 @@ void psi_memstall_leave(unsigned long *flags); int psi_show(struct seq_file *s, struct psi_group *group, enum psi_res res); struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, enum psi_res res, struct file *file, - struct kernfs_open_file *of); + struct kernfs_open_file *of, + bool *need_rtpoll_worker); +int psi_trigger_create_rtpoll_worker(struct psi_group *group); void psi_trigger_destroy(struct psi_trigger *t); __poll_t psi_trigger_poll(void **trigger_ptr, struct file *file, diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 38f8d9df8fbc..b5b461d4418b 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -3996,6 +3996,7 @@ static ssize_t pressure_write(struct kernfs_open_file *of, char *buf, struct psi_trigger *new; struct cgroup *cgrp; struct psi_group *psi; + bool need_rtpoll_worker; ssize_t ret = 0; cgrp = cgroup_kn_lock_live(of->kn, false); @@ -4015,12 +4016,32 @@ static ssize_t pressure_write(struct kernfs_open_file *of, char *buf, } psi = cgroup_psi(cgrp); - new = psi_trigger_create(psi, buf, res, of->file, of); + new = psi_trigger_create(psi, buf, res, of->file, of, + &need_rtpoll_worker); if (IS_ERR(new)) { ret = PTR_ERR(new); goto out_unlock; } + /* + * The worker fork must run with neither cgroup_mutex nor the file's + * kernfs active reference held. The latter is broken since + * cgroup_kn_lock_live(). @of->priv may be released while unlocked, so + * recheck before publishing @new. + */ + if (need_rtpoll_worker) { + cgroup_unlock(); + ret = psi_trigger_create_rtpoll_worker(psi); + cgroup_lock(); + + if (!ret && !of->priv) + ret = -ENODEV; + if (ret) { + psi_trigger_destroy(new); + goto out_unlock; + } + } + smp_store_release(&ctx->psi.trigger, new); out_unlock: diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index d9c9d9480a45..565ec7b80743 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -1292,9 +1292,44 @@ int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res) return 0; } +/* + * Create @group's rtpoll worker after psi_trigger_create() reported the need + * for one. kthread creation depends on the whole fork path and we don't want + * all of that nested inside cgroup_mutex, so the caller must drop it and any + * other lock that forks can wait behind. If two callers race, the loser stops + * its never-woken kthread. + */ +int psi_trigger_create_rtpoll_worker(struct psi_group *group) +{ + struct task_struct *task; + + task = kthread_create(psi_rtpoll_worker, group, "psimon"); + if (IS_ERR(task)) + return PTR_ERR(task); + + scoped_guard(mutex, &group->rtpoll_trigger_lock) { + if (!rcu_access_pointer(group->rtpoll_task)) { + atomic_set(&group->rtpoll_wakeup, 0); + wake_up_process(task); + rcu_assign_pointer(group->rtpoll_task, task); + + /* + * Poll once to catch up on scheduling attempts dropped + * while there was no rtpoll worker. + */ + psi_schedule_rtpoll_work(group, 1, true); + return 0; + } + } + + kthread_stop(task); + return 0; +} + struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, enum psi_res res, struct file *file, - struct kernfs_open_file *of) + struct kernfs_open_file *of, + bool *need_rtpoll_worker) { struct psi_trigger *t; enum psi_states state; @@ -1302,6 +1337,8 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, bool privileged; u32 window_us; + *need_rtpoll_worker = false; + if (static_branch_likely(&psi_disabled)) return ERR_PTR(-EOPNOTSUPP); @@ -1362,26 +1399,14 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, if (privileged) { mutex_lock(&group->rtpoll_trigger_lock); - if (!rcu_access_pointer(group->rtpoll_task)) { - struct task_struct *task; - - task = kthread_create(psi_rtpoll_worker, group, "psimon"); - if (IS_ERR(task)) { - kfree(t); - mutex_unlock(&group->rtpoll_trigger_lock); - return ERR_CAST(task); - } - atomic_set(&group->rtpoll_wakeup, 0); - wake_up_process(task); - rcu_assign_pointer(group->rtpoll_task, task); - } - list_add(&t->node, &group->rtpoll_triggers); group->rtpoll_min_period = min(group->rtpoll_min_period, div_u64(t->win.size, UPDATES_PER_WINDOW)); group->rtpoll_nr_triggers[t->state]++; group->rtpoll_states |= (1 << t->state); + *need_rtpoll_worker = !rcu_access_pointer(group->rtpoll_task); + mutex_unlock(&group->rtpoll_trigger_lock); } else { mutex_lock(&group->avgs_lock); @@ -1541,6 +1566,8 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf, size_t buf_size; struct seq_file *seq; struct psi_trigger *new; + bool need_rtpoll_worker; + int ret; if (static_branch_likely(&psi_disabled)) return -EOPNOTSUPP; @@ -1565,12 +1592,22 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf, return -EBUSY; } - new = psi_trigger_create(&psi_system, buf, res, file, NULL); + new = psi_trigger_create(&psi_system, buf, res, file, NULL, + &need_rtpoll_worker); if (IS_ERR(new)) { mutex_unlock(&seq->lock); return PTR_ERR(new); } + if (need_rtpoll_worker) { + ret = psi_trigger_create_rtpoll_worker(&psi_system); + if (ret) { + psi_trigger_destroy(new); + mutex_unlock(&seq->lock); + return ret; + } + } + smp_store_release(&seq->private, new); mutex_unlock(&seq->lock); From 5457025fa8ca3c0d2732109513de839e3e797190 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sun, 12 Jul 2026 07:23:55 -1000 Subject: [PATCH 2/3] sched/psi: Shut down rtpoll_timer in psi_cgroup_free() psi_schedule_rtpoll_work() is called locklessly from the scheduler hotpath and can race psi_trigger_destroy() taking down the last rtpoll trigger under rtpoll_trigger_lock: psi_schedule_rtpoll_work() psi_trigger_destroy() rcu_read_lock(); task = rcu_dereference(rtpoll_task); rcu_assign_pointer(rtpoll_task, NULL); timer_delete(&rtpoll_timer); mod_timer(&rtpoll_timer, ...); rcu_read_unlock(); synchronize_rcu(); kthread_stop(task_to_destroy); The group can then be freed with the re-armed timer still pending, and poll_timer_fn() runs on freed memory. 461daba06bdc ("psi: eliminate kthread_worker from psi trigger scheduling mechanism") deleted the timer synchronously after the synchronize_rcu(), which prevented this but raced trigger creation instead: the deletion could cancel the timer that a new trigger set armed during the grace period and, as creation also reinitialized the timer at the time, corrupt it. 8f91efd870ea ("psi: Fix race between psi_trigger_create/destroy") moved the initialization into group_init() and the deletion into the locked section, trading the creation races for the window above. Neither placement in the destruction path works. A pending timer firing while the group is alive is harmless though. poll_timer_fn() just wakes the rtpoll waitqueue and doesn't re-arm itself. Bind the timer to the group's lifetime instead and shut it down in psi_cgroup_free(). Nothing can arm it by then. timer_shutdown_sync() because the timer is never armed again. Fixes: 8f91efd870ea ("psi: Fix race between psi_trigger_create/destroy") Cc: stable@vger.kernel.org # v5.10+ Reported-by: Sashiko AI Closes: https://lore.kernel.org/all/20260711000434.36C4A1F000E9@smtp.kernel.org/ Signed-off-by: Tejun Heo Acked-by: Johannes Weiner Tested-by: Matt Fleming Acked-by: Suren Baghdasaryan --- kernel/sched/psi.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index 565ec7b80743..e2e825dcd088 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -1134,6 +1134,12 @@ void psi_cgroup_free(struct cgroup *cgroup) return; cancel_delayed_work_sync(&cgroup->psi->avgs_work); + /* + * A psi_schedule_rtpoll_work() call racing the last trigger's + * destruction may have re-armed the timer after psi_trigger_destroy() + * deleted it. Spurious firing while the group is alive is harmless. + */ + timer_shutdown_sync(&cgroup->psi->rtpoll_timer); free_percpu(cgroup->psi->pcpu); /* All triggers must be removed by now */ WARN_ONCE(cgroup->psi->rtpoll_states, "psi: trigger leak\n"); From 2fd9b4cfcefe30cb506072f78f2cd3b6dc8a29b1 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Tue, 28 Jul 2026 17:57:07 +0800 Subject: [PATCH 3/3] Docs/admin-guide/cgroup-v2: document io.latency rotational vs non-rotational behavior MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit io.latency is documented only in terms of average latency and the avg_lat stat, which matches rotational devices. On non-rotational devices a group misses its target once enough of the IOs in the window individually exceed it, and io.stat reports missed/total rather than avg_lat/win. Describe both cases: how a miss is detected, note that the avg_lat tuning guidance is rotational-only, and update the io.stat field list (mark avg_lat/win as rotational-only, document missed/total). Acked-by: Michal Koutný Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- Documentation/admin-guide/cgroup-v2.rst | 50 ++++++++++++++++++------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 14b8c571c0d1..aed195a71cbf 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -2239,9 +2239,12 @@ IO Latency ~~~~~~~~~~ This is a cgroup v2 controller for IO workload protection. You provide a group -with a latency target, and if the average latency exceeds that target the -controller will throttle any peers that have a lower latency target than the -protected workload. +with a latency target, and if the group misses its target the controller will +throttle any peers that have a lower latency target than the protected +workload. How a miss is detected depends on the device: on rotational devices +the average latency over the window must exceed the target, while on +non-rotational devices a miss is counted once enough of the IOs in the window +individually exceed the target. The limits are only applied at the peer level in the hierarchy. This means that in the diagram below, only groups A, B, and C will influence each other, and @@ -2258,10 +2261,12 @@ So the ideal way to configure this is to set io.latency in groups A, B, and C. Generally you do not want to set a value lower than the latency your device supports. Experiment to find the value that works best for your workload. Start at higher than the expected latency for your device and, with -blkcg_debug_stats enabled, watch the avg_lat value in io.stat for your -workload group to get an idea of the latency you see during normal operation. -Use the avg_lat value as a basis for your real setting, setting at 10-15% -higher than the value in io.stat. +blkcg_debug_stats enabled, observe io.stat for your workload group to get an +idea of the latency you see during normal operation. On rotational devices, +use the avg_lat value as a basis for your real setting, setting it 10-15% +higher. On non-rotational devices io.stat reports no average latency; set +the target based on your device and use the missed/total fields to verify it +is being met. How IO Latency Throttling Works ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -2303,19 +2308,36 @@ IO Latency Interface Files the blkcg_debug_stats module parameter is enabled (it is disabled by default). + The reported latency fields depend on the device. Rotational devices + report avg_lat and win; non-rotational devices report missed and total + instead. missed and total are live counters for the current window and + may change between reads. + depth This is the current queue depth for the group. avg_lat - This is an exponential moving average with a decay rate of 1/exp - bound by the sampling interval. The decay rate interval can be - calculated by multiplying the win value in io.stat by the - corresponding number of samples based on the win value. + (Rotational devices only.) This is an exponential moving + average with a decay rate of 1/exp bound by the sampling + interval. The decay rate interval can be calculated by + multiplying the win value in io.stat by the corresponding number + of samples based on the win value. win - The sampling window size in milliseconds. This is the minimum - duration of time between evaluation events. Windows only elapse - with IO activity. Idle periods extend the most recent window. + (Rotational devices only.) The sampling window size in + milliseconds. This is the minimum duration of time between + evaluation events. Windows only elapse with IO activity. Idle + periods extend the most recent window. + + missed + (Non-rotational devices only.) The number of IOs in the + current window whose latency exceeded the target. A group is + considered to be missing its target once missed reaches a + certain ratio of total. + + total + (Non-rotational devices only.) The total number of IOs + accounted in the current window. IO Priority ~~~~~~~~~~~