From a2ba161d17836d6c81aca32a35e75c913207142b Mon Sep 17 00:00:00 2001 From: Yousef Alhouseen Date: Wed, 24 Jun 2026 14:36:52 +0200 Subject: [PATCH 01/34] tools/cgroup: iocost_monitor: parse help before importing drgn iocost_monitor.py imports drgn before argparse can handle "-h" or report argument errors. That makes basic usage help fail on systems where drgn is not installed. Parse arguments before importing drgn so the help and argument-error paths work without the runtime debugging dependency. Normal execution still imports drgn before reading kernel state. Signed-off-by: Yousef Alhouseen Signed-off-by: Tejun Heo --- tools/cgroup/iocost_monitor.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/cgroup/iocost_monitor.py b/tools/cgroup/iocost_monitor.py index 933c750b319b..bdd78ba271b0 100644 --- a/tools/cgroup/iocost_monitor.py +++ b/tools/cgroup/iocost_monitor.py @@ -15,11 +15,6 @@ import time import json import math -import drgn -from drgn import container_of -from drgn.helpers.linux.list import list_for_each_entry,list_empty -from drgn.helpers.linux.radixtree import radix_tree_for_each,radix_tree_lookup - import argparse parser = argparse.ArgumentParser(description=desc, formatter_class=argparse.RawTextHelpFormatter) @@ -34,6 +29,11 @@ parser.add_argument('--json', action='store_true', help='Output in json') args = parser.parse_args() +import drgn +from drgn import container_of +from drgn.helpers.linux.list import list_for_each_entry,list_empty +from drgn.helpers.linux.radixtree import radix_tree_for_each,radix_tree_lookup + def err(s): print(s, file=sys.stderr, flush=True) sys.exit(1) From 866f587e9c70566a0391bf402123555605a82f81 Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Tue, 23 Jun 2026 19:04:12 -0400 Subject: [PATCH 02/34] cgroup/cpuset: Avoid unnecessary cpus & mems update in cpuset_hotplug_update_tasks() As reported by sashiko [1], cpuset_hotplug_update_tasks() may perform unnecessary task iteration and updating of tasks' CPU and node masks when mems_allowed and/or cpus_allowed are not set in cpuset v2. It is due to the fact that the temporary new_cpus and new_mems masks do not inherit parent's effective_cpus/mems when they are empty which is the expected behavior for cpuset v2 since commit 4ec22e9c5a90 ("cpuset: Enable cpuset controller in default hierarchy"). Fix that and avoid unnecessary work by enhancing compute_effective_cpumask() to add the empty cpumask check and inheriting the parent's versions if empty when in v2. A new compute_effective_nodemask() helper is also added to perform a similar function for new effective_mems. Add new test_cpuset_prs.sh test cases to confirm that effective_cpus will inherit the parent's version if cpuset.cpus is empty. [1] https://sashiko.dev/#/patchset/20260621032816.1806773-1-longman%40redhat.com Suggested-by: Ridong Chen Fixes: 4ec22e9c5a90 ("cpuset: Enable cpuset controller in default hierarchy") Signed-off-by: Waiman Long Reviewed-by: Ridong Chen Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 45 +++++++++++-------- .../selftests/cgroup/test_cpuset_prs.sh | 11 ++++- 2 files changed, 35 insertions(+), 21 deletions(-) diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 591e3aa487fc..a404894411dc 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -1089,12 +1089,35 @@ void cpuset_update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus) * @cs: the cpuset the need to recompute the new effective_cpus mask * @parent: the parent cpuset * + * For v2, the parent's effective_cpus is inherited if cpumask is empty. * The result is valid only if the given cpuset isn't a partition root. */ static void compute_effective_cpumask(struct cpumask *new_cpus, struct cpuset *cs, struct cpuset *parent) { - cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus); + bool has_cpus; + + has_cpus = cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus); + if (!has_cpus && is_in_v2_mode()) + cpumask_copy(new_cpus, parent->effective_cpus); +} + +/** + * compute_effective_nodemask - Compute the effective nodemask of the cpuset + * @new_mems: the temp variable for the new effective_mems mask + * @cs: the cpuset the need to recompute the new effective_mems mask + * @parent: the parent cpuset + * + * For v2, the parent's effective_mems is inherited if nodemask is empty. + */ +static void compute_effective_nodemask(nodemask_t *new_mems, + struct cpuset *cs, struct cpuset *parent) +{ + bool has_mems; + + has_mems = nodes_and(*new_mems, cs->mems_allowed, parent->effective_mems); + if (!has_mems && is_in_v2_mode()) + nodes_copy(*new_mems, parent->effective_mems); } /* @@ -2143,15 +2166,6 @@ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp, goto update_parent_effective; } - /* - * If it becomes empty, inherit the effective mask of the - * parent, which is guaranteed to have some CPUs unless - * it is a partition root that has explicitly distributed - * out all its CPUs. - */ - if (is_in_v2_mode() && !remote && cpumask_empty(tmp->new_cpus)) - cpumask_copy(tmp->new_cpus, parent->effective_cpus); - /* * Skip the whole subtree if * 1) the cpumask remains the same, @@ -2692,14 +2706,7 @@ static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems) cpuset_for_each_descendant_pre(cp, pos_css, cs) { struct cpuset *parent = parent_cs(cp); - bool has_mems = nodes_and(*new_mems, cp->mems_allowed, parent->effective_mems); - - /* - * If it becomes empty, inherit the effective mask of the - * parent, which is guaranteed to have some MEMs. - */ - if (is_in_v2_mode() && !has_mems) - *new_mems = parent->effective_mems; + compute_effective_nodemask(new_mems, cp, parent); /* Skip the whole subtree if the nodemask remains the same. */ if (nodes_equal(*new_mems, cp->effective_mems)) { @@ -3773,7 +3780,7 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp) parent = parent_cs(cs); compute_effective_cpumask(&new_cpus, cs, parent); - nodes_and(new_mems, cs->mems_allowed, parent->effective_mems); + compute_effective_nodemask(&new_mems, cs, parent); if (!tmp || !cs->partition_root_state) goto update_tasks; diff --git a/tools/testing/selftests/cgroup/test_cpuset_prs.sh b/tools/testing/selftests/cgroup/test_cpuset_prs.sh index 0d41aa0d343d..ca9bc38fdb95 100755 --- a/tools/testing/selftests/cgroup/test_cpuset_prs.sh +++ b/tools/testing/selftests/cgroup/test_cpuset_prs.sh @@ -495,13 +495,20 @@ REMOTE_TEST_MATRIX=( # Narrowing cpuset.cpus to previously sibling-excluded CPUs should # not return CPUs that were never actually owned. " C1-4:P1 . C1-2:P1 C1-3:P2 . . \ - . . . C3 . . p1:4|c11:1-2|c12:3 \ + . . . C3 . . p1:4|c11:1-2|c12:3 \ p1:P1|c11:P1|c12:P2 3" # Expanding cpuset.cpus to include a previously sibling-excluded CPU # after the sibling has become a member should correctly request it. " C1-4:P1 . C1-2:P1 C1-3:P2 . . \ - . . P0 C2-3 . . p1:1,4|c11:1|c12:2-3 \ + . . P0 C2-3 . . p1:1,4|c11:1|c12:2-3 \ p1:P1|c11:P0|c12:P2 2-3" + # Cpusets with empty cpuset.cpus should inherit parent's effective_cpus + " C1-4:P1 C5-6 C1-2 . C5 . \ + . P1 P1 . . . p1:3-4|p2:5-6|c11:1-2|c12:3-4|c21:5|c22:5-6 \ + p1:P1|p2:P1|c11:P1" + " C1-4:P1 C5-6 C1-2 . C5 . \ + . P1 P1 . O5=0 . p1:3-4|p2:6|c11:1-2|c12:3-4|c21:6|c22:6 \ + p1:P1|p2:P1|c11:P1" ) # From eda17a3a70845c78e160e9f9c39e6069ad749cb4 Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Tue, 23 Jun 2026 19:04:13 -0400 Subject: [PATCH 03/34] cgroup/cpuset: Rebind/migrate mm only for threadgroup leader in cpuset_update_tasks_nodemask() As reported by sashiko [1], cpuset_update_tasks_nodemask() will do mpol_rebind_mm() and possibly cpuset_migrate_mm() for all threads of a multithreaded process. Since commit 3df9ca0a2b8b ("cpuset: migrate memory only for threadgroup leaders"), cpuset_attach() had been updated to rebind and migrate memory only for threadgroup leaders to mark the group leader as the owner of the mm_struct. To be consistent and avoid unnecessary performance overhead for heavily multithreaded processes, follow the cpuset_attach() example and perform memory rebind and migration only for threadgroup leaders. Also add a paragraph in cgroup-v2.rst under cpuset.mems that the threadgroup leader is the memory owner of that threadgroup. Therefore the non-leading threads shouldn't be in other cgroups whose "cpuset.mems" doesn't fully overlap that of the group leader. [1] https://sashiko.dev/#/patchset/20260621032816.1806773-1-longman%40redhat.com Signed-off-by: Waiman Long Reviewed-by: Ridong Chen Signed-off-by: Tejun Heo --- Documentation/admin-guide/cgroup-v2.rst | 7 +++++++ kernel/cgroup/cpuset.c | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 993446ab66d0..1bf219829465 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -2527,6 +2527,13 @@ Cpuset Interface Files a need to change "cpuset.mems" with active tasks, it shouldn't be done frequently. + For a multithreaded process, the threadgroup leader is + considered the owner of the group's memory. Memory policy + rebinding and migration will only happen with respect to the + threadgroup leader. To avoid unexpected results, non-leading + threads shouldn't be put into another cgroup whose "cpuset.mems" + doesn't fully overlap that of the threadgroup leader. + cpuset.mems.effective A read-only multiple values file which exists on all cpuset-enabled cgroups. diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index a404894411dc..e53f35e2726f 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -2661,6 +2661,10 @@ void cpuset_update_tasks_nodemask(struct cpuset *cs) cpuset_change_task_nodemask(task, &newmems); + /* Rebind and migrate mm only for thread group leader */ + if (!thread_group_leader(task)) + continue; + mm = get_task_mm(task); if (!mm) continue; From 8a564dfdfd88f1c5262ad1a4957310fe907650fc Mon Sep 17 00:00:00 2001 From: "Zenghui Yu (Huawei)" Date: Mon, 22 Jun 2026 19:07:08 +0800 Subject: [PATCH 04/34] cgroup: Fix a typo of the function name in comment ... which was wrongly written as cgroup_threadcgroup_change_begin(). Signed-off-by: Zenghui Yu (Huawei) Signed-off-by: Tejun Heo --- include/linux/cgroup-defs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index de2cd6238c2a..7a631a257613 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -896,7 +896,7 @@ static inline void cgroup_threadgroup_change_begin(struct task_struct *tsk) * cgroup_threadgroup_change_end - threadgroup exclusion for cgroups * @tsk: target task * - * Counterpart of cgroup_threadcgroup_change_begin(). + * Counterpart of cgroup_threadgroup_change_begin(). */ static inline void cgroup_threadgroup_change_end(struct task_struct *tsk) { From da43ea213936494732e52212c59f027967b97173 Mon Sep 17 00:00:00 2001 From: Guopeng Zhang Date: Thu, 25 Jun 2026 09:39:44 +0800 Subject: [PATCH 05/34] cgroup: Use data_race() for task->flags in task_css_set_check() task_css_set_check() uses rcu_dereference_check() to verify that task->cgroups can be dereferenced. One accepted condition is that the task is already exiting, tested by checking PF_EXITING in task->flags. This check is only part of the CONFIG_PROVE_RCU lockdep predicate. This was found by KCSAN during fuzz testing. KCSAN can report a data race when another task flag bit is updated concurrently. One report shows pids_release() reading task->flags through task_css_set_check() while do_task_dead() sets PF_NOFREEZE: KCSAN: data-race in task_css() [inline] KCSAN: data-race in pids_release() task_css() pids_release() cgroup_release() release_task() wait_task_zombie() value changed: 0x0040004c -> 0x0040804c The changed bit is PF_NOFREEZE, not PF_EXITING. PF_EXITING remains set before and after the update, so the task_css_set_check() condition does not change. This is not a race on task->cgroups and does not indicate incorrect pids charging or uncharging. tools/memory-model/Documentation/access-marking.txt recommends data_race() for data-racy loads used only for diagnostic purposes. Use data_race() here to mark the intended diagnostic-only access. No functional change intended. Suggested-by: Tejun Heo Signed-off-by: Guopeng Zhang Signed-off-by: Tejun Heo --- include/linux/cgroup.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index f2aa46a4f871..b905208942bf 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -480,7 +480,7 @@ static inline void cgroup_unlock(void) rcu_read_lock_sched_held() || \ lockdep_is_held(&cgroup_mutex) || \ lockdep_is_held(&css_set_lock) || \ - ((task)->flags & PF_EXITING) || (__c)) + (data_race((task)->flags) & PF_EXITING) || (__c)) #else #define task_css_set_check(task, __c) \ rcu_dereference((task)->cgroups) From a2703c2980c0776e9be7d8f9e144afd054dddb12 Mon Sep 17 00:00:00 2001 From: Joe Simmons-Talbott Date: Fri, 26 Jun 2026 16:29:22 -0400 Subject: [PATCH 06/34] selftests/cgroup: Adjust cpu test duration based on HZ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For lower HZ values a quota of 1000us is much lower than the amount of microseconds per tick which makes the tests test_cpucg_max and test_cpugc_max_nested fail. Increase the test duration to accommodate for lower HZ values. Link: https://lore.kernel.org/lkml/20260625203307.1114538-1-joest@redhat.com/ Signed-off-by: Joe Simmons-Talbott Acked-by: Michal Koutný Signed-off-by: Tejun Heo --- .../cgroup/lib/include/cgroup_util.h | 1 + tools/testing/selftests/cgroup/test_cpu.c | 43 ++++++++++++++++--- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/cgroup/lib/include/cgroup_util.h b/tools/testing/selftests/cgroup/lib/include/cgroup_util.h index febc1723d090..8ebb2b4d4ec0 100644 --- a/tools/testing/selftests/cgroup/lib/include/cgroup_util.h +++ b/tools/testing/selftests/cgroup/lib/include/cgroup_util.h @@ -8,6 +8,7 @@ #define MB(x) (x << 20) +#define NSEC_PER_USEC 1000L #define USEC_PER_SEC 1000000L #define NSEC_PER_SEC 1000000000L diff --git a/tools/testing/selftests/cgroup/test_cpu.c b/tools/testing/selftests/cgroup/test_cpu.c index 7a40d76b9548..a5eccfcabef5 100644 --- a/tools/testing/selftests/cgroup/test_cpu.c +++ b/tools/testing/selftests/cgroup/test_cpu.c @@ -639,6 +639,31 @@ test_cpucg_nested_weight_underprovisioned(const char *root) return run_cpucg_nested_weight_test(root, false); } +/* + * Best effort attempt to get the kernel's HZ value from the config. + * Return the HZ value if found otherwise return 1000 (the default) to + * indicate failure. + */ +static long +get_config_hz(void) +{ + long hz = 1000; + FILE *f; + char cmd[256] = "zcat /proc/config.gz 2>/dev/null | grep '^CONFIG_HZ='"; + + f = popen(cmd, "r"); + + if (!f) + return hz; + + if (fscanf(f, "CONFIG_HZ=%ld", &hz) == EOF) + goto out; + +out: + pclose(f); + return hz; +} + /* * This test creates a cgroup with some maximum value within a period, and * verifies that a process in the cgroup is not overscheduled. @@ -646,15 +671,18 @@ test_cpucg_nested_weight_underprovisioned(const char *root) static int test_cpucg_max(const char *root) { int ret = KSFT_FAIL; + long hz = get_config_hz(); long quota_usec = 1000; long default_period_usec = 100000; /* cpu.max's default period */ long duration_seconds = 1; - long duration_usec = duration_seconds * USEC_PER_SEC; + long duration_usec; long usage_usec, n_periods, remainder_usec, expected_usage_usec; char *cpucg; char quota_buf[32]; + duration_usec = duration_seconds * USEC_PER_SEC * 1000 / hz; + snprintf(quota_buf, sizeof(quota_buf), "%ld", quota_usec); cpucg = cg_name(root, "cpucg_test"); @@ -670,8 +698,8 @@ static int test_cpucg_max(const char *root) struct cpu_hog_func_param param = { .nprocs = 1, .ts = { - .tv_sec = duration_seconds, - .tv_nsec = 0, + .tv_sec = duration_usec / USEC_PER_SEC, + .tv_nsec = duration_usec % USEC_PER_SEC * NSEC_PER_USEC, }, .clock_type = CPU_HOG_CLOCK_WALL, }; @@ -710,15 +738,18 @@ static int test_cpucg_max(const char *root) static int test_cpucg_max_nested(const char *root) { int ret = KSFT_FAIL; + long hz = get_config_hz(); long quota_usec = 1000; long default_period_usec = 100000; /* cpu.max's default period */ long duration_seconds = 1; - long duration_usec = duration_seconds * USEC_PER_SEC; + long duration_usec; long usage_usec, n_periods, remainder_usec, expected_usage_usec; char *parent, *child; char quota_buf[32]; + duration_usec = duration_seconds * USEC_PER_SEC * 1000 / hz; + snprintf(quota_buf, sizeof(quota_buf), "%ld", quota_usec); parent = cg_name(root, "cpucg_parent"); @@ -741,8 +772,8 @@ static int test_cpucg_max_nested(const char *root) struct cpu_hog_func_param param = { .nprocs = 1, .ts = { - .tv_sec = duration_seconds, - .tv_nsec = 0, + .tv_sec = duration_usec / USEC_PER_SEC, + .tv_nsec = duration_usec % USEC_PER_SEC * NSEC_PER_USEC, }, .clock_type = CPU_HOG_CLOCK_WALL, }; From 171569f8ee6724a4113a0100fea6ff83d9b70c6a Mon Sep 17 00:00:00 2001 From: Sun Shaojie Date: Mon, 29 Jun 2026 14:06:36 +0800 Subject: [PATCH 07/34] cgroup/cpu: document cpu.stat.local and clarify cpu.stat behavior MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add documentation for the cpu.stat.local interface file, which reports the throttled_usec stat -- the actual throttling time incurred by the cgroup's own runqueues, which may include throttling inherited from ancestor cgroup bandwidth limits. Unlike cpu.stat's throttled_usec which only accounts for throttling caused by the cgroup's own CFS bandwidth limit. When the controller is not enabled, the stat is not reported. Also clarify cpu.stat descriptions: note that the three base CPU usage stats (usage_usec, user_usec, system_usec) include descendant cgroups, and that the five CFS bandwidth stats are non-hierarchical -- they only account for throttling caused by the cgroup's own bandwidth limit. Signed-off-by: Sun Shaojie Acked-by: Michal Koutný Signed-off-by: Tejun Heo --- Documentation/admin-guide/cgroup-v2.rst | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 1bf219829465..851a1cf1bc22 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -1145,7 +1145,7 @@ will be referred to. All time durations are in microseconds. This file exists whether the controller is enabled or not. It always reports the following three stats, which account for all the - processes in the cgroup: + processes in the cgroup (including those in descendant cgroups): - usage_usec - user_usec @@ -1160,6 +1160,27 @@ will be referred to. All time durations are in microseconds. - nr_bursts - burst_usec + Note that the above five CFS bandwidth stats are non-hierarchical; + they only account for throttling caused by this cgroup's own bandwidth + limit, not including throttling inherited from ancestor cgroups. + + cpu.stat.local + A read-only flat-keyed file. + This file exists whether the controller is enabled or not. + + It reports the following stat when the controller is enabled: + + - throttled_usec + + Unlike the ``throttled_usec`` reported by ``cpu.stat`` which + accounts for throttling caused by this cgroup's own CFS + bandwidth limit, ``cpu.stat.local`` reports the actual + throttling time incurred by this cgroup's own runqueues, + which may include throttling inherited from ancestor + cgroup bandwidth limits. + + When the controller is not enabled, this stat is not reported. + cpu.weight A read-write single value file which exists on non-root cgroups. The default is "100". From e9d189aa4b2382e81b922c1a619edf78c8013386 Mon Sep 17 00:00:00 2001 From: Manuel Ebner Date: Fri, 3 Jul 2026 08:38:05 +0200 Subject: [PATCH 08/34] docs: cgroup: Fix bracket Remove single ')'. Signed-off-by: Manuel Ebner Signed-off-by: Tejun Heo --- Documentation/admin-guide/cgroup-v1/memcg_test.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/admin-guide/cgroup-v1/memcg_test.rst b/Documentation/admin-guide/cgroup-v1/memcg_test.rst index ebedbc3c3f9c..d9951c319ef5 100644 --- a/Documentation/admin-guide/cgroup-v1/memcg_test.rst +++ b/Documentation/admin-guide/cgroup-v1/memcg_test.rst @@ -10,7 +10,7 @@ Because VM is getting complex (one of reasons is memcg...), memcg's behavior is complex. This is a document for memcg's internal behavior. Please note that implementation details can be changed. -(*) Topics on API should be in Documentation/admin-guide/cgroup-v1/memory.rst) +(*) Topics on API should be in Documentation/admin-guide/cgroup-v1/memory.rst 0. How to record usage ? ======================== From 95220e1f18f6321008f021abc7d6f581f64bcb82 Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Thu, 2 Jul 2026 17:47:47 -0400 Subject: [PATCH 09/34] cgroup/cpuset: Make nr_deadline_tasks an atomic_t The nr_deadline_tasks variable in the cpuset structure was introduced by commit 6c24849f5515 ("sched/cpuset: Keep track of SCHED_DEADLINE task in cpusets"). It is reported by sashiko [1] that nr_deadline_tasks can currently be modified by inc_dl_tasks_cs() under rq->lock and by cpuset_attach() under cpuset_mutex. So if both updates happen simultaneously, the nr_deadline_tasks variable can be corrupted leading to incorrect operations down the road. Fix that by changing its type to atomic_t so that nr_deadline_tasks are always atomically updated. This fix patch is a low hanging fruit. It can handle some of the races between a concurrent sched_setscheduler() and cpuset_can_attach()/cpuset_attach() calls, but not all of them like the other issue raised by sashiko [2]. This will be handled hopefully in a future follow up patch. [1] https://sashiko.dev/#/patchset/20260626181923.133658-1-longman%40redhat.com [2] https://sashiko.dev/#/patchset/20260630033344.352702-1-longman%40redhat.com Fixes: 6c24849f5515 ("sched/cpuset: Keep track of SCHED_DEADLINE task in cpusets") Reviewed-by: Ridong Chen Signed-off-by: Waiman Long Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset-internal.h | 2 +- kernel/cgroup/cpuset.c | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/kernel/cgroup/cpuset-internal.h b/kernel/cgroup/cpuset-internal.h index f7aaf01f7cd5..140700e5e236 100644 --- a/kernel/cgroup/cpuset-internal.h +++ b/kernel/cgroup/cpuset-internal.h @@ -165,7 +165,7 @@ struct cpuset { * number of SCHED_DEADLINE tasks attached to this cpuset, so that we * know when to rebuild associated root domain bandwidth information. */ - int nr_deadline_tasks; + atomic_t nr_deadline_tasks; int nr_migrate_dl_tasks; /* DL bandwidth that needs destination reservation for this attach. */ u64 sum_migrate_dl_bw; diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index e53f35e2726f..7c54289edf40 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -222,14 +222,14 @@ void inc_dl_tasks_cs(struct task_struct *p) { struct cpuset *cs = task_cs(p); - cs->nr_deadline_tasks++; + atomic_inc(&cs->nr_deadline_tasks); } void dec_dl_tasks_cs(struct task_struct *p) { struct cpuset *cs = task_cs(p); - cs->nr_deadline_tasks--; + atomic_dec(&cs->nr_deadline_tasks); } static inline bool is_partition_valid(const struct cpuset *cs) @@ -918,7 +918,7 @@ static void dl_update_tasks_root_domain(struct cpuset *cs) struct css_task_iter it; struct task_struct *task; - if (cs->nr_deadline_tasks == 0) + if (atomic_read(&cs->nr_deadline_tasks) == 0) return; css_task_iter_start(&cs->css, 0, &it); @@ -3215,8 +3215,8 @@ static void cpuset_attach(struct cgroup_taskset *tset) cs->old_mems_allowed = cpuset_attach_nodemask_to; if (cs->nr_migrate_dl_tasks) { - cs->nr_deadline_tasks += cs->nr_migrate_dl_tasks; - oldcs->nr_deadline_tasks -= cs->nr_migrate_dl_tasks; + atomic_add(cs->nr_migrate_dl_tasks, &cs->nr_deadline_tasks); + atomic_sub(cs->nr_migrate_dl_tasks, &oldcs->nr_deadline_tasks); reset_migrate_dl_data(cs); } From 4d733685148c65eb6aa52aa9b248bc501a604e39 Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Thu, 2 Jul 2026 17:47:48 -0400 Subject: [PATCH 10/34] cgroup/cpuset: Fix node inconsistencies between cpuset_update_tasks_nodemask() and cpuset_attach() Whenever memory node mask is changed, there are 4 places where the node mask has to be updated or used. 1) task's node mask via cpuset_change_task_nodemask() 2) memory policy binding via mpol_rebind_mm() 3) if memory migration is enabled, migrate from old_mems_allowed to the new node mask via cpuset_migrate_mm(). 4) setting old_mems_allowed These memory actions are done in cpuset_update_tasks_nodemask() and cpuset_attach(). However there are inconsistencies in what node masks are being used in these 2 functions. In cpuset_update_tasks_nodemask(), - cpuset_change_task_nodemask(): guarantee_online_mems() - mpol_rebind_mm(): mems_allowed - cpuset_migrate_mm(): guarantee_online_mems() - old_mems_allowed: guarantee_online_mems() In cpuset_attach(), - cpuset_change_task_nodemask(): guarantee_online_mems() - mpol_rebind_mm(): effective_mems - cpuset_migrate_mm(): effective_mems - old_mems_allowed: effective_mems These inconsistencies dates back to quite a long time ago and it is hard to say what should be the correct values. The guarantee_online_mems() function returns a node mask from current or an ancestor cpuset that is a subset of node_states[N_MEMORY]. Nodes in node_states[N_MEMORY] are all online, i.e. in node_states[N_ONLINE]. However, node in node_states[N_ONLINE] may not have memory. So node_states[N_MEMORY] should be a subset of node_states[N_ONLINE]. The guarantee_online_mems() function should mostly be useful for v1 where mems_allowed is the same as effective_mems. With v2, the memory nodes in effective_mems should be a subset of node_states[N_MEMORY] except when a memory hot-unplug operation is in progress and a memory node is removed from node_states[N_MEMORY] but not yet reflected in the effective_mems's as cpuset_handle_hotplug() has not been called from cpuset_track_online_nodes(). Let use the following setup for both of them and make them consistent. - cpuset_change_task_nodemask(): guarantee_online_mems() - mpol_rebind_mm(): effective_mems - cpuset_migrate_mm(): guarantee_online_mems() - old_mems_allowed: guarantee_online_mems() So for v2, it is effectively all effective_mems most of the time. For v1, mpol_rebind_mm() uses mems_allowed which may differ from what guarantee_online_mems() returns, but it conforms to what the cpuset v1 documentation says with respect to setting memory policy. Signed-off-by: Waiman Long Reviewed-by: Ridong Chen Reviewed-by: Gregory Price Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 7c54289edf40..c2a7a3ccc80d 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -489,7 +489,10 @@ static void guarantee_active_cpus(struct task_struct *tsk, * Return in *pmask the portion of a cpusets's mems_allowed that * are online, with memory. If none are online with memory, walk * up the cpuset hierarchy until we find one that does have some - * online mems. The top cpuset always has some mems online. + * online mems. The top cpuset always has some mems online. With v2, + * effective_mems should always contain online memory nodes except + * during the transition period where a memory node hotunplug operation + * is in progress. * * One way or another, we guarantee to return some non-empty subset * of node_states[N_MEMORY]. @@ -2633,6 +2636,14 @@ static void *cpuset_being_rebound; * Iterate through each task of @cs updating its mems_allowed to the * effective cpuset's. As this function is called with cpuset_mutex held, * cpuset membership stays stable. + * + * - cpuset_change_task_nodemask(): guarantee_online_mems() + * - mpol_rebind_mm(): effective_mems + * - cpuset_migrate_mm(): guarantee_online_mems() + * - old_mems_allowed: guarantee_online_mems() + * + * For v2, guarantee_online_mems() should return a node mask that is the same + * as the effective_mems of current cpuset. */ void cpuset_update_tasks_nodemask(struct cpuset *cs) { @@ -2641,7 +2652,6 @@ void cpuset_update_tasks_nodemask(struct cpuset *cs) struct task_struct *task; cpuset_being_rebound = cs; /* causes mpol_dup() rebind */ - guarantee_online_mems(cs, &newmems); /* @@ -3159,19 +3169,16 @@ static void cpuset_attach(struct cgroup_taskset *tset) cpus_updated = !cpumask_equal(cs->effective_cpus, oldcs->effective_cpus); mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems); + guarantee_online_mems(cs, &cpuset_attach_nodemask_to); /* * In the default hierarchy, enabling cpuset in the child cgroups - * will trigger a number of cpuset_attach() calls with no change - * in effective cpus and mems. In that case, we can optimize out - * by skipping the task iteration and update. + * will trigger a cpuset_attach() call with no change in effective cpus + * and mems. In that case, we can optimize out by skipping the task + * iteration and update. */ - if (cpuset_v2() && !cpus_updated && !mems_updated) { - cpuset_attach_nodemask_to = cs->effective_mems; + if (cpuset_v2() && !cpus_updated && !mems_updated) goto out; - } - - guarantee_online_mems(cs, &cpuset_attach_nodemask_to); cgroup_taskset_for_each(task, css, tset) cpuset_attach_task(cs, task); @@ -3182,7 +3189,6 @@ static void cpuset_attach(struct cgroup_taskset *tset) * if there is no change in effective_mems and CS_MEMORY_MIGRATE is * not set. */ - cpuset_attach_nodemask_to = cs->effective_mems; if (!is_memory_migrate(cs) && !mems_updated) goto out; @@ -3190,7 +3196,7 @@ static void cpuset_attach(struct cgroup_taskset *tset) struct mm_struct *mm = get_task_mm(leader); if (mm) { - mpol_rebind_mm(mm, &cpuset_attach_nodemask_to); + mpol_rebind_mm(mm, &cs->effective_mems); /* * old_mems_allowed is the same with mems_allowed From 75f7a25ec6bb365225e60b5aa6bae397866b0cee Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Thu, 2 Jul 2026 17:47:49 -0400 Subject: [PATCH 11/34] cgroup/cpuset: Prevent race between task attach and cpuset state change Commit e44193d39e8d ("cpuset: let hotplug propagation work wait for task attaching") was introduced to let hotplug operation to wait until the completion of task attach operation. However, it is still possible that the states of the source or destination cpuset can be changed between the cpuset_can_attach() call and the subsequent cpuset_attach()/cpuset_cancel_attach() call. As a result, data gathered during cpuset_can_attach() cannot be reliably used in the subsequent cpuset_attach()/cpuset_cancel_attach() call at all. Make the task attach operation more robust and allow the sharing of data between cpuset_can_attach() and cpuset_attach()/cpuset_cancel_attach() by making cpuset_write_resmask() and cpuset_partition_write() wait for the completion of task attach as well. Ideally, an ongoing task attach operation should block any cpuset write operation that can change its internal state until the operation is completed. However, the attach_in_progress flag is currently per cpuset and only the destination cpuset will have this flag set. The flag is not set in the source cpuset where the tasks will be moved from. Even if we extend the scope to include the source cpuset, it will not block cpuset operation that changes the state of one of its ancestor cpuset which may indirectly impact the state of the source or destination cpuset. It may be too costly to set the flag for the whole subtree, it is far easier to just make the flag global and block all the cpuset write operation whenever a task attach operation is in progress. Make that change by creating a new cpuset attach context (attach_ctx) structure to hold the global in_progress flag and use it for blocking cpuset write operation if a cpuset attach operation is in progress. Also add a new wait_attach_done_lock() helper to do the waiting for an ongoing attach operation and acquire the cpuset_mutex. The comments about validate_change() are no longer valid as it won't be called at all if an attach operation is in progress. So the comments can be removed. The per-cpuset attach_in_progress flag is also currently used in partition_is_populated() and cpuset_is_populated() to determine if an empty cpuset will have incoming task. This check will no longer be needed as this function will not be called when there is a task attach in progress. So the flag check is now removed. Reviewed-by: Ridong Chen Signed-off-by: Waiman Long Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset-internal.h | 11 +--- kernel/cgroup/cpuset.c | 90 +++++++++++++++++++-------------- 2 files changed, 53 insertions(+), 48 deletions(-) diff --git a/kernel/cgroup/cpuset-internal.h b/kernel/cgroup/cpuset-internal.h index 140700e5e236..df662c7fd1a4 100644 --- a/kernel/cgroup/cpuset-internal.h +++ b/kernel/cgroup/cpuset-internal.h @@ -145,12 +145,6 @@ struct cpuset { */ nodemask_t old_mems_allowed; - /* - * Tasks are being attached to this cpuset. Used to prevent - * zeroing cpus/mems_allowed between ->can_attach() and ->attach(). - */ - int attach_in_progress; - /* partition root state */ int partition_root_state; @@ -269,10 +263,7 @@ static inline int nr_cpusets(void) static inline bool cpuset_is_populated(struct cpuset *cs) { lockdep_assert_cpuset_lock_held(); - - /* Cpusets in the process of attaching should be considered as populated */ - return cgroup_is_populated(cs->css.cgroup) || - cs->attach_in_progress; + return cgroup_is_populated(cs->css.cgroup); } /** diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index c2a7a3ccc80d..9b3f9524e139 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -356,6 +356,33 @@ static struct workqueue_struct *cpuset_migrate_mm_wq; static DECLARE_WAIT_QUEUE_HEAD(cpuset_attach_wq); +/* + * Cpuset task attach context + * Protected by cpuset_mutex + */ +static struct { + int in_progress; +} attach_ctx; + +/* + * Wait if task attach is in progress until it is done and then acquire + * cpuset_mutex before returning. + */ +static void wait_attach_done_lock(void) + __acquires(&cpuset_mutex) +{ + for (;;) { + mutex_lock(&cpuset_mutex); + if (!attach_ctx.in_progress) + return; + + mutex_unlock(&cpuset_mutex); + + /* Wait until attach operation is done to prevent racing */ + wait_event(cpuset_attach_wq, attach_ctx.in_progress == 0); + } +} + static inline void check_insane_mems_config(nodemask_t *nodes) { if (!cpusets_insane_config() && @@ -368,22 +395,22 @@ static inline void check_insane_mems_config(nodemask_t *nodes) } /* - * decrease cs->attach_in_progress. - * wake_up cpuset_attach_wq if cs->attach_in_progress==0. + * decrease attach_ctx.in_progress. + * wake_up cpuset_attach_wq if attach_ctx.in_progress==0. */ -static inline void dec_attach_in_progress_locked(struct cpuset *cs) +static inline void dec_attach_in_progress_locked(void) { lockdep_assert_cpuset_lock_held(); - cs->attach_in_progress--; - if (!cs->attach_in_progress) + attach_ctx.in_progress--; + if (!attach_ctx.in_progress) wake_up(&cpuset_attach_wq); } -static inline void dec_attach_in_progress(struct cpuset *cs) +static inline void dec_attach_in_progress(void) { mutex_lock(&cpuset_mutex); - dec_attach_in_progress_locked(cs); + dec_attach_in_progress_locked(); mutex_unlock(&cpuset_mutex); } @@ -432,8 +459,7 @@ static inline bool partition_is_populated(struct cpuset *cs, * nr_populated_domain_children may include populated * csets from descendants that are partitions. */ - if (cgroup_has_tasks(cs->css.cgroup) || - cs->attach_in_progress) + if (cgroup_has_tasks(cs->css.cgroup)) return true; rcu_read_lock(); @@ -3091,11 +3117,7 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) cs->dl_bw_cpu = cpu; out_success: - /* - * Mark attach is in progress. This makes validate_change() fail - * changes which zero cpus/mems_allowed. - */ - cs->attach_in_progress++; + attach_ctx.in_progress++; out_unlock: if (ret) @@ -3113,7 +3135,7 @@ static void cpuset_cancel_attach(struct cgroup_taskset *tset) cs = css_cs(css); mutex_lock(&cpuset_mutex); - dec_attach_in_progress_locked(cs); + dec_attach_in_progress_locked(); if (cs->dl_bw_cpu >= 0) dl_bw_free(cs->dl_bw_cpu, cs->sum_migrate_dl_bw); @@ -3226,7 +3248,7 @@ static void cpuset_attach(struct cgroup_taskset *tset) reset_migrate_dl_data(cs); } - dec_attach_in_progress_locked(cs); + dec_attach_in_progress_locked(); mutex_unlock(&cpuset_mutex); } @@ -3246,7 +3268,12 @@ ssize_t cpuset_write_resmask(struct kernfs_open_file *of, return -EACCES; buf = strstrip(buf); - cpuset_full_lock(); + + /* cpuset_mutex acquired in wait_attach_done_lock() */ + mutex_lock(&cpuset_top_mutex); + cpus_read_lock(); + wait_attach_done_lock(); + if (!is_cpuset_online(cs)) goto out_unlock; @@ -3377,7 +3404,10 @@ static ssize_t cpuset_partition_write(struct kernfs_open_file *of, char *buf, else return -EINVAL; - cpuset_full_lock(); + mutex_lock(&cpuset_top_mutex); + cpus_read_lock(); + wait_attach_done_lock(); + if (is_cpuset_online(cs)) retval = update_prstate(cs, val); cpuset_update_sd_hk_unlock(); @@ -3616,11 +3646,7 @@ static int cpuset_can_fork(struct task_struct *task, struct css_set *cset) if (ret) goto out_unlock; - /* - * Mark attach is in progress. This makes validate_change() fail - * changes which zero cpus/mems_allowed. - */ - cs->attach_in_progress++; + attach_ctx.in_progress++; out_unlock: mutex_unlock(&cpuset_mutex); return ret; @@ -3638,7 +3664,7 @@ static void cpuset_cancel_fork(struct task_struct *task, struct css_set *cset) if (same_cs) return; - dec_attach_in_progress(cs); + dec_attach_in_progress(); } /* @@ -3670,7 +3696,7 @@ static void cpuset_fork(struct task_struct *task) guarantee_online_mems(cs, &cpuset_attach_nodemask_to); cpuset_attach_task(cs, task); - dec_attach_in_progress_locked(cs); + dec_attach_in_progress_locked(); mutex_unlock(&cpuset_mutex); } @@ -3774,20 +3800,8 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp) bool remote; int partcmd = -1; struct cpuset *parent; -retry: - wait_event(cpuset_attach_wq, cs->attach_in_progress == 0); - - mutex_lock(&cpuset_mutex); - - /* - * We have raced with task attaching. We wait until attaching - * is finished, so we won't attach a task to an empty cpuset. - */ - if (cs->attach_in_progress) { - mutex_unlock(&cpuset_mutex); - goto retry; - } + wait_attach_done_lock(); parent = parent_cs(cs); compute_effective_cpumask(&new_cpus, cs, parent); compute_effective_nodemask(&new_mems, cs, parent); From 892b8bb3fb7c0a96f294fc8b34e4477f51c90ab0 Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Thu, 2 Jul 2026 17:47:50 -0400 Subject: [PATCH 12/34] cgroup/cpuset: Put all task attach related variables into attach_ctx Put the task attach related cpuset_attach_old_cs and cpuset_attach_nodemask_to static variables into the new attach_ctx structure to improve readability and ease maintanence. No functional change is expected. Suggested-by: Ridong Chen Signed-off-by: Waiman Long Reviewed-by: Ridong Chen Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 9b3f9524e139..8dbbac8a66fd 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -362,6 +362,8 @@ static DECLARE_WAIT_QUEUE_HEAD(cpuset_attach_wq); */ static struct { int in_progress; + struct cpuset *old_cs; /* Source cpuset */ + nodemask_t nodemask_to; } attach_ctx; /* @@ -3015,8 +3017,6 @@ static int update_prstate(struct cpuset *cs, int new_prs) return 0; } -static struct cpuset *cpuset_attach_old_cs; - /* * Check to see if a cpuset can accept a new task * For v1, cpus_allowed and mems_allowed can't be empty. @@ -3048,8 +3048,8 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) int cpu, ret; /* used later by cpuset_attach() */ - cpuset_attach_old_cs = task_cs(cgroup_taskset_first(tset, &css)); - oldcs = cpuset_attach_old_cs; + attach_ctx.old_cs = task_cs(cgroup_taskset_first(tset, &css)); + oldcs = attach_ctx.old_cs; cs = css_cs(css); mutex_lock(&cpuset_mutex); @@ -3152,7 +3152,6 @@ static void cpuset_cancel_attach(struct cgroup_taskset *tset) * allocate from cpuset_init(). */ static cpumask_var_t cpus_attach; -static nodemask_t cpuset_attach_nodemask_to; static void cpuset_attach_task(struct cpuset *cs, struct task_struct *task) { @@ -3169,7 +3168,7 @@ static void cpuset_attach_task(struct cpuset *cs, struct task_struct *task) */ WARN_ON_ONCE(set_cpus_allowed_ptr(task, cpus_attach)); - cpuset_change_task_nodemask(task, &cpuset_attach_nodemask_to); + cpuset_change_task_nodemask(task, &attach_ctx.nodemask_to); cpuset1_update_task_spread_flags(cs, task); } @@ -3179,7 +3178,7 @@ static void cpuset_attach(struct cgroup_taskset *tset) struct task_struct *leader; struct cgroup_subsys_state *css; struct cpuset *cs; - struct cpuset *oldcs = cpuset_attach_old_cs; + struct cpuset *oldcs = attach_ctx.old_cs; bool cpus_updated, mems_updated; bool queue_task_work = false; @@ -3191,7 +3190,7 @@ static void cpuset_attach(struct cgroup_taskset *tset) cpus_updated = !cpumask_equal(cs->effective_cpus, oldcs->effective_cpus); mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems); - guarantee_online_mems(cs, &cpuset_attach_nodemask_to); + guarantee_online_mems(cs, &attach_ctx.nodemask_to); /* * In the default hierarchy, enabling cpuset in the child cgroups @@ -3230,7 +3229,7 @@ static void cpuset_attach(struct cgroup_taskset *tset) */ if (is_memory_migrate(cs)) { cpuset_migrate_mm(mm, &oldcs->old_mems_allowed, - &cpuset_attach_nodemask_to); + &attach_ctx.nodemask_to); queue_task_work = true; } else mmput(mm); @@ -3240,7 +3239,7 @@ static void cpuset_attach(struct cgroup_taskset *tset) out: if (queue_task_work) schedule_flush_migrate_mm(); - cs->old_mems_allowed = cpuset_attach_nodemask_to; + cs->old_mems_allowed = attach_ctx.nodemask_to; if (cs->nr_migrate_dl_tasks) { atomic_add(cs->nr_migrate_dl_tasks, &cs->nr_deadline_tasks); @@ -3693,7 +3692,7 @@ static void cpuset_fork(struct task_struct *task) /* CLONE_INTO_CGROUP */ mutex_lock(&cpuset_mutex); - guarantee_online_mems(cs, &cpuset_attach_nodemask_to); + guarantee_online_mems(cs, &attach_ctx.nodemask_to); cpuset_attach_task(cs, task); dec_attach_in_progress_locked(); From e165f243fecce67898eaf986b3e768936ae4be8c Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Thu, 2 Jul 2026 17:47:51 -0400 Subject: [PATCH 13/34] cgroup/cpuset: Add a cpuset_reserve_dl_bw() helper Extract the DL bandwidth allocation code in cpuset_attach() to a new cpuset_reserve_dl_bw() helper to simplify code. No functional change is expected. Signed-off-by: Waiman Long Reviewed-by: Ridong Chen Reviewed-by: Gregory Price Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 8dbbac8a66fd..33a6f932249e 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -3031,6 +3031,25 @@ static int cpuset_can_attach_check(struct cpuset *cs) return 0; } +static int cpuset_reserve_dl_bw(struct cpuset *cs) +{ + int cpu, ret; + + if (!cs->sum_migrate_dl_bw) + return 0; + + cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus); + if (unlikely(cpu >= nr_cpu_ids)) + return -EINVAL; + + ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw); + if (ret) + return ret; + + cs->dl_bw_cpu = cpu; + return 0; +} + static void reset_migrate_dl_data(struct cpuset *cs) { cs->nr_migrate_dl_tasks = 0; @@ -3045,7 +3064,7 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) struct cpuset *cs, *oldcs; struct task_struct *task; bool setsched_check; - int cpu, ret; + int ret; /* used later by cpuset_attach() */ attach_ctx.old_cs = task_cs(cgroup_taskset_first(tset, &css)); @@ -3101,27 +3120,14 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) } } - if (!cs->sum_migrate_dl_bw) - goto out_success; - - cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus); - if (unlikely(cpu >= nr_cpu_ids)) { - ret = -EINVAL; - goto out_unlock; - } - - ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw); - if (ret) - goto out_unlock; - - cs->dl_bw_cpu = cpu; - -out_success: - attach_ctx.in_progress++; + ret = cpuset_reserve_dl_bw(cs); out_unlock: if (ret) reset_migrate_dl_data(cs); + else + attach_ctx.in_progress++; + mutex_unlock(&cpuset_mutex); return ret; } From 74eda6ea709e7effcd6678255fcaefad47d44695 Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Thu, 2 Jul 2026 17:47:52 -0400 Subject: [PATCH 14/34] cgroup/cpuset: Expand the scope of cpuset_can_attach_check() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expand the scope of cpuset_can_attach_check() by including the setting of setsched flag inside cpuset_can_attach_check() with the new @oldcs and @psetsched argument. As cpuset_can_attach_check() is also called from cpuset_can_fork(), set the new arguments to NULL from that caller. Signed-off-by: Waiman Long Reviewed-by: Ridong Chen Signed-off-by: Michal Koutný Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 52 ++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 33a6f932249e..7ada5e74346d 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -3022,12 +3022,39 @@ static int update_prstate(struct cpuset *cs, int new_prs) * For v1, cpus_allowed and mems_allowed can't be empty. * For v2, effective_cpus can't be empty. * Note that in v1, effective_cpus = cpus_allowed. + * + * Also set the boolean flag passed in by @psetsched depending on if + * security_task_setscheduler() call is needed and @oldcs is not NULL. */ -static int cpuset_can_attach_check(struct cpuset *cs) +static int cpuset_can_attach_check(struct cpuset *cs, struct cpuset *oldcs, + bool *psetsched) { if (cpumask_empty(cs->effective_cpus) || (!is_in_v2_mode() && nodes_empty(cs->mems_allowed))) return -ENOSPC; + + if (!oldcs) + return 0; + + /* + * Skip rights over task setsched check in v2 when nothing changes, + * migration permission derives from hierarchy ownership in + * cgroup_procs_write_permission()). + */ + *psetsched = !cpuset_v2() || + !cpumask_equal(cs->effective_cpus, oldcs->effective_cpus) || + !nodes_equal(cs->effective_mems, oldcs->effective_mems); + + /* + * A v1 cpuset with tasks will have no CPU left only when CPU hotplug + * brings the last online CPU offline as users are not allowed to empty + * cpuset.cpus when there are active tasks inside. When that happens, + * we should allow tasks to migrate out without security check to make + * sure they will be able to run after migration. + */ + if (!is_in_v2_mode() && cpumask_empty(oldcs->effective_cpus)) + *psetsched = false; + return 0; } @@ -3074,29 +3101,10 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) mutex_lock(&cpuset_mutex); /* Check to see if task is allowed in the cpuset */ - ret = cpuset_can_attach_check(cs); + ret = cpuset_can_attach_check(cs, oldcs, &setsched_check); if (ret) goto out_unlock; - /* - * Skip rights over task setsched check in v2 when nothing changes, - * migration permission derives from hierarchy ownership in - * cgroup_procs_write_permission()). - */ - setsched_check = !cpuset_v2() || - !cpumask_equal(cs->effective_cpus, oldcs->effective_cpus) || - !nodes_equal(cs->effective_mems, oldcs->effective_mems); - - /* - * A v1 cpuset with tasks will have no CPU left only when CPU hotplug - * brings the last online CPU offline as users are not allowed to empty - * cpuset.cpus when there are active tasks inside. When that happens, - * we should allow tasks to migrate out without security check to make - * sure they will be able to run after migration. - */ - if (!is_in_v2_mode() && cpumask_empty(oldcs->effective_cpus)) - setsched_check = false; - cgroup_taskset_for_each(task, css, tset) { ret = task_can_attach(task); if (ret) @@ -3639,7 +3647,7 @@ static int cpuset_can_fork(struct task_struct *task, struct css_set *cset) mutex_lock(&cpuset_mutex); /* Check to see if task is allowed in the cpuset */ - ret = cpuset_can_attach_check(cs); + ret = cpuset_can_attach_check(cs, NULL, NULL); if (ret) goto out_unlock; From 1bc48a502a43a4e42954ccd9e771af658e0e405f Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Thu, 2 Jul 2026 17:47:53 -0400 Subject: [PATCH 15/34] cgroup/cpuset: Make attach_ctx.old_cs track task group leader There are two possible ways that migration of tasks from multiple source cpusets to a target cpuset can happen. Either a multithread application with threads in different cpusets is wholely migrated to a new cpuset or disabling of v2 cpuset controller will move all the tasks in child cpusets to the parent cpuset. In the former case, it is the mm setting of the group leader that really matters. So attach_ctx.old_cs should track the oldcs of the thread leader. In the latter case, effective_mems of child cpusets must always be a subset of the parent. So no real page migration will not be necessary no matter which child cpuset is selected as attach_ctx.old_cs. IOW, attach_ctx.old_cs should be updated to match the latest task group leader in cpuset_can_attach(), but fall back to that of the first task if there is no group leader in the taskset. Suggested-by: Ridong Chen Signed-off-by: Waiman Long Reviewed-by: Ridong Chen Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 7ada5e74346d..ff02d763cc97 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -3105,11 +3105,32 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) if (ret) goto out_unlock; + /* + * The attach_ctx.old_cs is used mainly by cpuset_migrate_mm() to get + * the old_mems_allowed value. There are two ways that many-to-one + * cpuset migration can happen: + * 1) A multithread application with threads in different cpusets is + * wholely migrated to a new cpuset. + * 2) Disabling v2 cpuset controller will move all the tasks in child + * cpusets to the parent cpuset. + * + * In the former case, it is the mm setting of the group leader that + * really matters. So attach_ctx.old_cs should track the oldcs of the + * group leader. It falls back to the oldcs of the first task if there + * is no group leader in the taskset. In the latter case, effective_mems + * of child cpusets must always be a subset of the parent. So no real + * page migration will be necessary no matter which child cpuset is + * selected as attach_ctx.old_cs. + */ cgroup_taskset_for_each(task, css, tset) { ret = task_can_attach(task); if (ret) goto out_unlock; + /* Update attach_ctx.old_cs to the latest group leader */ + if (task == task->group_leader) + attach_ctx.old_cs = task_cs(task); + if (setsched_check) { ret = security_task_setscheduler(task); if (ret) From 65e510cd30d01663160b8bc5c7d9928ab7303977 Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Thu, 2 Jul 2026 17:47:54 -0400 Subject: [PATCH 16/34] cgroup/cpuset: Move mpol_rebind_mm/cpuset_migrate_mm() calls inside cpuset_attach_task() The cpuset_attach_task() was introduced in commit 42a11bf5c543 ("cgroup/cpuset: Make cpuset_fork() handle CLONE_INTO_CGROUP properly") to enable the CLONE_INTO_CGROUP flag of clone(2) to behave more like moving a task from one cpuset into another one. That commits didn't move the mpol_rebind_mm() and cpuset_migrate_mm() calls for group leader into cpuset_attach_task(). When the CLONE_INTO_CGROUP flag is used without CLONE_THREAD, the new task is its own group leader. So it is still not equivalent to moving task between cpusets in this case. Make CLONE_INTO_CGROUP behaves more close to cpuset_attach() by moving the mpol_rebind_mm() and cpuset_migrate_mm() calls inside cpuset_attach_task(). Also move the stack local cpus_updated, mems_updated and queue_task_work flags into attach_ctx so that these flags can be accessed inside and outside of cpuset_attach_task(). The cpuset_fork() function is updated to set up these flags and do memory migration if necessary. Reviewed-by: Ridong Chen Signed-off-by: Waiman Long Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 103 +++++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 44 deletions(-) diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index ff02d763cc97..947fe416d093 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -362,6 +362,9 @@ static DECLARE_WAIT_QUEUE_HEAD(cpuset_attach_wq); */ static struct { int in_progress; + bool cpus_updated; + bool mems_updated; + bool task_work_queued; struct cpuset *old_cs; /* Source cpuset */ nodemask_t nodemask_to; } attach_ctx; @@ -3190,6 +3193,8 @@ static cpumask_var_t cpus_attach; static void cpuset_attach_task(struct cpuset *cs, struct task_struct *task) { + struct mm_struct *mm; + lockdep_assert_cpuset_lock_held(); if (cs != &top_cpuset) @@ -3203,28 +3208,59 @@ static void cpuset_attach_task(struct cpuset *cs, struct task_struct *task) */ WARN_ON_ONCE(set_cpus_allowed_ptr(task, cpus_attach)); + if (cpuset_v2() && !attach_ctx.mems_updated) + return; + cpuset_change_task_nodemask(task, &attach_ctx.nodemask_to); cpuset1_update_task_spread_flags(cs, task); + + if ((task != task->group_leader) || !attach_ctx.mems_updated) + return; + + /* + * Change mm for threadgroup leader. This is expensive and may + * sleep and should be moved outside migration path proper. + */ + mm = get_task_mm(task); + if (mm) { + struct cpuset *oldcs = attach_ctx.old_cs; + + mpol_rebind_mm(mm, &cs->effective_mems); + + /* + * old_mems_allowed is the same with mems_allowed + * here, except if this task is being moved + * automatically due to hotplug. In that case + * @mems_allowed has been updated and is empty, so + * @old_mems_allowed is the right nodesets that we + * migrate mm from. + */ + if (is_memory_migrate(cs)) { + cpuset_migrate_mm(mm, &oldcs->old_mems_allowed, + &attach_ctx.nodemask_to); + attach_ctx.task_work_queued = true; + } else { + mmput(mm); + } + } } static void cpuset_attach(struct cgroup_taskset *tset) { struct task_struct *task; - struct task_struct *leader; struct cgroup_subsys_state *css; struct cpuset *cs; struct cpuset *oldcs = attach_ctx.old_cs; - bool cpus_updated, mems_updated; - bool queue_task_work = false; cgroup_taskset_first(tset, &css); cs = css_cs(css); lockdep_assert_cpus_held(); /* see cgroup_attach_lock() */ mutex_lock(&cpuset_mutex); - cpus_updated = !cpumask_equal(cs->effective_cpus, - oldcs->effective_cpus); - mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems); + attach_ctx.task_work_queued = false; + + attach_ctx.cpus_updated = !cpumask_equal(cs->effective_cpus, oldcs->effective_cpus); + attach_ctx.mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems); guarantee_online_mems(cs, &attach_ctx.nodemask_to); /* @@ -3233,46 +3269,14 @@ static void cpuset_attach(struct cgroup_taskset *tset) * and mems. In that case, we can optimize out by skipping the task * iteration and update. */ - if (cpuset_v2() && !cpus_updated && !mems_updated) + if (cpuset_v2() && !attach_ctx.cpus_updated && !attach_ctx.mems_updated) goto out; cgroup_taskset_for_each(task, css, tset) cpuset_attach_task(cs, task); - /* - * Change mm for all threadgroup leaders. This is expensive and may - * sleep and should be moved outside migration path proper. Skip it - * if there is no change in effective_mems and CS_MEMORY_MIGRATE is - * not set. - */ - if (!is_memory_migrate(cs) && !mems_updated) - goto out; - - cgroup_taskset_for_each_leader(leader, css, tset) { - struct mm_struct *mm = get_task_mm(leader); - - if (mm) { - mpol_rebind_mm(mm, &cs->effective_mems); - - /* - * old_mems_allowed is the same with mems_allowed - * here, except if this task is being moved - * automatically due to hotplug. In that case - * @mems_allowed has been updated and is empty, so - * @old_mems_allowed is the right nodesets that we - * migrate mm from. - */ - if (is_memory_migrate(cs)) { - cpuset_migrate_mm(mm, &oldcs->old_mems_allowed, - &attach_ctx.nodemask_to); - queue_task_work = true; - } else - mmput(mm); - } - } - out: - if (queue_task_work) + if (attach_ctx.task_work_queued) schedule_flush_migrate_mm(); cs->old_mems_allowed = attach_ctx.nodemask_to; @@ -3708,15 +3712,14 @@ static void cpuset_cancel_fork(struct task_struct *task, struct css_set *cset) */ static void cpuset_fork(struct task_struct *task) { - struct cpuset *cs; - bool same_cs; + struct cpuset *cs, *oldcs; rcu_read_lock(); cs = task_cs(task); - same_cs = (cs == task_cs(current)); + oldcs = task_cs(current); rcu_read_unlock(); - if (same_cs) { + if (cs == oldcs) { if (cs == &top_cpuset) return; @@ -3728,7 +3731,19 @@ static void cpuset_fork(struct task_struct *task) /* CLONE_INTO_CGROUP */ mutex_lock(&cpuset_mutex); guarantee_online_mems(cs, &attach_ctx.nodemask_to); + cs->old_mems_allowed = attach_ctx.nodemask_to; + + /* + * Assume CPUs and memory nodes are updated + * A CLONE_INTO_CGROUP operation should have taken the cgroup mutex + * and so there shouldn't be a competing cpuset_attach() operation. + */ + attach_ctx.cpus_updated = attach_ctx.mems_updated = true; + attach_ctx.task_work_queued = false; + attach_ctx.old_cs = oldcs; cpuset_attach_task(cs, task); + if (attach_ctx.task_work_queued) + schedule_flush_migrate_mm(); dec_attach_in_progress_locked(); mutex_unlock(&cpuset_mutex); From ac1607366c04ad833e37c14e7b70c8f7ebe42339 Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Thu, 2 Jul 2026 17:47:55 -0400 Subject: [PATCH 17/34] cgroup/cpuset: Support multiple source cpusets for cpuset_*attach() There are 2 possible scenarios where the cgroup_taskset structure passed into the cgroup can_attach() and attach() methods can contain task migration data with multiple source cpusets. - A multithread application with threads in different cpusets is fully migrated into a new cpuset. - Disabling v2 cpuset controller will move all the tasks in child cpusets to the parent cpuset. The current cpuset_can_attach() and cpuset_attach() functions still expect task migration is from one source cpuset to one destination cpuset. Fix that by tracking the set of source (old) cpusets in singly linked lists. The list will be iterated when necessary to properly update internal data. To ensure proper DL tasks accounting, the nr_migrate_dl_tasks in both the source and destination cpusets are decremented/incremented with their values added to nr_deadline_tasks when the migration is successful. The setting of the global attach_ctx.cpus_updated and attach_ctx.mems_updated flags are also moved from cpuset_attach() to cpuset_can_attach() as the correct source cpuset can no longer be determined in cpuset_attach() and cpuset states will not be changed between cpuset_attach() and cpuset_can_attach() with an earlier patch. Signed-off-by: Waiman Long Reviewed-by: Ridong Chen Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset-internal.h | 5 +++ kernel/cgroup/cpuset.c | 73 ++++++++++++++++++++++++++------- 2 files changed, 64 insertions(+), 14 deletions(-) diff --git a/kernel/cgroup/cpuset-internal.h b/kernel/cgroup/cpuset-internal.h index df662c7fd1a4..e7d010661fd3 100644 --- a/kernel/cgroup/cpuset-internal.h +++ b/kernel/cgroup/cpuset-internal.h @@ -145,6 +145,11 @@ struct cpuset { */ nodemask_t old_mems_allowed; + /* + * For linking impacted cpusets during an attach operation. + */ + struct llist_node attach_node; + /* partition root state */ int partition_root_state; diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 947fe416d093..9c1ccd84dfdd 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -37,6 +37,7 @@ #include #include #include +#include DEFINE_STATIC_KEY_FALSE(cpusets_pre_enable_key); DEFINE_STATIC_KEY_FALSE(cpusets_enabled_key); @@ -368,6 +369,7 @@ static struct { struct cpuset *old_cs; /* Source cpuset */ nodemask_t nodemask_to; } attach_ctx; +static LLIST_HEAD(src_cs_head); /* * Wait if task attach is in progress until it is done and then acquire @@ -615,6 +617,7 @@ static struct cpuset *dup_or_alloc_cpuset(struct cpuset *cs) return NULL; trial->dl_bw_cpu = -1; + init_llist_node(&trial->attach_node); /* Setup cpumask pointer array */ cpumask_var_t *pmask[4] = { @@ -3032,6 +3035,8 @@ static int update_prstate(struct cpuset *cs, int new_prs) static int cpuset_can_attach_check(struct cpuset *cs, struct cpuset *oldcs, bool *psetsched) { + bool cpus_updated, mems_updated; + if (cpumask_empty(cs->effective_cpus) || (!is_in_v2_mode() && nodes_empty(cs->mems_allowed))) return -ENOSPC; @@ -3039,14 +3044,23 @@ static int cpuset_can_attach_check(struct cpuset *cs, struct cpuset *oldcs, if (!oldcs) return 0; + if (!llist_on_list(&oldcs->attach_node)) + llist_add(&oldcs->attach_node, &src_cs_head); + + cpus_updated = !cpumask_equal(cs->effective_cpus, oldcs->effective_cpus); + mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems); + + if (cpus_updated) + attach_ctx.cpus_updated = true; + if (mems_updated) + attach_ctx.mems_updated = true; + /* - * Skip rights over task setsched check in v2 when nothing changes, - * migration permission derives from hierarchy ownership in - * cgroup_procs_write_permission()). + * Skip rights over task setsched check in v2 when nothing changes for + * the current oldcs/cs pair, migration permission derives from + * hierarchy ownership in cgroup_procs_write_permission()). */ - *psetsched = !cpuset_v2() || - !cpumask_equal(cs->effective_cpus, oldcs->effective_cpus) || - !nodes_equal(cs->effective_mems, oldcs->effective_mems); + *psetsched = !cpuset_v2() || cpus_updated || mems_updated; /* * A v1 cpuset with tasks will have no CPU left only when CPU hotplug @@ -3087,6 +3101,25 @@ static void reset_migrate_dl_data(struct cpuset *cs) cs->dl_bw_cpu = -1; } +/* + * Clear and optionally apply (@cancel is false) the attach related data in the + * source cpusets. + */ +static void clear_attach_data(struct llist_head *head, bool cancel) +{ + struct cpuset *cs, *next; + struct llist_node *lnode = __llist_del_all(head); + + llist_for_each_entry_safe(cs, next, lnode, attach_node) { + init_llist_node(&cs->attach_node); + if (cs->nr_migrate_dl_tasks) { + if (!cancel) + atomic_add(cs->nr_migrate_dl_tasks, &cs->nr_deadline_tasks); + cs->nr_migrate_dl_tasks = 0; + } + } +} + /* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */ static int cpuset_can_attach(struct cgroup_taskset *tset) { @@ -3102,6 +3135,8 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) cs = css_cs(css); mutex_lock(&cpuset_mutex); + attach_ctx.cpus_updated = false; + attach_ctx.mems_updated = false; /* Check to see if task is allowed in the cpuset */ ret = cpuset_can_attach_check(cs, oldcs, &setsched_check); @@ -3126,6 +3161,15 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) * selected as attach_ctx.old_cs. */ cgroup_taskset_for_each(task, css, tset) { + struct cpuset *new_oldcs = task_cs(task); + + if (new_oldcs != oldcs) { + oldcs = new_oldcs; + ret = cpuset_can_attach_check(cs, oldcs, &setsched_check); + if (ret) + goto out_unlock; + } + ret = task_can_attach(task); if (ret) goto out_unlock; @@ -3147,6 +3191,7 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) * contribute to sum_migrate_dl_bw. */ cs->nr_migrate_dl_tasks++; + oldcs->nr_migrate_dl_tasks--; if (dl_task_needs_bw_move(task, cs->effective_cpus)) cs->sum_migrate_dl_bw += task->dl.dl_bw; } @@ -3155,10 +3200,12 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) ret = cpuset_reserve_dl_bw(cs); out_unlock: - if (ret) - reset_migrate_dl_data(cs); - else + if (ret) { + reset_migrate_dl_data(cs); /* Destination cpuset only */ + clear_attach_data(&src_cs_head, true); + } else { attach_ctx.in_progress++; + } mutex_unlock(&cpuset_mutex); return ret; @@ -3174,6 +3221,7 @@ static void cpuset_cancel_attach(struct cgroup_taskset *tset) mutex_lock(&cpuset_mutex); dec_attach_in_progress_locked(); + clear_attach_data(&src_cs_head, true); if (cs->dl_bw_cpu >= 0) dl_bw_free(cs->dl_bw_cpu, cs->sum_migrate_dl_bw); @@ -3250,7 +3298,6 @@ static void cpuset_attach(struct cgroup_taskset *tset) struct task_struct *task; struct cgroup_subsys_state *css; struct cpuset *cs; - struct cpuset *oldcs = attach_ctx.old_cs; cgroup_taskset_first(tset, &css); cs = css_cs(css); @@ -3258,9 +3305,6 @@ static void cpuset_attach(struct cgroup_taskset *tset) lockdep_assert_cpus_held(); /* see cgroup_attach_lock() */ mutex_lock(&cpuset_mutex); attach_ctx.task_work_queued = false; - - attach_ctx.cpus_updated = !cpumask_equal(cs->effective_cpus, oldcs->effective_cpus); - attach_ctx.mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems); guarantee_online_mems(cs, &attach_ctx.nodemask_to); /* @@ -3282,10 +3326,10 @@ static void cpuset_attach(struct cgroup_taskset *tset) if (cs->nr_migrate_dl_tasks) { atomic_add(cs->nr_migrate_dl_tasks, &cs->nr_deadline_tasks); - atomic_sub(cs->nr_migrate_dl_tasks, &oldcs->nr_deadline_tasks); reset_migrate_dl_data(cs); } + clear_attach_data(&src_cs_head, false); dec_attach_in_progress_locked(); mutex_unlock(&cpuset_mutex); @@ -3792,6 +3836,7 @@ int __init cpuset_init(void) cpumask_setall(top_cpuset.effective_xcpus); cpumask_setall(top_cpuset.exclusive_cpus); nodes_setall(top_cpuset.effective_mems); + init_llist_node(&top_cpuset.attach_node); cpuset1_init(&top_cpuset); From 97e7efdda8a2b0c2ed2f15e4e49d25ab89d2bfa2 Mon Sep 17 00:00:00 2001 From: Song Hu Date: Tue, 14 Jul 2026 10:15:11 +0800 Subject: [PATCH 18/34] selftests/cgroup: fix missing TAP output in test_hugetlb_memcg main() in test_hugetlb_memcg never calls ksft_print_header(), ksft_set_plan(), or ksft_finished(), so its output has no TAP plan and is not valid TAP, unlike the sibling test_memcontrol and test_kmem tests. Add the header/plan/finished calls following the same pattern. Signed-off-by: Song Hu Reviewed-by: Tao Cui Signed-off-by: Tejun Heo --- tools/testing/selftests/cgroup/test_hugetlb_memcg.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/cgroup/test_hugetlb_memcg.c b/tools/testing/selftests/cgroup/test_hugetlb_memcg.c index b627d84358b1..8c5aced813b6 100644 --- a/tools/testing/selftests/cgroup/test_hugetlb_memcg.c +++ b/tools/testing/selftests/cgroup/test_hugetlb_memcg.c @@ -199,7 +199,10 @@ static int test_hugetlb_memcg(char *root) int main(int argc, char **argv) { char root[PATH_MAX]; - int ret = EXIT_SUCCESS, has_memory_hugetlb_acc; + int has_memory_hugetlb_acc; + + ksft_print_header(); + ksft_set_plan(1); has_memory_hugetlb_acc = proc_mount_contains("memory_hugetlb_accounting"); if (has_memory_hugetlb_acc < 0) @@ -211,7 +214,7 @@ int main(int argc, char **argv) if (get_hugepage_size() != 2048) { ksft_print_msg("test_hugetlb_memcg requires 2MB hugepages\n"); ksft_test_result_skip("test_hugetlb_memcg\n"); - return ret; + ksft_finished(); } if (cg_find_unified_root(root, sizeof(root), NULL)) @@ -233,10 +236,9 @@ int main(int argc, char **argv) ksft_test_result_skip("test_hugetlb_memcg\n"); break; default: - ret = EXIT_FAILURE; ksft_test_result_fail("test_hugetlb_memcg\n"); break; } - return ret; + ksft_finished(); } From 7309352a040017871bfc660aca27aa990e4170bb Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Sun, 12 Jul 2026 19:55:08 -0400 Subject: [PATCH 19/34] cgroup/cpuset: Support multiple destination cpusets for cpuset_*attach() The only case where the cgroup_taskset structure requires task migration to multiple cpusets is when enabling a cpuset controller in cgroup v2 where the newly created child cpusets inherits the same effective CPUs and memory nodes from the parent. In that case, task migration can happen directly with no update to tasks' CPU and memory nodes assignment and no further work needed from the cpuset side except updating nr_deadline_tasks when DL tasks are involved and setting old_mems_allowed in the child cpusets. Do that by tracking all the destination cpusets with a new dst_cs_head singly linked list. The reset_migrate_dl_data() function is integrated into clear_attach_data() so that it can be used for both source and destination cpusets. A warning will be printed if there are multiple destination cpusets but it is not on default hierarchy or when the CPUs or memory nodes change. Signed-off-by: Waiman Long Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 99 ++++++++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 43 deletions(-) diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 9c1ccd84dfdd..6a258ab1d66d 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -366,10 +366,12 @@ static struct { bool cpus_updated; bool mems_updated; bool task_work_queued; + bool many_dest_cs; /* Have many destination cpusets */ struct cpuset *old_cs; /* Source cpuset */ nodemask_t nodemask_to; } attach_ctx; static LLIST_HEAD(src_cs_head); +static LLIST_HEAD(dst_cs_head); /* * Wait if task attach is in progress until it is done and then acquire @@ -3047,6 +3049,9 @@ static int cpuset_can_attach_check(struct cpuset *cs, struct cpuset *oldcs, if (!llist_on_list(&oldcs->attach_node)) llist_add(&oldcs->attach_node, &src_cs_head); + if (!llist_on_list(&cs->attach_node)) + llist_add(&cs->attach_node, &dst_cs_head); + cpus_updated = !cpumask_equal(cs->effective_cpus, oldcs->effective_cpus); mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems); @@ -3075,35 +3080,31 @@ static int cpuset_can_attach_check(struct cpuset *cs, struct cpuset *oldcs, return 0; } -static int cpuset_reserve_dl_bw(struct cpuset *cs) +static int cpuset_reserve_dl_bw(void) { + struct cpuset *cs; int cpu, ret; - if (!cs->sum_migrate_dl_bw) - return 0; + llist_for_each_entry(cs, dst_cs_head.first, attach_node) { + if (!cs->sum_migrate_dl_bw) + continue; - cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus); - if (unlikely(cpu >= nr_cpu_ids)) - return -EINVAL; + cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus); + if (unlikely(cpu >= nr_cpu_ids)) + return -EINVAL; - ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw); - if (ret) - return ret; + ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw); + if (ret) + return ret; - cs->dl_bw_cpu = cpu; + cs->dl_bw_cpu = cpu; + } return 0; } -static void reset_migrate_dl_data(struct cpuset *cs) -{ - cs->nr_migrate_dl_tasks = 0; - cs->sum_migrate_dl_bw = 0; - cs->dl_bw_cpu = -1; -} - /* * Clear and optionally apply (@cancel is false) the attach related data in the - * source cpusets. + * source or destination cpuset. */ static void clear_attach_data(struct llist_head *head, bool cancel) { @@ -3115,7 +3116,11 @@ static void clear_attach_data(struct llist_head *head, bool cancel) if (cs->nr_migrate_dl_tasks) { if (!cancel) atomic_add(cs->nr_migrate_dl_tasks, &cs->nr_deadline_tasks); + else if (cs->dl_bw_cpu >= 0) /* && cancel */ + dl_bw_free(cs->dl_bw_cpu, cs->sum_migrate_dl_bw); cs->nr_migrate_dl_tasks = 0; + cs->sum_migrate_dl_bw = 0; + cs->dl_bw_cpu = -1; } } } @@ -3137,6 +3142,7 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) mutex_lock(&cpuset_mutex); attach_ctx.cpus_updated = false; attach_ctx.mems_updated = false; + attach_ctx.many_dest_cs = false; /* Check to see if task is allowed in the cpuset */ ret = cpuset_can_attach_check(cs, oldcs, &setsched_check); @@ -3161,9 +3167,13 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) * selected as attach_ctx.old_cs. */ cgroup_taskset_for_each(task, css, tset) { + struct cpuset *new_cs = css_cs(css); struct cpuset *new_oldcs = task_cs(task); - if (new_oldcs != oldcs) { + if ((new_oldcs != oldcs) || (new_cs != cs)) { + if (new_cs != cs) + attach_ctx.many_dest_cs = true; + cs = new_cs; oldcs = new_oldcs; ret = cpuset_can_attach_check(cs, oldcs, &setsched_check); if (ret) @@ -3197,12 +3207,28 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) } } - ret = cpuset_reserve_dl_bw(cs); + /* + * The only case where there are multiple destination cpusets for + * task migration is when enabling a v2 cpuset controllers where + * tasks will be migrated to multiple child cpusets from a parent + * cpuset with the same effective CPUs and memory nodes. IOW, + * both attach_cpus_updated and attach_mems_updated should be false. + * If not, it is a condition that the current code cannot handle. + * Print a warning and abort the attach operation as further code + * change may be needed. + */ + if (WARN_ON_ONCE(attach_ctx.many_dest_cs && (!cpuset_v2() || + attach_ctx.cpus_updated || attach_ctx.mems_updated))) { + ret = -EINVAL; + goto out_unlock; + } + + ret = cpuset_reserve_dl_bw(); out_unlock: if (ret) { - reset_migrate_dl_data(cs); /* Destination cpuset only */ clear_attach_data(&src_cs_head, true); + clear_attach_data(&dst_cs_head, true); } else { attach_ctx.in_progress++; } @@ -3213,22 +3239,10 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) static void cpuset_cancel_attach(struct cgroup_taskset *tset) { - struct cgroup_subsys_state *css; - struct cpuset *cs; - - cgroup_taskset_first(tset, &css); - cs = css_cs(css); - mutex_lock(&cpuset_mutex); dec_attach_in_progress_locked(); clear_attach_data(&src_cs_head, true); - - if (cs->dl_bw_cpu >= 0) - dl_bw_free(cs->dl_bw_cpu, cs->sum_migrate_dl_bw); - - if (cs->nr_migrate_dl_tasks) - reset_migrate_dl_data(cs); - + clear_attach_data(&dst_cs_head, true); mutex_unlock(&cpuset_mutex); } @@ -3311,25 +3325,24 @@ static void cpuset_attach(struct cgroup_taskset *tset) * In the default hierarchy, enabling cpuset in the child cgroups * will trigger a cpuset_attach() call with no change in effective cpus * and mems. In that case, we can optimize out by skipping the task - * iteration and update. + * iteration and the destination cpuset list is iterated to set + * old_mems_allowed. */ - if (cpuset_v2() && !attach_ctx.cpus_updated && !attach_ctx.mems_updated) + if (cpuset_v2() && !attach_ctx.cpus_updated && !attach_ctx.mems_updated) { + llist_for_each_entry(cs, dst_cs_head.first, attach_node) + cs->old_mems_allowed = attach_ctx.nodemask_to; goto out; + } cgroup_taskset_for_each(task, css, tset) cpuset_attach_task(cs, task); -out: if (attach_ctx.task_work_queued) schedule_flush_migrate_mm(); cs->old_mems_allowed = attach_ctx.nodemask_to; - - if (cs->nr_migrate_dl_tasks) { - atomic_add(cs->nr_migrate_dl_tasks, &cs->nr_deadline_tasks); - reset_migrate_dl_data(cs); - } - +out: clear_attach_data(&src_cs_head, false); + clear_attach_data(&dst_cs_head, false); dec_attach_in_progress_locked(); mutex_unlock(&cpuset_mutex); From 9637786d38d850aa4e7ba7a7b4fef03f13a1384d Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Sun, 12 Jul 2026 19:55:09 -0400 Subject: [PATCH 20/34] cgroup/cpuset: Handle the special case of non-moving tasks in cpuset_can_attach() With cgroup v2 migration of a multithreaded process having threads in different cgroups of a threaded subtree, it is possible that cpuset_can_attach() can be called with tasks that are not migrating with respect to cpuset if cpuset controller is not enabled in some of the subtree cgroups. IOW, the old cpuset can be the same as the new one. This can cause problem when we need to track the set of old cpusets and the new cpusets in singly linked lists as a cpuset cannot be in both lists. As reported by Tejun, the following is an example threaded subtree with partial cpuset delegation that can cause this issue to show up. P (+cpuset) |- R (cpuset) <- destination | `- C (no cpuset) -> effective cpuset == R `- W (cpuset) Group leader in R, thread_a in C, thread_b in W; migrate the whole process into R (echo $PID > R/cgroup.procs). thread_a moves C->R: its cgroup changes so compare_css_sets() keeps it in the taskset, but its cpuset css is unchanged (C inherits R's), so task_cs() == cs == R. cpuset is in ss_mask because thread_b (W->R) changed. can_attach() then tags R as a source (thread_a) and the destination (thread_b): Handle this special case by skipping tasks that are not migrating in cpuset_can_attach() and avoid calling cpuset_can_attach_check() in this case. By doing so, the destination cpuset will not be put into source cpuset linked list. As the source cpuset cannot be easily determined in cpuset_attach(), unnecessary work can be performed if a task is not actually migrating. However, no harm will be done except wasting some CPU cycles. If it happens that none of the tasks is migrating, attach_ctx.old_cs will be NULL and task iteration won't be needed. Reported-by: Tejun Heo Closes: https://lore.kernel.org/lkml/e254af713b5345aec3d086771ecf1e71@kernel.org Signed-off-by: Waiman Long Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 6a258ab1d66d..8690d987d746 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -3134,21 +3134,13 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) bool setsched_check; int ret; - /* used later by cpuset_attach() */ - attach_ctx.old_cs = task_cs(cgroup_taskset_first(tset, &css)); - oldcs = attach_ctx.old_cs; - cs = css_cs(css); - + cs = oldcs = NULL; mutex_lock(&cpuset_mutex); + attach_ctx.old_cs = NULL; /* Used later in cpuset_attach_task() */ attach_ctx.cpus_updated = false; attach_ctx.mems_updated = false; attach_ctx.many_dest_cs = false; - /* Check to see if task is allowed in the cpuset */ - ret = cpuset_can_attach_check(cs, oldcs, &setsched_check); - if (ret) - goto out_unlock; - /* * The attach_ctx.old_cs is used mainly by cpuset_migrate_mm() to get * the old_mems_allowed value. There are two ways that many-to-one @@ -3165,21 +3157,33 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) * of child cpusets must always be a subset of the parent. So no real * page migration will be necessary no matter which child cpuset is * selected as attach_ctx.old_cs. + * + * For a v2 threaded subtree where cpuset isn't enabled in some of the + * cgroups, it is possible that oldcs == cs for some of the tasks. + * In this case, we can skip checking on those tasks as there is no + * actual migration wrt cpuset. */ cgroup_taskset_for_each(task, css, tset) { struct cpuset *new_cs = css_cs(css); struct cpuset *new_oldcs = task_cs(task); if ((new_oldcs != oldcs) || (new_cs != cs)) { - if (new_cs != cs) + if (cs && (new_cs != cs)) attach_ctx.many_dest_cs = true; cs = new_cs; oldcs = new_oldcs; + if (oldcs == cs) + continue; + if (!attach_ctx.old_cs) + attach_ctx.old_cs = oldcs; ret = cpuset_can_attach_check(cs, oldcs, &setsched_check); if (ret) goto out_unlock; } + if (oldcs == cs) + continue; + ret = task_can_attach(task); if (ret) goto out_unlock; @@ -3321,6 +3325,14 @@ static void cpuset_attach(struct cgroup_taskset *tset) attach_ctx.task_work_queued = false; guarantee_online_mems(cs, &attach_ctx.nodemask_to); + /* + * attach_ctx.old_cs can only be NULL if no task is actually migrating. + * This is highly unlikely. If it happens at all, we can skip task + * iteration and setting old_mems_allowed. + */ + if (unlikely(!attach_ctx.old_cs)) + goto out; + /* * In the default hierarchy, enabling cpuset in the child cgroups * will trigger a cpuset_attach() call with no change in effective cpus From 98149f5425306cf0a1ed472b216da985b25350ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Koutn=C3=BD?= Date: Sun, 12 Jul 2026 19:55:10 -0400 Subject: [PATCH 21/34] selftests/cgroup: Add test for cpuset affinity on controller disable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new selftest that exposes a bug in cpuset_attach() where thread CPU affinity is not properly updated when the cpuset controller is disabled in a threaded cgroup hierarchy. The test creates a threaded cgroup hierarchy with two child cgroups (A and B) having different cpuset.cpus constraints: - Parent: cpuset.cpus=0-1 - Child A: cpuset.cpus=0-1 - Child B: cpuset.cpus=1 (restricted to CPU 1 only) A multithreaded process is created with threads placed in different cgroups. When the cpuset controller is disabled on the parent, thread affinities should be updated to match the parent's cpuset. Expected behavior: - thread_a affinity: {0-1} before and after (unchanged) - thread_b affinity: {1} before, {0-1} after (expanded) Current buggy behavior: - thread_b affinity remains {1} after controller disable Assisted-by: Claude:claude-sonnet-4-5 Signed-off-by: Michal Koutný Acked-by: Waiman Long Signed-off-by: Tejun Heo --- tools/testing/selftests/cgroup/test_cpuset.c | 243 +++++++++++++++++++ 1 file changed, 243 insertions(+) diff --git a/tools/testing/selftests/cgroup/test_cpuset.c b/tools/testing/selftests/cgroup/test_cpuset.c index c5cf8b56ceb8..8b4c4a9dd78b 100644 --- a/tools/testing/selftests/cgroup/test_cpuset.c +++ b/tools/testing/selftests/cgroup/test_cpuset.c @@ -1,7 +1,13 @@ // SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE +#include #include +#include +#include #include +#include +#include #include "kselftest.h" #include "cgroup_util.h" @@ -232,6 +238,242 @@ static int test_cpuset_perms_subtree(const char *root) return ret; } +static int get_cpu_affinity(cpu_set_t *mask) +{ + CPU_ZERO(mask); + return sched_getaffinity(0, sizeof(*mask), mask); +} + +static int cpu_set_equal(cpu_set_t *dst, unsigned long mask) +{ + cpu_set_t expected; + + CPU_ZERO(&expected); + assert(sizeof(mask) < CPU_SETSIZE); + + for (int cpu = 0; cpu < sizeof(mask); ++cpu) + if ((1UL << cpu) & mask) + CPU_SET(cpu, &expected); + + return CPU_EQUAL(&expected, dst); +} + +enum test_phase { + AFFINITY_SETUP, + AFFINITY_THREAD_A_READY, + AFFINITY_THREADS_READY, + AFFINITY_CONTROLLER_DISABLED, + AFFINITY_COMPLETE, + AFFINITY_ERROR +}; + +struct thread_args { + const char *cgroup; + cpu_set_t *affinity_before; + cpu_set_t *affinity_after; + enum test_phase ready_phase; +}; + +static pthread_mutex_t test_mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t test_cond = PTHREAD_COND_INITIALIZER; +static enum test_phase test_phase; + +static void *affinity_thread_fn(void *arg) +{ + struct thread_args *args = (struct thread_args *)arg; + + if (cg_enter_current_thread(args->cgroup)) + goto fail; + + if (get_cpu_affinity(args->affinity_before) != 0) + goto fail; + + pthread_mutex_lock(&test_mutex); + if (test_phase < args->ready_phase) + test_phase = args->ready_phase; + pthread_cond_broadcast(&test_cond); + + while (test_phase < AFFINITY_CONTROLLER_DISABLED) + pthread_cond_wait(&test_cond, &test_mutex); + pthread_mutex_unlock(&test_mutex); + + if (get_cpu_affinity(args->affinity_after) != 0) + goto fail; + + + return NULL; + +fail: + pthread_mutex_lock(&test_mutex); + test_phase = AFFINITY_ERROR; + pthread_cond_broadcast(&test_cond); + pthread_mutex_unlock(&test_mutex); + return NULL; +} + +/* + * Test that disabling cpuset controller properly updates thread affinity. + * + * This test exposes a bug in cpuset_attach() where threads in child cgroups + * don't get their affinity updated when the cpuset controller is disabled. + * + * Setup: + * - Create parent cgroup with cpuset.cpus=0-1 + * - Create child A with cpuset.cpus=0-1 + * - Create child B with cpuset.cpus=1 + * - Place multithreaded process: group leader + thread_a in A, thread_b in B + * - Disable cpuset controller on parent + * + * Expected: thread_b's affinity should expand from {1} to {0-1} + * Buggy: thread_b's affinity remains {1} + */ +static int test_cpuset_affinity_on_controller_disable(const char *root) +{ + char *parent = NULL, *child_a = NULL, *child_b = NULL; + pthread_t thread_a, thread_b; + int thread_a_created = 0, thread_b_created = 0; + cpu_set_t affinity_a_before, affinity_a_after; + cpu_set_t affinity_b_before, affinity_b_after; + int ret = KSFT_FAIL; + + parent = cg_name(root, "cpuset_affinity_test"); + if (!parent) + goto cleanup; + if (cg_create(parent)) + goto cleanup; + if (cg_write(parent, "cgroup.type", "threaded")) + goto cleanup; + + child_a = cg_name(parent, "A"); + if (!child_a) + goto cleanup; + if (cg_create(child_a)) + goto cleanup; + if (cg_write(child_a, "cgroup.type", "threaded")) + goto cleanup; + + child_b = cg_name(parent, "B"); + if (!child_b) + goto cleanup; + if (cg_create(child_b)) + goto cleanup; + if (cg_write(child_b, "cgroup.type", "threaded")) + goto cleanup; + + /* Now enable cpuset controller in parent */ + if (cg_write(parent, "cgroup.subtree_control", "+cpuset")) { + ret = KSFT_SKIP; + goto cleanup; + } + + /* Set CPU affinity constraints */ + if (cg_write(parent, "cpuset.cpus", "0-1")) + goto cleanup; + if (cg_write(child_a, "cpuset.cpus", "0-1")) + goto cleanup; + if (cg_write(child_b, "cpuset.cpus", "1")) + goto cleanup; + + /* Move group leader (main thread) to child A */ + if (cg_enter_current(child_a)) + goto cleanup; + + /* Create threads - they will move themselves to their respective cgroups */ + test_phase = AFFINITY_SETUP; + + struct thread_args args_a = { + .cgroup = child_a, + .affinity_before = &affinity_a_before, + .affinity_after = &affinity_a_after, + .ready_phase = AFFINITY_THREAD_A_READY, + }; + if (pthread_create(&thread_a, NULL, affinity_thread_fn, &args_a)) + goto cleanup; + thread_a_created = 1; + + struct thread_args args_b = { + .cgroup = child_b, + .affinity_before = &affinity_b_before, + .affinity_after = &affinity_b_after, + .ready_phase = AFFINITY_THREADS_READY, + }; + if (pthread_create(&thread_b, NULL, affinity_thread_fn, &args_b)) + goto cleanup_threads; + thread_b_created = 1; + + pthread_mutex_lock(&test_mutex); + while (test_phase < AFFINITY_THREADS_READY) + pthread_cond_wait(&test_cond, &test_mutex); + + /* If a thread failed during setup, bail out */ + if (test_phase == AFFINITY_ERROR) { + pthread_mutex_unlock(&test_mutex); + goto cleanup_threads; + } + pthread_mutex_unlock(&test_mutex); + + if (!cpu_set_equal(&affinity_a_before, 0x3)) { + ksft_print_msg("FAIL: thread_a initial affinity incorrect\n"); + goto cleanup_threads; + } + + if (!cpu_set_equal(&affinity_b_before, 0x2)) { + ksft_print_msg("FAIL: thread_b initial affinity incorrect\n"); + goto cleanup_threads; + } + + /* Disable cpuset controller - this should trigger affinity update */ + if (cg_write(parent, "cgroup.subtree_control", "-cpuset")) + goto cleanup_threads; + + /* Signal threads to save their final affinity and exit */ + pthread_mutex_lock(&test_mutex); + test_phase = AFFINITY_CONTROLLER_DISABLED; + pthread_cond_broadcast(&test_cond); + pthread_mutex_unlock(&test_mutex); + + pthread_join(thread_a, NULL); + pthread_join(thread_b, NULL); + + /* Verify thread affinities AFTER disabling controller */ + if (!cpu_set_equal(&affinity_a_after, 0x3)) { + ksft_print_msg("FAIL: thread_a final affinity incorrect\n"); + goto cleanup; + } + + if (!cpu_set_equal(&affinity_b_after, 0x3)) { + ksft_print_msg("FAIL: thread_b affinity did not expand to {0-1}\n"); + goto cleanup; + } + + ret = KSFT_PASS; + goto cleanup; + +cleanup_threads: + pthread_mutex_lock(&test_mutex); + test_phase = AFFINITY_COMPLETE; + pthread_cond_broadcast(&test_cond); + pthread_mutex_unlock(&test_mutex); + + if (thread_a_created) + pthread_join(thread_a, NULL); + if (thread_b_created) + pthread_join(thread_b, NULL); + +cleanup: + /* Move back to root before cleanup */ + cg_enter_current(root); + + cg_destroy(child_b); + free(child_b); + cg_destroy(child_a); + free(child_a); + cg_destroy(parent); + free(parent); + + return ret; +} + #define T(x) { x, #x } struct cpuset_test { @@ -241,6 +483,7 @@ struct cpuset_test { T(test_cpuset_perms_object_allow), T(test_cpuset_perms_object_deny), T(test_cpuset_perms_subtree), + T(test_cpuset_affinity_on_controller_disable), }; #undef T From 9133e0e2f7148044f0507211fd93724381f5122e Mon Sep 17 00:00:00 2001 From: Jian Guo Date: Fri, 17 Jul 2026 19:06:06 +0800 Subject: [PATCH 22/34] selftests/cgroup: Remove redundant cg_enter_current() call in test_core MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test_cgcore_no_internal_process_constraint_on_threads test has two back-to-back cg_enter_current(root) calls in its cleanup path. A single cg_enter_current() call atomically migrates the entire thread group to the target cgroup even for multi-threaded processes, and this test creates no extra threads or child processes that would require a second migration attempt. The second call is a harmless no-op once the process is already in the root cgroup, but it is redundant and inconsistent with the cleanup logic used in all other cgroup core selftest cases. Remove the duplicate call to clean up the code. No functional change is intended. Signed-off-by: Jian Guo Acked-by: Michal Koutný Tested-by: Tao Cui Signed-off-by: Tejun Heo --- tools/testing/selftests/cgroup/test_core.c | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/testing/selftests/cgroup/test_core.c b/tools/testing/selftests/cgroup/test_core.c index 88ca832d4fc1..8e3f9b391e44 100644 --- a/tools/testing/selftests/cgroup/test_core.c +++ b/tools/testing/selftests/cgroup/test_core.c @@ -426,7 +426,6 @@ static int test_cgcore_no_internal_process_constraint_on_threads(const char *roo ret = KSFT_PASS; cleanup: - cg_enter_current(root); cg_enter_current(root); if (child) cg_destroy(child); From 5da800b016a374a8d98ed270678ddaef3248a6ae Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Fri, 17 Jul 2026 14:02:25 +0800 Subject: [PATCH 23/34] Docs/admin-guide/cgroup-v2: fix delay_nsec unit in io.latency doc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The io.latency doc says the io.stat delay field counts microseconds. The field is delay_nsec and is reported in nanoseconds. Refer to it by its real name and correct the unit. Signed-off-by: Tao Cui Acked-by: Michal Koutný Signed-off-by: Tejun Heo --- Documentation/admin-guide/cgroup-v2.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 851a1cf1bc22..449f4897bbbb 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -2299,9 +2299,9 @@ This throttling takes 2 forms: throttled without possibly adversely affecting higher priority groups. This includes swapping and metadata IO. These types of IO are allowed to occur normally, however they are "charged" to the originating group. If the - originating group is being throttled you will see the use_delay and delay - fields in io.stat increase. The delay value is how many microseconds that are - being added to any process that runs in this group. Because this number can + originating group is being throttled you will see the use_delay and delay_nsec + fields in io.stat increase. The delay_nsec value is how many nanoseconds that + are being added to any process that runs in this group. Because this number can grow quite large if there is a lot of swapping or metadata IO occurring we limit the individual delay events to 1 second at a time. From d56281fefb19e9b3c1cabec303386f5bd8660106 Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Fri, 17 Jul 2026 15:18:14 -0400 Subject: [PATCH 24/34] selftests/cgroup: Fix minor defects in test_cpuset With commit 98149f542530 ("selftests/cgroup: Add test for cpuset affinity on controller disable"), sashiko [1] reported 3 different issues with the new test_cpuset_affinity_on_controller_disable() test. 1) `cpu_set_equal` iterates over mask bytes instead of bits, ignoring CPUs >= 8. 2) Thread synchronization logic allows the main thread to read uninitialized stack memory, causing test flakiness. 3) Test fails instead of skipping gracefully on uniprocessor systems or when CPU 1 is unavailable. Fix the reported issues by: 1) Iterates over the bit size of the mask. 2) Test the new ready flag for each thread to end the wait on the conditional variable and eliminate the now unneeded AFFINITY_THREAD_A_READY and AFFINITY_THREADS_READY test phases. 3) Return KSFT_SKIP on "cpuset.cpus" setting failure. [1] https://sashiko.dev/#/patchset/20260712235510.373125-1-longman%40redhat.com Fixes: 98149f542530 ("selftests/cgroup: Add test for cpuset affinity on controller disable") Signed-off-by: Waiman Long Signed-off-by: Tejun Heo --- tools/testing/selftests/cgroup/test_cpuset.c | 38 +++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/tools/testing/selftests/cgroup/test_cpuset.c b/tools/testing/selftests/cgroup/test_cpuset.c index 8b4c4a9dd78b..8c2d4d4ef1fc 100644 --- a/tools/testing/selftests/cgroup/test_cpuset.c +++ b/tools/testing/selftests/cgroup/test_cpuset.c @@ -251,7 +251,7 @@ static int cpu_set_equal(cpu_set_t *dst, unsigned long mask) CPU_ZERO(&expected); assert(sizeof(mask) < CPU_SETSIZE); - for (int cpu = 0; cpu < sizeof(mask); ++cpu) + for (int cpu = 0; cpu < sizeof(mask) * 8; ++cpu) if ((1UL << cpu) & mask) CPU_SET(cpu, &expected); @@ -260,8 +260,6 @@ static int cpu_set_equal(cpu_set_t *dst, unsigned long mask) enum test_phase { AFFINITY_SETUP, - AFFINITY_THREAD_A_READY, - AFFINITY_THREADS_READY, AFFINITY_CONTROLLER_DISABLED, AFFINITY_COMPLETE, AFFINITY_ERROR @@ -271,7 +269,7 @@ struct thread_args { const char *cgroup; cpu_set_t *affinity_before; cpu_set_t *affinity_after; - enum test_phase ready_phase; + int affinity_before_ready; }; static pthread_mutex_t test_mutex = PTHREAD_MUTEX_INITIALIZER; @@ -289,8 +287,7 @@ static void *affinity_thread_fn(void *arg) goto fail; pthread_mutex_lock(&test_mutex); - if (test_phase < args->ready_phase) - test_phase = args->ready_phase; + args->affinity_before_ready = 1; pthread_cond_broadcast(&test_cond); while (test_phase < AFFINITY_CONTROLLER_DISABLED) @@ -361,18 +358,20 @@ static int test_cpuset_affinity_on_controller_disable(const char *root) goto cleanup; /* Now enable cpuset controller in parent */ - if (cg_write(parent, "cgroup.subtree_control", "+cpuset")) { - ret = KSFT_SKIP; - goto cleanup; - } + if (cg_write(parent, "cgroup.subtree_control", "+cpuset")) + goto skip; - /* Set CPU affinity constraints */ + /* + * Set CPU affinity constraints + * Skip the test if the setting of "cpuset.cpus" fails as the test + * system may not have CPU 1. + */ if (cg_write(parent, "cpuset.cpus", "0-1")) - goto cleanup; + goto skip; if (cg_write(child_a, "cpuset.cpus", "0-1")) - goto cleanup; + goto skip; if (cg_write(child_b, "cpuset.cpus", "1")) - goto cleanup; + goto skip; /* Move group leader (main thread) to child A */ if (cg_enter_current(child_a)) @@ -385,7 +384,7 @@ static int test_cpuset_affinity_on_controller_disable(const char *root) .cgroup = child_a, .affinity_before = &affinity_a_before, .affinity_after = &affinity_a_after, - .ready_phase = AFFINITY_THREAD_A_READY, + .affinity_before_ready = 0, }; if (pthread_create(&thread_a, NULL, affinity_thread_fn, &args_a)) goto cleanup; @@ -395,14 +394,15 @@ static int test_cpuset_affinity_on_controller_disable(const char *root) .cgroup = child_b, .affinity_before = &affinity_b_before, .affinity_after = &affinity_b_after, - .ready_phase = AFFINITY_THREADS_READY, + .affinity_before_ready = 0, }; if (pthread_create(&thread_b, NULL, affinity_thread_fn, &args_b)) goto cleanup_threads; thread_b_created = 1; pthread_mutex_lock(&test_mutex); - while (test_phase < AFFINITY_THREADS_READY) + while ((test_phase < AFFINITY_ERROR) && + (args_a.affinity_before_ready + args_b.affinity_before_ready < 2)) pthread_cond_wait(&test_cond, &test_mutex); /* If a thread failed during setup, bail out */ @@ -449,6 +449,10 @@ static int test_cpuset_affinity_on_controller_disable(const char *root) ret = KSFT_PASS; goto cleanup; +skip: + ret = KSFT_SKIP; + goto cleanup; + cleanup_threads: pthread_mutex_lock(&test_mutex); test_phase = AFFINITY_COMPLETE; From 3864c95587a4b487cf231ec0ab8c3d1c11764eb7 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Fri, 24 Jul 2026 10:05:08 +0800 Subject: [PATCH 25/34] docs: cgroup-v2: mark memory.pressure and io.pressure as read-write The cgroup-v2 documentation describes memory.pressure and io.pressure as "read-only nested-keyed file", but both files accept trigger writes (cgroup_memory_pressure_write / cgroup_io_pressure_write) and are therefore read-write. cpu.pressure and irq.pressure are already documented as read-write, so this also resolves an internal inconsistency. Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- Documentation/admin-guide/cgroup-v2.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 449f4897bbbb..60fc6afe029e 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -1930,7 +1930,7 @@ The following nested keys are defined. is allowed unless memory.swap.max is set to 0. memory.pressure - A read-only nested-keyed file. + A read-write nested-keyed file. Shows pressure stall information for memory. See :ref:`Documentation/accounting/psi.rst ` for details. @@ -2190,7 +2190,7 @@ IO Interface Files 8:16 rbps=2097152 wbps=max riops=max wiops=max io.pressure - A read-only nested-keyed file. + A read-write nested-keyed file. Shows pressure stall information for IO. See :ref:`Documentation/accounting/psi.rst ` for details. From ae649c9636244a7529a1559c143d4ce9c5da62fa Mon Sep 17 00:00:00 2001 From: Julia Lawall Date: Sat, 1 Aug 2026 21:09:55 +0200 Subject: [PATCH 26/34] cgroup: drop unneeded semicolon The trailing semicolon belongs at the point of use, not in the macro definition. All uses have been verified to have their own semicolons. This was found using the following Coccinelle semantic patch: @r@ identifier i : script:ocaml() { String.lowercase_ascii i = i }; expression e; @@ *#define i(...) e; Signed-off-by: Julia Lawall Signed-off-by: Tejun Heo --- kernel/cgroup/cgroup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 38f8d9df8fbc..f87fc4550081 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -104,7 +104,7 @@ DEFINE_PERCPU_RWSEM(cgroup_threadgroup_rwsem); #define cgroup_assert_mutex_or_rcu_locked() \ RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \ !lockdep_is_held(&cgroup_mutex), \ - "cgroup_mutex or RCU read lock required"); + "cgroup_mutex or RCU read lock required") /* * cgroup destruction makes heavy use of work items and there can be a lot From 77bc7e952f660a7bab9741fe352f960f3675a650 Mon Sep 17 00:00:00 2001 From: Shaojie Sun Date: Thu, 30 Jul 2026 21:07:17 +0800 Subject: [PATCH 27/34] selftests/cgroup: add user_usec sanity check in test_cpucg_nice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In test_cpucg_nice, after the child process exits, user_usec is read from cpu.stat but the value is not checked. Add a sanity check to ensure user_usec > 0, analogous to test_cpucg_stats(), so that the test fails early if CPU usage wasn't properly accounted. Signed-off-by: Shaojie Sun Reviewed-by: Michal Koutný Acked-by: Tao Cui Signed-off-by: Tejun Heo --- tools/testing/selftests/cgroup/test_cpu.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/testing/selftests/cgroup/test_cpu.c b/tools/testing/selftests/cgroup/test_cpu.c index a5eccfcabef5..f9f7017d9299 100644 --- a/tools/testing/selftests/cgroup/test_cpu.c +++ b/tools/testing/selftests/cgroup/test_cpu.c @@ -291,6 +291,8 @@ static int test_cpucg_nice(const char *root) user_usec = cg_read_key_long(cpucg, "cpu.stat", "user_usec"); nice_usec = cg_read_key_long(cpucg, "cpu.stat", "nice_usec"); + if (user_usec <= 0) + goto cleanup; if (!values_close_report(nice_usec, expected_nice_usec, 1)) goto cleanup; From 26d3a59e0241c3ef9f66ae1ae990f9326acbc059 Mon Sep 17 00:00:00 2001 From: Guopeng Zhang Date: Mon, 10 Aug 2026 17:55:59 +0800 Subject: [PATCH 28/34] cgroup/cpuset: Use WRITE_ONCE() for shared prs_err updates cpuset_partition_show() reads cs->prs_err without cpuset_mutex using READ_ONCE(). The field is documented as not lock protected, but several updates to live cpusets still use plain stores. Convert the remaining prs_err stores on live cpusets to WRITE_ONCE(). Fixes: 0c7f293efc87 ("cgroup/cpuset: Add cpuset.cpus.exclusive.effective for v2") Assisted-by: LLM Signed-off-by: Guopeng Zhang Reviewed-by: Waiman Long Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 8690d987d746..550812e7c274 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -1587,7 +1587,7 @@ static int remote_partition_enable(struct cpuset *cs, int new_prs, cpumask_copy(cs->effective_xcpus, tmp->new_cpus); spin_unlock_irq(&callback_lock); cpuset_force_rebuild(); - cs->prs_err = 0; + WRITE_ONCE(cs->prs_err, 0); /* * Propagate changes in top_cpuset's effective_cpus down the hierarchy. @@ -1661,7 +1661,7 @@ static void remote_cpus_update(struct cpuset *cs, struct cpumask *xcpus, WARN_ON_ONCE(!cpumask_subset(cs->effective_xcpus, subpartitions_cpus)); if (cpumask_empty(excpus)) { - cs->prs_err = PERR_CPUSEMPTY; + WRITE_ONCE(cs->prs_err, PERR_CPUSEMPTY); goto invalidate; } @@ -1676,13 +1676,13 @@ static void remote_cpus_update(struct cpuset *cs, struct cpumask *xcpus, if (adding) { WARN_ON_ONCE(cpumask_intersects(tmp->addmask, subpartitions_cpus)); if (!capable(CAP_SYS_ADMIN)) - cs->prs_err = PERR_ACCESS; + WRITE_ONCE(cs->prs_err, PERR_ACCESS); else if (cpumask_intersects(tmp->addmask, subpartitions_cpus) || cpumask_subset(top_cpuset.effective_cpus, tmp->addmask)) - cs->prs_err = PERR_NOCPUS; + WRITE_ONCE(cs->prs_err, PERR_NOCPUS); else if ((prs == PRS_ISOLATED) && !isolated_cpus_can_update(tmp->addmask, tmp->delmask)) - cs->prs_err = PERR_HKEEPING; + WRITE_ONCE(cs->prs_err, PERR_HKEEPING); if (cs->prs_err) goto invalidate; } @@ -2110,13 +2110,13 @@ static void compute_partition_effective_cpumask(struct cpuset *cs, * partition root. */ WARN_ON_ONCE(is_remote_partition(child)); - child->prs_err = 0; + WRITE_ONCE(child->prs_err, 0); if (!cpumask_subset(child->effective_xcpus, cs->effective_xcpus)) - child->prs_err = PERR_INVCPUS; + WRITE_ONCE(child->prs_err, PERR_INVCPUS); else if (populated && cpumask_subset(new_ecpus, child->effective_xcpus)) - child->prs_err = PERR_NOCPUS; + WRITE_ONCE(child->prs_err, PERR_NOCPUS); if (child->prs_err) { int old_prs = child->partition_root_state; @@ -2420,8 +2420,10 @@ static void partition_cpus_change(struct cpuset *cs, struct cpuset *trialcs, return; prs_err = validate_partition(cs, trialcs); - if (prs_err) - trialcs->prs_err = cs->prs_err = prs_err; + if (prs_err) { + WRITE_ONCE(cs->prs_err, prs_err); + trialcs->prs_err = prs_err; + } if (is_remote_partition(cs)) { if (trialcs->prs_err) @@ -3939,7 +3941,7 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp) if (remote && (cpumask_empty(subpartitions_cpus) || (cpumask_empty(&new_cpus) && partition_is_populated(cs, NULL)))) { - cs->prs_err = PERR_HOTPLUG; + WRITE_ONCE(cs->prs_err, PERR_HOTPLUG); remote_partition_disable(cs, tmp); compute_effective_cpumask(&new_cpus, cs, parent); remote = false; From 75e79a20190040800445ec95a4c4a84193d16992 Mon Sep 17 00:00:00 2001 From: Rui Qi Date: Sun, 9 Aug 2026 16:21:10 +0800 Subject: [PATCH 29/34] selftests/cgroup: Avoid awk -e in cpuset tests The cpuset selftests use awk -e to parse cgroup mount points. This works with gawk, but mawk rejects the option. In test_cpuset_prs.sh, this leaves CGROUP2 empty and causes the test to skip as if cgroup v2 were not mounted. The same non-portable invocation exists in the cpuset v1 hotplug test. The scripts only need to pass a single awk program. Use the standard awk invocation without -e so mount point detection works with awk implementations that do not support the gawk extension. Fixes: a8c52eba880a ("kselftest/cgroup: Add cpuset v2 partition root state test") Fixes: 812c5945bdb8 ("cgroup/cpuset: Add test_cpuset_v1_hp.sh") Signed-off-by: Rui Qi Acked-by: Waiman Long Reviewed-by: Ridong Chen Signed-off-by: Tejun Heo --- tools/testing/selftests/cgroup/test_cpuset_prs.sh | 2 +- tools/testing/selftests/cgroup/test_cpuset_v1_hp.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/cgroup/test_cpuset_prs.sh b/tools/testing/selftests/cgroup/test_cpuset_prs.sh index ca9bc38fdb95..8e21e9fc439c 100755 --- a/tools/testing/selftests/cgroup/test_cpuset_prs.sh +++ b/tools/testing/selftests/cgroup/test_cpuset_prs.sh @@ -20,7 +20,7 @@ skip_test() { WAIT_INOTIFY=$(cd $(dirname $0); pwd)/wait_inotify # Find cgroup v2 mount point -CGROUP2=$(mount -t cgroup2 | head -1 | awk -e '{print $3}') +CGROUP2=$(mount -t cgroup2 | head -1 | awk '{print $3}') [[ -n "$CGROUP2" ]] || skip_test "Cgroup v2 mount point not found!" SUBPARTS_CPUS=$CGROUP2/.__DEBUG__.cpuset.cpus.subpartitions CPULIST=$(cat $CGROUP2/cpuset.cpus.effective) diff --git a/tools/testing/selftests/cgroup/test_cpuset_v1_hp.sh b/tools/testing/selftests/cgroup/test_cpuset_v1_hp.sh index 7406c24be1ac..da97f1643f9a 100755 --- a/tools/testing/selftests/cgroup/test_cpuset_v1_hp.sh +++ b/tools/testing/selftests/cgroup/test_cpuset_v1_hp.sh @@ -14,7 +14,7 @@ skip_test() { [[ $(id -u) -eq 0 ]] || skip_test "Test must be run as root!" # Find cpuset v1 mount point -CPUSET=$(mount -t cgroup | grep cpuset | head -1 | awk -e '{print $3}') +CPUSET=$(mount -t cgroup | grep cpuset | head -1 | awk '{print $3}') [[ -n "$CPUSET" ]] || skip_test "cpuset v1 mount point not found!" # From 3345c7248a9e2a91cb9e2477c29af16881706f4f Mon Sep 17 00:00:00 2001 From: Zhe Liu Date: Mon, 10 Aug 2026 14:35:35 +0800 Subject: [PATCH 30/34] docs: cgroup-v2: fix stale "io" controller introduction The introductory paragraph for the IO controller still states that weight based distribution is "available only if cfq-iosched is in use" and that "neither scheme is available for blk-mq devices". This text dates from when the cgroup v2 documentation was first written (2015) and was correct at the time, but is no longer accurate: * cfq-iosched was removed in v5.0; * blk-mq is now the only block I/O path, and both the absolute limit scheme (io.max via blk-throttle) and the weight based scheme (io.weight via iocost, or io.bfq.weight under BFQ) work on it; * latency based protection (iolatency) and I/O priority (ioprio) controllers have since been added. The rest of the section already documents io.weight, io.max, io.cost.{qos,model}, io.latency and io.prio.class correctly, so the introduction is the only part that contradicts them. Rewrite it to reflect the current state. Signed-off-by: Zhe Liu Reviewed-by: Tao Cui Signed-off-by: Tejun Heo --- Documentation/admin-guide/cgroup-v2.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 60fc6afe029e..0a4f4eb15626 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -2005,9 +2005,13 @@ IO The "io" controller regulates the distribution of IO resources. This controller implements both weight based and absolute bandwidth or IOPS -limit distribution; however, weight based distribution is available -only if cfq-iosched is in use and neither scheme is available for -blk-mq devices. +limit distribution. Absolute BPS and IOPS limits are enforced by +blk-throttle and apply to all devices, while weight based proportional +distribution is provided by the iocost cost model controller +(CONFIG_BLK_CGROUP_IOCOST) and, when the BFQ I/O scheduler is in use +for a device, by BFQ's own cgroup support. Latency-based protection +(CONFIG_BLK_CGROUP_IOLATENCY) and I/O priority assignment +(CONFIG_BLK_CGROUP_IOPRIO) are also available. IO Interface Files From 6dd5d93f6c48a461157c73b9e2ce7f6d19d57f5a Mon Sep 17 00:00:00 2001 From: Guopeng Zhang Date: Tue, 11 Aug 2026 16:30:27 +0800 Subject: [PATCH 31/34] cgroup/cpuset: Remove obsolete PFA_SPREAD_SLAB task flag Commit 16a1d968358a ("mm/slab: remove mm/slab.c and slab_def.h") removed the SLAB allocator, the only allocator that implemented cpuset slab spreading. Commit 61a182ab61a6 ("cgroup/cpuset: Remove cpuset_do_slab_mem_spread()") then removed the last task_spread_slab() caller. Commit 3ab67a9ce82f ("cgroup/cpuset: Mark memory_spread_slab as obsolete") marked the legacy control obsolete. cpuset still updates PFA_SPREAD_SLAB when tasks attach to a legacy cpuset and walks all tasks in a cpuset when memory_spread_slab changes. Remove the unused task flag and its helpers, and make spread task updates depend only on memory_spread_page. Keep the memory_spread_slab control and CS_SPREAD_SLAB state so legacy users retain the existing write, readback and inheritance behavior. Update the comments and documentation to describe only page-cache spreading as functional. Assisted-by: LLM Signed-off-by: Guopeng Zhang Reviewed-by: Waiman Long Signed-off-by: Tejun Heo --- .../admin-guide/cgroup-v1/cpusets.rst | 42 +++++++------------ include/linux/sched.h | 5 --- kernel/cgroup/cpuset-v1.c | 13 ++---- kernel/cgroup/cpuset.c | 19 ++++----- 4 files changed, 27 insertions(+), 52 deletions(-) diff --git a/Documentation/admin-guide/cgroup-v1/cpusets.rst b/Documentation/admin-guide/cgroup-v1/cpusets.rst index c7909e5ac136..8c8cda35be5e 100644 --- a/Documentation/admin-guide/cgroup-v1/cpusets.rst +++ b/Documentation/admin-guide/cgroup-v1/cpusets.rst @@ -179,7 +179,7 @@ files describing that cpuset: - cpuset.mem_hardwall flag: is memory allocation hardwalled - cpuset.memory_pressure: measure of how much paging pressure in cpuset - cpuset.memory_spread_page flag: if set, spread page cache evenly on allowed nodes - - cpuset.memory_spread_slab flag: OBSOLETE. Doesn't have any function. + - cpuset.memory_spread_slab flag: OBSOLETE. Has no effect on allocation behavior. - cpuset.sched_load_balance flag: if set, load balance within CPUs on that cpuset - cpuset.sched_relax_domain_level: the searching range when migrating tasks @@ -318,26 +318,20 @@ times 1000. 1.6 What is memory spread ? --------------------------- -There are two boolean flag files per cpuset that control where the -kernel allocates pages for the file system buffers and related in -kernel data structures. They are called 'cpuset.memory_spread_page' and -'cpuset.memory_spread_slab'. +The 'cpuset.memory_spread_page' boolean flag file controls where the kernel +allocates page-cache pages. +The 'cpuset.memory_spread_slab' file is obsolete and has no effect on +allocation behavior, but is retained for compatibility. If the per-cpuset boolean flag file 'cpuset.memory_spread_page' is set, then the kernel will spread the file system buffers (page cache) evenly over all the nodes that the faulting task is allowed to use, instead of preferring to put those pages on the node where the task is running. -If the per-cpuset boolean flag file 'cpuset.memory_spread_slab' is set, -then the kernel will spread some file system related slab caches, -such as for inodes and dentries evenly over all the nodes that the -faulting task is allowed to use, instead of preferring to put those -pages on the node where the task is running. - -The setting of these flags does not affect anonymous data segment or +The setting of this flag does not affect anonymous data segment or stack segment pages of a task. -By default, both kinds of memory spreading are off, and memory +By default, page cache memory spreading is off, and memory pages are allocated on the node local to where the task is running, except perhaps as modified by the task's NUMA mempolicy or cpuset configuration, so long as sufficient free memory pages are available. @@ -345,18 +339,18 @@ configuration, so long as sufficient free memory pages are available. When new cpusets are created, they inherit the memory spread settings of their parent. -Setting memory spreading causes allocations for the affected page -or slab caches to ignore the task's NUMA mempolicy and be spread -instead. Tasks using mbind() or set_mempolicy() calls to set NUMA -mempolicies will not notice any change in these calls as a result of -their containing task's memory spread settings. If memory spreading +Setting page cache memory spreading causes affected allocations to ignore the +task's NUMA mempolicy and be spread instead. Tasks using mbind() or +set_mempolicy() to set NUMA mempolicies will not notice any change as a +result of their containing task's memory spread settings. If memory spreading is turned off, then the currently specified NUMA mempolicy once again applies to memory page allocations. -Both 'cpuset.memory_spread_page' and 'cpuset.memory_spread_slab' are boolean flag -files. By default they contain "0", meaning that the feature is off -for that cpuset. If a "1" is written to that file, then that turns -the named feature on. +Both 'cpuset.memory_spread_page' and 'cpuset.memory_spread_slab' are boolean +flag files. In the root cpuset, both files initially contain "0". Writing "1" +or "0" to 'cpuset.memory_spread_page' enables or disables page-cache spreading, +respectively. The value of 'cpuset.memory_spread_slab' is retained, can be read +back and inherited, but it does not affect allocation behavior. The implementation is simple. @@ -367,10 +361,6 @@ is modified to perform an inline check for this PFA_SPREAD_PAGE task flag, and if set, a call to a new routine cpuset_mem_spread_node() returns the node to prefer for the allocation. -Similarly, setting 'cpuset.memory_spread_slab' turns on the flag -PFA_SPREAD_SLAB, and appropriately marked slab caches will allocate -pages from the node returned by cpuset_mem_spread_node(). - The cpuset_mem_spread_node() routine is also simple. It uses the value of a per-task rotor cpuset_mem_spread_rotor to select the next node in the current task's mems_allowed to prefer for the allocation. diff --git a/include/linux/sched.h b/include/linux/sched.h index 373bcc0598d1..40d6010be227 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1849,7 +1849,6 @@ static __always_inline bool is_user_task(struct task_struct *task) /* Per-process atomic flags. */ #define PFA_NO_NEW_PRIVS 0 /* May not gain new privileges. */ #define PFA_SPREAD_PAGE 1 /* Spread page cache over cpuset */ -#define PFA_SPREAD_SLAB 2 /* Spread some slab caches over cpuset */ #define PFA_SPEC_SSB_DISABLE 3 /* Speculative Store Bypass disabled */ #define PFA_SPEC_SSB_FORCE_DISABLE 4 /* Speculative Store Bypass force disabled*/ #define PFA_SPEC_IB_DISABLE 5 /* Indirect branch speculation restricted */ @@ -1875,10 +1874,6 @@ TASK_PFA_TEST(SPREAD_PAGE, spread_page) TASK_PFA_SET(SPREAD_PAGE, spread_page) TASK_PFA_CLEAR(SPREAD_PAGE, spread_page) -TASK_PFA_TEST(SPREAD_SLAB, spread_slab) -TASK_PFA_SET(SPREAD_SLAB, spread_slab) -TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab) - TASK_PFA_TEST(SPEC_SSB_DISABLE, spec_ssb_disable) TASK_PFA_SET(SPEC_SSB_DISABLE, spec_ssb_disable) TASK_PFA_CLEAR(SPEC_SSB_DISABLE, spec_ssb_disable) diff --git a/kernel/cgroup/cpuset-v1.c b/kernel/cgroup/cpuset-v1.c index 3e9968dd91e9..562ad35f00d0 100644 --- a/kernel/cgroup/cpuset-v1.c +++ b/kernel/cgroup/cpuset-v1.c @@ -204,7 +204,7 @@ static s64 cpuset_read_s64(struct cgroup_subsys_state *css, struct cftype *cft) } /* - * update task's spread flag if cpuset's page/slab spread flag is set + * Update a task's spread flag if the cpuset's page spread flag is set. * * Call with callback_lock or cpuset_mutex held. The check can be skipped * if on default hierarchy. @@ -219,18 +219,13 @@ void cpuset1_update_task_spread_flags(struct cpuset *cs, task_set_spread_page(tsk); else task_clear_spread_page(tsk); - - if (is_spread_slab(cs)) - task_set_spread_slab(tsk); - else - task_clear_spread_slab(tsk); } /** - * cpuset1_update_tasks_flags - update the spread flags of tasks in the cpuset. - * @cs: the cpuset in which each task's spread flags needs to be changed + * cpuset1_update_tasks_flags - update the page spread flag of cpuset tasks + * @cs: the cpuset whose tasks need their page spread flag updated * - * Iterate through each task of @cs updating its spread flags. As this + * Iterate through each task of @cs updating its page spread flag. As this * function is called with cpuset_mutex held, cpuset membership stays * stable. */ diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 550812e7c274..0c34013eda8e 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -2859,7 +2859,7 @@ int cpuset_update_flag(cpuset_flagbits_t bit, struct cpuset *cs, { struct cpuset *trialcs; int balance_flag_changed; - int spread_flag_changed; + int spread_page_changed; int err; trialcs = dup_or_alloc_cpuset(cs); @@ -2878,8 +2878,7 @@ int cpuset_update_flag(cpuset_flagbits_t bit, struct cpuset *cs, balance_flag_changed = (is_sched_load_balance(cs) != is_sched_load_balance(trialcs)); - spread_flag_changed = ((is_spread_slab(cs) != is_spread_slab(trialcs)) - || (is_spread_page(cs) != is_spread_page(trialcs))); + spread_page_changed = is_spread_page(cs) != is_spread_page(trialcs); spin_lock_irq(&callback_lock); cs->flags = trialcs->flags; @@ -2892,7 +2891,7 @@ int cpuset_update_flag(cpuset_flagbits_t bit, struct cpuset *cs, rebuild_sched_domains_locked(); } - if (spread_flag_changed) + if (spread_page_changed) cpuset1_update_tasks_flags(cs); out: free_cpuset(trialcs); @@ -4462,14 +4461,10 @@ void cpuset_nodes_allowed(struct cgroup *cgroup, nodemask_t *mask) * cpuset_spread_node() - On which node to begin search for a page * @rotor: round robin rotor * - * If a task is marked PF_SPREAD_PAGE or PF_SPREAD_SLAB (as for - * tasks in a cpuset with is_spread_page or is_spread_slab set), - * and if the memory allocation used cpuset_mem_spread_node() - * to determine on which node to start looking, as it will for - * certain page cache or slab cache pages such as used for file - * system buffers and inode caches, then instead of starting on the - * local node to look for a free page, rather spread the starting - * node around the tasks mems_allowed nodes. + * If a task is marked PFA_SPREAD_PAGE and a page cache allocation uses + * cpuset_mem_spread_node() to determine where to start looking, spread the + * starting node around the task's mems_allowed nodes instead of starting on + * the local node. * * We don't have to worry about the returned node being offline * because "it can't happen", and even if it did, it would be ok. From 44f57a2b1b9fabe3735a9b2669bb146ddb571fce Mon Sep 17 00:00:00 2001 From: Shaojie Sun Date: Tue, 11 Aug 2026 17:58:48 +0800 Subject: [PATCH 32/34] cgroup/cpuset: Add test for partition root invalidation returning wrong CPUs Add a test case to REMOTE_TEST_MATRIX covering the bug fixed by commit 345f40166694 ("cgroup/cpuset: Return only actually allocated CPUs during partition invalidation"). The test verifies that when a sibling partition root changes its cpuset.cpus to overlap with another partition root, only actually allocated CPUs (effective_xcpus) are returned to the parent, not all CPUs in cpus_allowed. Signed-off-by: Shaojie Sun Reviewed-by: Waiman Long Signed-off-by: Tejun Heo --- tools/testing/selftests/cgroup/test_cpuset_prs.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/testing/selftests/cgroup/test_cpuset_prs.sh b/tools/testing/selftests/cgroup/test_cpuset_prs.sh index 8e21e9fc439c..8380fd9d76bd 100755 --- a/tools/testing/selftests/cgroup/test_cpuset_prs.sh +++ b/tools/testing/selftests/cgroup/test_cpuset_prs.sh @@ -502,6 +502,12 @@ REMOTE_TEST_MATRIX=( " C1-4:P1 . C1-2:P1 C1-3:P2 . . \ . . P0 C2-3 . . p1:1,4|c11:1|c12:2-3 \ p1:P1|c11:P0|c12:P2 2-3" + # Changing a sibling partition's cpuset.cpus to overlap with another + # sibling partition should invalidate itself and return only actually + # allocated CPUs (effective_xcpus) to the parent. + " C1-4:P1 . C1-2:P1 C2-4:P2 . . \ + . . . C1-2 . . p1:3-4|c11:1-2|c12:3-4 \ + p1:P1|c11:P1|c12:P-2" # Cpusets with empty cpuset.cpus should inherit parent's effective_cpus " C1-4:P1 C5-6 C1-2 . C5 . \ . P1 P1 . . . p1:3-4|p2:5-6|c11:1-2|c12:3-4|c21:5|c22:5-6 \ From ddabc5dbd262f3ca981e679ddae99ed7a9adb279 Mon Sep 17 00:00:00 2001 From: Rui Qi Date: Thu, 13 Aug 2026 21:28:07 +0800 Subject: [PATCH 33/34] selftests/cgroup: Preserve CPU hotplug write errors The cpuset partition root state selftest checks several CPU hotplug transitions. If writing to a CPU online file fails, the helper still runs pause afterwards and returns the status of pause instead of the failed write. This hides the real hotplug failure and can make later checks run against expectations for a transition that never happened. Move the write before the bookkeeping and return when it fails, so callers can observe the hotplug error and the test does not record a CPU as offline unless the offline operation actually succeeded. Also change the O* command handler in set_ctrl_state() to use "eval $COMM $REDIRECT" like all other handlers. The previous version set COMM but still called write_cpu_online directly, bypassing the redirect that captures stderr for error reporting. Changes since v1: - Use eval $COMM $REDIRECT in the O* handler instead of calling write_cpu_online directly (Waiman Long) Fixes: a8c52eba880a ("kselftest/cgroup: Add cpuset v2 partition root state test") Signed-off-by: Rui Qi Signed-off-by: Tejun Heo --- tools/testing/selftests/cgroup/test_cpuset_prs.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/cgroup/test_cpuset_prs.sh b/tools/testing/selftests/cgroup/test_cpuset_prs.sh index 8380fd9d76bd..da8f7b920178 100755 --- a/tools/testing/selftests/cgroup/test_cpuset_prs.sh +++ b/tools/testing/selftests/cgroup/test_cpuset_prs.sh @@ -526,6 +526,7 @@ write_cpu_online() CPU=${1%=*} VAL=${1#*=} CPUFILE=//sys/devices/system/cpu/cpu${CPU}/online + echo $VAL > $CPUFILE || return 1 if [[ $VAL -eq 0 ]] then OFFLINE_CPUS="$OFFLINE_CPUS $CPU" @@ -535,7 +536,6 @@ write_cpu_online() sort | uniq -u) } fi - echo $VAL > $CPUFILE pause 0.05 } @@ -603,7 +603,8 @@ set_ctrl_state() eval $COMM $REDIRECT ;; O*) VAL=${CMD#?} - write_cpu_online $VAL + COMM="write_cpu_online $VAL" + eval $COMM $REDIRECT ;; T*) COMM="echo 0 > $TFILE" eval $COMM $REDIRECT From 2d19207f3fc8e08bab3270af2e3835358ab1473a Mon Sep 17 00:00:00 2001 From: Shaojie Sun Date: Fri, 14 Aug 2026 18:00:02 +0800 Subject: [PATCH 34/34] selftests/cgroup: Remove redundant chown in test_cgcore_lesser_ns_open test_cgcore_lesser_ns_open runs as root throughout and never changes its euid, so chowning the two cgroup.procs files to a non-root uid has no effect on the test. The ENOENT the test expects comes from the cgroup namespace delegation check in cgroup_procs_write_permission(): the source and destination cgroups must both be descendants of the namespace root captured at open time. That check does not depend on file ownership. In addition, the permission check only examines the common ancestor's cgroup.procs file (the test root here), which the chown calls do not touch. Remove the redundant chown calls and the now unused test_euid and cg_test_a_procs variables. Signed-off-by: Shaojie Sun Reviewed-by: Tao Cui Signed-off-by: Tejun Heo --- tools/testing/selftests/cgroup/test_core.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tools/testing/selftests/cgroup/test_core.c b/tools/testing/selftests/cgroup/test_core.c index 8e3f9b391e44..e9bee164bb70 100644 --- a/tools/testing/selftests/cgroup/test_core.c +++ b/tools/testing/selftests/cgroup/test_core.c @@ -794,10 +794,9 @@ static int lesser_ns_open_thread_fn(void *arg) static int test_cgcore_lesser_ns_open(const char *root) { static char stack[65536]; - const uid_t test_euid = 65534; /* usually nobody, any !root is fine */ int ret = KSFT_FAIL; char *cg_test_a = NULL, *cg_test_b = NULL; - char *cg_test_a_procs = NULL, *cg_test_b_procs = NULL; + char *cg_test_b_procs = NULL; int cg_test_b_procs_fd = -1; struct lesser_ns_open_thread_arg targ = { .fd = -1 }; pid_t pid; @@ -812,10 +811,9 @@ static int test_cgcore_lesser_ns_open(const char *root) if (!cg_test_a || !cg_test_b) goto cleanup; - cg_test_a_procs = cg_name(cg_test_a, "cgroup.procs"); cg_test_b_procs = cg_name(cg_test_b, "cgroup.procs"); - if (!cg_test_a_procs || !cg_test_b_procs) + if (!cg_test_b_procs) goto cleanup; if (cg_create(cg_test_a) || cg_create(cg_test_b)) @@ -824,10 +822,6 @@ static int test_cgcore_lesser_ns_open(const char *root) if (cg_enter_current(cg_test_b)) goto cleanup; - if (chown(cg_test_a_procs, test_euid, -1) || - chown(cg_test_b_procs, test_euid, -1)) - goto cleanup; - targ.path = cg_test_b_procs; pid = clone(lesser_ns_open_thread_fn, stack + sizeof(stack), CLONE_NEWCGROUP | CLONE_FILES | CLONE_VM | SIGCHLD, @@ -862,7 +856,6 @@ static int test_cgcore_lesser_ns_open(const char *root) if (cg_test_a) cg_destroy(cg_test_a); free(cg_test_b_procs); - free(cg_test_a_procs); free(cg_test_b); free(cg_test_a); return ret;