From 5bbb3453e8f35e1eacde30459e1fa7625c8d6d21 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Wed, 22 Apr 2026 10:17:09 +0800 Subject: [PATCH 01/22] cgroup/rdma: refactor resource parsing with match_table_t/match_token() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the hand-rolled strsep/strcmp/match_string parsing in rdmacg_resource_set_max() with a match_table_t and match_token() pattern, following the convention used by user_proactive_reclaim() and ioc_cost_model_write(). The old strncmp(value, RDMACG_MAX_STR, strlen(value)) also had two bugs that are fixed by this refactor: - It matched "ma" as "max" because strncmp only compared the shorter strlen(value) bytes. - It silently accepted "hca_handle=" (empty value) as "max" because strncmp with n=0 always returns 0. The match_token() approach also robustly handles extra whitespace in the input by splitting on " \t\n" and skipping empty tokens. Suggested-by: "Michal Koutný" Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- kernel/cgroup/rdma.c | 116 +++++++++++++++++++++---------------------- 1 file changed, 57 insertions(+), 59 deletions(-) diff --git a/kernel/cgroup/rdma.c b/kernel/cgroup/rdma.c index 4fdab4cf49e0..3df7c38ce481 100644 --- a/kernel/cgroup/rdma.c +++ b/kernel/cgroup/rdma.c @@ -9,6 +9,7 @@ */ #include +#include #include #include #include @@ -17,6 +18,22 @@ #define RDMACG_MAX_STR "max" +enum rdmacg_limit_tokens { + RDMACG_HCA_HANDLE_VAL, + RDMACG_HCA_HANDLE_MAX, + RDMACG_HCA_OBJECT_VAL, + RDMACG_HCA_OBJECT_MAX, + NR_RDMACG_LIMIT_TOKENS, +}; + +static const match_table_t rdmacg_limit_tokens = { + { RDMACG_HCA_HANDLE_VAL, "hca_handle=%d" }, + { RDMACG_HCA_HANDLE_MAX, "hca_handle=max" }, + { RDMACG_HCA_OBJECT_VAL, "hca_object=%d" }, + { RDMACG_HCA_OBJECT_MAX, "hca_object=max" }, + { NR_RDMACG_LIMIT_TOKENS, NULL }, +}; + /* * Protects list of resource pools maintained on per cgroup basis * and rdma device list. @@ -355,62 +372,6 @@ void rdmacg_unregister_device(struct rdmacg_device *device) } EXPORT_SYMBOL(rdmacg_unregister_device); -static int parse_resource(char *c, int *intval) -{ - substring_t argstr; - char *name, *value = c; - size_t len; - int ret, i; - - name = strsep(&value, "="); - if (!name || !value) - return -EINVAL; - - i = match_string(rdmacg_resource_names, RDMACG_RESOURCE_MAX, name); - if (i < 0) - return i; - - len = strlen(value); - - argstr.from = value; - argstr.to = value + len; - - ret = match_int(&argstr, intval); - if (ret >= 0) { - if (*intval < 0) - return -EINVAL; - return i; - } - if (strncmp(value, RDMACG_MAX_STR, len) == 0) { - *intval = S32_MAX; - return i; - } - return -EINVAL; -} - -static int rdmacg_parse_limits(char *options, - int *new_limits, unsigned long *enables) -{ - char *c; - int err = -EINVAL; - - /* parse resource options */ - while ((c = strsep(&options, " ")) != NULL) { - int index, intval; - - index = parse_resource(c, &intval); - if (index < 0) - goto err; - - new_limits[index] = intval; - *enables |= BIT(index); - } - return 0; - -err: - return err; -} - static struct rdmacg_device *rdmacg_get_device_locked(const char *name) { struct rdmacg_device *device; @@ -432,6 +393,7 @@ static ssize_t rdmacg_resource_set_max(struct kernfs_open_file *of, struct rdmacg_resource_pool *rpool; struct rdmacg_device *device; char *options = strstrip(buf); + char *p; int *new_limits; unsigned long enables = 0; int i = 0, ret = 0; @@ -449,9 +411,45 @@ static ssize_t rdmacg_resource_set_max(struct kernfs_open_file *of, goto err; } - ret = rdmacg_parse_limits(options, new_limits, &enables); - if (ret) - goto parse_err; + /* parse resource limit tokens */ + while ((p = strsep(&options, " \t\n"))) { + substring_t args[MAX_OPT_ARGS]; + int tok, intval; + + if (!*p) + continue; + + tok = match_token(p, rdmacg_limit_tokens, args); + switch (tok) { + case RDMACG_HCA_HANDLE_VAL: + if (match_int(&args[0], &intval) || intval < 0) { + ret = -EINVAL; + goto parse_err; + } + new_limits[RDMACG_RESOURCE_HCA_HANDLE] = intval; + enables |= BIT(RDMACG_RESOURCE_HCA_HANDLE); + break; + case RDMACG_HCA_HANDLE_MAX: + new_limits[RDMACG_RESOURCE_HCA_HANDLE] = S32_MAX; + enables |= BIT(RDMACG_RESOURCE_HCA_HANDLE); + break; + case RDMACG_HCA_OBJECT_VAL: + if (match_int(&args[0], &intval) || intval < 0) { + ret = -EINVAL; + goto parse_err; + } + new_limits[RDMACG_RESOURCE_HCA_OBJECT] = intval; + enables |= BIT(RDMACG_RESOURCE_HCA_OBJECT); + break; + case RDMACG_HCA_OBJECT_MAX: + new_limits[RDMACG_RESOURCE_HCA_OBJECT] = S32_MAX; + enables |= BIT(RDMACG_RESOURCE_HCA_OBJECT); + break; + default: + ret = -EINVAL; + goto parse_err; + } + } /* acquire lock to synchronize with hot plug devices */ mutex_lock(&rdmacg_mutex); From 7ea04cc4fe3cee2139cc2474cadaecc7c53d986d Mon Sep 17 00:00:00 2001 From: Hongfu Li Date: Fri, 8 May 2026 11:34:53 +0800 Subject: [PATCH 02/22] selftests/cgroup: Fix incorrect variable check in online_cpus() "OFFLINE_CPUS" is a literal string that is always non-empty. It should be "$OFFLINE_CPUS" to check the variable's value instead. Signed-off-by: Hongfu Li Signed-off-by: Tejun Heo --- tools/testing/selftests/cgroup/test_cpuset_prs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/cgroup/test_cpuset_prs.sh b/tools/testing/selftests/cgroup/test_cpuset_prs.sh index a56f4153c64d..df89ddfa073f 100755 --- a/tools/testing/selftests/cgroup/test_cpuset_prs.sh +++ b/tools/testing/selftests/cgroup/test_cpuset_prs.sh @@ -617,7 +617,7 @@ set_ctrl_state_noerr() online_cpus() { - [[ -n "OFFLINE_CPUS" ]] && { + [[ -n "$OFFLINE_CPUS" ]] && { for C in $OFFLINE_CPUS do write_cpu_online ${C}=1 From c26849d32167ec4f2051bb8c9798e850f6381ac9 Mon Sep 17 00:00:00 2001 From: Hongfu Li Date: Mon, 11 May 2026 14:08:53 +0800 Subject: [PATCH 03/22] selftests/cgroup: Add NULL check after malloc in cgroup_util.c Add NULL checks after malloc() in three helper functions to prevent NULL pointer dereference on memory allocation failure. - cg_name() - cg_name_indexed() - cg_control() These functions allocate memory with malloc() but previously called snprintf() unconditionally, which would trigger undefined behavior if allocation fails. Signed-off-by: Hongfu Li Signed-off-by: Tejun Heo --- tools/testing/selftests/cgroup/lib/cgroup_util.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/cgroup/lib/cgroup_util.c b/tools/testing/selftests/cgroup/lib/cgroup_util.c index 6a7295347e90..0c5e6d4bbc3a 100644 --- a/tools/testing/selftests/cgroup/lib/cgroup_util.c +++ b/tools/testing/selftests/cgroup/lib/cgroup_util.c @@ -59,7 +59,8 @@ char *cg_name(const char *root, const char *name) size_t len = strlen(root) + strlen(name) + 2; char *ret = malloc(len); - snprintf(ret, len, "%s/%s", root, name); + if (ret) + snprintf(ret, len, "%s/%s", root, name); return ret; } @@ -69,7 +70,8 @@ char *cg_name_indexed(const char *root, const char *name, int index) size_t len = strlen(root) + strlen(name) + 10; char *ret = malloc(len); - snprintf(ret, len, "%s/%s_%d", root, name, index); + if (ret) + snprintf(ret, len, "%s/%s_%d", root, name, index); return ret; } @@ -79,7 +81,8 @@ char *cg_control(const char *cgroup, const char *control) size_t len = strlen(cgroup) + strlen(control) + 2; char *ret = malloc(len); - snprintf(ret, len, "%s/%s", cgroup, control); + if (ret) + snprintf(ret, len, "%s/%s", cgroup, control); return ret; } From 5b72759aa0c9fb6fb44da0d62ef749bfaed868dc Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Mon, 11 May 2026 14:15:08 +0800 Subject: [PATCH 04/22] selftests/cgroup: fix child process escaping to parent cleanup in test_cpucg_nice In test_cpucg_nice, the forked child process incorrectly jumps to the parent's cleanup label on cg_write failure. This causes the child to attempt cg_destroy on cgroups the parent is still using, and then return to main() to continue executing tests as if it were the parent. Replace goto cleanup with exit(EXIT_FAILURE) in the child process. Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- tools/testing/selftests/cgroup/test_cpu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/cgroup/test_cpu.c b/tools/testing/selftests/cgroup/test_cpu.c index c83f05438d7c..7a40d76b9548 100644 --- a/tools/testing/selftests/cgroup/test_cpu.c +++ b/tools/testing/selftests/cgroup/test_cpu.c @@ -278,7 +278,7 @@ static int test_cpucg_nice(const char *root) char buf[64]; snprintf(buf, sizeof(buf), "%d", getpid()); if (cg_write(cpucg, "cgroup.procs", buf)) - goto cleanup; + exit(EXIT_FAILURE); /* Try to keep niced CPU usage as constrained to hog_cpu as possible */ nice(1); From 298e7f94fc0350c18801a1cfeb04cd053e78efb5 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Mon, 11 May 2026 14:25:20 +0800 Subject: [PATCH 05/22] selftests/cgroup: fix misleading debug message in test_cgfreezer_time_child The debug message says "Expect ctime <= ptime" when the test actually expects ctime > ptime (child's freeze time should exceed parent's, which is zero). Fix the message to match the actual expectation. Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- tools/testing/selftests/cgroup/test_freezer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/cgroup/test_freezer.c b/tools/testing/selftests/cgroup/test_freezer.c index 97fae92c8387..ead68542d45e 100644 --- a/tools/testing/selftests/cgroup/test_freezer.c +++ b/tools/testing/selftests/cgroup/test_freezer.c @@ -1353,7 +1353,7 @@ static int test_cgfreezer_time_child(const char *root) } if (ctime <= ptime) { - debug("Expect ctime (%ld) <= ptime (%ld)\n", ctime, ptime); + debug("Expect ctime (%ld) > ptime (%ld)\n", ctime, ptime); goto cleanup; } From be2689c9ed47bde6f15a24c9845f6fc1ec52502f Mon Sep 17 00:00:00 2001 From: Chen Wandun Date: Mon, 11 May 2026 16:18:38 +0800 Subject: [PATCH 06/22] cgroup/cpuset: Skip hardwall ancestor scan in cpuset v2 in cpuset_current_node_allowed() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cgroup v2 doesn't have the concept of memory hardwall, only top_cpuset has CS_MEM_EXCLUSIVE/CS_MEM_HARDWALL flags, nearest_hardwall_ancestor always returns top_cpuset with all nodes set, so no need to acquire callback_lock and scan up cpuset. Suggested-by: Michal Koutný Signed-off-by: Chen Wandun Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 3fbf6e7f68c3..74d5c494d6ae 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -4231,6 +4231,9 @@ bool cpuset_current_node_allowed(int node, gfp_t gfp_mask) if (gfp_mask & __GFP_HARDWALL) /* If hardwall request, stop here */ return false; + if (cpuset_v2()) + return true; + /* Not hardwall and node outside mems_allowed: scan up cpusets */ spin_lock_irqsave(&callback_lock, flags); From 2a7d34eba5f5a9b889c0f24a28c3b603ba1e7b2b Mon Sep 17 00:00:00 2001 From: Hongfu Li Date: Tue, 12 May 2026 15:14:24 +0800 Subject: [PATCH 07/22] selftests/cgroup: check malloc return value in alloc_anon functions The alloc_anon() function calls malloc() without checking for a NULL return. If memory allocation fails, a NULL pointer dereference will occur when accessing the buffer. Add proper error handling to return -1 when malloc() fails in all four alloc_anon variants: - alloc_anon() - alloc_anon_50M_check() - alloc_anon_noexit() - alloc_anon_50M_check_swap() Signed-off-by: Hongfu Li Reviewed-by: Vishal Moola Reviewed-by: Muchun Song Signed-off-by: Tejun Heo --- .../selftests/cgroup/test_memcontrol.c | 53 ++++++++++--------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/tools/testing/selftests/cgroup/test_memcontrol.c b/tools/testing/selftests/cgroup/test_memcontrol.c index b43da9bc20c4..21aedb35cc12 100644 --- a/tools/testing/selftests/cgroup/test_memcontrol.c +++ b/tools/testing/selftests/cgroup/test_memcontrol.c @@ -55,15 +55,31 @@ int alloc_pagecache(int fd, size_t size) return -1; } -int alloc_anon(const char *cgroup, void *arg) +static char *alloc_and_populate_anon(size_t size) { - size_t size = (unsigned long)arg; char *buf, *ptr; buf = malloc(size); + if (buf == NULL) { + fprintf(stderr, "malloc() failed\n"); + return NULL; + } + for (ptr = buf; ptr < buf + size; ptr += PAGE_SIZE) *ptr = 0; + return buf; +} + +int alloc_anon(const char *cgroup, void *arg) +{ + size_t size = (unsigned long)arg; + char *buf; + + buf = alloc_and_populate_anon(size); + if (!buf) + return -1; + free(buf); return 0; } @@ -174,18 +190,13 @@ static int test_memcg_subtree_control(const char *root) static int alloc_anon_50M_check(const char *cgroup, void *arg) { size_t size = MB(50); - char *buf, *ptr; + char *buf; long anon, current; int ret = -1; - buf = malloc(size); - if (buf == NULL) { - fprintf(stderr, "malloc() failed\n"); + buf = alloc_and_populate_anon(size); + if (!buf) return -1; - } - - for (ptr = buf; ptr < buf + size; ptr += PAGE_SIZE) - *ptr = 0; current = cg_read_long(cgroup, "memory.current"); if (current < size) @@ -406,16 +417,11 @@ static int alloc_anon_noexit(const char *cgroup, void *arg) { int ppid = getppid(); size_t size = (unsigned long)arg; - char *buf, *ptr; + char *buf; - buf = malloc(size); - if (buf == NULL) { - fprintf(stderr, "malloc() failed\n"); + buf = alloc_and_populate_anon(size); + if (!buf) return -1; - } - - for (ptr = buf; ptr < buf + size; ptr += PAGE_SIZE) - *ptr = 0; while (getppid() == ppid) sleep(1); @@ -990,18 +996,13 @@ static int alloc_anon_50M_check_swap(const char *cgroup, void *arg) { long mem_max = (long)arg; size_t size = MB(50); - char *buf, *ptr; + char *buf; long mem_current, swap_current; int ret = -1; - buf = malloc(size); - if (buf == NULL) { - fprintf(stderr, "malloc() failed\n"); + buf = alloc_and_populate_anon(size); + if (!buf) return -1; - } - - for (ptr = buf; ptr < buf + size; ptr += PAGE_SIZE) - *ptr = 0; mem_current = cg_read_long(cgroup, "memory.current"); if (!mem_current || !values_close(mem_current, mem_max, 3)) From c68095c4a4c919cbd7de016fdfa25d19fa918a74 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Thu, 14 May 2026 14:50:31 +0800 Subject: [PATCH 08/22] cgroup/rdma: add rdma.peak for per-device peak usage tracking rdma.peak tracks the high watermark of resource usage per device, giving a better baseline on which to set rdma.max. Polling rdma.current isn't feasible since it would miss short-lived spikes. This interface is analogous to memory.peak. Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- kernel/cgroup/rdma.c | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/kernel/cgroup/rdma.c b/kernel/cgroup/rdma.c index 3df7c38ce481..4e3bf0bade18 100644 --- a/kernel/cgroup/rdma.c +++ b/kernel/cgroup/rdma.c @@ -44,6 +44,7 @@ static LIST_HEAD(rdmacg_devices); enum rdmacg_file_type { RDMACG_RESOURCE_TYPE_MAX, RDMACG_RESOURCE_TYPE_STAT, + RDMACG_RESOURCE_TYPE_PEAK, }; /* @@ -60,6 +61,7 @@ static char const *rdmacg_resource_names[] = { struct rdmacg_resource { int max; int usage; + int peak; }; /* @@ -204,6 +206,17 @@ uncharge_cg_locked(struct rdma_cgroup *cg, rpool->usage_sum--; if (rpool->usage_sum == 0 && rpool->num_max_cnt == RDMACG_RESOURCE_MAX) { + int i; + + /* + * Keep the rpool alive if any peak value is non-zero, + * so that rdma.peak persists as a historical high- + * watermark even after all resources are freed. + */ + for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { + if (rpool->resources[i].peak) + return; + } /* * No user of the rpool and all entries are set to max, so * safe to delete this rpool. @@ -310,6 +323,12 @@ int rdmacg_try_charge(struct rdma_cgroup **rdmacg, } } } + /* Update peak only after all charges succeed */ + for (p = cg; p; p = parent_rdmacg(p)) { + rpool = find_cg_rpool_locked(p, device); + if (rpool && rpool->resources[index].usage > rpool->resources[index].peak) + rpool->resources[index].peak = rpool->resources[index].usage; + } mutex_unlock(&rdmacg_mutex); *rdmacg = cg; @@ -472,6 +491,12 @@ static ssize_t rdmacg_resource_set_max(struct kernfs_open_file *of, if (rpool->usage_sum == 0 && rpool->num_max_cnt == RDMACG_RESOURCE_MAX) { + int i; + + for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { + if (rpool->resources[i].peak) + goto dev_err; + } /* * No user of the rpool and all entries are set to max, so * safe to delete this rpool. @@ -506,6 +531,8 @@ static void print_rpool_values(struct seq_file *sf, value = rpool->resources[i].max; else value = S32_MAX; + } else if (sf_type == RDMACG_RESOURCE_TYPE_PEAK) { + value = rpool ? rpool->resources[i].peak : 0; } else { if (rpool) value = rpool->resources[i].usage; @@ -556,6 +583,12 @@ static struct cftype rdmacg_files[] = { .private = RDMACG_RESOURCE_TYPE_STAT, .flags = CFTYPE_NOT_ON_ROOT, }, + { + .name = "peak", + .seq_show = rdmacg_resource_read, + .private = RDMACG_RESOURCE_TYPE_PEAK, + .flags = CFTYPE_NOT_ON_ROOT, + }, { } /* terminate */ }; @@ -575,6 +608,13 @@ rdmacg_css_alloc(struct cgroup_subsys_state *parent) static void rdmacg_css_free(struct cgroup_subsys_state *css) { struct rdma_cgroup *cg = css_rdmacg(css); + struct rdmacg_resource_pool *rpool, *tmp; + + /* Clean up rpools kept alive by non-zero peak values */ + mutex_lock(&rdmacg_mutex); + list_for_each_entry_safe(rpool, tmp, &cg->rpools, cg_node) + free_cg_rpool_locked(rpool); + mutex_unlock(&rdmacg_mutex); kfree(cg); } From 009bcbd0b201d4dc125eb960a61cb6d4d9fdfc72 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Thu, 14 May 2026 14:50:32 +0800 Subject: [PATCH 09/22] cgroup/rdma: add rdma.events to track resource limit exhaustion Add per-device hierarchical event counters to track when RDMA resource limits are exceeded. The rdma.events file reports max event counts propagated upward from the cgroup whose limit was hit to all ancestors. This mirrors the design of pids.events, where events are attributed to the cgroup that imposed the limit, not necessarily the cgroup where the allocation was attempted. Userspace can monitor this file via poll/epoll for real-time notification of resource exhaustion. Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- include/linux/cgroup_rdma.h | 3 ++ kernel/cgroup/rdma.c | 72 +++++++++++++++++++++++++++++++++++-- 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/include/linux/cgroup_rdma.h b/include/linux/cgroup_rdma.h index 80edae03c313..ac691fe7d3f5 100644 --- a/include/linux/cgroup_rdma.h +++ b/include/linux/cgroup_rdma.h @@ -24,6 +24,9 @@ struct rdma_cgroup { * that belongs to this cgroup. */ struct list_head rpools; + + /* Handle for rdma.events */ + struct cgroup_file events_file; }; struct rdmacg_device { diff --git a/kernel/cgroup/rdma.c b/kernel/cgroup/rdma.c index 4e3bf0bade18..927bbf1eb949 100644 --- a/kernel/cgroup/rdma.c +++ b/kernel/cgroup/rdma.c @@ -81,6 +81,9 @@ struct rdmacg_resource_pool { u64 usage_sum; /* total number counts which are set to max */ int num_max_cnt; + + /* per-resource hierarchical max event counters */ + u64 events_max[RDMACG_RESOURCE_MAX]; }; static struct rdma_cgroup *css_rdmacg(struct cgroup_subsys_state *css) @@ -214,7 +217,8 @@ uncharge_cg_locked(struct rdma_cgroup *cg, * watermark even after all resources are freed. */ for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { - if (rpool->resources[i].peak) + if (rpool->resources[i].peak || + READ_ONCE(rpool->events_max[i])) return; } /* @@ -225,6 +229,34 @@ uncharge_cg_locked(struct rdma_cgroup *cg, } } +/** + * rdmacg_event_locked - fire hierarchical max event when resource limit is hit + * @over_cg: cgroup whose limit was exceeded + * @device: rdma device + * @index: resource type index + * + * Must be called under rdmacg_mutex. Propagates max event counts + * from @over_cg (including itself) upward to all ancestors with + * an rpool and notifies userspace. + */ +static void rdmacg_event_locked(struct rdma_cgroup *over_cg, + struct rdmacg_device *device, + enum rdmacg_resource_type index) +{ + struct rdmacg_resource_pool *rpool; + struct rdma_cgroup *p; + + lockdep_assert_held(&rdmacg_mutex); + + for (p = over_cg; parent_rdmacg(p); p = parent_rdmacg(p)) { + rpool = get_cg_rpool_locked(p, device); + if (!IS_ERR(rpool)) { + rpool->events_max[index]++; + cgroup_file_notify(&p->events_file); + } + } +} + /** * rdmacg_uncharge_hierarchy - hierarchically uncharge rdma resource count * @cg: pointer to cg to uncharge and all parents in hierarchy @@ -335,6 +367,8 @@ int rdmacg_try_charge(struct rdma_cgroup **rdmacg, return 0; err: + if (ret == -EAGAIN) + rdmacg_event_locked(p, device, index); mutex_unlock(&rdmacg_mutex); rdmacg_uncharge_hierarchy(cg, device, p, index); return ret; @@ -494,7 +528,8 @@ static ssize_t rdmacg_resource_set_max(struct kernfs_open_file *of, int i; for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { - if (rpool->resources[i].peak) + if (rpool->resources[i].peak || + READ_ONCE(rpool->events_max[i])) goto dev_err; } /* @@ -569,6 +604,33 @@ static int rdmacg_resource_read(struct seq_file *sf, void *v) return 0; } +static int rdmacg_events_show(struct seq_file *sf, void *v) +{ + struct rdma_cgroup *cg = css_rdmacg(seq_css(sf)); + struct rdmacg_resource_pool *rpool; + struct rdmacg_device *device; + int i; + + mutex_lock(&rdmacg_mutex); + + list_for_each_entry(device, &rdmacg_devices, dev_node) { + rpool = find_cg_rpool_locked(cg, device); + + seq_printf(sf, "%s ", device->name); + for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { + seq_printf(sf, "%s.max=%llu", + rdmacg_resource_names[i], + rpool ? READ_ONCE(rpool->events_max[i]) : 0ULL); + if (i < RDMACG_RESOURCE_MAX - 1) + seq_putc(sf, ' '); + } + seq_putc(sf, '\n'); + } + + mutex_unlock(&rdmacg_mutex); + return 0; +} + static struct cftype rdmacg_files[] = { { .name = "max", @@ -589,6 +651,12 @@ static struct cftype rdmacg_files[] = { .private = RDMACG_RESOURCE_TYPE_PEAK, .flags = CFTYPE_NOT_ON_ROOT, }, + { + .name = "events", + .seq_show = rdmacg_events_show, + .file_offset = offsetof(struct rdma_cgroup, events_file), + .flags = CFTYPE_NOT_ON_ROOT, + }, { } /* terminate */ }; From aefe4847f0891e2e71bedf5478d1cf350f86fc61 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Thu, 14 May 2026 14:50:33 +0800 Subject: [PATCH 10/22] cgroup/rdma: add rdma.events.local for per-cgroup allocation failure attribution Add per-cgroup local event counters to track RDMA resource limit exhaustion from the perspective of individual cgroups. The rdma.events.local file reports two per-resource counters: - max: number of times this cgroup's limit was the one that blocked an allocation in the subtree - alloc_fail: number of allocation attempts originating from this cgroup that failed due to an ancestor's limit This mirrors the design of pids.events.local, where events are attributed to the cgroup that imposed the limit, not necessarily the cgroup where the allocation was attempted. Also extend rdma.events with a hierarchical alloc_fail counter that tracks allocation failures propagating upward from the requesting cgroup, complementing the existing max counter, so that rdma.events and rdma.events.local share the same output format. Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- include/linux/cgroup_rdma.h | 3 +- kernel/cgroup/rdma.c | 143 +++++++++++++++++++++++++++--------- 2 files changed, 109 insertions(+), 37 deletions(-) diff --git a/include/linux/cgroup_rdma.h b/include/linux/cgroup_rdma.h index ac691fe7d3f5..404e746552ca 100644 --- a/include/linux/cgroup_rdma.h +++ b/include/linux/cgroup_rdma.h @@ -25,8 +25,9 @@ struct rdma_cgroup { */ struct list_head rpools; - /* Handle for rdma.events */ + /* Handles for rdma.events[.local] */ struct cgroup_file events_file; + struct cgroup_file events_local_file; }; struct rdmacg_device { diff --git a/kernel/cgroup/rdma.c b/kernel/cgroup/rdma.c index 927bbf1eb949..7c238a9d64d4 100644 --- a/kernel/cgroup/rdma.c +++ b/kernel/cgroup/rdma.c @@ -82,8 +82,11 @@ struct rdmacg_resource_pool { /* total number counts which are set to max */ int num_max_cnt; - /* per-resource hierarchical max event counters */ + /* per-resource event counters */ u64 events_max[RDMACG_RESOURCE_MAX]; + u64 events_alloc_fail[RDMACG_RESOURCE_MAX]; + u64 events_local_max[RDMACG_RESOURCE_MAX]; + u64 events_local_alloc_fail[RDMACG_RESOURCE_MAX]; }; static struct rdma_cgroup *css_rdmacg(struct cgroup_subsys_state *css) @@ -131,6 +134,26 @@ static void free_cg_rpool_locked(struct rdmacg_resource_pool *rpool) kfree(rpool); } +static bool rpool_has_persistent_state(struct rdmacg_resource_pool *rpool) +{ + int i; + + /* + * Keep the rpool alive if any peak value is non-zero, + * so that rdma.peak persists as a historical high- + * watermark even after all resources are freed. + */ + for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { + if (rpool->resources[i].peak || + READ_ONCE(rpool->events_max[i]) || + READ_ONCE(rpool->events_local_max[i]) || + READ_ONCE(rpool->events_alloc_fail[i]) || + READ_ONCE(rpool->events_local_alloc_fail[i])) + return true; + } + return false; +} + static struct rdmacg_resource_pool * find_cg_rpool_locked(struct rdma_cgroup *cg, struct rdmacg_device *device) @@ -209,37 +232,30 @@ uncharge_cg_locked(struct rdma_cgroup *cg, rpool->usage_sum--; if (rpool->usage_sum == 0 && rpool->num_max_cnt == RDMACG_RESOURCE_MAX) { - int i; - - /* - * Keep the rpool alive if any peak value is non-zero, - * so that rdma.peak persists as a historical high- - * watermark even after all resources are freed. - */ - for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { - if (rpool->resources[i].peak || - READ_ONCE(rpool->events_max[i])) - return; + if (!rpool_has_persistent_state(rpool)) { + /* + * No user of the rpool and all entries are set to max, so + * safe to delete this rpool. + */ + free_cg_rpool_locked(rpool); } - /* - * No user of the rpool and all entries are set to max, so - * safe to delete this rpool. - */ - free_cg_rpool_locked(rpool); } } /** - * rdmacg_event_locked - fire hierarchical max event when resource limit is hit + * rdmacg_event_locked - fire event when resource allocation exceeds limit + * @cg: requesting cgroup * @over_cg: cgroup whose limit was exceeded * @device: rdma device * @index: resource type index * - * Must be called under rdmacg_mutex. Propagates max event counts - * from @over_cg (including itself) upward to all ancestors with - * an rpool and notifies userspace. + * Must be called under rdmacg_mutex. Updates event counters in the + * resource pools of @cg and @over_cg, propagates hierarchical max + * events from @over_cg (including itself) upward, and notifies + * userspace via cgroup_file_notify(). */ -static void rdmacg_event_locked(struct rdma_cgroup *over_cg, +static void rdmacg_event_locked(struct rdma_cgroup *cg, + struct rdma_cgroup *over_cg, struct rdmacg_device *device, enum rdmacg_resource_type index) { @@ -248,6 +264,21 @@ static void rdmacg_event_locked(struct rdma_cgroup *over_cg, lockdep_assert_held(&rdmacg_mutex); + /* Increment local alloc_fail in requesting cgroup */ + rpool = find_cg_rpool_locked(cg, device); + if (rpool) { + rpool->events_local_alloc_fail[index]++; + cgroup_file_notify(&cg->events_local_file); + } + + /* Increment local max in the over-limit cgroup */ + rpool = find_cg_rpool_locked(over_cg, device); + if (rpool) { + rpool->events_local_max[index]++; + cgroup_file_notify(&over_cg->events_local_file); + } + + /* Propagate hierarchical max events upward */ for (p = over_cg; parent_rdmacg(p); p = parent_rdmacg(p)) { rpool = get_cg_rpool_locked(p, device); if (!IS_ERR(rpool)) { @@ -255,6 +286,14 @@ static void rdmacg_event_locked(struct rdma_cgroup *over_cg, cgroup_file_notify(&p->events_file); } } + /* Propagate hierarchical alloc_fail from requesting cgroup upward */ + for (p = cg; parent_rdmacg(p); p = parent_rdmacg(p)) { + rpool = get_cg_rpool_locked(p, device); + if (!IS_ERR(rpool)) { + rpool->events_alloc_fail[index]++; + cgroup_file_notify(&p->events_file); + } + } } /** @@ -368,7 +407,7 @@ int rdmacg_try_charge(struct rdma_cgroup **rdmacg, err: if (ret == -EAGAIN) - rdmacg_event_locked(p, device, index); + rdmacg_event_locked(cg, p, device, index); mutex_unlock(&rdmacg_mutex); rdmacg_uncharge_hierarchy(cg, device, p, index); return ret; @@ -525,18 +564,13 @@ static ssize_t rdmacg_resource_set_max(struct kernfs_open_file *of, if (rpool->usage_sum == 0 && rpool->num_max_cnt == RDMACG_RESOURCE_MAX) { - int i; - - for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { - if (rpool->resources[i].peak || - READ_ONCE(rpool->events_max[i])) - goto dev_err; + if (!rpool_has_persistent_state(rpool)) { + /* + * No user of the rpool and all entries are set to max, so + * safe to delete this rpool. + */ + free_cg_rpool_locked(rpool); } - /* - * No user of the rpool and all entries are set to max, so - * safe to delete this rpool. - */ - free_cg_rpool_locked(rpool); } dev_err: @@ -618,9 +652,40 @@ static int rdmacg_events_show(struct seq_file *sf, void *v) seq_printf(sf, "%s ", device->name); for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { - seq_printf(sf, "%s.max=%llu", + seq_printf(sf, "%s.max=%llu %s.alloc_fail=%llu", rdmacg_resource_names[i], - rpool ? READ_ONCE(rpool->events_max[i]) : 0ULL); + rpool ? READ_ONCE(rpool->events_max[i]) : 0ULL, + rdmacg_resource_names[i], + rpool ? READ_ONCE(rpool->events_alloc_fail[i]) : 0ULL); + if (i < RDMACG_RESOURCE_MAX - 1) + seq_putc(sf, ' '); + } + seq_putc(sf, '\n'); + } + + mutex_unlock(&rdmacg_mutex); + return 0; +} + +static int rdmacg_events_local_show(struct seq_file *sf, void *v) +{ + struct rdma_cgroup *cg = css_rdmacg(seq_css(sf)); + struct rdmacg_resource_pool *rpool; + struct rdmacg_device *device; + int i; + + mutex_lock(&rdmacg_mutex); + + list_for_each_entry(device, &rdmacg_devices, dev_node) { + rpool = find_cg_rpool_locked(cg, device); + + seq_printf(sf, "%s ", device->name); + for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { + seq_printf(sf, "%s.max=%llu %s.alloc_fail=%llu", + rdmacg_resource_names[i], + rpool ? READ_ONCE(rpool->events_local_max[i]) : 0ULL, + rdmacg_resource_names[i], + rpool ? READ_ONCE(rpool->events_local_alloc_fail[i]) : 0ULL); if (i < RDMACG_RESOURCE_MAX - 1) seq_putc(sf, ' '); } @@ -657,6 +722,12 @@ static struct cftype rdmacg_files[] = { .file_offset = offsetof(struct rdma_cgroup, events_file), .flags = CFTYPE_NOT_ON_ROOT, }, + { + .name = "events.local", + .seq_show = rdmacg_events_local_show, + .file_offset = offsetof(struct rdma_cgroup, events_local_file), + .flags = CFTYPE_NOT_ON_ROOT, + }, { } /* terminate */ }; From 4376352f2c651ed5308a46caf21d2ccb53c240eb Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Thu, 14 May 2026 14:50:34 +0800 Subject: [PATCH 11/22] cgroup/rdma: document rdma.peak, rdma.events and rdma.events.local Add interface file documentation for the new rdma cgroup files to Documentation/admin-guide/cgroup-v2.rst. Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- Documentation/admin-guide/cgroup-v2.rst | 53 +++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 6efd0095ed99..993446ab66d0 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -2785,6 +2785,59 @@ RDMA Interface Files mlx4_0 hca_handle=1 hca_object=20 ocrdma1 hca_handle=1 hca_object=23 + rdma.peak + A read-only nested-keyed file that exists for all the cgroups + except root. It shows the historical high watermark of + resource usage per device since the cgroup was created. + + An example for mlx4 and ocrdma device follows:: + + mlx4_0 hca_handle=1 hca_object=20 + ocrdma1 hca_handle=0 hca_object=23 + + rdma.events + A read-only nested-keyed file which exists on non-root + cgroups. The following nested keys are defined. + + max + The number of times a process in this cgroup or its + descendants attempted an RDMA resource allocation that + was rejected because a rdma.max limit in the subtree + was reached. This is a hierarchical counter: the event + is propagated upward to all ancestor cgroups. A value + change in this file generates a file modified event. + + alloc_fail + The number of RDMA resource allocation attempts that + originated in this cgroup or its descendants and failed + due to a rdma.max limit being reached. This is a + hierarchical counter propagated upward. + + An example for mlx4 device follows:: + + mlx4_0 hca_handle.max=5 hca_handle.alloc_fail=3 hca_object.max=0 hca_object.alloc_fail=0 + + rdma.events.local + Similar to rdma.events but the fields in the file are local + to the cgroup i.e. not hierarchical. The file modified event + generated on this file reflects only the local events. + + The following nested keys are defined. + + max + The number of times a process in this cgroup or its + descendants attempted an RDMA resource allocation that + was rejected because this cgroup's own rdma.max limit + was reached. + alloc_fail + The number of RDMA resource allocation attempts + originating from this cgroup that failed due to this + cgroup's or an ancestor's rdma.max limit. + + An example for mlx4 device follows:: + + mlx4_0 hca_handle.max=5 hca_handle.alloc_fail=0 hca_object.max=0 hca_object.alloc_fail=0 + DMEM ---- From 3360a5c16d87933fb74b530f5e016eb3dfffee5d Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 4 May 2026 14:51:17 -1000 Subject: [PATCH 12/22] cgroup: Inline cgroup_has_tasks() in cgroup.h cpuset reads cs->css.cgroup->nr_populated_csets directly in two places to test whether a cgroup has tasks. cgroup.c already has a matching helper, cgroup_has_tasks(). Move it to cgroup.h as static inline and use that instead. This is to prepare for relocation of cgroup->nr_populated_csets. No semantic change. Signed-off-by: Tejun Heo --- include/linux/cgroup.h | 5 +++++ kernel/cgroup/cgroup.c | 5 ----- kernel/cgroup/cpuset-v1.c | 2 +- kernel/cgroup/cpuset.c | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index e52160e85af4..ceb87507667e 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -639,6 +639,11 @@ static inline bool task_under_cgroup_hierarchy(struct task_struct *task, return cgroup_is_descendant(cset->dfl_cgrp, ancestor); } +static inline bool cgroup_has_tasks(struct cgroup *cgrp) +{ + return cgrp->nr_populated_csets; +} + /* no synchronization, the result can only be used as a hint */ static inline bool cgroup_is_populated(struct cgroup *cgrp) { diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index bd10a7e2f9c5..7a94c2ea1036 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -376,11 +376,6 @@ static void cgroup_idr_remove(struct idr *idr, int id) spin_unlock_bh(&cgroup_idr_lock); } -static bool cgroup_has_tasks(struct cgroup *cgrp) -{ - return cgrp->nr_populated_csets; -} - static bool cgroup_is_threaded(struct cgroup *cgrp) { return cgrp->dom_cgrp != cgrp; diff --git a/kernel/cgroup/cpuset-v1.c b/kernel/cgroup/cpuset-v1.c index 7308e9b02495..3e9968dd91e9 100644 --- a/kernel/cgroup/cpuset-v1.c +++ b/kernel/cgroup/cpuset-v1.c @@ -312,7 +312,7 @@ void cpuset1_hotplug_update_tasks(struct cpuset *cs, * This is full cgroup operation which will also call back into * cpuset. Execute it asynchronously using workqueue. */ - if (is_empty && cs->css.cgroup->nr_populated_csets && + if (is_empty && cgroup_has_tasks(cs->css.cgroup) && css_tryget_online(&cs->css)) { struct cpuset_remove_tasks_struct *s; diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 74d5c494d6ae..8500e4341c60 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -432,7 +432,7 @@ static inline bool partition_is_populated(struct cpuset *cs, * nr_populated_domain_children may include populated * csets from descendants that are partitions. */ - if (cs->css.cgroup->nr_populated_csets || + if (cgroup_has_tasks(cs->css.cgroup) || cs->attach_in_progress) return true; From 44fabf05634ce9e90b3fb179ea962995b7bbaa09 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 4 May 2026 14:51:18 -1000 Subject: [PATCH 13/22] cgroup: Annotate unlocked nr_populated_* accesses with READ_ONCE/WRITE_ONCE cgroup_update_populated() updates nr_populated_csets, nr_populated_domain_children, and nr_populated_threaded_children under css_set_lock, but cgroup_has_tasks(), cgroup_is_populated(), and cgroup_can_be_thread_root() read them without holding it. Use READ_ONCE/WRITE_ONCE. Signed-off-by: Tejun Heo --- include/linux/cgroup.h | 21 +++++++++++++++++---- kernel/cgroup/cgroup.c | 11 +++++++---- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index ceb87507667e..9f8bef8f3a60 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -639,16 +639,29 @@ static inline bool task_under_cgroup_hierarchy(struct task_struct *task, return cgroup_is_descendant(cset->dfl_cgrp, ancestor); } +/* + * Populated counters: writes happen under css_set_lock. The accessors below + * may read unlocked. What an unpopulated result means depends on context: + * + * - No lock held. Just a snapshot. May race with concurrent updates and is + * useful only as a hint. + * + * - cgroup_mutex held. Migration into the cgroup is blocked, so an observed + * !populated stays !populated until cgroup_mutex is dropped. + * + * - CSS_DYING set. The css can no longer be repopulated, so !populated is + * sticky once observed. + */ static inline bool cgroup_has_tasks(struct cgroup *cgrp) { - return cgrp->nr_populated_csets; + return READ_ONCE(cgrp->nr_populated_csets); } -/* no synchronization, the result can only be used as a hint */ static inline bool cgroup_is_populated(struct cgroup *cgrp) { - return cgrp->nr_populated_csets + cgrp->nr_populated_domain_children + - cgrp->nr_populated_threaded_children; + return READ_ONCE(cgrp->nr_populated_csets) + + READ_ONCE(cgrp->nr_populated_domain_children) + + READ_ONCE(cgrp->nr_populated_threaded_children); } /* returns ino associated with a cgroup */ diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 7a94c2ea1036..d1395784871a 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -404,7 +404,7 @@ static bool cgroup_can_be_thread_root(struct cgroup *cgrp) return false; /* can only have either domain or threaded children */ - if (cgrp->nr_populated_domain_children) + if (READ_ONCE(cgrp->nr_populated_domain_children)) return false; /* and no domain controllers can be enabled */ @@ -783,12 +783,15 @@ static void cgroup_update_populated(struct cgroup *cgrp, bool populated) bool was_populated = cgroup_is_populated(cgrp); if (!child) { - cgrp->nr_populated_csets += adj; + WRITE_ONCE(cgrp->nr_populated_csets, + cgrp->nr_populated_csets + adj); } else { if (cgroup_is_threaded(child)) - cgrp->nr_populated_threaded_children += adj; + WRITE_ONCE(cgrp->nr_populated_threaded_children, + cgrp->nr_populated_threaded_children + adj); else - cgrp->nr_populated_domain_children += adj; + WRITE_ONCE(cgrp->nr_populated_domain_children, + cgrp->nr_populated_domain_children + adj); } if (was_populated == cgroup_is_populated(cgrp)) From c4799253a3ee74ebb27be72fb991c597a5902c01 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 4 May 2026 14:51:19 -1000 Subject: [PATCH 14/22] cgroup: Move populated counters to cgroup_subsys_state Later patches replace the cgroup-level finish_destroy_work deferral added by 93618edf7538 ("cgroup: Defer css percpu_ref kill on rmdir until cgroup is depopulated") with a per-subsys-css deferral. That needs each subsystem css to track its own populated count. Move the populated counters from cgroup onto cgroup_subsys_state. cgroup->self is itself a cgroup_subsys_state and self.parent walks the same chain as cgroup_parent(), so cgroup_update_populated() generalizes to a single css_update_populated() taking a css. The cgroup-side bookkeeping runs only when the walk started from a self css. Keep nr_populated_{domain,threaded}_children on cgroup. Both sum to self.nr_populated_children, but staying as dedicated fields to allow readers like cgroup_can_be_thread_root() unlocked access. css_set_update_populated() also walks the per-subsys-css chain so each subsystem css's hierarchical populated count is maintained. No reader consumes those counts yet. Signed-off-by: Tejun Heo --- include/linux/cgroup-defs.h | 24 ++++++---- include/linux/cgroup.h | 11 +++-- kernel/cgroup/cgroup.c | 91 +++++++++++++++++++++---------------- 3 files changed, 74 insertions(+), 52 deletions(-) diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index 50a784da7a81..c4929f7bbe5a 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -253,6 +253,15 @@ struct cgroup_subsys_state { */ int nr_descendants; + /* + * Hierarchical populated state. For cgroup->self, nr_populated_csets + * counts populated csets linked via cgrp_cset_link. + * nr_populated_children counts immediate-child csses whose own + * populated state is nonzero. Protected by css_set_lock. + */ + int nr_populated_csets; + int nr_populated_children; + /* * A singly-linked list of css structures to be rstat flushed. * This is a scratch field to be used exclusively by @@ -504,17 +513,12 @@ struct cgroup { int max_descendants; /* - * Each non-empty css_set associated with this cgroup contributes - * one to nr_populated_csets. The counter is zero iff this cgroup - * doesn't have any tasks. - * - * All children which have non-zero nr_populated_csets and/or - * nr_populated_children of their own contribute one to either - * nr_populated_domain_children or nr_populated_threaded_children - * depending on their type. Each counter is zero iff all cgroups - * of the type in the subtree proper don't have any tasks. + * Domain/threaded split of self.nr_populated_children: each counts + * immediate-child cgroups whose subtree is populated and sums to + * self.nr_populated_children. Kept as separate fields to allow readers + * like cgroup_can_be_thread_root() unlocked access. Protected by + * css_set_lock; updated by css_update_populated(). */ - int nr_populated_csets; int nr_populated_domain_children; int nr_populated_threaded_children; diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index 9f8bef8f3a60..c2a8c38d8206 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -654,14 +654,17 @@ static inline bool task_under_cgroup_hierarchy(struct task_struct *task, */ static inline bool cgroup_has_tasks(struct cgroup *cgrp) { - return READ_ONCE(cgrp->nr_populated_csets); + return READ_ONCE(cgrp->self.nr_populated_csets); +} + +static inline bool css_is_populated(struct cgroup_subsys_state *css) +{ + return READ_ONCE(css->nr_populated_csets) || READ_ONCE(css->nr_populated_children); } static inline bool cgroup_is_populated(struct cgroup *cgrp) { - return READ_ONCE(cgrp->nr_populated_csets) + - READ_ONCE(cgrp->nr_populated_domain_children) + - READ_ONCE(cgrp->nr_populated_threaded_children); + return css_is_populated(&cgrp->self); } /* returns ino associated with a cgroup */ diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index d1395784871a..dd4ea9d83100 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -756,65 +756,70 @@ static bool css_set_populated(struct css_set *cset) } /** - * cgroup_update_populated - update the populated count of a cgroup - * @cgrp: the target cgroup - * @populated: inc or dec populated count + * css_update_populated - update the populated state of a css and ancestors + * @css: leaf css whose own populated count is changing + * @populated: inc or dec * - * One of the css_sets associated with @cgrp is either getting its first - * task or losing the last. Update @cgrp->nr_populated_* accordingly. The - * count is propagated towards root so that a given cgroup's - * nr_populated_children is zero iff none of its descendants contain any - * tasks. + * One of the css_sets pinned by @css is getting its first task or losing the + * last. Propagate the transition up the parent chain so that a css's + * nr_populated_children is zero iff none of its descendants contain any tasks. * - * @cgrp's interface file "cgroup.populated" is zero if both - * @cgrp->nr_populated_csets and @cgrp->nr_populated_children are zero and - * 1 otherwise. When the sum changes from or to zero, userland is notified - * that the content of the interface file has changed. This can be used to - * detect when @cgrp and its descendants become populated or empty. + * For a cgroup->self walk, also runs cgroup-side bookkeeping at each level: + * domain/threaded child split, deferred-destroy trigger, and notification via + * "cgroup.populated" (zero iff cgrp->self has neither populated csets nor + * populated children; userland is notified on transitions). */ -static void cgroup_update_populated(struct cgroup *cgrp, bool populated) +static void css_update_populated(struct cgroup_subsys_state *css, bool populated) { - struct cgroup *child = NULL; + struct cgroup_subsys_state *child = NULL; int adj = populated ? 1 : -1; lockdep_assert_held(&css_set_lock); do { - bool was_populated = cgroup_is_populated(cgrp); + /* non-NULL only on the cgroup->self walk */ + struct cgroup *cgrp = css_is_self(css) ? css->cgroup : NULL; + bool was_populated = css_is_populated(css); if (!child) { - WRITE_ONCE(cgrp->nr_populated_csets, - cgrp->nr_populated_csets + adj); + WRITE_ONCE(css->nr_populated_csets, + css->nr_populated_csets + adj); } else { - if (cgroup_is_threaded(child)) - WRITE_ONCE(cgrp->nr_populated_threaded_children, - cgrp->nr_populated_threaded_children + adj); - else - WRITE_ONCE(cgrp->nr_populated_domain_children, - cgrp->nr_populated_domain_children + adj); + WRITE_ONCE(css->nr_populated_children, + css->nr_populated_children + adj); + if (cgrp) { + if (cgroup_is_threaded(child->cgroup)) + WRITE_ONCE(cgrp->nr_populated_threaded_children, + cgrp->nr_populated_threaded_children + adj); + else + WRITE_ONCE(cgrp->nr_populated_domain_children, + cgrp->nr_populated_domain_children + adj); + } } - if (was_populated == cgroup_is_populated(cgrp)) + if (was_populated == css_is_populated(css)) break; /* * Subtree just emptied below an offlined cgrp. Fire deferred * destroy. The transition is one-shot. */ - if (was_populated && !css_is_online(&cgrp->self)) { + if (cgrp && was_populated && !css_is_online(css)) { cgroup_get(cgrp); WARN_ON_ONCE(!queue_work(cgroup_offline_wq, &cgrp->finish_destroy_work)); } - cgroup1_check_for_release(cgrp); - TRACE_CGROUP_PATH(notify_populated, cgrp, - cgroup_is_populated(cgrp)); - cgroup_file_notify(&cgrp->events_file); + if (cgrp) { + cgroup1_check_for_release(cgrp); + TRACE_CGROUP_PATH(notify_populated, cgrp, + cgroup_is_populated(cgrp)); + cgroup_file_notify(&cgrp->events_file); + } - child = cgrp; - cgrp = cgroup_parent(cgrp); - } while (cgrp); + child = css; + css = css->parent; + } while (css); } /** @@ -822,17 +827,27 @@ static void cgroup_update_populated(struct cgroup *cgrp, bool populated) * @cset: target css_set * @populated: whether @cset is populated or depopulated * - * @cset is either getting the first task or losing the last. Update the - * populated counters of all associated cgroups accordingly. + * @cset is either getting the first task or losing the last. Update the + * populated counters along each linked cgroup's self chain and each + * subsystem css that @cset pins. */ static void css_set_update_populated(struct css_set *cset, bool populated) { struct cgrp_cset_link *link; + struct cgroup_subsys *ss; + int ssid; lockdep_assert_held(&css_set_lock); list_for_each_entry(link, &cset->cgrp_links, cgrp_link) - cgroup_update_populated(link->cgrp, populated); + css_update_populated(&link->cgrp->self, populated); + + for_each_subsys(ss, ssid) { + struct cgroup_subsys_state *css = cset->subsys[ssid]; + + if (css) + css_update_populated(css, populated); + } } /* @@ -2190,7 +2205,7 @@ int cgroup_setup_root(struct cgroup_root *root, u32 ss_mask) hash_for_each(css_set_table, i, cset, hlist) { link_css_set(&tmp_links, cset, root_cgrp); if (css_set_populated(cset)) - cgroup_update_populated(root_cgrp, true); + css_update_populated(&root_cgrp->self, true); } spin_unlock_irq(&css_set_lock); @@ -6145,7 +6160,7 @@ static void kill_css_finish(struct cgroup_subsys_state *css) * * - cgroup_finish_destroy(): kicks the percpu_ref kill via kill_css_finish() on * each subsystem css. Fires once @cgrp's subtree is fully drained, either - * inline here or from cgroup_update_populated(). + * inline here or from css_update_populated(). * * - The percpu_ref kill chain: css_killed_ref_fn -> css_killed_work_fn -> * ->css_offline() -> release/free. From cfc1da7e1127b4c8787f4dc25d59987c10c9107f Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 4 May 2026 14:51:20 -1000 Subject: [PATCH 15/22] cgroup: Add per-subsys-css kill_css_finish deferral 93618edf7538 ("cgroup: Defer css percpu_ref kill on rmdir until cgroup is depopulated") deferred kill_css_finish() at the cgroup level: rmdir waits for the entire cgroup's populated count to drop to zero, then fires kill_css_finish() on every subsystem css at once. Replace that with per-subsys-css deferral. Each subsystem css now tracks its own hierarchical populated count and independently defers its kill_css_finish() until its own subtree drains. The rmdir-race fix carries through unchanged in shape. The dying css's ->css_offline() still waits until no PF_EXITING task references it, and v2's cgroup-level machinery goes away. cgroup_apply_control_disable() has the same race shape (PF_EXITING tasks pinning a css whose ->css_offline() is about to run) and stays synchronous here. This patch lays the groundwork for fixing it - per-cgroup waiting can't gate one subsys css being killed while the rest of the cgroup stays live, but per-css can. Subtree-wide invariant preserved: a dying ancestor css stays populated through nr_populated_children until every dying descendant's task drains, so the walker fires the ancestor's kill_finish_work only after all descendants have drained. Add paired smp_mb()s in kill_css_sync() and css_update_populated() to fence the StoreLoad on (CSS_DYING, populated counter), guaranteeing that either the walker queues kill_finish_work or the caller fires synchronously. cgroup_destroy_locked() was implicitly fenced by an unrelated css_set_lock pair; cgroup_apply_control_disable() in the next patch is not. Signed-off-by: Tejun Heo --- include/linux/cgroup-defs.h | 6 +-- kernel/cgroup/cgroup.c | 83 +++++++++++++++++++------------------ 2 files changed, 46 insertions(+), 43 deletions(-) diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index c4929f7bbe5a..de2cd6238c2a 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -262,6 +262,9 @@ struct cgroup_subsys_state { int nr_populated_csets; int nr_populated_children; + /* deferred kill_css_finish() queued by css_update_populated() */ + struct work_struct kill_finish_work; + /* * A singly-linked list of css structures to be rstat flushed. * This is a scratch field to be used exclusively by @@ -615,9 +618,6 @@ struct cgroup { /* used to wait for offlining of csses */ wait_queue_head_t offline_waitq; - /* defers killing csses after removal until cgroup is depopulated */ - struct work_struct finish_destroy_work; - /* used to schedule release agent */ struct work_struct release_agent_work; diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index dd4ea9d83100..fa24102535d9 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -264,7 +264,6 @@ static void cgroup_finalize_control(struct cgroup *cgrp, int ret); static void css_task_iter_skip(struct css_task_iter *it, struct task_struct *task); static int cgroup_destroy_locked(struct cgroup *cgrp); -static void cgroup_finish_destroy(struct cgroup *cgrp); static void kill_css_sync(struct cgroup_subsys_state *css); static void kill_css_finish(struct cgroup_subsys_state *css); static struct cgroup_subsys_state *css_create(struct cgroup *cgrp, @@ -801,13 +800,19 @@ static void css_update_populated(struct cgroup_subsys_state *css, bool populated break; /* - * Subtree just emptied below an offlined cgrp. Fire deferred - * destroy. The transition is one-shot. + * Pair with smp_mb() in kill_css_sync(). Either we observe + * CSS_DYING and queue, or the caller observes our decrement + * and fires synchronously. */ - if (cgrp && was_populated && !css_is_online(css)) { - cgroup_get(cgrp); - WARN_ON_ONCE(!queue_work(cgroup_offline_wq, - &cgrp->finish_destroy_work)); + smp_mb(); + + /* + * Subtree just emptied below a dying css. Fire deferred kill. + * The transition is one-shot for a dying css. + */ + if (was_populated && css_is_dying(css)) { + css_get(css); + WARN_ON_ONCE(!queue_work(cgroup_offline_wq, &css->kill_finish_work)); } if (cgrp) { @@ -2064,16 +2069,6 @@ static int cgroup_reconfigure(struct fs_context *fc) return 0; } -static void cgroup_finish_destroy_work_fn(struct work_struct *work) -{ - struct cgroup *cgrp = container_of(work, struct cgroup, finish_destroy_work); - - cgroup_lock(); - cgroup_finish_destroy(cgrp); - cgroup_unlock(); - cgroup_put(cgrp); -} - static void init_cgroup_housekeeping(struct cgroup *cgrp) { struct cgroup_subsys *ss; @@ -2100,7 +2095,6 @@ static void init_cgroup_housekeeping(struct cgroup *cgrp) #endif init_waitqueue_head(&cgrp->offline_waitq); - INIT_WORK(&cgrp->finish_destroy_work, cgroup_finish_destroy_work_fn); INIT_WORK(&cgrp->release_agent_work, cgroup1_release_agent); } @@ -5695,6 +5689,22 @@ static void css_release(struct percpu_ref *ref) queue_work(cgroup_release_wq, &css->destroy_work); } +/* + * Deferred kill_css_finish() fired from css_update_populated() once a dying + * css's hierarchical populated state drops to zero. Pinned by css_get() at the + * queue site; matched by css_put() here. + */ +static void kill_css_finish_work_fn(struct work_struct *work) +{ + struct cgroup_subsys_state *css = + container_of(work, struct cgroup_subsys_state, kill_finish_work); + + cgroup_lock(); + kill_css_finish(css); + cgroup_unlock(); + css_put(css); +} + static void init_and_link_css(struct cgroup_subsys_state *css, struct cgroup_subsys *ss, struct cgroup *cgrp) { @@ -5708,6 +5718,7 @@ static void init_and_link_css(struct cgroup_subsys_state *css, css->id = -1; INIT_LIST_HEAD(&css->sibling); INIT_LIST_HEAD(&css->children); + INIT_WORK(&css->kill_finish_work, kill_css_finish_work_fn); css->serial_nr = css_serial_nr_next++; atomic_set(&css->online_cnt, 0); @@ -6083,6 +6094,13 @@ static void kill_css_sync(struct cgroup_subsys_state *css) css->flags |= CSS_DYING; + /* + * Pair with smp_mb() in css_update_populated(). Either our + * caller observes the walker's decrement and fires + * synchronously, or the walker observes CSS_DYING and queues. + */ + smp_mb(); + /* * This must happen before css is disassociated with its cgroup. * See seq_css() for details. @@ -6158,9 +6176,9 @@ static void kill_css_finish(struct cgroup_subsys_state *css) * - This function: synchronous user-visible state teardown plus kill_css_sync() * on each subsystem css. * - * - cgroup_finish_destroy(): kicks the percpu_ref kill via kill_css_finish() on - * each subsystem css. Fires once @cgrp's subtree is fully drained, either - * inline here or from css_update_populated(). + * - For each subsys css: fire kill_css_finish() synchronously if the subtree is + * already drained, otherwise rely on css_update_populated() to queue + * kill_finish_work when the last populated cset under the css empties. * * - The percpu_ref kill chain: css_killed_ref_fn -> css_killed_work_fn -> * ->css_offline() -> release/free. @@ -6238,29 +6256,14 @@ static int cgroup_destroy_locked(struct cgroup *cgrp) /* put the base reference */ percpu_ref_kill(&cgrp->self.refcnt); - if (!cgroup_is_populated(cgrp)) - cgroup_finish_destroy(cgrp); + for_each_css(css, ssid, cgrp) { + if (!css_is_populated(css)) + kill_css_finish(css); + } return 0; }; -/** - * cgroup_finish_destroy - deferred half of @cgrp destruction - * @cgrp: cgroup whose subtree just became empty - * - * See cgroup_destroy_locked() for the rationale. - */ -static void cgroup_finish_destroy(struct cgroup *cgrp) -{ - struct cgroup_subsys_state *css; - int ssid; - - lockdep_assert_held(&cgroup_mutex); - - for_each_css(css, ssid, cgrp) - kill_css_finish(css); -} - int cgroup_rmdir(struct kernfs_node *kn) { struct cgroup *cgrp; From 1dffd95575eb05bc7ec20ec096ce73be4c5d1ed5 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 4 May 2026 14:51:21 -1000 Subject: [PATCH 16/22] cgroup: Defer kill_css_finish() in cgroup_apply_control_disable() Same race shape as the rmdir path that 93618edf7538 ("cgroup: Defer css percpu_ref kill on rmdir until cgroup is depopulated") fixed: a task past exit_signals() whose cset subsys[ssid] still pins the disabled controller's css can be touching subsys state while ->css_offline() runs. The earlier patches in this series built up the per-subsys-css deferral machinery and routed cgroup_destroy_locked() through it. Apply the same shape to cgroup_apply_control_disable(): kill_css_sync(css); if (!css_is_populated(css)) kill_css_finish(css); When the dying css is still populated, kill_css_finish() is deferred. The walker in css_update_populated() fires kill_finish_work once the css's hierarchical populated count drops to zero. cgroup_lock_and_drain_offline()'s wait predicate switches from percpu_ref_is_dying() to css_is_dying(). CSS_DYING is set by kill_css_sync() and is a strict superset of percpu_ref_is_dying. Without this change, a +cpu re-enable after a deferred -cpu disable would skip the drain (percpu_ref isn't killed yet) and observe the still-CSS_DYING css through cgroup_css(), treating it as live. Signed-off-by: Tejun Heo --- kernel/cgroup/cgroup.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index fa24102535d9..bdc8deedb4f7 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -3237,7 +3237,7 @@ void cgroup_lock_and_drain_offline(struct cgroup *cgrp) struct cgroup_subsys_state *css = cgroup_css(dsct, ss); DEFINE_WAIT(wait); - if (!css || !percpu_ref_is_dying(&css->refcnt)) + if (!css || !css_is_dying(css)) continue; cgroup_get_live(dsct); @@ -3405,7 +3405,8 @@ static void cgroup_apply_control_disable(struct cgroup *cgrp) if (css->parent && !(cgroup_ss_mask(dsct) & (1 << ss->id))) { kill_css_sync(css); - kill_css_finish(css); + if (!css_is_populated(css)) + kill_css_finish(css); } else if (!css_visible(css)) { css_clear_dir(css); if (ss->css_reset) From 0d25e3865841ea5edfedb5af42bf15cef075192e Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Sat, 16 May 2026 13:25:37 +0800 Subject: [PATCH 17/22] cgroup/rdma: Drop unnecessary READ_ONCE() on event counters All accesses to the event counters are serialized by rdmacg_mutex, making the READ_ONCE() annotations unnecessary. Remove them. Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- kernel/cgroup/rdma.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/kernel/cgroup/rdma.c b/kernel/cgroup/rdma.c index 7c238a9d64d4..5e82a03b3270 100644 --- a/kernel/cgroup/rdma.c +++ b/kernel/cgroup/rdma.c @@ -145,10 +145,10 @@ static bool rpool_has_persistent_state(struct rdmacg_resource_pool *rpool) */ for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { if (rpool->resources[i].peak || - READ_ONCE(rpool->events_max[i]) || - READ_ONCE(rpool->events_local_max[i]) || - READ_ONCE(rpool->events_alloc_fail[i]) || - READ_ONCE(rpool->events_local_alloc_fail[i])) + rpool->events_max[i] || + rpool->events_local_max[i] || + rpool->events_alloc_fail[i] || + rpool->events_local_alloc_fail[i]) return true; } return false; @@ -654,9 +654,9 @@ static int rdmacg_events_show(struct seq_file *sf, void *v) for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { seq_printf(sf, "%s.max=%llu %s.alloc_fail=%llu", rdmacg_resource_names[i], - rpool ? READ_ONCE(rpool->events_max[i]) : 0ULL, + rpool ? rpool->events_max[i] : 0ULL, rdmacg_resource_names[i], - rpool ? READ_ONCE(rpool->events_alloc_fail[i]) : 0ULL); + rpool ? rpool->events_alloc_fail[i] : 0ULL); if (i < RDMACG_RESOURCE_MAX - 1) seq_putc(sf, ' '); } @@ -683,9 +683,9 @@ static int rdmacg_events_local_show(struct seq_file *sf, void *v) for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { seq_printf(sf, "%s.max=%llu %s.alloc_fail=%llu", rdmacg_resource_names[i], - rpool ? READ_ONCE(rpool->events_local_max[i]) : 0ULL, + rpool ? rpool->events_local_max[i] : 0ULL, rdmacg_resource_names[i], - rpool ? READ_ONCE(rpool->events_local_alloc_fail[i]) : 0ULL); + rpool ? rpool->events_local_alloc_fail[i] : 0ULL); if (i < RDMACG_RESOURCE_MAX - 1) seq_putc(sf, ' '); } From 98f0adb2284a0a4599a40337a30bef429167bb7b Mon Sep 17 00:00:00 2001 From: Guopeng Zhang Date: Wed, 20 May 2026 17:31:30 +0800 Subject: [PATCH 18/22] selftests/cgroup: enable memory controller in hugetlb memcg test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_hugetlb_memcg creates a child cgroup and then writes memory.max and memory.swap.max. When the test is run standalone, the memory controller may not be enabled in the test root cgroup's subtree_control. In that case, the child cgroup is created without the memory control files, and the test fails during setup before reaching the hugetlb memcg accounting checks. Skip the test when the memory controller is unavailable. Otherwise, enable it in subtree_control before creating the test cgroup. Signed-off-by: Guopeng Zhang Acked-by: Michal Koutný Signed-off-by: Tejun Heo --- tools/testing/selftests/cgroup/test_hugetlb_memcg.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/testing/selftests/cgroup/test_hugetlb_memcg.c b/tools/testing/selftests/cgroup/test_hugetlb_memcg.c index f451aa449be6..b627d84358b1 100644 --- a/tools/testing/selftests/cgroup/test_hugetlb_memcg.c +++ b/tools/testing/selftests/cgroup/test_hugetlb_memcg.c @@ -217,6 +217,14 @@ int main(int argc, char **argv) if (cg_find_unified_root(root, sizeof(root), NULL)) ksft_exit_skip("cgroup v2 isn't mounted\n"); + if (cg_read_strstr(root, "cgroup.controllers", "memory")) + ksft_exit_skip("memory controller isn't available\n"); + + if (cg_read_strstr(root, "cgroup.subtree_control", "memory")) { + if (cg_write(root, "cgroup.subtree_control", "+memory")) + ksft_exit_skip("Failed to set memory controller\n"); + } + switch (test_hugetlb_memcg(root)) { case KSFT_PASS: ksft_test_result_pass("test_hugetlb_memcg\n"); From 336f87d742a616236006bb77275f79a3ac101637 Mon Sep 17 00:00:00 2001 From: Ren Tamura Date: Thu, 28 May 2026 13:28:39 +0900 Subject: [PATCH 19/22] cgroup: pair max limit READ_ONCE() with WRITE_ONCE() cgroup.max.descendants and cgroup.max.depth are shown through seq_file. Their show callbacks read cgrp->max_descendants and cgrp->max_depth with READ_ONCE(), respectively. The corresponding write callbacks update the same scalar fields while holding the cgroup lock, but the seq_file show path does not serialize against those stores. This leaves the lockless show-side loads annotated with READ_ONCE(), while the corresponding stores remain plain stores. Use WRITE_ONCE() for the updates so the intended lockless access is marked consistently on both sides. This does not change locking, ordering, or user-visible semantics. Assisted-by: OpenAI-Codex:gpt-5.5 Signed-off-by: Ren Tamura Signed-off-by: Tejun Heo --- kernel/cgroup/cgroup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index bdc8deedb4f7..6e92791d279e 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -3734,7 +3734,7 @@ static ssize_t cgroup_max_descendants_write(struct kernfs_open_file *of, if (!cgrp) return -ENOENT; - cgrp->max_descendants = descendants; + WRITE_ONCE(cgrp->max_descendants, descendants); cgroup_kn_unlock(of->kn); @@ -3777,7 +3777,7 @@ static ssize_t cgroup_max_depth_write(struct kernfs_open_file *of, if (!cgrp) return -ENOENT; - cgrp->max_depth = depth; + WRITE_ONCE(cgrp->max_depth, depth); cgroup_kn_unlock(of->kn); From 390f2d73bc99a888469f789f274c162da33bafe5 Mon Sep 17 00:00:00 2001 From: Guopeng Zhang Date: Thu, 28 May 2026 17:37:42 +0800 Subject: [PATCH 20/22] cgroup/cpuset: Free sched domains on rebuild guard failure generate_sched_domains() returns sched-domain masks and optional attributes that are normally handed to partition_sched_domains(), which takes ownership of them. rebuild_sched_domains_locked() has a WARN guard after generate_sched_domains() and before partition_sched_domains() to avoid passing offline CPUs into the scheduler domain rebuild path. If that guard fires, the function currently returns directly without freeing the generated doms and attr. Free the generated sched-domain masks and attributes before returning from the guard failure path. Signed-off-by: Guopeng Zhang Reviewed-by: Waiman Long Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 8500e4341c60..2a4122b8db29 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -1004,8 +1004,11 @@ void rebuild_sched_domains_locked(void) * prevent the panic. */ for (i = 0; doms && i < ndoms; i++) { - if (WARN_ON_ONCE(!cpumask_subset(doms[i], cpu_active_mask))) + if (WARN_ON_ONCE(!cpumask_subset(doms[i], cpu_active_mask))) { + free_sched_domains(doms, ndoms); + kfree(attr); return; + } } /* Have scheduler rebuild the domains */ From 6935f0496c2eb74dd3f57228a799d2db1a8a9ac8 Mon Sep 17 00:00:00 2001 From: Costa Shulyupin Date: Sun, 31 May 2026 17:00:45 +0300 Subject: [PATCH 21/22] docs: cgroup: Fix stale source file paths Update two references to files that were moved: - kernel/cgroup.c -> kernel/cgroup/cgroup.c - tools/cgroup/cgroup_event_listener.c -> samples/cgroup/cgroup_event_listener.c Assisted-by: Claude:claude-opus-4-6 Signed-off-by: Costa Shulyupin Signed-off-by: Tejun Heo --- Documentation/admin-guide/cgroup-v1/cgroups.rst | 2 +- Documentation/admin-guide/cgroup-v1/memcg_test.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/cgroup-v1/cgroups.rst b/Documentation/admin-guide/cgroup-v1/cgroups.rst index 463f98453323..e501f45ea93f 100644 --- a/Documentation/admin-guide/cgroup-v1/cgroups.rst +++ b/Documentation/admin-guide/cgroup-v1/cgroups.rst @@ -525,7 +525,7 @@ cgroup. It may also be taken to prevent cgroups from being modified, but more specific locks may be more appropriate in that situation. -See kernel/cgroup.c for more details. +See kernel/cgroup/cgroup.c for more details. Subsystems can take/release the cgroup_mutex via the functions cgroup_lock()/cgroup_unlock(). diff --git a/Documentation/admin-guide/cgroup-v1/memcg_test.rst b/Documentation/admin-guide/cgroup-v1/memcg_test.rst index 7c7cd457cf69..ebedbc3c3f9c 100644 --- a/Documentation/admin-guide/cgroup-v1/memcg_test.rst +++ b/Documentation/admin-guide/cgroup-v1/memcg_test.rst @@ -321,7 +321,7 @@ Under below explanation, we assume CONFIG_SWAP=y. ---------------------- Memory controller implements memory thresholds using cgroups notification - API. You can use tools/cgroup/cgroup_event_listener.c to test it. + API. You can use samples/cgroup/cgroup_event_listener.c to test it. (Shell-A) Create cgroup and run event listener:: From a99ce697ea5e27b867c9ba4ee55fa5ba3b8d1188 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 1 Jun 2026 08:56:04 -1000 Subject: [PATCH 22/22] cgroup: Migrate tasks to the root css when a controller is rebound cgroup_apply_control_disable() defers kill_css_finish() while a css is still populated, relying on css_update_populated() to fire the deferred kill once the populated count reaches zero. This deadlocks when a controller is rebound out of a hierarchy. Mounting an implicit_on_dfl controller such as perf_event as a v1 hierarchy steals it off the default hierarchy, and rebind_subsystems() kills its per-cgroup csses while they are still populated. The migration run in the same step keeps the old css for a controller no longer in the hierarchy's mask, so no task is migrated off the dying csses. Their populated count never reaches zero, the deferred kill_css_finish() never fires, and the next cgroup_lock_and_drain_offline() hangs forever under cgroup_mutex. That migration is already a no-op pass over the rebound subtree. Add cgroup_rebind_ss_mask so find_existing_css_set() resolves the leaving controllers to the root css. Their tasks are migrated there, the per-cgroup csses depopulate, and cgroup_apply_control_disable() kills them synchronously. The deferral stays correct for the rmdir and controller-disable paths it was meant for. Fixes: 1dffd95575eb ("cgroup: Defer kill_css_finish() in cgroup_apply_control_disable()") Reported-by: Mark Brown Closes: https://lore.kernel.org/all/41cd159c-54e5-45e0-81df-eaf36a6c028e@sirena.org.uk/ Reported-by: Bert Karwatzki Closes: https://lore.kernel.org/all/4e986b4ed7e16547805d54b6e67d09120bc4d2f2.camel@web.de/ Tested-by: Mark Brown Tested-by: Bert Karwatzki Signed-off-by: Tejun Heo --- kernel/cgroup/cgroup.c | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 6e92791d279e..4d442c224bf5 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -197,6 +197,14 @@ static u32 cgrp_dfl_implicit_ss_mask; /* some controllers can be threaded on the default hierarchy */ static u32 cgrp_dfl_threaded_ss_mask; +/* + * Set across rebind_subsystems() to the controllers leaving a hierarchy. + * Guarded by cgroup_mutex. Makes find_existing_css_set() resolve them to the + * root css so the affected tasks are migrated there before + * cgroup_apply_control_disable() kills the per-cgroup csses. + */ +static u32 cgroup_rebind_ss_mask; + /* The list of hierarchy roots */ LIST_HEAD(cgroup_roots); static int cgroup_root_count; @@ -1083,7 +1091,15 @@ static struct css_set *find_existing_css_set(struct css_set *old_cset, * won't change, so no need for locking. */ for_each_subsys(ss, i) { - if (root->subsys_mask & (1UL << i)) { + if (unlikely(cgroup_rebind_ss_mask & (1UL << i))) { + /* + * @ss is leaving this hierarchy and its per-cgroup + * csses are about to be killed. Resolve to the + * surviving root css so the tasks are migrated there. + */ + template[i] = cgroup_css(&root->cgrp, ss); + WARN_ON_ONCE(!template[i]); + } else if (root->subsys_mask & (1UL << i)) { /* * @ss is in this hierarchy, so we want the * effective css from @cgrp. @@ -1853,11 +1869,17 @@ int rebind_subsystems(struct cgroup_root *dst_root, u32 ss_mask) struct cgroup *scgrp = &cgrp_dfl_root.cgrp; /* - * Controllers from default hierarchy that need to be rebound - * are all disabled together in one go. + * Controllers leaving the default hierarchy are disabled + * together. cgroup_rebind_ss_mask makes cgroup_apply_control() + * migrate their tasks to the root css, so the per-cgroup csses + * are unpopulated when cgroup_finalize_control() kills them. + * Clear it before cgroup_finalize_control(), which does no + * css_set lookup. */ cgrp_dfl_root.subsys_mask &= ~dfl_disable_ss_mask; + cgroup_rebind_ss_mask = dfl_disable_ss_mask; WARN_ON(cgroup_apply_control(scgrp)); + cgroup_rebind_ss_mask = 0; cgroup_finalize_control(scgrp, 0); } @@ -1871,9 +1893,14 @@ int rebind_subsystems(struct cgroup_root *dst_root, u32 ss_mask) WARN_ON(!css || cgroup_css(dcgrp, ss)); if (src_root != &cgrp_dfl_root) { - /* disable from the source */ + /* + * Disable from the source, migrating its tasks to the + * root css first (see cgroup_rebind_ss_mask). + */ src_root->subsys_mask &= ~(1 << ssid); + cgroup_rebind_ss_mask = 1 << ssid; WARN_ON(cgroup_apply_control(scgrp)); + cgroup_rebind_ss_mask = 0; cgroup_finalize_control(scgrp, 0); }