From 29b3b6cde5f2dee1c51a1c03ba2e0015e205125b Mon Sep 17 00:00:00 2001 From: Hao Li Date: Wed, 24 Jun 2026 18:00:14 +0800 Subject: [PATCH 01/31] mm/slub: deduplicate NUMA policy calculation in allocation paths Currently, alloc_from_pcs() and __slab_alloc_node() both calculate the NUMA policy independently. Since they are called consecutively in paths like __kmalloc_nolock_noprof() and slab_alloc_node(), this leads to redundant code snippets. Introduce a helper function to resolve the NUMA policy once, eliminating the duplicated code and reducing execution overhead. Also remove __slab_alloc_node() function because it is almost empty. The callers of __slab_alloc_node now call ___slab_alloc() directly. Additional notes: Previously, when slab_strict_numa was enabled, alloc_from_pcs() and __slab_alloc_node() could each resolve the task mempolicy, so MPOL_INTERLEAVE or MPOL_WEIGHTED_INTERLEAVE could advance the interleave state twice for a single object allocation attempt. And each retry will also advance the interleave state. With this change, the strict NUMA node is resolved once and reused by both alloc_from_pcs() and ___slab_alloc() in each retry. This is a behavior change, but it better matches the intent of selecting one policy node for one allocation attempt. Signed-off-by: Hao Li Reviewed-by: Harry Yoo (Oracle) Link: https://patch.msgid.link/20260624100320.430115-1-hao.li@linux.dev Signed-off-by: Vlastimil Babka (SUSE) --- mm/slub.c | 42 +++++++++--------------------------------- 1 file changed, 9 insertions(+), 33 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 65febe957886..4d7e3067a5d6 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4526,11 +4526,8 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, return object; } -static void *__slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node, - const struct slab_alloc_context *ac) +static __always_inline int apply_strict_numa_policy(int node) { - void *object; - #ifdef CONFIG_NUMA if (static_branch_unlikely(&strict_numa) && node == NUMA_NO_NODE) { @@ -4551,10 +4548,7 @@ static void *__slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node, } } #endif - - object = ___slab_alloc(s, gfpflags, node, ac); - - return object; + return node; } static __fastpath_inline @@ -4759,28 +4753,6 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags, bool node_requested; void *object; -#ifdef CONFIG_NUMA - if (static_branch_unlikely(&strict_numa) && - node == NUMA_NO_NODE) { - - struct mempolicy *mpol = current->mempolicy; - - if (mpol) { - /* - * Special BIND rule support. If the local node - * is in permitted set then do not redirect - * to a particular node. - * Otherwise we apply the memory policy to get - * the node we need to allocate on. - */ - if (mpol->mode != MPOL_BIND || - !node_isset(numa_mem_id(), mpol->nodes)) - - node = mempolicy_slab_node(); - } - } -#endif - node_requested = IS_ENABLED(CONFIG_NUMA) && node != NUMA_NO_NODE; /* @@ -4930,10 +4902,12 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, if (unlikely(object)) goto out; + node = apply_strict_numa_policy(node); + object = alloc_from_pcs(s, gfpflags, ac->alloc_flags, node); if (unlikely(!object)) - object = __slab_alloc_node(s, gfpflags, node, ac); + object = ___slab_alloc(s, gfpflags, node, ac); maybe_wipe_obj_freeptr(s, object); @@ -5416,6 +5390,8 @@ static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_f if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) return NULL; + node = apply_strict_numa_policy(node); + retry: if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) return NULL; @@ -5440,10 +5416,10 @@ static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_f /* * Do not call slab_alloc_node(), since trylock mode isn't * compatible with slab_pre_alloc_hook/should_failslab and - * kfence_alloc. Hence call __slab_alloc_node() (at most twice) + * kfence_alloc. Hence call ___slab_alloc() (at most twice) * and slab_post_alloc_hook() directly. */ - ret = __slab_alloc_node(s, gfp_flags, node, ac); + ret = ___slab_alloc(s, gfp_flags, node, ac); /* * It's possible we failed due to trylock as we preempted someone with From a561e19e76ef1ebf3f9a56977dfe87526b37146c Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Thu, 4 Jun 2026 19:03:18 +0800 Subject: [PATCH 02/31] mm/mempool: Untangle CONFIG_SLUB_DEBUG_ON abuse and switch to static key The mempool subsystem historically wrapped its debugging logic inside an merely defines compile-time defaults for SLUB and caused two flaws: 1. On production kernels where CONFIG_SLUB_DEBUG=y but CONFIG_SLUB_DEBUG_ON=n, mempool debugging was completely compiled out at compile time. 2. On kernels with CONFIG_SLUB_DEBUG_ON=y, mempool debugging stayed active even if a user explicitly disabled slub debugging at boot time. Clean up this mess by removing the #ifdef and switching to a runtime static key (mempool_debug_enabled), allowing mempool debugging to be toggled cleanly via its own boot parameter. Suggested-by: Vlastimil Babka (SUSE) Signed-off-by: Li RongQing Cc: Vlastimil Babka Cc: Harry Yoo Cc: Andrew Morton Cc: Hao Li Cc: Christoph Lameter Cc: David Rientjes Cc: Roman Gushchin Cc: Matthew Wilcox Cc: Usama Arif Reviewed-by: SeongJae Park Reviewed-by: Harry Yoo (Oracle) Link: https://patch.msgid.link/20260604110318.2089-1-lirongqing@baidu.com Signed-off-by: Vlastimil Babka (SUSE) --- .../admin-guide/kernel-parameters.txt | 5 +++ mm/mempool.c | 35 +++++++++++++------ 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index b5493a7f8f22..ca755fe185e5 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3977,6 +3977,11 @@ Kernel parameters Note that even when enabled, there are a few cases where the feature is not effective. + mempool_debug [MM] + Enable mempool debugging. This enables element + poison checking when freeing elements back to the + pool. Useful for debugging mempool corruption. + memtest= [KNL,X86,ARM,M68K,PPC,RISCV,EARLY] Enable memtest Format: default : 0 diff --git a/mm/mempool.c b/mm/mempool.c index 473a029fa31f..cb74e718b2c6 100644 --- a/mm/mempool.c +++ b/mm/mempool.c @@ -16,11 +16,28 @@ #include #include #include +#include +#include #include "slab.h" static DECLARE_FAULT_ATTR(fail_mempool_alloc); static DECLARE_FAULT_ATTR(fail_mempool_alloc_bulk); +/* + * Debugging support for mempool using static key. + * + * This allows enabling mempool debug at boot time via: + * mempool_debug + */ +static DEFINE_STATIC_KEY_FALSE(mempool_debug_enabled); + +static int __init mempool_debug_setup(char *str) +{ + static_branch_enable(&mempool_debug_enabled); + return 1; +} +__setup("mempool_debug", mempool_debug_setup); + static int __init mempool_faul_inject_init(void) { int error; @@ -37,7 +54,6 @@ static int __init mempool_faul_inject_init(void) } late_initcall(mempool_faul_inject_init); -#ifdef CONFIG_SLUB_DEBUG_ON static void poison_error(struct mempool *pool, void *element, size_t size, size_t byte) { @@ -140,14 +156,6 @@ static void poison_element(struct mempool *pool, void *element) #endif } } -#else /* CONFIG_SLUB_DEBUG_ON */ -static inline void check_element(struct mempool *pool, void *element) -{ -} -static inline void poison_element(struct mempool *pool, void *element) -{ -} -#endif /* CONFIG_SLUB_DEBUG_ON */ static __always_inline bool kasan_poison_element(struct mempool *pool, void *element) @@ -175,7 +183,10 @@ static void kasan_unpoison_element(struct mempool *pool, void *element) static __always_inline void add_element(struct mempool *pool, void *element) { BUG_ON(pool->min_nr != 0 && pool->curr_nr >= pool->min_nr); - poison_element(pool, element); + + if (static_branch_unlikely(&mempool_debug_enabled)) + poison_element(pool, element); + if (kasan_poison_element(pool, element)) pool->elements[pool->curr_nr++] = element; } @@ -186,7 +197,9 @@ static void *remove_element(struct mempool *pool) BUG_ON(pool->curr_nr < 0); kasan_unpoison_element(pool, element); - check_element(pool, element); + + if (static_branch_unlikely(&mempool_debug_enabled)) + check_element(pool, element); return element; } From 0125c783248707a50819c1423cba8dc455b27612 Mon Sep 17 00:00:00 2001 From: Seongjun Hong Date: Wed, 1 Jul 2026 14:06:33 +0000 Subject: [PATCH 03/31] slab: remove unused SL_CPU slab_stat_type Since the removal of the per-cpu slab in commit 32c894c7274b ("slab: remove struct kmem_cache_cpu"), show_slab_objects() no longer has a branch handling SO_CPU, so cpu_slabs_show() always produces "0". Emit "0\n" directly instead, matching the sibling cpu_partial and slabs_cpu_partial stubs, and remove the now-unused SO_CPU macro and SL_CPU enum value. No functional change intended; the cpu_slabs sysfs attribute continues to read 0. Signed-off-by: Seongjun Hong Reviewed-by: Harry Yoo (Oracle) Reviewed-by: Hao Li Link: https://patch.msgid.link/20260701140634.71608-1-hsj0512@snu.ac.kr Signed-off-by: Vlastimil Babka (SUSE) --- mm/slub.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 4d7e3067a5d6..7a8f496b296c 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -8966,14 +8966,12 @@ static void process_slab(struct loc_track *t, struct kmem_cache *s, enum slab_stat_type { SL_ALL, /* All slabs */ SL_PARTIAL, /* Only partially allocated slabs */ - SL_CPU, /* Only slabs used for cpu caches */ SL_OBJECTS, /* Determine allocated objects not slabs */ SL_TOTAL /* Determine object capacity not slabs */ }; #define SO_ALL (1 << SL_ALL) #define SO_PARTIAL (1 << SL_PARTIAL) -#define SO_CPU (1 << SL_CPU) #define SO_OBJECTS (1 << SL_OBJECTS) #define SO_TOTAL (1 << SL_TOTAL) @@ -9162,7 +9160,7 @@ SLAB_ATTR_RO(partial); static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf) { - return show_slab_objects(s, buf, SO_CPU); + return sysfs_emit(buf, "0\n"); } SLAB_ATTR_RO(cpu_slabs); From bfec5d0b393f56b2f0ca42f50ee9939a9b3999d0 Mon Sep 17 00:00:00 2001 From: Seongjun Hong Date: Wed, 1 Jul 2026 14:17:46 +0000 Subject: [PATCH 04/31] docs: ABI: sysfs-kernel-slab: mark cpu_partial attributes deprecated The per-cpu slab and per-cpu partial slab mechanisms were removed when SLUB was fully converted to per-cpu sheaves in Linux 7.0. The cpu_slabs, slabs_cpu_partial and cpu_partial sysfs attributes were kept as stubs that always return 0 for backwards compatibility, but their documentation still described them as if they were functional. Update the three descriptions to state that the attributes are deprecated and always read 0, and note that they are retained only for compatibility. While here, fix a "partialli" typo in the slabs_cpu_partial description. Signed-off-by: Seongjun Hong Acked-by: Harry Yoo (Oracle) Link: https://patch.msgid.link/20260701141755.85119-1-hsj0512@snu.ac.kr Signed-off-by: Vlastimil Babka (SUSE) --- Documentation/ABI/testing/sysfs-kernel-slab | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-kernel-slab b/Documentation/ABI/testing/sysfs-kernel-slab index b26e4299f822..c52034e3e794 100644 --- a/Documentation/ABI/testing/sysfs-kernel-slab +++ b/Documentation/ABI/testing/sysfs-kernel-slab @@ -113,8 +113,10 @@ KernelVersion: 2.6.22 Contact: Pekka Enberg , Christoph Lameter Description: - The cpu_slabs file is read-only and displays how many cpu slabs - are active and their NUMA locality. + The cpu_slabs file is read-only. It is deprecated and always + reads "0" since the removal of per-cpu slabs in Linux 7.0. It + previously displayed how many cpu slabs were active and their + NUMA locality. The file is kept for backwards compatibility. What: /sys/kernel/slab//cpuslab_flush Date: April 2009 @@ -509,12 +511,16 @@ What: /sys/kernel/slab//slabs_cpu_partial Date: Aug 2011 Contact: Christoph Lameter Description: - This read-only file shows the number of partialli allocated - frozen slabs. + This read-only file is deprecated and always reads "0(0)" since + the removal of per-cpu partial slabs in Linux 7.0. It previously + showed the number of partially allocated frozen slabs. The file + is kept for backwards compatibility. What: /sys/kernel/slab//cpu_partial Date: Aug 2011 Contact: Christoph Lameter Description: - This read-only file shows the number of per cpu partial - pages to keep around. + This file is deprecated and always reads "0" since the removal of + per-cpu partial slabs in Linux 7.0. It previously showed the + number of per-cpu partial pages to keep around. The file is kept + for backwards compatibility. From e1fa26489025d6deac76d1dbfe2e0720a3ad84b1 Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Tue, 7 Jul 2026 14:16:00 +0200 Subject: [PATCH 05/31] mm/slab: extract __free_to_pcs_batch() from free_to_pcs_bulk() It has been noted that free_to_pcs_bulk() is difficult to follow, with a number of goto labels, and this has contributed to two memory leak bugs in there. Extract part of the code to __free_to_pcs_batch(), which focuses only on freeing free-hook-processed local objects to a percpu sheaf, and returning how many were freed. Zero means a trylock failure or no empty sheaf available, and thus the caller should fallback to __kmem_cache_free_bulk(). Make free_to_pcs_bulk() call this in a while loop, removing all goto labels from the function. __free_to_pcs_batch() retains two rather straightforward ones. Reviewed-by: Shengming Hu Link: https://patch.msgid.link/20260707-slab-simplify-bulk-pcs-v1-1-4850dbe0d904@kernel.org Reviewed-by: Hao Li Reviewed-by: Harry Yoo (Oracle) Signed-off-by: Vlastimil Babka (SUSE) --- mm/slub.c | 116 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 60 insertions(+), 56 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 7a8f496b296c..22045dc919ef 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -6179,51 +6179,21 @@ static __always_inline bool can_free_to_pcs(struct slab *slab) } /* - * Bulk free objects to the percpu sheaves. - * Unlike free_to_pcs() this includes the calls to all necessary hooks - * and the fallback to freeing to slab pages. + * Try to free as many objects (already processed by free hooks) as possible to + * a single per-cpu sheaf. + * + * Returns how many objects were freed. Zero means failure and the caller should + * fall back to __kmem_cache_free_bulk(). */ -static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) +static unsigned int __free_to_pcs_batch(struct kmem_cache *s, size_t size, void **p) { struct slub_percpu_sheaves *pcs; struct slab_sheaf *main, *empty; - bool init = slab_want_init_on_free(s); - unsigned int batch, i = 0; struct node_barn *barn; - void *remote_objects[PCS_BATCH_MAX]; - unsigned int remote_nr = 0; + unsigned int batch; - while (i < size) { - struct slab *slab = virt_to_slab(p[i]); - - memcg_slab_free_hook(s, slab, p + i, 1); - alloc_tagging_slab_free_hook(s, slab, p + i, 1); - - if (unlikely(!slab_free_hook(s, p[i], init, false))) { - p[i] = p[--size]; - continue; - } - - if (unlikely(!can_free_to_pcs(slab))) { - remote_objects[remote_nr] = p[i]; - p[i] = p[--size]; - if (++remote_nr >= PCS_BATCH_MAX) { - __kmem_cache_free_bulk(s, remote_nr, &remote_objects[0]); - stat_add(s, FREE_SLOWPATH, remote_nr); - remote_nr = 0; - } - continue; - } - - i++; - } - - if (!size) - goto flush_remote; - -next_batch: if (!local_trylock(&s->cpu_sheaves->lock)) - goto fallback; + return 0; pcs = this_cpu_ptr(s->cpu_sheaves); @@ -6269,29 +6239,63 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) stat_add(s, FREE_FASTPATH, batch); - if (batch < size) { - p += batch; - size -= batch; - goto next_batch; - } - - if (remote_nr) - goto flush_remote; - - return; + return batch; no_empty: local_unlock(&s->cpu_sheaves->lock); - /* - * if we depleted all empty sheaves in the barn or there are too - * many full sheaves, free the rest to slab pages - */ -fallback: - __kmem_cache_free_bulk(s, size, p); - stat_add(s, FREE_SLOWPATH, size); + return 0; +} + +/* + * Bulk free objects to the percpu sheaves. + * Unlike free_to_pcs() this includes the calls to all necessary hooks + * and the fallback to freeing to slab pages. + */ +static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) +{ + bool init = slab_want_init_on_free(s); + void *remote_objects[PCS_BATCH_MAX]; + unsigned int remote_nr = 0; + + for (unsigned int i = 0; i < size;) { + struct slab *slab = virt_to_slab(p[i]); + + memcg_slab_free_hook(s, slab, p + i, 1); + alloc_tagging_slab_free_hook(s, slab, p + i, 1); + + if (unlikely(!slab_free_hook(s, p[i], init, false))) { + p[i] = p[--size]; + continue; + } + + if (unlikely(!can_free_to_pcs(slab))) { + remote_objects[remote_nr] = p[i]; + p[i] = p[--size]; + if (++remote_nr >= PCS_BATCH_MAX) { + __kmem_cache_free_bulk(s, remote_nr, &remote_objects[0]); + stat_add(s, FREE_SLOWPATH, remote_nr); + remote_nr = 0; + } + continue; + } + + i++; + } + + while (size) { + unsigned int batch_freed = __free_to_pcs_batch(s, size, p); + + if (!batch_freed) { + __kmem_cache_free_bulk(s, size, p); + stat_add(s, FREE_SLOWPATH, size); + break; + } + + p += batch_freed; + size -= batch_freed; + } -flush_remote: if (remote_nr) { __kmem_cache_free_bulk(s, remote_nr, &remote_objects[0]); stat_add(s, FREE_SLOWPATH, remote_nr); From efa6a5b7bbb1bbdf304afc4a21da8ebacc2f758a Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Mon, 13 Jul 2026 09:52:30 +0200 Subject: [PATCH 06/31] mm/slab: simplify freeing remote objects in free_to_pcs_bulk() We have been moving remote objects to an on-stack array and flushing it when full. Instead, we can swap them towards the beginning of the supplied array and bulk-free it just once. Also add a comment to explain the rationale of freeing remote objects last, because now it would appear to be simpler to free them first. Reviewed-by: Pedro Falcato Reviewed-by: Shengming Hu Reviewed-by: Harry Yoo (Oracle) Reviewed-by: Hao Li Link: https://patch.msgid.link/20260713-bulk_free_remote-v2-1-24ee24771c2f@kernel.org Signed-off-by: Vlastimil Babka (SUSE) --- mm/slub.c | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 22045dc919ef..b650c1729437 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -6255,9 +6255,17 @@ static unsigned int __free_to_pcs_batch(struct kmem_cache *s, size_t size, void static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) { bool init = slab_want_init_on_free(s); - void *remote_objects[PCS_BATCH_MAX]; + void **remote_objects = p; unsigned int remote_nr = 0; + /* + * Process the free hooks and separate out remote objects by + * partitioning the 'p' array in place: + * + * [0, remote_nr) - processed remote objects + * [remote_nr, i) - processed local objects + * [i, size) - unprocessed objects + */ for (unsigned int i = 0; i < size;) { struct slab *slab = virt_to_slab(p[i]); @@ -6270,19 +6278,17 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) } if (unlikely(!can_free_to_pcs(slab))) { - remote_objects[remote_nr] = p[i]; - p[i] = p[--size]; - if (++remote_nr >= PCS_BATCH_MAX) { - __kmem_cache_free_bulk(s, remote_nr, &remote_objects[0]); - stat_add(s, FREE_SLOWPATH, remote_nr); - remote_nr = 0; - } - continue; + if (i != remote_nr) + swap(remote_objects[remote_nr], p[i]); + remote_nr++; } i++; } + p += remote_nr; + size -= remote_nr; + while (size) { unsigned int batch_freed = __free_to_pcs_batch(s, size, p); @@ -6296,8 +6302,12 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) size -= batch_freed; } + /* + * Processing remote objects last decreases the chances of cpu migration + * while freeing to sheaves and compromising object locality + */ if (remote_nr) { - __kmem_cache_free_bulk(s, remote_nr, &remote_objects[0]); + __kmem_cache_free_bulk(s, remote_nr, remote_objects); stat_add(s, FREE_SLOWPATH, remote_nr); } } From 56aa9b819edce4acfa879728af56534c951d1818 Mon Sep 17 00:00:00 2001 From: Hongling Zeng Date: Mon, 13 Jul 2026 15:00:24 +0800 Subject: [PATCH 07/31] mm/slub: add comment explaining intentional kobject handling in sysfs_slab_add Add a comment to clarify why we don't call kobject_put() when kobject_init_and_add() fails in sysfs_slab_add(). Per commit 2420baa8e046 ("mm/slab: Allow cache creation to proceed even if sysfs registration fails"), sysfs failures are treated as non-fatal and the cache continues to be used. Calling kobject_put() would trigger slab_kmem_cache_release() which frees the entire cache structure, so we intentionally skip it. Suggested-by: Harry Yoo Signed-off-by: Hongling Zeng Acked-by: Harry Yoo (Oracle) Link: https://patch.msgid.link/20260713070024.153552-1-zenghongling@kylinos.cn Signed-off-by: Vlastimil Babka (SUSE) --- mm/slub.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mm/slub.c b/mm/slub.c index b650c1729437..da9efb040040 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -9676,6 +9676,11 @@ static int sysfs_slab_add(struct kmem_cache *s) s->kobj.kset = kset; err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name); + /* + * Intentionally skip kobject_put(). See commit 2420baa8e046 + * ("mm/slab: Allow cache creation to proceed even if sysfs + * registration fails") + */ if (err) goto out; From 9338b189be6895e45cf6fcf819c55cd3c47e4a18 Mon Sep 17 00:00:00 2001 From: Shengming Hu Date: Tue, 21 Jul 2026 08:45:22 +0800 Subject: [PATCH 08/31] mm/slub: prevent pfmemalloc objects from entering the barn kmem_cache_return_sheaf() may refill a partially consumed sheaf before placing it in the barn. Without an explicit restriction, this refill may draw objects from pfmemalloc slabs and consume emergency reserves. Add __GFP_NOMEMALLOC so that returned sheaves are refilled only from non-pfmemalloc slabs. Also add __GFP_NOWARN, as suggested by Hao Li, because this refill is a best-effort attempt and failure is acceptable. If the refill fails, flush and free the sheaf instead. Fixes: 1ce20c28eafd ("slab: handle pfmemalloc slabs properly with sheaves") Cc: stable@vger.kernel.org Signed-off-by: Shengming Hu Reviewed-by: Harry Yoo (Oracle) Reviewed-by: Hao Li Link: https://patch.msgid.link/20260721084522552ZPa16p1SRj3PYat3sqxuN@zte.com.cn Signed-off-by: Vlastimil Babka (SUSE) --- mm/slub.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/slub.c b/mm/slub.c index da9efb040040..f528e5fbef82 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -5149,7 +5149,7 @@ void kmem_cache_return_sheaf(struct kmem_cache *s, gfp_t gfp, * simply flush and free it. */ if (!barn || data_race(barn->nr_full) >= MAX_FULL_SHEAVES || - refill_sheaf(s, sheaf, gfp)) { + refill_sheaf(s, sheaf, gfp | __GFP_NOMEMALLOC | __GFP_NOWARN)) { sheaf_flush_unused(s, sheaf); free_empty_sheaf(s, sheaf); return; From b0c4582862c2feae7c9d49fe77aff6215cd1cea9 Mon Sep 17 00:00:00 2001 From: "Harry Yoo (Oracle)" Date: Wed, 29 Jul 2026 17:20:09 +0900 Subject: [PATCH 09/31] mm/slab, slub_kunit: register kprobe to trigger _nolock APIs Since kmalloc_nolock() always fails in NMI and hardirq contexts on PREEMPT_RT, slub_kunit cannot properly test _nolock() APIs. Register a kprobe pre-handler to invoke kmalloc_nolock() and kfree_nolock() in the middle of the slab allocator. However, do not register the handler on UP kernels because that use case is not well supported [1] in the kernel. To attach the pre-handler while s->cpu_sheaves->lock or n->list_lock is held, add a wrapper function for lockdep_assert_held() that calls a no-op function slab_attach_kprobe_locked() on debug builds. The function is optimized away when neither CONFIG_PROVE_LOCKING nor CONFIG_DEBUG_VM is selected and register_kprobe() fails. The function calls barrier() to prevent the compiler from optimizing away its callsites. Otherwise, the compiler may consider the function does not have any side effect and remove callsites. Compared to using plain kprobe, this has two advantages: 1) it avoids hardcoding function names in the test, and 2) it can trigger those APIs in the middle of a function, where the lock is expected to be held as annotated with lockdep. While it was proposed [2] to use kunit function redirection to test this, it is currently infeasible as some lock helpers don't have symbols. Factor out the nested loop that calls kmalloc and friends to test_kmalloc_kfree(), and call them in test_kmalloc_kfree_nolock_{perf,kprobe}(), each being an independent test case. During the refactoring, drop alloc_fail handling as it doesn't provide much benefits. Link: https://lore.kernel.org/linux-mm/20260427-nolock-api-fix-v2-0-a6b83a92d9a4@kernel.org [1] Link: https://lore.kernel.org/linux-mm/6edebc2b-5f5a-4b9c-9a4c-564310acee1b@kernel.org [2] Acked-by: Vlastimil Babka (SUSE) Reviewed-by: Shengming Hu Signed-off-by: Harry Yoo (Oracle) Link: https://patch.msgid.link/20260729-kfree_rcu_nolock-v5-1-a28cdcda9673@kernel.org Signed-off-by: Vlastimil Babka (SUSE) --- lib/tests/slub_kunit.c | 167 +++++++++++++++++++++++++++++------------ mm/slub.c | 36 ++++++--- 2 files changed, 148 insertions(+), 55 deletions(-) diff --git a/lib/tests/slub_kunit.c b/lib/tests/slub_kunit.c index fa6d31dbca16..8c2b9911471e 100644 --- a/lib/tests/slub_kunit.c +++ b/lib/tests/slub_kunit.c @@ -8,6 +8,7 @@ #include #include #include +#include #include "../mm/slab.h" static struct kunit_resource resource; @@ -292,7 +293,7 @@ static void test_krealloc_redzone_zeroing(struct kunit *test) kmem_cache_destroy(s); } -#ifdef CONFIG_PERF_EVENTS +#if defined(CONFIG_PERF_EVENTS) || (defined(CONFIG_KPROBES) && defined(CONFIG_SMP)) #define NR_ITERATIONS 1000 #define NR_OBJECTS 1000 static void *objects[NR_OBJECTS]; @@ -302,26 +303,40 @@ struct test_nolock_context { int callback_count; int alloc_ok; int alloc_fail; +#ifdef CONFIG_PERF_EVENTS struct perf_event *event; +#endif +#if defined(CONFIG_KPROBES) && defined(CONFIG_SMP) + struct kprobe kprobe; +#endif }; -static struct perf_event_attr hw_attr = { - .type = PERF_TYPE_HARDWARE, - .config = PERF_COUNT_HW_CPU_CYCLES, - .size = sizeof(struct perf_event_attr), - .pinned = 1, - .disabled = 1, - .freq = 1, - .sample_freq = 100000, -}; +static void test_kmalloc_kfree(void) +{ + int i, j; -static void overflow_handler_test_kmalloc_kfree_nolock(struct perf_event *event, - struct perf_sample_data *data, - struct pt_regs *regs) + for (i = 0; i < NR_ITERATIONS; i++) { + for (j = 0; j < NR_OBJECTS; j++) { + gfp_t gfp = (i % 2) ? GFP_KERNEL : GFP_KERNEL_ACCOUNT; + + objects[j] = kmalloc(64, gfp); + if (!objects[j]) { + j--; + while (j >= 0) + kfree(objects[j--]); + return; + } + } + + for (j = 0; j < NR_OBJECTS; j++) + kfree(objects[j]); + } +} + +static void test_nolock(struct test_nolock_context *ctx) { void *objp; gfp_t gfp; - struct test_nolock_context *ctx = event->overflow_handler_context; /* __GFP_ACCOUNT to test kmalloc_nolock() in alloc_slab_obj_exts() */ gfp = (ctx->callback_count % 2) ? 0 : __GFP_ACCOUNT; @@ -335,47 +350,104 @@ static void overflow_handler_test_kmalloc_kfree_nolock(struct perf_event *event, kfree_nolock(objp); ctx->callback_count++; } +#endif -static void test_kmalloc_kfree_nolock(struct kunit *test) +#ifdef CONFIG_PERF_EVENTS +static struct perf_event_attr hw_attr = { + .type = PERF_TYPE_HARDWARE, + .config = PERF_COUNT_HW_CPU_CYCLES, + .size = sizeof(struct perf_event_attr), + .pinned = 1, + .disabled = 1, + .freq = 1, + .sample_freq = 100000, +}; + +static void overflow_handler_test_nolock(struct perf_event *event, + struct perf_sample_data *data, + struct pt_regs *regs) +{ + struct test_nolock_context *ctx = event->overflow_handler_context; + + test_nolock(ctx); +} + +static bool enable_perf_events(struct test_nolock_context *ctx) { - int i, j; - struct test_nolock_context ctx = { .test = test }; struct perf_event *event; - bool alloc_fail = false; event = perf_event_create_kernel_counter(&hw_attr, -1, current, - overflow_handler_test_kmalloc_kfree_nolock, - &ctx); + overflow_handler_test_nolock, + ctx); + if (IS_ERR(event)) - kunit_skip(test, "Failed to create perf event"); - ctx.event = event; - perf_event_enable(ctx.event); - for (i = 0; i < NR_ITERATIONS; i++) { - for (j = 0; j < NR_OBJECTS; j++) { - gfp_t gfp = (i % 2) ? GFP_KERNEL : GFP_KERNEL_ACCOUNT; + return false; - objects[j] = kmalloc(64, gfp); - if (!objects[j]) { - j--; - while (j >= 0) - kfree(objects[j--]); - alloc_fail = true; - goto cleanup; - } - } - for (j = 0; j < NR_OBJECTS; j++) - kfree(objects[j]); - } + ctx->event = event; + perf_event_enable(ctx->event); + return true; +} -cleanup: - perf_event_disable(ctx.event); - perf_event_release_kernel(ctx.event); +static void disable_perf_events(struct test_nolock_context *ctx) +{ + kunit_info(ctx->test, "HW perf events: callback_count: %d, alloc_ok: %d, alloc_fail: %d\n", + ctx->callback_count, ctx->alloc_ok, ctx->alloc_fail); - kunit_info(test, "callback_count: %d, alloc_ok: %d, alloc_fail: %d\n", - ctx.callback_count, ctx.alloc_ok, ctx.alloc_fail); + perf_event_disable(ctx->event); + perf_event_release_kernel(ctx->event); +} - if (alloc_fail) - kunit_skip(test, "Allocation failed"); +static void test_kmalloc_kfree_nolock_perf(struct kunit *test) +{ + struct test_nolock_context ctx = { .test = test }; + + if (!enable_perf_events(&ctx)) + kunit_skip(test, "Failed to enable perf event, skipping"); + + test_kmalloc_kfree(); + + disable_perf_events(&ctx); + KUNIT_EXPECT_EQ(test, 0, slab_errors); +} +#endif + +#if defined(CONFIG_KPROBES) && defined(CONFIG_SMP) +static int slab_kprobe_pre_handler(struct kprobe *p, struct pt_regs *regs) +{ + struct test_nolock_context *ctx; + + ctx = container_of(p, struct test_nolock_context, kprobe); + test_nolock(ctx); + return 0; +} + +static bool register_slab_kprobes(struct test_nolock_context *ctx) +{ + ctx->kprobe.symbol_name = "slab_attach_kprobe_locked"; + ctx->kprobe.pre_handler = slab_kprobe_pre_handler; + + if (register_kprobe(&ctx->kprobe)) + return false; + return true; +} + +static void unregister_slab_kprobes(struct test_nolock_context *ctx) +{ + kunit_info(ctx->test, "kprobes: callback_count: %d, alloc_ok: %d, alloc_fail: %d\n", + ctx->callback_count, ctx->alloc_ok, ctx->alloc_fail); + unregister_kprobe(&ctx->kprobe); +} + +static void test_kmalloc_kfree_nolock_kprobe(struct kunit *test) +{ + struct test_nolock_context ctx = { .test = test }; + + if (!register_slab_kprobes(&ctx)) + kunit_skip(test, "Failed to register kprobe, skipping"); + + test_kmalloc_kfree(); + + unregister_slab_kprobes(&ctx); KUNIT_EXPECT_EQ(test, 0, slab_errors); } #endif @@ -405,7 +477,10 @@ static struct kunit_case test_cases[] = { KUNIT_CASE(test_leak_destroy), KUNIT_CASE(test_krealloc_redzone_zeroing), #ifdef CONFIG_PERF_EVENTS - KUNIT_CASE_SLOW(test_kmalloc_kfree_nolock), + KUNIT_CASE_SLOW(test_kmalloc_kfree_nolock_perf), +#endif +#if defined(CONFIG_KPROBES) && defined(CONFIG_SMP) + KUNIT_CASE_SLOW(test_kmalloc_kfree_nolock_kprobe), #endif {} }; diff --git a/mm/slub.c b/mm/slub.c index 0337e60db5ac..8273780fb4ae 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -908,6 +908,24 @@ static inline unsigned int obj_exts_offset_in_object(struct kmem_cache *s) } #endif +/* + * A no-op function used to attach kprobe handlers in slub_kunit tests. + * The barrier is needed to prevent the compiler from optimizing out callsites. + */ +#if defined(CONFIG_DEBUG_VM) || defined(CONFIG_PROVE_LOCKING) +static noinline void slab_attach_kprobe_locked(void) +{ + barrier(); +} +#else +static inline void slab_attach_kprobe_locked(void) { } +#endif + +#define slab_lockdep_assert_held(lock) do { \ + lockdep_assert_held(lock); \ + slab_attach_kprobe_locked(); \ +} while (0) + #ifdef CONFIG_SLUB_DEBUG /* @@ -1665,7 +1683,7 @@ static void add_full(struct kmem_cache *s, if (!(s->flags & SLAB_STORE_USER)) return; - lockdep_assert_held(&n->list_lock); + slab_lockdep_assert_held(&n->list_lock); list_add(&slab->slab_list, &n->full); } @@ -1674,7 +1692,7 @@ static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct if (!(s->flags & SLAB_STORE_USER)) return; - lockdep_assert_held(&n->list_lock); + slab_lockdep_assert_held(&n->list_lock); list_del(&slab->slab_list); } @@ -2840,7 +2858,7 @@ static unsigned int __sheaf_flush_main_batch(struct kmem_cache *s) void *objects[PCS_BATCH_MAX]; struct slab_sheaf *sheaf; - lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); + slab_lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); pcs = this_cpu_ptr(s->cpu_sheaves); sheaf = pcs->main; @@ -3519,7 +3537,7 @@ __add_partial(struct kmem_cache_node *n, struct slab *slab, enum add_mode mode) static inline void add_partial(struct kmem_cache_node *n, struct slab *slab, enum add_mode mode) { - lockdep_assert_held(&n->list_lock); + slab_lockdep_assert_held(&n->list_lock); __add_partial(n, slab, mode); } @@ -3533,7 +3551,7 @@ static inline void clear_node_partial_state(struct kmem_cache_node *n, static inline void remove_partial(struct kmem_cache_node *n, struct slab *slab) { - lockdep_assert_held(&n->list_lock); + slab_lockdep_assert_held(&n->list_lock); list_del(&slab->slab_list); clear_node_partial_state(n, slab); } @@ -3549,7 +3567,7 @@ static void *alloc_single_from_partial(struct kmem_cache *s, { void *object; - lockdep_assert_held(&n->list_lock); + slab_lockdep_assert_held(&n->list_lock); #ifdef CONFIG_SLUB_DEBUG if (s->flags & SLAB_CONSISTENCY_CHECKS) { @@ -4620,7 +4638,7 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, struct node_barn *barn; bool allow_spin; - lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); + slab_lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); /* Bootstrap or debug cache, back off */ if (unlikely(!cache_has_sheaves(s))) { @@ -5763,7 +5781,7 @@ static void __pcs_install_empty_sheaf(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, struct slab_sheaf *empty, struct node_barn *barn) { - lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); + slab_lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); /* This is what we expect to find if nobody interrupted us. */ if (likely(!pcs->spare)) { @@ -5814,7 +5832,7 @@ __pcs_replace_full_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, bool put_fail; restart: - lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); + slab_lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); /* Bootstrap or debug cache, back off */ if (unlikely(!cache_has_sheaves(s))) { From 7e98f856395618011c517f767fb80ac3fe90de2b Mon Sep 17 00:00:00 2001 From: Li Xiasong Date: Wed, 29 Jul 2026 18:18:49 +0800 Subject: [PATCH 10/31] mm/slub: fix missing debugfs entries for caches created before sysfs init slab_debugfs_init() creates the slab debugfs root at device initcall time, while slab_sysfs_init() moves slab_state to FULL at late initcall time. SLAB_STORE_USER caches created in this window miss their debugfs entries because do_kmem_cache_create() skips debugfs_slab_add() when slab_state <= UP. This was observed with MPTCP's request_sock_subflow_v6 cache, whose slab debugfs directory was missing. The affected window is: slab_debugfs_init() slab_debugfs_root = debugfs_create_dir(...) list_for_each_entry(s, &slab_caches, list) debugfs_slab_add(s) kmem_cache_create(..., SLAB_STORE_USER, ...) do_kmem_cache_create() if (slab_state <= UP) return without debugfs entries slab_sysfs_init() slab_state = FULL Initialize the debugfs root and add debugfs entries while holding slab_mutex, walking slab_caches exactly once and handling both sysfs and debugfs entries in the same pass. This gives the sysfs and debugfs initialization an explicit order and prevents caches from being created between the debugfs scan and slab_state reaching FULL. Gate the new slab_late_init() on either sysfs or debugfs being enabled, with the slab_kset creation and alias_list processing factored into helpers that have empty no-sysfs variants, as suggested by Vlastimil Babka. On slab_kset_init() failure, slab_state stays below FULL so kmem_cache_create() keeps taking the early-boot path, matching prior behavior. Guard debugfs_slab_release() against an uninitialized debugfs root, since the root is now created later and a cache may be released before it exists. Fixes: 1a5ad30b89b4 ("mm: slub: make slab_sysfs_init() a late_initcall") Cc: stable@vger.kernel.org Suggested-by: Vlastimil Babka Signed-off-by: Li Xiasong Link: https://patch.msgid.link/20260729101849.3734287-1-lixiasong1@huawei.com Reviewed-by: Harry Yoo (Oracle) Signed-off-by: Vlastimil Babka (SUSE) --- mm/slub.c | 78 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 49 insertions(+), 29 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index f528e5fbef82..9656bdc94007 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -333,14 +333,20 @@ enum track_item { TRACK_ALLOC, TRACK_FREE }; #ifdef SLAB_SUPPORTS_SYSFS static int sysfs_slab_add(struct kmem_cache *); +static int __init slab_kset_init(void); +static void __init slab_sysfs_process_aliases(void); #else static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; } +static inline int slab_kset_init(void) { return 0; } +static inline void slab_sysfs_process_aliases(void) { } #endif #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG) static void debugfs_slab_add(struct kmem_cache *); +static void __init slab_debugfs_root_init(void); #else static inline void debugfs_slab_add(struct kmem_cache *s) { } +static inline void slab_debugfs_root_init(void) { } #endif enum add_mode { @@ -9746,28 +9752,20 @@ int sysfs_slab_alias(struct kmem_cache *s, const char *name) return 0; } -static int __init slab_sysfs_init(void) +static int __init slab_kset_init(void) { - struct kmem_cache *s; - int err; - - mutex_lock(&slab_mutex); - slab_kset = kset_create_and_add("slab", NULL, kernel_kobj); if (!slab_kset) { - mutex_unlock(&slab_mutex); pr_err("Cannot register slab subsystem.\n"); return -ENOMEM; } - slab_state = FULL; + return 0; +} - list_for_each_entry(s, &slab_caches, list) { - err = sysfs_slab_add(s); - if (err) - pr_err("SLUB: Unable to add boot slab %s to sysfs\n", - s->name); - } +static void __init slab_sysfs_process_aliases(void) +{ + int err; while (alias_list) { struct saved_alias *al = alias_list; @@ -9779,13 +9777,42 @@ static int __init slab_sysfs_init(void) al->name); kfree(al); } - - mutex_unlock(&slab_mutex); - return 0; } -late_initcall(slab_sysfs_init); #endif /* SLAB_SUPPORTS_SYSFS */ +#if defined(SLAB_SUPPORTS_SYSFS) || \ + (defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS)) +static int __init slab_late_init(void) +{ + struct kmem_cache *s; + int err; + + mutex_lock(&slab_mutex); + + err = slab_kset_init(); + if (err) + goto out; + + slab_debugfs_root_init(); + slab_state = FULL; + + list_for_each_entry(s, &slab_caches, list) { + if (sysfs_slab_add(s)) + pr_err("SLUB: Unable to add boot slab %s to sysfs\n", + s->name); + + if (s->flags & SLAB_STORE_USER) + debugfs_slab_add(s); + } + + slab_sysfs_process_aliases(); +out: + mutex_unlock(&slab_mutex); + return err; +} +late_initcall(slab_late_init); +#endif + #if defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS) static int slab_debugfs_show(struct seq_file *seq, void *v) { @@ -9976,23 +10003,16 @@ static void debugfs_slab_add(struct kmem_cache *s) void debugfs_slab_release(struct kmem_cache *s) { + if (unlikely(!slab_debugfs_root)) + return; + debugfs_lookup_and_remove(s->name, slab_debugfs_root); } -static int __init slab_debugfs_init(void) +static void __init slab_debugfs_root_init(void) { - struct kmem_cache *s; - slab_debugfs_root = debugfs_create_dir("slab", NULL); - - list_for_each_entry(s, &slab_caches, list) - if (s->flags & SLAB_STORE_USER) - debugfs_slab_add(s); - - return 0; - } -__initcall(slab_debugfs_init); #endif /* * The /proc/slabinfo ABI From 3a11935cd3e8c78a747c3b4144488926f9226329 Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Mon, 27 Jul 2026 14:53:55 +0200 Subject: [PATCH 11/31] mm/slab: skip kfence objects in allocation profiling struct kfence_metadata only contains struct slabobj_ext with CONFIG_MEMCG, which is then used for the "fake" slab's obj_exts field. If CONFIG_MEMCG is enabled, the struct can also end up used for memory allocation profiling. If CONFIG_MEMCG is disabled but profiling is enabled, it will end up allocating its obj_exts via prepare_slab_obj_exts_hook() and assigning them to the fake struct slab. These will probably then never be freed. So things sorta work, but not always in the intended and optimal way. The upcoming changes to slabobj_ext layout would additionally need a proper refactoring to keep working. However, there's little benefit in accounting KFENCE objects. KFENCE allocations are rare and there can be only CONFIG_KFENCE_NUM_OBJECTS (default to 255) outstanding ones at any time. For any callsite prominent enough in the memory allocation profiling stats, allocations served from KFENCE will be lost in the noise. Thus let's not complicate things and simply stop accounting KFENCE objects in allocation profiling and skip them in the related slab hooks. We also need to skip kfence objects in mark_obj_codetag_empty() in case a sheaf is allocated from kfence, per earlier sashiko review. Link: https://patch.msgid.link/20260727-b4-objext_split-v3-1-c29ef0f1f257@kernel.org Reviewed-by: Hao Li Signed-off-by: Vlastimil Babka (SUSE) --- Documentation/mm/allocation-profiling.rst | 7 +++++++ mm/slub.c | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/Documentation/mm/allocation-profiling.rst b/Documentation/mm/allocation-profiling.rst index 5389d241176a..d02eb54ee8f2 100644 --- a/Documentation/mm/allocation-profiling.rst +++ b/Documentation/mm/allocation-profiling.rst @@ -112,3 +112,10 @@ To do so: - Then, use the following form for your allocations: alloc_hooks_tag(ht->your_saved_tag, kmalloc_noprof(...)) + +Notes +===== + +- When a slab object is allocated from KFENCE, its accounting is skipped. + KFENCE allocations are rare and limited to a small number, so this omission + is negligible. diff --git a/mm/slub.c b/mm/slub.c index 0337e60db5ac..d702d273253c 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2067,6 +2067,9 @@ static inline void mark_obj_codetag_empty(const void *obj) struct slab *obj_slab; unsigned long slab_exts; + if (is_kfence_address(obj)) + return; + obj_slab = virt_to_slab(obj); slab_exts = slab_obj_exts(obj_slab); if (slab_exts) { @@ -2352,6 +2355,9 @@ __alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags, if (alloc_flags & SLAB_ALLOC_NO_RECURSE) return; + if (is_kfence_address(object)) + return; + slab = virt_to_slab(object); obj_exts = prepare_slab_obj_exts_hook(s, slab, flags, alloc_flags, object); /* @@ -2399,6 +2405,9 @@ __alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p for (i = 0; i < objects; i++) { unsigned int off = obj_to_index(s, slab, p[i]); + if (is_kfence_address(p[i])) + continue; + alloc_tag_sub(&slab_obj_ext(slab, obj_exts, off)->ref, s->size); } put_slab_obj_exts(obj_exts); From a867ffa398f24f22384eb5aa4fac182026b95692 Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Mon, 27 Jul 2026 14:53:56 +0200 Subject: [PATCH 12/31] mm/slab: remove objs_per_slab() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The function has an unused kmem_cache argument and almost nothing uses it anyway; doing slab->objects is simpler. Remove it with the last two users. KUNIT_EXPECT_EQ() needs a cast to avoid "error: ‘typeof’ applied to a bit-field" but we don't need to keep a wrapper just for that. Reviewed-by: Suren Baghdasaryan Reviewed-by: Harry Yoo (Oracle) Reviewed-by: Hao Li Link: https://patch.msgid.link/20260727-b4-objext_split-v3-2-c29ef0f1f257@kernel.org Signed-off-by: Vlastimil Babka (SUSE) --- mm/kfence/kfence_test.c | 2 +- mm/slab.h | 6 ------ mm/slub.c | 3 +-- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/mm/kfence/kfence_test.c b/mm/kfence/kfence_test.c index de2d0f7d62b1..9867c03ef0ae 100644 --- a/mm/kfence/kfence_test.c +++ b/mm/kfence/kfence_test.c @@ -295,7 +295,7 @@ static void *test_alloc(struct kunit *test, size_t size, gfp_t gfp, enum allocat * memcg accounting works correctly. */ KUNIT_EXPECT_EQ(test, obj_to_index(s, slab, alloc), 0U); - KUNIT_EXPECT_EQ(test, objs_per_slab(s, slab), 1); + KUNIT_EXPECT_EQ(test, ((unsigned int)slab->objects), 1); if (policy == ALLOCATE_ANY) return alloc; diff --git a/mm/slab.h b/mm/slab.h index f5e336b6b6b0..01535e1e2d3c 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -330,12 +330,6 @@ static inline unsigned int obj_to_index(const struct kmem_cache *cache, return __obj_to_index(cache, slab_address(slab), obj); } -static inline int objs_per_slab(const struct kmem_cache *cache, - const struct slab *slab) -{ - return slab->objects; -} - /* * State of the slab allocator. * diff --git a/mm/slub.c b/mm/slub.c index d702d273253c..b94482830637 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2130,7 +2130,6 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags) { const bool allow_spin = alloc_flags_allow_spinning(alloc_flags); - unsigned int objects = objs_per_slab(s, slab); bool new_slab = alloc_flags & SLAB_ALLOC_NEW_SLAB; unsigned long new_exts; unsigned long old_exts; @@ -2186,7 +2185,7 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, #endif retry: old_exts = READ_ONCE(slab->obj_exts); - handle_failed_objexts_alloc(old_exts, vec, objects); + handle_failed_objexts_alloc(old_exts, vec, slab->objects); if (new_slab) { /* From 215bb51fff467cdaa40d2df098a62ed2ae3e212d Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Mon, 27 Jul 2026 14:53:57 +0200 Subject: [PATCH 13/31] mm: move struct slabobj_ext to mm/slab.h Users of include/linux/memcontrol.h don't need to see this internal structure. Further changes to the struct will reduce recompiling. Reviewed-by: Suren Baghdasaryan Reviewed-by: Harry Yoo (Oracle) Reviewed-by: Hao Li Link: https://patch.msgid.link/20260727-b4-objext_split-v3-3-c29ef0f1f257@kernel.org Signed-off-by: Vlastimil Babka (SUSE) --- include/linux/memcontrol.h | 13 ------------- mm/slab.h | 13 +++++++++++++ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index e1f46a0016fc..93869cc35c25 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1440,19 +1440,6 @@ static inline void mem_cgroup_flush_workqueue(void) { } static inline int mem_cgroup_init(void) { return 0; } #endif /* CONFIG_MEMCG */ -/* - * Extended information for slab objects stored as an array in page->memcg_data - * if MEMCG_DATA_OBJEXTS is set. - */ -struct slabobj_ext { -#ifdef CONFIG_MEMCG - struct obj_cgroup *objcg; -#endif -#ifdef CONFIG_MEM_ALLOC_PROFILING - union codetag_ref ref; -#endif -} __aligned(8); - static inline struct lruvec *parent_lruvec(struct lruvec *lruvec) { struct mem_cgroup *memcg; diff --git a/mm/slab.h b/mm/slab.h index 01535e1e2d3c..7bd361447c54 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -549,6 +549,19 @@ static inline bool need_kmalloc_no_objext(void) return false; } +/* + * Extended information for slab objects stored as an array in page->memcg_data + * if MEMCG_DATA_OBJEXTS is set. + */ +struct slabobj_ext { +#ifdef CONFIG_MEMCG + struct obj_cgroup *objcg; +#endif +#ifdef CONFIG_MEM_ALLOC_PROFILING + union codetag_ref ref; +#endif +} __aligned(8); + #ifdef CONFIG_SLAB_OBJ_EXT /* From b215520d41db22dcac6d10e2e2a757d3433802b6 Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Mon, 27 Jul 2026 14:53:58 +0200 Subject: [PATCH 14/31] mm/slab: make slab_obj_ext() determine object index All callers perform the same obj_to_index() calculation to pass the index. Simplify by passing object pointer instead and determining the index by slab_obj_ext(). Reviewed-by: Suren Baghdasaryan Reviewed-by: Harry Yoo (Oracle) Reviewed-by: Hao Li Link: https://patch.msgid.link/20260727-b4-objext_split-v3-4-c29ef0f1f257@kernel.org Signed-off-by: Vlastimil Babka (SUSE) --- mm/memcontrol.c | 12 +++--------- mm/slab.h | 19 +++++++++++-------- mm/slub.c | 22 +++++++--------------- 3 files changed, 21 insertions(+), 32 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 6dc4888a90f3..4e427286a88a 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2865,15 +2865,13 @@ struct mem_cgroup *mem_cgroup_from_obj_slab(struct slab *slab, void *p) */ unsigned long obj_exts; struct slabobj_ext *obj_ext; - unsigned int off; obj_exts = slab_obj_exts(slab); if (!obj_exts) return NULL; get_slab_obj_exts(obj_exts); - off = obj_to_index(slab->slab_cache, slab, p); - obj_ext = slab_obj_ext(slab, obj_exts, off); + obj_ext = slab_obj_ext(slab->slab_cache, slab, obj_exts, p); if (obj_ext->objcg) { struct obj_cgroup *objcg = obj_ext->objcg; @@ -3541,7 +3539,6 @@ bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, size_t obj_size = obj_full_size(s); struct obj_cgroup *objcg; struct slab *slab; - unsigned long off; size_t i; /* @@ -3616,8 +3613,7 @@ bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, obj_exts = slab_obj_exts(slab); get_slab_obj_exts(obj_exts); - off = obj_to_index(s, slab, p[i]); - obj_ext = slab_obj_ext(slab, obj_exts, off); + obj_ext = slab_obj_ext(s, slab, obj_exts, p[i]); obj_cgroup_get(objcg); obj_ext->objcg = objcg; put_slab_obj_exts(obj_exts); @@ -3635,10 +3631,8 @@ void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, struct obj_cgroup *objcg; struct slabobj_ext *obj_ext; struct obj_stock_pcp *stock; - unsigned int off; - off = obj_to_index(s, slab, p[i]); - obj_ext = slab_obj_ext(slab, obj_exts, off); + obj_ext = slab_obj_ext(s, slab, obj_exts, p[i]); objcg = obj_ext->objcg; if (!objcg) continue; diff --git a/mm/slab.h b/mm/slab.h index 7bd361447c54..451b50b7f237 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -579,7 +579,7 @@ struct slabobj_ext { * obj_exts = slab_obj_exts(slab); * if (obj_exts) { * get_slab_obj_exts(obj_exts); - * obj_ext = slab_obj_ext(slab, obj_exts, obj_to_index(s, slab, obj)); + * obj_ext = slab_obj_ext(s, slab, obj_exts, obj); * // do something with obj_ext * put_slab_obj_exts(obj_exts); * } @@ -639,21 +639,24 @@ static inline unsigned int slab_get_stride(struct slab *slab) /* * slab_obj_ext - get the pointer to the slab object extension metadata * associated with an object in a slab. + * @s: cache that the slab belongs to * @slab: a pointer to the slab struct * @obj_exts: a pointer to the object extension vector - * @index: an index of the object + * @obj: a pointer to the object * * Returns a pointer to the object extension associated with the object. * Must be called within a section covered by get/put_slab_obj_exts(). */ -static inline struct slabobj_ext *slab_obj_ext(struct slab *slab, - unsigned long obj_exts, - unsigned int index) +static inline struct slabobj_ext * +slab_obj_ext(struct kmem_cache *s, struct slab *slab, unsigned long obj_exts, + const void *obj) { struct slabobj_ext *obj_ext; + unsigned int index; VM_WARN_ON_ONCE(obj_exts != slab_obj_exts(slab)); + index = obj_to_index(s, slab, obj); obj_ext = (struct slabobj_ext *)(obj_exts + slab_get_stride(slab) * index); return kasan_reset_tag(obj_ext); @@ -669,9 +672,9 @@ static inline unsigned long slab_obj_exts(struct slab *slab) return 0; } -static inline struct slabobj_ext *slab_obj_ext(struct slab *slab, - unsigned long obj_exts, - unsigned int index) +static inline struct slabobj_ext * +slab_obj_ext(struct kmem_cache *s, struct slab *slab, unsigned long obj_exts, + const void *obj) { return NULL; } diff --git a/mm/slub.c b/mm/slub.c index b94482830637..a74f1866c958 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2073,11 +2073,10 @@ static inline void mark_obj_codetag_empty(const void *obj) obj_slab = virt_to_slab(obj); slab_exts = slab_obj_exts(obj_slab); if (slab_exts) { + struct slabobj_ext *ext; + get_slab_obj_exts(slab_exts); - unsigned int offs = obj_to_index(obj_slab->slab_cache, - obj_slab, obj); - struct slabobj_ext *ext = slab_obj_ext(obj_slab, - slab_exts, offs); + ext = slab_obj_ext(obj_slab->slab_cache, obj_slab, slab_exts, obj); if (unlikely(is_codetag_empty(&ext->ref))) { put_slab_obj_exts(slab_exts); @@ -2365,10 +2364,8 @@ __alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags, * check should be added before alloc_tag_add(). */ if (obj_exts) { - unsigned int obj_idx = obj_to_index(s, slab, object); - get_slab_obj_exts(obj_exts); - obj_ext = slab_obj_ext(slab, obj_exts, obj_idx); + obj_ext = slab_obj_ext(s, slab, obj_exts, object); alloc_tag_add(&obj_ext->ref, current->alloc_tag, s->size); put_slab_obj_exts(obj_exts); } else { @@ -2389,7 +2386,6 @@ static noinline void __alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, int objects) { - int i; unsigned long obj_exts; /* slab->obj_exts might not be NULL if it was created for MEMCG accounting. */ @@ -2401,13 +2397,11 @@ __alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p return; get_slab_obj_exts(obj_exts); - for (i = 0; i < objects; i++) { - unsigned int off = obj_to_index(s, slab, p[i]); - + for (int i = 0; i < objects; i++) { if (is_kfence_address(p[i])) continue; - alloc_tag_sub(&slab_obj_ext(slab, obj_exts, off)->ref, s->size); + alloc_tag_sub(&slab_obj_ext(s, slab, obj_exts, p[i])->ref, s->size); } put_slab_obj_exts(obj_exts); } @@ -2492,7 +2486,6 @@ bool memcg_slab_post_charge(void *p, gfp_t flags) struct kmem_cache *s; struct page *page; struct slab *slab; - unsigned long off; page = virt_to_page(p); if (PageLargeKmalloc(page)) { @@ -2532,8 +2525,7 @@ bool memcg_slab_post_charge(void *p, gfp_t flags) obj_exts = slab_obj_exts(slab); if (obj_exts) { get_slab_obj_exts(obj_exts); - off = obj_to_index(s, slab, p); - obj_ext = slab_obj_ext(slab, obj_exts, off); + obj_ext = slab_obj_ext(s, slab, obj_exts, p); if (unlikely(obj_ext->objcg)) { put_slab_obj_exts(obj_exts); return true; From 060324c5531021f64514a0372790b177eed87559 Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Mon, 27 Jul 2026 14:53:59 +0200 Subject: [PATCH 15/31] mm/slab: abstract slabobj_ext.objcg access In preparation for changes to the structure, abstract getting and setting the objcg field with slab_obj_ext_objcg() and slab_obj_ext_set_objcg(). Rename the field to _objcg to make an unexpected direct access a compile error. The helpers take a slab pointer, which is currently unused, but will be used by a debug check later. Since there is no slab pointer easily available in __kfence_free(), just drop the debug check there. The whole memcg_kmem accounting in kfence is to be removed later anyway. Otherwise, no functional change intended. Reviewed-by: Hao Li Reviewed-by: Suren Baghdasaryan Reviewed-by: Harry Yoo (Oracle) Link: https://patch.msgid.link/20260727-b4-objext_split-v3-5-c29ef0f1f257@kernel.org Signed-off-by: Vlastimil Babka (SUSE) --- mm/kfence/core.c | 3 --- mm/memcontrol.c | 14 ++++++++------ mm/slab.h | 17 ++++++++++++++++- mm/slub.c | 2 +- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/mm/kfence/core.c b/mm/kfence/core.c index 6577bd76954e..5608a37f2b4d 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -1248,9 +1248,6 @@ void __kfence_free(void *addr) { struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr); -#ifdef CONFIG_MEMCG - KFENCE_WARN_ON(meta->obj_exts.objcg); -#endif /* * If the objects of the cache are SLAB_TYPESAFE_BY_RCU, defer freeing * the object, as the object page may be recycled for other-typed diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 4e427286a88a..68e98fb3350e 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2865,6 +2865,7 @@ struct mem_cgroup *mem_cgroup_from_obj_slab(struct slab *slab, void *p) */ unsigned long obj_exts; struct slabobj_ext *obj_ext; + struct obj_cgroup *objcg; obj_exts = slab_obj_exts(slab); if (!obj_exts) @@ -2872,9 +2873,8 @@ struct mem_cgroup *mem_cgroup_from_obj_slab(struct slab *slab, void *p) get_slab_obj_exts(obj_exts); obj_ext = slab_obj_ext(slab->slab_cache, slab, obj_exts, p); - if (obj_ext->objcg) { - struct obj_cgroup *objcg = obj_ext->objcg; - + objcg = slab_obj_ext_objcg(slab, obj_ext); + if (objcg) { put_slab_obj_exts(obj_exts); return obj_cgroup_memcg(objcg); } @@ -3614,8 +3614,10 @@ bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, obj_exts = slab_obj_exts(slab); get_slab_obj_exts(obj_exts); obj_ext = slab_obj_ext(s, slab, obj_exts, p[i]); + obj_cgroup_get(objcg); - obj_ext->objcg = objcg; + slab_obj_ext_set_objcg(slab, obj_ext, objcg); + put_slab_obj_exts(obj_exts); } @@ -3633,11 +3635,11 @@ void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, struct obj_stock_pcp *stock; obj_ext = slab_obj_ext(s, slab, obj_exts, p[i]); - objcg = obj_ext->objcg; + objcg = slab_obj_ext_objcg(slab, obj_ext); if (!objcg) continue; - obj_ext->objcg = NULL; + slab_obj_ext_set_objcg(slab, obj_ext, NULL); stock = trylock_stock(); __refill_obj_stock(objcg, stock, obj_size, true); diff --git a/mm/slab.h b/mm/slab.h index 451b50b7f237..ea014311e29f 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -555,7 +555,7 @@ static inline bool need_kmalloc_no_objext(void) */ struct slabobj_ext { #ifdef CONFIG_MEMCG - struct obj_cgroup *objcg; + struct obj_cgroup *_objcg; #endif #ifdef CONFIG_MEM_ALLOC_PROFILING union codetag_ref ref; @@ -662,6 +662,21 @@ slab_obj_ext(struct kmem_cache *s, struct slab *slab, unsigned long obj_exts, return kasan_reset_tag(obj_ext); } +#ifdef CONFIG_MEMCG +static inline struct obj_cgroup * +slab_obj_ext_objcg(struct slab *slab, struct slabobj_ext *obj_ext) +{ + return obj_ext->_objcg; +} + +static inline void +slab_obj_ext_set_objcg(struct slab *slab, struct slabobj_ext *obj_ext, + struct obj_cgroup *objcg) +{ + obj_ext->_objcg = objcg; +} +#endif + int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags); diff --git a/mm/slub.c b/mm/slub.c index a74f1866c958..ed18860fa0fd 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2526,7 +2526,7 @@ bool memcg_slab_post_charge(void *p, gfp_t flags) if (obj_exts) { get_slab_obj_exts(obj_exts); obj_ext = slab_obj_ext(s, slab, obj_exts, p); - if (unlikely(obj_ext->objcg)) { + if (unlikely(slab_obj_ext_objcg(slab, obj_ext))) { put_slab_obj_exts(obj_exts); return true; } From e684ee3bb54084eda60eb3d55e220f70b22b67ff Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Mon, 27 Jul 2026 14:54:00 +0200 Subject: [PATCH 16/31] mm/slab: abstract slabobj_ext.ref access In preparation for changes to the structure, abstract access to the ref field with a slab_obj_ext_codetag_ref() function. Rename the field to _ctref to make an unexpected direct access a compile error. No functional change intended. Reviewed-by: Suren Baghdasaryan Reviewed-by: Hao Li Reviewed-by: Harry Yoo (Oracle) Link: https://patch.msgid.link/20260727-b4-objext_split-v3-6-c29ef0f1f257@kernel.org Signed-off-by: Vlastimil Babka (SUSE) --- mm/slab.h | 10 +++++++++- mm/slub.c | 42 ++++++++++++++++++++++++++++-------------- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/mm/slab.h b/mm/slab.h index ea014311e29f..c806afdef2ee 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -558,7 +558,7 @@ struct slabobj_ext { struct obj_cgroup *_objcg; #endif #ifdef CONFIG_MEM_ALLOC_PROFILING - union codetag_ref ref; + union codetag_ref _ctref; #endif } __aligned(8); @@ -677,6 +677,14 @@ slab_obj_ext_set_objcg(struct slab *slab, struct slabobj_ext *obj_ext, } #endif +#ifdef CONFIG_MEM_ALLOC_PROFILING +static inline union codetag_ref * +slab_obj_ext_codetag_ref(struct slab *slab, struct slabobj_ext *obj_ext) +{ + return &obj_ext->_ctref; +} +#endif + int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags); diff --git a/mm/slub.c b/mm/slub.c index ed18860fa0fd..9d60d192737c 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2074,18 +2074,20 @@ static inline void mark_obj_codetag_empty(const void *obj) slab_exts = slab_obj_exts(obj_slab); if (slab_exts) { struct slabobj_ext *ext; + union codetag_ref *ref; get_slab_obj_exts(slab_exts); ext = slab_obj_ext(obj_slab->slab_cache, obj_slab, slab_exts, obj); + ref = slab_obj_ext_codetag_ref(obj_slab, ext); - if (unlikely(is_codetag_empty(&ext->ref))) { + if (unlikely(is_codetag_empty(ref))) { put_slab_obj_exts(slab_exts); return; } /* codetag should be NULL here */ - WARN_ON(ext->ref.ct); - set_codetag_empty(&ext->ref); + WARN_ON(ref->ct); + set_codetag_empty(ref); put_slab_obj_exts(slab_exts); } } @@ -2095,19 +2097,22 @@ static inline bool mark_failed_objexts_alloc(struct slab *slab) return cmpxchg(&slab->obj_exts, 0, OBJEXTS_ALLOC_FAIL) == 0; } -static inline void handle_failed_objexts_alloc(unsigned long obj_exts, - struct slabobj_ext *vec, unsigned int objects) +static inline void handle_failed_objexts_alloc(struct slab *slab, + unsigned long obj_exts, struct slabobj_ext *vec) { /* * If vector previously failed to allocate then we have live * objects with no tag reference. Mark all references in this * vector as empty to avoid warnings later on. */ - if (obj_exts == OBJEXTS_ALLOC_FAIL) { - unsigned int i; + if (obj_exts != OBJEXTS_ALLOC_FAIL) + return; - for (i = 0; i < objects; i++) - set_codetag_empty(&vec[i].ref); + for (unsigned int i = 0; i < slab->objects; i++) { + union codetag_ref *ref = slab_obj_ext_codetag_ref(slab, vec); + + set_codetag_empty(ref); + vec++; } } @@ -2115,8 +2120,8 @@ static inline void handle_failed_objexts_alloc(unsigned long obj_exts, static inline void mark_obj_codetag_empty(const void *obj) {} static inline bool mark_failed_objexts_alloc(struct slab *slab) { return false; } -static inline void handle_failed_objexts_alloc(unsigned long obj_exts, - struct slabobj_ext *vec, unsigned int objects) {} +static inline void handle_failed_objexts_alloc(struct slab *slab, + unsigned long obj_exts, struct slabobj_ext *vec) {} #endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ @@ -2184,7 +2189,7 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, #endif retry: old_exts = READ_ONCE(slab->obj_exts); - handle_failed_objexts_alloc(old_exts, vec, slab->objects); + handle_failed_objexts_alloc(slab, old_exts, vec); if (new_slab) { /* @@ -2364,9 +2369,15 @@ __alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags, * check should be added before alloc_tag_add(). */ if (obj_exts) { + union codetag_ref *ref; + get_slab_obj_exts(obj_exts); + obj_ext = slab_obj_ext(s, slab, obj_exts, object); - alloc_tag_add(&obj_ext->ref, current->alloc_tag, s->size); + ref = slab_obj_ext_codetag_ref(slab, obj_ext); + + alloc_tag_add(ref, current->alloc_tag, s->size); + put_slab_obj_exts(obj_exts); } else { alloc_tag_set_inaccurate(current->alloc_tag); @@ -2398,10 +2409,13 @@ __alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p get_slab_obj_exts(obj_exts); for (int i = 0; i < objects; i++) { + struct slabobj_ext *ext; + if (is_kfence_address(p[i])) continue; - alloc_tag_sub(&slab_obj_ext(s, slab, obj_exts, p[i])->ref, s->size); + ext = slab_obj_ext(s, slab, obj_exts, p[i]); + alloc_tag_sub(slab_obj_ext_codetag_ref(slab, ext), s->size); } put_slab_obj_exts(obj_exts); } From b5bc35ace2c5c03440fb6fcf09e855a90cd89ffd Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Mon, 27 Jul 2026 14:54:01 +0200 Subject: [PATCH 17/31] mm/slab: replace slab.stride with obj_exts_in_object The stride field is used to convert object index to an slabobj_ext so both compact arrays (kmalloc() or in-slab-leftover) and spread in-object-padding obj_ext layouts are supported. In practice thus the stride is always sizeof(slabobj_ext) or s->size. This simplifies the calculations, but with the upcoming slabobj_ext handling changes, it will be easier to stop storing the stride and instead just have a flag whether obj_ext is in the object padding. obj_exts_in_object() can then rely on this flag and slab_obj_ext() can use that to determine the stride. No functional change intended. Reviewed-by: Suren Baghdasaryan Reviewed-by: Hao Li Link: https://patch.msgid.link/20260727-b4-objext_split-v3-7-c29ef0f1f257@kernel.org Reviewed-by: Harry Yoo (Oracle) Signed-off-by: Vlastimil Babka (SUSE) --- mm/slab.h | 44 +++++++++++++++++++++++++------------------- mm/slub.c | 44 +++++++++++++++++--------------------------- 2 files changed, 42 insertions(+), 46 deletions(-) diff --git a/mm/slab.h b/mm/slab.h index c806afdef2ee..7c372c21b1c1 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -81,10 +81,10 @@ struct freelist_counters { #ifdef CONFIG_64BIT /* * Some optimizations use free bits in 'counters' field - * to save memory. In case ->stride field is not available, + * to save memory. If these free bits are not available, * such optimizations are disabled. */ - unsigned int stride; + unsigned obj_exts_in_object:1; #endif }; }; @@ -617,22 +617,20 @@ static inline void put_slab_obj_exts(unsigned long obj_exts) } #ifdef CONFIG_64BIT -static inline void slab_set_stride(struct slab *slab, unsigned int stride) +static inline bool obj_exts_in_object(struct slab *slab) { - slab->stride = stride; -} -static inline unsigned int slab_get_stride(struct slab *slab) -{ - return slab->stride; + /* + * Note we cannot rely on the SLAB_OBJ_EXT_IN_OBJ flag here and need to + * check the per-slab bit. A cache can have SLAB_OBJ_EXT_IN_OBJ set, but + * allocations within_slab_leftover are preferred. And those may be + * possible or not depending on the particular slab's size. + */ + return slab->obj_exts_in_object; } #else -static inline void slab_set_stride(struct slab *slab, unsigned int stride) +static inline bool obj_exts_in_object(struct slab *slab) { - VM_WARN_ON_ONCE(stride != sizeof(struct slabobj_ext)); -} -static inline unsigned int slab_get_stride(struct slab *slab) -{ - return sizeof(struct slabobj_ext); + return false; } #endif @@ -657,8 +655,15 @@ slab_obj_ext(struct kmem_cache *s, struct slab *slab, unsigned long obj_exts, VM_WARN_ON_ONCE(obj_exts != slab_obj_exts(slab)); index = obj_to_index(s, slab, obj); - obj_ext = (struct slabobj_ext *)(obj_exts + - slab_get_stride(slab) * index); + + if (!obj_exts_in_object(slab)) { + obj_ext = ((struct slabobj_ext *)obj_exts) + index; + } else { + unsigned int stride = s->size; + + obj_ext = (struct slabobj_ext *)(obj_exts + index * stride); + } + return kasan_reset_tag(obj_ext); } @@ -702,9 +707,10 @@ slab_obj_ext(struct kmem_cache *s, struct slab *slab, unsigned long obj_exts, return NULL; } -static inline void slab_set_stride(struct slab *slab, unsigned int stride) { } -static inline unsigned int slab_get_stride(struct slab *slab) { return 0; } - +static inline bool obj_exts_in_object(struct slab *slab) +{ + return false; +} #endif /* CONFIG_SLAB_OBJ_EXT */ diff --git a/mm/slub.c b/mm/slub.c index 9d60d192737c..1f3f51c42bbb 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -870,18 +870,6 @@ static inline bool obj_exts_in_slab(struct kmem_cache *s, struct slab *slab) #endif #if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT) -static bool obj_exts_in_object(struct kmem_cache *s, struct slab *slab) -{ - /* - * Note we cannot rely on the SLAB_OBJ_EXT_IN_OBJ flag here and need to - * check the stride. A cache can have SLAB_OBJ_EXT_IN_OBJ set, but - * allocations within_slab_leftover are preferred. And those may be - * possible or not depending on the particular slab's size. - */ - return obj_exts_in_slab(s, slab) && - (slab_get_stride(slab) == s->size); -} - static unsigned int obj_exts_offset_in_object(struct kmem_cache *s) { unsigned int offset = get_info_end(s); @@ -896,16 +884,20 @@ static unsigned int obj_exts_offset_in_object(struct kmem_cache *s) return offset; } -#else -static inline bool obj_exts_in_object(struct kmem_cache *s, struct slab *slab) -{ - return false; -} +static inline void slab_set_obj_exts_in_object(struct slab *slab) +{ + slab->obj_exts_in_object = 1; +} +#else static inline unsigned int obj_exts_offset_in_object(struct kmem_cache *s) { return 0; } + +static inline void slab_set_obj_exts_in_object(struct slab *slab) +{ +} #endif #ifdef CONFIG_SLUB_DEBUG @@ -1206,7 +1198,7 @@ static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p) off += kasan_metadata_size(s, false); - if (obj_exts_in_object(s, slab)) + if (obj_exts_in_object(slab)) off += sizeof(struct slabobj_ext); if (off != size_from_object(s)) @@ -1411,7 +1403,7 @@ static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p) off += kasan_metadata_size(s, false); - if (obj_exts_in_object(s, slab)) + if (obj_exts_in_object(slab)) off += sizeof(struct slabobj_ext); if (size_from_object(s) == off) @@ -1439,7 +1431,7 @@ slab_pad_check(struct kmem_cache *s, struct slab *slab) length = slab_size(slab); end = start + length; - if (obj_exts_in_slab(s, slab) && !obj_exts_in_object(s, slab)) { + if (obj_exts_in_slab(s, slab) && !obj_exts_in_object(slab)) { remainder = length; remainder -= obj_exts_offset_in_slab(s, slab); remainder -= obj_exts_size_in_slab(slab); @@ -2256,9 +2248,6 @@ static void alloc_slab_obj_exts_early(struct kmem_cache *s, struct slab *slab) void *addr; unsigned long obj_exts; - /* Initialize stride early to avoid memory ordering issues */ - slab_set_stride(slab, sizeof(struct slabobj_ext)); - if (!need_slab_obj_exts(s)) return; @@ -2292,7 +2281,7 @@ static void alloc_slab_obj_exts_early(struct kmem_cache *s, struct slab *slab) obj_exts |= MEMCG_DATA_OBJEXTS; #endif slab->obj_exts = obj_exts; - slab_set_stride(slab, s->size); + slab_set_obj_exts_in_object(slab); } } @@ -3405,9 +3394,10 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, stat(s, ORDER_FALLBACK); } + /* Initializes frozen, inuse, and any extra 64bit-only flags */ + slab->counters = 0; + slab->objects = oo_objects(oo); - slab->inuse = 0; - slab->frozen = 0; slab->slab_cache = s; @@ -6540,7 +6530,7 @@ static inline size_t slab_ksize(struct slab *slab) */ if (s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_STORE_USER)) return s->inuse; - else if (obj_exts_in_object(s, slab)) + else if (obj_exts_in_object(slab)) return s->inuse; /* * Else we can use all the padding etc for the allocation From f901ed647994859884c89620c26c8092d538af01 Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Mon, 27 Jul 2026 14:54:02 +0200 Subject: [PATCH 18/31] mm/slab: change struct slabobj_ext to a union Currently, struct slabobj_ext can hold both objcg pointer and codetag_ref (when both are compile-enabled) and there is an array of as many slabobj_ext instances as there are objects in a slab. This makes the layout fixed so even if codetag_ref is unused (because memory allocation profiling is disabled), the space for them is allocated and wasted. Similarly, some caches (currently kmalloc_normal) do not ever need objcg pointers, leading to wasted memory with memory allocation profiling enabled. To make this more flexible, change the layout so that struct slabobj_ext becomes a union of objcg pointer and codetag_ref (to ensure uniform size; in practice both are the same size anyway). The slabobj_ext array then can have twice as many elements as before. For cache locality purposes, the effective memory layout is unchanged, so objcg and codetag ref for a given object are still adjacent. cache_obj_ext_size() returns the effective size of (0-2) struct slabobj_ext's for a cache, slab_obj_ext_size() for a slab. Currently both return a constant value derived from the config options, but will be made dynamic later. Replace all sizeof(slabobj_ext) usage with these. No functional change intended, the layout is still effectively static. Reviewed-by: Suren Baghdasaryan Link: https://patch.msgid.link/20260727-b4-objext_split-v3-8-c29ef0f1f257@kernel.org Reviewed-by: Harry Yoo (Oracle) Reviewed-by: Hao Li Signed-off-by: Vlastimil Babka (SUSE) --- mm/slab.h | 49 +++++++++++++++++++++++++++++++++++++++---------- mm/slub.c | 19 +++++++++++-------- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/mm/slab.h b/mm/slab.h index 7c372c21b1c1..ef25c89b52d0 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -550,18 +550,42 @@ static inline bool need_kmalloc_no_objext(void) } /* - * Extended information for slab objects stored as an array in page->memcg_data - * if MEMCG_DATA_OBJEXTS is set. + * Extended information for slab objects stored as a pointer to an array in + * slab->obj_exts (aliasing page->memcg_data) if MEMCG_DATA_OBJEXTS is set. */ struct slabobj_ext { + /* + * All elements of the union should be pointer-sized to avoid memory + * waste + */ + union { #ifdef CONFIG_MEMCG - struct obj_cgroup *_objcg; + struct obj_cgroup *_objcg; #endif #ifdef CONFIG_MEM_ALLOC_PROFILING - union codetag_ref _ctref; + union codetag_ref _ctref; #endif + }; } __aligned(8); +static inline size_t cache_obj_ext_size(struct kmem_cache *s) +{ + size_t sz = 0; + + if (IS_ENABLED(CONFIG_MEMCG)) + sz += 1; + + if (IS_ENABLED(CONFIG_MEM_ALLOC_PROFILING)) + sz += 1; + + return sizeof(struct slabobj_ext) * sz; +} + +static inline size_t slab_obj_ext_size(struct slab *slab) +{ + return cache_obj_ext_size(slab->slab_cache); +} + #ifdef CONFIG_SLAB_OBJ_EXT /* @@ -651,18 +675,18 @@ slab_obj_ext(struct kmem_cache *s, struct slab *slab, unsigned long obj_exts, { struct slabobj_ext *obj_ext; unsigned int index; + unsigned int stride; VM_WARN_ON_ONCE(obj_exts != slab_obj_exts(slab)); index = obj_to_index(s, slab, obj); - if (!obj_exts_in_object(slab)) { - obj_ext = ((struct slabobj_ext *)obj_exts) + index; - } else { - unsigned int stride = s->size; + if (!obj_exts_in_object(slab)) + stride = slab_obj_ext_size(slab); + else + stride = s->size; - obj_ext = (struct slabobj_ext *)(obj_exts + index * stride); - } + obj_ext = (struct slabobj_ext *)(obj_exts + index * stride); return kasan_reset_tag(obj_ext); } @@ -671,6 +695,7 @@ slab_obj_ext(struct kmem_cache *s, struct slab *slab, unsigned long obj_exts, static inline struct obj_cgroup * slab_obj_ext_objcg(struct slab *slab, struct slabobj_ext *obj_ext) { + /* if objcg exists, it comes first, so we don't need to do anything */ return obj_ext->_objcg; } @@ -678,6 +703,7 @@ static inline void slab_obj_ext_set_objcg(struct slab *slab, struct slabobj_ext *obj_ext, struct obj_cgroup *objcg) { + /* if objcg exists, it comes first, so we don't need to do anything */ obj_ext->_objcg = objcg; } #endif @@ -686,6 +712,9 @@ slab_obj_ext_set_objcg(struct slab *slab, struct slabobj_ext *obj_ext, static inline union codetag_ref * slab_obj_ext_codetag_ref(struct slab *slab, struct slabobj_ext *obj_ext) { + if (IS_ENABLED(CONFIG_MEMCG)) + obj_ext += 1; + return &obj_ext->_ctref; } #endif diff --git a/mm/slub.c b/mm/slub.c index 1f3f51c42bbb..cfa370bc8130 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -803,7 +803,7 @@ static inline bool need_slab_obj_exts(struct kmem_cache *s) static inline unsigned int obj_exts_size_in_slab(struct slab *slab) { - return sizeof(struct slabobj_ext) * slab->objects; + return slab_obj_ext_size(slab) * slab->objects; } static inline unsigned long obj_exts_offset_in_slab(struct kmem_cache *s, @@ -1199,7 +1199,7 @@ static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p) off += kasan_metadata_size(s, false); if (obj_exts_in_object(slab)) - off += sizeof(struct slabobj_ext); + off += slab_obj_ext_size(slab); if (off != size_from_object(s)) /* Beginning of the filler is the free pointer */ @@ -1404,7 +1404,7 @@ static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p) off += kasan_metadata_size(s, false); if (obj_exts_in_object(slab)) - off += sizeof(struct slabobj_ext); + off += slab_obj_ext_size(slab); if (size_from_object(s) == off) return 1; @@ -2092,6 +2092,8 @@ static inline bool mark_failed_objexts_alloc(struct slab *slab) static inline void handle_failed_objexts_alloc(struct slab *slab, unsigned long obj_exts, struct slabobj_ext *vec) { + unsigned int stride; + /* * If vector previously failed to allocate then we have live * objects with no tag reference. Mark all references in this @@ -2100,11 +2102,13 @@ static inline void handle_failed_objexts_alloc(struct slab *slab, if (obj_exts != OBJEXTS_ALLOC_FAIL) return; + stride = slab_obj_ext_size(slab) / sizeof(*vec); + for (unsigned int i = 0; i < slab->objects; i++) { union codetag_ref *ref = slab_obj_ext_codetag_ref(slab, vec); set_codetag_empty(ref); - vec++; + vec += stride; } } @@ -2130,7 +2134,7 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, unsigned long new_exts; unsigned long old_exts; struct slabobj_ext *vec; - size_t sz = sizeof(struct slabobj_ext) * slab->objects; + size_t sz = slab_obj_ext_size(slab) * slab->objects; gfp &= ~OBJCGS_CLEAR_MASK; /* @@ -2273,8 +2277,7 @@ static void alloc_slab_obj_exts_early(struct kmem_cache *s, struct slab *slab) get_slab_obj_exts(obj_exts); for_each_object(addr, s, slab_address(slab), slab->objects) - memset(kasan_reset_tag(addr) + offset, 0, - sizeof(struct slabobj_ext)); + memset(kasan_reset_tag(addr) + offset, 0, slab_obj_ext_size(slab)); put_slab_obj_exts(obj_exts); #ifdef CONFIG_MEMCG @@ -7933,7 +7936,7 @@ static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s) aligned_size = ALIGN(size, s->align); #if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT) if (slab_args_unmergeable(args, s->flags) && - (aligned_size - size >= sizeof(struct slabobj_ext))) + (aligned_size - size >= cache_obj_ext_size(s))) s->flags |= SLAB_OBJ_EXT_IN_OBJ; #endif size = aligned_size; From f3521fbec2b2839698eba2b7013ed20119416e6d Mon Sep 17 00:00:00 2001 From: "Harry Yoo (Oracle)" Date: Wed, 29 Jul 2026 17:20:10 +0900 Subject: [PATCH 19/31] mm/slab: handle the !allow_spin case in kfree_rcu_sheaf() Teach kfree_rcu_sheaf() how to handle the !allow_spin case. Try to get an empty sheaf from pcs->spare or the barn even when spinning is not allowed. Unlike __pcs_replace_full_main(), try harder to allocate an empty sheaf because the fallback path will be more expensive than kfree_nolock(). Now that slab has internal alloc_flags to describe context, introduce free_flags analogously and convert free_flags to alloc_flags when allocating memory in the free path. When trylock fails or the kernel observes non-NULL pcs->rcu_free after lock acquisition, free the sheaf instead of putting it to the barn. This is rare and not worth complicating the code. Since call_rcu() cannot be called in an unknown context, kfree_rcu_sheaf() fails when the rcu sheaf becomes full. Link: https://lore.kernel.org/linux-mm/872bd673-3d45-4111-8a41-31185db3ece5@kernel.org Reviewed-by: Vlastimil Babka (SUSE) Signed-off-by: Harry Yoo (Oracle) Link: https://patch.msgid.link/20260729-kfree_rcu_nolock-v5-2-a28cdcda9673@kernel.org Reviewed-by: Shengming Hu Signed-off-by: Vlastimil Babka (SUSE) --- mm/slab.h | 18 +++++++++++++++++- mm/slab_common.c | 2 +- mm/slub.c | 34 ++++++++++++++++++++++++++-------- 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/mm/slab.h b/mm/slab.h index f5e336b6b6b0..ba08da09da36 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -24,11 +24,27 @@ #define SLAB_ALLOC_NO_RECURSE 0x04 /* prevent kmalloc() recursion */ #define SLAB_ALLOC_NO_OBJ_EXT 0x08 /* prevent obj_exts array allocation */ +#define SLAB_FREE_DEFAULT 0x00 /* no flags */ +#define SLAB_FREE_NOLOCK 0x01 /* spinning not allowed */ + +static inline unsigned int to_alloc_flags(unsigned int free_flags) +{ + if (free_flags & SLAB_FREE_NOLOCK) + return SLAB_ALLOC_NOLOCK; + else + return SLAB_ALLOC_DEFAULT; +} + static inline bool alloc_flags_allow_spinning(const unsigned int alloc_flags) { return !(alloc_flags & SLAB_ALLOC_NOLOCK); } +static inline bool free_flags_allow_spinning(const unsigned int free_flags) +{ + return !(free_flags & SLAB_FREE_NOLOCK); +} + void *__kmalloc_flags_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t flags, unsigned int alloc_flags, int node) __assume_kmalloc_alignment __alloc_size(1); @@ -436,7 +452,7 @@ static inline bool is_kmalloc_normal(struct kmem_cache *s) return !(s->flags & (SLAB_CACHE_DMA|SLAB_ACCOUNT|SLAB_RECLAIM_ACCOUNT|SLAB_NO_OBJ_EXT)); } -bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj); +bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags); void flush_all_rcu_sheaves(void); void flush_rcu_sheaves_on_cache(struct kmem_cache *s); diff --git a/mm/slab_common.c b/mm/slab_common.c index 03ecac12cd86..2475fc0e5e41 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -1622,7 +1622,7 @@ static bool kfree_rcu_sheaf(void *obj) s = slab->slab_cache; if (likely(!IS_ENABLED(CONFIG_NUMA) || slab_nid(slab) == numa_mem_id())) - return __kfree_rcu_sheaf(s, obj); + return __kfree_rcu_sheaf(s, obj, SLAB_FREE_DEFAULT); return false; } diff --git a/mm/slub.c b/mm/slub.c index 8273780fb4ae..325f3818cd6f 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2789,7 +2789,8 @@ static inline struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, return __alloc_empty_sheaf(s, gfp, alloc_flags, s->sheaf_capacity); } -static void free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf) +static void __free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf, + unsigned int free_flags) { /* * If the sheaf was created with SLAB_ALLOC_NO_RECURSE flag then its @@ -2801,11 +2802,20 @@ static void free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf) mark_obj_codetag_empty(sheaf); VM_WARN_ON_ONCE(sheaf->size > 0); - kfree(sheaf); + + if (unlikely(free_flags & SLAB_FREE_NOLOCK)) + kfree_nolock(sheaf); + else + kfree(sheaf); stat(s, SHEAF_FREE); } +static void free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf) +{ + __free_empty_sheaf(s, sheaf, SLAB_FREE_DEFAULT); +} + static unsigned int refill_objects(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int min, unsigned int max); @@ -6042,10 +6052,11 @@ static void rcu_free_sheaf(struct rcu_head *head) */ static DEFINE_WAIT_OVERRIDE_MAP(kfree_rcu_sheaf_map, LD_WAIT_CONFIG); -bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) +bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags) { struct slub_percpu_sheaves *pcs; struct slab_sheaf *rcu_sheaf; + bool allow_spin = free_flags_allow_spinning(free_flags); if (WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT))) return false; @@ -6058,9 +6069,10 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) pcs = this_cpu_ptr(s->cpu_sheaves); if (unlikely(!pcs->rcu_free)) { - struct slab_sheaf *empty; struct node_barn *barn; + unsigned int alloc_flags = to_alloc_flags(free_flags); + gfp_t gfp = allow_spin ? GFP_NOWAIT : __GFP_NOWARN; /* Bootstrap or debug cache, fall back */ if (unlikely(!cache_has_sheaves(s))) { @@ -6080,7 +6092,7 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) goto fail; } - empty = barn_get_empty_sheaf(barn, true); + empty = barn_get_empty_sheaf(barn, allow_spin); if (empty) { pcs->rcu_free = empty; @@ -6089,20 +6101,20 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) local_unlock(&s->cpu_sheaves->lock); - empty = alloc_empty_sheaf(s, GFP_NOWAIT, SLAB_ALLOC_DEFAULT); + empty = alloc_empty_sheaf(s, gfp, alloc_flags); if (!empty) goto fail; if (!local_trylock(&s->cpu_sheaves->lock)) { - barn_put_empty_sheaf(barn, empty); + __free_empty_sheaf(s, empty, free_flags); goto fail; } pcs = this_cpu_ptr(s->cpu_sheaves); if (unlikely(pcs->rcu_free)) - barn_put_empty_sheaf(barn, empty); + __free_empty_sheaf(s, empty, free_flags); else pcs->rcu_free = empty; } @@ -6120,6 +6132,12 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) if (likely(rcu_sheaf->size < s->sheaf_capacity)) { rcu_sheaf = NULL; } else { + if (unlikely(!allow_spin)) { + /* call_rcu() cannot be called in an unknown context */ + rcu_sheaf->size--; + local_unlock(&s->cpu_sheaves->lock); + goto fail; + } pcs->rcu_free = NULL; rcu_sheaf->node = numa_node_id(); } From f8e0c305995fa2931e77a6e669e8362458ae4eba Mon Sep 17 00:00:00 2001 From: "Harry Yoo (Oracle)" Date: Wed, 29 Jul 2026 17:20:11 +0900 Subject: [PATCH 20/31] mm/slab: use call_rcu() in unknown context if irqs are enabled call_rcu() disables IRQs with local_irq_save() to protect its per-cpu data structures. Therefore, if IRQs are not disabled, they cannot be corrupted by reentrance into call_rcu(). So fall back to the deferred path only when !allow_spin && irqs_disabled(). The RCU subsystem does not guarantee this contractually, and this optimization relies on RCU's implementation details. Ideally, it should be removed once call_rcu_nolock() is supported by the RCU subsystem. Link: https://lore.kernel.org/linux-mm/CAADnVQKRVD5ZSnEKbZZU7w86gHbGHUug2pvzpgZTngNS+fg4rw@mail.gmail.com Suggested-by: Alexei Starovoitov Signed-off-by: Harry Yoo (Oracle) Link: https://patch.msgid.link/20260729-kfree_rcu_nolock-v5-3-a28cdcda9673@kernel.org Reviewed-by: Shengming Hu Signed-off-by: Vlastimil Babka (SUSE) --- mm/slub.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 325f3818cd6f..31f94bdad3b0 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -6132,8 +6132,12 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags) if (likely(rcu_sheaf->size < s->sheaf_capacity)) { rcu_sheaf = NULL; } else { - if (unlikely(!allow_spin)) { - /* call_rcu() cannot be called in an unknown context */ + /* + * With !allow_spin, we might have interrupted call_rcu()'s + * IRQ-disabled critical section. If IRQs are not disabled, + * we know that's not the case. + */ + if (unlikely(!allow_spin && irqs_disabled())) { rcu_sheaf->size--; local_unlock(&s->cpu_sheaves->lock); goto fail; From 69abda97a09b2e7df26d21cb534f4930201b4186 Mon Sep 17 00:00:00 2001 From: "Harry Yoo (Oracle)" Date: Wed, 29 Jul 2026 17:20:12 +0900 Subject: [PATCH 21/31] mm/slab: extend deferred free mechanism to handle rcu sheaves __kfree_rcu_sheaf() cannot invoke call_rcu() when spinning is not allowed and IRQs are disabled. To relax the limitation, extend the deferred free fallback so that a full rcu sheaf can be submitted to call_rcu() via the existing IRQ work. Since the deferred mechanism does more than deferred freeing of objects, rename the struct to deferred_percpu_work and adjust names accordingly. When a sheaf is queued on an IRQ work, it is detached from pcs->rcu_free but call_rcu() is not invoked until the irq_work runs. To keep the kvfree_rcu barrier's promise, call irq_work_sync() on each CPU before calling rcu_barrier(). In the meantime, remove the TODO item as apparently there is no simple and effective way to achieve that. This is because, unlike sheaves, kfree_rcu() batches objects from different caches together. Suggested-by: Alexei Starovoitov Reviewed-by: Pedro Falcato Reviewed-by: Vlastimil Babka (SUSE) Signed-off-by: Harry Yoo (Oracle) Link: https://patch.msgid.link/20260729-kfree_rcu_nolock-v5-4-a28cdcda9673@kernel.org Signed-off-by: Vlastimil Babka (SUSE) --- mm/slab.h | 2 +- mm/slab_common.c | 7 ++-- mm/slub.c | 85 ++++++++++++++++++++++++++++-------------------- 3 files changed, 53 insertions(+), 41 deletions(-) diff --git a/mm/slab.h b/mm/slab.h index ba08da09da36..ddcf59230d2f 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -786,7 +786,7 @@ void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) void __check_heap_object(const void *ptr, unsigned long n, const struct slab *slab, bool to_user); -void defer_free_barrier(void); +void deferred_work_barrier(void); static inline bool slub_debug_orig_size(struct kmem_cache *s) { diff --git a/mm/slab_common.c b/mm/slab_common.c index 2475fc0e5e41..ed49b9abfab2 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -551,7 +551,7 @@ void kmem_cache_destroy(struct kmem_cache *s) } /* Wait for deferred work from kmalloc/kfree_nolock() */ - defer_free_barrier(); + deferred_work_barrier(); cpus_read_lock(); mutex_lock(&slab_mutex); @@ -2130,13 +2130,10 @@ void kvfree_rcu_barrier_on_cache(struct kmem_cache *s) cpus_read_lock(); flush_rcu_sheaves_on_cache(s); cpus_read_unlock(); + deferred_work_barrier(); rcu_barrier(); } - /* - * TODO: Introduce a version of __kvfree_rcu_barrier() that works - * on a specific slab cache. - */ __kvfree_rcu_barrier(); } EXPORT_SYMBOL_GPL(kvfree_rcu_barrier_on_cache); diff --git a/mm/slub.c b/mm/slub.c index 31f94bdad3b0..c0cc6d3126c7 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -418,6 +418,8 @@ struct slab_sheaf { union { struct rcu_head rcu_head; struct list_head barn_list; + /* only used to defer call_rcu() in unknown context */ + struct llist_node llnode; /* only used for prefilled sheafs */ struct { unsigned int capacity; @@ -4046,6 +4048,20 @@ static void flush_all(struct kmem_cache *s) cpus_read_unlock(); } +struct deferred_percpu_work { + struct llist_head objects; + struct llist_head rcu_sheaves; + struct irq_work work; +}; + +static void deferred_percpu_work_fn(struct irq_work *work); + +static DEFINE_PER_CPU(struct deferred_percpu_work, deferred_percpu_work) = { + .objects = LLIST_HEAD_INIT(objects), + .rcu_sheaves = LLIST_HEAD_INIT(rcu_sheaves), + .work = IRQ_WORK_INIT(deferred_percpu_work_fn), +}; + static void flush_rcu_sheaf(struct work_struct *w) { struct slub_percpu_sheaves *pcs; @@ -4117,6 +4133,7 @@ void flush_all_rcu_sheaves(void) mutex_unlock(&slab_mutex); cpus_read_unlock(); + deferred_work_barrier(); rcu_barrier(); } @@ -6132,16 +6149,6 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags) if (likely(rcu_sheaf->size < s->sheaf_capacity)) { rcu_sheaf = NULL; } else { - /* - * With !allow_spin, we might have interrupted call_rcu()'s - * IRQ-disabled critical section. If IRQs are not disabled, - * we know that's not the case. - */ - if (unlikely(!allow_spin && irqs_disabled())) { - rcu_sheaf->size--; - local_unlock(&s->cpu_sheaves->lock); - goto fail; - } pcs->rcu_free = NULL; rcu_sheaf->node = numa_node_id(); } @@ -6150,8 +6157,22 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags) * we flush before local_unlock to make sure a racing * flush_all_rcu_sheaves() doesn't miss this sheaf */ - if (rcu_sheaf) - call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf); + if (rcu_sheaf) { + /* + * With !allow_spin, we might have interrupted call_rcu()'s + * IRQ-disabled critical section. If IRQs are not disabled, + * we know that's not the case. + */ + if (unlikely(!allow_spin && irqs_disabled())) { + struct deferred_percpu_work *dpw; + + dpw = this_cpu_ptr(&deferred_percpu_work); + if (llist_add(&rcu_sheaf->llnode, &dpw->rcu_sheaves)) + irq_work_queue(&dpw->work); + } else { + call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf); + } + } local_unlock(&s->cpu_sheaves->lock); @@ -6336,31 +6357,21 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) } } -struct defer_free { - struct llist_head objects; - struct irq_work work; -}; - -static void free_deferred_objects(struct irq_work *work); - -static DEFINE_PER_CPU(struct defer_free, defer_free_objects) = { - .objects = LLIST_HEAD_INIT(objects), - .work = IRQ_WORK_INIT(free_deferred_objects), -}; - /* * In PREEMPT_RT irq_work runs in per-cpu kthread, so it's safe * to take sleeping spin_locks from __slab_free(). * In !PREEMPT_RT irq_work will run after local_unlock_irqrestore(). */ -static void free_deferred_objects(struct irq_work *work) +static void deferred_percpu_work_fn(struct irq_work *work) { - struct defer_free *df = container_of(work, struct defer_free, work); - struct llist_head *objs = &df->objects; + struct deferred_percpu_work *dpw; + struct llist_head *objs, *rcu_sheaves; struct llist_node *llnode, *pos, *t; + struct slab_sheaf *sheaf, *next; - if (llist_empty(objs)) - return; + dpw = container_of(work, struct deferred_percpu_work, work); + rcu_sheaves = &dpw->rcu_sheaves; + objs = &dpw->objects; llnode = llist_del_all(objs); llist_for_each_safe(pos, t, llnode) { @@ -6384,27 +6395,31 @@ static void free_deferred_objects(struct irq_work *work) __slab_free(s, slab, x, x, 1, _THIS_IP_); stat(s, FREE_SLOWPATH); } + + llnode = llist_del_all(rcu_sheaves); + llist_for_each_entry_safe(sheaf, next, llnode, llnode) + call_rcu(&sheaf->rcu_head, rcu_free_sheaf); } static void defer_free(struct kmem_cache *s, void *head) { - struct defer_free *df; + struct deferred_percpu_work *dpw; guard(preempt)(); head = kasan_reset_tag(head); - df = this_cpu_ptr(&defer_free_objects); - if (llist_add(head + s->offset, &df->objects)) - irq_work_queue(&df->work); + dpw = this_cpu_ptr(&deferred_percpu_work); + if (llist_add(head + s->offset, &dpw->objects)) + irq_work_queue(&dpw->work); } -void defer_free_barrier(void) +void deferred_work_barrier(void) { int cpu; for_each_possible_cpu(cpu) - irq_work_sync(&per_cpu_ptr(&defer_free_objects, cpu)->work); + irq_work_sync(&per_cpu_ptr(&deferred_percpu_work, cpu)->work); } static __fastpath_inline From 2a8bb29ec9b202d3d7bd094c800e6990fee278ba Mon Sep 17 00:00:00 2001 From: "Harry Yoo (Oracle)" Date: Wed, 29 Jul 2026 17:20:13 +0900 Subject: [PATCH 22/31] mm/slab: allow kfree_rcu_sheaf() on PREEMPT_RT As suggested by Vlastimil Babka [1], kfree_rcu_sheaf() can be used on PREEMPT_RT if we always assume spinning is not allowed on PREEMPT_RT. This is because local_trylock and spinlock_t are safe to use with trylock and unlock as long as the kernel does not spin and the context is not NMI and not hardirq. Now that __kfree_rcu_sheaf() knows how to handle SLAB_FREE_NOLOCK, relax the limitation and try the sheaves path on PREEMPT_RT as well. Keep the lockdep map on non RT kernels. However, do not use the lockdep map on PREEMPT_RT to avoid suppressing valid lockdep warnings. As pointed by Vlastimil Babka [2], on PREEMPT_RT it is unnecessary to defer call_rcu() under IRQ-disabled section or raw spinlock. However, let us avoid adding more complexity as the scenario is not supposed to be common on PREEMPT_RT, with a hope that call_rcu_nolock() will be soon supported in RCU. Link: https://lore.kernel.org/linux-mm/6811cc17-8ee4-48c8-8cbf-6bf4d9f98162@kernel.org [1] Link: https://lore.kernel.org/linux-mm/40591888-3a87-433e-b3d2-cda1cab543be@kernel.org [2] Suggested-by: Vlastimil Babka (SUSE) Reviewed-by: Vlastimil Babka (SUSE) Signed-off-by: Harry Yoo (Oracle) Link: https://patch.msgid.link/20260729-kfree_rcu_nolock-v5-5-a28cdcda9673@kernel.org Signed-off-by: Vlastimil Babka (SUSE) --- mm/slab_common.c | 12 ++++++++++-- mm/slub.c | 17 ++++++++++------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/mm/slab_common.c b/mm/slab_common.c index ed49b9abfab2..e776fc4516c8 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -1612,6 +1612,14 @@ static bool kfree_rcu_sheaf(void *obj) { struct kmem_cache *s; struct slab *slab; + unsigned int free_flags = SLAB_FREE_DEFAULT; + + /* + * It is not safe to spin on PREEMPT_RT because the kernel might be + * holding a raw spinlock and slab acquires sleeping locks. + */ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + free_flags = SLAB_FREE_NOLOCK; if (is_vmalloc_addr(obj)) return false; @@ -1622,7 +1630,7 @@ static bool kfree_rcu_sheaf(void *obj) s = slab->slab_cache; if (likely(!IS_ENABLED(CONFIG_NUMA) || slab_nid(slab) == numa_mem_id())) - return __kfree_rcu_sheaf(s, obj, SLAB_FREE_DEFAULT); + return __kfree_rcu_sheaf(s, obj, free_flags); return false; } @@ -1971,7 +1979,7 @@ void kvfree_call_rcu(struct rcu_head *head, void *ptr) if (!head) might_sleep(); - if (!IS_ENABLED(CONFIG_PREEMPT_RT) && kfree_rcu_sheaf(ptr)) + if (kfree_rcu_sheaf(ptr)) return; // Queue the object but don't yet schedule the batch. diff --git a/mm/slub.c b/mm/slub.c index c0cc6d3126c7..4a07ba92b64f 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -6060,12 +6060,13 @@ static void rcu_free_sheaf(struct rcu_head *head) * kvfree_call_rcu() can be called while holding a raw_spinlock_t. Since * __kfree_rcu_sheaf() may acquire a spinlock_t (sleeping lock on PREEMPT_RT), * this would violate lock nesting rules. Therefore, kvfree_call_rcu() avoids - * this problem by bypassing the sheaves layer entirely on PREEMPT_RT. + * this problem by passing SLAB_FREE_NOLOCK on PREEMPT_RT. * * However, lockdep still complains that it is invalid to acquire spinlock_t * while holding raw_spinlock_t, even on !PREEMPT_RT where spinlock_t is a * spinning lock. Tell lockdep that acquiring spinlock_t is valid here - * by temporarily raising the wait-type to LD_WAIT_CONFIG. + * by temporarily raising the wait-type to LD_WAIT_CONFIG. Skip the lockdep map + * on PREEMPT_RT to avoid suppressing valid lockdep warnings. */ static DEFINE_WAIT_OVERRIDE_MAP(kfree_rcu_sheaf_map, LD_WAIT_CONFIG); @@ -6075,10 +6076,10 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags) struct slab_sheaf *rcu_sheaf; bool allow_spin = free_flags_allow_spinning(free_flags); - if (WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT))) - return false; + VM_WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT) && allow_spin); - lock_map_acquire_try(&kfree_rcu_sheaf_map); + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + lock_map_acquire_try(&kfree_rcu_sheaf_map); if (!local_trylock(&s->cpu_sheaves->lock)) goto fail; @@ -6177,12 +6178,14 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags) local_unlock(&s->cpu_sheaves->lock); stat(s, FREE_RCU_SHEAF); - lock_map_release(&kfree_rcu_sheaf_map); + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + lock_map_release(&kfree_rcu_sheaf_map); return true; fail: stat(s, FREE_RCU_SHEAF_FAIL); - lock_map_release(&kfree_rcu_sheaf_map); + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + lock_map_release(&kfree_rcu_sheaf_map); return false; } From 580258a19284fce98de9013baa81633ab9d93515 Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Mon, 27 Jul 2026 14:54:03 +0200 Subject: [PATCH 23/31] mm/slab: introduce slab_obj_ext_has_codetag() mem_alloc_profiling_enabled() allows evaluating (with a static key) if memory profiling is currently enabled. mem_profiling_support is a variable where false means it's not possible to enable it anymore, because the system was booted with "never" or it was later shut down. This is possible to query by mem_alloc_profiling_permanently_disabled(). To make slabobj_ext array size handling dynamic, we need a snapshot of mem_alloc_profiling_permanently_disabled() early in boot, so that's not affected by a later shutdown. We also need it to be static key based for performance. Neither mem_alloc_profiling_enabled() nor mem_alloc_profiling_permanently_disabled() satisfy this. Therefore introduce slab_obj_ext_has_codetag() with an underlying static key for that use case. Its state is made to reflect the result of mem_alloc_profiling_permanently_disabled() during kmem_cache_init(), which does happen after setup_early_mem_profiling(). Reviewed-by: Suren Baghdasaryan Link: https://patch.msgid.link/20260727-b4-objext_split-v3-9-c29ef0f1f257@kernel.org Reviewed-by: Hao Li Reviewed-by: Harry Yoo Signed-off-by: Vlastimil Babka (SUSE) --- mm/slab.h | 18 ++++++++++++++++++ mm/slub.c | 30 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/mm/slab.h b/mm/slab.h index ef25c89b52d0..60e0df5bc5c2 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -568,6 +568,22 @@ struct slabobj_ext { }; } __aligned(8); +#ifdef CONFIG_MEM_ALLOC_PROFILING +DECLARE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT, + slab_obj_ext_has_codetag_key); + +static inline bool slab_obj_ext_has_codetag(void) +{ + return static_branch_maybe(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT, + &slab_obj_ext_has_codetag_key); +} +#else +static inline bool slab_obj_ext_has_codetag(void) +{ + return false; +} +#endif + static inline size_t cache_obj_ext_size(struct kmem_cache *s) { size_t sz = 0; @@ -712,6 +728,8 @@ slab_obj_ext_set_objcg(struct slab *slab, struct slabobj_ext *obj_ext, static inline union codetag_ref * slab_obj_ext_codetag_ref(struct slab *slab, struct slabobj_ext *obj_ext) { + VM_WARN_ON_ONCE(!slab_obj_ext_has_codetag()); + if (IS_ENABLED(CONFIG_MEMCG)) obj_ext += 1; diff --git a/mm/slub.c b/mm/slub.c index cfa370bc8130..d2cbca1ade22 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -213,6 +213,11 @@ DEFINE_STATIC_KEY_FALSE(slub_debug_enabled); static DEFINE_STATIC_KEY_FALSE(strict_numa); #endif +#ifdef CONFIG_MEM_ALLOC_PROFILING +DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT, + slab_obj_ext_has_codetag_key); +#endif + /* Structure holding extra parameters for slab allocations */ struct slab_alloc_context { unsigned long caller_addr; @@ -2420,6 +2425,25 @@ alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, __alloc_tagging_slab_free_hook(s, slab, p, objects); } +/* + * Make sure the static key used by slab_obj_ext_has_codetag() reflects the + * value of !mem_alloc_profiling_permanently_disabled() + * + * Any later mem alloc profiling shutdown won't be reflected in the static key + * because obj_exts with codetags might already exist. + */ +static void __init slab_obj_ext_has_codetag_init(void) +{ + bool need_codetag = !mem_alloc_profiling_permanently_disabled(); + + if (need_codetag != static_key_enabled(&slab_obj_ext_has_codetag_key)) { + if (need_codetag) + static_branch_enable(&slab_obj_ext_has_codetag_key); + else + static_branch_disable(&slab_obj_ext_has_codetag_key); + } +} + #else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void @@ -2434,6 +2458,10 @@ alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, { } +static inline void slab_obj_ext_has_codetag_init(void) +{ +} + #endif /* CONFIG_MEM_ALLOC_PROFILING */ @@ -8546,6 +8574,8 @@ void __init kmem_cache_init(void) boot_kmem_cache_node; int node; + slab_obj_ext_has_codetag_init(); + if (debug_guardpage_minorder()) slub_max_order = 0; From c4ec6cb55750c80fc3d43b310eb7ccab33df3dcd Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Mon, 27 Jul 2026 14:54:04 +0200 Subject: [PATCH 24/31] mm/slab: reduce slabobj_ext memory with allocation profiling disabled When memory allocation profiling is compiled in but permanently disabled on boot with (implicit or explicit) "never" parameter, stop allocating (thus wasting) memory for the codetag_ref parts of slabobj_ext metadata. Do this by using the new slab_obj_ext_has_codetag() helper in cache_obj_ext_size(). Additionally add a slab_obj_ext_has_codetag() check in handle_failed_objexts_alloc(). The function might get called with memory allocation profiling disabled, when the obj_ext array is allocated for objcg pointers only. Setting codetag refs as empty is unnecessary in that case, and with them not allocated anymore would now result in memory corruption. Reviewed-by: Suren Baghdasaryan Link: https://patch.msgid.link/20260727-b4-objext_split-v3-10-c29ef0f1f257@kernel.org Reviewed-by: Hao Li Reviewed-by: Harry Yoo Signed-off-by: Vlastimil Babka (SUSE) --- mm/slab.h | 2 +- mm/slub.c | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/mm/slab.h b/mm/slab.h index 60e0df5bc5c2..041cda2d7395 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -591,7 +591,7 @@ static inline size_t cache_obj_ext_size(struct kmem_cache *s) if (IS_ENABLED(CONFIG_MEMCG)) sz += 1; - if (IS_ENABLED(CONFIG_MEM_ALLOC_PROFILING)) + if (slab_obj_ext_has_codetag()) sz += 1; return sizeof(struct slabobj_ext) * sz; diff --git a/mm/slub.c b/mm/slub.c index d2cbca1ade22..a24428ae32fb 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2064,6 +2064,9 @@ static inline void mark_obj_codetag_empty(const void *obj) struct slab *obj_slab; unsigned long slab_exts; + if (!slab_obj_ext_has_codetag()) + return; + if (is_kfence_address(obj)) return; @@ -2099,6 +2102,9 @@ static inline void handle_failed_objexts_alloc(struct slab *slab, { unsigned int stride; + if (!slab_obj_ext_has_codetag()) + return; + /* * If vector previously failed to allocate then we have live * objects with no tag reference. Mark all references in this From acc6fdade62c11d822f7b73f16092ebd365fc1c2 Mon Sep 17 00:00:00 2001 From: "Harry Yoo (Oracle)" Date: Wed, 29 Jul 2026 17:20:14 +0900 Subject: [PATCH 25/31] mm/slab: introduce struct kvfree_rcu_head for kvfree_rcu batching rcu_head is overkill for kvfree_rcu() because the callback function is always either kfree(), vfree(), or free_large_kmalloc(), and thus there is no need for a function pointer. kvfree_rcu batching reuses the field to store the start address of an object, however, this is not strictly needed because we can calculate the start address in the slowpath. For the purpose of kvfree_rcu batching, it is sufficient to implement a linked list using a single pointer. Introduce a new struct called kvfree_rcu_head (the name was suggested by Vlastimil Babka), which is similar to rcu_head but is only a single pointer to build a linked list, without a function pointer, when CONFIG_KVFREE_RCU_BATCHED=y. When kvfree_rcu is not batched, kvfree_rcu_head is the same size as rcu_head. Note that shrinking struct kvfree_rcu_head on CONFIG_KVFREE_RCU_BATCHED=n kernels would inevitably require additional complexity and also some sort of batching (which defeats the purpose of the config option) because it cannot fall back to call_rcu(). For now there are no user-visible changes to the API. k[v]free_rcu() simply casts rcu_head to kvfree_rcu_head. While this does not affect the API, it allows kfree_rcu_nolock() to reuse kvfree_rcu batching as a fallback when trylock or sheaf allocation fails. Stop storing the object pointer in rcu_head.func and instead calculate the object's start address in kvfree_rcu_list(). Factor out the existing logic to calculate the start address from kvfree_rcu_cb() to kvmalloc_obj_start_addr(). To avoid losing the KASAN tag, calculate the offset and subtract it from the address of the kvfree_rcu_head. Signed-off-by: Harry Yoo (Oracle) Link: https://patch.msgid.link/20260729-kfree_rcu_nolock-v5-6-a28cdcda9673@kernel.org Signed-off-by: Vlastimil Babka (SUSE) --- include/linux/rcupdate.h | 10 ++++++---- include/linux/types.h | 10 ++++++++++ include/trace/events/rcu.h | 2 +- mm/slab.h | 32 ++++++++++++++++++++++++++++++ mm/slab_common.c | 21 ++++++++++---------- mm/slub.c | 40 +++++++++----------------------------- 6 files changed, 68 insertions(+), 47 deletions(-) diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 5e95acc33989..ef5bb6981133 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -1098,19 +1098,21 @@ static inline void rcu_read_unlock_migrate(void) /* * In mm/slab_common.c, no suitable header to include here. */ -void kvfree_call_rcu(struct rcu_head *head, void *ptr); +void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr); /* * The BUILD_BUG_ON() makes sure the rcu_head offset can be handled. See the * comment of kfree_rcu() for details. */ -#define kvfree_rcu_arg_2(ptr, rhf) \ +#define kvfree_rcu_arg_2(ptr, kvrhf) \ do { \ typeof (ptr) ___p = (ptr); \ + struct kvfree_rcu_head *___head; \ \ if (___p) { \ - BUILD_BUG_ON(offsetof(typeof(*(ptr)), rhf) >= 4096); \ - kvfree_call_rcu(&((___p)->rhf), (void *) (___p)); \ + BUILD_BUG_ON(offsetof(typeof(*(ptr)), kvrhf) >= 4096); \ + ___head = (struct kvfree_rcu_head *) &(___p)->kvrhf; \ + kvfree_call_rcu(___head, (void *) (___p)); \ } \ } while (0) diff --git a/include/linux/types.h b/include/linux/types.h index 93166b0b0617..7d1d305a763e 100644 --- a/include/linux/types.h +++ b/include/linux/types.h @@ -255,6 +255,16 @@ struct callback_head { } __attribute__((aligned(sizeof(void *)))); #define rcu_head callback_head +#ifdef CONFIG_KVFREE_RCU_BATCHED +struct kvfree_rcu_head { + struct kvfree_rcu_head *next; +}; +#else +struct kvfree_rcu_head { + struct rcu_head head; +}; +#endif + typedef void (*rcu_callback_t)(struct rcu_head *head); typedef void (*call_rcu_func_t)(struct rcu_head *head, rcu_callback_t func); diff --git a/include/trace/events/rcu.h b/include/trace/events/rcu.h index 5fbdabe3faea..a74027126a2f 100644 --- a/include/trace/events/rcu.h +++ b/include/trace/events/rcu.h @@ -625,7 +625,7 @@ TRACE_EVENT_RCU(rcu_invoke_callback, */ TRACE_EVENT_RCU(rcu_invoke_kvfree_callback, - TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset), + TP_PROTO(const char *rcuname, struct kvfree_rcu_head *rhp, unsigned long offset), TP_ARGS(rcuname, rhp, offset), diff --git a/mm/slab.h b/mm/slab.h index ddcf59230d2f..1dea45402794 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -352,6 +352,37 @@ static inline int objs_per_slab(const struct kmem_cache *cache, return slab->objects; } +/* + * kvfree_rcu_head offset can be only less than page size. + * Calculate the start address while preserving the KASAN tag. + */ +static inline void *kvmalloc_obj_start_addr(void *head) +{ + unsigned long offset; + + if (unlikely(is_vmalloc_addr(head))) { + offset = offset_in_page(head); + } else { + struct slab *slab = virt_to_slab(head); + + if (!slab) { + offset = offset_in_page(head); + } else if (is_kfence_address(head)) { + offset = head - kfence_object_start(head); + } else { + struct kmem_cache *s = slab->slab_cache; + unsigned int idx = __obj_to_index(s, slab_address(slab), head); + void *obj = slab_address(slab) + s->size * idx; + + obj = fixup_red_left(s, obj); + obj = kasan_reset_tag(obj); + offset = kasan_reset_tag(head) - obj; + } + } + + return head - offset; +} + /* * State of the slab allocator. * @@ -787,6 +818,7 @@ void __check_heap_object(const void *ptr, unsigned long n, const struct slab *slab, bool to_user); void deferred_work_barrier(void); +void defer_kfree_rcu(struct kvfree_rcu_head *head); static inline bool slub_debug_orig_size(struct kmem_cache *s) { diff --git a/mm/slab_common.c b/mm/slab_common.c index e776fc4516c8..661f1fc24c2d 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -1282,11 +1282,11 @@ EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free); #ifndef CONFIG_KVFREE_RCU_BATCHED -void kvfree_call_rcu(struct rcu_head *head, void *ptr) +void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr) { if (head) { kasan_record_aux_stack(ptr); - call_rcu(head, kvfree_rcu_cb); + call_rcu(&head->head, kvfree_rcu_cb); return; } @@ -1363,7 +1363,7 @@ struct kvfree_rcu_bulk_data { struct kfree_rcu_cpu_work { struct rcu_work rcu_work; - struct rcu_head *head_free; + struct kvfree_rcu_head *head_free; struct rcu_gp_oldstate head_free_gp_snap; struct list_head bulk_head_free[FREE_N_CHANNELS]; struct kfree_rcu_cpu *krcp; @@ -1399,7 +1399,7 @@ struct kfree_rcu_cpu_work { struct kfree_rcu_cpu { // Objects queued on a linked list // through their rcu_head structures. - struct rcu_head *head; + struct kvfree_rcu_head *head; unsigned long head_gp_snap; atomic_t head_count; @@ -1540,12 +1540,12 @@ kvfree_rcu_bulk(struct kfree_rcu_cpu *krcp, } static void -kvfree_rcu_list(struct rcu_head *head) +kvfree_rcu_list(struct kvfree_rcu_head *head) { - struct rcu_head *next; + struct kvfree_rcu_head *next; for (; head; head = next) { - void *ptr = (void *) head->func; + void *ptr = kvmalloc_obj_start_addr(head); unsigned long offset = (void *) head - ptr; next = head->next; @@ -1569,7 +1569,7 @@ static void kfree_rcu_work(struct work_struct *work) unsigned long flags; struct kvfree_rcu_bulk_data *bnode, *n; struct list_head bulk_head[FREE_N_CHANNELS]; - struct rcu_head *head; + struct kvfree_rcu_head *head; struct kfree_rcu_cpu *krcp; struct kfree_rcu_cpu_work *krwp; struct rcu_gp_oldstate head_gp_snap; @@ -1700,7 +1700,7 @@ kvfree_rcu_drain_ready(struct kfree_rcu_cpu *krcp) { struct list_head bulk_ready[FREE_N_CHANNELS]; struct kvfree_rcu_bulk_data *bnode, *n; - struct rcu_head *head_ready = NULL; + struct kvfree_rcu_head *head_ready = NULL; unsigned long flags; int i; @@ -1963,7 +1963,7 @@ void __init kfree_rcu_scheduler_running(void) * be free'd in workqueue context. This allows us to: batch requests together to * reduce the number of grace periods during heavy kfree_rcu()/kvfree_rcu() load. */ -void kvfree_call_rcu(struct rcu_head *head, void *ptr) +void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr) { unsigned long flags; struct kfree_rcu_cpu *krcp; @@ -2001,7 +2001,6 @@ void kvfree_call_rcu(struct rcu_head *head, void *ptr) // Inline if kvfree_rcu(one_arg) call. goto unlock_return; - head->func = ptr; head->next = krcp->head; WRITE_ONCE(krcp->head, head); atomic_inc(&krcp->head_count); diff --git a/mm/slub.c b/mm/slub.c index 4a07ba92b64f..0a7024c8c090 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -6681,43 +6681,21 @@ static void free_large_kmalloc(struct page *page, void *object) */ void kvfree_rcu_cb(struct rcu_head *head) { - void *obj = head; - struct page *page; - struct slab *slab; - struct kmem_cache *s; - void *slab_addr; + void *obj; + + obj = kvmalloc_obj_start_addr(head); if (is_vmalloc_addr(obj)) { - obj = (void *) PAGE_ALIGN_DOWN((unsigned long)obj); vfree(obj); - return; - } - - page = virt_to_page(obj); - slab = page_slab(page); - if (!slab) { - /* - * rcu_head offset can be only less than page size so no need to - * consider allocation order - */ - obj = (void *) PAGE_ALIGN_DOWN((unsigned long)obj); - free_large_kmalloc(page, obj); - return; - } - - s = slab->slab_cache; - slab_addr = slab_address(slab); - - if (is_kfence_address(obj)) { - obj = kfence_object_start(obj); } else { - unsigned int idx = __obj_to_index(s, slab_addr, obj); + struct page *page = virt_to_page(obj); + struct slab *slab = page_slab(page); - obj = slab_addr + s->size * idx; - obj = fixup_red_left(s, obj); + if (slab) + slab_free(slab->slab_cache, slab, obj, _RET_IP_); + else + free_large_kmalloc(page, obj); } - - slab_free(s, slab, obj, _RET_IP_); } /** From 3bc999d944b35dead1755b2bde1911cd5892225e Mon Sep 17 00:00:00 2001 From: "Harry Yoo (Oracle)" Date: Wed, 29 Jul 2026 17:20:15 +0900 Subject: [PATCH 26/31] mm/slab: introduce kfree_rcu_nolock() Currently, k[v]free_rcu() cannot be called in unknown context since it could lead to a deadlock when called in the middle of k[v]free_rcu(). Make users' lives easier by introducing kfree_rcu_nolock() variant, now that kfree_rcu_sheaf() is available on PREEMPT_RT and __kfree_rcu_sheaf() handles unknown context. When sheaves path fails, kfree_rcu_nolock() falls back to defer_kfree_rcu() that uses an irq work to free the object via kvfree_call_rcu(). In most cases, the sheaves path is expected to succeed and therefore it's unnecessary to introduce additional complexity to the existing kvfree_rcu batching by teaching it how to handle unknown context. Since defer_kfree_rcu() can be called on caches without sheaves, move deferred_work_barrier() and rcu_barrier() outside the branch in kvfree_rcu_barrier_on_cache(). Now that deferred kvfree_rcu objects are submitted to kvfree_call_rcu() after deferred_work_barrier() and may end up in RCU sheaves, deferred_work_barrier() must be invoked before flush_rcu_sheaves_on_cache(). Since the RCU sheaf path has not been used on !KVFREE_RCU_BATCHED kernels, always fall back when kvfree_rcu() is not batched, for consistency. kvfree_rcu_barrier{,_on_cache()}() on !KVFREE_RCU_BATCHED are moved to mm/slab_common.c to invoke deferred_work_barrier() before rcu_barrier(). Signed-off-by: Harry Yoo (Oracle) Link: https://patch.msgid.link/20260729-kfree_rcu_nolock-v5-7-a28cdcda9673@kernel.org Signed-off-by: Vlastimil Babka (SUSE) --- include/linux/rcupdate.h | 22 +++++++++++++++++++ include/linux/slab.h | 16 +++----------- mm/slab_common.c | 47 ++++++++++++++++++++++++++++++++++++++-- mm/slub.c | 28 ++++++++++++++++++++++-- 4 files changed, 96 insertions(+), 17 deletions(-) diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index ef5bb6981133..aede77bd0387 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -1099,6 +1099,7 @@ static inline void rcu_read_unlock_migrate(void) * In mm/slab_common.c, no suitable header to include here. */ void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr); +void kfree_call_rcu_nolock(struct kvfree_rcu_head *head, void *ptr); /* * The BUILD_BUG_ON() makes sure the rcu_head offset can be handled. See the @@ -1124,6 +1125,27 @@ do { \ kvfree_call_rcu(NULL, (void *) (___p)); \ } while (0) +/** + * kfree_rcu_nolock() - a version of kfree_rcu() that can be called in any context. + * @ptr: pointer to kfree for double-argument invocations. + * @kvrhf: the name of the struct kvfree_rcu_head within the type of @ptr. + * + * With KVFREE_RCU_BATCHED, kfree_rcu_nolock() tries hard to free objects + * without any deferred processing, but may still defer freeing. + * Large kmalloc and vmalloc objects are always deferred. + * + * kfree_rcu_nolock() supports 2-arg variant only. + */ +#define kfree_rcu_nolock(ptr, kvrhf) \ +do { \ + typeof (ptr) ___p = (ptr); \ + \ + if (___p) { \ + BUILD_BUG_ON(offsetof(typeof(*(ptr)), kvrhf) >= 4096); \ + kfree_call_rcu_nolock(&((___p)->kvrhf), (void *) (___p)); \ + } \ +} while (0) + /* * Place this after a lock-acquisition primitive to guarantee that * an UNLOCK+LOCK pair acts as a full barrier. This guarantee applies diff --git a/include/linux/slab.h b/include/linux/slab.h index 32c9f8ed7ae2..066a25cc6966 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -1427,25 +1427,15 @@ extern void kvfree_sensitive(const void *addr, size_t len); unsigned int kmem_cache_size(struct kmem_cache *s); #ifndef CONFIG_KVFREE_RCU_BATCHED -static inline void kvfree_rcu_barrier(void) -{ - rcu_barrier(); -} - -static inline void kvfree_rcu_barrier_on_cache(struct kmem_cache *s) -{ - rcu_barrier(); -} - static inline void kfree_rcu_scheduler_running(void) { } #else +void kfree_rcu_scheduler_running(void); +#endif + void kvfree_rcu_barrier(void); void kvfree_rcu_barrier_on_cache(struct kmem_cache *s); -void kfree_rcu_scheduler_running(void); -#endif - /** * kmalloc_size_roundup - Report allocation bucket size for the given size * diff --git a/mm/slab_common.c b/mm/slab_common.c index 661f1fc24c2d..39ca4b4a182a 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -1280,6 +1280,33 @@ EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc); EXPORT_TRACEPOINT_SYMBOL(kfree); EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free); +void kfree_call_rcu_nolock(struct kvfree_rcu_head *head, void *ptr) +{ + struct slab *slab; + + if (!IS_ENABLED(CONFIG_KVFREE_RCU_BATCHED)) + goto fallback; + + if (unlikely(is_vmalloc_addr(ptr))) + goto fallback; + + slab = virt_to_slab(ptr); + if (unlikely(!slab)) + goto fallback; + + if (unlikely(IS_ENABLED(CONFIG_NUMA) && slab_nid(slab) != numa_mem_id())) + goto fallback; + + if (unlikely(!__kfree_rcu_sheaf(slab->slab_cache, ptr, SLAB_FREE_NOLOCK))) + goto fallback; + + return; + +fallback: + defer_kfree_rcu(head); +} +EXPORT_SYMBOL_GPL(kfree_call_rcu_nolock); + #ifndef CONFIG_KVFREE_RCU_BATCHED void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr) @@ -1297,6 +1324,20 @@ void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr) } EXPORT_SYMBOL_GPL(kvfree_call_rcu); +void kvfree_rcu_barrier(void) +{ + deferred_work_barrier(); + rcu_barrier(); +} +EXPORT_SYMBOL_GPL(kvfree_rcu_barrier); + +void kvfree_rcu_barrier_on_cache(struct kmem_cache *s) +{ + deferred_work_barrier(); + rcu_barrier(); +} +EXPORT_SYMBOL_GPL(kvfree_rcu_barrier_on_cache); + void __init kvfree_rcu_init(void) { } @@ -2133,14 +2174,16 @@ EXPORT_SYMBOL_GPL(kvfree_rcu_barrier); */ void kvfree_rcu_barrier_on_cache(struct kmem_cache *s) { + /* kfree_rcu_nolock() might have deferred frees even without sheaves */ + deferred_work_barrier(); + if (cache_has_sheaves(s)) { cpus_read_lock(); flush_rcu_sheaves_on_cache(s); cpus_read_unlock(); - deferred_work_barrier(); - rcu_barrier(); } + rcu_barrier(); __kvfree_rcu_barrier(); } EXPORT_SYMBOL_GPL(kvfree_rcu_barrier_on_cache); diff --git a/mm/slub.c b/mm/slub.c index 0a7024c8c090..b9aeb02a880f 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4050,6 +4050,7 @@ static void flush_all(struct kmem_cache *s) struct deferred_percpu_work { struct llist_head objects; + struct llist_head objects_by_rcu; struct llist_head rcu_sheaves; struct irq_work work; }; @@ -4058,6 +4059,7 @@ static void deferred_percpu_work_fn(struct irq_work *work); static DEFINE_PER_CPU(struct deferred_percpu_work, deferred_percpu_work) = { .objects = LLIST_HEAD_INIT(objects), + .objects_by_rcu = LLIST_HEAD_INIT(objects_by_rcu), .rcu_sheaves = LLIST_HEAD_INIT(rcu_sheaves), .work = IRQ_WORK_INIT(deferred_percpu_work_fn), }; @@ -4121,6 +4123,8 @@ void flush_all_rcu_sheaves(void) { struct kmem_cache *s; + deferred_work_barrier(); + cpus_read_lock(); mutex_lock(&slab_mutex); @@ -4133,7 +4137,6 @@ void flush_all_rcu_sheaves(void) mutex_unlock(&slab_mutex); cpus_read_unlock(); - deferred_work_barrier(); rcu_barrier(); } @@ -6368,13 +6371,14 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) static void deferred_percpu_work_fn(struct irq_work *work) { struct deferred_percpu_work *dpw; - struct llist_head *objs, *rcu_sheaves; + struct llist_head *objs, *objs_by_rcu, *rcu_sheaves; struct llist_node *llnode, *pos, *t; struct slab_sheaf *sheaf, *next; dpw = container_of(work, struct deferred_percpu_work, work); rcu_sheaves = &dpw->rcu_sheaves; objs = &dpw->objects; + objs_by_rcu = &dpw->objects_by_rcu; llnode = llist_del_all(objs); llist_for_each_safe(pos, t, llnode) { @@ -6399,6 +6403,14 @@ static void deferred_percpu_work_fn(struct irq_work *work) stat(s, FREE_SLOWPATH); } + llnode = llist_del_all(objs_by_rcu); + llist_for_each_safe(pos, t, llnode) { + void *head = pos; + void *objp = kvmalloc_obj_start_addr(head); + + kvfree_call_rcu(head, objp); + } + llnode = llist_del_all(rcu_sheaves); llist_for_each_entry_safe(sheaf, next, llnode, llnode) call_rcu(&sheaf->rcu_head, rcu_free_sheaf); @@ -6417,6 +6429,18 @@ static void defer_free(struct kmem_cache *s, void *head) irq_work_queue(&dpw->work); } +void defer_kfree_rcu(struct kvfree_rcu_head *head) +{ + struct deferred_percpu_work *dpw; + + guard(preempt)(); + + dpw = this_cpu_ptr(&deferred_percpu_work); + if (llist_add((struct llist_node *)head, &dpw->objects_by_rcu)) + irq_work_queue(&dpw->work); +} + +/* Must be called before flush_rcu_sheaves_on_cache() */ void deferred_work_barrier(void) { int cpu; From 7df60eeb6736013ee1555a19e261a7d14e84f250 Mon Sep 17 00:00:00 2001 From: "Harry Yoo (Oracle)" Date: Wed, 29 Jul 2026 17:20:16 +0900 Subject: [PATCH 27/31] slub_kunit: extend the test for kfree_rcu_nolock() When slub_kunit is not built-in, call kfree_rcu() and kfree_rcu_nolock() to test kfree_rcu_nolock() in slub_kunit. Rename the test case as the test covers more _nolock() APIs. Acked-by: Vlastimil Babka (SUSE) Signed-off-by: Harry Yoo (Oracle) Link: https://patch.msgid.link/20260729-kfree_rcu_nolock-v5-8-a28cdcda9673@kernel.org Signed-off-by: Vlastimil Babka (SUSE) --- lib/tests/slub_kunit.c | 47 +++++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/lib/tests/slub_kunit.c b/lib/tests/slub_kunit.c index 8c2b9911471e..e3b63f0338d5 100644 --- a/lib/tests/slub_kunit.c +++ b/lib/tests/slub_kunit.c @@ -162,7 +162,10 @@ static void test_kmalloc_redzone_access(struct kunit *test) } struct test_kfree_rcu_struct { - struct rcu_head rcu; + union { + struct rcu_head rcu; + struct kvfree_rcu_head kvrcu; + }; }; static void test_kfree_rcu(struct kunit *test) @@ -296,7 +299,7 @@ static void test_krealloc_redzone_zeroing(struct kunit *test) #if defined(CONFIG_PERF_EVENTS) || (defined(CONFIG_KPROBES) && defined(CONFIG_SMP)) #define NR_ITERATIONS 1000 #define NR_OBJECTS 1000 -static void *objects[NR_OBJECTS]; +static struct test_kfree_rcu_struct *objects[NR_OBJECTS]; struct test_nolock_context { struct kunit *test; @@ -311,15 +314,16 @@ struct test_nolock_context { #endif }; -static void test_kmalloc_kfree(void) +static void test_kmalloc_and_friends(void) { int i, j; + bool can_use_kfree_rcu = !IS_BUILTIN(CONFIG_SLUB_KUNIT_TEST); for (i = 0; i < NR_ITERATIONS; i++) { for (j = 0; j < NR_OBJECTS; j++) { - gfp_t gfp = (i % 2) ? GFP_KERNEL : GFP_KERNEL_ACCOUNT; + gfp_t gfp = (i & 1) ? GFP_KERNEL : GFP_KERNEL_ACCOUNT; - objects[j] = kmalloc(64, gfp); + objects[j] = kmalloc_obj(*objects[j], gfp); if (!objects[j]) { j--; while (j >= 0) @@ -328,26 +332,35 @@ static void test_kmalloc_kfree(void) } } - for (j = 0; j < NR_OBJECTS; j++) - kfree(objects[j]); + for (j = 0; j < NR_OBJECTS; j++) { + if (can_use_kfree_rcu && (i & 2)) + kfree_rcu(objects[j], rcu); + else + kfree(objects[j]); + } } } static void test_nolock(struct test_nolock_context *ctx) { - void *objp; + struct test_kfree_rcu_struct *objp; gfp_t gfp; + bool can_use_kfree_rcu = !IS_BUILTIN(CONFIG_SLUB_KUNIT_TEST); /* __GFP_ACCOUNT to test kmalloc_nolock() in alloc_slab_obj_exts() */ - gfp = (ctx->callback_count % 2) ? 0 : __GFP_ACCOUNT; - objp = kmalloc_nolock(64, gfp, NUMA_NO_NODE); + gfp = (ctx->callback_count & 1) ? 0 : __GFP_ACCOUNT; + objp = kmalloc_nolock(sizeof(*objp), gfp, NUMA_NO_NODE); if (objp) ctx->alloc_ok++; else ctx->alloc_fail++; - kfree_nolock(objp); + if (can_use_kfree_rcu && (ctx->callback_count & 2)) + kfree_rcu_nolock(objp, kvrcu); + else + kfree_nolock(objp); + ctx->callback_count++; } #endif @@ -397,14 +410,14 @@ static void disable_perf_events(struct test_nolock_context *ctx) perf_event_release_kernel(ctx->event); } -static void test_kmalloc_kfree_nolock_perf(struct kunit *test) +static void test_kmalloc_nolock_and_friends_perf(struct kunit *test) { struct test_nolock_context ctx = { .test = test }; if (!enable_perf_events(&ctx)) kunit_skip(test, "Failed to enable perf event, skipping"); - test_kmalloc_kfree(); + test_kmalloc_and_friends(); disable_perf_events(&ctx); KUNIT_EXPECT_EQ(test, 0, slab_errors); @@ -438,14 +451,14 @@ static void unregister_slab_kprobes(struct test_nolock_context *ctx) unregister_kprobe(&ctx->kprobe); } -static void test_kmalloc_kfree_nolock_kprobe(struct kunit *test) +static void test_kmalloc_nolock_and_friends_kprobe(struct kunit *test) { struct test_nolock_context ctx = { .test = test }; if (!register_slab_kprobes(&ctx)) kunit_skip(test, "Failed to register kprobe, skipping"); - test_kmalloc_kfree(); + test_kmalloc_and_friends(); unregister_slab_kprobes(&ctx); KUNIT_EXPECT_EQ(test, 0, slab_errors); @@ -477,10 +490,10 @@ static struct kunit_case test_cases[] = { KUNIT_CASE(test_leak_destroy), KUNIT_CASE(test_krealloc_redzone_zeroing), #ifdef CONFIG_PERF_EVENTS - KUNIT_CASE_SLOW(test_kmalloc_kfree_nolock_perf), + KUNIT_CASE_SLOW(test_kmalloc_nolock_and_friends_perf), #endif #if defined(CONFIG_KPROBES) && defined(CONFIG_SMP) - KUNIT_CASE_SLOW(test_kmalloc_kfree_nolock_kprobe), + KUNIT_CASE_SLOW(test_kmalloc_nolock_and_friends_kprobe), #endif {} }; From 648294a02bfcd0eddae51877e3b30f8bbb2d4bb6 Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Thu, 30 Jul 2026 16:32:46 +0200 Subject: [PATCH 28/31] mm/slab: stop exporting kvfree_rcu_barrier[_on_cache]() No module code calls these functions directly. Seems it was always the case. Remove the exports. Acked-by: Paul E. McKenney Reviewed-by: Harry Yoo Link: https://patch.msgid.link/20260730-unexport-barriers-v1-1-852f6641abe9@kernel.org Signed-off-by: Vlastimil Babka (SUSE) --- mm/slab_common.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mm/slab_common.c b/mm/slab_common.c index 39ca4b4a182a..e71a1b346b36 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -1329,14 +1329,12 @@ void kvfree_rcu_barrier(void) deferred_work_barrier(); rcu_barrier(); } -EXPORT_SYMBOL_GPL(kvfree_rcu_barrier); void kvfree_rcu_barrier_on_cache(struct kmem_cache *s) { deferred_work_barrier(); rcu_barrier(); } -EXPORT_SYMBOL_GPL(kvfree_rcu_barrier_on_cache); void __init kvfree_rcu_init(void) { @@ -2163,7 +2161,6 @@ void kvfree_rcu_barrier(void) flush_all_rcu_sheaves(); __kvfree_rcu_barrier(); } -EXPORT_SYMBOL_GPL(kvfree_rcu_barrier); /** * kvfree_rcu_barrier_on_cache - Wait for in-flight kvfree_rcu() calls on a @@ -2186,7 +2183,6 @@ void kvfree_rcu_barrier_on_cache(struct kmem_cache *s) rcu_barrier(); __kvfree_rcu_barrier(); } -EXPORT_SYMBOL_GPL(kvfree_rcu_barrier_on_cache); static unsigned long kfree_rcu_shrink_count(struct shrinker *shrink, struct shrink_control *sc) From 7def2e8549e5186cd4de97ab5ce56f7944d3da59 Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Mon, 27 Jul 2026 14:54:05 +0200 Subject: [PATCH 29/31] mm/slab: add cache_ and slab_needs_objcg() helpers Slabs of some caches never need the objcg part of struct slabobj_ext. Introduce helpers to query this for a cache or a slab. Introduce SLAB_MAY_ACCOUNT flag that is currently only internal and all caches have it set except: - KMALLOC_NORMAL caches, as long as KMALLOC_RECLAIM caches are separate - KMALLOC_NO_OBJ_EXT caches, if they exist For named caches we currently can't derive SLAB_MAY_ACCOUNT from SLAB_ACCOUNT because some caches might be created without SLAB_ACCOUNT and then used both with and without __GFP_ACCOUNT concurrently, allocating obj_ext arrays on demand. So just add the SLAB_MAY_ACCOUNT to all kmem caches, unless kmem accounting is disabled. This can be improved later by finding out all caches used with __GFP_ACCOUNT, creating them with the SLAB_MAY_ACCOUNT flag explicitly, and then ignoring __GFP_ACCOUNT for all other caches (possibly with a warning). To make the evaluation of slab_needs_objcg() faster in the allocation and free fast paths, add a obj_exts_needs_objcg flag into slab itself. This optimization is only available on 64bit architectures where free bits are available for the flag. Reviewed-by: Hao Li Link: https://patch.msgid.link/20260727-b4-objext_split-v3-11-c29ef0f1f257@kernel.org Reviewed-by: Harry Yoo Signed-off-by: Vlastimil Babka (SUSE) --- include/linux/slab.h | 3 +++ mm/kfence/core.c | 3 +++ mm/slab.h | 31 +++++++++++++++++++++++++++++-- mm/slab_common.c | 26 +++++++++++++++++++++----- mm/slub.c | 6 +++++- 5 files changed, 61 insertions(+), 8 deletions(-) diff --git a/include/linux/slab.h b/include/linux/slab.h index 32c9f8ed7ae2..e1915026e030 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -45,6 +45,7 @@ enum _slab_flag_bits { #endif #ifdef CONFIG_MEMCG _SLAB_ACCOUNT, + _SLAB_MAY_ACCOUNT, #endif #ifdef CONFIG_KASAN_GENERIC _SLAB_KASAN, @@ -204,8 +205,10 @@ enum _slab_flag_bits { */ #ifdef CONFIG_MEMCG # define SLAB_ACCOUNT __SLAB_FLAG_BIT(_SLAB_ACCOUNT) +# define SLAB_MAY_ACCOUNT __SLAB_FLAG_BIT(_SLAB_MAY_ACCOUNT) #else # define SLAB_ACCOUNT __SLAB_FLAG_UNUSED +# define SLAB_MAY_ACCOUNT __SLAB_FLAG_UNUSED #endif #ifdef CONFIG_KASAN_GENERIC diff --git a/mm/kfence/core.c b/mm/kfence/core.c index 5608a37f2b4d..7da3be9fbf5d 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -640,6 +640,9 @@ static unsigned long kfence_init_pool(void) struct slab *slab = page_slab(page); slab->obj_exts = (unsigned long)&kfence_metadata_init[i / 2 - 1].obj_exts | MEMCG_DATA_OBJEXTS; +#ifdef CONFIG_64BIT + slab->obj_exts_needs_objcg = 1; +#endif #endif } diff --git a/mm/slab.h b/mm/slab.h index 041cda2d7395..69cd3d9631ff 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -81,10 +81,11 @@ struct freelist_counters { #ifdef CONFIG_64BIT /* * Some optimizations use free bits in 'counters' field - * to save memory. If these free bits are not available, - * such optimizations are disabled. + * to save memory or CPU. If these free bits are not + * available, such optimizations are disabled. */ unsigned obj_exts_in_object:1; + unsigned obj_exts_needs_objcg:1; #endif }; }; @@ -584,6 +585,32 @@ static inline bool slab_obj_ext_has_codetag(void) } #endif +#ifdef CONFIG_MEMCG +static inline bool cache_needs_objcg(struct kmem_cache *cache) +{ + return (cache->flags & SLAB_MAY_ACCOUNT); +} + +static inline bool slab_needs_objcg(struct slab *slab) +{ +#ifdef CONFIG_64BIT + return slab->obj_exts_needs_objcg; +#else + return cache_needs_objcg(slab->slab_cache); +#endif +} +#else +static inline bool cache_needs_objcg(struct kmem_cache *cache) +{ + return false; +} + +static inline bool slab_needs_objcg(struct slab *slab) +{ + return false; +} +#endif + static inline size_t cache_obj_ext_size(struct kmem_cache *s) { size_t sz = 0; diff --git a/mm/slab_common.c b/mm/slab_common.c index 03ecac12cd86..1e1d3feec353 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -52,7 +52,7 @@ struct kmem_cache *kmem_cache; SLAB_OBJ_EXT_IN_OBJ) #define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \ - SLAB_CACHE_DMA32 | SLAB_ACCOUNT) + SLAB_CACHE_DMA32 | SLAB_ACCOUNT | SLAB_MAY_ACCOUNT) /* * Merge control. If this is set then no merging of slab caches will occur. @@ -359,6 +359,13 @@ struct kmem_cache *__kmem_cache_create_args(const char *name, goto out_unlock; } + /* + * For now we assume any cache can be used with __GFP_ACCOUNT and thus + * may need to store objcg pointers for objects + */ + if (!mem_cgroup_kmem_disabled()) + flags |= SLAB_MAY_ACCOUNT; + /* Fail closed on bad usersize of useroffset values. */ if (!IS_ENABLED(CONFIG_HARDENED_USERCOPY) || WARN_ON(!args->usersize && args->useroffset) || @@ -984,11 +991,20 @@ new_kmalloc_cache(int idx, enum kmalloc_cache_type type) #endif /* - * If CONFIG_MEMCG is enabled, disable cache merging for - * KMALLOC_NORMAL caches. + * If memcg_kmem is enabled and this is a KMALLOC_NORMAL cache and not + * aliased with any other type, make sure it's never merged with any other + * cache. + * + * In other cases the kmalloc cache may end up being used for a + * __GFP_ACCOUNT allocation so mark it as such. The exception is a + * KMALLOC_NO_OBJ_EXT cache. */ - if (IS_ENABLED(CONFIG_MEMCG) && (type == KMALLOC_NORMAL)) - flags |= SLAB_NO_MERGE; + if (!mem_cgroup_kmem_disabled()) { + if (type == KMALLOC_NORMAL && KMALLOC_RECLAIM != KMALLOC_NORMAL) + flags |= SLAB_NO_MERGE; + else if (!(flags & SLAB_NO_OBJ_EXT)) + flags |= SLAB_MAY_ACCOUNT; + } if (minalign > ARCH_KMALLOC_MINALIGN) { aligned_size = ALIGN(aligned_size, minalign); diff --git a/mm/slub.c b/mm/slub.c index a24428ae32fb..331dd9a9b99d 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2558,7 +2558,7 @@ bool memcg_slab_post_charge(void *p, gfp_t flags) * of slab_obj_exts being allocated from the same slab and thus the slab * becoming effectively unfreeable. */ - if (is_kmalloc_normal(s)) + if (!cache_needs_objcg(s)) return true; /* Ignore already charged objects. */ @@ -3436,6 +3436,10 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, slab->objects = oo_objects(oo); +#ifdef CONFIG_64BIT + if (cache_needs_objcg(s)) + slab->obj_exts_needs_objcg = 1; +#endif slab->slab_cache = s; kasan_poison_slab(slab); From d4404b0f5b8b0ff4656d018a0ddfafdbd78879e2 Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Mon, 27 Jul 2026 14:54:06 +0200 Subject: [PATCH 30/31] mm/slab: stop allocating objcg pointers when unnecessary Start using the slab_needs_objcg() helper to calculate slabobj_ext size. Caches that we know to never need objcg pointers (currently KMALLOC_NORMAL caches) will thus stop wasting memory on them when memory allocation profiling is enabled. For things to work properly, we need to also add slab_needs_objcg() checks to mem_cgroup_from_obj_slab() and memcg_slab_free_hook(), because when obj_exts array exists for a slab only due to mem_alloc profiling, we would otherwise attempt to access a non-existing objcg pointer in that slab. In slab_obj_ext_[set_]objcg() add debug warnings if called on a slab where slab_needs_objcg() is false. Reviewed-by: Hao Li Link: https://patch.msgid.link/20260727-b4-objext_split-v3-12-c29ef0f1f257@kernel.org Reviewed-by: Harry Yoo Signed-off-by: Vlastimil Babka (SUSE) --- mm/memcontrol.c | 3 +++ mm/slab.h | 18 +++++++++++++++--- mm/slub.c | 3 +++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 68e98fb3350e..9971726406d6 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2871,6 +2871,9 @@ struct mem_cgroup *mem_cgroup_from_obj_slab(struct slab *slab, void *p) if (!obj_exts) return NULL; + if (!slab_needs_objcg(slab)) + return NULL; + get_slab_obj_exts(obj_exts); obj_ext = slab_obj_ext(slab->slab_cache, slab, obj_exts, p); objcg = slab_obj_ext_objcg(slab, obj_ext); diff --git a/mm/slab.h b/mm/slab.h index 69cd3d9631ff..f76cf2e2ed90 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -615,7 +615,7 @@ static inline size_t cache_obj_ext_size(struct kmem_cache *s) { size_t sz = 0; - if (IS_ENABLED(CONFIG_MEMCG)) + if (cache_needs_objcg(s)) sz += 1; if (slab_obj_ext_has_codetag()) @@ -626,7 +626,15 @@ static inline size_t cache_obj_ext_size(struct kmem_cache *s) static inline size_t slab_obj_ext_size(struct slab *slab) { - return cache_obj_ext_size(slab->slab_cache); + size_t sz = 0; + + if (slab_needs_objcg(slab)) + sz += 1; + + if (slab_obj_ext_has_codetag()) + sz += 1; + + return sizeof(struct slabobj_ext) * sz; } #ifdef CONFIG_SLAB_OBJ_EXT @@ -738,6 +746,8 @@ slab_obj_ext(struct kmem_cache *s, struct slab *slab, unsigned long obj_exts, static inline struct obj_cgroup * slab_obj_ext_objcg(struct slab *slab, struct slabobj_ext *obj_ext) { + VM_WARN_ON_ONCE(!slab_needs_objcg(slab)); + /* if objcg exists, it comes first, so we don't need to do anything */ return obj_ext->_objcg; } @@ -746,6 +756,8 @@ static inline void slab_obj_ext_set_objcg(struct slab *slab, struct slabobj_ext *obj_ext, struct obj_cgroup *objcg) { + VM_WARN_ON_ONCE(!slab_needs_objcg(slab)); + /* if objcg exists, it comes first, so we don't need to do anything */ obj_ext->_objcg = objcg; } @@ -757,7 +769,7 @@ slab_obj_ext_codetag_ref(struct slab *slab, struct slabobj_ext *obj_ext) { VM_WARN_ON_ONCE(!slab_obj_ext_has_codetag()); - if (IS_ENABLED(CONFIG_MEMCG)) + if (slab_needs_objcg(slab)) obj_ext += 1; return &obj_ext->_ctref; diff --git a/mm/slub.c b/mm/slub.c index 331dd9a9b99d..9142f0f4c72f 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2513,6 +2513,9 @@ void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, if (likely(!obj_exts)) return; + if (!slab_needs_objcg(slab)) + return; + get_slab_obj_exts(obj_exts); __memcg_slab_free_hook(s, slab, p, objects, obj_exts); put_slab_obj_exts(obj_exts); From a6172cca157f3f50c9744a8b9b563f8b14371ed9 Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Mon, 27 Jul 2026 14:54:07 +0200 Subject: [PATCH 31/31] mm/slab, kfence, memcg: completely remove obj_ext for kfence objects We have already disabled memory allocation profiling for objects allocated for KFENCE to avoid complexity. KFENCE allocations are rare and there can be only CONFIG_KFENCE_NUM_OBJECTS (default to 255) outstanding ones at any time, so they are among noise in the profiling stats. For the same reasons, we can stop memcg_kmem accounting of kfence objects as their memory usage will be negligible wrt any practical memcg limits. This allows us simplifying the code and getting rid of is_kfence_address() checks in various places, including slab_obj_ext()'s usage of obj_to_index(). Instead we rely on the fact that slab_obj_exts() will now always return 0 for a kfence object's fake slab, which makes those places unreachable. All we need to do to keep this assumption valid is not to allocate obj_exts for kfence objects, so the checks need to guard alloc_slab_obj_exts() where necessary. Suggested-by: Harry Yoo Link: https://patch.msgid.link/20260727-b4-objext_split-v3-13-c29ef0f1f257@kernel.org Reviewed-by: Hao Li Signed-off-by: Vlastimil Babka (SUSE) --- mm/kfence/core.c | 12 ------------ mm/kfence/kfence.h | 3 --- mm/memcontrol.c | 8 +++++--- mm/slab.h | 6 +++++- mm/slub.c | 31 ++++++++++++++++--------------- 5 files changed, 26 insertions(+), 34 deletions(-) diff --git a/mm/kfence/core.c b/mm/kfence/core.c index 7da3be9fbf5d..90925c646c4c 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -636,14 +636,6 @@ static unsigned long kfence_init_pool(void) page = pfn_to_page(start_pfn + i); __SetPageSlab(page); -#ifdef CONFIG_MEMCG - struct slab *slab = page_slab(page); - slab->obj_exts = (unsigned long)&kfence_metadata_init[i / 2 - 1].obj_exts | - MEMCG_DATA_OBJEXTS; -#ifdef CONFIG_64BIT - slab->obj_exts_needs_objcg = 1; -#endif -#endif } /* @@ -707,10 +699,6 @@ static unsigned long kfence_init_pool(void) continue; page = pfn_to_page(start_pfn + i); -#ifdef CONFIG_MEMCG - struct slab *slab = page_slab(page); - slab->obj_exts = 0; -#endif __ClearPageSlab(page); } diff --git a/mm/kfence/kfence.h b/mm/kfence/kfence.h index 1f618f9b0d12..e6b4bf349ff7 100644 --- a/mm/kfence/kfence.h +++ b/mm/kfence/kfence.h @@ -102,9 +102,6 @@ struct kfence_metadata { struct kfence_track free_track __guarded_by(&lock); /* For updating alloc_covered on frees. */ u32 alloc_stack_hash __guarded_by(&lock); -#ifdef CONFIG_MEMCG - struct slabobj_ext obj_exts; -#endif }; #define KFENCE_METADATA_SIZE PAGE_ALIGN(sizeof(struct kfence_metadata) * \ diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 9971726406d6..d1312441a02b 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -3583,9 +3583,11 @@ bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, slab = virt_to_slab(p[i]); - if (!slab_obj_exts(slab) && - alloc_slab_obj_exts(slab, s, flags, slab_alloc_flags)) { - continue; + if (!slab_obj_exts(slab)) { + if (is_kfence_address(p[i])) + continue; + if (alloc_slab_obj_exts(slab, s, flags, slab_alloc_flags)) + continue; } /* diff --git a/mm/slab.h b/mm/slab.h index f76cf2e2ed90..e64614749692 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -730,7 +730,11 @@ slab_obj_ext(struct kmem_cache *s, struct slab *slab, unsigned long obj_exts, VM_WARN_ON_ONCE(obj_exts != slab_obj_exts(slab)); - index = obj_to_index(s, slab, obj); + /* + * KFENCE objects have NULL obj_exts and thus can't reach this + * and we don't need obj_to_index() + */ + index = __obj_to_index(s, slab_address(slab), obj); if (!obj_exts_in_object(slab)) stride = slab_obj_ext_size(slab); diff --git a/mm/slub.c b/mm/slub.c index 9142f0f4c72f..9a0ee3d77230 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2067,9 +2067,6 @@ static inline void mark_obj_codetag_empty(const void *obj) if (!slab_obj_ext_has_codetag()) return; - if (is_kfence_address(obj)) - return; - obj_slab = virt_to_slab(obj); slab_exts = slab_obj_exts(obj_slab); if (slab_exts) { @@ -2332,11 +2329,15 @@ static inline unsigned long prepare_slab_obj_exts_hook(struct kmem_cache *s, struct slab *slab, gfp_t flags, unsigned int alloc_flags, void *p) { - if (!slab_obj_exts(slab) && - alloc_slab_obj_exts(slab, s, flags, alloc_flags)) { - pr_warn_once("%s, %s: Failed to create slab extension vector!\n", - __func__, s->name); - return 0; + if (!slab_obj_exts(slab)) { + if (is_kfence_address(p)) + return 0; + + if (alloc_slab_obj_exts(slab, s, flags, alloc_flags)) { + pr_warn_once("%s, %s: Failed to create slab extension vector!\n", + __func__, s->name); + return 0; + } } return slab_obj_exts(slab); @@ -2361,9 +2362,6 @@ __alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags, if (alloc_flags & SLAB_ALLOC_NO_RECURSE) return; - if (is_kfence_address(object)) - return; - slab = virt_to_slab(object); obj_exts = prepare_slab_obj_exts_hook(s, slab, flags, alloc_flags, object); /* @@ -2383,7 +2381,13 @@ __alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags, put_slab_obj_exts(obj_exts); } else { - alloc_tag_set_inaccurate(current->alloc_tag); + /* + * KFENCE allocations are rare and the amount of outstanding + * ones is limited to a small number so it's not worth setting + * tags as inaccurate because of them. + */ + if (!is_kfence_address(object)) + alloc_tag_set_inaccurate(current->alloc_tag); } } @@ -2414,9 +2418,6 @@ __alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p for (int i = 0; i < objects; i++) { struct slabobj_ext *ext; - if (is_kfence_address(p[i])) - continue; - ext = slab_obj_ext(s, slab, obj_exts, p[i]); alloc_tag_sub(slab_obj_ext_codetag_ref(slab, ext), s->size); }