mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-31 03:35:32 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'slab-for-7.3' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab

Browse Source 
Pull slab updates from Vlastimil Babka:

 - Add kfree_rcu_nolock() that can be used from contexts where spinning
   on a lock might be unsafe, such as a BPF program attached to an
   arbitrary function, or in NMI context. This complements the existing
   kfree_nolock() support (Harry Yoo)

 - Runtime instead of compile-time slabobj_ext sizing.

   Avoid wasting memory when memory allocation profiling is compiled but
   not enabled, with initial partial support to also avoid wasting
   memory for objcg pointers when those are not needed, while profiling
   is enabled (Vlastimil Babka)

 - Various non-urgent fixes, cleanups and optimizations (Hao Li,
   Hongling Zeng, Li RongQing, Li Xiasong, Seongjun Hong, Shengming Hu)

* tag 'slab-for-7.3' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab: (31 commits)
  mm/slab, kfence, memcg: completely remove obj_ext for kfence objects
  mm/slab: stop allocating objcg pointers when unnecessary
  mm/slab: add cache_ and slab_needs_objcg() helpers
  mm/slab: stop exporting kvfree_rcu_barrier[_on_cache]()
  slub_kunit: extend the test for kfree_rcu_nolock()
  mm/slab: introduce kfree_rcu_nolock()
  mm/slab: introduce struct kvfree_rcu_head for kvfree_rcu batching
  mm/slab: reduce slabobj_ext memory with allocation profiling disabled
  mm/slab: introduce slab_obj_ext_has_codetag()
  mm/slab: allow kfree_rcu_sheaf() on PREEMPT_RT
  mm/slab: extend deferred free mechanism to handle rcu sheaves
  mm/slab: use call_rcu() in unknown context if irqs are enabled
  mm/slab: handle the !allow_spin case in kfree_rcu_sheaf()
  mm/slab: change struct slabobj_ext to a union
  mm/slab: replace slab.stride with obj_exts_in_object
  mm/slab: abstract slabobj_ext.ref access
  mm/slab: abstract slabobj_ext.objcg access
  mm/slab: make slab_obj_ext() determine object index
  mm: move struct slabobj_ext to mm/slab.h
  mm/slab: remove objs_per_slab()
  ...
This commit is contained in:
Linus Torvalds
2026-08-24 10:58:57 -07:00
parent a0300e8cf0 564ed40708
commit 918e25291c
17 changed files with 945 additions and 466 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
Documentation/ABI/testing/sysfs-kernel-slab
View File

@@ -113,8 +113,10 @@ KernelVersion: 2.6.22
Contact: Pekka Enberg <penberg@cs.helsinki.fi>,
Christoph Lameter <cl@gentwo.org>
Description:
The cpu_slabs file is read-only and displays how many cpu slabs
are active and their NUMA locality.
The cpu_slabs file is read-only. It is deprecated and always
reads "0" since the removal of per-cpu slabs in Linux 7.0. It
previously displayed how many cpu slabs were active and their
NUMA locality. The file is kept for backwards compatibility.
What: /sys/kernel/slab/<cache>/cpuslab_flush
Date: April 2009
@@ -509,12 +511,16 @@ What: /sys/kernel/slab/<cache>/slabs_cpu_partial
Date: Aug 2011
Contact: Christoph Lameter <cl@gentwo.org>
Description:
This read-only file shows the number of partialli allocated
frozen slabs.
This read-only file is deprecated and always reads "0(0)" since
the removal of per-cpu partial slabs in Linux 7.0. It previously
showed the number of partially allocated frozen slabs. The file
is kept for backwards compatibility.
What: /sys/kernel/slab/<cache>/cpu_partial
Date: Aug 2011
Contact: Christoph Lameter <cl@gentwo.org>
Description:
This read-only file shows the number of per cpu partial
pages to keep around.
This file is deprecated and always reads "0" since the removal of
per-cpu partial slabs in Linux 7.0. It previously showed the
number of per-cpu partial pages to keep around. The file is kept
for backwards compatibility.

5
Documentation/admin-guide/kernel-parameters.txt
View File

@@ -4014,6 +4014,11 @@ Kernel parameters
Note that even when enabled, there are a few cases where
the feature is not effective.
mempool_debug [MM]
Enable mempool debugging. This enables element
poison checking when freeing elements back to the
pool. Useful for debugging mempool corruption.
memtest= [KNL,X86,ARM,M68K,PPC,RISCV,EARLY] Enable memtest
Format: <integer>
default : 0 <disable>

7
Documentation/mm/allocation-profiling.rst
View File

@@ -112,3 +112,10 @@ To do so:
- Then, use the following form for your allocations:
alloc_hooks_tag(ht->your_saved_tag, kmalloc_noprof(...))
Notes
=====
- When a slab object is allocated from KFENCE, its accounting is skipped.
KFENCE allocations are rare and limited to a small number, so this omission
is negligible.

13
include/linux/memcontrol.h
View File

@@ -1461,19 +1461,6 @@ static inline void mem_cgroup_flush_workqueue(void) { }
static inline int mem_cgroup_init(void) { return 0; }
#endif /* CONFIG_MEMCG */
/*
* Extended information for slab objects stored as an array in page->memcg_data
* if MEMCG_DATA_OBJEXTS is set.
*/
struct slabobj_ext {
#ifdef CONFIG_MEMCG
struct obj_cgroup *objcg;
#endif
#ifdef CONFIG_MEM_ALLOC_PROFILING
union codetag_ref ref;
#endif
} __aligned(8);
static inline struct lruvec *parent_lruvec(struct lruvec *lruvec)
{
struct mem_cgroup *memcg;

32
include/linux/rcupdate.h
View File

@@ -1107,19 +1107,22 @@ static inline void rcu_read_unlock_migrate(void)
/*
* In mm/slab_common.c, no suitable header to include here.
*/
void kvfree_call_rcu(struct rcu_head *head, void *ptr);
void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr);
void kfree_call_rcu_nolock(struct kvfree_rcu_head *head, void *ptr);
/*
* The BUILD_BUG_ON() makes sure the rcu_head offset can be handled. See the
* comment of kfree_rcu() for details.
*/
#define kvfree_rcu_arg_2(ptr, rhf) \
#define kvfree_rcu_arg_2(ptr, kvrhf) \
do { \
typeof (ptr) ___p = (ptr); \
struct kvfree_rcu_head *___head; \
\
if (___p) { \
BUILD_BUG_ON(offsetof(typeof(*(ptr)), rhf) >= 4096); \
kvfree_call_rcu(&((___p)->rhf), (void *) (___p)); \
BUILD_BUG_ON(offsetof(typeof(*(ptr)), kvrhf) >= 4096); \
___head = (struct kvfree_rcu_head *) &(___p)->kvrhf; \
kvfree_call_rcu(___head, (void *) (___p)); \
} \
} while (0)
@@ -1131,6 +1134,27 @@ do { \
kvfree_call_rcu(NULL, (void *) (___p)); \
} while (0)
/**
* kfree_rcu_nolock() - a version of kfree_rcu() that can be called in any context.
* @ptr: pointer to kfree for double-argument invocations.
* @kvrhf: the name of the struct kvfree_rcu_head within the type of @ptr.
*
* With KVFREE_RCU_BATCHED, kfree_rcu_nolock() tries hard to free objects
* without any deferred processing, but may still defer freeing.
* Large kmalloc and vmalloc objects are always deferred.
*
* kfree_rcu_nolock() supports 2-arg variant only.
*/
#define kfree_rcu_nolock(ptr, kvrhf) \
do { \
typeof (ptr) ___p = (ptr); \
\
if (___p) { \
BUILD_BUG_ON(offsetof(typeof(*(ptr)), kvrhf) >= 4096); \
kfree_call_rcu_nolock(&((___p)->kvrhf), (void *) (___p)); \
} \
} while (0)
/*
* Place this after a lock-acquisition primitive to guarantee that
* an UNLOCK+LOCK pair acts as a full barrier. This guarantee applies

19
include/linux/slab.h
View File

@@ -45,6 +45,7 @@ enum _slab_flag_bits {
#endif
#ifdef CONFIG_MEMCG
_SLAB_ACCOUNT,
_SLAB_MAY_ACCOUNT,
#endif
#ifdef CONFIG_KASAN_GENERIC
_SLAB_KASAN,
@@ -204,8 +205,10 @@ enum _slab_flag_bits {
*/
#ifdef CONFIG_MEMCG
# define SLAB_ACCOUNT __SLAB_FLAG_BIT(_SLAB_ACCOUNT)
# define SLAB_MAY_ACCOUNT __SLAB_FLAG_BIT(_SLAB_MAY_ACCOUNT)
#else
# define SLAB_ACCOUNT __SLAB_FLAG_UNUSED
# define SLAB_MAY_ACCOUNT __SLAB_FLAG_UNUSED
#endif
#ifdef CONFIG_KASAN_GENERIC
@@ -1427,25 +1430,15 @@ extern void kvfree_sensitive(const void *addr, size_t len);
unsigned int kmem_cache_size(struct kmem_cache *s);
#ifndef CONFIG_KVFREE_RCU_BATCHED
static inline void kvfree_rcu_barrier(void)
{
rcu_barrier();
}
static inline void kvfree_rcu_barrier_on_cache(struct kmem_cache *s)
{
rcu_barrier();
}
static inline void kfree_rcu_scheduler_running(void) { }
#else
void kfree_rcu_scheduler_running(void);
#endif
void kvfree_rcu_barrier(void);
void kvfree_rcu_barrier_on_cache(struct kmem_cache *s);
void kfree_rcu_scheduler_running(void);
#endif
/**
* kmalloc_size_roundup - Report allocation bucket size for the given size
*

10
include/linux/types.h
View File

@@ -257,6 +257,16 @@ struct callback_head {
} __attribute__((aligned(sizeof(void *))));
#define rcu_head callback_head
#ifdef CONFIG_KVFREE_RCU_BATCHED
struct kvfree_rcu_head {
struct kvfree_rcu_head *next;
};
#else
struct kvfree_rcu_head {
struct rcu_head head;
};
#endif
typedef void (*rcu_callback_t)(struct rcu_head *head);
typedef void (*call_rcu_func_t)(struct rcu_head *head, rcu_callback_t func);

2
include/trace/events/rcu.h
View File

@@ -626,7 +626,7 @@ TRACE_EVENT_RCU(rcu_invoke_callback,
*/
TRACE_EVENT_RCU(rcu_invoke_kvfree_callback,
TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset),
TP_PROTO(const char *rcuname, struct kvfree_rcu_head *rhp, unsigned long offset),
TP_ARGS(rcuname, rhp, offset),

190
lib/tests/slub_kunit.c
View File

@@ -8,6 +8,7 @@
#include <linux/rcupdate.h>
#include <linux/delay.h>
#include <linux/perf_event.h>
#include <linux/kprobes.h>
#include "../mm/slab.h"
static struct kunit_resource resource;
@@ -161,7 +162,10 @@ static void test_kmalloc_redzone_access(struct kunit *test)
}
struct test_kfree_rcu_struct {
struct rcu_head rcu;
union {
struct rcu_head rcu;
struct kvfree_rcu_head kvrcu;
};
};
static void test_kfree_rcu(struct kunit *test)
@@ -292,19 +296,76 @@ static void test_krealloc_redzone_zeroing(struct kunit *test)
kmem_cache_destroy(s);
}
#ifdef CONFIG_PERF_EVENTS
#if defined(CONFIG_PERF_EVENTS) || (defined(CONFIG_KPROBES) && defined(CONFIG_SMP))
#define NR_ITERATIONS 1000
#define NR_OBJECTS 1000
static void *objects[NR_OBJECTS];
static struct test_kfree_rcu_struct *objects[NR_OBJECTS];
struct test_nolock_context {
struct kunit *test;
int callback_count;
int alloc_ok;
int alloc_fail;
#ifdef CONFIG_PERF_EVENTS
struct perf_event *event;
#endif
#if defined(CONFIG_KPROBES) && defined(CONFIG_SMP)
struct kprobe kprobe;
#endif
};
static void test_kmalloc_and_friends(void)
{
int i, j;
bool can_use_kfree_rcu = !IS_BUILTIN(CONFIG_SLUB_KUNIT_TEST);
for (i = 0; i < NR_ITERATIONS; i++) {
for (j = 0; j < NR_OBJECTS; j++) {
gfp_t gfp = (i & 1) ? GFP_KERNEL : GFP_KERNEL_ACCOUNT;
objects[j] = kmalloc_obj(*objects[j], gfp);
if (!objects[j]) {
j--;
while (j >= 0)
kfree(objects[j--]);
return;
}
}
for (j = 0; j < NR_OBJECTS; j++) {
if (can_use_kfree_rcu && (i & 2))
kfree_rcu(objects[j], rcu);
else
kfree(objects[j]);
}
}
}
static void test_nolock(struct test_nolock_context *ctx)
{
struct test_kfree_rcu_struct *objp;
gfp_t gfp;
bool can_use_kfree_rcu = !IS_BUILTIN(CONFIG_SLUB_KUNIT_TEST);
/* __GFP_ACCOUNT to test kmalloc_nolock() in alloc_slab_obj_exts() */
gfp = (ctx->callback_count & 1) ? 0 : __GFP_ACCOUNT;
objp = kmalloc_nolock(sizeof(*objp), gfp, NUMA_NO_NODE);
if (objp)
ctx->alloc_ok++;
else
ctx->alloc_fail++;
if (can_use_kfree_rcu && (ctx->callback_count & 2))
kfree_rcu_nolock(objp, kvrcu);
else
kfree_nolock(objp);
ctx->callback_count++;
}
#endif
#ifdef CONFIG_PERF_EVENTS
static struct perf_event_attr hw_attr = {
.type = PERF_TYPE_HARDWARE,
.config = PERF_COUNT_HW_CPU_CYCLES,
@@ -315,67 +376,91 @@ static struct perf_event_attr hw_attr = {
.sample_freq = 100000,
};
static void overflow_handler_test_kmalloc_kfree_nolock(struct perf_event *event,
struct perf_sample_data *data,
struct pt_regs *regs)
static void overflow_handler_test_nolock(struct perf_event *event,
struct perf_sample_data *data,
struct pt_regs *regs)
{
void *objp;
gfp_t gfp;
struct test_nolock_context *ctx = event->overflow_handler_context;
/* __GFP_ACCOUNT to test kmalloc_nolock() in alloc_slab_obj_exts() */
gfp = (ctx->callback_count % 2) ? 0 : __GFP_ACCOUNT;
objp = kmalloc_nolock(64, gfp, NUMA_NO_NODE);
if (objp)
ctx->alloc_ok++;
else
ctx->alloc_fail++;
kfree_nolock(objp);
ctx->callback_count++;
test_nolock(ctx);
}
static void test_kmalloc_kfree_nolock(struct kunit *test)
static bool enable_perf_events(struct test_nolock_context *ctx)
{
int i, j;
struct test_nolock_context ctx = { .test = test };
struct perf_event *event;
bool alloc_fail = false;
event = perf_event_create_kernel_counter(&hw_attr, -1, current,
overflow_handler_test_kmalloc_kfree_nolock,
&ctx);
overflow_handler_test_nolock,
ctx);
if (IS_ERR(event))
kunit_skip(test, "Failed to create perf event");
ctx.event = event;
perf_event_enable(ctx.event);
for (i = 0; i < NR_ITERATIONS; i++) {
for (j = 0; j < NR_OBJECTS; j++) {
gfp_t gfp = (i % 2) ? GFP_KERNEL : GFP_KERNEL_ACCOUNT;
return false;
objects[j] = kmalloc(64, gfp);
if (!objects[j]) {
j--;
while (j >= 0)
kfree(objects[j--]);
alloc_fail = true;
goto cleanup;
}
}
for (j = 0; j < NR_OBJECTS; j++)
kfree(objects[j]);
}
ctx->event = event;
perf_event_enable(ctx->event);
return true;
}
cleanup:
perf_event_disable(ctx.event);
perf_event_release_kernel(ctx.event);
static void disable_perf_events(struct test_nolock_context *ctx)
{
kunit_info(ctx->test, "HW perf events: callback_count: %d, alloc_ok: %d, alloc_fail: %d\n",
ctx->callback_count, ctx->alloc_ok, ctx->alloc_fail);
kunit_info(test, "callback_count: %d, alloc_ok: %d, alloc_fail: %d\n",
ctx.callback_count, ctx.alloc_ok, ctx.alloc_fail);
perf_event_disable(ctx->event);
perf_event_release_kernel(ctx->event);
}
if (alloc_fail)
kunit_skip(test, "Allocation failed");
static void test_kmalloc_nolock_and_friends_perf(struct kunit *test)
{
struct test_nolock_context ctx = { .test = test };
if (!enable_perf_events(&ctx))
kunit_skip(test, "Failed to enable perf event, skipping");
test_kmalloc_and_friends();
disable_perf_events(&ctx);
KUNIT_EXPECT_EQ(test, 0, slab_errors);
}
#endif
#if defined(CONFIG_KPROBES) && defined(CONFIG_SMP)
static int slab_kprobe_pre_handler(struct kprobe *p, struct pt_regs *regs)
{
struct test_nolock_context *ctx;
ctx = container_of(p, struct test_nolock_context, kprobe);
test_nolock(ctx);
return 0;
}
static bool register_slab_kprobes(struct test_nolock_context *ctx)
{
ctx->kprobe.symbol_name = "slab_attach_kprobe_locked";
ctx->kprobe.pre_handler = slab_kprobe_pre_handler;
if (register_kprobe(&ctx->kprobe))
return false;
return true;
}
static void unregister_slab_kprobes(struct test_nolock_context *ctx)
{
kunit_info(ctx->test, "kprobes: callback_count: %d, alloc_ok: %d, alloc_fail: %d\n",
ctx->callback_count, ctx->alloc_ok, ctx->alloc_fail);
unregister_kprobe(&ctx->kprobe);
}
static void test_kmalloc_nolock_and_friends_kprobe(struct kunit *test)
{
struct test_nolock_context ctx = { .test = test };
if (!register_slab_kprobes(&ctx))
kunit_skip(test, "Failed to register kprobe, skipping");
test_kmalloc_and_friends();
unregister_slab_kprobes(&ctx);
KUNIT_EXPECT_EQ(test, 0, slab_errors);
}
#endif
@@ -405,7 +490,10 @@ static struct kunit_case test_cases[] = {
KUNIT_CASE(test_leak_destroy),
KUNIT_CASE(test_krealloc_redzone_zeroing),
#ifdef CONFIG_PERF_EVENTS
KUNIT_CASE_SLOW(test_kmalloc_kfree_nolock),
KUNIT_CASE_SLOW(test_kmalloc_nolock_and_friends_perf),
#endif
#if defined(CONFIG_KPROBES) && defined(CONFIG_SMP)
KUNIT_CASE_SLOW(test_kmalloc_nolock_and_friends_kprobe),
#endif
{}
};

12
mm/kfence/core.c
View File

@@ -636,11 +636,6 @@ static unsigned long kfence_init_pool(void)
page = pfn_to_page(start_pfn + i);
__SetPageSlab(page);
#ifdef CONFIG_MEMCG
struct slab *slab = page_slab(page);
slab->obj_exts = (unsigned long)&kfence_metadata_init[i / 2 - 1].obj_exts |
MEMCG_DATA_OBJEXTS;
#endif
}
/*
@@ -704,10 +699,6 @@ static unsigned long kfence_init_pool(void)
continue;
page = pfn_to_page(start_pfn + i);
#ifdef CONFIG_MEMCG
struct slab *slab = page_slab(page);
slab->obj_exts = 0;
#endif
__ClearPageSlab(page);
}
@@ -1248,9 +1239,6 @@ void __kfence_free(void *addr)
{
struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr);
#ifdef CONFIG_MEMCG
KFENCE_WARN_ON(meta->obj_exts.objcg);
#endif
/*
* If the objects of the cache are SLAB_TYPESAFE_BY_RCU, defer freeing
* the object, as the object page may be recycled for other-typed

3
mm/kfence/kfence.h
View File

@@ -102,9 +102,6 @@ struct kfence_metadata {
struct kfence_track free_track __guarded_by(&lock);
/* For updating alloc_covered on frees. */
u32 alloc_stack_hash __guarded_by(&lock);
#ifdef CONFIG_MEMCG
struct slabobj_ext obj_exts;
#endif
};
#define KFENCE_METADATA_SIZE PAGE_ALIGN(sizeof(struct kfence_metadata) * \

2
mm/kfence/kfence_test.c
View File

@@ -295,7 +295,7 @@ static void *test_alloc(struct kunit *test, size_t size, gfp_t gfp, enum allocat
* memcg accounting works correctly.
*/
KUNIT_EXPECT_EQ(test, obj_to_index(s, slab, alloc), 0U);
KUNIT_EXPECT_EQ(test, objs_per_slab(s, slab), 1);
KUNIT_EXPECT_EQ(test, ((unsigned int)slab->objects), 1);
if (policy == ALLOCATE_ANY)
return alloc;

37
mm/memcontrol.c
View File

@@ -2870,18 +2870,19 @@ struct mem_cgroup *mem_cgroup_from_obj_slab(struct slab *slab, void *p)
*/
unsigned long obj_exts;
struct slabobj_ext *obj_ext;
unsigned int off;
struct obj_cgroup *objcg;
obj_exts = slab_obj_exts(slab);
if (!obj_exts)
return NULL;
get_slab_obj_exts(obj_exts);
off = obj_to_index(slab->slab_cache, slab, p);
obj_ext = slab_obj_ext(slab, obj_exts, off);
if (obj_ext->objcg) {
struct obj_cgroup *objcg = obj_ext->objcg;
if (!slab_needs_objcg(slab))
return NULL;
get_slab_obj_exts(obj_exts);
obj_ext = slab_obj_ext(slab->slab_cache, slab, obj_exts, p);
objcg = slab_obj_ext_objcg(slab, obj_ext);
if (objcg) {
put_slab_obj_exts(obj_exts);
return obj_cgroup_memcg(objcg);
}
@@ -3543,7 +3544,6 @@ bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
size_t obj_size = obj_full_size(s);
struct obj_cgroup *objcg;
struct slab *slab;
unsigned long off;
size_t i;
/*
@@ -3585,9 +3585,11 @@ bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
slab = virt_to_slab(p[i]);
if (!slab_obj_exts(slab) &&
alloc_slab_obj_exts(slab, s, flags, slab_alloc_flags)) {
continue;
if (!slab_obj_exts(slab)) {
if (is_kfence_address(p[i]))
continue;
if (alloc_slab_obj_exts(slab, s, flags, slab_alloc_flags))
continue;
}
/*
@@ -3618,10 +3620,11 @@ bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
obj_exts = slab_obj_exts(slab);
get_slab_obj_exts(obj_exts);
off = obj_to_index(s, slab, p[i]);
obj_ext = slab_obj_ext(slab, obj_exts, off);
obj_ext = slab_obj_ext(s, slab, obj_exts, p[i]);
obj_cgroup_get(objcg);
obj_ext->objcg = objcg;
slab_obj_ext_set_objcg(slab, obj_ext, objcg);
put_slab_obj_exts(obj_exts);
}
@@ -3637,15 +3640,13 @@ void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
struct obj_cgroup *objcg;
struct slabobj_ext *obj_ext;
struct obj_stock_pcp *stock;
unsigned int off;
off = obj_to_index(s, slab, p[i]);
obj_ext = slab_obj_ext(slab, obj_exts, off);
objcg = obj_ext->objcg;
obj_ext = slab_obj_ext(s, slab, obj_exts, p[i]);
objcg = slab_obj_ext_objcg(slab, obj_ext);
if (!objcg)
continue;
obj_ext->objcg = NULL;
slab_obj_ext_set_objcg(slab, obj_ext, NULL);
stock = trylock_stock();
__refill_obj_stock(objcg, stock, obj_size, true);

35
mm/mempool.c
View File

@@ -16,11 +16,28 @@
#include <linux/export.h>
#include <linux/mempool.h>
#include <linux/writeback.h>
#include <linux/static_key.h>
#include <linux/init.h>
#include "slab.h"
static DECLARE_FAULT_ATTR(fail_mempool_alloc);
static DECLARE_FAULT_ATTR(fail_mempool_alloc_bulk);
/*
* Debugging support for mempool using static key.
*
* This allows enabling mempool debug at boot time via:
* mempool_debug
*/
static DEFINE_STATIC_KEY_FALSE(mempool_debug_enabled);
static int __init mempool_debug_setup(char *str)
{
static_branch_enable(&mempool_debug_enabled);
return 1;
}
__setup("mempool_debug", mempool_debug_setup);
static int __init mempool_faul_inject_init(void)
{
int error;
@@ -37,7 +54,6 @@ static int __init mempool_faul_inject_init(void)
}
late_initcall(mempool_faul_inject_init);
#ifdef CONFIG_SLUB_DEBUG_ON
static void poison_error(struct mempool *pool, void *element, size_t size,
size_t byte)
{
@@ -140,14 +156,6 @@ static void poison_element(struct mempool *pool, void *element)
#endif
}
}
#else /* CONFIG_SLUB_DEBUG_ON */
static inline void check_element(struct mempool *pool, void *element)
{
}
static inline void poison_element(struct mempool *pool, void *element)
{
}
#endif /* CONFIG_SLUB_DEBUG_ON */
static __always_inline bool kasan_poison_element(struct mempool *pool,
void *element)
@@ -175,7 +183,10 @@ static void kasan_unpoison_element(struct mempool *pool, void *element)
static __always_inline void add_element(struct mempool *pool, void *element)
{
BUG_ON(pool->min_nr != 0 && pool->curr_nr >= pool->min_nr);
poison_element(pool, element);
if (static_branch_unlikely(&mempool_debug_enabled))
poison_element(pool, element);
if (kasan_poison_element(pool, element))
pool->elements[pool->curr_nr++] = element;
}
@@ -186,7 +197,9 @@ static void *remove_element(struct mempool *pool)
BUG_ON(pool->curr_nr < 0);
kasan_unpoison_element(pool, element);
check_element(pool, element);
if (static_branch_unlikely(&mempool_debug_enabled))
check_element(pool, element);
return element;
}

243
mm/slab.h
View File

@@ -24,11 +24,27 @@
#define SLAB_ALLOC_NO_RECURSE 0x04 /* prevent kmalloc() recursion */
#define SLAB_ALLOC_NO_OBJ_EXT 0x08 /* prevent obj_exts array allocation */
#define SLAB_FREE_DEFAULT 0x00 /* no flags */
#define SLAB_FREE_NOLOCK 0x01 /* spinning not allowed */
static inline unsigned int to_alloc_flags(unsigned int free_flags)
{
if (free_flags & SLAB_FREE_NOLOCK)
return SLAB_ALLOC_NOLOCK;
else
return SLAB_ALLOC_DEFAULT;
}
static inline bool alloc_flags_allow_spinning(const unsigned int alloc_flags)
{
return !(alloc_flags & SLAB_ALLOC_NOLOCK);
}
static inline bool free_flags_allow_spinning(const unsigned int free_flags)
{
return !(free_flags & SLAB_FREE_NOLOCK);
}
void *__kmalloc_flags_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t flags,
unsigned int alloc_flags, int node)
__assume_kmalloc_alignment __alloc_size(1);
@@ -81,10 +97,11 @@ struct freelist_counters {
#ifdef CONFIG_64BIT
/*
* Some optimizations use free bits in 'counters' field
* to save memory. In case ->stride field is not available,
* such optimizations are disabled.
* to save memory or CPU. If these free bits are not
* available, such optimizations are disabled.
*/
unsigned int stride;
unsigned obj_exts_in_object:1;
unsigned obj_exts_needs_objcg:1;
#endif
};
};
@@ -330,10 +347,35 @@ static inline unsigned int obj_to_index(const struct kmem_cache *cache,
return __obj_to_index(cache, slab_address(slab), obj);
}
static inline int objs_per_slab(const struct kmem_cache *cache,
const struct slab *slab)
/*
* kvfree_rcu_head offset can be only less than page size.
* Calculate the start address while preserving the KASAN tag.
*/
static inline void *kvmalloc_obj_start_addr(void *head)
{
return slab->objects;
unsigned long offset;
if (unlikely(is_vmalloc_addr(head))) {
offset = offset_in_page(head);
} else {
struct slab *slab = virt_to_slab(head);
if (!slab) {
offset = offset_in_page(head);
} else if (is_kfence_address(head)) {
offset = head - kfence_object_start(head);
} else {
struct kmem_cache *s = slab->slab_cache;
unsigned int idx = __obj_to_index(s, slab_address(slab), head);
void *obj = slab_address(slab) + s->size * idx;
obj = fixup_red_left(s, obj);
obj = kasan_reset_tag(obj);
offset = kasan_reset_tag(head) - obj;
}
}
return head - offset;
}
/*
@@ -436,7 +478,7 @@ static inline bool is_kmalloc_normal(struct kmem_cache *s)
return !(s->flags & (SLAB_CACHE_DMA|SLAB_ACCOUNT|SLAB_RECLAIM_ACCOUNT|SLAB_NO_OBJ_EXT));
}
bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj);
bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags);
void flush_all_rcu_sheaves(void);
void flush_rcu_sheaves_on_cache(struct kmem_cache *s);
@@ -555,6 +597,93 @@ static inline bool need_kmalloc_no_objext(void)
return false;
}
/*
* Extended information for slab objects stored as a pointer to an array in
* slab->obj_exts (aliasing page->memcg_data) if MEMCG_DATA_OBJEXTS is set.
*/
struct slabobj_ext {
/*
* All elements of the union should be pointer-sized to avoid memory
* waste
*/
union {
#ifdef CONFIG_MEMCG
struct obj_cgroup *_objcg;
#endif
#ifdef CONFIG_MEM_ALLOC_PROFILING
union codetag_ref _ctref;
#endif
};
} __aligned(8);
#ifdef CONFIG_MEM_ALLOC_PROFILING
DECLARE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
slab_obj_ext_has_codetag_key);
static inline bool slab_obj_ext_has_codetag(void)
{
return static_branch_maybe(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
&slab_obj_ext_has_codetag_key);
}
#else
static inline bool slab_obj_ext_has_codetag(void)
{
return false;
}
#endif
#ifdef CONFIG_MEMCG
static inline bool cache_needs_objcg(struct kmem_cache *cache)
{
return (cache->flags & SLAB_MAY_ACCOUNT);
}
static inline bool slab_needs_objcg(struct slab *slab)
{
#ifdef CONFIG_64BIT
return slab->obj_exts_needs_objcg;
#else
return cache_needs_objcg(slab->slab_cache);
#endif
}
#else
static inline bool cache_needs_objcg(struct kmem_cache *cache)
{
return false;
}
static inline bool slab_needs_objcg(struct slab *slab)
{
return false;
}
#endif
static inline size_t cache_obj_ext_size(struct kmem_cache *s)
{
size_t sz = 0;
if (cache_needs_objcg(s))
sz += 1;
if (slab_obj_ext_has_codetag())
sz += 1;
return sizeof(struct slabobj_ext) * sz;
}
static inline size_t slab_obj_ext_size(struct slab *slab)
{
size_t sz = 0;
if (slab_needs_objcg(slab))
sz += 1;
if (slab_obj_ext_has_codetag())
sz += 1;
return sizeof(struct slabobj_ext) * sz;
}
#ifdef CONFIG_SLAB_OBJ_EXT
/*
@@ -572,7 +701,7 @@ static inline bool need_kmalloc_no_objext(void)
* obj_exts = slab_obj_exts(slab);
* if (obj_exts) {
* get_slab_obj_exts(obj_exts);
* obj_ext = slab_obj_ext(slab, obj_exts, obj_to_index(s, slab, obj));
* obj_ext = slab_obj_ext(s, slab, obj_exts, obj);
* // do something with obj_ext
* put_slab_obj_exts(obj_exts);
* }
@@ -610,48 +739,94 @@ static inline void put_slab_obj_exts(unsigned long obj_exts)
}
#ifdef CONFIG_64BIT
static inline void slab_set_stride(struct slab *slab, unsigned int stride)
static inline bool obj_exts_in_object(struct slab *slab)
{
slab->stride = stride;
}
static inline unsigned int slab_get_stride(struct slab *slab)
{
return slab->stride;
/*
* Note we cannot rely on the SLAB_OBJ_EXT_IN_OBJ flag here and need to
* check the per-slab bit. A cache can have SLAB_OBJ_EXT_IN_OBJ set, but
* allocations within_slab_leftover are preferred. And those may be
* possible or not depending on the particular slab's size.
*/
return slab->obj_exts_in_object;
}
#else
static inline void slab_set_stride(struct slab *slab, unsigned int stride)
static inline bool obj_exts_in_object(struct slab *slab)
{
VM_WARN_ON_ONCE(stride != sizeof(struct slabobj_ext));
}
static inline unsigned int slab_get_stride(struct slab *slab)
{
return sizeof(struct slabobj_ext);
return false;
}
#endif
/*
* slab_obj_ext - get the pointer to the slab object extension metadata
* associated with an object in a slab.
* @s: cache that the slab belongs to
* @slab: a pointer to the slab struct
* @obj_exts: a pointer to the object extension vector
* @index: an index of the object
* @obj: a pointer to the object
*
* Returns a pointer to the object extension associated with the object.
* Must be called within a section covered by get/put_slab_obj_exts().
*/
static inline struct slabobj_ext *slab_obj_ext(struct slab *slab,
unsigned long obj_exts,
unsigned int index)
static inline struct slabobj_ext *
slab_obj_ext(struct kmem_cache *s, struct slab *slab, unsigned long obj_exts,
const void *obj)
{
struct slabobj_ext *obj_ext;
unsigned int index;
unsigned int stride;
VM_WARN_ON_ONCE(obj_exts != slab_obj_exts(slab));
obj_ext = (struct slabobj_ext *)(obj_exts +
slab_get_stride(slab) * index);
/*
* KFENCE objects have NULL obj_exts and thus can't reach this
* and we don't need obj_to_index()
*/
index = __obj_to_index(s, slab_address(slab), obj);
if (!obj_exts_in_object(slab))
stride = slab_obj_ext_size(slab);
else
stride = s->size;
obj_ext = (struct slabobj_ext *)(obj_exts + index * stride);
return kasan_reset_tag(obj_ext);
}
#ifdef CONFIG_MEMCG
static inline struct obj_cgroup *
slab_obj_ext_objcg(struct slab *slab, struct slabobj_ext *obj_ext)
{
VM_WARN_ON_ONCE(!slab_needs_objcg(slab));
/* if objcg exists, it comes first, so we don't need to do anything */
return obj_ext->_objcg;
}
static inline void
slab_obj_ext_set_objcg(struct slab *slab, struct slabobj_ext *obj_ext,
struct obj_cgroup *objcg)
{
VM_WARN_ON_ONCE(!slab_needs_objcg(slab));
/* if objcg exists, it comes first, so we don't need to do anything */
obj_ext->_objcg = objcg;
}
#endif
#ifdef CONFIG_MEM_ALLOC_PROFILING
static inline union codetag_ref *
slab_obj_ext_codetag_ref(struct slab *slab, struct slabobj_ext *obj_ext)
{
VM_WARN_ON_ONCE(!slab_obj_ext_has_codetag());
if (slab_needs_objcg(slab))
obj_ext += 1;
return &obj_ext->_ctref;
}
#endif
int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
gfp_t gfp, unsigned int alloc_flags);
@@ -662,16 +837,17 @@ static inline unsigned long slab_obj_exts(struct slab *slab)
return 0;
}
static inline struct slabobj_ext *slab_obj_ext(struct slab *slab,
unsigned long obj_exts,
unsigned int index)
static inline struct slabobj_ext *
slab_obj_ext(struct kmem_cache *s, struct slab *slab, unsigned long obj_exts,
const void *obj)
{
return NULL;
}
static inline void slab_set_stride(struct slab *slab, unsigned int stride) { }
static inline unsigned int slab_get_stride(struct slab *slab) { return 0; }
static inline bool obj_exts_in_object(struct slab *slab)
{
return false;
}
#endif /* CONFIG_SLAB_OBJ_EXT */
@@ -770,7 +946,8 @@ void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
void __check_heap_object(const void *ptr, unsigned long n,
const struct slab *slab, bool to_user);
void defer_free_barrier(void);
void deferred_work_barrier(void);
void defer_kfree_rcu(struct kvfree_rcu_head *head);
static inline bool slub_debug_orig_size(struct kmem_cache *s)
{

111
mm/slab_common.c
View File

@@ -52,7 +52,7 @@ struct kmem_cache *kmem_cache;
SLAB_OBJ_EXT_IN_OBJ)
#define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \
SLAB_CACHE_DMA32 | SLAB_ACCOUNT)
SLAB_CACHE_DMA32 | SLAB_ACCOUNT | SLAB_MAY_ACCOUNT)
/*
* Merge control. If this is set then no merging of slab caches will occur.
@@ -359,6 +359,13 @@ struct kmem_cache *__kmem_cache_create_args(const char *name,
goto out_unlock;
}
/*
* For now we assume any cache can be used with __GFP_ACCOUNT and thus
* may need to store objcg pointers for objects
*/
if (!mem_cgroup_kmem_disabled())
flags |= SLAB_MAY_ACCOUNT;
/* Fail closed on bad usersize of useroffset values. */
if (!IS_ENABLED(CONFIG_HARDENED_USERCOPY) ||
WARN_ON(!args->usersize && args->useroffset) ||
@@ -551,7 +558,7 @@ void kmem_cache_destroy(struct kmem_cache *s)
}
/* Wait for deferred work from kmalloc/kfree_nolock() */
defer_free_barrier();
deferred_work_barrier();
cpus_read_lock();
mutex_lock(&slab_mutex);
@@ -984,11 +991,20 @@ new_kmalloc_cache(int idx, enum kmalloc_cache_type type)
#endif
/*
* If CONFIG_MEMCG is enabled, disable cache merging for
* KMALLOC_NORMAL caches.
* If memcg_kmem is enabled and this is a KMALLOC_NORMAL cache and not
* aliased with any other type, make sure it's never merged with any other
* cache.
*
* In other cases the kmalloc cache may end up being used for a
* __GFP_ACCOUNT allocation so mark it as such. The exception is a
* KMALLOC_NO_OBJ_EXT cache.
*/
if (IS_ENABLED(CONFIG_MEMCG) && (type == KMALLOC_NORMAL))
flags |= SLAB_NO_MERGE;
if (!mem_cgroup_kmem_disabled()) {
if (type == KMALLOC_NORMAL && KMALLOC_RECLAIM != KMALLOC_NORMAL)
flags |= SLAB_NO_MERGE;
else if (!(flags & SLAB_NO_OBJ_EXT))
flags |= SLAB_MAY_ACCOUNT;
}
if (minalign > ARCH_KMALLOC_MINALIGN) {
aligned_size = ALIGN(aligned_size, minalign);
@@ -1280,13 +1296,40 @@ EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc);
EXPORT_TRACEPOINT_SYMBOL(kfree);
EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free);
void kfree_call_rcu_nolock(struct kvfree_rcu_head *head, void *ptr)
{
struct slab *slab;
if (!IS_ENABLED(CONFIG_KVFREE_RCU_BATCHED))
goto fallback;
if (unlikely(is_vmalloc_addr(ptr)))
goto fallback;
slab = virt_to_slab(ptr);
if (unlikely(!slab))
goto fallback;
if (unlikely(IS_ENABLED(CONFIG_NUMA) && slab_nid(slab) != numa_mem_id()))
goto fallback;
if (unlikely(!__kfree_rcu_sheaf(slab->slab_cache, ptr, SLAB_FREE_NOLOCK)))
goto fallback;
return;
fallback:
defer_kfree_rcu(head);
}
EXPORT_SYMBOL_GPL(kfree_call_rcu_nolock);
#ifndef CONFIG_KVFREE_RCU_BATCHED
void kvfree_call_rcu(struct rcu_head *head, void *ptr)
void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr)
{
if (head) {
kasan_record_aux_stack(ptr);
call_rcu(head, kvfree_rcu_cb);
call_rcu(&head->head, kvfree_rcu_cb);
return;
}
@@ -1297,6 +1340,18 @@ void kvfree_call_rcu(struct rcu_head *head, void *ptr)
}
EXPORT_SYMBOL_GPL(kvfree_call_rcu);
void kvfree_rcu_barrier(void)
{
deferred_work_barrier();
rcu_barrier();
}
void kvfree_rcu_barrier_on_cache(struct kmem_cache *s)
{
deferred_work_barrier();
rcu_barrier();
}
void __init kvfree_rcu_init(void)
{
}
@@ -1363,7 +1418,7 @@ struct kvfree_rcu_bulk_data {
struct kfree_rcu_cpu_work {
struct rcu_work rcu_work;
struct rcu_head *head_free;
struct kvfree_rcu_head *head_free;
struct rcu_gp_seq head_free_gp_snap;
struct list_head bulk_head_free[FREE_N_CHANNELS];
struct kfree_rcu_cpu *krcp;
@@ -1399,7 +1454,7 @@ struct kfree_rcu_cpu_work {
struct kfree_rcu_cpu {
// Objects queued on a linked list
// through their rcu_head structures.
struct rcu_head *head;
struct kvfree_rcu_head *head;
unsigned long head_gp_snap;
atomic_t head_count;
@@ -1540,12 +1595,12 @@ kvfree_rcu_bulk(struct kfree_rcu_cpu *krcp,
}
static void
kvfree_rcu_list(struct rcu_head *head)
kvfree_rcu_list(struct kvfree_rcu_head *head)
{
struct rcu_head *next;
struct kvfree_rcu_head *next;
for (; head; head = next) {
void *ptr = (void *) head->func;
void *ptr = kvmalloc_obj_start_addr(head);
unsigned long offset = (void *) head - ptr;
next = head->next;
@@ -1569,7 +1624,7 @@ static void kfree_rcu_work(struct work_struct *work)
unsigned long flags;
struct kvfree_rcu_bulk_data *bnode, *n;
struct list_head bulk_head[FREE_N_CHANNELS];
struct rcu_head *head;
struct kvfree_rcu_head *head;
struct kfree_rcu_cpu *krcp;
struct kfree_rcu_cpu_work *krwp;
struct rcu_gp_seq head_gp_snap;
@@ -1612,6 +1667,14 @@ static bool kfree_rcu_sheaf(void *obj)
{
struct kmem_cache *s;
struct slab *slab;
unsigned int free_flags = SLAB_FREE_DEFAULT;
/*
* It is not safe to spin on PREEMPT_RT because the kernel might be
* holding a raw spinlock and slab acquires sleeping locks.
*/
if (IS_ENABLED(CONFIG_PREEMPT_RT))
free_flags = SLAB_FREE_NOLOCK;
if (is_vmalloc_addr(obj))
return false;
@@ -1622,7 +1685,7 @@ static bool kfree_rcu_sheaf(void *obj)
s = slab->slab_cache;
if (likely(!IS_ENABLED(CONFIG_NUMA) || slab_nid(slab) == numa_mem_id()))
return __kfree_rcu_sheaf(s, obj);
return __kfree_rcu_sheaf(s, obj, free_flags);
return false;
}
@@ -1692,7 +1755,7 @@ kvfree_rcu_drain_ready(struct kfree_rcu_cpu *krcp)
{
struct list_head bulk_ready[FREE_N_CHANNELS];
struct kvfree_rcu_bulk_data *bnode, *n;
struct rcu_head *head_ready = NULL;
struct kvfree_rcu_head *head_ready = NULL;
unsigned long flags;
int i;
@@ -1955,7 +2018,7 @@ void __init kfree_rcu_scheduler_running(void)
* be free'd in workqueue context. This allows us to: batch requests together to
* reduce the number of grace periods during heavy kfree_rcu()/kvfree_rcu() load.
*/
void kvfree_call_rcu(struct rcu_head *head, void *ptr)
void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr)
{
unsigned long flags;
struct kfree_rcu_cpu *krcp;
@@ -1971,7 +2034,7 @@ void kvfree_call_rcu(struct rcu_head *head, void *ptr)
if (!head)
might_sleep();
if (!IS_ENABLED(CONFIG_PREEMPT_RT) && kfree_rcu_sheaf(ptr))
if (kfree_rcu_sheaf(ptr))
return;
// Queue the object but don't yet schedule the batch.
@@ -1993,7 +2056,6 @@ void kvfree_call_rcu(struct rcu_head *head, void *ptr)
// Inline if kvfree_rcu(one_arg) call.
goto unlock_return;
head->func = ptr;
head->next = krcp->head;
WRITE_ONCE(krcp->head, head);
atomic_inc(&krcp->head_count);
@@ -2115,7 +2177,6 @@ void kvfree_rcu_barrier(void)
flush_all_rcu_sheaves();
__kvfree_rcu_barrier();
}
EXPORT_SYMBOL_GPL(kvfree_rcu_barrier);
/**
* kvfree_rcu_barrier_on_cache - Wait for in-flight kvfree_rcu() calls on a
@@ -2126,20 +2187,18 @@ EXPORT_SYMBOL_GPL(kvfree_rcu_barrier);
*/
void kvfree_rcu_barrier_on_cache(struct kmem_cache *s)
{
/* kfree_rcu_nolock() might have deferred frees even without sheaves */
deferred_work_barrier();
if (cache_has_sheaves(s)) {
cpus_read_lock();
flush_rcu_sheaves_on_cache(s);
cpus_read_unlock();
rcu_barrier();
}
/*
* TODO: Introduce a version of __kvfree_rcu_barrier() that works
* on a specific slab cache.
*/
rcu_barrier();
__kvfree_rcu_barrier();
}
EXPORT_SYMBOL_GPL(kvfree_rcu_barrier_on_cache);
static unsigned long
kfree_rcu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)

672
mm/slub.c
View File

File diff suppressed because it is too large Load Diff