diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 5e95acc33989..aede77bd0387 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -1098,19 +1098,22 @@ static inline void rcu_read_unlock_migrate(void) /* * In mm/slab_common.c, no suitable header to include here. */ -void kvfree_call_rcu(struct rcu_head *head, void *ptr); +void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr); +void kfree_call_rcu_nolock(struct kvfree_rcu_head *head, void *ptr); /* * The BUILD_BUG_ON() makes sure the rcu_head offset can be handled. See the * comment of kfree_rcu() for details. */ -#define kvfree_rcu_arg_2(ptr, rhf) \ +#define kvfree_rcu_arg_2(ptr, kvrhf) \ do { \ typeof (ptr) ___p = (ptr); \ + struct kvfree_rcu_head *___head; \ \ if (___p) { \ - BUILD_BUG_ON(offsetof(typeof(*(ptr)), rhf) >= 4096); \ - kvfree_call_rcu(&((___p)->rhf), (void *) (___p)); \ + BUILD_BUG_ON(offsetof(typeof(*(ptr)), kvrhf) >= 4096); \ + ___head = (struct kvfree_rcu_head *) &(___p)->kvrhf; \ + kvfree_call_rcu(___head, (void *) (___p)); \ } \ } while (0) @@ -1122,6 +1125,27 @@ do { \ kvfree_call_rcu(NULL, (void *) (___p)); \ } while (0) +/** + * kfree_rcu_nolock() - a version of kfree_rcu() that can be called in any context. + * @ptr: pointer to kfree for double-argument invocations. + * @kvrhf: the name of the struct kvfree_rcu_head within the type of @ptr. + * + * With KVFREE_RCU_BATCHED, kfree_rcu_nolock() tries hard to free objects + * without any deferred processing, but may still defer freeing. + * Large kmalloc and vmalloc objects are always deferred. + * + * kfree_rcu_nolock() supports 2-arg variant only. + */ +#define kfree_rcu_nolock(ptr, kvrhf) \ +do { \ + typeof (ptr) ___p = (ptr); \ + \ + if (___p) { \ + BUILD_BUG_ON(offsetof(typeof(*(ptr)), kvrhf) >= 4096); \ + kfree_call_rcu_nolock(&((___p)->kvrhf), (void *) (___p)); \ + } \ +} while (0) + /* * Place this after a lock-acquisition primitive to guarantee that * an UNLOCK+LOCK pair acts as a full barrier. This guarantee applies diff --git a/include/linux/slab.h b/include/linux/slab.h index e1915026e030..cda126def67a 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -1430,25 +1430,15 @@ extern void kvfree_sensitive(const void *addr, size_t len); unsigned int kmem_cache_size(struct kmem_cache *s); #ifndef CONFIG_KVFREE_RCU_BATCHED -static inline void kvfree_rcu_barrier(void) -{ - rcu_barrier(); -} - -static inline void kvfree_rcu_barrier_on_cache(struct kmem_cache *s) -{ - rcu_barrier(); -} - static inline void kfree_rcu_scheduler_running(void) { } #else +void kfree_rcu_scheduler_running(void); +#endif + void kvfree_rcu_barrier(void); void kvfree_rcu_barrier_on_cache(struct kmem_cache *s); -void kfree_rcu_scheduler_running(void); -#endif - /** * kmalloc_size_roundup - Report allocation bucket size for the given size * diff --git a/include/linux/types.h b/include/linux/types.h index 93166b0b0617..7d1d305a763e 100644 --- a/include/linux/types.h +++ b/include/linux/types.h @@ -255,6 +255,16 @@ struct callback_head { } __attribute__((aligned(sizeof(void *)))); #define rcu_head callback_head +#ifdef CONFIG_KVFREE_RCU_BATCHED +struct kvfree_rcu_head { + struct kvfree_rcu_head *next; +}; +#else +struct kvfree_rcu_head { + struct rcu_head head; +}; +#endif + typedef void (*rcu_callback_t)(struct rcu_head *head); typedef void (*call_rcu_func_t)(struct rcu_head *head, rcu_callback_t func); diff --git a/include/trace/events/rcu.h b/include/trace/events/rcu.h index 5fbdabe3faea..a74027126a2f 100644 --- a/include/trace/events/rcu.h +++ b/include/trace/events/rcu.h @@ -625,7 +625,7 @@ TRACE_EVENT_RCU(rcu_invoke_callback, */ TRACE_EVENT_RCU(rcu_invoke_kvfree_callback, - TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset), + TP_PROTO(const char *rcuname, struct kvfree_rcu_head *rhp, unsigned long offset), TP_ARGS(rcuname, rhp, offset), diff --git a/lib/tests/slub_kunit.c b/lib/tests/slub_kunit.c index fa6d31dbca16..e3b63f0338d5 100644 --- a/lib/tests/slub_kunit.c +++ b/lib/tests/slub_kunit.c @@ -8,6 +8,7 @@ #include #include #include +#include #include "../mm/slab.h" static struct kunit_resource resource; @@ -161,7 +162,10 @@ static void test_kmalloc_redzone_access(struct kunit *test) } struct test_kfree_rcu_struct { - struct rcu_head rcu; + union { + struct rcu_head rcu; + struct kvfree_rcu_head kvrcu; + }; }; static void test_kfree_rcu(struct kunit *test) @@ -292,19 +296,76 @@ static void test_krealloc_redzone_zeroing(struct kunit *test) kmem_cache_destroy(s); } -#ifdef CONFIG_PERF_EVENTS +#if defined(CONFIG_PERF_EVENTS) || (defined(CONFIG_KPROBES) && defined(CONFIG_SMP)) #define NR_ITERATIONS 1000 #define NR_OBJECTS 1000 -static void *objects[NR_OBJECTS]; +static struct test_kfree_rcu_struct *objects[NR_OBJECTS]; struct test_nolock_context { struct kunit *test; int callback_count; int alloc_ok; int alloc_fail; +#ifdef CONFIG_PERF_EVENTS struct perf_event *event; +#endif +#if defined(CONFIG_KPROBES) && defined(CONFIG_SMP) + struct kprobe kprobe; +#endif }; +static void test_kmalloc_and_friends(void) +{ + int i, j; + bool can_use_kfree_rcu = !IS_BUILTIN(CONFIG_SLUB_KUNIT_TEST); + + for (i = 0; i < NR_ITERATIONS; i++) { + for (j = 0; j < NR_OBJECTS; j++) { + gfp_t gfp = (i & 1) ? GFP_KERNEL : GFP_KERNEL_ACCOUNT; + + objects[j] = kmalloc_obj(*objects[j], gfp); + if (!objects[j]) { + j--; + while (j >= 0) + kfree(objects[j--]); + return; + } + } + + for (j = 0; j < NR_OBJECTS; j++) { + if (can_use_kfree_rcu && (i & 2)) + kfree_rcu(objects[j], rcu); + else + kfree(objects[j]); + } + } +} + +static void test_nolock(struct test_nolock_context *ctx) +{ + struct test_kfree_rcu_struct *objp; + gfp_t gfp; + bool can_use_kfree_rcu = !IS_BUILTIN(CONFIG_SLUB_KUNIT_TEST); + + /* __GFP_ACCOUNT to test kmalloc_nolock() in alloc_slab_obj_exts() */ + gfp = (ctx->callback_count & 1) ? 0 : __GFP_ACCOUNT; + objp = kmalloc_nolock(sizeof(*objp), gfp, NUMA_NO_NODE); + + if (objp) + ctx->alloc_ok++; + else + ctx->alloc_fail++; + + if (can_use_kfree_rcu && (ctx->callback_count & 2)) + kfree_rcu_nolock(objp, kvrcu); + else + kfree_nolock(objp); + + ctx->callback_count++; +} +#endif + +#ifdef CONFIG_PERF_EVENTS static struct perf_event_attr hw_attr = { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_CPU_CYCLES, @@ -315,67 +376,91 @@ static struct perf_event_attr hw_attr = { .sample_freq = 100000, }; -static void overflow_handler_test_kmalloc_kfree_nolock(struct perf_event *event, - struct perf_sample_data *data, - struct pt_regs *regs) +static void overflow_handler_test_nolock(struct perf_event *event, + struct perf_sample_data *data, + struct pt_regs *regs) { - void *objp; - gfp_t gfp; struct test_nolock_context *ctx = event->overflow_handler_context; - /* __GFP_ACCOUNT to test kmalloc_nolock() in alloc_slab_obj_exts() */ - gfp = (ctx->callback_count % 2) ? 0 : __GFP_ACCOUNT; - objp = kmalloc_nolock(64, gfp, NUMA_NO_NODE); - - if (objp) - ctx->alloc_ok++; - else - ctx->alloc_fail++; - - kfree_nolock(objp); - ctx->callback_count++; + test_nolock(ctx); } -static void test_kmalloc_kfree_nolock(struct kunit *test) +static bool enable_perf_events(struct test_nolock_context *ctx) { - int i, j; - struct test_nolock_context ctx = { .test = test }; struct perf_event *event; - bool alloc_fail = false; event = perf_event_create_kernel_counter(&hw_attr, -1, current, - overflow_handler_test_kmalloc_kfree_nolock, - &ctx); + overflow_handler_test_nolock, + ctx); + if (IS_ERR(event)) - kunit_skip(test, "Failed to create perf event"); - ctx.event = event; - perf_event_enable(ctx.event); - for (i = 0; i < NR_ITERATIONS; i++) { - for (j = 0; j < NR_OBJECTS; j++) { - gfp_t gfp = (i % 2) ? GFP_KERNEL : GFP_KERNEL_ACCOUNT; + return false; - objects[j] = kmalloc(64, gfp); - if (!objects[j]) { - j--; - while (j >= 0) - kfree(objects[j--]); - alloc_fail = true; - goto cleanup; - } - } - for (j = 0; j < NR_OBJECTS; j++) - kfree(objects[j]); - } + ctx->event = event; + perf_event_enable(ctx->event); + return true; +} -cleanup: - perf_event_disable(ctx.event); - perf_event_release_kernel(ctx.event); +static void disable_perf_events(struct test_nolock_context *ctx) +{ + kunit_info(ctx->test, "HW perf events: callback_count: %d, alloc_ok: %d, alloc_fail: %d\n", + ctx->callback_count, ctx->alloc_ok, ctx->alloc_fail); - kunit_info(test, "callback_count: %d, alloc_ok: %d, alloc_fail: %d\n", - ctx.callback_count, ctx.alloc_ok, ctx.alloc_fail); + perf_event_disable(ctx->event); + perf_event_release_kernel(ctx->event); +} - if (alloc_fail) - kunit_skip(test, "Allocation failed"); +static void test_kmalloc_nolock_and_friends_perf(struct kunit *test) +{ + struct test_nolock_context ctx = { .test = test }; + + if (!enable_perf_events(&ctx)) + kunit_skip(test, "Failed to enable perf event, skipping"); + + test_kmalloc_and_friends(); + + disable_perf_events(&ctx); + KUNIT_EXPECT_EQ(test, 0, slab_errors); +} +#endif + +#if defined(CONFIG_KPROBES) && defined(CONFIG_SMP) +static int slab_kprobe_pre_handler(struct kprobe *p, struct pt_regs *regs) +{ + struct test_nolock_context *ctx; + + ctx = container_of(p, struct test_nolock_context, kprobe); + test_nolock(ctx); + return 0; +} + +static bool register_slab_kprobes(struct test_nolock_context *ctx) +{ + ctx->kprobe.symbol_name = "slab_attach_kprobe_locked"; + ctx->kprobe.pre_handler = slab_kprobe_pre_handler; + + if (register_kprobe(&ctx->kprobe)) + return false; + return true; +} + +static void unregister_slab_kprobes(struct test_nolock_context *ctx) +{ + kunit_info(ctx->test, "kprobes: callback_count: %d, alloc_ok: %d, alloc_fail: %d\n", + ctx->callback_count, ctx->alloc_ok, ctx->alloc_fail); + unregister_kprobe(&ctx->kprobe); +} + +static void test_kmalloc_nolock_and_friends_kprobe(struct kunit *test) +{ + struct test_nolock_context ctx = { .test = test }; + + if (!register_slab_kprobes(&ctx)) + kunit_skip(test, "Failed to register kprobe, skipping"); + + test_kmalloc_and_friends(); + + unregister_slab_kprobes(&ctx); KUNIT_EXPECT_EQ(test, 0, slab_errors); } #endif @@ -405,7 +490,10 @@ static struct kunit_case test_cases[] = { KUNIT_CASE(test_leak_destroy), KUNIT_CASE(test_krealloc_redzone_zeroing), #ifdef CONFIG_PERF_EVENTS - KUNIT_CASE_SLOW(test_kmalloc_kfree_nolock), + KUNIT_CASE_SLOW(test_kmalloc_nolock_and_friends_perf), +#endif +#if defined(CONFIG_KPROBES) && defined(CONFIG_SMP) + KUNIT_CASE_SLOW(test_kmalloc_nolock_and_friends_kprobe), #endif {} }; diff --git a/mm/slab.h b/mm/slab.h index e64614749692..c6b73ed0fa17 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -24,11 +24,27 @@ #define SLAB_ALLOC_NO_RECURSE 0x04 /* prevent kmalloc() recursion */ #define SLAB_ALLOC_NO_OBJ_EXT 0x08 /* prevent obj_exts array allocation */ +#define SLAB_FREE_DEFAULT 0x00 /* no flags */ +#define SLAB_FREE_NOLOCK 0x01 /* spinning not allowed */ + +static inline unsigned int to_alloc_flags(unsigned int free_flags) +{ + if (free_flags & SLAB_FREE_NOLOCK) + return SLAB_ALLOC_NOLOCK; + else + return SLAB_ALLOC_DEFAULT; +} + static inline bool alloc_flags_allow_spinning(const unsigned int alloc_flags) { return !(alloc_flags & SLAB_ALLOC_NOLOCK); } +static inline bool free_flags_allow_spinning(const unsigned int free_flags) +{ + return !(free_flags & SLAB_FREE_NOLOCK); +} + void *__kmalloc_flags_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t flags, unsigned int alloc_flags, int node) __assume_kmalloc_alignment __alloc_size(1); @@ -331,6 +347,37 @@ static inline unsigned int obj_to_index(const struct kmem_cache *cache, return __obj_to_index(cache, slab_address(slab), obj); } +/* + * kvfree_rcu_head offset can be only less than page size. + * Calculate the start address while preserving the KASAN tag. + */ +static inline void *kvmalloc_obj_start_addr(void *head) +{ + unsigned long offset; + + if (unlikely(is_vmalloc_addr(head))) { + offset = offset_in_page(head); + } else { + struct slab *slab = virt_to_slab(head); + + if (!slab) { + offset = offset_in_page(head); + } else if (is_kfence_address(head)) { + offset = head - kfence_object_start(head); + } else { + struct kmem_cache *s = slab->slab_cache; + unsigned int idx = __obj_to_index(s, slab_address(slab), head); + void *obj = slab_address(slab) + s->size * idx; + + obj = fixup_red_left(s, obj); + obj = kasan_reset_tag(obj); + offset = kasan_reset_tag(head) - obj; + } + } + + return head - offset; +} + /* * State of the slab allocator. * @@ -431,7 +478,7 @@ static inline bool is_kmalloc_normal(struct kmem_cache *s) return !(s->flags & (SLAB_CACHE_DMA|SLAB_ACCOUNT|SLAB_RECLAIM_ACCOUNT|SLAB_NO_OBJ_EXT)); } -bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj); +bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags); void flush_all_rcu_sheaves(void); void flush_rcu_sheaves_on_cache(struct kmem_cache *s); @@ -899,7 +946,8 @@ void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) void __check_heap_object(const void *ptr, unsigned long n, const struct slab *slab, bool to_user); -void defer_free_barrier(void); +void deferred_work_barrier(void); +void defer_kfree_rcu(struct kvfree_rcu_head *head); static inline bool slub_debug_orig_size(struct kmem_cache *s) { diff --git a/mm/slab_common.c b/mm/slab_common.c index 1e1d3feec353..b55ef3531b67 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -558,7 +558,7 @@ void kmem_cache_destroy(struct kmem_cache *s) } /* Wait for deferred work from kmalloc/kfree_nolock() */ - defer_free_barrier(); + deferred_work_barrier(); cpus_read_lock(); mutex_lock(&slab_mutex); @@ -1296,13 +1296,40 @@ EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc); EXPORT_TRACEPOINT_SYMBOL(kfree); EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free); +void kfree_call_rcu_nolock(struct kvfree_rcu_head *head, void *ptr) +{ + struct slab *slab; + + if (!IS_ENABLED(CONFIG_KVFREE_RCU_BATCHED)) + goto fallback; + + if (unlikely(is_vmalloc_addr(ptr))) + goto fallback; + + slab = virt_to_slab(ptr); + if (unlikely(!slab)) + goto fallback; + + if (unlikely(IS_ENABLED(CONFIG_NUMA) && slab_nid(slab) != numa_mem_id())) + goto fallback; + + if (unlikely(!__kfree_rcu_sheaf(slab->slab_cache, ptr, SLAB_FREE_NOLOCK))) + goto fallback; + + return; + +fallback: + defer_kfree_rcu(head); +} +EXPORT_SYMBOL_GPL(kfree_call_rcu_nolock); + #ifndef CONFIG_KVFREE_RCU_BATCHED -void kvfree_call_rcu(struct rcu_head *head, void *ptr) +void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr) { if (head) { kasan_record_aux_stack(ptr); - call_rcu(head, kvfree_rcu_cb); + call_rcu(&head->head, kvfree_rcu_cb); return; } @@ -1313,6 +1340,18 @@ void kvfree_call_rcu(struct rcu_head *head, void *ptr) } EXPORT_SYMBOL_GPL(kvfree_call_rcu); +void kvfree_rcu_barrier(void) +{ + deferred_work_barrier(); + rcu_barrier(); +} + +void kvfree_rcu_barrier_on_cache(struct kmem_cache *s) +{ + deferred_work_barrier(); + rcu_barrier(); +} + void __init kvfree_rcu_init(void) { } @@ -1379,7 +1418,7 @@ struct kvfree_rcu_bulk_data { struct kfree_rcu_cpu_work { struct rcu_work rcu_work; - struct rcu_head *head_free; + struct kvfree_rcu_head *head_free; struct rcu_gp_oldstate head_free_gp_snap; struct list_head bulk_head_free[FREE_N_CHANNELS]; struct kfree_rcu_cpu *krcp; @@ -1415,7 +1454,7 @@ struct kfree_rcu_cpu_work { struct kfree_rcu_cpu { // Objects queued on a linked list // through their rcu_head structures. - struct rcu_head *head; + struct kvfree_rcu_head *head; unsigned long head_gp_snap; atomic_t head_count; @@ -1556,12 +1595,12 @@ kvfree_rcu_bulk(struct kfree_rcu_cpu *krcp, } static void -kvfree_rcu_list(struct rcu_head *head) +kvfree_rcu_list(struct kvfree_rcu_head *head) { - struct rcu_head *next; + struct kvfree_rcu_head *next; for (; head; head = next) { - void *ptr = (void *) head->func; + void *ptr = kvmalloc_obj_start_addr(head); unsigned long offset = (void *) head - ptr; next = head->next; @@ -1585,7 +1624,7 @@ static void kfree_rcu_work(struct work_struct *work) unsigned long flags; struct kvfree_rcu_bulk_data *bnode, *n; struct list_head bulk_head[FREE_N_CHANNELS]; - struct rcu_head *head; + struct kvfree_rcu_head *head; struct kfree_rcu_cpu *krcp; struct kfree_rcu_cpu_work *krwp; struct rcu_gp_oldstate head_gp_snap; @@ -1628,6 +1667,14 @@ static bool kfree_rcu_sheaf(void *obj) { struct kmem_cache *s; struct slab *slab; + unsigned int free_flags = SLAB_FREE_DEFAULT; + + /* + * It is not safe to spin on PREEMPT_RT because the kernel might be + * holding a raw spinlock and slab acquires sleeping locks. + */ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + free_flags = SLAB_FREE_NOLOCK; if (is_vmalloc_addr(obj)) return false; @@ -1638,7 +1685,7 @@ static bool kfree_rcu_sheaf(void *obj) s = slab->slab_cache; if (likely(!IS_ENABLED(CONFIG_NUMA) || slab_nid(slab) == numa_mem_id())) - return __kfree_rcu_sheaf(s, obj); + return __kfree_rcu_sheaf(s, obj, free_flags); return false; } @@ -1708,7 +1755,7 @@ kvfree_rcu_drain_ready(struct kfree_rcu_cpu *krcp) { struct list_head bulk_ready[FREE_N_CHANNELS]; struct kvfree_rcu_bulk_data *bnode, *n; - struct rcu_head *head_ready = NULL; + struct kvfree_rcu_head *head_ready = NULL; unsigned long flags; int i; @@ -1971,7 +2018,7 @@ void __init kfree_rcu_scheduler_running(void) * be free'd in workqueue context. This allows us to: batch requests together to * reduce the number of grace periods during heavy kfree_rcu()/kvfree_rcu() load. */ -void kvfree_call_rcu(struct rcu_head *head, void *ptr) +void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr) { unsigned long flags; struct kfree_rcu_cpu *krcp; @@ -1987,7 +2034,7 @@ void kvfree_call_rcu(struct rcu_head *head, void *ptr) if (!head) might_sleep(); - if (!IS_ENABLED(CONFIG_PREEMPT_RT) && kfree_rcu_sheaf(ptr)) + if (kfree_rcu_sheaf(ptr)) return; // Queue the object but don't yet schedule the batch. @@ -2009,7 +2056,6 @@ void kvfree_call_rcu(struct rcu_head *head, void *ptr) // Inline if kvfree_rcu(one_arg) call. goto unlock_return; - head->func = ptr; head->next = krcp->head; WRITE_ONCE(krcp->head, head); atomic_inc(&krcp->head_count); @@ -2131,7 +2177,6 @@ void kvfree_rcu_barrier(void) flush_all_rcu_sheaves(); __kvfree_rcu_barrier(); } -EXPORT_SYMBOL_GPL(kvfree_rcu_barrier); /** * kvfree_rcu_barrier_on_cache - Wait for in-flight kvfree_rcu() calls on a @@ -2142,20 +2187,18 @@ EXPORT_SYMBOL_GPL(kvfree_rcu_barrier); */ void kvfree_rcu_barrier_on_cache(struct kmem_cache *s) { + /* kfree_rcu_nolock() might have deferred frees even without sheaves */ + deferred_work_barrier(); + if (cache_has_sheaves(s)) { cpus_read_lock(); flush_rcu_sheaves_on_cache(s); cpus_read_unlock(); - rcu_barrier(); } - /* - * TODO: Introduce a version of __kvfree_rcu_barrier() that works - * on a specific slab cache. - */ + rcu_barrier(); __kvfree_rcu_barrier(); } -EXPORT_SYMBOL_GPL(kvfree_rcu_barrier_on_cache); static unsigned long kfree_rcu_shrink_count(struct shrinker *shrink, struct shrink_control *sc) diff --git a/mm/slub.c b/mm/slub.c index ec6f7504f76f..b0cd0572e2f2 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -429,6 +429,8 @@ struct slab_sheaf { union { struct rcu_head rcu_head; struct list_head barn_list; + /* only used to defer call_rcu() in unknown context */ + struct llist_node llnode; /* only used for prefilled sheafs */ struct { unsigned int capacity; @@ -911,6 +913,24 @@ static inline void slab_set_obj_exts_in_object(struct slab *slab) } #endif +/* + * A no-op function used to attach kprobe handlers in slub_kunit tests. + * The barrier is needed to prevent the compiler from optimizing out callsites. + */ +#if defined(CONFIG_DEBUG_VM) || defined(CONFIG_PROVE_LOCKING) +static noinline void slab_attach_kprobe_locked(void) +{ + barrier(); +} +#else +static inline void slab_attach_kprobe_locked(void) { } +#endif + +#define slab_lockdep_assert_held(lock) do { \ + lockdep_assert_held(lock); \ + slab_attach_kprobe_locked(); \ +} while (0) + #ifdef CONFIG_SLUB_DEBUG /* @@ -1668,7 +1688,7 @@ static void add_full(struct kmem_cache *s, if (!(s->flags & SLAB_STORE_USER)) return; - lockdep_assert_held(&n->list_lock); + slab_lockdep_assert_held(&n->list_lock); list_add(&slab->slab_list, &n->full); } @@ -1677,7 +1697,7 @@ static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct if (!(s->flags & SLAB_STORE_USER)) return; - lockdep_assert_held(&n->list_lock); + slab_lockdep_assert_held(&n->list_lock); list_del(&slab->slab_list); } @@ -2821,7 +2841,8 @@ static inline struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, return __alloc_empty_sheaf(s, gfp, alloc_flags, s->sheaf_capacity); } -static void free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf) +static void __free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf, + unsigned int free_flags) { /* * If the sheaf was created with SLAB_ALLOC_NO_RECURSE flag then its @@ -2833,11 +2854,20 @@ static void free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf) mark_obj_codetag_empty(sheaf); VM_WARN_ON_ONCE(sheaf->size > 0); - kfree(sheaf); + + if (unlikely(free_flags & SLAB_FREE_NOLOCK)) + kfree_nolock(sheaf); + else + kfree(sheaf); stat(s, SHEAF_FREE); } +static void free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf) +{ + __free_empty_sheaf(s, sheaf, SLAB_FREE_DEFAULT); +} + static unsigned int refill_objects(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int min, unsigned int max); @@ -2890,7 +2920,7 @@ static unsigned int __sheaf_flush_main_batch(struct kmem_cache *s) void *objects[PCS_BATCH_MAX]; struct slab_sheaf *sheaf; - lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); + slab_lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); pcs = this_cpu_ptr(s->cpu_sheaves); sheaf = pcs->main; @@ -3574,7 +3604,7 @@ __add_partial(struct kmem_cache_node *n, struct slab *slab, enum add_mode mode) static inline void add_partial(struct kmem_cache_node *n, struct slab *slab, enum add_mode mode) { - lockdep_assert_held(&n->list_lock); + slab_lockdep_assert_held(&n->list_lock); __add_partial(n, slab, mode); } @@ -3588,7 +3618,7 @@ static inline void clear_node_partial_state(struct kmem_cache_node *n, static inline void remove_partial(struct kmem_cache_node *n, struct slab *slab) { - lockdep_assert_held(&n->list_lock); + slab_lockdep_assert_held(&n->list_lock); list_del(&slab->slab_list); clear_node_partial_state(n, slab); } @@ -3604,7 +3634,7 @@ static void *alloc_single_from_partial(struct kmem_cache *s, { void *object; - lockdep_assert_held(&n->list_lock); + slab_lockdep_assert_held(&n->list_lock); #ifdef CONFIG_SLUB_DEBUG if (s->flags & SLAB_CONSISTENCY_CHECKS) { @@ -4073,6 +4103,22 @@ static void flush_all(struct kmem_cache *s) cpus_read_unlock(); } +struct deferred_percpu_work { + struct llist_head objects; + struct llist_head objects_by_rcu; + struct llist_head rcu_sheaves; + struct irq_work work; +}; + +static void deferred_percpu_work_fn(struct irq_work *work); + +static DEFINE_PER_CPU(struct deferred_percpu_work, deferred_percpu_work) = { + .objects = LLIST_HEAD_INIT(objects), + .objects_by_rcu = LLIST_HEAD_INIT(objects_by_rcu), + .rcu_sheaves = LLIST_HEAD_INIT(rcu_sheaves), + .work = IRQ_WORK_INIT(deferred_percpu_work_fn), +}; + static void flush_rcu_sheaf(struct work_struct *w) { struct slub_percpu_sheaves *pcs; @@ -4132,6 +4178,8 @@ void flush_all_rcu_sheaves(void) { struct kmem_cache *s; + deferred_work_barrier(); + cpus_read_lock(); mutex_lock(&slab_mutex); @@ -4669,7 +4717,7 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, struct node_barn *barn; bool allow_spin; - lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); + slab_lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); /* Bootstrap or debug cache, back off */ if (unlikely(!cache_has_sheaves(s))) { @@ -5794,7 +5842,7 @@ static void __pcs_install_empty_sheaf(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, struct slab_sheaf *empty, struct node_barn *barn) { - lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); + slab_lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); /* This is what we expect to find if nobody interrupted us. */ if (likely(!pcs->spare)) { @@ -5845,7 +5893,7 @@ __pcs_replace_full_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, bool put_fail; restart: - lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); + slab_lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); /* Bootstrap or debug cache, back off */ if (unlikely(!cache_has_sheaves(s))) { @@ -6046,24 +6094,26 @@ static void rcu_free_sheaf(struct rcu_head *head) * kvfree_call_rcu() can be called while holding a raw_spinlock_t. Since * __kfree_rcu_sheaf() may acquire a spinlock_t (sleeping lock on PREEMPT_RT), * this would violate lock nesting rules. Therefore, kvfree_call_rcu() avoids - * this problem by bypassing the sheaves layer entirely on PREEMPT_RT. + * this problem by passing SLAB_FREE_NOLOCK on PREEMPT_RT. * * However, lockdep still complains that it is invalid to acquire spinlock_t * while holding raw_spinlock_t, even on !PREEMPT_RT where spinlock_t is a * spinning lock. Tell lockdep that acquiring spinlock_t is valid here - * by temporarily raising the wait-type to LD_WAIT_CONFIG. + * by temporarily raising the wait-type to LD_WAIT_CONFIG. Skip the lockdep map + * on PREEMPT_RT to avoid suppressing valid lockdep warnings. */ static DEFINE_WAIT_OVERRIDE_MAP(kfree_rcu_sheaf_map, LD_WAIT_CONFIG); -bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) +bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags) { struct slub_percpu_sheaves *pcs; struct slab_sheaf *rcu_sheaf; + bool allow_spin = free_flags_allow_spinning(free_flags); - if (WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT))) - return false; + VM_WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT) && allow_spin); - lock_map_acquire_try(&kfree_rcu_sheaf_map); + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + lock_map_acquire_try(&kfree_rcu_sheaf_map); if (!local_trylock(&s->cpu_sheaves->lock)) goto fail; @@ -6071,9 +6121,10 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) pcs = this_cpu_ptr(s->cpu_sheaves); if (unlikely(!pcs->rcu_free)) { - struct slab_sheaf *empty; struct node_barn *barn; + unsigned int alloc_flags = to_alloc_flags(free_flags); + gfp_t gfp = allow_spin ? GFP_NOWAIT : __GFP_NOWARN; /* Bootstrap or debug cache, fall back */ if (unlikely(!cache_has_sheaves(s))) { @@ -6093,7 +6144,7 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) goto fail; } - empty = barn_get_empty_sheaf(barn, true); + empty = barn_get_empty_sheaf(barn, allow_spin); if (empty) { pcs->rcu_free = empty; @@ -6102,20 +6153,20 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) local_unlock(&s->cpu_sheaves->lock); - empty = alloc_empty_sheaf(s, GFP_NOWAIT, SLAB_ALLOC_DEFAULT); + empty = alloc_empty_sheaf(s, gfp, alloc_flags); if (!empty) goto fail; if (!local_trylock(&s->cpu_sheaves->lock)) { - barn_put_empty_sheaf(barn, empty); + __free_empty_sheaf(s, empty, free_flags); goto fail; } pcs = this_cpu_ptr(s->cpu_sheaves); if (unlikely(pcs->rcu_free)) - barn_put_empty_sheaf(barn, empty); + __free_empty_sheaf(s, empty, free_flags); else pcs->rcu_free = empty; } @@ -6141,18 +6192,34 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) * we flush before local_unlock to make sure a racing * flush_all_rcu_sheaves() doesn't miss this sheaf */ - if (rcu_sheaf) - call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf); + if (rcu_sheaf) { + /* + * With !allow_spin, we might have interrupted call_rcu()'s + * IRQ-disabled critical section. If IRQs are not disabled, + * we know that's not the case. + */ + if (unlikely(!allow_spin && irqs_disabled())) { + struct deferred_percpu_work *dpw; + + dpw = this_cpu_ptr(&deferred_percpu_work); + if (llist_add(&rcu_sheaf->llnode, &dpw->rcu_sheaves)) + irq_work_queue(&dpw->work); + } else { + call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf); + } + } local_unlock(&s->cpu_sheaves->lock); stat(s, FREE_RCU_SHEAF); - lock_map_release(&kfree_rcu_sheaf_map); + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + lock_map_release(&kfree_rcu_sheaf_map); return true; fail: stat(s, FREE_RCU_SHEAF_FAIL); - lock_map_release(&kfree_rcu_sheaf_map); + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + lock_map_release(&kfree_rcu_sheaf_map); return false; } @@ -6341,31 +6408,22 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) } } -struct defer_free { - struct llist_head objects; - struct irq_work work; -}; - -static void free_deferred_objects(struct irq_work *work); - -static DEFINE_PER_CPU(struct defer_free, defer_free_objects) = { - .objects = LLIST_HEAD_INIT(objects), - .work = IRQ_WORK_INIT(free_deferred_objects), -}; - /* * In PREEMPT_RT irq_work runs in per-cpu kthread, so it's safe * to take sleeping spin_locks from __slab_free(). * In !PREEMPT_RT irq_work will run after local_unlock_irqrestore(). */ -static void free_deferred_objects(struct irq_work *work) +static void deferred_percpu_work_fn(struct irq_work *work) { - struct defer_free *df = container_of(work, struct defer_free, work); - struct llist_head *objs = &df->objects; + struct deferred_percpu_work *dpw; + struct llist_head *objs, *objs_by_rcu, *rcu_sheaves; struct llist_node *llnode, *pos, *t; + struct slab_sheaf *sheaf, *next; - if (llist_empty(objs)) - return; + dpw = container_of(work, struct deferred_percpu_work, work); + rcu_sheaves = &dpw->rcu_sheaves; + objs = &dpw->objects; + objs_by_rcu = &dpw->objects_by_rcu; llnode = llist_del_all(objs); llist_for_each_safe(pos, t, llnode) { @@ -6389,27 +6447,51 @@ static void free_deferred_objects(struct irq_work *work) __slab_free(s, slab, x, x, 1, _THIS_IP_); stat(s, FREE_SLOWPATH); } + + llnode = llist_del_all(objs_by_rcu); + llist_for_each_safe(pos, t, llnode) { + void *head = pos; + void *objp = kvmalloc_obj_start_addr(head); + + kvfree_call_rcu(head, objp); + } + + llnode = llist_del_all(rcu_sheaves); + llist_for_each_entry_safe(sheaf, next, llnode, llnode) + call_rcu(&sheaf->rcu_head, rcu_free_sheaf); } static void defer_free(struct kmem_cache *s, void *head) { - struct defer_free *df; + struct deferred_percpu_work *dpw; guard(preempt)(); head = kasan_reset_tag(head); - df = this_cpu_ptr(&defer_free_objects); - if (llist_add(head + s->offset, &df->objects)) - irq_work_queue(&df->work); + dpw = this_cpu_ptr(&deferred_percpu_work); + if (llist_add(head + s->offset, &dpw->objects)) + irq_work_queue(&dpw->work); } -void defer_free_barrier(void) +void defer_kfree_rcu(struct kvfree_rcu_head *head) +{ + struct deferred_percpu_work *dpw; + + guard(preempt)(); + + dpw = this_cpu_ptr(&deferred_percpu_work); + if (llist_add((struct llist_node *)head, &dpw->objects_by_rcu)) + irq_work_queue(&dpw->work); +} + +/* Must be called before flush_rcu_sheaves_on_cache() */ +void deferred_work_barrier(void) { int cpu; for_each_possible_cpu(cpu) - irq_work_sync(&per_cpu_ptr(&defer_free_objects, cpu)->work); + irq_work_sync(&per_cpu_ptr(&deferred_percpu_work, cpu)->work); } static __fastpath_inline @@ -6668,43 +6750,21 @@ static void free_large_kmalloc(struct page *page, void *object) */ void kvfree_rcu_cb(struct rcu_head *head) { - void *obj = head; - struct page *page; - struct slab *slab; - struct kmem_cache *s; - void *slab_addr; + void *obj; + + obj = kvmalloc_obj_start_addr(head); if (is_vmalloc_addr(obj)) { - obj = (void *) PAGE_ALIGN_DOWN((unsigned long)obj); vfree(obj); - return; - } - - page = virt_to_page(obj); - slab = page_slab(page); - if (!slab) { - /* - * rcu_head offset can be only less than page size so no need to - * consider allocation order - */ - obj = (void *) PAGE_ALIGN_DOWN((unsigned long)obj); - free_large_kmalloc(page, obj); - return; - } - - s = slab->slab_cache; - slab_addr = slab_address(slab); - - if (is_kfence_address(obj)) { - obj = kfence_object_start(obj); } else { - unsigned int idx = __obj_to_index(s, slab_addr, obj); + struct page *page = virt_to_page(obj); + struct slab *slab = page_slab(page); - obj = slab_addr + s->size * idx; - obj = fixup_red_left(s, obj); + if (slab) + slab_free(slab->slab_cache, slab, obj, _RET_IP_); + else + free_large_kmalloc(page, obj); } - - slab_free(s, slab, obj, _RET_IP_); } /**