diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 5659f4b5a125..53207901d4c1 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -970,7 +970,10 @@ RUNTIME_CONST(ptr, __dentry_cache) \ RUNTIME_CONST(ptr, __names_cache) \ RUNTIME_CONST(ptr, __filp_cache) \ - RUNTIME_CONST(ptr, __bfilp_cache) + RUNTIME_CONST(ptr, __bfilp_cache) \ + RUNTIME_CONST(shift, __futex_shift) \ + RUNTIME_CONST(mask, __futex_mask) \ + RUNTIME_CONST(ptr, __futex_queues) /* Alignment must be consistent with (kunit_suite *) in include/kunit/test.h */ #define KUNIT_TABLE() \ diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 179b26e9c934..3bf6cb79f71a 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -48,23 +48,19 @@ #include +#include + #include "futex.h" #include "../locking/rtmutex_common.h" -/* - * The base of the bucket array and its size are always used together - * (after initialization only in futex_hash()), so ensure that they - * reside in the same cacheline. - */ -static struct { - unsigned long hashmask; - unsigned int hashshift; - struct futex_hash_bucket *queues[MAX_NUMNODES]; -} __futex_data __read_mostly __aligned(2*sizeof(long)); +static u32 __futex_mask __ro_after_init; +static u32 __futex_shift __ro_after_init; +static struct futex_hash_bucket **__futex_queues __ro_after_init; -#define futex_hashmask (__futex_data.hashmask) -#define futex_hashshift (__futex_data.hashshift) -#define futex_queues (__futex_data.queues) +static __always_inline struct futex_hash_bucket **futex_queues(void) +{ + return runtime_const_ptr(__futex_queues); +} struct futex_private_hash { int state; @@ -395,13 +391,13 @@ __futex_hash(union futex_key *key, struct futex_private_hash *fph, struct futex_ * NOTE: this isn't perfectly uniform, but it is fast and * handles sparse node masks. */ - node = (hash >> futex_hashshift) % nr_node_ids; + node = runtime_const_shift_right_32(hash, __futex_shift) % nr_node_ids; if (!node_possible(node)) { node = find_next_bit_wrap(node_possible_map.bits, nr_node_ids, node); } } - return &futex_queues[node][hash & futex_hashmask]; + return &futex_queues()[node][runtime_const_mask_32(hash, __futex_mask)]; } /** @@ -1922,7 +1918,7 @@ int futex_hash_allocate_default(void) * 16 <= threads * 4 <= global hash size */ buckets = roundup_pow_of_two(4 * threads); - buckets = clamp(buckets, 16, futex_hashmask + 1); + buckets = clamp(buckets, 16, __futex_mask + 1); if (current_buckets >= buckets) return 0; @@ -2020,10 +2016,21 @@ static int __init futex_init(void) hashsize = max(4, hashsize); hashsize = roundup_pow_of_two(hashsize); #endif - futex_hashshift = ilog2(hashsize); + __futex_mask = hashsize - 1; + __futex_shift = ilog2(hashsize); size = sizeof(struct futex_hash_bucket) * hashsize; order = get_order(size); + __futex_queues = kcalloc(nr_node_ids, sizeof(*__futex_queues), GFP_KERNEL); + + runtime_const_init(shift, __futex_shift); + runtime_const_init(mask, __futex_mask); + runtime_const_init(ptr, __futex_queues); + + barrier(); + + BUG_ON(!futex_queues()); + for_each_node(n) { struct futex_hash_bucket *table; @@ -2037,10 +2044,9 @@ static int __init futex_init(void) for (i = 0; i < hashsize; i++) futex_hash_bucket_init(&table[i]); - futex_queues[n] = table; + futex_queues()[n] = table; } - futex_hashmask = hashsize - 1; pr_info("futex hash table entries: %lu (%lu bytes on %d NUMA nodes, total %lu KiB, %s).\n", hashsize, size, num_possible_nodes(), size * num_possible_nodes() / 1024, order > MAX_PAGE_ORDER ? "vmalloc" : "linear");