mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2026-08-30 14:33:24 -04:00
mm/page_alloc: drop flag-conversion "optimisation"
This code uses flag equivalences to try to optimise conversion from GFP_
to ALLOC_ but there's no clear reason to believe it makes things faster.
Even if it gets rid of conditional branches, it just trades them for a
data dependency.
CPUs are pretty good at conditional branches. But, in my GCC x86 build it
doesn't look like there are any branches anyway, the compiler found some
conditional instruction tricks. (Caveat: This was extracted & annotated
by Gemini AI, I did not actually read the disasm myself)
Old code:
ae50: 8b 04 24 mov (%rsp),%eax # Load gfp_mask
...
ae5d: 41 89 c4 mov %eax,%r12d
ae64: 41 81 e4 20 08 00 00 and $0x820,%r12d # Mask both flags at once
...
ae6f: 44 89 e1 mov %r12d,%ecx
ae77: 83 c9 40 or $0x40,%ecx # OR with ALLOC_CPUSET (0x40)
ae7a: 89 4c 24 60 mov %ecx,0x60(%rsp) # Store to alloc_flags
New code:
For __GFP_HIGH ( 0x20 ):
It uses the Carry Flag (via sbb ) to conditionally add 0x20 to the base 0x40 ( ALLOC_CPUSET ) flag:
ae63: 83 e0 20 and $0x20,%eax # Test __GFP_HIGH
...
ae6a: 83 f8 01 cmp $0x1,%eax # Set carry flag if 0
ae6f: 45 19 e4 sbb %r12d,%r12d # %r12d = (gfp & 0x20) ? 0 : -1
ae80: 41 83 e4 e0 and $0xffffffe0,%r12d # %r12d = (gfp & 0x20) ? 0 : -32
ae87: 41 83 c4 60 add $0x60,%r12d # %r12d = (gfp & 0x20) ? 0x60 : 0x40
For __GFP_KSWAPD_RECLAIM ( 0x800 ):
It uses a conditional move ( cmov ) later in the function to set the ALLOC_KSWAPD ( 0x800 ) bit:
ae72: 25 00 08 00 00 and $0x800,%eax # Test __GFP_KSWAPD_RECLAIM
ae77: 89 44 24 30 mov %eax,0x30(%rsp) # Store result
...
af2c: 80 cf 08 or $0x8,%bh # Set ALLOC_KSWAPD (0x800) in temp reg
af2f: 45 85 c9 test %r9d,%r9d # Check if __GFP_KSWAPD_RECLAIM was set
af32: 0f 44 d8 cmove %eax,%ebx # If not, revert to flags without it
Testing with a modified version[0] of lib/free_pages_test.c (adding
printks with timing)...
Old results from a Sapphire Rapids consumer CPU:
[ 67.157118] page_alloc_test: Testing with GFP_KERNEL
[ 67.157122] page_alloc_test: Starting 1,000,000 allocations...
[ 70.704446] page_alloc_test: Completed. Time: 3543002 us (Avg: 3543.00 ns per alloc+free loop)
[ 70.704456] page_alloc_test: Testing with GFP_KERNEL | __GFP_COMP
[ 70.704460] page_alloc_test: Starting 1,000,000 allocations...
[ 70.944672] page_alloc_test: Completed. Time: 239980 us (Avg: 239.98 ns per alloc+free loop)
[ 70.944675] page_alloc_test: Test completed
New results:
[ 70.079015] page_alloc_test: Testing with GFP_KERNEL
[ 70.079020] page_alloc_test: Starting 1,000,000 allocations...
[ 73.669396] page_alloc_test: Completed. Time: 3586954 us (Avg: 3586.95 ns per alloc+free loop)
[ 73.669402] page_alloc_test: Testing with GFP_KERNEL | __GFP_COMP
[ 73.669405] page_alloc_test: Starting 1,000,000 allocations...
[ 73.905084] page_alloc_test: Completed. Time: 235496 us (Avg: 235.49 ns per alloc+free loop)
[ 73.905086] page_alloc_test: Test completed
Seems like a wash.
So, drop the flag value coupling here and let the compiler and CPU do
their job. Superscalar CPUs are pretty neat after all.
(Used AI for the disasm but the rest is all manual).
Link: https://lore.kernel.org/20260629-gfp-pessimisation-v2-1-311ece6a8637@google.com
Link: https://lore.kernel.org/20260615-gfp-pessimisation-v2-1-65f1319e6818@google.com
Link: 2ccdc84ef0/page-alloc-test/page-alloc-test.c [1]
Signed-off-by: Brendan Jackman <jackmanb@google.com>
Reviewed-by: Zi Yan <ziy@nvidia.com>
Reviewed-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
Reviewed-by: Gregory Price <gourry@gourry.net>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Harry Yoo (Oracle) <harry@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Brendan Jackman
committed by
Andrew Morton
Andrew Morton
parent
375c63b649
commit
52b71fe000
@@ -3741,13 +3741,10 @@ static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
|
||||
static inline unsigned int
|
||||
alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask)
|
||||
{
|
||||
unsigned int alloc_flags;
|
||||
unsigned int alloc_flags = 0;
|
||||
|
||||
/*
|
||||
* __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD
|
||||
* to save a branch.
|
||||
*/
|
||||
alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM);
|
||||
if (gfp_mask & __GFP_KSWAPD_RECLAIM)
|
||||
alloc_flags |= ALLOC_KSWAPD;
|
||||
|
||||
if (defrag_mode) {
|
||||
alloc_flags |= ALLOC_NOFRAGMENT;
|
||||
@@ -4480,22 +4477,16 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order)
|
||||
{
|
||||
unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
|
||||
|
||||
/*
|
||||
* __GFP_HIGH is assumed to be the same as ALLOC_MIN_RESERVE
|
||||
* and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD
|
||||
* to save two branches.
|
||||
*/
|
||||
BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_MIN_RESERVE);
|
||||
BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD);
|
||||
|
||||
/*
|
||||
* The caller may dip into page reserves a bit more if the caller
|
||||
* cannot run direct reclaim, or if the caller has realtime scheduling
|
||||
* policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will
|
||||
* set both ALLOC_NON_BLOCK and ALLOC_MIN_RESERVE(__GFP_HIGH).
|
||||
*/
|
||||
alloc_flags |= (__force int)
|
||||
(gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM));
|
||||
if (gfp_mask & __GFP_HIGH)
|
||||
alloc_flags |= ALLOC_MIN_RESERVE;
|
||||
if (gfp_mask & __GFP_KSWAPD_RECLAIM)
|
||||
alloc_flags |= ALLOC_KSWAPD;
|
||||
|
||||
if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) {
|
||||
/*
|
||||
|
||||
Reference in New Issue
Block a user