mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-07-23 00:27:30 -04:00
Code Issues Packages Projects Releases Wiki Activity

mm/slub: detach and reattach partial slabs in batch

Browse Source 
get_partial_node_bulk() moves each selected slab from the node's
partial list to the local pc->slabs list using a remove_partial() and
list_add() pair. In practice, the loop often detaches several adjacent
slabs. Doing this individually repeatedly manipulates list pointers
while holding n->list_lock, which causes unnecessary churn.

To demonstrate this, the counts below show how often single vs. multiple
consecutive slabs are retrieved during a will-it-scale mmap stress test:

consecutive_slabs_count        frequency
= 1                            277345324
= 2                            335238023
= 3                            175717884
>= 4                           88862337

The data confirms that retrieving multiple contiguous slabs is highly
frequent.

To optimize this, track contiguous runs of matching slabs and move each
run in a single operation using list_bulk_move_tail(). This reduces list
pointer churn inside the lock critical section.

Apply the same optimization to __refill_objects_node() when reattaching
leftover partial slabs back to the node's partial list.

The will-it-scale mmap benchmark shows a 2% ~ 5% performance improvement
after applying this patch.

Signed-off-by: Hao Li <hao.li@linux.dev>
Link: https://patch.msgid.link/20260529035120.81304-3-hao.li@linux.dev
Reviewed-by: Harry Yoo (Oracle) <harry@kernel.org>
Signed-off-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
This commit is contained in:
Hao Li
2026-05-29 11:50:52 +08:00
committed by Vlastimil Babka (SUSE)
parent 7e23073874
commit 0fc52deec1
1 changed files with 20 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
mm/slub.c
View File

@@ -3751,6 +3751,7 @@ static bool get_partial_node_bulk(struct kmem_cache *s,
bool allow_spin)
{
struct slab *slab, *slab2;
struct slab *first = NULL, *last = NULL;
unsigned int total_free = 0;
unsigned long flags;
@@ -3769,8 +3770,15 @@ static bool get_partial_node_bulk(struct kmem_cache *s,
struct freelist_counters flc;
unsigned int slab_free;
if (!pfmemalloc_match(slab, pc->flags))
if (!pfmemalloc_match(slab, pc->flags)) {
if (first) {
list_bulk_move_tail(&pc->slabs,
&first->slab_list,
&last->slab_list);
first = NULL;
}
continue;
}
/*
* determine the number of free objects in the slab racily
@@ -3787,15 +3795,20 @@ static bool get_partial_node_bulk(struct kmem_cache *s,
&& total_free + slab_free > pc->max_objects)
break;
remove_partial(n, slab);
list_add(&slab->slab_list, &pc->slabs);
if (!first)
first = slab;
last = slab;
clear_node_partial_state(n, slab);
total_free += slab_free;
if (total_free >= pc->max_objects)
break;
}
if (first)
list_bulk_move_tail(&pc->slabs, &first->slab_list,
&last->slab_list);
spin_unlock_irqrestore(&n->list_lock, flags);
return total_free > 0;
}
@@ -7205,11 +7218,10 @@ __refill_objects_node(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int mi
if (!list_empty(&pc.slabs)) {
spin_lock_irqsave(&n->list_lock, flags);
list_for_each_entry_safe(slab, slab2, &pc.slabs, slab_list) {
list_for_each_entry(slab, &pc.slabs, slab_list)
set_node_partial_state(n, slab);
list_del(&slab->slab_list);
add_partial(n, slab, ADD_TO_TAIL);
}
list_splice_tail(&pc.slabs, &n->partial);
spin_unlock_irqrestore(&n->list_lock, flags);
}