mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-28 12:13:39 -04:00
Code Issues Packages Projects Releases Wiki Activity

mm: page_alloc: move capture_control to the page allocator

Browse Source 
The compaction capturing code assumes the allocation request order and
compaction target order are the same.  That won't be true once defrag_mode
promotes sub-block allocations to pageblock-order compaction: compaction
targets the larger order, while capture should remain at the original
allocation order.

Move the capture_control to the page allocator and give it its own copies
of what the page freeing path matches against - zone, migratetype and the
allocation order - rather than reaching into compaction's live
compact_control.  __alloc_pages_direct_compact() fills in migratetype and
order, and installs and hides current->capture_control around the whole
compaction call; try_to_compact_pages() aims capc->zone at each zone while
it is being compacted.  compact_zone_order() no longer deals with capture
at all.

Pass the capture_control through try_to_compact_pages() /
compact_zone_order() in place of the bare struct page **.

No functional change.

Link: https://lore.kernel.org/20260722150006.3848560-4-hannes@cmpxchg.org
Fixes: e3aa7df331 ("mm: page_alloc: defrag_mode")
Signed-off-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
Co-developed-by: Johannes Weiner <hannes@cmpxchg.org>
Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Reviewed-by: Gregory Price <gourry@gourry.net>
Cc: Brendan Jackman <brendan.jackman@linux.dev>
Cc: Brendan Jackman <jackmanb@google.com>
Cc: David Hildenbrand <david@kernel.org>
Cc: Liam R. Howlett <liam@infradead.org>
Cc: Lorenzo Stoakes <ljs@kernel.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Shakeel Butt <shakeel.butt@linux.dev>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Zi Yan <ziy@nvidia.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Vlastimil Babka (SUSE)
2026-07-22 10:56:46 -04:00
committed by Andrew Morton
parent 1b4b697a57
commit aee220f565
4 changed files with 57 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
include/linux/compaction.h
View File

@@ -58,6 +58,7 @@ enum compact_result {
};
struct alloc_context; /* in mm/internal.h */
struct capture_control; /* in mm/internal.h */
/*
* Number of free order-0 pages that should be available above given watermark
@@ -92,7 +93,7 @@ extern int fragmentation_index(struct zone *zone, unsigned int order);
extern enum compact_result try_to_compact_pages(gfp_t gfp_mask,
unsigned int order, unsigned int alloc_flags,
const struct alloc_context *ac, enum compact_priority prio,
struct page **page);
struct capture_control *capc);
extern void reset_isolation_suitable(pg_data_t *pgdat);
extern bool compaction_suitable(struct zone *zone, int order,
unsigned long watermark, int highest_zoneidx);

50
mm/compaction.c
View File

@@ -2800,9 +2800,8 @@ compact_zone(struct compact_control *cc, struct capture_control *capc)
static enum compact_result compact_zone_order(struct zone *zone, int order,
gfp_t gfp_mask, enum compact_priority prio,
unsigned int alloc_flags, int highest_zoneidx,
struct page **capture)
struct capture_control *capc)
{
enum compact_result ret;
struct compact_control cc = {
.order = order,
.search_order = order,
@@ -2817,38 +2816,8 @@ static enum compact_result compact_zone_order(struct zone *zone, int order,
.ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY),
.ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY)
};
struct capture_control capc = {
.cc = &cc,
.page = NULL,
};
/*
* Make sure the structs are really initialized before we expose the
* capture control, in case we are interrupted and the interrupt handler
* frees a page.
*/
barrier();
WRITE_ONCE(current->capture_control, &capc);
ret = compact_zone(&cc, &capc);
/*
* Make sure we hide capture control first before we read the captured
* page pointer, otherwise an interrupt could free and capture a page
* and we would leak it.
*/
WRITE_ONCE(current->capture_control, NULL);
*capture = READ_ONCE(capc.page);
/*
* Technically, it is also possible that compaction is skipped but
* the page is still captured out of luck(IRQ came and freed the page).
* Returning COMPACT_SUCCESS in such cases helps in properly accounting
* the COMPACT[STALL|FAIL] when compaction is skipped.
*/
if (*capture)
ret = COMPACT_SUCCESS;
return ret;
return compact_zone(&cc, capc);
}
/**
@@ -2858,13 +2827,13 @@ static enum compact_result compact_zone_order(struct zone *zone, int order,
* @alloc_flags: The allocation flags of the current allocation
* @ac: The context of current allocation
* @prio: Determines how hard direct compaction should try to succeed
* @capture: Pointer to free page created by compaction will be stored here
* @capc: Free page capture bypassing the freelist
*
* This is the main entry point for direct page compaction.
*/
enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
unsigned int alloc_flags, const struct alloc_context *ac,
enum compact_priority prio, struct page **capture)
enum compact_priority prio, struct capture_control *capc)
{
struct zoneref *z;
struct zone *zone;
@@ -2891,8 +2860,17 @@ enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
continue;
}
WRITE_ONCE(capc->zone, zone);
status = compact_zone_order(zone, order, gfp_mask, prio,
alloc_flags, ac->highest_zoneidx, capture);
alloc_flags, ac->highest_zoneidx, capc);
WRITE_ONCE(capc->zone, NULL);
/* Stop if a page has been captured */
if (READ_ONCE(capc->page))
status = COMPACT_SUCCESS;
rc = max(status, rc);
/* The allocation should succeed, stop compacting */

4
mm/internal.h
View File

@@ -868,7 +868,9 @@ struct compact_control {
* immediately when one is created during the free path.
*/
struct capture_control {
struct compact_control *cc;
struct zone *zone;
int migratetype;
int order;
struct page *page;
};

45
mm/page_alloc.c
View File

@@ -725,14 +725,14 @@ static inline struct capture_control *task_capc(struct zone *zone)
return unlikely(capc) &&
!(current->flags & PF_KTHREAD) &&
!capc->page &&
capc->cc->zone == zone ? capc : NULL;
capc->zone == zone ? capc : NULL;
}
static inline bool
compaction_capture(struct capture_control *capc, struct page *page,
int order, int migratetype)
{
if (!capc || order != capc->cc->order)
if (!capc || order != capc->order)
return false;
/* Do not accidentally pollute CMA or isolated regions*/
@@ -748,12 +748,12 @@ compaction_capture(struct capture_control *capc, struct page *page,
* have trouble finding a high-order free page.
*/
if (order < pageblock_order && migratetype == MIGRATE_MOVABLE &&
capc->cc->migratetype != MIGRATE_MOVABLE)
capc->migratetype != MIGRATE_MOVABLE)
return false;
if (migratetype != capc->cc->migratetype)
trace_mm_page_alloc_extfrag(page, capc->cc->order, order,
capc->cc->migratetype, migratetype);
if (migratetype != capc->migratetype)
trace_mm_page_alloc_extfrag(page, capc->order, order,
capc->migratetype, migratetype);
capc->page = page;
return true;
@@ -4143,6 +4143,12 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
struct page *page = NULL;
unsigned long pflags;
unsigned int noreclaim_flag;
struct capture_control capc = {
.zone = NULL,
.migratetype = ac->migratetype,
.order = order,
.page = NULL,
};
if (!order)
return NULL;
@@ -4152,8 +4158,33 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
fs_reclaim_acquire(gfp_mask);
noreclaim_flag = memalloc_noreclaim_save();
/*
* Make sure the structs are really initialized before we expose the
* capture control, in case we are interrupted and the interrupt handler
* frees a page.
*/
barrier();
WRITE_ONCE(current->capture_control, &capc);
*compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
prio, &page);
prio, &capc);
/*
* Make sure we hide capture control first before we read the captured
* page pointer, otherwise an interrupt could free and capture a page
* and we would leak it.
*/
WRITE_ONCE(current->capture_control, NULL);
page = READ_ONCE(capc.page);
/*
* Technically, it is also possible that compaction is skipped but
* the page is still captured out of luck(IRQ came and freed the page).
* Returning COMPACT_SUCCESS in such cases helps in properly accounting
* the COMPACT[STALL|FAIL] when compaction is skipped.
*/
if (page)
*compact_result = COMPACT_SUCCESS;
memalloc_noreclaim_restore(noreclaim_flag);
fs_reclaim_release(gfp_mask);