mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-31 15:22:21 -04:00
Code Issues Packages Projects Releases Wiki Activity

mm: mglru: promote mapped executable folios after first usage

Browse Source 
Classical LRU protects mapped executable file folios through commit
8cab4754d2 ("vmscan: make mapped executable pages the first class
citizen") and commit c909e99364 ("vmscan: activate executable pages
after first usage"), giving executable code a better chance to stay in
memory, avoiding IO thrashing and improving workload performance.

However, MGLRU's protection of mapped executable file folios is less
reliable.  Although shrink_folio_list() checks references, the access flag
of mapped executable file folios may have already been checked and cleared
by lru_gen_look_around() or walk_mm().  Additionally, folio_update_gen()
or lru_gen_set_refs() only sets the 'PG_referenced' flag for mapped
executable file folios, which causes shrink_folio_list() to ignore the
first usage of these mapped executable file folios and reclaim them
easily.

Follow the classical LRU's logic, promoting mapped executable file folios
after their first usage in folio_update_gen() and lru_gen_set_refs(),
giving executable code a better chance to stay in memory.

On my 32-core Arm machine, with the memcg limit set to 2G, running 'make
-j32' to build kernel showed some improvement in sys time.

base			patched
9248.543s		7861.579s

Link: https://lore.kernel.org/f57d94b1d85bb3d620d89bb739128f0d929bf9c6.1784509721.git.baolin.wang@linux.alibaba.com
Signed-off-by: Baolin Wang <baolin.wang@linux.alibaba.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Reviewed-by: Axel Rasmussen <axelrasmussen@google.com>
Cc: Barry Song <baohua@kernel.org>
Cc: David Hildenbrand <david@kernel.org>
Cc: Harry Yoo <harry@kernel.org>
Cc: Jann Horn <jannh@google.com>
Cc: Kairui Song <kasong@tencent.com>
Cc: Lance Yang <lance.yang@linux.dev>
Cc: Liam R. Howlett <liam@infradead.org>
Cc: Lorenzo Stoakes <ljs@kernel.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Rik van Riel <riel@surriel.com>
Cc: Shakeel Butt <shakeel.butt@linux.dev>
Cc: Vlastimil Babka <vbabka@kernel.org>
Cc: Wei Xu <weixugc@google.com>
Cc: Yuanchu Xie <yuanchu@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Baolin Wang
2026-07-20 19:12:02 +08:00
committed by Andrew Morton
parent b64727d264
commit 0ee06ee38a
1 changed files with 27 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
mm/vmscan.c
View File

@@ -841,10 +841,16 @@ enum folio_references {
* with PG_active set. In contrast, the aging (page table walk) path uses
* folio_update_gen().
*/
static bool lru_gen_set_refs(struct folio *folio)
static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags)
{
/* see the comment on LRU_REFS_FLAGS */
if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) {
/* Activate file-backed executable folios after first usage. */
if (is_exec_file_folio(folio, vma_flags)) {
set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset));
return true;
}
set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
return false;
}
@@ -857,7 +863,7 @@ static bool lru_gen_set_refs(struct folio *folio)
return true;
}
#else
static bool lru_gen_set_refs(struct folio *folio)
static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags)
{
return false;
}
@@ -892,7 +898,7 @@ static enum folio_references folio_check_references(struct folio *folio,
if (!referenced_ptes)
return FOLIOREF_RECLAIM;
return lru_gen_set_refs(folio) ? FOLIOREF_ACTIVATE : FOLIOREF_KEEP;
return lru_gen_set_refs(folio, &vma_flags) ? FOLIOREF_ACTIVATE : FOLIOREF_KEEP;
}
referenced_folio = folio_test_clear_referenced(folio);
@@ -3208,14 +3214,19 @@ static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv)
******************************************************************************/
/* promote pages accessed through page tables */
static int folio_update_gen(struct folio *folio, int gen)
static int folio_update_gen(struct folio *folio, int gen, const vma_flags_t *vma_flags)
{
unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);
/* see the comment on LRU_REFS_FLAGS */
if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) {
/*
* See the comment on LRU_REFS_FLAGS, and activate file-backed
* executable folios after first usage to avoid typical IO
* thrashing from reclaiming.
*/
if (!folio_test_referenced(folio) && !folio_test_workingset(folio) &&
!is_exec_file_folio(folio, vma_flags)) {
set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
return -1;
}
@@ -3448,8 +3459,8 @@ static bool suitable_to_scan(int total, int young)
return young * n >= total;
}
static void walk_update_folio(struct lru_gen_mm_walk *walk, struct folio *folio,
int new_gen, bool dirty)
static void walk_update_folio(struct lru_gen_mm_walk *walk, struct vm_area_struct *vma,
struct folio *folio, int new_gen, bool dirty)
{
int old_gen;
@@ -3462,10 +3473,10 @@ static void walk_update_folio(struct lru_gen_mm_walk *walk, struct folio *folio,
folio_mark_dirty(folio);
if (walk) {
old_gen = folio_update_gen(folio, new_gen);
old_gen = folio_update_gen(folio, new_gen, &vma->flags);
if (old_gen >= 0 && old_gen != new_gen)
update_batch_size(walk, folio, old_gen, new_gen);
} else if (lru_gen_set_refs(folio)) {
} else if (lru_gen_set_refs(folio, &vma->flags)) {
old_gen = folio_lru_gen(folio);
if (old_gen >= 0 && old_gen != new_gen)
folio_activate(folio);
@@ -3538,7 +3549,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
continue;
if (last != folio) {
walk_update_folio(walk, last, gen, dirty);
walk_update_folio(walk, args->vma, last, gen, dirty);
last = folio;
dirty = false;
@@ -3551,7 +3562,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
walk->mm_stats[MM_LEAF_YOUNG] += nr;
}
walk_update_folio(walk, last, gen, dirty);
walk_update_folio(walk, args->vma, last, gen, dirty);
last = NULL;
if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end))
@@ -3629,7 +3640,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area
goto next;
if (last != folio) {
walk_update_folio(walk, last, gen, dirty);
walk_update_folio(walk, vma, last, gen, dirty);
last = folio;
dirty = false;
@@ -3643,7 +3654,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area
i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1;
} while (i <= MIN_LRU_BATCH);
walk_update_folio(walk, last, gen, dirty);
walk_update_folio(walk, vma, last, gen, dirty);
lazy_mmu_mode_disable();
spin_unlock(ptl);
@@ -4278,7 +4289,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr)
continue;
if (last != folio) {
walk_update_folio(walk, last, gen, dirty);
walk_update_folio(walk, vma, last, gen, dirty);
last = folio;
dirty = false;
@@ -4290,7 +4301,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr)
young += nr;
}
walk_update_folio(walk, last, gen, dirty);
walk_update_folio(walk, vma, last, gen, dirty);
lazy_mmu_mode_disable();