From ac0f3be693be9cbd1dd2c4464915afc54972dcd5 Mon Sep 17 00:00:00 2001 From: Joshua Hahn Date: Wed, 24 Jun 2026 11:36:59 -0700 Subject: [PATCH 001/501] mm/memcontrol: remove unused for_each_mem_cgroup macro and cleanup Commit 7e1c0d6f58207 ("memcg: switch lruvec stats to rstat") removed the last caller of for_each_mem_cgroup back in 2021, and there have not been any new callers since. Remove the macro. A comment in mem_cgroup_css_online has also been out of date since 2021, when 2bfd36374edd9 ("mm: vmscan: consolidate shrinker_maps handling code") open-coded the for_each_mem_cgroup iterator. Update the comment. Finally, 99430ab8b804c ("mm: introduce BPF kfuncs to access memcg statistics and events") added a second declaration for memcg_events to include/linux/memcontrol.h, duplicating the one in mm/memcontrol-v1.h. Let's clean that up too. No functional changes intended. Link: https://lore.kernel.org/20260624183700.1152742-1-joshua.hahnjy@gmail.com Signed-off-by: Joshua Hahn Acked-by: Shakeel Butt Reviewed-by: SeongJae Park Acked-by: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin Signed-off-by: Andrew Morton --- mm/memcontrol-v1.h | 6 ------ mm/memcontrol.c | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/mm/memcontrol-v1.h b/mm/memcontrol-v1.h index f92f81108d5e..d3ed5b93290f 100644 --- a/mm/memcontrol-v1.h +++ b/mm/memcontrol-v1.h @@ -17,14 +17,8 @@ iter != NULL; \ iter = mem_cgroup_iter(root, iter, NULL)) -#define for_each_mem_cgroup(iter) \ - for (iter = mem_cgroup_iter(NULL, NULL, NULL); \ - iter != NULL; \ - iter = mem_cgroup_iter(NULL, iter, NULL)) - void drain_all_stock(struct mem_cgroup *root_memcg); -unsigned long memcg_events(struct mem_cgroup *memcg, int event); int memory_stat_show(struct seq_file *m, void *v); struct mem_cgroup *mem_cgroup_private_id_get_online(struct mem_cgroup *memcg, diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 6dc4888a90f3..5e06109f1f66 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -4217,7 +4217,7 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css) /* * A memcg must be visible for expand_shrinker_info() * by the time the maps are allocated. So, we allocate maps - * here, when for_each_mem_cgroup() can't skip it. + * here, when mem_cgroup_iter() can't skip it. */ if (alloc_shrinker_info(memcg)) goto offline_kmem; From 7698d52e33fc9e53346ae783b23c707b7994e32b Mon Sep 17 00:00:00 2001 From: Zenghui Yu Date: Wed, 24 Jun 2026 23:06:42 +0800 Subject: [PATCH 002/501] tools/mm: add thp_swap_allocator_test binary to .gitignore Tell git to ignore the generated binary for thp_swap_allocator_test.c. Link: https://lore.kernel.org/20260624150642.19749-1-zenghui.yu@linux.dev Signed-off-by: Zenghui Yu Reviewed-by: SeongJae Park Signed-off-by: Andrew Morton --- tools/mm/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/mm/.gitignore b/tools/mm/.gitignore index 922879f93fc8..1446a659e540 100644 --- a/tools/mm/.gitignore +++ b/tools/mm/.gitignore @@ -2,3 +2,4 @@ slabinfo page-types page_owner_sort +thp_swap_allocator_test From 3ade88423958139e3c361a07906a9967c96a988f Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Tue, 23 Jun 2026 12:57:21 +0000 Subject: [PATCH 003/501] mm/swap: rename subpage->page in folio_dup_swap/folio_put_swap Patch series "mm: drop "sub" prefix from various places". Patch 1 converts subpage->page : folios have pages, not subpages. Patch 2 drops "sub" from a function and a variable because the context is clear enough. This patch (of 2): Folios have pages, not subpages. Rename 'subpage' parameters to 'page'. Link: https://lore.kernel.org/20260623125723.2503832-1-dev.jain@arm.com Link: https://lore.kernel.org/20260623125723.2503832-2-dev.jain@arm.com Signed-off-by: Dev Jain Acked-by: David Hildenbrand (Arm) Reviewed-by: Lance Yang Acked-by: Pedro Falcato Reviewed-by: Lorenzo Stoakes Reviewed-by: Nhat Pham Reviewed-by: Kairui Song Reviewed-by: Barry Song Cc: Anshuman Khandual Cc: Baoquan He Cc: Chris Li Cc: Jann Horn Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Ryan Roberts Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/swap.h | 4 ++-- mm/swapfile.c | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/mm/swap.h b/mm/swap.h index 77d2d14eda42..44ab8e1e595b 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -230,8 +230,8 @@ extern int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp); * folio_put_swap(): does the opposite thing of folio_dup_swap(). */ int folio_alloc_swap(struct folio *folio); -int folio_dup_swap(struct folio *folio, struct page *subpage); -void folio_put_swap(struct folio *folio, struct page *subpage); +int folio_dup_swap(struct folio *folio, struct page *page); +void folio_put_swap(struct folio *folio, struct page *page); /* For internal use */ extern void __swap_cluster_free_entries(struct swap_info_struct *si, diff --git a/mm/swapfile.c b/mm/swapfile.c index 78b49b0658ad..a602e5820513 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1781,7 +1781,7 @@ int folio_alloc_swap(struct folio *folio) /** * folio_dup_swap() - Increase swap count of swap entries of a folio. * @folio: folio with swap entries bounded. - * @subpage: if not NULL, only increase the swap count of this subpage. + * @page: if not NULL, only increase the swap count of this page. * * Typically called when the folio is unmapped and have its swap entry to * take its place: Swap entries allocated to a folio has count == 0 and pinned @@ -1795,7 +1795,7 @@ int folio_alloc_swap(struct folio *folio) * swap_put_entries_direct on its swap entry before this helper returns, or * the swap count may underflow. */ -int folio_dup_swap(struct folio *folio, struct page *subpage) +int folio_dup_swap(struct folio *folio, struct page *page) { swp_entry_t entry = folio->swap; unsigned long nr_pages = folio_nr_pages(folio); @@ -1803,8 +1803,8 @@ int folio_dup_swap(struct folio *folio, struct page *subpage) VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); VM_WARN_ON_FOLIO(!folio_test_swapcache(folio), folio); - if (subpage) { - entry.val += folio_page_idx(folio, subpage); + if (page) { + entry.val += folio_page_idx(folio, page); nr_pages = 1; } @@ -1815,13 +1815,13 @@ int folio_dup_swap(struct folio *folio, struct page *subpage) /** * folio_put_swap() - Decrease swap count of swap entries of a folio. * @folio: folio with swap entries bounded, must be in swap cache and locked. - * @subpage: if not NULL, only decrease the swap count of this subpage. + * @page: if not NULL, only decrease the swap count of this page. * * This won't free the swap slots even if swap count drops to zero, they are * still pinned by the swap cache. User may call folio_free_swap to free them. * Context: Caller must ensure the folio is locked and in the swap cache. */ -void folio_put_swap(struct folio *folio, struct page *subpage) +void folio_put_swap(struct folio *folio, struct page *page) { swp_entry_t entry = folio->swap; unsigned long nr_pages = folio_nr_pages(folio); @@ -1830,8 +1830,8 @@ void folio_put_swap(struct folio *folio, struct page *subpage) VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); VM_WARN_ON_FOLIO(!folio_test_swapcache(folio), folio); - if (subpage) { - entry.val += folio_page_idx(folio, subpage); + if (page) { + entry.val += folio_page_idx(folio, page); nr_pages = 1; } From 8a28b50d6fbf8252eb8d17b22524823c8deff713 Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Tue, 23 Jun 2026 12:57:22 +0000 Subject: [PATCH 004/501] mm/mprotect: drop 'sub' from batching context Shorten the name of page_anon_exclusive_sub_batch by dropping the "sub-batch" context - the function itself doesn't need this context. Similarly, drop "sub" from sub_batch_idx, it is unnecessary and the usage is clear enough. Link: https://lore.kernel.org/20260623125723.2503832-3-dev.jain@arm.com Signed-off-by: Dev Jain Reviewed-by: Lance Yang Reviewed-by: Pedro Falcato Reviewed-by: Lorenzo Stoakes Acked-by: David Hildenbrand (Arm) Reviewed-by: Barry Song Cc: Anshuman Khandual Cc: Baoquan He Cc: Chris Li Cc: Jann Horn Cc: Kairui Song Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Nhat Pham Cc: Ryan Roberts Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/mprotect.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mm/mprotect.c b/mm/mprotect.c index 9cbf932b028c..c0f5ab74bee2 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -143,7 +143,7 @@ static __always_inline void prot_commit_flush_ptes(struct vm_area_struct *vma, * !PageAnonExclusive() pages, starting from start_idx. Caller must enforce * that the ptes point to consecutive pages of the same anon large folio. */ -static __always_inline int page_anon_exclusive_sub_batch(int start_idx, int max_len, +static __always_inline int page_anon_exclusive_batch(int start_idx, int max_len, struct page *first_page, bool expected_anon_exclusive) { int idx; @@ -174,16 +174,16 @@ static __always_inline void commit_anon_folio_batch(struct vm_area_struct *vma, pte_t oldpte, pte_t ptent, int nr_ptes, struct mmu_gather *tlb) { bool expected_anon_exclusive; - int sub_batch_idx = 0; + int batch_idx = 0; int len; while (nr_ptes) { - expected_anon_exclusive = PageAnonExclusive(first_page + sub_batch_idx); - len = page_anon_exclusive_sub_batch(sub_batch_idx, nr_ptes, + expected_anon_exclusive = PageAnonExclusive(first_page + batch_idx); + len = page_anon_exclusive_batch(batch_idx, nr_ptes, first_page, expected_anon_exclusive); prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, len, - sub_batch_idx, expected_anon_exclusive, tlb); - sub_batch_idx += len; + batch_idx, expected_anon_exclusive, tlb); + batch_idx += len; nr_ptes -= len; } } From 731a624641d30fb7a43bee68cf90ae08c19382ac Mon Sep 17 00:00:00 2001 From: Igor Putko Date: Tue, 23 Jun 2026 14:47:42 +0300 Subject: [PATCH 005/501] mm/kasan: remove redundant initialization for kasan_flag_write_only Patch series "mm: remove redundant static variable initializations". This series removes explicit initializations of static bool variables to false within the mm/ subsystem. In C, static variables without explicit initialization are implicitly placed in the .bss section and initialized to zero/false by default. Removing these explicit initializations follows the Linux kernel coding style and avoids cluttering the data section. This patch (of 2): The static variable 'kasan_flag_write_only' is implicitly initialized to false. Remove the explicit initialization to follow the Linux kernel coding style. Link: https://lore.kernel.org/20260623114743.4565-1-igorpetindev@gmail.com Link: https://lore.kernel.org/20260623114743.4565-2-igorpetindev@gmail.com Signed-off-by: Igor Putko Reviewed-by: SeongJae Park Reviewed-by: Lance Yang Cc: Alexander Potapenko Cc: Andrey Konovalov Cc: Andrey Ryabinin Cc: Dmitry Vyukov Cc: Miaohe Lin Cc: Naoya Horiguchi Cc: Vincenzo Frascino Signed-off-by: Andrew Morton --- mm/kasan/hw_tags.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/kasan/hw_tags.c b/mm/kasan/hw_tags.c index cbef5e450954..a848eb2f9910 100644 --- a/mm/kasan/hw_tags.c +++ b/mm/kasan/hw_tags.c @@ -61,7 +61,7 @@ DEFINE_STATIC_KEY_FALSE(kasan_flag_vmalloc); EXPORT_SYMBOL_GPL(kasan_flag_vmalloc); /* Whether to check write accesses only. */ -static bool kasan_flag_write_only = false; +static bool kasan_flag_write_only; #define PAGE_ALLOC_SAMPLE_DEFAULT 1 #define PAGE_ALLOC_SAMPLE_ORDER_DEFAULT 3 From 253ed912fe65bf63aee3c073969af1ec768a531d Mon Sep 17 00:00:00 2001 From: Igor Putko Date: Tue, 23 Jun 2026 14:47:43 +0300 Subject: [PATCH 006/501] mm/memory-failure: remove redundant initialization for hw_memory_failure The static variable 'hw_memory_failure' is implicitly initialized to false. Remove the explicit initialization to follow the Linux kernel coding style. Link: https://lore.kernel.org/20260623114743.4565-3-igorpetindev@gmail.com Signed-off-by: Igor Putko Reviewed-by: SeongJae Park Reviewed-by: Lance Yang Acked-by: Miaohe Lin Cc: Alexander Potapenko Cc: Andrey Konovalov Cc: Andrey Ryabinin Cc: Dmitry Vyukov Cc: Naoya Horiguchi Cc: Vincenzo Frascino Signed-off-by: Andrew Morton --- mm/memory-failure.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 51508a55c405..4963ea9f6ec6 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -76,7 +76,7 @@ static int sysctl_enable_soft_offline __read_mostly = 1; atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0); -static bool hw_memory_failure __read_mostly = false; +static bool hw_memory_failure __read_mostly; static DEFINE_MUTEX(mf_mutex); From 4ac732c3705c218ce5da83cd6a4597b0e5f7df51 Mon Sep 17 00:00:00 2001 From: Guopeng Zhang Date: Tue, 23 Jun 2026 16:26:14 +0800 Subject: [PATCH 007/501] mm: memcg: remove stray text from obj_stock_pcp comment A patch filename was accidentally inserted into the comment describing the nr_bytes field of struct obj_stock_pcp. Remove it. No functional change. Link: https://lore.kernel.org/20260623082614.81621-1-guopeng.zhang@linux.dev Signed-off-by: Guopeng Zhang Acked-by: Harry Yoo (Oracle) Signed-off-by: Andrew Morton --- mm/memcontrol.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 5e06109f1f66..d20ffc827306 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2039,7 +2039,7 @@ struct obj_stock_pcp { /* * On rare archs with 256KiB base page size (hexagon and powerpc 44x) * keep nr_bytes to unsigned int as uint16_t cannot represent the full -e patches/memcg-uint16_t-for-nr_bytes-in-obj_stock_pcp.patch * sub-page remainder. Such archs are not cacheline optimization target. + * sub-page remainder. Such archs are not cacheline optimization targets. */ unsigned int nr_bytes[NR_OBJ_STOCK]; #else From 094470f3f45b9efdab2c6b27f920071e29ebf5c5 Mon Sep 17 00:00:00 2001 From: JP Kobryn Date: Mon, 22 Jun 2026 11:51:27 -0700 Subject: [PATCH 008/501] mm/lruvec: trace LRU add drains and drain-all requests LRU add batches can be drained before they reach capacity. This can be a source of LRU lock contention, but it is not currently possible to attribute these drains to callers with existing tracepoints. Add mm_lru_add_drain to report the CPU and lru_add batch count when an lru_add batch is drained. This allows tracing to distinguish full drains from partial drains and attribute them to the calling stack. Add mm_lru_add_drain_all to capture callers of __lru_add_drain_all and whether they set the force flag for all CPUs. The tracepoint resembles the signature of the enclosing function, but is needed because of potential inlining. Note that DECLARE_TRACE() is used for these new trace hooks to avoid creating a new trace event ABI. Link: https://lore.kernel.org/20260622185127.24579-1-jp.kobryn@linux.dev Signed-off-by: JP Kobryn Reviewed-by: Barry Song Acked-by: Shakeel Butt Cc: Axel Rasmussen Cc: Baoquan He Cc: Chris Li Cc: Kairui Song Cc: Kemeng Shi Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Nhat Pham Cc: Steven Rostedt Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie Signed-off-by: Andrew Morton --- include/trace/events/pagemap.h | 8 ++++++++ mm/swap.c | 7 ++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/include/trace/events/pagemap.h b/include/trace/events/pagemap.h index 171524d3526d..36c3a90f0acc 100644 --- a/include/trace/events/pagemap.h +++ b/include/trace/events/pagemap.h @@ -77,6 +77,14 @@ TRACE_EVENT(mm_lru_activate, TP_printk("folio=%p pfn=0x%lx", __entry->folio, __entry->pfn) ); +DECLARE_TRACE(mm_lru_add_drain, + TP_PROTO(int cpu, unsigned int nr_folios), + TP_ARGS(cpu, nr_folios)); + +DECLARE_TRACE(mm_lru_add_drain_all, + TP_PROTO(bool force_all_cpus), + TP_ARGS(force_all_cpus)); + #endif /* _TRACE_PAGEMAP_H */ /* This part must be outside protection */ diff --git a/mm/swap.c b/mm/swap.c index 588f50d8f1a8..460e56370b3c 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -694,9 +694,12 @@ void lru_add_drain_cpu(int cpu) { struct cpu_fbatches *fbatches = &per_cpu(cpu_fbatches, cpu); struct folio_batch *fbatch = &fbatches->lru_add; + unsigned int nr_folios = folio_batch_count(fbatch); - if (folio_batch_count(fbatch)) + if (nr_folios) { folio_batch_move_lru(fbatch, lru_add); + trace_mm_lru_add_drain_tp(cpu, nr_folios); + } fbatch = &fbatches->lru_move_tail; /* Disabling interrupts below acts as a compiler barrier. */ @@ -869,6 +872,8 @@ static inline void __lru_add_drain_all(bool force_all_cpus) if (WARN_ON(!mm_percpu_wq)) return; + trace_mm_lru_add_drain_all_tp(force_all_cpus); + /* * Guarantee folio_batch counter stores visible by this CPU * are visible to other CPUs before loading the current drain From e1c345582c979ed44f881dec0b3137d862097c6b Mon Sep 17 00:00:00 2001 From: Chi Zhiling Date: Sat, 20 Jun 2026 14:24:45 +0800 Subject: [PATCH 009/501] mm/filemap: reduce unnecessary xarray lookups when read cached pages Patch series "mm/filemap: reduce unnecessary xarray lookups". This series optimizes xarray lookups in filemap by avoiding redundant iterations after obtaining the last needed folio. The boundary check is moved to before advancing the xarray iterator, eliminating unnecessary lookups and branches in the fast path. This reduces the overhead of filemap_get_read_batch() from 2.91% to 2.53% in 4k read tests. This patch (of 2): When reading small amounts of data from the page cache, only a single folio is typically returned from filemap_read_get_batch(). In this case, calling xas_advance() or xas_next() after adding the folio to the batch is unnecessary and only introduces extra branches. The same issue exists for large reads, where one additional xarray walk is always performed before termination. Quit the loop once we get the last folio in the range, so the final redundant xarray advancement can be avoided. The xas_next() does not update xa_index when xas->xa_node is set to XAS_RESTART, so the put and retry path would not update xa_index, hence the warning should therefore never trigger. During the 4k reads test, the overhead of this function dropped from 2.91% to 2.53%. Link: https://lore.kernel.org/20260620062446.351475-2-chizhiling@163.com Signed-off-by: Chi Zhiling Suggested-by: Matthew Wilcox (Oracle) Reviewed-by: Jan Kara Cc: Chi Zhiling Cc: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- mm/filemap.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mm/filemap.c b/mm/filemap.c index 58eb9d240643..dfc22df1031a 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2467,11 +2467,14 @@ static void filemap_get_read_batch(struct address_space *mapping, XA_STATE(xas, &mapping->i_pages, index); struct folio *folio; + if (index > max) + return; + rcu_read_lock(); for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) { if (xas_retry(&xas, folio)) continue; - if (xas.xa_index > max || xa_is_value(folio)) + if (xa_is_value(folio)) break; if (xa_is_sibling(folio)) break; @@ -2488,6 +2491,8 @@ static void filemap_get_read_batch(struct address_space *mapping, if (folio_test_readahead(folio)) break; xas_advance(&xas, folio_next_index(folio) - 1); + if (xas.xa_index >= max) + break; continue; put_folio: folio_put(folio); From 32cd1afeca96076bf2408c61f552d98e3de94884 Mon Sep 17 00:00:00 2001 From: Chi Zhiling Date: Sat, 20 Jun 2026 14:24:46 +0800 Subject: [PATCH 010/501] mm/filemap: reduce unnecessary xarray lookups in filemap_get_folios_contig() Apply the same optimization used in filemap_get_read_batch() by moving the boundary check from the loop condition to before xas_next(), avoiding an unnecessary xarray lookup and reducing branches in the fast path. Link: https://lore.kernel.org/20260620062446.351475-3-chizhiling@163.com Signed-off-by: Chi Zhiling Reviewed-by: Jan Kara Cc: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- mm/filemap.c | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index dfc22df1031a..6e40f36c2bff 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2270,10 +2270,11 @@ unsigned filemap_get_folios_contig(struct address_space *mapping, unsigned long nr; struct folio *folio; - rcu_read_lock(); + if (*start > end) + return 0; - for (folio = xas_load(&xas); folio && xas.xa_index <= end; - folio = xas_next(&xas)) { + rcu_read_lock(); + for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) { if (xas_retry(&xas, folio)) continue; /* @@ -2281,11 +2282,11 @@ unsigned filemap_get_folios_contig(struct address_space *mapping, * No current caller is looking for DAX entries. */ if (xa_is_value(folio)) - goto update_start; + break; /* If we landed in the middle of a THP, continue at its end. */ if (xa_is_sibling(folio)) - goto update_start; + break; if (!folio_try_get(folio)) goto retry; @@ -2293,29 +2294,27 @@ unsigned filemap_get_folios_contig(struct address_space *mapping, if (unlikely(folio != xas_reload(&xas))) goto put_folio; - if (!folio_batch_add(fbatch, folio)) { - *start = folio_next_index(folio); - goto out; - } + if (!folio_batch_add(fbatch, folio)) + break; + xas_advance(&xas, folio_next_index(folio) - 1); + if (xas.xa_index >= end) + break; continue; + put_folio: folio_put(folio); - retry: xas_reset(&xas); } + rcu_read_unlock(); -update_start: nr = folio_batch_count(fbatch); - if (nr) { folio = fbatch->folios[nr - 1]; *start = folio_next_index(folio); } -out: - rcu_read_unlock(); - return folio_batch_count(fbatch); + return nr; } EXPORT_SYMBOL(filemap_get_folios_contig); From 5339ec29f9be211575674f55859e7fd1eca33d76 Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Fri, 19 Jun 2026 15:18:30 +0200 Subject: [PATCH 011/501] mm: replace __ASSEMBLY__ with __ASSEMBLER__ in memory management header files While the GCC and Clang compilers already define __ASSEMBLER__ automatically when compiling assembly code, __ASSEMBLY__ is a macro that only gets defined by the Makefiles in the kernel. This can be very confusing when switching between userspace and kernelspace coding, or when dealing with uapi headers that rather should use __ASSEMBLER__ instead. So let's standardize now on the __ASSEMBLER__ macro that is provided by the compilers. This is a completely mechanical patch (done with a simple "sed -i" statement). Link: https://lore.kernel.org/20260619131830.229804-1-thuth@redhat.com Signed-off-by: Thomas Huth Cc: Arnd Bergmann Cc: Axel Rasmussen Cc: Barry Song Cc: David Hildenbrand Cc: Kairui Song Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shakeel Butt Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie Signed-off-by: Andrew Morton --- include/asm-generic/fixmap.h | 4 ++-- include/asm-generic/getorder.h | 4 ++-- include/asm-generic/memory_model.h | 4 ++-- include/asm-generic/mmu.h | 2 +- include/asm-generic/pgtable-nop4d.h | 4 ++-- include/asm-generic/pgtable-nopmd.h | 4 ++-- include/asm-generic/pgtable-nopud.h | 4 ++-- include/linux/mmzone.h | 4 ++-- include/linux/pfn.h | 2 +- include/linux/pgtable.h | 4 ++-- 10 files changed, 18 insertions(+), 18 deletions(-) diff --git a/include/asm-generic/fixmap.h b/include/asm-generic/fixmap.h index 29cab7947980..3ff832ebcea5 100644 --- a/include/asm-generic/fixmap.h +++ b/include/asm-generic/fixmap.h @@ -21,7 +21,7 @@ #define __fix_to_virt(x) (FIXADDR_TOP - ((x) << PAGE_SHIFT)) #define __virt_to_fix(x) ((FIXADDR_TOP - ((x)&PAGE_MASK)) >> PAGE_SHIFT) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * 'index to address' translation. If anyone tries to use the idx * directly without translation, we catch the bug with a NULL-deference @@ -97,5 +97,5 @@ static inline unsigned long virt_to_fix(const unsigned long vaddr) #define set_fixmap_io(idx, phys) \ __set_fixmap(idx, phys, FIXMAP_PAGE_IO) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __ASM_GENERIC_FIXMAP_H */ diff --git a/include/asm-generic/getorder.h b/include/asm-generic/getorder.h index f2979e3a96b6..875ccae19683 100644 --- a/include/asm-generic/getorder.h +++ b/include/asm-generic/getorder.h @@ -2,7 +2,7 @@ #ifndef __ASM_GENERIC_GETORDER_H #define __ASM_GENERIC_GETORDER_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include #include @@ -47,6 +47,6 @@ static __always_inline __attribute_const__ int get_order(unsigned long size) #endif } -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __ASM_GENERIC_GETORDER_H */ diff --git a/include/asm-generic/memory_model.h b/include/asm-generic/memory_model.h index efa6610acbc7..fd74de50b054 100644 --- a/include/asm-generic/memory_model.h +++ b/include/asm-generic/memory_model.h @@ -4,7 +4,7 @@ #include -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * supports 3 memory models. @@ -86,6 +86,6 @@ static inline int pfn_valid(unsigned long pfn) #endif /* CONFIG_DEBUG_VIRTUAL */ #define phys_to_page(phys) pfn_to_page(PHYS_PFN(phys)) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif diff --git a/include/asm-generic/mmu.h b/include/asm-generic/mmu.h index 061838037542..5f78971e3ac2 100644 --- a/include/asm-generic/mmu.h +++ b/include/asm-generic/mmu.h @@ -6,7 +6,7 @@ * This is the mmu.h header for nommu implementations. * Architectures with an MMU need something more complex. */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ typedef struct { unsigned long end_brk; diff --git a/include/asm-generic/pgtable-nop4d.h b/include/asm-generic/pgtable-nop4d.h index 03b7dae47dd4..89c21f84cffb 100644 --- a/include/asm-generic/pgtable-nop4d.h +++ b/include/asm-generic/pgtable-nop4d.h @@ -2,7 +2,7 @@ #ifndef _PGTABLE_NOP4D_H #define _PGTABLE_NOP4D_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define __PAGETABLE_P4D_FOLDED 1 @@ -54,5 +54,5 @@ static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) #undef p4d_addr_end #define p4d_addr_end(addr, end) (end) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _PGTABLE_NOP4D_H */ diff --git a/include/asm-generic/pgtable-nopmd.h b/include/asm-generic/pgtable-nopmd.h index 8ffd64e7a24c..36b6490ed180 100644 --- a/include/asm-generic/pgtable-nopmd.h +++ b/include/asm-generic/pgtable-nopmd.h @@ -2,7 +2,7 @@ #ifndef _PGTABLE_NOPMD_H #define _PGTABLE_NOPMD_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include @@ -68,6 +68,6 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) #undef pmd_addr_end #define pmd_addr_end(addr, end) (end) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _PGTABLE_NOPMD_H */ diff --git a/include/asm-generic/pgtable-nopud.h b/include/asm-generic/pgtable-nopud.h index eb70c6d7ceff..356cbfbaab24 100644 --- a/include/asm-generic/pgtable-nopud.h +++ b/include/asm-generic/pgtable-nopud.h @@ -2,7 +2,7 @@ #ifndef _PGTABLE_NOPUD_H #define _PGTABLE_NOPUD_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include @@ -62,5 +62,5 @@ static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) #undef pud_addr_end #define pud_addr_end(addr, end) (end) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _PGTABLE_NOPUD_H */ diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index ca2712187147..e26b3d38fa25 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -2,7 +2,7 @@ #ifndef _LINUX_MMZONE_H #define _LINUX_MMZONE_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #ifndef __GENERATING_BOUNDS_H #include @@ -2389,5 +2389,5 @@ static inline unsigned long next_present_section_nr(unsigned long section_nr) #endif #endif /* !__GENERATING_BOUNDS.H */ -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* _LINUX_MMZONE_H */ diff --git a/include/linux/pfn.h b/include/linux/pfn.h index b90ca0b6c331..cfedf0f61bb3 100644 --- a/include/linux/pfn.h +++ b/include/linux/pfn.h @@ -2,7 +2,7 @@ #ifndef _LINUX_PFN_H_ #define _LINUX_PFN_H_ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include #endif diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 2981e386da7b..dc804296d78f 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -8,7 +8,7 @@ #define PMD_ORDER (PMD_SHIFT - PAGE_SHIFT) #define PUD_ORDER (PUD_SHIFT - PAGE_SHIFT) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #ifdef CONFIG_MMU #include @@ -2320,7 +2320,7 @@ static inline const char *pgtable_level_to_str(enum pgtable_level level) } } -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT) #ifdef CONFIG_PHYS_ADDR_T_64BIT From dec068de6dec6c8f94887303867c22c40549f1f2 Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Thu, 18 Jun 2026 21:04:11 +0800 Subject: [PATCH 012/501] mm/vmalloc: honor GFP constraints in pcpu_get_vm_areas() Patch series "mm/percpu: Fix possible NOFS/NOIO reclaim recursion", v4. Commit 9a5b183941b5 ("mm, percpu: do not consider sleepable allocations atomic") allowed GFP_NOFS and GFP_NOIO percpu allocations to use pcpu_alloc_mutex and the chunk creation slow path. This restored the allocation capability that was lost when those constrained allocations were treated as atomic, but it also makes the percpu slow path visible to callers from constrained reclaim contexts. There are two related problems. First, the create and populate slow paths do not fully preserve the caller's allocation constraints. pcpu_alloc_noprof() derives pcpu_gfp from the caller supplied GFP mask and passes it down to the percpu backing page allocator. However, chunk creation calls pcpu_get_vm_areas(), and chunk population can allocate temporary metadata or vmalloc page tables while mapping backing pages. Those internal allocations can still use GFP_KERNEL, so a caller using GFP_NOFS or GFP_NOIO can enter unconstrained FS or IO reclaim while holding pcpu_alloc_mutex. One possible case is blk-cgroup after commit 5d726c4dbeed ("blk-cgroup: fix possible deadlock while configuring policy"). blkg_conf_prep() now serializes against blkcg_deactivate_policy() with q->blkcg_mutex, and blkg_alloc() uses GFP_NOIO because queue freeze and IO reclaim dependencies can otherwise deadlock. If the percpu slow path loses that GFP_NOIO context, direct reclaim or writeback can issue IO to a frozen queue while q->blkcg_mutex is held. Second, allowing sleepable GFP_NOFS/GFP_NOIO allocations to take pcpu_alloc_mutex means that unconstrained backing allocations made under the mutex can create an FS/IO reclaim dependency against a constrained caller which already holds an FS or IO lock and then waits for pcpu_alloc_mutex. This series fixes those issues in three steps: - pass the caller supplied GFP mask into pcpu_get_vm_areas() and use it for vmalloc metadata and KASAN shadow allocations; - pass the GFP mask through the chunk population path, including the temporary pages array and vmalloc page table allocation scope; - restrict percpu backing allocations performed while holding pcpu_alloc_mutex to GFP_NOIO, so they cannot recurse into IO or FS reclaim. This keeps sleepable GFP_NOFS/GFP_NOIO percpu allocations working, while avoiding the reclaim recursion risks introduced by making those allocations eligible for the mutex-protected slow path. This patch (of 4): pcpu_alloc_noprof() derives pcpu_gfp from the caller supplied GFP mask and passes it down to the backing percpu allocator. However, when the percpu vmalloc allocator has to create a new chunk, pcpu_create_chunk() calls pcpu_get_vm_areas() to allocate the corresponding vmalloc areas. pcpu_get_vm_areas() currently performs its internal allocations with GFP_KERNEL, including vmap area metadata, vm_struct metadata and KASAN vmalloc shadow population. This means that a caller which deliberately uses GFP_NOFS or GFP_NOIO can still enter FS or IO reclaim while creating the vmalloc areas for a new percpu chunk. One possible case is blk-cgroup after commit 5d726c4dbeed ("blk-cgroup: fix possible deadlock while configuring policy"). blkg_conf_prep() now serializes against blkcg_deactivate_policy() with q->blkcg_mutex, and blkg_alloc() was changed to GFP_NOIO for that reason: CPU0: blkg_conf_prep() mutex_lock(q->blkcg_mutex) blkg_alloc(..., GFP_NOIO) alloc_percpu_gfp(..., GFP_NOIO) pcpu_alloc_noprof(..., GFP_NOIO) pcpu_create_chunk(GFP_NOIO) pcpu_get_vm_areas() -> if percpu chunks are exhausted, chunk create may do internal GFP_KERNEL allocations -> direct reclaim / writeback can issue IO to this queue -> IO waits because the queue is frozen CPU1: blkcg_deactivate_policy() blk_mq_freeze_queue(q) mutex_lock(q->blkcg_mutex) -> waits for CPU0 ... unfreeze only happens after q->blkcg_mutex is acquired/released So the concern is that the caller deliberately uses GFP_NOIO because it may hold a lock which can be acquired after queue freeze, but the percpu slow path can temporarily lose that allocation context. Pass the caller supplied GFP mask from pcpu_create_chunk() to pcpu_get_vm_areas(), and use it for the internal vmalloc metadata and KASAN shadow allocations. Link: https://lore.kernel.org/20260618130414.96383-1-kaitao.cheng@linux.dev Link: https://lore.kernel.org/20260618130414.96383-2-kaitao.cheng@linux.dev Fixes: 9a5b183941b5 ("mm, percpu: do not consider sleepable allocations atomic") Signed-off-by: Kaitao Cheng Reviewed-by: Uladzislau Rezki (Sony) Reviewed-by: Shivam Kalra Acked-by: Dennis Zhou Acked-by: Michal Hocko Cc: Christoph Lameter Cc: Pedro Falcato Cc: Tejun Heo Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/vmalloc.h | 4 ++-- mm/percpu-vm.c | 2 +- mm/vmalloc.c | 23 ++++++++++++----------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index d87dc7f77f4e..e4d8d0a9f30f 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -310,14 +310,14 @@ static inline void set_vm_flush_reset_perms(void *addr) {} #if defined(CONFIG_MMU) && defined(CONFIG_SMP) struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, const size_t *sizes, int nr_vms, - size_t align); + size_t align, gfp_t gfp); void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms); # else static inline struct vm_struct ** pcpu_get_vm_areas(const unsigned long *offsets, const size_t *sizes, int nr_vms, - size_t align) + size_t align, gfp_t gfp) { return NULL; } diff --git a/mm/percpu-vm.c b/mm/percpu-vm.c index 4f5937090590..69b00741dc68 100644 --- a/mm/percpu-vm.c +++ b/mm/percpu-vm.c @@ -340,7 +340,7 @@ static struct pcpu_chunk *pcpu_create_chunk(gfp_t gfp) return NULL; vms = pcpu_get_vm_areas(pcpu_group_offsets, pcpu_group_sizes, - pcpu_nr_groups, pcpu_atom_size); + pcpu_nr_groups, pcpu_atom_size, gfp); if (!vms) { pcpu_free_chunk(chunk); return NULL; diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 1afca3568b9b..08f468135e4d 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -4946,16 +4946,17 @@ pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align) * @sizes: array containing size of each area * @nr_vms: the number of areas to allocate * @align: alignment, all entries in @offsets and @sizes must be aligned to this + * @gfp: allocation flags passed to the underlying memory allocator * * Returns: kmalloc'd vm_struct pointer array pointing to allocated * vm_structs on success, %NULL on failure * * Percpu allocator wants to use congruent vm areas so that it can * maintain the offsets among percpu areas. This function allocates - * congruent vmalloc areas for it with GFP_KERNEL. These areas tend to - * be scattered pretty far, distance between two areas easily going up - * to gigabytes. To avoid interacting with regular vmallocs, these - * areas are allocated from top. + * congruent vmalloc areas for it. These areas tend to be scattered + * pretty far, distance between two areas easily going up to gigabytes. + * To avoid interacting with regular vmallocs, these areas are allocated + * from top. * * Despite its complicated look, this allocator is rather simple. It * does everything top-down and scans free blocks from the end looking @@ -4966,7 +4967,7 @@ pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align) */ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, const size_t *sizes, int nr_vms, - size_t align) + size_t align, gfp_t gfp) { const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align); const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); @@ -5004,14 +5005,14 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, return NULL; } - vms = kzalloc_objs(vms[0], nr_vms); - vas = kzalloc_objs(vas[0], nr_vms); + vms = kzalloc_objs(vms[0], nr_vms, gfp); + vas = kzalloc_objs(vas[0], nr_vms, gfp); if (!vas || !vms) goto err_free2; for (area = 0; area < nr_vms; area++) { - vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL); - vms[area] = kzalloc_obj(struct vm_struct); + vas[area] = kmem_cache_zalloc(vmap_area_cachep, gfp); + vms[area] = kzalloc_obj(struct vm_struct, gfp); if (!vas[area] || !vms[area]) goto err_free; } @@ -5101,7 +5102,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, /* populate the kasan shadow space */ for (area = 0; area < nr_vms; area++) { - if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area], GFP_KERNEL)) + if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area], gfp)) goto err_free_shadow; } @@ -5158,7 +5159,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, continue; vas[area] = kmem_cache_zalloc( - vmap_area_cachep, GFP_KERNEL); + vmap_area_cachep, gfp); if (!vas[area]) goto err_free; } From 881adc51b29ba004f55d2398037ce2619574017d Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Thu, 18 Jun 2026 21:04:12 +0800 Subject: [PATCH 013/501] mm/percpu: honor GFP constraints when populating chunks pcpu_alloc_noprof() derives pcpu_gfp from the caller supplied GFP mask and passes it down to pcpu_populate_chunk(). pcpu_alloc_pages() already uses that mask for backing page allocation. However, the populate slow path still has internal allocations and page table allocations which can lose the caller's allocation context. The temporary pages array is allocated by pcpu_get_pages() with GFP_KERNEL, and pcpu_map_pages() maps the backing pages through vmap_pages_range_noflush() using GFP_KERNEL. The latter can allocate vmalloc page tables implicitly, so a caller which deliberately uses GFP_NOFS or GFP_NOIO can still enter FS or IO reclaim while populating a percpu chunk. This has the same concern as chunk creation: callers such as blk-cgroup may use GFP_NOIO because they hold locks which can be involved in queue freeze or IO reclaim dependencies. If an allocation reaches the percpu slow path and needs to populate previously unbacked pages, the internal GFP_KERNEL allocations can defeat that context. One possible case is blk-cgroup after commit 5d726c4dbeed ("blk-cgroup: fix possible deadlock while configuring policy"). blkg_conf_prep() now serializes against blkcg_deactivate_policy() with q->blkcg_mutex, and blkg_alloc() was changed to GFP_NOIO for that reason: CPU0: blkg_conf_prep() mutex_lock(q->blkcg_mutex) blkg_alloc(..., GFP_NOIO) alloc_percpu_gfp(..., GFP_NOIO) pcpu_alloc_noprof(..., GFP_NOIO) pcpu_populate_chunk(GFP_NOIO) pcpu_get_pages() pcpu_map_pages() -> if the selected percpu chunk has unpopulated pages, chunk population may do internal GFP_KERNEL allocations -> direct reclaim / writeback can issue IO to this queue -> IO waits because the queue is frozen CPU1: blkcg_deactivate_policy() blk_mq_freeze_queue(q) mutex_lock(q->blkcg_mutex) -> waits for CPU0 ... unfreeze only happens after q->blkcg_mutex is acquired/released So the concern is that the caller deliberately uses GFP_NOIO because it may hold a lock which can be acquired after queue freeze, but the percpu slow path can temporarily lose that allocation context. Pass pcpu_gfp through pcpu_get_pages(), pcpu_map_pages() and __pcpu_map_pages(). Apply the corresponding memalloc scope around vmap_pages_range_noflush(), because vmalloc page table allocation does not pass the GFP mask down explicitly. Keep the first chunk setup path using GFP_KERNEL, matching the previous early-init behavior. Link: https://lore.kernel.org/20260618130414.96383-3-kaitao.cheng@linux.dev Fixes: 9a5b183941b5 ("mm, percpu: do not consider sleepable allocations atomic") Signed-off-by: Kaitao Cheng Acked-by: Dennis Zhou Acked-by: Michal Hocko Cc: Christoph Lameter Cc: Pedro Falcato Cc: Shivam Kalra Cc: Tejun Heo Cc: Uladzislau Rezki (Sony) Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/percpu-vm.c | 38 ++++++++++++++++++++++++++------------ mm/percpu.c | 2 +- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/mm/percpu-vm.c b/mm/percpu-vm.c index 69b00741dc68..ccd03cc152d4 100644 --- a/mm/percpu-vm.c +++ b/mm/percpu-vm.c @@ -21,6 +21,7 @@ static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk, /** * pcpu_get_pages - get temp pages array + * @gfp: allocation flags passed to the underlying allocator * * Returns pointer to array of pointers to struct page which can be indexed * with pcpu_page_idx(). Note that there is only one array and accesses @@ -29,7 +30,7 @@ static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk, * RETURNS: * Pointer to temp pages array on success. */ -static struct page **pcpu_get_pages(void) +static struct page **pcpu_get_pages(gfp_t gfp) { static struct page **pages; size_t pages_size = pcpu_nr_units * pcpu_unit_pages * sizeof(pages[0]); @@ -37,7 +38,7 @@ static struct page **pcpu_get_pages(void) lockdep_assert_held(&pcpu_alloc_mutex); if (!pages) - pages = pcpu_mem_zalloc(pages_size, GFP_KERNEL); + pages = pcpu_mem_zalloc(pages_size, gfp); return pages; } @@ -191,10 +192,22 @@ static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk, } static int __pcpu_map_pages(unsigned long addr, struct page **pages, - int nr_pages) + int nr_pages, gfp_t gfp) { - return vmap_pages_range_noflush(addr, addr + (nr_pages << PAGE_SHIFT), - PAGE_KERNEL, pages, PAGE_SHIFT, GFP_KERNEL); + unsigned int flags; + int ret; + + /* + * The vmalloc page table allocation path does not pass @gfp down + * explicitly. Apply the corresponding memalloc scope so implicit + * page table allocations preserve NOFS/NOIO constraints. + */ + flags = memalloc_apply_gfp_scope(gfp); + ret = vmap_pages_range_noflush(addr, addr + (nr_pages << PAGE_SHIFT), + PAGE_KERNEL, pages, PAGE_SHIFT, gfp); + memalloc_restore_scope(flags); + + return ret; } /** @@ -203,6 +216,7 @@ static int __pcpu_map_pages(unsigned long addr, struct page **pages, * @pages: pages array containing pages to be mapped * @page_start: page index of the first page to map * @page_end: page index of the last page to map + 1 + * @gfp: allocation flags passed to the underlying allocator * * For each cpu, map pages [@page_start,@page_end) into @chunk. The * caller is responsible for calling pcpu_post_map_flush() after all @@ -211,8 +225,8 @@ static int __pcpu_map_pages(unsigned long addr, struct page **pages, * This function is responsible for setting up whatever is necessary for * reverse lookup (addr -> chunk). */ -static int pcpu_map_pages(struct pcpu_chunk *chunk, - struct page **pages, int page_start, int page_end) +static int pcpu_map_pages(struct pcpu_chunk *chunk, struct page **pages, + int page_start, int page_end, gfp_t gfp) { unsigned int cpu, tcpu; int i, err; @@ -220,7 +234,7 @@ static int pcpu_map_pages(struct pcpu_chunk *chunk, for_each_possible_cpu(cpu) { err = __pcpu_map_pages(pcpu_chunk_addr(chunk, cpu, page_start), &pages[pcpu_page_idx(cpu, page_start)], - page_end - page_start); + page_end - page_start, gfp); if (err < 0) goto err; @@ -271,21 +285,21 @@ static void pcpu_post_map_flush(struct pcpu_chunk *chunk, * @chunk. * * CONTEXT: - * pcpu_alloc_mutex, does GFP_KERNEL allocation. + * pcpu_alloc_mutex, does @gfp allocation. */ static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int page_start, int page_end, gfp_t gfp) { struct page **pages; - pages = pcpu_get_pages(); + pages = pcpu_get_pages(gfp); if (!pages) return -ENOMEM; if (pcpu_alloc_pages(chunk, pages, page_start, page_end, gfp)) return -ENOMEM; - if (pcpu_map_pages(chunk, pages, page_start, page_end)) { + if (pcpu_map_pages(chunk, pages, page_start, page_end, gfp)) { pcpu_free_pages(chunk, pages, page_start, page_end); return -ENOMEM; } @@ -319,7 +333,7 @@ static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, * successful population attempt so the temp pages array must * be available now. */ - pages = pcpu_get_pages(); + pages = pcpu_get_pages(GFP_KERNEL); BUG_ON(!pages); /* unmap and free */ diff --git a/mm/percpu.c b/mm/percpu.c index b0676b8054ed..4d89965cba16 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -3256,7 +3256,7 @@ int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t /* pte already populated, the following shouldn't fail */ rc = __pcpu_map_pages(unit_addr, &pages[unit * unit_pages], - unit_pages); + unit_pages, GFP_KERNEL); if (rc < 0) panic("failed to map percpu area, err=%d\n", rc); From 17fcdd8699b2439e83e3c3aa6cf0f55bd35b4053 Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Thu, 18 Jun 2026 21:04:13 +0800 Subject: [PATCH 014/501] mm/percpu: make cached pages lookup explicit pcpu_depopulate_chunk() only needs the temporary pages array that was already allocated by an earlier successful population attempt. Passing GFP_KERNEL to pcpu_get_pages() in this path is misleading because the depopulation path is not expected to allocate the array. Teach pcpu_get_pages() to treat a zero gfp mask as a cached-only lookup and add pcpu_get_pages_cached() for that use case. This keeps allocation on the populate path tied to the caller supplied GFP mask while making the depopulate path's dependency on the cached array explicit. Link: https://lore.kernel.org/20260618130414.96383-4-kaitao.cheng@linux.dev Signed-off-by: Kaitao Cheng Suggested-by: Dennis Zhou Acked-by: Michal Hocko Cc: Christoph Lameter Cc: Pedro Falcato Cc: Shivam Kalra Cc: Tejun Heo Cc: Uladzislau Rezki (Sony) Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/percpu-vm.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mm/percpu-vm.c b/mm/percpu-vm.c index ccd03cc152d4..7ed216192fc0 100644 --- a/mm/percpu-vm.c +++ b/mm/percpu-vm.c @@ -21,7 +21,8 @@ static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk, /** * pcpu_get_pages - get temp pages array - * @gfp: allocation flags passed to the underlying allocator + * @gfp: allocation flags passed to the underlying allocator, 0 to only + * return the cached array * * Returns pointer to array of pointers to struct page which can be indexed * with pcpu_page_idx(). Note that there is only one array and accesses @@ -37,11 +38,16 @@ static struct page **pcpu_get_pages(gfp_t gfp) lockdep_assert_held(&pcpu_alloc_mutex); - if (!pages) + if (!pages && gfp) pages = pcpu_mem_zalloc(pages_size, gfp); return pages; } +static struct page **pcpu_get_pages_cached(void) +{ + return pcpu_get_pages(0); +} + /** * pcpu_free_pages - free pages which were allocated for @chunk * @chunk: chunk pages were allocated for @@ -333,7 +339,7 @@ static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, * successful population attempt so the temp pages array must * be available now. */ - pages = pcpu_get_pages(GFP_KERNEL); + pages = pcpu_get_pages_cached(); BUG_ON(!pages); /* unmap and free */ From 8725ae13f0ca89eff61cb6c256627b183d57ec7f Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Thu, 18 Jun 2026 21:04:14 +0800 Subject: [PATCH 015/501] mm/percpu: avoid IO/FS reclaim in backing allocations Commit 9a5b183941b5 ("mm, percpu: do not consider sleepable allocations atomic") allows sleepable GFP_NOIO and GFP_NOFS percpu allocations to take pcpu_alloc_mutex. This avoids premature allocation failures, but it also makes the mutex visible to callers from constrained IO/FS contexts. Thread A calls pcpu_alloc_noprof() with GFP_KERNEL and takes pcpu_alloc_mutex. Since the internal allocation is not constrained by NOFS, it may enter FS reclaim while still holding pcpu_alloc_mutex, creating a dependency like: pcpu_alloc_mutex -> fs_reclaim -> FS lock At the same time, Thread B may already hold an FS lock and then call pcpu_alloc_noprof() with GFP_NOFS. It will try to acquire pcpu_alloc_mutex and block, creating the reverse dependency: FS lock -> pcpu_alloc_mutex This can still form a potential deadlock cycle. Avoid the dependency by restricting percpu backing allocations to GFP_NOIO. The public allocation still uses the caller's GFP context to decide whether it may block, but the internal memory allocations performed while pcpu_alloc_mutex is held cannot recurse into IO or FS reclaim. Link: https://lore.kernel.org/20260618130414.96383-5-kaitao.cheng@linux.dev Fixes: 9a5b183941b5 ("mm, percpu: do not consider sleepable allocations atomic") Signed-off-by: Kaitao Cheng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Michal Hocko Cc: Pedro Falcato Cc: Shivam Kalra Cc: Tejun Heo Cc: Uladzislau Rezki (Sony) Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/percpu.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/mm/percpu.c b/mm/percpu.c index 4d89965cba16..a802d72c116f 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -1726,9 +1726,8 @@ static void pcpu_alloc_tag_free_hook(struct pcpu_chunk *chunk, int off, size_t s * @gfp: allocation flags * * Allocate percpu area of @size bytes aligned at @align. If @gfp doesn't - * contain %GFP_KERNEL, the allocation is atomic. If @gfp has __GFP_NOWARN - * then no warning will be triggered on invalid or failed allocation - * requests. + * allow blocking, the allocation is atomic. If @gfp has __GFP_NOWARN then no + * warning will be triggered on invalid or failed allocation requests. * * RETURNS: * Percpu pointer to the allocated area on success, NULL on failure. @@ -1749,8 +1748,17 @@ void __percpu *pcpu_alloc_noprof(size_t size, size_t align, bool reserved, size_t bits, bit_align; gfp = current_gfp_context(gfp); - /* whitelisted flags that can be passed to the backing allocators */ - pcpu_gfp = gfp & (GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN); + /* + * Allowlisted flags that can be passed to the backing allocators. + * Backing allocations under pcpu_alloc_mutex must not recurse into + * IO/FS reclaim. Otherwise a GFP_KERNEL caller holding the mutex can + * block on reclaim while a GFP_NOIO/NOFS caller holding an IO/FS lock + * waits for the same mutex. + * + * Do not pass __GFP_NOFAIL. A small percpu allocation may need many + * backing pages, making nofail reclaim too costly under NOIO/NOFS. + */ + pcpu_gfp = gfp & (GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN); is_atomic = !gfpflags_allow_blocking(gfp); do_warn = !(gfp & __GFP_NOWARN); From a35d8872dae7b2d73421bd7932d0d8d7781c6871 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Thu, 18 Jun 2026 19:35:23 +0800 Subject: [PATCH 016/501] mm: remove PageTransCompound() Remove the last user of PageTransCompound() in ksm and get rid of PageTransCompound(). Link: https://lore.kernel.org/20260618113523.3913307-1-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Acked-by: David Hildenbrand (Arm) Reviewed-by: Xu Xin Tested-by: Xu Xin Acked-by: Zi Yan Reviewed-by: SeongJae Park Cc: Chengming Zhou Signed-off-by: Andrew Morton --- include/linux/page-flags.h | 14 -------------- mm/ksm.c | 17 +++++++++-------- 2 files changed, 9 insertions(+), 22 deletions(-) diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 7223f6f4e2b4..7a863572adce 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -879,20 +879,6 @@ FOLIO_FLAG_FALSE(partially_mapped) #define PG_head_mask ((1UL << PG_head)) -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -/* - * PageTransCompound returns true for both transparent huge pages - * and hugetlbfs pages, so it should only be called when it's known - * that hugetlbfs pages aren't involved. - */ -static inline int PageTransCompound(const struct page *page) -{ - return PageCompound(page); -} -#else -TESTPAGEFLAG_FALSE(TransCompound, transcompound) -#endif - #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_TRANSPARENT_HUGEPAGE) /* * PageHasHWPoisoned indicates that at least one subpage is hwpoisoned in the diff --git a/mm/ksm.c b/mm/ksm.c index 7d5b76478f0b..41ab25aa2a82 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -2327,23 +2327,24 @@ static void cmp_and_merge_page(struct page *page, struct ksm_rmap_item *rmap_ite tree_rmap_item = unstable_tree_search_insert(rmap_item, page, &tree_page); if (tree_rmap_item) { + struct folio *tree_folio; bool split; kfolio = try_to_merge_two_pages(rmap_item, page, tree_rmap_item, tree_page); + tree_folio = page_folio(tree_page); /* - * If both pages we tried to merge belong to the same compound - * page, then we actually ended up increasing the reference - * count of the same compound page twice, and split_huge_page - * failed. + * If both pages we tried to merge belong to the same (large) + * folio, then we actually ended up increasing the reference + * count of the same folio twice, and split_huge_page failed. + * * Here we set a flag if that happened, and we use it later to - * try split_huge_page again. Since we call put_page right + * try split_huge_page again. Since we call folio_put() right * afterwards, the reference count will be correct and * split_huge_page should succeed. */ - split = PageTransCompound(page) - && compound_head(page) == compound_head(tree_page); - put_page(tree_page); + split = folio == tree_folio; + folio_put(tree_folio); if (kfolio) { /* * The pages were successfully merged: insert new From cdea9364e477b40fa73dc8e39468ea1701221fec Mon Sep 17 00:00:00 2001 From: Ben Dooks Date: Thu, 18 Jun 2026 11:06:14 +0100 Subject: [PATCH 017/501] mm/page_alloc: don't build vm_numa_stat_key if CONFIG_NUMA=n vm_numa_stat_key is only exported if CONFIG_NUMA is set, so avoid the following warning by guarding it in an #ifdef on CONFIG_NUMA: mm/page_alloc.c:165:1: warning: symbol 'vm_numa_stat_key' was not declared. Should it be static? Link: https://lore.kernel.org/20260618100614.1321950-1-ben.dooks@codethink.co.uk Signed-off-by: Ben Dooks Acked-by: Johannes Weiner Reviewed-by: Zi Yan Reviewed-by: SeongJae Park Cc: Brendan Jackman Cc: Michal Hocko Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/page_alloc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index ee902a468c2f..dbe632f6300d 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -165,7 +165,9 @@ DEFINE_PER_CPU(int, numa_node); EXPORT_PER_CPU_SYMBOL(numa_node); #endif +#ifdef CONFIG_NUMA DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); +#endif #ifdef CONFIG_HAVE_MEMORYLESS_NODES /* From d623ed9ba0896f100f7725b0134dedc745379669 Mon Sep 17 00:00:00 2001 From: Andrew Morton Date: Mon, 29 Jun 2026 17:07:51 -0700 Subject: [PATCH 018/501] csky: implement flush_cache_vmap() in C To avoid getting an unused-var warning from unsigned long start = something; ... flush_cache_vmap(start, ...); Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202606291606.9h8aGniQ-lkp@intel.com/ Reviewed-by: Guo Ren Reviewed-by: Barry Song Cc: Andrew Donnellan Cc: Anshuman Khandual Cc: Catalin Marinas Cc: David Hildenbrand Cc: Dev Jain Cc: Leo Yan Cc: Mike Rapoport Cc: Ryan Roberts Cc: Uladzislau Rezki Cc: Wen Jiang Cc: Wen Jiang Cc: Will Deacon Cc: Xueyuan Chen Signed-off-by: Andrew Morton --- arch/csky/abiv1/inc/abi/cacheflush.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/csky/abiv1/inc/abi/cacheflush.h b/arch/csky/abiv1/inc/abi/cacheflush.h index d011a81575d2..21cc7965c53d 100644 --- a/arch/csky/abiv1/inc/abi/cacheflush.h +++ b/arch/csky/abiv1/inc/abi/cacheflush.h @@ -42,7 +42,12 @@ static inline void flush_anon_page(struct vm_area_struct *vma, * Use cache_wbinv_all() here and need to be improved in future. */ extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); -#define flush_cache_vmap(start, end) cache_wbinv_all() + +static inline void flush_cache_vmap(unsigned long start, unsigned long end) +{ + cache_wbinv_all(); +} + #define flush_cache_vmap_early(start, end) do { } while (0) #define flush_cache_vunmap(start, end) cache_wbinv_all() From 73e28dccd095327390ba5111d08d9d19fdc8d5c8 Mon Sep 17 00:00:00 2001 From: Sang-Heon Jeon Date: Thu, 18 Jun 2026 01:39:19 +0900 Subject: [PATCH 019/501] arch_numa: remove redundant nodemask clears in numa_init() numa_init() clears numa_nodes_parsed, node_possible_map and node_online_map, then calls numa_memblks_init(), which clears the same nodemasks. Nothing uses them in between. These clears have been redundant since commit 767507654c22 ("arch_numa: switch over to numa_memblks") made numa_init() use numa_memblks_init(). No functional change. Link: https://lore.kernel.org/20260617163919.2544899-1-ekffu200098@gmail.com Signed-off-by: Sang-Heon Jeon Reviewed-by: Mike Rapoport (Microsoft) Cc: Danilo Krummrich Cc: Greg Kroah-Hartman Cc: "Rafael J. Wysocki" Signed-off-by: Andrew Morton --- drivers/base/arch_numa.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c index c99f2ab105e5..442ea239bba7 100644 --- a/drivers/base/arch_numa.c +++ b/drivers/base/arch_numa.c @@ -231,10 +231,6 @@ static int __init numa_init(int (*init_func)(void)) { int ret; - nodes_clear(numa_nodes_parsed); - nodes_clear(node_possible_map); - nodes_clear(node_online_map); - ret = numa_memblks_init(init_func, /* memblock_force_top_down */ false); if (ret < 0) goto out_free_distance; From 24e9b62985d8f40e72bf3b31ce908ae3c9547a67 Mon Sep 17 00:00:00 2001 From: Markus Elfring Date: Tue, 16 Jun 2026 18:14:52 +0200 Subject: [PATCH 020/501] mm/vmalloc: use more common error handling code in pcpu_get_vm_areas() Use an existing label once more so that a bit of exception handling can be better reused at the end of this function implementation. This issue was detected by using the Coccinelle software. Link: https://lore.kernel.org/453375c4-c3ca-4e6f-8880-0e6ff3c74ee3@web.de Signed-off-by: Markus Elfring Reviewed-by: Uladzislau Rezki (Sony) Cc: Alexander Potapenko Cc: Daniel Axtens Cc: Dmitry Vyukov Signed-off-by: Andrew Morton --- mm/vmalloc.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 08f468135e4d..12f4a39fdd0b 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -5199,9 +5199,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, kfree(vms[area]); } spin_unlock(&free_vmap_area_lock); - kfree(vas); - kfree(vms); - return NULL; + goto err_free2; } /** From aa38f2454b4ee8da82c01d8e878967b61d3e1792 Mon Sep 17 00:00:00 2001 From: Ethan Nelson-Moore Date: Mon, 15 Jun 2026 17:01:31 -0700 Subject: [PATCH 021/501] mm: hugetlb: correct CONFIG_CGROUP_HUGETLB macro name in comment A comment in incorrectly refers to CONFIG_MEM_RES_CTLR_HUGETLB, which has never existed in the kernel, instead of CONFIG_CGROUP_HUGETLB. Correct it. Discovered while searching for CONFIG_* symbols referenced in code but not defined in any Kconfig file. Link: https://lore.kernel.org/20260616000135.62815-1-enelsonmoore@gmail.com Signed-off-by: Ethan Nelson-Moore Reviewed-by: David Hildenbrand (Arm) Reviewed-by: Lorenzo Stoakes Cc: Anthony Yznaga Cc: Muchun Song Cc: Oscar Salvador Cc: Pedro Falcato Signed-off-by: Andrew Morton --- include/linux/hugetlb_cgroup.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/hugetlb_cgroup.h b/include/linux/hugetlb_cgroup.h index e5d64b8b59c2..16d72c8c71f6 100644 --- a/include/linux/hugetlb_cgroup.h +++ b/include/linux/hugetlb_cgroup.h @@ -267,5 +267,5 @@ static inline void hugetlb_cgroup_migrate(struct folio *old_folio, { } -#endif /* CONFIG_MEM_RES_CTLR_HUGETLB */ +#endif /* CONFIG_CGROUP_HUGETLB */ #endif From 5d10d4e19e6daa487f0cd0ea6cba472325de92f9 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Mon, 15 Jun 2026 10:49:06 -0700 Subject: [PATCH 022/501] mm/kmemleak: avoid soft lockup when scanning task stacks Patch series "mm/kmemleak: avoid soft lockup when scanning task", v3. kmemleak_scan() scans every task stack under one rcu_read_lock() with no reschedule point, which can trip the soft lockup watchdog on hosts with very many threads. That prints the following message, depending on the workload+host configuration: watchdog: BUG: soft lockup - CPU#35 stuck for 22s! [kmemleak:537] scan_block kmemleak_scan kmemleak_scan_thread kthread Patch 1 walks the tasks with find_ge_pid() so the scan reschedules between tasks Patches 2-3 let the scan loops stop early once a scan is interrupted. This patch (of 3): kmemleak_scan() walks every thread and scans its kernel stack under a single rcu_read_lock() with no reschedule point. On a host with very many threads -- amplified by KASAN/lockdep in debug builds -- this loop can hog a CPU long enough to trip the soft lockup watchdog: watchdog: BUG: soft lockup - CPU#35 stuck for 22s! [kmemleak:537] scan_block kmemleak_scan kmemleak_scan_thread kthread A cond_resched() cannot be added directly: the loop runs inside an RCU read-side critical section. Walk the tasks one PID at a time with find_ge_pid(), taking the RCU read lock only to look up and pin each task. The stack is then scanned with no lock held, so cond_resched() runs between tasks and the scan stops early on scan_should_stop(). This follows the next_tgid()/task_seq_get_next() iteration pattern and keeps each RCU critical section short. Link: https://lore.kernel.org/20260615-kmemleak-stack-resched-v3-0-acecd7d7fd92@debian.org Link: https://lore.kernel.org/20260615-kmemleak-stack-resched-v3-1-acecd7d7fd92@debian.org Fixes: c4b28963fd79 ("mm/kmemleak: rely on rcu for task stack scanning") Signed-off-by: Breno Leitao Reviewed-by: Catalin Marinas Reviewed-by: Davidlohr Bueso Reviewed-by: Lance Yang Reviewed-by: Oleg Nesterov Cc: Qian Cai Cc: SeongJae Park Cc: Signed-off-by: Andrew Morton --- mm/kmemleak.c | 51 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/mm/kmemleak.c b/mm/kmemleak.c index e196f53f9b46..16b72cead07d 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -1696,6 +1696,42 @@ static void kmemleak_cond_resched(struct kmemleak_object *object) put_object(object); } +/* + * Scan all task kernel stacks, rescheduling between tasks. Each task is looked + * up and pinned within its own RCU read-side section, so no lock is held across + * the scan and the walk cannot trip the soft lockup watchdog. + */ +static void kmemleak_scan_task_stacks(void) +{ + struct pid *pid; + int nr = 1; + + do { + struct task_struct *p = NULL; + + rcu_read_lock(); + pid = find_ge_pid(nr, &init_pid_ns); + if (pid) { + nr = pid_nr(pid) + 1; + p = pid_task(pid, PIDTYPE_PID); + if (p) + get_task_struct(p); + } + rcu_read_unlock(); + + if (p) { + void *stack = try_get_task_stack(p); + + if (stack) { + scan_block(stack, stack + THREAD_SIZE, NULL); + put_task_stack(p); + } + put_task_struct(p); + } + cond_resched(); + } while (pid && !scan_should_stop()); +} + /* * Print one leak inline. The hex dump is gated on OBJECT_ALLOCATED so it * does not touch user memory that was freed concurrently; the rest of the @@ -1885,19 +1921,8 @@ static void kmemleak_scan(void) /* * Scanning the task stacks (may introduce false negatives). */ - if (kmemleak_stack_scan) { - struct task_struct *p, *g; - - rcu_read_lock(); - for_each_process_thread(g, p) { - void *stack = try_get_task_stack(p); - if (stack) { - scan_block(stack, stack + THREAD_SIZE, NULL); - put_task_stack(p); - } - } - rcu_read_unlock(); - } + if (kmemleak_stack_scan) + kmemleak_scan_task_stacks(); /* * Scan the objects already referenced from the sections scanned From eb11f56eeca56069613d00bf623607e305b271d9 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Mon, 15 Jun 2026 10:49:07 -0700 Subject: [PATCH 023/501] mm/kmemleak: stop the task stack scan early when interrupted scan_block() already checks scan_should_stop() for every pointer and bails out of the current block, but the task stack walk cannot tell and keeps issuing a separate scan_should_stop() between every task. Return that status from scan_block() and use it as the task stack loop condition, so the walk stops as soon as a scan is interrupted. Link: https://lore.kernel.org/20260615-kmemleak-stack-resched-v3-2-acecd7d7fd92@debian.org Signed-off-by: Breno Leitao Suggested-by: Catalin Marinas Reviewed-by: Catalin Marinas Reviewed-by: Oleg Nesterov Cc: Davidlohr Bueso Cc: Lance Yang Cc: Qian Cai Cc: SeongJae Park Signed-off-by: Andrew Morton --- mm/kmemleak.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/mm/kmemleak.c b/mm/kmemleak.c index 16b72cead07d..bc79e293531b 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -1525,22 +1525,25 @@ static int scan_should_stop(void) /* * Scan a memory block (exclusive range) for valid pointers and add those - * found to the gray list. + * found to the gray list. Return non-zero if the scan was interrupted. */ -static void scan_block(void *_start, void *_end, - struct kmemleak_object *scanned) +static int scan_block(void *_start, void *_end, + struct kmemleak_object *scanned) { unsigned long *ptr; unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER); unsigned long *end = _end - (BYTES_PER_POINTER - 1); unsigned long flags; + int stop = 0; raw_spin_lock_irqsave(&kmemleak_lock, flags); for (ptr = start; ptr < end; ptr++) { unsigned long pointer; - if (scan_should_stop()) + if (scan_should_stop()) { + stop = 1; break; + } kasan_disable_current(); pointer = *(unsigned long *)kasan_reset_tag((void *)ptr); @@ -1550,6 +1553,8 @@ static void scan_block(void *_start, void *_end, pointer_update_refs(scanned, pointer, OBJECT_PERCPU); } raw_spin_unlock_irqrestore(&kmemleak_lock, flags); + + return stop; } /* @@ -1705,6 +1710,7 @@ static void kmemleak_scan_task_stacks(void) { struct pid *pid; int nr = 1; + int stop = 0; do { struct task_struct *p = NULL; @@ -1723,13 +1729,13 @@ static void kmemleak_scan_task_stacks(void) void *stack = try_get_task_stack(p); if (stack) { - scan_block(stack, stack + THREAD_SIZE, NULL); + stop = scan_block(stack, stack + THREAD_SIZE, NULL); put_task_stack(p); } put_task_struct(p); } cond_resched(); - } while (pid && !scan_should_stop()); + } while (pid && !stop); } /* From cdf95d6b3387b99a9377046491a0a51bad93b5af Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Mon, 15 Jun 2026 10:49:08 -0700 Subject: [PATCH 024/501] mm/kmemleak: stop the per-cpu and struct page scans early too The per-cpu and struct page scan loops have no reschedule-stop check of their own: once a scan is interrupted they keep calling scan_block() for every remaining block, which scans nothing useful. Propagate scan_block()'s interrupted status through scan_large_block() and break both loops as soon as it is set. Link: https://lore.kernel.org/20260615-kmemleak-stack-resched-v3-3-acecd7d7fd92@debian.org Signed-off-by: Breno Leitao Suggested-by: Catalin Marinas Reviewed-by: Catalin Marinas Reviewed-by: Oleg Nesterov Cc: Davidlohr Bueso Cc: Lance Yang Cc: Qian Cai Cc: SeongJae Park Signed-off-by: Andrew Morton --- mm/kmemleak.c | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/mm/kmemleak.c b/mm/kmemleak.c index bc79e293531b..ac2a44a1c4a5 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -1559,18 +1559,22 @@ static int scan_block(void *_start, void *_end, /* * Scan a large memory block in MAX_SCAN_SIZE chunks to reduce the latency. + * Return non-zero if the scan was interrupted. */ #ifdef CONFIG_SMP -static void scan_large_block(void *start, void *end) +static int scan_large_block(void *start, void *end) { void *next; while (start < end) { next = min(start + MAX_SCAN_SIZE, end); - scan_block(start, next, NULL); + if (scan_block(start, next, NULL)) + return 1; start = next; cond_resched(); } + + return 0; } #endif @@ -1890,9 +1894,11 @@ static void kmemleak_scan(void) #ifdef CONFIG_SMP /* per-cpu sections scanning */ - for_each_possible_cpu(i) - scan_large_block(__per_cpu_start + per_cpu_offset(i), - __per_cpu_end + per_cpu_offset(i)); + for_each_possible_cpu(i) { + if (scan_large_block(__per_cpu_start + per_cpu_offset(i), + __per_cpu_end + per_cpu_offset(i))) + break; + } #endif /* @@ -1903,6 +1909,7 @@ static void kmemleak_scan(void) unsigned long start_pfn = zone->zone_start_pfn; unsigned long end_pfn = zone_end_pfn(zone); unsigned long pfn; + int stop = 0; for (pfn = start_pfn; pfn < end_pfn; pfn++) { struct page *page = pfn_to_online_page(pfn); @@ -1919,8 +1926,12 @@ static void kmemleak_scan(void) /* only scan if page is in use */ if (page_count(page) == 0) continue; - scan_block(page, page + 1, NULL); + stop = scan_block(page, page + 1, NULL); + if (stop) + break; } + if (stop) + break; } put_online_mems(); From 53472eb262913d144df89ea959fcc30e43001f79 Mon Sep 17 00:00:00 2001 From: Ye Liu Date: Tue, 14 Jul 2026 09:51:00 +0800 Subject: [PATCH 025/501] mm/page_owner: extract skip_buddy_pages() helper to unify buddy page skipping Patch series "mm/page_owner: misc cleanups", v6. This series collects a few cleanups for mm/page_owner.c that have been accumulated while reading through the file. There is no functional change -- the goal is to make the code easier to read and maintain. Patch 1 consolidates three identical PageBuddy skip blocks into a single skip_buddy_pages() helper, eliminating the duplication and keeping the lockless-read comment in one place. Patch 2 replaces the -1 magic number used for "never migrated" with a proper MR_NEVER member in enum migrate_reason, adds the corresponding "never_migrated" string in the MIGRATE_REASON trace macro, and updates the GDB page_owner script to use MR_NEVER so that lx-dump-page-owner correctly detects unmigrated pages. Patch 3 follows up by converting the remaining 'int reason' parameters throughout the migration and hugetlb callchains to 'enum migrate_reason', making the type explicit and gaining compiler checking. The 'short last_migrate_reason' struct field in page_owner is intentionally left as 'short' since it is per-page metadata where size matters. Patch 4 hoists the CONFIG_MEMCG guard out of print_page_owner_memcg()'s body so that the real implementation and the empty stub are two clearly separate definitions, the common kernel idiom. Patch 5 adds a missing \n to the count_threshold debugfs attribute format string so that cat(1) output is properly terminated. Patch 6 moves free_ts_nsec from the allocation summary line to the free section in __dump_page_owner(), grouping it with free_pid and free_tgid where it logically belongs. This also makes the dump output consistent with print_page_owner(). Patch 7 drops the redundant page_owner_ prefix from file-scoped static symbols (stack_fops, threshold_fops, etc.). Since they cannot collide across translation units, the prefix carries no information. Patch 8 clamps the PFN advance in skip_buddy_pages() at the next MAX_ORDER_NR_PAGES boundary. The lockless buddy_order_unsafe() read can return a garbage order value if the page is concurrently allocated between the PageBuddy check and the private read, potentially causing the PFN to advance past the next bounadry whose pfn_valid() check would have caught an offline memory section. In read_page_owner(), which relies solely on boundary-aligned pfn_valid() to guard pfn_to_page(), this could lead to an unmapped mem_section access. Patch 9 avoids two TOCTOU issues in print_page_owner_memcg() by reusing the page->memcg_data snapshot already taken via READ_ONCE at the top of the function throughout, instead of calling page_memcg_check() and PageMemcgKmem() which re-read page->memcg_data locklessly with VM_BUG_ON assertions. If the page is concurrently freed and reallocated as a THP tail or slab page between the initial guards and these later calls, those assertions can fire on CONFIG_DEBUG_VM=y builds. The OBJEXTS (slab) case is also simplified with an early return since objcg != memcg for slabs. This patch (of 6): Three places in page_owner.c duplicate the same pattern: check if a page is PageBuddy, read its order via buddy_order_unsafe(), advance the pfn past the buddy block if the order is valid, and continue. Consolidate them into a single inline helper skip_buddy_pages(). The function returns true (skip) for any buddy page and advances @pfn past the block when the order is valid; returns false if the page is not a buddy page and should be processed normally. The old init_pages_in_zone() variant used "order > 0" as an extra guard before advancing pfn, but the continue was unconditional and (1UL << 0) - 1 == 0, so the behaviour is identical. The comment about zone->lock is preserved in the helper's kernel-doc. No functional change. Link: https://lore.kernel.org/20260714015117.78351-1-ye.liu@linux.dev Link: https://lore.kernel.org/20260714015117.78351-2-ye.liu@linux.dev Signed-off-by: Ye Liu Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Brendan Jackman Cc: Johannes Weiner Cc: Michal Hocko Cc: Suren Baghdasaryan Cc: Lorenzo Stoakes Cc: David Hildenbrand (Arm) Signed-off-by: Andrew Morton --- mm/page_owner.c | 52 ++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/mm/page_owner.c b/mm/page_owner.c index 2dddcb6510aa..342549891a8d 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -422,6 +422,29 @@ void __folio_copy_owner(struct folio *newfolio, struct folio *old) rcu_read_unlock(); } +/* + * Check if a page is a buddy page and advance @pfn past the entire buddy block. + * This safely reads the buddy order without the zone lock, which may cause us + * to skip less than the full buddy block, but that is acceptable for page owner + * iteration purposes. + * + * Return: true if the page was skipped (caller should continue its loop), + * false if the page is not a buddy page and should be processed normally. + */ +static inline bool skip_buddy_pages(unsigned long *pfn, struct page *page) +{ + unsigned long order; + + if (!PageBuddy(page)) + return false; + + order = buddy_order_unsafe(page); + if (order <= MAX_PAGE_ORDER) + *pfn += (1UL << order) - 1; + + return true; +} + void pagetypeinfo_showmixedcount_print(struct seq_file *m, pg_data_t *pgdat, struct zone *zone) { @@ -461,14 +484,8 @@ void pagetypeinfo_showmixedcount_print(struct seq_file *m, if (page_zone(page) != zone) continue; - if (PageBuddy(page)) { - unsigned long freepage_order; - - freepage_order = buddy_order_unsafe(page); - if (freepage_order <= MAX_PAGE_ORDER) - pfn += (1UL << freepage_order) - 1; + if (skip_buddy_pages(&pfn, page)) continue; - } if (PageReserved(page)) continue; @@ -697,13 +714,8 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) } page = pfn_to_page(pfn); - if (PageBuddy(page)) { - unsigned long freepage_order = buddy_order_unsafe(page); - - if (freepage_order <= MAX_PAGE_ORDER) - pfn += (1UL << freepage_order) - 1; + if (skip_buddy_pages(&pfn, page)) continue; - } page_ext = page_ext_get(page); if (unlikely(!page_ext)) @@ -798,20 +810,8 @@ static void init_pages_in_zone(struct zone *zone) if (page_zone(page) != zone) continue; - /* - * To avoid having to grab zone->lock, be a little - * careful when reading buddy page order. The only - * danger is that we skip too much and potentially miss - * some early allocated pages, which is better than - * heavy lock contention. - */ - if (PageBuddy(page)) { - unsigned long order = buddy_order_unsafe(page); - - if (order > 0 && order <= MAX_PAGE_ORDER) - pfn += (1UL << order) - 1; + if (skip_buddy_pages(&pfn, page)) continue; - } if (PageReserved(page)) continue; From 7eeed888c99a0619911165030bf3f82b265c141a Mon Sep 17 00:00:00 2001 From: Ye Liu Date: Tue, 14 Jul 2026 09:51:01 +0800 Subject: [PATCH 026/501] mm/page_owner: add MR_NEVER to enum migrate_reason and use it for last_migrate_reason The last_migrate_reason field uses -1 as a sentinel value to mean "no migration has happened". Replace the four bare -1 occurrences by adding a proper MR_NEVER member to enum migrate_reason, defining a corresponding "never_migrated" string in the MIGRATE_REASON trace macro, and updating the GDB page_owner script to use MR_NEVER instead of the hardcoded -1 so that lx-dump-page-owner does not incorrectly report unmigrated pages as migrated. No functional change. Link: https://lore.kernel.org/20260714015117.78351-3-ye.liu@linux.dev Signed-off-by: Ye Liu Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Brendan Jackman Cc: Johannes Weiner Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Suren Baghdasaryan Cc: David Hildenbrand (Arm) Signed-off-by: Andrew Morton --- include/linux/migrate_mode.h | 1 + include/trace/events/migrate.h | 3 ++- mm/page_owner.c | 8 ++++---- scripts/gdb/linux/page_owner.py | 4 +++- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/include/linux/migrate_mode.h b/include/linux/migrate_mode.h index 265c4328b36a..05102d4d2490 100644 --- a/include/linux/migrate_mode.h +++ b/include/linux/migrate_mode.h @@ -25,6 +25,7 @@ enum migrate_reason { MR_LONGTERM_PIN, MR_DEMOTION, MR_DAMON, + MR_NEVER, /* page has never been migrated */ MR_TYPES }; diff --git a/include/trace/events/migrate.h b/include/trace/events/migrate.h index cd01dd7b3640..11bc0aa14c7e 100644 --- a/include/trace/events/migrate.h +++ b/include/trace/events/migrate.h @@ -23,7 +23,8 @@ EM( MR_CONTIG_RANGE, "contig_range") \ EM( MR_LONGTERM_PIN, "longterm_pin") \ EM( MR_DEMOTION, "demotion") \ - EMe(MR_DAMON, "damon") + EM( MR_DAMON, "damon") \ + EMe(MR_NEVER, "never_migrated") /* * First define the enums in the above macros to be exported to userspace diff --git a/mm/page_owner.c b/mm/page_owner.c index 342549891a8d..c2f43ab860eb 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -339,7 +339,7 @@ noinline void __set_page_owner(struct page *page, unsigned short order, depot_stack_handle_t handle; handle = save_stack(gfp_mask); - __update_page_owner_handle(page, handle, order, gfp_mask, -1, + __update_page_owner_handle(page, handle, order, gfp_mask, MR_NEVER, ts_nsec, current->pid, current->tgid, current->comm); inc_stack_record_count(handle, gfp_mask, 1 << order); @@ -596,7 +596,7 @@ print_page_owner(char __user *buf, size_t count, unsigned long pfn, if (ret >= count) goto err; - if (page_owner->last_migrate_reason != -1) { + if (page_owner->last_migrate_reason != MR_NEVER) { ret += scnprintf(kbuf + ret, count - ret, "Page has been migrated, last migrate reason: %s\n", migrate_reason_names[page_owner->last_migrate_reason]); @@ -667,7 +667,7 @@ void __dump_page_owner(const struct page *page) stack_depot_print(handle); } - if (page_owner->last_migrate_reason != -1) + if (page_owner->last_migrate_reason != MR_NEVER) pr_alert("page has been migrated, last migrate reason: %s\n", migrate_reason_names[page_owner->last_migrate_reason]); page_ext_put(page_ext); @@ -826,7 +826,7 @@ static void init_pages_in_zone(struct zone *zone) /* Found early allocated page */ __update_page_owner_handle(page, early_handle, 0, 0, - -1, local_clock(), current->pid, + MR_NEVER, local_clock(), current->pid, current->tgid, current->comm); count++; ext_put_continue: diff --git a/scripts/gdb/linux/page_owner.py b/scripts/gdb/linux/page_owner.py index 8e713a09cfe7..eeabaeed438b 100644 --- a/scripts/gdb/linux/page_owner.py +++ b/scripts/gdb/linux/page_owner.py @@ -34,6 +34,7 @@ class DumpPageOwner(gdb.Command): max_pfn = None p_ops = None migrate_reason_names = None + mr_never = None def __init__(self): super(DumpPageOwner, self).__init__("lx-dump-page-owner", gdb.COMMAND_SUPPORT) @@ -65,6 +66,7 @@ class DumpPageOwner(gdb.Command): self.max_pfn = int(gdb.parse_and_eval("max_pfn")) self.page_ext_size = int(gdb.parse_and_eval("page_ext_size")) self.migrate_reason_names = gdb.parse_and_eval('migrate_reason_names') + self.mr_never = int(gdb.parse_and_eval('MR_NEVER')) def page_ext_invalid(self, page_ext): if page_ext == gdb.Value(0): @@ -138,7 +140,7 @@ class DumpPageOwner(gdb.Command): else: gdb.write('page last free stack trace:\n') stackdepot.stack_depot_print(page_owner["free_handle"]) - if page_owner['last_migrate_reason'] != -1: + if page_owner['last_migrate_reason'] != self.mr_never: gdb.write('page has been migrated, last migrate reason: %s\n' % self.migrate_reason_names[page_owner['last_migrate_reason']]) def read_page_owner(self): From c95dc0a84b4c84103bd688a93b28d84e8b752568 Mon Sep 17 00:00:00 2001 From: Ye Liu Date: Tue, 14 Jul 2026 09:51:02 +0800 Subject: [PATCH 027/501] mm: use enum migrate_reason instead of int for migration reason parameters Replace all 'int reason' function parameters that carry migrate_reason values with the proper 'enum migrate_reason' type. This makes the intent explicit and leverages compiler type checking. The affected subsystems are: - page_owner: __folio_set_owner_migrate_reason(), folio_set_owner_migrate_reason() - migrate: migrate_pages(), migrate_pages_sync(), migrate_pages_batch(), migrate_folios_move(), migrate_hugetlbs(), unmap_and_move_huge_page() - hugetlb: move_hugetlb_state(), htlb_allow_alloc_fallback() - trace: mm_migrate_pages and mm_migrate_pages_start events The 'short last_migrate_reason' struct field and internal helper parameter in page_owner are intentionally left as 'short' since they store per-page metadata where size matters. No functional change. Link: https://lore.kernel.org/20260714015117.78351-4-ye.liu@linux.dev Signed-off-by: Ye Liu Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Lorenzo Stoakes Acked-by: David Hildenbrand (Arm) Cc: Brendan Jackman Cc: Johannes Weiner Cc: Michal Hocko Cc: Suren Baghdasaryan Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 9 +++++---- include/linux/migrate.h | 8 +++++--- include/linux/page_owner.h | 7 ++++--- include/trace/events/migrate.h | 8 ++++---- mm/hugetlb.c | 3 ++- mm/migrate.c | 12 ++++++------ mm/page_owner.c | 2 +- 7 files changed, 27 insertions(+), 22 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 2abaf99321e9..d6967fc94a49 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -154,7 +154,8 @@ long hugetlb_unreserve_pages(struct inode *inode, long start, long end, bool folio_isolate_hugetlb(struct folio *folio, struct list_head *list); int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison); void folio_putback_hugetlb(struct folio *folio); -void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason); +void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, + enum migrate_reason reason); void hugetlb_fix_reserve_counts(struct inode *inode); extern struct mutex *hugetlb_fault_mutex_table; u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx); @@ -424,7 +425,7 @@ static inline void folio_putback_hugetlb(struct folio *folio) } static inline void move_hugetlb_state(struct folio *old_folio, - struct folio *new_folio, int reason) + struct folio *new_folio, enum migrate_reason reason) { } @@ -956,7 +957,7 @@ static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask) return modified_mask; } -static inline bool htlb_allow_alloc_fallback(int reason) +static inline bool htlb_allow_alloc_fallback(enum migrate_reason reason) { bool allowed_fallback = false; @@ -1238,7 +1239,7 @@ static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask) return 0; } -static inline bool htlb_allow_alloc_fallback(int reason) +static inline bool htlb_allow_alloc_fallback(enum migrate_reason reason) { return false; } diff --git a/include/linux/migrate.h b/include/linux/migrate.h index d5af2b7f577b..78424b3824c2 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -57,8 +57,9 @@ void putback_movable_pages(struct list_head *l); int migrate_folio(struct address_space *mapping, struct folio *dst, struct folio *src, enum migrate_mode mode); int migrate_pages(struct list_head *l, new_folio_t new, free_folio_t free, - unsigned long private, enum migrate_mode mode, int reason, - unsigned int *ret_succeeded); + unsigned long private, enum migrate_mode mode, + enum migrate_reason reason, + unsigned int *ret_succeeded); struct folio *alloc_migration_target(struct folio *src, unsigned long private); bool isolate_movable_ops_page(struct page *page, isolate_mode_t mode); bool isolate_folio_to_list(struct folio *folio, struct list_head *list); @@ -77,7 +78,8 @@ int set_movable_ops(const struct movable_operations *ops, enum pagetype type); static inline void putback_movable_pages(struct list_head *l) {} static inline int migrate_pages(struct list_head *l, new_folio_t new, free_folio_t free, unsigned long private, - enum migrate_mode mode, int reason, unsigned int *ret_succeeded) + enum migrate_mode mode, enum migrate_reason reason, + unsigned int *ret_succeeded) { return -ENOSYS; } static inline struct folio *alloc_migration_target(struct folio *src, unsigned long private) diff --git a/include/linux/page_owner.h b/include/linux/page_owner.h index 3328357f6dba..8188ddc5c412 100644 --- a/include/linux/page_owner.h +++ b/include/linux/page_owner.h @@ -3,6 +3,7 @@ #define __LINUX_PAGE_OWNER_H #include +#include #ifdef CONFIG_PAGE_OWNER extern struct static_key_false page_owner_inited; @@ -14,7 +15,7 @@ extern void __set_page_owner(struct page *page, extern void __split_page_owner(struct page *page, int old_order, int new_order); extern void __folio_copy_owner(struct folio *newfolio, struct folio *old); -extern void __folio_set_owner_migrate_reason(struct folio *folio, int reason); +void __folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason); extern void __dump_page_owner(const struct page *page); extern void pagetypeinfo_showmixedcount_print(struct seq_file *m, pg_data_t *pgdat, struct zone *zone); @@ -43,7 +44,7 @@ static inline void folio_copy_owner(struct folio *newfolio, struct folio *old) if (static_branch_unlikely(&page_owner_inited)) __folio_copy_owner(newfolio, old); } -static inline void folio_set_owner_migrate_reason(struct folio *folio, int reason) +static inline void folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason) { if (static_branch_unlikely(&page_owner_inited)) __folio_set_owner_migrate_reason(folio, reason); @@ -68,7 +69,7 @@ static inline void split_page_owner(struct page *page, int old_order, static inline void folio_copy_owner(struct folio *newfolio, struct folio *folio) { } -static inline void folio_set_owner_migrate_reason(struct folio *folio, int reason) +static inline void folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason) { } static inline void dump_page_owner(const struct page *page) diff --git a/include/trace/events/migrate.h b/include/trace/events/migrate.h index 11bc0aa14c7e..15ee2ef201b5 100644 --- a/include/trace/events/migrate.h +++ b/include/trace/events/migrate.h @@ -52,7 +52,7 @@ TRACE_EVENT(mm_migrate_pages, TP_PROTO(unsigned long succeeded, unsigned long failed, unsigned long thp_succeeded, unsigned long thp_failed, unsigned long thp_split, unsigned long large_folio_split, - enum migrate_mode mode, int reason), + enum migrate_mode mode, enum migrate_reason reason), TP_ARGS(succeeded, failed, thp_succeeded, thp_failed, thp_split, large_folio_split, mode, reason), @@ -65,7 +65,7 @@ TRACE_EVENT(mm_migrate_pages, __field( unsigned long, thp_split) __field( unsigned long, large_folio_split) __field( enum migrate_mode, mode) - __field( int, reason) + __field( enum migrate_reason, reason) ), TP_fast_assign( @@ -92,13 +92,13 @@ TRACE_EVENT(mm_migrate_pages, TRACE_EVENT(mm_migrate_pages_start, - TP_PROTO(enum migrate_mode mode, int reason), + TP_PROTO(enum migrate_mode mode, enum migrate_reason reason), TP_ARGS(mode, reason), TP_STRUCT__entry( __field(enum migrate_mode, mode) - __field(int, reason) + __field(enum migrate_reason, reason) ), TP_fast_assign( diff --git a/mm/hugetlb.c b/mm/hugetlb.c index e93c4d2456aa..b75774455d65 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -7186,7 +7186,8 @@ void folio_putback_hugetlb(struct folio *folio) folio_put(folio); } -void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason) +void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, + enum migrate_reason reason) { struct hstate *h = folio_hstate(old_folio); diff --git a/mm/migrate.c b/mm/migrate.c index d9b23909d716..49e10feeb094 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1469,7 +1469,7 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private, static int unmap_and_move_huge_page(new_folio_t get_new_folio, free_folio_t put_new_folio, unsigned long private, struct folio *src, int force, enum migrate_mode mode, - int reason, struct list_head *ret) + enum migrate_reason reason, struct list_head *ret) { struct folio *dst; int rc = -EAGAIN; @@ -1626,7 +1626,7 @@ struct migrate_pages_stats { */ static int migrate_hugetlbs(struct list_head *from, new_folio_t get_new_folio, free_folio_t put_new_folio, unsigned long private, - enum migrate_mode mode, int reason, + enum migrate_mode mode, enum migrate_reason reason, struct migrate_pages_stats *stats, struct list_head *ret_folios) { @@ -1716,7 +1716,7 @@ static int migrate_hugetlbs(struct list_head *from, new_folio_t get_new_folio, static void migrate_folios_move(struct list_head *src_folios, struct list_head *dst_folios, free_folio_t put_new_folio, unsigned long private, - enum migrate_mode mode, int reason, + enum migrate_mode mode, enum migrate_reason reason, struct list_head *ret_folios, struct migrate_pages_stats *stats, int *retry, int *thp_retry, int *nr_failed, @@ -1799,7 +1799,7 @@ static void migrate_folios_undo(struct list_head *src_folios, */ static int migrate_pages_batch(struct list_head *from, new_folio_t get_new_folio, free_folio_t put_new_folio, - unsigned long private, enum migrate_mode mode, int reason, + unsigned long private, enum migrate_mode mode, enum migrate_reason reason, struct list_head *ret_folios, struct list_head *split_folios, struct migrate_pages_stats *stats, int nr_pass) { @@ -2011,7 +2011,7 @@ static int migrate_pages_batch(struct list_head *from, static int migrate_pages_sync(struct list_head *from, new_folio_t get_new_folio, free_folio_t put_new_folio, unsigned long private, - enum migrate_mode mode, int reason, + enum migrate_mode mode, enum migrate_reason reason, struct list_head *ret_folios, struct list_head *split_folios, struct migrate_pages_stats *stats) { @@ -2088,7 +2088,7 @@ static int migrate_pages_sync(struct list_head *from, new_folio_t get_new_folio, */ int migrate_pages(struct list_head *from, new_folio_t get_new_folio, free_folio_t put_new_folio, unsigned long private, - enum migrate_mode mode, int reason, unsigned int *ret_succeeded) + enum migrate_mode mode, enum migrate_reason reason, unsigned int *ret_succeeded) { int rc, rc_gather; int nr_pages; diff --git a/mm/page_owner.c b/mm/page_owner.c index c2f43ab860eb..4e352941a6e2 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -345,7 +345,7 @@ noinline void __set_page_owner(struct page *page, unsigned short order, inc_stack_record_count(handle, gfp_mask, 1 << order); } -void __folio_set_owner_migrate_reason(struct folio *folio, int reason) +void __folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason) { struct page_ext *page_ext = page_ext_get(&folio->page); struct page_owner *page_owner; From e90ed519087d543432f333a1bad42a5c3c59abb8 Mon Sep 17 00:00:00 2001 From: Ye Liu Date: Tue, 14 Jul 2026 09:51:03 +0800 Subject: [PATCH 028/501] mm/page_owner: hoist CONFIG_MEMCG to function level for print_page_owner_memcg() The print_page_owner_memcg() function has CONFIG_MEMCG guarding its entire body via #ifdef inside the function, which leaves a no-op { return ret; } when the config is disabled. Hoist the #ifdef to the top level so the real implementation and the empty stub are two clearly separated definitions. No functional change. Link: https://lore.kernel.org/20260714015117.78351-5-ye.liu@linux.dev Signed-off-by: Ye Liu Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Brendan Jackman Cc: Johannes Weiner Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Suren Baghdasaryan Cc: David Hildenbrand (Arm) Signed-off-by: Andrew Morton --- mm/page_owner.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mm/page_owner.c b/mm/page_owner.c index 4e352941a6e2..fe2bf2274d8a 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -522,13 +522,13 @@ void pagetypeinfo_showmixedcount_print(struct seq_file *m, seq_putc(m, '\n'); } +#ifdef CONFIG_MEMCG /* * Looking for memcg information and print it out */ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, struct page *page) { -#ifdef CONFIG_MEMCG unsigned long memcg_data; struct mem_cgroup *memcg; bool online; @@ -556,10 +556,16 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, name); out_unlock: rcu_read_unlock(); -#endif /* CONFIG_MEMCG */ return ret; } +#else +static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, + struct page *page) +{ + return ret; +} +#endif static ssize_t print_page_owner(char __user *buf, size_t count, unsigned long pfn, From c1c94cf658630d240c7a8b64d9e001c6878a61cc Mon Sep 17 00:00:00 2001 From: Ye Liu Date: Tue, 14 Jul 2026 09:51:04 +0800 Subject: [PATCH 029/501] mm/page_owner: add missing newline to count_threshold format string The DEFINE_SIMPLE_ATTRIBUTE format string for page_owner_threshold_fops is missing a trailing \n. simple_attr_read() uses scnprintf() with the format string, which does not append a newline, so reading /sys/kernel/debug/page_owner_stacks/count_threshold produces output without a terminating newline. Add the missing \n to match the standard debugfs attribute convention. Link: https://lore.kernel.org/20260714015117.78351-6-ye.liu@linux.dev Signed-off-by: Ye Liu Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Brendan Jackman Cc: Johannes Weiner Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Suren Baghdasaryan Cc: David Hildenbrand (Arm) Signed-off-by: Andrew Morton --- mm/page_owner.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/page_owner.c b/mm/page_owner.c index fe2bf2274d8a..7520718f63f1 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -974,7 +974,7 @@ static int page_owner_threshold_set(void *data, u64 val) } DEFINE_SIMPLE_ATTRIBUTE(page_owner_threshold_fops, &page_owner_threshold_get, - &page_owner_threshold_set, "%llu"); + &page_owner_threshold_set, "%llu\n"); static int __init pageowner_init(void) From 1f5af031ffb724d0c7357119da8ef686ad77f9db Mon Sep 17 00:00:00 2001 From: Ye Liu Date: Tue, 14 Jul 2026 09:51:05 +0800 Subject: [PATCH 030/501] mm/page_owner: move free_ts_nsec output to free section in __dump_page_owner() The free_ts_nsec field is a free-event timestamp, but it was printed in the allocation summary line alongside ts_nsec (allocation time). Move it to the free section where it logically belongs, together with free_pid and free_tgid. This also makes __dump_page_owner() consistent with print_page_owner(), which only prints ts_nsec in the allocation summary. The output now groups all free-related information (pid, tgid, timestamp, stack trace) in one place. No functional change except output formatting. Link: https://lore.kernel.org/20260714015117.78351-7-ye.liu@linux.dev Signed-off-by: Ye Liu Acked-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Brendan Jackman Cc: Johannes Weiner Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Suren Baghdasaryan Cc: David Hildenbrand (Arm) Signed-off-by: Andrew Morton --- mm/page_owner.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mm/page_owner.c b/mm/page_owner.c index 7520718f63f1..84eb44459478 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -653,10 +653,10 @@ void __dump_page_owner(const struct page *page) else pr_alert("page_owner tracks the page as freed\n"); - pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu, free_ts %llu\n", + pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu\n", page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask, page_owner->pid, page_owner->tgid, page_owner->comm, - page_owner->ts_nsec, page_owner->free_ts_nsec); + page_owner->ts_nsec); handle = READ_ONCE(page_owner->handle); if (!handle) @@ -668,8 +668,9 @@ void __dump_page_owner(const struct page *page) if (!handle) { pr_alert("page_owner free stack trace missing\n"); } else { - pr_alert("page last free pid %d tgid %d stack trace:\n", - page_owner->free_pid, page_owner->free_tgid); + pr_alert("page last free pid %d tgid %d ts %llu stack trace:\n", + page_owner->free_pid, page_owner->free_tgid, + page_owner->free_ts_nsec); stack_depot_print(handle); } From f0bb69130e1443d0c0df2559612428c41ff44ff2 Mon Sep 17 00:00:00 2001 From: Ye Liu Date: Tue, 14 Jul 2026 09:51:06 +0800 Subject: [PATCH 031/501] mm/page_owner: drop redundant page_owner prefix from static symbols All of these symbols are file-scoped (static) in page_owner.c, so the page_owner_ prefix is pure noise. Rename them to shorter, still-clear names: page_owner_stack_op -> stack_op page_owner_stack_open -> stack_open page_owner_stack_fops -> stack_fops page_owner_pages_threshold -> pages_threshold page_owner_threshold_get -> threshold_get page_owner_threshold_set -> threshold_set page_owner_threshold_fops -> threshold_fops No functional change. Link: https://lore.kernel.org/20260714015117.78351-8-ye.liu@linux.dev Signed-off-by: Ye Liu Acked-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Brendan Jackman Cc: Johannes Weiner Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Suren Baghdasaryan Cc: David Hildenbrand (Arm) Signed-off-by: Andrew Morton --- mm/page_owner.c | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/mm/page_owner.c b/mm/page_owner.c index 84eb44459478..46a933f9c229 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -894,7 +894,7 @@ static void *stack_next(struct seq_file *m, void *v, loff_t *ppos) return stack; } -static unsigned long page_owner_pages_threshold; +static unsigned long pages_threshold; static int stack_print(struct seq_file *m, void *v) { @@ -911,7 +911,7 @@ static int stack_print(struct seq_file *m, void *v) nr_base_pages = refcount_read(&stack_record->count) - 1; if (ctx->flags & STACK_PRINT_FLAG_PAGES && - (nr_base_pages < 1 || nr_base_pages < page_owner_pages_threshold)) + (nr_base_pages < 1 || nr_base_pages < pages_threshold)) return 0; if (ctx->flags & STACK_PRINT_FLAG_STACK) { @@ -933,16 +933,16 @@ static void stack_stop(struct seq_file *m, void *v) { } -static const struct seq_operations page_owner_stack_op = { +static const struct seq_operations stack_op = { .start = stack_start, .next = stack_next, .stop = stack_stop, .show = stack_print }; -static int page_owner_stack_open(struct inode *inode, struct file *file) +static int stack_open(struct inode *inode, struct file *file) { - int ret = seq_open_private(file, &page_owner_stack_op, + int ret = seq_open_private(file, &stack_op, sizeof(struct stack_print_ctx)); if (!ret) { @@ -955,28 +955,26 @@ static int page_owner_stack_open(struct inode *inode, struct file *file) return ret; } -static const struct file_operations page_owner_stack_fops = { - .open = page_owner_stack_open, +static const struct file_operations stack_fops = { + .open = stack_open, .read = seq_read, .llseek = seq_lseek, .release = seq_release_private, }; -static int page_owner_threshold_get(void *data, u64 *val) +static int threshold_get(void *data, u64 *val) { - *val = READ_ONCE(page_owner_pages_threshold); + *val = READ_ONCE(pages_threshold); return 0; } -static int page_owner_threshold_set(void *data, u64 val) +static int threshold_set(void *data, u64 val) { - WRITE_ONCE(page_owner_pages_threshold, val); + WRITE_ONCE(pages_threshold, val); return 0; } -DEFINE_SIMPLE_ATTRIBUTE(page_owner_threshold_fops, &page_owner_threshold_get, - &page_owner_threshold_set, "%llu\n"); - +DEFINE_SIMPLE_ATTRIBUTE(threshold_fops, &threshold_get, &threshold_set, "%llu\n"); static int __init pageowner_init(void) { @@ -992,17 +990,17 @@ static int __init pageowner_init(void) debugfs_create_file("show_stacks", 0400, dir, (void *)(STACK_PRINT_FLAG_STACK | STACK_PRINT_FLAG_PAGES), - &page_owner_stack_fops); + &stack_fops); debugfs_create_file("show_handles", 0400, dir, (void *)(STACK_PRINT_FLAG_HANDLE | STACK_PRINT_FLAG_PAGES), - &page_owner_stack_fops); + &stack_fops); debugfs_create_file("show_stacks_handles", 0400, dir, (void *)(STACK_PRINT_FLAG_STACK | STACK_PRINT_FLAG_HANDLE), - &page_owner_stack_fops); + &stack_fops); debugfs_create_file("count_threshold", 0600, dir, NULL, - &page_owner_threshold_fops); + &threshold_fops); return 0; } late_initcall(pageowner_init) From 4fd1c85cc0314164944c4f1d225b421724456c24 Mon Sep 17 00:00:00 2001 From: Ye Liu Date: Tue, 14 Jul 2026 09:51:07 +0800 Subject: [PATCH 032/501] mm/page_owner: clamp skip_buddy_pages() PFN advance at MAX_ORDER_NR_PAGES boundary The lockless buddy_order_unsafe() read can return a garbage order value if the page is concurrently allocated between the PageBuddy check and the private read. If this bogus order is <= MAX_PAGE_ORDER, skip_buddy_pages() would arbitrarily advance the PFN, potentially jumping past a MAX_ORDER_NR_PAGES boundary whose pfn_valid() check would have caught an offline memory section. In read_page_owner(), which relies solely on boundary-aligned pfn_valid() to guard pfn_to_page(), skipping the boundary could cause pfn_to_page() to access an unmapped mem_section. Clamp the advance so it never crosses the next MAX_ORDER_NR_PAGES boundary. This is safe for all three callers: the pageblock-iterating ones already handle boundary transitions in their outer loops, and for read_page_owner() the worst case is one extra PageBuddy check per 1024 pages when a bogus order would otherwise push past the boundary. Link: https://lore.kernel.org/20260714015117.78351-9-ye.liu@linux.dev Signed-off-by: Ye Liu Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Brendan Jackman Cc: Johannes Weiner Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Suren Baghdasaryan Cc: David Hildenbrand (Arm) Signed-off-by: Andrew Morton --- mm/page_owner.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/mm/page_owner.c b/mm/page_owner.c index 46a933f9c229..2e3880053a34 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -428,6 +428,12 @@ void __folio_copy_owner(struct folio *newfolio, struct folio *old) * to skip less than the full buddy block, but that is acceptable for page owner * iteration purposes. * + * The lockless read of buddy_order_unsafe() can also return a garbage order if + * the page is concurrently allocated and PageBuddy is cleared between the check + * and the read. Clamp the advance at the next MAX_ORDER_NR_PAGES boundary so + * that a bogus order cannot carry @pfn into an unvalidated memory section, + * which would break callers that rely on boundary-aligned pfn_valid() checks. + * * Return: true if the page was skipped (caller should continue its loop), * false if the page is not a buddy page and should be processed normally. */ @@ -439,8 +445,12 @@ static inline bool skip_buddy_pages(unsigned long *pfn, struct page *page) return false; order = buddy_order_unsafe(page); - if (order <= MAX_PAGE_ORDER) - *pfn += (1UL << order) - 1; + if (order <= MAX_PAGE_ORDER) { + unsigned long new_pfn = *pfn + (1UL << order); + unsigned long boundary = ALIGN(*pfn + 1, MAX_ORDER_NR_PAGES); + + *pfn = min(new_pfn, boundary) - 1; + } return true; } From 90f095b816e25c6a9e4446d299bac5007fdcb3df Mon Sep 17 00:00:00 2001 From: Ye Liu Date: Tue, 14 Jul 2026 09:51:08 +0800 Subject: [PATCH 033/501] mm/page_owner: use memcg_data snapshot to avoid TOCTOU in print_page_owner_memcg() print_page_owner_memcg() reads page->memcg_data via READ_ONCE() at the start to guard against tail pages and NULL data. However, it later re-reads page->memcg_data locklessly in two places: 1: page_memcg_check(page) 2: PageMemcgKmem(page) (via folio_memcg_kmem(), which includes VM_BUG_ON assertions for tail pages and MEMCG_DATA_OBJEXTS) If the page is concurrently freed and reallocated as a THP tail page or slab page between these calls, the VM_BUG_ON assertions can trigger on CONFIG_DEBUG_VM=y builds, crashing the kernel. Fix both TOCTOU issues by using the memcg_data snapshot throughout. Link: https://lore.kernel.org/20260714015117.78351-10-ye.liu@linux.dev Fixes: fcf8935832b8 ("mm/page_owner: print memcg information") Signed-off-by: Ye Liu Reported-by: Sashiko Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Brendan Jackman Cc: Johannes Weiner Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Suren Baghdasaryan Cc: David Hildenbrand (Arm) Cc: Signed-off-by: Andrew Morton --- mm/page_owner.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mm/page_owner.c b/mm/page_owner.c index 2e3880053a34..e18512a49e38 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -540,6 +540,7 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, struct page *page) { unsigned long memcg_data; + struct obj_cgroup *objcg; struct mem_cgroup *memcg; bool online; char name[80]; @@ -549,11 +550,14 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, if (!memcg_data || PageTail(page)) goto out_unlock; - if (memcg_data & MEMCG_DATA_OBJEXTS) + if (memcg_data & MEMCG_DATA_OBJEXTS) { ret += scnprintf(kbuf + ret, count - ret, "Slab cache page\n"); + goto out_unlock; + } - memcg = page_memcg_check(page); + objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK); + memcg = objcg ? obj_cgroup_memcg(objcg) : NULL; if (!memcg) goto out_unlock; @@ -561,7 +565,7 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, cgroup_name(memcg->css.cgroup, name, sizeof(name)); ret += scnprintf(kbuf + ret, count - ret, "Charged %sto %smemcg %s\n", - PageMemcgKmem(page) ? "(via objcg) " : "", + (memcg_data & MEMCG_DATA_KMEM) ? "(via objcg) " : "", online ? "" : "offline ", name); out_unlock: From 8fb1ad47dcedfc40ce13dd1a74d6497cf5fff878 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Thu, 18 Jun 2026 17:28:42 +0800 Subject: [PATCH 034/501] mm: mincore: use walk_page_range_vma() in do_mincore() Patch series "mm: convert to walk_page_range_vma() to eliminate find_vma()", v2. walk_page_range() performs a find_vma() lookup on each page table walk. For callers that already hold a valid VMA and operate on a known single-VMA range, this lookup is redundant. Replace walk_page_range() with walk_page_range_vma() where the caller guarantees single-VMA semantics. This patch (of 4): do_mincore() uses walk_page_range() to walk the page table. Fortunately, the caller always passes start/end that falls within a single VMA, so it's safe to use the walk_page_range_vma() in do_mincore() to eliminate an unnecessary find_vma() lookup. Unlike walk_page_range(), walk_page_range_vma() does not call walk_page_test(), which handles VM_PFNMAP by invoking ->pte_hole() to skip the page table walk. Without this check, PFNMAP PTEs would be treated as present by mincore_pte_range(), changing the returned residency status. Handle VM_PFNMAP explicitly in do_mincore() to preserve the original behavior. [akpm@linux-foundation.org: simplify comment, per Pedro] Link: https://lore.kernel.org/ajP9bQhmvR9OX0VE@pedro-suse Link: https://lore.kernel.org/20260618092845.3905740-1-wangkefeng.wang@huawei.com Link: https://lore.kernel.org/20260618092845.3905740-2-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Acked-by: Zi Yan Acked-by: David Hildenbrand (Arm) Reviewed-by: Pedro Falcato Cc: Alistair Popple Cc: Byungchul Park Cc: Gregory Price Cc: Joshua Hahn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Brost Cc: Rakie Kim Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Ying Huang Signed-off-by: Andrew Morton --- mm/mincore.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/mm/mincore.c b/mm/mincore.c index c8757c5085bf..53b982803771 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -258,7 +258,16 @@ static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *v memset(vec, 1, pages); return pages; } - err = walk_page_range(vma->vm_mm, addr, end, &mincore_walk_ops, vec); + + /* + * mincore historically reports PFNMAP mappings as non-resident. + */ + if (vma->vm_flags & VM_PFNMAP) { + __mincore_unmapped_range(addr, end, vma, vec); + return (end - addr) >> PAGE_SHIFT; + } + + err = walk_page_range_vma(vma, addr, end, &mincore_walk_ops, vec); if (err < 0) return err; return (end - addr) >> PAGE_SHIFT; From 334509572d01e483104468e007661ace61436001 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Thu, 18 Jun 2026 17:28:43 +0800 Subject: [PATCH 035/501] mm: mprotect: use walk_page_range_vma() in mprotect_fixup() In mprotect_fixup(), the PROT_NONE PFN permission check uses walk_page_range() to walk the page table. Fortunately, the caller always passes start/end that falls within a single VMA, the do_mprotect_pkey() iterates per-VMA via for_each_vma_range(), and setup_arg_pages() passes the whole VMA. Note, walk_page_test() isn't called in walk_page_range_vma(), however, prot_none_test() in prot_none_walk_ops always return 0, so it's safe to replace walk_page_range() with walk_page_range_vma() to eliminate an unnecessary find_vma() lookup, also remove unneeded prot_none_test() too. Link: https://lore.kernel.org/20260618092845.3905740-3-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Reviewed-by: Zi Yan Reviewed-by: Pedro Falcato Cc: Alistair Popple Cc: Byungchul Park Cc: David Hildenbrand (Arm) Cc: Gregory Price Cc: Joshua Hahn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Brost Cc: Rakie Kim Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Ying Huang Signed-off-by: Andrew Morton --- mm/mprotect.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/mm/mprotect.c b/mm/mprotect.c index c0f5ab74bee2..8665a23f38d3 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -708,16 +708,9 @@ static int prot_none_hugetlb_entry(pte_t *pte, unsigned long hmask, 0 : -EACCES; } -static int prot_none_test(unsigned long addr, unsigned long next, - struct mm_walk *walk) -{ - return 0; -} - static const struct mm_walk_ops prot_none_walk_ops = { .pte_entry = prot_none_pte_entry, .hugetlb_entry = prot_none_hugetlb_entry, - .test_walk = prot_none_test, .walk_lock = PGWALK_WRLOCK, }; @@ -753,7 +746,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, !vma_flags_test_any_mask(&new_vma_flags, VMA_ACCESS_FLAGS)) { pgprot_t new_pgprot = vm_get_page_prot(newflags); - error = walk_page_range(current->mm, start, end, + error = walk_page_range_vma(vma, start, end, &prot_none_walk_ops, &new_pgprot); if (error) return error; From d4c63a378b014f63a0555ad651c14b23fd6e0922 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Thu, 18 Jun 2026 17:28:44 +0800 Subject: [PATCH 036/501] mm: mlock: use walk_page_range_vma() in mlock_vma_pages_range() The mlock_vma_pages_range() uses walk_page_range() to walk the page table. Fortunately, the caller always passes start/end that falls within a single VMA, apply_vma_lock_flags() iterates per-VMA, and apply_mlockall_flags() passes the whole VMA. Since there is no .test_walk in mlock_walk_ops and VM_PFNMAP was filtered by vma_supports_mlock(), it's safe to replace walk_page_range() with walk_page_range_vma() to eliminate an unnecessary find_vma() lookup. Link: https://lore.kernel.org/20260618092845.3905740-4-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Reviewed-by: Zi Yan Acked-by: David Hildenbrand (Arm) Reviewed-by: Pedro Falcato Cc: Alistair Popple Cc: Byungchul Park Cc: Gregory Price Cc: Joshua Hahn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Brost Cc: Rakie Kim Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Ying Huang Signed-off-by: Andrew Morton --- mm/mlock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/mlock.c b/mm/mlock.c index 8c227fefa2df..97e49038d8d3 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -446,7 +446,7 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma, vma_flags_reset_once(vma, new_vma_flags); lru_add_drain(); - walk_page_range(vma->vm_mm, start, end, &mlock_walk_ops, NULL); + walk_page_range_vma(vma, start, end, &mlock_walk_ops, NULL); lru_add_drain(); if (vma_flags_test(new_vma_flags, VMA_IO_BIT)) { From 9bde2240ec350a7eddaa98da0c13c5c72efec671 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Thu, 18 Jun 2026 17:28:45 +0800 Subject: [PATCH 037/501] mm: migrate_device: use walk_page_range_vma() in migrate_vma_collect() migrate_vma_collect() uses walk_page_range() to walk the page table. Fortunately, migrate_vma_setup() already validates that the entire range falls within a single VMA. Since there is no .test_walk in migrate_vma_walk_ops and VM_PFNMAP was filtered by migrate_vma_setup(), it's safe to replace walk_page_range() with walk_page_range_vma() to eliminate an unnecessary find_vma() lookup. Link: https://lore.kernel.org/20260618092845.3905740-5-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Reviewed-by: Zi Yan Acked-by: David Hildenbrand (Arm) Acked-by: Pedro Falcato Cc: Alistair Popple Cc: Byungchul Park Cc: Gregory Price Cc: Joshua Hahn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Brost Cc: Rakie Kim Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Ying Huang Signed-off-by: Andrew Morton --- mm/migrate_device.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 908d2d4ec43a..06a5536e8e63 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -514,7 +514,7 @@ static void migrate_vma_collect(struct migrate_vma *migrate) migrate->pgmap_owner); mmu_notifier_invalidate_range_start(&range); - walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end, + walk_page_range_vma(migrate->vma, migrate->start, migrate->end, &migrate_vma_walk_ops, migrate); mmu_notifier_invalidate_range_end(&range); From 9aaf5aa2444f5731609d0561b1bc183aec547edf Mon Sep 17 00:00:00 2001 From: Shivank Garg Date: Wed, 1 Jul 2026 05:17:20 +0000 Subject: [PATCH 038/501] mm/migrate: rename page to folio leftovers Patch series "mm/migrate: preparatory cleanups for batch copy and offload", v2. This is a small set of mm/migrate cleanups split out of the batch-copy and offload RFC [1], so they can be reviewed and merged independently ahead of that larger series. No functional change intended. This patch (of 3): Rename migrate_folio_undo_src()'s page_was_mapped parameter to was_mapped, unmap_and_move_huge_page() to unmap_and_move_hugetlb_folio(), its page_was_mapped variable to was_mapped and fix stale "page" wording in its comments. Also fix migrate_folio() kerneldoc to say "folio" instead of "page". Link: https://lore.kernel.org/20260701-migrate-cleanups-prep-v2-0-d9e8f17130b1@amd.com Link: https://lore.kernel.org/20260701-migrate-cleanups-prep-v2-1-d9e8f17130b1@amd.com Link: https://lore.kernel.org/all/20260428155043.39251-2-shivankg@amd.com [1] Signed-off-by: Shivank Garg Suggested-by: Dev Jain Suggested-by: David Hildenbrand (Arm) Reviewed-by: Zi Yan Reviewed-by: SJ Park Cc: Alistair Popple Cc: Byungchul Park Cc: Gregory Price Cc: "Huang, Ying" Cc: Joshua Hahn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Brost Cc: Michal Hocko Cc: Mike Rapoport Cc: Rakie Kim Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Jonathan Cameron Signed-off-by: Andrew Morton --- mm/migrate.c | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/mm/migrate.c b/mm/migrate.c index 49e10feeb094..858350ca58b4 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -884,7 +884,7 @@ static int __migrate_folio(struct address_space *mapping, struct folio *dst, * @mapping: The address_space containing the folio. * @dst: The folio to migrate the data to. * @src: The folio containing the current data. - * @mode: How to migrate the page. + * @mode: How to migrate the folio. * * Common logic to directly migrate a single LRU folio suitable for * folios that do not have private data. @@ -1157,13 +1157,10 @@ static void __migrate_folio_extract(struct folio *dst, } /* Restore the source folio to the original state upon failure */ -static void migrate_folio_undo_src(struct folio *src, - int page_was_mapped, - struct anon_vma *anon_vma, - bool locked, - struct list_head *ret) +static void migrate_folio_undo_src(struct folio *src, int was_mapped, + struct anon_vma *anon_vma, bool locked, struct list_head *ret) { - if (page_was_mapped) + if (was_mapped) remove_migration_ptes(src, src, 0); /* Drop an anon_vma reference if we took one */ if (anon_vma) @@ -1449,7 +1446,8 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private, } /* - * Counterpart of unmap_and_move_page() for hugepage migration. + * Counterpart of migrate_folio_unmap() and migrate_folio_move() for hugetlb + * folio migration. * * This function doesn't wait the completion of hugepage I/O * because there is no race between I/O and migration for hugepage. @@ -1466,20 +1464,20 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private, * because then pte is replaced with migration swap entry and direct I/O code * will wait in the page fault for migration to complete. */ -static int unmap_and_move_huge_page(new_folio_t get_new_folio, +static int unmap_and_move_hugetlb_folio(new_folio_t get_new_folio, free_folio_t put_new_folio, unsigned long private, struct folio *src, int force, enum migrate_mode mode, enum migrate_reason reason, struct list_head *ret) { struct folio *dst; int rc = -EAGAIN; - int page_was_mapped = 0; + int was_mapped = 0; struct anon_vma *anon_vma = NULL; struct address_space *mapping = NULL; enum ttu_flags ttu = 0; if (folio_ref_count(src) == 1) { - /* page was freed from under us. So we are done. */ + /* folio was freed from under us. So we are done. */ folio_putback_hugetlb(src); return 0; } @@ -1501,8 +1499,8 @@ static int unmap_and_move_huge_page(new_folio_t get_new_folio, } /* - * Check for pages which are in the process of being freed. Without - * folio_mapping() set, hugetlbfs specific move page routine will not + * Check for folios which are in the process of being freed. Without + * folio_mapping() set, hugetlbfs specific move folio routine will not * be called and we could leak usage counts for subpools. */ if (hugetlb_folio_subpool(src) && !folio_mapping(src)) { @@ -1532,13 +1530,13 @@ static int unmap_and_move_huge_page(new_folio_t get_new_folio, } try_to_migrate(src, ttu); - page_was_mapped = 1; + was_mapped = 1; } if (!folio_mapped(src)) rc = move_to_new_folio(dst, src, mode); - if (page_was_mapped) + if (was_mapped) remove_migration_ptes(src, !rc ? dst : src, ttu); if (ttu & TTU_RMAP_LOCKED) @@ -1663,10 +1661,10 @@ static int migrate_hugetlbs(struct list_head *from, new_folio_t get_new_folio, continue; } - rc = unmap_and_move_huge_page(get_new_folio, - put_new_folio, private, - folio, pass > 2, mode, - reason, ret_folios); + rc = unmap_and_move_hugetlb_folio(get_new_folio, + put_new_folio, private, + folio, pass > 2, mode, + reason, ret_folios); /* * The rules are: * 0: hugetlb folio will be put back From 8d7625771506a34866d594f8b59e32503d2fc4d0 Mon Sep 17 00:00:00 2001 From: Shivank Garg Date: Wed, 1 Jul 2026 05:17:21 +0000 Subject: [PATCH 039/501] mm/migrate: fix stale list name in migrate_folios_move() comment The return-value description in migrate_folios_move() still refers to unmap_folios, but that list no longer exists. Update this name to src_folios. Link: https://lore.kernel.org/20260701-migrate-cleanups-prep-v2-2-d9e8f17130b1@amd.com Signed-off-by: Shivank Garg Reviewed-by: Zi Yan Acked-by: David Hildenbrand (Arm) Cc: Alistair Popple Cc: Byungchul Park Cc: Dev Jain Cc: Gregory Price Cc: "Huang, Ying" Cc: Jonathan Cameron Cc: Joshua Hahn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Brost Cc: Michal Hocko Cc: Mike Rapoport Cc: Rakie Kim Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/migrate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/migrate.c b/mm/migrate.c index 858350ca58b4..5bc81d61116f 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1739,7 +1739,7 @@ static void migrate_folios_move(struct list_head *src_folios, /* * The rules are: * 0: folio will be freed - * -EAGAIN: stay on the unmap_folios list + * -EAGAIN: stay on the src_folios list * Other errno: put on ret_folios list */ switch (rc) { From 3dd5a633a0941eb818d4647e1f904252ca028dc6 Mon Sep 17 00:00:00 2001 From: Shivank Garg Date: Wed, 1 Jul 2026 05:17:22 +0000 Subject: [PATCH 040/501] mm/migrate: use migrate_info field instead of private Add an unsigned long migrate_info member to the struct folio union and use it to store migration state (anon_vma pointer and FOLIO_WAS_* markers) instead of using folio->private. While at it, switch to bitwise OR. No functional change. [rdunlap@infradead.org: add missing kernel-doc for @migrate_info] Link: https://lore.kernel.org/20260717022146.1290242-1-rdunlap@infradead.org Link: https://lore.kernel.org/20260701-migrate-cleanups-prep-v2-3-d9e8f17130b1@amd.com Signed-off-by: Shivank Garg Suggested-by: David Hildenbrand (Arm) Reviewed-by: Jonathan Cameron Acked-by: David Hildenbrand (Arm) Reviewed-by: Huang Ying Acked-by: Zi Yan Reviewed-by: SJ Park Signed-off-by: Randy Dunlap Cc: Alistair Popple Cc: Byungchul Park Cc: Dev Jain Cc: Gregory Price Cc: Joshua Hahn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Brost Cc: Michal Hocko Cc: Mike Rapoport Cc: Rakie Kim Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/mm_types.h | 3 +++ mm/migrate.c | 14 +++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index b18c2b2e7d2c..939b5ea8c9e0 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -368,6 +368,8 @@ typedef unsigned short mm_id_t; * dax_associate_entry. * @private: Filesystem per-folio data (see folio_attach_private()). * @swap: Used for swp_entry_t if folio_test_swapcache(). + * @migrate_info: Stores migration state (anon_vma pointer and + * FOLIO_WAS_* markers). * @_mapcount: Do not access this member directly. Use folio_mapcount() to * find out how many times this folio is mapped by userspace. * @_refcount: Do not access this member directly. Use folio_ref_count() @@ -427,6 +429,7 @@ struct folio { union { void *private; swp_entry_t swap; + unsigned long migrate_info; }; atomic_t _mapcount; atomic_t _refcount; diff --git a/mm/migrate.c b/mm/migrate.c index 5bc81d61116f..6fb6f46eea5f 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1130,7 +1130,7 @@ static int move_to_new_folio(struct folio *dst, struct folio *src, } /* - * To record some information during migration, we use unused private + * To record some information during migration, we use the migrate_info * field of struct folio of the newly allocated destination folio. * This is safe because nobody is using it except us. */ @@ -1143,17 +1143,17 @@ enum { static void __migrate_folio_record(struct folio *dst, int old_folio_state, struct anon_vma *anon_vma) { - dst->private = (void *)anon_vma + old_folio_state; + dst->migrate_info = (unsigned long)anon_vma | old_folio_state; } static void __migrate_folio_extract(struct folio *dst, int *old_folio_state, struct anon_vma **anon_vmap) { - unsigned long private = (unsigned long)dst->private; + unsigned long info = dst->migrate_info; - *anon_vmap = (struct anon_vma *)(private & ~FOLIO_OLD_STATES); - *old_folio_state = private & FOLIO_OLD_STATES; - dst->private = NULL; + *anon_vmap = (struct anon_vma *)(info & ~FOLIO_OLD_STATES); + *old_folio_state = info & FOLIO_OLD_STATES; + dst->migrate_info = 0; } /* Restore the source folio to the original state upon failure */ @@ -1214,7 +1214,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio, return -ENOMEM; *dstp = dst; - dst->private = NULL; + dst->migrate_info = 0; if (!folio_trylock(src)) { if (mode == MIGRATE_ASYNC) From 8570b65220e80f0dafa517edd07fb5e63eff5344 Mon Sep 17 00:00:00 2001 From: Zhen Ni Date: Tue, 7 Jul 2026 19:54:08 +0800 Subject: [PATCH 041/501] mm/page_owner: add print_mode filter Patch series "mm/page_owner: add per-fd filter infrastructure for print_mode and NUMA filtering", v12. This patch series introduces per-file-descriptor filtering capabilities to the page_owner feature. Problem Statement ================= In production environments with large memory configurations (e.g., 250GB+), collecting page_owner information often results in files ranging from several gigabytes to over 10GB. This creates significant challenges: 1. Storage pressure on production systems 2. Difficulty transferring large files from production environments 3. Post-processing overhead with tools/mm/page_owner_sort.c The primary contributor to file size is redundant stack trace information. While the kernel already deduplicates stacks via stackdepot, page_owner retrieves and stores full stack traces for each page, only to deduplicate them again during post-processing. Additionally, in NUMA-aware environments (e.g., DPDK-based cloud deployments where QEMU processes are bound to specific NUMA nodes), OOM events are often node-specific rather than system-wide. Previously, page_owner could not filter by NUMA node, forcing users to collect and analyze data for all nodes. Solution ======== This patch series introduces a per-file-descriptor filter infrastructure with two initial filters: 1. **Print Mode Filter**: Outputs only stack handles instead of full stack traces. The handle-to-stack mapping can be retrieved from the existing show_stacks_handles interface. This dramatically reduces output size while preserving all allocation metadata. 2. **NUMA Node Filter**: Allows filtering pages by specific NUMA node(s) using flexible nodelist format, enabling targeted analysis of memory issues in NUMA-aware deployments. The per-fd design allows multiple concurrent page_owner reads with different filters, solving coordination issues in multi-user production environments. Implementation ============== The series is structured as follows: - Patch 1: Implement print_mode filter infrastructure * Add file->private_data to store per-fd filter state * Add .open, .release, and .write file operations * Support "stack", "handle", and "stack_handle" modes via "mode=" write commands - Patch 2: Implement NUMA node filter infrastructure * Add nid_filter field to per-fd state * Support flexible nodelist format via "nid=" write commands (single, multiple, ranges) * Validate nodes and reject non-existent nodes using nodes_subset() - Patch 3: Add page_owner_filter userspace tool * Manages per-fd filters via write() interface * Provides user-friendly command-line interface * Includes comprehensive input validation - Patch 4: Document filter features and usage Usage Example ============= Using the page_owner_filter tool with per-fd filters: # ./page_owner_filter -m stack_handle -n "0,2-3" -o page_owner.txt The tool opens /sys/kernel/debug/page_owner, sets filters via write(), then reads the filtered output to the specified file (or stdout). Sample print_mode output (showing handles only): Page allocated via order 0, mask 0x0(), pid 0, tgid 0 (swapper), ts 0 ns PFN 0x40000 type Unmovable Block 512 type Unmovable Flags 0x3fffe0000000000(node=0|zone=0|lastcpupid=0x1ffff) handle: 1048577 Page allocated via order 0, mask 0x252000(__GFP_NOWARN| __GFP_NORETRY|__GFP_COMP|__GFP_THISNODE), pid 0, tgid 0 (swapper), ts 0 ns PFN 0x40002 type Unmovable Block 512 type Unmovable Flags 0x23fffe0000000200(workingset|node=0|zone=0|lastcpupid=0x1ffff) handle: 1048577 This patch (of 4): Add a print_mode filter to page_owner that allows users to choose between printing stack traces, stack handles, or both, providing flexibility for different debugging and analysis scenarios. The filter provides three modes via page_owner: - Writing "mode=stack" prints stack traces for each page (default) - Writing "mode=handle" prints only the handle number - Writing "mode=stack_handle" prints both stack traces and handles The default stack mode maintains backward compatibility with existing usage, displaying complete stack traces for each page allocation. The handle mode dramatically reduces log size and improves performance by showing only the handle number instead of the full stack trace. Testing shows handle mode reduces output size by ~66% (84MB vs 244MB) and improves read performance by ~4.4x compared to full stack output. The mapping from handles to actual stack traces can be obtained via the show_stacks_handles interface. The stack_handle mode prints both stack traces and handles, making it easier to identify pages with the same allocation pattern by comparing handle numbers instead of comparing large stack traces. Example usage: # Using the page_owner_filter tool (recommended) ./page_owner_filter -m stack # Print only stack traces (default) ./page_owner_filter -m handle # Print only handles ./page_owner_filter -m stack_handle # Print both stack and handles Sample output (handle mode): Page allocated via order 0, migratetype Unmovable, gfp_mask 0x1100ca, pid 1, tgid 1 (systemd), ts 123456789 ns PFN 0x1000 type Unmovable Block 1 type Unmovable Flags 0x3fffe800000084(referenced|lru|active|private|node=0|zone=1) handle: 17432583 ... This implementation uses per-file-descriptor filter state stored in file->private_data, allowing each opener to have independent filter configuration. Link: https://lore.kernel.org/20260707115411.1714314-1-zhen.ni@easystack.cn Link: https://lore.kernel.org/20260707115411.1714314-2-zhen.ni@easystack.cn Signed-off-by: Zhen Ni Tested-by: Zi Yan Acked-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Brendan Jackman Cc: Johannes Weiner Cc: Michal Hocko Cc: Suren Baghdasaryan Signed-off-by: Andrew Morton --- mm/page_owner.c | 116 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 110 insertions(+), 6 deletions(-) diff --git a/mm/page_owner.c b/mm/page_owner.c index e18512a49e38..5c475e655818 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -54,6 +54,22 @@ struct stack_print_ctx { u8 flags; }; +enum page_owner_print_mode { + PAGE_OWNER_PRINT_STACK, + PAGE_OWNER_PRINT_HANDLE, + PAGE_OWNER_PRINT_STACK_HANDLE, +}; + +static const char * const page_owner_print_mode_strings[] = { + [PAGE_OWNER_PRINT_STACK] = "stack", + [PAGE_OWNER_PRINT_HANDLE] = "handle", + [PAGE_OWNER_PRINT_STACK_HANDLE] = "stack_handle", +}; + +struct page_owner_filter_state { + enum page_owner_print_mode print_mode; +}; + static bool page_owner_enabled __initdata; DEFINE_STATIC_KEY_FALSE(page_owner_inited); @@ -584,16 +600,20 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, static ssize_t print_page_owner(char __user *buf, size_t count, unsigned long pfn, struct page *page, struct page_owner *page_owner, - depot_stack_handle_t handle) + depot_stack_handle_t handle, + struct page_owner_filter_state *state) { int ret, pageblock_mt, page_mt; char *kbuf; + enum page_owner_print_mode print_mode; count = min_t(size_t, count, PAGE_SIZE); kbuf = kmalloc(count, GFP_KERNEL); if (!kbuf) return -ENOMEM; + print_mode = state->print_mode; + ret = scnprintf(kbuf, count, "Page allocated via order %u, mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu ns\n", page_owner->order, page_owner->gfp_mask, @@ -612,9 +632,18 @@ print_page_owner(char __user *buf, size_t count, unsigned long pfn, migratetype_names[pageblock_mt], &page->flags.f); - ret += stack_depot_snprint(handle, kbuf + ret, count - ret, 0); - if (ret >= count) - goto err; + if (print_mode != PAGE_OWNER_PRINT_HANDLE) { + ret += stack_depot_snprint(handle, kbuf + ret, count - ret, 0); + if (ret >= count) + goto err; + } + + if (print_mode != PAGE_OWNER_PRINT_STACK) { + ret += scnprintf(kbuf + ret, count - ret, "handle: %u\n", + handle); + if (ret >= count) + goto err; + } if (page_owner->last_migrate_reason != MR_NEVER) { ret += scnprintf(kbuf + ret, count - ret, @@ -702,6 +731,7 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) struct page_ext *page_ext; struct page_owner *page_owner; depot_stack_handle_t handle; + struct page_owner_filter_state *state = file->private_data; if (!static_branch_unlikely(&page_owner_inited)) return -EINVAL; @@ -779,7 +809,7 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) page_owner_tmp = *page_owner; page_ext_put(page_ext); return print_page_owner(buf, count, pfn, page, - &page_owner_tmp, handle); + &page_owner_tmp, handle, state); ext_put_continue: page_ext_put(page_ext); } @@ -868,7 +898,81 @@ static void init_early_allocated_pages(void) init_pages_in_zone(zone); } +static int page_owner_open(struct inode *inode, struct file *file) +{ + struct page_owner_filter_state *state; + + state = kzalloc_obj(*state); + if (!state) + return -ENOMEM; + + state->print_mode = PAGE_OWNER_PRINT_STACK; + file->private_data = state; + return 0; +} + +static int page_owner_release(struct inode *inode, struct file *file) +{ + kfree(file->private_data); + return 0; +} + +static ssize_t page_owner_write(struct file *file, + const char __user *buf, + size_t count, loff_t *ppos) +{ + char *kbuf; + char *orig; + char *token; + int ret; + struct page_owner_filter_state *state = file->private_data; + enum page_owner_print_mode new_print_mode; + + /* + * Maximum input length for filter commands: + * 32: print_mode command max length is 17 ("mode=stack_handle"). + */ + if (count > 32) + return -EINVAL; + + kbuf = memdup_user_nul(buf, count); + if (IS_ERR(kbuf)) + return PTR_ERR(kbuf); + + orig = kbuf; + + new_print_mode = state->print_mode; + + while ((token = strsep(&kbuf, " \t\n")) != NULL) { + if (*token == '\0') + continue; + + if (!strncmp(token, "mode=", 5)) { + ret = sysfs_match_string(page_owner_print_mode_strings, + token + 5); + if (ret < 0) + goto out_free; + new_print_mode = ret; + } else { + ret = -EINVAL; + goto out_free; + } + } + + state->print_mode = new_print_mode; + + ret = count; + +out_free: + kfree(orig); + return ret; +} + static const struct file_operations page_owner_fops = { + .owner = THIS_MODULE, + .open = page_owner_open, + .release = page_owner_release, + .write = page_owner_write, .read = read_page_owner, .llseek = lseek_page_owner, }; @@ -999,7 +1103,7 @@ static int __init pageowner_init(void) return 0; } - debugfs_create_file("page_owner", 0400, NULL, NULL, &page_owner_fops); + debugfs_create_file("page_owner", 0600, NULL, NULL, &page_owner_fops); dir = debugfs_create_dir("page_owner_stacks", NULL); debugfs_create_file("show_stacks", 0400, dir, (void *)(STACK_PRINT_FLAG_STACK | From bed80036125a18bb3784c59e644a12027a17f4a7 Mon Sep 17 00:00:00 2001 From: Zhen Ni Date: Tue, 7 Jul 2026 19:54:09 +0800 Subject: [PATCH 042/501] mm/page_owner: add NUMA node filter Add NUMA node filtering functionality to page_owner to allow filtering pages by specific NUMA node(s). This is useful for NUMA-aware memory allocation analysis and debugging. The filter supports flexible input formats: - Single node: nid=0 - Multiple nodes: nid=0,2,3 - Node range: nid=0-3 - Mixed format: nid=0,2-4,7 Example usage: # Using the page_owner_filter tool (recommended) ./page_owner_filter -n 0-3 ./page_owner_filter -m stack_handle -n 0,2-4,7 The implementation uses per-file-descriptor filter state stored in file->private_data, allowing each opener to have independent filter configuration. It uses nodemask_t for efficient multi-node filtering and nodelist_parse() for flexible input parsing. Node validity is verified using nodes_subset() to reject nodes without memory. Link: https://lore.kernel.org/20260707115411.1714314-3-zhen.ni@easystack.cn Signed-off-by: Zhen Ni Tested-by: Zi Yan Acked-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Brendan Jackman Cc: Johannes Weiner Cc: Michal Hocko Cc: Suren Baghdasaryan Signed-off-by: Andrew Morton --- mm/page_owner.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/mm/page_owner.c b/mm/page_owner.c index 5c475e655818..7e235ee10930 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -68,6 +68,8 @@ static const char * const page_owner_print_mode_strings[] = { struct page_owner_filter_state { enum page_owner_print_mode print_mode; + nodemask_t nid_filter; + bool nid_filter_enabled; }; static bool page_owner_enabled __initdata; @@ -803,6 +805,21 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) if (!handle) goto ext_put_continue; + if (state->nid_filter_enabled) { + int nid; + memdesc_flags_t page_flags = READ_ONCE(page->flags); + + /* + * Bypass PF_POISONED_CHECK() in page_to_nid() to avoid + * VM_BUG_ON when accessing poisoned pages. + */ + if (page_flags.f == PAGE_POISON_PATTERN) + goto ext_put_continue; + nid = memdesc_nid(page_flags); + if (!node_isset(nid, state->nid_filter)) + goto ext_put_continue; + } + /* Record the next PFN to read in the file offset */ *ppos = pfn + 1; @@ -812,6 +829,7 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) &page_owner_tmp, handle, state); ext_put_continue: page_ext_put(page_ext); + cond_resched(); } return 0; @@ -907,6 +925,8 @@ static int page_owner_open(struct inode *inode, struct file *file) return -ENOMEM; state->print_mode = PAGE_OWNER_PRINT_STACK; + nodes_clear(state->nid_filter); + state->nid_filter_enabled = false; file->private_data = state; return 0; } @@ -927,12 +947,17 @@ static ssize_t page_owner_write(struct file *file, int ret; struct page_owner_filter_state *state = file->private_data; enum page_owner_print_mode new_print_mode; + nodemask_t new_nid_filter; + bool new_nid_filter_enabled; /* * Maximum input length for filter commands: - * 32: print_mode command max length is 17 ("mode=stack_handle"). + * - 32: print_mode command max length is 17 ("mode=stack_handle") + * with sufficient buffer + * - 6 * MAX_NUMNODES: worst case for nid list + * Worst case per node: ",NNNNN" (comma + 5-digit node number) = 6 bytes */ - if (count > 32) + if (count > 32 + 6 * MAX_NUMNODES) return -EINVAL; kbuf = memdup_user_nul(buf, count); @@ -942,6 +967,8 @@ static ssize_t page_owner_write(struct file *file, orig = kbuf; new_print_mode = state->print_mode; + new_nid_filter = state->nid_filter; + new_nid_filter_enabled = state->nid_filter_enabled; while ((token = strsep(&kbuf, " \t\n")) != NULL) { if (*token == '\0') @@ -953,13 +980,36 @@ static ssize_t page_owner_write(struct file *file, if (ret < 0) goto out_free; new_print_mode = ret; + } else if (!strncmp(token, "nid=", 4)) { + ret = nodelist_parse(token + 4, new_nid_filter); + if (ret < 0) + goto out_free; + + if (nodes_empty(new_nid_filter)) { + ret = -EINVAL; + goto out_free; + } + + /* + * We want to filter memory allocations by numa nodes, so make sure + * that the specified nodes have memory. + */ + if (!nodes_subset(new_nid_filter, node_states[N_MEMORY])) { + ret = -EINVAL; + goto out_free; + } + + new_nid_filter_enabled = true; } else { ret = -EINVAL; goto out_free; } } + /* Commit all filter changes */ state->print_mode = new_print_mode; + state->nid_filter = new_nid_filter; + state->nid_filter_enabled = new_nid_filter_enabled; ret = count; From 1279c14f176a6a15ae7e980811e1d81f796d29ca Mon Sep 17 00:00:00 2001 From: Zhen Ni Date: Tue, 7 Jul 2026 19:54:10 +0800 Subject: [PATCH 043/501] tools/mm: add page_owner_filter userspace tool Add a userspace filtering tool for page_owner that supports per-fd filtering with print_mode and NUMA node filters. Features: - Three print modes: stack (default), handle, stack_handle - NUMA node filtering with flexible formats (single: 0, multiple: 0,1,2, range: 0-3, mixed: 0,2-3) - Per-file-descriptor filter state for independent filtering Usage examples: # Filter by print mode ./page_owner_filter -m handle ./page_owner_filter -m stack_handle # Filter by NUMA node ./page_owner_filter -n 0 ./page_owner_filter -n 0-3 # Combined filters ./page_owner_filter -m stack -n 0,1,2 ./page_owner_filter -m handle -n 0,2-3 The tool validates inputs before sending commands to the kernel and provides clear error messages when the kernel does not support per-fd filtering. Link: https://lore.kernel.org/20260707115411.1714314-4-zhen.ni@easystack.cn Signed-off-by: Zhen Ni Tested-by: Zi Yan Acked-by: Vlastimil Babka (SUSE) Acked-by: Zi Yan Cc: Brendan Jackman Cc: Johannes Weiner Cc: Michal Hocko Cc: Suren Baghdasaryan Signed-off-by: Andrew Morton --- tools/mm/Makefile | 4 +- tools/mm/page_owner_filter.c | 310 +++++++++++++++++++++++++++++++++++ 2 files changed, 312 insertions(+), 2 deletions(-) create mode 100644 tools/mm/page_owner_filter.c diff --git a/tools/mm/Makefile b/tools/mm/Makefile index f5725b5c23aa..858186a6eefd 100644 --- a/tools/mm/Makefile +++ b/tools/mm/Makefile @@ -3,7 +3,7 @@ # include ../scripts/Makefile.include -BUILD_TARGETS=page-types slabinfo page_owner_sort thp_swap_allocator_test +BUILD_TARGETS=page-types slabinfo page_owner_sort page_owner_filter thp_swap_allocator_test INSTALL_TARGETS = $(BUILD_TARGETS) thpmaps LIB_DIR = ../lib/api @@ -23,7 +23,7 @@ $(LIBS): $(CC) $(CFLAGS) -o $@ $< $(LDFLAGS) clean: - $(RM) page-types slabinfo page_owner_sort thp_swap_allocator_test + $(RM) page-types slabinfo page_owner_sort page_owner_filter thp_swap_allocator_test make -C $(LIB_DIR) clean sbindir ?= /usr/sbin diff --git a/tools/mm/page_owner_filter.c b/tools/mm/page_owner_filter.c new file mode 100644 index 000000000000..1d1f0a38678a --- /dev/null +++ b/tools/mm/page_owner_filter.c @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * User-space helper to filter page_owner output per-fd + * + * Example use: + * ./page_owner_filter -m handle + * ./page_owner_filter -m stack_handle + * ./page_owner_filter -n 0,1,2 + * + * See Documentation/mm/page_owner.rst + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MAX_CMD_LEN 512 + +static void usage(const char *prog) +{ + fprintf(stderr, "Usage: %s [OPTIONS]\n", prog); + fprintf(stderr, "\nOptions:\n"); + fprintf(stderr, " -m, --mode MODE : print_mode (stack, handle, or stack_handle)\n"); + fprintf(stderr, " -n, --nid NID_LIST : NUMA node IDs (comma-separated or ranges)\n"); + fprintf(stderr, " -o, --output FILE : output file (default: stdout)\n"); + fprintf(stderr, " -h, --help : show this help message\n"); + fprintf(stderr, "\nExamples:\n"); + fprintf(stderr, " %s -m stack\n", prog); + fprintf(stderr, " %s -m handle\n", prog); + fprintf(stderr, " %s -m stack_handle\n", prog); + fprintf(stderr, " %s -m stack -o output.txt\n", prog); + fprintf(stderr, " %s -n 0,1,2\n", prog); + fprintf(stderr, " %s -m stack -n 0\n", prog); +} + +static int validate_mode(const char *mode) +{ + if (strcmp(mode, "stack") == 0 || + strcmp(mode, "handle") == 0 || + strcmp(mode, "stack_handle") == 0) + return 0; + + fprintf(stderr, "Error: Invalid mode '%s'\n", mode); + fprintf(stderr, "Valid modes: stack, handle, stack_handle\n"); + return -1; +} + +static int validate_nid_list(const char *nid_list) +{ + const char *p; + int i = 0; + int has_digit = 0; + int in_range = 0; + int prev_num = 0; + int curr_num = 0; + + if (!nid_list || strlen(nid_list) == 0) + return 0; + + for (p = nid_list; *p; p++) { + if (*p == ',') { + if (!has_digit) { + fprintf(stderr, "Error: Invalid nid_list format\n"); + return -1; + } + if (in_range && prev_num > curr_num) { + fprintf(stderr, + "Error: Invalid range %d-%d (start must be <= end)\n", + prev_num, curr_num); + return -1; + } + i = 0; + has_digit = 0; + in_range = 0; + prev_num = 0; + curr_num = 0; + continue; + } + + if (*p == '-') { + if (!has_digit) { + fprintf(stderr, + "Error: Invalid nid_list format "); + fprintf(stderr, + "(dash without preceding number)\n"); + return -1; + } + if (in_range) { + fprintf(stderr, "Error: Multiple dashes in nid_list\n"); + return -1; + } + prev_num = curr_num; + curr_num = 0; + i = 0; + has_digit = 0; + in_range = 1; + continue; + } + + if (!isdigit((unsigned char)*p)) { + fprintf(stderr, "Error: Invalid character '%c' in nid_list\n", *p); + return -1; + } + + if (i > 5) { + fprintf(stderr, "Error: NID too long (max 65536)\n"); + return -1; + } + curr_num = curr_num * 10 + (*p - '0'); + i++; + has_digit = 1; + } + + if (!has_digit) { + fprintf(stderr, "Error: Invalid nid_list format\n"); + return -1; + } + + if (in_range && prev_num > curr_num) { + fprintf(stderr, + "Error: Invalid range %d-%d (start must be <= end)\n", + prev_num, curr_num); + return -1; + } + + return 0; +} + +int main(int argc, char *argv[]) +{ + const char *output_file = NULL; + char filter_cmd[MAX_CMD_LEN]; + FILE *output = NULL; + int fd = -1; + ssize_t ret; + char buf[4096]; + int opt; + size_t cmd_len = 0; + + signal(SIGPIPE, SIG_IGN); + + static struct option long_options[] = { + {"mode", required_argument, 0, 'm'}, + {"nid", required_argument, 0, 'n'}, + {"output", required_argument, 0, 'o'}, + {"help", no_argument, 0, 'h'}, + {0, 0, 0, 0} + }; + + filter_cmd[0] = '\0'; + + if (argc > 1) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + usage(argv[0]); + return 0; + } + } + } + + /* Check if page_owner exists and is readable */ + if (access("/sys/kernel/debug/page_owner", F_OK) != 0) { + if (errno == ENOENT) + fprintf(stderr, "Error: /sys/kernel/debug/page_owner does not exist\n"); + else + perror("Error accessing /sys/kernel/debug/page_owner"); + fprintf(stderr, "Make sure page_owner is enabled in kernel\n"); + return 1; + } + + while ((opt = getopt_long(argc, argv, "m:n:o:h", long_options, NULL)) != -1) { + int len; + + switch (opt) { + case 'm': { + const char *mode = optarg; + + if (validate_mode(mode) < 0) + return 1; + len = snprintf(filter_cmd + cmd_len, MAX_CMD_LEN - cmd_len, + "%smode=%s", cmd_len > 0 ? " " : "", mode); + if (len < 0 || cmd_len + len >= MAX_CMD_LEN) { + fprintf(stderr, "Error: Command too long\n"); + return 1; + } + cmd_len += len; + break; + } + case 'n': { + const char *nid_list = optarg; + + if (validate_nid_list(nid_list) < 0) + return 1; + len = snprintf(filter_cmd + cmd_len, MAX_CMD_LEN - cmd_len, + "%snid=%s", cmd_len > 0 ? " " : "", nid_list); + if (len < 0 || cmd_len + len >= MAX_CMD_LEN) { + fprintf(stderr, "Error: Command too long\n"); + return 1; + } + cmd_len += len; + break; + } + case 'o': + output_file = optarg; + break; + case 'h': + /* Already handled above */ + break; + default: + usage(argv[0]); + return 1; + } + } + + /* At least one filter must be specified */ + if (cmd_len == 0) { + fprintf(stderr, "Error: At least one filter (-m or -n) must be specified\n\n"); + usage(argv[0]); + return 1; + } + + /* Open page_owner for read-write - this will fail if kernel doesn't support write */ + fd = open("/sys/kernel/debug/page_owner", O_RDWR); + if (fd < 0) { + if (errno == EACCES || errno == EPERM) { + fprintf(stderr, "Error: /sys/kernel/debug/page_owner "); + fprintf(stderr, "does not support write access\n"); + fprintf(stderr, "This kernel does not support "); + fprintf(stderr, "per-fd filtering.\n"); + fprintf(stderr, "Please ensure you have a kernel with "); + fprintf(stderr, "per-fd filtering support.\n"); + } else { + perror("Error opening /sys/kernel/debug/page_owner"); + } + return 1; + } + + if (output_file) { + output = fopen(output_file, "w"); + if (!output) { + perror("open output file"); + close(fd); + return 1; + } + } else { + output = stdout; + } + + ret = write(fd, filter_cmd, strlen(filter_cmd)); + + if (ret < 0) { + if (errno == EINVAL) { + fprintf(stderr, "Error: Kernel rejected the filter command.\n"); + fprintf(stderr, "Possible causes:\n"); + fprintf(stderr, " - Kernel does not support per-fd filtering\n"); + fprintf(stderr, " - NUMA node has no memory\n"); + fprintf(stderr, " - Unknown reason\n"); + } else { + perror("write filter command"); + } + goto out; + } + + if ((size_t)ret != strlen(filter_cmd)) + fprintf(stderr, "Warning: Partial write (%zd/%zu)\n", ret, strlen(filter_cmd)); + + /* Read and display filtered output */ + ret = 0; + while ((ret = read(fd, buf, sizeof(buf))) > 0) { + size_t written = fwrite(buf, 1, ret, output); + + if (written != (size_t)ret) { + if (errno == EPIPE) { + /* Pipe closed, treat as success */ + ret = 0; + goto out; + } + perror("write output"); + ret = -1; + goto out; + } + } + + if (ret < 0) { + perror("read page_owner"); + goto out; + } + + if (fflush(output)) { + if (errno == EPIPE) { + /* Pipe closed, treat as success */ + ret = 0; + } else { + perror("flush output"); + ret = -1; + } + } + +out: + close(fd); + if (output != stdout) + fclose(output); + return ret < 0 ? 1 : 0; +} From e2c75342c24f7420e4011a6745ba0f85c4fc047d Mon Sep 17 00:00:00 2001 From: Zhen Ni Date: Tue, 7 Jul 2026 19:54:11 +0800 Subject: [PATCH 044/501] mm/page_owner: document page_owner filter Add documentation for the page_owner_filter userspace tool and kernel-level filtering features. [rdunlap@infradead.org: avoid docs build warning] Link: https://lore.kernel.org/20260708213548.469155-1-rdunlap@infradead.org Link: https://lore.kernel.org/20260707115411.1714314-5-zhen.ni@easystack.cn Signed-off-by: Zhen Ni Signed-off-by: Randy Dunlap Acked-by: Vlastimil Babka (SUSE) Acked-by: Zi Yan Cc: Brendan Jackman Cc: Johannes Weiner Cc: Michal Hocko Cc: Suren Baghdasaryan Signed-off-by: Andrew Morton --- Documentation/mm/page_owner.rst | 78 ++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/Documentation/mm/page_owner.rst b/Documentation/mm/page_owner.rst index 6b12f3b007ec..a6bd3fe6423a 100644 --- a/Documentation/mm/page_owner.rst +++ b/Documentation/mm/page_owner.rst @@ -65,7 +65,15 @@ un-tracking state. Usage ===== -1) Build user-space helper:: +1) Build user-space helpers: +:: + + To filter page_owner output: + + cd tools/mm + make page_owner_filter + + To sort and analyze page_owner output: cd tools/mm make page_owner_sort @@ -74,7 +82,11 @@ Usage 3) Do the job that you want to debug. -4) Analyze information from page owner:: +4) (Optional) Filter page_owner output:: + + ./page_owner_filter -m handle -n 0,1,2 > filtered_page_owner.txt + +5) Analyze information from page owner:: cat /sys/kernel/debug/page_owner_stacks/show_stacks > stacks.txt cat stacks.txt @@ -263,3 +275,65 @@ STANDARD FORMAT SPECIFIERS f free whether the page has been released or not st stacktrace stack trace of the page allocation ator allocator memory allocator for pages + +Filtering page_owner output +============================ + +page_owner supports filtering output at the kernel level before reading, +which reduces the amount of data that needs to be processed in userspace. + +The page_owner_filter tool provides a convenient interface for this filtering +capability. It supports two types of filters: + +1. **print_mode filter**: Control what information is printed for each page + - ``stack``: Print full stack traces (default, compatible with existing usage) + - ``handle``: Print only stack handle numbers (much faster, smaller output) + - ``stack_handle``: Print both stack traces and handle numbers + + The ``handle`` mode uses numeric identifiers instead of full stack traces. + The mapping from handles to actual stack traces can be obtained via the + show_stacks_handles interface. + +2. **NUMA node filter**: Filter pages by NUMA node ID + - Supports single node: ``-n 0`` + - Multiple nodes: ``-n 0,1,2`` + - Ranges: ``-n 0-3`` + - Mixed format: ``-n 0,2-3,5`` + +Usage examples:: + + # Filter by print mode + ./page_owner_filter -m handle + ./page_owner_filter -m stack_handle + + # Filter by NUMA node + ./page_owner_filter -n 0 + ./page_owner_filter -n 0-3 + + # Combined filters + ./page_owner_filter -m stack -n 0,1,2 + ./page_owner_filter -m handle -n 0,2-3 + + # Save to file + ./page_owner_filter -m handle -o filtered_output.txt + +The handle mode is particularly useful for monitoring and performance-critical +scenarios as it dramatically reduces output size. Testing shows handle mode can +reduce output size by ~66% (84MB vs 244MB) and improve read performance by ~4.4x +compared to full stack output. + +The NUMA node filter is useful for NUMA-aware memory allocation analysis and debugging. + +Behind the scenes, page_owner_filter opens /sys/kernel/debug/page_owner and +writes filter commands before reading the filtered output. The filtering uses +per-file-descriptor state, allowing each open() to have independent filter settings. + +Each file descriptor maintains its own filter state, so you can have multiple +independent filtering operations running concurrently. For example, in different +terminals you can run different filters simultaneously:: + + # Terminal 1: Filter node 0 + ./page_owner_filter -n 0 > node0_output.txt + + # Terminal 2: Filter node 1 (runs concurrently) + ./page_owner_filter -n 1 > node1_output.txt From eafe0dcdc0441e32cb02a6850859116db6afb6b8 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 15 Jun 2026 17:22:41 +0100 Subject: [PATCH 045/501] mm: add writeback.h to docs build There's four functions in this header file with kernel-doc; add them to the htmldocs. Link: https://lore.kernel.org/20260615162244.2170866-1-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Cc: Andreas Gruenbacher Signed-off-by: Andrew Morton --- Documentation/core-api/mm-api.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/core-api/mm-api.rst b/Documentation/core-api/mm-api.rst index aabdd3cba58e..4df7d5edbee5 100644 --- a/Documentation/core-api/mm-api.rst +++ b/Documentation/core-api/mm-api.rst @@ -73,6 +73,7 @@ Readahead Writeback --------- +.. kernel-doc:: include/linux/writeback.h .. kernel-doc:: mm/page-writeback.c :export: From 375c63b649584b94756048b7656315a637c805b5 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Mon, 15 Jun 2026 17:22:42 +0100 Subject: [PATCH 046/501] writeback.h: fix a typo in the wbc_init_bio() description initializtion -> initialization (missing "a") Link: https://lore.kernel.org/20260615162244.2170866-2-willy@infradead.org Signed-off-by: Andreas Gruenbacher Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/writeback.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/writeback.h b/include/linux/writeback.h index 62552a2ce5b9..b749a9a5a5ee 100644 --- a/include/linux/writeback.h +++ b/include/linux/writeback.h @@ -244,7 +244,7 @@ void wbc_attach_fdatawrite_inode(struct writeback_control *wbc, struct inode *inode); /** - * wbc_init_bio - writeback specific initializtion of bio + * wbc_init_bio - writeback specific initialization of bio * @wbc: writeback_control for the writeback in progress * @bio: bio to be initialized * From 52b71fe00058e148f1d8e48a5fc2ce136a25992c Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Mon, 15 Jun 2026 10:54:34 +0000 Subject: [PATCH 047/501] mm/page_alloc: drop flag-conversion "optimisation" This code uses flag equivalences to try to optimise conversion from GFP_ to ALLOC_ but there's no clear reason to believe it makes things faster. Even if it gets rid of conditional branches, it just trades them for a data dependency. CPUs are pretty good at conditional branches. But, in my GCC x86 build it doesn't look like there are any branches anyway, the compiler found some conditional instruction tricks. (Caveat: This was extracted & annotated by Gemini AI, I did not actually read the disasm myself) Old code: ae50: 8b 04 24 mov (%rsp),%eax # Load gfp_mask ... ae5d: 41 89 c4 mov %eax,%r12d ae64: 41 81 e4 20 08 00 00 and $0x820,%r12d # Mask both flags at once ... ae6f: 44 89 e1 mov %r12d,%ecx ae77: 83 c9 40 or $0x40,%ecx # OR with ALLOC_CPUSET (0x40) ae7a: 89 4c 24 60 mov %ecx,0x60(%rsp) # Store to alloc_flags New code: For __GFP_HIGH ( 0x20 ): It uses the Carry Flag (via sbb ) to conditionally add 0x20 to the base 0x40 ( ALLOC_CPUSET ) flag: ae63: 83 e0 20 and $0x20,%eax # Test __GFP_HIGH ... ae6a: 83 f8 01 cmp $0x1,%eax # Set carry flag if 0 ae6f: 45 19 e4 sbb %r12d,%r12d # %r12d = (gfp & 0x20) ? 0 : -1 ae80: 41 83 e4 e0 and $0xffffffe0,%r12d # %r12d = (gfp & 0x20) ? 0 : -32 ae87: 41 83 c4 60 add $0x60,%r12d # %r12d = (gfp & 0x20) ? 0x60 : 0x40 For __GFP_KSWAPD_RECLAIM ( 0x800 ): It uses a conditional move ( cmov ) later in the function to set the ALLOC_KSWAPD ( 0x800 ) bit: ae72: 25 00 08 00 00 and $0x800,%eax # Test __GFP_KSWAPD_RECLAIM ae77: 89 44 24 30 mov %eax,0x30(%rsp) # Store result ... af2c: 80 cf 08 or $0x8,%bh # Set ALLOC_KSWAPD (0x800) in temp reg af2f: 45 85 c9 test %r9d,%r9d # Check if __GFP_KSWAPD_RECLAIM was set af32: 0f 44 d8 cmove %eax,%ebx # If not, revert to flags without it Testing with a modified version[0] of lib/free_pages_test.c (adding printks with timing)... Old results from a Sapphire Rapids consumer CPU: [ 67.157118] page_alloc_test: Testing with GFP_KERNEL [ 67.157122] page_alloc_test: Starting 1,000,000 allocations... [ 70.704446] page_alloc_test: Completed. Time: 3543002 us (Avg: 3543.00 ns per alloc+free loop) [ 70.704456] page_alloc_test: Testing with GFP_KERNEL | __GFP_COMP [ 70.704460] page_alloc_test: Starting 1,000,000 allocations... [ 70.944672] page_alloc_test: Completed. Time: 239980 us (Avg: 239.98 ns per alloc+free loop) [ 70.944675] page_alloc_test: Test completed New results: [ 70.079015] page_alloc_test: Testing with GFP_KERNEL [ 70.079020] page_alloc_test: Starting 1,000,000 allocations... [ 73.669396] page_alloc_test: Completed. Time: 3586954 us (Avg: 3586.95 ns per alloc+free loop) [ 73.669402] page_alloc_test: Testing with GFP_KERNEL | __GFP_COMP [ 73.669405] page_alloc_test: Starting 1,000,000 allocations... [ 73.905084] page_alloc_test: Completed. Time: 235496 us (Avg: 235.49 ns per alloc+free loop) [ 73.905086] page_alloc_test: Test completed Seems like a wash. So, drop the flag value coupling here and let the compiler and CPU do their job. Superscalar CPUs are pretty neat after all. (Used AI for the disasm but the rest is all manual). Link: https://lore.kernel.org/20260629-gfp-pessimisation-v2-1-311ece6a8637@google.com Link: https://lore.kernel.org/20260615-gfp-pessimisation-v2-1-65f1319e6818@google.com Link: https://github.com/bjackman/aethelred/blob/2ccdc84ef087c2a631914f58e106e99e19bd3b98/page-alloc-test/page-alloc-test.c [1] Signed-off-by: Brendan Jackman Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Gregory Price Acked-by: Johannes Weiner Acked-by: Harry Yoo (Oracle) Signed-off-by: Andrew Morton --- mm/page_alloc.c | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index dbe632f6300d..a460811eaa4d 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -3741,13 +3741,10 @@ static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) static inline unsigned int alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) { - unsigned int alloc_flags; + unsigned int alloc_flags = 0; - /* - * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD - * to save a branch. - */ - alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM); + if (gfp_mask & __GFP_KSWAPD_RECLAIM) + alloc_flags |= ALLOC_KSWAPD; if (defrag_mode) { alloc_flags |= ALLOC_NOFRAGMENT; @@ -4480,22 +4477,16 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) { unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; - /* - * __GFP_HIGH is assumed to be the same as ALLOC_MIN_RESERVE - * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD - * to save two branches. - */ - BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_MIN_RESERVE); - BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD); - /* * The caller may dip into page reserves a bit more if the caller * cannot run direct reclaim, or if the caller has realtime scheduling * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will * set both ALLOC_NON_BLOCK and ALLOC_MIN_RESERVE(__GFP_HIGH). */ - alloc_flags |= (__force int) - (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM)); + if (gfp_mask & __GFP_HIGH) + alloc_flags |= ALLOC_MIN_RESERVE; + if (gfp_mask & __GFP_KSWAPD_RECLAIM) + alloc_flags |= ALLOC_KSWAPD; if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) { /* From b55cc244d04f38f415b540795cd6974afa9847db Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 11 Dec 2024 20:40:14 +0000 Subject: [PATCH 048/501] percpu_ref: fix documentation of maximum value Tejun changd percpu_ref to use long instead of int back in 2014 but missed updating this bit of the documentation. Also add the documentation to the htmldocs. Link: https://lore.kernel.org/20241211204017.184512-1-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Acked-by: Tejun Heo Signed-off-by: Andrew Morton --- Documentation/driver-api/basics.rst | 3 +++ include/linux/percpu-refcount.h | 5 +++-- lib/percpu-refcount.c | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Documentation/driver-api/basics.rst b/Documentation/driver-api/basics.rst index 8b6a5888cb11..3b182cfdf135 100644 --- a/Documentation/driver-api/basics.rst +++ b/Documentation/driver-api/basics.rst @@ -90,6 +90,9 @@ Reference counting .. kernel-doc:: lib/refcount.c :export: +.. kernel-doc:: include/linux/percpu-refcount.h +.. kernel-doc:: lib/percpu-refcount.c + Atomics ------- diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h index d73a1c08c3e3..1e3212e2c827 100644 --- a/include/linux/percpu-refcount.h +++ b/include/linux/percpu-refcount.h @@ -12,8 +12,8 @@ * start shutting down you call percpu_ref_kill() _before_ dropping the initial * refcount. * - * The refcount will have a range of 0 to ((1U << 31) - 1), i.e. one bit less - * than an atomic_t - this is because of the way shutdown works, see + * The refcount will have a range of 0 to LONG_MAX, i.e. one bit less + * than an atomic_long_t - this is because of the way shutdown works, see * percpu_ref_kill()/PERCPU_COUNT_BIAS. * * Before you call percpu_ref_kill(), percpu_ref_put() does not check for the @@ -269,6 +269,7 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref) /** * percpu_ref_tryget_live_rcu - same as percpu_ref_tryget_live() but the * caller is responsible for taking RCU. + * @ref: percpu_ref to try-get * * This function is safe to call as long as @ref is between init and exit. */ diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c index 97772e42b9b2..f8d90689af9f 100644 --- a/lib/percpu-refcount.c +++ b/lib/percpu-refcount.c @@ -289,7 +289,7 @@ static void __percpu_ref_switch_mode(struct percpu_ref *ref, * @confirm_switch: optional confirmation callback * * There's no reason to use this function for the usual reference counting. - * Use percpu_ref_kill[_and_confirm](). + * Use percpu_ref_kill() or percpu_ref_kill_and_confirm(). * * Schedule switching of @ref to atomic mode. All its percpu counts will * be collected to the main atomic counter. On completion, when all CPUs From c0caeceb0c3899dc42844d3979093b27d1434108 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 12 Jun 2026 11:58:45 +0800 Subject: [PATCH 049/501] mm/hugetlb: fix boot panic with CONFIG_DEBUG_VM and HVO bootmem pages Patch series "mm: Refactor bootmem gigantic hugepage allocation", v4. This series is split out from the earlier larger series "mm: Generalize HVO for HugeTLB and device DAX" [1]. It collects the first 19 patches of that series as a standalone set of fixes and preparatory cleanups around bootmem HugeTLB handling, sparse initialization ordering, and related vmemmap setup. The first patches fix a few bugs found while reviewing the existing code, including incorrect bootmem HVO handling, wrong vmemmap registration arguments, a powerpc compound-vmemmap tracking bug, and too-late initialization of gigantic bootmem HugeTLB struct pages. The rest of the series reorders early memory initialization so the relevant zone state is available before sparse and HugeTLB boot-time setup runs, then simplifies the remaining bootmem gigantic hugepage allocation path and removes code made obsolete by that rework. At a high level: - patches [1-4] fix boot-time and arch-specific bugs - patches [5-12] reorder and simplify sparse/mm/hugetlb early init - patches [13-19] refactor bootmem gigantic hugepage allocation and remove obsolete helpers and state This patch (of 19): Commit 622026e87c40 ("mm/hugetlb: remove fake head pages") switched HVO to reuse per-zone shared tail pages from zone->vmemmap_tails[]. Those shared tail pages were initialized in hugetlb_vmemmap_init(), but bootmem HugeTLB folios are prepared earlier from gather_bootmem_prealloc(). With hugetlb_free_vmemmap=on, prep_and_add_bootmem_folios() can access pageblock flags on bootmem HugeTLB pages whose mirrored tail struct pages already point to the shared tail page. On CONFIG_DEBUG_VM kernels, get_pfnblock_bitmap_bitidx() then dereferences the still-uninitialized shared tail page and can panic during boot. Initialize zone->vmemmap_tails[] from gather_bootmem_prealloc(), before bootmem HugeTLB folios are processed, and drop the later initialization from hugetlb_vmemmap_init(). This bug only affects CONFIG_DEBUG_VM kernels, where the relevant assertion is evaluated. Link: https://lore.kernel.org/20260612035903.2468601-1-songmuchun@bytedance.com Link: https://lore.kernel.org/20260612035903.2468601-2-songmuchun@bytedance.com Fixes: 622026e87c40 ("mm/hugetlb: remove fake head pages") Signed-off-by: Muchun Song Acked-by: Oscar Salvador Tested-by: Michal Clapinski Reviewed-by: Michal Clapinski Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Mike Rapoport Cc: Nicholas Piggin Cc: "Ritesh Harjani (IBM)" Cc: Vlastimil Babka Cc: Frank van der Linden Cc: Oscar Salvador (SUSE) Cc: Usama Arif Cc: Signed-off-by: Andrew Morton --- mm/hugetlb.c | 25 +++++++++++++++++++++++++ mm/hugetlb_vmemmap.c | 17 ----------------- mm/sparse-vmemmap.c | 2 +- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index b75774455d65..4c234611e7a0 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3365,6 +3365,31 @@ static void __init gather_bootmem_prealloc(void) .max_threads = num_node_state(N_MEMORY), .numa_aware = true, }; +#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP + struct zone *zone; + + for_each_zone(zone) { + for (int i = 0; i < NR_VMEMMAP_TAILS; i++) { + struct page *tail, *p; + unsigned int order; + + tail = zone->vmemmap_tails[i]; + if (!tail) + continue; + + order = i + VMEMMAP_TAIL_MIN_ORDER; + p = page_to_virt(tail); + /* + * prep_and_add_bootmem_folios() can access pageblock + * flags on bootmem HugeTLB pages, so initialize the + * shared tail struct pages here before bootmem folios + * start using them. + */ + for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++) + init_compound_tail(p + j, NULL, order, zone); + } + } +#endif padata_do_multithreaded(&job); } diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index 133b46dfb09f..c713c0d2593a 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -870,27 +870,10 @@ static const struct ctl_table hugetlb_vmemmap_sysctls[] = { static int __init hugetlb_vmemmap_init(void) { const struct hstate *h; - struct zone *zone; /* HUGETLB_VMEMMAP_RESERVE_SIZE should cover all used struct pages */ BUILD_BUG_ON(__NR_USED_SUBPAGE > HUGETLB_VMEMMAP_RESERVE_PAGES); - for_each_zone(zone) { - for (int i = 0; i < NR_VMEMMAP_TAILS; i++) { - struct page *tail, *p; - unsigned int order; - - tail = zone->vmemmap_tails[i]; - if (!tail) - continue; - - order = i + VMEMMAP_TAIL_MIN_ORDER; - p = page_to_virt(tail); - for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++) - init_compound_tail(p + j, NULL, order, zone); - } - } - for_each_hstate(h) { if (hugetlb_vmemmap_optimizable(h)) { register_sysctl_init("vm", hugetlb_vmemmap_sysctls); diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index ebd3ac997f64..dff8da23eabb 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -342,7 +342,7 @@ static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone * * * Any initialization done here will be overwritten by memmap_init(). * - * hugetlb_vmemmap_init() will take care of initialization after + * gather_bootmem_prealloc() will take care of initialization after * memmap_init(). */ From 6e25c6a9c7eff141ecfc966ecdb7801db2a8810e Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 12 Jun 2026 11:58:46 +0800 Subject: [PATCH 050/501] mm/hugetlb_vmemmap: fix __hugetlb_vmemmap_optimize_folios() __hugetlb_vmemmap_optimize_folios() uses incorrect arguments when handling bootmem HugeTLB folios. The section number passed to register_page_bootmem_memmap() is derived from the vmemmap virtual address of folio->page instead of the folio PFN, so the bootmem memmap metadata can be registered against the wrong section. The helper is also given HUGETLB_VMEMMAP_RESERVE_SIZE even though it expects a page count, not a size in bytes. In addition, the write-protect range is based on pages_per_huge_page(h), which does not cover the full HugeTLB vmemmap area and can leave part of the shared tail vmemmap mapping writable. Fix the section lookup to use folio_pfn(folio), use HUGETLB_VMEMMAP_RESERVE_PAGES when registering the reserved memmap pages, and use hugetlb_vmemmap_size(h) for the write-protect range. Link: https://lore.kernel.org/20260612035903.2468601-3-songmuchun@bytedance.com Fixes: 752fe17af693 ("mm/hugetlb: add pre-HVO framework") Signed-off-by: Muchun Song Acked-by: Oscar Salvador Reviewed-by: Frank van der Linden Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Mike Rapoport (Microsoft) Cc: Nicholas Piggin Cc: Oscar Salvador (SUSE) Cc: "Ritesh Harjani (IBM)" Cc: Usama Arif Cc: Vlastimil Babka Cc: Signed-off-by: Andrew Morton --- mm/hugetlb_vmemmap.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index c713c0d2593a..ea6af85bfec1 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -635,12 +635,12 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h, * mirrored tail page structs RO. */ spfn = (unsigned long)&folio->page; - epfn = spfn + pages_per_huge_page(h); + epfn = spfn + hugetlb_vmemmap_size(h); vmemmap_wrprotect_hvo(spfn, epfn, folio_nid(folio), HUGETLB_VMEMMAP_RESERVE_SIZE); - register_page_bootmem_memmap(pfn_to_section_nr(spfn), + register_page_bootmem_memmap(pfn_to_section_nr(folio_pfn(folio)), &folio->page, - HUGETLB_VMEMMAP_RESERVE_SIZE); + HUGETLB_VMEMMAP_RESERVE_PAGES); continue; } From 89a4ae32764172468dea303eb6ae90fe6c859712 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 12 Jun 2026 11:58:47 +0800 Subject: [PATCH 051/501] powerpc/mm: fix wrong addr_pfn tracking in compound vmemmap population vmemmap_populate_compound_pages() uses addr_pfn to determine the PFN offset within a compound page and to decide whether the current vmemmap slot should be populated as a head page mapping or should reuse a tail page mapping. However, addr_pfn is advanced manually in parallel with addr. The loop itself progresses in vmemmap address space, so each PAGE_SIZE step in addr covers PAGE_SIZE / sizeof(struct page) struct page slots. Since addr_pfn is compared against nr_pages in data-PFN units, it should advance by the same number of PFNs. The existing manual increments do not match that and therefore do not reliably track the PFN corresponding to the current addr. As a result, pfn_offset can be computed from the wrong PFN and the code can make the head/tail decision for the wrong compound-page position. Fix this by deriving addr_pfn directly from the current vmemmap address instead of carrying it as loop state. Link: https://lore.kernel.org/20260612035903.2468601-4-songmuchun@bytedance.com Fixes: f2b79c0d7968 ("powerpc/book3s64/radix: add support for vmemmap optimization for radix") Signed-off-by: Muchun Song Acked-by: Oscar Salvador Reviewed-by: Ritesh Harjani (IBM) Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Frank van der Linden Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Mike Rapoport (Microsoft) Cc: Nicholas Piggin Cc: Oscar Salvador (SUSE) Cc: Usama Arif Cc: Vlastimil Babka Cc: Signed-off-by: Andrew Morton --- arch/powerpc/mm/book3s64/radix_pgtable.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c index 10aced261cff..cf692b2b5f7b 100644 --- a/arch/powerpc/mm/book3s64/radix_pgtable.c +++ b/arch/powerpc/mm/book3s64/radix_pgtable.c @@ -1314,7 +1314,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, * covering out both edges. */ unsigned long addr; - unsigned long addr_pfn = start_pfn; unsigned long next; pgd_t *pgd; p4d_t *p4d; @@ -1335,7 +1334,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, if (pmd_leaf(READ_ONCE(*pmd))) { /* existing huge mapping. Skip the range */ - addr_pfn += (PMD_SIZE >> PAGE_SHIFT); next = pmd_addr_end(addr, end); continue; } @@ -1348,11 +1346,11 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, * page whose VMEMMAP_RESERVE_NR pages were mapped and * this request fall in those pages. */ - addr_pfn += 1; next = addr + PAGE_SIZE; continue; } else { unsigned long nr_pages = pgmap_vmemmap_nr(pgmap); + unsigned long addr_pfn = page_to_pfn((struct page *)addr); unsigned long pfn_offset = addr_pfn - ALIGN_DOWN(addr_pfn, nr_pages); pte_t *tail_page_pte; @@ -1376,7 +1374,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, if (!pte) return -ENOMEM; - addr_pfn += 2; next = addr + 2 * PAGE_SIZE; continue; } @@ -1392,7 +1389,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, return -ENOMEM; vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); - addr_pfn += 1; next = addr + PAGE_SIZE; continue; } @@ -1402,7 +1398,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, return -ENOMEM; vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); - addr_pfn += 1; next = addr + PAGE_SIZE; continue; } From b1b7c045e808c761b1cc8c19b3040fadedda3fef Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 12 Jun 2026 11:58:48 +0800 Subject: [PATCH 052/501] mm/hugetlb: initialize gigantic bootmem hugepage struct pages earlier Gigantic bootmem HugeTLB pages are currently initialized from hugetlb_init(), but page_alloc_init_late() runs earlier and walks pageblocks to determine zone contiguity. If a bootmem HugeTLB region is marked noinit, set_zone_contiguous() can observe still-uninitialized struct pages through __pageblock_pfn_to_page(). This may not trigger an immediate failure, but it can make set_zone_contiguous() compute the wrong zone contiguity state. If extra poisoned-page checks are added in this path, such as PF_POISONED_CHECK() in page_zone_id(), it can also trigger an early boot panic. Initialize gigantic bootmem HugeTLB struct pages from page_alloc_init_late(), before zone contiguity is evaluated, so later page allocator setup only sees valid struct page state. This also makes the initialization order more natural, as struct pages should be initialized before later code inspects them. Link: https://lore.kernel.org/20260612035903.2468601-5-songmuchun@bytedance.com Fixes: fde1c4ecf916 ("mm: hugetlb: skip initialization of gigantic tail struct pages if freed by HVO") Signed-off-by: Muchun Song Acked-by: Mike Rapoport (Microsoft) Acked-by: Oscar Salvador Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Frank van der Linden Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Oscar Salvador (SUSE) Cc: "Ritesh Harjani (IBM)" Cc: Usama Arif Cc: Vlastimil Babka Cc: Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 5 +++++ mm/hugetlb.c | 5 ++--- mm/mm_init.c | 1 + mm/sparse-vmemmap.c | 4 ++-- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index d6967fc94a49..431848873370 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -172,6 +172,7 @@ extern int movable_gigantic_pages __read_mostly; extern int sysctl_hugetlb_shm_group __read_mostly; extern struct list_head huge_boot_pages[MAX_NUMNODES]; +void hugetlb_bootmem_struct_page_init(void); void hugetlb_bootmem_alloc(void); extern nodemask_t hugetlb_bootmem_nodes; void hugetlb_bootmem_set_nodes(void); @@ -1294,6 +1295,10 @@ static inline bool hugetlbfs_pagecache_present( static inline void hugetlb_bootmem_alloc(void) { } + +static inline void hugetlb_bootmem_struct_page_init(void) +{ +} #endif /* CONFIG_HUGETLB_PAGE */ static inline spinlock_t *huge_pte_lock(struct hstate *h, diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 4c234611e7a0..78fe77a690ad 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3353,7 +3353,7 @@ static void __init gather_bootmem_prealloc_parallel(unsigned long start, gather_bootmem_prealloc_node(nid); } -static void __init gather_bootmem_prealloc(void) +void __init hugetlb_bootmem_struct_page_init(void) { struct padata_mt_job job = { .thread_fn = gather_bootmem_prealloc_parallel, @@ -3582,7 +3582,7 @@ static unsigned long __init hugetlb_pages_alloc_boot(struct hstate *h) * - For gigantic pages, this is called early in the boot process and * pages are allocated from memblock allocated or something similar. * Gigantic pages are actually added to pools later with the routine - * gather_bootmem_prealloc. + * hugetlb_bootmem_struct_page_init. * - For non-gigantic pages, this is called later in the boot process after * all of mm is up and functional. Pages are allocated from buddy and * then added to hugetlb pools. @@ -4152,7 +4152,6 @@ static int __init hugetlb_init(void) } hugetlb_init_hstates(); - gather_bootmem_prealloc(); report_hugepages(); hugetlb_sysfs_init(); diff --git a/mm/mm_init.c b/mm/mm_init.c index 498d62c4ece3..eb7222f133e6 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -2330,6 +2330,7 @@ void __init page_alloc_init_late(void) /* Reinit limits that are based on free pages after the kernel is up */ files_maxfiles_init(); #endif + hugetlb_bootmem_struct_page_init(); /* Accounting of total+free memory is stable at this point. */ mem_init_print_info(); diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index dff8da23eabb..50df73b8f747 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -342,8 +342,8 @@ static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone * * * Any initialization done here will be overwritten by memmap_init(). * - * gather_bootmem_prealloc() will take care of initialization after - * memmap_init(). + * hugetlb_bootmem_struct_page_init() will take care of initialization + * after memmap_init(). */ p = vmemmap_alloc_block_zero(PAGE_SIZE, node); From 9c29f83d88d707e2d13d21017fefcb8ab9edb5f1 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 12 Jun 2026 11:58:49 +0800 Subject: [PATCH 053/501] mm/mm_init: simplify deferred_free_pages() migratetype init deferred_free_pages() open-codes two loops to initialize the pageblock migratetype for a range of pages. Replace them with pageblock_migratetype_init_range() to remove the duplication and make the code clearer (Note that deferred_free_pages() may be called from atomic context). Link: https://lore.kernel.org/20260612035903.2468601-6-songmuchun@bytedance.com Signed-off-by: Muchun Song Acked-by: Mike Rapoport (Microsoft) Acked-by: Oscar Salvador Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Frank van der Linden Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Oscar Salvador (SUSE) Cc: "Ritesh Harjani (IBM)" Cc: Usama Arif Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/mm_init.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index eb7222f133e6..405ecee15714 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -674,15 +674,15 @@ static inline void fixup_hashdist(void) static inline void fixup_hashdist(void) {} #endif /* CONFIG_NUMA */ -#ifdef CONFIG_ZONE_DEVICE +#if defined(CONFIG_ZONE_DEVICE) || defined(CONFIG_DEFERRED_STRUCT_PAGE_INIT) static __meminit void pageblock_migratetype_init_range(unsigned long pfn, - unsigned long nr_pages, int migratetype) + unsigned long nr_pages, int migratetype, bool atomic) { const unsigned long end = pfn + nr_pages; for (pfn = pageblock_align(pfn); pfn < end; pfn += pageblock_nr_pages) { init_pageblock_migratetype(pfn_to_page(pfn), migratetype, false); - if (IS_ALIGNED(pfn, PAGES_PER_SECTION)) + if (!atomic && IS_ALIGNED(pfn, PAGES_PER_SECTION)) cond_resched(); } } @@ -1142,7 +1142,7 @@ void __ref memmap_init_zone_device(struct zone *zone, compound_nr_pages(pfn, altmap, pgmap)); } - pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE); + pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE, false); pr_debug("%s initialised %lu pages in %ums\n", __func__, nr_pages, jiffies_to_msecs(jiffies - start)); @@ -1988,12 +1988,12 @@ static void __init deferred_free_pages(unsigned long pfn, if (!nr_pages) return; + pageblock_migratetype_init_range(pfn, nr_pages, mt, true); + page = pfn_to_page(pfn); /* Free a large naturally-aligned chunk if possible */ if (nr_pages == MAX_ORDER_NR_PAGES && IS_MAX_ORDER_ALIGNED(pfn)) { - for (i = 0; i < nr_pages; i += pageblock_nr_pages) - init_pageblock_migratetype(page + i, mt, false); __free_pages_core(page, MAX_PAGE_ORDER, MEMINIT_EARLY); return; } @@ -2001,11 +2001,8 @@ static void __init deferred_free_pages(unsigned long pfn, /* Accept chunks smaller than MAX_PAGE_ORDER upfront */ accept_memory(PFN_PHYS(pfn), nr_pages * PAGE_SIZE); - for (i = 0; i < nr_pages; i++, page++, pfn++) { - if (pageblock_aligned(pfn)) - init_pageblock_migratetype(page, mt, false); - __free_pages_core(page, 0, MEMINIT_EARLY); - } + for (i = 0; i < nr_pages; i++) + __free_pages_core(page + i, 0, MEMINIT_EARLY); } /* Completion tracking for deferred_init_memmap() threads */ From b3ef855262928c5e80e881895694891164c13fc8 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 12 Jun 2026 11:58:50 +0800 Subject: [PATCH 054/501] mm/sparse: panic on memmap and usemap allocation failure When vmemmap or usemap allocation fails, sparse_init_nid() currently marks the section non-present and continues. Later boot-time code can still walk PFNs in that section without checking for this partial setup, which leads to invalid accesses. subsection_map_init() can also touch an unallocated usemap. Auditing and fixing all early PFN walkers for this case is not worth the complexity. These allocation failures are expected to be fatal anyway, and other memory models already treat them that way. Make memmap and usemap allocation failures panic immediately instead of trying to recover and crashing later in less obvious ways. This is also consistent with how other memory model configurations handle memmap allocation failures. Link: https://lore.kernel.org/20260612035903.2468601-7-songmuchun@bytedance.com Signed-off-by: Muchun Song Acked-by: Mike Rapoport (Microsoft) Acked-by: Oscar Salvador Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Frank van der Linden Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Oscar Salvador (SUSE) Cc: "Ritesh Harjani (IBM)" Cc: Usama Arif Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/sparse.c | 44 +++++++++----------------------------------- 1 file changed, 9 insertions(+), 35 deletions(-) diff --git a/mm/sparse.c b/mm/sparse.c index 16ac6df3c89f..c92bbc3f3aa3 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -239,15 +239,8 @@ struct page __init *__populate_section_memmap(unsigned long pfn, struct dev_pagemap *pgmap) { unsigned long size = section_map_size(); - struct page *map; - phys_addr_t addr = __pa(MAX_DMA_ADDRESS); - map = memmap_alloc(size, size, addr, nid, false); - if (!map) - panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa\n", - __func__, size, PAGE_SIZE, nid, &addr); - - return map; + return memmap_alloc(size, size, __pa(MAX_DMA_ADDRESS), nid, false); } #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ @@ -300,17 +293,14 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin, unsigned long map_count) { unsigned long pnum; - struct page *map; - struct mem_section *ms; - if (sparse_usage_init(nid, map_count)) { - pr_err("%s: node[%d] usemap allocation failed", __func__, nid); - goto failed; - } + if (sparse_usage_init(nid, map_count)) + panic("Failed to allocate usemap for node %d\n", nid); sparse_vmemmap_init_nid_early(nid); for_each_present_section_nr(pnum_begin, pnum) { + struct mem_section *ms; unsigned long pfn = section_nr_to_pfn(pnum); if (pnum >= pnum_end) @@ -318,34 +308,18 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin, ms = __nr_to_section(pnum); if (!preinited_vmemmap_section(ms)) { + struct page *map; + map = __populate_section_memmap(pfn, PAGES_PER_SECTION, - nid, NULL, NULL); - if (!map) { - pr_err("%s: node[%d] memory map backing failed. Some memory will not be available.", - __func__, nid); - pnum_begin = pnum; - sparse_usage_fini(); - goto failed; - } + nid, NULL, NULL); + if (!map) + panic("Failed to allocate memmap for section %lu\n", pnum); memmap_boot_pages_add(DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page), PAGE_SIZE)); sparse_init_early_section(nid, map, pnum, 0); } } sparse_usage_fini(); - return; -failed: - /* - * We failed to allocate, mark all the following pnums as not present, - * except the ones already initialized earlier. - */ - for_each_present_section_nr(pnum_begin, pnum) { - if (pnum >= pnum_end) - break; - ms = __nr_to_section(pnum); - if (!preinited_vmemmap_section(ms)) - ms->section_mem_map = 0; - } } /* From 8e4ee7b1bdafb091c41c3f5b87b6db29094bf731 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 12 Jun 2026 11:58:51 +0800 Subject: [PATCH 055/501] mm/sparse: move subsection_map_init() into sparse_init() subsection_map_init() is part of sparse memory initialization, but it is currently called from free_area_init(). Move it into sparse_init() so the sparse-specific setup stays together instead of being split across the generic free_area_init() path. Link: https://lore.kernel.org/20260612035903.2468601-8-songmuchun@bytedance.com Signed-off-by: Muchun Song Acked-by: Mike Rapoport (Microsoft) Acked-by: Oscar Salvador Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Frank van der Linden Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Oscar Salvador (SUSE) Cc: "Ritesh Harjani (IBM)" Cc: Usama Arif Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/internal.h | 5 ++--- mm/mm_init.c | 10 ++-------- mm/sparse-vmemmap.c | 11 ++++++++++- mm/sparse.c | 1 + 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index 181e79f1d6a2..dccd4727de46 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -994,10 +994,9 @@ static inline void sparse_init(void) {} * mm/sparse-vmemmap.c */ #ifdef CONFIG_SPARSEMEM_VMEMMAP -void sparse_init_subsection_map(unsigned long pfn, unsigned long nr_pages); +void sparse_init_subsection_map(void); #else -static inline void sparse_init_subsection_map(unsigned long pfn, - unsigned long nr_pages) +static inline void sparse_init_subsection_map(void) { } #endif /* CONFIG_SPARSEMEM_VMEMMAP */ diff --git a/mm/mm_init.c b/mm/mm_init.c index 405ecee15714..0394150be24b 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1871,18 +1871,12 @@ static void __init free_area_init(void) (u64)zone_movable_pfn[i] << PAGE_SHIFT); } - /* - * Print out the early node map, and initialize the - * subsection-map relative to active online memory ranges to - * enable future "sub-section" extensions of the memory map. - */ + /* Print out the early node map. */ pr_info("Early memory node ranges\n"); - for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { + for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, (u64)start_pfn << PAGE_SHIFT, ((u64)end_pfn << PAGE_SHIFT) - 1); - sparse_init_subsection_map(start_pfn, end_pfn - start_pfn); - } /* Initialise every node */ mminit_verify_pageflags_layout(); diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 50df73b8f747..240704015ff6 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -603,7 +603,7 @@ static void subsection_mask_set(unsigned long *map, unsigned long pfn, bitmap_set(map, idx, end - idx + 1); } -void __init sparse_init_subsection_map(unsigned long pfn, unsigned long nr_pages) +static void __init sparse_init_subsection_map_range(unsigned long pfn, unsigned long nr_pages) { int end_sec_nr = pfn_to_section_nr(pfn + nr_pages - 1); unsigned long nr, start_sec_nr = pfn_to_section_nr(pfn); @@ -626,6 +626,15 @@ void __init sparse_init_subsection_map(unsigned long pfn, unsigned long nr_pages } } +void __init sparse_init_subsection_map(void) +{ + int i, nid; + unsigned long start, end; + + for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) + sparse_init_subsection_map_range(start, end - start); +} + #ifdef CONFIG_MEMORY_HOTPLUG /* Mark all memory sections within the pfn range as online */ diff --git a/mm/sparse.c b/mm/sparse.c index c92bbc3f3aa3..85557ef387c7 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -361,5 +361,6 @@ void __init sparse_init(void) } /* cover the last node */ sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count); + sparse_init_subsection_map(); vmemmap_populate_print_last(); } From ee6192a2c4321e8cfde0e09131a72ab304e2d4f0 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 12 Jun 2026 11:58:52 +0800 Subject: [PATCH 056/501] mm/mm_init: defer sparse_init() until after zone initialization free_area_init() is responsible for initializing pgdat and zone state. Calling sparse_init() from there mixes in later vmemmap and struct page setup, which makes the initialization flow less clear. Defer sparse_init(), sparse_vmemmap_init_nid_late(), and memmap_init() until after free_area_init() completes, when zone initialization is fully done. This keeps free_area_init() focused on zone setup and ensures that sparse_init() runs with the relevant zone state already available. This is also a prerequisite for later hugetlb vmemmap changes that need zone information during early sparse vmemmap setup. Link: https://lore.kernel.org/20260612035903.2468601-9-songmuchun@bytedance.com Signed-off-by: Muchun Song Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Oscar Salvador (SUSE) Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Frank van der Linden Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Oscar Salvador Cc: "Ritesh Harjani (IBM)" Cc: Usama Arif Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/mm_init.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 0394150be24b..bd18862ac30b 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1821,7 +1821,6 @@ static void __init free_area_init(void) bool descending; arch_zone_limits_init(max_zone_pfn); - sparse_init(); start_pfn = PHYS_PFN(memblock_start_of_DRAM()); descending = arch_has_descending_max_zone_pfns(); @@ -1910,11 +1909,7 @@ static void __init free_area_init(void) } } - for_each_node_state(nid, N_MEMORY) - sparse_vmemmap_init_nid_late(nid); - calc_nr_kernel_pages(); - memmap_init(); /* disable hash distribution for systems with a single node */ fixup_hashdist(); @@ -2686,10 +2681,17 @@ void __init __weak mem_init(void) void __init mm_core_init_early(void) { + int nid; + hugetlb_cma_reserve(); hugetlb_bootmem_alloc(); free_area_init(); + + sparse_init(); + for_each_node_state(nid, N_MEMORY) + sparse_vmemmap_init_nid_late(nid); + memmap_init(); } /* From 1221a19f2b039b61d81728cc4b7a62d3e1a71633 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 12 Jun 2026 11:58:53 +0800 Subject: [PATCH 057/501] mm/mm_init: defer hugetlb reservation until after zone initialization hugetlb_cma_reserve() and hugetlb_bootmem_alloc() currently run before free_area_init(), so HugeTLB reservation happens before zone state is initialized. Move the reservation step after free_area_init() so the relevant zone information is available before HugeTLB reserves memory. This is needed for later hugetlb changes that validate boot-time HugeTLB reservations against zone boundaries. Link: https://lore.kernel.org/20260612035903.2468601-10-songmuchun@bytedance.com Signed-off-by: Muchun Song Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Oscar Salvador (SUSE) Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Frank van der Linden Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Oscar Salvador Cc: "Ritesh Harjani (IBM)" Cc: Usama Arif Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/mm_init.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index bd18862ac30b..7cfe7301a107 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -2683,11 +2683,11 @@ void __init mm_core_init_early(void) { int nid; + free_area_init(); + hugetlb_cma_reserve(); hugetlb_bootmem_alloc(); - free_area_init(); - sparse_init(); for_each_node_state(nid, N_MEMORY) sparse_vmemmap_init_nid_late(nid); From 37424eab6da3ac6543472d7c5fbefbabd5c523dd Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 12 Jun 2026 11:58:54 +0800 Subject: [PATCH 058/501] mm/mm_init: remove set_pageblock_order() call from sparse_init() free_area_init() already sets pageblock_order before sparse_init() runs for CONFIG_HUGETLB_PAGE_SIZE_VARIABLE, so sparse_init() does not need to call set_pageblock_order() again. With that call removed, set_pageblock_order() is only used in mm/mm_init.c. Make it static. Link: https://lore.kernel.org/20260612035903.2468601-11-songmuchun@bytedance.com Signed-off-by: Muchun Song Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Oscar Salvador (SUSE) Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Frank van der Linden Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Oscar Salvador (SUSE) Cc: "Ritesh Harjani (IBM)" Cc: Usama Arif Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/internal.h | 1 - mm/mm_init.c | 4 ++-- mm/sparse.c | 3 --- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index dccd4727de46..09efb9f4d126 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1435,7 +1435,6 @@ extern unsigned long __must_check vm_mmap_pgoff(struct file *, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long); -extern void set_pageblock_order(void); unsigned long reclaim_pages(struct list_head *folio_list); unsigned int reclaim_clean_pages_from_list(struct zone *zone, struct list_head *folio_list); diff --git a/mm/mm_init.c b/mm/mm_init.c index 7cfe7301a107..6f9b02ad4f2c 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1496,7 +1496,7 @@ static inline void setup_usemap(struct zone *zone) {} #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ -void __init set_pageblock_order(void) +static void __init set_pageblock_order(void) { unsigned int order = PAGE_BLOCK_MAX_ORDER; @@ -1522,7 +1522,7 @@ void __init set_pageblock_order(void) * include/linux/pageblock-flags.h for the values of pageblock_order based on * the kernel config */ -void __init set_pageblock_order(void) +static inline void __init set_pageblock_order(void) { } diff --git a/mm/sparse.c b/mm/sparse.c index 85557ef387c7..324213d8bdcb 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -343,9 +343,6 @@ void __init sparse_init(void) pnum_begin = first_present_section_nr(); nid_begin = sparse_early_nid(__nr_to_section(pnum_begin)); - /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */ - set_pageblock_order(); - for_each_present_section_nr(pnum_begin + 1, pnum_end) { int nid = sparse_early_nid(__nr_to_section(pnum_end)); From 133b57fee93f1f23fe6e4aec694049940d40960b Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 12 Jun 2026 11:58:55 +0800 Subject: [PATCH 059/501] mm/sparse: move sparse_vmemmap_init_nid_late() into sparse_init_nid() sparse_vmemmap_init_nid_late() is still called separately from mm_core_init_early(), away from the rest of the sparse initialization path. Now that sparse_init() runs after zone initialization, call sparse_vmemmap_init_nid_late() from sparse_init_nid() instead. This keeps both sparse_vmemmap_init_nid_early() and sparse_vmemmap_init_nid_late() in the sparse setup path. Link: https://lore.kernel.org/20260612035903.2468601-12-songmuchun@bytedance.com Signed-off-by: Muchun Song Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Oscar Salvador (SUSE) Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Frank van der Linden Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Oscar Salvador Cc: "Ritesh Harjani (IBM)" Cc: Usama Arif Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/mm_init.c | 4 ---- mm/sparse.c | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 6f9b02ad4f2c..4ed4591dcdba 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -2681,16 +2681,12 @@ void __init __weak mem_init(void) void __init mm_core_init_early(void) { - int nid; - free_area_init(); hugetlb_cma_reserve(); hugetlb_bootmem_alloc(); sparse_init(); - for_each_node_state(nid, N_MEMORY) - sparse_vmemmap_init_nid_late(nid); memmap_init(); } diff --git a/mm/sparse.c b/mm/sparse.c index 324213d8bdcb..3917a47153d8 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -320,6 +320,7 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin, } } sparse_usage_fini(); + sparse_vmemmap_init_nid_late(nid); } /* From 7e250334158dabc2f9096b05979abe6f78569628 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 12 Jun 2026 11:58:56 +0800 Subject: [PATCH 060/501] mm/hugetlb_cma: validate hugetlb CMA range by zone at reserve time Hugetlb CMA allocation currently has to cope with CMA areas that span multiple zones. Validate the reserved CMA range up front in hugetlb_cma_reserve() so later hugetlb CMA allocations can assume a zone-consistent area. Also drop the pfn_valid() check from cma_validate_zones(). mem_section is not fully initialized at this point, so the check can trigger false warnings. Keep the sanity check in cma_activate_area() instead. Link: https://lore.kernel.org/20260612035903.2468601-13-songmuchun@bytedance.com Signed-off-by: Muchun Song Acked-by: Mike Rapoport (Microsoft) Reviewed-by: Oscar Salvador (SUSE) Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Frank van der Linden Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Oscar Salvador (SUSE) Cc: "Ritesh Harjani (IBM)" Cc: Usama Arif Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/cma.c | 3 ++- mm/hugetlb_cma.c | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mm/cma.c b/mm/cma.c index a13ce4999b39..31073738f2ac 100644 --- a/mm/cma.c +++ b/mm/cma.c @@ -126,7 +126,6 @@ bool cma_validate_zones(struct cma *cma) * to be in the same zone. Simplify by forcing the entire * CMA resv range to be in the same zone. */ - WARN_ON_ONCE(!pfn_valid(base_pfn)); if (pfn_range_intersects_zones(cma->nid, base_pfn, cmr->count)) { set_bit(CMA_ZONES_INVALID, &cma->flags); return false; @@ -165,6 +164,8 @@ static void __init cma_activate_area(struct cma *cma) bitmap_set(cmr->bitmap, 0, bitmap_count); } + WARN_ON_ONCE(!pfn_valid(cmr->base_pfn)); + for (pfn = early_pfn[r]; pfn < cmr->base_pfn + cmr->count; pfn += pageblock_nr_pages) init_cma_reserved_pageblock(pfn_to_page(pfn)); diff --git a/mm/hugetlb_cma.c b/mm/hugetlb_cma.c index 39344d6c78d8..ce999391cc14 100644 --- a/mm/hugetlb_cma.c +++ b/mm/hugetlb_cma.c @@ -231,9 +231,11 @@ void __init hugetlb_cma_reserve(void) res = cma_declare_contiguous_multi(size, gigantic_page_size, HUGETLB_PAGE_ORDER, name, &hugetlb_cma[nid], nid); - if (res) { - pr_warn("hugetlb_cma: reservation failed: err %d, node %d", + if (res || !cma_validate_zones(hugetlb_cma[nid])) { + pr_warn("hugetlb_cma: %s: err %d, node %d\n", + res ? "reservation failed" : "reserved area spans zones", res, nid); + hugetlb_cma[nid] = NULL; continue; } From 27ca02e689b2afdae361ab8701c8204a03cb879d Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 12 Jun 2026 11:58:57 +0800 Subject: [PATCH 061/501] mm/hugetlb: refactor early boot gigantic hugepage allocation The early boot gigantic hugepage allocation helpers currently mix allocation with huge_bootmem_page setup, and leave part of the initialization flow in architecture code. Refactor the interface to return the allocated huge page pointer and move the huge_bootmem_page setup into the generic hugetlb code. This makes the architecture-specific paths focus only on finding memory, while the common code handles node placement and early page metadata setup in one place. This also lets powerpc benefit from memblock_reserved_mark_noinit(), which it did not enable before. In addition, upcoming cross-zone validation for boot-time gigantic hugetlb reservation is common logic. With this refactoring, that logic can stay in the generic code instead of being duplicated in architecture-specific paths. Link: https://lore.kernel.org/20260612035903.2468601-14-songmuchun@bytedance.com Signed-off-by: Muchun Song Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Oscar Salvador (SUSE) Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Frank van der Linden Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Oscar Salvador (SUSE) Cc: "Ritesh Harjani (IBM)" Cc: Usama Arif Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- arch/powerpc/mm/hugetlbpage.c | 13 ++--- include/linux/hugetlb.h | 18 ++----- mm/hugetlb.c | 95 ++++++++++++++--------------------- mm/hugetlb_cma.c | 13 ++--- mm/hugetlb_cma.h | 8 ++- mm/internal.h | 9 ++++ 6 files changed, 64 insertions(+), 92 deletions(-) diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index 558fafb82b8a..a298746dc143 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -104,17 +104,14 @@ void __init pseries_add_gpage(u64 addr, u64 page_size, unsigned long number_of_p } } -static int __init pseries_alloc_bootmem_huge_page(struct hstate *hstate) +static __init void *pseries_alloc_bootmem_huge_page(struct hstate *hstate) { - struct huge_bootmem_page *m; + void *m; if (nr_gpages == 0) - return 0; + return NULL; m = phys_to_virt(gpage_freearray[--nr_gpages]); gpage_freearray[nr_gpages] = 0; - list_add(&m->list, &huge_boot_pages[0]); - m->hstate = hstate; - m->flags = 0; - return 1; + return m; } bool __init hugetlb_node_alloc_supported(void) @@ -124,7 +121,7 @@ bool __init hugetlb_node_alloc_supported(void) #endif -int __init alloc_bootmem_huge_page(struct hstate *h, int nid) +void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid) { #ifdef CONFIG_PPC_BOOK3S_64 diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 431848873370..cf39ac68e9bf 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -675,19 +675,11 @@ struct hstate { char name[HSTATE_NAME_LEN]; }; -struct cma; - -struct huge_bootmem_page { - struct list_head list; - struct hstate *hstate; - unsigned long flags; - struct cma *cma; -}; - #define HUGE_BOOTMEM_HVO 0x0001 #define HUGE_BOOTMEM_ZONES_VALID 0x0002 #define HUGE_BOOTMEM_CMA 0x0004 +struct huge_bootmem_page; bool hugetlb_bootmem_page_zones_valid(int nid, struct huge_bootmem_page *m); int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list); @@ -707,8 +699,8 @@ void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma, unsigned long address, struct folio *folio); /* arch callback */ -int __init __alloc_bootmem_huge_page(struct hstate *h, int nid); -int __init alloc_bootmem_huge_page(struct hstate *h, int nid); +void *__init __alloc_bootmem_huge_page(struct hstate *h, int nid); +void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid); bool __init hugetlb_node_alloc_supported(void); void __init hugetlb_add_hstate(unsigned order); @@ -1139,9 +1131,9 @@ alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, return NULL; } -static inline int __alloc_bootmem_huge_page(struct hstate *h) +static inline void *__alloc_bootmem_huge_page(struct hstate *h, int nid) { - return 0; + return NULL; } static inline struct hstate *hstate_file(struct file *f) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 78fe77a690ad..5224eb299795 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3027,79 +3027,58 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, static __init void *alloc_bootmem(struct hstate *h, int nid, bool node_exact) { - struct huge_bootmem_page *m; - int listnode = nid; - if (hugetlb_early_cma(h)) - m = hugetlb_cma_alloc_bootmem(h, &listnode, node_exact); - else { - if (node_exact) - m = memblock_alloc_exact_nid_raw(huge_page_size(h), + return hugetlb_cma_alloc_bootmem(h, nid, node_exact); + + if (node_exact) + return memblock_alloc_exact_nid_raw(huge_page_size(h), huge_page_size(h), 0, MEMBLOCK_ALLOC_ACCESSIBLE, nid); - else { - m = memblock_alloc_try_nid_raw(huge_page_size(h), + + return memblock_alloc_try_nid_raw(huge_page_size(h), huge_page_size(h), 0, MEMBLOCK_ALLOC_ACCESSIBLE, nid); - /* - * For pre-HVO to work correctly, pages need to be on - * the list for the node they were actually allocated - * from. That node may be different in the case of - * fallback by memblock_alloc_try_nid_raw. So, - * extract the actual node first. - */ - if (m) - listnode = early_pfn_to_nid(PHYS_PFN(__pa(m))); - } - - if (m) { - m->flags = 0; - m->cma = NULL; - } - } - - if (m) { - /* - * Use the beginning of the huge page to store the - * huge_bootmem_page struct (until gather_bootmem - * puts them into the mem_map). - * - * Put them into a private list first because mem_map - * is not up yet. - */ - INIT_LIST_HEAD(&m->list); - list_add(&m->list, &huge_boot_pages[listnode]); - m->hstate = h; - } - - return m; } -int alloc_bootmem_huge_page(struct hstate *h, int nid) +void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid) __attribute__ ((weak, alias("__alloc_bootmem_huge_page"))); -int __alloc_bootmem_huge_page(struct hstate *h, int nid) +void *__init __alloc_bootmem_huge_page(struct hstate *h, int nid) { - struct huge_bootmem_page *m = NULL; /* initialize for clang */ int nr_nodes, node = nid; /* do node specific alloc */ - if (nid != NUMA_NO_NODE) { - m = alloc_bootmem(h, node, true); - if (!m) - return 0; - goto found; - } + if (nid != NUMA_NO_NODE) + return alloc_bootmem(h, node, true); /* allocate from next node when distributing huge pages */ for_each_node_mask_to_alloc(&h->next_nid_to_alloc, nr_nodes, node, - &hugetlb_bootmem_nodes) { - m = alloc_bootmem(h, node, false); - if (!m) - return 0; - goto found; - } + &hugetlb_bootmem_nodes) + return alloc_bootmem(h, node, false); -found: + return NULL; +} + +static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid) +{ + struct huge_bootmem_page *m = arch_alloc_bootmem_huge_page(h, nid); + + if (!m) + return false; + + nid = early_pfn_to_nid(PHYS_PFN(__pa(m))); + /* + * Use the beginning of the huge page to store the huge_bootmem_page + * struct (until gather_bootmem puts them into the mem_map). + * + * Put them into a private list first because mem_map is not up yet. + */ + INIT_LIST_HEAD(&m->list); + list_add(&m->list, &huge_boot_pages[nid]); + m->hstate = h; + if (!hugetlb_early_cma(h)) { + m->cma = NULL; + m->flags = 0; + } /* * Only initialize the head struct page in memmap_init_reserved_pages, @@ -3111,7 +3090,7 @@ int __alloc_bootmem_huge_page(struct hstate *h, int nid) memblock_reserved_mark_noinit(__pa((void *)m + PAGE_SIZE), huge_page_size(h) - PAGE_SIZE); - return 1; + return true; } /* Initialize [start_page:end_page_number] tail struct pages of a hugepage */ diff --git a/mm/hugetlb_cma.c b/mm/hugetlb_cma.c index ce999391cc14..e487d0ffffc0 100644 --- a/mm/hugetlb_cma.c +++ b/mm/hugetlb_cma.c @@ -56,14 +56,13 @@ struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, return folio; } -struct huge_bootmem_page * __init -hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, bool node_exact) +void * __init hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, bool node_exact) { struct cma *cma; struct huge_bootmem_page *m; - int node = *nid; + int node; - cma = hugetlb_cma[*nid]; + cma = hugetlb_cma[nid]; m = cma_reserve_early(cma, huge_page_size(h)); if (!m) { if (node_exact) @@ -71,13 +70,11 @@ hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, bool node_exact) for_each_node_mask(node, hugetlb_bootmem_nodes) { cma = hugetlb_cma[node]; - if (!cma || node == *nid) + if (!cma || node == nid) continue; m = cma_reserve_early(cma, huge_page_size(h)); - if (m) { - *nid = node; + if (m) break; - } } } diff --git a/mm/hugetlb_cma.h b/mm/hugetlb_cma.h index c619c394b1ae..3aa483573d17 100644 --- a/mm/hugetlb_cma.h +++ b/mm/hugetlb_cma.h @@ -6,8 +6,7 @@ void hugetlb_cma_free_frozen_folio(struct folio *folio); struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, int nid, nodemask_t *nodemask); -struct huge_bootmem_page *hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, - bool node_exact); +void *hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, bool node_exact); bool hugetlb_cma_exclusive_alloc(void); unsigned long hugetlb_cma_total_size(void); void hugetlb_cma_validate_params(void); @@ -23,9 +22,8 @@ static inline struct folio *hugetlb_cma_alloc_frozen_folio(int order, return NULL; } -static inline -struct huge_bootmem_page *hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, - bool node_exact) +static inline void *hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, + bool node_exact) { return NULL; } diff --git a/mm/internal.h b/mm/internal.h index 09efb9f4d126..3401759924d9 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -23,6 +23,15 @@ #include "vma.h" struct folio_batch; +struct hstate; +struct cma; + +struct huge_bootmem_page { + struct list_head list; + struct hstate *hstate; + unsigned long flags; + struct cma *cma; +}; /* * Maintains state across a page table move. The operation assumes both source From 20549ccf4da49ff6dcb35ffc741f92aa92e07aeb Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 12 Jun 2026 11:58:58 +0800 Subject: [PATCH 062/501] mm/hugetlb: free cross-zone bootmem gigantic pages after allocation Now that hugetlb reservation runs after zone initialization, bootmem gigantic page allocation can detect pages that span multiple zones. Keep those cross-zone pages separate during allocation and free them after allocation completes, so later hugetlb initialization only sees zone-valid gigantic pages. This chooses to free cross-zone gigantic pages directly instead of retrying allocation. In practice, such cross-zone cases are expected to be very rare, so adding retry logic does not seem justified at this point. Keeping the handling simple also preserves the previous behavior. If similar real-world reports show up later, retry support can be reconsidered then. Link: https://lore.kernel.org/20260612035903.2468601-15-songmuchun@bytedance.com Signed-off-by: Muchun Song Acked-by: Mike Rapoport (Microsoft) Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Frank van der Linden Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Oscar Salvador Cc: Oscar Salvador (SUSE) Cc: "Ritesh Harjani (IBM)" Cc: Usama Arif Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/hugetlb.c | 75 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 11 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 5224eb299795..6af6895c4bf9 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3060,12 +3060,15 @@ void *__init __alloc_bootmem_huge_page(struct hstate *h, int nid) static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid) { + unsigned long pfn; + unsigned int nid_request = nid; struct huge_bootmem_page *m = arch_alloc_bootmem_huge_page(h, nid); if (!m) return false; - nid = early_pfn_to_nid(PHYS_PFN(__pa(m))); + pfn = PHYS_PFN(__pa(m)); + nid = early_pfn_to_nid(pfn); /* * Use the beginning of the huge page to store the huge_bootmem_page * struct (until gather_bootmem puts them into the mem_map). @@ -3073,22 +3076,38 @@ static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid) * Put them into a private list first because mem_map is not up yet. */ INIT_LIST_HEAD(&m->list); - list_add(&m->list, &huge_boot_pages[nid]); m->hstate = h; if (!hugetlb_early_cma(h)) { m->cma = NULL; m->flags = 0; } - /* - * Only initialize the head struct page in memmap_init_reserved_pages, - * rest of the struct pages will be initialized by the HugeTLB - * subsystem itself. - * The head struct page is used to get folio information by the HugeTLB - * subsystem like zone id and node id. - */ - memblock_reserved_mark_noinit(__pa((void *)m + PAGE_SIZE), - huge_page_size(h) - PAGE_SIZE); + /* CMA pages: zone-crossing is validated in hugetlb_cma_reserve(). */ + if (!hugetlb_early_cma(h) && + pfn_range_intersects_zones(nid, pfn, pages_per_huge_page(h))) { + /* + * If the allocated page is on a different node than requested + * (e.g., on PowerPC LPARs), put it on the requested node's list, + * because hugetlb_free_cross_zone_pages() only frees cross-zone + * pages belonging to the requested node. + */ + if (WARN_ON_ONCE(nid_request != NUMA_NO_NODE && nid != nid_request)) + list_add(&m->list, &huge_boot_pages[nid_request]); + else + list_add(&m->list, &huge_boot_pages[nid]); + } else { + list_add_tail(&m->list, &huge_boot_pages[nid]); + m->flags |= HUGE_BOOTMEM_ZONES_VALID; + /* + * Only initialize the head struct page in memmap_init_reserved_pages, + * rest of the struct pages will be initialized by the HugeTLB + * subsystem itself. + * The head struct page is used to get folio information by the HugeTLB + * subsystem like zone id and node id. + */ + memblock_reserved_mark_noinit(__pa((void *)m + PAGE_SIZE), + huge_page_size(h) - PAGE_SIZE); + } return true; } @@ -3373,6 +3392,34 @@ void __init hugetlb_bootmem_struct_page_init(void) padata_do_multithreaded(&job); } +static unsigned long __init hugetlb_free_cross_zone_pages(struct hstate *h, int nid) +{ + unsigned long freed = 0; + struct huge_bootmem_page *m, *tmp; + + if (!hstate_is_gigantic(h)) + return freed; + + list_for_each_entry_safe(m, tmp, &huge_boot_pages[nid], list) { + if (m->flags & HUGE_BOOTMEM_ZONES_VALID) + break; + + list_del(&m->list); + memblock_free(m, huge_page_size(h)); + freed++; + } + + if (freed) { + char buf[32]; + + string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, sizeof(buf)); + pr_warn("HugeTLB: freed %lu cross-zone hugepages of size %s on node %d.\n", + freed, buf, nid); + } + + return freed; +} + static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid) { unsigned long i; @@ -3403,6 +3450,8 @@ static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid) cond_resched(); } + i -= hugetlb_free_cross_zone_pages(h, nid); + if (!list_empty(&folio_list)) prep_and_add_allocated_folios(h, &folio_list); @@ -3476,6 +3525,7 @@ static void __init hugetlb_pages_alloc_boot_node(unsigned long start, unsigned l static unsigned long __init hugetlb_gigantic_pages_alloc_boot(struct hstate *h) { + int nid; unsigned long i; for (i = 0; i < h->max_huge_pages; ++i) { @@ -3484,6 +3534,9 @@ static unsigned long __init hugetlb_gigantic_pages_alloc_boot(struct hstate *h) cond_resched(); } + for_each_node(nid) + i -= hugetlb_free_cross_zone_pages(h, nid); + return i; } From 51d73513ddddb4733848a829030038035646437a Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 12 Jun 2026 11:58:59 +0800 Subject: [PATCH 063/501] mm/hugetlb_vmemmap: move bootmem HVO setup to early init Bootmem HugeTLB pages currently defer HVO setup to hugetlb_vmemmap_init_late(), because the optimization needs zone information. Now that zone initialization is available earlier, the bootmem HVO setup can be done directly from hugetlb_vmemmap_init_early(). This lets gigantic HugeTLB pages apply HVO as soon as they are allocated. Bootmem gigantic pages that span multiple zones are now filtered out when they are allocated, so the remaining bootmem gigantic pages seen by later hugetlb initialization are already zone-valid. As a result, hugetlb_vmemmap_init_late() no longer needs to handle bootmem HVO setup. Link: https://lore.kernel.org/20260612035903.2468601-16-songmuchun@bytedance.com Signed-off-by: Muchun Song Acked-by: Mike Rapoport (Microsoft) Acked-by: Usama Arif Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Frank van der Linden Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Oscar Salvador Cc: Oscar Salvador (SUSE) Cc: "Ritesh Harjani (IBM)" Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/hugetlb_vmemmap.c | 93 ++++++++++++-------------------------------- 1 file changed, 25 insertions(+), 68 deletions(-) diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index ea6af85bfec1..ee4fbd5fed0d 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -745,6 +745,20 @@ static bool vmemmap_should_optimize_bootmem_page(struct huge_bootmem_page *m) return true; } +static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn) +{ + struct zone *zone; + enum zone_type zone_type; + + for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { + zone = &NODE_DATA(nid)->node_zones[zone_type]; + if (zone_spans_pfn(zone, pfn)) + return zone; + } + + return NULL; +} + /* * Initialize memmap section for a gigantic page, HVO-style. */ @@ -752,6 +766,7 @@ void __init hugetlb_vmemmap_init_early(int nid) { unsigned long psize, paddr, section_size; unsigned long ns, i, pnum, pfn, nr_pages; + unsigned long start, end; struct huge_bootmem_page *m = NULL; void *map; @@ -761,6 +776,8 @@ void __init hugetlb_vmemmap_init_early(int nid) section_size = (1UL << PA_SECTION_SHIFT); list_for_each_entry(m, &huge_boot_pages[nid], list) { + struct zone *zone; + if (!vmemmap_should_optimize_bootmem_page(m)) continue; @@ -769,6 +786,14 @@ void __init hugetlb_vmemmap_init_early(int nid) paddr = virt_to_phys(m); pfn = PHYS_PFN(paddr); map = pfn_to_page(pfn); + start = (unsigned long)map; + end = start + hugetlb_vmemmap_size(m->hstate); + zone = pfn_to_zone(nid, pfn); + + if (vmemmap_populate_hvo(start, end, huge_page_order(m->hstate), + zone, HUGETLB_VMEMMAP_RESERVE_SIZE)) + panic("Failed to allocate memmap for HugeTLB page\n"); + memmap_boot_pages_add(DIV_ROUND_UP(HUGETLB_VMEMMAP_RESERVE_SIZE, PAGE_SIZE)); pnum = pfn_to_section_nr(pfn); ns = psize / section_size; @@ -784,76 +809,8 @@ void __init hugetlb_vmemmap_init_early(int nid) } } -static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn) -{ - struct zone *zone; - enum zone_type zone_type; - - for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { - zone = &NODE_DATA(nid)->node_zones[zone_type]; - if (zone_spans_pfn(zone, pfn)) - return zone; - } - - return NULL; -} - void __init hugetlb_vmemmap_init_late(int nid) { - struct huge_bootmem_page *m, *tm; - unsigned long phys, nr_pages, start, end; - unsigned long pfn, nr_mmap; - struct zone *zone = NULL; - struct hstate *h; - void *map; - - if (!READ_ONCE(vmemmap_optimize_enabled)) - return; - - list_for_each_entry_safe(m, tm, &huge_boot_pages[nid], list) { - if (!(m->flags & HUGE_BOOTMEM_HVO)) - continue; - - phys = virt_to_phys(m); - h = m->hstate; - pfn = PHYS_PFN(phys); - nr_pages = pages_per_huge_page(h); - map = pfn_to_page(pfn); - start = (unsigned long)map; - end = start + nr_pages * sizeof(struct page); - - if (!hugetlb_bootmem_page_zones_valid(nid, m)) { - /* - * Oops, the hugetlb page spans multiple zones. - * Remove it from the list, and populate it normally. - */ - list_del(&m->list); - - vmemmap_populate(start, end, nid, NULL); - nr_mmap = end - start; - memmap_boot_pages_add(DIV_ROUND_UP(nr_mmap, PAGE_SIZE)); - - memblock_phys_free(phys, huge_page_size(h)); - continue; - } - - if (!zone || !zone_spans_pfn(zone, pfn)) - zone = pfn_to_zone(nid, pfn); - if (WARN_ON_ONCE(!zone)) - continue; - - if (vmemmap_populate_hvo(start, end, huge_page_order(h), zone, - HUGETLB_VMEMMAP_RESERVE_SIZE) < 0) { - /* Fallback if HVO population fails */ - vmemmap_populate(start, end, nid, NULL); - nr_mmap = end - start; - } else { - m->flags |= HUGE_BOOTMEM_ZONES_VALID; - nr_mmap = HUGETLB_VMEMMAP_RESERVE_SIZE; - } - - memmap_boot_pages_add(DIV_ROUND_UP(nr_mmap, PAGE_SIZE)); - } } #endif From 4753a8220a803adc8add8f7338b26b6d476dbbfc Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 12 Jun 2026 11:59:00 +0800 Subject: [PATCH 064/501] mm/hugetlb: remove obsolete bootmem cross-zone checks Bootmem gigantic HugeTLB pages used to be validated again during gather_bootmem_prealloc_node() and any cross-zone pages were discarded there. That validation is no longer needed. Cross-zone bootmem gigantic pages are now detected during allocation and freed before they reach the later bootmem gathering path, so the remaining pages are already zone-valid. Remove the obsolete cross-zone validation, invalid-page freeing, and the associated discarded-page accounting. Link: https://lore.kernel.org/20260612035903.2468601-17-songmuchun@bytedance.com Signed-off-by: Muchun Song Acked-by: Mike Rapoport (Microsoft) Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Frank van der Linden Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Oscar Salvador Cc: Oscar Salvador (SUSE) Cc: "Ritesh Harjani (IBM)" Cc: Usama Arif Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 3 -- mm/hugetlb.c | 70 ----------------------------------------- 2 files changed, 73 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index cf39ac68e9bf..87c87f64bc05 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -679,9 +679,6 @@ struct hstate { #define HUGE_BOOTMEM_ZONES_VALID 0x0002 #define HUGE_BOOTMEM_CMA 0x0004 -struct huge_bootmem_page; -bool hugetlb_bootmem_page_zones_valid(int nid, struct huge_bootmem_page *m); - int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list); int replace_free_hugepage_folios(unsigned long start_pfn, unsigned long end_pfn); void wait_for_freed_hugetlb_folios(void); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 6af6895c4bf9..1bd9d12edb3e 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -58,7 +58,6 @@ struct hstate hstates[HUGE_MAX_HSTATE]; __initdata nodemask_t hugetlb_bootmem_nodes; __initdata struct list_head huge_boot_pages[MAX_NUMNODES]; -static unsigned long hstate_boot_nrinvalid[HUGE_MAX_HSTATE] __initdata; /* * Due to ordering constraints across the init code for various @@ -3221,57 +3220,6 @@ static void __init prep_and_add_bootmem_folios(struct hstate *h, } } -bool __init hugetlb_bootmem_page_zones_valid(int nid, - struct huge_bootmem_page *m) -{ - unsigned long start_pfn; - bool valid; - - if (m->flags & HUGE_BOOTMEM_ZONES_VALID) { - /* - * Already validated, skip check. - */ - return true; - } - - if (hugetlb_bootmem_page_earlycma(m)) { - valid = cma_validate_zones(m->cma); - goto out; - } - - start_pfn = virt_to_phys(m) >> PAGE_SHIFT; - - valid = !pfn_range_intersects_zones(nid, start_pfn, - pages_per_huge_page(m->hstate)); -out: - if (!valid) - hstate_boot_nrinvalid[hstate_index(m->hstate)]++; - - return valid; -} - -/* - * Free a bootmem page that was found to be invalid (intersecting with - * multiple zones). - * - * Since it intersects with multiple zones, we can't just do a free - * operation on all pages at once, but instead have to walk all - * pages, freeing them one by one. - */ -static void __init hugetlb_bootmem_free_invalid_page(int nid, struct page *page, - struct hstate *h) -{ - unsigned long npages = pages_per_huge_page(h); - unsigned long pfn; - - while (npages--) { - pfn = page_to_pfn(page); - __init_page_from_nid(pfn, nid); - free_reserved_page(page); - page++; - } -} - /* * Put bootmem huge pages into the standard lists after mem_map is up. * Note: This only applies to gigantic (order > MAX_PAGE_ORDER) pages. @@ -3287,17 +3235,6 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid) struct folio *folio = (void *)page; h = m->hstate; - if (!hugetlb_bootmem_page_zones_valid(nid, m)) { - /* - * Can't use this page. Initialize the - * page structures if that hasn't already - * been done, and give them to the page - * allocator. - */ - hugetlb_bootmem_free_invalid_page(nid, page, h); - continue; - } - /* * It is possible to have multiple huge page sizes (hstates) * in this list. If so, process each size separately. @@ -3692,20 +3629,13 @@ static void __init hugetlb_init_hstates(void) static void __init report_hugepages(void) { struct hstate *h; - unsigned long nrinvalid; for_each_hstate(h) { char buf[32]; - nrinvalid = hstate_boot_nrinvalid[hstate_index(h)]; - h->max_huge_pages -= nrinvalid; - string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32); pr_info("HugeTLB: registered %s page size, pre-allocated %ld pages\n", buf, h->nr_huge_pages); - if (nrinvalid) - pr_info("HugeTLB: %s page size: %lu invalid page%s discarded\n", - buf, nrinvalid, str_plural(nrinvalid)); pr_info("HugeTLB: %d KiB vmemmap can be freed for a %s page\n", hugetlb_vmemmap_optimizable_size(h) / SZ_1K, buf); } From ec1adc14c17438c72e9a1899bb17d9e2ae3a8aba Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 12 Jun 2026 11:59:01 +0800 Subject: [PATCH 065/501] mm/sparse-vmemmap: remove sparse_vmemmap_init_nid_late() hugetlb_vmemmap_init_late() no longer has any users, so the remaining late-init path in sparse_vmemmap_init_nid_late() is dead code. Remove sparse_vmemmap_init_nid_late() and its declarations. Link: https://lore.kernel.org/20260612035903.2468601-18-songmuchun@bytedance.com Signed-off-by: Muchun Song Acked-by: Mike Rapoport (Microsoft) Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Frank van der Linden Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Oscar Salvador Cc: Oscar Salvador (SUSE) Cc: "Ritesh Harjani (IBM)" Cc: Usama Arif Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/mmzone.h | 7 ------- mm/hugetlb_vmemmap.c | 4 ---- mm/hugetlb_vmemmap.h | 5 ----- mm/sparse-vmemmap.c | 11 ----------- mm/sparse.c | 1 - 5 files changed, 28 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index e26b3d38fa25..35d1a7643dc4 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -2156,8 +2156,6 @@ static inline int preinited_vmemmap_section(const struct mem_section *section) } void sparse_vmemmap_init_nid_early(int nid); -void sparse_vmemmap_init_nid_late(int nid); - #else static inline int preinited_vmemmap_section(const struct mem_section *section) { @@ -2166,10 +2164,6 @@ static inline int preinited_vmemmap_section(const struct mem_section *section) static inline void sparse_vmemmap_init_nid_early(int nid) { } - -static inline void sparse_vmemmap_init_nid_late(int nid) -{ -} #endif static inline int online_section_nr(unsigned long nr) @@ -2374,7 +2368,6 @@ static inline unsigned long next_present_section_nr(unsigned long section_nr) #else #define sparse_vmemmap_init_nid_early(_nid) do {} while (0) -#define sparse_vmemmap_init_nid_late(_nid) do {} while (0) #define pfn_in_present_section pfn_valid #endif /* CONFIG_SPARSEMEM */ diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index ee4fbd5fed0d..eefd6b5f9706 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -808,10 +808,6 @@ void __init hugetlb_vmemmap_init_early(int nid) m->flags |= HUGE_BOOTMEM_HVO; } } - -void __init hugetlb_vmemmap_init_late(int nid) -{ -} #endif static const struct ctl_table hugetlb_vmemmap_sysctls[] = { diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h index 18b490825215..7ac49c52457d 100644 --- a/mm/hugetlb_vmemmap.h +++ b/mm/hugetlb_vmemmap.h @@ -29,7 +29,6 @@ void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_l void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head *folio_list); #ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT void hugetlb_vmemmap_init_early(int nid); -void hugetlb_vmemmap_init_late(int nid); #endif @@ -81,10 +80,6 @@ static inline void hugetlb_vmemmap_init_early(int nid) { } -static inline void hugetlb_vmemmap_init_late(int nid) -{ -} - static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate *h) { return 0; diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 240704015ff6..1acc2924d814 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -581,17 +581,6 @@ void __init sparse_vmemmap_init_nid_early(int nid) { hugetlb_vmemmap_init_early(nid); } - -/* - * This is called just before the initialization of page structures - * through memmap_init. Zones are now initialized, so any work that - * needs to be done that needs zone information can be done from - * here. - */ -void __init sparse_vmemmap_init_nid_late(int nid) -{ - hugetlb_vmemmap_init_late(nid); -} #endif static void subsection_mask_set(unsigned long *map, unsigned long pfn, diff --git a/mm/sparse.c b/mm/sparse.c index 3917a47153d8..324213d8bdcb 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -320,7 +320,6 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin, } } sparse_usage_fini(); - sparse_vmemmap_init_nid_late(nid); } /* From 86693837593d7b4db461ccfbe58bd2211379c6c3 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 12 Jun 2026 11:59:02 +0800 Subject: [PATCH 066/501] mm/hugetlb: remove unused bootmem cma field struct huge_bootmem_page no longer needs to keep the CMA pointer. The bootmem path only needs to remember whether a huge page came from CMA, which is already encoded in the flags field. Set HUGE_BOOTMEM_CMA when the page is allocated, drop the unused cma field together with the redundant assignments, and simplify the early CMA bootmem allocation fallback path now that the cma pointer no longer has to be stored in struct huge_bootmem_page. Link: https://lore.kernel.org/20260612035903.2468601-19-songmuchun@bytedance.com Signed-off-by: Muchun Song Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Frank van der Linden Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Mike Rapoport (Microsoft) Cc: Nicholas Piggin Cc: Oscar Salvador Cc: Oscar Salvador (SUSE) Cc: "Ritesh Harjani (IBM)" Cc: Usama Arif Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/hugetlb.c | 5 +---- mm/hugetlb_cma.c | 29 +++++++++++------------------ mm/internal.h | 2 -- 3 files changed, 12 insertions(+), 24 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 1bd9d12edb3e..5bfbfc7e463e 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3076,10 +3076,7 @@ static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid) */ INIT_LIST_HEAD(&m->list); m->hstate = h; - if (!hugetlb_early_cma(h)) { - m->cma = NULL; - m->flags = 0; - } + m->flags = hugetlb_early_cma(h) ? HUGE_BOOTMEM_CMA : 0; /* CMA pages: zone-crossing is validated in hugetlb_cma_reserve(). */ if (!hugetlb_early_cma(h) && diff --git a/mm/hugetlb_cma.c b/mm/hugetlb_cma.c index e487d0ffffc0..4dfce68b354a 100644 --- a/mm/hugetlb_cma.c +++ b/mm/hugetlb_cma.c @@ -59,31 +59,24 @@ struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, void * __init hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, bool node_exact) { struct cma *cma; - struct huge_bootmem_page *m; + void *m; int node; cma = hugetlb_cma[nid]; m = cma_reserve_early(cma, huge_page_size(h)); - if (!m) { - if (node_exact) - return NULL; + if (m || node_exact) + return m; - for_each_node_mask(node, hugetlb_bootmem_nodes) { - cma = hugetlb_cma[node]; - if (!cma || node == nid) - continue; - m = cma_reserve_early(cma, huge_page_size(h)); - if (m) - break; - } + for_each_node_mask(node, hugetlb_bootmem_nodes) { + cma = hugetlb_cma[node]; + if (!cma || node == nid) + continue; + m = cma_reserve_early(cma, huge_page_size(h)); + if (m) + return m; } - if (m) { - m->flags = HUGE_BOOTMEM_CMA; - m->cma = cma; - } - - return m; + return NULL; } static int __init cmdline_parse_hugetlb_cma(char *p) diff --git a/mm/internal.h b/mm/internal.h index 3401759924d9..39c9564fba0e 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -24,13 +24,11 @@ struct folio_batch; struct hstate; -struct cma; struct huge_bootmem_page { struct list_head list; struct hstate *hstate; unsigned long flags; - struct cma *cma; }; /* From 6d098029de0902372fdaa07e049db3866cbd8907 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 12 Jun 2026 11:59:03 +0800 Subject: [PATCH 067/501] mm/mm_init: fold __init_page_from_nid() into __init_deferred_page() __init_page_from_nid() no longer has external users and is only used locally in mm/mm_init.c under CONFIG_DEFERRED_STRUCT_PAGE_INIT. Fold it into its sole caller __init_deferred_page() and remove the separate helper declaration. Link: https://lore.kernel.org/20260612035903.2468601-20-songmuchun@bytedance.com Signed-off-by: Muchun Song Reviewed-by: Mike Rapoport (Microsoft) Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Frank van der Linden Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Oscar Salvador Cc: Oscar Salvador (SUSE) Cc: "Ritesh Harjani (IBM)" Cc: Usama Arif Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/internal.h | 1 - mm/mm_init.c | 42 ++++++++++++++++-------------------------- 2 files changed, 16 insertions(+), 27 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index 39c9564fba0e..dc10291f14bb 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1760,7 +1760,6 @@ static inline bool pte_needs_soft_dirty_wp(struct vm_area_struct *vma, pte_t pte void __meminit __init_single_page(struct page *page, unsigned long pfn, unsigned long zone, int nid); -void __meminit __init_page_from_nid(unsigned long pfn, int nid); /* shrinker related functions */ unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg, diff --git a/mm/mm_init.c b/mm/mm_init.c index 4ed4591dcdba..cfd0b2722d83 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -688,31 +688,6 @@ static __meminit void pageblock_migratetype_init_range(unsigned long pfn, } #endif -/* - * Initialize a reserved page unconditionally, finding its zone first. - */ -void __meminit __init_page_from_nid(unsigned long pfn, int nid) -{ - pg_data_t *pgdat; - int zid; - - pgdat = NODE_DATA(nid); - - for (zid = 0; zid < MAX_NR_ZONES; zid++) { - struct zone *zone = &pgdat->node_zones[zid]; - - if (zone_spans_pfn(zone, pfn)) - break; - } - __init_single_page(pfn_to_page(pfn), pfn, zid, nid); - - if (pageblock_aligned(pfn)) { - enum migratetype mt = - kho_scratch_migratetype(pfn, MIGRATE_MOVABLE); - init_pageblock_migratetype(pfn_to_page(pfn), mt, false); - } -} - #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT static inline void pgdat_set_deferred_range(pg_data_t *pgdat) { @@ -771,10 +746,25 @@ defer_init(int nid, unsigned long pfn, unsigned long end_pfn) static void __meminit __init_deferred_page(unsigned long pfn, int nid) { + pg_data_t *pgdat = NODE_DATA(nid); + int zid; + if (early_page_initialised(pfn, nid)) return; - __init_page_from_nid(pfn, nid); + for (zid = 0; zid < MAX_NR_ZONES; zid++) { + struct zone *zone = &pgdat->node_zones[zid]; + + if (zone_spans_pfn(zone, pfn)) + break; + } + __init_single_page(pfn_to_page(pfn), pfn, zid, nid); + + if (pageblock_aligned(pfn)) { + enum migratetype mt = + kho_scratch_migratetype(pfn, MIGRATE_MOVABLE); + init_pageblock_migratetype(pfn_to_page(pfn), mt, false); + } } #else static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} From efcf17453c7c3661aa6ac1c5d5a5b31284220797 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Fri, 26 Jun 2026 10:56:23 -0700 Subject: [PATCH 068/501] mm/kmemleak: skip the remaining scan phases when interrupted kmemleak_scan() scans the per-cpu sections, the struct page ranges and the task stacks in sequence. Each loop now bails out once scan_block() reports the scan was interrupted, but the later phases are still entered and only bail on their first scan_block() call. Jump straight to the gray list scan once a phase reports an interrupted scan, so the remaining scan phases are not entered at all. This does not change the scan results, it only avoids the pointless re-entry. Link: https://lore.kernel.org/20260626-kmemleak_improve-v1-1-d40c7616f64f@debian.org Signed-off-by: Breno Leitao Suggested-by: Oleg Nesterov Reviewed-by: Catalin Marinas Signed-off-by: Andrew Morton --- mm/kmemleak.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mm/kmemleak.c b/mm/kmemleak.c index ac2a44a1c4a5..e96e9efd19b0 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -1851,6 +1851,7 @@ static void kmemleak_scan(void) int __maybe_unused i; struct xarray dedup; int new_leaks = 0; + int stop = 0; jiffies_last_scan = jiffies; @@ -1897,7 +1898,7 @@ static void kmemleak_scan(void) for_each_possible_cpu(i) { if (scan_large_block(__per_cpu_start + per_cpu_offset(i), __per_cpu_end + per_cpu_offset(i))) - break; + goto scan_gray; } #endif @@ -1909,7 +1910,6 @@ static void kmemleak_scan(void) unsigned long start_pfn = zone->zone_start_pfn; unsigned long end_pfn = zone_end_pfn(zone); unsigned long pfn; - int stop = 0; for (pfn = start_pfn; pfn < end_pfn; pfn++) { struct page *page = pfn_to_online_page(pfn); @@ -1934,6 +1934,8 @@ static void kmemleak_scan(void) break; } put_online_mems(); + if (stop) + goto scan_gray; /* * Scanning the task stacks (may introduce false negatives). @@ -1945,6 +1947,7 @@ static void kmemleak_scan(void) * Scan the objects already referenced from the sections scanned * above. */ +scan_gray: scan_gray_list(); /* From 31fd8876a3447b62785976feb99b92ba5ea8b387 Mon Sep 17 00:00:00 2001 From: Yousef Alhouseen Date: Sun, 28 Jun 2026 02:43:14 +0200 Subject: [PATCH 069/501] tmpfs: zero unused folio tail for long symlinks shmem_symlink() marks the entire folio uptodate after copying only the NUL-terminated link target. The remainder of the freshly allocated folio is left uninitialized. Reclaim may pass the whole folio to a swap compressor. KMSAN observed sw842_compress() computing a checksum over the uninitialized tail. If the folio is written to a swap device, those bytes can also leave the kernel. Zero the remainder of the folio before marking it uptodate and dirty. Link: https://lore.kernel.org/20260628004314.27370-1-alhouseenyousef@gmail.com Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2") Reported-by: syzbot+bf5586280a66e9ccdfa9@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=bf5586280a66e9ccdfa9 Signed-off-by: Yousef Alhouseen Reviewed-by: Baolin Wang Cc: Baolin Wang Signed-off-by: Andrew Morton --- mm/shmem.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/shmem.c b/mm/shmem.c index 9001aaf3b7b9..40bcaac105f4 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -4067,6 +4067,7 @@ static int shmem_symlink(struct mnt_idmap *idmap, struct inode *dir, goto out_remove_offset; inode->i_op = &shmem_symlink_inode_operations; memcpy(folio_address(folio), symname, len); + folio_zero_range(folio, len, folio_size(folio) - len); folio_mark_uptodate(folio); folio_mark_dirty(folio); folio_unlock(folio); From 83fd5cbe6526c61ce00295cec843dea6e3bb4cbd Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Sat, 27 Jun 2026 11:58:58 -0700 Subject: [PATCH 070/501] radix-tree: add/correct some kernel-doc Correct some kernel-doc issues in radix-tree.h: - use "DOC:" so that a kernel-doc comment is parsed correctly (or we could just use "/*" for that comment) - add one function parameter description - add one function parameter name inside the prototype to fix these warnings: Warning: include/linux/radix-tree.h:164 Incorrect use of kernel-doc format: * radix_tree_deref_slot - dereference a slot Warning: include/linux/radix-tree.h:177 cannot understand function prototype: '* @slot: slot pointer, returned by radix_tree_lookup_slot Warning: include/linux/radix-tree.h:192 function parameter 'treelock' not described in 'radix_tree_deref_slot_protected' Warning: include/linux/radix-tree.h:309 function parameter '' not described in 'radix_tree_next_chunk' Link: https://lore.kernel.org/20260627185859.1632928-1-rdunlap@infradead.org Signed-off-by: Randy Dunlap Cc: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/radix-tree.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h index eae67015ce51..057edc4cbb6e 100644 --- a/include/linux/radix-tree.h +++ b/include/linux/radix-tree.h @@ -111,7 +111,7 @@ struct radix_tree_iter { }; /** - * Radix-tree synchronization + * DOC: Radix-tree synchronization * * The radix-tree API requires that users provide all synchronisation (with * specific exceptions, noted below). @@ -182,6 +182,7 @@ static inline void *radix_tree_deref_slot(void __rcu **slot) /** * radix_tree_deref_slot_protected - dereference a slot with tree lock held * @slot: slot pointer, returned by radix_tree_lookup_slot + * @treelock: caller must hold this spinlock * * Similar to radix_tree_deref_slot. The caller does not hold the RCU read * lock but it must hold the tree lock to prevent parallel updates. @@ -306,7 +307,7 @@ radix_tree_iter_init(struct radix_tree_iter *iter, unsigned long start) * Also it fills @iter with data about chunk: position in the tree (index), * its end (next_index), and constructs a bit mask for tagged iterating (tags). */ -void __rcu **radix_tree_next_chunk(const struct radix_tree_root *, +void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root, struct radix_tree_iter *iter, unsigned flags); /** From f078b0a0727c723cba4e55b331571f9804d459ca Mon Sep 17 00:00:00 2001 From: Xuewen Wang Date: Fri, 26 Jun 2026 13:37:00 +0800 Subject: [PATCH 071/501] mm: annotate data-race in cpu_needs_drain() KCSAN reports a data-race when cpu_needs_drain() reads another CPU's per-cpu folio_batch->nr without locking, while the owning CPU writes to it via folio_batch_add(). Reading a slightly stale value is harmless -- cpu_needs_drain() only decides whether to schedule a drain, and the next iteration of __lru_add_drain_all() will re-check. Use data_race() to annotate the intentional race. [akpm@linux-foundation.org: reindent cpu_needs_drain, per David & Lorenzo] Link: https://lore.kernel.org/20260626053700.2036899-1-wangxuewen@kylinos.cn Signed-off-by: Xuewen Wang Acked-by: David Hildenbrand (Arm) Reviewed-by: Pedro Falcato Reviewed-by: Lorenzo Stoakes Cc: Axel Rasmussen Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: Jann Horn Cc: Kairui Song Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Nhat Pham Cc: Shakeel Butt Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie Signed-off-by: Andrew Morton --- mm/swap.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mm/swap.c b/mm/swap.c index 460e56370b3c..58e4eff698cc 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -831,13 +831,13 @@ static bool cpu_needs_drain(unsigned int cpu) struct cpu_fbatches *fbatches = &per_cpu(cpu_fbatches, cpu); /* Check these in order of likelihood that they're not zero */ - return folio_batch_count(&fbatches->lru_add) || - folio_batch_count(&fbatches->lru_move_tail) || - folio_batch_count(&fbatches->lru_deactivate_file) || - folio_batch_count(&fbatches->lru_deactivate) || - folio_batch_count(&fbatches->lru_lazyfree) || - folio_batch_count(&fbatches->lru_activate) || - need_mlock_drain(cpu) || + return data_race(folio_batch_count(&fbatches->lru_add) || + folio_batch_count(&fbatches->lru_move_tail) || + folio_batch_count(&fbatches->lru_deactivate_file) || + folio_batch_count(&fbatches->lru_deactivate) || + folio_batch_count(&fbatches->lru_lazyfree) || + folio_batch_count(&fbatches->lru_activate) || + need_mlock_drain(cpu)) || has_bh_in_lru(cpu, NULL); } From 9909b088b1f0b913aa318605544a38bbd25e9abf Mon Sep 17 00:00:00 2001 From: Wenchao Hao Date: Fri, 26 Jun 2026 09:50:00 +0800 Subject: [PATCH 072/501] mm/zsmalloc: encode class index in obj value for lockless class lookup Patch series "mm/zsmalloc: reduce lock contention in zs_free()", v6. This series reduces lock contention in zs_free(), which dominates the unmap path under memory pressure on Android (LMK kills) and on x86 servers running zswap-heavy workloads. The current zs_free() takes pool->lock (rwlock, read side) just to look up the size_class for a handle, then takes class->lock and holds it across __free_zspage() which can call into the buddy allocator and acquire zone->lock. Two costs follow: * pool->lock reader-counter cacheline bouncing among concurrent zs_free() callers. * class->lock held across folio_put(), so any zone->lock wait fans out to every other zs_free() on the same class. The series tackles both: Patch 1: encode size_class index into obj alongside PFN and obj_idx, so zs_free() can locate the class without pool->lock. Patch 2: drop pool->lock from zs_free() on 64-bit; 32-bit unchanged. Patch 3: move zspage page-freeing out of class->lock. Patch 4: document the three free_zspage helper variants that result from the split in patch 3. Performance results: Test: each process independently mmap 256MB, write data, madvise MADV_PAGEOUT to swap out via zram (lzo-rle), then concurrent munmap. Raspberry Pi 4B (4-core ARM64 Cortex-A72): mode Base Patched Speedup single 59.0ms 56.0ms 1.05x multi 2p 94.6ms 66.7ms 1.42x multi 4p 202.9ms 110.6ms 1.83x x86 (20-core Intel i7-12700, 16 concurrent processes): mode Base Patched Speedup single 11.7ms 9.8ms 1.19x multi 2p 24.1ms 17.2ms 1.40x multi 4p 63.0ms 45.3ms 1.39x This patch (of 4): Encode the size_class index (class_idx) into the obj value so that zs_free() can determine the correct size_class without dereferencing the handle->obj->PFN->zpdesc->zspage->class chain under pool->lock. class_idx is invariant across page migration (only PFN is rewritten), so a lockless read of obj always yields a valid class_idx. Where obj has more bits below the PFN field than obj_idx alone needs, split that space into class_idx and obj_idx subfields: |<-- _PFN_BITS -->|<-- ZS_OBJ_CLASS_BITS -->|<-- ZS_OBJ_IDX_BITS -->| +-----------------+-------------------------+-----------------------+ | PFN | class_idx | obj_idx | +-----------------+-------------------------+-----------------------+ MSB ^ LSB | +-- ZS_OBJ_PFN_SHIFT The macro layout changes as follows: Before After Meaning ---------------- ------------------ ---------------------------- OBJ_INDEX_BITS ZS_OBJ_IDX_BITS width of obj_idx subfield OBJ_INDEX_MASK ZS_OBJ_IDX_MASK mask of obj_idx subfield (n/a) ZS_OBJ_CLASS_BITS width of class_idx subfield (n/a) ZS_OBJ_CLASS_MASK mask of class_idx subfield (n/a) ZS_OBJ_PFN_SHIFT bit offset of PFN in obj ZS_OBJ_CLASS_BITS folds to 0 (and the layout collapses to [PFN | obj_idx]) when obj has no spare bits, i.e. on 32-bit or on 64-bit fallback paths where MAX_POSSIBLE_PHYSMEM_BITS == BITS_PER_LONG (e.g. UML); zs_free() then falls back to pool->lock. [akpm@linux-foundation.org: fix obj_to_class_idx() defined but not used, remove duplicated #ifdef] Link: https://lore.kernel.org/20260626015003.2965881-1-haowenchao22@gmail.com Link: https://lore.kernel.org/20260626015003.2965881-2-haowenchao22@gmail.com Signed-off-by: Wenchao Hao Reviewed-by: Nhat Pham Cc: Barry Song Cc: Joshua Hahn Cc: Minchan Kim Cc: Sergey Senozhatsky Cc: Xueyuan Chen Signed-off-by: Andrew Morton --- mm/zsmalloc.c | 106 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 90 insertions(+), 16 deletions(-) diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 83f5820c45f9..2c7c27b5b38a 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -67,8 +67,8 @@ #define MAX_POSSIBLE_PHYSMEM_BITS MAX_PHYSMEM_BITS #else /* - * If this definition of MAX_PHYSMEM_BITS is used, OBJ_INDEX_BITS will just - * be PAGE_SHIFT + * If this definition of MAX_PHYSMEM_BITS is used, ZS_OBJ_PFN_SHIFT will + * just be PAGE_SHIFT */ #define MAX_POSSIBLE_PHYSMEM_BITS BITS_PER_LONG #endif @@ -88,8 +88,23 @@ #define OBJ_TAG_BITS 1 #define OBJ_TAG_MASK OBJ_ALLOCATED_TAG -#define OBJ_INDEX_BITS (BITS_PER_LONG - _PFN_BITS) -#define OBJ_INDEX_MASK ((_AC(1, UL) << OBJ_INDEX_BITS) - 1) +/* + * obj is encoded as [PFN | class_idx | obj_idx] within an unsigned long: + * + * |<-- _PFN_BITS -->|<-- ZS_OBJ_CLASS_BITS -->|<-- ZS_OBJ_IDX_BITS -->| + * +-----------------+-------------------------+-----------------------+ + * | PFN | class_idx | obj_idx | + * +-----------------+-------------------------+-----------------------+ + * MSB ^ LSB + * | + * +-- ZS_OBJ_PFN_SHIFT + * + * Encoding class_idx into obj lets zs_free() locate the size_class + * without holding pool->lock; class_idx is invariant across page + * migration (only PFN changes), so a lockless read of the obj value + * always yields a valid class_idx. + */ +#define ZS_OBJ_PFN_SHIFT (BITS_PER_LONG - _PFN_BITS) #define HUGE_BITS 1 #define FULLNESS_BITS 4 @@ -98,9 +113,61 @@ #define ZS_MAX_PAGES_PER_ZSPAGE (_AC(CONFIG_ZSMALLOC_CHAIN_SIZE, UL)) +/* + * Bits to index a page within a zspage = ceil(log2(ZS_MAX_PAGES_PER_ZSPAGE)). + * Computed at preprocessor time, for use in #if below. Kconfig + * restricts ZSMALLOC_CHAIN_SIZE to [4, 16]. + */ +#if ZS_MAX_PAGES_PER_ZSPAGE <= 4 +#define ZS_PAGES_PER_ZSPAGE_BITS 2 +#elif ZS_MAX_PAGES_PER_ZSPAGE <= 8 +#define ZS_PAGES_PER_ZSPAGE_BITS 3 +#elif ZS_MAX_PAGES_PER_ZSPAGE <= 16 +#define ZS_PAGES_PER_ZSPAGE_BITS 4 +#else +#error "ZSMALLOC_CHAIN_SIZE out of expected range [4,16]" +#endif + +/* + * Bits to index an object within a single PAGE_SIZE at the smallest + * possible object size: log2(PAGE_SIZE / 32) = PAGE_SHIFT - 5. + * 32 is the hard floor of ZS_MIN_ALLOC_SIZE. + */ +#define ZS_OBJS_PER_PAGE_BITS (PAGE_SHIFT - 5) + +/* + * Bits to index any object in the densest possible zspage. Below this, + * ZS_MIN_ALLOC_SIZE is auto-raised by the MAX(32, ...) formula -- still + * correct, but objects are coarser. + */ +#define ZS_OBJS_PER_ZSPAGE_BITS \ + (ZS_PAGES_PER_ZSPAGE_BITS + ZS_OBJS_PER_PAGE_BITS) + +/* + * Encode class_idx only when obj has spare bits; otherwise + * ZS_OBJ_CLASS_BITS folds to 0 (32-bit, or 64-bit UML/fallback). + */ +#if BITS_PER_LONG >= 64 && \ + ZS_OBJ_PFN_SHIFT >= (CLASS_BITS + 1) + ZS_OBJS_PER_ZSPAGE_BITS +#define ZS_OBJ_CLASS_BITS (CLASS_BITS + 1) +#else +#define ZS_OBJ_CLASS_BITS 0 +#endif +#define ZS_OBJ_CLASS_MASK ((_AC(1, UL) << ZS_OBJ_CLASS_BITS) - 1) + +#define ZS_OBJ_IDX_BITS (ZS_OBJ_PFN_SHIFT - ZS_OBJ_CLASS_BITS) +#define ZS_OBJ_IDX_MASK ((_AC(1, UL) << ZS_OBJ_IDX_BITS) - 1) + +/* + * Belt-and-suspenders: the #if above already guarantees this when + * class_idx is enabled. Catches future tweaks that bypass it. + */ +static_assert(ZS_OBJ_IDX_BITS >= ZS_PAGES_PER_ZSPAGE_BITS, + "zsmalloc: ZS_MIN_ALLOC_SIZE would exceed ZS_MAX_ALLOC_SIZE"); + /* ZS_MIN_ALLOC_SIZE must be multiple of ZS_ALIGN */ #define ZS_MIN_ALLOC_SIZE \ - MAX(32, (ZS_MAX_PAGES_PER_ZSPAGE << PAGE_SHIFT >> OBJ_INDEX_BITS)) + MAX(32, (ZS_MAX_PAGES_PER_ZSPAGE << PAGE_SHIFT >> ZS_OBJ_IDX_BITS)) /* each chunk includes extra space to keep handle */ #define ZS_MAX_ALLOC_SIZE PAGE_SIZE @@ -720,26 +787,29 @@ static struct zpdesc *get_next_zpdesc(struct zpdesc *zpdesc) static void obj_to_location(unsigned long obj, struct zpdesc **zpdesc, unsigned int *obj_idx) { - *zpdesc = pfn_zpdesc(obj >> OBJ_INDEX_BITS); - *obj_idx = (obj & OBJ_INDEX_MASK); + *zpdesc = pfn_zpdesc(obj >> ZS_OBJ_PFN_SHIFT); + *obj_idx = (obj & ZS_OBJ_IDX_MASK); } static void obj_to_zpdesc(unsigned long obj, struct zpdesc **zpdesc) { - *zpdesc = pfn_zpdesc(obj >> OBJ_INDEX_BITS); + *zpdesc = pfn_zpdesc(obj >> ZS_OBJ_PFN_SHIFT); } /** - * location_to_obj - get obj value encoded from (, ) + * location_to_obj - encode (, , ) into obj value * @zpdesc: zpdesc object resides in zspage * @obj_idx: object index + * @class_idx: size class index; ignored when ZS_OBJ_CLASS_BITS == 0 */ -static unsigned long location_to_obj(struct zpdesc *zpdesc, unsigned int obj_idx) +static unsigned long location_to_obj(struct zpdesc *zpdesc, unsigned int obj_idx, + unsigned int class_idx) { unsigned long obj; - obj = zpdesc_pfn(zpdesc) << OBJ_INDEX_BITS; - obj |= obj_idx & OBJ_INDEX_MASK; + obj = zpdesc_pfn(zpdesc) << ZS_OBJ_PFN_SHIFT; + obj |= (unsigned long)(class_idx & ZS_OBJ_CLASS_MASK) << ZS_OBJ_IDX_BITS; + obj |= obj_idx & ZS_OBJ_IDX_MASK; return obj; } @@ -1275,7 +1345,7 @@ static unsigned long obj_malloc(struct zs_pool *pool, kunmap_local(vaddr); mod_zspage_inuse(zspage, 1); - obj = location_to_obj(m_zpdesc, obj); + obj = location_to_obj(m_zpdesc, obj, zspage->class); record_obj(handle, obj); return obj; @@ -1643,9 +1713,12 @@ static void lock_zspage(struct zspage *zspage) } zspage_read_unlock(zspage); } -#endif /* CONFIG_COMPACTION */ -#ifdef CONFIG_COMPACTION +/* Folds to 0 when ZS_OBJ_CLASS_BITS == 0; no ifdef needed at callers. */ +static unsigned int obj_to_class_idx(unsigned long obj) +{ + return (obj >> ZS_OBJ_IDX_BITS) & ZS_OBJ_CLASS_MASK; +} static void replace_sub_page(struct size_class *class, struct zspage *zspage, struct zpdesc *newzpdesc, struct zpdesc *oldzpdesc) @@ -1761,7 +1834,8 @@ static int zs_page_migrate(struct page *newpage, struct page *page, old_obj = handle_to_obj(handle); obj_to_location(old_obj, &dummy, &obj_idx); - new_obj = (unsigned long)location_to_obj(newzpdesc, obj_idx); + new_obj = location_to_obj(newzpdesc, obj_idx, + obj_to_class_idx(old_obj)); record_obj(handle, new_obj); } } From 59e88952a82706bb330063353964d8a2525c3428 Mon Sep 17 00:00:00 2001 From: Wenchao Hao Date: Fri, 26 Jun 2026 09:50:01 +0800 Subject: [PATCH 073/501] mm/zsmalloc: drop pool->lock from zs_free on 64-bit systems With class_idx encoded in obj, zs_free() can locate the size_class without holding pool->lock on 64-bit systems. Page migration also takes class->lock and only rewrites the PFN field of obj, so: 1. read obj locklessly, 2. lock the size_class derived from obj's class_idx, 3. re-read obj under class->lock to get a stable PFN. This eliminates the rwlock read-side cacheline bouncing between zs_free() and migration/compaction on multi-core systems. Annotate handle_to_obj()/record_obj() with READ_ONCE()/WRITE_ONCE() to prevent load/store tearing on the lockless read path and silence KCSAN data race reports. When ZS_OBJ_CLASS_BITS == 0 (32-bit, or 64-bit with obj too narrow to hold class_idx), zs_free() keeps pool->lock. [akpm@linux-foundation.org: build fix] [akpm@linux-foundation.org: fix obj_to_class_idx() warning yet again] [baohua@kernel.org: update the comment about pool lock] Link: https://lore.kernel.org/20260725035733.53241-1-baohua@kernel.org Link: https://lore.kernel.org/20260626015003.2965881-3-haowenchao22@gmail.com Signed-off-by: Wenchao Hao Reviewed-by: Nhat Pham Reviewed-by: Barry Song Cc: Joshua Hahn Cc: Minchan Kim Cc: Sergey Senozhatsky Cc: Xueyuan Chen Cc: kernel test robot Signed-off-by: Andrew Morton --- mm/zsmalloc.c | 98 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 73 insertions(+), 25 deletions(-) diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 2c7c27b5b38a..2c7198868de0 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -21,6 +21,10 @@ * pool->lock * class->lock * zspage->lock + * + * When ZS_OBJ_CLASS_BITS > 0, zs_free() skips pool->lock; it picks + * the size_class from obj's encoded class_idx and serializes against + * page migration via class->lock. */ #include @@ -463,10 +467,13 @@ static void cache_free_zspage(struct zspage *zspage) kmem_cache_free(zspage_cachep, zspage); } -/* class->lock(which owns the handle) synchronizes races */ +/* + * Pairs with READ_ONCE() in handle_to_obj(): zs_free() may read the + * handle locklessly, so prevent store tearing here. + */ static void record_obj(unsigned long handle, unsigned long obj) { - *(unsigned long *)handle = obj; + WRITE_ONCE(*(unsigned long *)handle, obj); } static inline bool __maybe_unused is_first_zpdesc(struct zpdesc *zpdesc) @@ -816,7 +823,7 @@ static unsigned long location_to_obj(struct zpdesc *zpdesc, unsigned int obj_idx static unsigned long handle_to_obj(unsigned long handle) { - return *(unsigned long *)handle; + return READ_ONCE(*(unsigned long *)handle); } static inline bool obj_allocated(struct zpdesc *zpdesc, void *obj, @@ -1450,10 +1457,66 @@ static void obj_free(int class_size, unsigned long obj) mod_zspage_inuse(zspage, -1); } +#if (ZS_OBJ_CLASS_BITS > 0) || defined(CONFIG_COMPACTION) +/* Folds to 0 when ZS_OBJ_CLASS_BITS == 0; no ifdef needed at callers. */ +static unsigned int obj_to_class_idx(unsigned long obj) +{ + return (obj >> ZS_OBJ_IDX_BITS) & ZS_OBJ_CLASS_MASK; +} +#endif + +/* + * Resolve @handle to its zspage / size_class and acquire class->lock. + * + * When class_idx is encoded in obj (ZS_OBJ_CLASS_BITS > 0), it is + * invariant under page migration, so the handle can be read locklessly + * to pick the size_class. Once class->lock is held migration is + * blocked and the handle is re-read to obtain a stable PFN. + * + * Otherwise (32-bit, or 64-bit fallback paths like UML where the + * encoding is disabled), fall back to pool->lock for the lookup. + */ +#if ZS_OBJ_CLASS_BITS > 0 +static inline void obj_class_get_and_lock(struct zs_pool *pool, unsigned long handle, + unsigned long *objp, struct zspage **zspagep, + struct size_class **classp) + __acquires(&(*classp)->lock) +{ + struct zpdesc *f_zpdesc; + unsigned long obj; + + obj = handle_to_obj(handle); + *classp = pool->size_class[obj_to_class_idx(obj)]; + spin_lock(&(*classp)->lock); + /* Re-read under class->lock: PFN is now stable vs migration. */ + obj = handle_to_obj(handle); + obj_to_zpdesc(obj, &f_zpdesc); + *zspagep = get_zspage(f_zpdesc); + *objp = obj; +} +#else +static inline void obj_class_get_and_lock(struct zs_pool *pool, unsigned long handle, + unsigned long *objp, struct zspage **zspagep, + struct size_class **classp) + __acquires(&(*classp)->lock) +{ + struct zpdesc *f_zpdesc; + unsigned long obj; + + read_lock(&pool->lock); + obj = handle_to_obj(handle); + obj_to_zpdesc(obj, &f_zpdesc); + *zspagep = get_zspage(f_zpdesc); + *classp = zspage_class(pool, *zspagep); + spin_lock(&(*classp)->lock); + read_unlock(&pool->lock); + *objp = obj; +} +#endif + void zs_free(struct zs_pool *pool, unsigned long handle) { struct zspage *zspage; - struct zpdesc *f_zpdesc; unsigned long obj; struct size_class *class; int fullness; @@ -1461,17 +1524,7 @@ void zs_free(struct zs_pool *pool, unsigned long handle) if (IS_ERR_OR_NULL((void *)handle)) return; - /* - * The pool->lock protects the race with zpage's migration - * so it's safe to get the page from handle. - */ - read_lock(&pool->lock); - obj = handle_to_obj(handle); - obj_to_zpdesc(obj, &f_zpdesc); - zspage = get_zspage(f_zpdesc); - class = zspage_class(pool, zspage); - spin_lock(&class->lock); - read_unlock(&pool->lock); + obj_class_get_and_lock(pool, handle, &obj, &zspage, &class); class_stat_sub(class, ZS_OBJS_INUSE, 1); obj_free(class->size, obj); @@ -1714,12 +1767,6 @@ static void lock_zspage(struct zspage *zspage) zspage_read_unlock(zspage); } -/* Folds to 0 when ZS_OBJ_CLASS_BITS == 0; no ifdef needed at callers. */ -static unsigned int obj_to_class_idx(unsigned long obj) -{ - return (obj >> ZS_OBJ_IDX_BITS) & ZS_OBJ_CLASS_MASK; -} - static void replace_sub_page(struct size_class *class, struct zspage *zspage, struct zpdesc *newzpdesc, struct zpdesc *oldzpdesc) { @@ -1785,8 +1832,8 @@ static int zs_page_migrate(struct page *newpage, struct page *page, pool = zspage->pool; /* - * The pool migrate_lock protects the race between zpage migration - * and zs_free. + * The pool migrate_lock protects against races between zpage migration + * and zs_free(), but only when ZS_OBJ_CLASS_BITS does not apply. */ write_lock(&pool->lock); class = zspage_class(pool, zspage); @@ -1965,8 +2012,9 @@ static unsigned long __zs_compact(struct zs_pool *pool, unsigned long pages_freed = 0; /* - * protect the race between zpage migration and zs_free - * as well as zpage allocation/free + * Protect against races between zpage migration and zs_free() + * (only when ZS_OBJ_CLASS_BITS does not apply), as well as + * zpage allocation and free. */ write_lock(&pool->lock); spin_lock(&class->lock); From 7ef28e8b81426375ddec0f1c1459767cc9b2df2c Mon Sep 17 00:00:00 2001 From: Xueyuan Chen Date: Fri, 26 Jun 2026 09:50:02 +0800 Subject: [PATCH 074/501] mm/zsmalloc: drop class lock before freeing zspage Currently in zs_free(), the class->lock is held until the zspage is completely freed and the counters are updated. However, freeing pages back to the buddy allocator requires acquiring the zone lock. Under heavy memory pressure, zone lock contention can be severe. When this happens, the CPU holding the class->lock will stall waiting for the zone lock, thereby blocking all other CPUs attempting to acquire the same class->lock. This patch shrinks the critical section of the class->lock to reduce lock contention. By moving the actual page freeing process outside the class->lock, we can improve the concurrency performance of zs_free(). Testing on the RADXA O6 platform shows that with 12 CPUs concurrently performing zs_free() operations, the execution time is reduced by 20%. Link: https://lore.kernel.org/20260626015003.2965881-4-haowenchao22@gmail.com Signed-off-by: Xueyuan Chen Signed-off-by: Wenchao Hao Reviewed-by: Nhat Pham Reviewed-by: Joshua Hahn Reviewed-by: Barry Song Cc: Minchan Kim Cc: Sergey Senozhatsky Signed-off-by: Andrew Morton --- mm/zsmalloc.c | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 2c7198868de0..6d3ed196f6f9 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -877,13 +877,10 @@ static int trylock_zspage(struct zspage *zspage) return 0; } -static void __free_zspage(struct zs_pool *pool, struct size_class *class, - struct zspage *zspage) +static inline void __free_zspage_lockless(struct zspage *zspage) { struct zpdesc *zpdesc, *next; - assert_spin_locked(&class->lock); - VM_BUG_ON(get_zspage_inuse(zspage)); VM_BUG_ON(zspage->fullness != ZS_INUSE_RATIO_0); @@ -899,7 +896,13 @@ static void __free_zspage(struct zs_pool *pool, struct size_class *class, } while (zpdesc != NULL); cache_free_zspage(zspage); +} +static void __free_zspage(struct zs_pool *pool, struct size_class *class, + struct zspage *zspage) +{ + assert_spin_locked(&class->lock); + __free_zspage_lockless(zspage); class_stat_sub(class, ZS_OBJS_ALLOCATED, class->objs_per_zspage); atomic_long_sub(class->pages_per_zspage, &pool->pages_allocated); } @@ -1520,6 +1523,7 @@ void zs_free(struct zs_pool *pool, unsigned long handle) unsigned long obj; struct size_class *class; int fullness; + struct zspage *zspage_to_free = NULL; if (IS_ERR_OR_NULL((void *)handle)) return; @@ -1530,10 +1534,23 @@ void zs_free(struct zs_pool *pool, unsigned long handle) obj_free(class->size, obj); fullness = fix_fullness_group(class, zspage); - if (fullness == ZS_INUSE_RATIO_0) - free_zspage(pool, class, zspage); + if (fullness == ZS_INUSE_RATIO_0) { + if (trylock_zspage(zspage)) { + remove_zspage(class, zspage); + class_stat_sub(class, ZS_OBJS_ALLOCATED, + class->objs_per_zspage); + zspage_to_free = zspage; + } else { + kick_deferred_free(pool); + } + } spin_unlock(&class->lock); + + if (zspage_to_free) { + __free_zspage_lockless(zspage_to_free); + atomic_long_sub(class->pages_per_zspage, &pool->pages_allocated); + } cache_free_handle(handle); } EXPORT_SYMBOL_GPL(zs_free); From 0abea609f37e984647f9a2af95eb6256c3227b7a Mon Sep 17 00:00:00 2001 From: Wenchao Hao Date: Fri, 26 Jun 2026 09:50:03 +0800 Subject: [PATCH 075/501] mm/zsmalloc: document free_zspage helper variants After splitting __free_zspage() into a lockless core and a wrapper that does the class-stat bookkeeping, three similarly-named helpers coexist: free_zspage / __free_zspage / __free_zspage_lockless. Add a comment block above them describing what each does and where it is used, so the names are not easy to confuse. No functional change. Link: https://lore.kernel.org/20260626015003.2965881-5-haowenchao22@gmail.com Signed-off-by: Wenchao Hao Suggested-by: Nhat Pham Reviewed-by: Nhat Pham Reviewed-by: Barry Song Cc: Joshua Hahn Cc: Minchan Kim Cc: Sergey Senozhatsky Cc: Xueyuan Chen Signed-off-by: Andrew Morton --- mm/zsmalloc.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 6d3ed196f6f9..b5eadee0e8f0 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -877,6 +877,22 @@ static int trylock_zspage(struct zspage *zspage) return 0; } +/* + * Three free helpers, kept apart here: + * + * __free_zspage_lockless(): bare core; walks zpdescs and returns pages + * to the buddy allocator. Caller owns all zpdesc locks and has + * removed the zspage from its class list. Used by zs_free() outside + * class->lock so the buddy-side work does not stall the class. + * + * __free_zspage(): __free_zspage_lockless() + per-class accounting, + * under class->lock. Used by async_free_zspage(), the worker for + * zspages whose trylock_zspage() failed. + * + * free_zspage(): full wrapper - trylock zpdescs, remove from class + * list, call __free_zspage(); kicks deferred free on contention. + * Used by compaction. + */ static inline void __free_zspage_lockless(struct zspage *zspage) { struct zpdesc *zpdesc, *next; From 7c717f37ee78e5c279bfae41c32867e03069838b Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 25 Jun 2026 11:48:56 -0700 Subject: [PATCH 076/501] MAINTAINERS: move inactive maintainer to CREDITS Patch series "move alloc_tag.c file under mm/". Memory allocation profiling is ultimately an mm feature and now that we need to use some internal mm definitions in it [1], the time is right to move its implementation under mm/. The move is straight-forward, involving just alloc_tag.c file. Update config, makefiles and maintainers as well. This patch (of 2): Move Kent Overstreet from maintainers for Memory Allocation Profiling to CREDITS in recognition of his co-authorship and contributions to this feature. Link: https://lore.kernel.org/20260625184857.2193482-2-surenb@google.com Link: https://lore.kernel.org/all/20260622-alloc-trylock-v2-13-31f31367d420@google.com/ [1] Signed-off-by: Suren Baghdasaryan Acked-by: Lorenzo Stoakes Cc: Kent Overstreet Cc: Brendan Jackman Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Mike Rapoport Cc: Vlastimil Babka Cc: Hao Ge Cc: Harry Yoo (Oracle) Cc: SeongJae Park Signed-off-by: Andrew Morton --- CREDITS | 4 ++++ MAINTAINERS | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CREDITS b/CREDITS index 91c51c14e993..fff8e1490714 100644 --- a/CREDITS +++ b/CREDITS @@ -3096,6 +3096,10 @@ N: Jens Osterkamp E: jens@de.ibm.com D: Maintainer of Spidernet network driver for Cell +N: Kent Overstreet +E: kent.overstreet@linux.dev +D: Co-authored and contributed to Memory Allocation Profiling + N: Gadi Oxman E: gadio@netvision.net.il D: Original author and maintainer of IDE/ATAPI floppy/tape drivers diff --git a/MAINTAINERS b/MAINTAINERS index 2f9472c1a090..c1509e331ff5 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -16889,7 +16889,6 @@ F: tools/testing/memblock/ MEMORY ALLOCATION PROFILING M: Suren Baghdasaryan -M: Kent Overstreet R: Hao Ge L: linux-mm@kvack.org S: Maintained From 3c77682d80d07e673c7e6eb04f2211688ab494c4 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Thu, 25 Jun 2026 11:48:57 -0700 Subject: [PATCH 077/501] mm: move alloc tag to mm The alloc tagging work is really mm-specific, so move alloc_tag.c to mm/ and additionally update the MAINTAINERS entry to place it within memory management and port over the Kconfig and Makefile code to mm. Link: https://lore.kernel.org/20260625184857.2193482-3-surenb@google.com Signed-off-by: Lorenzo Stoakes Signed-off-by: Suren Baghdasaryan Reviewed-by: SeongJae Park Tested-by: Hao Ge Acked-by: Hao Ge Acked-by: Vlastimil Babka (SUSE) Acked-by: David Hildenbrand (Arm) Acked-by: Mike Rapoport (Microsoft) Acked-by: Harry Yoo (Oracle) Reviewed-by: Lorenzo Stoakes Cc: Brendan Jackman Cc: Kent Overstreet Cc: Liam R. Howlett Signed-off-by: Andrew Morton --- MAINTAINERS | 20 +- lib/Kconfig.debug | 28 -- lib/Makefile | 1 - mm/Kconfig.debug | 28 ++ mm/Makefile | 1 + mm/alloc_tag.c | 1029 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 1068 insertions(+), 39 deletions(-) create mode 100644 mm/alloc_tag.c diff --git a/MAINTAINERS b/MAINTAINERS index c1509e331ff5..8ee9446da9df 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -16887,16 +16887,6 @@ F: mm/mm_init.c F: mm/rodata_test.c F: tools/testing/memblock/ -MEMORY ALLOCATION PROFILING -M: Suren Baghdasaryan -R: Hao Ge -L: linux-mm@kvack.org -S: Maintained -F: Documentation/mm/allocation-profiling.rst -F: include/linux/alloc_tag.h -F: include/linux/pgalloc_tag.h -F: lib/alloc_tag.c - MEMORY CONTROLLER DRIVERS M: Krzysztof Kozlowski L: linux-kernel@vger.kernel.org @@ -16941,6 +16931,16 @@ T: quilt git://git.kernel.org/pub/scm/linux/kernel/git/akpm/25-new F: mm/ F: tools/mm/ +MEMORY MANAGEMENT - ALLOCATION PROFILING (ALLOC TAG) +M: Suren Baghdasaryan +R: Hao Ge +L: linux-mm@kvack.org +S: Maintained +F: Documentation/mm/allocation-profiling.rst +F: include/linux/alloc_tag.h +F: include/linux/pgalloc_tag.h +F: mm/alloc_tag.c + MEMORY MANAGEMENT - BALLOON M: Andrew Morton M: David Hildenbrand diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 1244dcac2294..b82515cde538 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1048,34 +1048,6 @@ config CODE_TAGGING bool select KALLSYMS -config MEM_ALLOC_PROFILING - bool "Enable memory allocation profiling" - default n - depends on MMU - depends on PROC_FS - depends on !DEBUG_FORCE_WEAK_PER_CPU - select CODE_TAGGING - select PAGE_EXTENSION - select SLAB_OBJ_EXT - help - Track allocation source code and record total allocation size - initiated at that code location. The mechanism can be used to track - memory leaks with a low performance and memory impact. - -config MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT - bool "Enable memory allocation profiling by default" - default y - depends on MEM_ALLOC_PROFILING - -config MEM_ALLOC_PROFILING_DEBUG - bool "Memory allocation profiler debugging" - default n - depends on MEM_ALLOC_PROFILING - select MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT - help - Adds warnings with helpful error messages for memory allocation - profiling. - source "lib/Kconfig.kasan" source "lib/Kconfig.kfence" source "lib/Kconfig.kmsan" diff --git a/lib/Makefile b/lib/Makefile index 7f75cc6edf94..531a0be88062 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -196,7 +196,6 @@ obj-$(CONFIG_OF_RECONFIG_NOTIFIER_ERROR_INJECT) += \ obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o obj-$(CONFIG_CODE_TAGGING) += codetag.o -obj-$(CONFIG_MEM_ALLOC_PROFILING) += alloc_tag.o lib-$(CONFIG_GENERIC_BUG) += bug.o diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug index 91b3e027b753..5737a504efbb 100644 --- a/mm/Kconfig.debug +++ b/mm/Kconfig.debug @@ -320,3 +320,31 @@ config PER_VMA_LOCK_STATS overhead in the page fault path. If in doubt, say N. + +config MEM_ALLOC_PROFILING + bool "Enable memory allocation profiling" + default n + depends on MMU + depends on PROC_FS + depends on !DEBUG_FORCE_WEAK_PER_CPU + select CODE_TAGGING + select PAGE_EXTENSION + select SLAB_OBJ_EXT + help + Track allocation source code and record total allocation size + initiated at that code location. The mechanism can be used to track + memory leaks with a low performance and memory impact. + +config MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT + bool "Enable memory allocation profiling by default" + default y + depends on MEM_ALLOC_PROFILING + +config MEM_ALLOC_PROFILING_DEBUG + bool "Memory allocation profiler debugging" + default n + depends on MEM_ALLOC_PROFILING + select MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT + help + Adds warnings with helpful error messages for memory allocation + profiling. diff --git a/mm/Makefile b/mm/Makefile index eff9f9e7e061..4fc713867b9b 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -147,3 +147,4 @@ obj-$(CONFIG_SHRINKER_DEBUG) += shrinker_debug.o obj-$(CONFIG_EXECMEM) += execmem.o obj-$(CONFIG_TMPFS_QUOTA) += shmem_quota.o obj-$(CONFIG_LAZY_MMU_MODE_KUNIT_TEST) += tests/lazy_mmu_mode_kunit.o +obj-$(CONFIG_MEM_ALLOC_PROFILING) += alloc_tag.o diff --git a/mm/alloc_tag.c b/mm/alloc_tag.c new file mode 100644 index 000000000000..e5b218176c5a --- /dev/null +++ b/mm/alloc_tag.c @@ -0,0 +1,1029 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ALLOCINFO_FILE_NAME "allocinfo" +#define MODULE_ALLOC_TAG_VMAP_SIZE (100000UL * sizeof(struct alloc_tag)) +#define SECTION_START(NAME) (CODETAG_SECTION_START_PREFIX NAME) +#define SECTION_STOP(NAME) (CODETAG_SECTION_STOP_PREFIX NAME) + +#ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT +static bool mem_profiling_support = true; +#else +static bool mem_profiling_support; +#endif + +/* + * Memory allocation profiling is permanently disabled and cannot be enabled. + * Must be called after setup_early_mem_profiling(). + */ +bool mem_alloc_profiling_permanently_disabled(void) +{ + return !mem_profiling_support; +} + +static struct codetag_type *alloc_tag_cttype; + +#ifdef CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU +DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag); +EXPORT_SYMBOL(_shared_alloc_tag); +#endif + +DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT, + mem_alloc_profiling_key); +EXPORT_SYMBOL(mem_alloc_profiling_key); + +DEFINE_STATIC_KEY_FALSE(mem_profiling_compressed); + +struct alloc_tag_kernel_section kernel_tags = { NULL, 0 }; +unsigned long alloc_tag_ref_mask; +int alloc_tag_ref_offs; + +struct allocinfo_private { + struct codetag_iterator iter; + struct codetag_iterator reported_iter; + bool print_header; +}; + +static void *allocinfo_start(struct seq_file *m, loff_t *pos) +{ + struct allocinfo_private *priv; + loff_t node = *pos; + + priv = (struct allocinfo_private *)m->private; + codetag_lock_module_list(alloc_tag_cttype); + if (node == 0) { + priv->print_header = true; + priv->iter = codetag_get_ct_iter(alloc_tag_cttype); + } else { + priv->iter = priv->reported_iter; + } + codetag_next_ct(&priv->iter); + return priv->iter.ct ? priv : NULL; +} + +static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos) +{ + struct allocinfo_private *priv = (struct allocinfo_private *)arg; + struct codetag *ct; + + priv->reported_iter = priv->iter; + ct = codetag_next_ct(&priv->iter); + (*pos)++; + if (!ct) + return NULL; + + return priv; +} + +static void allocinfo_stop(struct seq_file *m, void *arg) +{ + codetag_unlock_module_list(alloc_tag_cttype); +} + +static void print_allocinfo_header(struct seq_buf *buf) +{ + /* Output format version, so we can change it. */ + seq_buf_printf(buf, "allocinfo - version: 2.0\n"); + seq_buf_printf(buf, "# \n"); +} + +static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct) +{ + struct alloc_tag *tag = ct_to_alloc_tag(ct); + struct alloc_tag_counters counter = alloc_tag_read(tag); + s64 bytes = counter.bytes; + + seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls); + codetag_to_text(out, ct); + if (unlikely(alloc_tag_is_inaccurate(tag))) + seq_buf_printf(out, " accurate:no"); + seq_buf_putc(out, ' '); + seq_buf_putc(out, '\n'); +} + +static int allocinfo_show(struct seq_file *m, void *arg) +{ + struct allocinfo_private *priv = (struct allocinfo_private *)arg; + char *bufp; + size_t n = seq_get_buf(m, &bufp); + struct seq_buf buf; + + seq_buf_init(&buf, bufp, n); + if (priv->print_header) { + print_allocinfo_header(&buf); + priv->print_header = false; + } + alloc_tag_to_text(&buf, priv->iter.ct); + seq_commit(m, seq_buf_used(&buf)); + return 0; +} + +static const struct seq_operations allocinfo_seq_op = { + .start = allocinfo_start, + .next = allocinfo_next, + .stop = allocinfo_stop, + .show = allocinfo_show, +}; + +size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep) +{ + struct codetag_iterator iter; + struct codetag *ct; + struct codetag_bytes n; + unsigned int i, nr = 0; + + if (IS_ERR_OR_NULL(alloc_tag_cttype)) + return 0; + + if (can_sleep) + codetag_lock_module_list(alloc_tag_cttype); + else if (!codetag_trylock_module_list(alloc_tag_cttype)) + return 0; + + iter = codetag_get_ct_iter(alloc_tag_cttype); + while ((ct = codetag_next_ct(&iter))) { + struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct)); + + n.ct = ct; + n.bytes = counter.bytes; + + for (i = 0; i < nr; i++) + if (n.bytes > tags[i].bytes) + break; + + if (i < count) { + nr -= nr == count; + memmove(&tags[i + 1], + &tags[i], + sizeof(tags[0]) * (nr - i)); + nr++; + tags[i] = n; + } + } + + codetag_unlock_module_list(alloc_tag_cttype); + + return nr; +} + +void pgalloc_tag_split(struct folio *folio, int old_order, int new_order) +{ + int i; + struct alloc_tag *tag; + unsigned int nr_pages = 1 << new_order; + + if (!mem_alloc_profiling_enabled()) + return; + + tag = __pgalloc_tag_get(&folio->page); + if (!tag) + return; + + for (i = nr_pages; i < (1 << old_order); i += nr_pages) { + union pgtag_ref_handle handle; + union codetag_ref ref; + + if (get_page_tag_ref(folio_page(folio, i), &ref, &handle)) { + /* Set new reference to point to the original tag */ + alloc_tag_ref_set(&ref, tag); + update_page_tag_ref(handle, &ref); + put_page_tag_ref(handle); + } + } +} + +void pgalloc_tag_swap(struct folio *new, struct folio *old) +{ + union pgtag_ref_handle handle_old, handle_new; + union codetag_ref ref_old, ref_new; + struct alloc_tag *tag_old, *tag_new; + + if (!mem_alloc_profiling_enabled()) + return; + + tag_old = __pgalloc_tag_get(&old->page); + if (!tag_old) + return; + tag_new = __pgalloc_tag_get(&new->page); + if (!tag_new) + return; + + if (!get_page_tag_ref(&old->page, &ref_old, &handle_old)) + return; + if (!get_page_tag_ref(&new->page, &ref_new, &handle_new)) { + put_page_tag_ref(handle_old); + return; + } + + /* + * Clear tag references to avoid debug warning when using + * __alloc_tag_ref_set() with non-empty reference. + */ + set_codetag_empty(&ref_old); + set_codetag_empty(&ref_new); + + /* swap tags */ + __alloc_tag_ref_set(&ref_old, tag_new); + update_page_tag_ref(handle_old, &ref_old); + __alloc_tag_ref_set(&ref_new, tag_old); + update_page_tag_ref(handle_new, &ref_new); + + put_page_tag_ref(handle_old); + put_page_tag_ref(handle_new); +} + +static void shutdown_mem_profiling(bool remove_file) +{ + if (mem_alloc_profiling_enabled()) + static_branch_disable(&mem_alloc_profiling_key); + + if (!mem_profiling_support) + return; + + if (remove_file) + remove_proc_entry(ALLOCINFO_FILE_NAME, NULL); + mem_profiling_support = false; +} + +void __init alloc_tag_sec_init(void) +{ + struct alloc_tag *last_codetag; + + if (!mem_profiling_support) + return; + + if (!static_key_enabled(&mem_profiling_compressed)) + return; + + kernel_tags.first_tag = (struct alloc_tag *)kallsyms_lookup_name( + SECTION_START(ALLOC_TAG_SECTION_NAME)); + last_codetag = (struct alloc_tag *)kallsyms_lookup_name( + SECTION_STOP(ALLOC_TAG_SECTION_NAME)); + kernel_tags.count = last_codetag - kernel_tags.first_tag; + + /* Check if kernel tags fit into page flags */ + if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) { + shutdown_mem_profiling(false); /* allocinfo file does not exist yet */ + pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n", + kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS); + return; + } + + alloc_tag_ref_offs = (LRU_REFS_PGOFF - NR_UNUSED_PAGEFLAG_BITS); + alloc_tag_ref_mask = ((1UL << NR_UNUSED_PAGEFLAG_BITS) - 1); + pr_debug("Memory allocation profiling compression is using %d page flag bits!\n", + NR_UNUSED_PAGEFLAG_BITS); +} + +#ifdef CONFIG_MODULES + +static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE); +static struct vm_struct *vm_module_tags; +/* A dummy object used to indicate an unloaded module */ +static struct module unloaded_mod; +/* A dummy object used to indicate a module prepended area */ +static struct module prepend_mod; + +struct alloc_tag_module_section module_tags; + +static inline unsigned long alloc_tag_align(unsigned long val) +{ + if (!static_key_enabled(&mem_profiling_compressed)) { + /* No alignment requirements when we are not indexing the tags */ + return val; + } + + if (val % sizeof(struct alloc_tag) == 0) + return val; + return ((val / sizeof(struct alloc_tag)) + 1) * sizeof(struct alloc_tag); +} + +static bool ensure_alignment(unsigned long align, unsigned int *prepend) +{ + if (!static_key_enabled(&mem_profiling_compressed)) { + /* No alignment requirements when we are not indexing the tags */ + return true; + } + + /* + * If alloc_tag size is not a multiple of required alignment, tag + * indexing does not work. + */ + if (!IS_ALIGNED(sizeof(struct alloc_tag), align)) + return false; + + /* Ensure prepend consumes multiple of alloc_tag-sized blocks */ + if (*prepend) + *prepend = alloc_tag_align(*prepend); + + return true; +} + +static inline bool tags_addressable(void) +{ + unsigned long tag_idx_count; + + if (!static_key_enabled(&mem_profiling_compressed)) + return true; /* with page_ext tags are always addressable */ + + tag_idx_count = CODETAG_ID_FIRST + kernel_tags.count + + module_tags.size / sizeof(struct alloc_tag); + + return tag_idx_count < (1UL << NR_UNUSED_PAGEFLAG_BITS); +} + +static bool needs_section_mem(struct module *mod, unsigned long size) +{ + if (!mem_profiling_support) + return false; + + return size >= sizeof(struct alloc_tag); +} + +static bool clean_unused_counters(struct alloc_tag *start_tag, + struct alloc_tag *end_tag) +{ + struct alloc_tag *tag; + bool ret = true; + + for (tag = start_tag; tag <= end_tag; tag++) { + struct alloc_tag_counters counter; + + if (!tag->counters) + continue; + + counter = alloc_tag_read(tag); + if (!counter.bytes) { + free_percpu(tag->counters); + tag->counters = NULL; + } else { + ret = false; + } + } + + return ret; +} + +/* Called with mod_area_mt locked */ +static void clean_unused_module_areas_locked(void) +{ + MA_STATE(mas, &mod_area_mt, 0, module_tags.size); + struct module *val; + + mas_for_each(&mas, val, module_tags.size) { + struct alloc_tag *start_tag; + struct alloc_tag *end_tag; + + if (val != &unloaded_mod) + continue; + + /* Release area if all tags are unused */ + start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index); + end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last); + if (clean_unused_counters(start_tag, end_tag)) + mas_erase(&mas); + } +} + +/* Called with mod_area_mt locked */ +static bool find_aligned_area(struct ma_state *mas, unsigned long section_size, + unsigned long size, unsigned int prepend, unsigned long align) +{ + bool cleanup_done = false; + +repeat: + /* Try finding exact size and hope the start is aligned */ + if (!mas_empty_area(mas, 0, section_size - 1, prepend + size)) { + if (IS_ALIGNED(mas->index + prepend, align)) + return true; + + /* Try finding larger area to align later */ + mas_reset(mas); + if (!mas_empty_area(mas, 0, section_size - 1, + size + prepend + align - 1)) + return true; + } + + /* No free area, try cleanup stale data and repeat the search once */ + if (!cleanup_done) { + clean_unused_module_areas_locked(); + cleanup_done = true; + mas_reset(mas); + goto repeat; + } + + return false; +} + +static int vm_module_tags_populate(void) +{ + unsigned long phys_end = ALIGN_DOWN(module_tags.start_addr, PAGE_SIZE) + + (vm_module_tags->nr_pages << PAGE_SHIFT); + unsigned long new_end = module_tags.start_addr + module_tags.size; + + if (phys_end < new_end) { + struct page **next_page = vm_module_tags->pages + vm_module_tags->nr_pages; + unsigned long old_shadow_end = ALIGN(phys_end, MODULE_ALIGN); + unsigned long new_shadow_end = ALIGN(new_end, MODULE_ALIGN); + unsigned long more_pages; + unsigned long nr = 0; + + more_pages = ALIGN(new_end - phys_end, PAGE_SIZE) >> PAGE_SHIFT; + while (nr < more_pages) { + unsigned long allocated; + + allocated = alloc_pages_bulk_node(GFP_KERNEL | __GFP_NOWARN, + NUMA_NO_NODE, more_pages - nr, next_page + nr); + + if (!allocated) + break; + nr += allocated; + } + + if (nr < more_pages || + vmap_pages_range(phys_end, phys_end + (nr << PAGE_SHIFT), PAGE_KERNEL, + next_page, PAGE_SHIFT) < 0) { + release_pages_arg arg = { .pages = next_page }; + + /* Clean up and error out */ + release_pages(arg, nr); + return -ENOMEM; + } + + vm_module_tags->nr_pages += nr; + + /* + * Kasan allocates 1 byte of shadow for every 8 bytes of data. + * When kasan_alloc_module_shadow allocates shadow memory, + * its unit of allocation is a page. + * Therefore, here we need to align to MODULE_ALIGN. + */ + if (old_shadow_end < new_shadow_end) + kasan_alloc_module_shadow((void *)old_shadow_end, + new_shadow_end - old_shadow_end, + GFP_KERNEL); + } + + /* + * Mark the pages as accessible, now that they are mapped. + * With hardware tag-based KASAN, marking is skipped for + * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). + */ + kasan_unpoison_vmalloc((void *)module_tags.start_addr, + new_end - module_tags.start_addr, + KASAN_VMALLOC_PROT_NORMAL); + + return 0; +} + +static void *reserve_module_tags(struct module *mod, unsigned long size, + unsigned int prepend, unsigned long align) +{ + unsigned long section_size = module_tags.end_addr - module_tags.start_addr; + MA_STATE(mas, &mod_area_mt, 0, section_size - 1); + unsigned long offset; + void *ret = NULL; + + /* If no tags return error */ + if (size < sizeof(struct alloc_tag)) + return ERR_PTR(-EINVAL); + + /* + * align is always power of 2, so we can use IS_ALIGNED and ALIGN. + * align 0 or 1 means no alignment, to simplify set to 1. + */ + if (!align) + align = 1; + + if (!ensure_alignment(align, &prepend)) { + shutdown_mem_profiling(true); + pr_err("%s: alignment %lu is incompatible with allocation tag indexing. Memory allocation profiling is disabled!\n", + mod->name, align); + return ERR_PTR(-EINVAL); + } + + mas_lock(&mas); + if (!find_aligned_area(&mas, section_size, size, prepend, align)) { + ret = ERR_PTR(-ENOMEM); + goto unlock; + } + + /* Mark found area as reserved */ + offset = mas.index; + offset += prepend; + offset = ALIGN(offset, align); + if (offset != mas.index) { + unsigned long pad_start = mas.index; + + mas.last = offset - 1; + mas_store(&mas, &prepend_mod); + if (mas_is_err(&mas)) { + ret = ERR_PTR(xa_err(mas.node)); + goto unlock; + } + mas.index = offset; + mas.last = offset + size - 1; + mas_store(&mas, mod); + if (mas_is_err(&mas)) { + mas.index = pad_start; + mas_erase(&mas); + ret = ERR_PTR(xa_err(mas.node)); + } + } else { + mas.last = offset + size - 1; + mas_store(&mas, mod); + if (mas_is_err(&mas)) + ret = ERR_PTR(xa_err(mas.node)); + } +unlock: + mas_unlock(&mas); + + if (IS_ERR(ret)) + return ret; + + if (module_tags.size < offset + size) { + int grow_res; + + module_tags.size = offset + size; + if (mem_alloc_profiling_enabled() && !tags_addressable()) { + shutdown_mem_profiling(true); + pr_warn("With module %s there are too many tags to fit in %d page flag bits. Memory allocation profiling is disabled!\n", + mod->name, NR_UNUSED_PAGEFLAG_BITS); + } + + grow_res = vm_module_tags_populate(); + if (grow_res) { + shutdown_mem_profiling(true); + pr_err("Failed to allocate memory for allocation tags in the module %s. Memory allocation profiling is disabled!\n", + mod->name); + return ERR_PTR(grow_res); + } + } + + return (struct alloc_tag *)(module_tags.start_addr + offset); +} + +static void release_module_tags(struct module *mod, bool used) +{ + MA_STATE(mas, &mod_area_mt, module_tags.size, module_tags.size); + struct alloc_tag *start_tag; + struct alloc_tag *end_tag; + struct module *val; + + mas_lock(&mas); + mas_for_each_rev(&mas, val, 0) + if (val == mod) + break; + + if (!val) /* module not found */ + goto out; + + if (!used) + goto release_area; + + start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index); + end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last); + if (!clean_unused_counters(start_tag, end_tag)) { + struct alloc_tag *tag; + + for (tag = start_tag; tag <= end_tag; tag++) { + struct alloc_tag_counters counter; + + if (!tag->counters) + continue; + + counter = alloc_tag_read(tag); + pr_info("%s:%u module %s func:%s has %llu allocated at module unload\n", + tag->ct.filename, tag->ct.lineno, tag->ct.modname, + tag->ct.function, counter.bytes); + } + } else { + used = false; + } +release_area: + mas_store(&mas, used ? &unloaded_mod : NULL); + val = mas_prev_range(&mas, 0); + if (val == &prepend_mod) + mas_store(&mas, NULL); +out: + mas_unlock(&mas); +} + +static int load_module(struct module *mod, struct codetag *start, struct codetag *stop) +{ + /* Allocate module alloc_tag percpu counters */ + struct alloc_tag *start_tag; + struct alloc_tag *stop_tag; + struct alloc_tag *tag; + + /* percpu counters for core allocations are already statically allocated */ + if (!mod) + return 0; + + start_tag = ct_to_alloc_tag(start); + stop_tag = ct_to_alloc_tag(stop); + for (tag = start_tag; tag < stop_tag; tag++) { + WARN_ON(tag->counters); + tag->counters = alloc_percpu(struct alloc_tag_counters); + if (!tag->counters) { + while (--tag >= start_tag) { + free_percpu(tag->counters); + tag->counters = NULL; + } + pr_err("Failed to allocate memory for allocation tag percpu counters in the module %s\n", + mod->name); + return -ENOMEM; + } + + /* + * Avoid a kmemleak false positive. The pointer to the counters is stored + * in the alloc_tag section of the module and cannot be directly accessed. + */ + kmemleak_ignore_percpu(tag->counters); + } + return 0; +} + +static void replace_module(struct module *mod, struct module *new_mod) +{ + MA_STATE(mas, &mod_area_mt, 0, module_tags.size); + struct module *val; + + mas_lock(&mas); + mas_for_each(&mas, val, module_tags.size) { + if (val != mod) + continue; + + mas_store_gfp(&mas, new_mod, GFP_KERNEL); + break; + } + mas_unlock(&mas); +} + +static int __init alloc_mod_tags_mem(void) +{ + /* Map space to copy allocation tags */ + vm_module_tags = execmem_vmap(MODULE_ALLOC_TAG_VMAP_SIZE); + if (!vm_module_tags) { + pr_err("Failed to map %lu bytes for module allocation tags\n", + MODULE_ALLOC_TAG_VMAP_SIZE); + module_tags.start_addr = 0; + return -ENOMEM; + } + + vm_module_tags->pages = kmalloc_objs(struct page *, + get_vm_area_size(vm_module_tags) >> PAGE_SHIFT, + GFP_KERNEL | __GFP_ZERO); + if (!vm_module_tags->pages) { + free_vm_area(vm_module_tags); + return -ENOMEM; + } + + module_tags.start_addr = (unsigned long)vm_module_tags->addr; + module_tags.end_addr = module_tags.start_addr + MODULE_ALLOC_TAG_VMAP_SIZE; + /* Ensure the base is alloc_tag aligned when required for indexing */ + module_tags.start_addr = alloc_tag_align(module_tags.start_addr); + + return 0; +} + +static void __init free_mod_tags_mem(void) +{ + release_pages_arg arg = { .pages = vm_module_tags->pages }; + + module_tags.start_addr = 0; + release_pages(arg, vm_module_tags->nr_pages); + kfree(vm_module_tags->pages); + free_vm_area(vm_module_tags); +} + +#else /* CONFIG_MODULES */ + +static inline int alloc_mod_tags_mem(void) { return 0; } +static inline void free_mod_tags_mem(void) {} + +#endif /* CONFIG_MODULES */ + +/* See: Documentation/mm/allocation-profiling.rst */ +static int __init setup_early_mem_profiling(char *str) +{ + bool compressed = false; + bool enable; + + if (!str || !str[0]) + return -EINVAL; + + if (!strncmp(str, "never", 5)) { + enable = false; + mem_profiling_support = false; + pr_info("Memory allocation profiling is disabled!\n"); + } else { + char *token = strsep(&str, ","); + + if (kstrtobool(token, &enable)) + return -EINVAL; + + if (str) { + + if (strcmp(str, "compressed")) + return -EINVAL; + + compressed = true; + } + mem_profiling_support = true; + pr_info("Memory allocation profiling is enabled %s compression and is turned %s!\n", + compressed ? "with" : "without", str_on_off(enable)); + } + + if (enable != mem_alloc_profiling_enabled()) { + if (enable) + static_branch_enable(&mem_alloc_profiling_key); + else + static_branch_disable(&mem_alloc_profiling_key); + } + if (compressed != static_key_enabled(&mem_profiling_compressed)) { + if (compressed) + static_branch_enable(&mem_profiling_compressed); + else + static_branch_disable(&mem_profiling_compressed); + } + + return 0; +} +early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling); + +static __init bool need_page_alloc_tagging(void) +{ + if (static_key_enabled(&mem_profiling_compressed)) + return false; + + return mem_profiling_support; +} + +#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG +/* + * Track page allocations before page_ext is initialized. + * Some pages are allocated before page_ext becomes available, leaving + * their codetag uninitialized. Track these early PFNs so we can clear + * their codetag refs later to avoid warnings when they are freed. + * + * Each page is cast to a pfn_pool: the first few bytes hold metadata + * (next pointer and slot count), the remainder stores PFNs. + */ +struct pfn_pool { + struct pfn_pool *next; + atomic_t count; + unsigned long pfns[]; +}; + +#define PFN_POOL_SIZE ((PAGE_SIZE - offsetof(struct pfn_pool, pfns)) / \ + sizeof(unsigned long)) + +/* + * Skip early PFN recording for a page allocation. Reuses the + * %__GFP_NO_OBJ_EXT bit. Used by __alloc_tag_add_early_pfn() to avoid + * recursion when allocating pages for the early PFN tracking list + * itself. + * + * Codetags of the pages allocated with __GFP_NO_CODETAG should be + * cleared (via clear_page_tag_ref()) before freeing the pages to prevent + * alloc_tag_sub_check() from triggering a warning. + */ +#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT + +static struct pfn_pool *current_pfn_pool __initdata; + +static void __init __alloc_tag_add_early_pfn(unsigned long pfn) +{ + struct pfn_pool *pool; + int idx; + + do { + pool = READ_ONCE(current_pfn_pool); + if (!pool || atomic_read(&pool->count) >= PFN_POOL_SIZE) { + struct page *new_page = alloc_page(__GFP_HIGH | __GFP_NO_CODETAG); + struct pfn_pool *new; + + if (!new_page) { + pr_warn_once("early PFN tracking page allocation failed\n"); + return; + } + new = page_address(new_page); + new->next = pool; + atomic_set(&new->count, 0); + if (cmpxchg(¤t_pfn_pool, pool, new) != pool) { + clear_page_tag_ref(new_page); + __free_page(new_page); + continue; + } + pool = new; + } + idx = atomic_read(&pool->count); + if (idx >= PFN_POOL_SIZE) + continue; + if (atomic_cmpxchg(&pool->count, idx, idx + 1) == idx) + break; + } while (1); + + pool->pfns[idx] = pfn; +} + +typedef void alloc_tag_add_func(unsigned long pfn); +static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr __refdata = + RCU_INITIALIZER(__alloc_tag_add_early_pfn); + +void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags) +{ + alloc_tag_add_func *alloc_tag_add; + + if (static_key_enabled(&mem_profiling_compressed)) + return; + + /* Skip allocations for the tracking list itself to avoid recursion. */ + if (gfp_flags & __GFP_NO_CODETAG) + return; + + rcu_read_lock(); + alloc_tag_add = rcu_dereference(alloc_tag_add_early_pfn_ptr); + if (alloc_tag_add) + alloc_tag_add(pfn); + rcu_read_unlock(); +} + +static void __init clear_early_alloc_pfn_tag_refs(void) +{ + struct pfn_pool *pool, *next; + struct page *page; + int i; + + if (static_key_enabled(&mem_profiling_compressed)) + return; + + rcu_assign_pointer(alloc_tag_add_early_pfn_ptr, NULL); + /* Make sure we are not racing with __alloc_tag_add_early_pfn() */ + synchronize_rcu(); + + for (pool = current_pfn_pool; pool; pool = next) { + int nr_pfns = atomic_read(&pool->count); + + for (i = 0; i < nr_pfns; i++) { + unsigned long pfn = pool->pfns[i]; + + if (pfn_valid(pfn)) { + union pgtag_ref_handle handle; + union codetag_ref ref; + + if (get_page_tag_ref(pfn_to_page(pfn), &ref, &handle)) { + /* + * An early-allocated page could be freed and reallocated + * after its page_ext is initialized but before we clear it. + * In that case, it already has a valid tag set. + * We should not overwrite that valid tag + * with CODETAG_EMPTY. + * + * Note: there is still a small race window between checking + * ref.ct and calling set_codetag_empty(). We accept this + * race as it's unlikely and the extra complexity of atomic + * cmpxchg is not worth it for this debug-only code path. + */ + if (ref.ct) { + put_page_tag_ref(handle); + continue; + } + + set_codetag_empty(&ref); + update_page_tag_ref(handle, &ref); + put_page_tag_ref(handle); + } + } + } + + next = pool->next; + page = virt_to_page(pool); + clear_page_tag_ref(page); + __free_page(page); + } +} +#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */ +static inline void __init clear_early_alloc_pfn_tag_refs(void) {} +#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ + +static __init void init_page_alloc_tagging(void) +{ + clear_early_alloc_pfn_tag_refs(); +} + +struct page_ext_operations page_alloc_tagging_ops = { + .size = sizeof(union codetag_ref), + .need = need_page_alloc_tagging, + .init = init_page_alloc_tagging, +}; +EXPORT_SYMBOL(page_alloc_tagging_ops); + +#ifdef CONFIG_SYSCTL +/* + * Not using proc_do_static_key() directly to prevent enabling profiling + * after it was shut down. + */ +static int proc_mem_profiling_handler(const struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + if (write) { + /* + * Call from do_sysctl_args() which is a no-op since the same + * value was already set by setup_early_mem_profiling. + * Return success to avoid warnings from do_sysctl_args(). + */ + if (!current->mm) + return 0; + +#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG + /* User can't toggle profiling while debugging */ + return -EACCES; +#endif + if (!mem_profiling_support) + return -EINVAL; + } + + return proc_do_static_key(table, write, buffer, lenp, ppos); +} + + +static const struct ctl_table memory_allocation_profiling_sysctls[] = { + { + .procname = "mem_profiling", + .data = &mem_alloc_profiling_key, + .mode = 0644, + .proc_handler = proc_mem_profiling_handler, + }, +}; + +static void __init sysctl_init(void) +{ + register_sysctl_init("vm", memory_allocation_profiling_sysctls); +} +#else /* CONFIG_SYSCTL */ +static inline void sysctl_init(void) {} +#endif /* CONFIG_SYSCTL */ + +static int __init alloc_tag_init(void) +{ + const struct codetag_type_desc desc = { + .section = ALLOC_TAG_SECTION_NAME, + .tag_size = sizeof(struct alloc_tag), +#ifdef CONFIG_MODULES + .needs_section_mem = needs_section_mem, + .alloc_section_mem = reserve_module_tags, + .free_section_mem = release_module_tags, + .module_load = load_module, + .module_replaced = replace_module, +#endif + }; + int res; + + sysctl_init(); + + if (!mem_profiling_support) { + pr_info("Memory allocation profiling is not supported!\n"); + return 0; + } + + if (!proc_create_seq_private(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_seq_op, + sizeof(struct allocinfo_private), NULL)) { + pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME); + shutdown_mem_profiling(false); + return -ENOMEM; + } + + res = alloc_mod_tags_mem(); + if (res) { + pr_err("Failed to reserve address space for module tags, errno = %d\n", res); + shutdown_mem_profiling(true); + return res; + } + + alloc_tag_cttype = codetag_register_type(&desc); + if (IS_ERR(alloc_tag_cttype)) { + pr_err("Allocation tags registration failed, errno = %pe\n", alloc_tag_cttype); + free_mod_tags_mem(); + shutdown_mem_profiling(true); + return PTR_ERR(alloc_tag_cttype); + } + + return 0; +} +module_init(alloc_tag_init); From 5a00cae64de1bf217b0d3819fe9d23991cbe39ee Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Thu, 11 Jun 2026 14:56:57 +0200 Subject: [PATCH 078/501] mm/damon/core: reduce kernel stack usage The main thread function has recently grown to the point of exceeding stack frame size warning limits in some configurations. This is what I hit on s390 with clang and CONFIG_KASAN: mm/damon/core.c:3440:31: error: stack frame size (1352) exceeds limit (1280) in 'kdamond_fn' [-Werror,-Wframe-larger-than] 3440 | static int kdamond_fn(struct damon_ctx *ctx) The largest stack usage here is inside of the kdamond_tune_intervals(), so by marking that one as noinline_for_stack, the functions individually stay below the warning limit, though kdamond_fn() itself still uses hundreds of kilobytes for some reason. Link: https://lore.kernel.org/20260611125704.3386176-1-arnd@kernel.org Signed-off-by: Arnd Bergmann Reviewed-by: SeongJae Park Cc: Bill Wendling Cc: Justin Stitt Cc: Nathan Chancellor Cc: Quanmin Yan Signed-off-by: Andrew Morton --- mm/damon/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index cff932b3317d..eab553fcb0b5 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -2052,7 +2052,7 @@ static unsigned long damon_get_intervals_adaptation_bp(struct damon_ctx *c) return adaptation_bp; } -static void kdamond_tune_intervals(struct damon_ctx *c) +static noinline_for_stack void kdamond_tune_intervals(struct damon_ctx *c) { unsigned long adaptation_bp; struct damon_attrs new_attrs; From 567c26a132574ebf63dd72b381f2e4037de697be Mon Sep 17 00:00:00 2001 From: "Ritesh Harjani (IBM)" Date: Thu, 11 Jun 2026 08:39:32 +0530 Subject: [PATCH 079/501] include/linux/swap.h: remove unused leftovers This removed unused leftovers, most of them are forward structure declarations. Also removes SWAP_BATCH macro which isn't used any where in the code. Found these during manual code review. Link: https://lore.kernel.org/68591daf0d679e5a0072d63751f187d14613e2b0.1781146877.git.ritesh.list@gmail.com Signed-off-by: Ritesh Harjani (IBM) Reviewed-by: Barry Song Acked-by: Chris Li Acked-by: David Hildenbrand (Arm) Cc: Baoquan He Cc: Kairui Song Cc: Kemeng Shi Cc: Nhat Pham Signed-off-by: Andrew Morton --- include/linux/swap.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/include/linux/swap.h b/include/linux/swap.h index 8f0f68e245ba..46c25523d7b8 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -16,10 +16,6 @@ #include #include -struct notifier_block; - -struct bio; - #define SWAP_FLAG_PREFER 0x8000 /* set if swap priority specified */ #define SWAP_FLAG_PRIO_MASK 0x7fff #define SWAP_FLAG_DISCARD 0x10000 /* enable discard for swap */ @@ -29,7 +25,6 @@ struct bio; #define SWAP_FLAGS_VALID (SWAP_FLAG_PRIO_MASK | SWAP_FLAG_PREFER | \ SWAP_FLAG_DISCARD | SWAP_FLAG_DISCARD_ONCE | \ SWAP_FLAG_DISCARD_PAGES) -#define SWAP_BATCH 64 static inline int current_is_kswapd(void) { @@ -175,7 +170,6 @@ static inline void mm_account_reclaimed_pages(unsigned long pages) struct address_space; struct sysinfo; -struct writeback_control; struct zone; /* @@ -442,7 +436,6 @@ extern sector_t swapdev_block(int, pgoff_t); extern int __swap_count(swp_entry_t entry); extern bool swap_entry_swapped(struct swap_info_struct *si, swp_entry_t entry); extern int swp_swapcount(swp_entry_t entry); -struct backing_dev_info; extern struct swap_info_struct *get_swap_device(swp_entry_t entry); sector_t swap_folio_sector(struct folio *folio); From baff6d2d2708e109d2cbfa653e7ad2c5394eb168 Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Mon, 8 Jun 2026 20:29:19 -0400 Subject: [PATCH 080/501] mm: constify oom_control, scan_control, and alloc_context nodemask MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nodemasks in these structures may come from a variety of sources, including tasks and cpusets - and should never be modified by any code when being passed around inside another context. Link: https://lore.kernel.org/20260609002919.3967782-1-gourry@gourry.net Signed-off-by: Gregory Price Reviewed-by: Lorenzo Stoakes Acked-by: David Hildenbrand (Arm) Reviewed-by: Barry Song Acked-by: Vlastimil Babka (SUSE) Tested-by: SeongJae Park Acked-by: SeongJae Park Acked-by: Waiman Long Acked-by: Zi Yan Cc: Axel Rasmussen Cc: Baoquan He Cc: Brendan Jackman Cc: Chris Li Cc: David Rientjes Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Michal Hocko Cc: Michal Koutný Cc: Mike Rapoport Cc: Nhat Pham Cc: Shakeel Butt Cc: Suren Baghdasaryan Cc: Tejun Heo Cc: Wei Xu Cc: Yuanchu Xie Signed-off-by: Andrew Morton --- include/linux/cpuset.h | 4 ++-- include/linux/mm.h | 4 ++-- include/linux/mmzone.h | 6 +++--- include/linux/oom.h | 2 +- include/linux/swap.h | 2 +- kernel/cgroup/cpuset.c | 2 +- mm/internal.h | 2 +- mm/mmzone.c | 5 +++-- mm/page_alloc.c | 6 +++--- mm/show_mem.c | 9 ++++++--- mm/vmscan.c | 6 +++--- 11 files changed, 26 insertions(+), 22 deletions(-) diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h index 65d76a38974b..a80d38e752d2 100644 --- a/include/linux/cpuset.h +++ b/include/linux/cpuset.h @@ -83,7 +83,7 @@ extern bool cpuset_cpus_allowed_fallback(struct task_struct *p); extern nodemask_t cpuset_mems_allowed(struct task_struct *p); #define cpuset_current_mems_allowed (current->mems_allowed) void cpuset_init_current_mems_allowed(void); -int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask); +int cpuset_nodemask_valid_mems_allowed(const nodemask_t *nodemask); extern bool cpuset_current_node_allowed(int node, gfp_t gfp_mask); @@ -224,7 +224,7 @@ static inline nodemask_t cpuset_mems_allowed(struct task_struct *p) #define cpuset_current_mems_allowed (node_states[N_MEMORY]) static inline void cpuset_init_current_mems_allowed(void) {} -static inline int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask) +static inline int cpuset_nodemask_valid_mems_allowed(const nodemask_t *nodemask) { return 1; } diff --git a/include/linux/mm.h b/include/linux/mm.h index 485df9c2dbdd..2101e5205fc0 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4042,7 +4042,7 @@ extern int __meminit early_pfn_to_nid(unsigned long pfn); extern void mem_init(void); extern void __init mmap_init(void); -extern void __show_mem(unsigned int flags, nodemask_t *nodemask, int max_zone_idx); +extern void __show_mem(unsigned int flags, const nodemask_t *nodemask, int max_zone_idx); static inline void show_mem(void) { __show_mem(0, NULL, MAX_NR_ZONES - 1); @@ -4052,7 +4052,7 @@ extern void si_meminfo(struct sysinfo * val); extern void si_meminfo_node(struct sysinfo *val, int nid); extern __printf(3, 4) -void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...); +void warn_alloc(gfp_t gfp_mask, const nodemask_t *nodemask, const char *fmt, ...); extern void setup_per_cpu_pageset(void); diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 35d1a7643dc4..242ec3cb8b52 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -1815,7 +1815,7 @@ static inline int zonelist_node_idx(const struct zoneref *zoneref) struct zoneref *__next_zones_zonelist(struct zoneref *z, enum zone_type highest_zoneidx, - nodemask_t *nodes); + const nodemask_t *nodes); /** * next_zones_zonelist - Returns the next zone at or below highest_zoneidx within the allowed nodemask using a cursor within a zonelist as a starting point @@ -1834,7 +1834,7 @@ struct zoneref *__next_zones_zonelist(struct zoneref *z, */ static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z, enum zone_type highest_zoneidx, - nodemask_t *nodes) + const nodemask_t *nodes) { if (likely(!nodes && zonelist_zone_idx(z) <= highest_zoneidx)) return z; @@ -1860,7 +1860,7 @@ static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z, */ static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist, enum zone_type highest_zoneidx, - nodemask_t *nodes) + const nodemask_t *nodes) { return next_zones_zonelist(zonelist->_zonerefs, highest_zoneidx, nodes); diff --git a/include/linux/oom.h b/include/linux/oom.h index 7b02bc1d0a7e..00da05d227e6 100644 --- a/include/linux/oom.h +++ b/include/linux/oom.h @@ -30,7 +30,7 @@ struct oom_control { struct zonelist *zonelist; /* Used to determine mempolicy */ - nodemask_t *nodemask; + const nodemask_t *nodemask; /* Memory cgroup in which oom is invoked, or NULL for global oom */ struct mem_cgroup *memcg; diff --git a/include/linux/swap.h b/include/linux/swap.h index 46c25523d7b8..3f31b6a56788 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -345,7 +345,7 @@ extern void swap_setup(void); /* linux/mm/vmscan.c */ extern unsigned long zone_reclaimable_pages(struct zone *zone); extern unsigned long try_to_free_pages(struct zonelist *zonelist, int order, - gfp_t gfp_mask, nodemask_t *mask); + gfp_t gfp_mask, const nodemask_t *mask); unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx); #define MEMCG_RECLAIM_MAY_SWAP (1 << 1) diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 45944b3e31ca..92484b293689 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -4162,7 +4162,7 @@ nodemask_t cpuset_mems_allowed(struct task_struct *tsk) * * Are any of the nodes in the nodemask allowed in current->mems_allowed? */ -int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask) +int cpuset_nodemask_valid_mems_allowed(const nodemask_t *nodemask) { return nodes_intersects(*nodemask, current->mems_allowed); } diff --git a/mm/internal.h b/mm/internal.h index dc10291f14bb..430aa72a4575 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -675,7 +675,7 @@ void page_alloc_sysctl_init(void); */ struct alloc_context { struct zonelist *zonelist; - nodemask_t *nodemask; + const nodemask_t *nodemask; struct zoneref *preferred_zoneref; int migratetype; diff --git a/mm/mmzone.c b/mm/mmzone.c index 0c8f181d9d50..59dc3f2076a6 100644 --- a/mm/mmzone.c +++ b/mm/mmzone.c @@ -43,7 +43,8 @@ struct zone *next_zone(struct zone *zone) return zone; } -static inline int zref_in_nodemask(struct zoneref *zref, nodemask_t *nodes) +static inline int zref_in_nodemask(struct zoneref *zref, + const nodemask_t *nodes) { #ifdef CONFIG_NUMA return node_isset(zonelist_node_idx(zref), *nodes); @@ -55,7 +56,7 @@ static inline int zref_in_nodemask(struct zoneref *zref, nodemask_t *nodes) /* Returns the next zone at or below highest_zoneidx in a zonelist */ struct zoneref *__next_zones_zonelist(struct zoneref *z, enum zone_type highest_zoneidx, - nodemask_t *nodes) + const nodemask_t *nodes) { /* * Find the next suitable zone to use for the allocation. diff --git a/mm/page_alloc.c b/mm/page_alloc.c index a460811eaa4d..549fa83045eb 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -3978,7 +3978,7 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, return NULL; } -static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) +static void warn_alloc_show_mem(gfp_t gfp_mask, const nodemask_t *nodemask) { unsigned int filter = SHOW_MEM_FILTER_NODES; @@ -3998,7 +3998,7 @@ static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) mem_cgroup_show_protected_memory(NULL); } -void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) +void warn_alloc(gfp_t gfp_mask, const nodemask_t *nodemask, const char *fmt, ...) { struct va_format vaf; va_list args; @@ -4680,7 +4680,7 @@ check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac) return false; } -static void check_alloc_stall_warn(gfp_t gfp_mask, nodemask_t *nodemask, +static void check_alloc_stall_warn(gfp_t gfp_mask, const nodemask_t *nodemask, unsigned int order, unsigned long alloc_start_time) { static DEFINE_SPINLOCK(alloc_stall_lock); diff --git a/mm/show_mem.c b/mm/show_mem.c index 43aca5a2ac99..1b721a8ade67 100644 --- a/mm/show_mem.c +++ b/mm/show_mem.c @@ -116,7 +116,8 @@ void si_meminfo_node(struct sysinfo *val, int nid) * Determine whether the node should be displayed or not, depending on whether * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). */ -static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask) +static bool show_mem_node_skip(unsigned int flags, int nid, + const nodemask_t *nodemask) { if (!(flags & SHOW_MEM_FILTER_NODES)) return false; @@ -177,7 +178,8 @@ static bool node_has_managed_zones(pg_data_t *pgdat, int max_zone_idx) * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's * cpuset. */ -static void show_free_areas(unsigned int filter, nodemask_t *nodemask, int max_zone_idx) +static void show_free_areas(unsigned int filter, const nodemask_t *nodemask, + int max_zone_idx) { unsigned long free_pcp = 0; int cpu, nid; @@ -402,7 +404,8 @@ static void show_free_areas(unsigned int filter, nodemask_t *nodemask, int max_z show_swap_cache_info(); } -void __show_mem(unsigned int filter, nodemask_t *nodemask, int max_zone_idx) +void __show_mem(unsigned int filter, const nodemask_t *nodemask, + int max_zone_idx) { unsigned long total = 0, reserved = 0, highmem = 0; struct zone *zone; diff --git a/mm/vmscan.c b/mm/vmscan.c index 1a142c58700d..de48b48470e6 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -79,7 +79,7 @@ struct scan_control { * Nodemask of nodes allowed by the caller. If NULL, all nodes * are scanned. */ - nodemask_t *nodemask; + const nodemask_t *nodemask; /* * The memory cgroup that hit its limit and as a result is the @@ -6591,7 +6591,7 @@ static bool allow_direct_reclaim(pg_data_t *pgdat) * happens, the page allocator should not consider triggering the OOM killer. */ static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist, - nodemask_t *nodemask) + const nodemask_t *nodemask) { struct zoneref *z; struct zone *zone; @@ -6671,7 +6671,7 @@ static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist, } unsigned long try_to_free_pages(struct zonelist *zonelist, int order, - gfp_t gfp_mask, nodemask_t *nodemask) + gfp_t gfp_mask, const nodemask_t *nodemask) { unsigned long nr_reclaimed; struct scan_control sc = { From a4519e5b648ae77d2ecb0343317169513e03dc6a Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Mon, 8 Jun 2026 07:32:42 -0700 Subject: [PATCH 081/501] mm/swap_state: remove unnecessary lru_add_drain() from readahead swap_cluster_readahead() and swap_vma_readahead() end the readahead loop with an explicit lru_add_drain() call. That drain is a leftover from 2.6.12 era code and serves no functional purpose for the callers: - do_swap_page() ignores LRU residency for the readahead folios; it only needs the target folio it called swapin_readahead() for, and if the write-fault path needs the target folio on the LRU to count references accurately, it runs its own lru_add_drain() at the wp_can_reuse_anon_folio() and do_swap_page() sites. - shmem_swapin_cluster() immediately locks the returned folio, waits for writeback, then operates on it - LRU residency of either the target or the readahead folios is irrelevant. - try_to_unuse() likewise locks the folio and calls unuse_pte() without depending on LRU presence. Folios newly added to the swap cache by the readahead loop sit in the per-CPU LRU folio_batch and will be drained naturally as the batch fills (FOLIO_BATCH_SIZE),by the next reclaim/compaction lru_add_drain_all() and so on. The unconditional drain only synchronously flushes a partial batch and forces contention on lruvec_lock. On a 176-CPU production host running a memory-pressured workload, this path was observed to call folio_batch_move_lru() from swap_cluster_readahead() ~28K/min, a very large source of LRU lock traffic. This is a direct continuation of the cleanup started in commit 1aa43598c03b ("mm: remove unnecessary calls to lru_add_drain") which removed the equivalent drain from free_pages_and_swap_cache() with the same rationale. A detailed reasoning for this is present in [1]. Remove both drains. Link: https://lore.kernel.org/20260608143242.2869392-1-usama.arif@linux.dev Link: https://lore.kernel.org/all/dca2824e8e88e826c6b260a831d79089b5b9c79d.camel@surriel.com/T/#u [1] Signed-off-by: Usama Arif Acked-by: Shakeel Butt Reviewed-by: Barry Song Reviewed-by: Kairui Song Acked-by: Johannes Weiner Cc: Baoquan He Cc: Chris Li Cc: David Hildenbrand Cc: Kemeng Shi Cc: Nhat Pham Cc: Rik van Riel Signed-off-by: Andrew Morton --- mm/swap_state.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/mm/swap_state.c b/mm/swap_state.c index 9c3a5cf99778..6fd6e3415b71 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -836,7 +836,6 @@ struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask, } blk_finish_plug(&plug); swap_read_unplug(splug); - lru_add_drain(); /* Push any new pages onto the LRU now */ skip: /* The page was likely read above, so no need for plugging here */ return swap_cache_read_folio(entry, gfp_mask, mpol, ilx, NULL, false); @@ -951,7 +950,6 @@ static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask, pte_unmap(pte); blk_finish_plug(&plug); swap_read_unplug(splug); - lru_add_drain(); skip: /* The folio was likely read above, so no need for plugging here */ folio = swap_cache_read_folio(targ_entry, gfp_mask, mpol, targ_ilx, From aaa98b100ea8d779d3c4dae516dc5a267f51418b Mon Sep 17 00:00:00 2001 From: Imran Khan Date: Thu, 4 Jun 2026 21:42:45 +0800 Subject: [PATCH 082/501] mm/vmstat: avoid taking zone lock in /proc/buddyinfo reads frag_show_print() just reads zone->free_area[order].nr_free, so it can safely do this without needing the zone->lock. Pass nolock=true from frag_show(), so that walk_zones_in_node() can skip the zone->lock acquisition. Link: https://lore.kernel.org/20260604134245.1580287-1-imran.f.khan@oracle.com Signed-off-by: Imran Khan Reviewed-by: Joshua Hahn Acked-by: Vlastimil Babka (SUSE) Reviewed-by: Lorenzo Stoakes Acked-by: David Hildenbrand (Arm) Cc: Liam R. Howlett Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Signed-off-by: Andrew Morton --- mm/vmstat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/vmstat.c b/mm/vmstat.c index f534972f517d..7b93fbf9af09 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1568,7 +1568,7 @@ static void frag_show_print(struct seq_file *m, pg_data_t *pgdat, static int frag_show(struct seq_file *m, void *arg) { pg_data_t *pgdat = (pg_data_t *)arg; - walk_zones_in_node(m, pgdat, true, false, frag_show_print); + walk_zones_in_node(m, pgdat, true, true, frag_show_print); return 0; } From e4742be45ea45bf554399ce89a09f71e525d7981 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 28 Jun 2026 14:54:40 -0700 Subject: [PATCH 083/501] samples/damon/wsse: handle damon_start() failure Patch series "samples/damon: handle damon_{start,stop}() failures". All DAMON sample modules are not correctly handling failures from damon_start(). Among those, mtier also has an additional problem for handling of damon_stop() failures. wsse and prcl also have a problem in their damon_call() failure handling. As a result, memory leaks, next DAMON operation disruptions, and use-after-free can happen. Fix those. Note that only the damon_start() failure caused issues can reliably be reproduced. Reproducing those issues require the admin permission, though. This patch (of 6): damon_sample_wsse_start() callers assume it will clean up resources when it fails. And the function does the cleanup for context buildup failures. However, it is not doing the cleanup for damon_start() failure. As a result, when damon_start() fails, it leaks the memory for DAMON context. Free the context in case of the failure to fix the issues. Note that the issue can reliably be reproduced because the module calls damon_start() in the exclusive mode. For example, $ sudo damo start $ echo $$ | sudo tee /sys/module/damon_sample_wsse/parameters/target_pid $ echo Y | sudo tee /sys/module/damon_sample_wsse/parameters/enabled $ sudo cat /proc/allocinfo | grep damon_new_ctx Because the first command is running another DAMON instance, the third command fails the damon_start() call because the new DAMON instance cannot exclusively run. And without this fix, by repeating the third and the fourth commands above, we can show the memory consumption is only increasing due to the leaks. It requires the sudo permission though. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260628215447.96166-2-sj@kernel.org Link: https://lore.kernel.org/20260609145814.70163-1-sj@kernel.org [1] Fixes: b757c6cfc696 ("samples/damon/wsse: start and stop DAMON as the user requests") Signed-off-by: SJ Park Reviewed-by: Zenghui Yu Cc: # 6.14.x Signed-off-by: Andrew Morton --- samples/damon/wsse.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/samples/damon/wsse.c b/samples/damon/wsse.c index 799ad4443943..bbd9392ab5b3 100644 --- a/samples/damon/wsse.c +++ b/samples/damon/wsse.c @@ -87,8 +87,10 @@ static int damon_sample_wsse_start(void) target->pid = target_pidp; err = damon_start(&ctx, 1, true); - if (err) + if (err) { + damon_destroy_ctx(ctx); return err; + } repeat_call_control.data = ctx; return damon_call(ctx, &repeat_call_control); } From 8b724349229bb6ebbf781178be011ba9bc2cca57 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 28 Jun 2026 14:54:41 -0700 Subject: [PATCH 084/501] samples/damon/prcl: handle damon_start() failure damon_sample_prcl_start() callers assume it will clean up resources when it fails. And the function does the cleanup for context buildup failures. However, it is not doing the cleanup for damon_start() failure. As a result, when damon_start() fails, it leaks the memory for DAMON context. Free the context in case of the failure to fix the issues. Note that the issue can reliably be reproduced because the module calls damon_start() in the exclusive mode. For example, $ sudo damo start $ echo $$ | sudo tee /sys/module/damon_sample_prcl/parameters/target_pid $ echo Y | sudo tee /sys/module/damon_sample_prcl/parameters/enabled $ sudo cat /proc/allocinfo | grep damon_new_ctx Because the first command is running another DAMON instance, the third command fails the damon_start() call because the new DAMON instance cannot exclusively run. And without this fix, by repeating the third and the fourth commands above, we can show the memory consumption is only increasing due to the leaks. It requires the sudo permission though. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260628215447.96166-3-sj@kernel.org Link: https://lore.kernel.org/20260609145814.70163-1-sj@kernel.org [1] Fixes: 2aca254620a8 ("samples/damon: introduce a skeleton of a smaple DAMON module for proactive reclamation") Signed-off-by: SJ Park Reviewed-by: Zenghui Yu Cc: # 6.14.x Signed-off-by: Andrew Morton --- samples/damon/prcl.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/samples/damon/prcl.c b/samples/damon/prcl.c index b7c50f2656ce..0db259894691 100644 --- a/samples/damon/prcl.c +++ b/samples/damon/prcl.c @@ -106,8 +106,10 @@ static int damon_sample_prcl_start(void) damon_set_schemes(ctx, &scheme, 1); err = damon_start(&ctx, 1, true); - if (err) + if (err) { + damon_destroy_ctx(ctx); return err; + } repeat_call_control.data = ctx; return damon_call(ctx, &repeat_call_control); From c7230d08ee79b13127bd2b45a3648d361ac912fc Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 28 Jun 2026 14:54:42 -0700 Subject: [PATCH 085/501] samples/damon/mtier: handle damon_start() failure damon_sample_mtier_start() callers assume it will clean up resources when it fails. And the function does the cleanup for context buildup failures. However, it is not doing the cleanup for damon_start() failure. As a result, when damon_start() fails, it could leak the memory for DAMON context. Also, if damon_start() fails for only the second context, the first context will indefinitely run, and avoid starting other DAMON contexts since it is running in the exclusive mode. Stop possibly started DAMON context and free the contexts in case of the failure to fix the issues. Note that the issue can reliably be reproduced because the module calls damon_start() in the exclusive mode. For example, $ sudo damo start $ echo Y | sudo tee /sys/module/damon_sample_mtier/parameters/enabled $ sudo cat /proc/allocinfo | grep damon_new_ctx Because the first command is running another DAMON instance, the second command fails the damon_start() call because the new DAMON instance cannot exclusively run. And without this fix, by repeating the second and the third commands above, we can show the memory consumption is only increasing due to the leaks. It requires the sudo permission though. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260628215447.96166-4-sj@kernel.org Link: https://lore.kernel.org/20260608112455.274231F00893@smtp.kernel.org [1] Fixes: 82a08bde3cf7 ("samples/damon: implement a DAMON module for memory tiering") Signed-off-by: SJ Park Reviewed-by: Zenghui Yu Cc: # 6.16.x Signed-off-by: Andrew Morton --- samples/damon/mtier.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/samples/damon/mtier.c b/samples/damon/mtier.c index 3785b0c7ffb1..564212e054de 100644 --- a/samples/damon/mtier.c +++ b/samples/damon/mtier.c @@ -177,6 +177,7 @@ static struct damon_ctx *damon_sample_mtier_build_ctx(bool promote) static int damon_sample_mtier_start(void) { struct damon_ctx *ctx; + int err; ctx = damon_sample_mtier_build_ctx(true); if (!ctx) @@ -188,7 +189,15 @@ static int damon_sample_mtier_start(void) return -ENOMEM; } ctxs[1] = ctx; - return damon_start(ctxs, 2, true); + err = damon_start(ctxs, 2, true); + if (!err) + return 0; + + if (damon_is_running(ctxs[0])) + damon_stop(ctxs, 1); + damon_destroy_ctx(ctxs[0]); + damon_destroy_ctx(ctxs[1]); + return err; } static void damon_sample_mtier_stop(void) From 9dc5b6d66fd51b103eff21ed0df3e292f489ebc0 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 28 Jun 2026 14:54:43 -0700 Subject: [PATCH 086/501] samples/damon/mtier: handle damon_stop() failure damon_sample_mtier_stop() assumes its damon_stop() call will always successfully stops the two DAMON contexts. Hence it deallocates the two DAMON contexts after the damon_stop() call. However, if a given context is already stopped, damon_stop() fails and returns an error while letting the DAMON contexts that have not yet stopped keep running. This kind of unexpected early DAMON context stops could happen due to memory allocation failures in kdamond_fn(). Because damon_sample_mtier_stop() just deallocates all DAMON contexts with damon_target and damon_region objects that are linked to the contexts, the execution of the unstopped DAMON context (kdamond) ends up using the memory that freed (use-after-free). Fix the issue by separating the damon_stop() to be invoked per context. Note that DAMON_SYSFS also allows multiple DAMON contexts execution. But, it calls damon_stop() for each context one by one. Hence this issue is only in mtier. For the long term, it would be better to refactor damon_stop() to always ensure stopping all contexts regardless of the failures in the middle. Make this fix in the current way, though, to keep it simple and easy to backport. I will do the refactoring later. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260628215447.96166-5-sj@kernel.org Link: https://lore.kernel.org/20260609014219.3013-1-sj@kernel.org [1] Fixes: 82a08bde3cf7 ("samples/damon: implement a DAMON module for memory tiering") Signed-off-by: SJ Park Reviewed-by: Zenghui Yu Cc: # 6.16.x Signed-off-by: Andrew Morton --- samples/damon/mtier.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/damon/mtier.c b/samples/damon/mtier.c index 564212e054de..e567f4edd80e 100644 --- a/samples/damon/mtier.c +++ b/samples/damon/mtier.c @@ -202,7 +202,8 @@ static int damon_sample_mtier_start(void) static void damon_sample_mtier_stop(void) { - damon_stop(ctxs, 2); + damon_stop(ctxs, 1); + damon_stop(&ctxs[1], 1); damon_destroy_ctx(ctxs[0]); damon_destroy_ctx(ctxs[1]); } From a2c6fa6c23ad87c61e1379b05dc05cf5fed4bf8d Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 28 Jun 2026 14:54:44 -0700 Subject: [PATCH 087/501] samples/damon/wsse: stop and free damon ctx when damon_call() fails damon_sample_wsse_start() calls damon_call() right after damon_start() is succeeded. The kdamond that has started by the damon_start() could be terminated by itself before or in the middle of the damon_call() execution. There could be multiple reasons for such a stop including monitoring target process termination and kdamond_fn() internal memory allocation failures. In the case, damon_call() will fail and return an error without cleaning up the DAMON context object. The damon_sample_wsse_start() caller assumes it would clean up the object, though. When the user requests to start DAMON again, damon_sample_wsse_start() is called again, allocates a new DAMON context object and overwrites the pointer for the previous object. As a result, the previous context object is leaked. Safely stop the kdamond and deallocate the context object when the failure is returned. Note that the kdamond should be stopped first, because damon_call() failure means not complete termination of the kdamond but only the fact that the termination process has started. The user impact shouldn't be that significant because the race is not easy to happen, and only up to one DAMON context object can be leaked per race. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260628215447.96166-6-sj@kernel.org Link: https://lore.kernel.org/20260610034828.4632-1-sj@kernel.org [1] Fixes: cc9c1b8c205b ("samples/damon/wsse: use damon_call() repeat mode instead of damon_callback") Signed-off-by: SJ Park Reviewed-by: Zenghui Yu Cc: # 6.17.x Signed-off-by: Andrew Morton --- samples/damon/wsse.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/samples/damon/wsse.c b/samples/damon/wsse.c index bbd9392ab5b3..ff5e8a890f44 100644 --- a/samples/damon/wsse.c +++ b/samples/damon/wsse.c @@ -92,7 +92,12 @@ static int damon_sample_wsse_start(void) return err; } repeat_call_control.data = ctx; - return damon_call(ctx, &repeat_call_control); + err = damon_call(ctx, &repeat_call_control); + if (err) { + damon_stop(&ctx, 1); + damon_destroy_ctx(ctx); + } + return err; } static void damon_sample_wsse_stop(void) From a73fa45d3f0f42c446ae55c5799e3d5ef044cd5d Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 28 Jun 2026 14:54:45 -0700 Subject: [PATCH 088/501] samples/damon/prcl: stop and free damon ctx when damon_call() fails damon_sample_prcl_start() calls damon_call() right after damon_start() is succeeded. The kdamond that has started by the damon_start() could be terminated by itself before or in the middle of the damon_call() execution. There could be multiple reasons for such a stop including monitoring target process termination and kdamond_fn() internal memory allocation failures. In the case, damon_call() will fail and return an error without cleaning up the DAMON context object. The damon_sample_prcl_start() caller assumes it would clean up the object, though. When the user requests to start DAMON again, damon_sample_prcl_start() is called again, allocates a new DAMON context object and overwrites the pointer for the previous object. As a result, the previous context object is leaked. Safely stop the kdamond and deallocate the context object when the failure is returned. Note that the kdamond should be stopped first, because damon_call() failure means not complete termination of the kdamond but only the fact that the termination process has started. The user impact shouldn't be that significant because the race is not easy to happen, and only up to one DAMON context object can be leaked per race. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260628215447.96166-7-sj@kernel.org Link: https://lore.kernel.org/20260610035214.4850-1-sj@kernel.org [1] Fixes: a6c33f1054e3 ("samples/damon/prcl: use damon_call() repeat mode instead of damon_callback") Signed-off-by: SJ Park Reviewed-by: Zenghui Yu Cc: # 6.17.x Signed-off-by: Andrew Morton --- samples/damon/prcl.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/samples/damon/prcl.c b/samples/damon/prcl.c index 0db259894691..edeae145c4a8 100644 --- a/samples/damon/prcl.c +++ b/samples/damon/prcl.c @@ -112,7 +112,12 @@ static int damon_sample_prcl_start(void) } repeat_call_control.data = ctx; - return damon_call(ctx, &repeat_call_control); + err = damon_call(ctx, &repeat_call_control); + if (err) { + damon_stop(&ctx, 1); + damon_destroy_ctx(ctx); + } + return err; } static void damon_sample_prcl_stop(void) From 263af33a72d1995ae6cdc22b08d527e2bda17259 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 28 Jun 2026 15:01:10 -0700 Subject: [PATCH 089/501] mm/damon/sysfs: kobject_del() target (normal), context and kdamond dirs Patch series "mm/damon/sysfs: kobject_del() directories that users can create/remove". DAMON sysfs interface allows users to create and remove arbitrary number of directories on sysfs, using a few files having 'nr_' prefix. For example, 'nr_kdamonds'. When the user writes a number 'N' to the files, directories having name starting from '0' to 'N - 1' are created in the same directory. The pre-existing number-named directories are removed before creating the new directories. For the removal of the existing directories, DAMON sysfs interface use only kobject_put(). Because DAMON sysfs interface is the only kernel component that manages the directories, there is no problem in normal situations. However, if CONFIG_DEBUG_KOBJECT_RELEASE is enabled, the removal of dirs are delayed. Let's suppose a user writes a non-zero number to the 'nr_*' files while there are pre-existing number-named directories, on the config enabled kernel. DAMON sysfs interface decreases the reference counts of the existing directories and immediately creates new directories. Because the removal of the sysfs directories is delayed, it shows some pre-existing directories of the same names when it tries to create the new directories, and fails. For example, the issue can be triggered like below: # grep DEBUG_KOBJECT_RELEASE /boot/config-$(uname -r) CONFIG_DEBUG_KOBJECT_RELEASE=y # ls nr_kdamonds # echo 1 > nr_kdamonds # echo 1 > nr_kdamonds bash: echo: write error: File exists # dmesg [...] [ 300.880458] kobject: kobject_add_internal failed for 0 with -EEXIST, don't try to register things with the same name in the same directory. [...] Some of the error handling paths of the directories also lack the kobject_del() call. If the user uses nr_* file right after the errors, similar issues can happen. This doesn't cause catastrophic issues like kernel panics or memory corruptions. Users can work around by removing all directories first (write 0 to the nr_* files) and then create new directories after confirming the old directories are gone. But, this is definitely a bug that causes a bad user experience. Fix the issues by calling kobject_del() before creating new directories. This patch (of 11) On CONFIG_DEBUG_KOBJECT_RELEASE enabled kernel, lack of kobject_del() could cause directories creation failures due to the name conflicts. Fix those issues for normal creation paths of target, context and kdamond directories, and error paths of context and kdamond directories by adding kobject_del() calls. Note that this fix for target directories is not complete since it has a similar issue in the damon_sysfs_targets_add_dirs() error path. Because the normal path issue and the error path issue are introduced by different commits, this commit is fixing only the normal path issue. A commit for the error path will be added next. Link: https://lore.kernel.org/20260628220121.97360-1-sj@kernel.org Link: https://lore.kernel.org/20260628220121.97360-2-sj@kernel.org Fixes: c951cd3b8901 ("mm/damon: implement a minimal stub for sysfs-based DAMON interface") Signed-off-by: SJ Park Cc: # 5.18.x Signed-off-by: Andrew Morton --- mm/damon/sysfs.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index a9e187158067..38f3b02481f0 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -331,6 +331,7 @@ static void damon_sysfs_targets_rm_dirs(struct damon_sysfs_targets *targets) for (i = 0; i < targets->nr; i++) { damon_sysfs_target_rm_dirs(targets_arr[i]); + kobject_del(&targets_arr[i]->kobj); kobject_put(&targets_arr[i]->kobj); } targets->nr = 0; @@ -1640,6 +1641,7 @@ static void damon_sysfs_contexts_rm_dirs(struct damon_sysfs_contexts *contexts) for (i = 0; i < contexts->nr; i++) { damon_sysfs_context_rm_dirs(contexts_arr[i]); + kobject_del(&contexts_arr[i]->kobj); kobject_put(&contexts_arr[i]->kobj); } contexts->nr = 0; @@ -1678,13 +1680,15 @@ static int damon_sysfs_contexts_add_dirs(struct damon_sysfs_contexts *contexts, err = damon_sysfs_context_add_dirs(context); if (err) - goto out; + goto del_out; contexts_arr[i] = context; contexts->nr++; } return 0; +del_out: + kobject_del(&context->kobj); out: damon_sysfs_contexts_rm_dirs(contexts); kobject_put(&context->kobj); @@ -2499,6 +2503,7 @@ static void damon_sysfs_kdamonds_rm_dirs(struct damon_sysfs_kdamonds *kdamonds) for (i = 0; i < kdamonds->nr; i++) { damon_sysfs_kdamond_rm_dirs(kdamonds_arr[i]); + kobject_del(&kdamonds_arr[i]->kobj); kobject_put(&kdamonds_arr[i]->kobj); } kdamonds->nr = 0; @@ -2553,13 +2558,15 @@ static int damon_sysfs_kdamonds_add_dirs(struct damon_sysfs_kdamonds *kdamonds, err = damon_sysfs_kdamond_add_dirs(kdamond); if (err) - goto out; + goto del_out; kdamonds_arr[i] = kdamond; kdamonds->nr++; } return 0; +del_out: + kobject_del(&kdamond->kobj); out: damon_sysfs_kdamonds_rm_dirs(kdamonds); kobject_put(&kdamond->kobj); From 2603ef6f6ec3d3f7de2d6a07e7c9a683cebac419 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 28 Jun 2026 15:01:11 -0700 Subject: [PATCH 090/501] mm/damon/sysfs: kobject_del() region and target (error) dirs On CONFIG_DEBUG_KOBJECT_RELEASE enabled kernel, lack of kobject_del() could cause directories creation failures due to the name conflicts. Fix those issues for the normal creation path of region directories and the error path of target directories, by adding kobject_del() calls. Link: https://lore.kernel.org/20260628220121.97360-3-sj@kernel.org Fixes: 2031b14ea757 ("mm/damon/sysfs: support the physical address space monitoring") Signed-off-by: SJ Park Cc: # 5.18.x Signed-off-by: Andrew Morton --- mm/damon/sysfs.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 38f3b02481f0..204aed6a3e5d 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -105,8 +105,10 @@ static void damon_sysfs_regions_rm_dirs(struct damon_sysfs_regions *regions) struct damon_sysfs_region **regions_arr = regions->regions_arr; int i; - for (i = 0; i < regions->nr; i++) + for (i = 0; i < regions->nr; i++) { + kobject_del(®ions_arr[i]->kobj); kobject_put(®ions_arr[i]->kobj); + } regions->nr = 0; kfree(regions_arr); regions->regions_arr = NULL; @@ -370,13 +372,15 @@ static int damon_sysfs_targets_add_dirs(struct damon_sysfs_targets *targets, err = damon_sysfs_target_add_dirs(target); if (err) - goto out; + goto del_out; targets_arr[i] = target; targets->nr++; } return 0; +del_out: + kobject_del(&target->kobj); out: damon_sysfs_targets_rm_dirs(targets); kobject_put(&target->kobj); From 05fb6ac98c49be870c5f9ccdfdf95f0107e564ae Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 28 Jun 2026 15:01:12 -0700 Subject: [PATCH 091/501] mm/damon/sysfs-schemes: kobject_del() scheme dirs On CONFIG_DEBUG_KOBJECT_RELEASE enabled kernel, lack of kobject_del() could cause directories creation failures due to the name conflicts. Fix those issues for scheme directories by adding kobject_del() calls. Link: https://lore.kernel.org/20260628220121.97360-4-sj@kernel.org Fixes: 7e84b1f8212a ("mm/damon/sysfs: support DAMON-based Operation Schemes") Signed-off-by: SJ Park Cc: # 5.18.x Signed-off-by: Andrew Morton --- mm/damon/sysfs-schemes.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 3cbeccd436e4..db496d2e493a 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -2681,6 +2681,7 @@ void damon_sysfs_schemes_rm_dirs(struct damon_sysfs_schemes *schemes) for (i = 0; i < schemes->nr; i++) { damon_sysfs_scheme_rm_dirs(schemes_arr[i]); + kobject_del(&schemes_arr[i]->kobj); kobject_put(&schemes_arr[i]->kobj); } schemes->nr = 0; @@ -2722,13 +2723,15 @@ static int damon_sysfs_schemes_add_dirs(struct damon_sysfs_schemes *schemes, goto out; err = damon_sysfs_scheme_add_dirs(scheme); if (err) - goto out; + goto del_out; schemes_arr[i] = scheme; schemes->nr++; } return 0; +del_out: + kobject_del(&scheme->kobj); out: damon_sysfs_schemes_rm_dirs(schemes); kobject_put(&scheme->kobj); From f3ec3271210781c255e737498b84d5790e8176b4 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 28 Jun 2026 15:01:13 -0700 Subject: [PATCH 092/501] mm/damon/sysfs-schemes: kobject_del() scheme region dirs On CONFIG_DEBUG_KOBJECT_RELEASE enabled kernel, lack of kobject_del() could cause directories creation failures due to the name conflicts. Fix those issues for scheme region directories by adding kobject_del() calls. This issue was discovered [1] by Sashiko, though its analysis was partially incorrect. Link: https://lore.kernel.org/20260628220121.97360-5-sj@kernel.org Link: https://lore.kernel.org/20260517205828.6204-1-sj@kernel.org [1] Fixes: 9277d0367ba1 ("mm/damon/sysfs-schemes: implement scheme region directory") Signed-off-by: SJ Park Cc: # 6.2.x Signed-off-by: Andrew Morton --- mm/damon/sysfs-schemes.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index db496d2e493a..9eb28fe77b5b 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -332,6 +332,7 @@ static void damon_sysfs_scheme_regions_rm_dirs( list_for_each_entry_safe(r, next, ®ions->regions_list, list) { damos_sysfs_region_rm_dirs(r); list_del(&r->list); + kobject_del(&r->kobj); kobject_put(&r->kobj); regions->nr_regions--; } From 3c453bddacd4c04ecb38cf79dbfa41e7dfe0531b Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 28 Jun 2026 15:01:14 -0700 Subject: [PATCH 093/501] mm/damon/sysfs-schemes: kobject_del() scheme filter dirs On CONFIG_DEBUG_KOBJECT_RELEASE enabled kernel, lack of kobject_del() could cause directories creation failures due to the name conflicts. Fix those issues for scheme filter directories by adding kobject_del() calls. Link: https://lore.kernel.org/20260628220121.97360-6-sj@kernel.org Fixes: 472e2b70eda6 ("mm/damon/sysfs-schemes: connect filter directory and filters directory") Signed-off-by: SJ Park Cc: # 6.3.x Signed-off-by: Andrew Morton --- mm/damon/sysfs-schemes.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 9eb28fe77b5b..e955fb916a7e 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -912,8 +912,10 @@ static void damon_sysfs_scheme_filters_rm_dirs( struct damon_sysfs_scheme_filter **filters_arr = filters->filters_arr; int i; - for (i = 0; i < filters->nr; i++) + for (i = 0; i < filters->nr; i++) { + kobject_del(&filters_arr[i]->kobj); kobject_put(&filters_arr[i]->kobj); + } filters->nr = 0; kfree(filters_arr); filters->filters_arr = NULL; From 0d4397ca921ceaf80fc3eca4c8194812ff79a979 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 28 Jun 2026 15:01:15 -0700 Subject: [PATCH 094/501] mm/damon/sysfs-schemes: kobject_del() scheme quota goal dirs On CONFIG_DEBUG_KOBJECT_RELEASE enabled kernel, lack of kobject_del() could cause directories creation failures due to the name conflicts. Fix those issues for scheme quota goal directories by adding kobject_del() calls. Link: https://lore.kernel.org/20260628220121.97360-7-sj@kernel.org Fixes: 7f262da0a30d ("mm/damon/sysfs-schemes: implement files for scheme quota goals setup") Signed-off-by: SJ Park Cc: # 6.8.x Signed-off-by: Andrew Morton --- mm/damon/sysfs-schemes.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index e955fb916a7e..58051185713c 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -1463,8 +1463,10 @@ static void damos_sysfs_quota_goals_rm_dirs( struct damos_sysfs_quota_goal **goals_arr = goals->goals_arr; int i; - for (i = 0; i < goals->nr; i++) + for (i = 0; i < goals->nr; i++) { + kobject_del(&goals_arr[i]->kobj); kobject_put(&goals_arr[i]->kobj); + } goals->nr = 0; kfree(goals_arr); goals->goals_arr = NULL; From 286380c78bc51e6c578621b9ae660bf9e5ad2563 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 28 Jun 2026 15:01:16 -0700 Subject: [PATCH 095/501] mm/damon/sysfs-schemes: kobject_del() scheme action destination dirs On CONFIG_DEBUG_KOBJECT_RELEASE enabled kernel, lack of kobject_del() could cause directories creation failures due to the name conflicts. Fix those issues for scheme action destination directories by adding kobject_del() calls. Link: https://lore.kernel.org/20260628220121.97360-8-sj@kernel.org Fixes: 2cd0bf85a203 ("mm/damon/sysfs-schemes: implement DAMOS action destinations directory") Signed-off-by: SJ Park Cc: # 6.17.x Signed-off-by: Andrew Morton --- mm/damon/sysfs-schemes.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 58051185713c..e2c8716be6c9 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -2143,8 +2143,10 @@ static void damos_sysfs_dests_rm_dirs( struct damos_sysfs_dest **dests_arr = dests->dests_arr; int i; - for (i = 0; i < dests->nr; i++) + for (i = 0; i < dests->nr; i++) { + kobject_del(&dests_arr[i]->kobj); kobject_put(&dests_arr[i]->kobj); + } dests->nr = 0; kfree(dests_arr); dests->dests_arr = NULL; From c37dfd69829702785963314fb8c4e644a908aa7d Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 28 Jun 2026 15:01:17 -0700 Subject: [PATCH 096/501] mm/damon/sysfs: kobject_del() probe dirs On CONFIG_DEBUG_KOBJECT_RELEASE enabled kernel, lack of kobject_del() could cause directories creation failures due to the name conflicts. Fix those issues for data attribute probe directories by adding kobject_del() calls. Link: https://lore.kernel.org/20260628220121.97360-9-sj@kernel.org Fixes: bf3ea3d30880 ("mm/damon/sysfs: implement probe dir") Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/sysfs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 204aed6a3e5d..9f92ebdb2857 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -1137,6 +1137,7 @@ static void damon_sysfs_probes_rm_dirs( for (i = 0; i < probes->nr; i++) { damon_sysfs_probe_rm_dirs(probes_arr[i]); + kobject_del(&probes_arr[i]->kobj); kobject_put(&probes_arr[i]->kobj); } probes->nr = 0; From 092a336674387a4453f6a82db0c41368bcd45612 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 28 Jun 2026 15:01:18 -0700 Subject: [PATCH 097/501] mm/damon/sysfs: kobject_del() probe filter dirs On CONFIG_DEBUG_KOBJECT_RELEASE enabled kernel, lack of kobject_del() could cause directories creation failures due to the name conflicts. Fix those issues for data attribute probe filter directories by adding kobject_del() calls. Link: https://lore.kernel.org/20260628220121.97360-10-sj@kernel.org Fixes: 82e66aef7714 ("mm/damon/sysfs: implement filter dir") Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/sysfs.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 9f92ebdb2857..0ccdc71275d5 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -960,8 +960,10 @@ static void damon_sysfs_filters_rm_dirs(struct damon_sysfs_filters *filters) struct damon_sysfs_filter **filters_arr = filters->filters_arr; int i; - for (i = 0; i < filters->nr; i++) + for (i = 0; i < filters->nr; i++) { + kobject_del(&filters_arr[i]->kobj); kobject_put(&filters_arr[i]->kobj); + } filters->nr = 0; kfree(filters_arr); filters->filters_arr = NULL; From f5b4541b851aaf58de8f47a078308d72f73b9cb4 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 28 Jun 2026 15:01:19 -0700 Subject: [PATCH 098/501] mm/damon/sysfs: kobject_del() probe dirs in probes_addd_dir error path On CONFIG_DEBUG_KOBJECT_RELEASE enabled kernel, lack of kobject_del() could cause directories creation failures due to the name conflicts. Fix the issue for data attribute probe filter directories in the error handling path of damon_sysfs_probes_add_dirs() by adding a kobject_del() call. Link: https://lore.kernel.org/20260628220121.97360-11-sj@kernel.org Fixes: af7cb41af9a9 ("mm/damon/sysfs: implement filters directory") Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/sysfs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 0ccdc71275d5..e3526a263e20 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -1181,6 +1181,7 @@ static int damon_sysfs_probes_add_dirs( err = damon_sysfs_probe_add_dirs(probe); if (err) { + kobject_del(&probe->kobj); kobject_put(&probe->kobj); damon_sysfs_probes_rm_dirs(probes); return err; From 5f0ead7e66f5c30e98f38e5a85b48123300e63c5 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 28 Jun 2026 15:01:20 -0700 Subject: [PATCH 099/501] mm/damon/sysfs-schemes: kobject_del() region for populate_region error On CONFIG_DEBUG_KOBJECT_RELEASE enabled kernel, lack of kobject_del() could cause directories creation failures due to the name conflicts. Fix the issue for tried region directories in the error handling path of damon_sysfs_populate_region_dir() by adding a kobject_del() call. Link: https://lore.kernel.org/20260628220121.97360-12-sj@kernel.org Fixes: b574a82d10de ("mm/damon/sysfs-schemes: implement tried_regions//probes/") Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/sysfs-schemes.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index e2c8716be6c9..41f93a1823bf 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -3132,12 +3132,14 @@ void damos_sysfs_populate_region_dir(struct damon_sysfs_schemes *sysfs_schemes, sysfs_regions->nr_regions)) goto out; if (damos_sysfs_region_add_dirs(region, ctx, r)) - goto out; + goto del_out; list_add_tail(®ion->list, &sysfs_regions->regions_list); sysfs_regions->nr_regions++; return; +del_out: + kobject_del(®ion->kobj); out: kobject_put(®ion->kobj); } From a71638ecc6424d5a2925e5ea04ff5f9f769295d6 Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Mon, 29 Jun 2026 15:49:47 +0200 Subject: [PATCH 100/501] sparc/mm: drop custom pte_clear_not_present_full() Patch series "mm: cleanup clear_not_present_full_ptes()", v2. While doing some review, I stumbled over clear_not_present_full_ptes() and concluded that it needs some love. Let's remove pte_clear_not_present_full() and cleanup clear_not_present_full_ptes(), renaming it to clear_non_present_ptes(). This patch (of 3): On sparc64, pte_clear_not_present_full() nowadays does a simple __set_pte_at(). In __set_pte_at() -> maybe_tlb_batch_add(), we check pte_accessible() to see whether to call tlb_batch_add(). However, non-present PTEs are surely not accessible, so tlb_batch_add() is never called and the "full" parameter is irrelevant. Let's drop the helper and just let common code do a pte_clear(). pte_clear() on sparc64 maps to set_pte_at()->set_ptes()->__set_pte_at() ... so it ends up calling the same function, just with "full=0". Given that "full" is irrelevant, there is no change. We added pte_clear_not_present_full() for sparc64 in commit 90f08e399d05 ("sparc: mmu_gather rework"), and I suspect that it was already not required back then. Link: https://lore.kernel.org/20260629-clear_not_present_full_ptes-v2-0-96089871a1e7@kernel.org Link: https://lore.kernel.org/20260629-clear_not_present_full_ptes-v2-1-96089871a1e7@kernel.org Signed-off-by: David Hildenbrand (Arm) Reviewed-by: Lance Yang Cc: Peter Zijlstra Cc: Andreas Larsson Cc: David S. Miller Cc: Jann Horn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Oscar Salvador (SUSE) Signed-off-by: Andrew Morton --- arch/sparc/include/asm/pgtable_64.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h index 74ede706fb32..0837ebbc5dce 100644 --- a/arch/sparc/include/asm/pgtable_64.h +++ b/arch/sparc/include/asm/pgtable_64.h @@ -945,10 +945,6 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr, #define pte_clear(mm,addr,ptep) \ set_pte_at((mm), (addr), (ptep), __pte(0UL)) -#define __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL -#define pte_clear_not_present_full(mm,addr,ptep,fullmm) \ - __set_pte_at((mm), (addr), (ptep), __pte(0UL), (fullmm)) - #ifdef DCACHE_ALIASING_POSSIBLE #define __HAVE_ARCH_MOVE_PTE #define move_pte(pte, old_addr, new_addr) \ From ef68364cd7767aba444cdb8bf6ceec099bc61cec Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Mon, 29 Jun 2026 15:49:48 +0200 Subject: [PATCH 101/501] mm: drop pte_clear_not_present_full() In general, there is no good reason to do anything special when clearing non-present PTEs. In theory, HW that does have to invalidate TLBs for non-present PTEs could benefit from a "full" parameter, but fortunately pte_clear_not_present_full() is not wired up anymore ... and there would have to be something very convincing for us to care about that to re-add it. So, let's just use pte_clear() directly now. To prevent the compiler complaining on some configs about "set but not used" addr parameter, silence that here. Link: https://lore.kernel.org/20260629-clear_not_present_full_ptes-v2-2-96089871a1e7@kernel.org Signed-off-by: David Hildenbrand (Arm) Reviewed-by: Oscar Salvador (SUSE) Reviewed-by: Lance Yang Cc: Andreas Larsson Cc: David S. Miller Cc: Jann Horn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Peter Zijlstra Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/pgtable.h | 21 ++++----------------- mm/madvise.c | 4 ++-- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index dc804296d78f..0b81e396816a 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -988,21 +988,6 @@ static inline void update_mmu_tlb(struct vm_area_struct *vma, update_mmu_tlb_range(vma, address, ptep, 1); } -/* - * Some architectures may be able to avoid expensive synchronization - * primitives when modifications are made to PTE's which are already - * not present, or in the process of an address space destruction. - */ -#ifndef __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL -static inline void pte_clear_not_present_full(struct mm_struct *mm, - unsigned long address, - pte_t *ptep, - int full) -{ - pte_clear(mm, address, ptep); -} -#endif - #ifndef clear_not_present_full_ptes /** * clear_not_present_full_ptes - Clear multiple not present PTEs which are @@ -1014,7 +999,7 @@ static inline void pte_clear_not_present_full(struct mm_struct *mm, * @full: Whether we are clearing a full mm. * * May be overridden by the architecture; otherwise, implemented as a simple - * loop over pte_clear_not_present_full(). + * loop over pte_clear(). * * Context: The caller holds the page table lock. The PTEs are all not present. * The PTEs are all in the same PMD. @@ -1022,8 +1007,10 @@ static inline void pte_clear_not_present_full(struct mm_struct *mm, static inline void clear_not_present_full_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned int nr, int full) { + (void)addr; + for (;;) { - pte_clear_not_present_full(mm, addr, ptep, full); + pte_clear(mm, addr, ptep); if (--nr == 0) break; ptep++; diff --git a/mm/madvise.c b/mm/madvise.c index 77552b03d318..e483b63bdcef 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -697,7 +697,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, clear_not_present_full_ptes(mm, addr, pte, nr, tlb->fullmm); } else if (softleaf_is_hwpoison(entry) || softleaf_is_poison_marker(entry)) { - pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); + pte_clear(mm, addr, pte); } continue; } @@ -1233,7 +1233,7 @@ static int guard_remove_pte_entry(pte_t *pte, unsigned long addr, if (is_guard_pte_marker(ptent)) { /* Simply clear the PTE marker. */ - pte_clear_not_present_full(walk->mm, addr, pte, false); + pte_clear(walk->mm, addr, pte); update_mmu_cache(walk->vma, addr, pte); } From d045e8a05d302c0670ba41015b1528d3694340e2 Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Mon, 29 Jun 2026 15:49:49 +0200 Subject: [PATCH 102/501] mm: cleanup clear_not_present_full_ptes() and rename to clear_non_present_ptes() Let's clean it up a bit: (1) There is no need to pass "full" anymore. (2) No architecture overwrites it, and there isn't really a good reason to do so when dealing with non-present PTEs. (3) While at it, call it "non-present", similar to copy_nonpresent_pte() and zap_nonpresent_ptes(). It's a shame that we have clear_non_present_ptes() correspond to pte_clear() and clear_ptes() correspond to ptep_get_and_clear*(). Link: https://lore.kernel.org/20260629-clear_not_present_full_ptes-v2-3-96089871a1e7@kernel.org Signed-off-by: David Hildenbrand (Arm) Reviewed-by: Oscar Salvador (SUSE) Reviewed-by: Lance Yang Cc: Andreas Larsson Cc: David S. Miller Cc: Jann Horn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Peter Zijlstra Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/pgtable.h | 14 ++++---------- mm/madvise.c | 2 +- mm/memory.c | 2 +- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 0b81e396816a..d52d2a976e5a 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -988,24 +988,19 @@ static inline void update_mmu_tlb(struct vm_area_struct *vma, update_mmu_tlb_range(vma, address, ptep, 1); } -#ifndef clear_not_present_full_ptes /** - * clear_not_present_full_ptes - Clear multiple not present PTEs which are - * consecutive in the pgtable. + * clear_nonpresent_ptes - Clear multiple non-present PTEs which are + * consecutive in the pgtable. * @mm: Address space the ptes represent. * @addr: Address of the first pte. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to clear. - * @full: Whether we are clearing a full mm. - * - * May be overridden by the architecture; otherwise, implemented as a simple - * loop over pte_clear(). * * Context: The caller holds the page table lock. The PTEs are all not present. * The PTEs are all in the same PMD. */ -static inline void clear_not_present_full_ptes(struct mm_struct *mm, - unsigned long addr, pte_t *ptep, unsigned int nr, int full) +static inline void clear_nonpresent_ptes(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, unsigned int nr) { (void)addr; @@ -1017,7 +1012,6 @@ static inline void clear_not_present_full_ptes(struct mm_struct *mm, addr += PAGE_SIZE; } } -#endif #ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH extern pte_t ptep_clear_flush(struct vm_area_struct *vma, diff --git a/mm/madvise.c b/mm/madvise.c index e483b63bdcef..9292f60b19aa 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -694,7 +694,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, nr = swap_pte_batch(pte, max_nr, ptent); nr_swap -= nr; swap_put_entries_direct(entry, nr); - clear_not_present_full_ptes(mm, addr, pte, nr, tlb->fullmm); + clear_nonpresent_ptes(mm, addr, pte, nr); } else if (softleaf_is_hwpoison(entry) || softleaf_is_poison_marker(entry)) { pte_clear(mm, addr, pte); diff --git a/mm/memory.c b/mm/memory.c index ff338c2abe92..a3fcaf8cfe8f 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1797,7 +1797,7 @@ static inline int zap_nonpresent_ptes(struct mmu_gather *tlb, pr_alert("unrecognized swap entry 0x%lx\n", entry.val); WARN_ON_ONCE(1); } - clear_not_present_full_ptes(vma->vm_mm, addr, pte, nr, tlb->fullmm); + clear_nonpresent_ptes(vma->vm_mm, addr, pte, nr); *any_skipped = zap_install_uffd_wp_if_needed(vma, addr, pte, nr, details, ptent); return nr; From 8319dadcbd8121d94f72332ab926de7f69d9b801 Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Mon, 29 Jun 2026 07:46:44 -0700 Subject: [PATCH 103/501] selftests/damon: prevent cross-context state pollution in DamonCtx Patch series "selftests/damon: misc fixes for test bugs", v3. This series fixes several bugs in the DAMON selftests. Most are trivial but makes test output wrong or even silently pass the one test case for 'avail_operation' file existence check. Patch 1 fixes mutable default arguments in DamonCtx.__init__() that cause state to leak between test instances. Patch 2 fixes wrong operator precedence and join TypeError in damos_tried_regions.py. Patch 3 fixes several wrong strings that produce dead elif branches, skipped file existence checks, and broken dict key lookups. This patch (of 3): DamonCtx.__init__() uses mutable default values for monitoring_attrs, targets, and schemes. In Python these are evaluated once at function definition time, so multiple DamonCtx instances can unintentionally share the same lists and DamonAttrs instance. Replace the mutable defaults with None sentinels and initialize the objects when needed. Link: https://lore.kernel.org/20260629144648.134092-1-sj@kernel.org Link: https://lore.kernel.org/20260601032314.424013-2-kunwu.chan@linux.dev Link: https://lore.kernel.org/20260629144648.134092-2-sj@kernel.org Co-developed-by: Wang Lian Signed-off-by: Wang Lian Signed-off-by: Kunwu Chan Signed-off-by: SJ Park Reviewed-by: SJ Park Cc: Kunwu Chan Cc: Wang Lian Signed-off-by: Andrew Morton --- tools/testing/selftests/damon/_damon_sysfs.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/damon/_damon_sysfs.py b/tools/testing/selftests/damon/_damon_sysfs.py index 8b12cc048440..2f6f2699db25 100644 --- a/tools/testing/selftests/damon/_damon_sysfs.py +++ b/tools/testing/selftests/damon/_damon_sysfs.py @@ -624,17 +624,23 @@ class DamonCtx: pause = None idx = None - def __init__(self, ops='paddr', monitoring_attrs=DamonAttrs(), targets=[], - schemes=[], pause=False): + def __init__(self, ops='paddr', monitoring_attrs=None, targets=None, + schemes=None, pause=False): self.ops = ops + if monitoring_attrs is None: + monitoring_attrs = DamonAttrs() self.monitoring_attrs = monitoring_attrs self.monitoring_attrs.context = self + if targets is None: + targets = [] self.targets = targets for idx, target in enumerate(self.targets): target.idx = idx target.context = self + if schemes is None: + schemes = [] self.schemes = schemes for idx, scheme in enumerate(self.schemes): scheme.idx = idx From 9ad3a4dbbc562a2b58517fb35a4635a980945dac Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Mon, 29 Jun 2026 07:46:45 -0700 Subject: [PATCH 104/501] selftests/damon/damos_tried_regions: fix expectation output and join TypeError The expectation print has wrong operator precedence: '%' binds before the conditional expression, so the else branch prints 'not met' without the prefix 'expectation (>= 14) is'. Add parentheses to fix it. Also, '\n'.join() on the list of ints raises TypeError; convert to str in the list comprehension. Link: https://lore.kernel.org/20260601032314.424013-3-kunwu.chan@linux.dev Link: https://lore.kernel.org/20260629144648.134092-3-sj@kernel.org Co-developed-by: Wang Lian Signed-off-by: Wang Lian Signed-off-by: Kunwu Chan Signed-off-by: SJ Park Reviewed-by: SJ Park Cc: Kunwu Chan Cc: Wang Lian Signed-off-by: Andrew Morton --- tools/testing/selftests/damon/damos_tried_regions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/damon/damos_tried_regions.py b/tools/testing/selftests/damon/damos_tried_regions.py index 3b347eb28bd2..d6472e6a6e08 100755 --- a/tools/testing/selftests/damon/damos_tried_regions.py +++ b/tools/testing/selftests/damon/damos_tried_regions.py @@ -55,10 +55,10 @@ def main(): collected_nr_regions.sort() sample = collected_nr_regions[4] print('50-th percentile nr_regions: %d' % sample) - print('expectation (>= 14) is %s' % 'met' if sample >= 14 else 'not met') + print('expectation (>= 14) is %s' % ('met' if sample >= 14 else 'not met')) if collected_nr_regions[4] < 14: print('full nr_regions:') - print('\n'.join(collected_nr_regions)) + print('\n'.join(['%d' % x for x in collected_nr_regions])) exit(1) if __name__ == '__main__': From 0185159faf9ec26a2d26b1f371f84109af9c49df Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Mon, 29 Jun 2026 07:46:46 -0700 Subject: [PATCH 105/501] selftests/damon: fix dead code, skipped checks, and broken lookups 'hugeapge_size' in drgn_dump_damon_status.py was a dead elif branch. $fail_reason in sysfs.sh was undefined, silently emptying the error message. 'exit' instead of 'exist' in sysfs.sh skipped a file existence check. 'nohugeapge' in sysfs.py broke an action dict lookup. Fix other wrong strings in the same files. Link: https://lore.kernel.org/20260601032314.424013-4-kunwu.chan@linux.dev Link: https://lore.kernel.org/20260629144648.134092-4-sj@kernel.org Co-developed-by: Wang Lian Signed-off-by: Wang Lian Signed-off-by: Kunwu Chan Signed-off-by: SJ Park Reviewed-by: SJ Park Cc: Kunwu Chan Cc: Wang Lian Signed-off-by: Andrew Morton --- tools/testing/selftests/damon/_damon_sysfs.py | 2 +- tools/testing/selftests/damon/damos_apply_interval.py | 2 +- tools/testing/selftests/damon/damos_quota_goal.py | 2 +- tools/testing/selftests/damon/drgn_dump_damon_status.py | 2 +- tools/testing/selftests/damon/sysfs.py | 4 ++-- tools/testing/selftests/damon/sysfs.sh | 6 +++--- .../sysfs_update_schemes_tried_regions_wss_estimation.py | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tools/testing/selftests/damon/_damon_sysfs.py b/tools/testing/selftests/damon/_damon_sysfs.py index 2f6f2699db25..f6127081dfb2 100644 --- a/tools/testing/selftests/damon/_damon_sysfs.py +++ b/tools/testing/selftests/damon/_damon_sysfs.py @@ -837,7 +837,7 @@ class Kdamond: for goal in scheme.quota.goals: err = goal.stage() if err is not None: - print('commit_schemes_quota_goals failed stagign: %s'% + print('commit_schemes_quota_goals failed staging: %s'% err) exit(1) return write_file(os.path.join(self.sysfs_dir(), 'state'), diff --git a/tools/testing/selftests/damon/damos_apply_interval.py b/tools/testing/selftests/damon/damos_apply_interval.py index f04d43702481..0f2f36584e48 100755 --- a/tools/testing/selftests/damon/damos_apply_interval.py +++ b/tools/testing/selftests/damon/damos_apply_interval.py @@ -56,7 +56,7 @@ def main(): # Because the second scheme was having the apply interval that is ten times # lower than that of the first scheme, the second scheme should be tried # about ten times more frequently than the first scheme. For possible - # timing errors, check if it was at least nine times more freuqnetly tried. + # timing errors, check if it was at least nine times more frequently tried. ratio = nr_tried_stats[1] / nr_tried_stats[0] if ratio < 9: print('%d / %d = %f (< 9)' % diff --git a/tools/testing/selftests/damon/damos_quota_goal.py b/tools/testing/selftests/damon/damos_quota_goal.py index f76e0412b564..661e4ba4765a 100755 --- a/tools/testing/selftests/damon/damos_quota_goal.py +++ b/tools/testing/selftests/damon/damos_quota_goal.py @@ -66,7 +66,7 @@ def main(): # effective quota was already minimum that cannot be more reduced if expect_increase is False and last_effective_bytes == 1: continue - print('efective bytes not changed: %d' % goal.effective_bytes) + print('effective bytes not changed: %d' % goal.effective_bytes) exit(1) increased = last_effective_bytes < goal.effective_bytes diff --git a/tools/testing/selftests/damon/drgn_dump_damon_status.py b/tools/testing/selftests/damon/drgn_dump_damon_status.py index 972948e6215f..26b207e44268 100755 --- a/tools/testing/selftests/damon/drgn_dump_damon_status.py +++ b/tools/testing/selftests/damon/drgn_dump_damon_status.py @@ -163,7 +163,7 @@ def damos_filter_to_dict(damos_filter): int(damos_filter.addr_range.end)] elif type_ == 'target': dict_['target_idx'] = int(damos_filter.target_idx) - elif type_ == 'hugeapge_size': + elif type_ == 'hugepage_size': dict_['sz_range'] = [int(damos_filter.sz_range.min), int(damos_filter.sz_range.max)] return dict_ diff --git a/tools/testing/selftests/damon/sysfs.py b/tools/testing/selftests/damon/sysfs.py index aa03a1187489..99412f0d31f3 100755 --- a/tools/testing/selftests/damon/sysfs.py +++ b/tools/testing/selftests/damon/sysfs.py @@ -119,7 +119,7 @@ def assert_access_pattern_committed(pattern, dump): 'max_nr_accesses', dump) assert_true(dump['min_age_region'] == pattern.age[0], 'min_age_region', dump) - assert_true(dump['max_age_region'] == pattern.age[1], 'miaxage_region', + assert_true(dump['max_age_region'] == pattern.age[1], 'max_age_region', dump) def assert_scheme_committed(scheme, dump): @@ -129,7 +129,7 @@ def assert_scheme_committed(scheme, dump): 'cold': 1, 'pageout': 2, 'hugepage': 3, - 'nohugeapge': 4, + 'nohugepage': 4, 'collapse': 5, 'lru_prio': 6, 'lru_deprio': 7, diff --git a/tools/testing/selftests/damon/sysfs.sh b/tools/testing/selftests/damon/sysfs.sh index 78f4badb5beb..2eaaa5ae3c5e 100755 --- a/tools/testing/selftests/damon/sysfs.sh +++ b/tools/testing/selftests/damon/sysfs.sh @@ -3,7 +3,7 @@ source _common.sh -# Kselftest frmework requirement - SKIP code is 4. +# Kselftest framework requirement - SKIP code is 4. ksft_skip=4 ensure_write_succ() @@ -28,7 +28,7 @@ ensure_write_fail() if (echo "$content" > "$file") 2> /dev/null then - echo "writing $content to $file succeed ($fail_reason)" + echo "writing $content to $file succeeded ($reason)" echo "expected failure because $reason" exit 1 fi @@ -363,7 +363,7 @@ test_context() { context_dir=$1 ensure_dir "$context_dir" "exist" - ensure_file "$context_dir/avail_operations" "exit" 400 + ensure_file "$context_dir/avail_operations" "exist" 400 ensure_file "$context_dir/operations" "exist" 600 ensure_file "$context_dir/addr_unit" "exist" 600 ensure_file "$context_dir/pause" "exist" 600 diff --git a/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py b/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py index 35c724a63f6c..16fdc6e7fc56 100755 --- a/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py +++ b/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py @@ -7,7 +7,7 @@ import time import _damon_sysfs def pass_wss_estimation(sz_region): - # access two regions of given size, 2 seocnds per each region + # access two regions of given size, 2 seconds per each region proc = subprocess.Popen( ['./access_memory', '2', '%d' % sz_region, '2000', 'repeat']) kdamonds = _damon_sysfs.Kdamonds([_damon_sysfs.Kdamond( From 477e99645fec6ac31d0cc339364f631878f90501 Mon Sep 17 00:00:00 2001 From: Cheng Nie Date: Mon, 29 Jun 2026 07:48:10 -0700 Subject: [PATCH 106/501] selftests/damon/_damon_sysfs.py: fix memcg_path assignment Patch series "selftests/damon: fix memcg_path staging handling", v5. Fix a bug in _damon_sysfs.py for damos_filter memcg_path setup, and add a test case of it in sysfs.py. This patch (of 2): DamosFilter stores memcg_path for sysfs staging, but the constructor assigns it with a trailing comma and therefore turns it into a tuple. Fix the assignment so memcg_path is stored as the intended string. This makes memcg filter staging and follow-up validation use the written path correctly. Link: https://lore.kernel.org/20260629144812.134159-1-sj@kernel.org Link: https://lore.kernel.org/464AE12D4BC6B6F4+20260601090519.240482-1-niecheng1@uniontech.com Link: https://lore.kernel.org/20260629144812.134159-2-sj@kernel.org Signed-off-by: Cheng Nie Signed-off-by: SJ Park Reviewed-by: SJ Park Signed-off-by: Andrew Morton --- tools/testing/selftests/damon/_damon_sysfs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/damon/_damon_sysfs.py b/tools/testing/selftests/damon/_damon_sysfs.py index f6127081dfb2..c197ab99bcc0 100644 --- a/tools/testing/selftests/damon/_damon_sysfs.py +++ b/tools/testing/selftests/damon/_damon_sysfs.py @@ -271,7 +271,7 @@ class DamosFilter: self.type_ = type_ self.matching = matching self.allow = allow - self.memcg_path = memcg_path, + self.memcg_path = memcg_path self.addr_start = addr_start self.addr_end = addr_end self.target_idx = target_idx From 9b413dff810a4945b564e823261075ea3525fd05 Mon Sep 17 00:00:00 2001 From: Cheng Nie Date: Mon, 29 Jun 2026 07:48:11 -0700 Subject: [PATCH 107/501] selftests/damon/sysfs.py: validate memcg_path staging readback Add a dedicated test at the end of main() that stages memcg_path via sysfs and verifies its readback. Configure the memcg filter before start(), do not call commit(), and ignore start() failures so the test does not depend on CONFIG_MEMCG or cgroup layout. Call stop() for cleanup without checking its return value. Link: https://lore.kernel.org/D2B37130D38E09AC+20260601090634.241864-1-niecheng1@uniontech.com Link: https://lore.kernel.org/20260629144812.134159-3-sj@kernel.org Signed-off-by: Cheng Nie Signed-off-by: SJ Park Reviewed-by: SJ Park Signed-off-by: Andrew Morton --- tools/testing/selftests/damon/sysfs.py | 31 ++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tools/testing/selftests/damon/sysfs.py b/tools/testing/selftests/damon/sysfs.py index 99412f0d31f3..3ffa054b6386 100755 --- a/tools/testing/selftests/damon/sysfs.py +++ b/tools/testing/selftests/damon/sysfs.py @@ -250,6 +250,35 @@ def assert_ctxs_committed(kdamonds): if ctx in ctxs_paused_for_dump: ctx.pause = False +def test_memcg_filter_memcg_path_staging(): + global kdamonds + memcg_filter = _damon_sysfs.DamosFilter( + type_='memcg', matching=True, allow=True, memcg_path='/') + kdamonds = _damon_sysfs.Kdamonds( + [_damon_sysfs.Kdamond( + contexts=[_damon_sysfs.DamonCtx( + targets=[_damon_sysfs.DamonTarget(pid=-1)], + schemes=[_damon_sysfs.Damos( + ops_filters=[memcg_filter])], + )])]) + kdamonds.start() + + shown, rd_err = _damon_sysfs.read_file( + os.path.join(memcg_filter.sysfs_dir(), 'memcg_path')) + if rd_err is not None: + print('memcg_path staging: sysfs read (%s)' % rd_err) + kdamonds.stop() + exit(1) + if shown.rstrip('\n') != memcg_filter.memcg_path: + print('memcg_path staging: memcg_path readback ' + '(shown=%s, expected=%s)' % + (shown.rstrip('\n'), memcg_filter.memcg_path)) + kdamonds.stop() + exit(1) + + kdamonds.stop() + kdamonds = None + def main(): global kdamonds kdamonds = _damon_sysfs.Kdamonds( @@ -356,5 +385,7 @@ def main(): assert_ctxs_committed(kdamonds) kdamonds.stop() + test_memcg_filter_memcg_path_staging() + if __name__ == '__main__': main() From b1c254e60b6b12a75470bf2766eee98b1fc2944b Mon Sep 17 00:00:00 2001 From: Ruslan Valiyev Date: Mon, 29 Jun 2026 07:49:25 -0700 Subject: [PATCH 108/501] selftests/damon/_damon_sysfs: support kdamond refresh_ms Patch series "selftests/damon: test kdamond refresh_ms", v2. The kdamond 'refresh_ms' sysfs file makes DAMON periodically update its read-only sysfs files (DAMOS stats, tuned monitoring intervals and the kdamond pid) on its own, so users don't have to write update keywords such as 'update_schemes_stats' to the 'state' file. It has no selftest coverage. The first patch adds refresh_ms support to the _damon_sysfs.py test control module. The second adds a test that sets refresh_ms and confirms a scheme's stats are updated under sysfs without an explicit update request; the test skips on kernels that predate the refresh_ms file. Tested on current mainline under a DAMON-enabled kernel: the new test passes and the existing DAMON selftests show no new failures. This patch (of 2): The Kdamond class has no way to set the kdamond-level 'refresh_ms' sysfs file, which makes DAMON periodically update the read-only sysfs files (DAMOS stats, tuned monitoring intervals and the kdamond pid) on its own. Add a 'refresh_ms' parameter to Kdamond. When it is set (including to zero, to disable the periodic update), write it before turning the kdamond on, so tests can exercise the auto-update behavior. Leaving it unset keeps the previous behavior of not touching the file, so callers running against kernels without the feature are unaffected. Link: https://lore.kernel.org/20260602131217.2210912-2-linuxoid@gmail.com Link: https://lore.kernel.org/20260629144927.134237-2-sj@kernel.org Signed-off-by: Ruslan Valiyev Reviewed-by: SJ Park Signed-off-by: SJ Park Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/damon/_damon_sysfs.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/damon/_damon_sysfs.py b/tools/testing/selftests/damon/_damon_sysfs.py index c197ab99bcc0..e6a2265d721e 100644 --- a/tools/testing/selftests/damon/_damon_sysfs.py +++ b/tools/testing/selftests/damon/_damon_sysfs.py @@ -698,12 +698,14 @@ class DamonCtx: class Kdamond: state = None pid = None + refresh_ms = None contexts = None idx = None # index of this kdamond between siblings kdamonds = None # parent - def __init__(self, contexts=[]): + def __init__(self, contexts=[], refresh_ms=None): self.contexts = contexts + self.refresh_ms = refresh_ms for idx, context in enumerate(self.contexts): context.idx = idx context.kdamond = self @@ -726,6 +728,11 @@ class Kdamond: err = context.stage() if err is not None: return err + if self.refresh_ms is not None: + err = write_file(os.path.join(self.sysfs_dir(), 'refresh_ms'), + '%d' % self.refresh_ms) + if err is not None: + return err err = write_file(os.path.join(self.sysfs_dir(), 'state'), 'on') if err is not None: return err From 99cea7eb5216ff688a597595e93c123823865674 Mon Sep 17 00:00:00 2001 From: Ruslan Valiyev Date: Mon, 29 Jun 2026 07:49:26 -0700 Subject: [PATCH 109/501] selftests/damon/sysfs_refresh: test kdamond refresh_ms Writing a non-zero value to a kdamond's 'refresh_ms' sysfs file should make DAMON periodically update the read-only sysfs files on its own, without the user writing update keywords such as 'update_schemes_stats' to the 'state' file. This behavior has no test coverage. Add a test that starts a kdamond with refresh_ms set and a 'stat' scheme whose default access pattern matches every monitored region, then polls the scheme's 'nr_tried' stats file directly, without requesting an update. The value can become non-zero only via the periodic refresh, so the test confirms refresh_ms works; with refresh_ms disabled the stat stays zero and the test fails. Link: https://lore.kernel.org/20260602131217.2210912-3-linuxoid@gmail.com Link: https://lore.kernel.org/20260629144927.134237-3-sj@kernel.org Signed-off-by: Ruslan Valiyev Reviewed-by: SJ Park Signed-off-by: SJ Park Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/damon/Makefile | 1 + .../testing/selftests/damon/sysfs_refresh.py | 75 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100755 tools/testing/selftests/damon/sysfs_refresh.py diff --git a/tools/testing/selftests/damon/Makefile b/tools/testing/selftests/damon/Makefile index 2180c328a825..ece244e5c5b9 100644 --- a/tools/testing/selftests/damon/Makefile +++ b/tools/testing/selftests/damon/Makefile @@ -13,6 +13,7 @@ TEST_PROGS += sysfs.py TEST_PROGS += sysfs_update_schemes_tried_regions_wss_estimation.py TEST_PROGS += damos_quota.py damos_quota_goal.py damos_apply_interval.py TEST_PROGS += damos_tried_regions.py damon_nr_regions.py +TEST_PROGS += sysfs_refresh.py TEST_PROGS += reclaim.sh lru_sort.sh # regression tests (reproducers of previously found bugs) diff --git a/tools/testing/selftests/damon/sysfs_refresh.py b/tools/testing/selftests/damon/sysfs_refresh.py new file mode 100755 index 000000000000..012b7e8f509f --- /dev/null +++ b/tools/testing/selftests/damon/sysfs_refresh.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 + +import os +import subprocess +import time + +import _damon_sysfs + +def main(): + # Continuously access a memory region for far longer than the test needs, + # so the kdamond always has a live target to monitor while we poll. + sz_region = 10 * 1024 * 1024 + proc = subprocess.Popen( + ['./access_memory', '1', '%d' % sz_region, '60000', 'repeat']) + + # A 'stat' scheme with the default (maximally wide) access pattern matches + # every monitored region, so its 'nr_tried' stat increases as the kdamond + # runs. refresh_ms should make DAMON update the schemes' stats files under + # sysfs on its own, without a manual 'update_schemes_stats' request. + kdamond = _damon_sysfs.Kdamond( + refresh_ms=100, + contexts=[_damon_sysfs.DamonCtx( + ops='vaddr', + targets=[_damon_sysfs.DamonTarget(pid=proc.pid)], + schemes=[_damon_sysfs.Damos(action='stat')], + )]) + kdamonds = _damon_sysfs.Kdamonds([kdamond]) + + err = kdamonds.start() + if err is not None: + # Kernels older than the refresh_ms feature have no such file; treat + # that as unsupported rather than a failure. + if not os.path.exists(os.path.join(kdamond.sysfs_dir(), 'refresh_ms')): + proc.terminate() + proc.wait() + print('kdamond has no refresh_ms file; skipping') + exit(_damon_sysfs.ksft_skip) + proc.terminate() + proc.wait() + print('kdamond start failed: %s' % err) + exit(1) + + scheme = kdamond.contexts[0].schemes[0] + nr_tried_path = os.path.join(scheme.sysfs_dir(), 'stats', 'nr_tried') + + try: + # Poll the stat file directly. We never request an update (e.g. + # 'update_schemes_stats'), so 'nr_tried' can become non-zero only + # through the periodic refresh that refresh_ms enables. + nr_tried = 0 + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + if proc.poll() is not None: + print('the access_memory target exited unexpectedly') + exit(1) + content, err = _damon_sysfs.read_file(nr_tried_path) + if err is not None: + print('reading %s failed: %s' % (nr_tried_path, err)) + exit(1) + nr_tried = int(content) + if nr_tried > 0: + break + time.sleep(0.1) + finally: + kdamonds.stop() + proc.terminate() + proc.wait() + + if nr_tried == 0: + print('refresh_ms did not auto-update the schemes stats') + exit(1) + +if __name__ == '__main__': + main() From 2817eaec9cfa3f819427be11abf11d11b8f6dd71 Mon Sep 17 00:00:00 2001 From: Akinobu Mita Date: Mon, 29 Jun 2026 07:55:32 -0700 Subject: [PATCH 110/501] mm/damon/core: use kvmalloc for target regions array Patch series "mm/damon: five misc fixups" Five patches for miscellaneous DAMON fixups. Use better fit kernel functions, cleanup/fixup documents, and add unit tests. The five patches were initially sent and revisioned by different individuals. Each patch contains changelog on their commentary area. The patches are curated into this series by SJ, for the convenience in reposting. This patch (of 5): damon_commit_target_regions() temporarily allocates a single contiguous memory region using kmalloc to store copies of all damon_regions of the damon_target. However, if the damon_target has a large number of damon_regions, the total size may exceed KMALLOC_MAX_SIZE. This problem can be avoided by using kvmalloc instead of kmalloc. Link: https://lore.kernel.org/20260629145538.134832-1-sj@kernel.org Link: https://lore.kernel.org/20260629145538.134832-2-sj@kernel.org Signed-off-by: Akinobu Mita Signed-off-by: SJ Park Reviewed-by: SJ Park Cc: Brendan Higgins Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Asier Gutierrez Cc: Doehyun Baek Cc: Philippe Laferriere Cc: Sailesh Nandanavanam Cc: Shuah Khan Signed-off-by: Andrew Morton --- mm/damon/core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index eab553fcb0b5..957cd8067bc0 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1369,14 +1369,14 @@ static int damon_commit_target_regions(struct damon_target *dst, if (!i) return 0; - ranges = kmalloc_objs(*ranges, i, GFP_KERNEL | __GFP_NOWARN); + ranges = kvmalloc_objs(*ranges, i, GFP_KERNEL | __GFP_NOWARN); if (!ranges) return -ENOMEM; i = 0; damon_for_each_region(src_region, src) ranges[i++] = src_region->ar; err = damon_set_regions(dst, ranges, i, src_min_region_sz); - kfree(ranges); + kvfree(ranges); return err; } From e214b3d81ac7f3353639df6dfa3d6947053f825d Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Mon, 29 Jun 2026 07:55:33 -0700 Subject: [PATCH 111/501] mm/damon/stat: use secs_to_jiffies() instead of msecs_to_jiffies() The conversion of a duration expressed in seconds reads as msecs_to_jiffies(5 * MSEC_PER_SEC), which obscures the intent and needlessly goes through milliseconds. Use the dedicated secs_to_jiffies() helper, which expresses the 5-second refresh interval directly. No functional change. Found using Coccinelle (scripts/coccinelle/misc/secs_to_jiffies.cocci). Link: https://lore.kernel.org/20260629145538.134832-3-sj@kernel.org Signed-off-by: Philippe Laferriere Signed-off-by: SJ Park Reviewed-by: SJ Park Cc: Akinobu Mita Cc: Asier Gutierrez Cc: Brendan Higgins Cc: David Hildenbrand Cc: Doehyun Baek Cc: Jonathan Corbet Cc: "Liam R. Howlett" Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Sailesh Nandanavanam Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/damon/stat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/damon/stat.c b/mm/damon/stat.c index 0e14f5bb8f75..b05b68f73e10 100644 --- a/mm/damon/stat.c +++ b/mm/damon/stat.c @@ -138,7 +138,7 @@ static int damon_stat_damon_call_fn(void *data) /* avoid unnecessarily frequent stat update */ if (time_before_eq(jiffies, damon_stat_last_refresh_jiffies + - msecs_to_jiffies(5 * MSEC_PER_SEC))) + secs_to_jiffies(5))) return 0; damon_stat_last_refresh_jiffies = jiffies; From 914a4582779871e737fd85f3402fbee72b2a8440 Mon Sep 17 00:00:00 2001 From: Doehyun Baek Date: Mon, 29 Jun 2026 07:55:34 -0700 Subject: [PATCH 112/501] Docs/{admin-guide,mm}/damon: fix DAMON documentation details Fix minor DAMON documentation issues. Correct the sysfs scheme file name apply_interval_us, the DAMON_STAT module count, a malformed reference, a misplaced label indentation, and a few typos. Link: https://lore.kernel.org/20260629145538.134832-4-sj@kernel.org Signed-off-by: Doehyun Baek Signed-off-by: SJ Park Reviewed-by: SJ Park Cc: David Hildenbrand Cc: Lorenzo Stoakes Cc: "Liam R. Howlett" Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Jonathan Corbet Cc: Shuah Khan Cc: Akinobu Mita Cc: Asier Gutierrez Cc: Brendan Higgins Cc: Philippe Laferriere Cc: Sailesh Nandanavanam Signed-off-by: Andrew Morton --- Documentation/admin-guide/mm/damon/usage.rst | 8 ++++---- Documentation/mm/damon/design.rst | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Documentation/admin-guide/mm/damon/usage.rst b/Documentation/admin-guide/mm/damon/usage.rst index 011296f1e7c2..b2649ea011f9 100644 --- a/Documentation/admin-guide/mm/damon/usage.rst +++ b/Documentation/admin-guide/mm/damon/usage.rst @@ -246,7 +246,7 @@ writing to and reading from the files. Under ``nr_regions`` directory, two files for the lower-bound and upper-bound of DAMON's monitoring regions (``min`` and ``max``, respectively), which controls the monitoring overhead, exist. You can set and get the values by -writing to and rading from the files. +writing to and reading from the files. For more details about the intervals and monitoring regions range, please refer to the Design document (:doc:`/mm/damon/design`). @@ -264,7 +264,7 @@ Please refer to the :ref:`design document of the feature ` for the internal of the tuning mechanism. Reading and writing the four files under ``intervals_goal`` directory shows and updates the tuning parameters that described in the -:ref:design doc ` with the same +:ref:`design doc ` with the same names. The tuning starts with the user-set ``sample_us`` and ``aggr_us``. The tuning-applied current values of the two intervals can be read from the ``sample_us`` and ``aggr_us`` files after writing ``update_tuned_intervals`` to @@ -377,7 +377,7 @@ schemes// In each scheme directory, nine directories (``access_pattern``, ``quotas``, ``watermarks``, ``core_filters``, ``ops_filters``, ``filters``, ``dests``, ``stats``, and ``tried_regions``) and three files (``action``, ``target_nid`` -and ``apply_interval``) exist. +and ``apply_interval_us``) exist. The ``action`` file is for setting and getting the scheme's :ref:`action `. The keywords that can be written to and read @@ -743,7 +743,7 @@ counter). Finally the tenth field (``X``) shows the ``age`` of the region (refer to :ref:`design ` for more details of the counter). -If the event was ``damon:damos_beofre_apply``, the ``perf script`` output would +If the event was ``damon:damos_before_apply``, the ``perf script`` output would be somewhat like below:: kdamond.0 47293 [000] 80801.060214: damon:damos_before_apply: ctx_idx=0 scheme_idx=0 target_idx=0 nr_regions=11 121932607488-135128711168: 0 136 diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index 2da7ca0d3d17..c16a3bb288d0 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -86,7 +86,7 @@ To know how user-space can do the configuration via :ref:`DAMON sysfs interface documentation. - .. _damon_design_vaddr_target_regions_construction: +.. _damon_design_vaddr_target_regions_construction: VMA-based Target Address Range Construction ------------------------------------------- @@ -930,11 +930,11 @@ control parameters for the usage would also need to be optimized for the purpose. To support such cases, yet more DAMON API user kernel modules that provide more -simple and optimized user space interfaces are available. Currently, two -modules for proactive reclamation and LRU lists manipulation are provided. For -more detail, please read the usage documents for those -(:doc:`/admin-guide/mm/damon/stat`, :doc:`/admin-guide/mm/damon/reclaim` and -:doc:`/admin-guide/mm/damon/lru_sort`). +simple and optimized user space interfaces are available. Currently, three +modules for access monitoring statistics, proactive reclamation, and LRU lists +manipulation are provided. For more detail, please read the usage documents for +those (:doc:`/admin-guide/mm/damon/stat`, :doc:`/admin-guide/mm/damon/reclaim` +and :doc:`/admin-guide/mm/damon/lru_sort`). .. _damon_design_special_purpose_modules_exclusivity: From 2ab5ed1c2bade928423991d526d879d134b60eec Mon Sep 17 00:00:00 2001 From: Asier Gutierrez Date: Mon, 29 Jun 2026 07:55:35 -0700 Subject: [PATCH 113/501] samples/damon: fix typos in Kconfig help text Fix a couple of typos in samples/damon/Kconfig help text. Change "Thps" to "This", and "tierign" to "tiering". Link: https://lore.kernel.org/20260629145538.134832-5-sj@kernel.org Signed-off-by: Asier Gutierrez Signed-off-by: SJ Park Reviewed-by: SJ Park Cc: Akinobu Mita Cc: Brendan Higgins Cc: David Hildenbrand Cc: Doehyun Baek Cc: Jonathan Corbet Cc: "Liam R. Howlett" Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Philippe Laferriere Cc: Sailesh Nandanavanam Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- samples/damon/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/damon/Kconfig b/samples/damon/Kconfig index cbf96fd8a8bf..00be3e6bdd65 100644 --- a/samples/damon/Kconfig +++ b/samples/damon/Kconfig @@ -31,7 +31,7 @@ config SAMPLE_DAMON_MTIER bool "DAMON sample module for memory tiering" depends on DAMON && DAMON_PADDR help - Thps builds DAMON sample module for memory tierign. + This builds DAMON sample module for memory tiering. The module assumes the system is constructed with two NUMA nodes, which seems as local and remote nodes to all CPUs. For example, From bafe4dbbd54cf167809c8c4a3c98f7592fa7a7a7 Mon Sep 17 00:00:00 2001 From: Sailesh Nandanavanam Date: Mon, 29 Jun 2026 07:55:36 -0700 Subject: [PATCH 114/501] mm/damon/tests/core-kunit: add KUnit test for walk_control_obsolete behavior Add a KUnit test to verify that damos_walk() rejects new requests when walk_control_obsolete is set. Commit 33c3f6c2b48c ("mm/damon/core: fix damos_walk() vs kdamond_fn() exit race") introduced walk_control_obsolete to prevent a race condition where new requests could be registered during kdamond shutdown and never handled. This test simulates the shutdown condition by setting walk_control_obsolete and verifies that damos_walk() returns -ECANCELED immediately. This validates the invariant introduced by the fix and helps prevent regressions. Link: https://patch.msgid.link/20260612062337.2459-1-saileshnandanavanam@gmail.com Link: https://lore.kernel.org/20260629145538.134832-6-sj@kernel.org Suggested-by: SJ Park Signed-off-by: Sailesh Nandanavanam Signed-off-by: SJ Park Reviewed-by: SJ Park Cc: Akinobu Mita Cc: Asier Gutierrez Cc: Brendan Higgins Cc: David Hildenbrand Cc: Doehyun Baek Cc: Jonathan Corbet Cc: "Liam R. Howlett" Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Philippe Laferriere Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/damon/tests/core-kunit.h | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index fcf7c7fadb5f..c5f5124c3d1f 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -1456,6 +1456,33 @@ static void damon_test_is_last_region(struct kunit *test) damon_free_target(t); } +/* + * Verify that damos_walk() rejects new requests when + * walk_control_obsolete is set. + * + * This tests the invariant introduced by: + * commit 33c3f6c2b48c ("mm/damon/core: fix damos_walk() vs kdamond_fn() exit race") + */ +static void damon_test_walk_control_obsolete(struct kunit *test) +{ + struct damon_ctx *ctx; + struct damos_walk_control control = {}; + int ret; + + ctx = damon_new_ctx(); + if (!ctx) + kunit_skip(test, "ctx alloc fail"); + + /* Simulate shutdown phase */ + ctx->walk_control_obsolete = true; + + ret = damos_walk(ctx, &control); + + KUNIT_EXPECT_EQ(test, ret, -ECANCELED); + + damon_destroy_ctx(ctx); +} + static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_target), KUNIT_CASE(damon_test_regions), @@ -1485,6 +1512,7 @@ static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_set_filters_default_reject), KUNIT_CASE(damon_test_apply_min_nr_regions), KUNIT_CASE(damon_test_is_last_region), + KUNIT_CASE(damon_test_walk_control_obsolete), {}, }; From 1a582e6e01744be5a70cdf77fa7ee8f19e6c3a5a Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Mon, 29 Jun 2026 07:56:28 -0700 Subject: [PATCH 115/501] mm/damon/core: split a fraction of regions when nr_regions exceeds max/2 Patch series "mm/damon/core: detect internal variation above max_nr_regions/2", v3. kdamond_split_regions() bails out early when nr_regions is already above max_nr_regions / 2. A large region that picks up new internal variation after that point never gets split, so we lose visibility into its hot/cold structure. We hit this with damon-paddr on hugepage workloads and damon-vaddr on processes that mmap a large anonymous range. Example with max_nr_regions == 1500. A target ends up with 799 small hot/cold regions plus one big region (an earlier merge collapsed a uniformly-accessed range into a single piece): H:hot C:cold r1 r2 r3 r800 HHHHHH|CCCCCC|HHHHHH|...|HHHHHH..........................| nr_regions = 800 > max_nr_regions / 2 = 750 Now a cold subarea shows up inside r800: r1 r2 r3 r800 HHHHHH|CCCCCC|HHHHHH|...|HHHHHH........CCCCCC.............| The small regions can't merge with each other (their access counts differ), so budget never frees up. r800 can't be split because nr_regions > max_nr_regions / 2 returns early. The cold subarea stays invisible. Patch 1 keeps refining on this path: when nr_regions is above max_nr_regions / 2 but still under the maximum, it splits a fraction of the regions instead of returning. The fraction shrinks as the remaining budget shrinks, so the count approaches max_nr_regions smoothly. A useless split is undone by the next merge cycle. Patch 2 adds a KUnit test for the case where nr_regions is already above max_nr_regions / 2. Thanks to SJ for the suggestion to drive the split fraction from the remaining budget rather than an age-based filter. This patch (of 2): kdamond_split_regions() returns early when nr_regions is above max_nr_regions / 2, leaving internal access variation inside a large region undetected. Such a layout is common with damon-paddr on hugepage workloads or damon-vaddr on processes with a large anonymous mmap. For example, with max_nr_regions == 1500, a target may end up with 799 small alternating-temperature regions plus one large region that absorbed a uniformly-accessed range during an earlier merge: H:hot C:cold r1 r2 r3 r800 HHHHHH|CCCCCC|HHHHHH|...|HHHHHH..........................| nr_regions = 800 > max_nr_regions / 2 = 750 If a cold subarea later emerges inside r800: r1 r2 r3 r800 HHHHHH|CCCCCC|HHHHHH|...|HHHHHH........CCCCCC.............| The small regions cannot merge with each other (different access counts), so the budget stays full. r800 cannot be split because nr_regions > max_nr_regions / 2 causes an early return. The cold subarea is never discovered. When nr_regions is above max_nr_regions / 2 but still under the maximum, split only a fraction of the regions instead of returning. One region in every 'max_nr_regions / budget' regions is split, where budget is the remaining room (max_nr_regions - nr_regions), starting from a rotating offset so different regions get picked over time. The fraction shrinks as the budget shrinks, so the region count keeps refining while approaching max_nr_regions smoothly rather than overshooting it. An unnecessary split is reverted by the next kdamond_merge_regions(). Link: https://lore.kernel.org/20260629145630.134891-1-sj@kernel.org Link: https://lore.kernel.org/20260626085851.70754-2-jiayuan.chen@linux.dev Link: https://lore.kernel.org/20260629145630.134891-2-sj@kernel.org Signed-off-by: Jiayuan Chen Signed-off-by: SJ Park Reviewed-by: SJ Park Cc: Shu Anzai Signed-off-by: Andrew Morton --- mm/damon/core.c | 49 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 957cd8067bc0..a99458c57851 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -3247,6 +3247,37 @@ static void damon_split_regions_of(struct damon_ctx *ctx, } } +/* Split one in every @split_step regions into two, from a rotating offset */ +static void damon_split_some_regions(struct damon_ctx *ctx, + unsigned long split_step) +{ + static unsigned long rotation; + struct damon_target *t; + struct damon_region *r, *next; + unsigned long offset = rotation++ % split_step; + unsigned long idx = 0; + + damon_for_each_target(t, ctx) { + damon_for_each_region_safe(r, next, t) { + unsigned long sz_region, sz_sub; + + if (idx++ % split_step != offset) + continue; + sz_region = damon_sz_region(r); + if (sz_region < 2 * ctx->min_region_sz) + continue; + + sz_sub = ALIGN_DOWN(damon_rand(ctx, 1, 10) * + sz_region / 10, ctx->min_region_sz); + /* Do not allow blank region */ + if (sz_sub == 0 || sz_sub >= sz_region) + continue; + + damon_split_region_at(t, r, sz_sub); + } + } +} + /* * Split every target region into randomly-sized small regions * @@ -3260,25 +3291,33 @@ static void damon_split_regions_of(struct damon_ctx *ctx, static void kdamond_split_regions(struct damon_ctx *ctx) { struct damon_target *t; - unsigned int nr_regions = 0; - static unsigned int last_nr_regions; + unsigned long nr_regions = 0; + unsigned long max_nr_regions = ctx->attrs.max_nr_regions; + static unsigned long last_nr_regions; int nr_subregions = 2; damon_for_each_target(t, ctx) nr_regions += damon_nr_regions(t); - if (nr_regions > ctx->attrs.max_nr_regions / 2) - return; + if (nr_regions >= max_nr_regions) + goto done; + + if (nr_regions > max_nr_regions / 2) { + damon_split_some_regions(ctx, + max_nr_regions / (max_nr_regions - nr_regions)); + goto done; + } /* Maybe the middle of the region has different access frequency */ if (last_nr_regions == nr_regions && - nr_regions < ctx->attrs.max_nr_regions / 3) + nr_regions < max_nr_regions / 3) nr_subregions = 3; damon_for_each_target(t, ctx) damon_split_regions_of(ctx, t, nr_subregions, ctx->min_region_sz); +done: last_nr_regions = nr_regions; } From 6958d08e3a6021190a496b4337569a0636fb3ffc Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Mon, 29 Jun 2026 07:56:29 -0700 Subject: [PATCH 116/501] mm/damon/tests/core-kunit: test split above max_nr_regions/2 Add a test that exercises kdamond_split_regions() when the total region count is already above max_nr_regions / 2, asserting that the function still splits a fraction of the regions (makes progress) and does not overshoot max_nr_regions. The region size and min_region_sz are picked so the split arithmetic does not depend on the page size. All tests pass: damon: pass:31 fail:0 skip:0 total:31 Totals: pass:31 fail:0 skip:0 total:31 Link: https://lore.kernel.org/20260626085851.70754-3-jiayuan.chen@linux.dev Link: https://lore.kernel.org/20260629145630.134891-3-sj@kernel.org Signed-off-by: Jiayuan Chen Signed-off-by: SJ Park Reviewed-by: SJ Park Cc: Shu Anzai Signed-off-by: Andrew Morton --- mm/damon/tests/core-kunit.h | 64 +++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index c5f5124c3d1f..a00168730445 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -335,6 +335,69 @@ static void damon_test_split_regions_of(struct kunit *test) damon_destroy_ctx(c); } +/* + * When the total region count is already above max_nr_regions / 2, + * kdamond_split_regions() must keep refining the resolution by splitting a + * fraction of the regions (making progress), without exceeding + * max_nr_regions. + */ +static void damon_test_split_above_half_progresses(struct kunit *test) +{ + struct damon_ctx *c; + struct damon_target *t; + struct damon_region *r; + unsigned long start; + unsigned int nr_before, nr_after, i; + const unsigned int nr_init = 760; + const unsigned long region_sz = 100; + + c = damon_new_ctx(); + if (!c) + kunit_skip(test, "ctx alloc fail"); + + /* Keep the split arithmetic independent of the page size */ + c->min_region_sz = 1; + c->attrs.min_nr_regions = 10; + c->attrs.max_nr_regions = 1500; + + t = damon_new_target(); + if (!t) { + damon_destroy_ctx(c); + kunit_skip(test, "target alloc fail"); + } + + for (i = 0; i < nr_init; i++) { + start = i * region_sz; + r = damon_new_region(start, start + region_sz); + if (!r) { + damon_free_target(t); + damon_destroy_ctx(c); + kunit_skip(test, "region alloc fail"); + } + r->nr_accesses = (i & 1) ? 0 : 100; + r->age = 5; + damon_add_region(r, t); + } + + damon_add_target(c, t); + + nr_before = damon_nr_regions(t); + /* Above max_nr_regions / 2, so the blanket-split path is skipped */ + KUNIT_EXPECT_GT(test, (unsigned long)nr_before, + c->attrs.max_nr_regions / 2); + + kdamond_split_regions(c); + + nr_after = damon_nr_regions(t); + /* Still made progress ... */ + KUNIT_EXPECT_GT(test, nr_after, nr_before); + /* ... but did not overshoot the configured maximum */ + KUNIT_EXPECT_LE(test, (unsigned long)nr_after, + c->attrs.max_nr_regions); + + damon_destroy_ctx(c); +} + static void damon_test_ops_registration(struct kunit *test) { struct damon_ctx *c = damon_new_ctx(); @@ -1491,6 +1554,7 @@ static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_merge_two), KUNIT_CASE(damon_test_merge_regions_of), KUNIT_CASE(damon_test_split_regions_of), + KUNIT_CASE(damon_test_split_above_half_progresses), KUNIT_CASE(damon_test_ops_registration), KUNIT_CASE(damon_test_set_regions), KUNIT_CASE(damon_test_nr_accesses_to_accesses_bp), From afa433c327fa34e48001ab6b50a2a7b756481b69 Mon Sep 17 00:00:00 2001 From: Baolin Wang Date: Mon, 29 Jun 2026 16:04:06 +0800 Subject: [PATCH 117/501] mm: vmscan: remove the redundant FOLIOREF_RECLAIM_CLEAN logic folio_check_references() will return FOLIOREF_RECLAIM_CLEAN for referenced file folios, indicating that we can proceed to reclaim clean file folios or keep them if they are dirty file folios. However, after commit 6b0dfabb3555 ("fs: Remove aops->writepage"), we no longer attempt to write back filesystem folios through reclaim. Instead, we always activate dirty file folios and wakeup the flush workers to write them back. As a result, the FOLIOREF_RECLAIM_CLEAN logic is now redundant: for dirty file folios, we will no longer reach the 'references == FOLIOREF_RECLAIM_CLEAN' branch in shrink_folio_list(). Additionally, lazyfree folios are also placed on the file LRU list, but if a lazyfree folio becomes dirty, try_to_unmap() will fail and thus prevent reclaim of the re-dirtied lazyfree folios. Therefore, we can drop the FOLIOREF_RECLAIM_CLEAN-related logic. Link: https://lore.kernel.org/def70a713e10bcbdf3b9fccc2139ecc07b64f2cb.1782715791.git.baolin.wang@linux.alibaba.com Signed-off-by: Baolin Wang Acked-by: Johannes Weiner Reviewed-by: Shakeel Butt Cc: Axel Rasmussen Cc: Barry Song Cc: David Hildenbrand Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Wei Xu Cc: Yuanchu Xie Signed-off-by: Andrew Morton --- mm/vmscan.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index de48b48470e6..c976cc8a2087 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -823,7 +823,6 @@ void folio_putback_lru(struct folio *folio) enum folio_references { FOLIOREF_RECLAIM, - FOLIOREF_RECLAIM_CLEAN, FOLIOREF_KEEP, FOLIOREF_ACTIVATE, }; @@ -920,10 +919,6 @@ static enum folio_references folio_check_references(struct folio *folio, return FOLIOREF_KEEP; } - /* Reclaim if clean, defer dirty folios to writeback */ - if (referenced_folio && folio_is_file_lru(folio)) - return FOLIOREF_RECLAIM_CLEAN; - return FOLIOREF_RECLAIM; } @@ -1235,7 +1230,6 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, stat->nr_ref_keep += nr_pages; goto keep_locked; case FOLIOREF_RECLAIM: - case FOLIOREF_RECLAIM_CLEAN: ; /* try to reclaim the folio below */ } @@ -1381,8 +1375,6 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, goto activate_locked; } - if (references == FOLIOREF_RECLAIM_CLEAN) - goto keep_locked; if (!may_enter_fs(folio, sc->gfp_mask)) goto keep_locked; if (!sc->may_writepage) From 608c74559718a2f3761a9806bc1115c184dae5dc Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Mon, 6 Jul 2026 04:42:03 -0700 Subject: [PATCH 118/501] mm: add softleaf_to_pmd() and convert existing callers Patch series "mm: preparatory patches for PMD level swap entries", v2. This is the preparatory part of the PMD page table swapin work. The full PMD swap entry series has been split into two parts: 1. this preparatory series, which contains the first 6 patches. Zi [1] and Lance [2] suggested to separate this out from the core series. 2. the PMD swap entry core series, which depends on this one. I will send this once the preparatory series is merged in mm-new as v3 as the combined is currently at v2 [1]. I have not marked this prep series as v3, as its not really adding support for PMD swap entries. This series does not introduce PMD swap entries and does not install any new page-table entry type. It only cleans up existing PMD softleaf helpers and call sites so the follow-up PMD swap entry series can be smaller and easier to review. It should be safe to merge independently. The patches are either helper additions, refactors of existing open-coded logic, defensive checks that preserve current migration/device-private behavior, or a mechanical rename of the PMD softleaf Kconfig gate. The follow-up series depends on these helpers, but this series does not depend on the follow-up series. Patch breakdown: 1. mm: add softleaf_to_pmd() and convert existing callers Add the PMD counterpart to softleaf_to_pte() and convert existing swp_entry_to_pmd() users that are constructing PMD softleaf entries. 2. mm: extract mm_prepare_for_swap_entries() helper Hoist the "register mm with swapoff" double-checked-locking pattern out of try_to_unmap_one() and copy_nonpresent_pte() so future PMD-level users do not need another open-coded copy. 3. fs/proc: use softleaf_has_pfn() in pagemap PMD walker Avoid assuming every non-present PMD softleaf entry encodes a PFN. Existing migration/device-private behavior is preserved. 4. mm/huge_memory: move softleaf_to_folio() inside migration branch Keep the folio lookup in change_non_present_huge_pmd() scoped to the migration-entry branch that actually needs it. 5. mm/migrate_device: move softleaf_to_folio() inside device-private branch Apply the same ordering cleanup to migrate_vma_collect_pmd(): only derive a folio after confirming the PMD entry is device-private. 6. mm: rename ARCH_ENABLE_THP_MIGRATION to ARCH_HAS_PMD_SOFTLEAVES Rename the architecture gate to describe what it actually enables: PMD softleaf entries. Migration remains the only current user in this series; the follow-up series adds PMD swap entries. This patch (of 6): Add softleaf_to_pmd() as the PMD counterpart to softleaf_to_pte(), completing the symmetry of the softleaf abstraction for page table leaf entries. The upcoming PMD swap entry support needs to construct PMD entries from swap entries. Converting existing swp_entry_to_pmd() callers to softleaf_to_pmd() in a prep patch keeps the feature patches focused on new functionality rather than mixing refactoring with new code. Link: https://lore.kernel.org/20260706114320.1643046-1-usama.arif@linux.dev Link: https://lore.kernel.org/20260706114320.1643046-2-usama.arif@linux.dev Link: https://lore.kernel.org/all/6E99CC4E-A026-4DE3-8A5A-34216771F521@nvidia.com/ [1] Link: https://lore.kernel.org/all/b08cafbb-a4b7-4609-84ae-dbb2cfcfc8be@linux.dev/#t [2] Link: https://lore.kernel.org/all/20260602142537.198755-1-usama.arif@linux.dev/ [3] Signed-off-by: Usama Arif Acked-by: David Hildenbrand (Arm) Reviewed-by: Zi Yan Reviewed-by: Lance Yang Cc: Alexandre Ghiti Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: Dev Jain Cc: "Huang, Ying" Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Kiryl Shutsemau Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Nhat Pham Cc: Nico Pache Cc: Rik van Riel Cc: Ryan Roberts Cc: Shakeel Butt Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/leafops.h | 20 ++++++++++++++++++++ mm/debug_vm_pgtable.c | 4 ++-- mm/huge_memory.c | 12 ++++++------ mm/migrate_device.c | 2 +- 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/include/linux/leafops.h b/include/linux/leafops.h index 992cd8bd8ed0..803d312437df 100644 --- a/include/linux/leafops.h +++ b/include/linux/leafops.h @@ -108,6 +108,21 @@ static inline softleaf_t softleaf_from_pmd(pmd_t pmd) return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry)); } +/** + * softleaf_to_pmd() - Obtain a PMD entry from a leaf entry. + * @entry: Leaf entry. + * + * This generates an architecture-specific PMD entry that can be utilised to + * encode the metadata the leaf entry encodes. + * + * Returns: Architecture-specific PMD entry encoding leaf entry. + */ +static inline pmd_t softleaf_to_pmd(softleaf_t entry) +{ + /* Temporary until swp_entry_t eliminated. */ + return swp_entry_to_pmd(entry); +} + #else static inline softleaf_t softleaf_from_pmd(pmd_t pmd) @@ -115,6 +130,11 @@ static inline softleaf_t softleaf_from_pmd(pmd_t pmd) return softleaf_mk_none(); } +static inline pmd_t softleaf_to_pmd(softleaf_t entry) +{ + return __pmd(0); +} + #endif /** diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c index 23dc3ee09561..18411fb09aab 100644 --- a/mm/debug_vm_pgtable.c +++ b/mm/debug_vm_pgtable.c @@ -758,7 +758,7 @@ static void __init pmd_leaf_soft_dirty_tests(struct pgtable_debug_args *args) return; pr_debug("Validating PMD swap soft dirty\n"); - pmd = swp_entry_to_pmd(args->leaf_entry); + pmd = softleaf_to_pmd(args->leaf_entry); WARN_ON(!pmd_is_huge(pmd)); WARN_ON(!pmd_is_valid_softleaf(pmd)); @@ -829,7 +829,7 @@ static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) return; pr_debug("Validating PMD swap\n"); - pmd1 = swp_entry_to_pmd(args->leaf_entry); + pmd1 = softleaf_to_pmd(args->leaf_entry); WARN_ON(!pmd_is_huge(pmd1)); WARN_ON(!pmd_is_valid_softleaf(pmd1)); diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 032702a4637b..b6f8c83c1000 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1819,7 +1819,7 @@ static void copy_huge_non_present_pmd( if (softleaf_is_migration_write(entry) || softleaf_is_migration_read_exclusive(entry)) { entry = make_readable_migration_entry(swp_offset(entry)); - pmd = swp_entry_to_pmd(entry); + pmd = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*src_pmd)) pmd = pmd_swp_mksoft_dirty(pmd); if (pmd_swp_uffd_wp(*src_pmd)) @@ -1832,7 +1832,7 @@ static void copy_huge_non_present_pmd( */ if (softleaf_is_device_private_write(entry)) { entry = make_readable_device_private_entry(swp_offset(entry)); - pmd = swp_entry_to_pmd(entry); + pmd = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*src_pmd)) pmd = pmd_swp_mksoft_dirty(pmd); @@ -2570,12 +2570,12 @@ static void change_non_present_huge_pmd(struct mm_struct *mm, entry = make_readable_exclusive_migration_entry(swp_offset(entry)); else entry = make_readable_migration_entry(swp_offset(entry)); - newpmd = swp_entry_to_pmd(entry); + newpmd = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*pmd)) newpmd = pmd_swp_mksoft_dirty(newpmd); } else if (softleaf_is_device_private_write(entry)) { entry = make_readable_device_private_entry(swp_offset(entry)); - newpmd = swp_entry_to_pmd(entry); + newpmd = softleaf_to_pmd(entry); if (pmd_swp_uffd_wp(*pmd)) newpmd = pmd_swp_mkuffd_wp(newpmd); } else { @@ -4927,7 +4927,7 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, } /* Set PMD. */ - pmdswp = swp_entry_to_pmd(entry); + pmdswp = softleaf_to_pmd(entry); if (softdirty) pmdswp = pmd_swp_mksoft_dirty(pmdswp); if (uffd_wp) @@ -4980,7 +4980,7 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) else entry = make_readable_device_private_entry( page_to_pfn(new)); - pmde = swp_entry_to_pmd(entry); + pmde = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*pvmw->pmd)) pmde = pmd_swp_mksoft_dirty(pmde); diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 06a5536e8e63..6f8b995810ff 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -836,7 +836,7 @@ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate, else swp_entry = make_readable_device_private_entry( page_to_pfn(page)); - entry = swp_entry_to_pmd(swp_entry); + entry = softleaf_to_pmd(swp_entry); } else { if (folio_is_zone_device(folio) && !folio_is_device_coherent(folio)) { From b707057443702993350c7a7740ef6135fee9befe Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Mon, 6 Jul 2026 04:42:04 -0700 Subject: [PATCH 119/501] mm: extract mm_prepare_for_swap_entries() helper When a swap entry is installed in a page table, the mm must be added to init_mm.mmlist so that swapoff can find and unuse its swap entries. This double-checked locking pattern is currently open-coded in try_to_unmap_one() and copy_nonpresent_pte(). Move it into mm_prepare_for_swap_entries() in mm/internal.h and convert both callers so it can be reused by upcoming PMD-level swap entry code paths that also need to register the mm with swapoff. copy_nonpresent_pte() previously inserted into &src_mm->mmlist rather than &init_mm.mmlist, but the insertion point is irrelevant, mmlist is a circular list and swapoff walks it entirely from init_mm.mmlist, so only membership matters, not position. Link: https://lore.kernel.org/20260706114320.1643046-3-usama.arif@linux.dev Signed-off-by: Usama Arif Reviewed-by: Dev Jain Reviewed-by: Zi Yan Acked-by: David Hildenbrand (Arm) Cc: Alexandre Ghiti Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: "Huang, Ying" Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Kiryl Shutsemau Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Nhat Pham Cc: Nico Pache Cc: Rik van Riel Cc: Ryan Roberts Cc: Shakeel Butt Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/internal.h | 10 ++++++++++ mm/memory.c | 9 +-------- mm/rmap.c | 7 +------ 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index 430aa72a4575..0360648e5726 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1955,4 +1955,14 @@ static inline int get_sysctl_max_map_count(void) bool may_expand_vm(struct mm_struct *mm, const vma_flags_t *vma_flags, unsigned long npages); +static inline void mm_prepare_for_swap_entries(struct mm_struct *mm) +{ + if (list_empty(&mm->mmlist)) { + spin_lock(&mmlist_lock); + if (list_empty(&mm->mmlist)) + list_add(&mm->mmlist, &init_mm.mmlist); + spin_unlock(&mmlist_lock); + } +} + #endif /* __MM_INTERNAL_H */ diff --git a/mm/memory.c b/mm/memory.c index a3fcaf8cfe8f..6637c5b13c9b 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -953,14 +953,7 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, if (swap_dup_entry_direct(entry) < 0) return -EIO; - /* make sure dst_mm is on swapoff's mmlist. */ - if (unlikely(list_empty(&dst_mm->mmlist))) { - spin_lock(&mmlist_lock); - if (list_empty(&dst_mm->mmlist)) - list_add(&dst_mm->mmlist, - &src_mm->mmlist); - spin_unlock(&mmlist_lock); - } + mm_prepare_for_swap_entries(dst_mm); /* Mark the swap entry as shared. */ if (pte_swp_exclusive(orig_pte)) { pte = pte_swp_clear_exclusive(orig_pte); diff --git a/mm/rmap.c b/mm/rmap.c index 1c77d5dc06e9..b93caabd186f 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2304,12 +2304,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, set_pte_at(mm, address, pvmw.pte, pteval); goto walk_abort; } - if (list_empty(&mm->mmlist)) { - spin_lock(&mmlist_lock); - if (list_empty(&mm->mmlist)) - list_add(&mm->mmlist, &init_mm.mmlist); - spin_unlock(&mmlist_lock); - } + mm_prepare_for_swap_entries(mm); dec_mm_counter(mm, MM_ANONPAGES); inc_mm_counter(mm, MM_SWAPENTS); swp_pte = swp_entry_to_pte(entry); From eecd06034bf494f985cbcd1f062391d6a38f61c4 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Mon, 6 Jul 2026 04:42:05 -0700 Subject: [PATCH 120/501] fs/proc: use softleaf_has_pfn() in pagemap PMD walker pagemap_pmd_range_thp() assumes that every non-present PMD is a migration entry and unconditionally calls softleaf_to_page(). This will crash on any non-present PMD type that does not encode a PFN, such as the upcoming PMD-level swap entries. Guard the page lookup with softleaf_has_pfn(), matching how pte_to_pagemap_entry() already handles non-present PTEs. Link: https://lore.kernel.org/20260706114320.1643046-4-usama.arif@linux.dev Signed-off-by: Usama Arif Acked-by: David Hildenbrand (Arm) Reviewed-by: Zi Yan Cc: Alexandre Ghiti Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: Dev Jain Cc: "Huang, Ying" Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Kiryl Shutsemau Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Nhat Pham Cc: Nico Pache Cc: Rik van Riel Cc: Ryan Roberts Cc: Shakeel Butt Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- fs/proc/task_mmu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 229d1fc3d7f1..95012243b9b3 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -2129,7 +2129,8 @@ static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr, flags |= PM_SOFT_DIRTY; if (pmd_swp_uffd_wp(pmd)) flags |= PM_UFFD_WP; - page = softleaf_to_page(entry); + if (softleaf_has_pfn(entry)) + page = softleaf_to_page(entry); } if (page) { From a6c30b073f75190bdfc6e1fd39eb6edd1e4d7eaf Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Mon, 6 Jul 2026 04:42:06 -0700 Subject: [PATCH 121/501] mm/huge_memory: move softleaf_to_folio() inside migration branch change_non_present_huge_pmd() calls softleaf_to_folio() unconditionally at the top of the function. softleaf_to_folio() extracts a PFN from the entry and converts it to a folio pointer, which is only meaningful for migration and device_private entries that encode a real PFN. A swap entry encodes a swap offset instead, so softleaf_to_folio() would produce a bogus pointer and crash on mprotect() when a PMD swap entry is present. Move the call into the migration_write branch where the folio is actually used, so the function is safe for any non-present PMD type. Link: https://lore.kernel.org/20260706114320.1643046-5-usama.arif@linux.dev Signed-off-by: Usama Arif Acked-by: David Hildenbrand (Arm) Reviewed-by: Dev Jain Reviewed-by: Zi Yan Cc: Alexandre Ghiti Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: "Huang, Ying" Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Kiryl Shutsemau Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Nhat Pham Cc: Nico Pache Cc: Rik van Riel Cc: Ryan Roberts Cc: Shakeel Butt Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/huge_memory.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index b6f8c83c1000..6ada1e862983 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2557,11 +2557,12 @@ static void change_non_present_huge_pmd(struct mm_struct *mm, bool uffd_wp_resolve) { softleaf_t entry = softleaf_from_pmd(*pmd); - const struct folio *folio = softleaf_to_folio(entry); pmd_t newpmd; VM_WARN_ON(!pmd_is_valid_softleaf(*pmd)); if (softleaf_is_migration_write(entry)) { + const struct folio *folio = softleaf_to_folio(entry); + /* * A protection check is difficult so * just be safe and disable write From 1f5a3d1afa5d89a1da77e3ad1654972d154831a3 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Mon, 6 Jul 2026 04:42:07 -0700 Subject: [PATCH 122/501] mm/migrate_device: move softleaf_to_folio() inside device-private branch migrate_vma_collect_pmd() calls softleaf_to_folio() on a non-present PMD before checking the entry's type. softleaf_to_folio() converts the entry's offset to a PFN, which is only meaningful for migration or device-private entries. A PMD swap entry's offset is a swap offset, not a PFN, so the lookup would either return a bogus folio pointer or trip pfn_to_page validation on a debug kernel. In the non-device-private path the returned folio is then unused (the OR short-circuits to migrate_vma_collect_skip()), but the lookup itself is already unsafe. Move the softleaf_to_folio() call inside the device-private branch where the folio is actually needed, mirroring the equivalent change_non_present_huge_pmd() fix. Link: https://lore.kernel.org/20260706114320.1643046-6-usama.arif@linux.dev Signed-off-by: Usama Arif Reviewed-by: Zi Yan Acked-by: David Hildenbrand (Arm) Cc: Alexandre Ghiti Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: Dev Jain Cc: "Huang, Ying" Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Kiryl Shutsemau Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Nhat Pham Cc: Nico Pache Cc: Rik van Riel Cc: Ryan Roberts Cc: Shakeel Butt Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/migrate_device.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 6f8b995810ff..39a6f5230a42 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -166,11 +166,14 @@ static int migrate_vma_collect_huge_pmd(pmd_t *pmdp, unsigned long start, } else if (!pmd_present(*pmdp)) { const softleaf_t entry = softleaf_from_pmd(*pmdp); - folio = softleaf_to_folio(entry); - if (!softleaf_is_device_private(entry) || - !(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_PRIVATE) || - (folio->pgmap->owner != migrate->pgmap_owner)) { + !(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_PRIVATE)) { + spin_unlock(ptl); + return migrate_vma_collect_skip(start, end, walk); + } + + folio = softleaf_to_folio(entry); + if (folio->pgmap->owner != migrate->pgmap_owner) { spin_unlock(ptl); return migrate_vma_collect_skip(start, end, walk); } From d547b1001755c170094c64bc45f9efba2c150feb Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Mon, 6 Jul 2026 04:42:08 -0700 Subject: [PATCH 123/501] mm: rename ARCH_ENABLE_THP_MIGRATION to ARCH_HAS_PMD_SOFTLEAVES CONFIG_ARCH_ENABLE_THP_MIGRATION gates PMD-level migration entries. PMD-level device-private entries use the same migration mechanism and therefore require the same architecture support. Upcoming PMD-level swap entries can use the same PMD softleaf helpers without depending on page migration, so rename the architecture gate to CONFIG_ARCH_HAS_PMD_SOFTLEAVES. This describes the PMD entry capability rather than one current user of it. This is a pure rename: the set of selecting architectures (x86, arm64, s390, riscv, loongarch, and powerpc on PPC_BOOK3S_64) and the gating semantics are unchanged. No functional change intended. Link: https://lore.kernel.org/20260706114320.1643046-7-usama.arif@linux.dev Signed-off-by: Usama Arif Reviewed-by: Zi Yan Acked-by: David Hildenbrand (Arm) Cc: Alexandre Ghiti Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: Dev Jain Cc: "Huang, Ying" Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Kiryl Shutsemau Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Nhat Pham Cc: Nico Pache Cc: Rik van Riel Cc: Ryan Roberts Cc: Shakeel Butt Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- arch/arm64/Kconfig | 2 +- arch/arm64/include/asm/pgtable.h | 4 ++-- arch/loongarch/Kconfig | 2 +- arch/powerpc/include/asm/book3s/64/pgtable.h | 2 +- arch/powerpc/platforms/Kconfig.cputype | 2 +- arch/riscv/Kconfig | 2 +- arch/riscv/include/asm/pgtable.h | 8 ++++---- arch/s390/Kconfig | 2 +- arch/s390/include/asm/pgtable.h | 2 +- arch/x86/Kconfig | 2 +- arch/x86/include/asm/pgtable.h | 2 +- include/linux/huge_mm.h | 2 +- include/linux/leafops.h | 8 ++++---- include/linux/pgtable.h | 2 +- include/linux/swapops.h | 6 +++--- mm/Kconfig | 2 +- mm/debug_vm_pgtable.c | 8 ++++---- mm/hmm.c | 4 ++-- mm/huge_memory.c | 2 +- mm/migrate.c | 4 ++-- mm/migrate_device.c | 6 +++--- mm/rmap.c | 2 +- 22 files changed, 38 insertions(+), 38 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b3afe0688919..e136e5abdb33 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -17,7 +17,7 @@ config ARM64 select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_MEMORY_HOTPLUG select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE select ARCH_HAS_CACHE_LINE_SIZE select ARCH_HAS_CC_PLATFORM select ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 27689c62bd25..ed1cc6dba780 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -1534,10 +1534,10 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) #define __swp_entry_to_pte(swp) ((pte_t) { (swp).val }) -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES #define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) }) #define __swp_entry_to_pmd(swp) __pmd((swp).val) -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ /* * Ensure that there are not more swap files than can be encoded in the kernel diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index d8d252325017..c6c611f44f3f 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -12,7 +12,7 @@ config LOONGARCH select ARCH_NEEDS_DEFER_KASAN select ARCH_DISABLE_KASAN_INLINE select ARCH_ENABLE_MEMORY_HOTPLUG - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_CPU_FINALIZE_INIT select ARCH_HAS_CURRENT_STACK_POINTER diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index e67e64ac6e8c..800ff3cc5f71 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -1060,7 +1060,7 @@ static inline pte_t *pmdp_ptep(pmd_t *pmd) #define pmd_mksoft_dirty(pmd) pte_pmd(pte_mksoft_dirty(pmd_pte(pmd))) #define pmd_clear_soft_dirty(pmd) pte_pmd(pte_clear_soft_dirty(pmd_pte(pmd))) -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES #define pmd_swp_mksoft_dirty(pmd) pte_pmd(pte_swp_mksoft_dirty(pmd_pte(pmd))) #define pmd_swp_soft_dirty(pmd) pte_swp_soft_dirty(pmd_pte(pmd)) #define pmd_swp_clear_soft_dirty(pmd) pte_pmd(pte_swp_clear_soft_dirty(pmd_pte(pmd))) diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype index bac02c83bb3e..9f6847b53e37 100644 --- a/arch/powerpc/platforms/Kconfig.cputype +++ b/arch/powerpc/platforms/Kconfig.cputype @@ -112,7 +112,7 @@ config PPC_THP depends on PPC_RADIX_MMU || (PPC_64S_HASH_MMU && PAGE_SIZE_64KB) select HAVE_ARCH_TRANSPARENT_HUGEPAGE select HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE choice prompt "CPU selection" diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index f7028caaeae0..760d0e0ae1f0 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -22,7 +22,7 @@ config RISCV select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM_VMEMMAP select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE select ARCH_HAS_BINFMT_FLAT select ARCH_HAS_CC_CAN_LINK select ARCH_HAS_CURRENT_STACK_POINTER diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 5d5756bda82e..23eae82c26a6 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -946,7 +946,7 @@ static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) return pte_pmd(pte_clear_soft_dirty(pmd_pte(pmd))); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static inline bool pmd_swp_soft_dirty(pmd_t pmd) { return pte_swp_soft_dirty(pmd_pte(pmd)); @@ -961,7 +961,7 @@ static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) { return pte_pmd(pte_swp_clear_soft_dirty(pmd_pte(pmd))); } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, @@ -1208,10 +1208,10 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) return __pte(pte_val(pte) & ~_PAGE_SWP_EXCLUSIVE); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES #define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) }) #define __swp_entry_to_pmd(swp) __pmd((swp).val) -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ /* * In the RV64 Linux scheme, we give the user half of the virtual-address space diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 84404e6778d5..6b317717bf48 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -82,7 +82,7 @@ config S390 select ARCH_CORRECT_STACKTRACE_ON_KRETPROBE select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE select ARCH_HAS_CC_CAN_LINK select ARCH_HAS_CPU_FINALIZE_INIT select ARCH_HAS_CURRENT_STACK_POINTER diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 859ce7c7d454..645fa2d99c6a 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -903,7 +903,7 @@ static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) return clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_SOFT_DIRTY)); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES #define pmd_swp_soft_dirty(pmd) pmd_soft_dirty(pmd) #define pmd_swp_mksoft_dirty(pmd) pmd_mksoft_dirty(pmd) #define pmd_swp_clear_soft_dirty(pmd) pmd_clear_soft_dirty(pmd) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index bdad90f210e4..50acb2830570 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -70,7 +70,7 @@ config X86 select ARCH_ENABLE_HUGEPAGE_MIGRATION if X86_64 && HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_MEMORY_HOTPLUG if X86_64 select ARCH_ENABLE_SPLIT_PMD_PTLOCK if (PGTABLE_LEVELS > 2) && (X86_64 || X86_PAE) - select ARCH_ENABLE_THP_MIGRATION if X86_64 && TRANSPARENT_HUGEPAGE + select ARCH_HAS_PMD_SOFTLEAVES if X86_64 && TRANSPARENT_HUGEPAGE select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_CPU_ATTACK_VECTORS if CPU_MITIGATIONS select ARCH_HAS_CACHE_LINE_SIZE diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index ac295ca6c92f..f69b02c1d3d3 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -1545,7 +1545,7 @@ static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) return pte_clear_flags(pte, _PAGE_SWP_SOFT_DIRTY); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) { return pmd_set_flags(pmd, _PAGE_SWP_SOFT_DIRTY); diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index ad20f7f8c179..b34e8e5e5f9a 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -567,7 +567,7 @@ static inline struct folio *get_persistent_huge_zero_folio(void) static inline bool thp_migration_supported(void) { - return IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION); + return IS_ENABLED(CONFIG_ARCH_HAS_PMD_SOFTLEAVES); } void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address, diff --git a/include/linux/leafops.h b/include/linux/leafops.h index 803d312437df..ef84ac1cce19 100644 --- a/include/linux/leafops.h +++ b/include/linux/leafops.h @@ -81,7 +81,7 @@ static inline pte_t softleaf_to_pte(softleaf_t entry) return swp_entry_to_pte(entry); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES /** * softleaf_from_pmd() - Obtain a leaf entry from a PMD entry. * @pmd: PMD entry. @@ -587,7 +587,7 @@ static inline bool pte_is_uffd_marker(pte_t pte) return false; } -#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_ENABLE_THP_MIGRATION) +#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_HAS_PMD_SOFTLEAVES) /** * pmd_is_device_private_entry() - Check if PMD contains a device private swap @@ -606,14 +606,14 @@ static inline bool pmd_is_device_private_entry(pmd_t pmd) return softleaf_is_device_private(softleaf_from_pmd(pmd)); } -#else /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static inline bool pmd_is_device_private_entry(pmd_t pmd) { return false; } -#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ /** * pmd_is_migration_entry() - Does this PMD entry encode a migration entry? diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index d52d2a976e5a..3df6db1e9e39 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1839,7 +1839,7 @@ static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) #endif #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY -#ifndef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifndef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) { return pmd; diff --git a/include/linux/swapops.h b/include/linux/swapops.h index 8cfc966eae48..1125401b787e 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -321,7 +321,7 @@ static inline swp_entry_t make_guard_swp_entry(void) struct page_vma_mapped_walk; -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page); @@ -338,7 +338,7 @@ static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) return __swp_entry_to_pmd(arch_entry); } -#else /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page) { @@ -358,7 +358,7 @@ static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) return __pmd(0); } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ #endif /* CONFIG_MMU */ #endif /* _LINUX_SWAPOPS_H */ diff --git a/mm/Kconfig b/mm/Kconfig index 9e0ca4824905..a360861c74fa 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -703,7 +703,7 @@ config DEVICE_MIGRATION config ARCH_ENABLE_HUGEPAGE_MIGRATION bool -config ARCH_ENABLE_THP_MIGRATION +config ARCH_HAS_PMD_SOFTLEAVES bool config HUGETLB_PAGE_SIZE_VARIABLE diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c index 18411fb09aab..7987962950c1 100644 --- a/mm/debug_vm_pgtable.c +++ b/mm/debug_vm_pgtable.c @@ -751,7 +751,7 @@ static void __init pmd_leaf_soft_dirty_tests(struct pgtable_debug_args *args) pmd_t pmd; if (!pgtable_supports_soft_dirty() || - !IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION)) + !IS_ENABLED(CONFIG_ARCH_HAS_PMD_SOFTLEAVES)) return; if (!has_transparent_hugepage()) @@ -819,7 +819,7 @@ static void __init pte_swap_tests(struct pgtable_debug_args *args) WARN_ON(memcmp(&pte1, &pte2, sizeof(pte1))); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) { swp_entry_t arch_entry; @@ -837,9 +837,9 @@ static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) pmd2 = __swp_entry_to_pmd(arch_entry); WARN_ON(memcmp(&pmd1, &pmd2, sizeof(pmd1))); } -#else /* !CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* !CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) { } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static void __init swap_migration_tests(struct pgtable_debug_args *args) { diff --git a/mm/hmm.c b/mm/hmm.c index c72c9ddfdb2f..e5c1f4deed24 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -331,7 +331,7 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr, return hmm_vma_fault(addr, end, required_fault, walk); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, unsigned long end, unsigned long *hmm_pfns, pmd_t pmd) @@ -391,7 +391,7 @@ static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, return -EFAULT; return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR); } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static int hmm_vma_walk_pmd(pmd_t *pmdp, unsigned long start, diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 6ada1e862983..fb2d7dd17de0 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -4868,7 +4868,7 @@ static int __init split_huge_pages_debugfs(void) late_initcall(split_huge_pages_debugfs); #endif -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page) { diff --git a/mm/migrate.c b/mm/migrate.c index 6fb6f46eea5f..3a426abc2c2b 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -362,7 +362,7 @@ static bool remove_migration_pte(struct folio *folio, idx = linear_page_index(vma, pvmw.address) - pvmw.pgoff; new = folio_page(folio, idx); -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES /* PMD-mapped THP migration entry */ if (!pvmw.pte) { VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) || @@ -545,7 +545,7 @@ void migration_entry_wait_huge(struct vm_area_struct *vma, unsigned long addr, p } #endif -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd) { spinlock_t *ptl; diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 39a6f5230a42..e74fee90489a 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -772,7 +772,7 @@ int migrate_vma_setup(struct migrate_vma *args) } EXPORT_SYMBOL(migrate_vma_setup); -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES /** * migrate_vma_insert_huge_pmd_page: Insert a huge folio into @migrate->vma->vm_mm * at @addr. folio is already allocated as a part of the migration process with @@ -927,7 +927,7 @@ static int migrate_vma_split_unmapped_folio(struct migrate_vma *migrate, migrate->src[i+idx] = migrate_pfn(pfn + i) | flags; return ret; } -#else /* !CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* !CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate, unsigned long addr, struct page *page, @@ -948,7 +948,7 @@ static int migrate_vma_split_unmapped_folio(struct migrate_vma *migrate, static unsigned long migrate_vma_nr_pages(unsigned long *src) { unsigned long nr = 1; -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES if (*src & MIGRATE_PFN_COMPOUND) nr = HPAGE_PMD_NR; #else diff --git a/mm/rmap.c b/mm/rmap.c index b93caabd186f..408e542649b5 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2472,7 +2472,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, page_vma_mapped_walk_restart(&pvmw); continue; } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES pmdval = pmdp_get(pvmw.pmd); if (likely(pmd_present(pmdval))) pfn = pmd_pfn(pmdval); From 6eceb61256ccbc54ae69e9f145840a6d0e8b1fe9 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 30 Jun 2026 07:17:15 -0700 Subject: [PATCH 124/501] Docs/mm/damon/design: update for DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP Patch series "mm/damon: update, optimize, and clean up doc, tests, and code". Patches 1 and 2 update the design and ABI documents for recently added DAMON features. Patches 3-7 add or update more unit and self tests for DAMON to cover recently changed or added functions and sysfs files. Patch 8 optimizes damon_commit_target_regions() to skip unnecessary adjacent ranges setup. Patches 9-11 clean and fix up recently added DAMON sysfs interface code for readability. This patch (of 11): Commit 9138e27a3bc3 ("mm/damon: add node_eligible_mem_bp goal metric") introduced DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP but forgot updating the DAMON design document for that. Update. Link: https://lore.kernel.org/20260630141726.92246-1-sj@kernel.org Link: https://lore.kernel.org/20260630141726.92246-2-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: SeongJae Park Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- Documentation/mm/damon/design.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index c16a3bb288d0..457d6e8bc787 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -686,9 +686,11 @@ mechanism tries to make ``current_value`` of ``target_metric`` be same to (1/10,000). - ``inactive_mem_bp``: Inactive to active + inactive (LRU) memory size ratio in bp (1/10,000). +- ``node_eligible_mem_bp``: Scheme target access pattern-eligible memory ratio + of a node in bp (1/10,000). -``nid`` is optionally required for only ``node_mem_used_bp``, -``node_mem_free_bp``, ``node_memcg_used_bp`` and ``node_memcg_free_bp`` to +``nid`` is optionally required for ``node_mem_used_bp``, ``node_mem_free_bp``, +``node_memcg_used_bp``, ``node_memcg_free_bp`` and ``node_eligible_mem_bp`` to point the specific NUMA node. ``path`` is optionally required for only ``node_memcg_used_bp`` and From fb6e06631d42a5833e423229f114dcc91cefae4c Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 30 Jun 2026 07:17:16 -0700 Subject: [PATCH 125/501] Docs/ABI/damon: document probe files DAMON ABI document is not updated for the DAMON probe sysfs files. Update. Link: https://lore.kernel.org/20260630141726.92246-3-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- .../ABI/testing/sysfs-kernel-mm-damon | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-damon b/Documentation/ABI/testing/sysfs-kernel-mm-damon index 4fdec63a47d4..dd6b5bd76e11 100644 --- a/Documentation/ABI/testing/sysfs-kernel-mm-damon +++ b/Documentation/ABI/testing/sysfs-kernel-mm-damon @@ -157,6 +157,46 @@ Description: Writing a value to this file sets the maximum number of monitoring regions of the DAMON context as the value. Reading this file returns the value. +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/nr_probes +Date: May 2026 +Contact: SJ Park +Description: Writing a number 'N' to this file creates the number of + directories for each DAMON probe named '0' to 'N-1' under the + probes/ directory. + +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/filters/nr_filters +Date: May 2026 +Contact: SJ Park +Description: Writing a number 'N' to this file creates the number of + directories for each DAMON probe filter named '0' to 'N-1' + under the filters/ directory. + +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/filters//type +Date: May 2026 +Contact: SJ Park +Description: Writing to and reading from this file sets and gets the type of + the memory of the interest. + +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/filters//path +Date: May 2026 +Contact: SJ Park +Description: If 'memcg' is written to the 'type' file, writing to and + reading from this file sets and gets the path to the memory + cgroup of the interest. + +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/filters//matching +Date: May 2026 +Contact: SJ Park +Description: Writing 'Y' or 'N' to this file sets whether the filter is for + the memory of the 'type', or all except the 'type'. + +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/filters//allow +Date: May 2026 +Contact: SJ Park +Description: Writing 'Y' or 'N' to this file sets whether to allow or reject + hitting the probe for the memory that satisfies the 'type' and + the 'matching' of the directory. + What: /sys/kernel/mm/damon/admin/kdamonds//contexts//targets/nr_targets Date: Mar 2022 Contact: SJ Park From 5b3c43405a68f1eee0399fc9f2e9264ed9a8f97a Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 30 Jun 2026 07:17:17 -0700 Subject: [PATCH 126/501] mm/damon/tests/core-kunit: test damon_rand() Commit 9012c4e647df ("mm/damon: replace damon_rand() with a per-ctx lockless PRNG") optimized DAMON for better performance. Add a kunit test for ensuring the bounds of the output. Link: https://lore.kernel.org/20260630141726.92246-4-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/damon/tests/core-kunit.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index a00168730445..0ec7d14d354e 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -1546,6 +1546,20 @@ static void damon_test_walk_control_obsolete(struct kunit *test) damon_destroy_ctx(ctx); } +static void damon_test_rand(struct kunit *test) +{ + struct damon_ctx ctx; + int i; + + prandom_seed_state(&ctx.rnd_state, get_random_u64()); + for (i = 0; i < 10000; i++) { + unsigned long rnd = damon_rand(&ctx, 0, 10); + + KUNIT_EXPECT_GE(test, rnd, 0); + KUNIT_EXPECT_LE(test, rnd, 9); + } +} + static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_target), KUNIT_CASE(damon_test_regions), @@ -1577,6 +1591,7 @@ static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_apply_min_nr_regions), KUNIT_CASE(damon_test_is_last_region), KUNIT_CASE(damon_test_walk_control_obsolete), + KUNIT_CASE(damon_test_rand), {}, }; From 1dfb454ea5590d17c228824c0c585ff9a30e565b Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 30 Jun 2026 07:17:18 -0700 Subject: [PATCH 127/501] selftests/damon/sysfs.sh: test multiple probe dirs creation DAMON sysfs essential file operations test (sysfs.sh) was extended to test DAMON probes sysfs directory, by commit 14885da09b0f ("selftests/damon/sysfs.sh: test probes dir"). Unlike other DAMON sysfs files, it is testing only a single directory case. Extend it for multiple directories. Link: https://lore.kernel.org/20260630141726.92246-5-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- tools/testing/selftests/damon/sysfs.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/testing/selftests/damon/sysfs.sh b/tools/testing/selftests/damon/sysfs.sh index 2eaaa5ae3c5e..d528dfea44c3 100755 --- a/tools/testing/selftests/damon/sysfs.sh +++ b/tools/testing/selftests/damon/sysfs.sh @@ -346,8 +346,13 @@ test_probes() ensure_write_succ "$probes_dir/nr_probes" "1" "valid input" test_probe "$probes_dir/0" + ensure_write_succ "$probes_dir/nr_probes" "2" "valid input" + test_probe "$probes_dir/0" + test_probe "$probes_dir/1" + ensure_write_succ "$probes_dir/nr_probes" "0" "valid input" ensure_dir "$probes_dir/0" "not_exist" + ensure_dir "$probes_dir/1" "not_exist" } test_monitoring_attrs() From 7acb8fafc14243b2d1a77f011065dcfced9539c9 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 30 Jun 2026 07:17:19 -0700 Subject: [PATCH 128/501] selftests/damon/sysfs.sh: test {core,ops}_filters/ directories DAMON sysfs interface essential file operations test (sysf.sh) is not testing DAMOS {core,ops}_filters directories. Add the tests. Link: https://lore.kernel.org/20260630141726.92246-6-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- tools/testing/selftests/damon/sysfs.sh | 28 ++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/damon/sysfs.sh b/tools/testing/selftests/damon/sysfs.sh index d528dfea44c3..78eea0d13c27 100755 --- a/tools/testing/selftests/damon/sysfs.sh +++ b/tools/testing/selftests/damon/sysfs.sh @@ -103,10 +103,28 @@ test_filter() { filter_dir=$1 ensure_file "$filter_dir/type" "exist" "600" - ensure_write_succ "$filter_dir/type" "anon" "valid input" - ensure_write_succ "$filter_dir/type" "memcg" "valid input" - ensure_write_succ "$filter_dir/type" "addr" "valid input" - ensure_write_succ "$filter_dir/type" "target" "valid input" + + local dir_name=$(basename "$(dirname "$filter_dir")") + if [ "$dir_name" = "filters" ] || [ "$dir_name" = "ops_filters" ] + then + ensure_write_succ "$filter_dir/type" "anon" "valid input" + ensure_write_succ "$filter_dir/type" "memcg" "valid input" + fi + if [ "$dir_name" = "filters" ] || [ "$dir_name" = "core_filters" ] + then + ensure_write_succ "$filter_dir/type" "addr" "valid input" + ensure_write_succ "$filter_dir/type" "target" "valid input" + fi + if [ "$dir_name" = "core_filters" ] + then + ensure_write_fail "$filter_dir/type" "anon" "ops type" + ensure_write_fail "$filter_dir/type" "memcg" "ops type" + fi + if [ "$dir_name" = "ops_filters" ] + then + ensure_write_fail "$filter_dir/type" "addr" "core type" + ensure_write_fail "$filter_dir/type" "target" "core type" + fi ensure_write_fail "$filter_dir/type" "foo" "invalid input" ensure_file "$filter_dir/matching" "exist" "600" ensure_file "$filter_dir/memcg_path" "exist" "600" @@ -208,6 +226,8 @@ test_scheme() test_quotas "$scheme_dir/quotas" test_watermarks "$scheme_dir/watermarks" test_filters "$scheme_dir/filters" + test_filters "$scheme_dir/core_filters" + test_filters "$scheme_dir/ops_filters" test_stats "$scheme_dir/stats" test_tried_regions "$scheme_dir/tried_regions" } From 01ff35b160bd286af825e3158d2e3a43da0e2f53 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 30 Jun 2026 07:17:20 -0700 Subject: [PATCH 129/501] selftests/damon/sysfs.sh: test dests dir DAMON selftest interface essential file operations test (sysfs.sh) is not testing DAMOS dests/ directory. Add the test. Link: https://lore.kernel.org/20260630141726.92246-7-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- tools/testing/selftests/damon/sysfs.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tools/testing/selftests/damon/sysfs.sh b/tools/testing/selftests/damon/sysfs.sh index 78eea0d13c27..f8d2092be004 100755 --- a/tools/testing/selftests/damon/sysfs.sh +++ b/tools/testing/selftests/damon/sysfs.sh @@ -99,6 +99,29 @@ test_stats() done } +test_dest() +{ + dest_dir=$1 + ensure_file "$dest_dir/id" "exist" "600" + ensure_file "$dest_dir/weight" "exist" "600" +} + +test_dests() +{ + dests_dir=$1 + ensure_file "$dests_dir/nr_dests" "exist" "600" + ensure_write_succ "$dests_dir/nr_dests" "1" "valid input" + test_dest "$dests_dir/0" + + ensure_write_succ "$dests_dir/nr_dests" "2" "valid input" + test_dest "$dests_dir/0" + test_dest "$dests_dir/1" + + ensure_write_succ "$dests_dir/nr_dests" "0" "valid input" + ensure_dir "$dests_dir/0" "not_exist" + ensure_dir "$dests_dir/1" "not_exist" +} + test_filter() { filter_dir=$1 @@ -225,6 +248,7 @@ test_scheme() ensure_file "$scheme_dir/apply_interval_us" "exist" "600" test_quotas "$scheme_dir/quotas" test_watermarks "$scheme_dir/watermarks" + test_dests "$scheme_dir/dests" test_filters "$scheme_dir/filters" test_filters "$scheme_dir/core_filters" test_filters "$scheme_dir/ops_filters" From 8fa52376da330889a425cc460396d4d2cac18f46 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 30 Jun 2026 07:17:21 -0700 Subject: [PATCH 130/501] selftests/damon/sysfs.sh: test all files in quota goal dir DAMON sysfs interface for DAMOS quota has quite extended since its initial introduction. The test case for that in DAMON sysfs interface essential file operations test (sysfs.sh) has not accordingly extended, though. Extend the test case to test all existing files. Link: https://lore.kernel.org/20260630141726.92246-8-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- tools/testing/selftests/damon/sysfs.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tools/testing/selftests/damon/sysfs.sh b/tools/testing/selftests/damon/sysfs.sh index f8d2092be004..b43deee60fe9 100755 --- a/tools/testing/selftests/damon/sysfs.sh +++ b/tools/testing/selftests/damon/sysfs.sh @@ -199,6 +199,20 @@ test_goal() ensure_dir "$goal_dir" "exist" ensure_file "$goal_dir/target_value" "exist" "600" ensure_file "$goal_dir/current_value" "exist" "600" + ensure_file "$goal_dir/target_metric" "exist" "600" + local fpath="$goal_dir/target_metric" + ensure_write_succ "$fpath" "user_input" "valid input" + ensure_write_succ "$fpath" "some_mem_psi_us" "valid input" + ensure_write_succ "$fpath" "node_mem_used_bp" "valid input" + ensure_write_succ "$fpath" "node_mem_free_bp" "valid input" + ensure_write_succ "$fpath" "node_memcg_used_bp" "valid input" + ensure_write_succ "$fpath" "node_memcg_free_bp" "valid input" + ensure_write_succ "$fpath" "active_mem_bp" "valid input" + ensure_write_succ "$fpath" "inactive_mem_bp" "valid input" + ensure_write_succ "$fpath" "node_eligible_mem_bp" "valid input" + ensure_write_fail "$fpath" "foo" "invalid input" + ensure_file "$goal_dir/nid" "exist" "600" + ensure_file "$goal_dir/path" "exist" "600" } test_goals() From 97ae89aa9f634b6cbf2560ffad47a93eda2bde4f Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 30 Jun 2026 07:17:22 -0700 Subject: [PATCH 131/501] mm/damon/core: reduce range setup in damon_commit_target_regions() damon_commit_target_regions() calls damon_set_regions() for updating the destination target's monitoring target region boundaries. It sets the boundaries same to source target's monitoring regions, even if they are adjacent. Meanwhile, damon_set_region() sets the destination target regions exactly the same to the source, only when the target regions are empty. When there are existing target regions, only a few regions are expanded or shrunk to fit on only the boundaries for disjoint regions in the source. Hence the adjacent source ranges mean nothing in common cases. When there are many regions, such adjacent range setup is only a waste of time and space. We recently found [1] it is actually causing memory overhead. Setup the ranges for only distinct ranges. Link: https://lore.kernel.org/20260630141726.92246-9-sj@kernel.org Link: https://lore.kernel.org/20260603112306.58490-1-akinobu.mita@gmail.com [1] Signed-off-by: SJ Park Cc: Brendan Higgins Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/damon/core.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index a99458c57851..dbff0e4c1c17 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1360,21 +1360,33 @@ static struct damon_target *damon_nth_target(int n, struct damon_ctx *ctx) static int damon_commit_target_regions(struct damon_target *dst, struct damon_target *src, unsigned long src_min_region_sz) { - struct damon_region *src_region; + struct damon_region *src_region, *prev = NULL; struct damon_addr_range *ranges; int i = 0, err; - damon_for_each_region(src_region, src) - i++; + damon_for_each_region(src_region, src) { + if (!prev || prev->ar.end != src_region->ar.start) + i++; + prev = src_region; + } if (!i) return 0; ranges = kvmalloc_objs(*ranges, i, GFP_KERNEL | __GFP_NOWARN); if (!ranges) return -ENOMEM; + prev = NULL; i = 0; - damon_for_each_region(src_region, src) - ranges[i++] = src_region->ar; + damon_for_each_region(src_region, src) { + if (!prev) { + ranges[i].start = src_region->ar.start; + } else if (prev->ar.end != src_region->ar.start) { + ranges[i++].end = prev->ar.end; + ranges[i].start = src_region->ar.start; + } + prev = src_region; + } + ranges[i++].end = damon_last_region(src)->ar.end; err = damon_set_regions(dst, ranges, i, src_min_region_sz); kvfree(ranges); return err; From 8c02207a064ddf66ae1d4ba1fbb9a50d1a127bd7 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 30 Jun 2026 07:17:23 -0700 Subject: [PATCH 132/501] mm/damon/sysfs: split probe setup function out damon_sysfs_set_probes() function is relatively long. It has two nested loop for setting two nested entities, namely probe and filter. Split out the probe level setup for readability. Link: https://lore.kernel.org/20260630141726.92246-10-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/damon/sysfs.c | 80 ++++++++++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 34 deletions(-) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index e3526a263e20..02afd37b04df 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -1909,47 +1909,59 @@ static int damon_sysfs_set_attrs(struct damon_ctx *ctx, return damon_set_attrs(ctx, &attrs); } +static int damon_sysfs_set_probe(struct damon_probe *probe, + struct damon_sysfs_probe *sys_probe) +{ + struct damon_sysfs_filters *sys_filters; + int i; + + sys_filters = sys_probe->filters; + if (!sys_filters) + return 0; + + for (i = 0; i < sys_filters->nr; i++) { + struct damon_sysfs_filter *sys_filter = + sys_filters->filters_arr[i]; + struct damon_filter *filter; + + filter = damon_new_filter(sys_filter->type, + sys_filter->matching, + sys_filter->allow); + if (!filter) + return -ENOMEM; + if (filter->type == DAMON_FILTER_TYPE_MEMCG) { + int err; + + err = damon_sysfs_memcg_path_to_id( + sys_filter->path, + &filter->memcg_id); + if (err) { + damon_destroy_filter(filter); + return err; + } + } + damon_add_filter(probe, filter); + } + return 0; +} + static int damon_sysfs_set_probes(struct damon_ctx *ctx, struct damon_sysfs_probes *sys_probes) { - int i; + int i, err; for (i = 0; i < sys_probes->nr; i++) { - struct damon_sysfs_filters *sys_filters = - sys_probes->probes_arr[i]->filters; - struct damon_probe *c; - int j; + struct damon_sysfs_probe *sys_probe; + struct damon_probe *p; - if (!sys_filters) - continue; - c = damon_new_probe(); - if (!c) + p = damon_new_probe(); + if (!p) return -ENOMEM; - damon_add_probe(ctx, c); - - for (j = 0; j < sys_filters->nr; j++) { - struct damon_sysfs_filter *sys_filter = - sys_filters->filters_arr[j]; - struct damon_filter *filter; - - filter = damon_new_filter(sys_filter->type, - sys_filter->matching, - sys_filter->allow); - if (!filter) - return -ENOMEM; - if (filter->type == DAMON_FILTER_TYPE_MEMCG) { - int err; - - err = damon_sysfs_memcg_path_to_id( - sys_filter->path, - &filter->memcg_id); - if (err) { - damon_destroy_filter(filter); - return err; - } - } - damon_add_filter(c, filter); - } + damon_add_probe(ctx, p); + sys_probe = sys_probes->probes_arr[i]; + err = damon_sysfs_set_probe(p, sys_probe); + if (err) + return err; } return 0; } From ea0e5f5786807ded4f0be8d195aa380e5d90f303 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 30 Jun 2026 07:17:24 -0700 Subject: [PATCH 133/501] mm/damon/sysfs: split out filters setup function damon_sysfs_set_probe() is doing not only probe setup but also filters setup. Split out filters setup for readability. Link: https://lore.kernel.org/20260630141726.92246-11-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/damon/sysfs.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 02afd37b04df..5b2ae2fe948f 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -1909,16 +1909,11 @@ static int damon_sysfs_set_attrs(struct damon_ctx *ctx, return damon_set_attrs(ctx, &attrs); } -static int damon_sysfs_set_probe(struct damon_probe *probe, - struct damon_sysfs_probe *sys_probe) +static int damon_sysfs_set_filters(struct damon_probe *probe, + struct damon_sysfs_filters *sys_filters) { - struct damon_sysfs_filters *sys_filters; int i; - sys_filters = sys_probe->filters; - if (!sys_filters) - return 0; - for (i = 0; i < sys_filters->nr; i++) { struct damon_sysfs_filter *sys_filter = sys_filters->filters_arr[i]; @@ -1945,6 +1940,17 @@ static int damon_sysfs_set_probe(struct damon_probe *probe, return 0; } +static int damon_sysfs_set_probe(struct damon_probe *probe, + struct damon_sysfs_probe *sys_probe) +{ + struct damon_sysfs_filters *sys_filters; + + sys_filters = sys_probe->filters; + if (!sys_filters) + return 0; + return damon_sysfs_set_filters(probe, sys_filters); +} + static int damon_sysfs_set_probes(struct damon_ctx *ctx, struct damon_sysfs_probes *sys_probes) { From 6028a90f33b4354f94cd4b5ec827b858b9a2aefa Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 30 Jun 2026 07:17:25 -0700 Subject: [PATCH 134/501] mm/damon/sysfs: fix typos in probe_{add,rm}_dirs: s/attr/probe/ damon_sysfs_probe_{add,rm}_dirs names a variable for damon_sysf_probe as 'attr'. Probably a trivial copy-pasta error, but it makes the code not pleasant to read. Fix those. Link: https://lore.kernel.org/20260630141726.92246-12-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/damon/sysfs.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 5b2ae2fe948f..6710b6d019bf 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -1073,7 +1073,7 @@ static struct damon_sysfs_probe *damon_sysfs_probe_alloc(void) return kzalloc_obj(struct damon_sysfs_probe); } -static int damon_sysfs_probe_add_dirs(struct damon_sysfs_probe *attr) +static int damon_sysfs_probe_add_dirs(struct damon_sysfs_probe *probe) { struct damon_sysfs_filters *filters; int err; @@ -1081,22 +1081,22 @@ static int damon_sysfs_probe_add_dirs(struct damon_sysfs_probe *attr) filters = damon_sysfs_filters_alloc(); if (!filters) return -ENOMEM; - attr->filters = filters; + probe->filters = filters; err = kobject_init_and_add(&filters->kobj, &damon_sysfs_filters_ktype, - &attr->kobj, "filters"); + &probe->kobj, "filters"); if (err) { kobject_put(&filters->kobj); - attr->filters = NULL; + probe->filters = NULL; } return err; } -static void damon_sysfs_probe_rm_dirs(struct damon_sysfs_probe *attr) +static void damon_sysfs_probe_rm_dirs(struct damon_sysfs_probe *probe) { - if (attr->filters) { - damon_sysfs_filters_rm_dirs(attr->filters); - kobject_put(&attr->filters->kobj); + if (probe->filters) { + damon_sysfs_filters_rm_dirs(probe->filters); + kobject_put(&probe->filters->kobj); } } From 9f2fd03c9bde366c928c9242fc9b2e2c5111a134 Mon Sep 17 00:00:00 2001 From: JP Kobryn Date: Mon, 22 Jun 2026 17:46:00 -0700 Subject: [PATCH 135/501] mm/page_alloc: use existing highatomic reserves on the buddy fastpath ALLOC_HIGHATOMIC currently provides both access to MIGRATE_HIGHATOMIC free pages and permission to create new highatomic pageblock reserves. This makes it unsuitable for the fastpath. However, the fastpath can reach rmqueue_buddy() while MIGRATE_HIGHATOMIC reserves have free pages available. In this situation, the allocation can fall back to other migratetypes without trying those reserves first. Allow high-priority non-blocking allocations to use existing MIGRATE_HIGHATOMIC reserves on the buddy fastpath without growing them. First tighten the criteria for reserving pageblocks so that growth may only occur in the slowpath. Then allow fastpath usage by enabling ALLOC_HIGHATOMIC when the GFP mask describes a non-blocking high-priority allocation. This logic has been factored out from gfp_to_alloc_flags() to a new function gfp_to_alloc_flags_nonblocking(). A UDP receive workload was run with free MIGRATE_HIGHATOMIC pageblocks available in the target zone. Before this patch, the workload did not consume these blocks. With this patch, eligible order-1 allocations reaching the buddy path consumed existing MIGRATE_HIGHATOMIC pageblocks, with no highatomic misses observed. The workload did not grow highatomic reserves and NAPI page-frag allocations remained healthy with no failures or order-0 fallbacks. Link: https://lore.kernel.org/20260623004600.113347-1-jp.kobryn@linux.dev Signed-off-by: JP Kobryn Reviewed-by: Vlastimil Babka (SUSE) Acked-by: Johannes Weiner Reviewed-by: Shakeel Butt Cc: Brendan Jackman Cc: David Hildenbrand Cc: Frank van der Linden Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/page_alloc.c | 44 ++++++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 549fa83045eb..62f71ece7ca1 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -3249,10 +3249,11 @@ struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone, } while (check_new_pages(page, order)); /* - * If this is a high-order atomic allocation then check - * if the pageblock should be reserved for the future + * Slowpath (precarious) high-atomic allocations may reserve + * a pageblock for future use. */ - if (unlikely(alloc_flags & ALLOC_HIGHATOMIC)) + if (unlikely((alloc_flags & ALLOC_HIGHATOMIC) && + ((alloc_flags & ALLOC_WMARK_MASK) == ALLOC_WMARK_MIN))) reserve_highatomic_pageblock(page, order, zone); __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); @@ -4472,6 +4473,29 @@ static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, } } +static inline unsigned int +gfp_to_alloc_flags_nonblocking(gfp_t gfp_mask, unsigned int order) +{ + unsigned int alloc_flags = 0; + + if (gfp_mask & __GFP_DIRECT_RECLAIM) + return 0; + + /* + * Not worth trying to allocate harder for __GFP_NOMEMALLOC even + * if it can't schedule. + */ + if (gfp_mask & __GFP_NOMEMALLOC) + return 0; + + alloc_flags |= ALLOC_NON_BLOCK; + + if (order > 0 && (gfp_mask & __GFP_HIGH)) + alloc_flags |= ALLOC_HIGHATOMIC; + + return alloc_flags; +} + static inline unsigned int gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) { @@ -4488,18 +4512,9 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) if (gfp_mask & __GFP_KSWAPD_RECLAIM) alloc_flags |= ALLOC_KSWAPD; + alloc_flags |= gfp_to_alloc_flags_nonblocking(gfp_mask, order); + if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) { - /* - * Not worth trying to allocate harder for __GFP_NOMEMALLOC even - * if it can't schedule. - */ - if (!(gfp_mask & __GFP_NOMEMALLOC)) { - alloc_flags |= ALLOC_NON_BLOCK; - - if (order > 0 && (alloc_flags & ALLOC_MIN_RESERVE)) - alloc_flags |= ALLOC_HIGHATOMIC; - } - /* * Ignore cpuset mems for non-blocking __GFP_HIGH (probably * GFP_ATOMIC) rather than fail, see the comment for @@ -5292,6 +5307,7 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, * memory until all local zones are considered. */ alloc_flags |= alloc_flags_nofragment(zonelist_zone(ac.preferred_zoneref), gfp); + alloc_flags |= gfp_to_alloc_flags_nonblocking(gfp, order) & ALLOC_HIGHATOMIC; /* First allocation attempt */ page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac); From bd1e4c4aa469c9e946d9aa2831b8cd4ff3c198f5 Mon Sep 17 00:00:00 2001 From: Wentao Guan Date: Tue, 30 Jun 2026 19:38:57 +0800 Subject: [PATCH 136/501] mm/hugetlb: avoid unnecessary TLB flush for empty folio list in vmemmap optimize Since 79359d6d24df ("hugetlb: perform vmemmap optimization on a list of pages") __hugetlb_vmemmap_optimize_folios() unconditionally issues a final flush_tlb_all() in its out path. However, a TLB flush must be paired with an actual page table modification. When the input folio list is empty, neither PMD splitting nor PTE remapping takes place, so no page tables are modified and the flush is pure overhead. An empty list is reached in common paths such as gather_bootmem_prealloc_node() on nodes without bootmem gigantic pages, hugetlb_pages_alloc_boot_node() when no pages were allocated, and runtime allocation failure paths in set_max_huge_pages(). Add an early return for empty lists. This restores the basic invariant that TLB flushes are only issued when page tables have been modified, and it also makes the NULL hstate passed by gather_bootmem_prealloc_node() on an empty list harmless. Assisted-by: kimi-cli:kimi-k2.7 code Assisted-by: Github Copilot:gpt-5.2 #Reported-by Link: https://lore.kernel.org/20260701053422.3664813-1-guanwentao@uniontech.com Link: https://lore.kernel.org/20260630113857.3319612-1-guanwentao@uniontech.com Fixes: 79359d6d24df ("hugetlb: perform vmemmap optimization on a list of pages") Signed-off-by: Wentao Guan Reviewed-by: Muchun Song Cc: David Hildenbrand Cc: Guan Wentao Cc: Oscar Salvador Signed-off-by: Andrew Morton --- mm/hugetlb_vmemmap.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index eefd6b5f9706..1430a5aa2de5 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -624,6 +624,9 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h, LIST_HEAD(vmemmap_pages); unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH; + if (list_empty(folio_list)) + return; + nr_to_optimize = 0; list_for_each_entry(folio, folio_list, lru) { int ret; From a33b5c9116554c48adec9e8476c137f49bdb2043 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Tue, 30 Jun 2026 04:23:32 -0700 Subject: [PATCH 137/501] mm/vmpressure: skip tree=true accounting on cgroup v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch series "mm/vmpressure: reduce CPU, memory and code overhead on cgroup v2", v3. The vmpressure subsystem has two distinct consumers, gated by the @tree argument: tree=false : in-kernel socket pressure, consumed by TCP/SCTP. This is cgroup v2 only; v1 sockets read memcg->tcpmem_pressure instead. tree=true : cgroup v1 userspace eventfd notifications via the memory.pressure_level / cgroup.event_control interface. v2 has no equivalent (userspace gets reclaim signals through memory.pressure / PSI, which doesn't touch vmpressure). So of the four (hierarchy, tree) combinations, only two carry data that anyone reads. The existing early return in vmpressure() covered v1 + tree=false; the symmetric v2 + tree=true case was falling through and doing the full lock / accumulate / schedule_work / parent-walk dance, even though the events list it eventually iterates is empty on cgroup v2 (vmpressure_register_event() is wired up only through the v1 cftype "memory.pressure_level" and can't be reached from a v2 memcg). Patch 1 extends the existing early return to also skip v2 + tree=true. On a v2-only host this eliminates a contended path where reclaimers can serialize on a single global sr_lock. bpftrace on a 176-core production host (cgroup v2, 285 memcgs, sustained reclaim) showed ~16,200 such calls per minute with tree = true. Patch 2 follows up with a cleanup: it splits the v1 userspace eventfd interface (struct vmpressure_event, the events list and its mutex, the work_struct and its handler, the parent walk, vmpressure_register_event / unregister_event, and vmpressure_prio) into a new mm/memcontrol-v1.c built only when CONFIG_MEMCG_V1=y, behind small no-op stubs in the header. mm/vmpressure.c keeps the shared bits and the tree=false socket-pressure path. The size of vmpressure.c goes down to half and the code is much more simpler. The only #ifdef CONFIG_MEMCG_V1 remaining in source is around the v1-only fields inside struct vmpressure itself. Memory savings on CONFIG_MEMCG_V1=n: struct vmpressure : 112B -> 24B struct mem_cgroup : 1664B -> 1536B This split is the first step toward eventually making vmpressure CONFIG_MEMCG_V1 only. The v2 in-kernel socket pressure path (tree=false) cannot be removed today immediately: PSI is not an exact replacement for vmpressure, and switching networking socket-buffer back-off to PSI may regress networking performance or increase memory pressure in workloads that today rely on vmpressure's hysteresis. The medium-term plan is to introduce a PSI-based socket-pressure path, keep vmpressure available for v2 behind a defconfig as an opt-out for several releases, and only then drop the tree=false path entirely, at which point everything that remains in mm/memcontrol-v1.c is the whole subsystem. This patch (of 2): vmpressure() has two outputs gated by the @tree argument: @tree=false drives in-kernel socket pressure (mem_cgroup_set_ socket_pressure), consumed by TCP/SCTP. This only applies on cgroup v2; on v1 socket memory is charged separately via tcpmem and the consumer reads memcg->tcpmem_pressure instead. @tree=true drives userspace eventfd notifications via the v1 memory.pressure_level / cgroup.event_control interface. v2 has no equivalent: userspace gets reclaim signals through memory.pressure (PSI), which does not touch vmpressure. The existing early return covered v1 + @tree=false. The symmetric v2 + @tree=true case was falling through and doing the full lock / accumulate / schedule_work / parent-walk dance for an events list that can never be populated. bpftrace on a 176-core production host (cgroup v2, CONFIG_MEMCG_V1=n, 285 memcgs, sustained reclaim) showed ~16,200 @tree=true vmpressure() calls per minute. Add an early return that skips cgroup v2 + tree = true which avoids us doing all this work. On a v2-only host this also eliminates a lock contention path that can serialise reclaimers on a single global sr_lock. [usama.arif@linux.dev: simplify the guard] Link: https://lore.kernel.org/e8e1a409-48d8-4fa7-ae98-49485a1607f6@linux.dev Link: https://lore.kernel.org/20260630112617.1198623-1-usama.arif@linux.dev Link: https://lore.kernel.org/20260630112617.1198623-2-usama.arif@linux.dev Signed-off-by: Usama Arif Acked-by: Shakeel Butt Acked-by: Johannes Weiner Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Michal Koutný Cc: Mike Rapoport Cc: Roman Gushchin Cc: Suren Baghdasaryan Cc: Tejun Heo Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/vmpressure.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mm/vmpressure.c b/mm/vmpressure.c index f053554e5826..93cc472bf2b2 100644 --- a/mm/vmpressure.c +++ b/mm/vmpressure.c @@ -246,11 +246,12 @@ void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree, return; /* - * The in-kernel users only care about the reclaim efficiency - * for this @memcg rather than the whole subtree, and there - * isn't and won't be any in-kernel user in a legacy cgroup. + * Only two combinations have a consumer: + * cgroup v2 + tree=false -> in-kernel socket pressure + * cgroup v1 + tree=true -> userspace eventfds (memory.pressure_level) + * Skip the other two: nothing consumes the result. */ - if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !tree) + if (cgroup_subsys_on_dfl(memory_cgrp_subsys) == tree) return; vmpr = memcg_to_vmpressure(memcg); From ea928e9e18da682e9a5bc40aa862bff7ce5ae42e Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Tue, 30 Jun 2026 04:23:33 -0700 Subject: [PATCH 138/501] mm/vmpressure: move v1 userspace eventfd code into memcontrol-v1.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clean up mm/vmpressure.c by separating the cgroup v1 userspace eventfd interface from the shared and v2 in-kernel code. Currently, almost half of mm/vmpressure.c exists to serve tree=true: struct vmpressure_event, the events list and its mutex, the work_struct and vmpressure_work_fn that drains tree_scanned/tree_reclaimed, the parent walk, vmpressure_event(), vmpressure_register_event(), vmpressure_unregister_event(), and vmpressure_prio() (which always calls vmpressure() with tree=true). Move it all into mm/memcontrol-v1.c (built only when CONFIG_MEMCG_V1=y) as a single contiguous block, following the per-component layout already used by that file. Keeping the v1 vmpressure code with the rest of the deprecated cgroup v1 memory controller makes the full footprint of the CONFIG_MEMCG_V1 option easy to see in one place, which matters more than component-level file separation for code that has no active development. vmpressure.c keeps the shared bits (constants, vmpressure_calc_level, the runtime hierarchy check, the tree=false body, init/cleanup plumbing) and calls into three small v1 hooks for the tree=true accumulator and the v1 portions of init/cleanup. The hooks have static-inline no-op stubs in include/linux/vmpressure.h for the !MEMCG_V1 case, so callers don't need ifdefs. vmpressure_prio() gets the same treatment, which means vmscan.c's call site disappears at compile time on v2-only kernels. The only #ifdef CONFIG_MEMCG_V1 in source remains around the v1-only fields inside struct vmpressure itself. Memory savings on CONFIG_MEMCG_V1=n (measured with pahole): struct vmpressure : 112B -> 24B struct mem_cgroup : 1664B -> 1536B This split is the first step toward eventually making vmpressure CONFIG_MEMCG_V1 only. The v2 in-kernel socket pressure path (tree=false) cannot be removed today immediately: PSI is not an exact replacement for vmpressure, and switching networking socket-buffer back-off to PSI may regress networking performance or increase memory pressure in workloads that today rely on vmpressure's hysteresis. The medium-term plan is to introduce a PSI-based socket-pressure path, keep vmpressure available for v2 behind a defconfig as an opt-out for several releases, and only then drop the tree=false path entirely, at which point everything that remains of the vmpressure block in mm/memcontrol-v1.c is the whole subsystem. Link: https://lore.kernel.org/20260630112617.1198623-3-usama.arif@linux.dev Signed-off-by: Usama Arif Acked-by: Shakeel Butt Cc: David Hildenbrand Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Michal Koutný Cc: Mike Rapoport Cc: Roman Gushchin Cc: Suren Baghdasaryan Cc: Tejun Heo Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/vmpressure.h | 48 +++++- mm/memcontrol-v1.c | 292 +++++++++++++++++++++++++++++++++++++ mm/vmpressure.c | 292 ++----------------------------------- 3 files changed, 344 insertions(+), 288 deletions(-) diff --git a/include/linux/vmpressure.h b/include/linux/vmpressure.h index faecd5522401..b4d13457bc2a 100644 --- a/include/linux/vmpressure.h +++ b/include/linux/vmpressure.h @@ -13,18 +13,31 @@ struct vmpressure { unsigned long scanned; unsigned long reclaimed; - - unsigned long tree_scanned; - unsigned long tree_reclaimed; - /* The lock is used to keep the scanned/reclaimed above in sync. */ + /* The lock is used to keep the scanned/reclaimed in sync. */ spinlock_t sr_lock; +#ifdef CONFIG_MEMCG_V1 + /* + * tree=true accumulators feed the v1 userspace eventfd interface + * (memory.pressure_level). Drained by @work. v2 has no equivalent + * interface, so this state is omitted on CONFIG_MEMCG_V1=n builds. + */ + unsigned long tree_scanned; + unsigned long tree_reclaimed; /* The list of vmpressure_event structs. */ struct list_head events; /* Have to grab the lock on events traversal or modifications. */ struct mutex events_lock; struct work_struct work; +#endif +}; + +enum vmpressure_levels { + VMPRESSURE_LOW = 0, + VMPRESSURE_MEDIUM, + VMPRESSURE_CRITICAL, + VMPRESSURE_NUM_LEVELS, }; struct mem_cgroup; @@ -32,18 +45,41 @@ struct mem_cgroup; #ifdef CONFIG_MEMCG void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree, unsigned long scanned, unsigned long reclaimed); -extern void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio); - extern void vmpressure_init(struct vmpressure *vmpr); extern void vmpressure_cleanup(struct vmpressure *vmpr); extern struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg); extern struct mem_cgroup *vmpressure_to_memcg(struct vmpressure *vmpr); + +/* Shared with the v1 vmpressure block in mm/memcontrol-v1.c. */ +extern const unsigned long vmpressure_win; +extern enum vmpressure_levels vmpressure_calc_level(unsigned long scanned, + unsigned long reclaimed); + +#ifdef CONFIG_MEMCG_V1 +extern void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio); extern int vmpressure_register_event(struct mem_cgroup *memcg, struct eventfd_ctx *eventfd, const char *args); extern void vmpressure_unregister_event(struct mem_cgroup *memcg, struct eventfd_ctx *eventfd); + +/* v1 hooks called from mm/vmpressure.c; no-ops below when !MEMCG_V1. */ +extern void vmpressure_v1_init(struct vmpressure *vmpr); +extern void vmpressure_v1_cleanup(struct vmpressure *vmpr); +extern void vmpressure_v1_account_tree(struct vmpressure *vmpr, + unsigned long scanned, + unsigned long reclaimed); #else +static inline void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, + int prio) {} +static inline void vmpressure_v1_init(struct vmpressure *vmpr) {} +static inline void vmpressure_v1_cleanup(struct vmpressure *vmpr) {} +static inline void vmpressure_v1_account_tree(struct vmpressure *vmpr, + unsigned long scanned, + unsigned long reclaimed) {} +#endif /* CONFIG_MEMCG_V1 */ + +#else /* !CONFIG_MEMCG */ static inline void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree, unsigned long scanned, unsigned long reclaimed) {} diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c index 765069211567..135622b6172b 100644 --- a/mm/memcontrol-v1.c +++ b/mm/memcontrol-v1.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -1476,6 +1477,297 @@ void memcg1_oom_finish(struct mem_cgroup *memcg, bool locked) mem_cgroup_oom_unlock(memcg); } +/* + * cgroup v1 userspace vmpressure interface (memory.pressure_level / + * cgroup.event_control). Kept here so v2-only kernels (CONFIG_MEMCG_V1=n) + * drop the whole eventfd accumulator, its work item, and the per-memcg + * state it requires. + * + * When there are too little pages left to scan, vmpressure() may miss the + * critical pressure as number of pages will be less than "window size". + * However, in that case the vmscan priority will raise fast as the + * reclaimer will try to scan LRUs more deeply. + * + * The vmscan logic considers these special priorities: + * + * prio == DEF_PRIORITY (12): reclaimer starts with that value + * prio <= DEF_PRIORITY - 2 : kswapd becomes somewhat overwhelmed + * prio == 0 : close to OOM, kernel scans every page in an lru + * + * Any value in this range is acceptable for this tunable (i.e. from 12 to + * 0). Current value for the vmpressure_level_critical_prio is chosen + * empirically, but the number, in essence, means that we consider + * critical level when scanning depth is ~10% of the lru size (vmscan + * scans 'lru_size >> prio' pages, so it is actually 12.5%, or one + * eights). + */ +static const unsigned int vmpressure_level_critical_prio = ilog2(100 / 10); + +enum vmpressure_modes { + VMPRESSURE_NO_PASSTHROUGH = 0, + VMPRESSURE_HIERARCHY, + VMPRESSURE_LOCAL, + VMPRESSURE_NUM_MODES, +}; + +static const char * const vmpressure_str_levels[] = { + [VMPRESSURE_LOW] = "low", + [VMPRESSURE_MEDIUM] = "medium", + [VMPRESSURE_CRITICAL] = "critical", +}; + +static const char * const vmpressure_str_modes[] = { + [VMPRESSURE_NO_PASSTHROUGH] = "default", + [VMPRESSURE_HIERARCHY] = "hierarchy", + [VMPRESSURE_LOCAL] = "local", +}; + +struct vmpressure_event { + struct eventfd_ctx *efd; + enum vmpressure_levels level; + enum vmpressure_modes mode; + struct list_head node; +}; + +static struct vmpressure *work_to_vmpressure(struct work_struct *work) +{ + return container_of(work, struct vmpressure, work); +} + +static struct vmpressure *vmpressure_parent(struct vmpressure *vmpr) +{ + struct mem_cgroup *memcg = vmpressure_to_memcg(vmpr); + + memcg = parent_mem_cgroup(memcg); + if (!memcg) + return NULL; + return memcg_to_vmpressure(memcg); +} + +static bool vmpressure_event(struct vmpressure *vmpr, + const enum vmpressure_levels level, + bool ancestor, bool signalled) +{ + struct vmpressure_event *ev; + bool ret = false; + + mutex_lock(&vmpr->events_lock); + list_for_each_entry(ev, &vmpr->events, node) { + if (ancestor && ev->mode == VMPRESSURE_LOCAL) + continue; + if (signalled && ev->mode == VMPRESSURE_NO_PASSTHROUGH) + continue; + if (level < ev->level) + continue; + eventfd_signal(ev->efd); + ret = true; + } + mutex_unlock(&vmpr->events_lock); + + return ret; +} + +static void vmpressure_work_fn(struct work_struct *work) +{ + struct vmpressure *vmpr = work_to_vmpressure(work); + unsigned long scanned; + unsigned long reclaimed; + enum vmpressure_levels level; + bool ancestor = false; + bool signalled = false; + + spin_lock(&vmpr->sr_lock); + /* + * Several contexts might be calling vmpressure(), so it is + * possible that the work was rescheduled again before the old + * work context cleared the counters. In that case we will run + * just after the old work returns, but then scanned might be zero + * here. No need for any locks here since we don't care if + * vmpr->reclaimed is in sync. + */ + scanned = vmpr->tree_scanned; + if (!scanned) { + spin_unlock(&vmpr->sr_lock); + return; + } + + reclaimed = vmpr->tree_reclaimed; + vmpr->tree_scanned = 0; + vmpr->tree_reclaimed = 0; + spin_unlock(&vmpr->sr_lock); + + level = vmpressure_calc_level(scanned, reclaimed); + + do { + if (vmpressure_event(vmpr, level, ancestor, signalled)) + signalled = true; + ancestor = true; + } while ((vmpr = vmpressure_parent(vmpr))); +} + +/* + * Tree-mode accumulator: accumulate per-memcg scanned/reclaimed and + * schedule the work that walks the parent chain and signals registered + * eventfd listeners once we cross the window threshold. + */ +void vmpressure_v1_account_tree(struct vmpressure *vmpr, + unsigned long scanned, + unsigned long reclaimed) +{ + spin_lock(&vmpr->sr_lock); + scanned = vmpr->tree_scanned += scanned; + vmpr->tree_reclaimed += reclaimed; + spin_unlock(&vmpr->sr_lock); + + if (scanned < vmpressure_win) + return; + schedule_work(&vmpr->work); +} + +void vmpressure_v1_init(struct vmpressure *vmpr) +{ + mutex_init(&vmpr->events_lock); + INIT_LIST_HEAD(&vmpr->events); + INIT_WORK(&vmpr->work, vmpressure_work_fn); +} + +void vmpressure_v1_cleanup(struct vmpressure *vmpr) +{ + /* + * Make sure there is no pending work before eventfd infrastructure + * goes away. + */ + flush_work(&vmpr->work); +} + +/** + * vmpressure_prio() - Account memory pressure through reclaimer priority level + * @gfp: reclaimer's gfp mask + * @memcg: cgroup memory controller handle + * @prio: reclaimer's priority + * + * This function should be called from the reclaim path every time when + * the vmscan's reclaiming priority (scanning depth) changes. + * + * This function does not return any value. + */ +void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio) +{ + /* + * We only use prio for accounting critical level. For more info + * see comment for vmpressure_level_critical_prio variable above. + */ + if (prio > vmpressure_level_critical_prio) + return; + + /* + * OK, the prio is below the threshold, updating vmpressure + * information before shrinker dives into long shrinking of long + * range vmscan. Passing scanned = vmpressure_win, reclaimed = 0 + * to the vmpressure() basically means that we signal 'critical' + * level. + */ + vmpressure(gfp, 0, memcg, true, vmpressure_win, 0); +} + +#define MAX_VMPRESSURE_ARGS_LEN (strlen("critical") + strlen("hierarchy") + 2) + +/** + * vmpressure_register_event() - Bind vmpressure notifications to an eventfd + * @memcg: memcg that is interested in vmpressure notifications + * @eventfd: eventfd context to link notifications with + * @args: event arguments (pressure level threshold, optional mode) + * + * This function associates eventfd context with the vmpressure + * infrastructure, so that the notifications will be delivered to the + * @eventfd. The @args parameter is a comma-delimited string that denotes a + * pressure level threshold (one of vmpressure_str_levels, i.e. "low", "medium", + * or "critical") and an optional mode (one of vmpressure_str_modes, i.e. + * "hierarchy" or "local"). + * + * To be used as memcg event method. + * + * Return: 0 on success, -ENOMEM on memory failure or -EINVAL if @args could + * not be parsed. + */ +int vmpressure_register_event(struct mem_cgroup *memcg, + struct eventfd_ctx *eventfd, const char *args) +{ + struct vmpressure *vmpr = memcg_to_vmpressure(memcg); + struct vmpressure_event *ev; + enum vmpressure_modes mode = VMPRESSURE_NO_PASSTHROUGH; + enum vmpressure_levels level; + char *spec, *spec_orig; + char *token; + int ret = 0; + + spec_orig = spec = kstrndup(args, MAX_VMPRESSURE_ARGS_LEN, GFP_KERNEL); + if (!spec) + return -ENOMEM; + + /* Find required level */ + token = strsep(&spec, ","); + ret = match_string(vmpressure_str_levels, VMPRESSURE_NUM_LEVELS, token); + if (ret < 0) + goto out; + level = ret; + + /* Find optional mode */ + token = strsep(&spec, ","); + if (token) { + ret = match_string(vmpressure_str_modes, VMPRESSURE_NUM_MODES, token); + if (ret < 0) + goto out; + mode = ret; + } + + ev = kzalloc_obj(*ev); + if (!ev) { + ret = -ENOMEM; + goto out; + } + + ev->efd = eventfd; + ev->level = level; + ev->mode = mode; + + mutex_lock(&vmpr->events_lock); + list_add(&ev->node, &vmpr->events); + mutex_unlock(&vmpr->events_lock); + ret = 0; +out: + kfree(spec_orig); + return ret; +} + +/** + * vmpressure_unregister_event() - Unbind eventfd from vmpressure + * @memcg: memcg handle + * @eventfd: eventfd context that was used to link vmpressure with the @cg + * + * This function does internal manipulations to detach the @eventfd from + * the vmpressure notifications, and then frees internal resources + * associated with the @eventfd (but the @eventfd itself is not freed). + * + * To be used as memcg event method. + */ +void vmpressure_unregister_event(struct mem_cgroup *memcg, + struct eventfd_ctx *eventfd) +{ + struct vmpressure *vmpr = memcg_to_vmpressure(memcg); + struct vmpressure_event *ev; + + mutex_lock(&vmpr->events_lock); + list_for_each_entry(ev, &vmpr->events, node) { + if (ev->efd != eventfd) + continue; + list_del(&ev->node); + kfree(ev); + break; + } + mutex_unlock(&vmpr->events_lock); +} + static DEFINE_MUTEX(memcg_max_mutex); static int mem_cgroup_resize_max(struct mem_cgroup *memcg, diff --git a/mm/vmpressure.c b/mm/vmpressure.c index 93cc472bf2b2..9629240d77ad 100644 --- a/mm/vmpressure.c +++ b/mm/vmpressure.c @@ -7,16 +7,15 @@ * * Based on ideas from Andrew Morton, David Rientjes, KOSAKI Motohiro, * Leonid Moiseichuk, Mel Gorman, Minchan Kim and Pekka Enberg. + * + * Tree-mode (cgroup v1 userspace eventfd) bookkeeping lives in + * mm/memcontrol-v1.c; this file holds the shared code and the in-kernel + * (tree=false) socket-pressure path that runs on cgroup v2. */ #include -#include #include -#include #include -#include -#include -#include #include #include #include @@ -35,7 +34,7 @@ * TODO: Make the window size depend on machine size, as we do for vmstat * thresholds. Currently we set it to 512 pages (2MB for 4KB pages). */ -static const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; +const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; /* * These thresholds are used when we account memory pressure through @@ -46,68 +45,6 @@ static const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; static const unsigned int vmpressure_level_med = 60; static const unsigned int vmpressure_level_critical = 95; -/* - * When there are too little pages left to scan, vmpressure() may miss the - * critical pressure as number of pages will be less than "window size". - * However, in that case the vmscan priority will raise fast as the - * reclaimer will try to scan LRUs more deeply. - * - * The vmscan logic considers these special priorities: - * - * prio == DEF_PRIORITY (12): reclaimer starts with that value - * prio <= DEF_PRIORITY - 2 : kswapd becomes somewhat overwhelmed - * prio == 0 : close to OOM, kernel scans every page in an lru - * - * Any value in this range is acceptable for this tunable (i.e. from 12 to - * 0). Current value for the vmpressure_level_critical_prio is chosen - * empirically, but the number, in essence, means that we consider - * critical level when scanning depth is ~10% of the lru size (vmscan - * scans 'lru_size >> prio' pages, so it is actually 12.5%, or one - * eights). - */ -static const unsigned int vmpressure_level_critical_prio = ilog2(100 / 10); - -static struct vmpressure *work_to_vmpressure(struct work_struct *work) -{ - return container_of(work, struct vmpressure, work); -} - -static struct vmpressure *vmpressure_parent(struct vmpressure *vmpr) -{ - struct mem_cgroup *memcg = vmpressure_to_memcg(vmpr); - - memcg = parent_mem_cgroup(memcg); - if (!memcg) - return NULL; - return memcg_to_vmpressure(memcg); -} - -enum vmpressure_levels { - VMPRESSURE_LOW = 0, - VMPRESSURE_MEDIUM, - VMPRESSURE_CRITICAL, - VMPRESSURE_NUM_LEVELS, -}; - -enum vmpressure_modes { - VMPRESSURE_NO_PASSTHROUGH = 0, - VMPRESSURE_HIERARCHY, - VMPRESSURE_LOCAL, - VMPRESSURE_NUM_MODES, -}; - -static const char * const vmpressure_str_levels[] = { - [VMPRESSURE_LOW] = "low", - [VMPRESSURE_MEDIUM] = "medium", - [VMPRESSURE_CRITICAL] = "critical", -}; - -static const char * const vmpressure_str_modes[] = { - [VMPRESSURE_NO_PASSTHROUGH] = "default", - [VMPRESSURE_HIERARCHY] = "hierarchy", - [VMPRESSURE_LOCAL] = "local", -}; - static enum vmpressure_levels vmpressure_level(unsigned long pressure) { if (pressure >= vmpressure_level_critical) @@ -117,8 +54,8 @@ static enum vmpressure_levels vmpressure_level(unsigned long pressure) return VMPRESSURE_LOW; } -static enum vmpressure_levels vmpressure_calc_level(unsigned long scanned, - unsigned long reclaimed) +enum vmpressure_levels vmpressure_calc_level(unsigned long scanned, + unsigned long reclaimed) { unsigned long scale = scanned + reclaimed; unsigned long pressure = 0; @@ -147,74 +84,6 @@ static enum vmpressure_levels vmpressure_calc_level(unsigned long scanned, return vmpressure_level(pressure); } -struct vmpressure_event { - struct eventfd_ctx *efd; - enum vmpressure_levels level; - enum vmpressure_modes mode; - struct list_head node; -}; - -static bool vmpressure_event(struct vmpressure *vmpr, - const enum vmpressure_levels level, - bool ancestor, bool signalled) -{ - struct vmpressure_event *ev; - bool ret = false; - - mutex_lock(&vmpr->events_lock); - list_for_each_entry(ev, &vmpr->events, node) { - if (ancestor && ev->mode == VMPRESSURE_LOCAL) - continue; - if (signalled && ev->mode == VMPRESSURE_NO_PASSTHROUGH) - continue; - if (level < ev->level) - continue; - eventfd_signal(ev->efd); - ret = true; - } - mutex_unlock(&vmpr->events_lock); - - return ret; -} - -static void vmpressure_work_fn(struct work_struct *work) -{ - struct vmpressure *vmpr = work_to_vmpressure(work); - unsigned long scanned; - unsigned long reclaimed; - enum vmpressure_levels level; - bool ancestor = false; - bool signalled = false; - - spin_lock(&vmpr->sr_lock); - /* - * Several contexts might be calling vmpressure(), so it is - * possible that the work was rescheduled again before the old - * work context cleared the counters. In that case we will run - * just after the old work returns, but then scanned might be zero - * here. No need for any locks here since we don't care if - * vmpr->reclaimed is in sync. - */ - scanned = vmpr->tree_scanned; - if (!scanned) { - spin_unlock(&vmpr->sr_lock); - return; - } - - reclaimed = vmpr->tree_reclaimed; - vmpr->tree_scanned = 0; - vmpr->tree_reclaimed = 0; - spin_unlock(&vmpr->sr_lock); - - level = vmpressure_calc_level(scanned, reclaimed); - - do { - if (vmpressure_event(vmpr, level, ancestor, signalled)) - signalled = true; - ancestor = true; - } while ((vmpr = vmpressure_parent(vmpr))); -} - /** * vmpressure() - Account memory pressure through scanned/reclaimed ratio * @gfp: reclaimer's gfp mask @@ -282,14 +151,7 @@ void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree, return; if (tree) { - spin_lock(&vmpr->sr_lock); - scanned = vmpr->tree_scanned += scanned; - vmpr->tree_reclaimed += reclaimed; - spin_unlock(&vmpr->sr_lock); - - if (scanned < vmpressure_win) - return; - schedule_work(&vmpr->work); + vmpressure_v1_account_tree(vmpr, scanned, reclaimed); } else { enum vmpressure_levels level; @@ -331,134 +193,6 @@ void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree, } } -/** - * vmpressure_prio() - Account memory pressure through reclaimer priority level - * @gfp: reclaimer's gfp mask - * @memcg: cgroup memory controller handle - * @prio: reclaimer's priority - * - * This function should be called from the reclaim path every time when - * the vmscan's reclaiming priority (scanning depth) changes. - * - * This function does not return any value. - */ -void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio) -{ - /* - * We only use prio for accounting critical level. For more info - * see comment for vmpressure_level_critical_prio variable above. - */ - if (prio > vmpressure_level_critical_prio) - return; - - /* - * OK, the prio is below the threshold, updating vmpressure - * information before shrinker dives into long shrinking of long - * range vmscan. Passing scanned = vmpressure_win, reclaimed = 0 - * to the vmpressure() basically means that we signal 'critical' - * level. - */ - vmpressure(gfp, 0, memcg, true, vmpressure_win, 0); -} - -#define MAX_VMPRESSURE_ARGS_LEN (strlen("critical") + strlen("hierarchy") + 2) - -/** - * vmpressure_register_event() - Bind vmpressure notifications to an eventfd - * @memcg: memcg that is interested in vmpressure notifications - * @eventfd: eventfd context to link notifications with - * @args: event arguments (pressure level threshold, optional mode) - * - * This function associates eventfd context with the vmpressure - * infrastructure, so that the notifications will be delivered to the - * @eventfd. The @args parameter is a comma-delimited string that denotes a - * pressure level threshold (one of vmpressure_str_levels, i.e. "low", "medium", - * or "critical") and an optional mode (one of vmpressure_str_modes, i.e. - * "hierarchy" or "local"). - * - * To be used as memcg event method. - * - * Return: 0 on success, -ENOMEM on memory failure or -EINVAL if @args could - * not be parsed. - */ -int vmpressure_register_event(struct mem_cgroup *memcg, - struct eventfd_ctx *eventfd, const char *args) -{ - struct vmpressure *vmpr = memcg_to_vmpressure(memcg); - struct vmpressure_event *ev; - enum vmpressure_modes mode = VMPRESSURE_NO_PASSTHROUGH; - enum vmpressure_levels level; - char *spec, *spec_orig; - char *token; - int ret = 0; - - spec_orig = spec = kstrndup(args, MAX_VMPRESSURE_ARGS_LEN, GFP_KERNEL); - if (!spec) - return -ENOMEM; - - /* Find required level */ - token = strsep(&spec, ","); - ret = match_string(vmpressure_str_levels, VMPRESSURE_NUM_LEVELS, token); - if (ret < 0) - goto out; - level = ret; - - /* Find optional mode */ - token = strsep(&spec, ","); - if (token) { - ret = match_string(vmpressure_str_modes, VMPRESSURE_NUM_MODES, token); - if (ret < 0) - goto out; - mode = ret; - } - - ev = kzalloc_obj(*ev); - if (!ev) { - ret = -ENOMEM; - goto out; - } - - ev->efd = eventfd; - ev->level = level; - ev->mode = mode; - - mutex_lock(&vmpr->events_lock); - list_add(&ev->node, &vmpr->events); - mutex_unlock(&vmpr->events_lock); - ret = 0; -out: - kfree(spec_orig); - return ret; -} - -/** - * vmpressure_unregister_event() - Unbind eventfd from vmpressure - * @memcg: memcg handle - * @eventfd: eventfd context that was used to link vmpressure with the @cg - * - * This function does internal manipulations to detach the @eventfd from - * the vmpressure notifications, and then frees internal resources - * associated with the @eventfd (but the @eventfd itself is not freed). - * - * To be used as memcg event method. - */ -void vmpressure_unregister_event(struct mem_cgroup *memcg, - struct eventfd_ctx *eventfd) -{ - struct vmpressure *vmpr = memcg_to_vmpressure(memcg); - struct vmpressure_event *ev; - - mutex_lock(&vmpr->events_lock); - list_for_each_entry(ev, &vmpr->events, node) { - if (ev->efd != eventfd) - continue; - list_del(&ev->node); - kfree(ev); - break; - } - mutex_unlock(&vmpr->events_lock); -} - /** * vmpressure_init() - Initialize vmpressure control structure * @vmpr: Structure to be initialized @@ -469,9 +203,7 @@ void vmpressure_unregister_event(struct mem_cgroup *memcg, void vmpressure_init(struct vmpressure *vmpr) { spin_lock_init(&vmpr->sr_lock); - mutex_init(&vmpr->events_lock); - INIT_LIST_HEAD(&vmpr->events); - INIT_WORK(&vmpr->work, vmpressure_work_fn); + vmpressure_v1_init(vmpr); } /** @@ -483,9 +215,5 @@ void vmpressure_init(struct vmpressure *vmpr) */ void vmpressure_cleanup(struct vmpressure *vmpr) { - /* - * Make sure there is no pending work before eventfd infrastructure - * goes away. - */ - flush_work(&vmpr->work); + vmpressure_v1_cleanup(vmpr); } From 5af3f83dcf2ea5e29278a9d18cb6b346e54e0622 Mon Sep 17 00:00:00 2001 From: yahia ahmed Date: Tue, 30 Jun 2026 15:02:22 +0300 Subject: [PATCH 139/501] mm/shmem: fix data-race in shmem_fault shmem_fault and shmem_writeout access inode->i_private without holding a lock, while shmem_fallocate is modifying it while holding a lock, thus a data-race is created. Fix this by using READ_ONCE and WRITE_ONCE, which provides an atomic, lockless read and write of inode->i_private which prevents compiler optimizations such as caching in registers and add writing to inode->i_private with WRITE_ONCE to prevent the compiler from writing in registers. Link: https://lore.kernel.org/20260630120222.11562-1-yahia.a.abdrabou@gmail.com Signed-off-by: yahia ahmed Reported-by: syzbot+76cc716982cf0254f302@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=76cc716982cf0254f302 Reviewed-by: Andrew Morton Cc: Baolin Wang Cc: Hugh Dickins Signed-off-by: Andrew Morton --- mm/shmem.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mm/shmem.c b/mm/shmem.c index 40bcaac105f4..fa8bb59f87b4 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1667,7 +1667,7 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug, * reactivate the folio, and let shmem_fallocate() quit when too many. */ if (!folio_test_uptodate(folio)) { - if (inode->i_private) { + if (READ_ONCE(inode->i_private)) { struct shmem_falloc *shmem_falloc; spin_lock(&inode->i_lock); shmem_falloc = inode->i_private; @@ -2703,7 +2703,7 @@ static vm_fault_t shmem_fault(struct vm_fault *vmf) * Trinity finds that probing a hole which tmpfs is punching can * prevent the hole-punch from ever completing: noted in i_private. */ - if (unlikely(inode->i_private)) { + if (unlikely(READ_ONCE(inode->i_private))) { ret = shmem_falloc_wait(vmf, inode); if (ret) return ret; @@ -3640,7 +3640,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT; shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT; spin_lock(&inode->i_lock); - inode->i_private = &shmem_falloc; + WRITE_ONCE(inode->i_private, &shmem_falloc); spin_unlock(&inode->i_lock); if ((u64)unmap_end > (u64)unmap_start) @@ -3650,7 +3650,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, /* No need to unmap again: hole-punching leaves COWed pages */ spin_lock(&inode->i_lock); - inode->i_private = NULL; + WRITE_ONCE(inode->i_private, NULL); wake_up_all(&shmem_falloc_waitq); WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head)); spin_unlock(&inode->i_lock); @@ -3682,7 +3682,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, shmem_falloc.nr_falloced = 0; shmem_falloc.nr_unswapped = 0; spin_lock(&inode->i_lock); - inode->i_private = &shmem_falloc; + WRITE_ONCE(inode->i_private, &shmem_falloc); spin_unlock(&inode->i_lock); /* @@ -3757,7 +3757,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, i_size_write(inode, offset + len); undone: spin_lock(&inode->i_lock); - inode->i_private = NULL; + WRITE_ONCE(inode->i_private, NULL); spin_unlock(&inode->i_lock); out: if (!error) From e453129b4c6de2ef6a8626e2c2d15abbe7022f2d Mon Sep 17 00:00:00 2001 From: Hongfu Li Date: Mon, 6 Jul 2026 16:15:55 +0800 Subject: [PATCH 140/501] selftests/mm: move pkey selftest helpers to pkey_util.c Patch series "selftests/mm: refactor pkey helpers and fix mmap error handling", v10. The main changes in this series are to refactor shared tracing and assertion helpers into a common file, unify both pkey selftests on pkey_assert() and per-test tracing for consistent diagnostics, and add missing mmap() return checks with MAP_FAILED used throughout for readability and consistency. This patch (of 10): Move pkey selftest debugging helpers into shared code so both pkey selftests can use the same tracing and abort-hook logic. Also fix cat_into_file() to print file, not str, in the open() failure message. Link: https://lore.kernel.org/20260706081600.3570203-1-lihongfu@kylinos.cn Link: https://lore.kernel.org/20260706081600.3570203-2-lihongfu@kylinos.cn Signed-off-by: Hongfu Li Acked-by: Mike Rapoport (Microsoft) Acked-by: Liam R. Howlett (Oracle) Reviewed-by: Kevin Brodsky Tested-by: Kevin Brodsky Cc: David Hildenbrand Cc: Joey Gouly Cc: John Hubbard Cc: Keith Lucas Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muhammad Usama Anjum Cc: Ross Zwisler Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Yury Khrustalev Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/pkey-helpers.h | 4 +- tools/testing/selftests/mm/pkey_util.c | 90 ++++++++++++++++++++ tools/testing/selftests/mm/protection_keys.c | 87 ------------------- 3 files changed, 93 insertions(+), 88 deletions(-) diff --git a/tools/testing/selftests/mm/pkey-helpers.h b/tools/testing/selftests/mm/pkey-helpers.h index 2c377f4e9df1..46a8a1878dc1 100644 --- a/tools/testing/selftests/mm/pkey-helpers.h +++ b/tools/testing/selftests/mm/pkey-helpers.h @@ -68,7 +68,9 @@ static inline void sigsafe_printf(const char *format, ...) #define dprintf3(args...) dprintf_level(3, args) #define dprintf4(args...) dprintf_level(4, args) -extern void abort_hooks(void); +void tracing_on(void); +void tracing_off(void); +void abort_hooks(void); #define pkey_assert(condition) do { \ if (!(condition)) { \ dprintf0("# assert() at %s::%d test_nr: %d iteration: %d\n", \ diff --git a/tools/testing/selftests/mm/pkey_util.c b/tools/testing/selftests/mm/pkey_util.c index 255b332f7a08..fbef3cd45447 100644 --- a/tools/testing/selftests/mm/pkey_util.c +++ b/tools/testing/selftests/mm/pkey_util.c @@ -2,9 +2,99 @@ #define __SANE_USERSPACE_TYPES__ #include #include +#include +#include +#include #include "pkey-helpers.h" +int iteration_nr = 1; +int test_nr; +int dprint_in_signal; + +#if CONTROL_TRACING > 0 +static void cat_into_file(char *str, char *file) +{ + int fd = open(file, O_RDWR); + int ret; + + dprintf2("%s(): writing '%s' to '%s'\n", __func__, str, file); + /* + * these need to be raw because they are called under + * pkey_assert() + */ + if (fd < 0) { + fprintf(stderr, "error opening '%s'\n", file); + perror("error: "); + exit(__LINE__); + } + + ret = write(fd, str, strlen(str)); + if (ret != strlen(str)) { + perror("write to file failed"); + fprintf(stderr, "filename: '%s' str: '%s'\n", file, str); + exit(__LINE__); + } + close(fd); +} + +static int warned_tracing; +static int tracing_root_ok(void) +{ + if (geteuid() != 0) { + if (!warned_tracing) + fprintf(stderr, "WARNING: not run as root, " + "can not do tracing control\n"); + warned_tracing = 1; + return 0; + } + return 1; +} +#endif + +void tracing_on(void) +{ +#if CONTROL_TRACING > 0 +#define TRACEDIR "/sys/kernel/tracing" + char pidstr[32]; + + if (!tracing_root_ok()) + return; + + sprintf(pidstr, "%d", getpid()); + cat_into_file("0", TRACEDIR "/tracing_on"); + cat_into_file("\n", TRACEDIR "/trace"); + if (1) { + cat_into_file("function_graph", TRACEDIR "/current_tracer"); + cat_into_file("1", TRACEDIR "/options/funcgraph-proc"); + } else { + cat_into_file("nop", TRACEDIR "/current_tracer"); + } + cat_into_file(pidstr, TRACEDIR "/set_ftrace_pid"); + cat_into_file("1", TRACEDIR "/tracing_on"); + dprintf1("enabled tracing\n"); +#endif +} + +void tracing_off(void) +{ +#if CONTROL_TRACING > 0 + if (!tracing_root_ok()) + return; + cat_into_file("0", "/sys/kernel/tracing/tracing_on"); +#endif +} + +void abort_hooks(void) +{ + fflush(stdout); + fprintf(stderr, "running %s()...\n", __func__); + tracing_off(); +#ifdef SLEEP_ON_ABORT + sleep(SLEEP_ON_ABORT); +#endif +} + int sys_pkey_alloc(unsigned long flags, unsigned long init_val) { int ret = syscall(SYS_pkey_alloc, flags, init_val); diff --git a/tools/testing/selftests/mm/protection_keys.c b/tools/testing/selftests/mm/protection_keys.c index 9a6d954ee371..be4a0470cc61 100644 --- a/tools/testing/selftests/mm/protection_keys.c +++ b/tools/testing/selftests/mm/protection_keys.c @@ -49,11 +49,7 @@ #include "hugepage_settings.h" #include "pkey-helpers.h" -int iteration_nr = 1; -int test_nr; - u64 shadow_pkey_reg; -int dprint_in_signal; noinline int read_ptr(int *ptr) { @@ -62,89 +58,6 @@ noinline int read_ptr(int *ptr) return *ptr; } -#if CONTROL_TRACING > 0 -static void cat_into_file(char *str, char *file) -{ - int fd = open(file, O_RDWR); - int ret; - - dprintf2("%s(): writing '%s' to '%s'\n", __func__, str, file); - /* - * these need to be raw because they are called under - * pkey_assert() - */ - if (fd < 0) { - fprintf(stderr, "error opening '%s'\n", str); - perror("error: "); - exit(__LINE__); - } - - ret = write(fd, str, strlen(str)); - if (ret != strlen(str)) { - perror("write to file failed"); - fprintf(stderr, "filename: '%s' str: '%s'\n", file, str); - exit(__LINE__); - } - close(fd); -} - -static int warned_tracing; -static int tracing_root_ok(void) -{ - if (geteuid() != 0) { - if (!warned_tracing) - fprintf(stderr, "WARNING: not run as root, " - "can not do tracing control\n"); - warned_tracing = 1; - return 0; - } - return 1; -} -#endif - -static void tracing_on(void) -{ -#if CONTROL_TRACING > 0 -#define TRACEDIR "/sys/kernel/tracing" - char pidstr[32]; - - if (!tracing_root_ok()) - return; - - sprintf(pidstr, "%d", getpid()); - cat_into_file("0", TRACEDIR "/tracing_on"); - cat_into_file("\n", TRACEDIR "/trace"); - if (1) { - cat_into_file("function_graph", TRACEDIR "/current_tracer"); - cat_into_file("1", TRACEDIR "/options/funcgraph-proc"); - } else { - cat_into_file("nop", TRACEDIR "/current_tracer"); - } - cat_into_file(pidstr, TRACEDIR "/set_ftrace_pid"); - cat_into_file("1", TRACEDIR "/tracing_on"); - dprintf1("enabled tracing\n"); -#endif -} - -static void tracing_off(void) -{ -#if CONTROL_TRACING > 0 - if (!tracing_root_ok()) - return; - cat_into_file("0", "/sys/kernel/tracing/tracing_on"); -#endif -} - -void abort_hooks(void) -{ - fflush(stdout); - fprintf(stderr, "running %s()...\n", __func__); - tracing_off(); -#ifdef SLEEP_ON_ABORT - sleep(SLEEP_ON_ABORT); -#endif -} - /* * This attempts to have roughly a page of instructions followed by a few * instructions that do a write, and another page of instructions. That From 9ee05ad72b84222e3e832b479e950a520bf0cdf3 Mon Sep 17 00:00:00 2001 From: Hongfu Li Date: Mon, 6 Jul 2026 16:15:56 +0800 Subject: [PATCH 141/501] selftests/mm: unify pkey sighandler selftest assertions and tracing Add per-test tracing to the pkey signal-handler selftest and use pkey_assert() for error handling. Each test enables tracing at start and disables it at end; on failure, pkey_assert() calls abort_hooks() to turn tracing off so ftrace is not left enabled. Link: https://lore.kernel.org/20260706081600.3570203-3-lihongfu@kylinos.cn Signed-off-by: Hongfu Li Acked-by: Mike Rapoport (Microsoft) Acked-by: Liam R. Howlett (Oracle) Reviewed-by: Kevin Brodsky Tested-by: Kevin Brodsky Cc: David Hildenbrand Cc: Joey Gouly Cc: John Hubbard Cc: Keith Lucas Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muhammad Usama Anjum Cc: Ross Zwisler Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Yury Khrustalev Signed-off-by: Andrew Morton --- .../selftests/mm/pkey_sighandler_tests.c | 89 +++++++++---------- 1 file changed, 44 insertions(+), 45 deletions(-) diff --git a/tools/testing/selftests/mm/pkey_sighandler_tests.c b/tools/testing/selftests/mm/pkey_sighandler_tests.c index 302fef54049c..8b3400379423 100644 --- a/tools/testing/selftests/mm/pkey_sighandler_tests.c +++ b/tools/testing/selftests/mm/pkey_sighandler_tests.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -207,15 +206,14 @@ static void test_sigsegv_handler_with_pkey0_disabled(void) struct sigaction sa; pthread_attr_t attr; pthread_t thr; + int ret; sa.sa_flags = SA_SIGINFO; sa.sa_sigaction = sigsegv_handler; sigemptyset(&sa.sa_mask); - if (sigaction(SIGSEGV, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGSEGV, &sa, NULL); + pkey_assert(ret == 0); memset(&siginfo, 0, sizeof(siginfo)); @@ -247,15 +245,14 @@ static void test_sigsegv_handler_cannot_access_stack(void) struct sigaction sa; pthread_attr_t attr; pthread_t thr; + int ret; sa.sa_flags = SA_SIGINFO; sa.sa_sigaction = sigsegv_handler; sigemptyset(&sa.sa_mask); - if (sigaction(SIGSEGV, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGSEGV, &sa, NULL); + pkey_assert(ret == 0); memset(&siginfo, 0, sizeof(siginfo)); @@ -288,21 +285,20 @@ static void test_sigsegv_handler_with_different_pkey_for_stack(void) int parent_pid = 0; int child_pid = 0; u64 pkey_reg; + long ret; sa.sa_flags = SA_SIGINFO | SA_ONSTACK; sa.sa_sigaction = sigsegv_handler; sigemptyset(&sa.sa_mask); - if (sigaction(SIGSEGV, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGSEGV, &sa, NULL); + pkey_assert(ret == 0); stack = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - assert(stack != MAP_FAILED); + pkey_assert(stack != MAP_FAILED); /* Allow access to MPK 0 and MPK 1 */ pkey_reg = pkey_reg_restrictive_default(); @@ -323,13 +319,13 @@ static void test_sigsegv_handler_with_different_pkey_for_stack(void) memset(&siginfo, 0, sizeof(siginfo)); /* Use clone to avoid newer glibcs using rseq on new threads */ - long ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES | - CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | - CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | - CLONE_DETACHED, - stack + STACK_SIZE, - &parent_pid, - &child_pid); + ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES | + CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | + CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | + CLONE_DETACHED, + stack + STACK_SIZE, + &parent_pid, + &child_pid); if (ret < 0) { errno = -ret; @@ -358,6 +354,7 @@ static void test_pkru_preserved_after_sigusr1(void) { struct sigaction sa; u64 pkey_reg; + int ret; /* Allow access to MPK 0 and an arbitrary set of keys */ pkey_reg = pkey_reg_restrictive_default(); @@ -369,10 +366,8 @@ static void test_pkru_preserved_after_sigusr1(void) sa.sa_sigaction = sigusr1_handler; sigemptyset(&sa.sa_mask); - if (sigaction(SIGUSR1, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGUSR1, &sa, NULL); + pkey_assert(ret == 0); memset(&siginfo, 0, sizeof(siginfo)); @@ -444,6 +439,13 @@ static void test_pkru_sigreturn(void) int parent_pid = 0; int child_pid = 0; u64 pkey_reg; + long ret; + + /* + * SIGSEGV handler is reset to SIG_DFL below; turn tracing off first + * so a crash does not leave ftrace enabled. + */ + tracing_off(); sa.sa_handler = SIG_DFL; sa.sa_flags = 0; @@ -453,24 +455,20 @@ static void test_pkru_sigreturn(void) * For this testcase, we do not want to handle SIGSEGV. Reset handler * to default so that the application can crash if it receives SIGSEGV. */ - if (sigaction(SIGSEGV, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGSEGV, &sa, NULL); + pkey_assert(ret == 0); sa.sa_flags = SA_SIGINFO | SA_ONSTACK; sa.sa_sigaction = sigusr2_handler; sigemptyset(&sa.sa_mask); - if (sigaction(SIGUSR2, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGUSR2, &sa, NULL); + pkey_assert(ret == 0); stack = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - assert(stack != MAP_FAILED); + pkey_assert(stack != MAP_FAILED); /* * Allow access to MPK 0 and MPK 2. The child thread (to be created @@ -494,13 +492,13 @@ static void test_pkru_sigreturn(void) sigstack.ss_size = STACK_SIZE; /* Use clone to avoid newer glibcs using rseq on new threads */ - long ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES | - CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | - CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | - CLONE_DETACHED, - stack + STACK_SIZE, - &parent_pid, - &child_pid); + ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES | + CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | + CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | + CLONE_DETACHED, + stack + STACK_SIZE, + &parent_pid, + &child_pid); if (ret < 0) { errno = -ret; @@ -530,16 +528,17 @@ static void (*pkey_tests[])(void) = { int main(int argc, char *argv[]) { - int i; - ksft_print_header(); ksft_set_plan(ARRAY_SIZE(pkey_tests)); if (!is_pkeys_supported()) ksft_exit_skip("pkeys not supported\n"); - for (i = 0; i < ARRAY_SIZE(pkey_tests); i++) - (*pkey_tests[i])(); + for (test_nr = 0; test_nr < ARRAY_SIZE(pkey_tests); test_nr++) { + tracing_on(); + (*pkey_tests[test_nr])(); + tracing_off(); + } ksft_finished(); return 0; From 4b87c918add02f3f5eb311f23803475bf9d792de Mon Sep 17 00:00:00 2001 From: Hongfu Li Date: Mon, 6 Jul 2026 16:15:57 +0800 Subject: [PATCH 142/501] selftests/mm: use pkey_assert on clone_raw failure in pkey test Use pkey_assert(0) instead of perror("clone") when clone_raw() fails. The old path only printed an error and continued; the test now exits via pkey_assert() on failure so it does not hang or proceed with an invalid child. Link: https://lore.kernel.org/20260706081600.3570203-4-lihongfu@kylinos.cn Signed-off-by: Hongfu Li Acked-by: Mike Rapoport (Microsoft) Acked-by: Liam R. Howlett (Oracle) Reviewed-by: Kevin Brodsky Tested-by: Kevin Brodsky Cc: David Hildenbrand Cc: Joey Gouly Cc: John Hubbard Cc: Keith Lucas Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muhammad Usama Anjum Cc: Ross Zwisler Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Yury Khrustalev Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/pkey_sighandler_tests.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/mm/pkey_sighandler_tests.c b/tools/testing/selftests/mm/pkey_sighandler_tests.c index 8b3400379423..188513e94fc5 100644 --- a/tools/testing/selftests/mm/pkey_sighandler_tests.c +++ b/tools/testing/selftests/mm/pkey_sighandler_tests.c @@ -329,7 +329,7 @@ static void test_sigsegv_handler_with_different_pkey_for_stack(void) if (ret < 0) { errno = -ret; - perror("clone"); + pkey_assert(0); } else if (ret == 0) { thread_segv_maperr_ptr(&sigstack); syscall_raw(SYS_exit, 0, 0, 0, 0, 0, 0); @@ -502,7 +502,7 @@ static void test_pkru_sigreturn(void) if (ret < 0) { errno = -ret; - perror("clone"); + pkey_assert(0); } else if (ret == 0) { thread_sigusr2_self(&sigstack); syscall_raw(SYS_exit, 0, 0, 0, 0, 0, 0); From 654b88fe7dfe980b81c20b4bbd74ed788fdb9b37 Mon Sep 17 00:00:00 2001 From: Hongfu Li Date: Mon, 6 Jul 2026 16:15:58 +0800 Subject: [PATCH 143/501] selftests/mm: add missing mmap() return checks in pkey tests Add missing checks against mmap() return value, replace (void *)-1 with MAP_FAILED for better readability and consistency. Link: https://lore.kernel.org/20260706081600.3570203-5-lihongfu@kylinos.cn Signed-off-by: Hongfu Li Acked-by: Mike Rapoport (Microsoft) Acked-by: Liam R. Howlett (Oracle) Reviewed-by: Kevin Brodsky Tested-by: Kevin Brodsky Cc: David Hildenbrand Cc: Joey Gouly Cc: John Hubbard Cc: Keith Lucas Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muhammad Usama Anjum Cc: Ross Zwisler Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Yury Khrustalev Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/pkey-powerpc.h | 2 +- tools/testing/selftests/mm/pkey_sighandler_tests.c | 2 ++ tools/testing/selftests/mm/protection_keys.c | 12 +++++++----- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/mm/pkey-powerpc.h b/tools/testing/selftests/mm/pkey-powerpc.h index 17bf2d1b0192..2ce85580b404 100644 --- a/tools/testing/selftests/mm/pkey-powerpc.h +++ b/tools/testing/selftests/mm/pkey-powerpc.h @@ -126,7 +126,7 @@ static inline void *malloc_pkey_with_mprotect_subpage(long size, int prot, u16 p size, prot, pkey); pkey_assert(pkey < NR_PKEYS); ptr = mmap(NULL, size, prot, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); - pkey_assert(ptr != (void *)-1); + pkey_assert(ptr != MAP_FAILED); ret = syscall(__NR_subpage_prot, ptr, size, NULL); if (ret) { diff --git a/tools/testing/selftests/mm/pkey_sighandler_tests.c b/tools/testing/selftests/mm/pkey_sighandler_tests.c index 188513e94fc5..8f98e6d9349f 100644 --- a/tools/testing/selftests/mm/pkey_sighandler_tests.c +++ b/tools/testing/selftests/mm/pkey_sighandler_tests.c @@ -313,6 +313,7 @@ static void test_sigsegv_handler_with_different_pkey_for_stack(void) /* Set up alternate signal stack that will use the default MPK */ sigstack.ss_sp = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + pkey_assert(sigstack.ss_sp != MAP_FAILED); sigstack.ss_flags = 0; sigstack.ss_size = STACK_SIZE; @@ -488,6 +489,7 @@ static void test_pkru_sigreturn(void) /* Set up alternate signal stack that will use the default MPK */ sigstack.ss_sp = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + pkey_assert(sigstack.ss_sp != MAP_FAILED); sigstack.ss_flags = 0; sigstack.ss_size = STACK_SIZE; diff --git a/tools/testing/selftests/mm/protection_keys.c b/tools/testing/selftests/mm/protection_keys.c index be4a0470cc61..ae6e1530b354 100644 --- a/tools/testing/selftests/mm/protection_keys.c +++ b/tools/testing/selftests/mm/protection_keys.c @@ -582,7 +582,7 @@ static void *malloc_pkey_with_mprotect(long size, int prot, u16 pkey) size, prot, pkey); pkey_assert(pkey < NR_PKEYS); ptr = mmap(NULL, size, prot, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); - pkey_assert(ptr != (void *)-1); + pkey_assert(ptr != MAP_FAILED); ret = mprotect_pkey((void *)ptr, PAGE_SIZE, prot, pkey); pkey_assert(!ret); record_pkey_malloc(ptr, size, prot); @@ -605,7 +605,7 @@ static void *malloc_pkey_anon_huge(long size, int prot, u16 pkey) */ size = ALIGN_UP(size, HPAGE_SIZE * 2); ptr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); - pkey_assert(ptr != (void *)-1); + pkey_assert(ptr != MAP_FAILED); record_pkey_malloc(ptr, size, prot); mprotect_pkey(ptr, size, prot, pkey); @@ -663,7 +663,7 @@ static void *malloc_pkey_hugetlb(long size, int prot, u16 pkey) size = ALIGN_UP(size, HPAGE_SIZE * 2); pkey_assert(pkey < NR_PKEYS); ptr = mmap(NULL, size, PROT_NONE, flags, -1, 0); - pkey_assert(ptr != (void *)-1); + pkey_assert(ptr != MAP_FAILED); mprotect_pkey(ptr, size, prot, pkey); record_pkey_malloc(ptr, size, prot); @@ -692,7 +692,7 @@ static void *malloc_pkey(long size, int prot, u16 pkey) pkey_assert(malloc_type < nr_malloc_types); ret = pkey_malloc[malloc_type](size, prot, pkey); - pkey_assert(ret != (void *)-1); + pkey_assert(ret != MAP_FAILED); malloc_type++; if (malloc_type >= nr_malloc_types) @@ -1110,6 +1110,7 @@ static void arch_force_pkey_reg_init(void) * doing the XSAVE size enumeration dance. */ buf = mmap(NULL, 1*MB, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); + pkey_assert(buf != MAP_FAILED); /* These __builtins require compiling with -mxsave */ @@ -1676,7 +1677,8 @@ int main(void) ksft_print_msg("running PKEY tests for unsupported CPU/OS\n"); ptr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); - assert(ptr != (void *)-1); + if (ptr == MAP_FAILED) + ksft_exit_fail_perror("mmap"); test_mprotect_pkey_on_unsupported_cpu(ptr, 1); ksft_test_result_pass("pkey on unsupported CPU/OS\n"); ksft_finished(); From 5a8e06833919369de0476327efbbfae6ebf0be7e Mon Sep 17 00:00:00 2001 From: Hongfu Li Date: Mon, 6 Jul 2026 16:15:59 +0800 Subject: [PATCH 144/501] selftests/mm: add missing pthread_create() return checks in pkey tests Add missing pthread_create() return checks in pkey sighandler tests to avoid hanging in pthread_cond_wait() when thread creation fails. Link: https://lore.kernel.org/20260706081600.3570203-6-lihongfu@kylinos.cn Signed-off-by: Hongfu Li Acked-by: Mike Rapoport (Microsoft) Acked-by: Liam R. Howlett (Oracle) Reviewed-by: Kevin Brodsky Tested-by: Kevin Brodsky Cc: David Hildenbrand Cc: Joey Gouly Cc: John Hubbard Cc: Keith Lucas Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muhammad Usama Anjum Cc: Ross Zwisler Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Yury Khrustalev Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/pkey_sighandler_tests.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/mm/pkey_sighandler_tests.c b/tools/testing/selftests/mm/pkey_sighandler_tests.c index 8f98e6d9349f..cbc24d6cf770 100644 --- a/tools/testing/selftests/mm/pkey_sighandler_tests.c +++ b/tools/testing/selftests/mm/pkey_sighandler_tests.c @@ -220,7 +220,11 @@ static void test_sigsegv_handler_with_pkey0_disabled(void) pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - pthread_create(&thr, &attr, thread_segv_with_pkey0_disabled, NULL); + ret = pthread_create(&thr, &attr, thread_segv_with_pkey0_disabled, NULL); + if (ret) { + errno = ret; + pkey_assert(0); + } pthread_mutex_lock(&mutex); while (siginfo.si_signo == 0) @@ -259,7 +263,11 @@ static void test_sigsegv_handler_cannot_access_stack(void) pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - pthread_create(&thr, &attr, thread_segv_pkuerr_stack, NULL); + ret = pthread_create(&thr, &attr, thread_segv_pkuerr_stack, NULL); + if (ret) { + errno = ret; + pkey_assert(0); + } pthread_mutex_lock(&mutex); while (siginfo.si_signo == 0) From c5efd0bded49eccabc54833cd532eefa667b172f Mon Sep 17 00:00:00 2001 From: Hongfu Li Date: Mon, 6 Jul 2026 16:16:00 +0800 Subject: [PATCH 145/501] selftests/mm: fix clone cleartid race in pkey sighandler tests Passing a stack-local child_pid to clone() with CLONE_CHILD_CLEARTID is unsafe: the kernel clears that address when the child exits, which may happen after the test function has returned and the stack slot has been reused. Neither testcase uses the settid/cleartid pointers for synchronization. Drop CLONE_PARENT_SETTID and CLONE_CHILD_CLEARTID and pass NULL for the clone tid arguments. Wait for the clone child to exit via tkill in test_sigsegv_handler_with_different_pkey_for_stack(), matching test_pkru_sigreturn(), so the detached thread cannot overlap with the next testcase. Link: https://lore.kernel.org/20260706081600.3570203-7-lihongfu@kylinos.cn Signed-off-by: Hongfu Li Cc: David Hildenbrand Cc: Joey Gouly Cc: John Hubbard Cc: Keith Lucas Cc: Kevin Brodsky Cc: Liam R. Howlett (Oracle) Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport (Microsoft) Cc: Muhammad Usama Anjum Cc: Ross Zwisler Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Yury Khrustalev Signed-off-by: Andrew Morton --- .../selftests/mm/pkey_sighandler_tests.c | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/mm/pkey_sighandler_tests.c b/tools/testing/selftests/mm/pkey_sighandler_tests.c index cbc24d6cf770..c218d0510a2a 100644 --- a/tools/testing/selftests/mm/pkey_sighandler_tests.c +++ b/tools/testing/selftests/mm/pkey_sighandler_tests.c @@ -290,7 +290,6 @@ static void test_sigsegv_handler_with_different_pkey_for_stack(void) static stack_t sigstack; void *stack; int pkey; - int parent_pid = 0; int child_pid = 0; u64 pkey_reg; long ret; @@ -330,11 +329,10 @@ static void test_sigsegv_handler_with_different_pkey_for_stack(void) /* Use clone to avoid newer glibcs using rseq on new threads */ ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | - CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | CLONE_DETACHED, stack + STACK_SIZE, - &parent_pid, - &child_pid); + NULL, + NULL); if (ret < 0) { errno = -ret; @@ -344,11 +342,19 @@ static void test_sigsegv_handler_with_different_pkey_for_stack(void) syscall_raw(SYS_exit, 0, 0, 0, 0, 0, 0); } + child_pid = ret; + pthread_mutex_lock(&mutex); while (siginfo.si_signo == 0) pthread_cond_wait(&cond, &mutex); pthread_mutex_unlock(&mutex); + /* Wait for child to exit before returning */ + do { + sched_yield(); + ret = syscall_raw(SYS_tkill, child_pid, 0, 0, 0, 0, 0); + } while (ret != -ESRCH && ret != -EINVAL); + ksft_test_result(siginfo.si_signo == SIGSEGV && siginfo.si_code == SEGV_MAPERR && siginfo.si_addr == NULL, @@ -445,7 +451,6 @@ static void test_pkru_sigreturn(void) static stack_t sigstack; void *stack; int pkey; - int parent_pid = 0; int child_pid = 0; u64 pkey_reg; long ret; @@ -504,11 +509,10 @@ static void test_pkru_sigreturn(void) /* Use clone to avoid newer glibcs using rseq on new threads */ ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | - CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | CLONE_DETACHED, stack + STACK_SIZE, - &parent_pid, - &child_pid); + NULL, + NULL); if (ret < 0) { errno = -ret; From 67a06fdfbf6f590d3e47b0592559e8fdc2a6998a Mon Sep 17 00:00:00 2001 From: Hui Zhu Date: Tue, 30 Jun 2026 15:08:10 +0800 Subject: [PATCH 146/501] mm: fix ASSERT_EXCLUSIVE_BITS by passing memdesc_flags_t by pointer KCSAN reports a data race between page_to_nid()/folio_pgdat() reading page->flags and folio_trylock()/folio_lock() concurrently doing test_and_set_bit_lock(PG_locked, ...) on the same word, e.g.: BUG: KCSAN: data-race in __lruvec_stat_mod_folio / shmem_get_folio_gfp The race is benign: nid/zone bits are set once at page init and never overlap with PG_locked. However, ASSERT_EXCLUSIVE_BITS() inside memdesc_nid/zonenum() was checking a by-value copy of the flags word, not the live page->flags, so it failed to annotate the real access. Change memdesc_nid(), memdesc_zonenum(), memdesc_section(), and memdesc_is_zone_device() to take a const memdesc_flags_t * and update all callers to pass &page->flags / &folio->flags, so ASSERT_EXCLUSIVE_BITS() operates on the actual shared word. Guard the ASSERT_EXCLUSIVE_BITS() call in memdesc_zonenum() under ZONES_WIDTH != 0 to avoid a zero-mask check on configs where the zone field is absent. memdesc_section() needs no such guard, since SECTIONS_WIDTH is never 0 wherever SECTION_IN_PAGE_FLAGS is defined. Under CONFIG_NUMA=n, memdesc_nid() itself is stubbed to "return 0" instead of reading page->flags, since NODES_MASK is 0 and the check can never fire; page_to_nid()/folio_nid() now just call memdesc_nid() unconditionally and rely on that stub, instead of duplicating the CONFIG_NUMA split at each call site. [zhuhui@kylinos.cn: v11] Link: https://lore.kernel.org/20260708083308.747930-1-hui.zhu@linux.dev [ziy@nvidia.com: build fix] Link: https://lore.kernel.org/DJSGLP9VFLRI.355AAWR2VHIEL@nvidia.com Link: https://lore.kernel.org/20260630070810.470763-1-hui.zhu@linux.dev Signed-off-by: Hui Zhu Co-developed-by: David Hildenbrand (Arm) Signed-off-by: David Hildenbrand (Arm) Acked-by: David Hildenbrand (Arm) Cc: Axel Rasmussen Cc: Barry Song Cc: Kairui Song Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shakeel Butt Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie Signed-off-by: Andrew Morton --- include/asm-generic/memory_model.h | 2 +- include/linux/mm.h | 26 ++++++++++++++++++-------- include/linux/mm_inline.h | 4 ++-- include/linux/mmzone.h | 26 ++++++++++++++------------ mm/page_alloc.c | 6 +++--- mm/page_owner.c | 2 +- mm/slab.h | 2 +- mm/sparse.c | 2 +- 8 files changed, 41 insertions(+), 29 deletions(-) diff --git a/include/asm-generic/memory_model.h b/include/asm-generic/memory_model.h index fd74de50b054..c6b4eafaf4cb 100644 --- a/include/asm-generic/memory_model.h +++ b/include/asm-generic/memory_model.h @@ -53,7 +53,7 @@ static inline int pfn_valid(unsigned long pfn) */ #define __page_to_pfn(pg) \ ({ const struct page *__pg = (pg); \ - int __sec = memdesc_section(__pg->flags); \ + int __sec = memdesc_section(&__pg->flags); \ (unsigned long)(__pg - __section_mem_map_addr(__nr_to_section(__sec))); \ }) diff --git a/include/linux/mm.h b/include/linux/mm.h index 2101e5205fc0..3dca9497ae03 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -37,6 +37,7 @@ #include #include #include +#include struct mempolicy; struct anon_vma; @@ -2286,22 +2287,30 @@ static inline int page_zone_id(struct page *page) } #ifdef NODE_NOT_IN_PAGE_FLAGS -int memdesc_nid(memdesc_flags_t mdf); +int memdesc_nid(const memdesc_flags_t *mdf); #else -static inline int memdesc_nid(memdesc_flags_t mdf) +#ifdef CONFIG_NUMA +static inline int memdesc_nid(const memdesc_flags_t *mdf) { - return (mdf.f >> NODES_PGSHIFT) & NODES_MASK; + ASSERT_EXCLUSIVE_BITS(mdf->f, NODES_MASK << NODES_PGSHIFT); + return (mdf->f >> NODES_PGSHIFT) & NODES_MASK; } +#else +static inline int memdesc_nid(const memdesc_flags_t *mdf) +{ + return 0; +} +#endif #endif static inline int page_to_nid(const struct page *page) { - return memdesc_nid(PF_POISONED_CHECK(page)->flags); + return memdesc_nid(&(PF_POISONED_CHECK(page)->flags)); } static inline int folio_nid(const struct folio *folio) { - return memdesc_nid(folio->flags); + return memdesc_nid(&folio->flags); } #ifdef CONFIG_NUMA_BALANCING @@ -2541,12 +2550,13 @@ static inline void set_page_section(struct page *page, unsigned long section) page->flags.f |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT; } -static inline unsigned long memdesc_section(memdesc_flags_t mdf) +static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) { - return (mdf.f >> SECTIONS_PGSHIFT) & SECTIONS_MASK; + ASSERT_EXCLUSIVE_BITS(mdf->f, SECTIONS_MASK << SECTIONS_PGSHIFT); + return (mdf->f >> SECTIONS_PGSHIFT) & SECTIONS_MASK; } #else /* !SECTION_IN_PAGE_FLAGS */ -static inline unsigned long memdesc_section(memdesc_flags_t mdf) +static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) { return 0; } diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index a8430a7ae054..efcddb9925ad 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -650,7 +650,7 @@ static inline bool vma_has_recency(const struct vm_area_struct *vma) static inline size_t num_pages_contiguous(struct page **pages, size_t nr_pages) { struct page *cur_page = pages[0]; - unsigned long section = memdesc_section(cur_page->flags); + unsigned long section = memdesc_section(&cur_page->flags); size_t i; for (i = 1; i < nr_pages; i++) { @@ -660,7 +660,7 @@ static inline size_t num_pages_contiguous(struct page **pages, size_t nr_pages) * In unproblematic kernel configs, page_to_section() == 0 and * the whole check will get optimized out. */ - if (memdesc_section(cur_page->flags) != section) + if (memdesc_section(&cur_page->flags) != section) break; } diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 242ec3cb8b52..0507193b3ae3 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -1272,31 +1272,33 @@ static inline bool zone_is_empty(const struct zone *zone) #define KASAN_TAG_MASK ((1UL << KASAN_TAG_WIDTH) - 1) #define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1) -static inline enum zone_type memdesc_zonenum(memdesc_flags_t flags) +static inline enum zone_type memdesc_zonenum(const memdesc_flags_t *flags) { - ASSERT_EXCLUSIVE_BITS(flags.f, ZONES_MASK << ZONES_PGSHIFT); - return (flags.f >> ZONES_PGSHIFT) & ZONES_MASK; +#if ZONES_WIDTH != 0 + ASSERT_EXCLUSIVE_BITS(flags->f, ZONES_MASK << ZONES_PGSHIFT); +#endif + return (flags->f >> ZONES_PGSHIFT) & ZONES_MASK; } static inline enum zone_type page_zonenum(const struct page *page) { - return memdesc_zonenum(page->flags); + return memdesc_zonenum(&page->flags); } static inline enum zone_type folio_zonenum(const struct folio *folio) { - return memdesc_zonenum(folio->flags); + return memdesc_zonenum(&folio->flags); } #ifdef CONFIG_ZONE_DEVICE -static inline bool memdesc_is_zone_device(memdesc_flags_t mdf) +static inline bool memdesc_is_zone_device(const memdesc_flags_t *mdf) { return memdesc_zonenum(mdf) == ZONE_DEVICE; } static inline struct dev_pagemap *page_pgmap(const struct page *page) { - VM_WARN_ON_ONCE_PAGE(!memdesc_is_zone_device(page->flags), page); + VM_WARN_ON_ONCE_PAGE(!memdesc_is_zone_device(&page->flags), page); return page_folio(page)->pgmap; } @@ -1311,9 +1313,9 @@ static inline struct dev_pagemap *page_pgmap(const struct page *page) static inline bool zone_device_pages_have_same_pgmap(const struct page *a, const struct page *b) { - if (memdesc_is_zone_device(a->flags) != memdesc_is_zone_device(b->flags)) + if (memdesc_is_zone_device(&a->flags) != memdesc_is_zone_device(&b->flags)) return false; - if (!memdesc_is_zone_device(a->flags)) + if (!memdesc_is_zone_device(&a->flags)) return true; return page_pgmap(a) == page_pgmap(b); } @@ -1321,7 +1323,7 @@ static inline bool zone_device_pages_have_same_pgmap(const struct page *a, extern void memmap_init_zone_device(struct zone *, unsigned long, unsigned long, struct dev_pagemap *); #else -static inline bool memdesc_is_zone_device(memdesc_flags_t mdf) +static inline bool memdesc_is_zone_device(const memdesc_flags_t *mdf) { return false; } @@ -1338,12 +1340,12 @@ static inline struct dev_pagemap *page_pgmap(const struct page *page) static inline bool is_zone_device_page(const struct page *page) { - return memdesc_is_zone_device(page->flags); + return memdesc_is_zone_device(&page->flags); } static inline bool folio_is_zone_device(const struct folio *folio) { - return memdesc_is_zone_device(folio->flags); + return memdesc_is_zone_device(&folio->flags); } static inline bool is_zone_movable_page(const struct page *page) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 62f71ece7ca1..762d9b6bc792 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -6913,15 +6913,15 @@ static void __free_contig_range_common(unsigned long pfn, unsigned long nr_pages continue; } - if (start && memdesc_section(page->flags) != start_sec) { + if (start && memdesc_section(&page->flags) != start_sec) { free_prepared_contig_range(start, i - nr_start); start = page; nr_start = i; - start_sec = memdesc_section(page->flags); + start_sec = memdesc_section(&page->flags); } else if (!start) { start = page; nr_start = i; - start_sec = memdesc_section(page->flags); + start_sec = memdesc_section(&page->flags); } } diff --git a/mm/page_owner.c b/mm/page_owner.c index 7e235ee10930..5cad1dfe057a 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -815,7 +815,7 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) */ if (page_flags.f == PAGE_POISON_PATTERN) goto ext_put_continue; - nid = memdesc_nid(page_flags); + nid = memdesc_nid(&page_flags); if (!node_isset(nid, state->nid_filter)) goto ext_put_continue; } diff --git a/mm/slab.h b/mm/slab.h index f5e336b6b6b0..c24c3daaa869 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -180,7 +180,7 @@ static inline void *slab_address(const struct slab *slab) static inline int slab_nid(const struct slab *slab) { - return memdesc_nid(slab->flags); + return memdesc_nid(&slab->flags); } static inline pg_data_t *slab_pgdat(const struct slab *slab) diff --git a/mm/sparse.c b/mm/sparse.c index 324213d8bdcb..058ef9300367 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -43,7 +43,7 @@ static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; #endif -int memdesc_nid(memdesc_flags_t mdf) +int memdesc_nid(const memdesc_flags_t *mdf) { return section_to_node_table[memdesc_section(mdf)]; } From 78b0ca26d00dfb585de60bdc79325d0c75c59b28 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Wed, 1 Jul 2026 07:06:38 -0700 Subject: [PATCH 147/501] mm/migrate_device: pin large folios before splitting migrate_vma_collect_pmd() can detect a large folio while holding the PTE lock, then drop the PTE lock before calling migrate_vma_split_folio(). The split helper took its own reference, but only after the lock had already been dropped. One way to hit this is device migration over a range that contains a large folio. The walker reads the PTE while holding the PTE lock and derives the folio either from a present PTE via vm_normal_page(), or from a non-present PTE that encodes a device-private softleaf entry. It then has to drop the PTE lock because split_folio() can block. Before migrate_vma_split_folio() gets a folio reference, concurrent reclaim, migration, or truncation can replace or clear the entry and drop the last reference to the folio. The split helper would then take a reference and lock on a stale folio pointer. Take a temporary reference before dropping the PTE lock and pass that reference into migrate_vma_split_folio(). The helper consumes the reference, so split_folio() still sees only the expected caller pin instead of an extra pin that could make the split fail. [usama.arif@linux.dev: condense comment about folio reference] Link: https://lore.kernel.org/87bbf335-648f-4065-abc8-3eaab5a3beeb@linux.dev Link: https://lore.kernel.org/20260701140638.840773-1-usama.arif@linux.dev Fixes: 022a12deda53 ("mm/migrate_device: handle partially mapped folios during collection") Signed-off-by: Usama Arif Reported-by: sashiko-bot Link: https://sashiko.dev/#/patchset/20260630164143.1595669-1-usama.arif%40linux.dev Acked-by: David Hildenbrand (Arm) Reviewed-by: Zi Yan Reviewed-by: Lance Yang Reviewed-by: SJ Park Cc: Alistair Popple Cc: Byungchul Park Cc: Gregory Price Cc: "Huang, Ying" Cc: Johannes Weiner Cc: Joshua Hahn Cc: Matthew Brost Cc: Rakie Kim Cc: Shakeel Butt Signed-off-by: Andrew Morton --- mm/migrate_device.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/mm/migrate_device.c b/mm/migrate_device.c index e74fee90489a..2fffeb1f9969 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -77,6 +77,9 @@ static int migrate_vma_collect_hole(unsigned long start, * @folio: the folio to split * @fault_page: struct page associated with the fault if any * + * If @folio is not the folio containing @fault_page, the caller must hold a + * reference on @folio. The helper consumes that reference. + * * Returns 0 on success */ static int migrate_vma_split_folio(struct folio *folio, @@ -86,10 +89,8 @@ static int migrate_vma_split_folio(struct folio *folio, struct folio *fault_folio = fault_page ? page_folio(fault_page) : NULL; struct folio *new_fault_folio = NULL; - if (folio != fault_folio) { - folio_get(folio); + if (folio != fault_folio) folio_lock(folio); - } ret = split_folio(folio); if (ret) { @@ -310,6 +311,9 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp, if (folio_test_large(folio)) { int ret; + /* migrate_vma_split_folio() consumes this reference */ + if (folio != fault_folio) + folio_get(folio); lazy_mmu_mode_disable(); pte_unmap_unlock(ptep, ptl); ret = migrate_vma_split_folio(folio, @@ -353,6 +357,9 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp, if (folio && folio_test_large(folio)) { int ret; + /* migrate_vma_split_folio() consumes this reference */ + if (folio != fault_folio) + folio_get(folio); lazy_mmu_mode_disable(); pte_unmap_unlock(ptep, ptl); ret = migrate_vma_split_folio(folio, From 9eed9653a35fa2ae9191d0025d279ef7694a4cdf Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Wed, 1 Jul 2026 18:16:13 -0400 Subject: [PATCH 148/501] mm/mm_init: handle alloc_percpu failure in free_area_init_core_hotplug We miss a failed allocation check for pgdat->per_cpu_nodestats, which results in a NULL deref when we offset into the per-cpu area. Propagate -ENOMEM up the stack and leave per_cpu_nodestats pointing at boot_nodestats so a later online can retry the allocation. hotadd_init_pgdat() returns NULL on failure, which __try_online_node() already maps to -ENOMEM. On failure nothing needs to be unwound: - the node is never marked online - per_cpu_nodestats is left pointing at boot_nodestats - __add_memory_resource() cleans up pending memblock resources - later online attempts retry the per_cpu_nodestats allocation Link: https://lore.kernel.org/20260701221613.2818148-1-gourry@gourry.net Fixes: 75ef71840539 ("mm, vmstat: add infrastructure for per-node vmstats") Signed-off-by: Gregory Price Reported-by: Sashiko Link: https://sashiko.dev/#/patchset/20260627202243.758289-1-gourry%40gourry.net Acked-by: David Hildenbrand (Arm) Cc: Johannes Weiner Cc: Mel Gorman Cc: Mike Rapoport Cc: Oscar Salvador Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/memory_hotplug.h | 2 +- mm/memory_hotplug.c | 3 ++- mm/mm_init.c | 14 +++++++++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 7c9d66729c60..06c58cb05779 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -289,7 +289,7 @@ static inline void __remove_memory(u64 start, u64 size) {} /* Default online_type (MMOP_*) when new memory blocks are added. */ extern enum mmop mhp_get_default_online_type(void); extern void mhp_set_default_online_type(enum mmop online_type); -extern void __ref free_area_init_core_hotplug(struct pglist_data *pgdat); +int __ref free_area_init_core_hotplug(struct pglist_data *pgdat); extern int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory_resource(int nid, struct resource *resource, diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 7ac19fab2263..8b137328dcf0 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1263,7 +1263,8 @@ static pg_data_t *hotadd_init_pgdat(int nid) pgdat = NODE_DATA(nid); /* init node's zones as empty zones, we don't have any present pages.*/ - free_area_init_core_hotplug(pgdat); + if (free_area_init_core_hotplug(pgdat)) + return NULL; /* * The node we allocated has no zone fallback lists. For avoiding diff --git a/mm/mm_init.c b/mm/mm_init.c index cfd0b2722d83..07a8c74cf7ad 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1526,7 +1526,7 @@ static inline void __init set_pageblock_order(void) * NOTE: this function is only called during memory hotplug */ #ifdef CONFIG_MEMORY_HOTPLUG -void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) +int __ref free_area_init_core_hotplug(struct pglist_data *pgdat) { int nid = pgdat->node_id; enum zone_type z; @@ -1534,8 +1534,14 @@ void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) pgdat_init_internals(pgdat); - if (pgdat->per_cpu_nodestats == &boot_nodestats) - pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat); + if (pgdat->per_cpu_nodestats == &boot_nodestats) { + struct per_cpu_nodestat __percpu *p; + + p = alloc_percpu(struct per_cpu_nodestat); + if (!p) + return -ENOMEM; + pgdat->per_cpu_nodestats = p; + } /* * Reset the nr_zones, order and highest_zoneidx before reuse. @@ -1573,6 +1579,8 @@ void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) zone->present_pages = 0; zone_init_internals(zone, z, nid, 0); } + + return 0; } #endif From 6a8024511ddf4877435c34fb3d6028aa8e590649 Mon Sep 17 00:00:00 2001 From: Stanislav Kinsburskii Date: Wed, 1 Jul 2026 15:02:20 -0700 Subject: [PATCH 149/501] lib/test_hmm: fail dmirror_fault() when the mirrored mm is gone dmirror_fault() is called from the dmirror_read() and dmirror_write() retry loops after dmirror_do_read() or dmirror_do_write() finds a missing device page table entry. If the mirrored mm has already exited, mmget_not_zero() fails. The current code returns 0 in that case, which tells the caller that faulting succeeded even though no page was faulted and no device page table entry was installed. The caller then retries the same address, hits -ENOENT again, and can loop forever without making progress. Return -EFAULT instead, so the ioctl fails when the mirrored mm is no longer faultable. Link: https://lore.kernel.org/178294308408.327222.3319445682023999403.stgit@skinsburskii Fixes: b2ef9f5a5cb37 ("mm/hmm/test: add selftest driver for HMM") Signed-off-by: Stanislav Kinsburskii Cc: Jason Gunthorpe Cc: Leon Romanovsky Cc: Ralph Campbell Signed-off-by: Andrew Morton --- lib/test_hmm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/test_hmm.c b/lib/test_hmm.c index c4adbf98fac7..45c0cb992218 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -407,7 +407,7 @@ static int dmirror_fault(struct dmirror *dmirror, unsigned long start, /* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) - return 0; + return -EFAULT; for (addr = start; addr < end; addr = range.end) { range.start = addr; From 5491b25d4ae99a59cada18aa03e0ac838b5fd725 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Wed, 1 Jul 2026 23:09:32 +0300 Subject: [PATCH 150/501] selftests/mm/uffd: don't treat UFFDIO_COPY -ENOENT as a failure Non-cooperarive uffd events are inherently racy and can happen in parallel with other userfaultfd operations. During event tests in uffd-unit-tests, the uffd monitor calls UFFDIO_UNREGISTER upon receiving UFFD_EVENT_REMOVE. In parallel, the faulting_process() verifies that the removed memory is actually zeroed. If a verification read wins the race with UFFDIO_UNREGISTER, it causes a missing fault that uffd monitor would receive after UFFDIO_UNREGISTER is complete. The monitor resolves the fault using UFFDIO_COPY that fails with -ENOENT which means that VMA has been changed (see commit 27d02568f529 ("userfaultfd: mcopy_atomic: return -ENOENT when no compatible VMA found")). Treat -ENOENT returned by UFFDIO_COPY as non-fatal, the same way -EEXIST is treated for concurrent faults, and don't fail the test. Link: https://lore.kernel.org/20260701200932.1470525-1-rppt@kernel.org Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: Lorenzo Stoakes Reviewed-by: David Hildenbrand (Arm) Cc: Liam R. Howlett Cc: Michal Hocko Cc: Peter Xu Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/uffd-common.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/mm/uffd-common.c b/tools/testing/selftests/mm/uffd-common.c index edd02328f77b..f48f5d4594ab 100644 --- a/tools/testing/selftests/mm/uffd-common.c +++ b/tools/testing/selftests/mm/uffd-common.c @@ -639,8 +639,13 @@ int __copy_page(uffd_global_test_opts_t *gopts, unsigned long offset, bool retry uffdio_copy.mode = 0; uffdio_copy.copy = 0; if (ioctl(gopts->uffd, UFFDIO_COPY, &uffdio_copy)) { - /* real retval in ufdio_copy.copy */ - if (uffdio_copy.copy != -EEXIST) + /* + * real retval in uffdio_copy.copy + * + * -EEXIST: the page was faulted in concurrently + * -ENOENT: the destination range was concurrently removed + */ + if (uffdio_copy.copy != -EEXIST && uffdio_copy.copy != -ENOENT) err("UFFDIO_COPY error: %"PRId64, (int64_t)uffdio_copy.copy); wake_range(gopts->uffd, uffdio_copy.dst, gopts->page_size); From f6c786a95e950931b47cf23cd3ced6028be71d62 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 1 Jul 2026 14:21:02 -0400 Subject: [PATCH 151/501] mm: gfp_types: fix __GFP_ACCOUNT, GFP_KERNEL_ACCOUNT documentation Gregory points out that these descriptions are cursed and confusing, considering what these flags actually do. This is mostly due to historic implementation choices and cgroup1 baggage. Improve the description of their actual effects. Link: https://lore.kernel.org/20260701182102.1586784-1-hannes@cmpxchg.org Signed-off-by: Johannes Weiner Reported-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Acked-by: Shakeel Butt Acked-by: Mike Rapoport (Microsoft) Reviewed-by: Gregory Price Acked-by: SJ Park Cc: David Hildenbrand Cc: Liam Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin Cc: Suren Baghdasaryan Signed-off-by: Andrew Morton --- include/linux/gfp_types.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h index 54ca0c88bab6..463b551d12d9 100644 --- a/include/linux/gfp_types.h +++ b/include/linux/gfp_types.h @@ -136,7 +136,8 @@ enum { * %__GFP_THISNODE forces the allocation to be satisfied from the requested * node with no fallbacks or placement policy enforcements. * - * %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg. + * %__GFP_ACCOUNT causes the allocation to be accounted to the active + * cgroup context. * * %__GFP_NO_OBJ_EXT causes slab allocation to have no object extension. * mark_obj_codetag_empty() should be called upon freeing for objects allocated @@ -320,7 +321,7 @@ enum { * %ZONE_NORMAL or a lower zone for direct access but can direct reclaim. * * %GFP_KERNEL_ACCOUNT is the same as GFP_KERNEL, except the allocation is - * accounted to kmemcg. + * accounted to the active cgroup context. * * %GFP_NOWAIT is for kernel allocations that should not stall for direct * reclaim, start physical IO or use any filesystem callback. It is very From 9f3786d8602fe6c78bb7e7b9db6b2be15ac4b7f3 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 29 Jun 2026 21:07:54 -0700 Subject: [PATCH 152/501] mm/damon/core: introduce damon_nr_accesses_mvsum() Patch series "mm/damon: optimize out nr_accesses_bp". TLDR: Replace damon_region->nr_accesses_bp, which is easy to be wrong, with a simpler on-demand moving sum function, damon_nr_accesses_mvsum(). Background ========== DAMON's monitoring output (access pattern snapshot, or more technically speaking, damon_region->nr_accesses) is completed once per aggregation interval, which is 100 ms by default. Users can arbitrarily increase the interval for demand. Under the suggested intervals auto-tuning setup, it can span up to 200 seconds. If the aggregation interval is too long, the snapshot users cannot use it in reasonable time. To mitigate this, we introduced a new field of damon_region, namely nr_accesses_bp. It contains a pseudo moving sum of nr_accesses in bp units and is updated for each sampling interval. It turned out keeping it correctly updated every sampling interval is not that easy. From online parameter update feature development and more experimental hacks, we found it is easy to be corrupted. Once it is corrupted, DAMON's monitoring outputs become quite insane. Hence we added a few validation checks. It is easy to be corrupted because it requires every update per sampling interval to be correct. Solution ======== There is no real reason to keep it updated every sampling interval. Due to the simple pseudo-moving sum mechanism and existing helper field (last_nr_accesses), we can also calculate the pseudo moving sum on demand in a much simpler way. Implement a function for getting the pseudo moving sum on demand, and replace nr_accessses_bp uses with the new function. Also remove no more needed tests for nr_accesses_bp and the per-sampling interval update functions. Finally, remove the nr_accesses_bp. The new function is quite simple. Discussion ========== Depending on the use case, multiple nr_accesses readers could be executed in the same kdamond_fn() main loop iteration, which is executed once per sampling interval. Such readers include DAMON region exporting tracepoints (damon_[region_]aggregated and damos_before_apply), DAMOS, and DAMON sysfs interface logic for update_schemes_tried_regions command. In this case, the new function will be called multiple times and this could be overhead compared to the old logic, which simply reads the field without any additional work. Nonetheless, the new function is quite simple. And the new approach does nothing while there is no need to read. The old approach had to execute its update function for each region for every sampling interval. Hence the new approach is believed to be even more lightweight in common case, and the overhead is anyway negligible. One more advantage of this change is that one field from the damon_region struct is removed. On setups that uses a high number of DAMON regions, this could be a potential memory space benefit. Patches Sequence ================ Patch 1 introduces the new function for getting the pseudo moving sum of nr_accesses on demands. Patch 2 implements a unit test for the new function's internal logic. Patch 3 and 4 update monitoring logic and the new function to ready for safe use on the existing logic. Patches 5-7 replace uses of nr_accesses_bp in DAMOS, tracepoints and DAMON sysfs interface with the new function, respectively. Patches 8-10 removes nr_accesses_bp validation functions in DAMON core, one by one. Patches 11 and 12 further remove tests and test helper for nr_accesses_bp, respectively. Patches 13 removes the setups and updates or nr_accesses_bp field. Patches 14-16 cleans up function parameters that are no more being used due to the previous patch. Patch 17 removes the function that was used for updating nr_accesses_bp field with its unit test, which is the single remaining caller of the function. Finally, patch 18 removes damon_region->nr_accesses_bp field. This patch (of 18): Introduce a new DAMON core function, damon_nr_accesses_mvsum(). It returns a pseudo moving sum value of a given region's nr_accesses for the last aggregation interval. The internal logic is the same to nr_accesses_bp. The difference is that nr_accesses_bp is updated for each sampling interval, while the new function needs to be executed only when requested. Hence the function's return value is the same as the value of nr_accesses_bp. Link: https://lore.kernel.org/20260630040812.149729-1-sj@kernel.org Link: https://lore.kernel.org/20260630040812.149729-2-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Shuah Khan Cc: Steven Rostedt Signed-off-by: Andrew Morton --- include/linux/damon.h | 2 ++ mm/damon/core.c | 62 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/include/linux/damon.h b/include/linux/damon.h index cfbbf8ba28f6..87c1ff479da8 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -1011,6 +1011,8 @@ struct damon_probe *damon_new_probe(void); void damon_add_probe(struct damon_ctx *ctx, struct damon_probe *probe); struct damon_region *damon_new_region(unsigned long start, unsigned long end); +unsigned int damon_nr_accesses_mvsum(struct damon_region *r, + struct damon_ctx *ctx); int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, unsigned int nr_ranges, unsigned long min_region_sz); diff --git a/mm/damon/core.c b/mm/damon/core.c index dbff0e4c1c17..21c15ffc26fd 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -208,6 +208,68 @@ static struct damon_probe *damon_nth_probe(int n, struct damon_ctx *ctx) return NULL; } +/* + * damon_mvsum() - Returns pseudo moving sum value for a time window. + * @current_nr: The value of the current aggregation window. + * @last_nr: The value of the last aggregation window. + * @left_window_bp: Left time of the current aggregation window. + * + * This function calculates a pseudo moving sum value of a counter that is + * aggregated for each time window. @current_nr is the value of the counter + * that aggregated so far (maybe not yet complete), from the beginning of the + * current aggregation time window. @last_nr is the value of the counter that + * has completely aggregated in the last aggregation time window. + * @left_window_bp represents how much time is left for the current aggregation + * time window in bp (1/10,000). For example, the aggregation time window is + * for every 10 seconds and 7 seconds has passed since the beginning of the + * current window, this parameter will be 3000 ((10 - 7) / 10 * 10000). + * + * The logic assumes the aggregation in the last phase was made in a single + * speed. Based on the assumption, the value from the last window that needs + * to be added to the current value is calculated as a portion of the last + * value based on the remaining time window. + */ +static unsigned long damon_mvsum(unsigned long current_nr, + unsigned long last_nr, unsigned long left_window_bp) +{ + return current_nr + mult_frac(last_nr, left_window_bp, 10000); +} + +/** + * damon_nr_accesses_mvsum() - Returns moving sum access frequency score. + * @r: Region to get the access frequency of. + * @ctx: DAMON context of @r. + * + * This function returns for how many sampling iterations in the last + * aggregation interval (&damon_attrs->aggr_interval) the region was found to + * be accessed. Hence the value can be interpreted as the relative access + * frequency score of the region (@r). The value is calculated as a pseudo + * moving sum, and hence it is not an exact value but just a best-effort + * reasonable estimation. + * + * Return: the pseudo moving sum access frequency score. + */ +unsigned int damon_nr_accesses_mvsum(struct damon_region *r, + struct damon_ctx *ctx) +{ + unsigned long sample_interval, aggr_interval; + unsigned long window_len, left_window, left_window_bp; + + sample_interval = ctx->attrs.sample_interval ? : 1; + aggr_interval = ctx->attrs.aggr_interval ? : 1; + window_len = aggr_interval / sample_interval; + if (time_after_eq(ctx->passed_sample_intervals, + ctx->next_aggregation_sis)) + left_window = 0; + else + left_window = ctx->next_aggregation_sis - + ctx->passed_sample_intervals; + left_window_bp = mult_frac(left_window, 10000, window_len); + + return damon_mvsum(r->nr_accesses, r->last_nr_accesses, + left_window_bp); +} + #ifdef CONFIG_DAMON_DEBUG_SANITY static void damon_verify_new_region(unsigned long start, unsigned long end) { From f8dbb118fa436be1ec4774dc8a4cfe5350d0a951 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 29 Jun 2026 21:07:55 -0700 Subject: [PATCH 153/501] mm/damon/tests/core-kunit: test damon_mvsum() Add a simple unit test for damon_nr_accesses_mvsum()'s internal core logic, damon_mvsum(). The test contains cases for just-started windows, partially completed windows, and just-completed windows. Link: https://lore.kernel.org/20260630040812.149729-3-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Shuah Khan Cc: Steven Rostedt Signed-off-by: Andrew Morton --- mm/damon/tests/core-kunit.h | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 0ec7d14d354e..c45377dcc97e 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -682,6 +682,30 @@ static void damon_test_moving_sum(struct kunit *test) } } +static void damon_test_mvsum(struct kunit *test) +{ + unsigned long input_expects[] = { + /* current value, last value, remaining window (bp) */ + 0, 49, 10000, 49, /* 0 + 49 * 1 */ + 3, 10, 7000, 10, /* 3 + 10 * 0.7 */ + 3, 10, 5000, 8, /* 3 + 10 * 0.5 */ + 32, 100, 1000, 42, /* 32 + 100 * 0.1 */ + 42, 49, 0, 42, /* 42 + 49 * 0 */ + }; + + int i; + + for (i = 0; i < ARRAY_SIZE(input_expects); i += 4) { + unsigned long current_nr = input_expects[i]; + unsigned long last_nr = input_expects[i + 1]; + unsigned long left_window_bp = input_expects[i + 2]; + unsigned long expect = input_expects[i + 3]; + + KUNIT_EXPECT_EQ(test, damon_mvsum(current_nr, last_nr, + left_window_bp), expect); + } +} + static void damos_test_new_filter(struct kunit *test) { struct damos_filter *filter; @@ -1575,6 +1599,7 @@ static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_update_monitoring_result), KUNIT_CASE(damon_test_set_attrs), KUNIT_CASE(damon_test_moving_sum), + KUNIT_CASE(damon_test_mvsum), KUNIT_CASE(damos_test_new_filter), KUNIT_CASE(damos_test_commit_quota_goal), KUNIT_CASE(damos_test_commit_quota_goals), From a44821c65c85102a657b3eb3daa56e662f1a175e Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 29 Jun 2026 21:07:56 -0700 Subject: [PATCH 154/501] mm/damon/core: always update ->last_nr_accesses for intervals change Each iteration of kdamond_fn() main loop caches and use the next aggregation time (next_aggregation_sis) because it can be updated in the middle, inside kdamond_call(). If that happens, damon_update_monitoring_result() is called for scaling the access frequency information of each region according to the changed intervals. The function does not update damon_region->last_nr_accesses when it is at the end of the aggregation, because it will anyway be reset after the function is executed, in kdamond_reset_aggregated(). Let's suppose damon_nr_accesses_mvsum() is called with the not yet updated last_nr_accesses. It will use the fresh next_aggregation_sis in the context instead of the cached one, unlike kdamond_fn(). As a result, use of not updated last_nr_acceses with the updated next_aggregation_sis result in returning wrong value. There is no such damon_nr_accesses_nvsum() call at the moment, so this is no problem. It is planned to add such calls, though. Prevent the issue by updating last_nr_accesses always. This adds overhead, but that's fine because the overhead is not big, and it is anyway not a fast path. Link: https://lore.kernel.org/20260630040812.149729-4-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Shuah Khan Cc: Steven Rostedt Signed-off-by: Andrew Morton --- mm/damon/core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 21c15ffc26fd..e6f0f7a6752d 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -884,6 +884,8 @@ static void damon_update_monitoring_result(struct damon_region *r, struct damon_attrs *old_attrs, struct damon_attrs *new_attrs, bool aggregating) { + r->last_nr_accesses = damon_nr_accesses_for_new_attrs( + r->last_nr_accesses, old_attrs, new_attrs); if (!aggregating) { r->nr_accesses = damon_nr_accesses_for_new_attrs( r->nr_accesses, old_attrs, new_attrs); @@ -895,8 +897,6 @@ static void damon_update_monitoring_result(struct damon_region *r, * interval. In other words, make the status like * kdamond_reset_aggregated() is called. */ - r->last_nr_accesses = damon_nr_accesses_for_new_attrs( - r->last_nr_accesses, old_attrs, new_attrs); r->nr_accesses_bp = r->last_nr_accesses * 10000; r->nr_accesses = 0; } From 1b43501d688d60964462151159f72cac1266c273 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 29 Jun 2026 21:07:57 -0700 Subject: [PATCH 155/501] mm/damon/core: handle unreset nr_accesses in damon_nr_accesses_mvsum() damon_set_attrs() works like reverting aggregations that were made so far for this aggregation window. If this is the end of the aggregation, however, kdamond_fn() will do the operations at the end of the aggregation interval, using cached timestamps. For such operations that rely on damon_region->nr_accesses, damon_update_monitoring_results() doesn't reset the nr_accesses if it is called at the end of the aggregation window. damon_nr_accesses_mvsum() works with fresh timestamps, though. The nr_accesses that are not reset in this case can make the logic to unnecessarily count nr_accesses, resulting in returning higher-than-expected pseudo moving sum nr_accesses. No code is using damon_nr_accesses_mvsum() yet, so this is not causing a real problem. Following commits will add usage of the function, though. For safe usages, calculate the pseudo moving sum without nr_accesses if the remaining window is full. Link: https://lore.kernel.org/20260630040812.149729-5-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Shuah Khan Cc: Steven Rostedt Signed-off-by: Andrew Morton --- mm/damon/core.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mm/damon/core.c b/mm/damon/core.c index e6f0f7a6752d..fcd7347e48e3 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -266,6 +266,9 @@ unsigned int damon_nr_accesses_mvsum(struct damon_region *r, ctx->passed_sample_intervals; left_window_bp = mult_frac(left_window, 10000, window_len); + if (left_window_bp == 10000) + return r->last_nr_accesses; + return damon_mvsum(r->nr_accesses, r->last_nr_accesses, left_window_bp); } From 8ea1a3a70abaa85343cd47d0add115d3410d3b35 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 29 Jun 2026 21:07:58 -0700 Subject: [PATCH 156/501] mm/damon/core: use damon_nr_accesses_mvsum() in __damos_valid_target() damon_nr_accesses_mvsum() returns a value same to nr_accesses_bp. Also the function is more simple and therefore more tolerant to errors. Execution of the function would be more expensive than the simple read of the field, but because the function is quite simple, the overhead should be negligible. Use it in __damos_valid_target() instead of the nr_accesses_bp. Link: https://lore.kernel.org/20260630040812.149729-6-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Shuah Khan Cc: Steven Rostedt Signed-off-by: Andrew Morton --- mm/damon/core.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index fcd7347e48e3..80ae08b704c6 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -2151,10 +2151,11 @@ static noinline_for_stack void kdamond_tune_intervals(struct damon_ctx *c) damon_set_attrs(c, &new_attrs); } -static bool __damos_valid_target(struct damon_region *r, struct damos *s) +static bool __damos_valid_target(struct damon_region *r, struct damos *s, + struct damon_ctx *c) { unsigned long sz; - unsigned int nr_accesses = r->nr_accesses_bp / 10000; + unsigned int nr_accesses = damon_nr_accesses_mvsum(r, c); sz = damon_sz_region(r); return s->pattern.min_sz_region <= sz && @@ -2180,7 +2181,7 @@ static bool damos_quota_is_set(struct damos_quota *quota) static bool damos_valid_target(struct damon_ctx *c, struct damon_region *r, struct damos *s) { - bool ret = __damos_valid_target(r, s); + bool ret = __damos_valid_target(r, s, c); if (!ret || !damos_quota_is_set(&s->quota) || !c->ops.get_scheme_score) return ret; @@ -2766,7 +2767,7 @@ static phys_addr_t damos_calc_eligible_bytes(struct damon_ctx *c, damon_for_each_region(r, t) { phys_addr_t addr, end_addr; - if (!__damos_valid_target(r, s)) + if (!__damos_valid_target(r, s, c)) continue; /* Convert from core address units to physical bytes */ @@ -3055,7 +3056,7 @@ static void damos_adjust_quota(struct damon_ctx *c, struct damos *s) (DAMOS_MAX_SCORE + 1)); damon_for_each_target(t, c) { damon_for_each_region(r, t) { - if (!__damos_valid_target(r, s)) + if (!__damos_valid_target(r, s, c)) continue; if (damos_core_filter_out(c, t, r, s)) continue; From bcea0202d4656f513be75431a05107c020eee24e Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 29 Jun 2026 21:07:59 -0700 Subject: [PATCH 157/501] mm/damon/core: use damon_nr_accesses_mvsum() for damos region tracing damon_nr_accesses_mvsum() returns a value same to nr_accesses_bp. Also the function is more simple and therefore more tolerant to errors. Execution of the function would be more expensive than the simple read of the field, but because the function is quite simple, the overhead should be negligible. Use it in the DAMON region exporting trace points instead of the nr_accesses_bp. Link: https://lore.kernel.org/20260630040812.149729-7-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Shuah Khan Cc: Steven Rostedt Signed-off-by: Andrew Morton --- include/trace/events/damon.h | 8 +++++--- mm/damon/core.c | 5 +++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/include/trace/events/damon.h b/include/trace/events/damon.h index 78388538acf4..8851727ae162 100644 --- a/include/trace/events/damon.h +++ b/include/trace/events/damon.h @@ -78,9 +78,11 @@ TRACE_EVENT_CONDITION(damos_before_apply, TP_PROTO(unsigned int context_idx, unsigned int scheme_idx, unsigned int target_idx, struct damon_region *r, - unsigned int nr_regions, bool do_trace), + unsigned int nr_accesses, unsigned int nr_regions, + bool do_trace), - TP_ARGS(context_idx, scheme_idx, target_idx, r, nr_regions, do_trace), + TP_ARGS(context_idx, scheme_idx, target_idx, r, nr_accesses, + nr_regions, do_trace), TP_CONDITION(do_trace), @@ -101,7 +103,7 @@ TRACE_EVENT_CONDITION(damos_before_apply, __entry->target_idx = target_idx; __entry->start = r->ar.start; __entry->end = r->ar.end; - __entry->nr_accesses = r->nr_accesses_bp / 10000; + __entry->nr_accesses = nr_accesses; __entry->age = r->age; __entry->nr_regions = nr_regions; ), diff --git a/mm/damon/core.c b/mm/damon/core.c index 80ae08b704c6..4628097fd9cb 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -2465,7 +2465,7 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t, struct damos *siter; /* schemes iterator */ unsigned int sidx = 0; struct damon_target *titer; /* targets iterator */ - unsigned int tidx = 0; + unsigned int tidx = 0, nr_accesses = 0; bool do_trace = false; /* get indices for trace_damos_before_apply() */ @@ -2480,6 +2480,7 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t, break; tidx++; } + nr_accesses = damon_nr_accesses_mvsum(r, c); do_trace = true; } @@ -2495,7 +2496,7 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t, if (damos_core_filter_out(c, t, r, s)) return; ktime_get_coarse_ts64(&begin); - trace_damos_before_apply(cidx, sidx, tidx, r, + trace_damos_before_apply(cidx, sidx, tidx, r, nr_accesses, damon_nr_regions(t), do_trace); sz_applied = c->ops.apply_scheme(c, t, r, s, &sz_ops_filter_passed); From e4375bcf0fb2514afc794569728defbf3d126869 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 29 Jun 2026 21:08:00 -0700 Subject: [PATCH 158/501] mm/damon/sysfs-schemes: use damon_nr_accesses_mvsum() for damo regions damon_nr_accesses_mvsum() returns a value same to nr_accesses_bp. Also the function is more simple and therefore more tolerant to errors. Execution of the function would be more expensive than the simple read of the field, but because the function is quite simple, the overhead should be negligible. Use it in the DAMON sysfs interface for scheme-tried regions, instead of the nr_accesses_bp. Link: https://lore.kernel.org/20260630040812.149729-8-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Shuah Khan Cc: Steven Rostedt Signed-off-by: Andrew Morton --- mm/damon/sysfs-schemes.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 41f93a1823bf..dbf2b0515d58 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -157,7 +157,7 @@ struct damon_sysfs_scheme_region { }; static struct damon_sysfs_scheme_region *damon_sysfs_scheme_region_alloc( - struct damon_region *region) + struct damon_region *region, struct damon_ctx *ctx) { struct damon_sysfs_scheme_region *sysfs_region = kmalloc_obj(*sysfs_region); @@ -165,7 +165,7 @@ static struct damon_sysfs_scheme_region *damon_sysfs_scheme_region_alloc( return NULL; sysfs_region->kobj = (struct kobject){}; sysfs_region->ar = region->ar; - sysfs_region->nr_accesses = region->nr_accesses_bp / 10000; + sysfs_region->nr_accesses = damon_nr_accesses_mvsum(region, ctx); sysfs_region->age = region->age; sysfs_region->probes = NULL; INIT_LIST_HEAD(&sysfs_region->list); @@ -3122,7 +3122,7 @@ void damos_sysfs_populate_region_dir(struct damon_sysfs_schemes *sysfs_schemes, if (total_bytes_only) return; - region = damon_sysfs_scheme_region_alloc(r); + region = damon_sysfs_scheme_region_alloc(r, ctx); if (!region) return; region->sz_filter_passed = sz_filter_passed; From 653902082cc80ee36300eb94c4499278f12b011c Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 29 Jun 2026 21:08:01 -0700 Subject: [PATCH 159/501] mm/damon/core: remove damon_warn_fix_nr_accesses_corruption() nr_accesses_bp is delicate. Once it is corrupted, the consequence is the bad madness of DAMON monitoring results. From developments of features of size, we historically found nr_accesses_bp can be corrupted by complicated bugs that are not easy to debug. Hence we added a function for finding the corruption and fixing it right away. There are no more uses of nr_accesses_bp. Hence the function for corruption detection and fix is no more needed. Rip it out. Link: https://lore.kernel.org/20260630040812.149729-9-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Shuah Khan Cc: Steven Rostedt Signed-off-by: Andrew Morton --- mm/damon/core.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 4628097fd9cb..19606c959503 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -2025,19 +2025,6 @@ int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control) return 0; } -/* - * Warn and fix corrupted ->nr_accesses[_bp] for investigations and preventing - * the problem being propagated. - */ -static void damon_warn_fix_nr_accesses_corruption(struct damon_region *r) -{ - if (r->nr_accesses_bp == r->nr_accesses * 10000) - return; - WARN_ONCE(true, "invalid nr_accesses_bp at reset: %u %u\n", - r->nr_accesses_bp, r->nr_accesses); - r->nr_accesses_bp = r->nr_accesses * 10000; -} - #ifdef CONFIG_DAMON_DEBUG_SANITY static void damon_verify_reset_aggregated(struct damon_region *r, struct damon_ctx *c) @@ -2079,7 +2066,6 @@ static void kdamond_reset_aggregated(struct damon_ctx *c) trace_damon_aggregated(ti, r, damon_nr_regions(t)); trace_damon_region_aggregated(ti, r, damon_nr_regions(t), nr_probes); - damon_warn_fix_nr_accesses_corruption(r); r->last_nr_accesses = r->nr_accesses; r->nr_accesses = 0; for (i = 0; i < DAMON_MAX_PROBES; i++) From e33eb22d718b0bedb654fee9718e7c9f70a57fea Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 29 Jun 2026 21:08:02 -0700 Subject: [PATCH 160/501] mm/damon/core: remove damon_verify_reset_aggregated() nr_accesses_bp is no longer being used in real use cases. Remove its validation function. Link: https://lore.kernel.org/20260630040812.149729-10-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Shuah Khan Cc: Steven Rostedt Signed-off-by: Andrew Morton --- mm/damon/core.c | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 19606c959503..aa0b76ccac4c 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -2025,23 +2025,6 @@ int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control) return 0; } -#ifdef CONFIG_DAMON_DEBUG_SANITY -static void damon_verify_reset_aggregated(struct damon_region *r, - struct damon_ctx *c) -{ - WARN_ONCE(r->nr_accesses_bp != r->last_nr_accesses * 10000, - "nr_accesses_bp %u last_nr_accesses %u sis %lu %lu\n", - r->nr_accesses_bp, r->last_nr_accesses, - c->passed_sample_intervals, c->next_aggregation_sis); -} -#else -static void damon_verify_reset_aggregated(struct damon_region *r, - struct damon_ctx *c) -{ -} -#endif - - /* * Reset the aggregated monitoring results ('nr_accesses' of each region). */ @@ -2070,7 +2053,6 @@ static void kdamond_reset_aggregated(struct damon_ctx *c) r->nr_accesses = 0; for (i = 0; i < DAMON_MAX_PROBES; i++) r->probe_hits[i] = 0; - damon_verify_reset_aggregated(r, c); } ti++; } From 5c1eece7c961f34e9900ecc16cf347baaf0b2af5 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 29 Jun 2026 21:08:03 -0700 Subject: [PATCH 161/501] mm/damon/core: remove damon_verify_merge_regions_of() damon_verify_merge_regions_of() is only for nr_accesses_bp validation. But nr_accesses_bp is no more being used for a real purpose. Remove the validation. Link: https://lore.kernel.org/20260630040812.149729-11-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Shuah Khan Cc: Steven Rostedt Signed-off-by: Andrew Morton --- mm/damon/core.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index aa0b76ccac4c..d2b7609dcf5a 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -3140,20 +3140,6 @@ static void damon_merge_two_regions(struct damon_target *t, damon_destroy_region(r, t); } -#ifdef CONFIG_DAMON_DEBUG_SANITY -static void damon_verify_merge_regions_of(struct damon_region *r) -{ - WARN_ONCE(r->nr_accesses != r->nr_accesses_bp / 10000, - "nr_accesses (%u) != nr_accesses_bp (%u)\n", - r->nr_accesses, r->nr_accesses_bp); -} -#else -static void damon_verify_merge_regions_of(struct damon_region *r) -{ -} -#endif - - /* * Merge adjacent regions having similar access frequencies * @@ -3167,7 +3153,6 @@ static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, struct damon_region *r, *prev = NULL, *next; damon_for_each_region_safe(r, next, t) { - damon_verify_merge_regions_of(r); if (abs(r->nr_accesses - r->last_nr_accesses) > thres) r->age = 0; else if ((r->nr_accesses == 0) != (r->last_nr_accesses == 0)) From ca7e1c3f068322d5ee9f69ee864a2c124bf5c992 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 29 Jun 2026 21:08:04 -0700 Subject: [PATCH 162/501] mm/damon/tests/core-kunit: remove nr_accesses_bp setup and tests DAMON core unit tests set up nr_accesses_bp for representing realistic damon_region, and also test the field. nr_acceses_bp is no longer being used for a real use case. Remove the setup and tests. Link: https://lore.kernel.org/20260630040812.149729-12-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Shuah Khan Cc: Steven Rostedt Signed-off-by: Andrew Morton --- mm/damon/tests/core-kunit.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index c45377dcc97e..0566fc3dc067 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -114,7 +114,6 @@ static void damon_test_aggregate(struct kunit *test) kunit_skip(test, "region alloc fail"); } r->nr_accesses = accesses[it][ir]; - r->nr_accesses_bp = accesses[it][ir] * 10000; damon_add_region(r, t); } it++; @@ -151,7 +150,6 @@ static void damon_test_split_at(struct kunit *test) damon_free_target(t); kunit_skip(test, "region alloc fail"); } - r->nr_accesses_bp = 420000; r->nr_accesses = 42; r->last_nr_accesses = 15; r->age = 10; @@ -164,7 +162,6 @@ static void damon_test_split_at(struct kunit *test) KUNIT_EXPECT_EQ(test, r_new->ar.start, 25ul); KUNIT_EXPECT_EQ(test, r_new->ar.end, 100ul); - KUNIT_EXPECT_EQ(test, r->nr_accesses_bp, r_new->nr_accesses_bp); KUNIT_EXPECT_EQ(test, r->nr_accesses, r_new->nr_accesses); KUNIT_EXPECT_EQ(test, r->last_nr_accesses, r_new->last_nr_accesses); KUNIT_EXPECT_EQ(test, r->age, r_new->age); @@ -187,7 +184,6 @@ static void damon_test_merge_two(struct kunit *test) kunit_skip(test, "region alloc fail"); } r->nr_accesses = 10; - r->nr_accesses_bp = 100000; r->age = 9; damon_add_region(r, t); r2 = damon_new_region(100, 300); @@ -196,7 +192,6 @@ static void damon_test_merge_two(struct kunit *test) kunit_skip(test, "second region alloc fail"); } r2->nr_accesses = 20; - r2->nr_accesses_bp = 200000; r2->age = 21; damon_add_region(r2, t); @@ -204,7 +199,6 @@ static void damon_test_merge_two(struct kunit *test) KUNIT_EXPECT_EQ(test, r->ar.start, 0ul); KUNIT_EXPECT_EQ(test, r->ar.end, 300ul); KUNIT_EXPECT_EQ(test, r->nr_accesses, 16u); - KUNIT_EXPECT_EQ(test, r->nr_accesses_bp, 160000u); KUNIT_EXPECT_EQ(test, r->age, 17u); i = 0; @@ -252,7 +246,6 @@ static void damon_test_merge_regions_of(struct kunit *test) kunit_skip(test, "region alloc fail"); } r->nr_accesses = nrs[i]; - r->nr_accesses_bp = nrs[i] * 10000; damon_add_region(r, t); } @@ -615,7 +608,6 @@ static void damon_test_update_monitoring_result(struct kunit *test) kunit_skip(test, "region alloc fail"); r->nr_accesses = 15; - r->nr_accesses_bp = 150000; r->age = 20; new_attrs = (struct damon_attrs){ From 51bd0de225a7c8488e53056cd35d89cd1cfc1e71 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 29 Jun 2026 21:08:05 -0700 Subject: [PATCH 163/501] selftests/damon/drgn_dump_damon_status: do not dump nr_accesses_bp drgn_dump_damon_status is dumping nr_accesses_bp field for future use case. nr_accesses_bp is not being used for a real purpose, though. Hence there will be no future test for it. Do not dump it. Link: https://lore.kernel.org/20260630040812.149729-13-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Shuah Khan Cc: Steven Rostedt Signed-off-by: Andrew Morton --- tools/testing/selftests/damon/drgn_dump_damon_status.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/testing/selftests/damon/drgn_dump_damon_status.py b/tools/testing/selftests/damon/drgn_dump_damon_status.py index 26b207e44268..09552e91bc78 100755 --- a/tools/testing/selftests/damon/drgn_dump_damon_status.py +++ b/tools/testing/selftests/damon/drgn_dump_damon_status.py @@ -59,7 +59,6 @@ def region_to_dict(region): ['ar', addr_range_to_dict], ['sampling_addr', int], ['nr_accesses', int], - ['nr_accesses_bp', int], ['age', int], ]) From 8bd5540d8eb3a1bb4551c97e292706fe3a3af924 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 29 Jun 2026 21:08:06 -0700 Subject: [PATCH 164/501] mm/damon/core: remove nr_accesses_bp setups and updates DAMON core sets and updates nr_accesses_bp in multiple places. It explains how delicate it is. The field is no more being used for any real purpose, and replaced by a simpler function. Remove the setups and updates. Link: https://lore.kernel.org/20260630040812.149729-14-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Shuah Khan Cc: Steven Rostedt Signed-off-by: Andrew Morton --- mm/damon/core.c | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index d2b7609dcf5a..ef0b87f2c035 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -302,7 +302,6 @@ struct damon_region *damon_new_region(unsigned long start, unsigned long end) region->ar.start = start; region->ar.end = end; region->nr_accesses = 0; - region->nr_accesses_bp = 0; for (i = 0; i < DAMON_MAX_PROBES; i++) region->probe_hits[i] = 0; INIT_LIST_HEAD(®ion->list); @@ -889,20 +888,17 @@ static void damon_update_monitoring_result(struct damon_region *r, { r->last_nr_accesses = damon_nr_accesses_for_new_attrs( r->last_nr_accesses, old_attrs, new_attrs); - if (!aggregating) { + if (!aggregating) r->nr_accesses = damon_nr_accesses_for_new_attrs( r->nr_accesses, old_attrs, new_attrs); - r->nr_accesses_bp = r->nr_accesses * 10000; - } else { + else /* * if this is called in the middle of the aggregation, reset * the aggregations we made so far for this aggregation * interval. In other words, make the status like * kdamond_reset_aggregated() is called. */ - r->nr_accesses_bp = r->last_nr_accesses * 10000; r->nr_accesses = 0; - } r->age = damon_age_for_new_attrs(r->age, old_attrs, new_attrs); } @@ -3129,7 +3125,6 @@ static void damon_merge_two_regions(struct damon_target *t, l->nr_accesses = (l->nr_accesses * sz_l + r->nr_accesses * sz_r) / (sz_l + sz_r); - l->nr_accesses_bp = l->nr_accesses * 10000; l->age = (l->age * sz_l + r->age * sz_r) / (sz_l + sz_r); l->ar.end = r->ar.end; /* todo: do this for only installed probes */ @@ -3241,7 +3236,6 @@ static void damon_split_region_at(struct damon_target *t, new->age = r->age; new->last_nr_accesses = r->last_nr_accesses; - new->nr_accesses_bp = r->nr_accesses_bp; new->nr_accesses = r->nr_accesses; /* todo: do this for only installed probes */ memcpy(new->probe_hits, r->probe_hits, sizeof(r->probe_hits)); @@ -3849,18 +3843,6 @@ static unsigned int damon_moving_sum(unsigned int mvsum, unsigned int nomvsum, void damon_update_region_access_rate(struct damon_region *r, bool accessed, struct damon_attrs *attrs) { - unsigned int len_window = 1; - - /* - * sample_interval can be zero, but cannot be larger than - * aggr_interval, owing to validation of damon_set_attrs(). - */ - if (attrs->sample_interval) - len_window = damon_max_nr_accesses(attrs); - r->nr_accesses_bp = damon_moving_sum(r->nr_accesses_bp, - r->last_nr_accesses * 10000, len_window, - accessed ? 10000 : 0); - if (accessed) r->nr_accesses++; } From 3a61be80e86cb498e353a76b195337a1d167a3ba Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 29 Jun 2026 21:08:07 -0700 Subject: [PATCH 165/501] mm/damon/core: remove attrs param from damon_update_region_access_rate() damon_update_region_access_rate() is not using attrs parameter. Remove it. Link: https://lore.kernel.org/20260630040812.149729-15-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Shuah Khan Cc: Steven Rostedt Signed-off-by: Andrew Morton --- include/linux/damon.h | 3 +-- mm/damon/core.c | 4 +--- mm/damon/paddr.c | 4 ++-- mm/damon/vaddr.c | 6 +++--- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 87c1ff479da8..02ed47c558cc 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -1016,8 +1016,7 @@ unsigned int damon_nr_accesses_mvsum(struct damon_region *r, int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, unsigned int nr_ranges, unsigned long min_region_sz); -void damon_update_region_access_rate(struct damon_region *r, bool accessed, - struct damon_attrs *attrs); +void damon_update_region_access_rate(struct damon_region *r, bool accessed); struct damos_filter *damos_new_filter(enum damos_filter_type type, bool matching, bool allow); diff --git a/mm/damon/core.c b/mm/damon/core.c index ef0b87f2c035..ce14e2929e9c 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -3833,15 +3833,13 @@ static unsigned int damon_moving_sum(unsigned int mvsum, unsigned int nomvsum, * damon_update_region_access_rate() - Update the access rate of a region. * @r: The DAMON region to update for its access check result. * @accessed: Whether the region has accessed during last sampling interval. - * @attrs: The damon_attrs of the DAMON context. * * Update the access rate of a region with the region's last sampling interval * access check result. * * Usually this will be called by &damon_operations->check_accesses callback. */ -void damon_update_region_access_rate(struct damon_region *r, bool accessed, - struct damon_attrs *attrs) +void damon_update_region_access_rate(struct damon_region *r, bool accessed) { if (accessed) r->nr_accesses++; diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 5c2da45f988c..853021308669 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -91,12 +91,12 @@ static void __damon_pa_check_access(struct damon_region *r, /* If the region is in the last checked page, reuse the result */ if (ALIGN_DOWN(last_addr, last_folio_sz) == ALIGN_DOWN(sampling_addr, last_folio_sz)) { - damon_update_region_access_rate(r, last_accessed, attrs); + damon_update_region_access_rate(r, last_accessed); return; } last_accessed = damon_pa_young(sampling_addr, &last_folio_sz); - damon_update_region_access_rate(r, last_accessed, attrs); + damon_update_region_access_rate(r, last_accessed); last_addr = sampling_addr; } diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index e73ec1ce016e..2eaced0765e2 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -501,19 +501,19 @@ static void __damon_va_check_access(struct mm_struct *mm, static bool last_accessed; if (!mm) { - damon_update_region_access_rate(r, false, attrs); + damon_update_region_access_rate(r, false); return; } /* If the region is in the last checked page, reuse the result */ if (same_target && (ALIGN_DOWN(last_addr, last_folio_sz) == ALIGN_DOWN(r->sampling_addr, last_folio_sz))) { - damon_update_region_access_rate(r, last_accessed, attrs); + damon_update_region_access_rate(r, last_accessed); return; } last_accessed = damon_va_young(mm, r->sampling_addr, &last_folio_sz); - damon_update_region_access_rate(r, last_accessed, attrs); + damon_update_region_access_rate(r, last_accessed); last_addr = r->sampling_addr; } From 74a03cad1b9d0b6b18b46ee248cb516c6da018e1 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 29 Jun 2026 21:08:08 -0700 Subject: [PATCH 166/501] mm/damon/paddr: remove attrs param from __damon_pa_check_access() The function is not using the parameter. Remove it. Link: https://lore.kernel.org/20260630040812.149729-16-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Shuah Khan Cc: Steven Rostedt Signed-off-by: Andrew Morton --- mm/damon/paddr.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 853021308669..617246498173 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -80,7 +80,7 @@ static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) } static void __damon_pa_check_access(struct damon_region *r, - struct damon_attrs *attrs, unsigned long addr_unit) + unsigned long addr_unit) { static phys_addr_t last_addr; static unsigned long last_folio_sz = PAGE_SIZE; @@ -109,8 +109,7 @@ static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) damon_for_each_target(t, ctx) { damon_for_each_region(r, t) { - __damon_pa_check_access( - r, &ctx->attrs, ctx->addr_unit); + __damon_pa_check_access(r, ctx->addr_unit); max_nr_accesses = max(r->nr_accesses, max_nr_accesses); } } From 5744423b4686a4bf2a638a20b833b25049949d5e Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 29 Jun 2026 21:08:09 -0700 Subject: [PATCH 167/501] mm/damon/vaddr: remove attrs param from __damon_va_check_access() The function is not using attrs parameter. Remove it. Link: https://lore.kernel.org/20260630040812.149729-17-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Shuah Khan Cc: Steven Rostedt Signed-off-by: Andrew Morton --- mm/damon/vaddr.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 2eaced0765e2..2058db9c01d5 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -493,8 +493,7 @@ static bool damon_va_young(struct mm_struct *mm, unsigned long addr, * r the region to be checked */ static void __damon_va_check_access(struct mm_struct *mm, - struct damon_region *r, bool same_target, - struct damon_attrs *attrs) + struct damon_region *r, bool same_target) { static unsigned long last_addr; static unsigned long last_folio_sz = PAGE_SIZE; @@ -530,8 +529,7 @@ static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) mm = damon_get_mm(t); same_target = false; damon_for_each_region(r, t) { - __damon_va_check_access(mm, r, same_target, - &ctx->attrs); + __damon_va_check_access(mm, r, same_target); max_nr_accesses = max(r->nr_accesses, max_nr_accesses); same_target = true; } From 9255add1fc85001b0aabfa94d918056b8a1817b4 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 29 Jun 2026 21:08:10 -0700 Subject: [PATCH 168/501] mm/damon/core: remove damon_moving_sum() and its unit test damon_moving_sum() is no longer being called for real purpose but its unit test. Testing a function that is not being used for real users makes no sense. Remove the test and the function. Link: https://lore.kernel.org/20260630040812.149729-18-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Shuah Khan Cc: Steven Rostedt Signed-off-by: Andrew Morton --- mm/damon/core.c | 40 ------------------------------------- mm/damon/tests/core-kunit.h | 16 --------------- 2 files changed, 56 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index ce14e2929e9c..0c32219fb1f8 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -3789,46 +3789,6 @@ int damon_set_region_system_rams_default(struct damon_target *t, return damon_set_regions(t, &addr_range, 1, min_region_sz); } -/* - * damon_moving_sum() - Calculate an inferred moving sum value. - * @mvsum: Inferred sum of the last @len_window values. - * @nomvsum: Non-moving sum of the last discrete @len_window window values. - * @len_window: The number of last values to take care of. - * @new_value: New value that will be added to the pseudo moving sum. - * - * Moving sum (moving average * window size) is good for handling noise, but - * the cost of keeping past values can be high for arbitrary window size. This - * function implements a lightweight pseudo moving sum function that doesn't - * keep the past window values. - * - * It simply assumes there was no noise in the past, and get the no-noise - * assumed past value to drop from @nomvsum and @len_window. @nomvsum is a - * non-moving sum of the last window. For example, if @len_window is 10 and we - * have 25 values, @nomvsum is the sum of the 11th to 20th values of the 25 - * values. Hence, this function simply drops @nomvsum / @len_window from - * given @mvsum and add @new_value. - * - * For example, if @len_window is 10 and @nomvsum is 50, the last 10 values for - * the last window could be vary, e.g., 0, 10, 0, 10, 0, 10, 0, 0, 0, 20. For - * calculating next moving sum with a new value, we should drop 0 from 50 and - * add the new value. However, this function assumes it got value 5 for each - * of the last ten times. Based on the assumption, when the next value is - * measured, it drops the assumed past value, 5 from the current sum, and add - * the new value to get the updated pseduo-moving average. - * - * This means the value could have errors, but the errors will be disappeared - * for every @len_window aligned calls. For example, if @len_window is 10, the - * pseudo moving sum with 11th value to 19th value would have an error. But - * the sum with 20th value will not have the error. - * - * Return: Pseudo-moving average after getting the @new_value. - */ -static unsigned int damon_moving_sum(unsigned int mvsum, unsigned int nomvsum, - unsigned int len_window, unsigned int new_value) -{ - return mvsum - nomvsum / len_window + new_value; -} - /** * damon_update_region_access_rate() - Update the access rate of a region. * @r: The DAMON region to update for its access check result. diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 0566fc3dc067..0124f83b39b8 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -659,21 +659,6 @@ static void damon_test_set_attrs(struct kunit *test) damon_destroy_ctx(c); } -static void damon_test_moving_sum(struct kunit *test) -{ - unsigned int mvsum = 50000, nomvsum = 50000, len_window = 10; - unsigned int new_values[] = {10000, 0, 10000, 0, 0, 0, 10000, 0, 0, 0}; - unsigned int expects[] = {55000, 50000, 55000, 50000, 45000, 40000, - 45000, 40000, 35000, 30000}; - int i; - - for (i = 0; i < ARRAY_SIZE(new_values); i++) { - mvsum = damon_moving_sum(mvsum, nomvsum, len_window, - new_values[i]); - KUNIT_EXPECT_EQ(test, mvsum, expects[i]); - } -} - static void damon_test_mvsum(struct kunit *test) { unsigned long input_expects[] = { @@ -1590,7 +1575,6 @@ static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_nr_accesses_to_accesses_bp), KUNIT_CASE(damon_test_update_monitoring_result), KUNIT_CASE(damon_test_set_attrs), - KUNIT_CASE(damon_test_moving_sum), KUNIT_CASE(damon_test_mvsum), KUNIT_CASE(damos_test_new_filter), KUNIT_CASE(damos_test_commit_quota_goal), From a5edf881c9523ccfd0e64a45f80176ffe90e12d1 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 29 Jun 2026 21:08:11 -0700 Subject: [PATCH 169/501] mm/damon/core: remove damon_region->nr_accesses_bp No code touches damon_region->nr_accesses_bp field. Remove it. Link: https://lore.kernel.org/20260630040812.149729-19-sj@kernel.org Signed-off-by: SJ Park Cc: Brendan Higgins Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Shuah Khan Cc: Steven Rostedt Signed-off-by: Andrew Morton --- include/linux/damon.h | 10 ---------- mm/damon/core.c | 3 +-- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 02ed47c558cc..805e089ff4f2 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -45,8 +45,6 @@ struct damon_size_range { * @ar: The address range of the region. * @sampling_addr: Address of the sample for the next access check. * @nr_accesses: Access frequency of this region. - * @nr_accesses_bp: @nr_accesses in basis point (0.01%) that updated for - * each sampling interval. * @probe_hits: Number of probe-positive region samples. * @list: List head for siblings. * @age: Age of this region. @@ -59,13 +57,6 @@ struct damon_size_range { * not be done with direct access but with the helper function, * damon_update_region_access_rate(). * - * @nr_accesses_bp is another representation of @nr_accesses in basis point - * (1 in 10,000) that updated for every &damon_attrs->sample_interval in a - * manner similar to moving sum. By the algorithm, this value becomes - * @nr_accesses * 10000 for every &struct damon_attrs->aggr_interval. This can - * be used when the aggregation interval is too huge and therefore cannot wait - * for it before getting the access monitoring results. - * * @age is initially zero, increased for each aggregation interval, and reset * to zero again if the access frequency is significantly changed. If two * regions are merged into a new region, both @nr_accesses and @age of the new @@ -75,7 +66,6 @@ struct damon_region { struct damon_addr_range ar; unsigned long sampling_addr; unsigned int nr_accesses; - unsigned int nr_accesses_bp; unsigned char probe_hits[DAMON_MAX_PROBES]; struct list_head list; diff --git a/mm/damon/core.c b/mm/damon/core.c index 0c32219fb1f8..b2fc15a3804f 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -3649,8 +3649,7 @@ static int kdamond_fn(void *data) * aggregation, and make aggregation * information reset for all regions. Then, * following kdamond_reset_aggregated() call - * will make the region information invalid, - * particularly for ->nr_accesses_bp. + * will make the region information invalid. * * Reset ->next_aggregation_sis to avoid that. * It will anyway correctly updated after this From d7a8934c078cce738b4b24e8d191d6a6102d6a77 Mon Sep 17 00:00:00 2001 From: Zhen Yan Date: Tue, 30 Jun 2026 20:50:47 +0800 Subject: [PATCH 170/501] mm: fix mapping_seek_hole_data() overflow on last page A local unprivileged process can create a shmem/tmpfs file with i_size == LLONG_MAX using memfd_create() and fallocate(). If the last page is present in the page cache, lseek(SEEK_HOLE) on that page returns 0x8000000000000000 as a successful offset, which is LLONG_MIN when stored in loff_t. The same file has readable data at the last byte, but SEEK_DATA from that offset returns ENXIO. The overflow is in mapping_seek_hole_data(): pos = round_up((u64)pos + 1, seek_size); For the final page below LLONG_MAX, the next page boundary is 0x8000000000000000, which is then used as a signed file offset. When assigned to the loff_t pos, this overflows to LLONG_MIN, so a subsequent "pos > end" comparison does not catch it. Keep mapping_seek_hole_data() inside its documented [start, end) search range: compute round_up() into a u64 variable and compare against (u64)end so the overflow is detected, then clamp pos to end when the rounded-up value goes past the search limit. Link: https://lore.kernel.org/20260630125047.703170-1-yanzhen20011121@163.com Signed-off-by: Zhen Yan Cc: Christian Brauner Cc: Hugh Dickins Cc: Jan Kara Cc: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- mm/filemap.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mm/filemap.c b/mm/filemap.c index 6e40f36c2bff..b39111abdc4b 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -3229,6 +3229,7 @@ loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start, while ((folio = find_get_entry(&xas, max, XA_PRESENT))) { loff_t pos = (u64)xas.xa_index << PAGE_SHIFT; size_t seek_size; + u64 next; if (start < pos) { if (!seek_data) @@ -3237,7 +3238,11 @@ loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start, } seek_size = seek_folio_size(&xas, folio); - pos = round_up((u64)pos + 1, seek_size); + next = round_up((u64)pos + 1, seek_size); + if (next > (u64)end) + pos = end; + else + pos = next; start = folio_seek_hole_data(&xas, mapping, folio, start, pos, seek_data); if (start < pos) From 88819619688c5cc76dc38f65003898200ddd590a Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 2 Jul 2026 09:21:45 -0700 Subject: [PATCH 171/501] mm: hugetlb: consolidate interpretation of gbl_chg within alloc_hugetlb_folio() Patch series "Open HugeTLB allocation routine for more generic use", v4. The motivation for this patch series is guest_memfd, which would like to use HugeTLB as a generic source of huge pages but not adopt HugeTLB's reservation at mmap() time. By refactoring alloc_hugetlb_folio() and some dependent functions, there is now an option to allocate HugeTLB folios without providing a VMA. Specifically, HugeTLB allocation used to be dependent on the VMA to 1. Look up reservations in the resv_map 2. Get mpol, stored at vma->vm_policy This refactoring provides hugetlb_alloc_folio(), which focuses on just the allocation itself, and associated memory and HugeTLB charging (cgroups). alloc_hugetlb_folio() still handles reservations in the resv_map and subpools. Regarding naming, I'm definitely open to alternative names :) I chose hugetlb_alloc_folio() because I'm seeing this function as a general allocation function that is provided by the HugeTLB subsystem (hence the hugetlb_ prefix). I'm intending for alloc_hugetlb_folio() to be later refactored as a static function for use just by HugeTLB, and HugeTLBfs should probably use hugetlb_alloc_folio() directly. To see how hugetlb_alloc_folio() is used by guest_memfd, the most recent patch series that uses this more generic HugeTLB allocation routine is at [1], and a newer revision of that patch series is at [2]. Independently of guest_memfd, I believe this change is useful in simplifying alloc_hugetlb_folio(). alloc_hugetlb_folio() was so coupled to a VMA that even HugeTLBfs allocates HugeTLB folios using a pseudo-VMA. This patch (of 6): The dequeue_hugetlb_folio_vma() function currently handles the gbl_chg parameter to determine if a folio can be dequeued based on global page availability. This leaks reservation-specific logic into the dequeueing path. Relocate this logic to alloc_hugetlb_folio() so that dequeue_hugetlb_folio_vma() focuses solely on selecting and dequeuing a folio. In alloc_hugetlb_folio(), only attempt to dequeue a folio if a reservation exists (gbl_chg == 0) or if there are available huge pages in the global pool. No functional change intended. Link: https://lore.kernel.org/20260702-hugetlb-open-up-v4-0-d53cefcccf34@google.com Link: https://lore.kernel.org/20260702-hugetlb-open-up-v4-1-d53cefcccf34@google.com Link: https://lore.kernel.org/all/cover.1747264138.git.ackerleytng@google.com/T/ [1] Link: https://github.com/googleprodkernel/linux-cc/tree/wip-gmem-conversions-hugetlb-restructuring-12-08-25 [2] Link: https://lore.kernel.org/all/agqaUcVp_hwH-VXr@localhost.localdomain/ [3] Link: https://sashiko.dev/#/patchset/20260518-hugetlb-open-up-v3-0-e14b302477f8@google.com [4] Signed-off-by: Ackerley Tng Reviewed-by: James Houghton Acked-by: Oscar Salvador Reviewed-by: Joshua Hahn Cc: Qi Zheng Cc: Alistair Popple Cc: Byungchul Park Cc: David Hildenbrand Cc: David Rientjes Cc: "Edgecombe, Rick P" Cc: Frank van der Linden Cc: Gregory Price Cc: "Huang, Ying" Cc: Jason Gunthorpe Cc: Jiaqi Yan Cc: Matthew Brost Cc: Michael Roth Cc: Michal Hocko Cc: Muchun Song Cc: Paolo Bonzini Cc: Pasha Tatashin Cc: Peter Xu Cc: Pratyush Yadav Cc: Rakie Kim Cc: Roman Gushchin Cc: Sean Christopherson Cc: Shakeel Butt Cc: Shivank Garg Cc: Vishal Annapurve Cc: Yan Zhao Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/hugetlb.c | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 5bfbfc7e463e..dcf8df3d71b9 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1318,7 +1318,7 @@ static unsigned long available_huge_pages(struct hstate *h) static struct folio *dequeue_hugetlb_folio_vma(struct hstate *h, struct vm_area_struct *vma, - unsigned long address, long gbl_chg) + unsigned long address) { struct folio *folio = NULL; struct mempolicy *mpol; @@ -1326,13 +1326,6 @@ static struct folio *dequeue_hugetlb_folio_vma(struct hstate *h, nodemask_t *nodemask; int nid; - /* - * gbl_chg==1 means the allocation requires a new page that was not - * reserved before. Making sure there's at least one free page. - */ - if (gbl_chg && !available_huge_pages(h)) - goto err; - gfp_mask = htlb_alloc_mask(h); nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask); @@ -1350,9 +1343,6 @@ static struct folio *dequeue_hugetlb_folio_vma(struct hstate *h, mpol_cond_put(mpol); return folio; - -err: - return NULL; } #if defined(CONFIG_ARCH_HAS_GIGANTIC_PAGE) && defined(CONFIG_CONTIG_ALLOC) @@ -2922,12 +2912,17 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, goto out_uncharge_cgroup_reservation; spin_lock_irq(&hugetlb_lock); + /* - * glb_chg is passed to indicate whether or not a page must be taken - * from the global free pool (global change). gbl_chg == 0 indicates - * a reservation exists for the allocation. + * gbl_chg == 0 indicates a reservation exists for the + * allocation, so try dequeuing a page. In case there was no + * reservation, try dequeuing a page if there are available + * pages in the global pool. */ - folio = dequeue_hugetlb_folio_vma(h, vma, addr, gbl_chg); + folio = NULL; + if (!gbl_chg || available_huge_pages(h)) + folio = dequeue_hugetlb_folio_vma(h, vma, addr); + if (!folio) { spin_unlock_irq(&hugetlb_lock); folio = alloc_buddy_hugetlb_folio_with_mpol(h, vma, addr); From 66a4e9e11e43f19123777d0a1b419a2d9bcb9ce7 Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 2 Jul 2026 09:21:46 -0700 Subject: [PATCH 172/501] mm: hugetlb: move mpol interpretation out of alloc_buddy_hugetlb_folio_with_mpol() Move memory policy interpretation out of alloc_buddy_hugetlb_folio_with_mpol() and into alloc_hugetlb_folio() to separate reading and interpretation of memory policy from actual allocation. This will later allow memory policy to be interpreted outside of the process of allocating a hugetlb folio entirely. This opens doors for other callers of the HugeTLB folio allocation function, such as guest_memfd, where memory may not always be mapped and hence may not have an associated vma. Introduce struct mempolicy_interpreted to hold all the components of an interpreted memory policy. Rename alloc_buddy_hugetlb_folio_with_mpol() to alloc_buddy_hugetlb_folio() since the function no longer interprets memory policy. No functional change intended. Link: https://lore.kernel.org/20260702-hugetlb-open-up-v4-2-d53cefcccf34@google.com Signed-off-by: Ackerley Tng Reviewed-by: James Houghton Acked-by: Oscar Salvador Cc: Alistair Popple Cc: Byungchul Park Cc: David Hildenbrand Cc: David Rientjes Cc: "Edgecombe, Rick P" Cc: Frank van der Linden Cc: Gregory Price Cc: "Huang, Ying" Cc: Jason Gunthorpe Cc: Jiaqi Yan Cc: Joshua Hahn Cc: Matthew Brost Cc: Michael Roth Cc: Michal Hocko Cc: Muchun Song Cc: Paolo Bonzini Cc: Pasha Tatashin Cc: Peter Xu Cc: Pratyush Yadav Cc: Qi Zheng Cc: Rakie Kim Cc: Roman Gushchin Cc: Sean Christopherson Cc: Shakeel Butt Cc: Shivank Garg Cc: Vishal Annapurve Cc: Yan Zhao Cc: Zi Yan Signed-off-by: Andrew Morton --- include/uapi/linux/mempolicy.h | 2 +- mm/hugetlb.c | 54 ++++++++++++++++++++++------------ 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/include/uapi/linux/mempolicy.h b/include/uapi/linux/mempolicy.h index 6c962d866e86..7f6fc9599693 100644 --- a/include/uapi/linux/mempolicy.h +++ b/include/uapi/linux/mempolicy.h @@ -16,7 +16,7 @@ */ /* Policies */ -enum { +enum mempolicy_mode { MPOL_DEFAULT, MPOL_PREFERRED, MPOL_BIND, diff --git a/mm/hugetlb.c b/mm/hugetlb.c index dcf8df3d71b9..622ef772c0ea 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1316,6 +1316,12 @@ static unsigned long available_huge_pages(struct hstate *h) return h->free_huge_pages - h->resv_huge_pages; } +struct mempolicy_interpreted { + int nid; + nodemask_t *nodemask; + enum mempolicy_mode mode; +}; + static struct folio *dequeue_hugetlb_folio_vma(struct hstate *h, struct vm_area_struct *vma, unsigned long address) @@ -2137,32 +2143,28 @@ static struct folio *alloc_migrate_hugetlb_folio(struct hstate *h, gfp_t gfp_mas return folio; } -/* - * Use the VMA's mpolicy to allocate a huge page from the buddy. - */ static -struct folio *alloc_buddy_hugetlb_folio_with_mpol(struct hstate *h, - struct vm_area_struct *vma, unsigned long addr) +struct folio *alloc_buddy_hugetlb_folio(struct hstate *h, + gfp_t gfp_mask, struct mempolicy_interpreted *mpoli) { struct folio *folio = NULL; - struct mempolicy *mpol; - gfp_t gfp_mask = htlb_alloc_mask(h); - int nid; - nodemask_t *nodemask; + nodemask_t *nodemask = mpoli->nodemask; - nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask); - if (mpol_is_preferred_many(mpol)) { + if (mpoli->mode == MPOL_PREFERRED_MANY) { gfp_t gfp = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); - folio = alloc_surplus_hugetlb_folio(h, gfp, nid, nodemask); + folio = alloc_surplus_hugetlb_folio(h, gfp, mpoli->nid, + nodemask); /* Fallback to all nodes if page==NULL */ nodemask = NULL; } - if (!folio) - folio = alloc_surplus_hugetlb_folio(h, gfp_mask, nid, nodemask); - mpol_cond_put(mpol); + if (!folio) { + folio = alloc_surplus_hugetlb_folio(h, gfp_mask, mpoli->nid, + nodemask); + } + return folio; } @@ -2852,7 +2854,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, int ret, idx; struct hugetlb_cgroup *h_cg = NULL; struct hugetlb_cgroup *h_cg_rsvd = NULL; - gfp_t gfp = htlb_alloc_mask(h) | __GFP_RETRY_MAYFAIL; + gfp_t gfp = htlb_alloc_mask(h); idx = hstate_index(h); @@ -2924,8 +2926,24 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, folio = dequeue_hugetlb_folio_vma(h, vma, addr); if (!folio) { + struct mempolicy_interpreted mpoli; + struct mempolicy *mpol; + nodemask_t *nodemask; + int nid; + spin_unlock_irq(&hugetlb_lock); - folio = alloc_buddy_hugetlb_folio_with_mpol(h, vma, addr); + nid = huge_node(vma, addr, gfp, &mpol, &nodemask); + mpoli = (struct mempolicy_interpreted){ + .nid = nid, +#ifdef CONFIG_NUMA + .mode = mpol ? mpol->mode : MPOL_DEFAULT, +#else + .mode = MPOL_DEFAULT, +#endif + .nodemask = nodemask, + }; + folio = alloc_buddy_hugetlb_folio(h, gfp, &mpoli); + mpol_cond_put(mpol); if (!folio) goto out_uncharge_cgroup; spin_lock_irq(&hugetlb_lock); @@ -2981,7 +2999,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, } } - ret = mem_cgroup_charge_hugetlb(folio, gfp); + ret = mem_cgroup_charge_hugetlb(folio, gfp | __GFP_RETRY_MAYFAIL); /* * Unconditionally increment NR_HUGETLB here. If it turns out that * mem_cgroup_charge_hugetlb failed, then immediately free the page and From a492abe28bece84fd1a6b116220f3e685e91cef4 Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 2 Jul 2026 09:21:47 -0700 Subject: [PATCH 173/501] mm: hugetlb: move mpol interpretation out of dequeue_hugetlb_folio_vma() Move memory policy interpretation out of dequeue_hugetlb_folio_vma() and into alloc_hugetlb_folio() to separate reading and interpretation of memory policy from actual allocation. Also rename dequeue_hugetlb_folio_vma() to dequeue_hugetlb_folio_with_mpol() to remove association with vma and to align with alloc_buddy_hugetlb_folio_with_mpol(). This will later allow memory policy to be interpreted outside of the process of allocating a hugetlb folio entirely. This opens doors for other callers of the HugeTLB folio allocation function, such as guest_memfd, where memory may not always be mapped and hence may not have an associated vma. No functional change intended. Link: https://lore.kernel.org/20260702-hugetlb-open-up-v4-3-d53cefcccf34@google.com Signed-off-by: Ackerley Tng Reviewed-by: James Houghton Cc: Alistair Popple Cc: Byungchul Park Cc: David Hildenbrand Cc: David Rientjes Cc: "Edgecombe, Rick P" Cc: Frank van der Linden Cc: Gregory Price Cc: "Huang, Ying" Cc: Jason Gunthorpe Cc: Jiaqi Yan Cc: Joshua Hahn Cc: Matthew Brost Cc: Michael Roth Cc: Michal Hocko Cc: Muchun Song Cc: Oscar Salvador Cc: Paolo Bonzini Cc: Pasha Tatashin Cc: Peter Xu Cc: Pratyush Yadav Cc: Qi Zheng Cc: Rakie Kim Cc: Roman Gushchin Cc: Sean Christopherson Cc: Shakeel Butt Cc: Shivank Garg Cc: Vishal Annapurve Cc: Yan Zhao Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/hugetlb.c | 66 +++++++++++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 622ef772c0ea..96211fba26f7 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1322,32 +1322,26 @@ struct mempolicy_interpreted { enum mempolicy_mode mode; }; -static struct folio *dequeue_hugetlb_folio_vma(struct hstate *h, - struct vm_area_struct *vma, - unsigned long address) +static struct folio *dequeue_hugetlb_folio(struct hstate *h, gfp_t gfp_mask, + struct mempolicy_interpreted *mpoli) { + nodemask_t *nodemask = mpoli->nodemask; struct folio *folio = NULL; - struct mempolicy *mpol; - gfp_t gfp_mask; - nodemask_t *nodemask; - int nid; - gfp_mask = htlb_alloc_mask(h); - nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask); - - if (mpol_is_preferred_many(mpol)) { + if (mpoli->mode == MPOL_PREFERRED_MANY) { folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, - nid, nodemask); + mpoli->nid, + nodemask); /* Fallback to all nodes if page==NULL */ nodemask = NULL; } - if (!folio) + if (!folio) { folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, - nid, nodemask); - - mpol_cond_put(mpol); + mpoli->nid, + nodemask); + } return folio; } @@ -2854,7 +2848,11 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, int ret, idx; struct hugetlb_cgroup *h_cg = NULL; struct hugetlb_cgroup *h_cg_rsvd = NULL; + struct mempolicy_interpreted mpoli; gfp_t gfp = htlb_alloc_mask(h); + struct mempolicy *mpol; + nodemask_t *nodemask; + int nid; idx = hstate_index(h); @@ -2913,6 +2911,18 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, if (ret) goto out_uncharge_cgroup_reservation; + /* Takes reference on mpol. */ + nid = huge_node(vma, addr, gfp, &mpol, &nodemask); + mpoli = (struct mempolicy_interpreted){ + .nid = nid, +#ifdef CONFIG_NUMA + .mode = mpol ? mpol->mode : MPOL_DEFAULT, +#else + .mode = MPOL_DEFAULT, +#endif + .nodemask = nodemask, + }; + spin_lock_irq(&hugetlb_lock); /* @@ -2923,35 +2933,23 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, */ folio = NULL; if (!gbl_chg || available_huge_pages(h)) - folio = dequeue_hugetlb_folio_vma(h, vma, addr); + folio = dequeue_hugetlb_folio(h, gfp, &mpoli); if (!folio) { - struct mempolicy_interpreted mpoli; - struct mempolicy *mpol; - nodemask_t *nodemask; - int nid; - spin_unlock_irq(&hugetlb_lock); - nid = huge_node(vma, addr, gfp, &mpol, &nodemask); - mpoli = (struct mempolicy_interpreted){ - .nid = nid, -#ifdef CONFIG_NUMA - .mode = mpol ? mpol->mode : MPOL_DEFAULT, -#else - .mode = MPOL_DEFAULT, -#endif - .nodemask = nodemask, - }; folio = alloc_buddy_hugetlb_folio(h, gfp, &mpoli); - mpol_cond_put(mpol); - if (!folio) + if (!folio) { + mpol_cond_put(mpol); goto out_uncharge_cgroup; + } spin_lock_irq(&hugetlb_lock); list_add(&folio->lru, &h->hugepage_activelist); folio_ref_unfreeze(folio, 1); /* Fall through */ } + mpol_cond_put(mpol); + /* * Either dequeued or buddy-allocated folio needs to add special * mark to the folio when it consumes a global reservation. From 564b2eeeb9ad1a51e023ae946ed1c4c2bb4de493 Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 2 Jul 2026 09:21:48 -0700 Subject: [PATCH 174/501] mm: hugetlb: use error variable in alloc_hugetlb_folio Refactor alloc_hugetlb_folio to use a local variable for returning error codes. Instead of returning ERR_PTR(-ENOSPC) at the end of the error path, assign -ENOSPC to a return variable at each failure point and return that variable at the end. This allows the cleanup goto targets to be used with other errors in a later patch. No functional change intended. Link: https://lore.kernel.org/20260702-hugetlb-open-up-v4-4-d53cefcccf34@google.com Signed-off-by: Ackerley Tng Cc: Alistair Popple Cc: Byungchul Park Cc: David Hildenbrand Cc: David Rientjes Cc: "Edgecombe, Rick P" Cc: Frank van der Linden Cc: Gregory Price Cc: "Huang, Ying" Cc: James Houghton Cc: Jason Gunthorpe Cc: Jiaqi Yan Cc: Joshua Hahn Cc: Matthew Brost Cc: Michael Roth Cc: Michal Hocko Cc: Muchun Song Cc: Oscar Salvador Cc: Paolo Bonzini Cc: Pasha Tatashin Cc: Peter Xu Cc: Pratyush Yadav Cc: Qi Zheng Cc: Rakie Kim Cc: Roman Gushchin Cc: Sean Christopherson Cc: Shakeel Butt Cc: Shivank Garg Cc: Vishal Annapurve Cc: Yan Zhao Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/hugetlb.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 96211fba26f7..b8c23ac5582f 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2886,8 +2886,10 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, */ if (map_chg) { gbl_chg = hugepage_subpool_get_pages(spool, 1); - if (gbl_chg < 0) + if (gbl_chg < 0) { + ret = -ENOSPC; goto out_end_reservation; + } } else { /* * If we have the vma reservation ready, no need for extra @@ -2903,13 +2905,17 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, if (map_chg) { ret = hugetlb_cgroup_charge_cgroup_rsvd( idx, pages_per_huge_page(h), &h_cg_rsvd); - if (ret) + if (ret) { + ret = -ENOSPC; goto out_subpool_put; + } } ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg); - if (ret) + if (ret) { + ret = -ENOSPC; goto out_uncharge_cgroup_reservation; + } /* Takes reference on mpol. */ nid = huge_node(vma, addr, gfp, &mpol, &nodemask); @@ -2940,6 +2946,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, folio = alloc_buddy_hugetlb_folio(h, gfp, &mpoli); if (!folio) { mpol_cond_put(mpol); + ret = -ENOSPC; goto out_uncharge_cgroup; } spin_lock_irq(&hugetlb_lock); @@ -3032,7 +3039,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, out_end_reservation: if (map_chg != MAP_CHG_ENFORCED) vma_end_reservation(h, vma, addr); - return ERR_PTR(-ENOSPC); + return ERR_PTR(ret); } static __init void *alloc_bootmem(struct hstate *h, int nid, bool node_exact) From 177e1cbbb58128a6895e081531e7ba7360919b74 Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 2 Jul 2026 09:21:49 -0700 Subject: [PATCH 175/501] mm: hugetlb: move mem_cgroup_charge_hugetlb() earlier in allocation Move mem_cgroup_charge_hugetlb() earlier in the folio allocation process. This change draws a cleaner line between memcg charging and the subsequent hugetlb-specific reservation logic for VMAs and subpools. While it would be ideal to make all accounting and reservations perfectly symmetric, mem_cgroup_charge_hugetlb() is a complex operation that cannot be performed under the hugetlb_lock. Moving the charge to this earlier point ensures that memcg charging is handled before the code begins manipulating subpool and VMA-specific state. These two types of accounting will be separated in a future patch. If mem_cgroup_charge_hugetlb() fails, the code now branches to out_subpool_put to ensure the folio is freed and the subpool references are handled correctly. Link: https://lore.kernel.org/20260702-hugetlb-open-up-v4-5-d53cefcccf34@google.com Signed-off-by: Ackerley Tng Cc: Alistair Popple Cc: Byungchul Park Cc: David Hildenbrand Cc: David Rientjes Cc: "Edgecombe, Rick P" Cc: Frank van der Linden Cc: Gregory Price Cc: "Huang, Ying" Cc: James Houghton Cc: Jason Gunthorpe Cc: Jiaqi Yan Cc: Joshua Hahn Cc: Matthew Brost Cc: Michael Roth Cc: Michal Hocko Cc: Muchun Song Cc: Oscar Salvador Cc: Paolo Bonzini Cc: Pasha Tatashin Cc: Peter Xu Cc: Pratyush Yadav Cc: Qi Zheng Cc: Rakie Kim Cc: Roman Gushchin Cc: Sean Christopherson Cc: Shakeel Butt Cc: Shivank Garg Cc: Vishal Annapurve Cc: Yan Zhao Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/hugetlb.c | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index b8c23ac5582f..4d4430c1b3f4 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2977,6 +2977,24 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, spin_unlock_irq(&hugetlb_lock); + ret = mem_cgroup_charge_hugetlb(folio, gfp | __GFP_RETRY_MAYFAIL); + /* + * Unconditionally increment NR_HUGETLB here. If it turns out that + * mem_cgroup_charge_hugetlb failed, then immediately free the page and + * decrement NR_HUGETLB. + */ + lruvec_stat_mod_folio(folio, NR_HUGETLB, pages_per_huge_page(h)); + + if (ret == -ENOMEM) { + free_huge_folio(folio); + /* + * Skip uncharging hugetlb_cgroup since the charges + * were committed to the folio and freeing the folio + * would have cleared those up. + */ + goto out_subpool_put; + } + hugetlb_set_folio_subpool(folio, spool); if (map_chg != MAP_CHG_ENFORCED) { @@ -3004,19 +3022,6 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, } } - ret = mem_cgroup_charge_hugetlb(folio, gfp | __GFP_RETRY_MAYFAIL); - /* - * Unconditionally increment NR_HUGETLB here. If it turns out that - * mem_cgroup_charge_hugetlb failed, then immediately free the page and - * decrement NR_HUGETLB. - */ - lruvec_stat_mod_folio(folio, NR_HUGETLB, pages_per_huge_page(h)); - - if (ret == -ENOMEM) { - free_huge_folio(folio); - return ERR_PTR(-ENOMEM); - } - return folio; out_uncharge_cgroup: From 5737df3826dee4287f986dc2eaf7dc352068710d Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 2 Jul 2026 09:21:50 -0700 Subject: [PATCH 176/501] mm: hugetlb: refactor out hugetlb_alloc_folio() Refactor out hugetlb_alloc_folio() from alloc_hugetlb_folio(), which handles allocation of a folio and memory and HugeTLB charging to cgroups. This refactoring decouples the HugeTLB page allocation from VMAs, specifically: 1. Reservations (as in resv_map) are stored in the vma 2. mpol is stored at vma->vm_policy 3. A vma must be used for allocation even if the pages are not meant to be used by host process. Without this coupling, VMAs are no longer a requirement for allocation. This opens up the allocation routine for usage without VMAs, which will allow guest_memfd to use HugeTLB as a more generic allocator of huge pages, since guest_memfd memory may not have any associated VMAs by design. In addition, direct allocations from HugeTLB could possibly be refactored to avoid the use of a pseudo-VMA. Also, this decouples HugeTLB page allocation from HugeTLBfs, where the subpool is stored at the fs mount. This is also a requirement for guest_memfd, where the plan is to have a subpool created per-fd and stored on the inode. Provide and use alloc_flags to allow more allocation knobs in future without expanding the number of parameters in hugetlb_alloc_folio(). No functional change intended. Link: https://lore.kernel.org/20260702-hugetlb-open-up-v4-6-d53cefcccf34@google.com Signed-off-by: Ackerley Tng Cc: Alistair Popple Cc: Byungchul Park Cc: David Hildenbrand Cc: David Rientjes Cc: "Edgecombe, Rick P" Cc: Frank van der Linden Cc: Gregory Price Cc: "Huang, Ying" Cc: James Houghton Cc: Jason Gunthorpe Cc: Jiaqi Yan Cc: Joshua Hahn Cc: Matthew Brost Cc: Michael Roth Cc: Michal Hocko Cc: Muchun Song Cc: Oscar Salvador Cc: Paolo Bonzini Cc: Pasha Tatashin Cc: Peter Xu Cc: Pratyush Yadav Cc: Rakie Kim Cc: Roman Gushchin Cc: Sean Christopherson Cc: Shakeel Butt Cc: Shivank Garg Cc: Vishal Annapurve Cc: Yan Zhao Cc: Zi Yan Cc: Qi Zheng Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 18 ++++ mm/hugetlb.c | 206 ++++++++++++++++++++++------------------ 2 files changed, 131 insertions(+), 93 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 87c87f64bc05..817b8b4223fa 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -2,6 +2,7 @@ #ifndef _LINUX_HUGETLB_H #define _LINUX_HUGETLB_H +#include #include #include #include @@ -682,6 +683,23 @@ struct hstate { int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list); int replace_free_hugepage_folios(unsigned long start_pfn, unsigned long end_pfn); void wait_for_freed_hugetlb_folios(void); + +struct mempolicy_interpreted { + int nid; + nodemask_t *nodemask; + enum mempolicy_mode mode; +}; + +enum hugetlb_alloc_flag { + HUGETLB_ALLOC_CHARGE_CGROUP_RSVD_BIT = 0, + HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS_BIT, +}; + +#define HUGETLB_ALLOC_CHARG_CGROUP_RSVD BIT(HUGETLB_ALLOC_CHARGE_CGROUP_RSVD_BIT) +#define HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS BIT(HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS_BIT) + +struct folio *hugetlb_alloc_folio(struct hstate *h, + struct mempolicy_interpreted *mpoli, u8 alloc_flags); struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, unsigned long addr, bool cow_from_owner); struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 4d4430c1b3f4..c0bee548a243 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1316,12 +1316,6 @@ static unsigned long available_huge_pages(struct hstate *h) return h->free_huge_pages - h->resv_huge_pages; } -struct mempolicy_interpreted { - int nid; - nodemask_t *nodemask; - enum mempolicy_mode mode; -}; - static struct folio *dequeue_hugetlb_folio(struct hstate *h, gfp_t gfp_mask, struct mempolicy_interpreted *mpoli) { @@ -2811,6 +2805,104 @@ void wait_for_freed_hugetlb_folios(void) flush_work(&free_hpage_work); } +/** + * hugetlb_alloc_folio - Allocate a hugetlb folio. + * @h: Hugetlb state control block. + * @mpoli: Interpreted memory policy to use for allocation. + * @alloc_flags: Flags controlling the allocation behavior. + * + * Allocates a hugetlb folio and handles cgroup charging and global hstate + * reservations. + * + * Return: A pointer to the allocated folio, or an ERR_PTR on failure. + * -ENOSPC if cgroup charging fails or no folio is available. + * -ENOMEM if mem cgroup charging fails. + */ +struct folio *hugetlb_alloc_folio(struct hstate *h, + struct mempolicy_interpreted *mpoli, u8 alloc_flags) +{ + bool charge_hugetlb_cgroup_rsvd = alloc_flags & + HUGETLB_ALLOC_CHARG_CGROUP_RSVD; + bool use_global_reservation = alloc_flags & + HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS; + size_t nr_pages = pages_per_huge_page(h); + struct hugetlb_cgroup *h_cg_rsvd = NULL; + struct hugetlb_cgroup *h_cg = NULL; + gfp_t gfp = htlb_alloc_mask(h); + int idx = hstate_index(h); + struct folio *folio; + int ret; + + if (charge_hugetlb_cgroup_rsvd && + hugetlb_cgroup_charge_cgroup_rsvd(idx, nr_pages, &h_cg_rsvd)) + return ERR_PTR(-ENOSPC); + + if (hugetlb_cgroup_charge_cgroup(idx, nr_pages, &h_cg)) { + ret = -ENOSPC; + goto err_uncharge_hugetlb_cgroup_rsvd; + } + + spin_lock_irq(&hugetlb_lock); + + folio = NULL; + if (use_global_reservation || available_huge_pages(h)) + folio = dequeue_hugetlb_folio(h, gfp, mpoli); + + if (!folio) { + spin_unlock_irq(&hugetlb_lock); + folio = alloc_buddy_hugetlb_folio(h, gfp, mpoli); + if (!folio) { + ret = -ENOSPC; + goto err_uncharge_hugetlb_cgroup; + } + spin_lock_irq(&hugetlb_lock); + list_add(&folio->lru, &h->hugepage_activelist); + folio_ref_unfreeze(folio, 1); + } + + if (use_global_reservation) { + folio_set_hugetlb_restore_reserve(folio); + h->resv_huge_pages--; + } + + hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, folio); + + if (charge_hugetlb_cgroup_rsvd) { + hugetlb_cgroup_commit_charge_rsvd(idx, nr_pages, h_cg_rsvd, + folio); + } + + spin_unlock_irq(&hugetlb_lock); + + ret = mem_cgroup_charge_hugetlb(folio, gfp | __GFP_RETRY_MAYFAIL); + /* + * Unconditionally increment NR_HUGETLB here because if + * mem_cgroup_charge_hugetlb failed, freeing the page will + * decrement NR_HUGETLB. + */ + lruvec_stat_mod_folio(folio, NR_HUGETLB, nr_pages); + + if (ret == -ENOMEM) { + free_huge_folio(folio); + /* + * Skip uncharging hugetlb_cgroup since the charges + * were committed to the folio and freeing the folio + * would have cleared those up. + */ + return ERR_PTR(ret); + } + + return folio; + + err_uncharge_hugetlb_cgroup: + hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg); + err_uncharge_hugetlb_cgroup_rsvd: + if (charge_hugetlb_cgroup_rsvd) + hugetlb_cgroup_uncharge_cgroup_rsvd(idx, nr_pages, h_cg_rsvd); + + return ERR_PTR(ret); +} + typedef enum { /* * For either 0/1: we checked the per-vma resv map, and one resv @@ -2845,16 +2937,13 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, struct folio *folio; long retval, gbl_chg, gbl_reserve; map_chg_state map_chg; - int ret, idx; - struct hugetlb_cgroup *h_cg = NULL; - struct hugetlb_cgroup *h_cg_rsvd = NULL; struct mempolicy_interpreted mpoli; gfp_t gfp = htlb_alloc_mask(h); struct mempolicy *mpol; nodemask_t *nodemask; + u8 alloc_flags = 0; int nid; - - idx = hstate_index(h); + int ret; /* Whether we need a separate per-vma reservation? */ if (cow_from_owner) { @@ -2899,23 +2988,18 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, } /* - * If this allocation is not consuming a per-vma reservation, - * charge the hugetlb cgroup now. + * If allocation doesn't reuse a reservation in the resv_map, + * charge for the reservation. */ - if (map_chg) { - ret = hugetlb_cgroup_charge_cgroup_rsvd( - idx, pages_per_huge_page(h), &h_cg_rsvd); - if (ret) { - ret = -ENOSPC; - goto out_subpool_put; - } - } + if (map_chg != MAP_CHG_REUSE) + alloc_flags |= HUGETLB_ALLOC_CHARG_CGROUP_RSVD; - ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg); - if (ret) { - ret = -ENOSPC; - goto out_uncharge_cgroup_reservation; - } + /* + * gbl_chg == 0 indicates a reservation exists for this + * allocation, so try to use it. + */ + if (gbl_chg == 0) + alloc_flags |= HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS; /* Takes reference on mpol. */ nid = huge_node(vma, addr, gfp, &mpol, &nodemask); @@ -2929,69 +3013,12 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, .nodemask = nodemask, }; - spin_lock_irq(&hugetlb_lock); - - /* - * gbl_chg == 0 indicates a reservation exists for the - * allocation, so try dequeuing a page. In case there was no - * reservation, try dequeuing a page if there are available - * pages in the global pool. - */ - folio = NULL; - if (!gbl_chg || available_huge_pages(h)) - folio = dequeue_hugetlb_folio(h, gfp, &mpoli); - - if (!folio) { - spin_unlock_irq(&hugetlb_lock); - folio = alloc_buddy_hugetlb_folio(h, gfp, &mpoli); - if (!folio) { - mpol_cond_put(mpol); - ret = -ENOSPC; - goto out_uncharge_cgroup; - } - spin_lock_irq(&hugetlb_lock); - list_add(&folio->lru, &h->hugepage_activelist); - folio_ref_unfreeze(folio, 1); - /* Fall through */ - } + folio = hugetlb_alloc_folio(h, &mpoli, alloc_flags); mpol_cond_put(mpol); - /* - * Either dequeued or buddy-allocated folio needs to add special - * mark to the folio when it consumes a global reservation. - */ - if (!gbl_chg) { - folio_set_hugetlb_restore_reserve(folio); - h->resv_huge_pages--; - } - - hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, folio); - /* If allocation is not consuming a reservation, also store the - * hugetlb_cgroup pointer on the page. - */ - if (map_chg) { - hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h), - h_cg_rsvd, folio); - } - - spin_unlock_irq(&hugetlb_lock); - - ret = mem_cgroup_charge_hugetlb(folio, gfp | __GFP_RETRY_MAYFAIL); - /* - * Unconditionally increment NR_HUGETLB here. If it turns out that - * mem_cgroup_charge_hugetlb failed, then immediately free the page and - * decrement NR_HUGETLB. - */ - lruvec_stat_mod_folio(folio, NR_HUGETLB, pages_per_huge_page(h)); - - if (ret == -ENOMEM) { - free_huge_folio(folio); - /* - * Skip uncharging hugetlb_cgroup since the charges - * were committed to the folio and freeing the folio - * would have cleared those up. - */ + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); goto out_subpool_put; } @@ -3024,12 +3051,6 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, return folio; -out_uncharge_cgroup: - hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg); -out_uncharge_cgroup_reservation: - if (map_chg) - hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h), - h_cg_rsvd); out_subpool_put: /* * put page to subpool iff the quota of subpool's rsv_hpages is used @@ -3040,7 +3061,6 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, hugetlb_acct_memory(h, -gbl_reserve); } - out_end_reservation: if (map_chg != MAP_CHG_ENFORCED) vma_end_reservation(h, vma, addr); From 923a89a91a7e07b51faa7b7fc72b62c3ca77e683 Mon Sep 17 00:00:00 2001 From: Feng Tang Date: Thu, 2 Jul 2026 19:26:10 +0800 Subject: [PATCH 177/501] mm/vmalloc: add alignment info in warning print as possible failure reason When running 'fix_align_alloc_test' case of test_vmalloc module with command: insmod ./test_vmalloc.ko run_test_mask=64 It will fail, which is the expected result, as the case increment the alignment parameter gradually to 64bit limit. And the dmesg has warning msg: "vmalloc_test/0: vmalloc error: size 4096, vm_struct allocation failed, mode:0xdc0(GFP_KERNEL|__GFP_ZERO), nodemask=(null),cpuset=/,mems_allowed=0" It doesn't give the alignment info, which is the real reason for the failure (not the 'size'). Add alignment info to the warning print to give the necessary hint for possible failure reason, and the message will be: "vmalloc_test/0: vmalloc error: size 4096, align 0x800000000000, vm_struct allocation failed, mode:0xdc0(GFP_KERNEL|__GFP_ZERO), nodemask=(null),cpuset=/,mems_allowed=0" Link: https://lore.kernel.org/20260702112610.21589-1-feng.tang@linux.alibaba.com Signed-off-by: Feng Tang Reviewed-by: Uladzislau Rezki (Sony) Signed-off-by: Andrew Morton --- mm/vmalloc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 12f4a39fdd0b..1191cda3b4e8 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -4045,8 +4045,8 @@ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align, if (!area) { bool nofail = gfp_mask & __GFP_NOFAIL; warn_alloc(gfp_mask, NULL, - "vmalloc error: size %lu, vm_struct allocation failed%s", - size, (nofail) ? ". Retrying." : ""); + "vmalloc error: size %lu, align 0x%lx, vm_struct allocation failed%s", + size, align, (nofail) ? ". Retrying." : ""); if (nofail) { schedule_timeout_uninterruptible(1); goto again; From aa497a270f68bec8fa0d83b70608cddb0c7275ff Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 3 Jul 2026 10:06:01 -0700 Subject: [PATCH 178/501] mm/damon: add damon_region->last_probe_hits Patch series "mm/damon: provide pseudo moving sum probe_hits". Data attribute counters (probe_hits) of DAMON are managed in the classical way. The counter value is accumulated every sampling interval, gets the complete view at the end of the aggregation interval, and is reset when the next aggregation interval starts. Hence, the complete view can be retrieved only once per aggregation interval, which can be quite long. With the suggested intervals autotuning setup, it becomes 2-4 seconds in common real production systems. It can span up to 200 seconds in theory. This will restrict online monitoring use case of DAMON. Actually DAMON is already providing online monitoring of probe_hits. DAMON sysfs interface exposes the values via schemes tried regions directory files. However, due to the above mentioned limitation, it usually shows only partially accumulated hit counters and therefore not useful. DAMOS is not using probe_hits at the moment. In the future, using it can further strengthen DAMOS. However, a recommended setup of DAMOS is utilizing sampling/aggregation intervals auto-tuning, and having its own DAMOS apply_interval (1 second is mostly recommended). In the setup, DAMOS will nearly always show incompletely accumulated probe_hits, which will not really be useful. Data frequency counter (nr_accesses) of DAMON solves this problem using the pseudo moving sum value. The infrastructure is not limited to nr_accesses but general sampling based counters. Maintain and provide the pseudo moving sum of probe_hits similar to nr_accesses, using the infrastructure. Tests ===== On an idle system, I ran DAMON with an attribute probe filter for non-anonymous page, using DAMON user-space tool, damo [1], like below. $ sudo ./damo start --probe_filter allow non anon Because the system is idle, nearly all memory is not an anonymous page but a free page, so the probe_hits are expected to be nearly always full. In this setup, since the sampling interval is 5ms and the aggregation interval is 100ms, the counter value is expected to always be near 20. On kernels not having this series, if we retrieve the probe hits in an arbitrary time that is likely not aligned to the aggregation interval, the values are usually much lower than the expectation like below. This is because the tool is showing the incompletely aggregated values. $ sudo ./damo report access --format append region "probe_hits: " heatmap: 00000000000000000000000000000000000000008999999711111111000000000000000000000000 # min/max temperatures: -1,630,000,000, 0, column size: 99.800 MiB intervals: sample 5 ms aggr 100 ms (max access hz 200) 0 addr 4.000 KiB size 3.898 GiB access 0 hz age 16.300 s probe_hits: 11 1 addr 3.898 GiB size 77.859 MiB access 0 hz age 1.500 s probe_hits: 11 2 addr 3.974 GiB size 700.770 MiB access 0 hz age 0 ns probe_hits: 11 3 addr 4.659 GiB size 791.078 MiB access 0 hz age 13.700 s probe_hits: 11 4 addr 5.431 GiB size 1.472 GiB access 0 hz age 15.800 s probe_hits: 11 5 addr 6.903 GiB size 915.059 MiB access 0 hz age 15.300 s probe_hits: 11 memory bw estimate: 0 B per second total size: 7.797 GiB record DAMON intervals: sample 5 ms, aggr 100 ms After applying this series, I was able to reliably show the expected results like below. $ sudo ./damo report access --format append region "probe_hits: " heatmap: 00000000333333330000000166666665111111139999999855555555333333333333333444444444 intervals: sample 5 ms aggr 100 ms (max access hz 200) 0 addr 4.000 KiB size 790.496 MiB access 0 hz age 1 m 33.300 s probe_hits: 20 1 addr 790.500 MiB size 791.160 MiB access 0 hz age 1 m 15.400 s probe_hits: 19 2 addr 1.545 GiB size 792.316 MiB access 0 hz age 1 m 32.400 s probe_hits: 19 3 addr 2.318 GiB size 795.465 MiB access 0 hz age 1 m 2.600 s probe_hits: 19 4 addr 3.095 GiB size 797.102 MiB access 0 hz age 1 m 23.500 s probe_hits: 20 5 addr 3.874 GiB size 797.293 MiB access 0 hz age 47.900 s probe_hits: 20 6 addr 4.652 GiB size 787.516 MiB access 0 hz age 1 m 3.800 s probe_hits: 20 7 addr 5.421 GiB size 784.461 MiB access 0 hz age 1 m 14.400 s probe_hits: 19 8 addr 6.187 GiB size 795.621 MiB access 0 hz age 1 m 15.700 s probe_hits: 20 9 addr 6.964 GiB size 798.000 MiB access 0 hz age 1 m 10.200 s probe_hits: 20 10 addr 7.744 GiB size 54.566 MiB access 0 hz age 1 m 9.300 s probe_hits: 20 memory bw estimate: 0 B per second total size: 7.797 GiB record DAMON intervals: sample 5 ms, aggr 100 ms FYI, 'damo report access' output format has changed on v3.3.0. Above outputs can be reproduced on <3.3.0 versions of damo. Patches Sequence ================ Patch 1 adds probe_hits counters for values that fully accumulated in the last aggregation interval. This is required for using the moving sum infrastructure. Patch 2 introduces a function for getting the moving sum values on demand, using the infrastructure. Finally, patch 3 updates the DAMON sysfs interface to expose the moving sum values to the schemes tried regions directory. This patch (of 3): Add new damon_region filed, last_probe_hits. Maintain fully accumulated probe_hits values from the last aggregation interval in the field. Link: https://lore.kernel.org/20260703170605.94472-1-sj@kernel.org Link: https://lore.kernel.org/20260703170605.94472-2-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 2 +- mm/damon/core.c | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 805e089ff4f2..4f7e305b261c 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -68,10 +68,10 @@ struct damon_region { unsigned int nr_accesses; unsigned char probe_hits[DAMON_MAX_PROBES]; struct list_head list; - unsigned int age; /* private: Internal value for age calculation. */ unsigned int last_nr_accesses; + unsigned char last_probe_hits[DAMON_MAX_PROBES]; }; /** diff --git a/mm/damon/core.c b/mm/damon/core.c index b2fc15a3804f..d45dc87fbbd9 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -302,8 +302,10 @@ struct damon_region *damon_new_region(unsigned long start, unsigned long end) region->ar.start = start; region->ar.end = end; region->nr_accesses = 0; - for (i = 0; i < DAMON_MAX_PROBES; i++) + for (i = 0; i < DAMON_MAX_PROBES; i++) { region->probe_hits[i] = 0; + region->last_probe_hits[i] = 0; + } INIT_LIST_HEAD(®ion->list); region->age = 0; @@ -2047,8 +2049,10 @@ static void kdamond_reset_aggregated(struct damon_ctx *c) damon_nr_regions(t), nr_probes); r->last_nr_accesses = r->nr_accesses; r->nr_accesses = 0; - for (i = 0; i < DAMON_MAX_PROBES; i++) + for (i = 0; i < DAMON_MAX_PROBES; i++) { + r->last_probe_hits[i] = r->probe_hits[i]; r->probe_hits[i] = 0; + } } ti++; } @@ -3239,6 +3243,8 @@ static void damon_split_region_at(struct damon_target *t, new->nr_accesses = r->nr_accesses; /* todo: do this for only installed probes */ memcpy(new->probe_hits, r->probe_hits, sizeof(r->probe_hits)); + memcpy(new->last_probe_hits, r->last_probe_hits, + sizeof(r->last_probe_hits)); damon_insert_region(new, r, damon_next_region(r), t); } From 2ec26dd341d123be8a5577207f25dbecf164e220 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 3 Jul 2026 10:06:02 -0700 Subject: [PATCH 179/501] mm/damon/core: introduce damon_probe_hits_mvsum() Implement a function for getting a reasonable best effort quality pseudo moving sums of probe_hits on demands. It reuses the internal function for the pseudo moving sum for data access frequency (nr_accesses). Link: https://lore.kernel.org/20260703170605.94472-3-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 2 ++ mm/damon/core.c | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/include/linux/damon.h b/include/linux/damon.h index 4f7e305b261c..616bdf0954b5 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -1003,6 +1003,8 @@ void damon_add_probe(struct damon_ctx *ctx, struct damon_probe *probe); struct damon_region *damon_new_region(unsigned long start, unsigned long end); unsigned int damon_nr_accesses_mvsum(struct damon_region *r, struct damon_ctx *ctx); +unsigned char damon_probe_hits_mvsum(int probe_idx, struct damon_region *r, + struct damon_ctx *ctx); int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, unsigned int nr_ranges, unsigned long min_region_sz); diff --git a/mm/damon/core.c b/mm/damon/core.c index d45dc87fbbd9..871c6f5257c9 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -273,6 +273,27 @@ unsigned int damon_nr_accesses_mvsum(struct damon_region *r, left_window_bp); } +unsigned char damon_probe_hits_mvsum(int probe_idx, struct damon_region *r, + struct damon_ctx *ctx) +{ + unsigned long sample_interval, aggr_interval; + unsigned long window_len, left_window, left_window_bp; + + sample_interval = ctx->attrs.sample_interval ? : 1; + aggr_interval = ctx->attrs.aggr_interval ? : 1; + window_len = aggr_interval / sample_interval; + if (time_after_eq(ctx->passed_sample_intervals, + ctx->next_aggregation_sis)) + left_window = 0; + else + left_window = ctx->next_aggregation_sis - + ctx->passed_sample_intervals; + left_window_bp = mult_frac(left_window, 10000, window_len); + + return damon_mvsum(r->probe_hits[probe_idx], + r->last_probe_hits[probe_idx], left_window_bp); +} + #ifdef CONFIG_DAMON_DEBUG_SANITY static void damon_verify_new_region(unsigned long start, unsigned long end) { From a9298dd6693355d55c82be64266f92b18a0fea90 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 3 Jul 2026 10:06:03 -0700 Subject: [PATCH 180/501] mm/damon/sysfs-schemes: set probe hits as pseudo moving sums DAMON sysfs interface exposes damon_region->probe_hits via probe hit files. Because the counters are completed only at the end of the aggregation interval, users can show incomplete values if they requested the file content update (update_schemes_tried_regions command) in the middle of an aggregation interval. Set the value as the pseudo moving sum value of the counter, similar to that for nr_accesses. Link: https://lore.kernel.org/20260703170605.94472-4-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/sysfs-schemes.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index dbf2b0515d58..32f495a96b17 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -110,7 +110,8 @@ static int damos_sysfs_probes_add_dirs(struct damos_sysfs_probes *probes, struct damos_sysfs_probe *sys_probe; int err; - sys_probe = damos_sysfs_probe_alloc(region->probe_hits[i]); + sys_probe = damos_sysfs_probe_alloc( + damon_probe_hits_mvsum(i, region, ctx)); if (!sys_probe) { damos_sysfs_probes_rm_dirs(probes); return -ENOMEM; From 03eaf4c445122993c162fe489396b8e704f47d67 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Fri, 3 Jul 2026 12:31:41 +0000 Subject: [PATCH 181/501] mm/page_alloc: rename ALLOC_TRYLOCK -> ALLOC_NOLOCK Patch series "mm: Some cleanups for page allocator APIs", v5. Some tweaks and cleanups for page allocator entrypoint and flags. This is motivated by preparation for __GFP_UNMAPPED [1] (which will probably become ALLOC_UNMAPPED in its next iteration), but all this is supposed to be an improvement to the codebase in its own right: unifying code paths, reducing API surface, and removing GFP flags. This started with unifying __alloc_frozen_pages[_nolock]_noprof() and expanded from there. Unifying the nolock allocator entrypoint with the normal allocator entrypoint means adding an alloc_flags argument to the later (only exposed within mm/). This presents an opportunity to take advantage of that arg to remove some GFP flags, if we add that alloc_flags arg a bit more broadly to allocator entrypoints. To distinguish between mm-internal and "public" allocator entrypoints, it makes sense to use the __ prefix. There are already some public APIs with that prefix. For *alloc_pages*, just removing those variants seems like a nice cleanup anyway, so do that. For get_free_pages, the "__" variant is the _only_ variant and it's very widely used, so it doesn't seem worthwhile to modify that. Therefore, scope this "__" change specifically to the *alloc_pages* API, which means we leave the *folio_alloc* API untouched too, even though that could probably be cleaned up if so desired. This patch (of 18): It's confusing that the function is called "nolock" but the flag is called "trylock", align them. The function's terminology is more visible and has more mindshare so use that. Link: https://lore.kernel.org/20260703-alloc-trylock-v5-0-c87b714e19d3@google.com Link: https://lore.kernel.org/20260703-alloc-trylock-v5-1-c87b714e19d3@google.com Link: https://lore.kernel.org/linux-mm/2399b3ad-4eac-4a14-94c3-27e9f07972a1@kernel.org/ Link: https://lore.kernel.org/all/20260320-page_alloc-unmapped-v2-0-28bf1bd54f41@google.com/ [1] Signed-off-by: Brendan Jackman Suggested-by: "Vlastimil Babka (SUSE)" Reviewed-by: Suren Baghdasaryan Reviewed-by: Harry Yoo (Oracle) Reviewed-by: Vlastimil Babka (SUSE) Acked-by: Zi Yan Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Alexei Starovoitov Cc: Alistair Popple Cc: Arnaldo Carvalho de Melo Cc: Arnd Bergmann Cc: "Borislav Petkov (AMD)" Cc: Byungchul Park Cc: David Hildenbrand Cc: David Rientjes Cc: David S. Miller Cc: Dimitris Michailidis Cc: Eric Dumazet Cc: Greg Kroah-Hartman Cc: Gregory Price Cc: Harry Yoo Cc: "H. Peter Anvin" Cc: "Huang, Ying" Cc: Ian Rogers Cc: Ingo Molnar Cc: Jakub Kacinski Cc: James Clark Cc: Jiri Olsa Cc: Johannes Weiner Cc: Joshua Hahn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Marc Rutland Cc: Matthew Brost Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Mike Rapoport Cc: Muchun Song Cc: Namhyung kim Cc: Oscar Salvador Cc: Paolo Abeni Cc: Paolo Bonzini Cc: Peter Zijlstra Cc: Rakie Kim Cc: Reiji Watanabe Cc: Robin Holt Cc: Roman Gushchin Cc: Sean Christopherson Cc: Sebastian Andrzej Siewior Cc: Steven Rostedt Cc: Steve Wahl Cc: Suren Baghdasaryan Signed-off-by: Andrew Morton --- mm/internal.h | 2 +- mm/page_alloc.c | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index 0360648e5726..812ff7f0f450 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1480,7 +1480,7 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone, #define ALLOC_NOFRAGMENT 0x0 #endif #define ALLOC_HIGHATOMIC 0x200 /* Allows access to MIGRATE_HIGHATOMIC */ -#define ALLOC_TRYLOCK 0x400 /* Only use spin_trylock in allocation path */ +#define ALLOC_NOLOCK 0x400 /* Only use spin_trylock in allocation path */ #define ALLOC_KSWAPD 0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */ /* Flags that allow allocations below the min watermark. */ diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 762d9b6bc792..6004fe6583d4 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2530,7 +2530,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, unsigned long flags; int i; - if (unlikely(alloc_flags & ALLOC_TRYLOCK)) { + if (unlikely(alloc_flags & ALLOC_NOLOCK)) { if (!spin_trylock_irqsave(&zone->lock, flags)) return 0; } else { @@ -3218,7 +3218,7 @@ struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone, do { page = NULL; - if (unlikely(alloc_flags & ALLOC_TRYLOCK)) { + if (unlikely(alloc_flags & ALLOC_NOLOCK)) { if (!spin_trylock_irqsave(&zone->lock, flags)) return NULL; } else { @@ -5059,7 +5059,7 @@ static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, * Don't invoke should_fail logic, since it may call * get_random_u32() and printk() which need to spin_lock. */ - if (!(*alloc_flags & ALLOC_TRYLOCK) && + if (!(*alloc_flags & ALLOC_NOLOCK) && should_fail_alloc_page(gfp_mask, order)) return false; @@ -7804,7 +7804,7 @@ static bool cond_accept_memory(struct zone *zone, unsigned int order, return false; /* Bailout, since try_to_accept_memory_one() needs to take a lock */ - if (alloc_flags & ALLOC_TRYLOCK) + if (alloc_flags & ALLOC_NOLOCK) return false; wmark = promo_wmark_pages(zone); @@ -7896,7 +7896,7 @@ struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned */ gfp_t alloc_gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC | __GFP_COMP | gfp_flags; - unsigned int alloc_flags = ALLOC_TRYLOCK; + unsigned int alloc_flags = ALLOC_NOLOCK; struct alloc_context ac = { }; struct page *page; From 4b2b50a9f899725f4434eb0ea6d508346e82b0d7 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Fri, 3 Jul 2026 12:31:42 +0000 Subject: [PATCH 182/501] mm/page_alloc: some renames to clarify alloc_flags scopes It's pretty confusing that: - The slowpath and fastpath have a totally distinct set of alloc_flags. - gfp_to_alloc_flags() sounds generic but it only influences the slowpath. Rename some variables to highlight which alloc_flags are fastpath-specific. Rename gfp_to_alloc_flags() to highlight that it's slowpath-specific. gfp_to_alloc_flags_cma() and gfp_to_alloc_flags_nonblocking() currently have perfectly harmless names, but to keep the naming consistent also rename those to the alloc_flags_*() pattern (which already exists for alloc_flags_nofragment()). Link: https://lore.kernel.org/20260703-alloc-trylock-v5-2-c87b714e19d3@google.com Signed-off-by: Brendan Jackman Reviewed-by: Vlastimil Babka (SUSE) Acked-by: JP Kobryn Reviewed-by: Zi Yan Signed-off-by: Andrew Morton --- include/linux/skbuff.h | 2 +- mm/page_alloc.c | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 22eda1d54a0e..4431b026e429 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3573,7 +3573,7 @@ static inline struct page *__dev_alloc_pages_noprof(gfp_t gfp_mask, * 3. If requesting a order 0 page it will not be compound * due to the check to see if order has a value in prep_new_page * 4. __GFP_MEMALLOC is ignored if __GFP_NOMEMALLOC is set due to - * code in gfp_to_alloc_flags that should be enforcing this. + * code in alloc_flags_slowpath() that should be enforcing this. */ gfp_mask |= __GFP_COMP | __GFP_MEMALLOC; diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 6004fe6583d4..df1345cde301 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -3774,8 +3774,8 @@ alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) } /* Must be called after current_gfp_context() which can change gfp_mask */ -static inline unsigned int gfp_to_alloc_flags_cma(gfp_t gfp_mask, - unsigned int alloc_flags) +static inline unsigned int alloc_flags_cma(gfp_t gfp_mask, + unsigned int alloc_flags) { #ifdef CONFIG_CMA if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) @@ -4474,7 +4474,7 @@ static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, } static inline unsigned int -gfp_to_alloc_flags_nonblocking(gfp_t gfp_mask, unsigned int order) +alloc_flags_nonblocking(gfp_t gfp_mask, unsigned int order) { unsigned int alloc_flags = 0; @@ -4497,7 +4497,7 @@ gfp_to_alloc_flags_nonblocking(gfp_t gfp_mask, unsigned int order) } static inline unsigned int -gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) +alloc_flags_slowpath(gfp_t gfp_mask, unsigned int order) { unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; @@ -4512,7 +4512,7 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) if (gfp_mask & __GFP_KSWAPD_RECLAIM) alloc_flags |= ALLOC_KSWAPD; - alloc_flags |= gfp_to_alloc_flags_nonblocking(gfp_mask, order); + alloc_flags |= alloc_flags_nonblocking(gfp_mask, order); if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) { /* @@ -4525,7 +4525,7 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) } else if (unlikely(rt_or_dl_task(current)) && in_task()) alloc_flags |= ALLOC_MIN_RESERVE; - alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, alloc_flags); + alloc_flags = alloc_flags_cma(gfp_mask, alloc_flags); if (defrag_mode) alloc_flags |= ALLOC_NOFRAGMENT; @@ -4791,7 +4791,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, * kswapd needs to be woken up, and to avoid the cost of setting up * alloc_flags precisely. So we do that now. */ - alloc_flags = gfp_to_alloc_flags(gfp_mask, order); + alloc_flags = alloc_flags_slowpath(gfp_mask, order); /* * We need to recalculate the starting point for the zonelist iterator @@ -4832,7 +4832,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); if (reserve_flags) - alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, reserve_flags) | + alloc_flags = alloc_flags_cma(gfp_mask, reserve_flags) | (alloc_flags & ALLOC_KSWAPD); /* @@ -5063,7 +5063,7 @@ static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, should_fail_alloc_page(gfp_mask, order)) return false; - *alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, *alloc_flags); + *alloc_flags = alloc_flags_cma(gfp_mask, *alloc_flags); /* Dirty zone balancing only done in the fast path */ ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); @@ -5277,7 +5277,7 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, int preferred_nid, nodemask_t *nodemask) { struct page *page; - unsigned int alloc_flags = ALLOC_WMARK_LOW; + unsigned int fastpath_alloc_flags = ALLOC_WMARK_LOW; gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */ struct alloc_context ac = { }; @@ -5299,18 +5299,18 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, gfp = current_gfp_context(gfp); alloc_gfp = gfp; if (!prepare_alloc_pages(gfp, order, preferred_nid, nodemask, &ac, - &alloc_gfp, &alloc_flags)) + &alloc_gfp, &fastpath_alloc_flags)) return NULL; /* * Forbid the first pass from falling back to types that fragment * memory until all local zones are considered. */ - alloc_flags |= alloc_flags_nofragment(zonelist_zone(ac.preferred_zoneref), gfp); - alloc_flags |= gfp_to_alloc_flags_nonblocking(gfp, order) & ALLOC_HIGHATOMIC; + fastpath_alloc_flags |= alloc_flags_nofragment(zonelist_zone(ac.preferred_zoneref), gfp); + fastpath_alloc_flags |= alloc_flags_nonblocking(gfp, order) & ALLOC_HIGHATOMIC; /* First allocation attempt */ - page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac); + page = get_page_from_freelist(alloc_gfp, order, fastpath_alloc_flags, &ac); if (likely(page)) goto out; From 2666382a4f075bc3e322abb4a690910efbd2efac Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Fri, 3 Jul 2026 12:31:43 +0000 Subject: [PATCH 183/501] mm: name some args in a function declaration Checkpatch complains about this, a later patch will move the code, fix it so that checkpatch doesn't complain about that patch. Do it in a separate patch so the "move the code" patch is trivial to review using Git's diff colouring. Link: https://lore.kernel.org/20260703-alloc-trylock-v5-3-c87b714e19d3@google.com Signed-off-by: Brendan Jackman Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Signed-off-by: Andrew Morton --- mm/internal.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index 812ff7f0f450..4d045653e5e8 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -919,8 +919,8 @@ extern bool free_pages_prepare(struct page *page, unsigned int order); extern int user_min_free_kbytes; -struct page *__alloc_frozen_pages_noprof(gfp_t, unsigned int order, int nid, - nodemask_t *); +struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, int nid, + nodemask_t *nodemask); #define __alloc_frozen_pages(...) \ alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__)) void free_frozen_pages(struct page *page, unsigned int order); From d0850de699b0cb7b02225747ae364f68f0bf86e1 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Fri, 3 Jul 2026 12:31:44 +0000 Subject: [PATCH 184/501] mm: split out internal page_alloc.h internal.h is a bit bloated, seems like time for a page_alloc.h. Where it wasn't obvious, the heuristic for deciding what goes into this new header was "does it support/correspond to a definition in mm/page_alloc.c?" Only need to include it from ~20 .c files out of ~150 so this does seem like a genuine reduction in scopes, which is nice. And there's no circular internal.h<->page_alloc.h dependency, so it seems worthwhile to split this up before that inevitably emerges! Link: https://lore.kernel.org/20260703-alloc-trylock-v5-4-c87b714e19d3@google.com Signed-off-by: Brendan Jackman Suggested-by: "David Hildenbrand (Arm)" Link: https://lore.kernel.org/all/41e92bab-6882-401a-8de9-154adbdcfb36@kernel.org/ Reviewed-by: Vlastimil Babka (SUSE) Acked-by: Zi Yan Signed-off-by: Andrew Morton --- MAINTAINERS | 1 + mm/compaction.c | 1 + mm/hugetlb.c | 1 + mm/internal.h | 252 ---------------------------------------- mm/khugepaged.c | 1 + mm/kmsan/init.c | 2 +- mm/memory-failure.c | 1 + mm/memory_hotplug.c | 1 + mm/mempolicy.c | 1 + mm/migrate.c | 1 + mm/mm_init.c | 1 + mm/page_alloc.c | 1 + mm/page_alloc.h | 269 +++++++++++++++++++++++++++++++++++++++++++ mm/page_frag_cache.c | 2 +- mm/page_isolation.c | 1 + mm/page_owner.c | 2 +- mm/page_reporting.c | 1 + mm/show_mem.c | 1 + mm/shuffle.c | 1 + mm/slub.c | 1 + mm/swap.c | 1 + mm/vmscan.c | 1 + 22 files changed, 289 insertions(+), 255 deletions(-) create mode 100644 mm/page_alloc.h diff --git a/MAINTAINERS b/MAINTAINERS index 8ee9446da9df..e55743fff56c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17173,6 +17173,7 @@ F: mm/debug_page_alloc.c F: mm/debug_page_ref.c F: mm/fail_page_alloc.c F: mm/page_alloc.c +F: mm/page_alloc.h F: mm/page_ext.c F: mm/page_frag_cache.c F: mm/page_isolation.c diff --git a/mm/compaction.c b/mm/compaction.c index f08765ade014..7d80735502d9 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -24,6 +24,7 @@ #include #include #include +#include "page_alloc.h" #include "internal.h" #ifdef CONFIG_COMPACTION diff --git a/mm/hugetlb.c b/mm/hugetlb.c index c0bee548a243..8ce6fe2a9a72 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -47,6 +47,7 @@ #include #include #include "internal.h" +#include "page_alloc.h" #include "hugetlb_vmemmap.h" #include "hugetlb_cma.h" #include "hugetlb_internal.h" diff --git a/mm/internal.h b/mm/internal.h index 4d045653e5e8..ffa72bd31708 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -658,165 +658,6 @@ extern int defrag_mode; void setup_per_zone_wmarks(void); void calculate_min_free_kbytes(void); int __meminit init_per_zone_wmark_min(void); -void page_alloc_sysctl_init(void); - -/* - * Structure for holding the mostly immutable allocation parameters passed - * between functions involved in allocations, including the alloc_pages* - * family of functions. - * - * nodemask, migratetype and highest_zoneidx are initialized only once in - * __alloc_pages() and then never change. - * - * zonelist, preferred_zone and highest_zoneidx are set first in - * __alloc_pages() for the fast path, and might be later changed - * in __alloc_pages_slowpath(). All other functions pass the whole structure - * by a const pointer. - */ -struct alloc_context { - struct zonelist *zonelist; - const nodemask_t *nodemask; - struct zoneref *preferred_zoneref; - int migratetype; - - /* - * highest_zoneidx represents highest usable zone index of - * the allocation request. Due to the nature of the zone, - * memory on lower zone than the highest_zoneidx will be - * protected by lowmem_reserve[highest_zoneidx]. - * - * highest_zoneidx is also used by reclaim/compaction to limit - * the target zone since higher zone than this index cannot be - * usable for this allocation request. - */ - enum zone_type highest_zoneidx; - bool spread_dirty_pages; -}; - -/* - * This function returns the order of a free page in the buddy system. In - * general, page_zone(page)->lock must be held by the caller to prevent the - * page from being allocated in parallel and returning garbage as the order. - * If a caller does not hold page_zone(page)->lock, it must guarantee that the - * page cannot be allocated or merged in parallel. Alternatively, it must - * handle invalid values gracefully, and use buddy_order_unsafe() below. - */ -static inline unsigned int buddy_order(struct page *page) -{ - /* PageBuddy() must be checked by the caller */ - return page_private(page); -} - -/* - * Like buddy_order(), but for callers who cannot afford to hold the zone lock. - * PageBuddy() should be checked first by the caller to minimize race window, - * and invalid values must be handled gracefully. - * - * READ_ONCE is used so that if the caller assigns the result into a local - * variable and e.g. tests it for valid range before using, the compiler cannot - * decide to remove the variable and inline the page_private(page) multiple - * times, potentially observing different values in the tests and the actual - * use of the result. - */ -#define buddy_order_unsafe(page) READ_ONCE(page_private(page)) - -/* - * This function checks whether a page is free && is the buddy - * we can coalesce a page and its buddy if - * (a) the buddy is not in a hole (check before calling!) && - * (b) the buddy is in the buddy system && - * (c) a page and its buddy have the same order && - * (d) a page and its buddy are in the same zone. - * - * For recording whether a page is in the buddy system, we set PageBuddy. - * Setting, clearing, and testing PageBuddy is serialized by zone->lock. - * - * For recording page's order, we use page_private(page). - */ -static inline bool page_is_buddy(struct page *page, struct page *buddy, - unsigned int order) -{ - if (!page_is_guard(buddy) && !PageBuddy(buddy)) - return false; - - if (buddy_order(buddy) != order) - return false; - - /* - * zone check is done late to avoid uselessly calculating - * zone/node ids for pages that could never merge. - */ - if (page_zone_id(page) != page_zone_id(buddy)) - return false; - - VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); - - return true; -} - -/* - * Locate the struct page for both the matching buddy in our - * pair (buddy1) and the combined O(n+1) page they form (page). - * - * 1) Any buddy B1 will have an order O twin B2 which satisfies - * the following equation: - * B2 = B1 ^ (1 << O) - * For example, if the starting buddy (buddy2) is #8 its order - * 1 buddy is #10: - * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10 - * - * 2) Any buddy B will have an order O+1 parent P which - * satisfies the following equation: - * P = B & ~(1 << O) - * - * Assumption: *_mem_map is contiguous at least up to MAX_PAGE_ORDER - */ -static inline unsigned long -__find_buddy_pfn(unsigned long page_pfn, unsigned int order) -{ - return page_pfn ^ (1 << order); -} - -/* - * Find the buddy of @page and validate it. - * @page: The input page - * @pfn: The pfn of the page, it saves a call to page_to_pfn() when the - * function is used in the performance-critical __free_one_page(). - * @order: The order of the page - * @buddy_pfn: The output pointer to the buddy pfn, it also saves a call to - * page_to_pfn(). - * - * The found buddy can be a non PageBuddy, out of @page's zone, or its order is - * not the same as @page. The validation is necessary before use it. - * - * Return: the found buddy page or NULL if not found. - */ -static inline struct page *find_buddy_page_pfn(struct page *page, - unsigned long pfn, unsigned int order, unsigned long *buddy_pfn) -{ - unsigned long __buddy_pfn = __find_buddy_pfn(pfn, order); - struct page *buddy; - - buddy = page + (__buddy_pfn - pfn); - if (buddy_pfn) - *buddy_pfn = __buddy_pfn; - - if (page_is_buddy(page, buddy, order)) - return buddy; - return NULL; -} - -extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn, - unsigned long end_pfn, struct zone *zone); - -static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn, - unsigned long end_pfn, struct zone *zone) -{ - if (zone->contiguous) - return pfn_to_page(start_pfn); - - return __pageblock_pfn_to_page(start_pfn, end_pfn, zone); -} void set_zone_contiguous(struct zone *zone); bool pfn_range_intersects_zones(int nid, unsigned long start_pfn, @@ -831,8 +672,6 @@ extern int __isolate_free_page(struct page *page, unsigned int order); extern void __putback_isolated_page(struct page *page, unsigned int order, int mt); extern void memblock_free_pages(unsigned long pfn, unsigned int order); -extern void __free_pages_core(struct page *page, unsigned int order, - enum meminit_context context); /* * This will have no effect, other than possibly generating a warning, if the @@ -914,40 +753,6 @@ static inline void init_compound_tail(struct page *tail, prep_compound_tail(tail, head, order); } -void post_alloc_hook(struct page *page, unsigned int order, gfp_t gfp_flags); -extern bool free_pages_prepare(struct page *page, unsigned int order); - -extern int user_min_free_kbytes; - -struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, int nid, - nodemask_t *nodemask); -#define __alloc_frozen_pages(...) \ - alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__)) -void free_frozen_pages(struct page *page, unsigned int order); -void free_unref_folios(struct folio_batch *fbatch); - -#ifdef CONFIG_NUMA -struct page *alloc_frozen_pages_noprof(gfp_t, unsigned int order); -#else -static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order) -{ - return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL); -} -#endif - -#define alloc_frozen_pages(...) \ - alloc_hooks(alloc_frozen_pages_noprof(__VA_ARGS__)) - -struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order); -#define alloc_frozen_pages_nolock(...) \ - alloc_hooks(alloc_frozen_pages_nolock_noprof(__VA_ARGS__)) -void free_frozen_pages_nolock(struct page *page, unsigned int order); - -extern void zone_pcp_reset(struct zone *zone); -extern void zone_pcp_disable(struct zone *zone); -extern void zone_pcp_enable(struct zone *zone); -extern void zone_pcp_init(struct zone *zone); - extern void *memmap_alloc(phys_addr_t size, phys_addr_t align, phys_addr_t min_addr, int nid, bool exact_nid); @@ -1101,23 +906,6 @@ static inline void init_cma_pageblock(struct page *page) } #endif -enum fallback_result { - /* Found suitable migratetype, *mt_out is valid. */ - FALLBACK_FOUND, - /* No fallback found in requested order. */ - FALLBACK_EMPTY, - /* Passed @claimable, but claiming whole block is a bad idea. */ - FALLBACK_NOCLAIM, -}; -enum fallback_result -find_suitable_fallback(struct free_area *area, unsigned int order, - int migratetype, bool claimable, int *mt_out); - -static inline bool free_area_empty(struct free_area *area, int migratetype) -{ - return list_empty(&area->free_list[migratetype]); -} - /* mm/util.c */ struct anon_vma *folio_anon_vma(const struct folio *folio); @@ -1445,46 +1233,6 @@ extern unsigned long __must_check vm_mmap_pgoff(struct file *, unsigned long, unsigned long reclaim_pages(struct list_head *folio_list); unsigned int reclaim_clean_pages_from_list(struct zone *zone, struct list_head *folio_list); -/* The ALLOC_WMARK bits are used as an index to zone->watermark */ -#define ALLOC_WMARK_MIN WMARK_MIN -#define ALLOC_WMARK_LOW WMARK_LOW -#define ALLOC_WMARK_HIGH WMARK_HIGH -#define ALLOC_NO_WATERMARKS 0x04 /* don't check watermarks at all */ - -/* Mask to get the watermark bits */ -#define ALLOC_WMARK_MASK (ALLOC_NO_WATERMARKS-1) - -/* - * Only MMU archs have async oom victim reclaim - aka oom_reaper so we - * cannot assume a reduced access to memory reserves is sufficient for - * !MMU - */ -#ifdef CONFIG_MMU -#define ALLOC_OOM 0x08 -#else -#define ALLOC_OOM ALLOC_NO_WATERMARKS -#endif - -#define ALLOC_NON_BLOCK 0x10 /* Caller cannot block. Allow access - * to 25% of the min watermark or - * 62.5% if __GFP_HIGH is set. - */ -#define ALLOC_MIN_RESERVE 0x20 /* __GFP_HIGH set. Allow access to 50% - * of the min watermark. - */ -#define ALLOC_CPUSET 0x40 /* check for correct cpuset */ -#define ALLOC_CMA 0x80 /* allow allocations from CMA areas */ -#ifdef CONFIG_ZONE_DMA32 -#define ALLOC_NOFRAGMENT 0x100 /* avoid mixing pageblock types */ -#else -#define ALLOC_NOFRAGMENT 0x0 -#endif -#define ALLOC_HIGHATOMIC 0x200 /* Allows access to MIGRATE_HIGHATOMIC */ -#define ALLOC_NOLOCK 0x400 /* Only use spin_trylock in allocation path */ -#define ALLOC_KSWAPD 0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */ - -/* Flags that allow allocations below the min watermark. */ -#define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM) enum ttu_flags; struct tlbflush_unmap_batch; diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 617bca76db49..58e14d1543ec 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -26,6 +26,7 @@ #include #include "internal.h" +#include "page_alloc.h" #include "mm_slot.h" enum scan_result { diff --git a/mm/kmsan/init.c b/mm/kmsan/init.c index b14ce3417e65..4983b6e9f7c9 100644 --- a/mm/kmsan/init.c +++ b/mm/kmsan/init.c @@ -13,7 +13,7 @@ #include #include -#include "../internal.h" +#include "../page_alloc.h" #define NUM_FUTURE_RANGES 128 struct start_end_pair { diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 4963ea9f6ec6..1fb4f6206e6b 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -66,6 +66,7 @@ #include #include "swap.h" +#include "page_alloc.h" #include "internal.h" static int sysctl_memory_failure_early_kill __read_mostly; diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 8b137328dcf0..11ab2f7bc7f3 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -40,6 +40,7 @@ #include #include "internal.h" +#include "page_alloc.h" #include "shuffle.h" enum { diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 36699fabd3c2..9c740324f916 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -119,6 +119,7 @@ #include #include "internal.h" +#include "page_alloc.h" /* Internal flags */ #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */ diff --git a/mm/migrate.c b/mm/migrate.c index 3a426abc2c2b..1977905ce12c 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -49,6 +49,7 @@ #include #include "internal.h" +#include "page_alloc.h" #include "swap.h" static const struct movable_operations *offline_movable_ops; diff --git a/mm/mm_init.c b/mm/mm_init.c index 07a8c74cf7ad..537664974ab1 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -33,6 +33,7 @@ #include #include #include "internal.h" +#include "page_alloc.h" #include "slab.h" #include "shuffle.h" diff --git a/mm/page_alloc.c b/mm/page_alloc.c index df1345cde301..85cee8a0031f 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -56,6 +56,7 @@ #include #include #include "internal.h" +#include "page_alloc.h" #include "shuffle.h" #include "page_reporting.h" diff --git a/mm/page_alloc.h b/mm/page_alloc.h new file mode 100644 index 000000000000..3250d44f9645 --- /dev/null +++ b/mm/page_alloc.h @@ -0,0 +1,269 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * mm-internal API for the page (buddy) allocator. Public API lives in + * include/linux/gfp.h. + */ +#ifndef __MM_PAGE_ALLOC_H +#define __MM_PAGE_ALLOC_H + +#include +#include +#include +#include + +/* The ALLOC_WMARK bits are used as an index to zone->watermark */ +#define ALLOC_WMARK_MIN WMARK_MIN +#define ALLOC_WMARK_LOW WMARK_LOW +#define ALLOC_WMARK_HIGH WMARK_HIGH +#define ALLOC_NO_WATERMARKS 0x04 /* don't check watermarks at all */ + +/* Mask to get the watermark bits */ +#define ALLOC_WMARK_MASK (ALLOC_NO_WATERMARKS-1) + +/* + * Only MMU archs have async oom victim reclaim - aka oom_reaper so we + * cannot assume a reduced access to memory reserves is sufficient for + * !MMU + */ +#ifdef CONFIG_MMU +#define ALLOC_OOM 0x08 +#else +#define ALLOC_OOM ALLOC_NO_WATERMARKS +#endif + +#define ALLOC_NON_BLOCK 0x10 /* Caller cannot block. Allow access + * to 25% of the min watermark or + * 62.5% if __GFP_HIGH is set. + */ +#define ALLOC_MIN_RESERVE 0x20 /* __GFP_HIGH set. Allow access to 50% + * of the min watermark. + */ +#define ALLOC_CPUSET 0x40 /* check for correct cpuset */ +#define ALLOC_CMA 0x80 /* allow allocations from CMA areas */ +#ifdef CONFIG_ZONE_DMA32 +#define ALLOC_NOFRAGMENT 0x100 /* avoid mixing pageblock types */ +#else +#define ALLOC_NOFRAGMENT 0x0 +#endif +#define ALLOC_HIGHATOMIC 0x200 /* Allows access to MIGRATE_HIGHATOMIC */ +#define ALLOC_NOLOCK 0x400 /* Only use spin_trylock in allocation path */ +#define ALLOC_KSWAPD 0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */ + +/* Flags that allow allocations below the min watermark. */ +#define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM) + +/* + * Structure for holding the mostly immutable allocation parameters passed + * between functions involved in allocations, including the alloc_pages* + * family of functions. + * + * nodemask, migratetype and highest_zoneidx are initialized only once in + * __alloc_pages() and then never change. + * + * zonelist, preferred_zone and highest_zoneidx are set first in + * __alloc_pages() for the fast path, and might be later changed + * in __alloc_pages_slowpath(). All other functions pass the whole structure + * by a const pointer. + */ +struct alloc_context { + struct zonelist *zonelist; + const nodemask_t *nodemask; + struct zoneref *preferred_zoneref; + int migratetype; + + /* + * highest_zoneidx represents highest usable zone index of + * the allocation request. Due to the nature of the zone, + * memory on lower zone than the highest_zoneidx will be + * protected by lowmem_reserve[highest_zoneidx]. + * + * highest_zoneidx is also used by reclaim/compaction to limit + * the target zone since higher zone than this index cannot be + * usable for this allocation request. + */ + enum zone_type highest_zoneidx; + bool spread_dirty_pages; +}; + +/* + * This function returns the order of a free page in the buddy system. In + * general, page_zone(page)->lock must be held by the caller to prevent the + * page from being allocated in parallel and returning garbage as the order. + * If a caller does not hold page_zone(page)->lock, it must guarantee that the + * page cannot be allocated or merged in parallel. Alternatively, it must + * handle invalid values gracefully, and use buddy_order_unsafe() below. + */ +static inline unsigned int buddy_order(struct page *page) +{ + /* PageBuddy() must be checked by the caller */ + return page_private(page); +} + +/* + * Like buddy_order(), but for callers who cannot afford to hold the zone lock. + * PageBuddy() should be checked first by the caller to minimize race window, + * and invalid values must be handled gracefully. + * + * READ_ONCE is used so that if the caller assigns the result into a local + * variable and e.g. tests it for valid range before using, the compiler cannot + * decide to remove the variable and inline the page_private(page) multiple + * times, potentially observing different values in the tests and the actual + * use of the result. + */ +#define buddy_order_unsafe(page) READ_ONCE(page_private(page)) + +/* + * This function checks whether a page is free && is the buddy + * we can coalesce a page and its buddy if + * (a) the buddy is not in a hole (check before calling!) && + * (b) the buddy is in the buddy system && + * (c) a page and its buddy have the same order && + * (d) a page and its buddy are in the same zone. + * + * For recording whether a page is in the buddy system, we set PageBuddy. + * Setting, clearing, and testing PageBuddy is serialized by zone->lock. + * + * For recording page's order, we use page_private(page). + */ +static inline bool page_is_buddy(struct page *page, struct page *buddy, + unsigned int order) +{ + if (!page_is_guard(buddy) && !PageBuddy(buddy)) + return false; + + if (buddy_order(buddy) != order) + return false; + + /* + * zone check is done late to avoid uselessly calculating + * zone/node ids for pages that could never merge. + */ + if (page_zone_id(page) != page_zone_id(buddy)) + return false; + + VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); + + return true; +} + +/* + * Locate the struct page for both the matching buddy in our + * pair (buddy1) and the combined O(n+1) page they form (page). + * + * 1) Any buddy B1 will have an order O twin B2 which satisfies + * the following equation: + * B2 = B1 ^ (1 << O) + * For example, if the starting buddy (buddy2) is #8 its order + * 1 buddy is #10: + * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10 + * + * 2) Any buddy B will have an order O+1 parent P which + * satisfies the following equation: + * P = B & ~(1 << O) + * + * Assumption: *_mem_map is contiguous at least up to MAX_PAGE_ORDER + */ +static inline unsigned long +__find_buddy_pfn(unsigned long page_pfn, unsigned int order) +{ + return page_pfn ^ (1 << order); +} + +/* + * Find the buddy of @page and validate it. + * @page: The input page + * @pfn: The pfn of the page, it saves a call to page_to_pfn() when the + * function is used in the performance-critical __free_one_page(). + * @order: The order of the page + * @buddy_pfn: The output pointer to the buddy pfn, it also saves a call to + * page_to_pfn(). + * + * The found buddy can be a non PageBuddy, out of @page's zone, or its order is + * not the same as @page. The validation is necessary before use it. + * + * Return: the found buddy page or NULL if not found. + */ +static inline struct page *find_buddy_page_pfn(struct page *page, + unsigned long pfn, unsigned int order, unsigned long *buddy_pfn) +{ + unsigned long __buddy_pfn = __find_buddy_pfn(pfn, order); + struct page *buddy; + + buddy = page + (__buddy_pfn - pfn); + if (buddy_pfn) + *buddy_pfn = __buddy_pfn; + + if (page_is_buddy(page, buddy, order)) + return buddy; + return NULL; +} + +extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn, + unsigned long end_pfn, struct zone *zone); + +static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn, + unsigned long end_pfn, struct zone *zone) +{ + if (zone->contiguous) + return pfn_to_page(start_pfn); + + return __pageblock_pfn_to_page(start_pfn, end_pfn, zone); +} + +extern void __free_pages_core(struct page *page, unsigned int order, + enum meminit_context context); + +void post_alloc_hook(struct page *page, unsigned int order, gfp_t gfp_flags); +extern bool free_pages_prepare(struct page *page, unsigned int order); + +extern int user_min_free_kbytes; + +struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, int nid, + nodemask_t *nodemask); +#define __alloc_frozen_pages(...) \ + alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__)) +void free_frozen_pages(struct page *page, unsigned int order); +void free_unref_folios(struct folio_batch *fbatch); + +#ifdef CONFIG_NUMA +struct page *alloc_frozen_pages_noprof(gfp_t, unsigned int order); +#else +static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order) +{ + return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL); +} +#endif + +#define alloc_frozen_pages(...) \ + alloc_hooks(alloc_frozen_pages_noprof(__VA_ARGS__)) + +struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order); +#define alloc_frozen_pages_nolock(...) \ + alloc_hooks(alloc_frozen_pages_nolock_noprof(__VA_ARGS__)) +void free_frozen_pages_nolock(struct page *page, unsigned int order); + +extern void zone_pcp_reset(struct zone *zone); +extern void zone_pcp_disable(struct zone *zone); +extern void zone_pcp_enable(struct zone *zone); +extern void zone_pcp_init(struct zone *zone); + +enum fallback_result { + /* Found suitable migratetype, *mt_out is valid. */ + FALLBACK_FOUND, + /* No fallback found in requested order. */ + FALLBACK_EMPTY, + /* Passed @claimable, but claiming whole block is a bad idea. */ + FALLBACK_NOCLAIM, +}; +enum fallback_result +find_suitable_fallback(struct free_area *area, unsigned int order, + int migratetype, bool claimable, int *mt_out); + +static inline bool free_area_empty(struct free_area *area, int migratetype) +{ + return list_empty(&area->free_list[migratetype]); +} + +void page_alloc_sysctl_init(void); + +#endif /* __MM_PAGE_ALLOC_H */ diff --git a/mm/page_frag_cache.c b/mm/page_frag_cache.c index d2423f30577e..a1077cef3a79 100644 --- a/mm/page_frag_cache.c +++ b/mm/page_frag_cache.c @@ -18,7 +18,7 @@ #include #include #include -#include "internal.h" +#include "page_alloc.h" static unsigned long encoded_page_create(struct page *page, unsigned int order, bool pfmemalloc) diff --git a/mm/page_isolation.c b/mm/page_isolation.c index 32ce8a7d9df3..e5dfc7bf4944 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -11,6 +11,7 @@ #include #include #include "internal.h" +#include "page_alloc.h" #define CREATE_TRACE_POINTS #include diff --git a/mm/page_owner.c b/mm/page_owner.c index 5cad1dfe057a..fbbda7ba914b 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -13,7 +13,7 @@ #include #include -#include "internal.h" +#include "page_alloc.h" /* * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack) diff --git a/mm/page_reporting.c b/mm/page_reporting.c index 942e84b6908a..9d21f9255779 100644 --- a/mm/page_reporting.c +++ b/mm/page_reporting.c @@ -8,6 +8,7 @@ #include #include +#include "page_alloc.h" #include "page_reporting.h" #include "internal.h" diff --git a/mm/show_mem.c b/mm/show_mem.c index 1b721a8ade67..d1288b4c2b64 100644 --- a/mm/show_mem.c +++ b/mm/show_mem.c @@ -16,6 +16,7 @@ #include #include "internal.h" +#include "page_alloc.h" #include "swap.h" atomic_long_t _totalram_pages __read_mostly; diff --git a/mm/shuffle.c b/mm/shuffle.c index fb1393b8b3a9..82a2c7725a08 100644 --- a/mm/shuffle.c +++ b/mm/shuffle.c @@ -7,6 +7,7 @@ #include #include #include "internal.h" +#include "page_alloc.h" #include "shuffle.h" DEFINE_STATIC_KEY_FALSE(page_alloc_shuffle_key); diff --git a/mm/slub.c b/mm/slub.c index 0337e60db5ac..1362a1f8f05c 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -53,6 +53,7 @@ #include #include "internal.h" +#include "page_alloc.h" /* * Lock order: diff --git a/mm/swap.c b/mm/swap.c index 58e4eff698cc..d25131305c94 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -39,6 +39,7 @@ #include #include "internal.h" +#include "page_alloc.h" #define CREATE_TRACE_POINTS #include diff --git a/mm/vmscan.c b/mm/vmscan.c index c976cc8a2087..454e9d0930ab 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -66,6 +66,7 @@ #include #include "internal.h" +#include "page_alloc.h" #include "swap.h" #define CREATE_TRACE_POINTS From 2fc4c1d51d878bcd615774d19a97dc29ac0ce970 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Fri, 3 Jul 2026 12:31:45 +0000 Subject: [PATCH 185/501] mm/page_alloc: unify __alloc_frozen_pages[_nolock]_noprof() Currently the core allocator code is controlled by ALLOC_NOLOCK, but the main entry point function is significantly different from the normal __alloc_frozen_pages_nolock(), this is tiring when reading the code. Plumb the ALLOC_NOLOCK control one layer up in the call stack: create an alloc_flags argument to __alloc_frozen_pages_nolock() (which is only exposed to mm/) and then turn the nolock variant into a thin wrapper that just sets that flag (as well as handling NUMA_NO_NODE, similar to how some of the wrappers in gfp.h do). For consistency, set ALLOC_WMARK_MIN explicitly in fastpath_alloc_flags for the new ALLOC_NOLOCK path. This was already "done" silently in __alloc_frozen_pages_nolock_noprof(): ALLOC_WMARK_MIN is 0. Rationale that this doesn't change anything: 1. Simple bits: A bunch of the nolock-specific handling is just moved to the new alloc_order_allowed(), alloc_nolock_allowed() and gfp_nolock. 2. __alloc_frozen_pages_noprof() has some extra logic that wasn't previously in the nolock variant: a. Application of gfp_allowed_mask; this only affects early boot, only flags that affect the slowpath get changed here, and the nolock allocation path isn't allowed to the GFP_BOOT_MASK flags. b. Application of current_gfp_context() - also only affects the slowpath 3. The slowpath itself: this is now just explicitly skipped under !ALLOC_NOLOCK. Ulterior motive: adding an alloc_flags arg to the allocator's mm-internal entrypoint can later be used to do more allocation customisation without needing to create new GFP flags. No functional change intended. Link: https://lore.kernel.org/20260703-alloc-trylock-v5-5-c87b714e19d3@google.com Signed-off-by: Brendan Jackman Reviewed-by: Vlastimil Babka (SUSE) Acked-by: Zi Yan Signed-off-by: Andrew Morton --- mm/hugetlb.c | 3 +- mm/mempolicy.c | 10 ++- mm/page_alloc.c | 192 +++++++++++++++++++++++++----------------------- mm/page_alloc.h | 6 +- mm/slub.c | 6 +- 5 files changed, 117 insertions(+), 100 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 8ce6fe2a9a72..53c4e2ef8f6d 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1790,7 +1790,8 @@ static struct folio *alloc_buddy_frozen_folio(int order, gfp_t gfp_mask, if (alloc_try_hard) gfp_mask |= __GFP_RETRY_MAYFAIL; - folio = (struct folio *)__alloc_frozen_pages(gfp_mask, order, nid, nmask); + folio = (struct folio *)__alloc_frozen_pages(gfp_mask, order, nid, nmask, + ALLOC_DEFAULT); /* * If we did not specify __GFP_RETRY_MAYFAIL, but still got a diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 9c740324f916..41d630f0ea82 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -2426,9 +2426,11 @@ static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order, */ preferred_gfp = gfp | __GFP_NOWARN; preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); - page = __alloc_frozen_pages_noprof(preferred_gfp, order, nid, nodemask); + page = __alloc_frozen_pages_noprof(preferred_gfp, order, nid, nodemask, + ALLOC_DEFAULT); if (!page) - page = __alloc_frozen_pages_noprof(gfp, order, nid, NULL); + page = __alloc_frozen_pages_noprof(gfp, order, nid, NULL, + ALLOC_DEFAULT); return page; } @@ -2476,7 +2478,7 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, */ page = __alloc_frozen_pages_noprof( gfp | __GFP_THISNODE | __GFP_NORETRY, order, - nid, NULL); + nid, NULL, ALLOC_DEFAULT); if (page || !(gfp & __GFP_DIRECT_RECLAIM)) return page; /* @@ -2488,7 +2490,7 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, } } - page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask); + page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask, ALLOC_DEFAULT); if (unlikely(pol->mode == MPOL_INTERLEAVE || pol->mode == MPOL_WEIGHTED_INTERLEAVE) && page) { diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 85cee8a0031f..f47a84855507 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5222,7 +5222,7 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid, } nr_account++; - prep_new_page(page, 0, gfp, 0); + prep_new_page(page, 0, gfp, ALLOC_DEFAULT); set_page_refcounted(page); page_array[nr_populated++] = page; } @@ -5271,24 +5271,99 @@ void free_pages_bulk(struct page **page_array, unsigned long nr_pages) } } -/* - * This is the 'heart' of the zoned buddy allocator. - */ -struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, - int preferred_nid, nodemask_t *nodemask) +static inline bool alloc_order_allowed(gfp_t gfp, unsigned int order, + unsigned int alloc_flags) { - struct page *page; - unsigned int fastpath_alloc_flags = ALLOC_WMARK_LOW; - gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */ - struct alloc_context ac = { }; + if (alloc_flags & ALLOC_NOLOCK) + return pcp_allowed_order(order); /* * There are several places where we assume that the order value is sane * so bail out early if the request is out of bound. */ - if (WARN_ON_ONCE_GFP(order > MAX_PAGE_ORDER, gfp)) + return !(WARN_ON_ONCE_GFP(order > MAX_PAGE_ORDER, gfp)); +} + +static inline bool alloc_nolock_allowed(void) +{ + /* + * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is + * unsafe in NMI. If spin_trylock() is called from hard IRQ the current + * task may be waiting for one rt_spin_lock, but rt_spin_trylock() will + * mark the task as the owner of another rt_spin_lock which will + * confuse PI logic, so return immediately if called from hard IRQ or + * NMI. + * + * Note, irqs_disabled() case is ok. This function can be called + * from raw_spin_lock_irqsave region. + */ + if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq())) + return false; + + /* On UP, spin_trylock() always succeeds even when it is locked */ + if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) + return false; + + /* Bailout, since _deferred_grow_zone() needs to take a lock */ + if (deferred_pages_enabled()) + return false; + + return true; +} + +/* + * GFP flags to set for ALLOC_NOLOCK i.e. alloc_pages_nolock(). + * + * Do not specify __GFP_DIRECT_RECLAIM, since direct claim is not allowed. + * Do not specify __GFP_KSWAPD_RECLAIM either, since wake up of kswapd + * is not safe in arbitrary context. + * + * These two are the conditions for gfpflags_allow_spinning() being true. + * + * Specify __GFP_NOWARN since failing alloc_pages_nolock() is not a reason + * to warn. Also warn would trigger printk() which is unsafe from + * various contexts. We cannot use printk_deferred_enter() to mitigate, + * since the running context is unknown. + * + * Specify __GFP_ZERO to make sure that call to kmsan_alloc_page() below + * is safe in any context. Also zeroing the page is mandatory for + * BPF use cases. + * + * Though __GFP_NOMEMALLOC is not checked in the code path below, + * specify it here to highlight that alloc_pages_nolock() + * doesn't want to deplete reserves. + */ +static const gfp_t gfp_nolock = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC | + __GFP_COMP; + +/* + * This is the 'heart' of the zoned buddy allocator. + */ +struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, + int preferred_nid, nodemask_t *nodemask, unsigned int alloc_flags) +{ + struct page *page; + gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */ + struct alloc_context ac = { }; + unsigned int fastpath_alloc_flags = alloc_flags; + + /* Other flags could be supported later if needed. */ + if (WARN_ON(alloc_flags & ~ALLOC_NOLOCK)) return NULL; + if (!alloc_order_allowed(gfp, order, alloc_flags)) + return NULL; + + if (alloc_flags & ALLOC_NOLOCK) { + VM_WARN_ON_ONCE(gfp & ~__GFP_ACCOUNT); + if (!alloc_nolock_allowed()) + return NULL; + gfp |= gfp_nolock; + fastpath_alloc_flags |= ALLOC_WMARK_MIN; + } else { + fastpath_alloc_flags |= ALLOC_WMARK_LOW; + } + gfp &= gfp_allowed_mask; /* * Apply scoped allocation constraints. This is mainly about GFP_NOFS @@ -5303,16 +5378,19 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, &alloc_gfp, &fastpath_alloc_flags)) return NULL; - /* - * Forbid the first pass from falling back to types that fragment - * memory until all local zones are considered. - */ - fastpath_alloc_flags |= alloc_flags_nofragment(zonelist_zone(ac.preferred_zoneref), gfp); + if (!(alloc_flags & ALLOC_NOLOCK)) { + /* + * Forbid the first pass from falling back to types that + * fragment memory until all local zones are considered. + */ + fastpath_alloc_flags |= alloc_flags_nofragment( + zonelist_zone(ac.preferred_zoneref), gfp); + } fastpath_alloc_flags |= alloc_flags_nonblocking(gfp, order) & ALLOC_HIGHATOMIC; - /* First allocation attempt */ + /* First allocation attempt (or, for nolock, only attempt) */ page = get_page_from_freelist(alloc_gfp, order, fastpath_alloc_flags, &ac); - if (likely(page)) + if (likely(page) || (alloc_flags & ALLOC_NOLOCK)) goto out; alloc_gfp = gfp; @@ -5329,7 +5407,8 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, out: if (memcg_kmem_online() && (gfp & __GFP_ACCOUNT) && page && unlikely(__memcg_kmem_charge_page(page, gfp, order) != 0)) { - free_frozen_pages(page, order); + __free_frozen_pages(page, order, + alloc_flags & ALLOC_NOLOCK ? FPI_TRYLOCK : 0); page = NULL; } @@ -5345,7 +5424,8 @@ struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, { struct page *page; - page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, nodemask); + page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, nodemask, + ALLOC_DEFAULT); if (page) set_page_refcounted(page); return page; @@ -7875,80 +7955,10 @@ static bool __free_unaccepted(struct page *page) struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order) { - /* - * Do not specify __GFP_DIRECT_RECLAIM, since direct claim is not allowed. - * Do not specify __GFP_KSWAPD_RECLAIM either, since wake up of kswapd - * is not safe in arbitrary context. - * - * These two are the conditions for gfpflags_allow_spinning() being true. - * - * Specify __GFP_NOWARN since failing alloc_pages_nolock() is not a reason - * to warn. Also warn would trigger printk() which is unsafe from - * various contexts. We cannot use printk_deferred_enter() to mitigate, - * since the running context is unknown. - * - * Specify __GFP_ZERO to make sure that call to kmsan_alloc_page() below - * is safe in any context. Also zeroing the page is mandatory for - * BPF use cases. - * - * Though __GFP_NOMEMALLOC is not checked in the code path below, - * specify it here to highlight that alloc_pages_nolock() - * doesn't want to deplete reserves. - */ - gfp_t alloc_gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC | __GFP_COMP - | gfp_flags; - unsigned int alloc_flags = ALLOC_NOLOCK; - struct alloc_context ac = { }; - struct page *page; - - VM_WARN_ON_ONCE(gfp_flags & ~__GFP_ACCOUNT); - /* - * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is - * unsafe in NMI. If spin_trylock() is called from hard IRQ the current - * task may be waiting for one rt_spin_lock, but rt_spin_trylock() will - * mark the task as the owner of another rt_spin_lock which will - * confuse PI logic, so return immediately if called from hard IRQ or - * NMI. - * - * Note, irqs_disabled() case is ok. This function can be called - * from raw_spin_lock_irqsave region. - */ - if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq())) - return NULL; - - /* On UP, spin_trylock() always succeeds even when it is locked */ - if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) - return NULL; - - if (!pcp_allowed_order(order)) - return NULL; - - /* Bailout, since _deferred_grow_zone() needs to take a lock */ - if (deferred_pages_enabled()) - return NULL; - if (nid == NUMA_NO_NODE) nid = numa_node_id(); - prepare_alloc_pages(alloc_gfp, order, nid, NULL, &ac, - &alloc_gfp, &alloc_flags); - - /* - * Best effort allocation from percpu free list. - * If it's empty attempt to spin_trylock zone->lock. - */ - page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac); - - /* Unlike regular alloc_pages() there is no __alloc_pages_slowpath(). */ - - if (memcg_kmem_online() && page && (gfp_flags & __GFP_ACCOUNT) && - unlikely(__memcg_kmem_charge_page(page, alloc_gfp, order) != 0)) { - __free_frozen_pages(page, order, FPI_TRYLOCK); - page = NULL; - } - trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype); - kmsan_alloc_page(page, order, alloc_gfp); - return page; + return __alloc_frozen_pages_noprof(gfp_flags, order, nid, NULL, ALLOC_NOLOCK); } /** * alloc_pages_nolock - opportunistic reentrant allocation from any context diff --git a/mm/page_alloc.h b/mm/page_alloc.h index 3250d44f9645..a4f4b325381a 100644 --- a/mm/page_alloc.h +++ b/mm/page_alloc.h @@ -11,6 +11,7 @@ #include #include +#define ALLOC_DEFAULT 0 /* The ALLOC_WMARK bits are used as an index to zone->watermark */ #define ALLOC_WMARK_MIN WMARK_MIN #define ALLOC_WMARK_LOW WMARK_LOW @@ -219,7 +220,7 @@ extern bool free_pages_prepare(struct page *page, unsigned int order); extern int user_min_free_kbytes; struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, int nid, - nodemask_t *nodemask); + nodemask_t *nodemask, unsigned int alloc_flags); #define __alloc_frozen_pages(...) \ alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__)) void free_frozen_pages(struct page *page, unsigned int order); @@ -230,7 +231,8 @@ struct page *alloc_frozen_pages_noprof(gfp_t, unsigned int order); #else static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order) { - return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL); + return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL, + ALLOC_DEFAULT); } #endif diff --git a/mm/slub.c b/mm/slub.c index 1362a1f8f05c..85599d227b9b 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -3264,7 +3264,8 @@ static inline struct slab *alloc_slab_page(gfp_t flags, int node, else if (node == NUMA_NO_NODE) page = alloc_frozen_pages(flags, order); else - page = __alloc_frozen_pages(flags, order, node, NULL); + page = __alloc_frozen_pages(flags, order, node, NULL, + ALLOC_DEFAULT); if (!page) return NULL; @@ -5274,7 +5275,8 @@ static void *___kmalloc_large_node(size_t size, gfp_t flags, int node) if (node == NUMA_NO_NODE) page = alloc_frozen_pages_noprof(flags, order); else - page = __alloc_frozen_pages_noprof(flags, order, node, NULL); + page = __alloc_frozen_pages_noprof(flags, order, node, NULL, + ALLOC_DEFAULT); if (page) { ptr = page_address(page); From ba26999e7c2e6fc9f32ac5e1321ad130d18ea7bd Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Fri, 3 Jul 2026 12:31:46 +0000 Subject: [PATCH 186/501] mm/page_alloc: relax GFP WARN in nolock allocs This WARN forbids setting other flags than __GFP_ACCOUNT but we unconditionally set the ones in gfp_nolock so they are certainly fine for the caller to set. There are other GFP flags that are almost certainly fine to set here; Willy noted GFP_HIGHMEM, GFP_DMA, GFP_MOVABLE and GFP_HARDWALL. But, nolock allocation is rather special, so be conservative to try and ensure we have a chance to think carefully before nontrivial new usecases arise. Link: https://lore.kernel.org/20260703-alloc-trylock-v5-6-c87b714e19d3@google.com Signed-off-by: Brendan Jackman Suggested-by: Matthew Wilcox Link: https://lore.kernel.org/linux-mm/ajS96fWbG4dzP3u3@casper.infradead.org/ Reviewed-by: Suren Baghdasaryan Reviewed-by: Vlastimil Babka (SUSE) Acked-by: Harry Yoo (Oracle) Acked-by: Zi Yan Signed-off-by: Andrew Morton --- mm/page_alloc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index f47a84855507..c2839959d790 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5355,7 +5355,8 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, return NULL; if (alloc_flags & ALLOC_NOLOCK) { - VM_WARN_ON_ONCE(gfp & ~__GFP_ACCOUNT); + /* Certain other flags could be supported later if needed. */ + VM_WARN_ON_ONCE(gfp & ~(__GFP_ACCOUNT | gfp_nolock)); if (!alloc_nolock_allowed()) return NULL; gfp |= gfp_nolock; From 6f414dbf21782ad8e906b391a429ddd21330e011 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Fri, 3 Jul 2026 12:31:47 +0000 Subject: [PATCH 187/501] mm: move some stuff to mm/page_alloc.h Some of this stuff in the public header is only used internally so shrink the scope to avoid silently growing new users. drain_local_pages() is still used from kernel/power/snapshot.c so that needs to stay behind. Link: https://lore.kernel.org/20260703-alloc-trylock-v5-7-c87b714e19d3@google.com Signed-off-by: Brendan Jackman Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Signed-off-by: Andrew Morton --- include/linux/gfp.h | 26 -------------------------- mm/page_alloc.h | 27 +++++++++++++++++++++++++++ mm/vmstat.c | 1 + 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/include/linux/gfp.h b/include/linux/gfp.h index cdf95a9f0b87..01d6d2591f49 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -17,28 +17,6 @@ struct mempolicy; #define __default_gfp(a,b,...) b #define default_gfp(...) __default_gfp(,##__VA_ARGS__,GFP_KERNEL) -/* Convert GFP flags to their corresponding migrate type */ -#define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE) -#define GFP_MOVABLE_SHIFT 3 - -static inline int gfp_migratetype(const gfp_t gfp_flags) -{ - VM_WARN_ON((gfp_flags & GFP_MOVABLE_MASK) == GFP_MOVABLE_MASK); - BUILD_BUG_ON((1UL << GFP_MOVABLE_SHIFT) != ___GFP_MOVABLE); - BUILD_BUG_ON((___GFP_MOVABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_MOVABLE); - BUILD_BUG_ON((___GFP_RECLAIMABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_RECLAIMABLE); - BUILD_BUG_ON(((___GFP_MOVABLE | ___GFP_RECLAIMABLE) >> - GFP_MOVABLE_SHIFT) != MIGRATE_HIGHATOMIC); - - if (unlikely(page_group_by_mobility_disabled)) - return MIGRATE_UNMOVABLE; - - /* Group based on mobility */ - return (__force unsigned long)(gfp_flags & GFP_MOVABLE_MASK) >> GFP_MOVABLE_SHIFT; -} -#undef GFP_MOVABLE_MASK -#undef GFP_MOVABLE_SHIFT - static inline bool gfpflags_allow_blocking(const gfp_t gfp_flags) { return !!(gfp_flags & __GFP_DIRECT_RECLAIM); @@ -395,10 +373,6 @@ extern void free_pages(unsigned long addr, unsigned int order); #define __free_page(page) __free_pages((page), 0) #define free_page(addr) free_pages((addr), 0) -void page_alloc_init_cpuhp(void); -bool decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp); -void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp); -void drain_all_pages(struct zone *zone); void drain_local_pages(struct zone *zone); void page_alloc_init_late(void); diff --git a/mm/page_alloc.h b/mm/page_alloc.h index a4f4b325381a..2d60551b4453 100644 --- a/mm/page_alloc.h +++ b/mm/page_alloc.h @@ -266,6 +266,33 @@ static inline bool free_area_empty(struct free_area *area, int migratetype) return list_empty(&area->free_list[migratetype]); } +/* Convert GFP flags to their corresponding migrate type */ +#define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE) +#define GFP_MOVABLE_SHIFT 3 + +static inline int gfp_migratetype(const gfp_t gfp_flags) +{ + VM_WARN_ON((gfp_flags & GFP_MOVABLE_MASK) == GFP_MOVABLE_MASK); + BUILD_BUG_ON((1UL << GFP_MOVABLE_SHIFT) != ___GFP_MOVABLE); + BUILD_BUG_ON((___GFP_MOVABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_MOVABLE); + BUILD_BUG_ON((___GFP_RECLAIMABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_RECLAIMABLE); + BUILD_BUG_ON(((___GFP_MOVABLE | ___GFP_RECLAIMABLE) >> + GFP_MOVABLE_SHIFT) != MIGRATE_HIGHATOMIC); + + if (unlikely(page_group_by_mobility_disabled)) + return MIGRATE_UNMOVABLE; + + /* Group based on mobility */ + return (__force unsigned long)(gfp_flags & GFP_MOVABLE_MASK) >> GFP_MOVABLE_SHIFT; +} +#undef GFP_MOVABLE_MASK +#undef GFP_MOVABLE_SHIFT + +bool decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp); +void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp); +void drain_all_pages(struct zone *zone); + +void page_alloc_init_cpuhp(void); void page_alloc_sysctl_init(void); #endif /* __MM_PAGE_ALLOC_H */ diff --git a/mm/vmstat.c b/mm/vmstat.c index 7b93fbf9af09..3b5cb1031f72 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -30,6 +30,7 @@ #include #include "internal.h" +#include "page_alloc.h" #ifdef CONFIG_PROC_FS #ifdef CONFIG_NUMA From ca74a8252289fc09cd69de891bb2cfbb34264e5a Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Fri, 3 Jul 2026 12:31:48 +0000 Subject: [PATCH 188/501] perf/x86/intel: use higher-level allocator API The difference between __alloc_pages_node() and alloc_pages_node() is that the latter allows you to pass NUMA_NO_NODE. The former is going away and the latter works fine here so switch over. No functional change intended. Link: https://lore.kernel.org/20260703-alloc-trylock-v5-8-c87b714e19d3@google.com Signed-off-by: Brendan Jackman Reviewed-by: Suren Baghdasaryan Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Cc: Peter Zijlstra Cc: Ingo Molnar Cc: Arnaldo Carvalho de Melo Cc: Namhyung Kim Cc: Mark Rutland Cc: Alexander Shishkin Cc: Jiri Olsa Cc: Ian Rogers Cc: Adrian Hunter Cc: James Clark Assisted-by: Gemini:unknown-version Signed-off-by: Andrew Morton --- arch/x86/events/intel/ds.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 91a093d8cf2e..70be80211d82 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -832,7 +832,7 @@ static void *dsalloc_pages(size_t size, gfp_t flags, int cpu) int node = cpu_to_node(cpu); struct page *page; - page = __alloc_pages_node(node, flags | __GFP_ZERO, order); + page = alloc_pages_node(node, flags | __GFP_ZERO, order); return page ? page_address(page) : NULL; } @@ -1088,9 +1088,9 @@ void init_arch_pebs_on_cpu(int cpu) /* * 4KB-aligned pointer of the output buffer - * (__alloc_pages_node() return page aligned address) + * (alloc_pages_node() returns page aligned address) * Buffer Size = 4KB * 2^SIZE - * contiguous physical buffer (__alloc_pages_node() with order) + * contiguous physical buffer (alloc_pages_node() with order) */ arch_pebs_base = virt_to_phys(cpuc->pebs_vaddr) | PEBS_BUFFER_SHIFT; wrmsrq_on_cpu(cpu, MSR_IA32_PEBS_BASE, arch_pebs_base); From bfceae3fd12c5295627bb7fe0fabcfa1b821f31a Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Fri, 3 Jul 2026 12:31:49 +0000 Subject: [PATCH 189/501] KVM: VMX: use higher-level allocator API The difference between __alloc_pages_node() and alloc_pages_node() is that the latter allows you to pass NUMA_NO_NODE. The former is going away and the latter works fine here so switch over. No functional change intended. Link: https://lore.kernel.org/20260703-alloc-trylock-v5-9-c87b714e19d3@google.com Signed-off-by: Brendan Jackman Reviewed-by: Suren Baghdasaryan Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Cc: Sean Christopherson Cc: Paolo Bonzini Assisted-by: Gemini:unknown-version Signed-off-by: Andrew Morton --- arch/x86/kvm/vmx/vmx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index cc75feec05da..6eaa337bd276 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -3027,7 +3027,7 @@ struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags) struct page *pages; struct vmcs *vmcs; - pages = __alloc_pages_node(node, flags, 0); + pages = alloc_pages_node(node, flags, 0); if (!pages) return NULL; vmcs = page_address(pages); From 9b5b281f21b458f0d56378981b6167ece362d3f6 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Fri, 3 Jul 2026 12:31:50 +0000 Subject: [PATCH 190/501] x86/virt: use higher-level allocator API The difference between __alloc_pages_node() and alloc_pages_node() is that the latter allows you to pass NUMA_NO_NODE. The former is going away and the latter works fine here so switch over. No functional change intended. Link: https://lore.kernel.org/20260703-alloc-trylock-v5-10-c87b714e19d3@google.com Signed-off-by: Brendan Jackman Reviewed-by: Suren Baghdasaryan Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: "H. Peter Anvin" Assisted-by: Gemini:unknown-version Signed-off-by: Andrew Morton --- arch/x86/virt/hw.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/virt/hw.c b/arch/x86/virt/hw.c index 7e9091c640be..a236447ac7a2 100644 --- a/arch/x86/virt/hw.c +++ b/arch/x86/virt/hw.c @@ -196,7 +196,7 @@ static __init int __x86_vmx_init(void) struct page *page; struct vmcs *vmcs; - page = __alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); + page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); if (WARN_ON_ONCE(!page)) { x86_vmx_exit(); return -ENOMEM; From 579c6de3def20f4247cb7621778d943aed653fca Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Fri, 3 Jul 2026 12:31:51 +0000 Subject: [PATCH 191/501] sgi-xp: use higher-level allocator API The difference between __alloc_pages_node() and alloc_pages_node() is that the latter allows you to pass NUMA_NO_NODE. The former is going away and the latter works fine here so switch over. No functional change intended. Link: https://lore.kernel.org/20260703-alloc-trylock-v5-11-c87b714e19d3@google.com Signed-off-by: Brendan Jackman Acked-by: Greg Kroah-Hartman Reviewed-by: Suren Baghdasaryan Acked-by: Steve Wahl Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Cc: Robin Holt Cc: Steve Wahl Cc: Arnd Bergmann Cc: Greg Kroah-Hartman Assisted-by: Gemini:unknown-model Signed-off-by: Andrew Morton --- drivers/misc/sgi-xp/xpc_uv.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/misc/sgi-xp/xpc_uv.c b/drivers/misc/sgi-xp/xpc_uv.c index 772c78726893..79c2f00ed4d7 100644 --- a/drivers/misc/sgi-xp/xpc_uv.c +++ b/drivers/misc/sgi-xp/xpc_uv.c @@ -170,9 +170,8 @@ xpc_create_gru_mq_uv(unsigned int mq_size, int cpu, char *irq_name, mq->mmr_blade = uv_cpu_to_blade_id(cpu); nid = cpu_to_node(cpu); - page = __alloc_pages_node(nid, - GFP_KERNEL | __GFP_ZERO | __GFP_THISNODE, - pg_order); + page = alloc_pages_node(nid, GFP_KERNEL | __GFP_ZERO | __GFP_THISNODE, + pg_order); if (page == NULL) { dev_err(xpc_part, "xpc_create_gru_mq_uv() failed to alloc %d " "bytes of memory on nid=%d for GRU mq\n", mq_size, nid); From 679e5aa1a299cc44121632363f9b345d1f5851a3 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Fri, 3 Jul 2026 12:31:52 +0000 Subject: [PATCH 192/501] net/funeth: switch to higher-level allocator API The difference between __alloc_pages_node() and alloc_pages_node() is that the latter allows you to pass NUMA_NO_NODE. The former is going away and the latter works fine here so switch over. No functional change intended. Link: https://lore.kernel.org/20260703-alloc-trylock-v5-12-c87b714e19d3@google.com Signed-off-by: Brendan Jackman Reviewed-by: Suren Baghdasaryan Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Cc: Dimitris Michailidis Cc: Andrew Lunn Cc: "David S. Miller" Cc: Eric Dumazet Cc: Jakub Kicinski Cc: Paolo Abeni Assisted-by: Gemini:unknown-version Signed-off-by: Andrew Morton --- drivers/net/ethernet/fungible/funeth/funeth_rx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/fungible/funeth/funeth_rx.c b/drivers/net/ethernet/fungible/funeth/funeth_rx.c index 7e2584895de3..d7000017ac2b 100644 --- a/drivers/net/ethernet/fungible/funeth/funeth_rx.c +++ b/drivers/net/ethernet/fungible/funeth/funeth_rx.c @@ -103,7 +103,7 @@ static int funeth_alloc_page(struct funeth_rxq *q, struct funeth_rxbuf *rb, if (cache_get(q, rb)) return 0; - p = __alloc_pages_node(node, gfp | __GFP_NOWARN, 0); + p = alloc_pages_node(node, gfp | __GFP_NOWARN, 0); if (unlikely(!p)) return -ENOMEM; From bcdd8d43a7cad8959f556e21b671b839fe4dc8e3 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Fri, 3 Jul 2026 12:31:53 +0000 Subject: [PATCH 193/501] mm: remove __alloc_pages_node() There were only a few users, which have been removed. The only advantage of this API over alloc_pages_node() is avoiding a single conditional branch. The disadvantages are: 1. More API surface, more sources of confusion, more maintenance. 2. Worse impact of CPU hotplug bugs: most users of __alloc_pages_node() were using the result of cpu_to_node(); if the CPU gets hotplugged out this will return NUMA_NO_NODE. If one of these paths fails to protect against a concurrent hotplug then page_alloc.c will use NUMA_NO_NODE as an index into NODE_DATA() and cause some horrible memory corruption or other. With alloc_pages_node(), the code might just work fine. Ulterior motive: this frees up the __* variants of the allocator APIs to serve specifically for use as mm-internal API. Link: https://lore.kernel.org/20260703-alloc-trylock-v5-13-c87b714e19d3@google.com Signed-off-by: Brendan Jackman Reviewed-by: Suren Baghdasaryan Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Signed-off-by: Andrew Morton --- include/linux/gfp.h | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 01d6d2591f49..3bf55a5f9143 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -256,21 +256,6 @@ static inline void warn_if_node_offline(int this_node, gfp_t gfp_mask) dump_stack(); } -/* - * Allocate pages, preferring the node given as nid. The node must be valid and - * online. For more general interface, see alloc_pages_node(). - */ -static inline struct page * -__alloc_pages_node_noprof(int nid, gfp_t gfp_mask, unsigned int order) -{ - VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES); - warn_if_node_offline(nid, gfp_mask); - - return __alloc_pages_noprof(gfp_mask, order, nid, NULL); -} - -#define __alloc_pages_node(...) alloc_hooks(__alloc_pages_node_noprof(__VA_ARGS__)) - static inline struct folio *__folio_alloc_node_noprof(gfp_t gfp, unsigned int order, int nid) { @@ -293,7 +278,10 @@ static inline struct page *alloc_pages_node_noprof(int nid, gfp_t gfp_mask, if (nid == NUMA_NO_NODE) nid = numa_mem_id(); - return __alloc_pages_node_noprof(nid, gfp_mask, order); + VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES); + warn_if_node_offline(nid, gfp_mask); + + return __alloc_pages_noprof(gfp_mask, order, nid, NULL); } #define alloc_pages_node(...) alloc_hooks(alloc_pages_node_noprof(__VA_ARGS__)) From a4f6c0f83d91ae093900eff55c4c4b0971206899 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Fri, 3 Jul 2026 12:31:54 +0000 Subject: [PATCH 194/501] mm: move __alloc_pages() to mm/page_alloc.h It's no longer used outside of mm/. Since this means __alloc_pages_noprof() is no longer visible from gfp.h, this also means moving the definition of alloc_pages_node_noprof into the .c file. Also remove references to this API from the documentation tree - referring to the specific function name was already questionable but now the function is not even public it definitely seems wrong. Link: https://lore.kernel.org/20260703-alloc-trylock-v5-14-c87b714e19d3@google.com Signed-off-by: Brendan Jackman Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Signed-off-by: Andrew Morton --- Documentation/admin-guide/cgroup-v1/cpusets.rst | 2 +- Documentation/admin-guide/mm/transhuge.rst | 2 +- include/linux/gfp.h | 16 +--------------- mm/page_alloc.c | 13 ++++++++++++- mm/page_alloc.h | 4 ++++ 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/Documentation/admin-guide/cgroup-v1/cpusets.rst b/Documentation/admin-guide/cgroup-v1/cpusets.rst index c7909e5ac136..52a213aff04e 100644 --- a/Documentation/admin-guide/cgroup-v1/cpusets.rst +++ b/Documentation/admin-guide/cgroup-v1/cpusets.rst @@ -284,7 +284,7 @@ take action. ==> Unless this feature is enabled by writing "1" to the special file /dev/cpuset/memory_pressure_enabled, the hook in the rebalance - code of __alloc_pages() for this metric reduces to simply noticing + code of the page allocator for this metric reduces to simply noticing that the cpuset_memory_pressure_enabled flag is zero. So only systems that enable this feature will compute the metric. diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst index 23f8d13c2629..16f37135ed80 100644 --- a/Documentation/admin-guide/mm/transhuge.rst +++ b/Documentation/admin-guide/mm/transhuge.rst @@ -761,7 +761,7 @@ compact_fail but failed. It is possible to establish how long the stalls were using the function -tracer to record how long was spent in __alloc_pages() and +tracer to record how long was spent in the page allocator and using the mm_page_alloc tracepoint to identify which allocations were for huge pages. diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 3bf55a5f9143..4d57e9c0bf20 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -204,10 +204,6 @@ static inline void arch_free_page(struct page *page, int order) { } static inline void arch_alloc_page(struct page *page, int order) { } #endif -struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, int preferred_nid, - nodemask_t *nodemask); -#define __alloc_pages(...) alloc_hooks(__alloc_pages_noprof(__VA_ARGS__)) - struct folio *__folio_alloc_noprof(gfp_t gfp, unsigned int order, int preferred_nid, nodemask_t *nodemask); #define __folio_alloc(...) alloc_hooks(__folio_alloc_noprof(__VA_ARGS__)) @@ -272,17 +268,7 @@ struct folio *__folio_alloc_node_noprof(gfp_t gfp, unsigned int order, int nid) * prefer the current CPU's closest node. Otherwise node must be valid and * online. */ -static inline struct page *alloc_pages_node_noprof(int nid, gfp_t gfp_mask, - unsigned int order) -{ - if (nid == NUMA_NO_NODE) - nid = numa_mem_id(); - - VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES); - warn_if_node_offline(nid, gfp_mask); - - return __alloc_pages_noprof(gfp_mask, order, nid, NULL); -} +struct page *alloc_pages_node_noprof(int nid, gfp_t gfp_mask, unsigned int order); #define alloc_pages_node(...) alloc_hooks(alloc_pages_node_noprof(__VA_ARGS__)) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index c2839959d790..f68b2b138a2e 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5431,7 +5431,18 @@ struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, set_page_refcounted(page); return page; } -EXPORT_SYMBOL(__alloc_pages_noprof); + +struct page *alloc_pages_node_noprof(int nid, gfp_t gfp_mask, unsigned int order) +{ + if (nid == NUMA_NO_NODE) + nid = numa_mem_id(); + + VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES); + warn_if_node_offline(nid, gfp_mask); + + return __alloc_pages_noprof(gfp_mask, order, nid, NULL); +} +EXPORT_SYMBOL(alloc_pages_node_noprof); struct folio *__folio_alloc_noprof(gfp_t gfp, unsigned int order, int preferred_nid, nodemask_t *nodemask) diff --git a/mm/page_alloc.h b/mm/page_alloc.h index 2d60551b4453..aa0c1481f7ca 100644 --- a/mm/page_alloc.h +++ b/mm/page_alloc.h @@ -244,6 +244,10 @@ struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned alloc_hooks(alloc_frozen_pages_nolock_noprof(__VA_ARGS__)) void free_frozen_pages_nolock(struct page *page, unsigned int order); +struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, int preferred_nid, + nodemask_t *nodemask); +#define __alloc_pages(...) alloc_hooks(__alloc_pages_noprof(__VA_ARGS__)) + extern void zone_pcp_reset(struct zone *zone); extern void zone_pcp_disable(struct zone *zone); extern void zone_pcp_enable(struct zone *zone); From 6372aac4d41fcdab6779b19e44544afed0a39cf0 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Fri, 3 Jul 2026 12:31:55 +0000 Subject: [PATCH 195/501] mm: replace __GFP_NO_CODETAG with ALLOC_NO_CODETAG Now that alloc_pages has an entrypoint that allows passing alloc_flags, we can take advantage of this to start removing GFP flags that are only used for mm-internal stuff. This requires also plumbing the alloc_flags into some more of the allocator code, in particular __alloc_pages[_noprof]() gets an alloc_flags arg to go along with its callees, and we now need to pass those flags deeper into the allocator so they can reach the alloc_tag code. While moving the flag definition into page_alloc.h, also update the comment per Hao's suggestion. No functional change intended. Link: https://lore.kernel.org/all/b4916118-3537-4e19-8bc8-1d103dd0d225@linux.dev/ Link: https://lore.kernel.org/20260703-alloc-trylock-v5-15-c87b714e19d3@google.com Signed-off-by: Brendan Jackman Reviewed-by: Vlastimil Babka (SUSE) Tested-by: Hao Ge Acked-by: Hao Ge Signed-off-by: Andrew Morton --- include/linux/alloc_tag.h | 4 +-- mm/alloc_tag.c | 23 ++++++----------- mm/compaction.c | 4 +-- mm/page_alloc.c | 52 ++++++++++++++++++++++----------------- mm/page_alloc.h | 14 +++++++++-- mm/page_frag_cache.c | 4 +-- 6 files changed, 55 insertions(+), 46 deletions(-) diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h index 6ed9f82e639f..7f2d80a59792 100644 --- a/include/linux/alloc_tag.h +++ b/include/linux/alloc_tag.h @@ -165,11 +165,11 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) { WARN_ONCE(ref && !ref->ct, "alloc_tag was not set\n"); } -void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags); +void alloc_tag_add_early_pfn(unsigned long pfn, unsigned int alloc_flags); #else static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag) {} static inline void alloc_tag_sub_check(union codetag_ref *ref) {} -static inline void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags) {} +static inline void alloc_tag_add_early_pfn(unsigned long pfn, unsigned int alloc_flags) {} #endif /* Caller should verify both ref and tag to be valid */ diff --git a/mm/alloc_tag.c b/mm/alloc_tag.c index e5b218176c5a..e93e7fec1f06 100644 --- a/mm/alloc_tag.c +++ b/mm/alloc_tag.c @@ -15,6 +15,9 @@ #include #include +#include "internal.h" +#include "page_alloc.h" + #define ALLOCINFO_FILE_NAME "allocinfo" #define MODULE_ALLOC_TAG_VMAP_SIZE (100000UL * sizeof(struct alloc_tag)) #define SECTION_START(NAME) (CODETAG_SECTION_START_PREFIX NAME) @@ -792,19 +795,6 @@ struct pfn_pool { #define PFN_POOL_SIZE ((PAGE_SIZE - offsetof(struct pfn_pool, pfns)) / \ sizeof(unsigned long)) - -/* - * Skip early PFN recording for a page allocation. Reuses the - * %__GFP_NO_OBJ_EXT bit. Used by __alloc_tag_add_early_pfn() to avoid - * recursion when allocating pages for the early PFN tracking list - * itself. - * - * Codetags of the pages allocated with __GFP_NO_CODETAG should be - * cleared (via clear_page_tag_ref()) before freeing the pages to prevent - * alloc_tag_sub_check() from triggering a warning. - */ -#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT - static struct pfn_pool *current_pfn_pool __initdata; static void __init __alloc_tag_add_early_pfn(unsigned long pfn) @@ -815,7 +805,8 @@ static void __init __alloc_tag_add_early_pfn(unsigned long pfn) do { pool = READ_ONCE(current_pfn_pool); if (!pool || atomic_read(&pool->count) >= PFN_POOL_SIZE) { - struct page *new_page = alloc_page(__GFP_HIGH | __GFP_NO_CODETAG); + struct page *new_page = __alloc_pages(__GFP_HIGH, 0, numa_mem_id(), + NULL, ALLOC_NO_CODETAG); struct pfn_pool *new; if (!new_page) { @@ -846,7 +837,7 @@ typedef void alloc_tag_add_func(unsigned long pfn); static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr __refdata = RCU_INITIALIZER(__alloc_tag_add_early_pfn); -void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags) +void alloc_tag_add_early_pfn(unsigned long pfn, unsigned int alloc_flags) { alloc_tag_add_func *alloc_tag_add; @@ -854,7 +845,7 @@ void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags) return; /* Skip allocations for the tracking list itself to avoid recursion. */ - if (gfp_flags & __GFP_NO_CODETAG) + if (alloc_flags & ALLOC_NO_CODETAG) return; rcu_read_lock(); diff --git a/mm/compaction.c b/mm/compaction.c index 7d80735502d9..4b2318fad4eb 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -83,7 +83,7 @@ static inline bool is_via_compact_memory(int order) { return false; } static struct page *mark_allocated_noprof(struct page *page, unsigned int order, gfp_t gfp_flags) { - post_alloc_hook(page, order, __GFP_MOVABLE); + post_alloc_hook(page, order, __GFP_MOVABLE, ALLOC_DEFAULT); set_page_refcounted(page); return page; } @@ -1851,7 +1851,7 @@ static struct folio *compaction_alloc_noprof(struct folio *src, unsigned long da } dst = (struct folio *)freepage; - post_alloc_hook(&dst->page, order, __GFP_MOVABLE); + post_alloc_hook(&dst->page, order, __GFP_MOVABLE, ALLOC_DEFAULT); set_page_refcounted(&dst->page); if (order) prep_compound_page(&dst->page, order); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index f68b2b138a2e..cfaf16244f56 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1249,7 +1249,7 @@ void __clear_page_tag_ref(struct page *page) /* Should be called only if mem_alloc_profiling_enabled() */ static noinline void __pgalloc_tag_add(struct page *page, struct task_struct *task, - unsigned int nr, gfp_t gfp_flags) + unsigned int nr, unsigned int alloc_flags) { union pgtag_ref_handle handle; union codetag_ref ref; @@ -1263,17 +1263,17 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task, * page_ext is not available yet, record the pfn so we can * clear the tag ref later when page_ext is initialized. */ - alloc_tag_add_early_pfn(page_to_pfn(page), gfp_flags); + alloc_tag_add_early_pfn(page_to_pfn(page), alloc_flags); if (task->alloc_tag) alloc_tag_set_inaccurate(task->alloc_tag); } } static inline void pgalloc_tag_add(struct page *page, struct task_struct *task, - unsigned int nr, gfp_t gfp_flags) + unsigned int nr, unsigned int alloc_flags) { if (mem_alloc_profiling_enabled()) - __pgalloc_tag_add(page, task, nr, gfp_flags); + __pgalloc_tag_add(page, task, nr, alloc_flags); } /* Should be called only if mem_alloc_profiling_enabled() */ @@ -1306,7 +1306,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) #else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add(struct page *page, struct task_struct *task, - unsigned int nr, gfp_t gfp_flags) {} + unsigned int nr, unsigned int alloc_flags) {} static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {} static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) {} @@ -1810,7 +1810,7 @@ static inline bool should_skip_init(gfp_t flags) } inline void post_alloc_hook(struct page *page, unsigned int order, - gfp_t gfp_flags) + gfp_t gfp_flags, unsigned int alloc_flags) { const bool zero_tags = gfp_flags & __GFP_ZEROTAGS; bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags) && @@ -1861,13 +1861,13 @@ inline void post_alloc_hook(struct page *page, unsigned int order, set_page_owner(page, order, gfp_flags); page_table_check_alloc(page, order); - pgalloc_tag_add(page, current, 1 << order, gfp_flags); + pgalloc_tag_add(page, current, 1 << order, alloc_flags); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, unsigned int alloc_flags) { - post_alloc_hook(page, order, gfp_flags); + post_alloc_hook(page, order, gfp_flags, alloc_flags); if (order && (gfp_flags & __GFP_COMP)) prep_compound_page(page, order); @@ -4078,7 +4078,7 @@ __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, */ page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & ~__GFP_DIRECT_RECLAIM, order, - ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); + ac->alloc_flags|ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); if (page) goto out; @@ -4124,7 +4124,7 @@ __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, */ if (gfp_mask & __GFP_NOFAIL) page = __alloc_pages_cpuset_fallback(gfp_mask, order, - ALLOC_NO_WATERMARKS, ac); + ac->alloc_flags|ALLOC_NO_WATERMARKS, ac); } out: mutex_unlock(&oom_lock); @@ -4791,8 +4791,12 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, * The fast path uses conservative alloc_flags to succeed only until * kswapd needs to be woken up, and to avoid the cost of setting up * alloc_flags precisely. So we do that now. + * + * Can't just or alloc_flags if it contains WMARK bits, but those flags + * shouldn't be set in ac->alloc_flags. */ - alloc_flags = alloc_flags_slowpath(gfp_mask, order); + VM_WARN_ON(ac->alloc_flags & ALLOC_WMARK_MASK); + alloc_flags = ac->alloc_flags | alloc_flags_slowpath(gfp_mask, order); /* * We need to recalculate the starting point for the zonelist iterator @@ -4834,7 +4838,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); if (reserve_flags) alloc_flags = alloc_flags_cma(gfp_mask, reserve_flags) | - (alloc_flags & ALLOC_KSWAPD); + ac->alloc_flags | (alloc_flags & ALLOC_KSWAPD); /* * Reset the nodemask and zonelist iterators if memory policies can be @@ -5003,6 +5007,8 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, * we always retry */ if (unlikely(nofail)) { + unsigned int alloc_flags = ac->alloc_flags | ALLOC_MIN_RESERVE; + /* * Lacking direct_reclaim we can't do anything to reclaim memory, * we disregard these unreasonable nofail requests and still @@ -5018,7 +5024,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, * could deplete whole memory reserves which would just make * the situation worse. */ - page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_MIN_RESERVE, ac); + page = __alloc_pages_cpuset_fallback(gfp_mask, order, alloc_flags, ac); if (page) goto got_pg; @@ -5236,7 +5242,7 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid, return nr_populated; failed: - page = __alloc_pages_noprof(gfp, 0, preferred_nid, nodemask); + page = __alloc_pages_noprof(gfp, 0, preferred_nid, nodemask, ALLOC_DEFAULT); if (page) page_array[nr_populated++] = page; goto out; @@ -5344,11 +5350,13 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, { struct page *page; gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */ - struct alloc_context ac = { }; + struct alloc_context ac = { + .alloc_flags = alloc_flags, + }; unsigned int fastpath_alloc_flags = alloc_flags; /* Other flags could be supported later if needed. */ - if (WARN_ON(alloc_flags & ~ALLOC_NOLOCK)) + if (WARN_ON(alloc_flags & ~(ALLOC_NOLOCK | ALLOC_NO_CODETAG))) return NULL; if (!alloc_order_allowed(gfp, order, alloc_flags)) @@ -5421,12 +5429,12 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, EXPORT_SYMBOL(__alloc_frozen_pages_noprof); struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, - int preferred_nid, nodemask_t *nodemask) + int preferred_nid, nodemask_t *nodemask, unsigned int alloc_flags) { struct page *page; page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, nodemask, - ALLOC_DEFAULT); + alloc_flags); if (page) set_page_refcounted(page); return page; @@ -5440,7 +5448,7 @@ struct page *alloc_pages_node_noprof(int nid, gfp_t gfp_mask, unsigned int order VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES); warn_if_node_offline(nid, gfp_mask); - return __alloc_pages_noprof(gfp_mask, order, nid, NULL); + return __alloc_pages_noprof(gfp_mask, order, nid, NULL, ALLOC_DEFAULT); } EXPORT_SYMBOL(alloc_pages_node_noprof); @@ -5448,7 +5456,7 @@ struct folio *__folio_alloc_noprof(gfp_t gfp, unsigned int order, int preferred_ nodemask_t *nodemask) { struct page *page = __alloc_pages_noprof(gfp | __GFP_COMP, order, - preferred_nid, nodemask); + preferred_nid, nodemask, ALLOC_DEFAULT); return page_rmappable_folio(page); } EXPORT_SYMBOL(__folio_alloc_noprof); @@ -7130,7 +7138,7 @@ static void split_free_frozen_pages(struct list_head *list, gfp_t gfp_mask) list_for_each_entry_safe(page, next, &list[order], lru) { int i; - post_alloc_hook(page, order, gfp_mask); + post_alloc_hook(page, order, gfp_mask, ALLOC_DEFAULT); if (!order) continue; @@ -7335,7 +7343,7 @@ int alloc_contig_frozen_range_noprof(unsigned long start, unsigned long end, struct page *head = pfn_to_page(start); check_new_pages(head, order); - prep_new_page(head, order, gfp_mask, 0); + prep_new_page(head, order, gfp_mask, ALLOC_DEFAULT); } else { ret = -EINVAL; WARN(true, "PFN range: requested [%lu, %lu), allocated [%lu, %lu)\n", diff --git a/mm/page_alloc.h b/mm/page_alloc.h index aa0c1481f7ca..b9259deddb59 100644 --- a/mm/page_alloc.h +++ b/mm/page_alloc.h @@ -49,6 +49,13 @@ #define ALLOC_HIGHATOMIC 0x200 /* Allows access to MIGRATE_HIGHATOMIC */ #define ALLOC_NOLOCK 0x400 /* Only use spin_trylock in allocation path */ #define ALLOC_KSWAPD 0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */ +/* + * Avoid alloc_tag recursion for internal allocations. + * + * Callers must clear_page_tag_ref() before freeing to avoid warnings from + * alloc_tag_sub_check(). + */ +#define ALLOC_NO_CODETAG 0x1000 /* Flags that allow allocations below the min watermark. */ #define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM) @@ -84,6 +91,8 @@ struct alloc_context { */ enum zone_type highest_zoneidx; bool spread_dirty_pages; + /* Only flags that are global to the whole allocation go here. */ + unsigned int alloc_flags; }; /* @@ -214,7 +223,8 @@ static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn, extern void __free_pages_core(struct page *page, unsigned int order, enum meminit_context context); -void post_alloc_hook(struct page *page, unsigned int order, gfp_t gfp_flags); +void post_alloc_hook(struct page *page, unsigned int order, gfp_t gfp_flags, + unsigned int alloc_flags); extern bool free_pages_prepare(struct page *page, unsigned int order); extern int user_min_free_kbytes; @@ -245,7 +255,7 @@ struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned void free_frozen_pages_nolock(struct page *page, unsigned int order); struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, int preferred_nid, - nodemask_t *nodemask); + nodemask_t *nodemask, unsigned int alloc_flags); #define __alloc_pages(...) alloc_hooks(__alloc_pages_noprof(__VA_ARGS__)) extern void zone_pcp_reset(struct zone *zone); diff --git a/mm/page_frag_cache.c b/mm/page_frag_cache.c index a1077cef3a79..e63efe78b7d4 100644 --- a/mm/page_frag_cache.c +++ b/mm/page_frag_cache.c @@ -57,10 +57,10 @@ static struct page *__page_frag_cache_refill(struct page_frag_cache *nc, gfp_mask = (gfp_mask & ~__GFP_DIRECT_RECLAIM) | __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC; page = __alloc_pages(gfp_mask, PAGE_FRAG_CACHE_MAX_ORDER, - numa_mem_id(), NULL); + numa_mem_id(), NULL, ALLOC_DEFAULT); #endif if (unlikely(!page)) { - page = __alloc_pages(gfp, 0, numa_mem_id(), NULL); + page = __alloc_pages(gfp, 0, numa_mem_id(), NULL, ALLOC_DEFAULT); order = 0; } From 01c69ef56dfa760651f86a34b243d0ef1d8827b8 Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Fri, 3 Jul 2026 12:31:56 +0000 Subject: [PATCH 196/501] mm: remove the __GFP_NO_OBJ_EXT flag All users of the flag are converted to SLAB_ALLOC_NO_RECURSE or ALLOC_NO_CODETAG (from __GFP_NO_CODETAG which reused the NO_OBJ_EXT bit). Free up the flag bit. [Rebased onto __GFP_NO_CODETAG removal] Link: https://lore.kernel.org/20260703-alloc-trylock-v5-16-c87b714e19d3@google.com Signed-off-by: Vlastimil Babka (SUSE) Signed-off-by: Brendan Jackman Acked-by: Hao Ge Signed-off-by: Andrew Morton --- include/linux/gfp_types.h | 7 ------- include/trace/events/mmflags.h | 10 +--------- tools/include/linux/gfp_types.h | 7 ------- 3 files changed, 1 insertion(+), 23 deletions(-) diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h index 463b551d12d9..190191411009 100644 --- a/include/linux/gfp_types.h +++ b/include/linux/gfp_types.h @@ -55,7 +55,6 @@ enum { #ifdef CONFIG_LOCKDEP ___GFP_NOLOCKDEP_BIT, #endif - ___GFP_NO_OBJ_EXT_BIT, ___GFP_LAST_BIT }; @@ -96,7 +95,6 @@ enum { #else #define ___GFP_NOLOCKDEP 0 #endif -#define ___GFP_NO_OBJ_EXT BIT(___GFP_NO_OBJ_EXT_BIT) /* * Physical address zone modifiers (see linux/mmzone.h - low four bits) @@ -138,17 +136,12 @@ enum { * * %__GFP_ACCOUNT causes the allocation to be accounted to the active * cgroup context. - * - * %__GFP_NO_OBJ_EXT causes slab allocation to have no object extension. - * mark_obj_codetag_empty() should be called upon freeing for objects allocated - * with this flag to indicate that their NULL tags are expected and normal. */ #define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE) #define __GFP_WRITE ((__force gfp_t)___GFP_WRITE) #define __GFP_HARDWALL ((__force gfp_t)___GFP_HARDWALL) #define __GFP_THISNODE ((__force gfp_t)___GFP_THISNODE) #define __GFP_ACCOUNT ((__force gfp_t)___GFP_ACCOUNT) -#define __GFP_NO_OBJ_EXT ((__force gfp_t)___GFP_NO_OBJ_EXT) /** * DOC: Watermark modifiers diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h index a6e5a44c9b42..c1a05ff0feab 100644 --- a/include/trace/events/mmflags.h +++ b/include/trace/events/mmflags.h @@ -54,18 +54,10 @@ # define TRACE_GFP_FLAGS_LOCKDEP #endif -#ifdef CONFIG_SLAB_OBJ_EXT -# define TRACE_GFP_FLAGS_SLAB \ - TRACE_GFP_EM(NO_OBJ_EXT) -#else -# define TRACE_GFP_FLAGS_SLAB -#endif - #define TRACE_GFP_FLAGS \ TRACE_GFP_FLAGS_GENERAL \ TRACE_GFP_FLAGS_KASAN \ - TRACE_GFP_FLAGS_LOCKDEP \ - TRACE_GFP_FLAGS_SLAB + TRACE_GFP_FLAGS_LOCKDEP #undef TRACE_GFP_EM #define TRACE_GFP_EM(a) TRACE_DEFINE_ENUM(___GFP_##a##_BIT); diff --git a/tools/include/linux/gfp_types.h b/tools/include/linux/gfp_types.h index 54ca0c88bab6..da47aa089f01 100644 --- a/tools/include/linux/gfp_types.h +++ b/tools/include/linux/gfp_types.h @@ -55,7 +55,6 @@ enum { #ifdef CONFIG_LOCKDEP ___GFP_NOLOCKDEP_BIT, #endif - ___GFP_NO_OBJ_EXT_BIT, ___GFP_LAST_BIT }; @@ -96,7 +95,6 @@ enum { #else #define ___GFP_NOLOCKDEP 0 #endif -#define ___GFP_NO_OBJ_EXT BIT(___GFP_NO_OBJ_EXT_BIT) /* * Physical address zone modifiers (see linux/mmzone.h - low four bits) @@ -137,17 +135,12 @@ enum { * node with no fallbacks or placement policy enforcements. * * %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg. - * - * %__GFP_NO_OBJ_EXT causes slab allocation to have no object extension. - * mark_obj_codetag_empty() should be called upon freeing for objects allocated - * with this flag to indicate that their NULL tags are expected and normal. */ #define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE) #define __GFP_WRITE ((__force gfp_t)___GFP_WRITE) #define __GFP_HARDWALL ((__force gfp_t)___GFP_HARDWALL) #define __GFP_THISNODE ((__force gfp_t)___GFP_THISNODE) #define __GFP_ACCOUNT ((__force gfp_t)___GFP_ACCOUNT) -#define __GFP_NO_OBJ_EXT ((__force gfp_t)___GFP_NO_OBJ_EXT) /** * DOC: Watermark modifiers From 5df11ba0eb5ae3c4e90edb291c5be7fbfde33076 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Fri, 3 Jul 2026 12:31:57 +0000 Subject: [PATCH 197/501] mm/page_alloc: drop alloc_flags arg from alloc_flags_cma() To align the style with other alloc_flags_*() functions, drop this additive argument and just have the callers do that themselves. Note you can't always freely or alloc_flags like these callers do (because of the WMARK bits that encode an enum) but this is fine for ALLOC_CMA, just like it's fine for e.g. ALLOC_NON_BLOCK returned by alloc_flags_nonblocking() and or'd by its caller. Link: https://lore.kernel.org/20260703-alloc-trylock-v5-17-c87b714e19d3@google.com Signed-off-by: Brendan Jackman Suggested-by: Vlastimil Babka (SUSE) Link: https://lore.kernel.org/all/5dcdd1ef-21ad-4ed0-9e8a-0e5cf96b4392@kernel.org/ Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Signed-off-by: Andrew Morton --- mm/page_alloc.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index cfaf16244f56..c3b246e67ed1 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -3775,14 +3775,13 @@ alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) } /* Must be called after current_gfp_context() which can change gfp_mask */ -static inline unsigned int alloc_flags_cma(gfp_t gfp_mask, - unsigned int alloc_flags) +static inline unsigned int alloc_flags_cma(gfp_t gfp_mask) { #ifdef CONFIG_CMA if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) - alloc_flags |= ALLOC_CMA; + return ALLOC_CMA; #endif - return alloc_flags; + return ALLOC_DEFAULT; } /* @@ -4526,7 +4525,7 @@ alloc_flags_slowpath(gfp_t gfp_mask, unsigned int order) } else if (unlikely(rt_or_dl_task(current)) && in_task()) alloc_flags |= ALLOC_MIN_RESERVE; - alloc_flags = alloc_flags_cma(gfp_mask, alloc_flags); + alloc_flags |= alloc_flags_cma(gfp_mask); if (defrag_mode) alloc_flags |= ALLOC_NOFRAGMENT; @@ -4837,7 +4836,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); if (reserve_flags) - alloc_flags = alloc_flags_cma(gfp_mask, reserve_flags) | + alloc_flags = alloc_flags_cma(gfp_mask) | reserve_flags | ac->alloc_flags | (alloc_flags & ALLOC_KSWAPD); /* @@ -5070,7 +5069,7 @@ static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, should_fail_alloc_page(gfp_mask, order)) return false; - *alloc_flags = alloc_flags_cma(gfp_mask, *alloc_flags); + *alloc_flags |= alloc_flags_cma(gfp_mask); /* Dirty zone balancing only done in the fast path */ ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); From 3a01af828138041eb4bd27403ad3bd25f8e89790 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Fri, 3 Jul 2026 12:31:58 +0000 Subject: [PATCH 198/501] mm: factor out can_spin_trylock() Deduplicate checks for whether the current context is safe for spin_trylock(). Does this function really belong in mm/internal.h or is it generic? Not sure. If someone ends up duplicating this logic elsewhere in the kernel, that would be a shame. But if it goes in some generic header, someone treats it as documentation about where it's guaranteed safe to spin_trylock(), and then it emerges that there are other subtle preconditions that didn't affect the mm usecase, that would be worse. So, just be conservative and keep it local. Link: https://lore.kernel.org/20260703-alloc-trylock-v5-18-c87b714e19d3@google.com Signed-off-by: Brendan Jackman Suggested-by: Harry Yoo Link: https://lore.kernel.org/all/397859cb-b127-4cc6-9c71-044afc99bf0c@kernel.org/ Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Harry Yoo (Oracle) Reviewed-by: Zi Yan Signed-off-by: Andrew Morton --- mm/internal.h | 23 +++++++++++++++++++++++ mm/page_alloc.c | 17 +---------------- mm/slub.c | 10 +--------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index ffa72bd31708..841c27611627 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1713,4 +1713,27 @@ static inline void mm_prepare_for_swap_entries(struct mm_struct *mm) } } +static inline bool can_spin_trylock(void) +{ + /* + * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is + * unsafe in NMI. If spin_trylock() is called from hard IRQ the current + * task may be waiting for one rt_spin_lock, but rt_spin_trylock() will + * mark the task as the owner of another rt_spin_lock which will + * confuse PI logic, so return immediately if called from hard IRQ or + * NMI. + * + * Note, irqs_disabled() case is ok. spin_trylock() can be called + * from raw_spin_lock_irqsave region. + */ + if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq())) + return false; + + /* On UP, spin_trylock() always succeeds even when it is locked */ + if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) + return false; + + return true; +} + #endif /* __MM_INTERNAL_H */ diff --git a/mm/page_alloc.c b/mm/page_alloc.c index c3b246e67ed1..a63733dac659 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5291,22 +5291,7 @@ static inline bool alloc_order_allowed(gfp_t gfp, unsigned int order, static inline bool alloc_nolock_allowed(void) { - /* - * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is - * unsafe in NMI. If spin_trylock() is called from hard IRQ the current - * task may be waiting for one rt_spin_lock, but rt_spin_trylock() will - * mark the task as the owner of another rt_spin_lock which will - * confuse PI logic, so return immediately if called from hard IRQ or - * NMI. - * - * Note, irqs_disabled() case is ok. This function can be called - * from raw_spin_lock_irqsave region. - */ - if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq())) - return false; - - /* On UP, spin_trylock() always succeeds even when it is locked */ - if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) + if (!can_spin_trylock()) return false; /* Bailout, since _deferred_grow_zone() needs to take a lock */ diff --git a/mm/slub.c b/mm/slub.c index 85599d227b9b..422bc3e12c02 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -5380,15 +5380,7 @@ static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_f if (unlikely(!size)) return ZERO_SIZE_PTR; - /* - * See the comment for the same check in - * alloc_frozen_pages_nolock_noprof() - */ - if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq())) - return NULL; - - /* On UP, spin_trylock() always succeeds even when it is locked */ - if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) + if (!can_spin_trylock()) return NULL; retry: From f5407e9b697c940e78b27ff63c6e14d8d171adc3 Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Fri, 3 Jul 2026 11:41:55 +0000 Subject: [PATCH 199/501] mm/rmap: use huge_ptep_get() in try_to_unmap_one() Patch series "Fix incorrect access of hugetlb pte entries", v3. There are various places which use ptep_get() to get the pte entry corresponding to a hugetlb folio. Some arches (like s390) have special handling to compute the pteval, so they provide huge_ptep_get(). Use this helper consistently. Additionally, some code paths may provide huge_ptep_get with an unaligned address. This is a problem on arm64 (I checked other arches and it looks fine for them), which is fixed in patch 1. The fix is made to be backport-friendly: the cleaner fix would be to perhaps pass the hstate to huge_ptep_get() - that is wider churn and we can do that later. This patch (of 5): try_to_unmap_one() handles hugetlb folios when memory failure needs to replace a poisoned hugetlb mapping with a hwpoison entry. In that case page_vma_mapped_walk() returns the pte pointer to the hugetlb folio in pvmw.pte, but the code reads it with ptep_get(). On arches which provide their own huge_ptep_get() to dereference a huge pte pointer, accessing via ptep_get() would cause pte_pfn(), pte_present() etc to misbehave. It is not clear whether this has a trivially visible effect to userspace. Just use huge_ptep_get() for dereferencing a huge pte pointer. Link: https://lore.kernel.org/20260703114202.365553-1-dev.jain@arm.com Link: https://lore.kernel.org/20260703114202.365553-3-dev.jain@arm.com Fixes: c7ab0d2fdc84 ("mm: convert try_to_unmap_one() to use page_vma_mapped_walk()") Signed-off-by: Dev Jain Reported-by: David Hildenbrand Reviewed-by: Muchun Song Acked-by: David Hildenbrand (Arm) Cc: Alistair Popple Cc: Andi Kleen Cc: Anshuman Khandual Cc: Byungchul Park Cc: Catalin Marinas Cc: Dave Hansen Cc: Gregory Price Cc: Harry Yoo Cc: "Huang, Ying" Cc: Jann Horn Cc: Josh Poimboeuf Cc: Joshua Hahn Cc: Jun'ichi "Nick" Nomura Cc: Kiryl Shutsemau Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Brost Cc: Mel Gorman Cc: Naoya Horiguchi Cc: Oscar Salvador Cc: Pedro Falcato Cc: Rakie Kim Cc: Ralph Campbell Cc: Rik van Riel Cc: Ryan Roberts Cc: Vlastimil Babka Cc: Will Deacon Cc: Zi Yan Cc: Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 2 ++ mm/rmap.c | 16 ++++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 817b8b4223fa..95b1b6ee1f96 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -1270,6 +1270,8 @@ static inline void hugetlb_count_sub(long l, struct mm_struct *mm) { } +pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep); + static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { diff --git a/mm/rmap.c b/mm/rmap.c index 408e542649b5..e0d7b33ceee4 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2095,14 +2095,19 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, /* Unexpected PMD-mapped THP? */ VM_BUG_ON_FOLIO(!pvmw.pte, folio); - /* - * Handle PFN swap PTEs, such as device-exclusive ones, that - * actually map pages. - */ - pteval = ptep_get(pvmw.pte); + address = pvmw.address; + if (folio_test_hugetlb(folio)) { + pteval = huge_ptep_get(mm, address, pvmw.pte); + } else { + pteval = ptep_get(pvmw.pte); + } if (likely(pte_present(pteval))) { pfn = pte_pfn(pteval); } else { + /* + * Handle PFN swap PTEs, such as device-exclusive ones, + * that actually map pages. + */ const softleaf_t entry = softleaf_from_pte(pteval); pfn = softleaf_to_pfn(entry); @@ -2110,7 +2115,6 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, } subpage = folio_page(folio, pfn - folio_pfn(folio)); - address = pvmw.address; anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(subpage); From 9be12ffa78cd087d41086f937a156435d4e588e9 Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Fri, 3 Jul 2026 11:41:56 +0000 Subject: [PATCH 200/501] mm/rmap: use huge_ptep_get() in try_to_migrate_one() try_to_migrate_one() is used by folio migration to replace a present mapping with a migration entry. For hugetlb folios, page_vma_mapped_walk() returns the pte pointer to the hugetlb folio in pvmw.pte, but the code reads the huge pte entry with ptep_get(). On arches which provide their own huge_ptep_get() to dereference a huge pte pointer, accessing via ptep_get() would cause pte_pfn(), pte_present() etc to misbehave. It is not clear whether this has a trivially visible effect to userspace. Use huge_ptep_get() to dereference a huge pte pointer. Commit a98a2f0c8ce1 copied the bug from try_to_unmap_one into try_to_migrate_one. [akpm@linux-foundation.org: coding-style cleanups] Link: https://lore.kernel.org/20260703114202.365553-4-dev.jain@arm.com Fixes: a98a2f0c8ce1 ("mm/rmap: split migration into its own function") Signed-off-by: Dev Jain Acked-by: Muchun Song Acked-by: David Hildenbrand (Arm) Cc: Alistair Popple Cc: Andi Kleen Cc: Anshuman Khandual Cc: Byungchul Park Cc: Catalin Marinas Cc: Dave Hansen Cc: Gregory Price Cc: Harry Yoo Cc: "Huang, Ying" Cc: Jann Horn Cc: Josh Poimboeuf Cc: Joshua Hahn Cc: Jun'ichi "Nick" Nomura Cc: Kiryl Shutsemau Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Brost Cc: Mel Gorman Cc: Naoya Horiguchi Cc: Oscar Salvador Cc: Pedro Falcato Cc: Rakie Kim Cc: Ralph Campbell Cc: Rik van Riel Cc: Ryan Roberts Cc: Vlastimil Babka Cc: Will Deacon Cc: Zi Yan Cc: Signed-off-by: Andrew Morton --- mm/rmap.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/mm/rmap.c b/mm/rmap.c index e0d7b33ceee4..9939400e77c7 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2500,14 +2500,18 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, /* Unexpected PMD-mapped THP? */ VM_BUG_ON_FOLIO(!pvmw.pte, folio); - /* - * Handle PFN swap PTEs, such as device-exclusive ones, that - * actually map pages. - */ - pteval = ptep_get(pvmw.pte); + address = pvmw.address; + if (folio_test_hugetlb(folio)) + pteval = huge_ptep_get(mm, address, pvmw.pte); + else + pteval = ptep_get(pvmw.pte); if (likely(pte_present(pteval))) { pfn = pte_pfn(pteval); } else { + /* + * Handle PFN swap PTEs, such as device-exclusive ones, + * that actually map pages. + */ const softleaf_t entry = softleaf_from_pte(pteval); pfn = softleaf_to_pfn(entry); @@ -2515,7 +2519,6 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, } subpage = folio_page(folio, pfn - folio_pfn(folio)); - address = pvmw.address; anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(subpage); From ac1ec50d71b953524100ddc09057ee61a3dfaff3 Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Fri, 3 Jul 2026 11:41:57 +0000 Subject: [PATCH 201/501] mm/migrate: use huge_ptep_get() in remove_migration_pte() remove_migration_pte() converts migration entries back to present PTEs after folio migration completes. For hugetlb folios, page_vma_mapped_walk() returns the pte pointer to the hugetlb folio in pvmw.pte, but the code reads it with ptep_get(). On arches which provide their own huge_ptep_get() to dereference a huge pte pointer, accessing via ptep_get() would cause pte_pfn(), pte_present() etc to misbehave. It is not clear whether this has a trivially visible effect to userspace. Use huge_ptep_get() to dereference a huge pte pointer. Link: https://lore.kernel.org/20260703114202.365553-5-dev.jain@arm.com Fixes: 290408d4a250 ("hugetlb: hugepage migration core") Signed-off-by: Dev Jain Acked-by: Muchun Song Acked-by: David Hildenbrand (Arm) Cc: Alistair Popple Cc: Andi Kleen Cc: Anshuman Khandual Cc: Byungchul Park Cc: Catalin Marinas Cc: Dave Hansen Cc: Gregory Price Cc: Harry Yoo Cc: "Huang, Ying" Cc: Jann Horn Cc: Josh Poimboeuf Cc: Joshua Hahn Cc: Jun'ichi "Nick" Nomura Cc: Kiryl Shutsemau Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Brost Cc: Mel Gorman Cc: Naoya Horiguchi Cc: Oscar Salvador Cc: Pedro Falcato Cc: Rakie Kim Cc: Ralph Campbell Cc: Rik van Riel Cc: Ryan Roberts Cc: Vlastimil Babka Cc: Will Deacon Cc: Zi Yan Cc: Signed-off-by: Andrew Morton --- mm/migrate.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mm/migrate.c b/mm/migrate.c index 1977905ce12c..b6f56b67e367 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -372,7 +372,11 @@ static bool remove_migration_pte(struct folio *folio, continue; } #endif - old_pte = ptep_get(pvmw.pte); + if (folio_test_hugetlb(folio)) + old_pte = huge_ptep_get(vma->vm_mm, pvmw.address, + pvmw.pte); + else + old_pte = ptep_get(pvmw.pte); if (rmap_walk_arg->map_unused_to_zeropage && try_to_map_unused_to_zeropage(&pvmw, folio, old_pte, idx)) continue; From e87df0d5d6962bde50f55f6d02b779daa394f894 Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Fri, 3 Jul 2026 11:41:58 +0000 Subject: [PATCH 202/501] mm/page_vma_mapped: use huge_ptep_get() for hugetlb check_pte() is the final validation step in page_vma_mapped_walk(). It reads pvmw->pte with ptep_get() to decide whether the entry maps the PFN range being walked. For hugetlb VMAs, that pointer refers to a hugetlb entry. On arches which provide their own huge_ptep_get() to dereference a huge pte pointer, accessing via ptep_get() would cause pte_pfn(), pte_present() etc to misbehave. It is not clear whether this has a trivially visible effect to userspace. Use huge_ptep_get() to dereference a huge pte pointer. Link: https://lore.kernel.org/20260703114202.365553-6-dev.jain@arm.com Fixes: ace71a19cec5 ("mm: introduce page_vma_mapped_walk()") Signed-off-by: Dev Jain Acked-by: David Hildenbrand (Arm) Reviewed-by: Muchun Song Cc: Alistair Popple Cc: Andi Kleen Cc: Anshuman Khandual Cc: Byungchul Park Cc: Catalin Marinas Cc: Dave Hansen Cc: Gregory Price Cc: Harry Yoo Cc: "Huang, Ying" Cc: Jann Horn Cc: Josh Poimboeuf Cc: Joshua Hahn Cc: Jun'ichi "Nick" Nomura Cc: Kiryl Shutsemau Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Brost Cc: Mel Gorman Cc: Naoya Horiguchi Cc: Oscar Salvador Cc: Pedro Falcato Cc: Rakie Kim Cc: Ralph Campbell Cc: Rik van Riel Cc: Ryan Roberts Cc: Vlastimil Babka Cc: Will Deacon Cc: Zi Yan Cc: Signed-off-by: Andrew Morton --- mm/page_vma_mapped.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index bac2eb5de63d..d7670ba4147b 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -107,7 +107,13 @@ static bool map_pte(struct page_vma_mapped_walk *pvmw, pmd_t *pmdvalp, static bool check_pte(struct page_vma_mapped_walk *pvmw, unsigned long pte_nr) { unsigned long pfn; - pte_t ptent = ptep_get(pvmw->pte); + pte_t ptent; + + if (is_vm_hugetlb_page(pvmw->vma)) + ptent = huge_ptep_get(pvmw->vma->vm_mm, pvmw->address, + pvmw->pte); + else + ptent = ptep_get(pvmw->pte); if (pvmw->flags & PVMW_MIGRATION) { const softleaf_t entry = softleaf_from_pte(ptent); From 399cfd133f88ec33881832053f9d162c4b48db43 Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Fri, 3 Jul 2026 11:41:59 +0000 Subject: [PATCH 203/501] mm/mprotect: use huge_ptep_get() for hugetlb prot_none_hugetlb_entry() is the hugetlb callback for the early mprotect(PROT_NONE) PFN permission walk on x86. The callback passes the decoded PFN to pfn_modify_allowed(). For a hugetlb callback, the pte pointer refers to a hugetlb entry. On architectures where hugetlb entries need huge_ptep_get(), reading that entry with ptep_get() can make the permission check use the wrong PFN. Use huge_ptep_get() before decoding the hugetlb PFN. Currently there is no path which can trigger a bug: huge_ptep_get() is a simple ptep_get() for x86, and the prot_none walk occurs only for x86. So no need to backport - use the correct helper anyways. [akpm@linux-foundation.org: s/EACCESS/EACCES/] Link: https://lore.kernel.org/20260703114202.365553-7-dev.jain@arm.com Fixes: 42e4089c7890 ("x86/speculation/l1tf: Disallow non privileged high MMIO PROT_NONE mappings") Signed-off-by: Dev Jain Reviewed-by: Muchun Song Acked-by: David Hildenbrand (Arm) Cc: Alistair Popple Cc: Andi Kleen Cc: Anshuman Khandual Cc: Byungchul Park Cc: Catalin Marinas Cc: Dave Hansen Cc: Gregory Price Cc: Harry Yoo Cc: "Huang, Ying" Cc: Jann Horn Cc: Josh Poimboeuf Cc: Joshua Hahn Cc: Jun'ichi "Nick" Nomura Cc: Kiryl Shutsemau Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Brost Cc: Mel Gorman Cc: Naoya Horiguchi Cc: Oscar Salvador Cc: Pedro Falcato Cc: Rakie Kim Cc: Ralph Campbell Cc: Rik van Riel Cc: Ryan Roberts Cc: Vlastimil Babka Cc: Will Deacon Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/mprotect.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mm/mprotect.c b/mm/mprotect.c index 8665a23f38d3..cec2a1eed539 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -699,14 +699,20 @@ static int prot_none_pte_entry(pte_t *pte, unsigned long addr, 0 : -EACCES; } +#ifdef CONFIG_HUGETLB_PAGE static int prot_none_hugetlb_entry(pte_t *pte, unsigned long hmask, unsigned long addr, unsigned long next, struct mm_walk *walk) { - return pfn_modify_allowed(pte_pfn(ptep_get(pte)), - *(pgprot_t *)(walk->private)) ? - 0 : -EACCES; + const pte_t entry = huge_ptep_get(walk->mm, addr, pte); + + if (pfn_modify_allowed(pte_pfn(entry), *(pgprot_t *)(walk->private))) + return 0; + return -EACCES; } +#else +#define prot_none_hugetlb_entry NULL +#endif static const struct mm_walk_ops prot_none_walk_ops = { .pte_entry = prot_none_pte_entry, From a3982653014631bbdc5b63d4b3bb5defbce998ef Mon Sep 17 00:00:00 2001 From: Arukonda Rahul Date: Tue, 7 Jul 2026 22:21:31 +0530 Subject: [PATCH 204/501] docs: ABI: zram: fix spelling mistakes Fix spelling mistakes in the zram sysfs ABI documentation. Link: https://lore.kernel.org/20260707165131.4408-1-rahularukonda2@gmail.com Signed-off-by: Arukonda Rahul Reviewed-by: Sergey Senozhatsky Cc: Minchan Kim Cc: Richard Chang Cc: Jonathan Corbet Signed-off-by: Andrew Morton --- Documentation/ABI/testing/sysfs-block-zram | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-block-zram b/Documentation/ABI/testing/sysfs-block-zram index 64c03010e951..1ea9018a627f 100644 --- a/Documentation/ABI/testing/sysfs-block-zram +++ b/Documentation/ABI/testing/sysfs-block-zram @@ -152,15 +152,15 @@ Description: compression algorithm parameters. What: /sys/block/zram/compressed_writeback -Date: Decemeber 2025 +Date: December 2025 Contact: Richard Chang Description: - The compressed_writeback device atrribute toggles compressed + The compressed_writeback device attribute toggles compressed writeback feature. What: /sys/block/zram/writeback_batch_size Date: November 2025 Contact: Sergey Senozhatsky Description: - The writeback_batch_size device atrribute sets the maximum + The writeback_batch_size device attribute sets the maximum number of in-flight writeback operations. From b90408ef116305770142b6d0d3cb077f28d8d41b Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 5 Jul 2026 08:55:52 -0700 Subject: [PATCH 205/501] mm/damon/core: safely validate src on damon_commit_ctx() Patch series "mm/damon: validate all parameters in the core". DAMON has a number of parameters. Some of the parameters are validated by DAMON, while some are validated by DAMON API callers. Each caller has their own set of parameters that are exposed to users. Hence each caller has their own ways to do the validation. There is no clear lines for the responsibilities. It is confusing and easy to make mistakes at validations. Actually we found a few bugs in the class. Define DAMON core as the place to do all the validations and implement those. damon_set_regions(), damon_start() and damon_commit_ctx() are the three main DAMON core API functions for setup of DAMON parameters. Make the three functions to do safe and holistic parameter checks. The first one is already providing the validation, so changes are only for the last two functions. This might add unnecessary validations for some use cases. The overhead should be negligible as parameters update is expected to only rarely happen. It reduces the number of places to check and fix for bugs of the class from all callers to the single component. The maintenance efficiency gain is obvious. Further cleanup documents and caller code. Make the lines of validation duties clearer. Remove validations that are no more needed owing to the core validations. Patches Sequence ================ Patch 1 implements the core holistic parameters validation for damon_commit_ctx(). Patch 2 extends the validation to damon_start(). Patch 3 removes the duplicated holistic parameters validation in DAMON sysfs interface, which is now embedded into the core layer. Patches 4 and 5 remove duplicated min_region_sz validation in DAMON modules. Patches 6 updates kernel doc to clarify damon_set_regions() is doing the range validation. Patches 7 and 8 remove monitoring target range validations that doesn't necessary thanks to the validation in damon_set_regions(). This patch (of 8): damon_commit_ctx() does its holistic parameter set validation while applying the new parameter in the set one by one. If it finds a parameter is invalid, because some invalid parameters may already be committed (it is called "commit" but not atomic and irreversable), it stops the running DAMON context. The callers of the function therefore have to validate the parameters before calling it. Because the function already embeds holistic validation, DAMON_SYSFS reuses it in a safe way. It creates a test-purpose context that is not running but mimics the running one, and calls damon_commit_ctx() against the test purpose context. If it succeeds, the parameters are considered valid, and a real damon_commit_ctx() call against the running context is made with those. Other callers such as DAMON_RECLAIM and DAMON_LRU_SORT do not expose full parameters to users. For efficiency, they validate only the known set of parameters. The efficiency gain is arguably small and doubtful, though. Meanwhile the maintenance overhead of the multiple different validations is clearly high. We actually found and fixed a few bugs in the class. Update damon_commit_ctx() to embed DAMON_SYSFS' safe and holistic validation approach. Callers can simply call damon_commit_ctx() without worrying if their parameters are invalid. Note that damon_commit_ctx() can still cause an unexpected stop of the running context, if internal memory allocation fails. It is arguably unlikely since those internal allocations are too small to fail, but theoretically possible. It should also be better addressed, but not necessarily a blocker of this small and incremental improvement effort. Link: https://lore.kernel.org/20260705155600.96555-1-sj@kernel.org Link: https://lore.kernel.org/20260705155600.96555-2-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/core.c | 61 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 871c6f5257c9..018dd5ff8032 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1664,20 +1664,7 @@ static int damon_commit_probes(struct damon_ctx *dst, struct damon_ctx *src) return 0; } -/** - * damon_commit_ctx() - Commit parameters of a DAMON context to another. - * @dst: The commit destination DAMON context. - * @src: The commit source DAMON context. - * - * This function copies user-specified parameters from @src to @dst and update - * the internal status and results accordingly. Users should use this function - * for context-level parameters update of running context, instead of manual - * in-place updates. - * - * This function should be called from parameters-update safe context, like - * damon_call(). - */ -int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) +static int __damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) { int err; struct damos *scheme; @@ -1732,6 +1719,52 @@ int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) return 0; } +static struct damon_ctx *damon_new_test_ctx(struct damon_ctx *dst) +{ + struct damon_ctx *test_ctx; + int err; + + test_ctx = damon_new_ctx(); + if (!test_ctx) + return NULL; + err = __damon_commit_ctx(test_ctx, dst); + if (err) { + damon_destroy_ctx(test_ctx); + return NULL; + } + return test_ctx; +} + +/** + * damon_commit_ctx() - Commit parameters of a DAMON context to another. + * @dst: The commit destination DAMON context. + * @src: The commit source DAMON context. + * + * This function copies user-specified parameters from @src to @dst and update + * the internal status and results accordingly. Users should use this function + * for context-level parameters update of running context, instead of manual + * in-place updates. + * + * This function should be called from parameters-update safe context, like + * damon_call(). + */ +int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) +{ + struct damon_ctx *test_ctx; + int err; + + test_ctx = damon_new_test_ctx(dst); + if (!test_ctx) + return -ENOMEM; + err = __damon_commit_ctx(test_ctx, src); + if (err) + goto out; + err = __damon_commit_ctx(dst, src); +out: + damon_destroy_ctx(test_ctx); + return err; +} + /** * damon_nr_running_ctxs() - Return number of currently running contexts. */ From b1471afe4d10eeea7c8f5935b52a3b2cac8ad779 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 5 Jul 2026 08:55:53 -0700 Subject: [PATCH 206/501] mm/damon/core: do parameter testing commit on damon_start() damon_start() and damon_commit_ctx() are two main DAMON core API functions for setting whole DAMON parameters. While damon_commit_ctx() does holistic parameters testing, damon_start() just believes the caller validated the whole thing. Embed the holistic parameter check that is already in damon_commit_ctx() into damon_start(). After this change, the callers can safely call damon_start() without validating the parameters. Link: https://lore.kernel.org/20260705155600.96555-3-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/core.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 018dd5ff8032..ca301abcb9ec 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1865,6 +1865,8 @@ static int __damon_start(struct damon_ctx *ctx) return err; } +static int __damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src); + /** * damon_start() - Starts the monitorings for a given group of contexts. * @ctxs: an array of the pointers for contexts to start monitoring @@ -1886,8 +1888,16 @@ int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive) int err = 0; for (i = 0; i < nr_ctxs; i++) { - if (!is_power_of_2(ctxs[i]->min_region_sz)) - return -EINVAL; + struct damon_ctx *test_ctx; + + test_ctx = damon_new_ctx(); + if (!test_ctx) + return -ENOMEM; + + err = __damon_commit_ctx(test_ctx, ctxs[i]); + damon_destroy_ctx(test_ctx); + if (err) + return err; } mutex_lock(&damon_lock); From fa93d1556238881aff0a56a963b89604efacbe4d Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 5 Jul 2026 08:55:54 -0700 Subject: [PATCH 207/501] mm/damon/sysfs: remove duplicated commit input validity check DAMON sysfs interface does parameters validation-purpose damon_commit_ctx() calls for parameters update. Now the same logic is embedded inside damon_commit_ctx() itself. Hence, the validation in DAMON sysfs interface is just an unnecessary duplicate. Remove it. Link: https://lore.kernel.org/20260705155600.96555-4-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/sysfs.c | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 6710b6d019bf..e666dddf1feb 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -2098,26 +2098,6 @@ static int damon_sysfs_apply_inputs(struct damon_ctx *ctx, static struct damon_ctx *damon_sysfs_build_ctx( struct damon_sysfs_context *sys_ctx); -/* - * Return a new damon_ctx for testing new parameters to commit. - */ -static struct damon_ctx *damon_sysfs_new_test_ctx( - struct damon_ctx *running_ctx) -{ - struct damon_ctx *test_ctx; - int err; - - test_ctx = damon_new_ctx(); - if (!test_ctx) - return NULL; - err = damon_commit_ctx(test_ctx, running_ctx); - if (err) { - damon_destroy_ctx(test_ctx); - return NULL; - } - return test_ctx; -} - /* * damon_sysfs_commit_input() - Commit user inputs to a running kdamond. * @kdamond: The kobject wrapper for the associated kdamond. @@ -2127,7 +2107,7 @@ static struct damon_ctx *damon_sysfs_new_test_ctx( static int damon_sysfs_commit_input(void *data) { struct damon_sysfs_kdamond *kdamond = data; - struct damon_ctx *param_ctx, *test_ctx; + struct damon_ctx *param_ctx; int err; if (!damon_sysfs_kdamond_running(kdamond)) @@ -2139,17 +2119,7 @@ static int damon_sysfs_commit_input(void *data) param_ctx = damon_sysfs_build_ctx(kdamond->contexts->contexts_arr[0]); if (IS_ERR(param_ctx)) return PTR_ERR(param_ctx); - test_ctx = damon_sysfs_new_test_ctx(kdamond->damon_ctx); - if (!test_ctx) { - damon_destroy_ctx(param_ctx); - return -ENOMEM; - } - err = damon_commit_ctx(test_ctx, param_ctx); - if (err) - goto out; err = damon_commit_ctx(kdamond->damon_ctx, param_ctx); -out: - damon_destroy_ctx(test_ctx); damon_destroy_ctx(param_ctx); return err; } From 529ba074821bea8aad5079a168c58aa7850be18f Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 5 Jul 2026 08:55:55 -0700 Subject: [PATCH 208/501] mm/damon/reclaim: remove duplicated min_region_sz power of 2 check DAMON_RECLAIM validates the user input for min_region_sz. The same validation is done inside damon_start() and damon_commit_ctx(). Remove the duplicate. Link: https://lore.kernel.org/20260705155600.96555-5-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/reclaim.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mm/damon/reclaim.c b/mm/damon/reclaim.c index 11b70d0a9a6f..6469b25cc34f 100644 --- a/mm/damon/reclaim.c +++ b/mm/damon/reclaim.c @@ -208,11 +208,6 @@ static int damon_reclaim_apply_parameters(void) param_ctx->addr_unit = addr_unit; param_ctx->min_region_sz = max(DAMON_MIN_REGION_SZ / addr_unit, 1); - if (!is_power_of_2(param_ctx->min_region_sz)) { - err = -EINVAL; - goto out; - } - if (!damon_reclaim_mon_attrs.aggr_interval) { err = -EINVAL; goto out; From cf4b20f9f96b8b4ddb57635f00dfeb2b8b5b6bbc Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 5 Jul 2026 08:55:56 -0700 Subject: [PATCH 209/501] mm/damon/lru_sort: remove duplicated min_region_sz power_of_2() check DAMON_LRU_SORT validates the user input for min_region_sz. The same validation is done inside damon_start() and damon_commit_ctx(). Remove the unnecessary duplicate. Link: https://lore.kernel.org/20260705155600.96555-6-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/lru_sort.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c index 32f41491b726..2dd0cd0d2627 100644 --- a/mm/damon/lru_sort.c +++ b/mm/damon/lru_sort.c @@ -284,11 +284,6 @@ static int damon_lru_sort_apply_parameters(void) param_ctx->addr_unit = addr_unit; param_ctx->min_region_sz = max(DAMON_MIN_REGION_SZ / addr_unit, 1); - if (!is_power_of_2(param_ctx->min_region_sz)) { - err = -EINVAL; - goto out; - } - if (!damon_lru_sort_mon_attrs.sample_interval) { err = -EINVAL; goto out; From f2be66be3ac8bdd3beafe68cb8935d47d42b5d64 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 5 Jul 2026 08:55:57 -0700 Subject: [PATCH 210/501] mm/damon: document region size validation in damon_set_regions() The kernel doc comment of damon_region clearly specifies every region should have positive size. But it is unclear who should verify it. damon_set_regions() is the recommended DAMON core function for setting regions from the callers, and has the verification. Update the comment to clarify the callers should be ok to pass any values for region addresses, as long as they use damon_set_regions(). Link: https://lore.kernel.org/20260705155600.96555-7-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 616bdf0954b5..2661231c0ae8 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -49,7 +49,8 @@ struct damon_size_range { * @list: List head for siblings. * @age: Age of this region. * - * For any use case, @ar should be non-zero positive size. + * For any use case, @ar should be non-zero positive size. damon_set_regions() + * does the validation. * * @nr_accesses is reset to zero for every &damon_attrs->aggr_interval and be * increased for every &damon_attrs->sample_interval if an access to the region From c31ad3c6dad30d932cd906e384efd000db06233a Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 5 Jul 2026 08:55:58 -0700 Subject: [PATCH 211/501] mm/damon/core: remove start, end check in damon_set_region_system_rams() damon_set_region_system_rams() validates user inputs to avoid creating a negative size region. But DAMON core avoids zero size, too. The check is incomplete. The complete check is done inside damon_set_regions(), which is eventually called from damon_set_region_system_rams_default(). Drop the incomplete and unnecessary check. Link: https://lore.kernel.org/20260705155600.96555-8-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/core.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index ca301abcb9ec..3c5a46e1c079 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -3846,9 +3846,6 @@ int damon_set_region_system_rams_default(struct damon_target *t, { struct damon_addr_range addr_range; - if (*start > *end) - return -EINVAL; - if (!*start && !*end && !damon_find_system_rams_range(start, end, addr_unit)) return -EINVAL; From d0a5d03b778af609f322ee0e1d4804302af29666 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 5 Jul 2026 08:55:59 -0700 Subject: [PATCH 212/501] mm/damon/sysfs: remove region size validation DAMON_SYSFS validates user inputs for monitoring target regions to disallow negative size regions. DAMON core assumes only positive size regions, though. The validation is incomplete. Fortunately damon_set_regions(), which is eventually used by DAMON_SYSFS, does the complete validation. Remove the incomplete and unnecessary validation. Link: https://lore.kernel.org/20260705155600.96555-9-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/sysfs.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index e666dddf1feb..b65651498e0d 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -1987,9 +1987,6 @@ static int damon_sysfs_set_regions(struct damon_target *t, struct damon_sysfs_region *sys_region = sysfs_regions->regions_arr[i]; - if (sys_region->ar.start > sys_region->ar.end) - goto out; - ranges[i].start = sys_region->ar.start; ranges[i].end = sys_region->ar.end; if (i == 0) From 9c9fa10839aeba7b33765bc706239ac979e21541 Mon Sep 17 00:00:00 2001 From: "Zenghui Yu (Huawei)" Date: Thu, 2 Jul 2026 16:56:55 +0800 Subject: [PATCH 213/501] MAINTAINERS: add ABI docs and selftests to ZRAM entry Ensure updates to these files get sent to the right place. Link: https://lore.kernel.org/20260702085655.51858-1-zenghui.yu@linux.dev Signed-off-by: Zenghui Yu (Huawei) Acked-by: SJ Park Reviewed-by: Sergey Senozhatsky Cc: Minchan Kim Signed-off-by: Andrew Morton --- MAINTAINERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index e55743fff56c..44b5555fc814 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -29787,8 +29787,11 @@ M: Minchan Kim M: Sergey Senozhatsky L: linux-kernel@vger.kernel.org S: Maintained +F: Documentation/ABI/testing/sysfs-block-zram +F: Documentation/ABI/testing/sysfs-class-zram F: Documentation/admin-guide/blockdev/zram.rst F: drivers/block/zram/ +F: tools/testing/selftests/zram/ ZS DECSTATION Z85C30 SERIAL DRIVER M: "Maciej W. Rozycki" From f11ff47ee9b06fc298e3cde5b85507048ff012e4 Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Thu, 2 Jul 2026 10:48:25 +0800 Subject: [PATCH 214/501] mm: memcg: reset zswap settings in css_reset mem_cgroup_css_reset() is called when the memory controller is disabled on a cgroup but the memcg cannot be destroyed because it is pinned by a subsystem dependency -- for example, the io controller declares .depends_on = 1 << memory_cgrp_id, so memory remains in the cgroup_ss_mask and the css is hidden rather than killed. The purpose of css_reset is to revert the memcg to its vanilla state so that no policies are applied and the css can be safely made visible again later. Currently, all page counters (memory.max, swap.max, kmem.max, tcpmem.max) and other limits (soft_limit, memory.high, swap.high) are reset to their defaults, but zswap_max and zswap_writeback are not. These fields are initialized in css_alloc (zswap_max = PAGE_COUNTER_MAX, zswap_writeback inherited from parent) but were missing from css_reset. As a result, stale zswap policies remain in effect after css_reset: the zswap charge path (obj_cgroup_may_zswap) continues to enforce the old zswap_max limit, and the writeback path continues to honor the old zswap_writeback setting, even though the memory controller has been "disabled" on this cgroup. Reset zswap_max to PAGE_COUNTER_MAX and zswap_writeback to true, matching their defaults in css_alloc. Test: echo "+memory +io" > /sys/fs/cgroup/cgroup.subtree_control mkdir /sys/fs/cgroup/test mkdir /sys/fs/cgroup/test/child echo "+memory +io" > /sys/fs/cgroup/test/cgroup.subtree_control echo 10000 > /sys/fs/cgroup/test/child/memory.zswap.max # child/memory.swap.max and child/memory.zswam.max disappear echo "-memory" > /sys/fs/cgroup/test/cgroup.subtree_control # re-enable memory control echo "+memory" > /sys/fs/cgroup/test/cgroup.subtree_control # before this patch cat /sys/fs/cgroup/test/child/memory.zswap.max 8192 # after this patch, same as memory.swap.max cat /sys/fs/cgroup/test/child/memory.zswap.max max Link: https://lore.kernel.org/20260703063826.306878-1-jiayuan.chen@linux.dev Link: https://lore.kernel.org/20260702024827.353185-1-jiayuan.chen@linux.dev Signed-off-by: Jiayuan Chen Reviewed-by: Tao Cui Reviewed-by: Muchun Song Cc: Chengming Zhou Cc: Johannes Weiner Cc: Michal Hocko Cc: Nhat Pham Cc: Roman Gushchin Cc: Shakeel Butt Signed-off-by: Andrew Morton --- mm/memcontrol.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index d20ffc827306..c20ef3c1d6fe 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -4362,6 +4362,10 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css) page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX); page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX); +#ifdef CONFIG_ZSWAP + WRITE_ONCE(memcg->zswap_max, PAGE_COUNTER_MAX); + WRITE_ONCE(memcg->zswap_writeback, true); +#endif #ifdef CONFIG_MEMCG_V1 page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX); page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX); From 74183a956dec832723b411ed8497f58d1fc84fec Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Fri, 3 Jul 2026 14:38:24 +0800 Subject: [PATCH 215/501] mm: memcg: reset oom_group in css_reset memory.oom.group defaults to disabled, but css_reset did not clear memcg->oom_group when a disabled memory css is kept alive by another controller dependency. Reset it with the other memory controller policies so a hidden memcg cannot keep applying stale group OOM kill policy. Link: https://lore.kernel.org/20260703063826.306878-2-jiayuan.chen@linux.dev Signed-off-by: Jiayuan Chen Acked-by: Roman Gushchin Reviewed-by: Muchun Song Cc: Jiayuan Chen Cc: Johannes Weiner Cc: Michal Hocko Cc: Shakeel Butt Signed-off-by: Andrew Morton --- mm/memcontrol.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index c20ef3c1d6fe..c7de62c8f86f 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -4362,6 +4362,7 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css) page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX); page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX); + WRITE_ONCE(memcg->oom_group, false); #ifdef CONFIG_ZSWAP WRITE_ONCE(memcg->zswap_max, PAGE_COUNTER_MAX); WRITE_ONCE(memcg->zswap_writeback, true); From 6251d650f4e71c07801015e460f930ccc034b652 Mon Sep 17 00:00:00 2001 From: Hajime Tazaki Date: Thu, 2 Jul 2026 10:28:30 +0900 Subject: [PATCH 216/501] mm: nommu: add sysctl_max_map_count() check for do_mmap() The sysctl variable vm.max_map_count (sysctl_max_map_count) is not exposed under !MMU configurations, but its default value (DEFAULT_MAX_MAP_COUNT) is still used as a allocation limit. Currently, this limit is enforced when a VMA entry is split into two chunks (split_vma()), but it is not checked during initial allocation (do_mmap()). As a result, if a user requests a large number of memory allocations, the system will continue allocating until it hits an Out-Of-Memory (OOM) condition. This commit introduces a check at the beginning of do_mmap() in nommu.c to prevent this situation. This issue was detected using the Linux Test Project (LTP) test linked below. Link: https://github.com/linux-test-project/ltp/blob/master/testcases/kernel/syscalls/munmap/munmap04.c Link: https://lore.kernel.org/20260702012830.667205-1-thehajime@gmail.com Signed-off-by: Hajime Tazaki Cc: Jann Horn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Pedro Falcato Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/nommu.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mm/nommu.c b/mm/nommu.c index ed3934bc2de4..11fd558be5ed 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -1035,6 +1035,9 @@ unsigned long do_mmap(struct file *file, if (ret < 0) return ret; + if (current->mm->map_count >= get_sysctl_max_map_count()) + return -ENOMEM; + /* we ignore the address hint */ addr = 0; len = PAGE_ALIGN(len); From aef750043c006540635809c90b06003a1c49c38c Mon Sep 17 00:00:00 2001 From: Yichong Chen Date: Mon, 29 Jun 2026 09:43:14 +0800 Subject: [PATCH 217/501] tools/mm/page_owner_sort: return explicit filter results Patch series "tools/mm/page_owner_sort: fix filtering and cleanup issues", v5. Patch 1 renames is_need() to filter_record() and makes the filter path return explicit results. Patch 2 fixes the per-record allocation leaks. Patch 3 bounds search_pattern() output copies, addressing the pre-existing issue reported by Sashiko/AI review. This patch (of 3): Rename is_need() to filter_record() and make the filter path return explicit error, skip, and match results. This lets callers distinguish allocation failures from records that simply do not match active filters. Link: https://lore.kernel.org/20260629014316.130307-1-chenyichong@uniontech.com Link: https://lore.kernel.org/20260629014316.130307-2-chenyichong@uniontech.com Signed-off-by: Yichong Chen Reviewed-by: Vishal Moola Cc: Ye Liu Cc: Zhen Ni Cc: Zi Yan Signed-off-by: Andrew Morton --- tools/mm/page_owner_sort.c | 40 +++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/tools/mm/page_owner_sort.c b/tools/mm/page_owner_sort.c index e6954909401c..3c754826cec5 100644 --- a/tools/mm/page_owner_sort.c +++ b/tools/mm/page_owner_sort.c @@ -43,6 +43,13 @@ enum FILTER_BIT { FILTER_TGID = 1<<2, FILTER_COMM = 1<<3 }; + +enum FILTER_RESULT { + FILTER_ERROR, + FILTER_SKIP, + FILTER_MATCH +}; + enum CULL_BIT { CULL_PID = 1<<1, CULL_TGID = 1<<2, @@ -372,6 +379,9 @@ static char *get_comm(char *buf) { char *comm_str = malloc(TASK_COMM_LEN); + if (!comm_str) + return NULL; + memset(comm_str, 0, TASK_COMM_LEN); search_pattern(&comm_pattern, comm_str, buf); @@ -450,32 +460,44 @@ static bool match_str_list(const char *str, char **list, int list_size) return false; } -static bool is_need(char *buf) +static enum FILTER_RESULT filter_record(char *buf) { + char *comm; + if ((filter & FILTER_PID) && !match_num_list(get_pid(buf), fc.pids, fc.pids_size)) - return false; + return FILTER_SKIP; if ((filter & FILTER_TGID) && !match_num_list(get_tgid(buf), fc.tgids, fc.tgids_size)) - return false; + return FILTER_SKIP; + if (!(filter & FILTER_COMM)) + return FILTER_MATCH; - char *comm = get_comm(buf); + comm = get_comm(buf); + if (!comm) + return FILTER_ERROR; - if ((filter & FILTER_COMM) && - !match_str_list(comm, fc.comms, fc.comms_size)) { + if (!match_str_list(comm, fc.comms, fc.comms_size)) { free(comm); - return false; + return FILTER_SKIP; } free(comm); - return true; + return FILTER_MATCH; } static bool add_list(char *buf, int len, char *ext_buf) { + enum FILTER_RESULT filter_result; + if (list_size == max_size) { fprintf(stderr, "max_size too small??\n"); return false; } - if (!is_need(buf)) + filter_result = filter_record(buf); + if (filter_result == FILTER_ERROR) { + fprintf(stderr, "Out of memory\n"); + return false; + } + if (filter_result == FILTER_SKIP) return true; list[list_size].pid = get_pid(buf); list[list_size].tgid = get_tgid(buf); From c88c52baa80c61810f39ecd61490859881a6f1af Mon Sep 17 00:00:00 2001 From: Yichong Chen Date: Mon, 29 Jun 2026 09:43:15 +0800 Subject: [PATCH 218/501] tools/mm/page_owner_sort: free per-record allocations add_list() allocates comm and txt for each page owner record, but the cleanup path only frees the outer list array. This leaks both buffers for every retained record. Free partial allocations in add_list(), discarded records during culling, and retained records on exit. Link: https://lore.kernel.org/20260629014316.130307-3-chenyichong@uniontech.com Signed-off-by: Yichong Chen Reviewed-by: Vishal Moola Cc: Ye Liu Cc: Zhen Ni Cc: Zi Yan Signed-off-by: Andrew Morton --- tools/mm/page_owner_sort.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tools/mm/page_owner_sort.c b/tools/mm/page_owner_sort.c index 3c754826cec5..4c9be28abe3b 100644 --- a/tools/mm/page_owner_sort.c +++ b/tools/mm/page_owner_sort.c @@ -396,6 +396,12 @@ static char *get_comm(char *buf) return comm_str; } +static void free_block_list(struct block_list *block) +{ + free(block->comm); + free(block->txt); +} + static int get_arg_type(const char *arg) { if (!strcmp(arg, "pid") || !strcmp(arg, "p")) @@ -502,9 +508,14 @@ static bool add_list(char *buf, int len, char *ext_buf) list[list_size].pid = get_pid(buf); list[list_size].tgid = get_tgid(buf); list[list_size].comm = get_comm(buf); - list[list_size].txt = malloc(len+1); + if (!list[list_size].comm) { + fprintf(stderr, "Out of memory\n"); + return false; + } + list[list_size].txt = malloc(len + 1); if (!list[list_size].txt) { fprintf(stderr, "Out of memory\n"); + free(list[list_size].comm); return false; } memcpy(list[list_size].txt, buf, len); @@ -863,8 +874,10 @@ int main(int argc, char **argv) } else { list[count-1].num += list[i].num; list[count-1].page_num += list[i].page_num; + free_block_list(&list[i]); } } + list_size = count; qsort(list, count, sizeof(list[0]), compare_sort_condition); @@ -898,8 +911,11 @@ int main(int argc, char **argv) free(ext_buf); if (buf) free(buf); - if (list) + if (list) { + for (i = 0; i < list_size; i++) + free_block_list(&list[i]); free(list); + } out_ts: regfree(&ts_nsec_pattern); out_comm: From 3ef8e3fdc8bafa05c872425851c8e2e1f5da443d Mon Sep 17 00:00:00 2001 From: Yichong Chen Date: Mon, 29 Jun 2026 09:43:16 +0800 Subject: [PATCH 219/501] tools/mm/page_owner_sort: bound pattern output copies search_pattern() copies a regex capture into caller-provided buffers without knowing their sizes. Several callers pass fixed-size buffers, including FIELD_BUFF and TASK_COMM_LEN. Pass the destination size to search_pattern(), reject captures that do not fit before copying them, and terminate the output string inside search_pattern(). Link: https://lore.kernel.org/20260629014316.130307-4-chenyichong@uniontech.com Signed-off-by: Yichong Chen Cc: Vishal Moola Cc: Ye Liu Cc: Zhen Ni Cc: Zi Yan Signed-off-by: Andrew Morton --- tools/mm/page_owner_sort.c | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/tools/mm/page_owner_sort.c b/tools/mm/page_owner_sort.c index 4c9be28abe3b..35d3d254941c 100644 --- a/tools/mm/page_owner_sort.c +++ b/tools/mm/page_owner_sort.c @@ -237,7 +237,8 @@ static int remove_pattern(regex_t *pattern, char *buf, int len) return len - (pmatch[1].rm_eo - pmatch[1].rm_so); } -static int search_pattern(regex_t *pattern, char *pattern_str, char *buf) +static int search_pattern(regex_t *pattern, char *pattern_str, + size_t pattern_str_size, char *buf) { int err, val_len; regmatch_t pmatch[2]; @@ -249,8 +250,14 @@ static int search_pattern(regex_t *pattern, char *pattern_str, char *buf) return -1; } val_len = pmatch[1].rm_eo - pmatch[1].rm_so; + if ((size_t)val_len >= pattern_str_size) { + if (debug_on) + fprintf(stderr, "pattern too long in %s\n", buf); + return -1; + } memcpy(pattern_str, buf + pmatch[1].rm_so, val_len); + pattern_str[val_len] = '\0'; return 0; } @@ -307,7 +314,8 @@ static int get_page_num(char *buf) char order_str[FIELD_BUFF] = {0}; char *endptr; - search_pattern(&order_pattern, order_str, buf); + if (search_pattern(&order_pattern, order_str, sizeof(order_str), buf) < 0) + return 0; errno = 0; order_val = strtol(order_str, &endptr, 10); if (order_val > 64 || errno != 0 || endptr == order_str || *endptr != '\0') { @@ -325,7 +333,8 @@ static pid_t get_pid(char *buf) char pid_str[FIELD_BUFF] = {0}; char *endptr; - search_pattern(&pid_pattern, pid_str, buf); + if (search_pattern(&pid_pattern, pid_str, sizeof(pid_str), buf) < 0) + return -1; errno = 0; pid = strtol(pid_str, &endptr, 10); if (errno != 0 || endptr == pid_str || *endptr != '\0') { @@ -344,7 +353,8 @@ static pid_t get_tgid(char *buf) char tgid_str[FIELD_BUFF] = {0}; char *endptr; - search_pattern(&tgid_pattern, tgid_str, buf); + if (search_pattern(&tgid_pattern, tgid_str, sizeof(tgid_str), buf) < 0) + return -1; errno = 0; tgid = strtol(tgid_str, &endptr, 10); if (errno != 0 || endptr == tgid_str || *endptr != '\0') { @@ -363,7 +373,9 @@ static __u64 get_ts_nsec(char *buf) char ts_nsec_str[FIELD_BUFF] = {0}; char *endptr; - search_pattern(&ts_nsec_pattern, ts_nsec_str, buf); + if (search_pattern(&ts_nsec_pattern, ts_nsec_str, + sizeof(ts_nsec_str), buf) < 0) + return -1; errno = 0; ts_nsec = strtoull(ts_nsec_str, &endptr, 10); if (errno != 0 || endptr == ts_nsec_str || *endptr != '\0') { @@ -384,7 +396,10 @@ static char *get_comm(char *buf) memset(comm_str, 0, TASK_COMM_LEN); - search_pattern(&comm_pattern, comm_str, buf); + if (search_pattern(&comm_pattern, comm_str, TASK_COMM_LEN, buf) < 0) { + free(comm_str); + return NULL; + } errno = 0; if (errno != 0) { if (debug_on) From 0beeaf14e7b96676f116847d57281800177bc3dc Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Thu, 2 Jul 2026 20:02:27 +0800 Subject: [PATCH 220/501] memcg: bail out memory.high when memcg is dying Patch series "memcg: bail out reclaim when memcg is dying", v3. This series mitigates a system-wide stall we hit when a cgroup is removed while one of its memory control files is doing synchronous reclaim. Problem Description =================== Writing to memory.high, memory.max or memory.reclaim runs reclaim synchronously in the writer's context, looping until the usage drops below the target (or, for memory.reclaim, until the requested amount has been reclaimed). On a large cgroup this can take a long time. The latency is especially bad when reclaim has to perform swap I/O, where it is bound by the swap device write bandwidth, and under thrashing it is effectively unbounded - each round reclaims a few pages that the workload immediately faults back in, so the loop keeps making "progress" and never converges. The legacy (v1) reclaim loops in memory.limit_in_bytes, memory.memsw.limit_in_bytes and memory.force_empty share the same pattern. These writes go through cgroup_file_write(), which does not take cgroup_mutex and does not pin the css. Instead, kernfs guarantees the node (and thus the css) stays alive for the duration of the operation by holding an active reference. So while the reclaim loop runs, the active reference on the file is held. If another task removes the same cgroup in parallel, cgroup_rmdir() takes cgroup_mutex and then blocks in kernfs_drain() waiting for that active reference to drain. Because cgroup_mutex is held throughout the wait, every other task that needs it piles up behind the remover - in our case the whole machine ground to a halt, with hung_task reports for the remover and for unrelated tasks merely reading /proc//cgroup: INFO: task cgdelete:366634 blocked for more than 159 seconds. Not tainted 6.6.102+ #1 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Call Trace: __schedule+0x3da/0x1650 schedule+0x58/0x100 kernfs_drain+0xe6/0x150 __kernfs_remove.part.0+0xd0/0x200 kernfs_remove_by_name_ns+0x75/0xd0 cgroup_addrm_files+0x325/0x410 css_clear_dir+0x50/0xf0 cgroup_destroy_locked+0xdf/0x1e0 cgroup_rmdir+0x2d/0xd0 kernfs_iop_rmdir+0x53/0x90 vfs_rmdir+0x98/0x240 do_rmdir+0x172/0x1b0 __x64_sys_rmdir+0x42/0x70 x64_sys_call+0xeb0/0x2210 do_syscall_64+0x56/0x90 entry_SYSCALL_64_after_hwframe+0x78/0xe2 INFO: task systemd-journal:2352 blocked for more than 182 seconds. Not tainted 6.6.102+ #1 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Call Trace: __schedule+0x3da/0x1650 schedule+0x58/0x100 schedule_preempt_disabled+0xe/0x20 __mutex_lock.constprop.0+0x3bb/0x640 __mutex_lock_slowpath+0x13/0x20 mutex_lock+0x3c/0x50 proc_cgroup_show+0x4d/0x380 proc_single_show+0x53/0xe0 seq_read_iter+0x12f/0x4b0 seq_read+0xcd/0x110 vfs_read+0xb1/0x360 ? __seccomp_filter+0x368/0x590 ksys_read+0x73/0x100 __x64_sys_read+0x19/0x30 x64_sys_call+0x18d3/0x2210 do_syscall_64+0x56/0x90 entry_SYSCALL_64_after_hwframe+0x78/0xe2 The system recovers only once the reclaim finally finishes and releases the active reference. The reclaim itself is pointless here: the cgroup is being torn down and its remaining pages will be reparented to the parent anyway. Even though we check signal_pending(current) in the reclaim loop, the typical symptom is that cat /proc//cgroup gets stuck. By the time someone looks for which task is actually stuck in reclaim, the hung task timeout has already been hit. This makes the problem particularly nasty to debug from a hung-task report alone, because the blocked tasks shown are often the victims, not the reclaim writer itself. Our Mitigation ============== cgroup destruction sets CSS_DYING in kill_css_sync() *before* css_clear_dir() triggers the kernfs_drain() that blocks the remover. The in-flight reclaim loop is therefore guaranteed to observe it before starting another reclaim iteration. This series checks memcg_is_dying() in the v2 reclaim loops (memory.high, memory.max and proactive reclaim) and the v1 reclaim loops (memory.limit_in_bytes, memory.memsw.limit_in_bytes and memory.force_empty), and bails out early, so the writer drops the active reference promptly and the remover can make progress. Unlike the no-progress guard (MAX_RECLAIM_RETRIES), which only fires when reclaim makes zero progress, the dying check also covers the slow swap I/O and thrashing cases, where reclaim keeps succeeding a little and the loop would otherwise never converge. For memory.reclaim, bailing out because the memcg is dying means the requested reclaim amount was not satisfied, so the write returns -EAGAIN. This is orthogonal to commit c8e6002bd611 ("memcg: introduce non-blocking limit setting option"): O_NONBLOCK lets a caller avoid the synchronous reclaim up front, while this series handles the case where reclaim is already running when the cgroup starts being removed. This patch (of 4): memory.high reclaims synchronously in the writer's context, and the latency can be very high - especially when reclaim performs swap I/O, or under thrashing where the loop may not converge for a long time. While this runs the kernfs active reference on the file is held, so a concurrent removal of the same cgroup blocks in kernfs_drain() under cgroup_mutex until it finishes. Reclaiming a dying cgroup is pointless, as its pages are reparented to the parent anyway. Mitigate this by bailing out of the reclaim loop once memcg_is_dying(). Link: https://lore.kernel.org/20260702120235.376752-1-jiayuan.chen@linux.dev Link: https://lore.kernel.org/20260702120235.376752-2-jiayuan.chen@linux.dev Signed-off-by: Jiayuan Chen Reported-by: Zhou Yingfu Acked-by: Johannes Weiner Cc: Axel Rasmussen Cc: Barry Song Cc: David Hildenbrand Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin Cc: Shakeel Butt Cc: Wei Xu Cc: Yuanchu Xie Signed-off-by: Andrew Morton --- mm/memcontrol.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index c7de62c8f86f..8ca42e79686c 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -4799,6 +4799,10 @@ static ssize_t memory_high_write(struct kernfs_open_file *of, if (signal_pending(current)) break; + /* cgroup_rmdir() waits for us with cgroup_mutex held. */ + if (memcg_is_dying(memcg)) + break; + if (!drained) { drain_all_stock(memcg); drained = true; From e13f634f50d5f3bf0f49c7e915cc31019b58297b Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Thu, 2 Jul 2026 20:02:28 +0800 Subject: [PATCH 221/501] memcg: bail out memory.max when memcg is dying memory.max has the same high-latency reclaim loop as memory.high, and may additionally invoke the OOM killer on a cgroup that is already going away, further delaying its removal. Mitigate this by bailing out of the loop once memcg_is_dying(). Link: https://lore.kernel.org/20260702120235.376752-3-jiayuan.chen@linux.dev Signed-off-by: Jiayuan Chen Reported-by: Zhou Yingfu Acked-by: Johannes Weiner Cc: Jiayuan Chen Cc: Axel Rasmussen Cc: Barry Song Cc: David Hildenbrand Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin Cc: Shakeel Butt Cc: Wei Xu Cc: Yuanchu Xie Signed-off-by: Andrew Morton --- mm/memcontrol.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 8ca42e79686c..22f55aeb94f3 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -4854,6 +4854,10 @@ static ssize_t memory_max_write(struct kernfs_open_file *of, if (signal_pending(current)) break; + /* cgroup_rmdir() waits for us with cgroup_mutex held. */ + if (memcg_is_dying(memcg)) + break; + if (!drained) { drain_all_stock(memcg); drained = true; From 757dd8193f6cc7197854d1af1a723aef2026b510 Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Thu, 2 Jul 2026 20:02:29 +0800 Subject: [PATCH 222/501] memcg: bail out proactive reclaim when memcg is dying Proactive reclaim via memory.reclaim can run for a long time - swap I/O or thrashing again dominating the latency - and delays cgroup removal in the same way. Mitigate this by stopping the reclaim once memcg_is_dying(). Link: https://lore.kernel.org/20260702120235.376752-4-jiayuan.chen@linux.dev Signed-off-by: Jiayuan Chen Reported-by: Zhou Yingfu Acked-by: Johannes Weiner Cc: Jiayuan Chen Cc: Axel Rasmussen Cc: Barry Song Cc: David Hildenbrand Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin Cc: Shakeel Butt Cc: Wei Xu Cc: Yuanchu Xie Signed-off-by: Andrew Morton --- mm/vmscan.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mm/vmscan.c b/mm/vmscan.c index 347565278a55..8e7b909e47e4 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -7912,6 +7912,10 @@ int user_proactive_reclaim(char *buf, if (signal_pending(current)) return -ERESTARTSYS; + /* cgroup_rmdir() waits for us with cgroup_mutex held. */ + if (memcg && memcg_is_dying(memcg)) + return -EAGAIN; + /* * This is the final attempt, drain percpu lru caches in the * hope of introducing more evictable pages. From 10228e0a51238b500a39831fa0a983f69c36e2dd Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Thu, 2 Jul 2026 20:02:30 +0800 Subject: [PATCH 223/501] memcg-v1: bail out reclaim when memcg is dying The legacy memory.limit_in_bytes and memory.memsw.limit_in_bytes writers retry page_counter_set_max() by reclaiming synchronously in the writer context. memory.force_empty similarly loops in synchronous reclaim until the cgroup is empty or reclaim stops making progress. These writes hold a kernfs active reference on the file. If cgroup removal starts in parallel, the remover sets CSS_DYING and then waits in kernfs_drain() under cgroup_mutex for the active reference to drain. Continuing reclaim after the memcg is dying can therefore delay cgroup removal and keep cgroup_mutex held for a long time. Stop the v1 reclaim loops once the memcg is dying. For limit resizing, keep the existing -EBUSY semantics when the new limit could not be installed. For memory.force_empty, keep the existing best-effort success semantics. Link: https://lore.kernel.org/20260702120235.376752-5-jiayuan.chen@linux.dev Signed-off-by: Jiayuan Chen Reported-by: Zhou Yingfu Acked-by: Johannes Weiner Cc: Jiayuan Chen Cc: Axel Rasmussen Cc: Barry Song Cc: David Hildenbrand Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin Cc: Shakeel Butt Cc: Wei Xu Cc: Yuanchu Xie Signed-off-by: Andrew Morton --- mm/memcontrol-v1.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c index 135622b6172b..e8b6e1560278 100644 --- a/mm/memcontrol-v1.c +++ b/mm/memcontrol-v1.c @@ -1805,6 +1805,10 @@ static int mem_cgroup_resize_max(struct mem_cgroup *memcg, if (!ret) break; + /* cgroup_rmdir() waits for us with cgroup_mutex held. */ + if (memcg_is_dying(memcg)) + break; + if (!drained) { drain_all_stock(memcg); drained = true; @@ -1843,6 +1847,10 @@ static int mem_cgroup_force_empty(struct mem_cgroup *memcg) if (signal_pending(current)) return -EINTR; + /* cgroup_rmdir() waits for us with cgroup_mutex held. */ + if (memcg_is_dying(memcg)) + break; + if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, MEMCG_RECLAIM_MAY_SWAP, NULL)) nr_retries--; From 8d7483174458577ca1b8134de7669a0ddef8c86b Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Tue, 30 Jun 2026 05:46:04 -0700 Subject: [PATCH 224/501] mm/memory-failure: drop dead error_states[] entry for reserved pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch series "mm/memory-failure: add panic option for unrecoverable pages", v10. A multi-bit ECC error on a kernel-owned page that the memory failure handler cannot recover is currently swallowed: PG_hwpoison is set, the event is logged, and the kernel keeps running. The corrupted memory remains accessible to the kernel and either drives silent data corruption or surfaces seconds-to-minutes later as an apparently unrelated crash. In a large fleet that delayed, unattributable crash turns into significant engineering effort to root-cause; in a kdump configuration, by the time the crash happens the original error context (faulting PFN, MCE/GHES record, page state) is long gone. This series adds an opt-in sysctl, vm.panic_on_unrecoverable_memory_failure, that converts an unrecoverable kernel-page hwpoison event into an immediate panic with a clean dmesg/vmcore that still contains the original failure context. The default is disabled so existing workloads see no change. There is a selftest that test different cases, and I tested it using the following variants: ┌─────────┬──────────┬───────────────────────────────────────────────────────────┐ │ Variant │ PFN │ Result │ ├─────────┼──────────┼───────────────────────────────────────────────────────────┤ │ rodata │ 0x2600 │ Panic with "Memory failure: 0x2600: unrecoverable page" │ ├─────────┼──────────┼───────────────────────────────────────────────────────────┤ │ slab │ 0x100032 │ Panic with "Memory failure: 0x100032: unrecoverable page" │ ├─────────┼──────────┼───────────────────────────────────────────────────────────┤ │ pgtable │ 0x100000 │ Panic with "Memory failure: 0x100000: unrecoverable page" │ └─────────┴──────────┴───────────────────────────────────────────────────────────┘ Each one shows the same call trace, exactly the path the series builds: hard_offline_page_store → memory_failure → action_result → panic("Memory failure: %#lx: unrecoverable page") This patch (of 5): The first entry of error_states[], { reserved, reserved, MF_MSG_KERNEL, me_kernel }, is unreachable. identify_page_state() has two callers, and neither one can dispatch a PG_reserved page to me_kernel(): * memory_failure() reaches identify_page_state() only after get_hwpoison_page() returned 1. get_any_page() reaches that return only via __get_hwpoison_page(), which only takes a refcount when the page is HWPoisonHandlable(). HWPoisonHandlable() is an allowlist for LRU, free-buddy, and (for soft-offline) movable_ops pages -- PG_reserved pages do not satisfy any of these, so they fail with -EBUSY/-EIO long before identify_page_state() runs. * try_memory_failure_hugetlb() reaches identify_page_state() only via the MF_HUGETLB_IN_USED branch, where the page is necessarily a hugetlb folio. hugetlb folios don't carry PG_reserved at that point: hugetlb_folio_init_vmemmap() calls __folio_clear_reserved() during init, so the reserved entry would not match even if it were still present. me_kernel() never executes and the entry exists only to be matched against by code that cannot see it. Drop the entry, the me_kernel() helper, and the now-unused "reserved" macro. Leave the MF_MSG_KERNEL enum value in place: it remains part of the tracepoint and pr_err() string tables, and follow-on work to classify unrecoverable kernel pages can reuse it without churning the user-visible enum. No functional change. Link: https://lore.kernel.org/20260630-ecc_panic-v10-0-c6ed5b62eea2@debian.org Link: https://lore.kernel.org/20260630-ecc_panic-v10-1-c6ed5b62eea2@debian.org Signed-off-by: Breno Leitao Suggested-by: David Hildenbrand Acked-by: David Hildenbrand (Arm) Reviewed-by: Lance Yang Acked-by: Miaohe Lin Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Michal Hocko Cc: Mike Rapoport Cc: Naoya Horiguchi Cc: Shuah Khan Cc: Steven Rostedt Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/memory-failure.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 1fb4f6206e6b..12efd03dc0ec 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -981,17 +981,6 @@ static bool has_extra_refcount(struct page_state *ps, struct page *p, return false; } -/* - * Error hit kernel page. - * Do nothing, try to be lucky and not touch this instead. For a few cases we - * could be more sophisticated. - */ -static int me_kernel(struct page_state *ps, struct page *p) -{ - unlock_page(p); - return MF_IGNORED; -} - /* * Page in unknown state. Do nothing. * This is a catch-all in case we fail to make sense of the page state. @@ -1200,10 +1189,8 @@ static int me_huge_page(struct page_state *ps, struct page *p) #define mlock (1UL << PG_mlocked) #define lru (1UL << PG_lru) #define head (1UL << PG_head) -#define reserved (1UL << PG_reserved) static struct page_state error_states[] = { - { reserved, reserved, MF_MSG_KERNEL, me_kernel }, /* * free pages are specially detected outside this table: * PG_buddy pages only make a small fraction of all free pages. @@ -1235,7 +1222,6 @@ static struct page_state error_states[] = { #undef mlock #undef lru #undef head -#undef reserved static void update_per_node_mf_stats(unsigned long pfn, enum mf_result result) From 8a276c0086cf713a49f4bcc461307955161e3b52 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Tue, 30 Jun 2026 05:46:05 -0700 Subject: [PATCH 225/501] mm/memory-failure: surface unhandlable kernel pages as -ENOTRECOVERABLE get_any_page() collapses every HWPoisonHandlable() rejection into a single -EIO via the __get_hwpoison_page() -> -EBUSY -> shake_page() -> retry path. That is correct for the transient case (a userspace folio briefly off LRU during migration or compaction, which a later shake can drag back), but wrong for stable kernel-owned pages: slab, page-table, large-kmalloc and PG_reserved pages will never become HWPoisonHandlable(), so the retry loop is wasted work and the final -EIO loses the "this is structurally unrecoverable" information. memory_failure() then maps -EIO into MF_MSG_GET_HWPOISON, which the panic-on-unrecoverable sysctl deliberately does not act on. Introduce is_kernel_owned_page(), a small predicate that positively identifies pages the hwpoison handler cannot recover from: is_kernel_owned_page(p) := PageReserved(p) || PageSlab(head) || PageTable(head) || PageLargeKmalloc(head) where head = compound_head(p). PG_reserved is a per-page flag (PF_NO_COMPOUND) and is tested on the page directly. The slab, page-table and large-kmalloc page-type bits are only stored on the head page, so those tests resolve the compound head first, then re-read compound_head(page) afterwards: a concurrent split or compound free that moves head invalidates the just-read flags and the loop retries. The lookup still takes no refcount, mirroring the rest of get_any_page(); the recheck closes the common split race, and a residual free->alloc->free in the same window can only mis-tag a genuinely poisoned page, never reclassify a handlable one. No MF_SOFT_OFFLINE / page_has_movable_ops() opt-out is needed: a movable_ops page is always PageOffline or PageZsmalloc, whose page_type is mutually exclusive with slab, page-table and large-kmalloc, and it never carries PG_reserved, so it can never match any of the checks above. The list is intentionally not exhaustive. vmalloc and kernel-stack pages, for example, do not carry a page_type bit and would need a different oracle; they keep going through the existing retry path unchanged. This is the smallest set we can identify with certainty by page type. Wire the helper into the top of get_any_page() to short-circuit those pages before the retry loop runs. On a hit, drop the caller's MF_COUNT_INCREASED reference (if any) and return -ENOTRECOVERABLE straight away. Pages outside the helper's positive list still take the existing retry path and return -EIO, leaving operator-visible behaviour for those cases unchanged. Extend the unhandlable-page pr_err() to fire for either errno and update the get_hwpoison_page() kerneldoc to document the new return. memory_failure() still folds every negative return into MF_MSG_GET_HWPOISON via its existing "else if (res < 0)" branch, so this patch on its own only changes the errno that soft_offline_page() can propagate to its callers. A follow-up wires -ENOTRECOVERABLE through memory_failure() and reports MF_MSG_KERNEL for the unrecoverable cases, which is what the panic_on_unrecoverable_memory_failure sysctl observes. Link: https://lore.kernel.org/20260630-ecc_panic-v10-2-c6ed5b62eea2@debian.org Signed-off-by: Breno Leitao Suggested-by: David Hildenbrand Suggested-by: Lance Yang Acked-by: Miaohe Lin Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Michal Hocko Cc: Mike Rapoport Cc: Naoya Horiguchi Cc: Shuah Khan Cc: Steven Rostedt Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/memory-failure.c | 52 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 12efd03dc0ec..dd21f37510b3 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -1326,6 +1326,38 @@ static inline bool HWPoisonHandlable(struct page *page, unsigned long flags) return PageLRU(page) || is_free_buddy_page(page); } +/* + * Positive identification of pages the hwpoison handler cannot recover: + * pages owned by kernel internals with no userspace mapping to unmap, no + * file mapping to invalidate, and no migration target. + */ +static inline bool is_kernel_owned_page(struct page *page) +{ + struct page *head; + bool kernel_owned; + + /* PG_reserved is a per-page flag, never set on a compound page. */ + if (PageReserved(page)) + return true; + + /* + * Page-type bits live only on the head page, so resolve any tail + * first. The check takes no refcount; recheck the head afterwards + * so a concurrent split or compound free cannot leave us trusting + * a stale view. A residual free->alloc->free cannot be closed here + * (frozen slab and large-kmalloc pages cannot be pinned), but is + * harmless: where a wrong verdict could panic, memory_failure() has + * already set PageHWPoison, which bars the page from the allocator. + */ +retry: + head = compound_head(page); + kernel_owned = PageSlab(head) || PageTable(head) || + PageLargeKmalloc(head); + if (head != compound_head(page)) + goto retry; + return kernel_owned; +} + static int __get_hwpoison_page(struct page *page, unsigned long flags) { struct folio *folio = page_folio(page); @@ -1372,6 +1404,19 @@ static int get_any_page(struct page *p, unsigned long flags) if (flags & MF_COUNT_INCREASED) count_increased = true; + /* + * Page types we know are kernel-owned and cannot be recovered. + * Short-circuit before the shake_page() / retry loop, which + * cannot turn any of these into something HWPoisonHandlable(). + * Drop the caller's reference if MF_COUNT_INCREASED took one. + */ + if (is_kernel_owned_page(p)) { + if (count_increased) + put_page(p); + ret = -ENOTRECOVERABLE; + goto out; + } + try_again: if (!count_increased) { ret = __get_hwpoison_page(p, flags); @@ -1419,7 +1464,7 @@ static int get_any_page(struct page *p, unsigned long flags) ret = -EIO; } out: - if (ret == -EIO) + if (ret == -EIO || ret == -ENOTRECOVERABLE) pr_err("%#lx: unhandlable page.\n", page_to_pfn(p)); return ret; @@ -1476,7 +1521,10 @@ static int __get_unpoison_page(struct page *page) * -EIO for pages on which we can not handle memory errors, * -EBUSY when get_hwpoison_page() has raced with page lifecycle * operations like allocation and free, - * -EHWPOISON when the page is hwpoisoned and taken off from buddy. + * -EHWPOISON when the page is hwpoisoned and taken off from buddy, + * -ENOTRECOVERABLE for kernel-owned pages identified by + * is_kernel_owned_page() (PG_reserved, slab, + * page-table, large-kmalloc) that the handler cannot recover. */ static int get_hwpoison_page(struct page *p, unsigned long flags) { From 737a97548c4a319041134251260466b6f4fe3563 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Tue, 30 Jun 2026 05:46:06 -0700 Subject: [PATCH 226/501] mm/memory-failure: report MF_MSG_KERNEL for unrecoverable kernel pages The previous patch teaches get_any_page() to return -ENOTRECOVERABLE for stable unhandlable kernel pages (PG_reserved, slab, page tables, large-kmalloc). memory_failure() still folds every negative return into MF_MSG_GET_HWPOISON, so callers that want to react to the unrecoverable cases (a panic option, smarter logging) cannot tell them apart from transient page-allocator races. Turn the post-call branch into a switch over the get_hwpoison_page() return code: map -ENOTRECOVERABLE to MF_MSG_KERNEL and any other negative return to MF_MSG_GET_HWPOISON. case 0 keeps the existing free-buddy / kernel-high-order handling and case 1 falls through to the rest of memory_failure() unchanged. The MF_MSG_KERNEL label and tracepoint string are kept as "reserved kernel page" to avoid breaking userspace tools that match on those literals; the enum value still adequately tags the failure even though it now also covers slab, page tables and large-kmalloc pages. Link: https://lore.kernel.org/20260630-ecc_panic-v10-3-c6ed5b62eea2@debian.org Signed-off-by: Breno Leitao Suggested-by: David Hildenbrand Acked-by: David Hildenbrand (Arm) Acked-by: Miaohe Lin Cc: Jonathan Corbet Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Michal Hocko Cc: Mike Rapoport Cc: Naoya Horiguchi Cc: Shuah Khan Cc: Steven Rostedt Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/memory-failure.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index dd21f37510b3..adc7135ca372 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -2437,7 +2437,8 @@ int memory_failure(unsigned long pfn, int flags) * that may make page_ref_freeze()/page_ref_unfreeze() mismatch. */ res = get_hwpoison_page(p, flags); - if (!res) { + switch (res) { + case 0: if (is_free_buddy_page(p)) { if (take_page_off_buddy(p)) { page_ref_inc(p); @@ -2456,7 +2457,19 @@ int memory_failure(unsigned long pfn, int flags) res = action_result(pfn, MF_MSG_KERNEL_HIGH_ORDER, MF_IGNORED); } goto unlock_mutex; - } else if (res < 0) { + case 1: + /* Got a refcount on a handlable page. */ + break; + case -ENOTRECOVERABLE: + /* + * Stable unhandlable kernel-owned page (PG_reserved, + * slab, page tables, large-kmalloc). + * No recovery possible. + */ + res = action_result(pfn, MF_MSG_KERNEL, MF_IGNORED); + goto unlock_mutex; + default: + /* Transient lifecycle race with the page allocator. */ res = action_result(pfn, MF_MSG_GET_HWPOISON, MF_IGNORED); goto unlock_mutex; } From 3e0659f93bae4df10124917ef7647f04fa09e372 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Tue, 30 Jun 2026 05:46:07 -0700 Subject: [PATCH 227/501] mm/memory-failure: add panic option for unrecoverable pages Add a sysctl panic_on_unrecoverable_memory_failure (disabled by default) that triggers a kernel panic when memory_failure() encounters pages that cannot be recovered. This provides a clean crash with useful debug information rather than allowing silent data corruption or a delayed crash at an unrelated code path. Panic eligibility is intentionally narrow: only MF_MSG_KERNEL with result == MF_IGNORED panics. After the previous patch, MF_MSG_KERNEL covers PG_reserved pages and the kernel-owned pages promoted from get_hwpoison_page() via -ENOTRECOVERABLE (slab, page tables, large-kmalloc). All other action types are excluded: - MF_MSG_GET_HWPOISON and MF_MSG_KERNEL_HIGH_ORDER can be reached by transient refcount races with the page allocator (an in-flight buddy allocation has refcount 0 and is no longer on the buddy free list, briefly), and panicking on them would risk killing the box for what is actually a recoverable userspace page. - MF_MSG_UNKNOWN means identify_page_state() could not classify the page; that is precisely the wrong basis for a panic decision. Link: https://lore.kernel.org/20260630-ecc_panic-v10-4-c6ed5b62eea2@debian.org Signed-off-by: Breno Leitao Acked-by: Miaohe Lin Cc: David Hildenbrand (Arm) Cc: Jonathan Corbet Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Michal Hocko Cc: Mike Rapoport Cc: Naoya Horiguchi Cc: Shuah Khan Cc: Steven Rostedt Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/memory-failure.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index adc7135ca372..bf717ec59508 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -75,6 +75,8 @@ static int sysctl_memory_failure_recovery __read_mostly = 1; static int sysctl_enable_soft_offline __read_mostly = 1; +static int sysctl_panic_on_unrecoverable_mf __read_mostly; + atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0); static bool hw_memory_failure __read_mostly; @@ -156,6 +158,15 @@ static const struct ctl_table memory_failure_table[] = { .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, + }, + { + .procname = "panic_on_unrecoverable_memory_failure", + .data = &sysctl_panic_on_unrecoverable_mf, + .maxlen = sizeof(sysctl_panic_on_unrecoverable_mf), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, } }; @@ -1256,6 +1267,15 @@ static void update_per_node_mf_stats(unsigned long pfn, ++mf_stats->total; } +static bool panic_on_unrecoverable_mf(enum mf_action_page_type type, + enum mf_result result) +{ + if (!sysctl_panic_on_unrecoverable_mf) + return false; + + return type == MF_MSG_KERNEL && result == MF_IGNORED; +} + /* * "Dirty/Clean" indication is not 100% accurate due to the possibility of * setting PG_dirty outside page lock. See also comment above set_page_dirty(). @@ -1273,6 +1293,9 @@ static int action_result(unsigned long pfn, enum mf_action_page_type type, pr_err("%#lx: recovery action for %s: %s\n", pfn, action_page_types[type], action_name[result]); + if (panic_on_unrecoverable_mf(type, result)) + panic("Memory failure: %#lx: unrecoverable page", pfn); + return (result == MF_RECOVERED || result == MF_DELAYED) ? 0 : -EBUSY; } From 97b03277ce77e680628c46b7638b844b564db3df Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Tue, 30 Jun 2026 05:46:08 -0700 Subject: [PATCH 228/501] Documentation: document panic_on_unrecoverable_memory_failure sysctl Add documentation for the new vm.panic_on_unrecoverable_memory_failure sysctl, describing which failures trigger a panic (kernel-owned pages the handler cannot recover) and which are intentionally left out (transient allocator races and unclassified pages). Link: https://lore.kernel.org/20260630-ecc_panic-v10-5-c6ed5b62eea2@debian.org Signed-off-by: Breno Leitao Acked-by: Miaohe Lin Cc: David Hildenbrand (Arm) Cc: Jonathan Corbet Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: "Masami Hiramatsu (Google)" Cc: Mathieu Desnoyers Cc: Michal Hocko Cc: Mike Rapoport Cc: Naoya Horiguchi Cc: Shuah Khan Cc: Steven Rostedt Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- Documentation/admin-guide/sysctl/vm.rst | 80 +++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst index b9b0c218bfb4..22cc54cac3b2 100644 --- a/Documentation/admin-guide/sysctl/vm.rst +++ b/Documentation/admin-guide/sysctl/vm.rst @@ -67,6 +67,7 @@ Currently, these files are in /proc/sys/vm: - page-cluster - page_lock_unfairness - panic_on_oom +- panic_on_unrecoverable_memory_failure - percpu_pagelist_high_fraction - stat_interval - stat_refresh @@ -925,6 +926,85 @@ panic_on_oom=2+kdump gives you very strong tool to investigate why oom happens. You can get snapshot. +panic_on_unrecoverable_memory_failure +====================================== + +When a hardware memory error (e.g. multi-bit ECC) hits a kernel page +that cannot be recovered by the memory failure handler, the default +behaviour is to ignore the error and continue operation. This is +dangerous because the corrupted data remains accessible to the kernel, +risking silent data corruption or a delayed crash when the poisoned +memory is next accessed. + +When enabled, this sysctl triggers a panic on memory failure events +hitting kernel-owned pages that the handler cannot recover: +``PageReserved`` (firmware reservations, kernel image, vDSO, zero +page, and similar memblock-reserved regions), ``PageSlab``, +``PageTable``, and ``PageLargeKmalloc``. These are owned by the +kernel and the memory failure handler cannot reliably evict their +contents. + +Other unrecoverable kernel-owned populations (vmalloc allocations, +kernel stack pages, ...) are not currently covered because the +handler has no page-type signal that distinguishes them from a +userspace folio temporarily off the LRU during migration or +compaction. Such pages still go through the standard +MF_MSG_GET_HWPOISON path: ``PG_hwpoison`` is set on them and a +delayed crash on the next access remains possible. Coverage may +grow as the handler gains stronger kernel-ownership signals. + +Recoverable failure paths are also intentionally left out: in-flight +buddy allocations and other transient races with the page allocator +can reach the same diagnostic, and panicking on them would risk +killing the box for a page destined for userspace where the standard +SIGBUS recovery path applies. Pages whose state could not be +classified at all are not covered either, since an unknown state is +not a sound basis for a panic decision. + +For many environments it is preferable to panic immediately with a clean +crash dump that captures the original error context, rather than to +continue and face a random crash later whose cause is difficult to +diagnose. + +Use cases +--------- + +This option is most useful in environments where unattributed crashes +are expensive to debug or where data integrity must take precedence +over availability: + +* Large fleets, where multi-bit ECC errors on kernel pages are observed + regularly and post-mortem analysis of an unrelated downstream crash + (often seconds to minutes after the original error) consumes + significant engineering effort. + +* Systems configured with kdump, where panicking at the moment of the + hardware error produces a vmcore that still contains the faulting + address, the affected page state, and the originating MCE/GHES + record — context that is typically lost by the time a delayed crash + occurs. + +* High-availability clusters that rely on fast, deterministic node + failure for failover, and prefer an immediate panic over silent data + corruption propagating to replicas or persistent storage. + +* Kernel and platform developers reproducing hwpoison issues with + tools such as ``mce-inject`` or error-injection debugfs interfaces, + where panicking on the unrecoverable path makes regressions + immediately visible instead of surfacing as later, unrelated + failures. + += ===================================================================== +0 Try to continue operation (default). +1 Panic immediately. If the ``panic`` sysctl is also non-zero then the + machine will be rebooted. += ===================================================================== + +Example:: + + echo 1 > /proc/sys/vm/panic_on_unrecoverable_memory_failure + + percpu_pagelist_high_fraction ============================= From 43469155c9f393fb30fbabc4ba53e6054d924ad1 Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Fri, 19 Jun 2026 17:47:04 +0200 Subject: [PATCH 229/501] percpu: replace __ASSEMBLY__ with __ASSEMBLER__ in percpu-defs.h header file While the GCC and Clang compilers already define __ASSEMBLER__ automatically when compiling assembly code, __ASSEMBLY__ is a macro that only gets defined by the Makefiles in the kernel. This can be very confusing when switching between userspace and kernelspace coding, or when dealing with uapi headers that rather should use __ASSEMBLER__ instead. So let's standardize now on the __ASSEMBLER__ macro that is provided by the compilers. This is a completely mechanical patch (done with a simple "sed -i" statement. Link: https://lore.kernel.org/20260619154704.41293-1-thuth@redhat.com Signed-off-by: Thomas Huth Cc: Dennis Zhou Cc: Tejun Heo Signed-off-by: Andrew Morton --- include/linux/percpu-defs.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h index 43c854a273c3..2cba7cc2b01f 100644 --- a/include/linux/percpu-defs.h +++ b/include/linux/percpu-defs.h @@ -203,7 +203,7 @@ /* * Accessors and operations. */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * __verify_pcpu_ptr() verifies @ptr is a percpu pointer without evaluating @@ -514,5 +514,5 @@ do { \ #define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1) #define this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _LINUX_PERCPU_DEFS_H */ From fe8c4d774dbacfc28e173eeb6b3fe263b527f8ae Mon Sep 17 00:00:00 2001 From: Watson Wheeler Date: Tue, 23 Jun 2026 16:17:12 +1000 Subject: [PATCH 230/501] lib/maple_tree: add missing spaces after switch keyword Add the required space before the opening parenthesis in switch statements to conform to kernel coding style. Link: https://lore.kernel.org/20260623061730.2024-1-git@tazy.dev Signed-off-by: Watson Wheeler Reviewed-by: Liam Howlett Signed-off-by: Andrew Morton --- lib/maple_tree.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/maple_tree.c b/lib/maple_tree.c index e52876435b77..529acc056e55 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -6447,7 +6447,7 @@ static void mt_dump_range(unsigned long min, unsigned long max, { static const char spaces[] = " "; - switch(format) { + switch (format) { case mt_dump_hex: if (min == max) pr_info("%.*s%lx: ", depth * 2, spaces, min); @@ -6489,7 +6489,7 @@ static void mt_dump_range64(const struct maple_tree *mt, void *entry, pr_cont(" contents: "); for (i = 0; i < MAPLE_RANGE64_SLOTS - 1; i++) { - switch(format) { + switch (format) { case mt_dump_hex: pr_cont(PTR_FMT " %lX ", node->slot[i], node->pivot[i]); break; @@ -6517,7 +6517,7 @@ static void mt_dump_range64(const struct maple_tree *mt, void *entry, if (last == max) break; if (last > max) { - switch(format) { + switch (format) { case mt_dump_hex: pr_err("node " PTR_FMT " last (%lx) > max (%lx) at pivot %d!\n", node, last, max, i); @@ -6576,7 +6576,7 @@ static void mt_dump_arange64(const struct maple_tree *mt, void *entry, if (last == max) break; if (last > max) { - switch(format) { + switch (format) { case mt_dump_hex: pr_err("node " PTR_FMT " last (%lx) > max (%lx) at pivot %d!\n", node, last, max, i); From d25711a9f32ac99d5c04a48d00d802596a8a1814 Mon Sep 17 00:00:00 2001 From: wang wei Date: Mon, 6 Jul 2026 21:43:04 +0800 Subject: [PATCH 231/501] selftests/damon: check correct path in ensure_file() not_exist case In the ensure_file() function, the "not_exist" code path checks whether $dir exists as a regular file. However, the intent is to verify that the target file ($file) does not exist, not the $dir. Testing $dir makes the existence check effectively useless -- it tests the wrong path and thus never catches the case where the file is unexpectedly present. Replace $dir with $file so the not_exist verification targets the correct path. Link: https://lore.kernel.org/20260706134305.5224-1-a929244872@163.com Signed-off-by: wang wei Reviewed-by: SJ Park Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/damon/sysfs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/damon/sysfs.sh b/tools/testing/selftests/damon/sysfs.sh index b43deee60fe9..f7fb94b84e71 100755 --- a/tools/testing/selftests/damon/sysfs.sh +++ b/tools/testing/selftests/damon/sysfs.sh @@ -67,7 +67,7 @@ ensure_file() echo "$file permission: expected $permission but $perm" exit 1 fi - elif [ "$to_ensure" = "not_exist" ] && [ -f "$dir" ] + elif [ "$to_ensure" = "not_exist" ] && [ -f "$file" ] then echo "$file is not expected but found" exit 1 From 5bf9c1f3a97e57e79b72be58394bc2cd3635f0ae Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 6 Jul 2026 07:06:16 -0700 Subject: [PATCH 232/501] mm/damon/core: stop ctxs in damon_start() before returning an error Patch series "mm/damon: refactor damon_{start,stop,commit}() for simple error handling". damon_start() and damon_stop() could leave a few of requested DAMON contexts running when they return an error. damon_commit() failure stops the DAMON context, but in an asynchronous way. Callers should stop the left-over DAMON contexts. It is easy to make mistakes, and indeed a few bugs from such mistakes were found and fixed. Refactor the core API functions to guarantee contexts are completely stopped under failures. Remove the caller side error handlers that are no longer needed due to the refactoring. Patches Sequence ================ Patch 1 refactors damon_start() to ensure all contexts are stopped for failures. Patch 2 updates unnecessary damon_start() error handling from mtier sample module. Patch 3 refactors damon_stop() to always success. Patches 4-6 updates callers (damon_{sysfs,reclaim lru_sort}) to ignore the return value. Patch 7 update damon_stop() return value to void. Patch 8 simplifies damon_stop() error handling in mtier sample module. Patch 9 refactors damon_call() to return errors only after the context is completely stopped. Patches 10 and 11 remove unnecessary error handlings from callers (wsse and prcl sample modules). This patch (of 11): When multiple contexts are passed to damon_start(), the function starts the contexts one by one. If any of the operations fails, it immediately returns an error. Contexts that successfully started before the failure keep running. The caller should catch this and stop the contexts. It is complicated and easy to make mistakes. Stop all contexts in damon_start() under the failures. Link: https://lore.kernel.org/20260706140628.87414-1-sj@kernel.org Link: https://lore.kernel.org/20260706140628.87414-2-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/core.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mm/damon/core.c b/mm/damon/core.c index 3c5a46e1c079..d89a7f2faf35 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1917,6 +1917,9 @@ int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive) running_exclusive_ctxs = true; mutex_unlock(&damon_lock); + if (i != nr_ctxs) + damon_stop(ctxs, i); + return err; } From 5c5df7aa649a5f9e0cc96f8aabd3aae66fce667e Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 6 Jul 2026 07:06:17 -0700 Subject: [PATCH 233/501] samples/damon/mtier: do not stop first context for damon_start() failure damon_start() was able to fail while keeping the first context running. mtier hence stops the first context in the case. damon_start() is refactored to avoid that. The error handling is no longer necessary. Remove it. Link: https://lore.kernel.org/20260706140628.87414-3-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- samples/damon/mtier.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/samples/damon/mtier.c b/samples/damon/mtier.c index e567f4edd80e..90881e8bd441 100644 --- a/samples/damon/mtier.c +++ b/samples/damon/mtier.c @@ -193,8 +193,6 @@ static int damon_sample_mtier_start(void) if (!err) return 0; - if (damon_is_running(ctxs[0])) - damon_stop(ctxs, 1); damon_destroy_ctx(ctxs[0]); damon_destroy_ctx(ctxs[1]); return err; From a0f1e3187399cdecd407579eebbed128264f29bd Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 6 Jul 2026 07:06:18 -0700 Subject: [PATCH 234/501] mm/damon/core: make damon_stop() never fail damon_stop() has no reason to fail. It returns an error code only for possible future changes that can make it fail. Such a change has not been made yet, and this only makes the error handling complicated and confusing. Ensure it returns no error. Link: https://lore.kernel.org/20260706140628.87414-4-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/core.c | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index d89a7f2faf35..84e400aa4e82 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1926,10 +1926,8 @@ int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive) /* * __damon_stop() - Stops monitoring of a given context. * @ctx: monitoring context - * - * Return: 0 on success, negative error code otherwise. */ -static int __damon_stop(struct damon_ctx *ctx) +static void __damon_stop(struct damon_ctx *ctx) { struct task_struct *tsk; @@ -1939,31 +1937,24 @@ static int __damon_stop(struct damon_ctx *ctx) get_task_struct(tsk); mutex_unlock(&ctx->kdamond_lock); kthread_stop_put(tsk); - return 0; + return; } mutex_unlock(&ctx->kdamond_lock); - - return -EPERM; } /** * damon_stop() - Stops the monitorings for a given group of contexts. * @ctxs: an array of the pointers for contexts to stop monitoring * @nr_ctxs: size of @ctxs - * - * Return: 0 on success, negative error code otherwise. */ int damon_stop(struct damon_ctx **ctxs, int nr_ctxs) { - int i, err = 0; + int i; - for (i = 0; i < nr_ctxs; i++) { + for (i = 0; i < nr_ctxs; i++) /* nr_running_ctxs is decremented in kdamond_fn */ - err = __damon_stop(ctxs[i]); - if (err) - break; - } - return err; + __damon_stop(ctxs[i]); + return 0; } /** From 1441fc0a2538c23f1f96c4cc7632d9d37a8e9c44 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 6 Jul 2026 07:06:19 -0700 Subject: [PATCH 235/501] mm/damon/sysfs: ignore damon_stop() return value damon_stop() return value is guaranteed to be 0. Ignore it. Link: https://lore.kernel.org/20260706140628.87414-5-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/sysfs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index b65651498e0d..6d7e36c9e509 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -2262,12 +2262,13 @@ static int damon_sysfs_turn_damon_off(struct damon_sysfs_kdamond *kdamond) { if (!kdamond->damon_ctx) return -EINVAL; - return damon_stop(&kdamond->damon_ctx, 1); + damon_stop(&kdamond->damon_ctx, 1); /* * To allow users show final monitoring results of already turned-off * DAMON, we free kdamond->damon_ctx in next * damon_sysfs_turn_damon_on(), or kdamonds_nr_store() */ + return 0; } static int damon_sysfs_damon_call(int (*fn)(void *data), From 3d88372457052d93d57dac922ae6ebcc004d9cb2 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 6 Jul 2026 07:06:20 -0700 Subject: [PATCH 236/501] mm/damon/reclaim: ignore damon_stop() return value damon_stop() return value is guaranteed to be 0. Ignore it. Link: https://lore.kernel.org/20260706140628.87414-6-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/reclaim.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mm/damon/reclaim.c b/mm/damon/reclaim.c index 6469b25cc34f..09e941d75f67 100644 --- a/mm/damon/reclaim.c +++ b/mm/damon/reclaim.c @@ -332,8 +332,10 @@ static int damon_reclaim_turn(bool on) { int err; - if (!on) - return damon_stop(&ctx, 1); + if (!on) { + damon_stop(&ctx, 1); + return 0; + } err = damon_reclaim_apply_parameters(); if (err) From c8518b971312182930ea8f06818df7a8a7145b51 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 6 Jul 2026 07:06:21 -0700 Subject: [PATCH 237/501] mm/damon/lru_sort: ignore damon_stop() return value damon_stop() return value is guaranteed to be 0. Ignore it. Link: https://lore.kernel.org/20260706140628.87414-7-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/lru_sort.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c index 2dd0cd0d2627..e8c389ad3226 100644 --- a/mm/damon/lru_sort.c +++ b/mm/damon/lru_sort.c @@ -409,8 +409,10 @@ static int damon_lru_sort_turn(bool on) { int err; - if (!on) - return damon_stop(&ctx, 1); + if (!on) { + damon_stop(&ctx, 1); + return 0; + } err = damon_lru_sort_apply_parameters(); if (err) From a4d91b8254a95f64bbdd20891f128d993f2dbc2f Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 6 Jul 2026 07:06:22 -0700 Subject: [PATCH 238/501] mm/damon/core: change damon_stop() return type to void damon_stop() always returns 0, and nobody cares. Change the return type to void. Link: https://lore.kernel.org/20260706140628.87414-8-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 2 +- mm/damon/core.c | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 2661231c0ae8..19b7e839bde0 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -1070,7 +1070,7 @@ static inline unsigned int damon_max_nr_accesses(const struct damon_attrs *attrs bool damon_initialized(void); int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive); -int damon_stop(struct damon_ctx **ctxs, int nr_ctxs); +void damon_stop(struct damon_ctx **ctxs, int nr_ctxs); bool damon_is_running(struct damon_ctx *ctx); int damon_kdamond_pid(struct damon_ctx *ctx); diff --git a/mm/damon/core.c b/mm/damon/core.c index 84e400aa4e82..08510bf9b38d 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1947,14 +1947,13 @@ static void __damon_stop(struct damon_ctx *ctx) * @ctxs: an array of the pointers for contexts to stop monitoring * @nr_ctxs: size of @ctxs */ -int damon_stop(struct damon_ctx **ctxs, int nr_ctxs) +void damon_stop(struct damon_ctx **ctxs, int nr_ctxs) { int i; for (i = 0; i < nr_ctxs; i++) /* nr_running_ctxs is decremented in kdamond_fn */ __damon_stop(ctxs[i]); - return 0; } /** From 3e734a0514f9d365773e91abe941decfc3e9331c Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 6 Jul 2026 07:06:23 -0700 Subject: [PATCH 239/501] samples/damon/mtier: stop all contexts with single damon_stop() call damon_stop() was theoretically able to return failure while keeping the second context for mtier running. mtier stops its contexts one by one with two damon_stop() call for the reason. damon_stop() is refactored to always successfully stop all requested DAMON contexts. The two calls are unnecessary. Use only single damon_stop() call for all contexts. Link: https://lore.kernel.org/20260706140628.87414-9-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- samples/damon/mtier.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/samples/damon/mtier.c b/samples/damon/mtier.c index 90881e8bd441..ac9c24b92ead 100644 --- a/samples/damon/mtier.c +++ b/samples/damon/mtier.c @@ -200,8 +200,7 @@ static int damon_sample_mtier_start(void) static void damon_sample_mtier_stop(void) { - damon_stop(ctxs, 1); - damon_stop(&ctxs[1], 1); + damon_stop(ctxs, 2); damon_destroy_ctx(ctxs[0]); damon_destroy_ctx(ctxs[1]); } From d079b6b890d128c396ee42320c027b7dd3c0a2a7 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 6 Jul 2026 07:06:24 -0700 Subject: [PATCH 240/501] mm/damon/core: wait ctx stop in damon_call() before reruning an error damon_call() failure means the DAMON context started its termination. The termination is asynchronously done in kdamond thread. The caller's error handling should handle the race, too. It is complicated and easy to make mistakes. Update damon_call() to ensure the context is stopped in the case, by waiting until the completion is confirmed. Link: https://lore.kernel.org/20260706140628.87414-10-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/core.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 08510bf9b38d..390e00b3685e 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -2010,6 +2010,8 @@ int damon_kdamond_pid(struct damon_ctx *ctx) * @ctx has succeeded. Otherwise, this function could fall into an indefinite * wait. * + * When this function is failed, the @ctx is guaranteed to be stopped. + * * Return: 0 on success, negative error code otherwise. */ int damon_call(struct damon_ctx *ctx, struct damon_call_control *control) @@ -2022,7 +2024,7 @@ int damon_call(struct damon_ctx *ctx, struct damon_call_control *control) mutex_lock(&ctx->call_controls_lock); if (ctx->call_controls_obsolete) { mutex_unlock(&ctx->call_controls_lock); - return -ECANCELED; + goto canceled; } list_add_tail(&control->list, &ctx->call_controls); mutex_unlock(&ctx->call_controls_lock); @@ -2030,8 +2032,14 @@ int damon_call(struct damon_ctx *ctx, struct damon_call_control *control) return 0; wait_for_completion(&control->completion); if (control->canceled) - return -ECANCELED; + goto canceled; return 0; + +canceled: + while (damon_is_running(ctx)) + schedule_timeout_idle(msecs_to_jiffies(100)); + return -ECANCELED; + } /** From cfd843b299298d7dff51bf86b51eee21e47796b7 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 6 Jul 2026 07:06:25 -0700 Subject: [PATCH 241/501] samples/damon/wsse: do not stop ctx for damon_call() failure damon_call() failure was causing the context to be stopped, but asynchronously by the kdamond thread. To handle the race, the caller had to stop the context. damon_call() is updated to do the stop on its own. Remove the damon_stop() call from the damon_call() error handling. Link: https://lore.kernel.org/20260706140628.87414-11-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- samples/damon/wsse.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/samples/damon/wsse.c b/samples/damon/wsse.c index ff5e8a890f44..37fd5da20158 100644 --- a/samples/damon/wsse.c +++ b/samples/damon/wsse.c @@ -93,10 +93,8 @@ static int damon_sample_wsse_start(void) } repeat_call_control.data = ctx; err = damon_call(ctx, &repeat_call_control); - if (err) { - damon_stop(&ctx, 1); + if (err) damon_destroy_ctx(ctx); - } return err; } From b1cc60c13be0e560ff651f23e6fea6df8822bf15 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Mon, 6 Jul 2026 07:06:26 -0700 Subject: [PATCH 242/501] samples/damon/prcl: do not stop DAMON for damon_call() failure damon_call() failure was causing the context to be stopped, but asynchronously by the kdamond thread. To handle the race, the caller had to stop the context. damon_call() is updated to do the stop on its own. Remove the damon_stop() call from the damon_call() error handling. Link: https://lore.kernel.org/20260706140628.87414-12-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- samples/damon/prcl.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/samples/damon/prcl.c b/samples/damon/prcl.c index edeae145c4a8..842099bd6228 100644 --- a/samples/damon/prcl.c +++ b/samples/damon/prcl.c @@ -113,10 +113,8 @@ static int damon_sample_prcl_start(void) repeat_call_control.data = ctx; err = damon_call(ctx, &repeat_call_control); - if (err) { - damon_stop(&ctx, 1); + if (err) damon_destroy_ctx(ctx); - } return err; } From f283f6a4e1ea7059355c438a157e476d7095bd2c Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Thu, 9 Jul 2026 14:06:01 -0400 Subject: [PATCH 243/501] mm/percpu-km: clear page->private before free them Patch series "Keep tail page private zero at free and folio split", v3. This patchset makes sure tail_page->private is zero before compound or high-order pages are returned to the allocator. It also checks tail pages that become new folio heads during large folio split, before their private fields are used by new folios. Note on ZONE_DEVICE and DAX page/folio === ZONE_DEVICE and DAX use prep_compound_tail() to reinitialize folios, so tail_page->private was reset before this patchset. There was a concern that after this patchset stale ->private can appear after ZONE_DEVICE/DAX folio initialization. My reasoning is that no code sets ZONE_DEVICE/DAX page->private, so their page->private stays zero all the time. ZONE_DEVICE_PRIVATE page migration only supports anonymous memory without swapcache, so after the migration ->private remains zero. But let me know if my reasoning is wrong. It can be fixed by adding ->private zeroing code in ZONE_DEVICE/DAX folio initialization code. Motivation === page->private is zeroed at page free time since commit ac1ea219590c0 ("mm/page_alloc: clear page->private in free_pages_prepare()"), since we concluded that it might be too much to ask every page user to free a page with ->private zeroed. The holder of the last page reference might not know whether ->private needs to be cleared. For compound and high-order pages, tail_page->private can also leak to later users if it is left uncleared. The page allocation path does not zero every tail_page->private field, so they can be seen by new users and cause unexpected issues[1]. Check tail_page->private at page free time, and check tail pages that become new folio heads during large folio split. With those checks in place, prep_compound_tail() no longer needs to clear tail_page->private when preparing compound page metadata. Overview === 1. Patch 1 clears all pages ->private before percpu-km frees them. 2. Patch 2 removes setting page->private in compaction code when a free page is taken out of the buddy allocator. cc->freepages is indexed by page order, so storing the free page order in page->private is redundant. In alloc_contig_frozen_range_noprof(), isolate_freepages_range() is used to grab free pages from buddy allocator and it leaves the aforementioned page->private set until either split_free_frozen_pages() or prep_new_page() is called. That stale value without resetting triggers the tail_page->private nonzero check once set_page_private(0) is removed from prep_compound_tail(). 3. Patch 3 adds back the page->private check for tail pages promoted to new folio heads in __split_folio_to_order(). 4. Patch 4 adds a tail_page->private check in the page free path. 5. Patch 5 removes tail_page->private zeroing from prep_compound_tail(). This patch (of 5): page->private is cleared in free page path. In a subsequent commit, tail_page->private will be checked and ensured to be zero. Clearing percpu-km allocated pages' ->private to prevent triggering warnings later, namely undo what we did in pcpu_create_chunk(). Link: https://lore.kernel.org/20260709-keep-subpage-private-zero-at-free-v3-0-7e4fe155f5b9@nvidia.com Link: https://lore.kernel.org/20260709-keep-subpage-private-zero-at-free-v3-1-7e4fe155f5b9@nvidia.com Link: https://lore.kernel.org/all/20260206174017.128673-1-mikhail.v.gavrilov@gmail.com/ [1] Signed-off-by: Zi Yan Acked-by: David Hildenbrand (Arm) Reviewed-by: Vlastimil Babka (SUSE) Cc: Alistair Popple Cc: Baolin Wang Cc: Barry Song Cc: Brendan Jackman Cc: Brendan Jackman Cc: Dennis Zhou Cc: Dev Jain Cc: Johannes Weiner Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Nico Pache Cc: Ryan Roberts Cc: Suren Baghdasaryan Cc: Tejun Heo Signed-off-by: Andrew Morton --- mm/percpu-km.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mm/percpu-km.c b/mm/percpu-km.c index dc096b5a6ce4..65fd5580e447 100644 --- a/mm/percpu-km.c +++ b/mm/percpu-km.c @@ -94,8 +94,15 @@ static void pcpu_destroy_chunk(struct pcpu_chunk *chunk) pcpu_stats_chunk_dealloc(); trace_percpu_destroy_chunk(chunk->base_addr); - if (chunk->data) + if (chunk->data) { + struct page *pages = (struct page *)chunk->data; + int i; + + /* clear chunk info from each page before free them */ + for (i = 0; i < nr_pages; i++) + pcpu_set_page_chunk(pages + i, NULL); __free_pages(chunk->data, order_base_2(nr_pages)); + } pcpu_free_chunk(chunk); } From 7ae5a5d8eca5eb7dd12c9d08e2cba1b2d8cf75e4 Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Thu, 9 Jul 2026 14:06:02 -0400 Subject: [PATCH 244/501] mm/compaction: stop recording free page order in page->private Commit 733aea0b3a7bb ("mm/compaction: add support for >0 order folio memory compaction.") stores isolated free pages in an array indexed by free page orders, it is no longer needed to store the order in each page's ->private field. And there is no code using the stored order. Stop doing that. It also prepares for an upcoming change that ensures subpage->private is zero at page free time and the removal of set_page_private(0) from prep_compound_tail(). Link: https://lore.kernel.org/20260709-keep-subpage-private-zero-at-free-v3-2-7e4fe155f5b9@nvidia.com Signed-off-by: Zi Yan Reviewed-by: David Hildenbrand (Arm) Reviewed-by: Lance Yang Reviewed-by: Baolin Wang Reviewed-by: Vlastimil Babka (SUSE) Cc: Alistair Popple Cc: Barry Song Cc: Brendan Jackman Cc: Brendan Jackman Cc: Dennis Zhou Cc: Dev Jain Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Nico Pache Cc: Ryan Roberts Cc: Suren Baghdasaryan Cc: Tejun Heo Signed-off-by: Andrew Morton --- mm/compaction.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/mm/compaction.c b/mm/compaction.c index 4b2318fad4eb..9f81055a358e 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -645,7 +645,6 @@ static unsigned long isolate_freepages_block(struct compact_control *cc, isolated = __isolate_free_page(page, order); if (!isolated) break; - set_page_private(page, order); nr_scanned += isolated - 1; total_isolated += isolated; @@ -1618,7 +1617,6 @@ static void fast_isolate_freepages(struct compact_control *cc) /* Isolate the page if available */ if (page) { if (__isolate_free_page(page, order)) { - set_page_private(page, order); nr_isolated = 1 << order; nr_scanned += nr_isolated - 1; total_isolated += nr_isolated; @@ -1847,7 +1845,6 @@ static struct folio *compaction_alloc_noprof(struct folio *src, unsigned long da size >>= 1; list_add(&freepage[size].lru, &cc->freepages[start_order]); - set_page_private(&freepage[size], start_order); } dst = (struct folio *)freepage; From d007658032970db4979ea483a2dcd5658f0184cc Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Thu, 9 Jul 2026 14:06:03 -0400 Subject: [PATCH 245/501] mm/huge_memory: add page->private check back in __split_folio_to_order() page->private should not be set in tail pages. Commit 4265d67e405a ("mm/migrate_device: add THP splitting during migration") removed a DEBUG_VM check without a proper reason[1]. Add it back. Link: https://lore.kernel.org/all/13f3fcda-7328-4aa5-afc6-75a294a82b2a@nvidia.com/ [1] Link: https://lore.kernel.org/20260709-keep-subpage-private-zero-at-free-v3-3-7e4fe155f5b9@nvidia.com Signed-off-by: Zi Yan Reviewed-by: Lance Yang Acked-by: David Hildenbrand (Arm) Reviewed-by: Baolin Wang Reviewed-by: Vlastimil Babka (SUSE) Cc: Alistair Popple Cc: Barry Song Cc: Brendan Jackman Cc: Brendan Jackman Cc: Dennis Zhou Cc: Dev Jain Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Nico Pache Cc: Ryan Roberts Cc: Suren Baghdasaryan Cc: Tejun Heo Signed-off-by: Andrew Morton --- mm/huge_memory.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 2289a0416cdc..7f3c53c0d3d4 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -3591,6 +3591,13 @@ static void __split_folio_to_order(struct folio *folio, int old_order, new_folio->mapping = folio->mapping; new_folio->index = folio->index + i; + /* + * page->private should not be set in tail pages. Warn once + * if private is unexpectedly set. Do it before swap.val assignment + * since private overlaps with swap.val. + */ + VM_WARN_ON_ONCE_PAGE(new_folio->private, new_head); + if (folio_test_swapcache(folio)) new_folio->swap.val = folio->swap.val + i; From 04ad9908b0319210ef9e33775dd8a2adb0951586 Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Thu, 9 Jul 2026 14:06:04 -0400 Subject: [PATCH 246/501] mm/page_alloc: make sure tail_page->private is zero at page free time Any code using tail_page->private of a folio, a compound page or a high-order page is supposed to reset it after use, otherwise ->private data can leak to new page user and cause unexpected issues. Add a bad_page() check at page free path for it. Link: https://lore.kernel.org/20260709-keep-subpage-private-zero-at-free-v3-4-7e4fe155f5b9@nvidia.com Signed-off-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Acked-by: David Hildenbrand (Arm) Cc: Alistair Popple Cc: Baolin Wang Cc: Barry Song Cc: Brendan Jackman Cc: Brendan Jackman Cc: Dennis Zhou Cc: Dev Jain Cc: Johannes Weiner Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Nico Pache Cc: Ryan Roberts Cc: Suren Baghdasaryan Cc: Tejun Heo Signed-off-by: Andrew Morton --- mm/page_alloc.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index a63733dac659..cc45dfe89c76 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1377,15 +1377,23 @@ static __always_inline bool __free_pages_prepare(struct page *page, #endif } for (i = 1; i < (1 << order); i++) { + struct page *tail_page = page + i; + if (compound) - bad += free_tail_page_prepare(page, page + i); + bad += free_tail_page_prepare(page, tail_page); if (is_check_pages_enabled()) { - if (free_page_is_bad(page + i)) { + if (free_page_is_bad(tail_page)) { + bad++; + continue; + } + + if (tail_page->private) { + bad_page(tail_page, "nonzero private"); bad++; continue; } } - (page + i)->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP; + tail_page->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP; } } if (folio_test_anon(folio)) { From 819bfbd47e49df893bd1ae26af2e772629452897 Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Thu, 9 Jul 2026 14:06:05 -0400 Subject: [PATCH 247/501] mm/page_alloc: remove set_page_private() in prep_compound_tail() Tail pages are expected to have (and optionally be checked) zeroed ->private when they are freed. It stays true during subsequent reallocation, so replace the tail_page->private initialization with a VM_WARN_ON_ONCE() in compound page preparation. Link: https://lore.kernel.org/20260709-keep-subpage-private-zero-at-free-v3-5-7e4fe155f5b9@nvidia.com Signed-off-by: Zi Yan Acked-by: Vlastimil Babka (SUSE) Acked-by: David Hildenbrand (Arm) Reviewed-by: Lance Yang Cc: Alistair Popple Cc: Baolin Wang Cc: Barry Song Cc: Brendan Jackman Cc: Brendan Jackman Cc: Dennis Zhou Cc: Dev Jain Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Nico Pache Cc: Ryan Roberts Cc: Suren Baghdasaryan Cc: Tejun Heo Signed-off-by: Andrew Morton --- mm/internal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/internal.h b/mm/internal.h index 841c27611627..398741a8ab09 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -741,7 +741,7 @@ static inline void prep_compound_tail(struct page *tail, { tail->mapping = TAIL_MAPPING; set_compound_head(tail, head, order); - set_page_private(tail, 0); + VM_WARN_ON_ONCE(tail->private); } static inline void init_compound_tail(struct page *tail, From 8d596d8b81ef56bf61538d6b4761a7b0d6138280 Mon Sep 17 00:00:00 2001 From: "Barry Song (Xiaomi)" Date: Thu, 2 Jul 2026 07:59:52 +0800 Subject: [PATCH 248/501] mm: avoid unnecessary lru drain for wp_can_reuse_anon_folio() Patch series "mm: drop redundant lru_add_drain in anon folio reuse paths", v3. We are doing a large number of redundant lru_add_drain() calls in both wp_can_reuse_anon_folio() and do_swap_page(), leading to LRU lock contention and unnecessary overhead. In wp_can_reuse_anon_folio(), we can check the refcount against the lru_cache before deciding to drain. In do_swap_page(), the drain is now entirely redundant after Kairui's work to route SYNC I/O through the swapcache in the same way as ASYNC I/O. Build the kernel within a 1 GB memcg using 20 threads with zRAM swap. The number of lru_add_drain() calls is reduced from 276,278 to 226,318, a reduction of about 18%. Build the kernel within an 800 MB memcg using 20 threads with zRAM swap. The number of lru_add_drain() calls is reduced from 778,950 to 541,149, a reduction of 30.5%. This patch (of 4): There is a case where `folio_ref_count(folio) == 3` and `!folio_test_swapcache(folio)`. In that case, both `folio_ref_count(folio) > 3` and `folio_ref_count(folio) > 1 + folio_test_swapcache(folio)` evaluate false, causing an unnecessary local LRU drain. During an Ubuntu boot, I observed over 5,000 redundant local LRU drains. For a kernel build with a minimal configuration, I observed more than 20,000 redundant drains. Fix this by checking against: `1 + in_swapcache + in_lrucache` instead of hardcoding `folio_ref_count(folio) > 3`. [baohua@kernel.org: rename in_lru_cache to maybe_in_lru_cache] Link: https://lore.kernel.org/20260708145718.82690-1-baohua@kernel.org Link: https://lore.kernel.org/20260701235955.36126-1-baohua@kernel.org Link: https://lore.kernel.org/20260701235955.36126-2-baohua@kernel.org Signed-off-by: Barry Song (Xiaomi) Suggested-by: David Hildenbrand (Arm) Reviewed-by: Kairui Song Acked-by: Shakeel Butt Acked-by: David Hildenbrand (Arm) Reviewed-by: Baoquan He Cc: Chris Li Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Nhat Pham Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Usama Arif Signed-off-by: Andrew Morton --- mm/memory.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index 6637c5b13c9b..c10b8a166828 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4174,6 +4174,9 @@ static bool __wp_can_reuse_large_anon_folio(struct folio *folio, static bool wp_can_reuse_anon_folio(struct folio *folio, struct vm_area_struct *vma) { + const bool maybe_in_lru_cache = !folio_test_lru(folio); + const bool in_swapcache = folio_test_swapcache(folio); + if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && folio_test_large(folio)) return __wp_can_reuse_large_anon_folio(folio, vma); @@ -4184,15 +4187,16 @@ static bool wp_can_reuse_anon_folio(struct folio *folio, * * KSM doesn't necessarily raise the folio refcount. */ - if (folio_test_ksm(folio) || folio_ref_count(folio) > 3) + if (folio_test_ksm(folio) || + folio_ref_count(folio) > 1 + maybe_in_lru_cache + in_swapcache) return false; - if (!folio_test_lru(folio)) + if (maybe_in_lru_cache) /* * We cannot easily detect+handle references from * remote LRU caches or references to LRU folios. */ lru_add_drain(); - if (folio_ref_count(folio) > 1 + folio_test_swapcache(folio)) + if (folio_ref_count(folio) > 1 + in_swapcache) return false; if (!folio_trylock(folio)) return false; From 34f6c8932e11db38f928ea99619bdb319a8c1b6b Mon Sep 17 00:00:00 2001 From: "Barry Song (Xiaomi)" Date: Thu, 2 Jul 2026 07:59:53 +0800 Subject: [PATCH 249/501] mm: drop stale folio_ref_count()==1 check in do_swap_page reuse logic The "we just allocated them without exposing them to the swapcache" case no longer exists, as Kairui has routed synchronous I/O through the swapcache as well in his series "unify swapin use swap cache and cleanup flags"[1]. As a result, folio_ref_count() should never be 1 in this path, since at least two references are held (base ref plus swapcache). Remove the folio_ref_count()==1 check and update the comment accordingly. The ksm_might_need_to_copy() check may allocate a fresh folio with folio_ref_count() == 1. Along that path, exclusive has already been set to true, so the folio can still be reused correctly. [akpm@linux-foundation.org: update comment, per David] Link: https://lore.kernel.org/3a185a5d-2f2c-4e9d-9cd9-8bdb236dfc5c@kernel.org Link: https://lore.kernel.org/20260701235955.36126-3-baohua@kernel.org Link: https://lore.kernel.org/all/20251220-swap-table-p2-v5-0-8862a265a033@tencent.com/ [1] Signed-off-by: Barry Song (Xiaomi) Acked-by: Usama Arif Reviewed-by: Kairui Song Reviewed-by: Baoquan He Acked-by: Shakeel Butt Acked-by: David Hildenbrand (Arm) Cc: Chris Li Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Nhat Pham Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/memory.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index c10b8a166828..bc0fded4a00a 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -5039,13 +5039,10 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) pte = pte_mkuffd_wp(pte); /* - * Same logic as in do_wp_page(); however, optimize for pages that are - * certainly not shared either because we just allocated them without - * exposing them to the swapcache or because the swap entry indicates - * exclusivity. + * Similar logic as in do_wp_page(); however, optimize for pages that + * are certainly exclusive. */ - if (!folio_test_ksm(folio) && - (exclusive || folio_ref_count(folio) == 1)) { + if (exclusive) { if ((vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) && !pte_needs_soft_dirty_wp(vma, pte)) { pte = pte_mkwrite(pte, vma); From 38f380f143f779738d725352f479056890488241 Mon Sep 17 00:00:00 2001 From: "Barry Song (Xiaomi)" Date: Thu, 2 Jul 2026 07:59:54 +0800 Subject: [PATCH 250/501] mm: entirely remove lru_add_drain in do_swap_page We are doing a lot of redundant lru_add_drain() calls in do_swap_page(), especially for synchronous I/O devices. For example, the test program below currently ends up draining lru_cache 100% of the time: int main(int argc, char *argv[]) { int i; #define SIZE 100*1024*1024 while(1) { volatile int *p = mmap(0, SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); for (int i = 0; i < SIZE/sizeof(int); i++) p[i] = i%64; madvise((void *)p, SIZE, MADV_PAGEOUT); for (int i = 0; i < SIZE/sizeof(int); i++) p[i] = i%64; munmap(p, SIZE); } return 0; } Folio reuse now relies primarily on the exclusive hint, making lru_cache draining to drop the refcount in lru_cache largely irrelevant. For a kernel build with a minimal configuration running in a 1 GB memcg, this patch skips more than 43,000 redundant local LRU drains. Link: https://lore.kernel.org/20260701235955.36126-4-baohua@kernel.org Signed-off-by: Barry Song (Xiaomi) Acked-by: David Hildenbrand (Arm) Acked-by: Shakeel Butt Cc: Baoquan He Cc: Chris Li Cc: Kairui Song Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Nhat Pham Cc: Suren Baghdasaryan Cc: Usama Arif Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/memory.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index bc0fded4a00a..1cc6a09fff8a 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4894,16 +4894,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) } else if (folio != swapcache) page = folio_page(folio, 0); - /* - * If we want to map a page that's in the swapcache writable, we - * have to detect via the refcount if we're really the exclusive - * owner. Try removing the extra reference from the local LRU - * caches if required. - */ - if ((vmf->flags & FAULT_FLAG_WRITE) && - !folio_test_ksm(folio) && !folio_test_lru(folio)) - lru_add_drain(); - folio_throttle_swaprate(folio, GFP_KERNEL); /* From 3cedb745d354dc539969ba76c7195028839ff9fb Mon Sep 17 00:00:00 2001 From: "Barry Song (Xiaomi)" Date: Thu, 2 Jul 2026 07:59:55 +0800 Subject: [PATCH 251/501] mm: clarify the folio_free_swap() for do_swap_page() Since commit 4b34f1d82c654 ("mm, swap: free the swap cache after folio is mapped"), we have relied on do_wp_page() to handle the non-exclusive case, where the folio may either be reused or require CoW. As a result, using the refcount in do_swap_page() to decide when to free the swap cache is no longer necessary, since do_wp_page() can handle this more cleanly and consistently. We can now simply use FAULT_FLAG_WRITE together with exclusivity to decide when to free the swap cache in do_swap_page(). Link: https://lore.kernel.org/20260701235955.36126-5-baohua@kernel.org Signed-off-by: Barry Song (Xiaomi) Suggested-by: David Hildenbrand (Arm) Acked-by: David Hildenbrand (Arm) Cc: Baoquan He Cc: Chris Li Cc: Kairui Song Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Nhat Pham Cc: Shakeel Butt Cc: Suren Baghdasaryan Cc: Usama Arif Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/memory.c | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index 1cc6a09fff8a..631c170f812a 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4509,7 +4509,7 @@ static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf) static inline bool should_try_to_free_swap(struct swap_info_struct *si, struct folio *folio, struct vm_area_struct *vma, - unsigned int extra_refs, + bool exclusive, unsigned int fault_flags) { if (!folio_test_swapcache(folio)) @@ -4525,14 +4525,12 @@ static inline bool should_try_to_free_swap(struct swap_info_struct *si, if (mem_cgroup_swap_full(folio) || (vma->vm_flags & VM_LOCKED) || folio_test_mlocked(folio)) return true; + /* - * If we want to map a page that's in the swapcache writable, we - * have to detect via the refcount if we're really the exclusive - * user. Try freeing the swapcache to get rid of the swapcache - * reference only in case it's likely that we'll be the exclusive user. + * Free the swapcache only if we are the exclusive user and + * this is a write fault. */ - return (fault_flags & FAULT_FLAG_WRITE) && !folio_test_ksm(folio) && - folio_ref_count(folio) == (extra_refs + folio_nr_pages(folio)); + return (fault_flags & FAULT_FLAG_WRITE) && exclusive; } static vm_fault_t pte_marker_clear(struct vm_fault *vmf) @@ -5036,10 +5034,8 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) if ((vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) && !pte_needs_soft_dirty_wp(vma, pte)) { pte = pte_mkwrite(pte, vma); - if (vmf->flags & FAULT_FLAG_WRITE) { + if (vmf->flags & FAULT_FLAG_WRITE) pte = pte_mkdirty(pte); - vmf->flags &= ~FAULT_FLAG_WRITE; - } } rmap_flags |= RMAP_EXCLUSIVE; } @@ -5079,7 +5075,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) * Do it after mapping, so raced page faults will likely see the folio * in swap cache and wait on the folio lock. */ - if (should_try_to_free_swap(si, folio, vma, nr_pages, vmf->flags)) + if (should_try_to_free_swap(si, folio, vma, exclusive, vmf->flags)) folio_free_swap(folio); folio_unlock(folio); @@ -5096,7 +5092,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) folio_put(swapcache); } - if (vmf->flags & FAULT_FLAG_WRITE) { + if ((vmf->flags & FAULT_FLAG_WRITE) && !pte_write(pte)) { ret |= do_wp_page(vmf); if (ret & VM_FAULT_ERROR) ret &= VM_FAULT_ERROR; From 2253208bec87270517bb7da81d6f5f7558f28973 Mon Sep 17 00:00:00 2001 From: Jianyue Wu Date: Wed, 8 Jul 2026 19:35:43 +0800 Subject: [PATCH 252/501] mm/swap: colocate page-cluster sysctl with swap readahead Patch series "mm: clean up folio LRU and swap declarations", v9. This series splits folio LRU and swap cleanup into three steps: - move the page-cluster sysctl next to swap readahead in mm/swap_state.c - rename mm/swap.c to mm/folio.c after the swap-specific code moves out - move MM-internal reclaim declarations out of include/linux/swap.h After this series, swap cache and swap-in readahead stay in mm/swap_state.c. Folio LRU helpers live in mm/folio.c, and MM-internal reclaim/workingset declarations move from include/linux/swap.h to mm/internal.h (public LRU helpers used outside mm/ remain in swap.h). The first patch handles the swap-specific page-cluster state before the file rename, so the rename commit only carries folio LRU code. The last patch keeps the LRU helpers used outside mm/ in include/linux/swap.h and moves the internal reclaim/workingset declarations to mm/internal.h. This patch (of 3): page_cluster and the vm.page-cluster sysctl are only used by swap-in readahead in swap_state.c. Move them out of swap.c together with swap_readahead_setup(), and make page_cluster static to that file. Rename swap_setup() while moving it as well. The helper is internal to MM and now only sets up swap readahead defaults and its sysctl hook, so the more specific name matches its reduced scope. Call swap_readahead_setup() from swap_init() after moving it, keeping the readahead defaults and sysctl registration with swap_state.c initialization. swap_setup() previously lived in mm/swap.c, which is built unconditionally, so the vm.page-cluster sysctl was registered also on CONFIG_SWAP=n kernels. After moving the setup into swap_state.c, which is built only when CONFIG_SWAP is enabled, vm.page-cluster is no longer registered there. The knob only tunes swap-in readahead and had no effect without swap. Link: https://lore.kernel.org/20260708-ch-swap-series-plus-folio-lru-cleanup-v9-0-2bc72b4f8730@gmail.com Link: https://lore.kernel.org/20260708-ch-swap-series-plus-folio-lru-cleanup-v9-1-2bc72b4f8730@gmail.com Signed-off-by: Jianyue Wu Suggested-by: Baoquan He Suggested-by: Barry Song Suggested-by: Johannes Weiner Suggested-by: Matthew Wilcox Reviewed-by: Kairui Song Acked-by: Johannes Weiner Acked-by: David Hildenbrand (Arm) Cc: Axel Rasmussen Cc: Baolin Wang Cc: Chris Li Cc: Hugh Dickins Cc: Jonathan Corbet Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Michal Hocko Cc: Mike Rapoport Cc: Nhat Pham Cc: Shakeel Butt Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie Signed-off-by: Andrew Morton --- include/linux/swap.h | 1 - mm/swap.c | 36 ------------------------------- mm/swap.h | 2 -- mm/swap_state.c | 50 ++++++++++++++++++++++++++++++++++++++++++-- mm/vmscan.c | 1 - 5 files changed, 48 insertions(+), 42 deletions(-) diff --git a/include/linux/swap.h b/include/linux/swap.h index 3f31b6a56788..45656ca9792d 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -340,7 +340,6 @@ extern void lru_add_drain_cpu_zone(struct zone *zone); extern void lru_add_drain_all(void); void folio_deactivate(struct folio *folio); void folio_mark_lazyfree(struct folio *folio); -extern void swap_setup(void); /* linux/mm/vmscan.c */ extern unsigned long zone_reclaimable_pages(struct zone *zone); diff --git a/mm/swap.c b/mm/swap.c index d25131305c94..c88643e56af7 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -44,10 +44,6 @@ #define CREATE_TRACE_POINTS #include -/* How many pages do we try to swap or page in/out together? As a power of 2 */ -int page_cluster; -static const int page_cluster_max = 31; - struct cpu_fbatches { /* * The following folio batches are grouped together because they are protected @@ -1177,35 +1173,3 @@ void lru_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int lruvec_reparent_lru(child_lruvec, parent_lruvec, lru, nid); } #endif - -static const struct ctl_table swap_sysctl_table[] = { - { - .procname = "page-cluster", - .data = &page_cluster, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = (void *)&page_cluster_max, - } -}; - -/* - * Perform any setup for the swap system - */ -void __init swap_setup(void) -{ - unsigned long megs = PAGES_TO_MB(totalram_pages()); - - /* Use a smaller cluster for small-memory machines */ - if (megs < 16) - page_cluster = 2; - else - page_cluster = 3; - /* - * Right now other parts of the system means that we - * _really_ don't want to cluster much more - */ - - register_sysctl_init("vm", swap_sysctl_table); -} diff --git a/mm/swap.h b/mm/swap.h index 44ab8e1e595b..b51ad3071a73 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -8,8 +8,6 @@ struct mempolicy; struct swap_iocb; struct swap_memcg_table; -extern int page_cluster; - #if defined(MAX_POSSIBLE_PHYSMEM_BITS) #define SWAP_CACHE_PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT) #elif defined(MAX_PHYSMEM_BITS) diff --git a/mm/swap_state.c b/mm/swap_state.c index 6fd6e3415b71..1444d20a40e9 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -22,10 +22,15 @@ #include #include #include +#include #include "internal.h" #include "swap_table.h" #include "swap.h" +/* Swap readahead cluster size, as a power of 2 pages. */ +static int page_cluster; +static const int page_cluster_max = 31; + /* * swapper_space is a fiction, retained to simplify the path through * vmscan's shrink_folio_list. @@ -985,6 +990,35 @@ struct folio *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask, return folio; } +static const struct ctl_table swap_readahead_sysctl_table[] = { + { + .procname = "page-cluster", + .data = &page_cluster, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = (void *)&page_cluster_max, + } +}; + +static void __init swap_readahead_setup(void) +{ + unsigned long megs = PAGES_TO_MB(totalram_pages()); + + /* Use a smaller cluster for small-memory machines */ + if (megs < 16) + page_cluster = 2; + else + page_cluster = 3; + /* + * Right now other parts of the system means that we + * _really_ don't want to cluster much more + */ + + register_sysctl_init("vm", swap_readahead_sysctl_table); +} + #ifdef CONFIG_SYSFS static ssize_t vma_ra_enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) @@ -1014,7 +1048,7 @@ static const struct attribute_group swap_attr_group = { .attrs = swap_attrs, }; -static int __init swap_init(void) +static int __init swap_sysfs_init(void) { int err; struct kobject *swap_kobj; @@ -1037,5 +1071,17 @@ static int __init swap_init(void) kobject_put(swap_kobj); return err; } -subsys_initcall(swap_init); +#else +static int __init swap_sysfs_init(void) +{ + return 0; +} #endif + +static int __init swap_init(void) +{ + swap_readahead_setup(); + + return swap_sysfs_init(); +} +subsys_initcall(swap_init); diff --git a/mm/vmscan.c b/mm/vmscan.c index 8e7b909e47e4..745c5c358d5e 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -7647,7 +7647,6 @@ static int __init kswapd_init(void) { int nid; - swap_setup(); for_each_node_state(nid, N_MEMORY) kswapd_run(nid); register_sysctl_init("vm", vmscan_sysctl_table); From 0ba14428abc22e31ac9b5227365524eb36f8994e Mon Sep 17 00:00:00 2001 From: Jianyue Wu Date: Wed, 8 Jul 2026 19:35:44 +0800 Subject: [PATCH 253/501] mm: rename swap.c to folio.c Rename mm/swap.c to mm/folio.c so the filename better matches the code's main responsibility. This keeps the implementation split from swap-specific code without changing the published LRU helper interfaces. Update MAINTAINERS and the remaining mm/swap.c documentation references after the rename. [wujianyue000@gmail.com: MAINTAINERS: move mm/folio.c to MM CORE] Link: https://lore.kernel.org/20260804111109.393127-1-wujianyue000@gmail.com Link: https://lore.kernel.org/20260708-ch-swap-series-plus-folio-lru-cleanup-v9-2-2bc72b4f8730@gmail.com Signed-off-by: Jianyue Wu Suggested-by: Baoquan He Suggested-by: David Hildenbrand Suggested-by: Matthew Wilcox Reviewed-by: Lorenzo Stoakes Acked-by: Johannes Weiner Acked-by: David Hildenbrand (Arm) Cc: Axel Rasmussen Cc: Baolin Wang Cc: Barry Song Cc: Chris Li Cc: Hugh Dickins Cc: Jonathan Corbet Cc: Kairui Song Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Michal Hocko Cc: Michal Hocko Cc: Mike Rapoport Cc: Nhat Pham Cc: Shakeel Butt Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie Signed-off-by: Andrew Morton --- Documentation/admin-guide/sysctl/vm.rst | 3 --- Documentation/core-api/mm-api.rst | 2 +- MAINTAINERS | 3 +-- mm/Makefile | 2 +- mm/{swap.c => folio.c} | 10 +++------- 5 files changed, 6 insertions(+), 14 deletions(-) rename mm/{swap.c => folio.c} (99%) diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst index 22cc54cac3b2..5b318d17aa4b 100644 --- a/Documentation/admin-guide/sysctl/vm.rst +++ b/Documentation/admin-guide/sysctl/vm.rst @@ -19,9 +19,6 @@ The files in this directory can be used to tune the operation of the virtual memory (VM) subsystem of the Linux kernel and the writeout of dirty data to disk. -Default values and initialization routines for most of these -files can be found in mm/swap.c. - Currently, these files are in /proc/sys/vm: - admin_reserve_kbytes diff --git a/Documentation/core-api/mm-api.rst b/Documentation/core-api/mm-api.rst index 4df7d5edbee5..c1d03a5a2a19 100644 --- a/Documentation/core-api/mm-api.rst +++ b/Documentation/core-api/mm-api.rst @@ -118,7 +118,7 @@ More Memory Management Functions .. #kernel-doc:: mm/hmm.c (build warnings) .. kernel-doc:: mm/memremap.c .. kernel-doc:: mm/hugetlb.c -.. kernel-doc:: mm/swap.c +.. kernel-doc:: mm/folio.c .. kernel-doc:: mm/memcontrol.c .. #kernel-doc:: mm/memory-tiers.c (build warnings) .. kernel-doc:: mm/shmem.c diff --git a/MAINTAINERS b/MAINTAINERS index bff676f08c63..0d47b358b2a4 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -16988,6 +16988,7 @@ F: kernel/fork.c F: mm/Kconfig F: mm/debug.c F: mm/folio-compat.c +F: mm/folio.c F: mm/highmem.c F: mm/init-mm.c F: mm/internal.h @@ -17088,7 +17089,6 @@ F: Documentation/admin-guide/mm/multigen_lru.rst F: Documentation/mm/multigen_lru.rst F: include/linux/mm_inline.h F: include/linux/mmzone.h -F: mm/swap.c F: mm/vmscan.c F: mm/workingset.c @@ -17240,7 +17240,6 @@ F: include/linux/swap.h F: include/linux/swapfile.h F: include/linux/swapops.h F: mm/page_io.c -F: mm/swap.c F: mm/swap.h F: mm/swap_table.h F: mm/swap_state.c diff --git a/mm/Makefile b/mm/Makefile index 4fc713867b9b..8e7e964b99a4 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -50,7 +50,7 @@ endif obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ maccess.o page-writeback.o folio-compat.o \ - readahead.o swap.o truncate.o vmscan.o shrinker.o \ + readahead.o folio.o truncate.o vmscan.o shrinker.o \ shmem.o util.o mmzone.o vmstat.o backing-dev.o \ mm_init.o percpu.o slab_common.o \ compaction.o show_mem.o \ diff --git a/mm/swap.c b/mm/folio.c similarity index 99% rename from mm/swap.c rename to mm/folio.c index c88643e56af7..d2937600cf72 100644 --- a/mm/swap.c +++ b/mm/folio.c @@ -1,17 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * linux/mm/swap.c + * linux/mm/folio.c * * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds */ /* - * This file contains the default values for the operation of the - * Linux VM subsystem. Fine-tuning documentation can be found in - * Documentation/admin-guide/sysctl/vm.rst. - * Started 18.12.91 - * Swap aging added 23.2.95, Stephen Tweedie. - * Buffermem limits added 12.3.98, Rik van Riel. + * Folio LRU helpers: add/remove folios from LRU lists, batching, + * activation/deactivation, and page cache release paths. */ #include From 3fe0620fae7c2a14e9aaad251c6eb225b6c16dc0 Mon Sep 17 00:00:00 2001 From: Jianyue Wu Date: Wed, 8 Jul 2026 19:35:45 +0800 Subject: [PATCH 254/501] mm: move reclaim-internal declarations out of swap.h Keep include/linux/swap.h focused on swap-facing interfaces by moving MM-internal reclaim and workingset declarations into mm/internal.h. Leave the small set of LRU helper declarations that are used outside mm/ in swap.h so this cleanup does not need a new public header under include/linux/. Link: https://lore.kernel.org/20260708-ch-swap-series-plus-folio-lru-cleanup-v9-3-2bc72b4f8730@gmail.com Signed-off-by: Jianyue Wu Suggested-by: Barry Song Suggested-by: Baoquan He Acked-by: Johannes Weiner Cc: Axel Rasmussen Cc: Baolin Wang Cc: Chris Li Cc: David Hildenbrand Cc: Hugh Dickins Cc: Jonathan Corbet Cc: Kairui Song Cc: Kemeng Shi Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Michal Hocko Cc: Mike Rapoport Cc: Nhat Pham Cc: Shakeel Butt Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie Signed-off-by: Andrew Morton --- include/linux/swap.h | 75 +++++--------------------------------------- mm/internal.h | 67 +++++++++++++++++++++++++++++++++++++++ mm/memfd.c | 1 + 3 files changed, 75 insertions(+), 68 deletions(-) diff --git a/include/linux/swap.h b/include/linux/swap.h index 45656ca9792d..696ed01709c2 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -287,39 +287,19 @@ static inline swp_entry_t page_swap_entry(struct page *page) return entry; } -/* linux/mm/workingset.c */ -bool workingset_test_recent(void *shadow, bool file, bool *workingset, - bool flush); -void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages); -void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg); -void workingset_refault(struct folio *folio, void *shadow); -void workingset_activation(struct folio *folio); - /* linux/mm/page_alloc.c */ extern unsigned long totalreserve_pages; /* Definition of global_zone_page_state not available yet */ #define nr_free_pages() global_zone_page_state(NR_FREE_PAGES) +/* linux/mm/folio.c */ +void folio_add_lru(struct folio *folio); +void folio_mark_accessed(struct folio *folio); +void lru_add_drain_all(void); -/* linux/mm/swap.c */ -void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file, - unsigned int nr_io, unsigned int nr_rotated); -void lru_note_cost_refault(struct folio *); -void folio_add_lru(struct folio *); -void folio_add_lru_vma(struct folio *, struct vm_area_struct *); -void mark_page_accessed(struct page *); -void folio_mark_accessed(struct folio *); - -static inline bool folio_may_be_lru_cached(struct folio *folio) -{ - /* - * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting. - * Holding small numbers of low-order mTHP folios in per-CPU LRU cache - * will be sensible, but nobody has implemented and tested that yet. - */ - return !folio_test_large(folio); -} +/* linux/mm/folio-compat.c */ +void mark_page_accessed(struct page *page); extern atomic_t lru_disable_count; @@ -328,42 +308,6 @@ static inline bool lru_cache_disabled(void) return atomic_read(&lru_disable_count); } -static inline void lru_cache_enable(void) -{ - atomic_dec(&lru_disable_count); -} - -extern void lru_cache_disable(void); -extern void lru_add_drain(void); -extern void lru_add_drain_cpu(int cpu); -extern void lru_add_drain_cpu_zone(struct zone *zone); -extern void lru_add_drain_all(void); -void folio_deactivate(struct folio *folio); -void folio_mark_lazyfree(struct folio *folio); - -/* linux/mm/vmscan.c */ -extern unsigned long zone_reclaimable_pages(struct zone *zone); -extern unsigned long try_to_free_pages(struct zonelist *zonelist, int order, - gfp_t gfp_mask, const nodemask_t *mask); -unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx); - -#define MEMCG_RECLAIM_MAY_SWAP (1 << 1) -#define MEMCG_RECLAIM_PROACTIVE (1 << 2) -#define MIN_SWAPPINESS 0 -#define MAX_SWAPPINESS 200 - -/* Just reclaim from anon folios in proactive memory reclaim */ -#define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1) - -extern unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, - unsigned long nr_pages, - gfp_t gfp_mask, - unsigned int reclaim_options, - int *swappiness); -extern unsigned long mem_cgroup_shrink_node(struct mem_cgroup *mem, - gfp_t gfp_mask, bool noswap, - pg_data_t *pgdat, - unsigned long *nr_scanned); extern unsigned long shrink_all_memory(unsigned long nr_pages); extern int vm_swappiness; long remove_mapping(struct address_space *mapping, struct folio *folio); @@ -384,11 +328,6 @@ static inline void reclaim_unregister_node(struct node *node) } #endif /* CONFIG_SYSFS && CONFIG_NUMA */ -#ifdef CONFIG_NUMA -extern int sysctl_min_unmapped_ratio; -extern int sysctl_min_slab_ratio; -#endif - void check_move_unevictable_folios(struct folio_batch *fbatch); extern void __meminit kswapd_run(int nid); @@ -544,7 +483,7 @@ static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg) void lru_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid); #else -static inline int mem_cgroup_swappiness(struct mem_cgroup *mem) +static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg) { return READ_ONCE(vm_swappiness); } diff --git a/mm/internal.h b/mm/internal.h index 398741a8ab09..c289d167ffac 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -31,6 +31,73 @@ struct huge_bootmem_page { unsigned long flags; }; +/* mm/workingset.c */ +bool workingset_test_recent(void *shadow, bool file, bool *workingset, + bool flush); +void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages); +void *workingset_eviction(struct folio *folio, + struct mem_cgroup *target_memcg); +void workingset_refault(struct folio *folio, void *shadow); +void workingset_activation(struct folio *folio); + +/* mm/folio.c */ +void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file, + unsigned int nr_io, unsigned int nr_rotated); +void lru_note_cost_refault(struct folio *folio); +void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma); + +static inline bool folio_may_be_lru_cached(struct folio *folio) +{ + /* + * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting. + * Holding small numbers of low-order mTHP folios in per-CPU LRU cache + * will be sensible, but nobody has implemented and tested that yet. + */ + return !folio_test_large(folio); +} + +static inline void lru_cache_enable(void) +{ + atomic_dec(&lru_disable_count); +} + +void lru_cache_disable(void); +void lru_add_drain(void); +void lru_add_drain_cpu(int cpu); +void lru_add_drain_cpu_zone(struct zone *zone); +void folio_deactivate(struct folio *folio); +void folio_mark_lazyfree(struct folio *folio); + +/* mm/vmscan.c */ +unsigned long zone_reclaimable_pages(struct zone *zone); +unsigned long try_to_free_pages(struct zonelist *zonelist, int order, + gfp_t gfp_mask, const nodemask_t *mask); +unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, + int zone_idx); + +#define MEMCG_RECLAIM_MAY_SWAP (1 << 1) +#define MEMCG_RECLAIM_PROACTIVE (1 << 2) +#define MIN_SWAPPINESS 0 +#define MAX_SWAPPINESS 200 + +/* Just reclaim from anon folios in proactive memory reclaim */ +#define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1) + +unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, + unsigned long nr_pages, + gfp_t gfp_mask, + unsigned int reclaim_options, + int *swappiness); +unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, + gfp_t gfp_mask, bool noswap, + pg_data_t *pgdat, + unsigned long *nr_scanned); + +#ifdef CONFIG_NUMA +extern int sysctl_min_unmapped_ratio; +extern int sysctl_min_slab_ratio; +#endif + /* * Maintains state across a page table move. The operation assumes both source * and destination VMAs already exist and are specified by the user. diff --git a/mm/memfd.c b/mm/memfd.c index abe13b291ddc..6c72fe6caef7 100644 --- a/mm/memfd.c +++ b/mm/memfd.c @@ -19,6 +19,7 @@ #include #include #include +#include "internal.h" #include "swap.h" /* From 889d3783ae3f78c8593bc7837a2a68b49db4166d Mon Sep 17 00:00:00 2001 From: Jaeyeon Lee Date: Wed, 8 Jul 2026 13:19:41 +0200 Subject: [PATCH 255/501] mm/shmem: annotate benign data-race in shmem_getattr() shmem_getattr() reads info->alloced, info->swapped and i_mapping->nrpages without info->lock to decide whether shmem_recalc_inode() should be called. shmem_recalc_inode() recomputes these under info->lock, so a stale read here only affects whether shmem_recalc_inode() runs. Annotate data_race() to silence KCSAN. Link: https://lore.kernel.org/20260708111941.35460-1-jaeyeon.lee.dev@gmail.com Signed-off-by: Jaeyeon Lee Reported-by: syzbot+dfb578404df369f6599b@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=dfb578404df369f6599b Cc: Baolin Wang Cc: Hugh Dickins Cc: Marco Elver Signed-off-by: Andrew Morton --- mm/shmem.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/shmem.c b/mm/shmem.c index fa8bb59f87b4..0a36b48f667e 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1297,7 +1297,8 @@ static int shmem_getattr(struct mnt_idmap *idmap, struct inode *inode = path->dentry->d_inode; struct shmem_inode_info *info = SHMEM_I(inode); - if (info->alloced - info->swapped != inode->i_mapping->nrpages) + /* Fast-path hint; recalc under info->lock corrects any stale read. */ + if (data_race(info->alloced - info->swapped != inode->i_mapping->nrpages)) shmem_recalc_inode(inode, 0, 0); if (info->fsflags & FS_APPEND_FL) From c6f11553c067d7d9ed2dcffe1e01780441d4e6e2 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Wed, 8 Jul 2026 12:14:02 +0100 Subject: [PATCH 256/501] mm: decouple protnone helpers from CONFIG_NUMA_BALANCING Patch series "userfaultfd: working set tracking for VM guest memory", v10. This series adds userfaultfd support for tracking the working set of VM guest memory, so a VMM can identify hot pages and reclaim cold ones to tiered or remote storage. This patch (of 15): pte_protnone() and pmd_protnone() detect present-but-inaccessible page table entries. This capability is useful beyond NUMA balancing -- for example, userfaultfd working set tracking uses protnone PTEs to track page access without unmapping pages. Introduce CONFIG_ARCH_HAS_PTE_PROTNONE to decouple the protnone PTE infrastructure from CONFIG_NUMA_BALANCING. The six architectures that support protnone PTEs (x86_64, arm64, powerpc, s390, riscv, loongarch) now select this option, and CONFIG_NUMA_BALANCING depends on it. No functional change -- the same set of architectures continues to have working protnone support, but the infrastructure is now available independently of NUMA balancing. Link: https://lore.kernel.org/20260708111417.173443-1-kirill@shutemov.name Link: https://lore.kernel.org/20260708111417.173443-2-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Assisted-by: Claude:claude-opus-4-6 Acked-by: SeongJae Park Acked-by: Mike Rapoport (Microsoft) Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: James Houghton Cc: Jonathan Corbet Cc: Liam Howlett Cc: Lorenzo Stoakes Cc: Paolo Bonzini Cc: Peter Xu Cc: Sean Christopherson Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/pgtable.h | 7 ++--- arch/loongarch/Kconfig | 1 + arch/loongarch/include/asm/pgtable.h | 4 +-- arch/powerpc/include/asm/book3s/64/pgtable.h | 8 ++--- arch/powerpc/platforms/Kconfig.cputype | 1 + arch/riscv/Kconfig | 1 + arch/riscv/include/asm/pgtable.h | 7 ++--- arch/s390/Kconfig | 1 + arch/s390/include/asm/pgtable.h | 4 +-- arch/x86/Kconfig | 1 + arch/x86/include/asm/pgtable.h | 8 ++--- include/linux/pgtable.h | 32 ++++++++++++++------ init/Kconfig | 8 +++++ mm/debug_vm_pgtable.c | 4 +-- 15 files changed, 52 insertions(+), 36 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index e136e5abdb33..11a9c534b7b4 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -78,6 +78,7 @@ config ARM64 select ARCH_SUPPORTS_CFI select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 + select ARCH_HAS_PTE_PROTNONE select ARCH_SUPPORTS_NUMA_BALANCING select ARCH_SUPPORTS_PAGE_TABLE_CHECK select ARCH_SUPPORTS_PER_VMA_LOCK diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index ed1cc6dba780..313f75af1207 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -553,10 +553,7 @@ static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ -#ifdef CONFIG_NUMA_BALANCING -/* - * See the comment in include/linux/pgtable.h - */ +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pte_protnone(pte_t pte) { /* @@ -575,7 +572,7 @@ static inline int pmd_protnone(pmd_t pmd) { return pte_protnone(pmd_pte(pmd)); } -#endif +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ #define pmd_present(pmd) pte_present(pmd_pte(pmd)) #define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd)) diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index c6c611f44f3f..e20acbe5fe7b 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -67,6 +67,7 @@ config LOONGARCH select ARCH_SUPPORTS_LTO_CLANG select ARCH_SUPPORTS_LTO_CLANG_THIN select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS + select ARCH_HAS_PTE_PROTNONE if 64BIT select ARCH_SUPPORTS_NUMA_BALANCING if NUMA select ARCH_SUPPORTS_PER_VMA_LOCK select ARCH_SUPPORTS_RT diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index 223528c04d73..1952e34bc8ee 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -623,7 +623,7 @@ static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -#ifdef CONFIG_NUMA_BALANCING +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline long pte_protnone(pte_t pte) { return (pte_val(pte) & _PAGE_PROTNONE); @@ -633,7 +633,7 @@ static inline long pmd_protnone(pmd_t pmd) { return (pmd_val(pmd) & _PAGE_PROTNONE); } -#endif /* CONFIG_NUMA_BALANCING */ +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ #define pmd_leaf(pmd) ((pmd_val(pmd) & _PAGE_HUGE) != 0) #define pud_leaf(pud) ((pud_val(pud) & _PAGE_HUGE) != 0) diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 800ff3cc5f71..f4db7d7fbd5c 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -490,13 +490,13 @@ static inline pte_t pte_clear_soft_dirty(pte_t pte) } #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ -#ifdef CONFIG_NUMA_BALANCING +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pte_protnone(pte_t pte) { return (pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE | _PAGE_RWX)) == cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE); } -#endif /* CONFIG_NUMA_BALANCING */ +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ static inline bool pte_hw_valid(pte_t pte) { @@ -1067,12 +1067,12 @@ static inline pte_t *pmdp_ptep(pmd_t *pmd) #endif #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ -#ifdef CONFIG_NUMA_BALANCING +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pmd_protnone(pmd_t pmd) { return pte_protnone(pmd_pte(pmd)); } -#endif /* CONFIG_NUMA_BALANCING */ +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ #define pmd_write(pmd) pte_write(pmd_pte(pmd)) diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype index 9f6847b53e37..f0adbf7ecb7f 100644 --- a/arch/powerpc/platforms/Kconfig.cputype +++ b/arch/powerpc/platforms/Kconfig.cputype @@ -87,6 +87,7 @@ config PPC_BOOK3S_64 select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_SPLIT_PMD_PTLOCK select ARCH_SUPPORTS_HUGETLBFS + select ARCH_HAS_PTE_PROTNONE select ARCH_SUPPORTS_NUMA_BALANCING select HAVE_MOVE_PMD select HAVE_MOVE_PUD diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 760d0e0ae1f0..7b9c373d82fa 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -71,6 +71,7 @@ config RISCV select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS if 64BIT && MMU select ARCH_SUPPORTS_PAGE_TABLE_CHECK if MMU select ARCH_SUPPORTS_PER_VMA_LOCK if MMU + select ARCH_HAS_PTE_PROTNONE if MMU select ARCH_SUPPORTS_RT select ARCH_SUPPORTS_SHADOW_CALL_STACK if HAVE_SHADOW_CALL_STACK select ARCH_SUPPORTS_SCHED_MC if SMP diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 23eae82c26a6..cbc58ec42752 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -534,10 +534,7 @@ static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) PAGE_SIZE) #endif -#ifdef CONFIG_NUMA_BALANCING -/* - * See the comment in include/asm-generic/pgtable.h - */ +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pte_protnone(pte_t pte) { return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE)) == _PAGE_PROT_NONE; @@ -547,7 +544,7 @@ static inline int pmd_protnone(pmd_t pmd) { return pte_protnone(pmd_pte(pmd)); } -#endif +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ /* Modify page protection bits */ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 6b317717bf48..ab8fccc2cc4e 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -148,6 +148,7 @@ config S390 select ARCH_SUPPORTS_HUGETLBFS select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && CC_IS_CLANG select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS + select ARCH_HAS_PTE_PROTNONE select ARCH_SUPPORTS_NUMA_BALANCING select ARCH_SUPPORTS_PAGE_TABLE_CHECK select ARCH_SUPPORTS_PER_VMA_LOCK diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 645fa2d99c6a..e882663a58e7 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -842,7 +842,7 @@ static inline int pte_same(pte_t a, pte_t b) return pte_val(a) == pte_val(b); } -#ifdef CONFIG_NUMA_BALANCING +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pte_protnone(pte_t pte) { return pte_present(pte) && !(pte_val(pte) & _PAGE_READ); @@ -853,7 +853,7 @@ static inline int pmd_protnone(pmd_t pmd) /* pmd_leaf(pmd) implies pmd_present(pmd) */ return pmd_leaf(pmd) && !(pmd_val(pmd) & _SEGMENT_ENTRY_READ); } -#endif +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ static inline bool pte_swp_exclusive(pte_t pte) { diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 50acb2830570..fb298e219179 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -124,6 +124,7 @@ config X86 select ARCH_SUPPORTS_DEBUG_PAGEALLOC select ARCH_SUPPORTS_HUGETLBFS select ARCH_SUPPORTS_PAGE_TABLE_CHECK if X86_64 + select ARCH_HAS_PTE_PROTNONE if X86_64 select ARCH_SUPPORTS_NUMA_BALANCING if X86_64 select ARCH_SUPPORTS_KMAP_LOCAL_FORCE_MAP if NR_CPUS <= 4096 select ARCH_SUPPORTS_CFI if X86_64 diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index f69b02c1d3d3..ef61a75cd98b 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -985,11 +985,7 @@ static inline int pmd_present(pmd_t pmd) return pmd_flags(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PSE); } -#ifdef CONFIG_NUMA_BALANCING -/* - * These work without NUMA balancing but the kernel does not care. See the - * comment in include/linux/pgtable.h - */ +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pte_protnone(pte_t pte) { return (pte_flags(pte) & (_PAGE_PROTNONE | _PAGE_PRESENT)) @@ -1001,7 +997,7 @@ static inline int pmd_protnone(pmd_t pmd) return (pmd_flags(pmd) & (_PAGE_PROTNONE | _PAGE_PRESENT)) == _PAGE_PROTNONE; } -#endif /* CONFIG_NUMA_BALANCING */ +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ static inline int pmd_none(pmd_t pmd) { diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 3df6db1e9e39..8c093c119e5a 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -2110,18 +2110,26 @@ static inline int pud_trans_unstable(pud_t *pud) return 0; } -#ifndef CONFIG_NUMA_BALANCING +#ifndef CONFIG_ARCH_HAS_PTE_PROTNONE /* - * In an inaccessible (PROT_NONE) VMA, pte_protnone() may indicate "yes". It is - * perfectly valid to indicate "no" in that case, which is why our default - * implementation defaults to "always no". + * In an inaccessible (PROT_NONE) VMA, pte_protnone() may indicate "yes". It + * is perfectly valid to indicate "no" in that case, which is why our + * default implementation defaults to "always no". * - * In an accessible VMA, however, pte_protnone() reliably indicates PROT_NONE - * page protection due to NUMA hinting. NUMA hinting faults only apply in - * accessible VMAs. + * In an accessible VMA, pte_protnone() reliably indicates a present + * PROT_NONE page protection. Today the kernel uses such PTEs for two + * purposes: NUMA hinting faults, and userfaultfd RWP tracking on + * VM_UFFD_RWP VMAs. The two are distinguished by the uffd PTE bit and + * the VMA flag; see include/linux/userfaultfd_k.h. * - * So, to reliably identify PROT_NONE PTEs that require a NUMA hinting fault, - * looking at the VMA accessibility is sufficient. + * So, to reliably identify PROT_NONE PTEs that require kernel handling, + * looking at the VMA accessibility (and the uffd bit on RWP VMAs) is + * sufficient. + * + * Architectures without CONFIG_ARCH_HAS_PTE_PROTNONE get the always-zero + * stubs below; PAGE_NONE references that survive to runtime fire the + * BUILD_BUG() fallback, since callers should have folded such paths to + * dead code via IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE). */ static inline int pte_protnone(pte_t pte) { @@ -2132,7 +2140,11 @@ static inline int pmd_protnone(pmd_t pmd) { return 0; } -#endif /* CONFIG_NUMA_BALANCING */ + +#ifndef PAGE_NONE +#define PAGE_NONE ({ BUILD_BUG(); (pgprot_t){0}; }) +#endif +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ #endif /* CONFIG_MMU */ diff --git a/init/Kconfig b/init/Kconfig index 10f2013b5321..c24f345f794b 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -947,6 +947,13 @@ config SCHED_PROXY_EXEC endmenu +# +# For architectures that support present-but-inaccessible (PROT_NONE) page +# table entries detectable via pte_protnone() / pmd_protnone(): +# +config ARCH_HAS_PTE_PROTNONE + bool + # # For architectures that want to enable the support for NUMA-affine scheduler # balancing logic: @@ -1013,6 +1020,7 @@ config ARCH_WANT_NUMA_VARIABLE_LOCALITY config NUMA_BALANCING bool "Memory placement aware NUMA scheduler" depends on ARCH_SUPPORTS_NUMA_BALANCING + depends on ARCH_HAS_PTE_PROTNONE depends on !ARCH_WANT_NUMA_VARIABLE_LOCALITY depends on SMP && NUMA_MIGRATION && !PREEMPT_RT help diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c index 7987962950c1..2875fd22d7bb 100644 --- a/mm/debug_vm_pgtable.c +++ b/mm/debug_vm_pgtable.c @@ -672,7 +672,7 @@ static void __init pte_protnone_tests(struct pgtable_debug_args *args) { pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot_none); - if (!IS_ENABLED(CONFIG_NUMA_BALANCING)) + if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE)) return; pr_debug("Validating PTE protnone\n"); @@ -685,7 +685,7 @@ static void __init pmd_protnone_tests(struct pgtable_debug_args *args) { pmd_t pmd; - if (!IS_ENABLED(CONFIG_NUMA_BALANCING)) + if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE)) return; if (!has_transparent_hugepage()) From 2bd507795e629e5039924685f42ebf90ac191680 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Wed, 8 Jul 2026 12:14:03 +0100 Subject: [PATCH 257/501] mm: rename uffd-wp PTE bit macros to uffd The uffd-wp PTE bit is about to gain a second consumer: userfaultfd RWP will use the same bit to mark access-tracking PTEs, distinct from mprotect(PROT_NONE) or NUMA-hinting PTEs. WP vs RWP semantics come from the VMA flag; the bit is just "uffd has claimed this entry." Drop the "_wp" suffix from the arch-private bit macros so they reflect that. x86: _PAGE_BIT_UFFD_WP -> _PAGE_BIT_UFFD _PAGE_UFFD_WP -> _PAGE_UFFD _PAGE_SWP_UFFD_WP -> _PAGE_SWP_UFFD arm64: PTE_UFFD_WP -> PTE_UFFD PTE_SWP_UFFD_WP -> PTE_SWP_UFFD riscv: _PAGE_UFFD_WP -> _PAGE_UFFD _PAGE_SWP_UFFD_WP -> _PAGE_SWP_UFFD Pure mechanical rename -- no behavior change. Link: https://lore.kernel.org/20260708111417.173443-3-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau Assisted-by: Claude:claude-opus-4-6 Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: SeongJae Park Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: James Houghton Cc: Jonathan Corbet Cc: Liam Howlett Cc: Lorenzo Stoakes Cc: Paolo Bonzini Cc: Peter Xu Cc: Sean Christopherson Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- arch/arm64/include/asm/pgtable-prot.h | 8 ++++---- arch/arm64/include/asm/pgtable.h | 12 ++++++------ arch/riscv/include/asm/pgtable-bits.h | 12 ++++++------ arch/riscv/include/asm/pgtable.h | 14 +++++++------- arch/x86/include/asm/pgtable.h | 24 ++++++++++++------------ arch/x86/include/asm/pgtable_types.h | 16 ++++++++-------- 6 files changed, 43 insertions(+), 43 deletions(-) diff --git a/arch/arm64/include/asm/pgtable-prot.h b/arch/arm64/include/asm/pgtable-prot.h index 212ce1b02e15..09d7c00cf405 100644 --- a/arch/arm64/include/asm/pgtable-prot.h +++ b/arch/arm64/include/asm/pgtable-prot.h @@ -28,11 +28,11 @@ #define PTE_PRESENT_VALID_KERNEL (PTE_VALID | PTE_MAYBE_NG) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -#define PTE_UFFD_WP (_AT(pteval_t, 1) << 58) /* uffd-wp tracking */ -#define PTE_SWP_UFFD_WP (_AT(pteval_t, 1) << 3) /* only for swp ptes */ +#define PTE_UFFD (_AT(pteval_t, 1) << 58) /* userfaultfd tracking */ +#define PTE_SWP_UFFD (_AT(pteval_t, 1) << 3) /* only for swp ptes */ #else -#define PTE_UFFD_WP (_AT(pteval_t, 0)) -#define PTE_SWP_UFFD_WP (_AT(pteval_t, 0)) +#define PTE_UFFD (_AT(pteval_t, 0)) +#define PTE_SWP_UFFD (_AT(pteval_t, 0)) #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ #define _PROT_DEFAULT (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 313f75af1207..0e490a2cf665 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -343,17 +343,17 @@ static inline pmd_t pmd_mknoncont(pmd_t pmd) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP static inline int pte_uffd_wp(pte_t pte) { - return !!(pte_val(pte) & PTE_UFFD_WP); + return !!(pte_val(pte) & PTE_UFFD); } static inline pte_t pte_mkuffd_wp(pte_t pte) { - return pte_wrprotect(set_pte_bit(pte, __pgprot(PTE_UFFD_WP))); + return pte_wrprotect(set_pte_bit(pte, __pgprot(PTE_UFFD))); } static inline pte_t pte_clear_uffd_wp(pte_t pte) { - return clear_pte_bit(pte, __pgprot(PTE_UFFD_WP)); + return clear_pte_bit(pte, __pgprot(PTE_UFFD)); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ @@ -539,17 +539,17 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP static inline pte_t pte_swp_mkuffd_wp(pte_t pte) { - return set_pte_bit(pte, __pgprot(PTE_SWP_UFFD_WP)); + return set_pte_bit(pte, __pgprot(PTE_SWP_UFFD)); } static inline int pte_swp_uffd_wp(pte_t pte) { - return !!(pte_val(pte) & PTE_SWP_UFFD_WP); + return !!(pte_val(pte) & PTE_SWP_UFFD); } static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) { - return clear_pte_bit(pte, __pgprot(PTE_SWP_UFFD_WP)); + return clear_pte_bit(pte, __pgprot(PTE_SWP_UFFD)); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ diff --git a/arch/riscv/include/asm/pgtable-bits.h b/arch/riscv/include/asm/pgtable-bits.h index b422d9691e60..d5a86b4df3ce 100644 --- a/arch/riscv/include/asm/pgtable-bits.h +++ b/arch/riscv/include/asm/pgtable-bits.h @@ -40,20 +40,20 @@ #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -/* ext_svrsw60t59b: Bit(60) for uffd-wp tracking */ -#define _PAGE_UFFD_WP \ +/* ext_svrsw60t59b: Bit(60) for userfaultfd tracking */ +#define _PAGE_UFFD \ ((riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B)) ? \ (1UL << 60) : 0) /* * Bit 4 is not involved into swap entry computation, so we - * can borrow it for swap page uffd-wp tracking. + * can borrow it for swap page userfaultfd tracking. */ -#define _PAGE_SWP_UFFD_WP \ +#define _PAGE_SWP_UFFD \ ((riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B)) ? \ _PAGE_USER : 0) #else -#define _PAGE_UFFD_WP 0 -#define _PAGE_SWP_UFFD_WP 0 +#define _PAGE_UFFD 0 +#define _PAGE_SWP_UFFD 0 #endif #define _PAGE_TABLE _PAGE_PRESENT diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index cbc58ec42752..ae4307cbaed5 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -415,32 +415,32 @@ static inline pte_t pte_wrprotect(pte_t pte) static inline bool pte_uffd_wp(pte_t pte) { - return !!(pte_val(pte) & _PAGE_UFFD_WP); + return !!(pte_val(pte) & _PAGE_UFFD); } static inline pte_t pte_mkuffd_wp(pte_t pte) { - return pte_wrprotect(__pte(pte_val(pte) | _PAGE_UFFD_WP)); + return pte_wrprotect(__pte(pte_val(pte) | _PAGE_UFFD)); } static inline pte_t pte_clear_uffd_wp(pte_t pte) { - return __pte(pte_val(pte) & ~(_PAGE_UFFD_WP)); + return __pte(pte_val(pte) & ~(_PAGE_UFFD)); } static inline bool pte_swp_uffd_wp(pte_t pte) { - return !!(pte_val(pte) & _PAGE_SWP_UFFD_WP); + return !!(pte_val(pte) & _PAGE_SWP_UFFD); } static inline pte_t pte_swp_mkuffd_wp(pte_t pte) { - return __pte(pte_val(pte) | _PAGE_SWP_UFFD_WP); + return __pte(pte_val(pte) | _PAGE_SWP_UFFD); } static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) { - return __pte(pte_val(pte) & ~(_PAGE_SWP_UFFD_WP)); + return __pte(pte_val(pte) & ~(_PAGE_SWP_UFFD)); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ @@ -1167,7 +1167,7 @@ static inline pud_t pud_modify(pud_t pud, pgprot_t newprot) * bit 0: _PAGE_PRESENT (zero) * bit 1 to 2: (zero) * bit 3: _PAGE_SWP_SOFT_DIRTY - * bit 4: _PAGE_SWP_UFFD_WP + * bit 4: _PAGE_SWP_UFFD * bit 5: _PAGE_PROT_NONE (zero) * bit 6: exclusive marker * bits 7 to 11: swap type diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index ef61a75cd98b..54432c308f26 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -413,17 +413,17 @@ static inline pte_t pte_wrprotect(pte_t pte) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP static inline int pte_uffd_wp(pte_t pte) { - return pte_flags(pte) & _PAGE_UFFD_WP; + return pte_flags(pte) & _PAGE_UFFD; } static inline pte_t pte_mkuffd_wp(pte_t pte) { - return pte_wrprotect(pte_set_flags(pte, _PAGE_UFFD_WP)); + return pte_wrprotect(pte_set_flags(pte, _PAGE_UFFD)); } static inline pte_t pte_clear_uffd_wp(pte_t pte) { - return pte_clear_flags(pte, _PAGE_UFFD_WP); + return pte_clear_flags(pte, _PAGE_UFFD); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ @@ -528,17 +528,17 @@ static inline pmd_t pmd_wrprotect(pmd_t pmd) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP static inline int pmd_uffd_wp(pmd_t pmd) { - return pmd_flags(pmd) & _PAGE_UFFD_WP; + return pmd_flags(pmd) & _PAGE_UFFD; } static inline pmd_t pmd_mkuffd_wp(pmd_t pmd) { - return pmd_wrprotect(pmd_set_flags(pmd, _PAGE_UFFD_WP)); + return pmd_wrprotect(pmd_set_flags(pmd, _PAGE_UFFD)); } static inline pmd_t pmd_clear_uffd_wp(pmd_t pmd) { - return pmd_clear_flags(pmd, _PAGE_UFFD_WP); + return pmd_clear_flags(pmd, _PAGE_UFFD); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ @@ -1562,32 +1562,32 @@ static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP static inline pte_t pte_swp_mkuffd_wp(pte_t pte) { - return pte_set_flags(pte, _PAGE_SWP_UFFD_WP); + return pte_set_flags(pte, _PAGE_SWP_UFFD); } static inline int pte_swp_uffd_wp(pte_t pte) { - return pte_flags(pte) & _PAGE_SWP_UFFD_WP; + return pte_flags(pte) & _PAGE_SWP_UFFD; } static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) { - return pte_clear_flags(pte, _PAGE_SWP_UFFD_WP); + return pte_clear_flags(pte, _PAGE_SWP_UFFD); } static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd) { - return pmd_set_flags(pmd, _PAGE_SWP_UFFD_WP); + return pmd_set_flags(pmd, _PAGE_SWP_UFFD); } static inline int pmd_swp_uffd_wp(pmd_t pmd) { - return pmd_flags(pmd) & _PAGE_SWP_UFFD_WP; + return pmd_flags(pmd) & _PAGE_SWP_UFFD; } static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd) { - return pmd_clear_flags(pmd, _PAGE_SWP_UFFD_WP); + return pmd_clear_flags(pmd, _PAGE_SWP_UFFD); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h index 2ec250ba467e..af08d98be930 100644 --- a/arch/x86/include/asm/pgtable_types.h +++ b/arch/x86/include/asm/pgtable_types.h @@ -31,7 +31,7 @@ #define _PAGE_BIT_SPECIAL _PAGE_BIT_SOFTW1 #define _PAGE_BIT_CPA_TEST _PAGE_BIT_SOFTW1 -#define _PAGE_BIT_UFFD_WP _PAGE_BIT_SOFTW2 /* userfaultfd wrprotected */ +#define _PAGE_BIT_UFFD _PAGE_BIT_SOFTW2 /* userfaultfd tracking */ #define _PAGE_BIT_SOFT_DIRTY _PAGE_BIT_SOFTW3 /* software dirty tracking */ #define _PAGE_BIT_KERNEL_4K _PAGE_BIT_SOFTW3 /* page must not be converted to large */ @@ -39,7 +39,7 @@ #define _PAGE_BIT_SAVED_DIRTY _PAGE_BIT_SOFTW5 /* Saved Dirty bit (leaf) */ #define _PAGE_BIT_NOPTISHADOW _PAGE_BIT_SOFTW5 /* No PTI shadow (root PGD) */ #else -/* Shared with _PAGE_BIT_UFFD_WP which is not supported on 32 bit */ +/* Shared with _PAGE_BIT_UFFD which is not supported on 32 bit */ #define _PAGE_BIT_SAVED_DIRTY _PAGE_BIT_SOFTW2 /* Saved Dirty bit (leaf) */ #define _PAGE_BIT_NOPTISHADOW _PAGE_BIT_SOFTW2 /* No PTI shadow (root PGD) */ #endif @@ -111,11 +111,11 @@ #endif #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -#define _PAGE_UFFD_WP (_AT(pteval_t, 1) << _PAGE_BIT_UFFD_WP) -#define _PAGE_SWP_UFFD_WP _PAGE_USER +#define _PAGE_UFFD (_AT(pteval_t, 1) << _PAGE_BIT_UFFD) +#define _PAGE_SWP_UFFD _PAGE_USER #else -#define _PAGE_UFFD_WP (_AT(pteval_t, 0)) -#define _PAGE_SWP_UFFD_WP (_AT(pteval_t, 0)) +#define _PAGE_UFFD (_AT(pteval_t, 0)) +#define _PAGE_SWP_UFFD (_AT(pteval_t, 0)) #endif #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) @@ -129,7 +129,7 @@ /* * The hardware requires shadow stack to be Write=0,Dirty=1. However, * there are valid cases where the kernel might create read-only PTEs that - * are dirty (e.g., fork(), mprotect(), uffd-wp(), soft-dirty tracking). In + * are dirty (e.g., fork(), mprotect(), userfaultfd, soft-dirty tracking). In * this case, the _PAGE_SAVED_DIRTY bit is used instead of the HW-dirty bit, * to avoid creating a wrong "shadow stack" PTEs. Such PTEs have * (Write=0,SavedDirty=1,Dirty=0) set. @@ -151,7 +151,7 @@ #define _COMMON_PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \ _PAGE_SPECIAL | _PAGE_ACCESSED | \ _PAGE_DIRTY_BITS | _PAGE_SOFT_DIRTY | \ - _PAGE_CC | _PAGE_UFFD_WP) + _PAGE_CC | _PAGE_UFFD) #define _PAGE_CHG_MASK (_COMMON_PAGE_CHG_MASK | _PAGE_PAT) #define _HPAGE_CHG_MASK (_COMMON_PAGE_CHG_MASK | _PAGE_PSE | _PAGE_PAT_LARGE) From c0ffe79eedc1d6f821e8fdac9106351a5a95e77d Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Wed, 8 Jul 2026 12:14:04 +0100 Subject: [PATCH 258/501] mm: rename uffd-wp PTE accessors to uffd Userfaultfd RWP will reuse the uffd-wp PTE bit to mark access-tracking PTEs, alongside the write-protected ones it already marks. The bit's meaning now depends on the VMA flag (WP or RWP), not on its name. Rename the kernel-internal names that describe the bit: - pte/pmd/huge_pte accessors (and swap variants) - pgtable_supports_uffd() capability query - SCAN_PTE_UFFD khugepaged enum The ftrace string emitted by mm_khugepaged_scan_pmd for this enum is kept as "pte_uffd_wp" so existing trace-based tooling keeps matching. Pure mechanical rename -- no behavior change. Link: https://lore.kernel.org/20260708111417.173443-4-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau Assisted-by: Claude:claude-opus-4-6 Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: SeongJae Park Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: James Houghton Cc: Jonathan Corbet Cc: Liam Howlett Cc: Lorenzo Stoakes Cc: Paolo Bonzini Cc: Peter Xu Cc: Sean Christopherson Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- arch/arm64/include/asm/pgtable.h | 28 +++++++-------- arch/riscv/include/asm/pgtable.h | 38 ++++++++++---------- arch/s390/include/asm/hugetlb.h | 12 +++---- arch/x86/include/asm/pgtable.h | 24 ++++++------- fs/proc/task_mmu.c | 44 +++++++++++------------ include/asm-generic/hugetlb.h | 18 +++++----- include/asm-generic/pgtable_uffd.h | 32 ++++++++--------- include/linux/leafops.h | 4 +-- include/linux/mm_inline.h | 4 +-- include/linux/swapops.h | 4 +-- include/linux/userfaultfd_k.h | 14 ++++---- include/trace/events/huge_memory.h | 2 +- mm/huge_memory.c | 58 +++++++++++++++--------------- mm/hugetlb.c | 44 +++++++++++------------ mm/internal.h | 4 +-- mm/khugepaged.c | 22 ++++++------ mm/memory.c | 34 +++++++++--------- mm/migrate.c | 12 +++---- mm/migrate_device.c | 8 ++--- mm/mprotect.c | 12 +++---- mm/mremap.c | 4 +-- mm/page_table_check.c | 8 ++--- mm/rmap.c | 16 ++++----- mm/swapfile.c | 4 +-- mm/userfaultfd.c | 6 ++-- 25 files changed, 228 insertions(+), 228 deletions(-) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 0e490a2cf665..a2681d755358 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -341,17 +341,17 @@ static inline pmd_t pmd_mknoncont(pmd_t pmd) } #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static inline int pte_uffd_wp(pte_t pte) +static inline int pte_uffd(pte_t pte) { return !!(pte_val(pte) & PTE_UFFD); } -static inline pte_t pte_mkuffd_wp(pte_t pte) +static inline pte_t pte_mkuffd(pte_t pte) { return pte_wrprotect(set_pte_bit(pte, __pgprot(PTE_UFFD))); } -static inline pte_t pte_clear_uffd_wp(pte_t pte) +static inline pte_t pte_clear_uffd(pte_t pte) { return clear_pte_bit(pte, __pgprot(PTE_UFFD)); } @@ -537,17 +537,17 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) } #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static inline pte_t pte_swp_mkuffd_wp(pte_t pte) +static inline pte_t pte_swp_mkuffd(pte_t pte) { return set_pte_bit(pte, __pgprot(PTE_SWP_UFFD)); } -static inline int pte_swp_uffd_wp(pte_t pte) +static inline int pte_swp_uffd(pte_t pte) { return !!(pte_val(pte) & PTE_SWP_UFFD); } -static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) +static inline pte_t pte_swp_clear_uffd(pte_t pte) { return clear_pte_bit(pte, __pgprot(PTE_SWP_UFFD)); } @@ -590,13 +590,13 @@ static inline int pmd_protnone(pmd_t pmd) #define pmd_mkvalid_k(pmd) pte_pmd(pte_mkvalid_k(pmd_pte(pmd))) #define pmd_mkinvalid(pmd) pte_pmd(pte_mkinvalid(pmd_pte(pmd))) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -#define pmd_uffd_wp(pmd) pte_uffd_wp(pmd_pte(pmd)) -#define pmd_mkuffd_wp(pmd) pte_pmd(pte_mkuffd_wp(pmd_pte(pmd))) -#define pmd_clear_uffd_wp(pmd) pte_pmd(pte_clear_uffd_wp(pmd_pte(pmd))) -#define pmd_swp_uffd_wp(pmd) pte_swp_uffd_wp(pmd_pte(pmd)) -#define pmd_swp_mkuffd_wp(pmd) pte_pmd(pte_swp_mkuffd_wp(pmd_pte(pmd))) -#define pmd_swp_clear_uffd_wp(pmd) \ - pte_pmd(pte_swp_clear_uffd_wp(pmd_pte(pmd))) +#define pmd_uffd(pmd) pte_uffd(pmd_pte(pmd)) +#define pmd_mkuffd(pmd) pte_pmd(pte_mkuffd(pmd_pte(pmd))) +#define pmd_clear_uffd(pmd) pte_pmd(pte_clear_uffd(pmd_pte(pmd))) +#define pmd_swp_uffd(pmd) pte_swp_uffd(pmd_pte(pmd)) +#define pmd_swp_mkuffd(pmd) pte_pmd(pte_swp_mkuffd(pmd_pte(pmd))) +#define pmd_swp_clear_uffd(pmd) \ + pte_pmd(pte_swp_clear_uffd(pmd_pte(pmd))) #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ #define pmd_write(pmd) pte_write(pmd_pte(pmd)) @@ -1512,7 +1512,7 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, * Encode and decode a swap entry: * bits 0-1: present (must be zero) * bits 2: remember PG_anon_exclusive - * bit 3: remember uffd-wp state + * bit 3: remember uffd state * bits 6-10: swap type * bit 11: PTE_PRESENT_INVALID (must be zero) * bits 12-61: swap offset diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index ae4307cbaed5..7f21742b2b6a 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -410,35 +410,35 @@ static inline pte_t pte_wrprotect(pte_t pte) } #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -#define pgtable_supports_uffd_wp() \ +#define pgtable_supports_uffd() \ riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B) -static inline bool pte_uffd_wp(pte_t pte) +static inline bool pte_uffd(pte_t pte) { return !!(pte_val(pte) & _PAGE_UFFD); } -static inline pte_t pte_mkuffd_wp(pte_t pte) +static inline pte_t pte_mkuffd(pte_t pte) { return pte_wrprotect(__pte(pte_val(pte) | _PAGE_UFFD)); } -static inline pte_t pte_clear_uffd_wp(pte_t pte) +static inline pte_t pte_clear_uffd(pte_t pte) { return __pte(pte_val(pte) & ~(_PAGE_UFFD)); } -static inline bool pte_swp_uffd_wp(pte_t pte) +static inline bool pte_swp_uffd(pte_t pte) { return !!(pte_val(pte) & _PAGE_SWP_UFFD); } -static inline pte_t pte_swp_mkuffd_wp(pte_t pte) +static inline pte_t pte_swp_mkuffd(pte_t pte) { return __pte(pte_val(pte) | _PAGE_SWP_UFFD); } -static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) +static inline pte_t pte_swp_clear_uffd(pte_t pte) { return __pte(pte_val(pte) & ~(_PAGE_SWP_UFFD)); } @@ -896,34 +896,34 @@ static inline pud_t pud_mkspecial(pud_t pud) #endif #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static inline bool pmd_uffd_wp(pmd_t pmd) +static inline bool pmd_uffd(pmd_t pmd) { - return pte_uffd_wp(pmd_pte(pmd)); + return pte_uffd(pmd_pte(pmd)); } -static inline pmd_t pmd_mkuffd_wp(pmd_t pmd) +static inline pmd_t pmd_mkuffd(pmd_t pmd) { - return pte_pmd(pte_mkuffd_wp(pmd_pte(pmd))); + return pte_pmd(pte_mkuffd(pmd_pte(pmd))); } -static inline pmd_t pmd_clear_uffd_wp(pmd_t pmd) +static inline pmd_t pmd_clear_uffd(pmd_t pmd) { - return pte_pmd(pte_clear_uffd_wp(pmd_pte(pmd))); + return pte_pmd(pte_clear_uffd(pmd_pte(pmd))); } -static inline bool pmd_swp_uffd_wp(pmd_t pmd) +static inline bool pmd_swp_uffd(pmd_t pmd) { - return pte_swp_uffd_wp(pmd_pte(pmd)); + return pte_swp_uffd(pmd_pte(pmd)); } -static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_mkuffd(pmd_t pmd) { - return pte_pmd(pte_swp_mkuffd_wp(pmd_pte(pmd))); + return pte_pmd(pte_swp_mkuffd(pmd_pte(pmd))); } -static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_clear_uffd(pmd_t pmd) { - return pte_pmd(pte_swp_clear_uffd_wp(pmd_pte(pmd))); + return pte_pmd(pte_swp_clear_uffd(pmd_pte(pmd))); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ diff --git a/arch/s390/include/asm/hugetlb.h b/arch/s390/include/asm/hugetlb.h index e33a5b587ee4..02821a7d9d91 100644 --- a/arch/s390/include/asm/hugetlb.h +++ b/arch/s390/include/asm/hugetlb.h @@ -77,20 +77,20 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, __set_huge_pte_at(mm, addr, ptep, pte_wrprotect(pte)); } -#define __HAVE_ARCH_HUGE_PTE_MKUFFD_WP -static inline pte_t huge_pte_mkuffd_wp(pte_t pte) +#define __HAVE_ARCH_HUGE_PTE_MKUFFD +static inline pte_t huge_pte_mkuffd(pte_t pte) { return pte; } -#define __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD_WP -static inline pte_t huge_pte_clear_uffd_wp(pte_t pte) +#define __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD +static inline pte_t huge_pte_clear_uffd(pte_t pte) { return pte; } -#define __HAVE_ARCH_HUGE_PTE_UFFD_WP -static inline int huge_pte_uffd_wp(pte_t pte) +#define __HAVE_ARCH_HUGE_PTE_UFFD +static inline int huge_pte_uffd(pte_t pte) { return 0; } diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 54432c308f26..8e0018fadd14 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -411,17 +411,17 @@ static inline pte_t pte_wrprotect(pte_t pte) } #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static inline int pte_uffd_wp(pte_t pte) +static inline int pte_uffd(pte_t pte) { return pte_flags(pte) & _PAGE_UFFD; } -static inline pte_t pte_mkuffd_wp(pte_t pte) +static inline pte_t pte_mkuffd(pte_t pte) { return pte_wrprotect(pte_set_flags(pte, _PAGE_UFFD)); } -static inline pte_t pte_clear_uffd_wp(pte_t pte) +static inline pte_t pte_clear_uffd(pte_t pte) { return pte_clear_flags(pte, _PAGE_UFFD); } @@ -526,17 +526,17 @@ static inline pmd_t pmd_wrprotect(pmd_t pmd) } #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static inline int pmd_uffd_wp(pmd_t pmd) +static inline int pmd_uffd(pmd_t pmd) { return pmd_flags(pmd) & _PAGE_UFFD; } -static inline pmd_t pmd_mkuffd_wp(pmd_t pmd) +static inline pmd_t pmd_mkuffd(pmd_t pmd) { return pmd_wrprotect(pmd_set_flags(pmd, _PAGE_UFFD)); } -static inline pmd_t pmd_clear_uffd_wp(pmd_t pmd) +static inline pmd_t pmd_clear_uffd(pmd_t pmd) { return pmd_clear_flags(pmd, _PAGE_UFFD); } @@ -1560,32 +1560,32 @@ static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) #endif #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static inline pte_t pte_swp_mkuffd_wp(pte_t pte) +static inline pte_t pte_swp_mkuffd(pte_t pte) { return pte_set_flags(pte, _PAGE_SWP_UFFD); } -static inline int pte_swp_uffd_wp(pte_t pte) +static inline int pte_swp_uffd(pte_t pte) { return pte_flags(pte) & _PAGE_SWP_UFFD; } -static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) +static inline pte_t pte_swp_clear_uffd(pte_t pte) { return pte_clear_flags(pte, _PAGE_SWP_UFFD); } -static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_mkuffd(pmd_t pmd) { return pmd_set_flags(pmd, _PAGE_SWP_UFFD); } -static inline int pmd_swp_uffd_wp(pmd_t pmd) +static inline int pmd_swp_uffd(pmd_t pmd) { return pmd_flags(pmd) & _PAGE_SWP_UFFD; } -static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_clear_uffd(pmd_t pmd) { return pmd_clear_flags(pmd, _PAGE_SWP_UFFD); } diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 95012243b9b3..75cfd9befb4d 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -2035,14 +2035,14 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, page = vm_normal_page(vma, addr, pte); if (pte_soft_dirty(pte)) flags |= PM_SOFT_DIRTY; - if (pte_uffd_wp(pte)) + if (pte_uffd(pte)) flags |= PM_UFFD_WP; } else { softleaf_t entry; if (pte_swp_soft_dirty(pte)) flags |= PM_SOFT_DIRTY; - if (pte_swp_uffd_wp(pte)) + if (pte_swp_uffd(pte)) flags |= PM_UFFD_WP; entry = softleaf_from_pte(pte); if (pm->show_pfn) { @@ -2108,7 +2108,7 @@ static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr, flags |= PM_PRESENT; if (pmd_soft_dirty(pmd)) flags |= PM_SOFT_DIRTY; - if (pmd_uffd_wp(pmd)) + if (pmd_uffd(pmd)) flags |= PM_UFFD_WP; if (pm->show_pfn) frame = pmd_pfn(pmd) + idx; @@ -2127,7 +2127,7 @@ static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr, flags |= PM_SWAP; if (pmd_swp_soft_dirty(pmd)) flags |= PM_SOFT_DIRTY; - if (pmd_swp_uffd_wp(pmd)) + if (pmd_swp_uffd(pmd)) flags |= PM_UFFD_WP; if (softleaf_has_pfn(entry)) page = softleaf_to_page(entry); @@ -2233,14 +2233,14 @@ static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask, !hugetlb_pmd_shared(ptep)) flags |= PM_MMAP_EXCLUSIVE; - if (huge_pte_uffd_wp(pte)) + if (huge_pte_uffd(pte)) flags |= PM_UFFD_WP; flags |= PM_PRESENT; if (pm->show_pfn) frame = pte_pfn(pte) + ((addr & ~hmask) >> PAGE_SHIFT); - } else if (pte_swp_uffd_wp_any(pte)) { + } else if (pte_swp_uffd_any(pte)) { flags |= PM_UFFD_WP; } @@ -2451,7 +2451,7 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p, categories = PAGE_IS_PRESENT; - if (!pte_uffd_wp(pte)) + if (!pte_uffd(pte)) categories |= PAGE_IS_WRITTEN; if (p->masks_of_interest & PAGE_IS_FILE) { @@ -2469,7 +2469,7 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p, categories = PAGE_IS_SWAPPED; - if (!pte_swp_uffd_wp_any(pte)) + if (!pte_swp_uffd_any(pte)) categories |= PAGE_IS_WRITTEN; entry = softleaf_from_pte(pte); @@ -2494,13 +2494,13 @@ static void make_uffd_wp_pte(struct vm_area_struct *vma, pte_t old_pte; old_pte = ptep_modify_prot_start(vma, addr, pte); - ptent = pte_mkuffd_wp(old_pte); + ptent = pte_mkuffd(old_pte); ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent); } else if (pte_none(ptent)) { set_pte_at(vma->vm_mm, addr, pte, make_pte_marker(PTE_MARKER_UFFD_WP)); } else { - ptent = pte_swp_mkuffd_wp(ptent); + ptent = pte_swp_mkuffd(ptent); set_pte_at(vma->vm_mm, addr, pte, ptent); } } @@ -2519,7 +2519,7 @@ static unsigned long pagemap_thp_category(struct pagemap_scan_private *p, struct page *page; categories |= PAGE_IS_PRESENT; - if (!pmd_uffd_wp(pmd)) + if (!pmd_uffd(pmd)) categories |= PAGE_IS_WRITTEN; if (p->masks_of_interest & PAGE_IS_FILE) { @@ -2534,7 +2534,7 @@ static unsigned long pagemap_thp_category(struct pagemap_scan_private *p, categories |= PAGE_IS_SOFT_DIRTY; } else { categories |= PAGE_IS_SWAPPED; - if (!pmd_swp_uffd_wp(pmd)) + if (!pmd_swp_uffd(pmd)) categories |= PAGE_IS_WRITTEN; if (pmd_swp_soft_dirty(pmd)) categories |= PAGE_IS_SOFT_DIRTY; @@ -2558,10 +2558,10 @@ static void make_uffd_wp_pmd(struct vm_area_struct *vma, if (pmd_present(pmd)) { old = pmdp_invalidate_ad(vma, addr, pmdp); - pmd = pmd_mkuffd_wp(old); + pmd = pmd_mkuffd(old); set_pmd_at(vma->vm_mm, addr, pmdp, pmd); } else if (pmd_is_migration_entry(pmd)) { - pmd = pmd_swp_mkuffd_wp(pmd); + pmd = pmd_swp_mkuffd(pmd); set_pmd_at(vma->vm_mm, addr, pmdp, pmd); } } @@ -2583,7 +2583,7 @@ static unsigned long pagemap_hugetlb_category(pte_t pte) if (pte_present(pte)) { categories |= PAGE_IS_PRESENT; - if (!huge_pte_uffd_wp(pte)) + if (!huge_pte_uffd(pte)) categories |= PAGE_IS_WRITTEN; if (!PageAnon(pte_page(pte))) categories |= PAGE_IS_FILE; @@ -2594,7 +2594,7 @@ static unsigned long pagemap_hugetlb_category(pte_t pte) } else { categories |= PAGE_IS_SWAPPED; - if (!pte_swp_uffd_wp_any(pte)) + if (!pte_swp_uffd_any(pte)) categories |= PAGE_IS_WRITTEN; if (pte_swp_soft_dirty(pte)) categories |= PAGE_IS_SOFT_DIRTY; @@ -2622,12 +2622,12 @@ static void make_uffd_wp_huge_pte(struct vm_area_struct *vma, if (softleaf_is_migration(entry)) { set_huge_pte_at(vma->vm_mm, addr, ptep, - pte_swp_mkuffd_wp(ptent), psize); + pte_swp_mkuffd(ptent), psize); } else { pte_t old_pte, new_pte; old_pte = huge_ptep_modify_prot_start(vma, addr, ptep); - new_pte = huge_pte_mkuffd_wp(old_pte); + new_pte = huge_pte_mkuffd(old_pte); huge_ptep_modify_prot_commit(vma, addr, ptep, old_pte, new_pte); } } @@ -2860,8 +2860,8 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { pte_t ptent = ptep_get(pte); - if ((pte_present(ptent) && pte_uffd_wp(ptent)) || - pte_swp_uffd_wp_any(ptent)) + if ((pte_present(ptent) && pte_uffd(ptent)) || + pte_swp_uffd_any(ptent)) continue; make_uffd_wp_pte(vma, addr, pte, ptent); if (!flush_end) @@ -2878,8 +2878,8 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, unsigned long next = addr + PAGE_SIZE; pte_t ptent = ptep_get(pte); - if ((pte_present(ptent) && pte_uffd_wp(ptent)) || - pte_swp_uffd_wp_any(ptent)) + if ((pte_present(ptent) && pte_uffd(ptent)) || + pte_swp_uffd_any(ptent)) continue; ret = pagemap_scan_output(p->cur_vma_category | PAGE_IS_WRITTEN, p, addr, &next); diff --git a/include/asm-generic/hugetlb.h b/include/asm-generic/hugetlb.h index e1a2e1b7c8e7..635c41cc3479 100644 --- a/include/asm-generic/hugetlb.h +++ b/include/asm-generic/hugetlb.h @@ -37,24 +37,24 @@ static inline pte_t huge_pte_modify(pte_t pte, pgprot_t newprot) return pte_modify(pte, newprot); } -#ifndef __HAVE_ARCH_HUGE_PTE_MKUFFD_WP -static inline pte_t huge_pte_mkuffd_wp(pte_t pte) +#ifndef __HAVE_ARCH_HUGE_PTE_MKUFFD +static inline pte_t huge_pte_mkuffd(pte_t pte) { - return huge_pte_wrprotect(pte_mkuffd_wp(pte)); + return huge_pte_wrprotect(pte_mkuffd(pte)); } #endif -#ifndef __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD_WP -static inline pte_t huge_pte_clear_uffd_wp(pte_t pte) +#ifndef __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD +static inline pte_t huge_pte_clear_uffd(pte_t pte) { - return pte_clear_uffd_wp(pte); + return pte_clear_uffd(pte); } #endif -#ifndef __HAVE_ARCH_HUGE_PTE_UFFD_WP -static inline int huge_pte_uffd_wp(pte_t pte) +#ifndef __HAVE_ARCH_HUGE_PTE_UFFD +static inline int huge_pte_uffd(pte_t pte) { - return pte_uffd_wp(pte); + return pte_uffd(pte); } #endif diff --git a/include/asm-generic/pgtable_uffd.h b/include/asm-generic/pgtable_uffd.h index 0d85791efdf7..30e88fc1de2f 100644 --- a/include/asm-generic/pgtable_uffd.h +++ b/include/asm-generic/pgtable_uffd.h @@ -2,79 +2,79 @@ #define _ASM_GENERIC_PGTABLE_UFFD_H /* - * Some platforms can customize the uffd-wp bit, making it unavailable + * Some platforms can customize the uffd PTE bit, making it unavailable * even if the architecture provides the resource. * Adding this API allows architectures to add their own checks for the * devices on which the kernel is running. * Note: When overriding it, please make sure the * CONFIG_HAVE_ARCH_USERFAULTFD_WP is part of this macro. */ -#ifndef pgtable_supports_uffd_wp -#define pgtable_supports_uffd_wp() IS_ENABLED(CONFIG_HAVE_ARCH_USERFAULTFD_WP) +#ifndef pgtable_supports_uffd +#define pgtable_supports_uffd() IS_ENABLED(CONFIG_HAVE_ARCH_USERFAULTFD_WP) #endif static inline bool uffd_supports_wp_marker(void) { - return pgtable_supports_uffd_wp() && IS_ENABLED(CONFIG_PTE_MARKER_UFFD_WP); + return pgtable_supports_uffd() && IS_ENABLED(CONFIG_PTE_MARKER_UFFD_WP); } #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static __always_inline int pte_uffd_wp(pte_t pte) +static __always_inline int pte_uffd(pte_t pte) { return 0; } -static __always_inline int pmd_uffd_wp(pmd_t pmd) +static __always_inline int pmd_uffd(pmd_t pmd) { return 0; } -static __always_inline pte_t pte_mkuffd_wp(pte_t pte) +static __always_inline pte_t pte_mkuffd(pte_t pte) { return pte; } -static __always_inline pmd_t pmd_mkuffd_wp(pmd_t pmd) +static __always_inline pmd_t pmd_mkuffd(pmd_t pmd) { return pmd; } -static __always_inline pte_t pte_clear_uffd_wp(pte_t pte) +static __always_inline pte_t pte_clear_uffd(pte_t pte) { return pte; } -static __always_inline pmd_t pmd_clear_uffd_wp(pmd_t pmd) +static __always_inline pmd_t pmd_clear_uffd(pmd_t pmd) { return pmd; } -static __always_inline pte_t pte_swp_mkuffd_wp(pte_t pte) +static __always_inline pte_t pte_swp_mkuffd(pte_t pte) { return pte; } -static __always_inline int pte_swp_uffd_wp(pte_t pte) +static __always_inline int pte_swp_uffd(pte_t pte) { return 0; } -static __always_inline pte_t pte_swp_clear_uffd_wp(pte_t pte) +static __always_inline pte_t pte_swp_clear_uffd(pte_t pte) { return pte; } -static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_mkuffd(pmd_t pmd) { return pmd; } -static inline int pmd_swp_uffd_wp(pmd_t pmd) +static inline int pmd_swp_uffd(pmd_t pmd) { return 0; } -static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_clear_uffd(pmd_t pmd) { return pmd; } diff --git a/include/linux/leafops.h b/include/linux/leafops.h index ef84ac1cce19..4c1476ae3234 100644 --- a/include/linux/leafops.h +++ b/include/linux/leafops.h @@ -100,8 +100,8 @@ static inline softleaf_t softleaf_from_pmd(pmd_t pmd) if (pmd_swp_soft_dirty(pmd)) pmd = pmd_swp_clear_soft_dirty(pmd); - if (pmd_swp_uffd_wp(pmd)) - pmd = pmd_swp_clear_uffd_wp(pmd); + if (pmd_swp_uffd(pmd)) + pmd = pmd_swp_clear_uffd(pmd); arch_entry = __pmd_to_swp_entry(pmd); /* Temporary until swp_entry_t eliminated. */ diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index efcddb9925ad..b5c4dc0f3fe3 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -600,14 +600,14 @@ pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr, return false; /* A uffd-wp wr-protected normal pte */ - if (unlikely(pte_present(pteval) && pte_uffd_wp(pteval))) + if (unlikely(pte_present(pteval) && pte_uffd(pteval))) arm_uffd_pte = true; /* * A uffd-wp wr-protected swap pte. Note: this should even cover an * existing pte marker with uffd-wp bit set. */ - if (unlikely(pte_swp_uffd_wp_any(pteval))) + if (unlikely(pte_swp_uffd_any(pteval))) arm_uffd_pte = true; if (unlikely(arm_uffd_pte)) { diff --git a/include/linux/swapops.h b/include/linux/swapops.h index 1125401b787e..c956bc445ee0 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -73,8 +73,8 @@ static inline pte_t pte_swp_clear_flags(pte_t pte) pte = pte_swp_clear_exclusive(pte); if (pte_swp_soft_dirty(pte)) pte = pte_swp_clear_soft_dirty(pte); - if (pte_swp_uffd_wp(pte)) - pte = pte_swp_clear_uffd_wp(pte); + if (pte_swp_uffd(pte)) + pte = pte_swp_clear_uffd(pte); return pte; } diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 68edac4dcd78..658740df2978 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -211,13 +211,13 @@ static inline bool userfaultfd_minor(struct vm_area_struct *vma) static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, pte_t pte) { - return userfaultfd_wp(vma) && pte_uffd_wp(pte); + return userfaultfd_wp(vma) && pte_uffd(pte); } static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, pmd_t pmd) { - return userfaultfd_wp(vma) && pmd_uffd_wp(pmd); + return userfaultfd_wp(vma) && pmd_uffd(pmd); } static inline bool userfaultfd_armed(struct vm_area_struct *vma) @@ -272,10 +272,10 @@ static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma) } /* - * Returns true if this is a swap pte and was uffd-wp wr-protected in either - * forms (pte marker or a normal swap pte), false otherwise. + * Returns true if this swap pte carries uffd-tracked state in either + * form (pte marker or a normal swap pte), false otherwise. */ -static inline bool pte_swp_uffd_wp_any(pte_t pte) +static inline bool pte_swp_uffd_any(pte_t pte) { if (!uffd_supports_wp_marker()) return false; @@ -283,7 +283,7 @@ static inline bool pte_swp_uffd_wp_any(pte_t pte) if (pte_present(pte)) return false; - if (pte_swp_uffd_wp(pte)) + if (pte_swp_uffd(pte)) return true; if (pte_is_uffd_wp_marker(pte)) @@ -424,7 +424,7 @@ static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma) * Returns true if this is a swap pte and was uffd-wp wr-protected in either * forms (pte marker or a normal swap pte), false otherwise. */ -static inline bool pte_swp_uffd_wp_any(pte_t pte) +static inline bool pte_swp_uffd_any(pte_t pte) { return false; } diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h index 291fae364c62..5a48c5406cce 100644 --- a/include/trace/events/huge_memory.h +++ b/include/trace/events/huge_memory.h @@ -16,7 +16,7 @@ EM( SCAN_EXCEED_SWAP_PTE, "exceed_swap_pte") \ EM( SCAN_EXCEED_SHARED_PTE, "exceed_shared_pte") \ EM( SCAN_PTE_NON_PRESENT, "pte_non_present") \ - EM( SCAN_PTE_UFFD_WP, "pte_uffd_wp") \ + EM( SCAN_PTE_UFFD, "pte_uffd_wp") \ EM( SCAN_PTE_MAPPED_HUGEPAGE, "pte_mapped_hugepage") \ EM( SCAN_LACK_REFERENCED_PAGE, "lack_referenced_page") \ EM( SCAN_PAGE_NULL, "page_null") \ diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 7f3c53c0d3d4..1d9bb7d7e68d 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1822,8 +1822,8 @@ static void copy_huge_non_present_pmd( pmd = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*src_pmd)) pmd = pmd_swp_mksoft_dirty(pmd); - if (pmd_swp_uffd_wp(*src_pmd)) - pmd = pmd_swp_mkuffd_wp(pmd); + if (pmd_swp_uffd(*src_pmd)) + pmd = pmd_swp_mkuffd(pmd); set_pmd_at(src_mm, addr, src_pmd, pmd); } else if (softleaf_is_device_private(entry)) { /* @@ -1836,8 +1836,8 @@ static void copy_huge_non_present_pmd( if (pmd_swp_soft_dirty(*src_pmd)) pmd = pmd_swp_mksoft_dirty(pmd); - if (pmd_swp_uffd_wp(*src_pmd)) - pmd = pmd_swp_mkuffd_wp(pmd); + if (pmd_swp_uffd(*src_pmd)) + pmd = pmd_swp_mkuffd(pmd); set_pmd_at(src_mm, addr, src_pmd, pmd); } @@ -1857,7 +1857,7 @@ static void copy_huge_non_present_pmd( mm_inc_nr_ptes(dst_mm); pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); if (!userfaultfd_wp(dst_vma)) - pmd = pmd_swp_clear_uffd_wp(pmd); + pmd = pmd_swp_clear_uffd(pmd); set_pmd_at(dst_mm, addr, dst_pmd, pmd); } @@ -1953,7 +1953,7 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); pmdp_set_wrprotect(src_mm, addr, src_pmd); if (!userfaultfd_wp(dst_vma)) - pmd = pmd_clear_uffd_wp(pmd); + pmd = pmd_clear_uffd(pmd); pmd = pmd_wrprotect(pmd); set_pmd: pmd = pmd_mkold(pmd); @@ -2494,9 +2494,9 @@ static pmd_t clear_uffd_wp_pmd(pmd_t pmd) if (pmd_none(pmd)) return pmd; if (pmd_present(pmd)) - pmd = pmd_clear_uffd_wp(pmd); + pmd = pmd_clear_uffd(pmd); else - pmd = pmd_swp_clear_uffd_wp(pmd); + pmd = pmd_swp_clear_uffd(pmd); return pmd; } @@ -2577,16 +2577,16 @@ static void change_non_present_huge_pmd(struct mm_struct *mm, } else if (softleaf_is_device_private_write(entry)) { entry = make_readable_device_private_entry(swp_offset(entry)); newpmd = softleaf_to_pmd(entry); - if (pmd_swp_uffd_wp(*pmd)) - newpmd = pmd_swp_mkuffd_wp(newpmd); + if (pmd_swp_uffd(*pmd)) + newpmd = pmd_swp_mkuffd(newpmd); } else { newpmd = *pmd; } if (uffd_wp) - newpmd = pmd_swp_mkuffd_wp(newpmd); + newpmd = pmd_swp_mkuffd(newpmd); else if (uffd_wp_resolve) - newpmd = pmd_swp_clear_uffd_wp(newpmd); + newpmd = pmd_swp_clear_uffd(newpmd); if (!pmd_same(*pmd, newpmd)) set_pmd_at(mm, addr, pmd, newpmd); } @@ -2667,14 +2667,14 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, entry = pmd_modify(oldpmd, newprot); if (uffd_wp) - entry = pmd_mkuffd_wp(entry); + entry = pmd_mkuffd(entry); else if (uffd_wp_resolve) /* * Leave the write bit to be handled by PF interrupt * handler, then things like COW could be properly * handled. */ - entry = pmd_clear_uffd_wp(entry); + entry = pmd_clear_uffd(entry); /* See change_pte_range(). */ if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) && @@ -3017,8 +3017,8 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, entry = pfn_pte(zero_pfn(addr), vma->vm_page_prot); entry = pte_mkspecial(entry); - if (pmd_uffd_wp(old_pmd)) - entry = pte_mkuffd_wp(entry); + if (pmd_uffd(old_pmd)) + entry = pte_mkuffd(entry); VM_BUG_ON(!pte_none(ptep_get(pte))); set_pte_at(mm, addr, pte, entry); pte++; @@ -3104,7 +3104,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, folio = page_folio(page); soft_dirty = pmd_swp_soft_dirty(old_pmd); - uffd_wp = pmd_swp_uffd_wp(old_pmd); + uffd_wp = pmd_swp_uffd(old_pmd); write = softleaf_is_migration_write(entry); if (PageAnon(page)) @@ -3120,7 +3120,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, folio = page_folio(page); soft_dirty = pmd_swp_soft_dirty(old_pmd); - uffd_wp = pmd_swp_uffd_wp(old_pmd); + uffd_wp = pmd_swp_uffd(old_pmd); write = softleaf_is_device_private_write(entry); anon_exclusive = PageAnonExclusive(page); @@ -3177,7 +3177,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, write = pmd_write(old_pmd); young = pmd_young(old_pmd); soft_dirty = pmd_soft_dirty(old_pmd); - uffd_wp = pmd_uffd_wp(old_pmd); + uffd_wp = pmd_uffd(old_pmd); VM_WARN_ON_FOLIO(!folio_ref_count(folio), folio); VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); @@ -3248,7 +3248,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, if (soft_dirty) entry = pte_swp_mksoft_dirty(entry); if (uffd_wp) - entry = pte_swp_mkuffd_wp(entry); + entry = pte_swp_mkuffd(entry); VM_WARN_ON(!pte_none(ptep_get(pte + i))); set_pte_at(mm, addr, pte + i, entry); } @@ -3275,7 +3275,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, if (soft_dirty) entry = pte_swp_mksoft_dirty(entry); if (uffd_wp) - entry = pte_swp_mkuffd_wp(entry); + entry = pte_swp_mkuffd(entry); VM_WARN_ON(!pte_none(ptep_get(pte + i))); set_pte_at(mm, addr, pte + i, entry); } @@ -3293,7 +3293,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, if (soft_dirty) entry = pte_mksoft_dirty(entry); if (uffd_wp) - entry = pte_mkuffd_wp(entry); + entry = pte_mkuffd(entry); for (i = 0; i < HPAGE_PMD_NR; i++) VM_WARN_ON(!pte_none(ptep_get(pte + i))); @@ -4911,7 +4911,7 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, writable = pmd_write(pmdval); softdirty = pmd_soft_dirty(pmdval); - uffd_wp = pmd_uffd_wp(pmdval); + uffd_wp = pmd_uffd(pmdval); } else { softleaf_t old_entry; @@ -4920,7 +4920,7 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, writable = softleaf_is_device_private_write(old_entry); softdirty = pmd_swp_soft_dirty(pmdval); - uffd_wp = pmd_swp_uffd_wp(pmdval); + uffd_wp = pmd_swp_uffd(pmdval); } /* See folio_try_share_anon_rmap_pmd(): invalidate PMD first. */ @@ -4951,7 +4951,7 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, if (softdirty) pmdswp = pmd_swp_mksoft_dirty(pmdswp); if (uffd_wp) - pmdswp = pmd_swp_mkuffd_wp(pmdswp); + pmdswp = pmd_swp_mkuffd(pmdswp); set_pmd_at(mm, address, pvmw->pmd, pmdswp); /* Migration entry installed: cleanup rmap, folio. */ @@ -4983,8 +4983,8 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) pmde = pmd_mksoft_dirty(pmde); if (softleaf_is_migration_write(entry)) pmde = pmd_mkwrite(pmde, vma); - if (pmd_swp_uffd_wp(*pvmw->pmd)) - pmde = pmd_mkuffd_wp(pmde); + if (pmd_swp_uffd(*pvmw->pmd)) + pmde = pmd_mkuffd(pmde); if (!softleaf_is_migration_young(entry)) pmde = pmd_mkold(pmde); /* NOTE: this may contain setting soft-dirty on some archs */ @@ -5004,8 +5004,8 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) if (pmd_swp_soft_dirty(*pvmw->pmd)) pmde = pmd_swp_mksoft_dirty(pmde); - if (pmd_swp_uffd_wp(*pvmw->pmd)) - pmde = pmd_swp_mkuffd_wp(pmde); + if (pmd_swp_uffd(*pvmw->pmd)) + pmde = pmd_swp_mkuffd(pmde); } if (folio_test_anon(folio)) { diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 53c4e2ef8f6d..40894ae263c6 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -4871,8 +4871,8 @@ hugetlb_install_folio(struct vm_area_struct *vma, pte_t *ptep, unsigned long add __folio_mark_uptodate(new_folio); hugetlb_add_new_anon_rmap(new_folio, vma, addr); - if (userfaultfd_wp(vma) && huge_pte_uffd_wp(old)) - newpte = huge_pte_mkuffd_wp(newpte); + if (userfaultfd_wp(vma) && huge_pte_uffd(old)) + newpte = huge_pte_mkuffd(newpte); set_huge_pte_at(vma->vm_mm, addr, ptep, newpte, sz); hugetlb_count_add(pages_per_huge_page(hstate_vma(vma)), vma->vm_mm); folio_set_hugetlb_migratable(new_folio); @@ -4953,7 +4953,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, */ set_huge_pte_at(dst, addr, dst_pte, entry, sz); } else if (unlikely(softleaf_is_migration(softleaf))) { - bool uffd_wp = pte_swp_uffd_wp(entry); + bool uffd = pte_swp_uffd(entry); if (!softleaf_is_migration_read(softleaf) && cow) { /* @@ -4963,12 +4963,12 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, softleaf = make_readable_migration_entry( swp_offset(softleaf)); entry = swp_entry_to_pte(softleaf); - if (userfaultfd_wp(src_vma) && uffd_wp) - entry = pte_swp_mkuffd_wp(entry); + if (userfaultfd_wp(src_vma) && uffd) + entry = pte_swp_mkuffd(entry); set_huge_pte_at(src, addr, src_pte, entry, sz); } if (!userfaultfd_wp(dst_vma)) - entry = pte_swp_clear_uffd_wp(entry); + entry = pte_swp_clear_uffd(entry); set_huge_pte_at(dst, addr, dst_pte, entry, sz); } else if (unlikely(pte_is_marker(entry))) { const pte_marker marker = copy_pte_marker(softleaf, dst_vma); @@ -5045,7 +5045,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, } if (!userfaultfd_wp(dst_vma)) - entry = huge_pte_clear_uffd_wp(entry); + entry = huge_pte_clear_uffd(entry); set_huge_pte_at(dst, addr, dst_pte, entry, sz); hugetlb_count_add(npages, dst); @@ -5093,9 +5093,9 @@ static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr, } else { if (need_clear_uffd_wp) { if (pte_present(pte)) - pte = huge_pte_clear_uffd_wp(pte); + pte = huge_pte_clear_uffd(pte); else - pte = pte_swp_clear_uffd_wp(pte); + pte = pte_swp_clear_uffd(pte); } set_huge_pte_at(mm, new_addr, dst_pte, pte, sz); } @@ -5229,7 +5229,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, * drop the uffd-wp bit in this zap, then replace the * pte with a marker. */ - if (pte_swp_uffd_wp_any(pte) && + if (pte_swp_uffd_any(pte) && !(zap_flags & ZAP_FLAG_DROP_MARKER)) set_huge_pte_at(mm, address, ptep, make_pte_marker(PTE_MARKER_UFFD_WP), @@ -5265,7 +5265,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, if (huge_pte_dirty(pte)) folio_mark_dirty(folio); /* Leave a uffd-wp pte marker if needed */ - if (huge_pte_uffd_wp(pte) && + if (huge_pte_uffd(pte) && !(zap_flags & ZAP_FLAG_DROP_MARKER)) set_huge_pte_at(mm, address, ptep, make_pte_marker(PTE_MARKER_UFFD_WP), @@ -5469,7 +5469,7 @@ static vm_fault_t hugetlb_wp(struct vm_fault *vmf) * can trigger this, because hugetlb_fault() will always resolve * uffd-wp bit first. */ - if (!unshare && huge_pte_uffd_wp(pte)) + if (!unshare && huge_pte_uffd(pte)) return 0; /* Let's take out MAP_SHARED mappings first. */ @@ -5613,8 +5613,8 @@ static vm_fault_t hugetlb_wp(struct vm_fault *vmf) huge_ptep_clear_flush(vma, vmf->address, vmf->pte); hugetlb_remove_rmap(old_folio); hugetlb_add_new_anon_rmap(new_folio, vma, vmf->address); - if (huge_pte_uffd_wp(pte)) - newpte = huge_pte_mkuffd_wp(newpte); + if (huge_pte_uffd(pte)) + newpte = huge_pte_mkuffd(newpte); set_huge_pte_at(mm, vmf->address, vmf->pte, newpte, huge_page_size(h)); folio_set_hugetlb_migratable(new_folio); @@ -5892,7 +5892,7 @@ static vm_fault_t hugetlb_no_page(struct address_space *mapping, * if populated. */ if (unlikely(pte_is_uffd_wp_marker(vmf->orig_pte))) - new_pte = huge_pte_mkuffd_wp(new_pte); + new_pte = huge_pte_mkuffd(new_pte); set_huge_pte_at(mm, vmf->address, vmf->pte, new_pte, huge_page_size(h)); hugetlb_count_add(pages_per_huge_page(h), mm); @@ -6090,7 +6090,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, goto out_ptl; /* Handle userfault-wp first, before trying to lock more pages */ - if (userfaultfd_wp(vma) && huge_pte_uffd_wp(huge_ptep_get(mm, vmf.address, vmf.pte)) && + if (userfaultfd_wp(vma) && huge_pte_uffd(huge_ptep_get(mm, vmf.address, vmf.pte)) && (flags & FAULT_FLAG_WRITE) && !huge_pte_write(vmf.orig_pte)) { if (!userfaultfd_wp_async(vma)) { spin_unlock(vmf.ptl); @@ -6099,7 +6099,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, return handle_userfault(&vmf, VM_UFFD_WP); } - vmf.orig_pte = huge_pte_clear_uffd_wp(vmf.orig_pte); + vmf.orig_pte = huge_pte_clear_uffd(vmf.orig_pte); set_huge_pte_at(mm, vmf.address, vmf.pte, vmf.orig_pte, huge_page_size(hstate_vma(vma))); /* Fallthrough to CoW */ @@ -6384,7 +6384,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, _dst_pte = pte_mkyoung(_dst_pte); if (wp_enabled) - _dst_pte = huge_pte_mkuffd_wp(_dst_pte); + _dst_pte = huge_pte_mkuffd(_dst_pte); set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte, size); @@ -6508,9 +6508,9 @@ long hugetlb_change_protection(struct vm_area_struct *vma, } if (uffd_wp) - newpte = pte_swp_mkuffd_wp(newpte); + newpte = pte_swp_mkuffd(newpte); else if (uffd_wp_resolve) - newpte = pte_swp_clear_uffd_wp(newpte); + newpte = pte_swp_clear_uffd(newpte); if (!pte_same(pte, newpte)) set_huge_pte_at(mm, address, ptep, newpte, psize); } else if (unlikely(pte_is_marker(pte))) { @@ -6531,9 +6531,9 @@ long hugetlb_change_protection(struct vm_area_struct *vma, pte = huge_pte_modify(old_pte, newprot); pte = arch_make_huge_pte(pte, shift, vma->vm_flags); if (uffd_wp) - pte = huge_pte_mkuffd_wp(pte); + pte = huge_pte_mkuffd(pte); else if (uffd_wp_resolve) - pte = huge_pte_clear_uffd_wp(pte); + pte = huge_pte_clear_uffd(pte); huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte); pages++; tlb_remove_huge_tlb_entry(h, &tlb, ptep, address); diff --git a/mm/internal.h b/mm/internal.h index c289d167ffac..3d30650a8440 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -486,8 +486,8 @@ static inline pte_t pte_move_swp_offset(pte_t pte, long delta) new = pte_swp_mksoft_dirty(new); if (pte_swp_exclusive(pte)) new = pte_swp_mkexclusive(new); - if (pte_swp_uffd_wp(pte)) - new = pte_swp_mkuffd_wp(new); + if (pte_swp_uffd(pte)) + new = pte_swp_mkuffd(new); return new; } diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 58e14d1543ec..0b5fb0d0a0c1 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -38,7 +38,7 @@ enum scan_result { SCAN_EXCEED_SWAP_PTE, SCAN_EXCEED_SHARED_PTE, SCAN_PTE_NON_PRESENT, - SCAN_PTE_UFFD_WP, + SCAN_PTE_UFFD, SCAN_PTE_MAPPED_HUGEPAGE, SCAN_LACK_REFERENCED_PAGE, SCAN_PAGE_NULL, @@ -696,8 +696,8 @@ static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma, result = SCAN_PTE_NON_PRESENT; goto out; } - if (pte_uffd_wp(pteval)) { - result = SCAN_PTE_UFFD_WP; + if (pte_uffd(pteval)) { + result = SCAN_PTE_UFFD; goto out; } page = vm_normal_page(vma, addr, pteval); @@ -1544,7 +1544,7 @@ static enum scan_result mthp_collapse(struct mm_struct *mm, case SCAN_PAGE_NULL: case SCAN_DEL_PAGE_LRU: case SCAN_PTE_NON_PRESENT: - case SCAN_PTE_UFFD_WP: + case SCAN_PTE_UFFD: case SCAN_PAGE_LAZYFREE: last_result = ret; goto next_order; @@ -1665,15 +1665,15 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, /* * Always be strict with uffd-wp * enabled swap entries. Please see - * comment below for pte_uffd_wp(). + * comment below for pte_uffd(). */ - if (pte_swp_uffd_wp_any(pteval)) { - result = SCAN_PTE_UFFD_WP; + if (pte_swp_uffd_any(pteval)) { + result = SCAN_PTE_UFFD; goto out_unmap; } continue; } - if (pte_uffd_wp(pteval)) { + if (pte_uffd(pteval)) { /* * Don't collapse the page if any of the small * PTEs are armed with uffd write protection. @@ -1683,7 +1683,7 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, * userfault messages that falls outside of * the registered range. So, just be simple. */ - result = SCAN_PTE_UFFD_WP; + result = SCAN_PTE_UFFD; goto out_unmap; } @@ -1895,7 +1895,7 @@ static enum scan_result try_collapse_pte_mapped_thp(struct mm_struct *mm, unsign /* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */ if (userfaultfd_wp(vma)) - return SCAN_PTE_UFFD_WP; + return SCAN_PTE_UFFD; folio = filemap_lock_folio(vma->vm_file->f_mapping, linear_page_index(vma, haddr)); @@ -3242,7 +3242,7 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start, /* Whitelisted set of results where continuing OK */ case SCAN_NO_PTE_TABLE: case SCAN_PTE_NON_PRESENT: - case SCAN_PTE_UFFD_WP: + case SCAN_PTE_UFFD: case SCAN_LACK_REFERENCED_PAGE: case SCAN_PAGE_NULL: case SCAN_PAGE_COUNT: diff --git a/mm/memory.c b/mm/memory.c index 631c170f812a..f4fb52e9e731 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -893,8 +893,8 @@ static void restore_exclusive_pte(struct vm_area_struct *vma, if (pte_swp_soft_dirty(orig_pte)) pte = pte_mksoft_dirty(pte); - if (pte_swp_uffd_wp(orig_pte)) - pte = pte_mkuffd_wp(pte); + if (pte_swp_uffd(orig_pte)) + pte = pte_mkuffd(pte); if ((vma->vm_flags & VM_WRITE) && can_change_pte_writable(vma, address, pte)) { @@ -977,8 +977,8 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, pte = softleaf_to_pte(entry); if (pte_swp_soft_dirty(orig_pte)) pte = pte_swp_mksoft_dirty(pte); - if (pte_swp_uffd_wp(orig_pte)) - pte = pte_swp_mkuffd_wp(pte); + if (pte_swp_uffd(orig_pte)) + pte = pte_swp_mkuffd(pte); set_pte_at(src_mm, addr, src_pte, pte); } } else if (softleaf_is_device_private(entry)) { @@ -1011,8 +1011,8 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, entry = make_readable_device_private_entry( swp_offset(entry)); pte = swp_entry_to_pte(entry); - if (pte_swp_uffd_wp(orig_pte)) - pte = pte_swp_mkuffd_wp(pte); + if (pte_swp_uffd(orig_pte)) + pte = pte_swp_mkuffd(pte); set_pte_at(src_mm, addr, src_pte, pte); } } else if (softleaf_is_device_exclusive(entry)) { @@ -1035,7 +1035,7 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, return 0; } if (!userfaultfd_wp(dst_vma)) - pte = pte_swp_clear_uffd_wp(pte); + pte = pte_swp_clear_uffd(pte); set_pte_at(dst_mm, addr, dst_pte, pte); return 0; } @@ -1083,7 +1083,7 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma); if (userfaultfd_pte_wp(dst_vma, ptep_get(src_pte))) /* Uffd-wp needs to be delivered to dest pte as well */ - pte = pte_mkuffd_wp(pte); + pte = pte_mkuffd(pte); set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); return 0; } @@ -1106,7 +1106,7 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma, pte = pte_mkold(pte); if (!userfaultfd_wp(dst_vma)) - pte = pte_clear_uffd_wp(pte); + pte = pte_clear_uffd(pte); set_ptes(dst_vma->vm_mm, addr, dst_pte, pte, nr); } @@ -3918,8 +3918,8 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) if (unlikely(unshare)) { if (pte_soft_dirty(vmf->orig_pte)) entry = pte_mksoft_dirty(entry); - if (pte_uffd_wp(vmf->orig_pte)) - entry = pte_mkuffd_wp(entry); + if (pte_uffd(vmf->orig_pte)) + entry = pte_mkuffd(entry); } else { entry = maybe_mkwrite(pte_mkdirty(entry), vma); } @@ -4258,7 +4258,7 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) * etc.) because we're only removing the uffd-wp bit, * which is completely invisible to the user. */ - pte = pte_clear_uffd_wp(ptep_get(vmf->pte)); + pte = pte_clear_uffd(ptep_get(vmf->pte)); set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); /* @@ -5023,8 +5023,8 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) pte = mk_pte(page, vma->vm_page_prot); if (pte_swp_soft_dirty(vmf->orig_pte)) pte = pte_mksoft_dirty(pte); - if (pte_swp_uffd_wp(vmf->orig_pte)) - pte = pte_mkuffd_wp(pte); + if (pte_swp_uffd(vmf->orig_pte)) + pte = pte_mkuffd(pte); /* * Similar logic as in do_wp_page(); however, optimize for pages that @@ -5239,7 +5239,7 @@ void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte, if (vma->vm_flags & VM_WRITE) entry = pte_mkwrite(pte_mkdirty(entry), vma); if (uffd_wp) - entry = pte_mkuffd_wp(entry); + entry = pte_mkuffd(entry); folio_ref_add(folio, nr_pages - 1); folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE); @@ -5306,7 +5306,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) return handle_userfault(vmf, VM_UFFD_MISSING); } if (vmf_orig_pte_uffd_wp(vmf)) - entry = pte_mkuffd_wp(entry); + entry = pte_mkuffd(entry); set_pte_at(vma->vm_mm, addr, vmf->pte, entry); /* No need to invalidate - it was non-present before */ @@ -5556,7 +5556,7 @@ void set_pte_range(struct vm_fault *vmf, struct folio *folio, else if (pte_write(entry) && folio_test_dirty(folio)) entry = pte_mkdirty(entry); if (unlikely(vmf_orig_pte_uffd_wp(vmf))) - entry = pte_mkuffd_wp(entry); + entry = pte_mkuffd(entry); /* copy-on-write page */ if (write && !(vma->vm_flags & VM_SHARED)) { VM_BUG_ON_FOLIO(nr != 1, folio); diff --git a/mm/migrate.c b/mm/migrate.c index d3bd5cc8f552..0ac0d8f51b0c 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -327,8 +327,8 @@ static bool try_to_map_unused_to_zeropage(struct page_vma_mapped_walk *pvmw, if (pte_swp_soft_dirty(old_pte)) newpte = pte_mksoft_dirty(newpte); - if (pte_swp_uffd_wp(old_pte)) - newpte = pte_mkuffd_wp(newpte); + if (pte_swp_uffd(old_pte)) + newpte = pte_mkuffd(newpte); set_pte_at(pvmw->vma->vm_mm, pvmw->address, pvmw->pte, newpte); @@ -396,8 +396,8 @@ static bool remove_migration_pte(struct folio *folio, if (softleaf_is_migration_write(entry)) pte = pte_mkwrite(pte, vma); - else if (pte_swp_uffd_wp(old_pte)) - pte = pte_mkuffd_wp(pte); + else if (pte_swp_uffd(old_pte)) + pte = pte_mkuffd(pte); if (folio_test_anon(folio) && !softleaf_is_migration_read(entry)) rmap_flags |= RMAP_EXCLUSIVE; @@ -412,8 +412,8 @@ static bool remove_migration_pte(struct folio *folio, pte = softleaf_to_pte(entry); if (pte_swp_soft_dirty(old_pte)) pte = pte_swp_mksoft_dirty(pte); - if (pte_swp_uffd_wp(old_pte)) - pte = pte_swp_mkuffd_wp(pte); + if (pte_swp_uffd(old_pte)) + pte = pte_swp_mkuffd(pte); } #ifdef CONFIG_HUGETLB_PAGE diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 2fffeb1f9969..18d097c38853 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -456,13 +456,13 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp, if (pte_present(pte)) { if (pte_soft_dirty(pte)) swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_uffd_wp(pte)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); + if (pte_uffd(pte)) + swp_pte = pte_swp_mkuffd(swp_pte); } else { if (pte_swp_soft_dirty(pte)) swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_swp_uffd_wp(pte)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); + if (pte_swp_uffd(pte)) + swp_pte = pte_swp_mkuffd(swp_pte); } set_pte_at(mm, addr, ptep, swp_pte); diff --git a/mm/mprotect.c b/mm/mprotect.c index cec2a1eed539..a971d86c06ca 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -240,8 +240,8 @@ static long change_softleaf_pte(struct vm_area_struct *vma, */ entry = make_readable_device_private_entry(swp_offset(entry)); newpte = swp_entry_to_pte(entry); - if (pte_swp_uffd_wp(oldpte)) - newpte = pte_swp_mkuffd_wp(newpte); + if (pte_swp_uffd(oldpte)) + newpte = pte_swp_mkuffd(newpte); } else if (softleaf_is_marker(entry)) { /* * Ignore error swap entries unconditionally, @@ -266,9 +266,9 @@ static long change_softleaf_pte(struct vm_area_struct *vma, } if (uffd_wp) - newpte = pte_swp_mkuffd_wp(newpte); + newpte = pte_swp_mkuffd(newpte); else if (uffd_wp_resolve) - newpte = pte_swp_clear_uffd_wp(newpte); + newpte = pte_swp_clear_uffd(newpte); if (!pte_same(oldpte, newpte)) { set_pte_at(vma->vm_mm, addr, pte, newpte); @@ -290,9 +290,9 @@ static __always_inline void change_present_ptes(struct mmu_gather *tlb, ptent = pte_modify(oldpte, newprot); if (uffd_wp) - ptent = pte_mkuffd_wp(ptent); + ptent = pte_mkuffd(ptent); else if (uffd_wp_resolve) - ptent = pte_clear_uffd_wp(ptent); + ptent = pte_clear_uffd(ptent); /* * In some writable, shared mappings, we might want diff --git a/mm/mremap.c b/mm/mremap.c index e9c8b1d05832..12732a5c547e 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -297,9 +297,9 @@ static int move_ptes(struct pagetable_move_control *pmc, else { if (need_clear_uffd_wp) { if (pte_present(pte)) - pte = pte_clear_uffd_wp(pte); + pte = pte_clear_uffd(pte); else - pte = pte_swp_clear_uffd_wp(pte); + pte = pte_swp_clear_uffd(pte); } set_ptes(mm, new_addr, new_ptep, pte, nr_ptes); } diff --git a/mm/page_table_check.c b/mm/page_table_check.c index 53a8997ec043..3fb995e5d40d 100644 --- a/mm/page_table_check.c +++ b/mm/page_table_check.c @@ -188,8 +188,8 @@ static inline bool softleaf_cached_writable(softleaf_t entry) static void page_table_check_pte_flags(pte_t pte) { if (pte_present(pte)) { - WARN_ON_ONCE(pte_uffd_wp(pte) && pte_write(pte)); - } else if (pte_swp_uffd_wp(pte)) { + WARN_ON_ONCE(pte_uffd(pte) && pte_write(pte)); + } else if (pte_swp_uffd(pte)) { const softleaf_t entry = softleaf_from_pte(pte); WARN_ON_ONCE(softleaf_cached_writable(entry)); @@ -216,9 +216,9 @@ EXPORT_SYMBOL(__page_table_check_ptes_set); static inline void page_table_check_pmd_flags(pmd_t pmd) { if (pmd_present(pmd)) { - if (pmd_uffd_wp(pmd)) + if (pmd_uffd(pmd)) WARN_ON_ONCE(pmd_write(pmd)); - } else if (pmd_swp_uffd_wp(pmd)) { + } else if (pmd_swp_uffd(pmd)) { const softleaf_t entry = softleaf_from_pmd(pmd); WARN_ON_ONCE(softleaf_cached_writable(entry)); diff --git a/mm/rmap.c b/mm/rmap.c index 9939400e77c7..50f973b63f1c 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2317,13 +2317,13 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, if (likely(pte_present(pteval))) { if (pte_soft_dirty(pteval)) swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_uffd_wp(pteval)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); + if (pte_uffd(pteval)) + swp_pte = pte_swp_mkuffd(swp_pte); } else { if (pte_swp_soft_dirty(pteval)) swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_swp_uffd_wp(pteval)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); + if (pte_swp_uffd(pteval)) + swp_pte = pte_swp_mkuffd(swp_pte); } set_pte_at(mm, address, pvmw.pte, swp_pte); } else { @@ -2694,14 +2694,14 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, swp_pte = swp_entry_to_pte(entry); if (pte_soft_dirty(pteval)) swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_uffd_wp(pteval)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); + if (pte_uffd(pteval)) + swp_pte = pte_swp_mkuffd(swp_pte); } else { swp_pte = swp_entry_to_pte(entry); if (pte_swp_soft_dirty(pteval)) swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_swp_uffd_wp(pteval)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); + if (pte_swp_uffd(pteval)) + swp_pte = pte_swp_mkuffd(swp_pte); } if (folio_test_hugetlb(folio)) set_huge_pte_at(mm, address, pvmw.pte, swp_pte, diff --git a/mm/swapfile.c b/mm/swapfile.c index a602e5820513..baa3462ce517 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -2496,8 +2496,8 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, new_pte = pte_mkold(mk_pte(page, vma->vm_page_prot)); if (pte_swp_soft_dirty(old_pte)) new_pte = pte_mksoft_dirty(new_pte); - if (pte_swp_uffd_wp(old_pte)) - new_pte = pte_mkuffd_wp(new_pte); + if (pte_swp_uffd(old_pte)) + new_pte = pte_mkuffd(new_pte); setpte: set_pte_at(vma->vm_mm, addr, pte, new_pte); folio_put_swap(swapcache, folio_file_page(swapcache, swp_offset(entry))); diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index c3adedaaf7d5..4147f247c441 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -376,7 +376,7 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, if (writable) _dst_pte = pte_mkwrite(_dst_pte, dst_vma); if (flags & MFILL_ATOMIC_WP) - _dst_pte = pte_mkuffd_wp(_dst_pte); + _dst_pte = pte_mkuffd(_dst_pte); ret = -EAGAIN; dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); @@ -3601,7 +3601,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING) vm_flags |= VM_UFFD_MISSING; if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) { - if (!pgtable_supports_uffd_wp()) + if (!pgtable_supports_uffd()) goto out; vm_flags |= VM_UFFD_WP; @@ -4311,7 +4311,7 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx, uffdio_api.features &= ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM); #endif - if (!pgtable_supports_uffd_wp()) + if (!pgtable_supports_uffd()) uffdio_api.features &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP; if (!uffd_supports_wp_marker()) { From 2730ee209f95102edb89a87267d656bf0c2df46a Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Wed, 8 Jul 2026 12:14:05 +0100 Subject: [PATCH 259/501] userfaultfd: test uffd VMA flags through the vma_flags_t API The uffd VMA-flag helpers read vma->vm_flags directly. Now that config-gated per-mode masks exist, switch them to the vma_flags_t accessor vma_test_any_mask(), which is the going-forward API and keeps a single place (the VMA_UFFD_* masks) that knows which modes are available on the current build. No functional change: vma_flags_t is in union with vm_flags, so the same bits are read, and the masks fold to the same code the open-coded vm_flags tests produced -- verified identical on gcc and clang, 32- and 64-bit. Link: https://lore.kernel.org/20260708111417.173443-5-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau Suggested-by: Lorenzo Stoakes Assisted-by: Claude:claude-opus-4-8 Acked-by: Mike Rapoport (Microsoft) Reviewed-by: Lorenzo Stoakes Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: James Houghton Cc: Jonathan Corbet Cc: Liam Howlett Cc: Paolo Bonzini Cc: Peter Xu Cc: Sean Christopherson Cc: SeongJae Park Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/userfaultfd_k.h | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 658740df2978..c4f2cc6dfcf0 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -178,7 +178,8 @@ static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, */ static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma) { - return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); + return vma_test_any_mask(vma, + mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_MINOR)); } /* @@ -190,22 +191,23 @@ static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma) */ static inline bool uffd_disable_fault_around(struct vm_area_struct *vma) { - return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); + return vma_test_any_mask(vma, + mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_MINOR)); } static inline bool userfaultfd_missing(struct vm_area_struct *vma) { - return vma->vm_flags & VM_UFFD_MISSING; + return vma_test_any_mask(vma, VMA_UFFD_MISSING); } static inline bool userfaultfd_wp(struct vm_area_struct *vma) { - return vma->vm_flags & VM_UFFD_WP; + return vma_test_any_mask(vma, VMA_UFFD_WP); } static inline bool userfaultfd_minor(struct vm_area_struct *vma) { - return vma->vm_flags & VM_UFFD_MINOR; + return vma_test_any_mask(vma, VMA_UFFD_MINOR); } static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, @@ -222,7 +224,7 @@ static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, static inline bool userfaultfd_armed(struct vm_area_struct *vma) { - return vma->vm_flags & __VM_UFFD_FLAGS; + return vma_test_any_mask(vma, __VMA_UFFD_FLAGS); } static inline bool vma_has_uffd_without_event_remap(struct vm_area_struct *vma) From 134c1d031a2e9d2f6a3f56fc358df8df7ffeef2b Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Wed, 8 Jul 2026 12:14:06 +0100 Subject: [PATCH 260/501] mm: add VM_UFFD_RWP VMA flag Preparatory patch for userfaultfd read-write protection (RWP). RWP extends userfaultfd protection from plain write-protection (WP) to full read-write protection: accesses to an RWP-protected range -- reads as well as writes -- trap through userfaultfd. Reserve VM_UFFD_RWP, add the userfaultfd_rwp() and userfaultfd_protected() helpers, and wire up the smaps "ur" entry and the trace-flag table the rest of the series will use. The flag is gated on CONFIG_USERFAULTFD_RWP, which is introduced together with the UAPI in a later patch; until then VM_UFFD_RWP aliases VM_NONE and every downstream check folds to dead code. Nothing sets or queries the flag yet. Link: https://lore.kernel.org/20260708111417.173443-6-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau Assisted-by: Claude:claude-opus-4-6 Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: SeongJae Park Reviewed-by: Lorenzo Stoakes Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: James Houghton Cc: Jonathan Corbet Cc: Liam Howlett Cc: Paolo Bonzini Cc: Peter Xu Cc: Sean Christopherson Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- Documentation/filesystems/proc.rst | 1 + fs/proc/task_mmu.c | 3 ++ include/linux/mm.h | 41 +++++++++++++++++---------- include/linux/userfaultfd_k.h | 45 ++++++++++++++++++++++++------ include/trace/events/mmflags.h | 7 +++++ 5 files changed, 74 insertions(+), 23 deletions(-) diff --git a/Documentation/filesystems/proc.rst b/Documentation/filesystems/proc.rst index 2ccd5b2dfdd3..27189f6e004e 100644 --- a/Documentation/filesystems/proc.rst +++ b/Documentation/filesystems/proc.rst @@ -608,6 +608,7 @@ encoded manner. The codes are the following: um userfaultfd missing tracking uw userfaultfd wr-protect tracking ui userfaultfd minor fault + ur userfaultfd read-write-protect tracking ss shadow/guarded control stack page sl sealed lf lock on fault pages diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 75cfd9befb4d..407d8dde2a90 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1237,6 +1237,9 @@ static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR [ilog2(VM_UFFD_MINOR)] = "ui", #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */ +#ifdef CONFIG_USERFAULTFD_RWP + [ilog2(VM_UFFD_RWP)] = "ur", +#endif #ifdef CONFIG_ARCH_HAS_USER_SHADOW_STACK [ilog2(VM_SHADOW_STACK)] = "ss", #endif diff --git a/include/linux/mm.h b/include/linux/mm.h index 3dca9497ae03..61bcd65cd029 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -354,6 +354,7 @@ enum { #endif DECLARE_VMA_BIT(UFFD_MINOR, 41), DECLARE_VMA_BIT(SEALED, 42), + DECLARE_VMA_BIT(UFFD_RWP, 43), /* Flags that reuse flags above. */ DECLARE_VMA_BIT_ALIAS(PKEY_BIT0, HIGH_ARCH_0), DECLARE_VMA_BIT_ALIAS(PKEY_BIT1, HIGH_ARCH_1), @@ -497,12 +498,17 @@ enum { #else #define VM_UFFD_MINOR VM_NONE #endif +#ifdef CONFIG_USERFAULTFD_RWP +#define VM_UFFD_RWP INIT_VM_FLAG(UFFD_RWP) +#else +#define VM_UFFD_RWP VM_NONE +#endif /* - * vma_flags_t masks for the userfaultfd VMA flags. VMA_UFFD_MINOR is gated on - * the same config as VM_UFFD_MINOR -- which implies 64BIT, where the bit fits - * -- so an out-of-range bit is never fed to mk_vma_flags() on a build whose - * bitmap cannot hold it. + * vma_flags_t masks for the userfaultfd VMA flags. The two high-bit modes are + * gated on the same configs as their VM_* flags above -- both of which imply + * 64BIT -- so an out-of-range bit is never fed to mk_vma_flags() on a build + * whose bitmap cannot hold it. */ #define VMA_UFFD_MISSING mk_vma_flags(VMA_UFFD_MISSING_BIT) #define VMA_UFFD_WP mk_vma_flags(VMA_UFFD_WP_BIT) @@ -511,6 +517,11 @@ enum { #else #define VMA_UFFD_MINOR EMPTY_VMA_FLAGS #endif +#ifdef CONFIG_USERFAULTFD_RWP +#define VMA_UFFD_RWP mk_vma_flags(VMA_UFFD_RWP_BIT) +#else +#define VMA_UFFD_RWP EMPTY_VMA_FLAGS +#endif #ifdef CONFIG_64BIT #define VM_ALLOW_ANY_UNCACHED INIT_VM_FLAG(ALLOW_ANY_UNCACHED) @@ -649,22 +660,24 @@ enum { * reconsistuted upon page fault, so necessitate page table copying upon fork. * * Note that these flags should be compared with the DESTINATION VMA not the - * source, as VM_UFFD_WP may not be propagated to destination, while all other - * flags will be. + * source: VM_UFFD_WP and VM_UFFD_RWP may be cleared on the destination + * (dup_userfaultfd() -> userfaultfd_reset_ctx() when the parent context did + * not negotiate UFFD_FEATURE_EVENT_FORK), while all other flags propagate. * * VM_PFNMAP / VM_MIXEDMAP - These contain kernel-mapped data which cannot be * reasonably reconstructed on page fault. * * VM_UFFD_WP - Encodes metadata about an installed uffd - * write protect handler, which cannot be - * reconstructed on page fault. + * VM_UFFD_RWP write- or read-write-protect handler, which + * cannot be reconstructed on page fault. * - * We always copy pgtables when dst_vma has uffd-wp - * enabled even if it's file-backed - * (e.g. shmem). Because when uffd-wp is enabled, - * pgtable contains uffd-wp protection information, - * that's something we can't retrieve from page cache, - * and skip copying will lose those info. + * We always copy pgtables when dst_vma has the + * uffd PTE bit in use even if it's file-backed + * (e.g. shmem). Because when the uffd bit is + * in use, the pgtable contains the protection + * information, that's something we can't + * retrieve from page cache, and skip copying + * will lose those info. * * VM_MAYBE_GUARD - Could contain page guard region markers which * by design are a property of the page tables diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index c4f2cc6dfcf0..6dad14b9abb9 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -21,10 +21,11 @@ #include /* The set of all possible UFFD-related VM flags. */ -#define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR) +#define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_MINOR | \ + VM_UFFD_WP | VM_UFFD_RWP) #define __VMA_UFFD_FLAGS mk_vma_flags_from_masks(VMA_UFFD_MISSING, VMA_UFFD_WP, \ - VMA_UFFD_MINOR) + VMA_UFFD_MINOR, VMA_UFFD_RWP) /* * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining @@ -168,7 +169,8 @@ static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, /* * Never enable huge pmd sharing on some uffd registered vmas: * - * - VM_UFFD_WP VMAs, because write protect information is per pgtable entry. + * - VM_UFFD_WP and VM_UFFD_RWP VMAs, because the write protect / access + * tracking information is per pgtable entry. * * - VM_UFFD_MINOR VMAs, because otherwise we would never get minor faults for * VMAs which share huge pmds. (If you have two mappings to the same @@ -179,20 +181,25 @@ static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma) { return vma_test_any_mask(vma, - mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_MINOR)); + mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_RWP, + VMA_UFFD_MINOR)); } /* - * Don't do fault around for either WP or MINOR registered uffd range. For + * Don't do fault around for WP, RWP or MINOR registered uffd range. For * MINOR registered range, fault around will be a total disaster and ptes can * be installed without notifications; for WP it should mostly be fine as long * as the fault around checks for pte_none() before the installation, however - * to be super safe we just forbid it. + * to be super safe we just forbid it; for RWP, pre-faulted neighbours would + * be indistinguishable from accessed pages in PAGEMAP_SCAN (PAGE_IS_ACCESSED) + * and pollute the tracked working set, so each page must be populated by its + * own fault. */ static inline bool uffd_disable_fault_around(struct vm_area_struct *vma) { return vma_test_any_mask(vma, - mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_MINOR)); + mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_RWP, + VMA_UFFD_MINOR)); } static inline bool userfaultfd_missing(struct vm_area_struct *vma) @@ -210,6 +217,16 @@ static inline bool userfaultfd_minor(struct vm_area_struct *vma) return vma_test_any_mask(vma, VMA_UFFD_MINOR); } +static inline bool userfaultfd_rwp(struct vm_area_struct *vma) +{ + return vma_test_single_mask(vma, VMA_UFFD_RWP); +} + +static inline bool userfaultfd_protected(struct vm_area_struct *vma) +{ + return userfaultfd_wp(vma) || userfaultfd_rwp(vma); +} + static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, pte_t pte) { @@ -330,6 +347,16 @@ static inline bool userfaultfd_minor(struct vm_area_struct *vma) return false; } +static inline bool userfaultfd_rwp(struct vm_area_struct *vma) +{ + return false; +} + +static inline bool userfaultfd_protected(struct vm_area_struct *vma) +{ + return false; +} + static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, pte_t pte) { @@ -423,8 +450,8 @@ static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma) } /* - * Returns true if this is a swap pte and was uffd-wp wr-protected in either - * forms (pte marker or a normal swap pte), false otherwise. + * Returns true if this swap pte carries uffd-tracked state in either + * form (pte marker or a normal swap pte), false otherwise. */ static inline bool pte_swp_uffd_any(pte_t pte) { diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h index c1a05ff0feab..935893e5ea53 100644 --- a/include/trace/events/mmflags.h +++ b/include/trace/events/mmflags.h @@ -186,6 +186,12 @@ IF_HAVE_PG_ARCH_3(arch_3) # define IF_HAVE_UFFD_MINOR(flag, name) #endif +#ifdef CONFIG_USERFAULTFD_RWP +# define IF_HAVE_UFFD_RWP(flag, name) {flag, name}, +#else +# define IF_HAVE_UFFD_RWP(flag, name) +#endif + #if defined(CONFIG_64BIT) || defined(CONFIG_PPC32) # define IF_HAVE_VM_DROPPABLE(flag, name) {flag, name}, #else @@ -207,6 +213,7 @@ IF_HAVE_UFFD_MINOR(VM_UFFD_MINOR, "uffd_minor" ) \ {VM_PFNMAP, "pfnmap" }, \ {VM_MAYBE_GUARD, "maybe_guard" }, \ {VM_UFFD_WP, "uffd_wp" }, \ +IF_HAVE_UFFD_RWP(VM_UFFD_RWP, "uffd_rwp" ) \ {VM_LOCKED, "locked" }, \ {VM_IO, "io" }, \ {VM_SEQ_READ, "seqread" }, \ From 9cf3c554acbdd3661e2ae5043de842a551c5d0ed Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Wed, 8 Jul 2026 12:14:07 +0100 Subject: [PATCH 261/501] mm: add MM_CP_UFFD_RWP change_protection() flag Preparatory patch. Add the change_protection() primitive that userfaultfd RWP will use. An RWP-protected PTE is PAGE_NONE with the uffd PTE bit set. The PROT_NONE half makes the CPU fault on any access; the uffd bit distinguishes an RWP fault from a plain mprotect(PROT_NONE) or NUMA hinting fault. MM_CP_UFFD_WP and MM_CP_UFFD_RWP share the same PTE bit, so the two cannot be used together on the same range. Two new change_protection() flags: MM_CP_UFFD_RWP install PAGE_NONE and set the uffd bit MM_CP_UFFD_RWP_RESOLVE restore vma->vm_page_prot, clear the uffd bit Both are wired through change_pte_range(), change_huge_pmd(), and hugetlb_change_protection() so anon, shmem, THP, and hugetlb all share the same semantics. Link: https://lore.kernel.org/20260708111417.173443-7-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau Assisted-by: Claude:claude-opus-4-6 Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: SeongJae Park Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: James Houghton Cc: Jonathan Corbet Cc: Liam Howlett Cc: Lorenzo Stoakes Cc: Paolo Bonzini Cc: Peter Xu Cc: Sean Christopherson Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/mm.h | 5 ++++ include/linux/userfaultfd_k.h | 1 - mm/huge_memory.c | 36 ++++++++++++++------------ mm/hugetlb.c | 25 +++++++++++++----- mm/mprotect.c | 48 +++++++++++++++++++++++++++-------- 5 files changed, 81 insertions(+), 34 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 61bcd65cd029..170f21f28c77 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3340,6 +3340,11 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen); #define MM_CP_UFFD_WP_RESOLVE (1UL << 3) /* Resolve wp */ #define MM_CP_UFFD_WP_ALL (MM_CP_UFFD_WP | \ MM_CP_UFFD_WP_RESOLVE) +/* Whether this change is for uffd RWP */ +#define MM_CP_UFFD_RWP (1UL << 4) /* do rwp */ +#define MM_CP_UFFD_RWP_RESOLVE (1UL << 5) /* resolve rwp */ +#define MM_CP_UFFD_RWP_ALL (MM_CP_UFFD_RWP | \ + MM_CP_UFFD_RWP_RESOLVE) bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, pte_t pte); diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 6dad14b9abb9..0a0763c6e9f4 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -369,7 +369,6 @@ static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, return false; } - static inline bool userfaultfd_armed(struct vm_area_struct *vma) { return false; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 1d9bb7d7e68d..7b5be6b8a64b 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2553,8 +2553,8 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, } static void change_non_present_huge_pmd(struct mm_struct *mm, - unsigned long addr, pmd_t *pmd, bool uffd_wp, - bool uffd_wp_resolve) + unsigned long addr, pmd_t *pmd, bool uffd_prot, + bool uffd_prot_resolve) { softleaf_t entry = softleaf_from_pmd(*pmd); pmd_t newpmd; @@ -2583,9 +2583,9 @@ static void change_non_present_huge_pmd(struct mm_struct *mm, newpmd = *pmd; } - if (uffd_wp) + if (uffd_prot) newpmd = pmd_swp_mkuffd(newpmd); - else if (uffd_wp_resolve) + else if (uffd_prot_resolve) newpmd = pmd_swp_clear_uffd(newpmd); if (!pmd_same(*pmd, newpmd)) set_pmd_at(mm, addr, pmd, newpmd); @@ -2606,8 +2606,9 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, spinlock_t *ptl; pmd_t oldpmd, entry; bool prot_numa = cp_flags & MM_CP_PROT_NUMA; - bool uffd_wp = cp_flags & MM_CP_UFFD_WP; - bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; + bool uffd_prot = cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP); + bool uffd_prot_resolve = cp_flags & + (MM_CP_UFFD_WP_RESOLVE | MM_CP_UFFD_RWP_RESOLVE); int ret = 1; tlb_change_page_size(tlb, HPAGE_PMD_SIZE); @@ -2620,11 +2621,17 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, return 0; if (thp_migration_supported() && pmd_is_valid_softleaf(*pmd)) { - change_non_present_huge_pmd(mm, addr, pmd, uffd_wp, - uffd_wp_resolve); + change_non_present_huge_pmd(mm, addr, pmd, uffd_prot, + uffd_prot_resolve); goto unlock; } + /* Already in the desired state */ + if (prot_numa && pmd_protnone(*pmd)) + goto unlock; + if ((cp_flags & MM_CP_UFFD_RWP) && pmd_protnone(*pmd) && pmd_uffd(*pmd)) + goto unlock; + if (prot_numa) { /* @@ -2635,9 +2642,6 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, if (is_huge_zero_pmd(*pmd)) goto unlock; - if (pmd_protnone(*pmd)) - goto unlock; - if (!folio_can_map_prot_numa(pmd_folio(*pmd), vma, vma_is_single_threaded_private(vma))) goto unlock; @@ -2666,9 +2670,9 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, oldpmd = pmdp_invalidate_ad(vma, addr, pmd); entry = pmd_modify(oldpmd, newprot); - if (uffd_wp) + if (uffd_prot) entry = pmd_mkuffd(entry); - else if (uffd_wp_resolve) + else if (uffd_prot_resolve) /* * Leave the write bit to be handled by PF interrupt * handler, then things like COW could be properly @@ -2714,10 +2718,10 @@ int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, return 1; /* - * Huge entries on userfault-wp only works with anonymous, while we - * don't have anonymous PUDs yet. + * Huge entries on userfault-wp or userfault-rwp only work with + * anonymous, while we don't have anonymous PUDs yet. */ - if (WARN_ON_ONCE(cp_flags & MM_CP_UFFD_WP_ALL)) + if (WARN_ON_ONCE(cp_flags & (MM_CP_UFFD_WP_ALL | MM_CP_UFFD_RWP_ALL))) return 1; ptl = __pud_trans_huge_lock(pudp, vma); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 40894ae263c6..1b5963a125fe 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -6427,6 +6427,8 @@ long hugetlb_change_protection(struct vm_area_struct *vma, unsigned long last_addr_mask; bool uffd_wp = cp_flags & MM_CP_UFFD_WP; bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; + bool uffd_rwp = cp_flags & MM_CP_UFFD_RWP; + bool uffd_rwp_resolve = cp_flags & MM_CP_UFFD_RWP_RESOLVE; struct mmu_gather tlb; /* @@ -6452,6 +6454,11 @@ long hugetlb_change_protection(struct vm_area_struct *vma, ptep = hugetlb_walk(vma, address, psize); if (!ptep) { + /* + * uffd_wp installs a pte marker on the unpopulated + * entry; uffd_rwp does not install markers so the + * allocation is unnecessary for it. + */ if (!uffd_wp) { address |= last_addr_mask; continue; @@ -6473,7 +6480,8 @@ long hugetlb_change_protection(struct vm_area_struct *vma, * shouldn't happen at all. Warn about it if it * happened due to some reason. */ - WARN_ON_ONCE(uffd_wp || uffd_wp_resolve); + WARN_ON_ONCE(uffd_wp || uffd_wp_resolve || + uffd_rwp || uffd_rwp_resolve); pages++; spin_unlock(ptl); address |= last_addr_mask; @@ -6507,9 +6515,9 @@ long hugetlb_change_protection(struct vm_area_struct *vma, pages++; } - if (uffd_wp) + if (uffd_wp || uffd_rwp) newpte = pte_swp_mkuffd(newpte); - else if (uffd_wp_resolve) + else if (uffd_wp_resolve || uffd_rwp_resolve) newpte = pte_swp_clear_uffd(newpte); if (!pte_same(pte, newpte)) set_huge_pte_at(mm, address, ptep, newpte, psize); @@ -6520,19 +6528,24 @@ long hugetlb_change_protection(struct vm_area_struct *vma, * pte_marker_uffd_wp()==true implies !poison * because they're mutual exclusive. */ - if (pte_is_uffd_wp_marker(pte) && uffd_wp_resolve) + if (pte_is_uffd_wp_marker(pte) && + (uffd_wp_resolve || uffd_rwp_resolve)) /* Safe to modify directly (non-present->none). */ huge_pte_clear(mm, address, ptep, psize); } else { pte_t old_pte; unsigned int shift = huge_page_shift(hstate_vma(vma)); + /* Already protnone with uffd bit set? Nothing to do. */ + if (uffd_rwp && pte_protnone(pte) && huge_pte_uffd(pte)) + goto next; + old_pte = huge_ptep_modify_prot_start(vma, address, ptep); pte = huge_pte_modify(old_pte, newprot); pte = arch_make_huge_pte(pte, shift, vma->vm_flags); - if (uffd_wp) + if (uffd_wp || uffd_rwp) pte = huge_pte_mkuffd(pte); - else if (uffd_wp_resolve) + else if (uffd_wp_resolve || uffd_rwp_resolve) pte = huge_pte_clear_uffd(pte); huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte); pages++; diff --git a/mm/mprotect.c b/mm/mprotect.c index a971d86c06ca..ea290f04d90f 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -214,8 +214,9 @@ static __always_inline void set_write_prot_commit_flush_ptes(struct vm_area_stru static long change_softleaf_pte(struct vm_area_struct *vma, unsigned long addr, pte_t *pte, pte_t oldpte, unsigned long cp_flags) { - const bool uffd_wp = cp_flags & MM_CP_UFFD_WP; - const bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; + const bool uffd_prot = cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP); + const bool uffd_prot_resolve = cp_flags & + (MM_CP_UFFD_WP_RESOLVE | MM_CP_UFFD_RWP_RESOLVE); softleaf_t entry = softleaf_from_pte(oldpte); pte_t newpte; @@ -256,7 +257,7 @@ static long change_softleaf_pte(struct vm_area_struct *vma, * to unprotect it, drop it; the next page * fault will trigger without uffd trapping. */ - if (uffd_wp_resolve) { + if (uffd_prot_resolve) { pte_clear(vma->vm_mm, addr, pte); return 1; } @@ -265,9 +266,9 @@ static long change_softleaf_pte(struct vm_area_struct *vma, newpte = oldpte; } - if (uffd_wp) + if (uffd_prot) newpte = pte_swp_mkuffd(newpte); - else if (uffd_wp_resolve) + else if (uffd_prot_resolve) newpte = pte_swp_clear_uffd(newpte); if (!pte_same(oldpte, newpte)) { @@ -282,16 +283,17 @@ static __always_inline void change_present_ptes(struct mmu_gather *tlb, int nr_ptes, unsigned long end, pgprot_t newprot, struct folio *folio, struct page *page, unsigned long cp_flags) { - const bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; - const bool uffd_wp = cp_flags & MM_CP_UFFD_WP; + const bool uffd_prot = cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP); + const bool uffd_prot_resolve = cp_flags & + (MM_CP_UFFD_WP_RESOLVE | MM_CP_UFFD_RWP_RESOLVE); pte_t ptent, oldpte; oldpte = modify_prot_start_ptes(vma, addr, ptep, nr_ptes); ptent = pte_modify(oldpte, newprot); - if (uffd_wp) + if (uffd_prot) ptent = pte_mkuffd(ptent); - else if (uffd_wp_resolve) + else if (uffd_prot_resolve) ptent = pte_clear_uffd(ptent); /* @@ -325,6 +327,7 @@ static long change_pte_range(struct mmu_gather *tlb, long pages = 0; bool is_private_single_threaded; bool prot_numa = cp_flags & MM_CP_PROT_NUMA; + bool uffd_rwp = cp_flags & MM_CP_UFFD_RWP; bool uffd_wp = cp_flags & MM_CP_UFFD_WP; int nr_ptes; @@ -350,6 +353,14 @@ static long change_pte_range(struct mmu_gather *tlb, /* Already in the desired state. */ if (prot_numa && pte_protnone(oldpte)) continue; + /* + * RWP-protected PTEs carry _PAGE_UFFD as a marker on + * top of PROT_NONE. Skip only entries already in that + * exact state; plain PROT_NONE from mprotect() still needs + * to be promoted so future faults can be distinguished. + */ + if (uffd_rwp && pte_protnone(oldpte) && pte_uffd(oldpte)) + continue; page = vm_normal_page(vma, addr, oldpte); if (page) @@ -358,6 +369,8 @@ static long change_pte_range(struct mmu_gather *tlb, /* * Avoid trapping faults against the zero or KSM * pages. See similar comment in change_huge_pmd. + * Skip this filter for uffd RWP which + * must set protnone regardless of NUMA placement. */ if (prot_numa && !folio_can_map_prot_numa(folio, vma, @@ -428,7 +441,7 @@ pgtable_split_needed(struct vm_area_struct *vma, unsigned long cp_flags) * (e.g. 2M shmem) because file thp is handled differently when * split by erasing the pmd so far. */ - return (cp_flags & MM_CP_UFFD_WP) && !vma_is_anonymous(vma); + return (cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP)) && !vma_is_anonymous(vma); } /* @@ -667,7 +680,16 @@ long change_protection(struct mmu_gather *tlb, pgprot_t newprot = vma->vm_page_prot; long pages; - BUG_ON((cp_flags & MM_CP_UFFD_WP_ALL) == MM_CP_UFFD_WP_ALL); + /* + * MM_CP_UFFD_{WP,RWP} and _RESOLVE are mutually exclusive within one + * change, and WP and RWP cannot mix. Miswired callers get a warn and + * a no-op; userspace cannot reach this state. + */ + if (WARN_ON_ONCE((cp_flags & MM_CP_UFFD_WP_ALL) == MM_CP_UFFD_WP_ALL || + (cp_flags & MM_CP_UFFD_RWP_ALL) == MM_CP_UFFD_RWP_ALL || + ((cp_flags & MM_CP_UFFD_WP_ALL) && + (cp_flags & MM_CP_UFFD_RWP_ALL)))) + return 0; #ifdef CONFIG_NUMA_BALANCING /* @@ -681,6 +703,10 @@ long change_protection(struct mmu_gather *tlb, WARN_ON_ONCE(cp_flags & MM_CP_PROT_NUMA); #endif + if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE) && + (cp_flags & MM_CP_UFFD_RWP)) + newprot = PAGE_NONE; + if (is_vm_hugetlb_page(vma)) pages = hugetlb_change_protection(vma, start, end, newprot, cp_flags); From 763f43865fc4136562ae84542604081f7989a951 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Wed, 8 Jul 2026 12:14:08 +0100 Subject: [PATCH 262/501] mm: preserve RWP marker across PTE rewrites The uffd PTE bit must survive any kernel path that rewrites a PTE on a VM_UFFD_RWP VMA, otherwise the marker that carries PAGE_NONE semantics is silently dropped and the next access leaks past RWP tracking. Wire the preservation through every path that rewrites a VM_UFFD_RWP PTE. Swap and device-exclusive: do_swap_page(), restore_exclusive_pte(), and unuse_pte() (swapoff()) re-apply PAGE_NONE when the swap PTE carries the uffd bit and the VMA has VM_UFFD_RWP. Migration: remove_migration_pte() and remove_migration_pmd() do the same after the migration entry is replaced with a real PTE/PMD. Fork: __copy_present_ptes(), copy_present_page(), copy_nonpresent_pte(), copy_huge_pmd(), copy_huge_non_present_pmd(), and copy_hugetlb_page_range() keep the uffd bit on the child when the destination VMA has VM_UFFD_RWP, matching the existing VM_UFFD_WP handling. Add VM_UFFD_RWP to VM_COPY_ON_FORK so the flag itself propagates. mprotect(): change_pte_range() and change_huge_pmd() restore PAGE_NONE after pte_modify()/pmd_modify() have recomputed the base protection from a (possibly user-changed) vm_page_prot. pte_modify() preserves _PAGE_UFFD, so the bit stays; we just have to force PAGE_NONE back on top. Link: https://lore.kernel.org/20260708111417.173443-8-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau Assisted-by: Claude:claude-opus-4-6 Acked-by: Mike Rapoport (Microsoft) Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: James Houghton Cc: Jonathan Corbet Cc: Liam Howlett Cc: Lorenzo Stoakes Cc: Paolo Bonzini Cc: Peter Xu Cc: Sean Christopherson Cc: SeongJae Park Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/mm.h | 3 ++- mm/huge_memory.c | 50 ++++++++++++++++++++++++++++++++--- mm/hugetlb.c | 51 +++++++++++++++++++++++++++++------ mm/memory.c | 66 +++++++++++++++++++++++++++++++++++++++------- mm/migrate.c | 8 ++++++ mm/mprotect.c | 10 +++++++ mm/mremap.c | 13 +++++++-- mm/swapfile.c | 5 ++++ mm/userfaultfd.c | 17 ++++++++++++ 9 files changed, 199 insertions(+), 24 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 170f21f28c77..be4457269a38 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -684,7 +684,8 @@ enum { * only and thus cannot be reconstructed on page * fault. */ -#define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_MAYBE_GUARD) +#define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_UFFD_RWP | \ + VM_MAYBE_GUARD) /* * mapping from the currently active vm_flags protection bits (the diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 7b5be6b8a64b..991f216f4f5b 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1856,7 +1856,7 @@ static void copy_huge_non_present_pmd( add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); mm_inc_nr_ptes(dst_mm); pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); - if (!userfaultfd_wp(dst_vma)) + if (!userfaultfd_protected(dst_vma)) pmd = pmd_swp_clear_uffd(pmd); set_pmd_at(dst_mm, addr, dst_pmd, pmd); } @@ -1951,9 +1951,15 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, out_zero_page: mm_inc_nr_ptes(dst_mm); pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); - pmdp_set_wrprotect(src_mm, addr, src_pmd); - if (!userfaultfd_wp(dst_vma)) + + /* See __copy_present_ptes(): restore accessible protection. */ + if (!userfaultfd_protected(dst_vma)) { + if (userfaultfd_rwp(src_vma) && pmd_uffd(pmd)) + pmd = pmd_modify(pmd, dst_vma->vm_page_prot); pmd = pmd_clear_uffd(pmd); + } + + pmdp_set_wrprotect(src_mm, addr, src_pmd); pmd = pmd_wrprotect(pmd); set_pmd: pmd = pmd_mkold(pmd); @@ -2539,8 +2545,19 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, pgtable_trans_huge_deposit(mm, new_pmd, pgtable); } pmd = move_soft_dirty_pmd(pmd); - if (vma_has_uffd_without_event_remap(vma)) + if (vma_has_uffd_without_event_remap(vma)) { + /* + * See __copy_present_ptes(): normalise the RWP marker + * so the destination starts accessible instead of + * taking a numa-hinting fault on first access. Only the + * marker (protnone + uffd) needs it; leave other present + * PMDs in the VMA untouched. + */ + if (pmd_present(pmd) && userfaultfd_rwp(vma) && + pmd_uffd(pmd)) + pmd = pmd_modify(pmd, vma->vm_page_prot); pmd = clear_uffd_wp_pmd(pmd); + } set_pmd_at(mm, new_addr, new_pmd, pmd); if (force_flush) flush_pmd_tlb_range(vma, old_addr, old_addr + PMD_SIZE); @@ -2680,6 +2697,10 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, */ entry = pmd_clear_uffd(entry); + /* See change_pte_range(): preserve RWP protection across mprotect() */ + if (userfaultfd_rwp(vma) && pmd_uffd(entry)) + entry = pmd_modify(entry, PAGE_NONE); + /* See change_pte_range(). */ if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) && can_change_pmd_writable(vma, addr, entry)) @@ -2847,6 +2868,13 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm _dst_pmd = move_soft_dirty_pmd(src_pmdval); _dst_pmd = clear_uffd_wp_pmd(_dst_pmd); } + + /* Re-arm RWP on the moved PMD if dst_vma is RWP-registered. */ + if (userfaultfd_rwp(dst_vma)) { + _dst_pmd = pmd_modify(_dst_pmd, PAGE_NONE); + _dst_pmd = pmd_mkuffd(_dst_pmd); + } + set_pmd_at(mm, dst_addr, dst_pmd, _dst_pmd); src_pgtable = pgtable_trans_huge_withdraw(mm, src_pmd); @@ -3023,6 +3051,11 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, entry = pte_mkspecial(entry); if (pmd_uffd(old_pmd)) entry = pte_mkuffd(entry); + + /* Restore PAGE_NONE so an RWP marker keeps trapping */ + if (userfaultfd_rwp(vma) && pmd_uffd(old_pmd)) + entry = pte_modify(entry, PAGE_NONE); + VM_BUG_ON(!pte_none(ptep_get(pte))); set_pte_at(mm, addr, pte, entry); pte++; @@ -3299,6 +3332,10 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, if (uffd_wp) entry = pte_mkuffd(entry); + /* Restore PAGE_NONE so an RWP marker keeps trapping */ + if (userfaultfd_rwp(vma) && uffd_wp) + entry = pte_modify(entry, PAGE_NONE); + for (i = 0; i < HPAGE_PMD_NR; i++) VM_WARN_ON(!pte_none(ptep_get(pte + i))); @@ -4989,6 +5026,11 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) pmde = pmd_mkwrite(pmde, vma); if (pmd_swp_uffd(*pvmw->pmd)) pmde = pmd_mkuffd(pmde); + + /* See do_swap_page(): restore PAGE_NONE for RWP */ + if (pmd_swp_uffd(*pvmw->pmd) && userfaultfd_rwp(vma)) + pmde = pmd_modify(pmde, PAGE_NONE); + if (!softleaf_is_migration_young(entry)) pmde = pmd_mkold(pmde); /* NOTE: this may contain setting soft-dirty on some archs */ diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 1b5963a125fe..bb6219002a5c 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -4871,8 +4871,16 @@ hugetlb_install_folio(struct vm_area_struct *vma, pte_t *ptep, unsigned long add __folio_mark_uptodate(new_folio); hugetlb_add_new_anon_rmap(new_folio, vma, addr); - if (userfaultfd_wp(vma) && huge_pte_uffd(old)) + if (userfaultfd_protected(vma) && huge_pte_uffd(old)) { newpte = huge_pte_mkuffd(newpte); + /* Restore PAGE_NONE so the RWP marker keeps trapping. */ + if (userfaultfd_rwp(vma)) { + unsigned int shift = huge_page_shift(hstate_vma(vma)); + + newpte = huge_pte_modify(newpte, PAGE_NONE); + newpte = arch_make_huge_pte(newpte, shift, vma->vm_flags); + } + } set_huge_pte_at(vma->vm_mm, addr, ptep, newpte, sz); hugetlb_count_add(pages_per_huge_page(hstate_vma(vma)), vma->vm_mm); folio_set_hugetlb_migratable(new_folio); @@ -4963,11 +4971,11 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, softleaf = make_readable_migration_entry( swp_offset(softleaf)); entry = swp_entry_to_pte(softleaf); - if (userfaultfd_wp(src_vma) && uffd) + if (userfaultfd_protected(src_vma) && uffd) entry = pte_swp_mkuffd(entry); set_huge_pte_at(src, addr, src_pte, entry, sz); } - if (!userfaultfd_wp(dst_vma)) + if (!userfaultfd_protected(dst_vma)) entry = pte_swp_clear_uffd(entry); set_huge_pte_at(dst, addr, dst_pte, entry, sz); } else if (unlikely(pte_is_marker(entry))) { @@ -5032,6 +5040,16 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, goto next; } + /* See __copy_present_ptes(): restore accessible protection. */ + if (!userfaultfd_protected(dst_vma)) { + if (userfaultfd_rwp(src_vma) && huge_pte_uffd(entry)) { + entry = huge_pte_modify(entry, dst_vma->vm_page_prot); + entry = arch_make_huge_pte(entry, huge_page_shift(h), + dst_vma->vm_flags); + } + entry = huge_pte_clear_uffd(entry); + } + if (cow) { /* * No need to notify as we are downgrading page @@ -5044,9 +5062,6 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, entry = huge_pte_wrprotect(entry); } - if (!userfaultfd_wp(dst_vma)) - entry = huge_pte_clear_uffd(entry); - set_huge_pte_at(dst, addr, dst_pte, entry, sz); hugetlb_count_add(npages, dst); } @@ -5092,10 +5107,23 @@ static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr, huge_pte_clear(mm, new_addr, dst_pte, sz); } else { if (need_clear_uffd_wp) { - if (pte_present(pte)) + if (pte_present(pte)) { + /* + * See __copy_present_ptes(): normalise the RWP + * marker so the destination starts accessible + * instead of taking a numa-hinting fault on + * first access. Only the marker (protnone + uffd) + * needs it; leave other present PTEs untouched. + */ + if (userfaultfd_rwp(vma) && huge_pte_uffd(pte)) { + pte = huge_pte_modify(pte, vma->vm_page_prot); + pte = arch_make_huge_pte(pte, huge_page_shift(h), + vma->vm_flags); + } pte = huge_pte_clear_uffd(pte); - else + } else { pte = pte_swp_clear_uffd(pte); + } } set_huge_pte_at(mm, new_addr, dst_pte, pte, sz); } @@ -6547,6 +6575,13 @@ long hugetlb_change_protection(struct vm_area_struct *vma, pte = huge_pte_mkuffd(pte); else if (uffd_wp_resolve || uffd_rwp_resolve) pte = huge_pte_clear_uffd(pte); + + /* Preserve RWP protection across mprotect() */ + if (userfaultfd_rwp(vma) && huge_pte_uffd(pte)) { + pte = huge_pte_modify(pte, PAGE_NONE); + pte = arch_make_huge_pte(pte, shift, vma->vm_flags); + } + huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte); pages++; tlb_remove_huge_tlb_entry(h, &tlb, ptep, address); diff --git a/mm/memory.c b/mm/memory.c index f4fb52e9e731..182d1516c0c1 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -896,6 +896,10 @@ static void restore_exclusive_pte(struct vm_area_struct *vma, if (pte_swp_uffd(orig_pte)) pte = pte_mkuffd(pte); + /* See do_swap_page(): restore PAGE_NONE for RWP */ + if (pte_swp_uffd(orig_pte) && userfaultfd_rwp(vma)) + pte = pte_modify(pte, PAGE_NONE); + if ((vma->vm_flags & VM_WRITE) && can_change_pte_writable(vma, address, pte)) { if (folio_test_dirty(folio)) @@ -1034,7 +1038,7 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, make_pte_marker(marker)); return 0; } - if (!userfaultfd_wp(dst_vma)) + if (!userfaultfd_protected(dst_vma)) pte = pte_swp_clear_uffd(pte); set_pte_at(dst_mm, addr, dst_pte, pte); return 0; @@ -1081,9 +1085,13 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma /* All done, just insert the new page copy in the child */ pte = folio_mk_pte(new_folio, dst_vma->vm_page_prot); pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma); - if (userfaultfd_pte_wp(dst_vma, ptep_get(src_pte))) - /* Uffd-wp needs to be delivered to dest pte as well */ + if (userfaultfd_protected(dst_vma) && pte_uffd(ptep_get(src_pte))) { + /* The uffd bit needs to be delivered to the dest pte as well */ pte = pte_mkuffd(pte); + /* Restore PAGE_NONE so the RWP marker keeps trapping */ + if (userfaultfd_rwp(dst_vma)) + pte = pte_modify(pte, PAGE_NONE); + } set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); return 0; } @@ -1093,9 +1101,31 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma, pte_t pte, unsigned long addr, int nr) { struct mm_struct *src_mm = src_vma->vm_mm; + bool writable; + + /* + * Snapshot writability before the RWP-disarm rewrite below: when the + * child is not RWP-armed, pte_modify(pte, dst_vma->vm_page_prot) can + * silently drop _PAGE_RW from a resolved (no-marker) writable PTE, + * so a later pte_write(pte) check would skip the COW wrprotect and + * leave the parent writable over a folio shared with the child. + */ + writable = pte_write(pte); + + /* + * Child is not RWP-armed: restore accessible protection so the + * inherited PAGE_NONE does not cost a fault on first read. Gate on + * pte_uffd(pte) so unrelated PAGE_NONE markers (e.g. NUMA balancing) + * are not normalised away. + */ + if (!userfaultfd_protected(dst_vma)) { + if (userfaultfd_rwp(src_vma) && pte_uffd(pte)) + pte = pte_modify(pte, dst_vma->vm_page_prot); + pte = pte_clear_uffd(pte); + } /* If it's a COW mapping, write protect it both processes. */ - if (is_cow_mapping(src_vma->vm_flags) && pte_write(pte)) { + if (is_cow_mapping(src_vma->vm_flags) && writable) { wrprotect_ptes(src_mm, addr, src_pte, nr); pte = pte_wrprotect(pte); } @@ -1105,9 +1135,6 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma, pte = pte_mkclean(pte); pte = pte_mkold(pte); - if (!userfaultfd_wp(dst_vma)) - pte = pte_clear_uffd(pte); - set_ptes(dst_vma->vm_mm, addr, dst_pte, pte, nr); } @@ -4747,6 +4774,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) struct swap_info_struct *si = NULL; rmap_t rmap_flags = RMAP_NONE; bool exclusive = false; + bool rwp_restore = false; softleaf_t entry; pte_t pte; vm_fault_t ret = 0; @@ -5026,12 +5054,27 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) if (pte_swp_uffd(vmf->orig_pte)) pte = pte_mkuffd(pte); + /* + * A page reclaimed while RWP-protected carries the uffd bit on + * its swap entry. Re-apply PAGE_NONE on swap-in so the first access + * still traps as an RWP fault. pte_modify() preserves _PAGE_UFFD. + */ + if (pte_swp_uffd(vmf->orig_pte) && userfaultfd_rwp(vma)) { + pte = pte_modify(pte, PAGE_NONE); + rwp_restore = true; + } + /* * Similar logic as in do_wp_page(); however, optimize for pages that * are certainly exclusive. + * + * Skip the write upgrade for an RWP-restored pte: it must stay + * PROT_NONE so the access retries through the RWP fault path + * (do_uffd_rwp()) rather than being made writable here. */ if (exclusive) { - if ((vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) && + if (!rwp_restore && + (vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) && !pte_needs_soft_dirty_wp(vma, pte)) { pte = pte_mkwrite(pte, vma); if (vmf->flags & FAULT_FLAG_WRITE) @@ -5092,7 +5135,12 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) folio_put(swapcache); } - if ((vmf->flags & FAULT_FLAG_WRITE) && !pte_write(pte)) { + /* + * For an RWP-restored pte, leave it PROT_NONE and let the write + * retry through the RWP fault path; do not COW it here, which would + * drop the marker for a non-exclusive page. + */ + if ((vmf->flags & FAULT_FLAG_WRITE) && !pte_write(pte) && !rwp_restore) { ret |= do_wp_page(vmf); if (ret & VM_FAULT_ERROR) ret &= VM_FAULT_ERROR; diff --git a/mm/migrate.c b/mm/migrate.c index 0ac0d8f51b0c..60355be9f652 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -330,6 +330,10 @@ static bool try_to_map_unused_to_zeropage(struct page_vma_mapped_walk *pvmw, if (pte_swp_uffd(old_pte)) newpte = pte_mkuffd(newpte); + /* See remove_migration_pte(): restore PAGE_NONE for RWP */ + if (pte_swp_uffd(old_pte) && userfaultfd_rwp(pvmw->vma)) + newpte = pte_modify(newpte, PAGE_NONE); + set_pte_at(pvmw->vma->vm_mm, pvmw->address, pvmw->pte, newpte); dec_mm_counter(pvmw->vma->vm_mm, mm_counter(folio)); @@ -399,6 +403,10 @@ static bool remove_migration_pte(struct folio *folio, else if (pte_swp_uffd(old_pte)) pte = pte_mkuffd(pte); + /* See do_swap_page(): restore PAGE_NONE for RWP */ + if (pte_swp_uffd(old_pte) && userfaultfd_rwp(vma)) + pte = pte_modify(pte, PAGE_NONE); + if (folio_test_anon(folio) && !softleaf_is_migration_read(entry)) rmap_flags |= RMAP_EXCLUSIVE; diff --git a/mm/mprotect.c b/mm/mprotect.c index ea290f04d90f..dc27bbc1712f 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -296,6 +296,16 @@ static __always_inline void change_present_ptes(struct mmu_gather *tlb, else if (uffd_prot_resolve) ptent = pte_clear_uffd(ptent); + /* + * The uffd bit on a VM_UFFD_RWP VMA carries PROT_NONE + * semantics. If mprotect() or NUMA hinting changed the + * base protection, restore PAGE_NONE so the PTE still + * traps on any access. pte_modify() preserves + * _PAGE_UFFD. + */ + if (userfaultfd_rwp(vma) && pte_uffd(ptent)) + ptent = pte_modify(ptent, PAGE_NONE); + /* * In some writable, shared mappings, we might want * to catch actual write access -- see diff --git a/mm/mremap.c b/mm/mremap.c index 12732a5c547e..8a46ec5831c8 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -296,10 +296,19 @@ static int move_ptes(struct pagetable_move_control *pmc, pte_clear(mm, new_addr, new_ptep); else { if (need_clear_uffd_wp) { - if (pte_present(pte)) + if (pte_present(pte)) { + /* + * See __copy_present_ptes(): normalise + * RWP PTEs so the destination starts + * accessible instead of taking a + * numa-hinting fault on first access. + */ + if (userfaultfd_rwp(vma) && pte_uffd(pte)) + pte = pte_modify(pte, vma->vm_page_prot); pte = pte_clear_uffd(pte); - else + } else { pte = pte_swp_clear_uffd(pte); + } } set_ptes(mm, new_addr, new_ptep, pte, nr_ptes); } diff --git a/mm/swapfile.c b/mm/swapfile.c index baa3462ce517..775c385490a5 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -2498,6 +2498,11 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, new_pte = pte_mksoft_dirty(new_pte); if (pte_swp_uffd(old_pte)) new_pte = pte_mkuffd(new_pte); + + /* See do_swap_page(): restore PAGE_NONE for RWP */ + if (pte_swp_uffd(old_pte) && userfaultfd_rwp(vma)) + new_pte = pte_modify(new_pte, PAGE_NONE); + setpte: set_pte_at(vma->vm_mm, addr, pte, new_pte); folio_put_swap(swapcache, folio_file_page(swapcache, swp_offset(entry))); diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 4147f247c441..c3c2fef8297c 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -1292,6 +1292,13 @@ static long move_present_ptes(struct mm_struct *mm, if (pte_dirty(orig_src_pte)) orig_dst_pte = pte_mkdirty(orig_dst_pte); orig_dst_pte = pte_mkwrite(orig_dst_pte, dst_vma); + + /* Re-arm RWP on the moved PTE if dst_vma is RWP-registered. */ + if (userfaultfd_rwp(dst_vma)) { + orig_dst_pte = pte_modify(orig_dst_pte, PAGE_NONE); + orig_dst_pte = pte_mkuffd(orig_dst_pte); + } + set_pte_at(mm, dst_addr, dst_pte, orig_dst_pte); src_addr += PAGE_SIZE; @@ -1373,6 +1380,9 @@ static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, orig_src_pte = ptep_get_and_clear(mm, src_addr, src_pte); if (pgtable_supports_soft_dirty()) orig_src_pte = pte_swp_mksoft_dirty(orig_src_pte); + /* Re-arm RWP on the moved swap entry if dst_vma is RWP-registered. */ + if (userfaultfd_rwp(dst_vma)) + orig_src_pte = pte_swp_mkuffd(orig_src_pte); set_pte_at(mm, dst_addr, dst_pte, orig_src_pte); double_pt_unlock(dst_ptl, src_ptl); @@ -1399,6 +1409,13 @@ static int move_zeropage_pte(struct mm_struct *mm, zero_pte = pte_mkspecial(pfn_pte(zero_pfn(dst_addr), dst_vma->vm_page_prot)); + + /* Re-arm RWP on the moved PTE if dst_vma is RWP-registered. */ + if (userfaultfd_rwp(dst_vma)) { + zero_pte = pte_modify(zero_pte, PAGE_NONE); + zero_pte = pte_mkuffd(zero_pte); + } + ptep_clear_flush(src_vma, src_addr, src_pte); set_pte_at(mm, dst_addr, dst_pte, zero_pte); double_pt_unlock(dst_ptl, src_ptl); From 7974c238531bfaa743878d8be8f062524a6f29ac Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Wed, 8 Jul 2026 12:14:09 +0100 Subject: [PATCH 263/501] mm: handle VM_UFFD_RWP in khugepaged, rmap, and GUP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three mm paths outside the fault handler gate on the uffd PTE bit today: khugepaged (skip collapse on ranges carrying markers), rmap (cap unmap batching), and GUP (force a fault through gup_can_follow_protnone). Extend each to treat VM_UFFD_RWP the same as VM_UFFD_WP; otherwise per-PTE RWP state is silently destroyed or bypassed. khugepaged: try_collapse_pte_mapped_thp() and file_backed_vma_is_retractable() already refuse to collapse or retract page tables on ranges carrying the uffd PTE bit. Broaden the VMA predicate from userfaultfd_wp() to userfaultfd_protected() so VM_UFFD_RWP ranges get the same protection. hpage_collapse_scan_pmd() needs no change — its existing pte_uffd() check already catches an RWP PTE because it carries the uffd bit. rmap: folio_unmap_pte_batch() caps batching at 1 for VM_UFFD_RWP so the restore path handles each PTE with its own marker. GUP: gup_can_follow_protnone() forces a fault on VM_UFFD_RWP VMAs regardless of FOLL_HONOR_NUMA_FAULT. RWP uses protnone as an access-tracking marker, not for NUMA hinting, so any GUP — read or write — must go through the userfaultfd fault path. Link: https://lore.kernel.org/20260708111417.173443-9-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau Assisted-by: Claude:claude-opus-4-6 Acked-by: Mike Rapoport (Microsoft) Reviewed-by: Lorenzo Stoakes Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: James Houghton Cc: Jonathan Corbet Cc: Liam Howlett Cc: Paolo Bonzini Cc: Peter Xu Cc: Sean Christopherson Cc: SeongJae Park Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/mm.h | 16 +++++++++++++++- mm/khugepaged.c | 18 +++++++++++------- mm/rmap.c | 2 +- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index be4457269a38..25e669632d2c 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4654,11 +4654,25 @@ static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags) /* * Indicates whether GUP can follow a PROT_NONE mapped page, or whether - * a (NUMA hinting) fault is required. + * a (NUMA hinting or userfaultfd RWP) fault is required. */ static inline bool gup_can_follow_protnone(const struct vm_area_struct *vma, unsigned int flags) { + /* + * VM_UFFD_RWP uses protnone as an access-tracking marker, not for + * NUMA hinting. GUP must always take a fault so the access is + * delivered to userfaultfd, regardless of FOLL_HONOR_NUMA_FAULT. + * + * Only do so while the VMA is accessible. If it has been made + * inaccessible (e.g. mprotect(PROT_NONE)), fall through to the guard + * below: forcing a fault there would loop, as handle_mm_fault() makes + * no progress on protnone in an inaccessible VMA, and the access is + * denied regardless of RWP anyway. + */ + if (vma_test_single_mask(vma, VMA_UFFD_RWP) && vma_is_accessible(vma)) + return false; + /* * If callers don't want to honor NUMA hinting faults, no need to * determine if we would actually have to trigger a NUMA hinting fault. diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 0b5fb0d0a0c1..55157567dc4c 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1893,8 +1893,11 @@ static enum scan_result try_collapse_pte_mapped_thp(struct mm_struct *mm, unsign if (!thp_vma_allowable_order(vma, vma->vm_flags, TVA_FORCED_COLLAPSE, PMD_ORDER)) return SCAN_VMA_CHECK; - /* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */ - if (userfaultfd_wp(vma)) + /* + * Keep pmd pgtable while the uffd bit is in use; see comment in + * retract_page_tables(). + */ + if (userfaultfd_protected(vma)) return SCAN_PTE_UFFD; folio = filemap_lock_folio(vma->vm_file->f_mapping, @@ -2107,13 +2110,14 @@ static bool file_backed_vma_is_retractable(struct vm_area_struct *vma) return false; /* - * When a vma is registered with uffd-wp, we cannot recycle + * When a vma is registered with uffd-wp or RWP, we cannot recycle * the page table because there may be pte markers installed. - * Other vmas can still have the same file mapped hugely, but - * skip this one: it will always be mapped in small page size - * for uffd-wp registered ranges. + * VM_UFFD_RWP ranges similarly rely on per-PTE uffd state + * and cannot be recycled to a shared PMD. Other vmas can still + * have the same file mapped hugely, but skip this one: it will + * always be mapped in small page size for these registrations. */ - if (userfaultfd_wp(vma)) + if (userfaultfd_protected(vma)) return false; /* diff --git a/mm/rmap.c b/mm/rmap.c index 50f973b63f1c..26166a6b8cb9 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1965,7 +1965,7 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio, if (pte_unused(pte)) return 1; - if (userfaultfd_wp(vma)) + if (userfaultfd_protected(vma)) return 1; /* From 6eab8f2cc6465ab093b5171ece815ec24e678d4e Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Wed, 8 Jul 2026 12:14:10 +0100 Subject: [PATCH 264/501] userfaultfd: add UFFDIO_REGISTER_MODE_RWP and UFFDIO_RWPROTECT plumbing Add the userspace interface for read-write protection tracking: - UFFDIO_REGISTER_MODE_RWP register a range for RWP tracking - UFFD_FEATURE_RWP capability bit - UFFDIO_RWPROTECT install / remove RWP on a range Introduce CONFIG_USERFAULTFD_RWP, auto-selected on 64-bit kernels with ARCH_HAS_PTE_PROTNONE and HAVE_ARCH_USERFAULTFD_WP. The symbol gates VM_UFFD_RWP (previously aliased to VM_NONE) and the smaps/trace-flag hooks added in the preparatory patches; without it the UAPI bits added here have nothing to drive and would be unreachable. Registration sets VM_UFFD_RWP on the VMA. Combining MODE_WP with MODE_RWP is rejected because both modes claim the uffd PTE bit. UFFDIO_RWPROTECT is the bidirectional counterpart of UFFDIO_WRITEPROTECT: - MODE_RWP change_protection() with MM_CP_UFFD_RWP installs PAGE_NONE and sets the uffd bit on present PTEs - !MODE_RWP change_protection() with MM_CP_UFFD_RWP_RESOLVE restores vma->vm_page_prot and clears the bit userfaultfd_clear_vma() runs the same resolve pass on unregister so RWP state cannot outlive the uffd. Re-registering a range must not drop a mode that installs per-PTE markers (WP or RWP); doing so returns -EBUSY. This also closes a pre-existing window where re-registering without MODE_WP would strand uffd-wp markers: before, those caused extra write-faults but were otherwise benign; with RWP preservation in place, a subsequent mprotect() on a VM_UFFD_RWP VMA would silently promote the stale markers to RWP. The feature is not yet advertised. UFFDIO_REGISTER_MODE_RWP, UFFD_FEATURE_RWP, and _UFFDIO_RWPROTECT are intentionally absent from UFFD_API_REGISTER_MODES, UFFD_API_FEATURES, and UFFD_API_RANGE_IOCTLS, so UFFDIO_API masks them out and the register-mode validator rejects the bit. The follow-up patch adds fault dispatch and exposes the UAPI. Link: https://lore.kernel.org/20260708111417.173443-10-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau Assisted-by: Claude:claude-opus-4-6 Reviewed-by: Mike Rapoport (Microsoft) Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: James Houghton Cc: Jonathan Corbet Cc: Liam Howlett Cc: Lorenzo Stoakes Cc: Paolo Bonzini Cc: Peter Xu Cc: Sean Christopherson Cc: SeongJae Park Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- Documentation/admin-guide/mm/userfaultfd.rst | 10 + include/linux/userfaultfd_k.h | 2 + include/uapi/linux/userfaultfd.h | 19 ++ mm/Kconfig | 9 + mm/userfaultfd.c | 191 ++++++++++++++++++- 5 files changed, 228 insertions(+), 3 deletions(-) diff --git a/Documentation/admin-guide/mm/userfaultfd.rst b/Documentation/admin-guide/mm/userfaultfd.rst index e5cc8848dcb3..1e533639fd50 100644 --- a/Documentation/admin-guide/mm/userfaultfd.rst +++ b/Documentation/admin-guide/mm/userfaultfd.rst @@ -131,6 +131,16 @@ userfaults on the range registered. Not all ioctls will necessarily be supported for all memory types (e.g. anonymous memory vs. shmem vs. hugetlbfs), or all types of intercepted faults. +.. note:: + + Re-registering an already-registered range must not drop any of the + modes that install per-PTE markers — currently + ``UFFDIO_REGISTER_MODE_WP`` and ``UFFDIO_REGISTER_MODE_RWP``. Doing + so would strand markers with no flag to describe them, so the call + is rejected with ``-EBUSY``; userspace must issue + ``UFFDIO_UNREGISTER`` first. This differs from older kernels, which + silently replaced the mode bits on re-registration. + Userland can use the ``uffdio_register.ioctls`` to manage the virtual address space in the background (to add or potentially also remove memory from the ``userfaultfd`` registered range). This means a userfault diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 0a0763c6e9f4..d31648ba898d 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -150,6 +150,8 @@ static inline uffd_flags_t uffd_flags_set_mode(uffd_flags_t flags, enum mfill_at extern long uffd_wp_range(struct vm_area_struct *vma, unsigned long start, unsigned long len, bool enable_wp); +extern int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, + unsigned long len, bool enable_rwp); /* move_pages */ void double_pt_lock(spinlock_t *ptl1, spinlock_t *ptl2); diff --git a/include/uapi/linux/userfaultfd.h b/include/uapi/linux/userfaultfd.h index 2841e4ea8f2c..7b78aa3b5318 100644 --- a/include/uapi/linux/userfaultfd.h +++ b/include/uapi/linux/userfaultfd.h @@ -79,6 +79,7 @@ #define _UFFDIO_WRITEPROTECT (0x06) #define _UFFDIO_CONTINUE (0x07) #define _UFFDIO_POISON (0x08) +#define _UFFDIO_RWPROTECT (0x09) #define _UFFDIO_API (0x3F) /* userfaultfd ioctl ids */ @@ -103,6 +104,8 @@ struct uffdio_continue) #define UFFDIO_POISON _IOWR(UFFDIO, _UFFDIO_POISON, \ struct uffdio_poison) +#define UFFDIO_RWPROTECT _IOWR(UFFDIO, _UFFDIO_RWPROTECT, \ + struct uffdio_rwprotect) /* read() structure */ struct uffd_msg { @@ -158,6 +161,7 @@ struct uffd_msg { #define UFFD_PAGEFAULT_FLAG_WRITE (1<<0) /* If this was a write fault */ #define UFFD_PAGEFAULT_FLAG_WP (1<<1) /* If reason is VM_UFFD_WP */ #define UFFD_PAGEFAULT_FLAG_MINOR (1<<2) /* If reason is VM_UFFD_MINOR */ +#define UFFD_PAGEFAULT_FLAG_RWP (1<<3) /* If reason is VM_UFFD_RWP */ struct uffdio_api { /* userland asks for an API number and the features to enable */ @@ -230,6 +234,11 @@ struct uffdio_api { * * UFFD_FEATURE_MOVE indicates that the kernel supports moving an * existing page contents from userspace. + * + * UFFD_FEATURE_RWP indicates that the kernel supports + * UFFDIO_REGISTER_MODE_RWP for read-write protection tracking. + * Pages are made inaccessible via UFFDIO_RWPROTECT and faults + * are delivered when the pages are re-accessed. */ #define UFFD_FEATURE_PAGEFAULT_FLAG_WP (1<<0) #define UFFD_FEATURE_EVENT_FORK (1<<1) @@ -248,6 +257,7 @@ struct uffdio_api { #define UFFD_FEATURE_POISON (1<<14) #define UFFD_FEATURE_WP_ASYNC (1<<15) #define UFFD_FEATURE_MOVE (1<<16) +#define UFFD_FEATURE_RWP (1<<17) __u64 features; __u64 ioctls; @@ -263,6 +273,7 @@ struct uffdio_register { #define UFFDIO_REGISTER_MODE_MISSING ((__u64)1<<0) #define UFFDIO_REGISTER_MODE_WP ((__u64)1<<1) #define UFFDIO_REGISTER_MODE_MINOR ((__u64)1<<2) +#define UFFDIO_REGISTER_MODE_RWP ((__u64)1<<3) __u64 mode; /* @@ -356,6 +367,14 @@ struct uffdio_poison { __s64 updated; }; +struct uffdio_rwprotect { + struct uffdio_range range; + /* !RWP means undo RWP-protection */ +#define UFFDIO_RWPROTECT_MODE_RWP ((__u64)1<<0) +#define UFFDIO_RWPROTECT_MODE_DONTWAKE ((__u64)1<<1) + __u64 mode; +}; + struct uffdio_move { __u64 dst; __u64 src; diff --git a/mm/Kconfig b/mm/Kconfig index a360861c74fa..4ea070f5664a 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1386,6 +1386,15 @@ config HAVE_ARCH_USERFAULTFD_MINOR help Arch has userfaultfd minor fault support +config USERFAULTFD_RWP + def_bool y + depends on 64BIT && ARCH_HAS_PTE_PROTNONE && HAVE_ARCH_USERFAULTFD_WP + help + Userfaultfd read-write protection (UFFDIO_RWPROTECT) delivers a + userfaultfd notification on every access -- read or write -- to a + protected range, letting userspace observe the working set of a + process. + menuconfig USERFAULTFD bool "Enable userfaultfd() system call" depends on MMU diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index c3c2fef8297c..83c571ecc377 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -1164,6 +1164,75 @@ static int mwriteprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, return err; } +int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, + unsigned long len, bool enable_rwp) +{ + struct mm_struct *dst_mm = ctx->mm; + unsigned long end = start + len; + struct vm_area_struct *dst_vma; + unsigned int mm_cp_flags; + struct mmu_gather tlb; + bool found = false; + VMA_ITERATOR(vmi, dst_mm, start); + + VM_WARN_ON_ONCE(start & ~PAGE_MASK); + VM_WARN_ON_ONCE(len & ~PAGE_MASK); + VM_WARN_ON_ONCE(start + len <= start); + + guard(mmap_read_lock)(dst_mm); + guard(rwsem_read)(&ctx->map_changing_lock); + + if (atomic_read(&ctx->mmap_changing)) + return -EAGAIN; + + if (enable_rwp) + mm_cp_flags = MM_CP_UFFD_RWP; + else + mm_cp_flags = MM_CP_UFFD_RWP_RESOLVE; + + /* + * Pre-scan the range: validate every spanned VMA before applying + * any change_protection() so a partial failure cannot leave the + * process with only a prefix of the range re-protected. + */ + for_each_vma_range(vmi, dst_vma, end) { + if (!userfaultfd_rwp(dst_vma)) + return -ENOENT; + + if (is_vm_hugetlb_page(dst_vma)) { + unsigned long page_mask; + + page_mask = vma_kernel_pagesize(dst_vma) - 1; + if ((start & page_mask) || (len & page_mask)) + return -EINVAL; + } + found = true; + } + if (!found) + return -ENOENT; + + vma_iter_set(&vmi, start); + tlb_gather_mmu(&tlb, dst_mm); + for_each_vma_range(vmi, dst_vma, end) { + unsigned long vma_start = max(dst_vma->vm_start, start); + unsigned long vma_end = min(dst_vma->vm_end, end); + unsigned int flags = mm_cp_flags; + + /* + * On resolve, try to upgrade writability per-VMA -- + * MM_CP_TRY_CHANGE_WRITABLE WARNs in + * maybe_change_pte_writable() if the VMA is not VM_WRITE, + * and RWP can be registered on PROT_READ-only mappings. + */ + if (!enable_rwp && vma_wants_manual_pte_write_upgrade(dst_vma)) + flags |= MM_CP_TRY_CHANGE_WRITABLE; + + change_protection(&tlb, dst_vma, vma_start, vma_end, flags); + } + tlb_finish_mmu(&tlb); + + return 0; +} void double_pt_lock(spinlock_t *ptl1, spinlock_t *ptl2) @@ -2207,9 +2276,22 @@ static struct vm_area_struct *userfaultfd_clear_vma(struct vma_iterator *vmi, if (start == vma->vm_start && end == vma->vm_end) give_up_on_oom = true; - /* Reset ptes for the whole vma range if wr-protected */ - if (userfaultfd_wp(vma)) - uffd_wp_range(vma, start, end - start, false); + /* Clear the uffd bit and/or restore protnone PTEs */ + if (userfaultfd_protected(vma)) { + unsigned int mm_cp_flags = 0; + struct mmu_gather tlb; + + if (userfaultfd_wp(vma)) + mm_cp_flags |= MM_CP_UFFD_WP_RESOLVE; + if (userfaultfd_rwp(vma)) + mm_cp_flags |= MM_CP_UFFD_RWP_RESOLVE; + if (vma_wants_manual_pte_write_upgrade(vma)) + mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE; + + tlb_gather_mmu(&tlb, vma->vm_mm); + change_protection(&tlb, vma, start, end, mm_cp_flags); + tlb_finish_mmu(&tlb); + } ret = vma_modify_flags_uffd(vmi, prev, vma, start, end, &new_vma_flags, NULL_VM_UFFD_CTX, @@ -2258,6 +2340,14 @@ static int userfaultfd_register_range(struct userfaultfd_ctx *ctx, vma_test_all_mask(vma, vma_flags)) goto skip; + /* + * Pre-scan in userfaultfd_register() already rejected mode + * switches that would drop VM_UFFD_WP or VM_UFFD_RWP, so a + * stray bit here is a bug. + */ + VM_WARN_ON_ONCE(vma->vm_userfaultfd_ctx.ctx == ctx && + vma->vm_flags & (VM_UFFD_WP | VM_UFFD_RWP) & ~vm_flags); + if (vma->vm_start > start) start = vma->vm_start; vma_end = min(end, vma->vm_end); @@ -2524,6 +2614,8 @@ static inline struct uffd_msg userfault_msg(unsigned long address, msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE; if (reason & VM_UFFD_WP) msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP; + if (reason & VM_UFFD_RWP) + msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_RWP; if (reason & VM_UFFD_MINOR) msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_MINOR; if (features & UFFD_FEATURE_THREAD_ID) @@ -3623,6 +3715,22 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, vm_flags |= VM_UFFD_WP; } + if (uffdio_register.mode & UFFDIO_REGISTER_MODE_RWP) { + if (!pgtable_supports_uffd() || VM_UFFD_RWP == VM_NONE) + goto out; + if (!(ctx->features & UFFD_FEATURE_RWP)) + goto out; + vm_flags |= VM_UFFD_RWP; + } + + /* + * WP and RWP share the uffd PTE bit and + * cannot coexist in the same VMA — the bit would carry ambiguous + * semantics. Reject the combination up front. + */ + if ((vm_flags & VM_UFFD_WP) && (vm_flags & VM_UFFD_RWP)) + goto out; + if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR) { #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR goto out; @@ -3677,6 +3785,17 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, if (!vma_can_userfault(cur, vm_flags, wp_async)) goto out_unlock; + /* + * RWP uses protnone as an access-tracking marker. PROT_NONE + * VMAs have vm_page_prot == PAGE_NONE, so RWP resolution + * cannot make a page accessible again. Reject at register + * time only: a VMA that later becomes inaccessible via + * mprotect() must still be unregisterable, so this is not + * part of vma_can_userfault(). + */ + if ((vm_flags & VM_UFFD_RWP) && !vma_is_accessible(cur)) + goto out_unlock; + /* * UFFDIO_COPY will fill file holes even without * PROT_WRITE. This check enforces that if this is a @@ -3716,6 +3835,16 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, cur->vm_userfaultfd_ctx.ctx != ctx) goto out_unlock; + /* + * Mode switches that drop VM_UFFD_WP or VM_UFFD_RWP would + * leave PTE markers without the flag that describes them; + * subsequent mprotect() would then promote stale markers + * into the other mode. Require an unregister first. + */ + if (cur->vm_userfaultfd_ctx.ctx == ctx && + cur->vm_flags & (VM_UFFD_WP | VM_UFFD_RWP) & ~vm_flags) + goto out_unlock; + /* * Note vmas containing huge pages */ @@ -3749,6 +3878,10 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR)) ioctls_out &= ~((__u64)1 << _UFFDIO_CONTINUE); + /* RWPROTECT is only supported for RWP ranges */ + if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_RWP)) + ioctls_out &= ~((__u64)1 << _UFFDIO_RWPROTECT); + /* * Now that we scanned all vmas we can already tell * userland which ioctls methods are guaranteed to @@ -4096,6 +4229,55 @@ static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx, return ret; } +static int userfaultfd_rwprotect(struct userfaultfd_ctx *ctx, + unsigned long arg) +{ + int ret; + struct uffdio_rwprotect uffdio_rwp; + struct userfaultfd_wake_range range; + bool mode_rwp, mode_dontwake; + + if (atomic_read(&ctx->mmap_changing)) + return -EAGAIN; + + if (copy_from_user(&uffdio_rwp, (void __user *)arg, + sizeof(uffdio_rwp))) + return -EFAULT; + + ret = validate_range(ctx->mm, uffdio_rwp.range.start, + uffdio_rwp.range.len); + if (ret) + return ret; + + if (uffdio_rwp.mode & ~(UFFDIO_RWPROTECT_MODE_DONTWAKE | + UFFDIO_RWPROTECT_MODE_RWP)) + return -EINVAL; + + mode_rwp = uffdio_rwp.mode & UFFDIO_RWPROTECT_MODE_RWP; + mode_dontwake = uffdio_rwp.mode & UFFDIO_RWPROTECT_MODE_DONTWAKE; + + if (mode_rwp && mode_dontwake) + return -EINVAL; + + if (mmget_not_zero(ctx->mm)) { + ret = mrwprotect_range(ctx, uffdio_rwp.range.start, + uffdio_rwp.range.len, mode_rwp); + mmput(ctx->mm); + } else { + return -ESRCH; + } + + if (ret) + return ret; + + if (!mode_rwp && !mode_dontwake) { + range.start = uffdio_rwp.range.start; + range.len = uffdio_rwp.range.len; + wake_userfault(ctx, &range); + } + return ret; +} + static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg) { __s64 ret; @@ -4402,6 +4584,9 @@ static long userfaultfd_ioctl(struct file *file, unsigned cmd, case UFFDIO_POISON: ret = userfaultfd_poison(ctx, arg); break; + case UFFDIO_RWPROTECT: + ret = userfaultfd_rwprotect(ctx, arg); + break; } return ret; } From 2d427bb0163978875dfbbcec7b7dd0c1320f419e Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Wed, 8 Jul 2026 12:14:11 +0100 Subject: [PATCH 265/501] mm/userfaultfd: add RWP fault delivery and expose UFFDIO_REGISTER_MODE_RWP Wire the fault side of read-write protection tracking and turn the userspace interface on. An RWP-protected PTE is PAGE_NONE with the uffd bit set. The PROT_NONE triggers a fault on any access; the uffd bit distinguishes it from plain mprotect(PROT_NONE) or NUMA hinting. Fault dispatch, per level: PTE handle_pte_fault() -> do_uffd_rwp() PMD __handle_mm_fault() -> do_huge_pmd_uffd_rwp() hugetlb hugetlb_fault() -> hugetlb_handle_userfault() The RWP branches gate on userfaultfd_pte_rwp() / userfaultfd_huge_pmd_rwp() (VM_UFFD_RWP plus the uffd bit) and fall through to do_numa_page() / do_huge_pmd_numa_page() otherwise. Each delivers a UFFD_PAGEFAULT_FLAG_RWP message through handle_userfault(); the handler resolves it with UFFDIO_RWPROTECT clearing MODE_RWP. userfaultfd_must_wait() and userfaultfd_huge_must_wait() add matching protnone+uffd waiters so sync-mode fault handlers block correctly. Expose the UAPI: UFFDIO_REGISTER_MODE_RWP -> UFFD_API_REGISTER_MODES UFFD_FEATURE_RWP -> UFFD_API_FEATURES _UFFDIO_RWPROTECT -> UFFD_API_RANGE_IOCTLS UFFD_API_RANGE_IOCTLS_BASIC UFFD_FEATURE_RWP is masked out at UFFDIO_API time when PROT_NONE is not available or VM_UFFD_RWP aliases VM_NONE (32-bit), so userspace never sees an advertised-but-broken feature. Works on anonymous, shmem, and hugetlb memory. Link: https://lore.kernel.org/20260708111417.173443-11-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau Assisted-by: Claude:claude-opus-4-6 Reviewed-by: Mike Rapoport (Microsoft) Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: James Houghton Cc: Jonathan Corbet Cc: Liam Howlett Cc: Lorenzo Stoakes Cc: Paolo Bonzini Cc: Peter Xu Cc: Sean Christopherson Cc: SeongJae Park Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/huge_mm.h | 7 +++++++ include/linux/userfaultfd_k.h | 24 ++++++++++++++++++++++++ include/uapi/linux/userfaultfd.h | 12 ++++++++---- mm/huge_memory.c | 5 +++++ mm/hugetlb.c | 11 +++++++++++ mm/memory.c | 31 +++++++++++++++++++++++++++++-- mm/userfaultfd.c | 32 ++++++++++++++++++++++++++++++-- 7 files changed, 114 insertions(+), 8 deletions(-) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index b34e8e5e5f9a..19a7bfce279e 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -529,6 +529,8 @@ static inline bool folio_test_pmd_mappable(struct folio *folio) vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf); +vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf); + vm_fault_t do_huge_pmd_device_private(struct vm_fault *vmf); extern struct folio *huge_zero_folio; @@ -723,6 +725,11 @@ static inline spinlock_t *pud_trans_huge_lock(pud_t *pud, return NULL; } +static inline vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf) +{ + return 0; +} + static inline vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) { return 0; diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index d31648ba898d..6f7245609bd4 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -241,6 +241,18 @@ static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, return userfaultfd_wp(vma) && pmd_uffd(pmd); } +static inline bool userfaultfd_pte_rwp(struct vm_area_struct *vma, + pte_t pte) +{ + return userfaultfd_rwp(vma) && pte_uffd(pte); +} + +static inline bool userfaultfd_huge_pmd_rwp(struct vm_area_struct *vma, + pmd_t pmd) +{ + return userfaultfd_rwp(vma) && pmd_uffd(pmd); +} + static inline bool userfaultfd_armed(struct vm_area_struct *vma) { return vma_test_any_mask(vma, __VMA_UFFD_FLAGS); @@ -371,6 +383,18 @@ static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, return false; } +static inline bool userfaultfd_pte_rwp(struct vm_area_struct *vma, + pte_t pte) +{ + return false; +} + +static inline bool userfaultfd_huge_pmd_rwp(struct vm_area_struct *vma, + pmd_t pmd) +{ + return false; +} + static inline bool userfaultfd_armed(struct vm_area_struct *vma) { return false; diff --git a/include/uapi/linux/userfaultfd.h b/include/uapi/linux/userfaultfd.h index 7b78aa3b5318..d803e76d47ad 100644 --- a/include/uapi/linux/userfaultfd.h +++ b/include/uapi/linux/userfaultfd.h @@ -25,7 +25,8 @@ #define UFFD_API ((__u64)0xAA) #define UFFD_API_REGISTER_MODES (UFFDIO_REGISTER_MODE_MISSING | \ UFFDIO_REGISTER_MODE_WP | \ - UFFDIO_REGISTER_MODE_MINOR) + UFFDIO_REGISTER_MODE_MINOR | \ + UFFDIO_REGISTER_MODE_RWP) #define UFFD_API_FEATURES (UFFD_FEATURE_PAGEFAULT_FLAG_WP | \ UFFD_FEATURE_EVENT_FORK | \ UFFD_FEATURE_EVENT_REMAP | \ @@ -42,7 +43,8 @@ UFFD_FEATURE_WP_UNPOPULATED | \ UFFD_FEATURE_POISON | \ UFFD_FEATURE_WP_ASYNC | \ - UFFD_FEATURE_MOVE) + UFFD_FEATURE_MOVE | \ + UFFD_FEATURE_RWP) #define UFFD_API_IOCTLS \ ((__u64)1 << _UFFDIO_REGISTER | \ (__u64)1 << _UFFDIO_UNREGISTER | \ @@ -54,13 +56,15 @@ (__u64)1 << _UFFDIO_MOVE | \ (__u64)1 << _UFFDIO_WRITEPROTECT | \ (__u64)1 << _UFFDIO_CONTINUE | \ - (__u64)1 << _UFFDIO_POISON) + (__u64)1 << _UFFDIO_POISON | \ + (__u64)1 << _UFFDIO_RWPROTECT) #define UFFD_API_RANGE_IOCTLS_BASIC \ ((__u64)1 << _UFFDIO_WAKE | \ (__u64)1 << _UFFDIO_COPY | \ (__u64)1 << _UFFDIO_WRITEPROTECT | \ (__u64)1 << _UFFDIO_CONTINUE | \ - (__u64)1 << _UFFDIO_POISON) + (__u64)1 << _UFFDIO_POISON | \ + (__u64)1 << _UFFDIO_RWPROTECT) /* * Valid ioctl command number range with this API is from 0x00 to diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 991f216f4f5b..dfbf6f3219ac 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2202,6 +2202,11 @@ static inline bool can_change_pmd_writable(struct vm_area_struct *vma, return pmd_dirty(pmd); } +vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf) +{ + return handle_userfault(vmf, VM_UFFD_RWP); +} + /* NUMA hinting page fault entry point for trans huge pmds */ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) { diff --git a/mm/hugetlb.c b/mm/hugetlb.c index bb6219002a5c..d6ebf18568fa 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -6095,6 +6095,17 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, goto out_mutex; } + /* + * Protnone hugetlb PTEs with the uffd bit are used by + * userfaultfd RWP for access tracking. Plain PROT_NONE (without the + * marker) is not an RWP fault and is not expected on hugetlb (no + * NUMA hinting), so let normal hugetlb fault handling proceed. + */ + if (pte_protnone(vmf.orig_pte) && vma_is_accessible(vma) && + userfaultfd_rwp(vma) && huge_pte_uffd(vmf.orig_pte)) { + return hugetlb_handle_userfault(&vmf, mapping, VM_UFFD_RWP); + } + /* * If we are going to COW/unshare the mapping later, we examine the * pending reservations for this page now. This will ensure that any diff --git a/mm/memory.c b/mm/memory.c index 182d1516c0c1..6e01690fd0e4 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -6119,6 +6119,16 @@ static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_stru if (!pte_present(ptent) || !pte_protnone(ptent)) continue; + /* + * RWP-armed PTEs are also protnone but carry _PAGE_UFFD as a + * marker. Leave them alone -- rewriting to vm_page_prot would + * stop the RWP trap. Gate on userfaultfd_rwp(vma) too: + * NUMA balancing preserves _PAGE_UFFD on UFFD_WP-marked PTEs + * when applying PROT_NONE, and those still need rebuilding. + */ + if (userfaultfd_rwp(vma) && pte_uffd(ptent)) + continue; + if (pfn_folio(pte_pfn(ptent)) != folio) continue; @@ -6134,6 +6144,12 @@ static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_stru } } +static vm_fault_t do_uffd_rwp(struct vm_fault *vmf) +{ + pte_unmap(vmf->pte); + return handle_userfault(vmf, VM_UFFD_RWP); +} + static vm_fault_t do_numa_page(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; @@ -6409,8 +6425,16 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) if (!pte_present(vmf->orig_pte)) return do_swap_page(vmf); - if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma)) + if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma)) { + /* + * RWP-protected PTEs are protnone plus the uffd bit. On a + * VM_UFFD_RWP VMA, a protnone PTE without the uffd bit is + * NUMA hinting and must still fall through to do_numa_page(). + */ + if (userfaultfd_pte_rwp(vmf->vma, vmf->orig_pte)) + return do_uffd_rwp(vmf); return do_numa_page(vmf); + } spin_lock(vmf->ptl); entry = vmf->orig_pte; @@ -6524,8 +6548,11 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, return 0; } if (pmd_trans_huge(vmf.orig_pmd)) { - if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma)) + if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma)) { + if (userfaultfd_huge_pmd_rwp(vma, vmf.orig_pmd)) + return do_huge_pmd_uffd_rwp(&vmf); return do_huge_pmd_numa_page(&vmf); + } if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) && !pmd_write(vmf.orig_pmd)) { diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 83c571ecc377..f4a64971d9dc 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -2669,6 +2669,12 @@ static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx, */ if (!huge_pte_write(pte) && (reason & VM_UFFD_WP)) return true; + /* + * PTE is still RW-protected (protnone with uffd bit), wait for + * resolution. Plain PROT_NONE without the marker is not an RWP fault. + */ + if (pte_protnone(pte) && huge_pte_uffd(pte) && (reason & VM_UFFD_RWP)) + return true; return false; } @@ -2729,8 +2735,14 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, if (!pmd_present(_pmd)) return false; - if (pmd_trans_huge(_pmd)) - return !pmd_write(_pmd) && (reason & VM_UFFD_WP); + if (pmd_trans_huge(_pmd)) { + if (!pmd_write(_pmd) && (reason & VM_UFFD_WP)) + return true; + if (pmd_protnone(_pmd) && pmd_uffd(_pmd) && + (reason & VM_UFFD_RWP)) + return true; + return false; + } pte = pte_offset_map(pmd, address); if (!pte) @@ -2766,6 +2778,13 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, */ if (!pte_write(ptent) && (reason & VM_UFFD_WP)) goto out; + /* + * PTE is still RW-protected (protnone with uffd bit), wait for + * userspace to resolve. Plain PROT_NONE without the marker is not + * an RWP fault. + */ + if (pte_protnone(ptent) && pte_uffd(ptent) && (reason & VM_UFFD_RWP)) + goto out; ret = false; out: @@ -4518,6 +4537,15 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx, uffdio_api.features &= ~UFFD_FEATURE_WP_UNPOPULATED; uffdio_api.features &= ~UFFD_FEATURE_WP_ASYNC; } + /* + * RWP needs both PROT_NONE support and the uffd-wp PTE bit. The + * VM_UFFD_RWP check covers compile-time unavailability; the + * pgtable_supports_uffd() check covers runtime (e.g. riscv + * without the SVRSW60T59B extension) where the PTE bit is declared + * but not actually usable. + */ + if (VM_UFFD_RWP == VM_NONE || !pgtable_supports_uffd()) + uffdio_api.features &= ~UFFD_FEATURE_RWP; ret = -EINVAL; if (features & ~uffdio_api.features) From 3389648819829299517a8d482fda213fd7166a2c Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Wed, 8 Jul 2026 12:14:12 +0100 Subject: [PATCH 266/501] mm/pagemap: add PAGE_IS_ACCESSED for RWP tracking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PAGEMAP_SCAN already reports PAGE_IS_WRITTEN from the inverted uffd PTE bit, targeting the UFFDIO_WRITEPROTECT workflow. UFFDIO_RWPROTECT reuses the same PTE bit as a marker for read-write protection, but "has been written" and "has been accessed" are distinct semantic signals — they happen to share one PTE bit today only because the two implementations share infrastructure. Give RWP its own pagemap category so the UAPI does not conflate them: PAGE_IS_WRITTEN reported on VM_UFFD_WP VMAs, !pte_uffd(pte) PAGE_IS_ACCESSED reported on VM_UFFD_RWP VMAs, !pte_uffd(pte) Both still read the same PTE bit today, but each is scoped to the VMA whose registered mode makes the bit meaningful. If a future implementation moves RWP to a separate PTE bit, only PAGE_IS_ACCESSED switches over. This is a UAPI narrowing. Outside VM_UFFD_WP VMAs the uffd bit is always clear, so PAGEMAP_SCAN used to flag PAGE_IS_WRITTEN on every present PTE there — a meaningless duplicate of PAGE_IS_PRESENT. Now PAGE_IS_WRITTEN fires only inside VM_UFFD_WP VMAs. pagemap_hugetlb_category() now takes the vma like its PTE/PMD peers. Link: https://lore.kernel.org/20260708111417.173443-12-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau Assisted-by: Claude:claude-opus-4-6 Acked-by: Mike Rapoport (Microsoft) Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: James Houghton Cc: Jonathan Corbet Cc: Liam Howlett Cc: Lorenzo Stoakes Cc: Paolo Bonzini Cc: Peter Xu Cc: Sean Christopherson Cc: SeongJae Park Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- Documentation/admin-guide/mm/pagemap.rst | 13 +++- fs/proc/task_mmu.c | 81 ++++++++++++++++-------- include/uapi/linux/fs.h | 1 + tools/include/uapi/linux/fs.h | 1 + 4 files changed, 67 insertions(+), 29 deletions(-) diff --git a/Documentation/admin-guide/mm/pagemap.rst b/Documentation/admin-guide/mm/pagemap.rst index c57e61b5d8aa..ffa690a171c8 100644 --- a/Documentation/admin-guide/mm/pagemap.rst +++ b/Documentation/admin-guide/mm/pagemap.rst @@ -19,8 +19,11 @@ There are four components to pagemap: * Bit 55 pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst) * Bit 56 page exclusively mapped (since 4.2) - * Bit 57 pte is uffd-wp write-protected (since 5.13) (see - Documentation/admin-guide/mm/userfaultfd.rst) + * Bit 57 pte is tracked by userfaultfd (since 5.13) — in a + ``VM_UFFD_WP`` VMA this indicates a write-protected PTE; in a + ``VM_UFFD_RWP`` VMA it indicates an RWP-protected PTE. WP and + RWP are mutually exclusive per VMA, so the meaning is + unambiguous. See Documentation/admin-guide/mm/userfaultfd.rst. * Bit 58 pte is a guard region (since 6.15) (see madvise (2) man page) * Bits 59-60 zero * Bit 61 page is file-page or shared-anon (since 3.5) @@ -244,7 +247,8 @@ in this IOCTL: Following flags about pages are currently supported: - ``PAGE_IS_WPALLOWED`` - Page has async-write-protection enabled -- ``PAGE_IS_WRITTEN`` - Page has been written to from the time it was write protected +- ``PAGE_IS_WRITTEN`` - Page in a ``UFFDIO_REGISTER_MODE_WP`` VMA has been + written to since it was write-protected. Only reported inside such VMAs. - ``PAGE_IS_FILE`` - Page is file backed - ``PAGE_IS_PRESENT`` - Page is present in the memory - ``PAGE_IS_SWAPPED`` - Page is in swapped @@ -252,6 +256,9 @@ Following flags about pages are currently supported: - ``PAGE_IS_HUGE`` - Page is PMD-mapped THP or Hugetlb backed - ``PAGE_IS_SOFT_DIRTY`` - Page is soft-dirty - ``PAGE_IS_GUARD`` - Page is a part of a guard region +- ``PAGE_IS_ACCESSED`` - Page in a ``UFFDIO_REGISTER_MODE_RWP`` VMA has been + accessed since RWP was applied. Only reported inside such VMAs. See + Documentation/admin-guide/mm/userfaultfd.rst for the RWP workflow. The ``struct pm_scan_arg`` is used as the argument of the IOCTL. diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 407d8dde2a90..817e3e0f9194 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -2284,7 +2284,7 @@ static const struct mm_walk_ops pagemap_ops = { * Bits 5-54 swap offset if swapped * Bit 55 pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst) * Bit 56 page exclusively mapped - * Bit 57 pte is uffd-wp write-protected + * Bit 57 pte is tracked by userfaultfd (uffd-wp or RWP) * Bit 58 pte is a guard region * Bits 59-60 zero * Bit 61 page is file-page or shared-anon @@ -2419,7 +2419,7 @@ static int pagemap_release(struct inode *inode, struct file *file) PAGE_IS_FILE | PAGE_IS_PRESENT | \ PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | \ PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY | \ - PAGE_IS_GUARD) + PAGE_IS_GUARD | PAGE_IS_ACCESSED) #define PM_SCAN_FLAGS (PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC) struct pagemap_scan_private { @@ -2438,15 +2438,17 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p, if (pte_none(pte)) { /* - * An unpopulated pte carries no uffd-wp marker, i.e. it is not - * write-protected, the same condition under which the present - * and swap cases below report PAGE_IS_WRITTEN. Report it here - * too so this generic path agrees with the PAGE_IS_WRITTEN fast - * path in pagemap_scan_pmd_entry(), which reports pte_none as - * written and, under PM_SCAN_WP_MATCHING, arms a marker. The - * fast path applies no VMA test, so neither does this. + * An unpopulated pte carries no uffd bit, i.e. it is not + * write-protected. The PAGE_IS_WRITTEN fast path in + * pagemap_scan_pmd_entry() is now gated on a VM_UFFD_WP VMA; + * gate the pte_none report here the same way so the two paths + * still agree. RWP has no such fast path and an unpopulated + * page is not part of the RWP working set, so it is reported as + * neither. */ - return PAGE_IS_WRITTEN; + if (userfaultfd_wp(vma)) + return PAGE_IS_WRITTEN; + return 0; } if (pte_present(pte)) { @@ -2454,8 +2456,12 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p, categories = PAGE_IS_PRESENT; - if (!pte_uffd(pte)) - categories |= PAGE_IS_WRITTEN; + if (!pte_uffd(pte)) { + if (userfaultfd_wp(vma)) + categories |= PAGE_IS_WRITTEN; + if (userfaultfd_rwp(vma)) + categories |= PAGE_IS_ACCESSED; + } if (p->masks_of_interest & PAGE_IS_FILE) { page = vm_normal_page(vma, addr, pte); @@ -2472,8 +2478,12 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p, categories = PAGE_IS_SWAPPED; - if (!pte_swp_uffd_any(pte)) - categories |= PAGE_IS_WRITTEN; + if (!pte_swp_uffd_any(pte)) { + if (userfaultfd_wp(vma)) + categories |= PAGE_IS_WRITTEN; + if (userfaultfd_rwp(vma)) + categories |= PAGE_IS_ACCESSED; + } entry = softleaf_from_pte(pte); if (softleaf_is_guard_marker(entry)) @@ -2522,8 +2532,12 @@ static unsigned long pagemap_thp_category(struct pagemap_scan_private *p, struct page *page; categories |= PAGE_IS_PRESENT; - if (!pmd_uffd(pmd)) - categories |= PAGE_IS_WRITTEN; + if (!pmd_uffd(pmd)) { + if (userfaultfd_wp(vma)) + categories |= PAGE_IS_WRITTEN; + if (userfaultfd_rwp(vma)) + categories |= PAGE_IS_ACCESSED; + } if (p->masks_of_interest & PAGE_IS_FILE) { page = vm_normal_page_pmd(vma, addr, pmd); @@ -2537,8 +2551,12 @@ static unsigned long pagemap_thp_category(struct pagemap_scan_private *p, categories |= PAGE_IS_SOFT_DIRTY; } else { categories |= PAGE_IS_SWAPPED; - if (!pmd_swp_uffd(pmd)) - categories |= PAGE_IS_WRITTEN; + if (!pmd_swp_uffd(pmd)) { + if (userfaultfd_wp(vma)) + categories |= PAGE_IS_WRITTEN; + if (userfaultfd_rwp(vma)) + categories |= PAGE_IS_ACCESSED; + } if (pmd_swp_soft_dirty(pmd)) categories |= PAGE_IS_SOFT_DIRTY; @@ -2571,7 +2589,8 @@ static void make_uffd_wp_pmd(struct vm_area_struct *vma, #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #ifdef CONFIG_HUGETLB_PAGE -static unsigned long pagemap_hugetlb_category(pte_t pte) +static unsigned long pagemap_hugetlb_category(struct vm_area_struct *vma, + pte_t pte) { unsigned long categories = PAGE_IS_HUGE; @@ -2586,8 +2605,12 @@ static unsigned long pagemap_hugetlb_category(pte_t pte) if (pte_present(pte)) { categories |= PAGE_IS_PRESENT; - if (!huge_pte_uffd(pte)) - categories |= PAGE_IS_WRITTEN; + if (!huge_pte_uffd(pte)) { + if (userfaultfd_wp(vma)) + categories |= PAGE_IS_WRITTEN; + if (userfaultfd_rwp(vma)) + categories |= PAGE_IS_ACCESSED; + } if (!PageAnon(pte_page(pte))) categories |= PAGE_IS_FILE; if (is_zero_pfn(pte_pfn(pte))) @@ -2597,8 +2620,12 @@ static unsigned long pagemap_hugetlb_category(pte_t pte) } else { categories |= PAGE_IS_SWAPPED; - if (!pte_swp_uffd_any(pte)) - categories |= PAGE_IS_WRITTEN; + if (!pte_swp_uffd_any(pte)) { + if (userfaultfd_wp(vma)) + categories |= PAGE_IS_WRITTEN; + if (userfaultfd_rwp(vma)) + categories |= PAGE_IS_ACCESSED; + } if (pte_swp_soft_dirty(pte)) categories |= PAGE_IS_SOFT_DIRTY; } @@ -2874,7 +2901,8 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, goto flush_and_return; } - if (!p->arg.category_anyof_mask && !p->arg.category_inverted && + if (userfaultfd_wp(vma) && !p->arg.category_anyof_mask && + !p->arg.category_inverted && p->arg.category_mask == PAGE_IS_WRITTEN && p->arg.return_mask == PAGE_IS_WRITTEN) { for (addr = start; addr < end; pte++, addr += PAGE_SIZE) { @@ -2949,7 +2977,8 @@ static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask, /* Go the short route when not write-protecting pages. */ pte = huge_ptep_get(walk->mm, start, ptep); - categories = p->cur_vma_category | pagemap_hugetlb_category(pte); + categories = p->cur_vma_category | + pagemap_hugetlb_category(vma, pte); if (!pagemap_scan_is_interesting_page(categories, p)) return 0; @@ -2961,7 +2990,7 @@ static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask, ptl = huge_pte_lock(hstate_vma(vma), vma->vm_mm, ptep); pte = huge_ptep_get(walk->mm, start, ptep); - categories = p->cur_vma_category | pagemap_hugetlb_category(pte); + categories = p->cur_vma_category | pagemap_hugetlb_category(vma, pte); if (!pagemap_scan_is_interesting_page(categories, p)) goto out_unlock; diff --git a/include/uapi/linux/fs.h b/include/uapi/linux/fs.h index bd87262f2e34..34c6f219462a 100644 --- a/include/uapi/linux/fs.h +++ b/include/uapi/linux/fs.h @@ -471,6 +471,7 @@ typedef int __bitwise __kernel_rwf_t; #define PAGE_IS_HUGE (1 << 6) #define PAGE_IS_SOFT_DIRTY (1 << 7) #define PAGE_IS_GUARD (1 << 8) +#define PAGE_IS_ACCESSED (1 << 9) /* * struct page_region - Page region with flags diff --git a/tools/include/uapi/linux/fs.h b/tools/include/uapi/linux/fs.h index 24ddf7bc4f25..f0a26309b6d5 100644 --- a/tools/include/uapi/linux/fs.h +++ b/tools/include/uapi/linux/fs.h @@ -364,6 +364,7 @@ typedef int __bitwise __kernel_rwf_t; #define PAGE_IS_HUGE (1 << 6) #define PAGE_IS_SOFT_DIRTY (1 << 7) #define PAGE_IS_GUARD (1 << 8) +#define PAGE_IS_ACCESSED (1 << 9) /* * struct page_region - Page region with flags From 45347e3d32c15bc70f097bef9ef1a7b90218d450 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Wed, 8 Jul 2026 12:14:13 +0100 Subject: [PATCH 267/501] userfaultfd: add UFFD_FEATURE_RWP_ASYNC for async fault resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sync RWP delivers a message and blocks the faulting thread until the handler resolves the fault. For working-set tracking the VMM does not need the message: it just needs to know, at scan time, which pages were touched. Async RWP serves that use case — the kernel restores access in-place and the faulting thread continues without blocking. The VMM reconstructs the access pattern after the fact via PAGEMAP_SCAN: pages whose uffd bit is still set (inverted PAGE_IS_ACCESSED) were not re-accessed since the last RWP cycle. Worth calling out: async resolution upgrades writable private anon PTEs via pte_mkwrite() when can_change_pte_writable() allows, mirroring do_numa_page(). Without it, every re-access of an RWP'd writable page would COW-fault a second time. UFFD_FEATURE_RWP_ASYNC requires UFFD_FEATURE_RWP. Link: https://lore.kernel.org/20260708111417.173443-13-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau Assisted-by: Claude:claude-opus-4-6 Acked-by: Mike Rapoport (Microsoft) Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: James Houghton Cc: Jonathan Corbet Cc: Liam Howlett Cc: Lorenzo Stoakes Cc: Paolo Bonzini Cc: Peter Xu Cc: Sean Christopherson Cc: SeongJae Park Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/userfaultfd_k.h | 6 ++++++ include/uapi/linux/userfaultfd.h | 11 ++++++++++- mm/huge_memory.c | 25 ++++++++++++++++++++++++- mm/hugetlb.c | 32 +++++++++++++++++++++++++++++++- mm/memory.c | 27 +++++++++++++++++++++++++-- mm/userfaultfd.c | 19 ++++++++++++++++++- 6 files changed, 114 insertions(+), 6 deletions(-) diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 6f7245609bd4..bfbd6a59909f 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -286,6 +286,7 @@ extern void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf); extern bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma); extern bool userfaultfd_wp_async(struct vm_area_struct *vma); +extern bool userfaultfd_rwp_async(struct vm_area_struct *vma); static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma) { @@ -464,6 +465,11 @@ static inline bool userfaultfd_wp_async(struct vm_area_struct *vma) return false; } +static inline bool userfaultfd_rwp_async(struct vm_area_struct *vma) +{ + return false; +} + static inline bool vma_has_uffd_without_event_remap(struct vm_area_struct *vma) { return false; diff --git a/include/uapi/linux/userfaultfd.h b/include/uapi/linux/userfaultfd.h index d803e76d47ad..c10f08f8a618 100644 --- a/include/uapi/linux/userfaultfd.h +++ b/include/uapi/linux/userfaultfd.h @@ -44,7 +44,8 @@ UFFD_FEATURE_POISON | \ UFFD_FEATURE_WP_ASYNC | \ UFFD_FEATURE_MOVE | \ - UFFD_FEATURE_RWP) + UFFD_FEATURE_RWP | \ + UFFD_FEATURE_RWP_ASYNC) #define UFFD_API_IOCTLS \ ((__u64)1 << _UFFDIO_REGISTER | \ (__u64)1 << _UFFDIO_UNREGISTER | \ @@ -243,6 +244,13 @@ struct uffdio_api { * UFFDIO_REGISTER_MODE_RWP for read-write protection tracking. * Pages are made inaccessible via UFFDIO_RWPROTECT and faults * are delivered when the pages are re-accessed. + * + * UFFD_FEATURE_RWP_ASYNC indicates asynchronous mode for + * UFFDIO_REGISTER_MODE_RWP. When set, faults on read-write + * protected pages are auto-resolved by the kernel (PTE + * permissions restored immediately) without delivering a message + * to the userfaultfd handler. Use PAGEMAP_SCAN with inverted + * PAGE_IS_ACCESSED to find pages that were not re-accessed. */ #define UFFD_FEATURE_PAGEFAULT_FLAG_WP (1<<0) #define UFFD_FEATURE_EVENT_FORK (1<<1) @@ -262,6 +270,7 @@ struct uffdio_api { #define UFFD_FEATURE_WP_ASYNC (1<<15) #define UFFD_FEATURE_MOVE (1<<16) #define UFFD_FEATURE_RWP (1<<17) +#define UFFD_FEATURE_RWP_ASYNC (1<<18) __u64 features; __u64 ioctls; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index dfbf6f3219ac..b6b0355268ea 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2204,7 +2204,30 @@ static inline bool can_change_pmd_writable(struct vm_area_struct *vma, vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf) { - return handle_userfault(vmf, VM_UFFD_RWP); + struct vm_area_struct *vma = vmf->vma; + pmd_t pmd; + + if (!userfaultfd_rwp_async(vma)) + return handle_userfault(vmf, VM_UFFD_RWP); + + vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); + if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) { + spin_unlock(vmf->ptl); + return 0; + } + pmd = pmd_modify(vmf->orig_pmd, vma->vm_page_prot); + /* pmd_modify() preserves _PAGE_UFFD; drop it on resolution */ + pmd = pmd_clear_uffd(pmd); + pmd = pmd_mkyoung(pmd); + if (!pmd_write(pmd) && + vma_wants_manual_pte_write_upgrade(vma) && + can_change_pmd_writable(vma, vmf->address, pmd)) + pmd = pmd_mkwrite(pmd, vma); + set_pmd_at(vma->vm_mm, vmf->address & HPAGE_PMD_MASK, + vmf->pmd, pmd); + update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); + spin_unlock(vmf->ptl); + return 0; } /* NUMA hinting page fault entry point for trans huge pmds */ diff --git a/mm/hugetlb.c b/mm/hugetlb.c index d6ebf18568fa..dcaca9309015 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -6103,7 +6103,37 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, */ if (pte_protnone(vmf.orig_pte) && vma_is_accessible(vma) && userfaultfd_rwp(vma) && huge_pte_uffd(vmf.orig_pte)) { - return hugetlb_handle_userfault(&vmf, mapping, VM_UFFD_RWP); + spinlock_t *ptl; + pte_t pte; + + /* Sync: drop hugetlb locks before blocking in handle_userfault() */ + if (!userfaultfd_rwp_async(vma)) + return hugetlb_handle_userfault(&vmf, mapping, VM_UFFD_RWP); + + ptl = huge_pte_lock(h, mm, vmf.pte); + pte = huge_ptep_get(mm, vmf.address, vmf.pte); + if (pte_protnone(pte) && huge_pte_uffd(pte)) { + unsigned int shift = huge_page_shift(h); + + pte = huge_pte_modify(pte, vma->vm_page_prot); + pte = arch_make_huge_pte(pte, shift, vma->vm_flags); + /* huge_pte_modify() preserves _PAGE_UFFD; drop it on resolution */ + pte = huge_pte_clear_uffd(pte); + pte = pte_mkyoung(pte); + /* + * Unlike do_uffd_rwp(), do not upgrade to writable + * here. Hugetlb lacks a can_change_huge_pte_writable() + * equivalent, so a write access will take a separate + * COW fault — acceptable for the rare private hugetlb + * case. + */ + set_huge_pte_at(mm, vmf.address, vmf.pte, pte, + huge_page_size(h)); + update_mmu_cache(vma, vmf.address, vmf.pte); + } + spin_unlock(ptl); + ret = 0; + goto out_mutex; } /* diff --git a/mm/memory.c b/mm/memory.c index 6e01690fd0e4..87db93f35bcf 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -6146,8 +6146,31 @@ static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_stru static vm_fault_t do_uffd_rwp(struct vm_fault *vmf) { - pte_unmap(vmf->pte); - return handle_userfault(vmf, VM_UFFD_RWP); + pte_t pte; + + if (!userfaultfd_rwp_async(vmf->vma)) { + /* Sync mode: unmap PTE and deliver to userfaultfd handler */ + pte_unmap(vmf->pte); + return handle_userfault(vmf, VM_UFFD_RWP); + } + + spin_lock(vmf->ptl); + if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + return 0; + } + pte = pte_modify(vmf->orig_pte, vmf->vma->vm_page_prot); + /* pte_modify() preserves _PAGE_UFFD; drop it on resolution */ + pte = pte_clear_uffd(pte); + pte = pte_mkyoung(pte); + if (!pte_write(pte) && + vma_wants_manual_pte_write_upgrade(vmf->vma) && + can_change_pte_writable(vmf->vma, vmf->address, pte)) + pte = pte_mkwrite(pte, vmf->vma); + set_pte_at(vmf->vma->vm_mm, vmf->address, vmf->pte, pte); + update_mmu_cache(vmf->vma, vmf->address, vmf->pte); + pte_unmap_unlock(vmf->pte, vmf->ptl); + return 0; } static vm_fault_t do_numa_page(struct vm_fault *vmf) diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index f4a64971d9dc..52e544296bf5 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -2488,6 +2488,11 @@ static bool userfaultfd_wp_async_ctx(struct userfaultfd_ctx *ctx) return ctx && (ctx->features & UFFD_FEATURE_WP_ASYNC); } +static bool userfaultfd_rwp_async_ctx(struct userfaultfd_ctx *ctx) +{ + return ctx && (ctx->features & UFFD_FEATURE_RWP_ASYNC); +} + /* * Whether WP_UNPOPULATED is enabled on the uffd context. It is only * meaningful when userfaultfd_wp()==true on the vma and when it's @@ -4420,6 +4425,11 @@ bool userfaultfd_wp_async(struct vm_area_struct *vma) return userfaultfd_wp_async_ctx(vma->vm_userfaultfd_ctx.ctx); } +bool userfaultfd_rwp_async(struct vm_area_struct *vma) +{ + return userfaultfd_rwp_async_ctx(vma->vm_userfaultfd_ctx.ctx); +} + static inline unsigned int uffd_ctx_features(__u64 user_features) { /* @@ -4523,6 +4533,12 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx, if (features & UFFD_FEATURE_WP_ASYNC) features |= UFFD_FEATURE_WP_UNPOPULATED; + ret = -EINVAL; + /* RWP_ASYNC requires RWP */ + if ((features & UFFD_FEATURE_RWP_ASYNC) && + !(features & UFFD_FEATURE_RWP)) + goto err_out; + /* report all available features and ioctls to userland */ uffdio_api.features = UFFD_API_FEATURES; #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR @@ -4545,7 +4561,8 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx, * but not actually usable. */ if (VM_UFFD_RWP == VM_NONE || !pgtable_supports_uffd()) - uffdio_api.features &= ~UFFD_FEATURE_RWP; + uffdio_api.features &= + ~(UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC); ret = -EINVAL; if (features & ~uffdio_api.features) From 39a7c34ea5082d721c980db2064e749c61068535 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Wed, 8 Jul 2026 12:14:14 +0100 Subject: [PATCH 268/501] userfaultfd: add UFFDIO_SET_MODE for runtime sync/async toggle Add an ioctl to toggle async mode at runtime without re-registering the userfaultfd. This allows a VMM to switch between sync and async RWP modes on-the-fly -- for example, starting in async mode for working set scanning, then switching to sync mode to intercept faults during page eviction. UFFDIO_SET_MODE takes an enable/disable bitmask of UFFD_FEATURE_* flags. Only UFFD_FEATURE_RWP_ASYNC is toggleable today; the ioctl rejects any other bit with -EINVAL. Enabling RWP_ASYNC also requires RWP to have been negotiated at UFFDIO_API time, mirroring the UFFDIO_API invariant. Fault-path readers of ctx->features run under mmap_read_lock or a per-VMA lock; the RMW takes mmap_write_lock and calls vma_start_write() on every UFFD-armed VMA, so those readers are fully excluded. userfaultfd_show_fdinfo(), however, reads ctx->features without any lock, so the RMW is written as a single WRITE_ONCE and fdinfo reads it with READ_ONCE. That keeps the lockless observer from seeing a mid-RMW intermediate and removes the audit burden when new toggleable bits are added later. When switching to async, pending sync waiters are woken so they retry and auto-resolve under the new mode. Link: https://lore.kernel.org/20260708111417.173443-14-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Assisted-by: Claude:claude-opus-4-6 Reviewed-by: Mike Rapoport (Microsoft) Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: James Houghton Cc: Jonathan Corbet Cc: Liam Howlett Cc: Lorenzo Stoakes Cc: Paolo Bonzini Cc: Peter Xu Cc: Sean Christopherson Cc: SeongJae Park Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- include/uapi/linux/userfaultfd.h | 14 +++ mm/userfaultfd.c | 152 +++++++++++++++++++++++++------ 2 files changed, 137 insertions(+), 29 deletions(-) diff --git a/include/uapi/linux/userfaultfd.h b/include/uapi/linux/userfaultfd.h index c10f08f8a618..cea11aad6b54 100644 --- a/include/uapi/linux/userfaultfd.h +++ b/include/uapi/linux/userfaultfd.h @@ -49,6 +49,7 @@ #define UFFD_API_IOCTLS \ ((__u64)1 << _UFFDIO_REGISTER | \ (__u64)1 << _UFFDIO_UNREGISTER | \ + (__u64)1 << _UFFDIO_SET_MODE | \ (__u64)1 << _UFFDIO_API) #define UFFD_API_RANGE_IOCTLS \ ((__u64)1 << _UFFDIO_WAKE | \ @@ -85,6 +86,7 @@ #define _UFFDIO_CONTINUE (0x07) #define _UFFDIO_POISON (0x08) #define _UFFDIO_RWPROTECT (0x09) +#define _UFFDIO_SET_MODE (0x0A) #define _UFFDIO_API (0x3F) /* userfaultfd ioctl ids */ @@ -111,6 +113,8 @@ struct uffdio_poison) #define UFFDIO_RWPROTECT _IOWR(UFFDIO, _UFFDIO_RWPROTECT, \ struct uffdio_rwprotect) +#define UFFDIO_SET_MODE _IOW(UFFDIO, _UFFDIO_SET_MODE, \ + struct uffdio_set_mode) /* read() structure */ struct uffd_msg { @@ -406,6 +410,16 @@ struct uffdio_move { __s64 move; }; +struct uffdio_set_mode { + /* + * Toggle async mode for features at runtime. + * Supported: UFFD_FEATURE_RWP_ASYNC. + * Setting a bit in both enable and disable is invalid. + */ + __u64 enable; + __u64 disable; +}; + /* * Flags for the userfaultfd(2) system call itself. */ diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 52e544296bf5..9d0357768bcf 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -2478,19 +2478,29 @@ struct userfaultfd_wake_range { /* internal indication that UFFD_API ioctl was successfully executed */ #define UFFD_FEATURE_INITIALIZED (1u << 31) +/* + * UFFDIO_SET_MODE updates ctx->features under mmap_write_lock with + * WRITE_ONCE; readers that run outside mmap_read_lock or the per-VMA + * lock (poll/read_iter/ioctl, fdinfo) must pair with READ_ONCE. + */ +static unsigned int userfaultfd_features(struct userfaultfd_ctx *ctx) +{ + return READ_ONCE(ctx->features); +} + static bool userfaultfd_is_initialized(struct userfaultfd_ctx *ctx) { - return ctx->features & UFFD_FEATURE_INITIALIZED; + return userfaultfd_features(ctx) & UFFD_FEATURE_INITIALIZED; } static bool userfaultfd_wp_async_ctx(struct userfaultfd_ctx *ctx) { - return ctx && (ctx->features & UFFD_FEATURE_WP_ASYNC); + return ctx && (userfaultfd_features(ctx) & UFFD_FEATURE_WP_ASYNC); } static bool userfaultfd_rwp_async_ctx(struct userfaultfd_ctx *ctx) { - return ctx && (ctx->features & UFFD_FEATURE_RWP_ASYNC); + return ctx && (userfaultfd_features(ctx) & UFFD_FEATURE_RWP_ASYNC); } /* @@ -2505,7 +2515,7 @@ bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma) if (!ctx) return false; - return ctx->features & UFFD_FEATURE_WP_UNPOPULATED; + return userfaultfd_features(ctx) & UFFD_FEATURE_WP_UNPOPULATED; } static int userfaultfd_wake_function(wait_queue_entry_t *wq, unsigned mode, @@ -3742,7 +3752,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, if (uffdio_register.mode & UFFDIO_REGISTER_MODE_RWP) { if (!pgtable_supports_uffd() || VM_UFFD_RWP == VM_NONE) goto out; - if (!(ctx->features & UFFD_FEATURE_RWP)) + if (!(userfaultfd_features(ctx) & UFFD_FEATURE_RWP)) goto out; vm_flags |= VM_UFFD_RWP; } @@ -4302,6 +4312,109 @@ static int userfaultfd_rwprotect(struct userfaultfd_ctx *ctx, return ret; } +/* Subset of UFFD_API_FEATURES actually supported by this kernel/arch */ +static __u64 uffd_api_available_features(void) +{ + __u64 f = UFFD_API_FEATURES; + + if (!IS_ENABLED(CONFIG_HAVE_ARCH_USERFAULTFD_MINOR)) + f &= ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM); + if (!pgtable_supports_uffd()) + f &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP; + if (!uffd_supports_wp_marker()) + f &= ~(UFFD_FEATURE_WP_HUGETLBFS_SHMEM | + UFFD_FEATURE_WP_UNPOPULATED | + UFFD_FEATURE_WP_ASYNC); + /* + * RWP needs both PROT_NONE support and the uffd PTE bit. The + * VM_UFFD_RWP check covers compile-time unavailability; the + * pgtable_supports_uffd() check covers runtime (e.g. riscv + * without the SVRSW60T59B extension) where the PTE bit is declared + * but not actually usable. + */ + if (VM_UFFD_RWP == VM_NONE || !pgtable_supports_uffd()) + f &= ~(UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC); + return f; +} + +/* Async features that can be toggled at runtime via UFFDIO_SET_MODE */ +#define UFFD_FEATURE_TOGGLEABLE UFFD_FEATURE_RWP_ASYNC + +static int userfaultfd_set_mode(struct userfaultfd_ctx *ctx, + unsigned long arg) +{ + struct uffdio_set_mode mode; + struct mm_struct *mm = ctx->mm; + + if (copy_from_user(&mode, (void __user *)arg, sizeof(mode))) + return -EFAULT; + + /* enable and disable must not overlap */ + if (mode.enable & mode.disable) + return -EINVAL; + + /* only toggleable features that this kernel/arch actually supports */ + if ((mode.enable | mode.disable) & + ~(uffd_api_available_features() & UFFD_FEATURE_TOGGLEABLE)) + return -EINVAL; + + /* RWP_ASYNC can only be enabled on contexts that negotiated RWP */ + if ((mode.enable & UFFD_FEATURE_RWP_ASYNC) && + !(userfaultfd_features(ctx) & UFFD_FEATURE_RWP)) + return -EINVAL; + + if (!mmget_not_zero(mm)) + return -ESRCH; + + /* + * Drain in-flight faults before flipping features. mmap_write_lock() + * blocks new mmap_read_lock() callers, but per-VMA locked faults + * (lock_vma_under_rcu() + FAULT_FLAG_VMA_LOCK) that acquired before + * this point keep running. Calling vma_start_write() on each UFFD- + * armed VMA waits for those readers to drop, so no in-flight fault + * can observe the old features after mmap_write_unlock(). + */ + mmap_write_lock(mm); + { + struct vm_area_struct *vma; + VMA_ITERATOR(vmi, mm, 0); + + for_each_vma(vmi, vma) { + if (vma->vm_userfaultfd_ctx.ctx == ctx) + vma_start_write(vma); + } + } + /* + * Single WRITE_ONCE so lockless readers (fdinfo, poll/read_iter + * via userfaultfd_is_initialized(), and the userfaultfd_features() + * helper used elsewhere) can't observe a mid-RMW intermediate + * value. Hot-path readers already serialise through the mmap lock + * + vma_start_write() drain above, so their load doesn't need an + * annotation. + */ + WRITE_ONCE(ctx->features, + (ctx->features | mode.enable) & ~mode.disable); + mmap_write_unlock(mm); + + /* + * If switching to async, wake threads blocked in handle_userfault(). + * They will retry the fault and auto-resolve under the new mode. + * len=0 means wake all pending faults on this context. + */ + if (mode.enable & UFFD_FEATURE_RWP_ASYNC) { + struct userfaultfd_wake_range range = { .len = 0 }; + + spin_lock_irq(&ctx->fault_pending_wqh.lock); + __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, + &range); + __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, &range); + spin_unlock_irq(&ctx->fault_pending_wqh.lock); + } + + mmput(mm); + return 0; +} + static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg) { __s64 ret; @@ -4540,29 +4653,7 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx, goto err_out; /* report all available features and ioctls to userland */ - uffdio_api.features = UFFD_API_FEATURES; -#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR - uffdio_api.features &= - ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM); -#endif - if (!pgtable_supports_uffd()) - uffdio_api.features &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP; - - if (!uffd_supports_wp_marker()) { - uffdio_api.features &= ~UFFD_FEATURE_WP_HUGETLBFS_SHMEM; - uffdio_api.features &= ~UFFD_FEATURE_WP_UNPOPULATED; - uffdio_api.features &= ~UFFD_FEATURE_WP_ASYNC; - } - /* - * RWP needs both PROT_NONE support and the uffd-wp PTE bit. The - * VM_UFFD_RWP check covers compile-time unavailability; the - * pgtable_supports_uffd() check covers runtime (e.g. riscv - * without the SVRSW60T59B extension) where the PTE bit is declared - * but not actually usable. - */ - if (VM_UFFD_RWP == VM_NONE || !pgtable_supports_uffd()) - uffdio_api.features &= - ~(UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC); + uffdio_api.features = uffd_api_available_features(); ret = -EINVAL; if (features & ~uffdio_api.features) @@ -4632,6 +4723,9 @@ static long userfaultfd_ioctl(struct file *file, unsigned cmd, case UFFDIO_RWPROTECT: ret = userfaultfd_rwprotect(ctx, arg); break; + case UFFDIO_SET_MODE: + ret = userfaultfd_set_mode(ctx, arg); + break; } return ret; } @@ -4659,7 +4753,7 @@ static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f) * protocols: aa:... bb:... */ seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n", - pending, total, UFFD_API, ctx->features, + pending, total, UFFD_API, userfaultfd_features(ctx), UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS); } #endif From 3628215f7fc682d45cb8cdf3451f1e13ad0e4602 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Thu, 16 Apr 2026 14:33:41 +0100 Subject: [PATCH 269/501] selftests/mm: add userfaultfd RWP tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Coverage for UFFDIO_REGISTER_MODE_RWP and UFFDIO_RWPROTECT: rwp-async async mode — touch pages, verify permissions are auto-restored without a message rwp-sync sync mode — access blocks, handler resolves via UFFDIO_RWPROTECT rwp-pagemap PAGEMAP_SCAN reports still-cold pages via inverted PAGE_IS_ACCESSED rwp-mprotect RWP survives mprotect(PROT_NONE) -> mprotect(PROT_READ|PROT_WRITE) round-trip rwp-gup GUP walks through a protnone RWP PTE (pipe write/read drives the GUP path) rwp-async-toggle UFFDIO_SET_MODE flips between sync and async without re-registering rwp-close closing the uffd restores page permissions rwp-fork RWP survives fork() with EVENT_FORK; child's PTEs keep the uffd bit rwp-fork-pin RWP survives fork() on an RO-longterm-pinned anon page (forces copy_present_page()); child read auto-resolves and clears the bit, proving PAGE_NONE was in place rwp-wp-exclusive register with MODE_WP|MODE_RWP returns -EINVAL All tests run against anon, shmem, shmem-private, hugetlb, and hugetlb-private memory, except rwp-fork-pin which is anon-only — copy_present_page() is the private-anon pinned-exclusive fork path. Snapshot the RWP additions into tools/include/uapi/linux/userfaultfd.h so the selftest builds without requiring "make headers" first, matching the mechanism established by commit 580ea358af0a ("selftests/mm: fix additional build errors for selftests"). Link: https://lore.kernel.org/ak-Z9KO2mP9HMOPW@thinkstation Signed-off-by: Kiryl Shutsemau Assisted-by: Claude:claude-opus-4-6 Reviewed-by: Mike Rapoport (Microsoft) Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: James Houghton Cc: Jonathan Corbet Cc: Liam Howlett Cc: Lorenzo Stoakes Cc: Paolo Bonzini Cc: Peter Xu Cc: Sean Christopherson Cc: SeongJae Park Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- tools/include/uapi/linux/userfaultfd.h | 54 +- tools/testing/selftests/mm/uffd-unit-tests.c | 919 +++++++++++++++++++ 2 files changed, 969 insertions(+), 4 deletions(-) diff --git a/tools/include/uapi/linux/userfaultfd.h b/tools/include/uapi/linux/userfaultfd.h index 4283de22d5b6..c8e99df95219 100644 --- a/tools/include/uapi/linux/userfaultfd.h +++ b/tools/include/uapi/linux/userfaultfd.h @@ -25,7 +25,8 @@ #define UFFD_API ((__u64)0xAA) #define UFFD_API_REGISTER_MODES (UFFDIO_REGISTER_MODE_MISSING | \ UFFDIO_REGISTER_MODE_WP | \ - UFFDIO_REGISTER_MODE_MINOR) + UFFDIO_REGISTER_MODE_MINOR | \ + UFFDIO_REGISTER_MODE_RWP) #define UFFD_API_FEATURES (UFFD_FEATURE_PAGEFAULT_FLAG_WP | \ UFFD_FEATURE_EVENT_FORK | \ UFFD_FEATURE_EVENT_REMAP | \ @@ -42,10 +43,13 @@ UFFD_FEATURE_WP_UNPOPULATED | \ UFFD_FEATURE_POISON | \ UFFD_FEATURE_WP_ASYNC | \ - UFFD_FEATURE_MOVE) + UFFD_FEATURE_MOVE | \ + UFFD_FEATURE_RWP | \ + UFFD_FEATURE_RWP_ASYNC) #define UFFD_API_IOCTLS \ ((__u64)1 << _UFFDIO_REGISTER | \ (__u64)1 << _UFFDIO_UNREGISTER | \ + (__u64)1 << _UFFDIO_SET_MODE | \ (__u64)1 << _UFFDIO_API) #define UFFD_API_RANGE_IOCTLS \ ((__u64)1 << _UFFDIO_WAKE | \ @@ -54,13 +58,15 @@ (__u64)1 << _UFFDIO_MOVE | \ (__u64)1 << _UFFDIO_WRITEPROTECT | \ (__u64)1 << _UFFDIO_CONTINUE | \ - (__u64)1 << _UFFDIO_POISON) + (__u64)1 << _UFFDIO_POISON | \ + (__u64)1 << _UFFDIO_RWPROTECT) #define UFFD_API_RANGE_IOCTLS_BASIC \ ((__u64)1 << _UFFDIO_WAKE | \ (__u64)1 << _UFFDIO_COPY | \ (__u64)1 << _UFFDIO_WRITEPROTECT | \ (__u64)1 << _UFFDIO_CONTINUE | \ - (__u64)1 << _UFFDIO_POISON) + (__u64)1 << _UFFDIO_POISON | \ + (__u64)1 << _UFFDIO_RWPROTECT) /* * Valid ioctl command number range with this API is from 0x00 to @@ -79,6 +85,8 @@ #define _UFFDIO_WRITEPROTECT (0x06) #define _UFFDIO_CONTINUE (0x07) #define _UFFDIO_POISON (0x08) +#define _UFFDIO_RWPROTECT (0x09) +#define _UFFDIO_SET_MODE (0x0A) #define _UFFDIO_API (0x3F) /* userfaultfd ioctl ids */ @@ -103,6 +111,10 @@ struct uffdio_continue) #define UFFDIO_POISON _IOWR(UFFDIO, _UFFDIO_POISON, \ struct uffdio_poison) +#define UFFDIO_RWPROTECT _IOWR(UFFDIO, _UFFDIO_RWPROTECT, \ + struct uffdio_rwprotect) +#define UFFDIO_SET_MODE _IOW(UFFDIO, _UFFDIO_SET_MODE, \ + struct uffdio_set_mode) /* read() structure */ struct uffd_msg { @@ -158,6 +170,7 @@ struct uffd_msg { #define UFFD_PAGEFAULT_FLAG_WRITE (1<<0) /* If this was a write fault */ #define UFFD_PAGEFAULT_FLAG_WP (1<<1) /* If reason is VM_UFFD_WP */ #define UFFD_PAGEFAULT_FLAG_MINOR (1<<2) /* If reason is VM_UFFD_MINOR */ +#define UFFD_PAGEFAULT_FLAG_RWP (1<<3) /* If reason is VM_UFFD_RWP */ struct uffdio_api { /* userland asks for an API number and the features to enable */ @@ -230,6 +243,18 @@ struct uffdio_api { * * UFFD_FEATURE_MOVE indicates that the kernel supports moving an * existing page contents from userspace. + * + * UFFD_FEATURE_RWP indicates that the kernel supports + * UFFDIO_REGISTER_MODE_RWP for read-write protection tracking. + * Pages are made inaccessible via UFFDIO_RWPROTECT and faults + * are delivered when the pages are re-accessed. + * + * UFFD_FEATURE_RWP_ASYNC indicates asynchronous mode for + * UFFDIO_REGISTER_MODE_RWP. When set, faults on read-write + * protected pages are auto-resolved by the kernel (PTE + * permissions restored immediately) without delivering a message + * to the userfaultfd handler. Use PAGEMAP_SCAN with inverted + * PAGE_IS_ACCESSED to find pages that were not re-accessed. */ #define UFFD_FEATURE_PAGEFAULT_FLAG_WP (1<<0) #define UFFD_FEATURE_EVENT_FORK (1<<1) @@ -248,6 +273,8 @@ struct uffdio_api { #define UFFD_FEATURE_POISON (1<<14) #define UFFD_FEATURE_WP_ASYNC (1<<15) #define UFFD_FEATURE_MOVE (1<<16) +#define UFFD_FEATURE_RWP (1<<17) +#define UFFD_FEATURE_RWP_ASYNC (1<<18) __u64 features; __u64 ioctls; @@ -263,6 +290,7 @@ struct uffdio_register { #define UFFDIO_REGISTER_MODE_MISSING ((__u64)1<<0) #define UFFDIO_REGISTER_MODE_WP ((__u64)1<<1) #define UFFDIO_REGISTER_MODE_MINOR ((__u64)1<<2) +#define UFFDIO_REGISTER_MODE_RWP ((__u64)1<<3) __u64 mode; /* @@ -356,6 +384,14 @@ struct uffdio_poison { __s64 updated; }; +struct uffdio_rwprotect { + struct uffdio_range range; + /* !RWP means undo RWP-protection */ +#define UFFDIO_RWPROTECT_MODE_RWP ((__u64)1<<0) +#define UFFDIO_RWPROTECT_MODE_DONTWAKE ((__u64)1<<1) + __u64 mode; +}; + struct uffdio_move { __u64 dst; __u64 src; @@ -374,6 +410,16 @@ struct uffdio_move { __s64 move; }; +struct uffdio_set_mode { + /* + * Toggle async mode for features at runtime. + * Supported: UFFD_FEATURE_RWP_ASYNC. + * Setting a bit in both enable and disable is invalid. + */ + __u64 enable; + __u64 disable; +}; + /* * Flags for the userfaultfd(2) system call itself. */ diff --git a/tools/testing/selftests/mm/uffd-unit-tests.c b/tools/testing/selftests/mm/uffd-unit-tests.c index a6c14109e818..ef9b3956bdcf 100644 --- a/tools/testing/selftests/mm/uffd-unit-tests.c +++ b/tools/testing/selftests/mm/uffd-unit-tests.c @@ -7,6 +7,8 @@ #include "uffd-common.h" +#include +#include #include "../../../../mm/gup_test.h" #ifdef __NR_userfaultfd @@ -109,6 +111,10 @@ static void uffd_test_skip(const char *message) static void test_uffd_api(bool use_dev) { + const uint64_t expected_ioctls = + BIT_ULL(_UFFDIO_REGISTER) | + BIT_ULL(_UFFDIO_UNREGISTER) | + BIT_ULL(_UFFDIO_API); struct uffdio_api uffdio_api; int uffd; @@ -148,6 +154,15 @@ static void test_uffd_api(bool use_dev) goto out; } + /* Verify returned fd-level ioctls bitmask */ + if ((uffdio_api.ioctls & expected_ioctls) != expected_ioctls) { + uffd_test_fail("UFFDIO_API missing expected ioctls: " + "got=0x%"PRIx64", expected=0x%"PRIx64, + (uint64_t)uffdio_api.ioctls, + expected_ioctls); + goto out; + } + /* Test double requests of UFFDIO_API with a random feature set */ uffdio_api.features = BIT_ULL(0); if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) { @@ -602,6 +617,814 @@ void uffd_minor_collapse_test(uffd_global_test_opts_t *gopts, uffd_test_args_t * uffd_minor_test_common(gopts, true, false); } +static int uffd_register_rwp(int uffd, void *addr, uint64_t len) +{ + struct uffdio_register reg = { + .range = { .start = (unsigned long)addr, .len = len }, + .mode = UFFDIO_REGISTER_MODE_RWP, + }; + + if (ioctl(uffd, UFFDIO_REGISTER, ®) == -1) + return -errno; + return 0; +} + +static void rwprotect_range(int uffd, __u64 start, __u64 len, bool protect) +{ + struct uffdio_rwprotect rwp = { + .range = { .start = start, .len = len }, + .mode = protect ? UFFDIO_RWPROTECT_MODE_RWP : 0, + }; + + if (ioctl(uffd, UFFDIO_RWPROTECT, &rwp)) + err("UFFDIO_RWPROTECT failed"); +} + +static void set_async_mode(int uffd, bool enable) +{ + struct uffdio_set_mode mode = { }; + + if (enable) + mode.enable = UFFD_FEATURE_RWP_ASYNC; + else + mode.disable = UFFD_FEATURE_RWP_ASYNC; + + if (ioctl(uffd, UFFDIO_SET_MODE, &mode)) + err("UFFDIO_SET_MODE failed"); +} + +/* + * Test async RWP faults on anonymous memory. + * Populate pages, register MODE_RWP with RWP_ASYNC, + * RW-protect, re-access, verify content preserved and no faults delivered. + */ +static void uffd_rwp_async_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + unsigned long p; + + /* Populate all pages with known content */ + for (p = 0; p < nr_pages; p++) + memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size); + + /* Register MODE_RWP */ + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, + nr_pages * page_size)) + err("register failure"); + + /* RW-protect all pages (sets protnone) */ + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + nr_pages * page_size, true); + + /* Access all pages — should auto-resolve, no faults */ + for (p = 0; p < nr_pages; p++) { + unsigned char *page = (unsigned char *)gopts->area_dst + + p * page_size; + unsigned char expected = p % 255 + 1; + + if (page[0] != expected) { + uffd_test_fail("page %lu content mismatch: %u != %u", + p, page[0], expected); + return; + } + } + + uffd_test_pass(); +} + +/* + * Fault handler for RWP — unprotect the page via UFFDIO_RWPROTECT. + */ +static void uffd_handle_rwp_fault(uffd_global_test_opts_t *gopts, + struct uffd_msg *msg, + struct uffd_args *uargs) +{ + if (!(msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_RWP)) + err("expected RWP fault, got 0x%llx", + msg->arg.pagefault.flags); + + rwprotect_range(gopts->uffd, msg->arg.pagefault.address, + gopts->page_size, false); + uargs->minor_faults++; +} + +/* + * Test sync RWP faults on anonymous memory. + * Populate pages, register MODE_RWP (sync), RW-protect, + * access from worker thread, verify fault delivered, UFFDIO_RWPROTECT resolves. + */ +static void uffd_rwp_sync_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + pthread_t uffd_mon; + struct uffd_args uargs = { }; + bool failed = false; + char c = '\0'; + unsigned long p; + + uargs.gopts = gopts; + uargs.handle_fault = uffd_handle_rwp_fault; + + /* Populate all pages */ + for (p = 0; p < nr_pages; p++) + memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size); + + /* Register MODE_RWP */ + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, + nr_pages * page_size)) + err("register failure"); + + /* RW-protect all pages */ + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + nr_pages * page_size, true); + + /* Start fault handler thread */ + if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &uargs)) + err("uffd_poll_thread create"); + + /* Access all pages — triggers sync RWP faults, handler unprotects */ + for (p = 0; p < nr_pages; p++) { + unsigned char *page = (unsigned char *)gopts->area_dst + + p * page_size; + + if (page[0] != (p % 255 + 1)) { + uffd_test_fail("page %lu content mismatch", p); + failed = true; + goto out; + } + } + +out: + /* + * Stop the handler before reading minor_faults: the last fault + * resolution rwprotect_range()s before incrementing the counter, + * so the main thread can race ahead of the increment. + */ + if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) + err("pipe write"); + if (pthread_join(uffd_mon, NULL)) + err("join() failed"); + + if (failed) + return; + if (uargs.minor_faults == 0) + uffd_test_fail("expected RWP faults, got 0"); + else + uffd_test_pass(); +} + +/* + * Test PAGEMAP_SCAN working-set discovery via the "hot" (accessed) scan. + * + * The working-set primitive is to find pages that were accessed: scan for + * PAGE_IS_ACCESSED, which is set once an access clears the protnone+uffd + * marker. A VMM treats every access fault as "hot" (RWP here; MINOR/MISSING + * for non-resident pages) and reclaims the rest from the backing file. + * + * We deliberately do NOT use an inverted "cold" scan: that only sees + * VMA-resident ptes, so for a file mapping it misses cached-but-unmapped (and + * never-faulted, pre-populated) pages, which are pte_none and thus invisible. + * Hot tracking + file-level reclaim covers them; a cold pte scan cannot. + */ +static void uffd_rwp_pagemap_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + unsigned long p; + struct page_region regions[16]; + struct pm_scan_arg pm_arg; + int pagemap_fd; + long ret; + + /* Need at least 4 pages */ + if (nr_pages < 4) { + uffd_test_skip("need at least 4 pages"); + return; + } + + /* Populate all pages */ + for (p = 0; p < nr_pages; p++) + memset(gopts->area_dst + p * page_size, 0xab, page_size); + + /* Register and RW-protect */ + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, + nr_pages * page_size)) + err("register failure"); + + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + nr_pages * page_size, true); + + /* Touch first half of pages to re-activate them (async auto-resolve) */ + for (p = 0; p < nr_pages / 2; p++) { + volatile char *page = gopts->area_dst + p * page_size; + (void)*page; + } + + uint64_t start = (uint64_t)gopts->area_dst; + uint64_t boundary = start + (nr_pages / 2) * page_size; + uint64_t end = start + nr_pages * page_size; + + pagemap_fd = open("/proc/self/pagemap", O_RDONLY); + if (pagemap_fd < 0) + err("open pagemap"); + + /* + * Hot scan: report the pages that were accessed. PAGE_IS_ACCESSED is + * set once the protnone+uffd marker is cleared (by the access, async + * auto-resolve here). The touched first half must come back as exactly + * one hot region [start, boundary); the untouched second half must not + * appear. + */ + memset(&pm_arg, 0, sizeof(pm_arg)); + pm_arg.size = sizeof(pm_arg); + pm_arg.start = start; + pm_arg.end = end; + pm_arg.vec = (uint64_t)regions; + pm_arg.vec_len = ARRAY_SIZE(regions); + pm_arg.category_mask = PAGE_IS_ACCESSED; + pm_arg.return_mask = PAGE_IS_ACCESSED; + + ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &pm_arg); + close(pagemap_fd); + + if (ret < 0) { + uffd_test_fail("PAGEMAP_SCAN failed: %s", strerror(errno)); + return; + } + + if (ret != 1 || regions[0].start != start || + regions[0].end != boundary) { + uffd_test_fail("hot set wrong: got %ld regions [0x%lx,0x%lx), expected 1 [0x%lx,0x%lx)", + ret, (unsigned long)regions[0].start, + (unsigned long)regions[0].end, + (unsigned long)start, (unsigned long)boundary); + return; + } + + uffd_test_pass(); +} + +/* + * Test that RWP protection survives a mprotect(PROT_NONE) -> + * mprotect(PROT_READ|PROT_WRITE) round-trip. The uffd-wp bit on a + * VM_UFFD_RWP VMA must continue to carry PROT_NONE semantics after + * mprotect() changes the base protection; otherwise accesses would + * silently succeed and the pagemap bit would stick without a fault + * ever clearing it. + */ +static void uffd_rwp_mprotect_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + unsigned long p; + struct page_region regions[16]; + struct pm_scan_arg pm_arg; + int pagemap_fd; + uint64_t value; + long ret; + + /* Populate all pages */ + for (p = 0; p < nr_pages; p++) + memset(gopts->area_dst + p * page_size, 0xab, page_size); + + /* Register and RW-protect the whole range */ + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, + nr_pages * page_size)) + err("register failure"); + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + nr_pages * page_size, true); + + /* Round-trip mprotect(): PROT_NONE -> PROT_READ|PROT_WRITE */ + if (mprotect(gopts->area_dst, nr_pages * page_size, PROT_NONE)) + err("mprotect() PROT_NONE"); + if (mprotect(gopts->area_dst, nr_pages * page_size, + PROT_READ | PROT_WRITE)) + err("mprotect() PROT_READ|PROT_WRITE"); + + /* + * The marker must survive the round-trip; if mprotect() dropped it, + * the touches below would not fault and the scan would pass + * vacuously. + */ + pagemap_fd = pagemap_open(); + value = pagemap_get_entry(pagemap_fd, gopts->area_dst); + close(pagemap_fd); + if (!(value & PM_UFFD_WP)) { + uffd_test_fail("RWP marker lost across mprotect()"); + return; + } + + /* Touch every page. Async RWP must auto-resolve each fault. */ + for (p = 0; p < nr_pages; p++) { + volatile char *page = gopts->area_dst + p * page_size; + (void)*page; + } + + /* + * After touching, no page should remain RW-protected. A stuck + * uffd-wp bit would mean mprotect() silently dropped PROT_NONE and + * the access never faulted. + */ + pagemap_fd = open("/proc/self/pagemap", O_RDONLY); + if (pagemap_fd < 0) + err("open pagemap"); + + memset(&pm_arg, 0, sizeof(pm_arg)); + pm_arg.size = sizeof(pm_arg); + pm_arg.start = (uint64_t)gopts->area_dst; + pm_arg.end = (uint64_t)gopts->area_dst + nr_pages * page_size; + pm_arg.vec = (uint64_t)regions; + pm_arg.vec_len = ARRAY_SIZE(regions); + pm_arg.category_mask = PAGE_IS_ACCESSED; + pm_arg.category_inverted = PAGE_IS_ACCESSED; + pm_arg.return_mask = PAGE_IS_ACCESSED; + + ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &pm_arg); + close(pagemap_fd); + + if (ret < 0) { + uffd_test_fail("PAGEMAP_SCAN failed: %s", strerror(errno)); + return; + } + if (ret != 0) { + uffd_test_fail("expected no cold pages after mprotect()+touch, got %ld regions", + ret); + return; + } + + uffd_test_pass(); +} + +/* + * Test that GUP resolves through protnone PTEs (async mode). + * vmsplice() into a pipe pins user pages via get_user_pages_fast() -- + * unlike write(), which goes through copy_from_user() and ordinary + * hardware page faults -- so it exercises gup_can_follow_protnone() on + * the RW-protected PTE. In async mode the kernel auto-restores + * permissions and GUP returns the page. + */ +static void uffd_rwp_gup_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + struct iovec iov; + char buf; + int pipefd[2]; + + /* Populate first page with known content */ + memset(gopts->area_dst, 0xCD, gopts->page_size); + + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, gopts->page_size)) + err("register failure"); + + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + gopts->page_size, true); + + if (pipe(pipefd)) + err("pipe"); + + /* + * One byte's worth of iov is enough to GUP the containing page and + * keeps the pipe transfer well under any pipe-capacity limit even on + * hugetlb-backed runs. + */ + iov.iov_base = gopts->area_dst; + iov.iov_len = 1; + if (vmsplice(pipefd[1], &iov, 1, 0) != 1) { + uffd_test_fail("vmsplice from RW-protected page failed: %s", + strerror(errno)); + goto out; + } + + if (read(pipefd[0], &buf, 1) != 1) { + uffd_test_fail("read from pipe failed"); + goto out; + } + + if (buf != (char)0xCD) { + uffd_test_fail("content mismatch: got 0x%02x, expected 0xCD", + (unsigned char)buf); + goto out; + } + + uffd_test_pass(); +out: + close(pipefd[0]); + close(pipefd[1]); +} + +/* + * Test runtime toggle between async and sync modes. + * Start in async mode (detection), flip to sync (eviction), verify faults + * block, resolve them, flip back to async. + */ +static void uffd_rwp_async_toggle_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + struct uffd_args uargs = { }; + pthread_t uffd_mon; + char c = '\0'; + unsigned long p; + + uargs.gopts = gopts; + uargs.handle_fault = uffd_handle_rwp_fault; + + /* Populate */ + for (p = 0; p < nr_pages; p++) + memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size); + + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, + nr_pages * page_size)) + err("register failure"); + + /* Phase 1: async detection — RW-protect, access first half */ + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + nr_pages * page_size, true); + + for (p = 0; p < nr_pages / 2; p++) { + volatile char *page = gopts->area_dst + p * page_size; + (void)*page; /* auto-resolves in async mode */ + } + + /* Phase 2: flip to sync for eviction */ + set_async_mode(gopts->uffd, false); + + /* Start handler — will receive faults for cold pages */ + if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &uargs)) + err("uffd_poll_thread create"); + + /* Access second half (cold pages) — should trigger sync faults */ + for (p = nr_pages / 2; p < nr_pages; p++) { + unsigned char *page = (unsigned char *)gopts->area_dst + + p * page_size; + if (page[0] != (p % 255 + 1)) { + uffd_test_fail("page %lu content mismatch", p); + goto out; + } + } + + /* + * Stop the handler before reading minor_faults: the last fault + * resolution rwprotect_range()s before incrementing the counter, + * so the main thread can race ahead of the increment. Stopping + * here also makes Phase 3 a clean async-only test -- with the + * handler still running it would silently resolve any sync fault + * the kernel erroneously delivers, masking a regression. + */ + if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) + err("pipe write"); + if (pthread_join(uffd_mon, NULL)) + err("join() failed"); + + if (uargs.minor_faults == 0) { + uffd_test_fail("expected sync faults, got 0"); + return; + } + + /* Phase 3: flip back to async */ + set_async_mode(gopts->uffd, true); + + /* RW-protect and access again — should auto-resolve */ + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + nr_pages * page_size, true); + + for (p = 0; p < nr_pages; p++) { + volatile char *page = gopts->area_dst + p * page_size; + (void)*page; + } + + uffd_test_pass(); + return; +out: + if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) + err("pipe write"); + if (pthread_join(uffd_mon, NULL)) + err("join() failed"); +} + +/* + * Test that RW-protected pages become accessible after closing uffd. + */ +static void uffd_rwp_close_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + unsigned long p; + + /* Populate */ + for (p = 0; p < nr_pages; p++) + memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size); + + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, + nr_pages * page_size)) + err("register failure"); + + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + nr_pages * page_size, true); + + /* Close uffd — should restore protnone PTEs */ + close(gopts->uffd); + gopts->uffd = -1; + + /* All pages should be accessible with original content */ + for (p = 0; p < nr_pages; p++) { + unsigned char *page = (unsigned char *)gopts->area_dst + + p * page_size; + unsigned char expected = p % 255 + 1; + + if (page[0] != expected) { + uffd_test_fail("page %lu not accessible after close", p); + return; + } + } + + uffd_test_pass(); +} + +/* + * Test that RWP protection is preserved across fork() when + * UFFD_FEATURE_EVENT_FORK is enabled. Without preservation, the child's + * PTEs would lose the uffd-wp marker and RWP-protected accesses would + * silently fall through to do_numa_page(). + */ +static void uffd_rwp_fork_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + int pagemap_fd; + uint64_t value; + + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, + nr_pages * page_size)) + err("register failed"); + + /* Populate + RWP-protect */ + *gopts->area_dst = 1; + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + page_size, true); + + /* Parent: verify uffd-wp bit is set before fork */ + pagemap_fd = pagemap_open(); + value = pagemap_get_entry(pagemap_fd, gopts->area_dst); + pagemap_check_wp(value, true); + + /* + * Fork with EVENT_FORK: child inherits VM_UFFD_RWP. Child reads + * its own pagemap and must still see the uffd-wp bit set. + */ + if (pagemap_test_fork(gopts, true, false)) { + uffd_test_fail("RWP marker lost in child after fork"); + goto out; + } + + uffd_test_pass(); +out: + close(pagemap_fd); +} + +/* + * Test that RWP protection on a pinned anon page is preserved across fork(). + * Pinning forces copy_present_page() in the child path, which must restore + * PAGE_NONE on top of the uffd bit. Using async mode, a read in the child + * auto-resolves if — and only if — the PTE was actually protnone+uffd; the + * cleared uffd bit afterward proves the fault path ran. + */ +static void uffd_rwp_fork_pin_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long page_size = gopts->page_size; + fork_event_args fevent_args = { .gopts = gopts, .child_uffd = -1 }; + pin_args pin_args = {}; + int pagemap_fd, status; + pthread_t fevent_thread; + uint64_t value; + pid_t child; + + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, page_size)) + err("register failed"); + + /* Populate. */ + *gopts->area_dst = 1; + + /* RO-longterm pin so fork() takes copy_present_page() for this PTE. */ + if (pin_pages(&pin_args, gopts->area_dst, page_size)) { + uffd_test_skip("Possibly CONFIG_GUP_TEST missing or unprivileged"); + uffd_unregister(gopts->uffd, gopts->area_dst, page_size); + return; + } + + /* RWP-protect: PTE is now PAGE_NONE + uffd bit. */ + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, page_size, true); + + pagemap_fd = pagemap_open(); + value = pagemap_get_entry(pagemap_fd, gopts->area_dst); + pagemap_check_wp(value, true); + + /* + * UFFD_FEATURE_EVENT_FORK is required so the child inherits + * VM_UFFD_RWP and the marker; without it dup_userfaultfd() resets + * the child VMA and the test would pass for the wrong reason. + * dup_userfaultfd() blocks until the EVENT_FORK message is consumed, + * so spawn a reader before the fork(). + */ + gopts->ready_for_fork = false; + if (pthread_create(&fevent_thread, NULL, fork_event_consumer, + &fevent_args)) + err("pthread_create() for fork event consumer"); + while (!gopts->ready_for_fork) + ; /* Wait for consumer to start polling. */ + + child = fork(); + if (child < 0) + err("fork"); + if (child == 0) { + volatile char c; + int cfd; + + /* + * Precondition: the child must have inherited the marker. + * If copy_present_page() dropped it together with PAGE_NONE, + * the read below would succeed without the fault path and + * the after-read check would pass for the wrong reason. + */ + cfd = pagemap_open(); + value = pagemap_get_entry(cfd, gopts->area_dst); + if (!(value & PM_UFFD_WP)) { + close(cfd); + _exit(2); + } + + /* + * Read the pinned page. Only reaches the fault path if the + * child PTE is protnone + uffd; async mode auto-resolves and + * clears the uffd bit. If copy_present_page() dropped + * PAGE_NONE, the read would silently succeed and the bit + * would still be set. + */ + c = *(volatile char *)gopts->area_dst; + (void)c; + + value = pagemap_get_entry(cfd, gopts->area_dst); + close(cfd); + _exit((value & PM_UFFD_WP) ? 1 : 0); + } + if (waitpid(child, &status, 0) < 0) + err("waitpid"); + if (pthread_join(fevent_thread, NULL)) + err("pthread_join() for fork event consumer"); + if (fevent_args.child_uffd >= 0) + close(fevent_args.child_uffd); + + unpin_pages(&pin_args); + close(pagemap_fd); + if (uffd_unregister(gopts->uffd, gopts->area_dst, page_size)) + err("unregister failed"); + + if (WIFEXITED(status) && WEXITSTATUS(status) == 2) { + uffd_test_fail("RWP marker not inherited by child"); + return; + } + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + uffd_test_fail("RWP not enforced in child after pinned fork"); + return; + } + + uffd_test_pass(); +} + +/* + * A non-exclusive (forked, COW-shared) anon page that is RWP-protected and + * then swapped out must keep tracking across swap-in. On the write that + * swaps it back in, do_swap_page() restores PAGE_NONE and the access retries + * through the RWP fault path, instead of being COWed straight to a fresh + * accessible page -- which would silently drop the marker for a non-exclusive + * folio. Sync mode lets us observe the fault directly: with the bug, the + * write COWs without delivering any RWP fault. + * + * Needs a swap device; skipped if MADV_PAGEOUT cannot evict the page. + */ +static void uffd_rwp_swap_cow_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long page_size = gopts->page_size; + struct uffd_args uargs = { }; + int pagemap_fd, go[2], i; + pthread_t uffd_mon; + char c = '\0'; + pid_t child; + + uargs.gopts = gopts; + uargs.handle_fault = uffd_handle_rwp_fault; + + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, page_size)) + err("register failed"); + + /* Populate one page (exclusive at this point). */ + *gopts->area_dst = 0x11; + + /* RWP-protect: PTE becomes PAGE_NONE + uffd bit (still exclusive). */ + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, page_size, true); + + /* + * Swap the page out while it is still exclusive: a shared (forked) + * folio does not get reclaimed by MADV_PAGEOUT. Retry, since a hot + * page may just be rotated on the first reclaim pass. + */ + pagemap_fd = pagemap_open(); + for (i = 0; i < 100; i++) { + if (madvise(gopts->area_dst, page_size, MADV_PAGEOUT)) + err("MADV_PAGEOUT"); + if (pagemap_is_swapped(pagemap_fd, gopts->area_dst)) + break; + usleep(10000); + } + if (!pagemap_is_swapped(pagemap_fd, gopts->area_dst)) { + uffd_test_skip("MADV_PAGEOUT did not swap the page; is swap enabled?"); + close(pagemap_fd); + uffd_unregister(gopts->uffd, gopts->area_dst, page_size); + return; + } + + /* + * fork() now: the child duplicates the swap entry, so the slot becomes + * non-exclusive. The child parks (keeping the reference) until the + * parent has faulted the page back in. + */ + if (pipe(go)) + err("pipe"); + child = fork(); + if (child < 0) + err("fork"); + if (child == 0) { + close(go[1]); + read(go[0], &c, 1); + _exit(0); + } + close(go[0]); + + if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &uargs)) + err("uffd_poll_thread create"); + + /* + * Write the page: swaps it back in (do_swap_page) on a non-exclusive + * folio with FAULT_FLAG_WRITE. The marker must survive and deliver an + * RWP fault rather than COW silently. + */ + *gopts->area_dst = 0x22; + + if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) + err("pipe write"); + if (pthread_join(uffd_mon, NULL)) + err("join failed"); + + if (uargs.minor_faults == 0) + uffd_test_fail("no RWP fault on swapped-in non-exclusive page"); + else + uffd_test_pass(); + + close(pagemap_fd); + if (write(go[1], &c, 1) != 1) + err("child release"); + close(go[1]); + waitpid(child, NULL, 0); +} + +/* + * WP and RWP share the uffd-wp PTE bit and cannot coexist in the same VMA. + * Registration requesting both modes must be rejected. + */ +static void uffd_rwp_wp_exclusive_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + struct uffdio_register reg = { }; + + reg.range.start = (unsigned long)gopts->area_dst; + reg.range.len = nr_pages * page_size; + reg.mode = UFFDIO_REGISTER_MODE_WP | UFFDIO_REGISTER_MODE_RWP; + + if (ioctl(gopts->uffd, UFFDIO_REGISTER, ®) == 0) { + uffd_test_fail("register with WP|RWP unexpectedly succeeded"); + return; + } + if (errno != EINVAL) { + uffd_test_fail("register with WP|RWP: expected EINVAL, got %d", + errno); + return; + } + uffd_test_pass(); +} + static sigjmp_buf jbuf, *sigbuf; static void sighndl(int sig, siginfo_t *siginfo, void *ptr) @@ -1604,6 +2427,83 @@ uffd_test_case_t uffd_tests[] = { /* We can't test MADV_COLLAPSE, so try our luck */ .uffd_feature_required = UFFD_FEATURE_MINOR_SHMEM, }, + { + .name = "rwp-async", + .uffd_fn = uffd_rwp_async_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = + UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC, + }, + { + .name = "rwp-sync", + .uffd_fn = uffd_rwp_sync_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = UFFD_FEATURE_RWP, + }, + { + .name = "rwp-pagemap", + .uffd_fn = uffd_rwp_pagemap_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = + UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC, + }, + { + .name = "rwp-mprotect", + .uffd_fn = uffd_rwp_mprotect_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = + UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC, + }, + { + .name = "rwp-gup", + .uffd_fn = uffd_rwp_gup_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = + UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC, + }, + { + .name = "rwp-async-toggle", + .uffd_fn = uffd_rwp_async_toggle_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = + UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC, + }, + { + .name = "rwp-close", + .uffd_fn = uffd_rwp_close_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = UFFD_FEATURE_RWP, + }, + { + .name = "rwp-fork", + .uffd_fn = uffd_rwp_fork_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = + UFFD_FEATURE_RWP | UFFD_FEATURE_EVENT_FORK, + }, + { + .name = "rwp-fork-pin", + .uffd_fn = uffd_rwp_fork_pin_test, + .mem_targets = MEM_ANON, + .uffd_feature_required = + UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC | + UFFD_FEATURE_EVENT_FORK, + }, + { + .name = "rwp-swap-cow", + .uffd_fn = uffd_rwp_swap_cow_test, + .mem_targets = MEM_ANON, + .uffd_feature_required = UFFD_FEATURE_RWP, + }, + { + .name = "rwp-wp-exclusive", + .uffd_fn = uffd_rwp_wp_exclusive_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = + UFFD_FEATURE_RWP | + UFFD_FEATURE_PAGEFAULT_FLAG_WP | + UFFD_FEATURE_WP_HUGETLBFS_SHMEM, + }, { .name = "sigbus", .uffd_fn = uffd_sigbus_test, @@ -1814,6 +2714,25 @@ int main(int argc, char *argv[]) uffd_test_skip(errmsg); continue; } + /* + * RWP tracks protection on ptes; a THP-backed shmem/anon + * range (e.g. shmem_enabled=always) would split on + * rwprotect and change behaviour under the test. Keep + * such ranges off THP. hugetlb is huge by definition and + * rejects MADV_NOHUGEPAGE, so skip it. + */ + if ((test->uffd_feature_required & UFFD_FEATURE_RWP) && + !(mem_type->mem_flag & (MEM_HUGETLB | MEM_HUGETLB_PRIVATE))) { + unsigned long len = gopts.nr_pages * gopts.page_size; + + /* + * EINVAL means CONFIG_TRANSPARENT_HUGEPAGE=n: + * nothing to opt out of. + */ + if (madvise(gopts.area_dst, len, MADV_NOHUGEPAGE) && + errno != EINVAL) + err("madvise(MADV_NOHUGEPAGE)"); + } test->uffd_fn(&gopts, &args); uffd_test_ctx_clear(&gopts); } From a137d9f8f51484d1b6268260610b7e64e897b27e Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Wed, 8 Jul 2026 12:14:16 +0100 Subject: [PATCH 270/501] Documentation/userfaultfd: document RWP working set tracking Add an admin-guide section covering UFFDIO_REGISTER_MODE_RWP: - sync and async fault models; - UFFDIO_RWPROTECT semantics; - UFFD_FEATURE_RWP_ASYNC; - UFFDIO_SET_MODE runtime mode flips. It also covers typical VMM working-set-tracking workflow from detection loop through sync-mode eviction and back to async. Link: https://lore.kernel.org/20260708111417.173443-16-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau Assisted-by: Claude:claude-opus-4-6 Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: James Houghton Cc: Jonathan Corbet Cc: Liam Howlett Cc: Lorenzo Stoakes Cc: Mike Rapoport (Microsoft) Cc: Paolo Bonzini Cc: Peter Xu Cc: Sean Christopherson Cc: SeongJae Park Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Zi Yan Cc: kernel test robot Signed-off-by: Andrew Morton --- Documentation/admin-guide/mm/userfaultfd.rst | 259 ++++++++++++++++++- 1 file changed, 253 insertions(+), 6 deletions(-) diff --git a/Documentation/admin-guide/mm/userfaultfd.rst b/Documentation/admin-guide/mm/userfaultfd.rst index 1e533639fd50..783d969f0e28 100644 --- a/Documentation/admin-guide/mm/userfaultfd.rst +++ b/Documentation/admin-guide/mm/userfaultfd.rst @@ -275,16 +275,16 @@ tracking and it can be different in a few ways: - Dirty information will not get lost if the pte was zapped due to various reasons (e.g. during split of a shmem transparent huge page). - - Due to a reverted meaning of soft-dirty (page clean when uffd-wp bit - set; dirty when uffd-wp bit cleared), it has different semantics on - some of the memory operations. For example: ``MADV_DONTNEED`` on + - Due to a reverted meaning of soft-dirty (page clean when the uffd bit + is set; dirty when the uffd bit is cleared), it has different semantics + on some of the memory operations. For example: ``MADV_DONTNEED`` on anonymous (or ``MADV_REMOVE`` on a file mapping) will be treated as - dirtying of memory by dropping uffd-wp bit during the procedure. + dirtying of memory by dropping the uffd bit during the procedure. The user app can collect the "written/dirty" status by looking up the -uffd-wp bit for the pages being interested in /proc/pagemap. +uffd bit for the pages being interested in /proc/pagemap. -The page will not be under track of uffd-wp async mode until the page is +The page will not be under track of userfaultfd-wp async mode until the page is explicitly write-protected by ``ioctl(UFFDIO_WRITEPROTECT)`` with the mode flag ``UFFDIO_WRITEPROTECT_MODE_WP`` set. Trying to resolve a page fault that was tracked by async mode userfaultfd-wp is invalid. @@ -307,6 +307,253 @@ transparent to the guest, we want that same address range to act as if it was still poisoned, even though it's on a new physical host which ostensibly doesn't have a memory error in the exact same spot. +Read-Write Protection +--------------------- + +``UFFDIO_REGISTER_MODE_RWP`` enables read-write protection tracking on a +memory range. It is similar to (but faster than) ``mprotect(PROT_NONE)`` +combined with a signal handler; unlike ``mprotect(PROT_NONE)``, RWP only +traps accesses to *present* PTEs, so accesses to unpopulated addresses in a +protected range fall through to the normal missing-page path. It uses the +PROT_NONE hinting mechanism (same as NUMA balancing) to make pages +inaccessible while keeping them resident in memory. Works on anonymous, +shmem, and hugetlbfs memory. + +RWP is designed for VM memory managers that need to track the working set +of guest memory for cold page eviction to tiered or remote storage. + +**Setup:** + +1. Open a userfaultfd and enable ``UFFD_FEATURE_RWP`` via ``UFFDIO_API``. + Optionally request ``UFFD_FEATURE_RWP_ASYNC`` as well — it requires + ``UFFD_FEATURE_RWP`` to be set in the same ``UFFDIO_API`` call. + +2. Register the guest memory range with ``UFFDIO_REGISTER_MODE_RWP`` + (and ``UFFDIO_REGISTER_MODE_MISSING`` if evicted pages will need to be + fetched back from storage). + +**Feature availability:** + +RWP is built on top of two kernel primitives: a spare PTE bit owned by +userfaultfd (``CONFIG_HAVE_ARCH_USERFAULTFD_WP``) and architecture support +for present-but-inaccessible PTEs (``CONFIG_ARCH_HAS_PTE_PROTNONE``). When both +are available on a 64-bit kernel, the build selects +``CONFIG_USERFAULTFD_RWP=y`` and the ``VM_UFFD_RWP`` VMA flag becomes +available. + +``UFFD_FEATURE_RWP`` and ``UFFD_FEATURE_RWP_ASYNC`` are unavailable when +the running kernel or architecture does not support them — for example +32-bit kernels (where ``VM_UFFD_RWP`` is unavailable), kernels built +without ``CONFIG_USERFAULTFD_RWP``, and architectures whose ptes cannot +carry the uffd bit at runtime (e.g. riscv without the ``SVRSW60T59B`` +extension). Requesting an unsupported feature in +``uffdio_api.features`` makes ``UFFDIO_API`` fail with ``EINVAL`` and +leaves the userfaultfd context uninitialized; the structure is returned +zeroed, so the error path cannot be used to discover what the kernel +supports. The recommended probe sequence is therefore to open a +throwaway userfaultfd, call ``UFFDIO_API`` once with ``features = 0``, +inspect the returned bitmask, close that fd, then open the real one +and call ``UFFDIO_API`` again with only the supported features set. + +**Protecting and Unprotecting:** + +Use ``UFFDIO_RWPROTECT`` to protect or unprotect a range, mirroring the +``UFFDIO_WRITEPROTECT`` interface:: + + struct uffdio_rwprotect rwp = { + .range = { .start = addr, .len = len }, + .mode = UFFDIO_RWPROTECT_MODE_RWP, /* protect */ + }; + ioctl(uffd, UFFDIO_RWPROTECT, &rwp); + +Setting ``UFFDIO_RWPROTECT_MODE_RWP`` sets PROT_NONE on present PTEs in the +range. Pages stay resident and their physical frames are preserved — only +access permissions are removed. + +Clearing ``UFFDIO_RWPROTECT_MODE_RWP`` restores normal VMA permissions and +wakes any faulting threads (unless ``UFFDIO_RWPROTECT_MODE_DONTWAKE`` is set). + +**Scope of protection:** + +RWP protection is a property of *present* PTEs. ``UFFDIO_RWPROTECT`` only +affects entries that are already populated. Unpopulated addresses within +the range remain unpopulated; when first accessed they fault through the +normal missing path (``do_anonymous_page()``, ``do_swap_page()``, +``finish_fault()``) and the resulting PTE is not RWP-protected. To observe +the population itself, co-register the range with +``UFFDIO_REGISTER_MODE_MISSING``. + +Protection is preserved across page reclaim: a page swapped out while +RWP-protected carries the marker on its swap entry, and swap-in restores +the PROT_NONE state so the first access after swap-in still faults. The +same applies to pages temporarily replaced by migration entries. + +Operations that drop the PTE entirely — ``MADV_DONTNEED`` on anonymous +memory, hole-punch on shmem, truncation of a file mapping — also drop the +RWP marker: the next access re-populates the range without protection. +Unlike WP (which persists via ``PTE_MARKER_UFFD_WP``), there is no +persistent RWP marker today. The user needs to re-arm the range with +``UFFDIO_RWPROTECT`` after any operation that explicitly frees PTEs. + +**Fault Handling:** + +When a protected page is accessed: + +- **Sync mode** (default): The faulting thread blocks and a + ``UFFD_PAGEFAULT_FLAG_RWP`` message is delivered to the userfaultfd + handler. The handler resolves the fault with ``UFFDIO_RWPROTECT`` + (clearing ``MODE_RWP``), which restores the PTE permissions and wakes + the faulting thread. + +- **Async mode** (``UFFD_FEATURE_RWP_ASYNC``): The kernel automatically + restores PTE permissions and the thread continues without blocking. No + message is delivered to the handler. + +**Runtime Mode Switching:** + +``UFFDIO_SET_MODE`` toggles ``UFFD_FEATURE_RWP_ASYNC`` at runtime, allowing +the VMM to switch between lightweight async detection and safe sync +eviction without re-registering. The toggle takes ``mmap_write_lock()`` +and calls ``vma_start_write()`` on each UFFD-armed VMA, draining +in-flight per-VMA-locked faults before the new mode takes effect. + +**Working-set detection with PAGEMAP_SCAN:** + +RWP-protected PTEs carry the uffd PTE bit; an access (and, in async mode, its +auto-resolution) clears it. ``PAGEMAP_SCAN`` reports ``PAGE_IS_ACCESSED`` once +the bit is clear on a ``VM_UFFD_RWP`` VMA, so a *non-inverted* scan reports the +pages that were touched during the interval -- the hot set:: + + struct pm_scan_arg arg = { + .size = sizeof(arg), + .start = guest_mem_start, + .end = guest_mem_end, + .vec = (uint64_t)regions, + .vec_len = regions_len, + .category_mask = PAGE_IS_ACCESSED, + .return_mask = PAGE_IS_ACCESSED, + }; + long n = ioctl(pagemap_fd, PAGEMAP_SCAN, &arg); + +The returned ``page_region`` array lists the hot ranges. ``PAGE_IS_ACCESSED`` +is set on an accessed page whether it is still present or has since been +swapped out, so the hot scan needs no ``PAGE_IS_PRESENT`` filter -- unpopulated +holes carry neither bit and are excluded on their own. + +Track the hot set and reclaim everything else from the backing file (see the +workflow below). Do **not** invert the scan to enumerate "cold" pages +directly: an inverted scan reports only the ``VM_UFFD_RWP`` PTEs that are still +protected, i.e. the resident portion of *this* VMA. For a file mapping the +working set spans the whole file -- pages that live in the page cache but are +not mapped into this VMA (a pre-populated tmpfs file, or memory populated +through another mapping) are ``pte_none`` here, never appear in the scan, and +would never be considered for eviction even though they occupy memory. Driving +eviction from "file offsets minus the hot set" avoids that blind spot; a cold +PTE scan cannot. To additionally record the *first* access to a cached but +unmapped page (e.g. pre-populated content) as hot, co-register the range with +``UFFDIO_REGISTER_MODE_MINOR``: such accesses then fault as minor faults +instead of mapping the page silently. + +**Cleanup:** + +When the userfaultfd is closed or the range is unregistered, all PROT_NONE +PTEs are automatically restored to their normal VMA permissions. This +prevents pages from becoming permanently inaccessible. + +**VMM Working Set Tracking Workflow:** + +A typical VMM lifecycle for cold page eviction to tiered storage. Two +mappings of the same shmem (or hugetlbfs) file are used: ``guest_mem`` is +the RWP-registered mapping that vCPUs access through, and ``io_mem`` is a +private mapping for VMM-side I/O. Reading ``io_mem`` does not go through +the RWP-protected PTEs of ``guest_mem``, so the VMM's own ``pwrite()`` +never traps on its own :: + + /* One-time setup */ + fd = memfd_create("guest", MFD_CLOEXEC); + ftruncate(fd, guest_size); + guest_mem = mmap(NULL, guest_size, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); /* vCPU view, RWP-registered */ + io_mem = mmap(NULL, guest_size, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); /* VMM I/O view, unprotected */ + + uffd = userfaultfd(O_CLOEXEC | O_NONBLOCK); + struct uffdio_api api = { + .api = UFFD_API, + .features = UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC, + }; + ioctl(uffd, UFFDIO_API, &api); + if (!(api.features & UFFD_FEATURE_RWP)) + /* RWP unavailable on this kernel/arch -- fall back. */ + ioctl(uffd, UFFDIO_REGISTER, &(struct uffdio_register){ + .range = { guest_mem, guest_size }, + .mode = UFFDIO_REGISTER_MODE_RWP | + UFFDIO_REGISTER_MODE_MISSING, + }); + + /* Tracking loop */ + while (vm_running) { + /* 1. Detection phase (async -- no vCPU stalls) */ + ioctl(uffd, UFFDIO_RWPROTECT, &(struct uffdio_rwprotect){ + .range = full_range, + .mode = UFFDIO_RWPROTECT_MODE_RWP }); + sleep(tracking_interval); + + /* + * 2. Switch to sync BEFORE scanning. In async mode a vCPU + * access races eviction: it would auto-resolve and mark the + * page hot just as the VMM writes it out and punches it, + * losing the update. Sync mode makes such accesses block and + * be delivered, freezing the hot snapshot for the rest of the + * iteration. + */ + ioctl(uffd, UFFDIO_SET_MODE, + &(struct uffdio_set_mode){ + .disable = UFFD_FEATURE_RWP_ASYNC }); + + /* 3. Read the hot set: pages touched this interval. */ + ioctl(pagemap_fd, PAGEMAP_SCAN, &(struct pm_scan_arg){ + .category_mask = PAGE_IS_ACCESSED, + .return_mask = PAGE_IS_ACCESSED, + ... + }); + + /* + * 4. Reclaim the file offsets that are NOT in the hot set. + * Driving this from the file's offset space (rather than from a + * cold PTE scan) also reclaims pages that are cached but not + * mapped into guest_mem, e.g. pre-populated content. + */ + for each non-hot offset range: + /* Read from io_mem -- bypasses RWP, no fault. */ + pwrite(storage_fd, (char *)io_mem + off, len, off); + /* Drop the page from the shared file. */ + fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + off, len); + /* + * Wake any vCPU blocked on the RWP fault for this range: + * fallocate() does not iterate ctx->fault_pending_wqh. + */ + ioctl(uffd, UFFDIO_WAKE, &(struct uffdio_range){ + .start = (uintptr_t)guest_mem + off, .len = len }); + + /* 5. Resume async tracking */ + ioctl(uffd, UFFDIO_SET_MODE, + &(struct uffdio_set_mode){ + .enable = UFFD_FEATURE_RWP_ASYNC }); + } + +During step 4, a vCPU that accesses a ``guest_mem`` offset being evicted +blocks with a ``UFFD_PAGEFAULT_FLAG_RWP`` fault while the eviction is in +progress. After ``fallocate()`` punches the page out and ``UFFDIO_WAKE`` +fires, the vCPU retries the access, faults as ``MISSING``, and the +handler resolves it with ``UFFDIO_COPY`` from storage. + +This workflow targets shmem and hugetlbfs (both support a private +``io_mem`` mapping over the same fd). Anonymous-memory backings need a +different inner-loop strategy because the VMM has no way to read the +page without going through the RWP-protected mapping. + QEMU/KVM ======== From 324853ce8d4f794827d7e59b48c6c89d42b14852 Mon Sep 17 00:00:00 2001 From: Hajime Tazaki Date: Wed, 8 Jul 2026 17:38:29 +0900 Subject: [PATCH 271/501] mm: nommu: fix the error path when vma_iter_prealloc() fails When vma_iter_prealloc() fails in do_mmap(), it jumps to error_just_free as a error path of this function, but there are several possible issues. 1) It jumps to error_just_free without updating ret to -ENOMEM, meaning do_mmap() will return 0 on failure. 2) The error path unconditionally frees the region struct. Since the region was already added to the global nommu_region_tree via add_nommu_region(), leaving it makes a potential dangling pointer in the tree and may cause a use-after-free on the next tree walk. 3) If do_mmap() finds an existing overlapping shared region, it increments its usage, sets region to this existing pregion, and jumps to share: region = pregion; result = start; goto share; When vma_iter_prealloc() fails and jumps to error_just_free, the error path unconditionally frees the region: error: ... if (region->vm_file) fput(region->vm_file); kmem_cache_free(vm_region_jar, region); This potentially leaves a dangling pointer in nommu_region_tree and causes RB-tree corruption. 4) When establishing a new private mapping, do_mmap_private() allocates physical pages and assigns them to region->vm_start: base = alloc_pages_exact(total << PAGE_SHIFT, GFP_KERNEL); ... region->vm_start = (unsigned long) base; If we later fail at vma_iter_prealloc() and jump to error_just_free, the region struct is freed, but the backing physical memory isn't freed via free_page_series(). 5) In the error label of do_mmap(), the vm_area_struct allocated is freed by vm_area_free(vma) but not called after vma_close(), leaving potential memory leak which should be handled by a custom .close handler of vm_ops. This commit fixes those issues by introducing new jump label, error_vma_iter_prealloc, to correctly handle the error case of vma_iter_prealloc(), updating ret value (1), and move the region updates after the place that the allocation is finished (2). Additionally, the commit removes the existing goto label, error, and consolidates to error_just_free as existing `goto error;` code blocks always release nommu_region_sem. Moreover, it only frees region allocated in this request to avoid freeing the shared, existing region shared by other processes (3), and free physical memory when do_mmap_private() allocates (4). It also add vma_close() before vm_area_free() to fix the potential leak (5). Those issues are discovered by Sashiko, linked below. Link: https://lore.kernel.org/20260708083829.576036-1-thehajime@gmail.com Link: https://sashiko.dev/#/patchset/20260702012830.667205-1-thehajime%40gmail.com Link: https://sashiko.dev/#/patchset/c8513ee5aa8444ec9bf6c276043c9f833016a2fa.1783304131.git.thehajime%40gmail.com Link: https://sashiko.dev/#/patchset/20260707235137.498738-1-thehajime%40gmail.com Fixes: b5df09226450 ("mm: set up vma iterator for vma_iter_prealloc() calls") Signed-off-by: Hajime Tazaki Cc: Geert Uytterhoeven Cc: Jann Horn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Pedro Falcato Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/nommu.c | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/mm/nommu.c b/mm/nommu.c index 11fd558be5ed..d9aa684d2bdf 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -1181,7 +1181,6 @@ unsigned long do_mmap(struct file *file, ret = do_mmap_private(vma, region, len, capabilities); if (ret < 0) goto error_just_free; - add_nommu_region(region); /* clear anonymous mappings that don't ask for uninitialized data */ if (!vma->vm_file && @@ -1199,7 +1198,9 @@ unsigned long do_mmap(struct file *file, BUG_ON(!vma->vm_region); vma_iter_config(&vmi, vma->vm_start, vma->vm_end); if (vma_iter_prealloc(&vmi, vma)) - goto error_just_free; + goto error_vma_iter_prealloc; + + add_nommu_region(region); setup_vma_to_mm(vma, current->mm); current->mm->map_count++; @@ -1218,22 +1219,41 @@ unsigned long do_mmap(struct file *file, return result; error_just_free: + vma_close(vma); + /* if the error was from shared mapping/existing region, don't free the region. + * this has to be before releasing semaphore. + */ + if (region->vm_usage == 1) { + if (region->vm_file) + fput(region->vm_file); + kmem_cache_free(vm_region_jar, region); + + } else + region->vm_usage--; + up_write(&nommu_region_sem); -error: vma_iter_free(&vmi); - if (region->vm_file) - fput(region->vm_file); - kmem_cache_free(vm_region_jar, region); + if (vma->vm_file) fput(vma->vm_file); vm_area_free(vma); return ret; sharing_violation: - up_write(&nommu_region_sem); pr_warn("Attempt to share mismatched mappings\n"); ret = -EINVAL; - goto error; + goto error_just_free; + +error_vma_iter_prealloc: + pr_warn("Allocation of vma iterator for process %d failed\n", current->pid); + show_mem(); + ret = -ENOMEM; + + /* in case that the region is allocated via do_mmap_private() */ + if ((region->vm_usage == 1) && (region->vm_flags & VM_MAPPED_COPY)) + free_page_series(region->vm_start, region->vm_top); + + goto error_just_free; error_getting_vma: kmem_cache_free(vm_region_jar, region); From 5e83b4944d4381787ba3f2b838cc5a0b48b3a81e Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Thu, 16 Jul 2026 16:51:00 +0200 Subject: [PATCH 272/501] x86/mm: drop order parameter from free_pagetable() Patch series "mm: remove CONFIG_HAVE_BOOTMEM_INFO_NODE (Part 2)", v2. Let's remove the remaining pieces of CONFIG_HAVE_BOOTMEM_INFO_NODE, performing some smaller cleanups around freeing of reserved vmemmap pages on the way. This patch (of 10): All callers pass 0, so let's drop the parameter. Link: https://lore.kernel.org/20260716-bootmem_info_part2-v2-0-4afc76c73d61@kernel.org Link: https://lore.kernel.org/20260716-bootmem_info_part2-v2-1-4afc76c73d61@kernel.org Signed-off-by: David Hildenbrand (Arm) Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Muchun Song Reviewed-by: Zi Yan Reviewed-by: Dave Hansen Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Brendan Jackman Cc: Christian Borntraeger Cc: Gerald Schaefer Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Peter Zijlstra Cc: Suren Baghdasaryan Cc: Sven Schnelle Cc: Vasily Gorbik Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- arch/x86/mm/init_64.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index d57f29ca23a5..69e36f02a663 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -1006,22 +1006,19 @@ static void free_reserved_pages(struct page *page, unsigned long nr_pages) free_reserved_page(page++); } -static void __meminit free_pagetable(struct page *page, int order) +static void __meminit free_pagetable(struct page *page) { /* bootmem page has reserved flag */ if (PageReserved(page)) { - unsigned long nr_pages = 1 << order; #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE enum bootmem_type type = bootmem_type(page); - if (type == MIX_SECTION_INFO) { - while (nr_pages--) - put_page_bootmem(page++); - } else { - free_reserved_pages(page, nr_pages); - } + if (type == MIX_SECTION_INFO) + put_page_bootmem(page); + else + free_reserved_page(page); #else - free_reserved_pages(page, nr_pages); + free_reserved_page(page); #endif } else { pagetable_free(page_ptdesc(page)); @@ -1060,7 +1057,7 @@ static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd) } /* free a pte table */ - free_pagetable(pmd_page(*pmd), 0); + free_pagetable(pmd_page(*pmd)); spin_lock(&init_mm.page_table_lock); pmd_clear(pmd); spin_unlock(&init_mm.page_table_lock); @@ -1078,7 +1075,7 @@ static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud) } /* free a pmd table */ - free_pagetable(pud_page(*pud), 0); + free_pagetable(pud_page(*pud)); spin_lock(&init_mm.page_table_lock); pud_clear(pud); spin_unlock(&init_mm.page_table_lock); @@ -1096,7 +1093,7 @@ static void __meminit free_pud_table(pud_t *pud_start, p4d_t *p4d) } /* free a pud table */ - free_pagetable(p4d_page(*p4d), 0); + free_pagetable(p4d_page(*p4d)); spin_lock(&init_mm.page_table_lock); p4d_clear(p4d); spin_unlock(&init_mm.page_table_lock); From d79db3f39684b008bd8ff6dbf509ef27802e4ef1 Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Thu, 16 Jul 2026 16:51:01 +0200 Subject: [PATCH 273/501] mm: provide free_reserved_pages(), removing x86 variant Let's extend free_reserved_page() in page_alloc.c to free_reserved_pages(), dropping the custom x86 variant. The common-code variant will consume an order, so adjust the x86 callers accordingly. Make free_reserved_pages() assume that we are freeing ordinary high-order pages, just with the special "reserved" flavor. The target use case for now is freeing vmemmap PMD pages. Set the refcount directly to 0 (instead of 1) and call __free_frozen_pages(). Set the page count to 0 before clearing PG_reserved, so someone checking PG_reserved (and not finding it set) to then try grabbing a ref would not suddenly have that ref be dropped. That is arguably cleaner and safer than the old way of doing it. Add some kerneldoc. Use a single adjust_managed_page_count() call. Link: https://lore.kernel.org/20260716-bootmem_info_part2-v2-2-4afc76c73d61@kernel.org Signed-off-by: David Hildenbrand (Arm) Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Muchun Song Reviewed-by: Zi Yan Reviewed-by: Dave Hansen Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Brendan Jackman Cc: Christian Borntraeger Cc: Gerald Schaefer Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Peter Zijlstra Cc: Suren Baghdasaryan Cc: Sven Schnelle Cc: Vasily Gorbik Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- arch/x86/mm/init_64.c | 8 +------- include/linux/mm.h | 8 ++++++-- mm/page_alloc.c | 33 ++++++++++++++++++++++++++------- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 69e36f02a663..320e742b0bb3 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -1000,12 +1000,6 @@ int arch_add_memory(int nid, u64 start, u64 size, return add_pages(nid, start_pfn, nr_pages, params); } -static void free_reserved_pages(struct page *page, unsigned long nr_pages) -{ - while (nr_pages--) - free_reserved_page(page++); -} - static void __meminit free_pagetable(struct page *page) { /* bootmem page has reserved flag */ @@ -1038,7 +1032,7 @@ static void __meminit free_vmemmap_pages(struct page *page, unsigned int order, while (nr_pages--) put_page_bootmem(page++); } else { - free_reserved_pages(page, nr_pages); + free_reserved_pages(page, order); } } else { __free_pages(page, order); diff --git a/include/linux/mm.h b/include/linux/mm.h index 25e669632d2c..e1e1154dafd7 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3996,8 +3996,12 @@ extern unsigned long free_reserved_area(void *start, void *end, extern void adjust_managed_page_count(struct page *page, long count); -/* Free the reserved page into the buddy system, so it gets managed. */ -void free_reserved_page(struct page *page); +void free_reserved_pages(struct page *page, unsigned int order); + +static inline void free_reserved_page(struct page *page) +{ + free_reserved_pages(page, 0); +} static inline void mark_page_reserved(struct page *page) { diff --git a/mm/page_alloc.c b/mm/page_alloc.c index cc45dfe89c76..f61e4203eced 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -6389,15 +6389,34 @@ void adjust_managed_page_count(struct page *page, long count) } EXPORT_SYMBOL(adjust_managed_page_count); -void free_reserved_page(struct page *page) +/** + * free_reserved_pages - free reserved pages + * @page: First page to free. + * @order: The page order to free. + * + * Free pages allocated through memblock during boot, letting the buddy + * manage them from now on. + * + * @page must be naturally aligned to the order and the order must not + * exceed MAX_PAGE_ORDER. All pages must be reserved. + */ +void free_reserved_pages(struct page *page, unsigned int order) { - clear_page_tag_ref(page); - ClearPageReserved(page); - init_page_count(page); - __free_page(page); - adjust_managed_page_count(page, 1); + const unsigned long nr_pages = 1UL << order; + int i; + + VM_WARN_ON_ONCE(!IS_ALIGNED(page_to_pfn(page), nr_pages)); + VM_WARN_ON_ONCE(order > MAX_PAGE_ORDER); + + for (i = 0; i < nr_pages; i++) { + clear_page_tag_ref(page + i); + set_page_count(page + i, 0); + ClearPageReserved(page + i); + } + adjust_managed_page_count(page, nr_pages); + __free_frozen_pages(page, order, FPI_NONE); } -EXPORT_SYMBOL(free_reserved_page); +EXPORT_SYMBOL(free_reserved_pages); static int page_alloc_cpu_dead(unsigned int cpu) { From cbbd561893f65e2b31cf9c18634405cb8d8bf6f0 Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Thu, 16 Jul 2026 16:51:02 +0200 Subject: [PATCH 274/501] s390/mm: use free_reserved_pages() in vmem_free_pages() Let's use our new generic helper. Link: https://lore.kernel.org/20260716-bootmem_info_part2-v2-3-4afc76c73d61@kernel.org Signed-off-by: David Hildenbrand (Arm) Acked-by: Heiko Carstens Reviewed-by: Muchun Song Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Zi Yan Reviewed-by: Dave Hansen Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Brendan Jackman Cc: Christian Borntraeger Cc: Gerald Schaefer Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Peter Zijlstra Cc: Suren Baghdasaryan Cc: Sven Schnelle Cc: Vasily Gorbik Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- arch/s390/mm/vmem.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 94ee06ae8913..d2879ce860a1 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -39,7 +39,6 @@ static void __ref *vmem_alloc_pages(unsigned int order) static void vmem_free_pages(unsigned long addr, int order, struct vmem_altmap *altmap) { - unsigned int nr_pages = 1 << order; struct page *page; if (altmap) { @@ -49,8 +48,7 @@ static void vmem_free_pages(unsigned long addr, int order, struct vmem_altmap *a page = virt_to_page((void *)addr); if (PageReserved(page)) { /* allocated from memblock */ - while (nr_pages--) - free_reserved_page(page++); + free_reserved_pages(page, order); } else { free_pages(addr, order); } From f4436442e5999bf63315a67803819c48a886d132 Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Thu, 16 Jul 2026 16:51:03 +0200 Subject: [PATCH 275/501] mm/bootmem_info: allow calling free_bootmem_page() on pages without a bootmem_type As preparation for further changes, let's temporarily allow freeing pages that were not previously registered. This will allow freeing unregistered vmemmap pages allocated during boot through free_bootmem_page() from hugetlb code, until we fully rip all of that out. Link: https://lore.kernel.org/20260716-bootmem_info_part2-v2-4-4afc76c73d61@kernel.org Signed-off-by: David Hildenbrand (Arm) Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Muchun Song Acked-by: Zi Yan Reviewed-by: Dave Hansen Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Brendan Jackman Cc: Christian Borntraeger Cc: Gerald Schaefer Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Peter Zijlstra Cc: Suren Baghdasaryan Cc: Sven Schnelle Cc: Vasily Gorbik Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/bootmem_info.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/linux/bootmem_info.h b/include/linux/bootmem_info.h index f724340755e5..486acc612703 100644 --- a/include/linux/bootmem_info.h +++ b/include/linux/bootmem_info.h @@ -44,12 +44,12 @@ static inline void free_bootmem_page(struct page *page) { enum bootmem_type type = bootmem_type(page); - VM_BUG_ON_PAGE(page_ref_count(page) != 2, page); - - if (type == SECTION_INFO || type == MIX_SECTION_INFO) + if (type == SECTION_INFO || type == MIX_SECTION_INFO) { + VM_WARN_ON_PAGE(page_ref_count(page) != 2, page); put_page_bootmem(page); - else - VM_BUG_ON_PAGE(1, page); + } else { + free_reserved_page(page); + } } #else static inline void register_page_bootmem_info_node(struct pglist_data *pgdat) From 7bfc04d6bd8d7fc60436cd8914928e8b80696439 Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Thu, 16 Jul 2026 16:51:04 +0200 Subject: [PATCH 276/501] x86/mm: stop marking vmemmap as SECTION_INFO We added the whole bootmem registration machinery in commit 04753278769f ("memory hotplug: register section/node id to free"). The main use case was to remember to which memory section memmap pages belonged, so the memmap could be handled accordingly when freeing memory. However, all that machinery is not required anymore: a memory section can only get offlined if *all* pages can get offlined; and it can only get unplugged once offline. If some of these pages are unmovable memmap pages: bad luck, doesn't work. Offlining will fail. Further, a lot of this machinery was required for pre-vmemmap support. Now we only support the vmemmap with memory hotplug. So the whole machinery is useless today. Let's start by removing the last pieces by first stopping to mark vmemmap pages as SECTION_INFO. In free_vmemmap_pages(), we can now always just free the reserved pages directly. Link: https://lore.kernel.org/20260716-bootmem_info_part2-v2-5-4afc76c73d61@kernel.org Signed-off-by: David Hildenbrand (Arm) Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Dave Hansen Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Brendan Jackman Cc: Christian Borntraeger Cc: Gerald Schaefer Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Oscar Salvador Cc: Peter Zijlstra Cc: Suren Baghdasaryan Cc: Sven Schnelle Cc: Vasily Gorbik Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- arch/x86/mm/init_64.c | 30 ++++-------------------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 320e742b0bb3..82bb0384b8ab 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -1024,19 +1024,12 @@ static void __meminit free_vmemmap_pages(struct page *page, unsigned int order, { unsigned long nr_pages = 1u << order; - if (altmap) { + if (altmap) vmem_altmap_free(altmap, nr_pages); - } else if (PageReserved(page)) { - if (IS_ENABLED(CONFIG_HAVE_BOOTMEM_INFO_NODE) && - bootmem_type(page) == SECTION_INFO) { - while (nr_pages--) - put_page_bootmem(page++); - } else { - free_reserved_pages(page, order); - } - } else { + else if (PageReserved(page)) + free_reserved_pages(page, order); + else __free_pages(page, order); - } } static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd) @@ -1593,12 +1586,8 @@ void register_page_bootmem_memmap(unsigned long section_nr, p4d_t *p4d; pud_t *pud; pmd_t *pmd; - unsigned int nr_pmd_pages; - struct page *page; for (; addr < end; addr = next) { - pte_t *pte = NULL; - pgd = pgd_offset_k(addr); if (pgd_none(*pgd)) { next = (addr + PAGE_SIZE) & PAGE_MASK; @@ -1630,19 +1619,8 @@ void register_page_bootmem_memmap(unsigned long section_nr, next = (addr + PAGE_SIZE) & PAGE_MASK; get_page_bootmem(section_nr, pmd_page(*pmd), MIX_SECTION_INFO); - - pte = pte_offset_kernel(pmd, addr); - if (pte_none(*pte)) - continue; - get_page_bootmem(section_nr, pte_page(*pte), - SECTION_INFO); } else { next = pmd_addr_end(addr, end); - nr_pmd_pages = (next - addr) >> PAGE_SHIFT; - page = pmd_page(*pmd); - while (nr_pmd_pages--) - get_page_bootmem(section_nr, page++, - SECTION_INFO); } } } From f20e92095f17c4a8be31eb8503e94cfd50c60949 Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Thu, 16 Jul 2026 16:51:05 +0200 Subject: [PATCH 277/501] x86/mm: stop marking page tables as MIX_SECTION_INFO There is no good reason to mark boot page tables as MIX_SECTION_INFO: we only free boot page tables when they are completely empty, and memory offlining/hotunplug doesn't benefit from it in any way. So just stop marking page tables as MIX_SECTION_INFO. In free_pagetable(), we can now simply free reserved pages directly. Link: https://lore.kernel.org/20260716-bootmem_info_part2-v2-6-4afc76c73d61@kernel.org Signed-off-by: David Hildenbrand (Arm) Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Dave Hansen Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Brendan Jackman Cc: Christian Borntraeger Cc: Gerald Schaefer Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Oscar Salvador Cc: Peter Zijlstra Cc: Suren Baghdasaryan Cc: Sven Schnelle Cc: Vasily Gorbik Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- arch/x86/mm/init_64.c | 59 ++----------------------------------------- 1 file changed, 2 insertions(+), 57 deletions(-) diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 82bb0384b8ab..1b17dcf781e6 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -1002,21 +1002,10 @@ int arch_add_memory(int nid, u64 start, u64 size, static void __meminit free_pagetable(struct page *page) { - /* bootmem page has reserved flag */ - if (PageReserved(page)) { -#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE - enum bootmem_type type = bootmem_type(page); - - if (type == MIX_SECTION_INFO) - put_page_bootmem(page); - else - free_reserved_page(page); -#else + if (PageReserved(page)) free_reserved_page(page); -#endif - } else { + else pagetable_free(page_ptdesc(page)); - } } static void __meminit free_vmemmap_pages(struct page *page, unsigned int order, @@ -1579,50 +1568,6 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, void register_page_bootmem_memmap(unsigned long section_nr, struct page *start_page, unsigned long nr_pages) { - unsigned long addr = (unsigned long)start_page; - unsigned long end = (unsigned long)(start_page + nr_pages); - unsigned long next; - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; - - for (; addr < end; addr = next) { - pgd = pgd_offset_k(addr); - if (pgd_none(*pgd)) { - next = (addr + PAGE_SIZE) & PAGE_MASK; - continue; - } - get_page_bootmem(section_nr, pgd_page(*pgd), MIX_SECTION_INFO); - - p4d = p4d_offset(pgd, addr); - if (p4d_none(*p4d)) { - next = (addr + PAGE_SIZE) & PAGE_MASK; - continue; - } - get_page_bootmem(section_nr, p4d_page(*p4d), MIX_SECTION_INFO); - - pud = pud_offset(p4d, addr); - if (pud_none(*pud)) { - next = (addr + PAGE_SIZE) & PAGE_MASK; - continue; - } - get_page_bootmem(section_nr, pud_page(*pud), MIX_SECTION_INFO); - - pmd = pmd_offset(pud, addr); - if (pmd_none(*pmd)) { - next = (addr + PAGE_SIZE) & PAGE_MASK; - continue; - } - - if (!boot_cpu_has(X86_FEATURE_PSE) || !pmd_leaf(*pmd)) { - next = (addr + PAGE_SIZE) & PAGE_MASK; - get_page_bootmem(section_nr, pmd_page(*pmd), - MIX_SECTION_INFO); - } else { - next = pmd_addr_end(addr, end); - } - } } #endif From ea963eab118b41f6ff2dcc4a8b0f43de375473b1 Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Thu, 16 Jul 2026 16:51:06 +0200 Subject: [PATCH 278/501] x86/mm: remove CONFIG_HAVE_BOOTMEM_INFO_NODE CONFIG_HAVE_BOOTMEM_INFO_NODE now essentially doesn't do anything. So let's remove support for CONFIG_HAVE_BOOTMEM_INFO_NODE. Link: https://lore.kernel.org/20260716-bootmem_info_part2-v2-7-4afc76c73d61@kernel.org Signed-off-by: David Hildenbrand (Arm) Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Dave Hansen Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Brendan Jackman Cc: Christian Borntraeger Cc: Gerald Schaefer Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Oscar Salvador Cc: Peter Zijlstra Cc: Suren Baghdasaryan Cc: Sven Schnelle Cc: Vasily Gorbik Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- arch/x86/mm/init_64.c | 26 -------------------------- mm/Kconfig | 1 - 2 files changed, 27 deletions(-) diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 1b17dcf781e6..ab4c5a02326f 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -33,7 +33,6 @@ #include #include #include -#include #include #include @@ -1286,16 +1285,6 @@ void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap, static struct kcore_list kcore_vsyscall; -static void __init register_page_bootmem_info(void) -{ -#if defined(CONFIG_NUMA) || defined(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP) - int i; - - for_each_online_node(i) - register_page_bootmem_info_node(NODE_DATA(i)); -#endif -} - /* * Pre-allocates page-table pages for the vmalloc area in the kernel page-table. * Only the level which needs to be synchronized between all page-tables is @@ -1358,14 +1347,6 @@ void __init mem_init(void) after_bootmem = 1; x86_init.hyper.init_after_bootmem(); - /* - * Must be done after boot memory is put on freelist, because here we - * might set fields in deferred struct pages that have not yet been - * initialized, and memblock_free_all() initializes all the reserved - * deferred pages for us. - */ - register_page_bootmem_info(); - /* Register memory areas for /proc/kcore */ if (get_gate_vma(&init_mm)) kclist_add(&kcore_vsyscall, (void *)VSYSCALL_ADDR, PAGE_SIZE, KCORE_USER); @@ -1564,13 +1545,6 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, return err; } -#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE -void register_page_bootmem_memmap(unsigned long section_nr, - struct page *start_page, unsigned long nr_pages) -{ -} -#endif - void __meminit vmemmap_populate_print_last(void) { if (p_start) { diff --git a/mm/Kconfig b/mm/Kconfig index 4ea070f5664a..6b3430ff8788 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -590,7 +590,6 @@ endchoice config MEMORY_HOTREMOVE bool "Allow for memory hot remove" - select HAVE_BOOTMEM_INFO_NODE if X86_64 depends on MEMORY_HOTPLUG select MIGRATION From 788a0efb959d78c4343afcff0cdd156b91083391 Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Thu, 16 Jul 2026 16:51:07 +0200 Subject: [PATCH 279/501] mm/hugetlb_vmemmap: remove bootmem_info leftovers We never set CONFIG_HAVE_BOOTMEM_INFO_NODE, so we can just switch to free_reserved_page() and drop the register_page_bootmem_memmap() call. Link: https://lore.kernel.org/20260716-bootmem_info_part2-v2-8-4afc76c73d61@kernel.org Signed-off-by: David Hildenbrand (Arm) Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Muchun Song Reviewed-by: Dave Hansen Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Brendan Jackman Cc: Christian Borntraeger Cc: Gerald Schaefer Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Peter Zijlstra Cc: Suren Baghdasaryan Cc: Sven Schnelle Cc: Vasily Gorbik Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/hugetlb_vmemmap.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index 1430a5aa2de5..917db0984143 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -12,7 +12,6 @@ #include #include -#include #include #include #include @@ -177,13 +176,13 @@ static int vmemmap_remap_range(unsigned long start, unsigned long end, * Free a vmemmap page. A vmemmap page can be allocated from the memblock * allocator or buddy allocator. If the PG_reserved flag is set, it means * that it allocated from the memblock allocator, just free it via the - * free_bootmem_page(). Otherwise, use __free_page(). + * free_reserved_page(). Otherwise, use __free_page(). */ static inline void free_vmemmap_page(struct page *page) { if (PageReserved(page)) { memmap_boot_pages_add(-1); - free_bootmem_page(page); + free_reserved_page(page); } else { memmap_pages_add(-1); __free_page(page); @@ -641,9 +640,6 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h, epfn = spfn + hugetlb_vmemmap_size(h); vmemmap_wrprotect_hvo(spfn, epfn, folio_nid(folio), HUGETLB_VMEMMAP_RESERVE_SIZE); - register_page_bootmem_memmap(pfn_to_section_nr(folio_pfn(folio)), - &folio->page, - HUGETLB_VMEMMAP_RESERVE_PAGES); continue; } From 4f9ec35df52b1d861690d369a9411460243cbc4e Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Thu, 16 Jul 2026 16:51:08 +0200 Subject: [PATCH 280/501] mm/sparse: remove bootmem_info.h include No longer required, so let's remove it. Link: https://lore.kernel.org/20260716-bootmem_info_part2-v2-9-4afc76c73d61@kernel.org Signed-off-by: David Hildenbrand (Arm) Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Muchun Song Reviewed-by: Dave Hansen Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Brendan Jackman Cc: Christian Borntraeger Cc: Gerald Schaefer Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Peter Zijlstra Cc: Suren Baghdasaryan Cc: Sven Schnelle Cc: Vasily Gorbik Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/sparse.c | 1 - 1 file changed, 1 deletion(-) diff --git a/mm/sparse.c b/mm/sparse.c index 058ef9300367..b5c74aab3379 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include "internal.h" #include From 3b7cee2e9a14467d05136f3b08a16c23d867b3bf Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Thu, 16 Jul 2026 16:51:09 +0200 Subject: [PATCH 281/501] mm/bootmem_info: remove CONFIG_HAVE_BOOTMEM_INFO_NODE The whole infrastructure is unused now. Let's remove the config option along with mm/bootmem_info. + include/linux/bootmem_info.h. Link: https://lore.kernel.org/20260716-bootmem_info_part2-v2-10-4afc76c73d61@kernel.org Signed-off-by: David Hildenbrand (Arm) Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Muchun Song Reviewed-by: Dave Hansen Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Brendan Jackman Cc: Christian Borntraeger Cc: Gerald Schaefer Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Peter Zijlstra Cc: Suren Baghdasaryan Cc: Sven Schnelle Cc: Vasily Gorbik Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- MAINTAINERS | 1 - include/linux/bootmem_info.h | 89 ------------------------------------ mm/Kconfig | 7 --- mm/Makefile | 1 - mm/bootmem_info.c | 72 ----------------------------- 5 files changed, 170 deletions(-) delete mode 100644 include/linux/bootmem_info.h delete mode 100644 mm/bootmem_info.c diff --git a/MAINTAINERS b/MAINTAINERS index 0d47b358b2a4..8acd1aeec9db 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -16880,7 +16880,6 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git fixes F: Documentation/core-api/boot-time-mm.rst F: include/linux/kho/abi/memblock.h F: include/linux/memblock.h -F: mm/bootmem_info.c F: mm/memblock.c F: mm/memtest.c F: mm/mm_init.c diff --git a/include/linux/bootmem_info.h b/include/linux/bootmem_info.h deleted file mode 100644 index 486acc612703..000000000000 --- a/include/linux/bootmem_info.h +++ /dev/null @@ -1,89 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __LINUX_BOOTMEM_INFO_H -#define __LINUX_BOOTMEM_INFO_H - -#include -#include - -/* - * Types for free bootmem stored in the low bits of page->private. - */ -enum bootmem_type { - MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE = 1, - SECTION_INFO = MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE, - MIX_SECTION_INFO, - NODE_INFO, - MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE = NODE_INFO, -}; - -#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE -void __init register_page_bootmem_info_node(struct pglist_data *pgdat); -void register_page_bootmem_memmap(unsigned long section_nr, struct page *map, - unsigned long nr_pages); - -void get_page_bootmem(unsigned long info, struct page *page, - enum bootmem_type type); -void put_page_bootmem(struct page *page); - -static inline enum bootmem_type bootmem_type(const struct page *page) -{ - return (unsigned long)page->private & 0xf; -} - -static inline unsigned long bootmem_info(const struct page *page) -{ - return (unsigned long)page->private >> 4; -} - -/* - * Any memory allocated via the memblock allocator and not via the - * buddy will be marked reserved already in the memmap. For those - * pages, we can call this function to free it to buddy allocator. - */ -static inline void free_bootmem_page(struct page *page) -{ - enum bootmem_type type = bootmem_type(page); - - if (type == SECTION_INFO || type == MIX_SECTION_INFO) { - VM_WARN_ON_PAGE(page_ref_count(page) != 2, page); - put_page_bootmem(page); - } else { - free_reserved_page(page); - } -} -#else -static inline void register_page_bootmem_info_node(struct pglist_data *pgdat) -{ -} - -static inline void register_page_bootmem_memmap(unsigned long section_nr, - struct page *map, unsigned long nr_pages) -{ -} - -static inline void put_page_bootmem(struct page *page) -{ -} - -static inline enum bootmem_type bootmem_type(const struct page *page) -{ - return SECTION_INFO; -} - -static inline unsigned long bootmem_info(const struct page *page) -{ - return 0; -} - -static inline void get_page_bootmem(unsigned long info, struct page *page, - enum bootmem_type type) -{ -} - -static inline void free_bootmem_page(struct page *page) -{ - free_reserved_page(page); -} -#endif - -#endif /* __LINUX_BOOTMEM_INFO_H */ diff --git a/mm/Kconfig b/mm/Kconfig index 6b3430ff8788..0d06e96ff3c9 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -509,13 +509,6 @@ config EXCLUSIVE_SYSTEM_RAM def_bool y depends on !DEVMEM || STRICT_DEVMEM -# -# Only be set on architectures that have completely implemented memory hotplug -# feature. If you are not sure, don't touch it. -# -config HAVE_BOOTMEM_INFO_NODE - def_bool n - config ARCH_ENABLE_MEMORY_HOTPLUG bool diff --git a/mm/Makefile b/mm/Makefile index 8e7e964b99a4..ab37ef428d98 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -141,7 +141,6 @@ obj-$(CONFIG_MEMFD_CREATE) += memfd.o obj-$(CONFIG_MAPPING_DIRTY_HELPERS) += mapping_dirty_helpers.o obj-$(CONFIG_PTDUMP) += ptdump.o obj-$(CONFIG_PAGE_REPORTING) += page_reporting.o -obj-$(CONFIG_HAVE_BOOTMEM_INFO_NODE) += bootmem_info.o obj-$(CONFIG_GENERIC_IOREMAP) += ioremap.o obj-$(CONFIG_SHRINKER_DEBUG) += shrinker_debug.o obj-$(CONFIG_EXECMEM) += execmem.o diff --git a/mm/bootmem_info.c b/mm/bootmem_info.c deleted file mode 100644 index 0fa78db7fbc0..000000000000 --- a/mm/bootmem_info.c +++ /dev/null @@ -1,72 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Bootmem core functions. - * - * Copyright (c) 2020, Bytedance. - * - * Author: Muchun Song - * - */ -#include -#include -#include -#include -#include -#include - -void get_page_bootmem(unsigned long info, struct page *page, - enum bootmem_type type) -{ - BUG_ON(type > 0xf); - BUG_ON(info > (ULONG_MAX >> 4)); - set_page_private(page, info << 4 | type); - page_ref_inc(page); -} - -void put_page_bootmem(struct page *page) -{ - enum bootmem_type type = bootmem_type(page); - - BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE || - type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE); - - if (page_ref_dec_return(page) == 1) { - set_page_private(page, 0); - free_reserved_page(page); - } -} - -static void __init register_page_bootmem_info_section(unsigned long start_pfn) -{ - unsigned long section_nr; - struct mem_section *ms; - - start_pfn = SECTION_ALIGN_DOWN(start_pfn); - section_nr = pfn_to_section_nr(start_pfn); - ms = __nr_to_section(section_nr); - - if (!preinited_vmemmap_section(ms)) - register_page_bootmem_memmap(section_nr, pfn_to_page(start_pfn), - PAGES_PER_SECTION); -} - -void __init register_page_bootmem_info_node(struct pglist_data *pgdat) -{ - unsigned long pfn, end_pfn; - int node = pgdat->node_id; - - pfn = pgdat->node_start_pfn; - end_pfn = pgdat_end_pfn(pgdat); - - /* register section info */ - for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { - /* - * Some platforms can assign the same pfn to multiple nodes - on - * node0 as well as nodeN. To avoid registering a pfn against - * multiple nodes we check that this pfn does not already - * reside in some other nodes. - */ - if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node)) - register_page_bootmem_info_section(pfn); - } -} From f3ab162a921c9554c01b81a1a1b022f3adbbf96c Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 8 Jul 2026 06:53:51 -0700 Subject: [PATCH 282/501] mm/damon/core: remove comment and test for nr_to_bp() divide-by-zero Patch series "mm/damon: update probe hits for runtime parameter commits". DAMON users can update DAMON parameters such as sampling and aggregation intervals at runtime. For such changes, monitoring results that depend on the intervals should be properly updated for better accuracy. For example, the access frequency counter (nr_accesses) is updated. The data attribute monitoring counter (probe_hits) is not being updated, though. Do the updates for new parameters. Patch 1 removes obsolete comments and test code for a function that this series will touch. Patches 2-5 rename functions that are being used for nr_accesses update, to be able to be used for probe_hits without confusion. Patch 6 does the probe_hits update. Patch 7 update damon_probe_hits_mvsum() to cover a corner case from the update for better accuracy. This patch (of 7): The comments on damon_nr_accesses_to_accesses_bp() and its unit test warn it can trigger division-by-zero when the aggregation interval is zero. Commit 35d4a3cf70a8 ("mm/damon/ops-common: handle extreme intervals in damon_hot_score()") modified damon_max_nr_accesses() to always return non-zero. Hence no division-by-zero of the note can happen. Remove the obsolete comment on the function. The test code was written to test the division-by-zero case, which cannot happen anymore. Having it makes no sense. Entirely remove the test code and its comment. Link: https://lore.kernel.org/20260708135359.122587-1-sj@kernel.org Link: https://lore.kernel.org/20260708135359.122587-2-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/core.c | 8 +------- mm/damon/tests/core-kunit.h | 23 ----------------------- 2 files changed, 1 insertion(+), 30 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 390e00b3685e..a15005db8289 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -883,13 +883,7 @@ static unsigned int damon_accesses_bp_to_nr_accesses( return accesses_bp * damon_max_nr_accesses(attrs) / 10000; } -/* - * Convert nr_accesses to access ratio in bp (per 10,000). - * - * Callers should ensure attrs.aggr_interval is not zero, like - * damon_update_monitoring_results() does . Otherwise, divide-by-zero would - * happen. - */ +/* Convert nr_accesses to access ratio in bp (per 10,000) */ static unsigned int damon_nr_accesses_to_accesses_bp( unsigned int nr_accesses, struct damon_attrs *attrs) { diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 0124f83b39b8..278d46632eff 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -575,28 +575,6 @@ static void damon_test_set_regions(struct kunit *test) }, 3); } -static void damon_test_nr_accesses_to_accesses_bp(struct kunit *test) -{ - struct damon_attrs attrs = { - .sample_interval = 10, - .aggr_interval = ((unsigned long)UINT_MAX + 1) * 10 - }; - - /* - * In some cases such as 32bit architectures where UINT_MAX is - * ULONG_MAX, attrs.aggr_interval becomes zero. Calling - * damon_nr_accesses_to_accesses_bp() in the case will cause - * divide-by-zero. Such case is prohibited in normal execution since - * the caution is documented on the comment for the function, and - * damon_update_monitoring_results() does the check. Skip the test in - * the case. - */ - if (!attrs.aggr_interval) - kunit_skip(test, "aggr_interval is zero."); - - KUNIT_EXPECT_EQ(test, damon_nr_accesses_to_accesses_bp(123, &attrs), 0); -} - static void damon_test_update_monitoring_result(struct kunit *test) { struct damon_attrs old_attrs = { @@ -1572,7 +1550,6 @@ static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_split_above_half_progresses), KUNIT_CASE(damon_test_ops_registration), KUNIT_CASE(damon_test_set_regions), - KUNIT_CASE(damon_test_nr_accesses_to_accesses_bp), KUNIT_CASE(damon_test_update_monitoring_result), KUNIT_CASE(damon_test_set_attrs), KUNIT_CASE(damon_test_mvsum), From 571cc9a34ef6c20673f0286c8e5a454ec12c91ef Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 8 Jul 2026 06:53:52 -0700 Subject: [PATCH 283/501] mm/damon/core: s/damon_max_nr_accesses()/damon_nr_samples_per_aggr()/ damon_max_nr_accesses() actually returns the number of samples DAMON checks for each region per each aggregation interval. Rename it to better describe what it really does and not confusing for more general uses. Link: https://lore.kernel.org/20260708135359.122587-3-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 10 ++++++---- mm/damon/core.c | 4 ++-- mm/damon/lru_sort.c | 2 +- mm/damon/ops-common.c | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 19b7e839bde0..63f596957c1f 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -1056,15 +1056,17 @@ static inline bool damon_target_has_pid(const struct damon_ctx *ctx) return ctx->ops.id == DAMON_OPS_VADDR || ctx->ops.id == DAMON_OPS_FVADDR; } -static inline unsigned int damon_max_nr_accesses(const struct damon_attrs *attrs) +/* Returns number of samples per aggregation interval */ +static inline unsigned int damon_nr_samples_per_aggr( + const struct damon_attrs *attrs) { unsigned long sample_interval; - unsigned long max_nr_accesses; + unsigned long nr_samples; sample_interval = attrs->sample_interval ? : 1; - max_nr_accesses = min(attrs->aggr_interval / sample_interval, + nr_samples = min(attrs->aggr_interval / sample_interval, (unsigned long)UINT_MAX); - return max_nr_accesses ? : 1; + return nr_samples ? : 1; } diff --git a/mm/damon/core.c b/mm/damon/core.c index a15005db8289..177e5abc981e 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -880,14 +880,14 @@ static unsigned int damon_age_for_new_attrs(unsigned int age, static unsigned int damon_accesses_bp_to_nr_accesses( unsigned int accesses_bp, struct damon_attrs *attrs) { - return accesses_bp * damon_max_nr_accesses(attrs) / 10000; + return accesses_bp * damon_nr_samples_per_aggr(attrs) / 10000; } /* Convert nr_accesses to access ratio in bp (per 10,000) */ static unsigned int damon_nr_accesses_to_accesses_bp( unsigned int nr_accesses, struct damon_attrs *attrs) { - return mult_frac(nr_accesses, 10000, damon_max_nr_accesses(attrs)); + return mult_frac(nr_accesses, 10000, damon_nr_samples_per_aggr(attrs)); } static unsigned int damon_nr_accesses_for_new_attrs(unsigned int nr_accesses, diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c index e8c389ad3226..7e077084cb03 100644 --- a/mm/damon/lru_sort.c +++ b/mm/damon/lru_sort.c @@ -303,7 +303,7 @@ static int damon_lru_sort_apply_parameters(void) goto out; err = -ENOMEM; - hot_thres = damon_max_nr_accesses(&attrs) * + hot_thres = damon_nr_samples_per_aggr(&attrs) * hot_thres_access_freq / 1000; hot_scheme = damon_lru_sort_new_hot_scheme(hot_thres); if (!hot_scheme) diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index 6bdd1cfd3863..89c694f6e4fa 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -112,7 +112,7 @@ int damon_hot_score(struct damon_ctx *c, struct damon_region *r, int hotness; freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / - damon_max_nr_accesses(&c->attrs); + damon_nr_samples_per_aggr(&c->attrs); age_in_sec = (unsigned long)r->age * c->attrs.aggr_interval / 1000000; if (age_in_sec) From 71599dc25351dbf56701191d8dea22058aade021 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 8 Jul 2026 06:53:53 -0700 Subject: [PATCH 284/501] mm/damon/core: s/accesses_bp_to_nr_accesses/sample_bp_to_count/ accesses_bp_to_nr_accesses() actually converts a positive samples ratio to the count. Rename it to better describe what it really does and not confusing for more general uses. Link: https://lore.kernel.org/20260708135359.122587-4-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/core.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 177e5abc981e..dea63083917b 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -876,11 +876,11 @@ static unsigned int damon_age_for_new_attrs(unsigned int age, return age * old_attrs->aggr_interval / new_attrs->aggr_interval; } -/* convert access ratio in bp (per 10,000) to nr_accesses */ -static unsigned int damon_accesses_bp_to_nr_accesses( - unsigned int accesses_bp, struct damon_attrs *attrs) +/* convert sample ratio in bp (per 10,000) to count */ +static unsigned int damon_sample_bp_to_count( + unsigned int bp, struct damon_attrs *attrs) { - return accesses_bp * damon_nr_samples_per_aggr(attrs) / 10000; + return bp * damon_nr_samples_per_aggr(attrs) / 10000; } /* Convert nr_accesses to access ratio in bp (per 10,000) */ @@ -893,7 +893,7 @@ static unsigned int damon_nr_accesses_to_accesses_bp( static unsigned int damon_nr_accesses_for_new_attrs(unsigned int nr_accesses, struct damon_attrs *old_attrs, struct damon_attrs *new_attrs) { - return damon_accesses_bp_to_nr_accesses( + return damon_sample_bp_to_count( damon_nr_accesses_to_accesses_bp( nr_accesses, old_attrs), new_attrs); From f9088c9845e7db83a35e5e6e41714719f8f39b0b Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 8 Jul 2026 06:53:54 -0700 Subject: [PATCH 285/501] mm/damon/core: s/nr_accesses_to_accesses_bp/sample_count_to_bp/ damon_nr_accesses_to_accesses_bp() actually converts a positive sample count to the ratio. Rename it to better describe what it really does and not confusing for more general uses. Link: https://lore.kernel.org/20260708135359.122587-5-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/core.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index dea63083917b..718268df27fa 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -883,19 +883,18 @@ static unsigned int damon_sample_bp_to_count( return bp * damon_nr_samples_per_aggr(attrs) / 10000; } -/* Convert nr_accesses to access ratio in bp (per 10,000) */ -static unsigned int damon_nr_accesses_to_accesses_bp( - unsigned int nr_accesses, struct damon_attrs *attrs) +/* convert sample count to ratio in bp (per 10,000) */ +static unsigned int damon_sample_count_to_bp( + unsigned int count, struct damon_attrs *attrs) { - return mult_frac(nr_accesses, 10000, damon_nr_samples_per_aggr(attrs)); + return mult_frac(count, 10000, damon_nr_samples_per_aggr(attrs)); } static unsigned int damon_nr_accesses_for_new_attrs(unsigned int nr_accesses, struct damon_attrs *old_attrs, struct damon_attrs *new_attrs) { return damon_sample_bp_to_count( - damon_nr_accesses_to_accesses_bp( - nr_accesses, old_attrs), + damon_sample_count_to_bp(nr_accesses, old_attrs), new_attrs); } From 84113a30a8a354374f5d68fd6fe008685ab8eac8 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 8 Jul 2026 06:53:55 -0700 Subject: [PATCH 286/501] mm/damon/core: s/nr_accesses_for_new_attrs/nr_samples_for_new_attrs/ damon_nr_accesses_for_new_attrs() can be used for not only nr_accesses but also any positive sample count, like probe_hits. Rename to be able to be used for such general uses without confusion. Link: https://lore.kernel.org/20260708135359.122587-6-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/core.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 718268df27fa..800ff6b6d247 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -890,22 +890,21 @@ static unsigned int damon_sample_count_to_bp( return mult_frac(count, 10000, damon_nr_samples_per_aggr(attrs)); } -static unsigned int damon_nr_accesses_for_new_attrs(unsigned int nr_accesses, +static unsigned int damon_nr_samples_for_new_attrs(unsigned int nr, struct damon_attrs *old_attrs, struct damon_attrs *new_attrs) { return damon_sample_bp_to_count( - damon_sample_count_to_bp(nr_accesses, old_attrs), - new_attrs); + damon_sample_count_to_bp(nr, old_attrs), new_attrs); } static void damon_update_monitoring_result(struct damon_region *r, struct damon_attrs *old_attrs, struct damon_attrs *new_attrs, bool aggregating) { - r->last_nr_accesses = damon_nr_accesses_for_new_attrs( + r->last_nr_accesses = damon_nr_samples_for_new_attrs( r->last_nr_accesses, old_attrs, new_attrs); if (!aggregating) - r->nr_accesses = damon_nr_accesses_for_new_attrs( + r->nr_accesses = damon_nr_samples_for_new_attrs( r->nr_accesses, old_attrs, new_attrs); else /* From e06b7f0cf8c88c582558839c853689cb3cbdf25b Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 8 Jul 2026 06:53:56 -0700 Subject: [PATCH 287/501] mm/damon/core: update probe hits for new parameter commit Users can update DAMON parameters at runtime. If the samples and/or aggregation intervals are updated in this way, monitoring results depending on the intervals should also be updated for a more accurate snapshot. The age and nr_accesses are properly updated, while probe_hits are not updated in the way. Do the update. Link: https://lore.kernel.org/20260708135359.122587-7-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/core.c | 28 +++++++++++++++++++++++++--- mm/damon/tests/core-kunit.h | 13 ++++++++++--- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 800ff6b6d247..15d31285515d 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -897,10 +897,32 @@ static unsigned int damon_nr_samples_for_new_attrs(unsigned int nr, damon_sample_count_to_bp(nr, old_attrs), new_attrs); } +static void damon_update_probe_hits(struct damon_region *r, + struct damon_attrs *old_attrs, struct damon_attrs *new_attrs, + bool aggregating, struct damon_ctx *ctx) +{ + struct damon_probe *p; + int i = 0; + + damon_for_each_probe(p, ctx) { + r->last_probe_hits[i] = damon_nr_samples_for_new_attrs( + r->last_probe_hits[i], old_attrs, new_attrs); + if (!aggregating) + r->probe_hits[i] = damon_nr_samples_for_new_attrs( + r->probe_hits[i], old_attrs, + new_attrs); + else + r->probe_hits[i] = 0; + i++; + } +} + static void damon_update_monitoring_result(struct damon_region *r, struct damon_attrs *old_attrs, struct damon_attrs *new_attrs, - bool aggregating) + bool aggregating, struct damon_ctx *ctx) { + damon_update_probe_hits(r, old_attrs, new_attrs, aggregating, ctx); + r->last_nr_accesses = damon_nr_samples_for_new_attrs( r->last_nr_accesses, old_attrs, new_attrs); if (!aggregating) @@ -940,8 +962,8 @@ static void damon_update_monitoring_results(struct damon_ctx *ctx, damon_for_each_target(t, ctx) damon_for_each_region(r, t) - damon_update_monitoring_result( - r, old_attrs, new_attrs, aggregating); + damon_update_monitoring_result(r, old_attrs, new_attrs, + aggregating, ctx); } /* diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 278d46632eff..d97280270c27 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -581,32 +581,39 @@ static void damon_test_update_monitoring_result(struct kunit *test) .sample_interval = 10, .aggr_interval = 1000,}; struct damon_attrs new_attrs; struct damon_region *r = damon_new_region(3, 7); + struct damon_ctx *ctx; if (!r) kunit_skip(test, "region alloc fail"); + ctx = damon_new_ctx(); + if (!ctx) { + damon_free_region(r); + kunit_skip(test, "ctx alloc fail"); + } r->nr_accesses = 15; r->age = 20; new_attrs = (struct damon_attrs){ .sample_interval = 100, .aggr_interval = 10000,}; - damon_update_monitoring_result(r, &old_attrs, &new_attrs, false); + damon_update_monitoring_result(r, &old_attrs, &new_attrs, false, ctx); KUNIT_EXPECT_EQ(test, r->nr_accesses, 15); KUNIT_EXPECT_EQ(test, r->age, 2); new_attrs = (struct damon_attrs){ .sample_interval = 1, .aggr_interval = 1000}; - damon_update_monitoring_result(r, &old_attrs, &new_attrs, false); + damon_update_monitoring_result(r, &old_attrs, &new_attrs, false, ctx); KUNIT_EXPECT_EQ(test, r->nr_accesses, 150); KUNIT_EXPECT_EQ(test, r->age, 2); new_attrs = (struct damon_attrs){ .sample_interval = 1, .aggr_interval = 100}; - damon_update_monitoring_result(r, &old_attrs, &new_attrs, false); + damon_update_monitoring_result(r, &old_attrs, &new_attrs, false, ctx); KUNIT_EXPECT_EQ(test, r->nr_accesses, 150); KUNIT_EXPECT_EQ(test, r->age, 20); damon_free_region(r); + damon_destroy_ctx(ctx); } static void damon_test_set_attrs(struct kunit *test) From d59bf2653c15be215257de936da357d7a40ba749 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Wed, 8 Jul 2026 06:53:57 -0700 Subject: [PATCH 288/501] mm/damon/core: handle unreset probe_hits in probe_hits_mvsum() If damon_update_monitoring_result() is called at the end of the aggregation interval, probe_hits is not reset. That's because the value will be exposed to the user via damon_region_aggregated trace event. Meanwhile, damon_probe_hits_mvsum() can be called in this state. Due to its logic, it will return a value that is incorrectly high. This could happen if the user requested DAMOS schemes applied regions sysfs files update exactly in the time sequence. The impact is minor, but better to avoid. Check the timing and simply return the fully aggregated last_probe_hits, like damon_nr_accesses_mvsum() also does. It is not 100% accurate since it is the last interval's aggregation. But better than the value that is completely reset. Link: https://lore.kernel.org/20260708135359.122587-8-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/core.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mm/damon/core.c b/mm/damon/core.c index 15d31285515d..11473aa1f70f 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -290,6 +290,9 @@ unsigned char damon_probe_hits_mvsum(int probe_idx, struct damon_region *r, ctx->passed_sample_intervals; left_window_bp = mult_frac(left_window, 10000, window_len); + if (left_window_bp == 10000) + return r->last_probe_hits[probe_idx]; + return damon_mvsum(r->probe_hits[probe_idx], r->last_probe_hits[probe_idx], left_window_bp); } From a5650de0413f333471fc235f5b1394567f433648 Mon Sep 17 00:00:00 2001 From: xu xin Date: Fri, 3 Jul 2026 16:23:57 +0800 Subject: [PATCH 289/501] ksm: add linear_page_index into ksm_rmap_item MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch series "KSM: performance optimizations for rmap_walk_ksm", v11. This series fixes a severe KSM reverse-mapping performance problem that can freeze applications for hundreds of milliseconds under memory pressure especially when a lot of unrelated VMAs sharing a single anon_vma. Two key highlights: 1. Lock hold time drops from >500ms to <2ms - In our benchmark (20,000 VMAs sharing an anon_vma), worst-case anon_vma lock hold time during KSM rmap walk went from 705ms down to 1.67ms (max) and 1.44ms (avg). 2. Real user impact - The anon_vma lock is also acquired by page faults, reclaim, migration, compaction, mlock, exit_mmap, and cgroup accounting. - A long hold due to inefficient rmap walks stalls application threads, causing latency spikes, reduced throughput, or even container timeouts. - The problem occurs even without fork() – VMA splitting (e.g., via mprotect or madvise over time) can create tens of thousands of VMAs all attached to the same anon_vma. Real-world examples: - JVM / Go runtime: These use mmap for heap regions and later call mprotect(PROT_NONE) for garbage collection barriers or guard pages, splitting the original VMA into thousands of small pieces over time. - Database engines (MySQL, PostgreSQL): Large shared memory buffers or anonymous mappings are managed with madvise(MADV_DONTNEED) to release specific pages, which also splits VMAs. Why the benchmark numbers are realistic: We observed ~20,000 VMAs sharing one anon_vma on a production system running a Java application with KSM enabled. The lock hold time before the patch was measured at 228 ms (max) during rmap walks triggered by memory compaction and page migration. The benchmark reproduces that VMA count and lock‑hold behavior in a controlled environment. For systems that do not have thousands of VMAs per anon_vma, the patch adds negligible overhead (a single pgoff comparison). For systems that do suffer from this issue, the improvement is dramatic: 1) Worst‑case anon_vma lock hold time drops from hundreds of milliseconds to under 2 ms.2)This directly reduces blocking of parallel operations that need the same lock – page faults, reclaim, migration, compaction, mlock, and exit_mmap. End‑users will see lower tail latency (fewer application stalls), higher throughput under memory pressure, and no more spurious lockup warnings or container timeouts caused by excessive lock hold times. In short: workloads that do not hit this pathological pattern are unaffected; those that do will see a 100x to 500x reduction in lock hold times, which translates directly into a more responsive system. This patch (of 3): As preparation for KSM rmap optimizations, let's track the original linear_page_index() of a de-duplicated page in its ksm_rmap_item, so we can efficiently search for the page in an address space, avoiding scanning the entire address space. This was previously discussed in [1, 2]. To avoid growing ksm_rmap_item, let's squeeze it into the existing structure by overlying some members (oldchecksum, age, remaining_skips) that are only relevant while on the unstable tree. The new entry will only be relevant for entries in the stable tree. However, as the age information is read by should_skip_rmap_item() with the smart-scanning approach even while we have an entry in the stable tree, but the page changes (no longer a KSM page, for example due to COW), we have to change the handling there a bit. We'll calculate the linear page index in try_to_merge_with_ksm_page(), when adding it to the stable tree, and reset the index (to reset overlayed data) when removing an item from the stable tree -- in remove_rmap_item_from_tree(), remove_node_from_stable_tree() and break_cow(). To be specially clarified, the reason for resetting the stored index at break_cow() is: - When a page successfully becomes a KSM page (i.e., after stable_tree_append() sets STABLE_FLAG), both anon_vma and the index are stored and remain valid. - However, during the merging process, there are several failure paths where we already prepared an rmap item to be added to the stable tree, but must revert that as some part of the merge process failed. Examples include: 1 The second call to try_to_merge_with_ksm_page() fails in try_to_merge_two_pages(). 2 stable_tree_insert() fails in cmp_and_merge_page(). In such cases, break_cow() is invoked to break the COW mapping and discard the KSM state. Currently, break_cow() already contains a put_anon_vma(rmap_item->anon_vma) to release the reference taken during the aborted merge. Because the index is logically paired with anon_vma (both are only meaningful when the rmap_item is in a stable state), it must also be cleared (or reset) in break_cow() to avoid leaving stale linear_page_index values that could confuse subsequent rmap walks or scanning logic. Link: https://lore.kernel.org/20260703162253688u8Str9eFLR8TGCmo7nIOF@zte.com.cn Link: https://lore.kernel.org/20260703162357853iIa-RP7if9hRlAIuTh5La@zte.com.cn Link: https://lore.kernel.org/all/adTPQSb-qSSHviJN@lucifer/ [1] Link: https://lore.kernel.org/all/202604091806051535BJWZ_FTtdIm3Snk24ei_@zte.com.cn/ [2] Signed-off-by: xu xin Acked-by: David Hildenbrand (Arm) Cc: Chengming Zhou Cc: Hugh Dickins Cc: "Liam R. Howlett" Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Wang Yaxin Signed-off-by: Andrew Morton --- mm/ksm.c | 48 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/mm/ksm.c b/mm/ksm.c index 41ab25aa2a82..ead1abb67495 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -195,22 +195,28 @@ struct ksm_stable_node { * @node: rb node of this rmap_item in the unstable tree * @head: pointer to stable_node heading this list in the stable tree * @hlist: link into hlist of rmap_items hanging off that stable_node - * @age: number of scan iterations since creation - * @remaining_skips: how many scans to skip + * @age: number of scan iterations since creation (unstable node) + * @remaining_skips: how many scans to skip (unstable node) + * @linear_page_index: the original page's index before merged by KSM (stable node) */ struct ksm_rmap_item { struct ksm_rmap_item *rmap_list; union { - struct anon_vma *anon_vma; /* when stable */ + struct anon_vma *anon_vma; /* for reverse mapping, when stable */ #ifdef CONFIG_NUMA int nid; /* when node of unstable tree */ #endif }; struct mm_struct *mm; unsigned long address; /* + low bits used for flags below */ - unsigned int oldchecksum; /* when unstable */ - rmap_age_t age; - rmap_age_t remaining_skips; + union { + struct { + unsigned int oldchecksum; + rmap_age_t age; + rmap_age_t remaining_skips; + }; /* when unstable */ + unsigned long linear_page_index; /* for reverse mapping, when stable */ + }; union { struct rb_node node; /* when node of unstable tree */ struct { /* when listed from stable tree */ @@ -776,6 +782,11 @@ static struct vm_area_struct *find_mergeable_vma(struct mm_struct *mm, return vma; } +/* + * break_cow: actively break COW, replacing the KSM page by a fresh anonymous + * page. This is called when rmap_item has not yet become stable, but page + * has been merged. + */ static void break_cow(struct ksm_rmap_item *rmap_item) { struct mm_struct *mm = rmap_item->mm; @@ -787,6 +798,11 @@ static void break_cow(struct ksm_rmap_item *rmap_item) * to undo, we also need to drop a reference to the anon_vma. */ put_anon_vma(rmap_item->anon_vma); + /* + * Reset linear_page_index that might overlay age-related + * information. (it's still unstable node) + */ + rmap_item->linear_page_index = 0; mmap_read_lock(mm); vma = find_mergeable_vma(mm, addr); @@ -899,6 +915,8 @@ static void remove_node_from_stable_tree(struct ksm_stable_node *stable_node) VM_BUG_ON(stable_node->rmap_hlist_len <= 0); stable_node->rmap_hlist_len--; put_anon_vma(rmap_item->anon_vma); + /* Reset linear_page_index that might overlay age-related information. */ + rmap_item->linear_page_index = 0; rmap_item->address &= PAGE_MASK; cond_resched(); } @@ -1052,6 +1070,8 @@ static void remove_rmap_item_from_tree(struct ksm_rmap_item *rmap_item) stable_node->rmap_hlist_len--; put_anon_vma(rmap_item->anon_vma); + /* Reset linear_page_index that might overlay age-related information. */ + rmap_item->linear_page_index = 0; rmap_item->head = NULL; rmap_item->address &= PAGE_MASK; @@ -1598,8 +1618,15 @@ static int try_to_merge_with_ksm_page(struct ksm_rmap_item *rmap_item, /* Unstable nid is in union with stable anon_vma: remove first */ remove_rmap_item_from_tree(rmap_item); - /* Must get reference to anon_vma while still holding mmap_lock */ + /* + * We can consider the VMA only while still holding the mmap lock, + * so lock, so reference the anon_vma and calculate the linear + * page index early, before stable_tree_append(). If anything goes + * wrong that prevents the rmap_item from being added to the + * stable_tree, break_cow() will clean it up. + */ rmap_item->anon_vma = vma->anon_vma; + rmap_item->linear_page_index = linear_page_index(vma, rmap_item->address); get_anon_vma(vma->anon_vma); out: mmap_read_unlock(mm); @@ -2459,6 +2486,13 @@ static bool should_skip_rmap_item(struct folio *folio, if (folio_test_ksm(folio)) return false; + /* + * There is no age information in stable-tree nodes. We might end up + * here without a KSM page for example after COW. + */ + if (rmap_item->address & STABLE_FLAG) + return false; + age = rmap_item->age; if (age != U8_MAX) rmap_item->age++; From 96d2d9acef497c95e5e781c6ff631ac3958ef2c1 Mon Sep 17 00:00:00 2001 From: xu xin Date: Fri, 3 Jul 2026 16:25:10 +0800 Subject: [PATCH 290/501] ksm: optimize rmap_walk_ksm by passing a suitable page index User impact / Why this matters to Linux users ============================================= When a system runs with KSM enabled and memory becomes tight, KSM pages may be swapped out or migrated. The kernel then performs a reverse map walk by rmap_walk_ksm to locate all page table entries that reference these pages. If A large number of unrelated VMAs can attach to a single anon_vma related with this KSM page, then rmap_walk might be severe performance bottleneck. In our embedded test environment, we observed ~20,000 VMAs sharing one anon_vma without any fork purely from VMA splits which cause 200~700ms duration of rmap_walk_ksm. When one of those VMAs mapped a KSM page, then this KSM page's rmapping will become bottleneck with hold its anon_vma lock for a long time. The anon_vma lock is not only used by KSM; it is a core lock protecting the VMA interval tree and is acquired by many critical memory operations: ' Page faults: do_anonymous_page(), do_wp_page() (during COW) ' Memory reclaim: try_to_unmap() ' Page migration & compaction: migrate_pages(), compact_zone() ' mlock / munlock: mlock_fixup() ' Process exit: exit_mmap() (tearing down VMAs) ' Cgroup memory accounting: mem_cgroup_move_charge() If one thread holds the anon_vma lock for hundreds of milliseconds because of an inefficient KSM rmap walk, any other thread that tries to acquire the same lock (e.g., an application taking a page fault, kswapd reclaiming pages, or a migration thread) will block. This leads to stalled application threads, increased latency spikes, and in extreme cases container timeouts or watchdog triggers. This patch reduces the worst-case anon_vma lock hold time during ksm_rmap_walk from >500 ms to <1 ms, thereby almost eliminating this source of lock contention and improving system responsiveness under memory pressure. Real-world examples: ==================== - JVM / Go runtime: These use mmap for heap regions and later call mprotect(PROT_NONE) for garbage collection barriers or guard pages, splitting the original VMA into thousands of small pieces over time. - Database engines (MySQL, PostgreSQL): Large shared memory buffers or anonymous mappings are managed with madvise(MADV_DONTNEED) to release specific pages, which also splits VMAs. Root Cause ========== Through local debugging trace analysis, we found that most of the latency of rmap_walk_ksm occurs within anon_vma_interval_tree_foreach, leading to an excessively long hold time on the anon_vma lock (even reaching 500ms or more), which in turn causes upper-layer applications (waiting for the anon_vma lock) to be blocked for extended periods. Further investigation revealed that 99.9% of iterations inside the anon_vma_interval_tree_foreach loop are skipped due to the first check "if (addr < vma->vm_start || addr >= vma->vm_end)), indicating that a large number of loop iterations are ineffective. This inefficiency arises because the start page index and the end page index parameters passed to anon_vma_interval_tree_foreach span the entire address space from 0 to ULONG_MAX, resulting in very poor loop efficiency. Solution ======== We cannot rely solely on anon_vma to locate all PTEs mapping this page but also need to have the original page's linear_page_index. Since the implementation of anon_vma_interval_tree_foreach it essentially iterates to find a suitable VMA such that the provided page index falls within the candidate's vm_pgoff range. vm_pgoff <= original linear page offset <= (vm_pgoff + vma_pages(v) - 1) Fortunately, an earlier commit introduced the linear_page_index to struct ksm_rmap_item, allowing for optimizing the RMAP walk. Test results ============ A rmap testbench can be obtained with two Out-Of-Tree patches at [1][2]. After applying the OOT patches and building rmap_benchmark from: tools/testing/rmap/rmap_benchmark.c, we can start the performance test. The testing result in QEMU is shown as follows: KSM rmapping Maximum duration Average duration Before: 705.12 ms (705119858 ns) 532.04 ms (532041586 ns) After: 1.67 ms (1665917 ns) 1.44 ms (1443784 ns) The benchmark numbers are realistic, since we observed ~20,000 VMAs sharing one anon_vma on a production system running a Java application with KSM enabled. The lock hold time before the patch was measured at 228ms (max) during rmap walks triggered by memory compaction and page migration. The benchmark reproduces that VMA count and lockhold behavior in a controlled environment. Link: https://lore.kernel.org/20260703162510242nxmjbcLy5ccp1dbZSK3EU@zte.com.cn Link: https://lore.kernel.org/all/202605301703094695zmVgcSC27BNR0rH0N8_x@zte.com.cn [1] Link: https://lore.kernel.org/all/20260530170404509QpJmBtpSjn3uQHeVKA2iA@zte.com.cn/ [2] Co-developed-by: Wang Yaxin Signed-off-by: Wang Yaxin Signed-off-by: xu xin Acked-by: David Hildenbrand (Arm) Cc: Chengming Zhou Cc: Hugh Dickins Cc: "Liam R. Howlett" Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/ksm.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/mm/ksm.c b/mm/ksm.c index ead1abb67495..c1dcc3cb4831 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -3208,6 +3208,7 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc) hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) { /* Ignore the stable/unstable/sqnr flags */ const unsigned long addr = rmap_item->address & PAGE_MASK; + const unsigned long index = rmap_item->linear_page_index; struct anon_vma *anon_vma = rmap_item->anon_vma; struct anon_vma_chain *vmac; struct vm_area_struct *vma; @@ -3221,8 +3222,18 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc) anon_vma_lock_read(anon_vma); } + /* + * Currently, KSM folios are always small folios, so it's + * sufficient to search for a single page. We can simply use + * the linear_page_index of the original de-duplicate + * anonymous page that we remembered in the rmap_item while + * de-duplicating. Note that mremap() always de-duplicates KSM + * folios: so if there was mremap() in our parent or our child, + * we wouldn't have the KSM folio mapped in these processes + * anymore. + */ anon_vma_interval_tree_foreach(vmac, &anon_vma->rb_root, - 0, ULONG_MAX) { + index, index) { cond_resched(); vma = vmac->vma; From 1695841621d7370ee13b82fd03294d00882b12d6 Mon Sep 17 00:00:00 2001 From: xu xin Date: Fri, 3 Jul 2026 16:26:37 +0800 Subject: [PATCH 291/501] ksm: add mremap selftests for ksm_rmap_walk The existing tools/testing/selftests/mm/rmap.c has already one testcase for ksm_rmap_walk in TEST_F(migrate, ksm), which takes use of migration of page from one NUMA node to another NUMA node. However, it just lacks the scenario of mremapped VMAs. We add the calling of mremap() and then trigger KSM to merge pages before migrating, which is specifically to test an optimization which is introduced by this patch ("ksm: Optimize rmap_walk_ksm by passing a suitable address pgoff"). This test can reproduce the issue that Hugh points out at https://lore.kernel.org/all/02e1b8df-d568-8cbb-b8f6-46d5476d9d75@google.com/ Link: https://lore.kernel.org/20260703162637070FU4ekl58Hw_Z7OSuJryZB@zte.com.cn Signed-off-by: xu xin Acked-by: David Hildenbrand (Arm) Cc: Chengming Zhou Cc: Hugh Dickins Cc: "Liam R. Howlett" Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Wang Yaxin Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/rmap.c | 81 +++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/tools/testing/selftests/mm/rmap.c b/tools/testing/selftests/mm/rmap.c index 53f2058b0ef2..1c293ad3f8b8 100644 --- a/tools/testing/selftests/mm/rmap.c +++ b/tools/testing/selftests/mm/rmap.c @@ -430,4 +430,85 @@ TEST_F(migrate, ksm) propagate_children(_metadata, data); } +static bool range_maps_the_same_pfn(int pagemap_fd, void *region, int nr_pages) +{ + int i; + int retries = 0; + unsigned long first_pfn; + +retry: + if (retries > 10) + return false; + + first_pfn = pagemap_get_pfn(pagemap_fd, region); + for (i = 0; i < nr_pages; i++) { + if (pagemap_get_pfn(pagemap_fd, region + i * getpagesize()) != first_pfn) { + /* + * Retry up to 10 times at most in case of the low chance of page + * compaction migrating the page while we check for pfn. + */ + retries++; + goto retry; + } + } + + return true; +} + +TEST_F(migrate, ksm_and_mremap) +{ + unsigned long old_pfn, new_pfn; + void *region, *mremap_region; + const int nr_pages = 16; + size_t mmap_size; + int pagemap_fd; + + /* Skip if KSM is not available */ + if (ksm_stop() < 0) + SKIP(return, "accessing \"/sys/kernel/mm/ksm/run\" failed"); + if (ksm_get_full_scans() < 0) + SKIP(return, "accessing \"/sys/kernel/mm/ksm/full_scan\" failed"); + + pagemap_fd = open("/proc/self/pagemap", O_RDONLY); + if (pagemap_fd < 0) + SKIP(return, "opening pagemap failed"); + + /* Allocate and populate twice the anon pages initially. */ + mmap_size = 2 * nr_pages * getpagesize(); + region = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, -1, 0); + ASSERT_NE(region, MAP_FAILED); + memset(region, 0x77, mmap_size); + + /* mremap the second half over the first half, to stress rmap handling */ + mmap_size /= 2; + mremap_region = mremap(region + mmap_size, mmap_size, mmap_size, + MREMAP_MAYMOVE | MREMAP_FIXED, region); + ASSERT_EQ(mremap_region, region); + + /* Merge all pages into a single KSM page. */ + madvise(region, mmap_size, MADV_MERGEABLE); + ASSERT_EQ(ksm_start(), 0); + + /* The whole range should map the same KSM page. */ + old_pfn = pagemap_get_pfn(pagemap_fd, region); + if (old_pfn == -1ul) + SKIP(return, "Obtaining PFN failed"); + ksm_start(); + ASSERT_TRUE(range_maps_the_same_pfn(pagemap_fd, region, nr_pages)); + + /* + * Migrate the KSM page; the whole range should map the new (migrated) + * KSM page. + */ + ASSERT_EQ(try_to_move_page(region), 0); + + new_pfn = pagemap_get_pfn(pagemap_fd, region); + if (new_pfn == -1ul) + SKIP(return, "Obtaining PFN failed"); + ASSERT_NE(new_pfn, old_pfn); + ASSERT_TRUE(range_maps_the_same_pfn(pagemap_fd, region, nr_pages)); +} + + TEST_HARNESS_MAIN From 55ed40abb2cd6c2a94adea43002be61faf4081e6 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Thu, 9 Jul 2026 13:00:03 +0300 Subject: [PATCH 292/501] mm: split out mm_init and memblock declarations from internal.h Patch series "mm: split a couple of headers from internal.h", v2. mm/internal.h becomes more and more bloated. Split declarations related to mm_init, memblock, vmalloc and sparse into new headers. This patch (of 3): mm/internal.h becomes more and more bloated. Move declarations for related to mm/mm_init.c and mm/memblock.c to a new mm/mm_init.h header. No functional changes. [rppt@kernel.org: split stubfs from internal.h to mm_init.h] Link: https://lore.kernel.org/alJd1BLypyK9Mpaw@kernel.org Link: https://lore.kernel.org/20260709-internal-h-v2-0-695631425968@kernel.org Link: https://lore.kernel.org/20260709-internal-h-v2-1-695631425968@kernel.org Signed-off-by: Mike Rapoport (Microsoft) Acked-by: Muchun Song Acked-by: Vlastimil Babka (SUSE) Acked-by: David Hildenbrand (Arm) Acked-by: Lorenzo Stoakes Acked-by: Pratyush Yadav Acked-by: SJ Park Cc: Alexander Graf Cc: Alexander Potapenko Cc: Brendan Jackman Cc: Brendan Jackman Cc: Dennis Zhou Cc: Dmitry Vyukov Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Marco Elver Cc: Michal Hocko Cc: Oscar Salvador Cc: Pasha Tatashin Cc: Suren Baghdasaryan Cc: Tejun Heo Cc: "Uladzislau Rezki (Sony)" Cc: Zi Yan Signed-off-by: Andrew Morton --- MAINTAINERS | 1 + kernel/liveupdate/kexec_handover.c | 1 + mm/cma.c | 1 + mm/hugetlb.c | 1 + mm/internal.h | 91 ---------------------- mm/memblock.c | 1 + mm/memory_hotplug.c | 1 + mm/mm_init.c | 1 + mm/mm_init.h | 120 +++++++++++++++++++++++++++++ mm/page_alloc.c | 1 + mm/sparse-vmemmap.c | 1 + mm/sparse.c | 1 + tools/testing/memblock/internal.h | 20 ----- tools/testing/memblock/mm_init.h | 24 ++++++ 14 files changed, 154 insertions(+), 111 deletions(-) create mode 100644 mm/mm_init.h create mode 100644 tools/testing/memblock/mm_init.h diff --git a/MAINTAINERS b/MAINTAINERS index 8acd1aeec9db..3be6db146cb9 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -16883,6 +16883,7 @@ F: include/linux/memblock.h F: mm/memblock.c F: mm/memtest.c F: mm/mm_init.c +F: mm/mm_init.h F: mm/rodata_test.c F: tools/testing/memblock/ diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c index 4834a809985a..fe11cbf1006e 100644 --- a/kernel/liveupdate/kexec_handover.c +++ b/kernel/liveupdate/kexec_handover.c @@ -35,6 +35,7 @@ * internal APIs. */ #include "../../mm/internal.h" +#include "../../mm/mm_init.h" #include "../kexec_internal.h" #include "kexec_handover_internal.h" diff --git a/mm/cma.c b/mm/cma.c index 31073738f2ac..a7929c758df1 100644 --- a/mm/cma.c +++ b/mm/cma.c @@ -33,6 +33,7 @@ #include "internal.h" #include "cma.h" +#include "mm_init.h" struct cma cma_areas[MAX_CMA_AREAS]; unsigned int cma_area_count; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index dcaca9309015..09c5783a3214 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -51,6 +51,7 @@ #include "hugetlb_vmemmap.h" #include "hugetlb_cma.h" #include "hugetlb_internal.h" +#include "mm_init.h" #include int hugetlb_max_hstate __read_mostly; diff --git a/mm/internal.h b/mm/internal.h index 3d30650a8440..59adb140fc7a 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -716,9 +716,6 @@ void set_recommended_min_free_kbytes(void); extern char * const zone_names[MAX_NR_ZONES]; -/* perform sanity checks on struct pages being allocated or freed */ -DECLARE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled); - extern int min_free_kbytes; extern int defrag_mode; @@ -726,19 +723,9 @@ void setup_per_zone_wmarks(void); void calculate_min_free_kbytes(void); int __meminit init_per_zone_wmark_min(void); -void set_zone_contiguous(struct zone *zone); -bool pfn_range_intersects_zones(int nid, unsigned long start_pfn, - unsigned long nr_pages); - -static inline void clear_zone_contiguous(struct zone *zone) -{ - zone->contiguous = false; -} - extern int __isolate_free_page(struct page *page, unsigned int order); extern void __putback_isolated_page(struct page *page, unsigned int order, int mt); -extern void memblock_free_pages(unsigned long pfn, unsigned int order); /* * This will have no effect, other than possibly generating a warning, if the @@ -820,14 +807,6 @@ static inline void init_compound_tail(struct page *tail, prep_compound_tail(tail, head, order); } -extern void *memmap_alloc(phys_addr_t size, phys_addr_t align, - phys_addr_t min_addr, - int nid, bool exact_nid); - -void memmap_init_range(unsigned long, int, unsigned long, unsigned long, - unsigned long, enum meminit_context, struct vmem_altmap *, int, - bool); - /* * mm/sparse.c */ @@ -948,9 +927,6 @@ int isolate_migratepages_range(struct compact_control *cc, unsigned long low_pfn, unsigned long end_pfn); -/* Free whole pageblock and set its migration type to MIGRATE_CMA. */ -void init_cma_reserved_pageblock(struct page *page); - #endif /* CONFIG_COMPACTION || CONFIG_CMA */ struct cma; @@ -958,7 +934,6 @@ struct cma; #ifdef CONFIG_CMA bool cma_validate_zones(struct cma *cma); void *cma_reserve_early(struct cma *cma, unsigned long size); -void init_cma_pageblock(struct page *page); #else static inline bool cma_validate_zones(struct cma *cma) { @@ -968,9 +943,6 @@ static inline void *cma_reserve_early(struct cma *cma, unsigned long size) { return NULL; } -static inline void init_cma_pageblock(struct page *page) -{ -} #endif /* mm/util.c */ @@ -1177,63 +1149,6 @@ static inline void vunmap_range_noflush(unsigned long start, unsigned long end) } #endif /* !CONFIG_MMU */ -/* Memory initialisation debug and verification */ -#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT -DECLARE_STATIC_KEY_TRUE(deferred_pages); - -static inline bool deferred_pages_enabled(void) -{ - return static_branch_unlikely(&deferred_pages); -} - -bool __init deferred_grow_zone(struct zone *zone, unsigned int order); -#else -static inline bool deferred_pages_enabled(void) -{ - return false; -} -#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ - -void init_deferred_page(unsigned long pfn, int nid); - -enum mminit_level { - MMINIT_WARNING, - MMINIT_VERIFY, - MMINIT_TRACE -}; - -#ifdef CONFIG_DEBUG_MEMORY_INIT - -extern int mminit_loglevel; - -#define mminit_dprintk(level, prefix, fmt, arg...) \ -do { \ - if (level < mminit_loglevel) { \ - if (level <= MMINIT_WARNING) \ - pr_warn("mminit::" prefix " " fmt, ##arg); \ - else \ - printk(KERN_DEBUG "mminit::" prefix " " fmt, ##arg); \ - } \ -} while (0) - -extern void mminit_verify_pageflags_layout(void); -extern void mminit_verify_zonelist(void); -#else - -static inline void mminit_dprintk(enum mminit_level level, - const char *prefix, const char *fmt, ...) -{ -} - -static inline void mminit_verify_pageflags_layout(void) -{ -} - -static inline void mminit_verify_zonelist(void) -{ -} -#endif /* CONFIG_DEBUG_MEMORY_INIT */ - #define NODE_RECLAIM_NOSCAN -2 #define NODE_RECLAIM_FULL -1 #define NODE_RECLAIM_SOME 0 @@ -1532,9 +1447,6 @@ static inline bool gup_must_unshare(struct vm_area_struct *vma, return !PageAnonExclusive(page); } -extern bool mirrored_kernelcore; -bool memblock_has_mirror(void); -void memblock_free_all(void); static __always_inline void vma_set_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, @@ -1573,9 +1485,6 @@ static inline bool pte_needs_soft_dirty_wp(struct vm_area_struct *vma, pte_t pte return vma_soft_dirty_enabled(vma) && !pte_soft_dirty(pte); } -void __meminit __init_single_page(struct page *page, unsigned long pfn, - unsigned long zone, int nid); - /* shrinker related functions */ unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg, int priority); diff --git a/mm/memblock.c b/mm/memblock.c index 6349c48154f4..777c69f05400 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -29,6 +29,7 @@ #include #include "internal.h" +#include "mm_init.h" #define INIT_MEMBLOCK_REGIONS 128 #define INIT_PHYSMEM_REGIONS 4 diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 11ab2f7bc7f3..99090f49d2df 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -40,6 +40,7 @@ #include #include "internal.h" +#include "mm_init.h" #include "page_alloc.h" #include "shuffle.h" diff --git a/mm/mm_init.c b/mm/mm_init.c index 537664974ab1..39583c028e34 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -33,6 +33,7 @@ #include #include #include "internal.h" +#include "mm_init.h" #include "page_alloc.h" #include "slab.h" #include "shuffle.h" diff --git a/mm/mm_init.h b/mm/mm_init.h new file mode 100644 index 000000000000..39f75df9be1c --- /dev/null +++ b/mm/mm_init.h @@ -0,0 +1,120 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * mm_init.h: + * + * mm/ internal mm_init and memblock declarations + */ + +#ifndef __MM_MM_INIT_H +#define __MM_MM_INIT_H + +#include +#include +#include +#include +#include + +struct page; +struct vmem_altmap; + +/* perform sanity checks on struct pages being allocated or freed */ +DECLARE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled); + +void set_zone_contiguous(struct zone *zone); +bool pfn_range_intersects_zones(int nid, unsigned long start_pfn, + unsigned long nr_pages); + +static inline void clear_zone_contiguous(struct zone *zone) +{ + zone->contiguous = false; +} + +void memblock_free_pages(unsigned long pfn, unsigned int order); + +void *memmap_alloc(phys_addr_t size, phys_addr_t align, phys_addr_t min_addr, + int nid, bool exact_nid); + +void memmap_init_range(unsigned long size, int nid, unsigned long zone, + unsigned long start_pfn, unsigned long zone_end_pfn, + enum meminit_context context, + struct vmem_altmap *altmap, int migratetype, + bool isolate_pageblock); + +#if defined CONFIG_COMPACTION || defined CONFIG_CMA +/* Free whole pageblock and set its migration type to MIGRATE_CMA. */ +void init_cma_reserved_pageblock(struct page *page); +#endif + +#ifdef CONFIG_CMA +void init_cma_pageblock(struct page *page); +#else +static inline void init_cma_pageblock(struct page *page) +{ +} +#endif + +/* Memory initialisation debug and verification */ +#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT +DECLARE_STATIC_KEY_TRUE(deferred_pages); + +static inline bool deferred_pages_enabled(void) +{ + return static_branch_unlikely(&deferred_pages); +} + +bool __init deferred_grow_zone(struct zone *zone, unsigned int order); +#else +static inline bool deferred_pages_enabled(void) +{ + return false; +} +#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ + +void init_deferred_page(unsigned long pfn, int nid); + +enum mminit_level { + MMINIT_WARNING, + MMINIT_VERIFY, + MMINIT_TRACE +}; + +#ifdef CONFIG_DEBUG_MEMORY_INIT + +extern int mminit_loglevel; + +#define mminit_dprintk(level, prefix, fmt, arg...) \ +do { \ + if (level < mminit_loglevel) { \ + if (level <= MMINIT_WARNING) \ + pr_warn("mminit::" prefix " " fmt, ##arg); \ + else \ + printk(KERN_DEBUG "mminit::" prefix " " fmt, ##arg); \ + } \ +} while (0) + +void mminit_verify_pageflags_layout(void); +void mminit_verify_zonelist(void); +#else + +static inline void mminit_dprintk(enum mminit_level level, + const char *prefix, const char *fmt, ...) +{ +} + +static inline void mminit_verify_pageflags_layout(void) +{ +} + +static inline void mminit_verify_zonelist(void) +{ +} +#endif /* CONFIG_DEBUG_MEMORY_INIT */ + +extern bool mirrored_kernelcore; +bool memblock_has_mirror(void); +void memblock_free_all(void); + +void __meminit __init_single_page(struct page *page, unsigned long pfn, + unsigned long zone, int nid); + +#endif /* __MM_MM_INIT_H */ diff --git a/mm/page_alloc.c b/mm/page_alloc.c index f61e4203eced..718aa08c3c97 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -56,6 +56,7 @@ #include #include #include "internal.h" +#include "mm_init.h" #include "page_alloc.h" #include "shuffle.h" #include "page_reporting.h" diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 1acc2924d814..49571676b860 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -41,6 +41,7 @@ #define VMEMMAP_POPULATE_PAGEREF 0x0001 #include "internal.h" +#include "mm_init.h" /* * Allocate a block of memory to be used to back the virtual memory map diff --git a/mm/sparse.c b/mm/sparse.c index b5c74aab3379..d54e30f58b93 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -15,6 +15,7 @@ #include #include #include "internal.h" +#include "mm_init.h" #include /* diff --git a/tools/testing/memblock/internal.h b/tools/testing/memblock/internal.h index b6b1d147fd75..4168e646af09 100644 --- a/tools/testing/memblock/internal.h +++ b/tools/testing/memblock/internal.h @@ -28,22 +28,6 @@ static inline struct page *virt_to_page(void *virt) return virt; } -void memblock_free_pages(unsigned long pfn, unsigned int order) -{ -} - -static inline void accept_memory(phys_addr_t start, unsigned long size) -{ -} - -unsigned long free_reserved_area(void *start, void *end, int poison, const char *s); -void free_reserved_page(struct page *page); - -static inline bool deferred_pages_enabled(void) -{ - return false; -} - #define for_each_valid_pfn(pfn, start_pfn, end_pfn) \ for ((pfn) = (start_pfn); (pfn) < (end_pfn); (pfn)++) @@ -60,10 +44,6 @@ static inline bool __is_kernel(unsigned long addr) #define for_each_valid_pfn(pfn, start_pfn, end_pfn) \ for ((pfn) = (start_pfn); (pfn) < (end_pfn); (pfn)++) -static inline void init_deferred_page(unsigned long pfn, int nid) -{ -} - #define __SetPageReserved(p) ((void)(p)) #endif diff --git a/tools/testing/memblock/mm_init.h b/tools/testing/memblock/mm_init.h new file mode 100644 index 000000000000..95bc5f2e8ed8 --- /dev/null +++ b/tools/testing/memblock/mm_init.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef __MM_MM_INIT_H +#define __MM_MM_INIT_H + +void memblock_free_pages(unsigned long pfn, unsigned int order) +{ +} + +static inline void accept_memory(phys_addr_t start, unsigned long size) +{ +} + +unsigned long free_reserved_area(void *start, void *end, int poison, const char *s); +void free_reserved_page(struct page *page); + +static inline bool deferred_pages_enabled(void) +{ + return false; +} + +static inline void init_deferred_page(unsigned long pfn, int nid) +{ +} +#endif /* __MM_MM_INIT_H */ From 2b65a42a0883e440d135945377a16d9b86330506 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Thu, 9 Jul 2026 13:00:04 +0300 Subject: [PATCH 293/501] mm: split out sparse declarations from internal.h mm/internal.h becomes more and more bloated. Move declarations related to SPARSE and SPARSE_VMEMMAP memory models to a new mm/sparse.h header. No functional changes. Link: https://lore.kernel.org/20260709-internal-h-v2-2-695631425968@kernel.org Signed-off-by: Mike Rapoport (Microsoft) Acked-by: Muchun Song Acked-by: Vlastimil Babka (SUSE) Acked-by: David Hildenbrand (Arm) Acked-by: Lorenzo Stoakes Acked-by: Pratyush Yadav Acked-by: SJ Park Cc: Alexander Graf Cc: Alexander Potapenko Cc: Brendan Jackman Cc: Brendan Jackman Cc: Dennis Zhou Cc: Dmitry Vyukov Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Marco Elver Cc: Michal Hocko Cc: Oscar Salvador Cc: Pasha Tatashin Cc: Suren Baghdasaryan Cc: Tejun Heo Cc: "Uladzislau Rezki (Sony)" Cc: Zi Yan Signed-off-by: Andrew Morton --- MAINTAINERS | 1 + mm/internal.h | 52 ------------------------------------- mm/mm_init.c | 1 + mm/sparse-vmemmap.c | 1 + mm/sparse.c | 1 + mm/sparse.h | 63 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 67 insertions(+), 52 deletions(-) create mode 100644 mm/sparse.h diff --git a/MAINTAINERS b/MAINTAINERS index 3be6db146cb9..16babc72f7c1 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17001,6 +17001,7 @@ F: mm/pgtable-generic.c F: mm/ptdump.c F: mm/sparse-vmemmap.c F: mm/sparse.c +F: mm/sparse.h F: mm/util.c F: mm/vmpressure.c F: mm/vmstat.c diff --git a/mm/internal.h b/mm/internal.h index 59adb140fc7a..a29a7dab2e6c 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -807,58 +807,6 @@ static inline void init_compound_tail(struct page *tail, prep_compound_tail(tail, head, order); } -/* - * mm/sparse.c - */ -#ifdef CONFIG_SPARSEMEM -void sparse_init(void); -int sparse_index_init(unsigned long section_nr, int nid); - -static inline void sparse_init_one_section(struct mem_section *ms, - unsigned long pnum, struct page *mem_map, - struct mem_section_usage *usage, unsigned long flags) -{ - unsigned long coded_mem_map; - - BUILD_BUG_ON(SECTION_MAP_LAST_BIT > PFN_SECTION_SHIFT); - - /* - * We encode the start PFN of the section into the mem_map such that - * page_to_pfn() on !CONFIG_SPARSEMEM_VMEMMAP can simply subtract it - * from the page pointer to obtain the PFN. - */ - coded_mem_map = (unsigned long)(mem_map - section_nr_to_pfn(pnum)); - VM_WARN_ON_ONCE(coded_mem_map & ~SECTION_MAP_MASK); - - ms->section_mem_map &= ~SECTION_MAP_MASK; - ms->section_mem_map |= coded_mem_map; - ms->section_mem_map |= flags | SECTION_HAS_MEM_MAP; - ms->usage = usage; -} - -static inline void __section_mark_present(struct mem_section *ms, - unsigned long section_nr) -{ - if (section_nr > __highest_present_section_nr) - __highest_present_section_nr = section_nr; - - ms->section_mem_map |= SECTION_MARKED_PRESENT; -} -#else -static inline void sparse_init(void) {} -#endif /* CONFIG_SPARSEMEM */ - -/* - * mm/sparse-vmemmap.c - */ -#ifdef CONFIG_SPARSEMEM_VMEMMAP -void sparse_init_subsection_map(void); -#else -static inline void sparse_init_subsection_map(void) -{ -} -#endif /* CONFIG_SPARSEMEM_VMEMMAP */ - #if defined CONFIG_COMPACTION || defined CONFIG_CMA /* diff --git a/mm/mm_init.c b/mm/mm_init.c index 39583c028e34..b557e5fe35f3 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -35,6 +35,7 @@ #include "internal.h" #include "mm_init.h" #include "page_alloc.h" +#include "sparse.h" #include "slab.h" #include "shuffle.h" diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 49571676b860..5a2469fb1838 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -42,6 +42,7 @@ #include "internal.h" #include "mm_init.h" +#include "sparse.h" /* * Allocate a block of memory to be used to back the virtual memory map diff --git a/mm/sparse.c b/mm/sparse.c index d54e30f58b93..704a9dec2b9a 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -16,6 +16,7 @@ #include #include "internal.h" #include "mm_init.h" +#include "sparse.h" #include /* diff --git a/mm/sparse.h b/mm/sparse.h new file mode 100644 index 000000000000..95aa031213f2 --- /dev/null +++ b/mm/sparse.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * sparse.h: + * + * mm/ internal sparse and sparse-vmemmap declarations + */ + +#ifndef __MM_SPARSE_H +#define __MM_SPARSE_H + +#include + +/* + * mm/sparse.c + */ +#ifdef CONFIG_SPARSEMEM +void sparse_init(void); +int sparse_index_init(unsigned long section_nr, int nid); + +static inline void sparse_init_one_section(struct mem_section *ms, + unsigned long pnum, struct page *mem_map, + struct mem_section_usage *usage, unsigned long flags) +{ + unsigned long coded_mem_map; + + BUILD_BUG_ON(SECTION_MAP_LAST_BIT > PFN_SECTION_SHIFT); + + /* + * We encode the start PFN of the section into the mem_map such that + * page_to_pfn() on !CONFIG_SPARSEMEM_VMEMMAP can simply subtract it + * from the page pointer to obtain the PFN. + */ + coded_mem_map = (unsigned long)(mem_map - section_nr_to_pfn(pnum)); + VM_WARN_ON_ONCE(coded_mem_map & ~SECTION_MAP_MASK); + + ms->section_mem_map &= ~SECTION_MAP_MASK; + ms->section_mem_map |= coded_mem_map; + ms->section_mem_map |= flags | SECTION_HAS_MEM_MAP; + ms->usage = usage; +} + +static inline void __section_mark_present(struct mem_section *ms, + unsigned long section_nr) +{ + if (section_nr > __highest_present_section_nr) + __highest_present_section_nr = section_nr; + + ms->section_mem_map |= SECTION_MARKED_PRESENT; +} +#else +static inline void sparse_init(void) {} +#endif /* CONFIG_SPARSEMEM */ + +/* + * mm/sparse-vmemmap.c + */ +#ifdef CONFIG_SPARSEMEM_VMEMMAP +void sparse_init_subsection_map(void); +#else +static inline void sparse_init_subsection_map(void) {} +#endif /* CONFIG_SPARSEMEM_VMEMMAP */ + +#endif /* __MM_SPARSE_H */ From ef79e0f5e31b870025f209101363655feb6f8e67 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Thu, 9 Jul 2026 13:00:05 +0300 Subject: [PATCH 294/501] mm: split out vmalloc declarations from internal.h mm/internal.h becomes more and more bloated. Move declarations related to vmalloc to a new mm/vmalloc.h header. No functional changes. Link: https://lore.kernel.org/20260709-internal-h-v2-3-695631425968@kernel.org Signed-off-by: Mike Rapoport (Microsoft) Acked-by: Muchun Song Acked-by: Vlastimil Babka (SUSE) Acked-by: David Hildenbrand (Arm) Acked-by: Lorenzo Stoakes Acked-by: Pratyush Yadav Acked-by: SJ Park Cc: Alexander Graf Cc: Alexander Potapenko Cc: Brendan Jackman Cc: Brendan Jackman Cc: Dennis Zhou Cc: Dmitry Vyukov Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Marco Elver Cc: Michal Hocko Cc: Oscar Salvador Cc: Pasha Tatashin Cc: Suren Baghdasaryan Cc: Tejun Heo Cc: "Uladzislau Rezki (Sony)" Cc: Zi Yan Signed-off-by: Andrew Morton --- MAINTAINERS | 1 + kernel/liveupdate/kexec_handover.c | 2 +- mm/execmem.c | 1 + mm/internal.h | 40 -------------------------- mm/kmsan/hooks.c | 1 + mm/kmsan/shadow.c | 1 + mm/mm_init.c | 1 + mm/percpu-vm.c | 1 + mm/vmalloc.c | 1 + mm/vmalloc.h | 46 ++++++++++++++++++++++++++++++ 10 files changed, 54 insertions(+), 41 deletions(-) create mode 100644 mm/vmalloc.h diff --git a/MAINTAINERS b/MAINTAINERS index 16babc72f7c1..ec4f60692bde 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -28818,6 +28818,7 @@ W: http://www.linux-mm.org T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm F: include/linux/vmalloc.h F: mm/vmalloc.c +F: mm/vmalloc.h F: lib/test_vmalloc.c VME SUBSYSTEM diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c index fe11cbf1006e..78ff267bf51e 100644 --- a/kernel/liveupdate/kexec_handover.c +++ b/kernel/liveupdate/kexec_handover.c @@ -34,8 +34,8 @@ * KHO is tightly coupled with mm init and needs access to some of mm * internal APIs. */ -#include "../../mm/internal.h" #include "../../mm/mm_init.h" +#include "../../mm/vmalloc.h" #include "../kexec_internal.h" #include "kexec_handover_internal.h" diff --git a/mm/execmem.c b/mm/execmem.c index 084a207e4278..74a178a87e75 100644 --- a/mm/execmem.c +++ b/mm/execmem.c @@ -20,6 +20,7 @@ #include #include "internal.h" +#include "vmalloc.h" static struct execmem_info *execmem_info __ro_after_init; static struct execmem_info default_execmem_info __ro_after_init; diff --git a/mm/internal.h b/mm/internal.h index a29a7dab2e6c..e5dcab69eb4f 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1092,9 +1092,6 @@ static inline void mlock_new_folio(struct folio *folio) { } static inline bool need_mlock_drain(int cpu) { return false; } static inline void mlock_drain_local(void) { } static inline void mlock_drain_remote(int cpu) { } -static inline void vunmap_range_noflush(unsigned long start, unsigned long end) -{ -} #endif /* !CONFIG_MMU */ #define NODE_RECLAIM_NOSCAN -2 @@ -1209,37 +1206,6 @@ struct migration_target_control { size_t splice_folio_into_pipe(struct pipe_inode_info *pipe, struct folio *folio, loff_t fpos, size_t size); -/* - * mm/vmalloc.c - */ -#ifdef CONFIG_MMU -void __init vmalloc_init(void); -int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end, - pgprot_t prot, struct page **pages, unsigned int page_shift, gfp_t gfp_mask); -unsigned int get_vm_area_page_order(struct vm_struct *vm); -#else -static inline void vmalloc_init(void) -{ -} - -static inline -int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end, - pgprot_t prot, struct page **pages, unsigned int page_shift, gfp_t gfp_mask) -{ - return -EINVAL; -} -#endif - -void clear_vm_uninitialized_flag(struct vm_struct *vm); - -int __must_check __vmap_pages_range_noflush(unsigned long addr, - unsigned long end, pgprot_t prot, - struct page **pages, unsigned int page_shift); - -void vunmap_range_noflush(unsigned long start, unsigned long end); - -void __vunmap_range_noflush(unsigned long start, unsigned long end); - static inline bool vma_is_single_threaded_private(struct vm_area_struct *vma) { if (vma->vm_flags & VM_SHARED) @@ -1267,12 +1233,6 @@ int numa_migrate_check(struct folio *folio, struct vm_fault *vmf, void free_zone_device_folio(struct folio *folio); int migrate_device_coherent_folio(struct folio *folio); -struct vm_struct *__get_vm_area_node(unsigned long size, - unsigned long align, unsigned long shift, - unsigned long vm_flags, unsigned long start, - unsigned long end, int node, gfp_t gfp_mask, - const void *caller); - /* * mm/gup.c */ diff --git a/mm/kmsan/hooks.c b/mm/kmsan/hooks.c index 8f22d1f22981..5f1b8053f9fa 100644 --- a/mm/kmsan/hooks.c +++ b/mm/kmsan/hooks.c @@ -21,6 +21,7 @@ #include #include "../internal.h" +#include "../vmalloc.h" #include "../slab.h" #include "kmsan.h" diff --git a/mm/kmsan/shadow.c b/mm/kmsan/shadow.c index 8fde939784a7..0c88d89bf0d6 100644 --- a/mm/kmsan/shadow.c +++ b/mm/kmsan/shadow.c @@ -17,6 +17,7 @@ #include #include "../internal.h" +#include "../vmalloc.h" #include "kmsan.h" #define shadow_page_for(page) ((page)->kmsan_shadow) diff --git a/mm/mm_init.c b/mm/mm_init.c index b557e5fe35f3..711f821f7b3c 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -38,6 +38,7 @@ #include "sparse.h" #include "slab.h" #include "shuffle.h" +#include "vmalloc.h" #include diff --git a/mm/percpu-vm.c b/mm/percpu-vm.c index 7ed216192fc0..509d8901835c 100644 --- a/mm/percpu-vm.c +++ b/mm/percpu-vm.c @@ -9,6 +9,7 @@ * This is the default chunk allocator. */ #include "internal.h" +#include "vmalloc.h" static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk, unsigned int cpu, int page_idx) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 1191cda3b4e8..4b0362cbdf5a 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -49,6 +49,7 @@ #include "internal.h" #include "pgalloc-track.h" +#include "vmalloc.h" #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP static unsigned int __ro_after_init ioremap_max_page_shift = BITS_PER_LONG - 1; diff --git a/mm/vmalloc.h b/mm/vmalloc.h new file mode 100644 index 000000000000..dcfe30eaa80c --- /dev/null +++ b/mm/vmalloc.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * mm-internal APIs for vmalloc + */ +#ifndef __MM_VMALLOC_H +#define __MM_VMALLOC_H + +#ifdef CONFIG_MMU +void __init vmalloc_init(void); +int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end, + pgprot_t prot, struct page **pages, + unsigned int page_shift, gfp_t gfp_mask); +unsigned int get_vm_area_page_order(struct vm_struct *vm); +#else +static inline void vmalloc_init(void) {} + +static inline +int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end, + pgprot_t prot, struct page **pages, + unsigned int page_shift, gfp_t gfp_mask) +{ + return -EINVAL; +} + +static inline void vunmap_range_noflush(unsigned long start, unsigned long end) +{ +} +#endif + +struct vm_struct *__get_vm_area_node(unsigned long size, + unsigned long align, unsigned long shift, + unsigned long vm_flags, unsigned long start, + unsigned long end, int node, gfp_t gfp_mask, + const void *caller); + +void clear_vm_uninitialized_flag(struct vm_struct *vm); + +int __must_check __vmap_pages_range_noflush(unsigned long addr, + unsigned long end, pgprot_t prot, + struct page **pages, unsigned int page_shift); + +void vunmap_range_noflush(unsigned long start, unsigned long end); + +void __vunmap_range_noflush(unsigned long start, unsigned long end); + +#endif /* __MM_VMALLOC_H */ From 887311e5cd67566a21ac8c70b9545bcc449714a8 Mon Sep 17 00:00:00 2001 From: xu xin Date: Thu, 9 Jul 2026 17:32:12 +0800 Subject: [PATCH 295/501] mm/ksm: initialize the addr only once in collect_procs_ksm Patch series "KSM: use linear_page_index in collect_procs_ksm()", v2. In collect_procs_ksm() which is used to collect processes when the error hit an ksm page, there is the same issue with rmap_walk_ksm (see the previous discussion at [1]). So we apply the similar logic changes to the collect_procs_ksm(). The patch [1/2] move the initializaion of addr from the position inside loop to the position before the loop, since the variable will not change in the loop. The patch [2/2] optimize collect_procs_ksm by passing a suitable page offset range to the anon_vma_interval_tree_foreach loop to reduce ineffective checks. This patch (of 2): Similar to 318d87b8fa7 ("ksm: initialize the addr only once in rmap_walk_ksm"), only initialize the addr once in rmap_walk_ksm because the addr variable doesn't change across iterations. Link: https://lore.kernel.org/20260709173212190rZdwynySRyLr9EtPuXBRU@zte.com.cn Link: https://lore.kernel.org/all/20260703162253688u8Str9eFLR8TGCmo7nIOF@zte.com.cn/ [1] Signed-off-by: xu xin Acked-by: David Hildenbrand (Arm) Cc: Chengming Zhou Signed-off-by: Andrew Morton --- mm/ksm.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mm/ksm.c b/mm/ksm.c index c1dcc3cb4831..337eb04a9340 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -3289,7 +3289,7 @@ void collect_procs_ksm(const struct folio *folio, const struct page *page, rcu_read_lock(); for_each_process(tsk) { struct anon_vma_chain *vmac; - unsigned long addr; + const unsigned long addr = rmap_item->address & PAGE_MASK; struct task_struct *t = task_early_kill(tsk, force_early); if (!t) @@ -3299,7 +3299,6 @@ void collect_procs_ksm(const struct folio *folio, const struct page *page, { vma = vmac->vma; if (vma->vm_mm == t->mm) { - addr = rmap_item->address & PAGE_MASK; add_to_kill_ksm(t, page, vma, to_kill, addr); } From 0471cade0a272803d192cbd8d8523e93e429ff1d Mon Sep 17 00:00:00 2001 From: xu xin Date: Thu, 9 Jul 2026 17:33:12 +0800 Subject: [PATCH 296/501] ksm: use precise linear_page_index instead of the whole address space Since we now have linear_page_index available that we can use here, allowing for optimizing the RMAP walk, we can also use it to locate more precisely all related-processes when error hits the KSM page, which will decrease a lot of invalid iterations. Link: https://lore.kernel.org/20260709173312403qgj1Af6pRkFMDSsmc19sM@zte.com.cn Signed-off-by: xu xin Acked-by: David Hildenbrand (Arm) Cc: Chengming Zhou Signed-off-by: Andrew Morton --- mm/ksm.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mm/ksm.c b/mm/ksm.c index 337eb04a9340..2791ce5bd44b 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -3290,12 +3290,13 @@ void collect_procs_ksm(const struct folio *folio, const struct page *page, for_each_process(tsk) { struct anon_vma_chain *vmac; const unsigned long addr = rmap_item->address & PAGE_MASK; + const unsigned long index = rmap_item->linear_page_index; struct task_struct *t = task_early_kill(tsk, force_early); if (!t) continue; - anon_vma_interval_tree_foreach(vmac, &av->rb_root, 0, - ULONG_MAX) + anon_vma_interval_tree_foreach(vmac, &av->rb_root, index, + index) { vma = vmac->vma; if (vma->vm_mm == t->mm) { From f421d67d2c2367ca4b4b16f3a241a390b3ed501a Mon Sep 17 00:00:00 2001 From: Hongfu Li Date: Thu, 9 Jul 2026 16:18:43 +0800 Subject: [PATCH 297/501] selftests/mm: fix memleak in migration benchmark Several early return paths in run_migration_benchmark() skip hmm_buffer_free(), leaking the buffer. Replace with a single cleanup label. Link: https://lore.kernel.org/20260709081843.1451202-1-lihongfu@kylinos.cn Fixes: 271a7b2e3c13 ("selftests/mm/hmm-tests: new throughput tests including THP") Signed-off-by: Hongfu Li Reviewed-by: David Hildenbrand (Arm) Reviewed-by: SJ Park Acked-by: Balbir Singh Reviewed-by: Lorenzo Stoakes Reviewed-by: Balbir Singh Reviewed-by: Lorenzo Stoakes (ARM) Cc: Hongfu Li Cc: Jason Gunthorpe Cc: Leon Romanovsky Cc: Liam R. Howlett Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/hmm-tests.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tools/testing/selftests/mm/hmm-tests.c b/tools/testing/selftests/mm/hmm-tests.c index 2f2b9879d100..6fccbdab02ee 100644 --- a/tools/testing/selftests/mm/hmm-tests.c +++ b/tools/testing/selftests/mm/hmm-tests.c @@ -2829,8 +2829,11 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz buffer->ptr = mmap(NULL, buffer_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (buffer->ptr == MAP_FAILED) - return -1; + if (buffer->ptr == MAP_FAILED) { + buffer->ptr = NULL; + ret = -1; + goto cleanup; + } /* Apply THP hint if requested */ if (use_thp) @@ -2839,7 +2842,7 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz ret = madvise(buffer->ptr, buffer_size, MADV_NOHUGEPAGE); if (ret) - return ret; + goto cleanup; /* Initialize memory to make sure pages are allocated */ ptr = (int *)buffer->ptr; @@ -2849,11 +2852,11 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz /* Warmup iteration */ ret = hmm_migrate_sys_to_dev(fd, buffer, npages); if (ret) - return ret; + goto cleanup; ret = hmm_migrate_dev_to_sys(fd, buffer, npages); if (ret) - return ret; + goto cleanup; /* Benchmark iterations */ for (i = 0; i < iterations; i++) { @@ -2862,7 +2865,7 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz ret = hmm_migrate_sys_to_dev(fd, buffer, npages); if (ret) - return ret; + goto cleanup; end = get_time_ms(); s2d_total += (end - start); @@ -2872,7 +2875,7 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz ret = hmm_migrate_dev_to_sys(fd, buffer, npages); if (ret) - return ret; + goto cleanup; end = get_time_ms(); d2s_total += (end - start); @@ -2886,9 +2889,9 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz results->throughput_d2s = (buffer_size / (1024.0 * 1024.0 * 1024.0)) / (results->dev_to_sys_time / 1000.0); - /* Cleanup */ +cleanup: hmm_buffer_free(buffer); - return 0; + return ret; } /* From 5c4c48ff5aa1240a689d6508326a0ce42a71e9c4 Mon Sep 17 00:00:00 2001 From: Sayali Patil Date: Wed, 8 Jul 2026 12:29:05 +0530 Subject: [PATCH 298/501] selftests/mm: handle EINVAL when configuring gigantic hugepages Patch series "selftests/mm: avoid false failures in hugetlb and KSM tests", v3. This series fixes issues in the hugetlb and KSM MM selftest categories that can report failures when the prerequisites for the tests are not satisfied. Patch 1 updates the hugetlb selftest helpers to handle -EINVAL when attempting to configure gigantic HugeTLB pages via nr_hugepages. PowerPC hash MMU pSeries systems expose gigantic hugepage sizes but do not allow runtime allocation of such pages, causing the sysfs write to fail. Handle this case gracefully and continue running the test instead of aborting. Patch 2 fixes the KSM NUMA merge test on systems with memoryless NUMA nodes. The test currently relies on the number of configured NUMA nodes and may attempt allocations on nodes that have no memory, resulting in spurious failures. Use the existing helpers to identify NUMA nodes that contain memory and skip the test when fewer than two such nodes are available. Patch 3 fixes a pre-existing operator precedence issue in ksm_tests, where a ternary expression combined with logical OR operators could be evaluated differently than intended. Added parentheses to ensure the correct evaluation order. These changes improve handling of unsupported test configurations and unmet test prerequisites, avoiding spurious failures. This patch (of 3): Some MM selftests attempt to configure the amount of HugeTLB pages of different sizes by writing to nr_hugepages. PowerPC hash MMU pSeries systems advertise gigantic hugepage sizes but do not support runtime allocation of such pages, writes to the corresponding nr_hugepages file fail with -EINVAL. This causes the test to bail out even though the failure is due to a platform limitation rather than the functionality being tested. Ignore -EINVAL when configuring nr_hugepages so that tests continue to run on systems where gigantic hugepage allocation is unsupported. Before patch: ------------------------- running ./hugetlb-madvise ------------------------- TAP version 13 1..1 [INFO] detected hugetlb page size: 16777216 KiB [INFO] detected hugetlb page size: 16384 KiB ok 1 MADV_DONTNEED and MADV_REMOVE on hugetlb Totals: pass:1 fail:0 xfail:0 xpass:0 skip:0 error:0 Bail out! /sys/kernel/mm/hugepages/hugepages-16777216kB/nr_hugepages write(0) failed: Invalid argument Totals: pass:0 fail:0 xfail:0 xpass:0 skip:0 error:0 [FAIL] After patch: ------------------------- running ./hugetlb-madvise ------------------------- TAP version 13 1..1 [INFO] detected hugetlb page size: 16777216 KiB [INFO] detected hugetlb page size: 16384 KiB ok 1 MADV_DONTNEED and MADV_REMOVE on hugetlb Totals: pass:1 fail:0 xfail:0 xpass:0 skip:0 error:0 [PASS] Link: https://lore.kernel.org/cover.1783446924.git.sayalip@linux.ibm.com Link: https://lore.kernel.org/2e3b585cbb30b2fc495dcd49d75de6f6da61861c.1783446924.git.sayalip@linux.ibm.com Fixes: 27477b28b74f ("selftests/mm: hugepage_settings: add APIs to get and set nr_hugepages") Co-developed-by: David Hildenbrand (Arm) Signed-off-by: David Hildenbrand (Arm) Signed-off-by: Sayali Patil Cc: Dev Jain Cc: Liam Howlett Cc: Miaohe Lin Cc: Michal Hocko Cc: Oscar Salvador Cc: "Ritesh Harjani (IBM)" Cc: Shuah Khan Cc: Zi Yan Signed-off-by: Andrew Morton --- .../testing/selftests/mm/hugepage_settings.c | 2 +- tools/testing/selftests/mm/vm_util.c | 26 ++++++++++++++++--- tools/testing/selftests/mm/vm_util.h | 1 + 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/mm/hugepage_settings.c b/tools/testing/selftests/mm/hugepage_settings.c index 2eab2110ac6a..d7917dce3aba 100644 --- a/tools/testing/selftests/mm/hugepage_settings.c +++ b/tools/testing/selftests/mm/hugepage_settings.c @@ -437,7 +437,7 @@ void hugetlb_set_nr_pages(unsigned long size, unsigned long nr) hugetlb_sysfs_path(path, sizeof(path), size, "nr_hugepages"); - write_num(path, nr); + write_num_ignore_einval(path, nr); } unsigned long hugetlb_free_pages(unsigned long size) diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c index 311fc5b4513e..ef1ea11981a7 100644 --- a/tools/testing/selftests/mm/vm_util.c +++ b/tools/testing/selftests/mm/vm_util.c @@ -719,7 +719,7 @@ int read_file(const char *path, char *buf, size_t buflen) return (unsigned int) numread; } -void write_file(const char *path, const char *buf, size_t buflen) +static void __write_file(const char *path, const char *buf, size_t buflen, bool ignore_einval) { int fd, saved_errno; ssize_t numwritten; @@ -735,14 +735,22 @@ void write_file(const char *path, const char *buf, size_t buflen) saved_errno = errno; close(fd); errno = saved_errno; - if (numwritten < 0) + if (numwritten < 0) { + if (ignore_einval && errno == EINVAL) + return; ksft_exit_fail_msg("%s write(%.*s) failed: %s\n", path, (int)(buflen - 1), buf, strerror(errno)); + } if (numwritten != buflen - 1) ksft_exit_fail_msg("%s write(%.*s) is truncated, expected %zu bytes, got %zd bytes\n", path, (int)(buflen - 1), buf, buflen - 1, numwritten); } +void write_file(const char *path, const char *buf, size_t buflen) +{ + __write_file(path, buf, buflen, /* ignore_einval = */ false); +} + unsigned long read_num(const char *path) { char buf[21]; @@ -753,12 +761,22 @@ unsigned long read_num(const char *path) return strtoul(buf, NULL, 10); } -void write_num(const char *path, unsigned long num) +static void __write_num(const char *path, unsigned long num, bool ignore_einval) { char buf[21]; sprintf(buf, "%lu", num); - write_file(path, buf, strlen(buf) + 1); + __write_file(path, buf, strlen(buf) + 1, ignore_einval); +} + +void write_num(const char *path, unsigned long num) +{ + return __write_num(path, num, /* ignore_einval = */ false); +} + +void write_num_ignore_einval(const char *path, unsigned long num) +{ + return __write_num(path, num, /* ignore_einval = */ true); } static unsigned long shmall, shmmax; diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index ea8fc8fdf0eb..7799154b67ee 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -168,6 +168,7 @@ void write_file(const char *path, const char *buf, size_t buflen); int read_file(const char *path, char *buf, size_t buflen); unsigned long read_num(const char *path); void write_num(const char *path, unsigned long num); +void write_num_ignore_einval(const char *path, unsigned long num); void shm_limits_prepare(unsigned long length); void __shm_limits_restore(void); From 15828a150c5806869b7feb9e38ad2c0284fbf18a Mon Sep 17 00:00:00 2001 From: Sayali Patil Date: Wed, 8 Jul 2026 12:29:06 +0530 Subject: [PATCH 299/501] selftests/mm: fix ksm NUMA merge test for systems with memoryless NUMA nodes The KSM NUMA merge test allocates identical pages on different NUMA nodes and verifies KSM behavior with merge_across_nodes enabled and disabled. On systems with memoryless NUMA nodes, for example: #numactl -H available: 2 nodes (0,4) ..... node 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 node 0 size: 14825 MB node 0 free: 1382 MB node 4 cpus: node 4 size: 0 MB node 4 free: 0 MB the test may attempt to allocate memory on a node without memory, causing numa_alloc_onnode() to fail and resulting in a spurious test failure. The test currently checks numa_num_configured_nodes() to determine whether sufficient NUMA nodes are available. However, configured nodes do not necessarily have memory. Reuse the existing get_first_mem_node() and get_next_mem_node() helpers to locate NUMA nodes that actually contain memory, and skip the test when fewer than two such nodes are available. Before patch: --------------------------- running ./ksm_tests -N -m 1 --------------------------- mbind: Invalid argument ok 1 KSM NUMA merging Totals: pass:1 fail:0 xfail:0 xpass:0 skip:0 error:0 [PASS] ok 1 ksm_tests -N -m 1 --------------------------- running ./ksm_tests -N -m 0 --------------------------- mbind: Invalid argument not ok 1 KSM NUMA merging Totals: pass:0 fail:1 xfail:0 xpass:0 skip:0 error:0 [FAIL] not ok 2 ksm_tests -N -m 0 # exit=1 After patch: --------------------------- running ./ksm_tests -N -m 1 --------------------------- At least 2 NUMA nodes with memory must be available ok 1 SKIP KSM NUMA merging Totals: pass:0 fail:0 xfail:0 xpass:0 skip:1 error:0 [PASS] ok 1 ksm_tests -N -m 1 --------------------------- running ./ksm_tests -N -m 0 --------------------------- At least 2 NUMA nodes with memory must be available ok 1 SKIP KSM NUMA merging Totals: pass:0 fail:0 xfail:0 xpass:0 skip:1 error:0 [PASS] ok 2 ksm_tests -N -m 0 Link: https://lore.kernel.org/78a3b0e3fb94004c0710872c5bab6f7381b7d63c.1783446924.git.sayalip@linux.ibm.com Fixes: e3820ab252dd ("selftest/vm: fix ksm selftest to run with different NUMA topologies") Co-developed-by: David Hildenbrand (Arm) Signed-off-by: David Hildenbrand (Arm) Signed-off-by: Sayali Patil Cc: Dev Jain Cc: Liam Howlett Cc: Miaohe Lin Cc: Michal Hocko Cc: Oscar Salvador Cc: "Ritesh Harjani (IBM)" Cc: Shuah Khan Cc: Zi Yan Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/ksm_tests.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tools/testing/selftests/mm/ksm_tests.c b/tools/testing/selftests/mm/ksm_tests.c index a050f4840cfa..2ebbb544c671 100644 --- a/tools/testing/selftests/mm/ksm_tests.c +++ b/tools/testing/selftests/mm/ksm_tests.c @@ -440,9 +440,9 @@ static int get_next_mem_node(int node) mem_node = i % (max_node + 1); node_size = numa_node_size(mem_node, NULL); if (node_size > 0) - break; + return mem_node; } - return mem_node; + return -ENODEV; } static int get_first_mem_node(void) @@ -455,8 +455,8 @@ static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeo { void *numa1_map_ptr, *numa2_map_ptr; struct timespec start_time; + int first_node, second_node; int page_count = 2; - int first_node; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { ksft_perror("clock_gettime"); @@ -467,17 +467,19 @@ static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeo ksft_print_msg("NUMA support not enabled\n"); return KSFT_SKIP; } - if (numa_num_configured_nodes() <= 1) { - ksft_print_msg("At least 2 NUMA nodes must be available\n"); + first_node = get_first_mem_node(); + second_node = get_next_mem_node(first_node); + + if (second_node < 0) { + ksft_print_msg("At least 2 NUMA nodes with memory must be available\n"); return KSFT_SKIP; } if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes)) return KSFT_FAIL; /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */ - first_node = get_first_mem_node(); numa1_map_ptr = numa_alloc_onnode(page_size, first_node); - numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node)); + numa2_map_ptr = numa_alloc_onnode(page_size, second_node); if (!numa1_map_ptr || !numa2_map_ptr) { ksft_perror("numa_alloc_onnode"); return KSFT_FAIL; From 4e1fbffb3333626682a011db7c4b4e9e40ba96d4 Mon Sep 17 00:00:00 2001 From: Sayali Patil Date: Wed, 8 Jul 2026 12:29:07 +0530 Subject: [PATCH 300/501] selftests/mm: fix ternary operator precedence in ksm_tests The KSM selftest uses conditional expressions to skip accesses to merge_across_nodes on systems without NUMA support. However, the ternary operator is combined with logical OR without parentheses: a || numa_available() ? 0 : b || c Due to operator precedence rules, this is parsed as: (a || numa_available()) ? 0 : (b || c) instead of the intended: a || (numa_available() ? 0 : b) || c Add parentheses around the conditional expressions to ensure the correct evaluation order. Link: https://lore.kernel.org/ce859430287ed2642848c933a90eb9a69da361f0.1783446924.git.sayalip@linux.ibm.com Fixes: 9aa1af954db0 ("selftests: vm: check numa_available() before operating "merge_across_nodes" in ksm_tests") Signed-off-by: Sayali Patil Acked-by: David Hildenbrand (Arm) Cc: Dev Jain Cc: Liam Howlett Cc: Miaohe Lin Cc: Michal Hocko Cc: Oscar Salvador Cc: "Ritesh Harjani (IBM)" Cc: Shuah Khan Cc: Zi Yan Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/ksm_tests.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/mm/ksm_tests.c b/tools/testing/selftests/mm/ksm_tests.c index 2ebbb544c671..5fd7792a0d47 100644 --- a/tools/testing/selftests/mm/ksm_tests.c +++ b/tools/testing/selftests/mm/ksm_tests.c @@ -288,8 +288,8 @@ static bool assert_ksm_pages_count(long dupl_page_count) static int ksm_save_def(struct ksm_sysfs *ksm_sysfs) { if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) || - numa_available() ? 0 : - ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes) || + (numa_available() ? 0 : + ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes)) || ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) || ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) || ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) || @@ -304,8 +304,8 @@ static int ksm_save_def(struct ksm_sysfs *ksm_sysfs) static int ksm_restore(struct ksm_sysfs *ksm_sysfs) { if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) || - numa_available() ? 0 : - ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes) || + (numa_available() ? 0 : + ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes)) || ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) || ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) || ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) || @@ -846,8 +846,8 @@ int main(int argc, char *argv[]) if (ksm_write_sysfs(KSM_FP("run"), 2) || ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) || - numa_available() ? 0 : - ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) || + (numa_available() ? 0 : + ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1)) || ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count)) ksft_exit_fail_msg("Cannot set up KSM tunables\n"); From c494788faffe67216c56623d240541fde50139c3 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Tue, 7 Jul 2026 17:57:45 -0700 Subject: [PATCH 301/501] mm/gup: fix GUP-fast fallback for NULL-mapping order-0 folios Since commit f002882ca369 ("mm: merge folio_is_secretmem() and folio_fast_pin_allowed() into gup_fast_folio_allowed()"), gup_fast_folio_allowed() falls back to the slow path for any order-0 folio with a NULL mapping when CONFIG_SECRETMEM=y. This causes a performance regression for drivers that allocate pages with alloc_page() and insert them into VMAs via vm_insert_page(). These pages legitimately have a NULL folio->mapping, but they cannot be secretmem pages. Secretmem pages are always added to the secretmem inode's page cache via filemap_add_folio(), which sets folio->mapping to the inode's i_mapping. A folio with a NULL mapping can never be a secretmem folio. The NULL-mapping check was intended to handle truncated file-backed pages (a reject_file_backed concern), not secretmem detection. When only check_secretmem is true (and reject_file_backed is false), a NULL mapping is sufficient to prove the folio is not secretmem, so the fast path can proceed. Link: https://lore.kernel.org/20260708005745.164928-1-jhubbard@nvidia.com Fixes: f002882ca369 ("mm: merge folio_is_secretmem() and folio_fast_pin_allowed() into gup_fast_folio_allowed()") Signed-off-by: John Hubbard Tested-by: Sourab Gupta Acked-by: David Hildenbrand (Arm) Cc: Alistair Popple Cc: Balbir Singh Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/gup.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index 0692119b7904..99902c15703b 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -2784,12 +2784,17 @@ static bool gup_fast_folio_allowed(struct folio *folio, unsigned int flags) mapping = READ_ONCE(folio->mapping); /* - * The mapping may have been truncated, in any case we cannot determine - * if this mapping is safe - fall back to slow path to determine how to - * proceed. + * If the mapping is NULL (truncated, or never set), we cannot + * determine whether the folio is file-backed, so a long-term writable + * pin must fall back to the slow path. + * + * Otherwise, a NULL mapping proves this is not a secretmem folio + * (secretmem folios always have a valid mapping to the secretmem + * inode's address_space), so in that case, we can continue with the + * fast path. */ if (!mapping) - return false; + return !reject_file_backed; /* Anonymous folios pose no problem. */ mapping_flags = (unsigned long)mapping & FOLIO_MAPPING_FLAGS; From 747beac679115b20984a722e59686c2f0bc118b3 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 10 Jul 2026 07:10:52 +0200 Subject: [PATCH 302/501] mm: remove wb_writeout_inc Remove this entirely unused but exported function. Link: https://lore.kernel.org/20260710051052.1839523-1-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: Jan Kara Acked-by: David Hildenbrand (Arm) Reviewed-by: Anshuman Khandual Acked-by: SJ Park Cc: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/backing-dev.h | 2 -- mm/page-writeback.c | 10 ---------- 2 files changed, 12 deletions(-) diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h index 5b7d12b40d5e..c2284466e7aa 100644 --- a/include/linux/backing-dev.h +++ b/include/linux/backing-dev.h @@ -76,8 +76,6 @@ static inline s64 wb_stat_sum(struct bdi_writeback *wb, enum wb_stat_item item) return percpu_counter_sum_positive(&wb->stat[item]); } -extern void wb_writeout_inc(struct bdi_writeback *wb); - /* * maximal error of a stat counter. */ diff --git a/mm/page-writeback.c b/mm/page-writeback.c index e98748112d1e..47495be68598 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -584,16 +584,6 @@ static inline void __wb_writeout_add(struct bdi_writeback *wb, long nr) wb->bdi->max_prop_frac, nr); } -void wb_writeout_inc(struct bdi_writeback *wb) -{ - unsigned long flags; - - local_irq_save(flags); - __wb_writeout_add(wb, 1); - local_irq_restore(flags); -} -EXPORT_SYMBOL_GPL(wb_writeout_inc); - /* * On idle system, we can be called long after we scheduled because we use * deferred timers so count with missed periods. From 37aae41c1056d44aae7988289920cdb504af6fb4 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 10 Jul 2026 06:46:30 -0700 Subject: [PATCH 303/501] mm/damon/core: introduce damon_probe->weight Patch series "mm/damon: introduce data attributes only monitoring". TL;DR: Introduce a way to get DAMON's best effort accuracy monitoring of user-demanding non-access data attributes. Background ========== DAMON was initially designed for only access monitoring. It turned out users want to get the information together with more data attributes. For example, some users want to know how much of a hot memory region belongs to huge pages or specific cgroups. Page level properties based monitoring was introduced with commit 626ffabe67c2 ("mm/damon: clarify trying vs applying on damos_stat kernel-doc comment") to fill the gap. Because it works only at snapshot level and snapshot capturing in the mode can induce high overhead, commit 45c49d9fd608 ("mm/damon/core: introduce struct damon_probe") introduced data attributes monitoring. Data attributes monitoring treats the attributes as only additional and subordinate information. Data access monitoring is always turned on, and regions are adjusted for best accuracy of the access information. In some cases, users may be primarily interested in the attributes more than the access. They might even not care about the access information at all. Because DAMON treats data accesses as the only primary information, such users cannot get high quality attributes information. Design and Implementation ========================= Introduce another way for treating data attributes as the primary information. Add 'weight' property to each data attribute probe. When any of the weights are set, the mode is enabled. Data access monitoring is completely turned off in the mode. For region adjustment, the weighted sum of probe hit counters is used instead of the nr_accesses. Using the weights, users can specify to what attributes they are interested in to what degree. DAMON will adjust the regions and provide the best-effort quality monitoring that is optimized for the user demands. Extend damon_operations for efficient use of probe hits. Update regions merge and kdamond main logic to support the new mode. Add a new struct field and a sysfs file for API callers and ABI users, respectively. Test ==== On ~7 GiB memory idle system, run a simple AI-assisted program. The program allocates and faults 2 GiB anonymous pages. Then, it does nothing but wait until the user terminates it. Hence, the system ~2 GiB of anonymous pages with no active accesses. Monitor the distribution of the anonymous pages using DAMON attributes monitoring mode, using DAMON user-space tool, damo [1]. $ sudo ./damo start --probe_filter allow anon $ sudo ./damo report access --dont_merge_regions heatmap: 00000000000000000000000000000000000000000000000399999995111111146666666666666666 # min/max temperatures: -2,470,000,000, -1,620,000,000, column size: 99.800 MiB intervals: sample 5 ms aggr 100 ms (max access hz 200) # 0 4.000 KiB 79.840 MiB 0 hz 24.700 s 2 1 79.844 MiB 718.562 MiB 0 hz 24.700 s 8 2 798.406 MiB 793.148 MiB 0 hz 24.700 s 7 3 1.554 GiB 797.828 MiB 0 hz 24.700 s 7 4 2.333 GiB 794.668 MiB 0 hz 24.600 s 8 5 3.109 GiB 791.117 MiB 0 hz 24.500 s 0 6 3.882 GiB 785.312 MiB 0 hz 24 s 2 7 4.649 GiB 787.867 MiB 0 hz 16.200 s 6 8 5.418 GiB 784.477 MiB 0 hz 23.300 s 6 9 6.184 GiB 783.820 MiB 0 hz 18.200 s 9 10 6.950 GiB 797.730 MiB 0 hz 18.900 s 7 11 7.729 GiB 69.625 MiB 0 hz 18.900 s 0 memory bw estimate: 0 B per second total size: 7.797 GiB record DAMON intervals: sample 5 ms, aggr 100 ms Note that the line after the line starting with "intervals:" is not provided by the current version of 'damo'. I manually added the legends line for easier understanding of these results. Each of the 12 lines after the legend line shows the DAMON-found regions. Each line shows 1) index of the region, 2) start address of the region, 3) size of the region, 4) access frequency of the region, 5) age (how long the access frequency on the region was kept) of the region, and finally 6) the probe hit count. Because data access is the primary information that adjusts region for, and there is only nearly zero access on the system, regions are naively adjusted with the same size. Still show different distribution of the anonymous pages, but it is obviously very rough information. Switch to the attributes only mode and show how it changes the picture: $ sudo ./damo tune --probe_filter allow anon --probe_weight 100 $ sudo ./damo report access --dont_merge_regions heatmap: 88888888888888888889888999999889999999000004888888888888889999988888898888888888 # min/max temperatures: -4,430,000,000, 0, column size: 99.800 MiB intervals: sample 5 ms aggr 100 ms (max access hz 200) # 0 4.000 KiB 60.445 MiB 0 hz 700 ms 0 1 60.449 MiB 1.363 MiB 0 hz 600 ms 18 2 61.812 MiB 144.000 KiB 0 hz 0 ns 1 3 61.953 MiB 1.922 MiB 0 hz 2.400 s 19 4 63.875 MiB 12.133 MiB 0 hz 200 ms 0 [...] 500 5.132 GiB 8.000 KiB 0 hz 2 m 15.800 s 20 501 5.132 GiB 8.000 KiB 0 hz 2 m 16.200 s 0 502 5.132 GiB 16.000 KiB 0 hz 2 m 16.900 s 20 503 5.132 GiB 24.000 KiB 0 hz 2 m 14.200 s 0 504 5.132 GiB 8.000 KiB 0 hz 2 m 14.900 s 20 [...] 923 7.534 GiB 126.637 MiB 0 hz 0 ns 6 924 7.658 GiB 252.000 KiB 0 hz 54.800 s 20 925 7.658 GiB 142.242 MiB 0 hz 300 ms 0 memory bw estimate: 0 B per second total size: 7.797 GiB record DAMON intervals: sample 5 ms, aggr 100 ms As expected, regions are adjusted to provide the best accurate picture for the anonymous pages distribution (). The region 0 (60.445 MiB memory from the address 4.000 KiB) has nearly zero anonymous pages. The region 1 (1.363 MiB memory from the address 60.449 MiB) is nearly full with anonymous pages. Region 500 (8 KiB memory from the address 5.132 GiB) is certainly two anonymous pages. Future Work =========== Attributes only monitoring disables access monitoring. We will enable that in future, by extending the supported attributes to include data accesses. This patch series, and the future work are parts of the ongoing project [2] for extending DAMON. The project aims to extend DAMON with primitives other than page table accessed bits such as AMD IBS, Intel PEBS, and Arm SPE, to provide more powerful and detailed information like per-CPUs/threads/reads/writes monitoring. Patches Sequence ================ Patch 1 introduces damon_probe->weight for specifying the weights of each attribute. Patches 2-6 extends apply_probe() damon_ops callback to efficiently support the new mode. Patch 7 fixes wrong use of abs() in the regions merge code. Patch 8 extends regions merge function to work with probe hits in the mode. Patch 8 also introduces the function for detecting the mode enablement but always returns false, for safe and incremental changes. Patches 9 and 10 adds user parameters validation to prevent theoretical overflow of probe hits and the weighted sum. Patches 11-14 incrementally update kdamond_fn() to support the mode. Patch 15 completes the mode detection function implementation, so that the new mode really works. Patch 16 introduces a new sysfs file for ABI users. Finally, patches 17-19 respectively updates design, usage and ABI documents for the new feature and interfaces. [1] https://github.com/damonitor/damo [2] https://lore.kernel.org/20260525225208.1179-1-sj@kernel.org/ This patch (of 19): Add a new field, weight to damon_probe struct. The field is used to specify the degree of the API caller's interest to the data attribute of the probe. Link: https://lore.kernel.org/20260710134651.18084-1-sj@kernel.org Link: https://lore.kernel.org/20260710134651.18084-2-sj@kernel.org Link: https://github.com/damonitor/damo [1] Link: https://lore.kernel.org/20260525225208.1179-1-sj@kernel.org/ [2] Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/damon.h | 2 ++ mm/damon/core.c | 3 +++ 2 files changed, 5 insertions(+) diff --git a/include/linux/damon.h b/include/linux/damon.h index 63f596957c1f..d8251c83e6e5 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -760,10 +760,12 @@ struct damon_filter { /** * struct damon_probe - Data region attribute probe. * + * @weight: Relative priority of the attribute for this probe. * @filters: Filters for assessing if a given region is for this probe. * @list: Siblings list. */ struct damon_probe { + unsigned int weight; struct list_head filters; struct list_head list; }; diff --git a/mm/damon/core.c b/mm/damon/core.c index 11473aa1f70f..327398fa4ee5 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -166,6 +166,7 @@ struct damon_probe *damon_new_probe(void) p = kmalloc_obj(*p); if (!p) return NULL; + p->weight = 0; INIT_LIST_HEAD(&p->filters); INIT_LIST_HEAD(&p->list); return p; @@ -1658,6 +1659,7 @@ static int damon_commit_probes(struct damon_ctx *dst, struct damon_ctx *src) damon_for_each_probe_safe(dst_probe, next, dst) { src_probe = damon_nth_probe(i++, src); if (src_probe) { + dst_probe->weight = src_probe->weight; err = damon_commit_filters(dst_probe, src_probe); if (err) return err; @@ -1674,6 +1676,7 @@ static int damon_commit_probes(struct damon_ctx *dst, struct damon_ctx *src) if (!new_probe) return -ENOMEM; damon_add_probe(dst, new_probe); + new_probe->weight = src_probe->weight; err = damon_commit_filters(new_probe, src_probe); if (err) return err; From 1138ec78dc6d9f14f7b14ed709020804f1243685 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 10 Jul 2026 06:46:31 -0700 Subject: [PATCH 304/501] mm/damon/core: ask apply_probes() ops callback to set sampling address prepare_access_checks() DAMON ops callback sets the monitoring sampling address per region. In future, DAMON will be able to call only apply_probes(). In this case, applyy_probes() may need to do the sampling address setup, to minimize unnecessary regions iteration. Update the protocol for the request. Link: https://lore.kernel.org/20260710134651.18084-3-sj@kernel.org Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/damon.h | 5 +++-- mm/damon/core.c | 2 +- mm/damon/paddr.c | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index d8251c83e6e5..aee28b05acc6 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -647,7 +647,8 @@ enum damon_ops_id { * It should also return max number of observed accesses that made as a result * of its update. The value will be used for regions adjustment threshold. * @apply_probes should apply the data attribute probes to each region and - * accordingly update the probe hits counter of the region. + * accordingly update the probe hits counter of the region. It should also + * set &damon_region->sampling_addr of each region if ``set_samples`` is true. * @get_scheme_score should return the priority score of a region for a scheme * as an integer in [0, &DAMOS_MAX_SCORE]. * @apply_scheme is called from @kdamond when a region for user provided @@ -665,7 +666,7 @@ struct damon_operations { void (*update)(struct damon_ctx *context); void (*prepare_access_checks)(struct damon_ctx *context); unsigned int (*check_accesses)(struct damon_ctx *context); - void (*apply_probes)(struct damon_ctx *context); + void (*apply_probes)(struct damon_ctx *context, bool set_samples); int (*get_scheme_score)(struct damon_ctx *context, struct damon_region *r, struct damos *scheme); unsigned long (*apply_scheme)(struct damon_ctx *context, diff --git a/mm/damon/core.c b/mm/damon/core.c index 327398fa4ee5..9fbea6204520 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -3692,7 +3692,7 @@ static int kdamond_fn(void *data) if (ctx->ops.check_accesses) max_nr_accesses = ctx->ops.check_accesses(ctx); if (ctx->ops.apply_probes) - ctx->ops.apply_probes(ctx); + ctx->ops.apply_probes(ctx, false); if (time_after_eq(ctx->passed_sample_intervals, next_aggregation_sis)) { diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 617246498173..b13bf7c6eade 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -166,7 +166,7 @@ static bool damon_pa_filter_pass(phys_addr_t pa, struct folio *folio, return pass; } -static void damon_pa_apply_probes(struct damon_ctx *ctx) +static void damon_pa_apply_probes(struct damon_ctx *ctx, bool set_samples) { struct damon_target *t; struct damon_region *r; From a75de62a5f2b56fb2cd2b8beb1c541dead65039e Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 10 Jul 2026 06:46:32 -0700 Subject: [PATCH 305/501] mm/damon/paddr: set samples in apply_probes() if requested apply_probe() callback implementation in DAMON_PADDR is ignoring set_samples parameter. Respect it. Link: https://lore.kernel.org/20260710134651.18084-4-sj@kernel.org Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/damon/paddr.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index b13bf7c6eade..ff4fcca94966 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -178,6 +178,10 @@ static void damon_pa_apply_probes(struct damon_ctx *ctx, bool set_samples) phys_addr_t pa; struct folio *folio; + if (set_samples) + r->sampling_addr = damon_rand(ctx, r->ar.start, + r->ar.end); + pa = damon_pa_phys_addr(r->sampling_addr, ctx->addr_unit); folio = damon_get_folio(PHYS_PFN(pa)); From d235513a7c6b4e93a907be3b5c6da1305db309f6 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 10 Jul 2026 06:46:33 -0700 Subject: [PATCH 306/501] mm/damon/core: ask apply_probe() to return max probe hits weighted sum check_accesses() DAMON ops callback returns the maximum nr_accesses of regions. DAMON core uses it to calculate a reasonable region merge threshold. The core will need to adjust regions for not nr_accesses but probe hits weighted sum in future. For that, the core needs to know the maximum weighted sum of the regions. Update the protocol for the task. Link: https://lore.kernel.org/20260710134651.18084-5-sj@kernel.org Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/damon.h | 5 ++++- mm/damon/core.c | 2 +- mm/damon/paddr.c | 4 +++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index aee28b05acc6..e7acd8e610a6 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -649,6 +649,8 @@ enum damon_ops_id { * @apply_probes should apply the data attribute probes to each region and * accordingly update the probe hits counter of the region. It should also * set &damon_region->sampling_addr of each region if ``set_samples`` is true. + * It should also return maximum probe hits weighted sum of regions if + * ``return_max_wsum`` is true. * @get_scheme_score should return the priority score of a region for a scheme * as an integer in [0, &DAMOS_MAX_SCORE]. * @apply_scheme is called from @kdamond when a region for user provided @@ -666,7 +668,8 @@ struct damon_operations { void (*update)(struct damon_ctx *context); void (*prepare_access_checks)(struct damon_ctx *context); unsigned int (*check_accesses)(struct damon_ctx *context); - void (*apply_probes)(struct damon_ctx *context, bool set_samples); + unsigned int (*apply_probes)(struct damon_ctx *context, + bool set_samples, bool return_max_wsum); int (*get_scheme_score)(struct damon_ctx *context, struct damon_region *r, struct damos *scheme); unsigned long (*apply_scheme)(struct damon_ctx *context, diff --git a/mm/damon/core.c b/mm/damon/core.c index 9fbea6204520..9f6cdd810bd8 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -3692,7 +3692,7 @@ static int kdamond_fn(void *data) if (ctx->ops.check_accesses) max_nr_accesses = ctx->ops.check_accesses(ctx); if (ctx->ops.apply_probes) - ctx->ops.apply_probes(ctx, false); + ctx->ops.apply_probes(ctx, false, false); if (time_after_eq(ctx->passed_sample_intervals, next_aggregation_sis)) { diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index ff4fcca94966..3eba3564a31f 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -166,7 +166,8 @@ static bool damon_pa_filter_pass(phys_addr_t pa, struct folio *folio, return pass; } -static void damon_pa_apply_probes(struct damon_ctx *ctx, bool set_samples) +static unsigned int damon_pa_apply_probes(struct damon_ctx *ctx, + bool set_samples, bool return_max_wsum) { struct damon_target *t; struct damon_region *r; @@ -194,6 +195,7 @@ static void damon_pa_apply_probes(struct damon_ctx *ctx, bool set_samples) folio_put(folio); } } + return 0; } /* From cfe9e8c738d9acfa09973040e2dfa3fbd17c12be Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 10 Jul 2026 06:46:34 -0700 Subject: [PATCH 307/501] mm/damon/core: implement damon_probe_hits_wsum() When damon_probe->weight is set, the weighted sum of probe hits will be useful. It will be useful for not only the users but also DAMON internal logics like regions merging. Implement a function for calculating it. Link: https://lore.kernel.org/20260710134651.18084-6-sj@kernel.org Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/damon.h | 2 ++ mm/damon/core.c | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/include/linux/damon.h b/include/linux/damon.h index e7acd8e610a6..f69442a9d431 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -1012,6 +1012,8 @@ unsigned int damon_nr_accesses_mvsum(struct damon_region *r, struct damon_ctx *ctx); unsigned char damon_probe_hits_mvsum(int probe_idx, struct damon_region *r, struct damon_ctx *ctx); +unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last, + struct damon_ctx *ctx); int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, unsigned int nr_ranges, unsigned long min_region_sz); diff --git a/mm/damon/core.c b/mm/damon/core.c index 9f6cdd810bd8..fea97399b575 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -393,6 +393,30 @@ static bool damon_is_last_region(struct damon_region *r, return list_is_last(&r->list, &t->regions_list); } +/** + * damon_probe_hits_wsum() - Returns probe hits weighted sum of a region. + * @r: region to get the weighted sum of. + * @last: if the request is for last-window aggregated probe hits. + * @ctx: context of &r. + * + * Return: the weighted sum of probe hits of the region. + */ +unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last, + struct damon_ctx *ctx) +{ + struct damon_probe *probe; + unsigned int sum = 0; + int i = 0; + + damon_for_each_probe(probe, ctx) { + if (last) + sum += r->last_probe_hits[i++] * probe->weight; + else + sum += r->probe_hits[i++] * probe->weight; + } + return sum; +} + /* * Check whether a region is intersecting an address range * From 3858025f48cf764c3c653b311a1fa1b2eb720a5c Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 10 Jul 2026 06:46:35 -0700 Subject: [PATCH 308/501] mm/damon/paddr: respect return_max_wsum apply_probes() ops implementation in DAMON_PADDR is ignoring return_max_wsum. Respect it. Link: https://lore.kernel.org/20260710134651.18084-7-sj@kernel.org Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/damon/paddr.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 3eba3564a31f..b85f88a7a38f 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -172,6 +172,7 @@ static unsigned int damon_pa_apply_probes(struct damon_ctx *ctx, struct damon_target *t; struct damon_region *r; struct damon_probe *p; + unsigned int max_wsum = 0; damon_for_each_target(t, ctx) { damon_for_each_region(r, t) { @@ -182,7 +183,6 @@ static unsigned int damon_pa_apply_probes(struct damon_ctx *ctx, if (set_samples) r->sampling_addr = damon_rand(ctx, r->ar.start, r->ar.end); - pa = damon_pa_phys_addr(r->sampling_addr, ctx->addr_unit); folio = damon_get_folio(PHYS_PFN(pa)); @@ -193,9 +193,12 @@ static unsigned int damon_pa_apply_probes(struct damon_ctx *ctx, } if (folio) folio_put(folio); + if (return_max_wsum) + max_wsum = max(damon_probe_hits_wsum(r, false, + ctx), max_wsum); } } - return 0; + return max_wsum; } /* From e1f150d41516c192e3b78f360a7fb2dd9879abda Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 10 Jul 2026 06:46:36 -0700 Subject: [PATCH 309/501] mm/damon/core: use abs_diff() instead of abs() Use of abs() in damon_merge_regions_of() could cause a silent integer overflow since the macro casts unsigned int to signed int. It is unlikely to have such a large value for nr_accesses. Even though it happens, the user impact is just degraded monitoring results. Users showing bad monitoring results for weird setup is quite trivial. But the code is obviously wrong. Use abs_diff() instead. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260710134651.18084-8-sj@kernel.org Link: https://lore.kernel.org/20260705213817.100841-1-sj@kernel.org/ [1] Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/damon/core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index fea97399b575..4c7e8a12a9b6 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -3261,7 +3261,7 @@ static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, struct damon_region *r, *prev = NULL, *next; damon_for_each_region_safe(r, next, t) { - if (abs(r->nr_accesses - r->last_nr_accesses) > thres) + if (abs_diff(r->nr_accesses, r->last_nr_accesses) > thres) r->age = 0; else if ((r->nr_accesses == 0) != (r->last_nr_accesses == 0)) r->age = 0; @@ -3269,7 +3269,7 @@ static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, r->age++; if (prev && prev->ar.end == r->ar.start && - abs(prev->nr_accesses - r->nr_accesses) <= thres && + abs_diff(prev->nr_accesses, r->nr_accesses) <= thres && damon_sz_region(prev) + damon_sz_region(r) <= sz_limit) damon_merge_two_regions(t, prev, r); else From 0d1daaed8beaaa97c59f9c5e3e4dfeb7674892c7 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 10 Jul 2026 06:46:37 -0700 Subject: [PATCH 310/501] mm/damon/core: extend merge function to work with probe hits When probe weights are set, users may want DAMON monitoring results to be optimized for the weights. For that, regions adjustment should work for the weighted sum of probe hits. Extend damon_merge_regions_of() to detect if the weights are set, and work with probe hits in the case. The weights setup detection function is incomplete. It always returns false. It is intentional, so that more changes to completely support weights can be made in an incremental but safe way. Until the function is completed, all changes depend on it is no-op, so DAMON works in the current mode. Link: https://lore.kernel.org/20260710134651.18084-9-sj@kernel.org Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/damon/core.c | 49 +++++++++++++++++++++++++++++-------- mm/damon/tests/core-kunit.h | 13 ++++++++-- 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 4c7e8a12a9b6..7d49420ea26c 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -209,6 +209,11 @@ static struct damon_probe *damon_nth_probe(int n, struct damon_ctx *ctx) return NULL; } +static bool damon_has_probe_weights(struct damon_ctx *c) +{ + return false; +} + /* * damon_mvsum() - Returns pseudo moving sum value for a time window. * @current_nr: The value of the current aggregation window. @@ -3248,6 +3253,16 @@ static void damon_merge_two_regions(struct damon_target *t, damon_destroy_region(r, t); } +static unsigned int damon_merge_score(struct damon_region *r, bool last, + struct damon_ctx *ctx, bool use_probe_hits) +{ + if (use_probe_hits) + return damon_probe_hits_wsum(r, last, ctx); + if (last) + return r->last_nr_accesses; + return r->nr_accesses; +} + /* * Merge adjacent regions having similar access frequencies * @@ -3256,24 +3271,38 @@ static void damon_merge_two_regions(struct damon_target *t, * sz_limit size upper limit of each region */ static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, - unsigned long sz_limit) + unsigned long sz_limit, struct damon_ctx *ctx) { struct damon_region *r, *prev = NULL, *next; + bool use_probe_hits = damon_has_probe_weights(ctx); damon_for_each_region_safe(r, next, t) { - if (abs_diff(r->nr_accesses, r->last_nr_accesses) > thres) + unsigned int score, last_score, diff; + + score = damon_merge_score(r, false, ctx, use_probe_hits); + last_score = damon_merge_score(r, true, ctx, use_probe_hits); + + if (abs_diff(score, last_score) > thres) r->age = 0; - else if ((r->nr_accesses == 0) != (r->last_nr_accesses == 0)) + else if ((score == 0) != (last_score == 0)) r->age = 0; else r->age++; - if (prev && prev->ar.end == r->ar.start && - abs_diff(prev->nr_accesses, r->nr_accesses) <= thres && - damon_sz_region(prev) + damon_sz_region(r) <= sz_limit) - damon_merge_two_regions(t, prev, r); - else - prev = r; + if (!prev) + goto set_prev_continue; + if (prev->ar.end != r->ar.start) + goto set_prev_continue; + diff = abs_diff(score, damon_merge_score(prev, false, ctx, + use_probe_hits)); + if (diff > thres) + goto set_prev_continue; + if (damon_sz_region(prev) + damon_sz_region(r) > sz_limit) + goto set_prev_continue; + damon_merge_two_regions(t, prev, r); + continue; +set_prev_continue: + prev = r; } } @@ -3306,7 +3335,7 @@ static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, do { nr_regions = 0; damon_for_each_target(t, c) { - damon_merge_regions_of(t, threshold, sz_limit); + damon_merge_regions_of(t, threshold, sz_limit, c); nr_regions += damon_nr_regions(t); } threshold = max(1, threshold * 2); diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index d97280270c27..6ad73559dd8e 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -226,6 +226,7 @@ static struct damon_region *__nth_region_of(struct damon_target *t, int idx) static void damon_test_merge_regions_of(struct kunit *test) { + struct damon_ctx *ctx; struct damon_target *t; struct damon_region *r; unsigned long sa[] = {0, 100, 114, 122, 130, 156, 170, 184, 230}; @@ -236,20 +237,27 @@ static void damon_test_merge_regions_of(struct kunit *test) unsigned long eaddrs[] = {112, 130, 156, 170, 230, 10170}; int i; + ctx = damon_new_ctx(); + if (!ctx) + kunit_skip(test, "ctx alloc fail"); + t = damon_new_target(); - if (!t) + if (!t) { + damon_destroy_ctx(ctx); kunit_skip(test, "target alloc fail"); + } for (i = 0; i < ARRAY_SIZE(sa); i++) { r = damon_new_region(sa[i], ea[i]); if (!r) { damon_free_target(t); + damon_destroy_ctx(ctx); kunit_skip(test, "region alloc fail"); } r->nr_accesses = nrs[i]; damon_add_region(r, t); } - damon_merge_regions_of(t, 9, 9999); + damon_merge_regions_of(t, 9, 9999, ctx); /* 0-112, 114-130, 130-156, 156-170, 170-230, 230-10170 */ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 6u); for (i = 0; i < 6; i++) { @@ -258,6 +266,7 @@ static void damon_test_merge_regions_of(struct kunit *test) KUNIT_EXPECT_EQ(test, r->ar.end, eaddrs[i]); } damon_free_target(t); + damon_destroy_ctx(ctx); } static void damon_test_split_regions_of(struct kunit *test) From 93508425db111473a6ce734d1918b0cb5ea823d9 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 10 Jul 2026 06:46:38 -0700 Subject: [PATCH 311/501] mm/damon/core: disallow probe_hits overflow on attrs only monitoring When any damon_probe->weight is set, DAMON will do only probe monitoring. probe_hits is 'unsigned char'. It could overflow when the aggregation interval is larger than the sampling interval times 256. damon_as_probe_weights() always return false, so such overflow cannot happen. Even if it happens, it only degrades the monitoring results. That said, the overflow is not intentional. It is better to be prevented as long as the cost is not expensive. Disallow the overflow by adding a validation logic on the core layer parameters validation function. Link: https://lore.kernel.org/20260710134651.18084-10-sj@kernel.org Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/damon/core.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/mm/damon/core.c b/mm/damon/core.c index 7d49420ea26c..4f1425e56950 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1332,6 +1332,19 @@ static void damos_set_filters_default_reject(struct damos *s) damos_filters_default_reject(&s->ops_filters); } +static bool damon_valid_probe_params(struct damon_ctx *ctx) +{ + unsigned long sample_interval; + + if (!damon_has_probe_weights(ctx)) + return true; + + sample_interval = ctx->attrs.sample_interval ? : 1; + if (ctx->attrs.aggr_interval / sample_interval > U8_MAX) + return false; + return true; +} + /* * damos_commit_dests() - Copy migration destinations from @src to @dst. * @dst: Destination structure to update. @@ -1736,6 +1749,9 @@ static int __damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) } } + if (!damon_valid_probe_params(src)) + return -EINVAL; + err = damon_commit_schemes(dst, src); if (err) return err; From 23c0623f802bf5bda6523d57ee8716711d1ecf29 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 10 Jul 2026 06:46:39 -0700 Subject: [PATCH 312/501] mm/damon/core: validate params for probe hits weighted sum overflow damon_probe_hits_wsum() could overflow in weird setups. Users could set the weight unreasonably high. They could also set the aggregation interval unreasonably high compared to the sampling interval. Such user setup is unlikely. Even if such setup is used, damon_has_probe_weights() always returns false, so the overflow cannot happen. The function may be completed in future, though. Even if the overflow happens, the consequence is degraded monitoring results for the unreasonable setup. It is just a trivial user experience issue. It is still better to be prevented unless the cost is expensive. Avoid the overflow by adding the parameter validation in the core layer parameters validation function. Link: https://lore.kernel.org/20260710134651.18084-11-sj@kernel.org Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/damon/core.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/mm/damon/core.c b/mm/damon/core.c index 4f1425e56950..a0e6a5129fbf 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1335,6 +1335,9 @@ static void damos_set_filters_default_reject(struct damos *s) static bool damon_valid_probe_params(struct damon_ctx *ctx) { unsigned long sample_interval; + unsigned char max_probe_hits; + struct damon_probe *probe; + unsigned int wsum, wsum_to_add; if (!damon_has_probe_weights(ctx)) return true; @@ -1342,6 +1345,18 @@ static bool damon_valid_probe_params(struct damon_ctx *ctx) sample_interval = ctx->attrs.sample_interval ? : 1; if (ctx->attrs.aggr_interval / sample_interval > U8_MAX) return false; + + /* invalid if probe hits weighted sum can overflow */ + max_probe_hits = damon_nr_samples_per_aggr(&ctx->attrs); + wsum = 0; + damon_for_each_probe(probe, ctx) { + if (probe->weight > UINT_MAX / max_probe_hits) + return false; + wsum_to_add = probe->weight * max_probe_hits; + if (UINT_MAX - wsum < wsum_to_add) + return false; + wsum += wsum_to_add; + } return true; } From 05c978da7f0222b7bd2407fbcd6d66a3f57a61de Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 10 Jul 2026 06:46:40 -0700 Subject: [PATCH 313/501] mm/damon/core: disable access monitoring when probe weights are set When users set probe weights, they are interested in the data attributes. Not necessarily they are interested in data access patterns. Disable the data access monitoring in the case. Link: https://lore.kernel.org/20260710134651.18084-12-sj@kernel.org Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/damon/core.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index a0e6a5129fbf..65f91e3b58d2 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -3763,17 +3763,18 @@ static int kdamond_fn(void *data) unsigned long next_aggregation_sis = ctx->next_aggregation_sis; unsigned long next_ops_update_sis = ctx->next_ops_update_sis; unsigned long sample_interval = ctx->attrs.sample_interval; + bool access_check_disabled = damon_has_probe_weights(ctx); if (kdamond_wait_activation(ctx)) break; - if (ctx->ops.prepare_access_checks) + if (!access_check_disabled && ctx->ops.prepare_access_checks) ctx->ops.prepare_access_checks(ctx); kdamond_usleep(sample_interval); ctx->passed_sample_intervals++; - if (ctx->ops.check_accesses) + if (!access_check_disabled && ctx->ops.check_accesses) max_nr_accesses = ctx->ops.check_accesses(ctx); if (ctx->ops.apply_probes) ctx->ops.apply_probes(ctx, false, false); From a8ba965a4cbb9863ed399c2095eb0ab601f2dd05 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 10 Jul 2026 06:46:41 -0700 Subject: [PATCH 314/501] mm/damon/core: set samples in apply_probes() if probe weights are set When probe weights are set, the prepare_access_checks() ops callback is not invoked, so sampling addresses of regions are unset. Ask apply_probes() to set those together in the case. Link: https://lore.kernel.org/20260710134651.18084-13-sj@kernel.org Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/damon/core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 65f91e3b58d2..dd0612e1a408 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -3777,7 +3777,8 @@ static int kdamond_fn(void *data) if (!access_check_disabled && ctx->ops.check_accesses) max_nr_accesses = ctx->ops.check_accesses(ctx); if (ctx->ops.apply_probes) - ctx->ops.apply_probes(ctx, false, false); + ctx->ops.apply_probes(ctx, access_check_disabled, + false); if (time_after_eq(ctx->passed_sample_intervals, next_aggregation_sis)) { From b9bc678bfb6bfdad9b2dcdc4bce33018f81c33ac Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 10 Jul 2026 06:46:42 -0700 Subject: [PATCH 315/501] mm/damon/core: s/max_nr_accesses/max_merge_score/ in kdamond_fn() When probe weights are set, DAMON merge logic works with the probe hits weighted sum. The core logic is setting the merge threshold based on the access frequency, though. Before using probe hits based threshold, rename the variables to look more general. Link: https://lore.kernel.org/20260710134651.18084-14-sj@kernel.org Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/damon/core.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index dd0612e1a408..f7782c69037d 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -3730,7 +3730,6 @@ static void kdamond_init_ctx(struct damon_ctx *ctx) static int kdamond_fn(void *data) { struct damon_ctx *ctx = data; - unsigned int max_nr_accesses = 0; unsigned long sz_limit = 0; pr_debug("kdamond (%d) starts\n", current->pid); @@ -3764,6 +3763,7 @@ static int kdamond_fn(void *data) unsigned long next_ops_update_sis = ctx->next_ops_update_sis; unsigned long sample_interval = ctx->attrs.sample_interval; bool access_check_disabled = damon_has_probe_weights(ctx); + unsigned int max_merge_score = 0; if (kdamond_wait_activation(ctx)) break; @@ -3775,7 +3775,7 @@ static int kdamond_fn(void *data) ctx->passed_sample_intervals++; if (!access_check_disabled && ctx->ops.check_accesses) - max_nr_accesses = ctx->ops.check_accesses(ctx); + max_merge_score = ctx->ops.check_accesses(ctx); if (ctx->ops.apply_probes) ctx->ops.apply_probes(ctx, access_check_disabled, false); @@ -3783,7 +3783,7 @@ static int kdamond_fn(void *data) if (time_after_eq(ctx->passed_sample_intervals, next_aggregation_sis)) { kdamond_merge_regions(ctx, - max_nr_accesses / 10, + max_merge_score / 10, sz_limit); /* online updates might be made */ sz_limit = damon_apply_min_nr_regions(ctx); From db697312bb572aab92601c71497f90c7e5c7fb45 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 10 Jul 2026 06:46:43 -0700 Subject: [PATCH 316/501] mm/damon/core: get merge threshold from probe hits when weights are set When probe weights are set, DAMON merges regions based on their probe hits weighted sum. But the merge threshold is calculated based on the access frequency. Update it to retrieve the maximum probe hits weighted sum in the snapshot from apply_probes() ops callback, and generate the threshold based on it. Link: https://lore.kernel.org/20260710134651.18084-15-sj@kernel.org Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/damon/core.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index f7782c69037d..91964336ce31 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -3763,7 +3763,8 @@ static int kdamond_fn(void *data) unsigned long next_ops_update_sis = ctx->next_ops_update_sis; unsigned long sample_interval = ctx->attrs.sample_interval; bool access_check_disabled = damon_has_probe_weights(ctx); - unsigned int max_merge_score = 0; + unsigned int max_merge_score = 0, max_wsum; + bool get_max_wsum; if (kdamond_wait_activation(ctx)) break; @@ -3776,9 +3777,18 @@ static int kdamond_fn(void *data) if (!access_check_disabled && ctx->ops.check_accesses) max_merge_score = ctx->ops.check_accesses(ctx); - if (ctx->ops.apply_probes) - ctx->ops.apply_probes(ctx, access_check_disabled, - false); + if (ctx->ops.apply_probes) { + if (time_after_eq(ctx->passed_sample_intervals, + next_aggregation_sis) && + access_check_disabled) + get_max_wsum = true; + else + get_max_wsum = false; + max_wsum = ctx->ops.apply_probes(ctx, + access_check_disabled, get_max_wsum); + if (get_max_wsum) + max_merge_score = max_wsum; + } if (time_after_eq(ctx->passed_sample_intervals, next_aggregation_sis)) { From 8d5e0ebac9e730d25f1657ffb9f5ae98b57164f6 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 10 Jul 2026 06:46:44 -0700 Subject: [PATCH 317/501] mm/damon/core: implement damon_has_probe_weight() DAMON monitors only data attributes when probe weights are set. However, the function for detecting it is intentionally returning false always. The intention was to make changes for complete support of the mode in a safe and incremental way. Now all the building blocks are made. Complete the mode detection function so that the mode can really start working. Link: https://lore.kernel.org/20260710134651.18084-16-sj@kernel.org Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/damon/core.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mm/damon/core.c b/mm/damon/core.c index 91964336ce31..806a67d02a6e 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -211,6 +211,12 @@ static struct damon_probe *damon_nth_probe(int n, struct damon_ctx *ctx) static bool damon_has_probe_weights(struct damon_ctx *c) { + struct damon_probe *p; + + damon_for_each_probe(p, c) { + if (p->weight) + return true; + } return false; } From 2430a8d5172c257158091368b81e27fdce535475 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 10 Jul 2026 06:46:45 -0700 Subject: [PATCH 318/501] mm/damon/sysfs: implement probe/weight file Implement a new sysfs file, 'weight', under probe directory. Users will be able to set the probe weight and enable the attributes only monitoring using it. Link: https://lore.kernel.org/20260710134651.18084-17-sj@kernel.org Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/damon/sysfs.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 6d7e36c9e509..b5fe036f7801 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -1065,6 +1065,7 @@ static const struct kobj_type damon_sysfs_filters_ktype = { struct damon_sysfs_probe { struct kobject kobj; + unsigned int weight; struct damon_sysfs_filters *filters; }; @@ -1100,12 +1101,35 @@ static void damon_sysfs_probe_rm_dirs(struct damon_sysfs_probe *probe) } } +static ssize_t weight_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + struct damon_sysfs_probe *probe = container_of(kobj, + struct damon_sysfs_probe, kobj); + + return sysfs_emit(buf, "%u\n", probe->weight); +} + +static ssize_t weight_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + struct damon_sysfs_probe *probe = container_of(kobj, + struct damon_sysfs_probe, kobj); + int err = kstrtouint(buf, 0, &probe->weight); + + return err ? err : count; +} + static void damon_sysfs_probe_release(struct kobject *kobj) { kfree(container_of(kobj, struct damon_sysfs_probe, kobj)); } +static struct kobj_attribute damon_sysfs_probe_weight_attr = + __ATTR_RW_MODE(weight, 0600); + static struct attribute *damon_sysfs_probe_attrs[] = { + &damon_sysfs_probe_weight_attr.attr, NULL, }; ATTRIBUTE_GROUPS(damon_sysfs_probe); @@ -1965,6 +1989,7 @@ static int damon_sysfs_set_probes(struct damon_ctx *ctx, return -ENOMEM; damon_add_probe(ctx, p); sys_probe = sys_probes->probes_arr[i]; + p->weight = sys_probe->weight; err = damon_sysfs_set_probe(p, sys_probe); if (err) return err; From f10489d2ef8e76bbb3eb78c5b963abae3a4a8dc9 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 10 Jul 2026 06:46:46 -0700 Subject: [PATCH 319/501] Docs/mm/damon/design: document attrs-only monitoring Update DAMON design document for the attributes only monitoring mode. Link: https://lore.kernel.org/20260710134651.18084-18-sj@kernel.org Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- Documentation/mm/damon/design.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index 457d6e8bc787..6d79e7d69be8 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -316,6 +316,26 @@ Another way to do this for higher accuracy is using :ref:`DAMOS filter information in page level. But, because it is operated in page level, the overhead is proportional to the size of the memory. +Data Attributes-only Monitoring +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Data access is the primary monitoring information for DAMON. Hence it +:ref:`adjusts ` regions using the access +:ref:`counter ` (``nr_accesses``). In some +use cases, however, users may want some of :ref:`attributes +` to be the primary information. + +Data attributes-only monitoring mode supports this use case. For the mode, +each attribute probe has their priority weight value. Users can describe by +what combination of the attributes the primary information is decided, by +setting the priority weight value. If the total sum of the weights is not +zero, the mode is enabled. The regions adjustment mechanism uses the weighted +sum of the :ref:`probe hit counts ` instead +of ``nr_accesses`` in the case. When the mode is enabled, access monitoring is +automatically turned off. The access counter (``nr_accesses``) will always be +zero and not updated. Hence the mode is called Data Attributes "only" +monitoring. + Dynamic Target Space Updates Handling ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 3baf78e04b0bc3ed285ff796ba905cd2a3185756 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 10 Jul 2026 06:46:47 -0700 Subject: [PATCH 320/501] Docs/admin-guide/mm/damon/usage: document weight sysfs file Update DAMON usage document for newly added probe weight file. Link: https://lore.kernel.org/20260710134651.18084-19-sj@kernel.org Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- Documentation/admin-guide/mm/damon/usage.rst | 11 ++++++++--- Documentation/mm/damon/design.rst | 5 +++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Documentation/admin-guide/mm/damon/usage.rst b/Documentation/admin-guide/mm/damon/usage.rst index b2649ea011f9..da5f9afd08ae 100644 --- a/Documentation/admin-guide/mm/damon/usage.rst +++ b/Documentation/admin-guide/mm/damon/usage.rst @@ -73,9 +73,10 @@ comma (","). │ │ │ │ │ │ │ intervals_goal/access_bp,aggrs,min_sample_us,max_sample_us │ │ │ │ │ │ nr_regions/min,max │ │ │ │ │ │ :ref:`probes `/nr_probes - │ │ │ │ │ │ │ 0/filters/nr_filters - │ │ │ │ │ │ │ │ 0/type,matching,allow,path - │ │ │ │ │ │ │ │ ... + │ │ │ │ │ │ │ 0/weight + │ │ │ │ │ │ │ │ filters/nr_filters + │ │ │ │ │ │ │ │ │ 0/type,matching,allow,path + │ │ │ │ │ │ │ │ │ ... │ │ │ │ │ │ │ ... │ │ │ │ │ :ref:`targets `/nr_targets │ │ │ │ │ │ :ref:`0 `/pid_target,obsolete_target @@ -286,6 +287,10 @@ In each probe directory, one directory, ``filters`` exists. The directory contains files for installing filters for the probe, that is used to determine the data attribute for the probe. +Each probe directory also contains ``weight`` file. Reading from and writing +to the file gets and sets the :ref:`attributes-only monitoring +` weight for the attribute of the probe. + In the beginning, ``filters`` directory has only one file, ``nr_filters``. Writing a number (``N``) to the file creates the number of child directories named ``0`` to ``N-1``. Each directory represents each filter and works in a diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index 6d79e7d69be8..aed6cb1cf483 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -316,6 +316,8 @@ Another way to do this for higher accuracy is using :ref:`DAMOS filter information in page level. But, because it is operated in page level, the overhead is proportional to the size of the memory. +.. _damon_design_attrs_only_monitoring: + Data Attributes-only Monitoring ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -336,6 +338,9 @@ automatically turned off. The access counter (``nr_accesses``) will always be zero and not updated. Hence the mode is called Data Attributes "only" monitoring. +Refer to the :ref:`admin guide ` to know how users +can use the mode. + Dynamic Target Space Updates Handling ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From ac59b45efd3cec28860f7d50816b888650a2eb02 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 10 Jul 2026 06:46:48 -0700 Subject: [PATCH 321/501] Docs/ABI/damon: document probe weight file Update DAMON ABI document for the newly added probe weight sysfs file. Link: https://lore.kernel.org/20260710134651.18084-20-sj@kernel.org Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- Documentation/ABI/testing/sysfs-kernel-mm-damon | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-damon b/Documentation/ABI/testing/sysfs-kernel-mm-damon index dd6b5bd76e11..907a504fb64c 100644 --- a/Documentation/ABI/testing/sysfs-kernel-mm-damon +++ b/Documentation/ABI/testing/sysfs-kernel-mm-damon @@ -164,6 +164,12 @@ Description: Writing a number 'N' to this file creates the number of directories for each DAMON probe named '0' to 'N-1' under the probes/ directory. +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/weight +Date: Jul 2026 +Contact: SJ Park +Description: Writing to and reading from this file sets and gets the + per-probe attribute weight. + What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/filters/nr_filters Date: May 2026 Contact: SJ Park From cf6daf24938809ad4777f189a0bce4febb378354 Mon Sep 17 00:00:00 2001 From: Andrew Morton Date: Mon, 27 Jul 2026 17:31:14 -0700 Subject: [PATCH 322/501] mm/hmm: make CONFIG_HMM_MIRROR select CONFIG_MMU_NOTIFIER hmm doesn't currently build well with MMU_NOTIFIER=n: With microblaze https://download.01.org/0day-ci/archive/20260702/202607021433.DYT5fDqE-lkp@intel.com/config (CONFIG_MMU_NOTIFIER=n): mm/hmm.c: In function 'hmm_range_fault_unlocked_timeout': mm/hmm.c:804:25: error: implicit declaration of function 'mmu_interval_read_begin'; did you mean 'mmu_interval_check_retry'? [-Wimplicit-function-declaration] 804 | mmu_interval_read_begin(range->notifier); | ^~~~~~~~~~~~~~~~~~~~~~~ | mmu_interval_check_retry Quoting Stanislav: : Documentation/mm/hmm.rst explicitly states: : : "Address space mirroring's main objective is to allow duplication of a : range of CPU page table into a device page table; HMM helps keep both : synchronized. A device driver that wants to mirror a process address : space must start with the registration of a mmu_interval_notifier" : : I think making CONFIG_HMM_MIRROR dependent on the CONFIG_MMU_NOTIFIER : is the right thing to do. So select MMU_NOTIFIER if hmm.c is to be compiled. Cc: David Hildenbrand Cc: Jason Gunthorpe Cc: Leon Romanovsky Cc: Stanislav Kinsburskii Signed-off-by: Andrew Morton --- mm/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/Kconfig b/mm/Kconfig index 0d06e96ff3c9..3185500ce7b7 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1252,6 +1252,7 @@ config ZONE_DEVICE config HMM_MIRROR bool depends on MMU + select MMU_NOTIFIER config GET_FREE_REGION bool From 1ef12d3b3068facad2c94bb84a7e1e3f8bc2252f Mon Sep 17 00:00:00 2001 From: Stanislav Kinsburskii Date: Thu, 23 Jul 2026 10:36:33 -0700 Subject: [PATCH 323/501] mm/hmm: move page fault handling out of walk callbacks Patch series "mm/hmm: Add mmap lock-drop support for userfaultfd-backed mappings", v11. This series extends the HMM framework to support userfaultfd-backed memory by allowing the mmap read lock to be dropped during hmm_range_fault(). Some page fault handlers most notably userfaultfd require the mmap lock to be released so that userspace can resolve the fault. The current HMM interface never sets FAULT_FLAG_ALLOW_RETRY, making it impossible to fault in pages from userfaultfd-registered regions. This series follows the established int *locked pattern from get_user_pages_remote() in mm/gup.c. A new helper function, hmm_range_fault_locked(), accepts an int *locked parameter. When the mmap lock is dropped during fault resolution (VM_FAULT_RETRY or VM_FAULT_COMPLETED), the function returns 0 with *locked = 0, signalling the caller to restart its walk. The existing hmm_range_fault() is refactored into a thin wrapper that passes NULL, preserving current behavior for all existing callers. Possible approaches to lift this limitation are documented in Documentation/mm/hmm.rst. This patch (of 8): hmm_range_fault() currently triggers page faults from inside the page-table walk callbacks: hmm_vma_walk_pmd(), hmm_vma_walk_pud(), hmm_vma_walk_hugetlb_entry() and the pte-level helper all call hmm_vma_fault(), which in turn calls handle_mm_fault() while the walker still holds nested locks. The pte spinlock is dropped explicitly by each caller, and the hugetlb path manually drops and retakes hugetlb_vma_lock_read around the fault to dodge a deadlock against the walk framework's unconditional unlock. This layering does not extend cleanly to fault handlers that may release mmap_lock (VM_FAULT_RETRY, VM_FAULT_COMPLETED). If the lock is dropped while walk_page_range() is mid-traversal, the VMA can be freed before the walk framework's matching hugetlb_vma_unlock_read(), turning that unlock into a use-after-free. Split the responsibilities the way get_user_pages() does. Walk callbacks become inspect-only: when they detect a range that needs to be faulted in, they record it in struct hmm_vma_walk and return a private sentinel (HMM_FAULT_PENDING). The outer loop in hmm_range_fault() then drops out of walk_page_range(), invokes a new helper hmm_do_fault() that calls handle_mm_fault() with only mmap_lock held, and restarts the walk so the now-present entries are collected into hmm_pfns. No functional change for existing callers. As a side effect the hugetlb callback no longer needs the hugetlb_vma_{un}lock_read dance, and every fault-path exit from the callbacks now releases the pte spinlock on a single, common path. This refactor is also a precursor for adding an unlockable variant of hmm_range_fault() in a follow-up patch. Link: https://lore.kernel.org/20260723-hmm-v10-v11-0-c55b003a4b61@gmail.com Link: https://lore.kernel.org/20260723-hmm-v10-v11-1-c55b003a4b61@gmail.com Signed-off-by: Stanislav Kinsburskii Reviewed-by: Jason Gunthorpe Reviewed-by: David Hildenbrand (Arm) Cc: Danilo Krummrich Cc: Dave Airlie Cc: Dexuan Cui Cc: Haiyang Zhang Cc: Jonathan Corbet Cc: K. Y. Srinivasan Cc: Leon Romanovsky Cc: Liam R. Howlett Cc: Lizhi Hou Cc: Long Li Cc: Lorenzo Stoakes Cc: Lyude Cc: Maarten Lankhorst Cc: Maxime Ripard Cc: Michal Hocko Cc: Mike Rapoport Cc: Oded Gabbay Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Thomas Zimemrmann Cc: Vlastimil Babka Cc: Wei Liu Signed-off-by: Andrew Morton --- mm/hmm.c | 118 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 75 insertions(+), 43 deletions(-) diff --git a/mm/hmm.c b/mm/hmm.c index e5c1f4deed24..bc9361a715fa 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -33,8 +33,17 @@ struct hmm_vma_walk { struct hmm_range *range; unsigned long last; + unsigned long end; + unsigned int required_fault; }; +/* + * Internal sentinel returned by walk callbacks when they need a page fault. + * The callback stores end/required_fault in hmm_vma_walk; the outer loop + * consumes the sentinel and never propagates it to the caller. + */ +#define HMM_FAULT_PENDING -EAGAIN + enum { HMM_NEED_FAULT = 1 << 0, HMM_NEED_WRITE_FAULT = 1 << 1, @@ -60,37 +69,25 @@ static int hmm_pfns_fill(unsigned long addr, unsigned long end, } /* - * hmm_vma_fault() - fault in a range lacking valid pmd or pte(s) - * @addr: range virtual start address (inclusive) - * @end: range virtual end address (exclusive) - * @required_fault: HMM_NEED_* flags - * @walk: mm_walk structure - * Return: -EBUSY after page fault, or page fault error + * hmm_record_fault() - record a range that needs to be faulted in * - * This function will be called whenever pmd_none() or pte_none() returns true, - * or whenever there is no page directory covering the virtual address range. + * Called by the walk callbacks when they discover that part of the range + * needs a page fault. The callback records what to fault and returns + * HMM_FAULT_PENDING; the outer loop in hmm_range_fault() drops back out of + * walk_page_range() and invokes handle_mm_fault() from a context where no + * page-table or hugetlb_vma_lock is held. */ -static int hmm_vma_fault(unsigned long addr, unsigned long end, - unsigned int required_fault, struct mm_walk *walk) +static int hmm_record_fault(unsigned long addr, unsigned long end, + unsigned int required_fault, + struct mm_walk *walk) { struct hmm_vma_walk *hmm_vma_walk = walk->private; - struct vm_area_struct *vma = walk->vma; - unsigned int fault_flags = FAULT_FLAG_REMOTE; WARN_ON_ONCE(!required_fault); hmm_vma_walk->last = addr; - - if (required_fault & HMM_NEED_WRITE_FAULT) { - if (!(vma->vm_flags & VM_WRITE)) - return -EPERM; - fault_flags |= FAULT_FLAG_WRITE; - } - - for (; addr < end; addr += PAGE_SIZE) - if (handle_mm_fault(vma, addr, fault_flags, NULL) & - VM_FAULT_ERROR) - return -EFAULT; - return -EBUSY; + hmm_vma_walk->end = end; + hmm_vma_walk->required_fault = required_fault; + return HMM_FAULT_PENDING; } static unsigned int hmm_pte_need_fault(const struct hmm_vma_walk *hmm_vma_walk, @@ -174,7 +171,7 @@ static int hmm_vma_walk_hole(unsigned long addr, unsigned long end, return hmm_pfns_fill(addr, end, range, HMM_PFN_ERROR); } if (required_fault) - return hmm_vma_fault(addr, end, required_fault, walk); + return hmm_record_fault(addr, end, required_fault, walk); return hmm_pfns_fill(addr, end, range, 0); } @@ -209,7 +206,7 @@ static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr, required_fault = hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, cpu_flags); if (required_fault) - return hmm_vma_fault(addr, end, required_fault, walk); + return hmm_record_fault(addr, end, required_fault, walk); pfn = pmd_pfn(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++) { @@ -328,7 +325,7 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr, fault: pte_unmap(ptep); /* Fault any virtual address we were asked to fault */ - return hmm_vma_fault(addr, end, required_fault, walk); + return hmm_record_fault(addr, end, required_fault, walk); } #ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES @@ -371,7 +368,7 @@ static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, npages, 0); if (required_fault) { if (softleaf_is_device_private(entry)) - return hmm_vma_fault(addr, end, required_fault, walk); + return hmm_record_fault(addr, end, required_fault, walk); else return -EFAULT; } @@ -517,7 +514,7 @@ static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end, npages, cpu_flags); if (required_fault) { spin_unlock(ptl); - return hmm_vma_fault(addr, end, required_fault, walk); + return hmm_record_fault(addr, end, required_fault, walk); } pfn = pud_pfn(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); @@ -564,21 +561,8 @@ static int hmm_vma_walk_hugetlb_entry(pte_t *pte, unsigned long hmask, required_fault = hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, cpu_flags); if (required_fault) { - int ret; - spin_unlock(ptl); - hugetlb_vma_unlock_read(vma); - /* - * Avoid deadlock: drop the vma lock before calling - * hmm_vma_fault(), which will itself potentially take and - * drop the vma lock. This is also correct from a - * protection point of view, because there is no further - * use here of either pte or ptl after dropping the vma - * lock. - */ - ret = hmm_vma_fault(addr, end, required_fault, walk); - hugetlb_vma_lock_read(vma); - return ret; + return hmm_record_fault(addr, end, required_fault, walk); } pfn = pte_pfn(entry) + ((start & ~hmask) >> PAGE_SHIFT); @@ -637,6 +621,44 @@ static const struct mm_walk_ops hmm_walk_ops = { .walk_lock = PGWALK_RDLOCK, }; +/* + * hmm_do_fault - fault in a range recorded by a walk callback + * + * Called from the outer loop in hmm_range_fault() after a callback + * returned HMM_FAULT_PENDING. At this point we hold only mmap_lock; + * the page-table spinlock and any hugetlb_vma_lock acquired by the walk + * framework have already been released by the unwind. + * + * Returns -EBUSY on success (all pages faulted, caller should re-walk). + * Returns a negative errno on failure. + */ +static int hmm_do_fault(struct mm_struct *mm, + struct hmm_vma_walk *hmm_vma_walk) +{ + unsigned long addr = hmm_vma_walk->last; + unsigned long end = hmm_vma_walk->end; + unsigned int required_fault = hmm_vma_walk->required_fault; + unsigned int fault_flags = FAULT_FLAG_REMOTE; + struct vm_area_struct *vma; + + vma = vma_lookup(mm, addr); + if (!vma) + return -EFAULT; + + if (required_fault & HMM_NEED_WRITE_FAULT) { + if (!(vma->vm_flags & VM_WRITE)) + return -EPERM; + fault_flags |= FAULT_FLAG_WRITE; + } + + for (; addr < end; addr += PAGE_SIZE) + if (handle_mm_fault(vma, addr, fault_flags, NULL) & + VM_FAULT_ERROR) + return -EFAULT; + + return -EBUSY; +} + /** * hmm_range_fault - try to fault some address in a virtual address range * @range: argument structure @@ -674,6 +696,16 @@ int hmm_range_fault(struct hmm_range *range) return -EBUSY; ret = walk_page_range(mm, hmm_vma_walk.last, range->end, &hmm_walk_ops, &hmm_vma_walk); + /* + * When HMM_FAULT_PENDING is returned a walk callback + * recorded a range that needs handle_mm_fault(); + * hmm_do_fault() runs the fault outside walk_page_range() + * (so no page-table or hugetlb_vma_lock is held) and + * returns -EBUSY so the loop re-walks and picks up the + * now-present entries. + */ + if (ret == HMM_FAULT_PENDING) + ret = hmm_do_fault(mm, &hmm_vma_walk); /* * When -EBUSY is returned the loop restarts with * hmm_vma_walk.last set to an address that has not been stored From 121170831228ef78be14b0d866b165a29c70dda3 Mon Sep 17 00:00:00 2001 From: Stanislav Kinsburskii Date: Thu, 23 Jul 2026 10:36:34 -0700 Subject: [PATCH 324/501] mm/hmm: add hmm_range_fault_unlocked_timeout() for mmap lock-drop support hmm_range_fault() requires the caller to hold the mmap read lock for the duration of the call. This is incompatible with mappings whose fault handler may release the mmap lock, notably userfaultfd-managed regions, where handle_mm_fault() can return VM_FAULT_RETRY or VM_FAULT_COMPLETED after dropping the lock. Drivers that need to populate device page tables for such mappings have no way to do so today. Add hmm_range_fault_unlocked_timeout() for callers that do not need to hold mmap_lock across any work outside the HMM fault itself. The helper takes mmap_read_lock_killable() internally, calls the common HMM fault implementation, and releases the lock before returning if it is still held. The timeout is specified in jiffies; passing 0 retries indefinitely, while a non-zero timeout makes the helper return -EBUSY when the retry budget expires. The retry deadline is set before refreshing the notifier sequence and acquiring mmap_lock, so contended mmap_lock acquisition is included in the retry budget. After acquiring mmap_lock, the helper also rejects unstable address spaces before walking page tables. When handle_mm_fault() drops mmap_lock, or when the range is invalidated, hmm_range_fault_unlocked_timeout() refreshes range->notifier_seq and retries the walk internally. If the lock was dropped, the retry deadline is also restarted because a lock-dropping fault handler made progress. Ordinary -EBUSY retries keep the existing deadline, preserving the caller's timeout policy for repeated mmu-notifier invalidations. The caller only needs to perform the usual post-success mmu_interval_read_retry() check while holding its update lock before consuming the pfns. If mmap_lock acquisition is interrupted or a fatal signal is pending during retry handling, -EINTR is returned instead. The common implementation conditionally sets FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_KILLABLE only for hmm_range_fault_unlocked_timeout(). The existing hmm_range_fault() path still passes no locked state, does not allow handle_mm_fault() to drop mmap_lock, and remains a thin wrapper preserving the existing API contract for current callers. The previous refactor that moved page fault handling out of the page-table walk callbacks is what makes this change small. Faults now run after walk_page_range() has unwound, with only mmap_lock held, so dropping it does not interact with the walker's pte spinlock or hugetlb_vma_lock. Hugetlb regions therefore participate in the unlocked path uniformly with PTE- and PMD-level mappings; no special case is required. Documentation/mm/hmm.rst is updated with a description of the new API and the recommended caller pattern. Link: https://lore.kernel.org/20260723-hmm-v10-v11-2-c55b003a4b61@gmail.com Signed-off-by: Stanislav Kinsburskii Cc: Danilo Krummrich Cc: Dave Airlie Cc: David Hildenbrand Cc: Dexuan Cui Cc: Haiyang Zhang Cc: Jason Gunthorpe Cc: Jonathan Corbet Cc: K. Y. Srinivasan Cc: Leon Romanovsky Cc: Liam R. Howlett Cc: Lizhi Hou Cc: Long Li Cc: Lorenzo Stoakes Cc: Lyude Cc: Maarten Lankhorst Cc: Maxime Ripard Cc: Michal Hocko Cc: Mike Rapoport Cc: Oded Gabbay Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Thomas Zimemrmann Cc: Vlastimil Babka Cc: Wei Liu Signed-off-by: Andrew Morton --- Documentation/mm/hmm.rst | 79 ++++++++++++------ include/linux/hmm.h | 2 + mm/hmm.c | 168 ++++++++++++++++++++++++++++++++------- 3 files changed, 197 insertions(+), 52 deletions(-) diff --git a/Documentation/mm/hmm.rst b/Documentation/mm/hmm.rst index 7d61b7a8b65b..e021218ada58 100644 --- a/Documentation/mm/hmm.rst +++ b/Documentation/mm/hmm.rst @@ -156,42 +156,57 @@ During the ops->invalidate() callback the device driver must perform the update action to the range (mark range read only, or fully unmap, etc.). The device must complete the update before the driver callback returns. -When the device driver wants to populate a range of virtual addresses, it can -use:: +When the device driver wants to populate a range of virtual addresses, the +normal interface is:: - int hmm_range_fault(struct hmm_range *range); + int hmm_range_fault_unlocked_timeout(struct hmm_range *range, + unsigned long timeout); It will trigger a page fault on missing or read-only entries if write access is requested (see below). Page faults use the generic mm page fault code path just -like a CPU page fault. The usage pattern is:: +like a CPU page fault. + +The caller must not hold ``mmap_read_lock`` before the call. +``hmm_range_fault_unlocked_timeout()`` takes the mmap read lock internally and +allows ``handle_mm_fault()`` to drop it during fault handling. This is required +for VMAs whose fault handlers may release the mmap lock, for example regions +managed by ``userfaultfd``. + +If the mmap lock is dropped or the range is invalidated, the function refreshes +``range->notifier_seq`` and restarts the walk internally. ``-EINTR`` is returned +if mmap lock acquisition is interrupted or a fatal signal is pending during +retry handling. + +The timeout is specified in jiffies; passing ``0`` means retry indefinitely. The +timeout exists to preserve caller policy for repeated mmu-notifier invalidation +and is checked between retry attempts. HMM does not interrupt page fault +handling when the timeout expires, but returns ``-EBUSY`` if the retry budget is +exhausted before a stable range is obtained. + +The usage pattern is:: int driver_populate_range(...) { struct hmm_range range; + unsigned long timeout; ... + timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); range.notifier = &interval_sub; range.start = ...; range.end = ...; range.hmm_pfns = ...; - if (!mmget_not_zero(interval_sub->notifier.mm)) + if (!mmget_not_zero(interval_sub.mm)) return -EFAULT; again: - range.notifier_seq = mmu_interval_read_begin(&interval_sub); - mmap_read_lock(mm); - ret = hmm_range_fault(&range); - if (ret) { - mmap_read_unlock(mm); - if (ret == -EBUSY) - goto again; - return ret; - } - mmap_read_unlock(mm); + ret = hmm_range_fault_unlocked_timeout(&range, timeout); + if (ret) + goto out_put; take_lock(driver->update); - if (mmu_interval_read_retry(&ni, range.notifier_seq) { + if (mmu_interval_read_retry(range.notifier, range.notifier_seq)) { release_lock(driver->update); goto again; } @@ -200,13 +215,31 @@ like a CPU page fault. The usage pattern is:: * under the update lock */ release_lock(driver->update); - return 0; + ret = 0; + + out_put: + mmput(interval_sub.mm); + return ret; } The driver->update lock is the same lock that the driver takes inside its invalidate() callback. That lock must be held before calling mmu_interval_read_retry() to avoid any race with a concurrent CPU page table -update. +update. The retry check must use the same notifier and sequence number stored +in ``range`` by ``hmm_range_fault_unlocked_timeout()``. + +Holding the mmap lock across HMM faults +======================================= + +Most callers should use ``hmm_range_fault_unlocked_timeout()``. If a driver +really needs to hold the mmap lock across work outside HMM, it can use:: + + int hmm_range_fault(struct hmm_range *range); + +The mmap lock must be held by the caller and will remain held on return. This +interface cannot support VMAs whose fault handlers need to drop the mmap lock. +New callers should prefer ``hmm_range_fault_unlocked_timeout()`` unless they +have a specific requirement to keep the mmap lock held across the call. Leverage default_flags and pfn_flags_mask ========================================= @@ -221,8 +254,8 @@ permission, it sets:: range->default_flags = HMM_PFN_REQ_FAULT; range->pfn_flags_mask = 0; -and calls hmm_range_fault() as described above. This will fill fault all pages -in the range with at least read permission. +and calls the HMM range fault helper as described above. This will fault +all pages in the range with at least read permission. Now let's say the driver wants to do the same except for one page in the range for which it wants to have write permission. Now driver set:: @@ -236,9 +269,9 @@ address == range->start + (index_of_write << PAGE_SHIFT) it will fault with write permission i.e., if the CPU pte does not have write permission set then HMM will call handle_mm_fault(). -After hmm_range_fault completes the flag bits are set to the current state of -the page tables, ie HMM_PFN_VALID | HMM_PFN_WRITE will be set if the page is -writable. +After the HMM range fault helper completes the flag bits are set to the +current state of the page tables, ie HMM_PFN_VALID | HMM_PFN_WRITE will be +set if the page is writable. Represent and manage device memory from core kernel point of view diff --git a/include/linux/hmm.h b/include/linux/hmm.h index db75ffc949a7..6f04e3932f5b 100644 --- a/include/linux/hmm.h +++ b/include/linux/hmm.h @@ -123,6 +123,8 @@ struct hmm_range { * Please see Documentation/mm/hmm.rst for how to use the range API. */ int hmm_range_fault(struct hmm_range *range); +int hmm_range_fault_unlocked_timeout(struct hmm_range *range, + unsigned long timeout); /* * HMM_RANGE_DEFAULT_TIMEOUT - default timeout (ms) when waiting for a range diff --git a/mm/hmm.c b/mm/hmm.c index bc9361a715fa..2b05c53b82dc 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -32,6 +33,7 @@ struct hmm_vma_walk { struct hmm_range *range; + bool *locked; unsigned long last; unsigned long end; unsigned int required_fault; @@ -44,6 +46,14 @@ struct hmm_vma_walk { */ #define HMM_FAULT_PENDING -EAGAIN +/* + * Internal sentinel returned by hmm_do_fault() when handle_mm_fault() + * completes a page fault with the mmap lock dropped. hmm_do_fault() sets + * *locked = false; the outer loop consumes the sentinel and never propagates + * it to the caller. + */ +#define HMM_FAULT_UNLOCKED -ENOLCK + enum { HMM_NEED_FAULT = 1 << 0, HMM_NEED_WRITE_FAULT = 1 << 1, @@ -73,9 +83,9 @@ static int hmm_pfns_fill(unsigned long addr, unsigned long end, * * Called by the walk callbacks when they discover that part of the range * needs a page fault. The callback records what to fault and returns - * HMM_FAULT_PENDING; the outer loop in hmm_range_fault() drops back out of - * walk_page_range() and invokes handle_mm_fault() from a context where no - * page-table or hugetlb_vma_lock is held. + * HMM_FAULT_PENDING; the outer loop in hmm_range_fault_locked() drops + * back out of walk_page_range() and invokes handle_mm_fault() from a context + * where no page-table or hugetlb_vma_lock is held. */ static int hmm_record_fault(unsigned long addr, unsigned long end, unsigned int required_fault, @@ -624,7 +634,7 @@ static const struct mm_walk_ops hmm_walk_ops = { /* * hmm_do_fault - fault in a range recorded by a walk callback * - * Called from the outer loop in hmm_range_fault() after a callback + * Called from the outer loop in hmm_range_fault_locked() after a callback * returned HMM_FAULT_PENDING. At this point we hold only mmap_lock; * the page-table spinlock and any hugetlb_vma_lock acquired by the walk * framework have already been released by the unwind. @@ -641,6 +651,9 @@ static int hmm_do_fault(struct mm_struct *mm, unsigned int fault_flags = FAULT_FLAG_REMOTE; struct vm_area_struct *vma; + if (hmm_vma_walk->locked) + fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + vma = vma_lookup(mm, addr); if (!vma) return -EFAULT; @@ -651,37 +664,34 @@ static int hmm_do_fault(struct mm_struct *mm, fault_flags |= FAULT_FLAG_WRITE; } - for (; addr < end; addr += PAGE_SIZE) - if (handle_mm_fault(vma, addr, fault_flags, NULL) & - VM_FAULT_ERROR) - return -EFAULT; + for (; addr < end; addr += PAGE_SIZE) { + vm_fault_t ret; + + ret = handle_mm_fault(vma, addr, fault_flags, NULL); + + if (ret & (VM_FAULT_COMPLETED | VM_FAULT_RETRY)) { + *hmm_vma_walk->locked = false; + return HMM_FAULT_UNLOCKED; + } + + if (ret & VM_FAULT_ERROR) { + int err = vm_fault_to_errno(ret, 0); + + if (WARN_ON(!err)) + err = -EINVAL; + + return err; + } + } return -EBUSY; } -/** - * hmm_range_fault - try to fault some address in a virtual address range - * @range: argument structure - * - * Returns 0 on success or one of the following error codes: - * - * -EINVAL: Invalid arguments or mm or virtual address is in an invalid vma - * (e.g., device file vma). - * -ENOMEM: Out of memory. - * -EPERM: Invalid permission (e.g., asking for write and range is read - * only). - * -EBUSY: The range has been invalidated and the caller needs to wait for - * the invalidation to finish. - * -EFAULT: A page was requested to be valid and could not be made valid - * ie it has no backing VMA or it is illegal to access - * - * This is similar to get_user_pages(), except that it can read the page tables - * without mutating them (ie causing faults). - */ -int hmm_range_fault(struct hmm_range *range) +static int hmm_range_fault_locked(struct hmm_range *range, bool *locked) { struct hmm_vma_walk hmm_vma_walk = { .range = range, + .locked = locked, .last = range->start, }; struct mm_struct *mm = range->notifier->mm; @@ -704,8 +714,14 @@ int hmm_range_fault(struct hmm_range *range) * returns -EBUSY so the loop re-walks and picks up the * now-present entries. */ - if (ret == HMM_FAULT_PENDING) + if (ret == HMM_FAULT_PENDING) { ret = hmm_do_fault(mm, &hmm_vma_walk); + if (ret == HMM_FAULT_UNLOCKED) { + if (fatal_signal_pending(current)) + return -EINTR; + return -EBUSY; + } + } /* * When -EBUSY is returned the loop restarts with * hmm_vma_walk.last set to an address that has not been stored @@ -715,8 +731,102 @@ int hmm_range_fault(struct hmm_range *range) } while (ret == -EBUSY); return ret; } + +/** + * hmm_range_fault - try to fault some address in a virtual address range + * @range: argument structure + * + * Returns 0 on success or one of the following error codes: + * + * -EINVAL: Invalid arguments or mm or virtual address is in an invalid vma + * (e.g., device file vma). + * -ENOMEM: Out of memory. + * -EPERM: Invalid permission (e.g., asking for write and range is read + * only). + * -EBUSY: The range has been invalidated and the caller needs to wait for + * the invalidation to finish. + * -EFAULT: A page was requested to be valid and could not be made valid + * ie it has no backing VMA or it is illegal to access + * + * This is similar to get_user_pages(), except that it can read the page tables + * without mutating them (ie causing faults). + * + * The mmap lock must be held by the caller and will remain held on return. + * New users should prefer hmm_range_fault_unlocked_timeout() unless they + * specifically need to keep the mmap lock held across the call. This helper + * cannot support VMAs whose fault handlers need to drop the mmap lock. + */ +int hmm_range_fault(struct hmm_range *range) +{ + return hmm_range_fault_locked(range, NULL); +} EXPORT_SYMBOL(hmm_range_fault); +/** + * hmm_range_fault_unlocked_timeout - fault in a range with a retry timeout + * @range: argument structure + * @timeout: timeout in jiffies for internal -EBUSY retries, or 0 to retry + * indefinitely + * + * The caller must not hold the mmap lock. The function takes the mmap read + * lock internally and allows handle_mm_fault() to drop it during faults. If + * the mmap lock is dropped or the range is invalidated, the function refreshes + * range->notifier_seq and restarts the walk internally. + * + * Passing 0 for @timeout retries indefinitely. A non-zero @timeout is a caller + * policy limit for repeated mmu-notifier invalidation retries. HMM does not + * interrupt page fault handling when the timeout expires, but returns -EBUSY + * if the retry budget is exhausted before a stable range is obtained. + * + * Returns 0 on success or one of the error codes documented for + * hmm_range_fault(). -EINTR is returned if mmap_lock acquisition is + * interrupted or a fatal signal is pending during retry handling. + */ +int hmm_range_fault_unlocked_timeout(struct hmm_range *range, + unsigned long timeout) +{ + struct mm_struct *mm = range->notifier->mm; + unsigned long deadline = 0; + bool locked = false; + int ret; + + do { + /* + * If the previous fault dropped mmap_lock, then the fault + * handler made progress. Restart the retry timeout in that + * case, but keep the existing deadline for ordinary -EBUSY + * retries. + */ + if (timeout && !locked) + deadline = jiffies + timeout; + + range->notifier_seq = + mmu_interval_read_begin(range->notifier); + + ret = mmap_read_lock_killable(mm); + if (ret) + return ret; + + if (check_stable_address_space(mm)) { + mmap_read_unlock(mm); + return -EFAULT; + } + + if (timeout && time_after(jiffies, deadline)) { + mmap_read_unlock(mm); + return -EBUSY; + } + + locked = true; + ret = hmm_range_fault_locked(range, &locked); + if (locked) + mmap_read_unlock(mm); + } while (ret == -EBUSY); + + return ret; +} +EXPORT_SYMBOL(hmm_range_fault_unlocked_timeout); + /** * hmm_dma_map_alloc - Allocate HMM map structure * @dev: device to allocate structure for From 9365813cadfb086c04040278dcc7f463183ba727 Mon Sep 17 00:00:00 2001 From: Stanislav Kinsburskii Date: Thu, 23 Jul 2026 10:36:35 -0700 Subject: [PATCH 325/501] selftests/mm: add HMM test for mmap lock-dropping faults Add test_hmm coverage for the HMM lock-dropping fault path. The test module gets a new HMM_DMIRROR_READ_UNLOCKED ioctl that calls hmm_range_fault_unlocked_timeout() with a timeout of 0, exercising the unbounded retry mode while allowing the mmap lock to be dropped during fault handling. Add a userfaultfd_read selftest that registers an anonymous mapping with UFFDIO_REGISTER_MODE_MISSING, services the faults from a handler thread with UFFDIO_COPY, and verifies that HMM can read back the data supplied by the handler. This exercises the path where handle_mm_fault() drops mmap_lock and hmm_range_fault_unlocked_timeout() restarts the walk internally. Assisted-by: GitHub-Copilot:claude-opus-4.6 Link: https://lore.kernel.org/20260723-hmm-v10-v11-3-c55b003a4b61@gmail.com Signed-off-by: Stanislav Kinsburskii Cc: Danilo Krummrich Cc: Dave Airlie Cc: David Hildenbrand Cc: Dexuan Cui Cc: Haiyang Zhang Cc: Jason Gunthorpe Cc: Jonathan Corbet Cc: K. Y. Srinivasan Cc: Leon Romanovsky Cc: Liam R. Howlett Cc: Lizhi Hou Cc: Long Li Cc: Lorenzo Stoakes Cc: Lyude Cc: Maarten Lankhorst Cc: Maxime Ripard Cc: Michal Hocko Cc: Mike Rapoport Cc: Oded Gabbay Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Thomas Zimemrmann Cc: Vlastimil Babka Cc: Wei Liu Signed-off-by: Andrew Morton --- lib/test_hmm.c | 107 +++++++++++++++++- lib/test_hmm_uapi.h | 1 + tools/testing/selftests/mm/hmm-tests.c | 150 +++++++++++++++++++++++++ 3 files changed, 257 insertions(+), 1 deletion(-) diff --git a/lib/test_hmm.c b/lib/test_hmm.c index 45c0cb992218..6205fb313bd0 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -389,6 +389,67 @@ static int dmirror_range_fault(struct dmirror *dmirror, return ret; } +static int dmirror_range_fault_unlocked(struct dmirror *dmirror, + struct hmm_range *range, + unsigned long timeout) +{ + int ret; + + while (true) { + ret = hmm_range_fault_unlocked_timeout(range, timeout); + if (ret) + goto out; + + mutex_lock(&dmirror->mutex); + if (mmu_interval_read_retry(range->notifier, + range->notifier_seq)) { + mutex_unlock(&dmirror->mutex); + continue; + } + break; + } + + ret = dmirror_do_fault(dmirror, range); + + mutex_unlock(&dmirror->mutex); +out: + return ret; +} + +static int dmirror_fault_unlocked(struct dmirror *dmirror, + unsigned long start, + unsigned long end, bool write, + unsigned long timeout) +{ + struct mm_struct *mm = dmirror->notifier.mm; + unsigned long addr; + unsigned long pfns[32]; + struct hmm_range range = { + .notifier = &dmirror->notifier, + .hmm_pfns = pfns, + .pfn_flags_mask = 0, + .default_flags = + HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), + .dev_private_owner = dmirror->mdevice, + }; + int ret = 0; + + if (!mmget_not_zero(mm)) + return -EFAULT; + + for (addr = start; addr < end; addr = range.end) { + range.start = addr; + range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); + + ret = dmirror_range_fault_unlocked(dmirror, &range, timeout); + if (ret) + break; + } + + mmput(mm); + return ret; +} + static int dmirror_fault(struct dmirror *dmirror, unsigned long start, unsigned long end, bool write) { @@ -488,6 +549,48 @@ static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) return ret; } +static int dmirror_read_unlocked(struct dmirror *dmirror, + struct hmm_dmirror_cmd *cmd, + unsigned long timeout) +{ + struct dmirror_bounce bounce; + unsigned long start, end; + unsigned long size = cmd->npages << PAGE_SHIFT; + int ret; + + start = cmd->addr; + end = start + size; + if (end < start) + return -EINVAL; + + ret = dmirror_bounce_init(&bounce, start, size); + if (ret) + return ret; + + while (1) { + mutex_lock(&dmirror->mutex); + ret = dmirror_do_read(dmirror, start, end, &bounce); + mutex_unlock(&dmirror->mutex); + if (ret != -ENOENT) + break; + + start = cmd->addr + (bounce.cpages << PAGE_SHIFT); + ret = dmirror_fault_unlocked(dmirror, start, end, false, timeout); + if (ret) + break; + cmd->faults++; + } + + if (ret == 0) { + if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, + bounce.size)) + ret = -EFAULT; + } + cmd->cpages = bounce.cpages; + dmirror_bounce_fini(&bounce); + return ret; +} + static int dmirror_do_write(struct dmirror *dmirror, unsigned long start, unsigned long end, struct dmirror_bounce *bounce) { @@ -1572,7 +1675,9 @@ static long dmirror_fops_unlocked_ioctl(struct file *filp, dmirror->flags = cmd.npages; ret = 0; break; - + case HMM_DMIRROR_READ_UNLOCKED: + ret = dmirror_read_unlocked(dmirror, &cmd, 0); + break; default: return -EINVAL; } diff --git a/lib/test_hmm_uapi.h b/lib/test_hmm_uapi.h index f94c6d457338..ea9b0ec404fb 100644 --- a/lib/test_hmm_uapi.h +++ b/lib/test_hmm_uapi.h @@ -38,6 +38,7 @@ struct hmm_dmirror_cmd { #define HMM_DMIRROR_CHECK_EXCLUSIVE _IOWR('H', 0x06, struct hmm_dmirror_cmd) #define HMM_DMIRROR_RELEASE _IOWR('H', 0x07, struct hmm_dmirror_cmd) #define HMM_DMIRROR_FLAGS _IOWR('H', 0x08, struct hmm_dmirror_cmd) +#define HMM_DMIRROR_READ_UNLOCKED _IOWR('H', 0x09, struct hmm_dmirror_cmd) #define HMM_DMIRROR_FLAG_FAIL_ALLOC (1ULL << 0) diff --git a/tools/testing/selftests/mm/hmm-tests.c b/tools/testing/selftests/mm/hmm-tests.c index 6fccbdab02ee..5acb728666f8 100644 --- a/tools/testing/selftests/mm/hmm-tests.c +++ b/tools/testing/selftests/mm/hmm-tests.c @@ -29,6 +29,10 @@ #include #include #include +#include +#include +#include +#include /* * This is a private UAPI to the kernel test module so it isn't exported @@ -2952,4 +2956,150 @@ TEST_F_TIMEOUT(hmm, benchmark_thp_migration, 120) &thp_results, ®ular_results); } } +/* + * Test that HMM can fault in pages backed by userfaultfd using the + * hmm_range_fault_unlocked_timeout() path with no timeout. This exercises + * the lock-drop retry logic in the HMM framework. + */ +struct uffd_thread_args { + int uffd; + int stop_fd; + void *page_buffer; + unsigned long page_size; +}; + +static void *uffd_handler_thread(void *arg) +{ + struct uffd_thread_args *args = arg; + struct uffd_msg msg; + struct uffdio_copy copy; + struct pollfd pollfd[2]; + int ret; + + pollfd[0].fd = args->uffd; + pollfd[0].events = POLLIN; + pollfd[1].fd = args->stop_fd; + pollfd[1].events = POLLIN; + + while (1) { + ret = poll(pollfd, 2, -1); + if (ret <= 0) + break; + if (pollfd[1].revents) + break; + if (!(pollfd[0].revents & POLLIN)) + break; + + ret = read(args->uffd, &msg, sizeof(msg)); + if (ret != sizeof(msg)) + break; + + if (msg.event != UFFD_EVENT_PAGEFAULT) + break; + + /* Fill the page with a known pattern */ + memset(args->page_buffer, 0xAB, args->page_size); + + copy.dst = msg.arg.pagefault.address & ~(args->page_size - 1); + copy.src = (unsigned long)args->page_buffer; + copy.len = args->page_size; + copy.mode = 0; + copy.copy = 0; + + ret = ioctl(args->uffd, UFFDIO_COPY, ©); + if (ret < 0) + break; + } + + return NULL; +} + +TEST_F(hmm, userfaultfd_read) +{ + struct hmm_buffer *buffer; + struct uffd_thread_args uffd_args; + unsigned long npages; + unsigned long size; + unsigned long i; + unsigned char *ptr; + pthread_t thread; + int uffd; + int stop_fd; + int ret; + struct uffdio_api api; + struct uffdio_register reg; + uint64_t stop = 1; + ssize_t nwrite; + + npages = 4; + size = npages << self->page_shift; + + /* Create userfaultfd */ + uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); + if (uffd < 0) + SKIP(return, "userfaultfd not available"); + + api.api = UFFD_API; + api.features = 0; + ret = ioctl(uffd, UFFDIO_API, &api); + ASSERT_EQ(ret, 0); + + buffer = malloc(sizeof(*buffer)); + ASSERT_NE(buffer, NULL); + + buffer->fd = -1; + buffer->size = size; + buffer->mirror = malloc(size); + ASSERT_NE(buffer->mirror, NULL); + + /* Create anonymous mapping */ + buffer->ptr = mmap(NULL, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + ASSERT_NE(buffer->ptr, MAP_FAILED); + + /* Register the region with userfaultfd */ + reg.range.start = (unsigned long)buffer->ptr; + reg.range.len = size; + reg.mode = UFFDIO_REGISTER_MODE_MISSING; + ret = ioctl(uffd, UFFDIO_REGISTER, ®); + ASSERT_EQ(ret, 0); + + /* Set up the handler thread */ + uffd_args.uffd = uffd; + stop_fd = eventfd(0, EFD_CLOEXEC); + ASSERT_GE(stop_fd, 0); + uffd_args.stop_fd = stop_fd; + uffd_args.page_buffer = malloc(self->page_size); + ASSERT_NE(uffd_args.page_buffer, NULL); + uffd_args.page_size = self->page_size; + + ret = pthread_create(&thread, NULL, uffd_handler_thread, &uffd_args); + ASSERT_EQ(ret, 0); + + /* + * Use the unlocked read path which allows the mmap lock to be + * dropped during the fault, enabling userfaultfd resolution. + */ + ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ_UNLOCKED, + buffer, npages); + ASSERT_EQ(ret, 0); + ASSERT_EQ(buffer->cpages, npages); + + /* Verify the device read the data filled by the uffd handler */ + ptr = buffer->mirror; + for (i = 0; i < size; ++i) + ASSERT_EQ(ptr[i], (unsigned char)0xAB); + + nwrite = write(stop_fd, &stop, sizeof(stop)); + ASSERT_EQ(nwrite, sizeof(stop)); + pthread_join(thread, NULL); + close(stop_fd); + free(uffd_args.page_buffer); + close(uffd); + hmm_buffer_free(buffer); +} + + TEST_HARNESS_MAIN From 39899823ad38b77160459970c46efec78bd26520 Mon Sep 17 00:00:00 2001 From: Stanislav Kinsburskii Date: Thu, 23 Jul 2026 10:36:36 -0700 Subject: [PATCH 326/501] mshv: use hmm_range_fault_unlocked_timeout() for region faults MSHV currently faults movable memory regions by taking mmap_read_lock() around hmm_range_fault(). That prevents the fault path from handling VMAs whose fault handlers need to drop mmap_lock, such as userfaultfd-backed mappings. Use hmm_range_fault_unlocked_timeout() instead. Passing a timeout of 0 preserves MSHV's existing unbounded retry behavior while letting the HMM helper own mmap_lock acquisition and refresh range->notifier_seq internally before walking the range. After the fault succeeds, MSHV still takes mreg_mutex and checks mmu_interval_read_retry() before installing the pages into the region, so the existing invalidation synchronization is preserved. Fold the small fault-and-lock helper into mshv_region_range_fault(), since the remaining retry path is just the standard "fault, take the driver lock, check the interval notifier sequence" pattern. Link: https://lore.kernel.org/20260723-hmm-v10-v11-4-c55b003a4b61@gmail.com Signed-off-by: Stanislav Kinsburskii Reviewed-by: Jason Gunthorpe Cc: Danilo Krummrich Cc: Dave Airlie Cc: David Hildenbrand Cc: Dexuan Cui Cc: Haiyang Zhang Cc: Jonathan Corbet Cc: K. Y. Srinivasan Cc: Leon Romanovsky Cc: Liam R. Howlett Cc: Lizhi Hou Cc: Long Li Cc: Lorenzo Stoakes Cc: Lyude Cc: Maarten Lankhorst Cc: Maxime Ripard Cc: Michal Hocko Cc: Mike Rapoport Cc: Oded Gabbay Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Thomas Zimemrmann Cc: Vlastimil Babka Cc: Wei Liu Signed-off-by: Andrew Morton --- drivers/hv/mshv_regions.c | 54 ++++++++------------------------------- 1 file changed, 10 insertions(+), 44 deletions(-) diff --git a/drivers/hv/mshv_regions.c b/drivers/hv/mshv_regions.c index 6d65e5b42152..dddaade31b5d 100644 --- a/drivers/hv/mshv_regions.c +++ b/drivers/hv/mshv_regions.c @@ -381,46 +381,6 @@ int mshv_region_get(struct mshv_mem_region *region) return kref_get_unless_zero(®ion->mreg_refcount); } -/** - * mshv_region_hmm_fault_and_lock - Handle HMM faults and lock the memory region - * @region: Pointer to the memory region structure - * @range: Pointer to the HMM range structure - * - * This function performs the following steps: - * 1. Reads the notifier sequence for the HMM range. - * 2. Acquires a read lock on the memory map. - * 3. Handles HMM faults for the specified range. - * 4. Releases the read lock on the memory map. - * 5. If successful, locks the memory region mutex. - * 6. Verifies if the notifier sequence has changed during the operation. - * If it has, releases the mutex and returns -EBUSY to match with - * hmm_range_fault() return code for repeating. - * - * Return: 0 on success, a negative error code otherwise. - */ -static int mshv_region_hmm_fault_and_lock(struct mshv_mem_region *region, - struct hmm_range *range) -{ - int ret; - - range->notifier_seq = mmu_interval_read_begin(range->notifier); - mmap_read_lock(region->mreg_mni.mm); - ret = hmm_range_fault(range); - mmap_read_unlock(region->mreg_mni.mm); - if (ret) - return ret; - - mutex_lock(®ion->mreg_mutex); - - if (mmu_interval_read_retry(range->notifier, range->notifier_seq)) { - mutex_unlock(®ion->mreg_mutex); - cond_resched(); - return -EBUSY; - } - - return 0; -} - /** * mshv_region_range_fault - Handle memory range faults for a given region. * @region: Pointer to the memory region structure. @@ -452,13 +412,19 @@ static int mshv_region_range_fault(struct mshv_mem_region *region, range.start = region->start_uaddr + page_offset * HV_HYP_PAGE_SIZE; range.end = range.start + page_count * HV_HYP_PAGE_SIZE; - do { - ret = mshv_region_hmm_fault_and_lock(region, &range); - } while (ret == -EBUSY); - +again: + ret = hmm_range_fault_unlocked_timeout(&range, 0); if (ret) goto out; + mutex_lock(®ion->mreg_mutex); + + if (mmu_interval_read_retry(range.notifier, range.notifier_seq)) { + mutex_unlock(®ion->mreg_mutex); + cond_resched(); + goto again; + } + for (i = 0; i < page_count; i++) region->mreg_pages[page_offset + i] = hmm_pfn_to_page(pfns[i]); From 74682a77744156ed6dcfbfc5da75888ffda1a3c7 Mon Sep 17 00:00:00 2001 From: Stanislav Kinsburskii Date: Thu, 23 Jul 2026 10:36:37 -0700 Subject: [PATCH 327/501] drm/nouveau: use hmm_range_fault_unlocked_timeout() for SVM faults nouveau_range_fault() takes mmap_read_lock() only to call hmm_range_fault(). It also keeps a single HMM_RANGE_DEFAULT_TIMEOUT deadline across both HMM -EBUSY retries and post-fault mmu_interval_read_retry() retries. Use hmm_range_fault_unlocked_timeout() instead. The HMM helper now owns the mmap lock and refreshes range->notifier_seq for its internal retries. Nouveau keeps its existing absolute deadline in the outer loop and passes the remaining jiffies to the helper for each fault attempt, so retries caused by mmu_interval_read_retry() do not reset the overall retry budget. Nouveau still validates the interval notifier sequence while holding svmm->mutex before programming the GPU mapping. Link: https://lore.kernel.org/20260723-hmm-v10-v11-5-c55b003a4b61@gmail.com Signed-off-by: Stanislav Kinsburskii Reviewed-by: Jason Gunthorpe Cc: Danilo Krummrich Cc: Dave Airlie Cc: David Hildenbrand Cc: Dexuan Cui Cc: Haiyang Zhang Cc: Jonathan Corbet Cc: K. Y. Srinivasan Cc: Leon Romanovsky Cc: Liam R. Howlett Cc: Lizhi Hou Cc: Long Li Cc: Lorenzo Stoakes Cc: Lyude Cc: Maarten Lankhorst Cc: Maxime Ripard Cc: Michal Hocko Cc: Mike Rapoport Cc: Oded Gabbay Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Thomas Zimemrmann Cc: Vlastimil Babka Cc: Wei Liu Signed-off-by: Andrew Morton --- drivers/gpu/drm/nouveau/nouveau_svm.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_svm.c b/drivers/gpu/drm/nouveau/nouveau_svm.c index dcc92131488e..58735446d783 100644 --- a/drivers/gpu/drm/nouveau/nouveau_svm.c +++ b/drivers/gpu/drm/nouveau/nouveau_svm.c @@ -678,20 +678,22 @@ static int nouveau_range_fault(struct nouveau_svmm *svmm, range.end = notifier->notifier.interval_tree.last + 1; while (true) { - if (time_after(jiffies, timeout)) { + long remaining = timeout - jiffies; + + /* + * The HMM timeout only bounds retries while HMM is walking and + * faulting the range. This fault is handled by a kernel worker, + * so fatal signals from the faulting process cannot stop an + * endless stream of invalidations here. + */ + if (time_after_eq(jiffies, timeout)) { ret = -EBUSY; goto out; } - range.notifier_seq = mmu_interval_read_begin(range.notifier); - mmap_read_lock(mm); - ret = hmm_range_fault(&range); - mmap_read_unlock(mm); - if (ret) { - if (ret == -EBUSY) - continue; + ret = hmm_range_fault_unlocked_timeout(&range, remaining); + if (ret) goto out; - } mutex_lock(&svmm->mutex); if (mmu_interval_read_retry(range.notifier, From e269fb946bf34f7c2bbb76e4fb6257de3cfe3cda Mon Sep 17 00:00:00 2001 From: Stanislav Kinsburskii Date: Thu, 23 Jul 2026 10:36:38 -0700 Subject: [PATCH 328/501] RDMA/umem: use hmm_range_fault_unlocked_timeout() for ODP faults ib_umem_odp_map_dma_and_lock() takes mmap_read_lock() only around hmm_range_fault(), then retries -EBUSY until HMM_RANGE_DEFAULT_TIMEOUT expires. Use hmm_range_fault_unlocked_timeout() instead. The HMM helper now owns the mmap lock and refreshes range->notifier_seq for its internal retries. ODP keeps using HMM_RANGE_DEFAULT_TIMEOUT for each HMM fault attempt, while interval invalidation retries continue to be handled by the existing outer loop. ODP still validates the interval notifier sequence while holding umem_mutex before DMA mapping pages. Link: https://lore.kernel.org/20260723-hmm-v10-v11-6-c55b003a4b61@gmail.com Signed-off-by: Stanislav Kinsburskii Reviewed-by: Jason Gunthorpe Cc: Danilo Krummrich Cc: Dave Airlie Cc: David Hildenbrand Cc: Dexuan Cui Cc: Haiyang Zhang Cc: Jonathan Corbet Cc: K. Y. Srinivasan Cc: Leon Romanovsky Cc: Liam R. Howlett Cc: Lizhi Hou Cc: Long Li Cc: Lorenzo Stoakes Cc: Lyude Cc: Maarten Lankhorst Cc: Maxime Ripard Cc: Michal Hocko Cc: Mike Rapoport Cc: Oded Gabbay Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Thomas Zimemrmann Cc: Vlastimil Babka Cc: Wei Liu Signed-off-by: Andrew Morton --- drivers/infiniband/core/umem_odp.c | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c index 404fa1cc3254..9cc21cd762d9 100644 --- a/drivers/infiniband/core/umem_odp.c +++ b/drivers/infiniband/core/umem_odp.c @@ -329,7 +329,7 @@ int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt, struct mm_struct *owning_mm = umem_odp->umem.owning_mm; int pfn_index, dma_index, ret = 0, start_idx; unsigned int page_shift, hmm_order, pfn_start_idx; - unsigned long num_pfns, current_seq; + unsigned long num_pfns; struct hmm_range range = {}; unsigned long timeout; @@ -363,26 +363,18 @@ int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt, } range.hmm_pfns = &(umem_odp->map.pfn_list[pfn_start_idx]); - timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); + timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); retry: - current_seq = range.notifier_seq = - mmu_interval_read_begin(&umem_odp->notifier); - - mmap_read_lock(owning_mm); - ret = hmm_range_fault(&range); - mmap_read_unlock(owning_mm); - if (unlikely(ret)) { - if (ret == -EBUSY && !time_after(jiffies, timeout)) - goto retry; + ret = hmm_range_fault_unlocked_timeout(&range, timeout); + if (unlikely(ret)) goto out_put_mm; - } start_idx = (range.start - ib_umem_start(umem_odp)) >> page_shift; dma_index = start_idx; mutex_lock(&umem_odp->umem_mutex); - if (mmu_interval_read_retry(&umem_odp->notifier, current_seq)) { + if (mmu_interval_read_retry(&umem_odp->notifier, range.notifier_seq)) { mutex_unlock(&umem_odp->umem_mutex); goto retry; } From a1b25d7cafdc9028eff03a9ef1d4a14f0f72c93b Mon Sep 17 00:00:00 2001 From: Stanislav Kinsburskii Date: Thu, 23 Jul 2026 10:36:39 -0700 Subject: [PATCH 329/501] accel/amdxdna: use hmm_range_fault_unlocked_timeout() for range population aie2_populate_range() takes mmap_read_lock() only around hmm_range_fault(). It also open-codes the mmu interval sequence setup before each HMM walk and retries -EBUSY until HMM_RANGE_DEFAULT_TIMEOUT expires. Use hmm_range_fault_unlocked_timeout() instead. The HMM helper now owns the mmap lock and refreshes mapp->range.notifier_seq for its internal retries, so the driver only needs to call the helper and then validate the sequence before marking the mapping populated. Pass HMM_RANGE_DEFAULT_TIMEOUT as the helper retry budget for each HMM population attempt. This scopes the timeout to repeated HMM notifier retries while preserving the existing outer loop that moves between invalid mappings and restarts when the interval is invalidated before the driver updates its mapping state. Keep returning -ETIME when the HMM retry budget expires, matching the driver's existing timeout error convention. Link: https://lore.kernel.org/20260723-hmm-v10-v11-7-c55b003a4b61@gmail.com Signed-off-by: Stanislav Kinsburskii Reviewed-by: Jason Gunthorpe Cc: Danilo Krummrich Cc: Dave Airlie Cc: David Hildenbrand Cc: Dexuan Cui Cc: Haiyang Zhang Cc: Jonathan Corbet Cc: K. Y. Srinivasan Cc: Leon Romanovsky Cc: Liam R. Howlett Cc: Lizhi Hou Cc: Long Li Cc: Lorenzo Stoakes Cc: Lyude Cc: Maarten Lankhorst Cc: Maxime Ripard Cc: Michal Hocko Cc: Mike Rapoport Cc: Oded Gabbay Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Thomas Zimemrmann Cc: Vlastimil Babka Cc: Wei Liu Signed-off-by: Andrew Morton --- drivers/accel/amdxdna/aie2_ctx.c | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c index 4b3a62aa8798..0a0ff9a10946 100644 --- a/drivers/accel/amdxdna/aie2_ctx.c +++ b/drivers/accel/amdxdna/aie2_ctx.c @@ -1047,7 +1047,7 @@ static int aie2_populate_range(struct amdxdna_gem_obj *abo) bool found; int ret; - timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); + timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); again: found = false; down_write(&xdna->notifier_lock); @@ -1072,24 +1072,9 @@ static int aie2_populate_range(struct amdxdna_gem_obj *abo) return -EFAULT; } - mapp->range.notifier_seq = mmu_interval_read_begin(&mapp->notifier); - mmap_read_lock(mm); - ret = hmm_range_fault(&mapp->range); - mmap_read_unlock(mm); - if (ret) { - if (time_after(jiffies, timeout)) { - ret = -ETIME; - goto put_mm; - } - - if (ret == -EBUSY) { - amdxdna_umap_put(mapp); - mmput(mm); - goto again; - } - + ret = hmm_range_fault_unlocked_timeout(&mapp->range, timeout); + if (ret) goto put_mm; - } down_write(&xdna->notifier_lock); if (mmu_interval_read_retry(&mapp->notifier, mapp->range.notifier_seq)) { @@ -1107,7 +1092,7 @@ static int aie2_populate_range(struct amdxdna_gem_obj *abo) put_mm: amdxdna_umap_put(mapp); mmput(mm); - return ret; + return ret == -EBUSY ? -ETIME : ret; } int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, u64 *seq) From aa8543e66268c38cd8a1974275dd8d8914fa7e38 Mon Sep 17 00:00:00 2001 From: Stanislav Kinsburskii Date: Thu, 23 Jul 2026 10:36:40 -0700 Subject: [PATCH 330/501] drm/gpusvm: use hmm_range_fault_unlocked_timeout() for range faults Several GPU SVM paths take mmap_read_lock() only to call hmm_range_fault() and open-code mmu interval sequence setup before each HMM walk. They also retry -EBUSY until HMM_RANGE_DEFAULT_TIMEOUT expires. Use hmm_range_fault_unlocked_timeout() for those faults. The HMM helper now owns mmap_lock acquisition and refreshes range->notifier_seq for its internal retries, while GPU SVM keeps its existing driver-lock validation with mmu_interval_read_retry() after a successful fault. drm_gpusvm_scan_mm() and drm_gpusvm_range_evict() pass HMM_RANGE_DEFAULT_TIMEOUT as the helper retry budget for each HMM fault attempt. drm_gpusvm_get_pages() keeps its existing absolute outer deadline because it can be reached from GPU page-fault workers, where fatal signals from the faulting process cannot stop an endless invalidation retry loop. It passes the remaining time from that deadline to HMM for each fault attempt. Leave drm_gpusvm_check_pages() on hmm_range_fault() because that path is called with the mmap lock already held by its caller. Link: https://lore.kernel.org/20260723-hmm-v10-v11-8-c55b003a4b61@gmail.com Signed-off-by: Stanislav Kinsburskii Reviewed-by: Jason Gunthorpe Cc: Danilo Krummrich Cc: Dave Airlie Cc: David Hildenbrand Cc: Dexuan Cui Cc: Haiyang Zhang Cc: Jonathan Corbet Cc: K. Y. Srinivasan Cc: Leon Romanovsky Cc: Liam R. Howlett Cc: Lizhi Hou Cc: Long Li Cc: Lorenzo Stoakes Cc: Lyude Cc: Maarten Lankhorst Cc: Maxime Ripard Cc: Michal Hocko Cc: Mike Rapoport Cc: Oded Gabbay Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Thomas Zimemrmann Cc: Vlastimil Babka Cc: Wei Liu Signed-off-by: Andrew Morton --- drivers/gpu/drm/drm_gpusvm.c | 60 ++++++------------------------------ 1 file changed, 10 insertions(+), 50 deletions(-) diff --git a/drivers/gpu/drm/drm_gpusvm.c b/drivers/gpu/drm/drm_gpusvm.c index 18ddef94da47..e5e105466f53 100644 --- a/drivers/gpu/drm/drm_gpusvm.c +++ b/drivers/gpu/drm/drm_gpusvm.c @@ -773,8 +773,7 @@ enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range, .end = end, .dev_private_owner = dev_private_owner, }; - unsigned long timeout = - jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); + unsigned long timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); enum drm_gpusvm_scan_result state = DRM_GPUSVM_SCAN_UNPOPULATED, new_state; unsigned long *pfns; unsigned long npages = npages_in_range(start, end); @@ -788,22 +787,7 @@ enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range, hmm_range.hmm_pfns = pfns; retry: - hmm_range.notifier_seq = mmu_interval_read_begin(notifier); - mmap_read_lock(range->gpusvm->mm); - - while (true) { - err = hmm_range_fault(&hmm_range); - if (err == -EBUSY) { - if (time_after(jiffies, timeout)) - break; - - hmm_range.notifier_seq = - mmu_interval_read_begin(notifier); - continue; - } - break; - } - mmap_read_unlock(range->gpusvm->mm); + err = hmm_range_fault_unlocked_timeout(&hmm_range, timeout); if (err) goto err_free; @@ -1423,6 +1407,7 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, void *zdd; unsigned long timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); + unsigned long remaining; unsigned long i, j; unsigned long npages = npages_in_range(pages_start, pages_end); unsigned long num_dma_mapped; @@ -1437,7 +1422,9 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, struct dma_iova_state *state = &svm_pages->state; retry: - if (time_after(jiffies, timeout)) + remaining = timeout - jiffies; + + if (time_after_eq(jiffies, timeout)) return -EBUSY; hmm_range.notifier_seq = mmu_interval_read_begin(notifier); @@ -1454,21 +1441,7 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, } hmm_range.hmm_pfns = pfns; - while (true) { - mmap_read_lock(mm); - err = hmm_range_fault(&hmm_range); - mmap_read_unlock(mm); - - if (err == -EBUSY) { - if (time_after(jiffies, timeout)) - break; - - hmm_range.notifier_seq = - mmu_interval_read_begin(notifier); - continue; - } - break; - } + err = hmm_range_fault_unlocked_timeout(&hmm_range, remaining); mmput(mm); if (err) goto err_free; @@ -1741,8 +1714,7 @@ int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm, .end = drm_gpusvm_range_end(range), .dev_private_owner = NULL, }; - unsigned long timeout = - jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); + unsigned long timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); unsigned long *pfns; unsigned long npages = npages_in_range(drm_gpusvm_range_start(range), drm_gpusvm_range_end(range)); @@ -1759,24 +1731,12 @@ int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm, } hmm_range.hmm_pfns = pfns; - while (!time_after(jiffies, timeout)) { - hmm_range.notifier_seq = mmu_interval_read_begin(notifier); - if (time_after(jiffies, timeout)) { - err = -ETIME; - break; - } - - mmap_read_lock(mm); - err = hmm_range_fault(&hmm_range); - mmap_read_unlock(mm); - if (err != -EBUSY) - break; - } + err = hmm_range_fault_unlocked_timeout(&hmm_range, timeout); kvfree(pfns); mmput(mm); - return err; + return err == -EBUSY ? -ETIME : err; } EXPORT_SYMBOL_GPL(drm_gpusvm_range_evict); From 995f0eed5e1e6c657b06a700c6037544ed9f742c Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:16:42 +0100 Subject: [PATCH 331/501] mm: move vma_start_pgoff() into mm.h and clean up Patch series "mm: make VMA page offset handling more consistent", v2. This series performs a series of cleanups and improvements around how the vma->vm_pgoff field is used. Folios belonging to file-backed mappings are simply indexed by the page offset within the file they map (excepting pfnmap and driver mappings which sometimes do things different). However, anonymous folios belonging to pure anonymous mappings are indexed by their "virtual" page offset, which is equal to addr >> PAGE_SHIFT at the time at which the VMA was first faulted in. The page offset of a VMA is stored in vma->vm_pgoff and indicates the page offset of the start of the VMA range, whether it be file-backed or anonymous. The work here both cleans up how we reference this field, as well as laying the foundations for a future series which addresses the inconsistency of CoW'd folios in MAP_PRIVATE-file backed mappings, which are indexed as if they were file-backed but behave as if they were anonymous. This future series will make it such that all anonymous folios are indexed by virtual page offset whether belonging to VMAs who satisfy vma_is_anonymous() or MAP_PRIVATE-mapped file-backed mappings. This series: * Exposes vma_start_pgoff() and updates the kernel to use it consistently. * Adds and uses the useful vma_end_pgoff() helper. * Parameterises the file-backed mapping helpers vma_interval_tree_*() by address_space rather than rb_root_cached. * Renames: the misleadingly-named vma_interval_tree_*() helpers to mapping_rmap_tree_*() to be consistent with the renamed anon_rmap_tree_*(). * Parameterises anon_rmap_tree_*() by anon_vma. * Moves mm/interval_tree.c to the rmap section. * Adds vmg_*() helpers for page offset. * Clarifies the confusing vmg_adjust_set_range() function. * Introduces linear_page_delta() to provide relative pgoff within a VMA. * Replaces open-coded versions of linear_page_delta() and linear_page_index() with invocations of these functions. * Introduces and uses vma_assert_can_modify() to account for whether a VMA can be modified (detached or write locked). * Adds and uses vma_[add,sub]_pgoff() to adjust VMA page offset. * Moves __install_special_mapping() to vma.c. * Makes vma_set_range() static and internal to vma.c. * Introduces and makes use of vma_set_pgoff(). * Fixes incorrect vma.h header inclusion. * Defaults VMA userland tests to 64-bit vma flags size. * Updates VMA userland tests to give better output on failure. * Various smaller cleanups. * Updated the vma_set_pgoff() assert to account for MAP_PRIVATE /dev/zero correctly. This patch (of 33): vma_last_pgoff() already lives there, so it's a bit odd to keep vma_start_pgoff() in mm/interval_tree.c. Move them together. These each return unsigned long, which pgoff_t is typedef'd to. Make this consistent and have these functions return pgoff_t instead. Additionally, express vma_last_pgoff() in terms of vma_start_pgoff(), since we wrap the vma->vm_pgoff access, we may as well use it here. Also while we're here, const-ify the VMA and cleanup a bit. Also update the VMA userland tests to reflect the change. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-0-2a5aa403d977@kernel.org Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-1-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Gregory Price Acked-by: David Hildenbrand (Arm) Reviewed-by: Pedro Falcato Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Liam R. Howlett (Oracle) Reviewed-by: Zi Yan Cc: Ackerley Tng Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Signed-off-by: Andrew Morton --- include/linux/mm.h | 9 +++++++-- mm/interval_tree.c | 5 ----- tools/testing/vma/include/dup.h | 5 +++++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index e1e1154dafd7..137a6269697f 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4311,9 +4311,14 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma) return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; } -static inline unsigned long vma_last_pgoff(struct vm_area_struct *vma) +static inline pgoff_t vma_start_pgoff(const struct vm_area_struct *vma) { - return vma->vm_pgoff + vma_pages(vma) - 1; + return vma->vm_pgoff; +} + +static inline pgoff_t vma_last_pgoff(const struct vm_area_struct *vma) +{ + return vma_start_pgoff(vma) + vma_pages(vma) - 1; } static inline unsigned long vma_desc_size(const struct vm_area_desc *desc) diff --git a/mm/interval_tree.c b/mm/interval_tree.c index 32bcfbfcf15f..344d1f5946c7 100644 --- a/mm/interval_tree.c +++ b/mm/interval_tree.c @@ -10,11 +10,6 @@ #include #include -static inline unsigned long vma_start_pgoff(struct vm_area_struct *v) -{ - return v->vm_pgoff; -} - INTERVAL_TREE_DEFINE(struct vm_area_struct, shared.rb, unsigned long, shared.rb_subtree_last, vma_start_pgoff, vma_last_pgoff, /* empty */, vma_interval_tree) diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index bf26b3f48d3a..668650067c7c 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -1301,6 +1301,11 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma) return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; } +static inline pgoff_t vma_start_pgoff(const struct vm_area_struct *vma) +{ + return vma->vm_pgoff; +} + static inline int vfs_mmap_prepare(struct file *file, struct vm_area_desc *desc) { return file->f_op->mmap_prepare(desc); From f9711438dfc19c9b7900ab7b9fa27b4d591080ed Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:16:43 +0100 Subject: [PATCH 332/501] mm: add kdoc comments for vma_start/last_pgoff() Describe what vma_start_pgoff() and vma_last_pgoff() actually provide in detail. This is in order that we can differentiate this between functions that will be added in a subsequent patch which provide a different page offset. We go to lengths to describe the edge cases that can be run into here. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-2-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Pedro Falcato Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Liam R. Howlett (Oracle) Reviewed-by: Zi Yan Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Signed-off-by: Andrew Morton --- include/linux/mm.h | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/include/linux/mm.h b/include/linux/mm.h index 137a6269697f..8ecc26d6bb86 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4311,11 +4311,41 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma) return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; } +/** + * vma_start_pgoff() - Get the page offset of the start of @vma + * @vma: The VMA whose page offset is required. + * + * If the VMA is file-backed, this is the page offset into the file. + * + * If @vma is anonymous, this is the virtual page offset of the start of the + * VMA - if unfaulted, then vma->vm_start >> PAGE_SHIFT, if faulted then the + * virtual page offset at the time of first fault. + * + * If @vma is a MAP_PRIVATE file-backed mapping, then this returns the + * page offset within the file. + * + * Edge cases: nommu does not abide by these, MAP_PRIVATE-/dev/zero satisfies + * vma_is_anonymous() but has file-backed page offset, and MAP_PRIVATE-pfnmap + * regions have their page offset set to the first PFN in the range. + * + * Returns: The page offset of the start of @vma. + */ static inline pgoff_t vma_start_pgoff(const struct vm_area_struct *vma) { return vma->vm_pgoff; } +/** + * vma_last_pgoff() - Get the page offset of the last page in @vma + * @vma: The VMA whose last page offset is required. + * + * This returns the last page offset contained within @vma. + * + * See the description of vma_start_pgoff() for a description of VMA page + * offsets. + * + * Returns: The last page offset of @vma. + */ static inline pgoff_t vma_last_pgoff(const struct vm_area_struct *vma) { return vma_start_pgoff(vma) + vma_pages(vma) - 1; From 3542bc07f44fd80689c6a7bc0363f02707f194c9 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:16:44 +0100 Subject: [PATCH 333/501] tools/testing/vma: use vma_start_pgoff() in merge tests Now we have the vma_start_pgoff() helper, update the merge tests to make use of it for consistency. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-3-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Pedro Falcato Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Liam R. Howlett (Oracle) Reviewed-by: Zi Yan Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Signed-off-by: Andrew Morton --- tools/testing/vma/tests/merge.c | 38 ++++++++++++++++----------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tools/testing/vma/tests/merge.c b/tools/testing/vma/tests/merge.c index 03b6f9820e0a..f8666a755749 100644 --- a/tools/testing/vma/tests/merge.c +++ b/tools/testing/vma/tests/merge.c @@ -118,7 +118,7 @@ static bool test_simple_merge(void) ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x3000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); ASSERT_FLAGS_SAME_MASK(&vma->flags, vma_flags); detach_free_vma(vma); @@ -150,7 +150,7 @@ static bool test_simple_modify(void) ASSERT_EQ(vma->vm_start, 0x1000); ASSERT_EQ(vma->vm_end, 0x2000); - ASSERT_EQ(vma->vm_pgoff, 1); + ASSERT_EQ(vma_start_pgoff(vma), 1); /* * Now walk through the three split VMAs and make sure they are as @@ -162,7 +162,7 @@ static bool test_simple_modify(void) ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x1000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); detach_free_vma(vma); vma_iter_clear(&vmi); @@ -171,7 +171,7 @@ static bool test_simple_modify(void) ASSERT_EQ(vma->vm_start, 0x1000); ASSERT_EQ(vma->vm_end, 0x2000); - ASSERT_EQ(vma->vm_pgoff, 1); + ASSERT_EQ(vma_start_pgoff(vma), 1); detach_free_vma(vma); vma_iter_clear(&vmi); @@ -180,7 +180,7 @@ static bool test_simple_modify(void) ASSERT_EQ(vma->vm_start, 0x2000); ASSERT_EQ(vma->vm_end, 0x3000); - ASSERT_EQ(vma->vm_pgoff, 2); + ASSERT_EQ(vma_start_pgoff(vma), 2); detach_free_vma(vma); mtree_destroy(&mm.mm_mt); @@ -209,7 +209,7 @@ static bool test_simple_expand(void) ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x3000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); detach_free_vma(vma); mtree_destroy(&mm.mm_mt); @@ -231,7 +231,7 @@ static bool test_simple_shrink(void) ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x1000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); detach_free_vma(vma); mtree_destroy(&mm.mm_mt); @@ -324,7 +324,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_TRUE(merged); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x4000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 3); @@ -343,7 +343,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_TRUE(merged); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x5000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 3); @@ -364,7 +364,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_TRUE(merged); ASSERT_EQ(vma->vm_start, 0x6000); ASSERT_EQ(vma->vm_end, 0x9000); - ASSERT_EQ(vma->vm_pgoff, 6); + ASSERT_EQ(vma_start_pgoff(vma), 6); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 3); @@ -384,7 +384,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_TRUE(merged); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x9000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 2); @@ -404,7 +404,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_TRUE(merged); ASSERT_EQ(vma->vm_start, 0xa000); ASSERT_EQ(vma->vm_end, 0xc000); - ASSERT_EQ(vma->vm_pgoff, 0xa); + ASSERT_EQ(vma_start_pgoff(vma), 0xa); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 2); @@ -423,7 +423,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_TRUE(merged); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0xc000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 1); @@ -443,7 +443,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_NE(vma, NULL); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0xc000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); detach_free_vma(vma); @@ -805,7 +805,7 @@ static bool test_vma_merge_new_with_close(void) ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x5000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); ASSERT_EQ(vma->vm_ops, &vm_ops); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 2); @@ -865,7 +865,7 @@ static bool __test_merge_existing(bool prev_is_sticky, bool middle_is_sticky, bo ASSERT_EQ(vma_next->anon_vma, &dummy_anon_vma); ASSERT_EQ(vma->vm_start, 0x2000); ASSERT_EQ(vma->vm_end, 0x3000); - ASSERT_EQ(vma->vm_pgoff, 2); + ASSERT_EQ(vma_start_pgoff(vma), 2); ASSERT_TRUE(vma_write_started(vma)); ASSERT_TRUE(vma_write_started(vma_next)); ASSERT_EQ(mm.map_count, 2); @@ -931,7 +931,7 @@ static bool __test_merge_existing(bool prev_is_sticky, bool middle_is_sticky, bo ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma); ASSERT_EQ(vma->vm_start, 0x6000); ASSERT_EQ(vma->vm_end, 0x7000); - ASSERT_EQ(vma->vm_pgoff, 6); + ASSERT_EQ(vma_start_pgoff(vma), 6); ASSERT_TRUE(vma_write_started(vma_prev)); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 2); @@ -1416,7 +1416,7 @@ static bool test_merge_extend(void) ASSERT_EQ(vma_merge_extend(&vmi, vma, 0x2000), vma); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x4000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 1); @@ -1456,7 +1456,7 @@ static bool test_expand_only_mode(void) ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma->vm_start, 0x3000); ASSERT_EQ(vma->vm_end, 0x9000); - ASSERT_EQ(vma->vm_pgoff, 3); + ASSERT_EQ(vma_start_pgoff(vma), 3); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(vma_iter_addr(&vmi), 0x3000); vma_assert_attached(vma); From 1219731c49e513afc83ee5245c0aefa5c31a65d2 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:16:45 +0100 Subject: [PATCH 334/501] mm: introduce and use vma_end_pgoff() We already have vma_last_pgoff() which retrieves the last page offset within a VMA. However, code often wishes to span a page offset range, which requires the exclusive end of this range. So provide this in vma_end_pgoff() and update vma_last_pgoff() to use this function. Also update the VMA userland tests to reflect the change. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-4-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Gregory Price Reviewed-by: Pedro Falcato Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Signed-off-by: Andrew Morton --- include/linux/mm.h | 19 ++++++++++++++++++- tools/testing/vma/include/dup.h | 5 +++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 8ecc26d6bb86..3e45978812e2 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4335,6 +4335,23 @@ static inline pgoff_t vma_start_pgoff(const struct vm_area_struct *vma) return vma->vm_pgoff; } +/** + * vma_end_pgoff() - Get the page offset of the exclusive end of @vma + * @vma: The VMA whose end page offset is required. + * + * This returns the exclusive end page offset of @vma, which is useful for + * expressing page offset ranges. + * + * See the description of vma_start_pgoff() for a description of VMA page + * offsets. + * + * Returns: The exclusive end page offset of @vma. + */ +static inline pgoff_t vma_end_pgoff(const struct vm_area_struct *vma) +{ + return vma_start_pgoff(vma) + vma_pages(vma); +} + /** * vma_last_pgoff() - Get the page offset of the last page in @vma * @vma: The VMA whose last page offset is required. @@ -4348,7 +4365,7 @@ static inline pgoff_t vma_start_pgoff(const struct vm_area_struct *vma) */ static inline pgoff_t vma_last_pgoff(const struct vm_area_struct *vma) { - return vma_start_pgoff(vma) + vma_pages(vma) - 1; + return vma_end_pgoff(vma) - 1; } static inline unsigned long vma_desc_size(const struct vm_area_desc *desc) diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index 668650067c7c..535747d7fee4 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -1306,6 +1306,11 @@ static inline pgoff_t vma_start_pgoff(const struct vm_area_struct *vma) return vma->vm_pgoff; } +static inline pgoff_t vma_end_pgoff(const struct vm_area_struct *vma) +{ + return vma_start_pgoff(vma) + vma_pages(vma); +} + static inline int vfs_mmap_prepare(struct file *file, struct vm_area_desc *desc) { return file->f_op->mmap_prepare(desc); From dbde17041cbf26606436f844457b12f9a45b90d7 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:16:46 +0100 Subject: [PATCH 335/501] mm/rmap: update mm/interval_tree.c comments Update the file comment to clarify that both file-backed and anonymous interval trees are provided, referencing the relevant data types for clarity. Also add comments to indicate which parts of the file apply to each. While we're here, convert the VM_BUG_ON_VMA() to VM_WARN_ON_ONCE_VMA(). Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-5-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Pedro Falcato Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Signed-off-by: Andrew Morton --- mm/interval_tree.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mm/interval_tree.c b/mm/interval_tree.c index 344d1f5946c7..2d50bc6228c4 100644 --- a/mm/interval_tree.c +++ b/mm/interval_tree.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * mm/interval_tree.c - interval tree for mapping->i_mmap + * mm/interval_tree.c - interval tree for address_space->i_mmap and + * anon_vma->rb_root * * Copyright (C) 2012, Michel Lespinasse */ @@ -10,6 +11,8 @@ #include #include +/* File-backed interval tree (address_space->i_mmap) */ + INTERVAL_TREE_DEFINE(struct vm_area_struct, shared.rb, unsigned long, shared.rb_subtree_last, vma_start_pgoff, vma_last_pgoff, /* empty */, vma_interval_tree) @@ -23,7 +26,7 @@ void vma_interval_tree_insert_after(struct vm_area_struct *node, struct vm_area_struct *parent; unsigned long last = vma_last_pgoff(node); - VM_BUG_ON_VMA(vma_start_pgoff(node) != vma_start_pgoff(prev), node); + VM_WARN_ON_ONCE_VMA(vma_start_pgoff(node) != vma_start_pgoff(prev), node); if (!prev->shared.rb.rb_right) { parent = prev; @@ -48,6 +51,8 @@ void vma_interval_tree_insert_after(struct vm_area_struct *node, &vma_interval_tree_augment); } +/* Anonymous interval tree (anon_vma->rb_root) */ + static inline unsigned long avc_start_pgoff(struct anon_vma_chain *avc) { return vma_start_pgoff(avc->vma); From 7a67b96af06929b515947ebd4786c9cf1847e58f Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:16:47 +0100 Subject: [PATCH 336/501] mm/rmap: parameterise vma_interval_tree_*() by address_space The file-backed mapping interval tree functions vma_interval_tree_*() accept a raw rb_root_cached pointer to determine the tree in which they are operating. However, in each case, this is always associated with an address_space data type. So simply pass a pointer to that instead to simplify the code, and more clearly differentiate between these operations and those concerning anonymous mappings. While we're here, make the generated interval tree functions static as they do not need to be used externally (any previously existing external users have now been removed). We also rename VMA parameters from 'node' to 'vma' as calling this a node is simply confusing, update the input index types to pgoff_t since they reference page offsets and rename the parameters to pgoff_start and pgoff_last. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-6-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Pedro Falcato Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Signed-off-by: Andrew Morton --- arch/arm/mm/fault-armv.c | 2 +- arch/arm/mm/flush.c | 2 +- arch/nios2/mm/cacheflush.c | 2 +- arch/parisc/kernel/cache.c | 2 +- fs/dax.c | 2 +- fs/hugetlbfs/inode.c | 15 ++++---- include/linux/mm.h | 32 +++++++++-------- kernel/events/uprobes.c | 2 +- mm/hugetlb.c | 4 +-- mm/interval_tree.c | 58 +++++++++++++++++++++++-------- mm/khugepaged.c | 4 +-- mm/memory-failure.c | 7 ++-- mm/memory.c | 8 ++--- mm/mmap.c | 3 +- mm/nommu.c | 8 ++--- mm/pagewalk.c | 2 +- mm/rmap.c | 3 +- mm/vma.c | 14 ++++---- tools/testing/vma/include/stubs.h | 4 +-- 19 files changed, 99 insertions(+), 75 deletions(-) diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index 91e488767783..cd52cf7f8874 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -140,7 +140,7 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, * cache coherency. */ flush_dcache_mmap_lock(mapping); - vma_interval_tree_foreach(mpnt, &mapping->i_mmap, pgoff, pgoff) { + vma_interval_tree_foreach(mpnt, mapping, pgoff, pgoff) { /* * If we are using split PTE locks, then we need to take the pte * lock. Otherwise we are using shared mm->page_table_lock which diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index 4d7ef5cc36b6..8c593e9898ee 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -251,7 +251,7 @@ static void __flush_dcache_aliases(struct address_space *mapping, struct folio * pgoff_end = pgoff + folio_nr_pages(folio) - 1; flush_dcache_mmap_lock(mapping); - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff_end) { + vma_interval_tree_foreach(vma, mapping, pgoff, pgoff_end) { unsigned long start, offset, pfn; unsigned int nr; diff --git a/arch/nios2/mm/cacheflush.c b/arch/nios2/mm/cacheflush.c index 8321182eb927..42e3bf892316 100644 --- a/arch/nios2/mm/cacheflush.c +++ b/arch/nios2/mm/cacheflush.c @@ -82,7 +82,7 @@ static void flush_aliases(struct address_space *mapping, struct folio *folio) pgoff = folio->index; flush_dcache_mmap_lock_irqsave(mapping, flags); - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff + nr - 1) { + vma_interval_tree_foreach(vma, mapping, pgoff, pgoff + nr - 1) { unsigned long start; if (vma->vm_mm != mm) diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c index 0170b69a21d3..f28aa7884cbf 100644 --- a/arch/parisc/kernel/cache.c +++ b/arch/parisc/kernel/cache.c @@ -503,7 +503,7 @@ void flush_dcache_folio(struct folio *folio) * on machines that support equivalent aliasing */ flush_dcache_mmap_lock_irqsave(mapping, flags); - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff + nr - 1) { + vma_interval_tree_foreach(vma, mapping, pgoff, pgoff + nr - 1) { unsigned long offset = pgoff - vma->vm_pgoff; unsigned long pfn = folio_pfn(folio); diff --git a/fs/dax.c b/fs/dax.c index 6d175cd47a99..2f0818a68a7f 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -1201,7 +1201,7 @@ static int dax_writeback_one(struct xa_state *xas, struct dax_device *dax_dev, /* Walk all mappings of a given index of a file and writeprotect them */ i_mmap_lock_read(mapping); - vma_interval_tree_foreach(vma, &mapping->i_mmap, index, end) { + vma_interval_tree_foreach(vma, mapping, index, end) { pfn_mkclean_range(pfn, count, index, vma); cond_resched(); } diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 216e1a0dd0b2..4ea1798f1ffb 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -380,7 +380,6 @@ static void hugetlb_unmap_file_folio(struct hstate *h, struct address_space *mapping, struct folio *folio, pgoff_t index) { - struct rb_root_cached *root = &mapping->i_mmap; struct hugetlb_vma_lock *vma_lock; unsigned long pfn = folio_pfn(folio); struct vm_area_struct *vma; @@ -394,7 +393,7 @@ static void hugetlb_unmap_file_folio(struct hstate *h, i_mmap_lock_write(mapping); retry: vma_lock = NULL; - vma_interval_tree_foreach(vma, root, start, end - 1) { + vma_interval_tree_foreach(vma, mapping, start, end - 1) { v_start = vma_offset_start(vma, start); v_end = vma_offset_end(vma, end); @@ -460,8 +459,8 @@ static void hugetlb_unmap_file_folio(struct hstate *h, } static void -hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end, - zap_flags_t zap_flags) +hugetlb_vmdelete_list(struct address_space *mapping, pgoff_t start, + pgoff_t end, zap_flags_t zap_flags) { struct vm_area_struct *vma; @@ -470,7 +469,8 @@ hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end, * unmapped. Note, end is exclusive, whereas the interval tree takes * an inclusive "last". */ - vma_interval_tree_foreach(vma, root, start, end ? end - 1 : ULONG_MAX) { + vma_interval_tree_foreach(vma, mapping, start, + end ? end - 1 : ULONG_MAX) { unsigned long v_start; unsigned long v_end; @@ -615,8 +615,7 @@ static void hugetlb_vmtruncate(struct inode *inode, loff_t offset) i_size_write(inode, offset); i_mmap_lock_write(mapping); if (mapping_mapped(mapping)) - hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0, - ZAP_FLAG_DROP_MARKER); + hugetlb_vmdelete_list(mapping, pgoff, 0, ZAP_FLAG_DROP_MARKER); i_mmap_unlock_write(mapping); remove_inode_hugepages(inode, offset, LLONG_MAX); } @@ -676,7 +675,7 @@ static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len) /* Unmap users of full pages in the hole. */ if (hole_end > hole_start) { if (mapping_mapped(mapping)) - hugetlb_vmdelete_list(&mapping->i_mmap, + hugetlb_vmdelete_list(mapping, hole_start >> PAGE_SHIFT, hole_end >> PAGE_SHIFT, 0); } diff --git a/include/linux/mm.h b/include/linux/mm.h index 3e45978812e2..349d59a737eb 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4094,23 +4094,25 @@ extern atomic_long_t mmap_pages_allocated; extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t); /* interval_tree.c */ -void vma_interval_tree_insert(struct vm_area_struct *node, - struct rb_root_cached *root); -void vma_interval_tree_insert_after(struct vm_area_struct *node, +void vma_interval_tree_insert(struct vm_area_struct *vma, + struct address_space *mapping); +void vma_interval_tree_insert_after(struct vm_area_struct *vma, struct vm_area_struct *prev, - struct rb_root_cached *root); -void vma_interval_tree_remove(struct vm_area_struct *node, - struct rb_root_cached *root); -struct vm_area_struct *vma_interval_tree_subtree_search(struct vm_area_struct *node, - unsigned long start, unsigned long last); -struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root_cached *root, - unsigned long start, unsigned long last); -struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node, - unsigned long start, unsigned long last); + struct address_space *mapping); +void vma_interval_tree_remove(struct vm_area_struct *vma, + struct address_space *mapping); +struct vm_area_struct * +vma_interval_tree_iter_first(struct address_space *mapping, + pgoff_t pgoff_start, pgoff_t pgoff_last); +struct vm_area_struct * +vma_interval_tree_iter_next(struct vm_area_struct *vma, + pgoff_t pgoff_start, pgoff_t pgoff_last); -#define vma_interval_tree_foreach(vma, root, start, last) \ - for (vma = vma_interval_tree_iter_first(root, start, last); \ - vma; vma = vma_interval_tree_iter_next(vma, start, last)) +#define vma_interval_tree_foreach(vma, mapping, pgoff_start, pgoff_last) \ + for (vma = vma_interval_tree_iter_first(mapping, pgoff_start, \ + pgoff_last); \ + vma; vma = vma_interval_tree_iter_next(vma, pgoff_start, \ + pgoff_last)) void anon_vma_interval_tree_insert(struct anon_vma_chain *node, struct rb_root_cached *root); diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 4084e926e284..50a96a4d812d 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -1210,7 +1210,7 @@ build_map_info(struct address_space *mapping, loff_t offset, bool is_register) again: i_mmap_lock_read(mapping); - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { + vma_interval_tree_foreach(vma, mapping, pgoff, pgoff) { if (!valid_vma(vma, is_register)) continue; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 09c5783a3214..e1d519341704 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5443,7 +5443,7 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, * __unmap_hugepage_range() is called as the lock is already held */ i_mmap_lock_write(mapping); - vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) { + vma_interval_tree_foreach(iter_vma, mapping, pgoff, pgoff) { /* Do not unmap the current VMA */ if (iter_vma == vma) continue; @@ -6986,7 +6986,7 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, pte_t *pte; i_mmap_lock_read(mapping); - vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) { + vma_interval_tree_foreach(svma, mapping, idx, idx) { if (svma == vma) continue; diff --git a/mm/interval_tree.c b/mm/interval_tree.c index 2d50bc6228c4..ff36fd14ef37 100644 --- a/mm/interval_tree.c +++ b/mm/interval_tree.c @@ -14,19 +14,26 @@ /* File-backed interval tree (address_space->i_mmap) */ INTERVAL_TREE_DEFINE(struct vm_area_struct, shared.rb, - unsigned long, shared.rb_subtree_last, - vma_start_pgoff, vma_last_pgoff, /* empty */, vma_interval_tree) + pgoff_t, shared.rb_subtree_last, + vma_start_pgoff, vma_last_pgoff, static, + __vma_interval_tree) -/* Insert node immediately after prev in the interval tree */ -void vma_interval_tree_insert_after(struct vm_area_struct *node, +void vma_interval_tree_insert(struct vm_area_struct *vma, + struct address_space *mapping) +{ + __vma_interval_tree_insert(vma, &mapping->i_mmap); +} + +/* Insert vma immediately after prev in the interval tree */ +void vma_interval_tree_insert_after(struct vm_area_struct *vma, struct vm_area_struct *prev, - struct rb_root_cached *root) + struct address_space *mapping) { struct rb_node **link; struct vm_area_struct *parent; - unsigned long last = vma_last_pgoff(node); + const pgoff_t pgoff_last = vma_last_pgoff(vma); - VM_WARN_ON_ONCE_VMA(vma_start_pgoff(node) != vma_start_pgoff(prev), node); + VM_WARN_ON_ONCE_VMA(vma_start_pgoff(vma) != vma_start_pgoff(prev), vma); if (!prev->shared.rb.rb_right) { parent = prev; @@ -34,21 +41,42 @@ void vma_interval_tree_insert_after(struct vm_area_struct *node, } else { parent = rb_entry(prev->shared.rb.rb_right, struct vm_area_struct, shared.rb); - if (parent->shared.rb_subtree_last < last) - parent->shared.rb_subtree_last = last; + if (parent->shared.rb_subtree_last < pgoff_last) + parent->shared.rb_subtree_last = pgoff_last; while (parent->shared.rb.rb_left) { parent = rb_entry(parent->shared.rb.rb_left, struct vm_area_struct, shared.rb); - if (parent->shared.rb_subtree_last < last) - parent->shared.rb_subtree_last = last; + if (parent->shared.rb_subtree_last < pgoff_last) + parent->shared.rb_subtree_last = pgoff_last; } link = &parent->shared.rb.rb_left; } - node->shared.rb_subtree_last = last; - rb_link_node(&node->shared.rb, &parent->shared.rb, link); - rb_insert_augmented(&node->shared.rb, &root->rb_root, - &vma_interval_tree_augment); + vma->shared.rb_subtree_last = pgoff_last; + rb_link_node(&vma->shared.rb, &parent->shared.rb, link); + rb_insert_augmented(&vma->shared.rb, &mapping->i_mmap.rb_root, + &__vma_interval_tree_augment); +} + +void vma_interval_tree_remove(struct vm_area_struct *vma, + struct address_space *mapping) +{ + __vma_interval_tree_remove(vma, &mapping->i_mmap); +} + +struct vm_area_struct * +vma_interval_tree_iter_first(struct address_space *mapping, + pgoff_t pgoff_start, pgoff_t pgoff_last) +{ + return __vma_interval_tree_iter_first(&mapping->i_mmap, + pgoff_start, pgoff_last); +} + +struct vm_area_struct * +vma_interval_tree_iter_next(struct vm_area_struct *vma, + pgoff_t pgoff_start, pgoff_t pgoff_last) +{ + return __vma_interval_tree_iter_next(vma, pgoff_start, pgoff_last); } /* Anonymous interval tree (anon_vma->rb_root) */ diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 55157567dc4c..e5462e026642 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -2141,7 +2141,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) struct vm_area_struct *vma; i_mmap_lock_read(mapping); - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { + vma_interval_tree_foreach(vma, mapping, pgoff, pgoff) { struct mmu_notifier_range range; struct mm_struct *mm; unsigned long addr; @@ -2573,7 +2573,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * not be able to observe any missing pages due to the * previously inserted retry entries. */ - vma_interval_tree_foreach(vma, &mapping->i_mmap, start, end) { + vma_interval_tree_foreach(vma, mapping, start, end) { if (userfaultfd_missing(vma)) { result = SCAN_EXCEED_NONE_PTE; goto immap_locked; diff --git a/mm/memory-failure.c b/mm/memory-failure.c index bf717ec59508..4358856e764f 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -598,8 +598,7 @@ static void collect_procs_file(const struct folio *folio, if (!t) continue; - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, - pgoff) { + vma_interval_tree_foreach(vma, mapping, pgoff, pgoff) { /* * Send early kill signal to tasks where a vma covers * the page but the corrupted page is not necessarily @@ -650,7 +649,7 @@ static void collect_procs_fsdax(const struct page *page, t = task_early_kill(tsk, true); if (!t) continue; - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { + vma_interval_tree_foreach(vma, mapping, pgoff, pgoff) { if (vma->vm_mm == t->mm) add_to_kill_fsdax(t, page, vma, to_kill, pgoff); } @@ -2297,7 +2296,7 @@ static void collect_procs_pfn(struct pfn_address_space *pfn_space, t = task_early_kill(tsk, true); if (!t) continue; - vma_interval_tree_foreach(vma, &mapping->i_mmap, 0, ULONG_MAX) { + vma_interval_tree_foreach(vma, mapping, 0, ULONG_MAX) { pgoff_t pgoff; if (vma->vm_mm == t->mm && diff --git a/mm/memory.c b/mm/memory.c index 87db93f35bcf..6ce92badc38a 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4360,7 +4360,7 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) return wp_page_copy(vmf); } -static inline void unmap_mapping_range_tree(struct rb_root_cached *root, +static inline void unmap_mapping_range_tree(struct address_space *mapping, pgoff_t first_index, pgoff_t last_index, struct zap_details *details) @@ -4369,7 +4369,7 @@ static inline void unmap_mapping_range_tree(struct rb_root_cached *root, unsigned long start, size; struct mmu_gather tlb; - vma_interval_tree_foreach(vma, root, first_index, last_index) { + vma_interval_tree_foreach(vma, mapping, first_index, last_index) { const pgoff_t start_idx = max(first_index, vma->vm_pgoff); const pgoff_t end_idx = min(last_index, vma_last_pgoff(vma)) + 1; @@ -4411,7 +4411,7 @@ void unmap_mapping_folio(struct folio *folio) i_mmap_lock_read(mapping); if (unlikely(mapping_mapped(mapping))) - unmap_mapping_range_tree(&mapping->i_mmap, first_index, + unmap_mapping_range_tree(mapping, first_index, last_index, &details); i_mmap_unlock_read(mapping); } @@ -4441,7 +4441,7 @@ void unmap_mapping_pages(struct address_space *mapping, pgoff_t start, i_mmap_lock_read(mapping); if (unlikely(mapping_mapped(mapping))) - unmap_mapping_range_tree(&mapping->i_mmap, first_index, + unmap_mapping_range_tree(mapping, first_index, last_index, &details); i_mmap_unlock_read(mapping); } diff --git a/mm/mmap.c b/mm/mmap.c index 2311ae7c2ff4..2f22fb0d068d 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1830,8 +1830,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) mapping_allow_writable(mapping); flush_dcache_mmap_lock(mapping); /* insert tmp into the share list, just after mpnt */ - vma_interval_tree_insert_after(tmp, mpnt, - &mapping->i_mmap); + vma_interval_tree_insert_after(tmp, mpnt, mapping); flush_dcache_mmap_unlock(mapping); i_mmap_unlock_write(mapping); } diff --git a/mm/nommu.c b/mm/nommu.c index d9aa684d2bdf..0ff4c8a84299 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -569,7 +569,7 @@ static void setup_vma_to_mm(struct vm_area_struct *vma, struct mm_struct *mm) i_mmap_lock_write(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_insert(vma, &mapping->i_mmap); + vma_interval_tree_insert(vma, mapping); flush_dcache_mmap_unlock(mapping); i_mmap_unlock_write(mapping); } @@ -585,7 +585,7 @@ static void cleanup_vma_from_mm(struct vm_area_struct *vma) i_mmap_lock_write(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_remove(vma, &mapping->i_mmap); + vma_interval_tree_remove(vma, mapping); flush_dcache_mmap_unlock(mapping); i_mmap_unlock_write(mapping); } @@ -1839,7 +1839,7 @@ int nommu_shrink_inode_mappings(struct inode *inode, size_t size, i_mmap_lock_read(inode->i_mapping); /* search for VMAs that fall within the dead zone */ - vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, low, high) { + vma_interval_tree_foreach(vma, inode->i_mapping, low, high) { /* found one - only interested if it's shared out of the page * cache */ if (vma->vm_flags & VM_SHARED) { @@ -1855,7 +1855,7 @@ int nommu_shrink_inode_mappings(struct inode *inode, size_t size, * we don't check for any regions that start beyond the EOF as there * shouldn't be any */ - vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, 0, ULONG_MAX) { + vma_interval_tree_foreach(vma, inode->i_mapping, 0, ULONG_MAX) { if (!(vma->vm_flags & VM_SHARED)) continue; diff --git a/mm/pagewalk.c b/mm/pagewalk.c index 3ae2586ff45b..490a14691660 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -810,7 +810,7 @@ int walk_page_mapping(struct address_space *mapping, pgoff_t first_index, return -EINVAL; lockdep_assert_held(&mapping->i_mmap_rwsem); - vma_interval_tree_foreach(vma, &mapping->i_mmap, first_index, + vma_interval_tree_foreach(vma, mapping, first_index, first_index + nr - 1) { /* Clip to the vma */ vba = vma->vm_pgoff; diff --git a/mm/rmap.c b/mm/rmap.c index 26166a6b8cb9..73673cd3961e 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -3053,8 +3053,7 @@ static void __rmap_walk_file(struct folio *folio, struct address_space *mapping, i_mmap_lock_read(mapping); } lookup: - vma_interval_tree_foreach(vma, &mapping->i_mmap, - pgoff_start, pgoff_end) { + vma_interval_tree_foreach(vma, mapping, pgoff_start, pgoff_end) { unsigned long address = vma_address(vma, pgoff_start, nr_pages); VM_BUG_ON_VMA(address == -EFAULT, vma); diff --git a/mm/vma.c b/mm/vma.c index 9eea2850818a..ce4ec4b71138 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -234,7 +234,7 @@ static void __vma_link_file(struct vm_area_struct *vma, mapping_allow_writable(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_insert(vma, &mapping->i_mmap); + vma_interval_tree_insert(vma, mapping); flush_dcache_mmap_unlock(mapping); } @@ -248,7 +248,7 @@ static void __remove_shared_vm_struct(struct vm_area_struct *vma, mapping_unmap_writable(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_remove(vma, &mapping->i_mmap); + vma_interval_tree_remove(vma, mapping); flush_dcache_mmap_unlock(mapping); } @@ -319,10 +319,9 @@ static void vma_prepare(struct vma_prepare *vp) if (vp->file) { flush_dcache_mmap_lock(vp->mapping); - vma_interval_tree_remove(vp->vma, &vp->mapping->i_mmap); + vma_interval_tree_remove(vp->vma, vp->mapping); if (vp->adj_next) - vma_interval_tree_remove(vp->adj_next, - &vp->mapping->i_mmap); + vma_interval_tree_remove(vp->adj_next, vp->mapping); } } @@ -340,9 +339,8 @@ static void vma_complete(struct vma_prepare *vp, struct vma_iterator *vmi, { if (vp->file) { if (vp->adj_next) - vma_interval_tree_insert(vp->adj_next, - &vp->mapping->i_mmap); - vma_interval_tree_insert(vp->vma, &vp->mapping->i_mmap); + vma_interval_tree_insert(vp->adj_next, vp->mapping); + vma_interval_tree_insert(vp->vma, vp->mapping); flush_dcache_mmap_unlock(vp->mapping); } diff --git a/tools/testing/vma/include/stubs.h b/tools/testing/vma/include/stubs.h index 64164e25658f..94442b29458d 100644 --- a/tools/testing/vma/include/stubs.h +++ b/tools/testing/vma/include/stubs.h @@ -258,12 +258,12 @@ static inline void vm_acct_memory(long pages) } static inline void vma_interval_tree_insert(struct vm_area_struct *vma, - struct rb_root_cached *rb) + struct address_space *mapping) { } static inline void vma_interval_tree_remove(struct vm_area_struct *vma, - struct rb_root_cached *rb) + struct address_space *mapping) { } From 5ec02309d774e46bcf92d4262dfbdcf6ee2998b3 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:16:48 +0100 Subject: [PATCH 337/501] mm/rmap: elide unnecessary static inline's in interval_tree.c It's not necessary to declare these functions static inline as they are contained within a single compilation unit. This makes the anonymous interval tree code consistent with the newly updated file-backed interval tree code. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-7-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Pedro Falcato Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Signed-off-by: Andrew Morton --- mm/interval_tree.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/interval_tree.c b/mm/interval_tree.c index ff36fd14ef37..b387d39e0547 100644 --- a/mm/interval_tree.c +++ b/mm/interval_tree.c @@ -81,19 +81,19 @@ vma_interval_tree_iter_next(struct vm_area_struct *vma, /* Anonymous interval tree (anon_vma->rb_root) */ -static inline unsigned long avc_start_pgoff(struct anon_vma_chain *avc) +static unsigned long avc_start_pgoff(struct anon_vma_chain *avc) { return vma_start_pgoff(avc->vma); } -static inline unsigned long avc_last_pgoff(struct anon_vma_chain *avc) +static unsigned long avc_last_pgoff(struct anon_vma_chain *avc) { return vma_last_pgoff(avc->vma); } INTERVAL_TREE_DEFINE(struct anon_vma_chain, rb, unsigned long, rb_subtree_last, avc_start_pgoff, avc_last_pgoff, - static inline, __anon_vma_interval_tree) + static, __anon_vma_interval_tree) void anon_vma_interval_tree_insert(struct anon_vma_chain *node, struct rb_root_cached *root) From 4b7a90a48b9999c33e06dcce067a2d1540df0703 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:16:49 +0100 Subject: [PATCH 338/501] mm/rmap: rename vma_interval_tree_*() to mapping_rmap_tree_*() The family of vma_interval_tree_() functions manipulate the address_space (which, of course, is generally referred to as 'mapping') reverse mapping, but are named the 'VMA' interval tree. VMAs may be mapped by an anon_vma, an address_space, or both. Therefore calling the mapping interval tree a 'VMA' interval tree is rather confusing. This is also inconsistent with the anon_vma_interval_tree_*() functions which explicitly reference the rmap object to which they pertain. Rename the vma_interval_tree_*() functions to mapping_rmap_tree_*() to correct this. We will rename the anon rmap functions similarly in a subsequent patch. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-8-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: Pedro Falcato Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Signed-off-by: Andrew Morton --- arch/arm/mm/fault-armv.c | 2 +- arch/arm/mm/flush.c | 2 +- arch/nios2/mm/cacheflush.c | 2 +- arch/parisc/kernel/cache.c | 2 +- fs/dax.c | 2 +- fs/hugetlbfs/inode.c | 4 ++-- include/linux/mm.h | 16 ++++++++-------- kernel/events/uprobes.c | 2 +- mm/hugetlb.c | 4 ++-- mm/interval_tree.c | 22 +++++++++++----------- mm/khugepaged.c | 4 ++-- mm/memory-failure.c | 6 +++--- mm/memory.c | 2 +- mm/mmap.c | 2 +- mm/nommu.c | 8 ++++---- mm/pagewalk.c | 2 +- mm/rmap.c | 2 +- mm/vma.c | 12 ++++++------ tools/testing/vma/include/stubs.h | 4 ++-- 19 files changed, 50 insertions(+), 50 deletions(-) diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index cd52cf7f8874..b505f1fd7984 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -140,7 +140,7 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, * cache coherency. */ flush_dcache_mmap_lock(mapping); - vma_interval_tree_foreach(mpnt, mapping, pgoff, pgoff) { + mapping_rmap_tree_foreach(mpnt, mapping, pgoff, pgoff) { /* * If we are using split PTE locks, then we need to take the pte * lock. Otherwise we are using shared mm->page_table_lock which diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index 8c593e9898ee..66270904f874 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -251,7 +251,7 @@ static void __flush_dcache_aliases(struct address_space *mapping, struct folio * pgoff_end = pgoff + folio_nr_pages(folio) - 1; flush_dcache_mmap_lock(mapping); - vma_interval_tree_foreach(vma, mapping, pgoff, pgoff_end) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff_end) { unsigned long start, offset, pfn; unsigned int nr; diff --git a/arch/nios2/mm/cacheflush.c b/arch/nios2/mm/cacheflush.c index 42e3bf892316..10232476031f 100644 --- a/arch/nios2/mm/cacheflush.c +++ b/arch/nios2/mm/cacheflush.c @@ -82,7 +82,7 @@ static void flush_aliases(struct address_space *mapping, struct folio *folio) pgoff = folio->index; flush_dcache_mmap_lock_irqsave(mapping, flags); - vma_interval_tree_foreach(vma, mapping, pgoff, pgoff + nr - 1) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff + nr - 1) { unsigned long start; if (vma->vm_mm != mm) diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c index f28aa7884cbf..98200b6b5b82 100644 --- a/arch/parisc/kernel/cache.c +++ b/arch/parisc/kernel/cache.c @@ -503,7 +503,7 @@ void flush_dcache_folio(struct folio *folio) * on machines that support equivalent aliasing */ flush_dcache_mmap_lock_irqsave(mapping, flags); - vma_interval_tree_foreach(vma, mapping, pgoff, pgoff + nr - 1) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff + nr - 1) { unsigned long offset = pgoff - vma->vm_pgoff; unsigned long pfn = folio_pfn(folio); diff --git a/fs/dax.c b/fs/dax.c index 2f0818a68a7f..4ea3384ecefd 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -1201,7 +1201,7 @@ static int dax_writeback_one(struct xa_state *xas, struct dax_device *dax_dev, /* Walk all mappings of a given index of a file and writeprotect them */ i_mmap_lock_read(mapping); - vma_interval_tree_foreach(vma, mapping, index, end) { + mapping_rmap_tree_foreach(vma, mapping, index, end) { pfn_mkclean_range(pfn, count, index, vma); cond_resched(); } diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 4ea1798f1ffb..9fddd1e0b813 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -393,7 +393,7 @@ static void hugetlb_unmap_file_folio(struct hstate *h, i_mmap_lock_write(mapping); retry: vma_lock = NULL; - vma_interval_tree_foreach(vma, mapping, start, end - 1) { + mapping_rmap_tree_foreach(vma, mapping, start, end - 1) { v_start = vma_offset_start(vma, start); v_end = vma_offset_end(vma, end); @@ -469,7 +469,7 @@ hugetlb_vmdelete_list(struct address_space *mapping, pgoff_t start, * unmapped. Note, end is exclusive, whereas the interval tree takes * an inclusive "last". */ - vma_interval_tree_foreach(vma, mapping, start, + mapping_rmap_tree_foreach(vma, mapping, start, end ? end - 1 : ULONG_MAX) { unsigned long v_start; unsigned long v_end; diff --git a/include/linux/mm.h b/include/linux/mm.h index 349d59a737eb..38132060ac58 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4094,24 +4094,24 @@ extern atomic_long_t mmap_pages_allocated; extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t); /* interval_tree.c */ -void vma_interval_tree_insert(struct vm_area_struct *vma, +void mapping_rmap_tree_insert(struct vm_area_struct *vma, struct address_space *mapping); -void vma_interval_tree_insert_after(struct vm_area_struct *vma, +void mapping_rmap_tree_insert_after(struct vm_area_struct *vma, struct vm_area_struct *prev, struct address_space *mapping); -void vma_interval_tree_remove(struct vm_area_struct *vma, +void mapping_rmap_tree_remove(struct vm_area_struct *vma, struct address_space *mapping); struct vm_area_struct * -vma_interval_tree_iter_first(struct address_space *mapping, +mapping_rmap_tree_iter_first(struct address_space *mapping, pgoff_t pgoff_start, pgoff_t pgoff_last); struct vm_area_struct * -vma_interval_tree_iter_next(struct vm_area_struct *vma, +mapping_rmap_tree_iter_next(struct vm_area_struct *vma, pgoff_t pgoff_start, pgoff_t pgoff_last); -#define vma_interval_tree_foreach(vma, mapping, pgoff_start, pgoff_last) \ - for (vma = vma_interval_tree_iter_first(mapping, pgoff_start, \ +#define mapping_rmap_tree_foreach(vma, mapping, pgoff_start, pgoff_last) \ + for (vma = mapping_rmap_tree_iter_first(mapping, pgoff_start, \ pgoff_last); \ - vma; vma = vma_interval_tree_iter_next(vma, pgoff_start, \ + vma; vma = mapping_rmap_tree_iter_next(vma, pgoff_start, \ pgoff_last)) void anon_vma_interval_tree_insert(struct anon_vma_chain *node, diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 50a96a4d812d..ac340a703d06 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -1210,7 +1210,7 @@ build_map_info(struct address_space *mapping, loff_t offset, bool is_register) again: i_mmap_lock_read(mapping); - vma_interval_tree_foreach(vma, mapping, pgoff, pgoff) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff) { if (!valid_vma(vma, is_register)) continue; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index e1d519341704..b5928d6766f6 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5443,7 +5443,7 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, * __unmap_hugepage_range() is called as the lock is already held */ i_mmap_lock_write(mapping); - vma_interval_tree_foreach(iter_vma, mapping, pgoff, pgoff) { + mapping_rmap_tree_foreach(iter_vma, mapping, pgoff, pgoff) { /* Do not unmap the current VMA */ if (iter_vma == vma) continue; @@ -6986,7 +6986,7 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, pte_t *pte; i_mmap_lock_read(mapping); - vma_interval_tree_foreach(svma, mapping, idx, idx) { + mapping_rmap_tree_foreach(svma, mapping, idx, idx) { if (svma == vma) continue; diff --git a/mm/interval_tree.c b/mm/interval_tree.c index b387d39e0547..39e020555910 100644 --- a/mm/interval_tree.c +++ b/mm/interval_tree.c @@ -16,16 +16,16 @@ INTERVAL_TREE_DEFINE(struct vm_area_struct, shared.rb, pgoff_t, shared.rb_subtree_last, vma_start_pgoff, vma_last_pgoff, static, - __vma_interval_tree) + __mapping_rmap_tree) -void vma_interval_tree_insert(struct vm_area_struct *vma, +void mapping_rmap_tree_insert(struct vm_area_struct *vma, struct address_space *mapping) { - __vma_interval_tree_insert(vma, &mapping->i_mmap); + __mapping_rmap_tree_insert(vma, &mapping->i_mmap); } /* Insert vma immediately after prev in the interval tree */ -void vma_interval_tree_insert_after(struct vm_area_struct *vma, +void mapping_rmap_tree_insert_after(struct vm_area_struct *vma, struct vm_area_struct *prev, struct address_space *mapping) { @@ -55,28 +55,28 @@ void vma_interval_tree_insert_after(struct vm_area_struct *vma, vma->shared.rb_subtree_last = pgoff_last; rb_link_node(&vma->shared.rb, &parent->shared.rb, link); rb_insert_augmented(&vma->shared.rb, &mapping->i_mmap.rb_root, - &__vma_interval_tree_augment); + &__mapping_rmap_tree_augment); } -void vma_interval_tree_remove(struct vm_area_struct *vma, +void mapping_rmap_tree_remove(struct vm_area_struct *vma, struct address_space *mapping) { - __vma_interval_tree_remove(vma, &mapping->i_mmap); + __mapping_rmap_tree_remove(vma, &mapping->i_mmap); } struct vm_area_struct * -vma_interval_tree_iter_first(struct address_space *mapping, +mapping_rmap_tree_iter_first(struct address_space *mapping, pgoff_t pgoff_start, pgoff_t pgoff_last) { - return __vma_interval_tree_iter_first(&mapping->i_mmap, + return __mapping_rmap_tree_iter_first(&mapping->i_mmap, pgoff_start, pgoff_last); } struct vm_area_struct * -vma_interval_tree_iter_next(struct vm_area_struct *vma, +mapping_rmap_tree_iter_next(struct vm_area_struct *vma, pgoff_t pgoff_start, pgoff_t pgoff_last) { - return __vma_interval_tree_iter_next(vma, pgoff_start, pgoff_last); + return __mapping_rmap_tree_iter_next(vma, pgoff_start, pgoff_last); } /* Anonymous interval tree (anon_vma->rb_root) */ diff --git a/mm/khugepaged.c b/mm/khugepaged.c index e5462e026642..912be3daf37e 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -2141,7 +2141,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) struct vm_area_struct *vma; i_mmap_lock_read(mapping); - vma_interval_tree_foreach(vma, mapping, pgoff, pgoff) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff) { struct mmu_notifier_range range; struct mm_struct *mm; unsigned long addr; @@ -2573,7 +2573,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * not be able to observe any missing pages due to the * previously inserted retry entries. */ - vma_interval_tree_foreach(vma, mapping, start, end) { + mapping_rmap_tree_foreach(vma, mapping, start, end) { if (userfaultfd_missing(vma)) { result = SCAN_EXCEED_NONE_PTE; goto immap_locked; diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 4358856e764f..66ae93c6d4b2 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -598,7 +598,7 @@ static void collect_procs_file(const struct folio *folio, if (!t) continue; - vma_interval_tree_foreach(vma, mapping, pgoff, pgoff) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff) { /* * Send early kill signal to tasks where a vma covers * the page but the corrupted page is not necessarily @@ -649,7 +649,7 @@ static void collect_procs_fsdax(const struct page *page, t = task_early_kill(tsk, true); if (!t) continue; - vma_interval_tree_foreach(vma, mapping, pgoff, pgoff) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff) { if (vma->vm_mm == t->mm) add_to_kill_fsdax(t, page, vma, to_kill, pgoff); } @@ -2296,7 +2296,7 @@ static void collect_procs_pfn(struct pfn_address_space *pfn_space, t = task_early_kill(tsk, true); if (!t) continue; - vma_interval_tree_foreach(vma, mapping, 0, ULONG_MAX) { + mapping_rmap_tree_foreach(vma, mapping, 0, ULONG_MAX) { pgoff_t pgoff; if (vma->vm_mm == t->mm && diff --git a/mm/memory.c b/mm/memory.c index 6ce92badc38a..b7e9a7e53516 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4369,7 +4369,7 @@ static inline void unmap_mapping_range_tree(struct address_space *mapping, unsigned long start, size; struct mmu_gather tlb; - vma_interval_tree_foreach(vma, mapping, first_index, last_index) { + mapping_rmap_tree_foreach(vma, mapping, first_index, last_index) { const pgoff_t start_idx = max(first_index, vma->vm_pgoff); const pgoff_t end_idx = min(last_index, vma_last_pgoff(vma)) + 1; diff --git a/mm/mmap.c b/mm/mmap.c index 2f22fb0d068d..91eaa0c9e7ee 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1830,7 +1830,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) mapping_allow_writable(mapping); flush_dcache_mmap_lock(mapping); /* insert tmp into the share list, just after mpnt */ - vma_interval_tree_insert_after(tmp, mpnt, mapping); + mapping_rmap_tree_insert_after(tmp, mpnt, mapping); flush_dcache_mmap_unlock(mapping); i_mmap_unlock_write(mapping); } diff --git a/mm/nommu.c b/mm/nommu.c index 0ff4c8a84299..355b3b1ab7ff 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -569,7 +569,7 @@ static void setup_vma_to_mm(struct vm_area_struct *vma, struct mm_struct *mm) i_mmap_lock_write(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_insert(vma, mapping); + mapping_rmap_tree_insert(vma, mapping); flush_dcache_mmap_unlock(mapping); i_mmap_unlock_write(mapping); } @@ -585,7 +585,7 @@ static void cleanup_vma_from_mm(struct vm_area_struct *vma) i_mmap_lock_write(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_remove(vma, mapping); + mapping_rmap_tree_remove(vma, mapping); flush_dcache_mmap_unlock(mapping); i_mmap_unlock_write(mapping); } @@ -1839,7 +1839,7 @@ int nommu_shrink_inode_mappings(struct inode *inode, size_t size, i_mmap_lock_read(inode->i_mapping); /* search for VMAs that fall within the dead zone */ - vma_interval_tree_foreach(vma, inode->i_mapping, low, high) { + mapping_rmap_tree_foreach(vma, inode->i_mapping, low, high) { /* found one - only interested if it's shared out of the page * cache */ if (vma->vm_flags & VM_SHARED) { @@ -1855,7 +1855,7 @@ int nommu_shrink_inode_mappings(struct inode *inode, size_t size, * we don't check for any regions that start beyond the EOF as there * shouldn't be any */ - vma_interval_tree_foreach(vma, inode->i_mapping, 0, ULONG_MAX) { + mapping_rmap_tree_foreach(vma, inode->i_mapping, 0, ULONG_MAX) { if (!(vma->vm_flags & VM_SHARED)) continue; diff --git a/mm/pagewalk.c b/mm/pagewalk.c index 490a14691660..7e7819cf5a3a 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -810,7 +810,7 @@ int walk_page_mapping(struct address_space *mapping, pgoff_t first_index, return -EINVAL; lockdep_assert_held(&mapping->i_mmap_rwsem); - vma_interval_tree_foreach(vma, mapping, first_index, + mapping_rmap_tree_foreach(vma, mapping, first_index, first_index + nr - 1) { /* Clip to the vma */ vba = vma->vm_pgoff; diff --git a/mm/rmap.c b/mm/rmap.c index 73673cd3961e..d3d3d6cb17b8 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -3053,7 +3053,7 @@ static void __rmap_walk_file(struct folio *folio, struct address_space *mapping, i_mmap_lock_read(mapping); } lookup: - vma_interval_tree_foreach(vma, mapping, pgoff_start, pgoff_end) { + mapping_rmap_tree_foreach(vma, mapping, pgoff_start, pgoff_end) { unsigned long address = vma_address(vma, pgoff_start, nr_pages); VM_BUG_ON_VMA(address == -EFAULT, vma); diff --git a/mm/vma.c b/mm/vma.c index ce4ec4b71138..d99e2088de4e 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -234,7 +234,7 @@ static void __vma_link_file(struct vm_area_struct *vma, mapping_allow_writable(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_insert(vma, mapping); + mapping_rmap_tree_insert(vma, mapping); flush_dcache_mmap_unlock(mapping); } @@ -248,7 +248,7 @@ static void __remove_shared_vm_struct(struct vm_area_struct *vma, mapping_unmap_writable(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_remove(vma, mapping); + mapping_rmap_tree_remove(vma, mapping); flush_dcache_mmap_unlock(mapping); } @@ -319,9 +319,9 @@ static void vma_prepare(struct vma_prepare *vp) if (vp->file) { flush_dcache_mmap_lock(vp->mapping); - vma_interval_tree_remove(vp->vma, vp->mapping); + mapping_rmap_tree_remove(vp->vma, vp->mapping); if (vp->adj_next) - vma_interval_tree_remove(vp->adj_next, vp->mapping); + mapping_rmap_tree_remove(vp->adj_next, vp->mapping); } } @@ -339,8 +339,8 @@ static void vma_complete(struct vma_prepare *vp, struct vma_iterator *vmi, { if (vp->file) { if (vp->adj_next) - vma_interval_tree_insert(vp->adj_next, vp->mapping); - vma_interval_tree_insert(vp->vma, vp->mapping); + mapping_rmap_tree_insert(vp->adj_next, vp->mapping); + mapping_rmap_tree_insert(vp->vma, vp->mapping); flush_dcache_mmap_unlock(vp->mapping); } diff --git a/tools/testing/vma/include/stubs.h b/tools/testing/vma/include/stubs.h index 94442b29458d..37f81b11e8b7 100644 --- a/tools/testing/vma/include/stubs.h +++ b/tools/testing/vma/include/stubs.h @@ -257,12 +257,12 @@ static inline void vm_acct_memory(long pages) { } -static inline void vma_interval_tree_insert(struct vm_area_struct *vma, +static inline void mapping_rmap_tree_insert(struct vm_area_struct *vma, struct address_space *mapping) { } -static inline void vma_interval_tree_remove(struct vm_area_struct *vma, +static inline void mapping_rmap_tree_remove(struct vm_area_struct *vma, struct address_space *mapping) { } From 68364437ec782b7dac7c7eca3b1f177a9815f3ff Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:16:50 +0100 Subject: [PATCH 339/501] mm/rmap: parameterise anon_vma_interval_tree_*() by anon_vma Similar to what we did with mapping_rmap_tree*(), let's declare anon_vma_interval_tree*() in terms of anon_vma rather than rb_root_cached. In each case the rb tree referenced is &anon_vma->rb_root, so just pass anon_vma and the functions can figure this out themselves. Also update the VMA userland tests to reflect the change. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-9-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Pedro Falcato Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Signed-off-by: Andrew Morton --- include/linux/mm.h | 11 ++++++----- mm/interval_tree.c | 13 +++++++------ mm/ksm.c | 6 ++---- mm/memory-failure.c | 3 +-- mm/rmap.c | 11 +++++------ mm/vma.c | 4 ++-- tools/testing/vma/include/stubs.h | 4 ++-- 7 files changed, 25 insertions(+), 27 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 38132060ac58..efbe0eebdcda 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4115,14 +4115,15 @@ mapping_rmap_tree_iter_next(struct vm_area_struct *vma, pgoff_last)) void anon_vma_interval_tree_insert(struct anon_vma_chain *node, - struct rb_root_cached *root); + struct anon_vma *anon_vma); void anon_vma_interval_tree_remove(struct anon_vma_chain *node, - struct rb_root_cached *root); + struct anon_vma *anon_vma); struct anon_vma_chain * -anon_vma_interval_tree_iter_first(struct rb_root_cached *root, +anon_vma_interval_tree_iter_first(struct anon_vma *anon_vma, unsigned long start, unsigned long last); -struct anon_vma_chain *anon_vma_interval_tree_iter_next( - struct anon_vma_chain *node, unsigned long start, unsigned long last); +struct anon_vma_chain * +anon_vma_interval_tree_iter_next(struct anon_vma_chain *avc, + unsigned long start, unsigned long last); #ifdef CONFIG_DEBUG_VM_RB void anon_vma_interval_tree_verify(struct anon_vma_chain *node); #endif diff --git a/mm/interval_tree.c b/mm/interval_tree.c index 39e020555910..b82d2a0d9c4e 100644 --- a/mm/interval_tree.c +++ b/mm/interval_tree.c @@ -96,26 +96,27 @@ INTERVAL_TREE_DEFINE(struct anon_vma_chain, rb, unsigned long, rb_subtree_last, static, __anon_vma_interval_tree) void anon_vma_interval_tree_insert(struct anon_vma_chain *node, - struct rb_root_cached *root) + struct anon_vma *anon_vma) { #ifdef CONFIG_DEBUG_VM_RB node->cached_vma_start = avc_start_pgoff(node); node->cached_vma_last = avc_last_pgoff(node); #endif - __anon_vma_interval_tree_insert(node, root); + __anon_vma_interval_tree_insert(node, &anon_vma->rb_root); } void anon_vma_interval_tree_remove(struct anon_vma_chain *node, - struct rb_root_cached *root) + struct anon_vma *anon_vma) { - __anon_vma_interval_tree_remove(node, root); + __anon_vma_interval_tree_remove(node, &anon_vma->rb_root); } struct anon_vma_chain * -anon_vma_interval_tree_iter_first(struct rb_root_cached *root, +anon_vma_interval_tree_iter_first(struct anon_vma *anon_vma, unsigned long first, unsigned long last) { - return __anon_vma_interval_tree_iter_first(root, first, last); + return __anon_vma_interval_tree_iter_first(&anon_vma->rb_root, + first, last); } struct anon_vma_chain * diff --git a/mm/ksm.c b/mm/ksm.c index 2791ce5bd44b..c0bd8eae763f 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -3232,8 +3232,7 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc) * we wouldn't have the KSM folio mapped in these processes * anymore. */ - anon_vma_interval_tree_foreach(vmac, &anon_vma->rb_root, - index, index) { + anon_vma_interval_tree_foreach(vmac, anon_vma, index, index) { cond_resched(); vma = vmac->vma; @@ -3295,8 +3294,7 @@ void collect_procs_ksm(const struct folio *folio, const struct page *page, task_early_kill(tsk, force_early); if (!t) continue; - anon_vma_interval_tree_foreach(vmac, &av->rb_root, index, - index) + anon_vma_interval_tree_foreach(vmac, av, index, index) { vma = vmac->vma; if (vma->vm_mm == t->mm) { diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 66ae93c6d4b2..24d660c32058 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -564,8 +564,7 @@ static void collect_procs_anon(const struct folio *folio, if (!t) continue; - anon_vma_interval_tree_foreach(vmac, &av->rb_root, - pgoff, pgoff) { + anon_vma_interval_tree_foreach(vmac, av, pgoff, pgoff) { vma = vmac->vma; if (vma->vm_mm != t->mm) continue; diff --git a/mm/rmap.c b/mm/rmap.c index d3d3d6cb17b8..6d4423a16c8b 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -211,7 +211,7 @@ int __anon_vma_prepare(struct vm_area_struct *vma) if (likely(!vma->anon_vma)) { vma->anon_vma = anon_vma; anon_vma_chain_assign(vma, avc, anon_vma); - anon_vma_interval_tree_insert(avc, &anon_vma->rb_root); + anon_vma_interval_tree_insert(avc, anon_vma); anon_vma->num_active_vmas++; allocated = NULL; avc = NULL; @@ -354,7 +354,7 @@ int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src, list_for_each_entry_reverse(avc, &dst->anon_vma_chain, same_vma) { struct anon_vma *anon_vma = avc->anon_vma; - anon_vma_interval_tree_insert(avc, &anon_vma->rb_root); + anon_vma_interval_tree_insert(avc, anon_vma); if (operation == VMA_OP_FORK) maybe_reuse_anon_vma(dst, anon_vma); } @@ -434,7 +434,7 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) anon_vma_chain_assign(vma, avc, anon_vma); /* Now let rmap see it. */ anon_vma_lock_write(anon_vma); - anon_vma_interval_tree_insert(avc, &anon_vma->rb_root); + anon_vma_interval_tree_insert(avc, anon_vma); anon_vma->parent->num_children++; anon_vma_unlock_write(anon_vma); @@ -499,7 +499,7 @@ void unlink_anon_vmas(struct vm_area_struct *vma) list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { struct anon_vma *anon_vma = avc->anon_vma; - anon_vma_interval_tree_remove(avc, &anon_vma->rb_root); + anon_vma_interval_tree_remove(avc, anon_vma); /* * Leave empty anon_vmas on the list - we'll need @@ -2988,8 +2988,7 @@ static void rmap_walk_anon(struct folio *folio, pgoff_start = folio_pgoff(folio); pgoff_end = pgoff_start + folio_nr_pages(folio) - 1; - anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, - pgoff_start, pgoff_end) { + anon_vma_interval_tree_foreach(avc, anon_vma, pgoff_start, pgoff_end) { struct vm_area_struct *vma = avc->vma; unsigned long address = vma_address(vma, pgoff_start, folio_nr_pages(folio)); diff --git a/mm/vma.c b/mm/vma.c index d99e2088de4e..2cd70f1d4db7 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -272,7 +272,7 @@ anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma) struct anon_vma_chain *avc; list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) - anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root); + anon_vma_interval_tree_remove(avc, avc->anon_vma); } static void @@ -281,7 +281,7 @@ anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma) struct anon_vma_chain *avc; list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) - anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root); + anon_vma_interval_tree_insert(avc, avc->anon_vma); } /* diff --git a/tools/testing/vma/include/stubs.h b/tools/testing/vma/include/stubs.h index 37f81b11e8b7..3a11547b5bd1 100644 --- a/tools/testing/vma/include/stubs.h +++ b/tools/testing/vma/include/stubs.h @@ -272,12 +272,12 @@ static inline void flush_dcache_mmap_unlock(struct address_space *mapping) } static inline void anon_vma_interval_tree_insert(struct anon_vma_chain *avc, - struct rb_root_cached *rb) + struct anon_vma *anon_vma) { } static inline void anon_vma_interval_tree_remove(struct anon_vma_chain *avc, - struct rb_root_cached *rb) + struct anon_vma *anon_vma) { } From 0b099cc22967420284ddd66689525ed6de10d45e Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:16:51 +0100 Subject: [PATCH 340/501] mm/rmap: rename anon_vma_interval_tree_*() params and use pgoff_t Rename parameters used by anon_vma_interval_tree_*() functions: 'node' to 'avc', 'start/first' to 'pgoff_start', and 'last' to 'pgoff_last' to make clear what is being passed. Also, express page offsets in terms of pgoff_t to be consistent. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-10-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: Pedro Falcato Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Signed-off-by: Andrew Morton --- include/linux/mm.h | 19 +++++++++++-------- mm/interval_tree.c | 34 +++++++++++++++++----------------- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index efbe0eebdcda..e108b03f536a 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4114,23 +4114,26 @@ mapping_rmap_tree_iter_next(struct vm_area_struct *vma, vma; vma = mapping_rmap_tree_iter_next(vma, pgoff_start, \ pgoff_last)) -void anon_vma_interval_tree_insert(struct anon_vma_chain *node, +void anon_vma_interval_tree_insert(struct anon_vma_chain *avc, struct anon_vma *anon_vma); -void anon_vma_interval_tree_remove(struct anon_vma_chain *node, +void anon_vma_interval_tree_remove(struct anon_vma_chain *avc, struct anon_vma *anon_vma); + struct anon_vma_chain * anon_vma_interval_tree_iter_first(struct anon_vma *anon_vma, - unsigned long start, unsigned long last); + pgoff_t pgoff_start, pgoff_t pgoff_last); struct anon_vma_chain * anon_vma_interval_tree_iter_next(struct anon_vma_chain *avc, - unsigned long start, unsigned long last); + pgoff_t pgoff_start, pgoff_t pgoff_last); #ifdef CONFIG_DEBUG_VM_RB -void anon_vma_interval_tree_verify(struct anon_vma_chain *node); +void anon_vma_interval_tree_verify(struct anon_vma_chain *avc); #endif -#define anon_vma_interval_tree_foreach(avc, root, start, last) \ - for (avc = anon_vma_interval_tree_iter_first(root, start, last); \ - avc; avc = anon_vma_interval_tree_iter_next(avc, start, last)) +#define anon_vma_interval_tree_foreach(avc, anon_vma, pgoff_start, pgoff_last) \ + for (avc = anon_vma_interval_tree_iter_first(anon_vma, pgoff_start, \ + pgoff_last); \ + avc; avc = anon_vma_interval_tree_iter_next(avc, pgoff_start, \ + pgoff_last)) /* mmap.c */ extern int __vm_enough_memory(const struct mm_struct *mm, long pages, int cap_sys_admin); diff --git a/mm/interval_tree.c b/mm/interval_tree.c index b82d2a0d9c4e..5bfe95b5cd28 100644 --- a/mm/interval_tree.c +++ b/mm/interval_tree.c @@ -81,55 +81,55 @@ mapping_rmap_tree_iter_next(struct vm_area_struct *vma, /* Anonymous interval tree (anon_vma->rb_root) */ -static unsigned long avc_start_pgoff(struct anon_vma_chain *avc) +static pgoff_t avc_start_pgoff(struct anon_vma_chain *avc) { return vma_start_pgoff(avc->vma); } -static unsigned long avc_last_pgoff(struct anon_vma_chain *avc) +static pgoff_t avc_last_pgoff(struct anon_vma_chain *avc) { return vma_last_pgoff(avc->vma); } -INTERVAL_TREE_DEFINE(struct anon_vma_chain, rb, unsigned long, rb_subtree_last, +INTERVAL_TREE_DEFINE(struct anon_vma_chain, rb, pgoff_t, rb_subtree_last, avc_start_pgoff, avc_last_pgoff, static, __anon_vma_interval_tree) -void anon_vma_interval_tree_insert(struct anon_vma_chain *node, +void anon_vma_interval_tree_insert(struct anon_vma_chain *avc, struct anon_vma *anon_vma) { #ifdef CONFIG_DEBUG_VM_RB - node->cached_vma_start = avc_start_pgoff(node); - node->cached_vma_last = avc_last_pgoff(node); + avc->cached_vma_start = avc_start_pgoff(avc); + avc->cached_vma_last = avc_last_pgoff(avc); #endif - __anon_vma_interval_tree_insert(node, &anon_vma->rb_root); + __anon_vma_interval_tree_insert(avc, &anon_vma->rb_root); } -void anon_vma_interval_tree_remove(struct anon_vma_chain *node, +void anon_vma_interval_tree_remove(struct anon_vma_chain *avc, struct anon_vma *anon_vma) { - __anon_vma_interval_tree_remove(node, &anon_vma->rb_root); + __anon_vma_interval_tree_remove(avc, &anon_vma->rb_root); } struct anon_vma_chain * anon_vma_interval_tree_iter_first(struct anon_vma *anon_vma, - unsigned long first, unsigned long last) + pgoff_t pgoff_start, pgoff_t pgoff_last) { return __anon_vma_interval_tree_iter_first(&anon_vma->rb_root, - first, last); + pgoff_start, pgoff_last); } struct anon_vma_chain * -anon_vma_interval_tree_iter_next(struct anon_vma_chain *node, - unsigned long first, unsigned long last) +anon_vma_interval_tree_iter_next(struct anon_vma_chain *avc, + pgoff_t pgoff_start, pgoff_t pgoff_last) { - return __anon_vma_interval_tree_iter_next(node, first, last); + return __anon_vma_interval_tree_iter_next(avc, pgoff_start, pgoff_last); } #ifdef CONFIG_DEBUG_VM_RB -void anon_vma_interval_tree_verify(struct anon_vma_chain *node) +void anon_vma_interval_tree_verify(struct anon_vma_chain *avc) { - WARN_ON_ONCE(node->cached_vma_start != avc_start_pgoff(node)); - WARN_ON_ONCE(node->cached_vma_last != avc_last_pgoff(node)); + WARN_ON_ONCE(avc->cached_vma_start != avc_start_pgoff(avc)); + WARN_ON_ONCE(avc->cached_vma_last != avc_last_pgoff(avc)); } #endif From b1e71c05fa3704d3e1306c8ac4e8bba2d9bd263c Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:16:52 +0100 Subject: [PATCH 341/501] mm/rmap: rename anon_vma_interval_tree_*() to anon_rmap_tree_*() To be consistent with the newly renamed mapping_rmap_tree_*(), rename the anon_vma_interval_tree_*() helpers to anon_rmap_tree_*(). No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-11-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: Pedro Falcato Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Signed-off-by: Andrew Morton --- include/linux/mm.h | 27 ++++++++++++--------------- mm/interval_tree.c | 30 +++++++++++++++--------------- mm/ksm.c | 4 ++-- mm/memory-failure.c | 2 +- mm/rmap.c | 10 +++++----- mm/vma.c | 30 +++++++++++++++--------------- tools/testing/vma/include/stubs.h | 8 ++++---- 7 files changed, 54 insertions(+), 57 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index e108b03f536a..498692da306c 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4114,26 +4114,23 @@ mapping_rmap_tree_iter_next(struct vm_area_struct *vma, vma; vma = mapping_rmap_tree_iter_next(vma, pgoff_start, \ pgoff_last)) -void anon_vma_interval_tree_insert(struct anon_vma_chain *avc, - struct anon_vma *anon_vma); -void anon_vma_interval_tree_remove(struct anon_vma_chain *avc, - struct anon_vma *anon_vma); - +void anon_rmap_tree_insert(struct anon_vma_chain *avc, + struct anon_vma *anon_vma); +void anon_rmap_tree_remove(struct anon_vma_chain *avc, + struct anon_vma *anon_vma); struct anon_vma_chain * -anon_vma_interval_tree_iter_first(struct anon_vma *anon_vma, - pgoff_t pgoff_start, pgoff_t pgoff_last); +anon_rmap_tree_iter_first(struct anon_vma *anon_vma, + pgoff_t pgoff_start, pgoff_t pgoff_last); struct anon_vma_chain * -anon_vma_interval_tree_iter_next(struct anon_vma_chain *avc, - pgoff_t pgoff_start, pgoff_t pgoff_last); +anon_rmap_tree_iter_next(struct anon_vma_chain *avc, + pgoff_t pgoff_start, pgoff_t pgoff_last); #ifdef CONFIG_DEBUG_VM_RB -void anon_vma_interval_tree_verify(struct anon_vma_chain *avc); +void anon_rmap_tree_verify(struct anon_vma_chain *avc); #endif -#define anon_vma_interval_tree_foreach(avc, anon_vma, pgoff_start, pgoff_last) \ - for (avc = anon_vma_interval_tree_iter_first(anon_vma, pgoff_start, \ - pgoff_last); \ - avc; avc = anon_vma_interval_tree_iter_next(avc, pgoff_start, \ - pgoff_last)) +#define anon_rmap_tree_foreach(avc, anon_vma, pgoff_start, pgoff_last) \ + for (avc = anon_rmap_tree_iter_first(anon_vma, pgoff_start, pgoff_last); \ + avc; avc = anon_rmap_tree_iter_next(avc, pgoff_start, pgoff_last)) /* mmap.c */ extern int __vm_enough_memory(const struct mm_struct *mm, long pages, int cap_sys_admin); diff --git a/mm/interval_tree.c b/mm/interval_tree.c index 5bfe95b5cd28..3ae9e106d3af 100644 --- a/mm/interval_tree.c +++ b/mm/interval_tree.c @@ -93,41 +93,41 @@ static pgoff_t avc_last_pgoff(struct anon_vma_chain *avc) INTERVAL_TREE_DEFINE(struct anon_vma_chain, rb, pgoff_t, rb_subtree_last, avc_start_pgoff, avc_last_pgoff, - static, __anon_vma_interval_tree) + static, __anon_rmap_tree) -void anon_vma_interval_tree_insert(struct anon_vma_chain *avc, - struct anon_vma *anon_vma) +void anon_rmap_tree_insert(struct anon_vma_chain *avc, + struct anon_vma *anon_vma) { #ifdef CONFIG_DEBUG_VM_RB avc->cached_vma_start = avc_start_pgoff(avc); avc->cached_vma_last = avc_last_pgoff(avc); #endif - __anon_vma_interval_tree_insert(avc, &anon_vma->rb_root); + __anon_rmap_tree_insert(avc, &anon_vma->rb_root); } -void anon_vma_interval_tree_remove(struct anon_vma_chain *avc, - struct anon_vma *anon_vma) +void anon_rmap_tree_remove(struct anon_vma_chain *avc, + struct anon_vma *anon_vma) { - __anon_vma_interval_tree_remove(avc, &anon_vma->rb_root); + __anon_rmap_tree_remove(avc, &anon_vma->rb_root); } struct anon_vma_chain * -anon_vma_interval_tree_iter_first(struct anon_vma *anon_vma, - pgoff_t pgoff_start, pgoff_t pgoff_last) +anon_rmap_tree_iter_first(struct anon_vma *anon_vma, + pgoff_t pgoff_start, pgoff_t pgoff_last) { - return __anon_vma_interval_tree_iter_first(&anon_vma->rb_root, - pgoff_start, pgoff_last); + return __anon_rmap_tree_iter_first(&anon_vma->rb_root, + pgoff_start, pgoff_last); } struct anon_vma_chain * -anon_vma_interval_tree_iter_next(struct anon_vma_chain *avc, - pgoff_t pgoff_start, pgoff_t pgoff_last) +anon_rmap_tree_iter_next(struct anon_vma_chain *avc, + pgoff_t pgoff_start, pgoff_t pgoff_last) { - return __anon_vma_interval_tree_iter_next(avc, pgoff_start, pgoff_last); + return __anon_rmap_tree_iter_next(avc, pgoff_start, pgoff_last); } #ifdef CONFIG_DEBUG_VM_RB -void anon_vma_interval_tree_verify(struct anon_vma_chain *avc) +void anon_rmap_tree_verify(struct anon_vma_chain *avc) { WARN_ON_ONCE(avc->cached_vma_start != avc_start_pgoff(avc)); WARN_ON_ONCE(avc->cached_vma_last != avc_last_pgoff(avc)); diff --git a/mm/ksm.c b/mm/ksm.c index c0bd8eae763f..14550f69cf02 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -3232,7 +3232,7 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc) * we wouldn't have the KSM folio mapped in these processes * anymore. */ - anon_vma_interval_tree_foreach(vmac, anon_vma, index, index) { + anon_rmap_tree_foreach(vmac, anon_vma, index, index) { cond_resched(); vma = vmac->vma; @@ -3294,7 +3294,7 @@ void collect_procs_ksm(const struct folio *folio, const struct page *page, task_early_kill(tsk, force_early); if (!t) continue; - anon_vma_interval_tree_foreach(vmac, av, index, index) + anon_rmap_tree_foreach(vmac, av, index, index) { vma = vmac->vma; if (vma->vm_mm == t->mm) { diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 24d660c32058..aaf14608b30e 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -564,7 +564,7 @@ static void collect_procs_anon(const struct folio *folio, if (!t) continue; - anon_vma_interval_tree_foreach(vmac, av, pgoff, pgoff) { + anon_rmap_tree_foreach(vmac, av, pgoff, pgoff) { vma = vmac->vma; if (vma->vm_mm != t->mm) continue; diff --git a/mm/rmap.c b/mm/rmap.c index 6d4423a16c8b..e854679553b9 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -211,7 +211,7 @@ int __anon_vma_prepare(struct vm_area_struct *vma) if (likely(!vma->anon_vma)) { vma->anon_vma = anon_vma; anon_vma_chain_assign(vma, avc, anon_vma); - anon_vma_interval_tree_insert(avc, anon_vma); + anon_rmap_tree_insert(avc, anon_vma); anon_vma->num_active_vmas++; allocated = NULL; avc = NULL; @@ -354,7 +354,7 @@ int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src, list_for_each_entry_reverse(avc, &dst->anon_vma_chain, same_vma) { struct anon_vma *anon_vma = avc->anon_vma; - anon_vma_interval_tree_insert(avc, anon_vma); + anon_rmap_tree_insert(avc, anon_vma); if (operation == VMA_OP_FORK) maybe_reuse_anon_vma(dst, anon_vma); } @@ -434,7 +434,7 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) anon_vma_chain_assign(vma, avc, anon_vma); /* Now let rmap see it. */ anon_vma_lock_write(anon_vma); - anon_vma_interval_tree_insert(avc, anon_vma); + anon_rmap_tree_insert(avc, anon_vma); anon_vma->parent->num_children++; anon_vma_unlock_write(anon_vma); @@ -499,7 +499,7 @@ void unlink_anon_vmas(struct vm_area_struct *vma) list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { struct anon_vma *anon_vma = avc->anon_vma; - anon_vma_interval_tree_remove(avc, anon_vma); + anon_rmap_tree_remove(avc, anon_vma); /* * Leave empty anon_vmas on the list - we'll need @@ -2988,7 +2988,7 @@ static void rmap_walk_anon(struct folio *folio, pgoff_start = folio_pgoff(folio); pgoff_end = pgoff_start + folio_nr_pages(folio) - 1; - anon_vma_interval_tree_foreach(avc, anon_vma, pgoff_start, pgoff_end) { + anon_rmap_tree_foreach(avc, anon_vma, pgoff_start, pgoff_end) { struct vm_area_struct *vma = avc->vma; unsigned long address = vma_address(vma, pgoff_start, folio_nr_pages(folio)); diff --git a/mm/vma.c b/mm/vma.c index 2cd70f1d4db7..35ba3475128f 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -258,30 +258,30 @@ static void __remove_shared_vm_struct(struct vm_area_struct *vma, * * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the * vma must be removed from the anon_vma's interval trees using - * anon_vma_interval_tree_pre_update_vma(). + * anon_rmap_tree_pre_update_vma(). * * After the update, the vma will be reinserted using - * anon_vma_interval_tree_post_update_vma(). + * anon_rmap_tree_post_update_vma(). * * The entire update must be protected by exclusive mmap_lock and by * the root anon_vma's mutex. */ static void -anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma) +anon_rmap_tree_pre_update_vma(struct vm_area_struct *vma) { struct anon_vma_chain *avc; list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) - anon_vma_interval_tree_remove(avc, avc->anon_vma); + anon_rmap_tree_remove(avc, avc->anon_vma); } static void -anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma) +anon_rmap_tree_post_update_vma(struct vm_area_struct *vma) { struct anon_vma_chain *avc; list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) - anon_vma_interval_tree_insert(avc, avc->anon_vma); + anon_rmap_tree_insert(avc, avc->anon_vma); } /* @@ -312,9 +312,9 @@ static void vma_prepare(struct vma_prepare *vp) if (vp->anon_vma) { anon_vma_lock_write(vp->anon_vma); - anon_vma_interval_tree_pre_update_vma(vp->vma); + anon_rmap_tree_pre_update_vma(vp->vma); if (vp->adj_next) - anon_vma_interval_tree_pre_update_vma(vp->adj_next); + anon_rmap_tree_pre_update_vma(vp->adj_next); } if (vp->file) { @@ -359,9 +359,9 @@ static void vma_complete(struct vma_prepare *vp, struct vma_iterator *vmi, } if (vp->anon_vma) { - anon_vma_interval_tree_post_update_vma(vp->vma); + anon_rmap_tree_post_update_vma(vp->vma); if (vp->adj_next) - anon_vma_interval_tree_post_update_vma(vp->adj_next); + anon_rmap_tree_post_update_vma(vp->adj_next); anon_vma_unlock_write(vp->anon_vma); } @@ -677,7 +677,7 @@ void validate_mm(struct mm_struct *mm) if (anon_vma) { anon_vma_lock_read(anon_vma); list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) - anon_vma_interval_tree_verify(avc); + anon_rmap_tree_verify(avc); anon_vma_unlock_read(anon_vma); } #endif @@ -3177,11 +3177,11 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) if (vma_test(vma, VMA_LOCKED_BIT)) mm->locked_vm += grow; vm_stat_account(mm, vma->vm_flags, grow); - anon_vma_interval_tree_pre_update_vma(vma); + anon_rmap_tree_pre_update_vma(vma); vma->vm_end = address; /* Overwrite old entry in mtree. */ vma_iter_store_overwrite(&vmi, vma); - anon_vma_interval_tree_post_update_vma(vma); + anon_rmap_tree_post_update_vma(vma); perf_event_mmap(vma); } @@ -3256,12 +3256,12 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address) if (vma_test(vma, VMA_LOCKED_BIT)) mm->locked_vm += grow; vm_stat_account(mm, vma->vm_flags, grow); - anon_vma_interval_tree_pre_update_vma(vma); + anon_rmap_tree_pre_update_vma(vma); vma->vm_start = address; vma->vm_pgoff -= grow; /* Overwrite old entry in mtree. */ vma_iter_store_overwrite(&vmi, vma); - anon_vma_interval_tree_post_update_vma(vma); + anon_rmap_tree_post_update_vma(vma); perf_event_mmap(vma); } diff --git a/tools/testing/vma/include/stubs.h b/tools/testing/vma/include/stubs.h index 3a11547b5bd1..d6136e19a8af 100644 --- a/tools/testing/vma/include/stubs.h +++ b/tools/testing/vma/include/stubs.h @@ -271,13 +271,13 @@ static inline void flush_dcache_mmap_unlock(struct address_space *mapping) { } -static inline void anon_vma_interval_tree_insert(struct anon_vma_chain *avc, - struct anon_vma *anon_vma) +static inline void anon_rmap_tree_insert(struct anon_vma_chain *avc, + struct anon_vma *anon_vma) { } -static inline void anon_vma_interval_tree_remove(struct anon_vma_chain *avc, - struct anon_vma *anon_vma) +static inline void anon_rmap_tree_remove(struct anon_vma_chain *avc, + struct anon_vma *anon_vma) { } From 271f11c7a5a17d9d5437fc61b37f8036f4e69789 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:16:53 +0100 Subject: [PATCH 342/501] MAINTAINERS: move mm/interval_tree.c to rmap section This file implements code for the interval trees used by the file and anon rmap implementation, so belongs in the rmap section. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-12-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Acked-by: Pedro Falcato Acked-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Acked-by: Liam R. Howlett (Oracle) Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Gregory Price Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Cc: Zi Yan Signed-off-by: Andrew Morton --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index ec4f60692bde..1db1811dbc5d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17212,6 +17212,7 @@ R: Lance Yang L: linux-mm@kvack.org S: Maintained F: include/linux/rmap.h +F: mm/interval_tree.c F: mm/page_vma_mapped.c F: mm/rmap.c F: tools/testing/selftests/mm/rmap.c @@ -17317,7 +17318,6 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm F: include/trace/events/mmap.h F: fs/proc/task_mmu.c F: fs/proc/task_nommu.c -F: mm/interval_tree.c F: mm/mincore.c F: mm/mlock.c F: mm/mmap.c From 8802c4069ee98f064b41d047cfa96475852e57f8 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:16:54 +0100 Subject: [PATCH 343/501] mm/vma: introduce and use vmg_pages(), vmg_[start, end]_pgoff() In the VMA logic we often need to determine the number of pages in the specified merge range, as well as the start and end page offsets of that range. Introduce and use helpers for these purposes. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-13-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Pedro Falcato Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Signed-off-by: Andrew Morton --- mm/vma.c | 11 ++++------- mm/vma.h | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/mm/vma.c b/mm/vma.c index 35ba3475128f..995de8198fbb 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -197,11 +197,9 @@ static void init_multi_vma_prep(struct vma_prepare *vp, */ static bool can_vma_merge_before(struct vma_merge_struct *vmg) { - pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start); - if (is_mergeable_vma(vmg, /* merge_next = */ true) && is_mergeable_anon_vma(vmg, /* merge_next = */ true)) { - if (vmg->next->vm_pgoff == vmg->pgoff + pglen) + if (vmg_end_pgoff(vmg) == vma_start_pgoff(vmg->next)) return true; } @@ -221,7 +219,7 @@ static bool can_vma_merge_after(struct vma_merge_struct *vmg) { if (is_mergeable_vma(vmg, /* merge_next = */ false) && is_mergeable_anon_vma(vmg, /* merge_next = */ false)) { - if (vmg->prev->vm_pgoff + vma_pages(vmg->prev) == vmg->pgoff) + if (vma_end_pgoff(vmg->prev) == vmg_start_pgoff(vmg)) return true; } return false; @@ -759,7 +757,7 @@ static int commit_merge(struct vma_merge_struct *vmg) */ vma_adjust_trans_huge(vma, vmg->start, vmg->end, vmg->__adjust_middle_start ? vmg->middle : NULL); - vma_set_range(vma, vmg->start, vmg->end, vmg->pgoff); + vma_set_range(vma, vmg->start, vmg->end, vmg_start_pgoff(vmg)); vmg_adjust_set_range(vmg); vma_iter_store_overwrite(vmg->vmi, vmg->target); @@ -962,8 +960,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( * middle next * shrink/delete extend */ - - pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start); + const pgoff_t pglen = vmg_pages(vmg); VM_WARN_ON_VMG(!merge_right, vmg); /* If we are offset into a VMA, then prev must be middle. */ diff --git a/mm/vma.h b/mm/vma.h index 8e4b61a7304c..527716c8739d 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -230,6 +230,23 @@ static inline bool vmg_nomem(struct vma_merge_struct *vmg) return vmg->state == VMA_MERGE_ERROR_NOMEM; } +static inline pgoff_t vmg_start_pgoff(const struct vma_merge_struct *vmg) +{ + return vmg->pgoff; +} + +static inline pgoff_t vmg_pages(const struct vma_merge_struct *vmg) +{ + const unsigned long size = vmg->end - vmg->start; + + return size >> PAGE_SHIFT; +} + +static inline pgoff_t vmg_end_pgoff(const struct vma_merge_struct *vmg) +{ + return vmg_start_pgoff(vmg) + vmg_pages(vmg); +} + /* Assumes addr >= vma->vm_start. */ static inline pgoff_t vma_pgoff_offset(struct vm_area_struct *vma, unsigned long addr) From dfe2b9abf65d4e046ef9ca53523776d5f3f97532 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:16:55 +0100 Subject: [PATCH 344/501] mm/vma: clean up anon_vma_compatible() Break up the existing very large conditional, add comments and use vma_[start/end]_pgoff() to make clearer what we're doing here. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-14-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Gregory Price Reviewed-by: Pedro Falcato Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Signed-off-by: Andrew Morton --- mm/vma.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/mm/vma.c b/mm/vma.c index 995de8198fbb..084afdf37385 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -1967,14 +1967,25 @@ static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct * { vma_flags_t diff = vma_flags_diff_pair(&a->flags, &b->flags); + /* Ignore flags that mprotect() can change. */ vma_flags_clear_mask(&diff, VMA_ACCESS_FLAGS); + /* Ignore flags that do not impact merging. */ vma_flags_clear_mask(&diff, VMA_IGNORE_MERGE_FLAGS); - return a->vm_end == b->vm_start && - mpol_equal(vma_policy(a), vma_policy(b)) && - a->vm_file == b->vm_file && - vma_flags_empty(&diff) && - b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT); + /* Must be adjacent. */ + if (a->vm_end != b->vm_start) + return false; + /* Must have matching policy. */ + if (!mpol_equal(vma_policy(a), vma_policy(b))) + return false; + /* Must both be anon or map the same file (MAP_PRIVATE case). */ + if (a->vm_file != b->vm_file) + return false; + /* Flags must be equivalent modulo mprotect(). */ + if (!vma_flags_empty(&diff)) + return false; + /* Page offset must align. */ + return vma_end_pgoff(a) == vma_start_pgoff(b); } /* From a30574d073c41cb8df26f487d8d9a01bc22432c7 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:16:56 +0100 Subject: [PATCH 345/501] mm/vma: refactor vmg_adjust_set_range() for clarity Add comments with ASCII diagrams to describe what we're doing, avoid dubious use of PHYS_PFN(), and use vma_start_pgoff(). The most complicated scenario represented here is vmg->__adjust_next_start - when this is set, vmg->[start, end] actually indicate the range to be retained, so take special care to describe this accurately. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-15-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Pedro Falcato Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/vma.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/mm/vma.c b/mm/vma.c index 084afdf37385..7265a054cfa3 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -704,11 +704,54 @@ static void vmg_adjust_set_range(struct vma_merge_struct *vmg) pgoff_t pgoff; if (vmg->__adjust_middle_start) { - adjust = vmg->middle; - pgoff = adjust->vm_pgoff + PHYS_PFN(vmg->end - adjust->vm_start); + /* + * vmg->start vmg->end + * | | + * v merge v + * <-------------> + * delta + * <------> + * |------|----------------| + * | prev | middle | + * |------|----------------| + * ^ + * | + * middle->vm_start + */ + struct vm_area_struct *middle = vmg->middle; + const unsigned long delta = vmg->end - middle->vm_start; + + pgoff = vma_start_pgoff(middle) + (delta >> PAGE_SHIFT); + adjust = middle; } else if (vmg->__adjust_next_start) { - adjust = vmg->next; - pgoff = adjust->vm_pgoff - PHYS_PFN(adjust->vm_start - vmg->end); + /* + * Originally: + * + * vmg->start vmg->end + * | | + * v merge v + * <------------> + * . . + * merge_existing_range() updates to: + * . . + * vmg->start vmg->end . + * | | . + * v retain v . + * <----------> . + * delta . + * <-----> . + * |----------------|------| + * | middle | next | + * |----------------|------| + * ^ + * | + * next->vm_start + */ + struct vm_area_struct *next = vmg->next; + const unsigned long delta = next->vm_start - vmg->end; + + pgoff = vma_start_pgoff(next) - (delta >> PAGE_SHIFT); + adjust = next; } else { return; } From 68d307442ba4ec7db5f10b45620a9c5d86145064 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:16:57 +0100 Subject: [PATCH 346/501] mm/vma: minor cleanup of expand_[upwards, downwards]() Adjust the stack expansion functions expand_upwards() and expand_downwards() such that they are expressed in terms of named constant values, and make use of vma_start_pgoff(). This clearly documents that we are referencing the page offset of the start of the VMA. Additionally this cleans up the overflow check in expand_upwards(). No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-16-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Pedro Falcato Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/vma.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/mm/vma.c b/mm/vma.c index 7265a054cfa3..5d5e60ea8a25 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -3216,13 +3216,12 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) /* Somebody else might have raced and expanded it already */ if (address > vma->vm_end) { - unsigned long size, grow; - - size = address - vma->vm_start; - grow = (address - vma->vm_end) >> PAGE_SHIFT; + const unsigned long size = address - vma->vm_start; + const unsigned long grow = (address - vma->vm_end) >> PAGE_SHIFT; + const pgoff_t pgoff = vma_start_pgoff(vma); error = -ENOMEM; - if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) { + if (pgoff + (size >> PAGE_SHIFT) >= pgoff) { error = acct_stack_growth(vma, size, grow); if (!error) { if (vma_test(vma, VMA_LOCKED_BIT)) @@ -3295,13 +3294,11 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address) /* Somebody else might have raced and expanded it already */ if (address < vma->vm_start) { - unsigned long size, grow; - - size = vma->vm_end - address; - grow = (vma->vm_start - address) >> PAGE_SHIFT; + const unsigned long size = vma->vm_end - address; + const unsigned long grow = (vma->vm_start - address) >> PAGE_SHIFT; error = -ENOMEM; - if (grow <= vma->vm_pgoff) { + if (grow <= vma_start_pgoff(vma)) { error = acct_stack_growth(vma, size, grow); if (!error) { if (vma_test(vma, VMA_LOCKED_BIT)) From 1268e1572ba545b75e11fbc33380f85a5d1ec1db Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:16:58 +0100 Subject: [PATCH 347/501] mm: introduce and use linear_page_delta() It's often useful to obtain the number of pages a given address lies at within a VMA. Add linear_page_delta() to determine this and update linear_page_index() to make use of it. Add comments to describe both linear_page_delta() and linear_page_index(). We refer the reader to the comment for vma_start_pgoff() so we don't duplicate information about the meaning of a VMA's page offset, which is also less likely to result in bitrot. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-17-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Ackerley Tng Reviewed-by: Gregory Price Reviewed-by: Pedro Falcato Reviewed-by: Vlastimil Babka (SUSE) Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/pagemap.h | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 2c3718d592d6..a950070b045f 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1063,11 +1063,40 @@ static inline pgoff_t folio_pgoff(const struct folio *folio) return folio->index; } +/** + * linear_page_delta() - Determine the relative page offset of @address within + * @vma. + * @vma: The VMA in which @address resides. + * @address: The address whose relative page offset is required. + * + * The result is identical for both file-backed and anonymous mappings and + * simply determines how many pages @address lies from @vma->vm_start. + * + * Returns: The number of pages @address is offset by within @vma. + */ +static inline pgoff_t linear_page_delta(const struct vm_area_struct *vma, + const unsigned long address) +{ + return (address - vma->vm_start) >> PAGE_SHIFT; +} + +/** + * linear_page_index() - Determine the absolute page offset of @address within + * @vma. + * @vma: The VMA in which @address resides. + * @address: The address whose absolute page offset is required. + * + * See the comment for vma_start_pgoff() for a description of what the page + * offset signifies. + * + * Returns: The absolute page offset of @address within @vma. + */ static inline pgoff_t linear_page_index(const struct vm_area_struct *vma, const unsigned long address) { pgoff_t pgoff; - pgoff = (address - vma->vm_start) >> PAGE_SHIFT; + + pgoff = linear_page_delta(vma, address); pgoff += vma->vm_pgoff; return pgoff; } @@ -1219,7 +1248,7 @@ static inline vm_fault_t folio_lock_or_retry(struct folio *folio, void folio_wait_bit(struct folio *folio, int bit_nr); int folio_wait_bit_killable(struct folio *folio, int bit_nr); -/* +/* * Wait for a folio to be unlocked. * * This must be called with the caller "holding" the folio, From 5872168de5d5c1f69a2efcf80e331c32aa08ec9a Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:16:59 +0100 Subject: [PATCH 348/501] mm/vma: use vma_start_pgoff(), linear_page_index() in mm code There are many instances in which linear_page_index() (as well as linear_page_delta()) is open-coded, which is confusing and inconsistent. Additionally, vma->vm_pgoff doesn't necessarily make it clear that this is the page offset of the start of the VMA range. Doing so also aids greppability. So use vma_start_pgoff() in favour of directly accessing vma->vm_pgoff, and linear_page_index() where we can. This also lays the ground for future changes which will add an anonymous page offset in order to be able to index MAP_PRIVATE-file backed anon folios in terms of their virtual page offset. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-18-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Gregory Price Reviewed-by: SJ Park Reviewed-by: Pedro Falcato Reviewed-by: Vlastimil Babka (SUSE) Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/huge_mm.h | 1 + include/linux/hugetlb.h | 3 +-- include/linux/pagemap.h | 2 +- mm/damon/vaddr.c | 5 +++-- mm/debug.c | 2 +- mm/filemap.c | 7 ++++--- mm/huge_memory.c | 2 +- mm/hugetlb.c | 11 ++++------- mm/internal.h | 24 ++++++++++++++---------- mm/khugepaged.c | 3 ++- mm/madvise.c | 6 +++--- mm/mapping_dirty_helpers.c | 2 +- mm/memory.c | 25 +++++++++++++------------ mm/mempolicy.c | 13 +++++++------ mm/mremap.c | 12 ++++-------- mm/msync.c | 4 ++-- mm/nommu.c | 7 ++++--- mm/pagewalk.c | 2 +- mm/shmem.c | 9 +++++---- mm/userfaultfd.c | 4 ++-- mm/util.c | 4 ++-- mm/vma.c | 15 +++++++-------- mm/vma_exec.c | 4 ++-- mm/vma_init.c | 2 +- 24 files changed, 86 insertions(+), 83 deletions(-) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 19a7bfce279e..cae97c307280 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -230,6 +230,7 @@ static inline bool thp_vma_suitable_order(struct vm_area_struct *vma, /* Don't have to check pgoff for anonymous vma */ if (!vma_is_anonymous(vma)) { + /* vma_start_pgoff() in mm.h so not available. */ if (!IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - vma->vm_pgoff, hpage_size >> PAGE_SHIFT)) return false; diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 95b1b6ee1f96..6ca232a61643 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -801,8 +801,7 @@ static inline pgoff_t hugetlb_linear_page_index(struct vm_area_struct *vma, { struct hstate *h = hstate_vma(vma); - return ((address - vma->vm_start) >> huge_page_shift(h)) + - (vma->vm_pgoff >> huge_page_order(h)); + return linear_page_index(vma, address) >> huge_page_order(h); } static inline bool order_is_gigantic(unsigned int order) diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index a950070b045f..c6fc783aaee5 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1097,7 +1097,7 @@ static inline pgoff_t linear_page_index(const struct vm_area_struct *vma, pgoff_t pgoff; pgoff = linear_page_delta(vma, address); - pgoff += vma->vm_pgoff; + pgoff += vma_start_pgoff(vma); return pgoff; } diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 2058db9c01d5..d10b8042adb5 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -623,8 +624,8 @@ static void damos_va_migrate_dests_add(struct folio *folio, } order = folio_order(folio); - ilx = vma->vm_pgoff >> order; - ilx += (addr - vma->vm_start) >> (PAGE_SHIFT + order); + ilx = vma_start_pgoff(vma) >> order; + ilx += linear_page_delta(vma, addr) >> order; for (i = 0; i < dests->nr_dests; i++) weight_total += dests->weight_arr[i]; diff --git a/mm/debug.c b/mm/debug.c index 77fa8fe1d641..497654b36f1a 100644 --- a/mm/debug.c +++ b/mm/debug.c @@ -163,7 +163,7 @@ void dump_vma(const struct vm_area_struct *vma) "flags: %#lx(%pGv)\n", vma, (void *)vma->vm_start, (void *)vma->vm_end, vma->vm_mm, (unsigned long)pgprot_val(vma->vm_page_prot), - vma->anon_vma, vma->vm_ops, vma->vm_pgoff, + vma->anon_vma, vma->vm_ops, vma_start_pgoff(vma), vma->vm_file, vma->vm_private_data, #ifdef CONFIG_PER_VMA_LOCK refcount_read(&vma->vm_refcnt), diff --git a/mm/filemap.c b/mm/filemap.c index b39111abdc4b..1dbb4c6f824e 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -3411,8 +3411,8 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) * of memory. */ struct vm_area_struct *vma = vmf->vma; - unsigned long start = vma->vm_pgoff; - unsigned long end = start + vma_pages(vma); + const unsigned long start = vma_start_pgoff(vma); + const unsigned long end = vma_end_pgoff(vma); unsigned long ra_end; ra->order = exec_folio_order(); @@ -3930,7 +3930,8 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, goto out; } - addr = vma->vm_start + ((start_pgoff - vma->vm_pgoff) << PAGE_SHIFT); + addr = vma->vm_start + + ((start_pgoff - vma_start_pgoff(vma)) << PAGE_SHIFT); vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl); if (!vmf->pte) { folio_unlock(folio); diff --git a/mm/huge_memory.c b/mm/huge_memory.c index b6b0355268ea..c89c1193c64d 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -180,7 +180,7 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, */ if (!in_pf && shmem_file(vma->vm_file)) return orders & shmem_allowable_huge_orders(file_inode(vma->vm_file), - vma, vma->vm_pgoff, 0, + vma, vma_start_pgoff(vma), 0, forced_collapse); if (!vma_is_anonymous(vma)) { diff --git a/mm/hugetlb.c b/mm/hugetlb.c index b5928d6766f6..0500b4e8bcd2 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1012,8 +1012,7 @@ static long region_count(struct resv_map *resv, long f, long t) static pgoff_t vma_hugecache_offset(struct hstate *h, struct vm_area_struct *vma, unsigned long address) { - return ((address - vma->vm_start) >> huge_page_shift(h)) + - (vma->vm_pgoff >> huge_page_order(h)); + return linear_page_index(vma, address) >> huge_page_order(h); } /* @@ -5433,8 +5432,7 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, * from page cache lookup which is in HPAGE_SIZE units. */ address = address & huge_page_mask(h); - pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + - vma->vm_pgoff; + pgoff = linear_page_index(vma, address); mapping = vma->vm_file->f_mapping; /* @@ -6893,7 +6891,7 @@ static unsigned long page_table_shareable(struct vm_area_struct *svma, struct vm_area_struct *vma, unsigned long addr, pgoff_t idx) { - unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) + + unsigned long saddr = ((idx - vma_start_pgoff(svma)) << PAGE_SHIFT) + svma->vm_start; unsigned long sbase = saddr & PUD_MASK; unsigned long s_end = sbase + PUD_SIZE; @@ -6978,8 +6976,7 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, pud_t *pud) { struct address_space *mapping = vma->vm_file->f_mapping; - pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + - vma->vm_pgoff; + const pgoff_t idx = linear_page_index(vma, addr); struct vm_area_struct *svma; unsigned long saddr; pte_t *spte = NULL; diff --git a/mm/internal.h b/mm/internal.h index e5dcab69eb4f..f950e88fdb52 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -924,26 +924,28 @@ static inline bool folio_within_range(struct folio *folio, struct vm_area_struct *vma, unsigned long start, unsigned long end) { - pgoff_t pgoff, addr; - unsigned long vma_pglen = vma_pages(vma); + const unsigned long vma_pglen = vma_pages(vma); + pgoff_t pgoff_folio, pgoff_vma_start; + unsigned long addr; VM_WARN_ON_FOLIO(folio_test_ksm(folio), folio); if (start > end) return false; + pgoff_folio = folio_pgoff(folio); + pgoff_vma_start = vma_start_pgoff(vma); + if (start < vma->vm_start) start = vma->vm_start; if (end > vma->vm_end) end = vma->vm_end; - pgoff = folio_pgoff(folio); - /* if folio start address is not in vma range */ - if (!in_range(pgoff, vma->vm_pgoff, vma_pglen)) + if (!in_range(pgoff_folio, pgoff_vma_start, vma_pglen)) return false; - addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + addr = vma->vm_start + ((pgoff_folio - pgoff_vma_start) << PAGE_SHIFT); return !(addr < start || end - addr < folio_size(folio)); } @@ -1015,15 +1017,16 @@ extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma); static inline unsigned long vma_address(const struct vm_area_struct *vma, pgoff_t pgoff, unsigned long nr_pages) { + const pgoff_t pgoff_start = vma_start_pgoff(vma); unsigned long address; - if (pgoff >= vma->vm_pgoff) { + if (pgoff >= pgoff_start) { address = vma->vm_start + - ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + ((pgoff - pgoff_start) << PAGE_SHIFT); /* Check for address beyond vma (or wrapped through 0?) */ if (address < vma->vm_start || address >= vma->vm_end) address = -EFAULT; - } else if (pgoff + nr_pages - 1 >= vma->vm_pgoff) { + } else if (pgoff + nr_pages - 1 >= pgoff_start) { /* Test above avoids possibility of wrap to 0 on 32-bit */ address = vma->vm_start; } else { @@ -1047,7 +1050,8 @@ static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw) return pvmw->address + PAGE_SIZE; pgoff = pvmw->pgoff + pvmw->nr_pages; - address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + address = vma->vm_start + + ((pgoff - vma_start_pgoff(vma)) << PAGE_SHIFT); /* Check for address beyond vma (or wrapped through 0?) */ if (address < vma->vm_start || address > vma->vm_end) address = vma->vm_end; diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 912be3daf37e..45e8245d80da 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -2150,7 +2150,8 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) spinlock_t *ptl; bool success = false; - addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + addr = vma->vm_start + + ((pgoff - vma_start_pgoff(vma)) << PAGE_SHIFT); if (addr & ~HPAGE_PMD_MASK || vma->vm_end < addr + HPAGE_PMD_SIZE) continue; diff --git a/mm/madvise.c b/mm/madvise.c index 9292f60b19aa..917f764fef80 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -253,7 +253,7 @@ static void shmem_swapin_range(struct vm_area_struct *vma, continue; addr = vma->vm_start + - ((xas.xa_index - vma->vm_pgoff) << PAGE_SHIFT); + ((xas.xa_index - vma_start_pgoff(vma)) << PAGE_SHIFT); xas_pause(&xas); rcu_read_unlock(); @@ -318,7 +318,7 @@ static long madvise_willneed(struct madvise_behavior *madv_behavior) mark_mmap_lock_dropped(madv_behavior); get_file(file); offset = (loff_t)(start - vma->vm_start) - + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + + ((loff_t)vma_start_pgoff(vma) << PAGE_SHIFT); mmap_read_unlock(mm); vfs_fadvise(file, offset, end - start, POSIX_FADV_WILLNEED); fput(file); @@ -1022,7 +1022,7 @@ static long madvise_remove(struct madvise_behavior *madv_behavior) return -EACCES; offset = (loff_t)(start - vma->vm_start) - + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + + ((loff_t)vma_start_pgoff(vma) << PAGE_SHIFT); /* * Filesystem's fallocate may need to take i_rwsem. We need to diff --git a/mm/mapping_dirty_helpers.c b/mm/mapping_dirty_helpers.c index 737c407f4081..e0efa36e0a07 100644 --- a/mm/mapping_dirty_helpers.c +++ b/mm/mapping_dirty_helpers.c @@ -95,7 +95,7 @@ static int clean_record_pte(pte_t *pte, unsigned long addr, if (pte_dirty(ptent)) { pgoff_t pgoff = ((addr - walk->vma->vm_start) >> PAGE_SHIFT) + - walk->vma->vm_pgoff - cwalk->bitmap_pgoff; + vma_start_pgoff(walk->vma) - cwalk->bitmap_pgoff; pte_t old_pte = ptep_modify_prot_start(walk->vma, addr, pte); ptent = pte_mkclean(old_pte); diff --git a/mm/memory.c b/mm/memory.c index b7e9a7e53516..fb7a246487ac 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -725,10 +725,10 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma, if (!pfn_valid(pfn)) return NULL; } else { - unsigned long off = (addr - vma->vm_start) >> PAGE_SHIFT; + const pgoff_t index = linear_page_index(vma, addr); /* Only CoW'ed anon folios are "normal". */ - if (pfn == vma->vm_pgoff + off) + if (pfn == index) return NULL; if (!is_cow_mapping(vma->vm_flags)) return NULL; @@ -2663,7 +2663,7 @@ static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages, int vm_map_pages(struct vm_area_struct *vma, struct page **pages, unsigned long num) { - return __vm_map_pages(vma, pages, num, vma->vm_pgoff); + return __vm_map_pages(vma, pages, num, vma_start_pgoff(vma)); } EXPORT_SYMBOL(vm_map_pages); @@ -3318,7 +3318,8 @@ int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long unsigned long pfn; int err; - err = __simple_ioremap_prep(vm_len, vma->vm_pgoff, start, len, &pfn); + err = __simple_ioremap_prep(vm_len, vma_start_pgoff(vma), start, len, + &pfn); if (err) return err; @@ -4366,15 +4367,15 @@ static inline void unmap_mapping_range_tree(struct address_space *mapping, struct zap_details *details) { struct vm_area_struct *vma; - unsigned long start, size; struct mmu_gather tlb; mapping_rmap_tree_foreach(vma, mapping, first_index, last_index) { - const pgoff_t start_idx = max(first_index, vma->vm_pgoff); + const pgoff_t start_idx = max(first_index, vma_start_pgoff(vma)); const pgoff_t end_idx = min(last_index, vma_last_pgoff(vma)) + 1; - - start = vma->vm_start + ((start_idx - vma->vm_pgoff) << PAGE_SHIFT); - size = (end_idx - start_idx) << PAGE_SHIFT; + const pgoff_t offset = start_idx - vma_start_pgoff(vma); + const unsigned long offset_bytes = offset << PAGE_SHIFT; + const unsigned long start = vma->vm_start + offset_bytes; + const unsigned long size = (end_idx - start_idx) << PAGE_SHIFT; tlb_gather_mmu(&tlb, vma->vm_mm); zap_vma_range_batched(&tlb, vma, start, size, details); @@ -5712,7 +5713,7 @@ vm_fault_t finish_fault(struct vm_fault *vmf) } else if (nr_pages > 1) { pgoff_t idx = folio_page_idx(folio, page); /* The page offset of vmf->address within the VMA. */ - pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff; + pgoff_t vma_off = vmf->pgoff - vma_start_pgoff(vmf->vma); /* The index of the entry in the pagetable for fault page. */ pgoff_t pte_off = pte_index(vmf->address); @@ -5824,7 +5825,7 @@ static vm_fault_t do_fault_around(struct vm_fault *vmf) pgoff_t nr_pages = READ_ONCE(fault_around_pages); pgoff_t pte_off = pte_index(vmf->address); /* The page offset of vmf->address within the VMA. */ - pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff; + pgoff_t vma_off = vmf->pgoff - vma_start_pgoff(vmf->vma); pgoff_t from_pte, to_pte; vm_fault_t ret; @@ -7352,7 +7353,7 @@ void print_vma_addr(char *prefix, unsigned long ip) if (vma && vma->vm_file) { struct file *f = vma->vm_file; ip -= vma->vm_start; - ip += vma->vm_pgoff << PAGE_SHIFT; + ip += vma_start_pgoff(vma) << PAGE_SHIFT; printk("%s%pD[%lx,%lx+%lx]", prefix, f, ip, vma->vm_start, vma->vm_end - vma->vm_start); diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 41d630f0ea82..a92bd29cf032 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -2049,8 +2049,8 @@ struct mempolicy *get_vma_policy(struct vm_area_struct *vma, pol = get_task_policy(current); if (pol->mode == MPOL_INTERLEAVE || pol->mode == MPOL_WEIGHTED_INTERLEAVE) { - *ilx += vma->vm_pgoff >> order; - *ilx += (addr - vma->vm_start) >> (PAGE_SHIFT + order); + *ilx += vma_start_pgoff(vma) >> order; + *ilx += linear_page_delta(vma, addr) >> order; } return pol; } @@ -3253,16 +3253,17 @@ EXPORT_SYMBOL_FOR_MODULES(mpol_shared_policy_init, "kvm"); int mpol_set_shared_policy(struct shared_policy *sp, struct vm_area_struct *vma, struct mempolicy *pol) { - int err; + const pgoff_t pgoff = vma_start_pgoff(vma); + const pgoff_t pgoff_end = vma_end_pgoff(vma); struct sp_node *new = NULL; - unsigned long sz = vma_pages(vma); + int err; if (pol) { - new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, pol); + new = sp_alloc(pgoff, pgoff_end, pol); if (!new) return -ENOMEM; } - err = shared_policy_replace(sp, vma->vm_pgoff, vma->vm_pgoff + sz, new); + err = shared_policy_replace(sp, pgoff, pgoff_end, new); if (err && new) sp_free(new); return err; diff --git a/mm/mremap.c b/mm/mremap.c index 8a46ec5831c8..384ef4cc2195 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -957,8 +957,7 @@ static unsigned long vrm_set_new_addr(struct vma_remap_struct *vrm) struct vm_area_struct *vma = vrm->vma; unsigned long map_flags = 0; /* Page Offset _into_ the VMA. */ - pgoff_t internal_pgoff = (vrm->addr - vma->vm_start) >> PAGE_SHIFT; - pgoff_t pgoff = vma->vm_pgoff + internal_pgoff; + const pgoff_t pgoff = linear_page_index(vma, vrm->addr); unsigned long new_addr = vrm_implies_new_addr(vrm) ? vrm->new_addr : 0; unsigned long res; @@ -1264,12 +1263,10 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) static int copy_vma_and_data(struct vma_remap_struct *vrm, struct vm_area_struct **new_vma_ptr) { - unsigned long internal_offset = vrm->addr - vrm->vma->vm_start; - unsigned long internal_pgoff = internal_offset >> PAGE_SHIFT; - unsigned long new_pgoff = vrm->vma->vm_pgoff + internal_pgoff; - unsigned long moved_len; + const unsigned long new_pgoff = linear_page_index(vrm->vma, vrm->addr); struct vm_area_struct *vma = vrm->vma; struct vm_area_struct *new_vma; + unsigned long moved_len; int err = 0; PAGETABLE_MOVE(pmc, NULL, NULL, vrm->addr, vrm->new_addr, vrm->old_len); @@ -1811,8 +1808,7 @@ static int check_prep_vma(struct vma_remap_struct *vrm) vrm->populate_expand = true; /* Need to be careful about a growing mapping */ - pgoff = (addr - vma->vm_start) >> PAGE_SHIFT; - pgoff += vma->vm_pgoff; + pgoff = linear_page_index(vma, addr); if (pgoff + (new_len >> PAGE_SHIFT) < pgoff) return -EINVAL; diff --git a/mm/msync.c b/mm/msync.c index ac4c9bfea2e7..90b491a27a14 100644 --- a/mm/msync.c +++ b/mm/msync.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -85,8 +86,7 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags) goto out_unlock; } file = vma->vm_file; - fstart = (start - vma->vm_start) + - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + fstart = (loff_t)linear_page_index(vma, start) << PAGE_SHIFT; fend = fstart + (min(end, vma->vm_end) - start) - 1; start = vma->vm_end; if ((flags & MS_SYNC) && file && diff --git a/mm/nommu.c b/mm/nommu.c index 355b3b1ab7ff..e392b884d0c8 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -975,7 +975,7 @@ static int do_mmap_private(struct vm_area_struct *vma, /* read the contents of a file into the copy */ loff_t fpos; - fpos = vma->vm_pgoff; + fpos = vma_start_pgoff(vma); fpos <<= PAGE_SHIFT; ret = kernel_read(vma->vm_file, base, len, &fpos); @@ -1378,7 +1378,8 @@ static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, delete_nommu_region(vma->vm_region); if (new_below) { vma->vm_region->vm_start = vma->vm_start = addr; - vma->vm_region->vm_pgoff = vma->vm_pgoff += npages; + vma->vm_pgoff += npages; + vma->vm_region->vm_pgoff = vma_start_pgoff(vma); } else { vma->vm_region->vm_end = vma->vm_end = addr; vma->vm_region->vm_top = addr; @@ -1626,7 +1627,7 @@ int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long unsigned long pfn = start >> PAGE_SHIFT; unsigned long vm_len = vma->vm_end - vma->vm_start; - pfn += vma->vm_pgoff; + pfn += vma_start_pgoff(vma); return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot); } EXPORT_SYMBOL(vm_iomap_memory); diff --git a/mm/pagewalk.c b/mm/pagewalk.c index 7e7819cf5a3a..dd854c734877 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -813,7 +813,7 @@ int walk_page_mapping(struct address_space *mapping, pgoff_t first_index, mapping_rmap_tree_foreach(vma, mapping, first_index, first_index + nr - 1) { /* Clip to the vma */ - vba = vma->vm_pgoff; + vba = vma_start_pgoff(vma); vea = vba + vma_pages(vma); cba = first_index; cba = max(cba, vba); diff --git a/mm/shmem.c b/mm/shmem.c index 0a36b48f667e..f1ab97c34a02 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1042,6 +1042,8 @@ unsigned long shmem_swap_usage(struct vm_area_struct *vma) struct inode *inode = file_inode(vma->vm_file); struct shmem_inode_info *info = SHMEM_I(inode); struct address_space *mapping = inode->i_mapping; + const pgoff_t pgoff = vma_start_pgoff(vma); + const pgoff_t pgoff_end = vma_end_pgoff(vma); unsigned long swapped; /* Be careful as we don't hold info->lock */ @@ -1055,12 +1057,11 @@ unsigned long shmem_swap_usage(struct vm_area_struct *vma) if (!swapped) return 0; - if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size) + if (!pgoff && vma->vm_end - vma->vm_start >= inode->i_size) return swapped << PAGE_SHIFT; /* Here comes the more involved part */ - return shmem_partial_swap_usage(mapping, vma->vm_pgoff, - vma->vm_pgoff + vma_pages(vma)); + return shmem_partial_swap_usage(mapping, pgoff, pgoff_end); } /* @@ -2850,7 +2851,7 @@ static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, * by page order, as in shmem_get_pgoff_policy() and get_vma_policy()). */ *ilx = inode->i_ino; - index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; + index = linear_page_index(vma, addr); return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); } diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 9d0357768bcf..de4cc2483562 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -481,7 +481,7 @@ static void mfill_retry_state_save(struct mfill_retry_state *s, { s->flags = vma_flags_and_mask(&vma->flags, MFILL_RETRY_STATE_VMA_FLAGS); s->ops = vma_uffd_ops(vma); - s->pgoff = vma->vm_pgoff; + s->pgoff = vma_start_pgoff(vma); if (vma->vm_file) s->file = get_file(vma->vm_file); @@ -507,7 +507,7 @@ static bool mfill_retry_state_changed(struct mfill_retry_state *state, /* VMA was file backed, but file, inode or offset has changed */ if (!vma->vm_file || vma->vm_file->f_inode != state->file->f_inode || - state->file != vma->vm_file || vma->vm_pgoff != state->pgoff) + state->file != vma->vm_file || vma_start_pgoff(vma) != state->pgoff) return true; return false; diff --git a/mm/util.c b/mm/util.c index 34cb43b3eaa4..5090457542ca 100644 --- a/mm/util.c +++ b/mm/util.c @@ -1188,7 +1188,7 @@ void compat_set_desc_from_vma(struct vm_area_desc *desc, desc->start = vma->vm_start; desc->end = vma->vm_end; - desc->pgoff = vma->vm_pgoff; + desc->pgoff = vma_start_pgoff(vma); desc->vm_file = vma->vm_file; desc->vma_flags = vma->flags; desc->page_prot = vma->vm_page_prot; @@ -1379,7 +1379,7 @@ static int call_vma_mapped(struct vm_area_struct *vma) if (!vm_ops || !vm_ops->mapped) return 0; - err = vm_ops->mapped(vma->vm_start, vma->vm_end, vma->vm_pgoff, + err = vm_ops->mapped(vma->vm_start, vma->vm_end, vma_start_pgoff(vma), vma->vm_file, &vm_private_data); if (err) return err; diff --git a/mm/vma.c b/mm/vma.c index 5d5e60ea8a25..4a4d096ba86a 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -967,10 +967,9 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( * prev middle next * extend delete delete */ - vmg->start = prev->vm_start; vmg->end = next->vm_end; - vmg->pgoff = prev->vm_pgoff; + vmg->pgoff = vma_start_pgoff(prev); /* * We already ensured anon_vma compatibility above, so now it's @@ -987,9 +986,8 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( * prev middle * extend shrink/delete */ - vmg->start = prev->vm_start; - vmg->pgoff = prev->vm_pgoff; + vmg->pgoff = vma_start_pgoff(prev); if (!vmg->__remove_middle) vmg->__adjust_middle_start = true; @@ -1011,13 +1009,13 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( if (vmg->__remove_middle) { vmg->end = next->vm_end; - vmg->pgoff = next->vm_pgoff - pglen; + vmg->pgoff = vma_start_pgoff(next) - pglen; } else { /* We shrink middle and expand next. */ vmg->__adjust_next_start = true; vmg->start = middle->vm_start; vmg->end = start; - vmg->pgoff = middle->vm_pgoff; + vmg->pgoff = vma_start_pgoff(middle); } err = dup_anon_vma(next, middle, &anon_dup); @@ -1126,7 +1124,7 @@ struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg) if (can_merge_left) { vmg->start = prev->vm_start; vmg->target = prev; - vmg->pgoff = prev->vm_pgoff; + vmg->pgoff = vma_start_pgoff(prev); /* * If this merge would result in removal of the next VMA but we @@ -1957,7 +1955,8 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma); *vmap = vma = new_vma; } - *need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff); + *need_rmap_locks = + (vma_start_pgoff(new_vma) <= vma_start_pgoff(vma)); } else { new_vma = vm_area_dup(vma); if (!new_vma) diff --git a/mm/vma_exec.c b/mm/vma_exec.c index 5cee8b7efa0f..e3644a3042e2 100644 --- a/mm/vma_exec.c +++ b/mm/vma_exec.c @@ -37,7 +37,7 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) unsigned long new_end = old_end - shift; VMA_ITERATOR(vmi, mm, new_start); VMG_STATE(vmg, mm, &vmi, new_start, old_end, EMPTY_VMA_FLAGS, - vma->vm_pgoff); + vma_start_pgoff(vma)); struct vm_area_struct *next; struct mmu_gather tlb; PAGETABLE_MOVE(pmc, vma, vma, old_start, new_start, length); @@ -89,7 +89,7 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) vma_prev(&vmi); /* Shrink the vma to just the new range */ - return vma_shrink(&vmi, vma, new_start, new_end, vma->vm_pgoff); + return vma_shrink(&vmi, vma, new_start, new_end, vma_start_pgoff(vma)); } /* diff --git a/mm/vma_init.c b/mm/vma_init.c index 3c0b65950510..a459669a1654 100644 --- a/mm/vma_init.c +++ b/mm/vma_init.c @@ -46,7 +46,7 @@ static void vm_area_init_from(const struct vm_area_struct *src, dest->vm_start = src->vm_start; dest->vm_end = src->vm_end; dest->anon_vma = src->anon_vma; - dest->vm_pgoff = src->vm_pgoff; + dest->vm_pgoff = vma_start_pgoff(src); dest->vm_file = src->vm_file; dest->vm_private_data = src->vm_private_data; vm_flags_init(dest, src->vm_flags); From 4276358f11a020dcc9686a3ac1843fafd79376f6 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:17:00 +0100 Subject: [PATCH 349/501] mm: prefer vma_[start,end]_pgoff() to vma->vm_pgoff in kernel/ Be consistent in using vma_start_pgoff() and vma_end_pgoff(), which clearly indicates which part of the VMA the page offset refers to and aids greppability. This is part of a broader series laying the ground to provide a virtual page offset for MAP_PRIVATE-file backed anon folios. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-19-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Acked-by: Marek Szyprowski # for kernel/dma Reviewed-by: Gregory Price Acked-by: Pedro Falcato Reviewed-by: Vlastimil Babka (SUSE) Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Cc: Zi Yan Signed-off-by: Andrew Morton --- kernel/dma/coherent.c | 7 ++++--- kernel/dma/direct.c | 6 ++++-- kernel/dma/mapping.c | 8 +++++--- kernel/dma/ops_helpers.c | 4 ++-- kernel/events/core.c | 20 +++++++++++--------- kernel/events/uprobes.c | 11 +++++++---- kernel/kcov.c | 2 +- kernel/trace/ring_buffer.c | 3 ++- 8 files changed, 36 insertions(+), 25 deletions(-) diff --git a/kernel/dma/coherent.c b/kernel/dma/coherent.c index bcdc0f76d2e8..2d3195eb7e83 100644 --- a/kernel/dma/coherent.c +++ b/kernel/dma/coherent.c @@ -236,14 +236,15 @@ static int __dma_mmap_from_coherent(struct dma_coherent_mem *mem, { if (mem && vaddr >= mem->virt_base && vaddr + size <= (mem->virt_base + ((dma_addr_t)mem->size << PAGE_SHIFT))) { - unsigned long off = vma->vm_pgoff; + const pgoff_t pgoff_start = vma_start_pgoff(vma); + const pgoff_t pgoff_end = vma_end_pgoff(vma); int start = (vaddr - mem->virt_base) >> PAGE_SHIFT; unsigned long user_count = vma_pages(vma); int count = PAGE_ALIGN(size) >> PAGE_SHIFT; *ret = -ENXIO; - if (off < count && user_count <= count - off) { - unsigned long pfn = mem->pfn_base + start + off; + if (pgoff_start < count && pgoff_end <= count) { + unsigned long pfn = mem->pfn_base + start + pgoff_start; *ret = remap_pfn_range(vma, vma->vm_start, pfn, user_count << PAGE_SHIFT, vma->vm_page_prot); diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 4391b797d4db..436310d6e4a2 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -534,6 +534,8 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma, unsigned long user_count = vma_pages(vma); unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; unsigned long pfn = PHYS_PFN(dma_to_phys(dev, dma_addr)); + const pgoff_t pgoff_start = vma_start_pgoff(vma); + const pgoff_t pgoff_end = vma_end_pgoff(vma); int ret = -ENXIO; vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs); @@ -545,9 +547,9 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma, if (dma_mmap_from_global_coherent(vma, cpu_addr, size, &ret)) return ret; - if (vma->vm_pgoff >= count || user_count > count - vma->vm_pgoff) + if (pgoff_start >= count || pgoff_end > count) return -ENXIO; - return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff, + return remap_pfn_range(vma, vma->vm_start, pfn + pgoff_start, user_count << PAGE_SHIFT, vma->vm_page_prot); } diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index 4fe04669e5e6..c986639044e9 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -761,12 +761,14 @@ EXPORT_SYMBOL_GPL(dma_free_pages); int dma_mmap_pages(struct device *dev, struct vm_area_struct *vma, size_t size, struct page *page) { - unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; + const pgoff_t pgoff_start = vma_start_pgoff(vma); + const pgoff_t pgoff_end = vma_end_pgoff(vma); + const unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; - if (vma->vm_pgoff >= count || vma_pages(vma) > count - vma->vm_pgoff) + if (pgoff_start >= count || pgoff_end > count) return -ENXIO; return remap_pfn_range(vma, vma->vm_start, - page_to_pfn(page) + vma->vm_pgoff, + page_to_pfn(page) + pgoff_start, vma_pages(vma) << PAGE_SHIFT, vma->vm_page_prot); } EXPORT_SYMBOL_GPL(dma_mmap_pages); diff --git a/kernel/dma/ops_helpers.c b/kernel/dma/ops_helpers.c index 20caf9cabf69..6b5f9208d31c 100644 --- a/kernel/dma/ops_helpers.c +++ b/kernel/dma/ops_helpers.c @@ -39,7 +39,7 @@ int dma_common_mmap(struct device *dev, struct vm_area_struct *vma, #ifdef CONFIG_MMU unsigned long user_count = vma_pages(vma); unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; - unsigned long off = vma->vm_pgoff; + unsigned long off = vma_start_pgoff(vma); struct page *page = dma_common_vaddr_to_page(cpu_addr); int ret = -ENXIO; @@ -52,7 +52,7 @@ int dma_common_mmap(struct device *dev, struct vm_area_struct *vma, return -ENXIO; return remap_pfn_range(vma, vma->vm_start, - page_to_pfn(page) + vma->vm_pgoff, + page_to_pfn(page) + vma_start_pgoff(vma), user_count << PAGE_SHIFT, vma->vm_page_prot); #else return -ENXIO; diff --git a/kernel/events/core.c b/kernel/events/core.c index ba5bd6a78fe7..cf78e892a4bb 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -6998,7 +6998,7 @@ static void perf_mmap_open(struct vm_area_struct *vma) refcount_inc(&event->mmap_count); refcount_inc(&event->rb->mmap_count); - if (vma->vm_pgoff) + if (vma_start_pgoff(vma)) refcount_inc(&event->rb->aux_mmap_count); if (mapped) @@ -7032,7 +7032,7 @@ static void perf_mmap_close(struct vm_area_struct *vma) * The AUX buffer is strictly a sub-buffer, serialize using aux_mutex * to avoid complications. */ - if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff && + if (rb_has_aux(rb) && vma_start_pgoff(vma) == rb->aux_pgoff && refcount_dec_and_mutex_lock(&rb->aux_mmap_count, &rb->aux_mutex)) { /* * Stop all AUX events that are writing to this buffer, @@ -7192,7 +7192,8 @@ static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma) */ for (pagenum = 0; pagenum < nr_pages; pagenum++) { unsigned long va = vma->vm_start + PAGE_SIZE * pagenum; - struct page *page = perf_mmap_to_page(rb, vma->vm_pgoff + pagenum); + struct page *page = perf_mmap_to_page(rb, + vma_start_pgoff(vma) + pagenum); if (page == NULL) { err = -EINVAL; @@ -7346,6 +7347,7 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, unsigned long nr_pages) { + const pgoff_t pgoff_start = vma_start_pgoff(vma); long extra = 0, user_extra = nr_pages; u64 aux_offset, aux_size; struct perf_buffer *rb; @@ -7368,11 +7370,11 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, if (aux_offset < perf_data_size(rb) + PAGE_SIZE) return -EINVAL; - if (aux_offset != vma->vm_pgoff << PAGE_SHIFT) + if (aux_offset != pgoff_start << PAGE_SHIFT) return -EINVAL; /* already mapped with a different offset */ - if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff) + if (rb_has_aux(rb) && rb->aux_pgoff != pgoff_start) return -EINVAL; if (aux_size != nr_pages * PAGE_SIZE) @@ -7402,7 +7404,7 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, if (vma->vm_flags & VM_WRITE) rb_flags |= RING_BUFFER_WRITABLE; - ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages, + ret = rb_alloc_aux(rb, event, pgoff_start, nr_pages, event->attr.aux_watermark, rb_flags); if (ret) { refcount_dec(&rb->mmap_count); @@ -7459,7 +7461,7 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) if (event->state <= PERF_EVENT_STATE_REVOKED) return -ENODEV; - if (vma->vm_pgoff == 0) + if (!vma_start_pgoff(vma)) ret = perf_mmap_rb(vma, event, nr_pages); else ret = perf_mmap_aux(vma, event, nr_pages); @@ -9886,7 +9888,7 @@ static bool perf_addr_filter_vma_adjust(struct perf_addr_filter *filter, struct perf_addr_filter_range *fr) { unsigned long vma_size = vma->vm_end - vma->vm_start; - unsigned long off = vma->vm_pgoff << PAGE_SHIFT; + unsigned long off = vma_start_pgoff(vma) << PAGE_SHIFT; struct file *file = vma->vm_file; if (!perf_addr_filter_match(filter, file, off, vma_size)) @@ -9976,7 +9978,7 @@ void perf_event_mmap(struct vm_area_struct *vma) /* .tid */ .start = vma->vm_start, .len = vma->vm_end - vma->vm_start, - .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT, + .pgoff = (u64)vma_start_pgoff(vma) << PAGE_SHIFT, }, /* .maj (attr_mmap2 only) */ /* .min (attr_mmap2 only) */ diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index ac340a703d06..ae2f3b9f8d50 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -144,12 +144,14 @@ static bool valid_vma(struct vm_area_struct *vma, bool is_register) static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset) { - return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + return vma->vm_start + offset - + ((loff_t)vma_start_pgoff(vma) << PAGE_SHIFT); } static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr) { - return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start); + return ((loff_t)vma_start_pgoff(vma) << PAGE_SHIFT) + + (vaddr - vma->vm_start); } /** @@ -1482,7 +1484,7 @@ static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) file_inode(vma->vm_file) != uprobe->inode) continue; - offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT; + offset = (loff_t)vma_start_pgoff(vma) << PAGE_SHIFT; if (uprobe->offset < offset || uprobe->offset >= offset + vma->vm_end - vma->vm_start) continue; @@ -2453,7 +2455,8 @@ static struct uprobe *find_active_uprobe_speculative(unsigned long bp_vaddr) if (!vm_file) return NULL; - offset = (loff_t)(vma->vm_pgoff << PAGE_SHIFT) + (bp_vaddr - vma->vm_start); + offset = (loff_t)(vma_start_pgoff(vma) << PAGE_SHIFT) + + (bp_vaddr - vma->vm_start); uprobe = find_uprobe_rcu(vm_file->f_inode, offset); if (!uprobe) return NULL; diff --git a/kernel/kcov.c b/kernel/kcov.c index 1df373fb562b..b19b473c366a 100644 --- a/kernel/kcov.c +++ b/kernel/kcov.c @@ -512,7 +512,7 @@ static int kcov_mmap(struct file *filep, struct vm_area_struct *vma) spin_lock_irqsave(&kcov->lock, flags); size = kcov->size * sizeof(unsigned long); - if (kcov->area == NULL || vma->vm_pgoff != 0 || + if (kcov->area == NULL || vma_start_pgoff(vma) || vma->vm_end - vma->vm_start != size) { res = -EINVAL; goto exit; diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 804ccae694d2..e863eb0d10f7 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -7611,7 +7611,8 @@ static int __rb_inc_dec_mapped(struct ring_buffer_per_cpu *cpu_buffer, static int __rb_map_vma(struct ring_buffer_per_cpu *cpu_buffer, struct vm_area_struct *vma) { - unsigned long nr_subbufs, nr_pages, nr_vma_pages, pgoff = vma->vm_pgoff; + unsigned long nr_subbufs, nr_pages, nr_vma_pages; + pgoff_t pgoff = vma_start_pgoff(vma); unsigned int subbuf_pages, subbuf_order; struct page **pages __free(kfree) = NULL; int p = 0, s = 0; From ff8fbc9e08739a1a754293f5aba7796be3649035 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:17:01 +0100 Subject: [PATCH 350/501] mm/vma: remove duplicative vma_pgoff_offset() helper This is doing what linear_page_index() does, so eliminate it and replace it with linear_page_index(). Update the VMA userland tests to reflect this change. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-20-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Pedro Falcato Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/vma.h | 9 +-------- tools/testing/vma/include/dup.h | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/mm/vma.h b/mm/vma.h index 527716c8739d..2342516ce00e 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -247,13 +247,6 @@ static inline pgoff_t vmg_end_pgoff(const struct vma_merge_struct *vmg) return vmg_start_pgoff(vmg) + vmg_pages(vmg); } -/* Assumes addr >= vma->vm_start. */ -static inline pgoff_t vma_pgoff_offset(struct vm_area_struct *vma, - unsigned long addr) -{ - return vma->vm_pgoff + PHYS_PFN(addr - vma->vm_start); -} - #define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_) \ struct vma_merge_struct name = { \ .mm = mm_, \ @@ -275,7 +268,7 @@ static inline pgoff_t vma_pgoff_offset(struct vm_area_struct *vma, .start = start_, \ .end = end_, \ .vm_flags = vma_->vm_flags, \ - .pgoff = vma_pgoff_offset(vma_, start_), \ + .pgoff = linear_page_index(vma_, start_), \ .file = vma_->vm_file, \ .anon_vma = vma_->anon_vma, \ .policy = vma_policy(vma_), \ diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index 535747d7fee4..7ed165c8d9bc 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -1548,3 +1548,19 @@ static inline pgprot_t vma_get_page_prot(vma_flags_t vma_flags) return vm_get_page_prot(vm_flags); } + +static inline pgoff_t linear_page_delta(const struct vm_area_struct *vma, + const unsigned long address) +{ + return (address - vma->vm_start) >> PAGE_SHIFT; +} + +static inline pgoff_t linear_page_index(const struct vm_area_struct *vma, + const unsigned long address) +{ + pgoff_t pgoff; + + pgoff = linear_page_delta(vma, address); + pgoff += vma_start_pgoff(vma); + return pgoff; +} From 93d23dff8eb5045a9cb54bf6202324da0c7010cb Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:17:02 +0100 Subject: [PATCH 351/501] mm: use linear_page_[index, delta]() consistently There are a number of places where we open code what linear_page_index() and linear_page_delta() calculate. Replace this code with the appropriate functions for consistency. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-21-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Thomas Zimmermann [DRM] Acked-by: Kai Huang # for sgx Reviewed-by: Gregory Price Reviewed-by: Pedro Falcato # for mm Reviewed-by: Ackerley Tng [guest_memfd] Reviewed-by: Vlastimil Babka (SUSE) Cc: David Hildenbrand (Arm) Cc: Marek Szyprowski Cc: SJ Park Cc: Liam R. Howlett (Oracle) Cc: Zi Yan Signed-off-by: Andrew Morton --- arch/arm/mm/fault-armv.c | 2 +- arch/x86/kernel/cpu/sgx/virt.c | 3 ++- drivers/gpu/drm/etnaviv/etnaviv_gem.c | 3 ++- drivers/gpu/drm/gma500/gem.c | 2 +- drivers/gpu/drm/msm/msm_gem.c | 3 ++- drivers/gpu/drm/omapdrm/omap_gem.c | 5 +++-- drivers/gpu/drm/tegra/gem.c | 3 ++- drivers/gpu/drm/ttm/ttm_bo_vm.c | 7 ++++--- drivers/vfio/pci/nvgrace-gpu/main.c | 3 ++- drivers/vfio/pci/vfio_pci_core.c | 3 ++- mm/nommu.c | 2 +- mm/vma.c | 2 +- virt/kvm/guest_memfd.c | 2 +- 13 files changed, 24 insertions(+), 16 deletions(-) diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index b505f1fd7984..2769b0f3325f 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -132,7 +132,7 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, pgoff_t pgoff; int aliases = 0; - pgoff = vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT); + pgoff = linear_page_index(vma, addr); /* * If we have any shared mappings that are in the same mm diff --git a/arch/x86/kernel/cpu/sgx/virt.c b/arch/x86/kernel/cpu/sgx/virt.c index db6806c40483..6a1933ddc6fc 100644 --- a/arch/x86/kernel/cpu/sgx/virt.c +++ b/arch/x86/kernel/cpu/sgx/virt.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -41,7 +42,7 @@ static int __sgx_vepc_fault(struct sgx_vepc *vepc, WARN_ON(!mutex_is_locked(&vepc->lock)); /* Calculate index of EPC page in virtual EPC's page_array */ - index = vma->vm_pgoff + PFN_DOWN(addr - vma->vm_start); + index = linear_page_index(vma, addr); epc_page = xa_load(&vepc->page_array, index); if (epc_page) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c index b0436a1e103f..2e4d6d117ee2 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -188,7 +189,7 @@ static vm_fault_t etnaviv_gem_fault(struct vm_fault *vmf) } /* We don't use vmf->pgoff since that has the fake offset: */ - pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + pgoff = linear_page_delta(vma, vmf->address); pfn = page_to_pfn(pages[pgoff]); diff --git a/drivers/gpu/drm/gma500/gem.c b/drivers/gpu/drm/gma500/gem.c index 88f1e86c8903..2708e8c68f4c 100644 --- a/drivers/gpu/drm/gma500/gem.c +++ b/drivers/gpu/drm/gma500/gem.c @@ -288,7 +288,7 @@ static vm_fault_t psb_gem_fault(struct vm_fault *vmf) /* Page relative to the VMA start - we must calculate this ourselves because vmf->pgoff is the fake GEM offset */ - page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + page_offset = linear_page_delta(vma, vmf->address); /* CPU view of the page, don't go via the GART for CPU writes */ if (pobj->stolen) diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c index efd3d3c9a449..cbf723a5d86f 100644 --- a/drivers/gpu/drm/msm/msm_gem.c +++ b/drivers/gpu/drm/msm/msm_gem.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -360,7 +361,7 @@ static vm_fault_t msm_gem_fault(struct vm_fault *vmf) } /* We don't use vmf->pgoff since that has the fake offset: */ - pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + pgoff = linear_page_delta(vma, vmf->address); pfn = page_to_pfn(pages[pgoff]); diff --git a/drivers/gpu/drm/omapdrm/omap_gem.c b/drivers/gpu/drm/omapdrm/omap_gem.c index 8e013e4f2c6b..00404fb6c29a 100644 --- a/drivers/gpu/drm/omapdrm/omap_gem.c +++ b/drivers/gpu/drm/omapdrm/omap_gem.c @@ -5,6 +5,7 @@ */ #include +#include #include #include #include @@ -359,7 +360,7 @@ static vm_fault_t omap_gem_fault_1d(struct drm_gem_object *obj, pgoff_t pgoff; /* We don't use vmf->pgoff since that has the fake offset: */ - pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + pgoff = linear_page_delta(vma, vmf->address); if (omap_obj->pages) { omap_gem_cpu_sync_page(obj, pgoff); @@ -407,7 +408,7 @@ static vm_fault_t omap_gem_fault_2d(struct drm_gem_object *obj, const int m = DIV_ROUND_UP(omap_obj->width << fmt, PAGE_SIZE); /* We don't use vmf->pgoff since that has the fake offset: */ - pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + pgoff = linear_page_delta(vma, vmf->address); /* * Actual address we start mapping at is rounded down to previous slot diff --git a/drivers/gpu/drm/tegra/gem.c b/drivers/gpu/drm/tegra/gem.c index 436394e04812..1d8d27a5ea89 100644 --- a/drivers/gpu/drm/tegra/gem.c +++ b/drivers/gpu/drm/tegra/gem.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -564,7 +565,7 @@ static vm_fault_t tegra_bo_fault(struct vm_fault *vmf) if (!bo->pages) return VM_FAULT_SIGBUS; - offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + offset = linear_page_delta(vma, vmf->address); page = bo->pages[offset]; return vmf_insert_page(vma, vmf->address, page); diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index a80510489c45..88babf435ac2 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -32,6 +32,7 @@ #define pr_fmt(fmt) "[TTM] " fmt #include +#include #include #include @@ -208,9 +209,9 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, if (unlikely(err != 0)) return VM_FAULT_SIGBUS; - page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) + - vma->vm_pgoff - drm_vma_node_start(&bo->base.vma_node); - page_last = vma_pages(vma) + vma->vm_pgoff - + page_offset = linear_page_index(vma, address) - + drm_vma_node_start(&bo->base.vma_node); + page_last = vma_end_pgoff(vma) - drm_vma_node_start(&bo->base.vma_node); if (unlikely(page_offset >= PFN_UP(bo->base.size))) diff --git a/drivers/vfio/pci/nvgrace-gpu/main.c b/drivers/vfio/pci/nvgrace-gpu/main.c index d07dcacb76bd..963fd8ded20d 100644 --- a/drivers/vfio/pci/nvgrace-gpu/main.c +++ b/drivers/vfio/pci/nvgrace-gpu/main.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -385,7 +386,7 @@ static unsigned long addr_to_pgoff(struct vm_area_struct *vma, u64 pgoff = vma->vm_pgoff & ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); - return ((addr - vma->vm_start) >> PAGE_SHIFT) + pgoff; + return linear_page_delta(vma, addr) + pgoff; } static vm_fault_t nvgrace_gpu_vfio_pci_huge_fault(struct vm_fault *vmf, diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 3f11a9624b9c..a113c55845e1 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -1780,7 +1781,7 @@ static vm_fault_t vfio_pci_mmap_huge_fault(struct vm_fault *vmf, struct vm_area_struct *vma = vmf->vma; struct vfio_pci_core_device *vdev = vma->vm_private_data; unsigned long addr = vmf->address & ~((PAGE_SIZE << order) - 1); - unsigned long pgoff = (addr - vma->vm_start) >> PAGE_SHIFT; + unsigned long pgoff = linear_page_delta(vma, addr); unsigned long pfn = vma_to_pfn(vma) + pgoff; vm_fault_t ret = VM_FAULT_FALLBACK; diff --git a/mm/nommu.c b/mm/nommu.c index e392b884d0c8..81caf5318aaf 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -1355,7 +1355,7 @@ static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, *region = *vma->vm_region; new->vm_region = region; - npages = (addr - vma->vm_start) >> PAGE_SHIFT; + npages = linear_page_delta(vma, addr); if (new_below) { region->vm_top = region->vm_end = new->vm_end = addr; diff --git a/mm/vma.c b/mm/vma.c index 4a4d096ba86a..7aa0149f076c 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -517,7 +517,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, new->vm_end = addr; } else { new->vm_start = addr; - new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT); + new->vm_pgoff += linear_page_delta(vma, addr); } err = -ENOMEM; diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index db57c5766ab6..f0e5da490866 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -440,7 +440,7 @@ static int kvm_gmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpo static struct mempolicy *kvm_gmem_get_policy(struct vm_area_struct *vma, unsigned long addr, pgoff_t *ilx) { - pgoff_t pgoff = vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT); + pgoff_t pgoff = linear_page_index(vma, addr); struct inode *inode = file_inode(vma->vm_file); *ilx = inode->i_ino; From e65f4dfac104d078862357eee5e602e1754de3f9 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:17:03 +0100 Subject: [PATCH 352/501] mm/vma: introduce vma_assert_can_modify() vma_assert_write_locked() and vma_assert_attached() are useful for their own purposes, however VMA code absolutely does allow the modification of non-write locked VMAs if they are at that point detached (i.e. unreachable from anywhere). It's therefore useful to be able to assert that a VMA is either detached (modification doesn't matter) or write locked (you're explicitly locked for modification). Therefore introduce vma_assert_can_modify() for this purpose. While we're here, make vma_is_attached() available generally - if !CONFIG_PER_VMA_LOCK, then there's no sense in which a VMA is detached (vma_mark_detached() is a noop), so have this default to true in this case. Also update VMA userland tests to reflect this change, correcting the previously open-coded vma_assert_[attached,detached]() there. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-22-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: Pedro Falcato Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/mmap_lock.h | 8 ++++++++ tools/testing/vma/include/dup.h | 15 +++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h index 04b8f61ece5d..d513286d8160 100644 --- a/include/linux/mmap_lock.h +++ b/include/linux/mmap_lock.h @@ -506,6 +506,8 @@ static inline __must_check int vma_start_write_killable(struct vm_area_struct *vma) { return 0; } static inline void vma_assert_write_locked(struct vm_area_struct *vma) { mmap_assert_write_locked(vma->vm_mm); } +static inline bool vma_is_attached(struct vm_area_struct *vma) + { return true; } static inline void vma_assert_attached(struct vm_area_struct *vma) {} static inline void vma_assert_detached(struct vm_area_struct *vma) {} static inline void vma_mark_attached(struct vm_area_struct *vma) {} @@ -530,6 +532,12 @@ static inline void vma_assert_stabilised(struct vm_area_struct *vma) #endif /* CONFIG_PER_VMA_LOCK */ +static inline void vma_assert_can_modify(struct vm_area_struct *vma) +{ + if (vma_is_attached(vma)) + vma_assert_write_locked(vma); +} + static inline void mmap_write_lock(struct mm_struct *mm) { __mmap_lock_trace_start_locking(mm, true); diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index 7ed165c8d9bc..e9ddc818f2c3 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -1163,6 +1163,11 @@ static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi) return mas_find(&vmi->mas, ULONG_MAX); } +static inline bool vma_is_attached(struct vm_area_struct *vma) +{ + return refcount_read(&vma->vm_refcnt); +} + /* * WARNING: to avoid racing with vma_mark_attached()/vma_mark_detached(), these * assertions should be made either under mmap_write_lock or when the object @@ -1170,12 +1175,12 @@ static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi) */ static inline void vma_assert_attached(struct vm_area_struct *vma) { - WARN_ON_ONCE(!refcount_read(&vma->vm_refcnt)); + WARN_ON_ONCE(!vma_is_attached(vma)); } static inline void vma_assert_detached(struct vm_area_struct *vma) { - WARN_ON_ONCE(refcount_read(&vma->vm_refcnt)); + WARN_ON_ONCE(vma_is_attached(vma)); } static inline void vma_assert_write_locked(struct vm_area_struct *); @@ -1564,3 +1569,9 @@ static inline pgoff_t linear_page_index(const struct vm_area_struct *vma, pgoff += vma_start_pgoff(vma); return pgoff; } + +static inline void vma_assert_can_modify(struct vm_area_struct *vma) +{ + if (vma_is_attached(vma)) + vma_assert_write_locked(vma); +} From 4ea24922ae3a1d48f97765c45dd004ffe312b52e Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:17:04 +0100 Subject: [PATCH 353/501] mm/vma: add and use vma_[add/sub]_pgoff() Add helpers for adding or subtracting to a VMA's page offset, exposed internally for VMA users within mm in mm/vma.h. This is to lay the foundations for tracking anonymous page offset for MAP_PRIVATE file-backed mappings, where adding and subtracting from this value must be reflected in both the file and anonymous offsets. These are used on VMA split and downward stack expansion. No functional change intended. [akpm@linux-foundation.org: use linear_page_delta() in __split_vma(), per Vlastimal & Lorenzo] Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-23-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Pedro Falcato Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/nommu.c | 6 ++++-- mm/vma.c | 6 +++--- mm/vma.h | 12 ++++++++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/mm/nommu.c b/mm/nommu.c index 81caf5318aaf..dc64834e6834 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -41,6 +41,7 @@ #include #include #include "internal.h" +#include "vma.h" unsigned long highest_memmap_pfn; int heap_stack_gap = 0; @@ -1361,7 +1362,8 @@ static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, region->vm_top = region->vm_end = new->vm_end = addr; } else { region->vm_start = new->vm_start = addr; - region->vm_pgoff = new->vm_pgoff += npages; + vma_add_pgoff(new, npages); + region->vm_pgoff = vma_start_pgoff(new); } vma_iter_config(vmi, new->vm_start, new->vm_end); @@ -1378,7 +1380,7 @@ static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, delete_nommu_region(vma->vm_region); if (new_below) { vma->vm_region->vm_start = vma->vm_start = addr; - vma->vm_pgoff += npages; + vma_add_pgoff(vma, npages); vma->vm_region->vm_pgoff = vma_start_pgoff(vma); } else { vma->vm_region->vm_end = vma->vm_end = addr; diff --git a/mm/vma.c b/mm/vma.c index 7aa0149f076c..6e62642e00a2 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -517,7 +517,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, new->vm_end = addr; } else { new->vm_start = addr; - new->vm_pgoff += linear_page_delta(vma, addr); + vma_add_pgoff(new, linear_page_delta(vma, addr)); } err = -ENOMEM; @@ -556,7 +556,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, if (new_below) { vma->vm_start = addr; - vma->vm_pgoff += (addr - new->vm_start) >> PAGE_SHIFT; + vma_add_pgoff(vma, linear_page_delta(new, addr)); } else { vma->vm_end = addr; } @@ -3305,7 +3305,7 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address) vm_stat_account(mm, vma->vm_flags, grow); anon_rmap_tree_pre_update_vma(vma); vma->vm_start = address; - vma->vm_pgoff -= grow; + vma_sub_pgoff(vma, grow); /* Overwrite old entry in mtree. */ vma_iter_store_overwrite(&vmi, vma); anon_rmap_tree_post_update_vma(vma); diff --git a/mm/vma.h b/mm/vma.h index 2342516ce00e..47fe35e5307e 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -247,6 +247,18 @@ static inline pgoff_t vmg_end_pgoff(const struct vma_merge_struct *vmg) return vmg_start_pgoff(vmg) + vmg_pages(vmg); } +static inline void vma_add_pgoff(struct vm_area_struct *vma, pgoff_t delta) +{ + vma_assert_can_modify(vma); + vma->vm_pgoff += delta; +} + +static inline void vma_sub_pgoff(struct vm_area_struct *vma, pgoff_t delta) +{ + vma_assert_can_modify(vma); + vma->vm_pgoff -= delta; +} + #define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_) \ struct vma_merge_struct name = { \ .mm = mm_, \ From 7578fc092a25240f21251b991cc35de131292a14 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:17:05 +0100 Subject: [PATCH 354/501] mm/vma: move __install_special_mapping() to vma.c This function is operating on VMAs and rightly belongs in vma.c, where it can be subject to VMA userland testing and allows us to isolate it from the rest of mm. The _install_special_mapping() function will remain in mmap.c as a wrapper, since this is used by architecture-specific code. Doing so allows us to isolate more functions in vma.c for the same reasons. This forms part of work to allow for tracking MAP_PRIVATE file-backed mappings by their anonymous virtual page offset, as doing so allows us to isolate and keep code that interacts with this together. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-24-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Pedro Falcato Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/mmap.c | 38 -------------------------------------- mm/vma.c | 38 ++++++++++++++++++++++++++++++++++++++ mm/vma.h | 5 +++++ 3 files changed, 43 insertions(+), 38 deletions(-) diff --git a/mm/mmap.c b/mm/mmap.c index 91eaa0c9e7ee..308a43eabd83 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1447,44 +1447,6 @@ static vm_fault_t special_mapping_fault(struct vm_fault *vmf) return VM_FAULT_SIGBUS; } -static struct vm_area_struct *__install_special_mapping( - struct mm_struct *mm, - unsigned long addr, unsigned long len, - vm_flags_t vm_flags, void *priv, - const struct vm_operations_struct *ops) -{ - int ret; - struct vm_area_struct *vma; - - vma = vm_area_alloc(mm); - if (unlikely(vma == NULL)) - return ERR_PTR(-ENOMEM); - - vma_set_range(vma, addr, addr + len, 0); - vm_flags |= mm->def_flags | VM_DONTEXPAND; - if (pgtable_supports_soft_dirty()) - vm_flags |= VM_SOFTDIRTY; - vm_flags_init(vma, vm_flags & ~VM_LOCKED_MASK); - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); - - vma->vm_ops = ops; - vma->vm_private_data = priv; - - ret = insert_vm_struct(mm, vma); - if (ret) - goto out; - - vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT); - - perf_event_mmap(vma); - - return vma; - -out: - vm_area_free(vma); - return ERR_PTR(ret); -} - bool vma_is_special_mapping(const struct vm_area_struct *vma, const struct vm_special_mapping *sm) { diff --git a/mm/vma.c b/mm/vma.c index 6e62642e00a2..c591eb410674 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -3399,3 +3399,41 @@ __weak unsigned long vma_mmu_pagesize(struct vm_area_struct *vma) { return vma_kernel_pagesize(vma); } + +struct vm_area_struct *__install_special_mapping( + struct mm_struct *mm, + unsigned long addr, unsigned long len, + vm_flags_t vm_flags, void *priv, + const struct vm_operations_struct *ops) +{ + int ret; + struct vm_area_struct *vma; + + vma = vm_area_alloc(mm); + if (unlikely(vma == NULL)) + return ERR_PTR(-ENOMEM); + + vma_set_range(vma, addr, addr + len, 0); + vm_flags |= mm->def_flags | VM_DONTEXPAND; + if (pgtable_supports_soft_dirty()) + vm_flags |= VM_SOFTDIRTY; + vm_flags_init(vma, vm_flags & ~VM_LOCKED_MASK); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + + vma->vm_ops = ops; + vma->vm_private_data = priv; + + ret = insert_vm_struct(mm, vma); + if (ret) + goto out; + + vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT); + + perf_event_mmap(vma); + + return vma; + +out: + vm_area_free(vma); + return ERR_PTR(ret); +} diff --git a/mm/vma.h b/mm/vma.h index 47fe35e5307e..14f026bf3be4 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -775,4 +775,9 @@ static inline bool map_deny_write_exec(const vma_flags_t *old, } #endif +struct vm_area_struct *__install_special_mapping(struct mm_struct *mm, + unsigned long addr, unsigned long len, + vm_flags_t vm_flags, void *priv, + const struct vm_operations_struct *ops); + #endif /* __MM_VMA_H */ From 775659f8fc144007165ffa38aadd4deab3041170 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:17:06 +0100 Subject: [PATCH 355/501] mm/vma: make vma_set_range() static, drop insert_vm_struct() decl With __install_special_mapping() moved to vma.c, vma_set_range() can be made into a static function there and is now completely isolated from the rest of mm. While we're here, we can also remove the insert_vm_struct() declaration from mm.h - the function is implemented in vma.c and already declared in vma.h, and has no users outside of mm. Also update the VMA userland tests to reflect this change. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-25-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Pedro Falcato Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/mm.h | 1 - mm/internal.h | 9 --------- mm/vma.c | 8 ++++++++ tools/testing/vma/shared.c | 9 --------- tools/testing/vma/shared.h | 5 ----- 5 files changed, 8 insertions(+), 24 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 498692da306c..32bb723ffbb9 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4134,7 +4134,6 @@ void anon_rmap_tree_verify(struct anon_vma_chain *avc); /* mmap.c */ extern int __vm_enough_memory(const struct mm_struct *mm, long pages, int cap_sys_admin); -extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *); extern void exit_mmap(struct mm_struct *); bool mmap_read_lock_maybe_expand(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, bool write); diff --git a/mm/internal.h b/mm/internal.h index f950e88fdb52..874be94cf257 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1360,15 +1360,6 @@ static inline bool gup_must_unshare(struct vm_area_struct *vma, } -static __always_inline void vma_set_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end, - pgoff_t pgoff) -{ - vma->vm_start = start; - vma->vm_end = end; - vma->vm_pgoff = pgoff; -} - static inline bool vma_soft_dirty_enabled(struct vm_area_struct *vma) { /* diff --git a/mm/vma.c b/mm/vma.c index c591eb410674..a3500a53a7c4 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -70,6 +70,14 @@ struct mmap_state { .state = VMA_MERGE_START, \ } +static void vma_set_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end, pgoff_t pgoff) +{ + vma->vm_start = start; + vma->vm_end = end; + vma->vm_pgoff = pgoff; +} + /* Was this VMA ever forked from a parent, i.e. maybe contains CoW mappings? */ static bool vma_is_fork_child(struct vm_area_struct *vma) { diff --git a/tools/testing/vma/shared.c b/tools/testing/vma/shared.c index 2565a5aecb80..bea9ea6db02a 100644 --- a/tools/testing/vma/shared.c +++ b/tools/testing/vma/shared.c @@ -120,12 +120,3 @@ unsigned long rlimit(unsigned int limit) { return (unsigned long)-1; } - -void vma_set_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end, - pgoff_t pgoff) -{ - vma->vm_start = start; - vma->vm_end = end; - vma->vm_pgoff = pgoff; -} diff --git a/tools/testing/vma/shared.h b/tools/testing/vma/shared.h index 8b9e3b11c3cb..ca4f1238f1c7 100644 --- a/tools/testing/vma/shared.h +++ b/tools/testing/vma/shared.h @@ -125,8 +125,3 @@ void __vma_set_dummy_anon_vma(struct vm_area_struct *vma, /* Provide a simple dummy VMA/anon_vma dummy setup for testing. */ void vma_set_dummy_anon_vma(struct vm_area_struct *vma, struct anon_vma_chain *avc); - -/* Helper function to specify a VMA's range. */ -void vma_set_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end, - pgoff_t pgoff); From 6387563e6ecdbd81d138525214475e66967801ab Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:17:07 +0100 Subject: [PATCH 356/501] mm/vma: update vma_shrink() to not pass start, pgoff parameters vma_shrink() is only used by relocate_vma_down() to shrink the tail of a VMA. Therefore neither the start nor the pgoff parameters make any sense. It seemed we were passing the pgoff parameter solely to satisfy vma_set_range()'s requirement for pgoff being specified. Since vma_set_range() is now isolated to vma.c, we can simply introduce __vma_set_range() which sets only vma->vm_[start, end], and invoke this instead, removing start and pgoff from vma_shrink() altogether. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-26-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Pedro Falcato Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/vma.c | 31 +++++++++++++++++-------------- mm/vma.h | 3 +-- mm/vma_exec.c | 2 +- tools/testing/vma/tests/merge.c | 2 +- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/mm/vma.c b/mm/vma.c index a3500a53a7c4..c75322d0ffc2 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -70,11 +70,17 @@ struct mmap_state { .state = VMA_MERGE_START, \ } -static void vma_set_range(struct vm_area_struct *vma, unsigned long start, - unsigned long end, pgoff_t pgoff) +static void __vma_set_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end) { vma->vm_start = start; vma->vm_end = end; +} + +static void vma_set_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end, pgoff_t pgoff) +{ + __vma_set_range(vma, start, end); vma->vm_pgoff = pgoff; } @@ -1279,27 +1285,24 @@ int vma_expand(struct vma_merge_struct *vmg) return -ENOMEM; } -/* - * vma_shrink() - Reduce an existing VMAs memory area +/** + * vma_shrink() - Shrink the end of a VMA * @vmi: The vma iterator * @vma: The VMA to modify - * @start: The new start * @end: The new end * + * Note that the caller may only shrink the end of the VMA. + * * Returns: 0 on success, -ENOMEM otherwise */ int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, - unsigned long start, unsigned long end, pgoff_t pgoff) + unsigned long end) { struct vma_prepare vp; - WARN_ON((vma->vm_start != start) && (vma->vm_end != end)); - - if (vma->vm_start < start) - vma_iter_config(vmi, vma->vm_start, start); - else - vma_iter_config(vmi, end, vma->vm_end); + VM_WARN_ON_ONCE(end > vma->vm_end); + vma_iter_config(vmi, end, vma->vm_end); if (vma_iter_prealloc(vmi, NULL)) return -ENOMEM; @@ -1307,10 +1310,10 @@ int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, init_vma_prep(&vp, vma); vma_prepare(&vp); - vma_adjust_trans_huge(vma, start, end, NULL); + vma_adjust_trans_huge(vma, vma->vm_start, end, NULL); vma_iter_clear(vmi); - vma_set_range(vma, start, end, pgoff); + __vma_set_range(vma, vma->vm_start, end); vma_complete(&vp, vmi, vma->vm_mm); validate_mm(vma->vm_mm); return 0; diff --git a/mm/vma.h b/mm/vma.h index 14f026bf3be4..40effaa3ebe4 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -297,8 +297,7 @@ void validate_mm(struct mm_struct *mm); __must_check int vma_expand(struct vma_merge_struct *vmg); __must_check int vma_shrink(struct vma_iterator *vmi, - struct vm_area_struct *vma, - unsigned long start, unsigned long end, pgoff_t pgoff); + struct vm_area_struct *vma, unsigned long end); static inline int vma_iter_store_gfp(struct vma_iterator *vmi, struct vm_area_struct *vma, gfp_t gfp) diff --git a/mm/vma_exec.c b/mm/vma_exec.c index e3644a3042e2..13a05e041195 100644 --- a/mm/vma_exec.c +++ b/mm/vma_exec.c @@ -89,7 +89,7 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) vma_prev(&vmi); /* Shrink the vma to just the new range */ - return vma_shrink(&vmi, vma, new_start, new_end, vma_start_pgoff(vma)); + return vma_shrink(&vmi, vma, new_end); } /* diff --git a/tools/testing/vma/tests/merge.c b/tools/testing/vma/tests/merge.c index f8666a755749..e357accc8499 100644 --- a/tools/testing/vma/tests/merge.c +++ b/tools/testing/vma/tests/merge.c @@ -227,7 +227,7 @@ static bool test_simple_shrink(void) ASSERT_FALSE(attach_vma(&mm, vma)); - ASSERT_FALSE(vma_shrink(&vmi, vma, 0, 0x1000, 0)); + ASSERT_FALSE(vma_shrink(&vmi, vma, 0x1000)); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x1000); From ce4ec991da01db884e96a96b2ac289b61f100d30 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:17:08 +0100 Subject: [PATCH 357/501] mm/vma: update vmg_adjust_set_range() to offset pgoff instead We are calculating the pgoff as an offset, since we have vma_add_pgoff() and vma_sub_pgoff() available, just offset this value directly and use __vma_set_range() for vma->vm_[start, end] values. We take care to update the range before offsetting the page offset, so the adjusted VMA's vm_start and vm_pgoff are mutually consistent at the point the page offset helpers operate - this matters once vma_set_pgoff() comes to assert invariants which relate the two. Doing so lays the foundation for future work which allows for use of virtual page offsets for MAP_PRIVATE-file backed mappings. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-27-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Pedro Falcato Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/vma.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/mm/vma.c b/mm/vma.c index c75322d0ffc2..78af0e530066 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -714,9 +714,6 @@ void validate_mm(struct mm_struct *mm) */ static void vmg_adjust_set_range(struct vma_merge_struct *vmg) { - struct vm_area_struct *adjust; - pgoff_t pgoff; - if (vmg->__adjust_middle_start) { /* * vmg->start vmg->end @@ -735,8 +732,8 @@ static void vmg_adjust_set_range(struct vma_merge_struct *vmg) struct vm_area_struct *middle = vmg->middle; const unsigned long delta = vmg->end - middle->vm_start; - pgoff = vma_start_pgoff(middle) + (delta >> PAGE_SHIFT); - adjust = middle; + __vma_set_range(middle, vmg->end, middle->vm_end); + vma_add_pgoff(middle, delta >> PAGE_SHIFT); } else if (vmg->__adjust_next_start) { /* * Originally: @@ -764,13 +761,9 @@ static void vmg_adjust_set_range(struct vma_merge_struct *vmg) struct vm_area_struct *next = vmg->next; const unsigned long delta = next->vm_start - vmg->end; - pgoff = vma_start_pgoff(next) - (delta >> PAGE_SHIFT); - adjust = next; - } else { - return; + __vma_set_range(next, vmg->end, next->vm_end); + vma_sub_pgoff(next, delta >> PAGE_SHIFT); } - - vma_set_range(adjust, vmg->end, adjust->vm_end, pgoff); } /* From bcb52fc56ee6a9aee942d512848b3c94f3432da3 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:17:09 +0100 Subject: [PATCH 358/501] mm/vma: slightly rework the anonymous check in __mmap_new_vma() Determine if the VMA is anonymous early on, so we separate the logic which invokes callbacks from the rest of the logic. This is in order that a subsequent commit which asserts correct virtual page offset for anonymous mappings correctly asserts this for new anonymous mappings. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-28-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: Pedro Falcato Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/vma.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/mm/vma.c b/mm/vma.c index 78af0e530066..4f2c56c57040 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -2587,6 +2587,8 @@ static int __mmap_new_file_vma(struct mmap_state *map, static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, struct mmap_action *action) { + const bool is_anon = !map->file && + !vma_flags_test(&map->vma_flags, VMA_SHARED_BIT); struct vma_iterator *vmi = map->vmi; int error = 0; struct vm_area_struct *vma; @@ -2601,6 +2603,10 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, return -ENOMEM; vma_iter_config(vmi, map->addr, map->end); + + if (is_anon) + vma_set_anonymous(vma); + vma_set_range(vma, map->addr, map->end, map->pgoff); vma->flags = map->vma_flags; vma->vm_page_prot = map->page_prot; @@ -2610,12 +2616,11 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, goto free_vma; } + /* Invoke callbacks. */ if (map->file) error = __mmap_new_file_vma(map, vma); - else if (vma_flags_test(&map->vma_flags, VMA_SHARED_BIT)) + else if (!is_anon) error = shmem_zero_setup(vma); - else - vma_set_anonymous(vma); if (error) goto free_iter_vma; From 613562c29fa69840f9a6239f1c3e0c24270a795d Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:17:10 +0100 Subject: [PATCH 359/501] mm/vma: introduce and use vma_set_pgoff() In order to lay the foundation for work that permits us to track the virtual page offset of MAP_PRIVATE file-backed mappings, we abstract the assignment of vma->vm_pgoff to vma_set_pgoff(). We additionally add a lock check here using the newly introduced vma_assert_can_modify(). This asserts the VMA write lock if the VMA is attached. We also assert that, if this is an anonymous VMA and unfaulted, that its (virtual) page offset is equal to the page offset of the VMA's address. We must be careful about MAP_PRIVATE-/dev/zero which violates fundamental assumptions about anonymous memory, so we check for !vma->vm_file after using vma_is_anonymous() which these mappings satisfy. Additionally, we only perform the assert if CONFIG_MMU is defined, as nommu does not set vma->vm_pgoff = addr >> PAGE_SHIFT. This isn't really relevant to rmap as it has no anon rmap (nor needs it), but we must avoid it asserting falsely. All of this logic is kept in assert_sane_pgoff() to keep things clear. In order to maintain correctness given this assert, we also update __install_special_mapping() to invoke vma_set_range() after it's set vma->vm_ops (which determine whether the VMA is anonymous or not). We do not use vma_set_pgoff() in vm_area_init_from(), as at the point of forking, we don't necessarily have correct locking state. Updating vma_set_range() covers most cases, but in addition to this we also update insert_vm_struct(), compat_set_vma_from_desc() and nommu callers. We also update vma_add_pgoff() and vma_sub_pgoff() to use vma_set_pgoff(). While we're here, we drop a BUG_ON() and update insert_vm_struct()'s comment to reflect the fact anonymous mappings can be added here. Finally, we update the CONFIG_MMU, CONFIG_PER_VMA_LOCK defines in the VMA userland tests so IS_ENABLED() will work correctly with them. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-29-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Pedro Falcato Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/nommu.c | 2 +- mm/vma.c | 14 ++++++------- mm/vma.h | 35 +++++++++++++++++++++++++++++--- tools/testing/vma/vma_internal.h | 4 ++-- 4 files changed, 42 insertions(+), 13 deletions(-) diff --git a/mm/nommu.c b/mm/nommu.c index dc64834e6834..82556301b356 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -1062,7 +1062,7 @@ unsigned long do_mmap(struct file *file, region->vm_pgoff = pgoff; vm_flags_init(vma, vm_flags); - vma->vm_pgoff = pgoff; + vma_set_pgoff(vma, pgoff); if (file) { region->vm_file = get_file(file); diff --git a/mm/vma.c b/mm/vma.c index 4f2c56c57040..c253c5498359 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -81,7 +81,7 @@ static void vma_set_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, pgoff_t pgoff) { __vma_set_range(vma, start, end); - vma->vm_pgoff = pgoff; + vma_set_pgoff(vma, pgoff); } /* Was this VMA ever forked from a parent, i.e. maybe contains CoW mappings? */ @@ -3347,9 +3347,9 @@ int __vm_munmap(unsigned long start, size_t len, bool unlock) return ret; } -/* Insert vm structure into process list sorted by address - * and into the inode's i_mmap tree. If vm_file is non-NULL - * then i_mmap_rwsem is taken here. +/* + * Insert vm structure into process list sorted by address + * and into the inode's i_mmap tree if file-backed. */ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) { @@ -3375,8 +3375,8 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) * Similarly in do_mmap and in do_brk_flags. */ if (vma_is_anonymous(vma)) { - BUG_ON(vma->anon_vma); - vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT; + WARN_ON_ONCE(vma->anon_vma); + vma_set_pgoff(vma, vma->vm_start >> PAGE_SHIFT); } if (vma_link(mm, vma)) { @@ -3422,7 +3422,6 @@ struct vm_area_struct *__install_special_mapping( if (unlikely(vma == NULL)) return ERR_PTR(-ENOMEM); - vma_set_range(vma, addr, addr + len, 0); vm_flags |= mm->def_flags | VM_DONTEXPAND; if (pgtable_supports_soft_dirty()) vm_flags |= VM_SOFTDIRTY; @@ -3431,6 +3430,7 @@ struct vm_area_struct *__install_special_mapping( vma->vm_ops = ops; vma->vm_private_data = priv; + vma_set_range(vma, addr, addr + len, 0); ret = insert_vm_struct(mm, vma); if (ret) diff --git a/mm/vma.h b/mm/vma.h index 40effaa3ebe4..58f48609ce22 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -247,16 +247,45 @@ static inline pgoff_t vmg_end_pgoff(const struct vma_merge_struct *vmg) return vmg_start_pgoff(vmg) + vmg_pages(vmg); } +static inline void assert_sane_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) +{ + /* nommu doesn't set a virtual pgoff for anon VMAs. */ + if (!IS_ENABLED(CONFIG_MMU)) + return; + /* + * File-backed VMAs have arbitrary page offset (either page offset into + * file or for pfnmap the PFN of the start of the range or drivers may + * set arbitrary page offset). + */ + if (!vma_is_anonymous(vma)) + return; + /* MAP_PRIVATE-/dev/zero is anon, non-NULL vm_file, but has file pgoff. */ + if (vma->vm_file) + return; + /* If faulted in, could have been remapped. */ + if (vma->anon_vma) + return; + /* OK this is really an anon VMA - expect virtual page offset. */ + VM_WARN_ON_ONCE(pgoff != vma->vm_start >> PAGE_SHIFT); +} + +static inline void vma_set_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) +{ + vma_assert_can_modify(vma); + assert_sane_pgoff(vma, pgoff); + vma->vm_pgoff = pgoff; +} + static inline void vma_add_pgoff(struct vm_area_struct *vma, pgoff_t delta) { vma_assert_can_modify(vma); - vma->vm_pgoff += delta; + vma_set_pgoff(vma, vma_start_pgoff(vma) + delta); } static inline void vma_sub_pgoff(struct vm_area_struct *vma, pgoff_t delta) { vma_assert_can_modify(vma); - vma->vm_pgoff -= delta; + vma_set_pgoff(vma, vma_start_pgoff(vma) - delta); } #define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_) \ @@ -331,7 +360,7 @@ static inline void compat_set_vma_from_desc(struct vm_area_struct *vma, */ /* Mutable fields. Populated with initial state. */ - vma->vm_pgoff = desc->pgoff; + vma_set_pgoff(vma, desc->pgoff); if (desc->vm_file != vma->vm_file) vma_set_file(vma, desc->vm_file); vma->flags = desc->vma_flags; diff --git a/tools/testing/vma/vma_internal.h b/tools/testing/vma/vma_internal.h index e12ab2c80f95..4f6c5666ac07 100644 --- a/tools/testing/vma/vma_internal.h +++ b/tools/testing/vma/vma_internal.h @@ -14,8 +14,8 @@ #include -#define CONFIG_MMU -#define CONFIG_PER_VMA_LOCK +#define CONFIG_MMU 1 +#define CONFIG_PER_VMA_LOCK 1 #ifdef __CONCAT #undef __CONCAT From dd7d7d00c073b742922ae1a82c7bd2f3a27b4140 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:17:11 +0100 Subject: [PATCH 360/501] mm/vma: correct incorrect vma.h inclusion The only files which should be including vma.h are the implementation files for the core VMA logic - vma.c, vma_init.c, and vma_exec.c. This is in order to allow for userland testing of core VMA logic. In this cases, vma_internal.h and vma.h are included, providing both the dependencies upon which the core VMA logic requires and its declarations. Userland testable VMA logic is achieved by having separate vma_internal.h implementations for userland and kernel. Callers other than the core VMA implementation should include internal.h instead. This header does not need to include vma_internal.h as it only contains the vma.h declarations, for which the includes already present suffice. Update code to reflect this, update comments to reflect the fact there are 3 VMA implementation files and document things more clearly. While we're here, slightly improve the language of the comment describing vma_exec.c. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-30-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Pedro Falcato Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/mmu_notifier.c | 2 +- mm/nommu.c | 1 - mm/vma.c | 4 ++++ mm/vma.h | 9 ++++++++- mm/vma_exec.c | 8 ++++++-- mm/vma_init.c | 4 ++++ mm/vma_internal.h | 4 ++-- 7 files changed, 25 insertions(+), 7 deletions(-) diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c index 245b74f39f91..df69ba6e797f 100644 --- a/mm/mmu_notifier.c +++ b/mm/mmu_notifier.c @@ -19,7 +19,7 @@ #include #include -#include "vma.h" +#include "internal.h" /* global SRCU for all MMs */ DEFINE_STATIC_SRCU(srcu); diff --git a/mm/nommu.c b/mm/nommu.c index 82556301b356..a7da5ec21f42 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -41,7 +41,6 @@ #include #include #include "internal.h" -#include "vma.h" unsigned long highest_memmap_pfn; int heap_stack_gap = 0; diff --git a/mm/vma.c b/mm/vma.c index c253c5498359..d74a8ae16bfa 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -4,6 +4,10 @@ * VMA-specific functions. */ +/* + * To allow for userland testing we place internal dependencies in + * vma_internal.h and external VMA API declarations in vma.h. + */ #include "vma_internal.h" #include "vma.h" diff --git a/mm/vma.h b/mm/vma.h index 58f48609ce22..adb7a0ba1192 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -2,7 +2,14 @@ /* * vma.h * - * Core VMA manipulation API implemented in vma.c. + * Core VMA manipulation API implemented in vma.c, vma_init.c and vma_exec.c. + * + * Note that, in order for VMA logic to be userland testable, this header + * intentionally includes no dependencies. + * + * This is specifically scoped to mm-only. Users of this functionality (other + * than the core VMA implementation itself) should not include this header + * directly, but rather include internal.h. */ #ifndef __MM_VMA_H #define __MM_VMA_H diff --git a/mm/vma_exec.c b/mm/vma_exec.c index 13a05e041195..ef1fa2b161f3 100644 --- a/mm/vma_exec.c +++ b/mm/vma_exec.c @@ -1,10 +1,14 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Functions explicitly implemented for exec functionality which however are - * explicitly VMA-only logic. + * Functions provided for exec functionality which however are + * specifically VMA-only logic. */ +/* + * To allow for userland testing we place internal dependencies in + * vma_internal.h and external VMA API declarations in vma.h. + */ #include "vma_internal.h" #include "vma.h" diff --git a/mm/vma_init.c b/mm/vma_init.c index a459669a1654..715feee283f0 100644 --- a/mm/vma_init.c +++ b/mm/vma_init.c @@ -5,6 +5,10 @@ * between CONFIG_MMU and non-CONFIG_MMU kernel configurations. */ +/* + * To allow for userland testing we place internal dependencies in + * vma_internal.h and external VMA API declarations in vma.h. + */ #include "vma_internal.h" #include "vma.h" diff --git a/mm/vma_internal.h b/mm/vma_internal.h index 2da6d224c1a8..4d300e7bbaf4 100644 --- a/mm/vma_internal.h +++ b/mm/vma_internal.h @@ -2,8 +2,8 @@ /* * vma_internal.h * - * Headers required by vma.c, which can be substituted accordingly when testing - * VMA functionality. + * Headers required by vma.c, vma_init.c and vma_exec.c, which can be + * substituted accordingly when testing VMA functionality. */ #ifndef __MM_VMA_INTERNAL_H From 9e98b6dd46f53ae02de62e4efa286c93f8610a0f Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:17:12 +0100 Subject: [PATCH 361/501] mm/vma: use guard clauses in can_vma_merge_[before, after]() Rather than combining a bunch of conditionals in a single expression, simplify by inverting the mergeability requirements into guard clauses. that is - instead of checking what must be true for the conditions to be met, instead check the inverse of the requirements and return false if any are true, defaulting to true. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-31-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Pedro Falcato Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/vma.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/mm/vma.c b/mm/vma.c index d74a8ae16bfa..12a684c36b6c 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -215,13 +215,13 @@ static void init_multi_vma_prep(struct vma_prepare *vp, */ static bool can_vma_merge_before(struct vma_merge_struct *vmg) { - if (is_mergeable_vma(vmg, /* merge_next = */ true) && - is_mergeable_anon_vma(vmg, /* merge_next = */ true)) { - if (vmg_end_pgoff(vmg) == vma_start_pgoff(vmg->next)) - return true; - } - - return false; + if (!is_mergeable_vma(vmg, /* merge_next = */ true)) + return false; + if (!is_mergeable_anon_vma(vmg, /* merge_next = */ true)) + return false; + if (vmg_end_pgoff(vmg) != vma_start_pgoff(vmg->next)) + return false; + return true; } /* @@ -235,12 +235,13 @@ static bool can_vma_merge_before(struct vma_merge_struct *vmg) */ static bool can_vma_merge_after(struct vma_merge_struct *vmg) { - if (is_mergeable_vma(vmg, /* merge_next = */ false) && - is_mergeable_anon_vma(vmg, /* merge_next = */ false)) { - if (vma_end_pgoff(vmg->prev) == vmg_start_pgoff(vmg)) - return true; - } - return false; + if (!is_mergeable_vma(vmg, /* merge_next = */ false)) + return false; + if (!is_mergeable_anon_vma(vmg, /* merge_next = */ false)) + return false; + if (vma_end_pgoff(vmg->prev) != vmg_start_pgoff(vmg)) + return false; + return true; } static void __vma_link_file(struct vm_area_struct *vma, From fe95ccb1ba7c9226765a0970381eb1c5a31701bb Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:17:13 +0100 Subject: [PATCH 362/501] tools/testing/vma: default VMA, mm flag bits to 64-bit With all of the sanitisers turned on, setting the VMA and mm flag bits depth to 128 by default results in overly long build times. Reduce this to 64 - we can always manipulate these later for testing of larger bitmaps as needed. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-32-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Gregory Price Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: Pedro Falcato Cc: SJ Park Cc: Thomas Zimmermann Cc: Vlastimil Babka (SUSE) Cc: Liam R. Howlett (Oracle) Cc: Zi Yan Signed-off-by: Andrew Morton --- tools/testing/vma/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/vma/Makefile b/tools/testing/vma/Makefile index e72b45dedda5..ef6cc558afe1 100644 --- a/tools/testing/vma/Makefile +++ b/tools/testing/vma/Makefile @@ -10,7 +10,7 @@ OFILES = $(SHARED_OFILES) main.o shared.o maple-shim.o TARGETS = vma # These can be varied to test different sizes. -CFLAGS += -DNUM_VMA_FLAG_BITS=128 -DNUM_MM_FLAG_BITS=128 +CFLAGS += -DNUM_VMA_FLAG_BITS=64 -DNUM_MM_FLAG_BITS=64 main.o: main.c shared.c shared.h vma_internal.h tests/merge.c tests/mmap.c tests/vma.c ../../../mm/vma.c ../../../mm/vma_init.c ../../../mm/vma_exec.c ../../../mm/vma.h include/custom.h include/dup.h include/stubs.h From f1966d954bc58093e933c23f7aa2a6c44c06fb38 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:17:14 +0100 Subject: [PATCH 363/501] tools/testing/vma: output compared expression on ASSERT_[EQ, NE]() Update the macros to output the compared values at hex for easier debugging when test asserts fail. We have to be careful not to re-evaluate expressions as they may have side-effects. So update the code to take local copies and use these for both the test and the debug output. Also remove unused IS_SET() macro. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-33-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Gregory Price Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: Pedro Falcato Cc: SJ Park Cc: Thomas Zimmermann Cc: Vlastimil Babka (SUSE) Cc: Liam R. Howlett (Oracle) Cc: Zi Yan Signed-off-by: Andrew Morton --- tools/testing/vma/shared.h | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/tools/testing/vma/shared.h b/tools/testing/vma/shared.h index ca4f1238f1c7..97cd7a679dc1 100644 --- a/tools/testing/vma/shared.h +++ b/tools/testing/vma/shared.h @@ -21,19 +21,35 @@ } \ } while (0) -#define ASSERT_TRUE(_expr) \ - do { \ - if (!(_expr)) { \ - fprintf(stderr, \ - "Assert FAILED at %s:%d:%s(): %s is FALSE.\n", \ - __FILE__, __LINE__, __FUNCTION__, #_expr); \ - return false; \ - } \ +#define __ASSERT_TRUE(_expr, _fmt, ...) \ + do { \ + if (!(_expr)) { \ + fprintf(stderr, \ + "Assert FAILED at %s:%d:%s(): %s is FALSE" \ + _fmt ".\n", \ + __FILE__, __LINE__, __FUNCTION__, #_expr \ + __VA_OPT__(,) __VA_ARGS__); \ + return false; \ + } \ } while (0) +#define __TO_SCALAR(x) ((unsigned long long)(uintptr_t)(x)) + +#define ASSERT_TRUE(_expr) __ASSERT_TRUE(_expr, "") #define ASSERT_FALSE(_expr) ASSERT_TRUE(!(_expr)) -#define ASSERT_EQ(_val1, _val2) ASSERT_TRUE((_val1) == (_val2)) -#define ASSERT_NE(_val1, _val2) ASSERT_TRUE((_val1) != (_val2)) +#define ASSERT_EQ(_val1, _val2) do { \ + __typeof__(_val1) __val1 = (_val1); \ + __typeof__(_val2) __val2 = (_val2); \ + __ASSERT_TRUE(__val1 == __val2, " (0x%llx != 0x%llx)", \ + __TO_SCALAR(__val1), __TO_SCALAR(__val2)); \ + } while (0) + +#define ASSERT_NE(_val1, _val2) do { \ + __typeof__(_val1) __val1 = (_val1); \ + __typeof__(_val2) __val2 = (_val2); \ + __ASSERT_TRUE(__val1 != __val2, " (0x%llx == 0x%llx)", \ + __TO_SCALAR(__val1), __TO_SCALAR(__val2)); \ + } while (0) #define ASSERT_FLAGS_SAME_MASK(_flags, _flags_other) \ ASSERT_TRUE(vma_flags_same_mask((_flags), (_flags_other))) @@ -53,8 +69,6 @@ #define ASSERT_FLAGS_NONEMPTY(_flags) \ ASSERT_FALSE(vma_flags_empty(_flags)) -#define IS_SET(_val, _flags) ((_val & _flags) == _flags) - extern bool fail_prealloc; /* Override vma_iter_prealloc() so we can choose to fail it. */ From 2858b4027f491e6fdb8ee2d8923798b619bf2791 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Fri, 10 Jul 2026 03:55:21 -0700 Subject: [PATCH 364/501] mm/mempolicy: skip non-present PMDs when queueing folios Patch series "mm: handle device-private PMDs in walk callbacks", v3. Since commit 368076f52ebe ("mm/huge_memory: add device-private THP support to PMD operations") a PMD may hold a device-private swap entry whenever an HMM-based GPU driver migrates an anonymous THP folio to device memory via migrate_vma_pages(). pmd_trans_huge_lock() succeeds for such PMDs (pmd_is_huge() returns true for any non-present, non-none huge PMD), so several MM walk callbacks that used to assume present THP or migration entry are now reachable with a device-private PMD. The results range from a VM_BUG_ON() firing on debug kernels, to an oops on a bogus vmemmap dereference, to silently isolating an unrelated live folio from LRU in the aliasing case. This patch (of 3): queue_folios_pmd() is called under pmd_trans_huge_lock(), whose pmd_is_huge() check returns true for any non-present, non-none PMD softleaf. Passing such a PMD to pmd_folio() treats the softleaf encoding as a hardware PFN and can return a bogus folio pointer. Mirror queue_folios_pte_range(): handle non-present entries before looking up a folio. Keep migration entries counted as failures, but skip other non-present PMDs such as device-private entries. Potential trigger: an HMM-based GPU driver migrates an anonymous THP folio to device memory via migrate_vma_pages(), leaving a device-private PMD. Userspace then calls mbind(), migrate_pages() or set_mempolicy_home_node() on that range. Link: https://lore.kernel.org/20260710105557.1987433-1-usama.arif@linux.dev Link: https://lore.kernel.org/20260710105557.1987433-2-usama.arif@linux.dev Fixes: 368076f52ebe ("mm/huge_memory: add device-private THP support to PMD operations") Signed-off-by: Usama Arif Reported-by: sashiko-bot Link: https://sashiko.dev/#/patchset/20260703173903.3789516-1-usama.arif%40linux.dev?part=6 Reviewed-by: Joshua Hahn Reviewed-by: Zi Yan Acked-by: David Hildenbrand (Arm) Reviewed-by: Balbir Singh Reviewed-by: Gregory Price Cc: Alistair Popple Cc: Baolin Wang Cc: Barry Song Cc: Byungchul Park Cc: Dev Jain Cc: "Huang, Ying" Cc: Jann Horn Cc: Johannes Weiner Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Brost Cc: Nico Pache Cc: Rakie Kim Cc: Ryan Roberts Cc: Shakeel Butt Cc: Vlastimil Babka Cc: Signed-off-by: Andrew Morton --- mm/mempolicy.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mm/mempolicy.c b/mm/mempolicy.c index a92bd29cf032..501e0b80d7da 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -654,12 +654,14 @@ static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk) { struct folio *folio; struct queue_pages *qp = walk->private; + pmd_t pmdval = pmdp_get(pmd); - if (unlikely(pmd_is_migration_entry(*pmd))) { - qp->nr_failed++; + if (unlikely(!pmd_present(pmdval))) { + if (pmd_is_migration_entry(pmdval)) + qp->nr_failed++; return; } - folio = pmd_folio(*pmd); + folio = pmd_folio(pmdval); if (is_huge_zero_folio(folio)) { walk->action = ACTION_CONTINUE; return; From d4b76d0b03cb49611312ecc4bcfb55ccdf0843e2 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Fri, 10 Jul 2026 03:55:22 -0700 Subject: [PATCH 365/501] mm/madvise: skip device-private PMDs in cold and pageout walks madvise_cold_or_pageout_pte_range() takes pmd_trans_huge_lock(), whose pmd_is_huge() check returns true for a device-private PMD. The subsequent !pmd_present() branch has a VM_BUG_ON() asserting migration is the only allowed non-present case; a device-private PMD trips it. Skip device-private PMDs in that non-present branch and continue to huge_unlock before calling pmd_folio(). Downgrade the check to VM_WARN_ON_ONCE() so an unexpected PMD softleaf logs a warning rather than panicking. Drop the thp_migration_supported() guard: it expands to IS_ENABLED(CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF), and both pmd_is_migration_entry() and pmd_is_device_private_entry() already return false when that config is not selected, so the guard suppresses only the case where the warning would already be silent. Potential trigger: an HMM-based GPU driver races with madvise(MADV_COLD)/MADV_PAGEOUT: pmd_trans_huge(*pmd) reads true, then migrate_vma_pages() flips the PMD to a device-private entry before the PMD lock is acquired. Link: https://lore.kernel.org/20260710105557.1987433-3-usama.arif@linux.dev Fixes: 368076f52ebe ("mm/huge_memory: add device-private THP support to PMD operations") Signed-off-by: Usama Arif Reported-by: sashiko-bot Link: https://sashiko.dev/#/patchset/20260703173903.3789516-1-usama.arif%40linux.dev?part=6 Reviewed-by: Joshua Hahn Reviewed-by: Zi Yan Reviewed-by: Balbir Singh Reviewed-by: Gregory Price Cc: Alistair Popple Cc: Baolin Wang Cc: Barry Song Cc: Byungchul Park Cc: David Hildenbrand (Arm) Cc: Dev Jain Cc: "Huang, Ying" Cc: Jann Horn Cc: Johannes Weiner Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Brost Cc: Nico Pache Cc: Rakie Kim Cc: Ryan Roberts Cc: Shakeel Butt Cc: Vlastimil Babka Cc: Signed-off-by: Andrew Morton --- mm/madvise.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/madvise.c b/mm/madvise.c index 917f764fef80..bf9ce199935a 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -388,8 +388,8 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, goto huge_unlock; if (unlikely(!pmd_present(orig_pmd))) { - VM_BUG_ON(thp_migration_supported() && - !pmd_is_migration_entry(orig_pmd)); + VM_WARN_ON_ONCE(!pmd_is_migration_entry(orig_pmd) && + !pmd_is_device_private_entry(orig_pmd)); goto huge_unlock; } From ce579dcf730ce5ed8043a5eeea18a3e6706a97e5 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Fri, 10 Jul 2026 03:55:23 -0700 Subject: [PATCH 366/501] mm/huge_memory: skip device-private PMDs in madvise_free_huge_pmd madvise_free_pte_range() checks pmd_trans_huge(*pmd) unlocked, then madvise_free_huge_pmd() takes pmd_trans_huge_lock(). pmd_is_huge() returns true for a device-private PMD, so orig_pmd can be device-private and enter the !pmd_present() branch. Skip device-private PMDs in that non-present branch and continue to out before calling pmd_folio(). Downgrade the check to VM_WARN_ON_ONCE() so an unexpected PMD softleaf logs a warning rather than panicking. Drop the thp_migration_supported() guard: it expands to IS_ENABLED(CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF), and both pmd_is_migration_entry() and pmd_is_device_private_entry() already return false when that config is not selected, so the guard suppresses only the case where the warning would already be silent. Potential trigger: an HMM-based GPU driver races with madvise(MADV_FREE): migrate_vma_pages() flips the PMD to a device-private entry between the caller's pmd_trans_huge() check and the callee's pmd_trans_huge_lock(). Link: https://lore.kernel.org/20260710105557.1987433-4-usama.arif@linux.dev Fixes: 368076f52ebe ("mm/huge_memory: add device-private THP support to PMD operations") Signed-off-by: Usama Arif Reviewed-by: Joshua Hahn Reviewed-by: Zi Yan Reviewed-by: Balbir Singh Acked-by: David Hildenbrand (Arm) Reviewed-by: Gregory Price Cc: Alistair Popple Cc: Baolin Wang Cc: Barry Song Cc: Byungchul Park Cc: Dev Jain Cc: "Huang, Ying" Cc: Jann Horn Cc: Johannes Weiner Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Brost Cc: Nico Pache Cc: Rakie Kim Cc: Ryan Roberts Cc: sashiko-bot Cc: Shakeel Butt Cc: Vlastimil Babka Cc: Signed-off-by: Andrew Morton --- mm/huge_memory.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index c89c1193c64d..c615e81620b9 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2331,8 +2331,8 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, goto out; if (unlikely(!pmd_present(orig_pmd))) { - VM_BUG_ON(thp_migration_supported() && - !pmd_is_migration_entry(orig_pmd)); + VM_WARN_ON_ONCE(!pmd_is_migration_entry(orig_pmd) && + !pmd_is_device_private_entry(orig_pmd)); goto out; } From 1edcd9d7e52f862de9f9b6d1b9fe7f172532ae82 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 10 Jul 2026 17:24:27 +0800 Subject: [PATCH 367/501] selftests/mm: remove obsolete hugetlb vmemmap test The hugetlb vmemmap selftest was added to check the old HVO layout where tail vmemmap pages reused the head page. That assumption no longer matches the current HVO mapping layout. HVO now keeps a private backing page for the head vmemmap page and remaps redundant tail vmemmap pages to a shared read-only backing page. The old page flag check is therefore testing an obsolete implementation detail rather than the current ABI or behavior. Remove the stale test and its build and run entries. [akpm@linux-foundation.org: don't clean old executables out of .gitignore] Link: https://lore.kernel.org/CAHk-=wiHK5_oBUdUiNAaevmN9f-ORe+QBqbRefAZaw-RbgEn3w@mail.gmail.com Link: https://lore.kernel.org/20260710092427.3459121-1-songmuchun@bytedance.com Signed-off-by: Muchun Song Reviewed-by: Lorenzo Stoakes Acked-by: David Hildenbrand (Arm) Reviewed-by: SJ Park Cc: Liam R. Howlett Cc: Michal Hocko Cc: Mike Rapoport Cc: Muchun Song Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/Makefile | 1 - tools/testing/selftests/mm/hugetlb-vmemmap.c | 132 ------------------- tools/testing/selftests/mm/run_vmtests.sh | 1 - 3 files changed, 134 deletions(-) delete mode 100644 tools/testing/selftests/mm/hugetlb-vmemmap.c diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile index e6df968f0971..277a141d662e 100644 --- a/tools/testing/selftests/mm/Makefile +++ b/tools/testing/selftests/mm/Makefile @@ -66,7 +66,6 @@ TEST_GEN_FILES += hugetlb-mremap TEST_GEN_FILES += hugetlb-read-hwpoison TEST_GEN_FILES += hugetlb-shm TEST_GEN_FILES += hugetlb-soft-offline -TEST_GEN_FILES += hugetlb-vmemmap TEST_GEN_FILES += khugepaged TEST_GEN_FILES += madv_populate TEST_GEN_FILES += map_fixed_noreplace diff --git a/tools/testing/selftests/mm/hugetlb-vmemmap.c b/tools/testing/selftests/mm/hugetlb-vmemmap.c deleted file mode 100644 index 507df78a158d..000000000000 --- a/tools/testing/selftests/mm/hugetlb-vmemmap.c +++ /dev/null @@ -1,132 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * A test case of using hugepage memory in a user application using the - * mmap system call with MAP_HUGETLB flag. Before running this program - * make sure the administrator has allocated enough default sized huge - * pages to cover the 2 MB allocation. - */ -#include -#include -#include -#include -#include -#include "vm_util.h" -#include "hugepage_settings.h" - -#define PAGE_COMPOUND_HEAD (1UL << 15) -#define PAGE_COMPOUND_TAIL (1UL << 16) -#define PAGE_HUGE (1UL << 17) - -#define HEAD_PAGE_FLAGS (PAGE_COMPOUND_HEAD | PAGE_HUGE) -#define TAIL_PAGE_FLAGS (PAGE_COMPOUND_TAIL | PAGE_HUGE) - -#define PM_PFRAME_BITS 55 -#define PM_PFRAME_MASK ~((1UL << PM_PFRAME_BITS) - 1) - -static size_t pagesize; -static size_t maplength; - -static void write_bytes(char *addr, size_t length) -{ - unsigned long i; - - for (i = 0; i < length; i++) - *(addr + i) = (char)i; -} - -static unsigned long virt_to_pfn(void *addr) -{ - int fd; - unsigned long pagemap; - - fd = open("/proc/self/pagemap", O_RDONLY); - if (fd < 0) - return -1UL; - - lseek(fd, (unsigned long)addr / pagesize * sizeof(pagemap), SEEK_SET); - read(fd, &pagemap, sizeof(pagemap)); - close(fd); - - return pagemap & ~PM_PFRAME_MASK; -} - -static int check_page_flags(unsigned long pfn) -{ - int fd, i; - unsigned long pageflags; - - fd = open("/proc/kpageflags", O_RDONLY); - if (fd < 0) - return -1; - - lseek(fd, pfn * sizeof(pageflags), SEEK_SET); - - read(fd, &pageflags, sizeof(pageflags)); - if ((pageflags & HEAD_PAGE_FLAGS) != HEAD_PAGE_FLAGS) { - close(fd); - ksft_print_msg("Head page flags (%lx) is invalid\n", pageflags); - return -1; - } - - /* - * pages other than the first page must be tail and shouldn't be head; - * this also verifies kernel has correctly set the fake page_head to tail - * while hugetlb_free_vmemmap is enabled. - */ - for (i = 1; i < maplength / pagesize; i++) { - read(fd, &pageflags, sizeof(pageflags)); - if ((pageflags & TAIL_PAGE_FLAGS) != TAIL_PAGE_FLAGS || - (pageflags & HEAD_PAGE_FLAGS) == HEAD_PAGE_FLAGS) { - close(fd); - ksft_print_msg("Tail page flags (%lx) is invalid\n", pageflags); - return -1; - } - } - - close(fd); - - return 0; -} - -int main(int argc, char **argv) -{ - void *addr; - unsigned long pfn; - int ret; - - ksft_print_header(); - ksft_set_plan(1); - - if (!hugetlb_setup_default(1)) - ksft_exit_skip("Not enough free huge pages\n"); - - pagesize = psize(); - maplength = default_huge_page_size(); - if (!maplength) - ksft_exit_skip("Unable to determine huge page size\n"); - - addr = mmap(NULL, maplength, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); - if (addr == MAP_FAILED) - ksft_exit_fail_perror("mmap"); - - /* Trigger allocation of HugeTLB page. */ - write_bytes(addr, maplength); - - pfn = virt_to_pfn(addr); - if (pfn == -1UL) { - ksft_perror("virt_to_pfn"); - munmap(addr, maplength); - ksft_exit_fail(); - } - - ksft_print_msg("Returned address is %p whose pfn is %lx\n", addr, pfn); - - ret = check_page_flags(pfn); - - if (munmap(addr, maplength)) - ksft_exit_fail_perror("munmap"); - - ksft_test_result(!ret, "HugeTLB vmemmap page flags\n"); - ksft_finished(); -} diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index 8c296dedf047..a60b9f9f16e7 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -262,7 +262,6 @@ echo "TAP version 13" | tap_output CATEGORY="hugetlb" run_test ./hugetlb-mmap CATEGORY="hugetlb" run_test ./hugetlb-shm CATEGORY="hugetlb" run_test ./hugetlb-mremap -CATEGORY="hugetlb" run_test ./hugetlb-vmemmap CATEGORY="hugetlb" run_test ./hugetlb-madvise CATEGORY="hugetlb" run_test ./hugetlb_dio CATEGORY="hugetlb" run_test ./hugetlb_fault_after_madv From af56d6acf494f7a003737b59e785a4e8e5e5b66a Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Thu, 30 Jul 2026 09:45:52 +0000 Subject: [PATCH 368/501] mm/rmap: convert page -> folio for hwpoison checks Patch series "mm/rmap: Refactor try_to_unmap_one", v4. In preparation for batching anonymous large folio unmapping to optimize it, refactor try_to_unmap_one. This series refactors hugetlb, anon-lazyfree and anon-swapbacked logic into their own functions, significantly reducing the length of the huge try_to_unmap_one. This patch (of 5): try_to_unmap() receives hugetlb folios only from the hwpoison path. hugetlb_update_hwpoison() sets the hugetlb folio's head-page hwpoison bit, and page_vma_mapped_walk() reports the hugetlb mapping at the head PFN, so the previous PageHWPoison(subpage) check happened to work for hugetlb. For non-hugetlb folios, unmap_poisoned_folio() currently rejects large folios before calling try_to_unmap(). Hence it is always the case that if try_to_unmap_one() handles an hwpoisoned folio, then the head page is marked with the poison bit. Therefore, convert the poisoned subpage checks to folio_test_hwpoison(). No functional change intended, except that, while at it, convert VM_BUG_* to VM_WARN_*. Link: https://lore.kernel.org/20260730094559.418003-1-dev.jain@arm.com Link: https://lore.kernel.org/20260730094559.418003-2-dev.jain@arm.com Signed-off-by: Dev Jain Reviewed-by: Lorenzo Stoakes (ARM) Acked-by: David Hildenbrand (Arm) Reviewed-by: Anshuman Khandual Cc: Harry Yoo Cc: Jann Horn Cc: Lance Yang Cc: Liam R. Howlett Cc: Muchun Song Cc: Oscar Salvador Cc: Rik van Riel Cc: Ryan Roberts Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/rmap.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mm/rmap.c b/mm/rmap.c index e854679553b9..77be9fcffa99 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2122,10 +2122,11 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, bool anon = folio_test_anon(folio); /* - * The try_to_unmap() is only passed a hugetlb page - * in the case where the hugetlb page is poisoned. + * The try_to_unmap() is only passed a hugetlb folio + * in the case where the hugetlb folio contains a + * poisoned page. */ - VM_BUG_ON_PAGE(!PageHWPoison(subpage), subpage); + VM_WARN_ON_FOLIO(!folio_test_hwpoison(folio), folio); /* * huge_pmd_unshare may unmap an entire PMD page. * There is no way of knowing exactly which PMDs may @@ -2204,7 +2205,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, /* Update high watermark before we lower rss */ update_hiwater_rss(mm); - if (PageHWPoison(subpage) && (flags & TTU_HWPOISON)) { + /* unmap_poisoned_folio() only refs order-0 or hugetlb folios */ + if (folio_test_hwpoison(folio) && (flags & TTU_HWPOISON)) { pteval = swp_entry_to_pte(make_hwpoison_entry(subpage)); if (folio_test_hugetlb(folio)) { hugetlb_count_sub(folio_nr_pages(folio), mm); From d4ec5572825abdfd5d76e69573ff12609f0b7a36 Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Thu, 30 Jul 2026 09:45:53 +0000 Subject: [PATCH 369/501] mm/rmap: add try_to_unmap_poisoned_hugetlb_one Simplify try_to_unmap_one() by separating the hugetlb parts into try_to_unmap_poisoned_hugetlb_one(). To understand the correctness of the refactoring, the following points are noted: 1. try_to_unmap() is called for hugetlb folios only when they are hwpoisoned. 2. A hugetlb VMA cannot be mlocked. 3. page_vma_mapped_walk() returns at most one hugetlb mapping in a VMA, and that mapping points at the head PFN. 4. We won't ever process a softleaf entry that encodes a hugetlb folio; hugetlb folios are never swapped out, migration entries will be skipped (PVMW_MIGRATION not passed), and device-exclusive does not work for hugetlb. 5. The hwpoison entry is constructed from the poisoned folio, just as in the pre-refactor code. Any previous uffd-wp state is deliberately not preserved for the hwpoison entry. 6. TTU_HWPOISON is always present; for it to not be present, either the folio has to be in swapcache, or mapping_can_writeback() is true (see unmap_poisoned_folio), none of which is true for hugetlb folios. 7. Hugetlb uses separate counters from normal rss counters, therefore update_highwater_rss() need not be called. While at it: - Change VM_BUG_* to VM_WARN_*. - Do not declare variables which are only used once. - Constify some variables. - Add some more VM_WARN_* to assert some invariants. Except the above 4 points, no functional change intended. Link: https://lore.kernel.org/20260730094559.418003-3-dev.jain@arm.com Signed-off-by: Dev Jain Reviewed-by: Lorenzo Stoakes (ARM) Suggested-by: David Hildenbrand (Arm) Acked-by: David Hildenbrand (Arm) Cc: Anshuman Khandual Cc: Harry Yoo Cc: Jann Horn Cc: Lance Yang Cc: Liam R. Howlett Cc: Muchun Song Cc: Oscar Salvador Cc: Rik van Riel Cc: Ryan Roberts Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 1 + mm/rmap.c | 180 ++++++++++++++++++++++------------------ 2 files changed, 98 insertions(+), 83 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 6ca232a61643..16c4c4caa126 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -1270,6 +1270,7 @@ static inline void hugetlb_count_sub(long l, struct mm_struct *mm) } pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep); +unsigned long huge_pte_dirty(pte_t pte); static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) diff --git a/mm/rmap.c b/mm/rmap.c index 77be9fcffa99..9cb496d7b375 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1978,6 +1978,96 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio, FPB_RESPECT_WRITE | FPB_RESPECT_SOFT_DIRTY); } +static bool try_to_unmap_poisoned_hugetlb_one(struct folio *folio, + struct vm_area_struct *vma, unsigned long address, void *arg) +{ + DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); + const unsigned long hsz = huge_page_size(hstate_vma(vma)); + const enum ttu_flags flags = (enum ttu_flags)(long)arg; + struct mm_struct *mm = vma->vm_mm; + struct mmu_notifier_range range; + bool ret = true; + pte_t pteval; + + /* + * The try_to_unmap() is only passed a hugetlb folio in the case + * where the hugetlb folio is poisoned. + */ + VM_WARN_ON_ONCE_FOLIO(!folio_test_hwpoison(folio), folio); + VM_WARN_ON_ONCE(!(flags & TTU_HWPOISON)); + + range.end = vma_address_end(&pvmw); + mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, + address, range.end); + adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end); + mmu_notifier_invalidate_range_start(&range); + + /* There is only a single mapping in a VMA. */ + if (!page_vma_mapped_walk(&pvmw)) + goto range_end; + + VM_WARN_ON_ONCE(address != pvmw.address); + + pteval = huge_ptep_get(mm, address, pvmw.pte); + VM_WARN_ON_ONCE(!pte_present(pteval)); + VM_WARN_ON_ONCE(pte_pfn(pteval) != folio_pfn(folio)); + + /* + * huge_pmd_unshare may unmap an entire PMD page. There is no way of + * knowing exactly which PMDs may be cached for this mm, so we must + * flush them all. start/end were already adjusted above to cover this + * range. + */ + flush_cache_range(vma, range.start, range.end); + + /* + * To call huge_pmd_unshare, i_mmap_rwsem must be held in write mode. + * Caller needs to explicitly do this outside rmap routines. + * + * We also must hold hugetlb vma_lock in write mode. Lock order dictates + * acquiring vma_lock BEFORE i_mmap_rwsem. We can only try lock here and + * fail if unsuccessful. + */ + if (!folio_test_anon(folio)) { + struct mmu_gather tlb; + + VM_WARN_ON_ONCE(!(flags & TTU_RMAP_LOCKED)); + if (!hugetlb_vma_trylock_write(vma)) { + ret = false; + goto walk_done; + } + + tlb_gather_mmu_vma(&tlb, vma); + if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) { + hugetlb_vma_unlock_write(vma); + huge_pmd_unshare_flush(&tlb, vma); + tlb_finish_mmu(&tlb); + /* + * The PMD table was unmapped, consequently unmapping + * the folio. + */ + goto walk_done; + } + hugetlb_vma_unlock_write(vma); + tlb_finish_mmu(&tlb); + } + pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); + if (huge_pte_dirty(pteval)) + folio_mark_dirty(folio); + + pteval = swp_entry_to_pte(make_hwpoison_entry(folio_page(folio, 0))); + hugetlb_count_sub(folio_nr_pages(folio), mm); + set_huge_pte_at(mm, address, pvmw.pte, pteval, hsz); + hugetlb_remove_rmap(folio); + folio_put_refs(folio, 1); + +walk_done: + page_vma_mapped_walk_done(&pvmw); +range_end: + mmu_notifier_invalidate_range_end(&range); + return ret; +} + /* * @arg: enum ttu_flags will be passed to this argument */ @@ -1993,7 +2083,6 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, enum ttu_flags flags = (enum ttu_flags)(long)arg; unsigned long nr_pages = 1, end_addr; unsigned long pfn; - unsigned long hsz = 0; int ptes = 0; /* @@ -2007,8 +2096,6 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, /* * For THP, we have to assume the worse case ie pmd for invalidation. - * For hugetlb, it could be much worse if we need to do pud - * invalidation in the case of pmd sharing. * * Note that the folio can not be freed in this function as call of * try_to_unmap() must hold a reference on the folio. @@ -2016,17 +2103,6 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, range.end = vma_address_end(&pvmw); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, address, range.end); - if (folio_test_hugetlb(folio)) { - /* - * If sharing is possible, start and end will be adjusted - * accordingly. - */ - adjust_range_if_pmd_sharing_possible(vma, &range.start, - &range.end); - - /* We need the huge page size for set_huge_pte_at() */ - hsz = huge_page_size(hstate_vma(vma)); - } mmu_notifier_invalidate_range_start(&range); while (page_vma_mapped_walk(&pvmw)) { @@ -2111,66 +2187,13 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, const softleaf_t entry = softleaf_from_pte(pteval); pfn = softleaf_to_pfn(entry); - VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio); } subpage = folio_page(folio, pfn - folio_pfn(folio)); anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(subpage); - if (folio_test_hugetlb(folio)) { - bool anon = folio_test_anon(folio); - - /* - * The try_to_unmap() is only passed a hugetlb folio - * in the case where the hugetlb folio contains a - * poisoned page. - */ - VM_WARN_ON_FOLIO(!folio_test_hwpoison(folio), folio); - /* - * huge_pmd_unshare may unmap an entire PMD page. - * There is no way of knowing exactly which PMDs may - * be cached for this mm, so we must flush them all. - * start/end were already adjusted above to cover this - * range. - */ - flush_cache_range(vma, range.start, range.end); - - /* - * To call huge_pmd_unshare, i_mmap_rwsem must be - * held in write mode. Caller needs to explicitly - * do this outside rmap routines. - * - * We also must hold hugetlb vma_lock in write mode. - * Lock order dictates acquiring vma_lock BEFORE - * i_mmap_rwsem. We can only try lock here and fail - * if unsuccessful. - */ - if (!anon) { - struct mmu_gather tlb; - - VM_BUG_ON(!(flags & TTU_RMAP_LOCKED)); - if (!hugetlb_vma_trylock_write(vma)) - goto walk_abort; - - tlb_gather_mmu_vma(&tlb, vma); - if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) { - hugetlb_vma_unlock_write(vma); - huge_pmd_unshare_flush(&tlb, vma); - tlb_finish_mmu(&tlb); - /* - * The PMD table was unmapped, - * consequently unmapping the folio. - */ - goto walk_done; - } - hugetlb_vma_unlock_write(vma); - tlb_finish_mmu(&tlb); - } - pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); - if (pte_dirty(pteval)) - folio_mark_dirty(folio); - } else if (likely(pte_present(pteval))) { + if (likely(pte_present(pteval))) { nr_pages = folio_unmap_pte_batch(folio, &pvmw, flags, pteval); end_addr = address + nr_pages * PAGE_SIZE; flush_cache_range(vma, address, end_addr); @@ -2205,17 +2228,11 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, /* Update high watermark before we lower rss */ update_hiwater_rss(mm); - /* unmap_poisoned_folio() only refs order-0 or hugetlb folios */ + /* unmap_poisoned_folio() only refs order-0 folios */ if (folio_test_hwpoison(folio) && (flags & TTU_HWPOISON)) { pteval = swp_entry_to_pte(make_hwpoison_entry(subpage)); - if (folio_test_hugetlb(folio)) { - hugetlb_count_sub(folio_nr_pages(folio), mm); - set_huge_pte_at(mm, address, pvmw.pte, pteval, - hsz); - } else { - dec_mm_counter(mm, mm_counter(folio)); - set_pte_at(mm, address, pvmw.pte, pteval); - } + dec_mm_counter(mm, mm_counter(folio)); + set_pte_at(mm, address, pvmw.pte, pteval); } else if (likely(pte_present(pteval)) && pte_unused(pteval) && !userfaultfd_armed(vma)) { /* @@ -2343,11 +2360,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, add_mm_counter(mm, mm_counter_file(folio), -nr_pages); } discard: - if (unlikely(folio_test_hugetlb(folio))) { - hugetlb_remove_rmap(folio); - } else { - folio_remove_rmap_ptes(folio, subpage, nr_pages, vma); - } + folio_remove_rmap_ptes(folio, subpage, nr_pages, vma); if (vma->vm_flags & VM_LOCKED) mlock_drain_local(); folio_put_refs(folio, nr_pages); @@ -2395,7 +2408,8 @@ static int folio_not_mapped(struct folio *folio) void try_to_unmap(struct folio *folio, enum ttu_flags flags) { struct rmap_walk_control rwc = { - .rmap_one = try_to_unmap_one, + .rmap_one = folio_test_hugetlb(folio) ? + try_to_unmap_poisoned_hugetlb_one : try_to_unmap_one, .arg = (void *)flags, .done = folio_not_mapped, .anon_lock = folio_lock_anon_vma_read, From 0955037b980e7dcf3858b079e9610dbd584a6507 Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Thu, 30 Jul 2026 09:45:54 +0000 Subject: [PATCH 370/501] mm/rmap: refactor some code around lazyfree folio unmapping For lazyfree folio unmapping, after clearing the ptes we must abort the operation if the folio got dirtied or it has unexpected references. Refactor this logic into a function which will return whether we need to abort or not. If we abort, we restore the ptes and bail out of try_to_unmap_one. Otherwise adjust the rss stats of the mm and jump to a label. Also rename that label from "discard" to "finish_unmap"; the former is appropriate in the lazyfree context, but the code following the label is executed for other successful unmap code paths too, so 'discard' does not sound correct for them. No functional change intended. Link: https://lore.kernel.org/20260730094559.418003-4-dev.jain@arm.com Signed-off-by: Dev Jain Acked-by: David Hildenbrand (Arm) Cc: Anshuman Khandual Cc: Harry Yoo Cc: Jann Horn Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes (ARM) Cc: Muchun Song Cc: Oscar Salvador Cc: Rik van Riel Cc: Ryan Roberts Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/rmap.c | 87 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 48 insertions(+), 39 deletions(-) diff --git a/mm/rmap.c b/mm/rmap.c index 9cb496d7b375..dd1309d72e37 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2068,6 +2068,50 @@ static bool try_to_unmap_poisoned_hugetlb_one(struct folio *folio, return ret; } +static bool ttu_anon_lazyfree_folio(struct vm_area_struct *vma, + struct folio *folio, unsigned long nr_pages) +{ + int ref_count, map_count; + + /* + * Synchronize with gup_pte_range(): + * - clear PTE; barrier; read refcount + * - inc refcount; barrier; read PTE + */ + smp_mb(); + + ref_count = folio_ref_count(folio); + map_count = folio_mapcount(folio); + + /* + * Order reads for page refcount and dirty flag + * (see comments in __remove_mapping()). + */ + smp_rmb(); + + if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) { + /* + * redirtied either using the page table or a previously + * obtained GUP reference. + */ + folio_set_swapbacked(folio); + return false; + } + + /* + * Additional references could be due to GUP or speculative lookups. + * GUP users must mark the folio dirty if there was a modification. + * This folio cannot be reclaimed right now either way, so act just + * like nothing happened. We'll come back here later and detect if the + * folio was dirtied when the additional reference is gone. + */ + if (ref_count != 1 + map_count) + return false; + + add_mm_counter(vma->vm_mm, MM_ANONPAGES, -nr_pages); + return true; +} + /* * @arg: enum ttu_flags will be passed to this argument */ @@ -2261,47 +2305,12 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, /* MADV_FREE page check */ if (!folio_test_swapbacked(folio)) { - int ref_count, map_count; - - /* - * Synchronize with gup_pte_range(): - * - clear PTE; barrier; read refcount - * - inc refcount; barrier; read PTE - */ - smp_mb(); - - ref_count = folio_ref_count(folio); - map_count = folio_mapcount(folio); - - /* - * Order reads for page refcount and dirty flag - * (see comments in __remove_mapping()). - */ - smp_rmb(); - - if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) { - /* - * redirtied either using the page table or a previously - * obtained GUP reference. - */ - set_ptes(mm, address, pvmw.pte, pteval, nr_pages); - folio_set_swapbacked(folio); - goto walk_abort; - } else if (ref_count != 1 + map_count) { - /* - * Additional reference. Could be a GUP reference or any - * speculative reference. GUP users must mark the folio - * dirty if there was a modification. This folio cannot be - * reclaimed right now either way, so act just like nothing - * happened. - * We'll come back here later and detect if the folio was - * dirtied when the additional reference is gone. - */ + if (!ttu_anon_lazyfree_folio(vma, folio, + nr_pages)) { set_ptes(mm, address, pvmw.pte, pteval, nr_pages); goto walk_abort; } - add_mm_counter(mm, MM_ANONPAGES, -nr_pages); - goto discard; + goto finish_unmap; } if (folio_dup_swap(folio, subpage) < 0) { @@ -2359,7 +2368,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, */ add_mm_counter(mm, mm_counter_file(folio), -nr_pages); } -discard: +finish_unmap: folio_remove_rmap_ptes(folio, subpage, nr_pages, vma); if (vma->vm_flags & VM_LOCKED) mlock_drain_local(); From 0caf74274d858addf197a7a53ca506aa2cbbf2bc Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Thu, 30 Jul 2026 09:45:55 +0000 Subject: [PATCH 371/501] mm/rmap: refactor anon folio unmap in try_to_unmap_one Refactor anonymous swapbacked folio unmap to ttu_anon_swapbacked_folio(). While at it, rename the remaining subpage variables to page; folios have pages, not subpages. No functional change intended. Link: https://lore.kernel.org/20260730094559.418003-5-dev.jain@arm.com Signed-off-by: Dev Jain Acked-by: David Hildenbrand (Arm) Cc: Anshuman Khandual Cc: Harry Yoo Cc: Jann Horn Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes (ARM) Cc: Muchun Song Cc: Oscar Salvador Cc: Rik van Riel Cc: Ryan Roberts Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/rmap.c | 115 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 68 insertions(+), 47 deletions(-) diff --git a/mm/rmap.c b/mm/rmap.c index dd1309d72e37..15efde4d1fc9 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2112,6 +2112,66 @@ static bool ttu_anon_lazyfree_folio(struct vm_area_struct *vma, return true; } +static pte_t swp_pte_prepare(swp_entry_t entry, pte_t old_pte, + bool anon_exclusive) +{ + pte_t swp_pte = swp_entry_to_pte(entry); + + if (anon_exclusive) + swp_pte = pte_swp_mkexclusive(swp_pte); + + if (likely(pte_present(old_pte))) { + if (pte_soft_dirty(old_pte)) + swp_pte = pte_swp_mksoft_dirty(swp_pte); + if (pte_uffd(old_pte)) + swp_pte = pte_swp_mkuffd(swp_pte); + } else { + /* Device-exclusive entry */ + if (pte_swp_soft_dirty(old_pte)) + swp_pte = pte_swp_mksoft_dirty(swp_pte); + if (pte_swp_uffd(old_pte)) + swp_pte = pte_swp_mkuffd(swp_pte); + } + + return swp_pte; +} + +static bool ttu_anon_swapbacked_folio(struct vm_area_struct *vma, + struct folio *folio, struct page *page, unsigned long address, + pte_t *ptep, pte_t pteval) +{ + const bool anon_exclusive = folio_test_anon(folio) && + PageAnonExclusive(page); + swp_entry_t entry = page_swap_entry(page); + struct mm_struct *mm = vma->vm_mm; + + if (folio_dup_swap(folio, page) < 0) + return false; + + /* + * arch_unmap_one() is expected to be a NOP on + * architectures where we could have PFN swap PTEs, + * so we'll not check/care. + */ + if (arch_unmap_one(mm, vma, address, pteval) < 0) { + folio_put_swap(folio, page); + return false; + } + + /* See folio_try_share_anon_rmap(): clear PTE first. */ + if (anon_exclusive && folio_try_share_anon_rmap_pte(folio, page)) { + folio_put_swap(folio, page); + return false; + } + + mm_prepare_for_swap_entries(mm); + dec_mm_counter(mm, MM_ANONPAGES); + inc_mm_counter(mm, MM_SWAPENTS); + set_pte_at(mm, address, ptep, + swp_pte_prepare(entry, pteval, anon_exclusive)); + return true; +} + /* * @arg: enum ttu_flags will be passed to this argument */ @@ -2120,9 +2180,9 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, { struct mm_struct *mm = vma->vm_mm; DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); - bool anon_exclusive, ret = true; + bool ret = true; pte_t pteval; - struct page *subpage; + struct page *page; struct mmu_notifier_range range; enum ttu_flags flags = (enum ttu_flags)(long)arg; unsigned long nr_pages = 1, end_addr; @@ -2233,9 +2293,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, pfn = softleaf_to_pfn(entry); } - subpage = folio_page(folio, pfn - folio_pfn(folio)); - anon_exclusive = folio_test_anon(folio) && - PageAnonExclusive(subpage); + page = folio_page(folio, pfn - folio_pfn(folio)); if (likely(pte_present(pteval))) { nr_pages = folio_unmap_pte_batch(folio, &pvmw, flags, pteval); @@ -2274,7 +2332,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, /* unmap_poisoned_folio() only refs order-0 folios */ if (folio_test_hwpoison(folio) && (flags & TTU_HWPOISON)) { - pteval = swp_entry_to_pte(make_hwpoison_entry(subpage)); + pteval = swp_entry_to_pte(make_hwpoison_entry(page)); dec_mm_counter(mm, mm_counter(folio)); set_pte_at(mm, address, pvmw.pte, pteval); } else if (likely(pte_present(pteval)) && pte_unused(pteval) && @@ -2291,8 +2349,6 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, */ dec_mm_counter(mm, mm_counter(folio)); } else if (folio_test_anon(folio)) { - swp_entry_t entry = page_swap_entry(subpage); - pte_t swp_pte; /* * Store the swap location in the pte. * See handle_pte_fault() ... @@ -2313,47 +2369,12 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, goto finish_unmap; } - if (folio_dup_swap(folio, subpage) < 0) { + if (!ttu_anon_swapbacked_folio(vma, folio, page, address, + pvmw.pte, pteval)) { set_pte_at(mm, address, pvmw.pte, pteval); goto walk_abort; } - - /* - * arch_unmap_one() is expected to be a NOP on - * architectures where we could have PFN swap PTEs, - * so we'll not check/care. - */ - if (arch_unmap_one(mm, vma, address, pteval) < 0) { - folio_put_swap(folio, subpage); - set_pte_at(mm, address, pvmw.pte, pteval); - goto walk_abort; - } - - /* See folio_try_share_anon_rmap(): clear PTE first. */ - if (anon_exclusive && - folio_try_share_anon_rmap_pte(folio, subpage)) { - folio_put_swap(folio, subpage); - set_pte_at(mm, address, pvmw.pte, pteval); - goto walk_abort; - } - mm_prepare_for_swap_entries(mm); - dec_mm_counter(mm, MM_ANONPAGES); - inc_mm_counter(mm, MM_SWAPENTS); - swp_pte = swp_entry_to_pte(entry); - if (anon_exclusive) - swp_pte = pte_swp_mkexclusive(swp_pte); - if (likely(pte_present(pteval))) { - if (pte_soft_dirty(pteval)) - swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_uffd(pteval)) - swp_pte = pte_swp_mkuffd(swp_pte); - } else { - if (pte_swp_soft_dirty(pteval)) - swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_swp_uffd(pteval)) - swp_pte = pte_swp_mkuffd(swp_pte); - } - set_pte_at(mm, address, pvmw.pte, swp_pte); + goto finish_unmap; } else { /* * This is a locked file-backed folio, @@ -2369,7 +2390,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, add_mm_counter(mm, mm_counter_file(folio), -nr_pages); } finish_unmap: - folio_remove_rmap_ptes(folio, subpage, nr_pages, vma); + folio_remove_rmap_ptes(folio, page, nr_pages, vma); if (vma->vm_flags & VM_LOCKED) mlock_drain_local(); folio_put_refs(folio, nr_pages); From 04b09ca57d485cdd3c85f810856bed756965a092 Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Thu, 30 Jul 2026 09:45:56 +0000 Subject: [PATCH 372/501] mm/rmap: add anon folio unmap dispatcher Add ttu_anon_folio() as the common entry point for anonymous folio unmapping. It dispatches to the lazyfree or swapbacked helper as appropriate. Keep restoration of cleared PTEs in the caller, where the PTEs were removed. No functional change intended. Link: https://lore.kernel.org/20260730094559.418003-6-dev.jain@arm.com Signed-off-by: Dev Jain Acked-by: David Hildenbrand (Arm) Cc: Anshuman Khandual Cc: Harry Yoo Cc: Jann Horn Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes (ARM) Cc: Muchun Song Cc: Oscar Salvador Cc: Rik van Riel Cc: Ryan Roberts Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/rmap.c | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/mm/rmap.c b/mm/rmap.c index 15efde4d1fc9..867db2b61573 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2172,6 +2172,26 @@ static bool ttu_anon_swapbacked_folio(struct vm_area_struct *vma, return true; } +static bool ttu_anon_folio(struct vm_area_struct *vma, struct folio *folio, + struct page *page, unsigned long address, pte_t *ptep, + pte_t pteval, unsigned long nr_pages) +{ + /* + * Store the swap location in the pte. + * See handle_pte_fault() ... + */ + if (WARN_ON_ONCE(folio_test_swapbacked(folio) != + folio_test_swapcache(folio))) + return false; + + if (!folio_test_swapbacked(folio)) + return ttu_anon_lazyfree_folio(vma, folio, nr_pages); + + /* nr_pages > 1 not supported yet */ + return ttu_anon_swapbacked_folio(vma, folio, page, address, ptep, + pteval); +} + /* * @arg: enum ttu_flags will be passed to this argument */ @@ -2349,31 +2369,12 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, */ dec_mm_counter(mm, mm_counter(folio)); } else if (folio_test_anon(folio)) { - /* - * Store the swap location in the pte. - * See handle_pte_fault() ... - */ - if (unlikely(folio_test_swapbacked(folio) != - folio_test_swapcache(folio))) { - WARN_ON_ONCE(1); + if (!ttu_anon_folio(vma, folio, page, address, + pvmw.pte, pteval, nr_pages)) { + set_ptes(mm, address, pvmw.pte, pteval, nr_pages); goto walk_abort; } - /* MADV_FREE page check */ - if (!folio_test_swapbacked(folio)) { - if (!ttu_anon_lazyfree_folio(vma, folio, - nr_pages)) { - set_ptes(mm, address, pvmw.pte, pteval, nr_pages); - goto walk_abort; - } - goto finish_unmap; - } - - if (!ttu_anon_swapbacked_folio(vma, folio, page, address, - pvmw.pte, pteval)) { - set_pte_at(mm, address, pvmw.pte, pteval); - goto walk_abort; - } goto finish_unmap; } else { /* From 72f522bbf473f03a77e83547b677dc1505c5d41a Mon Sep 17 00:00:00 2001 From: Guopeng Zhang Date: Mon, 13 Jul 2026 16:50:53 +0800 Subject: [PATCH 373/501] mm: memcontrol: update state_local when flushing NMI stats flush_nmi_stats() updates state[] for kmem and slab counters but leaves the corresponding state_local[] counters unchanged. Local kmem and slab statistics therefore miss updates collected through the NMI-safe atomic path. Update state_local[] together with state[]. Link: https://lore.kernel.org/20260713085053.2916813-1-guopeng.zhang@linux.dev Fixes: 940b01fc8dc1 ("memcg: nmi safe memcg stats for specific archs") Signed-off-by: Guopeng Zhang Acked-by: Tao Cui Acked-by: Johannes Weiner Acked-by: Shakeel Butt Cc: Alexandre Ghiti Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin Cc: Vlastimil Babka Cc: Signed-off-by: Andrew Morton --- mm/memcontrol.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 22f55aeb94f3..02599b8b6bd5 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -4443,6 +4443,7 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, int index = memcg_stats_index(MEMCG_KMEM); memcg->vmstats->state[index] += kmem; + memcg->vmstats->state_local[index] += kmem; if (parent) parent->vmstats->state_pending[index] += kmem; } @@ -4460,9 +4461,11 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, int index = memcg_stats_index(NR_SLAB_RECLAIMABLE_B); lstats->state[index] += slab; + lstats->state_local[index] += slab; if (plstats) plstats->state_pending[index] += slab; memcg->vmstats->state[index] += slab; + memcg->vmstats->state_local[index] += slab; if (parent) parent->vmstats->state_pending[index] += slab; } @@ -4471,9 +4474,11 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, int index = memcg_stats_index(NR_SLAB_UNRECLAIMABLE_B); lstats->state[index] += slab; + lstats->state_local[index] += slab; if (plstats) plstats->state_pending[index] += slab; memcg->vmstats->state[index] += slab; + memcg->vmstats->state_local[index] += slab; if (parent) parent->vmstats->state_pending[index] += slab; } From 466dd08cdc93a35e5ac1b5485308d3134a7d308b Mon Sep 17 00:00:00 2001 From: Guopeng Zhang Date: Mon, 13 Jul 2026 16:55:20 +0800 Subject: [PATCH 374/501] mm: memcg-v1: account vmpressure event allocations Commit 72797d218b43 ("mm/memcg: v1: account event registrations and drop world-writable cgroup.event_control") accounted cgroup v1 event registration allocations with GFP_KERNEL_ACCOUNT, but missed struct vmpressure_event. Use GFP_KERNEL_ACCOUNT for this allocation as well. Link: https://lore.kernel.org/20260713085520.2953121-1-guopeng.zhang@linux.dev Fixes: 72797d218b43 ("mm/memcg: v1: account event registrations and drop world-writable cgroup.event_control") Signed-off-by: Guopeng Zhang Acked-by: Tao Cui Acked-by: Johannes Weiner Reviewed-by: Muchun Song Acked-by: Shakeel Butt Cc: Michal Hocko Cc: Roman Gushchin Signed-off-by: Andrew Morton --- mm/memcontrol-v1.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c index e8b6e1560278..f8424ec3734b 100644 --- a/mm/memcontrol-v1.c +++ b/mm/memcontrol-v1.c @@ -1721,7 +1721,7 @@ int vmpressure_register_event(struct mem_cgroup *memcg, mode = ret; } - ev = kzalloc_obj(*ev); + ev = kzalloc_obj(*ev, GFP_KERNEL_ACCOUNT); if (!ev) { ret = -ENOMEM; goto out; From 45b1ec4887bdcc58af885540d9fefa8b67d17d57 Mon Sep 17 00:00:00 2001 From: Guopeng Zhang Date: Mon, 13 Jul 2026 16:57:56 +0800 Subject: [PATCH 375/501] mm: memcg-v1: fix wrong linux-mm list address in deprecation warnings The deprecation warnings for memory.oom_control and memory.pressure_level use linux-mm-@kvack.org instead of the linux-mm mailing list address. Remove the extra hyphen. Link: https://lore.kernel.org/20260713085756.2973549-1-guopeng.zhang@linux.dev Signed-off-by: Guopeng Zhang Acked-by: Michal Hocko Acked-by: Johannes Weiner Reviewed-by: Muchun Song Acked-by: Shakeel Butt Cc: Roman Gushchin Cc: Signed-off-by: Andrew Morton --- mm/memcontrol-v1.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c index f8424ec3734b..e43b13208eca 100644 --- a/mm/memcontrol-v1.c +++ b/mm/memcontrol-v1.c @@ -1182,13 +1182,13 @@ static ssize_t memcg_write_event_control(struct kernfs_open_file *of, event->unregister_event = mem_cgroup_usage_unregister_event; } else if (!strcmp(name, "memory.oom_control")) { pr_warn_once("oom_control is deprecated and will be removed. " - "Please report your usecase to linux-mm-@kvack.org" + "Please report your usecase to linux-mm@kvack.org" " if you depend on this functionality.\n"); event->register_event = mem_cgroup_oom_register_event; event->unregister_event = mem_cgroup_oom_unregister_event; } else if (!strcmp(name, "memory.pressure_level")) { pr_warn_once("pressure_level is deprecated and will be removed. " - "Please report your usecase to linux-mm-@kvack.org " + "Please report your usecase to linux-mm@kvack.org " "if you depend on this functionality.\n"); event->register_event = vmpressure_register_event; event->unregister_event = vmpressure_unregister_event; @@ -2340,7 +2340,7 @@ static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css, struct mem_cgroup *memcg = mem_cgroup_from_css(css); pr_warn_once("oom_control is deprecated and will be removed. " - "Please report your usecase to linux-mm-@kvack.org if you " + "Please report your usecase to linux-mm@kvack.org if you " "depend on this functionality.\n"); /* cannot set to root cgroup and only 0 and 1 are allowed */ From 219a53f6c84bafd3871afc5b007e777112074e51 Mon Sep 17 00:00:00 2001 From: Guopeng Zhang Date: Mon, 13 Jul 2026 17:00:10 +0800 Subject: [PATCH 376/501] mm: memcontrol: drop unused cpu argument from flush_nmi_stats flush_nmi_stats() does not use its cpu argument. Remove it from the function and its !CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC stub. The caller still uses cpu for the subsequent per-CPU rstat flush. No functional change. Link: https://lore.kernel.org/20260713090010.2991906-1-guopeng.zhang@linux.dev Signed-off-by: Guopeng Zhang Acked-by: Michal Hocko Acked-by: Johannes Weiner Reviewed-by: Muchun Song Acked-by: Shakeel Butt Cc: Roman Gushchin Signed-off-by: Andrew Morton --- mm/memcontrol.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 02599b8b6bd5..5dc103a98e74 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -4433,8 +4433,7 @@ static void mem_cgroup_stat_aggregate(struct aggregate_control *ac) } #ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC -static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, - int cpu) +static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent) { int nid; @@ -4485,8 +4484,7 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, } } #else -static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, - int cpu) +static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent) {} #endif @@ -4498,7 +4496,7 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu) struct aggregate_control ac; int nid; - flush_nmi_stats(memcg, parent, cpu); + flush_nmi_stats(memcg, parent); statc = per_cpu_ptr(memcg->vmstats_percpu, cpu); From afe9ae3fda96d1ccfd28f7b0c1f68527a570ccd4 Mon Sep 17 00:00:00 2001 From: Guopeng Zhang Date: Mon, 13 Jul 2026 17:03:04 +0800 Subject: [PATCH 377/501] mm: memcontrol: factor out memcg kmem uncharge sequence The kmem-uncharge sequence (mod_memcg_state(MEMCG_KMEM) + memcg1_account_kmem + conditional memcg_uncharge) is duplicated verbatim in obj_cgroup_release() and drain_obj_stock_slot(). Factor it into a small memcg_uncharge_kmem() helper. The reference get/put stays at the call sites, as they differ. No functional change. Link: https://lore.kernel.org/20260713090304.3015329-1-guopeng.zhang@linux.dev Signed-off-by: Guopeng Zhang Acked-by: Tao Cui Acked-by: Johannes Weiner Reviewed-by: Muchun Song Acked-by: Shakeel Butt Cc: Michal Hocko Cc: Roman Gushchin Signed-off-by: Andrew Morton --- mm/memcontrol.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 5dc103a98e74..8319ad8c5c23 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -137,6 +137,14 @@ bool mem_cgroup_kmem_disabled(void) static void memcg_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages); +static void memcg_uncharge_kmem(struct mem_cgroup *memcg, unsigned int nr_pages) +{ + mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages); + memcg1_account_kmem(memcg, -nr_pages); + if (!mem_cgroup_is_root(memcg)) + memcg_uncharge(memcg, nr_pages); +} + static void obj_cgroup_release(struct percpu_ref *ref) { struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt); @@ -172,10 +180,7 @@ static void obj_cgroup_release(struct percpu_ref *ref) struct mem_cgroup *memcg; memcg = get_mem_cgroup_from_objcg(objcg); - mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages); - memcg1_account_kmem(memcg, -nr_pages); - if (!mem_cgroup_is_root(memcg)) - memcg_uncharge(memcg, nr_pages); + memcg_uncharge_kmem(memcg, nr_pages); mem_cgroup_put(memcg); } @@ -3329,10 +3334,7 @@ static void drain_obj_stock_slot(struct obj_stock_pcp *stock, int i) memcg = get_mem_cgroup_from_objcg(old); - mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages); - memcg1_account_kmem(memcg, -nr_pages); - if (!mem_cgroup_is_root(memcg)) - memcg_uncharge(memcg, nr_pages); + memcg_uncharge_kmem(memcg, nr_pages); css_put(&memcg->css); } From 08d36b191f337dac643ec3365cc5d2e56c9f2d64 Mon Sep 17 00:00:00 2001 From: Guopeng Zhang Date: Mon, 13 Jul 2026 17:37:37 +0800 Subject: [PATCH 378/501] mm: memcg-v1: make mem_cgroup_oom_notify_cb() return void Commit 7d74b06f240f ("memcg: use for_each_mem_cgroup") replaced the mem_cgroup_walk_tree() call in mem_cgroup_oom_notify() with for_each_mem_cgroup_tree(), but left mem_cgroup_oom_notify_cb() with the int return type required by the old callback interface. The function now has a single direct caller and no failure path. Make it return void. Link: https://lore.kernel.org/20260713093737.3299646-1-guopeng.zhang@linux.dev Signed-off-by: Guopeng Zhang Acked-by: Michal Hocko Acked-by: Johannes Weiner Reviewed-by: Muchun Song Reviewed-by: SJ Park Acked-by: Shakeel Butt Cc: Roman Gushchin Signed-off-by: Andrew Morton --- mm/memcontrol-v1.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c index e43b13208eca..2dc599484d00 100644 --- a/mm/memcontrol-v1.c +++ b/mm/memcontrol-v1.c @@ -752,7 +752,7 @@ static int compare_thresholds(const void *a, const void *b) return 0; } -static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg) +static void mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg) { struct mem_cgroup_eventfd_list *ev; @@ -762,7 +762,6 @@ static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg) eventfd_signal(ev->eventfd); spin_unlock(&memcg_oom_lock); - return 0; } static void mem_cgroup_oom_notify(struct mem_cgroup *memcg) From 169c0a1ec49483228bb8f00fdc905b7b1438ce1c Mon Sep 17 00:00:00 2001 From: Song Hu Date: Mon, 13 Jul 2026 17:46:48 +0800 Subject: [PATCH 379/501] mm/damon/tests/core-kunit: test damon_nr_accesses_mvsum() damon_nr_accesses_mvsum() wraps damon_mvsum() with the monitoring intervals of the context to compute the pseudo moving sum of a region's access frequency, with a special case for when the whole aggregation window remains. damon_mvsum() itself is already covered by damon_test_mvsum(), but the wrapper is not. Add a table-driven KUnit test that exercises the full-window-remaining boundary (with both reset and not-yet-reset nr_accesses), partially elapsed windows, and the no-window-remaining case. Link: https://lore.kernel.org/20260713094648.897239-1-husong@kylinos.cn Signed-off-by: Song Hu Reviewed-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/tests/core-kunit.h | 49 +++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 6ad73559dd8e..73cb8fcdf07e 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -677,6 +677,54 @@ static void damon_test_mvsum(struct kunit *test) } } +/* + * Test damon_nr_accesses_mvsum(), which wraps damon_mvsum() with the + * monitoring intervals of the context. With a sample interval of 1 and an + * aggregation interval of 10, an aggregation window is 10 sample intervals + * long. Each row below specifies the passed sample intervals, the next + * aggregation time in sample intervals, the current and last nr_accesses of a + * region, and the expected return value. + */ +static void damon_test_nr_accesses_mvsum(struct kunit *test) +{ + unsigned long input_expects[] = { + /* passed, next_aggr, nr_accesses, last_nr_accesses, expect */ + 0, 10, 5, 3, 3, /* full window left, unreset */ + 0, 10, 0, 7, 7, /* full window left, reset */ + 5, 10, 3, 10, 8, /* half window left */ + 8, 10, 3, 10, 5, /* 20% window left */ + 10, 10, 42, 49, 42, /* no window left */ + }; + struct damon_ctx *c = damon_new_ctx(); + struct damon_region *r; + int i; + + if (!c) + kunit_skip(test, "ctx alloc fail"); + + r = damon_new_region(0, 4096); + if (!r) { + damon_destroy_ctx(c); + kunit_skip(test, "region alloc fail"); + } + + c->attrs.sample_interval = 1; + c->attrs.aggr_interval = 10; + + for (i = 0; i < ARRAY_SIZE(input_expects); i += 5) { + c->passed_sample_intervals = input_expects[i]; + c->next_aggregation_sis = input_expects[i + 1]; + r->nr_accesses = input_expects[i + 2]; + r->last_nr_accesses = input_expects[i + 3]; + + KUNIT_EXPECT_EQ(test, (unsigned int)input_expects[i + 4], + damon_nr_accesses_mvsum(r, c)); + } + + damon_free_region(r); + damon_destroy_ctx(c); +} + static void damos_test_new_filter(struct kunit *test) { struct damos_filter *filter; @@ -1569,6 +1617,7 @@ static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_update_monitoring_result), KUNIT_CASE(damon_test_set_attrs), KUNIT_CASE(damon_test_mvsum), + KUNIT_CASE(damon_test_nr_accesses_mvsum), KUNIT_CASE(damos_test_new_filter), KUNIT_CASE(damos_test_commit_quota_goal), KUNIT_CASE(damos_test_commit_quota_goals), From 51f4bd13c387091abfde2b36060da666b5160e11 Mon Sep 17 00:00:00 2001 From: Yijia Wang Date: Mon, 13 Jul 2026 17:43:19 +0800 Subject: [PATCH 380/501] selftests: mincore: count file-mmap readahead on both sides check_file_mmap() faults a page in the middle of a file mapping and expects the mmap read-around path to make neighbouring pages resident. The test currently counts only pages after the faulted page. That misses valid read-around on systems with large base page sizes. On arm64 with 64K pages and the default 128K readahead setting, the read-around window is two pages wide and centred on the faulting page. Faulting page 32 makes pages 31 and 32 resident, so the forward-only scan from page 33 reports ra_pages == 0 even though a neighbouring page was brought in. Keep the existing readahead assertion, but count resident neighbouring pages on both sides of the faulted page. This fixes the 64K-page false failure without teaching the selftest to compute the expected readahead window from sysfs or other implementation details. Link: https://lore.kernel.org/20260713094319.771550-1-wangyijia.yeah@bytedance.com Signed-off-by: Yijia Wang Cc: "Matthew Wilcox (Oracle)" Cc: Jan Kara Cc: Muchun Song Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/mincore/mincore_selftest.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/mincore/mincore_selftest.c b/tools/testing/selftests/mincore/mincore_selftest.c index cdd022c1c497..18dfbf318b1f 100644 --- a/tools/testing/selftests/mincore/mincore_selftest.c +++ b/tools/testing/selftests/mincore/mincore_selftest.c @@ -242,8 +242,10 @@ TEST(check_file_mmap) } /* - * Touch a page in the middle of the mapping. We expect the next - * few pages (the readahead window) to be populated too. + * Touch a page in the middle of the mapping. We expect some + * surrounding pages (the readahead window) to be populated too. + * Depending on the page size and readahead setting, the pages may + * land before the faulted page rather than after it. */ addr[FILE_SIZE / 2] = 1; retval = mincore(addr, FILE_SIZE, vec); @@ -252,6 +254,12 @@ TEST(check_file_mmap) TH_LOG("Page not found in memory after use"); } + i = FILE_SIZE / 2 / page_size - 1; + while (i >= 0 && vec[i]) { + ra_pages++; + i--; + } + i = FILE_SIZE / 2 / page_size + 1; while (i < vec_size && vec[i]) { ra_pages++; From 98df164036bed307a16e7c124ad023c2c13c4b76 Mon Sep 17 00:00:00 2001 From: Injae Ryou Date: Mon, 13 Jul 2026 18:27:00 +0900 Subject: [PATCH 381/501] selftests/mm: fix on-fault-limit false failure under sudo-rs run_vmtests.sh runs on-fault-limit as the nobody user via "sudo -u nobody ./on-fault-limit", guarded by a check that nobody can access the binary ("sudo -u nobody ls ./on-fault-limit"). The guard resolves the relative path from the inherited working directory, which only requires search permission on the test directory itself. Classic sudo passes the relative path through to execve() the same way, so the two agree. However, sudo-rs (the default sudo implementation since Ubuntu 25.10) canonicalizes the command to an absolute path before executing it, which requires search permission on every ancestor directory. When the kernel tree lives under a private home directory (mode 0750, the Ubuntu default for new users since 21.04), the guard passes but the execution fails with "command not found", and the test is reported as a false FAIL: # running sudo -u nobody ./on-fault-limit sudo: './on-fault-limit': command not found # [FAIL] Wrap the command in "sh -c" so that sudo only resolves the shell binary, and the relative path is resolved by nobody's shell from the inherited working directory, matching what the guard checks. This is the only "sudo -u nobody" invocation in the script; uid, cwd, rlimits (including RLIMIT_MEMLOCK, which this test exercises) and the exit status are unchanged through sh. Verified on Ubuntu 26.04 (sudo-rs 0.2.13): the test now runs and passes instead of failing. Verified on Ubuntu 24.04 (sudo 1.9.15p5): behavior is unchanged. Link: https://lore.kernel.org/20260713092700.464376-1-injaeryou@gmail.com Fixes: 5d2146a3354f ("selftests/mm: skip mlock tests if nobody user can't read it") Signed-off-by: Injae Ryou Cc: Brendan Jackman Cc: Brendan Jackman Cc: David Hildenbrand Cc: Lorenzo Stoakes Cc: Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/run_vmtests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index a60b9f9f16e7..687d115e3bd8 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -302,7 +302,7 @@ CATEGORY="compaction" run_test ./compaction_test if command -v sudo &> /dev/null && sudo -u nobody ls ./on-fault-limit >/dev/null; then - CATEGORY="mlock" run_test sudo -u nobody ./on-fault-limit + CATEGORY="mlock" run_test sudo -u nobody sh -c ./on-fault-limit else echo "# SKIP ./on-fault-limit" fi From 20dab9d72acdd2288d8a966a8e318274b1e26254 Mon Sep 17 00:00:00 2001 From: Hongling Zeng Date: Mon, 13 Jul 2026 13:41:54 +0800 Subject: [PATCH 382/501] mm: huge_memory: fix kobject cleanup in thpsize_create error When kobject_init_and_add() fails, the kobject API requires calling kobject_put() to properly clean up the memory, not direct kfree(). According to the kobject API documentation, kobject_init_and_add() calls kobject_init() internally. If the subsequent kobject_add() fails, the kobject has still been initialized and must be cleaned up via the reference count mechanism (kobject_put), not direct kfree(). Direct kfree() leaves the kobject's internal state (including the reference count and kset membership) uncleaned, which can cause: - Memory leaks of kobject internal structures - Potential use-after-free if there are pending references - Inconsistent state with the rest of the error handling code This fix matches the pattern used elsewhere in the kernel and in the same function (err_put label) which correctly uses kobject_put(). Link: https://lore.kernel.org/20260713054154.120915-1-zenghongling@kylinos.cn Fixes: 3485b88390b0 ("mm: thp: introduce multi-size THP sysfs interface") Signed-off-by: Hongling Zeng Suggested-by: Baolin Wang Reviewed-by: Lorenzo Stoakes (ARM) Acked-by: Zi Yan Reviewed-by: Baolin Wang Cc: Barry Song Cc: David Hildenbrand Cc: Dev Jain Cc: Hongling Zeng Cc: Lance Yang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Signed-off-by: Andrew Morton --- mm/huge_memory.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index c615e81620b9..c61871b5a333 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -818,10 +818,8 @@ static struct thpsize *thpsize_create(int order, struct kobject *parent) ret = kobject_init_and_add(&thpsize->kobj, &thpsize_ktype, parent, "hugepages-%lukB", size); - if (ret) { - kfree(thpsize); - goto err; - } + if (ret) + goto err_put; ret = sysfs_add_group(&thpsize->kobj, &any_ctrl_attr_grp); From 4f93f0956f446e270222c58f821de7500a163229 Mon Sep 17 00:00:00 2001 From: Youngjun Park Date: Mon, 13 Jul 2026 13:50:14 +0900 Subject: [PATCH 383/501] mm/swap: clarify unreachable cond_resched() in swap_reclaim_full_clusters Commit 66366d291f66 ("mm/swap: add cond_resched() in swap_reclaim_full_clusters to prevent softlockup") added cond_resched() to prevent soft lockups during heavy swap reclaim. Currently, there are two call sites for this function: 1. swap_reclaim_work(): called with force=true in a workqueue context. 2. cluster_alloc_swap_entry(): called with force=false, holding a local_lock and potentially a global cluster spinlock (atomic context). In the second case, calling cond_resched() would normally cause a "scheduling while atomic" bug. However, it is currently safe because when force=false, 'to_scan' is initialized to 1. The loop decrements it to 0 and breaks before ever reaching cond_resched(). This implicit dependency is hard to notice and recently triggered a false positive in AI Sashiko review. Add a comment to explicitly clarify that cond_resched() is unreachable in atomic contexts. This improves readability and prevents future misuse if the loop logic or 'to_scan' initialization is modified. Link: https://sashiko.dev/#/patchset/20260713025644.170839-1-youngjun.park@lge.com?part=4 Link: https://lore.kernel.org/20260713045014.219653-1-youngjun.park@lge.com Signed-off-by: Youngjun Park Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: Kairui Song Cc: Kemeng Shi Cc: Nhat Pham Signed-off-by: Andrew Morton --- mm/swapfile.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mm/swapfile.c b/mm/swapfile.c index 775c385490a5..095d43838d4e 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1072,6 +1072,12 @@ static void swap_reclaim_full_clusters(struct swap_info_struct *si, bool force) swap_cluster_unlock(ci); if (to_scan <= 0) break; + + /* + * When 'force' is false, 'to_scan' is initialized to 1. + * The loop breaks above, making this cond_resched() unreachable + * in atomic contexts. + */ cond_resched(); } } From 824d80962cf277f1c02e781bb18684b588111612 Mon Sep 17 00:00:00 2001 From: Hongfu Li Date: Mon, 13 Jul 2026 11:32:09 +0800 Subject: [PATCH 384/501] selftests/mm: factor out hmm_buffer_alloc() to consolidate buffer setup Each HMM test open-codes the same buffer initialization sequence: allocate main buffer, assign file descriptor and size, allocate mirror buffer, then perform mmap mapping. Factor out this repeated logic into a standalone hmm_buffer_alloc() helper to eliminate ~35 open-coded copies. The new helper supports distinct mmap_size and mirror_size parameters to fit scenarios with THP alignment padding or per-page snapshot flags. It also exposes prot, flags and fd arguments, enabling support for MAP_SHARED, MAP_HUGETLB and file-backed mappings. Eliminates ~360 lines of redundant boilerplate code. Fixes a missing NULL pointer check bug in the hmm_buffer_alloc() previously used only by the migration benchmark, now subsumed by this new unified helper. Link: https://lore.kernel.org/20260713033209.280435-1-lihongfu@kylinos.cn Signed-off-by: Hongfu Li Cc: David Hildenbrand Cc: Jason Gunthorpe Cc: Leon Romanovsky Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/hmm-tests.c | 625 +++++++------------------ 1 file changed, 165 insertions(+), 460 deletions(-) diff --git a/tools/testing/selftests/mm/hmm-tests.c b/tools/testing/selftests/mm/hmm-tests.c index 5acb728666f8..e2642eca0d02 100644 --- a/tools/testing/selftests/mm/hmm-tests.c +++ b/tools/testing/selftests/mm/hmm-tests.c @@ -228,6 +228,53 @@ static void hmm_buffer_free(struct hmm_buffer *buffer) free(buffer); } +/* + * Allocate a buffer structure with memory mapping and mirror. + * + * @mmap_size: total size of the mmap region (may differ from @mirror_size + * for alignment padding in THP tests). + * @mirror_size: size of the mirror data buffer (the actual working set). + * @prot: protection flags for the mmap (e.g. PROT_READ | PROT_WRITE). + * @flags: flags for the mmap (e.g. MAP_PRIVATE, MAP_SHARED, + * MAP_ANONYMOUS, MAP_HUGETLB). + * @fd: file descriptor for the mmap; pass -1 for MAP_ANONYMOUS. + * + * All internal allocations are checked; returns NULL and cleans up on any + * failure. Caller must ASSERT_NE or otherwise check the return value. + */ +static struct hmm_buffer *hmm_buffer_alloc(unsigned long mmap_size, + unsigned long mirror_size, + int prot, int flags, + int fd) +{ + struct hmm_buffer *buffer; + + buffer = malloc(sizeof(*buffer)); + if (!buffer) { + perror("malloc buffer"); + return NULL; + } + + buffer->fd = fd; + buffer->size = mmap_size; + buffer->mirror = malloc(mirror_size); + if (!buffer->mirror) { + perror("malloc mirror"); + free(buffer); + return NULL; + } + + buffer->ptr = mmap(NULL, mmap_size, prot, flags, fd, 0); + if (buffer->ptr == MAP_FAILED) { + perror("mmap"); + free(buffer->mirror); + free(buffer); + return NULL; + } + + return buffer; +} + /* * Create a temporary file that will be deleted on close. */ @@ -318,20 +365,11 @@ TEST_F(hmm, anon_read) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* * Initialize buffer in system memory but leave the first two pages * zero (pte_none and pfn_zero). @@ -381,20 +419,11 @@ TEST_F(hmm, anon_read_prot) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -440,20 +469,11 @@ TEST_F(hmm, anon_write) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize data that the device will write to buffer->ptr. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -488,20 +508,11 @@ TEST_F(hmm, anon_write_prot) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Simulate a device reading a zero page of memory. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1); ASSERT_EQ(ret, 0); @@ -563,20 +574,10 @@ TEST_F(hmm, anon_write_child) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size * 2, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = size * 2; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size * 2, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - old_ptr = buffer->ptr; if (use_thp) { map = (void *)ALIGN((uintptr_t)buffer->ptr, size); @@ -665,20 +666,11 @@ TEST_F(hmm, anon_write_child_shared) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer->ptr so we can tell if it is written. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -740,20 +732,10 @@ TEST_F(hmm, anon_write_huge) size = 2 * read_pmd_pagesize(); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - size /= 2; npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); @@ -799,22 +781,11 @@ TEST_F(hmm, anon_write_hugetlbfs) size = ALIGN(TWOMEG, default_hsize); npages = size >> self->page_shift; - buffer = malloc(sizeof(*buffer)); - ASSERT_NE(buffer, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, - -1, 0); - if (buffer->ptr == MAP_FAILED) { - free(buffer); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1); + if (!buffer) SKIP(return, "Huge page could not be allocated"); - } - - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); /* Initialize data that the device will write to buffer->ptr. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) @@ -856,14 +827,11 @@ TEST_F(hmm, file_read) fd = hmm_create_file(size); ASSERT_GE(fd, 0); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ, + MAP_SHARED, fd); ASSERT_NE(buffer, NULL); - buffer->fd = fd; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - /* Write initial contents of the file. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -871,12 +839,6 @@ TEST_F(hmm, file_read) ASSERT_EQ(len, size); memset(buffer->mirror, 0, size); - buffer->ptr = mmap(NULL, size, - PROT_READ, - MAP_SHARED, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Simulate a device reading system memory. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); ASSERT_EQ(ret, 0); @@ -911,20 +873,11 @@ TEST_F(hmm, file_write) fd = hmm_create_file(size); ASSERT_GE(fd, 0); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_SHARED, fd); ASSERT_NE(buffer, NULL); - buffer->fd = fd; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_SHARED, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize data that the device will write to buffer->ptr. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -964,20 +917,11 @@ TEST_F(hmm, migrate) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1014,20 +958,11 @@ TEST_F(hmm, migrate_file_private) fd = hmm_create_file(size); ASSERT_GE(fd, 0); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE, fd); ASSERT_NE(buffer, NULL); - buffer->fd = fd; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1062,20 +997,11 @@ TEST_F(hmm, migrate_fault) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1118,18 +1044,11 @@ TEST_F(hmm, migrate_release) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1168,20 +1087,11 @@ TEST_F(hmm, migrate_shared) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Migrate memory to device. */ ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); ASSERT_EQ(ret, -ENOENT); @@ -1205,20 +1115,10 @@ TEST_F(hmm2, migrate_mixed) npages = 6; size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - /* Reserve a range of addresses. */ - buffer->ptr = mmap(NULL, size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); p = buffer->ptr; /* Migrating a protected area should be an error. */ @@ -1299,20 +1199,11 @@ TEST_F(hmm, migrate_multiple) size = npages << self->page_shift; for (c = 0; c < NTIMES; c++) { - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1358,20 +1249,11 @@ TEST_F(hmm, anon_read_multiple) size = npages << self->page_shift; for (c = 0; c < NTIMES; c++) { - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i + c; @@ -1424,20 +1306,11 @@ TEST_F(hmm, anon_teardown) int *ptr; int rc; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i + c; @@ -1478,22 +1351,11 @@ TEST_F(hmm, mixedmap) npages = 1; size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, npages, + PROT_READ | PROT_WRITE, + MAP_PRIVATE, self->fd); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(npages); - ASSERT_NE(buffer->mirror, NULL); - - - /* Reserve a range of addresses. */ - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE, - self->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Simulate a device snapshotting CPU pagetables. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); ASSERT_EQ(ret, 0); @@ -1523,20 +1385,10 @@ TEST_F(hmm2, snapshot) npages = 7; size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, npages, + PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(npages); - ASSERT_NE(buffer->mirror, NULL); - - /* Reserve a range of addresses. */ - buffer->ptr = mmap(NULL, size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); p = buffer->ptr; /* Punch a hole after the first page address. */ @@ -1630,21 +1482,11 @@ TEST_F(hmm, compound) size = ALIGN(TWOMEG, default_hsize); npages = size >> self->page_shift; - buffer = malloc(sizeof(*buffer)); - ASSERT_NE(buffer, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, - -1, 0); - if (buffer->ptr == MAP_FAILED) { - free(buffer); - return; - } - - buffer->size = size; - buffer->mirror = malloc(npages); - ASSERT_NE(buffer->mirror, NULL); + buffer = hmm_buffer_alloc(size, npages, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1); + if (!buffer) + SKIP(return, "Huge page could not be allocated"); /* Initialize the pages the device will snapshot in buffer->ptr. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) @@ -1711,21 +1553,11 @@ TEST_F(hmm2, double_map) npages = 6; size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - /* Reserve a range of addresses. */ - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1787,20 +1619,11 @@ TEST_F(hmm, exclusive) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1841,20 +1664,11 @@ TEST_F(hmm, exclusive_mprotect) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1896,20 +1710,11 @@ TEST_F(hmm, exclusive_cow) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1997,20 +1802,11 @@ TEST_F(hmm, hmm_gup_test) npages = 4; size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -2083,20 +1879,11 @@ TEST_F(hmm, hmm_cow_in_device) npages = 4; size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -2156,21 +1943,12 @@ TEST_F(hmm, migrate_anon_huge_empty) size = read_pmd_pagesize(); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(2 * size, size, + PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 2 * size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, size); - buffer->ptr = mmap(NULL, 2 * size, - PROT_READ, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); ret = madvise(map, size, MADV_HUGEPAGE); @@ -2208,21 +1986,12 @@ TEST_F(hmm, migrate_anon_huge_zero) size = read_pmd_pagesize(); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(2 * size, size, + PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 2 * size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, size); - buffer->ptr = mmap(NULL, 2 * size, - PROT_READ, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); ret = madvise(map, size, MADV_HUGEPAGE); @@ -2271,21 +2040,12 @@ TEST_F(hmm, migrate_anon_huge_free) size = read_pmd_pagesize(); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(2 * size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 2 * size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, size); - buffer->ptr = mmap(NULL, 2 * size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); ret = madvise(map, size, MADV_HUGEPAGE); @@ -2333,21 +2093,12 @@ TEST_F(hmm, migrate_anon_huge_fault) size = read_pmd_pagesize(); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(2 * size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 2 * size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, size); - buffer->ptr = mmap(NULL, 2 * size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); old_ptr = buffer->ptr; @@ -2429,21 +2180,11 @@ TEST_F(hmm, migrate_partial_unmap_fault) for (use_thp = 0; use_thp < 2; ++use_thp) { for (j = 0; j < ARRAY_SIZE(offsets); ++j) { - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(2 * size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 2 * size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, size); - - buffer->ptr = mmap(NULL, 2 * size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); if (use_thp) @@ -2509,21 +2250,11 @@ TEST_F(hmm, migrate_remap_fault) if (dont_unmap) flags |= MREMAP_DONTUNMAP; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(8 * size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 8 * size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, size); - - buffer->ptr = mmap(NULL, buffer->size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); if (use_thp) @@ -2594,18 +2325,13 @@ TEST_F(hmm, migrate_anon_huge_err) size = read_pmd_pagesize(); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(2 * size, 2 * size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 2 * size; - buffer->mirror = malloc(2 * size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, 2 * size); - old_ptr = mmap(NULL, 2 * size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); - ASSERT_NE(old_ptr, MAP_FAILED); + old_ptr = buffer->ptr; npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)old_ptr, size); @@ -2689,18 +2415,13 @@ TEST_F(hmm, migrate_anon_huge_zero_err) size = read_pmd_pagesize(); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(2 * size, 2 * size, + PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 2 * size; - buffer->mirror = malloc(2 * size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, 2 * size); - old_ptr = mmap(NULL, 2 * size, PROT_READ, - MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); - ASSERT_NE(old_ptr, MAP_FAILED); + old_ptr = buffer->ptr; npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)old_ptr, size); @@ -2770,19 +2491,6 @@ static double get_time_ms(void) return (tv.tv_sec * 1000.0) + (tv.tv_usec / 1000.0); } -static inline struct hmm_buffer *hmm_buffer_alloc(unsigned long size) -{ - struct hmm_buffer *buffer; - - buffer = malloc(sizeof(*buffer)); - - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - memset(buffer->mirror, 0xFF, size); - return buffer; -} - static void print_benchmark_results(const char *test_name, size_t buffer_size, struct benchmark_results *thp, struct benchmark_results *regular) @@ -2827,17 +2535,14 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz int ret, i; int *ptr; - buffer = hmm_buffer_alloc(buffer_size); - - /* Map memory */ - buffer->ptr = mmap(NULL, buffer_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - - if (buffer->ptr == MAP_FAILED) { - buffer->ptr = NULL; + buffer = hmm_buffer_alloc(buffer_size, buffer_size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); + if (!buffer) { ret = -1; goto cleanup; } + memset(buffer->mirror, 0xFF, buffer_size); /* Apply THP hint if requested */ if (use_thp) From 0250dbe08c730d003ef9f484da56ae09a1ea0c4c Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 12 Jul 2026 09:54:30 -0700 Subject: [PATCH 385/501] mm/damon/core: skip aging from repeated aggressive merging The number of DAMON regions could temporarily exceed the user-defined maximum number of regions limit for corner cases. For example, users could lower the limit via runtime parameters update. For such a case, kdamond_merge_regions() repeats merging regions in the case doubling the merge threshold. The repeated merge operation could update the age of regions multiple times. This corrupts the monitoring results. Fix the issue by asking the merge operation to skip aging for the corner case. The user impact is degradation of the monitoring quality. The impact should be mild, since the degradation is only temporal, and it is not common to happen in realistic setups. The issue was discovered [1,2] by Sashiko. Link: https://lore.kernel.org/20260712165432.87609-1-sj@kernel.org Link: https://lore.kernel.org/20260621203548.10718-1-sj@kernel.org [1] Link: https://lore.kernel.org/20260709145425.96247-1-sj@kernel.org [2] Fixes: 310d6c15e910 ("mm/damon/core: merge regions aggressively when max_nr_regions is unmet") Signed-off-by: SJ Park Cc: # 6.10 Signed-off-by: Andrew Morton --- mm/damon/core.c | 21 +++++++++++++-------- mm/damon/tests/core-kunit.h | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 806a67d02a6e..6c4215cc809e 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -3308,7 +3308,7 @@ static unsigned int damon_merge_score(struct damon_region *r, bool last, * sz_limit size upper limit of each region */ static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, - unsigned long sz_limit, struct damon_ctx *ctx) + unsigned long sz_limit, struct damon_ctx *ctx, bool count_age) { struct damon_region *r, *prev = NULL, *next; bool use_probe_hits = damon_has_probe_weights(ctx); @@ -3319,12 +3319,14 @@ static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, score = damon_merge_score(r, false, ctx, use_probe_hits); last_score = damon_merge_score(r, true, ctx, use_probe_hits); - if (abs_diff(score, last_score) > thres) - r->age = 0; - else if ((score == 0) != (last_score == 0)) - r->age = 0; - else - r->age++; + if (count_age) { + if (abs_diff(score, last_score) > thres) + r->age = 0; + else if ((score == 0) != (last_score == 0)) + r->age = 0; + else + r->age++; + } if (!prev) goto set_prev_continue; @@ -3366,15 +3368,18 @@ static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, struct damon_target *t; unsigned int nr_regions; unsigned int max_thres; + bool count_age = true; max_thres = c->attrs.aggr_interval / (c->attrs.sample_interval ? c->attrs.sample_interval : 1); do { nr_regions = 0; damon_for_each_target(t, c) { - damon_merge_regions_of(t, threshold, sz_limit, c); + damon_merge_regions_of(t, threshold, sz_limit, c, + count_age); nr_regions += damon_nr_regions(t); } + count_age = false; threshold = max(1, threshold * 2); } while (nr_regions > c->attrs.max_nr_regions && threshold / 2 < max_thres); diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 73cb8fcdf07e..485472ddebd1 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -257,7 +257,7 @@ static void damon_test_merge_regions_of(struct kunit *test) damon_add_region(r, t); } - damon_merge_regions_of(t, 9, 9999, ctx); + damon_merge_regions_of(t, 9, 9999, ctx, true); /* 0-112, 114-130, 130-156, 156-170, 170-230, 230-10170 */ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 6u); for (i = 0; i < 6; i++) { From 1b492fc82368399123413c937f13da6ed312ee4d Mon Sep 17 00:00:00 2001 From: Song Hu Date: Tue, 14 Jul 2026 07:01:13 -0700 Subject: [PATCH 386/501] Docs/ABI/damon: fix typo in intervals_goal sysfs path Patch series "Docs/ABI/damon: sysfs ABI document fixes and additions", v2. This series fixes typos and fills in missing entries in the DAMON sysfs ABI document (Documentation/ABI/testing/sysfs-kernel-mm-damon). Patch 1 fixes a path typo, "intrvals_goal" -> "intervals_goal", in four What: entries; the documented path points to a non-existent directory, so it is Cc'ed to stable. Patch 2 fixes two further typos ("WDate:", "manimum"). Patches 3 and 4 add ABI entries that exist in the kernel and are already described in usage.rst but are missing from the canonical ABI document: the 'update_tuned_intervals' state command (patch 3) and the 'tried_regions//probes/

/hits' file (patch 4). This patch (of 4): The ABI document spells the DAMON sysfs directory as "intrvals_goal" (missing 'e') in four What: entries, but the kernel creates it as "intervals_goal" (mm/damon/sysfs.c). Following the documented path therefore yields a non-existent directory. Link: https://lore.kernel.org/20260714140117.94147-1-sj@kernel.org Link: https://lore.kernel.org/20260714140117.94147-2-sj@kernel.org Fixes: e2b23dc62369 ("Docs/ABI/damon: document intervals auto-tuning ABI") Signed-off-by: Song Hu Reviewed-by: SJ Park Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Jonathan Corbet Cc: Signed-off-by: Andrew Morton --- Documentation/ABI/testing/sysfs-kernel-mm-damon | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-damon b/Documentation/ABI/testing/sysfs-kernel-mm-damon index 907a504fb64c..a8269123b423 100644 --- a/Documentation/ABI/testing/sysfs-kernel-mm-damon +++ b/Documentation/ABI/testing/sysfs-kernel-mm-damon @@ -112,7 +112,7 @@ Description: Writing a value to this file sets the update interval of the DAMON context in microseconds as the value. Reading this file returns the value. -What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intrvals_goal/access_bp +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intervals_goal/access_bp Date: Feb 2025 Contact: SJ Park Description: Writing a value to this file sets the monitoring intervals @@ -120,7 +120,7 @@ Description: Writing a value to this file sets the monitoring intervals the given time interval (aggrs in same directory), in bp (1/10,000). Reading this file returns the value. -What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intrvals_goal/aggrs +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intervals_goal/aggrs Date: Feb 2025 Contact: SJ Park Description: Writing a value to this file sets the time interval to achieve @@ -128,14 +128,14 @@ Description: Writing a value to this file sets the time interval to achieve access events ratio (access_bp in same directory) within. Reading this file returns the value. -What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intrvals_goal/min_sample_us +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intervals_goal/min_sample_us Date: Feb 2025 Contact: SJ Park Description: Writing a value to this file sets the minimum value of auto-tuned sampling interval in microseconds. Reading this file returns the value. -What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intrvals_goal/max_sample_us +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intervals_goal/max_sample_us Date: Feb 2025 Contact: SJ Park Description: Writing a value to this file sets the maximum value of From 732c07c4d33eae6cd6d36eca156a167d6b067700 Mon Sep 17 00:00:00 2001 From: Song Hu Date: Tue, 14 Jul 2026 07:01:14 -0700 Subject: [PATCH 387/501] Docs/ABI/damon: fix typos Fix two typos in the DAMON sysfs ABI document: - "WDate:" -> "Date:" on the nr_regions/min entry - "manimum" -> "minimum" in the nr_accesses/min description Link: https://lore.kernel.org/20260714140117.94147-3-sj@kernel.org Signed-off-by: Song Hu Reviewed-by: SJ Park Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- Documentation/ABI/testing/sysfs-kernel-mm-damon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-damon b/Documentation/ABI/testing/sysfs-kernel-mm-damon index a8269123b423..f3c99c79b0f5 100644 --- a/Documentation/ABI/testing/sysfs-kernel-mm-damon +++ b/Documentation/ABI/testing/sysfs-kernel-mm-damon @@ -144,7 +144,7 @@ Description: Writing a value to this file sets the maximum value of What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/nr_regions/min -WDate: Mar 2022 +Date: Mar 2022 Contact: SJ Park Description: Writing a value to this file sets the minimum number of monitoring regions of the DAMON context as the value. Reading @@ -287,7 +287,7 @@ Description: Writing to and reading from this file sets and gets the maximum What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//access_pattern/nr_accesses/min Date: Mar 2022 Contact: SJ Park -Description: Writing to and reading from this file sets and gets the manimum +Description: Writing to and reading from this file sets and gets the minimum 'nr_accesses' of the scheme's target regions. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//access_pattern/nr_accesses/max From 66a0dae14a3c3dd60f7d03fd9c7a242c4cd97f2b Mon Sep 17 00:00:00 2001 From: Song Hu Date: Tue, 14 Jul 2026 07:01:15 -0700 Subject: [PATCH 388/501] Docs/ABI/damon: document update_tuned_intervals state command The 'update_tuned_intervals' command of the 'state' file, added in commit 1077605396b4 ("mm/damon/sysfs: implement a command to update auto-tuned monitoring intervals"), is described in usage.rst but missing from the canonical ABI document. Add it. Link: https://lore.kernel.org/20260714140117.94147-4-sj@kernel.org Signed-off-by: Song Hu Reviewed-by: SJ Park Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- Documentation/ABI/testing/sysfs-kernel-mm-damon | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-damon b/Documentation/ABI/testing/sysfs-kernel-mm-damon index f3c99c79b0f5..1507e63ab171 100644 --- a/Documentation/ABI/testing/sysfs-kernel-mm-damon +++ b/Documentation/ABI/testing/sysfs-kernel-mm-damon @@ -36,7 +36,10 @@ Description: Writing 'on' or 'off' to this file makes the kdamond starts or kdamond. Writing 'clear_schemes_tried_regions' to the file removes contents of the 'tried_regions' directory. Writing 'update_schemes_effective_quotas' to the file updates - '.../quotas/effective_bytes' files of this kdamond. + '.../quotas/effective_bytes' files of this kdamond. Writing + 'update_tuned_intervals' to the file updates 'sample_us' and + 'aggr_us' files under 'monitoring_attrs/intervals' with the + auto-tuned sampling and aggregation intervals. What: /sys/kernel/mm/damon/admin/kdamonds//pid Date: Mar 2022 From aace7a1f5f2032584ff57d8ca59ece4c803bb06b Mon Sep 17 00:00:00 2001 From: Song Hu Date: Tue, 14 Jul 2026 07:01:16 -0700 Subject: [PATCH 389/501] Docs/ABI/damon: document tried_regions probe hits The 'tried_regions//probes/

/hits' file, added in commit 5b0de1bc3325 ("mm/damon/sysfs-schemes: implement probe/hits file"), is described in usage.rst but missing from the canonical ABI document. Document the 'probes/' directory and the 'probes/

/hits' file. Link: https://lore.kernel.org/20260714140117.94147-5-sj@kernel.org Signed-off-by: Song Hu Reviewed-by: SJ Park Signed-off-by: SJ Park Cc: David Hildenbrand Cc: Jonathan Corbet Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- Documentation/ABI/testing/sysfs-kernel-mm-damon | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-damon b/Documentation/ABI/testing/sysfs-kernel-mm-damon index 1507e63ab171..e675a57145e3 100644 --- a/Documentation/ABI/testing/sysfs-kernel-mm-damon +++ b/Documentation/ABI/testing/sysfs-kernel-mm-damon @@ -650,3 +650,15 @@ Contact: SJ Park Description: Reading this file returns the size of the memory in the region that passed DAMON operations layer-handled filters of the scheme in bytes. + +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//tried_regions//probes/ +Date: May 2026 +Contact: SJ Park +Description: Directory for DAMON data-attribute monitoring probes of the + region. + +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//tried_regions//probes/

/hits +Date: May 2026 +Contact: SJ Park +Description: Reading this file returns the number of data-attribute + monitoring probe-hit positive samples of the region. From 463bd5b83b5da0d143b9c88a417a2121d5e333d6 Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Sun, 12 Jul 2026 11:44:55 -0400 Subject: [PATCH 390/501] mm/memory: add memory_block_aligned_range() helper Patch series "dax/kmem: atomic whole-device hotplug via sysfs", v7. The dax kmem driver onlines memory during probe using the system default policy, with no atomic control for the state of an entire region at runtime - only by toggling individual memory blocks. Offlining and removing a whole region therefore races with other userland controllers that interfere between the two steps. This series adds a sysfs "state" attribute for atomic whole-device hotplug control, plus the mm and dax plumbing to support it. Transitions are atomic across every range of the device. The state names mirror the per-block memoryX/state ABI with one modification: - "unplugged": memory blocks are not present - "online": online as system RAM, zone chosen by the kernel - "online_kernel": online in ZONE_NORMAL - "online_movable": online in ZONE_MOVABLE "offline" (blocks present but offline) is reportable for backward compatibility but is not writable because it entices the race condition we are trying to solve (separate atomic steps for offline and unplug). 'unplugged' (atomic offline+remove of the whole device) is the new capability provided by the new kmem sysfs attribute. dax/kmem probe still creates the memory blocks by default when the default policy is "offline", to preserve backwards compatibility. This patch (of 10): Memory hotplug operations require ranges aligned to memory block boundaries. This is a generic operation for hotplug. Add memory_block_aligned_range() as a common helper in that aligns the start address up and end address down to memory block boundaries. Guard against end underflow when the range falls below the first memory block boundary, returning an empty range instead. Update dax/kmem to use this helper. Link: https://lore.kernel.org/20260712154505.3564379-1-gourry@gourry.net Link: https://lore.kernel.org/20260712154505.3564379-2-gourry@gourry.net Signed-off-by: Gregory Price Reviewed-by: Dave Jiang Reviewed-by: Dan Williams Acked-by: David Hildenbrand (Arm) Cc: Alison Schofield Cc: Danilo Krummrich Cc: Greg Kroah-Hartman Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Oscar Salvador Cc: "Rafael J. Wysocki" Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vishal Verma Cc: Vlastimil Babka Cc: Hannes Reinecke Cc: Pankaj Gupta Signed-off-by: Andrew Morton --- drivers/dax/kmem.c | 4 +--- include/linux/memory.h | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c index a18e2b968e4d..592171ec10f4 100644 --- a/drivers/dax/kmem.c +++ b/drivers/dax/kmem.c @@ -33,9 +33,7 @@ static int dax_kmem_range(struct dev_dax *dev_dax, int i, struct range *r) struct dev_dax_range *dax_range = &dev_dax->ranges[i]; struct range *range = &dax_range->range; - /* memory-block align the hotplug range */ - r->start = ALIGN(range->start, memory_block_size_bytes()); - r->end = ALIGN_DOWN(range->end + 1, memory_block_size_bytes()) - 1; + *r = memory_block_aligned_range(range); if (r->start >= r->end) { r->start = range->start; r->end = range->end; diff --git a/include/linux/memory.h b/include/linux/memory.h index 463dc02f6cff..1783299073e4 100644 --- a/include/linux/memory.h +++ b/include/linux/memory.h @@ -20,6 +20,7 @@ #include #include #include +#include #define MIN_MEMORY_BLOCK_SIZE (1UL << SECTION_SIZE_BITS) @@ -100,6 +101,32 @@ int arch_get_memory_phys_device(unsigned long start_pfn); unsigned long memory_block_size_bytes(void); int set_memory_block_size_order(unsigned int order); +/** + * memory_block_aligned_range - align a physical address range to memory blocks + * @range: the input range to align + * + * Aligns the start address up and the end address down to memory block + * boundaries. This is required for memory hotplug operations which must + * operate on memory-block aligned ranges. + * + * Returns the aligned range. Callers should check that the returned + * range is valid (aligned.start < aligned.end) before using it. + */ +static inline struct range memory_block_aligned_range(const struct range *range) +{ + struct range aligned; + + aligned.start = ALIGN(range->start, memory_block_size_bytes()); + aligned.end = ALIGN_DOWN(range->end + 1, memory_block_size_bytes()); + /* No whole block fits (e.g. range below the first boundary): empty. */ + if (aligned.end <= aligned.start) + aligned.start = aligned.end; + else + aligned.end -= 1; + + return aligned; +} + struct memory_notify { unsigned long start_pfn; unsigned long nr_pages; From 5459c5be28c22c29afc1115c14911f7c5f8aff20 Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Sun, 12 Jul 2026 11:44:56 -0400 Subject: [PATCH 391/501] mm/memory_hotplug: add mhp_online_type_to_str() and export string helpers Add mhp_online_type_to_str() as the inverse of mhp_online_type_from_str(), and export both so a driver can render and parse the memory online type through its own sysfs interface. Link: https://lore.kernel.org/20260712154505.3564379-3-gourry@gourry.net Signed-off-by: Gregory Price Reviewed-by: Dave Jiang Acked-by: David Hildenbrand (Arm) Reviewed-by: Dan Williams Cc: Alison Schofield Cc: Danilo Krummrich Cc: Greg Kroah-Hartman Cc: Hannes Reinecke Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Oscar Salvador Cc: Pankaj Gupta Cc: "Rafael J. Wysocki" Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vishal Verma Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- drivers/base/memory.c | 9 +++++++++ include/linux/memory_hotplug.h | 1 + 2 files changed, 10 insertions(+) diff --git a/drivers/base/memory.c b/drivers/base/memory.c index bcfe2d9f4adb..5eead3346f1e 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -46,6 +46,15 @@ int mhp_online_type_from_str(const char *str) } return -EINVAL; } +EXPORT_SYMBOL_GPL(mhp_online_type_from_str); + +const char *mhp_online_type_to_str(int online_type) +{ + if (online_type < 0 || online_type >= (int)ARRAY_SIZE(online_type_to_str)) + return NULL; + return online_type_to_str[online_type]; +} +EXPORT_SYMBOL_GPL(mhp_online_type_to_str); #define to_memory_block(dev) container_of(dev, struct memory_block, dev) diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 06c58cb05779..3f5d63892d90 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -127,6 +127,7 @@ extern int arch_add_memory(int nid, u64 start, u64 size, extern u64 max_mem_size; extern int mhp_online_type_from_str(const char *str); +const char *mhp_online_type_to_str(int online_type); /* If movable_node boot option specified */ extern bool movable_node_enabled; From 65c1a93e438e42940bdc14b72288f42ca2e3ff06 Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Sun, 12 Jul 2026 11:44:57 -0400 Subject: [PATCH 392/501] mm/memory_hotplug: pass online_type to online_memory_block() via arg Modify online_memory_block() to accept the online type through its arg parameter rather than calling mhp_get_default_online_type() internally. This prepares for allowing callers to specify explicit online types. Update the caller in add_memory_resource() to pass the default online type via a local variable. No functional change. Link: https://lore.kernel.org/20260712154505.3564379-4-gourry@gourry.net Signed-off-by: Gregory Price Acked-by: David Hildenbrand (Red Hat) Reviewed-by: Pankaj Gupta Reviewed-by: Dave Jiang Reviewed-by: Dan Williams Cc: Alison Schofield Cc: Danilo Krummrich Cc: Greg Kroah-Hartman Cc: Hannes Reinecke Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Oscar Salvador Cc: "Rafael J. Wysocki" Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vishal Verma Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/memory_hotplug.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 99090f49d2df..1dc0a1bb3a75 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1340,7 +1340,9 @@ static int check_hotplug_memory_range(u64 start, u64 size) static int online_memory_block(struct memory_block *mem, void *arg) { - mem->online_type = mhp_get_default_online_type(); + enum mmop *online_type = arg; + + mem->online_type = *online_type; return device_online(&mem->dev); } @@ -1497,6 +1499,7 @@ static int create_altmaps_and_memory_blocks(int nid, struct memory_group *group, int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) { struct mhp_params params = { .pgprot = pgprot_mhp(PAGE_KERNEL) }; + enum mmop online_type = mhp_get_default_online_type(); enum memblock_flags memblock_flags = MEMBLOCK_NONE; struct memory_group *group = NULL; u64 start, size; @@ -1585,7 +1588,8 @@ int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) /* online pages if requested */ if (mhp_get_default_online_type() != MMOP_OFFLINE) - walk_memory_blocks(start, size, NULL, online_memory_block); + walk_memory_blocks(start, size, &online_type, + online_memory_block); return ret; error: From 7e00a85f783e33ce6106891d6e3ad3c0fd77668f Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Sun, 12 Jul 2026 11:44:58 -0400 Subject: [PATCH 393/501] mm/memory_hotplug: export mhp_get_default_online_type Drivers which may pass hotplug policy down to DAX need MMOP_ symbols and the mhp_get_default_online_type function for hotplug use cases. Some drivers (cxl) co-mingle their hotplug and devdax use-cases into the same driver code, and chose the dax_kmem path as the default driver path - making it difficult to require hotplug as a predicate to building the overall driver (it may break other non-hotplug use-cases). Export mhp_get_default_online_type function to allow these drivers to build when hotplug is disabled and still use the DAX use case. In the built-out case we simply return MMOP_OFFLINE as it's non-destructive. The internal function can never return -1 either, so we choose this to allow for defining the function with 'enum mmop'. Link: https://lore.kernel.org/20260712154505.3564379-5-gourry@gourry.net Signed-off-by: Gregory Price Reviewed-by: Dave Jiang Reviewed-by: Dan Williams Acked-by: David Hildenbrand (Arm) Cc: Alison Schofield Cc: Danilo Krummrich Cc: Greg Kroah-Hartman Cc: Hannes Reinecke Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Oscar Salvador Cc: Pankaj Gupta Cc: "Rafael J. Wysocki" Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vishal Verma Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/memory_hotplug.h | 2 ++ mm/memory_hotplug.c | 1 + 2 files changed, 3 insertions(+) diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 3f5d63892d90..5240aa28bc90 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -317,6 +317,8 @@ extern struct zone *zone_for_pfn_range(enum mmop online_type, extern int arch_create_linear_mapping(int nid, u64 start, u64 size, struct mhp_params *params); void arch_remove_linear_mapping(u64 start, u64 size); +#else +static inline enum mmop mhp_get_default_online_type(void) { return MMOP_OFFLINE; } #endif /* CONFIG_MEMORY_HOTPLUG */ #endif /* __LINUX_MEMORY_HOTPLUG_H */ diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 1dc0a1bb3a75..73c4a1b9895d 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -241,6 +241,7 @@ enum mmop mhp_get_default_online_type(void) return mhp_default_online_type; } +EXPORT_SYMBOL_GPL(mhp_get_default_online_type); void mhp_set_default_online_type(enum mmop online_type) { From 2b047c8a8c2769319a6ed17c47773c285b047080 Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Sun, 12 Jul 2026 11:44:59 -0400 Subject: [PATCH 394/501] mm/memory_hotplug: add __add_memory_driver_managed() with online_type arg Existing callers of add_memory_driver_managed cannot select the preferred online type (ZONE_NORMAL vs ZONE_MOVABLE), requiring it to hot-add memory as offline blocks, and then follow up by onlining each memory block individually. Most drivers prefer the system default, but the CXL driver wants to plumb a preferred policy through the dax kmem driver. Refactor APIs to add a new interface which allows the dax kmem module to select a preferred policy. Overriding the configured auto-online policy is only safe for known in-tree modules, where we know the override reflects a different, user-requested policy. We do not want arbitrary out-of-tree drivers silently overriding the system-wide onlining policy, so restrict the new interface to the kmem module using EXPORT_SYMBOL_FOR_MODULES() rather than a plain EXPORT_SYMBOL_GPL(). Other in-tree modules (e.g. cxl_core) can be added to the allowed list as the need arises. Refactor add_memory_driver_managed, extract __add_memory_driver_managed - Add proper kernel-doc for add_memory_driver_managed while refactoring - New helper accepts an explicit online_type. - New helper validates online_type is between OFFLINE and ONLINE_MOVABLE Refactor: add_memory_resource, extract __add_memory_resource - new helper accepts an explicit online_type Original APIs now explicitly pass the system-default to new helpers. No functional change for existing users. Link: https://lore.kernel.org/20260712154505.3564379-6-gourry@gourry.net Signed-off-by: Gregory Price Acked-by: David Hildenbrand (Arm) Reviewed-by: Pankaj Gupta Reviewed-by: Dave Jiang Reviewed-by: Dan Williams Cc: Alison Schofield Cc: Danilo Krummrich Cc: Greg Kroah-Hartman Cc: Hannes Reinecke Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Oscar Salvador Cc: "Rafael J. Wysocki" Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vishal Verma Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/memory_hotplug.h | 3 ++ mm/memory_hotplug.c | 61 +++++++++++++++++++++++++++++----- 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 5240aa28bc90..d69b273ed304 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -295,6 +295,9 @@ extern int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory_resource(int nid, struct resource *resource, mhp_t mhp_flags); +int __add_memory_driver_managed(int nid, u64 start, u64 size, + const char *resource_name, mhp_t mhp_flags, + enum mmop online_type); extern int add_memory_driver_managed(int nid, u64 start, u64 size, const char *resource_name, mhp_t mhp_flags); diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 73c4a1b9895d..fe6b38ef9aac 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1497,10 +1497,10 @@ static int create_altmaps_and_memory_blocks(int nid, struct memory_group *group, * * we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ -int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) +static int __add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags, + enum mmop online_type) { struct mhp_params params = { .pgprot = pgprot_mhp(PAGE_KERNEL) }; - enum mmop online_type = mhp_get_default_online_type(); enum memblock_flags memblock_flags = MEMBLOCK_NONE; struct memory_group *group = NULL; u64 start, size; @@ -1588,7 +1588,7 @@ int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) merge_system_ram_resource(res); /* online pages if requested */ - if (mhp_get_default_online_type() != MMOP_OFFLINE) + if (online_type != MMOP_OFFLINE) walk_memory_blocks(start, size, &online_type, online_memory_block); @@ -1606,7 +1606,13 @@ int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) return ret; } -/* requires device_hotplug_lock, see add_memory_resource() */ +int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) +{ + return __add_memory_resource(nid, res, mhp_flags, + mhp_get_default_online_type()); +} + +/* requires device_hotplug_lock, see __add_memory_resource() */ int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags) { struct resource *res; @@ -1634,7 +1640,15 @@ int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags) } EXPORT_SYMBOL_GPL(add_memory); -/* +/** + * __add_memory_driver_managed - add driver-managed memory with explicit online_type + * @nid: NUMA node ID where the memory will be added + * @start: Start physical address of the memory range + * @size: Size of the memory range in bytes + * @resource_name: Resource name in format "System RAM ($DRIVER)" + * @mhp_flags: Memory hotplug flags + * @online_type: Auto-Online behavior (offline, online, kernel, movable) + * * Add special, driver-managed memory to the system as system RAM. Such * memory is not exposed via the raw firmware-provided memmap as system * RAM, instead, it is detected and added by a driver - during cold boot, @@ -1642,6 +1656,7 @@ EXPORT_SYMBOL_GPL(add_memory); * * Reasons why this memory should not be used for the initial memmap of a * kexec kernel or for placing kexec images: + * * - The booting kernel is in charge of determining how this memory will be * used (e.g., use persistent memory as system RAM) * - Coordination with a hypervisor is required before this memory @@ -1654,9 +1669,12 @@ EXPORT_SYMBOL_GPL(add_memory); * * The resource_name (visible via /proc/iomem) has to have the format * "System RAM ($DRIVER)". + * + * Return: 0 on success, negative error code on failure. */ -int add_memory_driver_managed(int nid, u64 start, u64 size, - const char *resource_name, mhp_t mhp_flags) +int __add_memory_driver_managed(int nid, u64 start, u64 size, + const char *resource_name, mhp_t mhp_flags, + enum mmop online_type) { struct resource *res; int rc; @@ -1666,6 +1684,9 @@ int add_memory_driver_managed(int nid, u64 start, u64 size, resource_name[strlen(resource_name) - 1] != ')') return -EINVAL; + if (online_type < MMOP_OFFLINE || online_type > MMOP_ONLINE_MOVABLE) + return -EINVAL; + lock_device_hotplug(); res = register_memory_resource(start, size, resource_name); @@ -1674,7 +1695,7 @@ int add_memory_driver_managed(int nid, u64 start, u64 size, goto out_unlock; } - rc = add_memory_resource(nid, res, mhp_flags); + rc = __add_memory_resource(nid, res, mhp_flags, online_type); if (rc < 0) release_memory_resource(res); @@ -1682,6 +1703,30 @@ int add_memory_driver_managed(int nid, u64 start, u64 size, unlock_device_hotplug(); return rc; } +EXPORT_SYMBOL_FOR_MODULES(__add_memory_driver_managed, "kmem"); + +/** + * add_memory_driver_managed - add driver-managed memory + * @nid: NUMA node ID where the memory will be added + * @start: Start physical address of the memory range + * @size: Size of the memory range in bytes + * @resource_name: Resource name in format "System RAM ($DRIVER)" + * @mhp_flags: Memory hotplug flags + * + * Add driver-managed memory with the system default online type set by + * build config or kernel boot parameter. + * + * See __add_memory_driver_managed for more details. + * + * Return: 0 on success, negative error code on failure. + */ +int add_memory_driver_managed(int nid, u64 start, u64 size, + const char *resource_name, mhp_t mhp_flags) +{ + return __add_memory_driver_managed(nid, start, size, resource_name, + mhp_flags, + mhp_get_default_online_type()); +} EXPORT_SYMBOL_GPL(add_memory_driver_managed); /* From c17ba40c37e890987dce846cf6e1ec6a2bf79ea3 Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Sun, 12 Jul 2026 11:45:00 -0400 Subject: [PATCH 395/501] mm/memory_hotplug: add offline_and_remove_memory_ranges() offline_and_remove_memory() handles a single contiguous range. Callers that manage a device composed of several ranges (dax/kmem) currently have to call it in a loop, which gives up atomicity. In addition to pushing rollback logic into the driver, the lack of atomicity creates a race condition between system daemons trying to manage the same resource: - Manager 1: Offlines memory blocks. Removes device. ^^^^ - Manager 2: Detects offline memory blocks, re-onlines them. Add offline_and_remove_memory_ranges(), which takes an array of ranges and processes them as one operation under a single lock_device_hotplug(): - Phase 1 offlines every block of every range. - Phase 2 removes the ranges only if all ranges are offline. - If any offline fails, the whole operation is reverted. This gives callers all-or-nothing semantics for the offline step, so a failed or interrupted unplug leaves the device in a consistent state. This also resolves the battling managers race - the second manager's operation simply fails when the block is destroyed / cannot be onlined. offline_and_remove_memory() becomes a thin wrapper that passes its single range to the new helper, so the offline/rollback logic lives in one place. Link: https://lore.kernel.org/20260712154505.3564379-7-gourry@gourry.net Signed-off-by: Gregory Price Suggested-by: David Hildenbrand (Arm) Reviewed-by: Dave Jiang Acked-by: David Hildenbrand (Arm) Reviewed-by: Dan Williams Cc: Alison Schofield Cc: Danilo Krummrich Cc: Greg Kroah-Hartman Cc: Hannes Reinecke Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Oscar Salvador Cc: Pankaj Gupta Cc: "Rafael J. Wysocki" Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vishal Verma Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/memory_hotplug.h | 8 +++ mm/memory_hotplug.c | 96 ++++++++++++++++++++++++---------- 2 files changed, 76 insertions(+), 28 deletions(-) diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index d69b273ed304..b39605d30896 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -268,6 +268,8 @@ extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages, extern int remove_memory(u64 start, u64 size); extern void __remove_memory(u64 start, u64 size); extern int offline_and_remove_memory(u64 start, u64 size); +int offline_and_remove_memory_ranges(const struct range *ranges, + unsigned int nr_ranges); #else static inline void try_offline_node(int nid) {} @@ -284,6 +286,12 @@ static inline int remove_memory(u64 start, u64 size) } static inline void __remove_memory(u64 start, u64 size) {} + +static inline int offline_and_remove_memory_ranges(const struct range *ranges, + unsigned int nr_ranges) +{ + return -EBUSY; +} #endif /* CONFIG_MEMORY_HOTREMOVE */ #ifdef CONFIG_MEMORY_HOTPLUG diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index fe6b38ef9aac..226ab9cb078a 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -2432,58 +2432,98 @@ static int try_reonline_memory_block(struct memory_block *mem, void *arg) */ int offline_and_remove_memory(u64 start, u64 size) { - const unsigned long mb_count = size / memory_block_size_bytes(); - uint8_t *online_types, *tmp; - int rc; + struct range range = { + .start = start, + .end = start + size - 1, + }; - if (!IS_ALIGNED(start, memory_block_size_bytes()) || - !IS_ALIGNED(size, memory_block_size_bytes()) || !size) + return offline_and_remove_memory_ranges(&range, 1); +} +EXPORT_SYMBOL_GPL(offline_and_remove_memory); + +/** + * offline_and_remove_memory_ranges - offline and remove multiple memory ranges + * @ranges: array of physical address ranges to offline and remove + * @nr_ranges: number of entries in @ranges + * + * Offline and remove several memory ranges as one operation, serialized + * against other hotplug operations by a single lock_device_hotplug(). + * + * This offlines all ranges before removing any of them. If offlining any + * range fails, the entire process is reverted and nothing is removed. + * This provides a fully atomic semantic for unplugging an entire device. + * + * Each range must be memory-block aligned in start and size. + * + * Return: 0 on success, negative errno on failure (never positive). On + * failure no range has been removed. + */ +int offline_and_remove_memory_ranges(const struct range *ranges, + unsigned int nr_ranges) +{ + unsigned long mb_count = 0; + uint8_t *online_types, *tmp; + unsigned int i; + int rc = 0; + + if (!ranges || !nr_ranges) return -EINVAL; + for (i = 0; i < nr_ranges; i++) { + const u64 start = ranges[i].start; + const u64 size = range_len(&ranges[i]); + + if (!IS_ALIGNED(start, memory_block_size_bytes()) || + !IS_ALIGNED(size, memory_block_size_bytes()) || !size) + return -EINVAL; + mb_count += size / memory_block_size_bytes(); + } + /* - * We'll remember the old online type of each memory block, so we can - * try to revert whatever we did when offlining one memory block fails - * after offlining some others succeeded. + * Remember the old online type of every memory block across all ranges, + * so we can revert if offlining a later block fails. All entries start + * as MMOP_OFFLINE so blocks we never touched are skipped on rollback. */ online_types = kmalloc_array(mb_count, sizeof(*online_types), GFP_KERNEL); if (!online_types) return -ENOMEM; - /* - * Initialize all states to MMOP_OFFLINE, so when we abort processing in - * try_offline_memory_block(), we'll skip all unprocessed blocks in - * try_reonline_memory_block(). - */ memset(online_types, MMOP_OFFLINE, mb_count); lock_device_hotplug(); - tmp = online_types; - rc = walk_memory_blocks(start, size, &tmp, try_offline_memory_block); - /* - * In case we succeeded to offline all memory, remove it. - * This cannot fail as it cannot get onlined in the meantime. + * Phase 1: offline every block in every range. An already-offline + * block folds to success, so out-of-band offlining never blocks unplug. */ - if (!rc) { - rc = try_remove_memory(start, size); + tmp = online_types; + for (i = 0; i < nr_ranges; i++) { + rc = walk_memory_blocks(ranges[i].start, range_len(&ranges[i]), + &tmp, try_offline_memory_block); if (rc) - pr_err("%s: Failed to remove memory: %d", __func__, rc); + break; } - /* - * Rollback what we did. While memory onlining might theoretically fail - * (nacked by a notifier), it barely ever happens. - */ + /* If any failure occurred at all, rollback any changes and bail */ if (rc) { tmp = online_types; - walk_memory_blocks(start, size, &tmp, - try_reonline_memory_block); + for (i = 0; i < nr_ranges; i++) + walk_memory_blocks(ranges[i].start, + range_len(&ranges[i]), &tmp, + try_reonline_memory_block); + goto out_unlock; } + + /* Phase 2: Remove. This should never fail holding the hotplug lock */ + for (i = 0; i < nr_ranges; i++) + WARN_ON_ONCE(try_remove_memory(ranges[i].start, + range_len(&ranges[i]))); + +out_unlock: unlock_device_hotplug(); kfree(online_types); return rc; } -EXPORT_SYMBOL_GPL(offline_and_remove_memory); +EXPORT_SYMBOL_GPL(offline_and_remove_memory_ranges); #endif /* CONFIG_MEMORY_HOTREMOVE */ From 496a4cd6585cabddf56b98ac01f4d4f122ecfa7b Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Sun, 12 Jul 2026 11:45:01 -0400 Subject: [PATCH 396/501] dax/kmem: resolve default online type at probe time Resolve the default online type in kmem at probe time explicitly instead of implicitly inheriting the system default by calling add_memory_driver_managed(). No behavioral change at this point (still system default). Link: https://lore.kernel.org/20260712154505.3564379-8-gourry@gourry.net Signed-off-by: Gregory Price Cc: Alison Schofield Cc: Danilo Krummrich Cc: Dave Jiang Cc: David Hildenbrand (Arm) Cc: Greg Kroah-Hartman Cc: Hannes Reinecke Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Oscar Salvador Cc: Pankaj Gupta Cc: "Rafael J. Wysocki" Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vishal Verma Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- drivers/dax/kmem.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c index 592171ec10f4..38ed5c4e9c83 100644 --- a/drivers/dax/kmem.c +++ b/drivers/dax/kmem.c @@ -73,6 +73,7 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) mhp_t mhp_flags; int numa_node; int adist = MEMTIER_DEFAULT_DAX_ADISTANCE; + int online_type = mhp_get_default_online_type(); /* * Ensure good NUMA information for the persistent memory. @@ -172,8 +173,9 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) * Ensure that future kexec'd kernels will not treat * this as RAM automatically. */ - rc = add_memory_driver_managed(data->mgid, range.start, - range_len(&range), kmem_name, mhp_flags); + rc = __add_memory_driver_managed(data->mgid, range.start, + range_len(&range), kmem_name, mhp_flags, + online_type); if (rc) { dev_warn(dev, "mapping%d: %#llx-%#llx memory add failed\n", From bb76d369b6905879099d5a314f3b85b63a187ed4 Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Sun, 12 Jul 2026 11:45:02 -0400 Subject: [PATCH 397/501] dax/kmem: extract hotplug/hotremove helper functions Refactor kmem _probe() _remove() by extracting init, cleanup, hotplug, and hot-remove logic into separate helper functions: - dax_kmem_init_resources: inits IO_RESOURCE w/ request_mem_region - dax_kmem_cleanup_resources: cleans up initialized IO_RESOURCE - dax_kmem_do_hotplug: handles memory region reservation and adding - dax_kmem_do_hotremove: handles memory removal and resource cleanup This is a pure refactoring with no functional change. The helpers will enable future extensions to support more granular control over memory hotplug operations. We need to split hotplug/hotunplug and init/cleanup in order to have the resources available for hot-add. Otherwise, when probe occurs, the dax devices are never added to sysfs because the resources are never registered. Detaching hotunplug/cleanup allows us to re-use the hotunplug code without destroying the underlying resources. Link: https://lore.kernel.org/20260712154505.3564379-9-gourry@gourry.net Signed-off-by: Gregory Price Reviewed-by: Dave Jiang Cc: Alison Schofield Cc: Danilo Krummrich Cc: David Hildenbrand (Arm) Cc: Greg Kroah-Hartman Cc: Hannes Reinecke Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Oscar Salvador Cc: Pankaj Gupta Cc: "Rafael J. Wysocki" Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vishal Verma Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- drivers/dax/kmem.c | 325 +++++++++++++++++++++++++++++++-------------- 1 file changed, 224 insertions(+), 101 deletions(-) diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c index 38ed5c4e9c83..6174f7d3d05b 100644 --- a/drivers/dax/kmem.c +++ b/drivers/dax/kmem.c @@ -63,14 +63,206 @@ static void kmem_put_memory_types(void) mt_put_memory_types(&kmem_memory_types); } +/** + * dax_kmem_do_hotplug - hotplug memory for dax kmem device + * @dev_dax: the dev_dax instance + * @data: the dax_kmem_data structure with resource tracking + * + * Hotplugs all ranges in the dev_dax region as system memory. + * + * Returns the number of successfully mapped ranges, or negative error. + */ +static int dax_kmem_do_hotplug(struct dev_dax *dev_dax, + struct dax_kmem_data *data, + int online_type) +{ + struct device *dev = &dev_dax->dev; + int i, rc, onlined = 0; + mhp_t mhp_flags; + + for (i = 0; i < dev_dax->nr_range; i++) { + struct range range; + + rc = dax_kmem_range(dev_dax, i, &range); + if (rc) + continue; + + /* + * init_resources() is best-effort: if a reservation conflict + * occurs it keeps the range but leaves res[i]=NULL. For hotplug + * on probe systems, this means kmem will partially online. + * + * We have to keep this behavior not to break those systems. + * For those systems - atomicity only applies to valid ranges. + */ + if (!data->res[i]) + continue; + + mhp_flags = MHP_NID_IS_MGID; + if (dev_dax->memmap_on_memory) + mhp_flags |= MHP_MEMMAP_ON_MEMORY; + + /* + * Ensure that future kexec'd kernels will not treat + * this as RAM automatically. + */ + rc = __add_memory_driver_managed(data->mgid, range.start, + range_len(&range), kmem_name, mhp_flags, + online_type); + + if (rc) { + dev_warn(dev, "mapping%d: %#llx-%#llx memory add failed\n", + i, range.start, range.end); + /* + * Release the reservation for the range that failed to + * add so a later hotremove does not try to remove memory + * that was never added. + */ + if (data->res[i]) { + remove_resource(data->res[i]); + kfree(data->res[i]); + data->res[i] = NULL; + } + if (onlined) + continue; + return rc; + } + onlined++; + } + + return onlined; +} + +/** + * dax_kmem_init_resources - create memory regions for dax kmem + * @dev_dax: the dev_dax instance + * @data: the dax_kmem_data structure with resource tracking + * + * Initializes all the resources for the DAX + * + * Returns the number of successfully mapped ranges, or negative error. + */ +static int dax_kmem_init_resources(struct dev_dax *dev_dax, + struct dax_kmem_data *data) +{ + struct device *dev = &dev_dax->dev; + int i, rc, mapped = 0; + + for (i = 0; i < dev_dax->nr_range; i++) { + struct resource *res; + struct range range; + + rc = dax_kmem_range(dev_dax, i, &range); + if (rc) + continue; + + /* Skip ranges already added */ + if (data->res[i]) + continue; + + /* Region is permanently reserved if hotremove fails. */ + res = request_mem_region(range.start, range_len(&range), + data->res_name); + if (!res) { + dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve region\n", + i, range.start, range.end); + /* + * Once some memory has been onlined we can't + * assume that it can be un-onlined safely. + */ + if (mapped) + continue; + return -EBUSY; + } + data->res[i] = res; + /* + * Set flags appropriate for System RAM. Leave ..._BUSY clear + * so that add_memory() can add a child resource. Do not + * inherit flags from the parent since it may set new flags + * unknown to us that will break add_memory() later. + */ + res->flags = IORESOURCE_SYSTEM_RAM; + mapped++; + } + return mapped; +} + +#ifdef CONFIG_MEMORY_HOTREMOVE +/** + * dax_kmem_do_hotremove - hot-remove memory for dax kmem device + * @dev_dax: the dev_dax instance + * @data: the dax_kmem_data structure with resource tracking + * + * Removes all ranges in the dev_dax region. + * + * Returns the number of successfully removed ranges. + */ +static int dax_kmem_do_hotremove(struct dev_dax *dev_dax, + struct dax_kmem_data *data) +{ + struct device *dev = &dev_dax->dev; + int i, success = 0; + + for (i = 0; i < dev_dax->nr_range; i++) { + struct range range; + int rc; + + rc = dax_kmem_range(dev_dax, i, &range); + if (rc) + continue; + + /* range was never added during probe, count as removed */ + if (!data->res[i]) { + success++; + continue; + } + + rc = remove_memory(range.start, range_len(&range)); + if (rc == 0) { + /* Release the resource for the successfully removed range */ + remove_resource(data->res[i]); + kfree(data->res[i]); + data->res[i] = NULL; + success++; + continue; + } + any_hotremove_failed = true; + dev_err(dev, "mapping%d: %#llx-%#llx hotremove failed\n", + i, range.start, range.end); + } + + return success; +} +#endif /* CONFIG_MEMORY_HOTREMOVE */ + +/** + * dax_kmem_cleanup_resources - remove the dax memory resources + * @dev_dax: the dev_dax instance + * @data: the dax_kmem_data structure with resource tracking + * + * Removes all resources in the dev_dax region. + */ +static void dax_kmem_cleanup_resources(struct dev_dax *dev_dax, + struct dax_kmem_data *data) +{ + int i; + + for (i = 0; i < dev_dax->nr_range; i++) { + if (!data->res[i]) + continue; + remove_resource(data->res[i]); + kfree(data->res[i]); + data->res[i] = NULL; + } +} + static int dev_dax_kmem_probe(struct dev_dax *dev_dax) { struct device *dev = &dev_dax->dev; unsigned long total_len = 0, orig_len = 0; struct dax_kmem_data *data; struct memory_dev_type *mtype; - int i, rc, mapped = 0; - mhp_t mhp_flags; + int i, rc; int numa_node; int adist = MEMTIER_DEFAULT_DAX_ADISTANCE; int online_type = mhp_get_default_online_type(); @@ -133,68 +325,22 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) goto err_reg_mgid; data->mgid = rc; - for (i = 0; i < dev_dax->nr_range; i++) { - struct resource *res; - struct range range; - - rc = dax_kmem_range(dev_dax, i, &range); - if (rc) - continue; - - /* Region is permanently reserved if hotremove fails. */ - res = request_mem_region(range.start, range_len(&range), data->res_name); - if (!res) { - dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve region\n", - i, range.start, range.end); - /* - * Once some memory has been onlined we can't - * assume that it can be un-onlined safely. - */ - if (mapped) - continue; - rc = -EBUSY; - goto err_request_mem; - } - data->res[i] = res; - - /* - * Set flags appropriate for System RAM. Leave ..._BUSY clear - * so that add_memory() can add a child resource. Do not - * inherit flags from the parent since it may set new flags - * unknown to us that will break add_memory() below. - */ - res->flags = IORESOURCE_SYSTEM_RAM; - - mhp_flags = MHP_NID_IS_MGID; - if (dev_dax->memmap_on_memory) - mhp_flags |= MHP_MEMMAP_ON_MEMORY; - - /* - * Ensure that future kexec'd kernels will not treat - * this as RAM automatically. - */ - rc = __add_memory_driver_managed(data->mgid, range.start, - range_len(&range), kmem_name, mhp_flags, - online_type); - - if (rc) { - dev_warn(dev, "mapping%d: %#llx-%#llx memory add failed\n", - i, range.start, range.end); - remove_resource(res); - kfree(res); - data->res[i] = NULL; - if (mapped) - continue; - goto err_request_mem; - } - mapped++; - } - dev_set_drvdata(dev, data); + rc = dax_kmem_init_resources(dev_dax, data); + if (rc < 0) + goto err_resources; + + rc = dax_kmem_do_hotplug(dev_dax, data, online_type); + if (rc < 0) + goto err_hotplug; + return 0; -err_request_mem: +err_hotplug: + dax_kmem_cleanup_resources(dev_dax, data); +err_resources: + dev_set_drvdata(dev, NULL); memory_group_unregister(data->mgid); err_reg_mgid: kfree(data->res_name); @@ -208,7 +354,7 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) #ifdef CONFIG_MEMORY_HOTREMOVE static void dev_dax_kmem_remove(struct dev_dax *dev_dax) { - int i, success = 0; + int success; int node = dev_dax->target_node; struct device *dev = &dev_dax->dev; struct dax_kmem_data *data = dev_get_drvdata(dev); @@ -219,48 +365,25 @@ static void dev_dax_kmem_remove(struct dev_dax *dev_dax) * there is no way to hotremove this memory until reboot because device * unbind will succeed even if we return failure. */ - for (i = 0; i < dev_dax->nr_range; i++) { - struct range range; - int rc; - - rc = dax_kmem_range(dev_dax, i, &range); - if (rc) - continue; - - /* range was never added during probe */ - if (!data->res[i]) { - success++; - continue; - } - - rc = remove_memory(range.start, range_len(&range)); - if (rc == 0) { - remove_resource(data->res[i]); - kfree(data->res[i]); - data->res[i] = NULL; - success++; - continue; - } - any_hotremove_failed = true; - dev_err(dev, - "mapping%d: %#llx-%#llx cannot be hotremoved until the next reboot\n", - i, range.start, range.end); + success = dax_kmem_do_hotremove(dev_dax, data); + if (success < dev_dax->nr_range) { + dev_err(dev, "Hotplug regions stuck online until reboot\n"); + return; } - if (success >= dev_dax->nr_range) { - memory_group_unregister(data->mgid); - kfree(data->res_name); - kfree(data); - dev_set_drvdata(dev, NULL); - /* - * Clear the memtype association on successful unplug. - * If not, we have memory blocks left which can be - * offlined/onlined later. We need to keep memory_dev_type - * for that. This implies this reference will be around - * till next reboot. - */ - clear_node_memory_type(node, NULL); - } + dax_kmem_cleanup_resources(dev_dax, data); + memory_group_unregister(data->mgid); + kfree(data->res_name); + kfree(data); + dev_set_drvdata(dev, NULL); + /* + * Clear the memtype association on successful unplug. + * If not, we have memory blocks left which can be + * offlined/onlined later. We need to keep memory_dev_type + * for that. This implies this reference will be around + * till next reboot. + */ + clear_node_memory_type(node, NULL); } #else static void dev_dax_kmem_remove(struct dev_dax *dev_dax) From 2a6f2aef1126fcfe4ea002443a474e8cd905d5f4 Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Sun, 12 Jul 2026 11:45:03 -0400 Subject: [PATCH 398/501] dax/kmem: add sysfs interface for atomic whole-device hotplug There is no atomic mechanism to offline and remove an entire multi-block DAX kmem device. This is presently done in two steps: 1. offline all 2. remove all This creates a race condition where another entity operates directly on the memory blocks and can cause hot-unplug to fail / unbind to deadlock. Add a new 'state' sysfs attribute that enables an atomic whole-device hotplug operation across its entire memory region. daxX.Y/state mirrors the per-block memoryX/state ABI: - [offline, online, online_kernel, online_movable] - "unplugged" - is added specifically for dax0.0/state The valid writable states include: - "unplugged": memory blocks are not present - "online": memory is online, zone chosen by the kernel - "online_kernel": memory is online in ZONE_NORMAL - "online_movable": memory is online in ZONE_MOVABLE Valid transitions: - unplugged -> online[_kernel|_movable] - online[_kernel|_movable] -> unplugged - offline -> unplugged A device can only be onlined from "unplugged", so it must be returned there before being onlined into a different state. For backwards compatibility the memory blocks are always created at probe - existing tools expect them to be present after kmem binds. "offline" is therefore a reportable state but is not writable: it only arises from the legacy auto_online_blocks=offline policy. Onlining such a device through this attribute requires unplugging it first in an effort to get drivers creating DAX devices to set a default. Unplug is atomic across the whole device: dax_kmem_do_hotremove() collects every added range and offlines/removes them in one operation. Either the operation succeeds or is entirely rolled back. Unbind Note: An offline dax device memory is removed on unbind as before. If online at unbind, the resources are leaked (as before), but now we prevent deadlock if a memory region is impossible to hotremove. Link: https://lore.kernel.org/20260712154505.3564379-10-gourry@gourry.net Signed-off-by: Gregory Price Suggested-by: Hannes Reinecke Suggested-by: David Hildenbrand Reviewed-by: Dan Williams Cc: Alison Schofield Cc: Danilo Krummrich Cc: Dave Jiang Cc: Greg Kroah-Hartman Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Oscar Salvador Cc: Pankaj Gupta Cc: "Rafael J. Wysocki" Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vishal Verma Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- Documentation/ABI/testing/sysfs-bus-dax | 24 +++ drivers/dax/bus.h | 2 + drivers/dax/kmem.c | 247 ++++++++++++++++++++---- 3 files changed, 235 insertions(+), 38 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-bus-dax b/Documentation/ABI/testing/sysfs-bus-dax index b34266bfae49..60703fffa99e 100644 --- a/Documentation/ABI/testing/sysfs-bus-dax +++ b/Documentation/ABI/testing/sysfs-bus-dax @@ -151,3 +151,27 @@ Description: memmap_on_memory parameter for memory_hotplug. This is typically set on the kernel command line - memory_hotplug.memmap_on_memory set to 'true' or 'force'." + +What: /sys/bus/dax/devices/daxX.Y/state +Contact: nvdimm@lists.linux.dev +Description: + (RW) Controls the state of the memory region. + Applies to all memory blocks associated with the device. + Only applies to dax_kmem devices. + + Reading returns the current state; the writable states mirror + the per-block /sys/devices/system/memory/memoryX/state ABI:: + + "unplugged": memory blocks are not present + "online": memory is online, zone chosen by the kernel + "online_kernel": memory is online in ZONE_NORMAL + "online_movable": memory is online in ZONE_MOVABLE + + "offline" (memory blocks are present but offline) may also be + reported - this happens when the device is bound while the + auto_online_blocks policy is "offline". It cannot be written, + as it's not useful and creates device destruction races. + + A device can only be onlined from the "unplugged" state, so a + device must be returned to "unplugged" before it can be onlined + into a different state. diff --git a/drivers/dax/bus.h b/drivers/dax/bus.h index 5909171a4428..bfc31923f3ff 100644 --- a/drivers/dax/bus.h +++ b/drivers/dax/bus.h @@ -16,6 +16,8 @@ struct dax_region; #define IORESOURCE_DAX_STATIC BIT(0) #define IORESOURCE_DAX_KMEM BIT(1) +#define DAX_KMEM_UNPLUGGED (-1) /* Do not create memory blocks */ + struct dax_region *alloc_dax_region(struct device *parent, int region_id, struct range *range, int target_node, unsigned int align, unsigned long flags); diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c index 6174f7d3d05b..a48d699cf344 100644 --- a/drivers/dax/kmem.c +++ b/drivers/dax/kmem.c @@ -45,6 +45,8 @@ static int dax_kmem_range(struct dev_dax *dev_dax, int i, struct range *r) struct dax_kmem_data { const char *res_name; int mgid; + int state; + struct mutex lock; /* protects hotplug state transitions */ struct resource *res[]; }; @@ -63,12 +65,22 @@ static void kmem_put_memory_types(void) mt_put_memory_types(&kmem_memory_types); } +/* True for the online states a kmem dax device can hold. */ +static bool dax_kmem_state_is_online(int state) +{ + return state == MMOP_ONLINE || + state == MMOP_ONLINE_KERNEL || + state == MMOP_ONLINE_MOVABLE; +} + /** * dax_kmem_do_hotplug - hotplug memory for dax kmem device * @dev_dax: the dev_dax instance * @data: the dax_kmem_data structure with resource tracking + * @online_type: the online policy to use for the memory blocks * - * Hotplugs all ranges in the dev_dax region as system memory. + * Hotplugs all ranges in the dev_dax region as system memory with the + * provided online policy (offline, online, online_movable, online_kernel). * * Returns the number of successfully mapped ranges, or negative error. */ @@ -77,9 +89,15 @@ static int dax_kmem_do_hotplug(struct dev_dax *dev_dax, int online_type) { struct device *dev = &dev_dax->dev; - int i, rc, onlined = 0; + int i, rc, added = 0; mhp_t mhp_flags; + if (dax_kmem_state_is_online(data->state)) + return -EINVAL; + + if (online_type < MMOP_OFFLINE || online_type > MMOP_ONLINE_MOVABLE) + return -EINVAL; + for (i = 0; i < dev_dax->nr_range; i++) { struct range range; @@ -123,14 +141,14 @@ static int dax_kmem_do_hotplug(struct dev_dax *dev_dax, kfree(data->res[i]); data->res[i] = NULL; } - if (onlined) + if (added) continue; return rc; } - onlined++; + added++; } - return onlined; + return added; } /** @@ -193,45 +211,64 @@ static int dax_kmem_init_resources(struct dev_dax *dev_dax, * @dev_dax: the dev_dax instance * @data: the dax_kmem_data structure with resource tracking * - * Removes all ranges in the dev_dax region. + * Offlines and removes every currently-added range in the dev_dax region + * atomically: either all ranges are offlined and removed, or none are and + * the device is returned to its prior state. * - * Returns the number of successfully removed ranges. + * Returns 0 on success, or a negative errno on failure. */ static int dax_kmem_do_hotremove(struct dev_dax *dev_dax, struct dax_kmem_data *data) { struct device *dev = &dev_dax->dev; - int i, success = 0; + struct range *ranges; + int i, nr_ranges = 0, rc; + ranges = kmalloc_objs(*ranges, dev_dax->nr_range); + if (!ranges) + return -ENOMEM; + + /* Collect the ranges that were actually added during probe. */ for (i = 0; i < dev_dax->nr_range; i++) { struct range range; - int rc; - rc = dax_kmem_range(dev_dax, i, &range); - if (rc) + if (!data->res[i]) continue; - - /* range was never added during probe, count as removed */ - if (!data->res[i]) { - success++; + if (dax_kmem_range(dev_dax, i, &range)) continue; - } - - rc = remove_memory(range.start, range_len(&range)); - if (rc == 0) { - /* Release the resource for the successfully removed range */ - remove_resource(data->res[i]); - kfree(data->res[i]); - data->res[i] = NULL; - success++; - continue; - } - any_hotremove_failed = true; - dev_err(dev, "mapping%d: %#llx-%#llx hotremove failed\n", - i, range.start, range.end); + ranges[nr_ranges++] = range; } - return success; + /* Nothing added means nothing to remove. */ + if (!nr_ranges) { + kfree(ranges); + return 0; + } + + rc = offline_and_remove_memory_ranges(ranges, nr_ranges); + kfree(ranges); + if (rc) { + /* Recoverable: the ranges rolled back, nothing is leaked yet. */ + dev_err(dev, "hotremove failed, device left online: %d\n", rc); + return rc; + } + + /* All ranges removed; release the reserved resources. */ + for (i = 0; i < dev_dax->nr_range; i++) { + if (!data->res[i]) + continue; + remove_resource(data->res[i]); + kfree(data->res[i]); + data->res[i] = NULL; + } + + return 0; +} +#else +static int dax_kmem_do_hotremove(struct dev_dax *dev_dax, + struct dax_kmem_data *data) +{ + return -EBUSY; } #endif /* CONFIG_MEMORY_HOTREMOVE */ @@ -247,6 +284,18 @@ static void dax_kmem_cleanup_resources(struct dev_dax *dev_dax, { int i; + /* + * If the device unbind occurs before memory is hotremoved, we can never + * remove the memory (requires reboot). Attempting an offline operation + * here may cause deadlock and a failure to finish the unbind. + * + * Note: This leaks the resources. + */ + if (WARN(((data->state != DAX_KMEM_UNPLUGGED) && + (data->state != MMOP_OFFLINE)), + "Hotplug memory regions stuck online until reboot")) + return; + for (i = 0; i < dev_dax->nr_range; i++) { if (!data->res[i]) continue; @@ -256,6 +305,81 @@ static void dax_kmem_cleanup_resources(struct dev_dax *dev_dax, } } +static int dax_kmem_parse_state(const char *buf) +{ + int online_type; + + /* "unplugged" is kmem-specific - the rest map to MMOP_ */ + if (sysfs_streq(buf, "unplugged")) + return DAX_KMEM_UNPLUGGED; + + online_type = mhp_online_type_from_str(buf); + /* Disallow "offline": it's not useful and creates race conditions */ + if (online_type == MMOP_OFFLINE) + return -EINVAL; + return online_type; +} + +static ssize_t state_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dax_kmem_data *data = dev_get_drvdata(dev); + const char *state_str; + + if (data->state == DAX_KMEM_UNPLUGGED) + state_str = "unplugged"; + else + state_str = mhp_online_type_to_str(data->state); + + return sysfs_emit(buf, "%s\n", state_str ?: "unknown"); +} + +static ssize_t state_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t len) +{ + struct dev_dax *dev_dax = to_dev_dax(dev); + struct dax_kmem_data *data = dev_get_drvdata(dev); + int online_type; + int rc; + + online_type = dax_kmem_parse_state(buf); + if (online_type < DAX_KMEM_UNPLUGGED) + return online_type; + + guard(mutex)(&data->lock); + + /* Already in requested state */ + if (data->state == online_type) + return len; + + if (online_type == DAX_KMEM_UNPLUGGED) { + rc = dax_kmem_do_hotremove(dev_dax, data); + if (rc) + return rc; + data->state = DAX_KMEM_UNPLUGGED; + return len; + } + + /* Onlining is only allowed from the unplugged state. */ + if (data->state != DAX_KMEM_UNPLUGGED) + return -EBUSY; + + /* Re-acquire resources if previously unplugged, otherwise no-op */ + rc = dax_kmem_init_resources(dev_dax, data); + if (rc < 0) + return rc; + + rc = dax_kmem_do_hotplug(dev_dax, data, online_type); + if (rc < 0) { + /* Total failure, drop the reservations we took. */ + dax_kmem_cleanup_resources(dev_dax, data); + return rc; + } + + data->state = online_type; + return len; +} + static int dev_dax_kmem_probe(struct dev_dax *dev_dax) { struct device *dev = &dev_dax->dev; @@ -324,6 +448,8 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) if (rc < 0) goto err_reg_mgid; data->mgid = rc; + data->state = DAX_KMEM_UNPLUGGED; + mutex_init(&data->lock); dev_set_drvdata(dev, data); @@ -334,6 +460,7 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) rc = dax_kmem_do_hotplug(dev_dax, data, online_type); if (rc < 0) goto err_hotplug; + data->state = online_type; return 0; @@ -352,26 +479,59 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) } #ifdef CONFIG_MEMORY_HOTREMOVE +/* + * Remove the device's added ranges with remove_memory(). + * Unlike the sysfs unplug path it never offlines and fails if the blocks are + * online (-EBUSY), so it is safe from unbind. Failures leak until reboot. + * + * Returns 0 only if every added range was removed. + */ +static int dax_kmem_remove_ranges(struct dev_dax *dev_dax, + struct dax_kmem_data *data) +{ + struct device *dev = &dev_dax->dev; + int i, rc = 0; + + for (i = 0; i < dev_dax->nr_range; i++) { + struct range range; + + if (!data->res[i] || dax_kmem_range(dev_dax, i, &range)) + continue; + if (remove_memory(range.start, range_len(&range))) { + dev_warn(dev, "mapping%d: %#llx-%#llx stuck online until reboot\n", + i, range.start, range.end); + rc = -EBUSY; + continue; + } + remove_resource(data->res[i]); + kfree(data->res[i]); + data->res[i] = NULL; + } + return rc; +} + static void dev_dax_kmem_remove(struct dev_dax *dev_dax) { - int success; int node = dev_dax->target_node; struct device *dev = &dev_dax->dev; struct dax_kmem_data *data = dev_get_drvdata(dev); /* - * We have one shot for removing memory, if some memory blocks were not - * offline prior to calling this function remove_memory() will fail, and - * there is no way to hotremove this memory until reboot because device - * unbind will succeed even if we return failure. + * Remove every range that is still added. dax_kmem_remove_ranges() + * uses remove_memory(), which never offlines: an online block fails + * with -EBUSY rather than deadlocking an uninterruptible unbind. + * + * data->state only tracks daxX.Y/state writes, so it can be stale if + * blocks were toggled via memoryX/state. Do not trust it here and + * attempt simply remove_memory() - which reports the true state of + * each range anyway. Anything left online is leaked until reboot. */ - success = dax_kmem_do_hotremove(dev_dax, data); - if (success < dev_dax->nr_range) { + if (dax_kmem_remove_ranges(dev_dax, data)) { dev_err(dev, "Hotplug regions stuck online until reboot\n"); + any_hotremove_failed = true; return; } - dax_kmem_cleanup_resources(dev_dax, data); memory_group_unregister(data->mgid); kfree(data->res_name); kfree(data); @@ -399,10 +559,21 @@ static void dev_dax_kmem_remove(struct dev_dax *dev_dax) } #endif /* CONFIG_MEMORY_HOTREMOVE */ +static DEVICE_ATTR_RW(state); + +static struct attribute *dev_dax_kmem_attrs[] = { + &dev_attr_state.attr, + NULL, +}; +ATTRIBUTE_GROUPS(dev_dax_kmem); + static struct dax_device_driver device_dax_kmem_driver = { .probe = dev_dax_kmem_probe, .remove = dev_dax_kmem_remove, .type = DAXDRV_KMEM_TYPE, + .drv = { + .dev_groups = dev_dax_kmem_groups, + }, }; static int __init dax_kmem_init(void) From 249c5ee21a26867b3b165f59cc0f2de335bd35bb Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Sun, 12 Jul 2026 11:45:04 -0400 Subject: [PATCH 399/501] selftests/dax: add dax/kmem hotplug sysfs regression test Add a kselftest for the dax/kmem whole-device "state" sysfs attribute (/sys/bus/dax/devices/daxX.Y/state), which transitions a kmem-backed dax device between "unplugged", "online" and "online_movable". The kselftest also includes a test to demonstrate the force-unbind does not deadlock - but this is destructive (the dax device can never be rebound), so it only runs when DAX_KMEM_TEST_UNBIND=1 is set. Provisioning a devdax device and binding it to kmem needs daxctl/ndctl out of scope for an in-tree selftest. As the test mutates a device's memory, the operator opts in by naming it in DAX_KMEM_TEST_DEV (or "auto" to pick the first kmem-bound device); it SKIPs when unset, when no device is present, or when the memory cannot be freed to a baseline. When a device is available it validates the interface contract: - online / online_movable actually add memory (MemTotal grows), - online is idempotent, - switching between online types without unplug is rejected, - unplug removes memory and the reported state is "unplugged" - invalid input is rejected, - unplug and unbind tolerate blocks toggled out-of-band through the per-block memoryX/state interface. One specific regression test: online -> unplug -> online_movable -> unplug Re-online must re-reserve per-range resources so subsequent unplug actually offlines and removes instead of silently reporting success while the memory stays online. Link: https://lore.kernel.org/20260712154505.3564379-11-gourry@gourry.net Signed-off-by: Gregory Price Cc: Alison Schofield Cc: Danilo Krummrich Cc: Dave Jiang Cc: David Hildenbrand (Arm) Cc: Greg Kroah-Hartman Cc: Hannes Reinecke Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Oscar Salvador Cc: Pankaj Gupta Cc: "Rafael J. Wysocki" Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vishal Verma Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- tools/testing/selftests/Makefile | 1 + tools/testing/selftests/dax/Makefile | 6 + tools/testing/selftests/dax/config | 4 + .../testing/selftests/dax/dax-kmem-hotplug.sh | 317 ++++++++++++++++++ tools/testing/selftests/dax/settings | 1 + 5 files changed, 329 insertions(+) create mode 100644 tools/testing/selftests/dax/Makefile create mode 100644 tools/testing/selftests/dax/config create mode 100755 tools/testing/selftests/dax/dax-kmem-hotplug.sh create mode 100644 tools/testing/selftests/dax/settings diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 8d4db2241cc2..5528682a3a91 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -14,6 +14,7 @@ TARGETS += core TARGETS += cpufreq TARGETS += cpu-hotplug TARGETS += damon +TARGETS += dax TARGETS += devices/error_logs TARGETS += devices/probe TARGETS += dmabuf-heaps diff --git a/tools/testing/selftests/dax/Makefile b/tools/testing/selftests/dax/Makefile new file mode 100644 index 000000000000..25a4f3d73a5b --- /dev/null +++ b/tools/testing/selftests/dax/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +all: + +TEST_PROGS := dax-kmem-hotplug.sh + +include ../lib.mk diff --git a/tools/testing/selftests/dax/config b/tools/testing/selftests/dax/config new file mode 100644 index 000000000000..4c9aaeb6ceb4 --- /dev/null +++ b/tools/testing/selftests/dax/config @@ -0,0 +1,4 @@ +CONFIG_DEV_DAX=m +CONFIG_DEV_DAX_KMEM=m +CONFIG_MEMORY_HOTPLUG=y +CONFIG_MEMORY_HOTREMOVE=y diff --git a/tools/testing/selftests/dax/dax-kmem-hotplug.sh b/tools/testing/selftests/dax/dax-kmem-hotplug.sh new file mode 100755 index 000000000000..9299120ee509 --- /dev/null +++ b/tools/testing/selftests/dax/dax-kmem-hotplug.sh @@ -0,0 +1,317 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Exercise the dax/kmem "state" sysfs attribute: +# /sys/bus/dax/devices/daxX.Y/state -> unplugged | online | online_kernel | online_movable +# +# The test needs a dax device already bound to the kmem driver. +# +# This test mutates a device's memory: online/offline cycles migrate any +# in-use pages, and the optional unbind subtest wedges the device until +# reboot. The tester must identify the target device and opt into the +# destructive unbind tests. +# +# DAX_KMEM_TEST_DEV=daxX.Y test this specific device +# DAX_KMEM_TEST_DEV=auto auto-discover the first kmem-bound dax device +# (best-effort: it may be a device in use!) +# DAX_KMEM_TEST_UNBIND=1 also run the destructive unbind-while-online test +# +# If DAX_KMEM_TEST_DEV is unset the whole test SKIPs. +# +# A dax device can be provisioned with the memmap= boot param, e.g.: +# memmap=2G!4G +# +# then, in the booted system: +# +# ndctl create-namespace -m devdax -e namespace0.0 -f +# daxctl reconfigure-device -N -m system-ram dax0.0 # bind kmem +# DAX_KMEM_TEST_DEV=auto ./dax-kmem-hotplug.sh + +# shellcheck disable=SC1091 +DIR="$(dirname "$(readlink -f "$0")")" +. "$DIR"/../kselftest/ktap_helpers.sh + +DAX_BASE=/sys/bus/dax/devices +MEM_BASE=/sys/devices/system/memory + +memtotal_kb() { awk '/^MemTotal:/ {print $2}' /proc/meminfo; } +get_state() { cat "$HP" 2>/dev/null; } +# set_state STATE -- write a state to the state attribute; returns the +# write's exit status (0 = accepted by the kernel) +set_state() { echo "$1" > "$HP" 2>/dev/null; } + +is_kmem_dax() { + local drv + [ -e "$DAX_BASE/$1/state" ] || return 1 + drv=$(readlink "$DAX_BASE/$1/driver" 2>/dev/null) + [ "$(basename "${drv:-}")" = kmem ] +} + +find_kmem_dax() { + local d + for d in "$DAX_BASE"/dax*; do + is_kmem_dax "$(basename "$d")" || continue + basename "$d" + return 0 + done + return 1 +} + +# find_device_blocks -- print every memoryN block backing this dax device. +# The blocks are derived from the device's own range(s) in /proc/iomem (the +# reserved resource is named after the device), so we act on *its* blocks +# rather than guessing by NUMA node - the target node may also hold unrelated +# (and non-offlineable) memory. +find_device_blocks() { + local bs + bs=$(cat "$MEM_BASE/block_size_bytes" 2>/dev/null) # hex, no leading 0x + [ -n "$bs" ] || return 1 + grep -E " : ${DAX}\$" /proc/iomem | while read -r line; do + local range s e i + range=${line%% :*}; range=${range// /} + s=${range%-*}; e=${range#*-} + for (( i = 0x$s / 0x$bs; i <= 0x$e / 0x$bs; i++ )); do + echo "memory$i" + done + done +} + +# find_device_block -- print the first online block backing this dax device. +find_device_block() { + local b + for b in $(find_device_blocks); do + [ -f "$MEM_BASE/$b/state" ] || continue + [ "$(cat "$MEM_BASE/$b/state")" = online ] || continue + echo "$b" + return 0 + done + return 1 +} + +ktap_print_header + +if [ "$UID" != 0 ]; then + ktap_skip_all "must be run as root" + exit "$KSFT_SKIP" +fi + +# Device selection is opt-in - see the header for why. +DEV_SEL=${DAX_KMEM_TEST_DEV:-} +if [ -z "$DEV_SEL" ]; then + ktap_skip_all "set DAX_KMEM_TEST_DEV= to opt in (mutates device memory)" + exit "$KSFT_SKIP" +fi +if [ "$DEV_SEL" = auto ]; then + DAX=$(find_kmem_dax) +else + DAX=$DEV_SEL +fi +if [ -z "$DAX" ] || ! is_kmem_dax "$DAX"; then + ktap_skip_all "no kmem-bound dax device with a state attribute (${DEV_SEL})" + exit "$KSFT_SKIP" +fi +HP=$DAX_BASE/$DAX/state +ORIG=$(get_state) + +# A failure to reach the baseline is environmental (memory in use), not an +# interface failure, so skip rather than fail. +set_state unplugged; rc=$? +if [ "$rc" != 0 ] || [ "$(get_state)" != unplugged ]; then + ktap_skip_all "$DAX: cannot reach 'unplugged' baseline (memory in use?)" + [ -n "$ORIG" ] && set_state "$ORIG" + exit "$KSFT_SKIP" +fi +mt_unplugged=$(memtotal_kb) + +DRV=/sys/bus/dax/drivers/kmem +AOB=$MEM_BASE/auto_online_blocks + +ktap_print_msg "using $DAX (initial state was: $ORIG)" +ktap_set_plan 10 + +# A public (N_MEMORY) kmem node onlined into a kernel zone (online/online_kernel) +# collects unmovable allocations and can then never be offlined, which would +# wedge the device for the rest of this test. So this test only ever +# successfully onlines online_movable, the one mode that is reliably unpluggable. + +set_state online_movable; rc=$? +mt_online=$(memtotal_kb) +if [ "$rc" = 0 ] && [ "$(get_state)" = online_movable ] && [ "$mt_online" -gt "$mt_unplugged" ]; then + ktap_test_pass "online_movable: state=online_movable, MemTotal $mt_unplugged -> $mt_online kB" +else + ktap_test_fail "online_movable: rc=$rc state=$(get_state) MemTotal $mt_unplugged -> $mt_online" +fi + +set_state online_movable; rc=$? +if [ "$rc" = 0 ] && [ "$(get_state)" = online_movable ]; then + ktap_test_pass "online_movable idempotent" +else + ktap_test_fail "online_movable idempotent: rc=$rc state=$(get_state)" +fi + +# A different online type is rejected without an intervening unplug. The write +# is refused before any hotplug, so this never actually onlines a kernel zone. +set_state online_kernel; rc=$? +if [ "$rc" != 0 ] && [ "$(get_state)" = online_movable ]; then + ktap_test_pass "reject online_kernel without intervening unplug (no kernel-zone online)" +else + ktap_test_fail "online_movable->online_kernel not rejected: rc=$rc state=$(get_state)" +fi + +set_state unplugged; rc=$? +mt=$(memtotal_kb) +if [ "$rc" = 0 ] && [ "$(get_state)" = unplugged ] && [ "$mt" -lt "$mt_online" ]; then + ktap_test_pass "unplug from online_movable: MemTotal $mt_online -> $mt kB" +else + ktap_test_fail "unplug from online_movable: rc=$rc state=$(get_state) MemTotal $mt_online -> $mt" +fi + +before=$(get_state) +set_state bogus_state; rc=$? +if [ "$rc" != 0 ] && [ "$(get_state)" = "$before" ]; then + ktap_test_pass "reject invalid state string" +else + ktap_test_fail "invalid state not rejected: rc=$rc state=$(get_state)" +fi + +# An online_movable -> unplug cycle must re-acquire the per-range resources on +# each online and release them on each unplug. Assert every iteration grows +# MemTotal past the baseline and returns exactly to it; memory left online after +# unplug (off > baseline) is a partial-free failure. +set_state unplugged +cycle_ok=1; fail_i=0; on=0; off=0 +for i in 1 2 3; do + if ! set_state online_movable; then cycle_ok=0; fail_i=$i; break; fi + on=$(memtotal_kb) + if ! set_state unplugged; then cycle_ok=0; fail_i=$i; break; fi + off=$(memtotal_kb) + # online must grow past baseline, and unplug must return to it - a + # partial free (memory left online) is a failure, not just off == on. + if [ "$on" -le "$mt_unplugged" ] || [ "$off" -gt "$mt_unplugged" ]; then + cycle_ok=0; fail_i=$i; break + fi +done +if [ "$cycle_ok" = 1 ]; then + ktap_test_pass "online_movable/unplug cycle re-acquires resources (3x: added and freed each time)" +else + ktap_test_fail "online_movable/unplug cycle regressed at iteration $fail_i (on=$on off=$off baseline=$mt_unplugged)" +fi + +# Desync: toggle a block through the legacy per-block memoryN/state interface +# behind the driver's back, then unplug the whole device via daxX.Y/state. +# +# The driver only updates daxX.Y/state on its own writes, so it still reports +# online_movable while a block underneath is already offline. +# +# Whole-device unplug must still succeed (within reason, an actor changing a +# device from online_movable to online_kernel can no longer guarantee unplug). +# At the very least, an already-offline block should not produce an error. +set_state unplugged +set_state online_movable +blk=$(find_device_block) +if [ -n "$blk" ] && echo offline > "$MEM_BASE/$blk/state" 2>/dev/null; then + # daxX.Y/state is now stale (still online_movable); unplug the device. + set_state unplugged; rc=$? + mt=$(memtotal_kb) + if [ "$rc" = 0 ] && [ "$(get_state)" = unplugged ] && [ "$mt" -le "$mt_unplugged" ]; then + ktap_test_pass "unplug tolerates a block pre-offlined via memoryN/state ($blk)" + else + ktap_test_fail "desync unplug: rc=$rc state=$(get_state) MemTotal=$mt baseline=$mt_unplugged" + fi +else + set_state unplugged 2>/dev/null + ktap_test_skip "could not locate a device block to offline for desync test" +fi + +# change system default online policy while the device is unbound, and show +# the new system default policy is utilized across bindings. +set_state unplugged +if [ -w "$AOB" ] && [ -w "$DRV/unbind" ] && [ -w "$DRV/bind" ]; then + orig_aob=$(cat "$AOB") + echo "$DAX" > "$DRV/unbind" 2>/dev/null + echo offline > "$AOB" 2>/dev/null + echo "$DAX" > "$DRV/bind" 2>/dev/null + sleep 1 + st=$(get_state) + echo "$orig_aob" > "$AOB" 2>/dev/null # restore system policy + if [ "$st" = offline ]; then + ktap_test_pass "online policy resolved at bind: auto_online_blocks=offline -> state=offline" + else + ktap_test_fail "bind-time policy not honored: state=$st (expected offline)" + fi + set_state unplugged 2>/dev/null +else + ktap_test_skip "auto_online_blocks or driver bind/unbind not writable" +fi + +# Blocks offlined out-of-band (via memoryN/state) leave daxX.Y/state stale +# (still online_movable) while every block is actually offline. A driver unbind +# must still hot-remove the offline memory and free its resources rather than +# trust the stale state and leak until reboot. Unbind uses remove_memory(), +# which never offlines, so removing already-offline blocks is non-destructive and +# the device rebinds cleanly afterwards. +if [ -w "$DRV/unbind" ] && [ -w "$DRV/bind" ]; then + set_state unplugged + set_state online_movable + offl_ok=1 + for b in $(find_device_blocks); do + [ -f "$MEM_BASE/$b/state" ] || continue + [ "$(cat "$MEM_BASE/$b/state")" = online ] || continue + echo offline > "$MEM_BASE/$b/state" 2>/dev/null || offl_ok=0 + done + # daxX.Y/state is now stale (still online_movable) while all blocks are + # offline; the unbind must hot-remove them anyway. + if [ "$offl_ok" = 1 ] && [ "$(get_state)" = online_movable ]; then + echo "$DAX" > "$DRV/unbind" 2>/dev/null + mt_after=$(memtotal_kb) + leaked=$(grep -cE " : ${DAX}\$" /proc/iomem) # before rebind + echo "$DAX" > "$DRV/bind" 2>/dev/null # restore for later steps + sleep 1 + if [ "$mt_after" -le "$mt_unplugged" ] && [ "$leaked" = 0 ]; then + ktap_test_pass "unbind with stale online state hot-removes offlined blocks (no leak)" + else + ktap_test_fail "desync unbind leaked: MemTotal=$mt_after baseline=$mt_unplugged iomem_left=$leaked" + fi + set_state unplugged 2>/dev/null + else + ktap_test_skip "could not offline all device blocks for desync-unbind test" + fi +else + ktap_test_skip "driver bind/unbind not writable for desync-unbind test" +fi + +[ -n "$ORIG" ] && set_state "$ORIG" + +# DESTRUCTIVE and opt-in only (DAX_KMEM_TEST_UNBIND=1): +# +# unbinding the driver while memory is online causes the resources to leak - but +# the unbind should not deadlock. Instead the driver leaks it with a warning. + +# This leaves the memory online and the device unbound until reboot, so it runs +# last and only when explicitly requested. online_movable only: this test +# never onlines a public node into a kernel zone. + +if [ "${DAX_KMEM_TEST_UNBIND:-}" = 1 ] && [ -w "$DRV/unbind" ]; then + set_state unplugged; set_state online_movable +fi +if [ "${DAX_KMEM_TEST_UNBIND:-}" = 1 ] && [ "$(get_state)" = online_movable ] && + [ -w "$DRV/unbind" ]; then + mt_on=$(memtotal_kb) + dmesg -C 2>/dev/null + echo "$DAX" > "$DRV/unbind" 2>/dev/null + mt_after=$(memtotal_kb) + # The leaked "System RAM (kmem)" regions stay in the iomem tree; reading + # their names dereferences res_name, which a buggy unbind already freed. + # Walk /proc/iomem to provoke that use-after-free (caught by KASAN). + cat /proc/iomem > /dev/null 2>&1 + splat=$(dmesg 2>/dev/null | grep -ciE "KASAN|BUG:|use-after-free|general protection|Oops|refcount_t") + if [ "$splat" = 0 ] && [ "$mt_after" -ge "$mt_on" ]; then + ktap_test_pass "unbind while online: memory left online, no UAF/oops (MemTotal $mt_on -> $mt_after kB)" + else + ktap_test_fail "unbind while online regressed: splat=$splat MemTotal $mt_on -> $mt_after kB" + fi +else + ktap_test_skip "destructive unbind-while-online test (set DAX_KMEM_TEST_UNBIND=1)" +fi + +ktap_finished diff --git a/tools/testing/selftests/dax/settings b/tools/testing/selftests/dax/settings new file mode 100644 index 000000000000..ba4d85f74cd6 --- /dev/null +++ b/tools/testing/selftests/dax/settings @@ -0,0 +1 @@ +timeout=90 From 3a50308b3926b8b966145c618fbe3c3ca6ec95a6 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Sat, 11 Jul 2026 19:44:58 +0100 Subject: [PATCH 400/501] mm: introduce vma_flags_can_grow() and vma_can_grow() Patch series "mm: convert more vm_flags_t users to vma_flags_t", v2. This series makes further progress in converting usage of the deprecated vm_flags_t type to its replacement, vma_flags_t. It focuses on mm, though updates some users of mm APIs also. It updates: * The core do_mmap() code path for VMA mapping. * Unmapped area logic. * The usage of mm->def_vma_flags. * VMA page protection bit logic. * General usage of VMA flags in core mm code, mlock, mprotect, mremap. This patch (of 13): These test whether the VMA has stack semantics, i.e. is able to grow upwards or downwards depending on the architecture. In order to account for arches which do not support upward-growing stacks, introduce VMA_GROWSUP whose definition depends on the architecture supporting it, and use vma_flags_test_single_mask() in vma_flags_can_grow() to account for this. No functional change intended. Link: https://lore.kernel.org/20260711-b4-vma-flags-mm-v2-0-0fa2357d5431@kernel.org Link: https://lore.kernel.org/20260711-b4-vma-flags-mm-v2-1-0fa2357d5431@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Zi Yan Reviewed-by: Lance Yang Reviewed-by: Vlastimil Babka (SUSE) Cc: Baolin Wang Cc: Barry Song Cc: Christian Brauner Cc: Dave Airlie Cc: David Hildenbrand Cc: Dev Jain Cc: Jan Kara Cc: Jann Horn Cc: Lance Yang Cc: Mike Rapoport Cc: Muchun Song Cc: Nico Pache Cc: Oscar Salvador Cc: Pedro Falcato Cc: Suren Baghdasaryan Cc: Zi Yan Cc: Jani Nikula Cc: Thomas Zimmermann Signed-off-by: Andrew Morton --- include/linux/mm.h | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 32bb723ffbb9..7a7f559b3df0 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -474,6 +474,7 @@ enum { #define VM_SAO INIT_VM_FLAG(SAO) #elif defined(CONFIG_PARISC) #define VM_GROWSUP INIT_VM_FLAG(GROWSUP) +#define VMA_GROWSUP mk_vma_flags(VMA_GROWSUP_BIT) #elif defined(CONFIG_SPARC64) #define VM_SPARC_ADI INIT_VM_FLAG(SPARC_ADI) #define VM_ARCH_CLEAR INIT_VM_FLAG(ARCH_CLEAR) @@ -485,6 +486,7 @@ enum { #endif #ifndef VM_GROWSUP #define VM_GROWSUP VM_NONE +#define VMA_GROWSUP EMPTY_VMA_FLAGS #endif #ifdef CONFIG_ARM64_MTE #define VM_MTE INIT_VM_FLAG(MTE) @@ -1578,11 +1580,24 @@ static inline bool vma_is_initial_stack(const struct vm_area_struct *vma) vma->vm_end >= vma->vm_mm->start_stack; } +static inline bool vma_flags_can_grow(const vma_flags_t *flags) +{ + if (vma_flags_test_single_mask(flags, VMA_GROWSUP)) + return true; + if (vma_flags_test(flags, VMA_GROWSDOWN_BIT)) + return true; + + return false; +} + +static inline bool vma_can_grow(const struct vm_area_struct *vma) +{ + return vma_flags_can_grow(&vma->flags); +} + static inline bool vma_is_temporary_stack(const struct vm_area_struct *vma) { - int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP); - - if (!maybe_stack) + if (!vma_can_grow(vma)) return false; if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) == From 1944a0183c83265dceba1cb6d1f89dae7db23d40 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Sat, 11 Jul 2026 19:44:59 +0100 Subject: [PATCH 401/501] mm/vma: update do_mmap() to use vma_flags_t The core do_mmap() function accepts a vm_flags_t parameter which it then manipulates before passing to mmap_region() to do the heavy lifting of the memory mapping. Update do_mmap() to instead accept a vma_flags_t parameter, and adjust all the logic within do_mmap() to manipulate this instead. This is as part of the ongoing effort to convert VMA flags from a system word size to a bitmap type which allows us to unrestrict the number of VMA flags, as well as gain control over how VMA flag manipulation occurs. We do not cascade these changes to all functions which accept vm_flags_t, but rather use vma_flags_to_legacy() where necessary, specifically deferring converting calc_vm_prot_bits(), calc_vm_flag_bits() and __get_unmapped_area() to vma_flags_t. Also utilise the new vma_flags_can_grow() predicate which correctly handles the case of architectures without upward growing stacks. As part of this change, introduce VMA_SHADOW_STACK so we can correctly handle the case of the shadow stack not being defined. No functional change intended. Link: https://lore.kernel.org/20260711-b4-vma-flags-mm-v2-2-0fa2357d5431@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Lance Yang Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Baolin Wang Cc: Barry Song Cc: Christian Brauner Cc: Dave Airlie Cc: David Hildenbrand Cc: Dev Jain Cc: Jani Nikula Cc: Jan Kara Cc: Jann Horn Cc: Mike Rapoport Cc: Muchun Song Cc: Nico Pache Cc: Oscar Salvador Cc: Pedro Falcato Cc: Suren Baghdasaryan Cc: Thomas Zimmermann Signed-off-by: Andrew Morton --- arch/mips/kernel/vdso.c | 4 +-- fs/aio.c | 2 +- include/linux/memfd.h | 6 ++-- include/linux/mm.h | 6 ++-- ipc/shm.c | 3 +- mm/memfd.c | 15 ++++----- mm/mmap.c | 67 ++++++++++++++++++++++++----------------- mm/nommu.c | 3 +- mm/util.c | 10 +++--- mm/vma.c | 7 ++--- mm/vma.h | 2 +- 11 files changed, 69 insertions(+), 56 deletions(-) diff --git a/arch/mips/kernel/vdso.c b/arch/mips/kernel/vdso.c index bd1fc17d3975..94873775fc0f 100644 --- a/arch/mips/kernel/vdso.c +++ b/arch/mips/kernel/vdso.c @@ -91,8 +91,8 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) /* Map delay slot emulation page */ base = do_mmap(NULL, STACK_TOP, PAGE_SIZE, PROT_READ | PROT_EXEC, - MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, 0, 0, &unused, - NULL); + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, + EMPTY_VMA_FLAGS, 0, &unused, NULL); if (IS_ERR_VALUE(base)) { ret = base; goto out; diff --git a/fs/aio.c b/fs/aio.c index f57fa21a2503..3de586da197b 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -604,7 +604,7 @@ static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events) ctx->mmap_base = do_mmap(ctx->aio_ring_file, 0, ctx->mmap_size, PROT_READ | PROT_WRITE, - MAP_SHARED, 0, 0, &unused, NULL); + MAP_SHARED, EMPTY_VMA_FLAGS, 0, &unused, NULL); mmap_write_unlock(mm); if (IS_ERR((void *)ctx->mmap_base)) { ctx->mmap_size = 0; diff --git a/include/linux/memfd.h b/include/linux/memfd.h index b4fda09dab9f..c159e40e3f34 100644 --- a/include/linux/memfd.h +++ b/include/linux/memfd.h @@ -14,9 +14,9 @@ struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx); * to sealing, or 0 otherwise. * * We also update VMA flags if appropriate by manipulating the VMA flags pointed - * to by vm_flags_ptr. + * to by vma_flags_ptr. */ -int memfd_check_seals_mmap(struct file *file, vm_flags_t *vm_flags_ptr); +int memfd_check_seals_mmap(struct file *file, vma_flags_t *vma_flags_ptr); struct file *memfd_alloc_file(const char *name, unsigned int flags); int memfd_get_seals(struct file *file); int memfd_add_seals(struct file *file, unsigned int seals); @@ -30,7 +30,7 @@ static inline struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx) return ERR_PTR(-EINVAL); } static inline int memfd_check_seals_mmap(struct file *file, - vm_flags_t *vm_flags_ptr) + vma_flags_t *vma_flags_ptr) { return 0; } diff --git a/include/linux/mm.h b/include/linux/mm.h index 7a7f559b3df0..d20aa2f80472 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -465,9 +465,11 @@ enum { #if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_ARM64_GCS) || \ defined(CONFIG_RISCV_USER_CFI) #define VM_SHADOW_STACK INIT_VM_FLAG(SHADOW_STACK) +#define VMA_SHADOW_STACK mk_vma_flags(VMA_SHADOW_STACK_BIT) #define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT, VMA_SHADOW_STACK_BIT) #else #define VM_SHADOW_STACK VM_NONE +#define VMA_SHADOW_STACK EMPTY_VMA_FLAGS #define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT) #endif #if defined(CONFIG_PPC64) @@ -4198,9 +4200,9 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, return __get_unmapped_area(file, addr, len, pgoff, flags, 0); } -extern unsigned long do_mmap(struct file *file, unsigned long addr, +unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, - vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, + vma_flags_t vma_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf); extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf, diff --git a/ipc/shm.c b/ipc/shm.c index b3e8a58e177d..bb1a721a3e74 100644 --- a/ipc/shm.c +++ b/ipc/shm.c @@ -1661,7 +1661,8 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, goto invalid; } - addr = do_mmap(file, addr, size, prot, flags, 0, 0, &populate, NULL); + addr = do_mmap(file, addr, size, prot, flags, EMPTY_VMA_FLAGS, 0, + &populate, NULL); *raddr = addr; err = 0; if (IS_ERR_VALUE(addr)) diff --git a/mm/memfd.c b/mm/memfd.c index 6c72fe6caef7..c708d92533f4 100644 --- a/mm/memfd.c +++ b/mm/memfd.c @@ -370,39 +370,36 @@ static inline bool is_write_sealed(unsigned int seals) return seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE); } -static int check_write_seal(vm_flags_t *vm_flags_ptr) +static int check_write_seal(vma_flags_t *vma_flags_ptr) { - vm_flags_t vm_flags = *vm_flags_ptr; - vm_flags_t mask = vm_flags & (VM_SHARED | VM_WRITE); - /* If a private mapping then writability is irrelevant. */ - if (!(mask & VM_SHARED)) + if (!vma_flags_test(vma_flags_ptr, VMA_SHARED_BIT)) return 0; /* * New PROT_WRITE and MAP_SHARED mmaps are not allowed when * write seals are active. */ - if (mask & VM_WRITE) + if (vma_flags_test(vma_flags_ptr, VMA_WRITE_BIT)) return -EPERM; /* * This is a read-only mapping, disallow mprotect() from making a * write-sealed mapping writable in future. */ - *vm_flags_ptr &= ~VM_MAYWRITE; + vma_flags_clear(vma_flags_ptr, VMA_MAYWRITE_BIT); return 0; } -int memfd_check_seals_mmap(struct file *file, vm_flags_t *vm_flags_ptr) +int memfd_check_seals_mmap(struct file *file, vma_flags_t *vma_flags_ptr) { int err = 0; unsigned int *seals_ptr = memfd_file_seals_ptr(file); unsigned int seals = seals_ptr ? *seals_ptr : 0; if (is_write_sealed(seals)) - err = check_write_seal(vm_flags_ptr); + err = check_write_seal(vma_flags_ptr); return err; } diff --git a/mm/mmap.c b/mm/mmap.c index 308a43eabd83..bcc3793e532d 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -280,7 +280,7 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, * do_mmap() - Perform a userland memory mapping into the current process * address space of length @len with protection bits @prot, mmap flags @flags * (from which VMA flags will be inferred), and any additional VMA flags to - * apply @vm_flags. If this is a file-backed mapping then the file is specified + * apply @vma_flags. If this is a file-backed mapping then the file is specified * in @file and page offset into the file via @pgoff. * * This function does not perform security checks on the file and assumes, if @@ -320,7 +320,8 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, * (2) for details. * @flags: Flags specifying how the mapping should be performed, see mmap (2) * for details. - * @vm_flags: VMA flags which should be set by default, or 0 otherwise. + * @vma_flags: VMA flags which should be set by default, or EMPTY_VMA_FLAGS + * otherwise. * @pgoff: Page offset into the @file if file-backed, should be 0 otherwise. * @populate: A pointer to a value which will be set to 0 if no population of * the range is required, or the number of bytes to populate if it is. Must be @@ -335,7 +336,7 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, */ unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, - unsigned long flags, vm_flags_t vm_flags, + unsigned long flags, vma_flags_t vma_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf) { @@ -399,13 +400,19 @@ unsigned long do_mmap(struct file *file, unsigned long addr, * to. we assume access permissions have been handled by the open * of the memory object, so we don't do any here. */ - vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(file, flags) | - mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; + vma_flags_set_mask(&vma_flags, + legacy_to_vma_flags(calc_vm_prot_bits(prot, pkey))); + vma_flags_set_mask(&vma_flags, + legacy_to_vma_flags(calc_vm_flag_bits(file, flags))); + vma_flags_set_mask(&vma_flags, mm->def_vma_flags); + vma_flags_set(&vma_flags, VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, + VMA_MAYEXEC_BIT); /* Obtain the address to map to. we verify (or select) it and ensure * that it represents a valid section of the address space. */ - addr = __get_unmapped_area(file, addr, len, pgoff, flags, vm_flags); + addr = __get_unmapped_area(file, addr, len, pgoff, flags, + vma_flags_to_legacy(vma_flags)); if (IS_ERR_VALUE(addr)) return addr; @@ -418,7 +425,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, if (!can_do_mlock()) return -EPERM; - if (!mlock_future_ok(mm, vm_flags & VM_LOCKED, len)) + if (!mlock_future_ok(mm, vma_flags_test(&vma_flags, VMA_LOCKED_BIT), len)) return -EAGAIN; if (file) { @@ -461,22 +468,23 @@ unsigned long do_mmap(struct file *file, unsigned long addr, if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE)) return -EACCES; - vm_flags |= VM_SHARED | VM_MAYSHARE; + vma_flags_set(&vma_flags, VMA_SHARED_BIT, VMA_MAYSHARE_BIT); if (!(file->f_mode & FMODE_WRITE)) - vm_flags &= ~(VM_MAYWRITE | VM_SHARED); + vma_flags_clear(&vma_flags, VMA_MAYWRITE_BIT, + VMA_SHARED_BIT); fallthrough; case MAP_PRIVATE: if (!(file->f_mode & FMODE_READ)) return -EACCES; if (path_noexec(&file->f_path)) { - if (vm_flags & VM_EXEC) + if (vma_flags_test(&vma_flags, VMA_EXEC_BIT)) return -EPERM; - vm_flags &= ~VM_MAYEXEC; + vma_flags_clear(&vma_flags, VMA_MAYEXEC_BIT); } if (!can_mmap_file(file)) return -ENODEV; - if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP)) + if (vma_flags_can_grow(&vma_flags)) return -EINVAL; break; @@ -488,23 +496,27 @@ unsigned long do_mmap(struct file *file, unsigned long addr, * Check to see if we are violating any seals and update VMA * flags if necessary to avoid future seal violations. */ - err = memfd_check_seals_mmap(file, &vm_flags); + err = memfd_check_seals_mmap(file, &vma_flags); if (err) return (unsigned long)err; } else { switch (flags & MAP_TYPE) { case MAP_SHARED: - if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP)) + if (vma_flags_can_grow(&vma_flags)) return -EINVAL; /* * Ignore pgoff. */ pgoff = 0; - vm_flags |= VM_SHARED | VM_MAYSHARE; + vma_flags_set(&vma_flags, VMA_SHARED_BIT, VMA_MAYSHARE_BIT); break; - case MAP_DROPPABLE: - if (VM_DROPPABLE == VM_NONE) + case MAP_DROPPABLE: { + vma_flags_t droppable = VMA_DROPPABLE; + + if (vma_flags_empty(&droppable)) return -EOPNOTSUPP; + vma_flags_set_mask(&vma_flags, droppable); + /* * A locked or stack area makes no sense to be droppable. * @@ -515,23 +527,24 @@ unsigned long do_mmap(struct file *file, unsigned long addr, */ if (flags & (MAP_LOCKED | MAP_HUGETLB)) return -EINVAL; - if (vm_flags & (VM_GROWSDOWN | VM_GROWSUP)) + if (vma_flags_can_grow(&vma_flags)) return -EINVAL; - vm_flags |= VM_DROPPABLE; - /* * If the pages can be dropped, then it doesn't make * sense to reserve them. */ - vm_flags |= VM_NORESERVE; + vma_flags_set(&vma_flags, VMA_NORESERVE_BIT); /* * Likewise, they're volatile enough that they * shouldn't survive forks or coredumps. */ - vm_flags |= VM_WIPEONFORK | VM_DONTDUMP; + vma_flags_set(&vma_flags, VMA_WIPEONFORK_BIT, + VMA_DONTDUMP_BIT); + fallthrough; + } case MAP_PRIVATE: /* * Set pgoff according to addr for anon_vma. @@ -550,16 +563,16 @@ unsigned long do_mmap(struct file *file, unsigned long addr, if (flags & MAP_NORESERVE) { /* We honor MAP_NORESERVE if allowed to overcommit */ if (sysctl_overcommit_memory != OVERCOMMIT_NEVER) - vm_flags |= VM_NORESERVE; + vma_flags_set(&vma_flags, VMA_NORESERVE_BIT); /* hugetlb applies strict overcommit unless MAP_NORESERVE */ if (file && is_file_hugepages(file)) - vm_flags |= VM_NORESERVE; + vma_flags_set(&vma_flags, VMA_NORESERVE_BIT); } - addr = mmap_region(file, addr, len, vm_flags, pgoff, uf); + addr = mmap_region(file, addr, len, vma_flags, pgoff, uf); if (!IS_ERR_VALUE(addr) && - ((vm_flags & VM_LOCKED) || + (vma_flags_test(&vma_flags, VMA_LOCKED_BIT) || (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE)) *populate = len; return addr; @@ -1191,7 +1204,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, } ret = do_mmap(vma->vm_file, start, size, - prot, flags, 0, pgoff, &populate, NULL); + prot, flags, EMPTY_VMA_FLAGS, pgoff, &populate, NULL); out: mmap_write_unlock(mm); fput(file); diff --git a/mm/nommu.c b/mm/nommu.c index a7da5ec21f42..277f663e1c5b 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -1014,11 +1014,12 @@ unsigned long do_mmap(struct file *file, unsigned long len, unsigned long prot, unsigned long flags, - vm_flags_t vm_flags, + vma_flags_t vma_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf) { + vm_flags_t vm_flags = vma_flags_to_legacy(vma_flags); struct vm_area_struct *vma; struct vm_region *region; struct rb_node *rb; diff --git a/mm/util.c b/mm/util.c index 5090457542ca..bf0513d1d3d0 100644 --- a/mm/util.c +++ b/mm/util.c @@ -578,8 +578,8 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, if (!ret) { if (mmap_write_lock_killable(mm)) return -EINTR; - ret = do_mmap(file, addr, len, prot, flag, 0, pgoff, &populate, - &uf); + ret = do_mmap(file, addr, len, prot, flag, EMPTY_VMA_FLAGS, pgoff, + &populate, &uf); mmap_write_unlock(mm); userfaultfd_unmap_complete(mm, &uf); if (populate) @@ -627,20 +627,20 @@ EXPORT_SYMBOL(vm_mmap); unsigned long vm_mmap_shadow_stack(unsigned long addr, unsigned long len, unsigned long flags) { + vma_flags_t vma_flags = VMA_SHADOW_STACK; struct mm_struct *mm = current->mm; unsigned long ret, unused; - vm_flags_t vm_flags = VM_SHADOW_STACK; flags |= MAP_ANONYMOUS | MAP_PRIVATE; if (addr) flags |= MAP_FIXED_NOREPLACE; if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) - vm_flags |= VM_NOHUGEPAGE; + vma_flags_set(&vma_flags, VMA_NOHUGEPAGE_BIT); mmap_write_lock(mm); ret = do_mmap(NULL, addr, len, PROT_READ | PROT_WRITE, flags, - vm_flags, 0, &unused, NULL); + vma_flags, 0, &unused, NULL); mmap_write_unlock(mm); return ret; diff --git a/mm/vma.c b/mm/vma.c index 12a684c36b6c..dcf613b5e695 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -2883,7 +2883,7 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, * file to be mapped, otherwise NULL. * @addr: The page-aligned address at which to perform the mapping. * @len: The page-aligned, non-zero, length of the mapping. - * @vm_flags: The VMA flags which should be applied to the mapping. + * @vma_flags: The VMA flags which should be applied to the mapping. * @pgoff: If @file is specified, the page offset into the file, if not then * the virtual page offset in memory of the anonymous mapping. * @uf: Optionally, a pointer to a list head used for tracking userfaultfd unmap @@ -2893,12 +2893,11 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, * been performed. */ unsigned long mmap_region(struct file *file, unsigned long addr, - unsigned long len, vm_flags_t vm_flags, + unsigned long len, vma_flags_t vma_flags, unsigned long pgoff, struct list_head *uf) { unsigned long ret; bool writable_file_mapping = false; - const vma_flags_t vma_flags = legacy_to_vma_flags(vm_flags); mmap_assert_write_locked(current->mm); @@ -2907,7 +2906,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, return -EACCES; /* Allow architectures to sanity-check the vm_flags. */ - if (!arch_validate_flags(vm_flags)) + if (!arch_validate_flags(vma_flags_to_legacy(vma_flags))) return -EINVAL; /* Map writable and ensure this isn't a sealed memfd. */ diff --git a/mm/vma.h b/mm/vma.h index adb7a0ba1192..4eb85b4a6adc 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -517,7 +517,7 @@ int mm_take_all_locks(struct mm_struct *mm); void mm_drop_all_locks(struct mm_struct *mm); unsigned long mmap_region(struct file *file, unsigned long addr, - unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, + unsigned long len, vma_flags_t vma_flags, unsigned long pgoff, struct list_head *uf); int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma, From 2ec748152df7c0677ca3ef1c5de8d73ea433f020 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Sat, 11 Jul 2026 19:45:00 +0100 Subject: [PATCH 402/501] mm: convert __get_unmapped_area() to use vma_flags_t Update __get_unmapped_area() to be parameterised by vma_flags_t rather than vm_flags_t as part of the effort to move VMA flags from a system word to a bitmap. We cascade the changes up to arch_get_unmapped_area_topdown() and arch_get_unmapped_area(), where, for now, we use vma_flags_to_legacy() in order to propagate the VMA flags. No functional change intended. Link: https://lore.kernel.org/20260711-b4-vma-flags-mm-v2-3-0fa2357d5431@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Lance Yang Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Baolin Wang Cc: Barry Song Cc: Christian Brauner Cc: Dave Airlie Cc: David Hildenbrand Cc: Dev Jain Cc: Jani Nikula Cc: Jan Kara Cc: Jann Horn Cc: Mike Rapoport Cc: Muchun Song Cc: Nico Pache Cc: Oscar Salvador Cc: Pedro Falcato Cc: Suren Baghdasaryan Cc: Thomas Zimmermann Signed-off-by: Andrew Morton --- fs/hugetlbfs/inode.c | 3 ++- include/linux/huge_mm.h | 10 +++++----- include/linux/mm.h | 6 ++++-- include/linux/sched/mm.h | 9 +++------ mm/huge_memory.c | 21 ++++++++++++--------- mm/mmap.c | 27 ++++++++++++++------------- 6 files changed, 40 insertions(+), 36 deletions(-) diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 9fddd1e0b813..43066f9ddf52 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -184,7 +184,8 @@ hugetlb_get_unmapped_area(struct file *file, unsigned long addr, if (addr) addr0 = ALIGN(addr, huge_page_size(h)); - return mm_get_unmapped_area_vmflags(file, addr0, len, pgoff, flags, 0); + return mm_get_unmapped_area_vmaflags(file, addr0, len, pgoff, flags, + EMPTY_VMA_FLAGS); } /* diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index cae97c307280..9e43e138c0b7 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -391,9 +391,9 @@ static inline bool thp_disabled_by_hw(void) unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); -unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, +unsigned long thp_get_unmapped_area_vmaflags(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags, - vm_flags_t vm_flags); + vma_flags_t vma_flags); enum split_type { SPLIT_TYPE_UNIFORM, @@ -617,9 +617,9 @@ static inline unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma, #define thp_get_unmapped_area NULL static inline unsigned long -thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, - unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags) +thp_get_unmapped_area_vmaflags(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, + unsigned long flags, vma_flags_t vma_flags) { return 0; } diff --git a/include/linux/mm.h b/include/linux/mm.h index d20aa2f80472..5b3825fddf58 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4191,13 +4191,15 @@ unsigned long randomize_page(unsigned long start, unsigned long range); unsigned long __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, - unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags); + unsigned long pgoff, unsigned long flags, + vma_flags_t vma_flags); static inline unsigned long get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { - return __get_unmapped_area(file, addr, len, pgoff, flags, 0); + return __get_unmapped_area(file, addr, len, pgoff, flags, + EMPTY_VMA_FLAGS); } unsigned long do_mmap(struct file *file, unsigned long addr, diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 95d0040df584..10d723bee95c 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -193,12 +193,9 @@ unsigned long mm_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); -unsigned long mm_get_unmapped_area_vmflags(struct file *filp, - unsigned long addr, - unsigned long len, - unsigned long pgoff, - unsigned long flags, - vm_flags_t vm_flags); +unsigned long mm_get_unmapped_area_vmaflags(struct file *filp, + unsigned long addr, unsigned long len, unsigned long pgoff, + unsigned long flags, vma_flags_t vma_flags); unsigned long generic_get_unmapped_area(struct file *filp, unsigned long addr, diff --git a/mm/huge_memory.c b/mm/huge_memory.c index c61871b5a333..5bd8d4f59a7b 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1194,7 +1194,7 @@ static inline bool is_transparent_hugepage(const struct folio *folio) static unsigned long __thp_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, loff_t off, unsigned long flags, unsigned long size, - vm_flags_t vm_flags) + vma_flags_t vma_flags) { loff_t off_end = off + len; loff_t off_align = round_up(off, size); @@ -1210,8 +1210,9 @@ static unsigned long __thp_get_unmapped_area(struct file *filp, if (len_pad < len || (off + len_pad) < off) return 0; - ret = mm_get_unmapped_area_vmflags(filp, addr, len_pad, - off >> PAGE_SHIFT, flags, vm_flags); + ret = mm_get_unmapped_area_vmaflags(filp, addr, len_pad, + off >> PAGE_SHIFT, flags, + vma_flags); /* * The failure might be due to length padding. The caller will retry @@ -1236,25 +1237,27 @@ static unsigned long __thp_get_unmapped_area(struct file *filp, return ret; } -unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, +unsigned long thp_get_unmapped_area_vmaflags(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags, - vm_flags_t vm_flags) + vma_flags_t vma_flags) { unsigned long ret; loff_t off = (loff_t)pgoff << PAGE_SHIFT; - ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE, vm_flags); + ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE, + vma_flags); if (ret) return ret; - return mm_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, - vm_flags); + return mm_get_unmapped_area_vmaflags(filp, addr, len, pgoff, flags, + vma_flags); } unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { - return thp_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, 0); + return thp_get_unmapped_area_vmaflags(filp, addr, len, pgoff, flags, + EMPTY_VMA_FLAGS); } EXPORT_SYMBOL_GPL(thp_get_unmapped_area); diff --git a/mm/mmap.c b/mm/mmap.c index bcc3793e532d..4a88b2a233f7 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -411,8 +411,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, /* Obtain the address to map to. we verify (or select) it and ensure * that it represents a valid section of the address space. */ - addr = __get_unmapped_area(file, addr, len, pgoff, flags, - vma_flags_to_legacy(vma_flags)); + addr = __get_unmapped_area(file, addr, len, pgoff, flags, vma_flags); if (IS_ERR_VALUE(addr)) return addr; @@ -812,19 +811,20 @@ arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, } #endif -unsigned long mm_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, - unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags) +unsigned long mm_get_unmapped_area_vmaflags(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, unsigned long flags, + vma_flags_t vma_flags) { if (mm_flags_test(MMF_TOPDOWN, current->mm)) return arch_get_unmapped_area_topdown(filp, addr, len, pgoff, - flags, vm_flags); - return arch_get_unmapped_area(filp, addr, len, pgoff, flags, vm_flags); + flags, vma_flags_to_legacy(vma_flags)); + return arch_get_unmapped_area(filp, addr, len, pgoff, flags, + vma_flags_to_legacy(vma_flags)); } unsigned long __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, - unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags) + unsigned long pgoff, unsigned long flags, vma_flags_t vma_flags) { unsigned long (*get_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long) @@ -859,11 +859,11 @@ __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, && !addr /* no hint */ && IS_ALIGNED(len, PMD_SIZE)) { /* Ensures that larger anonymous mappings are THP aligned. */ - addr = thp_get_unmapped_area_vmflags(file, addr, len, - pgoff, flags, vm_flags); + addr = thp_get_unmapped_area_vmaflags(file, addr, len, + pgoff, flags, vma_flags); } else { - addr = mm_get_unmapped_area_vmflags(file, addr, len, - pgoff, flags, vm_flags); + addr = mm_get_unmapped_area_vmaflags(file, addr, len, + pgoff, flags, vma_flags); } if (IS_ERR_VALUE(addr)) return addr; @@ -881,7 +881,8 @@ unsigned long mm_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { - return mm_get_unmapped_area_vmflags(file, addr, len, pgoff, flags, 0); + return mm_get_unmapped_area_vmaflags(file, addr, len, pgoff, flags, + EMPTY_VMA_FLAGS); } EXPORT_SYMBOL(mm_get_unmapped_area); From 8d800431319cc045d0b44997055f70c93505c732 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Sat, 11 Jul 2026 19:45:01 +0100 Subject: [PATCH 403/501] mm: update generic_get_unmapped_area[_topdown]() to use vma_flags_t As part of the changes converting VMA flags from a system word size to a bitmap, extend this change to generic_get_unmapped_area() and generic_get_unmapped_area_topdown(), which also allows us to convert stack_guard_placement() as well. We retain arch_get_unmapped_area() and arch_get_unmapped_area_topdown() as-is for now, using legacy_to_vma_flags() as necessary to do so. No functional change intended. Link: https://lore.kernel.org/20260711-b4-vma-flags-mm-v2-4-0fa2357d5431@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Zi Yan Reviewed-by: Lance Yang Reviewed-by: Vlastimil Babka (SUSE) Cc: Baolin Wang Cc: Barry Song Cc: Christian Brauner Cc: Dave Airlie Cc: David Hildenbrand Cc: Dev Jain Cc: Jani Nikula Cc: Jan Kara Cc: Jann Horn Cc: Mike Rapoport Cc: Muchun Song Cc: Nico Pache Cc: Oscar Salvador Cc: Pedro Falcato Cc: Suren Baghdasaryan Cc: Thomas Zimmermann Signed-off-by: Andrew Morton --- arch/powerpc/mm/book3s64/slice.c | 6 ++++-- include/linux/sched/mm.h | 4 ++-- mm/mmap.c | 16 ++++++++-------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/arch/powerpc/mm/book3s64/slice.c b/arch/powerpc/mm/book3s64/slice.c index 28bec5bc7879..82127e31dca6 100644 --- a/arch/powerpc/mm/book3s64/slice.c +++ b/arch/powerpc/mm/book3s64/slice.c @@ -659,7 +659,8 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned int psize; if (radix_enabled()) - return generic_get_unmapped_area(filp, addr, len, pgoff, flags, vm_flags); + return generic_get_unmapped_area(filp, addr, len, pgoff, flags, + legacy_to_vma_flags(vm_flags)); if (filp && is_file_hugepages(filp)) psize = file_to_psize(filp); @@ -679,7 +680,8 @@ unsigned long arch_get_unmapped_area_topdown(struct file *filp, unsigned int psize; if (radix_enabled()) - return generic_get_unmapped_area_topdown(filp, addr0, len, pgoff, flags, vm_flags); + return generic_get_unmapped_area_topdown(filp, addr0, len, + pgoff, flags, legacy_to_vma_flags(vm_flags)); if (filp && is_file_hugepages(filp)) psize = file_to_psize(filp); diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 10d723bee95c..10be8a54b416 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -200,11 +200,11 @@ unsigned long mm_get_unmapped_area_vmaflags(struct file *filp, unsigned long generic_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags); + unsigned long flags, vma_flags_t vma_flags); unsigned long generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags); + unsigned long flags, vma_flags_t vma_flags); #else static inline void arch_pick_mmap_layout(struct mm_struct *mm, const struct rlimit *rlim_stack) {} diff --git a/mm/mmap.c b/mm/mmap.c index 4a88b2a233f7..39d33f3a55c2 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -657,9 +657,9 @@ SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg) * Determine if the allocation needs to ensure that there is no * existing mapping within it's guard gaps, for use as start_gap. */ -static inline unsigned long stack_guard_placement(vm_flags_t vm_flags) +static inline unsigned long stack_guard_placement(vma_flags_t vma_flags) { - if (vm_flags & VM_SHADOW_STACK) + if (vma_flags_test_single_mask(&vma_flags, VMA_SHADOW_STACK)) return PAGE_SIZE; return 0; @@ -701,7 +701,7 @@ unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info) unsigned long generic_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags) + unsigned long flags, vma_flags_t vma_flags) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma, *prev; @@ -726,7 +726,7 @@ generic_get_unmapped_area(struct file *filp, unsigned long addr, info.length = len; info.low_limit = mm->mmap_base; info.high_limit = mmap_end; - info.start_gap = stack_guard_placement(vm_flags); + info.start_gap = stack_guard_placement(vma_flags); if (filp && is_file_hugepages(filp)) info.align_mask = huge_page_mask_align(filp); return vm_unmapped_area(&info); @@ -739,7 +739,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long flags, vm_flags_t vm_flags) { return generic_get_unmapped_area(filp, addr, len, pgoff, flags, - vm_flags); + legacy_to_vma_flags(vm_flags)); } #endif @@ -750,7 +750,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags) + unsigned long flags, vma_flags_t vma_flags) { struct vm_area_struct *vma, *prev; struct mm_struct *mm = current->mm; @@ -778,7 +778,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, info.length = len; info.low_limit = PAGE_SIZE; info.high_limit = arch_get_mmap_base(addr, mm->mmap_base); - info.start_gap = stack_guard_placement(vm_flags); + info.start_gap = stack_guard_placement(vma_flags); if (filp && is_file_hugepages(filp)) info.align_mask = huge_page_mask_align(filp); addr = vm_unmapped_area(&info); @@ -807,7 +807,7 @@ arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, unsigned long flags, vm_flags_t vm_flags) { return generic_get_unmapped_area_topdown(filp, addr, len, pgoff, flags, - vm_flags); + legacy_to_vma_flags(vm_flags)); } #endif From 96c09ae60b5979dcd147832148066132f929e8f0 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Sat, 11 Jul 2026 19:45:02 +0100 Subject: [PATCH 404/501] mm: prefer mm->def_vma_flags in mm logic Currently mm->def_flags (of type vm_flags_t) is union'd with mm->def_vma_flags (of type vma_flags_t). As part of the effort to convert vm_flags_t usage to vma_flags_t (in order to no longer be arbitrarily limited to a system word size for VMA flags), prefer mm->def_vma_flags to mm->def_flags throughout the mm logic. We update dump_mm() to use the %*pb format which means we make no assumption about the number of VMA flag bits on output when outputting default VMA flags. No functional change intended. Link: https://lore.kernel.org/20260711-b4-vma-flags-mm-v2-5-0fa2357d5431@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Lance Yang Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Baolin Wang Cc: Barry Song Cc: Christian Brauner Cc: Dave Airlie Cc: David Hildenbrand Cc: Dev Jain Cc: Jani Nikula Cc: Jan Kara Cc: Jann Horn Cc: Mike Rapoport Cc: Muchun Song Cc: Nico Pache Cc: Oscar Salvador Cc: Pedro Falcato Cc: Suren Baghdasaryan Cc: Thomas Zimmermann Signed-off-by: Andrew Morton --- mm/debug.c | 5 +++-- mm/mlock.c | 13 +++++++------ mm/mmap.c | 11 ++++++----- mm/vma.c | 2 +- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/mm/debug.c b/mm/debug.c index 497654b36f1a..9a0297b3988d 100644 --- a/mm/debug.c +++ b/mm/debug.c @@ -197,7 +197,7 @@ void dump_mm(const struct mm_struct *mm) "numa_next_scan %lu numa_scan_offset %lu numa_scan_seq %d\n" #endif "tlb_flush_pending %d\n" - "def_flags: %#lx(%pGv)\n", + "def_flags: %*pb(%pGv)\n", mm, mm->task_size, mm->mmap_base, mm->mmap_legacy_base, @@ -226,7 +226,8 @@ void dump_mm(const struct mm_struct *mm) mm->numa_next_scan, mm->numa_scan_offset, mm->numa_scan_seq, #endif atomic_read(&mm->tlb_flush_pending), - mm->def_flags, &mm->def_flags + NUM_VMA_FLAG_BITS, mm->def_vma_flags.__vma_flags, + &mm->def_vma_flags ); } EXPORT_SYMBOL(dump_mm); diff --git a/mm/mlock.c b/mm/mlock.c index 97e49038d8d3..34ffa954006f 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -699,26 +699,27 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) /* * Take the MCL_* flags passed into mlockall (or 0 if called from munlockall) - * and translate into the appropriate modifications to mm->def_flags and/or the - * flags for all current VMAs. + * and translate into the appropriate modifications to mm->def_vma_flags and/or + * the flags for all current VMAs. * * There are a couple of subtleties with this. If mlockall() is called multiple * times with different flags, the values do not necessarily stack. If mlockall * is called once including the MCL_FUTURE flag and then a second time without - * it, VM_LOCKED and VM_LOCKONFAULT will be cleared from mm->def_flags. + * it, VM_LOCKED and VM_LOCKONFAULT will be cleared from mm->def_vma_flags. */ static int apply_mlockall_flags(int flags) { VMA_ITERATOR(vmi, current->mm, 0); + struct mm_struct *mm = current->mm; struct vm_area_struct *vma, *prev = NULL; vm_flags_t to_add = 0; - current->mm->def_flags &= ~VM_LOCKED_MASK; + vma_flags_clear_mask(&mm->def_vma_flags, VMA_LOCKED_MASK); if (flags & MCL_FUTURE) { - current->mm->def_flags |= VM_LOCKED; + vma_flags_set(&mm->def_vma_flags, VMA_LOCKED_BIT); if (flags & MCL_ONFAULT) - current->mm->def_flags |= VM_LOCKONFAULT; + vma_flags_set(&mm->def_vma_flags, VMA_LOCKONFAULT_BIT); if (!(flags & MCL_CURRENT)) goto out; diff --git a/mm/mmap.c b/mm/mmap.c index 39d33f3a55c2..ae9a5018e20e 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -102,15 +102,16 @@ void vma_set_page_prot(struct vm_area_struct *vma) */ static int check_brk_limits(unsigned long addr, unsigned long len) { + const struct mm_struct *mm = current->mm; + const bool is_def_locked = + vma_flags_test(&mm->def_vma_flags, VMA_LOCKED_BIT); unsigned long mapped_addr; mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED); if (IS_ERR_VALUE(mapped_addr)) return mapped_addr; - return mlock_future_ok(current->mm, - current->mm->def_flags & VM_LOCKED, len) - ? 0 : -EAGAIN; + return mlock_future_ok(mm, is_def_locked, len) ? 0 : -EAGAIN; } SYSCALL_DEFINE1(brk, unsigned long, brk) @@ -197,7 +198,7 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) goto out; mm->brk = brk; - if (mm->def_flags & VM_LOCKED) + if (vma_flags_test(&mm->def_vma_flags, VMA_LOCKED_BIT)) populate = true; success: @@ -1247,7 +1248,7 @@ int vm_brk_flags(unsigned long addr, unsigned long request, bool is_exec) vma = vma_prev(&vmi); ret = do_brk_flags(&vmi, vma, addr, len, vma_flags); - populate = ((mm->def_flags & VM_LOCKED) != 0); + populate = vma_flags_test(&mm->def_vma_flags, VMA_LOCKED_BIT); mmap_write_unlock(mm); userfaultfd_unmap_complete(mm, &uf); if (populate && !ret) diff --git a/mm/vma.c b/mm/vma.c index dcf613b5e695..49362ac267f9 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -3426,7 +3426,7 @@ struct vm_area_struct *__install_special_mapping( if (unlikely(vma == NULL)) return ERR_PTR(-ENOMEM); - vm_flags |= mm->def_flags | VM_DONTEXPAND; + vm_flags |= vma_flags_to_legacy(mm->def_vma_flags) | VM_DONTEXPAND; if (pgtable_supports_soft_dirty()) vm_flags |= VM_SOFTDIRTY; vm_flags_init(vma, vm_flags & ~VM_LOCKED_MASK); From cc593403b1460b7ac566375de441f9973084b2f6 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Sat, 11 Jul 2026 19:45:03 +0100 Subject: [PATCH 405/501] mm/vma: convert vm_pgprot_modify() to use vma_flags_t and rename Update vm_pgprot_modify() to use the new VMA flags type vma_flags_t and rename to vma_pgprot_modify() accordingly. This is part of the ongoing work to convert vm_flags_t to vma_flags_t, in order to eliminate the arbitrary limit of the number of bits in a system word on available VMA flags. Update VMA userland tests accordingly, updating vma_set_page_prot() to no longer inline vma_pgprot_modify(), rather we can simply define vma_pgprot_modify() as a static inline function and the tests will pick it up from vma.h. No functional change intended. Link: https://lore.kernel.org/20260711-b4-vma-flags-mm-v2-6-0fa2357d5431@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Lance Yang Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Baolin Wang Cc: Barry Song Cc: Christian Brauner Cc: Dave Airlie Cc: David Hildenbrand Cc: Dev Jain Cc: Jani Nikula Cc: Jan Kara Cc: Jann Horn Cc: Mike Rapoport Cc: Muchun Song Cc: Nico Pache Cc: Oscar Salvador Cc: Pedro Falcato Cc: Suren Baghdasaryan Cc: Thomas Zimmermann Signed-off-by: Andrew Morton --- mm/mmap.c | 8 ++++---- mm/vma.c | 2 +- mm/vma.h | 6 ++++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/mm/mmap.c b/mm/mmap.c index ae9a5018e20e..2076c70e7700 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -80,13 +80,13 @@ core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644); /* Update vma->vm_page_prot to reflect vma->vm_flags. */ void vma_set_page_prot(struct vm_area_struct *vma) { - vm_flags_t vm_flags = vma->vm_flags; + vma_flags_t vma_flags = vma->flags; pgprot_t vm_page_prot; - vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags); + vm_page_prot = vma_pgprot_modify(vma->vm_page_prot, vma_flags); if (vma_wants_writenotify(vma, vm_page_prot)) { - vm_flags &= ~VM_SHARED; - vm_page_prot = vm_pgprot_modify(vm_page_prot, vm_flags); + vma_flags_clear(&vma_flags, VMA_SHARED_BIT); + vm_page_prot = vma_pgprot_modify(vm_page_prot, vma_flags); } /* remove_protection_ptes reads vma->vm_page_prot without mmap_lock */ WRITE_ONCE(vma->vm_page_prot, vm_page_prot); diff --git a/mm/vma.c b/mm/vma.c index 49362ac267f9..d34c0be09a47 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -2176,7 +2176,7 @@ bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot) /* The open routine did something to the protections that pgprot_modify * won't preserve? */ if (pgprot_val(vm_page_prot) != - pgprot_val(vm_pgprot_modify(vm_page_prot, vma->vm_flags))) + pgprot_val(vma_pgprot_modify(vm_page_prot, vma->flags))) return false; /* diff --git a/mm/vma.h b/mm/vma.h index 4eb85b4a6adc..8ca6e7e8ae28 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -541,9 +541,11 @@ static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma } #ifdef CONFIG_MMU -static inline pgprot_t vm_pgprot_modify(pgprot_t oldprot, vm_flags_t vm_flags) +static inline pgprot_t vma_pgprot_modify(pgprot_t oldprot, vma_flags_t vma_flags) { - return pgprot_modify(oldprot, vm_get_page_prot(vm_flags)); + const pgprot_t prot = vma_get_page_prot(vma_flags); + + return pgprot_modify(oldprot, prot); } #endif From e28266b033c30aa0f6c888bea6f1d25f7854185d Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Sat, 11 Jul 2026 19:45:04 +0100 Subject: [PATCH 406/501] mm/vma: rename vma_get_page_prot to vma_flags_to_page_prot Having vma_get_page_prot() refer to VMA flags and vma_set_page_prot() refer to a VMA is confusing. Rename vma_get_page_prot() to vma_flags_to_page_prot() to resolve this confusion. No functional change intended. Link: https://lore.kernel.org/20260711-b4-vma-flags-mm-v2-7-0fa2357d5431@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Lance Yang Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Baolin Wang Cc: Barry Song Cc: Christian Brauner Cc: Dave Airlie Cc: David Hildenbrand Cc: Dev Jain Cc: Jani Nikula Cc: Jan Kara Cc: Jann Horn Cc: Mike Rapoport Cc: Muchun Song Cc: Nico Pache Cc: Oscar Salvador Cc: Pedro Falcato Cc: Suren Baghdasaryan Cc: Thomas Zimmermann Signed-off-by: Andrew Morton --- include/linux/mm.h | 4 ++-- mm/vma.c | 2 +- mm/vma.h | 2 +- tools/testing/vma/include/dup.h | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 5b3825fddf58..b8fe40f89d87 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4603,7 +4603,7 @@ static inline bool range_in_vma_desc(const struct vm_area_desc *desc, #ifdef CONFIG_MMU pgprot_t vm_get_page_prot(vm_flags_t vm_flags); -static inline pgprot_t vma_get_page_prot(vma_flags_t vma_flags) +static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags) { const vm_flags_t vm_flags = vma_flags_to_legacy(vma_flags); @@ -4616,7 +4616,7 @@ static inline pgprot_t vm_get_page_prot(vm_flags_t vm_flags) { return __pgprot(0); } -static inline pgprot_t vma_get_page_prot(vma_flags_t vma_flags) +static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags) { return __pgprot(0); } diff --git a/mm/vma.c b/mm/vma.c index d34c0be09a47..8b14e8201d66 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -56,7 +56,7 @@ struct mmap_state { .pglen = PHYS_PFN(len_), \ .vma_flags = vma_flags_, \ .file = file_, \ - .page_prot = vma_get_page_prot(vma_flags_), \ + .page_prot = vma_flags_to_page_prot(vma_flags_), \ } #define VMG_MMAP_STATE(name, map_, vma_) \ diff --git a/mm/vma.h b/mm/vma.h index 8ca6e7e8ae28..0bc7d521e976 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -543,7 +543,7 @@ static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma #ifdef CONFIG_MMU static inline pgprot_t vma_pgprot_modify(pgprot_t oldprot, vma_flags_t vma_flags) { - const pgprot_t prot = vma_get_page_prot(vma_flags); + const pgprot_t prot = vma_flags_to_page_prot(vma_flags); return pgprot_modify(oldprot, prot); } diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index e9ddc818f2c3..73a637d9472e 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -1547,7 +1547,7 @@ static inline int get_sysctl_max_map_count(void) #define pgtable_supports_soft_dirty() IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) #endif -static inline pgprot_t vma_get_page_prot(vma_flags_t vma_flags) +static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags) { const vm_flags_t vm_flags = vma_flags_to_legacy(vma_flags); From a2fdfd0080a3b15afa81f8495f8b82eb167441a5 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Sat, 11 Jul 2026 19:45:05 +0100 Subject: [PATCH 407/501] mm: introduce vma_get_page_prot() and use it There's a large number of vm_get_page_prot(vma->vm_flags) invocations. Make life easier by introducing vma_get_page_prot() parameterised by the VMA. This also makes converting vm_get_page_prot() to vma_flags_t easier. Also update the userland VMA tests to reflect the change. No functional change intended. Link: https://lore.kernel.org/20260711-b4-vma-flags-mm-v2-8-0fa2357d5431@kernel.org Signed-off-by: Lorenzo Stoakes Acked-by: Zi Yan Acked-by: Jani Nikula # for i915 Reviewed-by: Thomas Zimmermann [DRM] Reviewed-by: Vlastimil Babka (SUSE) Cc: Baolin Wang Cc: Barry Song Cc: Christian Brauner Cc: Dave Airlie Cc: David Hildenbrand Cc: Dev Jain Cc: Jan Kara Cc: Jann Horn Cc: Lance Yang Cc: Mike Rapoport Cc: Muchun Song Cc: Nico Pache Cc: Oscar Salvador Cc: Pedro Falcato Cc: Suren Baghdasaryan Signed-off-by: Andrew Morton --- drivers/gpu/drm/drm_gem.c | 2 +- drivers/gpu/drm/drm_gem_dma_helper.c | 2 +- drivers/gpu/drm/drm_gem_shmem_helper.c | 2 +- drivers/gpu/drm/etnaviv/etnaviv_gem.c | 2 +- drivers/gpu/drm/exynos/exynos_drm_gem.c | 6 +++--- drivers/gpu/drm/i915/gem/i915_gem_mman.c | 12 ++++++------ drivers/gpu/drm/msm/msm_gem.c | 2 +- drivers/gpu/drm/nouveau/nouveau_gem.c | 2 +- drivers/gpu/drm/omapdrm/omap_fbdev.c | 2 +- drivers/gpu/drm/omapdrm/omap_gem.c | 6 +++--- drivers/gpu/drm/panthor/panthor_gem.c | 2 +- drivers/gpu/drm/rockchip/rockchip_drm_gem.c | 2 +- drivers/gpu/drm/tegra/gem.c | 2 +- drivers/gpu/drm/virtio/virtgpu_vram.c | 2 +- drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c | 2 +- drivers/gpu/drm/xe/xe_device.c | 2 +- drivers/gpu/drm/xe/xe_mmio_gem.c | 2 +- drivers/gpu/drm/xen/xen_drm_front_gem.c | 2 +- drivers/video/fbdev/core/fb_io_fops.c | 2 +- include/linux/mm.h | 11 ++++++++++- mm/vma.c | 2 +- mm/vma_exec.c | 2 +- sound/core/memalloc.c | 2 +- tools/testing/vma/include/dup.h | 5 +++++ 24 files changed, 46 insertions(+), 32 deletions(-) diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c index e3ed684ddcf2..32a05d889b9a 100644 --- a/drivers/gpu/drm/drm_gem.c +++ b/drivers/gpu/drm/drm_gem.c @@ -1252,7 +1252,7 @@ int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size, } vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); - vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); + vma->vm_page_prot = pgprot_writecombine(vma_get_page_prot(vma)); vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); } diff --git a/drivers/gpu/drm/drm_gem_dma_helper.c b/drivers/gpu/drm/drm_gem_dma_helper.c index 1c00a71ab3c9..7d9612075d31 100644 --- a/drivers/gpu/drm/drm_gem_dma_helper.c +++ b/drivers/gpu/drm/drm_gem_dma_helper.c @@ -540,7 +540,7 @@ int drm_gem_dma_mmap(struct drm_gem_dma_object *dma_obj, struct vm_area_struct * vm_flags_mod(vma, VM_DONTDUMP | VM_DONTEXPAND, VM_PFNMAP); if (dma_obj->map_noncoherent) { - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); ret = dma_mmap_pages(drm_dev_dma_dev(dma_obj->base.dev), vma, vma->vm_end - vma->vm_start, diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index c989459eb215..06d019d51d3e 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -764,7 +764,7 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct return ret; vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); if (shmem->map_wc) vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c index 2e4d6d117ee2..f9c8b7b2bfc7 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c @@ -133,7 +133,7 @@ static int etnaviv_gem_mmap_obj(struct etnaviv_gem_object *etnaviv_obj, vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); - vm_page_prot = vm_get_page_prot(vma->vm_flags); + vm_page_prot = vma_get_page_prot(vma); if (etnaviv_obj->flags & ETNA_BO_WC) { vma->vm_page_prot = pgprot_writecombine(vm_page_prot); diff --git a/drivers/gpu/drm/exynos/exynos_drm_gem.c b/drivers/gpu/drm/exynos/exynos_drm_gem.c index 9a6270f3dca6..0208c9259572 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_gem.c +++ b/drivers/gpu/drm/exynos/exynos_drm_gem.c @@ -377,13 +377,13 @@ static int exynos_drm_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct /* non-cachable as default. */ if (exynos_gem->flags & EXYNOS_BO_CACHABLE) - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); else if (exynos_gem->flags & EXYNOS_BO_WC) vma->vm_page_prot = - pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); + pgprot_writecombine(vma_get_page_prot(vma)); else vma->vm_page_prot = - pgprot_noncached(vm_get_page_prot(vma->vm_flags)); + pgprot_noncached(vma_get_page_prot(vma)); ret = exynos_drm_gem_mmap_buffer(exynos_gem, vma); if (ret) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c index 0644f85c6c8e..9ca90c1bb5b4 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c @@ -112,7 +112,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data, vma = find_vma(mm, addr); if (vma && __vma_matches(vma, obj->base.filp, addr, args->size)) vma->vm_page_prot = - pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); + pgprot_writecombine(vma_get_page_prot(vma)); else addr = -ENOMEM; mmap_write_unlock(mm); @@ -1024,7 +1024,7 @@ i915_gem_object_mmap(struct drm_i915_gem_object *obj, fput(anon); if (obj->ops->mmap_ops) { - vma->vm_page_prot = pgprot_decrypted(vm_get_page_prot(vma->vm_flags)); + vma->vm_page_prot = pgprot_decrypted(vma_get_page_prot(vma)); vma->vm_ops = obj->ops->mmap_ops; vma->vm_private_data = obj->base.vma_node.driver_private; return 0; @@ -1035,7 +1035,7 @@ i915_gem_object_mmap(struct drm_i915_gem_object *obj, switch (mmo->mmap_type) { case I915_MMAP_TYPE_WC: vma->vm_page_prot = - pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); + pgprot_writecombine(vma_get_page_prot(vma)); vma->vm_ops = &vm_ops_cpu; break; @@ -1043,19 +1043,19 @@ i915_gem_object_mmap(struct drm_i915_gem_object *obj, GEM_WARN_ON(1); fallthrough; case I915_MMAP_TYPE_WB: - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); vma->vm_ops = &vm_ops_cpu; break; case I915_MMAP_TYPE_UC: vma->vm_page_prot = - pgprot_noncached(vm_get_page_prot(vma->vm_flags)); + pgprot_noncached(vma_get_page_prot(vma)); vma->vm_ops = &vm_ops_cpu; break; case I915_MMAP_TYPE_GTT: vma->vm_page_prot = - pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); + pgprot_writecombine(vma_get_page_prot(vma)); vma->vm_ops = &vm_ops_gtt; break; } diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c index cbf723a5d86f..6a78e242de7c 100644 --- a/drivers/gpu/drm/msm/msm_gem.c +++ b/drivers/gpu/drm/msm/msm_gem.c @@ -1125,7 +1125,7 @@ static int msm_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct struct msm_gem_object *msm_obj = to_msm_bo(obj); vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); - vma->vm_page_prot = msm_gem_pgprot(msm_obj, vm_get_page_prot(vma->vm_flags)); + vma->vm_page_prot = msm_gem_pgprot(msm_obj, vma_get_page_prot(vma)); return 0; } diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c index 20dba02d6175..9a6ee2e880c0 100644 --- a/drivers/gpu/drm/nouveau/nouveau_gem.c +++ b/drivers/gpu/drm/nouveau/nouveau_gem.c @@ -55,7 +55,7 @@ static vm_fault_t nouveau_ttm_fault(struct vm_fault *vmf) goto error_unlock; nouveau_bo_del_io_reserve_lru(bo); - prot = vm_get_page_prot(vma->vm_flags); + prot = vma_get_page_prot(vma); ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT); nouveau_bo_add_io_reserve_lru(bo); if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) diff --git a/drivers/gpu/drm/omapdrm/omap_fbdev.c b/drivers/gpu/drm/omapdrm/omap_fbdev.c index ca3fb186bf19..4881777642d2 100644 --- a/drivers/gpu/drm/omapdrm/omap_fbdev.c +++ b/drivers/gpu/drm/omapdrm/omap_fbdev.c @@ -84,7 +84,7 @@ static int omap_fbdev_pan_display(struct fb_var_screeninfo *var, struct fb_info static int omap_fbdev_fb_mmap(struct fb_info *info, struct vm_area_struct *vma) { - vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); + vma->vm_page_prot = pgprot_writecombine(vma_get_page_prot(vma)); return fb_deferred_io_mmap(info, vma); } diff --git a/drivers/gpu/drm/omapdrm/omap_gem.c b/drivers/gpu/drm/omapdrm/omap_gem.c index 00404fb6c29a..fb0e6f556b31 100644 --- a/drivers/gpu/drm/omapdrm/omap_gem.c +++ b/drivers/gpu/drm/omapdrm/omap_gem.c @@ -538,9 +538,9 @@ static int omap_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struc vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP | VM_IO | VM_MIXEDMAP); if (omap_obj->flags & OMAP_BO_WC) { - vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); + vma->vm_page_prot = pgprot_writecombine(vma_get_page_prot(vma)); } else if (omap_obj->flags & OMAP_BO_UNCACHED) { - vma->vm_page_prot = pgprot_noncached(vm_get_page_prot(vma->vm_flags)); + vma->vm_page_prot = pgprot_noncached(vma_get_page_prot(vma)); } else { /* * We do have some private objects, at least for scanout buffers @@ -558,7 +558,7 @@ static int omap_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struc vma->vm_pgoff -= drm_vma_node_start(&obj->vma_node); vma_set_file(vma, obj->filp); - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); } vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); diff --git a/drivers/gpu/drm/panthor/panthor_gem.c b/drivers/gpu/drm/panthor/panthor_gem.c index a1e2eb1ca7bb..770556353968 100644 --- a/drivers/gpu/drm/panthor/panthor_gem.c +++ b/drivers/gpu/drm/panthor/panthor_gem.c @@ -776,7 +776,7 @@ static int panthor_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *v } vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); if (should_map_wc(bo)) vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c index b188539dca0b..9a1dc9f12072 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c @@ -255,7 +255,7 @@ static int rockchip_drm_gem_object_mmap(struct drm_gem_object *obj, */ vm_flags_mod(vma, VM_IO | VM_DONTEXPAND | VM_DONTDUMP, VM_PFNMAP); - vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); + vma->vm_page_prot = pgprot_writecombine(vma_get_page_prot(vma)); vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); if (rk_obj->pages) diff --git a/drivers/gpu/drm/tegra/gem.c b/drivers/gpu/drm/tegra/gem.c index 1d8d27a5ea89..f76af733ea79 100644 --- a/drivers/gpu/drm/tegra/gem.c +++ b/drivers/gpu/drm/tegra/gem.c @@ -602,7 +602,7 @@ int __tegra_gem_mmap(struct drm_gem_object *gem, struct vm_area_struct *vma) vma->vm_pgoff = vm_pgoff; } else { - pgprot_t prot = vm_get_page_prot(vma->vm_flags); + pgprot_t prot = vma_get_page_prot(vma); vm_flags_mod(vma, VM_MIXEDMAP, VM_PFNMAP); diff --git a/drivers/gpu/drm/virtio/virtgpu_vram.c b/drivers/gpu/drm/virtio/virtgpu_vram.c index 4ae3cbc35dd3..544a6abddbc8 100644 --- a/drivers/gpu/drm/virtio/virtgpu_vram.c +++ b/drivers/gpu/drm/virtio/virtgpu_vram.c @@ -55,7 +55,7 @@ static int virtio_gpu_vram_mmap(struct drm_gem_object *obj, vma->vm_pgoff -= drm_vma_node_start(&obj->vma_node); vm_flags_set(vma, VM_MIXEDMAP | VM_DONTEXPAND); - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); vma->vm_ops = &virtio_gpu_vram_vm_ops; diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c b/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c index 45561bc1c9ef..a9fd4015a0ca 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c @@ -481,7 +481,7 @@ vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf) if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE) prot = vm_get_page_prot(vma->vm_flags & ~VM_SHARED); else - prot = vm_get_page_prot(vma->vm_flags); + prot = vma_get_page_prot(vma); ret = ttm_bo_vm_fault_reserved(vmf, prot, num_prefault); if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index dcb48caa485d..9d119c95a569 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -281,7 +281,7 @@ static vm_fault_t barrier_fault(struct vm_fault *vmf) pgprot_t prot; int idx; - prot = vm_get_page_prot(vma->vm_flags); + prot = vma_get_page_prot(vma); if (drm_dev_enter(dev, &idx)) { unsigned long pfn; diff --git a/drivers/gpu/drm/xe/xe_mmio_gem.c b/drivers/gpu/drm/xe/xe_mmio_gem.c index 8c803ef233cc..3741ae60f532 100644 --- a/drivers/gpu/drm/xe/xe_mmio_gem.c +++ b/drivers/gpu/drm/xe/xe_mmio_gem.c @@ -149,7 +149,7 @@ static int xe_mmio_gem_mmap(struct drm_gem_object *base, struct vm_area_struct * /* Set vm_pgoff (used as a fake buffer offset by DRM) to 0 */ vma->vm_pgoff = 0; - vma->vm_page_prot = pgprot_noncached(vm_get_page_prot(vma->vm_flags)); + vma->vm_page_prot = pgprot_noncached(vma_get_page_prot(vma)); vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_DONTCOPY | VM_NORESERVE); diff --git a/drivers/gpu/drm/xen/xen_drm_front_gem.c b/drivers/gpu/drm/xen/xen_drm_front_gem.c index eec4c1da3f9e..dd158443f55f 100644 --- a/drivers/gpu/drm/xen/xen_drm_front_gem.c +++ b/drivers/gpu/drm/xen/xen_drm_front_gem.c @@ -80,7 +80,7 @@ static int xen_drm_front_gem_object_mmap(struct drm_gem_object *gem_obj, * which is mapped as Normal Inner Write-Back Outer Write-Back * Inner-Shareable. */ - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); /* * vm_operations_struct.fault handler will be called if CPU access diff --git a/drivers/video/fbdev/core/fb_io_fops.c b/drivers/video/fbdev/core/fb_io_fops.c index 6ab60fcd0050..6d0a8c8e141a 100644 --- a/drivers/video/fbdev/core/fb_io_fops.c +++ b/drivers/video/fbdev/core/fb_io_fops.c @@ -161,7 +161,7 @@ int fb_io_mmap(struct fb_info *info, struct vm_area_struct *vma) len = info->fix.mmio_len; } - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); vma->vm_page_prot = pgprot_framebuffer(vma->vm_page_prot, vma->vm_start, vma->vm_end, start); diff --git a/include/linux/mm.h b/include/linux/mm.h index b8fe40f89d87..1209db1a4b92 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4610,6 +4610,11 @@ static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags) return vm_get_page_prot(vm_flags); } +static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma) +{ + return vma_flags_to_page_prot(vma->flags); +} + void vma_set_page_prot(struct vm_area_struct *vma); #else static inline pgprot_t vm_get_page_prot(vm_flags_t vm_flags) @@ -4620,9 +4625,13 @@ static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags) { return __pgprot(0); } +static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma) +{ + return __pgprot(0); +} static inline void vma_set_page_prot(struct vm_area_struct *vma) { - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); } #endif diff --git a/mm/vma.c b/mm/vma.c index 8b14e8201d66..074f0063b6b5 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -3430,7 +3430,7 @@ struct vm_area_struct *__install_special_mapping( if (pgtable_supports_soft_dirty()) vm_flags |= VM_SOFTDIRTY; vm_flags_init(vma, vm_flags & ~VM_LOCKED_MASK); - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); vma->vm_ops = ops; vma->vm_private_data = priv; diff --git a/mm/vma_exec.c b/mm/vma_exec.c index ef1fa2b161f3..a3c6b05c65fe 100644 --- a/mm/vma_exec.c +++ b/mm/vma_exec.c @@ -146,7 +146,7 @@ int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap, if (pgtable_supports_soft_dirty()) flags |= VM_SOFTDIRTY; vm_flags_init(vma, flags); - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); err = insert_vm_struct(mm, vma); if (err) diff --git a/sound/core/memalloc.c b/sound/core/memalloc.c index 9320671dfcc8..5bc7e586b430 100644 --- a/sound/core/memalloc.c +++ b/sound/core/memalloc.c @@ -851,7 +851,7 @@ static void snd_dma_noncoherent_free(struct snd_dma_buffer *dmab) static int snd_dma_noncoherent_mmap(struct snd_dma_buffer *dmab, struct vm_area_struct *area) { - area->vm_page_prot = vm_get_page_prot(area->vm_flags); + area->vm_page_prot = vma_get_page_prot(area); return dma_mmap_pages(dmab->dev.dev, area, area->vm_end - area->vm_start, virt_to_page(dmab->area)); diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index 73a637d9472e..ba02017eca2e 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -1575,3 +1575,8 @@ static inline void vma_assert_can_modify(struct vm_area_struct *vma) if (vma_is_attached(vma)) vma_assert_write_locked(vma); } + +static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma) +{ + return vma_flags_to_page_prot(vma->flags); +} From 8e7501a361d3f10611dcd1f8111c4fdb13aa6610 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Sat, 11 Jul 2026 19:45:06 +0100 Subject: [PATCH 408/501] mm/vma: update create_init_stack_vma() to use vma_flags_t Replace use of the legacy vm_flags_t flags with vma_flags_t values in create_init_stack_vma(). As part of this change we add VMA_STACK_EARLY and VMA_STACK_INCOMPLETE vma_flags_t defines, and slightly rework create_init_stack_vma() for clarity. No functional change intended. Link: https://lore.kernel.org/20260711-b4-vma-flags-mm-v2-9-0fa2357d5431@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Zi Yan Reviewed-by: Lance Yang Reviewed-by: Vlastimil Babka (SUSE) Cc: Baolin Wang Cc: Barry Song Cc: Christian Brauner Cc: Dave Airlie Cc: David Hildenbrand Cc: Dev Jain Cc: Jani Nikula Cc: Jan Kara Cc: Jann Horn Cc: Mike Rapoport Cc: Muchun Song Cc: Nico Pache Cc: Oscar Salvador Cc: Pedro Falcato Cc: Suren Baghdasaryan Cc: Thomas Zimmermann Signed-off-by: Andrew Morton --- include/linux/mm.h | 4 ++++ mm/vma_exec.c | 18 +++++++++++------- tools/testing/vma/include/dup.h | 4 ++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 1209db1a4b92..550fb92957d1 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -442,8 +442,10 @@ enum { #define VM_STACK INIT_VM_FLAG(STACK) #ifdef CONFIG_STACK_GROWSUP #define VM_STACK_EARLY INIT_VM_FLAG(STACK_EARLY) +#define VMA_STACK_EARLY mk_vma_flags(VMA_STACK_EARLY_BIT) #else #define VM_STACK_EARLY VM_NONE +#define VMA_STACK_EARLY EMPTY_VMA_FLAGS #endif #ifdef CONFIG_ARCH_HAS_PKEYS #define VM_PKEY_SHIFT ((__force int)VMA_HIGH_ARCH_0_BIT) @@ -544,6 +546,8 @@ enum { /* Bits set in the VMA until the stack is in its final location */ #define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY) +#define VMA_STACK_INCOMPLETE_SETUP append_vma_flags( \ + VMA_STACK_EARLY, VMA_RAND_READ_BIT, VMA_SEQ_READ_BIT) #define TASK_EXEC_BIT ((current->personality & READ_IMPLIES_EXEC) ? \ VMA_EXEC_BIT : VMA_READ_BIT) diff --git a/mm/vma_exec.c b/mm/vma_exec.c index a3c6b05c65fe..7af1260689b9 100644 --- a/mm/vma_exec.c +++ b/mm/vma_exec.c @@ -112,15 +112,17 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap, unsigned long *top_mem_p) { - unsigned long flags = VM_STACK_FLAGS | VM_STACK_INCOMPLETE_SETUP; + vma_flags_t flags = VMA_STACK_INCOMPLETE_SETUP; + struct vm_area_struct *vma; int err; - struct vm_area_struct *vma = vm_area_alloc(mm); + /* VMA_STACK_FLAGS and VMA_STACK_INCOMPLETE_SETUP must not overlap. */ + VM_WARN_ON_ONCE(vma_flags_test_any_mask(&flags, VMA_STACK_FLAGS)); + + vma = vm_area_alloc(mm); if (!vma) return -ENOMEM; - vma_set_anonymous(vma); - if (mmap_write_lock_killable(mm)) { err = -EINTR; goto err_free; @@ -134,18 +136,20 @@ int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap, if (err) goto err_ksm; + vma_flags_set_mask(&flags, VMA_STACK_FLAGS); + vma_set_anonymous(vma); + /* * Place the stack at the largest stack address the architecture * supports. Later, we'll move this to an appropriate place. We don't * use STACK_TOP because that can depend on attributes which aren't * configured yet. */ - VM_WARN_ON_ONCE(VM_STACK_FLAGS & VM_STACK_INCOMPLETE_SETUP); vma->vm_end = STACK_TOP_MAX; vma->vm_start = vma->vm_end - PAGE_SIZE; if (pgtable_supports_soft_dirty()) - flags |= VM_SOFTDIRTY; - vm_flags_init(vma, flags); + vma_flags_set(&flags, VMA_SOFTDIRTY_BIT); + vma->flags = flags; vma->vm_page_prot = vma_get_page_prot(vma); err = insert_vm_struct(mm, vma); diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index ba02017eca2e..cdeb53bbdd1b 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -245,8 +245,10 @@ enum { #define VM_STACK INIT_VM_FLAG(STACK) #ifdef CONFIG_STACK_GROWS_UP #define VM_STACK_EARLY INIT_VM_FLAG(STACK_EARLY) +#define VMA_STACK_EARLY mk_vma_flags(VMA_STACK_EARLY_BIT) #else #define VM_STACK_EARLY VM_NONE +#define VMA_STACK_EARLY EMPTY_VMA_FLAGS #endif #ifdef CONFIG_ARCH_HAS_PKEYS #define VM_PKEY_SHIFT ((__force int)VMA_HIGH_ARCH_0_BIT) @@ -315,6 +317,8 @@ enum { /* Bits set in the VMA until the stack is in its final location */ #define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY) +#define VMA_STACK_INCOMPLETE_SETUP append_vma_flags( \ + VMA_STACK_EARLY, VMA_RAND_READ_BIT, VMA_SEQ_READ_BIT) #define TASK_EXEC_BIT ((current->personality & READ_IMPLIES_EXEC) ? \ VM_EXEC_BIT : VM_READ_BIT) From df751fe672c8a10c0029d2e3fc19c22c4906b979 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Sat, 11 Jul 2026 19:45:07 +0100 Subject: [PATCH 409/501] mm/vma: convert miscellaneous uses of VMA flags in core mm Update various uses of legacy flags in vma.c and mmap.c to the new vma_flags_t type, updating comments alongside them to be consistent. No functional change intended. Link: https://lore.kernel.org/20260711-b4-vma-flags-mm-v2-10-0fa2357d5431@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Baolin Wang Cc: Barry Song Cc: Christian Brauner Cc: Dave Airlie Cc: David Hildenbrand Cc: Dev Jain Cc: Jani Nikula Cc: Jan Kara Cc: Jann Horn Cc: Lance Yang Cc: Mike Rapoport Cc: Muchun Song Cc: Nico Pache Cc: Oscar Salvador Cc: Pedro Falcato Cc: Suren Baghdasaryan Cc: Thomas Zimmermann Signed-off-by: Andrew Morton --- mm/mmap.c | 39 +++++++++++++++++++++------------------ mm/vma.c | 13 ++++++++----- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/mm/mmap.c b/mm/mmap.c index 2076c70e7700..4bf26b0f1e6e 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -557,8 +557,8 @@ unsigned long do_mmap(struct file *file, unsigned long addr, } /* - * Set 'VM_NORESERVE' if we should not account for the - * memory use of this mapping. + * Set VMA_NORESERVE_BIT if we should not account for the memory use + * of this mapping. */ if (flags & MAP_NORESERVE) { /* We honor MAP_NORESERVE if allowed to overcommit */ @@ -985,7 +985,7 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned lon return NULL; if (expand_stack_locked(prev, addr)) return NULL; - if (prev->vm_flags & VM_LOCKED) + if (vma_test(prev, VMA_LOCKED_BIT)) populate_vma_page_range(prev, addr, prev->vm_end, NULL); return prev; } @@ -1009,7 +1009,7 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned lon start = vma->vm_start; if (expand_stack_locked(vma, addr)) return NULL; - if (vma->vm_flags & VM_LOCKED) + if (vma_test(vma, VMA_LOCKED_BIT)) populate_vma_page_range(vma, addr, start, NULL); return vma; } @@ -1134,18 +1134,18 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, */ vma = vma_lookup(mm, start); - if (!vma || !(vma->vm_flags & VM_SHARED)) { + if (!vma || !vma_test(vma, VMA_SHARED_BIT)) { mmap_read_unlock(mm); return -EINVAL; } - prot |= vma->vm_flags & VM_READ ? PROT_READ : 0; - prot |= vma->vm_flags & VM_WRITE ? PROT_WRITE : 0; - prot |= vma->vm_flags & VM_EXEC ? PROT_EXEC : 0; + prot |= vma_test(vma, VMA_READ_BIT) ? PROT_READ : 0; + prot |= vma_test(vma, VMA_WRITE_BIT) ? PROT_WRITE : 0; + prot |= vma_test(vma, VMA_EXEC_BIT) ? PROT_EXEC : 0; flags &= MAP_NONBLOCK; flags |= MAP_SHARED | MAP_FIXED | MAP_POPULATE; - if (vma->vm_flags & VM_LOCKED) + if (vma_test(vma, VMA_LOCKED_BIT)) flags |= MAP_LOCKED; /* Save vm_flags used to calculate prot and flags, and recheck later. */ @@ -1271,7 +1271,7 @@ unsigned long tear_down_vmas(struct mm_struct *mm, struct vma_iterator *vmi, mmap_assert_write_locked(mm); vma_iter_set(vmi, vma->vm_end); do { - if (vma->vm_flags & VM_ACCOUNT) + if (vma_test(vma, VMA_ACCOUNT_BIT)) nr_accounted += vma_pages(vma); vma_mark_detached(vma); remove_vma(vma); @@ -1420,7 +1420,7 @@ static int special_mapping_split(struct vm_area_struct *vma, unsigned long addr) { /* * Forbid splitting special mappings - kernel has expectations over - * the number of pages in mapping. Together with VM_DONTEXPAND + * the number of pages in mapping. Together with VMA_DONTEXPAND_BIT * the size of vma should stay the same over the special mapping's * lifetime. */ @@ -1692,7 +1692,7 @@ bool mmap_read_lock_maybe_expand(struct mm_struct *mm, return true; } - if (!(new_vma->vm_flags & VM_GROWSDOWN)) + if (!vma_test(new_vma, VMA_GROWSDOWN_BIT)) return false; mmap_write_lock(mm); @@ -1742,7 +1742,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) retval = vma_start_write_killable(mpnt); if (retval < 0) goto loop_out; - if (mpnt->vm_flags & VM_DONTCOPY) { + if (vma_test(mpnt, VMA_DONTCOPY_BIT)) { retval = vma_iter_clear_gfp(&vmi, mpnt->vm_start, mpnt->vm_end, GFP_KERNEL); if (retval) @@ -1752,7 +1752,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) continue; } charge = 0; - if (mpnt->vm_flags & VM_ACCOUNT) { + if (vma_test(mpnt, VMA_ACCOUNT_BIT)) { unsigned long len = vma_pages(mpnt); if (security_vm_enough_memory_mm(oldmm, len)) /* sic */ @@ -1770,16 +1770,19 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) retval = dup_userfaultfd(tmp, &uf); if (retval) goto fail_nomem_anon_vma_fork; - if (tmp->vm_flags & VM_WIPEONFORK) { + + if (vma_test(tmp, VMA_WIPEONFORK_BIT)) { /* - * VM_WIPEONFORK gets a clean slate in the child. + * VMA_WIPEONFORK_BIT gets a clean slate in the child. * Don't prepare anon_vma until fault since we don't * copy page for current vma. */ tmp->anon_vma = NULL; } else if (anon_vma_fork(tmp, mpnt)) goto fail_nomem_anon_vma_fork; - vm_flags_clear(tmp, VM_LOCKED_MASK); + + vma_start_write(tmp); + vma_clear_flags_mask(tmp, VMA_LOCKED_MASK); /* * Copy/update hugetlb private vma information. */ @@ -1812,7 +1815,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) i_mmap_unlock_write(mapping); } - if (!(tmp->vm_flags & VM_WIPEONFORK)) + if (!vma_test(tmp, VMA_WIPEONFORK_BIT)) retval = copy_page_range(tmp, mpnt); if (retval) { diff --git a/mm/vma.c b/mm/vma.c index 074f0063b6b5..a325376e62ea 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -3419,17 +3419,20 @@ struct vm_area_struct *__install_special_mapping( vm_flags_t vm_flags, void *priv, const struct vm_operations_struct *ops) { - int ret; + vma_flags_t vma_flags = legacy_to_vma_flags(vm_flags); struct vm_area_struct *vma; + int ret; vma = vm_area_alloc(mm); - if (unlikely(vma == NULL)) + if (unlikely(!vma)) return ERR_PTR(-ENOMEM); - vm_flags |= vma_flags_to_legacy(mm->def_vma_flags) | VM_DONTEXPAND; + vma_flags_set_mask(&vma_flags, mm->def_vma_flags); + vma_flags_set(&vma_flags, VMA_DONTEXPAND_BIT); if (pgtable_supports_soft_dirty()) - vm_flags |= VM_SOFTDIRTY; - vm_flags_init(vma, vm_flags & ~VM_LOCKED_MASK); + vma_flags_set(&vma_flags, VMA_SOFTDIRTY_BIT); + vma_flags_clear_mask(&vma_flags, VMA_LOCKED_MASK); + vma->flags = vma_flags; vma->vm_page_prot = vma_get_page_prot(vma); vma->vm_ops = ops; From 2fa484226b18ce09b9077d0d2664ddc38f7332d3 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Sat, 11 Jul 2026 19:45:08 +0100 Subject: [PATCH 410/501] mm/mlock: convert mlock code to use vma_flags_t Replace use of the legacy vm_flags_t flags with vma_flags_t values throughout the mlock logic. Additionally update comments to reflect the changes to be consistent. No functional change intended. Link: https://lore.kernel.org/20260711-b4-vma-flags-mm-v2-11-0fa2357d5431@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Zi Yan Reviewed-by: Lance Yang Reviewed-by: Vlastimil Babka (SUSE) Cc: Baolin Wang Cc: Barry Song Cc: Christian Brauner Cc: Dave Airlie Cc: David Hildenbrand Cc: Dev Jain Cc: Jani Nikula Cc: Jan Kara Cc: Jann Horn Cc: Mike Rapoport Cc: Muchun Song Cc: Nico Pache Cc: Oscar Salvador Cc: Pedro Falcato Cc: Suren Baghdasaryan Cc: Thomas Zimmermann Signed-off-by: Andrew Morton --- mm/mlock.c | 88 +++++++++++++++++++++++++++++------------------------- 1 file changed, 47 insertions(+), 41 deletions(-) diff --git a/mm/mlock.c b/mm/mlock.c index 34ffa954006f..efa6716e4dfb 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -329,7 +329,7 @@ static inline bool allow_mlock_munlock(struct folio *folio, * be split. And the pages are not in VM_LOCKed VMA * can be reclaimed. */ - if (!(vma->vm_flags & VM_LOCKED)) + if (!vma_test(vma, VMA_LOCKED_BIT)) return true; /* folio_within_range() cannot take KSM, but any small folio is OK */ @@ -368,7 +368,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, folio = pmd_folio(*pmd); if (folio_is_zone_device(folio)) goto out; - if (vma->vm_flags & VM_LOCKED) + if (vma_test(vma, VMA_LOCKED_BIT)) mlock_folio(folio); else munlock_folio(folio); @@ -393,7 +393,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, if (!allow_mlock_munlock(folio, vma, start, end, step)) goto next_entry; - if (vma->vm_flags & VM_LOCKED) + if (vma_test(vma, VMA_LOCKED_BIT)) mlock_folio(folio); else munlock_folio(folio); @@ -417,8 +417,8 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, * @end - end of range in @vma * @new_vma_flags - the new set of flags for @vma. * - * Called for mlock(), mlock2() and mlockall(), to set @vma VM_LOCKED; - * called for munlock() and munlockall(), to clear VM_LOCKED from @vma. + * Called for mlock(), mlock2() and mlockall(), to set @vma VMA_LOCKED_BIT; + * called for munlock() and munlockall(), to clear VMA_LOCKED_BIT from @vma. */ static void mlock_vma_pages_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, @@ -431,14 +431,14 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma, /* * There is a slight chance that concurrent page migration, - * or page reclaim finding a page of this now-VM_LOCKED vma, + * or page reclaim finding a page of this now-VMA_LOCKED_BIT vma, * will call mlock_vma_folio() and raise page's mlock_count: * double counting, leaving the page unevictable indefinitely. - * Communicate this danger to mlock_vma_folio() with VM_IO, - * which is a VM_SPECIAL flag not allowed on VM_LOCKED vmas. + * Communicate this danger to mlock_vma_folio() with VMA_IO_BIT, + * which is a VMA_SPECIAL_FLAGS flag not allowed on VMA_LOCKED_BIT vmas. * mmap_lock is held in write mode here, so this weird * combination should not be visible to other mmap_lock users; - * but WRITE_ONCE so rmap walkers must see VM_IO if VM_LOCKED. + * but WRITE_ONCE so rmap walkers must see VMA_IO_BIT if VMA_LOCKED_BIT. */ if (vma_flags_test(new_vma_flags, VMA_LOCKED_BIT)) vma_flags_set(new_vma_flags, VMA_IO_BIT); @@ -458,7 +458,7 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma, /* * mlock_fixup - handle mlock[all]/munlock[all] requests. * - * Filters out "special" vmas -- VM_LOCKED never gets set for these, and + * Filters out "special" vmas -- VMA_LOCKED_BIT never gets set for these, and * munlock is a no-op. However, for some special vmas, we go ahead and * populate the ptes. * @@ -466,24 +466,23 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma, */ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, struct vm_area_struct **prev, unsigned long start, - unsigned long end, vm_flags_t newflags) + unsigned long end, vma_flags_t *new_vma_flags) { - vma_flags_t new_vma_flags = legacy_to_vma_flags(newflags); const vma_flags_t old_vma_flags = vma->flags; struct mm_struct *mm = vma->vm_mm; int nr_pages; int ret = 0; - if (vma_flags_same_pair(&old_vma_flags, &new_vma_flags) || + if (vma_flags_same_pair(&old_vma_flags, new_vma_flags) || vma_is_secretmem(vma) || !vma_supports_mlock(vma)) { /* - * Don't set VM_LOCKED or VM_LOCKONFAULT and don't count. - * For secretmem, don't allow the memory to be unlocked. + * Don't set VMA_LOCKED_BIT or VMA_LOCKONFAULT_BIT and don't + * count. For secretmem, don't allow the memory to be unlocked. */ goto out; } - vma = vma_modify_flags(vmi, *prev, vma, start, end, &new_vma_flags); + vma = vma_modify_flags(vmi, *prev, vma, start, end, new_vma_flags); if (IS_ERR(vma)) { ret = PTR_ERR(vma); goto out; @@ -493,7 +492,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, * Keep track of amount of locked VM. */ nr_pages = (end - start) >> PAGE_SHIFT; - if (!vma_flags_test(&new_vma_flags, VMA_LOCKED_BIT)) + if (!vma_flags_test(new_vma_flags, VMA_LOCKED_BIT)) nr_pages = -nr_pages; else if (vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT)) nr_pages = 0; @@ -502,15 +501,15 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, /* * vm_flags is protected by the mmap_lock held in write mode. * It's okay if try_to_unmap_one unmaps a page just after we - * set VM_LOCKED, populate_vma_page_range will bring it back. + * set VMA_LOCKED_BIT, populate_vma_page_range will bring it back. */ - if (vma_flags_test(&new_vma_flags, VMA_LOCKED_BIT) && + if (vma_flags_test(new_vma_flags, VMA_LOCKED_BIT) && vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT)) { /* No work to do, and mlocking twice would be wrong */ vma_start_write(vma); - vma->flags = new_vma_flags; + vma->flags = *new_vma_flags; } else { - mlock_vma_pages_range(vma, start, end, &new_vma_flags); + mlock_vma_pages_range(vma, start, end, new_vma_flags); } out: *prev = vma; @@ -518,7 +517,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, } static int apply_vma_lock_flags(unsigned long start, size_t len, - vm_flags_t flags) + const vma_flags_t *flags) { unsigned long nstart, end, tmp; struct vm_area_struct *vma, *prev; @@ -543,18 +542,20 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, tmp = vma->vm_start; for_each_vma_range(vmi, vma, end) { int error; - vm_flags_t newflags; + vma_flags_t newflags; if (vma->vm_start != tmp) return -ENOMEM; - newflags = vma->vm_flags & ~VM_LOCKED_MASK; - newflags |= flags; + newflags = vma->flags; + vma_flags_clear_mask(&newflags, VMA_LOCKED_MASK); + vma_flags_set_mask(&newflags, *flags); + /* Here we know that vma->vm_start <= nstart < vma->vm_end. */ tmp = vma->vm_end; if (tmp > end) tmp = end; - error = mlock_fixup(&vmi, vma, &prev, nstart, tmp, newflags); + error = mlock_fixup(&vmi, vma, &prev, nstart, tmp, &newflags); if (error) return error; tmp = vma_iter_end(&vmi); @@ -589,7 +590,7 @@ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm, end = start + len; for_each_vma_range(vmi, vma, end) { - if (vma->vm_flags & VM_LOCKED) { + if (vma_test(vma, VMA_LOCKED_BIT)) { if (start > vma->vm_start) count -= (start - vma->vm_start); if (end < vma->vm_end) { @@ -615,7 +616,8 @@ static int __mlock_posix_error_return(long retval) return retval; } -static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t flags) +static __must_check int do_mlock(unsigned long start, size_t len, + vma_flags_t *flags) { unsigned long locked; unsigned long lock_limit; @@ -664,24 +666,27 @@ static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t fla SYSCALL_DEFINE2(mlock, unsigned long, start, size_t, len) { - return do_mlock(start, len, VM_LOCKED); + vma_flags_t flags = mk_vma_flags(VMA_LOCKED_BIT); + + return do_mlock(start, len, &flags); } SYSCALL_DEFINE3(mlock2, unsigned long, start, size_t, len, int, flags) { - vm_flags_t vm_flags = VM_LOCKED; + vma_flags_t vma_flags = mk_vma_flags(VMA_LOCKED_BIT); if (flags & ~MLOCK_ONFAULT) return -EINVAL; if (flags & MLOCK_ONFAULT) - vm_flags |= VM_LOCKONFAULT; + vma_flags_set(&vma_flags, VMA_LOCKONFAULT_BIT); - return do_mlock(start, len, vm_flags); + return do_mlock(start, len, &vma_flags); } SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) { + vma_flags_t flags = EMPTY_VMA_FLAGS; int ret; start = untagged_addr(start); @@ -691,7 +696,7 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) if (mmap_write_lock_killable(current->mm)) return -EINTR; - ret = apply_vma_lock_flags(start, len, 0); + ret = apply_vma_lock_flags(start, len, &flags); mmap_write_unlock(current->mm); return ret; @@ -705,14 +710,15 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) * There are a couple of subtleties with this. If mlockall() is called multiple * times with different flags, the values do not necessarily stack. If mlockall * is called once including the MCL_FUTURE flag and then a second time without - * it, VM_LOCKED and VM_LOCKONFAULT will be cleared from mm->def_vma_flags. + * it, VMA_LOCKED_BIT and VMA_LOCKONFAULT_BIT will be cleared from + * mm->def_vma_flags. */ static int apply_mlockall_flags(int flags) { VMA_ITERATOR(vmi, current->mm, 0); struct mm_struct *mm = current->mm; struct vm_area_struct *vma, *prev = NULL; - vm_flags_t to_add = 0; + vma_flags_t to_add = EMPTY_VMA_FLAGS; vma_flags_clear_mask(&mm->def_vma_flags, VMA_LOCKED_MASK); if (flags & MCL_FUTURE) { @@ -726,20 +732,20 @@ static int apply_mlockall_flags(int flags) } if (flags & MCL_CURRENT) { - to_add |= VM_LOCKED; + vma_flags_set(&to_add, VMA_LOCKED_BIT); if (flags & MCL_ONFAULT) - to_add |= VM_LOCKONFAULT; + vma_flags_set(&to_add, VMA_LOCKONFAULT_BIT); } for_each_vma(vmi, vma) { int error; - vm_flags_t newflags; + vma_flags_t newflags = vma->flags; - newflags = vma->vm_flags & ~VM_LOCKED_MASK; - newflags |= to_add; + vma_flags_clear_mask(&newflags, VMA_LOCKED_MASK); + vma_flags_set_mask(&newflags, to_add); error = mlock_fixup(&vmi, vma, &prev, vma->vm_start, vma->vm_end, - newflags); + &newflags); /* Ignore errors, but prev needs fixing up. */ if (error) prev = vma; From 31b50515a737cb51431d6803c6b7e851b139498a Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Sat, 11 Jul 2026 19:45:09 +0100 Subject: [PATCH 411/501] mm/mprotect: convert mprotect code to use vma_flags_t Replace use of the legacy vm_flags_t flags with vma_flags_t values throughout the mprotect logic. Note that we retain the legacy vm_flags_t bit shifting code in do_mprotect_pkey(), deferring a vma_flags_t approach to this for the time being. Additionally update comments to reflect the changes to be consistent. No functional change intended. Link: https://lore.kernel.org/20260711-b4-vma-flags-mm-v2-12-0fa2357d5431@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Baolin Wang Cc: Barry Song Cc: Christian Brauner Cc: Dave Airlie Cc: David Hildenbrand Cc: Dev Jain Cc: Jani Nikula Cc: Jan Kara Cc: Jann Horn Cc: Lance Yang Cc: Mike Rapoport Cc: Muchun Song Cc: Nico Pache Cc: Oscar Salvador Cc: Pedro Falcato Cc: Suren Baghdasaryan Cc: Thomas Zimmermann Signed-off-by: Andrew Morton --- mm/mprotect.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mm/mprotect.c b/mm/mprotect.c index dc27bbc1712f..2888ee638d87 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -40,7 +40,7 @@ static bool maybe_change_pte_writable(struct vm_area_struct *vma, pte_t pte) { - if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE))) + if (WARN_ON_ONCE(!vma_test(vma, VMA_WRITE_BIT))) return false; /* Don't touch entries that are not even readable. */ @@ -97,7 +97,7 @@ static bool can_change_shared_pte_writable(struct vm_area_struct *vma, bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, pte_t pte) { - if (!(vma->vm_flags & VM_SHARED)) + if (!vma_test(vma, VMA_SHARED_BIT)) return can_change_private_pte_writable(vma, addr, pte); return can_change_shared_pte_writable(vma, pte); @@ -194,7 +194,7 @@ static __always_inline void set_write_prot_commit_flush_ptes(struct vm_area_stru { bool set_write; - if (vma->vm_flags & VM_SHARED) { + if (vma_test(vma, VMA_SHARED_BIT)) { set_write = can_change_shared_pte_writable(vma, ptent); prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, nr_ptes, /* idx = */ 0, set_write, tlb); @@ -846,8 +846,8 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, vm_unacct_memory(nrpages); /* - * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major - * fault on access. + * Private VMA_LOCKED_BIT VMA becoming writable: trigger COW to avoid + * major fault on access. */ if (vma_flags_test(&new_vma_flags, VMA_WRITE_BIT) && vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT) && @@ -921,7 +921,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, goto out; start = vma->vm_start; error = -EINVAL; - if (!(vma->vm_flags & VM_GROWSDOWN)) + if (!vma_test(vma, VMA_GROWSDOWN_BIT)) goto out; } else { if (vma->vm_start > start) @@ -929,7 +929,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, if (unlikely(grows & PROT_GROWSUP)) { end = vma->vm_end; error = -EINVAL; - if (!(vma->vm_flags & VM_GROWSUP)) + if (!vma_test_single_mask(vma, VMA_GROWSUP)) goto out; } } @@ -953,7 +953,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, } /* Does the application expect PROT_READ to imply PROT_EXEC */ - if (rier && (vma->vm_flags & VM_MAYEXEC)) + if (rier && vma_test(vma, VMA_MAYEXEC_BIT)) prot |= PROT_EXEC; /* From f4a31afd9647f9fcbaeb474172caab2a72596438 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Sat, 11 Jul 2026 19:45:10 +0100 Subject: [PATCH 412/501] mm/mremap: convert mremap code to use vma_flags_t Replace use of the legacy vm_flags_t flags with vma_flags_t values throughout the mremap logic. Note that, in replacing vm_flags_clear() (which takes the VMA write lock) with vma_clear_flags() and vma_clear_flags_mask() (which do not) respectively in unmap_source_vma() and dontunmap_complete(), we do not add a VMA write lock to account for htis. This is because, in both cases, move_vma() is their calling function and this has already acquired the VMA write lock on vrm->vma whose VMA flags are being cleared. In the case of vma_set_flags() in unmap_source_vma() we do need to do this - as prev and next were not necessarily write locked at this point. Additionally update comments to reflect the changes to be consistent. No functional change intended. Link: https://lore.kernel.org/20260711-b4-vma-flags-mm-v2-13-0fa2357d5431@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Baolin Wang Cc: Barry Song Cc: Christian Brauner Cc: Dave Airlie Cc: David Hildenbrand Cc: Dev Jain Cc: Jani Nikula Cc: Jan Kara Cc: Jann Horn Cc: Lance Yang Cc: Mike Rapoport Cc: Muchun Song Cc: Nico Pache Cc: Oscar Salvador Cc: Pedro Falcato Cc: Suren Baghdasaryan Cc: Thomas Zimmermann Signed-off-by: Andrew Morton --- mm/mremap.c | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/mm/mremap.c b/mm/mremap.c index 384ef4cc2195..b64aa1f6e07e 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -68,7 +68,7 @@ struct vma_remap_struct { bool populate_expand; /* mlock()'d expanded, must populate. */ enum mremap_type remap_type; /* expand, shrink, etc. */ bool mmap_locked; /* Is mm currently write-locked? */ - unsigned long charged; /* If VM_ACCOUNT, # pages to account. */ + unsigned long charged; /* If VMA_ACCOUNT_BIT, # pgs to account */ bool vmi_needs_invalidate; /* Is the VMA iterator invalidated? */ }; @@ -963,7 +963,7 @@ static unsigned long vrm_set_new_addr(struct vma_remap_struct *vrm) if (vrm->flags & MREMAP_FIXED) map_flags |= MAP_FIXED; - if (vma->vm_flags & VM_MAYSHARE) + if (vma_test(vma, VMA_MAYSHARE_BIT)) map_flags |= MAP_SHARED; res = get_unmapped_area(vma->vm_file, new_addr, vrm->new_len, pgoff, @@ -985,7 +985,7 @@ static bool vrm_calc_charge(struct vma_remap_struct *vrm) { unsigned long charged; - if (!(vrm->vma->vm_flags & VM_ACCOUNT)) + if (!vma_test(vrm->vma, VMA_ACCOUNT_BIT)) return true; /* @@ -1012,7 +1012,7 @@ static bool vrm_calc_charge(struct vma_remap_struct *vrm) */ static void vrm_uncharge(struct vma_remap_struct *vrm) { - if (!(vrm->vma->vm_flags & VM_ACCOUNT)) + if (!vma_test(vrm->vma, VMA_ACCOUNT_BIT)) return; vm_unacct_memory(vrm->charged); @@ -1032,7 +1032,7 @@ static void vrm_stat_account(struct vma_remap_struct *vrm, struct vm_area_struct *vma = vrm->vma; vm_stat_account(mm, vma->vm_flags, pages); - if (vma->vm_flags & VM_LOCKED) + if (vma_test(vma, VMA_LOCKED_BIT)) mm->locked_vm += pages; } @@ -1176,7 +1176,7 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) * arose, in which case we _do_ wish to unmap the _new_ VMA, which means * we actually _do_ want it be unaccounted. */ - bool accountable_move = (vma->vm_flags & VM_ACCOUNT) && + bool accountable_move = vma_test(vma, VMA_ACCOUNT_BIT) && !(vrm->flags & MREMAP_DONTUNMAP); /* @@ -1195,7 +1195,7 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) * portions of the original VMA that remain. */ if (accountable_move) { - vm_flags_clear(vma, VM_ACCOUNT); + vma_clear_flags(vma, VMA_ACCOUNT_BIT); /* We are about to split vma, so store the start/end. */ vm_start = vma->vm_start; vm_end = vma->vm_end; @@ -1220,8 +1220,8 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) * | | * |-------------| * - * Having cleared VM_ACCOUNT from the whole VMA, after we unmap above - * we'll end up with: + * Having cleared VMA_ACCOUNT_BIT from the whole VMA, after we unmap + * above we'll end up with: * * addr end * | | @@ -1241,13 +1241,15 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) if (vm_start < addr) { struct vm_area_struct *prev = vma_prev(&vmi); - vm_flags_set(prev, VM_ACCOUNT); /* Acquires VMA lock. */ + vma_start_write(prev); + vma_set_flags(prev, VMA_ACCOUNT_BIT); } if (vm_end > end) { struct vm_area_struct *next = vma_next(&vmi); - vm_flags_set(next, VM_ACCOUNT); /* Acquires VMA lock. */ + vma_start_write(next); + vma_set_flags(next, VMA_ACCOUNT_BIT); } } } @@ -1330,8 +1332,8 @@ static void dontunmap_complete(struct vma_remap_struct *vrm, unsigned long old_start = vrm->vma->vm_start; unsigned long old_end = vrm->vma->vm_end; - /* We always clear VM_LOCKED[ONFAULT] on the old VMA. */ - vm_flags_clear(vrm->vma, VM_LOCKED_MASK); + /* We always clear VMA_LOCKED[ONFAULT]_BIT on the old VMA. */ + vma_clear_flags_mask(vrm->vma, VMA_LOCKED_MASK); /* * anon_vma links of the old vma is no longer needed after its page @@ -1767,14 +1769,14 @@ static int check_prep_vma(struct vma_remap_struct *vrm) * based on the original. There are no known use cases for this * behavior. As a result, fail such attempts. */ - if (!old_len && !(vma->vm_flags & (VM_SHARED | VM_MAYSHARE))) { + if (!old_len && !vma_test_any(vma, VMA_SHARED_BIT, VMA_MAYSHARE_BIT)) { pr_warn_once("%s (%d): attempted to duplicate a private mapping with mremap. This is not supported.\n", current->comm, current->pid); return -EINVAL; } if ((vrm->flags & MREMAP_DONTUNMAP) && - (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP))) + vma_test_any(vma, VMA_DONTEXPAND_BIT, VMA_PFNMAP_BIT)) return -EINVAL; /* @@ -1804,7 +1806,7 @@ static int check_prep_vma(struct vma_remap_struct *vrm) return 0; /* We are expanding and the VMA is mlock()'d so we need to populate. */ - if (vma->vm_flags & VM_LOCKED) + if (vma_test(vma, VMA_LOCKED_BIT)) vrm->populate_expand = true; /* Need to be careful about a growing mapping */ @@ -1812,10 +1814,10 @@ static int check_prep_vma(struct vma_remap_struct *vrm) if (pgoff + (new_len >> PAGE_SHIFT) < pgoff) return -EINVAL; - if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)) + if (vma_test_any(vma, VMA_DONTEXPAND_BIT, VMA_PFNMAP_BIT)) return -EFAULT; - if (!mlock_future_ok(mm, vma->vm_flags & VM_LOCKED, vrm->delta)) + if (!mlock_future_ok(mm, vma_test(vma, VMA_LOCKED_BIT), vrm->delta)) return -EAGAIN; if (!may_expand_vm(mm, &vma->flags, vrm->delta >> PAGE_SHIFT)) From eed8da1ff61da86b7dd001d32614425de354ff29 Mon Sep 17 00:00:00 2001 From: xu xin Date: Tue, 14 Jul 2026 09:26:22 +0800 Subject: [PATCH 413/501] mm/mm_slot.h: add a helper function mm_slot_remove Patch series "Two small patches to clean up mm/mm_slot.h", v3. mm_slot.h is mainly used by THP and KSM. Patch 1 introduces mm_slot_remove() to abstract the common hash_del() + list_del() sequence used in both khugepaged and KSM. Patch 2 adds a comment explaining why mm_slot_lookup/insert cannot be converted to static inline functions. This patch (of 2): Both THP and KSM manage per-mm scanning slots using the mm_slot structure. The slot is kept in a hash table and a list, and removal from both containers requires the same two operations: hash_del() and list_del(). Introduce mm_slot_remove() to abstract the common hash_del() + list_del() sequence used in both khugepaged and KSM. No functional change is intended. Link: https://lore.kernel.org/202607140924549782dUh3YBPmy8g1NDMK2zIW@zte.com.cn Link: https://lore.kernel.org/20260714092622583ayzGzGYSjAareKKHt_T91@zte.com.cn Signed-off-by: xu xin Reviewed-by: Nico Pache Reviewed-by: Zi Yan Reviewed-by: Barry Song Acked-by: David Hildenbrand (Arm) Reviewed-by: Lorenzo Stoakes (ARM) Reviewed-by: SJ Park Cc: Baolin Wang Cc: Chengming Zhou Cc: Dev Jain Cc: Lance Yang Cc: Ryan Roberts Cc: Wang Yaxin Cc: Qi Zheng Signed-off-by: Andrew Morton --- mm/khugepaged.c | 6 ++---- mm/ksm.c | 9 +++------ mm/mm_slot.h | 5 +++++ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 45e8245d80da..27e8f3077e80 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -606,8 +606,7 @@ void __khugepaged_exit(struct mm_struct *mm) spin_lock(&khugepaged_mm_lock); slot = mm_slot_lookup(mm_slots_hash, mm); if (slot && khugepaged_scan.mm_slot != slot) { - hash_del(&slot->hash); - list_del(&slot->mm_node); + mm_slot_remove(slot); free = 1; } spin_unlock(&khugepaged_mm_lock); @@ -1802,8 +1801,7 @@ static void collect_mm_slot(struct mm_slot *slot) if (collapse_test_exit(mm)) { /* free mm_slot */ - hash_del(&slot->hash); - list_del(&slot->mm_node); + mm_slot_remove(slot); /* * Not strictly needed because the mm exited already. diff --git a/mm/ksm.c b/mm/ksm.c index 14550f69cf02..47006f494fcb 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -1257,8 +1257,7 @@ static int unmerge_and_remove_all_rmap_items(void) struct mm_slot, mm_node); ksm_scan.mm_slot = mm_slot_entry(slot, struct ksm_mm_slot, slot); if (ksm_test_exit(mm)) { - hash_del(&mm_slot->slot.hash); - list_del(&mm_slot->slot.mm_node); + mm_slot_remove(&mm_slot->slot); spin_unlock(&ksm_mmlist_lock); mm_slot_free(mm_slot_cache, mm_slot); @@ -2772,8 +2771,7 @@ static struct ksm_rmap_item *scan_get_next_rmap_item(struct page **page) * or when all VM_MERGEABLE areas have been unmapped (and * mmap_lock then protects against race with MADV_MERGEABLE). */ - hash_del(&mm_slot->slot.hash); - list_del(&mm_slot->slot.mm_node); + mm_slot_remove(&mm_slot->slot); spin_unlock(&ksm_mmlist_lock); mm_slot_free(mm_slot_cache, mm_slot); @@ -3116,8 +3114,7 @@ void __ksm_exit(struct mm_struct *mm) if (ksm_scan.mm_slot == mm_slot) goto unlock; if (!mm_slot->rmap_list) { - hash_del(&slot->hash); - list_del(&slot->mm_node); + mm_slot_remove(slot); easy_to_free = 1; } else { list_move(&slot->mm_node, diff --git a/mm/mm_slot.h b/mm/mm_slot.h index 83f18ed1c4bd..5de3e91d86b4 100644 --- a/mm/mm_slot.h +++ b/mm/mm_slot.h @@ -52,4 +52,9 @@ static inline void mm_slot_free(struct kmem_cache *cache, void *objp) hash_add(_hashtable, &_mm_slot->hash, (unsigned long)_mm); \ }) +static inline void mm_slot_remove(struct mm_slot *slot) +{ + hash_del(&slot->hash); + list_del(&slot->mm_node); +} #endif /* _LINUX_MM_SLOT_H */ From 5c3746a0fe380ae7ec5ee0e4aaef19b68b51d903 Mon Sep 17 00:00:00 2001 From: xu xin Date: Tue, 14 Jul 2026 09:28:15 +0800 Subject: [PATCH 414/501] mm/mm_slot.h: add comments for mm_slot_lookup/insert mm_slot_lookup() and mm_slot_insert() are the only helpers in this header that are implemented as macros rather than static inline functions. This may look inconsistent without explanation. Explain they must be macros because hash_for_each_possible() needs the table as an array (for sizeof), not a pointer. Link: https://lore.kernel.org/20260714092815120Wv-CFDlLKtsTmda--97Qw@zte.com.cn Signed-off-by: xu xin Reviewed-by: Barry Song Acked-by: David Hildenbrand (Arm) Reviewed-by: Qi Zheng Reviewed-by: Lorenzo Stoakes (ARM) Reviewed-by: SJ Park Reviewed-by: Zi Yan Cc: Baolin Wang Cc: Chengming Zhou Cc: Dev Jain Cc: Lance Yang Cc: Nico Pache Cc: Ryan Roberts Cc: Wang Yaxin Signed-off-by: Andrew Morton --- mm/mm_slot.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mm/mm_slot.h b/mm/mm_slot.h index 5de3e91d86b4..9b09b68e5742 100644 --- a/mm/mm_slot.h +++ b/mm/mm_slot.h @@ -33,6 +33,12 @@ static inline void mm_slot_free(struct kmem_cache *cache, void *objp) kmem_cache_free(cache, objp); } +/* + * Note: mm_slot_lookup and mm_slot_insert cannot be converted to static inline + * functions because the hash helpers (hash_for_each_possible and hash_add) rely + * on the actual array argument 'hashtable' for sizeof() instead of pointers. + */ + #define mm_slot_lookup(_hashtable, _mm) \ ({ \ struct mm_slot *tmp_slot, *mm_slot = NULL; \ From e0ea9467dbefc5803fd36b6f1714c2be51518977 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 14 Jul 2026 07:35:34 -0700 Subject: [PATCH 415/501] mm/damon/core: hide private damon_region fields Patch series "mm/damon/core: hide core-private struct fields". DAMON core structs hide core-private fields using 'private:' comment tags. It is incomplete and inconsistent. The linked list heads in a few structs, for example, are intended to be hidden, and always be used using the wrapper macros like damon_for_each_region(). But those were mistakenly marked as non-private. A few core layer-only fields were also mistakenly added as non-private. This only encourages callers to directly use the private fields. It is easy to make mistakes, and difficult to control. Mark all such DAMON core struct fields as private. Patches 1-8 mark the private fields for damon_region, damon_target, damos_quota_goal, damos_quota, damos_filter, damos, damon_filter and damon_probe, respectively. Patch 9 removes DAMON_SYSFS's direct access to core-private field, damon_ctx->ops. Finally patch 10 mark the private fields for damon_ctx. This patch (of 10): damon_region->list is intended to be used by only the DAMON core layer. But it is mistakenly not marked as private. Hide it from the callers by marking it private. Link: https://lore.kernel.org/20260714143544.101305-1-sj@kernel.org Link: https://lore.kernel.org/20260714143544.101305-2-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index f69442a9d431..d32377e7e535 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -46,7 +46,6 @@ struct damon_size_range { * @sampling_addr: Address of the sample for the next access check. * @nr_accesses: Access frequency of this region. * @probe_hits: Number of probe-positive region samples. - * @list: List head for siblings. * @age: Age of this region. * * For any use case, @ar should be non-zero positive size. damon_set_regions() @@ -68,9 +67,11 @@ struct damon_region { unsigned long sampling_addr; unsigned int nr_accesses; unsigned char probe_hits[DAMON_MAX_PROBES]; - struct list_head list; unsigned int age; -/* private: Internal value for age calculation. */ +/* private: internal use only. */ + /* List head for siblings. */ + struct list_head list; + /* for age calculation. */ unsigned int last_nr_accesses; unsigned char last_probe_hits[DAMON_MAX_PROBES]; }; From 300cb8592dd2722783eacb2fece0325570be1b0e Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 14 Jul 2026 07:35:35 -0700 Subject: [PATCH 416/501] mm/damon/core: hide private damon_target fields 'nr_regions', 'regions_list' and 'list' fields of damon_target are intended to be used by only the DAMON core layer. Those are mistakenly not marked as private. Mark as private. Link: https://lore.kernel.org/20260714143544.101305-3-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index d32377e7e535..6e160557a001 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -79,9 +79,6 @@ struct damon_region { /** * struct damon_target - Represents a monitoring target. * @pid: The PID of the virtual address space to monitor. - * @nr_regions: Number of monitoring target regions of this target. - * @regions_list: Head of the monitoring target regions of this target. - * @list: List head for siblings. * @obsolete: Whether the commit destination target is obsolete. * * Each monitoring context could have multiple targets. For example, a context @@ -95,10 +92,14 @@ struct damon_region { */ struct damon_target { struct pid *pid; - unsigned int nr_regions; - struct list_head regions_list; - struct list_head list; bool obsolete; +/* private: */ + /* Number of monitoring target regions of this target. */ + unsigned int nr_regions; + /* Head of the monitoring target regions of this target. */ + struct list_head regions_list; + /* List head for siblings. */ + struct list_head list; }; /** From 4e061ef21a09ca0256b60ed0cf05085ecae2ef70 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 14 Jul 2026 07:35:36 -0700 Subject: [PATCH 417/501] mm/damon/core: hide private damos_quota_goal fields 'last_psi_total' and 'list' of damos_quota_goal are intended to be used by only the DAMON core layer. Those are mistakenly not marked as private, though. Mark as private. Link: https://lore.kernel.org/20260714143544.101305-4-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 6e160557a001..d838cc27ee72 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -175,10 +175,8 @@ enum damos_quota_goal_metric { * @metric: Metric to be used for representing the goal. * @target_value: Target value of @metric to achieve with the tuning. * @current_value: Current value of @metric. - * @last_psi_total: Last measured total PSI * @nid: Node id. * @memcg_id: Memcg id. - * @list: List head for siblings. * * Data structure for getting the current score of the quota tuning goal. The * score is calculated by how close @current_value and @target_value are. Then @@ -201,12 +199,17 @@ struct damos_quota_goal { unsigned long current_value; /* metric-dependent fields */ union { - u64 last_psi_total; struct { int nid; u64 memcg_id; }; +/* private: */ + /* Last measured total PSI */ + u64 last_psi_total; + }; +/* private: */ + /* List head for siblings. */ struct list_head list; }; From f4dacc7d93231016a494ee2451a2351a95e017bb Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 14 Jul 2026 07:35:37 -0700 Subject: [PATCH 418/501] mm/damon/core: hide private damos_quota fields damos_quota->goals is intended to be used by only the DAMON core layer. But it is mistakenly not marked as private. Mark as private. Link: https://lore.kernel.org/20260714143544.101305-5-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index d838cc27ee72..bd024b89adfa 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -228,7 +228,6 @@ enum damos_quota_goal_tuner { * @reset_interval: Charge reset interval in milliseconds. * @ms: Maximum milliseconds that the scheme can use. * @sz: Maximum bytes of memory that the action can be applied. - * @goals: Head of quota tuning goals (&damos_quota_goal) list. * @goal_tuner: Goal-based @esz tuning algorithm to use. * @esz: Effective size quota in bytes. * @fail_charge_num: Failed regions charge rate numerator. @@ -255,7 +254,7 @@ enum damos_quota_goal_tuner { * the scheme's action. DAMON then compares it against &sz and uses smaller * one as the effective quota. * - * If @goals is not empty, DAMON calculates yet another size quota based on the + * If goals is not empty, DAMON calculates yet another size quota based on the * goals using its internal feedback loop algorithm, for every @reset_interval. * Then, if the new size quota is smaller than the effective quota, it uses the * new size quota as the effective quota. @@ -276,7 +275,7 @@ struct damos_quota { unsigned long reset_interval; unsigned long ms; unsigned long sz; - struct list_head goals; + enum damos_quota_goal_tuner goal_tuner; unsigned long esz; @@ -288,6 +287,8 @@ struct damos_quota { unsigned int weight_age; /* private: */ + /* Head of quota tuning goals (&damos_quota_goal) list. */ + struct list_head goals; /* For throughput estimation */ unsigned long total_charged_sz; unsigned long total_charged_ns; From b814d8f86a58a95a12147e0733c97ddef23bb261 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 14 Jul 2026 07:35:38 -0700 Subject: [PATCH 419/501] mm/damon/core: hide private damos_filter fields damos_filter->list is intended to be used by only the DAMON core layer. However, it is mistakenly not marked as private. Mark as private. Link: https://lore.kernel.org/20260714143544.101305-6-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index bd024b89adfa..15517be3e496 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -432,7 +432,6 @@ enum damos_filter_type { * &damon_ctx->adaptive_targets if @type is * DAMOS_FILTER_TYPE_TARGET. * @sz_range: Size range if @type is DAMOS_FILTER_TYPE_HUGEPAGE_SIZE. - * @list: List head for siblings. * * Before applying the &damos->action to a memory region, DAMOS checks if each * byte of the region matches to this given condition and avoid applying the @@ -450,6 +449,8 @@ struct damos_filter { int target_idx; struct damon_size_range sz_range; }; +/* private: */ + /* List head for siblings. */ struct list_head list; }; From c647b1ec5e9bba76b3fe594f1ff9e6b32ff6b2c2 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 14 Jul 2026 07:35:39 -0700 Subject: [PATCH 420/501] mm/damon/core: hide private damos fields 'core_filters', 'ops_filters', 'last_applied' and 'list' fields of damos are intended to be used by only the DAMON core layer. However, those are mistakenly not marked as private. Mark as private. Link: https://lore.kernel.org/20260714143544.101305-7-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 52 ++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 15517be3e496..907d65f03d08 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -522,12 +522,8 @@ struct damos_migrate_dests { * @wmarks: Watermarks for automated (in)activation of this scheme. * @migrate_dests: Destination nodes if @action is "migrate_{hot,cold}". * @target_nid: Destination node if @action is "migrate_{hot,cold}". - * @core_filters: Additional set of &struct damos_filter for &action. - * @ops_filters: ops layer handling &struct damos_filter objects list. - * @last_applied: Last @action applied ops-managing entity. * @stat: Statistics of this scheme. * @max_nr_snapshots: Upper limit of nr_snapshots stat. - * @list: List head for siblings. * * For each @apply_interval_us, DAMON finds regions which fit in the * &pattern and applies &action to those. To avoid consuming too much @@ -549,16 +545,7 @@ struct damos_migrate_dests { * * Before applying the &action to a memory region, &struct damon_operations * implementation could check pages of the region and skip &action to respect - * &core_filters - * - * The minimum entity that @action can be applied depends on the underlying - * &struct damon_operations. Since it may not be aligned with the core layer - * abstract, namely &struct damon_region, &struct damon_operations could apply - * @action to same entity multiple times. Large folios that underlying on - * multiple &struct damon region objects could be such examples. The &struct - * damon_operations can use @last_applied to avoid that. DAMOS core logic - * unsets @last_applied when each regions walking for applying the scheme is - * finished. + * &struct damos_filter. * * After applying the &action to each region, &stat is updated. * @@ -569,6 +556,16 @@ struct damos { struct damos_access_pattern pattern; enum damos_action action; unsigned long apply_interval_us; + struct damos_quota quota; + struct damos_watermarks wmarks; + union { + struct { + int target_nid; + struct damos_migrate_dests migrate_dests; + }; + }; + struct damos_stat stat; + unsigned long max_nr_snapshots; /* private: internal use only */ /* * number of sample intervals that should be passed before applying @@ -585,20 +582,25 @@ struct damos { /* whether to reject core/ops filters umatched regions */ bool core_filters_default_reject; bool ops_filters_default_reject; -/* public: */ - struct damos_quota quota; - struct damos_watermarks wmarks; - union { - struct { - int target_nid; - struct damos_migrate_dests migrate_dests; - }; - }; + /* Additional set of &struct damos_filter for &action. */ struct list_head core_filters; + /* ops layer handling &struct damos_filter objects list. */ struct list_head ops_filters; + /* + * Last @action applied ops-managing entity. + * + * The minimum entity that @action can be applied depends on the + * underlying &struct damon_operations. Since it may not be aligned + * with the core layer abstract, namely &struct damon_region, &struct + * damon_operations could apply @action to same entity multiple times. + * Large folios that underlying on multiple &struct damon region + * objects could be such examples. The &struct damon_operations can + * use @last_applied to avoid that. DAMOS core logic unsets + * @last_applied when each regions walking for applying the scheme is + * finished. + */ void *last_applied; - struct damos_stat stat; - unsigned long max_nr_snapshots; + /* List head for siblings. */ struct list_head list; }; From 55686d63f343cd738f70d120a1864bd5949d092d Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 14 Jul 2026 07:35:40 -0700 Subject: [PATCH 421/501] mm/damon/core: hide private damon_filter fields damon_filter->list is intended to be used by only the DAMON core layer. However, it is mistakenly not marked as private. Mark as private. Link: https://lore.kernel.org/20260714143544.101305-8-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 907d65f03d08..6c3c7c75b6ca 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -758,7 +758,6 @@ enum damon_filter_type { * @matching: Whether this filter is for the type-matching ones. * @allow: Whether the @type-@matching ones should pass this filter. * @memcg_id: Memcg id of the question if @type is DAMON_FILTER_MEMCG. - * @list: Siblings list. */ struct damon_filter { enum damon_filter_type type; @@ -767,6 +766,8 @@ struct damon_filter { union { u64 memcg_id; }; +/* private: */ + /* Siblings list. */ struct list_head list; }; From 6a0c38aba6040c92d2213b9fad4cdac4fcbf3c8a Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 14 Jul 2026 07:35:41 -0700 Subject: [PATCH 422/501] mm/damon/core: hide private damon_probe fields 'filters' and 'list' fields of damon_probe are intended to be used by only the DAMON core layer. However, those are mistakenly not marked as private. Mark as private. Link: https://lore.kernel.org/20260714143544.101305-9-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 6c3c7c75b6ca..537e59bfd707 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -775,12 +775,13 @@ struct damon_filter { * struct damon_probe - Data region attribute probe. * * @weight: Relative priority of the attribute for this probe. - * @filters: Filters for assessing if a given region is for this probe. - * @list: Siblings list. */ struct damon_probe { unsigned int weight; +/* private: */ + /* Filters for assessing if a given region is for this probe. */ struct list_head filters; + /* Siblings list. */ struct list_head list; }; From 131c20823ebcc039ae19b1bb4d8343a6b00026d2 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 14 Jul 2026 07:35:42 -0700 Subject: [PATCH 423/501] mm/damon/sysfs: do not directly access damon_ctx->ops damon_ctx->ops field is intended to be used by only the DAMON core layer. DAMON_SYSFS is directly reading the field to find if it is for physical address monitoring, though. Use the API function for the purpose, damon_target_has_pid(), instead. Link: https://lore.kernel.org/20260714143544.101305-10-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/sysfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index b5fe036f7801..208bb639f50f 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -2051,7 +2051,7 @@ static int damon_sysfs_add_targets(struct damon_ctx *ctx, int i, err; /* Multiple physical address space monitoring targets makes no sense */ - if (ctx->ops.id == DAMON_OPS_PADDR && sysfs_targets->nr > 1) + if (!damon_target_has_pid(ctx) && sysfs_targets->nr > 1) return -EINVAL; for (i = 0; i < sysfs_targets->nr; i++) { From 8315cf471eff1669bdc536829935ef2607caa108 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 14 Jul 2026 07:35:43 -0700 Subject: [PATCH 424/501] mm/damon/core: hide core-private damon_ctx fields 'ops', 'probes', 'adaptive_targets', 'schemes', and 'rnd_state' fields of damon_ctx are intended to be used by only the DAMON core layer. However, those are mistakenly not marked as private. Mark as private. Link: https://lore.kernel.org/20260714143544.101305-11-sj@kernel.org Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 537e59bfd707..327f534696aa 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -848,14 +848,9 @@ struct damon_attrs { * thread other than the kdamond should be made using safe DAMON APIs, * including damon_call() and damos_walk(). * - * @ops: Set of monitoring operations for given use cases. - * @probes: Head of probes (&damon_probe) list. * @addr_unit: Scale factor for core to ops address conversion. * @min_region_sz: Minimum region size. * @pause: Pause kdamond main loop. - * @adaptive_targets: Head of monitoring targets (&damon_target) list. - * @schemes: Head of schemes (&damos) list. - * @rnd_state: Per-ctx PRNG state for damon_rand(). */ struct damon_ctx { struct damon_attrs attrs; @@ -904,15 +899,21 @@ struct damon_ctx { struct mutex kdamond_lock; /* public: */ - struct damon_operations ops; - struct list_head probes; unsigned long addr_unit; unsigned long min_region_sz; bool pause; +/* private: */ + /* Set of monitoring operations for given use cases. */ + struct damon_operations ops; + /* Head of monitoring targets (&damon_target) list. */ struct list_head adaptive_targets; + /* Head of probes (&damon_probe) list. */ + struct list_head probes; + /* Head of schemes (&damos) list. */ struct list_head schemes; + /* @rnd_state: Per-ctx PRNG state for damon_rand(). */ struct rnd_state rnd_state; }; From ff3d40545f95d677e5c5e2baac30ca610c7135a4 Mon Sep 17 00:00:00 2001 From: Petr Tesarik Date: Tue, 14 Jul 2026 15:23:00 +0200 Subject: [PATCH 425/501] mm: let node_reclaim() return the number of pages reclaimed There is only one caller, get_page_from_freelist(), and it does not make any use of the reason for skipping the reclaim, nor does it make any distinction between a full and partially successful reclaim. Therefore, node_reclaim() can simply return the number of pages that have been reclaimed, same as __node_reclaim(), and the NODE_RECLAIM_xxx macros can be removed. There is one small change of behavior when __node_reclaim() was attempted but returned zero. The allocation now skips the zone immediately; before this patch, the zone watermarks were checked first. I believe it was an oversight rather than intention, because the chances that zone watermark is OK after __node_reclaim() did not reclaim any pages are very close to zero. Originally, I was looking for occurences of NODE_RECLAIM_SOME and NODE_RECLAIM_SUCCESS, but I couldn't find any. That's because they are typecast from the result of a relational operator. This seemed a bit fragile, so I dug a bit deeper and came up with this proposed cleanup. Link: https://lore.kernel.org/20260714132300.2136018-1-ptesarik@suse.com Signed-off-by: Petr Tesarik Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Acked-by: Johannes Weiner Cc: Brendan Jackman Cc: Brendan Jackman Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/internal.h | 14 +++++--------- mm/page_alloc.c | 19 ++++--------------- mm/vmscan.c | 16 ++++++++-------- 3 files changed, 17 insertions(+), 32 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index 874be94cf257..f26423de4ca2 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1098,23 +1098,19 @@ static inline void mlock_drain_local(void) { } static inline void mlock_drain_remote(int cpu) { } #endif /* !CONFIG_MMU */ -#define NODE_RECLAIM_NOSCAN -2 -#define NODE_RECLAIM_FULL -1 -#define NODE_RECLAIM_SOME 0 -#define NODE_RECLAIM_SUCCESS 1 - #ifdef CONFIG_NUMA extern int node_reclaim_mode; -extern int node_reclaim(struct pglist_data *, gfp_t, unsigned int); +extern unsigned long node_reclaim(struct pglist_data *pgdat, + gfp_t gfp_mask, unsigned int order); extern int find_next_best_node(int node, nodemask_t *used_node_mask); #else #define node_reclaim_mode 0 -static inline int node_reclaim(struct pglist_data *pgdat, gfp_t mask, - unsigned int order) +static inline unsigned long node_reclaim(struct pglist_data *pgdat, + gfp_t mask, unsigned int order) { - return NODE_RECLAIM_NOSCAN; + return 0; } static inline int find_next_best_node(int node, nodemask_t *used_node_mask) { diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 718aa08c3c97..520ebd2fa40b 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -3908,8 +3908,6 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, if (!zone_watermark_fast(zone, order, mark, ac->highest_zoneidx, alloc_flags, gfp_mask)) { - int ret; - if (cond_accept_memory(zone, order, alloc_flags)) goto try_this_zone; @@ -3930,22 +3928,13 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, !zone_allows_reclaim(zonelist_zone(ac->preferred_zoneref), zone)) continue; - ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); - switch (ret) { - case NODE_RECLAIM_NOSCAN: - /* did not scan */ + if (!node_reclaim(zone->zone_pgdat, gfp_mask, order)) continue; - case NODE_RECLAIM_FULL: - /* scanned but unreclaimable */ - continue; - default: - /* did we reclaim enough */ - if (zone_watermark_ok(zone, order, mark, - ac->highest_zoneidx, alloc_flags)) - goto try_this_zone; + /* did we reclaim enough */ + if (!zone_watermark_ok(zone, order, mark, + ac->highest_zoneidx, alloc_flags)) continue; - } } try_this_zone: diff --git a/mm/vmscan.c b/mm/vmscan.c index 745c5c358d5e..006829efceca 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -7773,9 +7773,9 @@ static unsigned long __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, return sc->nr_reclaimed; } -int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) +unsigned long node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) { - int ret; + unsigned long ret; /* Minimum pages needed in order to stay on node */ const unsigned long nr_pages = 1 << order; struct scan_control sc = { @@ -7802,13 +7802,13 @@ int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages && node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <= pgdat->min_slab_pages) - return NODE_RECLAIM_FULL; + return 0; /* * Do not scan if the allocation should not be delayed. */ if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC)) - return NODE_RECLAIM_NOSCAN; + return 0; /* * Only run node reclaim on the local node or on nodes that do not @@ -7817,15 +7817,15 @@ int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) * as wide as possible. */ if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id()) - return NODE_RECLAIM_NOSCAN; + return 0; if (test_and_set_bit_lock(PGDAT_RECLAIM_LOCKED, &pgdat->flags)) - return NODE_RECLAIM_NOSCAN; + return 0; - ret = __node_reclaim(pgdat, gfp_mask, nr_pages, &sc) >= nr_pages; + ret = __node_reclaim(pgdat, gfp_mask, nr_pages, &sc); clear_bit_unlock(PGDAT_RECLAIM_LOCKED, &pgdat->flags); - if (ret) + if (ret >= nr_pages) count_vm_event(PGSCAN_ZONE_RECLAIM_SUCCESS); else count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED); From 123e4619ab6c8ab1c4cb1d7a58311a2af13929cd Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 14 Jul 2026 20:09:56 -0700 Subject: [PATCH 426/501] mm/damon/core: avoid infinite kdamond_merge_regions() internal loop Patch series "mm/damon: unurgent fixes for infinite loop, NULL de-ref and races", v1.1. Sashiko found a few issues in DAMON that could cause infinite loop, NULL dereference and monitoring results degradation. The first two sounds scary but the infinite loop happens only under unreasonable user setup. The NULL dereference is only in a unit test. Monitoring results degradation is trivial since it is only best-effort, and those happens from only unlikely races. Still those are bugs that better to fix if possible. Fix those. This patch (of 6): Due to online parameter update like events, the number of DAMON regions could be higher than the user-set upper limit. kdamond_merge_regions() repeats merge regions until the number meets the limit, while doubling the merge threshold up to the theoretical maximum threshold. It is tried only up to the theoretical maximum threshold because even the aggressive merging can fail from reducing the number of regions under the user-defined upper limit. For example, there could be many user-defined non-contiguous regions that cannot be merged. The threshold based loop break condition is evaluated by comparing the threshold for the next merging try against the theoretical maximum threshold. If max_thres is larger than UINT_MAX / 2, doubling the threshold could make it overflow, and bypass the loop break condition. In the case, if the number of regions cannot be reduced under the upper limit like explained above, the loop will run infinitely. Prevent the case by doing the break condition check before doubling the threshold. Also, prevent the threshold exceeding the maximum threshold, as it could overflow and apply the wrong merge threshold. This issue is unlikely to occur in real world, since having the max_thres higher than UINT_MAX / 2 require unrealistically large aggregation intervals compared to the sampling interval. Also, it requires an unrealistically large number of uncontiguous regions setup. Nonetheless, the consequence is bad and the fix is simple. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260715031002.108504-1-sj@kernel.org Link: https://lore.kernel.org/20260715031002.108504-2-sj@kernel.org Link: https://lore.kernel.org/20260709145425.96247-1-sj@kernel.org [1] Fixes: 310d6c15e910 ("mm/damon/core: merge regions aggressively when max_nr_regions is unmet") Signed-off-by: SJ Park Cc: # 6.10.x Signed-off-by: Andrew Morton --- mm/damon/core.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 6c4215cc809e..603b102ff80f 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -3372,7 +3372,7 @@ static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, max_thres = c->attrs.aggr_interval / (c->attrs.sample_interval ? c->attrs.sample_interval : 1); - do { + while (true) { nr_regions = 0; damon_for_each_target(t, c) { damon_merge_regions_of(t, threshold, sz_limit, c, @@ -3380,9 +3380,14 @@ static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, nr_regions += damon_nr_regions(t); } count_age = false; - threshold = max(1, threshold * 2); - } while (nr_regions > c->attrs.max_nr_regions && - threshold / 2 < max_thres); + if (nr_regions <= c->attrs.max_nr_regions || + max_thres <= threshold) + break; + if (threshold < max_thres / 2) + threshold = max(1, threshold * 2); + else + threshold = max_thres; + } } #ifdef CONFIG_DAMON_DEBUG_SANITY From b640708929aa956235866bf7717d89018c661083 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 14 Jul 2026 20:09:57 -0700 Subject: [PATCH 427/501] mm/damon/tests/core-kunit: catch test failure in test_merge_regions_of() KUNIT_EXPECT_EQ() does not abort the execution of test code when the expectation is not met. But damon_test_merge_regions_of() code after its initial KUNIT_EXPECT_EQ() call assumes the expectation is met. It does a per-region test with a hard-coded number of regions that is correct only if the expectation was met. As a result, __nth_region_of() could return NULL, and the test code can dereference NULL pointers. Fix the issue by catching the expectation failure and skip the per-region tests. The user impact on realistic setups should be negligible, as it is a unit test. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260715031002.108504-3-sj@kernel.org Link: https://lore.kernel.org/20260710144937.26981-1-sj@kernel.org [1] Fixes: 17ccae8bb5c9 ("mm/damon: add kunit tests") Signed-off-by: SJ Park Cc: # 5.15.x Signed-off-by: Andrew Morton --- mm/damon/tests/core-kunit.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 485472ddebd1..eba643762132 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -260,11 +260,14 @@ static void damon_test_merge_regions_of(struct kunit *test) damon_merge_regions_of(t, 9, 9999, ctx, true); /* 0-112, 114-130, 130-156, 156-170, 170-230, 230-10170 */ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 6u); + if (damon_nr_regions(t) != 6) + goto out; for (i = 0; i < 6; i++) { r = __nth_region_of(t, i); KUNIT_EXPECT_EQ(test, r->ar.start, saddrs[i]); KUNIT_EXPECT_EQ(test, r->ar.end, eaddrs[i]); } +out: damon_free_target(t); damon_destroy_ctx(ctx); } From 831846078caa14b7d00b2ccca8b8fe522afe3204 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 14 Jul 2026 20:09:58 -0700 Subject: [PATCH 428/501] mm/damon/vaddr: drop last same folio access check optimization The optimization can race when multiple kdamonds are running. Meanwhile, the impact of the optimization is quite doubtful. Just remove it. The user impact of the issue should be quite trivial. After all, the race can happen only when the user intentionally setup DAMON in the way. Even if it happens, it would be rare and only degrade the best-effort monitoring results. No critical consequences like kernel panic or memory corruption happen. The race possibility was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260715031002.108504-4-sj@kernel.org Link: https://lore.kernel.org/20260621204050.10993-1-sj@kernel.org [1] Fixes: 3f49584b262c ("mm/damon: implement primitives for the virtual memory address spaces") Signed-off-by: SJ Park Cc: # 5.15.x Signed-off-by: Andrew Morton --- mm/damon/vaddr.c | 33 ++++++--------------------------- 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index d10b8042adb5..d487b7a4a104 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -383,8 +383,6 @@ static void damon_va_prepare_access_checks(struct damon_ctx *ctx) } struct damon_young_walk_private { - /* size of the folio for the access checked virtual memory address */ - unsigned long *folio_sz; bool young; }; @@ -411,7 +409,6 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr, mmu_notifier_test_young(walk->mm, addr)) priv->young = true; - *priv->folio_sz = HPAGE_PMD_SIZE; huge_out: spin_unlock(ptl); return 0; @@ -430,7 +427,6 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr, if (pte_young(ptent) || !folio_test_idle(folio) || mmu_notifier_test_young(walk->mm, addr)) priv->young = true; - *priv->folio_sz = folio_size(folio); out: pte_unmap_unlock(pte, ptl); return 0; @@ -458,7 +454,6 @@ static int damon_young_hugetlb_entry(pte_t *pte, unsigned long hmask, if (pte_young(entry) || !folio_test_idle(folio) || mmu_notifier_test_young(walk->mm, addr)) priv->young = true; - *priv->folio_sz = huge_page_size(h); folio_put(folio); @@ -470,11 +465,9 @@ static int damon_young_hugetlb_entry(pte_t *pte, unsigned long hmask, #define damon_young_hugetlb_entry NULL #endif /* CONFIG_HUGETLB_PAGE */ -static bool damon_va_young(struct mm_struct *mm, unsigned long addr, - unsigned long *folio_sz) +static bool damon_va_young(struct mm_struct *mm, unsigned long addr) { struct damon_young_walk_private arg = { - .folio_sz = folio_sz, .young = false, }; @@ -494,28 +487,17 @@ static bool damon_va_young(struct mm_struct *mm, unsigned long addr, * r the region to be checked */ static void __damon_va_check_access(struct mm_struct *mm, - struct damon_region *r, bool same_target) + struct damon_region *r) { - static unsigned long last_addr; - static unsigned long last_folio_sz = PAGE_SIZE; - static bool last_accessed; + bool accessed; if (!mm) { damon_update_region_access_rate(r, false); return; } - /* If the region is in the last checked page, reuse the result */ - if (same_target && (ALIGN_DOWN(last_addr, last_folio_sz) == - ALIGN_DOWN(r->sampling_addr, last_folio_sz))) { - damon_update_region_access_rate(r, last_accessed); - return; - } - - last_accessed = damon_va_young(mm, r->sampling_addr, &last_folio_sz); - damon_update_region_access_rate(r, last_accessed); - - last_addr = r->sampling_addr; + accessed = damon_va_young(mm, r->sampling_addr); + damon_update_region_access_rate(r, accessed); } static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) @@ -524,15 +506,12 @@ static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) struct mm_struct *mm; struct damon_region *r; unsigned int max_nr_accesses = 0; - bool same_target; damon_for_each_target(t, ctx) { mm = damon_get_mm(t); - same_target = false; damon_for_each_region(r, t) { - __damon_va_check_access(mm, r, same_target); + __damon_va_check_access(mm, r); max_nr_accesses = max(r->nr_accesses, max_nr_accesses); - same_target = true; } if (mm) mmput(mm); From f23f0aa62b2f32c2b12f95959fc4603ef81678b2 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 14 Jul 2026 20:09:59 -0700 Subject: [PATCH 429/501] mm/damon/paddr: drop last same folio access check reuse optimization It can race when multiple kdamonds are being used. The problem from the race is doubtful, but the gain from the optimization is also doubtful. Simply drop the optimization in favor of code simplicity. The user impact is doubtfully trivial. After all, this kind of interference can happen only by intentional user setup. Even if it happens, it will be rare, and the consequence is degradation of the best-effort monitoring results. No critical consequences like kernel panic or memory corruption happen. The race was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260715031002.108504-5-sj@kernel.org Link: https://lore.kernel.org/20260621204050.10993-1-sj@kernel.org [1] Fixes: a28397beb55b ("mm/damon: implement primitives for physical address space monitoring") Signed-off-by: SJ Park Cc: # 5.16.x Signed-off-by: Andrew Morton --- mm/damon/paddr.c | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index b85f88a7a38f..e4f98d67461f 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -65,7 +65,7 @@ static void damon_pa_prepare_access_checks(struct damon_ctx *ctx) } } -static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) +static bool damon_pa_young(phys_addr_t paddr) { struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); bool accessed; @@ -74,7 +74,6 @@ static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) return false; accessed = damon_folio_young(folio); - *folio_sz = folio_size(folio); folio_put(folio); return accessed; } @@ -82,23 +81,12 @@ static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) static void __damon_pa_check_access(struct damon_region *r, unsigned long addr_unit) { - static phys_addr_t last_addr; - static unsigned long last_folio_sz = PAGE_SIZE; - static bool last_accessed; + bool accessed; phys_addr_t sampling_addr = damon_pa_phys_addr( r->sampling_addr, addr_unit); - /* If the region is in the last checked page, reuse the result */ - if (ALIGN_DOWN(last_addr, last_folio_sz) == - ALIGN_DOWN(sampling_addr, last_folio_sz)) { - damon_update_region_access_rate(r, last_accessed); - return; - } - - last_accessed = damon_pa_young(sampling_addr, &last_folio_sz); - damon_update_region_access_rate(r, last_accessed); - - last_addr = sampling_addr; + accessed = damon_pa_young(sampling_addr); + damon_update_region_access_rate(r, accessed); } static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) From ab4d9358e32316fa39b5f1f5360292135978c3d9 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 14 Jul 2026 20:10:00 -0700 Subject: [PATCH 430/501] mm/damon/sysfs: read addr_unit only once in damon_sysfs_apply_inputs() damon_sysfs_apply_inputs() reads addr_unit twice. It could race with addr_unit_store(). As a result, the min_region_sz could wrongly be set up. Read it once. The user impact is trivial. Sane users ain't update the parameter in parallel. Even if it happens, the DAMON core layer handles the wrong min_region_sz (!is_power_of_2()). Even if somehow the race ended up making a min_region_sz that is different from the user's intention but still valid, only monitoring itself runs differently than expected. No critical consequences like kernel panic or memory corruption happen. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260715031002.108504-6-sj@kernel.org Link: https://lore.kernel.org/20260714142950.100711-1-sj@kernel.org [1] Fixes: 540a2aebc657 ("mm/damon/sysfs: implement addr_unit file under context dir") Signed-off-by: SJ Park Cc: # 6.18.x Signed-off-by: Andrew Morton --- mm/damon/sysfs.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 208bb639f50f..3db4da88fc72 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -2099,11 +2099,11 @@ static int damon_sysfs_apply_inputs(struct damon_ctx *ctx, err = damon_select_ops(ctx, sys_ctx->ops_id); if (err) return err; - ctx->addr_unit = sys_ctx->addr_unit; + ctx->addr_unit = READ_ONCE(sys_ctx->addr_unit); /* addr_unit is respected by only DAMON_OPS_PADDR */ if (sys_ctx->ops_id == DAMON_OPS_PADDR) ctx->min_region_sz = max( - DAMON_MIN_REGION_SZ / sys_ctx->addr_unit, 1); + DAMON_MIN_REGION_SZ / ctx->addr_unit, 1); ctx->pause = sys_ctx->pause; err = damon_sysfs_set_attrs(ctx, sys_ctx->attrs); if (err) From 5adaaa28be8a79ddd7e103b171f9d6e14e7fc26e Mon Sep 17 00:00:00 2001 From: SJ Park Date: Tue, 14 Jul 2026 20:10:01 -0700 Subject: [PATCH 431/501] mm/damon/sysfs: read ops_id only once in damon_sysfs_apply_inputs() damon_sysfs_apply_inputs() reads ops_id twice. It could race with ops_id_store(). As a result, the min_region_sz could wrongly be set up. Read it once. The user impact is trivial. Sane users ain't update the parameter in parallel. Even if it happens, the DAMON core layer handles the wrong min_region_sz (!is_power_of_2()). Even if somehow the race ended up making a min_region_sz that is different from the user's intention but still valid, only monitoring itself runs differently than expected. No critical consequences like kernel panic or memory corruption happen The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260715031002.108504-7-sj@kernel.org Link: https://lore.kernel.org/20260703172417.95426-1-sj@kernel.org [1] Fixes: 8d009da32f13 ("mm/damon/sysfs: set damon_ctx->min_sz_region only for paddr use case") Signed-off-by: SJ Park Cc: # 6.18.x Signed-off-by: Andrew Morton --- mm/damon/sysfs.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 3db4da88fc72..e3858ffab4b2 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -2094,14 +2094,16 @@ static inline bool damon_sysfs_kdamond_running( static int damon_sysfs_apply_inputs(struct damon_ctx *ctx, struct damon_sysfs_context *sys_ctx) { + enum damon_ops_id ops_id; int err; - err = damon_select_ops(ctx, sys_ctx->ops_id); + ops_id = READ_ONCE(sys_ctx->ops_id); + err = damon_select_ops(ctx, ops_id); if (err) return err; ctx->addr_unit = READ_ONCE(sys_ctx->addr_unit); /* addr_unit is respected by only DAMON_OPS_PADDR */ - if (sys_ctx->ops_id == DAMON_OPS_PADDR) + if (ops_id == DAMON_OPS_PADDR) ctx->min_region_sz = max( DAMON_MIN_REGION_SZ / ctx->addr_unit, 1); ctx->pause = sys_ctx->pause; From 76bdedfa62fc7c0d49046347d876baafe92472a2 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Fri, 17 Jul 2026 09:48:59 +0000 Subject: [PATCH 432/501] mm/secretmem: don't allow highmem folios secretmem_fault() allocates a folio with GFP_HIGHUSER and then calls set_direct_map_invalid_noflush() without checking folio_test_highmem(). This causes a warning and process crash (vibe-coded reproducer in Link below): Su[ 30.071284] ------------[ cut here ]------------ ccessfully allocated and mapped 2097152000 bytes at 0x3a449000 Populating memor[ 30.074614] CPA: called for zero pte. vaddr = 0 cpa->vaddr = 0 y... [ 30.078636] WARNING: arch/x86/mm/pat/set_memory.c:1840 at __cpa_process_fault+0x34d/0x360, CPU#5: allocate_secret/570 [ 30.084789] CPU: 5 UID: 0 PID: 570 Comm: allocate_secret Not tainted 7.1.0-14063-g4edcdefd4083-dirty #10 PREEMPTLAZY [ 30.090937] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.17.0-0-gb52ca86e094d-prebuilt.qemu.org 04/01/2014 [ 30.097543] EIP: __cpa_process_fault+0x34d/0x360 [ 30.100514] Code: ff ff 85 c0 0f 89 7d fe ff ff e9 3d fe ff ff 8b 03 8b 00 c7 04 24 c8 ff 64 c1 89 44 24 08 8b 45 e8 89 44 24 04 e8 53 7 a 00 00 <0f> 0b c7 45 f0 f2 ff ff ff e9 fc fc ff ff 90 8d 74 26 00 55 25 00 [ 30.110829] EAX: 00000000 EBX: f64afe98 ECX: 00000000 EDX: 00000000 [ 30.114799] ESI: 00000000 EDI: f64afe98 EBP: f64afe04 ESP: f64afdcc [ 30.118785] DS: 007b ES: 007b FS: 00d8 GS: 0033 SS: 0068 EFLAGS: 00010246 [ 30.123020] CR0: 80050033 CR2: 46c48ffc CR3: 038c8000 CR4: 00000690 [ 30.127010] Call Trace: [ 30.129078] __change_page_attr_set_clr+0x5e7/0x870 [ 30.132275] ? console_unlock+0x99/0x130 [ 30.135069] ? irq_work_queue+0x36/0x70 [ 30.137853] ? page_address+0xd3/0xf0 [ 30.140421] set_direct_map_invalid_noflush+0x52/0x60 [ 30.143782] secretmem_fault+0x128/0x210 [ 30.146560] __do_fault+0x25/0x90 [ 30.149053] handle_mm_fault+0x6d1/0xcb0 [ 30.151759] exc_page_fault+0x135/0x3b0 [ 30.154487] ? doublefault_shim+0x150/0x150 [ 30.157416] handle_exception+0x130/0x130 [ 30.160137] EIP: 0x804d29f [ 30.162307] Code: 89 54 08 e1 89 54 08 e5 89 54 08 e9 89 54 08 ed c3 0f b6 44 24 08 89 7c 24 0c 69 c0 01 01 01 01 8b 7c 24 04 f7 c7 0f 0 0 00 00 <89> 44 0f fc 75 0e c1 e9 02 f3 ab 8b 44 24 04 8b 7c 24 0c c3 31 d2 [ 30.172936] EAX: 5a5a5a5a EBX: 00000000 ECX: 0c800000 EDX: 3a449000 [ 30.176927] ESI: 00000000 EDI: 3a449000 EBP: bfbbae18 ESP: bfbbadac [ 30.180897] DS: 007b ES: 007b FS: 0000 GS: 0033 SS: 007b EFLAGS: 00010246 [ 30.185161] ? doublefault_shim+0x150/0x150 [ 30.187979] ---[ end trace 0000000000000000 ]--- Bus error (core dumped) ./allocate_secret_i686 2000M The equivalent bug was pointed out by a local Sashiko instance on https://lore.kernel.org/all/20260410151746.61150-3-kalyazin@amazon.com/ This hasn't been reproduced it on older kernel versions but from code inspection the bug seems to go back to the original introduction in commit 1507f51255c9f ("mm: introduce memfd_secret system call to create "secret" memory areas"). If this configuration has always been broken, there's no need to worry too much about feature regression here. Nonetheless, instead of just completely disabling secretmem under !HIGHMEM, just drop __GFP_HIGHMEM. This means that now where you previously got a crash, instead you'll just see the secretmem process OOM. Could secretmem just support highmem by saying "this isn't in the direct map anyway" and bailing out before the set_direct_map_invalid_noflush()? Maybe. That depends on requirements that are not well-defined (e.g. is it OK that kmap_local_page() is not a NOP for those pages?), and would require some research and deep thinking. Let's "defer" that until an actual usecase arises. Link: https://github.com/bjackman/limmat-kernel-nix/commit/7b2acba2d3a5ef01400d493a155beb1d135b6bb5 Link: https://lore.kernel.org/20260717-secretmem-highmem-v2-1-1f1a961ca91e@google.com Link: https://lore.kernel.org/all/20260704192603.40aa80cf9242b77aa75e8d8d@linux-foundation.org/ Fixes: 1507f51255c9 ("mm: introduce memfd_secret system call to create "secret" memory areas") Signed-off-by: Brendan Jackman Suggested-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Reviewed-by: Lorenzo Stoakes (ARM) Reviewed-by: Mike Rapoport (Microsoft) Cc: Brendan Jackman Cc: Liam R. Howlett Cc: Michal Hocko Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/secretmem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/secretmem.c b/mm/secretmem.c index 4877c262cb1f..d29865075b6e 100644 --- a/mm/secretmem.c +++ b/mm/secretmem.c @@ -202,7 +202,7 @@ static struct file *secretmem_file_create(unsigned long flags) if (IS_ERR(file)) goto err_free_inode; - mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); + mapping_set_gfp_mask(inode->i_mapping, GFP_USER); mapping_set_unevictable(inode->i_mapping); inode->i_op = &secretmem_iops; From e105f1aa271162728884a9561952ddac3a41c287 Mon Sep 17 00:00:00 2001 From: Manuel Ebner Date: Mon, 29 Jun 2026 18:11:56 +0200 Subject: [PATCH 433/501] docs/mm: fix braces Correct typos in mm documentation by balancing parentheses. Link: https://lore.kernel.org/20260629161156.90213-2-manuelebner@mailbox.org Signed-off-by: Manuel Ebner Acked-by: Randy Dunlap Reviewed-by: Lorenzo Stoakes Acked-by: David Hildenbrand (Arm) Reviewed-by: Vishal Moola Reviewed-by: SJ Park Cc: Jonathan Corbet Signed-off-by: Andrew Morton --- Documentation/mm/hmm.rst | 2 +- Documentation/mm/process_addrs.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/mm/hmm.rst b/Documentation/mm/hmm.rst index e021218ada58..fc1b8dc19825 100644 --- a/Documentation/mm/hmm.rst +++ b/Documentation/mm/hmm.rst @@ -349,7 +349,7 @@ between device driver specific code and shared common code: system memory and device private memory. One of the first steps migrate_vma_setup() does is to invalidate other - device's MMUs with the ``mmu_notifier_invalidate_range_start(()`` and + device's MMUs with the ``mmu_notifier_invalidate_range_start()`` and ``mmu_notifier_invalidate_range_end()`` calls around the page table walks to fill in the ``args->src`` array with PFNs to be migrated. The ``invalidate_range_start()`` callback is passed a diff --git a/Documentation/mm/process_addrs.rst b/Documentation/mm/process_addrs.rst index 042d64d72421..a7296f251799 100644 --- a/Documentation/mm/process_addrs.rst +++ b/Documentation/mm/process_addrs.rst @@ -724,7 +724,7 @@ the zap and the invocation of :c:func:`!free_pgtables`. Since it is assumed that all such steps have been taken, page table entries are cleared without page table locks (in the :c:func:`!pgd_clear`, :c:func:`!p4d_clear`, -:c:func:`!pud_clear`, and :c:func:`!pmd_clear` functions. +:c:func:`!pud_clear`, and :c:func:`!pmd_clear` functions). .. note:: It is possible for leaf page tables to be torn down independent of the page tables above it as is done by From fb0493ad1ad4260b0f41ed8d63806a2278747c8a Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Wed, 15 Jul 2026 15:42:34 +0100 Subject: [PATCH 434/501] selftests/mm: add PAGEMAP_SCAN test for THP PMD holes Add coverage for the PMD-hole case fixed by "fs/proc/task_mmu: fix PAGEMAP_SCAN written state for PMD holes": a MAP_PRIVATE|MAP_ANON THP that is uffd-wp'd and then dropped with MADV_DONTNEED leaves a pmd_none hole with no page table, which PAGEMAP_SCAN must still report as written. Factor the populate/drop/scan-both-paths sequence out of unpopulated_scan_test() into a helper, and add unpopulated_thp_scan_test() that reuses it with a THP. Include for MADV_COLLAPSE; lacks it on older glibc (e.g. 2.34). Same approach as commit fd5295afae91 ("selftests/mm: hmm-tests: include linux/mman.h to access MADV_COLLAPSE"). Link: https://lore.kernel.org/aljWYfPRCVc6IB2b@thinkstation Link: https://lore.kernel.org/20260715144234.442721-3-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau Cc: Muhammad Usama Anjum Cc: Peter Xu Cc: Zenghui Yu Cc: David Hildenbrand Cc: Jann Horn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Pedro Falcato Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Assisted-by: Claude:claude-fable-5 Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/pagemap_ioctl.c | 118 +++++++++++++++------ 1 file changed, 83 insertions(+), 35 deletions(-) diff --git a/tools/testing/selftests/mm/pagemap_ioctl.c b/tools/testing/selftests/mm/pagemap_ioctl.c index f9bcff8e78fa..f6ab626a90b4 100644 --- a/tools/testing/selftests/mm/pagemap_ioctl.c +++ b/tools/testing/selftests/mm/pagemap_ioctl.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -1058,50 +1059,96 @@ static void test_simple(void) * the generic path (reached e.g. via category_anyof_mask) must report every * page written. */ +/* + * Populate @mem (optionally collapsing it into a THP first), drop it with + * MADV_DONTNEED, then check PAGEMAP_SCAN reports the whole range written via + * both the fast and generic query paths. A dropped THP leaves a pmd_none hole + * with no page table, exercising pagemap_scan_pte_hole(); a base-page range + * leaves pte_none entries. + */ +static void unpopulated_written_test(const char *name, char *mem, long size, + bool use_thp) +{ + long npages = size / page_size, fast = 0, slow = 0, ret; + struct page_region regions[16]; + int i; + + wp_init(mem, size); + + /* Populate, optionally collapse to a THP, then drop it. */ + memset(mem, 1, size); + if (use_thp && + (madvise(mem, size, MADV_COLLAPSE) || + !check_huge_anon(mem, size / hpage_size, hpage_size))) { + ksft_test_result_skip("%s could not form a THP\n", name); + goto out; + } + if (madvise(mem, size, MADV_DONTNEED)) { + ksft_test_result_fail("%s MADV_DONTNEED failed\n", name); + goto out; + } + + /* Fast path: category_mask == return_mask == PAGE_IS_WRITTEN. */ + ret = pagemap_ioctl(mem, size, regions, ARRAY_SIZE(regions), 0, 0, + PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); + for (i = 0; ret > 0 && i < ret; i++) + fast += LEN(regions[i]); + + /* Generic path: same query expressed via category_anyof_mask. */ + ret = pagemap_ioctl(mem, size, regions, ARRAY_SIZE(regions), 0, 0, + 0, PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN); + for (i = 0; ret > 0 && i < ret; i++) + slow += LEN(regions[i]); + + ksft_test_result(fast == npages && slow == npages, + "%s unpopulated range reported written by both paths (%ld, %ld of %ld)\n", + name, fast, slow, npages); +out: + wp_free(mem, size); +} + static void unpopulated_scan_test(void) { - int npages = 16, i; - long mem_size = npages * page_size; - struct page_region regions[16]; - long fast = 0, slow = 0, ret; + long mem_size = 16 * page_size; char *mem; mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (mem == MAP_FAILED) - ksft_exit_fail_msg("%s mmap failed\n", __func__); + if (mem == MAP_FAILED) { + ksft_test_result_skip("%s mmap failed\n", __func__); + return; + } - wp_init(mem, mem_size); - - /* Populate, then drop: the ptes become pte_none without a marker. */ - memset(mem, 1, mem_size); - if (madvise(mem, mem_size, MADV_DONTNEED)) - ksft_exit_fail_msg("%s MADV_DONTNEED failed\n", __func__); - - /* Fast path: category_mask == return_mask == PAGE_IS_WRITTEN. */ - ret = pagemap_ioctl(mem, mem_size, regions, npages, 0, 0, - PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("%s fast scan failed\n", __func__); - for (i = 0; i < ret; i++) - fast += LEN(regions[i]); - - /* Generic path: same query expressed via category_anyof_mask. */ - ret = pagemap_ioctl(mem, mem_size, regions, npages, 0, 0, - 0, PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("%s generic scan failed\n", __func__); - for (i = 0; i < ret; i++) - slow += LEN(regions[i]); - - ksft_test_result(fast == npages && slow == npages, - "%s unpopulated ptes reported written by both paths (%ld, %ld of %d)\n", - __func__, fast, slow, npages); - - wp_free(mem, mem_size); + unpopulated_written_test(__func__, mem, mem_size, false); munmap(mem, mem_size); } +/* + * Same as unpopulated_scan_test(), but the range is a THP: a full-PMD + * MADV_DONTNEED leaves a pmd_none hole with no page table. + */ +static void unpopulated_thp_scan_test(void) +{ + char *area, *mem; + + if (!hpage_size) { + ksft_test_result_skip("%s THP not supported\n", __func__); + return; + } + + /* Over-allocate so a PMD-aligned, THP-sized range fits inside. */ + area = mmap(NULL, 2 * hpage_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (area == MAP_FAILED) { + ksft_test_result_skip("%s mmap failed\n", __func__); + return; + } + mem = (char *)(((unsigned long)area + hpage_size - 1) & ~(hpage_size - 1)); + + unpopulated_written_test(__func__, mem, hpage_size, true); + munmap(area, 2 * hpage_size); +} + int sanity_tests(void) { unsigned long long mem_size, vec_size; @@ -1610,7 +1657,7 @@ int main(int __attribute__((unused)) argc, char *argv[]) if (!hugetlb_setup_default(4)) ksft_print_msg("HugeTLB test will be skipped\n"); - ksft_set_plan(118); + ksft_set_plan(119); page_size = getpagesize(); hpage_size = read_pmd_pagesize(); @@ -1790,6 +1837,7 @@ int main(int __attribute__((unused)) argc, char *argv[]) /* 18. Unpopulated pte scan-path consistency */ unpopulated_scan_test(); + unpopulated_thp_scan_test(); close(pagemap_fd); ksft_finished(); From 3105ae628fb785d48b49256468be4f21a7b3cfc0 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Wed, 15 Jul 2026 09:03:58 +0000 Subject: [PATCH 435/501] mm/page_alloc: don't spin_trylock() in NMI on UP Patch series "mm/page_alloc: fixes for free_pages_nolock() on RT/UP". Pre-existing bugs found by Sashiko during review of this other series: https://lore.kernel.org/all/20260703-alloc-trylock-v5-0-c87b714e19d3@google.com/ I have not reproduced these bugs, and I suspect there is no real-world user that is affected by them. This patch (of 2): As noted in can_spin_trylock(), using this is unsafe in this context. commit 620b46ed6ae17 ("mm/page_alloc: return NULL early from alloc_frozen_pages_nolock() in NMI on UP") fixed this on the alloc side but missed the free side. Impact: If BPF programs using these features in NMI (probably tracing) are present on non-SMP builds this might crash the kernel and is probably exploitable by local attackers for privilege escalation. Link: https://lore.kernel.org/20260715-alloc-nolock-fixes-v1-0-fadc49952dda@google.com Link: https://lore.kernel.org/20260715-alloc-nolock-fixes-v1-1-fadc49952dda@google.com Fixes: 8c57b687e833 ("mm, bpf: Introduce free_pages_nolock()") Signed-off-by: Brendan Jackman Reported-by: sashiko-bot@kernel.org Closes: https://sashiko.dev/#/patchset/20260703-alloc-trylock-v5-0-c87b714e19d3%40google.com?part=18 Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Harry Yoo (Oracle) Cc: Brendan Jackman Cc: Alexei Starovoitov Cc: Johannes Weiner Cc: Michal Hocko Cc: Sebastian Andrzej Siewior Cc: Shakeel Butt Cc: Steven Rostedt Cc: Suren Baghdasaryan Cc: Zi Yan Cc: Signed-off-by: Andrew Morton --- mm/page_alloc.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 520ebd2fa40b..b3487c00971e 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2979,8 +2979,7 @@ static void __free_frozen_pages(struct page *page, unsigned int order, migratetype = MIGRATE_MOVABLE; } - if (unlikely((fpi_flags & FPI_TRYLOCK) && IS_ENABLED(CONFIG_PREEMPT_RT) - && (in_nmi() || in_hardirq()))) { + if (unlikely((fpi_flags & FPI_TRYLOCK) && !can_spin_trylock())) { add_page_to_zone_llist(zone, page, order); return; } From 52dcc33881e55189650404f833c0e6b7913d6c16 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Wed, 15 Jul 2026 09:03:59 +0000 Subject: [PATCH 436/501] mm/page_alloc: don't spin_trylock() when disallowed in free_one_page() __free_frozen_pages() checks can_spin_trylock() before entering into the main free_frozen_page_commit()/free_one_page() path, but before this it can get to free_one_page() via the !pcp_allowed_order() and MIGRATE_ISOLATE paths. The !pcp_allowed_order() path depends on usage by callers so might not be possible in practice. The MIGRATE_ISOLATE path probably means kernel crashes and privilege escalation if anyone ever did memory hotplug and BPF tracing on a PREEMPT_RT or !SMP build. Link: https://lore.kernel.org/20260715-alloc-nolock-fixes-v1-2-fadc49952dda@google.com Signed-off-by: Brendan Jackman Fixes: 8c57b687e833 ("mm, bpf: Introduce free_pages_nolock()") Reported-by: sashiko-bot@kernel.org Closes: https://sashiko.dev/#/patchset/20260710-spin-trylock-followup-v1-0-affb5fe5ed00%40google.com?part=2 Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Harry Yoo (Oracle) Cc: Brendan Jackman Cc: Alexei Starovoitov Cc: Johannes Weiner Cc: Michal Hocko Cc: Sebastian Andrzej Siewior Cc: Shakeel Butt Cc: Steven Rostedt Cc: Suren Baghdasaryan Cc: Zi Yan Cc: Signed-off-by: Andrew Morton --- mm/page_alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index b3487c00971e..891f687e622c 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1559,7 +1559,7 @@ static void free_one_page(struct zone *zone, struct page *page, unsigned long flags; if (unlikely(fpi_flags & FPI_TRYLOCK)) { - if (!spin_trylock_irqsave(&zone->lock, flags)) { + if (!can_spin_trylock() || !spin_trylock_irqsave(&zone->lock, flags)) { add_page_to_zone_llist(zone, page, order); return; } From 21be908d39e328b21111dff45c17ae9d58064396 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Wed, 15 Jul 2026 11:03:18 +0000 Subject: [PATCH 437/501] mm/page_alloc: rename FPI_TRYLOCK -> FPI_NOLOCK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch series "mm/page_alloc: couple of followups for recent cleanups", v3. This patch (of 4): As discussed in the linked patch, the there is some inconsistency between "trylock" and "nolock" nomenclature, let's align it. Since "nolock" is used in the public API it seems to have more mindshare so do that. The linked patch did this for the ALLOC_ flag but forgot about FPI_. Link: https://lore.kernel.org/20260715-spin-trylock-followup-v3-0-fc4d246f705d@google.com Link: https://lore.kernel.org/all/20260703-alloc-trylock-v5-1-c87b714e19d3@google.com/ Link: https://lore.kernel.org/20260715-spin-trylock-followup-v3-1-fc4d246f705d@google.com Signed-off-by: Brendan Jackman Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Brendan Jackman Cc: David Hildenbrand Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Michal Koutný Cc: Mike Rapoport Cc: Sebastian Andrzej Siewior Cc: Steven Rostedt Cc: Suren Baghdasaryan Cc: Tejun Heo Cc: Waiman Long Signed-off-by: Andrew Morton --- mm/page_alloc.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 891f687e622c..c971ac30d4f7 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -90,7 +90,7 @@ typedef int __bitwise fpi_t; #define FPI_TO_TAIL ((__force fpi_t)BIT(1)) /* Free the page without taking locks. Rely on trylock only. */ -#define FPI_TRYLOCK ((__force fpi_t)BIT(2)) +#define FPI_NOLOCK ((__force fpi_t)BIT(2)) /* free_pages_prepare() has already been called for page(s) being freed. */ #define FPI_PREPARED ((__force fpi_t)BIT(3)) @@ -1419,7 +1419,7 @@ static __always_inline bool __free_pages_prepare(struct page *page, page_table_check_free(page, order); pgalloc_tag_sub(page, 1 << order); - if (!PageHighMem(page) && !(fpi_flags & FPI_TRYLOCK)) { + if (!PageHighMem(page) && !(fpi_flags & FPI_NOLOCK)) { debug_check_no_locks_freed(page_address(page), PAGE_SIZE << order); debug_check_no_obj_freed(page_address(page), @@ -1558,7 +1558,7 @@ static void free_one_page(struct zone *zone, struct page *page, struct llist_head *llhead; unsigned long flags; - if (unlikely(fpi_flags & FPI_TRYLOCK)) { + if (unlikely(fpi_flags & FPI_NOLOCK)) { if (!can_spin_trylock() || !spin_trylock_irqsave(&zone->lock, flags)) { add_page_to_zone_llist(zone, page, order); return; @@ -1569,7 +1569,7 @@ static void free_one_page(struct zone *zone, struct page *page, /* The lock succeeded. Process deferred pages. */ llhead = &zone->trylock_free_pages; - if (unlikely(!llist_empty(llhead) && !(fpi_flags & FPI_TRYLOCK))) { + if (unlikely(!llist_empty(llhead) && !(fpi_flags & FPI_NOLOCK))) { struct llist_node *llnode; struct page *p, *tmp; @@ -2882,7 +2882,7 @@ static bool free_frozen_page_commit(struct zone *zone, if (pcp->free_count < (batch << CONFIG_PCP_BATCH_SCALE_MAX)) pcp->free_count += (1 << order); - if (unlikely(fpi_flags & FPI_TRYLOCK)) { + if (unlikely(fpi_flags & FPI_NOLOCK)) { /* * Do not attempt to take a zone lock. Let pcp->count get * over high mark temporarily. @@ -2979,7 +2979,7 @@ static void __free_frozen_pages(struct page *page, unsigned int order, migratetype = MIGRATE_MOVABLE; } - if (unlikely((fpi_flags & FPI_TRYLOCK) && !can_spin_trylock())) { + if (unlikely((fpi_flags & FPI_NOLOCK) && !can_spin_trylock())) { add_page_to_zone_llist(zone, page, order); return; } @@ -3001,7 +3001,7 @@ void free_frozen_pages(struct page *page, unsigned int order) void free_frozen_pages_nolock(struct page *page, unsigned int order) { - __free_frozen_pages(page, order, FPI_TRYLOCK); + __free_frozen_pages(page, order, FPI_NOLOCK); } /* @@ -5398,7 +5398,7 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, if (memcg_kmem_online() && (gfp & __GFP_ACCOUNT) && page && unlikely(__memcg_kmem_charge_page(page, gfp, order) != 0)) { __free_frozen_pages(page, order, - alloc_flags & ALLOC_NOLOCK ? FPI_TRYLOCK : 0); + alloc_flags & ALLOC_NOLOCK ? FPI_NOLOCK : 0); page = NULL; } @@ -5521,7 +5521,7 @@ EXPORT_SYMBOL(__free_pages); */ void free_pages_nolock(struct page *page, unsigned int order) { - ___free_pages(page, order, FPI_TRYLOCK); + ___free_pages(page, order, FPI_NOLOCK); } /** From 0cce423a1253c55a8b86fa9aaf82715d0a813dad Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Wed, 15 Jul 2026 11:03:19 +0000 Subject: [PATCH 438/501] cgroup/cpuset: update some comments about the page allocator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These comments describing the page allocator are out of date: - __alloc_pages() is no longer a public API and has no business being described outside of mm/. - The `wait` variable is gone. It may be out of date for other reasons too but this patch is just fixing the issues that stood out. To fix it: - Instead of referring to a specific function, instead to "the page allocator" - Completely drop out-of-date details of that function's internal behaviour, since they were irrelevant anyway. Link: https://lore.kernel.org/20260715-spin-trylock-followup-v3-2-fc4d246f705d@google.com Signed-off-by: Brendan Jackman Suggested-by: Zi Yan Link: https://lore.kernel.org/all/DJP11T5V7BDW.2FZZZ8R6LOY4I@nvidia.com/ Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Acked-by: Tejun Heo Cc: David Hildenbrand Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Michal Koutný Cc: Mike Rapoport Cc: Sebastian Andrzej Siewior Cc: Steven Rostedt Cc: Suren Baghdasaryan Cc: Waiman Long Cc: Brendan Jackman Signed-off-by: Andrew Morton --- kernel/cgroup/cpuset.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 92484b293689..ee4025b9707a 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -4198,7 +4198,7 @@ static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs) * nearest enclosing hardwalled ancestor cpuset. * * Scanning up parent cpusets requires callback_lock. The - * __alloc_pages() routine only calls here with __GFP_HARDWALL bit + * page allocator only calls here with __GFP_HARDWALL bit * _not_ set if it's a GFP_KERNEL allocation, and all nodes in the * current tasks mems_allowed came up empty on the first pass over * the zonelist. So only GFP_KERNEL allocations, if all nodes in the @@ -4211,11 +4211,8 @@ static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs) * come before the __GFP_HARDWALL check, otherwise a dying task * would be blocked on the fast path. * - * The second pass through get_page_from_freelist() doesn't even call - * here for GFP_ATOMIC calls. For those calls, the __alloc_pages() - * variable 'wait' is not set, and the bit ALLOC_CPUSET is not set - * in alloc_flags. That logic and the checks below have the combined - * affect that: + * The second pass through get_page_from_freelist() doesn't even call here for + * GFP_ATOMIC calls. That, and the checks below have the combined affect that: * in_interrupt - any node ok (current task context irrelevant) * GFP_ATOMIC - any node ok * tsk_is_oom_victim - any node ok @@ -4332,8 +4329,8 @@ void cpuset_nodes_allowed(struct cgroup *cgroup, nodemask_t *mask) * should not be possible for the following code to return an * offline node. But if it did, that would be ok, as this routine * is not returning the node where the allocation must be, only - * the node where the search should start. The zonelist passed to - * __alloc_pages() will include all nodes. If the slab allocator + * the node where the search should start. The zonelist used by + * the allocator will include all nodes. If the slab allocator * is passed an offline node, it will fall back to the local node. * See kmem_cache_alloc_node(). */ From d413d243c3dc8ef14428def3870f16f865fff45a Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Wed, 15 Jul 2026 11:03:20 +0000 Subject: [PATCH 439/501] mm/page_alloc: fixup alloc_pages_nolock_noprof() comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update the comment to reflect the recent change to allow flags in gfp_nolock. Link: https://lore.kernel.org/20260715-spin-trylock-followup-v3-3-fc4d246f705d@google.com Signed-off-by: Brendan Jackman Reported-by: sashiko-bot@kernel.org Link: https://sashiko.dev/#/patchset/20260703-alloc-trylock-v5-0-c87b714e19d3%40google.com?part=6 Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Brendan Jackman Cc: David Hildenbrand Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Michal Koutný Cc: Mike Rapoport Cc: Sebastian Andrzej Siewior Cc: Steven Rostedt Cc: Suren Baghdasaryan Cc: Tejun Heo Cc: Waiman Long Signed-off-by: Andrew Morton --- mm/page_alloc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index c971ac30d4f7..7ae536bb75f4 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -7982,7 +7982,8 @@ struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned } /** * alloc_pages_nolock - opportunistic reentrant allocation from any context - * @gfp_flags: GFP flags. Only __GFP_ACCOUNT allowed. + * @gfp_flags: GFP flags. Only __GFP_ACCOUNT, plus some flags that get set + * internally regardless (see %gfp_nolock) are allowed. * @nid: node to allocate from * @order: allocation order size * From e47b037e42084ba98f8598791cb1aaf84ddfd060 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Wed, 15 Jul 2026 11:03:21 +0000 Subject: [PATCH 440/501] mm/page_alloc: remove a couple of VM_BUG_ON()st MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit VM_BUG_ON() is out of favour and on the way to removal, since I recently touched alloc_pages_node_noprof() I am removing that invocation, and also removing the __folio_alloc_node_noprof() one for consistency. If this precondition is violated, the system will soon crash anyway. Link: https://lore.kernel.org/20260715-spin-trylock-followup-v3-4-fc4d246f705d@google.com Signed-off-by: Brendan Jackman Suggested-by: Zi Yan Link: https://lore.kernel.org/all/7F866265-3F2E-4765-B9D4-9AB898A9C4AC@nvidia.com/ Acked-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Brendan Jackman Cc: David Hildenbrand Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Michal Koutný Cc: Mike Rapoport Cc: Sebastian Andrzej Siewior Cc: Steven Rostedt Cc: Suren Baghdasaryan Cc: Tejun Heo Cc: Waiman Long Signed-off-by: Andrew Morton --- include/linux/gfp.h | 1 - mm/page_alloc.c | 1 - 2 files changed, 2 deletions(-) diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 4d57e9c0bf20..872bc53f32ec 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -255,7 +255,6 @@ static inline void warn_if_node_offline(int this_node, gfp_t gfp_mask) static inline struct folio *__folio_alloc_node_noprof(gfp_t gfp, unsigned int order, int nid) { - VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES); warn_if_node_offline(nid, gfp); return __folio_alloc_noprof(gfp, order, nid, NULL); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 7ae536bb75f4..f93a6bb9a872 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5426,7 +5426,6 @@ struct page *alloc_pages_node_noprof(int nid, gfp_t gfp_mask, unsigned int order if (nid == NUMA_NO_NODE) nid = numa_mem_id(); - VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES); warn_if_node_offline(nid, gfp_mask); return __alloc_pages_noprof(gfp_mask, order, nid, NULL, ALLOC_DEFAULT); From 14dad135d68b850f4506d7fc6872f1105dfde3cb Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Fri, 17 Jul 2026 18:27:09 +0100 Subject: [PATCH 441/501] mm/mseal: remove superfluous comments, fix confusion around mm Patch series "mm/mseal: further cleanups", v2. The mseal implementation is still rather confusing, so tighten things up a little. The only user of do_mseal() outside of the system call is the MMAP_PAGE_ZERO process personality - retain better control over how mseal is utilised by providing mseal_mmap_page_zero() for this instead. The comments are overly long and confusion, so cut them down so they're a lot clearer. Remove confusing mm_struct params (mseal can not be used on remote mm's) and wrap the actual system call logic into the system call declaration. This patch (of 3): Remove comment blocks that don't add value and eliminate any confusion about whether or not we permit mseal()'ing of remote mm's by not passing through an mm parameter but rather referencing current->mm in each function. Also while we're here, avoid an ugly goto by using an else branch, and move local parameters declarations into reverse xmas tree order. No functional change intended. Link: https://lore.kernel.org/20260717-mseal-fixups-v2-0-0daa0014b813@kernel.org Link: https://lore.kernel.org/20260717-mseal-fixups-v2-1-0daa0014b813@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Acked-by: David Hildenbrand (Arm) Reviewed-by: Pedro Falcato Cc: Al Viro Cc: Christian Brauner Cc: Jan Kara Cc: Jann Horn Cc: Kees Cook Cc: Liam R. Howlett Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/mseal.c | 53 +++++++++++------------------------------------------ 1 file changed, 11 insertions(+), 42 deletions(-) diff --git a/mm/mseal.c b/mm/mseal.c index 9781647483d1..430a252a6da4 100644 --- a/mm/mseal.c +++ b/mm/mseal.c @@ -16,32 +16,11 @@ #include #include "internal.h" -/* - * mseal() disallows an input range which contain unmapped ranges (VMA holes). - * - * It disallows unmapped regions from start to end whether they exist at the - * start, in the middle, or at the end of the range, or any combination thereof. - * - * This is because after sealing a range, there's nothing to stop memory mapping - * of ranges in the remaining gaps later, meaning that the user might then - * wrongly consider the entirety of the mseal()'d range to be sealed when it - * in fact isn't. - */ - -/* - * Does the [start, end) range contain any unmapped memory? - * - * We ensure that: - * - start is part of a valid VMA. - * - end is part of a valid VMA. - * - no gap (unallocated memory) exists between start and end. - */ -static bool range_contains_unmapped(struct mm_struct *mm, - unsigned long start, unsigned long end) +static bool range_contains_unmapped(unsigned long start, unsigned long end) { - struct vm_area_struct *vma; - unsigned long prev_end = start; VMA_ITERATOR(vmi, current->mm, start); + unsigned long prev_end = start; + struct vm_area_struct *vma; for_each_vma_range(vmi, vma, end) { if (vma->vm_start > prev_end) @@ -53,11 +32,10 @@ static bool range_contains_unmapped(struct mm_struct *mm, return prev_end < end; } -static int mseal_apply(struct mm_struct *mm, - unsigned long start, unsigned long end) +static int mseal_apply(unsigned long start, unsigned long end) { + VMA_ITERATOR(vmi, current->mm, start); struct vm_area_struct *vma, *prev; - VMA_ITERATOR(vmi, mm, start); /* We know there are no gaps so this will be non-NULL. */ vma = vma_iter_load(&vmi); @@ -142,10 +120,10 @@ static int mseal_apply(struct mm_struct *mm, */ int do_mseal(unsigned long start, size_t len_in, unsigned long flags) { - size_t len; - int ret = 0; - unsigned long end; struct mm_struct *mm = current->mm; + unsigned long end; + int ret = 0; + size_t len; /* Verify flags not set. */ if (flags) @@ -170,20 +148,11 @@ int do_mseal(unsigned long start, size_t len_in, unsigned long flags) if (mmap_write_lock_killable(mm)) return -EINTR; - if (range_contains_unmapped(mm, start, end)) { + if (range_contains_unmapped(start, end)) ret = -ENOMEM; - goto out; - } + else + ret = mseal_apply(start, end); - /* - * Second pass, this should success, unless there are errors - * from vma_modify_flags, e.g. merge/split error, or process - * reaching the max supported VMAs, however, those cases shall - * be rare. - */ - ret = mseal_apply(mm, start, end); - -out: mmap_write_unlock(mm); return ret; } From 4c1c2d2418c5395bf78b02d885112923e49e7f97 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Fri, 17 Jul 2026 18:27:10 +0100 Subject: [PATCH 442/501] mm/mseal: limit scope of mseal address zero to address zero Commit 44f65d900698 ("binfmt_elf: mseal address zero") unconditionally provided do_mseal() to any internal kernel caller in order to address a corner case slated for possible removal. It also incorrectly attempts to mseal without checking to see whether the mapping even succeeded. Restrict the scope to the corner case by providing mseal_mmap_page_zero() which asserts the MMAP_PAGE_ZERO personality. Avoid unnecessary checks in the start, end range by abstracting the actual mseal()'ing to mseal_range() and have mseal_mmap_page_zero() call that instead. Also only try to seal the VMA if we mapped the VMA. This isn't strictly necessary as the operation would error out anyway, but it's useless work and could be problematic if me make future changes to mseal semantics. Link: https://lore.kernel.org/20260717-mseal-fixups-v2-2-0daa0014b813@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Acked-by: David Hildenbrand (Arm) Cc: Al Viro Cc: Christian Brauner Cc: Jan Kara Cc: Jann Horn Cc: Kees Cook Cc: Liam R. Howlett Cc: Michal Hocko Cc: Mike Rapoport Cc: Pedro Falcato Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- fs/binfmt_elf.c | 7 ++----- include/linux/mm.h | 8 ++------ mm/mseal.c | 49 +++++++++++++++++++++++++++++++++------------- 3 files changed, 39 insertions(+), 25 deletions(-) diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index 16a56b6b3f6c..e3131a311995 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -1353,11 +1353,8 @@ static int load_elf_binary(struct linux_binprm *bprm) emulate the SVr4 behavior. Sigh. */ error = vm_mmap(NULL, 0, PAGE_SIZE, PROT_READ | PROT_EXEC, MAP_FIXED | MAP_PRIVATE, 0); - - retval = do_mseal(0, PAGE_SIZE, 0); - if (retval) - pr_warn_ratelimited("pid=%d, couldn't seal address 0, ret=%d.\n", - task_pid_nr(current), retval); + if (!error) + mseal_mmap_page_zero(); } regs = current_pt_regs(); diff --git a/include/linux/mm.h b/include/linux/mm.h index 550fb92957d1..87feaa5a2b78 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -5291,13 +5291,9 @@ int reserve_mem_find_by_name(const char *name, phys_addr_t *start, phys_addr_t * int reserve_mem_release_by_name(const char *name); #ifdef CONFIG_64BIT -int do_mseal(unsigned long start, size_t len_in, unsigned long flags); +void mseal_mmap_page_zero(void); #else -static inline int do_mseal(unsigned long start, size_t len_in, unsigned long flags) -{ - /* noop on 32 bit */ - return 0; -} +static inline void mseal_mmap_page_zero(void) {} #endif /* diff --git a/mm/mseal.c b/mm/mseal.c index 430a252a6da4..2a516da694c6 100644 --- a/mm/mseal.c +++ b/mm/mseal.c @@ -32,7 +32,7 @@ static bool range_contains_unmapped(unsigned long start, unsigned long end) return prev_end < end; } -static int mseal_apply(unsigned long start, unsigned long end) +static int __mseal_range(unsigned long start, unsigned long end) { VMA_ITERATOR(vmi, current->mm, start); struct vm_area_struct *vma, *prev; @@ -66,6 +66,38 @@ static int mseal_apply(unsigned long start, unsigned long end) return 0; } +static int mseal_range(unsigned long start, unsigned long end) +{ + int err; + + err = mmap_write_lock_killable(current->mm); + if (err) + return err; + if (range_contains_unmapped(start, end)) + err = -ENOMEM; + else + err = __mseal_range(start, end); + mmap_write_unlock(current->mm); + return err; +} + +/** + * mseal_mmap_page_zero() - If the MMAP_PAGE_ZERO personality is set, mseal() + * the page mapped at address zero. + */ +void mseal_mmap_page_zero(void) +{ + int err; + + if (WARN_ON_ONCE(!(current->personality & MMAP_PAGE_ZERO))) + return; + + err = mseal_range(0, PAGE_SIZE); + if (err) + pr_warn_ratelimited("pid=%d, couldn't seal address 0, ret=%d.\n", + task_pid_nr(current), err); +} + /* * mseal(2) seals the VM's meta data from * selected syscalls. @@ -118,11 +150,9 @@ static int mseal_apply(unsigned long start, unsigned long end) * * unseal() is not supported. */ -int do_mseal(unsigned long start, size_t len_in, unsigned long flags) +static int do_mseal(unsigned long start, size_t len_in, unsigned long flags) { - struct mm_struct *mm = current->mm; unsigned long end; - int ret = 0; size_t len; /* Verify flags not set. */ @@ -145,16 +175,7 @@ int do_mseal(unsigned long start, size_t len_in, unsigned long flags) if (end == start) return 0; - if (mmap_write_lock_killable(mm)) - return -EINTR; - - if (range_contains_unmapped(start, end)) - ret = -ENOMEM; - else - ret = mseal_apply(start, end); - - mmap_write_unlock(mm); - return ret; + return mseal_range(start, end); } SYSCALL_DEFINE3(mseal, unsigned long, start, size_t, len, unsigned long, From 1b300f679a4c27127effed80ede6a99fd1862dd8 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Fri, 17 Jul 2026 18:27:11 +0100 Subject: [PATCH 443/501] mm/mseal: remove further superfluous comments, do_mseal() There's no need to abstract do_mseal() any longer so put the system call implementation in the system call declaration. The comment around do_mseal() is strangely formatted, overly long and adds a lot of superfluous information that the code already provides, so boil it down to the essentials. Link: https://lore.kernel.org/20260717-mseal-fixups-v2-3-0daa0014b813@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Acked-by: David Hildenbrand (Arm) Reviewed-by: Pedro Falcato Cc: Al Viro Cc: Christian Brauner Cc: Jan Kara Cc: Jann Horn Cc: Kees Cook Cc: Liam R. Howlett Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/mseal.c | 74 ++++++++++++------------------------------------------ 1 file changed, 16 insertions(+), 58 deletions(-) diff --git a/mm/mseal.c b/mm/mseal.c index 2a516da694c6..7a8ac66dc215 100644 --- a/mm/mseal.c +++ b/mm/mseal.c @@ -99,61 +99,25 @@ void mseal_mmap_page_zero(void) } /* - * mseal(2) seals the VM's meta data from - * selected syscalls. + * Seal VMAs in the specified input range to prevent an attacker replacing what + * is mapped in the range with something else. * - * addr/len: VM address range. + * Disallows: + * - VMA unmapping, remapping or shrinking. + * - Overwriting the VMA with another one via mmap(), mremap() or similar. + * - Alteration of properties via mprotect()/pkey_mprotect(). + * - Destructive madvise() behaviours (like MADV_DONTNEED) on anonymous read-only + * ranges. * - * The address range by addr/len must meet: - * start (addr) must be in a valid VMA. - * end (addr + len) must be in a valid VMA. - * no gap (unallocated memory) between start and end. - * start (addr) must be page aligned. + * Since unmapped ranges can be mapped at any time, the input range must span + * mapped ranges only. * - * len: len will be page aligned implicitly. - * - * Below VMA operations are blocked after sealing. - * 1> Unmapping, moving to another location, and shrinking - * the size, via munmap() and mremap(), can leave an empty - * space, therefore can be replaced with a VMA with a new - * set of attributes. - * 2> Moving or expanding a different vma into the current location, - * via mremap(). - * 3> Modifying a VMA via mmap(MAP_FIXED). - * 4> Size expansion, via mremap(), does not appear to pose any - * specific risks to sealed VMAs. It is included anyway because - * the use case is unclear. In any case, users can rely on - * merging to expand a sealed VMA. - * 5> mprotect and pkey_mprotect. - * 6> Some destructive madvice() behavior (e.g. MADV_DONTNEED) - * for anonymous memory, when users don't have write permission to the - * memory. Those behaviors can alter region contents by discarding pages, - * effectively a memset(0) for anonymous memory. - * - * flags: reserved. - * - * return values: - * zero: success. - * -EINVAL: - * invalid input flags. - * start address is not page aligned. - * Address range (start + len) overflow. - * -ENOMEM: - * addr is not a valid address (not allocated). - * end (start + len) is not a valid address. - * a gap (unallocated memory) between start and end. - * -EPERM: - * - In 32 bit architecture, sealing is not supported. - * Note: - * user can call mseal(2) multiple times, adding a seal on an - * already sealed memory is a no-action (no error). - * - * unseal() is not supported. + * The flags parameter is currently reserved. */ -static int do_mseal(unsigned long start, size_t len_in, unsigned long flags) +SYSCALL_DEFINE3(mseal, unsigned long, start, size_t, len, unsigned long, flags) { + size_t len_aligned; unsigned long end; - size_t len; /* Verify flags not set. */ if (flags) @@ -163,12 +127,12 @@ static int do_mseal(unsigned long start, size_t len_in, unsigned long flags) if (!PAGE_ALIGNED(start)) return -EINVAL; - len = PAGE_ALIGN(len_in); + len_aligned = PAGE_ALIGN(len); /* Check to see whether len was rounded up from small -ve to zero. */ - if (len_in && !len) + if (len && !len_aligned) return -EINVAL; - end = start + len; + end = start + len_aligned; if (end < start) return -EINVAL; @@ -177,9 +141,3 @@ static int do_mseal(unsigned long start, size_t len_in, unsigned long flags) return mseal_range(start, end); } - -SYSCALL_DEFINE3(mseal, unsigned long, start, size_t, len, unsigned long, - flags) -{ - return do_mseal(start, len, flags); -} From e474ac24a79211f83e214d856630829f77363459 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Wed, 15 Jul 2026 21:12:58 +0800 Subject: [PATCH 444/501] mm/mseal: fix mseal documentation for 32-bit kernels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mseal.o is built only for 64-bit kernels, so 32-bit kernels fall back to sys_ni_syscall() and return -ENOSYS rather than -EPERM. Drop architecture description in mseal.rst, since the arch feature doc has the latest state of mseal for each architecture. Fix the CONFIG_MSEAL_SYSTEM_MAPPINGS typo in init/Kconfig. Link: https://lore.kernel.org/20260715131258.55499-1-leon.hwang@linux.dev Signed-off-by: Leon Hwang Acked-by: Lance Yang Cc: Alice Ryhl Cc: Anand Moon Cc: Doug Anderson Cc: Gary Guo Cc: Jann Horn Cc: Jonathan Corbet Cc: Leon Hwang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Miguel Ojeda Cc: Nathan Chancellor Cc: Pedro Falcato Cc: Peter Zijlstra Cc: Randy Dunlap Cc: Thomas Weißschuh Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- Documentation/userspace-api/mseal.rst | 14 ++++++-------- init/Kconfig | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Documentation/userspace-api/mseal.rst b/Documentation/userspace-api/mseal.rst index ea9b11a0bd89..c039d782f9f5 100644 --- a/Documentation/userspace-api/mseal.rst +++ b/Documentation/userspace-api/mseal.rst @@ -50,8 +50,8 @@ mseal syscall signature * The start address (``addr``) is not allocated. * The end address (``addr`` + ``len``) is not allocated. * A gap (unallocated memory) between start and end address. - - **-EPERM**: - * sealing is supported only on 64-bit CPUs, 32-bit is not supported. + - **-ENOSYS**: + * The kernel does not implement ``mseal()``. **Note about error return**: - For above error cases, users can expect the given memory range is @@ -62,7 +62,8 @@ mseal syscall signature memory range could happen. However, those cases should be rare. **Architecture support**: - mseal only works on 64-bit CPUs, not 32-bit CPUs. + mseal is built only for 64-bit kernels. 32-bit kernels return + ``-ENOSYS``. **Idempotent**: users can call mseal multiple times. mseal on an already sealed memory @@ -131,11 +132,11 @@ Use cases - Chrome browser: protect some security sensitive data structures. - System mappings: - The system mappings are created by the kernel and includes vdso, vvar, + The system mappings are created by the kernel and include vdso, vvar, vvar_vclock, vectors (arm compat-mode), sigpage (arm compat-mode), uprobes. Those system mappings are readonly only or execute only, memory sealing can - protect them from ever changing to writable or unmmap/remapped as different + protect them from ever changing to writable or unmapped/remapped as different attributes. This is useful to mitigate memory corruption issues where a corrupted pointer is passed to a memory management system. @@ -143,9 +144,6 @@ Use cases the CONFIG_MSEAL_SYSTEM_MAPPINGS seals all system mappings of this architecture. - The following architectures currently support this feature: x86-64, arm64, - loongarch and s390. - WARNING: This feature breaks programs which rely on relocating or unmapping system mappings. Known broken software at the time of writing includes CHECKPOINT_RESTORE, UML, gVisor, rr. Therefore diff --git a/init/Kconfig b/init/Kconfig index c24f345f794b..13cb6fdd9c37 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -2123,7 +2123,7 @@ config ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS from a kernel perspective. After the architecture enables this, a distribution can set - CONFIG_MSEAL_SYSTEM_MAPPING to manage access to the feature. + CONFIG_MSEAL_SYSTEM_MAPPINGS to manage access to the feature. For complete descriptions of memory sealing, please see Documentation/userspace-api/mseal.rst From e26f7a91de5ca806a3413ef7dbdd3343a6aca8b7 Mon Sep 17 00:00:00 2001 From: Ridong Chen Date: Fri, 24 Jul 2026 11:34:32 +0800 Subject: [PATCH 445/501] mm/vmscan: fix anon-only reclaim evicting file pages when swappiness=max Patch series "mm/vmscan: fix swappiness=max and clean up per-node proactive reclaim", v4. Fixes and one cleanup. Patch 1 fixes "swappiness=max": the anon-only test in get_scan_count() sat after the "cannot reclaim anon" check, so when no anon was reclaimable the request fell back to SCAN_FILE and evicted page cache instead. Patch 2 fixes reclaim_store() collapsing every error into -EAGAIN, so callers can no longer tell an invalid argument from a busy interface; propagate the real error code, matching the memcg path. Patch 3 drops the now-unused gfp_mask parameter from __node_reclaim(). Patch 4 fixes the same "swappiness=max" issue for MGLRU. This patch (of 4): As Qi mentioned [1], when swappiness=max (SWAPPINESS_ANON_ONLY) is set, the reclaim logic is expected to reclaim anonymous pages exclusively. However, due to the current ordering of checks in get_scan_count(), file pages may still be evicted if can_reclaim_anon_pages() returns false, which contradicts the semantics of SWAPPINESS_ANON_ONLY. Reproducer in a cgroup holding 64M of file cache, with no swap configured: Before (file cache is wrongly evicted): # cat memory.stat anon 196608 file 67178496 pgscan_proactive 0 # echo "64M swappiness=max" > memory.reclaim # cat memory.stat anon 208896 file 4096 <- page cache evicted pgsteal_proactive 16400 pgscan_proactive 16400 After (file cache is left intact): # cat memory.stat anon 200704 file 67178496 pgscan_proactive 0 # echo "64M swappiness=max" > memory.reclaim -bash: echo: write error: Resource temporarily unavailable # cat memory.stat anon 208896 file 67178496 <- page cache untouched pgsteal_proactive 0 pgscan_proactive 0 Fix this by bailing out early when SWAPPINESS_ANON_ONLY is set and no anonymous pages are reclaimable, before falling back to file reclaim. Link: https://lore.kernel.org/20260724033435.2573323-1-ridong.chen@linux.dev Link: https://lore.kernel.org/20260724033435.2573323-2-ridong.chen@linux.dev Link: https://lore.kernel.org/cgroups/7ddf3eee-5fe2-45f7-8614-c8936a039e04@linux.dev/ [1] Fixes: 68a1436bde00 ("mm: add swappiness=max arg to memory.reclaim for only anon reclaim") Signed-off-by: Ridong Chen Suggested-by: Qi Zheng Acked-by: Shakeel Butt Acked-by: Johannes Weiner Reviewed-by: Muchun Song Reviewed-by: Qi Zheng Reviewed-by: Barry Song Cc: Axel Rasmussen Cc: David Hildenbrand Cc: Davidlohr Bueso Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Roman Gushchin Cc: Wei Xu Cc: Yuanchu Xie Cc: Zhongkun He Cc: Baolin Wang Signed-off-by: Andrew Morton --- mm/vmscan.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 006829efceca..af0c42daa77d 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -2494,6 +2494,23 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, enum scan_balance scan_balance; enum lru_list lru; + /* + * Proactive reclaim initiated by userspace for anonymous memory only. + * SWAPPINESS_ANON_ONLY is set only on the proactive reclaim path, so + * warn if it shows up elsewhere. When anon cannot be reclaimed (e.g. + * no swap), bail out instead of falling back to evicting file pages, + * which would violate the anon-only semantics. + */ + if (swappiness == SWAPPINESS_ANON_ONLY) { + WARN_ON_ONCE(!sc->proactive); + if (!can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) { + memset(nr, 0, sizeof(*nr) * NR_LRU_LISTS); + return; + } + scan_balance = SCAN_ANON; + goto out; + } + /* If we have no swap space, do not bother scanning anon folios. */ if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) { scan_balance = SCAN_FILE; @@ -2512,13 +2529,6 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, goto out; } - /* Proactive reclaim initiated by userspace for anonymous memory only */ - if (swappiness == SWAPPINESS_ANON_ONLY) { - WARN_ON_ONCE(!sc->proactive); - scan_balance = SCAN_ANON; - goto out; - } - /* * Do not apply any pressure balancing cleverness when the * system is close to OOM, scan both anon and file equally From b27caeb46c9a8a0a023f90abe38fa2738856dc88 Mon Sep 17 00:00:00 2001 From: Ridong Chen Date: Fri, 24 Jul 2026 11:34:33 +0800 Subject: [PATCH 446/501] mm: vmscan: propagate real error code from per-node proactive reclaim It has been observed that per-node proactive reclaim always returns -EAGAIN when any error occurs. As discussed in the mailing list [1], the interface should distinguish between cases where no reclaimable memory is left and where another entity is concurrently using the same interface. Propagate the real error code, consistent with how memcg proactive reclaim handles errors. Link: https://lore.kernel.org/20260724033435.2573323-3-ridong.chen@linux.dev Link: https://lore.kernel.org/all/20250717235604.2atyx2aobwowpge3@offworld/T/#m3514718be82a31b05726a49da9b61fbfc69a589e [1] Fixes: b980077899ea ("mm: introduce per-node proactive reclaim interface") Signed-off-by: Ridong Chen Reviewed-by: Muchun Song Acked-by: Johannes Weiner Acked-by: Shakeel Butt Reviewed-by: Qi Zheng Reviewed-by: Barry Song Cc: Axel Rasmussen Cc: David Hildenbrand Cc: Davidlohr Bueso Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Roman Gushchin Cc: Wei Xu Cc: Yuanchu Xie Cc: Zhongkun He Cc: Baolin Wang Signed-off-by: Andrew Morton --- mm/vmscan.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index af0c42daa77d..b93bca48359a 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -8026,7 +8026,7 @@ static ssize_t reclaim_store(struct device *dev, int ret, nid = dev->id; ret = user_proactive_reclaim((char *)buf, NULL, NODE_DATA(nid)); - return ret ? -EAGAIN : count; + return ret ? ret : count; } static DEVICE_ATTR_WO(reclaim); From b33f6e98b36b6d83f92ba788b986dac8ba7fc653 Mon Sep 17 00:00:00 2001 From: Ridong Chen Date: Fri, 24 Jul 2026 11:34:34 +0800 Subject: [PATCH 447/501] mm: vmscan: drop unused gfp_mask parameter from __node_reclaim() Commit 57972c78e678 ("mm/vmscan: make __node_reclaim() more generic") moved the scan_control construction out to the callers and passed the struct in by pointer. After that change every use of the gfp mask inside __node_reclaim() goes through sc->gfp_mask, leaving the gfp_mask parameter unused. Just remove the dead parameter and update the callers accordingly. No functional change. Link: https://lore.kernel.org/20260724033435.2573323-4-ridong.chen@linux.dev Signed-off-by: Ridong Chen Acked-by: Shakeel Butt Acked-by: Johannes Weiner Reviewed-by: Muchun Song Reviewed-by: Barry Song Reviewed-by: Qi Zheng Cc: Axel Rasmussen Cc: David Hildenbrand Cc: Davidlohr Bueso Cc: Kairui Song Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Roman Gushchin Cc: Wei Xu Cc: Yuanchu Xie Cc: Zhongkun He Cc: Baolin Wang Signed-off-by: Andrew Morton --- mm/vmscan.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index b93bca48359a..b992f4e2e698 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -7740,7 +7740,7 @@ static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat) /* * Try to free up some pages from this node through reclaim. */ -static unsigned long __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, +static unsigned long __node_reclaim(struct pglist_data *pgdat, unsigned long nr_pages, struct scan_control *sc) { @@ -7832,7 +7832,7 @@ unsigned long node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned i if (test_and_set_bit_lock(PGDAT_RECLAIM_LOCKED, &pgdat->flags)) return 0; - ret = __node_reclaim(pgdat, gfp_mask, nr_pages, &sc); + ret = __node_reclaim(pgdat, nr_pages, &sc); clear_bit_unlock(PGDAT_RECLAIM_LOCKED, &pgdat->flags); if (ret >= nr_pages) @@ -7845,7 +7845,7 @@ unsigned long node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned i #else -static unsigned long __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, +static unsigned long __node_reclaim(struct pglist_data *pgdat, unsigned long nr_pages, struct scan_control *sc) { @@ -7958,8 +7958,7 @@ int user_proactive_reclaim(char *buf, &pgdat->flags)) return -EBUSY; - reclaimed = __node_reclaim(pgdat, gfp_mask, - batch_size, &sc); + reclaimed = __node_reclaim(pgdat, batch_size, &sc); clear_bit_unlock(PGDAT_RECLAIM_LOCKED, &pgdat->flags); } From 430e4cdcc6002db49814780e105b8e534b614fcc Mon Sep 17 00:00:00 2001 From: Ridong Chen Date: Fri, 24 Jul 2026 11:34:35 +0800 Subject: [PATCH 448/501] mm/mglru: fix anon-only reclaim evicting file pages when swappiness=max The previous patch fixed this issue for the traditional LRU. The same problem exists in MGLRU [1]: when swappiness=max (SWAPPINESS_ANON_ONLY) is set, reclaim is expected to evict anonymous pages exclusively, but file pages can still be reclaimed when anonymous pages cannot be reclaimed (e.g. no swap and no demotion target). Fix it the same way as the traditional LRU: keep returning SWAPPINESS_ANON_ONLY in get_swappiness(), and return 0 from get_nr_to_scan() when SWAPPINESS_ANON_ONLY is set but anon pages cannot be reclaimed. Since get_nr_to_scan() decides how much MGLRU scans, returning 0 skips the scan entirely and avoids the useless scan work when there is nothing eligible to reclaim. The test result: Before fix: # cat /sys/kernel/mm/lru_gen/enabled 0x0007 # cat memory.stat anon 204800 file 67108864 ... pgsteal_proactive 0 pgscan_proactive 0 # echo "64M swappiness=max" > memory.reclaim # cat memory.stat anon 208896 file 0 ... pgsteal_proactive 16384 pgscan_proactive 16384 After fix: # cat memory.stat anon 188416 file 67215360 kernel 1970176 ... pgsteal_proactive 0 pgscan_proactive 0 # echo "64M swappiness=max" > memory.reclaim -bash: echo: write error: Resource temporarily unavailable # cat memory.stat anon 204800 file 67215360 ... pgsteal_proactive 0 pgscan_proactive 0 Link: https://lore.kernel.org/20260724033435.2573323-5-ridong.chen@linux.dev Link: https://sashiko.dev/#/patchset/20260717113300.214717-1-ridong.chen@linux.dev [1] Fixes: 68a1436bde00 ("mm: add swappiness=max arg to memory.reclaim for only anon reclaim") Signed-off-by: Ridong Chen Reviewed-by: Kairui Song Reviewed-by: Barry Song Reviewed-by: Baolin Wang Cc: Axel Rasmussen Cc: David Hildenbrand Cc: Davidlohr Bueso Cc: Johannes Weiner Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Qi Zheng Cc: Roman Gushchin Cc: Shakeel Butt Cc: Wei Xu Cc: Yuanchu Xie Cc: Zhongkun He Signed-off-by: Andrew Morton --- mm/vmscan.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index b992f4e2e698..bc324e37c5f1 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -2701,6 +2701,10 @@ static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc) { struct mem_cgroup *memcg = lruvec_memcg(lruvec); struct pglist_data *pgdat = lruvec_pgdat(lruvec); + int swappiness = sc_swappiness(sc, memcg); + + if (swappiness == SWAPPINESS_ANON_ONLY) + return swappiness; if (!sc->may_swap) return 0; @@ -2709,7 +2713,7 @@ static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc) mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH) return 0; - return sc_swappiness(sc, memcg); + return swappiness; } static int get_nr_gens(struct lruvec *lruvec, int type) @@ -4911,6 +4915,20 @@ static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, struct mem_cgroup *memcg, int swappiness) { unsigned long nr_to_scan, evictable; + struct pglist_data *pgdat = lruvec_pgdat(lruvec); + + /* + * Proactive reclaim initiated by userspace for anonymous memory only. + * SWAPPINESS_ANON_ONLY is set only on the proactive reclaim path, so + * warn if it shows up elsewhere. When anon cannot be reclaimed (e.g. + * no swap), return 0 to skip the scan entirely, avoiding useless scan + * work when there is nothing eligible to reclaim. + */ + if (swappiness == SWAPPINESS_ANON_ONLY) { + WARN_ON_ONCE(!sc->proactive); + if (!can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) + return 0; + } evictable = lruvec_evictable_size(lruvec, swappiness); From 1605ce99afac80cd6080c821a664921c19eb3e00 Mon Sep 17 00:00:00 2001 From: Hongfu Li Date: Fri, 17 Jul 2026 15:02:51 +0800 Subject: [PATCH 449/501] selftests/mm: fix BUG_ON checking wrong variable in mremap_dontunmap In mremap_dontunmap_partial_mapping_overwrite(), mremap() stores its returned remapped address into remapped_mapping, while the BUG_ON incorrectly checks dest_mapping instead. If mremap() fails, dest_mapping still holds a valid pointer obtained from the prior mmap() call, making this BUG_ON never trigger at all. Correct the check to validate the actual mremap() return value stored in remapped_mapping. Link: https://lore.kernel.org/20260717070251.73212-1-hongfu.li@linux.dev Signed-off-by: Hongfu Li Reviewed-by: SJ Park Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/mremap_dontunmap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/mm/mremap_dontunmap.c b/tools/testing/selftests/mm/mremap_dontunmap.c index a4f75d836733..96ba537facf7 100644 --- a/tools/testing/selftests/mm/mremap_dontunmap.c +++ b/tools/testing/selftests/mm/mremap_dontunmap.c @@ -313,7 +313,7 @@ static void mremap_dontunmap_partial_mapping_overwrite(void) mremap(source_mapping, 5 * page_size, 5 * page_size, MREMAP_DONTUNMAP | MREMAP_MAYMOVE | MREMAP_FIXED, dest_mapping); - BUG_ON(dest_mapping == MAP_FAILED, "mremap"); + BUG_ON(remapped_mapping == MAP_FAILED, "mremap"); BUG_ON(dest_mapping != remapped_mapping, "expected to remap to dest_mapping"); BUG_ON(check_region_contains_byte(source_mapping, 5 * page_size, 0) != From 8a2081d845ac96b9426816d92cc21128cf64d823 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Fri, 17 Jul 2026 17:13:42 +0800 Subject: [PATCH 450/501] mm: introduce pud_is_huge() helper Patch series "mm: mincore: misc cleanups", v3. This series cleans up and simplifies the mincore. Most importantly, it removes the historical special behavior that always reports VM_PFNMAP pages as non-resident. This patch (of 6): Introduce the pud_is_huge() helper function to check whether a PUD is a huge PUD or a software leaf entry. Note that, unlike PMD, PUD currently does not support transparent migration entries or device private entries. However, the function still includes `!pud_none()` branches to accommodate potential future implementations. Link: https://lore.kernel.org/20260717091347.1144789-1-wangkefeng.wang@huawei.com Link: https://lore.kernel.org/20260717091347.1144789-2-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Acked-by: David Hildenbrand (Arm) Cc: Jann Horn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Pedro Falcato Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/huge_mm.h | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 9e43e138c0b7..98bf1766b57f 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -472,6 +472,24 @@ void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, unsigned long address); +/** + * pud_is_huge() - Is this PUD either a huge PUD entry or a software leaf entry? + * @pud: The PUD to check. + * + * This is similar to pmd_is_huge(), but it checks at the PUD level. + * + * Returns: true if this PUD is huge, false otherwise. + */ +static inline bool pud_is_huge(pud_t pud) +{ + if (pud_present(pud)) + return pud_trans_huge(pud); + else if (!pud_none(pud)) + return true; + + return false; +} + #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, pud_t *pudp, unsigned long addr, pgprot_t newprot, @@ -798,6 +816,11 @@ static inline bool pmd_is_huge(pmd_t pmd) { return false; } + +static inline bool pud_is_huge(pud_t pud) +{ + return false; +} #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ static inline bool is_pmd_order(unsigned int order) From 006943a9b437b2d4bd94f895ea9b69fe24b8d6da Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Fri, 17 Jul 2026 17:13:43 +0800 Subject: [PATCH 451/501] mm: mincore: remove special handling for VM_PFNMAP As David pointed out, "it's hard to believe that someone depends on pages in VM_PFNMAP to *not* be present", so remove the historical behavior that always reports VM_PFNMAP pages as non-resident. Adding mincore_pud_range() allows the page table walker to traverse the full page table hierarchy without splitting a huge PUD, enabling much more precise identification of resident pages in PFNMAP VMAs. [wangkefeng.wang@huawei.com: address David's comments] Link: https://lore.kernel.org/20260720124151.1483820-1-wangkefeng.wang@huawei.com Link: https://lore.kernel.org/linux-mm/0e619d71-1c3d-4534-8376-2982c7348c31@kernel.org/ Link: https://lore.kernel.org/20260717091347.1144789-3-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Suggested-by: David Hildenbrand (Arm) Acked-by: David Hildenbrand (Arm) Cc: Jann Horn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Pedro Falcato Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/mincore.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/mm/mincore.c b/mm/mincore.c index 53b982803771..d9d4381d7ed4 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -160,6 +160,20 @@ static int mincore_unmapped_range(unsigned long addr, unsigned long end, return 0; } +static int mincore_pud_entry(pud_t *pudp, unsigned long addr, unsigned long end, + struct mm_walk *walk) +{ + if (pud_is_huge(pudp_get(pudp))) { + const unsigned long nr = (end - addr) >> PAGE_SHIFT; + + memset(walk->private, 1, nr); + walk->private += nr; + walk->action = ACTION_CONTINUE; + } + + return 0; +} + static int mincore_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, struct mm_walk *walk) { @@ -232,6 +246,7 @@ static inline bool can_do_mincore(struct vm_area_struct *vma) } static const struct mm_walk_ops mincore_walk_ops = { + .pud_entry = mincore_pud_entry, .pmd_entry = mincore_pte_range, .pte_hole = mincore_unmapped_range, .hugetlb_entry = mincore_hugetlb, @@ -259,14 +274,6 @@ static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *v return pages; } - /* - * mincore historically reports PFNMAP mappings as non-resident. - */ - if (vma->vm_flags & VM_PFNMAP) { - __mincore_unmapped_range(addr, end, vma, vec); - return (end - addr) >> PAGE_SHIFT; - } - err = walk_page_range_vma(vma, addr, end, &mincore_walk_ops, vec); if (err < 0) return err; From 7625cb56733bfd470e9fae69a1b8b78aa8b1d3eb Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Fri, 17 Jul 2026 17:13:44 +0800 Subject: [PATCH 452/501] mm: mincore: replace __get_free_page() with kmalloc() Remove ugly casts by using the more natural kmalloc/kfree allocation, also replace GFP_USER(pointless here) with GFP_KERNEL. Link: https://lore.kernel.org/20260717091347.1144789-4-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Reviewed-by: Pedro Falcato Acked-by: David Hildenbrand (Arm) Cc: Jann Horn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/mincore.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mm/mincore.c b/mm/mincore.c index d9d4381d7ed4..a2119c50c50c 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -328,7 +329,7 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, if (!access_ok(vec, pages)) return -EFAULT; - tmp = (void *) __get_free_page(GFP_USER); + tmp = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!tmp) return -EAGAIN; @@ -353,6 +354,6 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, start += retval << PAGE_SHIFT; retval = 0; } - free_page((unsigned long) tmp); + kfree(tmp); return retval; } From 9218800076ec9257ed006f8d7edf802acc810319 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Fri, 17 Jul 2026 17:13:45 +0800 Subject: [PATCH 453/501] mm: mincore: remove xa_is_value() in mincore_swap() The swap_cache_get_folio() no longer returns shadow entries, so the xa_is_value() check is unnecessary. Link: https://lore.kernel.org/20260717091347.1144789-5-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Reviewed-by: Pedro Falcato Acked-by: David Hildenbrand (Arm) Cc: Jann Horn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/mincore.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mm/mincore.c b/mm/mincore.c index a2119c50c50c..16059ed47711 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -91,8 +91,7 @@ static unsigned char mincore_swap(swp_entry_t entry, bool shmem) folio = swap_cache_get_folio(entry); if (shmem) put_swap_device(si); - /* The swap cache space contains either folio, shadow or NULL */ - if (folio && !xa_is_value(folio)) { + if (folio) { present = folio_test_uptodate(folio); folio_put(folio); } From 751a1f061a7c412bf44e81782607b16866f4a6d6 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Fri, 17 Jul 2026 17:13:46 +0800 Subject: [PATCH 454/501] mm: mincore: improve mincore_hugetlb() walk_hugetlb_range() always passes a non-NULL pte, so remove the dead NULL check. Replace the per-page iteration loop with memset() for better performance. [wangkefeng.wang@huawei.com: fixup improve mincore_hugetlb()] Link: https://lore.kernel.org/20260720124252.1483984-1-wangkefeng.wang@huawei.com Link: https://lore.kernel.org/20260717091347.1144789-6-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Reviewed-by: Pedro Falcato Acked-by: David Hildenbrand (Arm) Cc: Jann Horn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/mincore.c | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/mm/mincore.c b/mm/mincore.c index 16059ed47711..40cffbc6f6fc 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -28,30 +28,16 @@ static int mincore_hugetlb(pte_t *pte, unsigned long hmask, unsigned long addr, unsigned long end, struct mm_walk *walk) { #ifdef CONFIG_HUGETLB_PAGE - unsigned char present; - unsigned char *vec = walk->private; + const unsigned long nr = (end - addr) >> PAGE_SHIFT; + unsigned char resident; spinlock_t *ptl; + pte_t ptep; ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte); - - /* - * Hugepages under user process are always in RAM and never - * swapped out, but theoretically it needs to be checked. - */ - if (!pte) { - present = 0; - } else { - const pte_t ptep = huge_ptep_get(walk->mm, addr, pte); - - if (huge_pte_none(ptep) || pte_is_marker(ptep)) - present = 0; - else - present = 1; - } - - for (; addr != end; vec++, addr += PAGE_SIZE) - *vec = present; - walk->private = vec; + ptep = huge_ptep_get(walk->mm, addr, pte); + resident = !huge_pte_none(ptep) && !pte_is_marker(ptep); + memset(walk->private, resident, nr); + walk->private += nr; spin_unlock(ptl); #else BUG(); From 557f842e25a9be7fc088818c308f5463c273a2bf Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Fri, 17 Jul 2026 17:13:47 +0800 Subject: [PATCH 455/501] mm: mincore: refactor mincore_page() Use early returns to reduce indentation and improve readability. Link: https://lore.kernel.org/20260717091347.1144789-7-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Acked-by: David Hildenbrand (Arm) Cc: Jann Horn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Pedro Falcato Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/mincore.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/mm/mincore.c b/mm/mincore.c index 40cffbc6f6fc..ff4ac8281768 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -93,7 +93,7 @@ static unsigned char mincore_swap(swp_entry_t entry, bool shmem) */ static unsigned char mincore_page(struct address_space *mapping, pgoff_t index) { - unsigned char present = 0; + unsigned char present; struct folio *folio; /* @@ -103,17 +103,16 @@ static unsigned char mincore_page(struct address_space *mapping, pgoff_t index) * tmpfs's .fault). So swapped out tmpfs mappings are tested here. */ folio = filemap_get_entry(mapping, index); - if (folio) { - if (xa_is_value(folio)) { - if (shmem_mapping(mapping)) - return mincore_swap(radix_to_swp_entry(folio), - true); - else - return 0; - } - present = folio_test_uptodate(folio); - folio_put(folio); + if (!folio) + return 0; + + if (xa_is_value(folio)) { + if (!shmem_mapping(mapping)) + return 0; + return mincore_swap(radix_to_swp_entry(folio), true); } + present = folio_test_uptodate(folio); + folio_put(folio); return present; } From d6f1586aa95ed8e9189395cc3a3b07233abe9b6f Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Thu, 16 Jul 2026 11:03:44 +0100 Subject: [PATCH 456/501] mm/huge_memory: remove unused can_split_folio() Patch series "mm/huge_memory: drop dead split helper variants". Two trivial cleanups in the folio split API, found while auditing the split helpers for the inode UAF fix [1]: - can_split_folio() is dead: only the !THP stub remains, no definition under CONFIG_TRANSPARENT_HUGEPAGE and no callers. Leftover from an earlier cleanup. - split_folio_to_list_to_order() has no direct callers; its only user is split_folio_to_order(), which always passes a NULL list. Fold it. No functional change. This patch (of 2): can_split_folio() has no callers and no definition under CONFIG_TRANSPARENT_HUGEPAGE; only the !THP stub remains, left behind by an earlier cleanup. Remove it. No functional change. Link: https://lore.kernel.org/20260716100345.473473-1-kirill@shutemov.name Link: https://lore.kernel.org/20260716100345.473473-2-kirill@shutemov.name Link: https://lore.kernel.org/linux-mm/aldjhtfVByHDQXe6@thinkstation [1] Link: https://lore.kernel.org/linux-mm/20260714122344.351895-1-kirill@shutemov.name [2] Signed-off-by: Kiryl Shutsemau (Meta) Acked-by: Usama Arif Reviewed-by: Lorenzo Stoakes (ARM) Reviewed-by: Zi Yan Acked-by: David Hildenbrand (Arm) Cc: Baolin Wang Cc: Barry Song Cc: Dev Jain Cc: Lance Yang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Signed-off-by: Andrew Morton --- include/linux/huge_mm.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 98bf1766b57f..fc27637e2688 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -642,11 +642,6 @@ thp_get_unmapped_area_vmaflags(struct file *filp, unsigned long addr, return 0; } -static inline bool -can_split_folio(struct folio *folio, int caller_pins, int *pextra_pins) -{ - return false; -} static inline int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, unsigned int new_order) From 2549ebcaa8361918d7e61e829f06366151a9623c Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Thu, 16 Jul 2026 11:03:45 +0100 Subject: [PATCH 457/501] mm/huge_memory: fold split_folio_to_list_to_order() into split_folio_to_order() split_folio_to_list_to_order() had no direct callers; its only user was split_folio_to_order(), which always passed a NULL list. Fold it into split_folio_to_order() and call split_huge_page_to_list_to_order() directly. No functional change. Link: https://lore.kernel.org/20260716100345.473473-3-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Acked-by: Usama Arif Reviewed-by: Lorenzo Stoakes (ARM) Reviewed-by: Zi Yan Acked-by: David Hildenbrand (Arm) Cc: Baolin Wang Cc: Barry Song Cc: Dev Jain Cc: Lance Yang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Signed-off-by: Andrew Morton --- include/linux/huge_mm.h | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index fc27637e2688..c745f7ad2298 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -823,15 +823,9 @@ static inline bool is_pmd_order(unsigned int order) return order == HPAGE_PMD_ORDER; } -static inline int split_folio_to_list_to_order(struct folio *folio, - struct list_head *list, int new_order) -{ - return split_huge_page_to_list_to_order(&folio->page, list, new_order); -} - static inline int split_folio_to_order(struct folio *folio, int new_order) { - return split_folio_to_list_to_order(folio, NULL, new_order); + return split_huge_page_to_list_to_order(&folio->page, NULL, new_order); } /** From af5f76aeb9c9aa9c77d5e1d94e0ee4618c805239 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 17 Jul 2026 17:14:35 -0700 Subject: [PATCH 458/501] mm/damon/core: initialize damos->last_applied Patch series "mm/damon: fix uninitialized DAMOS field and kunit exec expectation bugs". Fix a few Sashiko-found unurgent bugs. Patch 1 fixes use of uninitialized damos->last_applied field. Patches 2-7 fix DAMON kunit tests that do invalid memory access under test failures. The bugs are better to be fixed and eventually merged into stable@ kernel. That said, the fixes are arguably not urgent. Patch 1 only introduces negligible DAMOS efficiency degradation in occasional cases. Kunit fixes could introduce quite bad consequences but those are test code that affect only test run setups. This patch (of 7): Multiple DAMON regions could exist across a folio. If they fulfill the condition to apply a DAMOS scheme, the scheme could be applied multiple times to the folio. To avoid this, each DAMOS scheme stores the folio that the scheme was applied to last time in the damos->last_applied field and skips repeatedly applying the same scheme to the same folio. The field is being used without initialization, though. Hence, the mechanism could wrongly skip applying a scheme to a folio at the very first time of DAMOS run. The user impact is trivial. DAMON might unexpectedly skip applying DAMOS action for one folio for the first time per scheme. In the DAMON's best-effort world, this is never a real problem. No critical consequences such as kernel panic or memory corruption happen. It is a clear bug, though, and the fix is straightforward. Fix the issue by initializing the field in DAMOS scheme creation function, damon_new_scheme(). The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260718001442.87129-1-sj@kernel.org Link: https://lore.kernel.org/20260718001442.87129-2-sj@kernel.org Link: https://lore.kernel.org/20260714055436.120034-1-sj@kernel.org [1] Fixes: 94ba17adaba0 ("mm/damon: avoid applying DAMOS action to same entity multiple times") Signed-off-by: SJ Park Cc: Brendan Higgins Cc: # 6.15.x Signed-off-by: Andrew Morton --- mm/damon/core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/damon/core.c b/mm/damon/core.c index 603b102ff80f..20d267c615fa 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -705,6 +705,7 @@ struct damos *damon_new_scheme(struct damos_access_pattern *pattern, INIT_LIST_HEAD(&scheme->ops_filters); scheme->stat = (struct damos_stat){}; scheme->max_nr_snapshots = 0; + scheme->last_applied = NULL; INIT_LIST_HEAD(&scheme->list); scheme->quota = *(damos_quota_init(quota)); From 9b18ef3c3aa1ce24176e340061caf15fa2742564 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 17 Jul 2026 17:14:36 -0700 Subject: [PATCH 459/501] mm/damon/core-kunit: check region count before testing in split_at() damon_test_split_at() test next region that is assumed to be created by damon_split_region_at() invocation. But the split might fail. In this case, the succeeding test may dereference invalid pointers returned by damon_next_region(). The invalid pointer may not cause a really bad user impact, because of the implementation detail. It would only read wrong contents in the belonging damon_target struct. Depending on the future change of the offset from the link header to the accessing field, this could also be really dangerous, though. Still, the realistic user impact would be limited. It would affect only test run setups. Fix it by testing if the number of regions was also changed as expected and exit early for the failure. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260718001442.87129-3-sj@kernel.org Link: https://lore.kernel.org/20260714142352.100478-1-sj@kernel.org [1] Fixes: 17ccae8bb5c9 ("mm/damon: add kunit tests") Signed-off-by: SJ Park Cc: Brendan Higgins Cc: # 5.15.x Signed-off-by: Andrew Morton --- mm/damon/tests/core-kunit.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index eba643762132..322a295ac2ce 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -155,6 +155,10 @@ static void damon_test_split_at(struct kunit *test) r->age = 10; damon_add_region(r, t); damon_split_region_at(t, r, 25); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + if (damon_nr_regions(t) != 2) + goto out; + KUNIT_EXPECT_EQ(test, r->ar.start, 0ul); KUNIT_EXPECT_EQ(test, r->ar.end, 25ul); @@ -166,6 +170,7 @@ static void damon_test_split_at(struct kunit *test) KUNIT_EXPECT_EQ(test, r->last_nr_accesses, r_new->last_nr_accesses); KUNIT_EXPECT_EQ(test, r->age, r_new->age); +out: damon_free_target(t); } From 5fea07e460874c8c7cf00f728efbe22abc62c8d8 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 17 Jul 2026 17:14:37 -0700 Subject: [PATCH 460/501] mm/damon/vaddr-kunit: check region count in three_regions test damon_do_test_apply_three_regions() iterates regions after damon_set_regions() call assuming the function would succeed at setting the number of regions the same to the expected one. It might have failed. In this case, __nth_region_of() in the iteration could return NULL and NULL dereference can happen in the test. The consequent user impact (NULL dereference) is quite bad. The realistic user impact would be limited, though. It would affect only test run setups. Fix it by testing if the number of regions was also changed as expected and exit early for the failure. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260718001442.87129-4-sj@kernel.org Link: https://lore.kernel.org/20260713144757.39740-1-sj@kernel.org [1] Fixes: 17ccae8bb5c9 ("mm/damon: add kunit tests") Signed-off-by: SJ Park Cc: Brendan Higgins Cc: # 5.15.x Signed-off-by: Andrew Morton --- mm/damon/tests/vaddr-kunit.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mm/damon/tests/vaddr-kunit.h b/mm/damon/tests/vaddr-kunit.h index 61f844336ffb..6a95441d193a 100644 --- a/mm/damon/tests/vaddr-kunit.h +++ b/mm/damon/tests/vaddr-kunit.h @@ -158,12 +158,17 @@ static void damon_do_test_apply_three_regions(struct kunit *test, kunit_skip(test, "second damon_set_regions() fail"); } + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), nr_expected / 2); + if (damon_nr_regions(t) != nr_expected / 2) + goto out; + for (i = 0; i < nr_expected / 2; i++) { r = __nth_region_of(t, i); KUNIT_EXPECT_EQ(test, r->ar.start, expected[i * 2]); KUNIT_EXPECT_EQ(test, r->ar.end, expected[i * 2 + 1]); } +out: damon_destroy_target(t, NULL); } From 3423fe70395210e2f0cd795270292f6a27cd57b5 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 17 Jul 2026 17:14:38 -0700 Subject: [PATCH 461/501] mm/damon/core-kunit: handle region split failure in filter_out() damos_test_filter_out() test checks if damos_filter_match() of an address filter splits the region as expected under a given condition. But, the test continued regardless of the split successes. As a result, the later part of the test could dereference invalid pointers that returned from damon_next_region(). Further, it could corrupt memory from damon_destroy_region(). The consequent user impact (memory corruption) is quite bad. The realistic user impact would be limited, though. It would affect only test run setups. Fix it by exiting early for the number of regions test failure. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260718001442.87129-5-sj@kernel.org Link: https://lore.kernel.org/20260714142352.100478-1-sj@kernel.org [1] Fixes: 26713c890875 ("mm/damon/core-test: add a unit test for __damos_filter_out()") Signed-off-by: SJ Park Cc: Brendan Higgins Cc: # 6.6.x Signed-off-by: Andrew Morton --- mm/damon/tests/core-kunit.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 322a295ac2ce..eddf3a048458 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -1365,6 +1365,8 @@ static void damos_test_filter_out(struct kunit *test) KUNIT_EXPECT_EQ(test, r->ar.start, 1); KUNIT_EXPECT_EQ(test, r->ar.end, 2); KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + if (damon_nr_regions(t) != 2) + goto out; r2 = damon_next_region(r); KUNIT_EXPECT_EQ(test, r2->ar.start, 2); KUNIT_EXPECT_EQ(test, r2->ar.end, 4); @@ -1379,11 +1381,14 @@ static void damos_test_filter_out(struct kunit *test) KUNIT_EXPECT_EQ(test, r->ar.start, 2); KUNIT_EXPECT_EQ(test, r->ar.end, 6); KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + if (damon_nr_regions(t) != 2) + goto out; r2 = damon_next_region(r); KUNIT_EXPECT_EQ(test, r2->ar.start, 6); KUNIT_EXPECT_EQ(test, r2->ar.end, 8); damon_destroy_region(r2, t); +out: damon_free_target(t); damos_free_filter(f); } From 7e2f2c41b27f64caad6599073aaa0ccdd08745e1 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 17 Jul 2026 17:14:39 -0700 Subject: [PATCH 462/501] mm/damon/core-kunit: skip wrong dest walk in commit_dests_for() damos_test_commit_dests_for() traverse damos action destinations after damos_commit_dests(). It assumes damos_commit_dests() made expected numbers of destinations for source and destination structures. It might not. Because the traversal is made based on destination struct length, it could do out of bounds access for source value expectation. The consequent user impact (out-of-bound access ) is quite bad. The realistic user impact would be limited, though. It would affect only test run setups. Fix it by exiting early for the number of regions test failure. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260718001442.87129-6-sj@kernel.org Link: https://lore.kernel.org/20260713144757.39740-1-sj@kernel.org [1] Fixes: eec573b8dd65 ("mm/damon/tests/core-kunit: add damos_commit_dests() test") Signed-off-by: SJ Park Cc: Brendan Higgins Cc: # 6.19.x Signed-off-by: Andrew Morton --- mm/damon/tests/core-kunit.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index eddf3a048458..9b8c20a50ad5 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -1008,6 +1008,8 @@ static void damos_test_commit_dests_for(struct kunit *test, skip = false; KUNIT_EXPECT_EQ(test, dst.nr_dests, src_nr_dests); + if (dst.nr_dests != src_nr_dests) + goto out; for (i = 0; i < dst.nr_dests; i++) { KUNIT_EXPECT_EQ(test, dst.node_id_arr[i], src_node_id_arr[i]); KUNIT_EXPECT_EQ(test, dst.weight_arr[i], src_weight_arr[i]); From 867bbe6dbd5a68eb58e79cc7f068a6cf184c40d8 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 17 Jul 2026 17:14:40 -0700 Subject: [PATCH 463/501] mm/damon/core-kunit: skip wrong quota goal walk in commit_quota_goals() damos_test_commit_quota_goals_for() traverses damos quota goals after damos_commit_quota_goals() call. It assumes damos_commit_quota_goals() made expected numbers of goals. It might not. Because the traversal is made based on destination struct length, it could do out of bounds access for source expectation value array. The consequent user impact (out-of-bound access ) is quite bad. The realistic user impact would be limited though. It would affect only test run setups. Fix it by testing if the number of goals was also changed as expected and exit early for the failure. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260718001442.87129-7-sj@kernel.org Link: https://lore.kernel.org/20260713144757.39740-1-sj@kernel.org [1] Fixes: d9adfb8a28e7 ("mm/damon/tests/core-kunit: add damos_commit_quota_goals() test") Signed-off-by: SJ Park Cc: Brendan Higgins Cc: # 6.19.x Signed-off-by: Andrew Morton --- mm/damon/tests/core-kunit.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 9b8c20a50ad5..fbf986aee2de 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -839,6 +839,7 @@ static void damos_test_commit_quota_goals_for(struct kunit *test, struct damos_quota_goal *goal, *next; bool skip = true; int i; + int nr_dst = 0, nr_src = 0; INIT_LIST_HEAD(&dst.goals); INIT_LIST_HEAD(&src.goals); @@ -861,6 +862,14 @@ static void damos_test_commit_quota_goals_for(struct kunit *test, damos_commit_quota_goals(&dst, &src); + damos_for_each_quota_goal(goal, &dst) + nr_dst++; + damos_for_each_quota_goal(goal, &src) + nr_src++; + KUNIT_EXPECT_EQ(test, nr_dst, nr_src); + if (nr_dst != nr_src) + goto out; + i = 0; damos_for_each_quota_goal(goal, (&dst)) { KUNIT_EXPECT_EQ(test, goal->metric, src_goals[i].metric); From 488cf81c49955f83b7682dda880e9920c173e617 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Fri, 17 Jul 2026 17:14:41 -0700 Subject: [PATCH 464/501] mm/damon/core-kunit: skip wrong region walk in commit_target_regions() damon_test_commit_target_regions_for() traverses expected values array after damon_commit_target_regions() call. It assumes damon_commit_target_regions() made expected number of regions. It might not. Because the traversal is made based on the region count, it could do out of bounds access to the expectation value array. The consequent user impact (out-of-bound access) is quite bad. The realistic user impact would be limited, though. It would affect only test run setups. Fix it by testing if the number of regions was also changed as expected and exit early for the failure. The issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260718001442.87129-8-sj@kernel.org Link: https://lore.kernel.org/20260713144757.39740-1-sj@kernel.org [1] Fixes: 603f67eb91e0 ("mm/damon/tests/core-kunit: add damon_commit_target_regions() test") Signed-off-by: SJ Park Cc: Brendan Higgins Cc: # 6.19.x Signed-off-by: Andrew Morton --- mm/damon/tests/core-kunit.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index fbf986aee2de..4a536d41cdb2 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -1277,14 +1277,19 @@ static void damon_test_commit_target_regions_for(struct kunit *test, kunit_skip(test, "src target setup fail"); } damon_commit_target_regions(dst_target, src_target, 1); + + KUNIT_EXPECT_EQ(test, damon_nr_regions(dst_target), nr_expect_regions); + if (damon_nr_regions(dst_target) != nr_expect_regions) + goto out; + i = 0; damon_for_each_region(r, dst_target) { KUNIT_EXPECT_EQ(test, r->ar.start, expect_start_end[i][0]); KUNIT_EXPECT_EQ(test, r->ar.end, expect_start_end[i][1]); i++; } - KUNIT_EXPECT_EQ(test, damon_nr_regions(dst_target), nr_expect_regions); - KUNIT_EXPECT_EQ(test, i, nr_expect_regions); + +out: damon_free_target(dst_target); damon_free_target(src_target); } From a43ea999c56ab9c40f555d4387a096637e6d19ff Mon Sep 17 00:00:00 2001 From: Enze Li Date: Sat, 18 Jul 2026 08:21:25 +0800 Subject: [PATCH 465/501] mm/damon/ops-common: remove redundant local variable in damon_migrate_folio_list() In damon_migrate_folio_list(), we're redeclaring folio inside the first while loop, but it just shadows the outer one. Since the second loop uses the outer folio anyway, the inner declaration is pointless. Remove it to consistently reuse the same variable throughout the function and improve readability. Link: https://lore.kernel.org/20260718002125.637104-1-lienze@kylinos.cn Signed-off-by: Enze Li Reviewed-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/ops-common.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index 89c694f6e4fa..e59f5268d503 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -339,8 +339,6 @@ static unsigned int damon_migrate_folio_list(struct list_head *folio_list, LIST_HEAD(migrate_folios); while (!list_empty(folio_list)) { - struct folio *folio; - cond_resched(); folio = lru_to_folio(folio_list); From 308ab73e97c87bd0e142b11758faab7f88d82854 Mon Sep 17 00:00:00 2001 From: Yichong Chen Date: Mon, 20 Jul 2026 10:18:59 +0800 Subject: [PATCH 466/501] hugetlbfs: release subpool on fill_super failure hugetlbfs_fill_super() allocates a hugepage subpool when size or min_size mount options are specified. hugepage_new_subpool() may also reserve huge pages for min_size. If root dentry creation fails after the subpool is created, the failure path frees the subpool with kfree(). This bypasses hugepage_put_subpool() and can leave min_size reservations charged. Use hugepage_put_subpool() on the failure path, matching the normal put_super path. Link: https://lore.kernel.org/20260720021900.1376309-1-chenyichong@uniontech.com Fixes: 7ca02d0ae586 ("hugetlbfs: accept subpool min_size mount option and setup accordingly") Signed-off-by: Yichong Chen Cc: David Hildenbrand Cc: Muchun Song Cc: Oscar Salvador Signed-off-by: Andrew Morton --- fs/hugetlbfs/inode.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 43066f9ddf52..d71ed62cb783 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -1419,7 +1419,8 @@ hugetlbfs_fill_super(struct super_block *sb, struct fs_context *fc) goto out_free; return 0; out_free: - kfree(sbinfo->spool); + if (sbinfo->spool) + hugepage_put_subpool(sbinfo->spool); kfree(sbinfo); return -ENOMEM; } From 6c25083f7ae7e2660d766169e5b4d3e96010503f Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 19 Jul 2026 09:11:33 -0700 Subject: [PATCH 467/501] mm/damon/ops-common: use nr_accesses moving sum for quota score Since commit 42f994b71404 ("mm/damon/core: implement scheme-specific apply interval"), DAMOS scheme can be applied at any time. At that time, nr_accesses may not be fully aggregated. But the quota prioritization score is calculated using the not fully aggregated count. As a result, the performance of DAMOS could be degraded. Fix by using damon_nr_accesses_mvsum() instead. The user impact of the issue is suboptimum DAMOS performance under certain setups. Nonetheless, the bug was there from the beginning of the setup availability. In other words, the suboptimum performance is the baseline of the setup and hence it didn't cause regression. Also the extent of the suboptimality was not big enough to be found from users and testers. Still, this is a clear bug that is better to be fixed, and can be easily fixed. Link: https://lore.kernel.org/20260719161136.90191-1-sj@kernel.org Fixes: 42f994b71404 ("mm/damon/core: implement scheme-specific apply interval") Signed-off-by: SJ Park Cc: # 6.7.x Signed-off-by: Andrew Morton --- mm/damon/ops-common.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index e59f5268d503..d7d7f100389b 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -111,8 +111,9 @@ int damon_hot_score(struct damon_ctx *c, struct damon_region *r, unsigned int age_weight = s->quota.weight_age; int hotness; - freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / - damon_nr_samples_per_aggr(&c->attrs); + freq_subscore = mult_frac(damon_nr_accesses_mvsum(r, c), + DAMON_MAX_SUBSCORE, + damon_nr_samples_per_aggr(&c->attrs)); age_in_sec = (unsigned long)r->age * c->attrs.aggr_interval / 1000000; if (age_in_sec) From c608748607620f331196ed0ba9fe4017892c1457 Mon Sep 17 00:00:00 2001 From: SJ Park Date: Sun, 19 Jul 2026 08:54:40 -0700 Subject: [PATCH 468/501] mm/damon/core: handle region split failure in apply_min_nr_regions() damon_apply_min_nr_regions() repeatedly split each region until its size becomes small enough to meet the user-defined low limit of the number of regions. The loop assumes the split operation (damon_split_region_at()) will always succeed and create the new region. But the operation could silently fail for memory allocation failures, for example. If such failure happens and the region was the last region, the linked list-based next region fetching returns invalid pointer. As a result, invalid memory dereference and corruption could happen. Even if the corner case is handled, it imposes stress to the allocator by trying split regions for other targets. Fix the issue by breaking all the loops for any region split failure. This means there could be a min_nr_regions violation. It will only rarely happen since the allocation is arguably too small to fail. Even if it happens, it is only temporal. damon_apply_min_nr_regions() will be called again after the aggregation interval. The user impact of the issue should be minor, since the allocation is arguably too small to fail. But, it could still theoretically happen, and the consequence is very bad. This issue was discovered [1] by Sashiko. Link: https://lore.kernel.org/20260719155442.88794-1-sj@kernel.org Link: https://lore.kernel.org/20260717011834.120715-1-sj@kernel.org [1] Fixes: b1029f29eb1d ("mm/damon/core: split regions for min_nr_regions") Signed-off-by: SJ Park Cc: # 7.1.x Signed-off-by: Andrew Morton --- mm/damon/core.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 20d267c615fa..e0122376f43d 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1886,7 +1886,7 @@ static unsigned long damon_region_sz_limit(struct damon_ctx *ctx) return sz; } -static void damon_split_region_at(struct damon_target *t, +static int damon_split_region_at(struct damon_target *t, struct damon_region *r, unsigned long sz_r); /* @@ -1912,11 +1912,13 @@ static unsigned long damon_apply_min_nr_regions(struct damon_ctx *ctx) damon_for_each_target(t, ctx) { damon_for_each_region_safe(r, next, t) { while (damon_sz_region(r) > max_region_sz) { - damon_split_region_at(t, r, max_region_sz); + if (damon_split_region_at(t, r, max_region_sz)) + goto out; r = damon_next_region(r); } } } +out: return max_region_sz; } @@ -3411,8 +3413,10 @@ static void damon_verify_split_region_at(struct damon_region *r, * * r the region to be split * sz_r size of the first sub-region that will be made + * + * Return: 0 on success, negative error code otherwise. */ -static void damon_split_region_at(struct damon_target *t, +static int damon_split_region_at(struct damon_target *t, struct damon_region *r, unsigned long sz_r) { struct damon_region *new; @@ -3420,7 +3424,7 @@ static void damon_split_region_at(struct damon_target *t, damon_verify_split_region_at(r, sz_r); new = damon_new_region(r->ar.start + sz_r, r->ar.end); if (!new) - return; + return -ENOMEM; r->ar.end = new->ar.start; @@ -3433,6 +3437,7 @@ static void damon_split_region_at(struct damon_target *t, sizeof(r->last_probe_hits)); damon_insert_region(new, r, damon_next_region(r), t); + return 0; } /* Split every region in the given target into 'nr_subs' regions */ From c2875056e15a0cf4e80898292e9e6e8ba0750b62 Mon Sep 17 00:00:00 2001 From: "Zenghui Yu (Huawei)" Date: Tue, 21 Jul 2026 19:19:19 +0800 Subject: [PATCH 469/501] docs/mm: Physical Memory: remove deferred_split_queue deferred_split_queue was removed from struct pglist_data {} in commit fafaeceb89a5 ("mm: switch deferred split shrinker to list_lru"). Remove its description in doc as well. Link: https://lore.kernel.org/20260721111919.22923-1-zenghui.yu@linux.dev Signed-off-by: Zenghui Yu (Huawei) Acked-by: David Hildenbrand (Arm) Reviewed-by: Lorenzo Stoakes (ARM) Reviewed-by: SJ Park Signed-off-by: Andrew Morton --- Documentation/mm/physical_memory.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/Documentation/mm/physical_memory.rst b/Documentation/mm/physical_memory.rst index b76183545e5b..a09407d72973 100644 --- a/Documentation/mm/physical_memory.rst +++ b/Documentation/mm/physical_memory.rst @@ -262,9 +262,6 @@ General PFN that needs to be initialized. Defined only when ``CONFIG_DEFERRED_STRUCT_PAGE_INIT`` is enabled -``deferred_split_queue`` - Per-node queue of huge pages that their split was deferred. Defined only when ``CONFIG_TRANSPARENT_HUGEPAGE`` is enabled. - ``__lruvec`` Per-node lruvec holding LRU lists and related parameters. Used only when memory cgroups are disabled. It should not be accessed directly, use From e984520724bd40ccf0329e5c09606f05af1a3671 Mon Sep 17 00:00:00 2001 From: Jinjiang Tu Date: Mon, 20 Jul 2026 11:30:19 +0800 Subject: [PATCH 470/501] fs: stable_page_flags(): use BIT_ULL() for KPF flags Patch series "cleanup for stable_page_flags()", v2. This series cleans up stable_page_flags(), see the details in the commit messages. This patch (of 3): The stable_page_flags() function currently sets page flag bits using "1 << KPF_xxx" for various KPF_* definitions. KPF_* may be larger than 32, which will trigger -Wshift-count-overflow warning. All KPF_* values currently used in "1 << KPF_xxx" are smaller than 33, so no warning is triggered with current code. Replace all occurrences of "1 << KPF_*" with the BIT_ULL() macro to ensure all shifts are performed on a 64-bit unsigned type. This makes the code robust, and safe for future extension. No functional change is intended. Link: https://lore.kernel.org/20260720033021.4091944-1-tujinjiang@huawei.com Link: https://lore.kernel.org/20260720033021.4091944-2-tujinjiang@huawei.com Signed-off-by: Jinjiang Tu Acked-by: David Hildenbrand (Arm) Acked-by: Zi Yan Cc: Chengming Zhou Cc: Kefeng Wang Cc: Luiz Capitulino Cc: Matthew Wilcox (Oracle) Cc: Miaohe Lin Cc: Nanyong Sun Cc: Svetly Todorov Cc: xu xin Signed-off-by: Andrew Morton --- fs/proc/page.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/fs/proc/page.c b/fs/proc/page.c index 7d9387143435..ee39b321f1c6 100644 --- a/fs/proc/page.c +++ b/fs/proc/page.c @@ -157,7 +157,7 @@ u64 stable_page_flags(const struct page *page) * it differentiates a memory hole from a page with no flags */ if (!page) - return 1 << KPF_NOPAGE; + return BIT_ULL(KPF_NOPAGE); snapshot_page(&ps, page); folio = &ps.folio_snapshot; @@ -170,11 +170,11 @@ u64 stable_page_flags(const struct page *page) * pseudo flags for the well known (anonymous) memory mapped pages */ if (folio_mapped(folio)) - u |= 1 << KPF_MMAP; + u |= BIT_ULL(KPF_MMAP); if (is_anon) { - u |= 1 << KPF_ANON; + u |= BIT_ULL(KPF_ANON); if ((mapping & FOLIO_MAPPING_FLAGS) == FOLIO_MAPPING_KSM) - u |= 1 << KPF_KSM; + u |= BIT_ULL(KPF_KSM); } /* @@ -184,35 +184,35 @@ u64 stable_page_flags(const struct page *page) if (ps.idx == 0) u |= kpf_copy_bit(k, KPF_COMPOUND_HEAD, PG_head); else - u |= 1 << KPF_COMPOUND_TAIL; + u |= BIT_ULL(KPF_COMPOUND_TAIL); if (folio_test_hugetlb(folio)) - u |= 1 << KPF_HUGE; + u |= BIT_ULL(KPF_HUGE); else if (folio_test_large(folio) && folio_test_large_rmappable(folio)) { /* Note: we indicate any THPs here, not just PMD-sized ones */ - u |= 1 << KPF_THP; + u |= BIT_ULL(KPF_THP); } else if (is_huge_zero_pfn(ps.pfn)) { - u |= 1 << KPF_ZERO_PAGE; - u |= 1 << KPF_THP; + u |= BIT_ULL(KPF_ZERO_PAGE); + u |= BIT_ULL(KPF_THP); } else if (is_zero_pfn(ps.pfn)) { - u |= 1 << KPF_ZERO_PAGE; + u |= BIT_ULL(KPF_ZERO_PAGE); } if (ps.flags & PAGE_SNAPSHOT_PG_BUDDY) - u |= 1 << KPF_BUDDY; + u |= BIT_ULL(KPF_BUDDY); if (folio_test_offline(folio)) - u |= 1 << KPF_OFFLINE; + u |= BIT_ULL(KPF_OFFLINE); if (folio_test_pgtable(folio)) - u |= 1 << KPF_PGTABLE; + u |= BIT_ULL(KPF_PGTABLE); if (folio_test_slab(folio)) - u |= 1 << KPF_SLAB; + u |= BIT_ULL(KPF_SLAB); #if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT) u |= kpf_copy_bit(k, KPF_IDLE, PG_idle); #else if (ps.flags & PAGE_SNAPSHOT_PG_IDLE) - u |= 1 << KPF_IDLE; + u |= BIT_ULL(KPF_IDLE); #endif u |= kpf_copy_bit(k, KPF_LOCKED, PG_locked); @@ -227,14 +227,14 @@ u64 stable_page_flags(const struct page *page) #define SWAPCACHE ((1 << PG_swapbacked) | (1 << PG_swapcache)) if ((k & SWAPCACHE) == SWAPCACHE) - u |= 1 << KPF_SWAPCACHE; + u |= BIT_ULL(KPF_SWAPCACHE); u |= kpf_copy_bit(k, KPF_SWAPBACKED, PG_swapbacked); u |= kpf_copy_bit(k, KPF_UNEVICTABLE, PG_unevictable); u |= kpf_copy_bit(k, KPF_MLOCKED, PG_mlocked); #ifdef CONFIG_MEMORY_FAILURE - if (u & (1 << KPF_HUGE)) + if (u & BIT_ULL(KPF_HUGE)) u |= kpf_copy_bit(k, KPF_HWPOISON, PG_hwpoison); else u |= kpf_copy_bit(ps.page_snapshot.flags.f, KPF_HWPOISON, PG_hwpoison); From c52faf27a9a05071b7845560cdfcb5fa2dea2ba4 Mon Sep 17 00:00:00 2001 From: Jinjiang Tu Date: Mon, 20 Jul 2026 11:30:20 +0800 Subject: [PATCH 471/501] fs: stable_page_flags(): use folio_test_*() helpers Since commit 304daa8132a9 ("maps4: add /proc/kpageflags interface"), /proc/kpageflags directly operates on page->flags to determine page status. Later, commit 177975495914 ("proc: export more page flags in /proc/kpageflags") started using page helper functions when exposing new flags, leading to a mix of both approaches. For tail pages, the original code did not return corresponding status. commit 0a71649cb724 ("/proc/kpageflags: return KPF_SLAB for slab tail pages") and commit 832fc1de01ae ("/proc/kpageflags: return KPF_BUDDY for "tail" buddy pages") made tail slab/buddy pages also return corresponding status. Then commit dee3d0bef2b0 ("proc: rewrite stable_page_flags()") made all tail pages return the same status as their head page, except for hwpoison and mapped flags. It also cached the folio's flags and operate on the flags directly to avoid concurrency issues if using folio_test_*() helpers. Since commit 476d87d6a061 ("fs: stable_page_flags(): use snapshot_page()"), we can now safely switch to folio_test_*() helpers instead of directly operating on flags, which is more readable and consistent with the rest of the kernel. Only convert cfolio-specific flags (i.e., anon, ksm, swapcache) to folio_test_*() helpers, which reduces redundant code. Keep others unchanged due to they aren't folio-specific flags or coverting them doesn't cleanup. No functional change is intended. Link: https://lore.kernel.org/20260720033021.4091944-3-tujinjiang@huawei.com Signed-off-by: Jinjiang Tu Acked-by: David Hildenbrand (Arm) Cc: Chengming Zhou Cc: Kefeng Wang Cc: Luiz Capitulino Cc: Matthew Wilcox (Oracle) Cc: Miaohe Lin Cc: Nanyong Sun Cc: Svetly Todorov Cc: xu xin Cc: Zi Yan Signed-off-by: Andrew Morton --- fs/proc/page.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/fs/proc/page.c b/fs/proc/page.c index ee39b321f1c6..b3c1d7a7604d 100644 --- a/fs/proc/page.c +++ b/fs/proc/page.c @@ -148,8 +148,6 @@ u64 stable_page_flags(const struct page *page) const struct folio *folio; struct page_snapshot ps; unsigned long k; - unsigned long mapping; - bool is_anon; u64 u = 0; /* @@ -161,19 +159,16 @@ u64 stable_page_flags(const struct page *page) snapshot_page(&ps, page); folio = &ps.folio_snapshot; - k = folio->flags.f; - mapping = (unsigned long)folio->mapping; - is_anon = mapping & FOLIO_MAPPING_ANON; /* * pseudo flags for the well known (anonymous) memory mapped pages */ if (folio_mapped(folio)) u |= BIT_ULL(KPF_MMAP); - if (is_anon) { + if (folio_test_anon(folio)) { u |= BIT_ULL(KPF_ANON); - if ((mapping & FOLIO_MAPPING_FLAGS) == FOLIO_MAPPING_KSM) + if (folio_test_ksm(folio)) u |= BIT_ULL(KPF_KSM); } @@ -225,11 +220,10 @@ u64 stable_page_flags(const struct page *page) u |= kpf_copy_bit(k, KPF_ACTIVE, PG_active); u |= kpf_copy_bit(k, KPF_RECLAIM, PG_reclaim); -#define SWAPCACHE ((1 << PG_swapbacked) | (1 << PG_swapcache)) - if ((k & SWAPCACHE) == SWAPCACHE) + if (folio_test_swapcache(folio)) u |= BIT_ULL(KPF_SWAPCACHE); - u |= kpf_copy_bit(k, KPF_SWAPBACKED, PG_swapbacked); + u |= kpf_copy_bit(k, KPF_SWAPBACKED, PG_swapbacked); u |= kpf_copy_bit(k, KPF_UNEVICTABLE, PG_unevictable); u |= kpf_copy_bit(k, KPF_MLOCKED, PG_mlocked); From 39e9b8aeaa2629724e8131b20b2b7f93e02eb3d9 Mon Sep 17 00:00:00 2001 From: Jinjiang Tu Date: Mon, 20 Jul 2026 11:30:21 +0800 Subject: [PATCH 472/501] fs: stable_page_flags(): simplify KPF_IDLE handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For KPF_IDLE, folio_test_idle() check in set_ps_flags() already handles all config cases, so remove the duplicate code in stable_page_flags() that deals with CONFIG_PAGE_IDLE_FLAG for the 64‑bit case. No functional change is intended. Link: https://lore.kernel.org/20260720033021.4091944-4-tujinjiang@huawei.com Signed-off-by: Jinjiang Tu Acked-by: David Hildenbrand (Arm) Cc: Chengming Zhou Cc: Kefeng Wang Cc: Luiz Capitulino Cc: Matthew Wilcox (Oracle) Cc: Miaohe Lin Cc: Nanyong Sun Cc: Svetly Todorov Cc: xu xin Cc: Zi Yan Signed-off-by: Andrew Morton --- fs/proc/page.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/fs/proc/page.c b/fs/proc/page.c index b3c1d7a7604d..260772b20bd9 100644 --- a/fs/proc/page.c +++ b/fs/proc/page.c @@ -196,6 +196,9 @@ u64 stable_page_flags(const struct page *page) if (ps.flags & PAGE_SNAPSHOT_PG_BUDDY) u |= BIT_ULL(KPF_BUDDY); + if (ps.flags & PAGE_SNAPSHOT_PG_IDLE) + u |= BIT_ULL(KPF_IDLE); + if (folio_test_offline(folio)) u |= BIT_ULL(KPF_OFFLINE); if (folio_test_pgtable(folio)) @@ -203,13 +206,6 @@ u64 stable_page_flags(const struct page *page) if (folio_test_slab(folio)) u |= BIT_ULL(KPF_SLAB); -#if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT) - u |= kpf_copy_bit(k, KPF_IDLE, PG_idle); -#else - if (ps.flags & PAGE_SNAPSHOT_PG_IDLE) - u |= BIT_ULL(KPF_IDLE); -#endif - u |= kpf_copy_bit(k, KPF_LOCKED, PG_locked); u |= kpf_copy_bit(k, KPF_DIRTY, PG_dirty); u |= kpf_copy_bit(k, KPF_UPTODATE, PG_uptodate); From 60191388cad91a718b0c05fd6a8f45fd4ec3997f Mon Sep 17 00:00:00 2001 From: Yichong Chen Date: Mon, 20 Jul 2026 15:38:41 +0800 Subject: [PATCH 473/501] hugetlb: make hugepage_put_subpool() tolerate NULL Both callers of hugepage_put_subpool() check whether the subpool pointer is NULL before calling it. Move the NULL check into hugepage_put_subpool() so callers can use the helper unconditionally. This is a follow-up cleanup after using hugepage_put_subpool() from the hugetlbfs_fill_super() failure path. Link: https://lore.kernel.org/20260720073841.1389354-1-chenyichong@uniontech.com Signed-off-by: Yichong Chen Reviewed-by: Muchun Song Cc: David Hildenbrand Cc: Oscar Salvador Signed-off-by: Andrew Morton --- fs/hugetlbfs/inode.c | 6 ++---- mm/hugetlb.c | 3 +++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index d71ed62cb783..a578fba8e2fc 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -1129,8 +1129,7 @@ static void hugetlbfs_put_super(struct super_block *sb) if (sbi) { sb->s_fs_info = NULL; - if (sbi->spool) - hugepage_put_subpool(sbi->spool); + hugepage_put_subpool(sbi->spool); kfree(sbi); } @@ -1419,8 +1418,7 @@ hugetlbfs_fill_super(struct super_block *sb, struct fs_context *fc) goto out_free; return 0; out_free: - if (sbinfo->spool) - hugepage_put_subpool(sbinfo->spool); + hugepage_put_subpool(sbinfo->spool); kfree(sbinfo); return -ENOMEM; } diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 0500b4e8bcd2..6daf831b14c5 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -182,6 +182,9 @@ void hugepage_put_subpool(struct hugepage_subpool *spool) { unsigned long flags; + if (!spool) + return; + spin_lock_irqsave(&spool->lock, flags); BUG_ON(!spool->count); spool->count--; From 5d72b5d76de37c4a3f50b3520d1486eb1024b446 Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Mon, 20 Jul 2026 06:55:05 +0000 Subject: [PATCH 474/501] mm/memory: move pte_install_uffd_wp_if_needed() into memory.c Patch series "Batch unmap of uffd-wp file folios", v2. Currently, batched unmapping is supported if: 1) folio is a file folio, not belonging to uffd-wp VMA 2) folio is anonymous and not swapbacked (lazyfree), not belonging to uffd-wp VMA So the cases which are not supported are 1) folio belonging to uffd-wp VMA 2) folio is anonymous and swapbacked It is easy to see that this adds a lot of cognitive load while reading try_to_unmap_one - we need to remember throughout whether nr_pages == 1 or > 1. The uffd-wp handling in try_to_unmap_one is regarding preserving the uffd-wp state for file folios via pte_install_uffd_wp_if_needed (for anon folio, we handle that while constructing the swap pte). Stop special casing on uffd-wp VMAs by simply adding batching support to pte_install_uffd_wp_if_needed. This patch (of 3): pte_install_uffd_wp_if_needed() has grown too large for mm_inline.h. Move it to memory.c. This helper is only used inside mm/, so declare it in mm/internal.h instead of a public header. While at it, convert the comment to kerneldoc and rename the local arguments from pte/pteval to ptep/pte so the pointer and PTE value are easier to distinguish. Link: https://lore.kernel.org/20260720065508.2695106-1-dev.jain@arm.com Link: https://lore.kernel.org/20260720065508.2695106-2-dev.jain@arm.com Signed-off-by: Dev Jain Acked-by: David Hildenbrand (Arm) Cc: Anshuman Khandual Cc: Axel Rasmussen Cc: Barry Song Cc: Harry Yoo Cc: Jann Horn Cc: Kairui Song Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Rik van Riel Cc: Ryan Roberts Cc: Shakeel Butt Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie Signed-off-by: Andrew Morton --- include/linux/mm_inline.h | 53 ---------------------------------- mm/internal.h | 3 ++ mm/memory.c | 60 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 53 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index b5c4dc0f3fe3..621c8653d8f7 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -566,59 +566,6 @@ static inline pte_marker copy_pte_marker( return dstm; } -/* - * If this pte is wr-protected by uffd-wp in any form, arm the special pte to - * replace a none pte. NOTE! This should only be called when *pte is already - * cleared so we will never accidentally replace something valuable. Meanwhile - * none pte also means we are not demoting the pte so tlb flushed is not needed. - * E.g., when pte cleared the caller should have taken care of the tlb flush. - * - * Must be called with pgtable lock held so that no thread will see the none - * pte, and if they see it, they'll fault and serialize at the pgtable lock. - * - * Returns true if an uffd-wp pte was installed, false otherwise. - */ -static inline bool -pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr, - pte_t *pte, pte_t pteval) -{ - bool arm_uffd_pte = false; - - if (!uffd_supports_wp_marker()) - return false; - - /* The current status of the pte should be "cleared" before calling */ - WARN_ON_ONCE(!pte_none(ptep_get(pte))); - - /* - * NOTE: userfaultfd_wp_unpopulated() doesn't need this whole - * thing, because when zapping either it means it's dropping the - * page, or in TTU where the present pte will be quickly replaced - * with a swap pte. There's no way of leaking the bit. - */ - if (vma_is_anonymous(vma) || !userfaultfd_wp(vma)) - return false; - - /* A uffd-wp wr-protected normal pte */ - if (unlikely(pte_present(pteval) && pte_uffd(pteval))) - arm_uffd_pte = true; - - /* - * A uffd-wp wr-protected swap pte. Note: this should even cover an - * existing pte marker with uffd-wp bit set. - */ - if (unlikely(pte_swp_uffd_any(pteval))) - arm_uffd_pte = true; - - if (unlikely(arm_uffd_pte)) { - set_pte_at(vma->vm_mm, addr, pte, - make_pte_marker(PTE_MARKER_UFFD_WP)); - return true; - } - - return false; -} - static inline bool vma_has_recency(const struct vm_area_struct *vma) { if (vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ)) diff --git a/mm/internal.h b/mm/internal.h index f26423de4ca2..b6a3589a61c1 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -276,6 +276,9 @@ void unmap_vmas(struct mmu_gather *tlb, struct unmap_desc *unmap); #ifdef CONFIG_MMU +bool pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, pte_t pte); + static inline void get_anon_vma(struct anon_vma *anon_vma) { atomic_inc(&anon_vma->refcount); diff --git a/mm/memory.c b/mm/memory.c index fb7a246487ac..ed23e009217b 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1619,6 +1619,66 @@ static inline bool zap_drop_markers(struct zap_details *details) return details->zap_flags & ZAP_FLAG_DROP_MARKER; } +/** + * pte_install_uffd_wp_if_needed - install uffd-wp marker after clearing a PTE + * @vma: The VMA the page is mapped into. + * @addr: Address the page is mapped at. + * @ptep: Page table pointer for this entry. + * @pte: Old value of the entry pointed to by @ptep. + * + * If the PTE was write-protected by uffd-wp in any form, arm a special PTE + * to replace a none PTE. NOTE! This should only be called when the PTE is + * already cleared so we will never accidentally replace something valuable. + * Meanwhile none PTEs also mean we are not demoting the PTE so a TLB flush is + * not needed. E.g., when the PTE was cleared, the caller should have taken care + * of the TLB flush. + * + * Must be called with the page table lock held so that no thread will see the + * none PTE, and if they see it, they'll fault and serialize at the page table + * lock. + * + * Returns true if an uffd-wp PTE was installed, false otherwise. + */ +bool pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, pte_t pte) +{ + bool arm_uffd_pte = false; + + if (!uffd_supports_wp_marker()) + return false; + + /* The current status of the pte should be "cleared" before calling */ + WARN_ON_ONCE(!pte_none(ptep_get(ptep))); + + /* + * NOTE: userfaultfd_wp_unpopulated() doesn't need this whole + * thing, because when zapping either it means it's dropping the + * page, or in TTU where the present pte will be quickly replaced + * with a swap pte. There's no way of leaking the bit. + */ + if (vma_is_anonymous(vma) || !userfaultfd_wp(vma)) + return false; + + /* A uffd-wp wr-protected normal pte */ + if (unlikely(pte_present(pte) && pte_uffd(pte))) + arm_uffd_pte = true; + + /* + * A uffd-wp wr-protected swap pte. Note: this should even cover an + * existing pte marker with uffd-wp bit set. + */ + if (unlikely(pte_swp_uffd_any(pte))) + arm_uffd_pte = true; + + if (unlikely(arm_uffd_pte)) { + set_pte_at(vma->vm_mm, addr, ptep, + make_pte_marker(PTE_MARKER_UFFD_WP)); + return true; + } + + return false; +} + /* * This function makes sure that we'll replace the none pte with an uffd-wp * swap special pte marker when necessary. Must be with the pgtable lock held. From 4259767463b326261679718c09c58d44d1e9c4b6 Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Mon, 20 Jul 2026 06:55:06 +0000 Subject: [PATCH 475/501] mm/memory: batch set uffd-wp markers during zapping Enable batch setting of uffd-wp PTE markers. The code paths passing nr > 1 to zap_install_uffd_wp_if_needed() produce that nr through either folio_pte_batch() or swap_pte_batch(), therefore batching is correct: 1) All PTEs belong to the same type of VMA: anonymous or non-anonymous, wp-armed or non-wp-armed. 2) All PTEs are either marked with uffd-wp or not marked with uffd-wp; the same applies to the pte_swp_uffd_any() check. 3) uffd_supports_wp_marker() is independent of the function parameters. Use set_pte_at() in a loop instead of set_ptes(), because set_ptes() cannot handle nonpresent to nonpresent conversion for nr_pages > 1. Rename the helper to cond_install_uffd_wp_ptes(). Link: https://lore.kernel.org/20260720065508.2695106-3-dev.jain@arm.com Signed-off-by: Dev Jain Acked-by: David Hildenbrand (Arm) Cc: Anshuman Khandual Cc: Axel Rasmussen Cc: Barry Song Cc: Harry Yoo Cc: Jann Horn Cc: Kairui Song Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Rik van Riel Cc: Ryan Roberts Cc: Shakeel Butt Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie Signed-off-by: Andrew Morton --- mm/internal.h | 5 +++-- mm/memory.c | 61 +++++++++++++++++++++------------------------------ mm/rmap.c | 2 +- 3 files changed, 29 insertions(+), 39 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index b6a3589a61c1..0f49cff67a2b 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -276,8 +276,9 @@ void unmap_vmas(struct mmu_gather *tlb, struct unmap_desc *unmap); #ifdef CONFIG_MMU -bool pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, - unsigned long addr, pte_t *ptep, pte_t pte); +bool cond_install_uffd_wp_ptes(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, pte_t pte, + unsigned long nr_ptes); static inline void get_anon_vma(struct anon_vma *anon_vma) { diff --git a/mm/memory.c b/mm/memory.c index ed23e009217b..be743a9c6606 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1620,27 +1620,29 @@ static inline bool zap_drop_markers(struct zap_details *details) } /** - * pte_install_uffd_wp_if_needed - install uffd-wp marker after clearing a PTE - * @vma: The VMA the page is mapped into. - * @addr: Address the page is mapped at. - * @ptep: Page table pointer for this entry. + * cond_install_uffd_wp_ptes - install uffd-wp markers after clearing PTEs + * @vma: The VMA the pages are mapped into. + * @addr: Address the first page of this batch is mapped at. + * @ptep: Page table pointer for the first entry of this batch. * @pte: Old value of the entry pointed to by @ptep. + * @nr_ptes: Number of entries to install. * - * If the PTE was write-protected by uffd-wp in any form, arm a special PTE - * to replace a none PTE. NOTE! This should only be called when the PTE is + * If the PTEs were write-protected by uffd-wp in any form, arm special PTEs + * to replace none PTEs. NOTE! This should only be called when the PTEs are * already cleared so we will never accidentally replace something valuable. - * Meanwhile none PTEs also mean we are not demoting the PTE so a TLB flush is - * not needed. E.g., when the PTE was cleared, the caller should have taken care - * of the TLB flush. + * Meanwhile none PTEs also mean we are not demoting the PTEs so a TLB flush is + * not needed. E.g., when the PTEs were cleared, the caller should have taken + * care of the TLB flush. * * Must be called with the page table lock held so that no thread will see the - * none PTE, and if they see it, they'll fault and serialize at the page table + * none PTEs, and if they see them, they'll fault and serialize at the page table * lock. * - * Returns true if an uffd-wp PTE was installed, false otherwise. + * Returns true if uffd-wp PTEs were installed, false otherwise. */ -bool pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, - unsigned long addr, pte_t *ptep, pte_t pte) +bool cond_install_uffd_wp_ptes(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, pte_t pte, + unsigned long nr_ptes) { bool arm_uffd_pte = false; @@ -1670,13 +1672,19 @@ bool pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, if (unlikely(pte_swp_uffd_any(pte))) arm_uffd_pte = true; - if (unlikely(arm_uffd_pte)) { + if (likely(!arm_uffd_pte)) + return false; + + for (;;) { set_pte_at(vma->vm_mm, addr, ptep, make_pte_marker(PTE_MARKER_UFFD_WP)); - return true; + if (--nr_ptes == 0) + break; + ptep++; + addr += PAGE_SIZE; } - return false; + return true; } /* @@ -1690,29 +1698,10 @@ zap_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr, pte_t *pte, int nr, struct zap_details *details, pte_t pteval) { - bool was_installed = false; - - if (!uffd_supports_wp_marker()) - return false; - - /* Zap on anonymous always means dropping everything */ - if (vma_is_anonymous(vma)) - return false; - if (zap_drop_markers(details)) return false; - for (;;) { - /* the PFN in the PTE is irrelevant. */ - if (pte_install_uffd_wp_if_needed(vma, addr, pte, pteval)) - was_installed = true; - if (--nr == 0) - break; - pte++; - addr += PAGE_SIZE; - } - - return was_installed; + return cond_install_uffd_wp_ptes(vma, addr, pte, pteval, nr); } static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb, diff --git a/mm/rmap.c b/mm/rmap.c index 867db2b61573..e6ffb63fe19b 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2345,7 +2345,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * we may want to replace a none pte with a marker pte if * it's file-backed, so we don't lose the tracking info. */ - pte_install_uffd_wp_if_needed(vma, address, pvmw.pte, pteval); + cond_install_uffd_wp_ptes(vma, address, pvmw.pte, pteval, 1); /* Update high watermark before we lower rss */ update_hiwater_rss(mm); From 4fe4ea4cbe75f284fdebf2b124b1e84e33be767c Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Mon, 20 Jul 2026 06:55:07 +0000 Subject: [PATCH 476/501] mm/rmap: batch unmap file folios belonging to uffd-wp VMAs Commit a67fe41e214f ("mm: rmap: support batched unmapping for file large folios") extended batched unmapping for file folios. That also required making pte_install_uffd_wp_if_needed() support batching, but that was left out for the time being. Correctness was maintained by stopping batching if the VMA the folio belongs to is marked uffd-wp. Now that cond_install_uffd_wp_ptes() supports batching, call it with the full batch length and allow folio_unmap_pte_batch() to batch file folios belonging to uffd-wp VMAs. For file folios, if the uffd-wp bit is set, unmapping converts present PTEs into uffd-wp markers. We must ensure that the same PTE range is not reprocessed by the try_to_unmap_one() loop. The page_vma_mapped_walk API ensures this: check_pte() only returns true if any PFN in [pvmw->pfn, pvmw->pfn + nr_pages) is mapped by the PTE. There is no PFN underlying a uffd-wp marker PTE, so check_pte() returns false and the walk skips ahead until it reaches a present entry again. Link: https://lore.kernel.org/20260720065508.2695106-4-dev.jain@arm.com Signed-off-by: Dev Jain Acked-by: David Hildenbrand (Arm) Cc: Anshuman Khandual Cc: Axel Rasmussen Cc: Barry Song Cc: Harry Yoo Cc: Jann Horn Cc: Kairui Song Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Rik van Riel Cc: Ryan Roberts Cc: Shakeel Butt Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Wei Xu Cc: Yuanchu Xie Signed-off-by: Andrew Morton --- mm/rmap.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mm/rmap.c b/mm/rmap.c index e6ffb63fe19b..1f72d279ba68 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1965,9 +1965,6 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio, if (pte_unused(pte)) return 1; - if (userfaultfd_protected(vma)) - return 1; - /* * If unmap fails, we need to restore the ptes. To avoid accidentally * upgrading write permissions for ptes that were not originally @@ -2345,7 +2342,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * we may want to replace a none pte with a marker pte if * it's file-backed, so we don't lose the tracking info. */ - cond_install_uffd_wp_ptes(vma, address, pvmw.pte, pteval, 1); + cond_install_uffd_wp_ptes(vma, address, pvmw.pte, pteval, + nr_pages); /* Update high watermark before we lower rss */ update_hiwater_rss(mm); From 63b420948250550d2fe7bf50c4ccf95cb4afa9fd Mon Sep 17 00:00:00 2001 From: longlong yan Date: Mon, 20 Jul 2026 14:34:39 +0800 Subject: [PATCH 477/501] selftests/mm: use MAP_FAILED for mmap error check Replace the direct comparison with (void *)-1 with the standard MAP_FAILED macro when checking mmap() Link: https://lore.kernel.org/20260720063439.522-1-yanlonglong@kylinos.cn Signed-off-by: longlong yan Reviewed-by: Lorenzo Stoakes (ARM) Acked-by: David Hildenbrand (Arm) Cc: Liam R. Howlett Cc: Michal Hocko Cc: Mike Rapoport Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/mseal_test.c | 106 ++++++++++++------------ 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/tools/testing/selftests/mm/mseal_test.c b/tools/testing/selftests/mm/mseal_test.c index faad4833366a..93c2e13094d4 100644 --- a/tools/testing/selftests/mm/mseal_test.c +++ b/tools/testing/selftests/mm/mseal_test.c @@ -234,7 +234,7 @@ static void test_seal_addseal(void) unsigned long size = 4 * page_size; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); ret = sys_mseal(ptr, size); FAIL_TEST_IF_FALSE(!ret); @@ -250,7 +250,7 @@ static void test_seal_unmapped_start(void) unsigned long size = 4 * page_size; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); /* munmap 2 pages from ptr. */ ret = sys_munmap(ptr, 2 * page_size); @@ -278,7 +278,7 @@ static void test_seal_unmapped_middle(void) unsigned long size = 4 * page_size; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); /* munmap 2 pages from ptr + page. */ ret = sys_munmap(ptr + page_size, 2 * page_size); @@ -310,7 +310,7 @@ static void test_seal_unmapped_end(void) unsigned long size = 4 * page_size; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); /* unmap last 2 pages. */ ret = sys_munmap(ptr + 2 * page_size, 2 * page_size); @@ -339,7 +339,7 @@ static void test_seal_multiple_vmas(void) unsigned long size = 4 * page_size; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); /* use mprotect to split the vma into 3. */ ret = sys_mprotect(ptr + page_size, 2 * page_size, @@ -370,7 +370,7 @@ static void test_seal_split_start(void) unsigned long size = 4 * page_size; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); /* use mprotect to split at middle */ ret = sys_mprotect(ptr, 2 * page_size, PROT_READ | PROT_WRITE); @@ -395,7 +395,7 @@ static void test_seal_split_end(void) unsigned long size = 4 * page_size; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); /* use mprotect to split at middle */ ret = sys_mprotect(ptr, 2 * page_size, PROT_READ | PROT_WRITE); @@ -420,7 +420,7 @@ static void test_seal_invalid_input(void) int ret; setup_single_address(8 * page_size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); ret = clean_single_address(ptr + 4 * page_size, 4 * page_size); FAIL_TEST_IF_FALSE(!ret); @@ -455,7 +455,7 @@ static void test_seal_zero_length(void) int ret; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); ret = sys_mprotect(ptr, 0, PROT_READ | PROT_WRITE); FAIL_TEST_IF_FALSE(!ret); @@ -505,7 +505,7 @@ static void test_seal_twice(void) unsigned long size = 4 * page_size; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); ret = sys_mseal(ptr, size); FAIL_TEST_IF_FALSE(!ret); @@ -525,7 +525,7 @@ static void test_seal_mprotect(bool seal) int ret; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); if (seal) { ret = seal_single_address(ptr, size); @@ -549,7 +549,7 @@ static void test_seal_start_mprotect(bool seal) int ret; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); if (seal) { ret = seal_single_address(ptr, page_size); @@ -579,7 +579,7 @@ static void test_seal_end_mprotect(bool seal) int ret; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); if (seal) { ret = seal_single_address(ptr + page_size, 3 * page_size); @@ -609,7 +609,7 @@ static void test_seal_mprotect_unalign_len(bool seal) int ret; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); if (seal) { ret = seal_single_address(ptr, page_size * 2 - 1); @@ -638,7 +638,7 @@ static void test_seal_mprotect_unalign_len_variant_2(bool seal) int ret; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); if (seal) { ret = seal_single_address(ptr, page_size * 2 + 1); FAIL_TEST_IF_FALSE(!ret); @@ -666,7 +666,7 @@ static void test_seal_mprotect_two_vma(bool seal) int ret; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); /* use mprotect to split */ ret = sys_mprotect(ptr, page_size * 2, PROT_READ | PROT_WRITE); @@ -701,7 +701,7 @@ static void test_seal_mprotect_two_vma_with_split(bool seal) int ret; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); /* use mprotect to split as two vma. */ ret = sys_mprotect(ptr, page_size * 2, PROT_READ | PROT_WRITE); @@ -748,7 +748,7 @@ static void test_seal_mprotect_partial_mprotect(bool seal) int ret; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); /* seal one page. */ if (seal) { @@ -780,7 +780,7 @@ static void test_seal_mprotect_partial_mprotect_tail(bool seal) */ setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); if (seal) { ret = sys_mseal(ptr + page_size, page_size); @@ -810,7 +810,7 @@ static void test_seal_mprotect_two_vma_with_gap(void) int ret; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); /* use mprotect to split. */ ret = sys_mprotect(ptr, page_size, PROT_READ | PROT_WRITE); @@ -853,7 +853,7 @@ static void test_seal_mprotect_split(bool seal) int ret; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); /* use mprotect to split. */ ret = sys_mprotect(ptr, page_size, PROT_READ | PROT_WRITE); @@ -890,7 +890,7 @@ static void test_seal_mprotect_merge(bool seal) int ret; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); /* use mprotect to split one page. */ ret = sys_mprotect(ptr, page_size, PROT_READ | PROT_WRITE); @@ -924,7 +924,7 @@ static void test_seal_munmap(bool seal) int ret; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); if (seal) { ret = sys_mseal(ptr, size); @@ -955,7 +955,7 @@ static void test_seal_munmap_two_vma(bool seal) int ret; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); /* use mprotect to split */ ret = sys_mprotect(ptr, page_size * 2, PROT_READ | PROT_WRITE); @@ -996,7 +996,7 @@ static void test_seal_munmap_vma_with_gap(bool seal) int ret; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); ret = sys_munmap(ptr + page_size, page_size * 2); FAIL_TEST_IF_FALSE(!ret); @@ -1028,7 +1028,7 @@ static void test_seal_munmap_partial_across_vmas(bool seal) int prot; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); if (seal) { ret = sys_mseal(ptr + page_size, page_size); @@ -1058,7 +1058,7 @@ static void test_munmap_start_freed(bool seal) int prot; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); /* unmap the first page. */ ret = sys_munmap(ptr, page_size); @@ -1097,7 +1097,7 @@ static void test_munmap_end_freed(bool seal) int ret; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); /* unmap last page. */ ret = sys_munmap(ptr + page_size * 3, page_size); @@ -1128,7 +1128,7 @@ static void test_munmap_middle_freed(bool seal) int prot; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); /* unmap 2 pages in the middle. */ ret = sys_munmap(ptr + page_size, page_size * 2); @@ -1172,7 +1172,7 @@ static void test_seal_mremap_shrink(bool seal) void *ret2; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); if (seal) { ret = sys_mseal(ptr, size); @@ -1201,7 +1201,7 @@ static void test_seal_mremap_expand(bool seal) void *ret2; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); /* ummap last 2 pages. */ ret = sys_munmap(ptr + 2 * page_size, 2 * page_size); FAIL_TEST_IF_FALSE(!ret); @@ -1233,9 +1233,9 @@ static void test_seal_mremap_move(bool seal) void *ret2; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); setup_single_address(size, &newPtr); - FAIL_TEST_IF_FALSE(newPtr != (void *)-1); + FAIL_TEST_IF_FALSE(newPtr != MAP_FAILED); ret = clean_single_address(newPtr, size); FAIL_TEST_IF_FALSE(!ret); @@ -1266,7 +1266,7 @@ static void test_seal_mmap_overwrite_prot(bool seal) void *ret2; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); if (seal) { ret = sys_mseal(ptr, size); @@ -1294,7 +1294,7 @@ static void test_seal_mmap_expand(bool seal) void *ret2; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); /* ummap last 4 pages. */ ret = sys_munmap(ptr + 8 * page_size, 4 * page_size); FAIL_TEST_IF_FALSE(!ret); @@ -1325,7 +1325,7 @@ static void test_seal_mmap_shrink(bool seal) void *ret2; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); if (seal) { ret = sys_mseal(ptr, size); @@ -1354,9 +1354,9 @@ static void test_seal_mremap_shrink_fixed(bool seal) void *ret2; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); setup_single_address(size, &newAddr); - FAIL_TEST_IF_FALSE(newAddr != (void *)-1); + FAIL_TEST_IF_FALSE(newAddr != MAP_FAILED); if (seal) { ret = sys_mseal(ptr, size); @@ -1385,9 +1385,9 @@ static void test_seal_mremap_expand_fixed(bool seal) void *ret2; setup_single_address(page_size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); setup_single_address(size, &newAddr); - FAIL_TEST_IF_FALSE(newAddr != (void *)-1); + FAIL_TEST_IF_FALSE(newAddr != MAP_FAILED); if (seal) { ret = sys_mseal(newAddr, size); @@ -1416,9 +1416,9 @@ static void test_seal_mremap_move_fixed(bool seal) void *ret2; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); setup_single_address(size, &newAddr); - FAIL_TEST_IF_FALSE(newAddr != (void *)-1); + FAIL_TEST_IF_FALSE(newAddr != MAP_FAILED); if (seal) { ret = sys_mseal(newAddr, size); @@ -1445,7 +1445,7 @@ static void test_seal_mremap_move_fixed_zero(bool seal) void *ret2; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); if (seal) { ret = sys_mseal(ptr, size); @@ -1476,7 +1476,7 @@ static void test_seal_mremap_move_dontunmap(bool seal) void *ret2; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); if (seal) { ret = sys_mseal(ptr, size); @@ -1505,7 +1505,7 @@ static void test_seal_mremap_move_dontunmap_anyaddr(bool seal) void *ret2; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); if (seal) { ret = sys_mseal(ptr, size); @@ -1517,7 +1517,7 @@ static void test_seal_mremap_move_dontunmap_anyaddr(bool seal) * use allocate/free to similate that. */ setup_single_address(size, &ptr2); - FAIL_TEST_IF_FALSE(ptr2 != (void *)-1); + FAIL_TEST_IF_FALSE(ptr2 != MAP_FAILED); ret = sys_munmap(ptr2, size); FAIL_TEST_IF_FALSE(!ret); @@ -1547,7 +1547,7 @@ static void test_seal_merge_and_split(void) /* (24 RO) */ setup_single_address(24 * page_size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); /* use mprotect(NONE) to set out boundary */ /* (1 NONE) (22 RO) (1 NONE) */ @@ -1637,7 +1637,7 @@ static void test_seal_discard_ro_anon_on_rw(bool seal) int ret; setup_single_address_rw(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); if (seal) { ret = sys_mseal(ptr, size); @@ -1669,7 +1669,7 @@ static void test_seal_discard_ro_anon_on_pkey(bool seal) SKIP_TEST_IF_FALSE(pkey_supported()); setup_single_address_rw(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); pkey = sys_pkey_alloc(0, PKEY_UNRESTRICTED); FAIL_TEST_IF_FALSE(pkey > 0); @@ -1751,7 +1751,7 @@ static void test_seal_discard_ro_anon_on_shared(bool seal) unsigned long mapflags = MAP_ANONYMOUS | MAP_SHARED; ptr = mmap(NULL, size, PROT_READ, mapflags, -1, 0); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); if (seal) { ret = sys_mseal(ptr, size); @@ -1779,7 +1779,7 @@ static void test_seal_discard_ro_anon(bool seal) int ret; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); if (seal) { ret = seal_single_address(ptr, size); @@ -1809,7 +1809,7 @@ static void test_seal_discard_across_vmas(bool seal) int ret; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); if (seal) { ret = seal_single_address(ptr + page_size, page_size); @@ -1840,7 +1840,7 @@ static void test_seal_madvise_nodiscard(bool seal) int ret; setup_single_address(size, &ptr); - FAIL_TEST_IF_FALSE(ptr != (void *)-1); + FAIL_TEST_IF_FALSE(ptr != MAP_FAILED); if (seal) { ret = seal_single_address(ptr, size); From 9922d4ced6c0f80427c8d49738581c34bd2d64da Mon Sep 17 00:00:00 2001 From: longlong yan Date: Tue, 21 Jul 2026 14:36:11 +0800 Subject: [PATCH 478/501] selftests/mm/pagemap_ioctl: fix missing NULL checks after calloc() The pagemap_ioctl selftest allocates memory via calloc() in several places but does not check the return values. If calloc() fails, the subsequent code will dereference a NULL pointer and crash. Additionally, in sanity_tests(), the calloc() failure check incorrectly uses MAP_FAILED (the mmap() error constant) instead of NULL. Since calloc() returns NULL on failure, the check never triggers and a failed allocation goes undetected. Add NULL checks after each calloc() call, and fix the wrong error constant in sanity_tests(). Use ksft_exit_fail_msg() consistent with the existing error handling pattern in the file. Link: https://lore.kernel.org/20260721063611.342-1-yanlonglong@kylinos.cn Signed-off-by: longlong yan Acked-by: David Hildenbrand (Arm) Reviewed-by: SJ Park Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/pagemap_ioctl.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/mm/pagemap_ioctl.c b/tools/testing/selftests/mm/pagemap_ioctl.c index f6ab626a90b4..1b2dffcc999b 100644 --- a/tools/testing/selftests/mm/pagemap_ioctl.c +++ b/tools/testing/selftests/mm/pagemap_ioctl.c @@ -213,6 +213,8 @@ int userfaultfd_tests(void) vec_size = mem_size/page_size; vec = calloc(vec_size, sizeof(struct page_region)); + if (!vec) + ksft_exit_fail_msg("error nomem\n"); written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, vec_size - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); @@ -700,6 +702,8 @@ int base_tests(char *prefix, char *mem, unsigned long long mem_size, int skip) vec_size = mem_size/page_size; vec = calloc(vec_size, sizeof(struct page_region)); vec2 = calloc(vec_size, sizeof(struct page_region)); + if (!vec || !vec2) + ksft_exit_fail_msg("error nomem\n"); /* 1. all new pages must be not be written (dirty) */ written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, @@ -1001,6 +1005,8 @@ int unmapped_region_tests(void) int written, len = 0x00040000; long vec_size = len / page_size; struct page_region *vec = calloc(vec_size, sizeof(struct page_region)); + if (!vec) + ksft_exit_fail_msg("error nomem\n"); /* 1. Get written pages */ written = pagemap_ioctl(start, len, vec, vec_size, 0, 0, @@ -1163,7 +1169,7 @@ int sanity_tests(void) vec = calloc(vec_size, sizeof(struct page_region)); mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); - if (mem == MAP_FAILED || vec == MAP_FAILED) + if (mem == MAP_FAILED || !vec) ksft_exit_fail_msg("error nomem\n"); wp_init(mem, mem_size); From 6fa0d03840d0be5f91bfd00afc43177cda46ab1a Mon Sep 17 00:00:00 2001 From: Sang-Heon Jeon Date: Thu, 9 Jul 2026 02:06:45 +0900 Subject: [PATCH 479/501] mm/early_ioremap: clarify early_ioremap_reset() semantics Patch series "mm/early_ioremap: clarify and clean up early_ioremap_reset()". __late_set_fixmap() and __late_clear_fixmap() are only used after early_ioremap_reset() has been called, but the comment above them does not say anything about that. So arm64, riscv and powerpc, whose __set_fixmap() works before and after paging_init(), describe the same situation in three different ways: calls reset defines the macros arm64 yes yes riscv no yes powerpc no no Patch 1 documents when early_ioremap_reset() needs to be called and that only architectures calling it need to define the macros. Patches 2 and 3 remove the unneeded riscv macros, which are unreachable, and the arm64 reset call and macros, which change nothing. No functional change. This patch (of 3): __late_set_fixmap() and __late_clear_fixmap() are only used after early_ioremap_reset() has been called. arm64, riscv and powerpc all have a __set_fixmap() that works before and after paging_init(), so they do not need to call early_ioremap_reset() or define the macros, but they describe the same situation in three different ways: calls reset defines the macros arm64 yes yes riscv no yes powerpc no no The existing comment is vague and allows all three. Replace it with comments that make it clear when the reset and the macros are needed. No functional change. Link: https://lore.kernel.org/20260708170647.362562-1-ekffu200098@gmail.com Link: https://lore.kernel.org/20260708170647.362562-2-ekffu200098@gmail.com Signed-off-by: Sang-Heon Jeon Cc: Albert Ou Cc: Alexandre Ghiti Cc: Catalin Marinas Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Palmer Dabbelt Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Will Deacon Signed-off-by: Andrew Morton --- mm/early_ioremap.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mm/early_ioremap.c b/mm/early_ioremap.c index 96c29b9dc85d..6215b90dfef3 100644 --- a/mm/early_ioremap.c +++ b/mm/early_ioremap.c @@ -47,15 +47,19 @@ pgprot_t __init __weak early_memremap_pgprot_adjust(resource_size_t phys_addr, return prot; } +/* + * Only architectures whose early_ioremap() must stop using __early_set_fixmap() + * after paging_init() need to call this. + */ void __init early_ioremap_reset(void) { after_paging_init = 1; } /* - * Generally, ioremap() is available after paging_init() has been called. - * Architectures wanting to allow early_ioremap after paging_init() can - * define __late_set_fixmap and __late_clear_fixmap to do the right thing. + * Only architectures that call early_ioremap_reset() need to define + * __late_set_fixmap() and __late_clear_fixmap(), which early_ioremap() uses + * instead of __early_set_fixmap() after the reset. */ #ifndef __late_set_fixmap static inline void __init __late_set_fixmap(enum fixed_addresses idx, From 36d7f1b62b54a0036096ff93c965f1bd2b22420f Mon Sep 17 00:00:00 2001 From: Sang-Heon Jeon Date: Thu, 9 Jul 2026 02:06:46 +0900 Subject: [PATCH 480/501] riscv: remove unused __late_set_fixmap() and __late_clear_fixmap() __late_set_fixmap() and __late_clear_fixmap() are only used after early_ioremap_reset() has been called. riscv never calls it because __set_fixmap() works before and after paging_init(). So remove them. No functional change. Link: https://lore.kernel.org/20260708170647.362562-3-ekffu200098@gmail.com Signed-off-by: Sang-Heon Jeon Cc: Albert Ou Cc: Alexandre Ghiti Cc: Catalin Marinas Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Palmer Dabbelt Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/riscv/include/asm/fixmap.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/riscv/include/asm/fixmap.h b/arch/riscv/include/asm/fixmap.h index 0a55099bb734..a81823c5ee4a 100644 --- a/arch/riscv/include/asm/fixmap.h +++ b/arch/riscv/include/asm/fixmap.h @@ -55,9 +55,6 @@ enum fixed_addresses { #define __early_set_fixmap __set_fixmap -#define __late_set_fixmap __set_fixmap -#define __late_clear_fixmap(idx) __set_fixmap((idx), 0, FIXMAP_PAGE_CLEAR) - extern void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot); From 23e82f0223047c58c999f6147644d3bb237081ee Mon Sep 17 00:00:00 2001 From: Sang-Heon Jeon Date: Thu, 9 Jul 2026 02:06:47 +0900 Subject: [PATCH 481/501] arm64: remove early_ioremap_reset() call and __late_* macros On arm64, __early_set_fixmap(), __late_set_fixmap() and __late_clear_fixmap() are all __set_fixmap(). Calling early_ioremap_reset() changes nothing. So remove the call and the macros. No functional change. Link: https://lore.kernel.org/20260708170647.362562-4-ekffu200098@gmail.com Signed-off-by: Sang-Heon Jeon Acked-by: Will Deacon Cc: Albert Ou Cc: Alexandre Ghiti Cc: Catalin Marinas Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Palmer Dabbelt Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- arch/arm64/include/asm/fixmap.h | 3 --- arch/arm64/kernel/setup.c | 2 -- 2 files changed, 5 deletions(-) diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h index 7075a3bd2c61..170c3502d723 100644 --- a/arch/arm64/include/asm/fixmap.h +++ b/arch/arm64/include/asm/fixmap.h @@ -114,9 +114,6 @@ void __init early_fixmap_init(void); #define __early_set_fixmap __set_fixmap -#define __late_set_fixmap __set_fixmap -#define __late_clear_fixmap(idx) __set_fixmap((idx), 0, FIXMAP_PAGE_CLEAR) - extern void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot); #include diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 23c05dc7a8f2..29c6100f0c50 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -347,8 +347,6 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) request_standard_resources(); - early_ioremap_reset(); - if (acpi_disabled) psci_dt_init(); else From 51fa46c364039d5f44737f24e2bf22ca5e143d0f Mon Sep 17 00:00:00 2001 From: Song Hu Date: Tue, 21 Jul 2026 07:00:08 -0700 Subject: [PATCH 482/501] mm/damon: update outdated comment about DAMOS filter handling The kernel-doc comment above enum damos_filter_type states that only the anon and memcg type filters are handled by damon_operations (and therefore accounted as 'tried'), and that DAMON_OPS_VADDR and DAMON_OPS_FVADDR do not support those two filter types. Neither is accurate anymore. damos_filter_for_ops() routes every filter type except ADDR and TARGET to the operations layer, and the VADDR and FVADDR operations (the latter being a copy of the former) handle all of those types through damos_folio_filter_match() / damos_va_filter_out(). Update the comment to match the code. Link: https://lore.kernel.org/20260721140011.269802-1-sj@kernel.org Signed-off-by: Song Hu Reviewed-by: SJ Park Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 327f534696aa..0c8b7ddef9ab 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -398,16 +398,15 @@ struct damos_stat { * @DAMOS_FILTER_TYPE_TARGET: Data Access Monitoring target. * @NR_DAMOS_FILTER_TYPES: Number of filter types. * - * The anon pages type and memcg type filters are handled by underlying - * &struct damon_operations as a part of scheme action trying, and therefore - * accounted as 'tried'. In contrast, other types are handled by core layer - * before trying of the action and therefore not accounted as 'tried'. + * All types except &DAMOS_FILTER_TYPE_ADDR and &DAMOS_FILTER_TYPE_TARGET + * are handled by the underlying &struct damon_operations as a part of scheme + * action trying, and therefore accounted as 'tried'. In contrast, + * &DAMOS_FILTER_TYPE_ADDR and &DAMOS_FILTER_TYPE_TARGET filters are handled + * by the core layer before trying of the action, and therefore not accounted + * as 'tried'. * - * The support of the filters that handled by &struct damon_operations depend - * on the running &struct damon_operations. - * &enum DAMON_OPS_PADDR supports both anon pages type and memcg type filters, - * while &enum DAMON_OPS_VADDR and &enum DAMON_OPS_FVADDR don't support any of - * the two types. + * Support for the operations-handled filters depends on the running + * &struct damon_operations. */ enum damos_filter_type { DAMOS_FILTER_TYPE_ANON, From 613c2d5552d2f5f631a6738e01eb95f660854590 Mon Sep 17 00:00:00 2001 From: Jiahui Zhang Date: Tue, 21 Jul 2026 06:56:05 -0700 Subject: [PATCH 483/501] mm/damon/ops-common: prevent migration fallback to non-target nodes DAMOS_MIGRATE_{HOT,COLD} passes a target NUMA node to migrate_pages(). But alloc_migration_target() only treats mtc->nid as a preferred node unless __GFP_THISNODE is set. Hence target allocation can fall back to another node, and migrate_pages() can report success without placing the folio on the requested target node. Consider a two-node tiered system where node 0 is a fast tier and node 1 is a CPU-less slow tier such as CXL memory, and the user wants to promote hot regions from node 1 to node 0 with a command like: sudo damo start --ops vaddr --target_pid ${workload_pid} \ --damos_action migrate_hot 0 \ --damos_access_rate 70% max Without the __GFP_THISNODE flag, when the memory allocator finds that node 0 is nearly full, it can fall back to node 1 without waking up kswapd. Then the pages allocated for migrate_pages() are still on node 1, and the regions that are expected to be promoted to node 0 are only moved to different physical pages on node 1. Meanwhile, both the mm_migrate_pages tracepoint and DAMOS's own sz_applied statistics (reported via the damos_stat_after_apply_interval tracepoint) show the migrations as successful, which makes the failure practically invisible and hard to investigate. Running a demotion-purpose DAMOS scheme alongside the promotion scheme does not fully avoid this either. If demotion cannot keep up with the promotion rate, allocation can still fall back to node 1 during promotion, and the same misleading statistics show up. Make DAMON's migration target allocation strict by setting __GFP_THISNODE, so that a failed allocation on the target node is reported as a failure instead of silently landing on a different node. This is consistent with alloc_misplaced_dst_folio(), alloc_demote_folio(), and with do_move_pages_to_node(), which all use __GFP_THISNODE for migrations to an explicit destination node. Link: https://lore.kernel.org/20260721135607.251869-1-sj@kernel.org Signed-off-by: Jiahui Zhang Reviewed-by: SJ Park Signed-off-by: SJ Park Cc: Honggyu Kim Signed-off-by: Andrew Morton --- mm/damon/ops-common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index d7d7f100389b..e59f77eca83b 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -312,7 +312,7 @@ static unsigned int __damon_migrate_folio_list( * instead of migrated. */ .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | - __GFP_NOMEMALLOC | GFP_NOWAIT, + __GFP_NOMEMALLOC | GFP_NOWAIT | __GFP_THISNODE, .nid = target_nid, }; From 64a853c9a648be315ba92d50270791cf40ce0708 Mon Sep 17 00:00:00 2001 From: Xuewen Wang Date: Tue, 21 Jul 2026 06:53:31 -0700 Subject: [PATCH 484/501] mm/damon: remove trailing semicolons after function definitions Three function definitions terminate with '};' instead of '}', which is unnecessary and inconsistent with kernel coding style: - damon_pa_initcall() in paddr.c - damon_va_initcall() in vaddr.c - damos_get_some_mem_psi_total() in core.c No functional change intended. Link: https://lore.kernel.org/20260721135333.241106-1-sj@kernel.org Signed-off-by: Xuewen Wang Reviewed-by: SJ Park Signed-off-by: SJ Park Signed-off-by: Andrew Morton --- mm/damon/core.c | 2 +- mm/damon/paddr.c | 2 +- mm/damon/vaddr.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index e0122376f43d..644daf5a1656 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -2791,7 +2791,7 @@ static u64 damos_get_some_mem_psi_total(void) static inline u64 damos_get_some_mem_psi_total(void) { return 0; -}; +} #endif /* CONFIG_PSI */ diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index e4f98d67461f..5a6a78054784 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -447,6 +447,6 @@ static int __init damon_pa_initcall(void) }; return damon_register_ops(&ops); -}; +} subsys_initcall(damon_pa_initcall); diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index d487b7a4a104..4976775fe276 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -959,7 +959,7 @@ static int __init damon_va_initcall(void) if (err) return err; return damon_register_ops(&ops_fvaddr); -}; +} subsys_initcall(damon_va_initcall); From 47f3cecd720f8f8d94ad2f206e3bfe3bc2d3d4e5 Mon Sep 17 00:00:00 2001 From: Yichong Chen Date: Tue, 21 Jul 2026 11:52:07 +0800 Subject: [PATCH 485/501] hugetlb: evaluate subpool free state while locked unlock_or_release_subpool() drops spool->lock before calling subpool_is_free(). However, subpool_is_free() reads fields that are updated under spool->lock, including count, used_hpages and rsv_hpages. Keep the free-state evaluation under the same lock that protects those fields. The reservation accounting and kfree() calls still happen after dropping spool->lock. Link: https://lore.kernel.org/20260721035207.1437935-1-chenyichong@uniontech.com Signed-off-by: Yichong Chen Reviewed-by: Joshua Hahn Reviewed-by: Jane Chu Cc: David Hildenbrand Cc: Muchun Song Cc: Oscar Salvador Signed-off-by: Andrew Morton --- mm/hugetlb.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 6daf831b14c5..79e5c3b3e850 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -141,12 +141,14 @@ static inline bool subpool_is_free(struct hugepage_subpool *spool) static inline void unlock_or_release_subpool(struct hugepage_subpool *spool, unsigned long irq_flags) { - spin_unlock_irqrestore(&spool->lock, irq_flags); + bool free_subpool = subpool_is_free(spool); /* If no pages are used, and no other handles to the subpool * remain, give up any reservations based on minimum size and * free the subpool */ - if (subpool_is_free(spool)) { + spin_unlock_irqrestore(&spool->lock, irq_flags); + + if (free_subpool) { if (spool->min_hpages != -1) hugetlb_acct_memory(spool->hstate, -spool->min_hpages); From 37864d0bf5a4d60da6109b9078cc78ef3fc81216 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 22 Jul 2026 10:56:44 -0400 Subject: [PATCH 486/501] mm: page_alloc: __GFP_FS lockdep annotation for direct compaction Patch series "mm: fix reclaim storms in defrag_mode", v2. As we deployed vm.defrag_mode=1 in Meta production, some workloads regressed with recurring pressure spikes and swap storms (which in turn triggered userspace OOM rules on pressure and swap utilization levels). Tracing pinned this to non-movable requests spinning and reclaiming unproductively when kswapd/kcompactd are overwhelmed. Direct reclaim predominantly frees up pages in movable blocks, but those requests cannot use that space under defrag_mode rules; and it is unlikely to free up whole blocks incidentally for __rmqueue_claim() to work. This series fixes it by making non-movable requests participate in pageblock production in the allocator slowpath - meaning, they will invoke direct reclaim and direct compaction with pageblock_order. That requires some small-ish adjustments up front in the allocator and the compaction code: three prep patches and the fix last. The series has been in production against one of the affected workloads for several weeks and restores the OOM kill rate to !defrag_mode baseline. This patch (of 4): A subsequent patch will have some order-0 allocations participate in compaction under defrag_mode, to stave off extfrag events. Since this is a sprawling expansion of entry points, and compaction can enter filesystem paths, add lockdep annotations that catches __GFP_FS passing errors. Direct reclaim has had this annotation for a while, and since reclaim and compaction are usually used in conjunction, this is unlikely to unearth old bugs. It's more about future proofing and peace of mind. Link: https://lore.kernel.org/20260722150006.3848560-1-hannes@cmpxchg.org Link: https://lore.kernel.org/20260722150006.3848560-2-hannes@cmpxchg.org Fixes: e3aa7df331bc ("mm: page_alloc: defrag_mode") Signed-off-by: Johannes Weiner Reviewed-by: Vlastimil Babka (SUSE) Acked-by: Shakeel Butt Cc: Brendan Jackman Cc: Brendan Jackman Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Zi Yan Cc: Brendan Jackman Cc: Gregory Price Cc: Signed-off-by: Andrew Morton --- mm/page_alloc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index f93a6bb9a872..36cdec9c5c73 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -4149,12 +4149,14 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, psi_memstall_enter(&pflags); delayacct_compact_start(); + fs_reclaim_acquire(gfp_mask); noreclaim_flag = memalloc_noreclaim_save(); *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, prio, &page); memalloc_noreclaim_restore(noreclaim_flag); + fs_reclaim_release(gfp_mask); psi_memstall_leave(&pflags); delayacct_compact_end(); From 1b4b697a5743e624cc259d6c31a8d8716b796305 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 22 Jul 2026 10:56:45 -0400 Subject: [PATCH 487/501] mm: compaction: support non-movable compaction for pageblock requests While trying to fix a reclaim storm in defrag_mode, I noticed that non-movable direct compaction is extremely inefficient. When searching for space to evacuate, compaction only allows blocks of the same type as the incoming request. This is to prevent migratetype pollution, where a small non-movable request frees space in a movable block and provokes the allocator to fall back and pollute it. This protection is reasonable on one hand, but the downside is that it makes non-movable direct compaction nearly useless: if we get the type annotations right, by definition there aren't any movable pages inside the non-movable blocks it is allowed to scan. With defrag_mode, the goal is the production of whole blocks, which are essentially type neutral: __rmqueue_claim() will convert them wholesale on alloc. This makes type mixing and pollution a non-issue. Fix the pollution gates to take the requested order into account, and allow whole-block requests to scan blocks of other types. The only exception is CMA blocks. That type is sticky and these blocks cannot be claimed to other types. Continue to be strict with them, and allow only explicit ALLOC_CMA requests and kcompactd to evacuate them. Link: https://lore.kernel.org/20260722150006.3848560-3-hannes@cmpxchg.org Fixes: e3aa7df331bc ("mm: page_alloc: defrag_mode") Signed-off-by: Johannes Weiner Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Gregory Price Cc: Brendan Jackman Cc: Brendan Jackman Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shakeel Butt Cc: Suren Baghdasaryan Cc: Zi Yan Cc: Signed-off-by: Andrew Morton --- mm/compaction.c | 46 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/mm/compaction.c b/mm/compaction.c index 9f81055a358e..173a250309b7 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -1381,12 +1381,44 @@ static bool suitable_migration_source(struct compact_control *cc, if (pageblock_skip_persistent(page)) return false; - if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction) + /* + * Background compaction produces blocks for the zone at + * large, with no particular allocation context. Allow all + * block types, including CMA. + */ + if (!cc->direct_compaction) return true; block_mt = get_pageblock_migratetype(page); - if (cc->migratetype == MIGRATE_MOVABLE) + /* + * CMA pages can only be taken by ALLOC_CMA requests. For anybody + * else, vacating a CMA block consumes free pages the caller + * could have used, and produces free pages it cannot. + */ + if (is_migrate_cma(block_mt) && !(cc->alloc_flags & ALLOC_CMA)) + return false; + + /* + * Per default, scans are restricted to blocks compatible with + * the request, to prevent cross-contamination. Once + * compaction priority escalates to synchronous scans, though, + * scan all blocks to try to make forward progress. For + * movable request, this likely helps little: there shouldn't + * be many migratable pages inside non-movable blocks besides + * allocator fallbacks. For non-movable requests, this helps a + * lot, as they can finally scan movable blocks. + */ + if (cc->mode != MIGRATE_ASYNC) + return true; + + /* + * Prevent migratetype == MIGRATE_MOVABLE || cc->order >= pageblock_order) return is_migrate_movable(block_mt); else return block_mt == cc->migratetype; @@ -1972,12 +2004,12 @@ static unsigned long fast_find_migrateblock(struct compact_control *cc) return pfn; /* - * Only allow kcompactd and direct requests for movable pages to - * quickly clear out a MOVABLE pageblock for allocation. This - * reduces the risk that a large movable pageblock is freed for - * an unmovable/reclaimable small allocation. + * Prevent direct_compaction && cc->migratetype != MIGRATE_MOVABLE) + if (cc->direct_compaction && cc->migratetype != MIGRATE_MOVABLE && + cc->order < pageblock_order) return pfn; /* From aee220f565cce38f0efcff940ae2b44bdc495408 Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Wed, 22 Jul 2026 10:56:46 -0400 Subject: [PATCH 488/501] mm: page_alloc: move capture_control to the page allocator The compaction capturing code assumes the allocation request order and compaction target order are the same. That won't be true once defrag_mode promotes sub-block allocations to pageblock-order compaction: compaction targets the larger order, while capture should remain at the original allocation order. Move the capture_control to the page allocator and give it its own copies of what the page freeing path matches against - zone, migratetype and the allocation order - rather than reaching into compaction's live compact_control. __alloc_pages_direct_compact() fills in migratetype and order, and installs and hides current->capture_control around the whole compaction call; try_to_compact_pages() aims capc->zone at each zone while it is being compacted. compact_zone_order() no longer deals with capture at all. Pass the capture_control through try_to_compact_pages() / compact_zone_order() in place of the bare struct page **. No functional change. Link: https://lore.kernel.org/20260722150006.3848560-4-hannes@cmpxchg.org Fixes: e3aa7df331bc ("mm: page_alloc: defrag_mode") Signed-off-by: Vlastimil Babka (SUSE) Co-developed-by: Johannes Weiner Signed-off-by: Johannes Weiner Reviewed-by: Gregory Price Cc: Brendan Jackman Cc: Brendan Jackman Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shakeel Butt Cc: Suren Baghdasaryan Cc: Zi Yan Cc: Signed-off-by: Andrew Morton --- include/linux/compaction.h | 3 ++- mm/compaction.c | 50 +++++++++++--------------------------- mm/internal.h | 4 ++- mm/page_alloc.c | 45 ++++++++++++++++++++++++++++------ 4 files changed, 57 insertions(+), 45 deletions(-) diff --git a/include/linux/compaction.h b/include/linux/compaction.h index f29ef0653546..66a2f70e9e01 100644 --- a/include/linux/compaction.h +++ b/include/linux/compaction.h @@ -58,6 +58,7 @@ enum compact_result { }; struct alloc_context; /* in mm/internal.h */ +struct capture_control; /* in mm/internal.h */ /* * Number of free order-0 pages that should be available above given watermark @@ -92,7 +93,7 @@ extern int fragmentation_index(struct zone *zone, unsigned int order); extern enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order, unsigned int alloc_flags, const struct alloc_context *ac, enum compact_priority prio, - struct page **page); + struct capture_control *capc); extern void reset_isolation_suitable(pg_data_t *pgdat); extern bool compaction_suitable(struct zone *zone, int order, unsigned long watermark, int highest_zoneidx); diff --git a/mm/compaction.c b/mm/compaction.c index 173a250309b7..0568623d9384 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -2800,9 +2800,8 @@ compact_zone(struct compact_control *cc, struct capture_control *capc) static enum compact_result compact_zone_order(struct zone *zone, int order, gfp_t gfp_mask, enum compact_priority prio, unsigned int alloc_flags, int highest_zoneidx, - struct page **capture) + struct capture_control *capc) { - enum compact_result ret; struct compact_control cc = { .order = order, .search_order = order, @@ -2817,38 +2816,8 @@ static enum compact_result compact_zone_order(struct zone *zone, int order, .ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY), .ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY) }; - struct capture_control capc = { - .cc = &cc, - .page = NULL, - }; - /* - * Make sure the structs are really initialized before we expose the - * capture control, in case we are interrupted and the interrupt handler - * frees a page. - */ - barrier(); - WRITE_ONCE(current->capture_control, &capc); - - ret = compact_zone(&cc, &capc); - - /* - * Make sure we hide capture control first before we read the captured - * page pointer, otherwise an interrupt could free and capture a page - * and we would leak it. - */ - WRITE_ONCE(current->capture_control, NULL); - *capture = READ_ONCE(capc.page); - /* - * Technically, it is also possible that compaction is skipped but - * the page is still captured out of luck(IRQ came and freed the page). - * Returning COMPACT_SUCCESS in such cases helps in properly accounting - * the COMPACT[STALL|FAIL] when compaction is skipped. - */ - if (*capture) - ret = COMPACT_SUCCESS; - - return ret; + return compact_zone(&cc, capc); } /** @@ -2858,13 +2827,13 @@ static enum compact_result compact_zone_order(struct zone *zone, int order, * @alloc_flags: The allocation flags of the current allocation * @ac: The context of current allocation * @prio: Determines how hard direct compaction should try to succeed - * @capture: Pointer to free page created by compaction will be stored here + * @capc: Free page capture bypassing the freelist * * This is the main entry point for direct page compaction. */ enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order, unsigned int alloc_flags, const struct alloc_context *ac, - enum compact_priority prio, struct page **capture) + enum compact_priority prio, struct capture_control *capc) { struct zoneref *z; struct zone *zone; @@ -2891,8 +2860,17 @@ enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order, continue; } + WRITE_ONCE(capc->zone, zone); + status = compact_zone_order(zone, order, gfp_mask, prio, - alloc_flags, ac->highest_zoneidx, capture); + alloc_flags, ac->highest_zoneidx, capc); + + WRITE_ONCE(capc->zone, NULL); + + /* Stop if a page has been captured */ + if (READ_ONCE(capc->page)) + status = COMPACT_SUCCESS; + rc = max(status, rc); /* The allocation should succeed, stop compacting */ diff --git a/mm/internal.h b/mm/internal.h index 0f49cff67a2b..f28003f59812 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -868,7 +868,9 @@ struct compact_control { * immediately when one is created during the free path. */ struct capture_control { - struct compact_control *cc; + struct zone *zone; + int migratetype; + int order; struct page *page; }; diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 36cdec9c5c73..4905f3596998 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -725,14 +725,14 @@ static inline struct capture_control *task_capc(struct zone *zone) return unlikely(capc) && !(current->flags & PF_KTHREAD) && !capc->page && - capc->cc->zone == zone ? capc : NULL; + capc->zone == zone ? capc : NULL; } static inline bool compaction_capture(struct capture_control *capc, struct page *page, int order, int migratetype) { - if (!capc || order != capc->cc->order) + if (!capc || order != capc->order) return false; /* Do not accidentally pollute CMA or isolated regions*/ @@ -748,12 +748,12 @@ compaction_capture(struct capture_control *capc, struct page *page, * have trouble finding a high-order free page. */ if (order < pageblock_order && migratetype == MIGRATE_MOVABLE && - capc->cc->migratetype != MIGRATE_MOVABLE) + capc->migratetype != MIGRATE_MOVABLE) return false; - if (migratetype != capc->cc->migratetype) - trace_mm_page_alloc_extfrag(page, capc->cc->order, order, - capc->cc->migratetype, migratetype); + if (migratetype != capc->migratetype) + trace_mm_page_alloc_extfrag(page, capc->order, order, + capc->migratetype, migratetype); capc->page = page; return true; @@ -4143,6 +4143,12 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, struct page *page = NULL; unsigned long pflags; unsigned int noreclaim_flag; + struct capture_control capc = { + .zone = NULL, + .migratetype = ac->migratetype, + .order = order, + .page = NULL, + }; if (!order) return NULL; @@ -4152,8 +4158,33 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, fs_reclaim_acquire(gfp_mask); noreclaim_flag = memalloc_noreclaim_save(); + /* + * Make sure the structs are really initialized before we expose the + * capture control, in case we are interrupted and the interrupt handler + * frees a page. + */ + barrier(); + WRITE_ONCE(current->capture_control, &capc); + *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, - prio, &page); + prio, &capc); + + /* + * Make sure we hide capture control first before we read the captured + * page pointer, otherwise an interrupt could free and capture a page + * and we would leak it. + */ + WRITE_ONCE(current->capture_control, NULL); + page = READ_ONCE(capc.page); + + /* + * Technically, it is also possible that compaction is skipped but + * the page is still captured out of luck(IRQ came and freed the page). + * Returning COMPACT_SUCCESS in such cases helps in properly accounting + * the COMPACT[STALL|FAIL] when compaction is skipped. + */ + if (page) + *compact_result = COMPACT_SUCCESS; memalloc_noreclaim_restore(noreclaim_flag); fs_reclaim_release(gfp_mask); From 7e8756d7ad22655b935c384f123071aa9de07a27 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 22 Jul 2026 10:56:47 -0400 Subject: [PATCH 489/501] mm: page_alloc: fix non-movable reclaim storm in defrag_mode As we deployed defrag_mode into Meta production, pressure spikes and excessive swapping were observed on some workloads. Tracing confirmed that this is unmovable/reclaimable requests spinning in the allocator and direct reclaim, causing excessive amounts of swap. The initial plan for defrag_mode was to rely on kswapd/kcompactd to produce blocks, and if those are overwhelmed under high pressure, let the allocator fall back (__rmqueue_steal()) after its retry loops. However, that retrying results in more reclaim on some of these workloads than we'd hoped, sometimes excessively so, spurred on by the !costly order conditions in should_reclaim_retry(). The storms are dependent on the request type. Reclaim will inevitably make room in existing movable blocks, since that's where the LRU pages live. So if movable requests retry on reclaim, they make progress. When non-movable requests spin in reclaim that isn't productive. They cannot use the individually freed pages, and the process is unlikely to accidentally free whole blocks to meet the ALLOC_NOFRAGMENT bar. They spin and overreclaim excessively, which tanks performance and triggers userspace guards like swap exhaustion or pressure based OOM. To fix this, send non-movable requests, regardless of order, into pageblock reclaim/compaction. This way, they help move things along to meet the ALLOC_NOFRAGMENT bar. After this patch, the reclaim storms and excess OOM rates are no longer observed in production. The longer-term plan is still to have all requests, including the movable ones, help make blocks to spread the cost of defragmenting more evenly and fairly; combined with proper watermarking to reduce allocation latencies in the common case. However, doing this naively unearths scaling and concurrency limitations in compaction that need to be addressed first. Promoting just non-movables for now is the minimally viable bug fix for the above issue. [brendan.jackman@linux.dev: fix try_to_compact_pages() kerneldoc] Link: https://lore.kernel.org/DK7NM9RPUJOD.11PNJJ5N2OBED@linux.dev Link: https://lore.kernel.org/20260722150006.3848560-5-hannes@cmpxchg.org Fixes: e3aa7df331bc ("mm: page_alloc: defrag_mode") Signed-off-by: Johannes Weiner Signed-off-by: "Brendan Jackman" Reviewed-by: Vlastimil Babka (SUSE) Cc: Brendan Jackman Cc: David Hildenbrand Cc: Gregory Price Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Shakeel Butt Cc: Suren Baghdasaryan Cc: Zi Yan Cc: Signed-off-by: Andrew Morton --- mm/compaction.c | 2 +- mm/internal.h | 6 ++++++ mm/page_alloc.c | 31 ++++++++++++++++++++++++++----- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/mm/compaction.c b/mm/compaction.c index 0568623d9384..a049415512c6 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -2823,7 +2823,7 @@ static enum compact_result compact_zone_order(struct zone *zone, int order, /** * try_to_compact_pages - Direct compact to satisfy a high-order allocation * @gfp_mask: The GFP mask of the current allocation - * @order: The order of the current allocation + * @order: The order to try to make available * @alloc_flags: The allocation flags of the current allocation * @ac: The context of current allocation * @prio: Determines how hard direct compaction should try to succeed diff --git a/mm/internal.h b/mm/internal.h index f28003f59812..68db5abd0a4c 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -870,6 +870,12 @@ struct compact_control { struct capture_control { struct zone *zone; int migratetype; + /* + * Allocation request order. May differ from the compaction + * order: defrag_mode promotes sub-block allocations to + * pageblock-order compaction; capture still matches at the + * original allocation order so prep_new_page() is consistent. + */ int order; struct page *page; }; diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 4905f3596998..083cbcb5bdde 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -4149,8 +4149,24 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, .order = order, .page = NULL, }; + int compact_order = order; - if (!order) + /* + * If fallbacks are not permitted (defrag_mode), we either + * need to reclaim space in a block of matching type, or clear + * out an entire block to allow __rmqueue_claim() to convert. + * + * Reclaim by itself is primarily freeing space in movable + * blocks, since that's where the LRU pages live. So this + * works for movable requests, but not for others. + * + * For those, promote the order to help make blocks, instead + * of spinning in reclaim alone unproductively. + */ + if ((alloc_flags & ALLOC_NOFRAGMENT) && ac->migratetype != MIGRATE_MOVABLE) + compact_order = max(order, pageblock_order); + + if (!compact_order) return NULL; psi_memstall_enter(&pflags); @@ -4166,8 +4182,8 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, barrier(); WRITE_ONCE(current->capture_control, &capc); - *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, - prio, &capc); + *compact_result = try_to_compact_pages(gfp_mask, compact_order, + alloc_flags, ac, prio, &capc); /* * Make sure we hide capture control first before we read the captured @@ -4212,7 +4228,7 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, struct zone *zone = page_zone(page); zone->compact_blockskip_flush = false; - compaction_defer_reset(zone, order, true); + compaction_defer_reset(zone, compact_order, true); count_vm_event(COMPACTSUCCESS); return page; } @@ -4452,9 +4468,14 @@ __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, struct page *page = NULL; unsigned long pflags; bool drained = false; + int reclaim_order = order; + + /* Match the slowpath compaction promotion in __alloc_pages_direct_compact */ + if ((alloc_flags & ALLOC_NOFRAGMENT) && ac->migratetype != MIGRATE_MOVABLE) + reclaim_order = max(order, pageblock_order); psi_memstall_enter(&pflags); - *did_some_progress = __perform_reclaim(gfp_mask, order, ac); + *did_some_progress = __perform_reclaim(gfp_mask, reclaim_order, ac); if (unlikely(!(*did_some_progress))) goto out; From 5716a333c983dc374e9d4f8d599375fede76cf90 Mon Sep 17 00:00:00 2001 From: zhaozhengzhuo Date: Fri, 24 Jul 2026 15:46:03 +0800 Subject: [PATCH 490/501] selftests/mm: fix gup_longterm EINVAL error message The gup_longterm test prints a literal "n" when PIN_LONGTERM_TEST_START fails with EINVAL because the string is missing the newline escape sequence. Print a newline instead. Link: https://lore.kernel.org/23557F4CB8CF36FF+20260724074603.1479243-1-zhaozhengzhuo@uniontech.com Signed-off-by: zhaozhengzhuo Reviewed-by: Sarthak Sharma Reviewed-by: Dev Jain Acked-by: David Hildenbrand (arm) Cc: Jason Gunthorpe Cc: John Hubbard Cc: Peter Xu Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/gup_longterm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/mm/gup_longterm.c b/tools/testing/selftests/mm/gup_longterm.c index eb8963e9d98f..c03b4f8910c0 100644 --- a/tools/testing/selftests/mm/gup_longterm.c +++ b/tools/testing/selftests/mm/gup_longterm.c @@ -196,7 +196,7 @@ static void do_test(int fd, size_t size, enum test_type type, bool shared) args.flags |= rw ? PIN_LONGTERM_TEST_FLAG_USE_WRITE : 0; ret = ioctl(gup_fd, PIN_LONGTERM_TEST_START, &args); if (ret && errno == EINVAL) { - ksft_print_msg("PIN_LONGTERM_TEST_START failed (EINVAL)n"); + ksft_print_msg("PIN_LONGTERM_TEST_START failed (EINVAL)\n"); result = KSFT_SKIP; break; } else if (ret && errno == EFAULT) { From dbb7ba9c7fa81a48ed2a108ad684cf3655f3ae4b Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Tue, 14 Jul 2026 16:12:59 +0200 Subject: [PATCH 491/501] zram: move lockmap to be per-zram instead per table Patch series "zram: lockmap tweaks". This patch (of 2): The zram object contains an array zram_table_entry. Each one has a `lock' variable and each has a matching struct lockdep_map. This mimics a struct mutex. It uses always the same key for all lockdep_map instances. This makes it look like the same lock to lockdep. Therefore it could be reduced to have one lockdep_map per struct zram. Use only one struct lockdep_map per struct zram. Link: https://lore.kernel.org/20260714141300.3945672-1-bigeasy@linutronix.de Link: https://lore.kernel.org/20260714141300.3945672-2-bigeasy@linutronix.de Signed-off-by: Sebastian Andrzej Siewior Reviewed-by: Sergey Senozhatsky Tested-by: Sergey Senozhatsky Cc: Jens Axboe Cc: Minchan Kim Signed-off-by: Andrew Morton --- drivers/block/zram/zram_drv.c | 21 +++++++++------------ drivers/block/zram/zram_drv.h | 2 +- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index ace65c586072..4223002d80b4 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -57,14 +57,12 @@ static size_t huge_class_size; static const struct block_device_operations zram_devops; static void slot_free(struct zram *zram, u32 index); -#define slot_dep_map(zram, index) (&(zram)->table[(index)].dep_map) -static void slot_lock_init(struct zram *zram, u32 index) +static void slot_lock_init(struct zram *zram) { static struct lock_class_key __key; - lockdep_init_map(slot_dep_map(zram, index), "zram->table[index].lock", - &__key, 0); + lockdep_init_map(&zram->table_lock_map, "zram->table[index].lock", &__key, 0); } /* @@ -84,8 +82,8 @@ static __must_check bool slot_trylock(struct zram *zram, u32 index) unsigned long *lock = &zram->table[index].__lock; if (!test_and_set_bit_lock(ZRAM_ENTRY_LOCK, lock)) { - mutex_acquire(slot_dep_map(zram, index), 0, 1, _RET_IP_); - lock_acquired(slot_dep_map(zram, index), _RET_IP_); + mutex_acquire(&zram->table_lock_map, 0, 1, _RET_IP_); + lock_acquired(&zram->table_lock_map, _RET_IP_); return true; } @@ -96,16 +94,16 @@ static void slot_lock(struct zram *zram, u32 index) { unsigned long *lock = &zram->table[index].__lock; - mutex_acquire(slot_dep_map(zram, index), 0, 0, _RET_IP_); + mutex_acquire(&zram->table_lock_map, 0, 0, _RET_IP_); wait_on_bit_lock(lock, ZRAM_ENTRY_LOCK, TASK_UNINTERRUPTIBLE); - lock_acquired(slot_dep_map(zram, index), _RET_IP_); + lock_acquired(&zram->table_lock_map, _RET_IP_); } static void slot_unlock(struct zram *zram, u32 index) { unsigned long *lock = &zram->table[index].__lock; - mutex_release(slot_dep_map(zram, index), _RET_IP_); + mutex_release(&zram->table_lock_map, _RET_IP_); clear_and_wake_up_bit(ZRAM_ENTRY_LOCK, lock); } @@ -1984,7 +1982,7 @@ static void zram_meta_free(struct zram *zram, u64 disksize) static bool zram_meta_alloc(struct zram *zram, u64 disksize) { - size_t num_pages, index; + size_t num_pages; num_pages = disksize >> PAGE_SHIFT; zram->table = vzalloc(array_size(num_pages, sizeof(*zram->table))); @@ -2001,8 +1999,7 @@ static bool zram_meta_alloc(struct zram *zram, u64 disksize) if (!huge_class_size) huge_class_size = zs_huge_class_size(zram->mem_pool); - for (index = 0; index < num_pages; index++) - slot_lock_init(zram, index); + slot_lock_init(zram); return true; } diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h index 08d1774c15db..8290dc6e2335 100644 --- a/drivers/block/zram/zram_drv.h +++ b/drivers/block/zram/zram_drv.h @@ -74,7 +74,6 @@ struct zram_table_entry { #endif } attr; }; - struct lockdep_map dep_map; }; struct zram_stats { @@ -107,6 +106,7 @@ struct zram_stats { struct zram { struct zram_table_entry *table; + struct lockdep_map table_lock_map; struct zs_pool *mem_pool; struct zcomp *comps[ZRAM_MAX_COMPS]; struct zcomp_params params[ZRAM_MAX_COMPS]; From d56a82aa87ee5203952fc2c7251d3cf7e6129fa2 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Tue, 14 Jul 2026 16:13:00 +0200 Subject: [PATCH 492/501] zram: use a custom key for each zram object Each struct zram uses the same key for its struct lockdep_map which is used for locking analysis. According to Sergey the lock chains might be different if zram1 is used for and zram2 is for ext4. This might lead to false dead lock reports if it mixes a zram1 chain with a zram2. This can be avoided if each lockmap gets its own unique key.c Use a dynamic lock_class_key for the table_lock_map. Link: https://lore.kernel.org/20260714141300.3945672-3-bigeasy@linutronix.de Signed-off-by: Sebastian Andrzej Siewior Reviewed-by: Sergey Senozhatsky Tested-by: Sergey Senozhatsky Cc: Jens Axboe Cc: Minchan Kim Signed-off-by: Andrew Morton --- drivers/block/zram/zram_drv.c | 11 +++-------- drivers/block/zram/zram_drv.h | 1 + 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 4223002d80b4..4bfe63a5225d 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -58,13 +58,6 @@ static const struct block_device_operations zram_devops; static void slot_free(struct zram *zram, u32 index); -static void slot_lock_init(struct zram *zram) -{ - static struct lock_class_key __key; - - lockdep_init_map(&zram->table_lock_map, "zram->table[index].lock", &__key, 0); -} - /* * entry locking rules: * @@ -1978,6 +1971,7 @@ static void zram_meta_free(struct zram *zram, u64 disksize) zs_destroy_pool(zram->mem_pool); vfree(zram->table); zram->table = NULL; + lockdep_unregister_key(&zram->table_lock_key); } static bool zram_meta_alloc(struct zram *zram, u64 disksize) @@ -1999,7 +1993,8 @@ static bool zram_meta_alloc(struct zram *zram, u64 disksize) if (!huge_class_size) huge_class_size = zs_huge_class_size(zram->mem_pool); - slot_lock_init(zram); + lockdep_register_key(&zram->table_lock_key); + lockdep_init_map(&zram->table_lock_map, "zram->table[index].lock", &zram->table_lock_key, 0); return true; } diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h index 8290dc6e2335..4fddc582f3b8 100644 --- a/drivers/block/zram/zram_drv.h +++ b/drivers/block/zram/zram_drv.h @@ -107,6 +107,7 @@ struct zram_stats { struct zram { struct zram_table_entry *table; struct lockdep_map table_lock_map; + struct lock_class_key table_lock_key; struct zs_pool *mem_pool; struct zcomp *comps[ZRAM_MAX_COMPS]; struct zcomp_params params[ZRAM_MAX_COMPS]; From efe8f86c0916f0f74eea74ae21a3b37f728c6bad Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Mon, 27 Jul 2026 06:50:19 -0700 Subject: [PATCH 493/501] mm/migrate: report RCU-tasks quiescent states in migrate_pages_batch() migrate_pages_batch() unmaps each folio before moving it, and every unmap runs the mmu_notifier invalidate callbacks. On KVM hosts try_to_migrate() ends up in kvm_mmu_notifier_invalidate_range_start() -> tdp_mmu_zap_leafs(), which is expensive, so unmapping a large batch keeps the CPU busy for a long time. The loop already calls cond_resched(), but on PREEMPTION kernels that is a no-op, and involuntary preemption is not a Tasks-RCU quiescent state. A long batch therefore never reports a quiescent state, and the migrating task (e.g. kcompactd) becomes a Tasks-RCU holdout, stalling the Tasks-RCU grace period for minutes, which is common at Meta fleet: INFO: rcu_tasks detected stalls on tasks: 0000000055349ecc: .. nvcsw: 1157401/1157401 holdout: 1 idle_cpu: -1/56 task:kcompactd0 state:R running task Call Trace: tdp_mmu_zap_leafs tdp_mmu_next_root gfn_to_pfn_cache_invalidate_start kvm_mmu_notifier_invalidate_range_start __mmu_notifier_invalidate_range_start try_to_migrate_one try_to_migrate migrate_pages_batch migrate_pages compact_zone compact_node kcompactd kthread Use cond_resched_tasks_rcu_qs() so a quiescent state is reported even when cond_resched() does nothing. This has also been discussed at [1] Link: https://lore.kernel.org/20260727-kcompact-v1-1-bdfefddd6874@debian.org Link: https://lore.kernel.org/all/amdWVTs0WKOxguxP@gmail.com/ [1] Signed-off-by: Breno Leitao Acked-by: Zi Yan Reviewed-by: Gregory Price Reviewed-by: Paul E. McKenney Acked-by: David Hildenbrand (Arm) Cc: Alistair Popple Cc: Byungchul Park Cc: "Huang, Ying" Cc: Joshua Hahn Cc: Matthew Brost Cc: Rakie Kim Cc: Signed-off-by: Andrew Morton --- mm/migrate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/migrate.c b/mm/migrate.c index 60355be9f652..8aaafcea7bc1 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1841,7 +1841,7 @@ static int migrate_pages_batch(struct list_head *from, is_thp = folio_test_pmd_mappable(folio); nr_pages = folio_nr_pages(folio); - cond_resched(); + cond_resched_tasks_rcu_qs(); /* * The rare folio on the deferred split list should From 41f2c0f716e19b932a0653eaa462999e5e0d8126 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Mon, 27 Jul 2026 10:16:11 +0100 Subject: [PATCH 494/501] mm: fold userfaultfd_rwp() to false without CONFIG_ARCH_HAS_PTE_PROTNONE RWP tracks accesses by installing PAGE_NONE (protnone) PTEs, so its code paths are gated on userfaultfd_rwp(). Without CONFIG_ARCH_HAS_PTE_PROTNONE there is no PAGE_NONE -- defines it to a BUILD_BUG() stub, relying on callers folding such paths to dead code via IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE). userfaultfd_rwp() was not a compile-time constant, so the compiler could not fold those paths. With an older compiler (gcc 8.5.0, sparc64) the PAGE_NONE reference in move_pages_huge_pmd() survived to codegen: mm/huge_memory.c:2874: _dst_pmd = pmd_modify(_dst_pmd, PAGE_NONE); compiler_types.h:702: error: call to '__compiletime_assert_501' declared with attribute error: BUILD_BUG failed RWP cannot exist without protnone, so return a compile-time false when CONFIG_ARCH_HAS_PTE_PROTNONE is unset; every RWP path then folds away. Link: https://lore.kernel.org/amcitKvUvFYr8W38@thinkstation Signed-off-by: Kiryl Shutsemau (Meta) Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202607250853.VaJWGLeA-lkp@intel.com/ Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: James Houghton Cc: Jonathan Corbet Cc: Liam Howlett Cc: Lorenzo Stoakes Cc: Mike Rapoport (Microsoft) Cc: Paolo Bonzini Cc: Peter Xu Cc: Sean Christopherson Cc: SeongJae Park Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/userfaultfd_k.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index bfbd6a59909f..a4351cffc60c 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -221,6 +221,12 @@ static inline bool userfaultfd_minor(struct vm_area_struct *vma) static inline bool userfaultfd_rwp(struct vm_area_struct *vma) { + /* + * Callers gate PAGE_NONE usage on this; PAGE_NONE is a BUILD_BUG() + * without CONFIG_ARCH_HAS_PTE_PROTNONE, so fold to false. + */ + if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE)) + return false; return vma_test_single_mask(vma, VMA_UFFD_RWP); } From 6be12ddb52f9a3fa62f078a0b61573416de5ff5c Mon Sep 17 00:00:00 2001 From: Wang Wensheng Date: Sun, 26 Jul 2026 21:35:01 +0800 Subject: [PATCH 495/501] ksm: stop iterating VMAs when ksm_test_exit returns true In scan_get_next_rmap_item() the break statement only exits the inner while loop, leaving remaining VMAs to be iterated even if ksm_test_exit() returns true. Replace it with a goto statement to avoid the unnecessary work. Link: https://lore.kernel.org/20260726133501.504048-1-wsw9603@163.com Signed-off-by: Wang Wensheng Reviewed-by: Xu Xin Acked-by: David Hildenbrand (Arm) Cc: Chengming Zhou Signed-off-by: Andrew Morton --- mm/ksm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/ksm.c b/mm/ksm.c index 47006f494fcb..b4142746777e 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -2700,7 +2700,7 @@ static struct ksm_rmap_item *scan_get_next_rmap_item(struct page **page) struct folio *folio; if (ksm_test_exit(mm)) - break; + goto no_vmas; int found; From 2f99b6fe0b856065976f569929577fe587bf74b8 Mon Sep 17 00:00:00 2001 From: Sergey Senozhatsky Date: Tue, 28 Jul 2026 11:12:11 +0900 Subject: [PATCH 496/501] Documentation: zram: remove sections numbering Those numbers are difficult to maintain and in fact we can refer to sections by their names (in html). Link: https://lore.kernel.org/20260728021229.181627-1-senozhatsky@chromium.org Signed-off-by: Sergey Senozhatsky Suggested-by: Sebastian Andrzej Siewior Reviewed-by: SJ Park Cc: Jonathan Corbet Cc: Minchan Kim Signed-off-by: Andrew Morton --- Documentation/admin-guide/blockdev/zram.rst | 40 ++++++++++----------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/Documentation/admin-guide/blockdev/zram.rst b/Documentation/admin-guide/blockdev/zram.rst index 60b07a7e30cd..2f6bbfd991fe 100644 --- a/Documentation/admin-guide/blockdev/zram.rst +++ b/Documentation/admin-guide/blockdev/zram.rst @@ -61,8 +61,8 @@ and, in general case, something like:: should suffice. -1) Load Module -============== +Load Module +=========== :: @@ -73,8 +73,8 @@ This creates 4 devices: /dev/zram{0,1,2,3} num_devices parameter is optional and tells zram how many devices should be pre-created. Default: 1. -2) Select compression algorithm -=============================== +Select compression algorithm +============================ Using comp_algorithm device attribute one can see available and currently selected (shown in square brackets) compression algorithms, @@ -93,8 +93,8 @@ Examples:: For the time being, the `comp_algorithm` content shows only compression algorithms that are supported by zram. -3) Set compression algorithm parameters: Optional -================================================= +Set compression algorithm parameters: Optional +============================================== Compression algorithms may support specific parameters which can be tweaked for particular dataset. ZRAM has an `algorithm_params` device @@ -124,8 +124,8 @@ better the compression ratio, it even can take negatives values for some algorithms), for other algorithms `level` is acceleration level (the higher the value the lower the compression ratio). -4) Set Disksize -=============== +Set Disksize +============ Set disk size by writing the value to sysfs node 'disksize'. The value can be either in bytes or you can use mem suffixes. @@ -144,8 +144,8 @@ There is little point creating a zram of greater than twice the size of memory since we expect a 2:1 compression ratio. Note that zram uses about 0.1% of the size of the disk when not in use so a huge zram is wasteful. -5) Set memory limit: Optional -============================= +Set memory limit: Optional +========================== Set memory limit by writing the value to sysfs node 'mem_limit'. The value can be either in bytes or you can use mem suffixes. @@ -163,8 +163,8 @@ Examples:: # To disable memory limit echo 0 > /sys/block/zram0/mem_limit -6) Activate -=========== +Activate +======== :: @@ -174,8 +174,8 @@ Examples:: mkfs.ext4 /dev/zram1 mount /dev/zram1 /tmp -7) Add/remove zram devices -========================== +Add/remove zram devices +======================= zram provides a control interface, which enables dynamic (on-demand) device addition and removal. @@ -194,8 +194,8 @@ execute:: echo X > /sys/class/zram-control/hot_remove -8) Stats -======== +Stats +===== Per-device statistics are exported as various nodes under /sys/block/zram/ @@ -296,16 +296,16 @@ a single line of text and contains the following stats separated by whitespace: Unit: 4K bytes ============== ============================================================= -9) Deactivate -============== +Deactivate +========== :: swapoff /dev/zram0 umount /dev/zram1 -10) Reset -========= +Reset +===== Write any positive value to 'reset' sysfs node:: From f276408a816790115ccea1f6fcd6b160d80148d8 Mon Sep 17 00:00:00 2001 From: Richard Chang Date: Tue, 28 Jul 2026 05:53:33 +0000 Subject: [PATCH 497/501] mm/zsmalloc: fix release order of locks in zs_page_migrate() In zs_page_migrate(), locks are acquired in the following order: 1. write_lock(&pool->lock) 2. spin_lock(&class->lock) 3. zspage_write_trylock(zspage) However, upon successful page migration, they were being released in forward acquisition (FIFO) order: 1. write_unlock(&pool->lock) 2. spin_unlock(&class->lock) 3. zspage_write_unlock(zspage) Fix the unlocking order to release locks in strict reverse (LIFO) order of acquisition: 3. zspage_write_unlock(zspage) 2. spin_unlock(&class->lock) 1. write_unlock(&pool->lock) Releasing locks in reverse order of acquisition adheres to standard kernel locking hygiene, prevents potential lock ordering and lockdep inconsistencies. Link: https://lore.kernel.org/20260728055333.421080-1-richardycc@google.com Signed-off-by: Richard Chang Reviewed-by: Sergey Senozhatsky Tested-by: Sergey Senozhatsky Cc: Martin Liu Cc: Minchan Kim Signed-off-by: Andrew Morton --- mm/zsmalloc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index b5eadee0e8f0..8204b76f7830 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -1926,9 +1926,9 @@ static int zs_page_migrate(struct page *newpage, struct page *page, * Since we complete the data copy and set up new zspage structure, * it's okay to release migration_lock. */ - write_unlock(&pool->lock); - spin_unlock(&class->lock); zspage_write_unlock(zspage); + spin_unlock(&class->lock); + write_unlock(&pool->lock); zpdesc_get(newzpdesc); if (zpdesc_zone(newzpdesc) != zpdesc_zone(zpdesc)) { From 0ec282155c8458954b886a8d0045a753d6c5a207 Mon Sep 17 00:00:00 2001 From: Jakov Novak Date: Thu, 30 Jul 2026 22:47:24 +0200 Subject: [PATCH 498/501] mm/khugepaged: replace mutex_lock/mutex_unlock usage with guard macro Currently, khugepaged locks the khugepaged_mutex in two functions: start_stop_khugepaged and khugepaged_min_free_kbytes_update. Remove mutex_lock/mutex_unlock usage in these functions and replace it with the guard macro. This makes the code more readable (removing a goto statement) and makes it harder to introduce bugs in the future. No functional changes introduced. Link: https://lore.kernel.org/20260730204724.16912-1-jakovnovak30@gmail.com Signed-off-by: Jakov Novak Reviewed-by: Dev Jain Reviewed-by: Lorenzo Stoakes (ARM) Reviewed-by: Andrew Morton Reviewed-by: Zi Yan Acked-by: David Hildenbrand (Arm) Cc: Baolin Wang Cc: Barry Song Cc: Lance Yang Cc: Liam R. Howlett Cc: Nico Pache Cc: Ryan Roberts Signed-off-by: Andrew Morton --- mm/khugepaged.c | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 27e8f3077e80..b237f6e7662a 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include "internal.h" @@ -3115,18 +3116,19 @@ void set_recommended_min_free_kbytes(void) int start_stop_khugepaged(void) { - int err = 0; - - mutex_lock(&khugepaged_mutex); + guard(mutex)(&khugepaged_mutex); if (hugepage_enabled()) { - if (!khugepaged_thread) - khugepaged_thread = kthread_run(khugepaged, NULL, - "khugepaged"); - if (IS_ERR(khugepaged_thread)) { - pr_err("khugepaged: kthread_run(khugepaged) failed\n"); - err = PTR_ERR(khugepaged_thread); - khugepaged_thread = NULL; - goto fail; + if (!khugepaged_thread) { + struct task_struct *new_thread = kthread_run(khugepaged, + NULL, + "khugepaged"); + + if (IS_ERR(new_thread)) { + pr_err("khugepaged: kthread_run(khugepaged) failed\n"); + return PTR_ERR(new_thread); + } + + khugepaged_thread = new_thread; } if (!list_empty(&khugepaged_scan.mm_head)) @@ -3136,17 +3138,14 @@ int start_stop_khugepaged(void) khugepaged_thread = NULL; } set_recommended_min_free_kbytes(); -fail: - mutex_unlock(&khugepaged_mutex); - return err; + return 0; } void khugepaged_min_free_kbytes_update(void) { - mutex_lock(&khugepaged_mutex); + guard(mutex)(&khugepaged_mutex); if (hugepage_enabled() && khugepaged_thread) set_recommended_min_free_kbytes(); - mutex_unlock(&khugepaged_mutex); } bool current_is_khugepaged(void) From 1d3897b0244218f9c73bdec140cf3dab10a925a8 Mon Sep 17 00:00:00 2001 From: Jiacheng Yu Date: Tue, 28 Jul 2026 09:10:14 +0000 Subject: [PATCH 499/501] mm/shmem: downgrade final i_blocks check in shmem_evict_inode() to pr_warn() shmem_evict_inode() ends with WARN_ON(inode->i_blocks) as a final consistency check of shmem's block accounting. When it fires, the inode-local counters die with the inode; what may linger is a small residue in accounting kept outside the inode, such as per-mount or per-user charges. No data is lost, and no corruption follows. On kernels running with panic_on_warn=1, this accounting inconsistency escalates to a full machine panic, which is disproportionate to the impact. Downgrade the WARN_ON() to a pr_warn() that reports the inode together with its accounting counters (i_blocks, alloced, swapped, nrpages), keeping the inconsistency visible in the logs. The accounting bugs this check has caught over the years -- the swapout race described in commit 0f3c42f522dc ("tmpfs: change final i_blocks BUG to WARNING") and the error recovery race fixed in commit 267a4c76bbdb ("tmpfs: fix shmem_evict_inode() warnings on i_blocks") -- are real and should still be fixed; this change only removes the disproportionate escalation. One way to hit this race: soft_offline_in_use_page()'s fast path drops a clean, unmapped shmem folio via mapping_evict_folio(), where the xas_store() and the nrpages decrement are not atomic against a concurrent shmem_evict_inode(); the final shmem_recalc_inode() can then read the pre-decrement nrpages, compute freed = 0, and leave one page charged. Same class as the races in 0f3c42f522dc and 267a4c76bbdb, this time in the under-count direction; reproduced on 7.2-rc4 with madvise(MADV_SOFT_OFFLINE) racing MAP_FIXED replacement of a shared-anonymous VMA. [yujiacheng3@huawei.com: drop redundant casts in shmem_evict_inode() pr_warn] Link: https://lore.kernel.org/20260729121201.776566-1-yujiacheng3@huawei.com Link: https://lore.kernel.org/20260728091014.3876715-1-yujiacheng3@huawei.com Fixes: 0f3c42f522dc ("tmpfs: change final i_blocks BUG to WARNING") Signed-off-by: Jiacheng Yu Cc: Baolin Wang Cc: Hugh Dickins Cc: Yongqiang Liu Cc: Christian Brauner Signed-off-by: Andrew Morton --- mm/shmem.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mm/shmem.c b/mm/shmem.c index f1ab97c34a02..774f4b18ff5c 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1440,7 +1440,10 @@ static void shmem_evict_inode(struct inode *inode) simple_xattrs_free(&sbinfo->xa_cache, &info->xattrs, sbinfo->max_inodes ? &freed : NULL); shmem_free_inode(inode->i_sb, freed); - WARN_ON(inode->i_blocks); + if (inode->i_blocks) + pr_warn("%s: ino=%llu i_blocks=%llu alloced=%lu swapped=%lu nrpages=%lu\n", + __func__, inode->i_ino, inode->i_blocks, + info->alloced, info->swapped, inode->i_mapping->nrpages); clear_inode(inode); #ifdef CONFIG_TMPFS_QUOTA dquot_free_inode(inode); From 288058d8db5d35623228d84f48d9bea3707d5c85 Mon Sep 17 00:00:00 2001 From: Muhammad Usama Anjum Date: Wed, 29 Jul 2026 10:11:26 +0100 Subject: [PATCH 500/501] selftests/mm: unpoison pages in memory-failure teardown The memory-failure tests call cleanup() only after all result checks. A failed ASSERT_* invokes fixture teardown and aborts the test, so it skips cleanup() and leaves the injected page hardware-poisoned. Invoke cleanup() from FIXTURE_TEARDOWN() instead. Guard it with self->injection_attempted so tests that exit before injection do not try to unpoison a page when no injection was attempted. Injection can poison a page before returning an error or delivering SIGBUS, so teardown must clean up after every injection attempt. This runs the existing HWPoison and HardwareCorrupted checks on both normal and assertion-failure paths. Link: https://lore.kernel.org/20260729091127.1001179-1-usama.anjum@arm.com Fixes: ff4ef2fbd101 ("selftests/mm: add memory failure anonymous page test") Signed-off-by: Muhammad Usama Anjum Reviewed-by: David Hildenbrand (Arm) Acked-by: Miaohe Lin Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Naoya Horiguchi Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/memory-failure.c | 44 ++++++++++----------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tools/testing/selftests/mm/memory-failure.c b/tools/testing/selftests/mm/memory-failure.c index 032ed952057c..1a5a32e22cce 100644 --- a/tools/testing/selftests/mm/memory-failure.c +++ b/tools/testing/selftests/mm/memory-failure.c @@ -46,7 +46,7 @@ FIXTURE(memory_failure) unsigned long pfn; int pagemap_fd; int kpageflags_fd; - bool triggered; + bool injection_attempted; }; FIXTURE_VARIANT(memory_failure) @@ -122,13 +122,6 @@ static void teardown_sighandler(void) sigaction(SIGBUS, &sa, NULL); } -FIXTURE_TEARDOWN(memory_failure) -{ - close(self->kpageflags_fd); - close(self->pagemap_fd); - teardown_sighandler(); -} - static void prepare(struct __test_metadata *_metadata, FIXTURE_DATA(memory_failure) * self, void *vaddr) { @@ -200,8 +193,7 @@ static void check(struct __test_metadata *_metadata, FIXTURE_DATA(memory_failure ASSERT_EQ(pfn_flags & KPF_HWPOISON, KPF_HWPOISON); } -static void cleanup(struct __test_metadata *_metadata, FIXTURE_DATA(memory_failure) * self, - void *vaddr) +static void cleanup(struct __test_metadata *_metadata, FIXTURE_DATA(memory_failure) * self) { unsigned long size; uint64_t pfn_flags; @@ -217,6 +209,20 @@ static void cleanup(struct __test_metadata *_metadata, FIXTURE_DATA(memory_failu ASSERT_EQ(size, self->corrupted_size); } +FIXTURE_TEARDOWN(memory_failure) +{ + /* + * Injection may poison the page before failing or delivering SIGBUS, so + * clean up after every injection attempt. + */ + if (self->injection_attempted) + cleanup(_metadata, self); + + close(self->kpageflags_fd); + close(self->pagemap_fd); + teardown_sighandler(); +} + TEST_F(memory_failure, anon) { char *addr; @@ -231,8 +237,8 @@ TEST_F(memory_failure, anon) prepare(_metadata, self, addr); ret = sigsetjmp(signal_jmp_buf, 1); - if (!self->triggered) { - self->triggered = true; + if (!self->injection_attempted) { + self->injection_attempted = true; ASSERT_EQ(variant->inject(self, addr), 0); FORCE_READ(*addr); } @@ -242,8 +248,6 @@ TEST_F(memory_failure, anon) else check(_metadata, self, addr, MADV_SOFT_ANON, ret); - cleanup(_metadata, self, addr); - ASSERT_EQ(munmap(addr, self->page_size), 0); } @@ -296,8 +300,8 @@ TEST_F(memory_failure, clean_pagecache) prepare(_metadata, self, addr); ret = sigsetjmp(signal_jmp_buf, 1); - if (!self->triggered) { - self->triggered = true; + if (!self->injection_attempted) { + self->injection_attempted = true; ASSERT_EQ(variant->inject(self, addr), 0); FORCE_READ(*addr); } @@ -307,8 +311,6 @@ TEST_F(memory_failure, clean_pagecache) else check(_metadata, self, addr, MADV_SOFT_CLEAN_PAGECACHE, ret); - cleanup(_metadata, self, addr); - ASSERT_EQ(munmap(addr, self->page_size), 0); ASSERT_EQ(close(fd), 0); @@ -337,8 +339,8 @@ TEST_F(memory_failure, dirty_pagecache) prepare(_metadata, self, addr); ret = sigsetjmp(signal_jmp_buf, 1); - if (!self->triggered) { - self->triggered = true; + if (!self->injection_attempted) { + self->injection_attempted = true; ASSERT_EQ(variant->inject(self, addr), 0); FORCE_READ(*addr); } @@ -348,8 +350,6 @@ TEST_F(memory_failure, dirty_pagecache) else check(_metadata, self, addr, MADV_SOFT_DIRTY_PAGECACHE, ret); - cleanup(_metadata, self, addr); - ASSERT_EQ(munmap(addr, self->page_size), 0); ASSERT_EQ(close(fd), 0); From 08745c62350126bc31b09548137be87e2866f628 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Wed, 12 Aug 2026 11:17:40 +0300 Subject: [PATCH 501/501] selftests/mm: thuge-gen: fix test_shmget() for PAGE_SIZE check Commit 49a4e7186b08 ("selftests/mm: thuge-gen: add setup of HugeTLB pages") changed thuge-gen test to use common functions for reading hugetlb attributes from sysfs, but it missed that the original read_free() function special cased PAGE_SIZE tests. For PAGE_SIZE tests, failure to read sysfs was ignored and read_free() returned 0. This allowed test_shmget() to essentially skip the check of how many huge pages was consumed when it ran with PAGE_SIZE. Commit 3199b0c09efa ("selftests/mm: fix read_file() return value check") fixed checks for read_file() return value and this exposed the issue in test_shmget() that checks the number of free hugetlb pages even for PAGE_SIZE test, tries to access /sys/kernel/mm/hugepages/hugepages-/free_hugepages and obviously fails there. Gate the checks for free huge pages on size != getpagesize() and initialize before and after variables to values matching PAGE_SIZE test. Link: https://lore.kernel.org/20260812-selftests-thuge-gen-fix-v2-1-9adaa693e73b@kernel.org Fixes: 49a4e7186b08 ("selftests/mm: thuge-gen: add setup of HugeTLB pages") Acked-by: David Hildenbrand (Arm) Reviewed-by: Sarthak Sharma Acked-by: Lorenzo Stoakes (ARM) Signed-off-by: Mike Rapoport (Microsoft) Cc: Liam R. Howlett Cc: Michal Hocko Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/thuge-gen.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/mm/thuge-gen.c b/tools/testing/selftests/mm/thuge-gen.c index 22b9c2f1c35d..50d0805b65db 100644 --- a/tools/testing/selftests/mm/thuge-gen.c +++ b/tools/testing/selftests/mm/thuge-gen.c @@ -71,12 +71,16 @@ void test_mmap(unsigned long size, unsigned flags) void test_shmget(unsigned long size, unsigned flags) { - int id; - unsigned long before, after; + /* values for PAGE_SIZE test */ + unsigned long before = NUM_PAGES; + unsigned long after = 0; struct shm_info i; char *map; + int id; + + if (size != getpagesize()) + before = hugetlb_free_pages(size); - before = hugetlb_free_pages(size); id = shmget(IPC_PRIVATE, size * NUM_PAGES, IPC_CREAT|0600|flags); if (id < 0) { if (errno == EPERM) { @@ -97,10 +101,11 @@ void test_shmget(unsigned long size, unsigned flags) shmctl(id, IPC_RMID, NULL); memset(map, 0xff, size*NUM_PAGES); - after = hugetlb_free_pages(size); + if (size != getpagesize()) + after = hugetlb_free_pages(size); show(size); - ksft_test_result(size == getpagesize() || (before - after) == NUM_PAGES, + ksft_test_result((before - after) == NUM_PAGES, "%s: mmap %lu %x\n", __func__, size, flags); if (shmdt(map)) ksft_exit_fail_msg("%s: shmdt: %s\n", __func__, strerror(errno));