From e7616bb5ef5fe752e15db8a3dbd8657dc19c21f2 Mon Sep 17 00:00:00 2001 From: Vishal Moola Date: Mon, 29 Jun 2026 11:57:34 -0700 Subject: [PATCH 01/20] x86/mm/pat: Use IS_ENABLED() instead of ifdef Use IS_ENABLED() to check if we are on 32 bit. This standardizes this check with the other 32 bit check in the file. No functional changes. Signed-off-by: Vishal Moola Signed-off-by: Dave Hansen Reviewed-by: William Kucharski Acked-by: Mike Rapoport (Microsoft) Link: https://patch.msgid.link/20260629185742.126987-2-vishal.moola@gmail.com --- arch/x86/mm/pat/set_memory.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index d023a40a1e03..3b9032a3f315 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -888,8 +888,8 @@ static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte) { /* change init_mm */ set_pte_atomic(kpte, pte); -#ifdef CONFIG_X86_32 - { + + if (IS_ENABLED(CONFIG_X86_32)) { struct page *page; list_for_each_entry(page, &pgd_list, lru) { @@ -905,7 +905,6 @@ static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte) set_pte_atomic((pte_t *)pmd, pte); } } -#endif } static pgprot_t pgprot_clear_protnone_bits(pgprot_t prot) From e27d938b0de86304f12c89706b8034f89f015077 Mon Sep 17 00:00:00 2001 From: Vishal Moola Date: Mon, 29 Jun 2026 11:57:35 -0700 Subject: [PATCH 02/20] x86/mm/pat: Convert __set_pmd_pte() to ptdescs Convert __set_pmd_pte() to ptdescs in preparation for the eventual splitting of ptdescs from struct page. Signed-off-by: Vishal Moola Signed-off-by: Dave Hansen Reviewed-by: William Kucharski Acked-by: Mike Rapoport (Microsoft) Link: https://patch.msgid.link/20260629185742.126987-3-vishal.moola@gmail.com --- arch/x86/mm/pat/set_memory.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index 3b9032a3f315..93cb71451ce2 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -890,15 +890,15 @@ static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte) set_pte_atomic(kpte, pte); if (IS_ENABLED(CONFIG_X86_32)) { - struct page *page; + struct ptdesc *ptdesc; - list_for_each_entry(page, &pgd_list, lru) { + list_for_each_entry(ptdesc, &pgd_list, pt_list) { pgd_t *pgd; p4d_t *p4d; pud_t *pud; pmd_t *pmd; - pgd = (pgd_t *)page_address(page) + pgd_index(address); + pgd = (pgd_t *)ptdesc_address(ptdesc) + pgd_index(address); p4d = p4d_offset(pgd, address); pud = pud_offset(p4d, address); pmd = pmd_offset(pud, address); From 5fcb2abffe015017c252a5a720375145e808337e Mon Sep 17 00:00:00 2001 From: Vishal Moola Date: Mon, 29 Jun 2026 11:57:36 -0700 Subject: [PATCH 03/20] x86/mm/pat: Convert collapse_pmd_page() to ptdescs Convert collapse_pmd_page() to ptdescs in preparation for the eventual splitting of ptdescs from struct page. Signed-off-by: Vishal Moola Signed-off-by: Dave Hansen Reviewed-by: William Kucharski Acked-by: Mike Rapoport (Microsoft) Link: https://patch.msgid.link/20260629185742.126987-4-vishal.moola@gmail.com --- arch/x86/mm/pat/set_memory.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index 93cb71451ce2..45623d4c24c9 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -1297,11 +1297,11 @@ static int collapse_pmd_page(pmd_t *pmd, unsigned long addr, list_add(&page_ptdesc(pmd_page(old_pmd))->pt_list, pgtables); if (IS_ENABLED(CONFIG_X86_32)) { - struct page *page; + struct ptdesc *ptdesc; /* Update all PGD tables to use the same large page */ - list_for_each_entry(page, &pgd_list, lru) { - pgd_t *pgd = (pgd_t *)page_address(page) + pgd_index(addr); + list_for_each_entry(ptdesc, &pgd_list, pt_list) { + pgd_t *pgd = (pgd_t *)ptdesc_address(ptdesc) + pgd_index(addr); p4d_t *p4d = p4d_offset(pgd, addr); pud_t *pud = pud_offset(p4d, addr); pmd_t *pmd = pmd_offset(pud, addr); From 46ee485bb9dee952cab3e7b22138b9ad0e5cb468 Mon Sep 17 00:00:00 2001 From: Vishal Moola Date: Mon, 29 Jun 2026 11:57:37 -0700 Subject: [PATCH 04/20] x86/mm: Convert arch_sync_kernel_mappings() to ptdescs Convert arch_sync_kernel_mappings() to ptdescs in preparation for the eventual splitting of ptdescs from struct page. Following this patch, we can successfully boot a 32-bit x86 kernel with separately allocated ptdescs. Signed-off-by: Vishal Moola Signed-off-by: Dave Hansen Reviewed-by: William Kucharski Acked-by: Mike Rapoport (Microsoft) Link: https://patch.msgid.link/20260629185742.126987-5-vishal.moola@gmail.com --- arch/x86/mm/fault.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 45b99c3b1442..c31ffeac0714 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -275,17 +275,17 @@ void arch_sync_kernel_mappings(unsigned long start, unsigned long end) for (addr = start & PMD_MASK; addr >= TASK_SIZE_MAX && addr < VMALLOC_END; addr += PMD_SIZE) { - struct page *page; + struct ptdesc *ptdesc; spin_lock(&pgd_lock); - list_for_each_entry(page, &pgd_list, lru) { + list_for_each_entry(ptdesc, &pgd_list, pt_list) { spinlock_t *pgt_lock; /* the pgt_lock only for Xen */ - pgt_lock = &pgd_page_get_mm(page)->page_table_lock; + pgt_lock = &pgd_page_get_mm(ptdesc_page(ptdesc))->page_table_lock; spin_lock(pgt_lock); - vmalloc_sync_one(page_address(page), addr); + vmalloc_sync_one(ptdesc_address(ptdesc), addr); spin_unlock(pgt_lock); } spin_unlock(&pgd_lock); From 90c881895fab258924772846f2555a9c9eaac8f1 Mon Sep 17 00:00:00 2001 From: Vishal Moola Date: Mon, 29 Jun 2026 11:57:38 -0700 Subject: [PATCH 05/20] x86/mm: Convert sync_global_pgds_l5() to ptdescs Convert sync_global_pgds_l5() to ptdescs in preparation for the eventual splitting of ptdescs from struct page. Signed-off-by: Vishal Moola Signed-off-by: Dave Hansen Reviewed-by: William Kucharski Acked-by: Mike Rapoport (Microsoft) Link: https://patch.msgid.link/20260629185742.126987-6-vishal.moola@gmail.com --- arch/x86/mm/init_64.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index d57f29ca23a5..a097ec13bb0c 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -136,7 +136,7 @@ static void sync_global_pgds_l5(unsigned long start, unsigned long end) for (addr = start; addr <= end; addr = ALIGN(addr + 1, PGDIR_SIZE)) { const pgd_t *pgd_ref = pgd_offset_k(addr); - struct page *page; + struct ptdesc *ptdesc; /* Check for overflow */ if (addr < start) @@ -146,13 +146,13 @@ static void sync_global_pgds_l5(unsigned long start, unsigned long end) continue; spin_lock(&pgd_lock); - list_for_each_entry(page, &pgd_list, lru) { + list_for_each_entry(ptdesc, &pgd_list, pt_list) { pgd_t *pgd; spinlock_t *pgt_lock; - pgd = (pgd_t *)page_address(page) + pgd_index(addr); + pgd = (pgd_t *)ptdesc_address(ptdesc) + pgd_index(addr); /* the pgt_lock only for Xen */ - pgt_lock = &pgd_page_get_mm(page)->page_table_lock; + pgt_lock = &pgd_page_get_mm(ptdesc_page(ptdesc))->page_table_lock; spin_lock(pgt_lock); if (!pgd_none(*pgd_ref) && !pgd_none(*pgd)) From 9c453f8784dc033a35d06dff929be1eaedb3020c Mon Sep 17 00:00:00 2001 From: Vishal Moola Date: Mon, 29 Jun 2026 11:57:39 -0700 Subject: [PATCH 06/20] x86/mm: Convert sync_global_pgds_l4() to ptdescs Convert sync_global_pgds_l4() to ptdescs in preparation for the eventual splitting of ptdescs from struct page. Signed-off-by: Vishal Moola Signed-off-by: Dave Hansen Reviewed-by: William Kucharski Acked-by: Mike Rapoport (Microsoft) Link: https://patch.msgid.link/20260629185742.126987-7-vishal.moola@gmail.com --- arch/x86/mm/init_64.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index a097ec13bb0c..1441f5fbd84f 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -174,7 +174,7 @@ static void sync_global_pgds_l4(unsigned long start, unsigned long end) for (addr = start; addr <= end; addr = ALIGN(addr + 1, PGDIR_SIZE)) { pgd_t *pgd_ref = pgd_offset_k(addr); const p4d_t *p4d_ref; - struct page *page; + struct ptdesc *ptdesc; /* * With folded p4d, pgd_none() is always false, we need to @@ -187,15 +187,15 @@ static void sync_global_pgds_l4(unsigned long start, unsigned long end) continue; spin_lock(&pgd_lock); - list_for_each_entry(page, &pgd_list, lru) { + list_for_each_entry(ptdesc, &pgd_list, pt_list) { pgd_t *pgd; p4d_t *p4d; spinlock_t *pgt_lock; - pgd = (pgd_t *)page_address(page) + pgd_index(addr); + pgd = (pgd_t *)ptdesc_address(ptdesc) + pgd_index(addr); p4d = p4d_offset(pgd, addr); /* the pgt_lock only for Xen */ - pgt_lock = &pgd_page_get_mm(page)->page_table_lock; + pgt_lock = &pgd_page_get_mm(ptdesc_page(ptdesc))->page_table_lock; spin_lock(pgt_lock); if (!p4d_none(*p4d_ref) && !p4d_none(*p4d)) From dd9850c91665a56babd964df19de2cb90a98c538 Mon Sep 17 00:00:00 2001 From: Vishal Moola Date: Mon, 29 Jun 2026 11:57:40 -0700 Subject: [PATCH 07/20] x86/mm: Convert pgd_page_get_mm() to ptdescs Convert pgd_page_get_mm() to ptdescs. Define struct ptdesc in our pgtable_types so that our declarations recognize ptdesc as an appropriate page table type. Now that all callers are using ptdescs, we can pass in that ptdesc to get the underlying mm_struct. Signed-off-by: Vishal Moola Signed-off-by: Dave Hansen Reviewed-by: William Kucharski Acked-by: Mike Rapoport (Microsoft) Link: https://patch.msgid.link/20260629185742.126987-8-vishal.moola@gmail.com --- arch/x86/include/asm/pgtable.h | 2 +- arch/x86/include/asm/pgtable_types.h | 2 +- arch/x86/mm/fault.c | 2 +- arch/x86/mm/init_64.c | 4 ++-- arch/x86/mm/pgtable.c | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index ac295ca6c92f..dd046f29ec31 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -50,7 +50,7 @@ void ptdump_walk_user_pgd_level_checkwx(void); extern spinlock_t pgd_lock; extern struct list_head pgd_list; -extern struct mm_struct *pgd_page_get_mm(struct page *page); +struct mm_struct *pgd_page_get_mm(struct ptdesc *pt); extern pmdval_t early_pmd_flags; diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h index 2ec250ba467e..95c7f453db17 100644 --- a/arch/x86/include/asm/pgtable_types.h +++ b/arch/x86/include/asm/pgtable_types.h @@ -512,7 +512,7 @@ static inline pgprot_t pgprot_large_2_4k(pgprot_t pgprot) return __pgprot(protval_large_2_4k(pgprot_val(pgprot))); } - +struct ptdesc; typedef struct page *pgtable_t; extern pteval_t __supported_pte_mask; diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index c31ffeac0714..aa88370ce739 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -282,7 +282,7 @@ void arch_sync_kernel_mappings(unsigned long start, unsigned long end) spinlock_t *pgt_lock; /* the pgt_lock only for Xen */ - pgt_lock = &pgd_page_get_mm(ptdesc_page(ptdesc))->page_table_lock; + pgt_lock = &pgd_page_get_mm(ptdesc)->page_table_lock; spin_lock(pgt_lock); vmalloc_sync_one(ptdesc_address(ptdesc), addr); diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 1441f5fbd84f..bf529a99ebd1 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -152,7 +152,7 @@ static void sync_global_pgds_l5(unsigned long start, unsigned long end) pgd = (pgd_t *)ptdesc_address(ptdesc) + pgd_index(addr); /* the pgt_lock only for Xen */ - pgt_lock = &pgd_page_get_mm(ptdesc_page(ptdesc))->page_table_lock; + pgt_lock = &pgd_page_get_mm(ptdesc)->page_table_lock; spin_lock(pgt_lock); if (!pgd_none(*pgd_ref) && !pgd_none(*pgd)) @@ -195,7 +195,7 @@ static void sync_global_pgds_l4(unsigned long start, unsigned long end) pgd = (pgd_t *)ptdesc_address(ptdesc) + pgd_index(addr); p4d = p4d_offset(pgd, addr); /* the pgt_lock only for Xen */ - pgt_lock = &pgd_page_get_mm(ptdesc_page(ptdesc))->page_table_lock; + pgt_lock = &pgd_page_get_mm(ptdesc)->page_table_lock; spin_lock(pgt_lock); if (!p4d_none(*p4d_ref) && !p4d_none(*p4d)) diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index f32facdb3035..cb03f5a2b243 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -74,9 +74,9 @@ static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm) virt_to_ptdesc(pgd)->pt_mm = mm; } -struct mm_struct *pgd_page_get_mm(struct page *page) +struct mm_struct *pgd_page_get_mm(struct ptdesc *pt) { - return page_ptdesc(page)->pt_mm; + return pt->pt_mm; } static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd) From 278ddbd27427e4f51e6fdc0e1f11930b22b2f5a9 Mon Sep 17 00:00:00 2001 From: Vishal Moola Date: Mon, 29 Jun 2026 11:57:41 -0700 Subject: [PATCH 08/20] x86/xen: Convert xen_mm_pin_all() to ptdescs Convert xen_mm_pin_all() to ptdescs in preparation for the eventual splitting of ptdescs from struct page. Continue checking PagePinned through the underlying page as we do not have a per-memdesc api for page flags yet. Signed-off-by: Vishal Moola Signed-off-by: Dave Hansen Reviewed-by: William Kucharski Reviewed-by: Juergen Gross Acked-by: Mike Rapoport (Microsoft) Link: https://patch.msgid.link/20260629185742.126987-9-vishal.moola@gmail.com --- arch/x86/xen/mmu_pv.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c index 820af6f0aa57..d2b3bbd7100e 100644 --- a/arch/x86/xen/mmu_pv.c +++ b/arch/x86/xen/mmu_pv.c @@ -836,15 +836,15 @@ static void xen_pgd_pin(struct mm_struct *mm) */ void xen_mm_pin_all(void) { - struct page *page; + struct ptdesc *ptdesc; spin_lock(&init_mm.page_table_lock); spin_lock(&pgd_lock); - list_for_each_entry(page, &pgd_list, lru) { - if (!PagePinned(page)) { - __xen_pgd_pin(&init_mm, (pgd_t *)page_address(page)); - SetPageSavePinned(page); + list_for_each_entry(ptdesc, &pgd_list, pt_list) { + if (!PagePinned(ptdesc_page(ptdesc))) { + __xen_pgd_pin(&init_mm, (pgd_t *)ptdesc_address(ptdesc)); + SetPageSavePinned(ptdesc_page(ptdesc)); } } From defbd61c2ab01654b37750ed1515864523280cc4 Mon Sep 17 00:00:00 2001 From: Vishal Moola Date: Mon, 29 Jun 2026 11:57:42 -0700 Subject: [PATCH 09/20] x86/xen: Convert xen_mm_unpin_all() to ptdescs Convert xen_mm_unpin_all() to ptdescs in preparation for the eventual splitting of ptdescs from struct page. Continue checking PagePinned through the underlying page as we do not have a per-memdesc api yet. Signed-off-by: Vishal Moola Signed-off-by: Dave Hansen Reviewed-by: William Kucharski Reviewed-by: Juergen Gross Acked-by: Mike Rapoport (Microsoft) Link: https://patch.msgid.link/20260629185742.126987-10-vishal.moola@gmail.com --- arch/x86/xen/mmu_pv.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c index d2b3bbd7100e..377c3c0d9f20 100644 --- a/arch/x86/xen/mmu_pv.c +++ b/arch/x86/xen/mmu_pv.c @@ -947,16 +947,16 @@ static void xen_pgd_unpin(struct mm_struct *mm) */ void xen_mm_unpin_all(void) { - struct page *page; + struct ptdesc *ptdesc; spin_lock(&init_mm.page_table_lock); spin_lock(&pgd_lock); - list_for_each_entry(page, &pgd_list, lru) { - if (PageSavePinned(page)) { - BUG_ON(!PagePinned(page)); - __xen_pgd_unpin(&init_mm, (pgd_t *)page_address(page)); - ClearPageSavePinned(page); + list_for_each_entry(ptdesc, &pgd_list, pt_list) { + if (PageSavePinned(ptdesc_page(ptdesc))) { + BUG_ON(!PagePinned(ptdesc_page(ptdesc))); + __xen_pgd_unpin(&init_mm, (pgd_t *)ptdesc_address(ptdesc)); + ClearPageSavePinned(ptdesc_page(ptdesc)); } } From 5fce67641a3ed9a0782eaa228ddece526461a367 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Wed, 15 Jul 2026 17:45:19 +0300 Subject: [PATCH 10/20] x86/mm/pat: Don't gate cpa_lock on debug_pagealloc_enabled() The splitting and merging of kernel page table mappings between small and large is protected by cpa_lock. The merging is relatively new but the splitting is ancient. The splitting has a locking optimization: since DEBUG_PAGEALLOC forces all mappings to 4k, there are no large pages to split. So the code that *might* cause a split can just skip the locking (and a few other things). This is entertaining, but it adds complexity and makes for weird locking rules. Plus it's all for a debugging feature which makes the kernel super slow in the first place. Optimizing something which is already super slow and not used in production is not the best way to spend our complexity budget. Stop gating cpa_lock on debug_pagealloc_enabled() to simplify the code and the locking rules. [ dhansen: flesh out changelog ] Suggested-by: Dave Hansen Signed-off-by: Mike Rapoport (Microsoft) Signed-off-by: Dave Hansen Link: https://patch.msgid.link/20260715144519.934289-1-rppt@kernel.org Link: https://lore.kernel.org/all/aab44f08-89f8-47fe-bee4-0ab6b25968c6@intel.com/ --- arch/x86/mm/pat/set_memory.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index 45623d4c24c9..e9b408343c5d 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -62,10 +62,9 @@ enum cpa_warn { static const int cpa_warn_level = CPA_PROTECT; /* - * Serialize cpa() (for !DEBUG_PAGEALLOC which uses large identity mappings) - * using cpa_lock. So that we don't allow any other cpu, with stale large tlb - * entries change the page attribute in parallel to some other cpu - * splitting a large page entry along with changing the attribute. + * Serialize cpa() using cpa_lock so that we don't allow any other cpu, with + * stale large tlb entries, to change the page attribute in parallel to some + * other cpu splitting a large page entry along with changing the attribute. */ static DEFINE_SPINLOCK(cpa_lock); @@ -1234,11 +1233,9 @@ static int split_large_page(struct cpa_data *cpa, pte_t *kpte, { struct ptdesc *ptdesc; - if (!debug_pagealloc_enabled()) - spin_unlock(&cpa_lock); + spin_unlock(&cpa_lock); ptdesc = pagetable_alloc(GFP_KERNEL, 0); - if (!debug_pagealloc_enabled()) - spin_lock(&cpa_lock); + spin_lock(&cpa_lock); if (!ptdesc) return -ENOMEM; @@ -2022,11 +2019,9 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary) if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY)) cpa->numpages = 1; - if (!debug_pagealloc_enabled()) - spin_lock(&cpa_lock); + spin_lock(&cpa_lock); ret = __change_page_attr(cpa, primary); - if (!debug_pagealloc_enabled()) - spin_unlock(&cpa_lock); + spin_unlock(&cpa_lock); if (ret) goto out; From 1aac65f3e651334259ecb2a5f5ddb81c01f02599 Mon Sep 17 00:00:00 2001 From: "Denis V. Lunev" Date: Wed, 15 Jul 2026 20:34:52 +0200 Subject: [PATCH 11/20] x86/mm/pat: Take cpa_lock around large-page collapse Loading and unloading modules concurrently on several CPUs on a KASAN build, with a short delay injected at the CPA page-table lookup to widen the window, faults within minutes: BUG: KASAN: use-after-free in __change_page_attr+0x7cc/0x7e0 Write of size 8 at addr ffff888181139718 by task modprobe ... The buggy address belongs to the physical page: pfn:0x181139 ... page_type: f2(table) cpa_collapse_large_pages() rebuilds a leaf PMD from its 4K PTEs and frees the old PTE-table pages, while __change_page_attr() fetches a PTE pointer from a lockless lookup_address_in_pgd_attr() and writes it with set_pte_atomic() only later. When module text is served from a shared large ROX mapping the two run on the same PMD: CPU A (module load) CPU B (module finalize) ------------------- ----------------------- execmem_make_temp_rw set_memory_nx __change_page_attr split 2M -> 4K table P kpte = &P[i] (lockless) execmem_restore_rox set_memory_rox (CPA_COLLAPSE) cpa_collapse_large_pages rebuild leaf PMD flush_tlb_all pagetable_free(P) set_pte_atomic(kpte, ...) -> writes into freed P P is a page-table page (page_type: table), reused at once, so the write corrupts whatever got the page next: a bad-pte or bad-page splat, or a fatal fault once P has been turned into read-only text. The flush_tlb_all() before the free does not close this: its IPI only serializes against page-table walkers that run with interrupts off (e.g. GUP-fast); the walk in __change_page_attr() runs with interrupts on, so nothing stops it from holding a stale pointer into P. Serialize the collapse - the PMD rebuild, TLB flush and PTE-table free - under cpa_lock, the same lock __change_page_attr() now takes unconditionally since commit ("x86/mm/pat: stop gating cpa_lock on debug_pagealloc_enabled()"), so a concurrent walker can no longer hold a pointer into a table the collapse is about to free. Fixes: 41d88484c71c ("x86/mm/pat: restore large ROX pages after fragmentation") Signed-off-by: Denis V. Lunev Signed-off-by: Dave Hansen Acked-by: Kiryl Shutsemau (Meta) Link: https://patch.msgid.link/20260715183453.2381141-1-den@openvz.org --- arch/x86/mm/pat/set_memory.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index e9b408343c5d..b1e780a465b5 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -417,6 +417,8 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa) int collapsed = 0; int i; + spin_lock(&cpa_lock); + if (cpa->flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) { for (i = 0; i < cpa->numpages; i++) collapsed += collapse_large_pages(__cpa_addr(cpa, i), @@ -430,8 +432,10 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa) collapsed += collapse_large_pages(addr, &pgtables); } - if (!collapsed) + if (!collapsed) { + spin_unlock(&cpa_lock); return; + } flush_tlb_all(); @@ -439,6 +443,8 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa) list_del(&ptdesc->pt_list); pagetable_free(ptdesc); } + + spin_unlock(&cpa_lock); } static void cpa_flush(struct cpa_data *cpa, int cache) From 33c179b5020554722bcb4c5a3ff565192ac53f54 Mon Sep 17 00:00:00 2001 From: Chuyi Zhou Date: Thu, 9 Jul 2026 20:29:29 +0800 Subject: [PATCH 12/20] x86/mm: Factor out flush_tlb_info initialization get_flush_tlb_info() has two responsibilities: it reserves the per-CPU flush_tlb_info storage and it initializes the fields that describe the flush operation. The per-CPU storage also carries the DEBUG_VM reentrancy check and the matching put_flush_tlb_info() lifetime rules. Moving flush_tlb_info back to caller-provided storage requires the same field initialization without tying the caller to the per-CPU object. Leaving the field setup embedded in get_flush_tlb_info() would either keep those callers tied to the per-CPU object or duplicate the initialization logic. Split the field setup into init_flush_tlb_info(). Keep the per-CPU storage selection, DEBUG_VM reentrancy check and put_flush_tlb_info() lifetime rules in get_flush_tlb_info(). No functional change intended. Signed-off-by: Chuyi Zhou Signed-off-by: Thomas Gleixner Reviewed-by: Sebastian Andrzej Siewior Link: https://patch.msgid.link/20260709122933.4021501-11-zhouchuyi@bytedance.com --- arch/x86/mm/tlb.c | 50 ++++++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 1023acadd8f8..7f483424de1d 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -1379,10 +1379,33 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(struct flush_tlb_info, flush_tlb_info); static DEFINE_PER_CPU(unsigned int, flush_tlb_info_idx); #endif -static struct flush_tlb_info *get_flush_tlb_info(struct mm_struct *mm, - unsigned long start, unsigned long end, - unsigned int stride_shift, bool freed_tables, - u64 new_tlb_gen) +static void init_flush_tlb_info(struct flush_tlb_info *info, struct mm_struct *mm, + unsigned long start, unsigned long end, + unsigned int stride_shift, bool freed_tables, + u64 new_tlb_gen) +{ + /* + * If the number of flushes is so large that a full flush + * would be faster, do a full flush. + */ + if ((end - start) >> stride_shift > tlb_single_page_flush_ceiling) { + start = 0; + end = TLB_FLUSH_ALL; + } + + info->start = start; + info->end = end; + info->mm = mm; + info->stride_shift = stride_shift; + info->freed_tables = freed_tables; + info->new_tlb_gen = new_tlb_gen; + info->initiating_cpu = smp_processor_id(); + info->trim_cpumask = 0; +} + +static struct flush_tlb_info *get_flush_tlb_info(struct mm_struct *mm, unsigned long start, + unsigned long end, unsigned int stride_shift, + bool freed_tables, u64 new_tlb_gen) { struct flush_tlb_info *info = this_cpu_ptr(&flush_tlb_info); @@ -1395,24 +1418,7 @@ static struct flush_tlb_info *get_flush_tlb_info(struct mm_struct *mm, BUG_ON(this_cpu_inc_return(flush_tlb_info_idx) != 1); #endif - /* - * If the number of flushes is so large that a full flush - * would be faster, do a full flush. - */ - if ((end - start) >> stride_shift > tlb_single_page_flush_ceiling) { - start = 0; - end = TLB_FLUSH_ALL; - } - - info->start = start; - info->end = end; - info->mm = mm; - info->stride_shift = stride_shift; - info->freed_tables = freed_tables; - info->new_tlb_gen = new_tlb_gen; - info->initiating_cpu = smp_processor_id(); - info->trim_cpumask = 0; - + init_flush_tlb_info(info, mm, start, end, stride_shift, freed_tables, new_tlb_gen); return info; } From c86f522bb57b257befe2ff5fbcb4b94065abb011 Mon Sep 17 00:00:00 2001 From: Chuyi Zhou Date: Thu, 9 Jul 2026 20:29:30 +0800 Subject: [PATCH 13/20] x86/mm: Cap flush_tlb_info alignment at 64 bytes A stack allocated flush_tlb_info should keep cacheline alignment to avoid the regression that motivated the per-CPU storage, but using SMP_CACHE_BYTES directly can make the stack frame grow excessively on configurations with large cache lines. This was addressed by commit 780e0106d468 ("x86/mm/tlb: Revert "x86/mm: Align TLB invalidation info""), where the stack consumption reached 320 bytes. Add FLUSH_TLB_INFO_ALIGN and cap the type alignment at 64 bytes. The existing per-CPU flush_tlb_info instance remains DEFINE_PER_CPU_SHARED_ALIGNED(), so its per-CPU shared-cacheline alignment is unchanged. This prepares for moving flush_tlb_info back to stack storage without reintroducing the old large-cacheline stack usage problem. Signed-off-by: Chuyi Zhou Signed-off-by: Thomas Gleixner Reviewed-by: Sebastian Andrzej Siewior Link: https://patch.msgid.link/20260709122933.4021501-12-zhouchuyi@bytedance.com --- arch/x86/include/asm/tlbflush.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h index 0545fe75c3fa..70098d448e99 100644 --- a/arch/x86/include/asm/tlbflush.h +++ b/arch/x86/include/asm/tlbflush.h @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -211,6 +212,12 @@ extern u16 invlpgb_count_max; extern void initialize_tlbstate_and_flush(void); +/* + * Keep stack-allocated flush_tlb_info cacheline aligned, but cap the + * alignment to avoid excessive stack usage on large-cacheline systems. + */ +#define FLUSH_TLB_INFO_ALIGN MIN(SMP_CACHE_BYTES, 64) + /* * TLB flushing: * @@ -249,7 +256,7 @@ struct flush_tlb_info { u8 stride_shift; u8 freed_tables; u8 trim_cpumask; -}; +} __aligned(FLUSH_TLB_INFO_ALIGN); void flush_tlb_local(void); void flush_tlb_one_user(unsigned long addr); From ed15468787dd954ecfabe16378e4d27c8a55c430 Mon Sep 17 00:00:00 2001 From: Chuyi Zhou Date: Thu, 9 Jul 2026 20:29:31 +0800 Subject: [PATCH 14/20] x86/mm: Move flush_tlb_info back to the stack flush_tlb_info benefits from cacheline alignment, but using cacheline-aligned stack storage directly can grow stack usage too much on configurations with large SMP_CACHE_BYTES values. Commit 515ab7c41306 ("x86/mm: Align TLB invalidation info") attempted to align stack storage, and commit 780e0106d468 ("x86/mm/tlb: Revert "x86/mm: Align TLB invalidation info"") reverted it because using SMP_CACHE_BYTES led to 320 bytes of stack consumption. Commit 3db6d5a5ecaf ("x86/mm/tlb: Remove 'struct flush_tlb_info' from the stack") moved flush_tlb_info to per-CPU storage, which avoided the stack growth problem while preserving cacheline alignment. That was a good fit while the callers kept preemption disabled for the whole flush operation. However, a single per-CPU flush_tlb_info also requires all flush_tlb*() operations to keep preemption disabled while the object is in use, so that it cannot be overwritten by another flush on the same CPU. flush_tlb*() may send IPIs to remote CPUs and synchronously wait for all remote CPUs to complete their local TLB flushes. That wait can take tens of milliseconds when interrupts are disabled on a remote CPU or when a large number of remote CPUs are involved. To shorten the CPU-pinned and preemption-disabled section around those remote TLB flush waits, move flush_tlb_info back to caller-private stack storage. The caller then does not have to stay on the same CPU until the remote flush completes. The type alignment is capped at 64 bytes. This keeps the alignment benefit for stack objects without reintroducing the old large-cacheline stack usage problem. To evaluate the performance impact, use the following script to reproduce the microbenchmark mentioned in commit 3db6d5a5ecaf ("x86/mm/tlb: Remove 'struct flush_tlb_info' from the stack"). The test environment is an Ice Lake system (Intel(R) Xeon(R) Platinum 8336C) with 128 CPUs and 2 NUMA nodes. During the test, the threads were bound to specific CPUs, and both pti and mitigations were disabled: #include #include #include #include #include #include #define NUM_OPS 1000000 #define NUM_THREADS 3 #define NUM_RUNS 5 #define PAGE_SIZE 4096 volatile int stop_threads = 0; void *busy_wait_thread(void *arg) { while (!stop_threads) { __asm__ volatile ("nop"); } return NULL; } long long get_usec() { struct timeval tv; gettimeofday(&tv, NULL); return tv.tv_sec * 1000000LL + tv.tv_usec; } int main() { pthread_t threads[NUM_THREADS]; char *addr; int i, r; addr = mmap(NULL, PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (addr == MAP_FAILED) { perror("mmap"); exit(1); } for (i = 0; i < NUM_THREADS; i++) { if (pthread_create(&threads[i], NULL, busy_wait_thread, NULL)) exit(1); } printf("Running benchmark: %d runs, %d ops each, %d background\n" "threads\n", NUM_RUNS, NUM_OPS, NUM_THREADS); for (r = 0; r < NUM_RUNS; r++) { long long start, end; start = get_usec(); for (i = 0; i < NUM_OPS; i++) { addr[0] = 1; if (madvise(addr, PAGE_SIZE, MADV_DONTNEED)) { perror("madvise"); exit(1); } } end = get_usec(); double duration = (double)(end - start); double avg_lat = duration / NUM_OPS; printf("Run %d: Total time %.2f us, Avg latency %.4f us/op\n", r + 1, duration, avg_lat); } stop_threads = 1; for (i = 0; i < NUM_THREADS; i++) pthread_join(threads[i], NULL); munmap(addr, PAGE_SIZE); return 0; } base on-stack-aligned on-stack-not-aligned ---- --------- ----------- avg (usec/op) 2.5278 2.5261 2.5508 stddev 0.0007 0.0027 0.0023 The benchmark results show that the average latency difference between the baseline (base) and the properly aligned stack variable (on-stack-aligned) is within the standard deviation (stddev). This indicates that the variations are caused by testing noise, and reverting to a stack variable with proper alignment causes no performance regression compared to the per-CPU implementation. The unaligned version (on-stack-not-aligned) shows a minor performance drop. The CPU-pinned/preemption-disabled section can therefore be shortened without sacrificing performance. With caller-private storage there is no shared per-CPU object to protect, so remove the DEBUG_VM reentrancy counter as well. Signed-off-by: Chuyi Zhou Signed-off-by: Thomas Gleixner Tested-by: Paul E. McKenney Reviewed-by: Sebastian Andrzej Siewior Acked-by: Nadav Amit Link: https://patch.msgid.link/20260709122933.4021501-13-zhouchuyi@bytedance.com --- arch/x86/mm/tlb.c | 77 ++++++++++++----------------------------------- 1 file changed, 19 insertions(+), 58 deletions(-) diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 7f483424de1d..b464a733ef37 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -1373,13 +1373,8 @@ void flush_tlb_multi(const struct cpumask *cpumask, */ unsigned long tlb_single_page_flush_ceiling __read_mostly = 33; -static DEFINE_PER_CPU_SHARED_ALIGNED(struct flush_tlb_info, flush_tlb_info); - -#ifdef CONFIG_DEBUG_VM -static DEFINE_PER_CPU(unsigned int, flush_tlb_info_idx); -#endif - -static void init_flush_tlb_info(struct flush_tlb_info *info, struct mm_struct *mm, +static void init_flush_tlb_info(struct flush_tlb_info *info, + struct mm_struct *mm, unsigned long start, unsigned long end, unsigned int stride_shift, bool freed_tables, u64 new_tlb_gen) @@ -1403,47 +1398,18 @@ static void init_flush_tlb_info(struct flush_tlb_info *info, struct mm_struct *m info->trim_cpumask = 0; } -static struct flush_tlb_info *get_flush_tlb_info(struct mm_struct *mm, unsigned long start, - unsigned long end, unsigned int stride_shift, - bool freed_tables, u64 new_tlb_gen) -{ - struct flush_tlb_info *info = this_cpu_ptr(&flush_tlb_info); - -#ifdef CONFIG_DEBUG_VM - /* - * Ensure that the following code is non-reentrant and flush_tlb_info - * is not overwritten. This means no TLB flushing is initiated by - * interrupt handlers and machine-check exception handlers. - */ - BUG_ON(this_cpu_inc_return(flush_tlb_info_idx) != 1); -#endif - - init_flush_tlb_info(info, mm, start, end, stride_shift, freed_tables, new_tlb_gen); - return info; -} - -static void put_flush_tlb_info(void) -{ -#ifdef CONFIG_DEBUG_VM - /* Complete reentrancy prevention checks */ - barrier(); - this_cpu_dec(flush_tlb_info_idx); -#endif -} - void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, unsigned long end, unsigned int stride_shift, bool freed_tables) { - struct flush_tlb_info *info; + struct flush_tlb_info info; int cpu = get_cpu(); u64 new_tlb_gen; /* This is also a barrier that synchronizes with switch_mm(). */ new_tlb_gen = inc_mm_tlb_gen(mm); - info = get_flush_tlb_info(mm, start, end, stride_shift, freed_tables, - new_tlb_gen); + init_flush_tlb_info(&info, mm, start, end, stride_shift, freed_tables, new_tlb_gen); /* * flush_tlb_multi() is not optimized for the common case in which only @@ -1451,19 +1417,18 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, * flush_tlb_func_local() directly in this case. */ if (mm_global_asid(mm)) { - broadcast_tlb_flush(info); + broadcast_tlb_flush(&info); } else if (cpumask_any_but(mm_cpumask(mm), cpu) < nr_cpu_ids) { - info->trim_cpumask = should_trim_cpumask(mm); - flush_tlb_multi(mm_cpumask(mm), info); + info.trim_cpumask = should_trim_cpumask(mm); + flush_tlb_multi(mm_cpumask(mm), &info); consider_global_asid(mm); } else if (mm == this_cpu_read(cpu_tlbstate.loaded_mm)) { lockdep_assert_irqs_enabled(); local_irq_disable(); - flush_tlb_func(info); + flush_tlb_func(&info); local_irq_enable(); } - put_flush_tlb_info(); put_cpu(); mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end); } @@ -1533,19 +1498,16 @@ static void kernel_tlb_flush_range(struct flush_tlb_info *info) void flush_tlb_kernel_range(unsigned long start, unsigned long end) { - struct flush_tlb_info *info; + struct flush_tlb_info info; guard(preempt)(); + init_flush_tlb_info(&info, NULL, start, end, PAGE_SHIFT, false, + TLB_GENERATION_INVALID); - info = get_flush_tlb_info(NULL, start, end, PAGE_SHIFT, false, - TLB_GENERATION_INVALID); - - if (info->end == TLB_FLUSH_ALL) - kernel_tlb_flush_all(info); + if (info.end == TLB_FLUSH_ALL) + kernel_tlb_flush_all(&info); else - kernel_tlb_flush_range(info); - - put_flush_tlb_info(); + kernel_tlb_flush_range(&info); } /* @@ -1713,12 +1675,12 @@ EXPORT_SYMBOL_FOR_KVM(__flush_tlb_all); void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) { - struct flush_tlb_info *info; + struct flush_tlb_info info; int cpu = get_cpu(); - info = get_flush_tlb_info(NULL, 0, TLB_FLUSH_ALL, 0, false, - TLB_GENERATION_INVALID); + init_flush_tlb_info(&info, NULL, 0, TLB_FLUSH_ALL, 0, false, + TLB_GENERATION_INVALID); /* * flush_tlb_multi() is not optimized for the common case in which only * a local TLB flush is needed. Optimize this use-case by calling @@ -1728,17 +1690,16 @@ void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) invlpgb_flush_all_nonglobals(); batch->unmapped_pages = false; } else if (cpumask_any_but(&batch->cpumask, cpu) < nr_cpu_ids) { - flush_tlb_multi(&batch->cpumask, info); + flush_tlb_multi(&batch->cpumask, &info); } else if (cpumask_test_cpu(cpu, &batch->cpumask)) { lockdep_assert_irqs_enabled(); local_irq_disable(); - flush_tlb_func(info); + flush_tlb_func(&info); local_irq_enable(); } cpumask_clear(&batch->cpumask); - put_flush_tlb_info(); put_cpu(); } From ae397bb32d2a4cb97d93539a5e50d1d081a7569a Mon Sep 17 00:00:00 2001 From: Chuyi Zhou Date: Thu, 9 Jul 2026 20:29:32 +0800 Subject: [PATCH 15/20] x86/kvm: Disable preemption in kvm_flush_tlb_multi() kvm_flush_tlb_multi() is installed as an x86 PV TLB flush backend, so flush_tlb_multi() can reach it through pv_ops when running as a KVM guest. kvm_flush_tlb_multi() uses the per-CPU scratch cpumask __pv_cpu_mask. That buffer must remain tied to the current CPU until the mask has been copied, filtered, and consumed by native_flush_tlb_multi(). The x86/mm callers currently enter flush_tlb_multi() while pinned to a CPU. To let those callers drop CPU pinning before issuing the remote TLB flush, each PV backend must protect its own CPU-local scratch state. Make the KVM backend protect its per-CPU scratch cpumask by disabling preemption locally. This is harmless with the current callers, where the preemption disable is nested, and makes the KVM pv_ops dependency explicit before changing the x86/mm call sites. Signed-off-by: Chuyi Zhou Signed-off-by: Thomas Gleixner Reviewed-by: Sebastian Andrzej Siewior Link: https://patch.msgid.link/20260709122933.4021501-14-zhouchuyi@bytedance.com --- arch/x86/kernel/kvm.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index dcef84da304b..5b554e480c0e 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -663,8 +663,10 @@ static void kvm_flush_tlb_multi(const struct cpumask *cpumask, u8 state; int cpu; struct kvm_steal_time *src; - struct cpumask *flushmask = this_cpu_cpumask_var_ptr(__pv_cpu_mask); + struct cpumask *flushmask; + guard(preempt)(); + flushmask = this_cpu_cpumask_var_ptr(__pv_cpu_mask); cpumask_copy(flushmask, cpumask); /* * We have to call flush only on online vCPUs. And From a5a162fe1ae130e3d2ceefef3f43afe3773c1d56 Mon Sep 17 00:00:00 2001 From: Chuyi Zhou Date: Thu, 9 Jul 2026 20:29:33 +0800 Subject: [PATCH 16/20] x86/mm: Re-enable preemption before flush_tlb_multi() flush_tlb_mm_range() and arch_tlbbatch_flush() pin the current CPU while they decide whether the flush can be handled locally or must be sent to remote CPUs. The CPU pinning is needed for the current CPU number and for the local TLB flush path, which reads per-CPU TLB state. The caller does not need to remain pinned while waiting for a remote TLB flush to complete. After the remote-flush path has been selected, flush_tlb_info is caller-private stack storage, so the caller no longer has to stay on the same CPU to protect a shared per-CPU flush_tlb_info object. flush_tlb_multi() may also route through x86 PV backends. Those backends must protect their own CPU-local scratch state instead of relying on the caller to stay pinned. Hyper-V already does this by disabling interrupts while using hyperv_pcpu_input_arg, and Xen's multicall path brackets its per-CPU multicall buffer with xen_mc_batch() and xen_mc_issue(). kvm_flush_tlb_multi() also disables preemption while using __pv_cpu_mask. Remote TLB flushes may synchronously wait for many CPUs, and the wait can take tens of milliseconds when remote CPUs have interrupts disabled or when many CPUs are involved. Keeping preemption disabled for that whole wait unnecessarily increases scheduling latency on the initiating CPU. Drop the CPU pinning before calling flush_tlb_multi() in the remote paths of flush_tlb_mm_range() and arch_tlbbatch_flush(). Keep the local paths inside the pinned section because they still access this CPU's TLB state. Signed-off-by: Chuyi Zhou Signed-off-by: Thomas Gleixner Tested-by: Paul E. McKenney Reviewed-by: Sebastian Andrzej Siewior Link: https://patch.msgid.link/20260709122933.4021501-15-zhouchuyi@bytedance.com --- arch/x86/mm/tlb.c | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index b464a733ef37..e5a0c6e32d19 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -1403,6 +1403,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, bool freed_tables) { struct flush_tlb_info info; + bool remote_flush = false; int cpu = get_cpu(); u64 new_tlb_gen; @@ -1419,9 +1420,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, if (mm_global_asid(mm)) { broadcast_tlb_flush(&info); } else if (cpumask_any_but(mm_cpumask(mm), cpu) < nr_cpu_ids) { - info.trim_cpumask = should_trim_cpumask(mm); - flush_tlb_multi(mm_cpumask(mm), &info); - consider_global_asid(mm); + remote_flush = true; } else if (mm == this_cpu_read(cpu_tlbstate.loaded_mm)) { lockdep_assert_irqs_enabled(); local_irq_disable(); @@ -1430,6 +1429,13 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, } put_cpu(); + + if (remote_flush) { + info.trim_cpumask = should_trim_cpumask(mm); + flush_tlb_multi(mm_cpumask(mm), &info); + consider_global_asid(mm); + } + mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end); } @@ -1676,7 +1682,7 @@ EXPORT_SYMBOL_FOR_KVM(__flush_tlb_all); void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) { struct flush_tlb_info info; - + bool remote_flush = false; int cpu = get_cpu(); init_flush_tlb_info(&info, NULL, 0, TLB_FLUSH_ALL, 0, false, @@ -1690,7 +1696,7 @@ void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) invlpgb_flush_all_nonglobals(); batch->unmapped_pages = false; } else if (cpumask_any_but(&batch->cpumask, cpu) < nr_cpu_ids) { - flush_tlb_multi(&batch->cpumask, &info); + remote_flush = true; } else if (cpumask_test_cpu(cpu, &batch->cpumask)) { lockdep_assert_irqs_enabled(); local_irq_disable(); @@ -1698,9 +1704,12 @@ void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) local_irq_enable(); } - cpumask_clear(&batch->cpumask); - put_cpu(); + + if (remote_flush) + flush_tlb_multi(&batch->cpumask, &info); + + cpumask_clear(&batch->cpumask); } /* From c980eafc5b738f274b36fac634f06ff000a07038 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 29 Jul 2026 13:08:08 +0200 Subject: [PATCH 17/20] x86/mm: Use guard() in cpa_collapse_large_pages() Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260729111119.375614011@infradead.org --- arch/x86/mm/pat/set_memory.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index b1e780a465b5..fc6a1b28a701 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -417,7 +417,7 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa) int collapsed = 0; int i; - spin_lock(&cpa_lock); + guard(spinlock)(&cpa_lock); if (cpa->flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) { for (i = 0; i < cpa->numpages; i++) @@ -432,10 +432,8 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa) collapsed += collapse_large_pages(addr, &pgtables); } - if (!collapsed) { - spin_unlock(&cpa_lock); + if (!collapsed) return; - } flush_tlb_all(); @@ -443,8 +441,6 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa) list_del(&ptdesc->pt_list); pagetable_free(ptdesc); } - - spin_unlock(&cpa_lock); } static void cpa_flush(struct cpa_data *cpa, int cache) From 7558828805dae97ed82c8662119667189d062415 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 29 Jul 2026 13:08:09 +0200 Subject: [PATCH 18/20] x86/mm: Use guard() for pgd_lock Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260729111119.488154104@infradead.org --- arch/x86/mm/pat/set_memory.c | 34 +++++++++++----------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index fc6a1b28a701..ad52c9de48f4 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -85,9 +85,8 @@ static unsigned long direct_pages_count[PG_LEVEL_NUM]; void update_page_count(int level, unsigned long pages) { /* Protect against CPA */ - spin_lock(&pgd_lock); + guard(spinlock)(&pgd_lock); direct_pages_count[level] += pages; - spin_unlock(&pgd_lock); } static void split_page_count(int level) @@ -1075,16 +1074,11 @@ static int __should_split_large_page(pte_t *kpte, unsigned long address, static int should_split_large_page(pte_t *kpte, unsigned long address, struct cpa_data *cpa) { - int do_split; - if (cpa->force_split) return 1; - spin_lock(&pgd_lock); - do_split = __should_split_large_page(kpte, address, cpa); - spin_unlock(&pgd_lock); - - return do_split; + guard(spinlock)(&pgd_lock); + return __should_split_large_page(kpte, address, cpa); } static void split_set_pte(struct cpa_data *cpa, pte_t *pte, unsigned long pfn, @@ -1135,16 +1129,14 @@ __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address, bool nx, rw; pte_t *tmp; - spin_lock(&pgd_lock); + guard(spinlock)(&pgd_lock); /* * Check for races, another CPU might have split this page * up for us already: */ tmp = _lookup_address_cpa(cpa, address, &level, &nx, &rw); - if (tmp != kpte) { - spin_unlock(&pgd_lock); + if (tmp != kpte) return 1; - } paravirt_alloc_pte(&init_mm, page_to_pfn(base)); @@ -1177,7 +1169,6 @@ __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address, break; default: - spin_unlock(&pgd_lock); return 1; } @@ -1225,7 +1216,6 @@ __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address, * just split large page entry. */ flush_tlb_all(); - spin_unlock(&pgd_lock); return 0; } @@ -1374,7 +1364,7 @@ static int collapse_pud_page(pud_t *pud, unsigned long addr, */ static int collapse_large_pages(unsigned long addr, struct list_head *pgtables) { - int collapsed = 0; + int collapsed; pgd_t *pgd; p4d_t *p4d; pud_t *pud; @@ -1382,26 +1372,24 @@ static int collapse_large_pages(unsigned long addr, struct list_head *pgtables) addr &= PMD_MASK; - spin_lock(&pgd_lock); + guard(spinlock)(&pgd_lock); pgd = pgd_offset_k(addr); if (pgd_none(*pgd)) - goto out; + return 0; p4d = p4d_offset(pgd, addr); if (p4d_none(*p4d)) - goto out; + return 0; pud = pud_offset(p4d, addr); if (!pud_present(*pud) || pud_leaf(*pud)) - goto out; + return 0; pmd = pmd_offset(pud, addr); if (!pmd_present(*pmd) || pmd_leaf(*pmd)) - goto out; + return 0; collapsed = collapse_pmd_page(pmd, addr, pgtables); if (collapsed) collapsed += collapse_pud_page(pud, addr, pgtables); -out: - spin_unlock(&pgd_lock); return collapsed; } From 7da514d819a0afb148634aac92b3d190f34947c3 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 29 Jul 2026 13:08:10 +0200 Subject: [PATCH 19/20] x86/mm: Fix and document DEBUG_PAGEALLOC It turns out that commit 5fce67641a3e ("x86/mm/pat: Don't gate cpa_lock on debug_pagealloc_enabled()") was a little too quick to remove the debug_pagealloc exception for cpa_lock. Notably __kernel_map_pages() is used by the page-allocator from any context the page-allocator itself is used, which violates the cpa_lock rules. Re-instate the exception, except make it specific to the __kernel_map_pages() such that any other cpa() usage is still fully serialized by cpa_lock. Also note that since cpa() should not be used on memory that isn't allocated, the page-allocator locking and cpa are infact mutually exclusive and all cpa usage in fully serialized. Add a comment explaining this and other 'funnies' surrounding DEBUG_PAGEALLOC, including how pgd_lock is not affected and the TLB trickery. Fixes: 5fce67641a3e ("x86/mm/pat: Don't gate cpa_lock on debug_pagealloc_enabled()") Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260729111119.604452135@infradead.org --- arch/x86/mm/pat/set_memory.c | 80 ++++++++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 22 deletions(-) diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index ad52c9de48f4..d8d057f44417 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -68,11 +68,12 @@ static const int cpa_warn_level = CPA_PROTECT; */ static DEFINE_SPINLOCK(cpa_lock); -#define CPA_FLUSHTLB 1 -#define CPA_ARRAY 2 -#define CPA_PAGES_ARRAY 4 -#define CPA_NO_CHECK_ALIAS 8 /* Do not search for aliases */ -#define CPA_COLLAPSE 16 /* try to collapse large pages */ +#define CPA_FLUSHTLB 0x01 +#define CPA_ARRAY 0x02 +#define CPA_PAGES_ARRAY 0x04 +#define CPA_NO_CHECK_ALIAS 0x08 /* Do not search for aliases */ +#define CPA_COLLAPSE 0x10 /* try to collapse large pages */ +#define CPA_DEBUG_PAGEALLOC 0x20 static inline pgprot_t cachemode2pgprot(enum page_cache_mode pcm) { @@ -1990,6 +1991,7 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary) { unsigned long numpages = cpa->numpages; unsigned long rempages = numpages; + bool lock = true; int ret = 0; /* @@ -1999,6 +2001,29 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary) !cpa->force_split) return ret; + /* + * DEBUG_PAGEALLOC is special; it is called from any context the + * page-allocator is, which violates the normal cpa_lock locking + * rules. + * + * However, since it is part of the page-allocator, things are still + * properly serialized by the page-allocator locking and the fact that + * when a page is owned by the page-allocator, it isn't owned by + * anybody else. That is, you *SHOULD NOT* be calling cpa() on memory + * that isn't allocated. + * + * Additionally, DEBUG_PAGEALLOC ensures (per probe_page_size_mask()) + * that the kernel mapping is 4k pages, therefore there are no large + * pages to split/collapse. + * + * Furthermore, the page-allocator strictly manages pages that + * *exist*, avoiding pgd_lock. + * + * Therefore, it is safe to not take cpa_lock. + */ + if (debug_pagealloc_enabled() && (cpa->flags & CPA_DEBUG_PAGEALLOC)) + lock = false; + while (rempages) { /* * Store the remaining nr of pages for the large page @@ -2009,9 +2034,12 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary) if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY)) cpa->numpages = 1; - spin_lock(&cpa_lock); - ret = __change_page_attr(cpa, primary); - spin_unlock(&cpa_lock); + if (lock) { + guard(spinlock)(&cpa_lock); + ret = __change_page_attr(cpa, primary); + } else { + ret = __change_page_attr(cpa, primary); + } if (ret) goto out; @@ -2590,7 +2618,7 @@ int set_pages_rw(struct page *page, int numpages) return set_memory_rw(addr, numpages); } -static int __set_pages_p(struct page *page, int numpages) +static int __set_pages_p(struct page *page, int numpages, unsigned int cpa_flags) { unsigned long tempaddr = (unsigned long) page_address(page); struct cpa_data cpa = { .vaddr = &tempaddr, @@ -2598,7 +2626,7 @@ static int __set_pages_p(struct page *page, int numpages) .numpages = numpages, .mask_set = __pgprot(_PAGE_PRESENT | _PAGE_RW), .mask_clr = __pgprot(0), - .flags = CPA_NO_CHECK_ALIAS }; + .flags = CPA_NO_CHECK_ALIAS | cpa_flags }; /* * No alias checking needed for setting present flag. otherwise, @@ -2609,7 +2637,7 @@ static int __set_pages_p(struct page *page, int numpages) return __change_page_attr_set_clr(&cpa, 1); } -static int __set_pages_np(struct page *page, int numpages) +static int __set_pages_np(struct page *page, int numpages, unsigned int cpa_flags) { unsigned long tempaddr = (unsigned long) page_address(page); struct cpa_data cpa = { .vaddr = &tempaddr, @@ -2617,7 +2645,7 @@ static int __set_pages_np(struct page *page, int numpages) .numpages = numpages, .mask_set = __pgprot(0), .mask_clr = __pgprot(_PAGE_PRESENT | _PAGE_RW | _PAGE_DIRTY), - .flags = CPA_NO_CHECK_ALIAS }; + .flags = CPA_NO_CHECK_ALIAS | cpa_flags }; /* * No alias checking needed for setting not present flag. otherwise, @@ -2630,20 +2658,20 @@ static int __set_pages_np(struct page *page, int numpages) int set_direct_map_invalid_noflush(struct page *page) { - return __set_pages_np(page, 1); + return __set_pages_np(page, 1, 0); } int set_direct_map_default_noflush(struct page *page) { - return __set_pages_p(page, 1); + return __set_pages_p(page, 1, 0); } int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) { if (valid) - return __set_pages_p(page, nr); + return __set_pages_p(page, nr, 0); - return __set_pages_np(page, nr); + return __set_pages_np(page, nr, 0); } #ifdef CONFIG_DEBUG_PAGEALLOC @@ -2662,15 +2690,23 @@ void __kernel_map_pages(struct page *page, int numpages, int enable) * and hence no memory allocations during large page split. */ if (enable) - __set_pages_p(page, numpages); + __set_pages_p(page, numpages, CPA_DEBUG_PAGEALLOC); else - __set_pages_np(page, numpages); + __set_pages_np(page, numpages, CPA_DEBUG_PAGEALLOC); /* - * We should perform an IPI and flush all tlbs, - * but that can deadlock->flush only current cpu. - * Preemption needs to be disabled around __flush_tlb_all() due to - * CR3 reload in __native_flush_tlb(). + * We should perform an IPI and flush all tlbs, but that can + * deadlock, settle for a local flush. + * + * Not doing a global TLB flush means that remote CPUs will retain + * stale TLB entries. In case of P->NP (on free) this means the remote + * CPUs will not take the faults, making the debug scheme less + * reliable. On the NP->P (on alloc) this means the remote CPUs can + * take a spurious fault. However spurious_kernel_fault() will observe + * *_present() and fix it up. + * + * Preemption needs to be disabled around __flush_tlb_all() due to CR3 + * reload in __native_flush_tlb(). */ preempt_disable(); __flush_tlb_all(); From dee87e09b0dd63da9b1e1876167ccae37842dfd0 Mon Sep 17 00:00:00 2001 From: Bijan Tabatabai Date: Thu, 16 Jul 2026 17:06:04 -0500 Subject: [PATCH 20/20] x86/pkeys: Fix pkey_alloc() return value when pkeys are not supported The man page for pkey_alloc(2) specifies that it should return -1 with the errno set to ENOSPC when pkeys are not supported [1]. However, on x86 pkey_alloc() sets errno to EINVAL when called for the first time on a CPU that does not support pkeys. The root cause of this is the x86 implementation of mm_pkey_alloc() not directly checking if pkeys are supported. It only checks if all the pkeys have been allocated by comparing the allocation map against all_pkeys_mask. When OSPKE is not enabled, init_new_context() skips the initialization of the allocation map, leaving it as 0, while all_pkeys_mask is 1. mm_pkey_alloc() interprets this as there being a pkey available and it returns pkey 0. Then, pkey_alloc() fails with -EINVAL from arch_set_user_pkey_access() instead of returning -ENOSPC. Subsequent calls to pkey_alloc() do return -ENOSPC because pkey 0 is left marked as allocated. Change mm_pkey_alloc() to directly check if OSPKE is enabled, and return -1 if it is not, which causes pkey_alloc() to return -ENOSPC. The arm64 and powerpc implementations of mm_pkey_alloc() already do this check. [1] https://man7.org/linux/man-pages/man2/pkey_alloc.2.html [ dhansen: use arch_pkeys_enabled() to follow arm ] Fixes: e8c24d3a23a4 ("x86/pkeys: Allocation/free syscalls") Signed-off-by: Bijan Tabatabai Signed-off-by: Dave Hansen Link: https://patch.msgid.link/20260716220604.26452-1-bijan311@gmail.com --- arch/x86/include/asm/pkeys.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/x86/include/asm/pkeys.h b/arch/x86/include/asm/pkeys.h index 06ed2cd2592e..fcec52132fc7 100644 --- a/arch/x86/include/asm/pkeys.h +++ b/arch/x86/include/asm/pkeys.h @@ -88,6 +88,9 @@ int mm_pkey_alloc(struct mm_struct *mm) u16 all_pkeys_mask = ((1U << arch_max_pkey()) - 1); int ret; + if (!arch_pkeys_enabled()) + return -1; + /* * Are we out of pkeys? We must handle this specially * because ffz() behavior is undefined if there are no