diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 2822190daf2b..a8174d1d3848 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2930,7 +2930,7 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm } folio_move_anon_rmap(src_folio, dst_vma); - src_folio->index = linear_page_index(dst_vma, dst_addr); + src_folio->index = linear_anon_page_index(dst_vma, dst_addr); _dst_pmd = folio_mk_pmd(src_folio, dst_vma->vm_page_prot); /* Follow mremap() behavior and treat the entry dirty after the move */ diff --git a/mm/internal.h b/mm/internal.h index a38b5f7896b2..16750b130ec4 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -945,7 +945,8 @@ folio_within_range(struct folio *folio, struct vm_area_struct *vma, return false; pgoff_folio = folio_pgoff(folio); - pgoff_vma_start = vma_start_pgoff(vma); + pgoff_vma_start = folio_test_anon(folio) ? + vma_start_anon_pgoff(vma) : vma_start_pgoff(vma); if (start < vma->vm_start) start = vma->vm_start; @@ -1056,23 +1057,8 @@ static inline unsigned long vma_filebacked_address(const struct vm_area_struct * } /** - * vma_address - Find the virtual address a page range is mapped at. - * @vma: The vma which maps this object. - * @pgoff: The page offset within its object. - * @nr_pages: The number of pages to consider. - * - * If any page in this range is mapped by this VMA, return the first address - * where any of these pages appear. Otherwise, return -EFAULT. - */ -static inline unsigned long vma_address(const struct vm_area_struct *vma, - pgoff_t pgoff, unsigned long nr_pages) -{ - return __vma_address(vma, pgoff, vma_start_pgoff(vma), nr_pages); -} - -/** - * vma_anon_address - Find the address an anonymous folio with index @pgoff_anon - * is mapped at. + * vma_anon_address - Find the virtual address an anonymous page range is mapped + * at. * @vma: The vma which maps this object. * @pgoff_anon: The anonymous page index belonging to the folio. * @nr_pages: The number of pages to consider. @@ -1105,7 +1091,10 @@ static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw) if (pvmw->nr_pages == 1) return pvmw->address + PAGE_SIZE; - pgoff_vma_start = vma_start_pgoff(vma); + if (pvmw->pgoff_is_anon) + pgoff_vma_start = vma_start_anon_pgoff(vma); + else + pgoff_vma_start = vma_start_pgoff(vma); address = vma->vm_start + ((pgoff_end - pgoff_vma_start) << PAGE_SHIFT); diff --git a/mm/interval_tree.c b/mm/interval_tree.c index 3ae9e106d3af..7bbbf15cfbf0 100644 --- a/mm/interval_tree.c +++ b/mm/interval_tree.c @@ -83,12 +83,12 @@ mapping_rmap_tree_iter_next(struct vm_area_struct *vma, static pgoff_t avc_start_pgoff(struct anon_vma_chain *avc) { - return vma_start_pgoff(avc->vma); + return vma_start_anon_pgoff(avc->vma); } static pgoff_t avc_last_pgoff(struct anon_vma_chain *avc) { - return vma_last_pgoff(avc->vma); + return vma_last_anon_pgoff(avc->vma); } INTERVAL_TREE_DEFINE(struct anon_vma_chain, rb, pgoff_t, rb_subtree_last, diff --git a/mm/ksm.c b/mm/ksm.c index b4142746777e..b5854dc14a2e 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -1625,7 +1625,7 @@ static int try_to_merge_with_ksm_page(struct ksm_rmap_item *rmap_item, * stable_tree, break_cow() will clean it up. */ rmap_item->anon_vma = vma->anon_vma; - rmap_item->linear_page_index = linear_page_index(vma, rmap_item->address); + rmap_item->linear_page_index = linear_anon_page_index(vma, rmap_item->address); get_anon_vma(vma->anon_vma); out: mmap_read_unlock(mm); @@ -3152,7 +3152,7 @@ struct folio *ksm_might_need_to_copy(struct folio *folio, return folio; /* no need to copy it */ } else if (!anon_vma) { return folio; /* no need to copy it */ - } else if (folio->index == linear_page_index(vma, addr) && + } else if (folio->index == linear_anon_page_index(vma, addr) && anon_vma->root == vma->anon_vma->root) { return folio; /* still no need to copy it */ } @@ -3222,7 +3222,7 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc) /* * Currently, KSM folios are always small folios, so it's * sufficient to search for a single page. We can simply use - * the linear_page_index of the original de-duplicate + * the linear_anon_page_index of the original de-duplicate * anonymous page that we remembered in the rmap_item while * de-duplicating. Note that mremap() always de-duplicates KSM * folios: so if there was mremap() in our parent or our child, diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index 081e483cc7bf..4e964545e5e8 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -365,7 +365,7 @@ unsigned long page_mapped_in_vma(const struct page *page, }; if (folio_test_anon(folio)) - pvmw.address = vma_address(vma, pgoff, 1); + pvmw.address = vma_anon_address(vma, pgoff, 1); else pvmw.address = vma_filebacked_address(vma, pgoff, 1); if (pvmw.address == -EFAULT) diff --git a/mm/rmap.c b/mm/rmap.c index 1b23ac709f45..34ceeb600111 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -866,7 +866,7 @@ unsigned long page_address_in_vma(const struct folio *folio, vma->anon_vma->root != anon_vma->root) return -EFAULT; /* KSM folios don't reach here because of the !anon_vma check */ - return vma_address(vma, page_pgoff(folio, page), 1); + return vma_anon_address(vma, page_pgoff(folio, page), 1); } else if (!vma->vm_file) { return -EFAULT; } else if (vma->vm_file->f_mapping != folio->mapping) { @@ -1485,7 +1485,7 @@ static void __folio_set_anon(struct folio *folio, struct vm_area_struct *vma, */ anon_vma = (void *) anon_vma + FOLIO_MAPPING_ANON; WRITE_ONCE(folio->mapping, (struct address_space *) anon_vma); - folio->index = linear_page_index(vma, address); + folio->index = linear_anon_page_index(vma, address); } /** @@ -1512,8 +1512,8 @@ static void __page_check_anon_rmap(const struct folio *folio, */ VM_BUG_ON_FOLIO(folio_anon_vma(folio)->root != vma->anon_vma->root, folio); - VM_BUG_ON_PAGE(page_pgoff(folio, page) != linear_page_index(vma, address), - page); + VM_BUG_ON_PAGE(page_pgoff(folio, page) != + linear_anon_page_index(vma, address), page); } static __always_inline void __folio_add_anon_rmap(struct folio *folio, @@ -3038,10 +3038,10 @@ static void rmap_walk_anon(struct folio *folio, pgoff_end = pgoff_start + folio_nr_pages(folio) - 1; anon_rmap_tree_foreach(avc, anon_vma, pgoff_start, pgoff_end) { struct vm_area_struct *vma = avc->vma; - unsigned long address = vma_address(vma, pgoff_start, + const unsigned long address = vma_anon_address(vma, pgoff_start, folio_nr_pages(folio)); - VM_BUG_ON_VMA(address == -EFAULT, vma); + VM_WARN_ON_ONCE_VMA(address == -EFAULT, vma); cond_resched(); if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index de4cc2483562..23fb68fce000 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -1352,7 +1352,7 @@ static long move_present_ptes(struct mm_struct *mm, } folio_move_anon_rmap(src_folio, dst_vma); - src_folio->index = linear_page_index(dst_vma, dst_addr); + src_folio->index = linear_anon_page_index(dst_vma, dst_addr); orig_dst_pte = folio_mk_pte(src_folio, dst_vma->vm_page_prot); /* Set soft dirty bit so userspace can notice the pte was moved */ @@ -1428,7 +1428,7 @@ static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, */ if (src_folio) { folio_move_anon_rmap(src_folio, dst_vma); - src_folio->index = linear_page_index(dst_vma, dst_addr); + src_folio->index = linear_anon_page_index(dst_vma, dst_addr); } else { /* * Check if the swap entry is cached after acquiring the src_pte diff --git a/mm/vma.c b/mm/vma.c index e35b04ac12cb..35e7a64855fa 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -204,6 +204,25 @@ static void init_multi_vma_prep(struct vma_prepare *vp, vp->skip_vma_uprobe = true; } +/* + * Does this merge require that adjacent VMAs must have adjacent anonymous page + * offsets in addition to having adjacent vma->vm_pgoff? + * + * This is only required for MAP_PRIVATE-file backed mappings as the page offset + * for pure anonymous VMAs is equal to the anonymous page offset. + * + * Read-only shared mappings (with VMA_SHARED_BIT cleared) are always unfaulted + * so automatically have correct anonymous page offset (as it is always updated + * on remap). + * + * 'Special' mappings in the sense of VDSO, VVAR etc. have !file but would in + * any case not be candidates for merge nor be mergeable. + */ +static bool needs_adjacent_anon_pgoff(const struct vma_merge_struct *vmg) +{ + return vmg->file && vma_flags_is_cow_mapping(&vmg->vma_flags); +} + /* * Return true if we can merge this (vma_flags,anon_vma,file,vm_pgoff) * in front of (at a lower virtual address and file offset than) the vma. @@ -225,6 +244,9 @@ static bool can_vma_merge_before(struct vma_merge_struct *vmg) return false; if (vmg_end_pgoff(vmg) != vma_start_pgoff(vmg->next)) return false; + if (needs_adjacent_anon_pgoff(vmg) && + vmg_end_anon_pgoff(vmg) != vma_start_anon_pgoff(vmg->next)) + return false; return true; } @@ -245,6 +267,9 @@ static bool can_vma_merge_after(struct vma_merge_struct *vmg) return false; if (vma_end_pgoff(vmg->prev) != vmg_start_pgoff(vmg)) return false; + if (needs_adjacent_anon_pgoff(vmg) && + vma_end_anon_pgoff(vmg->prev) != vmg_start_anon_pgoff(vmg)) + return false; return true; } @@ -2048,7 +2073,12 @@ static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct * if (!vma_flags_empty(&diff)) return false; /* Page offset must align. */ - return vma_end_pgoff(a) == vma_start_pgoff(b); + if (vma_end_pgoff(a) != vma_start_pgoff(b)) + return false; + /* Only reached from anon path, so either MAP_PRIVATE file or anon. */ + if (vma_end_anon_pgoff(a) != vma_start_anon_pgoff(b)) + return false; + return true; } /*