mm/rmap: parameterise vma_interval_tree_*() by address_space

The file-backed mapping interval tree functions vma_interval_tree_*()
accept a raw rb_root_cached pointer to determine the tree in which they are
operating.

However, in each case, this is always associated with an address_space data
type.

So simply pass a pointer to that instead to simplify the code, and more
clearly differentiate between these operations and those concerning
anonymous mappings.

While we're here, make the generated interval tree functions static as they
do not need to be used externally (any previously existing external users
have now been removed).

We also rename VMA parameters from 'node' to 'vma' as calling this a node
is simply confusing, update the input index types to pgoff_t since they
reference page offsets and rename the parameters to pgoff_start and
pgoff_last.

No functional change intended.

Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-6-2a5aa403d977@kernel.org
Signed-off-by: Lorenzo Stoakes <ljs@kernel.org>
Reviewed-by: Pedro Falcato <pfalcato@suse.de>
Reviewed-by: Gregory Price <gourry@gourry.net>
Reviewed-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
Reviewed-by: Zi Yan <ziy@nvidia.com>
Cc: Ackerley Tng <ackerleytng@google.com>
Cc: David Hildenbrand (Arm) <david@kernel.org>
Cc: Kai Huang <kai.huang@intel.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: SJ Park <sj@kernel.org>
Cc: Thomas Zimmermann <tzimmermann@suse.de>
Cc: Liam R. Howlett (Oracle) <liam@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Lorenzo Stoakes
2026-07-10 21:16:47 +01:00
committed by Andrew Morton
parent dbde17041c
commit 7a67b96af0
19 changed files with 99 additions and 75 deletions

View File

@@ -140,7 +140,7 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma,
* cache coherency.
*/
flush_dcache_mmap_lock(mapping);
vma_interval_tree_foreach(mpnt, &mapping->i_mmap, pgoff, pgoff) {
vma_interval_tree_foreach(mpnt, mapping, pgoff, pgoff) {
/*
* If we are using split PTE locks, then we need to take the pte
* lock. Otherwise we are using shared mm->page_table_lock which

View File

@@ -251,7 +251,7 @@ static void __flush_dcache_aliases(struct address_space *mapping, struct folio *
pgoff_end = pgoff + folio_nr_pages(folio) - 1;
flush_dcache_mmap_lock(mapping);
vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff_end) {
vma_interval_tree_foreach(vma, mapping, pgoff, pgoff_end) {
unsigned long start, offset, pfn;
unsigned int nr;

View File

@@ -82,7 +82,7 @@ static void flush_aliases(struct address_space *mapping, struct folio *folio)
pgoff = folio->index;
flush_dcache_mmap_lock_irqsave(mapping, flags);
vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff + nr - 1) {
vma_interval_tree_foreach(vma, mapping, pgoff, pgoff + nr - 1) {
unsigned long start;
if (vma->vm_mm != mm)

View File

@@ -503,7 +503,7 @@ void flush_dcache_folio(struct folio *folio)
* on machines that support equivalent aliasing
*/
flush_dcache_mmap_lock_irqsave(mapping, flags);
vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff + nr - 1) {
vma_interval_tree_foreach(vma, mapping, pgoff, pgoff + nr - 1) {
unsigned long offset = pgoff - vma->vm_pgoff;
unsigned long pfn = folio_pfn(folio);

View File

@@ -1201,7 +1201,7 @@ static int dax_writeback_one(struct xa_state *xas, struct dax_device *dax_dev,
/* Walk all mappings of a given index of a file and writeprotect them */
i_mmap_lock_read(mapping);
vma_interval_tree_foreach(vma, &mapping->i_mmap, index, end) {
vma_interval_tree_foreach(vma, mapping, index, end) {
pfn_mkclean_range(pfn, count, index, vma);
cond_resched();
}

View File

@@ -380,7 +380,6 @@ static void hugetlb_unmap_file_folio(struct hstate *h,
struct address_space *mapping,
struct folio *folio, pgoff_t index)
{
struct rb_root_cached *root = &mapping->i_mmap;
struct hugetlb_vma_lock *vma_lock;
unsigned long pfn = folio_pfn(folio);
struct vm_area_struct *vma;
@@ -394,7 +393,7 @@ static void hugetlb_unmap_file_folio(struct hstate *h,
i_mmap_lock_write(mapping);
retry:
vma_lock = NULL;
vma_interval_tree_foreach(vma, root, start, end - 1) {
vma_interval_tree_foreach(vma, mapping, start, end - 1) {
v_start = vma_offset_start(vma, start);
v_end = vma_offset_end(vma, end);
@@ -460,8 +459,8 @@ static void hugetlb_unmap_file_folio(struct hstate *h,
}
static void
hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end,
zap_flags_t zap_flags)
hugetlb_vmdelete_list(struct address_space *mapping, pgoff_t start,
pgoff_t end, zap_flags_t zap_flags)
{
struct vm_area_struct *vma;
@@ -470,7 +469,8 @@ hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end,
* unmapped. Note, end is exclusive, whereas the interval tree takes
* an inclusive "last".
*/
vma_interval_tree_foreach(vma, root, start, end ? end - 1 : ULONG_MAX) {
vma_interval_tree_foreach(vma, mapping, start,
end ? end - 1 : ULONG_MAX) {
unsigned long v_start;
unsigned long v_end;
@@ -615,8 +615,7 @@ static void hugetlb_vmtruncate(struct inode *inode, loff_t offset)
i_size_write(inode, offset);
i_mmap_lock_write(mapping);
if (mapping_mapped(mapping))
hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0,
ZAP_FLAG_DROP_MARKER);
hugetlb_vmdelete_list(mapping, pgoff, 0, ZAP_FLAG_DROP_MARKER);
i_mmap_unlock_write(mapping);
remove_inode_hugepages(inode, offset, LLONG_MAX);
}
@@ -676,7 +675,7 @@ static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len)
/* Unmap users of full pages in the hole. */
if (hole_end > hole_start) {
if (mapping_mapped(mapping))
hugetlb_vmdelete_list(&mapping->i_mmap,
hugetlb_vmdelete_list(mapping,
hole_start >> PAGE_SHIFT,
hole_end >> PAGE_SHIFT, 0);
}

View File

@@ -4094,23 +4094,25 @@ extern atomic_long_t mmap_pages_allocated;
extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
/* interval_tree.c */
void vma_interval_tree_insert(struct vm_area_struct *node,
struct rb_root_cached *root);
void vma_interval_tree_insert_after(struct vm_area_struct *node,
void vma_interval_tree_insert(struct vm_area_struct *vma,
struct address_space *mapping);
void vma_interval_tree_insert_after(struct vm_area_struct *vma,
struct vm_area_struct *prev,
struct rb_root_cached *root);
void vma_interval_tree_remove(struct vm_area_struct *node,
struct rb_root_cached *root);
struct vm_area_struct *vma_interval_tree_subtree_search(struct vm_area_struct *node,
unsigned long start, unsigned long last);
struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root_cached *root,
unsigned long start, unsigned long last);
struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node,
unsigned long start, unsigned long last);
struct address_space *mapping);
void vma_interval_tree_remove(struct vm_area_struct *vma,
struct address_space *mapping);
struct vm_area_struct *
vma_interval_tree_iter_first(struct address_space *mapping,
pgoff_t pgoff_start, pgoff_t pgoff_last);
struct vm_area_struct *
vma_interval_tree_iter_next(struct vm_area_struct *vma,
pgoff_t pgoff_start, pgoff_t pgoff_last);
#define vma_interval_tree_foreach(vma, root, start, last) \
for (vma = vma_interval_tree_iter_first(root, start, last); \
vma; vma = vma_interval_tree_iter_next(vma, start, last))
#define vma_interval_tree_foreach(vma, mapping, pgoff_start, pgoff_last) \
for (vma = vma_interval_tree_iter_first(mapping, pgoff_start, \
pgoff_last); \
vma; vma = vma_interval_tree_iter_next(vma, pgoff_start, \
pgoff_last))
void anon_vma_interval_tree_insert(struct anon_vma_chain *node,
struct rb_root_cached *root);

View File

@@ -1210,7 +1210,7 @@ build_map_info(struct address_space *mapping, loff_t offset, bool is_register)
again:
i_mmap_lock_read(mapping);
vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
vma_interval_tree_foreach(vma, mapping, pgoff, pgoff) {
if (!valid_vma(vma, is_register))
continue;

View File

@@ -5443,7 +5443,7 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
* __unmap_hugepage_range() is called as the lock is already held
*/
i_mmap_lock_write(mapping);
vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
vma_interval_tree_foreach(iter_vma, mapping, pgoff, pgoff) {
/* Do not unmap the current VMA */
if (iter_vma == vma)
continue;
@@ -6986,7 +6986,7 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
pte_t *pte;
i_mmap_lock_read(mapping);
vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
vma_interval_tree_foreach(svma, mapping, idx, idx) {
if (svma == vma)
continue;

View File

@@ -14,19 +14,26 @@
/* File-backed interval tree (address_space->i_mmap) */
INTERVAL_TREE_DEFINE(struct vm_area_struct, shared.rb,
unsigned long, shared.rb_subtree_last,
vma_start_pgoff, vma_last_pgoff, /* empty */, vma_interval_tree)
pgoff_t, shared.rb_subtree_last,
vma_start_pgoff, vma_last_pgoff, static,
__vma_interval_tree)
/* Insert node immediately after prev in the interval tree */
void vma_interval_tree_insert_after(struct vm_area_struct *node,
void vma_interval_tree_insert(struct vm_area_struct *vma,
struct address_space *mapping)
{
__vma_interval_tree_insert(vma, &mapping->i_mmap);
}
/* Insert vma immediately after prev in the interval tree */
void vma_interval_tree_insert_after(struct vm_area_struct *vma,
struct vm_area_struct *prev,
struct rb_root_cached *root)
struct address_space *mapping)
{
struct rb_node **link;
struct vm_area_struct *parent;
unsigned long last = vma_last_pgoff(node);
const pgoff_t pgoff_last = vma_last_pgoff(vma);
VM_WARN_ON_ONCE_VMA(vma_start_pgoff(node) != vma_start_pgoff(prev), node);
VM_WARN_ON_ONCE_VMA(vma_start_pgoff(vma) != vma_start_pgoff(prev), vma);
if (!prev->shared.rb.rb_right) {
parent = prev;
@@ -34,21 +41,42 @@ void vma_interval_tree_insert_after(struct vm_area_struct *node,
} else {
parent = rb_entry(prev->shared.rb.rb_right,
struct vm_area_struct, shared.rb);
if (parent->shared.rb_subtree_last < last)
parent->shared.rb_subtree_last = last;
if (parent->shared.rb_subtree_last < pgoff_last)
parent->shared.rb_subtree_last = pgoff_last;
while (parent->shared.rb.rb_left) {
parent = rb_entry(parent->shared.rb.rb_left,
struct vm_area_struct, shared.rb);
if (parent->shared.rb_subtree_last < last)
parent->shared.rb_subtree_last = last;
if (parent->shared.rb_subtree_last < pgoff_last)
parent->shared.rb_subtree_last = pgoff_last;
}
link = &parent->shared.rb.rb_left;
}
node->shared.rb_subtree_last = last;
rb_link_node(&node->shared.rb, &parent->shared.rb, link);
rb_insert_augmented(&node->shared.rb, &root->rb_root,
&vma_interval_tree_augment);
vma->shared.rb_subtree_last = pgoff_last;
rb_link_node(&vma->shared.rb, &parent->shared.rb, link);
rb_insert_augmented(&vma->shared.rb, &mapping->i_mmap.rb_root,
&__vma_interval_tree_augment);
}
void vma_interval_tree_remove(struct vm_area_struct *vma,
struct address_space *mapping)
{
__vma_interval_tree_remove(vma, &mapping->i_mmap);
}
struct vm_area_struct *
vma_interval_tree_iter_first(struct address_space *mapping,
pgoff_t pgoff_start, pgoff_t pgoff_last)
{
return __vma_interval_tree_iter_first(&mapping->i_mmap,
pgoff_start, pgoff_last);
}
struct vm_area_struct *
vma_interval_tree_iter_next(struct vm_area_struct *vma,
pgoff_t pgoff_start, pgoff_t pgoff_last)
{
return __vma_interval_tree_iter_next(vma, pgoff_start, pgoff_last);
}
/* Anonymous interval tree (anon_vma->rb_root) */

View File

@@ -2141,7 +2141,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)
struct vm_area_struct *vma;
i_mmap_lock_read(mapping);
vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
vma_interval_tree_foreach(vma, mapping, pgoff, pgoff) {
struct mmu_notifier_range range;
struct mm_struct *mm;
unsigned long addr;
@@ -2573,7 +2573,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
* not be able to observe any missing pages due to the
* previously inserted retry entries.
*/
vma_interval_tree_foreach(vma, &mapping->i_mmap, start, end) {
vma_interval_tree_foreach(vma, mapping, start, end) {
if (userfaultfd_missing(vma)) {
result = SCAN_EXCEED_NONE_PTE;
goto immap_locked;

View File

@@ -598,8 +598,7 @@ static void collect_procs_file(const struct folio *folio,
if (!t)
continue;
vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff,
pgoff) {
vma_interval_tree_foreach(vma, mapping, pgoff, pgoff) {
/*
* Send early kill signal to tasks where a vma covers
* the page but the corrupted page is not necessarily
@@ -650,7 +649,7 @@ static void collect_procs_fsdax(const struct page *page,
t = task_early_kill(tsk, true);
if (!t)
continue;
vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
vma_interval_tree_foreach(vma, mapping, pgoff, pgoff) {
if (vma->vm_mm == t->mm)
add_to_kill_fsdax(t, page, vma, to_kill, pgoff);
}
@@ -2297,7 +2296,7 @@ static void collect_procs_pfn(struct pfn_address_space *pfn_space,
t = task_early_kill(tsk, true);
if (!t)
continue;
vma_interval_tree_foreach(vma, &mapping->i_mmap, 0, ULONG_MAX) {
vma_interval_tree_foreach(vma, mapping, 0, ULONG_MAX) {
pgoff_t pgoff;
if (vma->vm_mm == t->mm &&

View File

@@ -4360,7 +4360,7 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf)
return wp_page_copy(vmf);
}
static inline void unmap_mapping_range_tree(struct rb_root_cached *root,
static inline void unmap_mapping_range_tree(struct address_space *mapping,
pgoff_t first_index,
pgoff_t last_index,
struct zap_details *details)
@@ -4369,7 +4369,7 @@ static inline void unmap_mapping_range_tree(struct rb_root_cached *root,
unsigned long start, size;
struct mmu_gather tlb;
vma_interval_tree_foreach(vma, root, first_index, last_index) {
vma_interval_tree_foreach(vma, mapping, first_index, last_index) {
const pgoff_t start_idx = max(first_index, vma->vm_pgoff);
const pgoff_t end_idx = min(last_index, vma_last_pgoff(vma)) + 1;
@@ -4411,7 +4411,7 @@ void unmap_mapping_folio(struct folio *folio)
i_mmap_lock_read(mapping);
if (unlikely(mapping_mapped(mapping)))
unmap_mapping_range_tree(&mapping->i_mmap, first_index,
unmap_mapping_range_tree(mapping, first_index,
last_index, &details);
i_mmap_unlock_read(mapping);
}
@@ -4441,7 +4441,7 @@ void unmap_mapping_pages(struct address_space *mapping, pgoff_t start,
i_mmap_lock_read(mapping);
if (unlikely(mapping_mapped(mapping)))
unmap_mapping_range_tree(&mapping->i_mmap, first_index,
unmap_mapping_range_tree(mapping, first_index,
last_index, &details);
i_mmap_unlock_read(mapping);
}

View File

@@ -1830,8 +1830,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
mapping_allow_writable(mapping);
flush_dcache_mmap_lock(mapping);
/* insert tmp into the share list, just after mpnt */
vma_interval_tree_insert_after(tmp, mpnt,
&mapping->i_mmap);
vma_interval_tree_insert_after(tmp, mpnt, mapping);
flush_dcache_mmap_unlock(mapping);
i_mmap_unlock_write(mapping);
}

View File

@@ -569,7 +569,7 @@ static void setup_vma_to_mm(struct vm_area_struct *vma, struct mm_struct *mm)
i_mmap_lock_write(mapping);
flush_dcache_mmap_lock(mapping);
vma_interval_tree_insert(vma, &mapping->i_mmap);
vma_interval_tree_insert(vma, mapping);
flush_dcache_mmap_unlock(mapping);
i_mmap_unlock_write(mapping);
}
@@ -585,7 +585,7 @@ static void cleanup_vma_from_mm(struct vm_area_struct *vma)
i_mmap_lock_write(mapping);
flush_dcache_mmap_lock(mapping);
vma_interval_tree_remove(vma, &mapping->i_mmap);
vma_interval_tree_remove(vma, mapping);
flush_dcache_mmap_unlock(mapping);
i_mmap_unlock_write(mapping);
}
@@ -1839,7 +1839,7 @@ int nommu_shrink_inode_mappings(struct inode *inode, size_t size,
i_mmap_lock_read(inode->i_mapping);
/* search for VMAs that fall within the dead zone */
vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, low, high) {
vma_interval_tree_foreach(vma, inode->i_mapping, low, high) {
/* found one - only interested if it's shared out of the page
* cache */
if (vma->vm_flags & VM_SHARED) {
@@ -1855,7 +1855,7 @@ int nommu_shrink_inode_mappings(struct inode *inode, size_t size,
* we don't check for any regions that start beyond the EOF as there
* shouldn't be any
*/
vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, 0, ULONG_MAX) {
vma_interval_tree_foreach(vma, inode->i_mapping, 0, ULONG_MAX) {
if (!(vma->vm_flags & VM_SHARED))
continue;

View File

@@ -810,7 +810,7 @@ int walk_page_mapping(struct address_space *mapping, pgoff_t first_index,
return -EINVAL;
lockdep_assert_held(&mapping->i_mmap_rwsem);
vma_interval_tree_foreach(vma, &mapping->i_mmap, first_index,
vma_interval_tree_foreach(vma, mapping, first_index,
first_index + nr - 1) {
/* Clip to the vma */
vba = vma->vm_pgoff;

View File

@@ -3053,8 +3053,7 @@ static void __rmap_walk_file(struct folio *folio, struct address_space *mapping,
i_mmap_lock_read(mapping);
}
lookup:
vma_interval_tree_foreach(vma, &mapping->i_mmap,
pgoff_start, pgoff_end) {
vma_interval_tree_foreach(vma, mapping, pgoff_start, pgoff_end) {
unsigned long address = vma_address(vma, pgoff_start, nr_pages);
VM_BUG_ON_VMA(address == -EFAULT, vma);

View File

@@ -234,7 +234,7 @@ static void __vma_link_file(struct vm_area_struct *vma,
mapping_allow_writable(mapping);
flush_dcache_mmap_lock(mapping);
vma_interval_tree_insert(vma, &mapping->i_mmap);
vma_interval_tree_insert(vma, mapping);
flush_dcache_mmap_unlock(mapping);
}
@@ -248,7 +248,7 @@ static void __remove_shared_vm_struct(struct vm_area_struct *vma,
mapping_unmap_writable(mapping);
flush_dcache_mmap_lock(mapping);
vma_interval_tree_remove(vma, &mapping->i_mmap);
vma_interval_tree_remove(vma, mapping);
flush_dcache_mmap_unlock(mapping);
}
@@ -319,10 +319,9 @@ static void vma_prepare(struct vma_prepare *vp)
if (vp->file) {
flush_dcache_mmap_lock(vp->mapping);
vma_interval_tree_remove(vp->vma, &vp->mapping->i_mmap);
vma_interval_tree_remove(vp->vma, vp->mapping);
if (vp->adj_next)
vma_interval_tree_remove(vp->adj_next,
&vp->mapping->i_mmap);
vma_interval_tree_remove(vp->adj_next, vp->mapping);
}
}
@@ -340,9 +339,8 @@ static void vma_complete(struct vma_prepare *vp, struct vma_iterator *vmi,
{
if (vp->file) {
if (vp->adj_next)
vma_interval_tree_insert(vp->adj_next,
&vp->mapping->i_mmap);
vma_interval_tree_insert(vp->vma, &vp->mapping->i_mmap);
vma_interval_tree_insert(vp->adj_next, vp->mapping);
vma_interval_tree_insert(vp->vma, vp->mapping);
flush_dcache_mmap_unlock(vp->mapping);
}

View File

@@ -258,12 +258,12 @@ static inline void vm_acct_memory(long pages)
}
static inline void vma_interval_tree_insert(struct vm_area_struct *vma,
struct rb_root_cached *rb)
struct address_space *mapping)
{
}
static inline void vma_interval_tree_remove(struct vm_area_struct *vma,
struct rb_root_cached *rb)
struct address_space *mapping)
{
}