mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-31 03:35:32 -04:00
Code Issues Packages Projects Releases Wiki Activity

mm/mlock: convert mlock code to use vma_flags_t

Browse Source 
Replace use of the legacy vm_flags_t flags with vma_flags_t values
throughout the mlock logic.

Additionally update comments to reflect the changes to be consistent.

No functional change intended.

Link: https://lore.kernel.org/20260711-b4-vma-flags-mm-v2-11-0fa2357d5431@kernel.org
Signed-off-by: Lorenzo Stoakes <ljs@kernel.org>
Reviewed-by: Zi Yan <ziy@nvidia.com>
Reviewed-by: Lance Yang <lance.yang@linux.dev>
Reviewed-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Christian Brauner <brauner@kernel.org>
Cc: Dave Airlie <airlied@gmail.com>
Cc: David Hildenbrand <david@kernel.org>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Jani Nikula <jani.nikula@intel.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Jann Horn <jannh@google.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nico Pache <npache@redhat.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Pedro Falcato <pfalcato@suse.de>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Thomas Zimmermann <tzimmermann@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Lorenzo Stoakes
2026-07-11 19:45:08 +01:00
committed by Andrew Morton
parent df751fe672
commit 2fa484226b
1 changed files with 47 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
mm/mlock.c
View File

@@ -329,7 +329,7 @@ static inline bool allow_mlock_munlock(struct folio *folio,
* be split. And the pages are not in VM_LOCKed VMA
* can be reclaimed.
*/
if (!(vma->vm_flags & VM_LOCKED))
if (!vma_test(vma, VMA_LOCKED_BIT))
return true;
/* folio_within_range() cannot take KSM, but any small folio is OK */
@@ -368,7 +368,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr,
folio = pmd_folio(*pmd);
if (folio_is_zone_device(folio))
goto out;
if (vma->vm_flags & VM_LOCKED)
if (vma_test(vma, VMA_LOCKED_BIT))
mlock_folio(folio);
else
munlock_folio(folio);
@@ -393,7 +393,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr,
if (!allow_mlock_munlock(folio, vma, start, end, step))
goto next_entry;
if (vma->vm_flags & VM_LOCKED)
if (vma_test(vma, VMA_LOCKED_BIT))
mlock_folio(folio);
else
munlock_folio(folio);
@@ -417,8 +417,8 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr,
* @end - end of range in @vma
* @new_vma_flags - the new set of flags for @vma.
*
* Called for mlock(), mlock2() and mlockall(), to set @vma VM_LOCKED;
* called for munlock() and munlockall(), to clear VM_LOCKED from @vma.
* Called for mlock(), mlock2() and mlockall(), to set @vma VMA_LOCKED_BIT;
* called for munlock() and munlockall(), to clear VMA_LOCKED_BIT from @vma.
*/
static void mlock_vma_pages_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end,
@@ -431,14 +431,14 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma,
/*
* There is a slight chance that concurrent page migration,
* or page reclaim finding a page of this now-VM_LOCKED vma,
* or page reclaim finding a page of this now-VMA_LOCKED_BIT vma,
* will call mlock_vma_folio() and raise page's mlock_count:
* double counting, leaving the page unevictable indefinitely.
* Communicate this danger to mlock_vma_folio() with VM_IO,
* which is a VM_SPECIAL flag not allowed on VM_LOCKED vmas.
* Communicate this danger to mlock_vma_folio() with VMA_IO_BIT,
* which is a VMA_SPECIAL_FLAGS flag not allowed on VMA_LOCKED_BIT vmas.
* mmap_lock is held in write mode here, so this weird
* combination should not be visible to other mmap_lock users;
* but WRITE_ONCE so rmap walkers must see VM_IO if VM_LOCKED.
* but WRITE_ONCE so rmap walkers must see VMA_IO_BIT if VMA_LOCKED_BIT.
*/
if (vma_flags_test(new_vma_flags, VMA_LOCKED_BIT))
vma_flags_set(new_vma_flags, VMA_IO_BIT);
@@ -458,7 +458,7 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma,
/*
* mlock_fixup - handle mlock[all]/munlock[all] requests.
*
* Filters out "special" vmas -- VM_LOCKED never gets set for these, and
* Filters out "special" vmas -- VMA_LOCKED_BIT never gets set for these, and
* munlock is a no-op. However, for some special vmas, we go ahead and
* populate the ptes.
*
@@ -466,24 +466,23 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma,
*/
static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma,
struct vm_area_struct **prev, unsigned long start,
unsigned long end, vm_flags_t newflags)
unsigned long end, vma_flags_t *new_vma_flags)
{
vma_flags_t new_vma_flags = legacy_to_vma_flags(newflags);
const vma_flags_t old_vma_flags = vma->flags;
struct mm_struct *mm = vma->vm_mm;
int nr_pages;
int ret = 0;
if (vma_flags_same_pair(&old_vma_flags, &new_vma_flags) ||
if (vma_flags_same_pair(&old_vma_flags, new_vma_flags) ||
vma_is_secretmem(vma) || !vma_supports_mlock(vma)) {
/*
* Don't set VM_LOCKED or VM_LOCKONFAULT and don't count.
* For secretmem, don't allow the memory to be unlocked.
* Don't set VMA_LOCKED_BIT or VMA_LOCKONFAULT_BIT and don't
* count. For secretmem, don't allow the memory to be unlocked.
*/
goto out;
}
vma = vma_modify_flags(vmi, *prev, vma, start, end, &new_vma_flags);
vma = vma_modify_flags(vmi, *prev, vma, start, end, new_vma_flags);
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
goto out;
@@ -493,7 +492,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma,
* Keep track of amount of locked VM.
*/
nr_pages = (end - start) >> PAGE_SHIFT;
if (!vma_flags_test(&new_vma_flags, VMA_LOCKED_BIT))
if (!vma_flags_test(new_vma_flags, VMA_LOCKED_BIT))
nr_pages = -nr_pages;
else if (vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT))
nr_pages = 0;
@@ -502,15 +501,15 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma,
/*
* vm_flags is protected by the mmap_lock held in write mode.
* It's okay if try_to_unmap_one unmaps a page just after we
* set VM_LOCKED, populate_vma_page_range will bring it back.
* set VMA_LOCKED_BIT, populate_vma_page_range will bring it back.
*/
if (vma_flags_test(&new_vma_flags, VMA_LOCKED_BIT) &&
if (vma_flags_test(new_vma_flags, VMA_LOCKED_BIT) &&
vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT)) {
/* No work to do, and mlocking twice would be wrong */
vma_start_write(vma);
vma->flags = new_vma_flags;
vma->flags = *new_vma_flags;
} else {
mlock_vma_pages_range(vma, start, end, &new_vma_flags);
mlock_vma_pages_range(vma, start, end, new_vma_flags);
}
out:
*prev = vma;
@@ -518,7 +517,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma,
}
static int apply_vma_lock_flags(unsigned long start, size_t len,
vm_flags_t flags)
const vma_flags_t *flags)
{
unsigned long nstart, end, tmp;
struct vm_area_struct *vma, *prev;
@@ -543,18 +542,20 @@ static int apply_vma_lock_flags(unsigned long start, size_t len,
tmp = vma->vm_start;
for_each_vma_range(vmi, vma, end) {
int error;
vm_flags_t newflags;
vma_flags_t newflags;
if (vma->vm_start != tmp)
return -ENOMEM;
newflags = vma->vm_flags & ~VM_LOCKED_MASK;
newflags |= flags;
newflags = vma->flags;
vma_flags_clear_mask(&newflags, VMA_LOCKED_MASK);
vma_flags_set_mask(&newflags, *flags);
/* Here we know that vma->vm_start <= nstart < vma->vm_end. */
tmp = vma->vm_end;
if (tmp > end)
tmp = end;
error = mlock_fixup(&vmi, vma, &prev, nstart, tmp, newflags);
error = mlock_fixup(&vmi, vma, &prev, nstart, tmp, &newflags);
if (error)
return error;
tmp = vma_iter_end(&vmi);
@@ -589,7 +590,7 @@ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm,
end = start + len;
for_each_vma_range(vmi, vma, end) {
if (vma->vm_flags & VM_LOCKED) {
if (vma_test(vma, VMA_LOCKED_BIT)) {
if (start > vma->vm_start)
count -= (start - vma->vm_start);
if (end < vma->vm_end) {
@@ -615,7 +616,8 @@ static int __mlock_posix_error_return(long retval)
return retval;
}
static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t flags)
static __must_check int do_mlock(unsigned long start, size_t len,
vma_flags_t *flags)
{
unsigned long locked;
unsigned long lock_limit;
@@ -664,24 +666,27 @@ static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t fla
SYSCALL_DEFINE2(mlock, unsigned long, start, size_t, len)
{
return do_mlock(start, len, VM_LOCKED);
vma_flags_t flags = mk_vma_flags(VMA_LOCKED_BIT);
return do_mlock(start, len, &flags);
}
SYSCALL_DEFINE3(mlock2, unsigned long, start, size_t, len, int, flags)
{
vm_flags_t vm_flags = VM_LOCKED;
vma_flags_t vma_flags = mk_vma_flags(VMA_LOCKED_BIT);
if (flags & ~MLOCK_ONFAULT)
return -EINVAL;
if (flags & MLOCK_ONFAULT)
vm_flags |= VM_LOCKONFAULT;
vma_flags_set(&vma_flags, VMA_LOCKONFAULT_BIT);
return do_mlock(start, len, vm_flags);
return do_mlock(start, len, &vma_flags);
}
SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len)
{
vma_flags_t flags = EMPTY_VMA_FLAGS;
int ret;
start = untagged_addr(start);
@@ -691,7 +696,7 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len)
if (mmap_write_lock_killable(current->mm))
return -EINTR;
ret = apply_vma_lock_flags(start, len, 0);
ret = apply_vma_lock_flags(start, len, &flags);
mmap_write_unlock(current->mm);
return ret;
@@ -705,14 +710,15 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len)
* There are a couple of subtleties with this. If mlockall() is called multiple
* times with different flags, the values do not necessarily stack. If mlockall
* is called once including the MCL_FUTURE flag and then a second time without
* it, VM_LOCKED and VM_LOCKONFAULT will be cleared from mm->def_vma_flags.
* it, VMA_LOCKED_BIT and VMA_LOCKONFAULT_BIT will be cleared from
* mm->def_vma_flags.
*/
static int apply_mlockall_flags(int flags)
{
VMA_ITERATOR(vmi, current->mm, 0);
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma, *prev = NULL;
vm_flags_t to_add = 0;
vma_flags_t to_add = EMPTY_VMA_FLAGS;
vma_flags_clear_mask(&mm->def_vma_flags, VMA_LOCKED_MASK);
if (flags & MCL_FUTURE) {
@@ -726,20 +732,20 @@ static int apply_mlockall_flags(int flags)
}
if (flags & MCL_CURRENT) {
to_add |= VM_LOCKED;
vma_flags_set(&to_add, VMA_LOCKED_BIT);
if (flags & MCL_ONFAULT)
to_add |= VM_LOCKONFAULT;
vma_flags_set(&to_add, VMA_LOCKONFAULT_BIT);
}
for_each_vma(vmi, vma) {
int error;
vm_flags_t newflags;
vma_flags_t newflags = vma->flags;
newflags = vma->vm_flags & ~VM_LOCKED_MASK;
newflags |= to_add;
vma_flags_clear_mask(&newflags, VMA_LOCKED_MASK);
vma_flags_set_mask(&newflags, to_add);
error = mlock_fixup(&vmi, vma, &prev, vma->vm_start, vma->vm_end,
newflags);
&newflags);
/* Ignore errors, but prev needs fixing up. */
if (error)
prev = vma;