mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-28 05:14:41 -04:00
Code Issues Packages Projects Releases Wiki Activity

userfaultfd: add UFFDIO_REGISTER_MODE_RWP and UFFDIO_RWPROTECT plumbing

Browse Source 
Add the userspace interface for read-write protection tracking:

  - UFFDIO_REGISTER_MODE_RWP      register a range for RWP tracking
  - UFFD_FEATURE_RWP              capability bit
  - UFFDIO_RWPROTECT              install / remove RWP on a range

Introduce CONFIG_USERFAULTFD_RWP, auto-selected on 64-bit kernels with
ARCH_HAS_PTE_PROTNONE and HAVE_ARCH_USERFAULTFD_WP.  The symbol gates
VM_UFFD_RWP (previously aliased to VM_NONE) and the smaps/trace-flag hooks
added in the preparatory patches; without it the UAPI bits added here have
nothing to drive and would be unreachable.

Registration sets VM_UFFD_RWP on the VMA.  Combining MODE_WP with MODE_RWP
is rejected because both modes claim the uffd PTE bit.

UFFDIO_RWPROTECT is the bidirectional counterpart of
UFFDIO_WRITEPROTECT:

  - MODE_RWP              change_protection() with MM_CP_UFFD_RWP
                          installs PAGE_NONE and sets the uffd bit on
                          present PTEs
  - !MODE_RWP             change_protection() with MM_CP_UFFD_RWP_RESOLVE
                          restores vma->vm_page_prot and clears the bit

userfaultfd_clear_vma() runs the same resolve pass on unregister so RWP
state cannot outlive the uffd.

Re-registering a range must not drop a mode that installs per-PTE markers
(WP or RWP); doing so returns -EBUSY.  This also closes a pre-existing
window where re-registering without MODE_WP would strand uffd-wp markers:
before, those caused extra write-faults but were otherwise benign; with
RWP preservation in place, a subsequent mprotect() on a VM_UFFD_RWP VMA
would silently promote the stale markers to RWP.

The feature is not yet advertised.  UFFDIO_REGISTER_MODE_RWP,
UFFD_FEATURE_RWP, and _UFFDIO_RWPROTECT are intentionally absent from
UFFD_API_REGISTER_MODES, UFFD_API_FEATURES, and UFFD_API_RANGE_IOCTLS, so
UFFDIO_API masks them out and the register-mode validator rejects the bit.
The follow-up patch adds fault dispatch and exposes the UAPI.

Link: https://lore.kernel.org/20260708111417.173443-10-kirill@shutemov.name
Signed-off-by: Kiryl Shutsemau <kas@kernel.org>
Assisted-by: Claude:claude-opus-4-6
Reviewed-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: David Hildenbrand <david@kernel.org>
Cc: James Houghton <jthoughton@google.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Liam Howlett <liam@infradead.org>
Cc: Lorenzo Stoakes <ljs@kernel.org>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Peter Xu <peterx@redhat.com>
Cc: Sean Christopherson <seanjc@google.com>
Cc: SeongJae Park <sj@kernel.org>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@kernel.org>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Kiryl Shutsemau (Meta)
2026-07-08 12:14:10 +01:00
committed by Andrew Morton
parent 7974c23853
commit 6eab8f2cc6
5 changed files with 228 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
Documentation/admin-guide/mm/userfaultfd.rst
View File

@@ -131,6 +131,16 @@ userfaults on the range registered. Not all ioctls will necessarily be
supported for all memory types (e.g. anonymous memory vs. shmem vs.
hugetlbfs), or all types of intercepted faults.
.. note::
Re-registering an already-registered range must not drop any of the
modes that install per-PTE markers — currently
``UFFDIO_REGISTER_MODE_WP`` and ``UFFDIO_REGISTER_MODE_RWP``. Doing
so would strand markers with no flag to describe them, so the call
is rejected with ``-EBUSY``; userspace must issue
``UFFDIO_UNREGISTER`` first. This differs from older kernels, which
silently replaced the mode bits on re-registration.
Userland can use the ``uffdio_register.ioctls`` to manage the virtual
address space in the background (to add or potentially also remove
memory from the ``userfaultfd`` registered range). This means a userfault

2
include/linux/userfaultfd_k.h
View File

@@ -150,6 +150,8 @@ static inline uffd_flags_t uffd_flags_set_mode(uffd_flags_t flags, enum mfill_at
extern long uffd_wp_range(struct vm_area_struct *vma,
unsigned long start, unsigned long len, bool enable_wp);
extern int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start,
unsigned long len, bool enable_rwp);
/* move_pages */
void double_pt_lock(spinlock_t *ptl1, spinlock_t *ptl2);

19
include/uapi/linux/userfaultfd.h
View File

@@ -79,6 +79,7 @@
#define _UFFDIO_WRITEPROTECT (0x06)
#define _UFFDIO_CONTINUE (0x07)
#define _UFFDIO_POISON (0x08)
#define _UFFDIO_RWPROTECT (0x09)
#define _UFFDIO_API (0x3F)
/* userfaultfd ioctl ids */
@@ -103,6 +104,8 @@
struct uffdio_continue)
#define UFFDIO_POISON _IOWR(UFFDIO, _UFFDIO_POISON, \
struct uffdio_poison)
#define UFFDIO_RWPROTECT _IOWR(UFFDIO, _UFFDIO_RWPROTECT, \
struct uffdio_rwprotect)
/* read() structure */
struct uffd_msg {
@@ -158,6 +161,7 @@ struct uffd_msg {
#define UFFD_PAGEFAULT_FLAG_WRITE (1<<0) /* If this was a write fault */
#define UFFD_PAGEFAULT_FLAG_WP (1<<1) /* If reason is VM_UFFD_WP */
#define UFFD_PAGEFAULT_FLAG_MINOR (1<<2) /* If reason is VM_UFFD_MINOR */
#define UFFD_PAGEFAULT_FLAG_RWP (1<<3) /* If reason is VM_UFFD_RWP */
struct uffdio_api {
/* userland asks for an API number and the features to enable */
@@ -230,6 +234,11 @@ struct uffdio_api {
*
* UFFD_FEATURE_MOVE indicates that the kernel supports moving an
* existing page contents from userspace.
*
* UFFD_FEATURE_RWP indicates that the kernel supports
* UFFDIO_REGISTER_MODE_RWP for read-write protection tracking.
* Pages are made inaccessible via UFFDIO_RWPROTECT and faults
* are delivered when the pages are re-accessed.
*/
#define UFFD_FEATURE_PAGEFAULT_FLAG_WP (1<<0)
#define UFFD_FEATURE_EVENT_FORK (1<<1)
@@ -248,6 +257,7 @@ struct uffdio_api {
#define UFFD_FEATURE_POISON (1<<14)
#define UFFD_FEATURE_WP_ASYNC (1<<15)
#define UFFD_FEATURE_MOVE (1<<16)
#define UFFD_FEATURE_RWP (1<<17)
__u64 features;
__u64 ioctls;
@@ -263,6 +273,7 @@ struct uffdio_register {
#define UFFDIO_REGISTER_MODE_MISSING ((__u64)1<<0)
#define UFFDIO_REGISTER_MODE_WP ((__u64)1<<1)
#define UFFDIO_REGISTER_MODE_MINOR ((__u64)1<<2)
#define UFFDIO_REGISTER_MODE_RWP ((__u64)1<<3)
__u64 mode;
/*
@@ -356,6 +367,14 @@ struct uffdio_poison {
__s64 updated;
};
struct uffdio_rwprotect {
struct uffdio_range range;
/* !RWP means undo RWP-protection */
#define UFFDIO_RWPROTECT_MODE_RWP ((__u64)1<<0)
#define UFFDIO_RWPROTECT_MODE_DONTWAKE ((__u64)1<<1)
__u64 mode;
};
struct uffdio_move {
__u64 dst;
__u64 src;

9
mm/Kconfig
View File

@@ -1386,6 +1386,15 @@ config HAVE_ARCH_USERFAULTFD_MINOR
help
Arch has userfaultfd minor fault support
config USERFAULTFD_RWP
def_bool y
depends on 64BIT && ARCH_HAS_PTE_PROTNONE && HAVE_ARCH_USERFAULTFD_WP
help
Userfaultfd read-write protection (UFFDIO_RWPROTECT) delivers a
userfaultfd notification on every access -- read or write -- to a
protected range, letting userspace observe the working set of a
process.
menuconfig USERFAULTFD
bool "Enable userfaultfd() system call"
depends on MMU

191
mm/userfaultfd.c
View File

@@ -1164,6 +1164,75 @@ static int mwriteprotect_range(struct userfaultfd_ctx *ctx, unsigned long start,
return err;
}
int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start,
unsigned long len, bool enable_rwp)
{
struct mm_struct *dst_mm = ctx->mm;
unsigned long end = start + len;
struct vm_area_struct *dst_vma;
unsigned int mm_cp_flags;
struct mmu_gather tlb;
bool found = false;
VMA_ITERATOR(vmi, dst_mm, start);
VM_WARN_ON_ONCE(start & ~PAGE_MASK);
VM_WARN_ON_ONCE(len & ~PAGE_MASK);
VM_WARN_ON_ONCE(start + len <= start);
guard(mmap_read_lock)(dst_mm);
guard(rwsem_read)(&ctx->map_changing_lock);
if (atomic_read(&ctx->mmap_changing))
return -EAGAIN;
if (enable_rwp)
mm_cp_flags = MM_CP_UFFD_RWP;
else
mm_cp_flags = MM_CP_UFFD_RWP_RESOLVE;
/*
* Pre-scan the range: validate every spanned VMA before applying
* any change_protection() so a partial failure cannot leave the
* process with only a prefix of the range re-protected.
*/
for_each_vma_range(vmi, dst_vma, end) {
if (!userfaultfd_rwp(dst_vma))
return -ENOENT;
if (is_vm_hugetlb_page(dst_vma)) {
unsigned long page_mask;
page_mask = vma_kernel_pagesize(dst_vma) - 1;
if ((start & page_mask) || (len & page_mask))
return -EINVAL;
}
found = true;
}
if (!found)
return -ENOENT;
vma_iter_set(&vmi, start);
tlb_gather_mmu(&tlb, dst_mm);
for_each_vma_range(vmi, dst_vma, end) {
unsigned long vma_start = max(dst_vma->vm_start, start);
unsigned long vma_end = min(dst_vma->vm_end, end);
unsigned int flags = mm_cp_flags;
/*
* On resolve, try to upgrade writability per-VMA --
* MM_CP_TRY_CHANGE_WRITABLE WARNs in
* maybe_change_pte_writable() if the VMA is not VM_WRITE,
* and RWP can be registered on PROT_READ-only mappings.
*/
if (!enable_rwp && vma_wants_manual_pte_write_upgrade(dst_vma))
flags |= MM_CP_TRY_CHANGE_WRITABLE;
change_protection(&tlb, dst_vma, vma_start, vma_end, flags);
}
tlb_finish_mmu(&tlb);
return 0;
}
void double_pt_lock(spinlock_t *ptl1,
spinlock_t *ptl2)
@@ -2207,9 +2276,22 @@ static struct vm_area_struct *userfaultfd_clear_vma(struct vma_iterator *vmi,
if (start == vma->vm_start && end == vma->vm_end)
give_up_on_oom = true;
/* Reset ptes for the whole vma range if wr-protected */
if (userfaultfd_wp(vma))
uffd_wp_range(vma, start, end - start, false);
/* Clear the uffd bit and/or restore protnone PTEs */
if (userfaultfd_protected(vma)) {
unsigned int mm_cp_flags = 0;
struct mmu_gather tlb;
if (userfaultfd_wp(vma))
mm_cp_flags |= MM_CP_UFFD_WP_RESOLVE;
if (userfaultfd_rwp(vma))
mm_cp_flags |= MM_CP_UFFD_RWP_RESOLVE;
if (vma_wants_manual_pte_write_upgrade(vma))
mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE;
tlb_gather_mmu(&tlb, vma->vm_mm);
change_protection(&tlb, vma, start, end, mm_cp_flags);
tlb_finish_mmu(&tlb);
}
ret = vma_modify_flags_uffd(vmi, prev, vma, start, end,
&new_vma_flags, NULL_VM_UFFD_CTX,
@@ -2258,6 +2340,14 @@ static int userfaultfd_register_range(struct userfaultfd_ctx *ctx,
vma_test_all_mask(vma, vma_flags))
goto skip;
/*
* Pre-scan in userfaultfd_register() already rejected mode
* switches that would drop VM_UFFD_WP or VM_UFFD_RWP, so a
* stray bit here is a bug.
*/
VM_WARN_ON_ONCE(vma->vm_userfaultfd_ctx.ctx == ctx &&
vma->vm_flags & (VM_UFFD_WP | VM_UFFD_RWP) & ~vm_flags);
if (vma->vm_start > start)
start = vma->vm_start;
vma_end = min(end, vma->vm_end);
@@ -2524,6 +2614,8 @@ static inline struct uffd_msg userfault_msg(unsigned long address,
msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE;
if (reason & VM_UFFD_WP)
msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP;
if (reason & VM_UFFD_RWP)
msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_RWP;
if (reason & VM_UFFD_MINOR)
msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_MINOR;
if (features & UFFD_FEATURE_THREAD_ID)
@@ -3623,6 +3715,22 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
vm_flags |= VM_UFFD_WP;
}
if (uffdio_register.mode & UFFDIO_REGISTER_MODE_RWP) {
if (!pgtable_supports_uffd() || VM_UFFD_RWP == VM_NONE)
goto out;
if (!(ctx->features & UFFD_FEATURE_RWP))
goto out;
vm_flags |= VM_UFFD_RWP;
}
/*
* WP and RWP share the uffd PTE bit and
* cannot coexist in the same VMA — the bit would carry ambiguous
* semantics. Reject the combination up front.
*/
if ((vm_flags & VM_UFFD_WP) && (vm_flags & VM_UFFD_RWP))
goto out;
if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR) {
#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
goto out;
@@ -3677,6 +3785,17 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
if (!vma_can_userfault(cur, vm_flags, wp_async))
goto out_unlock;
/*
* RWP uses protnone as an access-tracking marker. PROT_NONE
* VMAs have vm_page_prot == PAGE_NONE, so RWP resolution
* cannot make a page accessible again. Reject at register
* time only: a VMA that later becomes inaccessible via
* mprotect() must still be unregisterable, so this is not
* part of vma_can_userfault().
*/
if ((vm_flags & VM_UFFD_RWP) && !vma_is_accessible(cur))
goto out_unlock;
/*
* UFFDIO_COPY will fill file holes even without
* PROT_WRITE. This check enforces that if this is a
@@ -3716,6 +3835,16 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
cur->vm_userfaultfd_ctx.ctx != ctx)
goto out_unlock;
/*
* Mode switches that drop VM_UFFD_WP or VM_UFFD_RWP would
* leave PTE markers without the flag that describes them;
* subsequent mprotect() would then promote stale markers
* into the other mode. Require an unregister first.
*/
if (cur->vm_userfaultfd_ctx.ctx == ctx &&
cur->vm_flags & (VM_UFFD_WP | VM_UFFD_RWP) & ~vm_flags)
goto out_unlock;
/*
* Note vmas containing huge pages
*/
@@ -3749,6 +3878,10 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR))
ioctls_out &= ~((__u64)1 << _UFFDIO_CONTINUE);
/* RWPROTECT is only supported for RWP ranges */
if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_RWP))
ioctls_out &= ~((__u64)1 << _UFFDIO_RWPROTECT);
/*
* Now that we scanned all vmas we can already tell
* userland which ioctls methods are guaranteed to
@@ -4096,6 +4229,55 @@ static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx,
return ret;
}
static int userfaultfd_rwprotect(struct userfaultfd_ctx *ctx,
unsigned long arg)
{
int ret;
struct uffdio_rwprotect uffdio_rwp;
struct userfaultfd_wake_range range;
bool mode_rwp, mode_dontwake;
if (atomic_read(&ctx->mmap_changing))
return -EAGAIN;
if (copy_from_user(&uffdio_rwp, (void __user *)arg,
sizeof(uffdio_rwp)))
return -EFAULT;
ret = validate_range(ctx->mm, uffdio_rwp.range.start,
uffdio_rwp.range.len);
if (ret)
return ret;
if (uffdio_rwp.mode & ~(UFFDIO_RWPROTECT_MODE_DONTWAKE |
UFFDIO_RWPROTECT_MODE_RWP))
return -EINVAL;
mode_rwp = uffdio_rwp.mode & UFFDIO_RWPROTECT_MODE_RWP;
mode_dontwake = uffdio_rwp.mode & UFFDIO_RWPROTECT_MODE_DONTWAKE;
if (mode_rwp && mode_dontwake)
return -EINVAL;
if (mmget_not_zero(ctx->mm)) {
ret = mrwprotect_range(ctx, uffdio_rwp.range.start,
uffdio_rwp.range.len, mode_rwp);
mmput(ctx->mm);
} else {
return -ESRCH;
}
if (ret)
return ret;
if (!mode_rwp && !mode_dontwake) {
range.start = uffdio_rwp.range.start;
range.len = uffdio_rwp.range.len;
wake_userfault(ctx, &range);
}
return ret;
}
static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg)
{
__s64 ret;
@@ -4402,6 +4584,9 @@ static long userfaultfd_ioctl(struct file *file, unsigned cmd,
case UFFDIO_POISON:
ret = userfaultfd_poison(ctx, arg);
break;
case UFFDIO_RWPROTECT:
ret = userfaultfd_rwprotect(ctx, arg);
break;
}
return ret;
}