mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2026-07-23 09:47:31 -04:00
mm: Add ptep_try_set() for lockless empty-slot installs
Add ptep_try_set(ptep, new_pte): atomically set *ptep to new_pte iff it is currently pte_none(). Returns true on success, false if the slot was already populated or the arch has no implementation. The intended caller is the upcoming bpf_arena kernel-side fault recovery path. The install runs from a page fault that can be nested under locks held by the faulting kernel caller (e.g. a BPF program holding raw_res_spin_lock_irqsave on its arena's spinlock), so trylock-and-retry would A-A deadlock. Lock-free cmpxchg is the only viable option, which constrains this helper to special kernel page tables where concurrent writers cooperate via atomic accessors. The generic version in <linux/pgtable.h> returns false. x86 and arm64 override with try_cmpxchg-based implementations on the underlying pteval. Other architectures get the false stub - the callers there already fall through to oops. v2: Rename to ptep_try_set(). Tighten kerneldoc. (David, Alexei) v3: Note that strict-zero cmpxchg is narrower than pte_none(). (Andrea) Suggested-by: Kumar Kartikeya Dwivedi <memxor@gmail.com> Suggested-by: Alexei Starovoitov <ast@kernel.org> Signed-off-by: Tejun Heo <tj@kernel.org> Reviewed-by: Andrea Righi <arighi@nvidia.com> Cc: David Hildenbrand <david@kernel.org> Acked-by: David Hildenbrand (arm) <david@kernel.org> Link: https://lore.kernel.org/r/20260522172219.1423324-2-tj@kernel.org Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
committed by
Alexei Starovoitov
parent
5200f5f493
commit
258df8fce4
@@ -1830,6 +1830,18 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
|
||||
return __ptep_get_and_clear(mm, addr, ptep);
|
||||
}
|
||||
|
||||
/*
|
||||
* Note: strictly-zero compare is narrower than pte_none(), but the gap is
|
||||
* harmless: a fresh kernel PTE has no software bits set.
|
||||
*/
|
||||
static inline bool ptep_try_set(pte_t *ptep, pte_t new_pte)
|
||||
{
|
||||
pteval_t old = 0;
|
||||
|
||||
return try_cmpxchg(&pte_val(*ptep), &old, pte_val(new_pte));
|
||||
}
|
||||
#define ptep_try_set ptep_try_set
|
||||
|
||||
#define test_and_clear_young_ptes test_and_clear_young_ptes
|
||||
static inline bool test_and_clear_young_ptes(struct vm_area_struct *vma,
|
||||
unsigned long addr, pte_t *ptep, unsigned int nr)
|
||||
|
||||
@@ -1284,6 +1284,18 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm,
|
||||
} while (!try_cmpxchg((long *)&ptep->pte, (long *)&old_pte, *(long *)&new_pte));
|
||||
}
|
||||
|
||||
/*
|
||||
* Note: strictly-zero compare is narrower than pte_none(), but the gap is
|
||||
* harmless: _PAGE_DIRTY and _PAGE_ACCESSED aren't set on untouched kernel PTEs.
|
||||
*/
|
||||
static inline bool ptep_try_set(pte_t *ptep, pte_t new_pte)
|
||||
{
|
||||
pte_t old_pte = __pte(0);
|
||||
|
||||
return try_cmpxchg((long *)&ptep->pte, (long *)&old_pte, *(long *)&new_pte);
|
||||
}
|
||||
#define ptep_try_set ptep_try_set
|
||||
|
||||
#define flush_tlb_fix_spurious_fault(vma, address, ptep) do { } while (0)
|
||||
|
||||
#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
|
||||
|
||||
@@ -1036,6 +1036,31 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addres
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef ptep_try_set
|
||||
/**
|
||||
* ptep_try_set - atomically set an empty kernel PTE
|
||||
* @ptep: page table entry
|
||||
* @new_pte: value to install
|
||||
*
|
||||
* Atomically set *@ptep to @new_pte iff *@ptep is pte_none(). Return true on
|
||||
* success, false if the slot was already populated or the arch has no
|
||||
* implementation.
|
||||
*
|
||||
* For special kernel page tables only - never user page tables. The caller must
|
||||
* prevent concurrent teardown of @ptep and must accept that other writers may
|
||||
* race. Concurrent clearers must use ptep_get_and_clear() so racing accesses
|
||||
* agree on the outcome.
|
||||
*
|
||||
* Architectures opt in by providing a cmpxchg-based override and defining
|
||||
* ptep_try_set as an identity macro. The generic stub returns false, which is
|
||||
* correct for callers that fall through to oops on failure.
|
||||
*/
|
||||
static inline bool ptep_try_set(pte_t *ptep, pte_t new_pte)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef wrprotect_ptes
|
||||
/**
|
||||
* wrprotect_ptes - Write-protect PTEs that map consecutive pages of the same
|
||||
|
||||
Reference in New Issue
Block a user