mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-05-16 02:01:18 -04:00
Code Issues Packages Projects Releases Wiki Activity

userfaultfd: retry copying with locks dropped in mfill_atomic_pte_copy()

Browse Source 
Implementation of UFFDIO_COPY for anonymous memory might fail to copy data
from userspace buffer when the destination VMA is locked (either with
mm_lock or with per-VMA lock).

In that case, mfill_atomic() releases the locks, retries copying the data
with locks dropped and then re-locks the destination VMA and
re-establishes PMD.

Since this retry-reget dance is only relevant for UFFDIO_COPY and it never
happens for other UFFDIO_ operations, make it a part of
mfill_atomic_pte_copy() that actually implements UFFDIO_COPY for anonymous
memory.

As a temporal safety measure to avoid breaking biscection
mfill_atomic_pte_copy() makes sure to never return -ENOENT so that the
loop in mfill_atomic() won't retry copiyng outside of mmap_lock.  This is
removed later when shmem implementation will be updated later and the loop
in mfill_atomic() will be adjusted.

[akpm@linux-foundation.org: update mfill_copy_folio_retry()]
  Link: https://lore.kernel.org/20260316173829.1126728-1-avagin@google.com
Link: https://lore.kernel.org/20260306171815.3160826-6-rppt@kernel.org
Link: https://lore.kernel.org/20260402041156.1377214-6-rppt@kernel.org
Signed-off-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Reviewed-by: Harry Yoo (Oracle) <harry@kernel.org>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Axel Rasmussen <axelrasmussen@google.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: David Hildenbrand (Arm) <david@kernel.org>
Cc: Hugh Dickins <hughd@google.com>
Cc: James Houghton <jthoughton@google.com>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Lorenzo Stoakes (Oracle) <ljs@kernel.org>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nikita Kalyazin <kalyazin@amazon.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Peter Xu <peterx@redhat.com>
Cc: Sean Christopherson <seanjc@google.com>
Cc: Shuah Khan <shuah@kernel.org>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: David Carlier <devnexen@gmail.com>
Cc: Harry Yoo <harry.yoo@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Mike Rapoport (Microsoft)
2026-04-02 07:11:46 +03:00
committed by Andrew Morton
parent b8c03b7f45
commit f5f035a724
1 changed files with 51 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
mm/userfaultfd.c
View File

@@ -405,35 +405,63 @@ static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr)
return ret;
}
static int mfill_copy_folio_retry(struct mfill_state *state, struct folio *folio)
{
unsigned long src_addr = state->src_addr;
void *kaddr;
int err;
/* retry copying with mm_lock dropped */
mfill_put_vma(state);
kaddr = kmap_local_folio(folio, 0);
err = copy_from_user(kaddr, (const void __user *) src_addr, PAGE_SIZE);
kunmap_local(kaddr);
if (unlikely(err))
return -EFAULT;
flush_dcache_folio(folio);
/* reget VMA and PMD, they could change underneath us */
err = mfill_get_vma(state);
if (err)
return err;
err = mfill_establish_pmd(state);
if (err)
return err;
return 0;
}
static int mfill_atomic_pte_copy(struct mfill_state *state)
{
struct vm_area_struct *dst_vma = state->vma;
unsigned long dst_addr = state->dst_addr;
unsigned long src_addr = state->src_addr;
uffd_flags_t flags = state->flags;
pmd_t *dst_pmd = state->pmd;
struct folio *folio;
int ret;
if (!state->folio) {
ret = -ENOMEM;
folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, dst_vma,
dst_addr);
if (!folio)
goto out;
folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, state->vma, dst_addr);
if (!folio)
return -ENOMEM;
ret = mfill_copy_folio_locked(folio, src_addr);
ret = -ENOMEM;
if (mem_cgroup_charge(folio, state->vma->vm_mm, GFP_KERNEL))
goto out_release;
/* fallback to copy_from_user outside mmap_lock */
if (unlikely(ret)) {
ret = -ENOENT;
state->folio = folio;
/* don't free the page */
goto out;
}
} else {
folio = state->folio;
state->folio = NULL;
ret = mfill_copy_folio_locked(folio, src_addr);
if (unlikely(ret)) {
/*
* Fallback to copy_from_user outside mmap_lock.
* If retry is successful, mfill_copy_folio_locked() returns
* with locks retaken by mfill_get_vma().
* If there was an error, we must mfill_put_vma() anyway and it
* will take care of unlocking if needed.
*/
ret = mfill_copy_folio_retry(state, folio);
if (ret)
goto out_release;
}
/*
@@ -443,17 +471,16 @@ static int mfill_atomic_pte_copy(struct mfill_state *state)
*/
__folio_mark_uptodate(folio);
ret = -ENOMEM;
if (mem_cgroup_charge(folio, dst_vma->vm_mm, GFP_KERNEL))
goto out_release;
ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr,
ret = mfill_atomic_install_pte(state->pmd, state->vma, dst_addr,
&folio->page, true, flags);
if (ret)
goto out_release;
out:
return ret;
out_release:
/* Don't return -ENOENT so that our caller won't retry */
if (ret == -ENOENT)
ret = -EFAULT;
folio_put(folio);
goto out;
}