mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-31 10:31:33 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'mm-hotfixes-stable' into mm-stable to pick up

Browse Source 
already-upstream changes to memcontrol.c, needed by "memcg: move
mem_cgroup_swappiness and vm_swappiness to mm/swap.h".
This commit is contained in:
Andrew Morton
2026-08-24 18:40:27 -07:00
parent 08745c6235 aedf2efd18
commit 4b47e81e56
22 changed files with 349 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.mailmap
View File

@@ -170,6 +170,7 @@ Boris Brezillon <bbrezillon@kernel.org> <b.brezillon@overkiz.com>
Boris Brezillon <bbrezillon@kernel.org> <boris.brezillon@bootlin.com>
Boris Brezillon <bbrezillon@kernel.org> <boris.brezillon@free-electrons.com>
Brendan Higgins <brendan.higgins@linux.dev> <brendanhiggins@google.com>
Brendan Jackman <brendan.jackman@linux.dev> <jackmanb@google.com>
Brian Avery <b.avery@hp.com>
Brian Cain <bcain@kernel.org> <brian.cain@oss.qualcomm.com>
Brian Cain <bcain@kernel.org> <bcain@quicinc.com>
@@ -232,6 +233,8 @@ Daniel Lezcano <daniel.lezcano@kernel.org> <daniel.lezcano@linexp.org>
Daniel Lezcano <daniel.lezcano@kernel.org> <dlezcano@fr.ibm.com>
Daniel Thompson <danielt@kernel.org> <daniel.thompson@linaro.org>
Daniele Alessandrelli <daniele.alessandrelli@gmail.com> <daniele.alessandrelli@intel.com>
Danila Tikhonov <danila@mainlining.org> <danila@jiaxyga.com>
Danila Tikhonov <danila@mainlining.org> <JIaxyga@protonmail.com>
Danilo Krummrich <dakr@kernel.org> <dakr@redhat.com>
David Brownell <david-b@pacbell.net>
David Collins <quic_collinsd@quicinc.com> <collinsd@codeaurora.org>
@@ -322,6 +325,8 @@ Gokul Sriram Palanisamy <quic_gokulsri@quicinc.com> <gokulsri@codeaurora.org>
Govindaraj Saminathan <quic_gsamin@quicinc.com> <gsamin@codeaurora.org>
Guo Ren <guoren@kernel.org> <guoren@linux.alibaba.com>
Guo Ren <guoren@kernel.org> <ren_guo@c-sky.com>
Guodong Xu <docular.xu@gmail.com> <guodong.xu@linaro.org>
Guodong Xu <docular.xu@gmail.com> <guodong@riscstar.com>
Guru Das Srinagesh <linux@gurudas.dev>
Guru Das Srinagesh <linux@gurudas.dev> <quic_gurus@quicinc.com>
Guru Das Srinagesh <linux@gurudas.dev> <gurus@codeaurora.org>
@@ -456,6 +461,7 @@ Jorge Ramirez-Ortiz <jorge.ramirez@oss.qualcomm.com> <jorge.ramirez-ortiz@linaro
Josh Poimboeuf <jpoimboe@kernel.org> <jpoimboe@redhat.com>
Josh Poimboeuf <jpoimboe@kernel.org> <jpoimboe@us.ibm.com>
Jouni Malinen <quic_jouni@quicinc.com> <jouni@codeaurora.org>
JP Kobryn <jp.kobryn@linux.dev> <inwardvessel@gmail.com>
Juha Yrjola <at solidboot.com>
Juha Yrjola <juha.yrjola@nokia.com>
Juha Yrjola <juha.yrjola@solidboot.com>

5
MAINTAINERS
View File

@@ -4982,7 +4982,7 @@ F: tools/lib/bpf/
BPF [MEMORY MANAGEMENT EXTENSIONS]
M: Roman Gushchin <roman.gushchin@linux.dev>
M: JP Kobryn <inwardvessel@gmail.com>
M: JP Kobryn <jp.kobryn@linux.dev>
M: Shakeel Butt <shakeel.butt@linux.dev>
L: bpf@vger.kernel.org
L: linux-mm@kvack.org
@@ -17160,7 +17160,7 @@ M: Andrew Morton <akpm@linux-foundation.org>
M: Vlastimil Babka <vbabka@kernel.org>
R: Suren Baghdasaryan <surenb@google.com>
R: Michal Hocko <mhocko@suse.com>
R: Brendan Jackman <jackmanb@google.com>
R: Brendan Jackman <brendan.jackman@linux.dev>
R: Johannes Weiner <hannes@cmpxchg.org>
R: Zi Yan <ziy@nvidia.com>
L: linux-mm@kvack.org
@@ -25553,7 +25553,6 @@ M: Phillip Lougher <phillip@squashfs.org.uk>
L: squashfs-devel@lists.sourceforge.net (subscribers-only)
S: Maintained
W: http://squashfs.org.uk
T: git git://git.kernel.org/pub/scm/linux/kernel/git/pkl/squashfs-next.git
F: Documentation/filesystems/squashfs.rst
F: fs/squashfs/

2
arch/arm64/include/asm/ptdump.h
View File

@@ -7,8 +7,6 @@
#include <linux/ptdump.h>
DECLARE_STATIC_KEY_FALSE(arm64_ptdump_lock_key);
#ifdef CONFIG_PTDUMP
#include <linux/mm_types.h>

43
arch/arm64/mm/mmu.c
View File

@@ -49,8 +49,6 @@
#define NO_CONT_MAPPINGS BIT(1)
#define NO_EXEC_MAPPINGS BIT(2) /* assumes FEAT_HPDS is not used */
DEFINE_STATIC_KEY_FALSE(arm64_ptdump_lock_key);
u64 kimage_voffset __ro_after_init;
EXPORT_SYMBOL(kimage_voffset);
@@ -1864,8 +1862,7 @@ int pmd_clear_huge(pmd_t *pmdp)
return 1;
}
static int __pmd_free_pte_page(pmd_t *pmdp, unsigned long addr,
bool acquire_mmap_lock)
int pmd_free_pte_page(pmd_t *pmdp, unsigned long addr)
{
pte_t *table;
pmd_t pmd;
@@ -1877,25 +1874,13 @@ static int __pmd_free_pte_page(pmd_t *pmdp, unsigned long addr,
return 1;
}
/* See comment in pud_free_pmd_page for static key logic */
table = pte_offset_kernel(pmdp, addr);
pmd_clear(pmdp);
__flush_tlb_kernel_pgtable(addr);
if (static_branch_unlikely(&arm64_ptdump_lock_key) && acquire_mmap_lock) {
mmap_read_lock(&init_mm);
mmap_read_unlock(&init_mm);
}
pte_free_kernel(NULL, table);
return 1;
}
int pmd_free_pte_page(pmd_t *pmdp, unsigned long addr)
{
/* If ptdump is walking the pagetables, acquire init_mm.mmap_lock */
return __pmd_free_pte_page(pmdp, addr, /* acquire_mmap_lock = */ true);
}
int pud_free_pmd_page(pud_t *pudp, unsigned long addr)
{
pmd_t *table;
@@ -1911,36 +1896,16 @@ int pud_free_pmd_page(pud_t *pudp, unsigned long addr)
}
table = pmd_offset(pudp, addr);
/*
* Our objective is to prevent ptdump from reading a PMD table which has
* been freed. In this race, if pud_free_pmd_page observes the key on
* (which got flipped by ptdump) then the mmap lock sequence here will,
* as a result of the mmap write lock/unlock sequence in ptdump, give
* us the correct synchronization. If not, this means that ptdump has
* yet not started walking the pagetables - the sequence of barriers
* issued by __flush_tlb_kernel_pgtable() guarantees that ptdump will
* observe an empty PUD.
*/
pud_clear(pudp);
__flush_tlb_kernel_pgtable(addr);
if (static_branch_unlikely(&arm64_ptdump_lock_key)) {
mmap_read_lock(&init_mm);
mmap_read_unlock(&init_mm);
}
pmdp = table;
next = addr;
end = addr + PUD_SIZE;
do {
if (pmd_present(pmdp_get(pmdp)))
/*
* PMD has been isolated, so ptdump won't see it. No
* need to acquire init_mm.mmap_lock.
*/
__pmd_free_pte_page(pmdp, next, /* acquire_mmap_lock = */ false);
pmd_free_pte_page(pmdp, next);
} while (pmdp++, next += PMD_SIZE, next != end);
pud_clear(pudp);
__flush_tlb_kernel_pgtable(addr);
pmd_free(NULL, table);
return 1;
}

11
arch/arm64/mm/ptdump.c
View File

@@ -283,13 +283,6 @@ void note_page_flush(struct ptdump_state *pt_st)
note_page(pt_st, 0, -1, pte_val(pte_zero));
}
static void arm64_ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm)
{
static_branch_inc(&arm64_ptdump_lock_key);
ptdump_walk_pgd(st, mm, NULL);
static_branch_dec(&arm64_ptdump_lock_key);
}
void ptdump_walk(struct seq_file *s, struct ptdump_info *info)
{
unsigned long end = ~0UL;
@@ -318,7 +311,7 @@ void ptdump_walk(struct seq_file *s, struct ptdump_info *info)
}
};
arm64_ptdump_walk_pgd(&st.ptdump, info->mm);
ptdump_walk_pgd(&st.ptdump, info->mm, NULL);
}
static void __init ptdump_initialize(void)
@@ -360,7 +353,7 @@ bool ptdump_check_wx(void)
}
};
arm64_ptdump_walk_pgd(&st.ptdump, &init_mm);
ptdump_walk_pgd(&st.ptdump, &init_mm, NULL);
if (st.wx_pages || st.uxn_pages) {
pr_warn("Checked W+X mappings: FAILED, %lu W+X pages found, %lu non-UXN pages found\n",

2
arch/microblaze/kernel/head.S
View File

@@ -39,6 +39,8 @@
#include <asm/processor.h>
.section .data
/* The MMU requires a page aligned page directory. */
.align 12
.global swapper_pg_dir
swapper_pg_dir:
.space PAGE_SIZE

1
include/linux/mmap_lock.h
View File

@@ -629,6 +629,7 @@ static inline void mmap_read_unlock(struct mm_struct *mm)
DEFINE_GUARD(mmap_read_lock, struct mm_struct *,
mmap_read_lock(_T), mmap_read_unlock(_T))
DEFINE_GUARD_COND(mmap_read_lock, _try, mmap_read_trylock(_T))
static inline void mmap_read_unlock_non_owner(struct mm_struct *mm)
{

13
mm/damon/lru_sort.c
View File

@@ -233,6 +233,8 @@ static int damon_lru_sort_add_quota_goals(struct damos *hot_scheme,
if (!active_mem_bp)
return 0;
if (10000 < active_mem_bp)
return -EINVAL;
goal = damos_new_quota_goal(DAMOS_QUOTA_ACTIVE_MEM_BP, active_mem_bp);
if (!goal)
return -ENOMEM;
@@ -344,6 +346,8 @@ static int damon_lru_sort_commit_inputs_fn(void *arg)
return damon_lru_sort_apply_parameters();
}
static bool damon_lru_sort_damon_has_started;
static int damon_lru_sort_commit_inputs_store(const char *val,
const struct kernel_param *kp)
{
@@ -364,11 +368,8 @@ static int damon_lru_sort_commit_inputs_store(const char *val,
if (!commit_inputs_request)
return 0;
/*
* Skip damon_call() if ctx is not initialized to avoid
* NULL pointer dereference.
*/
if (!ctx)
/* Skip damon_call() if ctx has not successfully started. */
if (!damon_lru_sort_damon_has_started)
return -EINVAL;
err = damon_call(ctx, &control);
@@ -421,6 +422,8 @@ static int damon_lru_sort_turn(bool on)
err = damon_start(&ctx, 1, true);
if (err)
return err;
if (!damon_lru_sort_damon_has_started)
damon_lru_sort_damon_has_started = true;
return damon_call(ctx, &call_control);
}

13
mm/damon/ops-common.c
View File

@@ -374,6 +374,8 @@ static unsigned int damon_migrate_folio_list(struct list_head *folio_list,
while (!list_empty(folio_list)) {
folio = lru_to_folio(folio_list);
list_del(&folio->lru);
node_stat_sub_folio(folio, NR_ISOLATED_ANON +
folio_is_file_lru(folio));
folio_putback_lru(folio);
}
@@ -391,8 +393,17 @@ unsigned long damon_migrate_pages(struct list_head *folio_list, int target_nid)
return nr_migrated;
if (target_nid < 0 || target_nid >= MAX_NUMNODES ||
!node_state(target_nid, N_MEMORY))
!node_state(target_nid, N_MEMORY)) {
while (!list_empty(folio_list)) {
struct folio *folio = lru_to_folio(folio_list);
list_del(&folio->lru);
node_stat_sub_folio(folio, NR_ISOLATED_ANON +
folio_is_file_lru(folio));
folio_putback_lru(folio);
}
return nr_migrated;
}
noreclaim_flag = memalloc_noreclaim_save();

2
mm/damon/paddr.c
View File

@@ -346,6 +346,8 @@ static unsigned long damon_pa_migrate(struct damon_region *r,
if (!folio_isolate_lru(folio))
goto put_folio;
node_stat_add_folio(folio, NR_ISOLATED_ANON +
folio_is_file_lru(folio));
list_add(&folio->lru, &folio_list);
put_folio:
addr += folio_size(folio);

11
mm/damon/reclaim.c
View File

@@ -271,6 +271,8 @@ static int damon_reclaim_commit_inputs_fn(void *arg)
return damon_reclaim_apply_parameters();
}
static bool damon_reclaim_damon_has_started;
static int damon_reclaim_commit_inputs_store(const char *val,
const struct kernel_param *kp)
{
@@ -291,11 +293,8 @@ static int damon_reclaim_commit_inputs_store(const char *val,
if (!commit_inputs_request)
return 0;
/*
* Skip damon_call() if ctx is not initialized to avoid
* NULL pointer dereference.
*/
if (!ctx)
/* Skip damon_call() if ctx has not successfully started. */
if (!damon_reclaim_damon_has_started)
return -EINVAL;
err = damon_call(ctx, &control);
@@ -344,6 +343,8 @@ static int damon_reclaim_turn(bool on)
err = damon_start(&ctx, 1, true);
if (err)
return err;
if (!damon_reclaim_damon_has_started)
damon_reclaim_damon_has_started = true;
return damon_call(ctx, &call_control);
}

3
mm/damon/vaddr.c
View File

@@ -627,7 +627,8 @@ static void damos_va_migrate_dests_add(struct folio *folio,
isolate:
if (!folio_isolate_lru(folio))
return;
node_stat_add_folio(folio, NR_ISOLATED_ANON +
folio_is_file_lru(folio));
list_add(&folio->lru, &migration_lists[i]);
}

6
mm/filemap.c
View File

@@ -931,6 +931,12 @@ noinline int __filemap_add_folio(struct address_space *mapping,
if (!xas_nomem(&xas, gfp))
break;
/*
* Lock has been dropped: start again with the original index
* and order (but now with the memory reserved by xas_nomem()).
*/
xas_set_order(&xas, index, forder);
}
if (xas_error(&xas))

199
mm/huge_memory.c
View File

@@ -41,6 +41,7 @@
#include <linux/pgalloc.h>
#include <linux/pgalloc_tag.h>
#include <linux/pagewalk.h>
#include <linux/cleanup.h>
#include <asm/tlb.h>
#include "internal.h"
@@ -77,9 +78,15 @@ static unsigned long deferred_split_scan(struct shrinker *shrink,
struct shrink_control *sc);
static bool split_underused_thp = true;
static atomic_t huge_zero_refcount;
#define HUGE_ZERO_UNSET_PFN (~0UL)
struct folio *huge_zero_folio __read_mostly;
unsigned long huge_zero_pfn __read_mostly = ~0UL;
unsigned long huge_zero_pfn __read_mostly = HUGE_ZERO_UNSET_PFN;
#ifndef CONFIG_PERSISTENT_HUGE_ZERO_FOLIO
static atomic_t huge_zero_refcount;
static DEFINE_SPINLOCK(huge_zero_lock);
static struct shrinker *huge_zero_folio_shrinker;
#endif
unsigned long huge_anon_orders_always __read_mostly;
unsigned long huge_anon_orders_madvise __read_mostly;
unsigned long huge_anon_orders_inherit __read_mostly;
@@ -221,33 +228,74 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
return orders;
}
static bool get_huge_zero_folio(void)
static struct folio *alloc_huge_zero_folio(void)
{
struct folio *zero_folio;
retry:
if (likely(atomic_inc_not_zero(&huge_zero_refcount)))
return true;
zero_folio = folio_alloc((GFP_TRANSHUGE | __GFP_ZERO | __GFP_ZEROTAGS) &
~__GFP_MOVABLE,
HPAGE_PMD_ORDER);
if (!zero_folio) {
count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED);
return false;
return NULL;
}
/* Ensure zero folio won't have large_rmappable flag set. */
folio_clear_large_rmappable(zero_folio);
preempt_disable();
if (cmpxchg(&huge_zero_folio, NULL, zero_folio)) {
preempt_enable();
folio_put(zero_folio);
goto retry;
}
WRITE_ONCE(huge_zero_pfn, folio_pfn(zero_folio));
folio_clear_large_rmappable(zero_folio); /* Explicitly not rmappable. */
return zero_folio;
}
#ifdef CONFIG_PERSISTENT_HUGE_ZERO_FOLIO
static int __init huge_zero_init(void)
{
huge_zero_folio = alloc_huge_zero_folio();
if (!huge_zero_folio) {
pr_warn("Allocating persistent huge zero folio failed\n");
} else {
huge_zero_pfn = folio_pfn(huge_zero_folio);
count_vm_event(THP_ZERO_PAGE_ALLOC);
}
return 0;
}
static void __init huge_zero_shrinker_exit(void)
{
}
struct folio *mm_get_huge_zero_folio(struct mm_struct *mm)
{
return huge_zero_folio;
}
void mm_put_huge_zero_folio(struct mm_struct *mm)
{
}
#else
static bool get_huge_zero_folio(void)
{
struct folio *zero_folio;
/* Paired with atomic_set_release(). */
if (likely(atomic_inc_not_zero(&huge_zero_refcount)))
return true;
zero_folio = alloc_huge_zero_folio();
if (unlikely(!zero_folio))
return false;
/* Paired with critical section in shrink_huge_zero_folio_scan(). */
spin_lock(&huge_zero_lock);
if (huge_zero_folio) {
/* Somebody else already installed it. */
atomic_inc(&huge_zero_refcount);
spin_unlock(&huge_zero_lock);
folio_put(zero_folio);
return true;
}
WRITE_ONCE(huge_zero_folio, zero_folio);
WRITE_ONCE(huge_zero_pfn, folio_pfn(zero_folio));
/* Paired with atomic_inc_not_zero(). +1 for shrinker pin. */
atomic_set_release(&huge_zero_refcount, 2);
spin_unlock(&huge_zero_lock);
/* We take additional reference here. It will be put back by shrinker */
atomic_set(&huge_zero_refcount, 2);
preempt_enable();
count_vm_event(THP_ZERO_PAGE_ALLOC);
return true;
}
@@ -258,14 +306,59 @@ static void put_huge_zero_folio(void)
* Counter should never go to zero here. Only shrinker can put
* last reference.
*/
BUG_ON(atomic_dec_and_test(&huge_zero_refcount));
WARN_ON_ONCE(atomic_dec_and_test(&huge_zero_refcount));
}
static unsigned long shrink_huge_zero_folio_count(struct shrinker *shrink,
struct shrink_control *sc)
{
/* we can free zero page only if last reference remains */
return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0;
}
static unsigned long shrink_huge_zero_folio_scan(struct shrinker *shrink,
struct shrink_control *sc)
{
struct folio *zero_folio;
/* Paired with critical section in get_huge_zero_folio(). */
scoped_guard(spinlock, &huge_zero_lock) {
/* Paired with atomic_inc_not_zero() in get_huge_zero_folio(). */
if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) != 1)
return 0;
zero_folio = huge_zero_folio;
VM_WARN_ON_ONCE(!zero_folio);
WRITE_ONCE(huge_zero_folio, NULL);
WRITE_ONCE(huge_zero_pfn, HUGE_ZERO_UNSET_PFN);
}
folio_put(zero_folio);
return HPAGE_PMD_NR;
}
static int __init huge_zero_init(void)
{
huge_zero_folio_shrinker = shrinker_alloc(0, "thp-zero");
if (!huge_zero_folio_shrinker) {
shrinker_free(deferred_split_shrinker);
list_lru_destroy(&deferred_split_lru);
return -ENOMEM;
}
huge_zero_folio_shrinker->count_objects = shrink_huge_zero_folio_count;
huge_zero_folio_shrinker->scan_objects = shrink_huge_zero_folio_scan;
shrinker_register(huge_zero_folio_shrinker);
return 0;
}
static void __init huge_zero_shrinker_exit(void)
{
shrinker_free(huge_zero_folio_shrinker);
}
struct folio *mm_get_huge_zero_folio(struct mm_struct *mm)
{
if (IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO))
return huge_zero_folio;
if (mm_flags_test(MMF_HUGE_ZERO_FOLIO, mm))
return READ_ONCE(huge_zero_folio);
@@ -280,35 +373,10 @@ struct folio *mm_get_huge_zero_folio(struct mm_struct *mm)
void mm_put_huge_zero_folio(struct mm_struct *mm)
{
if (IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO))
return;
if (mm_flags_test(MMF_HUGE_ZERO_FOLIO, mm))
put_huge_zero_folio();
}
static unsigned long shrink_huge_zero_folio_count(struct shrinker *shrink,
struct shrink_control *sc)
{
/* we can free zero page only if last reference remains */
return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0;
}
static unsigned long shrink_huge_zero_folio_scan(struct shrinker *shrink,
struct shrink_control *sc)
{
if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) {
struct folio *zero_folio = xchg(&huge_zero_folio, NULL);
BUG_ON(zero_folio == NULL);
WRITE_ONCE(huge_zero_pfn, ~0UL);
folio_put(zero_folio);
return HPAGE_PMD_NR;
}
return 0;
}
static struct shrinker *huge_zero_folio_shrinker;
#endif /* CONFIG_PERSISTENT_HUGE_ZERO_FOLIO */
#ifdef CONFIG_SYSFS
static ssize_t enabled_show(struct kobject *kobj,
@@ -970,39 +1038,14 @@ static int __init thp_shrinker_init(void)
deferred_split_shrinker->scan_objects = deferred_split_scan;
shrinker_register(deferred_split_shrinker);
if (IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO)) {
/*
* Bump the reference of the huge_zero_folio and do not
* initialize the shrinker.
*
* huge_zero_folio will always be NULL on failure. We assume
* that get_huge_zero_folio() will most likely not fail as
* thp_shrinker_init() is invoked early on during boot.
*/
if (!get_huge_zero_folio())
pr_warn("Allocating persistent huge zero folio failed\n");
return 0;
}
huge_zero_folio_shrinker = shrinker_alloc(0, "thp-zero");
if (!huge_zero_folio_shrinker) {
shrinker_free(deferred_split_shrinker);
list_lru_destroy(&deferred_split_lru);
return -ENOMEM;
}
huge_zero_folio_shrinker->count_objects = shrink_huge_zero_folio_count;
huge_zero_folio_shrinker->scan_objects = shrink_huge_zero_folio_scan;
shrinker_register(huge_zero_folio_shrinker);
return 0;
return huge_zero_init();
}
static void __init thp_shrinker_exit(void)
{
shrinker_free(huge_zero_folio_shrinker);
shrinker_free(deferred_split_shrinker);
list_lru_destroy(&deferred_split_lru);
huge_zero_shrinker_exit();
}
static int __init hugepage_init(void)
@@ -4111,7 +4154,7 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
gfp_t gfp;
mapping = folio->mapping;
min_order = mapping_min_folio_order(folio->mapping);
min_order = mapping_min_folio_order(mapping);
if (new_order < min_order) {
ret = -EINVAL;
goto out;
@@ -4125,6 +4168,8 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
goto out;
}
mapping_set_update(&xas, mapping);
if (split_type == SPLIT_TYPE_UNIFORM) {
xas_set_order(&xas, folio->index, new_order);
xas_split_alloc(&xas, folio, old_order, gfp);

102
mm/memcontrol.c
View File

@@ -2912,10 +2912,9 @@ struct mem_cgroup *mem_cgroup_from_virt(void *p)
return folio_memcg_check(virt_to_folio(p));
}
static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg)
static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg,
int nid)
{
int nid = numa_node_id();
for (; memcg; memcg = parent_mem_cgroup(memcg)) {
struct obj_cgroup *objcg = rcu_dereference(memcg->nodeinfo[nid]->objcg);
@@ -2926,12 +2925,13 @@ static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg)
return NULL;
}
static inline struct obj_cgroup *get_obj_cgroup_from_memcg(struct mem_cgroup *memcg)
static inline struct obj_cgroup *get_obj_cgroup_from_memcg(struct mem_cgroup *memcg,
int nid)
{
struct obj_cgroup *objcg;
rcu_read_lock();
objcg = __get_obj_cgroup_from_memcg(memcg);
objcg = __get_obj_cgroup_from_memcg(memcg, nid);
rcu_read_unlock();
return objcg;
@@ -2975,7 +2975,7 @@ static struct obj_cgroup *current_objcg_update(void)
rcu_read_lock();
memcg = mem_cgroup_from_task(current);
objcg = __get_obj_cgroup_from_memcg(memcg);
objcg = __get_obj_cgroup_from_memcg(memcg, numa_node_id());
rcu_read_unlock();
/*
@@ -4182,9 +4182,11 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
page_counter_init(&memcg->swap, &parent->swap, false);
#ifdef CONFIG_MEMCG_V1
memcg->memory.track_failcnt = !memcg_on_dfl;
memcg->memsw.track_failcnt = !memcg_on_dfl;
WRITE_ONCE(memcg->oom_kill_disable, READ_ONCE(parent->oom_kill_disable));
page_counter_init(&memcg->kmem, &parent->kmem, false);
page_counter_init(&memcg->tcpmem, &parent->tcpmem, false);
memcg->tcpmem.track_failcnt = !memcg_on_dfl;
#endif
} else {
init_memcg_stats();
@@ -5138,7 +5140,7 @@ static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg,
int ret = 0;
struct obj_cgroup *objcg;
objcg = get_obj_cgroup_from_memcg(memcg);
objcg = get_obj_cgroup_from_memcg(memcg, folio_nid(folio));
/* Do not account at the root objcg level. */
if (!obj_cgroup_is_root(objcg))
ret = try_charge_memcg(memcg, gfp, folio_nr_pages(folio));
@@ -5337,6 +5339,46 @@ void __mem_cgroup_uncharge_folios(struct folio_batch *folios)
uncharge_batch(&ug);
}
/*
* An LRU folio must hold the objcg belonging to its own node.
*
* memcg_reparent_objcgs() reparents a dying cgroup one node at a time: the
* folios on that node's LRU lists move to the parent and that node's objcg is
* redirected to the parent, atomically under the node's lru_lock.
* folio_lruvec_lock() relies on this to provide a stable folio<->lruvec
* binding. If a folio holds another node's objcg, its list membership and its
* lruvec resolution change in separate lock sections, and an LRU operation in
* between can re-add the folio to, and strand it on, the LRU list of a dead
* memcg.
*
* So when migration transfers the memcg state to a folio on another node,
* re-derive the objcg for the destination node. If the memcg is dying and the
* destination node has already been reparented, the lookup walks up to the
* nearest live ancestor - which is also where that node's LRU lists went.
*
* Returns the objcg to commit to @new, with a reference for the caller.
*/
static struct obj_cgroup *get_migration_objcg(struct folio *old,
struct folio *new)
{
struct obj_cgroup *old_objcg, *new_objcg;
int new_nid = folio_nid(new);
old_objcg = get_obj_cgroup_from_folio(old);
if (folio_nid(old) == new_nid)
return old_objcg;
rcu_read_lock();
new_objcg = __get_obj_cgroup_from_memcg(obj_cgroup_memcg(old_objcg),
new_nid);
rcu_read_unlock();
obj_cgroup_put(old_objcg);
return new_objcg;
}
/**
* mem_cgroup_replace_folio - Charge a folio's replacement.
* @old: Currently circulating folio.
@@ -5365,21 +5407,28 @@ void mem_cgroup_replace_folio(struct folio *old, struct folio *new)
if (folio_memcg_charged(new))
return;
objcg = folio_objcg(old);
VM_WARN_ON_ONCE_FOLIO(!objcg, old);
if (!objcg)
VM_WARN_ON_ONCE_FOLIO(!folio_objcg(old), old);
if (!folio_objcg(old))
return;
objcg = get_migration_objcg(old, new);
rcu_read_lock();
memcg = obj_cgroup_memcg(objcg);
/* Force-charge the new page. The old one will be freed soon */
/*
* Force-charge the new page. The old one will be freed soon.
*
* The rootness of the committed objcg decides whether the final
* uncharge of @new goes through the page counters (see
* uncharge_folio()); charge them only if the uncharge will.
*/
if (!obj_cgroup_is_root(objcg)) {
page_counter_charge(&memcg->memory, nr_pages);
if (do_memsw_account())
page_counter_charge(&memcg->memsw, nr_pages);
}
obj_cgroup_get(objcg);
commit_charge(new, objcg);
memcg1_commit_charge(new, memcg);
rcu_read_unlock();
@@ -5391,14 +5440,15 @@ void mem_cgroup_replace_folio(struct folio *old, struct folio *new)
* @new: Replacement folio.
*
* Transfer the memcg data from the old folio to the new folio for migration.
* The old folio's data info will be cleared. Note that the memory counters
* will remain unchanged throughout the process.
* The old folio's data info will be cleared. The memory counters remain
* unchanged, unless the charge moves out of a fully reparented ancestry
* and has to be settled (see below).
*
* Both folios must be locked, @new->mapping must be set up.
*/
void mem_cgroup_migrate(struct folio *old, struct folio *new)
{
struct obj_cgroup *objcg;
struct obj_cgroup *objcg, *new_objcg;
VM_BUG_ON_FOLIO(!folio_test_locked(old), old);
VM_BUG_ON_FOLIO(!folio_test_locked(new), new);
@@ -5419,12 +5469,30 @@ void mem_cgroup_migrate(struct folio *old, struct folio *new)
if (!objcg)
return;
/* Transfer the charge and the objcg ref */
commit_charge(new, objcg);
new_objcg = get_migration_objcg(old, new);
/*
* @old was charged through a non-root objcg, so its charge is in the
* page counters. If the re-derivation walked up to the root objcg -
* @old's entire ancestry is dying and already reparented - the final
* uncharge of @new will skip the page counters (see uncharge_folio()).
* Settle them now: this is @old's eventual uncharge, moved up to the
* point where its charge record ends.
*/
if (obj_cgroup_is_root(new_objcg) && !obj_cgroup_is_root(objcg)) {
rcu_read_lock();
memcg_uncharge(obj_cgroup_memcg(objcg), folio_nr_pages(old));
rcu_read_unlock();
}
commit_charge(new, new_objcg);
/* Warning should never happen, so don't worry about refcount non-0 */
WARN_ON_ONCE(folio_unqueue_deferred_split(old));
old->memcg_data = 0;
/* @new holds its own reference now, drop @old's */
obj_cgroup_put(objcg);
}
DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);

20
mm/page_table_check.c
View File

@@ -151,18 +151,29 @@ void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr,
if (&init_mm == mm)
return;
if (pte_user_accessible_page(mm, addr, pte))
if (pte_user_accessible_page(mm, addr, pte) && !pte_special(pte))
page_table_check_clear(pte_pfn(pte), PAGE_SIZE >> PAGE_SHIFT);
}
EXPORT_SYMBOL(__page_table_check_pte_clear);
static inline bool page_table_check_huge_zero_pmd(pmd_t pmd)
{
unsigned long pfn = pmd_pfn(pmd);
if (!pfn_valid(pfn))
return false;
return is_huge_zero_folio(page_folio(pfn_to_page(pfn)));
}
void __page_table_check_pmd_clear(struct mm_struct *mm, unsigned long addr,
pmd_t pmd)
{
if (&init_mm == mm)
return;
if (pmd_user_accessible_page(mm, addr, pmd))
if (pmd_user_accessible_page(mm, addr, pmd) &&
!page_table_check_huge_zero_pmd(pmd))
page_table_check_clear(pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT);
}
EXPORT_SYMBOL(__page_table_check_pmd_clear);
@@ -208,7 +219,7 @@ void __page_table_check_ptes_set(struct mm_struct *mm, unsigned long addr,
for (i = 0; i < nr; i++)
__page_table_check_pte_clear(mm, addr + PAGE_SIZE * i, ptep_get(ptep + i));
if (pte_user_accessible_page(mm, addr, pte))
if (pte_user_accessible_page(mm, addr, pte) && !pte_special(pte))
page_table_check_set(pte_pfn(pte), nr, pte_write(pte));
}
EXPORT_SYMBOL(__page_table_check_ptes_set);
@@ -238,7 +249,8 @@ void __page_table_check_pmds_set(struct mm_struct *mm, unsigned long addr,
for (i = 0; i < nr; i++)
__page_table_check_pmd_clear(mm, addr + PMD_SIZE * i, *(pmdp + i));
if (pmd_user_accessible_page(mm, addr, pmd))
if (pmd_user_accessible_page(mm, addr, pmd) &&
!page_table_check_huge_zero_pmd(pmd))
page_table_check_set(pmd_pfn(pmd), stride * nr, pmd_write(pmd));
}
EXPORT_SYMBOL(__page_table_check_pmds_set);

42
mm/pagewalk.c
View File

@@ -126,6 +126,7 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
pmd = pmd_offset(pud, addr);
do {
again:
walk->action = ACTION_SUBTREE;
next = pmd_addr_end(addr, end);
if (pmd_none(*pmd)) {
if (has_install)
@@ -138,8 +139,6 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
continue;
}
walk->action = ACTION_SUBTREE;
/*
* This implies that each ->pmd_entry() handler
* needs to know about pmd_trans_huge() pmds
@@ -196,6 +195,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
pud = pud_offset(p4d, addr);
do {
again:
walk->action = ACTION_SUBTREE;
next = pud_addr_end(addr, end);
if (pud_none(*pud)) {
if (has_install)
@@ -208,8 +208,6 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
continue;
}
walk->action = ACTION_SUBTREE;
if (ops->pud_entry)
err = ops->pud_entry(pud, addr, next, walk);
if (err)
@@ -678,6 +676,8 @@ int walk_kernel_page_table_range_lockless(unsigned long start, unsigned long end
* will also not lock the PTEs for the pte_entry() callback.
*
* This is for debugging purposes ONLY.
*
* The mmap write lock must be held.
*/
int walk_page_range_debug(struct mm_struct *mm, unsigned long start,
unsigned long end, const struct mm_walk_ops *ops,
@@ -691,25 +691,29 @@ int walk_page_range_debug(struct mm_struct *mm, unsigned long start,
.no_vma = true
};
/* For convenience, we allow traversal of kernel mappings. */
if (mm == &init_mm)
return walk_kernel_page_table_range(start, end, ops,
pgd, private);
if (start >= end || !walk.mm)
/*
* When walking userland page tables, an mmap write lock must be held to
* account for munmap() downgrading to an mmap read lock when tearing
* down page tables.
*
* When walking kernel page tables, an mmap write lock must also be held
* to account for page table freeing on vmap huge page mapping.
*/
mmap_assert_write_locked(mm);
/*
* x86, arm64 ptdump allow walks of efi mm's and x86 ptdump allows walks
* of arbitrary mm's.
*
* However, they both must also hold the init_mm lock to account for
* concurrent kernel page table freeing.
*/
mmap_assert_write_locked(&init_mm);
if (start >= end)
return -EINVAL;
if (!check_ops_safe(ops))
return -EINVAL;
/*
* The mmap lock protects the page walker from changes to the page
* tables during the walk. However a read lock is insufficient to
* protect those areas which don't have a VMA as munmap() detaches
* the VMAs before downgrading to a read lock and actually tearing
* down PTEs/page tables. In which case, the mmap write lock should
* be held.
*/
mmap_assert_write_locked(mm);
return walk_pgd_range(start, end, &walk);
}

7
mm/ptdump.c
View File

@@ -178,11 +178,18 @@ void ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm, pgd_t *pgd)
get_online_mems();
mmap_write_lock(mm);
/* To stabilise kernel page tables we must hold the init_mm lock too. */
if (mm != &init_mm)
mmap_write_lock_nested(&init_mm, SINGLE_DEPTH_NESTING);
while (range->start != range->end) {
walk_page_range_debug(mm, range->start, range->end,
&ptdump_ops, pgd, st);
range++;
}
if (mm != &init_mm)
mmap_write_unlock(&init_mm);
mmap_write_unlock(mm);
put_online_mems();

9
mm/swapfile.c
View File

@@ -2202,7 +2202,14 @@ void swap_free_hibernation_slot(swp_entry_t entry)
ci = swap_cluster_lock(si, offset);
__swap_cluster_put_entry(ci, offset % SWAPFILE_CLUSTER);
__swap_cluster_free_entries(si, ci, offset % SWAPFILE_CLUSTER, 1);
/*
* A slot with a folio in the swap cache is freed when the folio
* leaves the cache, the same rule swap_put_entries_cluster() follows.
* Readahead can put a folio here, and freeing the slot now would
* leave that folio with no entry behind it.
*/
if (!swp_tb_is_folio(__swap_table_get(ci, offset % SWAPFILE_CLUSTER)))
__swap_cluster_free_entries(si, ci, offset % SWAPFILE_CLUSTER, 1);
swap_cluster_unlock(ci);
/* In theory readahead might add it to the swap cache by accident */

40
mm/vmalloc.c
View File

@@ -43,6 +43,7 @@
#include <asm/tlbflush.h>
#include <asm/shmparam.h>
#include <linux/page_owner.h>
#include <linux/cleanup.h>
#define CREATE_TRACE_POINTS
#include <trace/events/vmalloc.h>
@@ -159,10 +160,21 @@ static int vmap_try_huge_pmd(pmd_t *pmd, unsigned long addr, unsigned long end,
if (!IS_ALIGNED(phys_addr, PMD_SIZE))
return 0;
if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr))
return 0;
if (!pmd_present(*pmd))
return pmd_set_huge(pmd, phys_addr, prot);
return pmd_set_huge(pmd, phys_addr, prot);
/*
* Acquire the mmap read lock to exclude ptdump, which walks
* kernel page tables it does not own under the mmap write lock.
*
* Concurrent read lock holders are safe: each exclusively owns
* the range it operates on and cannot reach this page table.
*/
scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm) {
if (!pmd_free_pte_page(pmd, addr))
return 0;
return pmd_set_huge(pmd, phys_addr, prot);
}
}
static int vmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
@@ -211,10 +223,15 @@ static int vmap_try_huge_pud(pud_t *pud, unsigned long addr, unsigned long end,
if (!IS_ALIGNED(phys_addr, PUD_SIZE))
return 0;
if (pud_present(*pud) && !pud_free_pmd_page(pud, addr))
return 0;
if (!pud_present(*pud))
return pud_set_huge(pud, phys_addr, prot);
return pud_set_huge(pud, phys_addr, prot);
/* See comment in vmap_try_huge_pmd(). */
scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm) {
if (!pud_free_pmd_page(pud, addr))
return 0;
return pud_set_huge(pud, phys_addr, prot);
}
}
static int vmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
@@ -263,10 +280,15 @@ static int vmap_try_huge_p4d(p4d_t *p4d, unsigned long addr, unsigned long end,
if (!IS_ALIGNED(phys_addr, P4D_SIZE))
return 0;
if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr))
return 0;
if (!p4d_present(*p4d))
return p4d_set_huge(p4d, phys_addr, prot);
return p4d_set_huge(p4d, phys_addr, prot);
/* See comment in vmap_try_huge_pmd(). */
scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm) {
if (!p4d_free_pud_page(p4d, addr))
return 0;
return p4d_set_huge(p4d, phys_addr, prot);
}
}
static int vmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,

2
mm/vmscan.c
View File

@@ -5948,7 +5948,7 @@ static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
}
}
cond_resched();
cond_resched_tasks_rcu_qs();
if (nr_reclaimed < nr_to_reclaim || proportional_reclaim)
continue;

3
samples/damon/mtier.c
View File

@@ -156,6 +156,9 @@ static struct damon_ctx *damon_sample_mtier_build_ctx(bool promote)
if (!scheme)
goto free_out;
damon_set_schemes(ctx, &scheme, 1);
/* zero target value causes division by zero in damos_quota_store() */
if (!node0_mem_used_bp || !node0_mem_free_bp)
goto free_out;
quota_goal = damos_new_quota_goal(
promote ? DAMOS_QUOTA_NODE_MEM_USED_BP :
DAMOS_QUOTA_NODE_MEM_FREE_BP,