mirrors/linux
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2025-12-27 12:21:22 -05:00
Code Issues Packages Projects Releases Wiki Activity

memcg: move preempt disable to callers of memcg_rstat_updated

Browse Source 
Let's move the explicit preempt disable code to the callers of
memcg_rstat_updated and also remove the memcg_stats_lock and related
functions which ensures the callers of stats update functions have
disabled preemption because now the stats update functions are explicitly
disabling preemption.

Link: https://lkml.kernel.org/r/20250514184158.3471331-3-shakeel.butt@linux.dev
Signed-off-by: Shakeel Butt <shakeel.butt@linux.dev>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Shakeel Butt
2025-05-14 11:41:53 -07:00
committed by Andrew Morton
parent 8a4b42b955
commit c7163535cd
1 changed files with 19 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
mm/memcontrol.c
View File

@@ -555,48 +555,22 @@ static u64 flush_last_time;
#define FLUSH_TIME (2UL*HZ)
/*
* Accessors to ensure that preemption is disabled on PREEMPT_RT because it can
* not rely on this as part of an acquired spinlock_t lock. These functions are
* never used in hardirq context on PREEMPT_RT and therefore disabling preemtion
* is sufficient.
*/
static void memcg_stats_lock(void)
{
preempt_disable_nested();
VM_WARN_ON_IRQS_ENABLED();
}
static void __memcg_stats_lock(void)
{
preempt_disable_nested();
}
static void memcg_stats_unlock(void)
{
preempt_enable_nested();
}
static bool memcg_vmstats_needs_flush(struct memcg_vmstats *vmstats)
{
return atomic64_read(&vmstats->stats_updates) >
MEMCG_CHARGE_BATCH * num_online_cpus();
}
static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val)
static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val,
int cpu)
{
struct memcg_vmstats_percpu __percpu *statc_pcpu;
struct memcg_vmstats_percpu *statc;
int cpu;
unsigned int stats_updates;
if (!val)
return;
/* Don't assume callers have preemption disabled. */
cpu = get_cpu();
cgroup_rstat_updated(memcg->css.cgroup, cpu);
statc_pcpu = memcg->vmstats_percpu;
for (; statc_pcpu; statc_pcpu = statc->parent_pcpu) {
@@ -617,7 +591,6 @@ static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val)
stats_updates = this_cpu_xchg(statc_pcpu->stats_updates, 0);
atomic64_add(stats_updates, &statc->vmstats->stats_updates);
}
put_cpu();
}
static void __mem_cgroup_flush_stats(struct mem_cgroup *memcg, bool force)
@@ -715,6 +688,7 @@ void __mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx,
int val)
{
int i = memcg_stats_index(idx);
int cpu;
if (mem_cgroup_disabled())
return;
@@ -722,12 +696,14 @@ void __mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx,
if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))
return;
memcg_stats_lock();
cpu = get_cpu();
__this_cpu_add(memcg->vmstats_percpu->state[i], val);
val = memcg_state_val_in_pages(idx, val);
memcg_rstat_updated(memcg, val);
memcg_rstat_updated(memcg, val, cpu);
trace_mod_memcg_state(memcg, idx, val);
memcg_stats_unlock();
put_cpu();
}
#ifdef CONFIG_MEMCG_V1
@@ -756,6 +732,7 @@ static void __mod_memcg_lruvec_state(struct lruvec *lruvec,
struct mem_cgroup_per_node *pn;
struct mem_cgroup *memcg;
int i = memcg_stats_index(idx);
int cpu;
if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))
return;
@@ -763,24 +740,7 @@ static void __mod_memcg_lruvec_state(struct lruvec *lruvec,
pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
memcg = pn->memcg;
/*
* The caller from rmap relies on disabled preemption because they never
* update their counter from in-interrupt context. For these two
* counters we check that the update is never performed from an
* interrupt context while other caller need to have disabled interrupt.
*/
__memcg_stats_lock();
if (IS_ENABLED(CONFIG_DEBUG_VM)) {
switch (idx) {
case NR_ANON_MAPPED:
case NR_FILE_MAPPED:
case NR_ANON_THPS:
WARN_ON_ONCE(!in_task());
break;
default:
VM_WARN_ON_IRQS_ENABLED();
}
}
cpu = get_cpu();
/* Update memcg */
__this_cpu_add(memcg->vmstats_percpu->state[i], val);
@@ -789,9 +749,10 @@ static void __mod_memcg_lruvec_state(struct lruvec *lruvec,
__this_cpu_add(pn->lruvec_stats_percpu->state[i], val);
val = memcg_state_val_in_pages(idx, val);
memcg_rstat_updated(memcg, val);
memcg_rstat_updated(memcg, val, cpu);
trace_mod_memcg_lruvec_state(memcg, idx, val);
memcg_stats_unlock();
put_cpu();
}
/**
@@ -871,6 +832,7 @@ void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
unsigned long count)
{
int i = memcg_events_index(idx);
int cpu;
if (mem_cgroup_disabled())
return;
@@ -878,11 +840,13 @@ void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))
return;
memcg_stats_lock();
cpu = get_cpu();
__this_cpu_add(memcg->vmstats_percpu->events[i], count);
memcg_rstat_updated(memcg, count);
memcg_rstat_updated(memcg, count, cpu);
trace_count_memcg_events(memcg, idx, count);
memcg_stats_unlock();
put_cpu();
}
unsigned long memcg_events(struct mem_cgroup *memcg, int event)