mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-07-24 17:23:29 -04:00
Code Issues Packages Projects Releases Wiki Activity

sched/fair: Drop redundant RCU read lock in NOHZ kick path

Browse Source 
nohz_balancer_kick() is reached from sched_balance_trigger(), which is
called from sched_tick(). sched_tick() runs with IRQs disabled, so the
additional rcu_read_lock/unlock() used around sched_domain accesses in
this path is redundant. Rely on the existing IRQ-disabled context (and
the rcu_dereference_all() checking) instead.

The same applies to set_cpu_sd_state_idle(), called from the idle entry
path with IRQs disabled, and to set_cpu_sd_state_busy(), reachable via
nohz_balance_exit_idle() from two contexts: nohz_balancer_kick() (IRQs
disabled, as above) and sched_cpu_deactivate() (the CPUHP_AP_ACTIVE
teardown, which runs under cpus_write_lock(), so it cannot race with
sched-domain rebuilds). In both cases the rcu_dereference_all()
validation is sufficient.

No functional change intended.

Suggested-by: K Prateek Nayak <kprateek.nayak@amd.com>
Signed-off-by: Andrea Righi <arighi@nvidia.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: K Prateek Nayak <kprateek.nayak@amd.com>
Reviewed-by: Vincent Guittot <vincent.guittot@linaro.org>
Link: https://patch.msgid.link/20260509180955.1840064-2-arighi@nvidia.com
This commit is contained in:
Andrea Righi
2026-05-09 20:07:25 +02:00
committed by Peter Zijlstra
parent acbdbab75f
commit c9d93a73ce
1 changed files with 11 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
kernel/sched/fair.c
View File

@@ -12715,8 +12715,6 @@ static void nohz_balancer_kick(struct rq *rq)
goto out;
}
rcu_read_lock();
sd = rcu_dereference_all(rq->sd);
if (sd) {
/*
@@ -12724,8 +12722,8 @@ static void nohz_balancer_kick(struct rq *rq)
* capacity, kick the ILB to see if there's a better CPU to run on:
*/
if (rq->cfs.h_nr_runnable >= 1 && check_cpu_capacity(rq, sd)) {
flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
goto unlock;
flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
goto out;
}
}
@@ -12741,8 +12739,8 @@ static void nohz_balancer_kick(struct rq *rq)
*/
for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) {
if (sched_asym(sd, i, cpu)) {
flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
goto unlock;
flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
goto out;
}
}
}
@@ -12753,10 +12751,8 @@ static void nohz_balancer_kick(struct rq *rq)
* When ASYM_CPUCAPACITY; see if there's a higher capacity CPU
* to run the misfit task on.
*/
if (check_misfit_status(rq)) {
flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
goto unlock;
}
if (check_misfit_status(rq))
flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
/*
* For asymmetric systems, we do not want to nicely balance
@@ -12765,7 +12761,7 @@ static void nohz_balancer_kick(struct rq *rq)
*
* Skip the LLC logic because it's not relevant in that case.
*/
goto unlock;
goto out;
}
sds = rcu_dereference_all(per_cpu(sd_llc_shared, cpu));
@@ -12780,13 +12776,9 @@ static void nohz_balancer_kick(struct rq *rq)
* like this LLC domain has tasks we could move.
*/
nr_busy = atomic_read(&sds->nr_busy_cpus);
if (nr_busy > 1) {
flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
goto unlock;
}
if (nr_busy > 1)
flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
}
unlock:
rcu_read_unlock();
out:
if (READ_ONCE(nohz.needs_update))
flags |= NOHZ_NEXT_KICK;
@@ -12798,17 +12790,13 @@ static void nohz_balancer_kick(struct rq *rq)
static void set_cpu_sd_state_busy(int cpu)
{
struct sched_domain *sd;
rcu_read_lock();
sd = rcu_dereference_all(per_cpu(sd_llc, cpu));
if (!sd || !sd->nohz_idle)
goto unlock;
return;
sd->nohz_idle = 0;
atomic_inc(&sd->shared->nr_busy_cpus);
unlock:
rcu_read_unlock();
}
void nohz_balance_exit_idle(struct rq *rq)
@@ -12827,17 +12815,13 @@ void nohz_balance_exit_idle(struct rq *rq)
static void set_cpu_sd_state_idle(int cpu)
{
struct sched_domain *sd;
rcu_read_lock();
sd = rcu_dereference_all(per_cpu(sd_llc, cpu));
if (!sd || sd->nohz_idle)
goto unlock;
return;
sd->nohz_idle = 1;
atomic_dec(&sd->shared->nr_busy_cpus);
unlock:
rcu_read_unlock();
}
/*