mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-07-16 17:57:38 -04:00
Code Issues Packages Projects Releases Wiki Activity

sched/fair: Co-locate cfs_rq and sched_entity in cfs_tg_state

Browse Source 
Improve data locality and reduce pointer chasing by allocating struct
cfs_rq and struct sched_entity together for non-root task groups. This
is achieved by introducing a new combined struct cfs_tg_state that
holds both objects in a single allocation.

This patch:

 - Introduces struct cfs_tg_state that embeds cfs_rq, sched_entity, and
   sched_statistics together in a single structure.

 - Updates __schedstats_from_se() in stats.h to use cfs_tg_state for accessing
   sched_statistics from a group sched_entity.

 - Modifies alloc_fair_sched_group() and free_fair_sched_group() to allocate
   and free the new struct as a single unit.

 - Modifies the per-CPU pointers in task_group->se and task_group->cfs_rq to
   point to the members in the new combined structure.

Signed-off-by: Zecheng Li <zecheng@google.com>
Signed-off-by: Zecheng Li <zli94@ncsu.edu>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: K Prateek Nayak <kprateek.nayak@amd.com>
Reviewed-by: Josh Don <joshdon@google.com>
Link: https://patch.msgid.link/20260522141623.600235-2-zli94@ncsu.edu
This commit is contained in:
Zecheng Li
2026-05-22 10:15:48 -04:00
committed by Peter Zijlstra
parent 63c1a12bc0
commit dfcfc97b6d
3 changed files with 19 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
kernel/sched/fair.c
View File

@@ -15083,8 +15083,6 @@ void free_fair_sched_group(struct task_group *tg)
for_each_possible_cpu(i) {
if (tg->cfs_rq)
kfree(tg->cfs_rq[i]);
if (tg->se)
kfree(tg->se[i]);
}
kfree(tg->cfs_rq);
@@ -15093,6 +15091,7 @@ void free_fair_sched_group(struct task_group *tg)
int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
{
struct cfs_tg_state *state;
struct sched_entity *se;
struct cfs_rq *cfs_rq;
int i;
@@ -15109,16 +15108,13 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
init_cfs_bandwidth(tg_cfs_bandwidth(tg), tg_cfs_bandwidth(parent));
for_each_possible_cpu(i) {
cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
GFP_KERNEL, cpu_to_node(i));
if (!cfs_rq)
state = kzalloc_node(sizeof(*state),
GFP_KERNEL, cpu_to_node(i));
if (!state)
goto err;
se = kzalloc_node(sizeof(struct sched_entity_stats),
GFP_KERNEL, cpu_to_node(i));
if (!se)
goto err_free_rq;
cfs_rq = &state->cfs_rq;
se = &state->se;
init_cfs_rq(cfs_rq);
init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]);
init_entity_runnable_average(se);
@@ -15126,8 +15122,6 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
return 1;
err_free_rq:
kfree(cfs_rq);
err:
return 0;
}

12
kernel/sched/sched.h
View File

@@ -2294,6 +2294,18 @@ static inline struct task_group *task_group(struct task_struct *p)
return p->sched_task_group;
}
#ifdef CONFIG_FAIR_GROUP_SCHED
/*
* Defined here to be available before stats.h is included, since
* stats.h has dependencies on things defined later in this file.
*/
struct cfs_tg_state {
struct cfs_rq cfs_rq;
struct sched_entity se;
struct sched_statistics stats;
} __no_randomize_layout;
#endif
/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
{

9
kernel/sched/stats.h
View File

@@ -89,19 +89,12 @@ static inline void rq_sched_info_depart (struct rq *rq, unsigned long long delt
#endif /* CONFIG_SCHEDSTATS */
#ifdef CONFIG_FAIR_GROUP_SCHED
struct sched_entity_stats {
struct sched_entity se;
struct sched_statistics stats;
} __no_randomize_layout;
#endif
static inline struct sched_statistics *
__schedstats_from_se(struct sched_entity *se)
{
#ifdef CONFIG_FAIR_GROUP_SCHED
if (!entity_is_task(se))
return &container_of(se, struct sched_entity_stats, se)->stats;
return &container_of(se, struct cfs_tg_state, se)->stats;
#endif
return &task_of(se)->stats;
}