mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-07-16 17:57:38 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'sched-core-2026-06-14' of gitolite.kernel.org:pub/scm/linux/kernel/git/tip/tip

Browse Source 
Pull scheduler updates from Ingo Molnar:
 "SMP load-balancing updates:

   - A large series to introduce infrastructure for cache-aware load
     balancing, with the goal of co-locating tasks that share data
     within the same Last Level Cache (LLC) domain. By improving cache
     locality, the scheduler can reduce cache bouncing and cache misses,
     ultimately improving data access efficiency.

     Implemented by Chen Yu and Tim Chen, based on early prototype work
     by Peter Zijlstra, with fixes by Jianyong Wu, Peter Zijlstra and
     Shrikanth Hegde.

   - A series to simplify CONFIG_SCHED_SMT ifdef usage (Shrikanth Hegde)

  Fair scheduler updates:

   - A series to improve SD_ASYM_CPUCAPACITY scheduling by introducing
     SMT awareness (Andrea Righi, K Prateek Nayak)

   - A series to optimize cfs_rq and sched_entity allocation for better
     data locality (Zecheng Li)

   - A preparatory series to change fair/cgroup scheduling to a single
     runqueue, without the final change (Peter Zijlstra)

   - Auto-manage ext/fair dl_server bandwidth (Andrea Righi)

   - Fix cpu_util runnable_avg arithmetic (Hongyan Xia)

   - Optimize update_tg_load_avg()'s rate-limiting code (Rik van Riel)

   - Allow account_cfs_rq_runtime() to throttle current hierarchy
     (K Prateek Nayak)

   - Update util_est after updating util_avg during dequeue, to fix the
     util signal update logic, which reduces signal noise (Vincent
     Guittot)

  Scheduler topology updates:

   - Allow multiple domains to claim sched_domain_shared (K Prateek
     Nayak)

   - Add parameter to split LLC (Peter Zijlstra)

  Core scheduler updates:

   - Use trace_call__<tp>() to save a static branch (Gabriele Monaco)

  Scheduler statistics updates:

   - Drop now-stale mul_u64_u64_div_u64() cputime over-approximation
     guard (Nicolas Pitre)

  Deadline scheduler updates:

   - Reject debugfs dl_server writes for offline CPUs (Andrea Righi)

   - Fix replenishment logic for non-deferred servers (Yuri Andriaccio)

  RT scheduling updates:

   - Turn RT_PUSH_IPI default off for non PREEMPT_RT (Steven Rostedt)

   - Update default bandwidth for real-time tasks to 1.0 (Yuri
     Andriaccio)

  Proxy scheduling updates:

   - A series to implement Optimized Donor Migration for Proxy Execution
     (John Stultz, Peter Zijlstra)

   - Various proxy scheduling cleanups and fixes (Peter Zijlstra,
     K Prateek Nayak)

  Misc fixes, improvements and cleanups by Aaron Lu, Andrea Righi,
  Zenghui Yu, Chen Yu, Guanyou.Chen, John Stultz, Shrikanth Hegde,
  Peter Zijlstra, Liang Luo and Yiyang Chen"

* tag 'sched-core-2026-06-14' of gitolite.kernel.org:pub/scm/linux/kernel/git/tip/tip: (91 commits)
  sched/fair: Fix newidle vs core-sched
  sched/deadline: Use task_on_rq_migrating() helper
  sched/core: Combine separate 'else' and 'if' statements
  sched/fair: Fix cpu_util runnable_avg arithmetic
  sched/fair: Unify cfs_rq throttling via account_cfs_rq_runtime()
  sched/fair: Move the throttled tasks to a local list in tg_unthrottle_up()
  sched/fair: Call update_curr() before unthrottling the hierarchy
  sched/fair: Use throttled_csd_list for local unthrottle
  sched/fair: Convert cfs bandwidth throttling to use guards
  sched/fair: Allocate cfs_tg_state with percpu allocator
  sched/fair: Remove task_group->se pointer array
  sched/fair: Co-locate cfs_rq and sched_entity in cfs_tg_state
  sched: restore timer_slack_ns when resetting RT policy on fork
  MAINTAINERS: Fix spelling mistake in Peter's name
  sched: Simplify ttwu_runnable()
  sched/proxy: Remove superfluous clear_task_blocked_in()
  sched/proxy: Remove PROXY_WAKING
  sched/proxy: Switch proxy to use p->is_blocked
  sched/proxy: Only return migrate when needed
  sched: Be more strict about p->is_blocked
  ...
This commit is contained in:
Linus Torvalds
2026-06-15 14:50:18 +05:30
parent 0bcc2dc22f c095741713
commit 2cbf335f8c
38 changed files with 3501 additions and 984 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
Documentation/admin-guide/kernel-parameters.txt
View File

@@ -7215,6 +7215,18 @@ Kernel parameters
Not specifying this option is equivalent to
spec_store_bypass_disable=auto.
split_llc=
[X86,EARLY] Split the LLC N-ways
When set, the LLC is split this many ways by matching
'core_id % n'. This is setup before SMP bringup and
used during SMP bringup before it knows the full
topology. If your core count doesn't nicely divide by
the number given, you get to keep the pieces.
This is mostly a debug feature to emulate multiple LLCs
on hardware that only have a single LLC.
split_lock_detect=
[X86] Enable split lock detection or bus lock detection

2
MAINTAINERS
View File

@@ -21350,7 +21350,7 @@ F: include/uapi/linux/pps_gen.h
PRESSURE STALL INFORMATION (PSI)
M: Johannes Weiner <hannes@cmpxchg.org>
M: Suren Baghdasaryan <surenb@google.com>
R: Peter Ziljstra <peterz@infradead.org>
R: Peter Zijlstra <peterz@infradead.org>
S: Maintained
F: include/linux/psi*
F: kernel/sched/psi.c

7
arch/powerpc/include/asm/topology.h
View File

@@ -135,6 +135,13 @@ struct cpumask *cpu_coregroup_mask(int cpu);
const struct cpumask *cpu_die_mask(int cpu);
int cpu_die_id(int cpu);
/*
* Points to where the LLC is. On power9 this will point at CACHE
* domain, On others it will point to SMT domain. In all cases
* cpu_l2_cache_mask points to where LLC is
*/
#define arch_llc_mask(cpu) cpu_l2_cache_mask(cpu)
#ifdef CONFIG_PPC64
#include <asm/smp.h>

5
arch/x86/include/asm/processor.h
View File

@@ -704,6 +704,11 @@ static inline u32 per_cpu_l2c_id(unsigned int cpu)
return per_cpu(cpu_info.topo.l2c_id, cpu);
}
static inline u32 per_cpu_core_id(unsigned int cpu)
{
return per_cpu(cpu_info.topo.core_id, cpu);
}
#ifdef CONFIG_CPU_SUP_AMD
/*
* Issue a DIV 0/1 insn to clear any division data from previous DIV

20
arch/x86/kernel/smpboot.c
View File

@@ -424,6 +424,21 @@ static const struct x86_cpu_id intel_cod_cpu[] = {
{}
};
/*
* Allows splitting the LLC by matching 'core_id % split_llc'.
*
* This is mostly a debug hack to emulate systems with multiple LLCs per node
* on systems that do not naturally have this.
*/
static unsigned int split_llc = 0;
static int __init split_llc_setup(char *str)
{
get_option(&str, &split_llc);
return 0;
}
early_param("split_llc", split_llc_setup);
static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
{
const struct x86_cpu_id *id = x86_match_cpu(intel_cod_cpu);
@@ -438,6 +453,11 @@ static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
if (per_cpu_llc_id(cpu1) != per_cpu_llc_id(cpu2))
return false;
if (split_llc &&
(per_cpu_core_id(cpu1) % split_llc) !=
(per_cpu_core_id(cpu2) % split_llc))
return false;
/*
* Allow the SNC topology without warning. Return of false
* means 'c' does not share the LLC of 'o'. This will be

23
drivers/base/cacheinfo.c
View File

@@ -17,6 +17,7 @@
#include <linux/init.h>
#include <linux/of.h>
#include <linux/sched.h>
#include <linux/sched/topology.h>
#include <linux/slab.h>
#include <linux/smp.h>
#include <linux/sysfs.h>
@@ -68,6 +69,24 @@ bool last_level_cache_is_valid(unsigned int cpu)
}
/*
* Get the cacheinfo of the LLC associated with @cpu.
* Derived from update_per_cpu_data_slice_size_cpu().
*/
struct cacheinfo *get_cpu_cacheinfo_llc(unsigned int cpu)
{
struct cacheinfo *llc;
if (!last_level_cache_is_valid(cpu))
return NULL;
llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1);
if (llc->type != CACHE_TYPE_DATA && llc->type != CACHE_TYPE_UNIFIED)
return NULL;
return llc;
}
bool last_level_cache_is_shared(unsigned int cpu_x, unsigned int cpu_y)
{
struct cacheinfo *llc_x, *llc_y;
@@ -1018,6 +1037,7 @@ static int cacheinfo_cpu_online(unsigned int cpu)
goto err;
if (cpu_map_shared_cache(true, cpu, &cpu_map))
update_per_cpu_data_slice_size(true, cpu, cpu_map);
sched_update_llc_bytes(cpu);
return 0;
err:
free_cache_attributes(cpu);
@@ -1036,6 +1056,9 @@ static int cacheinfo_cpu_pre_down(unsigned int cpu)
free_cache_attributes(cpu);
if (nr_shared > 1)
update_per_cpu_data_slice_size(false, cpu, cpu_map);
sched_update_llc_bytes(cpu);
return 0;
}

1
include/linux/cacheinfo.h
View File

@@ -89,6 +89,7 @@ int populate_cache_leaves(unsigned int cpu);
int cache_setup_acpi(unsigned int cpu);
bool last_level_cache_is_valid(unsigned int cpu);
bool last_level_cache_is_shared(unsigned int cpu_x, unsigned int cpu_y);
struct cacheinfo *get_cpu_cacheinfo_llc(unsigned int cpu);
int fetch_cache_info(unsigned int cpu);
int detect_cache_attributes(unsigned int cpu);
#ifndef CONFIG_ACPI_PPTT

32
include/linux/mm_types.h
View File

@@ -1223,6 +1223,8 @@ struct mm_struct {
/* MM CID related storage */
struct mm_mm_cid mm_cid;
/* sched_cache related statistics */
struct sched_cache_stat sc_stat;
#ifdef CONFIG_MMU
atomic_long_t pgtables_bytes; /* size of all page tables */
#endif
@@ -1619,6 +1621,36 @@ static inline unsigned int mm_cid_size(void)
# define MM_CID_STATIC_SIZE 0
#endif /* CONFIG_SCHED_MM_CID */
#ifdef CONFIG_SCHED_CACHE
void mm_init_sched(struct mm_struct *mm,
struct sched_cache_time __percpu *pcpu_sched);
static inline int mm_alloc_sched_noprof(struct mm_struct *mm)
{
struct sched_cache_time __percpu *pcpu_sched =
alloc_percpu_noprof(struct sched_cache_time);
if (!pcpu_sched)
return -ENOMEM;
mm_init_sched(mm, pcpu_sched);
return 0;
}
#define mm_alloc_sched(...) alloc_hooks(mm_alloc_sched_noprof(__VA_ARGS__))
static inline void mm_destroy_sched(struct mm_struct *mm)
{
free_percpu(mm->sc_stat.pcpu_sched);
mm->sc_stat.pcpu_sched = NULL;
}
#else /* !CONFIG_SCHED_CACHE */
static inline int mm_alloc_sched(struct mm_struct *mm) { return 0; }
static inline void mm_destroy_sched(struct mm_struct *mm) { }
#endif /* CONFIG_SCHED_CACHE */
struct mmu_gather;
extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm);
extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm);

102
include/linux/sched.h
View File

@@ -161,7 +161,7 @@ struct user_event_mm;
*/
#define is_special_task_state(state) \
((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_PARKED | \
TASK_DEAD | TASK_FROZEN))
TASK_DEAD | TASK_WAKING | TASK_FROZEN))
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
# define debug_normal_state_change(state_value) \
@@ -702,6 +702,11 @@ struct sched_dl_entity {
* running, skipping the defer phase.
*
* @dl_defer_idle tracks idle state
*
* @dl_bw_attached tells if this server's bandwidth currently
* contributes to the root domain's total_bw. Only meaningful for server
* entities (@dl_server == 1). Allows toggling the reservation on/off
* without losing the configured @dl_runtime/@dl_period.
*/
unsigned int dl_throttled : 1;
unsigned int dl_yielded : 1;
@@ -713,6 +718,7 @@ struct sched_dl_entity {
unsigned int dl_defer_armed : 1;
unsigned int dl_defer_running : 1;
unsigned int dl_defer_idle : 1;
unsigned int dl_bw_attached : 1;
/*
* Bandwidth enforcement timer. Each -deadline task has its
@@ -846,7 +852,11 @@ struct task_struct {
struct alloc_tag *alloc_tag;
#endif
int on_cpu;
u8 on_cpu;
u8 on_rq;
u8 is_blocked;
u8 __pad;
struct __call_single_node wake_entry;
unsigned int wakee_flips;
unsigned long wakee_flip_decay_ts;
@@ -861,7 +871,6 @@ struct task_struct {
*/
int recent_used_cpu;
int wake_cpu;
int on_rq;
int prio;
int static_prio;
@@ -1243,6 +1252,13 @@ struct task_struct {
struct mutex *blocked_on; /* lock we're blocked on */
raw_spinlock_t blocked_lock;
/*
* The task that is boosting this task; a back link for the current
* donor stack. Set in schedule() -> find_proxy_task() and only stable
* under preempt_disable().
*/
struct task_struct *blocked_donor;
#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
/*
* Encoded lock address causing task block (lower 2 bits = type from
@@ -1403,6 +1419,13 @@ struct task_struct {
unsigned long numa_pages_migrated;
#endif /* CONFIG_NUMA_BALANCING */
#ifdef CONFIG_SCHED_CACHE
struct callback_head cache_work;
int preferred_llc;
/* 1: task was enqueued to its preferred LLC, 0 otherwise */
int pref_llc_queued;
#endif
struct rseq_data rseq;
struct sched_mm_cid mm_cid;
@@ -2177,19 +2200,10 @@ extern int __cond_resched_rwlock_write(rwlock_t *lock) __must_hold(lock);
#ifndef CONFIG_PREEMPT_RT
/*
* With proxy exec, if a task has been proxy-migrated, it may be a donor
* on a cpu that it can't actually run on. Thus we need a special state
* to denote that the task is being woken, but that it needs to be
* evaluated for return-migration before it is run. So if the task is
* blocked_on PROXY_WAKING, return migrate it before running it.
*/
#define PROXY_WAKING ((struct mutex *)(-1L))
static inline struct mutex *__get_task_blocked_on(struct task_struct *p)
{
lockdep_assert_held_once(&p->blocked_lock);
return p->blocked_on == PROXY_WAKING ? NULL : p->blocked_on;
return p->blocked_on;
}
static inline void __set_task_blocked_on(struct task_struct *p, struct mutex *m)
@@ -2217,7 +2231,7 @@ static inline void __clear_task_blocked_on(struct task_struct *p, struct mutex *
* blocked_on relationships, but make sure we are not
* clearing the relationship with a different lock.
*/
WARN_ON_ONCE(m && p->blocked_on && p->blocked_on != m && p->blocked_on != PROXY_WAKING);
WARN_ON_ONCE(m && p->blocked_on && p->blocked_on != m);
p->blocked_on = NULL;
}
@@ -2226,35 +2240,6 @@ static inline void clear_task_blocked_on(struct task_struct *p, struct mutex *m)
guard(raw_spinlock_irqsave)(&p->blocked_lock);
__clear_task_blocked_on(p, m);
}
static inline void __set_task_blocked_on_waking(struct task_struct *p, struct mutex *m)
{
/* Currently we serialize blocked_on under the task::blocked_lock */
lockdep_assert_held_once(&p->blocked_lock);
if (!sched_proxy_exec()) {
__clear_task_blocked_on(p, m);
return;
}
/* Don't set PROXY_WAKING if blocked_on was already cleared */
if (!p->blocked_on)
return;
/*
* There may be cases where we set PROXY_WAKING on tasks that were
* already set to waking, but make sure we are not changing
* the relationship with a different lock.
*/
WARN_ON_ONCE(m && p->blocked_on != m && p->blocked_on != PROXY_WAKING);
p->blocked_on = PROXY_WAKING;
}
static inline void set_task_blocked_on_waking(struct task_struct *p, struct mutex *m)
{
guard(raw_spinlock_irqsave)(&p->blocked_lock);
__set_task_blocked_on_waking(p, m);
}
#else
static inline void __clear_task_blocked_on(struct task_struct *p, struct rt_mutex *m)
{
@@ -2263,14 +2248,6 @@ static inline void __clear_task_blocked_on(struct task_struct *p, struct rt_mute
static inline void clear_task_blocked_on(struct task_struct *p, struct rt_mutex *m)
{
}
static inline void __set_task_blocked_on_waking(struct task_struct *p, struct rt_mutex *m)
{
}
static inline void set_task_blocked_on_waking(struct task_struct *p, struct rt_mutex *m)
{
}
#endif /* !CONFIG_PREEMPT_RT */
static __always_inline bool need_resched(void)
@@ -2403,6 +2380,29 @@ static __always_inline int task_mm_cid(struct task_struct *t)
}
#endif
#ifdef CONFIG_SCHED_CACHE
struct sched_cache_time {
u64 runtime;
unsigned long epoch;
};
struct sched_cache_stat {
struct sched_cache_time __percpu *pcpu_sched;
raw_spinlock_t lock;
unsigned long epoch;
u64 nr_running_avg;
unsigned long next_scan;
unsigned long footprint;
int cpu;
} ____cacheline_aligned_in_smp;
#else
struct sched_cache_stat { };
#endif
#ifndef MODULE
#ifndef COMPILE_OFFSETS

5
include/linux/sched/clock.h
View File

@@ -33,6 +33,11 @@ extern u64 sched_clock_cpu(int cpu);
extern void sched_clock_init(void);
#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
static inline int sched_clock_stable(void)
{
return 1;
}
static inline void sched_clock_tick(void)
{
}

4
include/linux/sched/smt.h
View File

@@ -4,16 +4,12 @@
#include <linux/static_key.h>
#ifdef CONFIG_SCHED_SMT
extern struct static_key_false sched_smt_present;
static __always_inline bool sched_smt_active(void)
{
return static_branch_likely(&sched_smt_present);
}
#else
static __always_inline bool sched_smt_active(void) { return false; }
#endif
void arch_smt_update(void);

32
include/linux/sched/topology.h
View File

@@ -67,7 +67,25 @@ struct sched_domain_shared {
atomic_t ref;
atomic_t nr_busy_cpus;
int has_idle_cores;
int nr_idle_scan;
union {
int nr_idle_scan;
/*
* Used during allocation to claim the sched_domain_shared
* object at multiple levels.
*
* Note: between build and the first periodic LB tick, which
* rewrites the union via update_idle_cpu_scan(), readers of
* nr_idle_scan may observe the transient SD_* flag value as
* the scan bound. The flag bits are small positive integers,
* so the effect is just a slightly relaxed scan bound for one
* window and self-heals on the first tick.
*/
int alloc_flags;
};
#ifdef CONFIG_SCHED_CACHE
unsigned long util_avg;
unsigned long capacity;
#endif
};
struct sched_domain {
@@ -99,6 +117,12 @@ struct sched_domain {
u64 max_newidle_lb_cost;
unsigned long last_decay_max_lb_cost;
#ifdef CONFIG_SCHED_CACHE
unsigned int llc_max;
unsigned int *llc_counts __counted_by_ptr(llc_max);
unsigned long llc_bytes;
#endif
#ifdef CONFIG_SCHEDSTATS
/* sched_balance_rq() stats */
unsigned int lb_count[CPU_MAX_IDLE_TYPES];
@@ -256,4 +280,10 @@ static inline int task_node(const struct task_struct *p)
return cpu_to_node(task_cpu(p));
}
#ifdef CONFIG_SCHED_CACHE
extern void sched_update_llc_bytes(unsigned int cpu);
#else
static inline void sched_update_llc_bytes(unsigned int cpu) { }
#endif
#endif /* _LINUX_SCHED_TOPOLOGY_H */

15
include/linux/topology.h
View File

@@ -230,11 +230,24 @@ static inline int cpu_to_mem(int cpu)
#define topology_drawer_cpumask(cpu) cpumask_of(cpu)
#endif
#if defined(CONFIG_SCHED_SMT) && !defined(cpu_smt_mask)
/*
* Defining cpu_smt_mask as cpumask_of that CPU helps to get
* rid of lot of ifdeffery all around the codebase in case of
* CONFIG_SCHED_SMT=n. It just means there are no other siblings, which
* is what is expected.
*/
#if defined(CONFIG_SCHED_SMT)
# if !defined(cpu_smt_mask)
static inline const struct cpumask *cpu_smt_mask(int cpu)
{
return topology_sibling_cpumask(cpu);
}
# endif
#else /* !CONFIG_SCHED_SMT */
static inline const struct cpumask *cpu_smt_mask(int cpu)
{
return cpumask_of(cpu);
}
#endif
#ifndef topology_is_primary_thread

11
init/Kconfig
View File

@@ -1016,6 +1016,17 @@ config NUMA_BALANCING
This system will be inactive on UMA systems.
config SCHED_CACHE
bool "Cache aware load balance"
default y
depends on SMP
help
When enabled, the scheduler will attempt to aggregate tasks from
the same process onto a single Last Level Cache (LLC) domain when
possible. This improves cache locality by keeping tasks that share
resources within the same cache domain, reducing cache misses and
lowering data access latency.
config NUMA_BALANCING_DEFAULT_ENABLED
bool "Automatically enable NUMA aware memory/task placement"
default y

5
init/init_task.c
View File

@@ -210,6 +210,7 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
.mems_allowed_seq = SEQCNT_SPINLOCK_ZERO(init_task.mems_allowed_seq,
&init_task.alloc_lock),
#endif
.blocked_donor = NULL,
#ifdef CONFIG_RT_MUTEXES
.pi_waiters = RB_ROOT_CACHED,
.pi_top_task = NULL,
@@ -225,6 +226,10 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
.numa_group = NULL,
.numa_faults = NULL,
#endif
#ifdef CONFIG_SCHED_CACHE
.preferred_llc = -1,
.pref_llc_queued = 0,
#endif
#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
.kasan_depth = 1,
#endif

29
kernel/exit.c
View File

@@ -543,6 +543,32 @@ void mm_update_next_owner(struct mm_struct *mm)
}
#endif /* CONFIG_MEMCG */
#if defined(CONFIG_SCHED_CACHE) && defined(CONFIG_NUMA_BALANCING)
/*
* Subtract the memory footprint of the current task from
* mm.
*/
static void exit_mm_sched_cache(struct mm_struct *mm)
{
unsigned long fp, sub;
if (!current->total_numa_faults)
return;
/*
* No lock protection due to performance considerations.
* Make sure mm->sc_stat.footprint does not become
* negative.
*/
fp = READ_ONCE(mm->sc_stat.footprint);
sub = min(fp, current->total_numa_faults);
WRITE_ONCE(mm->sc_stat.footprint, fp - sub);
}
#else
static inline void exit_mm_sched_cache(struct mm_struct *mm)
{
}
#endif /* CONFIG_SCHED_CACHE CONFIG_NUMA_BALANCING */
/*
* Turn us into a lazy TLB process if we
* aren't already..
@@ -554,6 +580,9 @@ static void exit_mm(void)
exit_mm_release(current, mm);
if (!mm)
return;
exit_mm_sched_cache(mm);
mmap_read_lock(mm);
mmgrab_lazy_tlb(mm);
BUG_ON(mm != current->active_mm);

7
kernel/fork.c
View File

@@ -728,6 +728,7 @@ void __mmdrop(struct mm_struct *mm)
cleanup_lazy_tlbs(mm);
WARN_ON_ONCE(mm == current->active_mm);
mm_destroy_sched(mm);
mm_free_pgd(mm);
mm_free_id(mm);
destroy_context(mm);
@@ -1128,6 +1129,9 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p)
if (mm_alloc_cid(mm, p))
goto fail_cid;
if (mm_alloc_sched(mm))
goto fail_sched;
if (percpu_counter_init_many(mm->rss_stat, 0, GFP_KERNEL_ACCOUNT,
NR_MM_COUNTERS))
goto fail_pcpu;
@@ -1136,6 +1140,8 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p)
return mm;
fail_pcpu:
mm_destroy_sched(mm);
fail_sched:
mm_destroy_cid(mm);
fail_cid:
destroy_context(mm);
@@ -2234,6 +2240,7 @@ __latent_entropy struct task_struct *copy_process(
lockdep_init_task(p);
p->blocked_on = NULL; /* not blocked yet */
p->blocked_donor = NULL; /* nobody is boosting p yet */
#ifdef CONFIG_BCACHE
p->sequential_io = 0;

63
kernel/locking/mutex.c
View File

@@ -763,6 +763,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
raw_spin_lock_irqsave(&lock->wait_lock, flags);
raw_spin_lock(&current->blocked_lock);
__set_task_blocked_on(current, lock);
set_current_state(state);
if (opt_acquired)
break;
@@ -980,15 +981,22 @@ EXPORT_SYMBOL_GPL(ww_mutex_lock_interruptible);
static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip)
__releases(lock)
{
struct task_struct *next = NULL;
struct task_struct *donor, *next = NULL;
struct mutex_waiter *waiter;
DEFINE_WAKE_Q(wake_q);
unsigned long owner;
unsigned long flags;
mutex_release(&lock->dep_map, ip);
__release(lock);
/*
* Ensures the proxy donor stack is stable across unlock and handoff.
* Specifically, it avoids the case where current->blocked_donor is
* NULL when it is inspected while doing the unlock, but a preemption
* before taking the wake_lock would make it set and a hand-off is
* missed.
*/
guard(preempt)();
/*
* Release the lock before (potentially) taking the spinlock such that
* other contenders can get on with things ASAP.
@@ -1001,6 +1009,12 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
MUTEX_WARN_ON(__owner_task(owner) != current);
MUTEX_WARN_ON(owner & MUTEX_FLAG_PICKUP);
if (sched_proxy_exec() && current->blocked_donor) {
/* force handoff if we have a blocked_donor */
owner = MUTEX_FLAG_HANDOFF;
break;
}
if (owner & MUTEX_FLAG_HANDOFF)
break;
@@ -1013,14 +1027,42 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
}
raw_spin_lock_irqsave(&lock->wait_lock, flags);
raw_spin_lock(&current->blocked_lock);
debug_mutex_unlock(lock);
waiter = lock->first_waiter;
if (waiter) {
next = waiter->task;
if (sched_proxy_exec()) {
/*
* If we have a task boosting current, and that task was boosting
* current through this lock, hand the lock to that task, as that
* is the highest waiter, as selected by the scheduling function.
*/
donor = current->blocked_donor;
if (donor) {
struct mutex *next_lock;
raw_spin_lock_nested(&donor->blocked_lock, SINGLE_DEPTH_NESTING);
next_lock = __get_task_blocked_on(donor);
if (next_lock == lock) {
next = get_task_struct(donor);
__clear_task_blocked_on(next, lock);
current->blocked_donor = NULL;
}
raw_spin_unlock(&donor->blocked_lock);
}
}
/*
* Failing that, pick first on the wait list.
*/
waiter = lock->first_waiter;
if (!next && waiter) {
next = get_task_struct(waiter->task);
raw_spin_lock_nested(&next->blocked_lock, SINGLE_DEPTH_NESTING);
debug_mutex_wake_waiter(lock, waiter);
set_task_blocked_on_waking(next, lock);
wake_q_add(&wake_q, next);
__clear_task_blocked_on(next, lock);
raw_spin_unlock(&next->blocked_lock);
}
if (trace_contended_release_enabled() && waiter)
@@ -1029,7 +1071,12 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
if (owner & MUTEX_FLAG_HANDOFF)
__mutex_handoff(lock, next);
raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);
raw_spin_unlock(&current->blocked_lock);
raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
if (next) {
wake_up_process(next);
put_task_struct(next);
}
}
#ifndef CONFIG_DEBUG_LOCK_ALLOC

4
kernel/locking/ww_mutex.h
View File

@@ -324,7 +324,7 @@ __ww_mutex_die(struct MUTEX *lock, struct MUTEX_WAITER *waiter,
* blocked_on to PROXY_WAKING. Otherwise we can see
* circular blocked_on relationships that can't resolve.
*/
set_task_blocked_on_waking(waiter->task, lock);
clear_task_blocked_on(waiter->task, lock);
wake_q_add(wake_q, waiter->task);
}
@@ -383,7 +383,7 @@ static bool __ww_mutex_wound(struct MUTEX *lock,
* are waking the mutex owner, who may be currently
* blocked on a different mutex.
*/
set_task_blocked_on_waking(owner, NULL);
clear_task_blocked_on(owner, NULL);
wake_q_add(wake_q, owner);
}
return true;

439
kernel/sched/core.c
View File

@@ -537,13 +537,22 @@ sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { }
/* need a wrapper since we may need to trace from modules */
EXPORT_TRACEPOINT_SYMBOL(sched_set_state_tp);
/* Call via the helper macro trace_set_current_state. */
/*
* Call via the helper macro trace_set_current_state.
* Calls to this function MUST be guarded by a
* tracepoint_enabled(sched_set_state_tp)
*/
void __trace_set_current_state(int state_value)
{
trace_sched_set_state_tp(current, state_value);
trace_call__sched_set_state_tp(current, state_value);
}
EXPORT_SYMBOL(__trace_set_current_state);
int task_llc(const struct task_struct *p)
{
return per_cpu(sd_llc_id, task_cpu(p));
}
/*
* Serialization rules:
*
@@ -615,6 +624,12 @@ EXPORT_SYMBOL(__trace_set_current_state);
* [ The astute reader will observe that it is possible for two tasks on one
* CPU to have ->on_cpu = 1 at the same time. ]
*
* p->is_blocked <- { 0, 1 }:
*
* is set by try_to_block_task() and cleared by ttwu_do_wakeup() and tracks
* if the task is blocked. Traditionally this would mirror p->on_rq, however
* due things like DELAY_DEQUEUE and PROXY_EXEC, this can diverge.
*
* task_cpu(p): is changed by set_task_cpu(), the rules are:
*
* - Don't call set_task_cpu() on a blocked task:
@@ -1203,9 +1218,13 @@ static void __resched_curr(struct rq *rq, int tif)
}
}
/*
* Calls to this function MUST be guarded by a
* tracepoint_enabled(sched_set_need_resched_tp)
*/
void __trace_set_need_resched(struct task_struct *curr, int tif)
{
trace_sched_set_need_resched_tp(curr, smp_processor_id(), tif);
trace_call__sched_set_need_resched_tp(curr, smp_processor_id(), tif);
}
EXPORT_SYMBOL_GPL(__trace_set_need_resched);
@@ -2223,8 +2242,29 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
dequeue_task(rq, p, flags);
}
static void block_task(struct rq *rq, struct task_struct *p, int flags)
static void block_task(struct rq *rq, struct task_struct *p, unsigned long task_state)
{
int flags = DEQUEUE_NOCLOCK;
p->sched_contributes_to_load =
(task_state & TASK_UNINTERRUPTIBLE) &&
!(task_state & TASK_NOLOAD) &&
!(task_state & TASK_FROZEN);
if (unlikely(is_special_task_state(task_state)))
flags |= DEQUEUE_SPECIAL;
/*
* __schedule() ttwu()
* prev_state = prev->state; if (p->on_rq && ...)
* if (prev_state) goto out;
* p->on_rq = 0; smp_acquire__after_ctrl_dep();
* p->state = TASK_WAKING
*
* Where __schedule() and ttwu() have matching control dependencies.
*
* After this, schedule() must not care about p->state any more.
*/
if (dequeue_task(rq, p, DEQUEUE_SLEEP | flags))
__block_task(rq, p);
}
@@ -3685,6 +3725,7 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
*/
static inline void ttwu_do_wakeup(struct task_struct *p)
{
p->is_blocked = 0;
WRITE_ONCE(p->__state, TASK_RUNNING);
trace_sched_wakeup(p);
}
@@ -3701,6 +3742,65 @@ void update_rq_avg_idle(struct rq *rq)
rq->idle_stamp = 0;
}
#ifdef CONFIG_SCHED_PROXY_EXEC
static void zap_balance_callbacks(struct rq *rq);
static inline void proxy_reset_donor(struct rq *rq)
{
WARN_ON_ONCE(rq->donor == rq->curr);
put_prev_set_next_task(rq, rq->donor, rq->curr);
rq_set_donor(rq, rq->curr);
zap_balance_callbacks(rq);
resched_curr(rq);
}
/*
* Checks to see if task p has been proxy-migrated to another rq
* and needs to be returned. If so, we deactivate the task here
* so that it can be properly woken up on the p->wake_cpu
* (or whichever cpu select_task_rq() picks at the bottom of
* try_to_wake_up()
*/
static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p)
{
/*
* Typically per __set_task_cpu(), task_cpu(p) == p->wake_cpu.
*
* However, proxy_set_task_cpu() is such that it preserves the
* original cpu in p->wake_cpu while migrating p for proxy reasons
* (possibly outside of the allowed p->cpus_ptr).
*
* Furthermore, migration_cpu_stop() / __migrate_swap_task(), will
* only set p->wake_cpu when !p->on_rq, and since here p->on_rq, this
* will not apply. But if it did, this check is the safe way around
* and would migrate.
*/
if (task_cpu(p) == p->wake_cpu)
return false;
scoped_guard(raw_spinlock, &p->blocked_lock) {
/* Task is waking up; clear any blocked_on relationship */
__clear_task_blocked_on(p, NULL);
/* If already current, don't need to return migrate */
if (task_current(rq, p))
return false;
/* If we're return migrating the rq->donor, switch it out for idle */
if (task_current_donor(rq, p))
proxy_reset_donor(rq);
}
block_task(rq, p, TASK_WAKING);
return true;
}
#else /* !CONFIG_SCHED_PROXY_EXEC */
static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p)
{
return false;
}
#endif /* CONFIG_SCHED_PROXY_EXEC */
static void
ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
struct rq_flags *rf)
@@ -3716,8 +3816,7 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
en_flags |= ENQUEUE_RQ_SELECTED;
if (wake_flags & WF_MIGRATED)
en_flags |= ENQUEUE_MIGRATED;
else
if (p->in_iowait) {
else if (p->in_iowait) {
delayacct_blkio_end(p);
atomic_dec(&task_rq(p)->nr_iowait);
}
@@ -3765,28 +3864,28 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
*/
static int ttwu_runnable(struct task_struct *p, int wake_flags)
{
struct rq_flags rf;
struct rq *rq;
int ret = 0;
ACQUIRE(__task_rq_lock, guard)(p);
struct rq *rq = guard.rq;
rq = __task_rq_lock(p, &rf);
if (task_on_rq_queued(p)) {
update_rq_clock(rq);
if (!task_on_rq_queued(p))
return 0;
update_rq_clock(rq);
if (p->is_blocked) {
if (p->se.sched_delayed)
enqueue_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_DELAYED);
if (!task_on_cpu(rq, p)) {
/*
* When on_rq && !on_cpu the task is preempted, see if
* it should preempt the task that is current now.
*/
wakeup_preempt(rq, p, wake_flags);
}
ttwu_do_wakeup(p);
ret = 1;
if (proxy_needs_return(rq, p))
return 0;
}
__task_rq_unlock(rq, p, &rf);
return ret;
if (!task_on_cpu(rq, p)) {
/*
* When on_rq && !on_cpu the task is preempted, see if
* it should preempt the task that is current now.
*/
wakeup_preempt(rq, p, wake_flags);
}
ttwu_do_wakeup(p);
return 1;
}
void sched_ttwu_pending(void *arg)
@@ -4173,6 +4272,9 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
* it disabling IRQs (this allows not taking ->pi_lock).
*/
WARN_ON_ONCE(p->se.sched_delayed);
WARN_ON_ONCE(p->is_blocked);
/* If p is current, we know we can run here, so clear blocked_on */
clear_task_blocked_on(p, NULL);
if (!ttwu_state_match(p, state, &success))
goto out;
@@ -4189,6 +4291,7 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
*/
scoped_guard (raw_spinlock_irqsave, &p->pi_lock) {
smp_mb__after_spinlock();
if (!ttwu_state_match(p, state, &success))
break;
@@ -4297,6 +4400,16 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
wake_flags |= WF_MIGRATED;
psi_ttwu_dequeue(p);
set_task_cpu(p, cpu);
} else if (cpu != p->wake_cpu) {
/*
* If we were proxy-migrated to cpu, then
* select_task_rq() picks cpu instead of wake_cpu
* to return to, we won't call set_task_cpu(),
* leaving a stale wake_cpu pointing to where we
* proxy-migrated from. So just fixup wake_cpu here
* if its not correct
*/
p->wake_cpu = cpu;
}
ttwu_queue(p, cpu, wake_flags);
@@ -4463,6 +4576,7 @@ static void __sched_fork(u64 clone_flags, struct task_struct *p)
/* A delayed task cannot be in clone(). */
WARN_ON_ONCE(p->se.sched_delayed);
WARN_ON_ONCE(p->is_blocked);
#ifdef CONFIG_FAIR_GROUP_SCHED
p->se.cfs_rq = NULL;
@@ -4498,6 +4612,7 @@ static void __sched_fork(u64 clone_flags, struct task_struct *p)
init_numa_balancing(clone_flags, p);
p->wake_entry.u_flags = CSD_TYPE_TTWU;
p->migration_pending = NULL;
init_sched_mm(p);
}
DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
@@ -4710,6 +4825,7 @@ int sched_fork(u64 clone_flags, struct task_struct *p)
p->policy = SCHED_NORMAL;
p->static_prio = NICE_TO_PRIO(0);
p->rt_priority = 0;
p->timer_slack_ns = p->default_timer_slack_ns;
} else if (PRIO_TO_NICE(p->static_prio) < 0)
p->static_prio = NICE_TO_PRIO(0);
@@ -5976,10 +6092,9 @@ static inline void schedule_debug(struct task_struct *prev, bool preempt)
schedstat_inc(this_rq()->sched_count);
}
static void prev_balance(struct rq *rq, struct task_struct *prev,
struct rq_flags *rf)
static void prev_balance(struct rq *rq, struct rq_flags *rf)
{
const struct sched_class *start_class = prev->sched_class;
const struct sched_class *start_class = rq->donor->sched_class;
const struct sched_class *class;
/*
@@ -5991,7 +6106,7 @@ static void prev_balance(struct rq *rq, struct task_struct *prev,
* a runnable task of @class priority or higher.
*/
for_active_class_range(class, start_class, &idle_sched_class) {
if (class->balance && class->balance(rq, prev, rf))
if (class->balance && class->balance(rq, rf))
break;
}
}
@@ -6000,7 +6115,7 @@ static void prev_balance(struct rq *rq, struct task_struct *prev,
* Pick up the highest-prio task:
*/
static inline struct task_struct *
__pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
__pick_next_task(struct rq *rq, struct rq_flags *rf)
__must_hold(__rq_lockp(rq))
{
const struct sched_class *class;
@@ -6017,40 +6132,31 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
* higher scheduling class, because otherwise those lose the
* opportunity to pull in more work from other CPUs.
*/
if (likely(!sched_class_above(prev->sched_class, &fair_sched_class) &&
if (likely(!sched_class_above(rq->donor->sched_class, &fair_sched_class) &&
rq->nr_running == rq->cfs.h_nr_queued)) {
p = pick_next_task_fair(rq, prev, rf);
p = pick_task_fair(rq, rf);
if (unlikely(p == RETRY_TASK))
goto restart;
/* Assume the next prioritized class is idle_sched_class */
if (!p) {
if (!p)
p = pick_task_idle(rq, rf);
put_prev_set_next_task(rq, prev, p);
}
put_prev_set_next_task(rq, rq->donor, p);
return p;
}
restart:
prev_balance(rq, prev, rf);
prev_balance(rq, rf);
for_each_active_class(class) {
if (class->pick_next_task) {
p = class->pick_next_task(rq, prev, rf);
if (unlikely(p == RETRY_TASK))
goto restart;
if (p)
return p;
} else {
p = class->pick_task(rq, rf);
if (unlikely(p == RETRY_TASK))
goto restart;
if (p) {
put_prev_set_next_task(rq, prev, p);
return p;
}
p = class->pick_task(rq, rf);
if (unlikely(p == RETRY_TASK))
goto restart;
if (p) {
put_prev_set_next_task(rq, rq->donor, p);
return p;
}
}
@@ -6101,7 +6207,7 @@ extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_f
static void queue_core_balance(struct rq *rq);
static struct task_struct *
pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
pick_next_task(struct rq *rq, struct rq_flags *rf)
__must_hold(__rq_lockp(rq))
{
struct task_struct *next, *p, *max;
@@ -6114,7 +6220,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
bool need_sync;
if (!sched_core_enabled(rq))
return __pick_next_task(rq, prev, rf);
return __pick_next_task(rq, rf);
cpu = cpu_of(rq);
@@ -6127,7 +6233,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
*/
rq->core_pick = NULL;
rq->core_dl_server = NULL;
return __pick_next_task(rq, prev, rf);
return __pick_next_task(rq, rf);
}
/*
@@ -6151,7 +6257,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
goto out_set_next;
}
prev_balance(rq, prev, rf);
prev_balance(rq, rf);
smt_mask = cpu_smt_mask(cpu);
need_sync = !!rq->core->core_cookie;
@@ -6333,7 +6439,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
}
out_set_next:
put_prev_set_next_task(rq, prev, next);
put_prev_set_next_task(rq, rq->donor, next);
if (rq->core->core_forceidle_count && next == rq->idle)
queue_core_balance(rq);
@@ -6556,10 +6662,10 @@ static inline void sched_core_cpu_deactivate(unsigned int cpu) {}
static inline void sched_core_cpu_dying(unsigned int cpu) {}
static struct task_struct *
pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
pick_next_task(struct rq *rq, struct rq_flags *rf)
__must_hold(__rq_lockp(rq))
{
return __pick_next_task(rq, prev, rf);
return __pick_next_task(rq, rf);
}
#endif /* !CONFIG_SCHED_CORE */
@@ -6587,16 +6693,19 @@ static bool try_to_block_task(struct rq *rq, struct task_struct *p,
unsigned long *task_state_p, bool should_block)
{
unsigned long task_state = *task_state_p;
int flags = DEQUEUE_NOCLOCK;
WARN_ON_ONCE(p->is_blocked);
if (signal_pending_state(task_state, p)) {
WRITE_ONCE(p->__state, TASK_RUNNING);
*task_state_p = TASK_RUNNING;
set_task_blocked_on_waking(p, NULL);
clear_task_blocked_on(p, NULL);
return false;
}
p->is_blocked = 1;
/*
* We check should_block after signal_pending because we
* will want to wake the task in that case. But if
@@ -6607,26 +6716,7 @@ static bool try_to_block_task(struct rq *rq, struct task_struct *p,
if (!should_block)
return false;
p->sched_contributes_to_load =
(task_state & TASK_UNINTERRUPTIBLE) &&
!(task_state & TASK_NOLOAD) &&
!(task_state & TASK_FROZEN);
if (unlikely(is_special_task_state(task_state)))
flags |= DEQUEUE_SPECIAL;
/*
* __schedule() ttwu()
* prev_state = prev->state; if (p->on_rq && ...)
* if (prev_state) goto out;
* p->on_rq = 0; smp_acquire__after_ctrl_dep();
* p->state = TASK_WAKING
*
* Where __schedule() and ttwu() have matching control dependencies.
*
* After this, schedule() must not care about p->state any more.
*/
block_task(rq, p, flags);
block_task(rq, p, task_state);
return true;
}
@@ -6649,18 +6739,18 @@ static inline void proxy_set_task_cpu(struct task_struct *p, int cpu)
static inline struct task_struct *proxy_resched_idle(struct rq *rq)
{
put_prev_set_next_task(rq, rq->donor, rq->idle);
rq->next_class = &idle_sched_class;
rq_set_donor(rq, rq->idle);
set_tsk_need_resched(rq->idle);
return rq->idle;
}
static bool proxy_deactivate(struct rq *rq, struct task_struct *donor)
static void proxy_deactivate(struct rq *rq, struct task_struct *donor)
{
unsigned long state = READ_ONCE(donor->__state);
/* Don't deactivate if the state has been changed to TASK_RUNNING */
if (state == TASK_RUNNING)
return false;
WARN_ON_ONCE(state == TASK_RUNNING);
WARN_ON_ONCE(donor->blocked_on);
/*
* Because we got donor from pick_next_task(), it is *crucial*
* that we call proxy_resched_idle() before we deactivate it.
@@ -6671,7 +6761,7 @@ static bool proxy_deactivate(struct rq *rq, struct task_struct *donor)
* need to be changed from next *before* we deactivate.
*/
proxy_resched_idle(rq);
return try_to_block_task(rq, donor, &state, true);
block_task(rq, donor, state);
}
static inline void proxy_release_rq_lock(struct rq *rq, struct rq_flags *rf)
@@ -6745,76 +6835,21 @@ static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf,
proxy_reacquire_rq_lock(rq, rf);
}
static void proxy_force_return(struct rq *rq, struct rq_flags *rf,
struct task_struct *p)
__must_hold(__rq_lockp(rq))
{
struct rq *task_rq, *target_rq = NULL;
int cpu, wake_flag = WF_TTWU;
lockdep_assert_rq_held(rq);
WARN_ON(p == rq->curr);
if (p == rq->donor)
proxy_resched_idle(rq);
proxy_release_rq_lock(rq, rf);
/*
* We drop the rq lock, and re-grab task_rq_lock to get
* the pi_lock (needed for select_task_rq) as well.
*/
scoped_guard (task_rq_lock, p) {
task_rq = scope.rq;
/*
* Since we let go of the rq lock, the task may have been
* woken or migrated to another rq before we got the
* task_rq_lock. So re-check we're on the same RQ. If
* not, the task has already been migrated and that CPU
* will handle any futher migrations.
*/
if (task_rq != rq)
break;
/*
* Similarly, if we've been dequeued, someone else will
* wake us
*/
if (!task_on_rq_queued(p))
break;
/*
* Since we should only be calling here from __schedule()
* -> find_proxy_task(), no one else should have
* assigned current out from under us. But check and warn
* if we see this, then bail.
*/
if (task_current(task_rq, p) || task_on_cpu(task_rq, p)) {
WARN_ONCE(1, "%s rq: %i current/on_cpu task %s %d on_cpu: %i\n",
__func__, cpu_of(task_rq),
p->comm, p->pid, p->on_cpu);
break;
}
update_rq_clock(task_rq);
deactivate_task(task_rq, p, DEQUEUE_NOCLOCK);
cpu = select_task_rq(p, p->wake_cpu, &wake_flag);
set_task_cpu(p, cpu);
target_rq = cpu_rq(cpu);
clear_task_blocked_on(p, NULL);
}
if (target_rq)
attach_one_task(target_rq, p);
proxy_reacquire_rq_lock(rq, rf);
}
/*
* Find runnable lock owner to proxy for mutex blocked donor
*
* Follow the blocked-on relation:
* task->blocked_on -> mutex->owner -> task...
*
* ,-> task
* | | blocked-on
* | v
* blocked_donor | mutex
* | | owner
* | v
* `-- task
*
* and set the blocked_donor relation, this latter is used by the mutex
* code to find which (blocked) task to hand-off to.
*
* Lock order:
*
@@ -6834,18 +6869,19 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
bool curr_in_chain = false;
int this_cpu = cpu_of(rq);
struct task_struct *p;
struct mutex *mutex;
int owner_cpu;
/* Follow blocked_on chain. */
for (p = donor; (mutex = p->blocked_on); p = owner) {
for (p = donor; p->is_blocked; p = owner) {
/* if its PROXY_WAKING, do return migration or run if current */
if (mutex == PROXY_WAKING) {
struct mutex *mutex = p->blocked_on;
if (!mutex) {
clear_task_blocked_on(p, mutex);
if (task_current(rq, p)) {
clear_task_blocked_on(p, PROXY_WAKING);
p->is_blocked = 0;
return p;
}
goto force_return;
goto deactivate;
}
/*
@@ -6876,17 +6912,19 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
* and return p (if it is current and safe to
* just run on this rq), or return-migrate the task.
*/
__clear_task_blocked_on(p, NULL);
if (task_current(rq, p)) {
__clear_task_blocked_on(p, NULL);
p->is_blocked = 0;
return p;
}
goto force_return;
goto deactivate;
}
if (!READ_ONCE(owner->on_rq) || owner->se.sched_delayed) {
/* XXX Don't handle blocked owners/delayed dequeue yet */
if (curr_in_chain)
return proxy_resched_idle(rq);
__clear_task_blocked_on(p, NULL);
goto deactivate;
}
@@ -6954,17 +6992,13 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
* rq, therefore holding @rq->lock is sufficient to
* guarantee its existence, as per ttwu_remote().
*/
owner->blocked_donor = p;
}
WARN_ON_ONCE(owner && !owner->on_rq);
return owner;
deactivate:
if (proxy_deactivate(rq, donor))
return NULL;
/* If deactivate fails, force return */
p = donor;
force_return:
proxy_force_return(rq, rf, p);
proxy_deactivate(rq, p);
return NULL;
migrate_task:
proxy_migrate_task(rq, rf, p, owner_cpu);
@@ -7106,13 +7140,14 @@ static void __sched notrace __schedule(int sched_mode)
pick_again:
assert_balance_callbacks_empty(rq);
next = pick_next_task(rq, rq->donor, &rf);
next = pick_next_task(rq, &rf);
rq->next_class = next->sched_class;
if (sched_proxy_exec()) {
struct task_struct *prev_donor = rq->donor;
rq_set_donor(rq, next);
if (unlikely(next->blocked_on)) {
next->blocked_donor = NULL;
if (unlikely(next->is_blocked)) {
next = find_proxy_task(rq, next, &rf);
if (!next) {
zap_balance_callbacks(rq);
@@ -7968,7 +8003,7 @@ static void __sched_dynamic_update(int mode)
break;
}
preempt_dynamic_mode = mode;
WRITE_ONCE(preempt_dynamic_mode, mode);
}
void sched_dynamic_update(int mode)
@@ -8009,12 +8044,13 @@ static void __init preempt_dynamic_init(void)
}
}
# define PREEMPT_MODEL_ACCESSOR(mode) \
bool preempt_model_##mode(void) \
{ \
WARN_ON_ONCE(preempt_dynamic_mode == preempt_dynamic_undefined); \
return preempt_dynamic_mode == preempt_dynamic_##mode; \
} \
# define PREEMPT_MODEL_ACCESSOR(mode) \
bool preempt_model_##mode(void) \
{ \
int mode = READ_ONCE(preempt_dynamic_mode); \
WARN_ON_ONCE(mode == preempt_dynamic_undefined); \
return mode == preempt_dynamic_##mode; \
} \
EXPORT_SYMBOL_GPL(preempt_model_##mode)
PREEMPT_MODEL_ACCESSOR(none);
@@ -8608,18 +8644,14 @@ static void cpuset_cpu_inactive(unsigned int cpu)
static inline void sched_smt_present_inc(int cpu)
{
#ifdef CONFIG_SCHED_SMT
if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
static_branch_inc_cpuslocked(&sched_smt_present);
#endif
}
static inline void sched_smt_present_dec(int cpu)
{
#ifdef CONFIG_SCHED_SMT
if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
static_branch_dec_cpuslocked(&sched_smt_present);
#endif
}
int sched_cpu_activate(unsigned int cpu)
@@ -8674,7 +8706,8 @@ int sched_cpu_deactivate(unsigned int cpu)
* Remove CPU from nohz.idle_cpus_mask to prevent participating in
* load balancing when not active
*/
nohz_balance_exit_idle(rq);
scoped_guard (rcu)
nohz_balance_exit_idle(rq);
set_cpu_active(cpu, false);
@@ -8698,6 +8731,8 @@ int sched_cpu_deactivate(unsigned int cpu)
*/
synchronize_rcu();
sched_domains_free_llc_id(cpu);
sched_set_rq_offline(rq, cpu);
scx_rq_deactivate(rq);
@@ -8707,9 +8742,7 @@ int sched_cpu_deactivate(unsigned int cpu)
*/
sched_smt_present_dec(cpu);
#ifdef CONFIG_SCHED_SMT
sched_core_cpu_deactivate(cpu);
#endif
if (!sched_smp_initialized)
return 0;
@@ -8877,7 +8910,7 @@ static struct kmem_cache *task_group_cache __ro_after_init;
void __init sched_init(void)
{
unsigned long ptr = 0;
unsigned long __maybe_unused ptr = 0;
int i;
/* Make sure the linker didn't screw up */
@@ -8893,36 +8926,24 @@ void __init sched_init(void)
wait_bit_init();
#ifdef CONFIG_FAIR_GROUP_SCHED
ptr += 2 * nr_cpu_ids * sizeof(void **);
#endif
#ifdef CONFIG_RT_GROUP_SCHED
ptr += 2 * nr_cpu_ids * sizeof(void **);
#endif
if (ptr) {
ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);
root_task_group.cfs_rq = &runqueues.cfs;
#ifdef CONFIG_FAIR_GROUP_SCHED
root_task_group.se = (struct sched_entity **)ptr;
ptr += nr_cpu_ids * sizeof(void **);
root_task_group.cfs_rq = (struct cfs_rq **)ptr;
ptr += nr_cpu_ids * sizeof(void **);
root_task_group.shares = ROOT_TASK_GROUP_LOAD;
init_cfs_bandwidth(&root_task_group.cfs_bandwidth, NULL);
root_task_group.shares = ROOT_TASK_GROUP_LOAD;
init_cfs_bandwidth(&root_task_group.cfs_bandwidth, NULL);
#endif /* CONFIG_FAIR_GROUP_SCHED */
#ifdef CONFIG_EXT_GROUP_SCHED
scx_tg_init(&root_task_group);
scx_tg_init(&root_task_group);
#endif /* CONFIG_EXT_GROUP_SCHED */
#ifdef CONFIG_RT_GROUP_SCHED
root_task_group.rt_se = (struct sched_rt_entity **)ptr;
ptr += nr_cpu_ids * sizeof(void **);
ptr += 2 * nr_cpu_ids * sizeof(void **);
ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);
root_task_group.rt_se = (struct sched_rt_entity **)ptr;
ptr += nr_cpu_ids * sizeof(void **);
root_task_group.rt_rq = (struct rt_rq **)ptr;
ptr += nr_cpu_ids * sizeof(void **);
root_task_group.rt_rq = (struct rt_rq **)ptr;
ptr += nr_cpu_ids * sizeof(void **);
#endif /* CONFIG_RT_GROUP_SCHED */
}
init_defrootdomain();
@@ -9031,6 +9052,11 @@ void __init sched_init(void)
rq->core_cookie = 0UL;
#endif
#ifdef CONFIG_SCHED_CACHE
raw_spin_lock_init(&rq->cpu_epoch_lock);
rq->cpu_epoch_next = jiffies;
#endif
zalloc_cpumask_var_node(&rq->scratch_mask, GFP_KERNEL, cpu_to_node(i));
}
@@ -9832,15 +9858,18 @@ static int tg_set_cfs_bandwidth(struct task_group *tg,
}
for_each_online_cpu(i) {
struct cfs_rq *cfs_rq = tg->cfs_rq[i];
struct cfs_rq *cfs_rq = tg_cfs_rq(tg, i);
struct rq *rq = cfs_rq->rq;
guard(rq_lock_irq)(rq);
cfs_rq->runtime_enabled = runtime_enabled;
cfs_rq->runtime_remaining = 1;
if (cfs_rq->throttled)
if (cfs_rq->throttled) {
update_rq_clock(rq);
unthrottle_cfs_rq(cfs_rq);
}
}
if (runtime_was_enabled && !runtime_enabled)
@@ -9981,7 +10010,7 @@ static int cpu_cfs_stat_show(struct seq_file *sf, void *v)
int i;
for_each_possible_cpu(i) {
stats = __schedstats_from_se(tg->se[i]);
stats = __schedstats_from_se(tg_se(tg, i));
ws += schedstat_val(stats->wait_sum);
}
@@ -10000,7 +10029,7 @@ static u64 throttled_time_self(struct task_group *tg)
u64 total = 0;
for_each_possible_cpu(i) {
total += READ_ONCE(tg->cfs_rq[i]->throttled_clock_self_time);
total += READ_ONCE(tg_cfs_rq(tg, i)->throttled_clock_self_time);
}
return total;

2
kernel/sched/core_sched.c
View File

@@ -136,7 +136,7 @@ int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type,
struct pid *grp;
int err = 0;
if (!static_branch_likely(&sched_smt_present))
if (!sched_smt_active())
return -ENODEV;
BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_THREAD != PIDTYPE_PID);

6
kernel/sched/cputime.c
View File

@@ -776,12 +776,6 @@ void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev,
}
stime = mul_u64_u64_div_u64(stime, rtime, stime + utime);
/*
* Because mul_u64_u64_div_u64() can approximate on some
* achitectures; enforce the constraint that: a*b/(b+c) <= a.
*/
if (unlikely(stime > rtime))
stime = rtime;
update:
/*

262
kernel/sched/deadline.c
View File

@@ -1515,8 +1515,12 @@ static void update_curr_dl_se(struct rq *rq, struct sched_dl_entity *dl_se, s64
if (unlikely(is_dl_boosted(dl_se) || !start_dl_timer(dl_se))) {
if (dl_server(dl_se)) {
replenish_dl_new_period(dl_se, rq);
start_dl_timer(dl_se);
if (dl_se->dl_defer) {
replenish_dl_new_period(dl_se, rq);
start_dl_timer(dl_se);
} else {
enqueue_dl_entity(dl_se, ENQUEUE_REPLENISH);
}
} else {
enqueue_task_dl(rq, dl_task_of(dl_se), ENQUEUE_REPLENISH);
}
@@ -1793,7 +1797,8 @@ void dl_server_start(struct sched_dl_entity *dl_se)
struct rq *rq = dl_se->rq;
dl_se->dl_defer_idle = 0;
if (!dl_server(dl_se) || dl_se->dl_server_active || !dl_se->dl_runtime)
if (!dl_server(dl_se) || dl_se->dl_server_active || !dl_se->dl_runtime ||
!dl_se->dl_bw_attached)
return;
/*
@@ -1868,6 +1873,13 @@ void sched_init_dl_servers(void)
dl_se->dl_server = 1;
dl_se->dl_defer = 1;
setup_new_dl_entity(dl_se);
/*
* No BPF scheduler is loaded at boot, so the ext_server has no
* tasks to protect. Detach its bandwidth reservation, it will
* be attached when a BPF scheduler is loaded.
*/
dl_server_detach_bw(dl_se);
#endif
}
}
@@ -1878,6 +1890,9 @@ void __dl_server_attach_root(struct sched_dl_entity *dl_se, struct rq *rq)
int cpu = cpu_of(rq);
struct dl_bw *dl_b;
if (!dl_se->dl_bw_attached)
return;
dl_b = dl_bw_of(cpu_of(rq));
guard(raw_spinlock)(&dl_b->lock);
@@ -1889,7 +1904,8 @@ void __dl_server_attach_root(struct sched_dl_entity *dl_se, struct rq *rq)
int dl_server_apply_params(struct sched_dl_entity *dl_se, u64 runtime, u64 period, bool init)
{
u64 old_bw = init ? 0 : to_ratio(dl_se->dl_period, dl_se->dl_runtime);
u64 old_bw = (init || !dl_se->dl_bw_attached) ? 0 :
to_ratio(dl_se->dl_period, dl_se->dl_runtime);
u64 new_bw = to_ratio(period, runtime);
struct rq *rq = dl_se->rq;
int cpu = cpu_of(rq);
@@ -1909,7 +1925,8 @@ int dl_server_apply_params(struct sched_dl_entity *dl_se, u64 runtime, u64 perio
if (init) {
__add_rq_bw(new_bw, &rq->dl);
__dl_add(dl_b, new_bw, cpus);
} else {
dl_se->dl_bw_attached = 1;
} else if (dl_se->dl_bw_attached) {
__dl_sub(dl_b, dl_se->dl_bw, cpus);
__dl_add(dl_b, new_bw, cpus);
@@ -1929,6 +1946,181 @@ int dl_server_apply_params(struct sched_dl_entity *dl_se, u64 runtime, u64 perio
return 0;
}
/*
* Add @dl_se's bw to the root-domain accounting.
*
* Return -EBUSY if attaching would overflow root domain capacity.
*/
static int __dl_server_attach_bw_locked(struct sched_dl_entity *dl_se,
struct dl_bw *dl_b, int cpus)
{
struct rq *rq = dl_se->rq;
unsigned long cap;
/*
* Always update @rq->dl.this_bw, but only update @dl_b->total_bw
* (and run the overflow check it gates) while this CPU is active.
*
* This mirrors dl_server_add_bw() during root-domain rebuilds, which
* only publishes bandwidth from active CPUs into @dl_b.
*/
if (cpu_active(cpu_of(rq))) {
cap = dl_bw_capacity(cpu_of(rq));
if (__dl_overflow(dl_b, cap, 0, dl_se->dl_bw))
return -EBUSY;
__dl_add(dl_b, dl_se->dl_bw, cpus);
}
__add_rq_bw(dl_se->dl_bw, &rq->dl);
dl_se->dl_bw_attached = 1;
return 0;
}
/*
* Drain @dl_se and remove its bw from the root-domain accounting.
*/
static void __dl_server_detach_bw_locked(struct sched_dl_entity *dl_se,
struct dl_bw *dl_b, int cpus)
{
struct rq *rq = dl_se->rq;
/*
* If the server is still active (on_rq), dequeue it via
* dl_server_stop(); task_non_contending() will either subtract
* @dl_bw from running_bw immediately (0-lag passed) or set
* dl_non_contending and arm the inactive_timer.
*/
if (dl_se->dl_server_active)
dl_server_stop(dl_se);
/*
* Drop @dl_se's contribution from this rq's bandwidth accounting,
* mirroring the __add_rq_bw() done at attach time.
*/
dl_rq_change_utilization(rq, dl_se, 0);
/*
* Update @dl_b only while this CPU is active, matching
* dl_server_add_bw() during root-domain rebuilds.
*
* If this CPU is inactive, its bandwidth is not currently accounted in
* @dl_b->total_bw: either attach skipped adding it, or a rebuild
* already dropped it while re-publishing active CPUs only.
*
* In that case there is nothing to subtract from @dl_b. Just clear
* @dl_se->dl_bw_attached; if the CPU becomes active again, the next
* rebuild will re-publish its bandwidth.
*/
if (cpu_active(cpu_of(rq)))
__dl_sub(dl_b, dl_se->dl_bw, cpus);
dl_se->dl_bw_attached = 0;
}
/*
* Attach @dl_se's bandwidth to the root domain's total_bw accounting.
*
* Use to dynamically register a dl_server's bandwidth reservation while
* preserving its configured @dl_runtime / @dl_period. No-op if @dl_se is
* already attached.
*
* Returns -EBUSY if attaching would overflow the root domain capacity.
*/
int dl_server_attach_bw(struct sched_dl_entity *dl_se)
{
struct rq *rq = dl_se->rq;
int cpu = cpu_of(rq);
struct dl_bw *dl_b;
int cpus, ret;
if (dl_se->dl_bw_attached)
return 0;
scoped_guard (raw_spinlock, &dl_bw_of(cpu)->lock) {
dl_b = dl_bw_of(cpu);
cpus = dl_bw_cpus(cpu);
ret = __dl_server_attach_bw_locked(dl_se, dl_b, cpus);
}
if (ret)
return ret;
/*
* The natural 0->nr_running transition that triggers dl_server_start()
* may have happened while @dl_se was still detached (e.g., between
* scx_bypass(false) and the scx_enable() re-balance loop), so kick a
* start here.
*
* dl_server_start() bails out cleanly if there's nothing to schedule or
* it's already active. Skip if @cpu is offline; the server will be
* started naturally on the first enqueue once @cpu comes back.
*/
if (cpu_online(cpu))
dl_server_start(dl_se);
return 0;
}
/*
* Detach @dl_se's bandwidth from the root domain's total_bw accounting.
*
* Use to dynamically unregister a dl_server's bandwidth reservation while
* preserving its configured @dl_runtime / @dl_period. No-op if @dl_se is
* not currently attached.
*/
void dl_server_detach_bw(struct sched_dl_entity *dl_se)
{
int cpu = cpu_of(dl_se->rq);
struct dl_bw *dl_b;
int cpus;
if (!dl_se->dl_bw_attached)
return;
dl_b = dl_bw_of(cpu);
guard(raw_spinlock)(&dl_b->lock);
cpus = dl_bw_cpus(cpu);
__dl_server_detach_bw_locked(dl_se, dl_b, cpus);
}
/*
* Atomically detach @detach_se and attach @attach_se on the same rq, holding
* @dl_b->lock across both operations so a concurrent sched_setattr() cannot
* steal the bandwidth freed by the detach before the attach can claim it.
*
* Both entities must live on the same rq (same root domain). Returns the
* result of the attach: -EBUSY if attaching @attach_se would overflow root
* domain capacity (in which case both servers end up detached).
*/
int dl_server_swap_bw(struct sched_dl_entity *detach_se,
struct sched_dl_entity *attach_se)
{
struct rq *rq = detach_se->rq;
int cpu = cpu_of(rq);
struct dl_bw *dl_b;
int cpus, ret;
WARN_ON_ONCE(attach_se->rq != rq);
scoped_guard (raw_spinlock, &dl_bw_of(cpu)->lock) {
dl_b = dl_bw_of(cpu);
cpus = dl_bw_cpus(cpu);
if (detach_se->dl_bw_attached)
__dl_server_detach_bw_locked(detach_se, dl_b, cpus);
if (attach_se->dl_bw_attached)
ret = 0;
else
ret = __dl_server_attach_bw_locked(attach_se, dl_b, cpus);
}
if (ret)
return ret;
if (cpu_online(cpu))
dl_server_start(attach_se);
return 0;
}
/*
* Update the current task's runtime statistics (provided it is still
* a -deadline task and has not been removed from the dl_rq).
@@ -2292,7 +2484,10 @@ static void dequeue_dl_entity(struct sched_dl_entity *dl_se, int flags)
static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags)
{
if (is_dl_boosted(&p->dl)) {
struct sched_dl_entity *dl_se = &p->dl;
struct dl_rq *dl_rq = &rq->dl;
if (is_dl_boosted(dl_se)) {
/*
* Because of delays in the detection of the overrun of a
* thread's runtime, it might be the case that a thread
@@ -2305,14 +2500,14 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags)
*
* In this case, the boost overrides the throttle.
*/
if (p->dl.dl_throttled) {
if (dl_se->dl_throttled) {
/*
* The replenish timer needs to be canceled. No
* problem if it fires concurrently: boosted threads
* are ignored in dl_task_timer().
*/
cancel_replenish_timer(&p->dl);
p->dl.dl_throttled = 0;
cancel_replenish_timer(dl_se);
dl_se->dl_throttled = 0;
}
} else if (!dl_prio(p->normal_prio)) {
/*
@@ -2324,7 +2519,7 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags)
* being boosted again with no means to replenish the runtime and clear
* the throttle.
*/
p->dl.dl_throttled = 0;
dl_se->dl_throttled = 0;
if (!(flags & ENQUEUE_REPLENISH))
printk_deferred_once("sched: DL de-boosted task PID %d: REPLENISH flag missing\n",
task_pid_nr(p));
@@ -2333,20 +2528,23 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags)
}
check_schedstat_required();
update_stats_wait_start_dl(dl_rq_of_se(&p->dl), &p->dl);
update_stats_wait_start_dl(dl_rq, dl_se);
if (p->on_rq == TASK_ON_RQ_MIGRATING)
if (task_on_rq_migrating(p))
flags |= ENQUEUE_MIGRATING;
enqueue_dl_entity(&p->dl, flags);
enqueue_dl_entity(dl_se, flags);
if (dl_server(&p->dl))
if (dl_server(dl_se))
return;
if (task_is_blocked(p))
return;
if (!task_current(rq, p) && !p->dl.dl_throttled && p->nr_cpus_allowed > 1)
if (dl_rq->curr == dl_se)
return;
if (!task_current(rq, p) && !dl_se->dl_throttled && p->nr_cpus_allowed > 1)
enqueue_pushable_dl_task(rq, p);
}
@@ -2354,7 +2552,7 @@ static bool dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags)
{
update_curr_dl(rq);
if (p->on_rq == TASK_ON_RQ_MIGRATING)
if (task_on_rq_migrating(p))
flags |= DEQUEUE_MIGRATING;
dequeue_dl_entity(&p->dl, flags);
@@ -2506,8 +2704,14 @@ static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p)
resched_curr(rq);
}
static int balance_dl(struct rq *rq, struct task_struct *p, struct rq_flags *rf)
static int balance_dl(struct rq *rq, struct rq_flags *rf)
{
/*
* Note, rq->donor may change during rq lock drops,
* so don't re-use prev across lock drops
*/
struct task_struct *p = rq->donor;
if (!on_dl_rq(&p->dl) && need_pull_dl_task(rq, p)) {
/*
* This is OK, because current is on_cpu, which avoids it being
@@ -2562,6 +2766,10 @@ static void start_hrtick_dl(struct rq *rq, struct sched_dl_entity *dl_se)
}
#endif /* !CONFIG_SCHED_HRTICK */
/*
* DL keeps current in tree, because ->deadline is not typically changed while
* a task is runnable.
*/
static void set_next_task_dl(struct rq *rq, struct task_struct *p, bool first)
{
struct sched_dl_entity *dl_se = &p->dl;
@@ -2574,6 +2782,9 @@ static void set_next_task_dl(struct rq *rq, struct task_struct *p, bool first)
/* You can't push away the running task */
dequeue_pushable_dl_task(rq, p);
WARN_ON_ONCE(dl_rq->curr);
dl_rq->curr = dl_se;
if (!first)
return;
@@ -2637,17 +2848,20 @@ static void put_prev_task_dl(struct rq *rq, struct task_struct *p, struct task_s
struct sched_dl_entity *dl_se = &p->dl;
struct dl_rq *dl_rq = &rq->dl;
if (on_dl_rq(&p->dl))
if (on_dl_rq(dl_se))
update_stats_wait_start_dl(dl_rq, dl_se);
update_curr_dl(rq);
update_dl_rq_load_avg(rq_clock_pelt(rq), rq, 1);
WARN_ON_ONCE(dl_rq->curr != dl_se);
dl_rq->curr = NULL;
if (task_is_blocked(p))
return;
if (on_dl_rq(&p->dl) && p->nr_cpus_allowed > 1)
if (on_dl_rq(dl_se) && p->nr_cpus_allowed > 1)
enqueue_pushable_dl_task(rq, p);
}
@@ -3236,12 +3450,12 @@ static void dl_server_add_bw(struct root_domain *rd, int cpu)
struct sched_dl_entity *dl_se;
dl_se = &cpu_rq(cpu)->fair_server;
if (dl_server(dl_se) && cpu_active(cpu))
if (dl_server(dl_se) && dl_se->dl_bw_attached && cpu_active(cpu))
__dl_add(&rd->dl_bw, dl_se->dl_bw, dl_bw_cpus(cpu));
#ifdef CONFIG_SCHED_CLASS_EXT
dl_se = &cpu_rq(cpu)->ext_server;
if (dl_server(dl_se) && cpu_active(cpu))
if (dl_server(dl_se) && dl_se->dl_bw_attached && cpu_active(cpu))
__dl_add(&rd->dl_bw, dl_se->dl_bw, dl_bw_cpus(cpu));
#endif
}
@@ -3250,11 +3464,13 @@ static u64 dl_server_read_bw(int cpu)
{
u64 dl_bw = 0;
if (cpu_rq(cpu)->fair_server.dl_server)
if (cpu_rq(cpu)->fair_server.dl_server &&
cpu_rq(cpu)->fair_server.dl_bw_attached)
dl_bw += cpu_rq(cpu)->fair_server.dl_bw;
#ifdef CONFIG_SCHED_CLASS_EXT
if (cpu_rq(cpu)->ext_server.dl_server)
if (cpu_rq(cpu)->ext_server.dl_server &&
cpu_rq(cpu)->ext_server.dl_bw_attached)
dl_bw += cpu_rq(cpu)->ext_server.dl_bw;
#endif

188
kernel/sched/debug.c
View File

@@ -136,7 +136,7 @@ sched_feat_write(struct file *filp, const char __user *ubuf,
if (cnt > 63)
cnt = 63;
if (copy_from_user(&buf, ubuf, cnt))
if (copy_from_user(buf, ubuf, cnt))
return -EFAULT;
buf[cnt] = 0;
@@ -210,6 +210,48 @@ static const struct file_operations sched_scaling_fops = {
.release = single_release,
};
#ifdef CONFIG_SCHED_CACHE
static ssize_t
sched_cache_enable_write(struct file *filp, const char __user *ubuf,
size_t cnt, loff_t *ppos)
{
bool val;
int ret;
ret = kstrtobool_from_user(ubuf, cnt, &val);
if (ret)
return ret;
sysctl_sched_cache_user = val;
sched_cache_active_set();
*ppos += cnt;
return cnt;
}
static int sched_cache_enable_show(struct seq_file *m, void *v)
{
seq_printf(m, "%d\n", sysctl_sched_cache_user);
return 0;
}
static int sched_cache_enable_open(struct inode *inode,
struct file *filp)
{
return single_open(filp, sched_cache_enable_show, NULL);
}
static const struct file_operations sched_cache_enable_fops = {
.open = sched_cache_enable_open,
.write = sched_cache_enable_write,
.read = seq_read,
.llseek = seq_lseek,
.release = single_release,
};
#endif
#ifdef CONFIG_PREEMPT_DYNAMIC
static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf,
@@ -221,7 +263,7 @@ static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf,
if (cnt > 15)
cnt = 15;
if (copy_from_user(&buf, ubuf, cnt))
if (copy_from_user(buf, ubuf, cnt))
return -EFAULT;
buf[cnt] = 0;
@@ -239,6 +281,7 @@ static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf,
static int sched_dynamic_show(struct seq_file *m, void *v)
{
int i = (IS_ENABLED(CONFIG_PREEMPT_RT) || IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY)) * 2;
int mode = READ_ONCE(preempt_dynamic_mode);
int j;
/* Count entries in NULL terminated preempt_modes */
@@ -247,10 +290,10 @@ static int sched_dynamic_show(struct seq_file *m, void *v)
j -= !IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY);
for (; i < j; i++) {
if (preempt_dynamic_mode == i)
if (mode == i)
seq_puts(m, "(");
seq_puts(m, preempt_modes[i]);
if (preempt_dynamic_mode == i)
if (mode == i)
seq_puts(m, ")");
seq_puts(m, " ");
@@ -373,6 +416,9 @@ static ssize_t sched_server_write_common(struct file *filp, const char __user *u
return -EINVAL;
}
if (!cpu_online(cpu_of(rq)))
return -EBUSY;
update_rq_clock(rq);
dl_server_stop(dl_se);
retval = dl_server_apply_params(dl_se, runtime, period, 0);
@@ -445,6 +491,8 @@ static const struct file_operations fair_server_runtime_fops = {
.release = single_release,
};
static struct dentry *debugfs_sched;
#ifdef CONFIG_SCHED_CLASS_EXT
static ssize_t
sched_ext_server_runtime_write(struct file *filp, const char __user *ubuf,
@@ -477,6 +525,59 @@ static const struct file_operations ext_server_runtime_fops = {
.llseek = seq_lseek,
.release = single_release,
};
static ssize_t
sched_ext_server_period_write(struct file *filp, const char __user *ubuf,
size_t cnt, loff_t *ppos)
{
long cpu = (long) ((struct seq_file *) filp->private_data)->private;
struct rq *rq = cpu_rq(cpu);
return sched_server_write_common(filp, ubuf, cnt, ppos, DL_PERIOD,
&rq->ext_server);
}
static int sched_ext_server_period_show(struct seq_file *m, void *v)
{
unsigned long cpu = (unsigned long) m->private;
struct rq *rq = cpu_rq(cpu);
return sched_server_show_common(m, v, DL_PERIOD, &rq->ext_server);
}
static int sched_ext_server_period_open(struct inode *inode, struct file *filp)
{
return single_open(filp, sched_ext_server_period_show, inode->i_private);
}
static const struct file_operations ext_server_period_fops = {
.open = sched_ext_server_period_open,
.write = sched_ext_server_period_write,
.read = seq_read,
.llseek = seq_lseek,
.release = single_release,
};
static void debugfs_ext_server_init(void)
{
struct dentry *d_ext;
unsigned long cpu;
d_ext = debugfs_create_dir("ext_server", debugfs_sched);
if (!d_ext)
return;
for_each_possible_cpu(cpu) {
struct dentry *d_cpu;
char buf[32];
snprintf(buf, sizeof(buf), "cpu%lu", cpu);
d_cpu = debugfs_create_dir(buf, d_ext);
debugfs_create_file("runtime", 0644, d_cpu, (void *) cpu, &ext_server_runtime_fops);
debugfs_create_file("period", 0644, d_cpu, (void *) cpu, &ext_server_period_fops);
}
}
#endif /* CONFIG_SCHED_CLASS_EXT */
static ssize_t
@@ -511,42 +612,6 @@ static const struct file_operations fair_server_period_fops = {
.release = single_release,
};
#ifdef CONFIG_SCHED_CLASS_EXT
static ssize_t
sched_ext_server_period_write(struct file *filp, const char __user *ubuf,
size_t cnt, loff_t *ppos)
{
long cpu = (long) ((struct seq_file *) filp->private_data)->private;
struct rq *rq = cpu_rq(cpu);
return sched_server_write_common(filp, ubuf, cnt, ppos, DL_PERIOD,
&rq->ext_server);
}
static int sched_ext_server_period_show(struct seq_file *m, void *v)
{
unsigned long cpu = (unsigned long) m->private;
struct rq *rq = cpu_rq(cpu);
return sched_server_show_common(m, v, DL_PERIOD, &rq->ext_server);
}
static int sched_ext_server_period_open(struct inode *inode, struct file *filp)
{
return single_open(filp, sched_ext_server_period_show, inode->i_private);
}
static const struct file_operations ext_server_period_fops = {
.open = sched_ext_server_period_open,
.write = sched_ext_server_period_write,
.read = seq_read,
.llseek = seq_lseek,
.release = single_release,
};
#endif /* CONFIG_SCHED_CLASS_EXT */
static struct dentry *debugfs_sched;
static void debugfs_fair_server_init(void)
{
struct dentry *d_fair;
@@ -568,32 +633,9 @@ static void debugfs_fair_server_init(void)
}
}
#ifdef CONFIG_SCHED_CLASS_EXT
static void debugfs_ext_server_init(void)
{
struct dentry *d_ext;
unsigned long cpu;
d_ext = debugfs_create_dir("ext_server", debugfs_sched);
if (!d_ext)
return;
for_each_possible_cpu(cpu) {
struct dentry *d_cpu;
char buf[32];
snprintf(buf, sizeof(buf), "cpu%lu", cpu);
d_cpu = debugfs_create_dir(buf, d_ext);
debugfs_create_file("runtime", 0644, d_cpu, (void *) cpu, &ext_server_runtime_fops);
debugfs_create_file("period", 0644, d_cpu, (void *) cpu, &ext_server_period_fops);
}
}
#endif /* CONFIG_SCHED_CLASS_EXT */
static __init int sched_init_debug(void)
{
struct dentry __maybe_unused *numa;
struct dentry __maybe_unused *numa, *llc;
debugfs_sched = debugfs_create_dir("sched", NULL);
@@ -626,6 +668,22 @@ static __init int sched_init_debug(void)
debugfs_create_u32("hot_threshold_ms", 0644, numa, &sysctl_numa_balancing_hot_threshold);
#endif /* CONFIG_NUMA_BALANCING */
#ifdef CONFIG_SCHED_CACHE
llc = debugfs_create_dir("llc_balancing", debugfs_sched);
debugfs_create_file("enabled", 0644, llc, NULL,
&sched_cache_enable_fops);
debugfs_create_u32("aggr_tolerance", 0644, llc,
&llc_aggr_tolerance);
debugfs_create_u32("epoch_period", 0644, llc,
&llc_epoch_period);
debugfs_create_u32("epoch_affinity_timeout", 0644, llc,
&llc_epoch_affinity_timeout);
debugfs_create_u32("overaggr_pct", 0644, llc,
&llc_overaggr_pct);
debugfs_create_u32("imb_pct", 0644, llc,
&llc_imb_pct);
#endif
debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops);
debugfs_fair_server_init();
@@ -750,7 +808,7 @@ void dirty_sched_domain_sysctl(int cpu)
#ifdef CONFIG_FAIR_GROUP_SCHED
static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group *tg)
{
struct sched_entity *se = tg->se[cpu];
struct sched_entity *se = tg_se(tg, cpu);
#define P(F) SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)F)
#define P_SCHEDSTAT(F) SEQ_printf(m, " .%-30s: %lld\n", \

71
kernel/sched/ext.c
View File

@@ -5911,6 +5911,7 @@ static void scx_root_disable(struct scx_sched *sch)
struct scx_exit_info *ei = sch->exit_info;
struct scx_task_iter sti;
struct task_struct *p;
bool was_switched_all;
int cpu;
/* guarantee forward progress and wait for descendants to be disabled */
@@ -5937,6 +5938,8 @@ static void scx_root_disable(struct scx_sched *sch)
*/
mutex_lock(&scx_enable_mutex);
was_switched_all = scx_switched_all();
static_branch_disable(&__scx_switched_all);
WRITE_ONCE(scx_switching_all, false);
@@ -5986,10 +5989,34 @@ static void scx_root_disable(struct scx_sched *sch)
/*
* Invalidate all the rq clocks to prevent getting outdated
* rq clocks from a previous scx scheduler.
*
* Also re-balance the dl_server bandwidth reservations: detach
* ext_server (no more sched_ext tasks) and reinstate fair_server if it
* was previously detached because we were running in full mode.
*
* Unlike the enable path, this runs on a recovery path that cannot
* fail, so we use dl_server_swap_bw() to atomically free ext_server's
* bandwidth and reclaim it for fair_server under the same dl_b lock.
*
* The swap can still fail with -EBUSY if someone bumped ext_server's
* runtime via debugfs between enable and disable; in that narrow case
* both servers end up detached and we just WARN.
*/
for_each_possible_cpu(cpu) {
struct rq *rq = cpu_rq(cpu);
scx_rq_clock_invalidate(rq);
scoped_guard(rq_lock_irqsave, rq) {
update_rq_clock(rq);
if (was_switched_all) {
if (WARN_ON_ONCE(dl_server_swap_bw(&rq->ext_server,
&rq->fair_server)))
pr_warn("failed to re-attach fair_server on CPU %d\n", cpu);
} else {
dl_server_detach_bw(&rq->ext_server);
}
}
}
/* no task is on scx, turn off all the switches and flush in-progress calls */
@@ -6927,6 +6954,31 @@ static void scx_root_enable_workfn(struct kthread_work *work)
if (ret)
goto err_disable;
/*
* Attach the ext_server bandwidth reservation before anything is
* committed so that we can fail the enable if the root domain cannot
* accommodate it. The matching fair_server detach is deferred to the
* tail of this function, after the switch is fully committed and can no
* longer fail.
*
* On failure, err_disable funnels into scx_root_disable() which
* detaches ext_server, so partially-attached state is cleaned up
* automatically.
*/
for_each_possible_cpu(cpu) {
struct rq *rq = cpu_rq(cpu);
scoped_guard(rq_lock_irqsave, rq) {
update_rq_clock(rq);
ret = dl_server_attach_bw(&rq->ext_server);
}
if (ret) {
pr_warn("sched_ext: failed to attach ext_server on CPU %d (%d)\n",
cpu, ret);
goto err_disable;
}
}
/*
* Once __scx_enabled is set, %current can be switched to SCX anytime.
* This can lead to stalls as some BPF schedulers (e.g. userspace
@@ -7073,6 +7125,25 @@ static void scx_root_enable_workfn(struct kthread_work *work)
if (!(ops->flags & SCX_OPS_SWITCH_PARTIAL))
static_branch_enable(&__scx_switched_all);
/*
* Detach the fair_server bandwidth reservation now that the switch
* is fully committed. In full mode (!SCX_OPS_SWITCH_PARTIAL) no
* task will ever run in the fair class, so give that bandwidth
* back to the RT class. The matching ext_server attach already
* happened earlier; this only releases bandwidth and cannot fail.
*
* In partial mode keep fair_server attached.
*/
if (scx_switched_all()) {
for_each_possible_cpu(cpu) {
struct rq *rq = cpu_rq(cpu);
guard(rq_lock_irqsave)(rq);
update_rq_clock(rq);
dl_server_detach_bw(&rq->fair_server);
}
}
pr_info("sched_ext: BPF scheduler \"%s\" enabled%s\n",
sch->ops.name, scx_switched_all() ? "" : " (partial)");
kobject_uevent(&sch->kobj, KOBJ_ADD);

6
kernel/sched/ext_idle.c
View File

@@ -79,7 +79,6 @@ static bool scx_idle_test_and_clear_cpu(int cpu)
int node = scx_cpu_node_if_enabled(cpu);
struct cpumask *idle_cpus = idle_cpumask(node)->cpu;
#ifdef CONFIG_SCHED_SMT
/*
* SMT mask should be cleared whether we can claim @cpu or not. The SMT
* cluster is not wholly idle either way. This also prevents
@@ -104,7 +103,6 @@ static bool scx_idle_test_and_clear_cpu(int cpu)
else if (cpumask_test_cpu(cpu, idle_smts))
__cpumask_clear_cpu(cpu, idle_smts);
}
#endif
return cpumask_test_and_clear_cpu(cpu, idle_cpus);
}
@@ -622,7 +620,6 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags,
goto out_unlock;
}
#ifdef CONFIG_SCHED_SMT
/*
* Use @prev_cpu's sibling if it's idle.
*/
@@ -634,7 +631,6 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags,
goto out_unlock;
}
}
#endif
/*
* Search for any idle CPU in the same LLC domain.
@@ -714,7 +710,6 @@ static void update_builtin_idle(int cpu, bool idle)
assign_cpu(cpu, idle_cpus, idle);
#ifdef CONFIG_SCHED_SMT
if (sched_smt_active()) {
const struct cpumask *smt = cpu_smt_mask(cpu);
struct cpumask *idle_smts = idle_cpumask(node)->smt;
@@ -731,7 +726,6 @@ static void update_builtin_idle(int cpu, bool idle)
cpumask_andnot(idle_smts, idle_smts, smt);
}
}
#endif
}
/*

2221
kernel/sched/fair.c
View File

File diff suppressed because it is too large Load Diff

8
kernel/sched/features.h
View File

@@ -110,8 +110,16 @@ SCHED_FEAT(WARN_DOUBLE_CLOCK, false)
* rq lock and possibly create a large contention, sending an
* IPI to that CPU and let that CPU push the RT task to where
* it should go may be a better scenario.
*
* This is best for PREEMPT_RT, but for non-RT it can cause issues
* when preemption is disabled for long periods of time. Have
* it only default enabled for PREEMPT_RT.
*/
# ifdef CONFIG_PREEMPT_RT
SCHED_FEAT(RT_PUSH_IPI, true)
# else
SCHED_FEAT(RT_PUSH_IPI, false)
# endif
#endif
SCHED_FEAT(RT_RUNTIME_SHARE, false)

2
kernel/sched/idle.c
View File

@@ -465,7 +465,7 @@ select_task_rq_idle(struct task_struct *p, int cpu, int flags)
}
static int
balance_idle(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
balance_idle(struct rq *rq, struct rq_flags *rf)
{
return WARN_ON_ONCE(1);
}

12
kernel/sched/rt.c
View File

@@ -19,9 +19,9 @@ int sysctl_sched_rt_period = 1000000;
/*
* part of the period that we allow rt tasks to run in us.
* default: 0.95s
* default: 1s
*/
int sysctl_sched_rt_runtime = 950000;
int sysctl_sched_rt_runtime = 1000000;
#ifdef CONFIG_SYSCTL
static int sysctl_sched_rr_timeslice = (MSEC_PER_SEC * RR_TIMESLICE) / HZ;
@@ -1596,8 +1596,14 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
resched_curr(rq);
}
static int balance_rt(struct rq *rq, struct task_struct *p, struct rq_flags *rf)
static int balance_rt(struct rq *rq, struct rq_flags *rf)
{
/*
* Note, rq->donor may change during rq lock drops,
* so don't re-use p across lock drops
*/
struct task_struct *p = rq->donor;
if (!on_rt_rq(&p->rt) && need_pull_rt_task(rq, p)) {
/*
* This is OK, because current is on_cpu, which avoids it being

135
kernel/sched/sched.h
View File

@@ -421,6 +421,10 @@ extern void ext_server_init(struct rq *rq);
extern void __dl_server_attach_root(struct sched_dl_entity *dl_se, struct rq *rq);
extern int dl_server_apply_params(struct sched_dl_entity *dl_se,
u64 runtime, u64 period, bool init);
extern int dl_server_attach_bw(struct sched_dl_entity *dl_se);
extern void dl_server_detach_bw(struct sched_dl_entity *dl_se);
extern int dl_server_swap_bw(struct sched_dl_entity *detach_se,
struct sched_dl_entity *attach_se);
static inline bool dl_server_active(struct sched_dl_entity *dl_se)
{
@@ -480,10 +484,8 @@ struct task_group {
#endif
#ifdef CONFIG_FAIR_GROUP_SCHED
/* schedulable entities of this group on each CPU */
struct sched_entity **se;
/* runqueue "owned" by this group on each CPU */
struct cfs_rq **cfs_rq;
struct cfs_rq __percpu *cfs_rq;
unsigned long shares;
/*
* load_avg can be heavily contended at clock tick time, so put
@@ -889,6 +891,7 @@ struct dl_rq {
bool overloaded;
struct sched_dl_entity *curr;
/*
* Tasks on this rq that can be pushed away. They are kept in
* an rb-tree, ordered by tasks' deadlines, with caching
@@ -929,7 +932,8 @@ struct dl_rq {
};
#ifdef CONFIG_FAIR_GROUP_SCHED
/* Check whether a task group is root tg */
#define is_root_task_group(tg) ((tg) == &root_task_group)
/* An entity is a task if it doesn't "own" a runqueue */
#define entity_is_task(se) (!se->my_q)
@@ -1187,6 +1191,12 @@ struct rq {
struct scx_rq scx;
struct sched_dl_entity ext_server;
#endif
#ifdef CONFIG_SCHED_CACHE
raw_spinlock_t cpu_epoch_lock ____cacheline_aligned;
u64 cpu_runtime;
unsigned long cpu_epoch;
unsigned long cpu_epoch_next;
#endif
struct sched_dl_entity fair_server;
@@ -1199,6 +1209,12 @@ struct rq {
#ifdef CONFIG_NUMA_BALANCING
unsigned int numa_migrate_on;
#endif
#ifdef CONFIG_SCHED_CACHE
unsigned int nr_pref_llc_running;
unsigned int nr_llc_running;
#endif
/*
* This is part of a global counter where only the total sum
* over all CPUs matters. A task can increase this counter on
@@ -1546,6 +1562,14 @@ extern void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags);
extern void sched_core_get(void);
extern void sched_core_put(void);
static inline bool task_has_sched_core(struct task_struct *p)
{
if (sched_core_disabled())
return false;
return !!p->core_cookie;
}
#else /* !CONFIG_SCHED_CORE: */
static inline bool sched_core_enabled(struct rq *rq)
@@ -1586,6 +1610,11 @@ static inline bool sched_group_cookie_match(struct rq *rq,
return true;
}
static inline bool task_has_sched_core(struct task_struct *p)
{
return false;
}
#endif /* !CONFIG_SCHED_CORE */
#ifdef CONFIG_RT_GROUP_SCHED
@@ -1667,21 +1696,15 @@ do { \
flags = _raw_spin_rq_lock_irqsave(rq); \
} while (0)
#ifdef CONFIG_SCHED_SMT
extern void __update_idle_core(struct rq *rq);
static inline void update_idle_core(struct rq *rq)
{
if (static_branch_unlikely(&sched_smt_present))
if (sched_smt_active())
__update_idle_core(rq);
}
#else /* !CONFIG_SCHED_SMT: */
static inline void update_idle_core(struct rq *rq) { }
#endif /* !CONFIG_SCHED_SMT */
#ifdef CONFIG_FAIR_GROUP_SCHED
static inline struct task_struct *task_of(struct sched_entity *se)
{
WARN_ON_ONCE(!entity_is_task(se));
@@ -2082,6 +2105,8 @@ init_numa_balancing(u64 clone_flags, struct task_struct *p)
#endif /* !CONFIG_NUMA_BALANCING */
int task_llc(const struct task_struct *p);
static inline void
queue_balance_callback(struct rq *rq,
struct balance_callback *head,
@@ -2171,6 +2196,7 @@ DECLARE_PER_CPU(int, sd_llc_size);
DECLARE_PER_CPU(int, sd_llc_id);
DECLARE_PER_CPU(int, sd_share_id);
DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared);
DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_balance_shared);
DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa);
DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing);
DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);
@@ -2267,6 +2293,46 @@ static inline struct task_group *task_group(struct task_struct *p)
return p->sched_task_group;
}
#ifdef CONFIG_FAIR_GROUP_SCHED
/*
* Defined here to be available before stats.h is included, since
* stats.h has dependencies on things defined later in this file.
*/
struct cfs_tg_state {
struct cfs_rq cfs_rq;
struct sched_entity se;
struct sched_statistics stats;
} __no_randomize_layout;
/* Access a specific CPU's cfs_rq from a task group */
static inline struct cfs_rq *tg_cfs_rq(struct task_group *tg, int cpu)
{
return per_cpu_ptr(tg->cfs_rq, cpu);
}
static inline struct sched_entity *tg_se(struct task_group *tg, int cpu)
{
struct cfs_tg_state *state;
if (is_root_task_group(tg))
return NULL;
state = container_of(tg_cfs_rq(tg, cpu), struct cfs_tg_state, cfs_rq);
return &state->se;
}
static inline struct sched_entity *cfs_rq_se(struct cfs_rq *cfs_rq)
{
struct cfs_tg_state *state;
if (is_root_task_group(cfs_rq->tg))
return NULL;
state = container_of(cfs_rq, struct cfs_tg_state, cfs_rq);
return &state->se;
}
#endif
/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
{
@@ -2275,10 +2341,10 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
#endif
#ifdef CONFIG_FAIR_GROUP_SCHED
set_task_rq_fair(&p->se, p->se.cfs_rq, tg->cfs_rq[cpu]);
p->se.cfs_rq = tg->cfs_rq[cpu];
p->se.parent = tg->se[cpu];
p->se.depth = tg->se[cpu] ? tg->se[cpu]->depth + 1 : 0;
set_task_rq_fair(&p->se, p->se.cfs_rq, tg_cfs_rq(tg, cpu));
p->se.cfs_rq = tg_cfs_rq(tg, cpu);
p->se.parent = tg_se(tg, cpu);
p->se.depth = p->se.parent ? p->se.parent->depth + 1 : 0;
#endif
#ifdef CONFIG_RT_GROUP_SCHED
@@ -2561,23 +2627,12 @@ struct sched_class {
/*
* schedule/pick_next_task/prev_balance: rq->lock
*/
int (*balance)(struct rq *rq, struct task_struct *prev, struct rq_flags *rf);
int (*balance)(struct rq *rq, struct rq_flags *rf);
/*
* schedule/pick_next_task: rq->lock
*/
struct task_struct *(*pick_task)(struct rq *rq, struct rq_flags *rf);
/*
* Optional! When implemented pick_next_task() should be equivalent to:
*
* next = pick_task();
* if (next) {
* put_prev_task(prev);
* set_next_task_first(next);
* }
*/
struct task_struct *(*pick_next_task)(struct rq *rq, struct task_struct *prev,
struct rq_flags *rf);
/*
* sched_change:
@@ -2801,8 +2856,7 @@ static inline bool sched_fair_runnable(struct rq *rq)
return rq->cfs.nr_queued > 0;
}
extern struct task_struct *pick_next_task_fair(struct rq *rq, struct task_struct *prev,
struct rq_flags *rf);
extern struct task_struct *pick_task_fair(struct rq *rq, struct rq_flags *rf);
extern struct task_struct *pick_task_idle(struct rq *rq, struct rq_flags *rf);
#define SCA_CHECK 0x01
@@ -4037,6 +4091,29 @@ static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct
static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */
#ifdef CONFIG_SCHED_CACHE
DECLARE_STATIC_KEY_FALSE(sched_cache_present);
DECLARE_STATIC_KEY_FALSE(sched_cache_active);
extern int sysctl_sched_cache_user;
extern unsigned int llc_aggr_tolerance;
extern unsigned int llc_epoch_period;
extern unsigned int llc_epoch_affinity_timeout;
extern unsigned int llc_imb_pct;
extern unsigned int llc_overaggr_pct;
static inline bool sched_cache_enabled(void)
{
return static_branch_unlikely(&sched_cache_active);
}
extern void sched_cache_active_set(void);
#endif
void sched_domains_free_llc_id(int cpu);
extern void init_sched_mm(struct task_struct *p);
extern u64 avg_vruntime(struct cfs_rq *cfs_rq);
extern int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se);
static inline

9
kernel/sched/stats.h
View File

@@ -89,19 +89,12 @@ static inline void rq_sched_info_depart (struct rq *rq, unsigned long long delt
#endif /* CONFIG_SCHEDSTATS */
#ifdef CONFIG_FAIR_GROUP_SCHED
struct sched_entity_stats {
struct sched_entity se;
struct sched_statistics stats;
} __no_randomize_layout;
#endif
static inline struct sched_statistics *
__schedstats_from_se(struct sched_entity *se)
{
#ifdef CONFIG_FAIR_GROUP_SCHED
if (!entity_is_task(se))
return &container_of(se, struct sched_entity_stats, se)->stats;
return &container_of(se, struct cfs_tg_state, se)->stats;
#endif
return &task_of(se)->stats;
}

2
kernel/sched/stop_task.c
View File

@@ -16,7 +16,7 @@ select_task_rq_stop(struct task_struct *p, int cpu, int flags)
}
static int
balance_stop(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
balance_stop(struct rq *rq, struct rq_flags *rf)
{
return sched_stop_runnable(rq);
}

526
kernel/sched/topology.c
View File

@@ -19,8 +19,10 @@ void sched_domains_mutex_unlock(void)
}
/* Protected by sched_domains_mutex: */
static cpumask_var_t sched_domains_llc_id_allocmask;
static cpumask_var_t sched_domains_tmpmask;
static cpumask_var_t sched_domains_tmpmask2;
int max_lid;
static int __init sched_debug_setup(char *str)
{
@@ -621,6 +623,12 @@ static void free_sched_groups(struct sched_group *sg, int free_sgc)
} while (sg != first);
}
static void free_sched_domain_shared(struct sched_domain_shared *sds)
{
if (sds && atomic_dec_and_test(&sds->ref))
kfree(sds);
}
static void destroy_sched_domain(struct sched_domain *sd)
{
/*
@@ -629,9 +637,12 @@ static void destroy_sched_domain(struct sched_domain *sd)
* dropping group/capacity references, freeing where none remain.
*/
free_sched_groups(sd->groups, 1);
free_sched_domain_shared(sd->shared);
if (sd->shared && atomic_dec_and_test(&sd->shared->ref))
kfree(sd->shared);
#ifdef CONFIG_SCHED_CACHE
/* only the bottom sd has llc_counts array */
kfree(sd->llc_counts);
#endif
kfree(sd);
}
@@ -663,9 +674,10 @@ static void destroy_sched_domains(struct sched_domain *sd)
*/
DEFINE_PER_CPU(struct sched_domain __rcu *, sd_llc);
DEFINE_PER_CPU(int, sd_llc_size);
DEFINE_PER_CPU(int, sd_llc_id);
DEFINE_PER_CPU(int, sd_llc_id) = -1;
DEFINE_PER_CPU(int, sd_share_id);
DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared);
DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_balance_shared);
DEFINE_PER_CPU(struct sched_domain __rcu *, sd_numa);
DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing);
DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);
@@ -692,7 +704,6 @@ static void update_top_cache_domain(int cpu)
rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
per_cpu(sd_llc_size, cpu) = size;
per_cpu(sd_llc_id, cpu) = id;
rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds);
sd = lowest_flag_domain(cpu, SD_CLUSTER);
@@ -713,7 +724,18 @@ static void update_top_cache_domain(int cpu)
rcu_assign_pointer(per_cpu(sd_asym_packing, cpu), sd);
sd = lowest_flag_domain(cpu, SD_ASYM_CPUCAPACITY_FULL);
/*
* The shared object is attached to sd_asym_cpucapacity only when the
* asym domain is non-overlapping (i.e., not built from SD_NUMA).
* On overlapping (NUMA) asym domains we fall back to letting the
* SD_SHARE_LLC path own the shared object, so sd->shared may be NULL
* here.
*/
if (sd && sd->shared)
sds = sd->shared;
rcu_assign_pointer(per_cpu(sd_asym_cpucapacity, cpu), sd);
rcu_assign_pointer(per_cpu(sd_balance_shared, cpu), sds);
}
/*
@@ -737,7 +759,14 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
/* Pick reference to parent->shared. */
if (parent->shared) {
WARN_ON_ONCE(tmp->shared);
/*
* It is safe to free a sd->shared that
* has not been published yet. If a
* sd->shared was published, the refcount
* will end up being non-zero and it will
* not be freed here.
*/
free_sched_domain_shared(tmp->shared);
tmp->shared = parent->shared;
parent->shared = NULL;
}
@@ -762,10 +791,20 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
if (sd && sd_degenerate(sd)) {
tmp = sd;
sd = sd->parent;
destroy_sched_domain(tmp);
if (sd) {
struct sched_group *sg = sd->groups;
#ifdef CONFIG_SCHED_CACHE
/* move buffer to parent as child is being destroyed */
sd->llc_counts = tmp->llc_counts;
sd->llc_max = tmp->llc_max;
sd->llc_bytes = tmp->llc_bytes;
/* make sure destroy_sched_domain() does not free it */
tmp->llc_counts = NULL;
tmp->llc_max = 0;
tmp->llc_bytes = 0;
#endif
/*
* sched groups hold the flags of the child sched
* domain for convenience. Clear such flags since
@@ -777,6 +816,8 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
sd->child = NULL;
}
destroy_sched_domain(tmp);
}
sched_domain_debug(sd, cpu);
@@ -804,6 +845,239 @@ enum s_alloc {
sa_none,
};
#ifdef CONFIG_SCHED_CACHE
/* hardware support for cache aware scheduling */
DEFINE_STATIC_KEY_FALSE(sched_cache_present);
/*
* Indicator of whether cache aware scheduling
* is active, used by the scheduler.
*/
DEFINE_STATIC_KEY_FALSE(sched_cache_active);
/* user wants cache aware scheduling [0 or 1] */
int sysctl_sched_cache_user = 1;
/*
* Get the effective LLC size in bytes that @cpu's bottom sched_domain
* can use. A CPU within a cpuset partition can only use a proportion
* of the physical LLC, scaled by the ratio of the partition's span
* weight to the hardware LLC sharing weight. @sd should be the
* topmost domain with SD_SHARE_LLC.
*
* Returns 0 if cacheinfo is not yet populated. This happens during
* early boot when build_sched_domains() runs before the generic
* cacheinfo framework has been initialized (cacheinfo_cpu_online()
* is a device_initcall cpuhp callback). In that case,
* cacheinfo_cpu_online() will later call sched_update_llc_bytes()
* to fill in the bottom domain's llc_bytes once the cache attributes
* are available.
*/
static unsigned long get_effective_llc_bytes(int cpu,
struct sched_domain *sd)
{
struct cacheinfo *ci;
unsigned int hw_weight;
ci = get_cpu_cacheinfo_llc(cpu);
if (!ci)
return 0;
hw_weight = cpumask_weight(&ci->shared_cpu_map);
if (!hw_weight)
return 0;
return div_u64((u64)ci->size * sd->span_weight, hw_weight);
}
static bool alloc_sd_llc(const struct cpumask *cpu_map,
struct s_data *d)
{
struct sched_domain *sd, *top_llc, *parent;
unsigned int *p;
int i;
for_each_cpu(i, cpu_map) {
sd = *per_cpu_ptr(d->sd, i);
if (!sd)
goto err;
p = kcalloc_node(max_lid + 1, sizeof(unsigned int),
GFP_KERNEL, cpu_to_node(i));
if (!p)
goto err;
top_llc = sd;
/*
* Find the topmost SD_SHARE_LLC domain.
* Not yet attached to the CPU, so per_cpu(sd_llc, i)
* can not be used.
*/
while ((parent = rcu_dereference_protected(top_llc->parent, true)) &&
(parent->flags & SD_SHARE_LLC))
top_llc = parent;
if (top_llc->flags & SD_SHARE_LLC) {
sd->llc_max = max_lid + 1;
sd->llc_counts = p;
sd->llc_bytes = get_effective_llc_bytes(i, top_llc);
} else {
/* avoid memory leak */
kfree(p);
}
}
return true;
err:
for_each_cpu(i, cpu_map) {
sd = *per_cpu_ptr(d->sd, i);
if (sd) {
kfree(sd->llc_counts);
sd->llc_counts = NULL;
sd->llc_max = 0;
sd->llc_bytes = 0;
}
}
return false;
}
/*
* Enable/disable cache aware scheduling according to
* user input and the presence of hardware support.
*/
static void _sched_cache_active_set(void)
{
lockdep_assert_cpus_held();
lockdep_assert_held(&sched_domains_mutex);
/* hardware does not support */
if (!static_branch_likely(&sched_cache_present)) {
static_branch_disable_cpuslocked(&sched_cache_active);
if (sched_debug())
pr_info("%s: cache aware scheduling not supported on this platform\n", __func__);
return;
}
/*
* user wants it or not ?
* TBD: read before writing the static key.
* It is not in the critical path, leave as-is
* for now.
*/
if (sysctl_sched_cache_user) {
static_branch_enable_cpuslocked(&sched_cache_active);
if (sched_debug())
pr_info("%s: enabling cache aware scheduling\n", __func__);
} else {
static_branch_disable_cpuslocked(&sched_cache_active);
if (sched_debug())
pr_info("%s: disabling cache aware scheduling\n", __func__);
}
}
/* used by debugfs */
void sched_cache_active_set(void)
{
cpus_read_lock();
sched_domains_mutex_lock();
_sched_cache_active_set();
sched_domains_mutex_unlock();
cpus_read_unlock();
}
/*
* Update the bottom sched_domain's llc_bytes for @cpu and all its
* LLC siblings. Called from cacheinfo_cpu_online() or
* cacheinfo_cpu_pre_down() with cpu hotplug lock held.
*
* Note: get_effective_llc_bytes() returns 0 on PowerPC.
* thus cache aware scheduling is disabled on PowerPC for
* now. PowerPC does not use the generic cacheinfo framework --
* it has its own cacheinfo with a separate struct cache hierarchy
* and does not populates the per-CPU struct cpu_cacheinfo array
* that get_cpu_cacheinfo_llc() reads.
*/
void sched_update_llc_bytes(unsigned int cpu)
{
struct sched_domain *sd, *sdp;
unsigned int i;
sched_domains_mutex_lock();
sdp = rcu_dereference_sched_domain(per_cpu(sd_llc, cpu));
if (!sdp)
goto unlock;
/*
* ci->shared_cpu_map is built incrementally as CPUs come
* online, so the first CPU in an LLC initially sees
* hw_weight == 1 and computes an inflated llc_bytes in
* get_effective_llc_bytes(). Re-evaluating every LLC
* sibling on each online event corrects this once the full
* shared_cpu_map is known.
*/
for_each_cpu(i, sched_domain_span(sdp)) {
sd = rcu_dereference_sched_domain(cpu_rq(i)->sd);
if (sd)
sd->llc_bytes = get_effective_llc_bytes(i, sdp);
}
unlock:
sched_domains_mutex_unlock();
}
static void sched_cache_set(bool has_multi_llcs)
{
/*
* TBD: check before writing to it. sched domain rebuild
* is not in the critical path, leave as-is for now.
*/
if (has_multi_llcs)
static_branch_enable_cpuslocked(&sched_cache_present);
else
static_branch_disable_cpuslocked(&sched_cache_present);
_sched_cache_active_set();
}
#else
static bool alloc_sd_llc(const struct cpumask *cpu_map,
struct s_data *d)
{
return false;
}
static inline void sched_cache_set(bool has_multi_llcs) { }
#endif
/*
* Return true if @sd belongs to an LLC group whose enclosing
* partition spans more than one LLC. @sd must be the topmost
* SD_SHARE_LLC domain.
*
* Any duplicated parent domains with the same span as @sd are
* skipped: before cpu_attach_domain() degeneration these still
* exist, after degeneration the loop is a no-op. This makes the
* helper usable both during sched domain build and against an
* already-attached domain tree.
*
* Note: For systems with a single LLC per node, cache-aware
* scheduling is still enabled when multiple nodes exist.
* However, NUMA balancing decisions take precedence over
* cache-aware scheduling. Conversely, if there is only one
* LLC per partition, cache-aware scheduling should be disabled.
*/
static bool sd_in_multi_llcs(struct sched_domain *sd)
{
struct sched_domain *sdp = sd->parent;
/* it does not make sense to aggregate to 1 CPU */
if (sd->span_weight == 1)
return false;
while (sdp && sdp->span_weight == sd->span_weight)
sdp = sdp->parent;
return !!sdp;
}
/*
* Return the canonical balance CPU for this group, this is the first CPU
* of this group that's also in the balance mask.
@@ -1310,9 +1584,7 @@ static void init_sched_groups_capacity(int cpu, struct sched_domain *sd)
cpumask_copy(mask, sched_group_span(sg));
for_each_cpu(cpu, mask) {
cores++;
#ifdef CONFIG_SCHED_SMT
cpumask_andnot(mask, mask, cpu_smt_mask(cpu));
#endif
}
sg->cores = cores;
@@ -1790,8 +2062,22 @@ const struct cpumask *tl_mc_mask(struct sched_domain_topology_level *tl, int cpu
{
return cpu_coregroup_mask(cpu);
}
/*
* Majority of architectures have LLC at MC domain level with exception
* such as powerpc. Provide a way for arch to specify where its LLC is
* if it falls in exception category
*/
# ifndef arch_llc_mask
#define arch_llc_mask(cpu) cpu_coregroup_mask(cpu)
# endif
#else
#define arch_llc_mask(cpu) cpumask_of(cpu)
#endif
#define llc_mask(cpu) arch_llc_mask(cpu)
const struct cpumask *tl_pkg_mask(struct sched_domain_topology_level *tl, int cpu)
{
return cpu_node_mask(cpu);
@@ -2650,14 +2936,153 @@ static void adjust_numa_imbalance(struct sched_domain *sd_llc)
}
}
static void
init_sched_domain_shared(struct s_data *d, struct sched_domain *sd, int flags)
{
struct sched_domain_shared *sds = NULL;
int cpu;
/*
* Multiple domains can try to claim a shared object like
* SD_ASYM_CPUCAPACITY and SD_SHARE_LLC which can alias to
* same cpumask_first(sched_domain_span(sd)) CPU and can
* cause "nr_idle_scan" to be populated incorrectly during
* load balancing.
*
* Find the first CPU in sched_domain_span(sd) with an
* unclaimed domain (!alloc_flags) or where the alloc_flag
* matches the requested flag (SD_* flag)
*
* If the domain only has single CPU, allow temporary overlap
* in allocation since the domains will be degenerated later.
*/
for_each_cpu(cpu, sched_domain_span(sd)) {
sds = *per_cpu_ptr(d->sds, cpu);
if (!sds->alloc_flags ||
sd->span_weight == 1 ||
sds->alloc_flags == flags) {
sds->alloc_flags = flags;
sd->shared = sds;
break;
}
}
/*
* Use the sd_shared corresponding to the last
* CPU in the span if none are avaialable.
*/
if (WARN_ON_ONCE(!sd->shared))
sd->shared = sds;
/*
* nr_busy_cpus is consumed only by the NOHZ kick path via
* sd_balance_shared; on the asym-capacity path it is initialized but
* never read.
*/
atomic_set(&sd->shared->nr_busy_cpus, sd->span_weight);
atomic_inc(&sd->shared->ref);
}
/*
* For asymmetric CPU capacity, attach sched_domain_shared on the innermost
* SD_ASYM_CPUCAPACITY_FULL ancestor of @cpu's base domain when that ancestor is
* not an overlapping NUMA-built domain (then LLC should claim shared).
*
* A CPU may lack any FULL ancestor (e.g., exclusive cpuset symmetric island),
* then LLC must claim shared instead.
*
* Note: SD_ASYM_CPUCAPACITY_FULL is only set when all CPU capacity values
* are present in the domain span, so the asym domain we attach to cannot
* degenerate into a single-capacity group. The relevant edge cases are instead
* covered by the caveats above.
*
* Return true if this CPU's asym path claimed sd->shared, false otherwise.
*/
static bool claim_asym_sched_domain_shared(struct s_data *d, int cpu)
{
struct sched_domain *sd = *per_cpu_ptr(d->sd, cpu);
struct sched_domain *sd_asym;
if (!sd)
return false;
sd_asym = sd;
while (sd_asym && !(sd_asym->flags & SD_ASYM_CPUCAPACITY_FULL))
sd_asym = sd_asym->parent;
if (!sd_asym || (sd_asym->flags & SD_NUMA))
return false;
init_sched_domain_shared(d, sd_asym, SD_ASYM_CPUCAPACITY);
return true;
}
static int __sched_domains_alloc_llc_id(void)
{
int lid, max;
lockdep_assert_held(&sched_domains_mutex);
lid = cpumask_first_zero(sched_domains_llc_id_allocmask);
/*
* llc_id space should never grow larger than the
* possible number of CPUs in the system.
*/
if (lid >= nr_cpu_ids)
return -1;
__cpumask_set_cpu(lid, sched_domains_llc_id_allocmask);
max = cpumask_last(sched_domains_llc_id_allocmask);
if (max > max_lid)
max_lid = max;
return lid;
}
static void __sched_domains_free_llc_id(int cpu)
{
int i, lid, max;
lockdep_assert_held(&sched_domains_mutex);
lid = per_cpu(sd_llc_id, cpu);
if (lid == -1 || lid >= nr_cpu_ids)
return;
per_cpu(sd_llc_id, cpu) = -1;
for_each_cpu(i, llc_mask(cpu)) {
/* An online CPU owns the llc_id. */
if (per_cpu(sd_llc_id, i) == lid)
return;
}
__cpumask_clear_cpu(lid, sched_domains_llc_id_allocmask);
max = cpumask_last(sched_domains_llc_id_allocmask);
/* shrink max lid to save memory */
if (max < max_lid)
max_lid = max;
}
void sched_domains_free_llc_id(int cpu)
{
sched_domains_mutex_lock();
__sched_domains_free_llc_id(cpu);
sched_domains_mutex_unlock();
}
/*
* Build sched domains for a given set of CPUs and attach the sched domains
* to the individual CPUs
*/
static int
build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *attr)
build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *attr,
bool *multi_llcs)
{
enum s_alloc alloc_state = sa_none;
bool has_multi_llcs = false;
struct sched_domain *sd;
struct s_data d;
struct rq *rq = NULL;
@@ -2675,6 +3100,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
/* Set up domains for CPUs specified by the cpu_map: */
for_each_cpu(i, cpu_map) {
struct sched_domain_topology_level *tl;
int lid;
sd = NULL;
for_each_sd_topology(tl) {
@@ -2688,6 +3114,29 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
if (cpumask_equal(cpu_map, sched_domain_span(sd)))
break;
}
lid = per_cpu(sd_llc_id, i);
if (lid == -1) {
/* try to reuse the llc_id of its siblings */
for (int j = cpumask_first(llc_mask(i));
j < nr_cpu_ids;
j = cpumask_next(j, llc_mask(i))) {
if (i == j)
continue;
lid = per_cpu(sd_llc_id, j);
if (lid != -1) {
per_cpu(sd_llc_id, i) = lid;
break;
}
}
/* a new LLC is detected */
if (lid == -1)
per_cpu(sd_llc_id, i) = __sched_domains_alloc_llc_id();
}
}
if (WARN_ON(!topology_span_sane(cpu_map)))
@@ -2712,23 +3161,27 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
if (!sd)
continue;
if (has_asym)
claim_asym_sched_domain_shared(&d, i);
/* First, find the topmost SD_SHARE_LLC domain */
while (sd->parent && (sd->parent->flags & SD_SHARE_LLC))
sd = sd->parent;
if (sd->flags & SD_SHARE_LLC) {
int sd_id = cpumask_first(sched_domain_span(sd));
sd->shared = *per_cpu_ptr(d.sds, sd_id);
atomic_set(&sd->shared->nr_busy_cpus, sd->span_weight);
atomic_inc(&sd->shared->ref);
init_sched_domain_shared(&d, sd, SD_SHARE_LLC);
/*
* In presence of higher domains, adjust the
* NUMA imbalance stats for the hierarchy.
*/
if (IS_ENABLED(CONFIG_NUMA) && sd->parent)
adjust_numa_imbalance(sd);
if (sd->parent) {
if (IS_ENABLED(CONFIG_NUMA))
adjust_numa_imbalance(sd);
if (sd_in_multi_llcs(sd))
has_multi_llcs = true;
}
}
}
@@ -2743,6 +3196,8 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
init_sched_groups_capacity(i, sd);
}
alloc_sd_llc(cpu_map, &d);
/* Attach the domains */
rcu_read_lock();
for_each_cpu(i, cpu_map) {
@@ -2767,6 +3222,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
ret = 0;
error:
*multi_llcs = has_multi_llcs;
__free_domain_allocs(&d, alloc_state, cpu_map);
return ret;
@@ -2829,8 +3285,10 @@ void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
*/
int __init sched_init_domains(const struct cpumask *cpu_map)
{
bool multi_llcs;
int err;
zalloc_cpumask_var(&sched_domains_llc_id_allocmask, GFP_KERNEL);
zalloc_cpumask_var(&sched_domains_tmpmask, GFP_KERNEL);
zalloc_cpumask_var(&sched_domains_tmpmask2, GFP_KERNEL);
zalloc_cpumask_var(&fallback_doms, GFP_KERNEL);
@@ -2842,7 +3300,9 @@ int __init sched_init_domains(const struct cpumask *cpu_map)
if (!doms_cur)
doms_cur = &fallback_doms;
cpumask_and(doms_cur[0], cpu_map, housekeeping_cpumask(HK_TYPE_DOMAIN));
err = build_sched_domains(doms_cur[0], NULL);
err = build_sched_domains(doms_cur[0], NULL, &multi_llcs);
if (!err)
sched_cache_set(multi_llcs);
return err;
}
@@ -2915,6 +3375,7 @@ static void partition_sched_domains_locked(int ndoms_new, cpumask_var_t doms_new
struct sched_domain_attr *dattr_new)
{
bool __maybe_unused has_eas = false;
bool has_multi_llcs = false, multi_llcs;
int i, j, n;
int new_topology;
@@ -2964,14 +3425,41 @@ static void partition_sched_domains_locked(int ndoms_new, cpumask_var_t doms_new
for (i = 0; i < ndoms_new; i++) {
for (j = 0; j < n && !new_topology; j++) {
if (cpumask_equal(doms_new[i], doms_cur[j]) &&
dattrs_equal(dattr_new, i, dattr_cur, j))
dattrs_equal(dattr_new, i, dattr_cur, j)) {
/*
* Reused partition has to be taken care
* of here, because there could be a corner
* case that if the reused partition is skipped
* and only new partition is considered, an
* incorrect has_multi_llcs would be set. For
* example:
* If the only multi-LLC partition is reused
* and a new single-LLC partition is built,
* sched_cache_set(false) disables cache-aware
* scheduling globally despite the reused
* multi-LLC partition still being active.
*/
struct sched_domain *sd;
int cpu = cpumask_first(doms_cur[j]);
guard(rcu)();
sd = rcu_dereference(cpu_rq(cpu)->sd);
while (sd && sd->parent && (sd->parent->flags & SD_SHARE_LLC))
sd = sd->parent;
if (sd && (sd->flags & SD_SHARE_LLC) && sd->parent &&
sd_in_multi_llcs(sd))
has_multi_llcs = true;
goto match2;
}
}
/* No match - add a new doms_new */
build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL);
build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL,
&multi_llcs);
has_multi_llcs |= multi_llcs;
match2:
;
}
sched_cache_set(has_multi_llcs);
#if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL)
/* Build perf domains: */

5
kernel/stop_machine.c
View File

@@ -633,6 +633,11 @@ int stop_machine(cpu_stop_fn_t fn, void *data, const struct cpumask *cpus)
EXPORT_SYMBOL_GPL(stop_machine);
#ifdef CONFIG_SCHED_SMT
/*
* INTEL_IFS is the only user of this API. That selftest can
* only be compiled if SMP=y. On x86 it selects SCHED_SMT.
* Keep the ifdefs for now.
*/
int stop_core_cpuslocked(unsigned int cpu, cpu_stop_fn_t fn, void *data)
{
const struct cpumask *smt_mask = cpu_smt_mask(cpu);

7
kernel/trace/ring_buffer.c
View File

@@ -3791,13 +3791,6 @@ rb_add_time_stamp(struct ring_buffer_per_cpu *cpu_buffer,
return skip_time_extend(event);
}
#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
static inline bool sched_clock_stable(void)
{
return true;
}
#endif
static void
rb_check_timestamp(struct ring_buffer_per_cpu *cpu_buffer,
struct rb_event_info *info)

4
kernel/workqueue.c
View File

@@ -8212,11 +8212,7 @@ static bool __init cpus_dont_share(int cpu0, int cpu1)
static bool __init cpus_share_smt(int cpu0, int cpu1)
{
#ifdef CONFIG_SCHED_SMT
return cpumask_test_cpu(cpu0, cpu_smt_mask(cpu1));
#else
return false;
#endif
}
static bool __init cpus_share_numa(int cpu0, int cpu1)

201
tools/testing/selftests/sched_ext/total_bw.c
View File

@@ -100,6 +100,98 @@ static int read_total_bw_values(long *bw_values, int max_cpus)
return cpu_count;
}
/*
* Read a per-CPU dl_server param (runtime or period) from debugfs.
* Returns the value in nanoseconds, or -1 on failure.
*/
static long read_server_param(const char *server, const char *param, int cpu)
{
char path[128];
long value = -1;
FILE *fp;
snprintf(path, sizeof(path),
"/sys/kernel/debug/sched/%s_server/cpu%d/%s",
server, cpu, param);
fp = fopen(path, "r");
if (!fp)
return -1;
if (fscanf(fp, "%ld", &value) != 1)
value = -1;
fclose(fp);
return value;
}
/*
* Write a per-CPU dl_server param to debugfs. Returns 0 on success.
*/
static int write_server_param(const char *server, const char *param,
int cpu, long value)
{
char path[128];
FILE *fp;
int ret = 0;
snprintf(path, sizeof(path),
"/sys/kernel/debug/sched/%s_server/cpu%d/%s",
server, cpu, param);
fp = fopen(path, "w");
if (!fp)
return -1;
if (fprintf(fp, "%ld", value) < 0)
ret = -1;
if (fclose(fp) != 0)
ret = -1;
return ret;
}
static int read_fair_runtime_all(int nr_cpus, long *runtimes)
{
int i;
for (i = 0; i < nr_cpus; i++) {
runtimes[i] = read_server_param("fair", "runtime", i);
if (runtimes[i] <= 0)
return -1;
}
return 0;
}
static int write_fair_runtime_all(int nr_cpus, long value)
{
int i;
for (i = 0; i < nr_cpus; i++) {
if (write_server_param("fair", "runtime", i, value) < 0) {
SCX_ERR("Failed to write fair_server runtime on CPU %d", i);
return -1;
}
}
return 0;
}
/*
* Restore per-CPU fair_server runtimes.
*/
static int restore_fair_runtime_all(int nr_cpus, const long *runtimes)
{
int ret = 0;
int i;
for (i = 0; i < nr_cpus; i++) {
if (write_server_param("fair", "runtime", i, runtimes[i]) < 0) {
SCX_ERR("Failed to restore fair_server runtime on CPU %d", i);
ret = -1;
}
}
return ret;
}
static bool verify_total_bw_consistency(long *bw_values, int count)
{
int i;
@@ -217,6 +309,9 @@ static enum scx_test_status run(void *ctx)
struct bpf_link *link;
long loaded_bw[MAX_CPUS];
long unloaded_bw[MAX_CPUS];
long doubled_bw[MAX_CPUS];
long original_runtime[MAX_CPUS], doubled_runtime;
enum scx_test_status ret;
int i;
/* Test scenario 2: BPF program loaded */
@@ -257,7 +352,111 @@ static enum scx_test_status run(void *ctx)
}
fprintf(stderr, "All total_bw values are consistent across all scenarios\n");
return SCX_TEST_PASS;
/*
* Validate auto-register/unregister of dl_server bandwidth reservations.
*
* Doubling fair_server's runtime doubles its bw contribution. With a
* full-mode BPF scheduler (minimal_ops), the kernel should detach
* fair_server and attach ext_server, dropping total_bw back to its
* pre-customization (default ext_server-only) value. On unload, the
* fair_server reservation should come back with its customized runtime
* preserved, so total_bw doubles again.
*/
if (read_fair_runtime_all(test_ctx->nr_cpus, original_runtime) < 0) {
fprintf(stderr, "Skipping attach/detach validation: debugfs not accessible\n");
return SCX_TEST_PASS;
}
doubled_runtime = original_runtime[0] * 2;
fprintf(stderr,
"Setting fair_server runtime to %ld ns on all CPUs (orig %ld)\n",
doubled_runtime, original_runtime[0]);
if (write_fair_runtime_all(test_ctx->nr_cpus, doubled_runtime) < 0) {
ret = SCX_TEST_FAIL;
goto restore;
}
if (fetch_verify_total_bw(doubled_bw, test_ctx->nr_cpus) < 0) {
SCX_ERR("Failed to get stable values after doubling fair runtime");
ret = SCX_TEST_FAIL;
goto restore;
}
/*
* After doubling the runtime, fair_server's bw contribution must grow.
* We don't assert exactly 2x, because the kernel's to_ratio() truncates
* the value, so 2 * to_ratio(period, runtime) and
* to_ratio(period, 2 * runtime) can differ.
*/
for (i = 0; i < test_ctx->nr_cpus; i++) {
if (doubled_bw[i] <= test_ctx->baseline_bw[i]) {
SCX_ERR("CPU%d: fair did not increase total_bw (baseline=%ld, doubled=%ld)",
i, test_ctx->baseline_bw[i], doubled_bw[i]);
ret = SCX_TEST_FAIL;
goto restore;
}
}
link = bpf_map__attach_struct_ops(test_ctx->skel->maps.minimal_ops);
if (!link) {
SCX_ERR("Failed to attach scheduler for detach test");
ret = SCX_TEST_FAIL;
goto restore;
}
if (fetch_verify_total_bw(loaded_bw, test_ctx->nr_cpus) < 0) {
SCX_ERR("Failed to get stable values with BPF loaded (detach test)");
bpf_link__destroy(link);
ret = SCX_TEST_FAIL;
goto restore;
}
/*
* In full mode the customized fair_server is detached and ext_server is
* attached at its default runtime, total_bw must match baseline.
*/
for (i = 0; i < test_ctx->nr_cpus; i++) {
if (loaded_bw[i] != test_ctx->baseline_bw[i]) {
SCX_ERR("CPU%d: expected bw %ld (fair detached, ext default), got %ld",
i, test_ctx->baseline_bw[i], loaded_bw[i]);
bpf_link__destroy(link);
ret = SCX_TEST_FAIL;
goto restore;
}
}
bpf_link__destroy(link);
if (fetch_verify_total_bw(unloaded_bw, test_ctx->nr_cpus) < 0) {
SCX_ERR("Failed to get stable values after BPF unload (detach test)");
ret = SCX_TEST_FAIL;
goto restore;
}
/*
* After unload, fair_server is re-attached with its preserved 2x
* runtime, so total_bw should return to the doubled value.
*/
for (i = 0; i < test_ctx->nr_cpus; i++) {
if (unloaded_bw[i] != doubled_bw[i]) {
SCX_ERR("CPU%d: BPF unloaded: expected %ld (fair restored at 2x), got %ld",
i, doubled_bw[i], unloaded_bw[i]);
ret = SCX_TEST_FAIL;
goto restore;
}
}
fprintf(stderr,
"dl_server attach/detach with customized fair runtime verified\n");
ret = SCX_TEST_PASS;
restore:
if (restore_fair_runtime_all(test_ctx->nr_cpus, original_runtime) < 0)
SCX_ERR("Failed to fully restore per-CPU fair_server runtimes");
return ret;
}
static void cleanup(void *ctx)