mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-31 10:31:33 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'sched_ext-for-7.3' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext

Browse Source 
Pull sched_ext updates from Tejun Heo:
 "Most of this cycle completes the enqueue-path support for hierarchical
  sub-scheduling, which makes sub-scheduler support feature complete: a
  root BPF scheduler can now hand a cgroup subtree over to a nested
  sub-scheduler together with revocable CPU grants, and the
  sub-scheduler owns all scheduling decisions for its tasks on those
  CPUs.

  Development volume was high and a number of changes plugging holes in
  the new support landed late in the cycle. Also included are core
  scheduling fixes that were completed too late for the v7.2 release and
  are routed through this pull request.

  Sub-scheduler CPU delegation:

   - Parent schedulers now grant and revoke per-CPU capabilities
     (enqueueing, preemption, CPU frequency control) on their children,
     enforced on every path a scheduler can reach a CPU through.
     Previously only dispatching could be delegated; this lets
     sub-schedulers fully schedule their CPUs.

   - Rescue execution: a task whose scheduler doesn't have access to the
     CPUs the task needs to run on starved until the watchdog ejected
     the whole scheduler. The kernel now runs such tasks directly on a
     small bandwidth budget, turning a scheduler-killing failure into
     bounded degradation.

   - Cgroup integration: tasks migrating across a sub-scheduler boundary
     weren't re-homed to the new owner, causing wrong-scheduler
     scheduling and a use-after-free. Sub-schedulers now take over their
     cgroup subtree and receive its cgroup callbacks.

   - Arena objects now cross the kernel/BPF boundary as typed pointer
     arguments, translated transparently by the BPF tree's new arena
     argument support, replacing untyped arguments with manual
     translation.

   - scx_qmap now demonstrates full hierarchical sub-scheduling.

  Other fixes and updates:

   - Robustness improvements: the abort path is now NMI-safe, fixing
     deadlocks when errors are raised from NMI context and making
     hardlockup recovery direct. Reenqueue loops that could monopolize a
     CPU ahead of the watchdog now eject the offending scheduler, and
     stalls are blamed on the scheduler actually responsible.

   - Hardening: BPF-writable arena memory is validated before kernel
     use, and task slice and vtime writes got explicit synchronization
     rules, closing corruption vectors open to buggy or malicious
     schedulers.

   - Core scheduling: sched_ext dispatching can drop the rq lock inside
     the core-wide pick, which let interleaving selections corrupt each
     other's state and hard-hang the machine. The selection now restarts
     when the lock was released. The task ordering callback was also
     invoked with its arguments swapped, and the default ordering is
     updated to work across sub-scheduler boundaries. The fixes are
     marked for stable.

   - Other fixes headed for stable: a task init leak on fork failure
     during enable, tooling compat macros that silently failed to detect
     newer kernels, and a crash on reenqueueing against a destroyed
     dispatch queue.

   - Tooling: scx_pair moves off deprecated callbacks, and the
     deprecated scx_bpf_cpu_rq() kfunc is removed"

* tag 'sched_ext-for-7.3' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext: (144 commits)
  sched_ext: Drop the dead SCX_DEQ_CORE_SCHED_EXEC test in dequeue_task_scx()
  sched_ext: Make core-sched task ordering hierarchy-aware
  sched_ext: Use runnable_at for the default core-sched task ordering
  sched_ext: Fix inverted ops.core_sched_before() invocation
  sched_ext: Move the config-off sub-cap kfunc stubs into sub.c
  sched_ext: Rename balance-era identifiers to dispatch terms
  sched_ext: Drop the stale keep_prev fixup in dispatch_pick()
  sched_ext: Keep kick_sync waiting on the rq's own CPU
  sched_ext: Make SCHED_CLASS_EXT select GENERIC_ALLOCATOR
  sched_ext/scx_flatcg: Fix cvtime true-up on slice expiry
  sched_ext: Don't BUG_ON a destroyed DSQ in process_deferred_reenq_users
  sched_ext: Fix scx_bpf_dsq_move_to_local___v2 compat detection
  sched_ext: Make scx_bpf_events() read the calling scheduler's counters
  sched_ext: Drop unlocked scx_rq_clock_invalidate() from scx_root_disable()
  selftests/sched_ext: Fix flaky ddsp failure tests on busy systems
  selftests/sched_ext: Make numa idle validation race-free
  sched_ext: Fix scx_bpf_dsq_reenq___compat kfunc extern prototype
  sched_ext/scx_flatcg: expire cached hweights on weight changes
  sched_ext: Fix exit_task leak on fork failure during enable
  sched_ext: fix stale references in doc comments
  ...
This commit is contained in:
Linus Torvalds
2026-08-20 11:01:37 -07:00
parent 40d8c81577 fab183d632
commit 11260c335e
49 changed files with 8819 additions and 2602 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
Documentation/scheduler/sched-ext.rst
View File

@@ -93,20 +93,20 @@ scheduler has been loaded):
# cat /sys/kernel/sched_ext/enable_seq
1
Each running scheduler also exposes a per-scheduler ``events`` file under
``/sys/kernel/sched_ext/<scheduler-name>/events`` that tracks diagnostic
counters. Each counter occupies one ``name value`` line:
Each running scheduler exposes an ``events`` file under its sysfs kobject
(``/sys/kernel/sched_ext/root/events`` for the root scheduler) that tracks
diagnostic counters. Each counter occupies one ``name value`` line:
.. code-block:: none
# cat /sys/kernel/sched_ext/simple/events
# cat /sys/kernel/sched_ext/root/events
SCX_EV_SELECT_CPU_FALLBACK 0
SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE 0
SCX_EV_DISPATCH_KEEP_LAST 123
SCX_EV_ENQ_SKIP_EXITING 0
SCX_EV_ENQ_SKIP_MIGRATION_DISABLED 0
SCX_EV_REENQ_IMMED 0
SCX_EV_REENQ_LOCAL_REPEAT 0
SCX_EV_REENQ_REPEAT 0
SCX_EV_REFILL_SLICE_DFL 456789
SCX_EV_BYPASS_DURATION 0
SCX_EV_BYPASS_DISPATCH 0
@@ -129,9 +129,9 @@ The counters are described in ``kernel/sched/ext/internal.h``; briefly:
``SCX_OPS_ENQ_MIGRATION_DISABLED`` is not set).
* ``SCX_EV_REENQ_IMMED``: a task dispatched with ``SCX_ENQ_IMMED`` was
re-enqueued because the target CPU was not available for immediate execution.
* ``SCX_EV_REENQ_LOCAL_REPEAT``: a reenqueue of the local DSQ triggered
another reenqueue; recurring counts indicate incorrect ``SCX_ENQ_REENQ``
handling in the BPF scheduler.
* ``SCX_EV_REENQ_REPEAT``: a reenqueue led to another reenqueue without the
task running in between; recurring counts indicate that the BPF scheduler
keeps re-deciding placements it can't honor.
* ``SCX_EV_REFILL_SLICE_DFL``: a task's time slice was refilled with the
default value (``SCX_SLICE_DFL``).
* ``SCX_EV_BYPASS_DURATION``: total nanoseconds spent in bypass mode.
@@ -153,6 +153,7 @@ detailed information:
switching_all : 1
switched_all : 1
enable_state : enabled (2)
aborting : False
bypass_depth : 0
nr_rejected : 0
enable_seq : 1

26
include/linux/cgroup.h
View File

@@ -82,12 +82,38 @@ enum cgroup_lifetime_events {
CGROUP_LIFETIME_OFFLINE,
};
/*
* Events on cgroup_task_notifier, data is struct cgroup_task_migrate_ctx.
* MIGRATING fires per task before the migration commits and an error return
* from the chain fails the migration, in which case tasks that were already
* notified receive MIGRATE_CANCELED. MIGRATED fires per task after the
* migration is committed and can't fail. Only migrations that change a task's
* dfl cgroup are reported.
*/
enum cgroup_task_events {
CGROUP_TASK_MIGRATING,
CGROUP_TASK_MIGRATED,
CGROUP_TASK_MIGRATE_CANCELED,
};
/*
* @src_dcgrp and @dst_dcgrp are @task's dfl cgroups before and after the
* migration. @src_dcgrp is NULL for CGROUP_TASK_MIGRATED as per-task sources
* are not tracked past the commit point.
*/
struct cgroup_task_migrate_ctx {
struct task_struct *task;
struct cgroup *src_dcgrp;
struct cgroup *dst_dcgrp;
};
extern struct file_system_type cgroup_fs_type;
extern struct cgroup_root cgrp_dfl_root;
extern struct css_set init_css_set;
extern struct mutex cgroup_mutex;
extern spinlock_t css_set_lock;
extern struct blocking_notifier_head cgroup_lifetime_notifier;
extern struct blocking_notifier_head cgroup_task_notifier;
#define SUBSYS(_x) extern struct cgroup_subsys _x ## _cgrp_subsys;
#include <linux/cgroup_subsys.h>

61
include/linux/sched/ext.h
View File

@@ -58,6 +58,8 @@ enum scx_dsq_id_flags {
SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1,
SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2,
SCX_DSQ_BYPASS = SCX_DSQ_FLAG_BUILTIN | 3,
SCX_DSQ_REJECT = SCX_DSQ_FLAG_BUILTIN | 4, /* internal - see find_dsq_for_dispatch() */
SCX_DSQ_RESCUE = SCX_DSQ_FLAG_BUILTIN | 5, /* internal - see find_dsq_for_dispatch() */
SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON,
SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU,
};
@@ -101,6 +103,7 @@ enum scx_ent_flags {
SCX_TASK_DEQD_FOR_SLEEP = 1 << 3, /* last dequeue was for SLEEP */
SCX_TASK_SUB_INIT = 1 << 4, /* task being initialized for a sub sched */
SCX_TASK_IMMED = 1 << 5, /* task is on local DSQ with %SCX_ENQ_IMMED */
SCX_TASK_PROTECTED = 1 << 6, /* slice and DSQ head position protected */
/*
* Bits 8 to 10 are used to carry task state:
@@ -124,7 +127,7 @@ enum scx_ent_flags {
SCX_TASK_DEAD = 5 << SCX_TASK_STATE_SHIFT,
/*
* Bits 12 and 13 are used to carry reenqueue reason. In addition to
* Bits 12 to 14 are used to carry reenqueue reason. In addition to
* %SCX_ENQ_REENQ flag, ops.enqueue() can also test for
* %SCX_TASK_REENQ_REASON_NONE to distinguish reenqueues.
*
@@ -132,15 +135,17 @@ enum scx_ent_flags {
* KFUNC reenqueued by scx_bpf_dsq_reenq() and friends
* IMMED reenqueued due to failed ENQ_IMMED
* PREEMPTED preempted while running
* CAP sub-sched cap miss, see p->scx.reenq_reason_*
*/
SCX_TASK_REENQ_REASON_SHIFT = 12,
SCX_TASK_REENQ_REASON_BITS = 2,
SCX_TASK_REENQ_REASON_BITS = 3,
SCX_TASK_REENQ_REASON_MASK = ((1 << SCX_TASK_REENQ_REASON_BITS) - 1) << SCX_TASK_REENQ_REASON_SHIFT,
SCX_TASK_REENQ_NONE = 0 << SCX_TASK_REENQ_REASON_SHIFT,
SCX_TASK_REENQ_KFUNC = 1 << SCX_TASK_REENQ_REASON_SHIFT,
SCX_TASK_REENQ_IMMED = 2 << SCX_TASK_REENQ_REASON_SHIFT,
SCX_TASK_REENQ_PREEMPTED = 3 << SCX_TASK_REENQ_REASON_SHIFT,
SCX_TASK_REENQ_CAP = 4 << SCX_TASK_REENQ_REASON_SHIFT,
/* iteration cursor, not a task */
SCX_TASK_CURSOR = 1 << 31,
@@ -189,22 +194,26 @@ struct sched_ext_entity {
atomic_long_t ops_state;
u64 ddsp_dsq_id;
u64 ddsp_enq_flags;
u64 ddsp_slice;
u64 ddsp_vtime;
struct scx_dsq_list_node dsq_list; /* dispatch order */
struct rb_node dsq_priq; /* p->scx.dsq_vtime order */
u32 dsq_seq;
u32 dsq_flags; /* protected by DSQ lock */
u32 flags; /* protected by rq lock */
u32 weight;
u32 reenq_cnt; /* reenqueues since last run */
s32 sticky_cpu;
s32 holding_cpu;
s32 selected_cpu;
s32 runnable_cpu; /* cpu @p is runnable on, -1 if not */
struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */
struct list_head runnable_node; /* rq->scx.runnable_list */
unsigned long runnable_at;
#ifdef CONFIG_SCHED_CORE
u64 core_sched_at; /* see scx_prio_less() */
#ifdef CONFIG_EXT_SUB_SCHED
unsigned long rescue_at; /* queued on a rescue DSQ at, jiffies */
#endif
/*
@@ -219,10 +228,11 @@ struct sched_ext_entity {
/* BPF scheduler modifiable fields */
/*
* Runtime budget in nsecs. This is usually set through
* scx_bpf_dsq_insert() but can also be modified directly by the BPF
* scheduler. Automatically decreased by SCX as the task executes. On
* depletion, a scheduling event is triggered.
* Runtime budget in nsecs - how long the task may hold its cpu. Owned
* by the task's scheduler. Set it when enqueuing via
* scx_bpf_dsq_insert(), or otherwise via scx_bpf_task_set_slice().
* Automatically decreased as the task executes. On depletion a
* scheduling event is triggered.
*
* This value is cleared to zero if the task is preempted by
* %SCX_KICK_PREEMPT and shouldn't be used to determine how long the
@@ -239,6 +249,22 @@ struct sched_ext_entity {
*/
u64 dsq_vtime;
/*
* Out-of-band slice request from scx_bpf_task_set_slice() when the
* caller does not hold the rq lock, applied under the rq lock at the
* next slice consideration. One atomic64 packs the pending flag, the
* issuing sch's id, and the requested slice. See scx_slice_oob_consts.
*/
atomic64_t slice_oob;
/*
* Sub-sched cap rejected reenq context, valid only while
* %SCX_TASK_REENQ_CAP is set. @reenq_reason_caps is the SCX_CAP_* bits
* that were needed but missing. @reenq_reason_cid is the target cid.
*/
u64 reenq_reason_caps;
s32 reenq_reason_cid;
/*
* If set, reject future sched_setscheduler(2) calls updating the policy
* to %SCHED_EXT with -%EACCES.
@@ -263,7 +289,7 @@ void sched_ext_dead(struct task_struct *p);
void print_scx_info(const char *log_lvl, struct task_struct *p);
void scx_softlockup(u32 dur_s);
bool scx_hardlockup(int cpu);
bool scx_rcu_cpu_stall(void);
bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask);
#else /* !CONFIG_SCHED_CLASS_EXT */
@@ -271,12 +297,27 @@ static inline void sched_ext_dead(struct task_struct *p) {}
static inline void print_scx_info(const char *log_lvl, struct task_struct *p) {}
static inline void scx_softlockup(u32 dur_s) {}
static inline bool scx_hardlockup(int cpu) { return false; }
static inline bool scx_rcu_cpu_stall(void) { return false; }
static inline bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask) { return false; }
#endif /* CONFIG_SCHED_CLASS_EXT */
struct scx_task_group {
#ifdef CONFIG_EXT_GROUP_SCHED
/*
* The sched this tg is on, NULL if none. SCX_TG_INITED tracks whether
* ops.cgroup_init() succeeded on it. When a child sched exits and its
* tgs move to the parent, a failed init leaves the tg on the parent
* with INITED clear (see scx_cgroup_return_subtree()).
*
* This is tracked separately from cgrp->scx_sched because the tg
* hierarchy can diverge from the cgroup2 hierarchy in both lifetime and
* shape. A tg stays online past its cgroup's removal while the
* cgrp->scx_sched rewrites visit only live cgroups, leaving a removed
* cgroup's pointer stale. The cpu controller can also be mounted on
* cgroup1.
*/
struct scx_sched *sched;
u32 flags; /* SCX_TG_* */
u32 weight;
u64 bw_period_us;

28
include/trace/events/sched_ext.h
View File

@@ -84,6 +84,34 @@ TRACE_EVENT(sched_ext_bypass_lb,
)
);
TRACE_EVENT(sched_ext_exit,
TP_PROTO(struct scx_sched *sch, __u32 kind),
TP_ARGS(sch, kind),
TP_STRUCT__entry(
__string( name, sch->ops.name )
__field( __s32, level )
__field( __u64, sub_cgroup_id )
__string( cgrp_path, sch_cgrp_path(sch) )
__field( __u32, kind )
),
TP_fast_assign(
__assign_str(name);
__entry->level = sch->level;
__entry->sub_cgroup_id = sch->ops.sub_cgroup_id;
__assign_str(cgrp_path);
__entry->kind = kind;
),
TP_printk("sched %s level %d sub_cgroup_id %llu cgrp_path %s kind %u",
__get_str(name), __entry->level, __entry->sub_cgroup_id,
__get_str(cgrp_path), __entry->kind
)
);
#endif /* _TRACE_SCHED_EXT_H */
/* This part must be outside protection */

1
init/init_task.c
View File

@@ -145,6 +145,7 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
.dsq_list.node = LIST_HEAD_INIT(init_task.scx.dsq_list.node),
.sticky_cpu = -1,
.holding_cpu = -1,
.runnable_cpu = -1,
.runnable_node = LIST_HEAD_INIT(init_task.scx.runnable_node),
.runnable_at = INITIAL_JIFFIES,
.ddsp_dsq_id = SCX_DSQ_INVALID,

1
kernel/Kconfig.preempt
View File

@@ -173,6 +173,7 @@ config SCHED_CORE
config SCHED_CLASS_EXT
bool "Extensible Scheduling Class"
depends on BPF_SYSCALL && BPF_JIT && DEBUG_INFO_BTF
select GENERIC_ALLOCATOR
select STACKTRACE if STACKTRACE_SUPPORT
help
This option enables a new scheduler class sched_ext (SCX), which

93
kernel/cgroup/cgroup.c
View File

@@ -88,6 +88,8 @@ EXPORT_SYMBOL_GPL(css_set_lock);
struct blocking_notifier_head cgroup_lifetime_notifier =
BLOCKING_NOTIFIER_INIT(cgroup_lifetime_notifier);
struct blocking_notifier_head cgroup_task_notifier =
BLOCKING_NOTIFIER_INIT(cgroup_task_notifier);
DEFINE_SPINLOCK(trace_cgroup_path_lock);
char trace_cgroup_path[TRACE_CGROUP_PATH_LEN];
@@ -2676,14 +2678,27 @@ struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset,
return NULL;
}
static void cgroup_migrate_notify_canceled(struct css_set *src_cset,
struct task_struct *task)
{
struct cgroup_task_migrate_ctx ctx = {
.task = task,
.src_dcgrp = src_cset->dfl_cgrp,
.dst_dcgrp = src_cset->mg_dst_cset->dfl_cgrp,
};
blocking_notifier_call_chain(&cgroup_task_notifier,
CGROUP_TASK_MIGRATE_CANCELED, &ctx);
}
/**
* cgroup_migrate_execute - migrate a taskset
* @mgctx: migration context
*
* Migrate tasks in @mgctx as setup by migration preparation functions.
* This function fails iff one of the ->can_attach callbacks fails and
* guarantees that either all or none of the tasks in @mgctx are migrated.
* @mgctx is consumed regardless of success.
* Migrate tasks in @mgctx as setup by migration preparation functions. This
* function fails iff one of the ->can_attach callbacks or CGROUP_TASK_MIGRATING
* notifications fails and guarantees that either all or none of the tasks in
* @mgctx are migrated. @mgctx is consumed regardless of success.
*/
static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx)
{
@@ -2691,6 +2706,7 @@ static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx)
struct cgroup_subsys *ss;
struct task_struct *task, *tmp_task;
struct css_set *cset, *tmp_cset;
bool dfl_migration = false;
int ssid, failed_ssid, ret;
/* check that we can legitimately attach to the cgroup */
@@ -2707,6 +2723,33 @@ static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx)
} while_each_subsys_mask();
}
/*
* Notify each task about the impending migration. An error return fails
* the migration. Only migrations on the default hierarchy are reported:
* a migration modifies either every moved task's dfl cgroup or, on
* cgroup1 or for subtree_control writes, none.
*/
list_for_each_entry(cset, &tset->src_csets, mg_node) {
if (cset->dfl_cgrp == cset->mg_dst_cset->dfl_cgrp)
continue;
dfl_migration = true;
list_for_each_entry(task, &cset->mg_tasks, cg_list) {
struct cgroup_task_migrate_ctx ctx = {
.task = task,
.src_dcgrp = cset->dfl_cgrp,
.dst_dcgrp = cset->mg_dst_cset->dfl_cgrp,
};
ret = blocking_notifier_call_chain_robust(&cgroup_task_notifier,
CGROUP_TASK_MIGRATING,
CGROUP_TASK_MIGRATE_CANCELED,
&ctx);
ret = notifier_to_errno(ret);
if (ret)
goto out_cancel_migrating;
}
}
/*
* Now that we're guaranteed success, proceed to move all tasks to
* the new cgroup. There are no failure cases after here, so this
@@ -2750,9 +2793,41 @@ static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx)
} while_each_subsys_mask();
}
/*
* Notify each task after successful migration. The operation can no
* longer fail and the return value is ignored. The MIGRATING loop
* above explains why only dfl migrations are reported. Per-task
* sources are not tracked past the commit point, so src_dcgrp is
* NULL.
*/
if (dfl_migration) {
list_for_each_entry(cset, &tset->dst_csets, mg_node) {
list_for_each_entry(task, &cset->mg_tasks, cg_list) {
struct cgroup_task_migrate_ctx ctx = {
.task = task,
.dst_dcgrp = cset->dfl_cgrp,
};
blocking_notifier_call_chain(
&cgroup_task_notifier,
CGROUP_TASK_MIGRATED, &ctx);
}
}
}
ret = 0;
goto out_release_tset;
out_cancel_migrating:
list_for_each_entry_continue_reverse(task, &cset->mg_tasks, cg_list)
cgroup_migrate_notify_canceled(cset, task);
list_for_each_entry_continue_reverse(cset, &tset->src_csets, mg_node) {
if (cset->dfl_cgrp == cset->mg_dst_cset->dfl_cgrp)
continue;
list_for_each_entry_reverse(task, &cset->mg_tasks, cg_list)
cgroup_migrate_notify_canceled(cset, task);
}
failed_ssid = CGROUP_SUBSYS_COUNT;
out_cancel_attach:
if (tset->nr_tasks) {
do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {
@@ -2976,11 +3051,11 @@ int cgroup_migrate_prepare_dst(struct cgroup_mgctx *mgctx)
* cgroup_migrate_prepare_dst() on the targets before invoking this
* function and following up with cgroup_migrate_finish().
*
* As long as a controller's ->can_attach() doesn't fail, this function is
* guaranteed to succeed. This means that, excluding ->can_attach()
* failure, when migrating multiple targets, the success or failure can be
* decided for all targets by invoking group_migrate_prepare_dst() before
* actually starting migrating.
* As long as a controller's ->can_attach() or a CGROUP_TASK_MIGRATING
* notification doesn't fail, this function is guaranteed to succeed. This
* means that, excluding those failures, when migrating multiple targets,
* the success or failure can be decided for all targets by invoking
* group_migrate_prepare_dst() before actually starting migrating.
*/
int cgroup_migrate(struct task_struct *leader, bool threadgroup,
struct cgroup_mgctx *mgctx)

3
kernel/rcu/tree.c
View File

@@ -4900,6 +4900,9 @@ static void __init rcu_dump_rcu_node_tree(void)
struct workqueue_struct *rcu_gp_wq;
static struct cpumask rcu_stall_cpumask;
static struct cpumask rcu_exp_stall_cpumask;
void __init rcu_init(void)
{
int cpu = smp_processor_id();

5
kernel/rcu/tree_exp.h
View File

@@ -578,6 +578,7 @@ static void synchronize_rcu_expedited_stall(unsigned long jiffies_start, unsigne
if (!(READ_ONCE(rnp->expmask) & mask))
continue;
ndetected++;
cpumask_set_cpu(cpu, &rcu_exp_stall_cpumask);
rdp = per_cpu_ptr(&rcu_data, cpu);
pr_cont(" %d-%c%c%c%c", cpu,
"O."[!!cpu_online(cpu)],
@@ -665,6 +666,8 @@ static void synchronize_rcu_expedited_wait(void)
if (rcu_stall_is_suppressed())
continue;
cpumask_clear(&rcu_exp_stall_cpumask);
nbcon_cpu_emergency_enter();
j = jiffies;
@@ -675,7 +678,7 @@ static void synchronize_rcu_expedited_wait(void)
nbcon_cpu_emergency_exit();
panic_on_rcu_stall();
panic_on_rcu_stall(&rcu_exp_stall_cpumask);
}
}

13
kernel/rcu/tree_stall.h
View File

@@ -159,7 +159,7 @@ static int __init check_cpu_stall_init(void)
early_initcall(check_cpu_stall_init);
/* If so specified via sysctl, panic, yielding cleaner stall-warning output. */
static void panic_on_rcu_stall(void)
static void panic_on_rcu_stall(const struct cpumask *stalled_mask)
{
static int cpu_stall;
@@ -167,7 +167,7 @@ static void panic_on_rcu_stall(void)
* Attempt to kick out the BPF scheduler if it's installed and defer
* the panic to give the system a chance to recover.
*/
if (scx_rcu_cpu_stall())
if (scx_rcu_cpu_stall(stalled_mask))
return;
if (++cpu_stall < sysctl_max_rcu_stall_to_panic)
@@ -644,6 +644,8 @@ static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps)
if (rcu_stall_is_suppressed())
return;
cpumask_clear(&rcu_stall_cpumask);
nbcon_cpu_emergency_enter();
/*
@@ -659,6 +661,7 @@ static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps)
for_each_leaf_node_possible_cpu(rnp, cpu)
if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) {
print_cpu_stall_info(cpu);
cpumask_set_cpu(cpu, &rcu_stall_cpumask);
ndetected++;
}
}
@@ -700,7 +703,7 @@ static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps)
nbcon_cpu_emergency_exit();
panic_on_rcu_stall();
panic_on_rcu_stall(&rcu_stall_cpumask);
rcu_force_quiescent_state(); /* Kick them all. */
}
@@ -753,7 +756,9 @@ static void print_cpu_stall(unsigned long gp_seq, unsigned long gps)
nbcon_cpu_emergency_exit();
panic_on_rcu_stall();
cpumask_clear(&rcu_stall_cpumask);
cpumask_set_cpu(smp_processor_id(), &rcu_stall_cpumask);
panic_on_rcu_stall(&rcu_stall_cpumask);
/*
* Attempt to revive the RCU machinery by forcing a context switch.

3
kernel/sched/build_policy.c
View File

@@ -66,10 +66,13 @@
# include "ext/cid.h"
# include "ext/arena.h"
# include "ext/idle.h"
# include "ext/sub.h"
# include "ext/inlines.h"
# include "ext/ext.c"
# include "ext/cid.c"
# include "ext/arena.c"
# include "ext/idle.c"
# include "ext/sub.c"
#endif
#include "syscalls.c"

49
kernel/sched/core.c
View File

@@ -443,6 +443,17 @@ static void __sched_core_flip(bool enabled)
sched_core_lock(cpu, &flags);
/*
* A core-wide selection may have the shared rq lock temporarily
* released by a lock-dropping ->pick_task(). Flipping would
* rebind rq_lockp() under it. Wait it out.
*/
while (cpu_rq(cpu)->core->core_pick_in_flight) {
sched_core_unlock(cpu, &flags);
cpu_relax();
sched_core_lock(cpu, &flags);
}
for_each_cpu(t, smt_mask)
cpu_rq(t)->core_enabled = enabled;
@@ -6228,7 +6239,7 @@ pick_next_task(struct rq *rq, struct rq_flags *rf)
unsigned long cookie;
int i, cpu, occ = 0;
struct rq *rq_i;
bool need_sync;
bool need_sync = false;
if (!sched_core_enabled(rq))
return __pick_next_task(rq, rf);
@@ -6247,6 +6258,8 @@ pick_next_task(struct rq *rq, struct rq_flags *rf)
return __pick_next_task(rq, rf);
}
rq->core->core_pick_in_flight++;
/*
* If there were no {en,de}queues since we picked (IOW, the task
* pointers are all still valid), and we haven't scheduled the last
@@ -6271,7 +6284,9 @@ pick_next_task(struct rq *rq, struct rq_flags *rf)
prev_balance(rq, rf);
smt_mask = cpu_smt_mask(cpu);
need_sync = !!rq->core->core_cookie;
restart:
need_sync |= !!rq->core->core_cookie;
/* reset state */
rq->core->core_cookie = 0UL;
@@ -6306,10 +6321,15 @@ pick_next_task(struct rq *rq, struct rq_flags *rf)
* and there are no cookied tasks running on siblings.
*/
if (!need_sync) {
restart_single:
next = pick_task(rq, rf);
if (unlikely(next == RETRY_TASK))
goto restart_single;
if (unlikely(next == RETRY_TASK)) {
/* rq lock may have been dropped, clocks invalidated */
core_clock_updated = false;
if (!(rq->clock_update_flags & RQCF_UPDATED))
update_rq_clock(rq);
goto restart;
}
if (!next->core_cookie) {
rq->core_pick = NULL;
rq->core_dl_server = NULL;
@@ -6329,7 +6349,6 @@ pick_next_task(struct rq *rq, struct rq_flags *rf)
*
* Tie-break prio towards the current CPU
*/
restart_multi:
max = NULL;
for_each_cpu_wrap(i, smt_mask, cpu) {
rq_i = cpu_rq(i);
@@ -6343,8 +6362,13 @@ pick_next_task(struct rq *rq, struct rq_flags *rf)
update_rq_clock(rq_i);
p = pick_task(rq_i, rf);
if (unlikely(p == RETRY_TASK))
goto restart_multi;
if (unlikely(p == RETRY_TASK)) {
/* rq lock may have been dropped, clocks invalidated */
core_clock_updated = false;
if (!(rq->clock_update_flags & RQCF_UPDATED))
update_rq_clock(rq);
goto restart;
}
rq_i->core_pick = p;
rq_i->core_dl_server = rq_i->dl_server;
@@ -6450,6 +6474,7 @@ pick_next_task(struct rq *rq, struct rq_flags *rf)
}
out_set_next:
rq->core->core_pick_in_flight--;
put_prev_set_next_task(rq, rq->donor, next);
if (rq->core->core_forceidle_count && next == rq->idle)
queue_core_balance(rq);
@@ -6644,6 +6669,13 @@ static void sched_core_cpu_deactivate(unsigned int cpu)
core_rq->core_forceidle_seq = rq->core_forceidle_seq;
core_rq->core_forceidle_occupation = rq->core_forceidle_occupation;
/*
* A stale leftover would bias the count forever if this CPU later
* returns as its own leader. Move, don't copy.
*/
core_rq->core_pick_in_flight = rq->core_pick_in_flight;
rq->core_pick_in_flight = 0;
/*
* Accounting edge for forced idle is handled in pick_next_task().
* Don't need another one here, since the hotplug thread shouldn't
@@ -9058,6 +9090,7 @@ void __init sched_init(void)
rq->core_forceidle_count = 0;
rq->core_forceidle_occupation = 0;
rq->core_forceidle_start = 0;
rq->core_pick_in_flight = 0;
rq->core_cookie = 0UL;
#endif

16
kernel/sched/ext/arena.c
View File

@@ -6,8 +6,7 @@
*
* Each chunk added to @sch->arena_pool comes from one
* bpf_arena_alloc_pages_sleepable() call and is registered at the
* kernel-side mapping address. Callers translate to the BPF-arena form
* themselves if needed.
* kernel-side mapping address.
*
* Allocations grow the pool on demand. Underlying arena pages are released
* when the arena map itself is torn down.
@@ -70,8 +69,6 @@ void scx_arena_pool_destroy(struct scx_sched *sch)
*/
static int scx_arena_grow(struct scx_sched *sch, u32 page_cnt)
{
u64 kern_vm_start;
u32 uaddr32;
void *p;
int ret;
@@ -83,15 +80,8 @@ static int scx_arena_grow(struct scx_sched *sch, u32 page_cnt)
if (!p)
return -ENOMEM;
uaddr32 = (u32)(unsigned long)p;
/* arena.o, which defines these, is built only on MMU && 64BIT */
#if defined(CONFIG_MMU) && defined(CONFIG_64BIT)
kern_vm_start = bpf_arena_map_kern_vm_start(sch->arena_map);
#else
kern_vm_start = 0;
#endif
ret = gen_pool_add(sch->arena_pool, kern_vm_start + uaddr32,
ret = gen_pool_add(sch->arena_pool,
(unsigned long)scx_arena_to_kaddr(sch, p),
page_cnt * PAGE_SIZE, NUMA_NO_NODE);
if (ret) {
bpf_arena_free_pages_non_sleepable(sch->arena_map, p, page_cnt);

730
kernel/sched/ext/cid.c
View File

@@ -11,20 +11,26 @@
#include "cid.h"
/*
* cid tables.
*
* Pointers are published once on first enable and never revoked. The default
* mapping is populated before ops.init() runs; scx_bpf_cid_override() commits
* before it returns. As long as the BPF scheduler only uses the tables from
* those points onward, it sees a consistent view.
* cid tables. The cid kfuncs are available whether the root scheduler is
* cid-form or cpu-form, the latter to allow gradual migration to cids, so every
* root builds a default mapping. Each root enable allocates a fresh set, builds
* it privately and publishes the __rcu globals below once the layout is final.
* Root disable unpublishes and RCU-frees the set. kfuncs may run before the
* tables are published and must check for NULL.
*/
s16 *scx_cid_to_cpu_tbl;
s16 *scx_cpu_to_cid_tbl;
struct scx_cid_topo *scx_cid_topo;
u32 scx_nr_cid_shards;
s16 __rcu *scx_cid_to_cpu_tbl;
s16 __rcu *scx_cpu_to_cid_tbl;
s32 __rcu *scx_cid_to_shard;
s32 __rcu *scx_shard_node;
struct scx_cid_shard __rcu *scx_cid_shard_ranges;
struct scx_cid_topo __rcu *scx_cid_topo;
static struct scx_cid_tables *scx_cid_tables; /* used only during alloc/free */
#define SCX_CID_TOPO_NEG (struct scx_cid_topo) { \
.core_cid = -1, .core_idx = -1, .llc_cid = -1, .llc_idx = -1, \
.node_cid = -1, .node_idx = -1, \
.node_cid = -1, .node_idx = -1, .shard_cid = -1, .shard_idx = -1, \
}
/*
@@ -43,37 +49,128 @@ static const struct cpumask *cpu_llc_mask(int cpu, struct cpumask *fallbacks)
return &ci->info_list[ci->num_leaves - 1].shared_cpu_map;
}
/* Allocate the cid tables once on first enable; never freed. */
static s32 scx_cid_arrays_alloc(void)
/*
* Compute per-LLC shard layout. Each shard holds at most @shard_size cids, and
* in any case no more than SCX_CID_SHARD_MAX_CPUS. Cores are spread as evenly
* as possible across shards so cpu count is balanced: the first *@nr_large_p
* shards get (*@cores_per_shard_p + 1) cores, the rest get *@cores_per_shard_p.
*/
static void calc_shard_layout(const struct cpumask *llc_cpus, u32 shard_size,
u32 *cores_per_shard_p, u32 *nr_large_p)
{
u32 npossible = num_possible_cpus();
s16 *cid_to_cpu, *cpu_to_cid;
struct scx_cid_topo *cid_topo;
u32 nr_cores = 0, nr_cpus = 0, nr_shards;
int cpu;
if (scx_cid_to_cpu_tbl)
return 0;
cid_to_cpu = kzalloc_objs(*scx_cid_to_cpu_tbl, npossible, GFP_KERNEL);
cpu_to_cid = kzalloc_objs(*scx_cpu_to_cid_tbl, nr_cpu_ids, GFP_KERNEL);
cid_topo = kmalloc_objs(*scx_cid_topo, npossible, GFP_KERNEL);
if (!cid_to_cpu || !cpu_to_cid || !cid_topo) {
kfree(cid_to_cpu);
kfree(cpu_to_cid);
kfree(cid_topo);
return -ENOMEM;
for_each_cpu(cpu, llc_cpus) {
nr_cpus++;
if (cpumask_first(topology_sibling_cpumask(cpu)) == cpu)
nr_cores++;
}
WRITE_ONCE(scx_cid_to_cpu_tbl, cid_to_cpu);
WRITE_ONCE(scx_cpu_to_cid_tbl, cpu_to_cid);
WRITE_ONCE(scx_cid_topo, cid_topo);
return 0;
nr_shards = max_t(u32, 1, DIV_ROUND_UP(nr_cpus, shard_size));
nr_shards = max_t(u32, nr_shards,
DIV_ROUND_UP(nr_cpus, SCX_CID_SHARD_MAX_CPUS));
*cores_per_shard_p = nr_cores / nr_shards;
*nr_large_p = nr_cores % nr_shards;
}
static void scx_cid_tables_free(struct scx_cid_tables *tbls)
{
if (!tbls)
return;
kvfree(tbls->cid_to_cpu);
kvfree(tbls->cpu_to_cid);
kvfree(tbls->cid_to_shard);
kvfree(tbls->shard_node);
kvfree(tbls->shard_ranges);
kvfree(tbls->topo);
kfree(tbls);
}
static void scx_cid_tables_free_rcufn(struct rcu_head *rcu)
{
scx_cid_tables_free(container_of(rcu, struct scx_cid_tables, rcu));
}
static struct scx_cid_tables *scx_cid_alloc_tables(void)
{
u32 npossible = num_possible_cpus();
struct scx_cid_tables *tbls;
tbls = kzalloc_obj(*tbls, GFP_KERNEL);
if (!tbls)
return NULL;
tbls->cid_to_cpu = kvcalloc(npossible, sizeof(*tbls->cid_to_cpu), GFP_KERNEL);
tbls->cpu_to_cid = kvcalloc(nr_cpu_ids, sizeof(*tbls->cpu_to_cid), GFP_KERNEL);
tbls->cid_to_shard = kvcalloc(npossible, sizeof(*tbls->cid_to_shard), GFP_KERNEL);
tbls->shard_node = kvcalloc(npossible, sizeof(*tbls->shard_node), GFP_KERNEL);
tbls->shard_ranges = kvcalloc(npossible, sizeof(*tbls->shard_ranges), GFP_KERNEL);
tbls->topo = kvcalloc(npossible, sizeof(*tbls->topo), GFP_KERNEL);
if (!tbls->cid_to_cpu || !tbls->cpu_to_cid || !tbls->cid_to_shard ||
!tbls->shard_node || !tbls->shard_ranges || !tbls->topo) {
scx_cid_tables_free(tbls);
return NULL;
}
return tbls;
}
/**
* scx_cid_publish_tables - Publish the tables scx_cid_init() built
*
* Called after ops.init_cids() where the layout is final.
*/
void scx_cid_publish_tables(void)
{
struct scx_cid_tables *tbls = scx_cid_tables;
lockdep_assert_held(&scx_enable_mutex);
scx_nr_cid_shards = tbls->nr_shards;
rcu_assign_pointer(scx_cid_to_cpu_tbl, tbls->cid_to_cpu);
rcu_assign_pointer(scx_cpu_to_cid_tbl, tbls->cpu_to_cid);
rcu_assign_pointer(scx_cid_to_shard, tbls->cid_to_shard);
rcu_assign_pointer(scx_shard_node, tbls->shard_node);
rcu_assign_pointer(scx_cid_shard_ranges, tbls->shard_ranges);
rcu_assign_pointer(scx_cid_topo, tbls->topo);
}
/**
* scx_cid_retire_tables - Unpublish and retire the cid tables
*
* Called by root disable after the readers which dereference without NULL
* checks are drained, inside cpus_read_lock() to exclude the hotplug path.
*/
void scx_cid_retire_tables(void)
{
struct scx_cid_tables *tbls = scx_cid_tables;
lockdep_assert_held(&scx_enable_mutex);
lockdep_assert_cpus_held();
if (!tbls)
return;
scx_cid_tables = NULL;
RCU_INIT_POINTER(scx_cid_to_cpu_tbl, NULL);
RCU_INIT_POINTER(scx_cpu_to_cid_tbl, NULL);
RCU_INIT_POINTER(scx_cid_to_shard, NULL);
RCU_INIT_POINTER(scx_shard_node, NULL);
RCU_INIT_POINTER(scx_cid_shard_ranges, NULL);
RCU_INIT_POINTER(scx_cid_topo, NULL);
call_rcu(&tbls->rcu, scx_cid_tables_free_rcufn);
}
/**
* scx_cid_init - build the cid mapping
* @sch: the scx_sched being initialized; used as the scx_error() target
*
* Build a fresh table set. It becomes visible through scx_cid_publish_tables()
* and is retired by scx_cid_retire_tables() at disable.
*
* See "Topological CPU IDs" in cid.h for the model. Walk online cpus by
* intersection at each level (parent_scratch & this_level_mask), which keeps
* containment correct by construction and naturally splits a physical LLC
@@ -88,18 +185,32 @@ s32 scx_cid_init(struct scx_sched *sch)
cpumask_var_t core_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL;
cpumask_var_t llc_fallback __free(free_cpumask_var) = CPUMASK_VAR_NULL;
cpumask_var_t online_no_topo __free(free_cpumask_var) = CPUMASK_VAR_NULL;
struct scx_cid_tables *tbls;
u32 next_cid = 0;
s32 next_node_idx = 0, next_llc_idx = 0, next_core_idx = 0;
s32 cpu, ret;
s32 next_shard_idx = 0;
u32 shard_size, max_cids;
u32 notopo_in_shard;
s32 notopo_shard_cid, notopo_shard_idx;
s32 cpu, cid, si;
/* CMASK_MAX_WORDS in cid.bpf.h covers NR_CPUS up to 8192 */
BUILD_BUG_ON(NR_CPUS > 8192);
lockdep_assert_cpus_held();
lockdep_assert_held(&scx_enable_mutex);
ret = scx_cid_arrays_alloc();
if (ret)
return ret;
shard_size = sch->ops.cid_shard_size ?: SCX_CID_SHARD_SIZE_DFL;
max_cids = min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS);
tbls = scx_cid_alloc_tables();
if (!tbls)
return -ENOMEM;
scx_cid_tables = tbls;
for (si = 0; si < num_possible_cpus(); si++)
tbls->shard_node[si] = NUMA_NO_NODE;
if (!zalloc_cpumask_var(&to_walk, GFP_KERNEL) ||
!zalloc_cpumask_var(&node_scratch, GFP_KERNEL) ||
@@ -111,7 +222,7 @@ s32 scx_cid_init(struct scx_sched *sch)
/* -1 sentinels for sparse-possible cpu id holes (0 is a valid cid) */
for (cpu = 0; cpu < nr_cpu_ids; cpu++)
scx_cpu_to_cid_tbl[cpu] = -1;
tbls->cpu_to_cid[cpu] = -1;
cpumask_copy(to_walk, cpu_online_mask);
@@ -142,36 +253,69 @@ s32 scx_cid_init(struct scx_sched *sch)
const struct cpumask *llc_mask = cpu_llc_mask(ncpu, llc_fallback);
s32 llc_cid = next_cid;
s32 llc_idx = next_llc_idx++;
u32 cores_per_shard, nr_large;
u32 shard_local = 0, cores_in_shard = 0, cids_in_shard = 0;
s32 shard_cid, shard_idx;
/* llc_scratch = node_scratch & this llc */
cpumask_and(llc_scratch, node_scratch, llc_mask);
if (WARN_ON_ONCE(!cpumask_test_cpu(ncpu, llc_scratch)))
return -EINVAL;
calc_shard_layout(llc_scratch, shard_size, &cores_per_shard, &nr_large);
shard_cid = next_cid;
shard_idx = next_shard_idx++;
tbls->shard_node[shard_idx] = nid;
while (!cpumask_empty(llc_scratch)) {
s32 lcpu = cpumask_first(llc_scratch);
const struct cpumask *sib = topology_sibling_cpumask(lcpu);
s32 core_cid = next_cid;
s32 core_idx = next_core_idx++;
s32 ccpu;
u32 max_cores, cids_in_core;
/* core_scratch = llc_scratch & this core */
cpumask_and(core_scratch, llc_scratch, sib);
if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, core_scratch)))
return -EINVAL;
/*
* Advance to a new shard when either core or
* cid count reaches max. The latter bounds
* shard sizes under uneven SMT. Never start an
* empty shard.
*/
cids_in_core = cpumask_weight(core_scratch);
max_cores = cores_per_shard + (shard_local < nr_large ? 1 : 0);
if (cores_in_shard &&
(cores_in_shard >= max_cores ||
cids_in_shard + cids_in_core > max_cids)) {
shard_local++;
cores_in_shard = 0;
cids_in_shard = 0;
shard_cid = next_cid;
shard_idx = next_shard_idx++;
tbls->shard_node[shard_idx] = nid;
}
cores_in_shard++;
cids_in_shard += cids_in_core;
for_each_cpu(ccpu, core_scratch) {
s32 cid = next_cid++;
scx_cid_to_cpu_tbl[cid] = ccpu;
scx_cpu_to_cid_tbl[ccpu] = cid;
scx_cid_topo[cid] = (struct scx_cid_topo){
tbls->cid_to_cpu[cid] = ccpu;
tbls->cpu_to_cid[ccpu] = cid;
tbls->cid_to_shard[cid] = shard_idx;
tbls->topo[cid] = (struct scx_cid_topo){
.core_cid = core_cid,
.core_idx = core_idx,
.llc_cid = llc_cid,
.llc_idx = llc_idx,
.node_cid = node_cid,
.node_idx = node_idx,
.shard_cid = shard_cid,
.shard_idx = shard_idx,
};
cpumask_clear_cpu(ccpu, llc_scratch);
@@ -184,21 +328,37 @@ s32 scx_cid_init(struct scx_sched *sch)
/*
* No-topo section: any possible cpu without a cid - normally just the
* not-online ones. Collect any currently-online cpus that land here in
* @online_no_topo so we can warn about them at the end.
* not-online ones. Pack into shards of up to min(@shard_size,
* SCX_CID_SHARD_MAX_CPUS) cids so that every cid has a valid shard
* assignment and the hard cap holds even with a large @shard_size.
* Collect any currently-online cpus that land here in @online_no_topo
* so we can warn about them at the end.
*/
for_each_cpu(cpu, cpu_possible_mask) {
s32 cid;
notopo_in_shard = min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS);
notopo_shard_cid = -1;
notopo_shard_idx = -1;
if (__scx_cpu_to_cid(cpu) != -1)
for_each_cpu(cpu, cpu_possible_mask) {
if (tbls->cpu_to_cid[cpu] != -1)
continue;
if (cpu_online(cpu))
cpumask_set_cpu(cpu, online_no_topo);
cid = next_cid++;
scx_cid_to_cpu_tbl[cid] = cpu;
scx_cpu_to_cid_tbl[cpu] = cid;
scx_cid_topo[cid] = SCX_CID_TOPO_NEG;
tbls->cid_to_cpu[cid] = cpu;
tbls->cpu_to_cid[cpu] = cid;
if (notopo_in_shard >= min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS)) {
notopo_shard_cid = cid;
notopo_shard_idx = next_shard_idx++;
notopo_in_shard = 0;
}
notopo_in_shard++;
tbls->cid_to_shard[cid] = notopo_shard_idx;
tbls->topo[cid] = SCX_CID_TOPO_NEG;
tbls->topo[cid].shard_cid = notopo_shard_cid;
tbls->topo[cid].shard_idx = notopo_shard_idx;
}
if (!cpumask_empty(llc_fallback))
@@ -208,6 +368,20 @@ s32 scx_cid_init(struct scx_sched *sch)
pr_warn("scx_cid: online cpus with no usable topology: %*pbl\n",
cpumask_pr_args(online_no_topo));
/*
* Fill cid_shard_ranges[] from cid_to_shard[]. Shards are contiguous
* cid ranges by construction: base_cid is the first cid landing in a
* shard, nr_cids is the count.
*/
for (cid = 0; cid < next_cid; cid++) {
s32 sidx = tbls->cid_to_shard[cid];
if (tbls->shard_ranges[sidx].nr_cids == 0)
tbls->shard_ranges[sidx].base_cid = cid;
tbls->shard_ranges[sidx].nr_cids++;
}
tbls->nr_shards = next_shard_idx;
return 0;
}
@@ -253,50 +427,76 @@ void scx_cmask_fill(struct scx_cmask *m)
m->bits[nr_words - 1] &= (1ULL << tail_bits) - 1;
}
/**
* scx_cpumask_to_cmask - Translate a kernel cpumask into a cmask
* @src: source cpumask
* @dst: cmask to write
*
* Clear @dst's active range and set the bit for each cid whose cpu is in
* @src and lies within that range. Out-of-range cids are silently ignored.
/*
* Return the index of the largest entry in @counts, or NUMA_NO_NODE if all
* entries are zero. Ties resolve to the lowest index.
*/
void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst)
static s32 pick_max_node(const u32 *counts, u32 n)
{
s32 cpu;
s32 best = NUMA_NO_NODE;
u32 best_count = 0, i;
scx_cmask_clear(dst);
for_each_cpu(cpu, src) {
s32 cid = __scx_cpu_to_cid(cpu);
if (cid >= 0)
__scx_cmask_set(cid, dst);
for (i = 0; i < n; i++) {
if (counts[i] > best_count) {
best_count = counts[i];
best = i;
}
}
return best;
}
__bpf_kfunc_start_defs();
/**
* scx_bpf_cid_override - Install an explicit cpu->cid mapping
* @cpu_to_cid: array of nr_cpu_ids s32 entries (cid for each cpu)
* @cpu_to_cid__sz: must be nr_cpu_ids * sizeof(s32) bytes
* scx_bpf_cid_override - Install an explicit cpu->cid mapping with shard info
* @cpu_to_cid__arena: array of nr_cpu_ids s32 entries (cid for each cpu)
* @cpu_to_cid_cnt: number of entries, must be nr_cpu_ids
* @shard_start__arena: array of first-cid-of-each-shard, one entry per shard
* @shard_start_cnt: number of shards
* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
*
* May only be called from ops.init() of the root scheduler. Replace the
* topology-probed cid mapping with the caller-provided one. Each possible cpu
* must map to a unique cid in [0, num_possible_cpus()). Topo info is cleared.
* On invalid input, trigger scx_error() to abort the scheduler.
* May only be called from ops.init_cids() of the root scheduler. Replace the
* topology-probed cid mapping and shard layout with caller-provided ones. Each
* possible cpu must map to a unique cid in [0, num_possible_cpus()). The shard
* starts must be strictly increasing with the first entry 0 and all values <
* num_possible_cpus(). The last shard extends to num_possible_cpus() and no
* shard may span more than SCX_CID_SHARD_MAX_CPUS cids. Topo info
* (core/LLC/node) is cleared and the shard layout is set from the input. On
* invalid input, abort the scheduler.
*/
__bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz,
__bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid__arena, u32 cpu_to_cid_cnt,
const s32 *shard_start__arena, u32 shard_start_cnt,
const struct bpf_prog_aux *aux)
{
cpumask_var_t seen __free(free_cpumask_var) = CPUMASK_VAR_NULL;
u32 *node_counts __free(kfree) = NULL;
s32 *cpu_to_cid __free(kfree) = NULL;
s32 *shard_start __free(kfree) = NULL;
u32 npossible = num_possible_cpus();
struct scx_cid_tables *tbls;
struct scx_sched *sch;
u32 nr_shards = shard_start_cnt;
bool alloced;
s32 cpu, cid;
s32 cpu, cid, si;
/* GFP_KERNEL alloc must happen before the rcu read section */
/*
* GFP_KERNEL allocs must happen before the rcu read section. Snapshot
* the BPF-supplied arrays so a concurrent arena write can't change
* them between validation and use.
*
* The BPF-supplied counts size the snapshots and thus the arena reads.
* Gate the copies on the count bounds, reported below once @sch is
* available. The bounded reads, at most 32KB, stay within the guard
* region that arena fault recovery covers.
*/
alloced = zalloc_cpumask_var(&seen, GFP_KERNEL);
node_counts = kcalloc(nr_node_ids, sizeof(*node_counts), GFP_KERNEL);
if (cpu_to_cid_cnt == nr_cpu_ids)
cpu_to_cid = kmemdup(cpu_to_cid__arena, cpu_to_cid_cnt * sizeof(s32),
GFP_KERNEL);
if (nr_shards && nr_shards <= npossible)
shard_start = kmemdup(shard_start__arena, nr_shards * sizeof(s32),
GFP_KERNEL);
guard(rcu)();
@@ -304,22 +504,59 @@ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz,
if (unlikely(!sch))
return;
if (!alloced) {
scx_error(sch, "scx_bpf_cid_override: failed to allocate cpumask");
/* called from ops.init_cids(), so the tables exist and are unpublished */
lockdep_assert_held(&scx_enable_mutex);
tbls = scx_cid_tables;
if (cpu_to_cid_cnt != nr_cpu_ids) {
scx_error(sch, "scx_bpf_cid_override: cpu_to_cid expected %u entries, got %u",
nr_cpu_ids, cpu_to_cid_cnt);
return;
}
if (scx_parent(sch)) {
scx_error(sch, "scx_bpf_cid_override() only allowed from root sched");
if (!nr_shards || nr_shards > npossible) {
scx_error(sch, "scx_bpf_cid_override: invalid shard_start count %u",
nr_shards);
return;
}
if (cpu_to_cid__sz != nr_cpu_ids * sizeof(s32)) {
scx_error(sch, "scx_bpf_cid_override: expected %zu bytes, got %u",
nr_cpu_ids * sizeof(s32), cpu_to_cid__sz);
if (!alloced || !node_counts || !cpu_to_cid || !shard_start) {
scx_error(sch, "scx_bpf_cid_override: allocation failed");
return;
}
/* validate shard_start[]: starts at 0, strictly increasing, in range */
if (shard_start[0] != 0) {
scx_error(sch, "scx_bpf_cid_override: shard_start[0] must be 0, got %d",
shard_start[0]);
return;
}
for (si = 1; si < nr_shards; si++) {
if (shard_start[si] <= shard_start[si - 1]) {
scx_error(sch, "scx_bpf_cid_override: shard_start not increasing at [%d]",
si);
return;
}
if (shard_start[si] >= npossible) {
scx_error(sch, "scx_bpf_cid_override: shard_start[%d]=%d >= %u",
si, shard_start[si], npossible);
return;
}
if (shard_start[si] - shard_start[si - 1] > SCX_CID_SHARD_MAX_CPUS) {
scx_error(sch, "scx_bpf_cid_override: shard[%d] span %d exceeds max %d",
si - 1, shard_start[si] - shard_start[si - 1],
SCX_CID_SHARD_MAX_CPUS);
return;
}
}
if (npossible - shard_start[nr_shards - 1] > SCX_CID_SHARD_MAX_CPUS) {
scx_error(sch, "scx_bpf_cid_override: shard[%d] span %d exceeds max %d",
nr_shards - 1, npossible - shard_start[nr_shards - 1],
SCX_CID_SHARD_MAX_CPUS);
return;
}
/* validate first so that invalid input leaves the tables untouched */
for_each_possible_cpu(cpu) {
s32 c = cpu_to_cid[cpu];
@@ -329,13 +566,56 @@ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz,
scx_error(sch, "cid %d assigned to multiple cpus", c);
return;
}
scx_cpu_to_cid_tbl[cpu] = c;
scx_cid_to_cpu_tbl[c] = cpu;
}
/* Invalidate stale topo info - the override carries no topology. */
for (cid = 0; cid < num_possible_cpus(); cid++)
scx_cid_topo[cid] = SCX_CID_TOPO_NEG;
for_each_possible_cpu(cpu) {
s32 c = cpu_to_cid[cpu];
tbls->cpu_to_cid[cpu] = c;
tbls->cid_to_cpu[c] = cpu;
}
/*
* Derive shard_node[] by majority count: an overridden shard may
* span NUMA nodes, so assign each to the node that owns the most cpus.
*/
for (si = 0; si < nr_shards; si++) {
u32 end = (si + 1 < nr_shards) ? shard_start[si + 1] : npossible;
memset(node_counts, 0, nr_node_ids * sizeof(*node_counts));
for (cid = shard_start[si]; cid < end; cid++) {
s32 node = cpu_to_node(tbls->cid_to_cpu[cid]);
if (numa_valid_node(node))
node_counts[node]++;
}
tbls->shard_node[si] = pick_max_node(node_counts, nr_node_ids);
}
/*
* Invalidate stale topo info and install shard layout from
* @shard_start. Walk shards to derive shard_cid/shard_idx for each cid.
*/
si = 0;
for (cid = 0; cid < npossible; cid++) {
if (si + 1 < nr_shards && cid >= shard_start[si + 1])
si++;
tbls->cid_to_shard[cid] = si;
tbls->topo[cid] = SCX_CID_TOPO_NEG;
tbls->topo[cid].shard_cid = shard_start[si];
tbls->topo[cid].shard_idx = si;
}
/* Rebuild shard_ranges[] for the new layout. */
memset(tbls->shard_ranges, 0, npossible * sizeof(*tbls->shard_ranges));
for (si = 0; si < nr_shards; si++) {
u32 end = (si + 1 < nr_shards) ? shard_start[si + 1] : npossible;
tbls->shard_ranges[si].base_cid = shard_start[si];
tbls->shard_ranges[si].nr_cids = end - shard_start[si];
}
tbls->nr_shards = nr_shards;
}
/**
@@ -395,21 +675,25 @@ __bpf_kfunc s32 scx_bpf_cpu_to_cid(s32 cpu, const struct bpf_prog_aux *aux)
* bits outside stay untouched. In particular, scx_cmask_copy() does NOT zero
* @dst bits that lie outside @src's range.
*
* The _RACY variants are otherwise identical to their non-racy counterpart but
* read @src word-by-word via data_race(). Memory ordering with concurrent
* writers is the caller's responsibility.
* Word accesses use READ_ONCE/WRITE_ONCE so a caller may read @src
* locklessly. Memory ordering against concurrent writers is the caller's
* responsibility.
*/
enum cmask_op2 {
/* mutating */
CMASK_OP2_AND,
CMASK_OP2_OR,
CMASK_OP2_OR_RACY,
CMASK_OP2_COPY,
CMASK_OP2_COPY_RACY,
CMASK_OP2_ANDNOT,
/* predicates - short-circuit when the per-word result is true */
CMASK_OP2_SUBSET,
CMASK_OP2_INTERSECTS,
/*
* @a is a BPF-arena cmask. Words on @a use READ_ONCE/WRITE_ONCE since
* BPF may read/write concurrently. See scx_cmask_ref_or() / _copy().
*/
CMASK_OP2_REF_OR,
CMASK_OP2_REF_COPY,
};
static __always_inline bool cmask_op2_is_pred(const enum cmask_op2 op)
@@ -422,28 +706,28 @@ static __always_inline bool cmask_word_op2(u64 *av, const u64 *bp, u64 mask,
{
switch (op) {
case CMASK_OP2_AND:
*av &= ~mask | *bp;
WRITE_ONCE(*av, *av & (~mask | READ_ONCE(*bp)));
return false;
case CMASK_OP2_OR:
*av |= *bp & mask;
return false;
case CMASK_OP2_OR_RACY:
*av |= data_race(*bp) & mask;
WRITE_ONCE(*av, *av | (READ_ONCE(*bp) & mask));
return false;
case CMASK_OP2_COPY:
*av = (*av & ~mask) | (*bp & mask);
return false;
case CMASK_OP2_COPY_RACY:
*av = (*av & ~mask) | (data_race(*bp) & mask);
WRITE_ONCE(*av, (*av & ~mask) | (READ_ONCE(*bp) & mask));
return false;
case CMASK_OP2_ANDNOT:
*av &= ~(*bp & mask);
WRITE_ONCE(*av, *av & ~(READ_ONCE(*bp) & mask));
return false;
case CMASK_OP2_SUBSET:
/* stop on the first bit in @sub not set in @super */
return (*bp & ~*av) & mask;
return (READ_ONCE(*bp) & ~READ_ONCE(*av)) & mask;
case CMASK_OP2_INTERSECTS:
return (*av & *bp) & mask;
return (READ_ONCE(*av) & READ_ONCE(*bp)) & mask;
case CMASK_OP2_REF_OR:
WRITE_ONCE(*av, READ_ONCE(*av) | (READ_ONCE(*bp) & mask));
return false;
case CMASK_OP2_REF_COPY:
WRITE_ONCE(*av, (READ_ONCE(*av) & ~mask) | (READ_ONCE(*bp) & mask));
return false;
}
unreachable();
}
@@ -504,7 +788,7 @@ static __always_inline bool cmask_word_op1(const u64 *ap, u64 mask,
{
switch (op) {
case CMASK_OP1_ANY_SET:
return *ap & mask;
return READ_ONCE(*ap) & mask;
}
unreachable();
}
@@ -556,39 +840,12 @@ void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src)
src->bits, src->base, src->nr_cids, CMASK_OP2_OR);
}
/**
* scx_cmask_or_racy - OR @src into @dst, reading @src without locking
*
* @src is read word-by-word through data_race(). Same per-bit independence
* rationale as scx_cmask_copy_racy(). Memory ordering with writers is the
* caller's responsibility.
*/
void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src)
{
cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,
src->bits, src->base, src->nr_cids, CMASK_OP2_OR_RACY);
}
void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src)
{
cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,
src->bits, src->base, src->nr_cids, CMASK_OP2_COPY);
}
/**
* scx_cmask_copy_racy - Snapshot @src into @dst without locking
*
* @src is read word-by-word through data_race(). Head/tail masking matches
* scx_cmask_copy(). Each bit in a cmask is independent, so partial updates
* just leave some bits fresher than others. Memory ordering with writers is
* the caller's responsibility.
*/
void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src)
{
cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,
src->bits, src->base, src->nr_cids, CMASK_OP2_COPY_RACY);
}
void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src)
{
cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,
@@ -660,33 +917,36 @@ bool scx_cmask_empty(const struct scx_cmask *m)
*
* Fill @out__uninit with the topology info for @cid. Trigger scx_error() if
* @cid is out of range. If @cid is valid but in the no-topo section, all fields
* are set to -1.
* are set to -1. All fields are also set to -1 when no cid tables have been
* published yet, which a program may observe while racing the root enable.
*/
__bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out__uninit,
const struct bpf_prog_aux *aux)
{
struct scx_cid_topo *topo;
struct scx_sched *sch;
guard(rcu)();
sch = scx_prog_sched(aux);
if (unlikely(!sch) || !cid_valid(sch, cid)) {
topo = rcu_dereference(scx_cid_topo);
if (unlikely(!sch) || !cid_valid(sch, cid) || unlikely(!topo)) {
*out__uninit = SCX_CID_TOPO_NEG;
return;
}
*out__uninit = READ_ONCE(scx_cid_topo)[cid];
*out__uninit = topo[cid];
}
__bpf_kfunc_end_defs();
BTF_KFUNCS_START(scx_kfunc_ids_init)
BTF_KFUNCS_START(scx_kfunc_ids_init_cids)
BTF_ID_FLAGS(func, scx_bpf_cid_override, KF_IMPLICIT_ARGS | KF_SLEEPABLE)
BTF_KFUNCS_END(scx_kfunc_ids_init)
BTF_KFUNCS_END(scx_kfunc_ids_init_cids)
static const struct btf_kfunc_id_set scx_kfunc_set_init = {
static const struct btf_kfunc_id_set scx_kfunc_set_init_cids = {
.owner = THIS_MODULE,
.set = &scx_kfunc_ids_init,
.set = &scx_kfunc_ids_init_cids,
.filter = scx_kfunc_context_filter,
};
@@ -701,9 +961,205 @@ static const struct btf_kfunc_id_set scx_kfunc_set_cid = {
.set = &scx_kfunc_ids_cid,
};
/**
* scx_cmask_ref_init - Bind a scx_cmask_ref to a BPF-arena cmask
* @sch: scheduler whose arena hosts @src
* @src: BPF-supplied cmask, rebased to its kernel address
* @ref: output ref
*
* Snapshot @src's @base, @nr_cids and @alloc_words. The snapshot is necessary
* because BPF may mutate the live header asynchronously.
*
* Return 0 on success, -EINVAL if the range is out of bounds or @alloc_words
* doesn't cover it.
*/
int scx_cmask_ref_init(struct scx_sched *sch, const struct scx_cmask *src,
struct scx_cmask_ref *ref)
{
u32 base, nr_cids, alloc_words, npossible = num_possible_cpus();
s32 *cid_to_shard;
base = READ_ONCE(src->base);
nr_cids = READ_ONCE(src->nr_cids);
alloc_words = READ_ONCE(src->alloc_words);
if (unlikely(base >= npossible || nr_cids > npossible - base ||
SCX_CMASK_NR_WORDS(nr_cids) > alloc_words))
return -EINVAL;
ref->sch = sch;
ref->src = (struct scx_cmask *)src;
ref->base = base;
ref->nr_cids = nr_cids;
cid_to_shard = rcu_dereference_all(scx_cid_to_shard);
ref->shard_first = cid_to_shard[base];
if (likely(nr_cids))
ref->shard_end = cid_to_shard[base + nr_cids - 1] + 1;
else
ref->shard_end = ref->shard_first;
return 0;
}
/**
* scx_cmask_ref_init_kern - Bind a scx_cmask_ref to a kernel-owned cmask
* @sch: scheduler the cmask belongs to
* @m: kernel address of the target cmask, storage sized for @nr_cids at @base
* @base: first cid of the active range
* @nr_cids: active range length
* @ref: output ref
*
* Like scx_cmask_ref_init() but the geometry is supplied by the caller, not
* read from @m's header, so a concurrent BPF write to the header can't steer
* later sizing or offsets. Rewrite the header from the trusted geometry and
* bind @ref to it.
*/
void scx_cmask_ref_init_kern(struct scx_sched *sch, struct scx_cmask *m,
u32 base, u32 nr_cids, struct scx_cmask_ref *ref)
{
s32 *cid_to_shard;
WRITE_ONCE(m->base, base);
WRITE_ONCE(m->nr_cids, nr_cids);
WRITE_ONCE(m->alloc_words, SCX_CMASK_NR_WORDS(nr_cids));
ref->sch = sch;
ref->src = m;
ref->base = base;
ref->nr_cids = nr_cids;
cid_to_shard = rcu_dereference_all(scx_cid_to_shard);
ref->shard_first = cid_to_shard[base];
if (likely(nr_cids))
ref->shard_end = cid_to_shard[base + nr_cids - 1] + 1;
else
ref->shard_end = ref->shard_first;
}
/**
* scx_cmask_ref_shard - Read one shard from @ref into @out
* @ref: validated ref
* @shard_idx: target shard, in [@ref->shard_first, @ref->shard_end)
* @out: output cmask whose @out->alloc_words must hold the shard
*
* Set @out to the intersection of @ref's range with @shard_idx's cid range,
* with bits[] read from @ref->src via READ_ONCE. Empty intersection sets
* @out->nr_cids to 0. scx_error()s on @ref's sched if @out can't hold the
* shard.
*/
void scx_cmask_ref_shard(const struct scx_cmask_ref *ref, s32 shard_idx,
struct scx_cmask *out)
{
const struct scx_cid_shard *shard =
&rcu_dereference_all(scx_cid_shard_ranges)[shard_idx];
u32 shard_base = shard->base_cid;
u32 shard_end = shard_base + shard->nr_cids;
u32 isect_base, isect_end, nr_words, src_off, wi;
u64 head_mask, tail_mask;
isect_base = max(ref->base, shard_base);
isect_end = min(ref->base + ref->nr_cids, shard_end);
if (isect_base >= isect_end) {
out->base = shard_base;
out->nr_cids = 0;
return;
}
nr_words = ((isect_end - 1) / 64) - (isect_base / 64) + 1;
if (nr_words > out->alloc_words) {
scx_error(ref->sch, "scx_cmask_ref_shard: out alloc_words=%u < %u for shard %d",
out->alloc_words, nr_words, shard_idx);
out->base = shard_base;
out->nr_cids = 0;
return;
}
out->base = isect_base;
out->nr_cids = isect_end - isect_base;
src_off = (isect_base / 64) - (ref->base / 64);
for (wi = 0; wi < nr_words; wi++)
out->bits[wi] = READ_ONCE(ref->src->bits[src_off + wi]);
head_mask = GENMASK_U64(63, isect_base & 63);
out->bits[0] &= head_mask;
tail_mask = GENMASK_U64((isect_end - 1) & 63, 0);
out->bits[nr_words - 1] &= tail_mask;
}
/**
* scx_cmask_ref_or - OR @src into the arena cmask referenced by @ref
* @ref: validated ref
* @src: stable kernel cmask
*
* Bits inside the intersection of @ref's snapshotted range with @src's range
* are OR'd into @ref->src and bits outside are left unchanged. Stores on
* @ref->src use WRITE_ONCE since BPF may read/write concurrently.
*/
void scx_cmask_ref_or(const struct scx_cmask_ref *ref, const struct scx_cmask *src)
{
cmask_walk_op2(ref->src->bits, ref->base, ref->nr_cids,
src->bits, src->base, src->nr_cids, CMASK_OP2_REF_OR);
}
/**
* scx_cmask_ref_copy - Copy @src into the arena cmask referenced by @ref
* @ref: validated ref
* @src: stable kernel cmask
*
* Bits inside the intersection of @ref's snapshotted range with @src's range
* take @src's values and bits outside are left unchanged. Stores on @ref->src
* use WRITE_ONCE since BPF may read/write concurrently.
*/
void scx_cmask_ref_copy(const struct scx_cmask_ref *ref, const struct scx_cmask *src)
{
cmask_walk_op2(ref->src->bits, ref->base, ref->nr_cids,
src->bits, src->base, src->nr_cids, CMASK_OP2_REF_COPY);
}
/**
* scx_cmask_ref_from_cpumask - Populate @ref's arena cmask from a cpumask
* @ref: kern-bound ref, see scx_cmask_ref_init_kern()
* @cpumask: cpus to translate into cids
*
* Write @ref's active range one word at a time, setting each cid's bit when
* its cpu is in @cpumask. Offsets and length come from @ref's trusted geometry
* and stores use WRITE_ONCE since BPF may read concurrently, so the arena
* header is never read.
*/
void scx_cmask_ref_from_cpumask(const struct scx_cmask_ref *ref,
const struct cpumask *cpumask)
{
struct scx_cmask *m = ref->src;
u32 base = ref->base, nr_cids = ref->nr_cids;
u32 wi, nr_words;
if (!nr_cids)
return;
nr_words = (base + nr_cids - 1) / 64 - base / 64 + 1;
for (wi = 0; wi < nr_words; wi++) {
u32 word_first_cid = (base / 64 + wi) * 64;
u64 word = 0;
u32 bit;
for (bit = 0; bit < 64; bit++) {
u32 cid = word_first_cid + bit;
if (cid < base || cid >= base + nr_cids)
continue;
if (cpumask_test_cpu(__scx_cid_to_cpu(cid), cpumask))
word |= BIT_U64(bit);
}
WRITE_ONCE(m->bits[wi], word);
}
}
int scx_cid_kfunc_init(void)
{
return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_init) ?:
return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_init_cids) ?:
register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_cid) ?:
register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &scx_kfunc_set_cid) ?:
register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_cid);

72
kernel/sched/ext/cid.h
View File

@@ -48,25 +48,40 @@ struct scx_sched;
* See the comment above the table definitions in cid.c for the
* memory-ordering and visibility contract.
*/
extern s16 *scx_cid_to_cpu_tbl;
extern s16 *scx_cpu_to_cid_tbl;
extern struct scx_cid_topo *scx_cid_topo;
extern struct btf_id_set8 scx_kfunc_ids_init;
struct scx_cid_tables {
u32 nr_shards;
s16 *cid_to_cpu; /* [num_possible_cpus()] */
s16 *cpu_to_cid; /* [nr_cpu_ids] */
s32 *cid_to_shard; /* [num_possible_cpus()] */
s32 *shard_node; /* [num_possible_cpus()] */
struct scx_cid_shard *shard_ranges; /* [num_possible_cpus()] */
struct scx_cid_topo *topo; /* [num_possible_cpus()] */
struct rcu_head rcu;
};
extern u32 scx_nr_cid_shards;
extern s16 __rcu *scx_cid_to_cpu_tbl;
extern s16 __rcu *scx_cpu_to_cid_tbl;
extern s32 __rcu *scx_cid_to_shard;
extern s32 __rcu *scx_shard_node;
extern struct scx_cid_shard __rcu *scx_cid_shard_ranges;
extern struct scx_cid_topo __rcu *scx_cid_topo;
extern struct btf_id_set8 scx_kfunc_ids_init_cids;
extern struct btf_id_set8 scx_kfunc_ids_cid;
void scx_cmask_clear(struct scx_cmask *m);
void scx_cmask_fill(struct scx_cmask *m);
void scx_cmask_and(struct scx_cmask *dst, const struct scx_cmask *src);
void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src);
void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src);
void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src);
void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src);
void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src);
bool scx_cmask_subset(const struct scx_cmask *sub, const struct scx_cmask *super);
bool scx_cmask_intersects(const struct scx_cmask *a, const struct scx_cmask *b);
bool scx_cmask_empty(const struct scx_cmask *m);
s32 scx_cid_init(struct scx_sched *sch);
void scx_cid_publish_tables(void);
void scx_cid_retire_tables(void);
int scx_cid_kfunc_init(void);
void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst);
/**
* cid_valid - Verify a cid value, to be used on ops input args
@@ -88,14 +103,12 @@ static inline bool cid_valid(struct scx_sched *sch, s32 cid)
* __scx_cid_to_cpu - Unchecked cid->cpu table lookup
* @cid: cid to look up. Must be in [0, num_possible_cpus()).
*
* Intended for callsites that have already validated @cid and that hold a
* non-NULL @sch from scx_prog_sched() - a live sched implies the table has
* been allocated, so no NULL check is needed here.
* Intended for callsites that have already validated @cid and that run on a
* live scheduler, which guarantees the tables are published and stable.
*/
static inline s32 __scx_cid_to_cpu(s32 cid)
{
/* READ_ONCE pairs with WRITE_ONCE in scx_cid_arrays_alloc() */
return READ_ONCE(scx_cid_to_cpu_tbl)[cid];
return rcu_dereference_all(scx_cid_to_cpu_tbl)[cid];
}
/**
@@ -106,7 +119,7 @@ static inline s32 __scx_cid_to_cpu(s32 cid)
*/
static inline s32 __scx_cpu_to_cid(s32 cpu)
{
return READ_ONCE(scx_cpu_to_cid_tbl)[cpu];
return rcu_dereference_all(scx_cpu_to_cid_tbl)[cpu];
}
/**
@@ -115,15 +128,19 @@ static inline s32 __scx_cpu_to_cid(s32 cpu)
* @cid: cid to look up
*
* Return the cpu for @cid or a negative errno on failure. Invalid cid triggers
* scx_error() on @sch. The cid arrays are allocated on first scheduler enable
* and never freed, so the returned cpu is stable for the lifetime of the loaded
* scheduler.
* scx_error() on @sch. The mapping is stable while the scheduler is live.
*
* Return -EINVAL without triggering scx_error() if no tables have been
* published yet, which a prog-facing kfunc can observe while racing the root
* scheduler enable.
*/
static inline s32 scx_cid_to_cpu(struct scx_sched *sch, s32 cid)
{
if (!cid_valid(sch, cid))
s16 *tbl = rcu_dereference_all(scx_cid_to_cpu_tbl);
if (!cid_valid(sch, cid) || unlikely(!tbl))
return -EINVAL;
return __scx_cid_to_cpu(cid);
return tbl[cid];
}
/**
@@ -132,13 +149,15 @@ static inline s32 scx_cid_to_cpu(struct scx_sched *sch, s32 cid)
* @cpu: cpu to look up
*
* Return the cid for @cpu or a negative errno on failure. Invalid cpu triggers
* scx_error() on @sch. Same lifetime guarantee as scx_cid_to_cpu().
* scx_error() on @sch. Same usage rules as scx_cid_to_cpu().
*/
static inline s32 scx_cpu_to_cid(struct scx_sched *sch, s32 cpu)
{
if (!scx_cpu_valid(sch, cpu, NULL))
s16 *tbl = rcu_dereference_all(scx_cpu_to_cid_tbl);
if (!scx_cpu_valid(sch, cpu, NULL) || unlikely(!tbl))
return -EINVAL;
return __scx_cpu_to_cid(cpu);
return tbl[cpu];
}
/**
@@ -291,4 +310,15 @@ static inline s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid)
return scx_cid_to_cpu(sch, cpu_or_cid);
}
int scx_cmask_ref_init(struct scx_sched *sch, const struct scx_cmask *src,
struct scx_cmask_ref *ref);
void scx_cmask_ref_init_kern(struct scx_sched *sch, struct scx_cmask *m,
u32 base, u32 nr_cids, struct scx_cmask_ref *ref);
void scx_cmask_ref_shard(const struct scx_cmask_ref *ref, s32 shard_idx,
struct scx_cmask *out);
void scx_cmask_ref_from_cpumask(const struct scx_cmask_ref *ref,
const struct cpumask *cpumask);
void scx_cmask_ref_or(const struct scx_cmask_ref *ref, const struct scx_cmask *src);
void scx_cmask_ref_copy(const struct scx_cmask_ref *ref, const struct scx_cmask *src);
#endif /* _KERNEL_SCHED_EXT_CID_H */

4057
kernel/sched/ext/ext.c
View File

File diff suppressed because it is too large Load Diff

117
kernel/sched/ext/idle.c
View File

@@ -12,6 +12,7 @@
#include "internal.h"
#include "cid.h"
#include "idle.h"
#include "sub.h"
/* Enable/disable built-in idle CPU selection policy */
static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled);
@@ -732,6 +733,55 @@ static void update_builtin_idle(int cpu, bool idle)
}
}
/*
* Notify schedulers of an idle transition on @cpu's cid, delivering to every
* sched that holds %SCX_CAP_BASE on the cid (the root holds every cap). A real
* transition (@do_notify) reaches all holders. A forced one (@root_renotify for
* the root, a sub-sched's idle_renotify marker for a sub) reaches only the owed
* scheds.
*/
static void scx_idle_notify(struct rq *rq, bool idle, bool do_notify, bool root_renotify)
{
s32 cpu = cpu_of(rq);
s32 cid = scx_cpu_arg(cpu);
struct scx_sched *root = scx_root_protected_live();
struct scx_sched *pos;
lockdep_assert_rq_held(rq);
/* with no sub-sched, only the root can be owed a notification */
if (!scx_has_subs()) {
if ((do_notify || root_renotify) &&
SCX_HAS_OP(root, update_idle) && !scx_bypassing(root, cpu))
SCX_CALL_OP(root, update_idle, rq, cid, idle);
return;
}
pos = scx_next_descendant_pre(NULL, root);
while (pos) {
bool forced = false;
if (unlikely(scx_missing_caps(pos, cpu, SCX_CAP_BASE))) {
pos = scx_skip_subtree_pre(pos, root);
continue;
}
if (!pos->level) {
forced = root_renotify;
}
#ifdef CONFIG_EXT_SUB_SCHED
else if (per_cpu_ptr(pos->pcpu, cpu)->idle_renotify) {
per_cpu_ptr(pos->pcpu, cpu)->idle_renotify = false;
forced = true;
}
#endif
if ((do_notify || forced) && SCX_HAS_OP(pos, update_idle) &&
!scx_bypassing(pos, cpu))
SCX_CALL_OP(pos, update_idle, rq, cid, idle);
pos = scx_next_descendant_pre(pos, root);
}
}
/*
* Update the idle state of a CPU to @idle.
*
@@ -750,44 +800,39 @@ static void update_builtin_idle(int cpu, bool idle)
*/
void __scx_update_idle(struct rq *rq, bool idle, bool do_notify)
{
struct scx_sched *sch = scx_root;
int cpu = cpu_of(rq);
lockdep_assert_rq_held(rq);
/*
* Update the idle masks:
* - for real idle transitions (do_notify == true)
* - for idle-to-idle transitions (indicated by the previous task
* being the idle thread, managed by pick_task_idle())
*
* Skip updating idle masks if the previous task is not the idle
* thread, since set_next_task_idle() has already handled it when
* transitioning from a task to the idle thread (calling this
* function with do_notify == true).
*
* In this way we can avoid updating the idle masks twice,
* unnecessarily.
* pick_task_idle() calls here only on an idle-to-idle re-pick and the
* transitions call with @do_notify, so every reaching call updates the
* masks.
*/
if (static_branch_likely(&scx_builtin_idle_enabled))
if (do_notify || is_idle_task(rq->curr))
update_builtin_idle(cpu, idle);
update_builtin_idle(cpu, idle);
/*
* Trigger ops.update_idle() only when transitioning from a task to
* the idle thread and vice versa.
* ops.update_idle() fires on real idle transitions, indicated by
* @do_notify and managed by put_prev_task_idle()/set_next_task_idle().
* An idle pick also fires it to flush a forced notify owed to a sched
* that missed transitions while bypassed or on a cid it just gained.
* unbypass_renotify_idle() and scx_process_sync_ecaps() arm the per-rq
* gates, and scx_idle_notify() targets the owed scheds.
*
* Idle transitions are indicated by do_notify being set to true,
* managed by put_prev_task_idle()/set_next_task_idle().
*
* This must come after builtin idle update so that BPF schedulers can
* create interlocking between ops.update_idle() and ops.enqueue() -
* This must come after the builtin idle update so that BPF schedulers
* can create interlocking between ops.update_idle() and ops.enqueue() -
* either enqueue() sees the idle bit or update_idle() sees the task
* that enqueue() queued.
*/
if (SCX_HAS_OP(sch, update_idle) && do_notify &&
!scx_bypassing(sch, cpu_of(rq)))
SCX_CALL_OP(sch, update_idle, rq, scx_cpu_arg(cpu_of(rq)), idle);
if (do_notify ||
(idle && (rq->scx.flags &
(SCX_RQ_SUB_IDLE_RENOTIFY | SCX_RQ_ROOT_IDLE_RENOTIFY)))) {
bool root_renotify = rq->scx.flags & SCX_RQ_ROOT_IDLE_RENOTIFY;
rq->scx.flags &= ~(SCX_RQ_SUB_IDLE_RENOTIFY | SCX_RQ_ROOT_IDLE_RENOTIFY);
scx_idle_notify(rq, idle, do_notify, root_renotify);
}
}
static void reset_idle_masks(struct sched_ext_ops *ops)
@@ -795,20 +840,20 @@ static void reset_idle_masks(struct sched_ext_ops *ops)
int node;
/*
* Consider all online cpus idle. Should converge to the actual state
* quickly.
* Start with all CPUs marked busy. The idle masks are populated when
* bypass is lifted and each idle CPU is forced through an idle re-pick.
* This may temporarily omit idle CPUs but never advertises a busy CPU as
* idle.
*/
if (!(ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE)) {
cpumask_copy(idle_cpumask(NUMA_NO_NODE)->cpu, cpu_online_mask);
cpumask_copy(idle_cpumask(NUMA_NO_NODE)->smt, cpu_online_mask);
cpumask_clear(idle_cpumask(NUMA_NO_NODE)->cpu);
cpumask_clear(idle_cpumask(NUMA_NO_NODE)->smt);
return;
}
for_each_node(node) {
const struct cpumask *node_mask = cpumask_of_node(node);
cpumask_and(idle_cpumask(node)->cpu, cpu_online_mask, node_mask);
cpumask_and(idle_cpumask(node)->smt, cpu_online_mask, node_mask);
cpumask_clear(idle_cpumask(node)->cpu);
cpumask_clear(idle_cpumask(node)->smt);
}
}
@@ -1318,7 +1363,7 @@ __bpf_kfunc s32 scx_bpf_pick_idle_cpu_node(const struct cpumask *cpus_allowed,
/**
* scx_bpf_pick_idle_cpu - Pick and claim an idle cpu
* @cpus_allowed: Allowed cpumask
* @flags: %SCX_PICK_IDLE_CPU_* flags
* @flags: %SCX_PICK_IDLE_* flags
* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
*
* Pick and claim an idle cpu in @cpus_allowed. Returns the picked idle cpu
@@ -1365,7 +1410,7 @@ __bpf_kfunc s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed,
* or pick any CPU from @node
* @cpus_allowed: Allowed cpumask
* @node: target NUMA node
* @flags: %SCX_PICK_IDLE_CPU_* flags
* @flags: %SCX_PICK_IDLE_* flags
* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
*
* Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any
@@ -1416,7 +1461,7 @@ __bpf_kfunc s32 scx_bpf_pick_any_cpu_node(const struct cpumask *cpus_allowed,
/**
* scx_bpf_pick_any_cpu - Pick and claim an idle cpu if available or pick any CPU
* @cpus_allowed: Allowed cpumask
* @flags: %SCX_PICK_IDLE_CPU_* flags
* @flags: %SCX_PICK_IDLE_* flags
* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
*
* Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any

138
kernel/sched/ext/inlines.h Normal file
View File

@@ -0,0 +1,138 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
*
* Inline definitions layered on top of internal.h and cid.h.
*
* Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
* Copyright (c) 2026 Tejun Heo <tj@kernel.org>
*/
#ifndef _KERNEL_SCHED_EXT_INLINES_H
#define _KERNEL_SCHED_EXT_INLINES_H
#include "internal.h"
#include "cid.h"
/* what dispatch concluded, consumed by the pick that follows */
enum scx_dsp_verdict {
SCX_DSP_NONE, /* nothing to run */
SCX_DSP_LOCAL, /* local DSQ has tasks */
SCX_DSP_PREV, /* keep running @prev */
SCX_DSP_RETRY, /* pick helpers only: restart the pick */
};
/*
* One user of this function is scx_bpf_sub_dispatch() which can be called
* recursively as sub-sched dispatches nest. Always inline to reduce stack usage
* from the call frame.
*/
static __always_inline enum scx_dsp_verdict
scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
struct task_struct *prev, bool nested)
{
struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
int nr_loops = SCX_DSP_MAX_LOOPS;
s32 cpu = cpu_of(rq);
bool prev_on_sch = (prev->sched_class == &ext_sched_class) &&
scx_task_on_sched(sch, prev);
if (scx_consume_global_dsq(sch, rq))
return SCX_DSP_LOCAL;
if (scx_bypass_dsp_enabled(sch)) {
/* if @sch is bypassing, only the bypass DSQs are active */
if (scx_bypassing(sch, cpu)) {
if (scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0))
return SCX_DSP_LOCAL;
return SCX_DSP_NONE;
}
#ifdef CONFIG_EXT_SUB_SCHED
/*
* If @sch isn't bypassing but its children are, @sch is
* responsible for making forward progress for both its own
* tasks that aren't bypassing and the bypassing descendants'
* tasks. The following implements a simple built-in behavior -
* let each CPU try to run the bypass DSQ every Nth time.
*
* Later, if necessary, we can add an ops flag to suppress the
* auto-consumption and a kfunc to consume the bypass DSQ and,
* so that the BPF scheduler can fully control scheduling of
* bypassed tasks.
*/
struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) &&
scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) {
__scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1);
return SCX_DSP_LOCAL;
}
#endif /* CONFIG_EXT_SUB_SCHED */
}
if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq))
return SCX_DSP_NONE;
dspc->rq = rq;
/*
* The dispatch loop. Because scx_flush_dispatch_buf() may drop the rq
* lock, the local DSQ might still end up empty after a successful
* ops.dispatch(). If the local DSQ is empty even after ops.dispatch()
* produced some tasks, retry. The BPF scheduler may depend on this
* looping behavior to simplify its implementation.
*/
do {
dspc->nr_tasks = 0;
#ifdef CONFIG_EXT_SUB_SCHED
/* stash @prev so that nested invocations can access it */
if (!nested)
rq->scx.sub_dispatch_prev = prev;
#endif
SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu),
prev_on_sch ? prev : NULL);
#ifdef CONFIG_EXT_SUB_SCHED
if (!nested)
rq->scx.sub_dispatch_prev = NULL;
#endif
scx_flush_dispatch_buf(sch, rq);
if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice)
return SCX_DSP_PREV;
if (rq->scx.local_dsq.nr)
return SCX_DSP_LOCAL;
if (scx_consume_global_dsq(sch, rq))
return SCX_DSP_LOCAL;
/*
* ops.dispatch() can trap us in this loop by repeatedly
* dispatching ineligible tasks. Break out once in a while to
* allow the watchdog to run. As IRQ can't be enabled in
* dispatch, we want to complete this scheduling cycle and then
* start a new one. IOW, we want to call resched_curr() on the
* next, most likely idle, task, not the current one. Use
* __scx_bpf_kick_cpu() for deferred kicking.
*/
if (unlikely(!--nr_loops)) {
scx_kick_cpu(sch, cpu, 0);
break;
}
} while (dspc->nr_tasks);
/*
* Prevent the CPU from going idle while bypassed descendants have tasks
* queued. Without this fallback, bypassed tasks could stall if the host
* scheduler's ops.dispatch() doesn't yield any tasks.
*/
if (scx_bypass_dsp_enabled(sch) &&
scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0))
return SCX_DSP_LOCAL;
return SCX_DSP_NONE;
}
#endif /* _KERNEL_SCHED_EXT_INLINES_H */

804
kernel/sched/ext/internal.h
View File

File diff suppressed because it is too large Load Diff

2681
kernel/sched/ext/sub.c Normal file
View File

File diff suppressed because it is too large Load Diff

232
kernel/sched/ext/sub.h Normal file
View File

@@ -0,0 +1,232 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
*
* Sub-scheduler hierarchy support.
*
* Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
* Copyright (c) 2026 Tejun Heo <tj@kernel.org>
*/
#ifndef _KERNEL_SCHED_EXT_SUB_H
#define _KERNEL_SCHED_EXT_SUB_H
#include "internal.h"
#ifdef CONFIG_EXT_SUB_SCHED
struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root);
struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root);
void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch);
struct cgroup *sch_cgroup(struct scx_sched *sch);
void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch);
void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog);
void scx_pstack_recursion_on_caps_updated(struct bpf_prog *prog);
void drain_descendants(struct scx_sched *sch);
void scx_sub_disable(struct scx_sched *sch);
void scx_sub_enable_workfn(struct kthread_work *work);
bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux);
void scx_free_pshards(struct scx_sched *sch);
s32 scx_alloc_pshards(struct scx_sched *sch);
void scx_init_root_caps(struct scx_sched *sch);
void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev);
void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch);
void scx_online_ecaps(struct rq *rq);
void scx_offline_ecaps(struct rq *rq);
void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu);
void scx_discard_stale_ecaps_syncs(void);
struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
struct task_struct *p, u64 *enq_flags);
bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p);
void scx_reenq_reject(struct rq *rq);
void scx_rescue_charge(struct rq *rq, s64 delta_exec);
void scx_rescue_end(struct rq *rq);
bool scx_rescue_keep(struct rq *rq, struct task_struct *p);
void scx_rescue_flush(struct rq *rq);
void scx_rescue_dump(struct seq_buf *s, struct rq *rq);
void scx_rescue_set_knobs(struct scx_sched *sch);
void scx_rescue_init(struct rq *rq);
/*
* cgrp->scx_sched is written by root/sub enable/disable under all of
* scx_enable_mutex, scx_fork_rwsem and cgroup_mutex. A new cgroup inherits the
* parent's sched under just cgroup_mutex but is not yet reachable by the other
* two lock holders. Any one of the three locks stabilizes the association.
*/
static inline struct scx_sched *scx_cgroup_sched(struct cgroup *cgrp)
{
return rcu_dereference_check(cgrp->scx_sched,
lockdep_is_held(&cgroup_mutex) ||
percpu_rwsem_is_held(&scx_fork_rwsem) ||
lockdep_is_held(&scx_enable_mutex));
}
static inline const char *sch_cgrp_path(struct scx_sched *sch)
{
return sch->cgrp_path;
}
/* a dying sub's hot-path influence ends in scx_sched_free_rcu_work() */
static inline void scx_dec_has_subs(struct scx_sched *sch)
{
if (sch->level)
static_branch_dec(&__scx_has_subs);
}
#else /* CONFIG_EXT_SUB_SCHED */
static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; }
static inline struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root) { return NULL; }
static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {}
static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; }
static inline const char *sch_cgrp_path(struct scx_sched *sch) { return "/"; }
static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {}
static inline void drain_descendants(struct scx_sched *sch) { }
static inline void scx_sub_disable(struct scx_sched *sch) { }
static inline void scx_free_pshards(struct scx_sched *sch) {}
static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; }
static inline void scx_init_root_caps(struct scx_sched *sch) {}
static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) {}
static inline void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch) {}
static inline void scx_online_ecaps(struct rq *rq) {}
static inline void scx_offline_ecaps(struct rq *rq) {}
static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) {}
static inline void scx_discard_stale_ecaps_syncs(void) {}
static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; }
static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; }
static inline void scx_reenq_reject(struct rq *rq) {}
static inline void scx_rescue_charge(struct rq *rq, s64 delta_exec) {}
static inline void scx_rescue_end(struct rq *rq) {}
static inline bool scx_rescue_keep(struct rq *rq, struct task_struct *p) { return false; }
static inline void scx_rescue_flush(struct rq *rq) {}
static inline void scx_rescue_dump(struct seq_buf *s, struct rq *rq) {}
static inline void scx_rescue_set_knobs(struct scx_sched *sch) {}
static inline void scx_rescue_init(struct rq *rq) {}
static inline void scx_dec_has_subs(struct scx_sched *sch) {}
#endif /* CONFIG_EXT_SUB_SCHED */
/**
* scx_for_each_descendant_pre - pre-order walk of a sched's descendants
* @pos: iteration cursor
* @root: sched to walk the descendants of
*
* Walk @root's descendants. @root is included in the iteration and the first
* node to be visited. Must be called with scx_enable_mutex, scx_sched_lock, or
* RCU read lock.
*/
#define scx_for_each_descendant_pre(pos, root) \
for ((pos) = scx_next_descendant_pre(NULL, (root)); (pos); \
(pos) = scx_next_descendant_pre((pos), (root)))
#ifdef CONFIG_EXT_SUB_SCHED
/**
* scx_missing_caps - The caps in @needed that @sch lacks on @cpu
* @sch: sched to test
* @cpu: cpu to test on
* @needed: bitmask of SCX_CAP_* values
*
* Return the caps in @needed that @sch lacks for @cpu, 0 if it holds them all.
*/
static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed)
{
u64 ecaps;
/* no sub-scheds, no missing caps */
if (!scx_has_subs())
return 0;
/* root holds every cap on every cpu */
if (!sch->level)
return 0;
ecaps = READ_ONCE(per_cpu_ptr(sch->pcpu, cpu)->ecaps);
return needed & ~ecaps;
}
/*
* Cap semantics: which caps an action requires, and which caps a cap implies.
* Keep all such mappings collected here.
*/
/* map @enq_flags to the SCX_CAP_* bit required for the local-DSQ insert */
static inline u64 scx_caps_for_enq(u64 enq_flags)
{
/* a restored task must be put into the local DSQ regardless of caps */
if (unlikely(enq_flags & SCX_ENQ_IGNORE_CAPS))
return 0;
if (enq_flags & SCX_ENQ_IMMED)
return SCX_CAP_ENQ_IMMED;
return SCX_CAP_ENQ;
}
/* map queued @p to the SCX_CAP_* bit required to stay on its local DSQ */
static inline u64 scx_caps_for_task(struct task_struct *p)
{
if (p->scx.flags & SCX_TASK_IMMED)
return SCX_CAP_ENQ_IMMED;
return SCX_CAP_ENQ;
}
/* the cap @sch needs to preempt @rq's current task, 0 if none */
static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags)
{
struct task_struct *curr = rq->curr;
/* a kernel-forced placement preempts regardless of caps */
if (unlikely(enq_flags & SCX_ENQ_IGNORE_CAPS))
return 0;
/* a non-ext task can't be preempted by ext, own-subtree needs no cap */
if (curr->sched_class != &ext_sched_class ||
scx_is_descendant(scx_task_sched(curr), sch))
return 0;
return SCX_CAP_PREEMPT;
}
/* caps implied by holding @cap */
static inline u64 scx_caps_implied(u64 cap)
{
switch (cap) {
case SCX_CAP_PREEMPT:
return SCX_CAP_ENQ | SCX_CAP_ENQ_IMMED;
case SCX_CAP_ENQ:
return SCX_CAP_ENQ_IMMED;
}
return 0;
}
/* may @p keep running on @rq's cpu? requires baseline cpu access */
static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p)
{
if (!scx_has_subs())
return true;
/* a migration-disabled task is let in without caps, keep it likewise */
if (unlikely(is_migration_disabled(p)))
return true;
return likely(!scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE));
}
/* the task admitted for rescue on @rq, NULL if none */
static inline struct task_struct *scx_rescuee(struct rq *rq)
{
lockdep_assert_rq_held(rq);
if (!scx_has_subs())
return NULL;
return rq->scx.rescue.curr;
}
#else /* CONFIG_EXT_SUB_SCHED */
static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) { return 0; }
static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags) { return 0; }
static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { return true; }
static inline struct task_struct *scx_rescuee(struct rq *rq) { return NULL; }
#endif /* CONFIG_EXT_SUB_SCHED */
#endif /* _KERNEL_SCHED_EXT_SUB_H */

88
kernel/sched/ext/types.h
View File

@@ -19,6 +19,18 @@ enum scx_consts {
SCX_DSP_MAX_LOOPS = 32,
SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ,
/* rescue knob defaults and limits, see scx_rescue_timerfn() */
SCX_RESCUE_DFL_BW_PPT = 20, /* parts per thousand, 2% */
SCX_RESCUE_MAX_BW_PPT = 250, /* 25% */
SCX_RESCUE_DISABLE = U32_MAX, /* disables rescue */
SCX_RESCUE_DFL_QUANTUM_US = 5000,
SCX_RESCUE_MIN_QUANTUM_US = 1000,
SCX_RESCUE_MAX_QUANTUM_US = 100000,
SCX_RESCUE_MIN_SLICE_US = 1000, /* floor of the divided slice */
SCX_RESCUE_OVERLOAD_MULT = 16, /* overload threshold in funding periods */
SCX_RESCUE_MIN_OVERLOAD_MS = 1000,
SCX_RESCUE_MAX_OVERLOAD_MS = 15000,
/* per-CPU chunk size for p->scx.tid allocation, see scx_alloc_tid() */
SCX_TID_CHUNK = 1024,
@@ -41,17 +53,22 @@ enum scx_consts {
SCX_BYPASS_LB_MIN_DELTA_DIV = 4,
SCX_BYPASS_LB_BATCH = 256,
SCX_REENQ_LOCAL_MAX_REPEAT = 256,
SCX_REENQ_MAX_REPEAT = 256,
SCX_SUB_MAX_DEPTH = 4,
};
/*
* Per-cid topology info. For each topology level (core, LLC, node), records
* the first cid in the unit and its global index. Global indices are
* consecutive integers assigned in cid-walk order, so e.g. core_idx ranges
* over [0, nr_cores_at_init) with no gaps. No-topo cids have all fields set
* to -1.
* Per-cid topology info. For each topology level (core, LLC, node) and shard,
* records the first cid in the unit and its global index. Global indices are
* consecutive integers assigned in cid-walk order, so e.g. core_idx ranges over
* [0, nr_cores_at_init) with no gaps. No-topo cids have core/LLC/node fields
* set to -1 but always have valid shard assignments.
*
* Shards are contiguous CID ranges used as scalable locking/work domains for
* sub-scheduler operations. By default each LLC becomes one shard, split into
* smaller shards if the LLC exceeds the target size. No-topo cids are packed
* into their own max-sized shards.
*
* @core_cid: first cid of this cid's core (smt-sibling group)
* @core_idx: global index of that core, in [0, nr_cores_at_init)
@@ -59,6 +76,8 @@ enum scx_consts {
* @llc_idx: global index of that LLC, in [0, nr_llcs_at_init)
* @node_cid: first cid of this cid's NUMA node
* @node_idx: global index of that node, in [0, nr_nodes_at_init)
* @shard_cid: first cid of this cid's shard
* @shard_idx: global index of that shard, in [0, scx_nr_cid_shards)
*/
struct scx_cid_topo {
s32 core_cid;
@@ -67,6 +86,24 @@ struct scx_cid_topo {
s32 llc_idx;
s32 node_cid;
s32 node_idx;
s32 shard_cid;
s32 shard_idx;
};
enum scx_cid_consts {
SCX_CID_SHARD_SIZE_DFL = 24,
SCX_CID_SHARD_MAX_CPUS = 512,
};
/*
* Per-shard metadata for O(1) shard->cid-range lookup.
*
* @base_cid: first cid of the shard
* @nr_cids: number of cids in the shard
*/
struct scx_cid_shard {
s32 base_cid;
s32 nr_cids;
};
/*
@@ -91,7 +128,7 @@ struct scx_cmask {
u32 base;
u32 nr_cids;
u32 alloc_words;
u64 bits[] __counted_by(alloc_words);
u64 bits[];
};
/*
@@ -147,4 +184,41 @@ struct scx_cmask {
#define SCX_CMASK_DEFINE_SHARD(NAME, BASE, NR_CIDS) \
__SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, SCX_CID_SHARD_MAX_CPUS)
/*
* scx_cmask_ref: validated reference to a BPF-arena cmask.
*
* scx_cmask_ref_init() snapshots @base/@nr_cids. The snapshot is what
* downstream code uses for sizing - the live header can be mutated concurrently
* by BPF.
*
* scx_cmask_ref_shard() reads one shard into a cmask. scx_cmask_ref_or() and
* scx_cmask_ref_copy() write back into the referenced arena cmask, bounded by
* the snapshot.
*
* Typical input use:
*
* struct scx_cmask_ref ref;
* SCX_CMASK_DEFINE(shard, 0, SCX_CID_SHARD_MAX_CPUS);
* s32 idx, ret;
*
* ret = scx_cmask_ref_init(sch, src, &ref);
* if (ret < 0)
* return ret;
*
* for (idx = ref.shard_first; idx < ref.shard_end; idx++) {
* scx_cmask_ref_shard(&ref, idx, shard);
* if (!shard->nr_cids)
* continue;
* ... use idx and shard ...
* }
*/
struct scx_cmask_ref {
struct scx_sched *sch;
struct scx_cmask *src;
u32 base;
u32 nr_cids;
s32 shard_first;
s32 shard_end;
};
#endif /* _KERNEL_SCHED_EXT_TYPES_H */

8
kernel/sched/idle.c
View File

@@ -503,7 +503,13 @@ static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool fir
struct task_struct *pick_task_idle(struct rq *rq, struct rq_flags *rf)
{
scx_update_idle(rq, true, false);
/*
* Notify scx only on an idle-to-idle re-pick (the cpu was already idle).
* A real task->idle transition is delivered by set_next_task_idle(), so
* calling here too would duplicate it.
*/
if (scx_enabled() && is_idle_task(rq->curr))
scx_update_idle(rq, true, false);
return rq->idle;
}

40
kernel/sched/sched.h
View File

@@ -787,39 +787,60 @@ enum scx_rq_flags {
*/
SCX_RQ_ONLINE = 1 << 0,
SCX_RQ_CAN_STOP_TICK = 1 << 1,
SCX_RQ_BAL_KEEP = 1 << 3, /* balance decided to keep current */
SCX_RQ_CLK_VALID = 1 << 5, /* RQ clock is fresh and valid */
SCX_RQ_BAL_CB_PENDING = 1 << 6, /* must queue a cb after dispatching */
SCX_RQ_SUB_IDLE_RENOTIFY = 1 << 7, /* sub-scheds are owed update_idle() */
SCX_RQ_ROOT_IDLE_RENOTIFY = 1 << 8, /* the root is owed update_idle() */
SCX_RQ_IN_WAKEUP = 1 << 16,
SCX_RQ_IN_BALANCE = 1 << 17,
SCX_RQ_IN_DISPATCH = 1 << 17,
};
/* per-rq rescue execution state, see scx_rescue_timerfn() */
struct scx_rq_rescue {
struct scx_dispatch_q dsq; /* stranded tasks awaiting rescue */
s64 budget; /* execution token bucket, ns */
u64 clock; /* last budget accrual timestamp */
struct task_struct *curr; /* task being rescued, one at a time */
s64 slice; /* curr's admitted slice */
u64 exec_snap; /* sum_exec_runtime at admission */
struct timer_list timer; /* paces admission and escalation */
u64 kill_at; /* last ejection, init before any */
};
struct scx_rq {
struct scx_dispatch_q local_dsq;
#ifdef CONFIG_EXT_SUB_SCHED
struct scx_dispatch_q reject_dsq; /* staging for cap-rejected tasks */
struct scx_rq_rescue rescue;
#endif
struct list_head runnable_list; /* runnable tasks on this rq */
struct list_head ddsp_deferred_locals; /* deferred ddsps from enq */
unsigned long ops_qseq;
u64 extra_enq_flags; /* see move_task_to_local_dsq() */
/* both stashed across the activate_task() in move_remote_task_to_local_dsq() */
u64 remote_activate_enq_flags;
struct scx_sched *remote_activate_sch;
u32 nr_running;
u32 cpuperf_target; /* [0, SCHED_CAPACITY_SCALE] */
bool in_select_cpu;
bool cpu_released;
u32 flags;
u32 nr_immed; /* ENQ_IMMED tasks on local_dsq */
#ifdef CONFIG_SCHED_CORE
u32 lock_drop_seq; /* nr dispatch lock releases */
#endif
u64 clock; /* current per-rq clock -- see scx_bpf_now() */
cpumask_var_t cpus_to_kick;
cpumask_var_t cpus_to_kick_if_idle;
cpumask_var_t cpus_to_preempt;
cpumask_var_t cpus_to_wait;
#ifdef CONFIG_EXT_SUB_SCHED
struct llist_head ecaps_to_sync; /* pending ecaps syncs */
struct task_struct *sub_dispatch_prev;
#endif
cpumask_var_t cpus_to_sync;
bool kick_sync_pending;
unsigned long kick_sync;
struct task_struct *sub_dispatch_prev;
struct list_head sched_pcpus_to_kick; /* see kick_cpus_irq_workfn() */
raw_spinlock_t deferred_reenq_lock;
u64 deferred_reenq_locals_seq;
struct list_head deferred_reenq_locals; /* scheds requesting reenq of local DSQ */
struct list_head deferred_reenq_users; /* user DSQs requesting reenq */
struct balance_callback deferred_bal_cb;
@@ -1361,6 +1382,7 @@ struct rq {
unsigned int core_forceidle_seq;
unsigned int core_forceidle_occupation;
u64 core_forceidle_start;
unsigned int core_pick_in_flight;
#endif /* CONFIG_SCHED_CORE */
/* Scratch cpumask to be temporarily used under rq_lock */

77
tools/sched_ext/include/scx/cid.bpf.h
View File

@@ -668,6 +668,83 @@ static __always_inline u32 cmask_next_and_set_wrap(const struct scx_cmask __aren
return found < start ? found : a_end;
}
/*
* Like cmask_next_and_set() but over the intersection of THREE masks. Return
* a->base + a->nr_cids if no cid is set in all three at or after @start.
*/
static __always_inline u32 cmask_next_and2_set(const struct scx_cmask __arena *a,
const struct scx_cmask __arena *b,
const struct scx_cmask __arena *c,
u32 start)
{
u32 a_end = a->base + a->nr_cids;
u32 b_end = b->base + b->nr_cids;
u32 c_end = c->base + c->nr_cids;
u32 a_wbase = a->base / 64;
u32 b_wbase = b->base / 64;
u32 c_wbase = c->base / 64;
u32 lo = a->base > b->base ? a->base : b->base;
u32 hi = a_end < b_end ? a_end : b_end;
u32 last_wi, start_wi, start_bit, i;
lo = lo > c->base ? lo : c->base;
hi = hi < c_end ? hi : c_end;
if (lo >= hi)
return a_end;
if (start < lo)
start = lo;
if (start >= hi)
return a_end;
last_wi = (hi - 1) / 64;
start_wi = start / 64;
start_bit = start & 63;
bpf_for(i, 0, CMASK_MAX_WORDS) {
u32 abs_wi = start_wi + i;
u64 word;
u32 found;
if (abs_wi > last_wi)
break;
word = a->bits[abs_wi - a_wbase] & b->bits[abs_wi - b_wbase] &
c->bits[abs_wi - c_wbase];
if (i == 0)
word &= GENMASK_U64(63, start_bit);
if (!word)
continue;
found = abs_wi * 64 + ctzll(word);
if (found >= hi)
return a_end;
return found;
}
return a_end;
}
/*
* Round-robin variant of cmask_next_and2_set(): wrap to @a->base if the
* three-way intersection has no cid in the forward half. Return a->base +
* a->nr_cids if empty.
*/
static __always_inline u32 cmask_next_and2_set_wrap(const struct scx_cmask __arena *a,
const struct scx_cmask __arena *b,
const struct scx_cmask __arena *c,
u32 start)
{
u32 a_end = a->base + a->nr_cids;
u32 found;
found = cmask_next_and2_set(a, b, c, start);
if (found < a_end || start <= a->base)
return found;
found = cmask_next_and2_set(a, b, c, a->base);
return found < start ? found : a_end;
}
/**
* cmask_from_cpumask - translate a kernel cpumask to a cid-space cmask
* @m: cmask to fill. Zeroed first; only bits within [@m->base, @m->base +

32
tools/sched_ext/include/scx/common.bpf.h
View File

@@ -26,6 +26,7 @@
#include <asm-generic/errno.h>
#include "user_exit_info.bpf.h"
#include "enum_defs.autogen.h"
#include "bpf_arena_common.bpf.h"
#define PF_IDLE 0x00000002 /* I am an IDLE thread */
#define PF_IO_WORKER 0x00000010 /* Task is an IO worker */
@@ -96,7 +97,6 @@ s32 scx_bpf_pick_any_cpu_node(const cpumask_t *cpus_allowed, int node, u64 flags
s32 scx_bpf_pick_any_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym;
bool scx_bpf_task_running(const struct task_struct *p) __ksym;
s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym;
struct rq *scx_bpf_cpu_rq(s32 cpu) __ksym;
struct rq *scx_bpf_locked_rq(void) __ksym;
struct task_struct *scx_bpf_cpu_curr(s32 cpu) __ksym __weak;
struct task_struct *scx_bpf_tid_to_task(u64 tid) __ksym __weak;
@@ -105,7 +105,7 @@ void scx_bpf_events(struct scx_event_stats *events, size_t events__sz) __ksym __
s32 scx_bpf_cpu_to_cid(s32 cpu) __ksym __weak;
s32 scx_bpf_cid_to_cpu(s32 cid) __ksym __weak;
void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out) __ksym __weak;
s32 scx_bpf_kick_cid(s32 cid, u64 flags) __ksym __weak;
void scx_bpf_kick_cid(s32 cid, u64 flags) __ksym __weak;
s32 scx_bpf_task_cid(const struct task_struct *p) __ksym __weak;
s32 scx_bpf_this_cid(void) __ksym __weak;
struct task_struct *scx_bpf_cid_curr(s32 cid) __ksym __weak;
@@ -113,7 +113,13 @@ u32 scx_bpf_nr_cids(void) __ksym __weak;
u32 scx_bpf_nr_online_cids(void) __ksym __weak;
u32 scx_bpf_cidperf_cap(s32 cid) __ksym __weak;
u32 scx_bpf_cidperf_cur(s32 cid) __ksym __weak;
void scx_bpf_cidperf_set(s32 cid, u32 perf) __ksym __weak;
s32 scx_bpf_cidperf_set(s32 cid, u32 perf) __ksym __weak;
/* sub-scheduler cap control, scx_bpf_sub_caps() cgroup_id 0 == self */
s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, const struct scx_cmask __arena *cmask__arena, struct scx_cmask __arena *denied_out__arena__nullable) __ksym __weak;
void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, const struct scx_cmask __arena *cmask__arena) __ksym __weak;
s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask __arena *out__arena) __ksym __weak;
s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt, unsigned long long *data, u32 data__sz) __ksym __weak;
/*
* Use the following as @it__iter when calling scx_bpf_dsq_move[_vtime]() from
@@ -160,6 +166,22 @@ void ___scx_bpf_bstr_format_checker(const char *fmt, ...) {}
___scx_bpf_bstr_format_checker(fmt, ##args); \
})
/*
* scx_bpf_sub_kill() wraps the scx_bpf_sub_kill_bstr() kfunc with variadic
* arguments instead of an array of u64. It kills the direct child sub-scheduler
* @cgid, passing the formatted reason to its user space, and evaluates to the
* kfunc's return value. On a kernel without sub-scheduler support the kfunc is
* absent and it returns -EOPNOTSUPP.
*/
#define scx_bpf_sub_kill(cgid, fmt, args...) \
({ \
scx_bpf_bstr_preamble(fmt, args) \
___scx_bpf_bstr_format_checker(fmt, ##args); \
bpf_ksym_exists(scx_bpf_sub_kill_bstr) ? \
scx_bpf_sub_kill_bstr((cgid), ___fmt, ___param, \
sizeof(___param)) : -EOPNOTSUPP; \
})
/*
* scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments
* instead of an array of u64. Invoking this macro will cause the scheduler to
@@ -983,8 +1005,8 @@ extern struct irqtime___local cpu_irqtime __ksym __weak;
static inline struct rq___local *get_current_rq(u32 cpu)
{
/*
* This is a workaround to get an rq pointer since we decided to
* deprecate scx_bpf_cpu_rq().
* This is a workaround to get an rq pointer now that
* scx_bpf_cpu_rq() has been removed.
*
* WARNING: The caller must hold the rq lock for @cpu. This is
* guaranteed when called from scheduling callbacks (ops.running,

42
tools/sched_ext/include/scx/compat.bpf.h
View File

@@ -31,7 +31,7 @@ struct cgroup *scx_bpf_task_cgroup___new(struct task_struct *p) __ksym __weak;
*
* v7.1: scx_bpf_dsq_move_to_local___v2() to add @enq_flags.
*/
bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags) __ksym __weak;
bool scx_bpf_dsq_move_to_local___v2___compat(u64 dsq_id, u64 enq_flags) __ksym __weak;
bool scx_bpf_dsq_move_to_local___v1(u64 dsq_id) __ksym __weak;
void scx_bpf_dsq_move_set_slice___new(struct bpf_iter_scx_dsq *it__iter, u64 slice) __ksym __weak;
void scx_bpf_dsq_move_set_vtime___new(struct bpf_iter_scx_dsq *it__iter, u64 vtime) __ksym __weak;
@@ -45,8 +45,8 @@ bool scx_bpf_dispatch_from_dsq___old(struct bpf_iter_scx_dsq *it__iter, struct t
bool scx_bpf_dispatch_vtime_from_dsq___old(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak;
#define scx_bpf_dsq_move_to_local(dsq_id, enq_flags) \
(bpf_ksym_exists(scx_bpf_dsq_move_to_local___v2) ? \
scx_bpf_dsq_move_to_local___v2((dsq_id), (enq_flags)) : \
(bpf_ksym_exists(scx_bpf_dsq_move_to_local___v2___compat) ? \
scx_bpf_dsq_move_to_local___v2___compat((dsq_id), (enq_flags)) : \
(bpf_ksym_exists(scx_bpf_dsq_move_to_local___v1) ? \
scx_bpf_dsq_move_to_local___v1((dsq_id)) : \
scx_bpf_consume___old((dsq_id))))
@@ -122,15 +122,20 @@ static inline bool scx_bpf_sub_dispatch(u64 cgroup_id)
}
/*
* v7.2: scx_bpf_cid_override() for explicit cpu->cid mapping. Ignore if
* v7.3: scx_bpf_cid_override() for explicit cid and shard mapping. Ignore if
* missing.
*/
void scx_bpf_cid_override___compat(const s32 *cpu_to_cid, u32 cpu_to_cid__sz) __ksym __weak;
void scx_bpf_cid_override___compat(const s32 __arena *cpu_to_cid__arena,
u32 cpu_to_cid_cnt,
const s32 __arena *shard_start__arena,
u32 shard_start_cnt) __ksym __weak;
static inline void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz)
static inline void scx_bpf_cid_override(const s32 __arena *cpu_to_cid, u32 cpu_to_cid_cnt,
const s32 __arena *shard_start, u32 shard_start_cnt)
{
if (bpf_ksym_exists(scx_bpf_cid_override___compat))
return scx_bpf_cid_override___compat(cpu_to_cid, cpu_to_cid__sz);
scx_bpf_cid_override___compat(cpu_to_cid, cpu_to_cid_cnt,
shard_start, shard_start_cnt);
}
/**
@@ -233,23 +238,6 @@ static inline bool __COMPAT_is_enq_cpu_selected(u64 enq_flags)
scx_bpf_pick_any_cpu_node(cpus_allowed, node, flags) : \
scx_bpf_pick_any_cpu(cpus_allowed, flags))
/*
* v6.18: Add a helper to retrieve the current task running on a CPU.
*
* Keep this helper available until v6.20 for compatibility.
*/
static inline struct task_struct *__COMPAT_scx_bpf_cpu_curr(int cpu)
{
struct rq *rq;
if (bpf_ksym_exists(scx_bpf_cpu_curr))
return scx_bpf_cpu_curr(cpu);
rq = scx_bpf_cpu_rq(cpu);
return rq ? rq->curr : NULL;
}
/*
* v6.19: To work around BPF maximum parameter limit, the following kfuncs are
* replaced with variants that pack scalar arguments in a struct. Wrappers are
@@ -414,10 +402,10 @@ static inline void scx_bpf_reenqueue_local(void)
}
/*
* v6.20: New scx_bpf_dsq_reenq() that allows re-enqueues on more DSQs. This
* v7.1: New scx_bpf_dsq_reenq() that allows re-enqueues on more DSQs. This
* will eventually deprecate scx_bpf_reenqueue_local().
*/
void scx_bpf_dsq_reenq___compat(u64 dsq_id, u64 reenq_flags, const struct bpf_prog_aux *aux__prog) __ksym __weak;
void scx_bpf_dsq_reenq___compat(u64 dsq_id, u64 reenq_flags) __ksym __weak;
static inline bool __COMPAT_has_generic_reenq(void)
{
@@ -427,7 +415,7 @@ static inline bool __COMPAT_has_generic_reenq(void)
static inline void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags)
{
if (bpf_ksym_exists(scx_bpf_dsq_reenq___compat))
scx_bpf_dsq_reenq___compat(dsq_id, reenq_flags, NULL);
scx_bpf_dsq_reenq___compat(dsq_id, reenq_flags);
else if (dsq_id == SCX_DSQ_LOCAL && reenq_flags == 0)
scx_bpf_reenqueue_local();
else

51
tools/sched_ext/include/scx/compat.h
View File

@@ -28,7 +28,7 @@ static inline bool __COMPAT_read_enum(const char *type, const char *name, u64 *v
const struct btf_type *t;
const char *n;
s32 tid;
int i;
__u32 i;
__COMPAT_load_vmlinux_btf();
@@ -42,7 +42,7 @@ static inline bool __COMPAT_read_enum(const char *type, const char *name, u64 *v
if (btf_is_enum(t)) {
struct btf_enum *e = btf_enum(t);
for (i = 0; i < BTF_INFO_VLEN(t->info); i++) {
for (i = 0; i < btf_vlen(t); i++) {
n = btf__name_by_offset(__COMPAT_vmlinux_btf, e[i].name_off);
SCX_BUG_ON(!n, "btf__name_by_offset()");
if (!strcmp(n, name)) {
@@ -53,7 +53,7 @@ static inline bool __COMPAT_read_enum(const char *type, const char *name, u64 *v
} else if (btf_is_enum64(t)) {
struct btf_enum64 *e = btf_enum64(t);
for (i = 0; i < BTF_INFO_VLEN(t->info); i++) {
for (i = 0; i < btf_vlen(t); i++) {
n = btf__name_by_offset(__COMPAT_vmlinux_btf, e[i].name_off);
SCX_BUG_ON(!n, "btf__name_by_offset()");
if (!strcmp(n, name)) {
@@ -85,7 +85,7 @@ static inline bool __COMPAT_struct_has_field(const char *type, const char *field
const struct btf_member *m;
const char *n;
s32 tid;
int i;
__u32 i;
__COMPAT_load_vmlinux_btf();
tid = btf__find_by_name_kind(__COMPAT_vmlinux_btf, type, BTF_KIND_STRUCT);
@@ -97,7 +97,7 @@ static inline bool __COMPAT_struct_has_field(const char *type, const char *field
m = btf_members(t);
for (i = 0; i < BTF_INFO_VLEN(t->info); i++) {
for (i = 0; i < btf_vlen(t); i++) {
n = btf__name_by_offset(__COMPAT_vmlinux_btf, m[i].name_off);
SCX_BUG_ON(!n, "btf__name_by_offset()");
if (!strcmp(n, field))
@@ -154,7 +154,7 @@ static inline long scx_hotplug_seq(void)
* struct sched_ext_ops can change over time. Two complementary mechanisms
* keep BPF schedulers built against newer headers running on older kernels:
*
* 1. Load-time fix-up (this macro). For each optional ops callback or field
* 1. Load-time fix-up (SCX_OPS_OPEN()). For each optional ops callback or field
* added to struct sched_ext_ops, an explicit stanza below probes the
* running kernel's BTF via __COMPAT_struct_has_field() and, if the field
* is missing, clears it in the in-memory struct_ops (with a warning to
@@ -175,17 +175,25 @@ static inline long scx_hotplug_seq(void)
* - v6.17: ops.cgroup_set_bandwidth()
* - v6.19: ops.cgroup_set_idle()
* - v7.1: ops.sub_attach(), ops.sub_detach(), ops.sub_cgroup_id
* - v7.3: ops.rescue_bandwidth_ppt, ops.rescue_quantum_us
*/
#define __SCX_OPS_OPEN(__ops_name, __scx_name, __ops_struct) ({ \
struct __scx_name *__oskel; \
\
SCX_BUG_ON(!__COMPAT_struct_has_field(__ops_struct, "dump"), \
__ops_struct ".dump() missing, kernel too old?"); \
\
__oskel = __scx_name##__open(); \
SCX_BUG_ON(!__oskel, "Could not open " #__scx_name); \
__oskel->struct_ops.__ops_name->hotplug_seq = scx_hotplug_seq(); \
SCX_ENUM_INIT(__oskel); \
__oskel; \
})
#define SCX_OPS_OPEN(__ops_name, __scx_name) ({ \
struct __scx_name *__skel; \
\
SCX_BUG_ON(!__COMPAT_struct_has_field("sched_ext_ops", "dump"), \
"sched_ext_ops.dump() missing, kernel too old?"); \
\
__skel = __scx_name##__open(); \
SCX_BUG_ON(!__skel, "Could not open " #__scx_name); \
__skel->struct_ops.__ops_name->hotplug_seq = scx_hotplug_seq(); \
SCX_ENUM_INIT(__skel); \
__skel = __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops"); \
if (__skel->struct_ops.__ops_name->cgroup_set_bandwidth && \
!__COMPAT_struct_has_field("sched_ext_ops", "cgroup_set_bandwidth")) { \
fprintf(stderr, "WARNING: kernel doesn't support ops.cgroup_set_bandwidth()\n"); \
@@ -211,9 +219,26 @@ static inline long scx_hotplug_seq(void)
fprintf(stderr, "WARNING: kernel doesn't support ops.sub_cgroup_id\n"); \
__skel->struct_ops.__ops_name->sub_cgroup_id = 0; \
} \
if (__skel->struct_ops.__ops_name->rescue_bandwidth_ppt > 0 && \
!__COMPAT_struct_has_field("sched_ext_ops", "rescue_bandwidth_ppt")) { \
fprintf(stderr, "WARNING: kernel doesn't support ops.rescue_bandwidth_ppt\n"); \
__skel->struct_ops.__ops_name->rescue_bandwidth_ppt = 0; \
} \
if (__skel->struct_ops.__ops_name->rescue_quantum_us > 0 && \
!__COMPAT_struct_has_field("sched_ext_ops", "rescue_quantum_us")) { \
fprintf(stderr, "WARNING: kernel doesn't support ops.rescue_quantum_us\n"); \
__skel->struct_ops.__ops_name->rescue_quantum_us = 0; \
} \
__skel; \
})
/*
* Open a cid-form (struct sched_ext_ops_cid) skeleton. The cid form postdates
* every op the load-time fix-ups above handle, so none of them apply.
*/
#define SCX_OPS_CID_OPEN(__ops_name, __scx_name) \
__SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops_cid")
/*
* Associate non-struct_ops BPF programs with the scheduler's struct_ops map so
* that scx_prog_sched() can determine which scheduler a BPF program belongs

69
tools/sched_ext/include/scx/enum_defs.autogen.h
View File

@@ -7,9 +7,36 @@
#ifndef __ENUM_DEFS_AUTOGEN_H__
#define __ENUM_DEFS_AUTOGEN_H__
#define HAVE_SCX_ARENA_MIN_ORDER
#define HAVE_SCX_ARENA_GROW_PAGES
#define HAVE___SCX_CAP_ENQ_IMMED
#define HAVE___SCX_CAP_ENQ
#define HAVE___SCX_CAP_PREEMPT
#define HAVE___SCX_CAP_PERF
#define HAVE___SCX_NR_CAPS
#define HAVE___SCX_CAP_ALL
#define HAVE_SCX_CAP_ENQ_IMMED
#define HAVE_SCX_CAP_ENQ
#define HAVE_SCX_CAP_PREEMPT
#define HAVE_SCX_CAP_PERF
#define HAVE_SCX_CAP_BASE
#define HAVE_SCX_CAPS_REENQ_ON_LOSS
#define HAVE_SCX_CID_SHARD_SIZE_DFL
#define HAVE_SCX_CID_SHARD_MAX_CPUS
#define HAVE_SCX_DSP_DFL_MAX_BATCH
#define HAVE_SCX_DSP_MAX_LOOPS
#define HAVE_SCX_WATCHDOG_MAX_TIMEOUT
#define HAVE_SCX_RESCUE_DFL_BW_PPT
#define HAVE_SCX_RESCUE_MAX_BW_PPT
#define HAVE_SCX_RESCUE_DISABLE
#define HAVE_SCX_RESCUE_DFL_QUANTUM_US
#define HAVE_SCX_RESCUE_MIN_QUANTUM_US
#define HAVE_SCX_RESCUE_MAX_QUANTUM_US
#define HAVE_SCX_RESCUE_MIN_SLICE_US
#define HAVE_SCX_RESCUE_OVERLOAD_MULT
#define HAVE_SCX_RESCUE_MIN_OVERLOAD_MS
#define HAVE_SCX_RESCUE_MAX_OVERLOAD_MS
#define HAVE_SCX_TID_CHUNK
#define HAVE_SCX_EXIT_BT_LEN
#define HAVE_SCX_EXIT_MSG_LEN
#define HAVE_SCX_EXIT_DUMP_DFL_LEN
@@ -20,7 +47,7 @@
#define HAVE_SCX_BYPASS_LB_DONOR_PCT
#define HAVE_SCX_BYPASS_LB_MIN_DELTA_DIV
#define HAVE_SCX_BYPASS_LB_BATCH
#define HAVE_SCX_REENQ_LOCAL_MAX_REPEAT
#define HAVE_SCX_REENQ_MAX_REPEAT
#define HAVE_SCX_SUB_MAX_DEPTH
#define HAVE_SCX_CPU_PREEMPT_RT
#define HAVE_SCX_CPU_PREEMPT_DL
@@ -35,6 +62,8 @@
#define HAVE_SCX_DSQ_GLOBAL
#define HAVE_SCX_DSQ_LOCAL
#define HAVE_SCX_DSQ_BYPASS
#define HAVE_SCX_DSQ_REJECT
#define HAVE_SCX_DSQ_RESCUE
#define HAVE_SCX_DSQ_LOCAL_ON
#define HAVE_SCX_DSQ_LOCAL_CPU_MASK
#define HAVE_SCX_DSQ_ITER_REV
@@ -53,6 +82,7 @@
#define HAVE_SCX_ENQ_CPU_SELECTED
#define HAVE_SCX_ENQ_PREEMPT
#define HAVE_SCX_ENQ_IMMED
#define HAVE_SCX_ENQ_RESCUE
#define HAVE_SCX_ENQ_REENQ
#define HAVE_SCX_ENQ_LAST
#define HAVE___SCX_ENQ_INTERNAL_MASK
@@ -60,6 +90,9 @@
#define HAVE_SCX_ENQ_DSQ_PRIQ
#define HAVE_SCX_ENQ_NESTED
#define HAVE_SCX_ENQ_GDSQ_FALLBACK
#define HAVE_SCX_ENQ_IGNORE_CAPS
#define HAVE_SCX_ENQ_APPLY_SLICE
#define HAVE_SCX_ENQ_SLICE_DFL
#define HAVE_SCX_TASK_DSQ_ON_PRIQ
#define HAVE_SCX_TASK_QUEUED
#define HAVE_SCX_TASK_IN_CUSTODY
@@ -67,13 +100,16 @@
#define HAVE_SCX_TASK_DEQD_FOR_SLEEP
#define HAVE_SCX_TASK_SUB_INIT
#define HAVE_SCX_TASK_IMMED
#define HAVE_SCX_TASK_PROTECTED
#define HAVE_SCX_TASK_STATE_SHIFT
#define HAVE_SCX_TASK_STATE_BITS
#define HAVE_SCX_TASK_STATE_MASK
#define HAVE_SCX_TASK_NONE
#define HAVE_SCX_TASK_INIT_BEGIN
#define HAVE_SCX_TASK_INIT
#define HAVE_SCX_TASK_READY
#define HAVE_SCX_TASK_ENABLED
#define HAVE_SCX_TASK_DEAD
#define HAVE_SCX_TASK_REENQ_REASON_SHIFT
#define HAVE_SCX_TASK_REENQ_REASON_BITS
#define HAVE_SCX_TASK_REENQ_REASON_MASK
@@ -81,6 +117,7 @@
#define HAVE_SCX_TASK_REENQ_KFUNC
#define HAVE_SCX_TASK_REENQ_IMMED
#define HAVE_SCX_TASK_REENQ_PREEMPTED
#define HAVE_SCX_TASK_REENQ_CAP
#define HAVE_SCX_TASK_CURSOR
#define HAVE_SCX_ECODE_RSN_HOTPLUG
#define HAVE_SCX_ECODE_RSN_CGROUP_OFFLINE
@@ -93,17 +130,18 @@
#define HAVE_SCX_EXIT_UNREG_KERN
#define HAVE_SCX_EXIT_SYSRQ
#define HAVE_SCX_EXIT_PARENT
#define HAVE_SCX_EXIT_PARENT_KILL
#define HAVE_SCX_EXIT_ERROR
#define HAVE_SCX_EXIT_ERROR_BPF
#define HAVE_SCX_EXIT_ERROR_STALL
#define HAVE_SCX_KF_UNLOCKED
#define HAVE_SCX_KF_CPU_RELEASE
#define HAVE_SCX_KF_DISPATCH
#define HAVE_SCX_KF_ENQUEUE
#define HAVE_SCX_KF_SELECT_CPU
#define HAVE_SCX_KF_REST
#define HAVE___SCX_KF_RQ_LOCKED
#define HAVE___SCX_KF_TERMINAL
#define HAVE_SCX_EXIT_ERROR_REENQ
#define HAVE_SCX_EXIT_ERROR_RESCUE
#define HAVE_SCX_KF_ALLOW_UNLOCKED
#define HAVE_SCX_KF_ALLOW_INIT_CIDS
#define HAVE_SCX_KF_ALLOW_CPU_RELEASE
#define HAVE_SCX_KF_ALLOW_DISPATCH
#define HAVE_SCX_KF_ALLOW_ENQUEUE
#define HAVE_SCX_KF_ALLOW_SELECT_CPU
#define HAVE_SCX_KICK_IDLE
#define HAVE_SCX_KICK_PREEMPT
#define HAVE_SCX_KICK_WAIT
@@ -121,6 +159,7 @@
#define HAVE_SCX_OPS_ALLOW_QUEUED_WAKEUP
#define HAVE_SCX_OPS_BUILTIN_IDLE_PER_NODE
#define HAVE_SCX_OPS_ALWAYS_ENQ_IMMED
#define HAVE_SCX_OPS_TID_TO_TASK
#define HAVE_SCX_OPS_ALL_FLAGS
#define HAVE___SCX_OPS_INTERNAL_MASK
#define HAVE_SCX_OPS_HAS_CPU_PREEMPT
@@ -136,6 +175,7 @@
#define HAVE_SCX_SLICE_BYPASS
#define HAVE_SCX_SLICE_INF
#define HAVE_SCX_REENQ_ANY
#define HAVE_SCX_REENQ_CAP_REVOKE
#define HAVE___SCX_REENQ_FILTER_MASK
#define HAVE___SCX_REENQ_USER_MASK
#define HAVE_SCX_REENQ_TSR_RQ_OPEN
@@ -143,14 +183,23 @@
#define HAVE___SCX_REENQ_TSR_MASK
#define HAVE_SCX_RQ_ONLINE
#define HAVE_SCX_RQ_CAN_STOP_TICK
#define HAVE_SCX_RQ_BAL_KEEP
#define HAVE_SCX_RQ_CLK_VALID
#define HAVE_SCX_RQ_BAL_CB_PENDING
#define HAVE_SCX_RQ_SUB_IDLE_RENOTIFY
#define HAVE_SCX_RQ_ROOT_IDLE_RENOTIFY
#define HAVE_SCX_RQ_IN_WAKEUP
#define HAVE_SCX_RQ_IN_BALANCE
#define HAVE_SCX_RQ_IN_DISPATCH
#define HAVE_SCX_SCHED_PCPU_BYPASSING
#define HAVE_SCX_SLICE_OOB_DUR_BITS
#define HAVE_SCX_SLICE_OOB_ID_BITS
#define HAVE_SCX_SLICE_OOB_DUR_MASK
#define HAVE_SCX_SLICE_OOB_ID_SHIFT
#define HAVE_SCX_SLICE_OOB_ID_MASK
#define HAVE_SCX_SLICE_OOB_PENDING
#define HAVE_SCX_TG_ONLINE
#define HAVE_SCX_TG_INITED
#define HAVE_SCX_TG_SUB_INIT
#define HAVE_SCX_WAKE_FORK
#define HAVE_SCX_WAKE_TTWU
#define HAVE_SCX_WAKE_SYNC

9
tools/sched_ext/include/scx/enums.autogen.bpf.h
View File

@@ -22,9 +22,6 @@ const volatile u64 __SCX_RQ_CAN_STOP_TICK __weak;
const volatile u64 __SCX_RQ_BAL_PENDING __weak;
#define SCX_RQ_BAL_PENDING __SCX_RQ_BAL_PENDING
const volatile u64 __SCX_RQ_BAL_KEEP __weak;
#define SCX_RQ_BAL_KEEP __SCX_RQ_BAL_KEEP
const volatile u64 __SCX_RQ_BYPASSING __weak;
#define SCX_RQ_BYPASSING __SCX_RQ_BYPASSING
@@ -37,6 +34,9 @@ const volatile u64 __SCX_RQ_IN_WAKEUP __weak;
const volatile u64 __SCX_RQ_IN_BALANCE __weak;
#define SCX_RQ_IN_BALANCE __SCX_RQ_IN_BALANCE
const volatile u64 __SCX_RQ_IN_DISPATCH __weak;
#define SCX_RQ_IN_DISPATCH __SCX_RQ_IN_DISPATCH
const volatile u64 __SCX_DSQ_FLAG_BUILTIN __weak;
#define SCX_DSQ_FLAG_BUILTIN __SCX_DSQ_FLAG_BUILTIN
@@ -124,6 +124,9 @@ const volatile u64 __SCX_ENQ_PREEMPT __weak;
const volatile u64 __SCX_ENQ_IMMED __weak;
#define SCX_ENQ_IMMED __SCX_ENQ_IMMED
const volatile u64 __SCX_ENQ_RESCUE __weak;
#define SCX_ENQ_RESCUE __SCX_ENQ_RESCUE
const volatile u64 __SCX_ENQ_REENQ __weak;
#define SCX_ENQ_REENQ __SCX_ENQ_REENQ

3
tools/sched_ext/include/scx/enums.autogen.h
View File

@@ -11,11 +11,11 @@
SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_ONLINE); \
SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_CAN_STOP_TICK); \
SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BAL_PENDING); \
SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BAL_KEEP); \
SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BYPASSING); \
SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_CLK_VALID); \
SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_IN_WAKEUP); \
SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_IN_BALANCE); \
SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_IN_DISPATCH); \
SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_FLAG_BUILTIN); \
SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_FLAG_LOCAL_ON); \
SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_INVALID); \
@@ -45,6 +45,7 @@
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_HEAD); \
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_PREEMPT); \
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_IMMED); \
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_RESCUE); \
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_REENQ); \
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_LAST); \
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_CLEAR_OPSS); \

2
tools/sched_ext/scx_central.c
View File

@@ -120,7 +120,7 @@ int main(int argc, char **argv)
ecode = UEI_REPORT(skel, uei);
scx_central__destroy(skel);
if (UEI_ECODE_RESTART(ecode))
if (!exit_req && UEI_ECODE_RESTART(ecode))
goto restart;
return 0;
}

4
tools/sched_ext/scx_cpu0.c
View File

@@ -62,7 +62,7 @@ int main(int argc, char **argv)
{
struct scx_cpu0 *skel;
struct bpf_link *link;
__u32 opt;
__s32 opt;
__u64 ecode;
libbpf_set_print(libbpf_print_fn);
@@ -99,7 +99,7 @@ int main(int argc, char **argv)
ecode = UEI_REPORT(skel, uei);
scx_cpu0__destroy(skel);
if (UEI_ECODE_RESTART(ecode))
if (!exit_req && UEI_ECODE_RESTART(ecode))
goto restart;
return 0;
}

22
tools/sched_ext/scx_flatcg.bpf.c
View File

@@ -256,7 +256,7 @@ static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc)
* and thus can't be updated and repositioned. Instead, we collect the
* vtime deltas separately and apply it asynchronously here.
*/
delta = __sync_fetch_and_sub(&cgc->cvtime_delta, cgc->cvtime_delta);
delta = __sync_fetch_and_and(&cgc->cvtime_delta, 0);
cvtime = cgv_node->cvtime + delta;
/*
@@ -570,7 +570,8 @@ void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable)
cgc = find_cgrp_ctx(cgrp);
if (cgc) {
__sync_fetch_and_add(&cgc->cvtime_delta,
p->se.sum_exec_runtime - taskc->bypassed_at);
(p->se.sum_exec_runtime - taskc->bypassed_at) *
FCG_HWEIGHT_ONE / (cgc->hweight ?: 1));
taskc->bypassed_at = 0;
}
bpf_cgroup_release(cgrp);
@@ -604,6 +605,9 @@ void BPF_STRUCT_OPS(fcg_cgroup_set_weight, struct cgroup *cgrp, u32 weight)
pcgc->child_weight_sum += (s64)weight - cgc->weight;
cgc->weight = weight;
bpf_spin_unlock(&cgv_tree_lock);
/* expire cached hweights so the new weight propagates */
__sync_fetch_and_add(&hweight_gen, 1);
}
static bool try_pick_next_cgroup(u64 *cgidp)
@@ -768,10 +772,18 @@ void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev)
* cgroup to execute but the latter needs to be done in a loop
* and we can't keep the lock held. Oh well...
*/
s64 delta = now - cpuc->cur_at - cgrp_slice_ns;
bpf_spin_lock(&cgv_tree_lock);
__sync_fetch_and_add(&cgc->cvtime_delta,
(cpuc->cur_at + cgrp_slice_ns - now) *
FCG_HWEIGHT_ONE / (cgc->hweight ?: 1));
/* keep the dividends positive, BPF division is unsigned */
if (delta >= 0)
__sync_fetch_and_add(&cgc->cvtime_delta,
(u64)delta * FCG_HWEIGHT_ONE /
(cgc->hweight ?: 1));
else
__sync_fetch_and_sub(&cgc->cvtime_delta,
(u64)-delta * FCG_HWEIGHT_ONE /
(cgc->hweight ?: 1));
bpf_spin_unlock(&cgv_tree_lock);
} else {
stat_inc(FCG_STAT_CNS_GONE);

6
tools/sched_ext/scx_flatcg.c
View File

@@ -105,12 +105,12 @@ static void fcg_read_stats(struct scx_flatcg *skel, __u64 *stats)
__u64 *cnts;
__u32 idx;
memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS);
cnts = calloc(skel->rodata->nr_cpus, sizeof(__u64));
if (!cnts)
return;
memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS);
for (idx = 0; idx < FCG_NR_STATS; idx++) {
int ret, cpu;
@@ -233,7 +233,7 @@ int main(int argc, char **argv)
ecode = UEI_REPORT(skel, uei);
scx_flatcg__destroy(skel);
if (UEI_ECODE_RESTART(ecode))
if (!exit_req && UEI_ECODE_RESTART(ecode))
goto restart;
return 0;
}

138
tools/sched_ext/scx_pair.bpf.c
View File

@@ -93,12 +93,13 @@
* -----------------------
*
* SCX is the lowest priority sched_class, and could be preempted by them at
* any time. To address this, the scheduler implements pair_cpu_release() and
* pair_cpu_acquire() callbacks which are invoked by the core scheduler when
* the scheduler loses and gains control of the CPU respectively.
* any time. To address this, the scheduler watches every sched_switch from
* a tracepoint and edge-detects when a CPU leaves and returns to SCX
* control.
*
* In pair_cpu_release(), we mark the pair_ctx as having been preempted, and
* then invoke:
* When a higher-priority class takes a CPU away from a running SCX task -
* a sched_switch from an SCX task to a higher-priority task - we mark the
* pair_ctx as having been preempted and then invoke:
*
* scx_bpf_kick_cpu(pair_cpu, SCX_KICK_PREEMPT | SCX_KICK_WAIT);
*
@@ -107,9 +108,19 @@
* sched_class that preempted our scheduler does not schedule a task
* concurrently with our pair CPU.
*
* When the CPU is re-acquired in pair_cpu_acquire(), we unmark the preemption
* in the pair_ctx, and send another resched IPI to the pair CPU to re-enable
* pair scheduling.
* When the CPU returns to SCX or idle, we unmark the preemption in the
* pair_ctx and send another resched IPI to the pair CPU to re-enable pair
* scheduling.
*
* A switch from idle straight to a higher-priority task is not a release:
* the CPU was not running an SCX task, so there is nothing to drain and no
* reason to make the pair wait. Kicking SCX_KICK_WAIT on every such wakeup
* would stall the pair CPU behind rt bursts it was never coupled to.
*
* Note that sched_setscheduler() on a running task changes its class in
* place without a context switch, so such transitions are only observed at
* the task's next switch. Until then the stale active_mask bit makes the
* pair wait in try_dispatch(), which is bounded by that next switch.
*
* Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
* Copyright (c) 2022 Tejun Heo <tj@kernel.org>
@@ -118,6 +129,8 @@
#include <scx/common.bpf.h>
#include "scx_pair.h"
#define MAX_RT_PRIO 100
char _license[] SEC("license") = "GPL";
/* !0 for veristat, set during init */
@@ -308,6 +321,40 @@ static int lookup_pairc_and_mask(s32 cpu, struct pair_ctx **pairc, u32 *mask)
return 0;
}
/*
* A task is above SCX whenever its effective priority is in the rt/dl
* range. Test p->prio rather than p->policy: rt_mutex_setprio() boosts
* a PI beneficiary into the rt/dl classes with its policy left
* untouched, so a policy test would misclassify boosted tasks in both
* directions. p->prio follows the boost and the deboost.
*
* This still cannot tell fair and SCX tasks apart. It is complete only
* because scx_pair runs in switch-all mode, where no fair class task
* exists; in partial mode fair is also above SCX and can take the CPU.
*/
static bool pair_task_is_highpri(struct task_struct *p)
{
return p->prio < MAX_RT_PRIO;
}
static void pair_cpu_acquire_locked(struct pair_ctx *pairc, u32 in_pair_mask,
u32 *kick_flags)
{
pairc->preempted_mask &= ~in_pair_mask;
/* Kick the pair CPU, unless it was also preempted. */
*kick_flags = !pairc->preempted_mask ? SCX_KICK_PREEMPT : 0;
}
static void pair_cpu_release_locked(struct pair_ctx *pairc, u32 in_pair_mask,
u32 *kick_flags)
{
pairc->preempted_mask |= in_pair_mask;
pairc->active_mask &= ~in_pair_mask;
/* Kick the pair CPU if it's still running. */
*kick_flags = pairc->active_mask ? SCX_KICK_PREEMPT | SCX_KICK_WAIT : 0;
pairc->draining = true;
}
__attribute__((noinline))
static int try_dispatch(s32 cpu)
{
@@ -500,61 +547,60 @@ void BPF_STRUCT_OPS(pair_dispatch, s32 cpu, struct task_struct *prev)
}
}
void BPF_STRUCT_OPS(pair_cpu_acquire, s32 cpu, struct scx_cpu_acquire_args *args)
SEC("tp_btf/sched_switch")
int BPF_PROG(pair_sched_switch, bool preempt, struct task_struct *prev,
struct task_struct *next, unsigned int prev_state)
{
int ret;
s32 cpu = bpf_get_smp_processor_id();
u32 in_pair_mask;
struct pair_ctx *pairc;
bool kick_pair;
u32 kick_flags = 0;
bool preempted;
bool release, acquire;
ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask);
if (ret)
return;
return 0;
/*
* This runs on every context switch in the system. A CPU's own
* preempted_mask bit is only ever written by this tracepoint
* running on that CPU, so the unlocked read is exact and the
* pair-shared lock is only taken on actual transitions.
*/
preempted = pairc->preempted_mask & in_pair_mask;
if (next->pid && pair_task_is_highpri(next)) {
/* an SCX task lost the CPU to a higher-priority class */
release = !preempted && prev->pid && !pair_task_is_highpri(prev);
acquire = false;
} else {
/* the CPU is back under SCX control (or idle) */
release = false;
acquire = preempted;
}
if (!release && !acquire)
return 0;
bpf_spin_lock(&pairc->lock);
pairc->preempted_mask &= ~in_pair_mask;
/* Kick the pair CPU, unless it was also preempted. */
kick_pair = !pairc->preempted_mask;
if (release) {
pair_cpu_release_locked(pairc, in_pair_mask, &kick_flags);
__sync_fetch_and_add(&nr_preemptions, 1);
} else {
pair_cpu_acquire_locked(pairc, in_pair_mask, &kick_flags);
}
bpf_spin_unlock(&pairc->lock);
if (kick_pair) {
if (kick_flags) {
s32 *pair = (s32 *)ARRAY_ELEM_PTR(pair_cpu, cpu, nr_cpu_ids);
if (pair) {
__sync_fetch_and_add(&nr_kicks, 1);
scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT);
scx_bpf_kick_cpu(*pair, kick_flags);
}
}
}
void BPF_STRUCT_OPS(pair_cpu_release, s32 cpu, struct scx_cpu_release_args *args)
{
int ret;
u32 in_pair_mask;
struct pair_ctx *pairc;
bool kick_pair;
ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask);
if (ret)
return;
bpf_spin_lock(&pairc->lock);
pairc->preempted_mask |= in_pair_mask;
pairc->active_mask &= ~in_pair_mask;
/* Kick the pair CPU if it's still running. */
kick_pair = pairc->active_mask;
pairc->draining = true;
bpf_spin_unlock(&pairc->lock);
if (kick_pair) {
s32 *pair = (s32 *)ARRAY_ELEM_PTR(pair_cpu, cpu, nr_cpu_ids);
if (pair) {
__sync_fetch_and_add(&nr_kicks, 1);
scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT | SCX_KICK_WAIT);
}
}
__sync_fetch_and_add(&nr_preemptions, 1);
return 0;
}
s32 BPF_STRUCT_OPS(pair_cgroup_init, struct cgroup *cgrp)
@@ -602,8 +648,6 @@ void BPF_STRUCT_OPS(pair_exit, struct scx_exit_info *ei)
SCX_OPS_DEFINE(pair_ops,
.enqueue = (void *)pair_enqueue,
.dispatch = (void *)pair_dispatch,
.cpu_acquire = (void *)pair_cpu_acquire,
.cpu_release = (void *)pair_cpu_release,
.cgroup_init = (void *)pair_cgroup_init,
.cgroup_exit = (void *)pair_cgroup_exit,
.exit = (void *)pair_exit,

2
tools/sched_ext/scx_pair.c
View File

@@ -190,7 +190,7 @@ int main(int argc, char **argv)
ecode = UEI_REPORT(skel, uei);
scx_pair__destroy(skel);
if (UEI_ECODE_RESTART(ecode))
if (!exit_req && UEI_ECODE_RESTART(ecode))
goto restart;
return 0;
}

1024
tools/sched_ext/scx_qmap.bpf.c
View File

File diff suppressed because it is too large Load Diff

336
tools/sched_ext/scx_qmap.c
View File

@@ -4,6 +4,9 @@
* Copyright (c) 2022 Tejun Heo <tj@kernel.org>
* Copyright (c) 2022 David Vernet <dvernet@meta.com>
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
@@ -12,6 +15,10 @@
#include <libgen.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <limits.h>
#include <string.h>
#include <time.h>
#include <bpf/bpf.h>
#include <scx/common.h>
#include "scx_qmap.h"
@@ -20,11 +27,27 @@
const char help_fmt[] =
"A simple five-level FIFO queue sched_ext scheduler.\n"
"\n"
"See the top-level comment in .bpf.c for more details.\n"
"It also demonstrates hierarchical sub-scheduling: a scheduler can hand some\n"
"of its cpus to a child cgroup that runs its own scheduler. Run one qmap as\n"
"the parent, then run another qmap on a child cgroup with -c to attach it\n"
"beneath the parent.\n"
"\n"
"The policy below is deliberately simplistic and the resulting behavior can\n"
"look odd. qmap is a demo: it exists to exercise every sub-scheduling\n"
"primitive the kernel offers with as little code as possible, not to schedule\n"
"well.\n"
"\n"
"A parent divides the full cpus it holds among itself and its children in\n"
"proportion to cpu.weight. The cpus left over by rounding are time-shared,\n"
"handed to each participant in turn every -R ms. A cpu a scheduler only\n"
"holds a time-share of is never handed further down, and a parent left with\n"
"no full cpu of its own shuts its children down.\n"
"\n"
"See the top-of-file comment in .bpf.c for the design.\n"
"\n"
"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-b COUNT]\n"
" [-N COUNT] [-P] [-M] [-H] [-c CG_PATH] [-d PID] [-D LEN] [-S] [-p] [-I]\n"
" [-F COUNT] [-v]\n"
" [-F COUNT] [-i SEC] [-R MS] [-J MODE] [-v]\n"
"\n"
" -s SLICE_US Override slice duration\n"
" -e COUNT Trigger scx_bpf_error() after COUNT enqueues\n"
@@ -43,7 +66,14 @@ const char help_fmt[] =
" -p Switch only tasks on SCHED_EXT policy instead of all\n"
" -I Turn on SCX_OPS_ALWAYS_ENQ_IMMED\n"
" -F COUNT IMMED stress: force every COUNT'th enqueue to a busy local DSQ (use with -I)\n"
" -C MODE cid-override test (shuffle|bad-dup|bad-range)\n"
" -C MODE cid-override test (shuffle|bad-dup|bad-range|bad-mono)\n"
" -i SEC Stats interval, seconds (default 5)\n"
" -R MS Round-robin period for time-shared cpus, ms (default 200)\n"
" -J MODE Fault injection (wrong-cid: dispatch to a cid not held,\n"
" init-fail/cgrp-init-fail: fail init_task/cpuctl_init for\n"
" \"qmfail*\" comms/cgroups)\n"
" -B PPT Rescue bandwidth in parts per thousand, 0 disables (root only, default 20)\n"
" -q US Rescue batch quantum in microseconds (root only, default 5000)\n"
" -v Print libbpf debug messages\n"
" -h Display this help and exit\n";
@@ -62,14 +92,157 @@ static void sigint_handler(int dummy)
exit_req = 1;
}
static void invoke_flush_alloc(struct scx_qmap *skel)
{
LIBBPF_OPTS(bpf_test_run_opts, opts);
bpf_prog_test_run_opts(bpf_program__fd(skel->progs.flush_alloc), &opts);
}
/* previous counter snapshots for the per-interval hier stats */
struct hier_prev {
u64 alloc_ns[MAX_SUB_SCHEDS];
u64 self_alloc_ns;
u64 alloc_window_ns;
u64 nr_dsps[MAX_SUB_SCHEDS];
u64 nr_reenq_cap;
u64 nr_reenq_immed;
u64 nr_inject_attempts;
u64 nr_rescue_dsp;
};
/* current wall-clock time as "HH:MM:SS" for the startup and interval headers */
static const char *tstamp(char *buf, size_t sz)
{
time_t now = time(NULL);
strftime(buf, sz, "%H:%M:%S", localtime(&now));
return buf;
}
/* format the cids whose cid_owner[] matches @owner as "0-3,8", "-" if none */
static void format_cid_ranges(struct qmap_arena *qa, s32 owner, char *buf, size_t sz)
{
u32 nr = qa->nr_cids, cid;
size_t off = 0;
s32 start = -1;
buf[0] = '\0';
for (cid = 0; cid <= nr; cid++) {
bool match = cid < nr && qa->part.cid_owner[cid] == owner;
int n;
if (match) {
if (start < 0)
start = cid;
continue;
}
if (start < 0)
continue;
if (start == (s32)cid - 1)
n = snprintf(buf + off, sz - off, "%s%d",
off ? "," : "", start);
else
n = snprintf(buf + off, sz - off, "%s%d-%d",
off ? "," : "", start, cid - 1);
if (n < 0 || (size_t)n >= sz - off) {
strcpy(&buf[sz - 4], "...");
return;
}
off += n;
start = -1;
}
if (!off)
strcpy(buf, "-");
}
/* partition summary + one row per sched: weight, cpus, dispatch rate, cids */
static void print_hier(struct qmap_arena *qa, struct hier_prev *prev, u64 own_cgid)
{
char ranges[128], who[16];
const char *rr = "-";
double secs;
u32 i;
/*
* account_alloc() bumps alloc_window_ns together with the per-owner
* counters, so dividing by the same window yields exact cid counts.
*/
secs = (qa->alloc_window_ns - prev->alloc_window_ns) / 1e9;
prev->alloc_window_ns = qa->alloc_window_ns;
/* resolve the live shared-pool holder */
if (qa->part.nr_shared && qa->part.nr_rr) {
u64 cgid = qa->part.rr_slots[qa->part.rr_pos];
rr = "self";
if (cgid) {
rr = "?";
for (i = 0; i < MAX_SUB_SCHEDS; i++) {
if (qa->sub_sched_ctxs[i].cgroup_id == cgid) {
snprintf(who, sizeof(who), "sub%u", i);
rr = who;
break;
}
}
}
}
format_cid_ranges(qa, CID_SHARED, ranges, sizeof(ranges));
printf("hier : nsub=%llu excl=%u shared=%s rr=%s reenq cap/immed +%llu/+%llu inj=+%llu rescue=+%llu\n",
(unsigned long long)qa->nr_sub_scheds, qa->part.nr_excl, ranges, rr,
(unsigned long long)(qa->nr_reenq_cap - prev->nr_reenq_cap),
(unsigned long long)(qa->nr_reenq_immed - prev->nr_reenq_immed),
(unsigned long long)(qa->nr_inject_attempts - prev->nr_inject_attempts),
(unsigned long long)(qa->nr_rescue_dsp - prev->nr_rescue_dsp));
prev->nr_reenq_cap = qa->nr_reenq_cap;
prev->nr_reenq_immed = qa->nr_reenq_immed;
prev->nr_inject_attempts = qa->nr_inject_attempts;
prev->nr_rescue_dsp = qa->nr_rescue_dsp;
printf("hier : %-4s %10s %4s %6s %8s %s\n",
"", "cgroup", "w", "alloc", "disp/s", "cids");
format_cid_ranges(qa, CID_SELF, ranges, sizeof(ranges));
printf("hier : %-4s %10llu %4u %6.2f %8s %s\n", "self",
(unsigned long long)own_cgid, 100,
secs > 0 ? (qa->self_alloc_ns - prev->self_alloc_ns) / (secs * 1e9) : 0.0,
"-", ranges);
prev->self_alloc_ns = qa->self_alloc_ns;
for (i = 0; i < MAX_SUB_SCHEDS; i++) {
struct sub_sched_ctx *sc = &qa->sub_sched_ctxs[i];
if (!sc->cgroup_id)
continue;
snprintf(who, sizeof(who), "sub%u", i);
format_cid_ranges(qa, i, ranges, sizeof(ranges));
printf("hier : %-4s %10llu %4u %6.2f %8.1f %s\n", who,
(unsigned long long)sc->cgroup_id, sc->weight,
secs > 0 ? (qa->alloc_ns[i] - prev->alloc_ns[i]) / (secs * 1e9) : 0.0,
secs > 0 ? (sc->nr_dsps - prev->nr_dsps[i]) / secs : 0.0,
ranges);
prev->alloc_ns[i] = qa->alloc_ns[i];
prev->nr_dsps[i] = sc->nr_dsps;
}
}
int main(int argc, char **argv)
{
struct scx_qmap *skel;
struct bpf_link *link;
struct qmap_arena *qa;
__u32 test_error_cnt = 0;
__u64 ecode;
int opt;
u32 test_error_cnt = 0;
u64 ecode;
int opt, stats_intv = 5, i, round_robin_ms = 200;
struct hier_prev hprev = {};
const char *sub_cg_path = NULL;
char tbuf[32];
u32 inject_mode = 0;
u64 own_cgid = 0;
s32 cid_override_shard_sz = 4;
libbpf_set_print(libbpf_print_fn);
signal(SIGINT, sigint_handler);
@@ -84,12 +257,13 @@ int main(int argc, char **argv)
}
restart:
optind = 1;
skel = SCX_OPS_OPEN(qmap_ops, scx_qmap);
skel = SCX_OPS_CID_OPEN(qmap_ops, scx_qmap);
skel->rodata->slice_ns = __COMPAT_ENUM_OR_ZERO("scx_public_consts", "SCX_SLICE_DFL");
skel->rodata->max_tasks = 16384;
while ((opt = getopt(argc, argv, "s:e:t:T:l:b:N:PMHc:d:D:SpIF:C:vh")) != -1) {
while ((opt = getopt(argc, argv,
"s:e:t:T:l:b:N:PMHc:d:D:SpIF:C:i:R:J:B:q:vh")) != -1) {
switch (opt) {
case 's':
skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000;
@@ -129,6 +303,8 @@ int main(int argc, char **argv)
}
skel->struct_ops.qmap_ops->sub_cgroup_id = st.st_ino;
skel->rodata->sub_cgroup_id = st.st_ino;
own_cgid = st.st_ino;
sub_cg_path = optarg;
break;
}
case 'd':
@@ -146,7 +322,6 @@ int main(int argc, char **argv)
skel->struct_ops.qmap_ops->flags |= SCX_OPS_SWITCH_PARTIAL;
break;
case 'I':
skel->rodata->always_enq_immed = true;
skel->struct_ops.qmap_ops->flags |= SCX_OPS_ALWAYS_ENQ_IMMED;
break;
case 'F':
@@ -154,33 +329,73 @@ int main(int argc, char **argv)
break;
case 'C': {
u32 nr_cpus = libbpf_num_possible_cpus();
u32 mode, i;
u32 mode;
if (!strcmp(optarg, "shuffle"))
mode = 1;
mode = QMAP_CID_OVR_SHUFFLE;
else if (!strcmp(optarg, "bad-dup"))
mode = 2;
mode = QMAP_CID_OVR_BAD_DUP;
else if (!strcmp(optarg, "bad-range"))
mode = 3;
mode = QMAP_CID_OVR_BAD_RANGE;
else if (!strcmp(optarg, "bad-mono"))
mode = QMAP_CID_OVR_BAD_MONO;
else {
fprintf(stderr, "unknown cid-override mode '%s'\n", optarg);
return 1;
}
skel->rodata->cid_override_mode = mode;
cid_override_shard_sz = 4;
/* shuffle: reversed cpu_to_cid, bad-dup: dup cid 0, bad-range: identity */
for (i = 0; i < nr_cpus; i++) {
if (mode == 1)
skel->bss->cid_override_cpu_to_cid[i] = nr_cpus - 1 - i;
else
skel->bss->cid_override_cpu_to_cid[i] = i;
/*
* bad-mono needs >= 3 shards to build a 0-based but
* non-monotonic shard_start. Shrink the shard size so
* the test runs on any machine with >= 3 cpus.
*/
if (mode == QMAP_CID_OVR_BAD_MONO) {
if (nr_cpus < 3) {
fprintf(stderr, "bad-mono needs >= 3 cpus (have %u)\n",
nr_cpus);
return 1;
}
cid_override_shard_sz = nr_cpus / 3;
}
if (mode == 2 && nr_cpus >= 2)
skel->bss->cid_override_cpu_to_cid[1] = 0;
if (mode == 3)
skel->bss->cid_override_cpu_to_cid[0] = (s32)nr_cpus;
/* shards of shard_sz each */
skel->rodata->cid_override_nr_shards =
(nr_cpus + cid_override_shard_sz - 1) / cid_override_shard_sz;
break;
}
case 'i':
stats_intv = atoi(optarg);
if (stats_intv < 1)
stats_intv = 1;
break;
case 'R':
round_robin_ms = atoi(optarg);
if (round_robin_ms < 10)
round_robin_ms = 10;
break;
case 'J':
if (!strcmp(optarg, "wrong-cid"))
inject_mode = QMAP_INJ_WRONG_CID;
else if (!strcmp(optarg, "init-fail"))
inject_mode = QMAP_INJ_INIT_FAIL;
else if (!strcmp(optarg, "cgrp-init-fail"))
inject_mode = QMAP_INJ_CGRP_INIT_FAIL;
else
inject_mode = strtoul(optarg, NULL, 0);
break;
case 'B': {
u32 ppt = strtoul(optarg, NULL, 0);
if (!ppt)
ppt = __COMPAT_ENUM_OR_ZERO("scx_consts", "SCX_RESCUE_DISABLE");
skel->struct_ops.qmap_ops->rescue_bandwidth_ppt = ppt;
break;
}
case 'q':
skel->struct_ops.qmap_ops->rescue_quantum_us = strtoul(optarg, NULL, 0);
break;
case 'v':
verbose = true;
break;
@@ -190,27 +405,75 @@ int main(int argc, char **argv)
}
}
skel->rodata->round_robin_ns = (u64)round_robin_ms * 1000000;
SCX_OPS_LOAD(skel, qmap_ops, scx_qmap, uei);
link = SCX_OPS_ATTACH(skel, qmap_ops, scx_qmap);
qa = &skel->arena->qa;
/*
* The cid-override arrays live in the arena, which is mmapped at load.
* Populate them before qmap_init_cids() consumes them at attach.
*/
if (skel->rodata->cid_override_mode) {
u32 mode = skel->rodata->cid_override_mode;
u32 nr_cpus = libbpf_num_possible_cpus();
u32 i;
/* shuffle: reversed cpu_to_cid; others: identity */
for (i = 0; i < nr_cpus; i++) {
if (mode == QMAP_CID_OVR_SHUFFLE)
qa->cid_override_cpu_to_cid[i] = nr_cpus - 1 - i;
else
qa->cid_override_cpu_to_cid[i] = i;
}
if (mode == QMAP_CID_OVR_BAD_DUP && nr_cpus >= 2)
qa->cid_override_cpu_to_cid[1] = 0;
if (mode == QMAP_CID_OVR_BAD_RANGE)
qa->cid_override_cpu_to_cid[0] = (s32)nr_cpus;
for (i = 0; i < skel->rodata->cid_override_nr_shards; i++)
qa->cid_override_shard_start[i] = i * cid_override_shard_sz;
if (mode == QMAP_CID_OVR_BAD_MONO) {
/* swap [1] and [2] to break monotonicity */
s32 tmp = qa->cid_override_shard_start[1];
qa->cid_override_shard_start[1] = qa->cid_override_shard_start[2];
qa->cid_override_shard_start[2] = tmp;
}
}
link = SCX_OPS_ATTACH(skel, qmap_ops, scx_qmap);
qa->test_error_cnt = test_error_cnt;
qa->inject_mode = inject_mode;
if (sub_cg_path)
printf("%s scx_qmap started: sub-scheduler on %s, stats every %ds\n",
tstamp(tbuf, sizeof(tbuf)), sub_cg_path, stats_intv);
else
printf("%s scx_qmap started: root scheduler, stats every %ds\n",
tstamp(tbuf, sizeof(tbuf)), stats_intv);
fflush(stdout);
while (!exit_req && !UEI_EXITED(skel, uei)) {
long nr_enqueued = qa->nr_enqueued;
long nr_dispatched = qa->nr_dispatched;
printf("---- %s ----\n",
tstamp(tbuf, sizeof(tbuf)));
printf("stats : enq=%lu dsp=%lu delta=%ld reenq/cid0=%llu/%llu deq=%llu core=%llu enq_ddsp=%llu\n",
nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched,
qa->nr_reenqueued, qa->nr_reenqueued_cid0,
qa->nr_dequeued,
qa->nr_core_sched_execed,
qa->nr_ddsp_from_enq);
(unsigned long long)qa->nr_reenqueued,
(unsigned long long)qa->nr_reenqueued_cid0,
(unsigned long long)qa->nr_dequeued,
(unsigned long long)qa->nr_core_sched_execed,
(unsigned long long)qa->nr_ddsp_from_enq);
printf(" exp_local=%llu exp_remote=%llu exp_timer=%llu exp_lost=%llu\n",
qa->nr_expedited_local,
qa->nr_expedited_remote,
qa->nr_expedited_from_timer,
qa->nr_expedited_lost);
(unsigned long long)qa->nr_expedited_local,
(unsigned long long)qa->nr_expedited_remote,
(unsigned long long)qa->nr_expedited_from_timer,
(unsigned long long)qa->nr_expedited_lost);
if (__COMPAT_has_ksym("scx_bpf_cidperf_cur"))
printf("cpuperf: cur min/avg/max=%u/%u/%u target min/avg/max=%u/%u/%u\n",
qa->cpuperf_min,
@@ -219,15 +482,20 @@ int main(int argc, char **argv)
qa->cpuperf_target_min,
qa->cpuperf_target_avg,
qa->cpuperf_target_max);
invoke_flush_alloc(skel);
print_hier(qa, &hprev, own_cgid);
fflush(stdout);
sleep(1);
for (i = 0; i < stats_intv && !exit_req && !UEI_EXITED(skel, uei); i++)
sleep(1);
}
bpf_link__destroy(link);
ecode = UEI_REPORT(skel, uei);
scx_qmap__destroy(skel);
if (UEI_ECODE_RESTART(ecode))
if (!exit_req && UEI_ECODE_RESTART(ecode))
goto restart;
return 0;
}

153
tools/sched_ext/scx_qmap.h
View File

@@ -20,6 +20,7 @@
#endif
#define MAX_SUB_SCHEDS 8
#define MAX_PARTS (MAX_SUB_SCHEDS + 1) /* participants: children + self */
/*
* cpu_ctxs[] is sized to a fixed cap so the layout is shared between BPF and
@@ -27,47 +28,159 @@
*/
#define SCX_QMAP_MAX_CPUS 1024
/*
* An owner id identifies who holds a cid: a child slot in [0, MAX_SUB_SCHEDS),
* CID_SELF for this node, CID_NONE for a cid not currently held, or CID_SHARED
* for a cid in the round-robin pool (its live holder is rr_slots[rr_pos]). Used
* by the partition's cid_owner[].
*/
#define CID_SELF (-1)
#define CID_NONE (-2)
#define CID_SHARED (-3)
/* -C cid-override test modes. Selects cid_override_mode in scx_qmap.bpf.c. */
enum qmap_cid_override {
QMAP_CID_OVR_OFF = 0, /* disabled */
QMAP_CID_OVR_SHUFFLE = 1, /* valid reversed cpu->cid mapping */
QMAP_CID_OVR_BAD_DUP = 2, /* invalid: duplicate cid assignment */
QMAP_CID_OVR_BAD_RANGE = 3, /* invalid: out-of-range cid */
QMAP_CID_OVR_BAD_MONO = 4, /* invalid: non-monotonic shard_start */
};
struct cpu_ctx {
__u64 dsp_idx; /* dispatch index */
__u64 dsp_cnt; /* remaining count */
__u32 avg_weight;
__u32 cpuperf_target;
u64 dsp_idx; /* dispatch index */
u64 dsp_cnt; /* remaining count */
u32 avg_weight;
u32 cpuperf_target;
};
struct qmap_fifo {
struct task_ctx __arena *head;
struct task_ctx __arena *tail;
s32 idx;
};
/* -J fault-injection modes. Selects inject_mode in struct qmap_arena. */
enum qmap_inject {
QMAP_INJ_OFF = 0,
QMAP_INJ_WRONG_CID = 1, /* dispatch to a cid we don't hold */
QMAP_INJ_INIT_FAIL = 2, /* fail init_task for "qmfail*" comms */
QMAP_INJ_CGRP_INIT_FAIL = 3, /* fail cpuctl_init for "qmfail*" cgroups */
};
/*
* scx_cmask's are embedded in struct qmap_arena with inline backing storage.
* The bpf side uses &field.mask with the normal cmask_* helpers. Userspace
* doesn't have access to the type definition and sees same-sized opaque words.
* _Static_assert()'s in .bpf.c ensure that they are in sync.
*/
#define QMAP_CMASK_WORDS (((SCX_QMAP_MAX_CPUS) + 63) / 64 + 1)
struct qmap_cmask {
#ifdef __BPF__
union {
struct scx_cmask mask;
u64 words[QMAP_CMASK_WORDS + 2];
};
#else
u64 words[QMAP_CMASK_WORDS + 2];
#endif
};
/* Opaque to userspace; defined in scx_qmap.bpf.c. */
struct task_ctx;
struct qmap_fifo {
struct task_ctx __arena *head;
struct task_ctx __arena *tail;
__s32 idx;
/* per-direct-child state for the sub-scheduler */
struct sub_sched_ctx {
u64 cgroup_id;
u32 weight; /* cpu.weight, seeded at attach, then set_weight */
u64 nr_dsps;
struct qmap_cmask granted_cids; /* cids granted excl to this child */
struct qmap_cmask prev_granted; /* last grant, for delta calculation */
};
/*
* compute_partition() builds the following from this node's held caps, and
* apply_partition()/rr_advance() execute it. Userspace only reads for the
* hierarchy display.
*/
struct qmap_partition {
u32 nr_excl; /* number of excl-held (delegatable) cids */
s32 cid_owner[SCX_QMAP_MAX_CPUS]; /* per cid: owner id, or CID_NONE */
s32 shared_cids[MAX_PARTS]; /* the round-robin cid pool */
u32 nr_shared; /* number of shared_cids entries */
u64 rr_slots[MAX_PARTS]; /* rotation order: holder cgroup_id, 0 = self */
u32 nr_rr; /* number of rr_slots entries */
u32 rr_pos; /* current rotation index */
};
struct qmap_arena {
/* userspace-visible stats */
__u64 nr_enqueued, nr_dispatched, nr_reenqueued, nr_reenqueued_cid0;
__u64 nr_dequeued, nr_ddsp_from_enq;
__u64 nr_core_sched_execed;
__u64 nr_expedited_local, nr_expedited_remote;
__u64 nr_expedited_lost, nr_expedited_from_timer;
__u64 nr_highpri_queued;
__u32 test_error_cnt;
__u32 cpuperf_min, cpuperf_avg, cpuperf_max;
__u32 cpuperf_target_min, cpuperf_target_avg, cpuperf_target_max;
u64 nr_enqueued, nr_dispatched, nr_reenqueued, nr_reenqueued_cid0;
u64 nr_dequeued, nr_ddsp_from_enq;
u64 nr_core_sched_execed;
u64 nr_expedited_local, nr_expedited_remote;
u64 nr_expedited_lost, nr_expedited_from_timer;
u64 nr_highpri_queued;
u32 test_error_cnt;
u32 cpuperf_min, cpuperf_avg, cpuperf_max;
u32 cpuperf_target_min, cpuperf_target_avg, cpuperf_target_max;
/* kernel-side runtime state */
__u64 sub_sched_cgroup_ids[MAX_SUB_SCHEDS];
__u64 core_sched_head_seqs[5];
__u64 core_sched_tail_seqs[5];
u64 core_sched_head_seqs[5];
u64 core_sched_tail_seqs[5];
struct cpu_ctx cpu_ctxs[SCX_QMAP_MAX_CPUS];
/* cid-override test input, populated by the loader before attach */
__s32 cid_override_cpu_to_cid[SCX_QMAP_MAX_CPUS];
__s32 cid_override_shard_start[SCX_QMAP_MAX_CPUS];
/* task_ctx slab; allocated and threaded by qmap_init() */
struct task_ctx __arena *task_ctxs;
struct task_ctx __arena *task_free_head;
/* five priority FIFOs, each a doubly-linked list through task_ctx */
struct qmap_fifo fifos[5];
/*
* Hierarchical sub-scheduling state. See the design comment at the top
* of scx_qmap.bpf.c.
*/
u32 nr_cids; /* cid count, cached at init */
/* bpf-owned partition: read by userspace for display */
struct qmap_partition part;
struct sub_sched_ctx sub_sched_ctxs[MAX_SUB_SCHEDS]; /* per-child context */
u64 nr_sub_scheds; /* number of attached children */
/* bpf-internal per-cid state */
u8 cid_shared[SCX_QMAP_MAX_CPUS]; /* per cid: 1 if held shared (ENQ_IMMED-only) */
/* allocated cid-time, charged per owner by account_alloc() */
u64 alloc_ns[MAX_SUB_SCHEDS]; /* per child slot */
u64 self_alloc_ns;
u64 alloc_ts; /* last accounting timestamp */
u64 alloc_window_ns; /* total accounted time, the alloc denominator */
/* bpf-internal cmasks (embedded, see struct qmap_cmask) */
struct qmap_cmask self_cids; /* cids this node runs its own tasks on */
struct qmap_cmask idle_cids; /* idle state of all cids regardless of delegation */
struct qmap_cmask rr_cids; /* the shared pool, as a mask for grant/revoke */
/* scratch cmasks */
struct qmap_cmask to_revoke_cids; /* delta cids to revoke */
struct qmap_cmask to_grant_cids; /* delta cids to grant */
struct qmap_cmask prev_rr_cids; /* previous shared pool, to clear stale grants */
struct qmap_cmask held_excl; /* cids held excl (ENQ): delegatable */
struct qmap_cmask held_shared; /* cids held shared (ENQ_IMMED only): self-local */
/* bpf -> userspace: stats */
u64 nr_reenq_cap; /* SCX_TASK_REENQ_CAP bounces */
u64 nr_reenq_immed; /* SCX_TASK_REENQ_IMMED bounces */
u64 nr_inject_attempts; /* fault-injection: dispatches to an unheld cid */
u64 nr_rescue_dsp; /* SCX_ENQ_RESCUE dispatch attempts */
u32 inject_mode; /* fault-injection mode (QMAP_INJ_*) */
};
#endif /* __SCX_QMAP_H */

4
tools/sched_ext/scx_sdt.c
View File

@@ -44,7 +44,7 @@ int main(int argc, char **argv)
{
struct scx_sdt *skel;
struct bpf_link *link;
__u32 opt;
__s32 opt;
__u64 ecode;
libbpf_set_print(libbpf_print_fn);
@@ -96,7 +96,7 @@ int main(int argc, char **argv)
ecode = UEI_REPORT(skel, uei);
scx_sdt__destroy(skel);
if (UEI_ECODE_RESTART(ecode))
if (!exit_req && UEI_ECODE_RESTART(ecode))
goto restart;
return 0;
}

4
tools/sched_ext/scx_simple.c
View File

@@ -64,7 +64,7 @@ int main(int argc, char **argv)
{
struct scx_simple *skel;
struct bpf_link *link;
__u32 opt;
__s32 opt;
__u64 ecode;
libbpf_set_print(libbpf_print_fn);
@@ -104,7 +104,7 @@ int main(int argc, char **argv)
ecode = UEI_REPORT(skel, uei);
scx_simple__destroy(skel);
if (UEI_ECODE_RESTART(ecode))
if (!exit_req && UEI_ECODE_RESTART(ecode))
goto restart;
return 0;
}

13
tools/sched_ext/scx_userland.c
View File

@@ -52,6 +52,7 @@ static __u32 batch_size = 8;
static bool verbose;
static volatile int exit_req;
static volatile int stats_stop;
static int enqueued_fd, dispatched_fd;
static pthread_t stats_printer;
@@ -286,7 +287,7 @@ static void dispatch_batch(void)
static void *run_stats_printer(void *arg)
{
while (!exit_req) {
while (!stats_stop) {
__u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues, total;
nr_failed_enqueues = skel->bss->nr_failed_enqueues;
@@ -326,7 +327,7 @@ static int spawn_stats_thread(void)
static void pre_bootstrap(int argc, char **argv)
{
int err;
__u32 opt;
__s32 opt;
struct sched_param sched_param = {
.sched_priority = sched_get_priority_max(SCHED_EXT),
};
@@ -374,7 +375,7 @@ static void pre_bootstrap(int argc, char **argv)
static void bootstrap(char *comm)
{
exit_req = 0;
stats_stop = 0;
min_vruntime = 0.0;
__atomic_store_n(&nr_vruntime_enqueues, 0, __ATOMIC_RELAXED);
__atomic_store_n(&nr_vruntime_dispatches, 0, __ATOMIC_RELAXED);
@@ -404,7 +405,7 @@ static void bootstrap(char *comm)
static void sched_main_loop(void)
{
while (!exit_req) {
while (!exit_req && !UEI_EXITED(skel, uei)) {
/*
* Perform the following work in the main user space scheduler
* loop:
@@ -434,13 +435,13 @@ int main(int argc, char **argv)
bootstrap(argv[0]);
sched_main_loop();
exit_req = 1;
stats_stop = 1;
bpf_link__destroy(ops_link);
pthread_join(stats_printer, NULL);
ecode = UEI_REPORT(skel, uei);
scx_userland__destroy(skel);
if (UEI_ECODE_RESTART(ecode))
if (!exit_req && UEI_ECODE_RESTART(ecode))
goto restart;
return 0;
}

49
tools/testing/selftests/sched_ext/allowed_cpus.bpf.c
View File

@@ -15,15 +15,48 @@ UEI_DEFINE(uei);
private(PREF_CPUS) struct bpf_cpumask __kptr * allowed_cpumask;
static void
validate_idle_cpu(const struct task_struct *p, const struct cpumask *allowed, s32 cpu)
validate_local_idle_state(void)
{
if (scx_bpf_test_and_clear_cpu_idle(cpu))
scx_bpf_error("CPU %d should be marked as busy", cpu);
const struct cpumask *idle;
struct task_struct *curr;
s32 cpu = bpf_get_smp_processor_id();
bool cpu_is_idle, curr_is_idle;
if (bpf_cpumask_subset(allowed, p->cpus_ptr) &&
!bpf_cpumask_test_cpu(cpu, allowed))
bpf_rcu_read_lock();
curr = scx_bpf_cpu_curr(cpu);
curr_is_idle = curr && (curr->flags & PF_IDLE);
bpf_rcu_read_unlock();
idle = scx_bpf_get_idle_cpumask();
cpu_is_idle = bpf_cpumask_test_cpu(cpu, idle);
scx_bpf_put_idle_cpumask(idle);
/*
* Unlike a remote selected CPU, the local CPU cannot go through an
* idle re-pick while this callback is running. If it is running a
* non-idle scheduling context, it must not be advertised as idle.
*/
if (!curr_is_idle && cpu_is_idle)
scx_bpf_error("running CPU %d should be marked as busy", cpu);
}
static void
validate_selected_cpu(const struct task_struct *p, s32 cpu)
{
const struct cpumask *allowed = cast_mask(allowed_cpumask);
if (!allowed) {
scx_bpf_error("allowed domain not initialized");
return;
}
if (!bpf_cpumask_test_cpu(cpu, allowed))
scx_bpf_error("CPU %d not in the allowed domain for %d (%s)",
cpu, p->pid, p->comm);
if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr))
scx_bpf_error("CPU %d not in the affinity mask for %d (%s)",
cpu, p->pid, p->comm);
}
s32 BPF_STRUCT_OPS(allowed_cpus_select_cpu,
@@ -32,6 +65,7 @@ s32 BPF_STRUCT_OPS(allowed_cpus_select_cpu,
const struct cpumask *allowed;
s32 cpu;
validate_local_idle_state();
allowed = cast_mask(allowed_cpumask);
if (!allowed) {
scx_bpf_error("allowed domain not initialized");
@@ -43,7 +77,7 @@ s32 BPF_STRUCT_OPS(allowed_cpus_select_cpu,
*/
cpu = scx_bpf_select_cpu_and(p, prev_cpu, wake_flags, allowed, 0);
if (cpu >= 0) {
validate_idle_cpu(p, allowed, cpu);
validate_selected_cpu(p, cpu);
scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, 0);
return cpu;
@@ -59,6 +93,7 @@ void BPF_STRUCT_OPS(allowed_cpus_enqueue, struct task_struct *p, u64 enq_flags)
scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0);
validate_local_idle_state();
allowed = cast_mask(allowed_cpumask);
if (!allowed) {
scx_bpf_error("allowed domain not initialized");
@@ -71,7 +106,7 @@ void BPF_STRUCT_OPS(allowed_cpus_enqueue, struct task_struct *p, u64 enq_flags)
*/
cpu = scx_bpf_select_cpu_and(p, prev_cpu, 0, allowed, 0);
if (cpu >= 0) {
validate_idle_cpu(p, allowed, cpu);
validate_selected_cpu(p, cpu);
scx_bpf_kick_cpu(cpu, SCX_KICK_IDLE);
}
}

20
tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c
View File

@@ -14,18 +14,16 @@ s32 BPF_STRUCT_OPS(ddsp_bogus_dsq_fail_select_cpu, struct task_struct *p,
s32 prev_cpu, u64 wake_flags)
{
s32 cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr, 0);
if (cpu < 0)
cpu = prev_cpu;
if (cpu >= 0) {
/*
* If we dispatch to a bogus DSQ that will fall back to the
* builtin global DSQ, we fail gracefully.
*/
scx_bpf_dsq_insert_vtime(p, 0xcafef00d, SCX_SLICE_DFL,
p->scx.dsq_vtime, 0);
return cpu;
}
return prev_cpu;
/*
* If we dispatch to a bogus DSQ that will fall back to the
* builtin global DSQ, we fail gracefully.
*/
scx_bpf_dsq_insert_vtime(p, 0xcafef00d, SCX_SLICE_DFL,
p->scx.dsq_vtime, 0);
return cpu;
}
void BPF_STRUCT_OPS(ddsp_bogus_dsq_fail_exit, struct scx_exit_info *ei)

13
tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c
View File

@@ -14,15 +14,14 @@ s32 BPF_STRUCT_OPS(ddsp_vtimelocal_fail_select_cpu, struct task_struct *p,
s32 prev_cpu, u64 wake_flags)
{
s32 cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr, 0);
if (cpu < 0)
cpu = prev_cpu;
if (cpu >= 0) {
/* Shouldn't be allowed to vtime dispatch to a builtin DSQ. */
scx_bpf_dsq_insert_vtime(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL,
p->scx.dsq_vtime, 0);
return cpu;
}
/* Shouldn't be allowed to vtime dispatch to a builtin DSQ. */
scx_bpf_dsq_insert_vtime(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL,
p->scx.dsq_vtime, 0);
return prev_cpu;
return cpu;
}
void BPF_STRUCT_OPS(ddsp_vtimelocal_fail_exit, struct scx_exit_info *ei)

1
tools/testing/selftests/sched_ext/exit.c
View File

@@ -31,6 +31,7 @@ static enum scx_test_status run(void *ctx)
continue;
skel = exit__open();
SCX_FAIL_IF(!skel, "Failed to open");
SCX_ENUM_INIT(skel);
skel->rodata->exit_point = tc;
SCX_FAIL_IF(exit__load(skel), "Failed to load skel");

28
tools/testing/selftests/sched_ext/numa.bpf.c
View File

@@ -19,16 +19,31 @@ UEI_DEFINE(uei);
const volatile unsigned int __COMPAT_SCX_PICK_IDLE_IN_NODE;
static bool is_cpu_idle(s32 cpu, int node)
static void validate_local_idle_state(void)
{
const struct cpumask *idle_cpumask;
bool idle;
struct task_struct *curr;
s32 cpu = bpf_get_smp_processor_id();
int node = __COMPAT_scx_bpf_cpu_node(cpu);
bool cpu_is_idle, curr_is_idle;
bpf_rcu_read_lock();
curr = scx_bpf_cpu_curr(cpu);
curr_is_idle = curr && (curr->flags & PF_IDLE);
bpf_rcu_read_unlock();
idle_cpumask = __COMPAT_scx_bpf_get_idle_cpumask_node(node);
idle = bpf_cpumask_test_cpu(cpu, idle_cpumask);
cpu_is_idle = bpf_cpumask_test_cpu(cpu, idle_cpumask);
scx_bpf_put_cpumask(idle_cpumask);
return idle;
/*
* Unlike a remote picked CPU, the local CPU cannot go through an
* idle re-pick while this callback is running. If it is running a
* non-idle scheduling context, it must not be advertised as idle
* in its node's idle cpumask.
*/
if (!curr_is_idle && cpu_is_idle)
scx_bpf_error("running CPU %d should be marked as busy", cpu);
}
s32 BPF_STRUCT_OPS(numa_select_cpu,
@@ -38,6 +53,8 @@ s32 BPF_STRUCT_OPS(numa_select_cpu,
int node = __COMPAT_scx_bpf_cpu_node(task_cpu);
s32 cpu;
validate_local_idle_state();
/*
* We could just use __COMPAT_scx_bpf_pick_any_cpu_node() here,
* since it already tries to pick an idle CPU within the node
@@ -59,9 +76,6 @@ s32 BPF_STRUCT_OPS(numa_select_cpu,
if (cpu < 0 && !bpf_cpumask_test_cpu(task_cpu, p->cpus_ptr))
return prev_cpu;
if (is_cpu_idle(cpu, node))
scx_bpf_error("CPU %d should be marked as busy", cpu);
if (__COMPAT_scx_bpf_cpu_node(cpu) != node)
scx_bpf_error("CPU %d should be in node %d", cpu, node);

36
tools/testing/selftests/sched_ext/prog_run.c
View File

@@ -28,7 +28,8 @@ static enum scx_test_status setup(void **ctx)
static enum scx_test_status run(void *ctx)
{
struct prog_run *skel = ctx;
struct bpf_link *link;
struct bpf_link *link = NULL;
enum scx_test_status status = SCX_TEST_PASS;
int prog_fd, err = 0;
prog_fd = bpf_program__fd(skel->progs.prog_run_syscall);
@@ -42,23 +43,40 @@ static enum scx_test_status run(void *ctx)
link = bpf_map__attach_struct_ops(skel->maps.prog_run_ops);
if (!link) {
SCX_ERR("Failed to attach scheduler");
close(prog_fd);
return SCX_TEST_FAIL;
status = SCX_TEST_FAIL;
goto out;
}
err = bpf_prog_test_run_opts(prog_fd, &topts);
SCX_EQ(err, 0);
if (err) {
SCX_ERR("BPF_PROG_RUN failed (%d)", err);
status = SCX_TEST_FAIL;
goto out;
}
/* Assumes uei.kind is written last */
while (skel->data->uei.kind == EXIT_KIND(SCX_EXIT_NONE))
sched_yield();
SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG_BPF));
SCX_EQ(skel->data->uei.exit_code, 0xdeadbeef);
close(prog_fd);
bpf_link__destroy(link);
if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_UNREG_BPF)) {
SCX_ERR("Unexpected exit kind: %llu",
(unsigned long long)skel->data->uei.kind);
status = SCX_TEST_FAIL;
goto out;
}
if (skel->data->uei.exit_code != 0xdeadbeef) {
SCX_ERR("Unexpected exit code: %lld",
(long long)skel->data->uei.exit_code);
status = SCX_TEST_FAIL;
goto out;
}
return SCX_TEST_PASS;
out:
close(prog_fd);
if (link)
bpf_link__destroy(link);
return status;
}
static void cleanup(void *ctx)