mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-27 11:46:46 -04:00
Code Issues Packages Projects Releases Wiki Activity
Files
9cebfe6504488198b012e746bc6b313f88b95439
linux/init/init_task.c
Linus Torvalds 11260c335e Merge tag 'sched_ext-for-7.3' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext 
Pull sched_ext updates from Tejun Heo:
 "Most of this cycle completes the enqueue-path support for hierarchical
  sub-scheduling, which makes sub-scheduler support feature complete: a
  root BPF scheduler can now hand a cgroup subtree over to a nested
  sub-scheduler together with revocable CPU grants, and the
  sub-scheduler owns all scheduling decisions for its tasks on those
  CPUs.

  Development volume was high and a number of changes plugging holes in
  the new support landed late in the cycle. Also included are core
  scheduling fixes that were completed too late for the v7.2 release and
  are routed through this pull request.

  Sub-scheduler CPU delegation:

   - Parent schedulers now grant and revoke per-CPU capabilities
     (enqueueing, preemption, CPU frequency control) on their children,
     enforced on every path a scheduler can reach a CPU through.
     Previously only dispatching could be delegated; this lets
     sub-schedulers fully schedule their CPUs.

   - Rescue execution: a task whose scheduler doesn't have access to the
     CPUs the task needs to run on starved until the watchdog ejected
     the whole scheduler. The kernel now runs such tasks directly on a
     small bandwidth budget, turning a scheduler-killing failure into
     bounded degradation.

   - Cgroup integration: tasks migrating across a sub-scheduler boundary
     weren't re-homed to the new owner, causing wrong-scheduler
     scheduling and a use-after-free. Sub-schedulers now take over their
     cgroup subtree and receive its cgroup callbacks.

   - Arena objects now cross the kernel/BPF boundary as typed pointer
     arguments, translated transparently by the BPF tree's new arena
     argument support, replacing untyped arguments with manual
     translation.

   - scx_qmap now demonstrates full hierarchical sub-scheduling.

  Other fixes and updates:

   - Robustness improvements: the abort path is now NMI-safe, fixing
     deadlocks when errors are raised from NMI context and making
     hardlockup recovery direct. Reenqueue loops that could monopolize a
     CPU ahead of the watchdog now eject the offending scheduler, and
     stalls are blamed on the scheduler actually responsible.

   - Hardening: BPF-writable arena memory is validated before kernel
     use, and task slice and vtime writes got explicit synchronization
     rules, closing corruption vectors open to buggy or malicious
     schedulers.

   - Core scheduling: sched_ext dispatching can drop the rq lock inside
     the core-wide pick, which let interleaving selections corrupt each
     other's state and hard-hang the machine. The selection now restarts
     when the lock was released. The task ordering callback was also
     invoked with its arguments swapped, and the default ordering is
     updated to work across sub-scheduler boundaries. The fixes are
     marked for stable.

   - Other fixes headed for stable: a task init leak on fork failure
     during enable, tooling compat macros that silently failed to detect
     newer kernels, and a crash on reenqueueing against a destroyed
     dispatch queue.

   - Tooling: scx_pair moves off deprecated callbacks, and the
     deprecated scx_bpf_cpu_rq() kfunc is removed"

* tag 'sched_ext-for-7.3' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext: (144 commits)
  sched_ext: Drop the dead SCX_DEQ_CORE_SCHED_EXEC test in dequeue_task_scx()
  sched_ext: Make core-sched task ordering hierarchy-aware
  sched_ext: Use runnable_at for the default core-sched task ordering
  sched_ext: Fix inverted ops.core_sched_before() invocation
  sched_ext: Move the config-off sub-cap kfunc stubs into sub.c
  sched_ext: Rename balance-era identifiers to dispatch terms
  sched_ext: Drop the stale keep_prev fixup in dispatch_pick()
  sched_ext: Keep kick_sync waiting on the rq's own CPU
  sched_ext: Make SCHED_CLASS_EXT select GENERIC_ALLOCATOR
  sched_ext/scx_flatcg: Fix cvtime true-up on slice expiry
  sched_ext: Don't BUG_ON a destroyed DSQ in process_deferred_reenq_users
  sched_ext: Fix scx_bpf_dsq_move_to_local___v2 compat detection
  sched_ext: Make scx_bpf_events() read the calling scheduler's counters
  sched_ext: Drop unlocked scx_rq_clock_invalidate() from scx_root_disable()
  selftests/sched_ext: Fix flaky ddsp failure tests on busy systems
  selftests/sched_ext: Make numa idle validation race-free
  sched_ext: Fix scx_bpf_dsq_reenq___compat kfunc extern prototype
  sched_ext/scx_flatcg: expire cached hweights on weight changes
  sched_ext: Fix exit_task leak on fork failure during enable
  sched_ext: fix stale references in doc comments
  ...
2026-08-20 11:01:37 -07:00

280 lines
7.8 KiB
C
Raw Blame History

// SPDX-License-Identifier: GPL-2.0
#include <linux/init_task.h>
#include <linux/export.h>
#include <linux/mqueue.h>
#include <linux/sched.h>
#include <linux/sched/sysctl.h>
#include <linux/sched/rt.h>
#include <linux/sched/task.h>
#include <linux/sched/ext.h>
#include <linux/sched/exec_state.h>
#include <linux/user_namespace.h>
#include <linux/init.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/audit.h>
#include <linux/numa.h>
#include <linux/scs.h>
#include <linux/plist.h>
#include <linux/uaccess.h>
static struct signal_struct init_signals = {
.nr_threads = 1,
.thread_head = LIST_HEAD_INIT(init_task.thread_node),
.wait_chldexit = __WAIT_QUEUE_HEAD_INITIALIZER(init_signals.wait_chldexit),
.shared_pending = {
.list = LIST_HEAD_INIT(init_signals.shared_pending.list),
.signal = {{0}}
},
.multiprocess = HLIST_HEAD_INIT,
.rlim = INIT_RLIMITS,
#ifdef CONFIG_CGROUPS
.cgroup_threadgroup_rwsem = __RWSEM_INITIALIZER(init_signals.cgroup_threadgroup_rwsem),
#endif
.cred_guard_mutex = __MUTEX_INITIALIZER(init_signals.cred_guard_mutex),
.exec_update_lock = __RWSEM_INITIALIZER(init_signals.exec_update_lock),
#ifdef CONFIG_POSIX_TIMERS
.posix_timers = HLIST_HEAD_INIT,
.ignored_posix_timers = HLIST_HEAD_INIT,
.cputimer = {
.cputime_atomic = INIT_CPUTIME_ATOMIC,
},
#endif
INIT_CPU_TIMERS(init_signals)
.pids = {
[PIDTYPE_PID] = &init_struct_pid,
[PIDTYPE_TGID] = &init_struct_pid,
[PIDTYPE_PGID] = &init_struct_pid,
[PIDTYPE_SID] = &init_struct_pid,
},
INIT_PREV_CPUTIME(init_signals)
};
static struct sighand_struct init_sighand = {
.count = REFCOUNT_INIT(1),
.action = { { { .sa_handler = SIG_DFL, } }, },
.siglock = __SPIN_LOCK_UNLOCKED(init_sighand.siglock),
.signalfd_wqh = __WAIT_QUEUE_HEAD_INITIALIZER(init_sighand.signalfd_wqh),
};
/* init to 2 - one for init_task, one to ensure it is never freed */
struct task_exec_state init_task_exec_state = {
.count = REFCOUNT_INIT(2),
.dumpable = TASK_DUMPABLE_OWNER,
.user_ns = &init_user_ns,
};
#ifdef CONFIG_SHADOW_CALL_STACK
unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)] = {
[(SCS_SIZE / sizeof(long)) - 1] = SCS_END_MAGIC
};
#endif
/* init to 2 - one for init_task, one to ensure it is never freed */
static struct group_info init_groups = { .usage = REFCOUNT_INIT(2) };
/*
* The initial credentials for the initial task
*/
static struct cred init_cred = {
.usage = ATOMIC_INIT(4),
.uid = GLOBAL_ROOT_UID,
.gid = GLOBAL_ROOT_GID,
.suid = GLOBAL_ROOT_UID,
.sgid = GLOBAL_ROOT_GID,
.euid = GLOBAL_ROOT_UID,
.egid = GLOBAL_ROOT_GID,
.fsuid = GLOBAL_ROOT_UID,
.fsgid = GLOBAL_ROOT_GID,
.securebits = SECUREBITS_DEFAULT,
.cap_inheritable = CAP_EMPTY_SET,
.cap_permitted = CAP_FULL_SET,
.cap_effective = CAP_FULL_SET,
.cap_bset = CAP_FULL_SET,
.user = INIT_USER,
.user_ns = &init_user_ns,
.group_info = &init_groups,
.ucounts = &init_ucounts,
};
/*
* Set up the first task table, touch at your own risk!. Base=0,
* limit=0x1fffff (=2MB)
*/
struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
#ifdef CONFIG_THREAD_INFO_IN_TASK
.thread_info = INIT_THREAD_INFO(init_task),
.stack_refcount = REFCOUNT_INIT(1),
#endif
.__state = 0,
.stack = init_stack,
.usage = REFCOUNT_INIT(2),
.flags = PF_KTHREAD,
.prio = MAX_PRIO - 20,
.static_prio = MAX_PRIO - 20,
.normal_prio = MAX_PRIO - 20,
.policy = SCHED_NORMAL,
.cpus_ptr = &init_task.cpus_mask,
.user_cpus_ptr = NULL,
.cpus_mask = CPU_MASK_ALL,
.max_allowed_capacity = SCHED_CAPACITY_SCALE,
.nr_cpus_allowed= NR_CPUS,
.mm = NULL,
.active_mm = &init_mm,
.exec_state = &init_task_exec_state,
.restart_block = {
.fn = do_no_restart_syscall,
},
.se = {
.group_node = LIST_HEAD_INIT(init_task.se.group_node),
},
.rt = {
.run_list = LIST_HEAD_INIT(init_task.rt.run_list),
.time_slice = RR_TIMESLICE,
},
.tasks = LIST_HEAD_INIT(init_task.tasks),
#ifdef CONFIG_SMP
.pushable_tasks = PLIST_NODE_INIT(init_task.pushable_tasks, MAX_PRIO),
#endif
#ifdef CONFIG_CGROUP_SCHED
.sched_task_group = &root_task_group,
#endif
#ifdef CONFIG_SCHED_CLASS_EXT
.scx = {
.dsq_list.node = LIST_HEAD_INIT(init_task.scx.dsq_list.node),
.sticky_cpu = -1,
.holding_cpu = -1,
.runnable_cpu = -1,
.runnable_node = LIST_HEAD_INIT(init_task.scx.runnable_node),
.runnable_at = INITIAL_JIFFIES,
.ddsp_dsq_id = SCX_DSQ_INVALID,
.slice = SCX_SLICE_DFL,
},
#endif
.ptraced = LIST_HEAD_INIT(init_task.ptraced),
.ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry),
.real_parent = &init_task,
.parent = &init_task,
.children = LIST_HEAD_INIT(init_task.children),
.sibling = LIST_HEAD_INIT(init_task.sibling),
.group_leader = &init_task,
RCU_POINTER_INITIALIZER(real_cred, &init_cred),
RCU_POINTER_INITIALIZER(cred, &init_cred),
.comm = INIT_TASK_COMM,
.thread = INIT_THREAD,
.real_fs = &init_fs,
.fs = &init_fs,
.files = &init_files,
#ifdef CONFIG_IO_URING
.io_uring = NULL,
#endif
.signal = &init_signals,
.sighand = &init_sighand,
.nsproxy = &init_nsproxy,
.pending = {
.list = LIST_HEAD_INIT(init_task.pending.list),
.signal = {{0}}
},
.blocked = {{0}},
.alloc_lock = __SPIN_LOCK_UNLOCKED(init_task.alloc_lock),
.journal_info = NULL,
INIT_CPU_TIMERS(init_task)
.pi_lock = __RAW_SPIN_LOCK_UNLOCKED(init_task.pi_lock),
.blocked_lock = __RAW_SPIN_LOCK_UNLOCKED(init_task.blocked_lock),
.timer_slack_ns = 50000, /* 50 usec default slack */
.thread_pid = &init_struct_pid,
.thread_node = LIST_HEAD_INIT(init_signals.thread_head),
#ifdef CONFIG_AUDIT
.loginuid = INVALID_UID,
.sessionid = AUDIT_SID_UNSET,
#endif
#ifdef CONFIG_PERF_EVENTS
.perf_event_mutex = __MUTEX_INITIALIZER(init_task.perf_event_mutex),
.perf_event_list = LIST_HEAD_INIT(init_task.perf_event_list),
#endif
#ifdef CONFIG_PREEMPT_RCU
.rcu_read_lock_nesting = 0,
.rcu_read_unlock_special.s = 0,
.rcu_node_entry = LIST_HEAD_INIT(init_task.rcu_node_entry),
.rcu_blocked_node = NULL,
#endif
#ifdef CONFIG_TASKS_RCU
.rcu_tasks_holdout = false,
.rcu_tasks_holdout_list = LIST_HEAD_INIT(init_task.rcu_tasks_holdout_list),
.rcu_tasks_idle_cpu = -1,
.rcu_tasks_exit_list = LIST_HEAD_INIT(init_task.rcu_tasks_exit_list),
#endif
#ifdef CONFIG_TASKS_TRACE_RCU
.trc_reader_nesting = 0,
#endif
#ifdef CONFIG_CPUSETS
.mems_allowed_seq = SEQCNT_SPINLOCK_ZERO(init_task.mems_allowed_seq,
&init_task.alloc_lock),
#endif
.blocked_donor = NULL,
#ifdef CONFIG_RT_MUTEXES
.pi_waiters = RB_ROOT_CACHED,
.pi_top_task = NULL,
#endif
INIT_PREV_CPUTIME(init_task)
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
.vtime.seqcount = SEQCNT_ZERO(init_task.vtime_seqcount),
.vtime.starttime = 0,
.vtime.state = VTIME_SYS,
#endif
#ifdef CONFIG_NUMA_BALANCING
.numa_preferred_nid = NUMA_NO_NODE,
.numa_group = NULL,
.numa_faults = NULL,
#endif
#ifdef CONFIG_SCHED_CACHE
.preferred_llc = -1,
.pref_llc_queued = 0,
#endif
#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
.kasan_depth = 1,
#endif
#ifdef CONFIG_KCSAN
.kcsan_ctx = {
.scoped_accesses = {LIST_POISON1, NULL},
},
#endif
#ifdef CONFIG_TRACE_IRQFLAGS
.softirqs_enabled = 1,
#endif
#ifdef CONFIG_LOCKDEP
.lockdep_depth = 0, /* no locks held yet */
.curr_chain_key = INITIAL_CHAIN_KEY,
.lockdep_recursion = 0,
#endif
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
.ret_stack = NULL,
.tracing_graph_pause = ATOMIC_INIT(0),
#endif
#if defined(CONFIG_TRACING) && defined(CONFIG_PREEMPTION)
.trace_recursion = 0,
#endif
#ifdef CONFIG_LIVEPATCH
.patch_state = KLP_TRANSITION_IDLE,
#endif
#ifdef CONFIG_SECURITY
.security = NULL,
#endif
#ifdef CONFIG_SECCOMP_FILTER
.seccomp = { .filter_count = ATOMIC_INIT(0) },
#endif
#ifdef CONFIG_SCHED_MM_CID
.mm_cid = { .cid = MM_CID_UNSET, },
#endif
};
EXPORT_SYMBOL(init_task);
/*
* Initial thread structure. Alignment of this is handled by a special
* linker map entry.
*/
#ifndef CONFIG_THREAD_INFO_IN_TASK
struct thread_info init_thread_info __init_thread_info = INIT_THREAD_INFO(init_task);
#endif
Reference in New Issue View Git Blame Copy Permalink