diff --git a/Documentation/admin-guide/pm/cpufreq.rst b/Documentation/admin-guide/pm/cpufreq.rst index 8831cface585..34baf20cc202 100644 --- a/Documentation/admin-guide/pm/cpufreq.rst +++ b/Documentation/admin-guide/pm/cpufreq.rst @@ -497,7 +497,7 @@ This governor exposes the following tunables: represented by it to be 1.5 times as high as the transition latency (the default):: - # echo `$(($(cat cpuinfo_transition_latency) * 3 / 2))` > ondemand/sampling_rate + # echo $(($(cat cpuinfo_transition_latency) * 3 / 2)) > ondemand/sampling_rate ``up_threshold`` If the estimated CPU load is above this value (in percent), the governor diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index a74a4cf99d22..f8d8288d644f 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -782,6 +782,7 @@ static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy, static void amd_pstate_adjust_perf(struct cpufreq_policy *policy, unsigned long _min_perf, unsigned long target_perf, + unsigned long _max_perf, unsigned long capacity) { u8 max_perf, min_perf, des_perf, cap_perf; diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index b898b6544069..0d0df986fa3d 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -2225,14 +2225,17 @@ EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch); * @policy: cpufreq policy object of the target CPU. * @min_perf: Minimum (required) performance level (units of @capacity). * @target_perf: Target (desired) performance level (units of @capacity). + * @max_perf: Maximum (allowed) performance level (units of @capacity). * @capacity: Capacity of the target CPU. * - * Carry out a fast performance level switch of @cpu without sleeping. + * Carry out a fast performance level adjustment for the CPU represented by + * @policy without sleeping. * * The driver's ->adjust_perf() callback invoked by this function must be - * suitable for being called from within RCU-sched read-side critical sections - * and it is expected to select a suitable performance level equal to or above - * @min_perf and preferably equal to or below @target_perf. + * suitable for calling from within RCU-sched read-side critical sections and + * it is expected to program the processor to select suitable performance + * levels between @min_perf and @max_perf inclusive and preferably close to + * @target_perf going forward for the CPU represented by @policy. * * This function must not be called if policy->fast_switch_enabled is unset. * @@ -2244,9 +2247,10 @@ EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch); void cpufreq_driver_adjust_perf(struct cpufreq_policy *policy, unsigned long min_perf, unsigned long target_perf, + unsigned long max_perf, unsigned long capacity) { - cpufreq_driver->adjust_perf(policy, min_perf, target_perf, capacity); + cpufreq_driver->adjust_perf(policy, min_perf, target_perf, max_perf, capacity); } /** diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 6e984c114d96..2cbf810db66e 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -299,7 +299,6 @@ struct pstate_funcs { static struct pstate_funcs pstate_funcs __read_mostly; static bool hwp_active __ro_after_init; -static int hwp_mode_bdw __ro_after_init; static bool per_cpu_limits __ro_after_init; static bool hwp_forced __ro_after_init; static bool hwp_boost __read_mostly; @@ -587,21 +586,14 @@ static void intel_pstate_hybrid_hwp_adjust(struct cpudata *cpu) hwp_is_hybrid = true; - cpu->pstate.turbo_freq = rounddown(cpu->pstate.turbo_pstate * scaling, - perf_ctl_scaling); - cpu->pstate.max_freq = rounddown(cpu->pstate.max_pstate * scaling, - perf_ctl_scaling); - freq = perf_ctl_max_phys * perf_ctl_scaling; cpu->pstate.max_pstate_physical = intel_pstate_freq_to_hwp(cpu, freq); - freq = cpu->pstate.min_pstate * perf_ctl_scaling; - cpu->pstate.min_freq = freq; /* * Cast the min P-state value retrieved via pstate_funcs.get_min() to * the effective range of HWP performance levels. */ - cpu->pstate.min_pstate = intel_pstate_freq_to_hwp(cpu, freq); + cpu->pstate.min_pstate = intel_pstate_freq_to_hwp(cpu, cpu->pstate.min_freq); } static bool turbo_is_disabled(void) @@ -979,12 +971,10 @@ static int hybrid_get_cost(struct device *dev, unsigned long freq, * capacity. Similarly, P-cores start to be populated when E-cores are * utilized above 60% of the capacity. */ - if (hybrid_get_cpu_type(dev->id) == INTEL_CPU_TYPE_ATOM) { - if (hybrid_has_l3(dev->id)) /* E-core */ - *cost += 1; - } else { /* P-core */ + if (hybrid_get_cpu_type(dev->id) == INTEL_CPU_TYPE_CORE) /* P-core */ *cost += 2; - } + else if (hybrid_has_l3(dev->id)) /* E-core */ + *cost += 1; return 0; } @@ -1185,6 +1175,22 @@ static bool hybrid_clear_max_perf_cpu(void) return ret; } +static void intel_pstate_update_freq_limits(struct cpudata *cpu) +{ + int scaling = cpu->pstate.scaling; + unsigned int turbo_freq = cpu->pstate.turbo_pstate * scaling; + unsigned int max_freq = cpu->pstate.max_pstate * scaling; + int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling; + + if (scaling != perf_ctl_scaling) { + turbo_freq = rounddown(turbo_freq, perf_ctl_scaling); + max_freq = rounddown(max_freq, perf_ctl_scaling); + } + + cpu->pstate.turbo_freq = turbo_freq; + cpu->pstate.max_freq = max_freq; +} + static void __intel_pstate_get_hwp_cap(struct cpudata *cpu) { u64 cap; @@ -1197,20 +1203,8 @@ static void __intel_pstate_get_hwp_cap(struct cpudata *cpu) static void intel_pstate_get_hwp_cap(struct cpudata *cpu) { - int scaling = cpu->pstate.scaling; - __intel_pstate_get_hwp_cap(cpu); - - cpu->pstate.max_freq = cpu->pstate.max_pstate * scaling; - cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * scaling; - if (scaling != cpu->pstate.perf_ctl_scaling) { - int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling; - - cpu->pstate.max_freq = rounddown(cpu->pstate.max_freq, - perf_ctl_scaling); - cpu->pstate.turbo_freq = rounddown(cpu->pstate.turbo_freq, - perf_ctl_scaling); - } + intel_pstate_update_freq_limits(cpu); } static void hybrid_update_capacity(struct cpudata *cpu) @@ -2299,33 +2293,16 @@ static int hwp_get_cpu_scaling(int cpu) return intel_pstate_cppc_get_scaling(cpu); } -static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate) -{ - trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu); - cpu->pstate.current_pstate = pstate; - /* - * Generally, there is no guarantee that this code will always run on - * the CPU being updated, so force the register update to run on the - * right CPU. - */ - wrmsrq_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL, - pstate_funcs.get_val(cpu, pstate)); -} - -static void intel_pstate_set_min_pstate(struct cpudata *cpu) -{ - intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate); -} - static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) { int perf_ctl_scaling = pstate_funcs.get_scaling(); cpu->pstate.max_pstate_physical = pstate_funcs.get_max_physical(cpu->cpu); cpu->pstate.min_pstate = pstate_funcs.get_min(cpu->cpu); + cpu->pstate.min_freq = cpu->pstate.min_pstate * perf_ctl_scaling; cpu->pstate.perf_ctl_scaling = perf_ctl_scaling; - if (hwp_active && !hwp_mode_bdw) { + if (hwp_active) { __intel_pstate_get_hwp_cap(cpu); if (pstate_funcs.get_cpu_scaling) { @@ -2345,19 +2322,13 @@ static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) cpu->pstate.turbo_pstate = pstate_funcs.get_turbo(cpu->cpu); } - if (cpu->pstate.scaling == perf_ctl_scaling) { - cpu->pstate.min_freq = cpu->pstate.min_pstate * perf_ctl_scaling; - cpu->pstate.max_freq = cpu->pstate.max_pstate * perf_ctl_scaling; - cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * perf_ctl_scaling; - } + intel_pstate_update_freq_limits(cpu); if (pstate_funcs.get_aperf_mperf_shift) cpu->aperf_mperf_shift = pstate_funcs.get_aperf_mperf_shift(); if (pstate_funcs.get_vid) pstate_funcs.get_vid(cpu); - - intel_pstate_set_min_pstate(cpu); } /* @@ -2884,6 +2855,19 @@ static void intel_pstate_update_perf_limits(struct cpudata *cpu, cpu->min_perf_ratio); } +static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate) +{ + trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu); + cpu->pstate.current_pstate = pstate; + /* + * Generally, there is no guarantee that this code will always run on + * the CPU being updated, so force the register update to run on the + * right CPU. + */ + wrmsrq_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL, + pstate_funcs.get_val(cpu, pstate)); +} + static int intel_pstate_set_policy(struct cpufreq_policy *policy) { struct cpudata *cpu; @@ -2971,6 +2955,11 @@ static int intel_pstate_verify_policy(struct cpufreq_policy_data *policy) return 0; } +static void intel_pstate_set_min_pstate(struct cpudata *cpu) +{ + intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate); +} + static int intel_cpufreq_cpu_offline(struct cpufreq_policy *policy) { struct cpudata *cpu = all_cpu_data[policy->cpu]; @@ -3063,6 +3052,7 @@ static int __intel_pstate_cpu_init(struct cpufreq_policy *policy) static int intel_pstate_cpu_init(struct cpufreq_policy *policy) { int ret = __intel_pstate_cpu_init(policy); + struct cpudata *cpu; if (ret) return ret; @@ -3073,11 +3063,11 @@ static int intel_pstate_cpu_init(struct cpufreq_policy *policy) */ policy->policy = CPUFREQ_POLICY_POWERSAVE; - if (hwp_active) { - struct cpudata *cpu = all_cpu_data[policy->cpu]; - + cpu = all_cpu_data[policy->cpu]; + if (hwp_active) cpu->epp_cached = intel_pstate_get_epp(cpu, 0); - } + else + intel_pstate_set_min_pstate(cpu); return 0; } @@ -3243,6 +3233,7 @@ static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy *policy, static void intel_cpufreq_adjust_perf(struct cpufreq_policy *policy, unsigned long min_perf, unsigned long target_perf, + unsigned long max_perf, unsigned long capacity) { struct cpudata *cpu = all_cpu_data[policy->cpu]; @@ -3273,7 +3264,13 @@ static void intel_cpufreq_adjust_perf(struct cpufreq_policy *policy, if (min_pstate > cpu->max_perf_ratio) min_pstate = cpu->max_perf_ratio; - max_pstate = min(cap_pstate, cpu->max_perf_ratio); + max_pstate = cap_pstate; + if (max_perf < capacity) + max_pstate = DIV_ROUND_UP(cap_pstate * max_perf, capacity); + + if (max_pstate > cpu->max_perf_ratio) + max_pstate = cpu->max_perf_ratio; + if (max_pstate < min_pstate) max_pstate = min_pstate; @@ -3301,8 +3298,6 @@ static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy) return ret; policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY; - /* This reflects the intel_pstate_get_cpu_pstates() setting. */ - policy->cur = policy->cpuinfo.min_freq; req = kzalloc_objs(*req, 2); if (!req) { @@ -3323,9 +3318,15 @@ static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy) WRITE_ONCE(cpu->hwp_req_cached, value); cpu->epp_cached = intel_pstate_get_epp(cpu, value); + + intel_cpufreq_hwp_update(cpu, cpu->pstate.min_pstate, + cpu->pstate.max_pstate, + cpu->pstate.min_pstate, false); } else { policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY; + intel_pstate_set_min_pstate(cpu); } + policy->cur = policy->cpuinfo.min_freq; freq = DIV_ROUND_UP(cpu->pstate.turbo_freq * global.min_perf_pct, 100); @@ -3676,14 +3677,14 @@ static inline bool intel_pstate_has_acpi_ppc(void) { return false; } static inline void intel_pstate_request_control_from_smm(void) {} #endif /* CONFIG_ACPI */ -#define INTEL_PSTATE_HWP_BROADWELL 0x01 +#define INTEL_PSTATE_HWP_NOT_HYBRID 0x01 #define X86_MATCH_HWP(vfm, hwp_mode) \ X86_MATCH_VFM_FEATURE(vfm, X86_FEATURE_HWP, hwp_mode) static const struct x86_cpu_id hwp_support_ids[] __initconst = { - X86_MATCH_HWP(INTEL_BROADWELL_X, INTEL_PSTATE_HWP_BROADWELL), - X86_MATCH_HWP(INTEL_BROADWELL_D, INTEL_PSTATE_HWP_BROADWELL), + X86_MATCH_HWP(INTEL_BROADWELL_X, INTEL_PSTATE_HWP_NOT_HYBRID), + X86_MATCH_HWP(INTEL_BROADWELL_D, INTEL_PSTATE_HWP_NOT_HYBRID), X86_MATCH_HWP(INTEL_ANY, 0), {} }; @@ -3808,7 +3809,6 @@ static int __init intel_pstate_init(void) if (!no_hwp) { hwp_active = true; - hwp_mode_bdw = id->driver_data; intel_pstate.attr = hwp_cpufreq_attrs; intel_cpufreq.attr = hwp_cpufreq_attrs; intel_cpufreq.flags |= CPUFREQ_NEED_UPDATE_LIMITS; @@ -3816,7 +3816,8 @@ static int __init intel_pstate_init(void) if (!default_driver) default_driver = &intel_pstate; - pstate_funcs.get_cpu_scaling = hwp_get_cpu_scaling; + if (!id->driver_data) + pstate_funcs.get_cpu_scaling = hwp_get_cpu_scaling; goto hwp_cpu_matched; } diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index ae9d1ce4f49c..35ce665edfd8 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -379,6 +379,7 @@ struct cpufreq_driver { void (*adjust_perf)(struct cpufreq_policy *policy, unsigned long min_perf, unsigned long target_perf, + unsigned long max_perf, unsigned long capacity); /* @@ -624,6 +625,7 @@ unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy, void cpufreq_driver_adjust_perf(struct cpufreq_policy *policy, unsigned long min_perf, unsigned long target_perf, + unsigned long max_perf, unsigned long capacity); bool cpufreq_driver_has_adjust_perf(void); int cpufreq_driver_target(struct cpufreq_policy *policy, diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index a4e689eefdfb..dc0835ceee98 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -50,6 +50,7 @@ struct sugov_cpu { unsigned long util; unsigned long bw_min; + unsigned long bw_max; /* The field below is for single-CPU policies only: */ #ifdef CONFIG_NO_HZ_COMMON @@ -232,6 +233,7 @@ static void sugov_get_util(struct sugov_cpu *sg_cpu, unsigned long boost) util = effective_cpu_util(sg_cpu->cpu, util, &min, &max); util = max(util, boost); sg_cpu->bw_min = min; + sg_cpu->bw_max = max; sg_cpu->util = sugov_effective_cpu_perf(sg_cpu->cpu, util, min, max); } @@ -314,7 +316,7 @@ static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, u64 time, * A CPU running a task which woken up after an IO operation can have its * utilization boosted to speed up the completion of those IO operations. * The IO boost value is increased each time a task wakes up from IO, in - * sugov_iowait_apply(), and it's instead decreased by this function, + * sugov_iowait_boost(), and it's instead decreased by this function, * each time an increase has not been requested (!iowait_boost_pending). * * A CPU which also appears to have been idle for at least one tick has also @@ -484,7 +486,7 @@ static void sugov_update_single_perf(struct update_util_data *hook, u64 time, sg_cpu->util = prev_util; cpufreq_driver_adjust_perf(sg_policy->policy, sg_cpu->bw_min, - sg_cpu->util, max_cap); + sg_cpu->util, sg_cpu->bw_max, max_cap); sg_policy->need_freq_update = false; sg_policy->last_freq_update_time = time; diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index 58ac04c650a1..d624cccb6540 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -792,7 +792,13 @@ fn fast_switch(_policy: &mut Policy, _target_freq: u32) -> u32 { } /// Driver's `adjust_perf` callback. - fn adjust_perf(_policy: &mut Policy, _min_perf: usize, _target_perf: usize, _capacity: usize) { + fn adjust_perf( + _policy: &mut Policy, + _min_perf: usize, + _target_perf: usize, + _max_perf: usize, + _capacity: usize, + ) { build_error!(VTABLE_DEFAULT_ERROR) } @@ -1263,12 +1269,13 @@ impl Registration { ptr: *mut bindings::cpufreq_policy, min_perf: c_ulong, target_perf: c_ulong, + max_perf: c_ulong, capacity: c_ulong, ) { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. let policy = unsafe { Policy::from_raw_mut(ptr) }; - T::adjust_perf(policy, min_perf, target_perf, capacity); + T::adjust_perf(policy, min_perf, target_perf, max_perf, capacity); } /// Driver's `get_intermediate` callback.