diff --git a/Documentation/arch/arm64/cpu-hotplug.rst b/Documentation/arch/arm64/cpu-hotplug.rst index 8fb438bf7781..7c3379b704aa 100644 --- a/Documentation/arch/arm64/cpu-hotplug.rst +++ b/Documentation/arch/arm64/cpu-hotplug.rst @@ -47,11 +47,12 @@ ever have can be described at boot. There are no power-domain considerations as such devices are emulated. CPU Hotplug on virtual systems is supported. It is distinct from physical -CPU Hotplug as all resources are described as ``present``, but CPUs may be -marked as disabled by firmware. Only the CPU's online/offline behaviour is -influenced by firmware. An example is where a virtual machine boots with a -single CPU, and additional CPUs are added once a cloud orchestrator deploys -the workload. +CPU Hotplug as all vCPU resources are statically described in the firmware +configuration tables (e.g. MADT), meaning their maximum possible count is +known at boot. However, vCPUs that are not enabled at boot are not marked +as ``present`` by the kernel until they are hotplugged. An example is where +a virtual machine boots with a single CPU, and additional CPUs are added +once a cloud orchestrator deploys the workload. For a virtual machine, the VMM (e.g. Qemu) plays the part of firmware. @@ -60,16 +61,19 @@ brought online. Firmware can enforce its policy via PSCI's return codes. e.g. ``DENIED``. The ACPI tables must describe all the resources of the virtual machine. CPUs -that firmware wishes to disable either from boot (or later) should not be -``enabled`` in the MADT GICC structures, but should have the ``online capable`` -bit set, to indicate they can be enabled later. The boot CPU must be marked as -``enabled``. The 'always on' GICR structure must be used to describe the -redistributors. +that are hot-pluggable must have the ``online capable`` bit set and the +``enabled`` bit cleared in the MADT GICC structures to indicate they can be +enabled later. The boot CPU must be marked as ``enabled`` with its +``online capable`` bit cleared. The 'always on' GICR structure must be used +to describe the redistributors. CPUs described as ``online capable`` but not ``enabled`` can be set to enabled by the DSDT's Processor object's _STA method. On virtual systems the _STA method -must always report the CPU as ``present``. Changes to the firmware policy can -be notified to the OS via device-check or eject-request. +must always set the ``ACPI_STA_DEVICE_PRESENT`` bit, while toggling the +``ACPI_STA_DEVICE_ENABLED`` bit to reflect its plug status. The kernel will +then dynamically mark the vCPU as ``present`` within the OS when the +``ACPI_STA_DEVICE_ENABLED`` bit becomes set during hot-add. Changes to the +firmware policy can be notified to the OS via device-check or eject-request. CPUs described as ``enabled`` in the static table, should not have their _STA modified dynamically by firmware. Soft-restart features such as kexec will diff --git a/arch/arm64/include/asm/tlbbatch.h b/arch/arm64/include/asm/tlbbatch.h index 6297631532e5..767f35ea62b3 100644 --- a/arch/arm64/include/asm/tlbbatch.h +++ b/arch/arm64/include/asm/tlbbatch.h @@ -2,17 +2,11 @@ #ifndef _ARCH_ARM64_TLBBATCH_H #define _ARCH_ARM64_TLBBATCH_H -#include - struct arch_tlbflush_unmap_batch { -#ifdef CONFIG_ARM64_ERRATUM_4193714 /* - * Track CPUs that need SME DVMSync on completion of this batch. - * Otherwise, the arm64 HW can do tlb shootdown, so we don't need to - * record cpumask for sending IPI + * For arm64, HW can do TLB shootdown, so we don't need to record a + * cpumask for sending IPIs. */ - cpumask_var_t cpumask; -#endif }; #endif /* _ARCH_ARM64_TLBBATCH_H */ diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index d52ac8c17190..e0e84332f51b 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -82,6 +82,8 @@ static inline unsigned long get_trans_granule(void) #ifdef CONFIG_ARM64_ERRATUM_4193714 +extern cpumask_t sme_active_cpus; + void sme_do_dvmsync(const struct cpumask *mask); static inline void sme_dvmsync(struct mm_struct *mm) @@ -92,42 +94,12 @@ static inline void sme_dvmsync(struct mm_struct *mm) sme_do_dvmsync(mm_cpumask(mm)); } -static inline void sme_dvmsync_add_pending(struct arch_tlbflush_unmap_batch *batch, - struct mm_struct *mm) +static inline void sme_dvmsync_batch(void) { if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) return; - /* - * Order the mm_cpumask() read after the hardware DVMSync. - */ - dsb(ish); - if (cpumask_empty(mm_cpumask(mm))) - return; - - /* - * Allocate the batch cpumask on first use. Fall back to an immediate - * IPI for this mm in case of failure. - */ - if (!cpumask_available(batch->cpumask) && - !zalloc_cpumask_var(&batch->cpumask, GFP_ATOMIC)) { - sme_do_dvmsync(mm_cpumask(mm)); - return; - } - - cpumask_or(batch->cpumask, batch->cpumask, mm_cpumask(mm)); -} - -static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch) -{ - if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) - return; - - if (!cpumask_available(batch->cpumask)) - return; - - sme_do_dvmsync(batch->cpumask); - cpumask_clear(batch->cpumask); + sme_do_dvmsync(&sme_active_cpus); } #else @@ -135,11 +107,7 @@ static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch) static inline void sme_dvmsync(struct mm_struct *mm) { } -static inline void sme_dvmsync_add_pending(struct arch_tlbflush_unmap_batch *batch, - struct mm_struct *mm) -{ -} -static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch) +static inline void sme_dvmsync_batch(void) { } @@ -285,11 +253,11 @@ static inline void __tlbi_sync_s1ish(struct mm_struct *mm) sme_dvmsync(mm); } -static inline void __tlbi_sync_s1ish_batch(struct arch_tlbflush_unmap_batch *batch) +static inline void __tlbi_sync_s1ish_batch(void) { dsb(ish); __repeat_tlbi_sync(vale1is, 0); - sme_dvmsync_batch(batch); + sme_dvmsync_batch(); } static inline void __tlbi_sync_s1ish_kernel(void) @@ -434,7 +402,7 @@ static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm) */ static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) { - __tlbi_sync_s1ish_batch(batch); + __tlbi_sync_s1ish_batch(); } /* @@ -722,7 +690,6 @@ static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *b __flush_tlb_range(&vma, start, end, PAGE_SIZE, 3, TLBF_NOWALKCACHE | TLBF_NOSYNC); - sme_dvmsync_add_pending(batch, mm); } static inline bool __pte_flags_need_flush(ptval_t oldval, ptval_t newval) diff --git a/arch/arm64/kernel/acpi.c b/arch/arm64/kernel/acpi.c index 5891f92c2035..681aa2bbc399 100644 --- a/arch/arm64/kernel/acpi.c +++ b/arch/arm64/kernel/acpi.c @@ -448,12 +448,14 @@ int acpi_map_cpu(acpi_handle handle, phys_cpuid_t physid, u32 apci_id, return *pcpu; } + set_cpu_present(*pcpu, true); return 0; } EXPORT_SYMBOL(acpi_map_cpu); int acpi_unmap_cpu(int cpu) { + set_cpu_present(cpu, false); return 0; } EXPORT_SYMBOL(acpi_unmap_cpu); diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 25dc5afe9ba0..e7f1682a3059 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -1355,6 +1355,7 @@ void do_sve_acc(unsigned long esr, struct pt_regs *regs) * SME/CME erratum handling. */ static cpumask_t sme_dvmsync_cpus; +cpumask_t sme_active_cpus; /* * These helpers are only called from non-preemptible contexts, so @@ -1368,13 +1369,15 @@ void sme_set_active(void) return; cpumask_set_cpu(cpu, mm_cpumask(current->mm)); + cpumask_set_cpu(cpu, &sme_active_cpus); /* * A subsequent (post ERET) SME access may use a stale address * translation. On C1-Pro, a TLBI+DSB on a different CPU will wait for - * the completion of cpumask_set_cpu() above as it appears in program - * order before the SME access. The post-TLBI+DSB read of mm_cpumask() - * will lead to the IPI being issued. + * the completion of the cpumask_set_cpu() operations above as they + * appear in program order before the SME access. The post-TLBI+DSB + * read of mm_cpumask() or sme_active_cpus will lead to the IPI being + * issued. * * https://lore.kernel.org/r/ablEXwhfKyJW1i7l@J2N7QTR9R3 */ @@ -1392,6 +1395,7 @@ void sme_clear_active(void) * completed on entering EL1. */ cpumask_clear_cpu(cpu, mm_cpumask(current->mm)); + cpumask_clear_cpu(cpu, &sme_active_cpus); } static void sme_dvmsync_ipi(void *unused) diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 033643cd4e5e..581f80e9b9b7 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -341,41 +341,8 @@ void flush_thread(void) flush_gcs(); } -#ifdef CONFIG_ARM64_ERRATUM_4193714 - -static void arch_dup_tlbbatch_mask(struct task_struct *dst) -{ - /* - * Clear the inherited cpumask with memset() to cover both cases where - * cpumask_var_t is a pointer or an array. It will be allocated lazily - * in sme_dvmsync_add_pending() if CPUMASK_OFFSTACK=y. - */ - if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) - memset(&dst->tlb_ubc.arch.cpumask, 0, - sizeof(dst->tlb_ubc.arch.cpumask)); -} - -static void arch_release_tlbbatch_mask(struct task_struct *tsk) -{ - if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) - free_cpumask_var(tsk->tlb_ubc.arch.cpumask); -} - -#else - -static void arch_dup_tlbbatch_mask(struct task_struct *dst) -{ -} - -static void arch_release_tlbbatch_mask(struct task_struct *tsk) -{ -} - -#endif /* CONFIG_ARM64_ERRATUM_4193714 */ - void arch_release_task_struct(struct task_struct *tsk) { - arch_release_tlbbatch_mask(tsk); fpsimd_release_task(tsk); } @@ -391,8 +358,6 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) *dst = *src; - arch_dup_tlbbatch_mask(dst); - /* * Drop stale reference to src's sve_state and convert dst to * non-streaming FPSIMD mode. diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index d46022f72075..cdcdd160e5b6 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -535,23 +535,13 @@ void arch_unregister_cpu(int cpu) { acpi_handle acpi_handle = acpi_get_processor_handle(cpu); struct cpu *c = &per_cpu(cpu_devices, cpu); - acpi_status status; unsigned long long sta; - - if (!acpi_handle) { - pr_err_once("Removing a CPU without associated ACPI handle\n"); - return; - } + acpi_status status; status = acpi_evaluate_integer(acpi_handle, "_STA", NULL, &sta); - if (ACPI_FAILURE(status)) - return; - - /* For now do not allow anything that looks like physical CPU HP */ - if (cpu_present(cpu) && !(sta & ACPI_STA_DEVICE_PRESENT)) { + if (!ACPI_FAILURE(status) && + cpu_present(cpu) && !(sta & ACPI_STA_DEVICE_PRESENT)) pr_err_once("Changing CPU present bit is not supported\n"); - return; - } unregister_cpu(c); } @@ -566,6 +556,11 @@ struct acpi_madt_generic_interrupt *acpi_cpu_get_madt_gicc(int cpu) } EXPORT_SYMBOL_GPL(acpi_cpu_get_madt_gicc); +static bool acpi_cpu_is_present(int cpu) +{ + return acpi_cpu_get_madt_gicc(cpu)->flags & ACPI_MADT_ENABLED; +} + /* * acpi_map_gic_cpu_interface - parse processor MADT entry * @@ -670,6 +665,10 @@ static void __init acpi_parse_and_init_cpus(void) early_map_cpu_to_node(i, acpi_numa_get_nid(i)); } #else +static bool acpi_cpu_is_present(int cpu) +{ + return false; +} #define acpi_parse_and_init_cpus(...) do { } while (0) #endif @@ -814,7 +813,8 @@ void __init smp_prepare_cpus(unsigned int max_cpus) if (err) continue; - set_cpu_present(cpu, true); + if (acpi_disabled || acpi_cpu_is_present(cpu)) + set_cpu_present(cpu, true); numa_store_cpu_info(cpu); } } diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index f2be501468ce..a25d8beacc83 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1515,7 +1515,13 @@ static void unmap_hotplug_pmd_range(pud_t *pudp, unsigned long addr, if (free_mapped) { /* CONT blocks are not supported in the vmemmap */ WARN_ON(pmd_cont(pmd)); - flush_tlb_kernel_range(addr, addr + PMD_SIZE); + /* + * Invalidating a block entry requires just + * a single overlapping TLB invalidation, + * so limit the range of the flush to a single + * page. + */ + flush_tlb_kernel_range(addr, addr + PAGE_SIZE); free_hotplug_page_range(pmd_page(pmd), PMD_SIZE, altmap); } @@ -1545,7 +1551,8 @@ static void unmap_hotplug_pud_range(p4d_t *p4dp, unsigned long addr, if (pud_leaf(pud)) { pud_clear(pudp); if (free_mapped) { - flush_tlb_kernel_range(addr, addr + PUD_SIZE); + /* See comment in unmap_hotplug_pmd_range(). */ + flush_tlb_kernel_range(addr, addr + PAGE_SIZE); free_hotplug_page_range(pud_page(pud), PUD_SIZE, altmap); } diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index bc1788b1662b..7cb61aca3797 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -1806,7 +1806,7 @@ Res0 15:8 UnsignedEnum 7:4 BWE 0b0000 NI 0b0001 FEAT_BWE - 0b0002 FEAT_BWE2 + 0b0010 FEAT_BWE2 EndEnum UnsignedEnum 3:0 STEP 0b0000 NI diff --git a/tools/testing/selftests/arm64/gcs/libc-gcs.c b/tools/testing/selftests/arm64/gcs/libc-gcs.c index 72e82bfbecc9..ddb0b2b20155 100644 --- a/tools/testing/selftests/arm64/gcs/libc-gcs.c +++ b/tools/testing/selftests/arm64/gcs/libc-gcs.c @@ -130,7 +130,7 @@ TEST(gcs_find_terminator) * We can access a GCS via ptrace * * This could usefully have a fixture but note that each test is - * fork()ed into a new child whcih causes issues. Might be better to + * fork()ed into a new child which causes issues. Might be better to * lift at least some of this out into a separate, non-harness, test * program. */ diff --git a/tools/testing/selftests/arm64/pauth/pac.c b/tools/testing/selftests/arm64/pauth/pac.c index 67d138057707..f4b859c75a5a 100644 --- a/tools/testing/selftests/arm64/pauth/pac.c +++ b/tools/testing/selftests/arm64/pauth/pac.c @@ -292,7 +292,7 @@ TEST(single_thread_different_keys) /* * fork() does not change keys. Only exec() does so call a worker program. - * Its only job is to sign a value and report back the resutls + * Its only job is to sign a value and report back the results */ TEST(exec_changed_keys) {