diff --git a/Documentation/arch/riscv/hwprobe.rst b/Documentation/arch/riscv/hwprobe.rst index d9928641deb9..893e1a1215d2 100644 --- a/Documentation/arch/riscv/hwprobe.rst +++ b/Documentation/arch/riscv/hwprobe.rst @@ -401,3 +401,23 @@ The following keys are defined: as defined in version 1.0 of the RISC-V Control-flow Integrity (CFI) extensions specification, ratified in commit 302a2d45c243 ("Update build-pdf.yml") of riscv-cfi. + + * :c:macro:`RISCV_HWPROBE_EXT_ZICCLSM`: The Zicclsm extension is supported, + as defined in the RISC-V Profiles specification starting from commit + b1d80660 ("Updated to ratified state.") + + * :c:macro:`RISCV_HWPROBE_EXT_ZICCAMOA`: The Ziccamoa extension is supported, + as defined in the RISC-V Profiles specification starting from commit + b1d80660 ("Updated to ratified state.") + + * :c:macro:`RISCV_HWPROBE_EXT_ZICCIF`: The Ziccif extension is supported, + as defined in the RISC-V Profiles specification starting from commit + b1d80660 ("Updated to ratified state.") + + * :c:macro:`RISCV_HWPROBE_EXT_ZICCRSE`: The Ziccrse extension is supported, + as defined in the RISC-V Profiles specification starting from commit + b1d80660 ("Updated to ratified state.") + + * :c:macro:`RISCV_HWPROBE_EXT_ZA64RS`: The Za64rs extension is supported, + as defined in the RISC-V Profiles specification starting from commit + b1d80660 ("Updated to ratified state.") diff --git a/Documentation/devicetree/bindings/riscv/extensions.yaml b/Documentation/devicetree/bindings/riscv/extensions.yaml index 5ffc40d599c0..577d7ae5576b 100644 --- a/Documentation/devicetree/bindings/riscv/extensions.yaml +++ b/Documentation/devicetree/bindings/riscv/extensions.yaml @@ -181,6 +181,27 @@ properties: changes to interrupts as frozen at commit ccbddab ("Merge pull request #42 from riscv/jhauser-2023-RC4") of riscv-aia. + - const: smcdeleg + description: | + The standard Smcdeleg machine-level extension for the machine mode + to delegate the hpmcounters to supervisor mode so that they are + directly accessible in the supervisor mode as ratified in the + 20240213 version of the privileged ISA specification. + + - const: smcntrpmf + description: | + The standard Smcntrpmf machine-level extension for the machine mode + to enable privilege mode filtering for cycle and instret counters as + ratified in the 20240326 version of the privileged ISA specification. + + - const: smcsrind + description: | + The standard Smcsrind machine-level extension extends the + indirect CSR access mechanism defined by the Smaia extension. This + extension allows other ISA extensions to use indirect CSR access + mechanism in M-mode as ratified in the 20240326 version of the + privileged ISA specification. + - const: smmpm description: | The standard Smmpm extension for M-mode pointer masking as @@ -206,6 +227,14 @@ properties: behavioural changes to interrupts as frozen at commit ccbddab ("Merge pull request #42 from riscv/jhauser-2023-RC4") of riscv-aia. + - const: ssccfg + description: | + The standard Ssccfg supervisor-level extension for configuring + the delegated hpmcounters to be accessible directly in supervisor + mode as ratified in the 20240213 version of the privileged ISA + specification. This extension depends on Sscsrind, Smcdeleg, Zihpm, + Zicntr extensions. + - const: ssccptr description: | The standard Ssccptr extension for main memory (cacheability and @@ -226,12 +255,26 @@ properties: Profiles Version 1.0, with commit b1d806605f87 ("Updated to ratified state.") + - const: sscsrind + description: | + The standard Sscsrind supervisor-level extension extends the + indirect CSR access mechanism defined by the Ssaia extension. This + extension allows other ISA extensions to use indirect CSR access + mechanism in S-mode as ratified in the 20240326 version of the + privileged ISA specification. + - const: ssnpm description: | The standard Ssnpm extension for next-mode pointer masking as ratified at commit d70011dde6c2 ("Update to ratified state") of riscv-j-extension. + - const: ssqosid + description: | + The standard Ssqosid extension for Quality of Service ID is + ratified as v1.0 in commit d9c616497fde ("Merge pull + request #7 from ved-rivos/Ratified") of riscv-ssqosid. + - const: ssstateen description: | The standard Ssstateen extension for supervisor-mode view of the @@ -1113,6 +1156,32 @@ properties: allOf: - const: zilsd - const: zca + # Smcdeleg depends on Sscsrind, Zihpm, Zicntr + - if: + contains: + const: smcdeleg + then: + allOf: + - contains: + const: sscsrind + - contains: + const: zihpm + - contains: + const: zicntr + # Ssccfg depends on Smcdeleg, Sscsrind, Zihpm, Zicntr + - if: + contains: + const: ssccfg + then: + allOf: + - contains: + const: smcdeleg + - contains: + const: sscsrind + - contains: + const: zihpm + - contains: + const: zicntr allOf: # Zcf extension does not exist on rv64 diff --git a/MAINTAINERS b/MAINTAINERS index 169ffe10e815..a830d3b252e2 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -23389,6 +23389,14 @@ F: drivers/perf/riscv_pmu.c F: drivers/perf/riscv_pmu_legacy.c F: drivers/perf/riscv_pmu_sbi.c +RISC-V QOS RESCTRL SUPPORT +M: Drew Fustini +R: yunhui cui +L: linux-riscv@lists.infradead.org +S: Supported +F: arch/riscv/include/asm/qos.h +F: arch/riscv/kernel/qos.c + RISC-V RPMI AND MPXY DRIVERS M: Rahul Pathak M: Anup Patel diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index ccd10d77bad7..bf2e81d258c4 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -23,6 +23,7 @@ config RISCV select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM_VMEMMAP select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_BINFMT_FLAT select ARCH_HAS_CC_CAN_LINK select ARCH_HAS_CURRENT_STACK_POINTER @@ -592,6 +593,24 @@ config RISCV_ISA_SVNAPOT If you don't know what to do here, say Y. +config RISCV_ISA_SSQOSID + bool "Ssqosid extension support for supervisor mode Quality of Service ID" + depends on 64BIT + default n + help + Adds support for the Ssqosid ISA extension (Supervisor-mode + Quality of Service ID). + + Ssqosid defines the srmcfg CSR which allows the system to tag the + running process with an RCID (Resource Control ID) and MCID + (Monitoring Counter ID). The RCID is used to determine resource + allocation. The MCID is used to track resource usage in event + counters. + + For example, a cache controller may use the RCID to apply a + cache partitioning scheme and use the MCID to track how much + cache a process, or a group of processes, is using. + config RISCV_ISA_SVPBMT bool "Svpbmt extension support for supervisor mode page-based memory types" depends on 64BIT && MMU diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile index 1363e5bef35c..44f5a439741a 100644 --- a/arch/riscv/Makefile +++ b/arch/riscv/Makefile @@ -75,12 +75,6 @@ else riscv-march-$(CONFIG_TOOLCHAIN_NEEDS_EXPLICIT_ZICSR_ZIFENCEI) := $(riscv-march-y)_zicsr_zifencei endif -# Check if the toolchain supports Zacas -riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZACAS) := $(riscv-march-y)_zacas - -# Check if the toolchain supports Zabha -riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZABHA) := $(riscv-march-y)_zabha - KBUILD_BASE_ISA = -march=$(shell echo $(riscv-march-y) | sed -E 's/(rv32ima|rv64ima)fd([^v_]*)v?/\1\2/') export KBUILD_BASE_ISA @@ -172,12 +166,12 @@ vdso-install-$(CONFIG_MMU) += arch/riscv/kernel/vdso/vdso.so.dbg vdso-install-$(CONFIG_RISCV_USER_CFI) += arch/riscv/kernel/vdso_cfi/vdso-cfi.so.dbg vdso-install-$(CONFIG_COMPAT) += arch/riscv/kernel/compat_vdso/compat_vdso.so.dbg -BOOT_TARGETS := Image Image.gz Image.bz2 Image.lz4 Image.lzma Image.lzo Image.zst Image.xz loader loader.bin xipImage vmlinuz.efi +BOOT_TARGETS := Image Image.gz Image.bz2 Image.lz4 Image.lzma Image.lzo Image.zst Image.xz loader loader.bin vmlinuz.efi all: $(notdir $(KBUILD_IMAGE)) loader.bin: loader -Image.gz Image.bz2 Image.lz4 Image.lzma Image.lzo Image.zst Image.xz loader xipImage vmlinuz.efi: Image +Image.gz Image.bz2 Image.lz4 Image.lzma Image.lzo Image.zst Image.xz loader vmlinuz.efi: Image $(BOOT_TARGETS): vmlinux $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ diff --git a/arch/riscv/boot/.gitignore b/arch/riscv/boot/.gitignore index e1bc507e8cb2..5d123ceaa012 100644 --- a/arch/riscv/boot/.gitignore +++ b/arch/riscv/boot/.gitignore @@ -5,4 +5,3 @@ loader loader.lds loader.bin vmlinuz* -xipImage diff --git a/arch/riscv/boot/Makefile b/arch/riscv/boot/Makefile index fcfbe3f814d6..4a2d75b7e42d 100644 --- a/arch/riscv/boot/Makefile +++ b/arch/riscv/boot/Makefile @@ -16,9 +16,8 @@ OBJCOPYFLAGS_Image :=-O binary -R .note -R .note.gnu.build-id -R .comment -S OBJCOPYFLAGS_loader.bin :=-O binary -OBJCOPYFLAGS_xipImage :=-O binary -R .note -R .note.gnu.build-id -R .comment -S -targets := Image Image.* loader loader.o loader.lds loader.bin xipImage +targets := Image Image.* loader loader.o loader.lds loader.bin $(obj)/Image: vmlinux FORCE $(call if_changed,objcopy) diff --git a/arch/riscv/include/asm/acpi.h b/arch/riscv/include/asm/acpi.h index f598520ac903..cdefa1f2d67d 100644 --- a/arch/riscv/include/asm/acpi.h +++ b/arch/riscv/include/asm/acpi.h @@ -12,6 +12,8 @@ #ifndef _ASM_ACPI_H #define _ASM_ACPI_H +#include + /* Basic configuration for ACPI */ #ifdef CONFIG_ACPI @@ -68,6 +70,23 @@ int acpi_get_riscv_isa(struct acpi_table_header *table, void acpi_get_cbo_block_size(struct acpi_table_header *table, u32 *cbom_size, u32 *cboz_size, u32 *cbop_size); acpi_handle acpi_get_riscv_gsi_handle(u32 gsi); + +/* + * RISC-V Functional Fixed Hardware Specification Version v1.0.1, + * Chapter 3.1.2, Table 4: Arch. Context Lost Flags + */ +#define RISCV_LPI_HART_TIMER_CTXT_LOST BIT(0) + +static inline unsigned int arch_get_idle_state_flags(u32 arch_flags) +{ + if (arch_flags & RISCV_LPI_HART_TIMER_CTXT_LOST) + return CPUIDLE_FLAG_TIMER_STOP; + + return 0; +} + +#define arch_get_idle_state_flags arch_get_idle_state_flags + #else static inline void acpi_init_rintc_map(void) { } static inline struct acpi_madt_rintc *acpi_cpu_get_madt_rintc(int cpu) @@ -93,4 +112,6 @@ void acpi_map_cpus_to_nodes(void); static inline void acpi_map_cpus_to_nodes(void) { } #endif /* CONFIG_ACPI_NUMA */ +#define ACPI_TABLE_UPGRADE_MAX_PHYS MEMBLOCK_ALLOC_ACCESSIBLE + #endif /*_ASM_ACPI_H*/ diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h index 8712cf9c69dc..662e160b0522 100644 --- a/arch/riscv/include/asm/cmpxchg.h +++ b/arch/riscv/include/asm/cmpxchg.h @@ -23,7 +23,10 @@ riscv_has_extension_unlikely(RISCV_ISA_EXT_ZABHA)) { \ __asm__ __volatile__ ( \ prepend \ + " .option push\n" \ + " .option arch, +zabha\n" \ " amoswap" swap_sfx " %0, %z2, %1\n" \ + " .option pop\n" \ swap_append \ : "=&r" (r), "+A" (*(p)) \ : "rJ" (n) \ @@ -141,7 +144,10 @@ \ __asm__ __volatile__ ( \ cas_prepend \ + " .option push\n" \ + " .option arch, +zacas, +zabha\n" \ " amocas" cas_sfx " %0, %z2, %1\n" \ + " .option pop\n" \ cas_append \ : "+&r" (r), "+A" (*(p)) \ : "rJ" (n) \ @@ -188,7 +194,10 @@ \ __asm__ __volatile__ ( \ cas_prepend \ + " .option push\n" \ + " .option arch, +zacas\n" \ " amocas" cas_sfx " %0, %z2, %1\n" \ + " .option pop\n" \ cas_append \ : "+&r" (r), "+A" (*(p)) \ : "rJ" (n) \ @@ -340,7 +349,10 @@ union __u128_halves { register unsigned long t4 asm ("t4") = __ho.high; \ \ __asm__ __volatile__ ( \ - " amocas.q" cas_sfx " %0, %z3, %2" \ + " .option push\n" \ + " .option arch, +zacas\n" \ + " amocas.q" cas_sfx " %0, %z3, %2\n" \ + " .option pop\n" \ : "+&r" (t3), "+&r" (t4), "+A" (*(p)) \ : "rJ" (t1), "rJ" (t2) \ : "memory"); \ diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h index 31b8988f4488..7f909629d21d 100644 --- a/arch/riscv/include/asm/csr.h +++ b/arch/riscv/include/asm/csr.h @@ -84,6 +84,10 @@ #define SATP_ASID_MASK _AC(0xFFFF, UL) #endif +/* SRMCFG fields */ +#define SRMCFG_RCID_MASK GENMASK(11, 0) +#define SRMCFG_MCID_MASK GENMASK(27, 16) + /* Exception cause high bit - is an interrupt if set */ #define CAUSE_IRQ_FLAG (_AC(1, UL) << (__riscv_xlen - 1)) @@ -241,6 +245,23 @@ #define SMSTATEEN0_HSENVCFG (_ULL(1) << SMSTATEEN0_HSENVCFG_SHIFT) #define SMSTATEEN0_SSTATEEN0_SHIFT 63 #define SMSTATEEN0_SSTATEEN0 (_ULL(1) << SMSTATEEN0_SSTATEEN0_SHIFT) +/* HPMEVENT bits. These are accessible in S-mode via Smcdeleg/Ssccfg */ +#define HPMEVENT_OF (BIT_ULL(63)) +#define HPMEVENT_MINH (BIT_ULL(62)) +#define HPMEVENT_SINH (BIT_ULL(61)) +#define HPMEVENT_UINH (BIT_ULL(60)) +#define HPMEVENT_VSINH (BIT_ULL(59)) +#define HPMEVENT_VUINH (BIT_ULL(58)) +#ifndef CONFIG_64BIT +#define HPMEVENTH_OF (BIT(31)) +#define HPMEVENTH_MINH (BIT(30)) +#define HPMEVENTH_SINH (BIT(29)) +#define HPMEVENTH_UINH (BIT(28)) +#define HPMEVENTH_VSINH (BIT(27)) +#define HPMEVENTH_VUINH (BIT(26)) +#endif + +#define SISELECT_SSCCFG_BASE 0x40 /* mseccfg bits */ #define MSECCFG_PMM ENVCFG_PMM @@ -322,12 +343,14 @@ #define CSR_SCOUNTEREN 0x106 #define CSR_SENVCFG 0x10a #define CSR_SSTATEEN0 0x10c +#define CSR_SCOUNTINHIBIT 0x120 #define CSR_SSCRATCH 0x140 #define CSR_SEPC 0x141 #define CSR_SCAUSE 0x142 #define CSR_STVAL 0x143 #define CSR_SIP 0x144 #define CSR_SATP 0x180 +#define CSR_SRMCFG 0x181 #define CSR_STIMECMP 0x14D #define CSR_STIMECMPH 0x15D @@ -347,6 +370,12 @@ /* Supervisor-Level Window to Indirectly Accessed Registers (AIA) */ #define CSR_SISELECT 0x150 #define CSR_SIREG 0x151 +/* Supervisor-Level Window to Indirectly Accessed Registers (Sscsrind) */ +#define CSR_SIREG2 0x152 +#define CSR_SIREG3 0x153 +#define CSR_SIREG4 0x155 +#define CSR_SIREG5 0x156 +#define CSR_SIREG6 0x157 /* Supervisor-Level Interrupts (AIA) */ #define CSR_STOPEI 0x15c @@ -394,6 +423,14 @@ /* VS-Level Window to Indirectly Accessed Registers (H-extension with AIA) */ #define CSR_VSISELECT 0x250 #define CSR_VSIREG 0x251 +/* + * VS-Level Window to Indirectly Accessed Registers (H-extension with Sscsrind) + */ +#define CSR_VSIREG2 0x252 +#define CSR_VSIREG3 0x253 +#define CSR_VSIREG4 0x255 +#define CSR_VSIREG5 0x256 +#define CSR_VSIREG6 0x257 /* VS-Level Interrupts (H-extension with AIA) */ #define CSR_VSTOPEI 0x25c @@ -436,6 +473,12 @@ /* Machine-Level Window to Indirectly Accessed Registers (AIA) */ #define CSR_MISELECT 0x350 #define CSR_MIREG 0x351 +/* Machine-Level Window to Indirectly Accessed Registers (Smcsrind) */ +#define CSR_MIREG2 0x352 +#define CSR_MIREG3 0x353 +#define CSR_MIREG4 0x355 +#define CSR_MIREG5 0x356 +#define CSR_MIREG6 0x357 /* Machine-Level Interrupts (AIA) */ #define CSR_MTOPEI 0x35c @@ -498,6 +541,11 @@ # define CSR_IEH CSR_MIEH # define CSR_ISELECT CSR_MISELECT # define CSR_IREG CSR_MIREG +# define CSR_IREG2 CSR_MIREG2 +# define CSR_IREG3 CSR_MIREG3 +# define CSR_IREG4 CSR_MIREG4 +# define CSR_IREG5 CSR_MIREG5 +# define CSR_IREG6 CSR_MIREG6 # define CSR_IPH CSR_MIPH # define CSR_TOPEI CSR_MTOPEI # define CSR_TOPI CSR_MTOPI @@ -523,6 +571,11 @@ # define CSR_IEH CSR_SIEH # define CSR_ISELECT CSR_SISELECT # define CSR_IREG CSR_SIREG +# define CSR_IREG2 CSR_SIREG2 +# define CSR_IREG3 CSR_SIREG3 +# define CSR_IREG4 CSR_SIREG4 +# define CSR_IREG5 CSR_SIREG5 +# define CSR_IREG6 CSR_SIREG6 # define CSR_IPH CSR_SIPH # define CSR_TOPEI CSR_STOPEI # define CSR_TOPI CSR_STOPI diff --git a/arch/riscv/include/asm/csr_indirect.h b/arch/riscv/include/asm/csr_indirect.h new file mode 100644 index 000000000000..0f558fac8f5f --- /dev/null +++ b/arch/riscv/include/asm/csr_indirect.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _ASM_RISCV_CSR_INDIRECT_H +#define _ASM_RISCV_CSR_INDIRECT_H + +#include + +#include + +/* + * These have to be macros rather than functions: csr_read()/csr_write() + * stringify their CSR argument into the inline asm template via __ASM_STR(), + * so the CSR number must be a literal token at preprocessing time. Passing it + * as a function parameter emits "csrr %0, iregcsr", which no assembler can + * resolve. RISC-V has no register-indirect form of csrr/csrw - the CSR is a + * 12-bit immediate - so the sireg CSR selecting the indirect window cannot + * itself be a variable. + */ +#define csr_indirect_read(iregcsr, iselbase, iseloff) ({ \ + unsigned long __value = 0; \ + unsigned long __flags; \ + local_irq_save(__flags); \ + csr_write(CSR_ISELECT, (iselbase) + (iseloff)); \ + __value = csr_read(iregcsr); \ + local_irq_restore(__flags); \ + __value; \ +}) + +#define csr_indirect_write(iregcsr, iselbase, iseloff, value) ({ \ + unsigned long __flags; \ + local_irq_save(__flags); \ + csr_write(CSR_ISELECT, (iselbase) + (iseloff)); \ + csr_write(iregcsr, (value)); \ + local_irq_restore(__flags); \ +}) + +#define csr_indirect_warl(iregcsr, iselbase, iseloff, warl_val) ({ \ + unsigned long __old_val = 0, __value = 0; \ + unsigned long __flags; \ + local_irq_save(__flags); \ + csr_write(CSR_ISELECT, (iselbase) + (iseloff)); \ + __old_val = csr_read(iregcsr); \ + csr_write(iregcsr, (warl_val)); \ + __value = csr_read(iregcsr); \ + csr_write(iregcsr, __old_val); \ + local_irq_restore(__flags); \ + __value; \ +}) + +#endif diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 7ef8e5f55c8d..f8db798b2654 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -10,15 +10,15 @@ #include -#define RISCV_ISA_EXT_a ('a' - 'a') -#define RISCV_ISA_EXT_c ('c' - 'a') -#define RISCV_ISA_EXT_d ('d' - 'a') -#define RISCV_ISA_EXT_f ('f' - 'a') -#define RISCV_ISA_EXT_h ('h' - 'a') -#define RISCV_ISA_EXT_i ('i' - 'a') -#define RISCV_ISA_EXT_m ('m' - 'a') -#define RISCV_ISA_EXT_q ('q' - 'a') -#define RISCV_ISA_EXT_v ('v' - 'a') +#define RISCV_ISA_EXT_A ('a' - 'a') +#define RISCV_ISA_EXT_C ('c' - 'a') +#define RISCV_ISA_EXT_D ('d' - 'a') +#define RISCV_ISA_EXT_F ('f' - 'a') +#define RISCV_ISA_EXT_H ('h' - 'a') +#define RISCV_ISA_EXT_I ('i' - 'a') +#define RISCV_ISA_EXT_M ('m' - 'a') +#define RISCV_ISA_EXT_Q ('q' - 'a') +#define RISCV_ISA_EXT_V ('v' - 'a') /* * These macros represent the logical IDs of each multi-letter RISC-V ISA @@ -112,6 +112,16 @@ #define RISCV_ISA_EXT_ZCLSD 103 #define RISCV_ISA_EXT_ZICFILP 104 #define RISCV_ISA_EXT_ZICFISS 105 +#define RISCV_ISA_EXT_SSCSRIND 106 +#define RISCV_ISA_EXT_SMCSRIND 107 +#define RISCV_ISA_EXT_SMCNTRPMF 108 +#define RISCV_ISA_EXT_SSCCFG 109 +#define RISCV_ISA_EXT_SMCDELEG 110 +#define RISCV_ISA_EXT_SSQOSID 111 +#define RISCV_ISA_EXT_ZICCLSM 112 +#define RISCV_ISA_EXT_ZICCAMOA 113 +#define RISCV_ISA_EXT_ZICCIF 114 +#define RISCV_ISA_EXT_ZA64RS 115 #define RISCV_ISA_EXT_XLINUXENVCFG 127 @@ -121,9 +131,11 @@ #ifdef CONFIG_RISCV_M_MODE #define RISCV_ISA_EXT_SxAIA RISCV_ISA_EXT_SMAIA #define RISCV_ISA_EXT_SUPM RISCV_ISA_EXT_SMNPM +#define RISCV_ISA_EXT_SxCSRIND RISCV_ISA_EXT_SMCSRIND #else #define RISCV_ISA_EXT_SxAIA RISCV_ISA_EXT_SSAIA #define RISCV_ISA_EXT_SUPM RISCV_ISA_EXT_SSNPM +#define RISCV_ISA_EXT_SxCSRIND RISCV_ISA_EXT_SSCSRIND #endif #endif /* _ASM_RISCV_HWCAP_H */ diff --git a/arch/riscv/include/asm/hwprobe.h b/arch/riscv/include/asm/hwprobe.h index 8b9f5e1cf4cb..9b04377c0f98 100644 --- a/arch/riscv/include/asm/hwprobe.h +++ b/arch/riscv/include/asm/hwprobe.h @@ -43,11 +43,4 @@ static inline bool riscv_hwprobe_pair_cmp(struct riscv_hwprobe *pair, return pair->value == other_pair->value; } -#ifdef CONFIG_MMU -void riscv_hwprobe_register_async_probe(void); -void riscv_hwprobe_complete_async_probe(void); -#else -static inline void riscv_hwprobe_register_async_probe(void) {} -static inline void riscv_hwprobe_complete_async_probe(void) {} -#endif #endif diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 5d5756bda82e..823805cc465a 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { + unsigned long asid = get_mm_asid(vma->vm_mm); + /* * Svvptc guarantees that the new valid pte will be visible within * a bounded timeframe, so when the uarch does not cache invalid @@ -576,6 +578,14 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC)) return; + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)) { + local_sfence_w_inval(); + while (nr--) + local_sinval_vma(address + nr * PAGE_SIZE, asid); + local_sfence_inval_ir(); + return; + } + /* * The kernel assumes that TLBs don't cache invalid entries, but * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a @@ -584,7 +594,7 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, * the extra traps reduce performance. So, eagerly SFENCE.VMA. */ while (nr--) - local_flush_tlb_page(address + nr * PAGE_SIZE); + local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid); } #define update_mmu_cache(vma, addr, ptep) \ diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h index 812517b2cec1..49a386d74cd3 100644 --- a/arch/riscv/include/asm/processor.h +++ b/arch/riscv/include/asm/processor.h @@ -123,6 +123,9 @@ struct thread_struct { /* A forced icache flush is not needed if migrating to the previous cpu. */ unsigned int prev_cpu; #endif +#ifdef CONFIG_RISCV_ISA_SSQOSID + u32 srmcfg; +#endif }; /* Whitelist the fstate from the task_struct for hardened usercopy */ diff --git a/arch/riscv/include/asm/qos.h b/arch/riscv/include/asm/qos.h new file mode 100644 index 000000000000..cf19e8438bb9 --- /dev/null +++ b/arch/riscv/include/asm/qos.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_RISCV_QOS_H +#define _ASM_RISCV_QOS_H + +#include + +#ifdef CONFIG_RISCV_ISA_SSQOSID + +#include +#include +#include + +#include +#include + +/* cached value of srmcfg csr for each cpu */ +DECLARE_PER_CPU(u32, cpu_srmcfg); + +/* default srmcfg value for each cpu, set via resctrl cpu assignment */ +DECLARE_PER_CPU(u32, cpu_srmcfg_default); + +static inline void __switch_to_srmcfg(struct task_struct *next) +{ + u32 thread_srmcfg, default_srmcfg; + + thread_srmcfg = READ_ONCE(next->thread.srmcfg); + default_srmcfg = __this_cpu_read(cpu_srmcfg_default); + + /* + * RCID and MCID inherit from cpu_srmcfg_default independently. + * RESCTRL_RESERVED_CLOSID and RESCTRL_RESERVED_RMID are both 0, so a + * zero field means "unassigned" and takes the CPU default. + */ + if (thread_srmcfg == 0) { + thread_srmcfg = default_srmcfg; + } else { + u32 rcid = FIELD_GET(SRMCFG_RCID_MASK, thread_srmcfg); + u32 mcid = FIELD_GET(SRMCFG_MCID_MASK, thread_srmcfg); + + if (rcid == 0 || mcid == 0) { + if (rcid == 0) + rcid = FIELD_GET(SRMCFG_RCID_MASK, default_srmcfg); + if (mcid == 0) + mcid = FIELD_GET(SRMCFG_MCID_MASK, default_srmcfg); + thread_srmcfg = FIELD_PREP(SRMCFG_RCID_MASK, rcid) | + FIELD_PREP(SRMCFG_MCID_MASK, mcid); + } + } + + if (thread_srmcfg != __this_cpu_read(cpu_srmcfg)) { + /* + * No fence around the csrw. Ssqosid is silent on srmcfg + * ordering versus memory accesses, so a few accesses at the + * switch boundary may carry the previous RCID/MCID. The + * tagging inaccuracy is bounded and acceptable for QoS. + */ + __this_cpu_write(cpu_srmcfg, thread_srmcfg); + csr_write(CSR_SRMCFG, thread_srmcfg); + } +} + +static __always_inline bool has_srmcfg(void) +{ + return riscv_has_extension_unlikely(RISCV_ISA_EXT_SSQOSID); +} + +#else /* ! CONFIG_RISCV_ISA_SSQOSID */ + +struct task_struct; +static __always_inline bool has_srmcfg(void) { return false; } +static inline void __switch_to_srmcfg(struct task_struct *next) { } + +#endif /* CONFIG_RISCV_ISA_SSQOSID */ +#endif /* _ASM_RISCV_QOS_H */ diff --git a/arch/riscv/include/asm/switch_to.h b/arch/riscv/include/asm/switch_to.h index 0e71eb82f920..04f10a949066 100644 --- a/arch/riscv/include/asm/switch_to.h +++ b/arch/riscv/include/asm/switch_to.h @@ -14,6 +14,7 @@ #include #include #include +#include #ifdef CONFIG_FPU extern void __fstate_save(struct task_struct *save_to); @@ -60,8 +61,8 @@ static inline void __switch_to_fpu(struct task_struct *prev, static __always_inline bool has_fpu(void) { - return riscv_has_extension_likely(RISCV_ISA_EXT_f) || - riscv_has_extension_likely(RISCV_ISA_EXT_d); + return riscv_has_extension_likely(RISCV_ISA_EXT_F) || + riscv_has_extension_likely(RISCV_ISA_EXT_D); } #else static __always_inline bool has_fpu(void) { return false; } @@ -119,6 +120,8 @@ do { \ __switch_to_fpu(__prev, __next); \ if (has_vector() || has_xtheadvector()) \ __switch_to_vector(__prev, __next); \ + if (has_srmcfg()) \ + __switch_to_srmcfg(__next); \ if (switch_to_should_flush_icache(__next)) \ local_flush_icache_all(); \ __switch_to_envcfg(__next); \ diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h index eed0abc40514..9636d07fe9ee 100644 --- a/arch/riscv/include/asm/tlbflush.h +++ b/arch/riscv/include/asm/tlbflush.h @@ -15,6 +15,29 @@ #define FLUSH_TLB_NO_ASID ((unsigned long)-1) #ifdef CONFIG_MMU +static inline unsigned long get_mm_asid(struct mm_struct *mm) +{ + return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID; +} + +static inline void local_sfence_inval_ir(void) +{ + asm volatile(SFENCE_INVAL_IR() ::: "memory"); +} + +static inline void local_sfence_w_inval(void) +{ + asm volatile(SFENCE_W_INVAL() ::: "memory"); +} + +static inline void local_sinval_vma(unsigned long vma, unsigned long asid) +{ + if (asid != FLUSH_TLB_NO_ASID) + asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory"); + else + asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory"); +} + static inline void local_flush_tlb_all(void) { __asm__ __volatile__ ("sfence.vma" : : : "memory"); diff --git a/arch/riscv/include/asm/vdso.h b/arch/riscv/include/asm/vdso.h index f7998d9ad9b2..0259bbc98e93 100644 --- a/arch/riscv/include/asm/vdso.h +++ b/arch/riscv/include/asm/vdso.h @@ -19,10 +19,14 @@ #ifdef CONFIG_MMU #include +#else +extern size_t __vdso_alternatives_start_offset, __vdso_alternatives_end_offset; #endif #ifdef CONFIG_RISCV_USER_CFI #include +#else +extern size_t __vdso_alternatives_start_cfi_offset, __vdso_alternatives_end_cfi_offset; #endif #ifdef CONFIG_RISCV_USER_CFI @@ -41,6 +45,8 @@ #define COMPAT_VDSO_SYMBOL(base, name) \ (void __user *)((unsigned long)(base) + compat__vdso_##name##_offset) +#else +extern size_t compat__vdso_alternatives_start_offset, compat__vdso_alternatives_end_offset; #endif /* CONFIG_COMPAT */ extern char vdso_start[], vdso_end[]; diff --git a/arch/riscv/include/asm/vdso/arch_data.h b/arch/riscv/include/asm/vdso/arch_data.h index 88b37af55175..da57a3786f7a 100644 --- a/arch/riscv/include/asm/vdso/arch_data.h +++ b/arch/riscv/include/asm/vdso/arch_data.h @@ -12,12 +12,6 @@ struct vdso_arch_data { /* Boolean indicating all CPUs have the same static hwprobe values. */ __u8 homogeneous_cpus; - - /* - * A gate to check and see if the hwprobe data is actually ready, as - * probing is deferred to avoid boot slowdowns. - */ - __u8 ready; }; #endif /* __RISCV_ASM_VDSO_ARCH_DATA_H */ diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h index 9139edba0aec..e6a394fbe7fc 100644 --- a/arch/riscv/include/uapi/asm/hwprobe.h +++ b/arch/riscv/include/uapi/asm/hwprobe.h @@ -7,6 +7,7 @@ #define _UAPI_ASM_HWPROBE_H #include +#include /* * Interface for probing hardware capabilities from userspace, see @@ -21,80 +22,80 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_KEY_MARCHID 1 #define RISCV_HWPROBE_KEY_MIMPID 2 #define RISCV_HWPROBE_KEY_BASE_BEHAVIOR 3 -#define RISCV_HWPROBE_BASE_BEHAVIOR_IMA (1 << 0) +#define RISCV_HWPROBE_BASE_BEHAVIOR_IMA _BITULL(0) #define RISCV_HWPROBE_KEY_IMA_EXT_0 4 -#define RISCV_HWPROBE_IMA_FD (1 << 0) -#define RISCV_HWPROBE_IMA_C (1 << 1) -#define RISCV_HWPROBE_IMA_V (1 << 2) -#define RISCV_HWPROBE_EXT_ZBA (1 << 3) -#define RISCV_HWPROBE_EXT_ZBB (1 << 4) -#define RISCV_HWPROBE_EXT_ZBS (1 << 5) -#define RISCV_HWPROBE_EXT_ZICBOZ (1 << 6) -#define RISCV_HWPROBE_EXT_ZBC (1 << 7) -#define RISCV_HWPROBE_EXT_ZBKB (1 << 8) -#define RISCV_HWPROBE_EXT_ZBKC (1 << 9) -#define RISCV_HWPROBE_EXT_ZBKX (1 << 10) -#define RISCV_HWPROBE_EXT_ZKND (1 << 11) -#define RISCV_HWPROBE_EXT_ZKNE (1 << 12) -#define RISCV_HWPROBE_EXT_ZKNH (1 << 13) -#define RISCV_HWPROBE_EXT_ZKSED (1 << 14) -#define RISCV_HWPROBE_EXT_ZKSH (1 << 15) -#define RISCV_HWPROBE_EXT_ZKT (1 << 16) -#define RISCV_HWPROBE_EXT_ZVBB (1 << 17) -#define RISCV_HWPROBE_EXT_ZVBC (1 << 18) -#define RISCV_HWPROBE_EXT_ZVKB (1 << 19) -#define RISCV_HWPROBE_EXT_ZVKG (1 << 20) -#define RISCV_HWPROBE_EXT_ZVKNED (1 << 21) -#define RISCV_HWPROBE_EXT_ZVKNHA (1 << 22) -#define RISCV_HWPROBE_EXT_ZVKNHB (1 << 23) -#define RISCV_HWPROBE_EXT_ZVKSED (1 << 24) -#define RISCV_HWPROBE_EXT_ZVKSH (1 << 25) -#define RISCV_HWPROBE_EXT_ZVKT (1 << 26) -#define RISCV_HWPROBE_EXT_ZFH (1 << 27) -#define RISCV_HWPROBE_EXT_ZFHMIN (1 << 28) -#define RISCV_HWPROBE_EXT_ZIHINTNTL (1 << 29) -#define RISCV_HWPROBE_EXT_ZVFH (1 << 30) -#define RISCV_HWPROBE_EXT_ZVFHMIN (1ULL << 31) -#define RISCV_HWPROBE_EXT_ZFA (1ULL << 32) -#define RISCV_HWPROBE_EXT_ZTSO (1ULL << 33) -#define RISCV_HWPROBE_EXT_ZACAS (1ULL << 34) -#define RISCV_HWPROBE_EXT_ZICOND (1ULL << 35) -#define RISCV_HWPROBE_EXT_ZIHINTPAUSE (1ULL << 36) -#define RISCV_HWPROBE_EXT_ZVE32X (1ULL << 37) -#define RISCV_HWPROBE_EXT_ZVE32F (1ULL << 38) -#define RISCV_HWPROBE_EXT_ZVE64X (1ULL << 39) -#define RISCV_HWPROBE_EXT_ZVE64F (1ULL << 40) -#define RISCV_HWPROBE_EXT_ZVE64D (1ULL << 41) -#define RISCV_HWPROBE_EXT_ZIMOP (1ULL << 42) -#define RISCV_HWPROBE_EXT_ZCA (1ULL << 43) -#define RISCV_HWPROBE_EXT_ZCB (1ULL << 44) -#define RISCV_HWPROBE_EXT_ZCD (1ULL << 45) -#define RISCV_HWPROBE_EXT_ZCF (1ULL << 46) -#define RISCV_HWPROBE_EXT_ZCMOP (1ULL << 47) -#define RISCV_HWPROBE_EXT_ZAWRS (1ULL << 48) -#define RISCV_HWPROBE_EXT_SUPM (1ULL << 49) -#define RISCV_HWPROBE_EXT_ZICNTR (1ULL << 50) -#define RISCV_HWPROBE_EXT_ZIHPM (1ULL << 51) -#define RISCV_HWPROBE_EXT_ZFBFMIN (1ULL << 52) -#define RISCV_HWPROBE_EXT_ZVFBFMIN (1ULL << 53) -#define RISCV_HWPROBE_EXT_ZVFBFWMA (1ULL << 54) -#define RISCV_HWPROBE_EXT_ZICBOM (1ULL << 55) -#define RISCV_HWPROBE_EXT_ZAAMO (1ULL << 56) -#define RISCV_HWPROBE_EXT_ZALRSC (1ULL << 57) -#define RISCV_HWPROBE_EXT_ZABHA (1ULL << 58) -#define RISCV_HWPROBE_EXT_ZALASR (1ULL << 59) -#define RISCV_HWPROBE_EXT_ZICBOP (1ULL << 60) -#define RISCV_HWPROBE_EXT_ZILSD (1ULL << 61) -#define RISCV_HWPROBE_EXT_ZCLSD (1ULL << 62) -#define RISCV_HWPROBE_EXT_ZICFILP (1ULL << 63) +#define RISCV_HWPROBE_IMA_FD _BITULL(0) +#define RISCV_HWPROBE_IMA_C _BITULL(1) +#define RISCV_HWPROBE_IMA_V _BITULL(2) +#define RISCV_HWPROBE_EXT_ZBA _BITULL(3) +#define RISCV_HWPROBE_EXT_ZBB _BITULL(4) +#define RISCV_HWPROBE_EXT_ZBS _BITULL(5) +#define RISCV_HWPROBE_EXT_ZICBOZ _BITULL(6) +#define RISCV_HWPROBE_EXT_ZBC _BITULL(7) +#define RISCV_HWPROBE_EXT_ZBKB _BITULL(8) +#define RISCV_HWPROBE_EXT_ZBKC _BITULL(9) +#define RISCV_HWPROBE_EXT_ZBKX _BITULL(10) +#define RISCV_HWPROBE_EXT_ZKND _BITULL(11) +#define RISCV_HWPROBE_EXT_ZKNE _BITULL(12) +#define RISCV_HWPROBE_EXT_ZKNH _BITULL(13) +#define RISCV_HWPROBE_EXT_ZKSED _BITULL(14) +#define RISCV_HWPROBE_EXT_ZKSH _BITULL(15) +#define RISCV_HWPROBE_EXT_ZKT _BITULL(16) +#define RISCV_HWPROBE_EXT_ZVBB _BITULL(17) +#define RISCV_HWPROBE_EXT_ZVBC _BITULL(18) +#define RISCV_HWPROBE_EXT_ZVKB _BITULL(19) +#define RISCV_HWPROBE_EXT_ZVKG _BITULL(20) +#define RISCV_HWPROBE_EXT_ZVKNED _BITULL(21) +#define RISCV_HWPROBE_EXT_ZVKNHA _BITULL(22) +#define RISCV_HWPROBE_EXT_ZVKNHB _BITULL(23) +#define RISCV_HWPROBE_EXT_ZVKSED _BITULL(24) +#define RISCV_HWPROBE_EXT_ZVKSH _BITULL(25) +#define RISCV_HWPROBE_EXT_ZVKT _BITULL(26) +#define RISCV_HWPROBE_EXT_ZFH _BITULL(27) +#define RISCV_HWPROBE_EXT_ZFHMIN _BITULL(28) +#define RISCV_HWPROBE_EXT_ZIHINTNTL _BITULL(29) +#define RISCV_HWPROBE_EXT_ZVFH _BITULL(30) +#define RISCV_HWPROBE_EXT_ZVFHMIN _BITULL(31) +#define RISCV_HWPROBE_EXT_ZFA _BITULL(32) +#define RISCV_HWPROBE_EXT_ZTSO _BITULL(33) +#define RISCV_HWPROBE_EXT_ZACAS _BITULL(34) +#define RISCV_HWPROBE_EXT_ZICOND _BITULL(35) +#define RISCV_HWPROBE_EXT_ZIHINTPAUSE _BITULL(36) +#define RISCV_HWPROBE_EXT_ZVE32X _BITULL(37) +#define RISCV_HWPROBE_EXT_ZVE32F _BITULL(38) +#define RISCV_HWPROBE_EXT_ZVE64X _BITULL(39) +#define RISCV_HWPROBE_EXT_ZVE64F _BITULL(40) +#define RISCV_HWPROBE_EXT_ZVE64D _BITULL(41) +#define RISCV_HWPROBE_EXT_ZIMOP _BITULL(42) +#define RISCV_HWPROBE_EXT_ZCA _BITULL(43) +#define RISCV_HWPROBE_EXT_ZCB _BITULL(44) +#define RISCV_HWPROBE_EXT_ZCD _BITULL(45) +#define RISCV_HWPROBE_EXT_ZCF _BITULL(46) +#define RISCV_HWPROBE_EXT_ZCMOP _BITULL(47) +#define RISCV_HWPROBE_EXT_ZAWRS _BITULL(48) +#define RISCV_HWPROBE_EXT_SUPM _BITULL(49) +#define RISCV_HWPROBE_EXT_ZICNTR _BITULL(50) +#define RISCV_HWPROBE_EXT_ZIHPM _BITULL(51) +#define RISCV_HWPROBE_EXT_ZFBFMIN _BITULL(52) +#define RISCV_HWPROBE_EXT_ZVFBFMIN _BITULL(53) +#define RISCV_HWPROBE_EXT_ZVFBFWMA _BITULL(54) +#define RISCV_HWPROBE_EXT_ZICBOM _BITULL(55) +#define RISCV_HWPROBE_EXT_ZAAMO _BITULL(56) +#define RISCV_HWPROBE_EXT_ZALRSC _BITULL(57) +#define RISCV_HWPROBE_EXT_ZABHA _BITULL(58) +#define RISCV_HWPROBE_EXT_ZALASR _BITULL(59) +#define RISCV_HWPROBE_EXT_ZICBOP _BITULL(60) +#define RISCV_HWPROBE_EXT_ZILSD _BITULL(61) +#define RISCV_HWPROBE_EXT_ZCLSD _BITULL(62) +#define RISCV_HWPROBE_EXT_ZICFILP _BITULL(63) #define RISCV_HWPROBE_KEY_CPUPERF_0 5 -#define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0) -#define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0) -#define RISCV_HWPROBE_MISALIGNED_SLOW (2 << 0) -#define RISCV_HWPROBE_MISALIGNED_FAST (3 << 0) -#define RISCV_HWPROBE_MISALIGNED_UNSUPPORTED (4 << 0) -#define RISCV_HWPROBE_MISALIGNED_MASK (7 << 0) +#define RISCV_HWPROBE_MISALIGNED_UNKNOWN 0 +#define RISCV_HWPROBE_MISALIGNED_EMULATED 1 +#define RISCV_HWPROBE_MISALIGNED_SLOW 2 +#define RISCV_HWPROBE_MISALIGNED_FAST 3 +#define RISCV_HWPROBE_MISALIGNED_UNSUPPORTED 4 +#define RISCV_HWPROBE_MISALIGNED_MASK 7 #define RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE 6 #define RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS 7 #define RISCV_HWPROBE_KEY_TIME_CSR_FREQ 8 @@ -115,11 +116,16 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_KEY_VENDOR_EXT_MIPS_0 14 #define RISCV_HWPROBE_KEY_ZICBOP_BLOCK_SIZE 15 #define RISCV_HWPROBE_KEY_IMA_EXT_1 16 -#define RISCV_HWPROBE_EXT_ZICFISS (1ULL << 0) +#define RISCV_HWPROBE_EXT_ZICFISS _BITULL(0) +#define RISCV_HWPROBE_EXT_ZICCLSM _BITULL(1) +#define RISCV_HWPROBE_EXT_ZICCAMOA _BITULL(2) +#define RISCV_HWPROBE_EXT_ZICCIF _BITULL(3) +#define RISCV_HWPROBE_EXT_ZICCRSE _BITULL(4) +#define RISCV_HWPROBE_EXT_ZA64RS _BITULL(5) /* Increase RISCV_HWPROBE_MAX_KEY when adding items. */ /* Flags */ -#define RISCV_HWPROBE_WHICH_CPUS (1 << 0) +#define RISCV_HWPROBE_WHICH_CPUS _BITUL(0) #endif diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index cabb99cadfb6..ebe1c3588177 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -128,3 +128,5 @@ obj-$(CONFIG_ACPI_NUMA) += acpi_numa.o obj-$(CONFIG_GENERIC_CPU_VULNERABILITIES) += bugs.o obj-$(CONFIG_RISCV_USER_CFI) += usercfi.o + +obj-$(CONFIG_RISCV_ISA_SSQOSID) += qos.o diff --git a/arch/riscv/kernel/acpi.c b/arch/riscv/kernel/acpi.c index 068e0b404b6f..efd52a5d05b5 100644 --- a/arch/riscv/kernel/acpi.c +++ b/arch/riscv/kernel/acpi.c @@ -353,3 +353,8 @@ int acpi_get_cpu_uid(unsigned int cpu, u32 *uid) return 0; } EXPORT_SYMBOL_GPL(acpi_get_cpu_uid); + +void __init arch_reserve_mem_area(acpi_physical_address addr, size_t size) +{ + memblock_mark_nomap(addr, size); +} diff --git a/arch/riscv/kernel/alternative.c b/arch/riscv/kernel/alternative.c index 104dc0862c5c..c0c9306022c5 100644 --- a/arch/riscv/kernel/alternative.c +++ b/arch/riscv/kernel/alternative.c @@ -173,24 +173,14 @@ static void __init_or_module _apply_alternatives(struct alt_entry *begin, stage); } -static void __init apply_vdso_alternatives(void *start) +static __always_inline +void __init apply_vdso_alternatives(void *base, size_t alternatives_begin, size_t alternatives_end) { - const Elf_Ehdr *hdr; - const Elf_Shdr *shdr; - const Elf_Shdr *alt; - struct alt_entry *begin, *end; - - hdr = (Elf_Ehdr *)start; - shdr = (void *)hdr + hdr->e_shoff; - alt = find_section(hdr, shdr, ".alternative"); - if (!alt) + if (alternatives_begin == alternatives_end) return; - begin = (void *)hdr + alt->sh_offset, - end = (void *)hdr + alt->sh_offset + alt->sh_size, - - _apply_alternatives((struct alt_entry *)begin, - (struct alt_entry *)end, + _apply_alternatives(base + alternatives_begin, + base + alternatives_end, RISCV_ALTERNATIVES_BOOT); } @@ -204,10 +194,19 @@ void __init apply_boot_alternatives(void) RISCV_ALTERNATIVES_BOOT); if (IS_ENABLED(CONFIG_MMU)) - apply_vdso_alternatives(vdso_start); + apply_vdso_alternatives(vdso_start, + __vdso_alternatives_start_offset, + __vdso_alternatives_end_offset); if (IS_ENABLED(CONFIG_RISCV_USER_CFI)) - apply_vdso_alternatives(vdso_cfi_start); + apply_vdso_alternatives(vdso_cfi_start, + __vdso_alternatives_start_cfi_offset, + __vdso_alternatives_end_cfi_offset); + + if (IS_ENABLED(CONFIG_COMPAT)) + apply_vdso_alternatives(compat_vdso_start, + compat__vdso_alternatives_start_offset, + compat__vdso_alternatives_end_offset); } /* diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index f46aa5602d74..d2ec96843456 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -84,7 +84,7 @@ EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); static int riscv_ext_f_depends(const struct riscv_isa_ext_data *data, const unsigned long *isa_bitmap) { - if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_f)) + if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_F)) return 0; return -EPROBE_DEFER; @@ -146,7 +146,7 @@ static int riscv_ext_f_validate(const struct riscv_isa_ext_data *data, * Due to extension ordering, d is checked before f, so no deferral * is required. */ - if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d)) { + if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_D)) { pr_warn_once("This kernel does not support systems with F but not D\n"); return -EINVAL; } @@ -189,7 +189,7 @@ static int riscv_ext_vector_float_validate(const struct riscv_isa_ext_data *data * Since this function validates vector only, and v/Zve* are probed * after f/d, there's no need for a deferral here. */ - if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d)) + if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_D)) return -EINVAL; return 0; @@ -224,7 +224,7 @@ static int riscv_ext_zcd_validate(const struct riscv_isa_ext_data *data, const unsigned long *isa_bitmap) { if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA) && - __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d)) + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_D)) return 0; return -EPROBE_DEFER; @@ -237,7 +237,7 @@ static int riscv_ext_zcf_validate(const struct riscv_isa_ext_data *data, return -EINVAL; if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA) && - __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_f)) + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_F)) return 0; return -EPROBE_DEFER; @@ -330,6 +330,27 @@ static const unsigned int riscv_a_exts[] = { RISCV_ISA_EXT_ZKNE, \ RISCV_ISA_EXT_ZKNH +static int riscv_ext_smcdeleg_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_SSCSRIND) && + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZIHPM) && + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZICNTR)) + return 0; + + return -EPROBE_DEFER; +} + +static int riscv_ext_ssccfg_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (!riscv_ext_smcdeleg_validate(data, isa_bitmap) && + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_SMCDELEG)) + return 0; + + return -EPROBE_DEFER; +} + static const unsigned int riscv_zk_bundled_exts[] = { RISCV_ISA_EXT_ZKN, RISCV_ISA_EXT_ZKR, @@ -452,7 +473,8 @@ static const unsigned int riscv_c_exts[] = { /* * The canonical order of ISA extension names in the ISA string is defined in - * chapter 27 of the unprivileged specification. + * Chapter 27 of the RISC-V Instruction Set Manual Volume I Unprivileged ISA + * (Document Version 20191213). * * Ordinarily, for in-kernel data structures, this order is unimportant but * isa_ext_arr defines the order of the ISA string in /proc/cpuinfo. @@ -490,18 +512,22 @@ static const unsigned int riscv_c_exts[] = { * New entries to this struct should follow the ordering rules described above. */ const struct riscv_isa_ext_data riscv_isa_ext[] = { - __RISCV_ISA_EXT_DATA(i, RISCV_ISA_EXT_i), - __RISCV_ISA_EXT_DATA(m, RISCV_ISA_EXT_m), - __RISCV_ISA_EXT_SUPERSET(a, RISCV_ISA_EXT_a, riscv_a_exts), - __RISCV_ISA_EXT_DATA_VALIDATE(f, RISCV_ISA_EXT_f, riscv_ext_f_validate), - __RISCV_ISA_EXT_DATA_VALIDATE(d, RISCV_ISA_EXT_d, riscv_ext_d_validate), - __RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_q), - __RISCV_ISA_EXT_SUPERSET(c, RISCV_ISA_EXT_c, riscv_c_exts), - __RISCV_ISA_EXT_SUPERSET_VALIDATE(v, RISCV_ISA_EXT_v, riscv_v_exts, riscv_ext_vector_float_validate), - __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_h), + __RISCV_ISA_EXT_DATA(i, RISCV_ISA_EXT_I), + __RISCV_ISA_EXT_DATA(m, RISCV_ISA_EXT_M), + __RISCV_ISA_EXT_SUPERSET(a, RISCV_ISA_EXT_A, riscv_a_exts), + __RISCV_ISA_EXT_DATA_VALIDATE(f, RISCV_ISA_EXT_F, riscv_ext_f_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(d, RISCV_ISA_EXT_D, riscv_ext_d_validate), + __RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_Q), + __RISCV_ISA_EXT_SUPERSET(c, RISCV_ISA_EXT_C, riscv_c_exts), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(v, RISCV_ISA_EXT_V, riscv_v_exts, + riscv_ext_vector_float_validate), + __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_H), __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicbom, RISCV_ISA_EXT_ZICBOM, riscv_xlinuxenvcfg_exts, riscv_ext_zicbom_validate), __RISCV_ISA_EXT_DATA_VALIDATE(zicbop, RISCV_ISA_EXT_ZICBOP, riscv_ext_zicbop_validate), __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts, riscv_ext_zicboz_validate), + __RISCV_ISA_EXT_DATA(ziccamoa, RISCV_ISA_EXT_ZICCAMOA), + __RISCV_ISA_EXT_DATA(ziccif, RISCV_ISA_EXT_ZICCIF), + __RISCV_ISA_EXT_DATA(zicclsm, RISCV_ISA_EXT_ZICCLSM), __RISCV_ISA_EXT_DATA(ziccrse, RISCV_ISA_EXT_ZICCRSE), __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicfilp, RISCV_ISA_EXT_ZICFILP, riscv_xlinuxenvcfg_exts, riscv_cfilp_validate), @@ -515,6 +541,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM), __RISCV_ISA_EXT_DATA(zimop, RISCV_ISA_EXT_ZIMOP), + __RISCV_ISA_EXT_DATA(za64rs, RISCV_ISA_EXT_ZA64RS), __RISCV_ISA_EXT_DATA(zaamo, RISCV_ISA_EXT_ZAAMO), __RISCV_ISA_EXT_DATA(zabha, RISCV_ISA_EXT_ZABHA), __RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS), @@ -576,12 +603,19 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvksg, riscv_zvksg_bundled_exts, riscv_ext_vector_crypto_validate), __RISCV_ISA_EXT_DATA_VALIDATE(zvkt, RISCV_ISA_EXT_ZVKT, riscv_ext_vector_crypto_validate), __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA), + __RISCV_ISA_EXT_DATA_VALIDATE(smcdeleg, RISCV_ISA_EXT_SMCDELEG, + riscv_ext_smcdeleg_validate), + __RISCV_ISA_EXT_DATA(smcntrpmf, RISCV_ISA_EXT_SMCNTRPMF), + __RISCV_ISA_EXT_DATA(smcsrind, RISCV_ISA_EXT_SMCSRIND), __RISCV_ISA_EXT_DATA(smmpm, RISCV_ISA_EXT_SMMPM), __RISCV_ISA_EXT_SUPERSET(smnpm, RISCV_ISA_EXT_SMNPM, riscv_xlinuxenvcfg_exts), __RISCV_ISA_EXT_DATA(smstateen, RISCV_ISA_EXT_SMSTATEEN), __RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA), + __RISCV_ISA_EXT_DATA_VALIDATE(ssccfg, RISCV_ISA_EXT_SSCCFG, riscv_ext_ssccfg_validate), __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF), + __RISCV_ISA_EXT_DATA(sscsrind, RISCV_ISA_EXT_SSCSRIND), __RISCV_ISA_EXT_SUPERSET(ssnpm, RISCV_ISA_EXT_SSNPM, riscv_xlinuxenvcfg_exts), + __RISCV_ISA_EXT_DATA(ssqosid, RISCV_ISA_EXT_SSQOSID), __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC), __RISCV_ISA_EXT_DATA(svade, RISCV_ISA_EXT_SVADE), __RISCV_ISA_EXT_DATA_VALIDATE(svadu, RISCV_ISA_EXT_SVADU, riscv_ext_svadu_validate), @@ -897,7 +931,7 @@ static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap) * marchid. */ if (acpi_disabled && boot_vendorid == THEAD_VENDOR_ID && boot_archid == 0x0) - clear_bit(RISCV_ISA_EXT_v, source_isa); + clear_bit(RISCV_ISA_EXT_V, source_isa); riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap); @@ -1105,13 +1139,13 @@ void __init riscv_fill_hwcap(void) unsigned long isa2hwcap[RISCV_ISA_EXT_BASE] = {0}; int i, j; - isa2hwcap[RISCV_ISA_EXT_i] = COMPAT_HWCAP_ISA_I; - isa2hwcap[RISCV_ISA_EXT_m] = COMPAT_HWCAP_ISA_M; - isa2hwcap[RISCV_ISA_EXT_a] = COMPAT_HWCAP_ISA_A; - isa2hwcap[RISCV_ISA_EXT_f] = COMPAT_HWCAP_ISA_F; - isa2hwcap[RISCV_ISA_EXT_d] = COMPAT_HWCAP_ISA_D; - isa2hwcap[RISCV_ISA_EXT_c] = COMPAT_HWCAP_ISA_C; - isa2hwcap[RISCV_ISA_EXT_v] = COMPAT_HWCAP_ISA_V; + isa2hwcap[RISCV_ISA_EXT_I] = COMPAT_HWCAP_ISA_I; + isa2hwcap[RISCV_ISA_EXT_M] = COMPAT_HWCAP_ISA_M; + isa2hwcap[RISCV_ISA_EXT_A] = COMPAT_HWCAP_ISA_A; + isa2hwcap[RISCV_ISA_EXT_F] = COMPAT_HWCAP_ISA_F; + isa2hwcap[RISCV_ISA_EXT_D] = COMPAT_HWCAP_ISA_D; + isa2hwcap[RISCV_ISA_EXT_C] = COMPAT_HWCAP_ISA_C; + isa2hwcap[RISCV_ISA_EXT_V] = COMPAT_HWCAP_ISA_V; if (!acpi_disabled) { riscv_fill_hwcap_from_isa_string(isa2hwcap); diff --git a/arch/riscv/kernel/efi.c b/arch/riscv/kernel/efi.c index b64bf1624a05..2d3cc57b4535 100644 --- a/arch/riscv/kernel/efi.c +++ b/arch/riscv/kernel/efi.c @@ -95,3 +95,8 @@ int __init efi_set_mapping_permissions(struct mm_struct *mm, md->num_pages << EFI_PAGE_SHIFT, set_permissions, md); } + +bool efi_poweroff_required(void) +{ + return efi_enabled(EFI_RUNTIME_SERVICES); +} diff --git a/arch/riscv/kernel/probes/decode-insn.c b/arch/riscv/kernel/probes/decode-insn.c index 65d9590bfb9f..433d9035b888 100644 --- a/arch/riscv/kernel/probes/decode-insn.c +++ b/arch/riscv/kernel/probes/decode-insn.c @@ -29,12 +29,12 @@ riscv_probe_decode_insn(probe_opcode_t *addr, struct arch_probe_insn *api) * TODO: the REJECTED ones below need to be implemented */ #ifdef CONFIG_RISCV_ISA_C - RISCV_INSN_REJECTED(c_jal, insn); RISCV_INSN_REJECTED(c_ebreak, insn); RISCV_INSN_SET_SIMULATE(c_j, insn); RISCV_INSN_SET_SIMULATE(c_jr, insn); RISCV_INSN_SET_SIMULATE(c_jalr, insn); + RISCV_INSN_SET_SIMULATE(c_jal, insn); RISCV_INSN_SET_SIMULATE(c_beqz, insn); RISCV_INSN_SET_SIMULATE(c_bnez, insn); #endif diff --git a/arch/riscv/kernel/probes/simulate-insn.c b/arch/riscv/kernel/probes/simulate-insn.c index fa581590c1f8..f8a2f6857877 100644 --- a/arch/riscv/kernel/probes/simulate-insn.c +++ b/arch/riscv/kernel/probes/simulate-insn.c @@ -163,6 +163,13 @@ bool __kprobes simulate_c_j(u32 opcode, unsigned long addr, struct pt_regs *regs return true; } +bool __kprobes simulate_c_jal(u32 opcode, unsigned long addr, struct pt_regs *regs) +{ + regs->ra = addr + 2; + + return simulate_c_j(opcode, addr, regs); +} + static bool __kprobes simulate_c_jr_jalr(u32 opcode, unsigned long addr, struct pt_regs *regs, bool is_jalr) { diff --git a/arch/riscv/kernel/probes/simulate-insn.h b/arch/riscv/kernel/probes/simulate-insn.h index 44ebbc444db9..b89e1bb01842 100644 --- a/arch/riscv/kernel/probes/simulate-insn.h +++ b/arch/riscv/kernel/probes/simulate-insn.h @@ -25,6 +25,7 @@ bool simulate_branch(u32 opcode, unsigned long addr, struct pt_regs *regs); bool simulate_jal(u32 opcode, unsigned long addr, struct pt_regs *regs); bool simulate_jalr(u32 opcode, unsigned long addr, struct pt_regs *regs); bool simulate_c_j(u32 opcode, unsigned long addr, struct pt_regs *regs); +bool simulate_c_jal(u32 opcode, unsigned long addr, struct pt_regs *regs); bool simulate_c_jr(u32 opcode, unsigned long addr, struct pt_regs *regs); bool simulate_c_jalr(u32 opcode, unsigned long addr, struct pt_regs *regs); bool simulate_c_bnez(u32 opcode, unsigned long addr, struct pt_regs *regs); diff --git a/arch/riscv/kernel/qos.c b/arch/riscv/kernel/qos.c new file mode 100644 index 000000000000..99925c0e93dd --- /dev/null +++ b/arch/riscv/kernel/qos.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/* + * Cached value of srmcfg csr for each cpu. Seeded to U32_MAX so the next + * __switch_to_srmcfg() unconditionally writes the CSR. The encoding + * MCID << 16 | RCID with both fields well under 16 bits can never + * produce this sentinel. This covers early-boot context switches that + * happen before riscv_srmcfg_init() runs as an arch_initcall. + */ +DEFINE_PER_CPU(u32, cpu_srmcfg) = U32_MAX; + +/* default srmcfg value for each cpu, set via resctrl cpu assignment */ +DEFINE_PER_CPU(u32, cpu_srmcfg_default); + +/* + * Invalidate the per-CPU srmcfg cache. Used as both the cpuhp startup + * and teardown callback. U32_MAX is not a valid srmcfg value + * (MCID << 16 | RCID, both fields under 16 bits), so the next + * __switch_to_srmcfg() always writes the CSR. + * + * Ssqosid leaves the CSR implementation-defined across hart stop/start, + * so the cached value cannot be trusted after online. The startup + * callback runs at CPUHP_AP_ONLINE_DYN, before CPUHP_AP_ACTIVE makes + * the CPU schedulable, so the cache is invalidated before any normal + * task runs and the CSR is written on that task's first switch. + * The teardown callback is not relied on. Idle and per-CPU kthreads keep + * switching as the CPU goes down and overwrite the sentinel with the CPU + * default, so it does not survive the offline period. + */ +static int riscv_srmcfg_reset_cache(unsigned int cpu) +{ + per_cpu(cpu_srmcfg, cpu) = U32_MAX; + return 0; +} + +/* + * CPU PM notifier: invalidate the cached srmcfg on resume from a deep + * idle / suspend. Ssqosid leaves CSR_SRMCFG state across low-power + * transitions implementation-defined, and the boot CPU never goes + * through the cpuhp online callback during system suspend, so without + * this hook __switch_to_srmcfg() would skip the CSR write when the + * outgoing task happens to share its srmcfg with the pre-suspend cache. + */ +static int riscv_srmcfg_pm_notify(struct notifier_block *nb, + unsigned long action, void *unused) +{ + switch (action) { + case CPU_PM_EXIT: + case CPU_PM_ENTER_FAILED: + /* + * The CSR is implementation-defined across the low-power + * transition. Invalidate the cache and eagerly rewrite the + * CSR for the current task so it does not run mis-tagged + * until the next context switch. + */ + __this_cpu_write(cpu_srmcfg, U32_MAX); + __switch_to_srmcfg(current); + break; + } + return NOTIFY_OK; +} + +static struct notifier_block riscv_srmcfg_pm_nb = { + .notifier_call = riscv_srmcfg_pm_notify, +}; + +static int __init riscv_srmcfg_init(void) +{ + int err; + + if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_SSQOSID)) + return 0; + + /* + * cpuhp_setup_state() invokes the startup callback locally on every + * already-online CPU, so no separate seed loop is needed here. + */ + err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "riscv/srmcfg:online", + riscv_srmcfg_reset_cache, riscv_srmcfg_reset_cache); + if (err < 0) + pr_warn("srmcfg: cpuhp setup failed (%d), cache not invalidated on CPU online\n", + err); + + /* + * Register the PM notifier even if the cpuhp setup failed. It is + * independent of the cpuhp state and guards suspend/resume. + */ + cpu_pm_register_notifier(&riscv_srmcfg_pm_nb); + return 0; +} +arch_initcall(riscv_srmcfg_init); diff --git a/arch/riscv/kernel/reset.c b/arch/riscv/kernel/reset.c index 912288572226..14eb08a6db85 100644 --- a/arch/riscv/kernel/reset.c +++ b/arch/riscv/kernel/reset.c @@ -3,10 +3,11 @@ * Copyright (C) 2012 Regents of the University of California */ +#include #include #include -static void default_power_off(void) +static void __noreturn default_power_off(void) { while (1) wait_for_interrupt(); @@ -17,6 +18,12 @@ EXPORT_SYMBOL(pm_power_off); void machine_restart(char *cmd) { + /* + * UpdateCapsule() depends on the system being reset via ResetSystem(). + */ + if (efi_enabled(EFI_RUNTIME_SERVICES)) + efi_reboot(reboot_mode, NULL); + do_kernel_restart(cmd); while (1); } diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index 52d1d2b8f338..a32344bb220d 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -321,6 +321,8 @@ void __init setup_arch(char **cmdline_p) efi_init(); paging_init(); + acpi_table_upgrade(); + /* Parse the ACPI tables for possible boot-time configuration */ acpi_boot_table_init(); diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c index caf6762427c8..bd6ca7d769da 100644 --- a/arch/riscv/kernel/sys_hwprobe.c +++ b/arch/riscv/kernel/sys_hwprobe.c @@ -5,9 +5,6 @@ * more details. */ #include -#include -#include -#include #include #include #include @@ -88,10 +85,10 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, if (has_fpu()) pair->value |= RISCV_HWPROBE_IMA_FD; - if (riscv_isa_extension_available(NULL, c)) + if (riscv_isa_extension_available(NULL, C)) pair->value |= RISCV_HWPROBE_IMA_C; - if (has_vector() && riscv_isa_extension_available(NULL, v)) + if (has_vector() && riscv_isa_extension_available(NULL, V)) pair->value |= RISCV_HWPROBE_IMA_V; /* @@ -205,6 +202,11 @@ static void hwprobe_isa_ext1(struct riscv_hwprobe *pair, * in the hart_isa bitmap, are made. */ EXT_KEY(isainfo->isa, ZICFISS, pair->value, missing); + EXT_KEY(isainfo->isa, ZICCLSM, pair->value, missing); + EXT_KEY(isainfo->isa, ZICCAMOA, pair->value, missing); + EXT_KEY(isainfo->isa, ZICCIF, pair->value, missing); + EXT_KEY(isainfo->isa, ZICCRSE, pair->value, missing); + EXT_KEY(isainfo->isa, ZA64RS, pair->value, missing); } /* Now turn off reporting features if any CPU is missing it. */ @@ -505,32 +507,28 @@ static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs, return 0; } +static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs, + size_t pair_count, size_t cpusetsize, + unsigned long __user *cpus_user, + unsigned int flags) +{ + if (flags & RISCV_HWPROBE_WHICH_CPUS) + return hwprobe_get_cpus(pairs, pair_count, cpusetsize, + cpus_user, flags); + + return hwprobe_get_values(pairs, pair_count, cpusetsize, + cpus_user, flags); +} + #ifdef CONFIG_MMU -static DECLARE_COMPLETION(boot_probes_done); -static atomic_t pending_boot_probes = ATOMIC_INIT(1); - -void riscv_hwprobe_register_async_probe(void) -{ - atomic_inc(&pending_boot_probes); -} - -void riscv_hwprobe_complete_async_probe(void) -{ - if (atomic_dec_and_test(&pending_boot_probes)) - complete(&boot_probes_done); -} - -static int complete_hwprobe_vdso_data(void) +static int __init init_hwprobe_vdso_data(void) { struct vdso_arch_data *avd = vdso_k_arch_data; u64 id_bitsmash = 0; struct riscv_hwprobe pair; int key; - if (unlikely(!atomic_dec_and_test(&pending_boot_probes))) - wait_for_completion(&boot_probes_done); - /* * Initialize vDSO data with the answers for the "all CPUs" case, to * save a syscall in the common case. @@ -558,52 +556,13 @@ static int complete_hwprobe_vdso_data(void) * vDSO should defer to the kernel for exotic cpu masks. */ avd->homogeneous_cpus = id_bitsmash != 0 && id_bitsmash != -1; - - /* - * Make sure all the VDSO values are visible before we look at them. - * This pairs with the implicit "no speculativly visible accesses" - * barrier in the VDSO hwprobe code. - */ - smp_wmb(); - avd->ready = true; - return 0; -} - -static int __init init_hwprobe_vdso_data(void) -{ - struct vdso_arch_data *avd = vdso_k_arch_data; - - /* - * Prevent the vDSO cached values from being used, as they're not ready - * yet. - */ - avd->ready = false; return 0; } arch_initcall_sync(init_hwprobe_vdso_data); -#else - -static int complete_hwprobe_vdso_data(void) { return 0; } - #endif /* CONFIG_MMU */ -static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs, - size_t pair_count, size_t cpusetsize, - unsigned long __user *cpus_user, - unsigned int flags) -{ - DO_ONCE_SLEEPABLE(complete_hwprobe_vdso_data); - - if (flags & RISCV_HWPROBE_WHICH_CPUS) - return hwprobe_get_cpus(pairs, pair_count, cpusetsize, - cpus_user, flags); - - return hwprobe_get_values(pairs, pair_count, cpusetsize, - cpus_user, flags); -} - SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs, size_t, pair_count, size_t, cpusetsize, unsigned long __user *, cpus, unsigned int, flags) diff --git a/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S b/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S index f16deee9e091..15e50934bbd7 100644 --- a/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S +++ b/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S @@ -179,6 +179,23 @@ test_kprobes_c_bnez_addr3: ret SYM_FUNC_END(test_kprobes_c_bnez) +#ifdef CONFIG_32BIT +SYM_FUNC_START(test_kprobes_c_jal) + li a0, 0 + mv a1, ra +test_kprobes_c_jal_addr1: + c.jal 2f + ret +1: li a0, KPROBE_TEST_MAGIC_UPPER + ret +test_kprobes_c_jal_addr2: +2: c.jal 1b + li a2, KPROBE_TEST_MAGIC_LOWER + add a0, a0, a2 + jr a1 +SYM_FUNC_END(test_kprobes_c_jal) +#endif + #endif /* CONFIG_RISCV_ISA_C */ .section .rodata @@ -209,6 +226,10 @@ SYM_DATA_START(test_kprobes_addresses) RISCV_PTR test_kprobes_c_bnez_addr1 RISCV_PTR test_kprobes_c_bnez_addr2 RISCV_PTR test_kprobes_c_bnez_addr3 +#ifdef CONFIG_32BIT + RISCV_PTR test_kprobes_c_jal_addr1 + RISCV_PTR test_kprobes_c_jal_addr2 +#endif #endif /* CONFIG_RISCV_ISA_C */ RISCV_PTR 0 SYM_DATA_END(test_kprobes_addresses) @@ -226,6 +247,9 @@ SYM_DATA_START(test_kprobes_functions) RISCV_PTR test_kprobes_c_jalr RISCV_PTR test_kprobes_c_beqz RISCV_PTR test_kprobes_c_bnez +#ifdef CONFIG_32BIT + RISCV_PTR test_kprobes_c_jal +#endif #endif /* CONFIG_RISCV_ISA_C */ RISCV_PTR 0 SYM_DATA_END(test_kprobes_functions) diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index 2f57fd48ec28..f8292adda099 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -269,6 +269,7 @@ static bool probe_single_step_handler(struct pt_regs *regs) return user ? uprobe_single_step_handler(regs) : kprobe_single_step_handler(regs); } +NOKPROBE_SYMBOL(probe_single_step_handler); static bool probe_breakpoint_handler(struct pt_regs *regs) { @@ -276,6 +277,7 @@ static bool probe_breakpoint_handler(struct pt_regs *regs) return user ? uprobe_breakpoint_handler(regs) : kprobe_breakpoint_handler(regs); } +NOKPROBE_SYMBOL(probe_breakpoint_handler); void handle_break(struct pt_regs *regs) { @@ -300,6 +302,7 @@ void handle_break(struct pt_regs *regs) else die(regs, "Kernel BUG"); } +NOKPROBE_SYMBOL(handle_break); asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs) { diff --git a/arch/riscv/kernel/unaligned_access_speed.c b/arch/riscv/kernel/unaligned_access_speed.c index 36201613d2ca..67f1a0371659 100644 --- a/arch/riscv/kernel/unaligned_access_speed.c +++ b/arch/riscv/kernel/unaligned_access_speed.c @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include @@ -288,18 +287,9 @@ static void check_vector_unaligned_access(struct work_struct *work __always_unus __free_pages(page, MISALIGNED_BUFFER_ORDER); } -/* Measure unaligned access speed on all CPUs present at boot in parallel. */ -static int vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) -{ - schedule_on_each_cpu(check_vector_unaligned_access); - riscv_hwprobe_complete_async_probe(); - - return 0; -} #else /* CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS */ -static int vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) +static void check_vector_unaligned_access(struct work_struct *work __always_unused) { - return 0; } #endif @@ -387,12 +377,7 @@ static int __init check_unaligned_access_all_cpus(void) per_cpu(vector_misaligned_access, cpu) = unaligned_vector_speed_param; } else if (!check_vector_unaligned_access_emulated_all_cpus() && IS_ENABLED(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS)) { - riscv_hwprobe_register_async_probe(); - if (IS_ERR(kthread_run(vec_check_unaligned_access_speed_all_cpus, - NULL, "vec_check_unaligned_access_speed_all_cpus"))) { - pr_warn("Failed to create vec_unalign_check kthread\n"); - riscv_hwprobe_complete_async_probe(); - } + schedule_on_each_cpu(check_vector_unaligned_access); } /* diff --git a/arch/riscv/kernel/usercfi.c b/arch/riscv/kernel/usercfi.c index 5a7113d69bad..f027e6e05251 100644 --- a/arch/riscv/kernel/usercfi.c +++ b/arch/riscv/kernel/usercfi.c @@ -110,7 +110,7 @@ void set_indir_lp_lock(struct task_struct *task, bool lock) } /* * The shadow stack only stores the return address and not any variables - * this should be more than sufficient for most applications. + * 512M should be more than sufficient for most applications. * Else PAGE_ALIGN it and return back */ static unsigned long calc_shstk_size(unsigned long size) @@ -118,7 +118,7 @@ static unsigned long calc_shstk_size(unsigned long size) if (size) return PAGE_ALIGN(size); - return PAGE_ALIGN(min(rlimit(RLIMIT_STACK) / 2, SZ_2G)); + return PAGE_ALIGN(min(rlimit(RLIMIT_STACK) / 8, SZ_512M)); } /* diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index 43ee881f6c6f..8dbf2532a573 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -41,6 +41,8 @@ ccflags-y += $(CFI_FULL) asflags-y += $(KBUILD_BASE_ISA)$(CFI_MARCH) asflags-y += $(CFI_FULL) +ccflags-remove-y += $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_LTO) + ifneq ($(c-gettimeofday-y),) CFLAGS_vgettimeofday.o += -fPIC -include $(c-gettimeofday-y) endif @@ -68,11 +70,6 @@ ifneq ($(filter vgettimeofday, $(vdso-syms)),) CPPFLAGS_$(vdso_lds) += -DHAS_VGETTIMEOFDAY endif -# Disable -pg to prevent insert call site -CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_LTO) -CFLAGS_REMOVE_getrandom.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_LTO) -CFLAGS_REMOVE_hwprobe.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_LTO) - # Force dependency $(obj)/$(vdso_o): $(obj)/$(vdso_so) diff --git a/arch/riscv/kernel/vdso/hwprobe.c b/arch/riscv/kernel/vdso/hwprobe.c index 8f45500d0a6e..2ddeba6c68dd 100644 --- a/arch/riscv/kernel/vdso/hwprobe.c +++ b/arch/riscv/kernel/vdso/hwprobe.c @@ -27,7 +27,7 @@ static int riscv_vdso_get_values(struct riscv_hwprobe *pairs, size_t pair_count, * homogeneous, then this function can handle requests for arbitrary * masks. */ - if (flags != 0 || (!all_cpus && !avd->homogeneous_cpus) || unlikely(!avd->ready)) + if ((flags != 0) || (!all_cpus && !avd->homogeneous_cpus)) return riscv_hwprobe(pairs, pair_count, cpusetsize, cpus, flags); /* This is something we can handle, fill out the pairs. */ diff --git a/arch/riscv/kernel/vdso/vdso.lds.S b/arch/riscv/kernel/vdso/vdso.lds.S index c29ef12a63bb..5e9424df932b 100644 --- a/arch/riscv/kernel/vdso/vdso.lds.S +++ b/arch/riscv/kernel/vdso/vdso.lds.S @@ -45,9 +45,11 @@ SECTIONS .text : { *(.text .text.*) } :text . = ALIGN(4); + __vdso_alternatives_start = .; .alternative : { *(.alternative) } + __vdso_alternatives_end = .; } /* diff --git a/arch/riscv/kernel/vmlinux-xip.lds.S b/arch/riscv/kernel/vmlinux-xip.lds.S deleted file mode 100644 index a7611789bad5..000000000000 --- a/arch/riscv/kernel/vmlinux-xip.lds.S +++ /dev/null @@ -1,143 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 Regents of the University of California - * Copyright (C) 2017 SiFive - * Copyright (C) 2020 Vitaly Wool, Konsulko AB - */ - -#include -#define LOAD_OFFSET KERNEL_LINK_ADDR -/* No __ro_after_init data in the .rodata section - which will always be ro */ -#define RO_AFTER_INIT_DATA - -#include -#include -#include -#include -#include - -OUTPUT_ARCH(riscv) -ENTRY(_start) - -jiffies = jiffies_64; - -SECTIONS -{ - /* Beginning of code and text segment */ - . = LOAD_OFFSET; - _xiprom = .; - _start = .; - HEAD_TEXT_SECTION - INIT_TEXT_SECTION(PAGE_SIZE) - /* we have to discard exit text and such at runtime, not link time */ - __exittext_begin = .; - .exit.text : - { - EXIT_TEXT - } - __exittext_end = .; - - .text : { - _text = .; - _stext = .; - TEXT_TEXT - SCHED_TEXT - LOCK_TEXT - KPROBES_TEXT - ENTRY_TEXT - IRQENTRY_TEXT - SOFTIRQENTRY_TEXT - _etext = .; - } - RO_DATA(L1_CACHE_BYTES) - .srodata : { - *(.srodata*) - } - .init.rodata : { - INIT_SETUP(16) - INIT_CALLS - CON_INITCALL - INIT_RAM_FS - } - _exiprom = .; /* End of XIP ROM area */ - - -/* - * From this point, stuff is considered writable and will be copied to RAM - */ - __data_loc = ALIGN(PAGE_SIZE); /* location in file */ - . = ALIGN(SECTION_ALIGN); /* location in memory */ - -#undef LOAD_OFFSET -#define LOAD_OFFSET (KERNEL_LINK_ADDR + _sdata - __data_loc) - - _sdata = .; /* Start of data section */ - _data = .; - RW_DATA(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE) - _edata = .; - __start_ro_after_init = .; - .data.ro_after_init : AT(ADDR(.data.ro_after_init) - LOAD_OFFSET) { - *(.data..ro_after_init) - } - __end_ro_after_init = .; - - . = ALIGN(PAGE_SIZE); - __init_begin = .; - .init.data : { - INIT_DATA - } - .exit.data : { - EXIT_DATA - } - . = ALIGN(8); - __soc_early_init_table : { - __soc_early_init_table_start = .; - KEEP(*(__soc_early_init_table)) - __soc_early_init_table_end = .; - } - __soc_builtin_dtb_table : { - __soc_builtin_dtb_table_start = .; - KEEP(*(__soc_builtin_dtb_table)) - __soc_builtin_dtb_table_end = .; - } - - __init_end = .; - - . = ALIGN(16); - .xip.traps : { - __xip_traps_start = .; - *(.xip.traps) - __xip_traps_end = .; - } - - . = ALIGN(PAGE_SIZE); - .sdata : { - __global_pointer$ = . + 0x800; - *(.sdata*) - *(.sbss*) - } - - BSS_SECTION(PAGE_SIZE, PAGE_SIZE, 0) - - PERCPU_SECTION(L1_CACHE_BYTES) - - .rel.dyn : AT(ADDR(.rel.dyn) - LOAD_OFFSET) { - *(.rel.dyn*) - } - - /* - * End of copied data. We need a dummy section to get its LMA. - * Also located before final ALIGN() as trailing padding is not stored - * in the resulting binary file and useless to copy. - */ - .data.endmark : AT(ADDR(.data.endmark) - LOAD_OFFSET) { } - _edata_loc = LOADADDR(.data.endmark); - - . = ALIGN(PAGE_SIZE); - _end = .; - - STABS_DEBUG - DWARF_DEBUG - - DISCARDS -} diff --git a/arch/riscv/kvm/isa.c b/arch/riscv/kvm/isa.c index 1132d909cc25..94077117d113 100644 --- a/arch/riscv/kvm/isa.c +++ b/arch/riscv/kvm/isa.c @@ -17,14 +17,14 @@ /* Mapping between KVM ISA Extension ID & guest ISA extension ID */ static const unsigned long kvm_isa_ext_arr[] = { /* Single letter extensions (alphabetically sorted) */ - [KVM_RISCV_ISA_EXT_A] = RISCV_ISA_EXT_a, - [KVM_RISCV_ISA_EXT_C] = RISCV_ISA_EXT_c, - [KVM_RISCV_ISA_EXT_D] = RISCV_ISA_EXT_d, - [KVM_RISCV_ISA_EXT_F] = RISCV_ISA_EXT_f, - [KVM_RISCV_ISA_EXT_H] = RISCV_ISA_EXT_h, - [KVM_RISCV_ISA_EXT_I] = RISCV_ISA_EXT_i, - [KVM_RISCV_ISA_EXT_M] = RISCV_ISA_EXT_m, - [KVM_RISCV_ISA_EXT_V] = RISCV_ISA_EXT_v, + KVM_ISA_EXT_ARR(A), + KVM_ISA_EXT_ARR(C), + KVM_ISA_EXT_ARR(D), + KVM_ISA_EXT_ARR(F), + KVM_ISA_EXT_ARR(H), + KVM_ISA_EXT_ARR(I), + KVM_ISA_EXT_ARR(M), + KVM_ISA_EXT_ARR(V), /* Multi letter extensions (alphabetically sorted) */ KVM_ISA_EXT_ARR(SMNPM), KVM_ISA_EXT_ARR(SMSTATEEN), diff --git a/arch/riscv/kvm/main.c b/arch/riscv/kvm/main.c index 0924c75100a2..0b8bf5f9eab1 100644 --- a/arch/riscv/kvm/main.c +++ b/arch/riscv/kvm/main.c @@ -85,7 +85,7 @@ static int __init riscv_kvm_init(void) char slist[64]; const char *str; - if (!riscv_isa_extension_available(NULL, h)) { + if (!riscv_isa_extension_available(NULL, H)) { kvm_info("hypervisor extension not available\n"); return -ENODEV; } diff --git a/arch/riscv/kvm/vcpu_fp.c b/arch/riscv/kvm/vcpu_fp.c index 6ad6df26a2fd..bb11e6757d34 100644 --- a/arch/riscv/kvm/vcpu_fp.c +++ b/arch/riscv/kvm/vcpu_fp.c @@ -21,8 +21,8 @@ void kvm_riscv_vcpu_fp_reset(struct kvm_vcpu *vcpu) struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; cntx->sstatus &= ~SR_FS; - if (riscv_isa_extension_available(vcpu->arch.isa, f) || - riscv_isa_extension_available(vcpu->arch.isa, d)) + if (riscv_isa_extension_available(vcpu->arch.isa, F) || + riscv_isa_extension_available(vcpu->arch.isa, D)) cntx->sstatus |= SR_FS_INITIAL; else cntx->sstatus |= SR_FS_OFF; @@ -38,9 +38,9 @@ void kvm_riscv_vcpu_guest_fp_save(struct kvm_cpu_context *cntx, const unsigned long *isa) { if ((cntx->sstatus & SR_FS) == SR_FS_DIRTY) { - if (riscv_isa_extension_available(isa, d)) + if (riscv_isa_extension_available(isa, D)) __kvm_riscv_fp_d_save(cntx); - else if (riscv_isa_extension_available(isa, f)) + else if (riscv_isa_extension_available(isa, F)) __kvm_riscv_fp_f_save(cntx); kvm_riscv_vcpu_fp_clean(cntx); } @@ -50,9 +50,9 @@ void kvm_riscv_vcpu_guest_fp_restore(struct kvm_cpu_context *cntx, const unsigned long *isa) { if ((cntx->sstatus & SR_FS) != SR_FS_OFF) { - if (riscv_isa_extension_available(isa, d)) + if (riscv_isa_extension_available(isa, D)) __kvm_riscv_fp_d_restore(cntx); - else if (riscv_isa_extension_available(isa, f)) + else if (riscv_isa_extension_available(isa, F)) __kvm_riscv_fp_f_restore(cntx); kvm_riscv_vcpu_fp_clean(cntx); } @@ -89,7 +89,7 @@ int kvm_riscv_vcpu_get_reg_fp(struct kvm_vcpu *vcpu, void *reg_val; if ((rtype == KVM_REG_RISCV_FP_F) && - riscv_isa_extension_available(vcpu->arch.isa, f)) { + riscv_isa_extension_available(vcpu->arch.isa, F)) { if (KVM_REG_SIZE(reg->id) != sizeof(u32)) return -EINVAL; if (reg_num == KVM_REG_RISCV_FP_F_REG(fcsr)) @@ -102,7 +102,7 @@ int kvm_riscv_vcpu_get_reg_fp(struct kvm_vcpu *vcpu, } else return -ENOENT; } else if ((rtype == KVM_REG_RISCV_FP_D) && - riscv_isa_extension_available(vcpu->arch.isa, d)) { + riscv_isa_extension_available(vcpu->arch.isa, D)) { if (reg_num == KVM_REG_RISCV_FP_D_REG(fcsr)) { if (KVM_REG_SIZE(reg->id) != sizeof(u32)) return -EINVAL; @@ -138,7 +138,7 @@ int kvm_riscv_vcpu_set_reg_fp(struct kvm_vcpu *vcpu, void *reg_val; if ((rtype == KVM_REG_RISCV_FP_F) && - riscv_isa_extension_available(vcpu->arch.isa, f)) { + riscv_isa_extension_available(vcpu->arch.isa, F)) { if (KVM_REG_SIZE(reg->id) != sizeof(u32)) return -EINVAL; if (reg_num == KVM_REG_RISCV_FP_F_REG(fcsr)) @@ -151,7 +151,7 @@ int kvm_riscv_vcpu_set_reg_fp(struct kvm_vcpu *vcpu, } else return -ENOENT; } else if ((rtype == KVM_REG_RISCV_FP_D) && - riscv_isa_extension_available(vcpu->arch.isa, d)) { + riscv_isa_extension_available(vcpu->arch.isa, D)) { if (reg_num == KVM_REG_RISCV_FP_D_REG(fcsr)) { if (KVM_REG_SIZE(reg->id) != sizeof(u32)) return -EINVAL; diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c index 99b9107b1ac1..afcfdc79a145 100644 --- a/arch/riscv/kvm/vcpu_onereg.c +++ b/arch/riscv/kvm/vcpu_onereg.c @@ -748,7 +748,7 @@ static inline unsigned long num_fp_f_regs(const struct kvm_vcpu *vcpu) { const struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; - if (riscv_isa_extension_available(vcpu->arch.isa, f)) + if (riscv_isa_extension_available(vcpu->arch.isa, F)) return sizeof(cntx->fp.f) / sizeof(u32); else return 0; @@ -777,7 +777,7 @@ static inline unsigned long num_fp_d_regs(const struct kvm_vcpu *vcpu) { const struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; - if (riscv_isa_extension_available(vcpu->arch.isa, d)) + if (riscv_isa_extension_available(vcpu->arch.isa, D)) return sizeof(cntx->fp.d.f) / sizeof(u64) + 1; else return 0; @@ -856,7 +856,7 @@ static inline unsigned long num_sbi_regs(struct kvm_vcpu *vcpu) static inline unsigned long num_vector_regs(const struct kvm_vcpu *vcpu) { - if (!riscv_isa_extension_available(vcpu->arch.isa, v)) + if (!riscv_isa_extension_available(vcpu->arch.isa, V)) return 0; /* vstart, vl, vtype, vcsr, vlenb and 32 vector regs */ diff --git a/arch/riscv/kvm/vcpu_vector.c b/arch/riscv/kvm/vcpu_vector.c index 3708616e2c32..3c3183b39ae2 100644 --- a/arch/riscv/kvm/vcpu_vector.c +++ b/arch/riscv/kvm/vcpu_vector.c @@ -27,7 +27,7 @@ void kvm_riscv_vcpu_vector_reset(struct kvm_vcpu *vcpu) cntx->vector.vlenb = riscv_v_vsize / 32; - if (riscv_isa_extension_available(isa, v)) { + if (riscv_isa_extension_available(isa, V)) { cntx->sstatus |= SR_VS_INITIAL; WARN_ON(!cntx->vector.datap); memset(cntx->vector.datap, 0, riscv_v_vsize); @@ -46,7 +46,7 @@ void kvm_riscv_vcpu_guest_vector_save(struct kvm_cpu_context *cntx, unsigned long *isa) { if ((cntx->sstatus & SR_VS) == SR_VS_DIRTY) { - if (riscv_isa_extension_available(isa, v)) + if (riscv_isa_extension_available(isa, V)) __kvm_riscv_vector_save(cntx); kvm_riscv_vcpu_vector_clean(cntx); } @@ -56,7 +56,7 @@ void kvm_riscv_vcpu_guest_vector_restore(struct kvm_cpu_context *cntx, unsigned long *isa) { if ((cntx->sstatus & SR_VS) != SR_VS_OFF) { - if (riscv_isa_extension_available(isa, v)) + if (riscv_isa_extension_available(isa, V)) __kvm_riscv_vector_restore(cntx); kvm_riscv_vcpu_vector_clean(cntx); } @@ -164,7 +164,7 @@ int kvm_riscv_vcpu_get_reg_vector(struct kvm_vcpu *vcpu, void *reg_addr; int rc; - if (!riscv_isa_extension_available(isa, v)) + if (!riscv_isa_extension_available(isa, V)) return -ENOENT; rc = kvm_riscv_vcpu_vreg_addr(vcpu, reg_num, reg_size, ®_addr); @@ -190,7 +190,7 @@ int kvm_riscv_vcpu_set_reg_vector(struct kvm_vcpu *vcpu, void *reg_addr; int rc; - if (!riscv_isa_extension_available(isa, v)) + if (!riscv_isa_extension_available(isa, V)) return -ENOENT; if (reg_num == KVM_REG_RISCV_VECTOR_CSR_REG(vlenb)) { diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index a46558b457e8..f8994caefc70 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -305,8 +305,6 @@ static void __init setup_bootmem(void) */ if (!IS_ENABLED(CONFIG_BUILTIN_DTB)) memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va)); - - dma_contiguous_reserve(dma32_phys_limit); } #ifdef CONFIG_RELOCATABLE @@ -1359,6 +1357,7 @@ void __init misc_mem_init(void) { early_memtest(min_low_pfn << PAGE_SHIFT, max_low_pfn << PAGE_SHIFT); arch_numa_init(); + dma_contiguous_reserve(dma32_phys_limit); #ifdef CONFIG_SPARSEMEM_VMEMMAP /* The entire VMEMMAP region has been populated. Flush TLB for this region */ local_flush_tlb_kernel_range(VMEMMAP_START, VMEMMAP_END); diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c index 8404530ec00f..962db300a166 100644 --- a/arch/riscv/mm/tlbflush.c +++ b/arch/riscv/mm/tlbflush.c @@ -11,24 +11,6 @@ #define has_svinval() riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL) -static inline void local_sfence_inval_ir(void) -{ - asm volatile(SFENCE_INVAL_IR() ::: "memory"); -} - -static inline void local_sfence_w_inval(void) -{ - asm volatile(SFENCE_W_INVAL() ::: "memory"); -} - -static inline void local_sinval_vma(unsigned long vma, unsigned long asid) -{ - if (asid != FLUSH_TLB_NO_ASID) - asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory"); - else - asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory"); -} - /* * Flush entire TLB if number of entries to be flushed is greater * than the threshold below. @@ -110,11 +92,6 @@ static void __ipi_flush_tlb_range_asid(void *info) local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid); } -static inline unsigned long get_mm_asid(struct mm_struct *mm) -{ - return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID; -} - static void __flush_tlb_range(struct mm_struct *mm, const struct cpumask *cmask, unsigned long start, unsigned long size, diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c index 4286e4af1092..eb93c060426f 100644 --- a/drivers/acpi/tables.c +++ b/drivers/acpi/tables.c @@ -22,6 +22,7 @@ #include #include #include +#include #include "internal.h" #ifdef CONFIG_ACPI_CUSTOM_DSDT diff --git a/drivers/perf/riscv_pmu_sbi.c b/drivers/perf/riscv_pmu_sbi.c index dfc886dee5ad..50220f7b46d9 100644 --- a/drivers/perf/riscv_pmu_sbi.c +++ b/drivers/perf/riscv_pmu_sbi.c @@ -1219,22 +1219,29 @@ static int pmu_sbi_setup_irqs(struct riscv_pmu *pmu, struct platform_device *pde DOMAIN_BUS_ANY); if (!domain) { pr_err("Failed to find INTC IRQ root domain\n"); - return -ENODEV; + ret = -ENODEV; + goto err; } riscv_pmu_irq = irq_create_mapping(domain, riscv_pmu_irq_num); if (!riscv_pmu_irq) { pr_err("Failed to map PMU interrupt for node\n"); - return -ENODEV; + ret = -ENODEV; + goto err; } ret = request_percpu_irq(riscv_pmu_irq, pmu_sbi_ovf_handler, "riscv-pmu", hw_events); if (ret) { pr_err("registering percpu irq failed [%d]\n", ret); - return ret; + irq_dispose_mapping(riscv_pmu_irq); + riscv_pmu_irq = 0; + goto err; } return 0; +err: + riscv_pmu_use_irq = false; + return ret; } #ifdef CONFIG_CPU_PM @@ -1301,7 +1308,8 @@ static void riscv_pmu_destroy(struct riscv_pmu *pmu) } } riscv_pm_pmu_unregister(pmu); - cpuhp_state_remove_instance(CPUHP_AP_PERF_RISCV_STARTING, &pmu->node); + if (!hlist_unhashed(&pmu->node)) + cpuhp_state_remove_instance(CPUHP_AP_PERF_RISCV_STARTING, &pmu->node); } static void pmu_sbi_event_init(struct perf_event *event) @@ -1423,6 +1431,7 @@ static int pmu_sbi_device_probe(struct platform_device *pdev) struct riscv_pmu *pmu = NULL; int ret = -ENODEV; int num_counters; + bool irq_requested = false; pr_info("SBI PMU extension is available\n"); pmu = riscv_pmu_alloc(); @@ -1451,6 +1460,7 @@ static int pmu_sbi_device_probe(struct platform_device *pdev) pmu->pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT; pmu->pmu.capabilities |= PERF_PMU_CAP_NO_EXCLUDE; } + irq_requested = (ret == 0); pmu->pmu.attr_groups = riscv_pmu_attr_groups; pmu->pmu.parent = &pdev->dev; @@ -1469,11 +1479,11 @@ static int pmu_sbi_device_probe(struct platform_device *pdev) ret = riscv_pm_pmu_register(pmu); if (ret) - goto out_unregister; + goto out_destroy; ret = perf_pmu_register(&pmu->pmu, "cpu", PERF_TYPE_RAW); if (ret) - goto out_unregister; + goto out_destroy; /* SBI PMU Snapsphot is only available in SBI v2.0 */ if (sbi_v2_available) { @@ -1514,9 +1524,20 @@ static int pmu_sbi_device_probe(struct platform_device *pdev) return 0; out_unregister: + perf_pmu_unregister(&pmu->pmu); + +out_destroy: riscv_pmu_destroy(pmu); + if (irq_requested) { + free_percpu_irq(riscv_pmu_irq, pmu->hw_events); + irq_dispose_mapping(riscv_pmu_irq); + riscv_pmu_irq = 0; + } out_free: + free_percpu(pmu->hw_events); + kfree(pmu_ctr_list); + pmu_ctr_list = NULL; kfree(pmu); return ret; }