diff --git a/arch/arm/include/asm/arch_gicv3.h b/arch/arm/include/asm/arch_gicv3.h index 847590df7551..d4ac8d3271b1 100644 --- a/arch/arm/include/asm/arch_gicv3.h +++ b/arch/arm/include/asm/arch_gicv3.h @@ -246,11 +246,7 @@ static inline void gic_pmr_mask_irqs(void) WARN_ON_ONCE(true); } -static inline void gic_arch_enable_irqs(void) -{ - /* Should not get called. */ - WARN_ON_ONCE(true); -} +static inline void gic_unmask_pnmis(void) {} static inline bool gic_has_relaxed_pmr_sync(void) { diff --git a/arch/arm64/include/asm/arch_gicv3.h b/arch/arm64/include/asm/arch_gicv3.h index d20b03931a8d..3dcb7b8309d9 100644 --- a/arch/arm64/include/asm/arch_gicv3.h +++ b/arch/arm64/include/asm/arch_gicv3.h @@ -178,9 +178,12 @@ static inline void gic_pmr_mask_irqs(void) gic_write_pmr(GIC_PRIO_IRQOFF); } -static inline void gic_arch_enable_irqs(void) +static inline void gic_unmask_pnmis(void) { - asm volatile ("msr daifclr, #3" : : : "memory"); + if (gic_prio_masking_enabled()) { + gic_pmr_mask_irqs(); + asm volatile ("msr daifclr, #3" : : : "memory"); + } } static inline bool gic_has_relaxed_pmr_sync(void) diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index effae53e9739..0b58b550e8dc 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -34,10 +34,6 @@ wx\n .req w\n .endr - .macro disable_daif - msr daifset, #0xf - .endm - /* * Save/restore interrupts. */ diff --git a/arch/arm64/include/asm/entry-common.h b/arch/arm64/include/asm/entry-common.h index cab8cd78f693..1905765159aa 100644 --- a/arch/arm64/include/asm/entry-common.h +++ b/arch/arm64/include/asm/entry-common.h @@ -32,7 +32,7 @@ static inline bool arch_irqentry_exit_need_resched(void) /* * DAIF.DA are cleared at the start of IRQ/FIQ handling, and when GIC * priority masking is used the GIC irqchip driver will clear DAIF.IF - * using gic_arch_enable_irqs() for normal IRQs. If anything is set in + * in gic_unmask_pnmis() for normal IRQs. If anything is set in * DAIF we must have handled an NMI, so skip preemption. */ if (system_uses_irq_prio_masking() && read_sysreg(daif)) diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h index 39582511ad72..f7dc5fb9427d 100644 --- a/arch/arm64/include/asm/ptrace.h +++ b/arch/arm64/include/asm/ptrace.h @@ -18,8 +18,6 @@ #define INIT_PSTATE_EL1 \ (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | PSR_MODE_EL1h) -#define INIT_PSTATE_EL2 \ - (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | PSR_MODE_EL2h) #include diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c index 29307642f4c9..e271fbac5f82 100644 --- a/arch/arm64/kernel/debug-monitors.c +++ b/arch/arm64/kernel/debug-monitors.c @@ -40,10 +40,7 @@ u8 debug_monitors_arch(void) */ static void mdscr_write(u64 mdscr) { - unsigned long flags; - flags = local_daif_save(); write_sysreg(mdscr, mdscr_el1); - local_daif_restore(flags); } NOKPROBE_SYMBOL(mdscr_write); diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index ceb4eb11232a..72c03ccea59f 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -52,16 +52,36 @@ static noinstr irqentry_state_t arm64_enter_from_kernel_mode(struct pt_regs *reg * After this function returns it is not safe to call regular kernel code, * instrumentable code, or any code which may trigger an exception. */ -static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs, - irqentry_state_t state) +static void noinstr __arm64_exit_to_kernel_mode(struct pt_regs *regs, + irqentry_state_t state) { - local_irq_disable(); - irqentry_exit_to_kernel_mode_preempt(regs, state); local_daif_mask(); mte_check_tfsr_exit(); irqentry_exit_to_kernel_mode_after_preempt(regs, state); } +/* + * We are returning from the context which allows involuntary kernel preemption + */ +static void noinstr arm64_exit_to_kernel_mode_preempt(struct pt_regs *regs, + irqentry_state_t state) +{ + irqentry_exit_to_kernel_mode_preempt(regs, state); + __arm64_exit_to_kernel_mode(regs, state); +} + +static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs, + irqentry_state_t state) +{ + if (!regs_irqs_disabled(regs)) { + local_irq_disable(); + arm64_exit_to_kernel_mode_preempt(regs, state); + return; + } + + __arm64_exit_to_kernel_mode(regs, state); +} + static __always_inline void arm64_syscall_enter_from_user_mode(struct pt_regs *regs) { enter_from_user_mode(regs); @@ -495,6 +515,7 @@ static __always_inline void __el1_pnmi(struct pt_regs *regs, state = irqentry_nmi_enter(regs); do_interrupt_handler(regs, handler); + local_daif_mask(); irqentry_nmi_exit(regs, state); } @@ -509,7 +530,7 @@ static __always_inline void __el1_irq(struct pt_regs *regs, do_interrupt_handler(regs, handler); irq_exit_rcu(); - arm64_exit_to_kernel_mode(regs, state); + arm64_exit_to_kernel_mode_preempt(regs, state); } static void noinstr el1_interrupt(struct pt_regs *regs, void (*handler)(struct pt_regs *)) @@ -540,6 +561,7 @@ asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs) local_daif_restore(DAIF_ERRCTX); state = irqentry_nmi_enter(regs); do_serror(regs, esr); + local_daif_mask(); irqentry_nmi_exit(regs, state); } diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index e0db14e9c843..f63049ac32dc 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -333,10 +333,6 @@ alternative_else_nop_endif .endm .macro kernel_exit, el - .if \el != 0 - disable_daif - .endif - #ifdef CONFIG_ARM64_PSEUDO_NMI alternative_if_not ARM64_HAS_GIC_PRIO_MASKING b .Lskip_pmr_restore\@ diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index 9717568518ba..7bf117427777 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -348,8 +348,10 @@ int swsusp_arch_suspend(void) crash_prepare_suspend(); ret = swsusp_mte_save_tags(); - if (ret) + if (ret) { + local_daif_restore(flags); return ret; + } sleep_cpu = smp_processor_id(); ret = swsusp_save(); @@ -465,9 +467,21 @@ int __nocfi swsusp_arch_resume(void) if (el2_reset_needed()) __hyp_set_vectors(el2_vectors); + /* + * It is necessary to mask all DAIF exceptions here as: + * + * - The copy of swsusp_arch_suspend_exit() in the hibernation + * text cannot handle taking any exceptions. + * + * - The suspended kernel masked all DAIF exceptions in + * swsusp_arch_resume(), and expects to be re-entered in the + * same state : with all DAIF exceptions masked. + */ + local_daif_save(); hibernate_exit(virt_to_phys(tmp_pg_dir), resume_hdr.ttbr1_el1, resume_hdr.reenter_kernel, restore_pblist, resume_hdr.__hyp_stub_vectors, virt_to_phys(zero_page)); + unreachable(); return 0; } diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index eaaff94329cd..c41724a40b75 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -99,7 +99,6 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) int ret = 0; unsigned long flags; struct sleep_stack_data state; - struct arm_cpuidle_irq_context context; /* * Some portions of CPU state (e.g. PSTATE.{PAN,DIT}) are initialized @@ -121,6 +120,9 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) * Strictly speaking the trace_hardirqs_off() here is superfluous, * hardirqs should be firmly off by now. This really ought to use * something like raw_local_daif_save(). + * + * This also unmasks interrupts in PMR in order to reliably + * resume if we're using pseudo-NMIs. */ flags = local_daif_save(); @@ -131,12 +133,6 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) */ pause_graph_tracing(); - /* - * Switch to using DAIF.IF instead of PMR in order to reliably - * resume if we're using pseudo-NMIs. - */ - arm_cpuidle_save_irq_context(&context); - ct_cpuidle_enter(); if (__cpu_suspend_enter(&state)) { @@ -159,8 +155,6 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) __cpu_suspend_exit(); } - arm_cpuidle_restore_irq_context(&context); - unpause_graph_tracing(); /* diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index f4e4e71a0ea8..12aacc74f764 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -169,6 +169,13 @@ alternative_if ARM64_HAS_RAS_EXTN msr_s SYS_DISR_EL1, xzr alternative_else_nop_endif +#ifdef CONFIG_ARM64_PSEUDO_NMI +alternative_if ARM64_HAS_GIC_PRIO_MASKING + mov x1, #GIC_PRIO_IRQON + msr_s SYS_ICC_PMR_EL1, x1 +alternative_else_nop_endif +#endif + ptrauth_keys_install_kernel_nosync x14, x1, x2, x3 isb ret diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 99444a1b2ffa..94c6a3f2b009 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -867,10 +867,7 @@ static void __gic_handle_irq_from_irqson(struct pt_regs *regs) nmi_exit(); } - if (gic_prio_masking_enabled()) { - gic_pmr_mask_irqs(); - gic_arch_enable_irqs(); - } + gic_unmask_pnmis(); if (!is_nmi) __gic_handle_irq(irqnr, regs);