From b7f741717d1e7bcdca2d5eaf3babbb5026cdb3a4 Mon Sep 17 00:00:00 2001 From: Vladimir Murzin Date: Mon, 27 Jul 2026 17:34:09 +0100 Subject: [PATCH 1/9] arm64: ptrace: Remove INIT_PSTATE_EL2 Last user of INIT_PSTATE_EL2 has gone with ae4b7e38e9a9 ("arm64: Allow sticky E2H when entering EL1"), so remove it. Signed-off-by: Vladimir Murzin Reviewed-by: Jinjie Ruan Signed-off-by: Will Deacon --- arch/arm64/include/asm/ptrace.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h index 39582511ad72..f7dc5fb9427d 100644 --- a/arch/arm64/include/asm/ptrace.h +++ b/arch/arm64/include/asm/ptrace.h @@ -18,8 +18,6 @@ #define INIT_PSTATE_EL1 \ (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | PSR_MODE_EL1h) -#define INIT_PSTATE_EL2 \ - (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | PSR_MODE_EL2h) #include From 7cf2d6efb86fc98c4f5f78e438e2e32d88d76ff7 Mon Sep 17 00:00:00 2001 From: Ada Couprie Diaz Date: Mon, 27 Jul 2026 17:34:10 +0100 Subject: [PATCH 2/9] arm64: debug: don't mask DAIF for mdscr_write() Masking DAIF around the write to MDSCR_EL1 doesn't do anything: we can write to sysregs with interrupts unmasked, and writing to PSTATE is not a context synchronization event so it does not synchronize it. This is done in the context of a general interrupt handling cleanup, so it does not address the missing context synchronization for the MDSCR_EL1 write, staying consistent with the current state. This should be addressed in a future patch. Signed-off-by: Ada Couprie Diaz Signed-off-by: Vladimir Murzin Signed-off-by: Will Deacon --- arch/arm64/kernel/debug-monitors.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c index 29307642f4c9..e271fbac5f82 100644 --- a/arch/arm64/kernel/debug-monitors.c +++ b/arch/arm64/kernel/debug-monitors.c @@ -40,10 +40,7 @@ u8 debug_monitors_arch(void) */ static void mdscr_write(u64 mdscr) { - unsigned long flags; - flags = local_daif_save(); write_sysreg(mdscr, mdscr_el1); - local_daif_restore(flags); } NOKPROBE_SYMBOL(mdscr_write); From 684bde100117931f4c51c644a95f42f2dab041bc Mon Sep 17 00:00:00 2001 From: Ada Couprie Diaz Date: Mon, 27 Jul 2026 17:34:11 +0100 Subject: [PATCH 3/9] arm64: hibernate: mask DAIF before restoring hibernated kernel The arm64 hibernate code manages the exception masking in an unsound way, leading to potential crashes and/or warnings during resume. When a hibernation image is saved in `swsusp_arch_suspend()`, all DAIF exceptions are masked (by virtue of `local_daif_save()`), and the suspended image is saved assuming that all DAIF exceptions will remain masked when the image is restored. When a hibernation image is resumed by `swsusp_arch_resume()`, only interrupts are masked (by virtue of `local_irq_disable()` in `resume_target_kernel()`). When pseudo-NMI is enabled the DAIF.IF bits will be clear, and regardless of pseudo-NMI the DAIF.DA bits will be clear. This means that there are two problems: (1) It is possible to take Debug, SError, or pseudo-NMI exceptions during the resume process. This is unsafe, as during the resume process both the old ane new kernels will tranisently be in an inconsistent state, and swsusp_arch_suspend_exit() won't retain an executable mapping of any exception vectors. Any exception taken here will be fatal and silent. (2) When re-entering the resumed kernel, some DAIF bits will be clear unexpectedly. This permits Debug, SError, or pseudo-NMI exceptions to be taken for a short period while the resumed kernel is not yet in a consistent state. This is detected by CONFIG_ARM64_DEBUG_PRIORITY_MASKING. Avoid these issues by masking all DAIF exceptions during resume. Fixes: 82869ac57b5d ("arm64: kernel: Add support for hibernate/suspend-to-disk") Signed-off-by: Ada Couprie Diaz Signed-off-by: Vladimir Murzin Reviewed-by: Jinjie Ruan Signed-off-by: Will Deacon --- arch/arm64/kernel/hibernate.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index 9717568518ba..1eb1c1074c5b 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -465,9 +465,21 @@ int __nocfi swsusp_arch_resume(void) if (el2_reset_needed()) __hyp_set_vectors(el2_vectors); + /* + * It is necessary to mask all DAIF exceptions here as: + * + * - The copy of swsusp_arch_suspend_exit() in the hibernation + * text cannot handle taking any exceptions. + * + * - The suspended kernel masked all DAIF exceptions in + * swsusp_arch_resume(), and expects to be re-entered in the + * same state : with all DAIF exceptions masked. + */ + local_daif_save(); hibernate_exit(virt_to_phys(tmp_pg_dir), resume_hdr.ttbr1_el1, resume_hdr.reenter_kernel, restore_pblist, resume_hdr.__hyp_stub_vectors, virt_to_phys(zero_page)); + unreachable(); return 0; } From 541549827889d0380fd73f8aacb5de6ef7a5a1ac Mon Sep 17 00:00:00 2001 From: Vladimir Murzin Date: Mon, 27 Jul 2026 17:34:12 +0100 Subject: [PATCH 4/9] arm64: hibernate: Restore DAIF state on error Sashiko AI has reported that if swsusp_mte_save_tags() for some reason fails we return from swsusp_arch_suspend() with DAIF being masked - that is not what we'd expect. Restore the saved DAIF state before returning from the error path. Fixes: ee11f332af96 ("arm64: mte: Save tags when hibernating") Signed-off-by: Vladimir Murzin Reviewed-by: Jinjie Ruan Signed-off-by: Will Deacon --- arch/arm64/kernel/hibernate.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index 1eb1c1074c5b..7bf117427777 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -348,8 +348,10 @@ int swsusp_arch_suspend(void) crash_prepare_suspend(); ret = swsusp_mte_save_tags(); - if (ret) + if (ret) { + local_daif_restore(flags); return ret; + } sleep_cpu = smp_processor_id(); ret = swsusp_save(); From 630ca6c58fcfeaddd96dd46e7b11826013977955 Mon Sep 17 00:00:00 2001 From: Ada Couprie Diaz Date: Mon, 27 Jul 2026 17:34:13 +0100 Subject: [PATCH 5/9] arm64: suspend: rely on daif helpers to handle PMR Commit 77345ef70445 ("arm64: suspend: Use cpuidle context helpers in cpu_suspend()") added cpuidle helpers to handle PMR manipulation and restoration to ensure that the CPU receives interrupts when suspended and pseudo-NMIs are enabled. However, those helpers are called in between a pair of `local_daif_save()` and `local_daif_restore()`, which already configure the PMR as expected. Effectively, `arm_cpuidle_save_irq_context()` is a no-op here, even when using pseudo-NMIs, and `arm_cpuidle_restore_irq_context()` would not restore proper interrupt masking configuration early enough if there were unexpected changes during suspend or resume. (This can be observed with Trusted Firmware A (TF-A) at EL3 handling suspend through PSCI. Even though it should not be the case, TF-A can reset `ICC_PMR_EL1` during CPU_SUSPEND, thus resuming the kernel with an inconsistent priority mask value on hardware implementing more than the minimum number of priority levels, such as Morello.) Thus : remove the cpuidle context helpers as they do not do anything, but keep the comment mentioning the need for interrupts to reach the CPU if we are using pseudo-NMIs. Signed-off-by: Ada Couprie Diaz Signed-off-by: Vladimir Murzin Reviewed-by: Jinjie Ruan Signed-off-by: Will Deacon --- arch/arm64/kernel/suspend.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index eaaff94329cd..c41724a40b75 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -99,7 +99,6 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) int ret = 0; unsigned long flags; struct sleep_stack_data state; - struct arm_cpuidle_irq_context context; /* * Some portions of CPU state (e.g. PSTATE.{PAN,DIT}) are initialized @@ -121,6 +120,9 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) * Strictly speaking the trace_hardirqs_off() here is superfluous, * hardirqs should be firmly off by now. This really ought to use * something like raw_local_daif_save(). + * + * This also unmasks interrupts in PMR in order to reliably + * resume if we're using pseudo-NMIs. */ flags = local_daif_save(); @@ -131,12 +133,6 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) */ pause_graph_tracing(); - /* - * Switch to using DAIF.IF instead of PMR in order to reliably - * resume if we're using pseudo-NMIs. - */ - arm_cpuidle_save_irq_context(&context); - ct_cpuidle_enter(); if (__cpu_suspend_enter(&state)) { @@ -159,8 +155,6 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) __cpu_suspend_exit(); } - arm_cpuidle_restore_irq_context(&context); - unpause_graph_tracing(); /* From d3ebdc1772c0714faca14eec789ff082f6e00535 Mon Sep 17 00:00:00 2001 From: Vladimir Murzin Date: Mon, 27 Jul 2026 17:34:14 +0100 Subject: [PATCH 6/9] arm64: suspend: Initialize PMR on resume When we resume from cpu_suspend() context tracking, specially, ct_idle_exit() performs IRQ save/restore sequence. It doesn't cause any functional issues since we have masked all exceptions prior suspend and have not restored them. However, in case of pseudo-NMI PMR can be set by firmware to arbitrary value, thus IRQ save/restore routines manipulates this arbitrary value. Again, it doesn't cause any issues since PMR variant of IRQ save helper carries a __pmr_irqs_disabled_flags() guard. Going forward __pmr_irqs_disabled_flags() guard will be gone and we will call __pmr_local_irq_disable() unconditionally - that would cause warning in case CONFIG_ARM64_DEBUG_PRIORITY_MASKING is set. Initialize PMR to a value known to Linux on resume until the normal exception restore path restores the saved DAIF and PMR state. Signed-off-by: Vladimir Murzin Signed-off-by: Will Deacon --- arch/arm64/mm/proc.S | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index 22866b49be37..06c8bc9a85d6 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -169,6 +169,13 @@ alternative_if ARM64_HAS_RAS_EXTN msr_s SYS_DISR_EL1, xzr alternative_else_nop_endif +#ifdef CONFIG_ARM64_PSEUDO_NMI +alternative_if ARM64_HAS_GIC_PRIO_MASKING + mov x1, #GIC_PRIO_IRQON + msr_s SYS_ICC_PMR_EL1, x1 +alternative_else_nop_endif +#endif + ptrauth_keys_install_kernel_nosync x14, x1, x2, x3 isb ret From 0d774e0517f30b9684e936c2559fb50681a45329 Mon Sep 17 00:00:00 2001 From: Ada Couprie Diaz Date: Mon, 27 Jul 2026 17:34:15 +0100 Subject: [PATCH 7/9] arm64: entry: mask DAIF before returning from C EL1 handlers Most EL1 exceptions already call local_daif_mask() before returning, with the exception of debug exception handlers which do not change DAIF, and the IRQ/FIQ/Error handlers. However, DAIF get masked in kernel_exit() in all cases when returning from EL1 C handlers anyway. Move this masking from assembly to C by calling local_daif_mask() before irqentry_nmi_exit(). Unlike the raw DAIF masking helper, local_daif_mask() invokes trace_hardirqs_off(), so it must execute while RCU is still watching. Remove the disable_daif assembly macro, as this was its only use. Signed-off-by: Ada Couprie Diaz Signed-off-by: Vladimir Murzin Reviewed-by: Jinjie Ruan Signed-off-by: Will Deacon --- arch/arm64/include/asm/assembler.h | 4 ---- arch/arm64/kernel/entry-common.c | 2 ++ arch/arm64/kernel/entry.S | 4 ---- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index effae53e9739..0b58b550e8dc 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -34,10 +34,6 @@ wx\n .req w\n .endr - .macro disable_daif - msr daifset, #0xf - .endm - /* * Save/restore interrupts. */ diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index ceb4eb11232a..2be42d7f4eaa 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -495,6 +495,7 @@ static __always_inline void __el1_pnmi(struct pt_regs *regs, state = irqentry_nmi_enter(regs); do_interrupt_handler(regs, handler); + local_daif_mask(); irqentry_nmi_exit(regs, state); } @@ -540,6 +541,7 @@ asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs) local_daif_restore(DAIF_ERRCTX); state = irqentry_nmi_enter(regs); do_serror(regs, esr); + local_daif_mask(); irqentry_nmi_exit(regs, state); } diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index e0db14e9c843..f63049ac32dc 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -333,10 +333,6 @@ alternative_else_nop_endif .endm .macro kernel_exit, el - .if \el != 0 - disable_daif - .endif - #ifdef CONFIG_ARM64_PSEUDO_NMI alternative_if_not ARM64_HAS_GIC_PRIO_MASKING b .Lskip_pmr_restore\@ From 067f029c6463ab3bc980053bd0d50f98d7edfa3e Mon Sep 17 00:00:00 2001 From: Ada Couprie Diaz Date: Mon, 27 Jul 2026 17:34:16 +0100 Subject: [PATCH 8/9] irqchip/gic-v3: make the unmasking of pseudo-NMIs explicit when handling IRQs `gic_arch_enable_irqs()` is only used when handling IRQs (which could be pseudo-NMIs) and unmasking pseudo-NMIs. The chain of `gic_pmr_mask_irqs()` and `gic_arch_enable_irqs()` for it is slightly confusing without further explanation. Remove `gic_arch_enable_irqs()` and instead do the whole pseudo-NMI umasking in `gic_unmask_pnmis()`, making the operation explicit. Signed-off-by: Ada Couprie Diaz Signed-off-by: Vladimir Murzin Reviewed-by: Jinjie Ruan Reviewed-by: Marc Zyngier Signed-off-by: Will Deacon --- arch/arm/include/asm/arch_gicv3.h | 6 +----- arch/arm64/include/asm/arch_gicv3.h | 7 +++++-- arch/arm64/include/asm/entry-common.h | 2 +- drivers/irqchip/irq-gic-v3.c | 5 +---- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/arch/arm/include/asm/arch_gicv3.h b/arch/arm/include/asm/arch_gicv3.h index 847590df7551..d4ac8d3271b1 100644 --- a/arch/arm/include/asm/arch_gicv3.h +++ b/arch/arm/include/asm/arch_gicv3.h @@ -246,11 +246,7 @@ static inline void gic_pmr_mask_irqs(void) WARN_ON_ONCE(true); } -static inline void gic_arch_enable_irqs(void) -{ - /* Should not get called. */ - WARN_ON_ONCE(true); -} +static inline void gic_unmask_pnmis(void) {} static inline bool gic_has_relaxed_pmr_sync(void) { diff --git a/arch/arm64/include/asm/arch_gicv3.h b/arch/arm64/include/asm/arch_gicv3.h index d20b03931a8d..3dcb7b8309d9 100644 --- a/arch/arm64/include/asm/arch_gicv3.h +++ b/arch/arm64/include/asm/arch_gicv3.h @@ -178,9 +178,12 @@ static inline void gic_pmr_mask_irqs(void) gic_write_pmr(GIC_PRIO_IRQOFF); } -static inline void gic_arch_enable_irqs(void) +static inline void gic_unmask_pnmis(void) { - asm volatile ("msr daifclr, #3" : : : "memory"); + if (gic_prio_masking_enabled()) { + gic_pmr_mask_irqs(); + asm volatile ("msr daifclr, #3" : : : "memory"); + } } static inline bool gic_has_relaxed_pmr_sync(void) diff --git a/arch/arm64/include/asm/entry-common.h b/arch/arm64/include/asm/entry-common.h index cab8cd78f693..1905765159aa 100644 --- a/arch/arm64/include/asm/entry-common.h +++ b/arch/arm64/include/asm/entry-common.h @@ -32,7 +32,7 @@ static inline bool arch_irqentry_exit_need_resched(void) /* * DAIF.DA are cleared at the start of IRQ/FIQ handling, and when GIC * priority masking is used the GIC irqchip driver will clear DAIF.IF - * using gic_arch_enable_irqs() for normal IRQs. If anything is set in + * in gic_unmask_pnmis() for normal IRQs. If anything is set in * DAIF we must have handled an NMI, so skip preemption. */ if (system_uses_irq_prio_masking() && read_sysreg(daif)) diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 99444a1b2ffa..94c6a3f2b009 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -867,10 +867,7 @@ static void __gic_handle_irq_from_irqson(struct pt_regs *regs) nmi_exit(); } - if (gic_prio_masking_enabled()) { - gic_pmr_mask_irqs(); - gic_arch_enable_irqs(); - } + gic_unmask_pnmis(); if (!is_nmi) __gic_handle_irq(irqnr, regs); From 39aebe0e89469c2904e60b1e977e0d4dbf33326b Mon Sep 17 00:00:00 2001 From: Vladimir Murzin Date: Mon, 27 Jul 2026 17:34:17 +0100 Subject: [PATCH 9/9] arm64: entry: Avoid unnecessary local_irq_disable() on kernel exit Currently, when exiting to kernel mode, we attempt involuntary preemption. The preemption logic expects IRQs to be disabled, which is why we call local_irq_disable() before attempting preemption. However, depending on the context, local_irq_disable() may be unnecessary: - __el1_irq(), the non-NMI EL1 IRQ path, already has IRQs disabled, so local_irq_disable() is redundant. - irqentry_exit_to_kernel_mode_preempt() immediately returns when exiting from an NMI-like context, so calling local_irq_disable() beforehand is unnecessary work. Furthermore, it confuses the pNMI state tracking when we are in a context with interrupts disabled and the GIC_PRIO_PSR_I_SET bit is set in the PMR, leading to a warning when CONFIG_ARM64_DEBUG_PRIORITY_MASKING=y: WARNING: ./arch/arm64/include/asm/irqflags.h:63 at arm64_exit_to_kernel_mode+0xb8/0xc0, CPU#40: retsnoop/31805 CPU: 40 UID: 0 PID: 31805 Comm: retsnoop Not tainted 7.2.0-rc6-next-20260805 #7 PREEMPTLAZY pstate: 234013c9 (nzCv DAIF +PAN -UAO +TCO +DIT +SSBS BTYPE=--) pc : arm64_exit_to_kernel_mode (arch/arm64/kernel/entry-common.c:63) lr : el1_abort (arch/arm64/kernel/entry-common.c:323) pmr: 000000f0 Call trace: arm64_exit_to_kernel_mode (arch/arm64/kernel/entry-common.c:63) (P) el1_abort (arch/arm64/kernel/entry-common.c:323) el1h_64_sync_handler (arch/arm64/kernel/entry-common.c:449) el1h_64_sync (arch/arm64/kernel/entry.S:589) [...] Split arm64_exit_to_kernel_mode() into preempt, non-preempt, and dispatch parts so that we can avoid this extra work where it is not needed and avoid breaking the pNMI tracking logic. Reported-by: Breno Leitao Fixes: ae654112eac0 ("arm64: entry: Use split preemption logic") Link: https://lore.kernel.org/all/20260807-arm64_fix-v1-1-d069ccf9d71b@debian.org/ Reviewed-by: Jinjie Ruan Signed-off-by: Vladimir Murzin Signed-off-by: Will Deacon --- arch/arm64/kernel/entry-common.c | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 2be42d7f4eaa..72c03ccea59f 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -52,16 +52,36 @@ static noinstr irqentry_state_t arm64_enter_from_kernel_mode(struct pt_regs *reg * After this function returns it is not safe to call regular kernel code, * instrumentable code, or any code which may trigger an exception. */ -static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs, - irqentry_state_t state) +static void noinstr __arm64_exit_to_kernel_mode(struct pt_regs *regs, + irqentry_state_t state) { - local_irq_disable(); - irqentry_exit_to_kernel_mode_preempt(regs, state); local_daif_mask(); mte_check_tfsr_exit(); irqentry_exit_to_kernel_mode_after_preempt(regs, state); } +/* + * We are returning from the context which allows involuntary kernel preemption + */ +static void noinstr arm64_exit_to_kernel_mode_preempt(struct pt_regs *regs, + irqentry_state_t state) +{ + irqentry_exit_to_kernel_mode_preempt(regs, state); + __arm64_exit_to_kernel_mode(regs, state); +} + +static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs, + irqentry_state_t state) +{ + if (!regs_irqs_disabled(regs)) { + local_irq_disable(); + arm64_exit_to_kernel_mode_preempt(regs, state); + return; + } + + __arm64_exit_to_kernel_mode(regs, state); +} + static __always_inline void arm64_syscall_enter_from_user_mode(struct pt_regs *regs) { enter_from_user_mode(regs); @@ -510,7 +530,7 @@ static __always_inline void __el1_irq(struct pt_regs *regs, do_interrupt_handler(regs, handler); irq_exit_rcu(); - arm64_exit_to_kernel_mode(regs, state); + arm64_exit_to_kernel_mode_preempt(regs, state); } static void noinstr el1_interrupt(struct pt_regs *regs, void (*handler)(struct pt_regs *))