From dc0dddb1d66de88c571cf1a5bc3b484521a578af Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 14 Aug 2024 13:34:28 +0100 Subject: [PATCH 1/2] KVM: arm64: Invalidate EL1&0 TLB entries for all VMIDs in nvhe hyp init When initialising the nVHE hypervisor, we invalidate potentially stale TLB entries for the EL1&0 regime using a 'vmalls12e1' invalidation. However, this invalidation operation applies only to the active VMID and therefore we could proceed with stale TLB entries for other VMIDs. Replace the operation with an 'alle1' which applies to all entries for the EL1&0 regime, regardless of the VMID. Cc: Marc Zyngier Cc: Oliver Upton Fixes: 1025c8c0c6ac ("KVM: arm64: Wrap the host with a stage 2") Signed-off-by: Will Deacon Link: https://lore.kernel.org/r/20240814123429.20457-2-will@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/hyp-init.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index 07120b37da35..401af1835be6 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -130,7 +130,7 @@ alternative_else_nop_endif /* Invalidate the stale TLBs from Bootloader */ tlbi alle2 - tlbi vmalls12e1 + tlbi alle1 dsb sy mov_q x0, INIT_SCTLR_EL2_MMU_ON From ed49fe5a6fb9c1a1bbbf4b5b648c7d34a756cb6d Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 14 Aug 2024 13:34:29 +0100 Subject: [PATCH 2/2] KVM: arm64: Ensure TLBI uses correct VMID after changing context When the target context passed to enter_vmid_context() matches the current running context, the function returns early without manipulating the registers of the stage-2 MMU. This can result in a stale VMID due to the lack of an ISB instruction in exit_vmid_context() after writing the VTTBR when ARM64_WORKAROUND_SPECULATIVE_AT is not enabled. For example, with pKVM enabled: // Initially running in host context enter_vmid_context(guest); -> __load_stage2(guest); isb // Writes VTCR & VTTBR exit_vmid_context(guest); -> __load_stage2(host); // Restores VTCR & VTTBR enter_vmid_context(host); -> Returns early as we're already in host context tlbi vmalls12e1is // !!! Can use the stale VMID as we // haven't performed context // synchronisation since restoring // VTTBR.VMID Add an unconditional ISB instruction to exit_vmid_context() after restoring the VTTBR. This already existed for the ARM64_WORKAROUND_SPECULATIVE_AT path, so we can simply hoist that onto the common path. Cc: Marc Zyngier Cc: Oliver Upton Cc: Fuad Tabba Fixes: 58f3b0fc3b87 ("KVM: arm64: Support TLB invalidation in guest context") Signed-off-by: Will Deacon Link: https://lore.kernel.org/r/20240814123429.20457-3-will@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/tlb.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/tlb.c b/arch/arm64/kvm/hyp/nvhe/tlb.c index ca3c09df8d7c..48da9ca9763f 100644 --- a/arch/arm64/kvm/hyp/nvhe/tlb.c +++ b/arch/arm64/kvm/hyp/nvhe/tlb.c @@ -132,10 +132,10 @@ static void exit_vmid_context(struct tlb_inv_context *cxt) else __load_host_stage2(); - if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) { - /* Ensure write of the old VMID */ - isb(); + /* Ensure write of the old VMID */ + isb(); + if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) { if (!(cxt->sctlr & SCTLR_ELx_M)) { write_sysreg_el1(cxt->sctlr, SYS_SCTLR); isb();