diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index 60017ceec9d2..e2d5808169e4 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -209,13 +209,13 @@ struct kvm_vcpu_arch { /* * VCPU interrupts * - * We have a lockless approach for tracking pending VCPU interrupts - * implemented using atomic bitops. The irqs_pending bitmap represent - * pending interrupts whereas irqs_pending_mask represent bits changed - * in irqs_pending. Our approach is modeled around multiple producer - * and single consumer problem where the consumer is the VCPU itself. + * The irqs_pending bitmap represents pending interrupts whereas + * irqs_pending_mask represents bits changed in irqs_pending. Updates + * to these bitmaps are serialized so vcpu interrupt sync/flush cannot + * drop a newly injected interrupt while syncing guest-visible HVIP. */ #define KVM_RISCV_VCPU_NR_IRQS 64 + raw_spinlock_t irqs_pending_lock; DECLARE_BITMAP(irqs_pending, KVM_RISCV_VCPU_NR_IRQS); DECLARE_BITMAP(irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS); diff --git a/arch/riscv/kvm/aia.c b/arch/riscv/kvm/aia.c index bafb009c5ce5..9a653b4ad40a 100644 --- a/arch/riscv/kvm/aia.c +++ b/arch/riscv/kvm/aia.c @@ -53,12 +53,15 @@ void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu) struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; unsigned long mask, val; + lockdep_assert_held(&vcpu->arch.irqs_pending_lock); + if (!kvm_riscv_aia_available()) return; - if (READ_ONCE(vcpu->arch.irqs_pending_mask[1])) { - mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[1], 0); - val = READ_ONCE(vcpu->arch.irqs_pending[1]) & mask; + mask = vcpu->arch.irqs_pending_mask[1]; + if (mask) { + vcpu->arch.irqs_pending_mask[1] = 0; + val = vcpu->arch.irqs_pending[1] & mask; csr->hviph &= ~mask; csr->hviph |= val; @@ -69,6 +72,8 @@ void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu) { struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; + lockdep_assert_held(&vcpu->arch.irqs_pending_lock); + if (kvm_riscv_aia_available()) csr->vsieh = ncsr_read(CSR_VSIEH); } @@ -77,13 +82,22 @@ void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu) bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) { unsigned long seip; +#ifdef CONFIG_32BIT + unsigned long flags; + bool pending; +#endif if (!kvm_riscv_aia_available()) return false; #ifdef CONFIG_32BIT - if (READ_ONCE(vcpu->arch.irqs_pending[1]) & - (vcpu->arch.aia_context.guest_csr.vsieh & upper_32_bits(mask))) + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + pending = vcpu->arch.irqs_pending[1] & + (vcpu->arch.aia_context.guest_csr.vsieh & + upper_32_bits(mask)); + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); + + if (pending) return true; #endif @@ -207,6 +221,9 @@ int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu, { struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; unsigned long regs_max = sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); +#ifdef CONFIG_32BIT + unsigned long flags; +#endif if (!riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) return -ENOENT; @@ -219,8 +236,12 @@ int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu, ((unsigned long *)csr)[reg_num] = val; #ifdef CONFIG_32BIT - if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph)) - WRITE_ONCE(vcpu->arch.irqs_pending_mask[1], 0); + if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph)) { + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + vcpu->arch.irqs_pending_mask[1] = 0; + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, + flags); + } #endif } diff --git a/arch/riscv/kvm/gstage.c b/arch/riscv/kvm/gstage.c index c4c3b79567f1..b0474fcf065a 100644 --- a/arch/riscv/kvm/gstage.c +++ b/arch/riscv/kvm/gstage.c @@ -5,11 +5,13 @@ */ #include +#include #include #include #include #include #include +#include #ifdef CONFIG_64BIT unsigned long kvm_riscv_gstage_max_pgd_levels __ro_after_init = 3; @@ -171,8 +173,10 @@ int kvm_riscv_gstage_set_pte(struct kvm_gstage *gstage, } if (pte_val(*ptep) != pte_val(map->pte)) { + bool was_invalid = !pte_val(*ptep); set_pte(ptep, map->pte); - if (gstage_pte_leaf(ptep)) + if (gstage_pte_leaf(ptep) && + !(was_invalid && riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC))) gstage_tlb_flush(gstage, current_level, map->addr); } diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index 082f9b261733..8a0aa5e0e216 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -41,6 +41,7 @@ int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa, pgprot_t prot; unsigned long pfn; phys_addr_t addr, end; + unsigned long pgd_levels = kvm->arch.pgd_levels; struct kvm_mmu_memory_cache pcache = { .gfp_custom = (in_atomic) ? GFP_ATOMIC | __GFP_ACCOUNT : 0, .gfp_zero = __GFP_ZERO, @@ -63,7 +64,7 @@ int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa, if (!writable) map.pte = pte_wrprotect(map.pte); - ret = kvm_mmu_topup_memory_cache(&pcache, kvm->arch.pgd_levels); + ret = __kvm_mmu_topup_memory_cache(&pcache, pgd_levels, pgd_levels); if (ret) goto out; diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index cf6e231e76e2..977e36ab83d3 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -80,6 +80,7 @@ static void kvm_riscv_vcpu_context_reset(struct kvm_vcpu *vcpu, static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu, bool kvm_sbi_reset) { + unsigned long flags; bool loaded; /** @@ -104,8 +105,10 @@ static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu, bool kvm_sbi_reset) kvm_riscv_vcpu_aia_reset(vcpu); + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); bitmap_zero(vcpu->arch.irqs_pending, KVM_RISCV_VCPU_NR_IRQS); bitmap_zero(vcpu->arch.irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS); + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); kvm_riscv_vcpu_pmu_reset(vcpu); @@ -151,6 +154,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) /* Setup VCPU hfence queue */ spin_lock_init(&vcpu->arch.hfence_lock); + raw_spin_lock_init(&vcpu->arch.irqs_pending_lock); spin_lock_init(&vcpu->arch.reset_state.lock); @@ -352,10 +356,14 @@ void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu) { struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; unsigned long mask, val; + unsigned long flags; - if (READ_ONCE(vcpu->arch.irqs_pending_mask[0])) { - mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[0], 0); - val = READ_ONCE(vcpu->arch.irqs_pending[0]) & mask; + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + + mask = vcpu->arch.irqs_pending_mask[0]; + if (mask) { + vcpu->arch.irqs_pending_mask[0] = 0; + val = vcpu->arch.irqs_pending[0] & mask; csr->hvip &= ~mask; csr->hvip |= val; @@ -363,11 +371,14 @@ void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu) /* Flush AIA high interrupts */ kvm_riscv_vcpu_aia_flush_interrupts(vcpu); + + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); } void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu) { unsigned long hvip; + unsigned long flags; struct kvm_vcpu_arch *v = &vcpu->arch; struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; @@ -376,34 +387,41 @@ void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu) /* Sync-up HVIP.VSSIP bit changes does by Guest */ hvip = ncsr_read(CSR_HVIP); + + raw_spin_lock_irqsave(&v->irqs_pending_lock, flags); + if ((csr->hvip ^ hvip) & (1UL << IRQ_VS_SOFT)) { if (hvip & (1UL << IRQ_VS_SOFT)) { - if (!test_and_set_bit(IRQ_VS_SOFT, - v->irqs_pending_mask)) - set_bit(IRQ_VS_SOFT, v->irqs_pending); + if (!__test_and_set_bit(IRQ_VS_SOFT, + v->irqs_pending_mask)) + __set_bit(IRQ_VS_SOFT, v->irqs_pending); } else { - if (!test_and_set_bit(IRQ_VS_SOFT, - v->irqs_pending_mask)) - clear_bit(IRQ_VS_SOFT, v->irqs_pending); + if (!__test_and_set_bit(IRQ_VS_SOFT, + v->irqs_pending_mask)) + __clear_bit(IRQ_VS_SOFT, v->irqs_pending); } } /* Sync up the HVIP.LCOFIP bit changes (only clear) by the guest */ if ((csr->hvip ^ hvip) & (1UL << IRQ_PMU_OVF)) { if (!(hvip & (1UL << IRQ_PMU_OVF)) && - !test_and_set_bit(IRQ_PMU_OVF, v->irqs_pending_mask)) - clear_bit(IRQ_PMU_OVF, v->irqs_pending); + !__test_and_set_bit(IRQ_PMU_OVF, v->irqs_pending_mask)) + __clear_bit(IRQ_PMU_OVF, v->irqs_pending); } /* Sync-up AIA high interrupts */ kvm_riscv_vcpu_aia_sync_interrupts(vcpu); + raw_spin_unlock_irqrestore(&v->irqs_pending_lock, flags); + /* Sync-up timer CSRs */ kvm_riscv_vcpu_timer_sync(vcpu); } int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) { + unsigned long flags; + /* * We only allow VS-mode software, timer, and external * interrupts when irq is one of the local interrupts @@ -416,9 +434,10 @@ int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) irq != IRQ_PMU_OVF) return -EINVAL; - set_bit(irq, vcpu->arch.irqs_pending); - smp_mb__before_atomic(); - set_bit(irq, vcpu->arch.irqs_pending_mask); + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + __set_bit(irq, vcpu->arch.irqs_pending); + __set_bit(irq, vcpu->arch.irqs_pending_mask); + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); kvm_vcpu_kick(vcpu); @@ -427,6 +446,8 @@ int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) { + unsigned long flags; + /* * We only allow VS-mode software, timer, counter overflow and external * interrupts when irq is one of the local interrupts @@ -439,26 +460,33 @@ int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) irq != IRQ_PMU_OVF) return -EINVAL; - clear_bit(irq, vcpu->arch.irqs_pending); - smp_mb__before_atomic(); - set_bit(irq, vcpu->arch.irqs_pending_mask); + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + __clear_bit(irq, vcpu->arch.irqs_pending); + __set_bit(irq, vcpu->arch.irqs_pending_mask); + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); return 0; } bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) { + unsigned long flags; unsigned long ie; + bool ret; + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK) << VSIP_TO_HVIP_SHIFT) & (unsigned long)mask; ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK & (unsigned long)mask; - if (READ_ONCE(vcpu->arch.irqs_pending[0]) & ie) - return true; + ret = vcpu->arch.irqs_pending[0] & ie; + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); /* Check AIA high interrupts */ - return kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask); + if (!ret) + ret = kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask); + + return ret; } void __kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu) diff --git a/arch/riscv/kvm/vcpu_exit.c b/arch/riscv/kvm/vcpu_exit.c index 0bb0c51e3c89..6c8530b9f29e 100644 --- a/arch/riscv/kvm/vcpu_exit.c +++ b/arch/riscv/kvm/vcpu_exit.c @@ -38,6 +38,25 @@ static int gstage_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run, return kvm_riscv_vcpu_mmio_store(vcpu, run, fault_addr, trap->htinst); + case EXC_INST_GUEST_PAGE_FAULT: { + /* + * No memslot backs this GPA and an instruction fetch + * cannot be emulated as MMIO. On bare metal a fetch + * from an unbacked physical address raises an + * instruction access fault, so reflect that back to + * the guest. + */ + struct kvm_cpu_trap inst_trap = { + .sepc = trap->sepc, + .scause = EXC_INST_ACCESS, + .stval = trap->stval, + .htval = 0, + .htinst = 0, + }; + + kvm_riscv_vcpu_trap_redirect(vcpu, &inst_trap); + return 1; + } default: return -EOPNOTSUPP; }; diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c index bb920e8923c9..99b9107b1ac1 100644 --- a/arch/riscv/kvm/vcpu_onereg.c +++ b/arch/riscv/kvm/vcpu_onereg.c @@ -50,19 +50,13 @@ static int kvm_riscv_vcpu_get_reg_config(struct kvm_vcpu *vcpu, reg_val = vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK; break; case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): - if (kvm_riscv_isa_check_host(ZICBOM)) - return -ENOENT; - reg_val = riscv_cbom_block_size; + reg_val = (kvm_riscv_isa_check_host(ZICBOM)) ? 0 : riscv_cbom_block_size; break; case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): - if (kvm_riscv_isa_check_host(ZICBOZ)) - return -ENOENT; - reg_val = riscv_cboz_block_size; + reg_val = (kvm_riscv_isa_check_host(ZICBOZ)) ? 0 : riscv_cboz_block_size; break; case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size): - if (kvm_riscv_isa_check_host(ZICBOP)) - return -ENOENT; - reg_val = riscv_cbop_block_size; + reg_val = (kvm_riscv_isa_check_host(ZICBOP)) ? 0 : riscv_cbop_block_size; break; case KVM_REG_RISCV_CONFIG_REG(mvendorid): reg_val = vcpu->arch.mvendorid; @@ -144,21 +138,15 @@ static int kvm_riscv_vcpu_set_reg_config(struct kvm_vcpu *vcpu, } break; case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): - if (kvm_riscv_isa_check_host(ZICBOM)) - return -ENOENT; - if (reg_val != riscv_cbom_block_size) + if (reg_val && reg_val != riscv_cbom_block_size) return -EINVAL; break; case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): - if (kvm_riscv_isa_check_host(ZICBOZ)) - return -ENOENT; - if (reg_val != riscv_cboz_block_size) + if (reg_val && reg_val != riscv_cboz_block_size) return -EINVAL; break; case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size): - if (kvm_riscv_isa_check_host(ZICBOP)) - return -ENOENT; - if (reg_val != riscv_cbop_block_size) + if (reg_val && reg_val != riscv_cbop_block_size) return -EINVAL; break; case KVM_REG_RISCV_CONFIG_REG(mvendorid): @@ -298,6 +286,7 @@ static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu, { struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; unsigned long regs_max = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); + unsigned long flags; if (reg_num >= regs_max) return -ENOENT; @@ -311,8 +300,11 @@ static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu, ((unsigned long *)csr)[reg_num] = reg_val; - if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) - WRITE_ONCE(vcpu->arch.irqs_pending_mask[0], 0); + if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + vcpu->arch.irqs_pending_mask[0] = 0; + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); + } return 0; } @@ -614,20 +606,6 @@ static int copy_config_reg_indices(const struct kvm_vcpu *vcpu, u64 size; u64 reg; - /* - * Avoid reporting config reg if the corresponding extension - * was not available. - */ - if (i == KVM_REG_RISCV_CONFIG_REG(zicbom_block_size) && - kvm_riscv_isa_check_host(ZICBOM)) - continue; - else if (i == KVM_REG_RISCV_CONFIG_REG(zicboz_block_size) && - kvm_riscv_isa_check_host(ZICBOZ)) - continue; - else if (i == KVM_REG_RISCV_CONFIG_REG(zicbop_block_size) && - kvm_riscv_isa_check_host(ZICBOP)) - continue; - size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CONFIG | i; diff --git a/arch/riscv/kvm/vcpu_pmu.c b/arch/riscv/kvm/vcpu_pmu.c index bb46dcbfb24d..2025b664961c 100644 --- a/arch/riscv/kvm/vcpu_pmu.c +++ b/arch/riscv/kvm/vcpu_pmu.c @@ -586,7 +586,7 @@ int kvm_riscv_vcpu_pmu_ctr_start(struct kvm_vcpu *vcpu, unsigned long ctr_base, } } /* Start the counters that have been configured and requested by the guest */ - for_each_set_bit(i, &ctr_mask, RISCV_MAX_COUNTERS) { + for_each_set_bit(i, &ctr_mask, BITS_PER_LONG) { pmc_index = array_index_nospec(i + ctr_base, RISCV_KVM_MAX_COUNTERS); if (!test_bit(pmc_index, kvpmu->pmc_in_use)) @@ -658,7 +658,7 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, } /* Stop the counters that have been configured and requested by the guest */ - for_each_set_bit(i, &ctr_mask, RISCV_MAX_COUNTERS) { + for_each_set_bit(i, &ctr_mask, BITS_PER_LONG) { pmc_index = array_index_nospec(i + ctr_base, RISCV_KVM_MAX_COUNTERS); if (!test_bit(pmc_index, kvpmu->pmc_in_use)) diff --git a/arch/riscv/kvm/vcpu_sbi_fwft.c b/arch/riscv/kvm/vcpu_sbi_fwft.c index ab39ac464ffd..1342adb3180c 100644 --- a/arch/riscv/kvm/vcpu_sbi_fwft.c +++ b/arch/riscv/kvm/vcpu_sbi_fwft.c @@ -327,9 +327,11 @@ static int kvm_sbi_fwft_set(struct kvm_vcpu *vcpu, u32 feature, if (conf->flags & SBI_FWFT_SET_FLAG_LOCK) return SBI_ERR_DENIED_LOCKED; - conf->flags = flags; + ret = conf->feature->set(vcpu, conf, false, value); + if (ret == SBI_SUCCESS) + conf->flags = flags; - return conf->feature->set(vcpu, conf, false, value); + return ret; } static int kvm_sbi_fwft_get(struct kvm_vcpu *vcpu, unsigned long feature, diff --git a/arch/riscv/kvm/vcpu_vector.c b/arch/riscv/kvm/vcpu_vector.c index 62d2fb77bb9b..3708616e2c32 100644 --- a/arch/riscv/kvm/vcpu_vector.c +++ b/arch/riscv/kvm/vcpu_vector.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -129,11 +130,20 @@ static int kvm_riscv_vcpu_vreg_addr(struct kvm_vcpu *vcpu, return -ENOENT; } } else if (reg_num <= KVM_REG_RISCV_VECTOR_REG(31)) { + unsigned long reg_offset; + if (reg_size != vlenb) return -EINVAL; WARN_ON(!cntx->vector.datap); - *reg_addr = cntx->vector.datap + - (reg_num - KVM_REG_RISCV_VECTOR_REG(0)) * vlenb; + /* + * The reg_num is derived from the userspace-provided ONE_REG + * id. Sanitize it with array_index_nospec() to prevent + * speculative out-of-bounds access to the vector register + * buffer (32 vector registers: v0..v31). + */ + reg_offset = array_index_nospec( + reg_num - KVM_REG_RISCV_VECTOR_REG(0), 32); + *reg_addr = cntx->vector.datap + reg_offset * vlenb; } else { return -ENOENT; } diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 38bba9a1114c..48b019114c19 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -2052,7 +2052,7 @@ static void apic_timer_expired(struct kvm_lapic *apic, bool from_timer_fn) if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use) ktimer->expired_tscdeadline = ktimer->tscdeadline; - if (!from_timer_fn && apic->apicv_active) { + if (!from_timer_fn && apic->apicv_active && vcpu->wants_to_run) { WARN_ON(kvm_get_running_vcpu() != vcpu); kvm_apic_inject_pending_timer_irqs(apic); return; diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 234d0a95abf5..22cf222d3033 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -2642,6 +2642,7 @@ static int mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp, */ if (tdp_enabled && invalid_list && child->role.guest_mode && + !child->root_count && !atomic_long_read(&child->parent_ptes.val)) return kvm_mmu_prepare_zap_page(kvm, child, invalid_list); @@ -4852,16 +4853,17 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault if (r != RET_PF_CONTINUE) return r; - r = RET_PF_RETRY; write_lock(&vcpu->kvm->mmu_lock); - if (is_page_fault_stale(vcpu, fault)) - goto out_unlock; - r = make_mmu_pages_available(vcpu); if (r) goto out_unlock; + if (is_page_fault_stale(vcpu, fault)) { + r = RET_PF_RETRY; + goto out_unlock; + } + r = direct_map(vcpu, fault); out_unlock: @@ -7574,7 +7576,9 @@ void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen) static void mmu_destroy_caches(void) { kmem_cache_destroy(pte_list_desc_cache); + pte_list_desc_cache = NULL; kmem_cache_destroy(mmu_page_header_cache); + mmu_page_header_cache = NULL; } static void kvm_wake_nx_recovery_thread(struct kvm *kvm) diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index df3ae0c7ec2c..1ba840a73b7a 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -864,15 +864,17 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault } #endif - r = RET_PF_RETRY; write_lock(&vcpu->kvm->mmu_lock); - if (is_page_fault_stale(vcpu, fault)) - goto out_unlock; - r = make_mmu_pages_available(vcpu); if (r) goto out_unlock; + + if (is_page_fault_stale(vcpu, fault)) { + r = RET_PF_RETRY; + goto out_unlock; + } + r = FNAME(fetch)(vcpu, fault, &walker); out_unlock: diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 220d42ebc82e..ddf6df7bee93 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -336,6 +336,7 @@ static void nested_put_vmcs12_pages(struct kvm_vcpu *vcpu) static void free_nested(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); + struct vmcs *shadow_vmcs; if (WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01)) vmx_switch_vmcs(vcpu, &vmx->vmcs01); @@ -353,9 +354,15 @@ static void free_nested(struct kvm_vcpu *vcpu) vmx->nested.current_vmptr = INVALID_GPA; if (enable_shadow_vmcs) { vmx_disable_shadow_vmcs(vmx); - vmcs_clear(vmx->vmcs01.shadow_vmcs); - free_vmcs(vmx->vmcs01.shadow_vmcs); + + /* + * Keep the pointer visible until after VMCLEAR, so migration + * can clear an active shadow VMCS on the old CPU. + */ + shadow_vmcs = vmx->vmcs01.shadow_vmcs; + vmcs_clear(shadow_vmcs); vmx->vmcs01.shadow_vmcs = NULL; + free_vmcs(shadow_vmcs); } kfree(vmx->nested.cached_vmcs12); vmx->nested.cached_vmcs12 = NULL; diff --git a/tools/testing/selftests/kvm/x86/sev_init2_tests.c b/tools/testing/selftests/kvm/x86/sev_init2_tests.c index 8db88c355f16..689390c10f7c 100644 --- a/tools/testing/selftests/kvm/x86/sev_init2_tests.c +++ b/tools/testing/selftests/kvm/x86/sev_init2_tests.c @@ -130,12 +130,18 @@ int main(int argc, char *argv[]) KVM_X86_SEV_VMSA_FEATURES, &supported_vmsa_features); - have_sev = kvm_cpu_has(X86_FEATURE_SEV); - TEST_ASSERT(have_sev == !!(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SEV_VM)), - "sev: KVM_CAP_VM_TYPES (%x) does not match cpuid (checking %x)", - kvm_check_cap(KVM_CAP_VM_TYPES), 1 << KVM_X86_SEV_VM); + /* + * Whether a VM type is available depends on KVM, not just CPUID: e.g. + * when all SEV ASIDs are assigned to SEV-SNP, KVM does not offer the + * SEV VM type even though X86_FEATURE_SEV is set. Derive availability + * from KVM_CAP_VM_TYPES and only assert the one-way implication that a + * type offered by KVM must also be reported in CPUID. + */ + have_sev = kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SEV_VM); + TEST_ASSERT(!have_sev || kvm_cpu_has(X86_FEATURE_SEV), + "sev: SEV_VM supported without SEV in CPUID"); - TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SEV_VM)); + TEST_REQUIRE(have_sev); have_sev_es = kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SEV_ES_VM); TEST_ASSERT(!have_sev_es || kvm_cpu_has(X86_FEATURE_SEV_ES), diff --git a/tools/testing/selftests/kvm/x86/sev_smoke_test.c b/tools/testing/selftests/kvm/x86/sev_smoke_test.c index 6b2cbe2a90b7..bf27b6187afa 100644 --- a/tools/testing/selftests/kvm/x86/sev_smoke_test.c +++ b/tools/testing/selftests/kvm/x86/sev_smoke_test.c @@ -247,7 +247,14 @@ int main(int argc, char *argv[]) { TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SEV)); - test_sev_smoke(guest_sev_code, KVM_X86_SEV_VM, 0); + /* + * Only exercise VM types the host actually offers. CPUID reporting + * SEV does not guarantee KVM offers the SEV VM type: when all SEV + * ASIDs are assigned to SEV-SNP, KVM_X86_SEV_VM is unavailable even + * though X86_FEATURE_SEV is set. Gate every type on KVM_CAP_VM_TYPES. + */ + if (kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SEV_VM)) + test_sev_smoke(guest_sev_code, KVM_X86_SEV_VM, 0); if (kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SEV_ES_VM)) test_sev_smoke(guest_sev_es_code, KVM_X86_SEV_ES_VM, SEV_POLICY_ES);