mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-28 14:34:17 -04:00
Code Issues Packages Projects Releases Wiki Activity
Files
185a4caeecabc150106deda1da170b09f2ad803f
linux/arch/x86/kvm/regs.h
Paolo Bonzini 4e6df93968 Merge branch 'kvm-single-pdptrs' into HEAD 
The non-MMU changes/preliminary cleanups from the "split kvm_mmu in
three" series[1].  The final outcome is to have a single copy of the
PDPTRs (in vcpu->arch) instead of two (in root_mmu and nested_mmu).

[1] https://lore.kernel.org/kvm/20260603105814.10236-1-pbonzini@redhat.com/T/#t
2026-06-12 10:47:24 +02:00

424 lines
12 KiB
C
Raw Blame History

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef ARCH_X86_KVM_REGS_H
#define ARCH_X86_KVM_REGS_H
#include <linux/kvm_host.h>
#define KVM_POSSIBLE_CR0_GUEST_BITS (X86_CR0_TS | X86_CR0_WP)
#define KVM_POSSIBLE_CR4_GUEST_BITS \
(X86_CR4_PVI | X86_CR4_DE | X86_CR4_PCE | X86_CR4_OSFXSR \
| X86_CR4_OSXMMEXCPT | X86_CR4_PGE | X86_CR4_TSD | X86_CR4_FSGSBASE \
| X86_CR4_CET)
#define X86_CR0_PDPTR_BITS (X86_CR0_CD | X86_CR0_NW | X86_CR0_PG)
#define X86_CR4_TLBFLUSH_BITS (X86_CR4_PGE | X86_CR4_PCIDE | X86_CR4_PAE | X86_CR4_SMEP)
#define X86_CR4_PDPTR_BITS (X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_SMEP)
static_assert(!(KVM_POSSIBLE_CR0_GUEST_BITS & X86_CR0_PDPTR_BITS));
static inline bool is_long_mode(struct kvm_vcpu *vcpu)
{
#ifdef CONFIG_X86_64
return !!(vcpu->arch.efer & EFER_LMA);
#else
return false;
#endif
}
static inline bool is_64_bit_mode(struct kvm_vcpu *vcpu)
{
int cs_db, cs_l;
WARN_ON_ONCE(vcpu->arch.guest_state_protected);
if (!is_long_mode(vcpu))
return false;
kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
return cs_l;
}
static inline bool is_64_bit_hypercall(struct kvm_vcpu *vcpu)
{
#ifdef CONFIG_X86_64
/*
* If running with protected guest state, the CS register is not
* accessible. The hypercall register values will have had to been
* provided in 64-bit mode, so assume the guest is in 64-bit.
*/
return vcpu->arch.guest_state_protected || is_64_bit_mode(vcpu);
#else
return false;
#endif
}
static __always_inline unsigned long kvm_reg_mode_mask(struct kvm_vcpu *vcpu)
{
#ifdef CONFIG_X86_64
return is_64_bit_mode(vcpu) ? GENMASK(63, 0) : GENMASK(31, 0);
#else
return GENMASK(31, 0);
#endif
}
#define __BUILD_KVM_GPR_ACCESSORS(lname, uname) \
static __always_inline unsigned long kvm_##lname##_read(struct kvm_vcpu *vcpu) \
{ \
return vcpu->arch.regs[VCPU_REGS_##uname] & kvm_reg_mode_mask(vcpu); \
} \
static __always_inline unsigned long kvm_##lname##_read_raw(struct kvm_vcpu *vcpu) \
{ \
return vcpu->arch.regs[VCPU_REGS_##uname]; \
} \
static __always_inline void kvm_##lname##_write_raw(struct kvm_vcpu *vcpu, \
unsigned long val) \
{ \
vcpu->arch.regs[VCPU_REGS_##uname] = val; \
}
#define BUILD_KVM_GPR_ACCESSORS(lname, uname) \
static __always_inline u32 kvm_e##lname##_read(struct kvm_vcpu *vcpu) \
{ \
return vcpu->arch.regs[VCPU_REGS_##uname]; \
} \
static __always_inline void kvm_e##lname##_write(struct kvm_vcpu *vcpu, u32 val) \
{ \
vcpu->arch.regs[VCPU_REGS_##uname] = val; \
} \
__BUILD_KVM_GPR_ACCESSORS(r##lname, uname)
BUILD_KVM_GPR_ACCESSORS(ax, RAX)
BUILD_KVM_GPR_ACCESSORS(bx, RBX)
BUILD_KVM_GPR_ACCESSORS(cx, RCX)
BUILD_KVM_GPR_ACCESSORS(dx, RDX)
BUILD_KVM_GPR_ACCESSORS(bp, RBP)
BUILD_KVM_GPR_ACCESSORS(si, RSI)
BUILD_KVM_GPR_ACCESSORS(di, RDI)
#ifdef CONFIG_X86_64
__BUILD_KVM_GPR_ACCESSORS(r8, R8)
__BUILD_KVM_GPR_ACCESSORS(r9, R9)
__BUILD_KVM_GPR_ACCESSORS(r10, R10)
__BUILD_KVM_GPR_ACCESSORS(r11, R11)
__BUILD_KVM_GPR_ACCESSORS(r12, R12)
__BUILD_KVM_GPR_ACCESSORS(r13, R13)
__BUILD_KVM_GPR_ACCESSORS(r14, R14)
__BUILD_KVM_GPR_ACCESSORS(r15, R15)
#endif
/*
* Using the register cache from interrupt context is generally not allowed, as
* caching a register and marking it available/dirty can't be done atomically,
* i.e. accesses from interrupt context may clobber state or read stale data if
* the vCPU task is in the process of updating the cache. The exception is if
* KVM is handling a PMI IRQ/NMI VM-Exit, as that bound code sequence doesn't
* touch the cache, it runs after the cache is reset (post VM-Exit), and PMIs
* need to access several registers that are cacheable.
*/
#define kvm_assert_register_caching_allowed(vcpu) \
lockdep_assert_once(in_task() || kvm_arch_pmi_in_guest(vcpu))
/*
* avail dirty
* 0 0 register in VMCS/VMCB
* 0 1 *INVALID*
* 1 0 register in vcpu->arch
* 1 1 register in vcpu->arch, needs to be stored back
*/
static inline bool kvm_register_is_available(struct kvm_vcpu *vcpu,
enum kvm_reg reg)
{
kvm_assert_register_caching_allowed(vcpu);
return test_bit(reg, vcpu->arch.regs_avail);
}
static inline bool kvm_register_is_dirty(struct kvm_vcpu *vcpu,
enum kvm_reg reg)
{
kvm_assert_register_caching_allowed(vcpu);
return test_bit(reg, vcpu->arch.regs_dirty);
}
static inline void kvm_register_mark_for_reload(struct kvm_vcpu *vcpu,
enum kvm_reg reg)
{
kvm_assert_register_caching_allowed(vcpu);
__clear_bit(reg, vcpu->arch.regs_avail);
__clear_bit(reg, vcpu->arch.regs_dirty);
}
static inline void kvm_register_mark_available(struct kvm_vcpu *vcpu,
enum kvm_reg reg)
{
kvm_assert_register_caching_allowed(vcpu);
__set_bit(reg, vcpu->arch.regs_avail);
}
static inline void kvm_register_mark_dirty(struct kvm_vcpu *vcpu,
enum kvm_reg reg)
{
kvm_assert_register_caching_allowed(vcpu);
__set_bit(reg, vcpu->arch.regs_avail);
__set_bit(reg, vcpu->arch.regs_dirty);
}
/*
* kvm_register_test_and_mark_available() is a special snowflake that uses an
* arch bitop directly to avoid the explicit instrumentation that comes with
* the generic bitops. This allows code that cannot be instrumented (noinstr
* functions), e.g. the low level VM-Enter/VM-Exit paths, to cache registers.
*/
static __always_inline bool kvm_register_test_and_mark_available(struct kvm_vcpu *vcpu,
enum kvm_reg reg)
{
kvm_assert_register_caching_allowed(vcpu);
return arch___test_and_set_bit(reg, vcpu->arch.regs_avail);
}
static __always_inline void kvm_clear_available_registers(struct kvm_vcpu *vcpu,
unsigned long clear_mask)
{
BUILD_BUG_ON(sizeof(clear_mask) != sizeof(vcpu->arch.regs_avail[0]));
BUILD_BUG_ON(ARRAY_SIZE(vcpu->arch.regs_avail) != 1);
/*
* Note the bitwise-AND! In practice, a straight write would also work
* as KVM initializes the mask to all ones and never clears registers
* that are eagerly synchronized. Using a bitwise-AND adds a bit of
* sanity checking as incorrectly marking an eagerly sync'd register
* unavailable will generate a WARN due to an unexpected cache request.
*/
vcpu->arch.regs_avail[0] &= ~clear_mask;
}
static __always_inline void kvm_reset_dirty_registers(struct kvm_vcpu *vcpu)
{
BUILD_BUG_ON(ARRAY_SIZE(vcpu->arch.regs_dirty) != 1);
vcpu->arch.regs_dirty[0] = 0;
}
/*
* The "raw" register helpers are only for cases where the full 64 bits of a
* register are read/written irrespective of current vCPU mode. In other words,
* odds are good you shouldn't be using the raw variants.
*/
static inline unsigned long kvm_register_read_raw(struct kvm_vcpu *vcpu, int reg)
{
if (WARN_ON_ONCE((unsigned int)reg >= NR_VCPU_GENERAL_PURPOSE_REGS))
return 0;
if (!kvm_register_is_available(vcpu, reg))
kvm_x86_call(cache_reg)(vcpu, reg);
return vcpu->arch.regs[reg];
}
static inline unsigned long kvm_register_read(struct kvm_vcpu *vcpu, int reg)
{
return kvm_register_read_raw(vcpu, reg) & kvm_reg_mode_mask(vcpu);
}
static inline void kvm_register_write_raw(struct kvm_vcpu *vcpu, int reg,
unsigned long val)
{
if (WARN_ON_ONCE((unsigned int)reg >= NR_VCPU_GENERAL_PURPOSE_REGS))
return;
vcpu->arch.regs[reg] = val;
kvm_register_mark_dirty(vcpu, reg);
}
static inline void kvm_register_write(struct kvm_vcpu *vcpu,
int reg, unsigned long val)
{
return kvm_register_write_raw(vcpu, reg, val & kvm_reg_mode_mask(vcpu));
}
static inline unsigned long kvm_rip_read(struct kvm_vcpu *vcpu)
{
if (!kvm_register_is_available(vcpu, VCPU_REG_RIP))
kvm_x86_call(cache_reg)(vcpu, VCPU_REG_RIP);
return vcpu->arch.rip;
}
static inline void kvm_rip_write(struct kvm_vcpu *vcpu, unsigned long val)
{
vcpu->arch.rip = val;
kvm_register_mark_dirty(vcpu, VCPU_REG_RIP);
}
static inline unsigned long kvm_rsp_read(struct kvm_vcpu *vcpu)
{
return kvm_register_read_raw(vcpu, VCPU_REGS_RSP);
}
static inline void kvm_rsp_write(struct kvm_vcpu *vcpu, unsigned long val)
{
kvm_register_write_raw(vcpu, VCPU_REGS_RSP, val);
}
static inline u64 kvm_read_edx_eax(struct kvm_vcpu *vcpu)
{
return kvm_eax_read(vcpu) | (u64)(kvm_edx_read(vcpu)) << 32;
}
static inline u64 kvm_pdptr_read(struct kvm_vcpu *vcpu, int index)
{
might_sleep(); /* on svm */
if (!kvm_register_is_available(vcpu, VCPU_REG_PDPTR))
kvm_x86_call(cache_reg)(vcpu, VCPU_REG_PDPTR);
return vcpu->arch.pdptrs[index];
}
static inline void kvm_pdptr_write(struct kvm_vcpu *vcpu, int index, u64 value)
{
vcpu->arch.pdptrs[index] = value;
}
static inline ulong kvm_read_cr0_bits(struct kvm_vcpu *vcpu, ulong mask)
{
ulong tmask = mask & KVM_POSSIBLE_CR0_GUEST_BITS;
if ((tmask & vcpu->arch.cr0_guest_owned_bits) &&
!kvm_register_is_available(vcpu, VCPU_REG_CR0))
kvm_x86_call(cache_reg)(vcpu, VCPU_REG_CR0);
return vcpu->arch.cr0 & mask;
}
static __always_inline bool kvm_is_cr0_bit_set(struct kvm_vcpu *vcpu,
unsigned long cr0_bit)
{
BUILD_BUG_ON(!is_power_of_2(cr0_bit));
return !!kvm_read_cr0_bits(vcpu, cr0_bit);
}
static inline ulong kvm_read_cr0(struct kvm_vcpu *vcpu)
{
return kvm_read_cr0_bits(vcpu, ~0UL);
}
static inline ulong kvm_read_cr4_bits(struct kvm_vcpu *vcpu, ulong mask)
{
ulong tmask = mask & KVM_POSSIBLE_CR4_GUEST_BITS;
if ((tmask & vcpu->arch.cr4_guest_owned_bits) &&
!kvm_register_is_available(vcpu, VCPU_REG_CR4))
kvm_x86_call(cache_reg)(vcpu, VCPU_REG_CR4);
return vcpu->arch.cr4 & mask;
}
static __always_inline bool kvm_is_cr4_bit_set(struct kvm_vcpu *vcpu,
unsigned long cr4_bit)
{
BUILD_BUG_ON(!is_power_of_2(cr4_bit));
return !!kvm_read_cr4_bits(vcpu, cr4_bit);
}
static inline ulong kvm_read_cr3(struct kvm_vcpu *vcpu)
{
if (!kvm_register_is_available(vcpu, VCPU_REG_CR3))
kvm_x86_call(cache_reg)(vcpu, VCPU_REG_CR3);
return vcpu->arch.cr3;
}
static inline ulong kvm_read_cr4(struct kvm_vcpu *vcpu)
{
return kvm_read_cr4_bits(vcpu, ~0UL);
}
static inline bool __kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
{
return !(cr4 & vcpu->arch.cr4_guest_rsvd_bits);
}
#define __cr4_reserved_bits(__cpu_has, __c) \
({ \
u64 __reserved_bits = CR4_RESERVED_BITS; \
\
if (!__cpu_has(__c, X86_FEATURE_XSAVE)) \
__reserved_bits |= X86_CR4_OSXSAVE; \
if (!__cpu_has(__c, X86_FEATURE_SMEP)) \
__reserved_bits |= X86_CR4_SMEP; \
if (!__cpu_has(__c, X86_FEATURE_SMAP)) \
__reserved_bits |= X86_CR4_SMAP; \
if (!__cpu_has(__c, X86_FEATURE_FSGSBASE)) \
__reserved_bits |= X86_CR4_FSGSBASE; \
if (!__cpu_has(__c, X86_FEATURE_PKU)) \
__reserved_bits |= X86_CR4_PKE; \
if (!__cpu_has(__c, X86_FEATURE_LA57)) \
__reserved_bits |= X86_CR4_LA57; \
if (!__cpu_has(__c, X86_FEATURE_UMIP)) \
__reserved_bits |= X86_CR4_UMIP; \
if (!__cpu_has(__c, X86_FEATURE_VMX)) \
__reserved_bits |= X86_CR4_VMXE; \
if (!__cpu_has(__c, X86_FEATURE_PCID)) \
__reserved_bits |= X86_CR4_PCIDE; \
if (!__cpu_has(__c, X86_FEATURE_LAM)) \
__reserved_bits |= X86_CR4_LAM_SUP; \
if (!__cpu_has(__c, X86_FEATURE_SHSTK) && \
!__cpu_has(__c, X86_FEATURE_IBT)) \
__reserved_bits |= X86_CR4_CET; \
__reserved_bits; \
})
static inline bool is_protmode(struct kvm_vcpu *vcpu)
{
return kvm_is_cr0_bit_set(vcpu, X86_CR0_PE);
}
static inline bool is_pae(struct kvm_vcpu *vcpu)
{
return kvm_is_cr4_bit_set(vcpu, X86_CR4_PAE);
}
static inline bool is_pse(struct kvm_vcpu *vcpu)
{
return kvm_is_cr4_bit_set(vcpu, X86_CR4_PSE);
}
static inline bool is_paging(struct kvm_vcpu *vcpu)
{
return likely(kvm_is_cr0_bit_set(vcpu, X86_CR0_PG));
}
static inline bool is_pae_paging(struct kvm_vcpu *vcpu)
{
return !is_long_mode(vcpu) && is_pae(vcpu) && is_paging(vcpu);
}
static inline bool kvm_dr7_valid(u64 data)
{
/* Bits [63:32] are reserved */
return !(data >> 32);
}
static inline bool kvm_dr6_valid(u64 data)
{
/* Bits [63:32] are reserved */
return !(data >> 32);
}
static inline void enter_guest_mode(struct kvm_vcpu *vcpu)
{
vcpu->arch.hflags |= HF_GUEST_MASK;
vcpu->stat.guest_mode = 1;
}
static inline void leave_guest_mode(struct kvm_vcpu *vcpu)
{
vcpu->arch.hflags &= ~HF_GUEST_MASK;
if (vcpu->arch.load_eoi_exitmap_pending) {
vcpu->arch.load_eoi_exitmap_pending = false;
kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu);
}
vcpu->stat.guest_mode = 0;
}
static inline bool is_guest_mode(struct kvm_vcpu *vcpu)
{
return vcpu->arch.hflags & HF_GUEST_MASK;
}
#endif
Reference in New Issue View Git Blame Copy Permalink