mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-07-22 02:17:36 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'kvm-x86-sev-7.2' of https://github.com/kvm-x86/linux into HEAD

Browse Source 
KVM SEV changes for 7.2

 - Don't advertise support for unusuable VM types, and account for VM types
   that are disabled by firmware, e.g. to mitigate security vulnerabilities.

 - Rewrite the SEV {en,de}crypt debug ioctls as they were riddle with bugs and
   unnecessarily complicated, and add comprehensive tests.

 - Clean up and deduplicate the SEV page pinning code.

 - Fix minor goofs related to writing back CPUID information after firmware
   rejects a CPUID page for an SNP vCPU.
This commit is contained in:
Paolo Bonzini
2026-06-12 10:13:03 +02:00
parent b02a4f8c42 97cd21d57e
commit b39c6bd6fd
12 changed files with 528 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files

483
arch/x86/kvm/svm/sev.c
View File

@@ -720,14 +720,50 @@ static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
return ret;
}
static int sev_check_pin_count(struct kvm *kvm, unsigned long npages)
{
unsigned long total_npages, lock_limit;
total_npages = to_kvm_sev_info(kvm)->pages_locked + npages;
if (total_npages > totalram_pages())
return -EINVAL;
lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
if (total_npages > lock_limit && !capable(CAP_IPC_LOCK)) {
pr_err_ratelimited("SEV: %lu total pages would exceed the lock limit of %lu.\n",
total_npages, lock_limit);
return -ENOMEM;
}
return 0;
}
static int sev_pin_user_pages(struct kvm *kvm, unsigned long addr, int npages,
unsigned int gup_flags, struct page **pages)
{
int npinned;
lockdep_assert_held(&kvm->lock);
npinned = pin_user_pages_fast(addr, npages, gup_flags, pages);
if (npinned != npages) {
if (npinned > 0)
unpin_user_pages(pages, npinned);
pr_err_ratelimited("SEV: Failure locking %u pages.\n", npages);
return -ENOMEM;
}
to_kvm_sev_info(kvm)->pages_locked += npages;
return 0;
}
static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
unsigned long ulen, unsigned long *n,
unsigned int flags)
{
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
unsigned long npages, total_npages, lock_limit;
unsigned long npages;
struct page **pages;
int npinned, ret;
int ret;
lockdep_assert_held(&kvm->lock);
@@ -743,16 +779,9 @@ static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
if (npages > INT_MAX)
return ERR_PTR(-EINVAL);
total_npages = sev->pages_locked + npages;
if (total_npages > totalram_pages())
return ERR_PTR(-EINVAL);
lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
if (total_npages > lock_limit && !capable(CAP_IPC_LOCK)) {
pr_err("SEV: %lu total pages would exceed the lock limit of %lu.\n",
total_npages, lock_limit);
return ERR_PTR(-ENOMEM);
}
ret = sev_check_pin_count(kvm, npages);
if (ret)
return ERR_PTR(ret);
/*
* Don't WARN if the kernel (rightly) thinks the total size is absurd,
@@ -764,25 +793,14 @@ static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
if (!pages)
return ERR_PTR(-ENOMEM);
/* Pin the user virtual address. */
npinned = pin_user_pages_fast(uaddr, npages, flags, pages);
if (npinned != npages) {
pr_err("SEV: Failure locking %lu pages.\n", npages);
ret = -ENOMEM;
goto err;
ret = sev_pin_user_pages(kvm, uaddr, npages, flags, pages);
if (ret) {
kvfree(pages);
return ERR_PTR(ret);
}
*n = npages;
sev->pages_locked = total_npages;
return pages;
err:
if (npinned > 0)
unpin_user_pages(pages, npinned);
kvfree(pages);
return ERR_PTR(ret);
}
static void sev_unpin_memory(struct kvm *kvm, struct page **pages,
@@ -793,6 +811,29 @@ static void sev_unpin_memory(struct kvm *kvm, struct page **pages,
to_kvm_sev_info(kvm)->pages_locked -= npages;
}
static struct page *sev_pin_page(struct kvm *kvm, unsigned long addr,
unsigned int flags)
{
struct page *page;
int r;
r = sev_check_pin_count(kvm, 1);
if (r)
return ERR_PTR(r);
r = sev_pin_user_pages(kvm, addr, 1, flags, &page);
if (r)
return ERR_PTR(r);
return page;
}
static void sev_unpin_page(struct kvm *kvm, struct page *page)
{
unpin_user_pages(&page, 1);
to_kvm_sev_info(kvm)->pages_locked -= 1;
}
static void sev_clflush_pages(struct page *pages[], unsigned long npages)
{
uint8_t *page_virtual;
@@ -1196,160 +1237,115 @@ static int sev_guest_status(struct kvm *kvm, struct kvm_sev_cmd *argp)
return ret;
}
static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src,
unsigned long dst, int size,
int *error, bool enc)
static int sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src_pa,
unsigned long dst_pa, unsigned int size,
unsigned int ioctl, int *error)
{
struct sev_data_dbg data;
int cmd = ioctl == KVM_SEV_DBG_DECRYPT ? SEV_CMD_DBG_DECRYPT :
SEV_CMD_DBG_ENCRYPT;
struct sev_data_dbg data = {
.handle = to_kvm_sev_info(kvm)->handle,
.dst_addr = dst_pa,
.src_addr = src_pa,
.len = size,
};
data.reserved = 0;
data.handle = to_kvm_sev_info(kvm)->handle;
data.dst_addr = dst;
data.src_addr = src;
data.len = size;
return sev_issue_cmd(kvm,
enc ? SEV_CMD_DBG_ENCRYPT : SEV_CMD_DBG_DECRYPT,
&data, error);
return sev_issue_cmd(kvm, cmd, &data, error);
}
static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr,
unsigned long dst_paddr, int sz, int *err)
static void *sev_dbg_crypt_slow_alloc(struct page *page, unsigned long __va,
unsigned int len, unsigned long *pa,
unsigned int *nr_bytes)
{
int offset;
unsigned long va = ALIGN_DOWN(__va, 16);
/* The number of bytes to {de,en}crypt must be 16-byte aligned. */
*nr_bytes = round_up(len, 16);
/*
* Its safe to read more than we are asked, caller should ensure that
* destination has enough space.
* Increase the number of bytes to {de,en}crypt by one chunk (16 bytes)
* if the aligned address and length doesn't cover the unaligned range,
* e.g. if the address is unaligned _and_ the access will split a chunk
* at the tail.
*/
offset = src_paddr & 15;
src_paddr = round_down(src_paddr, 16);
sz = round_up(sz + offset, 16);
if (va + *nr_bytes < __va + len)
*nr_bytes += 16;
return __sev_issue_dbg_cmd(kvm, src_paddr, dst_paddr, sz, err, false);
}
static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr,
void __user *dst_uaddr,
unsigned long dst_paddr,
int size, int *err)
{
struct page *tpage = NULL;
int ret, offset;
/* if inputs are not 16-byte then use intermediate buffer */
if (!IS_ALIGNED(dst_paddr, 16) ||
!IS_ALIGNED(paddr, 16) ||
!IS_ALIGNED(size, 16)) {
tpage = (void *)alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
if (!tpage)
return -ENOMEM;
dst_paddr = __sme_page_pa(tpage);
}
ret = __sev_dbg_decrypt(kvm, paddr, dst_paddr, size, err);
if (ret)
goto e_free;
if (tpage) {
offset = paddr & 15;
if (copy_to_user(dst_uaddr, page_address(tpage) + offset, size))
ret = -EFAULT;
}
e_free:
if (tpage)
__free_page(tpage);
return ret;
}
static int __sev_dbg_encrypt_user(struct kvm *kvm, unsigned long paddr,
void __user *vaddr,
unsigned long dst_paddr,
void __user *dst_vaddr,
int size, int *error)
{
struct page *src_tpage = NULL;
struct page *dst_tpage = NULL;
int ret, len = size;
/* If source buffer is not aligned then use an intermediate buffer */
if (!IS_ALIGNED((unsigned long)vaddr, 16)) {
src_tpage = alloc_page(GFP_KERNEL_ACCOUNT);
if (!src_tpage)
return -ENOMEM;
if (copy_from_user(page_address(src_tpage), vaddr, size)) {
__free_page(src_tpage);
return -EFAULT;
}
paddr = __sme_page_pa(src_tpage);
}
*pa = __sme_page_pa(page) + (va & ~PAGE_MASK);
/*
* If destination buffer or length is not aligned then do read-modify-write:
* - decrypt destination in an intermediate buffer
* - copy the source buffer in an intermediate buffer
* - use the intermediate buffer as source buffer
* Sanity check that the new access won't split a page. This should
* never happen; just pretend the allocation failed.
*/
if (!IS_ALIGNED((unsigned long)dst_vaddr, 16) || !IS_ALIGNED(size, 16)) {
int dst_offset;
if (WARN_ON_ONCE((*pa & PAGE_MASK) != ((*pa + *nr_bytes - 1) & PAGE_MASK)))
return NULL;
dst_tpage = alloc_page(GFP_KERNEL_ACCOUNT);
if (!dst_tpage) {
ret = -ENOMEM;
goto e_free;
}
ret = __sev_dbg_decrypt(kvm, dst_paddr,
__sme_page_pa(dst_tpage), size, error);
if (ret)
goto e_free;
/*
* If source is kernel buffer then use memcpy() otherwise
* copy_from_user().
*/
dst_offset = dst_paddr & 15;
if (src_tpage)
memcpy(page_address(dst_tpage) + dst_offset,
page_address(src_tpage), size);
else {
if (copy_from_user(page_address(dst_tpage) + dst_offset,
vaddr, size)) {
ret = -EFAULT;
goto e_free;
}
}
paddr = __sme_page_pa(dst_tpage);
dst_paddr = round_down(dst_paddr, 16);
len = round_up(size, 16);
}
ret = __sev_issue_dbg_cmd(kvm, paddr, dst_paddr, len, error, true);
e_free:
if (src_tpage)
__free_page(src_tpage);
if (dst_tpage)
__free_page(dst_tpage);
return ret;
return kmalloc(*nr_bytes, GFP_KERNEL);
}
static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
static int sev_dbg_decrypt_slow(struct kvm *kvm, unsigned long src,
struct page *src_p, unsigned long dst,
unsigned int len, int *err)
{
unsigned int nr_bytes;
unsigned long src_pa;
void *buf;
int r;
buf = sev_dbg_crypt_slow_alloc(src_p, src, len, &src_pa, &nr_bytes);
if (!buf)
return -ENOMEM;
r = sev_issue_dbg_cmd(kvm, src_pa, __sme_set(__pa(buf)),
nr_bytes, KVM_SEV_DBG_DECRYPT, err);
if (r)
goto out;
if (copy_to_user((void __user *)dst, buf + (src & 15), len))
r = -EFAULT;
out:
kfree(buf);
return r;
}
static int sev_dbg_encrypt_slow(struct kvm *kvm, unsigned long src,
unsigned long dst, struct page *dst_p,
unsigned int len, int *err)
{
unsigned int nr_bytes;
unsigned long dst_pa;
void *buf;
int r;
/* Decrypt the _destination_ to do a RMW on plaintext. */
buf = sev_dbg_crypt_slow_alloc(dst_p, dst, len, &dst_pa, &nr_bytes);
if (!buf)
return -ENOMEM;
r = sev_issue_dbg_cmd(kvm, dst_pa, __sme_set(__pa(buf)),
nr_bytes, KVM_SEV_DBG_DECRYPT, err);
if (r)
goto out;
/*
* Copy from the source into the intermediate buffer, and then
* re-encrypt the buffer into the destination.
*/
if (copy_from_user(buf + (dst & 15), (void __user *)src, len))
r = -EFAULT;
else
r = sev_issue_dbg_cmd(kvm, __sme_set(__pa(buf)), dst_pa,
nr_bytes, KVM_SEV_DBG_ENCRYPT, err);
out:
kfree(buf);
return r;
}
static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp,
unsigned int cmd)
{
unsigned long vaddr, vaddr_end, next_vaddr;
unsigned long dst_vaddr;
struct page **src_p, **dst_p;
struct kvm_sev_dbg debug;
unsigned long n;
unsigned int size;
int ret;
unsigned int i, len;
if (!sev_guest(kvm))
return -ENOTTY;
@@ -1357,27 +1353,38 @@ static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
if (copy_from_user(&debug, u64_to_user_ptr(argp->data), sizeof(debug)))
return -EFAULT;
if (!debug.len || debug.src_uaddr + debug.len < debug.src_uaddr)
return -EINVAL;
if (!debug.dst_uaddr)
if (!debug.len || !debug.src_uaddr || !debug.dst_uaddr)
return -EINVAL;
vaddr = debug.src_uaddr;
size = debug.len;
vaddr_end = vaddr + size;
dst_vaddr = debug.dst_uaddr;
if (debug.src_uaddr + debug.len < debug.src_uaddr ||
debug.dst_uaddr + debug.len < debug.dst_uaddr)
return -EINVAL;
for (; vaddr < vaddr_end; vaddr = next_vaddr) {
int len, s_off, d_off;
for (i = 0; i < debug.len; i += len) {
unsigned long src = debug.src_uaddr + i;
unsigned long dst = debug.dst_uaddr + i;
unsigned long s_off = src & ~PAGE_MASK;
unsigned long d_off = dst & ~PAGE_MASK;
struct page *src_p, *dst_p;
int ret;
/* lock userspace source and destination page */
src_p = sev_pin_memory(kvm, vaddr & PAGE_MASK, PAGE_SIZE, &n, 0);
/*
* Copy as many remaining bytes as possible while staying in a
* single page for both the source and destination.
*/
len = min3(debug.len - i, PAGE_SIZE - s_off, PAGE_SIZE - d_off);
/*
* Pin the source and destination pages; firmware operates on
* physical addresses.
*/
src_p = sev_pin_page(kvm, src & PAGE_MASK, 0);
if (IS_ERR(src_p))
return PTR_ERR(src_p);
dst_p = sev_pin_memory(kvm, dst_vaddr & PAGE_MASK, PAGE_SIZE, &n, FOLL_WRITE);
dst_p = sev_pin_page(kvm, dst & PAGE_MASK, FOLL_WRITE);
if (IS_ERR(dst_p)) {
sev_unpin_memory(kvm, src_p, n);
sev_unpin_page(kvm, src_p);
return PTR_ERR(dst_p);
}
@@ -1386,43 +1393,28 @@ static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
* the pages; flush the destination too so that future accesses do not
* see stale data.
*/
sev_clflush_pages(src_p, 1);
sev_clflush_pages(dst_p, 1);
sev_clflush_pages(&src_p, 1);
sev_clflush_pages(&dst_p, 1);
/*
* Since user buffer may not be page aligned, calculate the
* offset within the page.
*/
s_off = vaddr & ~PAGE_MASK;
d_off = dst_vaddr & ~PAGE_MASK;
len = min_t(size_t, (PAGE_SIZE - s_off), size);
if (dec)
ret = __sev_dbg_decrypt_user(kvm,
__sme_page_pa(src_p[0]) + s_off,
(void __user *)dst_vaddr,
__sme_page_pa(dst_p[0]) + d_off,
len, &argp->error);
if (IS_ALIGNED(src, 16) && IS_ALIGNED(dst, 16) && IS_ALIGNED(len, 16))
ret = sev_issue_dbg_cmd(kvm,
__sme_page_pa(src_p) + s_off,
__sme_page_pa(dst_p) + d_off,
len, cmd, &argp->error);
else if (cmd == KVM_SEV_DBG_DECRYPT)
ret = sev_dbg_decrypt_slow(kvm, src, src_p, dst,
len, &argp->error);
else
ret = __sev_dbg_encrypt_user(kvm,
__sme_page_pa(src_p[0]) + s_off,
(void __user *)vaddr,
__sme_page_pa(dst_p[0]) + d_off,
(void __user *)dst_vaddr,
len, &argp->error);
ret = sev_dbg_encrypt_slow(kvm, src, dst, dst_p,
len, &argp->error);
sev_unpin_memory(kvm, src_p, n);
sev_unpin_memory(kvm, dst_p, n);
sev_unpin_page(kvm, src_p);
sev_unpin_page(kvm, dst_p);
if (ret)
goto err;
next_vaddr = vaddr + len;
dst_vaddr = dst_vaddr + len;
size -= len;
return ret;
}
err:
return ret;
return 0;
}
static int sev_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp)
@@ -1695,8 +1687,7 @@ static int sev_send_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
struct sev_data_send_update_data data;
struct kvm_sev_send_update_data params;
void *hdr, *trans_data;
struct page **guest_page;
unsigned long n;
struct page *guest_page;
int ret, offset;
if (!sev_guest(kvm))
@@ -1720,8 +1711,7 @@ static int sev_send_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
return -EINVAL;
/* Pin guest memory */
guest_page = sev_pin_memory(kvm, params.guest_uaddr & PAGE_MASK,
PAGE_SIZE, &n, 0);
guest_page = sev_pin_page(kvm, params.guest_uaddr & PAGE_MASK, 0);
if (IS_ERR(guest_page))
return PTR_ERR(guest_page);
@@ -1742,7 +1732,7 @@ static int sev_send_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
data.trans_len = params.trans_len;
/* The SEND_UPDATE_DATA command requires C-bit to be always set. */
data.guest_address = (page_to_pfn(guest_page[0]) << PAGE_SHIFT) + offset;
data.guest_address = page_to_phys(guest_page) + offset;
data.guest_address |= sev_me_mask;
data.guest_len = params.guest_len;
data.handle = to_kvm_sev_info(kvm)->handle;
@@ -1769,8 +1759,7 @@ static int sev_send_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
e_free_hdr:
kfree(hdr);
e_unpin:
sev_unpin_memory(kvm, guest_page, n);
sev_unpin_page(kvm, guest_page);
return ret;
}
@@ -1875,8 +1864,7 @@ static int sev_receive_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
struct kvm_sev_receive_update_data params;
struct sev_data_receive_update_data data;
void *hdr = NULL, *trans = NULL;
struct page **guest_page;
unsigned long n;
struct page *guest_page;
int ret, offset;
if (!sev_guest(kvm))
@@ -1913,8 +1901,7 @@ static int sev_receive_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
data.trans_len = params.trans_len;
/* Pin guest memory */
guest_page = sev_pin_memory(kvm, params.guest_uaddr & PAGE_MASK,
PAGE_SIZE, &n, FOLL_WRITE);
guest_page = sev_pin_page(kvm, params.guest_uaddr & PAGE_MASK, FOLL_WRITE);
if (IS_ERR(guest_page)) {
ret = PTR_ERR(guest_page);
goto e_free_trans;
@@ -1925,10 +1912,10 @@ static int sev_receive_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
* encrypts the written data with the guest's key, and the cache may
* contain dirty, unencrypted data.
*/
sev_clflush_pages(guest_page, n);
sev_clflush_pages(&guest_page, 1);
/* The RECEIVE_UPDATE_DATA command requires C-bit to be always set. */
data.guest_address = (page_to_pfn(guest_page[0]) << PAGE_SHIFT) + offset;
data.guest_address = page_to_phys(guest_page) + offset;
data.guest_address |= sev_me_mask;
data.guest_len = params.guest_len;
data.handle = to_kvm_sev_info(kvm)->handle;
@@ -1936,7 +1923,7 @@ static int sev_receive_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
ret = sev_issue_cmd(kvm, SEV_CMD_RECEIVE_UPDATE_DATA, &data,
&argp->error);
sev_unpin_memory(kvm, guest_page, n);
sev_unpin_page(kvm, guest_page);
e_free_trans:
kfree(trans);
@@ -2360,8 +2347,8 @@ static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
memcpy(dst_vaddr, src_vaddr, PAGE_SIZE);
kunmap_local(src_vaddr);
kunmap_local(dst_vaddr);
kunmap_local(src_vaddr);
}
ret = rmp_make_private(pfn, gfn << PAGE_SHIFT, PG_LEVEL_4K,
@@ -2395,9 +2382,10 @@ static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
void *dst_vaddr = kmap_local_pfn(pfn);
memcpy(src_vaddr, dst_vaddr, PAGE_SIZE);
set_page_dirty(src_page);
kunmap_local(src_vaddr);
kunmap_local(dst_vaddr);
kunmap_local(src_vaddr);
}
out:
@@ -2469,6 +2457,7 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp)
sev_populate_args.type = params.type;
count = kvm_gmem_populate(kvm, params.gfn_start, src, npages,
params.type == KVM_SEV_SNP_PAGE_TYPE_CPUID,
sev_gmem_post_populate, &sev_populate_args);
if (count < 0) {
argp->error = sev_populate_args.fw_error;
@@ -2689,10 +2678,8 @@ int sev_mem_enc_ioctl(struct kvm *kvm, void __user *argp)
r = sev_guest_status(kvm, &sev_cmd);
break;
case KVM_SEV_DBG_DECRYPT:
r = sev_dbg_crypt(kvm, &sev_cmd, true);
break;
case KVM_SEV_DBG_ENCRYPT:
r = sev_dbg_crypt(kvm, &sev_cmd, false);
r = sev_dbg_crypt(kvm, &sev_cmd, sev_cmd.id);
break;
case KVM_SEV_LAUNCH_SECRET:
r = sev_launch_secret(kvm, &sev_cmd);
@@ -3013,18 +3000,14 @@ void sev_vm_destroy(struct kvm *kvm)
void __init sev_set_cpu_caps(void)
{
if (sev_enabled) {
if (sev_enabled)
kvm_cpu_cap_set(X86_FEATURE_SEV);
kvm_caps.supported_vm_types |= BIT(KVM_X86_SEV_VM);
}
if (sev_es_enabled) {
if (sev_es_enabled)
kvm_cpu_cap_set(X86_FEATURE_SEV_ES);
kvm_caps.supported_vm_types |= BIT(KVM_X86_SEV_ES_VM);
}
if (sev_snp_enabled) {
if (sev_snp_enabled)
kvm_cpu_cap_set(X86_FEATURE_SEV_SNP);
kvm_caps.supported_vm_types |= BIT(KVM_X86_SNP_VM);
}
}
static bool is_sev_snp_initialized(void)
@@ -3054,6 +3037,11 @@ static bool is_sev_snp_initialized(void)
return initialized;
}
static const char * __init sev_str_feature_state(bool is_supported, bool is_usable)
{
return is_supported ? is_usable ? "enabled" : "unusable" : "disabled";
}
void __init sev_hardware_setup(void)
{
unsigned int eax, ebx, ecx, edx, sev_asid_count, sev_es_asid_count;
@@ -3061,6 +3049,7 @@ void __init sev_hardware_setup(void)
bool sev_snp_supported = false;
bool sev_es_supported = false;
bool sev_supported = false;
u32 vm_types = 0;
if (!sev_enabled || !npt_enabled || !nrips)
goto out;
@@ -3194,21 +3183,27 @@ void __init sev_hardware_setup(void)
}
}
if (sev_supported && min_sev_asid <= max_sev_asid)
vm_types |= BIT(KVM_X86_SEV_VM);
if (sev_es_supported && min_sev_es_asid <= max_sev_es_asid)
vm_types |= BIT(KVM_X86_SEV_ES_VM);
if (sev_snp_supported)
vm_types |= BIT(KVM_X86_SNP_VM);
vm_types &= sev_firmware_supported_vm_types();
kvm_caps.supported_vm_types |= vm_types;
if (boot_cpu_has(X86_FEATURE_SEV))
pr_info("SEV %s (ASIDs %u - %u)\n",
sev_supported ? min_sev_asid <= max_sev_asid ? "enabled" :
"unusable" :
"disabled",
sev_str_feature_state(sev_supported, vm_types & BIT(KVM_X86_SEV_VM)),
min_sev_asid, max_sev_asid);
if (boot_cpu_has(X86_FEATURE_SEV_ES))
pr_info("SEV-ES %s (ASIDs %u - %u)\n",
sev_es_supported ? min_sev_es_asid <= max_sev_es_asid ? "enabled" :
"unusable" :
"disabled",
sev_str_feature_state(sev_es_supported, vm_types & BIT(KVM_X86_SEV_ES_VM)),
min_sev_es_asid, max_sev_es_asid);
if (boot_cpu_has(X86_FEATURE_SEV_SNP))
pr_info("SEV-SNP %s (ASIDs %u - %u)\n",
str_enabled_disabled(sev_snp_supported),
sev_str_feature_state(sev_snp_supported, vm_types & BIT(KVM_X86_SNP_VM)),
min_snp_asid, max_snp_asid);
sev_enabled = sev_supported;

2
arch/x86/kvm/vmx/tdx.c
View File

@@ -3261,7 +3261,7 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c
};
gmem_ret = kvm_gmem_populate(kvm, gpa_to_gfn(region.gpa),
u64_to_user_ptr(region.source_addr),
1, tdx_gmem_post_populate, &arg);
1, false, tdx_gmem_post_populate, &arg);
if (gmem_ret < 0) {
ret = gmem_ret;
break;

101
drivers/crypto/ccp/sev-dev.c
View File

@@ -2381,7 +2381,8 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable)
return ret;
}
static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp)
static int __sev_do_snp_platform_status(struct sev_user_data_snp_status *status,
int *error)
{
struct sev_device *sev = psp_master->sev_data;
struct sev_data_snp_addr buf;
@@ -2389,9 +2390,6 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp)
void *data;
int ret;
if (!argp->data)
return -EINVAL;
status_page = alloc_page(GFP_KERNEL_ACCOUNT);
if (!status_page)
return -ENOMEM;
@@ -2414,7 +2412,7 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp)
}
buf.address = __psp_pa(data);
ret = __sev_do_cmd_locked(SEV_CMD_SNP_PLATFORM_STATUS, &buf, &argp->error);
ret = __sev_do_cmd_locked(SEV_CMD_SNP_PLATFORM_STATUS, &buf, error);
if (sev->snp_initialized) {
/*
@@ -2429,15 +2427,32 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp)
if (ret)
goto cleanup;
if (copy_to_user((void __user *)argp->data, data,
sizeof(struct sev_user_data_snp_status)))
ret = -EFAULT;
memcpy(status, data, sizeof(*status));
cleanup:
__free_pages(status_page, 0);
return ret;
}
static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp)
{
struct sev_user_data_snp_status status;
int ret;
if (!argp->data)
return -EINVAL;
ret = __sev_do_snp_platform_status(&status, &argp->error);
if (ret < 0)
return ret;
if (copy_to_user((void __user *)argp->data, &status,
sizeof(struct sev_user_data_snp_status)))
ret = -EFAULT;
return ret;
}
static int sev_ioctl_do_snp_commit(struct sev_issue_cmd *argp)
{
struct sev_device *sev = psp_master->sev_data;
@@ -2939,3 +2954,73 @@ void sev_pci_exit(void)
sev_firmware_shutdown(sev);
}
static int get_v1_svn(struct sev_device *sev)
{
struct sev_snp_tcb_version_genoa_milan *tcb;
struct sev_user_data_snp_status status;
int ret, error = 0;
mutex_lock(&sev_cmd_mutex);
ret = __sev_do_snp_platform_status(&status, &error);
mutex_unlock(&sev_cmd_mutex);
if (ret < 0)
return ret;
tcb = (struct sev_snp_tcb_version_genoa_milan *)&status
.current_tcb_version;
return tcb->snp;
}
static int get_v2_svn(struct sev_device *sev)
{
struct sev_user_data_snp_status status;
struct sev_snp_tcb_version_turin *tcb;
int ret, error = 0;
mutex_lock(&sev_cmd_mutex);
ret = __sev_do_snp_platform_status(&status, &error);
mutex_unlock(&sev_cmd_mutex);
if (ret < 0)
return ret;
tcb = (struct sev_snp_tcb_version_turin *)&status
.current_tcb_version;
return tcb->snp;
}
static bool sev_firmware_allows_es(struct sev_device *sev)
{
/* Documented in AMD-SB-3023 */
if (boot_cpu_has(X86_FEATURE_ZEN4) || boot_cpu_has(X86_FEATURE_ZEN3))
return get_v1_svn(sev) < 0x1b;
else if (boot_cpu_has(X86_FEATURE_ZEN5))
return get_v2_svn(sev) < 0x4;
else
return true;
}
int sev_firmware_supported_vm_types(void)
{
int supported_vm_types = 0;
struct sev_device *sev;
if (!psp_master || !psp_master->sev_data)
return supported_vm_types;
sev = psp_master->sev_data;
supported_vm_types |= BIT(KVM_X86_SEV_VM);
supported_vm_types |= BIT(KVM_X86_SEV_ES_VM);
if (!sev->snp_initialized)
return supported_vm_types;
supported_vm_types |= BIT(KVM_X86_SNP_VM);
if (!sev_firmware_allows_es(sev))
supported_vm_types &= ~BIT(KVM_X86_SEV_ES_VM);
return supported_vm_types;
}
EXPORT_SYMBOL_FOR_MODULES(sev_firmware_supported_vm_types, "kvm-amd");

3
include/linux/kvm_host.h
View File

@@ -2596,7 +2596,8 @@ int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_ord
typedef int (*kvm_gmem_populate_cb)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
struct page *page, void *opaque);
long kvm_gmem_populate(struct kvm *kvm, gfn_t gfn, void __user *src, long npages,
long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src,
long npages, bool may_writeback_src,
kvm_gmem_populate_cb post_populate, void *opaque);
#endif

37
include/linux/psp-sev.h
View File

@@ -902,6 +902,42 @@ struct snp_feature_info {
/* Feature bits in EBX */
#define SNP_SEV_TIO_SUPPORTED BIT(1)
/**
* struct sev_snp_tcb_version_genoa_milan
*
* @boot_loader: SVN of PSP bootloader
* @tee: SVN of PSP operating system
* @reserved: reserved
* @snp: SVN of SNP firmware
* @microcode: Lowest current patch level of all cores
*/
struct sev_snp_tcb_version_genoa_milan {
u8 boot_loader;
u8 tee;
u8 reserved[4];
u8 snp;
u8 microcode;
};
/**
* struct sev_snp_tcb_version_turin
*
* @fmc: SVN of FMC firmware
* @boot_loader: SVN of PSP bootloader
* @tee: SVN of PSP operating system
* @snp: SVN of SNP firmware
* @reserved: reserved
* @microcode: Lowest current patch level of all cores
*/
struct sev_snp_tcb_version_turin {
u8 fmc;
u8 boot_loader;
u8 tee;
u8 snp;
u8 reserved[3];
u8 microcode;
};
#ifdef CONFIG_CRYPTO_DEV_SP_PSP
/**
@@ -1048,6 +1084,7 @@ void snp_free_firmware_page(void *addr);
void sev_platform_shutdown(void);
bool sev_is_snp_ciphertext_hiding_supported(void);
u64 sev_get_snp_policy_bits(void);
int sev_firmware_supported_vm_types(void);
#else /* !CONFIG_CRYPTO_DEV_SP_PSP */

1
tools/testing/selftests/kvm/Makefile.kvm
View File

@@ -141,6 +141,7 @@ TEST_GEN_PROGS_x86 += x86/tsc_msrs_test
TEST_GEN_PROGS_x86 += x86/vmx_pmu_caps_test
TEST_GEN_PROGS_x86 += x86/xen_shinfo_test
TEST_GEN_PROGS_x86 += x86/xen_vmcall_test
TEST_GEN_PROGS_x86 += x86/sev_dbg_test
TEST_GEN_PROGS_x86 += x86/sev_init2_tests
TEST_GEN_PROGS_x86 += x86/sev_migrate_tests
TEST_GEN_PROGS_x86 += x86/sev_smoke_test

24
tools/testing/selftests/kvm/include/x86/sev.h
View File

@@ -144,4 +144,28 @@ static inline void snp_launch_update_data(struct kvm_vm *vm, gpa_t gpa,
vm_sev_ioctl(vm, KVM_SEV_SNP_LAUNCH_UPDATE, &update_data);
}
static inline void sev_dbg_crypt_memory(struct kvm_vm *vm, unsigned int cmd,
void *dst, void *src, unsigned int len)
{
struct kvm_sev_dbg dbg = {
.src_uaddr = (unsigned long)src,
.dst_uaddr = (unsigned long)dst,
.len = len,
};
vm_sev_ioctl(vm, cmd, &dbg);
}
static inline void sev_decrypt_memory(struct kvm_vm *vm, void *dst, void *src,
unsigned int len)
{
sev_dbg_crypt_memory(vm, KVM_SEV_DBG_DECRYPT, dst, src, len);
}
static inline void sev_encrypt_memory(struct kvm_vm *vm, void *dst, void *src,
unsigned int len)
{
sev_dbg_crypt_memory(vm, KVM_SEV_DBG_ENCRYPT, dst, src, len);
}
#endif /* SELFTEST_KVM_SEV_H */

118
tools/testing/selftests/kvm/x86/sev_dbg_test.c Normal file
View File

@@ -0,0 +1,118 @@
// SPDX-License-Identifier: GPL-2.0-only
#include <fcntl.h>
#include <string.h>
#include <sys/ioctl.h>
#include "test_util.h"
#include "kvm_util.h"
#include "processor.h"
#include "sev.h"
#define BUFFER_SIZE (PAGE_SIZE * 2)
static u8 *data;
static u8 src[BUFFER_SIZE] __aligned(PAGE_SIZE);
static u8 dst[BUFFER_SIZE] __aligned(PAGE_SIZE);
static void validate_dst(int i, int nr_bytes, u8 pattern)
{
for ( ; i < nr_bytes; i++)
TEST_ASSERT(dst[i] == pattern,
"Expected 0x%x at byte %u, got 0x%x",
pattern, i, dst[i]);
}
static void validate_buffers(void)
{
int i;
for (i = 0; i < BUFFER_SIZE; i++)
TEST_ASSERT(src[i] == dst[i],
"Expected src[%u] (0x%x) == dst[%u] (0x%x)",
i, src[i], i, dst[i]);
}
static void ____test_sev_dbg(struct kvm_vm *vm, int i, int j, int nr_bytes)
{
u8 pattern = guest_random_u32(&guest_rng);
if (i + nr_bytes > BUFFER_SIZE || j + nr_bytes > BUFFER_SIZE)
return;
memset(&src[i], pattern, nr_bytes);
sev_encrypt_memory(vm, &data[j], &src[i], nr_bytes);
sev_decrypt_memory(vm, &dst[i], &data[j], nr_bytes);
validate_buffers();
validate_dst(i, nr_bytes, pattern);
}
static void __test_sev_dbg(struct kvm_vm *vm, int nr_bytes)
{
/*
* In a perfect world, all sizes at all combinations within the buffers
* would be tested. In reality, even this much testing is quite slow.
* Target sizes and offsets around the chunk (16 bytes) and page (4096
* bytes) sizes.
*/
int x[] = { 1, 8, 15, 16, 23 };
int p = PAGE_SIZE - 24;
int i, j;
____test_sev_dbg(vm, 0, 0, nr_bytes);
for (i = 0; i < ARRAY_SIZE(x); i++) {
for (j = 0; j < ARRAY_SIZE(x); j++) {
____test_sev_dbg(vm, x[i], x[j], nr_bytes);
____test_sev_dbg(vm, x[i], p + x[j], nr_bytes);
____test_sev_dbg(vm, p + x[i], x[j], nr_bytes);
____test_sev_dbg(vm, p + x[i], p + x[j], nr_bytes);
}
}
}
static void test_sev_dbg(u32 type, u64 policy)
{
int sizes[] = { 1, 8, 15, 16, 17, 32, 33 };
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
int i;
if (!(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(type)))
return;
vm = vm_sev_create_with_one_vcpu(type, NULL, &vcpu);
data = addr_gva2hva(vm, vm_alloc(vm, BUFFER_SIZE, KVM_UTIL_MIN_VADDR));
memset(data, 0xaa, BUFFER_SIZE);
vm_sev_launch(vm, policy, NULL);
sev_decrypt_memory(vm, dst, data, BUFFER_SIZE);
validate_dst(0, BUFFER_SIZE, 0xaa);
memset(src, 0x55, BUFFER_SIZE);
sev_encrypt_memory(vm, data, src, BUFFER_SIZE);
sev_decrypt_memory(vm, dst, data, BUFFER_SIZE);
validate_dst(0, BUFFER_SIZE, 0x55);
__test_sev_dbg(vm, PAGE_SIZE);
for (i = 0; i < ARRAY_SIZE(sizes); i++) {
__test_sev_dbg(vm, sizes[i]);
__test_sev_dbg(vm, PAGE_SIZE - sizes[i]);
__test_sev_dbg(vm, PAGE_SIZE + sizes[i]);
__test_sev_dbg(vm, BUFFER_SIZE - sizes[i]);
}
kvm_vm_free(vm);
}
int main(int argc, char *argv[])
{
TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SEV));
/* Note, KVM doesn't support {de,en}crypt commands for SNP. */
test_sev_dbg(KVM_X86_SEV_VM, 0);
test_sev_dbg(KVM_X86_SEV_ES_VM, SEV_POLICY_ES);
return 0;
}

14
tools/testing/selftests/kvm/x86/sev_init2_tests.c
View File

@@ -136,16 +136,14 @@ int main(int argc, char *argv[])
kvm_check_cap(KVM_CAP_VM_TYPES), 1 << KVM_X86_SEV_VM);
TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SEV_VM));
have_sev_es = kvm_cpu_has(X86_FEATURE_SEV_ES);
have_sev_es = kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SEV_ES_VM);
TEST_ASSERT(have_sev_es == !!(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SEV_ES_VM)),
"sev-es: KVM_CAP_VM_TYPES (%x) does not match cpuid (checking %x)",
kvm_check_cap(KVM_CAP_VM_TYPES), 1 << KVM_X86_SEV_ES_VM);
TEST_ASSERT(!have_sev_es || kvm_cpu_has(X86_FEATURE_SEV_ES),
"sev-es: SEV_ES_VM supported without SEV_ES in CPUID");
have_snp = kvm_cpu_has(X86_FEATURE_SEV_SNP);
TEST_ASSERT(have_snp == !!(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SNP_VM)),
"sev-snp: KVM_CAP_VM_TYPES (%x) indicates SNP support (bit %d), but CPUID does not",
kvm_check_cap(KVM_CAP_VM_TYPES), KVM_X86_SNP_VM);
have_snp = kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SNP_VM);
TEST_ASSERT(!have_snp || kvm_cpu_has(X86_FEATURE_SEV_SNP),
"sev-snp: SNP_VM supported without SEV_SNP in CPUID");
test_vm_types();

2
tools/testing/selftests/kvm/x86/sev_migrate_tests.c
View File

@@ -374,7 +374,7 @@ int main(int argc, char *argv[])
TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SEV));
have_sev_es = kvm_cpu_has(X86_FEATURE_SEV_ES);
have_sev_es = kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SEV_ES_VM);
if (kvm_has_cap(KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM)) {
test_sev_migrate_from(/* es= */ false);

4
tools/testing/selftests/kvm/x86/sev_smoke_test.c
View File

@@ -249,10 +249,10 @@ int main(int argc, char *argv[])
test_sev_smoke(guest_sev_code, KVM_X86_SEV_VM, 0);
if (kvm_cpu_has(X86_FEATURE_SEV_ES))
if (kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SEV_ES_VM))
test_sev_smoke(guest_sev_es_code, KVM_X86_SEV_ES_VM, SEV_POLICY_ES);
if (kvm_cpu_has(X86_FEATURE_SEV_SNP))
if (kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SNP_VM))
test_sev_smoke(guest_snp_code, KVM_X86_SNP_VM, snp_default_policy());
return 0;

6
virt/kvm/guest_memfd.c
View File

@@ -860,7 +860,8 @@ static long __kvm_gmem_populate(struct kvm *kvm, struct kvm_memory_slot *slot,
return ret;
}
long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long npages,
long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src,
long npages, bool may_writeback_src,
kvm_gmem_populate_cb post_populate, void *opaque)
{
struct kvm_memory_slot *slot;
@@ -894,8 +895,9 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long
if (src) {
unsigned long uaddr = (unsigned long)src + i * PAGE_SIZE;
unsigned int flags = may_writeback_src ? FOLL_WRITE : 0;
ret = get_user_pages_fast(uaddr, 1, 0, &src_page);
ret = get_user_pages_fast(uaddr, 1, flags, &src_page);
if (ret < 0)
break;
if (ret != 1) {