From: Peter Gonda pgonda@google.com

SEV-ES guests need additional pages allocated for their GHCBs. Add arch specific function definition with __weak to allow for overriding for X86 specific SEV-ES functionality.

Cc: Vishal Annapurve vannapurve@google.com Cc: Ackerley Tng ackerleytng@google.com Cc: Paolo Bonzini pbonzini@redhat.com Cc: Claudio Imbrenda imbrenda@linux.ibm.com Cc: Sean Christopherson seanjc@google.com Cc: Carlos Bilbao carlos.bilbao@amd.com Cc: Tom Lendacky thomas.lendacky@amd.com Cc: Michael Roth michael.roth@amd.com Cc: kvm@vger.kernel.org Cc: linux-kselftest@vger.kernel.org Signed-off-by: Peter Gonda pgonda@google.com Signed-off-by: Neeraj Upadhyay Neeraj.Upadhyay@amd.com --- tools/testing/selftests/kvm/include/kvm_util.h | 2 ++ tools/testing/selftests/kvm/lib/kvm_util.c | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 4c4e5a847f67..b621a0f1907e 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -254,6 +254,8 @@ int get_kvm_param_integer(const char *param); int get_kvm_intel_param_integer(const char *param); int get_kvm_amd_param_integer(const char *param);

+int kvm_arch_vm_additional_pages_required(struct vm_shape shape, + uint64_t page_size); unsigned int kvm_check_cap(long cap);

static inline bool kvm_has_cap(long cap) diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 089488e2eaf6..18a44d8220ef 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -367,11 +367,11 @@ struct kvm_vm *____vm_create(struct vm_shape shape) return vm; }

-static uint64_t vm_nr_pages_required(enum vm_guest_mode mode, +static uint64_t vm_nr_pages_required(struct vm_shape shape, uint32_t nr_runnable_vcpus, uint64_t extra_mem_pages) { - uint64_t page_size = vm_guest_mode_params[mode].page_size; + uint64_t page_size = vm_guest_mode_params[shape.mode].page_size; uint64_t nr_pages;

TEST_ASSERT(nr_runnable_vcpus, @@ -403,13 +403,15 @@ static uint64_t vm_nr_pages_required(enum vm_guest_mode mode, /* Account for the number of pages needed by ucall. */ nr_pages += ucall_nr_pages_required(page_size);

- return vm_adjust_num_guest_pages(mode, nr_pages); + nr_pages += kvm_arch_vm_additional_pages_required(shape, page_size); + + return vm_adjust_num_guest_pages(shape.mode, nr_pages); }

struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus, uint64_t nr_extra_pages) { - uint64_t nr_pages = vm_nr_pages_required(shape.mode, nr_runnable_vcpus, + uint64_t nr_pages = vm_nr_pages_required(shape, nr_runnable_vcpus, nr_extra_pages); struct userspace_mem_region *slot0; struct kvm_vm *vm; @@ -2247,6 +2249,12 @@ __weak void kvm_arch_vm_post_create(struct kvm_vm *vm) { }

+__weak int kvm_arch_vm_additional_pages_required(struct vm_shape shape, + uint64_t page_size) +{ + return 0; +} + __weak void kvm_selftest_arch_init(void) { }

From: Peter Gonda pgonda@google.com

Add GHCB management functionality similar to the ucall management. Allows for selftest vCPUs to acquire GHCBs for their usage.

Cc: Vishal Annapurve vannapurve@google.com Cc: Ackerley Tng ackerleytng@google.com Cc: Paolo Bonzini pbonzini@redhat.com Cc: Claudio Imbrenda imbrenda@linux.ibm.com Cc: Sean Christopherson seanjc@google.com Cc: Carlos Bilbao carlos.bilbao@amd.com Cc: Tom Lendacky thomas.lendacky@amd.com Cc: Michael Roth michael.roth@amd.com Cc: kvm@vger.kernel.org Cc: linux-kselftest@vger.kernel.org Signed-off-by: Peter Gonda pgonda@google.com Signed-off-by: Neeraj Upadhyay Neeraj.Upadhyay@amd.com --- tools/testing/selftests/kvm/include/x86/sev.h | 1 + .../testing/selftests/kvm/lib/x86/processor.c | 9 +++ tools/testing/selftests/kvm/lib/x86/sev.c | 78 +++++++++++++++++++ 3 files changed, 88 insertions(+)

diff --git a/tools/testing/selftests/kvm/include/x86/sev.h b/tools/testing/selftests/kvm/include/x86/sev.h index fd5d5261e10e..0b4411847cbf 100644 --- a/tools/testing/selftests/kvm/include/x86/sev.h +++ b/tools/testing/selftests/kvm/include/x86/sev.h @@ -43,6 +43,7 @@ enum sev_guest_state { bool is_sev_vm(struct kvm_vm *vm); bool is_sev_es_vm(struct kvm_vm *vm); bool is_sev_snp_vm(struct kvm_vm *vm); +int ghcb_nr_pages_required(uint64_t page_size);

void sev_vm_launch(struct kvm_vm *vm, uint32_t policy); void sev_vm_launch_measure(struct kvm_vm *vm, uint8_t *measurement); diff --git a/tools/testing/selftests/kvm/lib/x86/processor.c b/tools/testing/selftests/kvm/lib/x86/processor.c index a92dc1dad085..7129dfb652c4 100644 --- a/tools/testing/selftests/kvm/lib/x86/processor.c +++ b/tools/testing/selftests/kvm/lib/x86/processor.c @@ -651,6 +651,15 @@ void kvm_arch_vm_post_create(struct kvm_vm *vm) sync_global_to_guest(vm, guest_tsc_khz); }

+int kvm_arch_vm_additional_pages_required(struct vm_shape shape, uint64_t page_size) +{ + if (shape.type == KVM_X86_SEV_ES_VM || + shape.type == KVM_X86_SNP_VM) + return ghcb_nr_pages_required(page_size); + + return 0; +} + void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code) { struct kvm_regs regs; diff --git a/tools/testing/selftests/kvm/lib/x86/sev.c b/tools/testing/selftests/kvm/lib/x86/sev.c index 17d493e9907a..dd7ccf0324c5 100644 --- a/tools/testing/selftests/kvm/lib/x86/sev.c +++ b/tools/testing/selftests/kvm/lib/x86/sev.c @@ -3,6 +3,80 @@ #include <stdbool.h>

#include "sev.h" +#include "linux/bitmap.h" +#include "svm.h" +#include "svm_util.h" + +struct ghcb_entry { + struct ghcb ghcb; + + /* Guest physical address of this GHCB. */ + void *gpa; + + /* Host virtual address of this struct. */ + struct ghcb_entry *hva; +}; + +struct ghcb_header { + struct ghcb_entry ghcbs[KVM_MAX_VCPUS]; + DECLARE_BITMAP(in_use, KVM_MAX_VCPUS); +}; + +static struct ghcb_header *ghcb_pool; + +int ghcb_nr_pages_required(uint64_t page_size) +{ + return align_up(sizeof(struct ghcb_header), page_size) / page_size; +} + +void ghcb_init(struct kvm_vm *vm) +{ + struct ghcb_header *hdr; + struct ghcb_entry *entry; + vm_vaddr_t vaddr; + int i; + + vaddr = vm_vaddr_alloc_shared(vm, sizeof(*hdr), KVM_UTIL_MIN_VADDR, + MEM_REGION_DATA); + hdr = (struct ghcb_header *)addr_gva2hva(vm, vaddr); + memset(hdr, 0, sizeof(*hdr)); + + for (i = 0; i < KVM_MAX_VCPUS; ++i) { + entry = &hdr->ghcbs[i]; + entry->hva = entry; + entry->gpa = (void *)addr_hva2gpa(vm, &entry->ghcb); + } + + write_guest_global(vm, ghcb_pool, (struct ghcb_header *)vaddr); +} + +static struct ghcb_entry *ghcb_alloc(void) +{ + return &ghcb_pool->ghcbs[0]; + struct ghcb_entry *entry; + int i; + + if (!ghcb_pool) + goto ucall_failed; + + for (i = 0; i < KVM_MAX_VCPUS; ++i) { + if (!test_and_set_bit(i, ghcb_pool->in_use)) { + entry = &ghcb_pool->ghcbs[i]; + memset(&entry->ghcb, 0, sizeof(entry->ghcb)); + return entry; + } + } + +ucall_failed: + return NULL; +} + +static void ghcb_free(struct ghcb_entry *entry) +{ + /* Beware, here be pointer arithmetic. */ + clear_bit(entry - ghcb_pool->ghcbs, ghcb_pool->in_use); +} +

bool is_sev_snp_vm(struct kvm_vm *vm) { @@ -117,7 +191,11 @@ void sev_vm_launch(struct kvm_vm *vm, uint32_t policy) struct kvm_sev_guest_status status; int ctr;

+ if (is_sev_es_vm(vm)) + ghcb_init(vm); + vm_sev_ioctl(vm, KVM_SEV_LAUNCH_START, &launch_start); + vm_sev_ioctl(vm, KVM_SEV_GUEST_STATUS, &status);

TEST_ASSERT_EQ(status.policy, policy);

From: Peter Gonda pgonda@google.com

Modifies ucall handling for SEV-ES VMs. Instead of using an out instruction and storing the ucall pointer in RDI, SEV-ES guests use a outsb VMGEXIT to move the ucall pointer as the data. Allows for SEV-ES to use ucalls instead of relying the SEV-ES MSR based termination protocol.

Cc: Vishal Annapurve vannapurve@google.com Cc: Ackerley Tng ackerleytng@google.com Cc: Paolo Bonzini pbonzini@redhat.com Cc: Claudio Imbrenda imbrenda@linux.ibm.com Cc: Sean Christopherson seanjc@google.com Cc: Carlos Bilbao carlos.bilbao@amd.com Cc: Tom Lendacky thomas.lendacky@amd.com Cc: Michael Roth michael.roth@amd.com Cc: kvm@vger.kernel.org Cc: linux-kselftest@vger.kernel.org Signed-off-by: Peter Gonda pgonda@google.com Signed-off-by: Neeraj Upadhyay Neeraj.Upadhyay@amd.com --- tools/testing/selftests/kvm/include/x86/sev.h | 2 + tools/testing/selftests/kvm/lib/x86/sev.c | 98 +++++++++++++++++-- tools/testing/selftests/kvm/lib/x86/ucall.c | 18 ++++ .../selftests/kvm/x86/sev_smoke_test.c | 27 +---- 4 files changed, 113 insertions(+), 32 deletions(-)

diff --git a/tools/testing/selftests/kvm/include/x86/sev.h b/tools/testing/selftests/kvm/include/x86/sev.h index 437e397ddd29..bd6ab3f38679 100644 --- a/tools/testing/selftests/kvm/include/x86/sev.h +++ b/tools/testing/selftests/kvm/include/x86/sev.h @@ -151,4 +151,6 @@ bool is_sev_enabled(void); bool is_sev_es_enabled(void); bool is_sev_snp_enabled(void);

+void sev_es_ucall_port_write(uint32_t port, uint64_t data); + #endif /* SELFTEST_KVM_SEV_H */ diff --git a/tools/testing/selftests/kvm/lib/x86/sev.c b/tools/testing/selftests/kvm/lib/x86/sev.c index 0c542eae4184..425ec8a3a3c7 100644 --- a/tools/testing/selftests/kvm/lib/x86/sev.c +++ b/tools/testing/selftests/kvm/lib/x86/sev.c @@ -7,11 +7,18 @@ #include "svm.h" #include "svm_util.h"

+#define IOIO_TYPE_STR (1 << 2) +#define IOIO_SEG_DS (1 << 11 | 1 << 10) +#define IOIO_DATA_8 (1 << 4) +#define IOIO_REP (1 << 3) + +#define SW_EXIT_CODE_IOIO 0x7b + struct ghcb_entry { struct ghcb ghcb;

/* Guest physical address of this GHCB. */ - void *gpa; + uint64_t gpa;

/* Host virtual address of this struct. */ struct ghcb_entry *hva; @@ -35,25 +42,35 @@ void ghcb_init(struct kvm_vm *vm) struct ghcb_entry *entry; vm_vaddr_t vaddr; int i; + size_t sz = align_up(sizeof(struct ghcb_header), vm_guest_mode_params[vm->mode].page_size);

- vaddr = vm_vaddr_alloc_shared(vm, sizeof(*hdr), KVM_UTIL_MIN_VADDR, + vaddr = vm_vaddr_alloc_shared(vm, sz, KVM_UTIL_MIN_VADDR, MEM_REGION_DATA); hdr = (struct ghcb_header *)addr_gva2hva(vm, vaddr); - memset(hdr, 0, sizeof(*hdr)); + memset(hdr, 0, sz);

for (i = 0; i < KVM_MAX_VCPUS; ++i) { entry = &hdr->ghcbs[i]; entry->hva = entry; - entry->gpa = (void *)addr_hva2gpa(vm, &entry->ghcb); + entry->gpa = (uint64_t)addr_hva2gpa(vm, &entry->ghcb); }

+ if (is_sev_snp_vm(vm)) + vm_mem_set_shared(vm, addr_hva2gpa(vm, hdr), sz); + write_guest_global(vm, ghcb_pool, (struct ghcb_header *)vaddr); }

+static void sev_es_terminate(void) +{ + wrmsr(MSR_AMD64_SEV_ES_GHCB, GHCB_MSR_TERM_REQ); +} + static struct ghcb_entry *ghcb_alloc(void) { return &ghcb_pool->ghcbs[0]; struct ghcb_entry *entry; + struct ghcb *ghcb; int i;

if (!ghcb_pool) @@ -62,12 +79,18 @@ static struct ghcb_entry *ghcb_alloc(void) for (i = 0; i < KVM_MAX_VCPUS; ++i) { if (!test_and_set_bit(i, ghcb_pool->in_use)) { entry = &ghcb_pool->ghcbs[i]; - memset(&entry->ghcb, 0, sizeof(entry->ghcb)); + ghcb = &entry->ghcb; + + memset(&ghcb, 0, sizeof(*ghcb)); + ghcb->ghcb_usage = 0; + ghcb->protocol_version = 1; + return entry; } }

ucall_failed: + sev_es_terminate(); return NULL; }

@@ -191,9 +214,6 @@ void sev_vm_launch(struct kvm_vm *vm, uint32_t policy) struct kvm_sev_guest_status status; int ctr;

- if (is_sev_es_vm(vm)) - ghcb_init(vm); - vm_sev_ioctl(vm, KVM_SEV_LAUNCH_START, &launch_start);

vm_sev_ioctl(vm, KVM_SEV_GUEST_STATUS, &status); @@ -285,6 +305,9 @@ struct kvm_vm *vm_sev_create_with_one_vcpu(uint32_t type, void *guest_code,

void vm_sev_launch(struct kvm_vm *vm, uint64_t policy, uint8_t *measurement) { + if (is_sev_es_vm(vm)) + ghcb_init(vm); + if (is_sev_snp_vm(vm)) { vm_enable_cap(vm, KVM_CAP_EXIT_HYPERCALL, (1 << KVM_HC_MAP_GPA_RANGE));

@@ -323,3 +346,62 @@ bool is_sev_snp_enabled(void) return is_sev_es_enabled() && rdmsr(MSR_AMD64_SEV) & MSR_AMD64_SEV_SNP_ENABLED; } + +static uint64_t setup_exitinfo1_portio(uint32_t port) +{ + uint64_t exitinfo1 = 0; + + exitinfo1 |= IOIO_TYPE_STR; + exitinfo1 |= ((port & 0xffff) << 16); + exitinfo1 |= IOIO_SEG_DS; + exitinfo1 |= IOIO_DATA_8; + exitinfo1 |= IOIO_REP; + + return exitinfo1; +} + +#define GHCB_MSR_REG_GPA_REQ 0x012 +#define GHCB_MSR_REG_GPA_REQ_VAL(v) \ + /* GHCBData[63:12] */ \ + (((u64)((v) & GENMASK_ULL(51, 0)) << 12) | \ + /* GHCBData[11:0] */ \ + GHCB_MSR_REG_GPA_REQ) + +static void register_ghcb_page(uint64_t ghcb_gpa) +{ + if (is_sev_snp_enabled()) { + wrmsr(MSR_AMD64_SEV_ES_GHCB, GHCB_MSR_REG_GPA_REQ_VAL(ghcb_gpa >> 12)); + VMGEXIT(); + } +} + +static void do_vmg_exit(uint64_t ghcb_gpa) +{ + wrmsr(MSR_AMD64_SEV_ES_GHCB, ghcb_gpa); + VMGEXIT(); +} + +void sev_es_ucall_port_write(uint32_t port, uint64_t data) +{ + struct ghcb_entry *entry; + struct ghcb *ghcb; + const uint64_t exitinfo1 = setup_exitinfo1_portio(port); + + entry = ghcb_alloc(); + ghcb = &entry->ghcb; + + register_ghcb_page(entry->gpa); + + ghcb_set_sw_exit_code(ghcb, SW_EXIT_CODE_IOIO); + ghcb_set_sw_exit_info_1(ghcb, exitinfo1); + ghcb_set_sw_exit_info_2(ghcb, sizeof(data)); + + // Setup the SW Stratch buffer pointer. + ghcb_set_sw_scratch(ghcb, + entry->gpa + offsetof(struct ghcb, shared_buffer)); + memcpy(&ghcb->shared_buffer, &data, sizeof(data)); + + do_vmg_exit(entry->gpa); + + ghcb_free(entry); +} diff --git a/tools/testing/selftests/kvm/lib/x86/ucall.c b/tools/testing/selftests/kvm/lib/x86/ucall.c index 1265cecc7dd1..711e58a3a356 100644 --- a/tools/testing/selftests/kvm/lib/x86/ucall.c +++ b/tools/testing/selftests/kvm/lib/x86/ucall.c @@ -5,6 +5,8 @@ * Copyright (C) 2018, Red Hat, Inc. */ #include "kvm_util.h" +#include "processor.h" +#include "sev.h"

#define UCALL_PIO_PORT ((uint16_t)0x1000)

@@ -21,6 +23,11 @@ void ucall_arch_do_ucall(vm_vaddr_t uc) #define HORRIFIC_L2_UCALL_CLOBBER_HACK \ "rcx", "rsi", "r8", "r9", "r10", "r11"

+ if (is_sev_es_enabled()) { + sev_es_ucall_port_write(UCALL_PIO_PORT, uc); + return; + } + asm volatile("push %%rbp\n\t" "push %%r15\n\t" "push %%r14\n\t" @@ -48,8 +55,19 @@ void *ucall_arch_get_ucall(struct kvm_vcpu *vcpu)

if (run->exit_reason == KVM_EXIT_IO && run->io.port == UCALL_PIO_PORT) { struct kvm_regs regs; + uint64_t addr; + + if (is_sev_es_vm(vcpu->vm)) { + TEST_ASSERT( + run->io.count == 8 && run->io.size == 1, + "SEV-ES ucall exit requires 8 byte string out\n"); + + addr = *(uint64_t *)((uint8_t *)(run) + run->io.data_offset); + return (void *)addr; + }

vcpu_regs_get(vcpu, &regs); + return (void *)regs.rdi; } return NULL; diff --git a/tools/testing/selftests/kvm/x86/sev_smoke_test.c b/tools/testing/selftests/kvm/x86/sev_smoke_test.c index 29382dcab18c..3834d3664219 100644 --- a/tools/testing/selftests/kvm/x86/sev_smoke_test.c +++ b/tools/testing/selftests/kvm/x86/sev_smoke_test.c @@ -20,8 +20,7 @@ static void guest_snp_code(void) { GUEST_ASSERT(is_sev_snp_enabled());

- wrmsr(MSR_AMD64_SEV_ES_GHCB, GHCB_MSR_TERM_REQ); - VMGEXIT(); + GUEST_DONE(); }

static void guest_sev_es_code(void) @@ -29,12 +28,7 @@ static void guest_sev_es_code(void) /* TODO: Check CPUID after GHCB-based hypercall support is added. */ GUEST_ASSERT(is_sev_es_enabled());

- /* - * TODO: Add GHCB and ucall support for SEV-ES guests. For now, simply - * force "termination" to signal "done" via the GHCB MSR protocol. - */ - wrmsr(MSR_AMD64_SEV_ES_GHCB, GHCB_MSR_TERM_REQ); - VMGEXIT(); + GUEST_DONE(); }

static void guest_sev_code(void) @@ -102,12 +96,7 @@ static void __test_sync_vmsa(uint32_t type, uint64_t policy)

vcpu_run(vcpu);

- TEST_ASSERT(vcpu->run->exit_reason == KVM_EXIT_SYSTEM_EVENT, - "Wanted SYSTEM_EVENT, got %s", - exit_reason_str(vcpu->run->exit_reason)); - TEST_ASSERT_EQ(vcpu->run->system_event.type, KVM_SYSTEM_EVENT_SEV_TERM); - TEST_ASSERT_EQ(vcpu->run->system_event.ndata, 1); - TEST_ASSERT_EQ(vcpu->run->system_event.data[0], GHCB_MSR_TERM_REQ); + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);

compare_xsave((u8 *)&xsave, (u8 *)hva);

@@ -128,16 +117,6 @@ static void __test_sev(void *guest_code, uint32_t type, uint64_t policy) for (;;) { vcpu_run(vcpu);

- if (is_sev_es_vm(vm)) { - TEST_ASSERT(vcpu->run->exit_reason == KVM_EXIT_SYSTEM_EVENT, - "Wanted SYSTEM_EVENT, got %s", - exit_reason_str(vcpu->run->exit_reason)); - TEST_ASSERT_EQ(vcpu->run->system_event.type, KVM_SYSTEM_EVENT_SEV_TERM); - TEST_ASSERT_EQ(vcpu->run->system_event.ndata, 1); - TEST_ASSERT_EQ(vcpu->run->system_event.data[0], GHCB_MSR_TERM_REQ); - break; - } - switch (get_ucall(vcpu, &uc)) { case UCALL_SYNC: continue;

Make GHCB entry page size aligned. Without this change GHCB GPA registration gets incorrectly interpreted in host as a GHCB MSR protocol request.

Signed-off-by: Neeraj Upadhyay Neeraj.Upadhyay@amd.com --- tools/testing/selftests/kvm/lib/x86/sev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/testing/selftests/kvm/lib/x86/sev.c b/tools/testing/selftests/kvm/lib/x86/sev.c index 228c446072a8..38813f60c252 100644 --- a/tools/testing/selftests/kvm/lib/x86/sev.c +++ b/tools/testing/selftests/kvm/lib/x86/sev.c @@ -22,7 +22,7 @@ struct ghcb_entry {

/* Host virtual address of this struct. */ struct ghcb_entry *hva; -}; +} __attribute__((__aligned__(PAGE_SIZE)));

struct ghcb_header { struct ghcb_entry ghcbs[KVM_MAX_VCPUS];

Add #VC exception handling on rdmsr/wrmsr accesses for SEV-ES guests. In addition, add PV interface for direct msr read/write from SEV-ES guests without going through #VC exception path.

Signed-off-by: Neeraj Upadhyay Neeraj.Upadhyay@amd.com --- tools/testing/selftests/kvm/include/x86/sev.h | 2 + tools/testing/selftests/kvm/lib/x86/sev.c | 83 ++++++++++++++++++- 2 files changed, 84 insertions(+), 1 deletion(-)

diff --git a/tools/testing/selftests/kvm/include/x86/sev.h b/tools/testing/selftests/kvm/include/x86/sev.h index bd6ab3f38679..5556ee891260 100644 --- a/tools/testing/selftests/kvm/include/x86/sev.h +++ b/tools/testing/selftests/kvm/include/x86/sev.h @@ -153,4 +153,6 @@ bool is_sev_snp_enabled(void);

void sev_es_ucall_port_write(uint32_t port, uint64_t data);

+void sev_es_vc_handler(struct ex_regs *regs); +void sev_es_pv_msr_rw(uint64_t msr, uint64_t *data, bool write); #endif /* SELFTEST_KVM_SEV_H */ diff --git a/tools/testing/selftests/kvm/lib/x86/sev.c b/tools/testing/selftests/kvm/lib/x86/sev.c index 38813f60c252..ff8f02b83871 100644 --- a/tools/testing/selftests/kvm/lib/x86/sev.c +++ b/tools/testing/selftests/kvm/lib/x86/sev.c @@ -12,7 +12,8 @@ #define IOIO_DATA_8 (1 << 4) #define IOIO_REP (1 << 3)

-#define SW_EXIT_CODE_IOIO 0x7b +#define SW_EXIT_CODE_IOIO 0x7b +#define SW_EXIT_CODE_MSR 0x7c

struct ghcb_entry { struct ghcb ghcb; @@ -404,3 +405,83 @@ void sev_es_ucall_port_write(uint32_t port, uint64_t data)

ghcb_free(entry); } + +static void __sev_es_msr_rw(struct ghcb_entry *entry, uint64_t msr, + uint32_t *low, uint32_t *high, bool write) +{ + uint64_t exitinfo1 = write ? 1 : 0; + struct ghcb *ghcb = &entry->ghcb; + uint32_t ret; + + ghcb_set_sw_exit_code(ghcb, SW_EXIT_CODE_MSR); + ghcb_set_sw_exit_info_1(ghcb, exitinfo1); + ghcb_set_sw_exit_info_2(ghcb, 0); + + ghcb_set_rcx(ghcb, msr); + if (write) { + ghcb_set_rax(ghcb, *low); + ghcb_set_rdx(ghcb, *high); + } + + do_vmg_exit(entry->gpa); + + ret = ghcb->save.sw_exit_info_1 & 0xffffffff; + __GUEST_ASSERT(!ret, "%smsr failed, ret: %u", write ? "wr" : "rd", ret); + + if (!write) { + *low = ghcb->save.rax; + *high = ghcb->save.rdx; + } +} + +void sev_es_pv_msr_rw(uint64_t msr, uint64_t *data, bool write) +{ + struct ghcb_entry *entry; + uint32_t low, high; + + entry = ghcb_alloc(); + register_ghcb_page(entry->gpa); + + if (write) { + low = *data & ((1ULL << 32) - 1); + high = *data >> 32; + } + __sev_es_msr_rw(entry, msr, &low, &high, write); + + if (!write) + *data = low | (uint64_t)high << 32; + + ghcb_free(entry); +} + +static void sev_es_vc_msr_handler(struct ex_regs *regs) +{ + struct ghcb_entry *entry; + bool write; + + /* wrmsr encoding has second byte = 0x30 */ + write = (*((char *)regs->rip + 1) == 0x30); + + entry = ghcb_alloc(); + register_ghcb_page(entry->gpa); + + __sev_es_msr_rw(entry, regs->rcx, (uint32_t *)&regs->rax, + (uint32_t *)&regs->rdx, write); + + ghcb_free(entry); +} + +void sev_es_vc_handler(struct ex_regs *regs) +{ + uint64_t exit_code = regs->error_code; + + switch (exit_code) { + case SVM_EXIT_MSR: + sev_es_vc_msr_handler(regs); + /* rdmsr/wrmsr instruction size = 2 */ + regs->rip += 2; + break; + default: + __GUEST_ASSERT(0, "No VC handler\n"); + } +}

Add instruction deconding support for MMIO instruction decoding in SEV-ES VC handler path. For ease of review, file insn-eval.c is a snapshot of the file from arch/x86/lib/. It will be updated in subsequent commits with the modifications to just support minimal instruction deconding.

Signed-off-by: Neeraj Upadhyay Neeraj.Upadhyay@amd.com --- tools/testing/selftests/kvm/.gitignore | 3 +- .../testing/selftests/kvm/lib/x86/insn-eval.c | 1670 +++++++++++++++++ 2 files changed, 1672 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/kvm/lib/x86/insn-eval.c

diff --git a/tools/testing/selftests/kvm/.gitignore b/tools/testing/selftests/kvm/.gitignore index 1d41a046a7bf..7f75c8a2f722 100644 --- a/tools/testing/selftests/kvm/.gitignore +++ b/tools/testing/selftests/kvm/.gitignore @@ -9,4 +9,5 @@ !config !settings !Makefile -!Makefile.kvm \ No newline at end of file +!Makefile.kvm + diff --git a/tools/testing/selftests/kvm/lib/x86/insn-eval.c b/tools/testing/selftests/kvm/lib/x86/insn-eval.c new file mode 100644 index 000000000000..98631c0e7a11 --- /dev/null +++ b/tools/testing/selftests/kvm/lib/x86/insn-eval.c @@ -0,0 +1,1670 @@ +/* + * Utility functions for x86 operand and address decoding + * + * Copyright (C) Intel Corporation 2017 + */ +#include <linux/kernel.h> +#include <linux/string.h> +#include <linux/ratelimit.h> +#include <linux/mmu_context.h> +#include <asm/desc_defs.h> +#include <asm/desc.h> +#include <asm/inat.h> +#include <asm/insn.h> +#include <asm/insn-eval.h> +#include <asm/ldt.h> +#include <asm/vm86.h> + +#undef pr_fmt +#define pr_fmt(fmt) "insn: " fmt + +enum reg_type { + REG_TYPE_RM = 0, + REG_TYPE_REG, + REG_TYPE_INDEX, + REG_TYPE_BASE, +}; + +/** + * is_string_insn() - Determine if instruction is a string instruction + * @insn: Instruction containing the opcode to inspect + * + * Returns: + * + * true if the instruction, determined by the opcode, is any of the + * string instructions as defined in the Intel Software Development manual. + * False otherwise. + */ +static bool is_string_insn(struct insn *insn) +{ + /* All string instructions have a 1-byte opcode. */ + if (insn->opcode.nbytes != 1) + return false; + + switch (insn->opcode.bytes[0]) { + case 0x6c ... 0x6f: /* INS, OUTS */ + case 0xa4 ... 0xa7: /* MOVS, CMPS */ + case 0xaa ... 0xaf: /* STOS, LODS, SCAS */ + return true; + default: + return false; + } +} + +/** + * insn_has_rep_prefix() - Determine if instruction has a REP prefix + * @insn: Instruction containing the prefix to inspect + * + * Returns: + * + * true if the instruction has a REP prefix, false if not. + */ +bool insn_has_rep_prefix(struct insn *insn) +{ + insn_byte_t p; + int i; + + insn_get_prefixes(insn); + + for_each_insn_prefix(insn, i, p) { + if (p == 0xf2 || p == 0xf3) + return true; + } + + return false; +} + +/** + * get_seg_reg_override_idx() - obtain segment register override index + * @insn: Valid instruction with segment override prefixes + * + * Inspect the instruction prefixes in @insn and find segment overrides, if any. + * + * Returns: + * + * A constant identifying the segment register to use, among CS, SS, DS, + * ES, FS, or GS. INAT_SEG_REG_DEFAULT is returned if no segment override + * prefixes were found. + * + * -EINVAL in case of error. + */ +static int get_seg_reg_override_idx(struct insn *insn) +{ + int idx = INAT_SEG_REG_DEFAULT; + int num_overrides = 0, i; + insn_byte_t p; + + insn_get_prefixes(insn); + + /* Look for any segment override prefixes. */ + for_each_insn_prefix(insn, i, p) { + insn_attr_t attr; + + attr = inat_get_opcode_attribute(p); + switch (attr) { + case INAT_MAKE_PREFIX(INAT_PFX_CS): + idx = INAT_SEG_REG_CS; + num_overrides++; + break; + case INAT_MAKE_PREFIX(INAT_PFX_SS): + idx = INAT_SEG_REG_SS; + num_overrides++; + break; + case INAT_MAKE_PREFIX(INAT_PFX_DS): + idx = INAT_SEG_REG_DS; + num_overrides++; + break; + case INAT_MAKE_PREFIX(INAT_PFX_ES): + idx = INAT_SEG_REG_ES; + num_overrides++; + break; + case INAT_MAKE_PREFIX(INAT_PFX_FS): + idx = INAT_SEG_REG_FS; + num_overrides++; + break; + case INAT_MAKE_PREFIX(INAT_PFX_GS): + idx = INAT_SEG_REG_GS; + num_overrides++; + break; + /* No default action needed. */ + } + } + + /* More than one segment override prefix leads to undefined behavior. */ + if (num_overrides > 1) + return -EINVAL; + + return idx; +} + +/** + * check_seg_overrides() - check if segment override prefixes are allowed + * @insn: Valid instruction with segment override prefixes + * @regoff: Operand offset, in pt_regs, for which the check is performed + * + * For a particular register used in register-indirect addressing, determine if + * segment override prefixes can be used. Specifically, no overrides are allowed + * for rDI if used with a string instruction. + * + * Returns: + * + * True if segment override prefixes can be used with the register indicated + * in @regoff. False if otherwise. + */ +static bool check_seg_overrides(struct insn *insn, int regoff) +{ + if (regoff == offsetof(struct pt_regs, di) && is_string_insn(insn)) + return false; + + return true; +} + +/** + * resolve_default_seg() - resolve default segment register index for an operand + * @insn: Instruction with opcode and address size. Must be valid. + * @regs: Register values as seen when entering kernel mode + * @off: Operand offset, in pt_regs, for which resolution is needed + * + * Resolve the default segment register index associated with the instruction + * operand register indicated by @off. Such index is resolved based on defaults + * described in the Intel Software Development Manual. + * + * Returns: + * + * If in protected mode, a constant identifying the segment register to use, + * among CS, SS, ES or DS. If in long mode, INAT_SEG_REG_IGNORE. + * + * -EINVAL in case of error. + */ +static int resolve_default_seg(struct insn *insn, struct pt_regs *regs, int off) +{ + if (any_64bit_mode(regs)) + return INAT_SEG_REG_IGNORE; + /* + * Resolve the default segment register as described in Section 3.7.4 + * of the Intel Software Development Manual Vol. 1: + * + * + DS for all references involving r[ABCD]X, and rSI. + * + If used in a string instruction, ES for rDI. Otherwise, DS. + * + AX, CX and DX are not valid register operands in 16-bit address + * encodings but are valid for 32-bit and 64-bit encodings. + * + -EDOM is reserved to identify for cases in which no register + * is used (i.e., displacement-only addressing). Use DS. + * + SS for rSP or rBP. + * + CS for rIP. + */ + + switch (off) { + case offsetof(struct pt_regs, ax): + case offsetof(struct pt_regs, cx): + case offsetof(struct pt_regs, dx): + /* Need insn to verify address size. */ + if (insn->addr_bytes == 2) + return -EINVAL; + + fallthrough; + + case -EDOM: + case offsetof(struct pt_regs, bx): + case offsetof(struct pt_regs, si): + return INAT_SEG_REG_DS; + + case offsetof(struct pt_regs, di): + if (is_string_insn(insn)) + return INAT_SEG_REG_ES; + return INAT_SEG_REG_DS; + + case offsetof(struct pt_regs, bp): + case offsetof(struct pt_regs, sp): + return INAT_SEG_REG_SS; + + case offsetof(struct pt_regs, ip): + return INAT_SEG_REG_CS; + + default: + return -EINVAL; + } +} + +/** + * resolve_seg_reg() - obtain segment register index + * @insn: Instruction with operands + * @regs: Register values as seen when entering kernel mode + * @regoff: Operand offset, in pt_regs, used to determine segment register + * + * Determine the segment register associated with the operands and, if + * applicable, prefixes and the instruction pointed by @insn. + * + * The segment register associated to an operand used in register-indirect + * addressing depends on: + * + * a) Whether running in long mode (in such a case segments are ignored, except + * if FS or GS are used). + * + * b) Whether segment override prefixes can be used. Certain instructions and + * registers do not allow override prefixes. + * + * c) Whether segment overrides prefixes are found in the instruction prefixes. + * + * d) If there are not segment override prefixes or they cannot be used, the + * default segment register associated with the operand register is used. + * + * The function checks first if segment override prefixes can be used with the + * operand indicated by @regoff. If allowed, obtain such overridden segment + * register index. Lastly, if not prefixes were found or cannot be used, resolve + * the segment register index to use based on the defaults described in the + * Intel documentation. In long mode, all segment register indexes will be + * ignored, except if overrides were found for FS or GS. All these operations + * are done using helper functions. + * + * The operand register, @regoff, is represented as the offset from the base of + * pt_regs. + * + * As stated, the main use of this function is to determine the segment register + * index based on the instruction, its operands and prefixes. Hence, @insn + * must be valid. However, if @regoff indicates rIP, we don't need to inspect + * @insn at all as in this case CS is used in all cases. This case is checked + * before proceeding further. + * + * Please note that this function does not return the value in the segment + * register (i.e., the segment selector) but our defined index. The segment + * selector needs to be obtained using get_segment_selector() and passing the + * segment register index resolved by this function. + * + * Returns: + * + * An index identifying the segment register to use, among CS, SS, DS, + * ES, FS, or GS. INAT_SEG_REG_IGNORE is returned if running in long mode. + * + * -EINVAL in case of error. + */ +static int resolve_seg_reg(struct insn *insn, struct pt_regs *regs, int regoff) +{ + int idx; + + /* + * In the unlikely event of having to resolve the segment register + * index for rIP, do it first. Segment override prefixes should not + * be used. Hence, it is not necessary to inspect the instruction, + * which may be invalid at this point. + */ + if (regoff == offsetof(struct pt_regs, ip)) { + if (any_64bit_mode(regs)) + return INAT_SEG_REG_IGNORE; + else + return INAT_SEG_REG_CS; + } + + if (!insn) + return -EINVAL; + + if (!check_seg_overrides(insn, regoff)) + return resolve_default_seg(insn, regs, regoff); + + idx = get_seg_reg_override_idx(insn); + if (idx < 0) + return idx; + + if (idx == INAT_SEG_REG_DEFAULT) + return resolve_default_seg(insn, regs, regoff); + + /* + * In long mode, segment override prefixes are ignored, except for + * overrides for FS and GS. + */ + if (any_64bit_mode(regs)) { + if (idx != INAT_SEG_REG_FS && + idx != INAT_SEG_REG_GS) + idx = INAT_SEG_REG_IGNORE; + } + + return idx; +} + +/** + * get_segment_selector() - obtain segment selector + * @regs: Register values as seen when entering kernel mode + * @seg_reg_idx: Segment register index to use + * + * Obtain the segment selector from any of the CS, SS, DS, ES, FS, GS segment + * registers. In CONFIG_X86_32, the segment is obtained from either pt_regs or + * kernel_vm86_regs as applicable. In CONFIG_X86_64, CS and SS are obtained + * from pt_regs. DS, ES, FS and GS are obtained by reading the actual CPU + * registers. This done for only for completeness as in CONFIG_X86_64 segment + * registers are ignored. + * + * Returns: + * + * Value of the segment selector, including null when running in + * long mode. + * + * -EINVAL on error. + */ +static short get_segment_selector(struct pt_regs *regs, int seg_reg_idx) +{ + unsigned short sel; + +#ifdef CONFIG_X86_64 + switch (seg_reg_idx) { + case INAT_SEG_REG_IGNORE: + return 0; + case INAT_SEG_REG_CS: + return (unsigned short)(regs->cs & 0xffff); + case INAT_SEG_REG_SS: + return (unsigned short)(regs->ss & 0xffff); + case INAT_SEG_REG_DS: + savesegment(ds, sel); + return sel; + case INAT_SEG_REG_ES: + savesegment(es, sel); + return sel; + case INAT_SEG_REG_FS: + savesegment(fs, sel); + return sel; + case INAT_SEG_REG_GS: + savesegment(gs, sel); + return sel; + default: + return -EINVAL; + } +#else /* CONFIG_X86_32 */ + struct kernel_vm86_regs *vm86regs = (struct kernel_vm86_regs *)regs; + + if (v8086_mode(regs)) { + switch (seg_reg_idx) { + case INAT_SEG_REG_CS: + return (unsigned short)(regs->cs & 0xffff); + case INAT_SEG_REG_SS: + return (unsigned short)(regs->ss & 0xffff); + case INAT_SEG_REG_DS: + return vm86regs->ds; + case INAT_SEG_REG_ES: + return vm86regs->es; + case INAT_SEG_REG_FS: + return vm86regs->fs; + case INAT_SEG_REG_GS: + return vm86regs->gs; + case INAT_SEG_REG_IGNORE: + default: + return -EINVAL; + } + } + + switch (seg_reg_idx) { + case INAT_SEG_REG_CS: + return (unsigned short)(regs->cs & 0xffff); + case INAT_SEG_REG_SS: + return (unsigned short)(regs->ss & 0xffff); + case INAT_SEG_REG_DS: + return (unsigned short)(regs->ds & 0xffff); + case INAT_SEG_REG_ES: + return (unsigned short)(regs->es & 0xffff); + case INAT_SEG_REG_FS: + return (unsigned short)(regs->fs & 0xffff); + case INAT_SEG_REG_GS: + savesegment(gs, sel); + return sel; + case INAT_SEG_REG_IGNORE: + default: + return -EINVAL; + } +#endif /* CONFIG_X86_64 */ +} + +static const int pt_regoff[] = { + offsetof(struct pt_regs, ax), + offsetof(struct pt_regs, cx), + offsetof(struct pt_regs, dx), + offsetof(struct pt_regs, bx), + offsetof(struct pt_regs, sp), + offsetof(struct pt_regs, bp), + offsetof(struct pt_regs, si), + offsetof(struct pt_regs, di), +#ifdef CONFIG_X86_64 + offsetof(struct pt_regs, r8), + offsetof(struct pt_regs, r9), + offsetof(struct pt_regs, r10), + offsetof(struct pt_regs, r11), + offsetof(struct pt_regs, r12), + offsetof(struct pt_regs, r13), + offsetof(struct pt_regs, r14), + offsetof(struct pt_regs, r15), +#else + offsetof(struct pt_regs, ds), + offsetof(struct pt_regs, es), + offsetof(struct pt_regs, fs), + offsetof(struct pt_regs, gs), +#endif +}; + +int pt_regs_offset(struct pt_regs *regs, int regno) +{ + if ((unsigned)regno < ARRAY_SIZE(pt_regoff)) + return pt_regoff[regno]; + return -EDOM; +} + +static int get_regno(struct insn *insn, enum reg_type type) +{ + int nr_registers = ARRAY_SIZE(pt_regoff); + int regno = 0; + + /* + * Don't possibly decode a 32-bit instructions as + * reading a 64-bit-only register. + */ + if (IS_ENABLED(CONFIG_X86_64) && !insn->x86_64) + nr_registers -= 8; + + switch (type) { + case REG_TYPE_RM: + regno = X86_MODRM_RM(insn->modrm.value); + + /* + * ModRM.mod == 0 and ModRM.rm == 5 means a 32-bit displacement + * follows the ModRM byte. + */ + if (!X86_MODRM_MOD(insn->modrm.value) && regno == 5) + return -EDOM; + + if (X86_REX_B(insn->rex_prefix.value)) + regno += 8; + break; + + case REG_TYPE_REG: + regno = X86_MODRM_REG(insn->modrm.value); + + if (X86_REX_R(insn->rex_prefix.value)) + regno += 8; + break; + + case REG_TYPE_INDEX: + regno = X86_SIB_INDEX(insn->sib.value); + if (X86_REX_X(insn->rex_prefix.value)) + regno += 8; + + /* + * If ModRM.mod != 3 and SIB.index = 4 the scale*index + * portion of the address computation is null. This is + * true only if REX.X is 0. In such a case, the SIB index + * is used in the address computation. + */ + if (X86_MODRM_MOD(insn->modrm.value) != 3 && regno == 4) + return -EDOM; + break; + + case REG_TYPE_BASE: + regno = X86_SIB_BASE(insn->sib.value); + /* + * If ModRM.mod is 0 and SIB.base == 5, the base of the + * register-indirect addressing is 0. In this case, a + * 32-bit displacement follows the SIB byte. + */ + if (!X86_MODRM_MOD(insn->modrm.value) && regno == 5) + return -EDOM; + + if (X86_REX_B(insn->rex_prefix.value)) + regno += 8; + break; + + default: + pr_err_ratelimited("invalid register type: %d\n", type); + return -EINVAL; + } + + if (regno >= nr_registers) { + WARN_ONCE(1, "decoded an instruction with an invalid register"); + return -EINVAL; + } + return regno; +} + +static int get_reg_offset(struct insn *insn, struct pt_regs *regs, + enum reg_type type) +{ + int regno = get_regno(insn, type); + + if (regno < 0) + return regno; + + return pt_regs_offset(regs, regno); +} + +/** + * get_reg_offset_16() - Obtain offset of register indicated by instruction + * @insn: Instruction containing ModRM byte + * @regs: Register values as seen when entering kernel mode + * @offs1: Offset of the first operand register + * @offs2: Offset of the second operand register, if applicable + * + * Obtain the offset, in pt_regs, of the registers indicated by the ModRM byte + * in @insn. This function is to be used with 16-bit address encodings. The + * @offs1 and @offs2 will be written with the offset of the two registers + * indicated by the instruction. In cases where any of the registers is not + * referenced by the instruction, the value will be set to -EDOM. + * + * Returns: + * + * 0 on success, -EINVAL on error. + */ +static int get_reg_offset_16(struct insn *insn, struct pt_regs *regs, + int *offs1, int *offs2) +{ + /* + * 16-bit addressing can use one or two registers. Specifics of + * encodings are given in Table 2-1. "16-Bit Addressing Forms with the + * ModR/M Byte" of the Intel Software Development Manual. + */ + static const int regoff1[] = { + offsetof(struct pt_regs, bx), + offsetof(struct pt_regs, bx), + offsetof(struct pt_regs, bp), + offsetof(struct pt_regs, bp), + offsetof(struct pt_regs, si), + offsetof(struct pt_regs, di), + offsetof(struct pt_regs, bp), + offsetof(struct pt_regs, bx), + }; + + static const int regoff2[] = { + offsetof(struct pt_regs, si), + offsetof(struct pt_regs, di), + offsetof(struct pt_regs, si), + offsetof(struct pt_regs, di), + -EDOM, + -EDOM, + -EDOM, + -EDOM, + }; + + if (!offs1 || !offs2) + return -EINVAL; + + /* Operand is a register, use the generic function. */ + if (X86_MODRM_MOD(insn->modrm.value) == 3) { + *offs1 = insn_get_modrm_rm_off(insn, regs); + *offs2 = -EDOM; + return 0; + } + + *offs1 = regoff1[X86_MODRM_RM(insn->modrm.value)]; + *offs2 = regoff2[X86_MODRM_RM(insn->modrm.value)]; + + /* + * If ModRM.mod is 0 and ModRM.rm is 110b, then we use displacement- + * only addressing. This means that no registers are involved in + * computing the effective address. Thus, ensure that the first + * register offset is invalid. The second register offset is already + * invalid under the aforementioned conditions. + */ + if ((X86_MODRM_MOD(insn->modrm.value) == 0) && + (X86_MODRM_RM(insn->modrm.value) == 6)) + *offs1 = -EDOM; + + return 0; +} + +/** + * get_desc() - Obtain contents of a segment descriptor + * @out: Segment descriptor contents on success + * @sel: Segment selector + * + * Given a segment selector, obtain a pointer to the segment descriptor. + * Both global and local descriptor tables are supported. + * + * Returns: + * + * True on success, false on failure. + * + * NULL on error. + */ +static bool get_desc(struct desc_struct *out, unsigned short sel) +{ + struct desc_ptr gdt_desc = {0, 0}; + unsigned long desc_base; + +#ifdef CONFIG_MODIFY_LDT_SYSCALL + if ((sel & SEGMENT_TI_MASK) == SEGMENT_LDT) { + bool success = false; + struct ldt_struct *ldt; + + /* Bits [15:3] contain the index of the desired entry. */ + sel >>= 3; + + mutex_lock(&current->active_mm->context.lock); + ldt = current->active_mm->context.ldt; + if (ldt && sel < ldt->nr_entries) { + *out = ldt->entries[sel]; + success = true; + } + + mutex_unlock(&current->active_mm->context.lock); + + return success; + } +#endif + native_store_gdt(&gdt_desc); + + /* + * Segment descriptors have a size of 8 bytes. Thus, the index is + * multiplied by 8 to obtain the memory offset of the desired descriptor + * from the base of the GDT. As bits [15:3] of the segment selector + * contain the index, it can be regarded as multiplied by 8 already. + * All that remains is to clear bits [2:0]. + */ + desc_base = sel & ~(SEGMENT_RPL_MASK | SEGMENT_TI_MASK); + + if (desc_base > gdt_desc.size) + return false; + + *out = *(struct desc_struct *)(gdt_desc.address + desc_base); + return true; +} + +/** + * insn_get_seg_base() - Obtain base address of segment descriptor. + * @regs: Register values as seen when entering kernel mode + * @seg_reg_idx: Index of the segment register pointing to seg descriptor + * + * Obtain the base address of the segment as indicated by the segment descriptor + * pointed by the segment selector. The segment selector is obtained from the + * input segment register index @seg_reg_idx. + * + * Returns: + * + * In protected mode, base address of the segment. Zero in long mode, + * except when FS or GS are used. In virtual-8086 mode, the segment + * selector shifted 4 bits to the right. + * + * -1L in case of error. + */ +unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx) +{ + struct desc_struct desc; + short sel; + + sel = get_segment_selector(regs, seg_reg_idx); + if (sel < 0) + return -1L; + + if (v8086_mode(regs)) + /* + * Base is simply the segment selector shifted 4 + * bits to the right. + */ + return (unsigned long)(sel << 4); + + if (any_64bit_mode(regs)) { + /* + * Only FS or GS will have a base address, the rest of + * the segments' bases are forced to 0. + */ + unsigned long base; + + if (seg_reg_idx == INAT_SEG_REG_FS) { + rdmsrl(MSR_FS_BASE, base); + } else if (seg_reg_idx == INAT_SEG_REG_GS) { + /* + * swapgs was called at the kernel entry point. Thus, + * MSR_KERNEL_GS_BASE will have the user-space GS base. + */ + if (user_mode(regs)) + rdmsrl(MSR_KERNEL_GS_BASE, base); + else + rdmsrl(MSR_GS_BASE, base); + } else { + base = 0; + } + return base; + } + + /* In protected mode the segment selector cannot be null. */ + if (!sel) + return -1L; + + if (!get_desc(&desc, sel)) + return -1L; + + return get_desc_base(&desc); +} + +/** + * get_seg_limit() - Obtain the limit of a segment descriptor + * @regs: Register values as seen when entering kernel mode + * @seg_reg_idx: Index of the segment register pointing to seg descriptor + * + * Obtain the limit of the segment as indicated by the segment descriptor + * pointed by the segment selector. The segment selector is obtained from the + * input segment register index @seg_reg_idx. + * + * Returns: + * + * In protected mode, the limit of the segment descriptor in bytes. + * In long mode and virtual-8086 mode, segment limits are not enforced. Thus, + * limit is returned as -1L to imply a limit-less segment. + * + * Zero is returned on error. + */ +static unsigned long get_seg_limit(struct pt_regs *regs, int seg_reg_idx) +{ + struct desc_struct desc; + unsigned long limit; + short sel; + + sel = get_segment_selector(regs, seg_reg_idx); + if (sel < 0) + return 0; + + if (any_64bit_mode(regs) || v8086_mode(regs)) + return -1L; + + if (!sel) + return 0; + + if (!get_desc(&desc, sel)) + return 0; + + /* + * If the granularity bit is set, the limit is given in multiples + * of 4096. This also means that the 12 least significant bits are + * not tested when checking the segment limits. In practice, + * this means that the segment ends in (limit << 12) + 0xfff. + */ + limit = get_desc_limit(&desc); + if (desc.g) + limit = (limit << 12) + 0xfff; + + return limit; +} + +/** + * insn_get_code_seg_params() - Obtain code segment parameters + * @regs: Structure with register values as seen when entering kernel mode + * + * Obtain address and operand sizes of the code segment. It is obtained from the + * selector contained in the CS register in regs. In protected mode, the default + * address is determined by inspecting the L and D bits of the segment + * descriptor. In virtual-8086 mode, the default is always two bytes for both + * address and operand sizes. + * + * Returns: + * + * An int containing ORed-in default parameters on success. + * + * -EINVAL on error. + */ +int insn_get_code_seg_params(struct pt_regs *regs) +{ + struct desc_struct desc; + short sel; + + if (v8086_mode(regs)) + /* Address and operand size are both 16-bit. */ + return INSN_CODE_SEG_PARAMS(2, 2); + + sel = get_segment_selector(regs, INAT_SEG_REG_CS); + if (sel < 0) + return sel; + + if (!get_desc(&desc, sel)) + return -EINVAL; + + /* + * The most significant byte of the Type field of the segment descriptor + * determines whether a segment contains data or code. If this is a data + * segment, return error. + */ + if (!(desc.type & BIT(3))) + return -EINVAL; + + switch ((desc.l << 1) | desc.d) { + case 0: /* + * Legacy mode. CS.L=0, CS.D=0. Address and operand size are + * both 16-bit. + */ + return INSN_CODE_SEG_PARAMS(2, 2); + case 1: /* + * Legacy mode. CS.L=0, CS.D=1. Address and operand size are + * both 32-bit. + */ + return INSN_CODE_SEG_PARAMS(4, 4); + case 2: /* + * IA-32e 64-bit mode. CS.L=1, CS.D=0. Address size is 64-bit; + * operand size is 32-bit. + */ + return INSN_CODE_SEG_PARAMS(4, 8); + case 3: /* Invalid setting. CS.L=1, CS.D=1 */ + fallthrough; + default: + return -EINVAL; + } +} + +/** + * insn_get_modrm_rm_off() - Obtain register in r/m part of the ModRM byte + * @insn: Instruction containing the ModRM byte + * @regs: Register values as seen when entering kernel mode + * + * Returns: + * + * The register indicated by the r/m part of the ModRM byte. The + * register is obtained as an offset from the base of pt_regs. In specific + * cases, the returned value can be -EDOM to indicate that the particular value + * of ModRM does not refer to a register and shall be ignored. + */ +int insn_get_modrm_rm_off(struct insn *insn, struct pt_regs *regs) +{ + return get_reg_offset(insn, regs, REG_TYPE_RM); +} + +/** + * insn_get_modrm_reg_off() - Obtain register in reg part of the ModRM byte + * @insn: Instruction containing the ModRM byte + * @regs: Register values as seen when entering kernel mode + * + * Returns: + * + * The register indicated by the reg part of the ModRM byte. The + * register is obtained as an offset from the base of pt_regs. + */ +int insn_get_modrm_reg_off(struct insn *insn, struct pt_regs *regs) +{ + return get_reg_offset(insn, regs, REG_TYPE_REG); +} + +/** + * insn_get_modrm_reg_ptr() - Obtain register pointer based on ModRM byte + * @insn: Instruction containing the ModRM byte + * @regs: Register values as seen when entering kernel mode + * + * Returns: + * + * The register indicated by the reg part of the ModRM byte. + * The register is obtained as a pointer within pt_regs. + */ +unsigned long *insn_get_modrm_reg_ptr(struct insn *insn, struct pt_regs *regs) +{ + int offset; + + offset = insn_get_modrm_reg_off(insn, regs); + if (offset < 0) + return NULL; + return (void *)regs + offset; +} + +/** + * get_seg_base_limit() - obtain base address and limit of a segment + * @insn: Instruction. Must be valid. + * @regs: Register values as seen when entering kernel mode + * @regoff: Operand offset, in pt_regs, used to resolve segment descriptor + * @base: Obtained segment base + * @limit: Obtained segment limit + * + * Obtain the base address and limit of the segment associated with the operand + * @regoff and, if any or allowed, override prefixes in @insn. This function is + * different from insn_get_seg_base() as the latter does not resolve the segment + * associated with the instruction operand. If a limit is not needed (e.g., + * when running in long mode), @limit can be NULL. + * + * Returns: + * + * 0 on success. @base and @limit will contain the base address and of the + * resolved segment, respectively. + * + * -EINVAL on error. + */ +static int get_seg_base_limit(struct insn *insn, struct pt_regs *regs, + int regoff, unsigned long *base, + unsigned long *limit) +{ + int seg_reg_idx; + + if (!base) + return -EINVAL; + + seg_reg_idx = resolve_seg_reg(insn, regs, regoff); + if (seg_reg_idx < 0) + return seg_reg_idx; + + *base = insn_get_seg_base(regs, seg_reg_idx); + if (*base == -1L) + return -EINVAL; + + if (!limit) + return 0; + + *limit = get_seg_limit(regs, seg_reg_idx); + if (!(*limit)) + return -EINVAL; + + return 0; +} + +/** + * get_eff_addr_reg() - Obtain effective address from register operand + * @insn: Instruction. Must be valid. + * @regs: Register values as seen when entering kernel mode + * @regoff: Obtained operand offset, in pt_regs, with the effective address + * @eff_addr: Obtained effective address + * + * Obtain the effective address stored in the register operand as indicated by + * the ModRM byte. This function is to be used only with register addressing + * (i.e., ModRM.mod is 3). The effective address is saved in @eff_addr. The + * register operand, as an offset from the base of pt_regs, is saved in @regoff; + * such offset can then be used to resolve the segment associated with the + * operand. This function can be used with any of the supported address sizes + * in x86. + * + * Returns: + * + * 0 on success. @eff_addr will have the effective address stored in the + * operand indicated by ModRM. @regoff will have such operand as an offset from + * the base of pt_regs. + * + * -EINVAL on error. + */ +static int get_eff_addr_reg(struct insn *insn, struct pt_regs *regs, + int *regoff, long *eff_addr) +{ + int ret; + + ret = insn_get_modrm(insn); + if (ret) + return ret; + + if (X86_MODRM_MOD(insn->modrm.value) != 3) + return -EINVAL; + + *regoff = get_reg_offset(insn, regs, REG_TYPE_RM); + if (*regoff < 0) + return -EINVAL; + + /* Ignore bytes that are outside the address size. */ + if (insn->addr_bytes == 2) + *eff_addr = regs_get_register(regs, *regoff) & 0xffff; + else if (insn->addr_bytes == 4) + *eff_addr = regs_get_register(regs, *regoff) & 0xffffffff; + else /* 64-bit address */ + *eff_addr = regs_get_register(regs, *regoff); + + return 0; +} + +/** + * get_eff_addr_modrm() - Obtain referenced effective address via ModRM + * @insn: Instruction. Must be valid. + * @regs: Register values as seen when entering kernel mode + * @regoff: Obtained operand offset, in pt_regs, associated with segment + * @eff_addr: Obtained effective address + * + * Obtain the effective address referenced by the ModRM byte of @insn. After + * identifying the registers involved in the register-indirect memory reference, + * its value is obtained from the operands in @regs. The computed address is + * stored @eff_addr. Also, the register operand that indicates the associated + * segment is stored in @regoff, this parameter can later be used to determine + * such segment. + * + * Returns: + * + * 0 on success. @eff_addr will have the referenced effective address. @regoff + * will have a register, as an offset from the base of pt_regs, that can be used + * to resolve the associated segment. + * + * -EINVAL on error. + */ +static int get_eff_addr_modrm(struct insn *insn, struct pt_regs *regs, + int *regoff, long *eff_addr) +{ + long tmp; + int ret; + + if (insn->addr_bytes != 8 && insn->addr_bytes != 4) + return -EINVAL; + + ret = insn_get_modrm(insn); + if (ret) + return ret; + + if (X86_MODRM_MOD(insn->modrm.value) > 2) + return -EINVAL; + + *regoff = get_reg_offset(insn, regs, REG_TYPE_RM); + + /* + * -EDOM means that we must ignore the address_offset. In such a case, + * in 64-bit mode the effective address relative to the rIP of the + * following instruction. + */ + if (*regoff == -EDOM) { + if (any_64bit_mode(regs)) + tmp = regs->ip + insn->length; + else + tmp = 0; + } else if (*regoff < 0) { + return -EINVAL; + } else { + tmp = regs_get_register(regs, *regoff); + } + + if (insn->addr_bytes == 4) { + int addr32 = (int)(tmp & 0xffffffff) + insn->displacement.value; + + *eff_addr = addr32 & 0xffffffff; + } else { + *eff_addr = tmp + insn->displacement.value; + } + + return 0; +} + +/** + * get_eff_addr_modrm_16() - Obtain referenced effective address via ModRM + * @insn: Instruction. Must be valid. + * @regs: Register values as seen when entering kernel mode + * @regoff: Obtained operand offset, in pt_regs, associated with segment + * @eff_addr: Obtained effective address + * + * Obtain the 16-bit effective address referenced by the ModRM byte of @insn. + * After identifying the registers involved in the register-indirect memory + * reference, its value is obtained from the operands in @regs. The computed + * address is stored @eff_addr. Also, the register operand that indicates + * the associated segment is stored in @regoff, this parameter can later be used + * to determine such segment. + * + * Returns: + * + * 0 on success. @eff_addr will have the referenced effective address. @regoff + * will have a register, as an offset from the base of pt_regs, that can be used + * to resolve the associated segment. + * + * -EINVAL on error. + */ +static int get_eff_addr_modrm_16(struct insn *insn, struct pt_regs *regs, + int *regoff, short *eff_addr) +{ + int addr_offset1, addr_offset2, ret; + short addr1 = 0, addr2 = 0, displacement; + + if (insn->addr_bytes != 2) + return -EINVAL; + + insn_get_modrm(insn); + + if (!insn->modrm.nbytes) + return -EINVAL; + + if (X86_MODRM_MOD(insn->modrm.value) > 2) + return -EINVAL; + + ret = get_reg_offset_16(insn, regs, &addr_offset1, &addr_offset2); + if (ret < 0) + return -EINVAL; + + /* + * Don't fail on invalid offset values. They might be invalid because + * they cannot be used for this particular value of ModRM. Instead, use + * them in the computation only if they contain a valid value. + */ + if (addr_offset1 != -EDOM) + addr1 = regs_get_register(regs, addr_offset1) & 0xffff; + + if (addr_offset2 != -EDOM) + addr2 = regs_get_register(regs, addr_offset2) & 0xffff; + + displacement = insn->displacement.value & 0xffff; + *eff_addr = addr1 + addr2 + displacement; + + /* + * The first operand register could indicate to use of either SS or DS + * registers to obtain the segment selector. The second operand + * register can only indicate the use of DS. Thus, the first operand + * will be used to obtain the segment selector. + */ + *regoff = addr_offset1; + + return 0; +} + +/** + * get_eff_addr_sib() - Obtain referenced effective address via SIB + * @insn: Instruction. Must be valid. + * @regs: Register values as seen when entering kernel mode + * @base_offset: Obtained operand offset, in pt_regs, associated with segment + * @eff_addr: Obtained effective address + * + * Obtain the effective address referenced by the SIB byte of @insn. After + * identifying the registers involved in the indexed, register-indirect memory + * reference, its value is obtained from the operands in @regs. The computed + * address is stored @eff_addr. Also, the register operand that indicates the + * associated segment is stored in @base_offset; this parameter can later be + * used to determine such segment. + * + * Returns: + * + * 0 on success. @eff_addr will have the referenced effective address. + * @base_offset will have a register, as an offset from the base of pt_regs, + * that can be used to resolve the associated segment. + * + * Negative value on error. + */ +static int get_eff_addr_sib(struct insn *insn, struct pt_regs *regs, + int *base_offset, long *eff_addr) +{ + long base, indx; + int indx_offset; + int ret; + + if (insn->addr_bytes != 8 && insn->addr_bytes != 4) + return -EINVAL; + + ret = insn_get_modrm(insn); + if (ret) + return ret; + + if (!insn->modrm.nbytes) + return -EINVAL; + + if (X86_MODRM_MOD(insn->modrm.value) > 2) + return -EINVAL; + + ret = insn_get_sib(insn); + if (ret) + return ret; + + if (!insn->sib.nbytes) + return -EINVAL; + + *base_offset = get_reg_offset(insn, regs, REG_TYPE_BASE); + indx_offset = get_reg_offset(insn, regs, REG_TYPE_INDEX); + + /* + * Negative values in the base and index offset means an error when + * decoding the SIB byte. Except -EDOM, which means that the registers + * should not be used in the address computation. + */ + if (*base_offset == -EDOM) + base = 0; + else if (*base_offset < 0) + return -EINVAL; + else + base = regs_get_register(regs, *base_offset); + + if (indx_offset == -EDOM) + indx = 0; + else if (indx_offset < 0) + return -EINVAL; + else + indx = regs_get_register(regs, indx_offset); + + if (insn->addr_bytes == 4) { + int addr32, base32, idx32; + + base32 = base & 0xffffffff; + idx32 = indx & 0xffffffff; + + addr32 = base32 + idx32 * (1 << X86_SIB_SCALE(insn->sib.value)); + addr32 += insn->displacement.value; + + *eff_addr = addr32 & 0xffffffff; + } else { + *eff_addr = base + indx * (1 << X86_SIB_SCALE(insn->sib.value)); + *eff_addr += insn->displacement.value; + } + + return 0; +} + +/** + * get_addr_ref_16() - Obtain the 16-bit address referred by instruction + * @insn: Instruction containing ModRM byte and displacement + * @regs: Register values as seen when entering kernel mode + * + * This function is to be used with 16-bit address encodings. Obtain the memory + * address referred by the instruction's ModRM and displacement bytes. Also, the + * segment used as base is determined by either any segment override prefixes in + * @insn or the default segment of the registers involved in the address + * computation. In protected mode, segment limits are enforced. + * + * Returns: + * + * Linear address referenced by the instruction operands on success. + * + * -1L on error. + */ +static void __user *get_addr_ref_16(struct insn *insn, struct pt_regs *regs) +{ + unsigned long linear_addr = -1L, seg_base, seg_limit; + int ret, regoff; + short eff_addr; + long tmp; + + if (insn_get_displacement(insn)) + goto out; + + if (insn->addr_bytes != 2) + goto out; + + if (X86_MODRM_MOD(insn->modrm.value) == 3) { + ret = get_eff_addr_reg(insn, regs, &regoff, &tmp); + if (ret) + goto out; + + eff_addr = tmp; + } else { + ret = get_eff_addr_modrm_16(insn, regs, &regoff, &eff_addr); + if (ret) + goto out; + } + + ret = get_seg_base_limit(insn, regs, regoff, &seg_base, &seg_limit); + if (ret) + goto out; + + /* + * Before computing the linear address, make sure the effective address + * is within the limits of the segment. In virtual-8086 mode, segment + * limits are not enforced. In such a case, the segment limit is -1L to + * reflect this fact. + */ + if ((unsigned long)(eff_addr & 0xffff) > seg_limit) + goto out; + + linear_addr = (unsigned long)(eff_addr & 0xffff) + seg_base; + + /* Limit linear address to 20 bits */ + if (v8086_mode(regs)) + linear_addr &= 0xfffff; + +out: + return (void __user *)linear_addr; +} + +/** + * get_addr_ref_32() - Obtain a 32-bit linear address + * @insn: Instruction with ModRM, SIB bytes and displacement + * @regs: Register values as seen when entering kernel mode + * + * This function is to be used with 32-bit address encodings to obtain the + * linear memory address referred by the instruction's ModRM, SIB, + * displacement bytes and segment base address, as applicable. If in protected + * mode, segment limits are enforced. + * + * Returns: + * + * Linear address referenced by instruction and registers on success. + * + * -1L on error. + */ +static void __user *get_addr_ref_32(struct insn *insn, struct pt_regs *regs) +{ + unsigned long linear_addr = -1L, seg_base, seg_limit; + int eff_addr, regoff; + long tmp; + int ret; + + if (insn->addr_bytes != 4) + goto out; + + if (X86_MODRM_MOD(insn->modrm.value) == 3) { + ret = get_eff_addr_reg(insn, regs, &regoff, &tmp); + if (ret) + goto out; + + eff_addr = tmp; + + } else { + if (insn->sib.nbytes) { + ret = get_eff_addr_sib(insn, regs, &regoff, &tmp); + if (ret) + goto out; + + eff_addr = tmp; + } else { + ret = get_eff_addr_modrm(insn, regs, &regoff, &tmp); + if (ret) + goto out; + + eff_addr = tmp; + } + } + + ret = get_seg_base_limit(insn, regs, regoff, &seg_base, &seg_limit); + if (ret) + goto out; + + /* + * In protected mode, before computing the linear address, make sure + * the effective address is within the limits of the segment. + * 32-bit addresses can be used in long and virtual-8086 modes if an + * address override prefix is used. In such cases, segment limits are + * not enforced. When in virtual-8086 mode, the segment limit is -1L + * to reflect this situation. + * + * After computed, the effective address is treated as an unsigned + * quantity. + */ + if (!any_64bit_mode(regs) && ((unsigned int)eff_addr > seg_limit)) + goto out; + + /* + * Even though 32-bit address encodings are allowed in virtual-8086 + * mode, the address range is still limited to [0x-0xffff]. + */ + if (v8086_mode(regs) && (eff_addr & ~0xffff)) + goto out; + + /* + * Data type long could be 64 bits in size. Ensure that our 32-bit + * effective address is not sign-extended when computing the linear + * address. + */ + linear_addr = (unsigned long)(eff_addr & 0xffffffff) + seg_base; + + /* Limit linear address to 20 bits */ + if (v8086_mode(regs)) + linear_addr &= 0xfffff; + +out: + return (void __user *)linear_addr; +} + +/** + * get_addr_ref_64() - Obtain a 64-bit linear address + * @insn: Instruction struct with ModRM and SIB bytes and displacement + * @regs: Structure with register values as seen when entering kernel mode + * + * This function is to be used with 64-bit address encodings to obtain the + * linear memory address referred by the instruction's ModRM, SIB, + * displacement bytes and segment base address, as applicable. + * + * Returns: + * + * Linear address referenced by instruction and registers on success. + * + * -1L on error. + */ +#ifndef CONFIG_X86_64 +static void __user *get_addr_ref_64(struct insn *insn, struct pt_regs *regs) +{ + return (void __user *)-1L; +} +#else +static void __user *get_addr_ref_64(struct insn *insn, struct pt_regs *regs) +{ + unsigned long linear_addr = -1L, seg_base; + int regoff, ret; + long eff_addr; + + if (insn->addr_bytes != 8) + goto out; + + if (X86_MODRM_MOD(insn->modrm.value) == 3) { + ret = get_eff_addr_reg(insn, regs, &regoff, &eff_addr); + if (ret) + goto out; + + } else { + if (insn->sib.nbytes) { + ret = get_eff_addr_sib(insn, regs, &regoff, &eff_addr); + if (ret) + goto out; + } else { + ret = get_eff_addr_modrm(insn, regs, &regoff, &eff_addr); + if (ret) + goto out; + } + + } + + ret = get_seg_base_limit(insn, regs, regoff, &seg_base, NULL); + if (ret) + goto out; + + linear_addr = (unsigned long)eff_addr + seg_base; + +out: + return (void __user *)linear_addr; +} +#endif /* CONFIG_X86_64 */ + +/** + * insn_get_addr_ref() - Obtain the linear address referred by instruction + * @insn: Instruction structure containing ModRM byte and displacement + * @regs: Structure with register values as seen when entering kernel mode + * + * Obtain the linear address referred by the instruction's ModRM, SIB and + * displacement bytes, and segment base, as applicable. In protected mode, + * segment limits are enforced. + * + * Returns: + * + * Linear address referenced by instruction and registers on success. + * + * -1L on error. + */ +void __user *insn_get_addr_ref(struct insn *insn, struct pt_regs *regs) +{ + if (!insn || !regs) + return (void __user *)-1L; + + if (insn_get_opcode(insn)) + return (void __user *)-1L; + + switch (insn->addr_bytes) { + case 2: + return get_addr_ref_16(insn, regs); + case 4: + return get_addr_ref_32(insn, regs); + case 8: + return get_addr_ref_64(insn, regs); + default: + return (void __user *)-1L; + } +} + +int insn_get_effective_ip(struct pt_regs *regs, unsigned long *ip) +{ + unsigned long seg_base = 0; + + /* + * If not in user-space long mode, a custom code segment could be in + * use. This is true in protected mode (if the process defined a local + * descriptor table), or virtual-8086 mode. In most of the cases + * seg_base will be zero as in USER_CS. + */ + if (!user_64bit_mode(regs)) { + seg_base = insn_get_seg_base(regs, INAT_SEG_REG_CS); + if (seg_base == -1L) + return -EINVAL; + } + + *ip = seg_base + regs->ip; + + return 0; +} + +/** + * insn_fetch_from_user() - Copy instruction bytes from user-space memory + * @regs: Structure with register values as seen when entering kernel mode + * @buf: Array to store the fetched instruction + * + * Gets the linear address of the instruction and copies the instruction bytes + * to the buf. + * + * Returns: + * + * - number of instruction bytes copied. + * - 0 if nothing was copied. + * - -EINVAL if the linear address of the instruction could not be calculated + */ +int insn_fetch_from_user(struct pt_regs *regs, unsigned char buf[MAX_INSN_SIZE]) +{ + unsigned long ip; + int not_copied; + + if (insn_get_effective_ip(regs, &ip)) + return -EINVAL; + + not_copied = copy_from_user(buf, (void __user *)ip, MAX_INSN_SIZE); + + return MAX_INSN_SIZE - not_copied; +} + +/** + * insn_fetch_from_user_inatomic() - Copy instruction bytes from user-space memory + * while in atomic code + * @regs: Structure with register values as seen when entering kernel mode + * @buf: Array to store the fetched instruction + * + * Gets the linear address of the instruction and copies the instruction bytes + * to the buf. This function must be used in atomic context. + * + * Returns: + * + * - number of instruction bytes copied. + * - 0 if nothing was copied. + * - -EINVAL if the linear address of the instruction could not be calculated. + */ +int insn_fetch_from_user_inatomic(struct pt_regs *regs, unsigned char buf[MAX_INSN_SIZE]) +{ + unsigned long ip; + int not_copied; + + if (insn_get_effective_ip(regs, &ip)) + return -EINVAL; + + not_copied = __copy_from_user_inatomic(buf, (void __user *)ip, MAX_INSN_SIZE); + + return MAX_INSN_SIZE - not_copied; +} + +/** + * insn_decode_from_regs() - Decode an instruction + * @insn: Structure to store decoded instruction + * @regs: Structure with register values as seen when entering kernel mode + * @buf: Buffer containing the instruction bytes + * @buf_size: Number of instruction bytes available in buf + * + * Decodes the instruction provided in buf and stores the decoding results in + * insn. Also determines the correct address and operand sizes. + * + * Returns: + * + * True if instruction was decoded, False otherwise. + */ +bool insn_decode_from_regs(struct insn *insn, struct pt_regs *regs, + unsigned char buf[MAX_INSN_SIZE], int buf_size) +{ + int seg_defs; + + insn_init(insn, buf, buf_size, user_64bit_mode(regs)); + + /* + * Override the default operand and address sizes with what is specified + * in the code segment descriptor. The instruction decoder only sets + * the address size it to either 4 or 8 address bytes and does nothing + * for the operand bytes. This OK for most of the cases, but we could + * have special cases where, for instance, a 16-bit code segment + * descriptor is used. + * If there is an address override prefix, the instruction decoder + * correctly updates these values, even for 16-bit defaults. + */ + seg_defs = insn_get_code_seg_params(regs); + if (seg_defs == -EINVAL) + return false; + + insn->addr_bytes = INSN_CODE_SEG_ADDR_SZ(seg_defs); + insn->opnd_bytes = INSN_CODE_SEG_OPND_SZ(seg_defs); + + if (insn_get_length(insn)) + return false; + + if (buf_size < insn->length) + return false; + + return true; +} + +/** + * insn_decode_mmio() - Decode a MMIO instruction + * @insn: Structure to store decoded instruction + * @bytes: Returns size of memory operand + * + * Decodes instruction that used for Memory-mapped I/O. + * + * Returns: + * + * Type of the instruction. Size of the memory operand is stored in + * @bytes. If decode failed, INSN_MMIO_DECODE_FAILED returned. + */ +enum insn_mmio_type insn_decode_mmio(struct insn *insn, int *bytes) +{ + enum insn_mmio_type type = INSN_MMIO_DECODE_FAILED; + + *bytes = 0; + + if (insn_get_opcode(insn)) + return INSN_MMIO_DECODE_FAILED; + + switch (insn->opcode.bytes[0]) { + case 0x88: /* MOV m8,r8 */ + *bytes = 1; + fallthrough; + case 0x89: /* MOV m16/m32/m64, r16/m32/m64 */ + if (!*bytes) + *bytes = insn->opnd_bytes; + type = INSN_MMIO_WRITE; + break; + + case 0xc6: /* MOV m8, imm8 */ + *bytes = 1; + fallthrough; + case 0xc7: /* MOV m16/m32/m64, imm16/imm32/imm64 */ + if (!*bytes) + *bytes = insn->opnd_bytes; + type = INSN_MMIO_WRITE_IMM; + break; + + case 0x8a: /* MOV r8, m8 */ + *bytes = 1; + fallthrough; + case 0x8b: /* MOV r16/r32/r64, m16/m32/m64 */ + if (!*bytes) + *bytes = insn->opnd_bytes; + type = INSN_MMIO_READ; + break; + + case 0xa4: /* MOVS m8, m8 */ + *bytes = 1; + fallthrough; + case 0xa5: /* MOVS m16/m32/m64, m16/m32/m64 */ + if (!*bytes) + *bytes = insn->opnd_bytes; + type = INSN_MMIO_MOVS; + break; + + case 0x0f: /* Two-byte instruction */ + switch (insn->opcode.bytes[1]) { + case 0xb6: /* MOVZX r16/r32/r64, m8 */ + *bytes = 1; + fallthrough; + case 0xb7: /* MOVZX r32/r64, m16 */ + if (!*bytes) + *bytes = 2; + type = INSN_MMIO_READ_ZERO_EXTEND; + break; + + case 0xbe: /* MOVSX r16/r32/r64, m8 */ + *bytes = 1; + fallthrough; + case 0xbf: /* MOVSX r32/r64, m16 */ + if (!*bytes) + *bytes = 2; + type = INSN_MMIO_READ_SIGN_EXTEND; + break; + } + break; + } + + return type; +}

Add MMIO VC exception handling support to allow xapic mmio access for SEV-ES guests. In addition, add a PV interface for xapic MMIO guest accesses from outside of VC exception handler context.

Signed-off-by: Neeraj Upadhyay Neeraj.Upadhyay@amd.com --- .../selftests/kvm/include/x86/ex_regs.h | 2 +- tools/testing/selftests/kvm/include/x86/sev.h | 1 + tools/testing/selftests/kvm/lib/x86/sev.c | 259 +++++++++++++++++- 3 files changed, 257 insertions(+), 5 deletions(-)

diff --git a/tools/testing/selftests/kvm/include/x86/ex_regs.h b/tools/testing/selftests/kvm/include/x86/ex_regs.h index 172cfbb0a2d0..fd773b6614bc 100644 --- a/tools/testing/selftests/kvm/include/x86/ex_regs.h +++ b/tools/testing/selftests/kvm/include/x86/ex_regs.h @@ -4,7 +4,7 @@ */

#ifndef SELFTEST_KVM_EX_REGS_H -#define SELFTEST_KVM_EX_REG_H +#define SELFTEST_KVM_EX_REGS_H

struct ex_regs { uint64_t rax, rcx, rdx, rbx; diff --git a/tools/testing/selftests/kvm/include/x86/sev.h b/tools/testing/selftests/kvm/include/x86/sev.h index 5556ee891260..3756805197c3 100644 --- a/tools/testing/selftests/kvm/include/x86/sev.h +++ b/tools/testing/selftests/kvm/include/x86/sev.h @@ -155,4 +155,5 @@ void sev_es_ucall_port_write(uint32_t port, uint64_t data);

void sev_es_vc_handler(struct ex_regs *regs); void sev_es_pv_msr_rw(uint64_t msr, uint64_t *data, bool write); +void sev_es_pv_mmio_rw(uint32_t *reg_gpa, uint32_t *data, bool write); #endif /* SELFTEST_KVM_SEV_H */ diff --git a/tools/testing/selftests/kvm/lib/x86/sev.c b/tools/testing/selftests/kvm/lib/x86/sev.c index ff8f02b83871..16d6b21649d1 100644 --- a/tools/testing/selftests/kvm/lib/x86/sev.c +++ b/tools/testing/selftests/kvm/lib/x86/sev.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-only #include <stdint.h> #include <stdbool.h> +#include <asm/insn.h>

+#include "insn-eval.h" #include "sev.h" #include "linux/bitmap.h" #include "svm.h" @@ -14,6 +16,8 @@

#define SW_EXIT_CODE_IOIO 0x7b #define SW_EXIT_CODE_MSR 0x7c +#define SVM_VMGEXIT_MMIO_READ 0x80000001 +#define SVM_VMGEXIT_MMIO_WRITE 0x80000002

struct ghcb_entry { struct ghcb ghcb; @@ -362,10 +366,10 @@ static uint64_t setup_exitinfo1_portio(uint32_t port)

#define GHCB_MSR_REG_GPA_REQ 0x012 #define GHCB_MSR_REG_GPA_REQ_VAL(v) \ - /* GHCBData[63:12] */ \ - (((u64)((v) & GENMASK_ULL(51, 0)) << 12) | \ - /* GHCBData[11:0] */ \ - GHCB_MSR_REG_GPA_REQ) + /* GHCBData[63:12] */ \ + (((u64)((v) & GENMASK_ULL(51, 0)) << 12) | \ + /* GHCBData[11:0] */ \ + GHCB_MSR_REG_GPA_REQ)

static void register_ghcb_page(uint64_t ghcb_gpa) { @@ -471,6 +475,250 @@ static void sev_es_vc_msr_handler(struct ex_regs *regs) ghcb_free(entry); }

+static void __sev_es_hv_mmio_rw(struct ghcb_entry *entry, uint32_t *reg_gpa, + unsigned int bytes, bool write) +{ + uint64_t exitinfo1 = (uint64_t)reg_gpa; + struct ghcb *ghcb = &entry->ghcb; + int ret; + + if (write) + ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_MMIO_WRITE); + else + ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_MMIO_READ); + ghcb_set_sw_exit_info_1(ghcb, exitinfo1); + ghcb_set_sw_exit_info_2(ghcb, bytes); + ghcb_set_sw_scratch(ghcb, entry->gpa + offsetof(struct ghcb, shared_buffer)); + do_vmg_exit(entry->gpa); + ret = ghcb->save.sw_exit_info_1 & 0xffffffff; + __GUEST_ASSERT(!ret, "mmio %s failed, ret: %d", + write ? "write" : "read", ret); +} + +void sev_es_pv_mmio_rw(uint32_t *reg_gpa, uint32_t *data, bool write) +{ + struct ghcb_entry *entry; + struct ghcb *ghcb; + + entry = ghcb_alloc(); + ghcb = &entry->ghcb; + + register_ghcb_page(entry->gpa); + + if (write) + memcpy(&ghcb->shared_buffer, data, sizeof(*data)); + + __sev_es_hv_mmio_rw(entry, reg_gpa, sizeof(*data), write); + + if (!write) + memcpy(data, &ghcb->shared_buffer, sizeof(*data)); + + ghcb_free(entry); +} + +static void do_mmio(struct ghcb_entry *entry, struct ex_regs *regs, + struct insn *insn, unsigned int bytes, bool read) +{ + void *ref = insn_get_addr_ref(insn, regs); + + register_ghcb_page(entry->gpa); + __sev_es_hv_mmio_rw(entry, ref, bytes, !read); +} + +static int vc_write_mem(struct ex_regs *regs, struct insn *insn, + unsigned char *dst, unsigned char *buf, size_t size) +{ + uint8_t *buffer = (uint8_t *)buf; + + switch (size) { + case 1: { + uint8_t *target = (uint8_t *)dst; + + memcpy(target, buffer, 1); + break; + } + case 2: { + uint16_t *target = (uint16_t *)dst; + + memcpy(target, buffer, 2); + break; + } + case 4: { + uint32_t *target = (uint32_t *)dst; + + memcpy(target, buffer, 4); + break; + } + case 8: { + uint64_t *target = (uint64_t *)dst; + + memcpy(target, buffer, 8); + break; + } + default: + return -1; + } + + return 0; +} + +static int vc_read_mem(struct ex_regs *regs, struct insn *insn, + unsigned char *src, unsigned char *buf, size_t size) +{ + switch (size) { + case 1: { + uint8_t *s = (uint8_t *)src; + + memcpy(buf, s, 1); + break; + } + case 2: { + uint16_t *s = (uint16_t *)src; + + memcpy(buf, s, 1); + break; + } + case 4: { + uint32_t *s = (uint32_t *)src; + + memcpy(buf, s, 1); + break; + } + case 8: { + uint64_t *s = (uint64_t *)src; + + memcpy(buf, s, 1); + break; + } + default: + return -1; + } + + return 0; +} + + +static int vc_handle_mmio_movs(struct ex_regs *regs, struct insn *insn, unsigned int bytes) +{ + unsigned long ds_base, es_base; + unsigned char *src, *dst; + unsigned char buffer[8]; + int ret; + bool rep; + int off; + + ds_base = insn_get_seg_base(regs, INAT_SEG_REG_DS); + es_base = insn_get_seg_base(regs, INAT_SEG_REG_ES); + + if (ds_base == -1L || es_base == -1L) + return -1; + + src = ds_base + (unsigned char *)regs->rsi; + dst = es_base + (unsigned char *)regs->rdi; + + ret = vc_read_mem(regs, insn, src, buffer, bytes); + if (ret != 0) + return ret; + + ret = vc_write_mem(regs, insn, dst, buffer, bytes); + if (ret != 0) + return ret; + +#define X86_EFLAGS_DF (1UL << 10) + if (regs->rflags & X86_EFLAGS_DF) + off = -bytes; + else + off = bytes; + + regs->rsi += off; + regs->rdi += off; + + rep = insn_has_rep_prefix(insn); + if (rep) + regs->rcx -= 1; + + if (!rep || regs->rcx == 0) + return 0; + else + return 1; +} + +static void sev_es_vc_mmio_handler(struct ex_regs *regs) +{ + char buffer[MAX_INSN_SIZE]; + struct ghcb_entry *entry; + enum insn_mmio_type mmio; + unsigned long *reg_data; + unsigned int bytes; + struct ghcb *ghcb; + uint8_t sign_byte; + struct insn insn; + int ret; + + memcpy(buffer, (uint8_t *)regs->rip, MAX_INSN_SIZE); + ret = insn_decode(&insn, buffer, MAX_INSN_SIZE, INSN_MODE_64); + + if (ret < 0) + __GUEST_ASSERT(0, "Instruction decode failed, ret: %d\n", ret); + + mmio = insn_decode_mmio(&insn, (int *)&bytes); + __GUEST_ASSERT(!(mmio == INSN_MMIO_DECODE_FAILED), " MMIO decode failed\n"); + + if (mmio != INSN_MMIO_WRITE_IMM && mmio != INSN_MMIO_MOVS) { + reg_data = insn_get_modrm_reg_ptr(&insn, regs); + __GUEST_ASSERT(reg_data, "insn_get_modrm_reg_ptr failed\n"); + } + + entry = ghcb_alloc(); + ghcb = &entry->ghcb; + + switch (mmio) { + case INSN_MMIO_WRITE: + memcpy(ghcb->shared_buffer, reg_data, bytes); + do_mmio(entry, regs, &insn, bytes, false); + break; + case INSN_MMIO_WRITE_IMM: + memcpy(ghcb->shared_buffer, insn.immediate1.bytes, bytes); + do_mmio(entry, regs, &insn, bytes, false); + break; + case INSN_MMIO_READ: + do_mmio(entry, regs, &insn, bytes, true); + if (bytes == 4) + *reg_data = 0; + memcpy(reg_data, ghcb->shared_buffer, bytes); + break; + case INSN_MMIO_READ_ZERO_EXTEND: + do_mmio(entry, regs, &insn, bytes, true); + memset(reg_data, 0, insn.opnd_bytes); + memcpy(reg_data, ghcb->shared_buffer, bytes); + break; + case INSN_MMIO_READ_SIGN_EXTEND: + do_mmio(entry, regs, &insn, bytes, true); + if (bytes == 1) { + uint8_t *val = (uint8_t *)ghcb->shared_buffer; + + sign_byte = (*val & 0x80) ? 0xff : 0x00; + } else { + uint16_t *val = (uint16_t *)ghcb->shared_buffer; + + sign_byte = (*val & 0x8000) ? 0xff : 0x00; + } + + /* Sign extend based on operand size */ + memset(reg_data, sign_byte, insn.opnd_bytes); + memcpy(reg_data, ghcb->shared_buffer, bytes); + break; + case INSN_MMIO_MOVS: + ret = vc_handle_mmio_movs(regs, &insn, bytes); + break; + default: + break; + } + + ghcb_free(entry); + regs->rip += insn.length; +} + void sev_es_vc_handler(struct ex_regs *regs) { uint64_t exit_code = regs->error_code; @@ -481,6 +729,9 @@ void sev_es_vc_handler(struct ex_regs *regs) /* rdmsr/wrmsr instruction size = 2 */ regs->rip += 2; break; + case SVM_EXIT_NPF: + sev_es_vc_mmio_handler(regs); + break; default: __GUEST_ASSERT(0, "No VC handler\n"); }

Now with xapic/x2apic acceses being supported for SEV-ES and SNP guests, add support for testing these VMs in xapic_state_test.

Signed-off-by: Neeraj Upadhyay Neeraj.Upadhyay@amd.com --- .../selftests/kvm/x86/xapic_state_test.c | 117 ++++++++++++++++-- 1 file changed, 109 insertions(+), 8 deletions(-)

diff --git a/tools/testing/selftests/kvm/x86/xapic_state_test.c b/tools/testing/selftests/kvm/x86/xapic_state_test.c index 88bcca188799..efbc98f04d45 100644 --- a/tools/testing/selftests/kvm/x86/xapic_state_test.c +++ b/tools/testing/selftests/kvm/x86/xapic_state_test.c @@ -9,6 +9,7 @@ #include "kvm_util.h" #include "processor.h" #include "test_util.h" +#include "sev.h"

struct xapic_vcpu { struct kvm_vcpu *vcpu; @@ -160,6 +161,27 @@ static void __test_apic_id(struct kvm_vcpu *vcpu, uint64_t apic_base) expected, apic_id); }

+static inline bool is_sev_vm_type(int type) +{ + return type == KVM_X86_SEV_VM || + type == KVM_X86_SEV_ES_VM || + type == KVM_X86_SNP_VM; +} + +static inline uint64_t get_sev_policy(int vm_type) +{ + switch (vm_type) { + case KVM_X86_SEV_VM: + return SEV_POLICY_NO_DBG; + case KVM_X86_SEV_ES_VM: + return SEV_POLICY_ES; + case KVM_X86_SNP_VM: + return snp_default_policy(); + default: + return 0; + } +} + /* * Verify that KVM switches the APIC_ID between xAPIC and x2APIC when userspace * stuffs MSR_IA32_APICBASE. Setting the APIC_ID when x2APIC is enabled and @@ -168,16 +190,22 @@ static void __test_apic_id(struct kvm_vcpu *vcpu, uint64_t apic_base) * attempted to transition from x2APIC to xAPIC without disabling the APIC is * architecturally disallowed. */ -static void test_apic_id(void) +static void test_apic_id(int vm_type) { const uint32_t NR_VCPUS = 3; struct kvm_vcpu *vcpus[NR_VCPUS]; uint64_t apic_base; struct kvm_vm *vm; int i; + struct vm_shape shape = { + .mode = VM_MODE_DEFAULT, + .type = vm_type, + };

- vm = vm_create_with_vcpus(NR_VCPUS, NULL, vcpus); + vm = __vm_create_with_vcpus(shape, NR_VCPUS, 0, NULL, vcpus); vm_enable_cap(vm, KVM_CAP_X2APIC_API, KVM_X2APIC_API_USE_32BIT_IDS); + if (is_sev_vm(vm)) + vm_sev_launch(vm, get_sev_policy(vm_type), NULL);

for (i = 0; i < NR_VCPUS; i++) { apic_base = vcpu_get_msr(vcpus[i], MSR_IA32_APICBASE); @@ -195,15 +223,21 @@ static void test_apic_id(void) kvm_vm_free(vm); }

-static void test_x2apic_id(void) +static void test_x2apic_id(int vm_type) { struct kvm_lapic_state lapic = {}; struct kvm_vcpu *vcpu; struct kvm_vm *vm; int i; + bool is_sev = is_sev_vm_type(vm_type);

- vm = vm_create_with_one_vcpu(&vcpu, NULL); + if (is_sev) + vm = vm_sev_create_with_one_vcpu(vm_type, NULL, &vcpu); + else + vm = vm_create_with_one_vcpu(&vcpu, NULL); vcpu_set_msr(vcpu, MSR_IA32_APICBASE, MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE); + if (is_sev) + vm_sev_launch(vm, get_sev_policy(vm_type), NULL);

/* * Try stuffing a modified x2APIC ID, KVM should ignore the value and @@ -222,6 +256,46 @@ static void test_x2apic_id(void) kvm_vm_free(vm); }

+void get_cmdline_args(int argc, char *argv[], int *vm_type) +{ + for (;;) { + int opt = getopt(argc, argv, "t:"); + + if (opt == -1) + break; + switch (opt) { + case 't': + *vm_type = parse_size(optarg); + switch (*vm_type) { + case KVM_X86_DEFAULT_VM: + break; + case KVM_X86_SEV_VM: + TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SEV)); + break; + case KVM_X86_SEV_ES_VM: + TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SEV_ES)); + break; + case KVM_X86_SNP_VM: + TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SNP)); + break; + default: + TEST_ASSERT(false, "Unsupported VM type :%d", + *vm_type); + } + break; + default: + TEST_ASSERT(false, + "Usage: -t <vm type>. Default is %d.\n" + "Supported values:\n" + "0 - default\n" + "2 - SEV\n" + "3 - SEV-ES\n" + "4 - SNP", + KVM_X86_DEFAULT_VM); + } + } +} + int main(int argc, char *argv[]) { struct xapic_vcpu x = { @@ -229,8 +303,24 @@ int main(int argc, char *argv[]) .is_x2apic = true, }; struct kvm_vm *vm; + int vm_type = KVM_X86_DEFAULT_VM; + bool is_sev; + + get_cmdline_args(argc, argv, &vm_type); + is_sev = is_sev_vm_type(vm_type); + + if (is_sev) + vm = vm_sev_create_with_one_vcpu(vm_type, x2apic_guest_code, + &x.vcpu); + else + vm = vm_create_with_one_vcpu(&x.vcpu, x2apic_guest_code); + + if (is_sev_es_vm(vm)) + vm_install_exception_handler(vm, 29, sev_es_vc_handler); + + if (is_sev) + vm_sev_launch(vm, get_sev_policy(vm_type), NULL);

- vm = vm_create_with_one_vcpu(&x.vcpu, x2apic_guest_code); test_icr(&x); kvm_vm_free(vm);

@@ -239,7 +329,15 @@ int main(int argc, char *argv[]) * the guest in order to test AVIC. KVM disallows changing CPUID after * KVM_RUN and AVIC is disabled if _any_ vCPU is allowed to use x2APIC. */ - vm = vm_create_with_one_vcpu(&x.vcpu, xapic_guest_code); + if (is_sev) + vm = vm_sev_create_with_one_vcpu(vm_type, xapic_guest_code, + &x.vcpu); + else + vm = vm_create_with_one_vcpu(&x.vcpu, xapic_guest_code); + + if (is_sev_es_vm(vm)) + vm_install_exception_handler(vm, 29, sev_es_vc_handler); + x.is_x2apic = false;

/* @@ -254,9 +352,12 @@ int main(int argc, char *argv[]) vcpu_clear_cpuid_feature(x.vcpu, X86_FEATURE_X2APIC);

virt_pg_map(vm, APIC_DEFAULT_GPA, APIC_DEFAULT_GPA); + if (is_sev) + vm_sev_launch(vm, get_sev_policy(vm_type), NULL); + test_icr(&x); kvm_vm_free(vm);

- test_apic_id(); - test_x2apic_id(); + test_apic_id(vm_type); + test_x2apic_id(vm_type); }

Add support to test SEV VMs - SEV, SEV-ES, SEV-SNP in both xapic and x2apic modes. Convert the stats page to a shared page so that it is accesible from host.

Signed-off-by: Neeraj Upadhyay Neeraj.Upadhyay@amd.com --- .../testing/selftests/kvm/include/kvm_util.h | 1 + tools/testing/selftests/kvm/lib/kvm_util.c | 19 ++++ .../selftests/kvm/x86/xapic_ipi_test.c | 94 ++++++++++++++++--- 3 files changed, 100 insertions(+), 14 deletions(-)

diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index bd963ff49bf0..a160e1ac7cbc 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -609,6 +609,7 @@ vm_vaddr_t vm_vaddr_alloc_pages_shared(struct kvm_vm *vm, int nr_pages); vm_vaddr_t __vm_vaddr_alloc_page(struct kvm_vm *vm, enum kvm_mem_region_type type); vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm); +vm_vaddr_t vm_vaddr_alloc_page_shared(struct kvm_vm *vm);

void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, unsigned int npages); diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 4f3240976f6c..34e586d4fec4 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -1500,6 +1500,25 @@ vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm) return vm_vaddr_alloc_pages(vm, 1); }

+/* + * VM Virtual Address Allocate shared Page + * + * Input Args: + * vm - Virtual Machine + * + * Output Args: None + * + * Return: + * Starting guest virtual address + * + * Allocates at least one system page (in shared state) worth of bytes within the + * virtual address space of the vm. + */ +vm_vaddr_t vm_vaddr_alloc_page_shared(struct kvm_vm *vm) +{ + return vm_vaddr_alloc_pages_shared(vm, 1); +} + /* * Map a range of VM virtual address to the VM's physical address * diff --git a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c index 024514089766..3a54d828dc69 100644 --- a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c +++ b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c @@ -30,6 +30,7 @@ #include "processor.h" #include "test_util.h" #include "vmx.h" +#include "sev.h"

/* Default running time for the test */ #define DEFAULT_RUN_SECS 3 @@ -48,7 +49,7 @@ * Incremented in the IPI handler. Provides evidence to the sender that the IPI * arrived at the destination */ -static volatile uint64_t ipis_rcvd; +static volatile uint64_t *ipis_rcvd;

static bool x2apic;

@@ -93,6 +94,7 @@ struct test_data_page { * to determine whether APIC access exits are working. */ uint32_t halter_lvr; + uint64_t ipis_rcvd; };

struct thread_params { @@ -141,7 +143,7 @@ static void halter_guest_code(struct test_data_page *data) */ static void guest_ipi_handler(struct ex_regs *regs) { - ipis_rcvd++; + (*ipis_rcvd)++; apic_write_reg(APIC_EOI, 77); }

@@ -175,7 +177,7 @@ static void sender_guest_code(struct test_data_page *data)

last_wake_count = data->wake_count; last_hlt_count = data->hlt_count; - last_ipis_rcvd_count = ipis_rcvd; + last_ipis_rcvd_count = *ipis_rcvd; for (;;) { /* * Send IPI to halter vCPU. @@ -200,19 +202,19 @@ static void sender_guest_code(struct test_data_page *data) */ tsc_start = rdtsc(); while (rdtsc() - tsc_start < 2000000000) { - if ((ipis_rcvd != last_ipis_rcvd_count) && + if ((*ipis_rcvd != last_ipis_rcvd_count) && (data->wake_count != last_wake_count) && (data->hlt_count != last_hlt_count)) break; }

- GUEST_ASSERT((ipis_rcvd != last_ipis_rcvd_count) && + GUEST_ASSERT((*ipis_rcvd != last_ipis_rcvd_count) && (data->wake_count != last_wake_count) && (data->hlt_count != last_hlt_count));

last_wake_count = data->wake_count; last_hlt_count = data->hlt_count; - last_ipis_rcvd_count = ipis_rcvd; + last_ipis_rcvd_count = *ipis_rcvd; } }

@@ -383,10 +385,10 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, }

void get_cmdline_args(int argc, char *argv[], int *run_secs, - bool *migrate, int *delay_usecs, bool *x2apic) + bool *migrate, int *delay_usecs, bool *x2apic, int *vm_type) { for (;;) { - int opt = getopt(argc, argv, "s:d:v:me:"); + int opt = getopt(argc, argv, "s:d:v:me:t:");

if (opt == -1) break; @@ -403,6 +405,25 @@ void get_cmdline_args(int argc, char *argv[], int *run_secs, case 'e': *x2apic = parse_size(optarg) == 1; break; + case 't': + *vm_type = parse_size(optarg); + switch (*vm_type) { + case KVM_X86_DEFAULT_VM: + break; + case KVM_X86_SEV_VM: + TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SEV)); + break; + case KVM_X86_SEV_ES_VM: + TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SEV_ES)); + break; + case KVM_X86_SNP_VM: + TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SNP)); + break; + default: + TEST_ASSERT(false, "Unsupported VM type :%d", + *vm_type); + } + break; default: TEST_ASSERT(false, "Usage: -s <runtime seconds>. Default is %d seconds.\n" @@ -411,12 +432,39 @@ void get_cmdline_args(int argc, char *argv[], int *run_secs, "-d <delay microseconds> - delay between migrate_pages() calls." " Default is %d microseconds.\n" "-e <apic mode> - APIC mode 0 - xapic , 1 - x2apic" - " Default is xAPIC.\n", - DEFAULT_RUN_SECS, DEFAULT_DELAY_USECS); + " Default is xAPIC.\n" + "-t <vm type>. Default is %d.\n" + "Supported values:\n" + "0 - default\n" + "2 - SEV\n" + "3 - SEV-ES\n" + "4 - SNP", + DEFAULT_RUN_SECS, DEFAULT_DELAY_USECS, KVM_X86_DEFAULT_VM); } } }

+static inline bool is_sev_vm_type(int type) +{ + return type == KVM_X86_SEV_VM || + type == KVM_X86_SEV_ES_VM || + type == KVM_X86_SNP_VM; +} + +static inline uint64_t get_sev_policy(int vm_type) +{ + switch (vm_type) { + case KVM_X86_SEV_VM: + return SEV_POLICY_NO_DBG; + case KVM_X86_SEV_ES_VM: + return SEV_POLICY_ES; + case KVM_X86_SNP_VM: + return snp_default_policy(); + default: + return 0; + } +} + int main(int argc, char *argv[]) { int r; @@ -431,8 +479,13 @@ int main(int argc, char *argv[]) struct thread_params params[2]; struct kvm_vm *vm; uint64_t *pipis_rcvd; + int vm_type = KVM_X86_DEFAULT_VM; + bool is_sev; + + get_cmdline_args(argc, argv, &run_secs, &migrate, &delay_usecs, + &x2apic, &vm_type); + is_sev = is_sev_vm_type(vm_type);

- get_cmdline_args(argc, argv, &run_secs, &migrate, &delay_usecs, &x2apic); if (x2apic) migrate = 0;

@@ -441,9 +494,15 @@ int main(int argc, char *argv[]) if (delay_usecs <= 0) delay_usecs = DEFAULT_DELAY_USECS;

- vm = vm_create_with_one_vcpu(&params[0].vcpu, halter_guest_code); + if (is_sev) + vm = vm_sev_create_with_one_vcpu(vm_type, halter_guest_code, + &params[0].vcpu); + else + vm = vm_create_with_one_vcpu(&params[0].vcpu, halter_guest_code);

vm_install_exception_handler(vm, IPI_VECTOR, guest_ipi_handler); + if (is_sev_es_vm(vm)) + vm_install_exception_handler(vm, 29, sev_es_vc_handler);

sync_global_to_guest(vm, x2apic); if (!x2apic) @@ -451,8 +510,10 @@ int main(int argc, char *argv[])

params[1].vcpu = vm_vcpu_add(vm, 1, sender_guest_code);

- test_data_page_vaddr = vm_vaddr_alloc_page(vm); + test_data_page_vaddr = vm_vaddr_alloc_page_shared(vm); data = addr_gva2hva(vm, test_data_page_vaddr); + if (is_sev_snp_vm(vm)) + vm_mem_set_shared(vm, addr_hva2gpa(vm, data), getpagesize()); memset(data, 0, sizeof(*data)); params[0].data = data; params[1].data = data; @@ -460,10 +521,15 @@ int main(int argc, char *argv[]) vcpu_args_set(params[0].vcpu, 1, test_data_page_vaddr); vcpu_args_set(params[1].vcpu, 1, test_data_page_vaddr);

- pipis_rcvd = (uint64_t *)addr_gva2hva(vm, (uint64_t)&ipis_rcvd); + ipis_rcvd = &((struct test_data_page *)test_data_page_vaddr)->ipis_rcvd; + sync_global_to_guest(vm, ipis_rcvd); + pipis_rcvd = (uint64_t *)addr_gva2hva(vm, (uint64_t)ipis_rcvd); params[0].pipis_rcvd = pipis_rcvd; params[1].pipis_rcvd = pipis_rcvd;

+ if (is_sev) + vm_sev_launch(vm, get_sev_policy(vm_type), NULL); + /* Start halter vCPU thread and wait for it to execute first HLT. */ r = pthread_create(&threads[0], NULL, vcpu_thread, &params[0]); TEST_ASSERT(r == 0,

Add #VC exception handling support for unaccelerated msrs reads/writes for Secure AVIC guests.

Signed-off-by: Neeraj Upadhyay Neeraj.Upadhyay@amd.com --- .../testing/selftests/kvm/include/x86/apic.h | 1 + .../testing/selftests/kvm/include/x86/savic.h | 1 + tools/testing/selftests/kvm/lib/x86/savic.c | 145 ++++++++++++++++++ 3 files changed, 147 insertions(+)

diff --git a/tools/testing/selftests/kvm/include/x86/apic.h b/tools/testing/selftests/kvm/include/x86/apic.h index 6ba5d0545bf8..aa3a5d54c404 100644 --- a/tools/testing/selftests/kvm/include/x86/apic.h +++ b/tools/testing/selftests/kvm/include/x86/apic.h @@ -33,6 +33,7 @@ #define APIC_SPIV 0xF0 #define APIC_SPIV_FOCUS_DISABLED (1 << 9) #define APIC_SPIV_APIC_ENABLED (1 << 8) +#define APIC_TMR 0x180 #define APIC_IRR 0x200 #define APIC_ICR 0x300 #define APIC_LVTCMCI 0x2f0 diff --git a/tools/testing/selftests/kvm/include/x86/savic.h b/tools/testing/selftests/kvm/include/x86/savic.h index 1ab92dad00c1..238d7450ab6e 100644 --- a/tools/testing/selftests/kvm/include/x86/savic.h +++ b/tools/testing/selftests/kvm/include/x86/savic.h @@ -16,4 +16,5 @@ void savic_hv_write_reg(uint32_t reg, uint64_t val); uint64_t savic_hv_read_reg(uint32_t reg); void savic_enable(void); int savic_nr_pages_required(uint64_t page_size); +void savic_vc_handler(struct ex_regs *regs); #endif diff --git a/tools/testing/selftests/kvm/lib/x86/savic.c b/tools/testing/selftests/kvm/lib/x86/savic.c index f4a765b6040a..141d31637e51 100644 --- a/tools/testing/selftests/kvm/lib/x86/savic.c +++ b/tools/testing/selftests/kvm/lib/x86/savic.c @@ -42,6 +42,8 @@ enum lapic_lvt_entry { #define MSR_AMD64_SECURE_AVIC_EN_BIT 0 #define MSR_AMD64_SECURE_AVIC_ALLOWED_NMI_BIT 1

+#define SVM_EXIT_AVIC_UNACCELERATED_ACCESS 0x402 + /* * Initial pool of guest apic backing page. */ @@ -204,3 +206,146 @@ void savic_enable(void) "SAVIC Control msr unexpected val : 0x%lx, expected : 0x%lx", savic_ctrl_msr_val, exp_msr_val); } + +static bool savic_reg_access_is_trapped(uint32_t reg) +{ + switch (reg) { + case APIC_ID: + case APIC_TASKPRI: + case APIC_EOI: + case APIC_LDR: + case APIC_SPIV: + case APIC_ICR: + case APIC_ICR2: + case APIC_LVTT: + case APIC_LVTTHMR: + case APIC_LVTPC: + case APIC_LVT0: + case APIC_LVT1: + case APIC_LVTERR: + case APIC_TMICT: + case APIC_TDCR: + return true; + case APIC_LVR: + case APIC_PROCPRI: + case APIC_TMR: + case APIC_IRR ... APIC_IRR + 0x70: + case APIC_TMCCT: + return false; + default: + return false; + } +} + +static void savic_unaccel_apic_msrs_read(struct guest_apic_page *apic_page, + uint32_t reg, uint64_t *val) +{ + switch (reg) { + case APIC_TMICT: + case APIC_TMCCT: + case APIC_TDCR: + case APIC_LVTT: + case APIC_LVTTHMR: + case APIC_LVTPC: + case APIC_LVT0: + case APIC_LVT1: + case APIC_LVTERR: + *val = savic_hv_read_reg(reg); + break; + default: + __GUEST_ASSERT(0, "Unexpected unaccelerated read trap for reg: %x\n", reg); + } +} + +static void savic_unaccel_apic_msrs_write(struct guest_apic_page *apic_page, + uint32_t reg, uint64_t val) +{ + switch (reg) { + /* + * APIC_ID value is in sync between guest apic backing page and + * hv. + * LVT* registers and APIC timer register updates are propagated to hv. + */ + case APIC_ID: + case APIC_LVTT: + case APIC_LVTTHMR: + case APIC_LVTPC: + case APIC_LVT0: + case APIC_LVT1: + case APIC_LVTERR: + case APIC_TMICT: + case APIC_TMCCT: + case APIC_TDCR: + savic_write_reg(apic_page, reg, val); + savic_hv_write_reg(reg, val); + break; + /* + * LDR is derived in hv from APIC_ID. + * TPR, SPIV, IRR information is not propagated to hv. + */ + case APIC_LDR: + case APIC_TASKPRI: + case APIC_SPIV: + case APIC_IRR: + savic_write_reg(apic_page, reg, val); + break; + /* + * EOI write need to be propagated to hv for level-triggered + * interrupts. + */ + case APIC_EOI: + savic_hv_write_reg(reg, val); + break; + default: + __GUEST_ASSERT(0, "Write not permitted for reg: %x\n", reg); + } +} + +static void handle_savic_unaccel_access(struct ex_regs *regs) +{ + bool write;; + uint64_t msr = regs->rcx; + uint32_t reg = (msr - APIC_BASE_MSR) << 4; + struct guest_apic_page *apic_page; + uint64_t low = regs->rax; + uint64_t high = regs->rdx; + uint64_t val = 0; + + apic_page = &apic_page_pool->guest_apic_page[x2apic_read_reg(APIC_ID)]; + + switch (msr) { + case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff: + if (savic_reg_access_is_trapped(reg)) + write = *((uint8_t *)regs->rip - 1) == 0x30; + else + write = *((uint8_t *)regs->rip + 1) == 0x30; + if (write) { + savic_unaccel_apic_msrs_write(apic_page, reg, + high << 32 | low); + } else { + savic_unaccel_apic_msrs_read(apic_page, reg, &val); + regs->rax = val & ((1ULL << 32) - 1); + regs->rdx = val >> 32; + } + if (!savic_reg_access_is_trapped(reg)) + regs->rip += 2; + break; + default: + __GUEST_ASSERT(0, "Unknown unaccelerated msr: %lx\n", msr); + break; + } +} + +void savic_vc_handler(struct ex_regs *regs) +{ + uint64_t exit_code = regs->error_code; + + switch (exit_code) { + case SVM_EXIT_AVIC_UNACCELERATED_ACCESS: + handle_savic_unaccel_access(regs); + break; + default: + sev_es_vc_handler(regs); + break; + } +}

Add provision to pass custom args to kvm_arch_vm_post_create(). This will be used to pass sev init args (vmsa features) for SEV VMs.

Signed-off-by: Neeraj Upadhyay Neeraj.Upadhyay@amd.com --- .../testing/selftests/kvm/include/kvm_util.h | 9 +++- tools/testing/selftests/kvm/include/x86/sev.h | 3 ++ tools/testing/selftests/kvm/lib/kvm_util.c | 51 +++++++++++++------ .../testing/selftests/kvm/lib/x86/processor.c | 6 ++- tools/testing/selftests/kvm/lib/x86/sev.c | 13 +++-- tools/testing/selftests/kvm/s390/cmma_test.c | 2 +- 6 files changed, 62 insertions(+), 22 deletions(-)

diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index a160e1ac7cbc..7f97bade5797 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -901,6 +901,9 @@ static inline vm_paddr_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, struct kvm_vm *____vm_create(struct vm_shape shape); struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus, uint64_t nr_extra_pages); +struct kvm_vm *__vm_create_with_args(struct vm_shape shape, + uint32_t nr_runnable_vcpus, uint64_t nr_extra_pages, + void *args);

static inline struct kvm_vm *vm_create_barebones(void) { @@ -925,6 +928,10 @@ static inline struct kvm_vm *vm_create(uint32_t nr_runnable_vcpus) struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus, uint64_t extra_mem_pages, void *guest_code, struct kvm_vcpu *vcpus[]); +struct kvm_vm *___vm_create_with_vcpus(struct vm_shape shape, + uint32_t nr_vcpus, uint64_t extra_mem_pages, + void *guest_code, struct kvm_vcpu *vcpus[], + void *args);

static inline struct kvm_vm *vm_create_with_vcpus(uint32_t nr_vcpus, void *guest_code, @@ -1141,7 +1148,7 @@ static inline int __vm_disable_nx_huge_pages(struct kvm_vm *vm) */ void kvm_selftest_arch_init(void);

-void kvm_arch_vm_post_create(struct kvm_vm *vm); +void kvm_arch_vm_post_create(struct kvm_vm *vm, void *args);

bool vm_is_gpa_protected(struct kvm_vm *vm, vm_paddr_t paddr);

diff --git a/tools/testing/selftests/kvm/include/x86/sev.h b/tools/testing/selftests/kvm/include/x86/sev.h index 3756805197c3..ffb5ded0a35a 100644 --- a/tools/testing/selftests/kvm/include/x86/sev.h +++ b/tools/testing/selftests/kvm/include/x86/sev.h @@ -52,6 +52,9 @@ void snp_vm_launch_start(struct kvm_vm *vm, uint64_t policy); void snp_vm_launch_update(struct kvm_vm *vm); void snp_vm_launch_finish(struct kvm_vm *vm);

+struct kvm_vm *_vm_sev_create_with_one_vcpu(uint32_t type, void *guest_code, + struct kvm_vcpu **cpu, + void *init_args); struct kvm_vm *vm_sev_create_with_one_vcpu(uint32_t type, void *guest_code, struct kvm_vcpu **cpu); void vm_sev_launch(struct kvm_vm *vm, uint64_t policy, uint8_t *measurement); diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 34e586d4fec4..93b8e2ccc7b3 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -408,8 +408,8 @@ static uint64_t vm_nr_pages_required(struct vm_shape shape, return vm_adjust_num_guest_pages(shape.mode, nr_pages); }

-struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus, - uint64_t nr_extra_pages) +static struct kvm_vm *___vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus, + uint64_t nr_extra_pages, void *args) { uint64_t nr_pages = vm_nr_pages_required(shape, nr_runnable_vcpus, nr_extra_pages); @@ -447,7 +447,37 @@ struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus, guest_rng = new_guest_random_state(guest_random_seed); sync_global_to_guest(vm, guest_rng);

- kvm_arch_vm_post_create(vm); + kvm_arch_vm_post_create(vm, args); + + return vm; +} + +struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus, + uint64_t nr_extra_pages) +{ + return ___vm_create(shape, nr_runnable_vcpus, nr_extra_pages, NULL); +} + +struct kvm_vm *__vm_create_with_args(struct vm_shape shape, uint32_t nr_runnable_vcpus, + uint64_t nr_extra_pages, void *args) +{ + return ___vm_create(shape, nr_runnable_vcpus, nr_extra_pages, args); +} + +struct kvm_vm *___vm_create_with_vcpus(struct vm_shape shape, + uint32_t nr_vcpus, uint64_t extra_mem_pages, + void *guest_code, struct kvm_vcpu *vcpus[], + void *args) +{ + struct kvm_vm *vm; + int i; + + TEST_ASSERT(!nr_vcpus || vcpus, "Must provide vCPU array"); + + vm = ___vm_create(shape, nr_vcpus, extra_mem_pages, args); + + for (i = 0; i < nr_vcpus; ++i) + vcpus[i] = vm_vcpu_add(vm, i, guest_code);

return vm; } @@ -475,17 +505,8 @@ struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus, uint64_t extra_mem_pages, void *guest_code, struct kvm_vcpu *vcpus[]) { - struct kvm_vm *vm; - int i; - - TEST_ASSERT(!nr_vcpus || vcpus, "Must provide vCPU array"); - - vm = __vm_create(shape, nr_vcpus, extra_mem_pages); - - for (i = 0; i < nr_vcpus; ++i) - vcpus[i] = vm_vcpu_add(vm, i, guest_code); - - return vm; + return ___vm_create_with_vcpus(shape, nr_vcpus, extra_mem_pages, guest_code, + vcpus, NULL); }

struct kvm_vm *__vm_create_shape_with_one_vcpu(struct vm_shape shape, @@ -2270,7 +2291,7 @@ void __vm_get_stat(struct kvm_vm *vm, const char *stat_name, uint64_t *data, } }

-__weak void kvm_arch_vm_post_create(struct kvm_vm *vm) +__weak void kvm_arch_vm_post_create(struct kvm_vm *vm, void *args) { }

diff --git a/tools/testing/selftests/kvm/lib/x86/processor.c b/tools/testing/selftests/kvm/lib/x86/processor.c index 2d6105b1f610..09474be27986 100644 --- a/tools/testing/selftests/kvm/lib/x86/processor.c +++ b/tools/testing/selftests/kvm/lib/x86/processor.c @@ -9,6 +9,7 @@ #include "processor.h" #include "sev.h" #include "apic.h" +#include "savic.h"

#ifndef NUM_INTERRUPTS #define NUM_INTERRUPTS 256 @@ -631,7 +632,7 @@ void assert_on_unhandled_exception(struct kvm_vcpu *vcpu) REPORT_GUEST_ASSERT(uc); }

-void kvm_arch_vm_post_create(struct kvm_vm *vm) +void kvm_arch_vm_post_create(struct kvm_vm *vm, void *sev_init_args) { int r;

@@ -648,7 +649,8 @@ void kvm_arch_vm_post_create(struct kvm_vm *vm) if (is_sev_vm(vm)) { struct kvm_sev_init init = { 0 };

- vm_sev_ioctl(vm, KVM_SEV_INIT2, &init); + vm_sev_ioctl(vm, KVM_SEV_INIT2, sev_init_args ? + (struct kvm_sev_init *)sev_init_args : &init); }

r = __vm_ioctl(vm, KVM_GET_TSC_KHZ, NULL); diff --git a/tools/testing/selftests/kvm/lib/x86/sev.c b/tools/testing/selftests/kvm/lib/x86/sev.c index 518e30275960..7675950efe56 100644 --- a/tools/testing/selftests/kvm/lib/x86/sev.c +++ b/tools/testing/selftests/kvm/lib/x86/sev.c @@ -291,8 +291,9 @@ void snp_vm_launch_finish(struct kvm_vm *vm) vm_sev_ioctl(vm, KVM_SEV_SNP_LAUNCH_FINISH, &launch_finish); }

-struct kvm_vm *vm_sev_create_with_one_vcpu(uint32_t type, void *guest_code, - struct kvm_vcpu **cpu) +struct kvm_vm *_vm_sev_create_with_one_vcpu(uint32_t type, void *guest_code, + struct kvm_vcpu **cpu, + void *init_args) { struct vm_shape shape = { .mode = VM_MODE_DEFAULT, @@ -301,7 +302,7 @@ struct kvm_vm *vm_sev_create_with_one_vcpu(uint32_t type, void *guest_code, struct kvm_vm *vm; struct kvm_vcpu *cpus[1];

- vm = __vm_create_with_vcpus(shape, 1, 0, guest_code, cpus); + vm = ___vm_create_with_vcpus(shape, 1, 0, guest_code, cpus, init_args); *cpu = cpus[0];

return vm; @@ -319,6 +320,12 @@ static bool is_savic_enabled(void) return supported_vmsa_features & BIT_ULL(SVM_FEAT_SECURE_AVIC); }

+struct kvm_vm *vm_sev_create_with_one_vcpu(uint32_t type, void *guest_code, + struct kvm_vcpu **cpu) +{ + return _vm_sev_create_with_one_vcpu(type, guest_code, cpu, NULL); +} + void vm_sev_launch(struct kvm_vm *vm, uint64_t policy, uint8_t *measurement) { if (is_sev_es_vm(vm)) diff --git a/tools/testing/selftests/kvm/s390/cmma_test.c b/tools/testing/selftests/kvm/s390/cmma_test.c index e32dd59703a0..b6a3fa1d71aa 100644 --- a/tools/testing/selftests/kvm/s390/cmma_test.c +++ b/tools/testing/selftests/kvm/s390/cmma_test.c @@ -145,7 +145,7 @@ static void finish_vm_setup(struct kvm_vm *vm) slot0 = memslot2region(vm, 0); ucall_init(vm, slot0->region.guest_phys_addr + slot0->region.memory_size);

- kvm_arch_vm_post_create(vm); + kvm_arch_vm_post_create(vm, NULL); }

static struct kvm_vm *create_vm_two_memslots(void)

Test cross-vCPU IPI for Secure AVIC guests using xapic_ipi_test. Add new "SAVIC" apic mode to the test for this.

Signed-off-by: Neeraj Upadhyay Neeraj.Upadhyay@amd.com --- tools/arch/x86/include/asm/msr-index.h | 4 +- .../selftests/kvm/include/x86/processor.h | 1 + .../selftests/kvm/x86/xapic_ipi_test.c | 70 ++++++++++++++----- 3 files changed, 55 insertions(+), 20 deletions(-)

diff --git a/tools/arch/x86/include/asm/msr-index.h b/tools/arch/x86/include/asm/msr-index.h index 3ae84c3b8e6d..93f3ff9991cc 100644 --- a/tools/arch/x86/include/asm/msr-index.h +++ b/tools/arch/x86/include/asm/msr-index.h @@ -680,7 +680,9 @@ #define MSR_AMD64_SNP_VMSA_REG_PROT BIT_ULL(MSR_AMD64_SNP_VMSA_REG_PROT_BIT) #define MSR_AMD64_SNP_SMT_PROT_BIT 17 #define MSR_AMD64_SNP_SMT_PROT BIT_ULL(MSR_AMD64_SNP_SMT_PROT_BIT) -#define MSR_AMD64_SNP_RESV_BIT 18 +#define MSR_AMD64_SNP_SECURE_AVIC_BIT 18 +#define MSR_AMD64_SNP_SECURE_AVIC BIT_ULL(MSR_AMD64_SNP_SECURE_AVIC_BIT) +#define MSR_AMD64_SNP_RESV_BIT 19 #define MSR_AMD64_SNP_RESERVED_MASK GENMASK_ULL(63, MSR_AMD64_SNP_RESV_BIT)

#define MSR_AMD64_VIRT_SPEC_CTRL 0xc001011f diff --git a/tools/testing/selftests/kvm/include/x86/processor.h b/tools/testing/selftests/kvm/include/x86/processor.h index 3f9369644962..f09b18944c47 100644 --- a/tools/testing/selftests/kvm/include/x86/processor.h +++ b/tools/testing/selftests/kvm/include/x86/processor.h @@ -201,6 +201,7 @@ struct kvm_x86_cpu_feature { #define X86_FEATURE_SEV KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 1) #define X86_FEATURE_SEV_ES KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 3) #define X86_FEATURE_SNP KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 4) +#define X86_FEATURE_SECURE_AVIC KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 26)

/* * KVM defined paravirt features. diff --git a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c index 3a54d828dc69..e11c7ded8b8a 100644 --- a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c +++ b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c @@ -31,6 +31,7 @@ #include "test_util.h" #include "vmx.h" #include "sev.h" +#include "savic.h"

/* Default running time for the test */ #define DEFAULT_RUN_SECS 3 @@ -45,30 +46,44 @@ */ #define IPI_VECTOR 0xa5

+enum apic_mode { + XAPIC, + X2APIC, + SAVIC, +}; + /* * Incremented in the IPI handler. Provides evidence to the sender that the IPI * arrived at the destination */ static volatile uint64_t *ipis_rcvd;

-static bool x2apic; +static int apic_mode;

static void apic_enable(void) { - if (x2apic) - x2apic_enable(); - else + switch (apic_mode) { + case XAPIC: xapic_enable(); + break; + case X2APIC: + x2apic_enable(); + break; + case SAVIC: + x2apic_enable(); + savic_enable(); + break; + } }

static uint32_t apic_read_reg(unsigned int reg) { - return x2apic ? x2apic_read_reg(reg) : xapic_read_reg(reg); + return apic_mode != XAPIC ? x2apic_read_reg(reg) : xapic_read_reg(reg); }

static void apic_write_reg(unsigned int reg, uint64_t val) { - if (x2apic) + if (apic_mode != XAPIC) x2apic_write_reg(reg, val); else xapic_write_reg(reg, (uint32_t)val); @@ -144,7 +159,7 @@ static void halter_guest_code(struct test_data_page *data) static void guest_ipi_handler(struct ex_regs *regs) { (*ipis_rcvd)++; - apic_write_reg(APIC_EOI, 77); + apic_write_reg(APIC_EOI, 0); }

static void sender_guest_code(struct test_data_page *data) @@ -184,7 +199,7 @@ static void sender_guest_code(struct test_data_page *data) * First IPI can be sent unconditionally because halter vCPU * starts earlier. */ - if (!x2apic) { + if (apic_mode == XAPIC) { apic_write_reg(APIC_ICR2, icr2_val); apic_write_reg(APIC_ICR, icr_val); } else { @@ -385,10 +400,10 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, }

void get_cmdline_args(int argc, char *argv[], int *run_secs, - bool *migrate, int *delay_usecs, bool *x2apic, int *vm_type) + bool *migrate, int *delay_usecs, int *apic_mode, int *vm_type) { for (;;) { - int opt = getopt(argc, argv, "s:d:v:me:t:"); + int opt = getopt(argc, argv, "s:d:v:me:t:g");

if (opt == -1) break; @@ -403,7 +418,7 @@ void get_cmdline_args(int argc, char *argv[], int *run_secs, *delay_usecs = parse_size(optarg); break; case 'e': - *x2apic = parse_size(optarg) == 1; + *apic_mode = parse_size(optarg); break; case 't': *vm_type = parse_size(optarg); @@ -431,7 +446,7 @@ void get_cmdline_args(int argc, char *argv[], int *run_secs, " Default is no migrations.\n" "-d <delay microseconds> - delay between migrate_pages() calls." " Default is %d microseconds.\n" - "-e <apic mode> - APIC mode 0 - xapic , 1 - x2apic" + "-e <apic mode> - APIC mode 0 - xapic , 1 - x2apic, 3 - Secure AVIC" " Default is xAPIC.\n" "-t <vm type>. Default is %d.\n" "Supported values:\n" @@ -483,10 +498,17 @@ int main(int argc, char *argv[]) bool is_sev;

get_cmdline_args(argc, argv, &run_secs, &migrate, &delay_usecs, - &x2apic, &vm_type); + &apic_mode, &vm_type); + if (apic_mode == SAVIC) { + vm_type = KVM_X86_SNP_VM; + TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SNP)); + TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_IDLE_HLT)); + TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SECURE_AVIC)); + } + is_sev = is_sev_vm_type(vm_type);

- if (x2apic) + if (apic_mode != XAPIC) migrate = 0;

if (run_secs <= 0) @@ -494,18 +516,28 @@ int main(int argc, char *argv[]) if (delay_usecs <= 0) delay_usecs = DEFAULT_DELAY_USECS;

- if (is_sev) + if (apic_mode == SAVIC) { + struct kvm_sev_init args = { .vmsa_features = BIT_ULL(SVM_FEAT_SECURE_AVIC) | + BIT_ULL(SVM_FEAT_ALLOWED_SEV_FEATURES_VALID) + }; + + vm = _vm_sev_create_with_one_vcpu(vm_type, halter_guest_code, + &params[0].vcpu, &args); + } else if (is_sev) { vm = vm_sev_create_with_one_vcpu(vm_type, halter_guest_code, &params[0].vcpu); - else + } else { vm = vm_create_with_one_vcpu(&params[0].vcpu, halter_guest_code); + }

vm_install_exception_handler(vm, IPI_VECTOR, guest_ipi_handler); - if (is_sev_es_vm(vm)) + if (apic_mode == SAVIC) + vm_install_exception_handler(vm, 29, savic_vc_handler); + else if (is_sev_es_vm(vm)) vm_install_exception_handler(vm, 29, sev_es_vc_handler);

- sync_global_to_guest(vm, x2apic); - if (!x2apic) + sync_global_to_guest(vm, apic_mode); + if (apic_mode == XAPIC) virt_pg_map(vm, APIC_DEFAULT_GPA, APIC_DEFAULT_GPA);

params[1].vcpu = vm_vcpu_add(vm, 1, sender_guest_code);

Extend SAVIC test to verify APIC MSR accesses in SAVIC enabled mode. Verify the behavior of reads and writes using rdmsr/wrmsr for various APIC registers. In addition, test whether wrmsr based writes are propagated to guest backing page.

Signed-off-by: Neeraj Upadhyay Neeraj.Upadhyay@amd.com --- .../testing/selftests/kvm/include/x86/apic.h | 1 + .../testing/selftests/kvm/include/x86/savic.h | 3 + tools/testing/selftests/kvm/lib/x86/savic.c | 7 +- tools/testing/selftests/kvm/x86/savic_test.c | 192 +++++++++++++++++- 4 files changed, 198 insertions(+), 5 deletions(-)

diff --git a/tools/testing/selftests/kvm/include/x86/apic.h b/tools/testing/selftests/kvm/include/x86/apic.h index aa3a5d54c404..af555638086f 100644 --- a/tools/testing/selftests/kvm/include/x86/apic.h +++ b/tools/testing/selftests/kvm/include/x86/apic.h @@ -33,6 +33,7 @@ #define APIC_SPIV 0xF0 #define APIC_SPIV_FOCUS_DISABLED (1 << 9) #define APIC_SPIV_APIC_ENABLED (1 << 8) +#define APIC_ISR 0x100 #define APIC_TMR 0x180 #define APIC_IRR 0x200 #define APIC_ICR 0x300 diff --git a/tools/testing/selftests/kvm/include/x86/savic.h b/tools/testing/selftests/kvm/include/x86/savic.h index cb432eb527b3..33f19f5e39b3 100644 --- a/tools/testing/selftests/kvm/include/x86/savic.h +++ b/tools/testing/selftests/kvm/include/x86/savic.h @@ -6,6 +6,9 @@ #ifndef SELFTEST_KVM_SAVIC_H #define SELFTEST_KVM_SAVIC_H

+#define APIC_REG_OFF(VEC) (VEC / 32 * 16) +#define APIC_VEC_POS(VEC) (VEC % 32) + struct guest_apic_page;

void guest_apic_pages_init(struct kvm_vm *vm); diff --git a/tools/testing/selftests/kvm/lib/x86/savic.c b/tools/testing/selftests/kvm/lib/x86/savic.c index d4c9fcf835ad..c637e486b6e8 100644 --- a/tools/testing/selftests/kvm/lib/x86/savic.c +++ b/tools/testing/selftests/kvm/lib/x86/savic.c @@ -9,6 +9,7 @@ #include "kvm_util.h" #include "sev.h" #include "ex_regs.h" +#include "savic.h"

struct apic_page { u8 apic_regs[PAGE_SIZE]; @@ -45,9 +46,6 @@ enum lapic_lvt_entry { #define SVM_EXIT_AVIC_UNACCELERATED_ACCESS 0x402 #define SVM_EXIT_AVIC_INCOMPLETE_IPI 0x401

-#define REG_OFF(VEC) (VEC / 32 * 16) -#define VEC_POS(VEC) (VEC % 32) - #define SAVIC_NMI_REQ_OFFSET 0x278

/* @@ -363,7 +361,8 @@ static void send_ipi(int cpu, int vector, bool nmi) if (nmi) savic_write_reg(apic_page, SAVIC_NMI_REQ_OFFSET, 1); else - savic_write_reg(apic_page, APIC_IRR + REG_OFF(vector), BIT(VEC_POS(vector))); + savic_write_reg(apic_page, APIC_IRR + APIC_REG_OFF(vector), + BIT(APIC_VEC_POS(vector))); }

static bool is_cpu_present(int cpu) diff --git a/tools/testing/selftests/kvm/x86/savic_test.c b/tools/testing/selftests/kvm/x86/savic_test.c index ca1d7352bc3e..8cba7a81bce2 100644 --- a/tools/testing/selftests/kvm/x86/savic_test.c +++ b/tools/testing/selftests/kvm/x86/savic_test.c @@ -29,6 +29,7 @@ enum savic_test_state { SAVIC_TEST_STATE(X2APIC_ENABLE), /* APIC regs state on Secure AVIC enablement */ SAVIC_TEST_STATE(SAVIC_EN), + SAVIC_TEST_STATE(SAVIC_APIC_MSR_ACCESSES), };

/* APIC reg values written by host. */ @@ -288,6 +289,193 @@ static void guest_savic_enabled(int id) savic_write_apic_regs(apage); }

+static int savic_wrmsr(uint32_t reg, uint64_t val) +{ + switch (reg) { + case APIC_LVR: + case APIC_LDR: + case APIC_ISR: + case APIC_TMR: + case APIC_IRR: + case APIC_TMCCT: + x2apic_write_reg_fault(reg, val); + return -1; + default: + x2apic_write_reg(reg, val); + break; + } + + return 0; +} + +static uint64_t savic_rdmsr(uint32_t reg) +{ + uint64_t val; + uint32_t msr = APIC_BASE_MSR + (reg >> 4); + + switch (reg) { + case APIC_EOI: + uint8_t fault = rdmsr_safe(msr, &val); + + __GUEST_ASSERT(fault == GP_VECTOR, + "Wanted #GP on RDMSR(%x) = %x, got 0x%x\n", + msr, GP_VECTOR, fault); + return val; + default: + return x2apic_read_reg(reg); + } +} + +static void guest_verify_host_guest_reg(struct guest_apic_page *apage, uint32_t reg, + uint64_t val, char *regname) +{ + uint64_t hval, gval, gval2; + + if (savic_wrmsr(reg, val) == -1) { + savic_write_reg(apage, reg, val); + /* + * Write using PV interface if wrmsr fails. Skip for + * regs which trigger GP + */ + if (reg != APIC_LVR && reg != APIC_TMR && reg != APIC_IRR) + savic_hv_write_reg(reg, val); + } + + gval = savic_read_reg(apage, reg); + gval2 = savic_rdmsr(reg); + hval = savic_hv_read_reg(reg); + __GUEST_ASSERT(gval == val, "Unexpected Guest %s 0x%lx, expected val:0x%lx\n", + regname, gval, val); + __GUEST_ASSERT(gval == gval2, "Unexpected Guest %s backing page value : 0x%lx, msr read val:0x%lx\n", + regname, gval, gval2); + + switch (reg) { + case APIC_LVR: + case APIC_LDR: + case APIC_ISR: + case APIC_TMICT: + case APIC_TDCR: + case APIC_LVTT: + case APIC_LVTTHMR: + case APIC_LVTPC: + case APIC_LVT0: + case APIC_LVT1: + case APIC_LVTERR: + __GUEST_ASSERT(hval == gval, "Guest 0x%lx host 0x%lx %s mismatch\n", + gval, hval, regname); + break; + case APIC_TASKPRI: + case APIC_SPIV: + case APIC_ICR: + case APIC_TMR: + case APIC_IRR: + __GUEST_ASSERT(hval != gval, "Guest 0x%lx host 0x%lx reg: %x %s must not match\n", + gval, hval, reg, regname); + break; + default: + break; + } +} + +static inline uint32_t x2apic_ldr(uint32_t id) +{ + return ((id >> 4) << 16) | (1 << (id & 0xf)); +} + +static void guest_savic_apic_msr_accesses(int id) +{ + struct guest_apic_page *apage = get_guest_apic_page(); + uint64_t val, hval; + uint32_t reg; + int vec; + int i; + uint32_t lvt_regs[] = { + APIC_LVTT, APIC_LVTTHMR, APIC_LVTPC, + APIC_LVT0, APIC_LVT1, APIC_LVTERR + }; + + reg = APIC_LVR; + val = savic_hv_read_reg(reg); + /* APIC_LVR state is in sync between host and guest. */ + guest_verify_host_guest_reg(apage, reg, val, "APIC_LVR"); + + reg = APIC_TASKPRI; + val = 0x30; + /* Write new TASKPRI to host using PV interface. */ + savic_hv_write_reg(reg, val); + val = 0x40; + /* TASKPRI is accelerated and state is not up-to-date in host. */ + guest_verify_host_guest_reg(apage, reg, val, "APIC_TASKPRI"); + + reg = APIC_PROCPRI; + val = x2apic_read_reg(reg); + /* APIC_PROCPRI is updated with the APIC_TASKPRI update above. */ + GUEST_ASSERT((val & 0xf0) == (x2apic_read_reg(APIC_TASKPRI) & 0xf0)); + GUEST_ASSERT((val & 0xf0) == 0x40); + vec = 0x20; + x2apic_write_reg(APIC_ICR, APIC_DEST_SELF | APIC_INT_ASSERT | vec); + /* Interrupt remains pending in APIC_IRR. */ + val = savic_read_reg(apage, APIC_IRR + APIC_REG_OFF(vec)); + GUEST_ASSERT((val & BIT_ULL(APIC_VEC_POS(vec))) == BIT_ULL(APIC_VEC_POS(vec))); + savic_wrmsr(APIC_TASKPRI, 0x0); + + /* Triggers GP fault */ + savic_rdmsr(APIC_EOI); + + reg = APIC_LDR; + val = x2apic_ldr(savic_rdmsr(APIC_ID)); + hval = savic_hv_read_reg(APIC_LDR); + __GUEST_ASSERT(val == hval, "APIC_LDR mismatch between host %lx and guest %lx", + hval, val); + + /* APIC_SPIV state is not visible to host. */ + reg = APIC_SPIV; + val = savic_rdmsr(APIC_SPIV) & ~APIC_SPIV_APIC_ENABLED; + savic_hv_write_reg(reg, val); + val = savic_rdmsr(APIC_SPIV) | APIC_SPIV_APIC_ENABLED; + guest_verify_host_guest_reg(apage, reg, val, "APIC_SPIV"); + + reg = APIC_ISR; + (void) savic_rdmsr(reg); + /* Triggers GP fault */ + savic_wrmsr(reg, 0x10); + + /* APIC_TMR is not synced to host. */ + reg = APIC_TMR; + val = 0x10000; + guest_verify_host_guest_reg(apage, reg, val, "APIC_TMR"); + vec = 0x20; + savic_write_reg(apage, reg + APIC_REG_OFF(vec), BIT_ULL(APIC_VEC_POS(vec))); + GUEST_ASSERT(x2apic_read_reg(reg + APIC_REG_OFF(vec)) & BIT_ULL(APIC_VEC_POS(vec))); + + reg = APIC_IRR; + val = 0x10000; + guest_verify_host_guest_reg(apage, reg, val, "APIC_IRR"); + savic_write_reg(apage, reg, 0x0); + + reg = APIC_TMICT; + val = 0x555; + guest_verify_host_guest_reg(apage, reg, val, "APIC_TMICT"); + + reg = APIC_TMCCT; + savic_rdmsr(reg); + savic_wrmsr(reg, 0xf); + + reg = APIC_TDCR; + val = 0x1; + savic_hv_write_reg(reg, val); + val = 0x3; + guest_verify_host_guest_reg(apage, reg, val, "APIC_TDCR"); + + for (i = 0; i < ARRAY_SIZE(lvt_regs); i++) { + reg = lvt_regs[i]; + val = 0x41; + savic_hv_write_reg(reg, val); + val = 0x42; + guest_verify_host_guest_reg(apage, reg, val, "APIC_LVTx"); + } +} + static void guest_code(int id) { GUEST_ASSERT(rdmsr(MSR_AMD64_SEV) & MSR_AMD64_SNP_SECURE_AVIC); @@ -302,6 +490,8 @@ static void guest_code(int id)

SAVIC_GUEST_SYNC(SAVIC_EN, guest_savic_enabled);

+ SAVIC_GUEST_SYNC(SAVIC_APIC_MSR_ACCESSES, guest_savic_apic_msr_accesses); + GUEST_DONE(); }

@@ -448,7 +638,7 @@ int main(int argc, char *argv[])

vcpu_args_set(vcpus[0], 1, vcpus[0]->id);

- vm_install_exception_handler(vm, 29, sev_es_vc_handler); + vm_install_exception_handler(vm, 29, savic_vc_handler); vm_sev_launch(vm, snp_default_policy(), NULL);

r = pthread_create(&threads[0], NULL, vcpu_thread, vcpus[0]);

Extend Secure AVIC tests to test edge and level ioapic based interrupt flow. In addition, test RTC_GSI IOAPIC line which has special casing in KVM. Add new interfaces to read and write to IOAPIC redirect tables.

Signed-off-by: Neeraj Upadhyay Neeraj.Upadhyay@amd.com --- .../testing/selftests/kvm/include/kvm_util.h | 1 + .../testing/selftests/kvm/include/x86/apic.h | 49 ++++ .../testing/selftests/kvm/include/x86/savic.h | 1 + tools/testing/selftests/kvm/lib/kvm_util.c | 17 ++ .../testing/selftests/kvm/lib/x86/processor.c | 2 +- tools/testing/selftests/kvm/lib/x86/savic.c | 11 + tools/testing/selftests/kvm/x86/savic_test.c | 270 +++++++++++++++++- 7 files changed, 347 insertions(+), 4 deletions(-)

diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 7f97bade5797..33326c5975c0 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -861,6 +861,7 @@ void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu); void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...);

void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level); +void kvm_irq_line_status(struct kvm_vm *vm, uint32_t irq, int level); int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level);

#define KVM_MAX_IRQ_ROUTES 4096 diff --git a/tools/testing/selftests/kvm/include/x86/apic.h b/tools/testing/selftests/kvm/include/x86/apic.h index af555638086f..765c463dff33 100644 --- a/tools/testing/selftests/kvm/include/x86/apic.h +++ b/tools/testing/selftests/kvm/include/x86/apic.h @@ -12,6 +12,7 @@ #include "ucall_common.h"

#define APIC_DEFAULT_GPA 0xfee00000ULL +#define IOAPIC_DEFAULT_GPA 0xfec00000ULL

/* APIC base address MSR and fields */ #define MSR_IA32_APICBASE 0x0000001b @@ -122,5 +123,53 @@ static inline void x2apic_write_reg_fault(unsigned int reg, uint64_t value) APIC_BASE_MSR + (reg >> 4), value, fault); }

+struct ioapic_redirect_entry { + uint8_t vector; + uint8_t delivery_mode:3; + uint8_t dest_mode:1; + uint8_t delivery_status:1; + uint8_t polarity:1; + uint8_t remote_irr:1; + uint8_t trig_mode:1; + uint8_t mask:1; + uint8_t reserve:7; + uint8_t reserved[4]; + uint8_t dest_id; +}; + +enum trigger_mode { + TRIGGER_EDGE = 0, + TRIGGER_LEVEL, + TRIGGER_MAX, +}; + +static void *ioapic_addr = (void *)IOAPIC_DEFAULT_GPA; + +static inline void ioapic_write_reg(uint32_t reg, uint32_t val) +{ + *(volatile uint32_t *)ioapic_addr = reg; + *(volatile u32 *)(ioapic_addr + 0x10) = val; +} + +static inline void ioapic_write_redir(unsigned int line, struct ioapic_redirect_entry e) +{ + ioapic_write_reg(0x10 + line * 2 + 0, ((uint32_t *)&e)[0]); + ioapic_write_reg(0x10 + line * 2 + 1, ((uint32_t *)&e)[1]); +} + +static inline uint32_t ioapic_read_reg(unsigned int reg) +{ + *(volatile uint32_t *)ioapic_addr = reg; + return *(volatile uint32_t *)(ioapic_addr + 0x10); +} + +static inline struct ioapic_redirect_entry ioapic_read_redir(unsigned int line) +{ + struct ioapic_redirect_entry e; + + ((u32 *)&e)[0] = ioapic_read_reg(0x10 + line * 2 + 0); + ((u32 *)&e)[1] = ioapic_read_reg(0x10 + line * 2 + 1);

+ return e; +} #endif /* SELFTEST_KVM_APIC_H */ diff --git a/tools/testing/selftests/kvm/include/x86/savic.h b/tools/testing/selftests/kvm/include/x86/savic.h index 33f19f5e39b3..73965edfef5c 100644 --- a/tools/testing/selftests/kvm/include/x86/savic.h +++ b/tools/testing/selftests/kvm/include/x86/savic.h @@ -21,4 +21,5 @@ void savic_enable(void); int savic_nr_pages_required(uint64_t page_size); void savic_vc_handler(struct ex_regs *regs); struct guest_apic_page *get_guest_apic_page(void); +void savic_allow_vector(int vec); #endif diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 93b8e2ccc7b3..900fbad7258c 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -1861,6 +1861,23 @@ void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level) TEST_ASSERT(ret >= 0, KVM_IOCTL_ERROR(KVM_IRQ_LINE, ret)); }

+int _kvm_irq_line_status(struct kvm_vm *vm, uint32_t irq, int level) +{ + struct kvm_irq_level irq_level = { + .irq = irq, + .level = level, + }; + + return __vm_ioctl(vm, KVM_IRQ_LINE_STATUS, &irq_level); +} + +void kvm_irq_line_status(struct kvm_vm *vm, uint32_t irq, int level) +{ + int ret = _kvm_irq_line_status(vm, irq, level); + + TEST_ASSERT(ret >= 0, KVM_IOCTL_ERROR(KVM_IRQ_LINE_STATUS, ret)); +} + struct kvm_irq_routing *kvm_gsi_routing_create(void) { struct kvm_irq_routing *routing; diff --git a/tools/testing/selftests/kvm/lib/x86/processor.c b/tools/testing/selftests/kvm/lib/x86/processor.c index 09474be27986..da2f8d974f60 100644 --- a/tools/testing/selftests/kvm/lib/x86/processor.c +++ b/tools/testing/selftests/kvm/lib/x86/processor.c @@ -229,7 +229,7 @@ void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, int level) "PTE already present for 4k page at vaddr: 0x%lx", vaddr); *pte = PTE_PRESENT_MASK | PTE_WRITABLE_MASK | (paddr & PHYSICAL_PAGE_MASK);

- if (paddr == APIC_DEFAULT_GPA) { + if (paddr == APIC_DEFAULT_GPA || paddr == IOAPIC_DEFAULT_GPA) { *pte |= vm->arch.s_bit; return; } diff --git a/tools/testing/selftests/kvm/lib/x86/savic.c b/tools/testing/selftests/kvm/lib/x86/savic.c index c637e486b6e8..ee2d6b66d6ce 100644 --- a/tools/testing/selftests/kvm/lib/x86/savic.c +++ b/tools/testing/selftests/kvm/lib/x86/savic.c @@ -46,6 +46,7 @@ enum lapic_lvt_entry { #define SVM_EXIT_AVIC_UNACCELERATED_ACCESS 0x402 #define SVM_EXIT_AVIC_INCOMPLETE_IPI 0x401

+#define SAVIC_ALLOWED_IRR (APIC_IRR + 0x4) #define SAVIC_NMI_REQ_OFFSET 0x278

/* @@ -108,6 +109,16 @@ struct guest_apic_page *get_guest_apic_page(void) return &apic_page_pool->guest_apic_page[x2apic_read_reg(APIC_ID)]; }

+void savic_allow_vector(int vec) +{ + struct guest_apic_page *apage = get_guest_apic_page(); + + savic_write_reg(apage, SAVIC_ALLOWED_IRR + APIC_REG_OFF(vec), + savic_read_reg(apage, SAVIC_ALLOWED_IRR + APIC_REG_OFF(vec)) | + BIT_ULL(APIC_VEC_POS(vec))); + +} + /* * Write APIC reg offset in the guest APIC backing page. * diff --git a/tools/testing/selftests/kvm/x86/savic_test.c b/tools/testing/selftests/kvm/x86/savic_test.c index 5c52254f7b1c..3399ccefc37d 100644 --- a/tools/testing/selftests/kvm/x86/savic_test.c +++ b/tools/testing/selftests/kvm/x86/savic_test.c @@ -14,9 +14,15 @@ #include "savic.h"

#define NR_SAVIC_VCPUS 1 -#define IDLE_HLT_INTR_VECTOR 0x30 #define NUM_ITERATIONS 2000

+#define IDLE_HLT_INTR_VECTOR 0x30 +#define IOAPIC_VECTOR_START 0x81 +#define IOAPIC_NUM_EDGE_VECTORS 2 +#define IOAPIC_NUM_LEVEL_VECTORS 2 +#define RTC_GSI 8 +#define RTC_GSI_IRQ 0x85 + static bool irq_received; static struct kvm_vcpu *vcpus[NR_SAVIC_VCPUS]; static pthread_t threads[NR_SAVIC_VCPUS]; @@ -34,6 +40,8 @@ enum savic_test_state { SAVIC_TEST_STATE(SAVIC_EN), SAVIC_TEST_STATE(SAVIC_APIC_MSR_ACCESSES), SAVIC_TEST_STATE(SAVIC_IDLE_HALT), + SAVIC_TEST_STATE(SAVIC_IOAPIC), + SAVIC_TEST_STATE(SAVIC_IOAPIC2), };

/* APIC reg values written by host. */ @@ -108,6 +116,17 @@ enum context {

static struct kvm_lapic_state apic_state[NR_SAVIC_VCPUS];

+/* Data struct shared between host main thread and vCPUs */ +struct test_data_page { + uint64_t ioapic_eirq1_count; + uint64_t ioapic_eirq2_count; + uint64_t ioapic_lirq1_count; + uint64_t ioapic_lirq2_count; + uint64_t ioapic_rtc_gsi_irq_count; +}; + +static struct test_data_page *test_data[NR_SAVIC_VCPUS]; + /* * Write APIC reg from host or guest context. * @@ -519,6 +538,177 @@ static void guest_savic_idle_halt(int id) } }

+static void _ioapic_level_irq_handler(int vec) +{ + uint32_t isr, tmr; + int offset, pos; + + offset = APIC_REG_OFF(vec); + pos = APIC_VEC_POS(vec); + isr = savic_hv_read_reg(APIC_ISR + offset); + tmr = savic_hv_read_reg(APIC_TMR + offset); + + __GUEST_ASSERT(tmr & BIT_ULL(pos), + "IOAPIC level vector %d trigger mode in not set in host TMR: %x", + vec, tmr); + __GUEST_ASSERT(isr & BIT_ULL(pos), + "IOAPIC level vector %d in not set in host ISR: %x", + vec, isr); + + x2apic_write_reg(APIC_EOI, 0x00); + savic_hv_write_reg(APIC_EOI, 0); + + isr = savic_hv_read_reg(APIC_ISR + offset); + __GUEST_ASSERT(!(isr & BIT_ULL(pos)), + "IOAPIC level vector %d set in host ISR after EOI", + vec); +} + +static void ioapic_level_irq1_intr_handler(struct ex_regs *regs) +{ + struct test_data_page *data = test_data[x2apic_read_reg(APIC_ID)]; + int vec; + + vec = IOAPIC_VECTOR_START + IOAPIC_NUM_EDGE_VECTORS; + WRITE_ONCE(data->ioapic_lirq1_count, data->ioapic_lirq1_count + 1); + _ioapic_level_irq_handler(vec); +} + +static void ioapic_level_irq2_intr_handler(struct ex_regs *regs) +{ + struct test_data_page *data = test_data[x2apic_read_reg(APIC_ID)]; + int vec; + + vec = IOAPIC_VECTOR_START + IOAPIC_NUM_EDGE_VECTORS + 1; + WRITE_ONCE(data->ioapic_lirq2_count, data->ioapic_lirq2_count + 1); + _ioapic_level_irq_handler(vec); +} + +static void ioapic_edge_irq1_intr_handler(struct ex_regs *regs) +{ + struct test_data_page *data = test_data[x2apic_read_reg(APIC_ID)]; + + WRITE_ONCE(data->ioapic_eirq1_count, data->ioapic_eirq1_count + 1); + x2apic_write_reg(APIC_EOI, 0x00); +} + +static void ioapic_edge_irq2_intr_handler(struct ex_regs *regs) +{ + struct test_data_page *data = test_data[x2apic_read_reg(APIC_ID)]; + + WRITE_ONCE(data->ioapic_eirq2_count, data->ioapic_eirq2_count + 1); + x2apic_write_reg(APIC_EOI, 0x00); +} + +static void ioapic_rtc_gsi_intr_handler(struct ex_regs *regs) +{ + struct test_data_page *data = test_data[x2apic_read_reg(APIC_ID)]; + + WRITE_ONCE(data->ioapic_rtc_gsi_irq_count, data->ioapic_rtc_gsi_irq_count + 1); + x2apic_write_reg(APIC_EOI, 0x00); +} + +static void __savic_ioapic(int count) +{ + struct test_data_page *data = test_data[x2apic_read_reg(APIC_ID)]; + int vec = IOAPIC_VECTOR_START; + + __GUEST_ASSERT(READ_ONCE(data->ioapic_eirq1_count) == count, + "Invalid ioapic edge irq %d count: %ld, expected: %d", + vec, READ_ONCE(data->ioapic_eirq1_count), count); + __GUEST_ASSERT(READ_ONCE(data->ioapic_eirq2_count) == count, + "Invalid ioapic edge irq %d count: %ld, expected: %d", + vec + 1, READ_ONCE(data->ioapic_eirq2_count), count); + __GUEST_ASSERT(READ_ONCE(data->ioapic_lirq1_count) == count, + "Invalid ioapic level irq %d count: %ld, expected: %d", + vec + 2, READ_ONCE(data->ioapic_lirq1_count), count); + __GUEST_ASSERT(READ_ONCE(data->ioapic_lirq2_count) == count, + "Invalid ioapic level irq %d count: %ld, expected: %d", + vec + 3, READ_ONCE(data->ioapic_lirq2_count), count); + __GUEST_ASSERT(READ_ONCE(data->ioapic_rtc_gsi_irq_count) == count, + "Invalid ioapic RTC irq %d count: %ld, expected: %d", + RTC_GSI_IRQ, READ_ONCE(data->ioapic_rtc_gsi_irq_count), + count); +} + +static void savic_ioapic(int id) +{ + __savic_ioapic(1); +} + +static void savic_ioapic2(int id) +{ + __savic_ioapic(2); +} + +static void ioapic_set_redir(unsigned int line, unsigned int vec, + enum trigger_mode trig_mode) +{ + struct ioapic_redirect_entry e = { + .vector = vec, + .delivery_mode = 0, + .dest_mode = 0, + .trig_mode = trig_mode, + .mask = 0, + .dest_id = 0, + .delivery_status = 0, + .remote_irr = 0, + }; + + ioapic_write_redir(line, e); +} + +static void guest_setup_ioapic(int id) +{ + int vec = IOAPIC_VECTOR_START; + struct ioapic_redirect_entry e; + int i, line = 0; + + for (i = 0; i < IOAPIC_NUM_EDGE_VECTORS; i++) { + ioapic_set_redir(line, vec, TRIGGER_EDGE); + e = ioapic_read_redir(line); + __GUEST_ASSERT( + e.vector == vec && e.trig_mode == TRIGGER_EDGE && + e.dest_id == 0, + "Invalid IOAPIC redir entry for line : %d, trig_mode: %d vector: %d", + line, e.trig_mode, e.vector); + vec++; + line++; + } + + for (i = 0; i < IOAPIC_NUM_LEVEL_VECTORS; i++) { + ioapic_set_redir(line, vec, TRIGGER_LEVEL); + e = ioapic_read_redir(line); + __GUEST_ASSERT( + e.vector == vec && e.trig_mode == TRIGGER_LEVEL && + e.dest_id == 0, + "Invalid IOAPIC redir entry for line : %d, trig_mode: %d vector: %d", + line, e.trig_mode, e.vector); + line++; + vec++; + } + + vec = RTC_GSI_IRQ; + line = RTC_GSI; + ioapic_set_redir(line, vec, TRIGGER_EDGE); + e = ioapic_read_redir(line); + __GUEST_ASSERT( + e.vector == vec && e.trig_mode == TRIGGER_EDGE && + e.dest_id == 0, + "Invalid IOAPIC redir entry for line : %d, trig_mode: %d vector: %d", + line, e.trig_mode, e.vector); + + x2apic_write_reg(APIC_TASKPRI, 0); + + for (i = 0; i < (IOAPIC_NUM_EDGE_VECTORS + IOAPIC_NUM_LEVEL_VECTORS); i++) { + vec = IOAPIC_VECTOR_START + i; + savic_allow_vector(vec); + } + + vec = RTC_GSI_IRQ; + savic_allow_vector(vec); +} + static void guest_code(int id) { GUEST_ASSERT(rdmsr(MSR_AMD64_SEV) & MSR_AMD64_SNP_SECURE_AVIC); @@ -537,6 +727,10 @@ static void guest_code(int id)

SAVIC_GUEST_SYNC(SAVIC_IDLE_HALT, guest_savic_idle_halt);

+ guest_setup_ioapic(id); + SAVIC_GUEST_SYNC(SAVIC_IOAPIC, savic_ioapic); + SAVIC_GUEST_SYNC(SAVIC_IOAPIC2, savic_ioapic2); + GUEST_DONE(); }

@@ -611,6 +805,46 @@ static void host_post_savic_en_end(int id) test_apic_vals(id, false, CTXT_HOST, true, true); }

+static void host_send_ioapic_irq(struct kvm_vm *vm, int id) +{ + int i; + int vec, offset, pos; + uint32_t regval; + + kvm_irq_line(vm, 0, 1); + kvm_irq_line(vm, 1, 1); + kvm_irq_line(vm, 0, 0); + kvm_irq_line(vm, 1, 0); + kvm_irq_line(vm, 2, 1); + kvm_irq_line(vm, 2, 0); + kvm_irq_line(vm, 3, 1); + kvm_irq_line(vm, 3, 0); + kvm_irq_line_status(vm, RTC_GSI, 1); + kvm_irq_line_status(vm, RTC_GSI, 0); + + vcpu_ioctl(vcpus[id], KVM_GET_LAPIC, &apic_state[id]); + + for (i = 0; i < IOAPIC_NUM_EDGE_VECTORS; i++) { + vec = IOAPIC_VECTOR_START + i; + offset = APIC_REG_OFF(vec); + regval = *(uint32_t *)&apic_state[id].regs[APIC_TMR + offset]; + pos = APIC_VEC_POS(vec); + TEST_ASSERT(!(regval & BIT_ULL(pos)), "TMR should be 0 for vector: %x", vec); + regval = *(uint32_t *)&apic_state[id].regs[APIC_IRR + offset]; + TEST_ASSERT(regval & BIT_ULL(pos), "IRR should be 1 for vector: %x", vec); + } + + for (i = 0; i < IOAPIC_NUM_LEVEL_VECTORS; i++) { + vec = IOAPIC_VECTOR_START + 2 + i; + offset = APIC_REG_OFF(vec); + regval = *(uint32_t *)&apic_state[id].regs[APIC_TMR + offset]; + pos = APIC_VEC_POS(vec); + TEST_ASSERT(regval & BIT_ULL(pos), "TMR should be 1 for vector: %x", vec); + regval = *(uint32_t *)&apic_state[id].regs[APIC_IRR + offset]; + TEST_ASSERT(regval & BIT_ULL(pos), "IRR should be 1 for vector: %x", vec); + } +} + static void host_test_savic(struct kvm_vm *vm, int id, enum savic_test_state test_state) { switch (test_state) { @@ -632,6 +866,12 @@ static void host_test_savic(struct kvm_vm *vm, int id, enum savic_test_state tes case SAVIC_EN_END: host_post_savic_en_end(id); break; + case SAVIC_IOAPIC_START: + host_send_ioapic_irq(vm, id); + break; + case SAVIC_IOAPIC2_START: + host_send_ioapic_irq(vm, id); + break; default: break; } @@ -660,7 +900,6 @@ static void *vcpu_thread(void *arg) default: TEST_FAIL("Unknown ucall 0x%lx.", uc.cmd); } - }

return NULL; @@ -670,6 +909,11 @@ static void install_exception_handlers(struct kvm_vm *vm) { vm_install_exception_handler(vm, IDLE_HLT_INTR_VECTOR, guest_idle_hlt_intr_handler); vm_install_exception_handler(vm, 29, savic_vc_handler); + vm_install_exception_handler(vm, IOAPIC_VECTOR_START, ioapic_edge_irq1_intr_handler); + vm_install_exception_handler(vm, IOAPIC_VECTOR_START + 1, ioapic_edge_irq2_intr_handler); + vm_install_exception_handler(vm, IOAPIC_VECTOR_START + 2, ioapic_level_irq1_intr_handler); + vm_install_exception_handler(vm, IOAPIC_VECTOR_START + 3, ioapic_level_irq2_intr_handler); + vm_install_exception_handler(vm, RTC_GSI_IRQ, ioapic_rtc_gsi_intr_handler); }

int main(int argc, char *argv[]) @@ -677,8 +921,10 @@ int main(int argc, char *argv[]) struct kvm_sev_init args = { .vmsa_features = BIT_ULL(SVM_FEAT_SECURE_AVIC) | BIT_ULL(SVM_FEAT_ALLOWED_SEV_FEATURES_VALID) }; + struct test_data_page *shared_data[NR_SAVIC_VCPUS]; + vm_vaddr_t test_data_page_vaddr; struct kvm_vm *vm; - int r; + int i, r;

TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SNP)); TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SECURE_AVIC)); @@ -687,11 +933,21 @@ int main(int argc, char *argv[]) vm = _vm_sev_create_with_one_vcpu(KVM_X86_SNP_VM, guest_code, &vcpus[0], &args);

virt_pg_map(vm, APIC_DEFAULT_GPA, APIC_DEFAULT_GPA); + virt_pg_map(vm, IOAPIC_DEFAULT_GPA, IOAPIC_DEFAULT_GPA);

install_exception_handlers(vm);

vcpu_args_set(vcpus[0], 1, vcpus[0]->id);

+ for (i = 0; i < NR_SAVIC_VCPUS; i++) { + test_data_page_vaddr = vm_vaddr_alloc_page_shared(vm); + test_data[i] = (struct test_data_page *)test_data_page_vaddr; + shared_data[i] = addr_gva2hva(vm, test_data_page_vaddr); + vm_mem_set_shared(vm, addr_hva2gpa(vm, shared_data[i]), getpagesize()); + } + + sync_global_to_guest(vm, test_data); + vm_sev_launch(vm, snp_default_policy(), NULL);

r = pthread_create(&threads[0], NULL, vcpu_thread, vcpus[0]); @@ -699,6 +955,14 @@ int main(int argc, char *argv[])

pthread_join(threads[0], NULL);

+ for (i = 0; i < NR_SAVIC_VCPUS; i++) { + struct test_data_page *shared_state = shared_data[i]; + + fprintf(stderr, "VCPU %d ioapic edge irq1 count: %ld edge irq2 count: %ld\n", i, shared_state->ioapic_eirq1_count, shared_state->ioapic_eirq2_count); + fprintf(stderr, "VCPU %d ioapic level irq1 count: %ld level irq2 count: %ld\n", i, shared_state->ioapic_lirq1_count, shared_state->ioapic_lirq2_count); + fprintf(stderr, "VCPU %d ioapic RTC GSI irq1 count: %ld\n", i, shared_state->ioapic_rtc_gsi_irq_count); + } + kvm_vm_free(vm);

return 0;

Extend SAVIC test with various NMI tests:

- Test NMI injection failure when AllowedNMI is not set by guest. - Test NMI injection when 2 NMIs are pending. - Test ICR based cross-VCPU NMI for fixed-phys, fixed-logical, broadcast-all and broadcast-exclself modes.

Signed-off-by: Neeraj Upadhyay Neeraj.Upadhyay@amd.com --- .../selftests/kvm/include/x86/processor.h | 5 + tools/testing/selftests/kvm/include/x86/sev.h | 1 + tools/testing/selftests/kvm/lib/x86/savic.c | 15 +- tools/testing/selftests/kvm/lib/x86/sev.c | 22 + tools/testing/selftests/kvm/x86/savic_test.c | 388 ++++++++++++++---- 5 files changed, 351 insertions(+), 80 deletions(-)

diff --git a/tools/testing/selftests/kvm/include/x86/processor.h b/tools/testing/selftests/kvm/include/x86/processor.h index f09b18944c47..ebd30c4515e7 100644 --- a/tools/testing/selftests/kvm/include/x86/processor.h +++ b/tools/testing/selftests/kvm/include/x86/processor.h @@ -917,6 +917,11 @@ static inline void vcpu_xcrs_set(struct kvm_vcpu *vcpu, struct kvm_xcrs *xcrs) vcpu_ioctl(vcpu, KVM_SET_XCRS, xcrs); }

+static inline void vcpu_nmi(struct kvm_vcpu *vcpu) +{ + vcpu_ioctl(vcpu, KVM_NMI, NULL); +} + const struct kvm_cpuid_entry2 *get_cpuid_entry(const struct kvm_cpuid2 *cpuid, uint32_t function, uint32_t index); const struct kvm_cpuid2 *kvm_get_supported_cpuid(void); diff --git a/tools/testing/selftests/kvm/include/x86/sev.h b/tools/testing/selftests/kvm/include/x86/sev.h index 81ee79f63b7f..4e8944ca8440 100644 --- a/tools/testing/selftests/kvm/include/x86/sev.h +++ b/tools/testing/selftests/kvm/include/x86/sev.h @@ -160,4 +160,5 @@ void sev_es_vc_handler(struct ex_regs *regs); void sev_es_pv_msr_rw(uint64_t msr, uint64_t *data, bool write); void sev_es_pv_mmio_rw(uint32_t *reg_gpa, uint32_t *data, bool write); void sev_es_savic_notify_gpa(uint64_t gpa); +void sev_es_nmi_complete(void); #endif /* SELFTEST_KVM_SEV_H */ diff --git a/tools/testing/selftests/kvm/lib/x86/savic.c b/tools/testing/selftests/kvm/lib/x86/savic.c index ee2d6b66d6ce..f7301ec6d45f 100644 --- a/tools/testing/selftests/kvm/lib/x86/savic.c +++ b/tools/testing/selftests/kvm/lib/x86/savic.c @@ -91,13 +91,15 @@ int savic_nr_pages_required(uint64_t page_size) */ void set_savic_control_msr(struct guest_apic_page *apic_page, bool enable, bool enable_nmi) { - uint64_t val = apic_page->gpa | BIT_ULL(MSR_AMD64_SECURE_AVIC_EN_BIT); + uint64_t val;

if (!enable) { wrmsr(MSR_AMD64_SECURE_AVIC_CONTROL, 0); return; }

+ val = apic_page->gpa | BIT_ULL(MSR_AMD64_SECURE_AVIC_EN_BIT); + if (enable_nmi) val |= BIT_ULL(MSR_AMD64_SECURE_AVIC_ALLOWED_NMI_BIT);

@@ -442,15 +444,18 @@ static void savic_handle_icr_write(uint64_t icr_data) bool logical = icr_data & APIC_DEST_LOGICAL; bool nmi = (icr_data & APIC_DM_FIXED_MASK) == APIC_DM_NMI; uint64_t self_icr_data = APIC_DEST_SELF | APIC_INT_ASSERT | vector; - - if (nmi) - self_icr_data |= APIC_DM_NMI; + struct guest_apic_page *apic_page;

switch (dsh) { case APIC_DEST_ALLINC: savic_send_ipi_all_but(vector, nmi); savic_hv_write_reg(APIC_ICR, icr_data); - x2apic_write_reg(APIC_ICR, self_icr_data); + if (nmi) { + apic_page = &apic_page_pool->guest_apic_page[x2apic_read_reg(APIC_ID)]; + savic_write_reg(apic_page, SAVIC_NMI_REQ_OFFSET, 1); + } else { + x2apic_write_reg(APIC_ICR, self_icr_data); + } break; case APIC_DEST_ALLBUT: savic_send_ipi_all_but(vector, nmi); diff --git a/tools/testing/selftests/kvm/lib/x86/sev.c b/tools/testing/selftests/kvm/lib/x86/sev.c index 24a2a29a575b..446f7d7dafda 100644 --- a/tools/testing/selftests/kvm/lib/x86/sev.c +++ b/tools/testing/selftests/kvm/lib/x86/sev.c @@ -20,6 +20,7 @@ #define SVM_VMGEXIT_MMIO_READ 0x80000001 #define SVM_VMGEXIT_MMIO_WRITE 0x80000002 #define SVM_VMGEXIT_SECURE_AVIC 0x8000001a +#define SVM_VMGEXIT_NMI_COMPLETE 0x80000003

struct ghcb_entry { struct ghcb ghcb; @@ -798,3 +799,24 @@ void sev_es_savic_notify_gpa(uint64_t gpa) __GUEST_ASSERT(!ret, "Secure AVIC GPA notification failed, ret: %d", ret); ghcb_free(entry); } + +void sev_es_nmi_complete(void) +{ + struct ghcb_entry *entry; + struct ghcb *ghcb; + int ret; + + entry = ghcb_alloc(); + ghcb = &entry->ghcb; + register_ghcb_page(entry->gpa); + + ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_NMI_COMPLETE); + ghcb_set_sw_exit_info_1(ghcb, 0); + ghcb_set_sw_exit_info_2(ghcb, 0); + + do_vmg_exit(entry->gpa); + ret = ghcb->save.sw_exit_info_1 & 0xffffffff; + __GUEST_ASSERT(!ret, "NMI completion failed, ret: %d", ret); + + ghcb_free(entry); +} diff --git a/tools/testing/selftests/kvm/x86/savic_test.c b/tools/testing/selftests/kvm/x86/savic_test.c index d677b68aa6c6..277ee18a0cbd 100644 --- a/tools/testing/selftests/kvm/x86/savic_test.c +++ b/tools/testing/selftests/kvm/x86/savic_test.c @@ -13,8 +13,8 @@ #include "test_util.h" #include "savic.h"

-#define NR_SAVIC_VCPUS 2 -#define NUM_ITERATIONS 2000 +#define NR_SAVIC_VCPUS 4 +#define NUM_ITERATIONS 1000

#define IDLE_HLT_INTR_VECTOR 0x30 #define IOAPIC_VECTOR_START 0x81 @@ -47,6 +47,13 @@ enum savic_test_state { SAVIC_TEST_STATE(SAVIC_IOAPIC), SAVIC_TEST_STATE(SAVIC_IOAPIC2), SAVIC_TEST_STATE(SAVIC_IPI), + SAVIC_TEST_STATE(SAVIC_NMI), + SAVIC_TEST_STATE(SAVIC_NMI2), + SAVIC_TEST_STATE(SAVIC_NMI3), + SAVIC_TEST_STATE(SAVIC_ICR_FIXED_PHYS_NMI), + SAVIC_TEST_STATE(SAVIC_ICR_FIXED_LOGICAL_NMI), + SAVIC_TEST_STATE(SAVIC_ICR_BROADCAST_NMI), + SAVIC_TEST_STATE(SAVIC_ICR_BROADCAST_NOSELF_NMI), };

/* APIC reg values written by host. */ @@ -128,18 +135,32 @@ struct test_data_page { uint64_t ioapic_lirq1_count; uint64_t ioapic_lirq2_count; uint64_t ioapic_rtc_gsi_irq_count; - uint64_t fixed_ipi_wake_count; - uint64_t fixed_ipi_hlt_count; + uint64_t fixed_phys_ipi_wake_count; + uint64_t fixed_phys_ipi_hlt_count; uint64_t fixed_logical_ipi_hlt_count; uint64_t fixed_logical_ipi_wake_count; uint64_t broadcast_ipi_hlt_count; uint64_t broadcast_ipi_wake_count; uint64_t broadcast_noself_ipi_hlt_count; uint64_t broadcast_noself_ipi_wake_count; - uint64_t fixed_ipi_count; + uint64_t fixed_phys_ipi_count; uint64_t fixed_logical_ipi_count; uint64_t broadcast_ipi_count; uint64_t broadcast_noself_ipi_count; + uint64_t *nmi_count_p; + uint64_t nmi_count; + uint64_t fixed_phys_nmi_hlt_count; + uint64_t fixed_phys_nmi_wake_count; + uint64_t fixed_phys_nmi_count; + uint64_t fixed_logical_nmi_hlt_count; + uint64_t fixed_logical_nmi_wake_count; + uint64_t fixed_logical_nmi_count; + uint64_t broadcast_nmi_hlt_count; + uint64_t broadcast_nmi_wake_count; + uint64_t broadcast_nmi_count; + uint64_t broadcast_noself_nmi_hlt_count; + uint64_t broadcast_noself_nmi_wake_count; + uint64_t broadcast_noself_nmi_count; };

static struct test_data_page *test_data[NR_SAVIC_VCPUS]; @@ -581,9 +602,14 @@ static void _ioapic_level_irq_handler(int vec) vec); }

+static inline struct test_data_page *get_test_data(void) +{ + return test_data[x2apic_read_reg(APIC_ID)]; +} + static void ioapic_level_irq1_intr_handler(struct ex_regs *regs) { - struct test_data_page *data = test_data[x2apic_read_reg(APIC_ID)]; + struct test_data_page *data = get_test_data(); int vec;

vec = IOAPIC_VECTOR_START + IOAPIC_NUM_EDGE_VECTORS; @@ -593,7 +619,7 @@ static void ioapic_level_irq1_intr_handler(struct ex_regs *regs)

static void ioapic_level_irq2_intr_handler(struct ex_regs *regs) { - struct test_data_page *data = test_data[x2apic_read_reg(APIC_ID)]; + struct test_data_page *data = get_test_data(); int vec;

vec = IOAPIC_VECTOR_START + IOAPIC_NUM_EDGE_VECTORS + 1; @@ -603,7 +629,7 @@ static void ioapic_level_irq2_intr_handler(struct ex_regs *regs)

static void ioapic_edge_irq1_intr_handler(struct ex_regs *regs) { - struct test_data_page *data = test_data[x2apic_read_reg(APIC_ID)]; + struct test_data_page *data = get_test_data();

WRITE_ONCE(data->ioapic_eirq1_count, data->ioapic_eirq1_count + 1); x2apic_write_reg(APIC_EOI, 0x00); @@ -611,7 +637,7 @@ static void ioapic_edge_irq1_intr_handler(struct ex_regs *regs)

static void ioapic_edge_irq2_intr_handler(struct ex_regs *regs) { - struct test_data_page *data = test_data[x2apic_read_reg(APIC_ID)]; + struct test_data_page *data = get_test_data();

WRITE_ONCE(data->ioapic_eirq2_count, data->ioapic_eirq2_count + 1); x2apic_write_reg(APIC_EOI, 0x00); @@ -619,7 +645,7 @@ static void ioapic_edge_irq2_intr_handler(struct ex_regs *regs)

static void ioapic_rtc_gsi_intr_handler(struct ex_regs *regs) { - struct test_data_page *data = test_data[x2apic_read_reg(APIC_ID)]; + struct test_data_page *data = get_test_data();

WRITE_ONCE(data->ioapic_rtc_gsi_irq_count, data->ioapic_rtc_gsi_irq_count + 1); x2apic_write_reg(APIC_EOI, 0x00); @@ -627,7 +653,7 @@ static void ioapic_rtc_gsi_intr_handler(struct ex_regs *regs)

static void __savic_ioapic(int count) { - struct test_data_page *data = test_data[x2apic_read_reg(APIC_ID)]; + struct test_data_page *data = get_test_data(); int vec = IOAPIC_VECTOR_START;

__GUEST_ASSERT(READ_ONCE(data->ioapic_eirq1_count) == count, @@ -726,10 +752,40 @@ static void guest_setup_ioapic(int id) savic_allow_vector(vec); }

-static void savic_fixed_ipi(bool logical) +static void set_fixed_counters( + struct test_data_page *data, + uint64_t **fixed_ipi_p, + uint64_t **fixed_ipi_hlt_cnt_p, + uint64_t **fixed_ipi_wake_cnt_p, + bool logical, bool nmi) +{ + if (logical) { + *fixed_ipi_p = + nmi ? &data->fixed_logical_nmi_count : + &data->fixed_logical_ipi_count; + *fixed_ipi_hlt_cnt_p = + nmi ? &data->fixed_logical_nmi_hlt_count : + &data->fixed_logical_ipi_hlt_count; + *fixed_ipi_wake_cnt_p = + nmi ? &data->fixed_logical_nmi_wake_count : + &data->fixed_logical_ipi_wake_count; + } else { + *fixed_ipi_p = + nmi ? &data->fixed_phys_nmi_count : + &data->fixed_phys_ipi_count; + *fixed_ipi_hlt_cnt_p = + nmi ? &data->fixed_phys_nmi_hlt_count : + &data->fixed_phys_ipi_hlt_count; + *fixed_ipi_wake_cnt_p = + nmi ? &data->fixed_phys_nmi_wake_count : + &data->fixed_phys_ipi_wake_count; + } +} + +static void savic_fixed_ipi(bool logical, bool nmi) { uint64_t last_wake_cnt, last_hlt_cnt; - uint64_t last_fixed_ipi_cnt; + uint64_t last_fixed_phys_ipi_cnt; uint64_t tsc_start; uint64_t *fixed_ipi_p; uint64_t *fixed_ipi_hlt_cnt_p; @@ -741,26 +797,26 @@ static void savic_fixed_ipi(bool logical) struct test_data_page *data = test_data[i]; uint64_t dst_apic_id = i;

+ set_fixed_counters(data, &fixed_ipi_p, + &fixed_ipi_hlt_cnt_p, &fixed_ipi_wake_cnt_p, + logical, nmi); if (logical) { - fixed_ipi_p = &data->fixed_logical_ipi_count; - fixed_ipi_hlt_cnt_p = &data->fixed_logical_ipi_hlt_count; - fixed_ipi_wake_cnt_p = &data->fixed_logical_ipi_wake_count; vec = FIXED_LOGICAL_IPI_VEC | APIC_DEST_LOGICAL; dst_apic_id = 1 << i; } else { - fixed_ipi_p = &data->fixed_ipi_count; - fixed_ipi_hlt_cnt_p = &data->fixed_ipi_hlt_count; - fixed_ipi_wake_cnt_p = &data->fixed_ipi_wake_count; vec = FIXED_IPI_VEC; dst_apic_id = i; }

+ if (nmi) + vec |= APIC_DM_NMI; + last_wake_cnt = READ_ONCE(*fixed_ipi_wake_cnt_p); while (!READ_ONCE(*fixed_ipi_hlt_cnt_p)) ;

last_hlt_cnt = READ_ONCE(*fixed_ipi_hlt_cnt_p); - last_fixed_ipi_cnt = READ_ONCE(*fixed_ipi_p); + last_fixed_phys_ipi_cnt = READ_ONCE(*fixed_ipi_p);

for (j = 0; j < NUM_ITERATIONS; j++) { tsc_start = rdtsc(); @@ -769,31 +825,73 @@ static void savic_fixed_ipi(bool logical) while (rdtsc() - tsc_start < 1000000000) { if (READ_ONCE(*fixed_ipi_wake_cnt_p) != last_wake_cnt && READ_ONCE(*fixed_ipi_hlt_cnt_p) != last_hlt_cnt && - READ_ONCE(*fixed_ipi_p) != last_fixed_ipi_cnt) + READ_ONCE(*fixed_ipi_p) != last_fixed_phys_ipi_cnt) break; }

__GUEST_ASSERT(READ_ONCE(*fixed_ipi_wake_cnt_p) != last_wake_cnt && READ_ONCE(*fixed_ipi_hlt_cnt_p) != last_hlt_cnt && - READ_ONCE(*fixed_ipi_p) != last_fixed_ipi_cnt, - "wakeup_cnt: %ld last_wake_cnt: %ld hlt_count: %ld last_hlt_cnt: %ld d_ipi_count: %ld last_d_ipi_count: %ld", + READ_ONCE(*fixed_ipi_p) != last_fixed_phys_ipi_cnt, + "%s fixed-%s wake: %ld last_wake: %ld hlt: %ld last_hlt: %ld ipi: %ld last_ipi: %ld", + nmi ? "nmi" : "ipi", + logical ? "logical" : "phys", READ_ONCE(*fixed_ipi_wake_cnt_p), last_wake_cnt, READ_ONCE(*fixed_ipi_hlt_cnt_p), last_hlt_cnt, - READ_ONCE(*fixed_ipi_p), last_fixed_ipi_cnt); + READ_ONCE(*fixed_ipi_p), last_fixed_phys_ipi_cnt);

last_wake_cnt = READ_ONCE(*fixed_ipi_wake_cnt_p); last_hlt_cnt = READ_ONCE(*fixed_ipi_hlt_cnt_p); - last_fixed_ipi_cnt = READ_ONCE(*fixed_ipi_p); + last_fixed_phys_ipi_cnt = READ_ONCE(*fixed_ipi_p); } } }

-static void savic_send_broadcast(int dsh) +static uint64_t *get_broadcast_ipi_counter(struct test_data_page *data, + int dsh, bool nmi) +{ + if (dsh == APIC_DEST_ALLINC) + return nmi ? + &data->broadcast_nmi_count : + &data->broadcast_ipi_count; + else + return nmi ? + &data->broadcast_noself_nmi_count : + &data->broadcast_noself_ipi_count; +} + + +static uint64_t *get_broadcast_hlt_counter(struct test_data_page *data, + int dsh, bool nmi) +{ + if (dsh == APIC_DEST_ALLINC) + return nmi ? + &data->broadcast_nmi_hlt_count : + &data->broadcast_ipi_hlt_count; + else + return nmi ? + &data->broadcast_noself_nmi_hlt_count : + &data->broadcast_noself_ipi_hlt_count; +} + +static uint64_t *get_broadcast_wake_counter(struct test_data_page *data, + int dsh, bool nmi) +{ + if (dsh == APIC_DEST_ALLINC) + return nmi ? + &data->broadcast_nmi_wake_count : + &data->broadcast_ipi_wake_count; + else + return nmi ? + &data->broadcast_noself_nmi_wake_count : + &data->broadcast_noself_ipi_wake_count; +} + +static void savic_send_broadcast(int dsh, bool nmi) { uint64_t last_wake_cnt[NR_SAVIC_VCPUS], last_hlt_cnt[NR_SAVIC_VCPUS]; uint64_t last_ipi_cnt[NR_SAVIC_VCPUS]; uint64_t tsc_start; - uint64_t *broadcast_ipi_p; + uint64_t *broadcast_ipi_cnt_p; uint64_t *broadcast_ipi_hlt_cnt_p; uint64_t *broadcast_ipi_wake_cnt_p; struct test_data_page *data; @@ -808,10 +906,8 @@ static void savic_send_broadcast(int dsh) for (i = 1; i < NR_SAVIC_VCPUS; i++) { data = test_data[i];

- if (dsh == APIC_DEST_ALLINC) - broadcast_ipi_hlt_cnt_p = &data->broadcast_ipi_hlt_count; - else - broadcast_ipi_hlt_cnt_p = &data->broadcast_noself_ipi_hlt_count; + broadcast_ipi_hlt_cnt_p = get_broadcast_hlt_counter( + data, dsh, nmi);

while (!READ_ONCE(*broadcast_ipi_hlt_cnt_p)) ; @@ -821,17 +917,20 @@ static void savic_send_broadcast(int dsh) for (i = 1; i < NR_SAVIC_VCPUS; i++) { data = test_data[i];

- if (dsh == APIC_DEST_ALLINC) { - last_hlt_cnt[i] = READ_ONCE(data->broadcast_ipi_hlt_count); - last_ipi_cnt[i] = READ_ONCE(data->broadcast_ipi_count); - last_wake_cnt[i] = READ_ONCE(data->broadcast_ipi_wake_count); - } else { - last_hlt_cnt[i] = READ_ONCE(data->broadcast_noself_ipi_hlt_count); - last_ipi_cnt[i] = READ_ONCE(data->broadcast_noself_ipi_count); - last_wake_cnt[i] = READ_ONCE(data->broadcast_noself_ipi_wake_count); - } + broadcast_ipi_cnt_p = get_broadcast_ipi_counter( + data, dsh, nmi); + broadcast_ipi_hlt_cnt_p = get_broadcast_hlt_counter( + data, dsh, nmi); + broadcast_ipi_wake_cnt_p = get_broadcast_wake_counter( + data, dsh, nmi); + last_ipi_cnt[i] = *broadcast_ipi_cnt_p; + last_hlt_cnt[i] = *broadcast_ipi_hlt_cnt_p; + last_wake_cnt[i] = *broadcast_ipi_wake_cnt_p; }

+ if (nmi) + vec |= APIC_DM_NMI; + x2apic_write_reg(APIC_ICR, APIC_INT_ASSERT | dsh | vec);

tsc_start = rdtsc(); @@ -839,60 +938,59 @@ static void savic_send_broadcast(int dsh) for (i = 1; i < NR_SAVIC_VCPUS; i++) { data = test_data[i];

- if (dsh == APIC_DEST_ALLINC) { - broadcast_ipi_p = &data->broadcast_ipi_count; - broadcast_ipi_hlt_cnt_p = &data->broadcast_ipi_hlt_count; - broadcast_ipi_wake_cnt_p = &data->broadcast_ipi_wake_count; - } else { - broadcast_ipi_p = &data->broadcast_noself_ipi_count; - broadcast_ipi_hlt_cnt_p = &data->broadcast_noself_ipi_hlt_count; - broadcast_ipi_wake_cnt_p = &data->broadcast_noself_ipi_wake_count; - } + broadcast_ipi_cnt_p = get_broadcast_ipi_counter( + data, dsh, nmi); + broadcast_ipi_hlt_cnt_p = get_broadcast_hlt_counter( + data, dsh, nmi); + broadcast_ipi_wake_cnt_p = get_broadcast_wake_counter( + data, dsh, nmi);

while (rdtsc() - tsc_start < 1000000000) { if (READ_ONCE(*broadcast_ipi_wake_cnt_p) != last_wake_cnt[i] && READ_ONCE(*broadcast_ipi_hlt_cnt_p) != last_hlt_cnt[i] && - READ_ONCE(*broadcast_ipi_p) != last_ipi_cnt[i]) + READ_ONCE(*broadcast_ipi_cnt_p) != last_ipi_cnt[i]) break; }

__GUEST_ASSERT(READ_ONCE(*broadcast_ipi_wake_cnt_p) != last_wake_cnt[i] && READ_ONCE(*broadcast_ipi_hlt_cnt_p) != last_hlt_cnt[i] && - READ_ONCE(*broadcast_ipi_p) != last_ipi_cnt[i], - "wakeup_cnt: %ld last_wake_cnt: %ld hlt_count: %ld last_hlt_cnt: %ld b_ipi_count: %ld last_b_ipi_count: %ld", + READ_ONCE(*broadcast_ipi_cnt_p) != last_ipi_cnt[i], + "%s broadcast-%s wake: %ld last_wake: %ld hlt: %ld last_hlt: %ld ipi: %ld last_ipi: %ld", + nmi ? "nmi" : "ipi", + dsh == APIC_DEST_ALLINC ? "all" : "excl-self", READ_ONCE(*broadcast_ipi_wake_cnt_p), last_wake_cnt[i], READ_ONCE(*broadcast_ipi_hlt_cnt_p), last_hlt_cnt[i], - READ_ONCE(*broadcast_ipi_p), last_ipi_cnt[i]); + READ_ONCE(*broadcast_ipi_cnt_p), last_ipi_cnt[i]);

last_wake_cnt[i] = READ_ONCE(*broadcast_ipi_wake_cnt_p); last_hlt_cnt[i] = READ_ONCE(*broadcast_ipi_hlt_cnt_p); - last_ipi_cnt[i] = READ_ONCE(*broadcast_ipi_p); + last_ipi_cnt[i] = READ_ONCE(*broadcast_ipi_cnt_p); } } }

void savic_ipi(int id) { - savic_fixed_ipi(false); - savic_fixed_ipi(true); + savic_fixed_ipi(false, false); + savic_fixed_ipi(true, false);

asm volatile("sti;":::"memory"); x2apic_write_reg(APIC_TASKPRI, 0); - savic_send_broadcast(APIC_DEST_ALLINC); - savic_send_broadcast(APIC_DEST_ALLBUT); + savic_send_broadcast(APIC_DEST_ALLINC, false); + savic_send_broadcast(APIC_DEST_ALLBUT, false); }

-void guest_fixed_ipi_handler(struct ex_regs *regs) +void guest_fixed_phys_ipi_handler(struct ex_regs *regs) { - struct test_data_page *data = test_data[x2apic_read_reg(APIC_ID)]; + struct test_data_page *data = get_test_data();

- WRITE_ONCE(data->fixed_ipi_count, data->fixed_ipi_count + 1); + WRITE_ONCE(data->fixed_phys_ipi_count, data->fixed_phys_ipi_count + 1); x2apic_write_reg(APIC_EOI, 0x00); }

void guest_fixed_logical_ipi_handler(struct ex_regs *regs) { - struct test_data_page *data = test_data[x2apic_read_reg(APIC_ID)]; + struct test_data_page *data = get_test_data();

WRITE_ONCE(data->fixed_logical_ipi_count, data->fixed_logical_ipi_count + 1); x2apic_write_reg(APIC_EOI, 0x00); @@ -900,7 +998,7 @@ void guest_fixed_logical_ipi_handler(struct ex_regs *regs)

void guest_broadcast_ipi_handler(struct ex_regs *regs) { - struct test_data_page *data = test_data[x2apic_read_reg(APIC_ID)]; + struct test_data_page *data = get_test_data();

WRITE_ONCE(data->broadcast_ipi_count, data->broadcast_ipi_count + 1); x2apic_write_reg(APIC_EOI, 0x00); @@ -908,13 +1006,65 @@ void guest_broadcast_ipi_handler(struct ex_regs *regs)

void guest_broadcast_noself_ipi_handler(struct ex_regs *regs) { - struct test_data_page *data = test_data[x2apic_read_reg(APIC_ID)]; + struct test_data_page *data = get_test_data();

WRITE_ONCE(data->broadcast_noself_ipi_count, data->broadcast_noself_ipi_count + 1); x2apic_write_reg(APIC_EOI, 0x00); }

-static void ipi_guest_code(int id, unsigned long secondary_entry) +static void savic_nmi(int id) +{ + struct test_data_page *data = get_test_data(); + + __GUEST_ASSERT(!data->nmi_count, "Invalid NMI count: %ld\n", data->nmi_count); + set_savic_control_msr(get_guest_apic_page(), true, true); +} + +static void savic_nmi2(int id) +{ + struct test_data_page *data = get_test_data(); + + __GUEST_ASSERT(data->nmi_count == 2, "Invalid NMI count: %ld\n", data->nmi_count); +} + +static void savic_nmi3(int id) +{ + struct test_data_page *data = get_test_data(); + + __GUEST_ASSERT(data->nmi_count == 4, "Invalid NMI count: %ld\n", data->nmi_count); +} + +static void savic_icr_fixed_phys(int id) +{ + savic_fixed_ipi(false, true); +} + +static void savic_icr_fixed_logical(int id) +{ + savic_fixed_ipi(true, true); +} + +static void savic_icr_bcast(int id) +{ + savic_send_broadcast(APIC_DEST_ALLINC, true); +} + +static void savic_icr_bcast_noself(int id) +{ + savic_send_broadcast(APIC_DEST_ALLBUT, true); +} + +static void guest_nmi_handler(struct ex_regs *regs) +{ + struct test_data_page *data = get_test_data(); + + WRITE_ONCE(*data->nmi_count_p, *data->nmi_count_p + 1); + /* Skip NMI completed notification for ICR based NMI. */ + if (data->nmi_count_p == &data->nmi_count) + sev_es_nmi_complete(); +} + +static void ipi_guest_code(int id) { struct test_data_page *data; uint64_t *ipi_count_p, *hlt_count_p, *wake_count_p; @@ -928,9 +1078,9 @@ static void ipi_guest_code(int id, unsigned long secondary_entry)

uint64_t *ipi_count_types[][3] = { { - &data->fixed_ipi_hlt_count, - &data->fixed_ipi_count, - &data->fixed_ipi_wake_count + &data->fixed_phys_ipi_hlt_count, + &data->fixed_phys_ipi_count, + &data->fixed_phys_ipi_wake_count }, { &data->fixed_logical_ipi_hlt_count, @@ -947,6 +1097,26 @@ static void ipi_guest_code(int id, unsigned long secondary_entry) &data->broadcast_noself_ipi_count, &data->broadcast_noself_ipi_wake_count }, + { + &data->fixed_phys_nmi_hlt_count, + &data->fixed_phys_nmi_count, + &data->fixed_phys_nmi_wake_count + }, + { + &data->fixed_logical_nmi_hlt_count, + &data->fixed_logical_nmi_count, + &data->fixed_logical_nmi_wake_count + }, + { + &data->broadcast_nmi_hlt_count, + &data->broadcast_nmi_count, + &data->broadcast_nmi_wake_count + }, + { + &data->broadcast_noself_nmi_hlt_count, + &data->broadcast_noself_nmi_count, + &data->broadcast_noself_nmi_wake_count + }, };

for (i = 0; i < ARRAY_SIZE(ipi_count_types); i++) { @@ -955,9 +1125,11 @@ static void ipi_guest_code(int id, unsigned long secondary_entry) wake_count_p = ipi_count_types[i][2];

while (READ_ONCE(*ipi_count_p) != NUM_ITERATIONS) { - asm volatile("cli"); + if (i < 4) + asm volatile("cli"); WRITE_ONCE(*hlt_count_p, *hlt_count_p + 1); - asm volatile("sti; hlt" : : : "memory"); + if (i < 4) + asm volatile("sti; hlt" : : : "memory"); WRITE_ONCE(*wake_count_p, *wake_count_p + 1); }

@@ -969,6 +1141,9 @@ static void ipi_guest_code(int id, unsigned long secondary_entry)

static void guest_code(int id) { + struct test_data_page *data; + int i; + GUEST_ASSERT(rdmsr(MSR_AMD64_SEV) & MSR_AMD64_SNP_SECURE_AVIC);

SAVIC_GUEST_SYNC(RESET, guest_savic_start); @@ -991,6 +1166,39 @@ static void guest_code(int id)

SAVIC_GUEST_SYNC(SAVIC_IPI, savic_ipi);

+ /* Disable host NMI injection in control MSR. */ + set_savic_control_msr(get_guest_apic_page(), true, false); + + data = test_data[id]; + data->nmi_count_p = &data->nmi_count; + SAVIC_GUEST_SYNC(SAVIC_NMI, savic_nmi); + SAVIC_GUEST_SYNC(SAVIC_NMI2, savic_nmi2); + SAVIC_GUEST_SYNC(SAVIC_NMI3, savic_nmi3); + + for (i = 0; i < NR_SAVIC_VCPUS; i++) { + data = test_data[i]; + data->nmi_count_p = &data->fixed_phys_nmi_count; + } + SAVIC_GUEST_SYNC(SAVIC_ICR_FIXED_PHYS_NMI, savic_icr_fixed_phys); + + for (i = 0; i < NR_SAVIC_VCPUS; i++) { + data = test_data[i]; + data->nmi_count_p = &data->fixed_logical_nmi_count; + } + SAVIC_GUEST_SYNC(SAVIC_ICR_FIXED_LOGICAL_NMI, savic_icr_fixed_logical); + + for (i = 0; i < NR_SAVIC_VCPUS; i++) { + data = test_data[i]; + data->nmi_count_p = &data->broadcast_nmi_count; + } + SAVIC_GUEST_SYNC(SAVIC_ICR_BROADCAST_NMI, savic_icr_bcast); + + for (i = 0; i < NR_SAVIC_VCPUS; i++) { + data = test_data[i]; + data->nmi_count_p = &data->broadcast_noself_nmi_count; + } + SAVIC_GUEST_SYNC(SAVIC_ICR_BROADCAST_NOSELF_NMI, savic_icr_bcast_noself); + GUEST_DONE(); }

@@ -1105,6 +1313,12 @@ static void host_send_ioapic_irq(struct kvm_vm *vm, int id) } }

+static void host_send_nmi(int id) +{ + vcpu_nmi(vcpus[id]); + vcpu_nmi(vcpus[id]); +} + static void host_test_savic(struct kvm_vm *vm, int id, enum savic_test_state test_state) { switch (test_state) { @@ -1132,6 +1346,11 @@ static void host_test_savic(struct kvm_vm *vm, int id, enum savic_test_state tes case SAVIC_IOAPIC2_START: host_send_ioapic_irq(vm, id); break; + case SAVIC_NMI_START: + case SAVIC_NMI2_START: + case SAVIC_NMI3_START: + host_send_nmi(id); + break; default: break; } @@ -1174,11 +1393,12 @@ static void install_exception_handlers(struct kvm_vm *vm) vm_install_exception_handler(vm, IOAPIC_VECTOR_START + 2, ioapic_level_irq1_intr_handler); vm_install_exception_handler(vm, IOAPIC_VECTOR_START + 3, ioapic_level_irq2_intr_handler); vm_install_exception_handler(vm, RTC_GSI_IRQ, ioapic_rtc_gsi_intr_handler); - vm_install_exception_handler(vm, FIXED_IPI_VEC, guest_fixed_ipi_handler); + vm_install_exception_handler(vm, FIXED_IPI_VEC, guest_fixed_phys_ipi_handler); vm_install_exception_handler(vm, FIXED_LOGICAL_IPI_VEC, guest_fixed_logical_ipi_handler); vm_install_exception_handler(vm, BROADCAST_ALL_IPI_VEC, guest_broadcast_ipi_handler); vm_install_exception_handler(vm, BROADCAST_NOSELF_IPI_VEC, guest_broadcast_noself_ipi_handler); + vm_install_exception_handler(vm, NMI_VECTOR, guest_nmi_handler); }

int main(int argc, char *argv[]) @@ -1239,9 +1459,9 @@ int main(int argc, char *argv[]) fprintf(stderr, "VCPU %d ioapic level irq1 count: %ld level irq2 count: %ld\n", i, shared_state->ioapic_lirq1_count, shared_state->ioapic_lirq2_count); fprintf(stderr, "VCPU %d ioapic RTC GSI irq1 count: %ld\n", i, shared_state->ioapic_rtc_gsi_irq_count); fprintf(stderr, "vCPU %d fixed IPI counts wake: %ld hlt: %ld num-IPI: %ld\n", - i, shared_state->fixed_ipi_wake_count, - shared_state->fixed_ipi_hlt_count, - shared_state->fixed_ipi_count); + i, shared_state->fixed_phys_ipi_wake_count, + shared_state->fixed_phys_ipi_hlt_count, + shared_state->fixed_phys_ipi_count); fprintf(stderr, "vCPU %d fixed-logical IPI counts wake: %ld hlt: %ld num-IPI: %ld\n", i, shared_state->fixed_logical_ipi_wake_count, shared_state->fixed_logical_ipi_hlt_count, @@ -1250,10 +1470,28 @@ int main(int argc, char *argv[]) i, shared_state->broadcast_ipi_wake_count, shared_state->broadcast_ipi_hlt_count, shared_state->broadcast_ipi_count); - fprintf(stderr, "vCPU %d broadcast exluding self IPI counts wake: %ld hlt: %ld num-IPI: %ld\n", + fprintf(stderr, "vCPU %d broadcast excluding self IPI counts wake: %ld hlt: %ld num-IPI: %ld\n", i, shared_state->broadcast_noself_ipi_wake_count, shared_state->broadcast_noself_ipi_hlt_count, shared_state->broadcast_noself_ipi_count); + fprintf(stderr, "vCPU %d nmi count: %ld\n", + i, shared_state->nmi_count); + fprintf(stderr, "vCPU %d nmi fixed IPI counts wake: %ld hlt: %ld num-IPI: %ld\n", + i, shared_state->fixed_phys_nmi_wake_count, + shared_state->fixed_phys_nmi_hlt_count, + shared_state->fixed_phys_nmi_count); + fprintf(stderr, "vCPU %d nmi fixed-logical IPI counts wake: %ld hlt: %ld num-IPI: %ld\n", + i, shared_state->fixed_logical_nmi_wake_count, + shared_state->fixed_logical_nmi_hlt_count, + shared_state->fixed_logical_nmi_count); + fprintf(stderr, "vCPU %d nmi broadcast IPI counts wake: %ld hlt: %ld num-IPI: %ld\n", + i, shared_state->broadcast_nmi_wake_count, + shared_state->broadcast_nmi_hlt_count, + shared_state->broadcast_nmi_count); + fprintf(stderr, "vCPU %d nmi broadcast excluding self IPI counts wake: %ld hlt: %ld num-IPI: %ld\n", + i, shared_state->broadcast_noself_nmi_wake_count, + shared_state->broadcast_noself_nmi_hlt_count, + shared_state->broadcast_noself_nmi_count); }

kvm_vm_free(vm);