Re: [RFC PATCH v2 1/9] KVM: arm64: nv: selftests: Add support to run guest code in vEL2.

Hi,

On 5/12/25 12:52 PM, Ganapatrao Kulkarni wrote:

This patch adds required changes to vcpu init to run a guest code in vEL2 context and also adds NV specific helper functions.

Signed-off-by: Ganapatrao Kulkarni gankulkarni@os.amperecomputing.com

tools/testing/selftests/kvm/Makefile.kvm | 2 + .../kvm/include/arm64/kvm_util_arch.h | 3 + .../selftests/kvm/include/arm64/nv_util.h | 45 ++++++++++++++ .../selftests/kvm/include/arm64/vgic.h | 1 + .../testing/selftests/kvm/include/kvm_util.h | 3 + tools/testing/selftests/kvm/lib/arm64/nv.c | 46 ++++++++++++++ .../selftests/kvm/lib/arm64/processor.c | 61 ++++++++++++++----- tools/testing/selftests/kvm/lib/arm64/vgic.c | 8 +++ 8 files changed, 155 insertions(+), 14 deletions(-) create mode 100644 tools/testing/selftests/kvm/include/arm64/nv_util.h create mode 100644 tools/testing/selftests/kvm/lib/arm64/nv.c

diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index f773f8f99249..3348f729d3b2 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -37,6 +37,7 @@ LIBKVM_arm64 += lib/arm64/processor.c LIBKVM_arm64 += lib/arm64/spinlock.c LIBKVM_arm64 += lib/arm64/ucall.c LIBKVM_arm64 += lib/arm64/vgic.c +LIBKVM_arm64 += lib/arm64/nv.c LIBKVM_s390 += lib/s390/diag318_test_handler.c LIBKVM_s390 += lib/s390/processor.c @@ -155,6 +156,7 @@ TEST_GEN_PROGS_arm64 += arm64/vgic_irq TEST_GEN_PROGS_arm64 += arm64/vgic_lpi_stress TEST_GEN_PROGS_arm64 += arm64/vpmu_counter_access TEST_GEN_PROGS_arm64 += arm64/no-vgic-v3 +TEST_GEN_PROGS_arm64 += arm64/nv_guest_hypervisor TEST_GEN_PROGS_arm64 += access_tracking_perf_test TEST_GEN_PROGS_arm64 += arch_timer TEST_GEN_PROGS_arm64 += coalesced_io_test diff --git a/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h b/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h index e43a57d99b56..ab5279c24413 100644 --- a/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h +++ b/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h @@ -2,6 +2,9 @@ #ifndef SELFTEST_KVM_UTIL_ARCH_H #define SELFTEST_KVM_UTIL_ARCH_H +#define CurrentEL_EL1 (1 << 2) +#define CurrentEL_EL2 (2 << 2)

nit: not yet used in this patch.

struct kvm_vm_arch {}; #endif // SELFTEST_KVM_UTIL_ARCH_H diff --git a/tools/testing/selftests/kvm/include/arm64/nv_util.h b/tools/testing/selftests/kvm/include/arm64/nv_util.h new file mode 100644 index 000000000000..622a17c9d142 --- /dev/null +++ b/tools/testing/selftests/kvm/include/arm64/nv_util.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/*

• • Copyright (c) 2025 Ampere Computing
• */

+#ifndef SELFTEST_NV_UTIL_H +#define SELFTEST_NV_UTIL_H

+#include <linux/bitmap.h> +#include <vgic.h>

+#define HCR_NV2 (UL(1) << 45) +#define HCR_AT (UL(1) << 44) +#define HCR_NV (UL(1) << 42) +#define HCR_E2H (UL(1) << 34) +#define HCR_TTLB (UL(1) << 25)

+/* Enable NV2 and guest in VHE mode */ +#define HCR_EL2_NV_EANBLE (HCR_E2H | HCR_NV | HCR_NV2 | HCR_AT | HCR_TTLB)

ENABLE

+struct kvm_vm *nv_vm_create_with_vcpus_gic(uint32_t nr_vcpus,

• struct kvm_vcpu **vcpus, int *gic_fd, void *guest_code);
  

+struct kvm_vm *__nv_vm_create_with_vcpus_gic(struct vm_shape shape,

• uint32_t nr_vcpus, struct kvm_vcpu **vcpus,
  

• uint64_t extra_mem_pages, int *gic_fd, void *guest_code);
  

+/* NV helpers */ +static inline void init_vcpu_nested(struct kvm_vcpu_init *init) +{

• init->features[0] |= (1 << KVM_ARM_VCPU_HAS_EL2);

+}

+static inline bool kvm_arm_vcpu_has_el2(struct kvm_vcpu_init *init) +{

• unsigned long features = init->features[0];
• return test_bit(KVM_ARM_VCPU_HAS_EL2, &features);

+}

+static inline bool is_vcpu_nested(struct kvm_vcpu *vcpu) +{

• return vcpu->nested;

+}

+#endif /* SELFTEST_NV_UTIL_H */ diff --git a/tools/testing/selftests/kvm/include/arm64/vgic.h b/tools/testing/selftests/kvm/include/arm64/vgic.h index c481d0c00a5d..46142fa36199 100644 --- a/tools/testing/selftests/kvm/include/arm64/vgic.h +++ b/tools/testing/selftests/kvm/include/arm64/vgic.h @@ -17,6 +17,7 @@ index) int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, uint32_t nr_irqs); +void vgic_v3_close(int gic_fd); #define VGIC_MAX_RESERVED 1023 diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 373912464fb4..9b26b9124dc3 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -65,6 +65,9 @@ struct kvm_vcpu { struct kvm_dirty_gfn *dirty_gfns; uint32_t fetch_index; uint32_t dirty_gfns_count; +#ifdef __aarch64__

• bool nested;

+#endif }; struct userspace_mem_regions { diff --git a/tools/testing/selftests/kvm/lib/arm64/nv.c b/tools/testing/selftests/kvm/lib/arm64/nv.c new file mode 100644 index 000000000000..e930808a7ed4 --- /dev/null +++ b/tools/testing/selftests/kvm/lib/arm64/nv.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0 +/*

• • Copyright (c) 2025 Ampere Computing LLC
• */

+#include <linux/compiler.h> +#include <assert.h>

+#include "guest_modes.h" +#include "kvm_util.h" +#include "nv_util.h" +#include "processor.h"

+struct kvm_vm *__nv_vm_create_with_vcpus_gic(struct vm_shape shape, uint32_t nr_vcpus,

• struct kvm_vcpu **vcpus, uint64_t extra_mem_pages, int *gic_fd, void *guest_code)
  

+{

• struct kvm_vcpu_init init;
• struct kvm_vm *vm;
• int i;
• TEST_REQUIRE(kvm_has_cap(KVM_CAP_ARM_EL2));
• vm = __vm_create(shape, nr_vcpus, extra_mem_pages);
• vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &init);
• init_vcpu_nested(&init);
• for (i = 0; i < nr_vcpus; ++i) {

• vcpus[i] = aarch64_vcpu_add(vm, i, &init, guest_code);
  

• __TEST_REQUIRE(is_vcpu_nested(vcpus[i]), "Failed to Enable NV");
  

• }
• /* vgic is not created, If gic_fd argument is NULL */
• if (gic_fd) {

• *gic_fd = vgic_v3_setup(vm, nr_vcpus, 64);
  

• __TEST_REQUIRE(*gic_fd >= 0, "Failed to create vgic-v3");
  

• }
• return vm;

+}

+struct kvm_vm *nv_vm_create_with_vcpus_gic(uint32_t nr_vcpus,

• struct kvm_vcpu **vcpus, int *gic_fd, void *guest_code)
  

+{

• return __nv_vm_create_with_vcpus_gic(VM_SHAPE_DEFAULT,

• 		nr_vcpus, vcpus, 0, gic_fd, guest_code);
  

+} diff --git a/tools/testing/selftests/kvm/lib/arm64/processor.c b/tools/testing/selftests/kvm/lib/arm64/processor.c index 7ba3aa3755f3..6e759981bf9e 100644 --- a/tools/testing/selftests/kvm/lib/arm64/processor.c +++ b/tools/testing/selftests/kvm/lib/arm64/processor.c @@ -10,6 +10,7 @@ #include "guest_modes.h" #include "kvm_util.h" +#include "nv_util.h" #include "processor.h" #include "ucall_common.h" @@ -258,14 +259,49 @@ void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) } } +static void aarch64_vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t sctlr_el1,

• 	uint64_t tcr_el1, uint64_t ttbr0_el1)
  

I am not sure it is worth introducing this helper. Some reg values are set externally while fpen is set here. That looks odd to me.

+{

• uint64_t fpen;
• /*

• * Enable FP/ASIMD to avoid trapping when accessing Q0-Q15
  

• * registers, which the variable argument list macros do.
  

• */
  

• fpen = 3 << 20;
• if (is_vcpu_nested(vcpu)) {

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPTR_EL2), fpen);
  

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL2), sctlr_el1);
  

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL2), tcr_el1);
  

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL2), DEFAULT_MAIR_EL1);
  

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL2), ttbr0_el1);
  

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_HCR_EL2), HCR_EL2_NV_EANBLE);
  

• } else {

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPACR_EL1), fpen);
  

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1), sctlr_el1);
  

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL1), tcr_el1);
  

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL1), DEFAULT_MAIR_EL1);
  

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL1), ttbr0_el1);
  

• }
• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL1), vcpu->id);

+}

void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init) { struct kvm_vcpu_init default_init = { .target = -1, }; struct kvm_vm *vm = vcpu->vm; uint64_t sctlr_el1, tcr_el1, ttbr0_el1;

• if (!init)

• if (!init) { init = &default_init;
• } else {

• /* Is this vcpu a Guest-Hypersior */
  

• if (kvm_arm_vcpu_has_el2(init))
  

• 	vcpu->nested = true;
  

• }

if (init->target == -1) { struct kvm_vcpu_init preferred; @@ -275,12 +311,6 @@ void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init) vcpu_ioctl(vcpu, KVM_ARM_VCPU_INIT, init);

• /*

• * Enable FP/ASIMD to avoid trapping when accessing Q0-Q15
  

• * registers, which the variable argument list macros do.
  

• */
  

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPACR_EL1), 3 << 20);
• sctlr_el1 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1)); tcr_el1 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL1));

@@ -349,11 +379,7 @@ void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init) if (use_lpa2_pte_format(vm)) tcr_el1 |= (1ul << 59) /* DS */;

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1), sctlr_el1);
• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL1), tcr_el1);
• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL1), DEFAULT_MAIR_EL1);
• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL1), ttbr0_el1);
• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL1), vcpu->id);

• aarch64_vcpu_set_reg(vcpu, sctlr_el1, tcr_el1, ttbr0_el1);

} void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, uint8_t indent) @@ -387,7 +413,11 @@ static struct kvm_vcpu *__aarch64_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id, aarch64_vcpu_setup(vcpu, init);

• vcpu_set_reg(vcpu, ARM64_CORE_REG(sp_el1), stack_vaddr + stack_size);

• if (is_vcpu_nested(vcpu))

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SP_EL2), stack_vaddr + stack_size);
  

• else

• vcpu_set_reg(vcpu, ARM64_CORE_REG(sp_el1), stack_vaddr + stack_size);
  

• return vcpu;

} @@ -457,7 +487,10 @@ void vcpu_init_descriptor_tables(struct kvm_vcpu *vcpu) { extern char vectors;

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_VBAR_EL1), (uint64_t)&vectors);

• if (is_vcpu_nested(vcpu))

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_VBAR_EL2), (uint64_t)&vectors);
  

• else

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_VBAR_EL1), (uint64_t)&vectors);
  

} void route_exception(struct ex_regs *regs, int vector) diff --git a/tools/testing/selftests/kvm/lib/arm64/vgic.c b/tools/testing/selftests/kvm/lib/arm64/vgic.c index 4427f43f73ea..67822b803d0f 100644 --- a/tools/testing/selftests/kvm/lib/arm64/vgic.c +++ b/tools/testing/selftests/kvm/lib/arm64/vgic.c @@ -79,6 +79,14 @@ int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, uint32_t nr_irqs) return gic_fd; } +void vgic_v3_close(int gic_fd) +{

• if (gic_fd < 0)

• return;
  

• close(gic_fd);

+}

/* should only work for level sensitive interrupts */ int _kvm_irq_set_level_info(int gic_fd, uint32_t intid, int level) {

Thanks

Eric