Hi Ganapatrao On Mon, May 12, 2025 at 03:52:43AM -0700, Ganapatrao Kulkarni wrote:

This patch adds required changes to vcpu init to run a guest code in vEL2 context and also adds NV specific helper functions.

Signed-off-by: Ganapatrao Kulkarni gankulkarni@os.amperecomputing.com

I'm interested in testing this, but is this series against kvmarm? which branch I can cleanly do git am?

Thanks, Itaru.

---

tools/testing/selftests/kvm/Makefile.kvm | 2 + .../kvm/include/arm64/kvm_util_arch.h | 3 + .../selftests/kvm/include/arm64/nv_util.h | 45 ++++++++++++++ .../selftests/kvm/include/arm64/vgic.h | 1 + .../testing/selftests/kvm/include/kvm_util.h | 3 + tools/testing/selftests/kvm/lib/arm64/nv.c | 46 ++++++++++++++ .../selftests/kvm/lib/arm64/processor.c | 61 ++++++++++++++----- tools/testing/selftests/kvm/lib/arm64/vgic.c | 8 +++ 8 files changed, 155 insertions(+), 14 deletions(-) create mode 100644 tools/testing/selftests/kvm/include/arm64/nv_util.h create mode 100644 tools/testing/selftests/kvm/lib/arm64/nv.c

diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index f773f8f99249..3348f729d3b2 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -37,6 +37,7 @@ LIBKVM_arm64 += lib/arm64/processor.c LIBKVM_arm64 += lib/arm64/spinlock.c LIBKVM_arm64 += lib/arm64/ucall.c LIBKVM_arm64 += lib/arm64/vgic.c +LIBKVM_arm64 += lib/arm64/nv.c LIBKVM_s390 += lib/s390/diag318_test_handler.c LIBKVM_s390 += lib/s390/processor.c @@ -155,6 +156,7 @@ TEST_GEN_PROGS_arm64 += arm64/vgic_irq TEST_GEN_PROGS_arm64 += arm64/vgic_lpi_stress TEST_GEN_PROGS_arm64 += arm64/vpmu_counter_access TEST_GEN_PROGS_arm64 += arm64/no-vgic-v3 +TEST_GEN_PROGS_arm64 += arm64/nv_guest_hypervisor TEST_GEN_PROGS_arm64 += access_tracking_perf_test TEST_GEN_PROGS_arm64 += arch_timer TEST_GEN_PROGS_arm64 += coalesced_io_test diff --git a/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h b/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h index e43a57d99b56..ab5279c24413 100644 --- a/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h +++ b/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h @@ -2,6 +2,9 @@ #ifndef SELFTEST_KVM_UTIL_ARCH_H #define SELFTEST_KVM_UTIL_ARCH_H +#define CurrentEL_EL1 (1 << 2) +#define CurrentEL_EL2 (2 << 2)

struct kvm_vm_arch {}; #endif // SELFTEST_KVM_UTIL_ARCH_H diff --git a/tools/testing/selftests/kvm/include/arm64/nv_util.h b/tools/testing/selftests/kvm/include/arm64/nv_util.h new file mode 100644 index 000000000000..622a17c9d142 --- /dev/null +++ b/tools/testing/selftests/kvm/include/arm64/nv_util.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/*

• • Copyright (c) 2025 Ampere Computing
• */

+#ifndef SELFTEST_NV_UTIL_H +#define SELFTEST_NV_UTIL_H

+#include <linux/bitmap.h> +#include <vgic.h>

+#define HCR_NV2 (UL(1) << 45) +#define HCR_AT (UL(1) << 44) +#define HCR_NV (UL(1) << 42) +#define HCR_E2H (UL(1) << 34) +#define HCR_TTLB (UL(1) << 25)

+/* Enable NV2 and guest in VHE mode */ +#define HCR_EL2_NV_EANBLE (HCR_E2H | HCR_NV | HCR_NV2 | HCR_AT | HCR_TTLB)

+struct kvm_vm *nv_vm_create_with_vcpus_gic(uint32_t nr_vcpus,

• struct kvm_vcpu **vcpus, int *gic_fd, void *guest_code);
  

+struct kvm_vm *__nv_vm_create_with_vcpus_gic(struct vm_shape shape,

• uint32_t nr_vcpus, struct kvm_vcpu **vcpus,
  

• uint64_t extra_mem_pages, int *gic_fd, void *guest_code);
  

+/* NV helpers */ +static inline void init_vcpu_nested(struct kvm_vcpu_init *init) +{

• init->features[0] |= (1 << KVM_ARM_VCPU_HAS_EL2);

+}

+static inline bool kvm_arm_vcpu_has_el2(struct kvm_vcpu_init *init) +{

• unsigned long features = init->features[0];
• return test_bit(KVM_ARM_VCPU_HAS_EL2, &features);

+}

+static inline bool is_vcpu_nested(struct kvm_vcpu *vcpu) +{

• return vcpu->nested;

+}

+#endif /* SELFTEST_NV_UTIL_H */ diff --git a/tools/testing/selftests/kvm/include/arm64/vgic.h b/tools/testing/selftests/kvm/include/arm64/vgic.h index c481d0c00a5d..46142fa36199 100644 --- a/tools/testing/selftests/kvm/include/arm64/vgic.h +++ b/tools/testing/selftests/kvm/include/arm64/vgic.h @@ -17,6 +17,7 @@ index) int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, uint32_t nr_irqs); +void vgic_v3_close(int gic_fd); #define VGIC_MAX_RESERVED 1023 diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 373912464fb4..9b26b9124dc3 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -65,6 +65,9 @@ struct kvm_vcpu { struct kvm_dirty_gfn *dirty_gfns; uint32_t fetch_index; uint32_t dirty_gfns_count; +#ifdef __aarch64__

• bool nested;

+#endif }; struct userspace_mem_regions { diff --git a/tools/testing/selftests/kvm/lib/arm64/nv.c b/tools/testing/selftests/kvm/lib/arm64/nv.c new file mode 100644 index 000000000000..e930808a7ed4 --- /dev/null +++ b/tools/testing/selftests/kvm/lib/arm64/nv.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0 +/*

• • Copyright (c) 2025 Ampere Computing LLC
• */

+#include <linux/compiler.h> +#include <assert.h>

+#include "guest_modes.h" +#include "kvm_util.h" +#include "nv_util.h" +#include "processor.h"

+struct kvm_vm *__nv_vm_create_with_vcpus_gic(struct vm_shape shape, uint32_t nr_vcpus,

• struct kvm_vcpu **vcpus, uint64_t extra_mem_pages, int *gic_fd, void *guest_code)
  

+{

• struct kvm_vcpu_init init;
• struct kvm_vm *vm;
• int i;
• TEST_REQUIRE(kvm_has_cap(KVM_CAP_ARM_EL2));
• vm = __vm_create(shape, nr_vcpus, extra_mem_pages);
• vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &init);
• init_vcpu_nested(&init);
• for (i = 0; i < nr_vcpus; ++i) {

• vcpus[i] = aarch64_vcpu_add(vm, i, &init, guest_code);
  

• __TEST_REQUIRE(is_vcpu_nested(vcpus[i]), "Failed to Enable NV");
  

• }
• /* vgic is not created, If gic_fd argument is NULL */
• if (gic_fd) {

• *gic_fd = vgic_v3_setup(vm, nr_vcpus, 64);
  

• __TEST_REQUIRE(*gic_fd >= 0, "Failed to create vgic-v3");
  

• }
• return vm;

+}

+struct kvm_vm *nv_vm_create_with_vcpus_gic(uint32_t nr_vcpus,

• struct kvm_vcpu **vcpus, int *gic_fd, void *guest_code)
  

+{

• return __nv_vm_create_with_vcpus_gic(VM_SHAPE_DEFAULT,

• 		nr_vcpus, vcpus, 0, gic_fd, guest_code);
  

+} diff --git a/tools/testing/selftests/kvm/lib/arm64/processor.c b/tools/testing/selftests/kvm/lib/arm64/processor.c index 7ba3aa3755f3..6e759981bf9e 100644 --- a/tools/testing/selftests/kvm/lib/arm64/processor.c +++ b/tools/testing/selftests/kvm/lib/arm64/processor.c @@ -10,6 +10,7 @@ #include "guest_modes.h" #include "kvm_util.h" +#include "nv_util.h" #include "processor.h" #include "ucall_common.h" @@ -258,14 +259,49 @@ void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) } } +static void aarch64_vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t sctlr_el1,

• 	uint64_t tcr_el1, uint64_t ttbr0_el1)
  

+{

• uint64_t fpen;
• /*

• * Enable FP/ASIMD to avoid trapping when accessing Q0-Q15
  

• * registers, which the variable argument list macros do.
  

• */
  

• fpen = 3 << 20;
• if (is_vcpu_nested(vcpu)) {

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPTR_EL2), fpen);
  

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL2), sctlr_el1);
  

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL2), tcr_el1);
  

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL2), DEFAULT_MAIR_EL1);
  

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL2), ttbr0_el1);
  

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_HCR_EL2), HCR_EL2_NV_EANBLE);
  

• } else {

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPACR_EL1), fpen);
  

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1), sctlr_el1);
  

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL1), tcr_el1);
  

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL1), DEFAULT_MAIR_EL1);
  

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL1), ttbr0_el1);
  

• }
• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL1), vcpu->id);

+}

void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init) { struct kvm_vcpu_init default_init = { .target = -1, }; struct kvm_vm *vm = vcpu->vm; uint64_t sctlr_el1, tcr_el1, ttbr0_el1;

• if (!init)

• if (!init) { init = &default_init;
• } else {

• /* Is this vcpu a Guest-Hypersior */
  

• if (kvm_arm_vcpu_has_el2(init))
  

• 	vcpu->nested = true;
  

• }

if (init->target == -1) { struct kvm_vcpu_init preferred; @@ -275,12 +311,6 @@ void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init) vcpu_ioctl(vcpu, KVM_ARM_VCPU_INIT, init);

• /*

• * Enable FP/ASIMD to avoid trapping when accessing Q0-Q15
  

• * registers, which the variable argument list macros do.
  

• */
  

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPACR_EL1), 3 << 20);
• sctlr_el1 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1)); tcr_el1 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL1));

@@ -349,11 +379,7 @@ void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init) if (use_lpa2_pte_format(vm)) tcr_el1 |= (1ul << 59) /* DS */;

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1), sctlr_el1);
• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL1), tcr_el1);
• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL1), DEFAULT_MAIR_EL1);
• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL1), ttbr0_el1);
• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL1), vcpu->id);

• aarch64_vcpu_set_reg(vcpu, sctlr_el1, tcr_el1, ttbr0_el1);

} void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, uint8_t indent) @@ -387,7 +413,11 @@ static struct kvm_vcpu *__aarch64_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id, aarch64_vcpu_setup(vcpu, init);

• vcpu_set_reg(vcpu, ARM64_CORE_REG(sp_el1), stack_vaddr + stack_size);

• if (is_vcpu_nested(vcpu))

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SP_EL2), stack_vaddr + stack_size);
  

• else

• vcpu_set_reg(vcpu, ARM64_CORE_REG(sp_el1), stack_vaddr + stack_size);
  

• return vcpu;

} @@ -457,7 +487,10 @@ void vcpu_init_descriptor_tables(struct kvm_vcpu *vcpu) { extern char vectors;

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_VBAR_EL1), (uint64_t)&vectors);

• if (is_vcpu_nested(vcpu))

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_VBAR_EL2), (uint64_t)&vectors);
  

• else

• vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_VBAR_EL1), (uint64_t)&vectors);
  

} void route_exception(struct ex_regs *regs, int vector) diff --git a/tools/testing/selftests/kvm/lib/arm64/vgic.c b/tools/testing/selftests/kvm/lib/arm64/vgic.c index 4427f43f73ea..67822b803d0f 100644 --- a/tools/testing/selftests/kvm/lib/arm64/vgic.c +++ b/tools/testing/selftests/kvm/lib/arm64/vgic.c @@ -79,6 +79,14 @@ int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, uint32_t nr_irqs) return gic_fd; } +void vgic_v3_close(int gic_fd) +{

• if (gic_fd < 0)

• return;
  

• close(gic_fd);

+}

/* should only work for level sensitive interrupts */ int _kvm_irq_set_level_info(int gic_fd, uint32_t intid, int level) {

-- 2.48.1

On Mon, 12 May 2025 11:52:43 +0100, Ganapatrao Kulkarni gankulkarni@os.amperecomputing.com wrote:

This patch adds required changes to vcpu init to run a guest code in vEL2 context and also adds NV specific helper functions.

Signed-off-by: Ganapatrao Kulkarni gankulkarni@os.amperecomputing.com

tools/testing/selftests/kvm/Makefile.kvm | 2 + .../kvm/include/arm64/kvm_util_arch.h | 3 + .../selftests/kvm/include/arm64/nv_util.h | 45 ++++++++++++++ .../selftests/kvm/include/arm64/vgic.h | 1 + .../testing/selftests/kvm/include/kvm_util.h | 3 + tools/testing/selftests/kvm/lib/arm64/nv.c | 46 ++++++++++++++ .../selftests/kvm/lib/arm64/processor.c | 61 ++++++++++++++----- tools/testing/selftests/kvm/lib/arm64/vgic.c | 8 +++ 8 files changed, 155 insertions(+), 14 deletions(-) create mode 100644 tools/testing/selftests/kvm/include/arm64/nv_util.h create mode 100644 tools/testing/selftests/kvm/lib/arm64/nv.c

diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index f773f8f99249..3348f729d3b2 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -37,6 +37,7 @@ LIBKVM_arm64 += lib/arm64/processor.c LIBKVM_arm64 += lib/arm64/spinlock.c LIBKVM_arm64 += lib/arm64/ucall.c LIBKVM_arm64 += lib/arm64/vgic.c +LIBKVM_arm64 += lib/arm64/nv.c LIBKVM_s390 += lib/s390/diag318_test_handler.c LIBKVM_s390 += lib/s390/processor.c @@ -155,6 +156,7 @@ TEST_GEN_PROGS_arm64 += arm64/vgic_irq TEST_GEN_PROGS_arm64 += arm64/vgic_lpi_stress TEST_GEN_PROGS_arm64 += arm64/vpmu_counter_access TEST_GEN_PROGS_arm64 += arm64/no-vgic-v3 +TEST_GEN_PROGS_arm64 += arm64/nv_guest_hypervisor TEST_GEN_PROGS_arm64 += access_tracking_perf_test TEST_GEN_PROGS_arm64 += arch_timer TEST_GEN_PROGS_arm64 += coalesced_io_test diff --git a/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h b/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h index e43a57d99b56..ab5279c24413 100644 --- a/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h +++ b/tools/testing/selftests/kvm/include/arm64/kvm_util_arch.h @@ -2,6 +2,9 @@ #ifndef SELFTEST_KVM_UTIL_ARCH_H #define SELFTEST_KVM_UTIL_ARCH_H +#define CurrentEL_EL1 (1 << 2) +#define CurrentEL_EL2 (2 << 2)

struct kvm_vm_arch {}; #endif // SELFTEST_KVM_UTIL_ARCH_H diff --git a/tools/testing/selftests/kvm/include/arm64/nv_util.h b/tools/testing/selftests/kvm/include/arm64/nv_util.h new file mode 100644 index 000000000000..622a17c9d142 --- /dev/null +++ b/tools/testing/selftests/kvm/include/arm64/nv_util.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/*

• • Copyright (c) 2025 Ampere Computing
• */

+#ifndef SELFTEST_NV_UTIL_H +#define SELFTEST_NV_UTIL_H

+#include <linux/bitmap.h> +#include <vgic.h>

+#define HCR_NV2 (UL(1) << 45) +#define HCR_AT (UL(1) << 44) +#define HCR_NV (UL(1) << 42) +#define HCR_E2H (UL(1) << 34) +#define HCR_TTLB (UL(1) << 25)

+/* Enable NV2 and guest in VHE mode */ +#define HCR_EL2_NV_EANBLE (HCR_E2H | HCR_NV | HCR_NV2 | HCR_AT | HCR_TTLB)

What is the point of enabling NV and trapping all sort of things, given that you're not even enabling a guest? How do you expect to deal with EL0 without TGE being set?

M.

On Mon, May 12, 2025 at 03:52:44AM -0700, Ganapatrao Kulkarni wrote:

Add simple test to run guest code with NV enabled. With NV enabled, guest code runs in vEL2 context.

Signed-off-by: Ganapatrao Kulkarni gankulkarni@os.amperecomputing.com

.../selftests/kvm/arm64/nv_guest_hypervisor.c | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c

diff --git a/tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c b/tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c new file mode 100644 index 000000000000..7d7b3944e229 --- /dev/null +++ b/tools/testing/selftests/kvm/arm64/nv_guest_hypervisor.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0-only +/*

• • Copyright (c) 2025 Ampere Computing LLC
• */

+#include <kvm_util.h> +#include <nv_util.h> +#include <processor.h>

+static void guest_code(void) +{

• if (read_sysreg(CurrentEL) == CurrentEL_EL2)

• GUEST_PRINTF("Test PASS\n");
  

• else

• GUEST_FAIL("Fail to run in vEL2\n");
  

• GUEST_DONE();

+}

+static void guest_undef_handler(struct ex_regs *regs) +{

• GUEST_FAIL("Unexpected exception far_el1 = 0x%lx", read_sysreg(far_el1));

+}

+static void test_run_vcpu(struct kvm_vcpu *vcpu) +{

• struct ucall uc;
• do {

• vcpu_run(vcpu);
  

• switch (get_ucall(vcpu, &uc)) {
  

• case UCALL_ABORT:
  

• 	REPORT_GUEST_ASSERT(uc);
  

• 	break;
  

• case UCALL_PRINTF:
  

• 	printf("%s", uc.buffer);
  

• 	break;
  

• case UCALL_DONE:
  

• 	break;
  

• default:
  

• 	TEST_FAIL("Unknown ucall %lu", uc.cmd);
  

• }
  

• } while (uc.cmd != UCALL_DONE);

+}

+static void test_nv_guest_hypervisor(void) +{

• struct kvm_vcpu *vcpu;
• struct kvm_vm *vm;
• int gic_fd = -1;
• vm = nv_vm_create_with_vcpus_gic(1, &vcpu, &gic_fd, guest_code);
• vm_init_descriptor_tables(vm);
• vcpu_init_descriptor_tables(vcpu);
• vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT,

• 		ESR_ELx_EC_UNKNOWN, guest_undef_handler);
  

• test_run_vcpu(vcpu);
• vgic_v3_close(gic_fd);
• kvm_vm_free(vm);

+}

+int main(int argc, char *argv[]) +{

• test_nv_guest_hypervisor();
• return 0;

+}

The kernel this v2 series applied is based off of the latest kvmarm-next and tested this on QEMU TCG mode with the kvm-arm.mode=nested option added at boot.

Tested-by: Itaru Kitayama itaru.kitayama@fujitsu.com

-- 2.48.1

Hi Ganapatrao,

On 5/12/25 12:52 PM, Ganapatrao Kulkarni wrote:

Adding required changes to enable and test HVTIMER and HPTIMER in vEL2. In default case, PTIMER and VTIMER are validated and with NV enabled (with argument "-g 1"), HPTIMER and HVTIMER are validated by injecting respective timer interrupts.

Signed-off-by: Ganapatrao Kulkarni gankulkarni@os.amperecomputing.com

tools/testing/selftests/kvm/arch_timer.c | 8 +- .../testing/selftests/kvm/arm64/arch_timer.c | 118 +++++++++++++++--- .../selftests/kvm/include/arm64/arch_timer.h | 16 +++ .../selftests/kvm/include/timer_test.h | 1 + 4 files changed, 123 insertions(+), 20 deletions(-)

diff --git a/tools/testing/selftests/kvm/arch_timer.c b/tools/testing/selftests/kvm/arch_timer.c index acb2cb596332..5c30bda0462b 100644 --- a/tools/testing/selftests/kvm/arch_timer.c +++ b/tools/testing/selftests/kvm/arch_timer.c @@ -35,6 +35,7 @@ struct test_args test_args = { .migration_freq_ms = TIMER_TEST_MIGRATION_FREQ_MS, .timer_err_margin_us = TIMER_TEST_ERR_MARGIN_US, .reserved = 1,

• .is_nested = false,

}; struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; @@ -43,6 +44,7 @@ struct test_vcpu_shared_data vcpu_shared_data[KVM_MAX_VCPUS]; static pthread_t pt_vcpu_run[KVM_MAX_VCPUS]; static unsigned long *vcpu_done_map; static pthread_mutex_t vcpu_done_map_lock; +bool is_nested; static void *test_vcpu_run(void *arg) { @@ -193,6 +195,7 @@ static void test_print_help(char *name) pr_info("\t-o: Counter offset (in counter cycles, default: 0) [aarch64-only]\n"); pr_info("\t-e: Interrupt arrival error margin (in us) of the guest timer (default: %u)\n", TIMER_TEST_ERR_MARGIN_US);

• pr_info("\t-g: Enable Nested Virtualization, run guest code as guest hypervisor (default: Disabled)\n");

I would recommend "default: 0" in all enhanced tests to clarify that 0/1 are the values to be used.

Thanks

Eric

pr_info("\t-h: print this help screen\n"); } @@ -200,7 +203,7 @@ static bool parse_args(int argc, char *argv[]) { int opt;

• while ((opt = getopt(argc, argv, "hn:i:p:m:o:e:")) != -1) {

• while ((opt = getopt(argc, argv, "hn:i:p:m:o:e:g:")) != -1) { switch (opt) { case 'n': test_args.nr_vcpus = atoi_positive("Number of vCPUs", optarg);

@@ -226,6 +229,9 @@ static bool parse_args(int argc, char *argv[]) test_args.counter_offset = strtol(optarg, NULL, 0); test_args.reserved = 0; break;

• case 'g':
  

• 	test_args.is_nested = atoi_non_negative("Is Nested", optarg);
  

• 	break;
  

  case 'h': default: goto err;

diff --git a/tools/testing/selftests/kvm/arm64/arch_timer.c b/tools/testing/selftests/kvm/arm64/arch_timer.c index eeba1cc87ff8..50bf05bb6f85 100644 --- a/tools/testing/selftests/kvm/arm64/arch_timer.c +++ b/tools/testing/selftests/kvm/arm64/arch_timer.c @@ -12,16 +12,22 @@ #include "timer_test.h" #include "ucall_common.h" #include "vgic.h" +#include <nv_util.h> enum guest_stage { GUEST_STAGE_VTIMER_CVAL = 1, GUEST_STAGE_VTIMER_TVAL, GUEST_STAGE_PTIMER_CVAL, GUEST_STAGE_PTIMER_TVAL,

• GUEST_STAGE_HVTIMER_CVAL,
• GUEST_STAGE_HVTIMER_TVAL,
• GUEST_STAGE_HPTIMER_CVAL,
• GUEST_STAGE_HPTIMER_TVAL, GUEST_STAGE_MAX,

}; static int vtimer_irq, ptimer_irq; +static int hvtimer_irq, hptimer_irq; static void guest_configure_timer_action(struct test_vcpu_shared_data *shared_data) @@ -47,6 +53,26 @@ guest_configure_timer_action(struct test_vcpu_shared_data *shared_data) shared_data->xcnt = timer_get_cntct(PHYSICAL); timer_set_ctl(PHYSICAL, CTL_ENABLE); break;

• case GUEST_STAGE_HVTIMER_CVAL:

• timer_set_next_cval_ms(HVIRTUAL, test_args.timer_period_ms);
  

• shared_data->xcnt = timer_get_cntct(HVIRTUAL);
  

• timer_set_ctl(HVIRTUAL, CTL_ENABLE);
  

• break;
  

• case GUEST_STAGE_HVTIMER_TVAL:

• timer_set_next_tval_ms(HVIRTUAL, test_args.timer_period_ms);
  

• shared_data->xcnt = timer_get_cntct(HVIRTUAL);
  

• timer_set_ctl(HVIRTUAL, CTL_ENABLE);
  

• break;
  

• case GUEST_STAGE_HPTIMER_CVAL:

• timer_set_next_cval_ms(HPHYSICAL, test_args.timer_period_ms);
  

• shared_data->xcnt = timer_get_cntct(HPHYSICAL);
  

• timer_set_ctl(HPHYSICAL, CTL_ENABLE);
  

• break;
  

• case GUEST_STAGE_HPTIMER_TVAL:

• timer_set_next_tval_ms(HPHYSICAL, test_args.timer_period_ms);
  

• shared_data->xcnt = timer_get_cntct(HPHYSICAL);
  

• timer_set_ctl(HPHYSICAL, CTL_ENABLE);
  

• break;
  

  default: GUEST_ASSERT(0); }

@@ -75,6 +101,16 @@ static void guest_validate_irq(unsigned int intid, accessor = PHYSICAL; timer_irq = ptimer_irq; break;

• case GUEST_STAGE_HVTIMER_CVAL:
• case GUEST_STAGE_HVTIMER_TVAL:

• accessor = HVIRTUAL;
  

• timer_irq = hvtimer_irq;
  

• break;
  

• case GUEST_STAGE_HPTIMER_CVAL:
• case GUEST_STAGE_HPTIMER_TVAL:

• accessor = HPHYSICAL;
  

• timer_irq = hptimer_irq;
  

• break;
  

  default: GUEST_ASSERT(0); return;

@@ -142,38 +178,79 @@ static void guest_code(void) { uint32_t cpu = guest_get_vcpuid(); struct test_vcpu_shared_data *shared_data = &vcpu_shared_data[cpu];

• bool is_nested = false;
• enum arch_timer vtimer, ptimer;
• int vtmr_irq, ptmr_irq;
• enum guest_stage stage_vtimer_cval, stage_vtimer_tval;
• enum guest_stage stage_ptimer_cval, stage_ptimer_tval;

• local_irq_disable();

• if (read_sysreg(CurrentEL) == CurrentEL_EL2)

• is_nested = true;
  

• local_irq_disable(); gic_init(GIC_V3, test_args.nr_vcpus);

• timer_set_ctl(VIRTUAL, CTL_IMASK);
• timer_set_ctl(PHYSICAL, CTL_IMASK);

• if (is_nested) {

• vtimer = HVIRTUAL;
  

• ptimer = HPHYSICAL;
  

• vtmr_irq = hvtimer_irq;
  

• ptmr_irq = hptimer_irq;
  

• stage_vtimer_cval = GUEST_STAGE_HVTIMER_CVAL;
  

• stage_vtimer_tval = GUEST_STAGE_HVTIMER_TVAL;
  

• stage_ptimer_cval = GUEST_STAGE_HPTIMER_CVAL;
  

• stage_ptimer_tval = GUEST_STAGE_HPTIMER_TVAL;
  

• } else {

• vtimer = VIRTUAL;
  

• ptimer = PHYSICAL;
  

• vtmr_irq = vtimer_irq;
  

• ptmr_irq = ptimer_irq;
  

• stage_vtimer_cval = GUEST_STAGE_VTIMER_CVAL;
  

• stage_vtimer_tval = GUEST_STAGE_VTIMER_TVAL;
  

• stage_ptimer_cval = GUEST_STAGE_PTIMER_CVAL;
  

• stage_ptimer_tval = GUEST_STAGE_PTIMER_TVAL;
  

• }
• timer_set_ctl(vtimer, CTL_IMASK);
• timer_set_ctl(ptimer, CTL_IMASK);
• gic_irq_enable(vtmr_irq);
• gic_irq_enable(ptmr_irq);

• gic_irq_enable(vtimer_irq);
• gic_irq_enable(ptimer_irq); local_irq_enable();

• guest_run_stage(shared_data, GUEST_STAGE_VTIMER_CVAL);
• guest_run_stage(shared_data, GUEST_STAGE_VTIMER_TVAL);
• guest_run_stage(shared_data, GUEST_STAGE_PTIMER_CVAL);
• guest_run_stage(shared_data, GUEST_STAGE_PTIMER_TVAL);

• guest_run_stage(shared_data, stage_vtimer_cval);
• guest_run_stage(shared_data, stage_vtimer_tval);
• guest_run_stage(shared_data, stage_ptimer_cval);
• guest_run_stage(shared_data, stage_ptimer_tval);

GUEST_DONE(); } static void test_init_timer_irq(struct kvm_vm *vm) {

• /* Timer initid should be same for all the vCPUs, so query only vCPU-0 */
• vcpu_device_attr_get(vcpus[0], KVM_ARM_VCPU_TIMER_CTRL,

• 	     KVM_ARM_VCPU_TIMER_IRQ_PTIMER, &ptimer_irq);
  

• vcpu_device_attr_get(vcpus[0], KVM_ARM_VCPU_TIMER_CTRL,

• 	     KVM_ARM_VCPU_TIMER_IRQ_VTIMER, &vtimer_irq);
  

• sync_global_to_guest(vm, ptimer_irq);
• sync_global_to_guest(vm, vtimer_irq);
• pr_debug("ptimer_irq: %d; vtimer_irq: %d\n", ptimer_irq, vtimer_irq);

• /* Timer initid should be same for all the vCPUs, so query only vCPU-0 */
• if (is_vcpu_nested(vcpus[0])) {

• vcpu_device_attr_get(vcpus[0], KVM_ARM_VCPU_TIMER_CTRL,
  

• 		KVM_ARM_VCPU_TIMER_IRQ_HPTIMER, &hptimer_irq);
  

• vcpu_device_attr_get(vcpus[0], KVM_ARM_VCPU_TIMER_CTRL,
  

• 		KVM_ARM_VCPU_TIMER_IRQ_HVTIMER, &hvtimer_irq);
  

• sync_global_to_guest(vm, hptimer_irq);
  

• sync_global_to_guest(vm, hvtimer_irq);
  

• pr_debug("hptimer_irq: %d; hvtimer_irq: %d\n", hptimer_irq, hvtimer_irq);
  

• } else {

• vcpu_device_attr_get(vcpus[0], KVM_ARM_VCPU_TIMER_CTRL,
  

• 		KVM_ARM_VCPU_TIMER_IRQ_PTIMER, &ptimer_irq);
  

• vcpu_device_attr_get(vcpus[0], KVM_ARM_VCPU_TIMER_CTRL,
  

• 		KVM_ARM_VCPU_TIMER_IRQ_VTIMER, &vtimer_irq);
  

• sync_global_to_guest(vm, ptimer_irq);
  

• sync_global_to_guest(vm, vtimer_irq);
  

• pr_debug("ptimer_irq: %d; vtimer_irq: %d\n", ptimer_irq, vtimer_irq);
  

• }

} static int gic_fd; @@ -184,7 +261,10 @@ struct kvm_vm *test_vm_create(void) unsigned int i; int nr_vcpus = test_args.nr_vcpus;

• vm = vm_create_with_vcpus(nr_vcpus, guest_code, vcpus);

• if (test_args.is_nested)

• vm = nv_vm_create_with_vcpus_gic(nr_vcpus, vcpus, NULL, guest_code);
  

• else

• vm = vm_create_with_vcpus(nr_vcpus, guest_code, vcpus);
  

vm_init_descriptor_tables(vm); vm_install_exception_handler(vm, VECTOR_IRQ_CURRENT, guest_irq_handler); diff --git a/tools/testing/selftests/kvm/include/arm64/arch_timer.h b/tools/testing/selftests/kvm/include/arm64/arch_timer.h index bf461de34785..82ba908fba8b 100644 --- a/tools/testing/selftests/kvm/include/arm64/arch_timer.h +++ b/tools/testing/selftests/kvm/include/arm64/arch_timer.h @@ -11,6 +11,8 @@ enum arch_timer { VIRTUAL, PHYSICAL,

• HVIRTUAL,
• HPHYSICAL,

}; #define CTL_ENABLE (1 << 0) @@ -37,8 +39,10 @@ static inline uint64_t timer_get_cntct(enum arch_timer timer) switch (timer) { case VIRTUAL:

• case HVIRTUAL: return read_sysreg(cntvct_el0); case PHYSICAL:
• case HPHYSICAL: return read_sysreg(cntpct_el0); default: GUEST_FAIL("Unexpected timer type = %u", timer);

@@ -52,9 +56,11 @@ static inline void timer_set_cval(enum arch_timer timer, uint64_t cval) { switch (timer) { case VIRTUAL:

• case HVIRTUAL: write_sysreg(cval, cntv_cval_el0); break; case PHYSICAL:
• case HPHYSICAL: write_sysreg(cval, cntp_cval_el0); break; default:

@@ -68,8 +74,10 @@ static inline uint64_t timer_get_cval(enum arch_timer timer) { switch (timer) { case VIRTUAL:

• case HVIRTUAL: return read_sysreg(cntv_cval_el0); case PHYSICAL:
• case HPHYSICAL: return read_sysreg(cntp_cval_el0); default: GUEST_FAIL("Unexpected timer type = %u", timer);

@@ -83,9 +91,11 @@ static inline void timer_set_tval(enum arch_timer timer, int32_t tval) { switch (timer) { case VIRTUAL:

• case HVIRTUAL: write_sysreg(tval, cntv_tval_el0); break; case PHYSICAL:
• case HPHYSICAL: write_sysreg(tval, cntp_tval_el0); break; default:

@@ -100,8 +110,10 @@ static inline int32_t timer_get_tval(enum arch_timer timer) isb(); switch (timer) { case VIRTUAL:

• case HVIRTUAL: return read_sysreg(cntv_tval_el0); case PHYSICAL:
• case HPHYSICAL: return read_sysreg(cntp_tval_el0); default: GUEST_FAIL("Could not get timer %d\n", timer);

@@ -115,9 +127,11 @@ static inline void timer_set_ctl(enum arch_timer timer, uint32_t ctl) { switch (timer) { case VIRTUAL:

• case HVIRTUAL: write_sysreg(ctl, cntv_ctl_el0); break; case PHYSICAL:
• case HPHYSICAL: write_sysreg(ctl, cntp_ctl_el0); break; default:

@@ -131,8 +145,10 @@ static inline uint32_t timer_get_ctl(enum arch_timer timer) { switch (timer) { case VIRTUAL:

• case HVIRTUAL: return read_sysreg(cntv_ctl_el0); case PHYSICAL:
• case HPHYSICAL: return read_sysreg(cntp_ctl_el0); default: GUEST_FAIL("Unexpected timer type = %u", timer);

diff --git a/tools/testing/selftests/kvm/include/timer_test.h b/tools/testing/selftests/kvm/include/timer_test.h index 9b6edaafe6d4..95f61c4a8a80 100644 --- a/tools/testing/selftests/kvm/include/timer_test.h +++ b/tools/testing/selftests/kvm/include/timer_test.h @@ -26,6 +26,7 @@ struct test_args { /* Members of struct kvm_arm_counter_offset */ uint64_t counter_offset; uint64_t reserved;

• bool is_nested;

}; /* Shared variables between host and guest */

On Mon, May 12, 2025 at 03:52:50AM -0700, Ganapatrao Kulkarni wrote:

Adding code to run guest_code in vEL2. NV is enabled using command line argument and it is disabled by default.

NV is only enabled on ARM64, for other architectures the test will exit with an ASSERT, if tried to run with NV enabled.

I'm seeing this in QEMU TCG mode, does this mean the limitation of the emulation?

$ sudo /mnt/projects/linux/tools/testing/selftests/kvm/arm64/page_fault_test -m 3 -s anonymous Random seed: 0x6b8b4567 ==== Test Assertion Failure ==== arm64/page_fault_test.c:632: test->expected_events.uffd_faults == events.uffd_faults pid=769 tid=769 errno=9 - Bad file descriptor 1 0x000000000040325b: check_event_counts at page_fault_test.c:632 2 (inlined by) run_test at page_fault_test.c:739 3 0x0000000000403cbf: for_each_guest_mode at guest_modes.c:96 4 0x0000000000401cfb: for_each_test_and_guest_mode at page_fault_test.c:1107 5 (inlined by) main at page_fault_test.c:1133 6 0x0000ffff848122db: ?? ??:0 7 0x0000ffff848123bb: ?? ??:0 8 0x0000000000401def: _start at ??:? 0x2 != 0x1 (test->expected_events.uffd_faults != events.uffd_faults)

Thanks, Itaru.

Signed-off-by: Ganapatrao Kulkarni gankulkarni@os.amperecomputing.com

.../selftests/kvm/kvm_page_table_test.c | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-)

diff --git a/tools/testing/selftests/kvm/kvm_page_table_test.c b/tools/testing/selftests/kvm/kvm_page_table_test.c index dd8b12f626d3..383f9d134ecb 100644 --- a/tools/testing/selftests/kvm/kvm_page_table_test.c +++ b/tools/testing/selftests/kvm/kvm_page_table_test.c @@ -20,6 +20,10 @@ #include "guest_modes.h" #include "ucall_common.h" +#ifdef __aarch64__ +#include <nv_util.h> +#endif

#define TEST_MEM_SLOT_INDEX 1 /* Default size(1GB) of the memory for testing */ @@ -229,7 +233,9 @@ static void *vcpu_worker(void *data) struct test_params { uint64_t phys_offset; uint64_t test_mem_size;

• bool is_nested; enum vm_mem_backing_src_type src_type;
• int fd;

}; static struct kvm_vm *pre_init_before_test(enum vm_guest_mode mode, void *arg) @@ -252,8 +258,17 @@ static struct kvm_vm *pre_init_before_test(enum vm_guest_mode mode, void *arg) /* Create a VM with enough guest pages */ guest_num_pages = test_mem_size / guest_page_size;

• vm = __vm_create_with_vcpus(VM_SHAPE(mode), nr_vcpus, guest_num_pages,

• if (p->is_nested) {

+#ifdef __aarch64__

• vm = __nv_vm_create_with_vcpus_gic(VM_SHAPE(mode), nr_vcpus,
  

• 		test_args.vcpus, guest_num_pages, &p->fd, guest_code);
  

+#else

• TEST_FAIL("Nested Not Supported");
  

+#endif

• } else {

• vm = __vm_create_with_vcpus(VM_SHAPE(mode), nr_vcpus, guest_num_pages,
  	    guest_code, test_args.vcpus);
  

• }

/* Align down GPA of the testing memslot */ if (!p->phys_offset) @@ -345,6 +360,7 @@ static void run_test(enum vm_guest_mode mode, void *arg) struct timespec start; struct timespec ts_diff; int ret, i;

• struct test_params *p = (struct test_params *)arg;

/* Create VM with vCPUs and make some pre-initialization */ vm = pre_init_before_test(mode, arg); @@ -414,6 +430,8 @@ static void run_test(enum vm_guest_mode mode, void *arg) TEST_ASSERT(ret == 0, "Error in sem_destroy"); free(vcpu_threads);

• if (p->is_nested)

• close(p->fd);
  

  kvm_vm_free(vm);

} @@ -421,7 +439,7 @@ static void help(char *name) { puts(""); printf("usage: %s [-h] [-p offset] [-m mode] "

•       "[-b mem-size] [-v vcpus] [-s mem-type]\n", name);
  

•       "[-b mem-size] [-v vcpus] [-s mem-type] [-g nv]\n", name);
  

  puts(""); printf(" -p: specify guest physical test memory offset\n" " Warning: a low offset can conflict with the loaded test code.\n");

@@ -430,6 +448,8 @@ static void help(char *name) " (default: 1G)\n"); printf(" -v: specify the number of vCPUs to run\n" " (default: 1)\n");

• printf(" -g: Enable Nested Virtualization, run guest code as guest hypervisor.\n"

•       "     (default: Disabled)\n");
  

  backing_src_help("-s"); puts("");

} @@ -440,12 +460,13 @@ int main(int argc, char *argv[]) struct test_params p = { .test_mem_size = DEFAULT_TEST_MEM_SIZE, .src_type = DEFAULT_VM_MEM_SRC,

• .is_nested = false,
  

  }; int opt;

guest_modes_append_default();

• while ((opt = getopt(argc, argv, "hp:m:b:v:s:")) != -1) {

• while ((opt = getopt(argc, argv, "hp:m:b:v:s:g:")) != -1) { switch (opt) { case 'p': p.phys_offset = strtoull(optarg, NULL, 0);

@@ -464,6 +485,9 @@ int main(int argc, char *argv[]) case 's': p.src_type = parse_backing_src_type(optarg); break;

• case 'g':
  

• 	p.is_nested = atoi_non_negative("Is Nested", optarg);
  

• 	break;
  

  case 'h': default: help(argv[0]);

-- 2.48.1