Add a new selftest called vfio_pci_device_irq_test that routes and delivers an MSI from a vfio-pci device into a guest.
Note that this test currently uses vfio_pci_irq_trigger(), in which VFIO emulates a device sending the MSI. In the future we want to replace this with a real MSI so we can test the hardware path (e.g. VT-d on Intel).
This test only supports x86_64 for now, but can be ported to other architectures in the future.
Signed-off-by: David Matlack dmatlack@google.com --- tools/testing/selftests/kvm/Makefile.kvm | 1 + .../testing/selftests/kvm/include/kvm_util.h | 4 + tools/testing/selftests/kvm/lib/kvm_util.c | 21 +++ .../selftests/kvm/vfio_pci_device_irq_test.c | 129 ++++++++++++++++++ 4 files changed, 155 insertions(+) create mode 100644 tools/testing/selftests/kvm/vfio_pci_device_irq_test.c
diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 85f651743325..fefb6408b2e9 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -139,6 +139,7 @@ TEST_GEN_PROGS_x86 += rseq_test TEST_GEN_PROGS_x86 += steal_time TEST_GEN_PROGS_x86 += system_counter_offset_test TEST_GEN_PROGS_x86 += pre_fault_memory_test +TEST_GEN_PROGS_x86 += vfio_pci_device_irq_test
# Compiled outputs used by test targets TEST_GEN_PROGS_EXTENDED_x86 += x86/nx_huge_pages_test diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 373912464fb4..860883ac53e1 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -866,6 +866,8 @@ void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...); void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level); int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level);
+void kvm_add_irqfd(struct kvm_vm *vm, u32 gsi, int fd); + #define KVM_MAX_IRQ_ROUTES 4096
struct kvm_irq_routing *kvm_gsi_routing_create(void); @@ -874,6 +876,8 @@ void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing, int _kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing); void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing);
+void kvm_route_gsi(struct kvm_vm *vm, struct kvm_irq_routing_entry *entry); + const char *exit_reason_str(unsigned int exit_reason);
vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min, diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 815bc45dd8dc..3ae752da5065 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -1871,6 +1871,16 @@ void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level) TEST_ASSERT(ret >= 0, KVM_IOCTL_ERROR(KVM_IRQ_LINE, ret)); }
+void kvm_add_irqfd(struct kvm_vm *vm, u32 gsi, int fd) +{ + struct kvm_irqfd arg = { + .gsi = gsi, + .fd = fd, + }; + + vm_ioctl(vm, KVM_IRQFD, &arg); +} + struct kvm_irq_routing *kvm_gsi_routing_create(void) { struct kvm_irq_routing *routing; @@ -1921,6 +1931,17 @@ void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing) TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_SET_GSI_ROUTING, ret)); }
+void kvm_route_gsi(struct kvm_vm *vm, struct kvm_irq_routing_entry *entry) +{ + u8 buf[sizeof(struct kvm_irq_routing) + sizeof(*entry)] = {}; + struct kvm_irq_routing *routes = (void *)&buf; + + routes->nr = 1; + routes->entries[0] = *entry; + + vm_ioctl(vm, KVM_SET_GSI_ROUTING, routes); +} + /* * VM Dump * diff --git a/tools/testing/selftests/kvm/vfio_pci_device_irq_test.c b/tools/testing/selftests/kvm/vfio_pci_device_irq_test.c new file mode 100644 index 000000000000..c792fc169028 --- /dev/null +++ b/tools/testing/selftests/kvm/vfio_pci_device_irq_test.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "kvm_util.h" +#include "test_util.h" +#include "apic.h" +#include "processor.h" + +#include <pthread.h> +#include <time.h> +#include <linux/vfio.h> +#include <vfio_util.h> + +static bool guest_ready_for_irq; +static bool guest_received_irq; + +#define TIMEOUT_NS (10ULL * 1000 * 1000 * 1000) + +static void guest_enable_interrupts(void) +{ + x2apic_enable(); + asm volatile ("sti"); +} + +void kvm_route_msi(struct kvm_vm *vm, u32 gsi, struct kvm_vcpu *vcpu, u8 vector) +{ + struct kvm_irq_routing_entry entry = { + .gsi = gsi, + .type = KVM_IRQ_ROUTING_MSI, + .u.msi.address_lo = 0xFEE00000 | (vcpu->id << 12), + .u.msi.data = vector, + }; + + kvm_route_gsi(vm, &entry); +} + +static void guest_irq_handler(struct ex_regs *regs) +{ + WRITE_ONCE(guest_received_irq, true); + GUEST_DONE(); +} + +static void guest_code(void) +{ + guest_enable_interrupts(); + WRITE_ONCE(guest_ready_for_irq, true); + + for (;;) + continue; +} + +void *vcpu_thread_main(void *arg) +{ + struct kvm_vcpu *vcpu = arg; + struct ucall uc; + + vcpu_run(vcpu); + TEST_ASSERT_EQ(UCALL_DONE, get_ucall(vcpu, &uc)); + + return NULL; +} + +static void help(const char *name) +{ + printf("Usage: %s [-i iommu_mode] segment:bus:device.function\n", name); + iommu_mode_help("-i"); + exit(1); +} + +int main(int argc, char **argv) +{ + /* Random non-reserved vector and GSI to use for the device IRQ */ + const u8 vector = 0xe0; + const u32 gsi = 32; + + struct timespec start, elapsed; + struct vfio_pci_device *device; + const char *iommu_mode = NULL; + struct kvm_vcpu *vcpu; + pthread_t vcpu_thread; + struct kvm_vm *vm; + int c; + + while ((c = getopt(argc, argv, "i:")) != -1) { + switch (c) { + case 'i': + iommu_mode = optarg; + break; + default: + help(argv[0]); + } + } + + if (optind >= argc) + help(argv[0]); + + vm = vm_create_with_one_vcpu(&vcpu, guest_code); + vm_install_exception_handler(vm, vector, guest_irq_handler); + + device = vfio_pci_device_init(argv[optind], iommu_mode); + TEST_REQUIRE(device->msix_info.count > 0); + + vfio_pci_msix_enable(device, 0, 1); + kvm_add_irqfd(vm, gsi, device->msi_eventfds[0]); + kvm_route_msi(vm, gsi, vcpu, vector); + + pthread_create(&vcpu_thread, NULL, vcpu_thread_main, vcpu); + + while (!READ_ONCE(guest_ready_for_irq)) + sync_global_from_guest(vm, guest_ready_for_irq); + + /* + * TODO: Get the device to send a physical MSI to exercise IRQ Bypass + * (e.g. VT-d on Intel), rather than manually synthesizing a + * notification from VFIO. + */ + vfio_pci_irq_trigger(device, VFIO_PCI_MSIX_IRQ_INDEX, 0); + + clock_gettime(CLOCK_MONOTONIC, &start); + + while (!READ_ONCE(guest_received_irq)) { + elapsed = timespec_elapsed(start); + TEST_ASSERT(timespec_to_ns(elapsed) < TIMEOUT_NS, "vCPU never received IRQ\n"); + sync_global_from_guest(vm, guest_received_irq); + } + + pthread_join(vcpu_thread, NULL); + vfio_pci_device_cleanup(device); + + return 0; +}