The demand paging test is currently a simple page access test which, while potentially useful, doesn't add much versus the existing dirty logging test. To improve the demand paging test, add a basic userfaultfd demand paging implementation.

Signed-off-by: Ben Gardon bgardon@google.com --- .../selftests/kvm/demand_paging_test.c | 196 +++++++++++++++++- 1 file changed, 192 insertions(+), 4 deletions(-)

diff --git a/tools/testing/selftests/kvm/demand_paging_test.c b/tools/testing/selftests/kvm/demand_paging_test.c index 5f214517ba1de..6fab4468f97f6 100644 --- a/tools/testing/selftests/kvm/demand_paging_test.c +++ b/tools/testing/selftests/kvm/demand_paging_test.c @@ -11,11 +11,14 @@

#include <stdio.h> #include <stdlib.h> +#include <sys/syscall.h> #include <unistd.h> #include <time.h> +#include <poll.h> #include <pthread.h> #include <linux/bitmap.h> #include <linux/bitops.h> +#include <linux/userfaultfd.h>

#include "test_util.h" #include "kvm_util.h" @@ -39,6 +42,8 @@ static uint64_t host_page_size; static uint64_t guest_page_size; static uint64_t guest_num_pages;

+static char *guest_data_prototype; + /* * Guest physical memory offset of the testing memory slot. * This will be set to the topmost valid physical address minus @@ -110,13 +115,169 @@ static struct kvm_vm *create_vm(enum vm_guest_mode mode, uint32_t vcpuid, return vm; }

+static int handle_uffd_page_request(int uffd, uint64_t addr) +{ + pid_t tid; + struct uffdio_copy copy; + int r; + + tid = syscall(__NR_gettid); + + copy.src = (uint64_t)guest_data_prototype; + copy.dst = addr; + copy.len = host_page_size; + copy.mode = 0; + + r = ioctl(uffd, UFFDIO_COPY, &copy); + if (r == -1) { + DEBUG("Failed Paged in 0x%lx from thread %d with errno: %d\n", + addr, tid, errno); + return r; + } + + return 0; +} + +bool quit_uffd_thread; + +struct uffd_handler_args { + int uffd; + int pipefd; +}; + +static void *uffd_handler_thread_fn(void *arg) +{ + struct uffd_handler_args *uffd_args = (struct uffd_handler_args *)arg; + int uffd = uffd_args->uffd; + int pipefd = uffd_args->pipefd; + int64_t pages = 0; + + while (!quit_uffd_thread) { + struct uffd_msg msg; + struct pollfd pollfd[2]; + char tmp_chr; + int r; + uint64_t addr; + + pollfd[0].fd = uffd; + pollfd[0].events = POLLIN; + pollfd[1].fd = pipefd; + pollfd[1].events = POLLIN; + + r = poll(pollfd, 2, -1); + switch (r) { + case -1: + DEBUG("poll err"); + continue; + case 0: + continue; + case 1: + break; + default: + DEBUG("Polling uffd returned %d", r); + return NULL; + } + + if (pollfd[0].revents & POLLERR) { + DEBUG("uffd revents has POLLERR"); + return NULL; + } + + if (pollfd[1].revents & POLLIN) { + r = read(pollfd[1].fd, &tmp_chr, 1); + TEST_ASSERT(r == 1, + "Error reading pipefd in UFFD thread\n"); + return NULL; + } + + if (!pollfd[0].revents & POLLIN) + continue; + + r = read(uffd, &msg, sizeof(msg)); + if (r == -1) { + if (errno == EAGAIN) + continue; + DEBUG("Read of uffd gor errno %d", errno); + return NULL; + } + + if (r != sizeof(msg)) { + DEBUG("Read on uffd returned unexpected size: %d bytes", + r); + return NULL; + } + + if (!(msg.event & UFFD_EVENT_PAGEFAULT)) + continue; + + addr = msg.arg.pagefault.address; + r = handle_uffd_page_request(uffd, addr); + if (r < 0) + return NULL; + pages++; + } + + return NULL; +} + +static int setup_demand_paging(struct kvm_vm *vm, + pthread_t *uffd_handler_thread, int pipefd) +{ + int uffd; + struct uffdio_api uffdio_api; + struct uffdio_register uffdio_register; + struct uffd_handler_args uffd_args; + + guest_data_prototype = malloc(host_page_size); + TEST_ASSERT(guest_data_prototype, + "Failed to allocate buffer for guest data pattern"); + memset(guest_data_prototype, 0xAB, host_page_size); + + uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); + if (uffd == -1) { + DEBUG("uffd creation failed\n"); + return -1; + } + + uffdio_api.api = UFFD_API; + uffdio_api.features = 0; + if (ioctl(uffd, UFFDIO_API, &uffdio_api) == -1) { + DEBUG("ioctl uffdio_api failed\n"); + return -1; + } + + uffdio_register.range.start = (uint64_t)host_test_mem; + uffdio_register.range.len = host_num_pages * host_page_size; + uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING; + if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1) { + DEBUG("ioctl uffdio_register failed\n"); + return -1; + } + + if ((uffdio_register.ioctls & UFFD_API_RANGE_IOCTLS) != + UFFD_API_RANGE_IOCTLS) { + DEBUG("unexpected userfaultfd ioctl set\n"); + return -1; + } + + uffd_args.uffd = uffd; + uffd_args.pipefd = pipefd; + pthread_create(uffd_handler_thread, NULL, uffd_handler_thread_fn, + &uffd_args); + + return 0; +} + #define GUEST_MEM_SHIFT 30 /* 1G */ #define PAGE_SHIFT_4K 12

-static void run_test(enum vm_guest_mode mode) +static void run_test(enum vm_guest_mode mode, bool use_uffd) { pthread_t vcpu_thread; + pthread_t uffd_handler_thread; + int pipefd[2]; struct kvm_vm *vm; + int r;

/* * We reserve page table for 2 times of extra dirty mem which @@ -173,6 +334,16 @@ static void run_test(enum vm_guest_mode mode) /* Cache the HVA pointer of the region */ host_test_mem = addr_gpa2hva(vm, (vm_paddr_t)guest_test_phys_mem);

+ if (use_uffd) { + /* Set up user fault fd to handle demand paging requests. */ + r = pipe2(pipefd, O_CLOEXEC | O_NONBLOCK); + TEST_ASSERT(!r, "Failed to set up pipefd"); + + r = setup_demand_paging(vm, &uffd_handler_thread, pipefd[0]); + if (r < 0) + exit(-r); + } + #ifdef __x86_64__ vcpu_set_cpuid(vm, VCPU_ID, kvm_get_supported_cpuid()); #endif @@ -191,8 +362,20 @@ static void run_test(enum vm_guest_mode mode) /* Wait for the vcpu thread to quit */ pthread_join(vcpu_thread, NULL);

+ if (use_uffd) { + char c; + + /* Tell the user fault fd handler thread to quit */ + r = write(pipefd[1], &c, 1); + TEST_ASSERT(r == 1, "Unable to write to pipefd"); + + pthread_join(uffd_handler_thread, NULL); + } + ucall_uninit(vm); kvm_vm_free(vm); + + free(guest_data_prototype); }

struct vm_guest_mode_params { @@ -212,7 +395,7 @@ static void help(char *name) int i;

puts(""); - printf("usage: %s [-h] [-m mode]\n", name); + printf("usage: %s [-h] [-m mode] [-u]\n", name); printf(" -m: specify the guest mode ID to test\n" " (default: test all supported modes)\n" " This option may be used multiple times.\n" @@ -221,6 +404,7 @@ static void help(char *name) printf(" %d: %s%s\n", i, vm_guest_mode_string(i), vm_guest_mode_params[i].supported ? " (supported)" : ""); } + printf(" -u: Use User Fault FD to handle vCPU page faults.\n"); puts(""); exit(0); } @@ -230,6 +414,7 @@ int main(int argc, char *argv[]) bool mode_selected = false; unsigned int mode; int opt, i; + bool use_uffd = false; #ifdef __aarch64__ unsigned int host_ipa_limit; #endif @@ -253,7 +438,7 @@ int main(int argc, char *argv[]) vm_guest_mode_params_init(VM_MODE_P40V48_4K, true, true); #endif

- while ((opt = getopt(argc, argv, "hm:")) != -1) { + while ((opt = getopt(argc, argv, "hm:u")) != -1) { switch (opt) { case 'm': if (!mode_selected) { @@ -266,6 +451,9 @@ int main(int argc, char *argv[]) "Guest mode ID %d too big", mode); vm_guest_mode_params[mode].enabled = true; break; + case 'u': + use_uffd = true; + break; case 'h': default: help(argv[0]); @@ -279,7 +467,7 @@ int main(int argc, char *argv[]) TEST_ASSERT(vm_guest_mode_params[i].supported, "Guest mode ID %d (%s) not supported.", i, vm_guest_mode_string(i)); - run_test(i); + run_test(i, use_uffd); }

return 0;

Add an argument to allow the demand paging test to work on larger and smaller guest sizes.

Signed-off-by: Ben Gardon bgardon@google.com --- tools/testing/selftests/kvm/Makefile | 2 +- .../selftests/kvm/demand_paging_test.c | 57 ++++++++++------- .../testing/selftests/kvm/include/test_util.h | 2 + tools/testing/selftests/kvm/lib/test_util.c | 61 +++++++++++++++++++ 4 files changed, 100 insertions(+), 22 deletions(-) create mode 100644 tools/testing/selftests/kvm/lib/test_util.c

diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index e2e1b92faee3b..89bf05d4c2f3e 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -7,7 +7,7 @@ top_srcdir = ../../../.. KSFT_KHDR_INSTALL := 1 UNAME_M := $(shell uname -m)

-LIBKVM = lib/assert.c lib/elf.c lib/io.c lib/kvm_util.c lib/sparsebit.c +LIBKVM = lib/assert.c lib/elf.c lib/io.c lib/kvm_util.c lib/sparsebit.c lib/test_util.c LIBKVM_x86_64 = lib/x86_64/processor.c lib/x86_64/vmx.c lib/x86_64/ucall.c LIBKVM_aarch64 = lib/aarch64/processor.c lib/aarch64/ucall.c LIBKVM_s390x = lib/s390x/processor.c lib/s390x/ucall.c diff --git a/tools/testing/selftests/kvm/demand_paging_test.c b/tools/testing/selftests/kvm/demand_paging_test.c index 01d2c76ada55d..9d7514e96a639 100644 --- a/tools/testing/selftests/kvm/demand_paging_test.c +++ b/tools/testing/selftests/kvm/demand_paging_test.c @@ -32,6 +32,8 @@ /* Default guest test virtual memory offset */ #define DEFAULT_GUEST_TEST_MEM 0xc0000000

+#define DEFAULT_GUEST_TEST_MEM_SIZE (1 << 30) /* 1G */ + /* * Guest/Host shared variables. Ensure addr_gva2hva() and/or * sync_global_to/from_guest() are used when accessing from @@ -274,11 +276,10 @@ static int setup_demand_paging(struct kvm_vm *vm, return 0; }

-#define GUEST_MEM_SHIFT 30 /* 1G */ #define PAGE_SHIFT_4K 12

static void run_test(enum vm_guest_mode mode, bool use_uffd, - useconds_t uffd_delay) + useconds_t uffd_delay, uint64_t guest_memory_bytes) { pthread_t vcpu_thread; pthread_t uffd_handler_thread; @@ -287,33 +288,40 @@ static void run_test(enum vm_guest_mode mode, bool use_uffd, int r;

/* - * We reserve page table for 2 times of extra dirty mem which - * will definitely cover the original (1G+) test range. Here - * we do the calculation with 4K page size which is the - * smallest so the page number will be enough for all archs - * (e.g., 64K page size guest will need even less memory for - * page tables). + * We reserve page table for twice the ammount of memory we intend + * to use in the test region for demand paging. Here we do the + * calculation with 4K page size which is the smallest so the page + * number will be enough for all archs. (e.g., 64K page size guest + * will need even less memory for page tables). */ vm = create_vm(mode, VCPU_ID, - 2ul << (GUEST_MEM_SHIFT - PAGE_SHIFT_4K), + (2 * guest_memory_bytes) >> PAGE_SHIFT_4K, guest_code);

guest_page_size = vm_get_page_size(vm); - /* - * A little more than 1G of guest page sized pages. Cover the - * case where the size is not aligned to 64 pages. - */ - guest_num_pages = (1ul << (GUEST_MEM_SHIFT - - vm_get_page_shift(vm))) + 16; + + TEST_ASSERT(guest_memory_bytes % guest_page_size == 0, + "Guest memory size is not guest page size aligned."); + + guest_num_pages = guest_memory_bytes / guest_page_size; + #ifdef __s390x__ /* Round up to multiple of 1M (segment size) */ guest_num_pages = (guest_num_pages + 0xff) & ~0xffUL; #endif + /* + * If there should be more memory in the guest test region than there + * can be pages in the guest, it will definitely cause problems. + */ + TEST_ASSERT(guest_num_pages < vm_get_max_gfn(vm), + "Requested more guest memory than address space allows.\n" + " guest pages: %lx max gfn: %lx\n", + guest_num_pages, vm_get_max_gfn(vm));

host_page_size = getpagesize(); - host_num_pages = (guest_num_pages * guest_page_size) / host_page_size + - !!((guest_num_pages * guest_page_size) % - host_page_size); + TEST_ASSERT(guest_memory_bytes % host_page_size == 0, + "Guest memory size is not host page size aligned."); + host_num_pages = guest_memory_bytes / host_page_size;

guest_test_phys_mem = (vm_get_max_gfn(vm) - guest_num_pages) * guest_page_size; @@ -403,7 +411,8 @@ static void help(char *name) int i;

puts(""); - printf("usage: %s [-h] [-m mode] [-u] [-d uffd_delay_usec]\n", name); + printf("usage: %s [-h] [-m mode] [-u] [-d uffd_delay_usec]\n" + " [-b memory]\n", name); printf(" -m: specify the guest mode ID to test\n" " (default: test all supported modes)\n" " This option may be used multiple times.\n" @@ -417,6 +426,8 @@ static void help(char *name) printf(" -d: add a delay in usec to the User Fault\n" " FD handler to simulate demand paging\n" " overheads. Ignored without -u.\n"); + printf(" -b: specify the size of the memory region which should be\n" + " demand paged. e.g. 10M or 3G. Default: 1G\n"); puts(""); exit(0); } @@ -424,6 +435,7 @@ static void help(char *name) int main(int argc, char *argv[]) { bool mode_selected = false; + uint64_t guest_memory_bytes = DEFAULT_GUEST_TEST_MEM_SIZE; unsigned int mode; int opt, i; bool use_uffd = false; @@ -451,7 +463,7 @@ int main(int argc, char *argv[]) vm_guest_mode_params_init(VM_MODE_P40V48_4K, true, true); #endif

- while ((opt = getopt(argc, argv, "hm:ud:")) != -1) { + while ((opt = getopt(argc, argv, "hm:ud:b:")) != -1) { switch (opt) { case 'm': if (!mode_selected) { @@ -472,6 +484,9 @@ int main(int argc, char *argv[]) TEST_ASSERT(uffd_delay >= 0, "A negative UFFD delay is not supported."); break; + case 'b': + guest_memory_bytes = parse_size(optarg); + break; case 'h': default: help(argv[0]); @@ -485,7 +500,7 @@ int main(int argc, char *argv[]) TEST_ASSERT(vm_guest_mode_params[i].supported, "Guest mode ID %d (%s) not supported.", i, vm_guest_mode_string(i)); - run_test(i, use_uffd, uffd_delay); + run_test(i, use_uffd, uffd_delay, guest_memory_bytes); }

return 0; diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h index a41db6fb7e24b..e696c8219d69a 100644 --- a/tools/testing/selftests/kvm/include/test_util.h +++ b/tools/testing/selftests/kvm/include/test_util.h @@ -39,4 +39,6 @@ void test_assert(bool exp, const char *exp_str, #a, #b, #a, (unsigned long) __a, #b, (unsigned long) __b); \ } while (0)

+size_t parse_size(const char *size); + #endif /* SELFTEST_KVM_TEST_UTIL_H */ diff --git a/tools/testing/selftests/kvm/lib/test_util.c b/tools/testing/selftests/kvm/lib/test_util.c new file mode 100644 index 0000000000000..706e0f963a44b --- /dev/null +++ b/tools/testing/selftests/kvm/lib/test_util.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * tools/testing/selftests/kvm/lib/test_util.c + * + * Copyright (C) 2020, Google LLC. + */ + +#include "test_util.h" + +#include <ctype.h> + +/* + * Parses "[0-9]+[kmgt]?". + */ +size_t parse_size(const char *size) +{ + size_t len = strlen(size); + size_t i; + size_t scale_shift = 0; + size_t base; + + TEST_ASSERT(len > 0, "Need at least 1 digit in '%s'", size); + + /* Find the first letter in the string, indicating scale. */ + for (i = 0; i < len; i++) { + if (!isdigit(size[i])) { + TEST_ASSERT(i > 0, "Need at least 1 digit in '%s'", + size); + TEST_ASSERT(i == len - 1, + "Expected letter at the end in '%s'.", + size); + switch (tolower(size[i])) { + case 't': + scale_shift = 40; + break; + case 'g': + scale_shift = 30; + break; + case 'm': + scale_shift = 20; + break; + case 'k': + scale_shift = 10; + break; + default: + TEST_ASSERT(false, "Unknown size letter %c", + size[i]); + } + } + } + + TEST_ASSERT(scale_shift < 8 * sizeof(size_t), + "Overflow parsing scale!"); + + base = atoi(size); + + TEST_ASSERT(!(base & ~((1 << (sizeof(size_t) - scale_shift)) - 1)), + "Overflow parsing size!"); + + return base << scale_shift; +}

In preparation for supporting multiple vCPUs in the demand paging test, pass arguments to the vCPU in a consolidated global struct instead of syncing multiple globals.

Signed-off-by: Ben Gardon bgardon@google.com --- .../selftests/kvm/demand_paging_test.c | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-)

diff --git a/tools/testing/selftests/kvm/demand_paging_test.c b/tools/testing/selftests/kvm/demand_paging_test.c index 9d7514e96a639..9e2a5f7dfa140 100644 --- a/tools/testing/selftests/kvm/demand_paging_test.c +++ b/tools/testing/selftests/kvm/demand_paging_test.c @@ -42,7 +42,6 @@ */ static uint64_t host_page_size; static uint64_t guest_page_size; -static uint64_t guest_num_pages;

static char *guest_data_prototype;

@@ -59,18 +58,30 @@ static uint64_t guest_test_phys_mem; */ static uint64_t guest_test_virt_mem = DEFAULT_GUEST_TEST_MEM;

+struct vcpu_args { + uint64_t gva; + uint64_t pages; + + /* Only used by the host userspace part of the vCPU thread */ + int vcpu_id; + struct kvm_vm *vm; +}; + +static struct vcpu_args vcpu_args; + /* * Continuously write to the first 8 bytes of each page in the demand paging * memory region. */ static void guest_code(void) { + uint64_t gva = vcpu_args.gva; + uint64_t pages = vcpu_args.pages; int i;

- for (i = 0; i < guest_num_pages; i++) { - uint64_t addr = guest_test_virt_mem; + for (i = 0; i < pages; i++) { + uint64_t addr = gva + (i * guest_page_size);

- addr += i * guest_page_size; addr &= ~(host_page_size - 1); *(uint64_t *)addr = 0x0123456789ABCDEF; } @@ -85,15 +96,16 @@ static uint64_t host_num_pages; static void *vcpu_worker(void *data) { int ret; - struct kvm_vm *vm = data; + struct kvm_vm *vm = vcpu_args.vm; + int vcpu_id = vcpu_args.vcpu_id; struct kvm_run *run;

- run = vcpu_state(vm, VCPU_ID); + run = vcpu_state(vm, vcpu_id);

/* Let the guest access its memory */ - ret = _vcpu_run(vm, VCPU_ID); + ret = _vcpu_run(vm, vcpu_id); TEST_ASSERT(ret == 0, "vcpu_run failed: %d\n", ret); - if (get_ucall(vm, VCPU_ID, NULL) != UCALL_SYNC) { + if (get_ucall(vm, vcpu_id, NULL) != UCALL_SYNC) { TEST_ASSERT(false, "Invalid guest sync status: exit_reason=%s\n", exit_reason_str(run->exit_reason)); @@ -285,6 +297,7 @@ static void run_test(enum vm_guest_mode mode, bool use_uffd, pthread_t uffd_handler_thread; int pipefd[2]; struct kvm_vm *vm; + uint64_t guest_num_pages; int r;

/* @@ -370,10 +383,13 @@ static void run_test(enum vm_guest_mode mode, bool use_uffd, /* Export the shared variables to the guest */ sync_global_to_guest(vm, host_page_size); sync_global_to_guest(vm, guest_page_size); - sync_global_to_guest(vm, guest_test_virt_mem); - sync_global_to_guest(vm, guest_num_pages);

- pthread_create(&vcpu_thread, NULL, vcpu_worker, vm); + vcpu_args.vm = vm; + vcpu_args.vcpu_id = VCPU_ID; + vcpu_args.gva = guest_test_virt_mem; + vcpu_args.pages = guest_num_pages; + sync_global_to_guest(vm, vcpu_args); + pthread_create(&vcpu_thread, NULL, vcpu_worker, &vcpu_args);

/* Wait for the vcpu thread to quit */ pthread_join(vcpu_thread, NULL);

Most VMs have multiple vCPUs, the concurrent execution of which has a substantial impact on demand paging performance. Add an option to create multiple vCPUs to each access disjoint regions of memory.

Signed-off-by: Ben Gardon bgardon@google.com --- .../selftests/kvm/demand_paging_test.c | 255 ++++++++++++------ 1 file changed, 172 insertions(+), 83 deletions(-)

diff --git a/tools/testing/selftests/kvm/demand_paging_test.c b/tools/testing/selftests/kvm/demand_paging_test.c index 9e2a5f7dfa140..2002032df32cc 100644 --- a/tools/testing/selftests/kvm/demand_paging_test.c +++ b/tools/testing/selftests/kvm/demand_paging_test.c @@ -24,8 +24,6 @@ #include "kvm_util.h" #include "processor.h"

-#define VCPU_ID 1 - /* The memory slot index demand page */ #define TEST_MEM_SLOT_INDEX 1

@@ -34,6 +32,14 @@

#define DEFAULT_GUEST_TEST_MEM_SIZE (1 << 30) /* 1G */

+#ifdef PRINT_PER_VCPU_UPDATES +#define PER_VCPU_DEBUG(...) DEBUG(__VA_ARGS__) +#else +#define PER_VCPU_DEBUG(...) +#endif + +#define MAX_VCPUS 512 + /* * Guest/Host shared variables. Ensure addr_gva2hva() and/or * sync_global_to/from_guest() are used when accessing from @@ -67,18 +73,25 @@ struct vcpu_args { struct kvm_vm *vm; };

-static struct vcpu_args vcpu_args; +static struct vcpu_args vcpu_args[MAX_VCPUS];

/* * Continuously write to the first 8 bytes of each page in the demand paging * memory region. */ -static void guest_code(void) +static void guest_code(uint32_t vcpu_id) { - uint64_t gva = vcpu_args.gva; - uint64_t pages = vcpu_args.pages; + uint64_t gva; + uint64_t pages; int i;

+ /* Return to signal error if vCPU args data structure is courrupt. */ + if (vcpu_args[vcpu_id].vcpu_id != vcpu_id) + return; + + gva = vcpu_args[vcpu_id].gva; + pages = vcpu_args[vcpu_id].pages; + for (i = 0; i < pages; i++) { uint64_t addr = gva + (i * guest_page_size);

@@ -89,17 +102,15 @@ static void guest_code(void) GUEST_SYNC(1); }

-/* Points to the test VM memory region on which we are doing demand paging */ -static void *host_test_mem; -static uint64_t host_num_pages; - static void *vcpu_worker(void *data) { int ret; - struct kvm_vm *vm = vcpu_args.vm; - int vcpu_id = vcpu_args.vcpu_id; + struct vcpu_args *args = (struct vcpu_args *)data; + struct kvm_vm *vm = args->vm; + int vcpu_id = args->vcpu_id; struct kvm_run *run;

+ vcpu_args_set(vm, vcpu_id, 1, vcpu_id); run = vcpu_state(vm, vcpu_id);

/* Let the guest access its memory */ @@ -114,18 +125,33 @@ static void *vcpu_worker(void *data) return NULL; }

-static struct kvm_vm *create_vm(enum vm_guest_mode mode, uint32_t vcpuid, - uint64_t extra_mem_pages, void *guest_code) +#define PAGE_SHIFT_4K 12 +#define PTES_PER_4K_PT 512 + +static struct kvm_vm *create_vm(enum vm_guest_mode mode, int vcpus, + uint64_t vcpu_memory_bytes) { struct kvm_vm *vm; - uint64_t extra_pg_pages = extra_mem_pages / 512 * 2; + uint64_t pages = DEFAULT_GUEST_PHY_PAGES; + + /* Account for a few pages per-vCPU for stacks */ + pages += DEFAULT_STACK_PGS * vcpus; + + /* + * Reserve twice the ammount of memory needed to map the test region and + * the page table / stacks region, at 4k, for page tables. Do the + * calculation with 4K page size: the smallest of all archs. (e.g., 64K + * page size guest will need even less memory for page tables). + */ + pages += (2 * pages) / PTES_PER_4K_PT; + pages += ((2 * vcpus * vcpu_memory_bytes) >> PAGE_SHIFT_4K) / + PTES_PER_4K_PT;

- vm = _vm_create(mode, DEFAULT_GUEST_PHY_PAGES + extra_pg_pages, O_RDWR); + vm = _vm_create(mode, pages, O_RDWR); kvm_vm_elf_load(vm, program_invocation_name, 0, 0); #ifdef __x86_64__ vm_create_irqchip(vm); #endif - vm_vcpu_add_default(vm, vcpuid, guest_code); return vm; }

@@ -240,17 +266,13 @@ static void *uffd_handler_thread_fn(void *arg)

static int setup_demand_paging(struct kvm_vm *vm, pthread_t *uffd_handler_thread, int pipefd, - useconds_t uffd_delay) + useconds_t uffd_delay, + struct uffd_handler_args *uffd_args, + void *hva, uint64_t len) { int uffd; struct uffdio_api uffdio_api; struct uffdio_register uffdio_register; - struct uffd_handler_args uffd_args; - - guest_data_prototype = malloc(host_page_size); - TEST_ASSERT(guest_data_prototype, - "Failed to allocate buffer for guest data pattern"); - memset(guest_data_prototype, 0xAB, host_page_size);

uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); if (uffd == -1) { @@ -265,8 +287,8 @@ static int setup_demand_paging(struct kvm_vm *vm, return -1; }

- uffdio_register.range.start = (uint64_t)host_test_mem; - uffdio_register.range.len = host_num_pages * host_page_size; + uffdio_register.range.start = (uint64_t)hva; + uffdio_register.range.len = len; uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING; if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1) { DEBUG("ioctl uffdio_register failed\n"); @@ -279,44 +301,39 @@ static int setup_demand_paging(struct kvm_vm *vm, return -1; }

- uffd_args.uffd = uffd; - uffd_args.pipefd = pipefd; - uffd_args.delay = uffd_delay; + uffd_args->uffd = uffd; + uffd_args->pipefd = pipefd; + uffd_args->delay = uffd_delay; pthread_create(uffd_handler_thread, NULL, uffd_handler_thread_fn, - &uffd_args); + uffd_args); + + PER_VCPU_DEBUG("Created uffd thread for HVA range [%p, %p)\n", + hva, hva + len);

return 0; }

-#define PAGE_SHIFT_4K 12 - static void run_test(enum vm_guest_mode mode, bool use_uffd, - useconds_t uffd_delay, uint64_t guest_memory_bytes) + useconds_t uffd_delay, int vcpus, + uint64_t vcpu_memory_bytes) { - pthread_t vcpu_thread; - pthread_t uffd_handler_thread; - int pipefd[2]; + pthread_t *vcpu_threads; + pthread_t *uffd_handler_threads = NULL; + struct uffd_handler_args *uffd_args = NULL; + int *pipefds; struct kvm_vm *vm; uint64_t guest_num_pages; + int vcpu_id; int r;

- /* - * We reserve page table for twice the ammount of memory we intend - * to use in the test region for demand paging. Here we do the - * calculation with 4K page size which is the smallest so the page - * number will be enough for all archs. (e.g., 64K page size guest - * will need even less memory for page tables). - */ - vm = create_vm(mode, VCPU_ID, - (2 * guest_memory_bytes) >> PAGE_SHIFT_4K, - guest_code); + vm = create_vm(mode, vcpus, vcpu_memory_bytes);

guest_page_size = vm_get_page_size(vm);

- TEST_ASSERT(guest_memory_bytes % guest_page_size == 0, + TEST_ASSERT(vcpu_memory_bytes % guest_page_size == 0, "Guest memory size is not guest page size aligned.");

- guest_num_pages = guest_memory_bytes / guest_page_size; + guest_num_pages = (vcpus * vcpu_memory_bytes) / guest_page_size;

#ifdef __s390x__ /* Round up to multiple of 1M (segment size) */ @@ -328,13 +345,13 @@ static void run_test(enum vm_guest_mode mode, bool use_uffd, */ TEST_ASSERT(guest_num_pages < vm_get_max_gfn(vm), "Requested more guest memory than address space allows.\n" - " guest pages: %lx max gfn: %lx\n", - guest_num_pages, vm_get_max_gfn(vm)); + " guest pages: %lx max gfn: %lx vcpus: %d wss: %lx]\n", + guest_num_pages, vm_get_max_gfn(vm), vcpus, + vcpu_memory_bytes);

host_page_size = getpagesize(); - TEST_ASSERT(guest_memory_bytes % host_page_size == 0, + TEST_ASSERT(vcpu_memory_bytes % host_page_size == 0, "Guest memory size is not host page size aligned."); - host_num_pages = guest_memory_bytes / host_page_size;

guest_test_phys_mem = (vm_get_max_gfn(vm) - guest_num_pages) * guest_page_size; @@ -359,55 +376,116 @@ static void run_test(enum vm_guest_mode mode, bool use_uffd, virt_map(vm, guest_test_virt_mem, guest_test_phys_mem, guest_num_pages * guest_page_size, 0);

- /* Cache the HVA pointer of the region */ - host_test_mem = addr_gpa2hva(vm, (vm_paddr_t)guest_test_phys_mem); +#ifdef __aarch64__ + ucall_init(vm, NULL); +#endif + + guest_data_prototype = malloc(host_page_size); + TEST_ASSERT(guest_data_prototype, + "Failed to allocate buffer for guest data pattern"); + memset(guest_data_prototype, 0xAB, host_page_size); + + vcpu_threads = malloc(vcpus * sizeof(*vcpu_threads)); + TEST_ASSERT(vcpu_threads, "Memory allocation failed");

if (use_uffd) { - /* Set up user fault fd to handle demand paging requests. */ - r = pipe2(pipefd, O_CLOEXEC | O_NONBLOCK); - TEST_ASSERT(!r, "Failed to set up pipefd"); + uffd_handler_threads = + malloc(vcpus * sizeof(*uffd_handler_threads)); + TEST_ASSERT(uffd_handler_threads, "Memory allocation failed");

- r = setup_demand_paging(vm, &uffd_handler_thread, pipefd[0], - uffd_delay); - if (r < 0) - exit(-r); + uffd_args = malloc(vcpus * sizeof(*uffd_args)); + TEST_ASSERT(uffd_args, "Memory allocation failed"); + + pipefds = malloc(sizeof(int) * vcpus * 2); + TEST_ASSERT(pipefds, "Unable to allocate memory for pipefd"); }

+ for (vcpu_id = 0; vcpu_id < vcpus; vcpu_id++) { + vm_paddr_t vcpu_gpa; + void *vcpu_hva; + + vm_vcpu_add_default(vm, vcpu_id, guest_code); + + vcpu_gpa = guest_test_phys_mem + (vcpu_id * vcpu_memory_bytes); + PER_VCPU_DEBUG("Added VCPU %d with test mem gpa [%lx, %lx)\n", + vcpu_id, vcpu_gpa, vcpu_gpa + vcpu_memory_bytes); + + /* Cache the HVA pointer of the region */ + vcpu_hva = addr_gpa2hva(vm, vcpu_gpa); + + if (use_uffd) { + /* + * Set up user fault fd to handle demand paging + * requests. + */ + r = pipe2(&pipefds[vcpu_id * 2], + O_CLOEXEC | O_NONBLOCK); + TEST_ASSERT(!r, "Failed to set up pipefd"); + + r = setup_demand_paging(vm, + &uffd_handler_threads[vcpu_id], + pipefds[vcpu_id * 2], + uffd_delay, &uffd_args[vcpu_id], + vcpu_hva, vcpu_memory_bytes); + if (r < 0) + exit(-r); + } + #ifdef __x86_64__ - vcpu_set_cpuid(vm, VCPU_ID, kvm_get_supported_cpuid()); -#endif -#ifdef __aarch64__ - ucall_init(vm, NULL); + vcpu_set_cpuid(vm, vcpu_id, kvm_get_supported_cpuid()); #endif

+ vcpu_args[vcpu_id].vm = vm; + vcpu_args[vcpu_id].vcpu_id = vcpu_id; + vcpu_args[vcpu_id].gva = guest_test_virt_mem + + (vcpu_id * vcpu_memory_bytes); + vcpu_args[vcpu_id].pages = vcpu_memory_bytes / guest_page_size; + } + /* Export the shared variables to the guest */ sync_global_to_guest(vm, host_page_size); sync_global_to_guest(vm, guest_page_size); - - vcpu_args.vm = vm; - vcpu_args.vcpu_id = VCPU_ID; - vcpu_args.gva = guest_test_virt_mem; - vcpu_args.pages = guest_num_pages; sync_global_to_guest(vm, vcpu_args); - pthread_create(&vcpu_thread, NULL, vcpu_worker, &vcpu_args);

- /* Wait for the vcpu thread to quit */ - pthread_join(vcpu_thread, NULL); + DEBUG("Finished creating vCPUs and starting uffd threads\n"); + + for (vcpu_id = 0; vcpu_id < vcpus; vcpu_id++) { + pthread_create(&vcpu_threads[vcpu_id], NULL, vcpu_worker, + &vcpu_args[vcpu_id]); + } + + DEBUG("Started all vCPUs\n"); + + /* Wait for the vcpu threads to quit */ + for (vcpu_id = 0; vcpu_id < vcpus; vcpu_id++) { + pthread_join(vcpu_threads[vcpu_id], NULL); + PER_VCPU_DEBUG("Joined thread for vCPU %d\n", vcpu_id); + } + + DEBUG("All vCPU threads joined\n");

if (use_uffd) { char c;

- /* Tell the user fault fd handler thread to quit */ - r = write(pipefd[1], &c, 1); - TEST_ASSERT(r == 1, "Unable to write to pipefd"); + /* Tell the user fault fd handler threads to quit */ + for (vcpu_id = 0; vcpu_id < vcpus; vcpu_id++) { + r = write(pipefds[vcpu_id * 2 + 1], &c, 1); + TEST_ASSERT(r == 1, "Unable to write to pipefd");

- pthread_join(uffd_handler_thread, NULL); + pthread_join(uffd_handler_threads[vcpu_id], NULL); + } }

ucall_uninit(vm); kvm_vm_free(vm);

free(guest_data_prototype); + free(vcpu_threads); + if (use_uffd) { + free(uffd_handler_threads); + free(uffd_args); + free(pipefds); + } }

struct vm_guest_mode_params { @@ -428,7 +506,7 @@ static void help(char *name)

puts(""); printf("usage: %s [-h] [-m mode] [-u] [-d uffd_delay_usec]\n" - " [-b memory]\n", name); + " [-b memory] [-v vcpus]\n", name); printf(" -m: specify the guest mode ID to test\n" " (default: test all supported modes)\n" " This option may be used multiple times.\n" @@ -443,7 +521,9 @@ static void help(char *name) " FD handler to simulate demand paging\n" " overheads. Ignored without -u.\n"); printf(" -b: specify the size of the memory region which should be\n" - " demand paged. e.g. 10M or 3G. Default: 1G\n"); + " demand paged by each vCPU. e.g. 10M or 3G.\n" + " Default: 1G\n"); + printf(" -v: specify the number of vCPUs to run.\n"); puts(""); exit(0); } @@ -451,7 +531,8 @@ static void help(char *name) int main(int argc, char *argv[]) { bool mode_selected = false; - uint64_t guest_memory_bytes = DEFAULT_GUEST_TEST_MEM_SIZE; + uint64_t vcpu_memory_bytes = DEFAULT_GUEST_TEST_MEM_SIZE; + int vcpus = 1; unsigned int mode; int opt, i; bool use_uffd = false; @@ -479,7 +560,7 @@ int main(int argc, char *argv[]) vm_guest_mode_params_init(VM_MODE_P40V48_4K, true, true); #endif

- while ((opt = getopt(argc, argv, "hm:ud:b:")) != -1) { + while ((opt = getopt(argc, argv, "hm:ud:b:v:")) != -1) { switch (opt) { case 'm': if (!mode_selected) { @@ -501,7 +582,15 @@ int main(int argc, char *argv[]) "A negative UFFD delay is not supported."); break; case 'b': - guest_memory_bytes = parse_size(optarg); + vcpu_memory_bytes = parse_size(optarg); + break; + case 'v': + vcpus = atoi(optarg); + TEST_ASSERT(vcpus > 0, + "Must have a positive number of vCPUs"); + TEST_ASSERT(vcpus <= MAX_VCPUS, + "This test does not currently support\n" + "more than %d vCPUs.", MAX_VCPUS); break; case 'h': default: @@ -516,7 +605,7 @@ int main(int argc, char *argv[]) TEST_ASSERT(vm_guest_mode_params[i].supported, "Guest mode ID %d (%s) not supported.", i, vm_guest_mode_string(i)); - run_test(i, use_uffd, uffd_delay, guest_memory_bytes); + run_test(i, use_uffd, uffd_delay, vcpus, vcpu_memory_bytes); }

return 0;

KVM creates internal memslots between 3 and 4 GiB paddrs on the first vCPU creation. If memslot 0 is large enough it collides with these memslots an causes vCPU creation to fail. Instead of creating memslot 0 at paddr 0, start it 4G into the guest physical address space.

Signed-off-by: Ben Gardon bgardon@google.com --- tools/testing/selftests/kvm/lib/kvm_util.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 5b971c04f1643..427c88d32e988 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -130,9 +130,11 @@ _Static_assert(sizeof(vm_guest_mode_string)/sizeof(char *) == NUM_VM_MODES, * * Creates a VM with the mode specified by mode (e.g. VM_MODE_P52V48_4K). * When phy_pages is non-zero, a memory region of phy_pages physical pages - * is created and mapped starting at guest physical address 0. The file - * descriptor to control the created VM is created with the permissions - * given by perm (e.g. O_RDWR). + * is created, starting at 4G into the guest physical address space to avoid + * KVM internal memslots which map the region between 3G and 4G. If tests need + * to use the physical region between 0 and 3G, they can allocate another + * memslot for that region. The file descriptor to control the created VM is + * created with the permissions given by perm (e.g. O_RDWR). */ struct kvm_vm *_vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm) { @@ -231,7 +233,8 @@ struct kvm_vm *_vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm) vm->vpages_mapped = sparsebit_alloc(); if (phy_pages != 0) vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, - 0, 0, phy_pages, 0); + KVM_INTERNAL_MEMSLOTS_END_PADDR, + 0, phy_pages, 0);

return vm; }