[Sorry for the spam]
Greeting from Linaro ! We are running kvm-unit-tests on our CI Continuous Integration and testing on x86_64 and arm64 Juno-r2. Linux stable branches and Linux mainline and Linux next.
Few tests getting fail and skipped, we are interested in increasing the test coverage by adding required kernel config fragments, kernel command line arguments and user space tools.
Your help is much appreciated.
Here is the details of the LKFT kvm unit test logs,
x86_64 test output log, + ./run_tests.sh -a -v <> TESTNAME=apic-split TIMEOUT=90s ACCEL= ./x86/run x86/apic.flat -smp 2 -cpu qemu64,+x2apic,+tsc-deadline -machine kernel_irqchip=split PASS apic-split (53 tests) TESTNAME=ioapic-split TIMEOUT=90s ACCEL= ./x86/run x86/ioapic.flat -smp 1 -cpu qemu64 -machine kernel_irqchip=split PASS ioapic-split (19 tests) TESTNAME=apic TIMEOUT=30 ACCEL= ./x86/run x86/apic.flat -smp 2 -cpu qemu64,+x2apic,+tsc-deadline PASS apic (53 tests) TESTNAME=ioapic TIMEOUT=90s ACCEL= ./x86/run x86/ioapic.flat -smp 4 -cpu qemu64 PASS ioapic (26 tests) SKIP cmpxchg8b (i386 only) TESTNAME=smptest TIMEOUT=90s ACCEL= ./x86/run x86/smptest.flat -smp 2 PASS smptest (1 tests) TESTNAME=smptest3 TIMEOUT=90s ACCEL= ./x86/run x86/smptest.flat -smp 3 PASS smptest3 (1 tests) TESTNAME=vmexit_cpuid TIMEOUT=90s ACCEL= ./x86/run x86/vmexit.flat -smp 1 -append 'cpuid' PASS vmexit_cpuid TESTNAME=vmexit_vmcall TIMEOUT=90s ACCEL= ./x86/run x86/vmexit.flat -smp 1 -append 'vmcall' PASS vmexit_vmcall TESTNAME=vmexit_mov_from_cr8 TIMEOUT=90s ACCEL= ./x86/run x86/vmexit.flat -smp 1 -append 'mov_from_cr8' PASS vmexit_mov_from_cr8 [ 68.348949] APIC base relocation is unsupported by KVM [ 91.669828] grep (2166) used greatest stack depth: 11928 bytes left TESTNAME=vmexit_mov_to_cr8 TIMEOUT=90s ACCEL= ./x86/run x86/vmexit.flat -smp 1 -append 'mov_to_cr8' PASS vmexit_mov_to_cr8 TESTNAME=vmexit_inl_pmtimer TIMEOUT=90s ACCEL= ./x86/run x86/vmexit.flat -smp 1 -append 'inl_from_pmtimer' PASS vmexit_inl_pmtimer TESTNAME=vmexit_ipi TIMEOUT=90s ACCEL= ./x86/run x86/vmexit.flat -smp 2 -append 'ipi' PASS vmexit_ipi TESTNAME=vmexit_ipi_halt TIMEOUT=90s ACCEL= ./x86/run x86/vmexit.flat -smp 2 -append 'ipi_halt' PASS vmexit_ipi_halt TESTNAME=vmexit_ple_round_robin TIMEOUT=90s ACCEL= ./x86/run x86/vmexit.flat -smp 1 -append 'ple_round_robin' PASS vmexit_ple_round_robin TESTNAME=vmexit_tscdeadline TIMEOUT=90s ACCEL= ./x86/run x86/vmexit.flat -smp 1 -cpu qemu64,+x2apic,+tsc-deadline -append tscdeadline PASS vmexit_tscdeadline TESTNAME=vmexit_tscdeadline_immed TIMEOUT=90s ACCEL= ./x86/run x86/vmexit.flat -smp 1 -cpu qemu64,+x2apic,+tsc-deadline -append tscdeadline_immed PASS vmexit_tscdeadline_immed TESTNAME=access TIMEOUT=90s ACCEL= ./x86/run x86/access.flat -smp 1 -cpu host PASS access TESTNAME=smap TIMEOUT=90s ACCEL= ./x86/run x86/smap.flat -smp 1 -cpu host PASS smap (18 tests) TESTNAME=pku TIMEOUT=90s ACCEL= ./x86/run x86/pku.flat -smp 1 -cpu host SKIP pku (0 tests) TESTNAME=asyncpf TIMEOUT=90s ACCEL= ./x86/run x86/asyncpf.flat -smp 1 -m 2048 PASS asyncpf (1 tests) TESTNAME=emulator TIMEOUT=90s ACCEL= ./x86/run x86/emulator.flat -smp 1 [ 117.428273] kvm: emulating exchange as write PASS emulator (131 tests, 1 skipped) TESTNAME=eventinj TIMEOUT=90s ACCEL= ./x86/run x86/eventinj.flat -smp 1 PASS eventinj (13 tests) TESTNAME=hypercall TIMEOUT=90s ACCEL= ./x86/run x86/hypercall.flat -smp 1 PASS hypercall (2 tests) TESTNAME=idt_test TIMEOUT=90s ACCEL= ./x86/run x86/idt_test.flat -smp 1 PASS idt_test (4 tests) TESTNAME=memory TIMEOUT=90s ACCEL= ./x86/run x86/memory.flat -smp 1 -cpu host PASS memory (8 tests) TESTNAME=msr TIMEOUT=90s ACCEL= ./x86/run x86/msr.flat -smp 1 PASS msr (12 tests) cat: /proc/sys/kernel/nmi_watchdog: No such file or directory SKIP pmu (/proc/sys/kernel/nmi_watchdog not equal to 0) TESTNAME=vmware_backdoors TIMEOUT=90s ACCEL= ./x86/run x86/vmware_backdoors.flat -smp 1 -machine vmport=on -cpu host PASS vmware_backdoors (11 tests) TESTNAME=port80 TIMEOUT=90s ACCEL= ./x86/run x86/port80.flat -smp 1 PASS port80 TESTNAME=realmode TIMEOUT=90s ACCEL= ./x86/run x86/realmode.flat -smp 1 PASS realmode TESTNAME=s3 TIMEOUT=90s ACCEL= ./x86/run x86/s3.flat -smp 1 PASS s3 TESTNAME=setjmp TIMEOUT=90s ACCEL= ./x86/run x86/setjmp.flat -smp 1 PASS setjmp (10 tests) TESTNAME=sieve TIMEOUT=180 ACCEL= ./x86/run x86/sieve.flat -smp 1 PASS sieve TESTNAME=syscall TIMEOUT=90s ACCEL= ./x86/run x86/syscall.flat -smp 1 -cpu Opteron_G1,vendor=AuthenticAMD PASS syscall (2 tests) TESTNAME=tsc TIMEOUT=90s ACCEL= ./x86/run x86/tsc.flat -smp 1 -cpu kvm64,+rdtscp PASS tsc (3 tests) TESTNAME=tsc_adjust TIMEOUT=90s ACCEL= ./x86/run x86/tsc_adjust.flat -smp 1 -cpu host PASS tsc_adjust (5 tests) TESTNAME=xsave TIMEOUT=90s ACCEL= ./x86/run x86/xsave.flat -smp 1 -cpu host PASS xsave (17 tests) TESTNAME=rmap_chain TIMEOUT=90s ACCEL= ./x86/run x86/rmap_chain.flat -smp 1 PASS rmap_chain TESTNAME=svm TIMEOUT=90s ACCEL= ./x86/run x86/svm.flat -smp 2 -cpu host,+svm SKIP svm (0 tests) SKIP taskswitch (i386 only) SKIP taskswitch2 (i386 only) TESTNAME=kvmclock_test TIMEOUT=90s ACCEL= ./x86/run x86/kvmclock_test.flat -smp 2 --append "10000000 `date +%s`" PASS kvmclock_test TESTNAME=pcid TIMEOUT=90s ACCEL= ./x86/run x86/pcid.flat -smp 1 -cpu qemu64,+pcid PASS pcid (3 tests) TESTNAME=rdpru TIMEOUT=90s ACCEL= ./x86/run x86/rdpru.flat -smp 1 -cpu host PASS rdpru (1 tests) TESTNAME=umip TIMEOUT=90s ACCEL= ./x86/run x86/umip.flat -smp 1 -cpu qemu64,+umip PASS umip (21 tests) TESTNAME=vmx TIMEOUT=90s ACCEL= ./x86/run x86/vmx.flat -smp 1 -cpu host,+vmx -append "-exit_monitor_from_l2_test -ept_access* -vmx_smp* -vmx_vmcs_shadow_test -atomic_switch_overflow_msrs_test -vmx_init_signal_test -vmx_apic_passthrough_tpr_threshold_test" [ 176.338834] kvm [6439]: vcpu0, guest rIP: 0x4041d2 kvm_set_msr_common: MSR_IA32_DEBUGCTLMSR 0x1, nop [ 176.349414] kvm [6439]: vcpu0, guest rIP: 0x409eb6 kvm_set_msr_common: MSR_IA32_DEBUGCTLMSR 0x3, nop [ 176.359036] kvm [6439]: vcpu0, guest rIP: 0x40cc37 kvm_set_msr_common: MSR_IA32_DEBUGCTLMSR 0x1, nop [ 176.368929] kvm [6439]: vcpu0, guest rIP: 0x409f57 kvm_set_msr_common: MSR_IA32_DEBUGCTLMSR 0x3, nop [31mFAIL vmx (408624 tests, 3 unexpected failures, 2 expected failures, 5 skipped) TESTNAME=ept TIMEOUT=90s ACCEL= ./x86/run x86/vmx.flat -smp 1 -cpu host,host-phys-bits,+vmx -m 2560 -append "ept_access*" SKIP ept (0 tests) TESTNAME=vmx_eoi_bitmap_ioapic_scan TIMEOUT=90s ACCEL= ./x86/run x86/vmx.flat -smp 2 -cpu host,+vmx -m 2048 -append vmx_eoi_bitmap_ioapic_scan_test SKIP vmx_eoi_bitmap_ioapic_scan (1 tests, 1 skipped) TESTNAME=vmx_hlt_with_rvi_test TIMEOUT=10 ACCEL= ./x86/run x86/vmx.flat -smp 1 -cpu host,+vmx -append vmx_hlt_with_rvi_test SKIP vmx_hlt_with_rvi_test (1 tests, 1 skipped) TESTNAME=vmx_apicv_test TIMEOUT=10 ACCEL= ./x86/run x86/vmx.flat -smp 1 -cpu host,+vmx -append "apic_reg_virt_test virt_x2apic_mode_test" SKIP vmx_apicv_test (2 tests, 2 skipped) TESTNAME=vmx_apic_passthrough_thread TIMEOUT=90s ACCEL= ./x86/run x86/vmx.flat -smp 2 -cpu host,+vmx -m 2048 -append vmx_apic_passthrough_thread_test PASS vmx_apic_passthrough_thread (8 tests) TESTNAME=vmx_init_signal_test TIMEOUT=10 ACCEL= ./x86/run x86/vmx.flat -smp 2 -cpu host,+vmx -m 2048 -append vmx_init_signal_test PASS vmx_init_signal_test (9 tests) TESTNAME=vmx_apic_passthrough_tpr_threshold_test TIMEOUT=10 ACCEL= ./x86/run x86/vmx.flat -smp 1 -cpu host,+vmx -m 2048 -append vmx_apic_passthrough_tpr_threshold_test PASS vmx_apic_passthrough_tpr_threshold_test (6 tests) TESTNAME=vmx_vmcs_shadow_test TIMEOUT=90s ACCEL= ./x86/run x86/vmx.flat -smp 1 -cpu host,+vmx -append vmx_vmcs_shadow_test PASS vmx_vmcs_shadow_test (142218 tests) TESTNAME=debug TIMEOUT=90s ACCEL= ./x86/run x86/debug.flat -smp 1 PASS debug (11 tests) TESTNAME=hyperv_synic TIMEOUT=90s ACCEL= ./x86/run x86/hyperv_synic.flat -smp 2 -cpu kvm64,hv_vpindex,hv_synic -device hyperv-testdev PASS hyperv_synic (1 tests) TESTNAME=hyperv_connections TIMEOUT=90s ACCEL= ./x86/run x86/hyperv_connections.flat -smp 2 -cpu kvm64,hv_vpindex,hv_synic -device hyperv-testdev SKIP hyperv_connections (1 tests, 1 skipped) TESTNAME=hyperv_stimer TIMEOUT=90s ACCEL= ./x86/run x86/hyperv_stimer.flat -smp 2 -cpu kvm64,hv_vpindex,hv_time,hv_synic,hv_stimer -device hyperv-testdev PASS hyperv_stimer (12 tests) TESTNAME=hyperv_clock TIMEOUT=90s ACCEL= ./x86/run x86/hyperv_clock.flat -smp 2 -cpu kvm64,hv_time PASS hyperv_clock TESTNAME=intel_iommu TIMEOUT=30 ACCEL= ./x86/run x86/intel-iommu.flat -smp 4 -M q35,kernel-irqchip=split -device intel-iommu,intremap=on,eim=off -device edu PASS intel_iommu (11 tests) TESTNAME=tsx-ctrl TIMEOUT=90s ACCEL= ./x86/run x86/tsx-ctrl.flat -smp 1 -cpu host PASS tsx-ctrl
summary: -------------- access: pass apic: pass apic-split: pass asyncpf: pass cmpxchg8b: skip debug: pass emulator: pass ept: skip eventinj: pass hypercall: pass hyperv_clock: pass hyperv_connections: skip hyperv_stimer: pass hyperv_synic: pass idt_test: pass intel_iommu: pass ioapic: pass ioapic-split: pass kvmclock_test: pass memory: pass msr: pass pcid: pass pku: skip pmu: skip port80: pass rdpru: pass realmode: pass rmap_chain: pass s3: pass setjmp: pass sieve: pass smap: pass smptest: pass smptest3: pass svm: skip syscall: pass taskswitch: skip taskswitch2: skip tsc: pass tsc_adjust: pass tsx-ctrl: pass umip: pass vmexit_cpuid: pass vmexit_inl_pmtimer: pass vmexit_ipi: pass vmexit_ipi_halt: pass vmexit_mov_from_cr8: pass vmexit_mov_to_cr8: pass vmexit_ple_round_robin: pass vmexit_tscdeadline: pass vmexit_tscdeadline_immed: pass vmexit_vmcall: pass vmware_backdoors: pass vmx: fail vmx_apic_passthrough_thread: pass vmx_apic_passthrough_tpr_threshold_test: pass vmx_apicv_test: skip vmx_eoi_bitmap_ioapic_scan: skip vmx_hlt_with_rvi_test: skip vmx_init_signal_test: pass vmx_vmcs_shadow_test: pass xsave: pass
Reference links, test sources link, https://git.kernel.org/pub/scm/virt/kvm/kvm-unit-tests.git/
extra_kernel_args: kvm.enable_vmware_backdoor=1 kvm.force_emulation_prefix=1
x86 kvm unit test run, https://qa-reports.linaro.org/lkft/linux-mainline-oe/build/v5.6-rc2-17-g0a44...
Test job on lava link, x86: https://lkft.validation.linaro.org/scheduler/job/1232942 Juno-r2: https://lkft.validation.linaro.org/scheduler/job/1242488
Configs: x86: http://snapshots.linaro.org/openembedded/lkft/lkft/sumo/intel-corei7-64/lkft... Juno-r2: http://builds.tuxbuild.com/wI08H1E5eDb63gYjzt2OUg/kernel.config
On 24/02/20 13:53, Naresh Kamboju wrote:
[Sorry for the spam]
Greeting from Linaro ! We are running kvm-unit-tests on our CI Continuous Integration and testing on x86_64 and arm64 Juno-r2. Linux stable branches and Linux mainline and Linux next.
Few tests getting fail and skipped, we are interested in increasing the test coverage by adding required kernel config fragments, kernel command line arguments and user space tools.
The remainins SKIPs mostly depend on hardware, for example "svm" only runs on AMD machines and "pku" only on more recent Intel processors than you have.
FAIL vmx (408624 tests, 3 unexpected failures, 2 expected failures, 5 skipped)
This could be fixed in a more recent kernel.
Paolo
Hi Paolo,
On Mon, 24 Feb 2020 at 18:36, Paolo Bonzini pbonzini@redhat.com wrote:
On 24/02/20 13:53, Naresh Kamboju wrote:
[Sorry for the spam]
Greeting from Linaro ! We are running kvm-unit-tests on our CI Continuous Integration and testing on x86_64 and arm64 Juno-r2. Linux stable branches and Linux mainline and Linux next.
Few tests getting fail and skipped, we are interested in increasing the test coverage by adding required kernel config fragments, kernel command line arguments and user space tools.
The remainins SKIPs mostly depend on hardware, for example "svm" only runs on AMD machines and "pku" only on more recent Intel processors than you have.
Thanks for explaining the reason for skip.
FAIL vmx (408624 tests, 3 unexpected failures, 2 expected failures, 5 skipped)
This could be fixed in a more recent kernel.
I will keep running these tests on most recent kernels.
My two cents, OTOH, It would be great if we have monthly tag release for kvm-unit-tests.
LKFT plan to keep track of metadata / release tag version of each test suites and kernel branches and versions details.
Currently LKFT sending out kselftests results test summary on each linux-next release tag for x86_64, i386, arm and arm64 devices.
The next plan is to enable kvm-unit-tests results reporting from LKFT.
Paolo
On Mon, Feb 24, 2020 at 10:36:54PM +0530, Naresh Kamboju wrote:
Hi Paolo,
On Mon, 24 Feb 2020 at 18:36, Paolo Bonzini pbonzini@redhat.com wrote:
On 24/02/20 13:53, Naresh Kamboju wrote:
FAIL vmx (408624 tests, 3 unexpected failures, 2 expected failures, 5 skipped)
This could be fixed in a more recent kernel.
I will keep running these tests on most recent kernels.
My two cents, OTOH, It would be great if we have monthly tag release for kvm-unit-tests.
LKFT plan to keep track of metadata / release tag version of each test suites and kernel branches and versions details.
Currently LKFT sending out kselftests results test summary on each linux-next release tag for x86_64, i386, arm and arm64 devices.
The next plan is to enable kvm-unit-tests results reporting from LKFT.
Rather than monthly tags, what about tagging a release for each major kernel version? E.g. for v5.5, v5.6, etc... That way the compatibility is embedded in the tag itself, i.e. there's no need to cross reference release dates against kernel/KVM releases to figure out why version of kvm-unit-tests should be run.
Paolo more or less agreed to the idea[*], it's just never been implemented.
[*] https://lkml.kernel.org/r/dc5ff4ed-c6dd-74ea-03ae-4f65c5d58073@redhat.com
On Mon, Feb 24, 2020 at 09:30:33AM -0800, Sean Christopherson wrote:
On Mon, Feb 24, 2020 at 10:36:54PM +0530, Naresh Kamboju wrote:
Hi Paolo,
On Mon, 24 Feb 2020 at 18:36, Paolo Bonzini pbonzini@redhat.com wrote:
On 24/02/20 13:53, Naresh Kamboju wrote:
FAIL vmx (408624 tests, 3 unexpected failures, 2 expected failures, 5 skipped)
This could be fixed in a more recent kernel.
I will keep running these tests on most recent kernels.
My two cents, OTOH, It would be great if we have monthly tag release for kvm-unit-tests.
LKFT plan to keep track of metadata / release tag version of each test suites and kernel branches and versions details.
Currently LKFT sending out kselftests results test summary on each linux-next release tag for x86_64, i386, arm and arm64 devices.
The next plan is to enable kvm-unit-tests results reporting from LKFT.
Rather than monthly tags, what about tagging a release for each major kernel version? E.g. for v5.5, v5.6, etc... That way the compatibility is embedded in the tag itself, i.e. there's no need to cross reference release dates against kernel/KVM releases to figure out why version of kvm-unit-tests should be run.
Paolo more or less agreed to the idea[*], it's just never been implemented.
[*] https://lkml.kernel.org/r/dc5ff4ed-c6dd-74ea-03ae-4f65c5d58073@redhat.com
The behavior of kvm in LTS kernels will change over time. In general, as I wrote in that original thread as well, we would much prefer to use the latest version of kvm-unit-tests against older kernels.
I think this is a valid example (right?): v4.19 (Oct 22 2018) until now shows 245 kvm-related patches backported:
$ git log --oneline v4.19..v4.19.106 | grep -i kvm | wc -l 245
Just for curiosity I took a look at patch counts per recent releases, and it seems to average around 250 or so. This means that a 6-year extended LTS kernel branch will likely receive several releases worth of fixes.
$ git log --oneline v5.1..v5.2 | grep -i kvm | wc -l 238 $ git log --oneline v5.2..v5.3 | grep -i kvm | wc -l 239 $ git log --oneline v5.3..v5.4 | grep -i kvm | wc -l 246 $ git log --oneline v5.4..v5.5 | grep -i kvm | wc -l 172
Indeed, 4.9 has received almost two releases worth of changes since Dec 2016, and is scheduled to still be supported until 2023.
$ git log --oneline v4.9..v4.9.214 | grep -i kvm | wc -l 387
We would also like to be able to verify the additional test coverage that may be added to kvm-unit-tests against older kernels (I assume it's not all just new features that tests are added for?)
Dan
On 24/02/20 22:22, Dan Rue wrote:
We would also like to be able to verify the additional test coverage that may be added to kvm-unit-tests against older kernels (I assume it's not all just new features that tests are added for?)
Depending on what you mean by "features". Most changes to kvm-unit-tests are for aspects of the processor that are emulated more correctly. These are rarely if ever backported to stable kernels, and they show up as test failures.
Paolo
Hi Naresh,
On 2/24/20 12:53 PM, Naresh Kamboju wrote:
[Sorry for the spam]
Greeting from Linaro ! We are running kvm-unit-tests on our CI Continuous Integration and testing on x86_64 and arm64 Juno-r2. Linux stable branches and Linux mainline and Linux next.
Few tests getting fail and skipped, we are interested in increasing the test coverage by adding required kernel config fragments, kernel command line arguments and user space tools.
Your help is much appreciated.
Here is the details of the LKFT kvm unit test logs,
[..]
I am going to comment on the arm64 tests. As far as I am aware, you don't need any kernel configs to run the tests.
From looking at the java log [1], I can point out a few things:
- The gicv3 tests are failing because Juno has a gicv2 and the kernel refuses to create a virtual gicv3. It's normal.
- I am not familiar with the PMU test, so I cannot help you with that.
- Without the logs, it's hard for me to say why the micro-bench test is failing. Can you post the logs for that particular run? They are located in /path/to/kvm-unit-tests/logs/micro-bench.log. My guess is that it has to do with the fact that you are using taskset to keep the tests on one CPU. Micro-bench will use 2 VCPUs to send 2^28 IPIs which will run on the same physical CPU, and sending and receiving them will be serialized which will incur a *lot* of overhead. I tried the same test without taskset, and it worked. With taskset -c 0, it timed out like in your log.
- there are also other tests that spawn multiple VCPUs, using taskset will serialize the VCPUs and will probably hide any potential locking issues.
[1]|https://lkft.validation.linaro.org/scheduler/job/1242488%7C
|Thanks,| |Alex| ||||
On Mon, Feb 24, 2020 at 01:21:23PM +0000, Alexandru Elisei wrote:
Hi Naresh,
On 2/24/20 12:53 PM, Naresh Kamboju wrote:
[Sorry for the spam]
Greeting from Linaro ! We are running kvm-unit-tests on our CI Continuous Integration and testing on x86_64 and arm64 Juno-r2. Linux stable branches and Linux mainline and Linux next.
Few tests getting fail and skipped, we are interested in increasing the test coverage by adding required kernel config fragments, kernel command line arguments and user space tools.
Your help is much appreciated.
Here is the details of the LKFT kvm unit test logs,
[..]
I am going to comment on the arm64 tests. As far as I am aware, you don't need any kernel configs to run the tests.
From looking at the java log [1], I can point out a few things:
- The gicv3 tests are failing because Juno has a gicv2 and the kernel refuses to
create a virtual gicv3. It's normal.
Yup
- I am not familiar with the PMU test, so I cannot help you with that.
Where is the output from running the PMU test? I didn't see it in the link below.
- Without the logs, it's hard for me to say why the micro-bench test is failing.
Can you post the logs for that particular run? They are located in /path/to/kvm-unit-tests/logs/micro-bench.log. My guess is that it has to do with the fact that you are using taskset to keep the tests on one CPU. Micro-bench will use 2 VCPUs to send 2^28 IPIs which will run on the same physical CPU, and sending and receiving them will be serialized which will incur a *lot* of overhead. I tried the same test without taskset, and it worked. With taskset -c 0, it timed out like in your log.
We've also had "failures" of the micro-bench test when run under avocado reported. The problem was/is the assert_msg() on line 107 is firing. We could probably increase the number of tries or change the assert to a warning. Of course micro-bench isn't a "test" anyway so it can't "fail". Well, not unless one goes through the trouble of preparing expected times for each measurement for a given host and then compares new results to those expectations. Then it could fail when the results are too large (some threshold must be defined too).
- there are also other tests that spawn multiple VCPUs, using taskset will
serialize the VCPUs and will probably hide any potential locking issues.
Indeed.
Thanks, drew
[1]|https://lkft.validation.linaro.org/scheduler/job/1242488%7C
|Thanks,| |Alex| ||||
Hi,
On 2/24/20 1:38 PM, Andrew Jones wrote:
On Mon, Feb 24, 2020 at 01:21:23PM +0000, Alexandru Elisei wrote:
Hi Naresh,
On 2/24/20 12:53 PM, Naresh Kamboju wrote:
[Sorry for the spam]
Greeting from Linaro ! We are running kvm-unit-tests on our CI Continuous Integration and testing on x86_64 and arm64 Juno-r2. Linux stable branches and Linux mainline and Linux next.
Few tests getting fail and skipped, we are interested in increasing the test coverage by adding required kernel config fragments, kernel command line arguments and user space tools.
Your help is much appreciated.
Here is the details of the LKFT kvm unit test logs,
[..]
I am going to comment on the arm64 tests. As far as I am aware, you don't need any kernel configs to run the tests.
From looking at the java log [1], I can point out a few things:
- The gicv3 tests are failing because Juno has a gicv2 and the kernel refuses to
create a virtual gicv3. It's normal.
Yup
- I am not familiar with the PMU test, so I cannot help you with that.
Where is the output from running the PMU test? I didn't see it in the link below.
It's toward the end, it just says that 2 tests failed:
|TESTNAME=pmu TIMEOUT=90s ACCEL= ./arm/run arm/pmu.flat -smp 1| |[31mFAIL[0m pmu (3 tests, 2 unexpected failures)|
- Without the logs, it's hard for me to say why the micro-bench test is failing.
Can you post the logs for that particular run? They are located in /path/to/kvm-unit-tests/logs/micro-bench.log. My guess is that it has to do with the fact that you are using taskset to keep the tests on one CPU. Micro-bench will use 2 VCPUs to send 2^28 IPIs which will run on the same physical CPU, and sending and receiving them will be serialized which will incur a *lot* of overhead. I tried the same test without taskset, and it worked. With taskset -c 0, it timed out like in your log.
We've also had "failures" of the micro-bench test when run under avocado reported. The problem was/is the assert_msg() on line 107 is firing. We could probably increase the number of tries or change the assert to a warning. Of course micro-bench isn't a "test" anyway so it can't "fail". Well, not unless one goes through the trouble of preparing expected times for each measurement for a given host and then compares new results to those expectations. Then it could fail when the results are too large (some threshold must be defined too).
That happens to me too on occasions when running under kvmtool. When it does I just rerun the test and it passes almost always. But I think that's not the case here, since the test times out:
|TESTNAME=micro-bench TIMEOUT=90s ACCEL=kvm ./arm/run arm/micro-bench.flat -smp 2| |[31mFAIL[0m micro-bench (timeout; duration=90s)|
I tried it and I got the same message, and the in the log:
$ cat logs/micro-bench.log timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/micro-bench.flat -smp 2 # -initrd /tmp/tmp.XXOYQIrjIM Timer Frequency 40000000 Hz (Output in microseconds)
name total ns avg ns -------------------------------------------------------------------------------------------- hvc 87727475.0 1338.0 mmio_read_user 348083225.0 5311.0 mmio_read_vgic 125456300.0 1914.0 eoi 820875.0 12.0 qemu-system-aarch64: terminating on signal 15 from pid 23273 (timeout)
Thanks, Alex
On Mon, Feb 24, 2020 at 01:47:44PM +0000, Alexandru Elisei wrote:
Hi,
On 2/24/20 1:38 PM, Andrew Jones wrote:
On Mon, Feb 24, 2020 at 01:21:23PM +0000, Alexandru Elisei wrote:
Hi Naresh,
On 2/24/20 12:53 PM, Naresh Kamboju wrote:
[Sorry for the spam]
Greeting from Linaro ! We are running kvm-unit-tests on our CI Continuous Integration and testing on x86_64 and arm64 Juno-r2. Linux stable branches and Linux mainline and Linux next.
Few tests getting fail and skipped, we are interested in increasing the test coverage by adding required kernel config fragments, kernel command line arguments and user space tools.
Your help is much appreciated.
Here is the details of the LKFT kvm unit test logs,
[..]
I am going to comment on the arm64 tests. As far as I am aware, you don't need any kernel configs to run the tests.
From looking at the java log [1], I can point out a few things:
- The gicv3 tests are failing because Juno has a gicv2 and the kernel refuses to
create a virtual gicv3. It's normal.
Yup
- I am not familiar with the PMU test, so I cannot help you with that.
Where is the output from running the PMU test? I didn't see it in the link below.
It's toward the end, it just says that 2 tests failed:
If the test runner isn't capturing all the output of the tests somewhere, then it should. Naresh, is the pmu.log file somewhere?
Thanks, drew
|TESTNAME=pmu TIMEOUT=90s ACCEL= ./arm/run arm/pmu.flat -smp 1| |[31mFAIL[0m pmu (3 tests, 2 unexpected failures)|
- Without the logs, it's hard for me to say why the micro-bench test is failing.
Can you post the logs for that particular run? They are located in /path/to/kvm-unit-tests/logs/micro-bench.log. My guess is that it has to do with the fact that you are using taskset to keep the tests on one CPU. Micro-bench will use 2 VCPUs to send 2^28 IPIs which will run on the same physical CPU, and sending and receiving them will be serialized which will incur a *lot* of overhead. I tried the same test without taskset, and it worked. With taskset -c 0, it timed out like in your log.
We've also had "failures" of the micro-bench test when run under avocado reported. The problem was/is the assert_msg() on line 107 is firing. We could probably increase the number of tries or change the assert to a warning. Of course micro-bench isn't a "test" anyway so it can't "fail". Well, not unless one goes through the trouble of preparing expected times for each measurement for a given host and then compares new results to those expectations. Then it could fail when the results are too large (some threshold must be defined too).
That happens to me too on occasions when running under kvmtool. When it does I just rerun the test and it passes almost always. But I think that's not the case here, since the test times out:
|TESTNAME=micro-bench TIMEOUT=90s ACCEL=kvm ./arm/run arm/micro-bench.flat -smp 2| |[31mFAIL[0m micro-bench (timeout; duration=90s)|
I tried it and I got the same message, and the in the log:
$ cat logs/micro-bench.log timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/micro-bench.flat -smp 2 # -initrd /tmp/tmp.XXOYQIrjIM Timer Frequency 40000000 Hz (Output in microseconds)
name total ns avg ns
hvc 87727475.0 1338.0 mmio_read_user 348083225.0 5311.0 mmio_read_vgic 125456300.0 1914.0 eoi 820875.0 12.0 qemu-system-aarch64: terminating on signal 15 from pid 23273 (timeout)
Thanks, Alex
On Mon, 24 Feb 2020 at 20:29, Andrew Jones drjones@redhat.com wrote:
On Mon, Feb 24, 2020 at 01:47:44PM +0000, Alexandru Elisei wrote:
Hi,
On 2/24/20 1:38 PM, Andrew Jones wrote:
On Mon, Feb 24, 2020 at 01:21:23PM +0000, Alexandru Elisei wrote:
Hi Naresh,
On 2/24/20 12:53 PM, Naresh Kamboju wrote:
[Sorry for the spam]
Greeting from Linaro ! We are running kvm-unit-tests on our CI Continuous Integration and testing on x86_64 and arm64 Juno-r2. Linux stable branches and Linux mainline and Linux next.
Few tests getting fail and skipped, we are interested in increasing the test coverage by adding required kernel config fragments, kernel command line arguments and user space tools.
Your help is much appreciated.
Here is the details of the LKFT kvm unit test logs,
[..]
I am going to comment on the arm64 tests. As far as I am aware, you don't need any kernel configs to run the tests.
Thanks for the confirmation on Kconfig part for arm64. The next question is, How to enable and run nested virtual testing ?
From looking at the java log [1], I can point out a few things:
- The gicv3 tests are failing because Juno has a gicv2 and the kernel refuses to
create a virtual gicv3. It's normal.
Got it. Because of heterogeneous big.LITTLE CPU architecture of Juno device caused test hang and "taskset -c 0 ./run_tests.sh -a -v -t " solved this problem.
Yup
timers test is intermittent failure due to timeout on the CPU 0 which is configured as LITTLE cpu cortext a53. If i change test to run on big CPU then it always PASS. "taskset -c $BIG_CPU_ID ./run_tests.sh -a -v -t"
- I am not familiar with the PMU test, so I cannot help you with that.
Where is the output from running the PMU test? I didn't see it in the link below.
It's toward the end, it just says that 2 tests failed:
If the test runner isn't capturing all the output of the tests somewhere, then it should. Naresh, is the pmu.log file somewhere?
For more detail I have shared LAVA log [1] and attached detail run output.
timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/pmu.flat -smp 1 # -initrd /tmp/tmp.ZJ05lRvgc4 INFO: PMU version: 3 INFO: pmu: PMU implementer/ID code/counters: 0x41("A")/0x3/6 PASS: pmu: Control register Read 0 then 0. FAIL: pmu: Monotonically increasing cycle count instrs : cycles0 cycles1 ... 4: 0 cycles not incrementing! FAIL: pmu: Cycle/instruction ratio SUMMARY: 3 tests, 2 unexpected failures
arm64 juno-r2 TAP13 output and log file details. ok 1 - selftest: setup: smp: number of CPUs matches expectation ok 2 - selftest: setup: mem: memory size matches expectation ok 3 - selftest: vectors-kernel: und ok 4 - selftest: vectors-kernel: svc ok 5 - selftest: vectors-kernel: pabt ok 6 - selftest: vectors-user: und ok 7 - selftest: vectors-user: svc ok 8 - selftest: smp: PSCI version ok 9 - selftest: smp: MPIDR test on all CPUs ok 10 - pci: PCI test device passed 6/6 tests ok 11 - pmu: Control register not ok 12 - pmu: Monotonically increasing cycle count not ok 13 - pmu: Cycle/instruction ratio ok 14 - gicv2: ipi: self: IPI: self ok 15 - gicv2: ipi: target-list: IPI: directed ok 16 - gicv2: ipi: broadcast: IPI: broadcast ok 17 - gicv2: mmio: all CPUs have interrupts ok 18 - gicv2: mmio: GICD_TYPER is read-only ok 19 - gicv2: mmio: GICD_IIDR is read-only ok 20 - gicv2: mmio: ICPIDR2 is read-only ok 21 - gicv2: mmio: IPRIORITYR: consistent priority masking ok 22 - gicv2: mmio: IPRIORITYR: implements at least 4 priority bits ok 23 - gicv2: mmio: IPRIORITYR: clearing priorities ok 24 - gicv2: mmio: IPRIORITYR: accesses beyond limit RAZ/WI ok 25 - gicv2: mmio: IPRIORITYR: accessing last SPIs ok 26 - gicv2: mmio: IPRIORITYR: priorities are preserved ok 27 - gicv2: mmio: IPRIORITYR: byte reads successful ok 28 - gicv2: mmio: IPRIORITYR: byte writes successful ok 29 - gicv2: mmio: ITARGETSR: bits for non-existent CPUs masked ok 30 - gicv2: mmio: ITARGETSR: accesses beyond limit RAZ/WI ok 31 - gicv2: mmio: ITARGETSR: register content preserved ok 32 - gicv2: mmio: ITARGETSR: byte reads successful ok 33 - gicv2: mmio: ITARGETSR: byte writes successful ok 34 - gicv2: mmio: all CPUs have interrupts ok 35 - gicv2: mmio: GICD_TYPER is read-only ok 36 - gicv2: mmio: GICD_IIDR is read-only ok 37 - gicv2: mmio: ICPIDR2 is read-only ok 38 - gicv2: mmio: IPRIORITYR: consistent priority masking ok 39 - gicv2: mmio: IPRIORITYR: implements at least 4 priority bits ok 40 - gicv2: mmio: IPRIORITYR: clearing priorities ok 41 - gicv2: mmio: IPRIORITYR: accesses beyond limit RAZ/WI ok 42 - gicv2: mmio: IPRIORITYR: accessing last SPIs ok 43 - gicv2: mmio: IPRIORITYR: priorities are preserved ok 44 - gicv2: mmio: IPRIORITYR: byte reads successful ok 45 - gicv2: mmio: IPRIORITYR: byte writes successful ok 46 - gicv2: mmio: ITARGETSR: bits for non-existent CPUs masked ok 47 - gicv2: mmio: ITARGETSR: accesses beyond limit RAZ/WI ok 48 - gicv2: mmio: ITARGETSR: register content preserved ok 49 - gicv2: mmio: ITARGETSR: byte reads successful ok 50 - gicv2: mmio: ITARGETSR: byte writes successful ok 51 - gicv2: mmio: all CPUs have interrupts ok 52 - gicv2: mmio: GICD_TYPER is read-only ok 53 - gicv2: mmio: GICD_IIDR is read-only ok 54 - gicv2: mmio: ICPIDR2 is read-only ok 55 - gicv2: mmio: IPRIORITYR: consistent priority masking ok 56 - gicv2: mmio: IPRIORITYR: implements at least 4 priority bits ok 57 - gicv2: mmio: IPRIORITYR: clearing priorities ok 58 - gicv2: mmio: IPRIORITYR: accesses beyond limit RAZ/WI ok 59 - gicv2: mmio: IPRIORITYR: accessing last SPIs ok 60 - gicv2: mmio: IPRIORITYR: priorities are preserved ok 61 - gicv2: mmio: IPRIORITYR: byte reads successful ok 62 - gicv2: mmio: IPRIORITYR: byte writes successful ok 63 - gicv2: mmio: ITARGETSR: bits for non-existent CPUs masked ok 64 - gicv2: mmio: ITARGETSR: accesses beyond limit RAZ/WI ok 65 - gicv2: mmio: ITARGETSR: register content preserved ok 66 - gicv2: mmio: ITARGETSR: byte reads successful ok 67 - gicv2: mmio: ITARGETSR: byte writes successful ok 68 - gicv2: active: self: IPI: self ok 69 - psci: invalid-function ok 70 - psci: affinity-info-on ok 71 - psci: affinity-info-off ok 72 - psci: cpu-on ok 73 - vtimer-busy-loop: not pending before ok 74 - vtimer-busy-loop: interrupt signal pending ok 75 - vtimer-busy-loop: interrupt signal no longer pending ok 76 - vtimer-busy-loop: latency within 10 ms ok 77 - vtimer-busy-loop: interrupt received ok 78 - vtimer-busy-loop: interrupt received after TVAL/WFI ok 79 - vtimer-busy-loop: timer has expired ok 80 - ptimer-busy-loop: not pending before ok 81 - ptimer-busy-loop: interrupt signal pending ok 82 - ptimer-busy-loop: interrupt signal no longer pending ok 83 - ptimer-busy-loop: latency within 10 ms ok 84 - ptimer-busy-loop: interrupt received ok 85 - ptimer-busy-loop: interrupt received after TVAL/WFI ok 86 - ptimer-busy-loop: timer has expired ok 87 - IDC-DIC: code generation 1..87 + ls logs/cache.log logs/gicv2-active.log logs/gicv2-ipi.log logs/gicv2-mmio-3p.log logs/gicv2-mmio-up.log logs/gicv2-mmio.log logs/gicv3-active.log logs/gicv3-ipi.log logs/micro-bench.log logs/pci-test.log logs/pmu.log logs/psci.log logs/selftest-setup.log logs/selftest-smp.log logs/selftest-vectors-kernel.log logs/selftest-vectors-user.log logs/timer.log logs/cache.log logs/gicv3-active.log logs/selftest-setup.log logs/gicv2-active.log logs/gicv3-ipi.log logs/selftest-smp.log logs/gicv2-ipi.log logs/micro-bench.log logs/selftest-vectors-kernel.log logs/gicv2-mmio-3p.log logs/pci-test.log logs/selftest-vectors-user.log logs/gicv2-mmio-up.log logs/pmu.log logs/timer.log logs/gicv2-mmio.log logs/psci.log
+ cat logs/cache.log logs/gicv2-active.log logs/gicv2-ipi.log logs/gicv2-mmio-3p.log logs/gicv2-mmio-up.log logs/gicv2-mmio.log logs/gicv3-active.log logs/gicv3-ipi.log logs/micro-bench.log logs/pci-test.log logs/pmu.log logs/psci.log logs/selftest-setup.log logs/selftest-smp.log logs/selftest-vectors-kernel.log logs/selftest-vectors-user.log logs/timer.log timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/cache.flat -smp 1 # -initrd /tmp/tmp.m99CM6guY4 INFO: IDC-DIC: dcache clean to PoU required INFO: IDC-DIC: icache invalidation to PoU required PASS: IDC-DIC: code generation SUMMARY: 1 tests timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/gic.flat -smp 6 -machine gic-version=2 -append active # -initrd /tmp/tmp.T4AwRe5sDM PASS: gicv2: active: self: IPI: self SUMMARY: 1 tests timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/gic.flat -smp 6 -machine gic-version=2 -append ipi # -initrd /tmp/tmp.Lm1tKKNrS6 PASS: gicv2: ipi: self: IPI: self PASS: gicv2: ipi: target-list: IPI: directed PASS: gicv2: ipi: broadcast: IPI: broadcast SUMMARY: 3 tests timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/gic.flat -smp 3 -machine gic-version=2 -append mmio # -initrd /tmp/tmp.IRrNzwPw2R INFO: gicv2: mmio: number of implemented SPIs: 256 INFO: gicv2: mmio: nr_cpus=3 PASS: gicv2: mmio: all CPUs have interrupts INFO: gicv2: mmio: IIDR: 0x4b00243b PASS: gicv2: mmio: GICD_TYPER is read-only PASS: gicv2: mmio: GICD_IIDR is read-only PASS: gicv2: mmio: ICPIDR2 is read-only INFO: gicv2: mmio: value of ICPIDR2: 0x00000000 PASS: gicv2: mmio: IPRIORITYR: consistent priority masking INFO: gicv2: mmio: IPRIORITYR: priority mask is 0xf8f8f8f8 PASS: gicv2: mmio: IPRIORITYR: implements at least 4 priority bits INFO: gicv2: mmio: IPRIORITYR: 5 priority bits implemented PASS: gicv2: mmio: IPRIORITYR: clearing priorities PASS: gicv2: mmio: IPRIORITYR: accesses beyond limit RAZ/WI PASS: gicv2: mmio: IPRIORITYR: accessing last SPIs PASS: gicv2: mmio: IPRIORITYR: priorities are preserved PASS: gicv2: mmio: IPRIORITYR: byte reads successful PASS: gicv2: mmio: IPRIORITYR: byte writes successful PASS: gicv2: mmio: ITARGETSR: bits for non-existent CPUs masked INFO: gicv2: mmio: ITARGETSR: 5 non-existent CPUs PASS: gicv2: mmio: ITARGETSR: accesses beyond limit RAZ/WI PASS: gicv2: mmio: ITARGETSR: register content preserved PASS: gicv2: mmio: ITARGETSR: byte reads successful PASS: gicv2: mmio: ITARGETSR: byte writes successful SUMMARY: 17 tests timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/gic.flat -smp 1 -machine gic-version=2 -append mmio # -initrd /tmp/tmp.mLksHaaXvI INFO: gicv2: mmio: number of implemented SPIs: 256 INFO: gicv2: mmio: nr_cpus=1 PASS: gicv2: mmio: all CPUs have interrupts INFO: gicv2: mmio: IIDR: 0x4b00243b PASS: gicv2: mmio: GICD_TYPER is read-only PASS: gicv2: mmio: GICD_IIDR is read-only PASS: gicv2: mmio: ICPIDR2 is read-only INFO: gicv2: mmio: value of ICPIDR2: 0x00000000 PASS: gicv2: mmio: IPRIORITYR: consistent priority masking INFO: gicv2: mmio: IPRIORITYR: priority mask is 0xf8f8f8f8 PASS: gicv2: mmio: IPRIORITYR: implements at least 4 priority bits INFO: gicv2: mmio: IPRIORITYR: 5 priority bits implemented PASS: gicv2: mmio: IPRIORITYR: clearing priorities PASS: gicv2: mmio: IPRIORITYR: accesses beyond limit RAZ/WI PASS: gicv2: mmio: IPRIORITYR: accessing last SPIs PASS: gicv2: mmio: IPRIORITYR: priorities are preserved PASS: gicv2: mmio: IPRIORITYR: byte reads successful PASS: gicv2: mmio: IPRIORITYR: byte writes successful PASS: gicv2: mmio: ITARGETSR: bits for non-existent CPUs masked INFO: gicv2: mmio: ITARGETSR: 7 non-existent CPUs PASS: gicv2: mmio: ITARGETSR: accesses beyond limit RAZ/WI PASS: gicv2: mmio: ITARGETSR: register content preserved PASS: gicv2: mmio: ITARGETSR: byte reads successful PASS: gicv2: mmio: ITARGETSR: byte writes successful SUMMARY: 17 tests timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/gic.flat -smp 6 -machine gic-version=2 -append mmio # -initrd /tmp/tmp.tboRkZvIS4 INFO: gicv2: mmio: number of implemented SPIs: 256 INFO: gicv2: mmio: nr_cpus=6 PASS: gicv2: mmio: all CPUs have interrupts INFO: gicv2: mmio: IIDR: 0x4b00243b PASS: gicv2: mmio: GICD_TYPER is read-only PASS: gicv2: mmio: GICD_IIDR is read-only PASS: gicv2: mmio: ICPIDR2 is read-only INFO: gicv2: mmio: value of ICPIDR2: 0x00000000 PASS: gicv2: mmio: IPRIORITYR: consistent priority masking INFO: gicv2: mmio: IPRIORITYR: priority mask is 0xf8f8f8f8 PASS: gicv2: mmio: IPRIORITYR: implements at least 4 priority bits INFO: gicv2: mmio: IPRIORITYR: 5 priority bits implemented PASS: gicv2: mmio: IPRIORITYR: clearing priorities PASS: gicv2: mmio: IPRIORITYR: accesses beyond limit RAZ/WI PASS: gicv2: mmio: IPRIORITYR: accessing last SPIs PASS: gicv2: mmio: IPRIORITYR: priorities are preserved PASS: gicv2: mmio: IPRIORITYR: byte reads successful PASS: gicv2: mmio: IPRIORITYR: byte writes successful PASS: gicv2: mmio: ITARGETSR: bits for non-existent CPUs masked INFO: gicv2: mmio: ITARGETSR: 2 non-existent CPUs PASS: gicv2: mmio: ITARGETSR: accesses beyond limit RAZ/WI PASS: gicv2: mmio: ITARGETSR: register content preserved PASS: gicv2: mmio: ITARGETSR: byte reads successful PASS: gicv2: mmio: ITARGETSR: byte writes successful SUMMARY: 17 tests timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel _NO_FILE_4Uhere_ -smp 6 -machine gic-version=3 -append active # -initrd /tmp/tmp.N6d5klgltd qemu-system-aarch64: Initialization of device kvm-arm-gicv3 failed: error creating in-kernel VGIC: No such device timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel _NO_FILE_4Uhere_ -smp 6 -machine gic-version=3 -append ipi # -initrd /tmp/tmp.PUI0VvHEuT qemu-system-aarch64: Initialization of device kvm-arm-gicv3 failed: error creating in-kernel VGIC: No such device timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/micro-bench.flat -smp 2 # -initrd /tmp/tmp.urqlMsBpJd Timer Frequency 50000000 Hz (Output in microseconds) name total ns avg ns -------------------------------------------------------------------------------------------- hvc 296915440.0 4530.0 mmio_read_user 1322325100.0 20177.0 mmio_read_vgic 462255460.0 7053.0 eoi 6779880.0 103.0 qemu-system-aarch64: terminating on signal 15 from pid 3097 (timeout) timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/pci-test.flat -smp 1 # -initrd /tmp/tmp.7bpqvRWF8Z 00.00.0 1b36:0008 type 00 progif 00 class 06 subclass 00 00.01.0 1b36:0005 type 00 progif 00 class 00 subclass ff BAR#0 [10000000-10000fff MEM32] BAR#1 [3eff0000-3eff00ff PIO] pci-testdev mem: mmio-no-eventfd pci-testdev mem: mmio-wildcard-eventfd pci-testdev mem: mmio-datamatch-eventfd pci-testdev io: portio-no-eventfd pci-testdev io: portio-wildcard-eventfd pci-testdev io: portio-datamatch-eventfd PASS: pci: PCI test device passed 6/6 tests SUMMARY: 1 tests timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/pmu.flat -smp 1 # -initrd /tmp/tmp.ZJ05lRvgc4 INFO: PMU version: 3 INFO: pmu: PMU implementer/ID code/counters: 0x41("A")/0x3/6 PASS: pmu: Control register Read 0 then 0. FAIL: pmu: Monotonically increasing cycle count instrs : cycles0 cycles1 ... 4: 0 cycles not incrementing! FAIL: pmu: Cycle/instruction ratio SUMMARY: 3 tests, 2 unexpected failures timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/psci.flat -smp 6 # -initrd /tmp/tmp.F1CbWD7wLo INFO: psci: PSCI version 1.0 PASS: psci: invalid-function PASS: psci: affinity-info-on PASS: psci: affinity-info-off PASS: psci: cpu-on SUMMARY: 4 tests timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/selftest.flat -smp 2 -m 256 -append setup smp=2 mem=256 # -initrd /tmp/tmp.VNiBOSgM9H PASS: selftest: setup: smp: number of CPUs matches expectation INFO: selftest: setup: smp: found 2 CPUs PASS: selftest: setup: mem: memory size matches expectation INFO: selftest: setup: mem: found 256 MB SUMMARY: 2 tests timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/selftest.flat -smp 6 -append smp # -initrd /tmp/tmp.OoV3Gn6wac PSCI version 1.0 PASS: selftest: smp: PSCI version INFO: selftest: smp: CPU 1: MPIDR=0080000001 INFO: selftest: smp: CPU 2: MPIDR=0080000002 INFO: selftest: smp: CPU 3: MPIDR=0080000003 INFO: selftest: smp: CPU 4: MPIDR=0080000004 INFO: selftest: smp: CPU 0: MPIDR=0080000000 INFO: selftest: smp: CPU 5: MPIDR=0080000005 PASS: selftest: smp: MPIDR test on all CPUs INFO: selftest: smp: 6 CPUs reported back SUMMARY: 2 tests timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/selftest.flat -smp 1 -append vectors-kernel # -initrd /tmp/tmp.dBkubcCv7q PASS: selftest: vectors-kernel: und PASS: selftest: vectors-kernel: svc PASS: selftest: vectors-kernel: pabt SUMMARY: 3 tests timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/selftest.flat -smp 1 -append vectors-user # -initrd /tmp/tmp.sYzNVU4dPt PASS: selftest: vectors-user: und PASS: selftest: vectors-user: svc SUMMARY: 2 tests timeout -k 1s --foreground 2s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/timer.flat -smp 1 # -initrd /tmp/tmp.n02ue5jzon CNTFRQ_EL0 : 0x0000000002faf080 CNTPCT_EL0 : 0x00000002833698b6 CNTP_CTL_EL0 : 0x0000000000000004 CNTP_CVAL_EL0: 0x0000000000000000 CNTVCT_EL0 : 0x0000000001043c98 CNTV_CTL_EL0 : 0x0000000000000000 CNTV_CVAL_EL0: 0x0000000000000000 PASS: vtimer-busy-loop: not pending before PASS: vtimer-busy-loop: interrupt signal pending PASS: vtimer-busy-loop: interrupt signal no longer pending INFO: vtimer-busy-loop: After timer: 0x0000000001265c31 INFO: vtimer-busy-loop: Expected : 0x00000000012659d1 INFO: vtimer-busy-loop: Difference : 12 us PASS: vtimer-busy-loop: latency within 10 ms PASS: vtimer-busy-loop: interrupt received INFO: vtimer-busy-loop: waiting for interrupt... PASS: vtimer-busy-loop: interrupt received after TVAL/WFI PASS: vtimer-busy-loop: timer has expired INFO: vtimer-busy-loop: TVAL is -1848 ticks PASS: ptimer-busy-loop: not pending before PASS: ptimer-busy-loop: interrupt signal pending PASS: ptimer-busy-loop: interrupt signal no longer pending INFO: ptimer-busy-loop: After timer: 0x0000000283ab412b INFO: ptimer-busy-loop: Expected : 0x0000000283ab3c47 INFO: ptimer-busy-loop: Difference : 25 us PASS: ptimer-busy-loop: latency within 10 ms PASS: ptimer-busy-loop: interrupt received INFO: ptimer-busy-loop: waiting for interrupt... PASS: ptimer-busy-loop: interrupt received after TVAL/WFI PASS: ptimer-busy-loop: timer has expired INFO: ptimer-busy-loop: TVAL is -2365 ticks SUMMARY: 14 tests
Test log link, https://lkft.validation.linaro.org/scheduler/job/1249193#L1513
Hi,
On 2/24/20 4:55 PM, Naresh Kamboju wrote:
On Mon, 24 Feb 2020 at 20:29, Andrew Jones drjones@redhat.com wrote:
On Mon, Feb 24, 2020 at 01:47:44PM +0000, Alexandru Elisei wrote:
Hi,
On 2/24/20 1:38 PM, Andrew Jones wrote:
On Mon, Feb 24, 2020 at 01:21:23PM +0000, Alexandru Elisei wrote:
Hi Naresh,
On 2/24/20 12:53 PM, Naresh Kamboju wrote:
[Sorry for the spam]
Greeting from Linaro ! We are running kvm-unit-tests on our CI Continuous Integration and testing on x86_64 and arm64 Juno-r2. Linux stable branches and Linux mainline and Linux next.
Few tests getting fail and skipped, we are interested in increasing the test coverage by adding required kernel config fragments, kernel command line arguments and user space tools.
Your help is much appreciated.
Here is the details of the LKFT kvm unit test logs,
[..]
I am going to comment on the arm64 tests. As far as I am aware, you don't need any kernel configs to run the tests.
Thanks for the confirmation on Kconfig part for arm64. The next question is, How to enable and run nested virtual testing ?
There's not support in KVM to run nested guests (yet [1]) and no support in kvm-unit-tests to run at EL2 (yet [2]) and no hardware that has supported for nested virtualization (yet).
[1] https://www.spinics.net/lists/arm-kernel/msg784744.html [2] https://www.spinics.net/lists/kvm/msg203527.html
From looking at the java log [1], I can point out a few things:
- The gicv3 tests are failing because Juno has a gicv2 and the kernel refuses to
create a virtual gicv3. It's normal.
Got it. Because of heterogeneous big.LITTLE CPU architecture of Juno device caused test hang and "taskset -c 0 ./run_tests.sh -a -v -t " solved this problem.
KVM doesn't normally care about big.little configurations, and kvm-unit-tests definitely doesn't do anything that is specific to a certain microarchitecture. I would say something else is wrong here. I'll try and reproduce it on my Juno when I get the time, but that might not happen until next week. Can you trigger this behaviour every run?
Yup
timers test is intermittent failure due to timeout on the CPU 0 which is configured as LITTLE cpu cortext a53. If i change test to run on big CPU then it always PASS. "taskset -c $BIG_CPU_ID ./run_tests.sh -a -v -t"
This might just be an unfortunate mix of events and kernel scheduling decisions for the VCPU thread that is causing an unexpected delay in receiving timer interrupts. Hard to know without a log.
- I am not familiar with the PMU test, so I cannot help you with that.
Where is the output from running the PMU test? I didn't see it in the link below.
It's toward the end, it just says that 2 tests failed:
If the test runner isn't capturing all the output of the tests somewhere, then it should. Naresh, is the pmu.log file somewhere?
For more detail I have shared LAVA log [1] and attached detail run output.
timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/pmu.flat -smp 1 # -initrd /tmp/tmp.ZJ05lRvgc4 INFO: PMU version: 3 INFO: pmu: PMU implementer/ID code/counters: 0x41("A")/0x3/6 PASS: pmu: Control register Read 0 then 0. FAIL: pmu: Monotonically increasing cycle count instrs : cycles0 cycles1 ... 4: 0 cycles not incrementing! FAIL: pmu: Cycle/instruction ratio SUMMARY: 3 tests, 2 unexpected failures
This when running the tests with taskset, right?
[..] timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/micro-bench.flat -smp 2 # -initrd /tmp/tmp.urqlMsBpJd Timer Frequency 50000000 Hz (Output in microseconds) name total ns avg ns
hvc 296915440.0 4530.0 mmio_read_user 1322325100.0 20177.0 mmio_read_vgic 462255460.0 7053.0 eoi 6779880.0 103.0 qemu-system-aarch64: terminating on signal 15 from pid 3097 (timeout)
[..]
I think this is because you are running it on one physical CPU (it's exactly the same message I am getting when I use taskset to run the tests). Can you try and run it without taskset and see if it solves your issue?
Thanks, Alex
On Mon, 24 Feb 2020 at 23:06, Alexandru Elisei alexandru.elisei@arm.com wrote:
Hi,
On 2/24/20 4:55 PM, Naresh Kamboju wrote:
On Mon, 24 Feb 2020 at 20:29, Andrew Jones drjones@redhat.com wrote:
On Mon, Feb 24, 2020 at 01:47:44PM +0000, Alexandru Elisei wrote:
Hi,
On 2/24/20 1:38 PM, Andrew Jones wrote:
On Mon, Feb 24, 2020 at 01:21:23PM +0000, Alexandru Elisei wrote:
Hi Naresh,
On 2/24/20 12:53 PM, Naresh Kamboju wrote: > [..] I am going to comment on the arm64 tests. As far as I am aware, you don't need any kernel configs to run the tests.
Thanks for the confirmation on Kconfig part for arm64. The next question is, How to enable and run nested virtual testing ?
There's not support in KVM to run nested guests (yet [1]) and no support in kvm-unit-tests to run at EL2 (yet [2]) and no hardware that has supported for nested virtualization (yet).
ack.
[1] https://www.spinics.net/lists/arm-kernel/msg784744.html [2] https://www.spinics.net/lists/kvm/msg203527.html
From looking at the java log [1], I can point out a few things:
- The gicv3 tests are failing because Juno has a gicv2 and the kernel refuses to
create a virtual gicv3. It's normal.
Got it. Because of heterogeneous big.LITTLE CPU architecture of Juno device caused test hang and "taskset -c 0 ./run_tests.sh -a -v -t " solved this problem.
KVM doesn't normally care about big.little configurations, and kvm-unit-tests definitely doesn't do anything that is specific to a certain microarchitecture. I would say something else is wrong here. I'll try and reproduce it on my Juno when I get the time, but that might not happen until next week. Can you trigger this behaviour every run?
I will schedule multiple runs and get an answer by tomorrow.
Yup
timers test is intermittent failure due to timeout on the CPU 0 which is configured as LITTLE cpu cortext a53. If i change test to run on big CPU then it always PASS. "taskset -c $BIG_CPU_ID ./run_tests.sh -a -v -t"
This might just be an unfortunate mix of events and kernel scheduling decisions for the VCPU thread that is causing an unexpected delay in receiving timer interrupts. Hard to know without a log.
- I am not familiar with the PMU test, so I cannot help you with that.
Where is the output from running the PMU test? I didn't see it in the link below.
It's toward the end, it just says that 2 tests failed:
If the test runner isn't capturing all the output of the tests somewhere, then it should. Naresh, is the pmu.log file somewhere?
For more detail I have shared LAVA log [1] and attached detail run output.
timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/pmu.flat -smp 1 # -initrd /tmp/tmp.ZJ05lRvgc4 INFO: PMU version: 3 INFO: pmu: PMU implementer/ID code/counters: 0x41("A")/0x3/6 PASS: pmu: Control register Read 0 then 0. FAIL: pmu: Monotonically increasing cycle count instrs : cycles0 cycles1 ... 4: 0 cycles not incrementing! FAIL: pmu: Cycle/instruction ratio SUMMARY: 3 tests, 2 unexpected failures
This when running the tests with taskset, right?
Right.
[..] timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/micro-bench.flat -smp 2 # -initrd /tmp/tmp.urqlMsBpJd Timer Frequency 50000000 Hz (Output in microseconds) name total ns avg ns
hvc 296915440.0 4530.0 mmio_read_user 1322325100.0 20177.0 mmio_read_vgic 462255460.0 7053.0 eoi 6779880.0 103.0 qemu-system-aarch64: terminating on signal 15 from pid 3097 (timeout)
[..]
I think this is because you are running it on one physical CPU (it's exactly the same message I am getting when I use taskset to run the tests). Can you try and run it without taskset and see if it solves your issue?
Alright. I will run without taskset and share logs here.
Thanks, Alex
Hi Alexandru,
On Mon, 24 Feb 2020 at 23:14, Naresh Kamboju naresh.kamboju@linaro.org wrote:
I think this is because you are running it on one physical CPU (it's exactly the same message I am getting when I use taskset to run the tests). Can you try and run it without taskset and see if it solves your issue?
We have a new problem when running [1] without taskset on Juno-r2. None of the test got pass [2] when running without taskset on Juno-r2.
Test results summary on arm64 juno-r2. selftest-setup fail selftest-vectors-kernel fail selftest-vectors-user fail selftest-smp fail pci-test skip pmu skip gicv2-ipi skip gicv2-mmio fail gicv2-mmio-up skip gicv2-mmio-3p fail gicv3-ipi skip gicv2-active skip gicv3-active skip psci fail timer skip micro-bench skip
TAP 13 version and logs output on arm64 juno-r2.
+ ./run_tests.sh -a -v -t + tee -a /lava-1250334/1/tests/2_kvm-unit-tests-tap13-1/automated/linux/kvm-unit-tests/output/result_log.txt TAP version 13 [ 129.497212] audit: type=1701 audit(1582614457.002:23): auid=4294967295 uid=0 gid=0 ses=4294967295 pid=3718 comm="qemu-system-aar" exe="/usr/bin/qemu-system-aarch64" sig=6 res=1 [ 129.513615] audit: type=1701 audit(1582614457.018:24): auid=4294967295 uid=0 gid=0 ses=4294967295 pid=3715 comm="timeout" exe="/usr/bin/timeout.coreutils" sig=6 res=1 [ 131.516562] audit: type=1701 audit(1582614459.022:25): auid=4294967295 uid=0 gid=0 ses=4294967295 pid=4008 comm="qemu-system-aar" exe="/usr/bin/qemu-system-aarch64" sig=6 res=1 [ 131.535440] audit: type=1701 audit(1582614459.038:26): auid=4294967295 uid=0 gid=0 ses=4294967295 pid=4005 comm="timeout" exe="/usr/bin/timeout.coreutils" sig=6 res=1 [ 137.918204] audit: type=1701 audit(1582614465.418:27): auid=4294967295 uid=0 gid=0 ses=4294967295 pid=5020 comm="qemu-system-aar" exe="/usr/bin/qemu-system-aarch64" sig=6 res=1 [ 137.944969] audit: type=1701 audit(1582614465.446:28): auid=4294967295 uid=0 gid=0 ses=4294967295 pid=5017 comm="timeout" exe="/usr/bin/timeout.coreutils" sig=6 res=1 [ 138.934361] audit: type=1701 audit(1582614466.438:29): auid=4294967295 uid=0 gid=0 ses=4294967295 pid=5169 comm="qemu-system-aar" exe="/usr/bin/qemu-system-aarch64" sig=6 res=1 [ 138.950974] audit: type=1701 audit(1582614466.450:30): auid=4294967295 uid=0 gid=0 ses=4294967295 pid=5166 comm="timeout" exe="/usr/bin/timeout.coreutils" sig=6 res=1 1..0 + ls logs/cache.log logs/gicv2-active.log logs/gicv2-ipi.log logs/gicv2-mmio-3p.log logs/gicv2-mmio-up.log logs/gicv2-mmio.log logs/gicv3-active.log logs/gicv3-ipi.log logs/micro-bench.log logs/pci-test.log logs/pmu.log logs/psci.log logs/selftest-setup.log logs/selftest-smp.log logs/selftest-vectors-kernel.log logs/selftest-vectors-user.log logs/timer.log logs/cache.log logs/gicv3-active.log logs/selftest-setup.log logs/gicv2-active.log logs/gicv3-ipi.log logs/selftest-smp.log logs/gicv2-ipi.log logs/micro-bench.log logs/selftest-vectors-kernel.log logs/gicv2-mmio-3p.log logs/pci-test.log logs/selftest-vectors-user.log logs/gicv2-mmio-up.log logs/pmu.log logs/timer.log logs/gicv2-mmio.log logs/psci.log + cat logs/cache.log logs/gicv2-active.log logs/gicv2-ipi.log logs/gicv2-mmio-3p.log logs/gicv2-mmio-up.log logs/gicv2-mmio.log logs/gicv3-active.log logs/gicv3-ipi.log logs/micro-bench.log logs/pci-test.log logs/pmu.log logs/psci.log logs/selftest-setup.log logs/selftest-smp.log logs/selftest-vectors-kernel.log logs/selftest-vectors-user.log logs/timer.log timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/cache.flat -smp 1 # -initrd /tmp/tmp.34XgPJlN9a kvm_init_vcpu failed: Invalid argument timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/gic.flat -smp 6 -machine gic-version=2 -append active # -initrd /tmp/tmp.OjpYr51BSm kvm_init_vcpu failed: Invalid argument timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel _NO_FILE_4Uhere_ -smp 6 -machine gic-version=2 -append ipi # -initrd /tmp/tmp.Tu8YjEjozY kvm_init_vcpu failed: Invalid argument timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel _NO_FILE_4Uhere_ -smp 3 -machine gic-version=2 -append mmio # -initrd /tmp/tmp.tP5NfvXIN1 kvm_init_vcpu failed: Invalid argument timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel _NO_FILE_4Uhere_ -smp 1 -machine gic-version=2 -append mmio # -initrd /tmp/tmp.YCCzSHAXbL kvm_init_vcpu failed: Invalid argument timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel _NO_FILE_4Uhere_ -smp 6 -machine gic-version=2 -append mmio # -initrd /tmp/tmp.uotwZXn3uL kvm_init_vcpu failed: Invalid argument timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel _NO_FILE_4Uhere_ -smp 6 -machine gic-version=3 -append active # -initrd /tmp/tmp.W2NfOxzsXx kvm_init_vcpu failed: Invalid argument timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel _NO_FILE_4Uhere_ -smp 6 -machine gic-version=3 -append ipi # -initrd /tmp/tmp.VoJWeZB3q5 kvm_init_vcpu failed: Invalid argument timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/micro-bench.flat -smp 2 # -initrd /tmp/tmp.oOL0QMcBXU kvm_init_vcpu failed: Invalid argument timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel _NO_FILE_4Uhere_ -smp 1 # -initrd /tmp/tmp.SxqK7GfycP kvm_init_vcpu failed: Invalid argument timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/pmu.flat -smp 1 # -initrd /tmp/tmp.ikueOgLcuz kvm_init_vcpu failed: Invalid argument timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/psci.flat -smp 6 # -initrd /tmp/tmp.DnL8Q0I9uT kvm_arm_vcpu_init failed: Invalid argument timeout: the monitored command dumped core QEMU Aborted timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/selftest.flat -smp 2 -m 256 -append setup smp=2 mem=256 # -initrd /tmp/tmp.BXVafuVRMR kvm_arm_vcpu_init failed: Invalid argument timeout: the monitored command dumped core QEMU Aborted timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel _NO_FILE_4Uhere_ -smp 6 -append smp # -initrd /tmp/tmp.oxILXsU6Lz kvm_init_vcpu failed: Invalid argument timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/selftest.flat -smp 1 -append vectors-kernel # -initrd /tmp/tmp.04kmwQgtRW kvm_init_vcpu failed: Invalid argument timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/selftest.flat -smp 1 -append vectors-user # -initrd /tmp/tmp.xd7EZ8XBHo kvm_arm_vcpu_init failed: Invalid argument timeout: the monitored command dumped core QEMU Aborted timeout -k 1s --foreground 2s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/timer.flat -smp 1 # -initrd /tmp/tmp.NmNBOxyyxP kvm_arm_vcpu_init failed: Invalid argument timeout: the monitored command dumped core QEMU Aborted
[1] https://lkft.validation.linaro.org/scheduler/job/1250335#L2594 [2] https://lkft.validation.linaro.org/scheduler/job/1250336#L1578
- Naresh
Hi,
On 2/25/20 8:20 AM, Naresh Kamboju wrote:
Hi Alexandru,
On Mon, 24 Feb 2020 at 23:14, Naresh Kamboju naresh.kamboju@linaro.org wrote:
I think this is because you are running it on one physical CPU (it's exactly the same message I am getting when I use taskset to run the tests). Can you try and run it without taskset and see if it solves your issue?
We have a new problem when running [1] without taskset on Juno-r2. None of the test got pass [2] when running without taskset on Juno-r2.
I think I have an explanation for why all the tests fail. qemu creates a vcpu to match the host cpu in kvm_arm_create_scratch_host_vcpu and it sets the target to whatever the result of the KVM_ARM_PREFERRED_TARGET ioctl is. If it's run on the little core, the target will be KVM_ARM_TARGET_CORTEX_A53. If it's run on the big core, the target will be KVM_ARM_TARGET_GENERIC_V8. I tried it a few times, and for me it has always been the big core.
The vcpu is created from a different thread by doing a KVM_ARM_VCPU_INIT ioctl and KVM makes sure that the vcpu target matches the target corresponding to the physical CPU the thread is running on. What is happening is that the vcpu thread is run on a little core, so the target as far as KVM is concerned should be KVM_ARM_TARGET_CORTEX_A53, but qemu (correctly) set it to KVM_ARM_TARGET_GENERIC_V8. The ioctl return -EINVAL (-22) and qemu dies.
To get around this, I ran the tests either only on the big cores or on the little cores.
I also managed to reliably trigger the PMU failures that you are seeing. They only happen when kvm-unit-tests is run on the little cores (ran them 10 times in a loop). When run on the big cores, everything is fine (also ran them 10 times in a loop). Log output when it fails:
# taskset -c 0,3,4,5 arm/run arm/pmu.flat /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/pmu.flat # -initrd /tmp/tmp.s4ld4DX4uK INFO: PMU version: 3 INFO: pmu: PMU implementer/ID code/counters: 0x41("A")/0x3/6 PASS: pmu: Control register Read 0 then 0. FAIL: pmu: Monotonically increasing cycle count instrs : cycles0 cycles1 ... 4: 0 cycles not incrementing! FAIL: pmu: Cycle/instruction ratio SUMMARY: 3 tests, 2 unexpected failures
I'm looking into it.
Thanks, Alex
On Tue, 3 Mar 2020 at 21:32, Alexandru Elisei alexandru.elisei@arm.com wrote:
Hi,
On 2/25/20 8:20 AM, Naresh Kamboju wrote:
Hi Alexandru,
On Mon, 24 Feb 2020 at 23:14, Naresh Kamboju naresh.kamboju@linaro.org wrote:
I think this is because you are running it on one physical CPU (it's exactly the same message I am getting when I use taskset to run the tests). Can you try and run it without taskset and see if it solves your issue?
We have a new problem when running [1] without taskset on Juno-r2. None of the test got pass [2] when running without taskset on Juno-r2.
I think I have an explanation for why all the tests fail. qemu creates a vcpu to match the host cpu in kvm_arm_create_scratch_host_vcpu and it sets the target to whatever the result of the KVM_ARM_PREFERRED_TARGET ioctl is. If it's run on the little core, the target will be KVM_ARM_TARGET_CORTEX_A53. If it's run on the big core, the target will be KVM_ARM_TARGET_GENERIC_V8. I tried it a few times, and for me it has always been the big core.
The vcpu is created from a different thread by doing a KVM_ARM_VCPU_INIT ioctl and KVM makes sure that the vcpu target matches the target corresponding to the physical CPU the thread is running on. What is happening is that the vcpu thread is run on a little core, so the target as far as KVM is concerned should be KVM_ARM_TARGET_CORTEX_A53, but qemu (correctly) set it to KVM_ARM_TARGET_GENERIC_V8. The ioctl return -EINVAL (-22) and qemu dies.
To get around this, I ran the tests either only on the big cores or on the little cores.
Thanks for explaining in details. I have seen this scenario and defined my test to run only on CPU 0. The CPU 0 on my Juno-r2 devices found to be LITTLE CPU.
I also managed to reliably trigger the PMU failures that you are seeing. They only happen when kvm-unit-tests is run on the little cores (ran them 10 times in a loop). When run on the big cores, everything is fine (also ran them 10 times in a loop). Log output when it fails:
Thanks for reproducing this PMU failure.
# taskset -c 0,3,4,5 arm/run arm/pmu.flat
CPU 0,3,4,5 are seem to be on little cores.
/usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/pmu.flat # -initrd /tmp/tmp.s4ld4DX4uK INFO: PMU version: 3 INFO: pmu: PMU implementer/ID code/counters: 0x41("A")/0x3/6 PASS: pmu: Control register Read 0 then 0. FAIL: pmu: Monotonically increasing cycle count instrs : cycles0 cycles1 ... 4: 0 cycles not incrementing! FAIL: pmu: Cycle/instruction ratio SUMMARY: 3 tests, 2 unexpected failures
I'm looking into it.
- Naresh
On 2020-03-03 18:53, Naresh Kamboju wrote:
On Tue, 3 Mar 2020 at 21:32, Alexandru Elisei alexandru.elisei@arm.com wrote:
Hi,
On 2/25/20 8:20 AM, Naresh Kamboju wrote:
Hi Alexandru,
On Mon, 24 Feb 2020 at 23:14, Naresh Kamboju naresh.kamboju@linaro.org wrote:
I think this is because you are running it on one physical CPU (it's exactly the same message I am getting when I use taskset to run the tests). Can you try and run it without taskset and see if it solves your issue?
We have a new problem when running [1] without taskset on Juno-r2. None of the test got pass [2] when running without taskset on Juno-r2.
I think I have an explanation for why all the tests fail. qemu creates a vcpu to match the host cpu in kvm_arm_create_scratch_host_vcpu and it sets the target to whatever the result of the KVM_ARM_PREFERRED_TARGET ioctl is. If it's run on the little core, the target will be KVM_ARM_TARGET_CORTEX_A53. If it's run on the big core, the target will be KVM_ARM_TARGET_GENERIC_V8. I tried it a few times, and for me it has always been the big core.
The vcpu is created from a different thread by doing a KVM_ARM_VCPU_INIT ioctl and KVM makes sure that the vcpu target matches the target corresponding to the physical CPU the thread is running on. What is happening is that the vcpu thread is run on a little core, so the target as far as KVM is concerned should be KVM_ARM_TARGET_CORTEX_A53, but qemu (correctly) set it to KVM_ARM_TARGET_GENERIC_V8. The ioctl return -EINVAL (-22) and qemu dies.
To get around this, I ran the tests either only on the big cores or on the little cores.
Thanks for explaining in details. I have seen this scenario and defined my test to run only on CPU 0. The CPU 0 on my Juno-r2 devices found to be LITTLE CPU.
big-little? Just say no.
To be clear: this isn't a workaround. big-little is a fundamentally broken paradigm when it comes to virtualization. If you let vcpus roam of different u-archs, you will end-up with unpredictable results. So QEMU does the right thing, and refuses to start a VM in these conditions.
I suggest you drop your Juno at the nearest museum, department of Bad Ideas, and get yourself a sensible machine. Even a RPi4 (cough!) is marginally better.
I also managed to reliably trigger the PMU failures that you are seeing. They only happen when kvm-unit-tests is run on the little cores (ran them 10 times in a loop). When run on the big cores, everything is fine (also ran them 10 times in a loop). Log output when it fails:
Thanks for reproducing this PMU failure.
This one is slightly more convoluted: Nothing in the KVM PMU code expects *two* independent PMUs. What we need is a way to tie a physical PMU to the virtual PMU that gets emulated with perf on the host. I'm working on something similar for SPE, so maybe we can come up with a common approach.
Thanks,
M.
Hi Alexandru,
On Mon, 24 Feb 2020 at 19:17, Alexandru Elisei alexandru.elisei@arm.com wrote:
Hi,
On 2/24/20 1:38 PM, Andrew Jones wrote:
On Mon, Feb 24, 2020 at 01:21:23PM +0000, Alexandru Elisei wrote:
Hi Naresh,
On 2/24/20 12:53 PM, Naresh Kamboju wrote:
[Sorry for the spam]
Greeting from Linaro ! We are running kvm-unit-tests on our CI Continuous Integration and testing on x86_64 and arm64 Juno-r2. Linux stable branches and Linux mainline and Linux next.
Few tests getting fail and skipped, we are interested in increasing the test coverage by adding required kernel config fragments, kernel command line arguments and user space tools.
Your help is much appreciated.
Here is the details of the LKFT kvm unit test logs,
[..]
I am going to comment on the arm64 tests. As far as I am aware, you don't need any kernel configs to run the tests.
From looking at the java log [1], I can point out a few things:
- The gicv3 tests are failing because Juno has a gicv2 and the kernel refuses to
create a virtual gicv3. It's normal.
Yup
- I am not familiar with the PMU test, so I cannot help you with that.
Where is the output from running the PMU test? I didn't see it in the link below.
It's toward the end, it just says that 2 tests failed:
|TESTNAME=pmu TIMEOUT=90s ACCEL= ./arm/run arm/pmu.flat -smp 1| |[31mFAIL[0m pmu (3 tests, 2 unexpected failures)|
- Without the logs, it's hard for me to say why the micro-bench test is failing.
Can you post the logs for that particular run?
Would it be a good idea to print failed log on console when test fails ?
timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/pmu.flat -smp 1 # -initrd /tmp/tmp.ZJ05lRvgc4 INFO: PMU version: 3 INFO: pmu: PMU implementer/ID code/counters: 0x41("A")/0x3/6 PASS: pmu: Control register Read 0 then 0. FAIL: pmu: Monotonically increasing cycle count instrs : cycles0 cycles1 ... 4: 0 cycles not incrementing! FAIL: pmu: Cycle/instruction ratio SUMMARY: 3 tests, 2 unexpected failures
They are located in /path/to/kvm-unit-tests/logs/micro-bench.log. My guess is that it has to do with the fact that you are using taskset to keep the tests on one CPU. Micro-bench will use 2 VCPUs to send 2^28 IPIs which will run on the same physical CPU, and sending and receiving them will be serialized which will incur a *lot* of overhead. I tried the same test without taskset, and it worked. With taskset -c 0, it timed out like in your log.
We've also had "failures" of the micro-bench test when run under avocado reported. The problem was/is the assert_msg() on line 107 is firing. We could probably increase the number of tries or change the assert to a warning. Of course micro-bench isn't a "test" anyway so it can't "fail". Well, not unless one goes through the trouble of preparing expected times for each measurement for a given host and then compares new results to those expectations. Then it could fail when the results are too large (some threshold must be defined too).
That happens to me too on occasions when running under kvmtool. When it does I just rerun the test and it passes almost always. But I think that's not the case here, since the test times out:
micro-bench [1] always timeout on arm64 juno-r2 running with taskset -c 0 with new test code it failed it used to be skipped.
|TESTNAME=micro-bench TIMEOUT=90s ACCEL=kvm ./arm/run arm/micro-bench.flat -smp 2| |[31mFAIL[0m micro-bench (timeout; duration=90s)|
I tried it and I got the same message, and the in the log:
$ cat logs/micro-bench.log timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/micro-bench.flat -smp 2 # -initrd /tmp/tmp.XXOYQIrjIM Timer Frequency 40000000 Hz (Output in microseconds)
name total ns avg ns
hvc 87727475.0 1338.0 mmio_read_user 348083225.0 5311.0 mmio_read_vgic 125456300.0 1914.0 eoi 820875.0 12.0 qemu-system-aarch64: terminating on signal 15 from pid 23273 (timeout)
timeout -k 1s --foreground 90s /usr/bin/qemu-system-aarch64 -nodefaults -machine virt,gic-version=host,accel=kvm -cpu host -device virtio-serial-device -device virtconsole,chardev=ctd -chardev testdev,id=ctd -device pci-testdev -display none -serial stdio -kernel arm/micro-bench.flat -smp 2 # -initrd /tmp/tmp.urqlMsBpJd Timer Frequency 50000000 Hz (Output in microseconds) name total ns avg ns -------------------------------------------------------------------------------------------- hvc 296915440.0 4530.0 mmio_read_user 1322325100.0 20177.0 mmio_read_vgic 462255460.0 7053.0 eoi 6779880.0 103.0 qemu-system-aarch64: terminating on signal 15 from pid 3097 (timeout)
[1] https://qa-reports.linaro.org/lkft/linux-mainline-oe/tests/kvm-unit-tests/mi...
Thanks, Alex
On Tue, 25 Feb 2020 at 01:33, Paolo Bonzini pbonzini@redhat.com wrote:
On 24/02/20 13:53, Naresh Kamboju wrote:
FAIL vmx (408624 tests, 3 unexpected failures, 2 expected failures, 5 skipped)
This is fixed in the latest kvm-unit-tests.git.
Running latest kvm-unit-tests on x86_64 got this output [1]. "VMX enabled and locked by BIOS" We will enable on our BIOS and re-run these tests.
+ cat logs/vmx.log timeout -k 1s --foreground 90s /usr/bin/qemu-system-x86_64 -nodefaults -device pc-testdev -device isa-debug-exit,iobase=0xf4,iosize=0x4 -vnc none -serial stdio -device pci-testdev -machine accel=kvm -kernel x86/vmx.flat -smp 1 -cpu host,+vmx -append -exit_monitor_from_l2_test -ept_access* -vmx_smp* -vmx_vmcs_shadow_test -atomic_switch_overflow_msrs_test -vmx_init_signal_test -vmx_apic_passthrough_tpr_threshold_test # -initrd /tmp/tmp.XZcnThGaPI enabling apic paging enabled cr0 = 80010011 cr3 = 63d000 cr4 = 20 VMX enabled and locked by BIOS PASS: test vmxon with unaligned vmxon region <> PASS: ENT_LOAD_DBGCTLS enabled, GUEST_DR7 80000000 Unhandled exception 13 #GP at ip 0000000000409af5 error_code=0000 rflags=00010046 cs=00000008 rax=0000000000000001 rcx=0000000000000000 rdx=0000000000000000 rbx=0000000000635a10 rbp=0000000000644fdf rsi=0000000000000000 rdi=0000000000000000 r8=0000000000000000 r9=0000000000000000 r10=0000000000000000 r11=0000000000000000 r12=0000000000000000 r13=0000000000000000 r14=0000000000000000 r15=0000000000000000 cr0=0000000080010031 cr2=ffffffffffffe000 cr3=000000000063d000 cr4=0000000000002020 cr8=0000000000000000 STACK: @409af5 401765 400535 0x0000000000409af5: guest_state_test_main at x86/vmx_tests.c:5072 Error pretty printing stack: Traceback (most recent call last): File "./scripts/pretty_print_stacks.py", line 83, in main pretty_print_stack(binary, line) File "./scripts/pretty_print_stacks.py", line 49, in pretty_print_stack lines = open(path).readlines() File "/usr/lib/python3.5/encodings/ascii.py", line 26, in decode return codecs.ascii_decode(input, self.errors)[0] UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 7651: ordinal not in range(128) Continuing without pretty printing... STACK: @409af5 401765 400535
[1] https://lkft.validation.linaro.org/scheduler/job/1250342#L1761
Paolo
-
Alexandru Elisei -
Andrew Jones -
Dan Rue -
Marc Zyngier -
Naresh Kamboju -
Paolo Bonzini -
Sean Christopherson