Define VMX basic information fields with BIT_ULL()/GENMASK_ULL(), and replace hardcoded VMX basic numbers with these field macros.

Save the full/raw value of MSR_IA32_VMX_BASIC in the global vmcs_config as type u64 to get rid of the hi/lo crud, and then use VMX_BASIC helpers to extract info as needed.

VMX_EPTP_MT_{WB,UC} values 0x6 and 0x0 are generic x86 memory type values, no need to prefix them with VMX_EPTP_.

Signed-off-by: Xin Li xin3.li@intel.com Tested-by: Shan Kang shan.kang@intel.com Acked-by: Kai Huang kai.huang@intel.com ---

Changes since v4: * Do not split VMX_BASIC bit definitions across multiple files (Kai Huang). * Put some words to the changelog to justify changes around memory type macros (Kai Huang). * Remove a leftover ';' (Kai Huang).

Changes since v3: * Remove vmx_basic_vmcs_basic_cap() (Kai Huang). * Add 2 macros VMX_BASIC_VMCS12_SIZE and VMX_BASIC_MEM_TYPE_WB to avoid keeping 2 their bit shift macros (Kai Huang).

Changes since v2: * Simply save the full/raw value of MSR_IA32_VMX_BASIC in the global vmcs_config, and then use the helpers to extract info from it as needed (Sean Christopherson). * Move all VMX_MISC related changes to the second patch (Kai Huang). * Commonize memory type definitions used in the VMX files, as memory types are architectural.

Changes since v1: * Don't add field shift macros unless it's really needed, extra layer of indirect makes it harder to read (Sean Christopherson). * Add a static_assert() to ensure that VMX_BASIC_FEATURES_MASK doesn't overlap with VMX_BASIC_RESERVED_BITS (Sean Christopherson). * read MSR_IA32_VMX_BASIC into an u64 rather than 2 u32 (Sean Christopherson). * Add 2 new functions for extracting fields from VMX basic (Sean Christopherson). * Drop the tools header update (Sean Christopherson). * Move VMX basic field macros to arch/x86/include/asm/vmx.h. --- arch/x86/include/asm/msr-index.h | 9 --------- arch/x86/include/asm/vmx.h | 18 ++++++++++++++++-- arch/x86/kvm/vmx/capabilities.h | 6 ++---- arch/x86/kvm/vmx/nested.c | 31 ++++++++++++++++++++----------- arch/x86/kvm/vmx/vmx.c | 24 ++++++++++-------------- 5 files changed, 48 insertions(+), 40 deletions(-)

diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 1f9dc9bd13eb..e8af4cf01e89 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -1113,15 +1113,6 @@ #define MSR_IA32_VMX_VMFUNC 0x00000491 #define MSR_IA32_VMX_PROCBASED_CTLS3 0x00000492

-/* VMX_BASIC bits and bitmasks */ -#define VMX_BASIC_VMCS_SIZE_SHIFT 32 -#define VMX_BASIC_TRUE_CTLS (1ULL << 55) -#define VMX_BASIC_64 0x0001000000000000LLU -#define VMX_BASIC_MEM_TYPE_SHIFT 50 -#define VMX_BASIC_MEM_TYPE_MASK 0x003c000000000000LLU -#define VMX_BASIC_MEM_TYPE_WB 6LLU -#define VMX_BASIC_INOUT 0x0040000000000000LLU - /* Resctrl MSRs: */ /* - Intel: */ #define MSR_IA32_L3_QOS_CFG 0xc81 diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index 4dba17363008..353538b79ce5 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -121,6 +121,17 @@

#define VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR 0x000011ff

+/* x86 memory types, explicitly used in VMX only */ +#define MEM_TYPE_WB 0x6ULL +#define MEM_TYPE_UC 0x0ULL + +/* VMX_BASIC bits */ +#define VMX_BASIC_32BIT_PHYS_ADDR_ONLY BIT_ULL(48) +#define VMX_BASIC_DUAL_MONITOR_TREATMENT BIT_ULL(49) +#define VMX_BASIC_INOUT BIT_ULL(54) +#define VMX_BASIC_TRUE_CTLS BIT_ULL(55) + + #define VMX_MISC_PREEMPTION_TIMER_RATE_MASK 0x0000001f #define VMX_MISC_SAVE_EFER_LMA 0x00000020 #define VMX_MISC_ACTIVITY_HLT 0x00000040 @@ -144,6 +155,11 @@ static inline u32 vmx_basic_vmcs_size(u64 vmx_basic) return (vmx_basic & GENMASK_ULL(44, 32)) >> 32; }

+static inline u32 vmx_basic_vmcs_mem_type(u64 vmx_basic) +{ + return (vmx_basic & GENMASK_ULL(53, 50)) >> 50; +} + static inline int vmx_misc_preemption_timer_rate(u64 vmx_misc) { return vmx_misc & VMX_MISC_PREEMPTION_TIMER_RATE_MASK; @@ -506,8 +522,6 @@ enum vmcs_field { #define VMX_EPTP_PWL_5 0x20ull #define VMX_EPTP_AD_ENABLE_BIT (1ull << 6) #define VMX_EPTP_MT_MASK 0x7ull -#define VMX_EPTP_MT_WB 0x6ull -#define VMX_EPTP_MT_UC 0x0ull #define VMX_EPT_READABLE_MASK 0x1ull #define VMX_EPT_WRITABLE_MASK 0x2ull #define VMX_EPT_EXECUTABLE_MASK 0x4ull diff --git a/arch/x86/kvm/vmx/capabilities.h b/arch/x86/kvm/vmx/capabilities.h index 41a4533f9989..86ce8bb96bed 100644 --- a/arch/x86/kvm/vmx/capabilities.h +++ b/arch/x86/kvm/vmx/capabilities.h @@ -54,9 +54,7 @@ struct nested_vmx_msrs { };

struct vmcs_config { - int size; - u32 basic_cap; - u32 revision_id; + u64 basic; u32 pin_based_exec_ctrl; u32 cpu_based_exec_ctrl; u32 cpu_based_2nd_exec_ctrl; @@ -76,7 +74,7 @@ extern struct vmx_capability vmx_capability __ro_after_init;

static inline bool cpu_has_vmx_basic_inout(void) { - return (((u64)vmcs_config.basic_cap << 32) & VMX_BASIC_INOUT); + return vmcs_config.basic & VMX_BASIC_INOUT; }

static inline bool cpu_has_virtual_nmis(void) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 6329a306856b..14d0167825dd 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -1226,23 +1226,29 @@ static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask) return (superset | subset) == superset; }

+#define VMX_BASIC_FEATURES_MASK \ + (VMX_BASIC_DUAL_MONITOR_TREATMENT | \ + VMX_BASIC_INOUT | \ + VMX_BASIC_TRUE_CTLS) + +#define VMX_BASIC_RESERVED_BITS \ + (GENMASK_ULL(63, 56) | GENMASK_ULL(47, 45) | BIT_ULL(31)) + static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data) { - const u64 feature_and_reserved = - /* feature (except bit 48; see below) */ - BIT_ULL(49) | BIT_ULL(54) | BIT_ULL(55) | - /* reserved */ - BIT_ULL(31) | GENMASK_ULL(47, 45) | GENMASK_ULL(63, 56); u64 vmx_basic = vmcs_config.nested.basic;

- if (!is_bitwise_subset(vmx_basic, data, feature_and_reserved)) + static_assert(!(VMX_BASIC_FEATURES_MASK & VMX_BASIC_RESERVED_BITS)); + + if (!is_bitwise_subset(vmx_basic, data, + VMX_BASIC_FEATURES_MASK | VMX_BASIC_RESERVED_BITS)) return -EINVAL;

/* * KVM does not emulate a version of VMX that constrains physical * addresses of VMX structures (e.g. VMCS) to 32-bits. */ - if (data & BIT_ULL(48)) + if (data & VMX_BASIC_32BIT_PHYS_ADDR_ONLY) return -EINVAL;

if (vmx_basic_vmcs_revision_id(vmx_basic) != @@ -2726,11 +2732,11 @@ static bool nested_vmx_check_eptp(struct kvm_vcpu *vcpu, u64 new_eptp)

/* Check for memory type validity */ switch (new_eptp & VMX_EPTP_MT_MASK) { - case VMX_EPTP_MT_UC: + case MEM_TYPE_UC: if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT))) return false; break; - case VMX_EPTP_MT_WB: + case MEM_TYPE_WB: if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT))) return false; break; @@ -6994,6 +7000,9 @@ static void nested_vmx_setup_misc_data(struct vmcs_config *vmcs_conf, msrs->misc_high = 0; }

+#define VMX_BSAIC_VMCS12_SIZE ((u64)VMCS12_SIZE << 32) +#define VMX_BASIC_MEM_TYPE_WB (MEM_TYPE_WB << 50) + static void nested_vmx_setup_basic(struct nested_vmx_msrs *msrs) { /* @@ -7005,8 +7014,8 @@ static void nested_vmx_setup_basic(struct nested_vmx_msrs *msrs) msrs->basic = VMCS12_REVISION | VMX_BASIC_TRUE_CTLS | - ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) | - (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT); + VMX_BSAIC_VMCS12_SIZE | + VMX_BASIC_MEM_TYPE_WB;

if (cpu_has_vmx_basic_inout()) msrs->basic |= VMX_BASIC_INOUT; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index cce92f701dee..a16b3de01e3f 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2564,13 +2564,13 @@ static u64 adjust_vmx_controls64(u64 ctl_opt, u32 msr) static int setup_vmcs_config(struct vmcs_config *vmcs_conf, struct vmx_capability *vmx_cap) { - u32 vmx_msr_low, vmx_msr_high; u32 _pin_based_exec_control = 0; u32 _cpu_based_exec_control = 0; u32 _cpu_based_2nd_exec_control = 0; u64 _cpu_based_3rd_exec_control = 0; u32 _vmexit_control = 0; u32 _vmentry_control = 0; + u64 basic_msr; u64 misc_msr; int i;

@@ -2689,29 +2689,25 @@ static int setup_vmcs_config(struct vmcs_config *vmcs_conf, _vmexit_control &= ~x_ctrl; }

- rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high); + rdmsrl(MSR_IA32_VMX_BASIC, basic_msr);

/* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */ - if ((vmx_msr_high & 0x1fff) > PAGE_SIZE) + if ((vmx_basic_vmcs_size(basic_msr) > PAGE_SIZE)) return -EIO;

#ifdef CONFIG_X86_64 /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */ - if (vmx_msr_high & (1u<<16)) + if (basic_msr & VMX_BASIC_32BIT_PHYS_ADDR_ONLY) return -EIO; #endif

/* Require Write-Back (WB) memory type for VMCS accesses. */ - if (((vmx_msr_high >> 18) & 15) != 6) + if (vmx_basic_vmcs_mem_type(basic_msr) != MEM_TYPE_WB) return -EIO;

rdmsrl(MSR_IA32_VMX_MISC, misc_msr);

- vmcs_conf->size = vmx_msr_high & 0x1fff; - vmcs_conf->basic_cap = vmx_msr_high & ~0x1fff; - - vmcs_conf->revision_id = vmx_msr_low; - + vmcs_conf->basic = basic_msr; vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control; vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control; vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control; @@ -2861,13 +2857,13 @@ struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags) if (!pages) return NULL; vmcs = page_address(pages); - memset(vmcs, 0, vmcs_config.size); + memset(vmcs, 0, vmx_basic_vmcs_size(vmcs_config.basic));

/* KVM supports Enlightened VMCS v1 only */ if (kvm_is_using_evmcs()) vmcs->hdr.revision_id = KVM_EVMCS_VERSION; else - vmcs->hdr.revision_id = vmcs_config.revision_id; + vmcs->hdr.revision_id = vmx_basic_vmcs_revision_id(vmcs_config.basic);

if (shadow) vmcs->hdr.shadow_vmcs = 1; @@ -2960,7 +2956,7 @@ static __init int alloc_kvm_area(void) * physical CPU. */ if (kvm_is_using_evmcs()) - vmcs->hdr.revision_id = vmcs_config.revision_id; + vmcs->hdr.revision_id = vmx_basic_vmcs_revision_id(vmcs_config.basic);

per_cpu(vmxarea, cpu) = vmcs; } @@ -3362,7 +3358,7 @@ static int vmx_get_max_ept_level(void)

u64 construct_eptp(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level) { - u64 eptp = VMX_EPTP_MT_WB; + u64 eptp = MEM_TYPE_WB;

eptp |= (root_level == 5) ? VMX_EPTP_PWL_5 : VMX_EPTP_PWL_4;

Enable the secondary VM exit controls to prepare for FRED enabling.

The activation of the secondary VM exit controls is off now, and it will be switched on when a VMX feature needing it is enabled.

Signed-off-by: Xin Li xin3.li@intel.com Tested-by: Shan Kang shan.kang@intel.com ---

Change since v1: * Always load the secondary VM exit controls (Sean Christopherson). --- arch/x86/include/asm/msr-index.h | 1 + arch/x86/include/asm/vmx.h | 3 +++ arch/x86/kvm/vmx/capabilities.h | 9 ++++++++- arch/x86/kvm/vmx/vmcs.h | 1 + arch/x86/kvm/vmx/vmx.c | 17 ++++++++++++++++- arch/x86/kvm/vmx/vmx.h | 7 ++++++- 6 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 4fa2b3dd743e..ab9ec10a3fff 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -1112,6 +1112,7 @@ #define MSR_IA32_VMX_TRUE_ENTRY_CTLS 0x00000490 #define MSR_IA32_VMX_VMFUNC 0x00000491 #define MSR_IA32_VMX_PROCBASED_CTLS3 0x00000492 +#define MSR_IA32_VMX_EXIT_CTLS2 0x00000493

/* Resctrl MSRs: */ /* - Intel: */ diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index 76518e21c54d..272af2004111 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -105,6 +105,7 @@ #define VM_EXIT_CLEAR_BNDCFGS 0x00800000 #define VM_EXIT_PT_CONCEAL_PIP 0x01000000 #define VM_EXIT_CLEAR_IA32_RTIT_CTL 0x02000000 +#define VM_EXIT_ACTIVATE_SECONDARY_CONTROLS 0x80000000

#define VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR 0x00036dff

@@ -250,6 +251,8 @@ enum vmcs_field { TERTIARY_VM_EXEC_CONTROL_HIGH = 0x00002035, PID_POINTER_TABLE = 0x00002042, PID_POINTER_TABLE_HIGH = 0x00002043, + SECONDARY_VM_EXIT_CONTROLS = 0x00002044, + SECONDARY_VM_EXIT_CONTROLS_HIGH = 0x00002045, GUEST_PHYSICAL_ADDRESS = 0x00002400, GUEST_PHYSICAL_ADDRESS_HIGH = 0x00002401, VMCS_LINK_POINTER = 0x00002800, diff --git a/arch/x86/kvm/vmx/capabilities.h b/arch/x86/kvm/vmx/capabilities.h index cb6588238f46..e8f3ad0f79ee 100644 --- a/arch/x86/kvm/vmx/capabilities.h +++ b/arch/x86/kvm/vmx/capabilities.h @@ -59,8 +59,9 @@ struct vmcs_config { u32 cpu_based_exec_ctrl; u32 cpu_based_2nd_exec_ctrl; u64 cpu_based_3rd_exec_ctrl; - u32 vmexit_ctrl; u32 vmentry_ctrl; + u32 vmexit_ctrl; + u64 secondary_vmexit_ctrl; u64 misc; struct nested_vmx_msrs nested; }; @@ -136,6 +137,12 @@ static inline bool cpu_has_tertiary_exec_ctrls(void) CPU_BASED_ACTIVATE_TERTIARY_CONTROLS; }

+static inline bool cpu_has_secondary_vmexit_ctrls(void) +{ + return vmcs_config.vmexit_ctrl & + VM_EXIT_ACTIVATE_SECONDARY_CONTROLS; +} + static inline bool cpu_has_vmx_virtualize_apic_accesses(void) { return vmcs_config.cpu_based_2nd_exec_ctrl & diff --git a/arch/x86/kvm/vmx/vmcs.h b/arch/x86/kvm/vmx/vmcs.h index 7c1996b433e2..7d45a6504200 100644 --- a/arch/x86/kvm/vmx/vmcs.h +++ b/arch/x86/kvm/vmx/vmcs.h @@ -47,6 +47,7 @@ struct vmcs_host_state { struct vmcs_controls_shadow { u32 vm_entry; u32 vm_exit; + u64 secondary_vm_exit; u32 pin; u32 exec; u32 secondary_exec; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 581967d20659..4023474ea002 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2569,6 +2569,7 @@ static int setup_vmcs_config(struct vmcs_config *vmcs_conf, u32 _cpu_based_2nd_exec_control = 0; u64 _cpu_based_3rd_exec_control = 0; u32 _vmexit_control = 0; + u64 _secondary_vmexit_control = 0; u32 _vmentry_control = 0; u64 basic_msr; int i; @@ -2688,6 +2689,11 @@ static int setup_vmcs_config(struct vmcs_config *vmcs_conf, _vmexit_control &= ~x_ctrl; }

+ if (_vmexit_control & VM_EXIT_ACTIVATE_SECONDARY_CONTROLS) + _secondary_vmexit_control = + adjust_vmx_controls64(KVM_OPTIONAL_VMX_SECONDARY_VM_EXIT_CONTROLS, + MSR_IA32_VMX_EXIT_CTLS2); + rdmsrl(MSR_IA32_VMX_BASIC, basic_msr);

/* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */ @@ -2709,8 +2715,9 @@ static int setup_vmcs_config(struct vmcs_config *vmcs_conf, vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control; vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control; vmcs_conf->cpu_based_3rd_exec_ctrl = _cpu_based_3rd_exec_control; - vmcs_conf->vmexit_ctrl = _vmexit_control; vmcs_conf->vmentry_ctrl = _vmentry_control; + vmcs_conf->vmexit_ctrl = _vmexit_control; + vmcs_conf->secondary_vmexit_ctrl = _secondary_vmexit_control;

rdmsrl(MSR_IA32_VMX_MISC, vmcs_conf->misc);

@@ -4421,6 +4428,11 @@ static u32 vmx_vmexit_ctrl(void) ~(VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | VM_EXIT_LOAD_IA32_EFER); }

+static u64 vmx_secondary_vmexit_ctrl(void) +{ + return vmcs_config.secondary_vmexit_ctrl; +} + static void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -4766,6 +4778,9 @@ static void init_vmcs(struct vcpu_vmx *vmx)

vm_exit_controls_set(vmx, vmx_vmexit_ctrl());

+ if (cpu_has_secondary_vmexit_ctrls()) + secondary_vm_exit_controls_set(vmx, vmx_secondary_vmexit_ctrl()); + /* 22.2.1, 20.8.1 */ vm_entry_controls_set(vmx, vmx_vmentry_ctrl());

diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index e3b0985bb74a..f470eeb2a5c8 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -506,7 +506,11 @@ static inline u8 vmx_get_rvi(void) VM_EXIT_LOAD_IA32_EFER | \ VM_EXIT_CLEAR_BNDCFGS | \ VM_EXIT_PT_CONCEAL_PIP | \ - VM_EXIT_CLEAR_IA32_RTIT_CTL) + VM_EXIT_CLEAR_IA32_RTIT_CTL | \ + VM_EXIT_ACTIVATE_SECONDARY_CONTROLS) + +#define KVM_REQUIRED_VMX_SECONDARY_VM_EXIT_CONTROLS (0) +#define KVM_OPTIONAL_VMX_SECONDARY_VM_EXIT_CONTROLS (0)

#define KVM_REQUIRED_VMX_PIN_BASED_VM_EXEC_CONTROL \ (PIN_BASED_EXT_INTR_MASK | \ @@ -610,6 +614,7 @@ static __always_inline void lname##_controls_clearbit(struct vcpu_vmx *vmx, u##b } BUILD_CONTROLS_SHADOW(vm_entry, VM_ENTRY_CONTROLS, 32) BUILD_CONTROLS_SHADOW(vm_exit, VM_EXIT_CONTROLS, 32) +BUILD_CONTROLS_SHADOW(secondary_vm_exit, SECONDARY_VM_EXIT_CONTROLS, 64) BUILD_CONTROLS_SHADOW(pin, PIN_BASED_VM_EXEC_CONTROL, 32) BUILD_CONTROLS_SHADOW(exec, CPU_BASED_VM_EXEC_CONTROL, 32) BUILD_CONTROLS_SHADOW(secondary_exec, SECONDARY_VM_EXEC_CONTROL, 32)

Setup the global vmcs_config for FRED: 1) Add VM_ENTRY_LOAD_IA32_FRED to KVM_OPTIONAL_VMX_VM_ENTRY_CONTROLS to have a FRED CPU load guest FRED MSRs from VMCS upon VM entry. 2) Add SECONDARY_VM_EXIT_SAVE_IA32_FRED to KVM_OPTIONAL_VMX_SECONDARY_VM_EXIT_CONTROLS to have a FRED CPU save guest FRED MSRs to VMCS during VM exit. 3) add SECONDARY_VM_EXIT_LOAD_IA32_FRED to KVM_OPTIONAL_VMX_SECONDARY_VM_EXIT_CONTROLS to have a FRED CPU load host FRED MSRs from VMCS during VM exit.

Signed-off-by: Xin Li xin3.li@intel.com Tested-by: Shan Kang shan.kang@intel.com ---

Change since v1: * Remove FRED VM entry/exit controls consistency checks in setup_vmcs_config() (Sean Christopherson). --- arch/x86/include/asm/vmx.h | 3 +++ arch/x86/kvm/vmx/vmx.h | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index 272af2004111..cb14f7e315f5 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -106,6 +106,8 @@ #define VM_EXIT_PT_CONCEAL_PIP 0x01000000 #define VM_EXIT_CLEAR_IA32_RTIT_CTL 0x02000000 #define VM_EXIT_ACTIVATE_SECONDARY_CONTROLS 0x80000000 +#define SECONDARY_VM_EXIT_SAVE_IA32_FRED 0x00000001 +#define SECONDARY_VM_EXIT_LOAD_IA32_FRED 0x00000002

#define VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR 0x00036dff

@@ -119,6 +121,7 @@ #define VM_ENTRY_LOAD_BNDCFGS 0x00010000 #define VM_ENTRY_PT_CONCEAL_PIP 0x00020000 #define VM_ENTRY_LOAD_IA32_RTIT_CTL 0x00040000 +#define VM_ENTRY_LOAD_IA32_FRED 0x00800000

#define VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR 0x000011ff

diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index f470eeb2a5c8..3ad52437f426 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -484,7 +484,8 @@ static inline u8 vmx_get_rvi(void) VM_ENTRY_LOAD_IA32_EFER | \ VM_ENTRY_LOAD_BNDCFGS | \ VM_ENTRY_PT_CONCEAL_PIP | \ - VM_ENTRY_LOAD_IA32_RTIT_CTL) + VM_ENTRY_LOAD_IA32_RTIT_CTL | \ + VM_ENTRY_LOAD_IA32_FRED)

#define __KVM_REQUIRED_VMX_VM_EXIT_CONTROLS \ (VM_EXIT_SAVE_DEBUG_CONTROLS | \ @@ -510,7 +511,9 @@ static inline u8 vmx_get_rvi(void) VM_EXIT_ACTIVATE_SECONDARY_CONTROLS)

#define KVM_REQUIRED_VMX_SECONDARY_VM_EXIT_CONTROLS (0) -#define KVM_OPTIONAL_VMX_SECONDARY_VM_EXIT_CONTROLS (0) +#define KVM_OPTIONAL_VMX_SECONDARY_VM_EXIT_CONTROLS \ + (SECONDARY_VM_EXIT_SAVE_IA32_FRED | \ + SECONDARY_VM_EXIT_LOAD_IA32_FRED)

#define KVM_REQUIRED_VMX_PIN_BASED_VM_EXEC_CONTROL \ (PIN_BASED_EXT_INTR_MASK | \

Add FRED MSRs to the valid passthrough MSR list and set FRED MSRs intercept based on FRED enumeration.

Signed-off-by: Xin Li xin3.li@intel.com Tested-by: Shan Kang shan.kang@intel.com ---

Change since v1: * Enable FRED MSRs intercept if FRED is no longer enumerated in CPUID (Chao Gao). --- arch/x86/kvm/vmx/vmx.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 34b6676f60d8..d58ed2d3d379 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -693,6 +693,9 @@ static bool is_valid_passthrough_msr(u32 msr) case MSR_LBR_CORE_TO ... MSR_LBR_CORE_TO + 8: /* LBR MSRs. These are handled in vmx_update_intercept_for_lbr_msrs() */ return true; + case MSR_IA32_FRED_RSP0 ... MSR_IA32_FRED_CONFIG: + /* FRED MSRs should be passthrough to FRED guests only */ + return true; }

r = possible_passthrough_msr_slot(msr) != -ENOENT; @@ -7774,10 +7777,12 @@ static void update_intel_pt_cfg(struct kvm_vcpu *vcpu) static void vmx_vcpu_config_fred_after_set_cpuid(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); + bool fred_enumerated;

kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_FRED); + fred_enumerated = guest_can_use(vcpu, X86_FEATURE_FRED);

- if (guest_can_use(vcpu, X86_FEATURE_FRED)) { + if (fred_enumerated) { vm_entry_controls_setbit(vmx, VM_ENTRY_LOAD_IA32_FRED); secondary_vm_exit_controls_setbit(vmx, SECONDARY_VM_EXIT_SAVE_IA32_FRED | @@ -7788,6 +7793,16 @@ static void vmx_vcpu_config_fred_after_set_cpuid(struct kvm_vcpu *vcpu) SECONDARY_VM_EXIT_SAVE_IA32_FRED | SECONDARY_VM_EXIT_LOAD_IA32_FRED); } + + vmx_set_intercept_for_msr(vcpu, MSR_IA32_FRED_RSP0, MSR_TYPE_RW, !fred_enumerated); + vmx_set_intercept_for_msr(vcpu, MSR_IA32_FRED_RSP1, MSR_TYPE_RW, !fred_enumerated); + vmx_set_intercept_for_msr(vcpu, MSR_IA32_FRED_RSP2, MSR_TYPE_RW, !fred_enumerated); + vmx_set_intercept_for_msr(vcpu, MSR_IA32_FRED_RSP3, MSR_TYPE_RW, !fred_enumerated); + vmx_set_intercept_for_msr(vcpu, MSR_IA32_FRED_STKLVLS, MSR_TYPE_RW, !fred_enumerated); + vmx_set_intercept_for_msr(vcpu, MSR_IA32_FRED_SSP1, MSR_TYPE_RW, !fred_enumerated); + vmx_set_intercept_for_msr(vcpu, MSR_IA32_FRED_SSP2, MSR_TYPE_RW, !fred_enumerated); + vmx_set_intercept_for_msr(vcpu, MSR_IA32_FRED_SSP3, MSR_TYPE_RW, !fred_enumerated); + vmx_set_intercept_for_msr(vcpu, MSR_IA32_FRED_CONFIG, MSR_TYPE_RW, !fred_enumerated); }

static void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)

Switch MSR_IA32_FRED_RSP0 between host and guest in vmx_prepare_switch_to_{host,guest}().

MSR_IA32_FRED_RSP0 is used during ring 3 event delivery only, thus KVM, running on ring 0, can run safely with guest FRED RSP0, i.e., no need to switch between host/guest FRED RSP0 during VM entry and exit.

KVM should switch to host FRED RSP0 before returning to user level, and switch to guest FRED RSP0 before entering guest mode.

Signed-off-by: Xin Li xin3.li@intel.com Tested-by: Shan Kang shan.kang@intel.com ---

Changes since v1: * Don't use guest_cpuid_has() in vmx_prepare_switch_to_{host,guest}(), which are called from IRQ-disabled context (Chao Gao). * Reset msr_guest_fred_rsp0 in __vmx_vcpu_reset() (Chao Gao). --- arch/x86/kvm/vmx/vmx.c | 17 +++++++++++++++++ arch/x86/kvm/vmx/vmx.h | 2 ++ 2 files changed, 19 insertions(+)

diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index b7b772183ee4..264378c3b784 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1337,6 +1337,16 @@ void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu) }

wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); + + if (guest_can_use(vcpu, X86_FEATURE_FRED)) { + /* + * MSR_IA32_FRED_RSP0 is top of task stack, which never changes. + * Thus it should be initialized only once. + */ + if (unlikely(vmx->msr_host_fred_rsp0 == 0)) + vmx->msr_host_fred_rsp0 = read_msr(MSR_IA32_FRED_RSP0); + wrmsrl(MSR_IA32_FRED_RSP0, vmx->msr_guest_fred_rsp0); + } #else savesegment(fs, fs_sel); savesegment(gs, gs_sel); @@ -1381,6 +1391,11 @@ static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx) invalidate_tss_limit(); #ifdef CONFIG_X86_64 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base); + + if (guest_can_use(&vmx->vcpu, X86_FEATURE_FRED)) { + vmx->msr_guest_fred_rsp0 = read_msr(MSR_IA32_FRED_RSP0); + wrmsrl(MSR_IA32_FRED_RSP0, vmx->msr_host_fred_rsp0); + } #endif load_fixmap_gdt(raw_smp_processor_id()); vmx->guest_state_loaded = false; @@ -4889,6 +4904,8 @@ static void __vmx_vcpu_reset(struct kvm_vcpu *vcpu)

#ifdef CONFIG_X86_64 if (kvm_cpu_cap_has(X86_FEATURE_FRED)) { + vmx->msr_guest_fred_rsp0 = 0; + vmcs_write64(GUEST_IA32_FRED_CONFIG, 0); vmcs_write64(GUEST_IA32_FRED_RSP1, 0); vmcs_write64(GUEST_IA32_FRED_RSP2, 0); diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 3ad52437f426..176ad39be406 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -278,6 +278,8 @@ struct vcpu_vmx { #ifdef CONFIG_X86_64 u64 msr_host_kernel_gs_base; u64 msr_guest_kernel_gs_base; + u64 msr_host_fred_rsp0; + u64 msr_guest_fred_rsp0; #endif

u64 spec_ctrl;

Add kvm_is_fred_enabled() to get if FRED is enabled on a vCPU.

Signed-off-by: Xin Li xin3.li@intel.com Tested-by: Shan Kang shan.kang@intel.com ---

Change since v1: * Explain why it is ok to only check CR4.FRED (Chao Gao). --- arch/x86/kvm/kvm_cache_regs.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+)

diff --git a/arch/x86/kvm/kvm_cache_regs.h b/arch/x86/kvm/kvm_cache_regs.h index 75eae9c4998a..1d431c703fdf 100644 --- a/arch/x86/kvm/kvm_cache_regs.h +++ b/arch/x86/kvm/kvm_cache_regs.h @@ -187,6 +187,23 @@ static __always_inline bool kvm_is_cr4_bit_set(struct kvm_vcpu *vcpu, return !!kvm_read_cr4_bits(vcpu, cr4_bit); }

+/* + * It's enough to check just CR4.FRED (X86_CR4_FRED) to tell if + * a vCPU is running with FRED enabled, because: + * 1) CR4.FRED can be set to 1 only _after_ IA32_EFER.LMA = 1. + * 2) To leave IA-32e mode, CR4.FRED must be cleared first. + * + * More details at FRED Spec 6.0 Section 4.2 Enabling in CR4. + */ +static __always_inline bool kvm_is_fred_enabled(struct kvm_vcpu *vcpu) +{ +#ifdef CONFIG_X86_64 + return kvm_is_cr4_bit_set(vcpu, X86_CR4_FRED); +#else + return false; +#endif +} + static inline ulong kvm_read_cr3(struct kvm_vcpu *vcpu) { if (!kvm_register_is_available(vcpu, VCPU_EXREG_CR3))

Set VMX nested exception bit in the VM-entry interruption information VMCS field when injecting a nested exception using FRED event delivery to ensure: 1) The nested exception is injected on a correct stack level. 2) The nested bit defined in FRED stack frame is set.

The event stack level used by FRED event delivery depends on whether the event was a nested exception encountered during delivery of another event, because a nested exception is "regarded" as happening on ring 0. E.g., when #PF is configured to use stack level 1 in IA32_FRED_STKLVLS MSR: - nested #PF will be delivered on stack level 1 when encountered in ring 3. - normal #PF will be delivered on stack level 0 when encountered in ring 3.

The VMX nested-exception support ensures the correct event stack level is chosen when a VM entry injects a nested exception.

Signed-off-by: Xin Li xin3.li@intel.com Tested-by: Shan Kang shan.kang@intel.com ---

Changes since v1: * Set the nested flag when there is an original interrupt (Chao Gao). --- arch/x86/include/asm/kvm_host.h | 6 +++-- arch/x86/include/asm/vmx.h | 5 ++-- arch/x86/kvm/svm/svm.c | 4 +-- arch/x86/kvm/vmx/vmx.c | 8 ++++-- arch/x86/kvm/x86.c | 46 ++++++++++++++++++++++++++------- arch/x86/kvm/x86.h | 1 + 6 files changed, 53 insertions(+), 17 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 0d88873eba63..ef278ee0b6ca 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -736,6 +736,7 @@ struct kvm_queued_exception { u32 error_code; unsigned long payload; bool has_payload; + bool nested; };

struct kvm_vcpu_arch { @@ -2060,8 +2061,9 @@ int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu); void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr); void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr, unsigned long payload); -void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr); -void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); +void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr, bool nested); +void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, + u32 error_code, bool nested); void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault); void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault); diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index 6b796c5c9c2b..68af74e48788 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -134,7 +134,7 @@ #define VMX_BASIC_DUAL_MONITOR_TREATMENT BIT_ULL(49) #define VMX_BASIC_INOUT BIT_ULL(54) #define VMX_BASIC_TRUE_CTLS BIT_ULL(55) - +#define VMX_BASIC_NESTED_EXCEPTION BIT_ULL(58)

/* VMX_MISC bits and bitmasks */ #define VMX_MISC_INTEL_PT BIT_ULL(14) @@ -407,8 +407,9 @@ enum vmcs_field { #define INTR_INFO_INTR_TYPE_MASK 0x700 /* 10:8 */ #define INTR_INFO_DELIVER_CODE_MASK 0x800 /* 11 */ #define INTR_INFO_UNBLOCK_NMI 0x1000 /* 12 */ +#define INTR_INFO_NESTED_EXCEPTION_MASK 0x2000 /* 13 */ #define INTR_INFO_VALID_MASK 0x80000000 /* 31 */ -#define INTR_INFO_RESVD_BITS_MASK 0x7ffff000 +#define INTR_INFO_RESVD_BITS_MASK 0x7fffd000

#define VECTORING_INFO_VECTOR_MASK INTR_INFO_VECTOR_MASK #define VECTORING_INFO_TYPE_MASK INTR_INFO_INTR_TYPE_MASK diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index e90b429c84f1..c220b690a37c 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -4057,10 +4057,10 @@ static void svm_complete_interrupts(struct kvm_vcpu *vcpu)

if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) { u32 err = svm->vmcb->control.exit_int_info_err; - kvm_requeue_exception_e(vcpu, vector, err); + kvm_requeue_exception_e(vcpu, vector, err, false);

} else - kvm_requeue_exception(vcpu, vector); + kvm_requeue_exception(vcpu, vector, false); break; case SVM_EXITINTINFO_TYPE_INTR: kvm_queue_interrupt(vcpu, vector, false); diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index f622fb90a098..1f265d526daf 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1891,6 +1891,8 @@ static void vmx_inject_exception(struct kvm_vcpu *vcpu) event_data = to_vmx(vcpu)->fred_xfd_event_data;

vmcs_write64(INJECTED_EVENT_DATA, event_data); + + intr_info |= ex->nested ? INTR_INFO_NESTED_EXCEPTION_MASK : 0; } }

@@ -7281,9 +7283,11 @@ static void __vmx_complete_interrupts(struct kvm_vcpu *vcpu, bool vectoring) }

if (event_id & INTR_INFO_DELIVER_CODE_MASK) - kvm_requeue_exception_e(vcpu, vector, vmcs_read32(error_code_field)); + kvm_requeue_exception_e(vcpu, vector, vmcs_read32(error_code_field), + event_id & INTR_INFO_NESTED_EXCEPTION_MASK); else - kvm_requeue_exception(vcpu, vector); + kvm_requeue_exception(vcpu, vector, + event_id & INTR_INFO_NESTED_EXCEPTION_MASK); break; case INTR_TYPE_SOFT_INTR: vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 00c0062726ae..725819262085 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -645,7 +645,8 @@ static void kvm_leave_nested(struct kvm_vcpu *vcpu)

static void kvm_multiple_exception(struct kvm_vcpu *vcpu, unsigned nr, bool has_error, u32 error_code, - bool has_payload, unsigned long payload, bool reinject) + bool has_payload, unsigned long payload, + bool reinject, bool nested) { u32 prev_nr; int class1, class2; @@ -696,6 +697,13 @@ static void kvm_multiple_exception(struct kvm_vcpu *vcpu, vcpu->arch.exception.pending = true; vcpu->arch.exception.injected = false; } + + vcpu->arch.exception.nested = vcpu->arch.exception.nested || + (kvm_is_fred_enabled(vcpu) && + ((reinject && nested) || + vcpu->arch.nmi_injected || + vcpu->arch.interrupt.injected)); + vcpu->arch.exception.has_error_code = has_error; vcpu->arch.exception.vector = nr; vcpu->arch.exception.error_code = error_code; @@ -725,8 +733,28 @@ static void kvm_multiple_exception(struct kvm_vcpu *vcpu, vcpu->arch.exception.injected = false; vcpu->arch.exception.pending = false;

+ /* + * A #DF is NOT a nested event per its definition, however per + * FRED spec 5.0 Appendix B, its delivery determines the new + * stack level as is done for events occurring when CPL = 0. + */ + vcpu->arch.exception.nested = false; + kvm_queue_exception_e(vcpu, DF_VECTOR, 0); } else { + /* + * FRED spec 5.0 Appendix B: delivery of a nested exception + * determines the new stack level as is done for events + * occurring when CPL = 0. + * + * IOW, FRED event delivery of an event encountered in ring 3 + * normally uses stack level 0 unconditionally. However, if + * the event is an exception nested on any earlier event, + * delivery of the nested exception will consult the FRED MSR + * IA32_FRED_STKLVLS to determine which stack level to use. + */ + vcpu->arch.exception.nested = kvm_is_fred_enabled(vcpu); + /* replace previous exception with a new one in a hope that instruction re-execution will regenerate lost exception */ @@ -736,20 +764,20 @@ static void kvm_multiple_exception(struct kvm_vcpu *vcpu,

void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr) { - kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false); + kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false, false); } EXPORT_SYMBOL_GPL(kvm_queue_exception);

-void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr) +void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr, bool nested) { - kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true); + kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true, nested); } EXPORT_SYMBOL_GPL(kvm_requeue_exception);

void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr, unsigned long payload) { - kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false); + kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false, false); } EXPORT_SYMBOL_GPL(kvm_queue_exception_p);

@@ -757,7 +785,7 @@ static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code, unsigned long payload) { kvm_multiple_exception(vcpu, nr, true, error_code, - true, payload, false); + true, payload, false, false); }

int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err) @@ -829,13 +857,13 @@ void kvm_inject_nmi(struct kvm_vcpu *vcpu)

void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) { - kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false); + kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false, false); } EXPORT_SYMBOL_GPL(kvm_queue_exception_e);

-void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) +void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code, bool nested) { - kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true); + kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true, nested); } EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);

diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 9a52016ebf5a..c1f1d5696080 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -108,6 +108,7 @@ static inline void kvm_clear_exception_queue(struct kvm_vcpu *vcpu) { vcpu->arch.exception.pending = false; vcpu->arch.exception.injected = false; + vcpu->arch.exception.nested = false; vcpu->arch.exception_vmexit.pending = false; }

Add FRED related VMCS fields to dump_vmcs() to have it dump FRED context.

Signed-off-by: Xin Li xin3.li@intel.com Tested-by: Shan Kang shan.kang@intel.com ---

Change since v1: * Use kvm_cpu_cap_has() instead of cpu_feature_enabled() (Chao Gao). * Dump guest FRED states only if guest has FRED enabled (Nikolay Borisov). --- arch/x86/kvm/vmx/vmx.c | 46 +++++++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 7 deletions(-)

diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index a484b9ac2400..e3409607122d 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6392,7 +6392,7 @@ void dump_vmcs(struct kvm_vcpu *vcpu) struct vcpu_vmx *vmx = to_vmx(vcpu); u32 vmentry_ctl, vmexit_ctl; u32 cpu_based_exec_ctrl, pin_based_exec_ctrl, secondary_exec_control; - u64 tertiary_exec_control; + u64 tertiary_exec_control, secondary_vmexit_ctl; unsigned long cr4; int efer_slot;

@@ -6403,6 +6403,8 @@ void dump_vmcs(struct kvm_vcpu *vcpu)

vmentry_ctl = vmcs_read32(VM_ENTRY_CONTROLS); vmexit_ctl = vmcs_read32(VM_EXIT_CONTROLS); + secondary_vmexit_ctl = cpu_has_secondary_vmexit_ctrls() ? + vmcs_read64(SECONDARY_VM_EXIT_CONTROLS) : 0; cpu_based_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL); pin_based_exec_ctrl = vmcs_read32(PIN_BASED_VM_EXEC_CONTROL); cr4 = vmcs_readl(GUEST_CR4); @@ -6449,6 +6451,19 @@ void dump_vmcs(struct kvm_vcpu *vcpu) vmx_dump_sel("LDTR:", GUEST_LDTR_SELECTOR); vmx_dump_dtsel("IDTR:", GUEST_IDTR_LIMIT); vmx_dump_sel("TR: ", GUEST_TR_SELECTOR); +#ifdef CONFIG_X86_64 + if (kvm_is_fred_enabled(vcpu)) { + pr_err("FRED guest: config=0x%016llx, stack levels=0x%016llx\n" + "RSP0=0x%016lx, RSP1=0x%016llx\n" + "RSP2=0x%016llx, RSP3=0x%016llx\n", + vmcs_read64(GUEST_IA32_FRED_CONFIG), + vmcs_read64(GUEST_IA32_FRED_STKLVLS), + read_msr(MSR_IA32_FRED_RSP0), + vmcs_read64(GUEST_IA32_FRED_RSP1), + vmcs_read64(GUEST_IA32_FRED_RSP2), + vmcs_read64(GUEST_IA32_FRED_RSP3)); + } +#endif efer_slot = vmx_find_loadstore_msr_slot(&vmx->msr_autoload.guest, MSR_EFER); if (vmentry_ctl & VM_ENTRY_LOAD_IA32_EFER) pr_err("EFER= 0x%016llx\n", vmcs_read64(GUEST_IA32_EFER)); @@ -6496,6 +6511,19 @@ void dump_vmcs(struct kvm_vcpu *vcpu) vmcs_readl(HOST_TR_BASE)); pr_err("GDTBase=%016lx IDTBase=%016lx\n", vmcs_readl(HOST_GDTR_BASE), vmcs_readl(HOST_IDTR_BASE)); +#ifdef CONFIG_X86_64 + if (kvm_cpu_cap_has(X86_FEATURE_FRED)) { + pr_err("FRED host: config=0x%016llx, stack levels=0x%016llx\n" + "RSP0=0x%016llx, RSP1=0x%016llx\n" + "RSP2=0x%016llx, RSP3=0x%016llx\n", + vmcs_read64(HOST_IA32_FRED_CONFIG), + vmcs_read64(HOST_IA32_FRED_STKLVLS), + vmx->msr_host_fred_rsp0, + vmcs_read64(HOST_IA32_FRED_RSP1), + vmcs_read64(HOST_IA32_FRED_RSP2), + vmcs_read64(HOST_IA32_FRED_RSP3)); + } +#endif pr_err("CR0=%016lx CR3=%016lx CR4=%016lx\n", vmcs_readl(HOST_CR0), vmcs_readl(HOST_CR3), vmcs_readl(HOST_CR4)); @@ -6517,25 +6545,29 @@ void dump_vmcs(struct kvm_vcpu *vcpu) pr_err("*** Control State ***\n"); pr_err("CPUBased=0x%08x SecondaryExec=0x%08x TertiaryExec=0x%016llx\n", cpu_based_exec_ctrl, secondary_exec_control, tertiary_exec_control); - pr_err("PinBased=0x%08x EntryControls=%08x ExitControls=%08x\n", - pin_based_exec_ctrl, vmentry_ctl, vmexit_ctl); + pr_err("PinBased=0x%08x EntryControls=0x%08x\n", + pin_based_exec_ctrl, vmentry_ctl); + pr_err("ExitControls=0x%08x SecondaryExitControls=0x%016llx\n", + vmexit_ctl, secondary_vmexit_ctl); pr_err("ExceptionBitmap=%08x PFECmask=%08x PFECmatch=%08x\n", vmcs_read32(EXCEPTION_BITMAP), vmcs_read32(PAGE_FAULT_ERROR_CODE_MASK), vmcs_read32(PAGE_FAULT_ERROR_CODE_MATCH)); - pr_err("VMEntry: intr_info=%08x errcode=%08x ilen=%08x\n", + pr_err("VMEntry: intr_info=%08x errcode=%08x ilen=%08x event data=%016llx\n", vmcs_read32(VM_ENTRY_INTR_INFO_FIELD), vmcs_read32(VM_ENTRY_EXCEPTION_ERROR_CODE), - vmcs_read32(VM_ENTRY_INSTRUCTION_LEN)); + vmcs_read32(VM_ENTRY_INSTRUCTION_LEN), + kvm_cpu_cap_has(X86_FEATURE_FRED) ? vmcs_read64(INJECTED_EVENT_DATA) : 0); pr_err("VMExit: intr_info=%08x errcode=%08x ilen=%08x\n", vmcs_read32(VM_EXIT_INTR_INFO), vmcs_read32(VM_EXIT_INTR_ERROR_CODE), vmcs_read32(VM_EXIT_INSTRUCTION_LEN)); pr_err(" reason=%08x qualification=%016lx\n", vmcs_read32(VM_EXIT_REASON), vmcs_readl(EXIT_QUALIFICATION)); - pr_err("IDTVectoring: info=%08x errcode=%08x\n", + pr_err("IDTVectoring: info=%08x errcode=%08x event data=%016llx\n", vmcs_read32(IDT_VECTORING_INFO_FIELD), - vmcs_read32(IDT_VECTORING_ERROR_CODE)); + vmcs_read32(IDT_VECTORING_ERROR_CODE), + kvm_cpu_cap_has(X86_FEATURE_FRED) ? vmcs_read64(ORIGINAL_EVENT_DATA) : 0); pr_err("TSC Offset = 0x%016llx\n", vmcs_read64(TSC_OFFSET)); if (secondary_exec_control & SECONDARY_EXEC_TSC_SCALING) pr_err("TSC Multiplier = 0x%016llx\n",

Enable the secondary VM exit controls to prepare for nested FRED.

Signed-off-by: Xin Li xin3.li@intel.com Tested-by: Shan Kang shan.kang@intel.com --- Documentation/virt/kvm/x86/nested-vmx.rst | 1 + arch/x86/kvm/vmx/capabilities.h | 1 + arch/x86/kvm/vmx/nested.c | 15 ++++++++++++++- arch/x86/kvm/vmx/vmcs12.c | 1 + arch/x86/kvm/vmx/vmcs12.h | 2 ++ arch/x86/kvm/x86.h | 2 +- 6 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/Documentation/virt/kvm/x86/nested-vmx.rst b/Documentation/virt/kvm/x86/nested-vmx.rst index ac2095d41f02..e64ef231f310 100644 --- a/Documentation/virt/kvm/x86/nested-vmx.rst +++ b/Documentation/virt/kvm/x86/nested-vmx.rst @@ -217,6 +217,7 @@ struct shadow_vmcs is ever changed. u16 host_fs_selector; u16 host_gs_selector; u16 host_tr_selector; + u64 secondary_vm_exit_controls; };

diff --git a/arch/x86/kvm/vmx/capabilities.h b/arch/x86/kvm/vmx/capabilities.h index 73bf6618c425..b41c2cde811d 100644 --- a/arch/x86/kvm/vmx/capabilities.h +++ b/arch/x86/kvm/vmx/capabilities.h @@ -38,6 +38,7 @@ struct nested_vmx_msrs { u32 pinbased_ctls_high; u32 exit_ctls_low; u32 exit_ctls_high; + u64 secondary_exit_ctls; u32 entry_ctls_low; u32 entry_ctls_high; u32 misc_low; diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 8a5fda04e2de..1132e360ff13 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -1431,6 +1431,7 @@ int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) case MSR_IA32_VMX_PINBASED_CTLS: case MSR_IA32_VMX_PROCBASED_CTLS: case MSR_IA32_VMX_EXIT_CTLS: + case MSR_IA32_VMX_EXIT_CTLS2: case MSR_IA32_VMX_ENTRY_CTLS: /* * The "non-true" VMX capability MSRs are generated from the @@ -1509,6 +1510,9 @@ int vmx_get_vmx_msr(struct nested_vmx_msrs *msrs, u32 msr_index, u64 *pdata) if (msr_index == MSR_IA32_VMX_EXIT_CTLS) *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; break; + case MSR_IA32_VMX_EXIT_CTLS2: + *pdata = msrs->secondary_exit_ctls; + break; case MSR_IA32_VMX_TRUE_ENTRY_CTLS: case MSR_IA32_VMX_ENTRY_CTLS: *pdata = vmx_control_msr( @@ -2443,6 +2447,11 @@ static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct loaded_vmcs *vmcs0 exec_control &= ~VM_EXIT_LOAD_IA32_EFER; vm_exit_controls_set(vmx, exec_control);

+ if (exec_control & VM_EXIT_ACTIVATE_SECONDARY_CONTROLS) { + exec_control = __secondary_vm_exit_controls_get(vmcs01); + secondary_vm_exit_controls_set(vmx, exec_control); + } + /* * Interrupt/Exception Fields */ @@ -6856,13 +6865,17 @@ static void nested_vmx_setup_exit_ctls(struct vmcs_config *vmcs_conf, VM_EXIT_HOST_ADDR_SPACE_SIZE | #endif VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT | - VM_EXIT_CLEAR_BNDCFGS; + VM_EXIT_CLEAR_BNDCFGS | VM_EXIT_ACTIVATE_SECONDARY_CONTROLS; msrs->exit_ctls_high |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR | VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER | VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT | VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL;

+ /* secondary exit controls */ + if (msrs->exit_ctls_high & VM_EXIT_ACTIVATE_SECONDARY_CONTROLS) + rdmsrl(MSR_IA32_VMX_EXIT_CTLS2, msrs->secondary_exit_ctls); + /* We support free control of debug control saving. */ msrs->exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS; } diff --git a/arch/x86/kvm/vmx/vmcs12.c b/arch/x86/kvm/vmx/vmcs12.c index 106a72c923ca..98457d7b2b23 100644 --- a/arch/x86/kvm/vmx/vmcs12.c +++ b/arch/x86/kvm/vmx/vmcs12.c @@ -73,6 +73,7 @@ const unsigned short vmcs12_field_offsets[] = { FIELD(PAGE_FAULT_ERROR_CODE_MATCH, page_fault_error_code_match), FIELD(CR3_TARGET_COUNT, cr3_target_count), FIELD(VM_EXIT_CONTROLS, vm_exit_controls), + FIELD(SECONDARY_VM_EXIT_CONTROLS, secondary_vm_exit_controls), FIELD(VM_EXIT_MSR_STORE_COUNT, vm_exit_msr_store_count), FIELD(VM_EXIT_MSR_LOAD_COUNT, vm_exit_msr_load_count), FIELD(VM_ENTRY_CONTROLS, vm_entry_controls), diff --git a/arch/x86/kvm/vmx/vmcs12.h b/arch/x86/kvm/vmx/vmcs12.h index 01936013428b..f50f897b9b5f 100644 --- a/arch/x86/kvm/vmx/vmcs12.h +++ b/arch/x86/kvm/vmx/vmcs12.h @@ -185,6 +185,7 @@ struct __packed vmcs12 { u16 host_gs_selector; u16 host_tr_selector; u16 guest_pml_index; + u64 secondary_vm_exit_controls; };

/* @@ -358,6 +359,7 @@ static inline void vmx_check_vmcs12_offsets(void) CHECK_OFFSET(host_gs_selector, 992); CHECK_OFFSET(host_tr_selector, 994); CHECK_OFFSET(guest_pml_index, 996); + CHECK_OFFSET(secondary_vm_exit_controls, 998); }

extern const unsigned short vmcs12_field_offsets[]; diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index c1f1d5696080..498bb6090b1e 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -47,7 +47,7 @@ void kvm_spurious_fault(void); * associated feature that KVM supports for nested virtualization. */ #define KVM_FIRST_EMULATED_VMX_MSR MSR_IA32_VMX_BASIC -#define KVM_LAST_EMULATED_VMX_MSR MSR_IA32_VMX_VMFUNC +#define KVM_LAST_EMULATED_VMX_MSR MSR_IA32_VMX_EXIT_CTLS2

#define KVM_DEFAULT_PLE_GAP 128 #define KVM_VMX_DEFAULT_PLE_WINDOW 4096

Add FRED VMCS fields to nested VMX context management.

Todo: change VMCS12_REVISION, as struct vmcs12 is changed.

Signed-off-by: Xin Li xin3.li@intel.com Tested-by: Shan Kang shan.kang@intel.com ---

Change since v1: * Remove hyperv TLFS related changes (Jeremi Piotrowski). * Use kvm_cpu_cap_has() instead of cpu_feature_enabled() (Chao Gao). --- Documentation/virt/kvm/x86/nested-vmx.rst | 18 +++++ arch/x86/kvm/vmx/nested.c | 91 +++++++++++++++++++---- arch/x86/kvm/vmx/vmcs12.c | 18 +++++ arch/x86/kvm/vmx/vmcs12.h | 36 +++++++++ arch/x86/kvm/vmx/vmcs_shadow_fields.h | 4 + 5 files changed, 152 insertions(+), 15 deletions(-)

diff --git a/Documentation/virt/kvm/x86/nested-vmx.rst b/Documentation/virt/kvm/x86/nested-vmx.rst index e64ef231f310..87fa9f3877ab 100644 --- a/Documentation/virt/kvm/x86/nested-vmx.rst +++ b/Documentation/virt/kvm/x86/nested-vmx.rst @@ -218,6 +218,24 @@ struct shadow_vmcs is ever changed. u16 host_gs_selector; u16 host_tr_selector; u64 secondary_vm_exit_controls; + u64 guest_ia32_fred_config; + u64 guest_ia32_fred_rsp1; + u64 guest_ia32_fred_rsp2; + u64 guest_ia32_fred_rsp3; + u64 guest_ia32_fred_stklvls; + u64 guest_ia32_fred_ssp1; + u64 guest_ia32_fred_ssp2; + u64 guest_ia32_fred_ssp3; + u64 host_ia32_fred_config; + u64 host_ia32_fred_rsp1; + u64 host_ia32_fred_rsp2; + u64 host_ia32_fred_rsp3; + u64 host_ia32_fred_stklvls; + u64 host_ia32_fred_ssp1; + u64 host_ia32_fred_ssp2; + u64 host_ia32_fred_ssp3; + u64 injected_event_data; + u64 original_event_data; };

diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 94da6a0a2f81..f9c1fbeac302 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -686,6 +686,9 @@ static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu,

nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, MSR_KERNEL_GS_BASE, MSR_TYPE_RW); + + nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, + MSR_IA32_FRED_RSP0, MSR_TYPE_RW); #endif nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, MSR_IA32_SPEC_CTRL, MSR_TYPE_RW); @@ -2498,6 +2501,8 @@ static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct loaded_vmcs *vmcs0 vmcs12->vm_entry_instruction_len); vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, vmcs12->guest_interruptibility_info); + if (kvm_cpu_cap_has(X86_FEATURE_FRED)) + vmcs_write64(INJECTED_EVENT_DATA, vmcs12->injected_event_data); vmx->loaded_vmcs->nmi_known_unmasked = !(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_NMI); } else { @@ -2548,6 +2553,17 @@ static void prepare_vmcs02_rare(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base); vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base);

+ if (kvm_cpu_cap_has(X86_FEATURE_FRED)) { + vmcs_write64(GUEST_IA32_FRED_CONFIG, vmcs12->guest_ia32_fred_config); + vmcs_write64(GUEST_IA32_FRED_RSP1, vmcs12->guest_ia32_fred_rsp1); + vmcs_write64(GUEST_IA32_FRED_RSP2, vmcs12->guest_ia32_fred_rsp2); + vmcs_write64(GUEST_IA32_FRED_RSP3, vmcs12->guest_ia32_fred_rsp3); + vmcs_write64(GUEST_IA32_FRED_STKLVLS, vmcs12->guest_ia32_fred_stklvls); + vmcs_write64(GUEST_IA32_FRED_SSP1, vmcs12->guest_ia32_fred_ssp1); + vmcs_write64(GUEST_IA32_FRED_SSP2, vmcs12->guest_ia32_fred_ssp2); + vmcs_write64(GUEST_IA32_FRED_SSP3, vmcs12->guest_ia32_fred_ssp3); + } + vmx->segment_cache.bitmask = 0; }

@@ -3835,6 +3851,22 @@ vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) vcpu->arch.cr4_guest_owned_bits)); }

+static inline unsigned long +nested_vmx_get_event_data(struct kvm_vcpu *vcpu, bool for_ex_vmexit) +{ + struct kvm_queued_exception *ex = for_ex_vmexit ? + &vcpu->arch.exception_vmexit : &vcpu->arch.exception; + + if (ex->has_payload) + return ex->payload; + else if (ex->vector == PF_VECTOR) + return vcpu->arch.cr2; + else if (ex->vector == DB_VECTOR) + return (vcpu->arch.dr6 & ~DR6_BT) ^ DR6_ACTIVE_LOW; + else + return 0; +} + static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, u32 vm_exit_reason, u32 exit_intr_info) @@ -3842,6 +3874,8 @@ static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu, u32 idt_vectoring; unsigned int nr;

+ vmcs12->original_event_data = 0; + /* * Per the SDM, VM-Exits due to double and triple faults are never * considered to occur during event delivery, even if the double/triple @@ -3880,6 +3914,12 @@ static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu, vcpu->arch.exception.error_code; }

+ idt_vectoring |= vcpu->arch.exception.nested ? + INTR_INFO_NESTED_EXCEPTION_MASK : 0; + + vmcs12->original_event_data = + nested_vmx_get_event_data(vcpu, false); + vmcs12->idt_vectoring_info_field = idt_vectoring; } else if (vcpu->arch.nmi_injected) { vmcs12->idt_vectoring_info_field = @@ -3970,19 +4010,7 @@ static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu) struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit; u32 intr_info = ex->vector | INTR_INFO_VALID_MASK; struct vmcs12 *vmcs12 = get_vmcs12(vcpu); - unsigned long exit_qual; - - if (ex->has_payload) { - exit_qual = ex->payload; - } else if (ex->vector == PF_VECTOR) { - exit_qual = vcpu->arch.cr2; - } else if (ex->vector == DB_VECTOR) { - exit_qual = vcpu->arch.dr6; - exit_qual &= ~DR6_BT; - exit_qual ^= DR6_ACTIVE_LOW; - } else { - exit_qual = 0; - } + unsigned long exit_qual = nested_vmx_get_event_data(vcpu, true);

/* * Unlike AMD's Paged Real Mode, which reports an error code on #PF @@ -4003,10 +4031,12 @@ static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu) intr_info |= INTR_INFO_DELIVER_CODE_MASK; }

- if (kvm_exception_is_soft(ex->vector)) + if (kvm_exception_is_soft(ex->vector)) { intr_info |= INTR_TYPE_SOFT_EXCEPTION; - else + } else { intr_info |= INTR_TYPE_HARD_EXCEPTION; + intr_info |= ex->nested ? INTR_INFO_NESTED_EXCEPTION_MASK : 0; + }

if (!(vmcs12->idt_vectoring_info_field & VECTORING_INFO_VALID_MASK) && vmx_get_nmi_mask(vcpu)) @@ -4352,6 +4382,14 @@ static bool is_vmcs12_ext_field(unsigned long field) case GUEST_TR_BASE: case GUEST_GDTR_BASE: case GUEST_IDTR_BASE: + case GUEST_IA32_FRED_CONFIG: + case GUEST_IA32_FRED_RSP1: + case GUEST_IA32_FRED_RSP2: + case GUEST_IA32_FRED_RSP3: + case GUEST_IA32_FRED_STKLVLS: + case GUEST_IA32_FRED_SSP1: + case GUEST_IA32_FRED_SSP2: + case GUEST_IA32_FRED_SSP3: case GUEST_PENDING_DBG_EXCEPTIONS: case GUEST_BNDCFGS: return true; @@ -4401,6 +4439,18 @@ static void sync_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu, vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE); vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE); vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE); + + if (kvm_cpu_cap_has(X86_FEATURE_FRED)) { + vmcs12->guest_ia32_fred_config = vmcs_read64(GUEST_IA32_FRED_CONFIG); + vmcs12->guest_ia32_fred_rsp1 = vmcs_read64(GUEST_IA32_FRED_RSP1); + vmcs12->guest_ia32_fred_rsp2 = vmcs_read64(GUEST_IA32_FRED_RSP2); + vmcs12->guest_ia32_fred_rsp3 = vmcs_read64(GUEST_IA32_FRED_RSP3); + vmcs12->guest_ia32_fred_stklvls = vmcs_read64(GUEST_IA32_FRED_STKLVLS); + vmcs12->guest_ia32_fred_ssp1 = vmcs_read64(GUEST_IA32_FRED_SSP1); + vmcs12->guest_ia32_fred_ssp2 = vmcs_read64(GUEST_IA32_FRED_SSP2); + vmcs12->guest_ia32_fred_ssp3 = vmcs_read64(GUEST_IA32_FRED_SSP3); + } + vmcs12->guest_pending_dbg_exceptions = vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS);

@@ -4625,6 +4675,17 @@ static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, vmcs_write32(GUEST_IDTR_LIMIT, 0xFFFF); vmcs_write32(GUEST_GDTR_LIMIT, 0xFFFF);

+ if (kvm_cpu_cap_has(X86_FEATURE_FRED)) { + vmcs_write64(GUEST_IA32_FRED_CONFIG, vmcs12->host_ia32_fred_config); + vmcs_write64(GUEST_IA32_FRED_RSP1, vmcs12->host_ia32_fred_rsp1); + vmcs_write64(GUEST_IA32_FRED_RSP2, vmcs12->host_ia32_fred_rsp2); + vmcs_write64(GUEST_IA32_FRED_RSP3, vmcs12->host_ia32_fred_rsp3); + vmcs_write64(GUEST_IA32_FRED_STKLVLS, vmcs12->host_ia32_fred_stklvls); + vmcs_write64(GUEST_IA32_FRED_SSP1, vmcs12->host_ia32_fred_ssp1); + vmcs_write64(GUEST_IA32_FRED_SSP2, vmcs12->host_ia32_fred_ssp2); + vmcs_write64(GUEST_IA32_FRED_SSP3, vmcs12->host_ia32_fred_ssp3); + } + /* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1. */ if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS) vmcs_write64(GUEST_BNDCFGS, 0); diff --git a/arch/x86/kvm/vmx/vmcs12.c b/arch/x86/kvm/vmx/vmcs12.c index 98457d7b2b23..59f17fdfad11 100644 --- a/arch/x86/kvm/vmx/vmcs12.c +++ b/arch/x86/kvm/vmx/vmcs12.c @@ -80,6 +80,7 @@ const unsigned short vmcs12_field_offsets[] = { FIELD(VM_ENTRY_MSR_LOAD_COUNT, vm_entry_msr_load_count), FIELD(VM_ENTRY_INTR_INFO_FIELD, vm_entry_intr_info_field), FIELD(VM_ENTRY_EXCEPTION_ERROR_CODE, vm_entry_exception_error_code), + FIELD(INJECTED_EVENT_DATA, injected_event_data), FIELD(VM_ENTRY_INSTRUCTION_LEN, vm_entry_instruction_len), FIELD(TPR_THRESHOLD, tpr_threshold), FIELD(SECONDARY_VM_EXEC_CONTROL, secondary_vm_exec_control), @@ -89,6 +90,7 @@ const unsigned short vmcs12_field_offsets[] = { FIELD(VM_EXIT_INTR_ERROR_CODE, vm_exit_intr_error_code), FIELD(IDT_VECTORING_INFO_FIELD, idt_vectoring_info_field), FIELD(IDT_VECTORING_ERROR_CODE, idt_vectoring_error_code), + FIELD(ORIGINAL_EVENT_DATA, original_event_data), FIELD(VM_EXIT_INSTRUCTION_LEN, vm_exit_instruction_len), FIELD(VMX_INSTRUCTION_INFO, vmx_instruction_info), FIELD(GUEST_ES_LIMIT, guest_es_limit), @@ -152,5 +154,21 @@ const unsigned short vmcs12_field_offsets[] = { FIELD(HOST_IA32_SYSENTER_EIP, host_ia32_sysenter_eip), FIELD(HOST_RSP, host_rsp), FIELD(HOST_RIP, host_rip), + FIELD(GUEST_IA32_FRED_CONFIG, guest_ia32_fred_config), + FIELD(GUEST_IA32_FRED_RSP1, guest_ia32_fred_rsp1), + FIELD(GUEST_IA32_FRED_RSP2, guest_ia32_fred_rsp2), + FIELD(GUEST_IA32_FRED_RSP3, guest_ia32_fred_rsp3), + FIELD(GUEST_IA32_FRED_STKLVLS, guest_ia32_fred_stklvls), + FIELD(GUEST_IA32_FRED_SSP1, guest_ia32_fred_ssp1), + FIELD(GUEST_IA32_FRED_SSP2, guest_ia32_fred_ssp2), + FIELD(GUEST_IA32_FRED_SSP3, guest_ia32_fred_ssp3), + FIELD(HOST_IA32_FRED_CONFIG, host_ia32_fred_config), + FIELD(HOST_IA32_FRED_RSP1, host_ia32_fred_rsp1), + FIELD(HOST_IA32_FRED_RSP2, host_ia32_fred_rsp2), + FIELD(HOST_IA32_FRED_RSP3, host_ia32_fred_rsp3), + FIELD(HOST_IA32_FRED_STKLVLS, host_ia32_fred_stklvls), + FIELD(HOST_IA32_FRED_SSP1, host_ia32_fred_ssp1), + FIELD(HOST_IA32_FRED_SSP2, host_ia32_fred_ssp2), + FIELD(HOST_IA32_FRED_SSP3, host_ia32_fred_ssp3), }; const unsigned int nr_vmcs12_fields = ARRAY_SIZE(vmcs12_field_offsets); diff --git a/arch/x86/kvm/vmx/vmcs12.h b/arch/x86/kvm/vmx/vmcs12.h index f50f897b9b5f..edf7fcef8ccf 100644 --- a/arch/x86/kvm/vmx/vmcs12.h +++ b/arch/x86/kvm/vmx/vmcs12.h @@ -186,6 +186,24 @@ struct __packed vmcs12 { u16 host_tr_selector; u16 guest_pml_index; u64 secondary_vm_exit_controls; + u64 guest_ia32_fred_config; + u64 guest_ia32_fred_rsp1; + u64 guest_ia32_fred_rsp2; + u64 guest_ia32_fred_rsp3; + u64 guest_ia32_fred_stklvls; + u64 guest_ia32_fred_ssp1; + u64 guest_ia32_fred_ssp2; + u64 guest_ia32_fred_ssp3; + u64 host_ia32_fred_config; + u64 host_ia32_fred_rsp1; + u64 host_ia32_fred_rsp2; + u64 host_ia32_fred_rsp3; + u64 host_ia32_fred_stklvls; + u64 host_ia32_fred_ssp1; + u64 host_ia32_fred_ssp2; + u64 host_ia32_fred_ssp3; + u64 injected_event_data; + u64 original_event_data; };

/* @@ -360,6 +378,24 @@ static inline void vmx_check_vmcs12_offsets(void) CHECK_OFFSET(host_tr_selector, 994); CHECK_OFFSET(guest_pml_index, 996); CHECK_OFFSET(secondary_vm_exit_controls, 998); + CHECK_OFFSET(guest_ia32_fred_config, 1006); + CHECK_OFFSET(guest_ia32_fred_rsp1, 1014); + CHECK_OFFSET(guest_ia32_fred_rsp2, 1022); + CHECK_OFFSET(guest_ia32_fred_rsp3, 1030); + CHECK_OFFSET(guest_ia32_fred_stklvls, 1038); + CHECK_OFFSET(guest_ia32_fred_ssp1, 1046); + CHECK_OFFSET(guest_ia32_fred_ssp2, 1054); + CHECK_OFFSET(guest_ia32_fred_ssp3, 1062); + CHECK_OFFSET(host_ia32_fred_config, 1070); + CHECK_OFFSET(host_ia32_fred_rsp1, 1078); + CHECK_OFFSET(host_ia32_fred_rsp2, 1086); + CHECK_OFFSET(host_ia32_fred_rsp3, 1094); + CHECK_OFFSET(host_ia32_fred_stklvls, 1102); + CHECK_OFFSET(host_ia32_fred_ssp1, 1110); + CHECK_OFFSET(host_ia32_fred_ssp2, 1118); + CHECK_OFFSET(host_ia32_fred_ssp3, 1126); + CHECK_OFFSET(injected_event_data, 1134); + CHECK_OFFSET(original_event_data, 1142); }

extern const unsigned short vmcs12_field_offsets[]; diff --git a/arch/x86/kvm/vmx/vmcs_shadow_fields.h b/arch/x86/kvm/vmx/vmcs_shadow_fields.h index 7f48056fe351..3885a3e0fbe8 100644 --- a/arch/x86/kvm/vmx/vmcs_shadow_fields.h +++ b/arch/x86/kvm/vmx/vmcs_shadow_fields.h @@ -74,6 +74,10 @@ SHADOW_FIELD_RW(HOST_GS_BASE, host_gs_base, true) /* 64-bit */ SHADOW_FIELD_RO(GUEST_PHYSICAL_ADDRESS, guest_physical_address, true) SHADOW_FIELD_RO(GUEST_PHYSICAL_ADDRESS_HIGH, guest_physical_address, true) +SHADOW_FIELD_RO(ORIGINAL_EVENT_DATA, original_event_data, kvm_cpu_cap_has(X86_FEATURE_FRED)) +SHADOW_FIELD_RO(ORIGINAL_EVENT_DATA_HIGH, original_event_data, kvm_cpu_cap_has(X86_FEATURE_FRED)) +SHADOW_FIELD_RW(INJECTED_EVENT_DATA, injected_event_data, kvm_cpu_cap_has(X86_FEATURE_FRED)) +SHADOW_FIELD_RW(INJECTED_EVENT_DATA_HIGH, injected_event_data, kvm_cpu_cap_has(X86_FEATURE_FRED))

#undef SHADOW_FIELD_RO #undef SHADOW_FIELD_RW

Add FRED related VMCS fields checkings.

As real hardware, nested VMX performs checks on various VMCS fields, including both controls and guest/host states. With the introduction of VMX FRED, add FRED related VMCS fields checkings.

Signed-off-by: Xin Li xin3.li@intel.com Tested-by: Shan Kang shan.kang@intel.com --- arch/x86/kvm/vmx/nested.c | 80 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-)

diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 04a9cdb0361f..ef0bd46eb0ce 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -2933,6 +2933,8 @@ static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { struct vcpu_vmx *vmx = to_vmx(vcpu); + bool fred_enabled = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) && + (vmcs12->guest_cr4 & X86_CR4_FRED);

if (CC(!vmx_control_verify(vmcs12->vm_entry_controls, vmx->nested.msrs.entry_ctls_low, @@ -2951,6 +2953,7 @@ static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, u32 intr_type = intr_info & INTR_INFO_INTR_TYPE_MASK; bool has_error_code = intr_info & INTR_INFO_DELIVER_CODE_MASK; bool should_have_error_code; + bool has_nested_exception = vmx->nested.msrs.basic & VMX_BASIC_NESTED_EXCEPTION; bool urg = nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST); bool prot_mode = !urg || vmcs12->guest_cr0 & X86_CR0_PE; @@ -2964,7 +2967,9 @@ static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, /* VM-entry interruption-info field: vector */ if (CC(intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) || CC(intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) || - CC(intr_type == INTR_TYPE_OTHER_EVENT && vector != 0)) + CC(intr_type == INTR_TYPE_OTHER_EVENT && + ((!fred_enabled && vector > 0) || + (fred_enabled && vector > 2)))) return -EINVAL;

/* VM-entry interruption-info field: deliver error code */ @@ -2983,6 +2988,15 @@ static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, if (CC(intr_info & INTR_INFO_RESVD_BITS_MASK)) return -EINVAL;

+ /* + * When the CPU enumerates VMX nested-exception support, bit 13 + * (set to indicate a nested exception) of the intr info field + * may have value 1. Otherwise bit 13 is reserved. + */ + if (CC(!has_nested_exception && + (intr_info & INTR_INFO_NESTED_EXCEPTION_MASK))) + return -EINVAL; + /* VM-entry instruction length */ switch (intr_type) { case INTR_TYPE_SOFT_EXCEPTION: @@ -2992,6 +3006,12 @@ static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, CC(vmcs12->vm_entry_instruction_len == 0 && CC(!nested_cpu_has_zero_length_injection(vcpu)))) return -EINVAL; + break; + case INTR_TYPE_OTHER_EVENT: + if (fred_enabled && (vector == 1 || vector == 2)) + if (CC(vmcs12->vm_entry_instruction_len > 15)) + return -EINVAL; + break; } }

@@ -3054,9 +3074,30 @@ static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu, if (ia32e) { if (CC(!(vmcs12->host_cr4 & X86_CR4_PAE))) return -EINVAL; + if (vmcs12->vm_exit_controls & VM_EXIT_ACTIVATE_SECONDARY_CONTROLS && + vmcs12->secondary_vm_exit_controls & SECONDARY_VM_EXIT_LOAD_IA32_FRED) { + /* Bit 11, bits 5:4, and bit 2 of the IA32_FRED_CONFIG must be zero */ + if (CC(vmcs12->host_ia32_fred_config & + (BIT_ULL(11) | GENMASK_ULL(5, 4) | BIT_ULL(2))) || + CC(vmcs12->host_ia32_fred_rsp1 & GENMASK_ULL(5, 0)) || + CC(vmcs12->host_ia32_fred_rsp2 & GENMASK_ULL(5, 0)) || + CC(vmcs12->host_ia32_fred_rsp3 & GENMASK_ULL(5, 0)) || + CC(vmcs12->host_ia32_fred_ssp1 & GENMASK_ULL(2, 0)) || + CC(vmcs12->host_ia32_fred_ssp2 & GENMASK_ULL(2, 0)) || + CC(vmcs12->host_ia32_fred_ssp3 & GENMASK_ULL(2, 0)) || + CC(is_noncanonical_address(vmcs12->host_ia32_fred_config & PAGE_MASK, vcpu)) || + CC(is_noncanonical_address(vmcs12->host_ia32_fred_rsp1, vcpu)) || + CC(is_noncanonical_address(vmcs12->host_ia32_fred_rsp2, vcpu)) || + CC(is_noncanonical_address(vmcs12->host_ia32_fred_rsp3, vcpu)) || + CC(is_noncanonical_address(vmcs12->host_ia32_fred_ssp1, vcpu)) || + CC(is_noncanonical_address(vmcs12->host_ia32_fred_ssp2, vcpu)) || + CC(is_noncanonical_address(vmcs12->host_ia32_fred_ssp3, vcpu))) + return -EINVAL; + } } else { if (CC(vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) || CC(vmcs12->host_cr4 & X86_CR4_PCIDE) || + CC(vmcs12->host_cr4 & X86_CR4_FRED) || CC((vmcs12->host_rip) >> 32)) return -EINVAL; } @@ -3200,6 +3241,43 @@ static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu, CC((vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD)))) return -EINVAL;

+ if (ia32e) { + if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_FRED) { + /* Bit 11, bits 5:4, and bit 2 of the IA32_FRED_CONFIG must be zero */ + if (CC(vmcs12->guest_ia32_fred_config & + (BIT_ULL(11) | GENMASK_ULL(5, 4) | BIT_ULL(2))) || + CC(vmcs12->guest_ia32_fred_rsp1 & GENMASK_ULL(5, 0)) || + CC(vmcs12->guest_ia32_fred_rsp2 & GENMASK_ULL(5, 0)) || + CC(vmcs12->guest_ia32_fred_rsp3 & GENMASK_ULL(5, 0)) || + CC(vmcs12->guest_ia32_fred_ssp1 & GENMASK_ULL(2, 0)) || + CC(vmcs12->guest_ia32_fred_ssp2 & GENMASK_ULL(2, 0)) || + CC(vmcs12->guest_ia32_fred_ssp3 & GENMASK_ULL(2, 0)) || + CC(is_noncanonical_address(vmcs12->guest_ia32_fred_config & PAGE_MASK, vcpu)) || + CC(is_noncanonical_address(vmcs12->guest_ia32_fred_rsp1, vcpu)) || + CC(is_noncanonical_address(vmcs12->guest_ia32_fred_rsp2, vcpu)) || + CC(is_noncanonical_address(vmcs12->guest_ia32_fred_rsp3, vcpu)) || + CC(is_noncanonical_address(vmcs12->guest_ia32_fred_ssp1, vcpu)) || + CC(is_noncanonical_address(vmcs12->guest_ia32_fred_ssp2, vcpu)) || + CC(is_noncanonical_address(vmcs12->guest_ia32_fred_ssp3, vcpu))) + return -EINVAL; + } + if (vmcs12->guest_cr4 & X86_CR4_FRED) { + unsigned int ss_dpl = VMX_AR_DPL(vmcs12->guest_ss_ar_bytes); + if (CC(ss_dpl == 1 || ss_dpl == 2)) + return -EINVAL; + if (ss_dpl == 0 && + CC(!(vmcs12->guest_cs_ar_bytes & VMX_AR_L_MASK))) + return -EINVAL; + if (ss_dpl == 3 && + (CC(vmcs12->guest_rflags & X86_EFLAGS_IOPL) || + CC(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_STI))) + return -EINVAL; + } + } else { + if (CC(vmcs12->guest_cr4 & X86_CR4_FRED)) + return -EINVAL; + } + if (nested_check_guest_non_reg_state(vmcs12)) return -EINVAL;

Run another round of debug_regs test with FRED enabled if FRED is available.

Signed-off-by: Xin Li xin3.li@intel.com --- .../selftests/kvm/include/x86_64/processor.h | 4 ++ .../testing/selftests/kvm/x86_64/debug_regs.c | 50 ++++++++++++++----- 2 files changed, 41 insertions(+), 13 deletions(-)

diff --git a/tools/testing/selftests/kvm/include/x86_64/processor.h b/tools/testing/selftests/kvm/include/x86_64/processor.h index a84863503fcb..bc5cd8628a20 100644 --- a/tools/testing/selftests/kvm/include/x86_64/processor.h +++ b/tools/testing/selftests/kvm/include/x86_64/processor.h @@ -48,6 +48,7 @@ extern bool host_cpu_is_amd; #define X86_CR4_SMEP (1ul << 20) #define X86_CR4_SMAP (1ul << 21) #define X86_CR4_PKE (1ul << 22) +#define X86_CR4_FRED (1ul << 32)

struct xstate_header { u64 xstate_bv; @@ -164,6 +165,9 @@ struct kvm_x86_cpu_feature { #define X86_FEATURE_SPEC_CTRL KVM_X86_CPU_FEATURE(0x7, 0, EDX, 26) #define X86_FEATURE_ARCH_CAPABILITIES KVM_X86_CPU_FEATURE(0x7, 0, EDX, 29) #define X86_FEATURE_PKS KVM_X86_CPU_FEATURE(0x7, 0, ECX, 31) +#define X86_FEATURE_FRED KVM_X86_CPU_FEATURE(0x7, 1, EAX, 17) +#define X86_FEATURE_LKGS KVM_X86_CPU_FEATURE(0x7, 1, EAX, 18) +#define X86_FEATURE_WRMSRNS KVM_X86_CPU_FEATURE(0x7, 1, EAX, 19) #define X86_FEATURE_XTILECFG KVM_X86_CPU_FEATURE(0xD, 0, EAX, 17) #define X86_FEATURE_XTILEDATA KVM_X86_CPU_FEATURE(0xD, 0, EAX, 18) #define X86_FEATURE_XSAVES KVM_X86_CPU_FEATURE(0xD, 1, EAX, 3) diff --git a/tools/testing/selftests/kvm/x86_64/debug_regs.c b/tools/testing/selftests/kvm/x86_64/debug_regs.c index f6b295e0b2d2..69055e764f15 100644 --- a/tools/testing/selftests/kvm/x86_64/debug_regs.c +++ b/tools/testing/selftests/kvm/x86_64/debug_regs.c @@ -20,7 +20,7 @@ uint32_t guest_value;

extern unsigned char sw_bp, hw_bp, write_data, ss_start, bd_start;

-static void guest_code(void) +static void guest_test_code(void) { /* Create a pending interrupt on current vCPU */ x2apic_enable(); @@ -61,6 +61,15 @@ static void guest_code(void)

/* DR6.BD test */ asm volatile("bd_start: mov %%dr0, %%rax" : : : "rax"); +} + +static void guest_code(void) +{ + guest_test_code(); + + if (get_cr4() & X86_CR4_FRED) + guest_test_code(); + GUEST_DONE(); }

@@ -75,19 +84,15 @@ static void vcpu_skip_insn(struct kvm_vcpu *vcpu, int insn_len) vcpu_regs_set(vcpu, &regs); }

-int main(void) +void run_test(struct kvm_vcpu *vcpu) { struct kvm_guest_debug debug; + struct kvm_run *run = vcpu->run; unsigned long long target_dr6, target_rip; - struct kvm_vcpu *vcpu; - struct kvm_run *run; - struct kvm_vm *vm; - struct ucall uc; - uint64_t cmd; int i; /* Instruction lengths starting at ss_start */ int ss_size[6] = { - 1, /* sti*/ + 1, /* sti */ 2, /* xor */ 2, /* cpuid */ 5, /* mov */ @@ -95,11 +100,6 @@ int main(void) 1, /* cli */ };

- TEST_REQUIRE(kvm_has_cap(KVM_CAP_SET_GUEST_DEBUG)); - - vm = vm_create_with_one_vcpu(&vcpu, guest_code); - run = vcpu->run; - /* Test software BPs - int3 */ memset(&debug, 0, sizeof(debug)); debug.control = KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP; @@ -202,6 +202,30 @@ int main(void) /* Disable all debug controls, run to the end */ memset(&debug, 0, sizeof(debug)); vcpu_guest_debug_set(vcpu, &debug); +} + +int main(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct ucall uc; + uint64_t cmd; + + TEST_REQUIRE(kvm_has_cap(KVM_CAP_SET_GUEST_DEBUG)); + + vm = vm_create_with_one_vcpu(&vcpu, guest_code); + + run_test(vcpu); + + if (kvm_cpu_has(X86_FEATURE_FRED)) { + struct kvm_sregs sregs; + + vcpu_sregs_get(vcpu, &sregs); + sregs.cr4 |= X86_CR4_FRED; + vcpu_sregs_set(vcpu, &sregs); + + run_test(vcpu); + }

vcpu_run(vcpu); TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);

Add tests for FRED event data and VMX nested-exception.

FRED is designed to save a complete event context in its stack frame, e.g., FRED saves the faulting linear address of a #PF into a 64-bit event data field defined in FRED stack frame. As such, FRED VMX adds event data handling during VMX transitions.

Besides, FRED introduces event stack levels to dispatch an event handler onto a stack baesd on current stack level and stack levels defined in IA32_FRED_STKLVLS MSR for each exception vector. VMX nested-exception support ensures a correct event stack level is chosen when a VM entry injects a nested exception, which is regarded as occurred in ring 0.

To fully test the underlying FRED VMX code, this test should be run one more round with EPT disabled to inject page faults as nested exceptions.

Originally-by: Shan Kang shan.kang@intel.com Signed-off-by: Xin Li xin3.li@intel.com --- tools/testing/selftests/kvm/Makefile | 1 + .../selftests/kvm/include/x86_64/processor.h | 32 ++ .../testing/selftests/kvm/x86_64/fred_test.c | 297 ++++++++++++++++++ 3 files changed, 330 insertions(+) create mode 100644 tools/testing/selftests/kvm/x86_64/fred_test.c

diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index 492e937fab00..eaac13a605f2 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -67,6 +67,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/get_msr_index_features TEST_GEN_PROGS_x86_64 += x86_64/exit_on_emulation_failure_test TEST_GEN_PROGS_x86_64 += x86_64/fix_hypercall_test TEST_GEN_PROGS_x86_64 += x86_64/hwcr_msr_test +TEST_GEN_PROGS_x86_64 += x86_64/fred_test TEST_GEN_PROGS_x86_64 += x86_64/hyperv_clock TEST_GEN_PROGS_x86_64 += x86_64/hyperv_cpuid TEST_GEN_PROGS_x86_64 += x86_64/hyperv_evmcs diff --git a/tools/testing/selftests/kvm/include/x86_64/processor.h b/tools/testing/selftests/kvm/include/x86_64/processor.h index bc5cd8628a20..ef7aaab790e0 100644 --- a/tools/testing/selftests/kvm/include/x86_64/processor.h +++ b/tools/testing/selftests/kvm/include/x86_64/processor.h @@ -1275,4 +1275,36 @@ void virt_map_level(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, #define PFERR_GUEST_PAGE_MASK BIT_ULL(PFERR_GUEST_PAGE_BIT) #define PFERR_IMPLICIT_ACCESS BIT_ULL(PFERR_IMPLICIT_ACCESS_BIT)

+/* + * FRED related data structures and functions + */ + +#define FRED_SSX_NMI BIT_ULL(18) + +struct fred_stack { + u64 r15; + u64 r14; + u64 r13; + u64 r12; + u64 bp; + u64 bx; + u64 r11; + u64 r10; + u64 r9; + u64 r8; + u64 ax; + u64 cx; + u64 dx; + u64 si; + u64 di; + u64 error_code; + u64 ip; + u64 csx; + u64 flags; + u64 sp; + u64 ssx; + u64 event_data; + u64 reserved; +}; + #endif /* SELFTEST_KVM_PROCESSOR_H */ diff --git a/tools/testing/selftests/kvm/x86_64/fred_test.c b/tools/testing/selftests/kvm/x86_64/fred_test.c new file mode 100644 index 000000000000..412afa919568 --- /dev/null +++ b/tools/testing/selftests/kvm/x86_64/fred_test.c @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * FRED nested exception tests + * + * Copyright (C) 2023, Intel, Inc. + */ +#define _GNU_SOURCE /* for program_invocation_short_name */ +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/ioctl.h> +#include <asm/msr-index.h> + +#include "apic.h" +#include "kvm_util.h" +#include "test_util.h" +#include "guest_modes.h" +#include "processor.h" + +#define IRQ_VECTOR 0xAA + +#define FRED_STKLVL(v,l) (_AT(unsigned long, l) << (2 * (v))) +#define FRED_CONFIG_ENTRYPOINT(p) _AT(unsigned long, (p)) + +/* This address is already mapped in guest page table. */ +#define FRED_VALID_RSP 0x8000 + +/* + * The following addresses are not yet mapped in both EPT and guest page + * tables at the beginning. As a result, it causes an EPT violation VM + * exit with an original guest #PF to access any of them for the first + * time. + * + * Use these addresses as guest FRED RSP0 to generate nested #PFs to test + * if event data are properly virtualized. + */ +static unsigned long fred_invalid_rsp[4] = { + 0x0, + 0xf0000000, + 0xe0000000, + 0xd0000000, +}; + +extern char asm_user_nop[]; +extern char asm_user_ud[]; +extern char asm_done_fault[]; + +extern void asm_test_fault(int test); + +/* + * user level code for triggering faults. + */ +asm(".pushsection .text\n" + ".align 4096\n" + + ".type asm_user_nop, @function\n" + "asm_user_nop:\n" + "1: .byte 0x90\n" + "jmp 1b\n" + + ".fill asm_user_ud - ., 1, 0xcc\n" + + ".type asm_user_ud, @function\n" + ".org asm_user_nop + 16\n" + "asm_user_ud:\n" + /* Trigger a #UD */ + "ud2\n" + + ".align 4096, 0xcc\n" + ".popsection"); + +/* Send current stack level and #PF address */ +#define GUEST_SYNC_CSL_FA(__stage, __pf_address) \ + GUEST_SYNC_ARGS(__stage, __pf_address, 0, 0, 0) + +void fred_entry_from_user(struct fred_stack *stack) +{ + u32 current_stack_level = rdmsr(MSR_IA32_FRED_CONFIG) & 0x3; + + GUEST_SYNC_CSL_FA(current_stack_level, stack->event_data); + + /* Do NOT go back to user level, continue the next test instead */ + stack->ssx = 0x18; + stack->csx = 0x10; + stack->ip = (u64)&asm_done_fault; +} + +void fred_entry_from_kernel(struct fred_stack *stack) +{ + /* + * Keep NMI blocked to delay the delivery of the next NMI until + * returning to user level. + * */ + stack->ssx &= ~FRED_SSX_NMI; +} + +#define PUSH_REGS \ + "push %rdi\n" \ + "push %rsi\n" \ + "push %rdx\n" \ + "push %rcx\n" \ + "push %rax\n" \ + "push %r8\n" \ + "push %r9\n" \ + "push %r10\n" \ + "push %r11\n" \ + "push %rbx\n" \ + "push %rbp\n" \ + "push %r12\n" \ + "push %r13\n" \ + "push %r14\n" \ + "push %r15\n" + +#define POP_REGS \ + "pop %r15\n" \ + "pop %r14\n" \ + "pop %r13\n" \ + "pop %r12\n" \ + "pop %rbp\n" \ + "pop %rbx\n" \ + "pop %r11\n" \ + "pop %r10\n" \ + "pop %r9\n" \ + "pop %r8\n" \ + "pop %rax\n" \ + "pop %rcx\n" \ + "pop %rdx\n" \ + "pop %rsi\n" \ + "pop %rdi\n" + +/* + * FRED entry points. + */ +asm(".pushsection .text\n" + ".type asm_fred_entrypoint_user, @function\n" + ".align 4096\n" + "asm_fred_entrypoint_user:\n" + "endbr64\n" + PUSH_REGS + "movq %rsp, %rdi\n" + "call fred_entry_from_user\n" + POP_REGS + /* Do NOT go back to user level, continue the next test instead */ + ".byte 0xf2,0x0f,0x01,0xca\n" /* ERETS */ + + ".fill asm_fred_entrypoint_kernel - ., 1, 0xcc\n" + + ".type asm_fred_entrypoint_kernel, @function\n" + ".org asm_fred_entrypoint_user + 256\n" + "asm_fred_entrypoint_kernel:\n" + "endbr64\n" + PUSH_REGS + "movq %rsp, %rdi\n" + "call fred_entry_from_kernel\n" + POP_REGS + ".byte 0xf2,0x0f,0x01,0xca\n" /* ERETS */ + ".align 4096, 0xcc\n" + ".popsection"); + +extern char asm_fred_entrypoint_user[]; + +/* + * Prepare a FRED stack frame for ERETU to return to user level code, + * nop or ud2. + * + * Because FRED RSP0 is deliberately not mapped in guest page table, + * the delivery of interrupt/NMI or #UD from ring 3 causes a nested + * #PF, which is then delivered on FRED RSPx (x is 1, 2 or 3, + * determinated by MSR FRED_STKLVL[PF_VECTOR]). + */ +asm(".pushsection .text\n" + ".type asm_test_fault, @function\n" + ".align 4096\n" + "asm_test_fault:\n" + "endbr64\n" + "push %rbp\n" + "mov %rsp, %rbp\n" + "and $(~0x3f), %rsp\n" + "push $0\n" + "push $0\n" + "mov $0x2b, %rax\n" + /* Unblock NMI */ + "bts $18, %rax\n" + /* Set long mode bit */ + "bts $57, %rax\n" + "push %rax\n" + /* No stack required for the FRED user level test code */ + "push $0\n" + "pushf\n" + "pop %rax\n" + /* Allow external interrupts */ + "bts $9, %rax\n" + "push %rax\n" + "mov $0x33, %rax\n" + "push %rax\n" + "cmp $0, %edi\n" + "jne 1f\n" + "lea asm_user_nop(%rip), %rax\n" + "jmp 2f\n" + "1: lea asm_user_ud(%rip), %rax\n" + "2: push %rax\n" + "push $0\n" + /* ERETU to user level code to allow event delivery immediately */ + ".byte 0xf3,0x0f,0x01,0xca\n" + "asm_done_fault:\n" + "mov %rbp, %rsp\n" + "pop %rbp\n" + "ret\n" + ".align 4096, 0xcc\n" + ".popsection"); + +/* + * To fully test the underlying FRED VMX code, this test should be run one + * more round with EPT disabled to inject page faults as nested exceptions. + */ +static void guest_code(void) +{ + wrmsr(MSR_IA32_FRED_CONFIG, + FRED_CONFIG_ENTRYPOINT(asm_fred_entrypoint_user)); + + wrmsr(MSR_IA32_FRED_RSP1, FRED_VALID_RSP); + wrmsr(MSR_IA32_FRED_RSP2, FRED_VALID_RSP); + wrmsr(MSR_IA32_FRED_RSP3, FRED_VALID_RSP); + + /* Enable FRED */ + set_cr4(get_cr4() | X86_CR4_FRED); + + x2apic_enable(); + + wrmsr(MSR_IA32_FRED_STKLVLS, FRED_STKLVL(PF_VECTOR, 1)); + wrmsr(MSR_IA32_FRED_RSP0, fred_invalid_rsp[1]); + /* 1: ud2 to generate #UD */ + asm_test_fault(1); + + wrmsr(MSR_IA32_FRED_STKLVLS, FRED_STKLVL(PF_VECTOR, 2)); + wrmsr(MSR_IA32_FRED_RSP0, fred_invalid_rsp[2]); + asm volatile("cli"); + /* Create a pending interrupt on current vCPU */ + x2apic_write_reg(APIC_ICR, APIC_DEST_SELF | APIC_INT_ASSERT | + APIC_DM_FIXED | IRQ_VECTOR); + /* Return to ring 3 */ + asm_test_fault(0); + x2apic_write_reg(APIC_EOI, 0); + + wrmsr(MSR_IA32_FRED_STKLVLS, FRED_STKLVL(PF_VECTOR, 3)); + wrmsr(MSR_IA32_FRED_RSP0, fred_invalid_rsp[3]); + /* + * The first NMI is just to have NMI blocked in ring 0, because + * fred_entry_from_kernel() deliberately clears the NMI bit in + * FRED stack frame. + */ + x2apic_write_reg(APIC_ICR, APIC_DEST_SELF | APIC_INT_ASSERT | + APIC_DM_NMI | NMI_VECTOR); + /* The second NMI will be delivered after returning to ring 3 */ + x2apic_write_reg(APIC_ICR, APIC_DEST_SELF | APIC_INT_ASSERT | + APIC_DM_NMI | NMI_VECTOR); + /* Return to ring 3 */ + asm_test_fault(0); + + GUEST_DONE(); +} + +int main(int argc, char *argv[]) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct ucall uc; + uint64_t expected_current_stack_level = 1; + + TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_FRED)); + + vm = __vm_create_with_vcpus(VM_SHAPE(VM_MODE_PXXV48_4K_USER), 1, 0, + guest_code, &vcpu); + + while (true) { + uint64_t r; + + vcpu_run(vcpu); + + r = get_ucall(vcpu, &uc); + + if (r == UCALL_DONE) + break; + + if (r == UCALL_SYNC) { + TEST_ASSERT((uc.args[1] == expected_current_stack_level) && + (uc.args[2] == fred_invalid_rsp[expected_current_stack_level] - 1), + "Incorrect stack level %lx and #PF address %lx\n", + uc.args[1], uc.args[2]); + expected_current_stack_level++; + } + } + + kvm_vm_free(vm); + return 0; +}

On 2/7/24 10:26 PM, Xin Li wrote:

Add tests for FRED event data and VMX nested-exception.

FRED is designed to save a complete event context in its stack frame, e.g., FRED saves the faulting linear address of a #PF into a 64-bit event data field defined in FRED stack frame. As such, FRED VMX adds event data handling during VMX transitions.

Besides, FRED introduces event stack levels to dispatch an event handler onto a stack baesd on current stack level and stack levels defined in IA32_FRED_STKLVLS MSR for each exception vector. VMX nested-exception support ensures a correct event stack level is chosen when a VM entry injects a nested exception, which is regarded as occurred in ring 0.

To fully test the underlying FRED VMX code, this test should be run one more round with EPT disabled to inject page faults as nested exceptions.

Originally-by: Shan Kang shan.kang@intel.com Signed-off-by: Xin Li xin3.li@intel.com

Thank you for the new test patch. We have been trying to ensure TAP conformance for tests which cannot be achieved if new tests aren't using TAP already. Please make your test TAP compliant.

---

tools/testing/selftests/kvm/Makefile | 1 + .../selftests/kvm/include/x86_64/processor.h | 32 ++ .../testing/selftests/kvm/x86_64/fred_test.c | 297 ++++++++++++++++++

Add generated binary object to .gitignore.

3 files changed, 330 insertions(+) create mode 100644 tools/testing/selftests/kvm/x86_64/fred_test.c

diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index 492e937fab00..eaac13a605f2 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -67,6 +67,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/get_msr_index_features TEST_GEN_PROGS_x86_64 += x86_64/exit_on_emulation_failure_test TEST_GEN_PROGS_x86_64 += x86_64/fix_hypercall_test TEST_GEN_PROGS_x86_64 += x86_64/hwcr_msr_test +TEST_GEN_PROGS_x86_64 += x86_64/fred_test TEST_GEN_PROGS_x86_64 += x86_64/hyperv_clock TEST_GEN_PROGS_x86_64 += x86_64/hyperv_cpuid TEST_GEN_PROGS_x86_64 += x86_64/hyperv_evmcs diff --git a/tools/testing/selftests/kvm/include/x86_64/processor.h b/tools/testing/selftests/kvm/include/x86_64/processor.h index bc5cd8628a20..ef7aaab790e0 100644 --- a/tools/testing/selftests/kvm/include/x86_64/processor.h +++ b/tools/testing/selftests/kvm/include/x86_64/processor.h @@ -1275,4 +1275,36 @@ void virt_map_level(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, #define PFERR_GUEST_PAGE_MASK BIT_ULL(PFERR_GUEST_PAGE_BIT) #define PFERR_IMPLICIT_ACCESS BIT_ULL(PFERR_IMPLICIT_ACCESS_BIT) +/*

• • FRED related data structures and functions
• */

+#define FRED_SSX_NMI BIT_ULL(18)

+struct fred_stack {

• u64 r15;
• u64 r14;
• u64 r13;
• u64 r12;
• u64 bp;
• u64 bx;
• u64 r11;
• u64 r10;
• u64 r9;
• u64 r8;
• u64 ax;
• u64 cx;
• u64 dx;
• u64 si;
• u64 di;
• u64 error_code;
• u64 ip;
• u64 csx;
• u64 flags;
• u64 sp;
• u64 ssx;
• u64 event_data;
• u64 reserved;

+};

#endif /* SELFTEST_KVM_PROCESSOR_H */ diff --git a/tools/testing/selftests/kvm/x86_64/fred_test.c b/tools/testing/selftests/kvm/x86_64/fred_test.c new file mode 100644 index 000000000000..412afa919568 --- /dev/null +++ b/tools/testing/selftests/kvm/x86_64/fred_test.c @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0-only +/*

• • FRED nested exception tests
• • Copyright (C) 2023, Intel, Inc.
• */

+#define _GNU_SOURCE /* for program_invocation_short_name */ +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/ioctl.h> +#include <asm/msr-index.h>

+#include "apic.h" +#include "kvm_util.h" +#include "test_util.h" +#include "guest_modes.h" +#include "processor.h"

+#define IRQ_VECTOR 0xAA

+#define FRED_STKLVL(v,l) (_AT(unsigned long, l) << (2 * (v))) +#define FRED_CONFIG_ENTRYPOINT(p) _AT(unsigned long, (p))

+/* This address is already mapped in guest page table. */ +#define FRED_VALID_RSP 0x8000

+/*

• • The following addresses are not yet mapped in both EPT and guest page
• • tables at the beginning. As a result, it causes an EPT violation VM
• • exit with an original guest #PF to access any of them for the first
• • time.
• • Use these addresses as guest FRED RSP0 to generate nested #PFs to test
• • if event data are properly virtualized.
• */

+static unsigned long fred_invalid_rsp[4] = {

• 0x0,
• 0xf0000000,
• 0xe0000000,
• 0xd0000000,

+};

+extern char asm_user_nop[]; +extern char asm_user_ud[]; +extern char asm_done_fault[];

+extern void asm_test_fault(int test);

+/*

• • user level code for triggering faults.
• */

+asm(".pushsection .text\n"

• ".align 4096\n"
• ".type asm_user_nop, @function\n"
• "asm_user_nop:\n"
• "1: .byte 0x90\n"
• "jmp 1b\n"
• ".fill asm_user_ud - ., 1, 0xcc\n"
• ".type asm_user_ud, @function\n"
• ".org asm_user_nop + 16\n"
• "asm_user_ud:\n"
• /* Trigger a #UD */
• "ud2\n"
• ".align 4096, 0xcc\n"
• ".popsection");

+/* Send current stack level and #PF address */ +#define GUEST_SYNC_CSL_FA(__stage, __pf_address) \

• GUEST_SYNC_ARGS(__stage, __pf_address, 0, 0, 0)

+void fred_entry_from_user(struct fred_stack *stack) +{

• u32 current_stack_level = rdmsr(MSR_IA32_FRED_CONFIG) & 0x3;
• GUEST_SYNC_CSL_FA(current_stack_level, stack->event_data);
• /* Do NOT go back to user level, continue the next test instead */
• stack->ssx = 0x18;
• stack->csx = 0x10;
• stack->ip = (u64)&asm_done_fault;

+}

+void fred_entry_from_kernel(struct fred_stack *stack) +{

• /*

• * Keep NMI blocked to delay the delivery of the next NMI until
  

• * returning to user level.
  

• * */
  

• stack->ssx &= ~FRED_SSX_NMI;

+}

+#define PUSH_REGS \

• "push %rdi\n" \
• "push %rsi\n" \
• "push %rdx\n" \
• "push %rcx\n" \
• "push %rax\n" \
• "push %r8\n" \
• "push %r9\n" \
• "push %r10\n" \
• "push %r11\n" \
• "push %rbx\n" \
• "push %rbp\n" \
• "push %r12\n" \
• "push %r13\n" \
• "push %r14\n" \
• "push %r15\n"

+#define POP_REGS \

• "pop %r15\n" \
• "pop %r14\n" \
• "pop %r13\n" \
• "pop %r12\n" \
• "pop %rbp\n" \
• "pop %rbx\n" \
• "pop %r11\n" \
• "pop %r10\n" \
• "pop %r9\n" \
• "pop %r8\n" \
• "pop %rax\n" \
• "pop %rcx\n" \
• "pop %rdx\n" \
• "pop %rsi\n" \
• "pop %rdi\n"

+/*

• • FRED entry points.
• */

+asm(".pushsection .text\n"

• ".type asm_fred_entrypoint_user, @function\n"
• ".align 4096\n"
• "asm_fred_entrypoint_user:\n"
• "endbr64\n"
• PUSH_REGS
• "movq %rsp, %rdi\n"
• "call fred_entry_from_user\n"
• POP_REGS
• /* Do NOT go back to user level, continue the next test instead */
• ".byte 0xf2,0x0f,0x01,0xca\n" /* ERETS */
• ".fill asm_fred_entrypoint_kernel - ., 1, 0xcc\n"
• ".type asm_fred_entrypoint_kernel, @function\n"
• ".org asm_fred_entrypoint_user + 256\n"
• "asm_fred_entrypoint_kernel:\n"
• "endbr64\n"
• PUSH_REGS
• "movq %rsp, %rdi\n"
• "call fred_entry_from_kernel\n"
• POP_REGS
• ".byte 0xf2,0x0f,0x01,0xca\n" /* ERETS */
• ".align 4096, 0xcc\n"
• ".popsection");

+extern char asm_fred_entrypoint_user[];

+/*

• • Prepare a FRED stack frame for ERETU to return to user level code,
• • nop or ud2.
• • Because FRED RSP0 is deliberately not mapped in guest page table,
• • the delivery of interrupt/NMI or #UD from ring 3 causes a nested
• • #PF, which is then delivered on FRED RSPx (x is 1, 2 or 3,
• • determinated by MSR FRED_STKLVL[PF_VECTOR]).
• */

+asm(".pushsection .text\n"

• ".type asm_test_fault, @function\n"
• ".align 4096\n"
• "asm_test_fault:\n"
• "endbr64\n"
• "push %rbp\n"
• "mov %rsp, %rbp\n"
• "and $(~0x3f), %rsp\n"
• "push $0\n"
• "push $0\n"
• "mov $0x2b, %rax\n"
• /* Unblock NMI */
• "bts $18, %rax\n"
• /* Set long mode bit */
• "bts $57, %rax\n"
• "push %rax\n"
• /* No stack required for the FRED user level test code */
• "push $0\n"
• "pushf\n"
• "pop %rax\n"
• /* Allow external interrupts */
• "bts $9, %rax\n"
• "push %rax\n"
• "mov $0x33, %rax\n"
• "push %rax\n"
• "cmp $0, %edi\n"
• "jne 1f\n"
• "lea asm_user_nop(%rip), %rax\n"
• "jmp 2f\n"
• "1: lea asm_user_ud(%rip), %rax\n"
• "2: push %rax\n"
• "push $0\n"
• /* ERETU to user level code to allow event delivery immediately */
• ".byte 0xf3,0x0f,0x01,0xca\n"
• "asm_done_fault:\n"
• "mov %rbp, %rsp\n"
• "pop %rbp\n"
• "ret\n"
• ".align 4096, 0xcc\n"
• ".popsection");

+/*

• • To fully test the underlying FRED VMX code, this test should be run one
• • more round with EPT disabled to inject page faults as nested exceptions.
• */

+static void guest_code(void) +{

• wrmsr(MSR_IA32_FRED_CONFIG,

•      FRED_CONFIG_ENTRYPOINT(asm_fred_entrypoint_user));
  

• wrmsr(MSR_IA32_FRED_RSP1, FRED_VALID_RSP);
• wrmsr(MSR_IA32_FRED_RSP2, FRED_VALID_RSP);
• wrmsr(MSR_IA32_FRED_RSP3, FRED_VALID_RSP);
• /* Enable FRED */
• set_cr4(get_cr4() | X86_CR4_FRED);
• x2apic_enable();
• wrmsr(MSR_IA32_FRED_STKLVLS, FRED_STKLVL(PF_VECTOR, 1));
• wrmsr(MSR_IA32_FRED_RSP0, fred_invalid_rsp[1]);
• /* 1: ud2 to generate #UD */
• asm_test_fault(1);
• wrmsr(MSR_IA32_FRED_STKLVLS, FRED_STKLVL(PF_VECTOR, 2));
• wrmsr(MSR_IA32_FRED_RSP0, fred_invalid_rsp[2]);
• asm volatile("cli");
• /* Create a pending interrupt on current vCPU */
• x2apic_write_reg(APIC_ICR, APIC_DEST_SELF | APIC_INT_ASSERT |

• 	 APIC_DM_FIXED | IRQ_VECTOR);
  

• /* Return to ring 3 */
• asm_test_fault(0);
• x2apic_write_reg(APIC_EOI, 0);
• wrmsr(MSR_IA32_FRED_STKLVLS, FRED_STKLVL(PF_VECTOR, 3));
• wrmsr(MSR_IA32_FRED_RSP0, fred_invalid_rsp[3]);
• /*

• * The first NMI is just to have NMI blocked in ring 0, because
  

• * fred_entry_from_kernel() deliberately clears the NMI bit in
  

• * FRED stack frame.
  

• */
  

• x2apic_write_reg(APIC_ICR, APIC_DEST_SELF | APIC_INT_ASSERT |

• 	 APIC_DM_NMI | NMI_VECTOR);
  

• /* The second NMI will be delivered after returning to ring 3 */
• x2apic_write_reg(APIC_ICR, APIC_DEST_SELF | APIC_INT_ASSERT |

• 	 APIC_DM_NMI | NMI_VECTOR);
  

• /* Return to ring 3 */
• asm_test_fault(0);
• GUEST_DONE();

+}

+int main(int argc, char *argv[]) +{

• struct kvm_vcpu *vcpu;
• struct kvm_vm *vm;
• struct ucall uc;
• uint64_t expected_current_stack_level = 1;
• TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_FRED));
• vm = __vm_create_with_vcpus(VM_SHAPE(VM_MODE_PXXV48_4K_USER), 1, 0,

• 		    guest_code, &vcpu);
  

• while (true) {

• uint64_t r;
  

• vcpu_run(vcpu);
  

• r = get_ucall(vcpu, &uc);
  

• if (r == UCALL_DONE)
  

• 	break;
  

• if (r == UCALL_SYNC) {
  

• 	TEST_ASSERT((uc.args[1] == expected_current_stack_level) &&
  

• 		    (uc.args[2] == fred_invalid_rsp[expected_current_stack_level] - 1),
  

• 		    "Incorrect stack level %lx and #PF address %lx\n",
  

• 		    uc.args[1], uc.args[2]);
  

• 	expected_current_stack_level++;
  

• }
  

• }
• kvm_vm_free(vm);
• return 0;

+}

On Wed, Feb 07, 2024 at 09:26:27AM -0800, Xin Li wrote:

Add FRED MSRs to the valid passthrough MSR list and set FRED MSRs intercept based on FRED enumeration.

Signed-off-by: Xin Li xin3.li@intel.com Tested-by: Shan Kang shan.kang@intel.com

Reviewed-by: Chao Gao chao.gao@intel.com

two nits below.

---

Change since v1:

• Enable FRED MSRs intercept if FRED is no longer enumerated in CPUID

(Chao Gao).

arch/x86/kvm/vmx/vmx.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 34b6676f60d8..d58ed2d3d379 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -693,6 +693,9 @@ static bool is_valid_passthrough_msr(u32 msr) case MSR_LBR_CORE_TO ... MSR_LBR_CORE_TO + 8: /* LBR MSRs. These are handled in vmx_update_intercept_for_lbr_msrs() */ return true;

• case MSR_IA32_FRED_RSP0 ... MSR_IA32_FRED_CONFIG:

• /* FRED MSRs should be passthrough to FRED guests only */
  

This comment sounds weird. It sounds like the code will be something like: if guest supports FRED return true else return false

how about "FRED MSRs are pass-thru'd to guests which enumerate FRED"?

Or to align with above comment for LBR MSRs, just say

/* FRED MSRs. These are handled in vmx_vcpu_config_fred_after_set_cpuid() */

• return true;
  

  }

  r = possible_passthrough_msr_slot(msr) != -ENOENT;

@@ -7774,10 +7777,12 @@ static void update_intel_pt_cfg(struct kvm_vcpu *vcpu) static void vmx_vcpu_config_fred_after_set_cpuid(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu);

• bool fred_enumerated;

  kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_FRED);

• fred_enumerated = guest_can_use(vcpu, X86_FEATURE_FRED);

• if (guest_can_use(vcpu, X86_FEATURE_FRED)) {

• if (fred_enumerated) { vm_entry_controls_setbit(vmx, VM_ENTRY_LOAD_IA32_FRED); secondary_vm_exit_controls_setbit(vmx, SECONDARY_VM_EXIT_SAVE_IA32_FRED |

@@ -7788,6 +7793,16 @@ static void vmx_vcpu_config_fred_after_set_cpuid(struct kvm_vcpu *vcpu) SECONDARY_VM_EXIT_SAVE_IA32_FRED | SECONDARY_VM_EXIT_LOAD_IA32_FRED); }

• vmx_set_intercept_for_msr(vcpu, MSR_IA32_FRED_RSP0, MSR_TYPE_RW, !fred_enumerated);
• vmx_set_intercept_for_msr(vcpu, MSR_IA32_FRED_RSP1, MSR_TYPE_RW, !fred_enumerated);
• vmx_set_intercept_for_msr(vcpu, MSR_IA32_FRED_RSP2, MSR_TYPE_RW, !fred_enumerated);
• vmx_set_intercept_for_msr(vcpu, MSR_IA32_FRED_RSP3, MSR_TYPE_RW, !fred_enumerated);
• vmx_set_intercept_for_msr(vcpu, MSR_IA32_FRED_STKLVLS, MSR_TYPE_RW, !fred_enumerated);
• vmx_set_intercept_for_msr(vcpu, MSR_IA32_FRED_SSP1, MSR_TYPE_RW, !fred_enumerated);
• vmx_set_intercept_for_msr(vcpu, MSR_IA32_FRED_SSP2, MSR_TYPE_RW, !fred_enumerated);
• vmx_set_intercept_for_msr(vcpu, MSR_IA32_FRED_SSP3, MSR_TYPE_RW, !fred_enumerated);
• vmx_set_intercept_for_msr(vcpu, MSR_IA32_FRED_CONFIG, MSR_TYPE_RW, !fred_enumerated);

Use a for-loop here? e.g., for (i = MSR_IA32_FRED_RSP0; i <= MSR_IA32_FRED_CONFIG; i++)

}

static void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)

2.43.0

On Wed, Feb 07, 2024 at 09:26:29AM -0800, Xin Li wrote:

Switch MSR_IA32_FRED_RSP0 between host and guest in vmx_prepare_switch_to_{host,guest}().

MSR_IA32_FRED_RSP0 is used during ring 3 event delivery only, thus KVM, running on ring 0, can run safely with guest FRED RSP0, i.e., no need to switch between host/guest FRED RSP0 during VM entry and exit.

KVM should switch to host FRED RSP0 before returning to user level, and switch to guest FRED RSP0 before entering guest mode.

Signed-off-by: Xin Li xin3.li@intel.com Tested-by: Shan Kang shan.kang@intel.com

---

Changes since v1:

• Don't use guest_cpuid_has() in vmx_prepare_switch_to_{host,guest}(),

which are called from IRQ-disabled context (Chao Gao).

• Reset msr_guest_fred_rsp0 in __vmx_vcpu_reset() (Chao Gao).

---

arch/x86/kvm/vmx/vmx.c | 17 +++++++++++++++++ arch/x86/kvm/vmx/vmx.h | 2 ++ 2 files changed, 19 insertions(+)

diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index b7b772183ee4..264378c3b784 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1337,6 +1337,16 @@ void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu) }

wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);

• if (guest_can_use(vcpu, X86_FEATURE_FRED)) {

• /*
  

•  * MSR_IA32_FRED_RSP0 is top of task stack, which never changes.
  

•  * Thus it should be initialized only once.
  

•  */
  

• if (unlikely(vmx->msr_host_fred_rsp0 == 0))
  

• 	vmx->msr_host_fred_rsp0 = read_msr(MSR_IA32_FRED_RSP0);
  

can we just drop this and use "(unsigned long)task_stack_page(current) + THREAD_SIZE" as host fred rsp0?

• wrmsrl(MSR_IA32_FRED_RSP0, vmx->msr_guest_fred_rsp0);
  

any reason to not use wrmsrns?

• }

#else savesegment(fs, fs_sel); savesegment(gs, gs_sel); @@ -1381,6 +1391,11 @@ static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx) invalidate_tss_limit(); #ifdef CONFIG_X86_64 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);

• if (guest_can_use(&vmx->vcpu, X86_FEATURE_FRED)) {

• vmx->msr_guest_fred_rsp0 = read_msr(MSR_IA32_FRED_RSP0);
  

• wrmsrl(MSR_IA32_FRED_RSP0, vmx->msr_host_fred_rsp0);
  

same question.

• }

#endif load_fixmap_gdt(raw_smp_processor_id()); vmx->guest_state_loaded = false; @@ -4889,6 +4904,8 @@ static void __vmx_vcpu_reset(struct kvm_vcpu *vcpu)

#ifdef CONFIG_X86_64 if (kvm_cpu_cap_has(X86_FEATURE_FRED)) {

• vmx->msr_guest_fred_rsp0 = 0;
  

• vmcs_write64(GUEST_IA32_FRED_CONFIG, 0); vmcs_write64(GUEST_IA32_FRED_RSP1, 0); vmcs_write64(GUEST_IA32_FRED_RSP2, 0);

diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 3ad52437f426..176ad39be406 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -278,6 +278,8 @@ struct vcpu_vmx { #ifdef CONFIG_X86_64 u64 msr_host_kernel_gs_base; u64 msr_guest_kernel_gs_base;

• u64 msr_host_fred_rsp0;
• u64 msr_guest_fred_rsp0;

#endif

u64 spec_ctrl;

2.43.0

On Thu, Feb 08, 2024 at 01:26:30AM +0800, Xin Li wrote:

Handle host initiated FRED MSR access requests to allow FRED context to be set/get from user level.

The changelog isn't accurate because guest accesses are also handled by this patch, specifically in the "else" branch.

>+ if (host_initiated) { >+ if (!kvm_cpu_cap_has(X86_FEATURE_FRED)) >+ return 1; >+ } else {

void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, @@ -2019,6 +2037,33 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) case MSR_KERNEL_GS_BASE: msr_info->data = vmx_read_guest_kernel_gs_base(vmx); break;

• case MSR_IA32_FRED_RSP0:

• msr_info->data = vmx_read_guest_fred_rsp0(vmx);
  

• break;
  

• case MSR_IA32_FRED_RSP1:

• msr_info->data = vmcs_read64(GUEST_IA32_FRED_RSP1);
  

• break;
  

• case MSR_IA32_FRED_RSP2:

• msr_info->data = vmcs_read64(GUEST_IA32_FRED_RSP2);
  

• break;
  

• case MSR_IA32_FRED_RSP3:

• msr_info->data = vmcs_read64(GUEST_IA32_FRED_RSP3);
  

• break;
  

• case MSR_IA32_FRED_STKLVLS:

• msr_info->data = vmcs_read64(GUEST_IA32_FRED_STKLVLS);
  

• break;
  

• case MSR_IA32_FRED_SSP1:

• msr_info->data = vmcs_read64(GUEST_IA32_FRED_SSP1);
  

• break;
  

• case MSR_IA32_FRED_SSP2:

• msr_info->data = vmcs_read64(GUEST_IA32_FRED_SSP2);
  

• break;
  

• case MSR_IA32_FRED_SSP3:

• msr_info->data = vmcs_read64(GUEST_IA32_FRED_SSP3);
  

• break;
  

• case MSR_IA32_FRED_CONFIG:

• msr_info->data = vmcs_read64(GUEST_IA32_FRED_CONFIG);
  

• break;
  

how about adding a helper function to convert MSR index to the VMCS field id? Then do:

case MSR_IA32_FRED_RSP1 ... MSR_IA32_FRED_STKLVLS: case MSR_IA32_FRED_SSP1 ... MSR_IA32_FRED_CONFIG: msr_info->data = vmcs_read64(msr_to_vmcs(index)); break;

and ...

#endif case MSR_EFER: return kvm_get_msr_common(vcpu, msr_info); @@ -2226,6 +2271,33 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) vmx_update_exception_bitmap(vcpu); } break;

• case MSR_IA32_FRED_RSP0:

• vmx_write_guest_fred_rsp0(vmx, data);
  

• break;
  

• case MSR_IA32_FRED_RSP1:

• vmcs_write64(GUEST_IA32_FRED_RSP1, data);
  

• break;
  

• case MSR_IA32_FRED_RSP2:

• vmcs_write64(GUEST_IA32_FRED_RSP2, data);
  

• break;
  

• case MSR_IA32_FRED_RSP3:

• vmcs_write64(GUEST_IA32_FRED_RSP3, data);
  

• break;
  

• case MSR_IA32_FRED_STKLVLS:

• vmcs_write64(GUEST_IA32_FRED_STKLVLS, data);
  

• break;
  

• case MSR_IA32_FRED_SSP1:

• vmcs_write64(GUEST_IA32_FRED_SSP1, data);
  

• break;
  

• case MSR_IA32_FRED_SSP2:

• vmcs_write64(GUEST_IA32_FRED_SSP2, data);
  

• break;
  

• case MSR_IA32_FRED_SSP3:

• vmcs_write64(GUEST_IA32_FRED_SSP3, data);
  

• break;
  

• case MSR_IA32_FRED_CONFIG:

• vmcs_write64(GUEST_IA32_FRED_CONFIG, data);
  

• break;
  

case MSR_IA32_FRED_RSP1 ... MSR_IA32_FRED_STKLVLS: case MSR_IA32_FRED_SSP1 ... MSR_IA32_FRED_CONFIG: vmcs_write64(msr_to_vmcs(index), data); break;

The code will be more compact and generate less instructions. I believe CET series can do the same change [*]. Performance here isn't critical. I just think it looks cumbersome to repeat the same pattern for 8 (and more with CET considered) MSRs.

[*]: https://lore.kernel.org/kvm/20240219074733.122080-21-weijiang.yang@intel.com...

#endif case MSR_IA32_SYSENTER_CS: if (is_guest_mode(vcpu)) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 363b1c080205..4e8d60f248e3 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1451,6 +1451,9 @@ static const u32 msrs_to_save_base[] = { MSR_STAR, #ifdef CONFIG_X86_64 MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,

• MSR_IA32_FRED_RSP0, MSR_IA32_FRED_RSP1, MSR_IA32_FRED_RSP2,
• MSR_IA32_FRED_RSP3, MSR_IA32_FRED_STKLVLS, MSR_IA32_FRED_SSP1,
• MSR_IA32_FRED_SSP2, MSR_IA32_FRED_SSP3, MSR_IA32_FRED_CONFIG,

#endif MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA, MSR_IA32_FEAT_CTL, MSR_IA32_BNDCFGS, MSR_TSC_AUX, @@ -1892,6 +1895,30 @@ static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data, return 1;

data = (u32)data;

• break;
  

• case MSR_IA32_FRED_RSP0 ... MSR_IA32_FRED_CONFIG:

• if (index != MSR_IA32_FRED_STKLVLS && is_noncanonical_address(data, vcpu))
  

• 	return 1;
  

• if ((index >= MSR_IA32_FRED_RSP0 && index <= MSR_IA32_FRED_RSP3) &&
  

•     (data & GENMASK_ULL(5, 0)))
  

• 	return 1;
  

• if ((index >= MSR_IA32_FRED_SSP1 && index <= MSR_IA32_FRED_SSP3) &&
  

•     (data & GENMASK_ULL(2, 0)))
  

• 	return 1;
  

• if (host_initiated) {
  

• 	if (!kvm_cpu_cap_has(X86_FEATURE_FRED))
  

• 		return 1;
  

Should be: if (!kvm_cpu_cap_has(X86_FEATURE_FRED) && data)

KVM ABI allows userspace to write only 0 if guests cannot enumerate the feature. And even better, your next version can be on top of Sean's series

https://lore.kernel.org/kvm/20240425181422.3250947-1-seanjc@google.com/T/#md...

this way, you can get rid of the "host_initiated" check.

index ee61d2c25cb0..f622fb90a098 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1871,9 +1871,29 @@ static void vmx_inject_exception(struct kvm_vcpu *vcpu) vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, vmx->vcpu.arch.event_exit_inst_len); intr_info |= INTR_TYPE_SOFT_EXCEPTION;

•   } else
  

•   } else {
           intr_info |= INTR_TYPE_HARD_EXCEPTION;
  
  

•           if (kvm_is_fred_enabled(vcpu)) {
  

•                   u64 event_data = 0;
  

•                   if (is_debug(intr_info))
  

•                           /*
  

•                            * Compared to DR6, FRED #DB event data saved on
  

•                            * the stack frame have bits 4 ~ 11 and 16 ~ 31
  

•                            * inverted, i.e.,
  

•                            *   fred_db_event_data = dr6 ^ 0xFFFF0FF0UL
  

•                            */
  

•                           event_data = vcpu->arch.dr6 ^ DR6_RESERVED;
  

•                   else if (is_page_fault(intr_info))
  

•                           event_data = vcpu->arch.cr2;
  

•                   else if (is_nm_fault(intr_info))
  

•                           event_data = to_vmx(vcpu)->fred_xfd_event_data;
  

IMO, deriving an event_data from CR2/DR6 is a little short-sighted because the event_data and CR2/DR6 __can__ be different, e.g., L1 VMM __can__ set CR2 to A and event_data field to B (!=A) when injecting #PF.

And this approach cannot be extended to handle a (future) exception whose event_data isn't tied to a dedicated register like CR2/DR6.

Adding a new field fred_xfd_event_data in struct vcpu has problems too: fred_xfd_event_data gets lost during migration; strickly speaking, event_data is tied to an exception rather than a CPU. e.g., the CPU may detect a nested exception when delivering one and both have their own event_data.

I think we can make event_data a property of exceptions. i.e., add a payload2 to struct kvm_queued_exception. and add new APIs to kvm_queue_exception* family to accept a payload2 and in VMX code, just program payload2 to the VMCS event_data field if FRED is enabled. KVM ABI should be extended as well to pass payload2 to userspace like how the payload is handled in kvm_vcpu_ioctl_x86_get/put_vcpu_events.