From: Ira Weiny ira.weiny@intel.com

Define a helper, update_pkey_val(), which will be used to support both Protection Key User (PKU) and the new Protection Key for Supervisor (PKS) in subsequent patches.

Reviewed-by: Dan Williams dan.j.williams@intel.com Co-developed-by: Peter Zijlstra peterz@infradead.org Signed-off-by: Peter Zijlstra peterz@infradead.org Signed-off-by: Ira Weiny ira.weiny@intel.com

--- Changes from RFC V3: Per Dave Hansen Update and add comments per Dave's review Per Peter Correct attribution --- arch/x86/include/asm/pkeys.h | 2 ++ arch/x86/kernel/fpu/xstate.c | 22 ++++------------------ arch/x86/mm/pkeys.c | 23 +++++++++++++++++++++++ 3 files changed, 29 insertions(+), 18 deletions(-)

diff --git a/arch/x86/include/asm/pkeys.h b/arch/x86/include/asm/pkeys.h index f9feba80894b..4526245b03e5 100644 --- a/arch/x86/include/asm/pkeys.h +++ b/arch/x86/include/asm/pkeys.h @@ -136,4 +136,6 @@ static inline int vma_pkey(struct vm_area_struct *vma) return (vma->vm_flags & vma_pkey_mask) >> VM_PKEY_SHIFT; }

+u32 update_pkey_val(u32 pk_reg, int pkey, unsigned int flags); + #endif /*_ASM_X86_PKEYS_H */ diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index face29dab0e3..00251bdf759b 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -994,9 +994,7 @@ const void *get_xsave_field_ptr(int xfeature_nr) int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, unsigned long init_val) { - u32 old_pkru; - int pkey_shift = (pkey * PKR_BITS_PER_PKEY); - u32 new_pkru_bits = 0; + u32 pkru;

/* * This check implies XSAVE support. OSPKE only gets @@ -1012,21 +1010,9 @@ int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, */ WARN_ON_ONCE(pkey >= arch_max_pkey());

- /* Set the bits we need in PKRU: */ - if (init_val & PKEY_DISABLE_ACCESS) - new_pkru_bits |= PKR_AD_BIT; - if (init_val & PKEY_DISABLE_WRITE) - new_pkru_bits |= PKR_WD_BIT; - - /* Shift the bits in to the correct place in PKRU for pkey: */ - new_pkru_bits <<= pkey_shift; - - /* Get old PKRU and mask off any old bits in place: */ - old_pkru = read_pkru(); - old_pkru &= ~((PKR_AD_BIT|PKR_WD_BIT) << pkey_shift); - - /* Write old part along with new part: */ - write_pkru(old_pkru | new_pkru_bits); + pkru = read_pkru(); + pkru = update_pkey_val(pkru, pkey, init_val); + write_pkru(pkru);

return 0; } diff --git a/arch/x86/mm/pkeys.c b/arch/x86/mm/pkeys.c index f5efb4007e74..d1dfe743e79f 100644 --- a/arch/x86/mm/pkeys.c +++ b/arch/x86/mm/pkeys.c @@ -208,3 +208,26 @@ static __init int setup_init_pkru(char *opt) return 1; } __setup("init_pkru=", setup_init_pkru); + +/* + * Replace disable bits for @pkey with values from @flags + * + * Kernel users use the same flags as user space: + * PKEY_DISABLE_ACCESS + * PKEY_DISABLE_WRITE + */ +u32 update_pkey_val(u32 pk_reg, int pkey, unsigned int flags) +{ + int pkey_shift = pkey * PKR_BITS_PER_PKEY; + + /* Mask out old bit values */ + pk_reg &= ~(((1 << PKR_BITS_PER_PKEY) - 1) << pkey_shift); + + /* Or in new values */ + if (flags & PKEY_DISABLE_ACCESS) + pk_reg |= PKR_AD_BIT << pkey_shift; + if (flags & PKEY_DISABLE_WRITE) + pk_reg |= PKR_WD_BIT << pkey_shift; + + return pk_reg; +}

From: Ira Weiny ira.weiny@intel.com

Protection Keys for Supervisor pages (PKS) enables fast, hardware thread specific, manipulation of permission restrictions on supervisor page mappings. It uses the same mechanism of Protection Keys as those on User mappings but applies that mechanism to supervisor mappings using a supervisor specific MSR.

Kernel users can define domains of page mappings which have an extra level of protection beyond those specified in the supervisor page table entries.

Define the PKS CPU feature bits.

Add the Kconfig ARCH_HAS_SUPERVISOR_PKEYS to indicate to consumers that an architecture supports pkeys.

Introduce ARCH_ENABLE_SUPERVISOR_PKEYS to allow kernel users to specify to the arch that they wish to use the supervisor key support if ARCH_HAS_SUPERVISOR_PKEYS is available. ARCH_ENABLE_SUPERVISOR_PKEYS remains off until the first use case sets it.

Reviewed-by: Dan Williams dan.j.williams@intel.com Co-developed-by: Fenghua Yu fenghua.yu@intel.com Signed-off-by: Fenghua Yu fenghua.yu@intel.com Signed-off-by: Ira Weiny ira.weiny@intel.com

--- Changes from V3: From Dan Clean up commit message Add ARCH_ENABLE_SUPERVISOR_PKEYS option so we don't have the overhead of PKS unless there is a user Clean up commit message grammar

Changes from V2 New patch for V3: Split this off from the enable patch to be able to create cleaner bisectability --- arch/x86/Kconfig | 1 + arch/x86/include/asm/cpufeatures.h | 1 + arch/x86/include/asm/disabled-features.h | 8 +++++++- arch/x86/include/uapi/asm/processor-flags.h | 2 ++ mm/Kconfig | 4 ++++ 5 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 2792879d398e..5e3a7c2bc342 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1870,6 +1870,7 @@ config X86_INTEL_MEMORY_PROTECTION_KEYS depends on X86_64 && (CPU_SUP_INTEL || CPU_SUP_AMD) select ARCH_USES_HIGH_VMA_FLAGS select ARCH_HAS_PKEYS + select ARCH_HAS_SUPERVISOR_PKEYS help Memory Protection Keys provides a mechanism for enforcing page-based protections, but without requiring modification of the diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index cc96e26d69f7..83ed73407417 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -359,6 +359,7 @@ #define X86_FEATURE_MOVDIR64B (16*32+28) /* MOVDIR64B instruction */ #define X86_FEATURE_ENQCMD (16*32+29) /* ENQCMD and ENQCMDS instructions */ #define X86_FEATURE_SGX_LC (16*32+30) /* Software Guard Extensions Launch Control */ +#define X86_FEATURE_PKS (16*32+31) /* Protection Keys for Supervisor pages */

/* AMD-defined CPU features, CPUID level 0x80000007 (EBX), word 17 */ #define X86_FEATURE_OVERFLOW_RECOV (17*32+ 0) /* MCA overflow recovery support */ diff --git a/arch/x86/include/asm/disabled-features.h b/arch/x86/include/asm/disabled-features.h index b7dd944dc867..fd09ae852c04 100644 --- a/arch/x86/include/asm/disabled-features.h +++ b/arch/x86/include/asm/disabled-features.h @@ -44,6 +44,12 @@ # define DISABLE_OSPKE (1<<(X86_FEATURE_OSPKE & 31)) #endif /* CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS */

+#ifdef CONFIG_ARCH_ENABLE_SUPERVISOR_PKEYS +# define DISABLE_PKS 0 +#else +# define DISABLE_PKS (1<<(X86_FEATURE_PKS & 31)) +#endif + #ifdef CONFIG_X86_5LEVEL # define DISABLE_LA57 0 #else @@ -88,7 +94,7 @@ #define DISABLED_MASK14 0 #define DISABLED_MASK15 0 #define DISABLED_MASK16 (DISABLE_PKU|DISABLE_OSPKE|DISABLE_LA57|DISABLE_UMIP| \ - DISABLE_ENQCMD) + DISABLE_ENQCMD|DISABLE_PKS) #define DISABLED_MASK17 0 #define DISABLED_MASK18 0 #define DISABLED_MASK19 0 diff --git a/arch/x86/include/uapi/asm/processor-flags.h b/arch/x86/include/uapi/asm/processor-flags.h index bcba3c643e63..191c574b2390 100644 --- a/arch/x86/include/uapi/asm/processor-flags.h +++ b/arch/x86/include/uapi/asm/processor-flags.h @@ -130,6 +130,8 @@ #define X86_CR4_SMAP _BITUL(X86_CR4_SMAP_BIT) #define X86_CR4_PKE_BIT 22 /* enable Protection Keys support */ #define X86_CR4_PKE _BITUL(X86_CR4_PKE_BIT) +#define X86_CR4_PKS_BIT 24 /* enable Protection Keys for Supervisor */ +#define X86_CR4_PKS _BITUL(X86_CR4_PKS_BIT)

/* * x86-64 Task Priority Register, CR8 diff --git a/mm/Kconfig b/mm/Kconfig index 24c045b24b95..c7d1fc780358 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -808,6 +808,10 @@ config ARCH_USES_HIGH_VMA_FLAGS bool config ARCH_HAS_PKEYS bool +config ARCH_HAS_SUPERVISOR_PKEYS + bool +config ARCH_ENABLE_SUPERVISOR_PKEYS + bool

config PERCPU_STATS bool "Collect percpu memory statistics"

From: Ira Weiny ira.weiny@intel.com

Previously if a Protection key fault occurred it indicated something very wrong because user page mappings are not supposed to be in the kernel address space.

Now PKey faults may happen on kernel mappings if the feature is enabled.

Remove the warning in the fault path and allow the oops to occur without extra debugging if PKS is enabled.

Cc: Sean Christopherson seanjc@google.com Cc: Dan Williams dan.j.williams@intel.com Signed-off-by: Ira Weiny ira.weiny@intel.com

--- Changes from V4: From Sean Christopherson Clean up commit message and comment Change cpu_feature_enabled to be in WARN_ON check --- arch/x86/mm/fault.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index a73347e2cdfc..c32d01803e26 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -1141,11 +1141,15 @@ do_kern_addr_fault(struct pt_regs *regs, unsigned long hw_error_code, unsigned long address) { /* - * Protection keys exceptions only happen on user pages. We - * have no user pages in the kernel portion of the address - * space, so do not expect them here. + * X86_PF_PK (Protection key exceptions) may occur on kernel addresses + * when PKS (PKeys Supervisor) are enabled. + * + * If PKS is not enabled an exception should only happen on user pages. + * Because, we have no user pages in the kernel portion of the address + * space something must have gone very wrong and we should WARN. */ - WARN_ON_ONCE(hw_error_code & X86_PF_PK); + WARN_ON_ONCE(!cpu_feature_enabled(X86_FEATURE_PKS) && + (hw_error_code & X86_PF_PK));

#ifdef CONFIG_X86_32 /*

From: Ira Weiny ira.weiny@intel.com

The PKRS MSR is not managed by XSAVE. It is preserved through a context switch but this support leaves exception handling code open to memory accesses during exceptions.

2 possible places for preserving this state were considered, irqentry_state_t or pt_regs.[1] pt_regs was much more complicated and was potentially fraught with unintended consequences.[2] However, Andy came up with a way to hide additional values on the stack which could be accessed as "extended_pt_regs".[3] This method allows for; any place which has struct pt_regs can get access to the extra information; no extra information is added to irq_state; and pt_regs is left intact for compatibility with outside tools like BPF.

To simplify, the assembly code only adds space on the stack. The setting or use of any needed values are left to the C code. While some entry points may not use this space it is still added where ever pt_regs is passed to the C code for consistency.

Each nested exception gets another copy of this extended space allowing for any number of levels of exception handling.

In the assembly, a macro is defined to allow a central place to add space for other uses should the need arise.

Finally export pkrs_{save_set|restore}_irq to the common code to allow it to preserve the current task's PKRS in the new extended pt_regs if enabled.

Peter, Thomas, Andy, Dave, and Dan all suggested parts of the patch or aided in the development of the patch..

[1] https://lore.kernel.org/lkml/CALCETrVe1i5JdyzD_BcctxQJn+ZE3T38EFPgjxN1F577M3... [2] https://lore.kernel.org/lkml/874kpxx4jf.fsf@nanos.tec.linutronix.de/#t [3] https://lore.kernel.org/lkml/CALCETrUHwZPic89oExMMe-WyDY8-O3W68NcZvse3=PGW+i...

Acked-by: Dave Hansen dave.hansen@linux.intel.com Reviewed-by: Dan Williams dan.j.williams@intel.com Suggested-by: Dave Hansen dave.hansen@linux.intel.com Suggested-by: Dan Williams dan.j.williams@intel.com Suggested-by: Peter Zijlstra peterz@infradead.org Suggested-by: Thomas Gleixner tglx@linutronix.de Suggested-by: Andy Lutomirski luto@kernel.org Signed-off-by: Ira Weiny ira.weiny@intel.com

--- Changes from V4: Fix checkpatch errors

Changes from V3: Fix 0-day issues Move all extended regs stuff to pks.h From Dan Williams Move show_extended_regs_oops ifdefery to pks.h Remove a bad comment s/irq_save_set_pkrs/pkrs_save_set_irq s/irq_restore_pkrs/pkrs_restore_irq s/ARCH_HAS/ARCH_ENABLE_SUPERVISOR_PKEYS From Dave Hansen: remove extra macro parameter for most calls clarify with comments Add BUILD check for extend regs size use subq/addq vs push/pop From Dan Williams and Dave Hansen: Use a macro call to wrap the c function calls with push/pop extended_pt_regs From Dave Hansen: Guidance on where to find each of the pt_regs being passed to C code From Thomas Gleixner: Remove unnecessary noinstr's From Andy Lutomirski: Convert to using the extended pt_regs Add in showing pks on fault through the extended pt_regs

Changes from V1 remove redundant irq_state->pkrs This value is only needed for the global tracking. So it should be included in that patch and not in this one.

Changes from RFC V3 Standardize on 'irq_state' variable name Per Dave Hansen irq_save_pkrs() -> irq_save_set_pkrs() Rebased based on clean up patch by Thomas Gleixner This includes moving irq_[save_set|restore]_pkrs() to the core as well. --- arch/x86/entry/calling.h | 26 ++++++++++++ arch/x86/entry/common.c | 58 ++++++++++++++++++++++++++ arch/x86/entry/entry_64.S | 22 +++++----- arch/x86/entry/entry_64_compat.S | 6 +-- arch/x86/include/asm/pks.h | 16 +++++++ arch/x86/include/asm/processor-flags.h | 2 + arch/x86/kernel/head_64.S | 7 ++-- arch/x86/mm/fault.c | 3 ++ include/linux/pkeys.h | 17 ++++++++ kernel/entry/common.c | 14 ++++++- 10 files changed, 152 insertions(+), 19 deletions(-)

diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h index 07a9331d55e7..ec85f8f675be 100644 --- a/arch/x86/entry/calling.h +++ b/arch/x86/entry/calling.h @@ -97,6 +97,32 @@ For 32-bit we have the following conventions - kernel is built with

#define SIZEOF_PTREGS 21*8

+/* + * __call_ext_ptregs - Helper macro to call into C with extended pt_regs + * @cfunc: C function to be called + * + * This will ensure that extended_ptregs is added and removed as needed during + * a call into C code. + */ +.macro __call_ext_ptregs cfunc annotate_retpoline_safe:req +#ifdef CONFIG_ARCH_ENABLE_SUPERVISOR_PKEYS + /* add space for extended_pt_regs */ + subq $EXTENDED_PT_REGS_SIZE, %rsp +#endif + .if \annotate_retpoline_safe == 1 + ANNOTATE_RETPOLINE_SAFE + .endif + call \cfunc +#ifdef CONFIG_ARCH_ENABLE_SUPERVISOR_PKEYS + /* remove space for extended_pt_regs */ + addq $EXTENDED_PT_REGS_SIZE, %rsp +#endif +.endm + +.macro call_ext_ptregs cfunc + __call_ext_ptregs \cfunc, annotate_retpoline_safe=0 +.endm + .macro PUSH_AND_CLEAR_REGS rdx=%rdx rax=%rax save_ret=0 .if \save_ret pushq %rsi /* pt_regs->si */ diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c index 4efd39aacb9f..075668972401 100644 --- a/arch/x86/entry/common.c +++ b/arch/x86/entry/common.c @@ -19,6 +19,7 @@ #include <linux/nospec.h> #include <linux/syscalls.h> #include <linux/uaccess.h> +#include <linux/pkeys.h>

#ifdef CONFIG_XEN_PV #include <xen/xen-ops.h> @@ -34,6 +35,7 @@ #include <asm/io_bitmap.h> #include <asm/syscall.h> #include <asm/irq_stack.h> +#include <asm/pks.h>

#ifdef CONFIG_X86_64 __visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs) @@ -214,6 +216,60 @@ SYSCALL_DEFINE0(ni_syscall) return -ENOSYS; }

+#ifdef CONFIG_ARCH_ENABLE_SUPERVISOR_PKEYS + +void show_extended_regs_oops(struct pt_regs *regs, unsigned long error_code) +{ + struct extended_pt_regs *ept_regs = extended_pt_regs(regs); + + if (cpu_feature_enabled(X86_FEATURE_PKS) && (error_code & X86_PF_PK)) + pr_alert("PKRS: 0x%x\n", ept_regs->thread_pkrs); +} + +/* + * PKRS is a per-logical-processor MSR which overlays additional protection for + * pages which have been mapped with a protection key. + * + * The register is not maintained with XSAVE so we have to maintain the MSR + * value in software during context switch and exception handling. + * + * Context switches save the MSR in the task struct thus taking that value to + * other processors if necessary. + * + * To protect against exceptions having access to this memory we save the + * current running value and sets the PKRS value to be used during the + * exception. + */ +void pkrs_save_set_irq(struct pt_regs *regs, u32 val) +{ + struct extended_pt_regs *ept_regs; + + BUILD_BUG_ON(sizeof(struct extended_pt_regs) + != EXTENDED_PT_REGS_SIZE + + sizeof(struct pt_regs)); + + if (!cpu_feature_enabled(X86_FEATURE_PKS)) + return; + + ept_regs = extended_pt_regs(regs); + ept_regs->thread_pkrs = current->thread.saved_pkrs; + write_pkrs(val); +} + +void pkrs_restore_irq(struct pt_regs *regs) +{ + struct extended_pt_regs *ept_regs; + + if (!cpu_feature_enabled(X86_FEATURE_PKS)) + return; + + ept_regs = extended_pt_regs(regs); + write_pkrs(ept_regs->thread_pkrs); + current->thread.saved_pkrs = ept_regs->thread_pkrs; +} + +#endif /* CONFIG_ARCH_ENABLE_SUPERVISOR_PKEYS */ + #ifdef CONFIG_XEN_PV #ifndef CONFIG_PREEMPTION /* @@ -270,6 +326,8 @@ __visible noinstr void xen_pv_evtchn_do_upcall(struct pt_regs *regs)

inhcall = get_and_clear_inhcall(); if (inhcall && !WARN_ON_ONCE(state.exit_rcu)) { + /* Normally called by irqentry_exit, we must restore pkrs here */ + pkrs_restore_irq(regs); instrumentation_begin(); irqentry_exit_cond_resched(); instrumentation_end(); diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 400908dff42e..d65952a18ad7 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -331,7 +331,7 @@ SYM_CODE_END(ret_from_fork) movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */ .endif

- call \cfunc + call_ext_ptregs \cfunc

jmp error_return .endm @@ -434,7 +434,7 @@ SYM_CODE_START(\asmsym)

movq %rsp, %rdi /* pt_regs pointer */

- call \cfunc + call_ext_ptregs \cfunc

jmp paranoid_exit

@@ -495,7 +495,7 @@ SYM_CODE_START(\asmsym) * stack. */ movq %rsp, %rdi /* pt_regs pointer */ - call vc_switch_off_ist + call_ext_ptregs vc_switch_off_ist movq %rax, %rsp /* Switch to new stack */

UNWIND_HINT_REGS @@ -506,7 +506,7 @@ SYM_CODE_START(\asmsym)

movq %rsp, %rdi /* pt_regs pointer */

- call \cfunc + call_ext_ptregs \cfunc

/* * No need to switch back to the IST stack. The current stack is either @@ -541,7 +541,7 @@ SYM_CODE_START(\asmsym) movq %rsp, %rdi /* pt_regs pointer into first argument */ movq ORIG_RAX(%rsp), %rsi /* get error code into 2nd argument*/ movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */ - call \cfunc + call_ext_ptregs \cfunc

jmp paranoid_exit

@@ -780,7 +780,7 @@ SYM_CODE_START_LOCAL(exc_xen_hypervisor_callback) movq %rdi, %rsp /* we don't return, adjust the stack frame */ UNWIND_HINT_REGS

- call xen_pv_evtchn_do_upcall + call_ext_ptregs xen_pv_evtchn_do_upcall

jmp error_return SYM_CODE_END(exc_xen_hypervisor_callback) @@ -986,7 +986,7 @@ SYM_CODE_START_LOCAL(error_entry) /* Put us onto the real thread stack. */ popq %r12 /* save return addr in %12 */ movq %rsp, %rdi /* arg0 = pt_regs pointer */ - call sync_regs + call_ext_ptregs sync_regs movq %rax, %rsp /* switch stack */ ENCODE_FRAME_POINTER pushq %r12 @@ -1041,7 +1041,7 @@ SYM_CODE_START_LOCAL(error_entry) * as if we faulted immediately after IRET. */ mov %rsp, %rdi - call fixup_bad_iret + call_ext_ptregs fixup_bad_iret mov %rax, %rsp jmp .Lerror_entry_from_usermode_after_swapgs SYM_CODE_END(error_entry) @@ -1147,7 +1147,7 @@ SYM_CODE_START(asm_exc_nmi)

movq %rsp, %rdi movq $-1, %rsi - call exc_nmi + call_ext_ptregs exc_nmi

/* * Return back to user mode. We must *not* do the normal exit @@ -1183,6 +1183,8 @@ SYM_CODE_START(asm_exc_nmi) * +---------------------------------------------------------+ * | pt_regs | * +---------------------------------------------------------+ + * | (Optionally) extended_pt_regs | + * +---------------------------------------------------------+ * * The "original" frame is used by hardware. Before re-enabling * NMIs, we need to be done with it, and we need to leave enough @@ -1359,7 +1361,7 @@ end_repeat_nmi:

movq %rsp, %rdi movq $-1, %rsi - call exc_nmi + call_ext_ptregs exc_nmi

/* Always restore stashed CR3 value (see paranoid_entry) */ RESTORE_CR3 scratch_reg=%r15 save_reg=%r14 diff --git a/arch/x86/entry/entry_64_compat.S b/arch/x86/entry/entry_64_compat.S index 0051cf5c792d..53254d29d5c7 100644 --- a/arch/x86/entry/entry_64_compat.S +++ b/arch/x86/entry/entry_64_compat.S @@ -136,7 +136,7 @@ SYM_INNER_LABEL(entry_SYSENTER_compat_after_hwframe, SYM_L_GLOBAL) .Lsysenter_flags_fixed:

movq %rsp, %rdi - call do_SYSENTER_32 + call_ext_ptregs do_SYSENTER_32 /* XEN PV guests always use IRET path */ ALTERNATIVE "testl %eax, %eax; jz swapgs_restore_regs_and_return_to_usermode", \ "jmp swapgs_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV @@ -253,7 +253,7 @@ SYM_INNER_LABEL(entry_SYSCALL_compat_after_hwframe, SYM_L_GLOBAL) UNWIND_HINT_REGS

movq %rsp, %rdi - call do_fast_syscall_32 + call_ext_ptregs do_fast_syscall_32 /* XEN PV guests always use IRET path */ ALTERNATIVE "testl %eax, %eax; jz swapgs_restore_regs_and_return_to_usermode", \ "jmp swapgs_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV @@ -410,6 +410,6 @@ SYM_CODE_START(entry_INT80_compat) cld

movq %rsp, %rdi - call do_int80_syscall_32 + call_ext_ptregs do_int80_syscall_32 jmp swapgs_restore_regs_and_return_to_usermode SYM_CODE_END(entry_INT80_compat) diff --git a/arch/x86/include/asm/pks.h b/arch/x86/include/asm/pks.h index 5d7067ada8fb..516d82f032b6 100644 --- a/arch/x86/include/asm/pks.h +++ b/arch/x86/include/asm/pks.h @@ -4,11 +4,27 @@

#ifdef CONFIG_ARCH_ENABLE_SUPERVISOR_PKEYS

+struct extended_pt_regs { + u32 thread_pkrs; + /* Keep stack 8 byte aligned */ + u32 pad; + struct pt_regs pt_regs; +}; + void setup_pks(void);

+static inline struct extended_pt_regs *extended_pt_regs(struct pt_regs *regs) +{ + return container_of(regs, struct extended_pt_regs, pt_regs); +} + +void show_extended_regs_oops(struct pt_regs *regs, unsigned long error_code); + #else /* !CONFIG_ARCH_ENABLE_SUPERVISOR_PKEYS */

static inline void setup_pks(void) { } +static inline void show_extended_regs_oops(struct pt_regs *regs, + unsigned long error_code) { }

#endif /* CONFIG_ARCH_ENABLE_SUPERVISOR_PKEYS */

diff --git a/arch/x86/include/asm/processor-flags.h b/arch/x86/include/asm/processor-flags.h index 02c2cbda4a74..4a41fc4cf028 100644 --- a/arch/x86/include/asm/processor-flags.h +++ b/arch/x86/include/asm/processor-flags.h @@ -53,4 +53,6 @@ # define X86_CR3_PTI_PCID_USER_BIT 11 #endif

+#define EXTENDED_PT_REGS_SIZE 8 + #endif /* _ASM_X86_PROCESSOR_FLAGS_H */ diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 04bddaaba8e2..80531526b0d2 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -319,8 +319,7 @@ SYM_CODE_START_NOALIGN(vc_boot_ghcb) movq %rsp, %rdi movq ORIG_RAX(%rsp), %rsi movq initial_vc_handler(%rip), %rax - ANNOTATE_RETPOLINE_SAFE - call *%rax + __call_ext_ptregs *%rax, annotate_retpoline_safe=1

/* Unwind pt_regs */ POP_REGS @@ -397,7 +396,7 @@ SYM_CODE_START_LOCAL(early_idt_handler_common) UNWIND_HINT_REGS

movq %rsp,%rdi /* RDI = pt_regs; RSI is already trapnr */ - call do_early_exception + call_ext_ptregs do_early_exception

decl early_recursion_flag(%rip) jmp restore_regs_and_return_to_kernel @@ -421,7 +420,7 @@ SYM_CODE_START_NOALIGN(vc_no_ghcb) /* Call C handler */ movq %rsp, %rdi movq ORIG_RAX(%rsp), %rsi - call do_vc_no_ghcb + call_ext_ptregs do_vc_no_ghcb

/* Unwind pt_regs */ POP_REGS diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index c32d01803e26..3e6bea196366 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -32,6 +32,7 @@ #include <asm/pgtable_areas.h> /* VMALLOC_START, ... */ #include <asm/kvm_para.h> /* kvm_handle_async_pf */ #include <asm/vdso.h> /* fixup_vdso_exception() */ +#include <asm/pks.h>

#define CREATE_TRACE_POINTS #include <asm/trace/exceptions.h> @@ -547,6 +548,8 @@ show_fault_oops(struct pt_regs *regs, unsigned long error_code, unsigned long ad (error_code & X86_PF_PK) ? "protection keys violation" : "permissions violation");

+ show_extended_regs_oops(regs, error_code); + if (!(error_code & X86_PF_USER) && user_mode(regs)) { struct desc_ptr idt, gdt; u16 ldtr, tr; diff --git a/include/linux/pkeys.h b/include/linux/pkeys.h index 2955ba976048..a3d17a8e4e81 100644 --- a/include/linux/pkeys.h +++ b/include/linux/pkeys.h @@ -50,4 +50,21 @@ static inline void copy_init_pkru_to_fpregs(void)

#endif /* ! CONFIG_ARCH_HAS_PKEYS */

+ +#ifdef CONFIG_ARCH_ENABLE_SUPERVISOR_PKEYS + +void pkrs_save_set_irq(struct pt_regs *regs, u32 val); +void pkrs_restore_irq(struct pt_regs *regs); + +#else /* !CONFIG_ARCH_ENABLE_SUPERVISOR_PKEYS */ + +#ifndef INIT_PKRS_VALUE +#define INIT_PKRS_VALUE 0 +#endif + +static inline void pkrs_save_set_irq(struct pt_regs *regs, u32 val) { } +static inline void pkrs_restore_irq(struct pt_regs *regs) { } + +#endif /* CONFIG_ARCH_ENABLE_SUPERVISOR_PKEYS */ + #endif /* _LINUX_PKEYS_H */ diff --git a/kernel/entry/common.c b/kernel/entry/common.c index 8442e5c9cfa2..b50bcc2d3ea5 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -5,6 +5,7 @@ #include <linux/highmem.h> #include <linux/livepatch.h> #include <linux/audit.h> +#include <linux/pkeys.h>

#include "common.h"

@@ -363,7 +364,7 @@ noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) instrumentation_end();

ret.exit_rcu = true; - return ret; + goto done; }

/* @@ -378,6 +379,8 @@ noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) trace_hardirqs_off_finish(); instrumentation_end();

+done: + pkrs_save_set_irq(regs, INIT_PKRS_VALUE); return ret; }

@@ -403,7 +406,12 @@ noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state) /* Check whether this returns to user mode */ if (user_mode(regs)) { irqentry_exit_to_user_mode(regs); - } else if (!regs_irqs_disabled(regs)) { + return; + } + + pkrs_restore_irq(regs); + + if (!regs_irqs_disabled(regs)) { /* * If RCU was not watching on entry this needs to be done * carefully and needs the same ordering of lockdep/tracing @@ -457,11 +465,13 @@ irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs) ftrace_nmi_enter(); instrumentation_end();

+ pkrs_save_set_irq(regs, INIT_PKRS_VALUE); return irq_state; }

void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state) { + pkrs_restore_irq(regs); instrumentation_begin(); ftrace_nmi_exit(); if (irq_state.lockdep) {

From: Ira Weiny ira.weiny@intel.com

The core PKS functionality provides an interface for kernel users to reserve keys to their domains set up the page tables with those keys and control access to those domains when needed.

Define test code which exercises the core functionality of PKS via a debugfs entry. Basic checks can be triggered on boot with a kernel command line option while both basic and preemption checks can be triggered with separate debugfs values.

debugfs controls are:

'0' -- Run access tests with a single pkey '1' -- Set up the pkey register with no access for the pkey allocated to this fd '2' -- Check that the pkey register updated in '1' is still the same. (To be used after a forced context switch.) '3' -- Allocate all pkeys possible and run tests on each pkey allocated. DEFAULT when run at boot.

Closing the fd will cleanup and release the pkey, therefore to exercise context switch testing a user space program is provided in:

.../tools/testing/selftests/x86/test_pks.c

Reviewed-by: Dan Williams dan.j.williams@intel.com Reviewed-by: Dave Hansen dave.hansen@linux.intel.com Co-developed-by: Fenghua Yu fenghua.yu@intel.com Signed-off-by: Fenghua Yu fenghua.yu@intel.com Signed-off-by: Ira Weiny ira.weiny@intel.com

--- Changes from V3 Add test into ARCH_ENABLE_SUPERVISOR_PKEYS Fix allocate context error handling Callback must now take pt_regs instead of irq_state Use pipes to ensure code switches contexts Add more verbose output Add --debug opt to trigger more kernel debug output Reduce kernel output by default Use #defines for the various options Add ability to chose cpu for testing Work out how to make pkrs_cache global when CONFIG_PKS_TEST=y Comments from Dan Williams: Remove walk_table in favor of follow_pte Adjust for new MASK and SHIFT macros Remove unneeded pkey.h header Handle_pks_testing -> handle_pks_test Retain static pkrs_cache when not test s/PKS_TESTING/PKS_TEST/ Put pks_test_callback declaration in pks_common.h Don't export pks_test_callback Add comment explaining context creation Remove module boilerplate

Changes for V2 Fix compilation errors

Changes for V1 Update for new pks_key_alloc()

Changes from RFC V3 Comments from Dave Hansen clean up whitespace dmanage Clean up Kconfig help Clean up user test error output s/pks_mknoaccess/pks_mk_noaccess/ s/pks_mkread/pks_mk_readonly/ s/pks_mkrdwr/pks_mk_readwrite/ Comments from Jing Han Remove duplicate stdio.h --- Documentation/core-api/protection-keys.rst | 5 +- arch/x86/include/asm/pks.h | 19 + arch/x86/mm/fault.c | 16 + arch/x86/mm/pkeys.c | 2 +- lib/Kconfig.debug | 11 + lib/Makefile | 3 + lib/pks/Makefile | 3 + lib/pks/pks_test.c | 693 +++++++++++++++++++++ mm/Kconfig | 3 +- tools/testing/selftests/x86/Makefile | 3 +- tools/testing/selftests/x86/test_pks.c | 150 +++++ 11 files changed, 904 insertions(+), 4 deletions(-) create mode 100644 lib/pks/Makefile create mode 100644 lib/pks/pks_test.c create mode 100644 tools/testing/selftests/x86/test_pks.c

diff --git a/Documentation/core-api/protection-keys.rst b/Documentation/core-api/protection-keys.rst index 5b1b90d8bdab..f1245eb172c7 100644 --- a/Documentation/core-api/protection-keys.rst +++ b/Documentation/core-api/protection-keys.rst @@ -121,7 +121,8 @@ PTE adds this additional protection to the page.

Kernel users intending to use PKS support should check (depend on) ARCH_HAS_SUPERVISOR_PKEYS and add their config to ARCH_ENABLE_SUPERVISOR_PKEYS -to turn on this support within the core. +to turn on this support within the core. See the test configuration option +'PKS_TEST' for an example.

int pks_key_alloc(const char * const pkey_user); #define PAGE_KERNEL_PKEY(pkey) @@ -171,3 +172,5 @@ text: affected by PKRU register will not execute (even transiently) until all prior executions of WRPKRU have completed execution and updated the PKRU register. + +Example code can be found in lib/pks/pks_test.c diff --git a/arch/x86/include/asm/pks.h b/arch/x86/include/asm/pks.h index 2fff4ac484b9..67c3bc782c32 100644 --- a/arch/x86/include/asm/pks.h +++ b/arch/x86/include/asm/pks.h @@ -32,4 +32,23 @@ static inline void show_extended_regs_oops(struct pt_regs *regs,

#endif /* CONFIG_ARCH_ENABLE_SUPERVISOR_PKEYS */

+ +#ifdef CONFIG_PKS_TEST + +#define __static_or_pks_test + +bool handle_pks_test(unsigned long hw_error_code, struct pt_regs *regs); +bool pks_test_callback(struct pt_regs *regs); + +#else /* !CONFIG_PKS_TEST */ + +#define __static_or_pks_test static + +static inline bool handle_pks_test(unsigned long hw_error_code, struct pt_regs *regs) +{ + return false; +} + +#endif /* CONFIG_PKS_TEST */ + #endif /* _ASM_X86_PKS_H */ diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 3e6bea196366..1d59dc9c850e 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -1134,6 +1134,19 @@ bool fault_in_kernel_space(unsigned long address) return address >= TASK_SIZE_MAX; }

+#ifdef CONFIG_PKS_TEST +bool handle_pks_test(unsigned long hw_error_code, struct pt_regs *regs) +{ + /* + * If we get a protection key exception it could be because we + * are running the PKS test. If so, pks_test_callback() will + * clear the protection mechanism and return true to indicate + * the fault was handled. + */ + return (hw_error_code & X86_PF_PK) && pks_test_callback(regs); +} +#endif + /* * Called for all faults where 'address' is part of the kernel address * space. Might get called for faults that originate from *code* that @@ -1154,6 +1167,9 @@ do_kern_addr_fault(struct pt_regs *regs, unsigned long hw_error_code, WARN_ON_ONCE(!cpu_feature_enabled(X86_FEATURE_PKS) && (hw_error_code & X86_PF_PK));

+ if (handle_pks_test(hw_error_code, regs)) + return; + #ifdef CONFIG_X86_32 /* * We can fault-in kernel-space virtual memory on-demand. The diff --git a/arch/x86/mm/pkeys.c b/arch/x86/mm/pkeys.c index 47d29707ac39..2dd1feeff9f6 100644 --- a/arch/x86/mm/pkeys.c +++ b/arch/x86/mm/pkeys.c @@ -236,7 +236,7 @@ u32 update_pkey_val(u32 pk_reg, int pkey, unsigned int flags)

#ifdef CONFIG_ARCH_ENABLE_SUPERVISOR_PKEYS

-static DEFINE_PER_CPU(u32, pkrs_cache); +__static_or_pks_test DEFINE_PER_CPU(u32, pkrs_cache);

/* * write_pkrs() optimizes MSR writes by maintaining a per cpu cache which can diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 2779c29d9981..b7728ed139f9 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2535,6 +2535,17 @@ config HYPERV_TESTING help Select this option to enable Hyper-V vmbus testing.

+config PKS_TEST + bool "PKey (S)upervisor testing" + depends on ARCH_HAS_SUPERVISOR_PKEYS + help + Select this option to enable testing of PKS core software and + hardware. The PKS core provides a mechanism to allocate keys as well + as maintain the protection settings across context switches. + Answer N if you don't know what supervisor keys are. + + If unsure, say N. + endmenu # "Kernel Testing and Coverage"

source "Documentation/Kconfig" diff --git a/lib/Makefile b/lib/Makefile index b5307d3eec1a..3606b97e12e0 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -354,3 +354,6 @@ obj-$(CONFIG_BITS_TEST) += test_bits.o obj-$(CONFIG_CMDLINE_KUNIT_TEST) += cmdline_kunit.o

obj-$(CONFIG_GENERIC_LIB_DEVMEM_IS_ALLOWED) += devmem_is_allowed.o + +# PKS test +obj-y += pks/ diff --git a/lib/pks/Makefile b/lib/pks/Makefile new file mode 100644 index 000000000000..9daccba4f7c4 --- /dev/null +++ b/lib/pks/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_PKS_TEST) += pks_test.o diff --git a/lib/pks/pks_test.c b/lib/pks/pks_test.c new file mode 100644 index 000000000000..ca308f3ff5aa --- /dev/null +++ b/lib/pks/pks_test.c @@ -0,0 +1,693 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright(c) 2020 Intel Corporation. All rights reserved. + * + * Implement PKS testing + * Access to run this test can be with a command line parameter + * ("pks-test-on-boot") or more detailed tests can be triggered through: + * + * /sys/kernel/debug/x86/run_pks + * + * debugfs controls are: + * + * '0' -- Run access tests with a single pkey + * + * '1' -- Set up the pkey register with no access for the pkey allocated to + * this fd + * '2' -- Check that the pkey register updated in '1' is still the same. (To + * be used after a forced context switch.) + * + * '3' -- Allocate all pkeys possible and run tests on each pkey allocated. + * DEFAULT when run at boot. + * + * Closing the fd will cleanup and release the pkey. + * + * A companion user space program is provided in: + * + * .../tools/testing/selftests/x86/test_pks.c + * + * which will better test the context switching. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/debugfs.h> +#include <linux/delay.h> +#include <linux/entry-common.h> +#include <linux/fs.h> +#include <linux/list.h> +#include <linux/mman.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/percpu-defs.h> +#include <linux/pgtable.h> +#include <linux/pkeys.h> +#include <linux/slab.h> +#include <linux/vmalloc.h> + +#include <asm/ptrace.h> /* for struct pt_regs */ +#include <asm/pkeys_common.h> +#include <asm/processor.h> +#include <asm/pks.h> + +#define PKS_TEST_MEM_SIZE (PAGE_SIZE) + +#define RUN_ALLOCATE "0" +#define ARM_CTX_SWITCH "1" +#define CHECK_CTX_SWITCH "2" +#define RUN_ALLOCATE_ALL "3" +#define RUN_ALLOCATE_DEBUG "4" +#define RUN_ALLOCATE_ALL_DEBUG "5" +#define RUN_CRASH_TEST "9" + +DECLARE_PER_CPU(u32, pkrs_cache); + +/* + * run_on_boot default '= false' which checkpatch complains about initializing; + * so we don't + */ +static bool run_on_boot; +static struct dentry *pks_test_dentry; +static bool run_9; + +/* + * We must lock the following globals for brief periods while the fault handler + * checks/updates them. + */ +static DEFINE_SPINLOCK(test_lock); +static int test_armed_key; +static unsigned long prev_cnt; +static unsigned long fault_cnt; + +struct pks_test_ctx { + bool pass; + bool pks_cpu_enabled; + bool debug; + int pkey; + char data[64]; +}; +static struct pks_test_ctx *test_exception_ctx; + +static bool check_pkey_val(u32 pk_reg, int pkey, u32 expected) +{ + pk_reg = (pk_reg & PKR_PKEY_MASK(pkey)) >> PKR_PKEY_SHIFT(pkey); + return (pk_reg == expected); +} + +/* + * Check if the register @pkey value matches @expected value + * + * Both the cached and actual MSR must match. + */ +static bool check_pkrs(int pkey, u32 expected) +{ + bool ret = true; + u64 pkrs; + u32 *tmp_cache; + + tmp_cache = get_cpu_ptr(&pkrs_cache); + if (!check_pkey_val(*tmp_cache, pkey, expected)) + ret = false; + put_cpu_ptr(tmp_cache); + + rdmsrl(MSR_IA32_PKRS, pkrs); + if (!check_pkey_val(pkrs, pkey, expected)) + ret = false; + + return ret; +} + +static void check_exception(u32 thread_pkrs) +{ + /* Check the thread saved state */ + if (!check_pkey_val(thread_pkrs, test_armed_key, PKEY_DISABLE_WRITE)) { + pr_err(" FAIL: checking ept_regs->thread_pkrs\n"); + test_exception_ctx->pass = false; + } + + /* Check the exception state */ + if (!check_pkrs(test_armed_key, PKEY_DISABLE_ACCESS)) { + pr_err(" FAIL: PKRS cache and MSR\n"); + test_exception_ctx->pass = false; + } + + /* + * Check we can update the value during exception without affecting the + * calling thread. The calling thread is checked after exception... + */ + pks_mk_readwrite(test_armed_key); + if (!check_pkrs(test_armed_key, 0)) { + pr_err(" FAIL: exception did not change register to 0\n"); + test_exception_ctx->pass = false; + } + pks_mk_noaccess(test_armed_key); + if (!check_pkrs(test_armed_key, PKEY_DISABLE_ACCESS)) { + pr_err(" FAIL: exception did not change register to 0x%x\n", + PKEY_DISABLE_ACCESS); + test_exception_ctx->pass = false; + } +} + +/** + * pks_test_callback() is exported so that the fault handler can detect + * and report back status of intentional faults. + * + * NOTE: It clears the protection key from the page such that the fault handler + * will not re-trigger. + */ +bool pks_test_callback(struct pt_regs *regs) +{ + struct extended_pt_regs *ept_regs = extended_pt_regs(regs); + bool armed = (test_armed_key != 0); + + if (test_exception_ctx) { + check_exception(ept_regs->thread_pkrs); + /* + * We stop this check within the exception because the + * fault handler clean up code will call us 2x while checking + * the PMD entry and we don't need to check this again + */ + test_exception_ctx = NULL; + } + + if (armed) { + /* Enable read and write to stop faults */ + ept_regs->thread_pkrs = update_pkey_val(ept_regs->thread_pkrs, + test_armed_key, 0); + fault_cnt++; + } + + return armed; +} + +static bool exception_caught(void) +{ + bool ret = (fault_cnt != prev_cnt); + + prev_cnt = fault_cnt; + return ret; +} + +static void report_pkey_settings(void *info) +{ + u8 pkey; + unsigned long long msr = 0; + unsigned int cpu = smp_processor_id(); + struct pks_test_ctx *ctx = info; + + rdmsrl(MSR_IA32_PKRS, msr); + + pr_info("for CPU %d : 0x%llx\n", cpu, msr); + + if (ctx->debug) { + for (pkey = 0; pkey < PKS_NUM_KEYS; pkey++) { + int ad, wd; + + ad = (msr >> PKR_PKEY_SHIFT(pkey)) & PKEY_DISABLE_ACCESS; + wd = (msr >> PKR_PKEY_SHIFT(pkey)) & PKEY_DISABLE_WRITE; + pr_info(" %u: A:%d W:%d\n", pkey, ad, wd); + } + } +} + +enum pks_access_mode { + PKS_TEST_NO_ACCESS, + PKS_TEST_RDWR, + PKS_TEST_RDONLY +}; + +static char *get_mode_str(enum pks_access_mode mode) +{ + switch (mode) { + case PKS_TEST_NO_ACCESS: + return "No Access"; + case PKS_TEST_RDWR: + return "Read Write"; + case PKS_TEST_RDONLY: + return "Read Only"; + default: + pr_err("BUG in test invalid mode\n"); + break; + } + + return ""; +} + +struct pks_access_test { + enum pks_access_mode mode; + bool write; + bool exception; +}; + +static struct pks_access_test pkey_test_ary[] = { + /* disable both */ + { PKS_TEST_NO_ACCESS, true, true }, + { PKS_TEST_NO_ACCESS, false, true }, + + /* enable both */ + { PKS_TEST_RDWR, true, false }, + { PKS_TEST_RDWR, false, false }, + + /* enable read only */ + { PKS_TEST_RDONLY, true, true }, + { PKS_TEST_RDONLY, false, false }, +}; + +static int test_it(struct pks_test_ctx *ctx, struct pks_access_test *test, void *ptr) +{ + bool exception; + int ret = 0; + + spin_lock(&test_lock); + WRITE_ONCE(test_armed_key, ctx->pkey); + + if (test->write) + memcpy(ptr, ctx->data, 8); + else + memcpy(ctx->data, ptr, 8); + + exception = exception_caught(); + + WRITE_ONCE(test_armed_key, 0); + spin_unlock(&test_lock); + + if (test->exception != exception) { + pr_err("pkey test FAILED: mode %s; write %s; exception %s != %s\n", + get_mode_str(test->mode), + test->write ? "TRUE" : "FALSE", + test->exception ? "TRUE" : "FALSE", + exception ? "TRUE" : "FALSE"); + ret = -EFAULT; + } + + return ret; +} + +static int run_access_test(struct pks_test_ctx *ctx, + struct pks_access_test *test, + void *ptr) +{ + switch (test->mode) { + case PKS_TEST_NO_ACCESS: + pks_mk_noaccess(ctx->pkey); + break; + case PKS_TEST_RDWR: + pks_mk_readwrite(ctx->pkey); + break; + case PKS_TEST_RDONLY: + pks_mk_readonly(ctx->pkey); + break; + default: + pr_err("BUG in test invalid mode\n"); + break; + } + + return test_it(ctx, test, ptr); +} + +static void *alloc_test_page(int pkey) +{ + return __vmalloc_node_range(PKS_TEST_MEM_SIZE, 1, VMALLOC_START, VMALLOC_END, + GFP_KERNEL, PAGE_KERNEL_PKEY(pkey), 0, + NUMA_NO_NODE, __builtin_return_address(0)); +} + +static void test_mem_access(struct pks_test_ctx *ctx) +{ + int i, rc; + u8 pkey; + void *ptr = NULL; + pte_t *ptep = NULL; + spinlock_t *ptl; + + ptr = alloc_test_page(ctx->pkey); + if (!ptr) { + pr_err("Failed to vmalloc page???\n"); + ctx->pass = false; + return; + } + + if (follow_pte(&init_mm, (unsigned long)ptr, &ptep, &ptl)) { + pr_err("Failed to walk table???\n"); + ctx->pass = false; + goto done; + } + + pkey = pte_flags_pkey(ptep->pte); + pr_info("ptep flags 0x%lx pkey %u\n", + (unsigned long)ptep->pte, pkey); + pte_unmap_unlock(ptep, ptl); + + if (pkey != ctx->pkey) { + pr_err("invalid pkey found: %u, test_pkey: %u\n", + pkey, ctx->pkey); + ctx->pass = false; + goto done; + } + + if (!ctx->pks_cpu_enabled) { + pr_err("not CPU enabled; skipping access tests...\n"); + ctx->pass = true; + goto done; + } + + for (i = 0; i < ARRAY_SIZE(pkey_test_ary); i++) { + rc = run_access_test(ctx, &pkey_test_ary[i], ptr); + + /* only save last error is fine */ + if (rc) + ctx->pass = false; + } + +done: + vfree(ptr); +} + +static void pks_run_test(struct pks_test_ctx *ctx) +{ + ctx->pass = true; + + pr_info("\n"); + pr_info("\n"); + pr_info(" ***** BEGIN: Testing (CPU enabled : %s) *****\n", + ctx->pks_cpu_enabled ? "TRUE" : "FALSE"); + + if (ctx->pks_cpu_enabled) + on_each_cpu(report_pkey_settings, ctx, 1); + + pr_info(" BEGIN: pkey %d Testing\n", ctx->pkey); + test_mem_access(ctx); + pr_info(" END: PAGE_KERNEL_PKEY Testing : %s\n", + ctx->pass ? "PASS" : "FAIL"); + + pr_info(" ***** END: Testing *****\n"); + pr_info("\n"); + pr_info("\n"); +} + +static ssize_t pks_read_file(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct pks_test_ctx *ctx = file->private_data; + char buf[32]; + unsigned int len; + + if (!ctx) + len = sprintf(buf, "not run\n"); + else + len = sprintf(buf, "%s\n", ctx->pass ? "PASS" : "FAIL"); + + return simple_read_from_buffer(user_buf, count, ppos, buf, len); +} + +static struct pks_test_ctx *alloc_ctx(const char *name) +{ + struct pks_test_ctx *ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + + if (!ctx) { + pr_err("Failed to allocate memory for test context\n"); + return ERR_PTR(-ENOMEM); + } + + ctx->pkey = pks_key_alloc(name); + if (ctx->pkey <= 0) { + pr_err("Failed to allocate memory for test context\n"); + kfree(ctx); + return ERR_PTR(-ENOMEM); + } + + ctx->pks_cpu_enabled = cpu_feature_enabled(X86_FEATURE_PKS); + sprintf(ctx->data, "%s", "DEADBEEF"); + return ctx; +} + +static void free_ctx(struct pks_test_ctx *ctx) +{ + pks_key_free(ctx->pkey); + kfree(ctx); +} + +static void run_exception_test(void) +{ + void *ptr = NULL; + bool pass = true; + struct pks_test_ctx *ctx; + + pr_info(" ***** BEGIN: exception checking\n"); + + ctx = alloc_ctx("Exception test"); + if (IS_ERR(ctx)) { + pr_err(" FAIL: no context\n"); + pass = false; + goto result; + } + ctx->pass = true; + + ptr = alloc_test_page(ctx->pkey); + if (!ptr) { + pr_err(" FAIL: no vmalloc page\n"); + pass = false; + goto free_context; + } + + pks_mk_readonly(ctx->pkey); + + spin_lock(&test_lock); + WRITE_ONCE(test_exception_ctx, ctx); + WRITE_ONCE(test_armed_key, ctx->pkey); + + memcpy(ptr, ctx->data, 8); + + if (!exception_caught()) { + pr_err(" FAIL: did not get an exception\n"); + pass = false; + } + + /* + * NOTE The exception code has to enable access (b00) to keep the + * fault from looping forever. So we don't see the write disabled + * restored but rather full access restored. Also note that as part + * of this test the exception callback attempted to disable access + * completely (b11) and so we ensure that we are seeing the proper + * thread value restored here. + */ + if (!check_pkrs(test_armed_key, 0)) { + pr_err(" FAIL: PKRS not restored\n"); + pass = false; + } + + if (!ctx->pass) + pass = false; + + WRITE_ONCE(test_armed_key, 0); + spin_unlock(&test_lock); + + vfree(ptr); +free_context: + free_ctx(ctx); +result: + pr_info(" ***** END: exception checking : %s\n", + pass ? "PASS" : "FAIL"); +} + +static void run_all(bool debug) +{ + struct pks_test_ctx *ctx[PKS_NUM_KEYS]; + static char name[PKS_NUM_KEYS][64]; + int i; + + for (i = 1; i < PKS_NUM_KEYS; i++) { + sprintf(name[i], "pks ctx %d", i); + ctx[i] = alloc_ctx((const char *)name[i]); + if (!IS_ERR(ctx[i])) + ctx[i]->debug = debug; + } + + for (i = 1; i < PKS_NUM_KEYS; i++) { + if (!IS_ERR(ctx[i])) + pks_run_test(ctx[i]); + } + + for (i = 1; i < PKS_NUM_KEYS; i++) { + if (!IS_ERR(ctx[i])) + free_ctx(ctx[i]); + } + + run_exception_test(); +} + +static void crash_it(void) +{ + struct pks_test_ctx *ctx; + void *ptr; + + pr_warn(" ***** BEGIN: Unhandled fault test *****\n"); + + ctx = alloc_ctx("crashing kernel\n"); + if (IS_ERR(ctx)) { + pr_err("Failed to allocate context???\n"); + return; + } + + ptr = alloc_test_page(ctx->pkey); + if (!ptr) { + pr_err("Failed to vmalloc page???\n"); + ctx->pass = false; + return; + } + + pks_mk_noaccess(ctx->pkey); + + spin_lock(&test_lock); + WRITE_ONCE(test_armed_key, 0); + /* This purposely faults */ + memcpy(ptr, ctx->data, 8); + spin_unlock(&test_lock); + + vfree(ptr); + free_ctx(ctx); +} + +static ssize_t pks_write_file(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + char buf[2]; + struct pks_test_ctx *ctx = file->private_data; + + if (copy_from_user(buf, user_buf, 1)) + return -EFAULT; + buf[1] = '\0'; + + /* + * WARNING: Test "9" will crash the kernel. + * + * So we arm the test and print a warning. A second "9" will run the + * test. + */ + if (!strcmp(buf, RUN_CRASH_TEST)) { + if (run_9) { + crash_it(); + run_9 = false; + } else { + pr_warn("CAUTION: Test 9 will crash in the kernel.\n"); + pr_warn(" Specify 9 a second time to run\n"); + pr_warn(" run any other test to clear\n"); + run_9 = true; + } + } else { + run_9 = false; + } + + /* + * Test "3" will test allocating all keys. Do it first without + * using "ctx". + */ + if (!strcmp(buf, RUN_ALLOCATE_ALL)) + run_all(false); + if (!strcmp(buf, RUN_ALLOCATE_ALL_DEBUG)) + run_all(true); + + /* + * This context is only required if the file is held open for the below + * tests. Otherwise the context just get's freed in pks_release_file. + */ + if (!ctx) { + ctx = alloc_ctx("pks test"); + if (IS_ERR(ctx)) + return -ENOMEM; + file->private_data = ctx; + } + + if (!strcmp(buf, RUN_ALLOCATE)) { + ctx->debug = false; + pks_run_test(ctx); + } + if (!strcmp(buf, RUN_ALLOCATE_DEBUG)) { + ctx->debug = true; + pks_run_test(ctx); + } + + /* start of context switch test */ + if (!strcmp(buf, ARM_CTX_SWITCH)) { + unsigned long reg_pkrs; + int access; + + /* Ensure a known state to test context switch */ + pks_mk_readwrite(ctx->pkey); + + rdmsrl(MSR_IA32_PKRS, reg_pkrs); + + access = (reg_pkrs >> PKR_PKEY_SHIFT(ctx->pkey)) & + PKEY_ACCESS_MASK; + pr_info("Context switch armed : pkey %d: 0x%x reg: 0x%lx\n", + ctx->pkey, access, reg_pkrs); + } + + /* After context switch msr should be restored */ + if (!strcmp(buf, CHECK_CTX_SWITCH) && ctx->pks_cpu_enabled) { + unsigned long reg_pkrs; + int access; + + rdmsrl(MSR_IA32_PKRS, reg_pkrs); + + access = (reg_pkrs >> PKR_PKEY_SHIFT(ctx->pkey)) & + PKEY_ACCESS_MASK; + if (access != 0) { + ctx->pass = false; + pr_err("Context switch check failed: pkey %d: 0x%x reg: 0x%lx\n", + ctx->pkey, access, reg_pkrs); + } else { + pr_err("Context switch check passed: pkey %d: 0x%x reg: 0x%lx\n", + ctx->pkey, access, reg_pkrs); + } + } + + return count; +} + +static int pks_release_file(struct inode *inode, struct file *file) +{ + struct pks_test_ctx *ctx = file->private_data; + + if (!ctx) + return 0; + + free_ctx(ctx); + return 0; +} + +static const struct file_operations fops_init_pks = { + .read = pks_read_file, + .write = pks_write_file, + .llseek = default_llseek, + .release = pks_release_file, +}; + +static int __init parse_pks_test_options(char *str) +{ + run_on_boot = true; + + return 0; +} +early_param("pks-test-on-boot", parse_pks_test_options); + +static int __init pks_test_init(void) +{ + if (cpu_feature_enabled(X86_FEATURE_PKS)) { + if (run_on_boot) + run_all(true); + + pks_test_dentry = debugfs_create_file("run_pks", 0600, arch_debugfs_dir, + NULL, &fops_init_pks); + } + + return 0; +} +late_initcall(pks_test_init); + +static void __exit pks_test_exit(void) +{ + debugfs_remove(pks_test_dentry); + pr_info("test exit\n"); +} diff --git a/mm/Kconfig b/mm/Kconfig index c7d1fc780358..463e95ea0df1 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -811,7 +811,8 @@ config ARCH_HAS_PKEYS config ARCH_HAS_SUPERVISOR_PKEYS bool config ARCH_ENABLE_SUPERVISOR_PKEYS - bool + def_bool y + depends on PKS_TEST

config PERCPU_STATS bool "Collect percpu memory statistics" diff --git a/tools/testing/selftests/x86/Makefile b/tools/testing/selftests/x86/Makefile index 333980375bc7..32fe0414c6af 100644 --- a/tools/testing/selftests/x86/Makefile +++ b/tools/testing/selftests/x86/Makefile @@ -13,7 +13,8 @@ CAN_BUILD_WITH_NOPIE := $(shell ./check_cc.sh $(CC) trivial_program.c -no-pie) TARGETS_C_BOTHBITS := single_step_syscall sysret_ss_attrs syscall_nt test_mremap_vdso \ check_initial_reg_state sigreturn iopl ioperm \ test_vsyscall mov_ss_trap \ - syscall_arg_fault fsgsbase_restore + syscall_arg_fault fsgsbase_restore test_pks + TARGETS_C_32BIT_ONLY := entry_from_vm86 test_syscall_vdso unwind_vdso \ test_FCMOV test_FCOMI test_FISTTP \ vdso_restorer diff --git a/tools/testing/selftests/x86/test_pks.c b/tools/testing/selftests/x86/test_pks.c new file mode 100644 index 000000000000..62146cd59eb5 --- /dev/null +++ b/tools/testing/selftests/x86/test_pks.c @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#define _GNU_SOURCE +#include <sched.h> +#include <stdlib.h> +#include <getopt.h> +#include <unistd.h> +#include <assert.h> +#include <stdio.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <string.h> +#include <stdbool.h> + +#define PKS_TEST_FILE "/sys/kernel/debug/x86/run_pks" + +#define RUN_ALLOCATE "0" +#define SETUP_CTX_SWITCH "1" +#define CHECK_CTX_SWITCH "2" +#define RUN_ALLOCATE_ALL "3" +#define RUN_ALLOCATE_DEBUG "4" +#define RUN_ALLOCATE_ALL_DEBUG "5" +#define RUN_CRASH_TEST "9" + +int main(int argc, char *argv[]) +{ + cpu_set_t cpuset; + char result[32]; + pid_t pid; + int fd; + int setup_done[2]; + int switch_done[2]; + int cpu = 0; + int rc = 0; + int c; + bool debug = false; + + while (1) { + int option_index = 0; + static struct option long_options[] = { + {"debug", no_argument, 0, 0 }, + {0, 0, 0, 0 } + }; + + c = getopt_long(argc, argv, "", long_options, &option_index); + if (c == -1) + break; + + switch (c) { + case 0: + debug = true; + break; + } + } + + if (optind < argc) + cpu = strtoul(argv[optind], NULL, 0); + + if (cpu >= sysconf(_SC_NPROCESSORS_ONLN)) { + printf("CPU %d is invalid\n", cpu); + cpu = sysconf(_SC_NPROCESSORS_ONLN) - 1; + printf(" running on max CPU: %d\n", cpu); + } + + CPU_ZERO(&cpuset); + CPU_SET(cpu, &cpuset); + /* Two processes run on CPU 0 so that they go through context switch. */ + sched_setaffinity(getpid(), sizeof(cpu_set_t), &cpuset); + + if (pipe(setup_done)) + printf("Failed to create pipe\n"); + if (pipe(switch_done)) + printf("Failed to create pipe\n"); + + pid = fork(); + if (pid == 0) { + char done = 'y'; + + fd = open(PKS_TEST_FILE, O_RDWR); + if (fd < 0) { + printf("cannot open %s\n", PKS_TEST_FILE); + return -1; + } + + cpu = sched_getcpu(); + printf("Child running on cpu %d...\n", cpu); + + /* Allocate test_pkey1 and run test. */ + if (debug) + write(fd, RUN_ALLOCATE_DEBUG, 1); + else + write(fd, RUN_ALLOCATE, 1); + + /* Arm for context switch test */ + write(fd, SETUP_CTX_SWITCH, 1); + + printf(" tell parent to go\n"); + write(setup_done[1], &done, sizeof(done)); + + /* Context switch out... */ + printf(" Waiting for parent...\n"); + read(switch_done[0], &done, sizeof(done)); + + /* Check msr restored */ + printf("Checking result\n"); + write(fd, CHECK_CTX_SWITCH, 1); + + read(fd, result, 10); + printf(" #PF, context switch, pkey allocation and free tests: %s\n", result); + if (!strncmp(result, "PASS", 10)) { + rc = -1; + done = 'F'; + } + + /* Signal result */ + write(setup_done[1], &done, sizeof(done)); + } else { + char done = 'y'; + + read(setup_done[0], &done, sizeof(done)); + cpu = sched_getcpu(); + printf("Parent running on cpu %d\n", cpu); + + fd = open(PKS_TEST_FILE, O_RDWR); + if (fd < 0) { + printf("cannot open %s\n", PKS_TEST_FILE); + return -1; + } + + /* run test with alternate pkey */ + if (debug) + write(fd, RUN_ALLOCATE_DEBUG, 1); + else + write(fd, RUN_ALLOCATE, 1); + + /* Signal child we are done. */ + printf(" Telling child we are done.\n"); + write(switch_done[1], &done, sizeof(done)); + + /* Wait for result */ + read(setup_done[0], &done, sizeof(done)); + if (done == 'F') + rc = -1; + } + + close(fd); + + return rc; +}