From: Ira Weiny ira.weiny@intel.com

Protection Keys User (PKU) and Protection Keys Supervisor (PKS) work in similar fashions and can share common defines. Specifically PKS and PKU each have:

1. A single control register 2. The same number of keys 3. The same number of bits in the register per key 4. Access and Write disable in the same bit locations

That means that we can share all the macros that synthesize and manipulate register values between the two features. Normally, these macros would be put in asm/pkeys.h to be used internally and externally to the arch code. However, the defines are required in pgtable.h and inclusion of pkeys.h in that header creates complex dependencies which are best resolved in a separate header.

Share these defines by moving them into a new header, change their names to reflect the common use, and include the header where needed.

Reviewed-by: Dave Hansen dave.hansen@linux.intel.com Signed-off-by: Ira Weiny ira.weiny@intel.com

--- NOTE: The initialization of init_pkru_value cause checkpatch errors because of the space after the '(' in the macros. We leave this as is because it is more readable in this format. And it was existing code.

--- Changes from RFC V3 Per Dave Hansen Update commit message Add comment to PKR_AD_KEY macro --- arch/x86/include/asm/pgtable.h | 13 ++++++------- arch/x86/include/asm/pkeys.h | 2 ++ arch/x86/include/asm/pkeys_common.h | 15 +++++++++++++++ arch/x86/kernel/fpu/xstate.c | 8 ++++---- arch/x86/mm/pkeys.c | 14 ++++++-------- 5 files changed, 33 insertions(+), 19 deletions(-) create mode 100644 arch/x86/include/asm/pkeys_common.h

diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index a02c67291cfc..bfbfb951fe65 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -1360,9 +1360,7 @@ static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd) } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */

-#define PKRU_AD_BIT 0x1 -#define PKRU_WD_BIT 0x2 -#define PKRU_BITS_PER_PKEY 2 +#include <asm/pkeys_common.h>

#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS extern u32 init_pkru_value; @@ -1372,18 +1370,19 @@ extern u32 init_pkru_value;

static inline bool __pkru_allows_read(u32 pkru, u16 pkey) { - int pkru_pkey_bits = pkey * PKRU_BITS_PER_PKEY; - return !(pkru & (PKRU_AD_BIT << pkru_pkey_bits)); + int pkru_pkey_bits = pkey * PKR_BITS_PER_PKEY; + + return !(pkru & (PKR_AD_BIT << pkru_pkey_bits)); }

static inline bool __pkru_allows_write(u32 pkru, u16 pkey) { - int pkru_pkey_bits = pkey * PKRU_BITS_PER_PKEY; + int pkru_pkey_bits = pkey * PKR_BITS_PER_PKEY; /* * Access-disable disables writes too so we need to check * both bits here. */ - return !(pkru & ((PKRU_AD_BIT|PKRU_WD_BIT) << pkru_pkey_bits)); + return !(pkru & ((PKR_AD_BIT|PKR_WD_BIT) << pkru_pkey_bits)); }

static inline u16 pte_flags_pkey(unsigned long pte_flags) diff --git a/arch/x86/include/asm/pkeys.h b/arch/x86/include/asm/pkeys.h index 2ff9b98812b7..f9feba80894b 100644 --- a/arch/x86/include/asm/pkeys.h +++ b/arch/x86/include/asm/pkeys.h @@ -2,6 +2,8 @@ #ifndef _ASM_X86_PKEYS_H #define _ASM_X86_PKEYS_H

+#include <asm/pkeys_common.h> + #define ARCH_DEFAULT_PKEY 0

/* diff --git a/arch/x86/include/asm/pkeys_common.h b/arch/x86/include/asm/pkeys_common.h new file mode 100644 index 000000000000..737d916f476c --- /dev/null +++ b/arch/x86/include/asm/pkeys_common.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_PKEYS_INTERNAL_H +#define _ASM_X86_PKEYS_INTERNAL_H + +#define PKR_AD_BIT 0x1 +#define PKR_WD_BIT 0x2 +#define PKR_BITS_PER_PKEY 2 + +/* + * Generate an Access-Disable mask for the given pkey. Several of these can be + * OR'd together to generate pkey register values. + */ +#define PKR_AD_KEY(pkey) (PKR_AD_BIT << ((pkey) * PKR_BITS_PER_PKEY)) + +#endif /*_ASM_X86_PKEYS_INTERNAL_H */ diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 5d8047441a0a..a99afc70cc0a 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -995,7 +995,7 @@ int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, unsigned long init_val) { u32 old_pkru; - int pkey_shift = (pkey * PKRU_BITS_PER_PKEY); + int pkey_shift = (pkey * PKR_BITS_PER_PKEY); u32 new_pkru_bits = 0;

/* @@ -1014,16 +1014,16 @@ int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,

/* Set the bits we need in PKRU: */ if (init_val & PKEY_DISABLE_ACCESS) - new_pkru_bits |= PKRU_AD_BIT; + new_pkru_bits |= PKR_AD_BIT; if (init_val & PKEY_DISABLE_WRITE) - new_pkru_bits |= PKRU_WD_BIT; + new_pkru_bits |= PKR_WD_BIT;

/* Shift the bits in to the correct place in PKRU for pkey: */ new_pkru_bits <<= pkey_shift;

/* Get old PKRU and mask off any old bits in place: */ old_pkru = read_pkru(); - old_pkru &= ~((PKRU_AD_BIT|PKRU_WD_BIT) << pkey_shift); + old_pkru &= ~((PKR_AD_BIT|PKR_WD_BIT) << pkey_shift);

/* Write old part along with new part: */ write_pkru(old_pkru | new_pkru_bits); diff --git a/arch/x86/mm/pkeys.c b/arch/x86/mm/pkeys.c index 8873ed1438a9..f5efb4007e74 100644 --- a/arch/x86/mm/pkeys.c +++ b/arch/x86/mm/pkeys.c @@ -111,19 +111,17 @@ int __arch_override_mprotect_pkey(struct vm_area_struct *vma, int prot, int pkey return vma_pkey(vma); }

-#define PKRU_AD_KEY(pkey) (PKRU_AD_BIT << ((pkey) * PKRU_BITS_PER_PKEY)) - /* * Make the default PKRU value (at execve() time) as restrictive * as possible. This ensures that any threads clone()'d early * in the process's lifetime will not accidentally get access * to data which is pkey-protected later on. */ -u32 init_pkru_value = PKRU_AD_KEY( 1) | PKRU_AD_KEY( 2) | PKRU_AD_KEY( 3) | - PKRU_AD_KEY( 4) | PKRU_AD_KEY( 5) | PKRU_AD_KEY( 6) | - PKRU_AD_KEY( 7) | PKRU_AD_KEY( 8) | PKRU_AD_KEY( 9) | - PKRU_AD_KEY(10) | PKRU_AD_KEY(11) | PKRU_AD_KEY(12) | - PKRU_AD_KEY(13) | PKRU_AD_KEY(14) | PKRU_AD_KEY(15); +u32 init_pkru_value = PKR_AD_KEY( 1) | PKR_AD_KEY( 2) | PKR_AD_KEY( 3) | + PKR_AD_KEY( 4) | PKR_AD_KEY( 5) | PKR_AD_KEY( 6) | + PKR_AD_KEY( 7) | PKR_AD_KEY( 8) | PKR_AD_KEY( 9) | + PKR_AD_KEY(10) | PKR_AD_KEY(11) | PKR_AD_KEY(12) | + PKR_AD_KEY(13) | PKR_AD_KEY(14) | PKR_AD_KEY(15);

/* * Called from the FPU code when creating a fresh set of FPU @@ -173,7 +171,7 @@ static ssize_t init_pkru_write_file(struct file *file, * up immediately if someone attempts to disable access * or writes to pkey 0. */ - if (new_init_pkru & (PKRU_AD_BIT|PKRU_WD_BIT)) + if (new_init_pkru & (PKR_AD_BIT|PKR_WD_BIT)) return -EINVAL;

WRITE_ONCE(init_pkru_value, new_init_pkru);

From: Fenghua Yu fenghua.yu@intel.com

Protection Keys for Supervisor pages (PKS) enables fast, hardware thread specific, manipulation of permission restrictions on supervisor page mappings. It uses the same mechanism of Protection Keys as those on User mappings but applies that mechanism to supervisor mappings using a supervisor specific MSR.

Kernel users can thus defines 'domains' of page mappings which have an extra level of protection beyond those specified in the supervisor page table entries.

Define ARCH_HAS_SUPERVISOR_PKEYS to distinguish this functionality from the existing ARCH_HAS_PKEYS and then enable PKS when configured and indicated by the CPU instance. While not strictly necessary in this patch, ARCH_HAS_SUPERVISOR_PKEYS separates this functionality through the patch series so it is introduced here.

Co-developed-by: Ira Weiny ira.weiny@intel.com Signed-off-by: Ira Weiny ira.weiny@intel.com Signed-off-by: Fenghua Yu fenghua.yu@intel.com

--- Changes since RFC V3 Per Dave Hansen Update comment Add X86_FEATURE_PKS to disabled-features.h Rebase based on latest TIP tree --- arch/x86/Kconfig | 1 + arch/x86/include/asm/cpufeatures.h | 1 + arch/x86/include/asm/disabled-features.h | 8 +++++++- arch/x86/include/uapi/asm/processor-flags.h | 2 ++ arch/x86/kernel/cpu/common.c | 13 +++++++++++++ mm/Kconfig | 2 ++ 6 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index f6946b81f74a..78c4c749c6a9 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1876,6 +1876,7 @@ config X86_INTEL_MEMORY_PROTECTION_KEYS depends on X86_64 && (CPU_SUP_INTEL || CPU_SUP_AMD) select ARCH_USES_HIGH_VMA_FLAGS select ARCH_HAS_PKEYS + select ARCH_HAS_SUPERVISOR_PKEYS help Memory Protection Keys provides a mechanism for enforcing page-based protections, but without requiring modification of the diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index dad350d42ecf..4deb580324e8 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -356,6 +356,7 @@ #define X86_FEATURE_MOVDIRI (16*32+27) /* MOVDIRI instruction */ #define X86_FEATURE_MOVDIR64B (16*32+28) /* MOVDIR64B instruction */ #define X86_FEATURE_ENQCMD (16*32+29) /* ENQCMD and ENQCMDS instructions */ +#define X86_FEATURE_PKS (16*32+31) /* Protection Keys for Supervisor pages */

/* AMD-defined CPU features, CPUID level 0x80000007 (EBX), word 17 */ #define X86_FEATURE_OVERFLOW_RECOV (17*32+ 0) /* MCA overflow recovery support */ diff --git a/arch/x86/include/asm/disabled-features.h b/arch/x86/include/asm/disabled-features.h index 5861d34f9771..82540f0c5b6c 100644 --- a/arch/x86/include/asm/disabled-features.h +++ b/arch/x86/include/asm/disabled-features.h @@ -44,6 +44,12 @@ # define DISABLE_OSPKE (1<<(X86_FEATURE_OSPKE & 31)) #endif /* CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS */

+#ifdef CONFIG_ARCH_HAS_SUPERVISOR_PKEYS +# define DISABLE_PKS 0 +#else +# define DISABLE_PKS (1<<(X86_FEATURE_PKS & 31)) +#endif + #ifdef CONFIG_X86_5LEVEL # define DISABLE_LA57 0 #else @@ -82,7 +88,7 @@ #define DISABLED_MASK14 0 #define DISABLED_MASK15 0 #define DISABLED_MASK16 (DISABLE_PKU|DISABLE_OSPKE|DISABLE_LA57|DISABLE_UMIP| \ - DISABLE_ENQCMD) + DISABLE_ENQCMD|DISABLE_PKS) #define DISABLED_MASK17 0 #define DISABLED_MASK18 0 #define DISABLED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 19) diff --git a/arch/x86/include/uapi/asm/processor-flags.h b/arch/x86/include/uapi/asm/processor-flags.h index bcba3c643e63..191c574b2390 100644 --- a/arch/x86/include/uapi/asm/processor-flags.h +++ b/arch/x86/include/uapi/asm/processor-flags.h @@ -130,6 +130,8 @@ #define X86_CR4_SMAP _BITUL(X86_CR4_SMAP_BIT) #define X86_CR4_PKE_BIT 22 /* enable Protection Keys support */ #define X86_CR4_PKE _BITUL(X86_CR4_PKE_BIT) +#define X86_CR4_PKS_BIT 24 /* enable Protection Keys for Supervisor */ +#define X86_CR4_PKS _BITUL(X86_CR4_PKS_BIT)

/* * x86-64 Task Priority Register, CR8 diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 35ad8480c464..6a9ca938d9a9 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1494,6 +1494,18 @@ static void validate_apic_and_package_id(struct cpuinfo_x86 *c) #endif }

+/* + * PKS is independent of PKU and either or both may be supported on a CPU. + * Configure PKS if the CPU supports the feature. + */ +static void setup_pks(void) +{ + if (!cpu_feature_enabled(X86_FEATURE_PKS)) + return; + + cr4_set_bits(X86_CR4_PKS); +} + /* * This does the hard work of actually picking apart the CPU stuff... */ @@ -1591,6 +1603,7 @@ static void identify_cpu(struct cpuinfo_x86 *c)

x86_init_rdrand(c); setup_pku(c); + setup_pks();

/* * Clear/Set all flags overridden by options, need do it diff --git a/mm/Kconfig b/mm/Kconfig index d42423f884a7..fc9ce7f65683 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -826,6 +826,8 @@ config ARCH_USES_HIGH_VMA_FLAGS bool config ARCH_HAS_PKEYS bool +config ARCH_HAS_SUPERVISOR_PKEYS + bool

config PERCPU_STATS bool "Collect percpu memory statistics"

From: Fenghua Yu fenghua.yu@intel.com

PKS allows kernel users to define domains of page mappings which have additional protections beyond the paging protections.

Add an API to allocate, use, and free a protection key which identifies such a domain. Export 5 new symbols pks_key_alloc(), pks_mknoaccess(), pks_mkread(), pks_mkrdwr(), and pks_key_free(). Add 2 new macros; PAGE_KERNEL_PKEY(key) and _PAGE_PKEY(pkey).

Update the protection key documentation to cover pkeys on supervisor pages.

Co-developed-by: Ira Weiny ira.weiny@intel.com Signed-off-by: Ira Weiny ira.weiny@intel.com Signed-off-by: Fenghua Yu fenghua.yu@intel.com

--- Changes from V1 Per Dave Hansen Add flags to pks_key_alloc() to help future proof the interface if/when the key space is exhausted.

Changes from RFC V3 Per Dave Hansen Put WARN_ON_ONCE in pks_key_free() s/pks_mknoaccess/pks_mk_noaccess/ s/pks_mkread/pks_mk_readonly/ s/pks_mkrdwr/pks_mk_readwrite/ Change return pks_key_alloc() to EOPNOTSUPP when not supported or configured Per Peter Zijlstra Remove unneeded preempt disable/enable --- Documentation/core-api/protection-keys.rst | 102 ++++++++++++++--- arch/x86/include/asm/pgtable_types.h | 12 ++ arch/x86/include/asm/pkeys.h | 11 ++ arch/x86/include/asm/pkeys_common.h | 4 + arch/x86/mm/pkeys.c | 126 +++++++++++++++++++++ include/linux/pgtable.h | 4 + include/linux/pkeys.h | 24 ++++ 7 files changed, 265 insertions(+), 18 deletions(-)

diff --git a/Documentation/core-api/protection-keys.rst b/Documentation/core-api/protection-keys.rst index ec575e72d0b2..c4e6c480562f 100644 --- a/Documentation/core-api/protection-keys.rst +++ b/Documentation/core-api/protection-keys.rst @@ -4,25 +4,33 @@ Memory Protection Keys ======================

-Memory Protection Keys for Userspace (PKU aka PKEYs) is a feature -which is found on Intel's Skylake (and later) "Scalable Processor" -Server CPUs. It will be available in future non-server Intel parts -and future AMD processors. - -For anyone wishing to test or use this feature, it is available in -Amazon's EC2 C5 instances and is known to work there using an Ubuntu -17.04 image. - Memory Protection Keys provides a mechanism for enforcing page-based protections, but without requiring modification of the page tables -when an application changes protection domains. It works by -dedicating 4 previously ignored bits in each page table entry to a -"protection key", giving 16 possible keys. +when an application changes protection domains. + +PKeys Userspace (PKU) is a feature which is found on Intel's Skylake "Scalable +Processor" Server CPUs and later. And It will be available in future +non-server Intel parts and future AMD processors. + +Future Intel processors will support Protection Keys for Supervisor pages +(PKS). + +For anyone wishing to test or use user space pkeys, it is available in Amazon's +EC2 C5 instances and is known to work there using an Ubuntu 17.04 image. + +pkeys work by dedicating 4 previously Reserved bits in each page table entry to +a "protection key", giving 16 possible keys. User and Supervisor pages are +treated separately. + +Protections for each page are controlled with per CPU registers for each type +of page User and Supervisor. Each of these 32 bit register stores two separate +bits (Access Disable and Write Disable) for each key.

-There is also a new user-accessible register (PKRU) with two separate -bits (Access Disable and Write Disable) for each key. Being a CPU -register, PKRU is inherently thread-local, potentially giving each -thread a different set of protections from every other thread. +For Userspace the register is user-accessible (rdpkru/wrpkru). For +Supervisor, the register (MSR_IA32_PKRS) is accessible only to the kernel. + +Being a CPU register, pkeys are inherently thread-local, potentially giving +each thread an independent set of protections from every other thread.

There are two new instructions (RDPKRU/WRPKRU) for reading and writing to the new register. The feature is only available in 64-bit mode, @@ -30,8 +38,11 @@ even though there is theoretically space in the PAE PTEs. These permissions are enforced on data access only and have no effect on instruction fetches.

-Syscalls -======== +For kernel space rdmsr/wrmsr are used to access the kernel MSRs. + + +Syscalls for user space keys +============================

There are 3 system calls which directly interact with pkeys::

@@ -98,3 +109,58 @@ with a read():: The kernel will send a SIGSEGV in both cases, but si_code will be set to SEGV_PKERR when violating protection keys versus SEGV_ACCERR when the plain mprotect() permissions are violated. + + +Kernel API for PKS support +========================== + +The following interface is used to allocate, use, and free a pkey which defines +a 'protection domain' within the kernel. Setting a pkey value in a supervisor +mapping adds that mapping to the protection domain. + + int pks_key_alloc(const char * const pkey_user, int flags); + #define PAGE_KERNEL_PKEY(pkey) + #define _PAGE_KEY(pkey) + void pks_mk_noaccess(int pkey); + void pks_mk_readonly(int pkey); + void pks_mk_readwrite(int pkey); + void pks_key_free(int pkey); + +pks_key_alloc() allocates keys dynamically to allow better use of the limited +key space. 'flags' alter the allocation based on the users need. Currently +they can request an exclusive key. + +Callers of pks_key_alloc() _must_ be prepared for it to fail and take +appropriate action. This is due mainly to the fact that PKS may not be +available on all arch's. Failure to check the return of pks_key_alloc() and +using any of the rest of the API is undefined. + +Kernel users must set the PTE permissions in the page table entries for the +mappings they want to protect. This can be done with PAGE_KERNEL_PKEY() or +_PAGE_KEY(). + +The pks_mk*() family of calls allows kernel users the ability to change the +protections for the domain identified by the pkey specified. 3 states are +available pks_mk_noaccess(), pks_mk_readonly(), and pks_mk_readwrite() which +set the access to none, read, and read/write respectively. + +Finally, pks_key_free() allows a user to return the key to the allocator for +use by others. + +The interface maintains pks_mk_noaccess() (Access Disabled (AD=1)) for all keys +not currently allocated. Therefore, the user can depend on access being +disabled when pks_key_alloc() returns a key and the user should remove mappings +from the domain (remove the pkey from the PTE) prior to calling pks_key_free(). + +It should be noted that the underlying WRMSR(MSR_IA32_PKRS) is not serializing +but still maintains ordering properties similar to WRPKRU. Thus it is safe to +immediately use a mapping when the pks_mk*() functions returns. + +The current SDM section on PKRS needs updating but should be the same as that +of WRPKRU. So to quote from the WRPKRU text: + + WRPKRU will never execute transiently. Memory accesses + affected by PKRU register will not execute (even transiently) + until all prior executions of WRPKRU have completed execution + and updated the PKRU register. + diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h index 816b31c68550..c9fdfbdcbbfb 100644 --- a/arch/x86/include/asm/pgtable_types.h +++ b/arch/x86/include/asm/pgtable_types.h @@ -73,6 +73,12 @@ _PAGE_PKEY_BIT2 | \ _PAGE_PKEY_BIT3)

+#ifdef CONFIG_ARCH_HAS_SUPERVISOR_PKEYS +#define _PAGE_PKEY(pkey) (_AT(pteval_t, pkey) << _PAGE_BIT_PKEY_BIT0) +#else +#define _PAGE_PKEY(pkey) (_AT(pteval_t, 0)) +#endif + #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) #define _PAGE_KNL_ERRATUM_MASK (_PAGE_DIRTY | _PAGE_ACCESSED) #else @@ -229,6 +235,12 @@ enum page_cache_mode { #define PAGE_KERNEL_IO __pgprot_mask(__PAGE_KERNEL_IO) #define PAGE_KERNEL_IO_NOCACHE __pgprot_mask(__PAGE_KERNEL_IO_NOCACHE)

+#ifdef CONFIG_ARCH_HAS_SUPERVISOR_PKEYS +#define PAGE_KERNEL_PKEY(pkey) __pgprot_mask(__PAGE_KERNEL | _PAGE_PKEY(pkey)) +#else +#define PAGE_KERNEL_PKEY(pkey) PAGE_KERNEL +#endif + #endif /* __ASSEMBLY__ */

/* xwr */ diff --git a/arch/x86/include/asm/pkeys.h b/arch/x86/include/asm/pkeys.h index 4526245b03e5..f84351b4ac7c 100644 --- a/arch/x86/include/asm/pkeys.h +++ b/arch/x86/include/asm/pkeys.h @@ -3,6 +3,7 @@ #define _ASM_X86_PKEYS_H

#include <asm/pkeys_common.h> +#include <asm-generic/mman-common.h>

#define ARCH_DEFAULT_PKEY 0

@@ -138,4 +139,14 @@ static inline int vma_pkey(struct vm_area_struct *vma)

u32 update_pkey_val(u32 pk_reg, int pkey, unsigned int flags);

+#ifdef CONFIG_ARCH_HAS_SUPERVISOR_PKEYS +int pks_key_alloc(const char *const pkey_user, int flags); +void pks_key_free(int pkey); + +void pks_mk_noaccess(int pkey); +void pks_mk_readonly(int pkey); +void pks_mk_readwrite(int pkey); + +#endif /* CONFIG_ARCH_HAS_SUPERVISOR_PKEYS */ + #endif /*_ASM_X86_PKEYS_H */ diff --git a/arch/x86/include/asm/pkeys_common.h b/arch/x86/include/asm/pkeys_common.h index 801a75615209..cd492c23b28c 100644 --- a/arch/x86/include/asm/pkeys_common.h +++ b/arch/x86/include/asm/pkeys_common.h @@ -26,6 +26,10 @@ PKR_AD_KEY(10) | PKR_AD_KEY(11) | PKR_AD_KEY(12) | \ PKR_AD_KEY(13) | PKR_AD_KEY(14) | PKR_AD_KEY(15))

+/* PKS supports 16 keys. Key 0 is reserved for the kernel. */ +#define PKS_KERN_DEFAULT_KEY 0 +#define PKS_NUM_KEYS 16 + #ifdef CONFIG_ARCH_HAS_SUPERVISOR_PKEYS void write_pkrs(u32 new_pkrs); #else diff --git a/arch/x86/mm/pkeys.c b/arch/x86/mm/pkeys.c index 76a62419c446..0dc77409957a 100644 --- a/arch/x86/mm/pkeys.c +++ b/arch/x86/mm/pkeys.c @@ -3,6 +3,9 @@ * Intel Memory Protection Keys management * Copyright (c) 2015, Intel Corporation. */ +#undef pr_fmt +#define pr_fmt(fmt) "x86/pkeys: " fmt + #include <linux/debugfs.h> /* debugfs_create_u32() */ #include <linux/mm_types.h> /* mm_struct, vma, etc... */ #include <linux/pkeys.h> /* PKEY_* */ @@ -231,6 +234,7 @@ u32 update_pkey_val(u32 pk_reg, int pkey, unsigned int flags)

return pk_reg; } +EXPORT_SYMBOL_GPL(update_pkey_val);

DEFINE_PER_CPU(u32, pkrs_cache);

@@ -262,3 +266,125 @@ void write_pkrs(u32 new_pkrs) } put_cpu_ptr(pkrs); } +EXPORT_SYMBOL_GPL(write_pkrs); + +/** + * Do not call this directly, see pks_mk*() below. + * + * @pkey: Key for the domain to change + * @protection: protection bits to be used + * + * Protection utilizes the same protection bits specified for User pkeys + * PKEY_DISABLE_ACCESS + * PKEY_DISABLE_WRITE + * + */ +static inline void pks_update_protection(int pkey, unsigned long protection) +{ + current->thread.saved_pkrs = update_pkey_val(current->thread.saved_pkrs, + pkey, protection); + write_pkrs(current->thread.saved_pkrs); +} + +/** + * PKS access control functions + * + * Change the access of the domain specified by the pkey. These are global + * updates. They only affects the current running thread. It is undefined and + * a bug for users to call this without having allocated a pkey and using it as + * pkey here. + * + * pks_mk_noaccess() + * Disable all access to the domain + * pks_mk_readonly() + * Make the domain Read only + * pks_mk_readwrite() + * Make the domain Read/Write + * + * @pkey the pkey for which the access should change. + * + */ +void pks_mk_noaccess(int pkey) +{ + pks_update_protection(pkey, PKEY_DISABLE_ACCESS); +} +EXPORT_SYMBOL_GPL(pks_mk_noaccess); + +void pks_mk_readonly(int pkey) +{ + pks_update_protection(pkey, PKEY_DISABLE_WRITE); +} +EXPORT_SYMBOL_GPL(pks_mk_readonly); + +void pks_mk_readwrite(int pkey) +{ + pks_update_protection(pkey, 0); +} +EXPORT_SYMBOL_GPL(pks_mk_readwrite); + +static const char pks_key_user0[] = "kernel"; + +/* Store names of allocated keys for debug. Key 0 is reserved for the kernel. */ +static const char *pks_key_users[PKS_NUM_KEYS] = { + pks_key_user0 +}; + +/* + * Each key is represented by a bit. Bit 0 is set for key 0 and reserved for + * its use. We use ulong for the bit operations but only 16 bits are used. + */ +static unsigned long pks_key_allocation_map = 1 << PKS_KERN_DEFAULT_KEY; + +/* + * pks_key_alloc - Allocate a PKS key + * @pkey_user: String stored for debugging of key exhaustion. The caller is + * responsible to maintain this memory until pks_key_free(). + * @flags: Flags to modify behavior: see pks_alloc_flags + * + * Returns: pkey if success + * -EOPNOTSUPP if pks is not supported or not enabled + * -ENOSPC if no keys are available (even for sharing) + */ +int pks_key_alloc(const char * const pkey_user, int flags) +{ + int nr; + + if (!cpu_feature_enabled(X86_FEATURE_PKS)) + return -EOPNOTSUPP; + + while (1) { + nr = find_first_zero_bit(&pks_key_allocation_map, PKS_NUM_KEYS); + if (nr >= PKS_NUM_KEYS) { + pr_info("Cannot allocate supervisor key for %s.\n", + pkey_user); + return -ENOSPC; + } + if (!test_and_set_bit_lock(nr, &pks_key_allocation_map)) + break; + } + + /* for debugging key exhaustion */ + pks_key_users[nr] = pkey_user; + + return nr; +} +EXPORT_SYMBOL_GPL(pks_key_alloc); + +/* + * pks_key_free - Free a previously allocate PKS key + * @pkey: Key to be free'ed + */ +void pks_key_free(int pkey) +{ + if (!cpu_feature_enabled(X86_FEATURE_PKS)) + return; + + if (WARN_ON_ONCE(pkey >= PKS_NUM_KEYS || pkey <= PKS_KERN_DEFAULT_KEY)) + return; + + /* Restore to default of no access */ + pks_mk_noaccess(pkey); + pks_key_users[pkey] = NULL; + __clear_bit(pkey, &pks_key_allocation_map); +} +EXPORT_SYMBOL_GPL(pks_key_free); diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 38c33eabea89..cd72d73e8e1c 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1402,6 +1402,10 @@ static inline bool arch_has_pfn_modify_check(void) # define PAGE_KERNEL_EXEC PAGE_KERNEL #endif

+#ifndef PAGE_KERNEL_PKEY +#define PAGE_KERNEL_PKEY(pkey) PAGE_KERNEL +#endif + /* * Page Table Modification bits for pgtbl_mod_mask. * diff --git a/include/linux/pkeys.h b/include/linux/pkeys.h index 2955ba976048..0959a4c0ca64 100644 --- a/include/linux/pkeys.h +++ b/include/linux/pkeys.h @@ -50,4 +50,28 @@ static inline void copy_init_pkru_to_fpregs(void)

#endif /* ! CONFIG_ARCH_HAS_PKEYS */

+#define PKS_FLAG_EXCLUSIVE 0x00 + +#ifndef CONFIG_ARCH_HAS_SUPERVISOR_PKEYS +static inline int pks_key_alloc(const char * const pkey_user, int flags) +{ + return -EOPNOTSUPP; +} +static inline void pks_key_free(int pkey) +{ +} +static inline void pks_mk_noaccess(int pkey) +{ + WARN_ON_ONCE(1); +} +static inline void pks_mk_readonly(int pkey) +{ + WARN_ON_ONCE(1); +} +static inline void pks_mk_readwrite(int pkey) +{ + WARN_ON_ONCE(1); +} +#endif /* ! CONFIG_ARCH_HAS_SUPERVISOR_PKEYS */ + #endif /* _LINUX_PKEYS_H */

From: Ira Weiny ira.weiny@intel.com

Currently struct irqentry_state_t only contains a single bool value which makes passing it by value is reasonable. However, future patches propose to add information to this struct, for example the PKRS register/thread state.

Adding information to irqentry_state_t makes passing by value less efficient. Therefore, change the entry/exit calls to pass irq_state by reference.

While at it, make the code easier to follow by changing all the usage sites to consistently use the variable name 'irq_state'.

Signed-off-by: Ira Weiny ira.weiny@intel.com

--- Changes from V1 From Thomas: Update commit message Further clean up Kernel doc and comments Missed some 'return' comments which are no longer valid

Changes from RFC V3 Clean up @irq_state comments Standardize on 'irq_state' for the state variable name Refactor based on new patch from Thomas Gleixner Also addresses Peter Zijlstra's comment --- arch/x86/entry/common.c | 8 ++++---- arch/x86/include/asm/idtentry.h | 25 ++++++++++++++---------- arch/x86/kernel/cpu/mce/core.c | 4 ++-- arch/x86/kernel/kvm.c | 6 +++--- arch/x86/kernel/nmi.c | 4 ++-- arch/x86/kernel/traps.c | 21 ++++++++++++-------- arch/x86/mm/fault.c | 6 +++--- include/linux/entry-common.h | 18 +++++++++-------- kernel/entry/common.c | 34 +++++++++++++-------------------- 9 files changed, 65 insertions(+), 61 deletions(-)

diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c index 18d8f17f755c..87dea56a15d2 100644 --- a/arch/x86/entry/common.c +++ b/arch/x86/entry/common.c @@ -259,9 +259,9 @@ __visible noinstr void xen_pv_evtchn_do_upcall(struct pt_regs *regs) { struct pt_regs *old_regs; bool inhcall; - irqentry_state_t state; + irqentry_state_t irq_state;

- state = irqentry_enter(regs); + irqentry_enter(regs, &irq_state); old_regs = set_irq_regs(regs);

instrumentation_begin(); @@ -271,13 +271,13 @@ __visible noinstr void xen_pv_evtchn_do_upcall(struct pt_regs *regs) set_irq_regs(old_regs);

inhcall = get_and_clear_inhcall(); - if (inhcall && !WARN_ON_ONCE(state.exit_rcu)) { + if (inhcall && !WARN_ON_ONCE(irq_state.exit_rcu)) { instrumentation_begin(); irqentry_exit_cond_resched(); instrumentation_end(); restore_inhcall(inhcall); } else { - irqentry_exit(regs, state); + irqentry_exit(regs, &irq_state); } } #endif /* CONFIG_XEN_PV */ diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h index 247a60a47331..282d2413b6a1 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -49,12 +49,13 @@ static __always_inline void __##func(struct pt_regs *regs); \ \ __visible noinstr void func(struct pt_regs *regs) \ { \ - irqentry_state_t state = irqentry_enter(regs); \ + irqentry_state_t irq_state; \ \ + irqentry_enter(regs, &irq_state); \ instrumentation_begin(); \ __##func (regs); \ instrumentation_end(); \ - irqentry_exit(regs, state); \ + irqentry_exit(regs, &irq_state); \ } \ \ static __always_inline void __##func(struct pt_regs *regs) @@ -96,12 +97,13 @@ static __always_inline void __##func(struct pt_regs *regs, \ __visible noinstr void func(struct pt_regs *regs, \ unsigned long error_code) \ { \ - irqentry_state_t state = irqentry_enter(regs); \ + irqentry_state_t irq_state; \ \ + irqentry_enter(regs, &irq_state); \ instrumentation_begin(); \ __##func (regs, error_code); \ instrumentation_end(); \ - irqentry_exit(regs, state); \ + irqentry_exit(regs, &irq_state); \ } \ \ static __always_inline void __##func(struct pt_regs *regs, \ @@ -192,15 +194,16 @@ static __always_inline void __##func(struct pt_regs *regs, u8 vector); \ __visible noinstr void func(struct pt_regs *regs, \ unsigned long error_code) \ { \ - irqentry_state_t state = irqentry_enter(regs); \ + irqentry_state_t irq_state; \ \ + irqentry_enter(regs, &irq_state); \ instrumentation_begin(); \ irq_enter_rcu(); \ kvm_set_cpu_l1tf_flush_l1d(); \ __##func (regs, (u8)error_code); \ irq_exit_rcu(); \ instrumentation_end(); \ - irqentry_exit(regs, state); \ + irqentry_exit(regs, &irq_state); \ } \ \ static __always_inline void __##func(struct pt_regs *regs, u8 vector) @@ -234,15 +237,16 @@ static void __##func(struct pt_regs *regs); \ \ __visible noinstr void func(struct pt_regs *regs) \ { \ - irqentry_state_t state = irqentry_enter(regs); \ + irqentry_state_t irq_state; \ \ + irqentry_enter(regs, &irq_state); \ instrumentation_begin(); \ irq_enter_rcu(); \ kvm_set_cpu_l1tf_flush_l1d(); \ run_sysvec_on_irqstack_cond(__##func, regs); \ irq_exit_rcu(); \ instrumentation_end(); \ - irqentry_exit(regs, state); \ + irqentry_exit(regs, &irq_state); \ } \ \ static noinline void __##func(struct pt_regs *regs) @@ -263,15 +267,16 @@ static __always_inline void __##func(struct pt_regs *regs); \ \ __visible noinstr void func(struct pt_regs *regs) \ { \ - irqentry_state_t state = irqentry_enter(regs); \ + irqentry_state_t irq_state; \ \ + irqentry_enter(regs, &irq_state); \ instrumentation_begin(); \ __irq_enter_raw(); \ kvm_set_cpu_l1tf_flush_l1d(); \ __##func (regs); \ __irq_exit_raw(); \ instrumentation_end(); \ - irqentry_exit(regs, state); \ + irqentry_exit(regs, &irq_state); \ } \ \ static __always_inline void __##func(struct pt_regs *regs) diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 403561a89c28..169168e535f2 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -1993,7 +1993,7 @@ static __always_inline void exc_machine_check_kernel(struct pt_regs *regs) mce_check_crashing_cpu()) return;

- irq_state = irqentry_nmi_enter(regs); + irqentry_nmi_enter(regs, &irq_state); /* * The call targets are marked noinstr, but objtool can't figure * that out because it's an indirect call. Annotate it. @@ -2004,7 +2004,7 @@ static __always_inline void exc_machine_check_kernel(struct pt_regs *regs) if (regs->flags & X86_EFLAGS_IF) trace_hardirqs_on_prepare(); instrumentation_end(); - irqentry_nmi_exit(regs, irq_state); + irqentry_nmi_exit(regs, &irq_state); }

static __always_inline void exc_machine_check_user(struct pt_regs *regs) diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 7f57ede3cb8e..ed7427c6e74d 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -238,12 +238,12 @@ EXPORT_SYMBOL_GPL(kvm_read_and_reset_apf_flags); noinstr bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token) { u32 flags = kvm_read_and_reset_apf_flags(); - irqentry_state_t state; + irqentry_state_t irq_state;

if (!flags) return false;

- state = irqentry_enter(regs); + irqentry_enter(regs, &irq_state); instrumentation_begin();

/* @@ -264,7 +264,7 @@ noinstr bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token) }

instrumentation_end(); - irqentry_exit(regs, state); + irqentry_exit(regs, &irq_state); return true; }

diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c index bf250a339655..1fd7780e99dd 100644 --- a/arch/x86/kernel/nmi.c +++ b/arch/x86/kernel/nmi.c @@ -502,14 +502,14 @@ DEFINE_IDTENTRY_RAW(exc_nmi)

this_cpu_write(nmi_dr7, local_db_save());

- irq_state = irqentry_nmi_enter(regs); + irqentry_nmi_enter(regs, &irq_state);

inc_irq_stat(__nmi_count);

if (!ignore_nmis) default_do_nmi(regs);

- irqentry_nmi_exit(regs, irq_state); + irqentry_nmi_exit(regs, &irq_state);

local_db_restore(this_cpu_read(nmi_dr7));

diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index e1b78829d909..8481cc373794 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -245,7 +245,7 @@ static noinstr bool handle_bug(struct pt_regs *regs)

DEFINE_IDTENTRY_RAW(exc_invalid_op) { - irqentry_state_t state; + irqentry_state_t irq_state;

/* * We use UD2 as a short encoding for 'CALL __WARN', as such @@ -255,11 +255,11 @@ DEFINE_IDTENTRY_RAW(exc_invalid_op) if (!user_mode(regs) && handle_bug(regs)) return;

- state = irqentry_enter(regs); + irqentry_enter(regs, &irq_state); instrumentation_begin(); handle_invalid_op(regs); instrumentation_end(); - irqentry_exit(regs, state); + irqentry_exit(regs, &irq_state); }

DEFINE_IDTENTRY(exc_coproc_segment_overrun) @@ -343,6 +343,7 @@ __visible void __noreturn handle_stack_overflow(const char *message, */ DEFINE_IDTENTRY_DF(exc_double_fault) { + irqentry_state_t irq_state; static const char str[] = "double fault"; struct task_struct *tsk = current;

@@ -405,7 +406,7 @@ DEFINE_IDTENTRY_DF(exc_double_fault) } #endif

- irqentry_nmi_enter(regs); + irqentry_nmi_enter(regs, &irq_state); instrumentation_begin(); notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV);

@@ -651,13 +652,15 @@ DEFINE_IDTENTRY_RAW(exc_int3) instrumentation_end(); irqentry_exit_to_user_mode(regs); } else { - irqentry_state_t irq_state = irqentry_nmi_enter(regs); + irqentry_state_t irq_state; + + irqentry_nmi_enter(regs, &irq_state);

instrumentation_begin(); if (!do_int3(regs)) die("int3", regs, 0); instrumentation_end(); - irqentry_nmi_exit(regs, irq_state); + irqentry_nmi_exit(regs, &irq_state); } }

@@ -852,7 +855,9 @@ static __always_inline void exc_debug_kernel(struct pt_regs *regs, * includes the entry stack is excluded for everything. */ unsigned long dr7 = local_db_save(); - irqentry_state_t irq_state = irqentry_nmi_enter(regs); + irqentry_state_t irq_state; + + irqentry_nmi_enter(regs, &irq_state); instrumentation_begin();

/* @@ -909,7 +914,7 @@ static __always_inline void exc_debug_kernel(struct pt_regs *regs, regs->flags &= ~X86_EFLAGS_TF; out: instrumentation_end(); - irqentry_nmi_exit(regs, irq_state); + irqentry_nmi_exit(regs, &irq_state);

local_db_restore(dr7); } diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 82bf37a5c9ec..8d20c4c13abf 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -1441,7 +1441,7 @@ handle_page_fault(struct pt_regs *regs, unsigned long error_code, DEFINE_IDTENTRY_RAW_ERRORCODE(exc_page_fault) { unsigned long address = read_cr2(); - irqentry_state_t state; + irqentry_state_t irq_state;

prefetchw(&current->mm->mmap_lock);

@@ -1479,11 +1479,11 @@ DEFINE_IDTENTRY_RAW_ERRORCODE(exc_page_fault) * code reenabled RCU to avoid subsequent wreckage which helps * debugability. */ - state = irqentry_enter(regs); + irqentry_enter(regs, &irq_state);

instrumentation_begin(); handle_page_fault(regs, error_code, address); instrumentation_end();

- irqentry_exit(regs, state); + irqentry_exit(regs, &irq_state); } diff --git a/include/linux/entry-common.h b/include/linux/entry-common.h index 7dff07713a07..0e10d0b85817 100644 --- a/include/linux/entry-common.h +++ b/include/linux/entry-common.h @@ -367,6 +367,8 @@ typedef struct irqentry_state { /** * irqentry_enter - Handle state tracking on ordinary interrupt entries * @regs: Pointer to pt_regs of interrupted context + * @irq_state: Pointer to an opaque object to store state information; to be + * passed back to irqentry_exit() * * Invokes: * - lockdep irqflag state tracking as low level ASM entry disabled @@ -392,10 +394,8 @@ typedef struct irqentry_state { * For user mode entries irqentry_enter_from_user_mode() is invoked to * establish the proper context for NOHZ_FULL. Otherwise scheduling on exit * would not be possible. - * - * Returns: An opaque object that must be passed to idtentry_exit() */ -irqentry_state_t noinstr irqentry_enter(struct pt_regs *regs); +void noinstr irqentry_enter(struct pt_regs *regs, irqentry_state_t *irq_state);

/** * irqentry_exit_cond_resched - Conditionally reschedule on return from interrupt @@ -407,7 +407,7 @@ void irqentry_exit_cond_resched(void); /** * irqentry_exit - Handle return from exception that used irqentry_enter() * @regs: Pointer to pt_regs (exception entry regs) - * @state: Return value from matching call to irqentry_enter() + * @irq_state: Pointer to state information passed to irqentry_enter() * * Depending on the return target (kernel/user) this runs the necessary * preemption and work checks if possible and reguired and returns to @@ -418,25 +418,27 @@ void irqentry_exit_cond_resched(void); * * Counterpart to irqentry_enter(). */ -void noinstr irqentry_exit(struct pt_regs *regs, irqentry_state_t state); +void noinstr irqentry_exit(struct pt_regs *regs, irqentry_state_t *irq_state);

/** * irqentry_nmi_enter - Handle NMI entry * @regs: Pointer to currents pt_regs + * @irq_state: Pointer to an opaque object to store state information; to be + * passed back to irqentry_nmi_exit() * * Similar to irqentry_enter() but taking care of the NMI constraints. */ -irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs); +void noinstr irqentry_nmi_enter(struct pt_regs *regs, irqentry_state_t *irq_state);

/** * irqentry_nmi_exit - Handle return from NMI handling * @regs: Pointer to pt_regs (NMI entry regs) - * @irq_state: Return value from matching call to irqentry_nmi_enter() + * @irq_state: Pointer to state information passed to irqentry_nmi_enter() * * Last action before returning to the low level assmenbly code. * * Counterpart to irqentry_nmi_enter(). */ -void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state); +void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t *irq_state);

#endif diff --git a/kernel/entry/common.c b/kernel/entry/common.c index 372f14cc347f..359688400c6b 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -281,15 +281,13 @@ noinstr void irqentry_exit_to_user_mode(struct pt_regs *regs) exit_to_user_mode(); }

-noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) +noinstr void irqentry_enter(struct pt_regs *regs, irqentry_state_t *irq_state) { - irqentry_state_t ret = { - .exit_rcu = false, - }; + irq_state->exit_rcu = false;

if (user_mode(regs)) { irqentry_enter_from_user_mode(regs); - return ret; + return; }

/* @@ -327,8 +325,8 @@ noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) trace_hardirqs_off_finish(); instrumentation_end();

- ret.exit_rcu = true; - return ret; + irq_state->exit_rcu = true; + return; }

/* @@ -342,8 +340,6 @@ noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) /* Use the combo lockdep/tracing function */ trace_hardirqs_off(); instrumentation_end(); - - return ret; }

void irqentry_exit_cond_resched(void) @@ -358,7 +354,7 @@ void irqentry_exit_cond_resched(void) } }

-noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state) +noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t *irq_state) { lockdep_assert_irqs_disabled();

@@ -371,7 +367,7 @@ noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state) * carefully and needs the same ordering of lockdep/tracing * and RCU as the return to user mode path. */ - if (state.exit_rcu) { + if (irq_state->exit_rcu) { instrumentation_begin(); /* Tell the tracer that IRET will enable interrupts */ trace_hardirqs_on_prepare(); @@ -393,16 +389,14 @@ noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state) * IRQ flags state is correct already. Just tell RCU if it * was not watching on entry. */ - if (state.exit_rcu) + if (irq_state->exit_rcu) rcu_irq_exit(); } }

-irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs) +void noinstr irqentry_nmi_enter(struct pt_regs *regs, irqentry_state_t *irq_state) { - irqentry_state_t irq_state; - - irq_state.lockdep = lockdep_hardirqs_enabled(); + irq_state->lockdep = lockdep_hardirqs_enabled();

__nmi_enter(); lockdep_hardirqs_off(CALLER_ADDR0); @@ -413,15 +407,13 @@ irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs) trace_hardirqs_off_finish(); ftrace_nmi_enter(); instrumentation_end(); - - return irq_state; }

-void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state) +void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t *irq_state) { instrumentation_begin(); ftrace_nmi_exit(); - if (irq_state.lockdep) { + if (irq_state->lockdep) { trace_hardirqs_on_prepare(); lockdep_hardirqs_on_prepare(CALLER_ADDR0); } @@ -429,7 +421,7 @@ void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state)

rcu_nmi_exit(); lockdep_hardirq_exit(); - if (irq_state.lockdep) + if (irq_state->lockdep) lockdep_hardirqs_on(CALLER_ADDR0); __nmi_exit(); }

From: Ira Weiny ira.weiny@intel.com

When only user space pkeys are enabled faulting within the kernel was an unexpected condition which should never happen. Therefore a WARN_ON in the kernel fault handler would detect if it ever did. Now this is no longer the case if PKS is enabled and supported.

Report a Pkey fault with a normal splat and add the PKRS state to the fault splat text. Note the PKS register is reset during an exception therefore the saved PKRS value from before the beginning of the exception is passed down.

If PKS is not enabled, or not active, maintain the WARN_ON_ONCE() from before.

Because each fault has its own state the pkrs information will be correctly reported even if a fault 'faults'.

Suggested-by: Andy Lutomirski luto@kernel.org Signed-off-by: Ira Weiny ira.weiny@intel.com

--- Changes from RFC V3 Update commit message Per Dave Hansen Don't print PKRS if !cpu_feature_enabled(X86_FEATURE_PKS) Fix comment Remove check on CONFIG_ARCH_HAS_SUPERVISOR_PKEYS in favor of disabled-features.h --- arch/x86/mm/fault.c | 58 ++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 25 deletions(-)

diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 8d20c4c13abf..931603102010 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -504,7 +504,8 @@ static void show_ldttss(const struct desc_ptr *gdt, const char *name, u16 index) }

static void -show_fault_oops(struct pt_regs *regs, unsigned long error_code, unsigned long address) +show_fault_oops(struct pt_regs *regs, unsigned long error_code, unsigned long address, + irqentry_state_t *irq_state) { if (!oops_may_print()) return; @@ -548,6 +549,11 @@ show_fault_oops(struct pt_regs *regs, unsigned long error_code, unsigned long ad (error_code & X86_PF_PK) ? "protection keys violation" : "permissions violation");

+#ifdef CONFIG_ARCH_HAS_SUPERVISOR_PKEYS + if (cpu_feature_enabled(X86_FEATURE_PKS) && irq_state && (error_code & X86_PF_PK)) + pr_alert("PKRS: 0x%x\n", irq_state->pkrs); +#endif + if (!(error_code & X86_PF_USER) && user_mode(regs)) { struct desc_ptr idt, gdt; u16 ldtr, tr; @@ -626,7 +632,8 @@ static void set_signal_archinfo(unsigned long address,

static noinline void no_context(struct pt_regs *regs, unsigned long error_code, - unsigned long address, int signal, int si_code) + unsigned long address, int signal, int si_code, + irqentry_state_t *irq_state) { struct task_struct *tsk = current; unsigned long flags; @@ -732,7 +739,7 @@ no_context(struct pt_regs *regs, unsigned long error_code, */ flags = oops_begin();

- show_fault_oops(regs, error_code, address); + show_fault_oops(regs, error_code, address, irq_state);

if (task_stack_end_corrupted(tsk)) printk(KERN_EMERG "Thread overran stack, or stack corrupted\n"); @@ -785,7 +792,8 @@ static bool is_vsyscall_vaddr(unsigned long vaddr)

static void __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code, - unsigned long address, u32 pkey, int si_code) + unsigned long address, u32 pkey, int si_code, + irqentry_state_t *irq_state) { struct task_struct *tsk = current;

@@ -832,14 +840,14 @@ __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code, if (is_f00f_bug(regs, address)) return;

- no_context(regs, error_code, address, SIGSEGV, si_code); + no_context(regs, error_code, address, SIGSEGV, si_code, irq_state); }

static noinline void bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code, - unsigned long address) + unsigned long address, irqentry_state_t *irq_state) { - __bad_area_nosemaphore(regs, error_code, address, 0, SEGV_MAPERR); + __bad_area_nosemaphore(regs, error_code, address, 0, SEGV_MAPERR, irq_state); }

static void @@ -853,7 +861,7 @@ __bad_area(struct pt_regs *regs, unsigned long error_code, */ mmap_read_unlock(mm);

- __bad_area_nosemaphore(regs, error_code, address, pkey, si_code); + __bad_area_nosemaphore(regs, error_code, address, pkey, si_code, NULL); }

static noinline void @@ -923,7 +931,7 @@ do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address, { /* Kernel mode? Handle exceptions or die: */ if (!(error_code & X86_PF_USER)) { - no_context(regs, error_code, address, SIGBUS, BUS_ADRERR); + no_context(regs, error_code, address, SIGBUS, BUS_ADRERR, NULL); return; }

@@ -957,7 +965,7 @@ mm_fault_error(struct pt_regs *regs, unsigned long error_code, unsigned long address, vm_fault_t fault) { if (fatal_signal_pending(current) && !(error_code & X86_PF_USER)) { - no_context(regs, error_code, address, 0, 0); + no_context(regs, error_code, address, 0, 0, NULL); return; }

@@ -965,7 +973,7 @@ mm_fault_error(struct pt_regs *regs, unsigned long error_code, /* Kernel mode? Handle exceptions or die: */ if (!(error_code & X86_PF_USER)) { no_context(regs, error_code, address, - SIGSEGV, SEGV_MAPERR); + SIGSEGV, SEGV_MAPERR, NULL); return; }

@@ -980,7 +988,7 @@ mm_fault_error(struct pt_regs *regs, unsigned long error_code, VM_FAULT_HWPOISON_LARGE)) do_sigbus(regs, error_code, address, fault); else if (fault & VM_FAULT_SIGSEGV) - bad_area_nosemaphore(regs, error_code, address); + bad_area_nosemaphore(regs, error_code, address, NULL); else BUG(); } @@ -1148,14 +1156,14 @@ bool fault_in_kernel_space(unsigned long address) */ static void do_kern_addr_fault(struct pt_regs *regs, unsigned long hw_error_code, - unsigned long address) + unsigned long address, irqentry_state_t *irq_state) { /* - * Protection keys exceptions only happen on user pages. We - * have no user pages in the kernel portion of the address - * space, so do not expect them here. + * PF_PK is only expected on kernel addresses when supervisor pkeys are + * enabled. */ - WARN_ON_ONCE(hw_error_code & X86_PF_PK); + if (!cpu_feature_enabled(X86_FEATURE_PKS)) + WARN_ON_ONCE(hw_error_code & X86_PF_PK);

#ifdef CONFIG_X86_32 /* @@ -1204,7 +1212,7 @@ do_kern_addr_fault(struct pt_regs *regs, unsigned long hw_error_code, * Don't take the mm semaphore here. If we fixup a prefetch * fault we could otherwise deadlock: */ - bad_area_nosemaphore(regs, hw_error_code, address); + bad_area_nosemaphore(regs, hw_error_code, address, irq_state); } NOKPROBE_SYMBOL(do_kern_addr_fault);

@@ -1245,7 +1253,7 @@ void do_user_addr_fault(struct pt_regs *regs, !(hw_error_code & X86_PF_USER) && !(regs->flags & X86_EFLAGS_AC))) { - bad_area_nosemaphore(regs, hw_error_code, address); + bad_area_nosemaphore(regs, hw_error_code, address, NULL); return; }

@@ -1254,7 +1262,7 @@ void do_user_addr_fault(struct pt_regs *regs, * in a region with pagefaults disabled then we must not take the fault */ if (unlikely(faulthandler_disabled() || !mm)) { - bad_area_nosemaphore(regs, hw_error_code, address); + bad_area_nosemaphore(regs, hw_error_code, address, NULL); return; }

@@ -1316,7 +1324,7 @@ void do_user_addr_fault(struct pt_regs *regs, * Fault from code in kernel from * which we do not expect faults. */ - bad_area_nosemaphore(regs, hw_error_code, address); + bad_area_nosemaphore(regs, hw_error_code, address, NULL); return; } retry: @@ -1375,7 +1383,7 @@ void do_user_addr_fault(struct pt_regs *regs, if (fault_signal_pending(fault, regs)) { if (!user_mode(regs)) no_context(regs, hw_error_code, address, SIGBUS, - BUS_ADRERR); + BUS_ADRERR, NULL); return; }

@@ -1415,7 +1423,7 @@ trace_page_fault_entries(struct pt_regs *regs, unsigned long error_code,

static __always_inline void handle_page_fault(struct pt_regs *regs, unsigned long error_code, - unsigned long address) + unsigned long address, irqentry_state_t *irq_state) { trace_page_fault_entries(regs, error_code, address);

@@ -1424,7 +1432,7 @@ handle_page_fault(struct pt_regs *regs, unsigned long error_code,

/* Was the fault on kernel-controlled part of the address space? */ if (unlikely(fault_in_kernel_space(address))) { - do_kern_addr_fault(regs, error_code, address); + do_kern_addr_fault(regs, error_code, address, irq_state); } else { do_user_addr_fault(regs, error_code, address); /* @@ -1482,7 +1490,7 @@ DEFINE_IDTENTRY_RAW_ERRORCODE(exc_page_fault) irqentry_enter(regs, &irq_state);

instrumentation_begin(); - handle_page_fault(regs, error_code, address); + handle_page_fault(regs, error_code, address, &irq_state); instrumentation_end();

irqentry_exit(regs, &irq_state);