[PATCH 6.10 375/375] LoongArch: Use accessors to page table entries instead of direct dereference

10 Sep 2024

6.10-stable review patch.  If anyone has any objections, please let me know.
------------------
From: Huacai Chen chenhuacai@loongson.cn
commit 4574815abf43e2bf05643e1b3f7a2e5d6df894f0 upstream.
As very well explained in commit 20a004e7b017cce282 ("arm64: mm: Use
READ_ONCE/WRITE_ONCE when accessing page tables"), an architecture whose
page table walker can modify the PTE in parallel must use READ_ONCE()/
WRITE_ONCE() macro to avoid any compiler transformation.
So apply that to LoongArch which is such an architecture, in order to
avoid potential problems.
Similar to commit edf955647269422e ("riscv: Use accessors to page table
entries instead of direct dereference").
Signed-off-by: Huacai Chen chenhuacai@loongson.cn
Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org
---
 arch/loongarch/include/asm/hugetlb.h |    4 +-
 arch/loongarch/include/asm/kfence.h  |    6 ++--
 arch/loongarch/include/asm/pgtable.h |   48 +++++++++++++++++++++--------------
 arch/loongarch/kvm/mmu.c             |    8 ++---
 arch/loongarch/mm/hugetlbpage.c      |    6 ++--
 arch/loongarch/mm/init.c             |   10 +++----
 arch/loongarch/mm/kasan_init.c       |   10 +++----
 arch/loongarch/mm/pgtable.c          |    2 -
 8 files changed, 52 insertions(+), 42 deletions(-)

--- a/arch/loongarch/include/asm/hugetlb.h
+++ b/arch/loongarch/include/asm/hugetlb.h
@@ -34,7 +34,7 @@ static inline pte_t huge_ptep_get_and_cl
    				    unsigned long addr, pte_t *ptep)
 {
    pte_t clear;
-	pte_t pte = *ptep;
+	pte_t pte = ptep_get(ptep);
pte_val(clear) = (unsigned long)invalid_pte_table;
    set_pte_at(mm, addr, ptep, clear);
@@ -65,7 +65,7 @@ static inline int huge_ptep_set_access_f
    				     pte_t *ptep, pte_t pte,
    				     int dirty)
 {
-	int changed = !pte_same(*ptep, pte);
+	int changed = !pte_same(ptep_get(ptep), pte);
if (changed) {
    	set_pte_at(vma->vm_mm, addr, ptep, pte);
--- a/arch/loongarch/include/asm/kfence.h
+++ b/arch/loongarch/include/asm/kfence.h
@@ -53,13 +53,13 @@ static inline bool kfence_protect_page(u
 {
    pte_t *pte = virt_to_kpte(addr);
-	if (WARN_ON(!pte) || pte_none(*pte))
+	if (WARN_ON(!pte) || pte_none(ptep_get(pte)))
    	return false;
if (protect)
-		set_pte(pte, __pte(pte_val(*pte) & ~(_PAGE_VALID | _PAGE_PRESENT)));
+		set_pte(pte, __pte(pte_val(ptep_get(pte)) & ~(_PAGE_VALID | _PAGE_PRESENT)));
    else
-		set_pte(pte, __pte(pte_val(*pte) | (_PAGE_VALID | _PAGE_PRESENT)));
+		set_pte(pte, __pte(pte_val(ptep_get(pte)) | (_PAGE_VALID | _PAGE_PRESENT)));
preempt_disable();
    local_flush_tlb_one(addr);
--- a/arch/loongarch/include/asm/pgtable.h
+++ b/arch/loongarch/include/asm/pgtable.h
@@ -106,6 +106,9 @@ extern unsigned long empty_zero_page[PAG
 #define KFENCE_AREA_START	(VMEMMAP_END + 1)
 #define KFENCE_AREA_END		(KFENCE_AREA_START + KFENCE_AREA_SIZE - 1)
+#define ptep_get(ptep) READ_ONCE(*(ptep))
+#define pmdp_get(pmdp) READ_ONCE(*(pmdp))
+
 #define pte_ERROR(e) \
    pr_err("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e))
 #ifndef __PAGETABLE_PMD_FOLDED
@@ -147,11 +150,6 @@ static inline int p4d_present(p4d_t p4d)
    return p4d_val(p4d) != (unsigned long)invalid_pud_table;
 }
-static inline void p4d_clear(p4d_t *p4dp)
-{
-	p4d_val(*p4dp) = (unsigned long)invalid_pud_table;
-}
-
 static inline pud_t *p4d_pgtable(p4d_t p4d)
 {
    return (pud_t *)p4d_val(p4d);
@@ -159,7 +157,12 @@ static inline pud_t *p4d_pgtable(p4d_t p
static inline void set_p4d(p4d_t *p4d, p4d_t p4dval)
 {
-	*p4d = p4dval;
+	WRITE_ONCE(*p4d, p4dval);
+}
+
+static inline void p4d_clear(p4d_t *p4dp)
+{
+	set_p4d(p4dp, __p4d((unsigned long)invalid_pud_table));
 }
#define p4d_phys(p4d)		PHYSADDR(p4d_val(p4d))
@@ -193,17 +196,20 @@ static inline int pud_present(pud_t pud)
    return pud_val(pud) != (unsigned long)invalid_pmd_table;
 }
-static inline void pud_clear(pud_t *pudp)
+static inline pmd_t *pud_pgtable(pud_t pud)
 {
-	pud_val(*pudp) = ((unsigned long)invalid_pmd_table);
+	return (pmd_t *)pud_val(pud);
 }
-static inline pmd_t *pud_pgtable(pud_t pud)
+static inline void set_pud(pud_t *pud, pud_t pudval)
 {
-	return (pmd_t *)pud_val(pud);
+	WRITE_ONCE(*pud, pudval);
 }
-#define set_pud(pudptr, pudval) do { *(pudptr) = (pudval); } while (0)
+static inline void pud_clear(pud_t *pudp)
+{
+	set_pud(pudp, __pud((unsigned long)invalid_pmd_table));
+}
#define pud_phys(pud)		PHYSADDR(pud_val(pud))
 #define pud_page(pud)		(pfn_to_page(pud_phys(pud) >> PAGE_SHIFT))
@@ -231,12 +237,15 @@ static inline int pmd_present(pmd_t pmd)
    return pmd_val(pmd) != (unsigned long)invalid_pte_table;
 }
-static inline void pmd_clear(pmd_t *pmdp)
+static inline void set_pmd(pmd_t *pmd, pmd_t pmdval)
 {
-	pmd_val(*pmdp) = ((unsigned long)invalid_pte_table);
+	WRITE_ONCE(*pmd, pmdval);
 }
-#define set_pmd(pmdptr, pmdval) do { *(pmdptr) = (pmdval); } while (0)
+static inline void pmd_clear(pmd_t *pmdp)
+{
+	set_pmd(pmdp, __pmd((unsigned long)invalid_pte_table));
+}
#define pmd_phys(pmd)		PHYSADDR(pmd_val(pmd))
@@ -314,7 +323,8 @@ extern void paging_init(void);
static inline void set_pte(pte_t *ptep, pte_t pteval)
 {
-	*ptep = pteval;
+	WRITE_ONCE(*ptep, pteval);
+
    if (pte_val(pteval) & _PAGE_GLOBAL) {
    	pte_t *buddy = ptep_buddy(ptep);
    	/*
@@ -341,8 +351,8 @@ static inline void set_pte(pte_t *ptep,
    	: [buddy] "+m" (buddy->pte), [tmp] "=&r" (tmp)
    	: [global] "r" (page_global));
 #else /* !CONFIG_SMP */
-		if (pte_none(*buddy))
-			pte_val(*buddy) = pte_val(*buddy) | _PAGE_GLOBAL;
+		if (pte_none(ptep_get(buddy)))
+			WRITE_ONCE(*buddy, __pte(pte_val(ptep_get(buddy)) | _PAGE_GLOBAL));
 #endif /* CONFIG_SMP */
    }
 }
@@ -350,7 +360,7 @@ static inline void set_pte(pte_t *ptep,
 static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
 {
    /* Preserve global status for the pair */
-	if (pte_val(*ptep_buddy(ptep)) & _PAGE_GLOBAL)
+	if (pte_val(ptep_get(ptep_buddy(ptep))) & _PAGE_GLOBAL)
    	set_pte(ptep, __pte(_PAGE_GLOBAL));
    else
    	set_pte(ptep, __pte(0));
@@ -589,7 +599,7 @@ static inline pmd_t pmd_mkinvalid(pmd_t
 static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
    				    unsigned long address, pmd_t *pmdp)
 {
-	pmd_t old = *pmdp;
+	pmd_t old = pmdp_get(pmdp);
pmd_clear(pmdp);
--- a/arch/loongarch/kvm/mmu.c
+++ b/arch/loongarch/kvm/mmu.c
@@ -695,19 +695,19 @@ static int host_pfn_mapping_level(struct
     * value) and then p*d_offset() walks into the target huge page instead
     * of the old page table (sees the new value).
     */
-	pgd = READ_ONCE(*pgd_offset(kvm->mm, hva));
+	pgd = pgdp_get(pgd_offset(kvm->mm, hva));
    if (pgd_none(pgd))
    	goto out;
-	p4d = READ_ONCE(*p4d_offset(&pgd, hva));
+	p4d = p4dp_get(p4d_offset(&pgd, hva));
    if (p4d_none(p4d) || !p4d_present(p4d))
    	goto out;
-	pud = READ_ONCE(*pud_offset(&p4d, hva));
+	pud = pudp_get(pud_offset(&p4d, hva));
    if (pud_none(pud) || !pud_present(pud))
    	goto out;
-	pmd = READ_ONCE(*pmd_offset(&pud, hva));
+	pmd = pmdp_get(pmd_offset(&pud, hva));
    if (pmd_none(pmd) || !pmd_present(pmd))
    	goto out;
--- a/arch/loongarch/mm/hugetlbpage.c
+++ b/arch/loongarch/mm/hugetlbpage.c
@@ -39,11 +39,11 @@ pte_t *huge_pte_offset(struct mm_struct
    pmd_t *pmd = NULL;
pgd = pgd_offset(mm, addr);
-	if (pgd_present(*pgd)) {
+	if (pgd_present(pgdp_get(pgd))) {
    	p4d = p4d_offset(pgd, addr);
-		if (p4d_present(*p4d)) {
+		if (p4d_present(p4dp_get(p4d))) {
    		pud = pud_offset(p4d, addr);
-			if (pud_present(*pud))
+			if (pud_present(pudp_get(pud)))
    			pmd = pmd_offset(pud, addr);
    	}
    }
--- a/arch/loongarch/mm/init.c
+++ b/arch/loongarch/mm/init.c
@@ -141,7 +141,7 @@ void __meminit vmemmap_set_pmd(pmd_t *pm
 int __meminit vmemmap_check_pmd(pmd_t *pmd, int node,
    			unsigned long addr, unsigned long next)
 {
-	int huge = pmd_val(*pmd) & _PAGE_HUGE;
+	int huge = pmd_val(pmdp_get(pmd)) & _PAGE_HUGE;
if (huge)
    	vmemmap_verify((pte_t *)pmd, node, addr, next);
@@ -173,7 +173,7 @@ pte_t * __init populate_kernel_pte(unsig
    pud_t *pud;
    pmd_t *pmd;
-	if (p4d_none(*p4d)) {
+	if (p4d_none(p4dp_get(p4d))) {
    	pud = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
    	if (!pud)
    		panic("%s: Failed to allocate memory\n", __func__);
@@ -184,7 +184,7 @@ pte_t * __init populate_kernel_pte(unsig
    }
pud = pud_offset(p4d, addr);
-	if (pud_none(*pud)) {
+	if (pud_none(pudp_get(pud))) {
    	pmd = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
    	if (!pmd)
    		panic("%s: Failed to allocate memory\n", __func__);
@@ -195,7 +195,7 @@ pte_t * __init populate_kernel_pte(unsig
    }
pmd = pmd_offset(pud, addr);
-	if (!pmd_present(*pmd)) {
+	if (!pmd_present(pmdp_get(pmd))) {
    	pte_t *pte;
pte = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
@@ -216,7 +216,7 @@ void __init __set_fixmap(enum fixed_addr
    BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
ptep = populate_kernel_pte(addr);
-	if (!pte_none(*ptep)) {
+	if (!pte_none(ptep_get(ptep))) {
    	pte_ERROR(*ptep);
    	return;
    }
--- a/arch/loongarch/mm/kasan_init.c
+++ b/arch/loongarch/mm/kasan_init.c
@@ -105,7 +105,7 @@ static phys_addr_t __init kasan_alloc_ze
static pte_t *__init kasan_pte_offset(pmd_t *pmdp, unsigned long addr, int node, bool early)
 {
-	if (__pmd_none(early, READ_ONCE(*pmdp))) {
+	if (__pmd_none(early, pmdp_get(pmdp))) {
    	phys_addr_t pte_phys = early ?
    			__pa_symbol(kasan_early_shadow_pte) : kasan_alloc_zeroed_page(node);
    	if (!early)
@@ -118,7 +118,7 @@ static pte_t *__init kasan_pte_offset(pm
static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node, bool early)
 {
-	if (__pud_none(early, READ_ONCE(*pudp))) {
+	if (__pud_none(early, pudp_get(pudp))) {
    	phys_addr_t pmd_phys = early ?
    			__pa_symbol(kasan_early_shadow_pmd) : kasan_alloc_zeroed_page(node);
    	if (!early)
@@ -131,7 +131,7 @@ static pmd_t *__init kasan_pmd_offset(pu
static pud_t *__init kasan_pud_offset(p4d_t *p4dp, unsigned long addr, int node, bool early)
 {
-	if (__p4d_none(early, READ_ONCE(*p4dp))) {
+	if (__p4d_none(early, p4dp_get(p4dp))) {
    	phys_addr_t pud_phys = early ?
    		__pa_symbol(kasan_early_shadow_pud) : kasan_alloc_zeroed_page(node);
    	if (!early)
@@ -154,7 +154,7 @@ static void __init kasan_pte_populate(pm
    				      : kasan_alloc_zeroed_page(node);
    	next = addr + PAGE_SIZE;
    	set_pte(ptep, pfn_pte(__phys_to_pfn(page_phys), PAGE_KERNEL));
-	} while (ptep++, addr = next, addr != end && __pte_none(early, READ_ONCE(*ptep)));
+	} while (ptep++, addr = next, addr != end && __pte_none(early, ptep_get(ptep)));
 }
static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr,
@@ -166,7 +166,7 @@ static void __init kasan_pmd_populate(pu
    do {
    	next = pmd_addr_end(addr, end);
    	kasan_pte_populate(pmdp, addr, next, node, early);
-	} while (pmdp++, addr = next, addr != end && __pmd_none(early, READ_ONCE(*pmdp)));
+	} while (pmdp++, addr = next, addr != end && __pmd_none(early, pmdp_get(pmdp)));
 }
static void __init kasan_pud_populate(p4d_t *p4dp, unsigned long addr,
--- a/arch/loongarch/mm/pgtable.c
+++ b/arch/loongarch/mm/pgtable.c
@@ -128,7 +128,7 @@ pmd_t mk_pmd(struct page *page, pgprot_t
 void set_pmd_at(struct mm_struct *mm, unsigned long addr,
    	pmd_t *pmdp, pmd_t pmd)
 {
-	*pmdp = pmd;
+	WRITE_ONCE(*pmdp, pmd);
    flush_tlb_all();
 }

    

2024

2023

2022

2021

2020

2019

2018

2017

[PATCH 6.10 375/375] LoongArch: Use accessors to page table entries instead of direct dereference