From f5397c3ee0a3e2ca0a6d66d079ffcd5386b45b81 Mon Sep 17 00:00:00 2001 From: Nanyong Sun Date: Fri, 30 Apr 2021 16:28:47 +0800 Subject: riscv: mm: add _PAGE_LEAF macro In riscv, a page table entry is leaf when any bit of read, write, or execute bit is set. So add a macro:_PAGE_LEAF instead of (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC), which is frequently used to determine if it is a leaf page. This make code easier to read, without any functional change. Signed-off-by: Nanyong Sun Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/pgtable.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'arch/riscv/include/asm/pgtable.h') diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 9469f464e71a..dbced7d37768 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -190,8 +190,7 @@ static inline int pmd_bad(pmd_t pmd) #define pmd_leaf pmd_leaf static inline int pmd_leaf(pmd_t pmd) { - return pmd_present(pmd) && - (pmd_val(pmd) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC)); + return pmd_present(pmd) && (pmd_val(pmd) & _PAGE_LEAF); } static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) @@ -267,8 +266,7 @@ static inline int pte_exec(pte_t pte) static inline int pte_huge(pte_t pte) { - return pte_present(pte) - && (pte_val(pte) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC)); + return pte_present(pte) && (pte_val(pte) & _PAGE_LEAF); } static inline int pte_dirty(pte_t pte) -- cgit From 141682f5b9d658b5fba7c33cf8574329a7840cdc Mon Sep 17 00:00:00 2001 From: Nanyong Sun Date: Fri, 30 Apr 2021 16:28:48 +0800 Subject: riscv: mm: make pmd_bad() check leaf condition In the definition in Documentation/vm/arch_pgtable_helpers.rst, pmd_bad() means test a non-table mapped PMD, so it should also return true when it is a leaf page. Signed-off-by: Nanyong Sun Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/pgtable.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/riscv/include/asm/pgtable.h') diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index dbced7d37768..b06eb8394e4e 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -184,7 +184,7 @@ static inline int pmd_none(pmd_t pmd) static inline int pmd_bad(pmd_t pmd) { - return !pmd_present(pmd); + return !pmd_present(pmd) || (pmd_val(pmd) & _PAGE_LEAF); } #define pmd_leaf pmd_leaf -- cgit From e88b333142e4aba7410d6d3292ad97b3a8588bfe Mon Sep 17 00:00:00 2001 From: Nanyong Sun Date: Fri, 30 Apr 2021 16:28:50 +0800 Subject: riscv: mm: add THP support on 64-bit Bring Transparent HugePage support to riscv. A transparent huge page is always represented as a pmd. Signed-off-by: Nanyong Sun Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/pgtable.h | 156 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) (limited to 'arch/riscv/include/asm/pgtable.h') diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index b06eb8394e4e..4b708ae08910 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -172,10 +172,23 @@ extern pgd_t swapper_pg_dir[]; #define __S110 PAGE_SHARED_EXEC #define __S111 PAGE_SHARED_EXEC +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline int pmd_present(pmd_t pmd) +{ + /* + * Checking for _PAGE_LEAF is needed too because: + * When splitting a THP, split_huge_page() will temporarily clear + * the present bit, in this situation, pmd_present() and + * pmd_trans_huge() still needs to return true. + */ + return (pmd_val(pmd) & (_PAGE_PRESENT | _PAGE_PROT_NONE | _PAGE_LEAF)); +} +#else static inline int pmd_present(pmd_t pmd) { return (pmd_val(pmd) & (_PAGE_PRESENT | _PAGE_PROT_NONE)); } +#endif static inline int pmd_none(pmd_t pmd) { @@ -369,6 +382,14 @@ static inline void update_mmu_cache(struct vm_area_struct *vma, local_flush_tlb_page(address); } +static inline void update_mmu_cache_pmd(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp) +{ + pte_t *ptep = (pte_t *)pmdp; + + update_mmu_cache(vma, address, ptep); +} + #define __HAVE_ARCH_PTE_SAME static inline int pte_same(pte_t pte_a, pte_t pte_b) { @@ -462,6 +483,141 @@ static inline int ptep_clear_flush_young(struct vm_area_struct *vma, return ptep_test_and_clear_young(vma, address, ptep); } +/* + * THP functions + */ +static inline pmd_t pte_pmd(pte_t pte) +{ + return __pmd(pte_val(pte)); +} + +static inline pmd_t pmd_mkhuge(pmd_t pmd) +{ + return pmd; +} + +static inline pmd_t pmd_mkinvalid(pmd_t pmd) +{ + return __pmd(pmd_val(pmd) & ~(_PAGE_PRESENT|_PAGE_PROT_NONE)); +} + +#define __pmd_to_phys(pmd) (pmd_val(pmd) >> _PAGE_PFN_SHIFT << PAGE_SHIFT) + +static inline unsigned long pmd_pfn(pmd_t pmd) +{ + return ((__pmd_to_phys(pmd) & PMD_MASK) >> PAGE_SHIFT); +} + +static inline pmd_t mk_pmd(struct page *page, pgprot_t prot) +{ + return pfn_pmd(page_to_pfn(page), prot); +} + +static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) +{ + return pte_pmd(pte_modify(pmd_pte(pmd), newprot)); +} + +#define pmd_write pmd_write +static inline int pmd_write(pmd_t pmd) +{ + return pte_write(pmd_pte(pmd)); +} + +static inline int pmd_dirty(pmd_t pmd) +{ + return pte_dirty(pmd_pte(pmd)); +} + +static inline int pmd_young(pmd_t pmd) +{ + return pte_young(pmd_pte(pmd)); +} + +static inline pmd_t pmd_mkold(pmd_t pmd) +{ + return pte_pmd(pte_mkold(pmd_pte(pmd))); +} + +static inline pmd_t pmd_mkyoung(pmd_t pmd) +{ + return pte_pmd(pte_mkyoung(pmd_pte(pmd))); +} + +static inline pmd_t pmd_mkwrite(pmd_t pmd) +{ + return pte_pmd(pte_mkwrite(pmd_pte(pmd))); +} + +static inline pmd_t pmd_wrprotect(pmd_t pmd) +{ + return pte_pmd(pte_wrprotect(pmd_pte(pmd))); +} + +static inline pmd_t pmd_mkclean(pmd_t pmd) +{ + return pte_pmd(pte_mkclean(pmd_pte(pmd))); +} + +static inline pmd_t pmd_mkdirty(pmd_t pmd) +{ + return pte_pmd(pte_mkdirty(pmd_pte(pmd))); +} + +static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd) +{ + return set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd)); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline int pmd_trans_huge(pmd_t pmd) +{ + return pmd_leaf(pmd); +} + +#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS +static inline int pmdp_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, + pmd_t entry, int dirty) +{ + return ptep_set_access_flags(vma, address, (pte_t *)pmdp, pmd_pte(entry), dirty); +} + +#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG +static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp) +{ + return ptep_test_and_clear_young(vma, address, (pte_t *)pmdp); +} + +#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR +static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, + unsigned long address, pmd_t *pmdp) +{ + return pte_pmd(ptep_get_and_clear(mm, address, (pte_t *)pmdp)); +} + +#define __HAVE_ARCH_PMDP_SET_WRPROTECT +static inline void pmdp_set_wrprotect(struct mm_struct *mm, + unsigned long address, pmd_t *pmdp) +{ + ptep_set_wrprotect(mm, address, (pte_t *)pmdp); +} + +#define pmdp_establish pmdp_establish +static inline pmd_t pmdp_establish(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, pmd_t pmd) +{ + return __pmd(atomic_long_xchg((atomic_long_t *)pmdp, pmd_val(pmd))); +} + +#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE +void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end); + +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + /* * Encode and decode a swap entry * -- cgit From 3332f4190674114e08daaf6859c11a7e464bceff Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Sat, 17 Apr 2021 00:37:22 +0800 Subject: riscv: mremap speedup - enable HAVE_MOVE_PUD and HAVE_MOVE_PMD HAVE_MOVE_PUD enables remapping pages at the PUD level if both the source and destination addresses are PUD-aligned. HAVE_MOVE_PMD does similar speedup on the PMD level. With HAVE_MOVE_PUD enabled, there is about a 143x improvement on qemu With HAVE_MOVE_PMD enabled, there is about a 5x improvement on qemu Signed-off-by: Jisheng Zhang Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/pgtable.h | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'arch/riscv/include/asm/pgtable.h') diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 4b708ae08910..3b72862a83fa 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -241,6 +241,11 @@ static inline pte_t pmd_pte(pmd_t pmd) return __pte(pmd_val(pmd)); } +static inline pte_t pud_pte(pud_t pud) +{ + return __pte(pud_val(pud)); +} + /* Yields the page frame number (PFN) of a page table entry */ static inline unsigned long pte_pfn(pte_t pte) { @@ -570,6 +575,12 @@ static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, return set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd)); } +static inline void set_pud_at(struct mm_struct *mm, unsigned long addr, + pud_t *pudp, pud_t pud) +{ + return set_pte_at(mm, addr, (pte_t *)pudp, pud_pte(pud)); +} + #ifdef CONFIG_TRANSPARENT_HUGEPAGE static inline int pmd_trans_huge(pmd_t pmd) { -- cgit From f842f5ff6aafc2752580ed99ee757652c08684e7 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Mon, 10 May 2021 19:42:22 +0800 Subject: riscv: Move setup_bootmem into paging_init Make setup_bootmem() static. Signed-off-by: Kefeng Wang Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/pgtable.h | 1 - 1 file changed, 1 deletion(-) (limited to 'arch/riscv/include/asm/pgtable.h') diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 3b72862a83fa..bde8ce3bfe7c 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -698,7 +698,6 @@ extern uintptr_t _dtb_early_pa; #define dtb_early_pa _dtb_early_pa #endif /* CONFIG_XIP_KERNEL */ -void setup_bootmem(void); void paging_init(void); void misc_mem_init(void); -- cgit From cba43c31f14b08f193ebb5b4a72751b0947436c1 Mon Sep 17 00:00:00 2001 From: Guo Ren Date: Wed, 26 May 2021 05:49:20 +0000 Subject: riscv: Use global mappings for kernel pages We map kernel pages into all addresses spages, so they can be marked as global. This allows hardware to avoid flushing the kernel mappings when moving between address spaces. Signed-off-by: Guo Ren Reviewed-by: Anup Patel Reviewed-by: Christoph Hellwig [Palmer: commit text] Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/pgtable.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'arch/riscv/include/asm/pgtable.h') diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index bde8ce3bfe7c..c103f0a278e5 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -134,7 +134,8 @@ | _PAGE_WRITE \ | _PAGE_PRESENT \ | _PAGE_ACCESSED \ - | _PAGE_DIRTY) + | _PAGE_DIRTY \ + | _PAGE_GLOBAL) #define PAGE_KERNEL __pgprot(_PAGE_KERNEL) #define PAGE_KERNEL_READ __pgprot(_PAGE_KERNEL & ~_PAGE_WRITE) -- cgit From ff76e3d7c3c958b51f095dfdb7d451177312896b Mon Sep 17 00:00:00 2001 From: Bixuan Cui Date: Sat, 29 May 2021 16:06:57 +0800 Subject: riscv: fix build error when CONFIG_SMP is disabled Fix build error when disable CONFIG_SMP: mm/pgtable-generic.o: In function `.L19': pgtable-generic.c:(.text+0x42): undefined reference to `flush_pmd_tlb_range' mm/pgtable-generic.o: In function `pmdp_huge_clear_flush': pgtable-generic.c:(.text+0x6c): undefined reference to `flush_pmd_tlb_range' mm/pgtable-generic.o: In function `pmdp_invalidate': pgtable-generic.c:(.text+0x162): undefined reference to `flush_pmd_tlb_range' Fixes: e88b333142e4 ("riscv: mm: add THP support on 64-bit") Reported-by: Hulk Robot Signed-off-by: Bixuan Cui Acked-by: Nanyong Sun Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/pgtable.h | 5 ----- 1 file changed, 5 deletions(-) (limited to 'arch/riscv/include/asm/pgtable.h') diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index c103f0a278e5..f282f7a375e2 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -623,11 +623,6 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, { return __pmd(atomic_long_xchg((atomic_long_t *)pmdp, pmd_val(pmd))); } - -#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE -void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, - unsigned long end); - #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ /* -- cgit From 7094e6acaf7ab869a1f1c34825ba1fe3173fe350 Mon Sep 17 00:00:00 2001 From: Alexandre Ghiti Date: Fri, 4 Jun 2021 13:49:48 +0200 Subject: riscv: Simplify xip and !xip kernel address conversion macros To simplify the kernel address conversion code, make the same definition of kernel_mapping_pa_to_va and kernel_mapping_va_to_pa compatible for both xip and !xip kernel by defining XIP_OFFSET to 0 in !xip kernel. Signed-off-by: Alexandre Ghiti Reviewed-by: Anup Patel Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/pgtable.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'arch/riscv/include/asm/pgtable.h') diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 9469f464e71a..0344c060ca41 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -77,6 +77,8 @@ #ifdef CONFIG_XIP_KERNEL #define XIP_OFFSET SZ_8M +#else +#define XIP_OFFSET 0 #endif #ifndef __ASSEMBLY__ -- cgit From 9eb4fcff220790f4afadf59160f2c696e99f0a84 Mon Sep 17 00:00:00 2001 From: Nanyong Sun Date: Thu, 17 Jun 2021 17:58:31 +0800 Subject: riscv: mm: fix build errors caused by mk_pmd() With "riscv: mm: add THP support on 64-bit", mk_pmd() function introduce build errors, 1.build with CONFIG_ARCH_RV32I=y: arch/riscv/include/asm/pgtable.h: In function 'mk_pmd': arch/riscv/include/asm/pgtable.h:513:9: error: implicit declaration of function 'pfn_pmd'; did you mean 'pfn_pgd'? [-Werror=implicit-function-declaration] 2.build with CONFIG_SPARSEMEM=y && CONFIG_SPARSEMEM_VMEMMAP=n arch/riscv/include/asm/pgtable.h: In function 'mk_pmd': include/asm-generic/memory_model.h:64:14: error: implicit declaration of function 'page_to_section'; did you mean 'present_section'? [-Werror=implicit-function-declaration] Move the definition of mk_pmd to pgtable-64.h to fix the first error. Use macro definition instead of inline function for mk_pmd to fix the second problem. It is similar to the mk_pte macro. Reported-by: kernel test robot Signed-off-by: Nanyong Sun Tested-by: Geert Uytterhoeven Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/pgtable.h | 5 ----- 1 file changed, 5 deletions(-) (limited to 'arch/riscv/include/asm/pgtable.h') diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 2056faf06f13..559ca4186b5c 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -516,11 +516,6 @@ static inline unsigned long pmd_pfn(pmd_t pmd) return ((__pmd_to_phys(pmd) & PMD_MASK) >> PAGE_SHIFT); } -static inline pmd_t mk_pmd(struct page *page, pgprot_t prot) -{ - return pfn_pmd(page_to_pfn(page), prot); -} - static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) { return pte_pmd(pte_modify(pmd_pte(pmd), newprot)); -- cgit