diff options
Diffstat (limited to 'arch/riscv/include/asm/pgalloc.h')
| -rw-r--r-- | arch/riscv/include/asm/pgalloc.h | 124 |
1 files changed, 71 insertions, 53 deletions
diff --git a/arch/riscv/include/asm/pgalloc.h b/arch/riscv/include/asm/pgalloc.h index 94043cf83c90..770ce18a7328 100644 --- a/arch/riscv/include/asm/pgalloc.h +++ b/arch/riscv/include/asm/pgalloc.h @@ -1,23 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (C) 2009 Chen Liqin <liqin.chen@sunplusct.com> * Copyright (C) 2012 Regents of the University of California - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation, version 2. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef _ASM_RISCV_PGALLOC_H #define _ASM_RISCV_PGALLOC_H #include <linux/mm.h> +#include <asm/sbi.h> #include <asm/tlb.h> +#ifdef CONFIG_MMU +#define __HAVE_ARCH_PUD_FREE +#include <asm-generic/pgalloc.h> + static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte) { @@ -41,82 +38,103 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) set_pud(pud, __pud((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE)); } -#endif /* __PAGETABLE_PMD_FOLDED */ -#define pmd_pgtable(pmd) pmd_page(pmd) - -static inline pgd_t *pgd_alloc(struct mm_struct *mm) +static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) { - pgd_t *pgd; + if (pgtable_l4_enabled) { + unsigned long pfn = virt_to_pfn(pud); - pgd = (pgd_t *)__get_free_page(GFP_KERNEL); - if (likely(pgd != NULL)) { - memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); - /* Copy kernel mappings */ - memcpy(pgd + USER_PTRS_PER_PGD, - init_mm.pgd + USER_PTRS_PER_PGD, - (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); + set_p4d(p4d, __p4d((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE)); } - return pgd; } -static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) +static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, + pud_t *pud) { - free_page((unsigned long)pgd); + if (pgtable_l4_enabled) { + unsigned long pfn = virt_to_pfn(pud); + + set_p4d_safe(p4d, + __p4d((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE)); + } } -#ifndef __PAGETABLE_PMD_FOLDED +static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) +{ + if (pgtable_l5_enabled) { + unsigned long pfn = virt_to_pfn(p4d); -static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) + set_pgd(pgd, __pgd((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE)); + } +} + +static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, + p4d_t *p4d) { - return (pmd_t *)__get_free_page( - GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_ZERO); + if (pgtable_l5_enabled) { + unsigned long pfn = virt_to_pfn(p4d); + + set_pgd_safe(pgd, + __pgd((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE)); + } } -static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) +#define pud_free pud_free +static inline void pud_free(struct mm_struct *mm, pud_t *pud) { - free_page((unsigned long)pmd); + if (pgtable_l4_enabled) + __pud_free(mm, pud); } -#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) +static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, + unsigned long addr) +{ + if (pgtable_l4_enabled) + tlb_remove_ptdesc(tlb, virt_to_ptdesc(pud)); +} +static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, + unsigned long addr) +{ + if (pgtable_l5_enabled) + tlb_remove_ptdesc(tlb, virt_to_ptdesc(p4d)); +} #endif /* __PAGETABLE_PMD_FOLDED */ -static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm) +static inline void sync_kernel_mappings(pgd_t *pgd) { - return (pte_t *)__get_free_page( - GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_ZERO); + memcpy(pgd + USER_PTRS_PER_PGD, + init_mm.pgd + USER_PTRS_PER_PGD, + (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); } -static inline struct page *pte_alloc_one(struct mm_struct *mm) +static inline pgd_t *pgd_alloc(struct mm_struct *mm) { - struct page *pte; + pgd_t *pgd; - pte = alloc_page(GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_ZERO); - if (likely(pte != NULL)) - pgtable_page_ctor(pte); - return pte; + pgd = __pgd_alloc(mm, 0); + if (likely(pgd != NULL)) { + /* Copy kernel mappings */ + sync_kernel_mappings(pgd); + } + return pgd; } -static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) -{ - free_page((unsigned long)pte); -} +#ifndef __PAGETABLE_PMD_FOLDED -static inline void pte_free(struct mm_struct *mm, pgtable_t pte) +static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, + unsigned long addr) { - pgtable_page_dtor(pte); - __free_page(pte); + tlb_remove_ptdesc(tlb, virt_to_ptdesc(pmd)); } -#define __pte_free_tlb(tlb, pte, buf) \ -do { \ - pgtable_page_dtor(pte); \ - tlb_remove_page((tlb), pte); \ -} while (0) +#endif /* __PAGETABLE_PMD_FOLDED */ -static inline void check_pgt_cache(void) +static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, + unsigned long addr) { + tlb_remove_ptdesc(tlb, page_ptdesc(pte)); } +#endif /* CONFIG_MMU */ #endif /* _ASM_RISCV_PGALLOC_H */ |