diff options
Diffstat (limited to 'arch/powerpc/include/asm/mmu.h')
| -rw-r--r-- | arch/powerpc/include/asm/mmu.h | 346 |
1 files changed, 266 insertions, 80 deletions
diff --git a/arch/powerpc/include/asm/mmu.h b/arch/powerpc/include/asm/mmu.h index 691fd8aca939..5f9c5d436e17 100644 --- a/arch/powerpc/include/asm/mmu.h +++ b/arch/powerpc/include/asm/mmu.h @@ -1,36 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_MMU_H_ #define _ASM_POWERPC_MMU_H_ #ifdef __KERNEL__ #include <linux/types.h> -#include <asm/asm-compat.h> -#include <asm/feature-fixups.h> +#include <asm/asm-const.h> /* * MMU features bit definitions */ /* - * First half is MMU families + * MMU families */ #define MMU_FTR_HPTE_TABLE ASM_CONST(0x00000001) #define MMU_FTR_TYPE_8xx ASM_CONST(0x00000002) -#define MMU_FTR_TYPE_40x ASM_CONST(0x00000004) #define MMU_FTR_TYPE_44x ASM_CONST(0x00000008) #define MMU_FTR_TYPE_FSL_E ASM_CONST(0x00000010) -#define MMU_FTR_TYPE_3E ASM_CONST(0x00000020) -#define MMU_FTR_TYPE_47x ASM_CONST(0x00000040) +#define MMU_FTR_TYPE_47x ASM_CONST(0x00000020) + +/* Radix page table supported and enabled */ +#define MMU_FTR_TYPE_RADIX ASM_CONST(0x00000040) + +/* + * Individual features below. + */ + +/* + * Supports KUAP feature + * key 0 controlling userspace addresses on radix + * Key 3 on hash + */ +#define MMU_FTR_KUAP ASM_CONST(0x00000200) /* - * This is individual features + * Supports KUEP feature + * key 0 controlling userspace addresses on radix + * Key 3 on hash */ +#define MMU_FTR_BOOK3S_KUEP ASM_CONST(0x00000400) + +/* + * Support for memory protection keys. + */ +#define MMU_FTR_PKEY ASM_CONST(0x00000800) + +/* Guest Translation Shootdown Enable */ +#define MMU_FTR_GTSE ASM_CONST(0x00001000) + +/* + * Support for 68 bit VA space. We added that from ISA 2.05 + */ +#define MMU_FTR_68_BIT_VA ASM_CONST(0x00002000) +/* + * Kernel read only support. + * We added the ppp value 0b110 in ISA 2.04. + */ +#define MMU_FTR_KERNEL_RO ASM_CONST(0x00004000) + +/* + * We need to clear top 16bits of va (from the remaining 64 bits )in + * tlbie* instructions + */ +#define MMU_FTR_TLBIE_CROP_VA ASM_CONST(0x00008000) /* Enable use of high BAT registers */ #define MMU_FTR_USE_HIGH_BATS ASM_CONST(0x00010000) /* Enable >32-bit physical addresses on 32-bit processor, only used - * by CONFIG_6xx currently as BookE supports that from day 1 + * by CONFIG_PPC_BOOK3S_32 currently as BookE supports that from day 1 */ #define MMU_FTR_BIG_PHYS ASM_CONST(0x00020000) @@ -56,18 +95,9 @@ */ #define MMU_FTR_NEED_DTLB_SW_LRU ASM_CONST(0x00200000) -/* Enable use of TLB reservation. Processor should support tlbsrx. - * instruction and MAS0[WQ]. - */ -#define MMU_FTR_USE_TLBRSRV ASM_CONST(0x00800000) - -/* Use paired MAS registers (MAS7||MAS3, etc.) - */ -#define MMU_FTR_USE_PAIRED_MAS ASM_CONST(0x01000000) - -/* MMU is SLB-based +/* Doesn't support the B bit (1T segment) in SLBIE */ -#define MMU_FTR_SLB ASM_CONST(0x02000000) +#define MMU_FTR_NO_SLBIE_B ASM_CONST(0x02000000) /* Support 16M large pages */ @@ -89,62 +119,189 @@ */ #define MMU_FTR_1T_SEGMENT ASM_CONST(0x40000000) -/* Doesn't support the B bit (1T segment) in SLBIE - */ -#define MMU_FTR_NO_SLBIE_B ASM_CONST(0x80000000) +// NX paste RMA reject in DSI +#define MMU_FTR_NX_DSI ASM_CONST(0x80000000) /* MMU feature bit sets for various CPUs */ -#define MMU_FTRS_DEFAULT_HPTE_ARCH_V2 \ - MMU_FTR_HPTE_TABLE | MMU_FTR_PPCAS_ARCH_V2 -#define MMU_FTRS_POWER4 MMU_FTRS_DEFAULT_HPTE_ARCH_V2 -#define MMU_FTRS_PPC970 MMU_FTRS_POWER4 -#define MMU_FTRS_POWER5 MMU_FTRS_POWER4 | MMU_FTR_LOCKLESS_TLBIE -#define MMU_FTRS_POWER6 MMU_FTRS_POWER4 | MMU_FTR_LOCKLESS_TLBIE -#define MMU_FTRS_POWER7 MMU_FTRS_POWER4 | MMU_FTR_LOCKLESS_TLBIE -#define MMU_FTRS_POWER8 MMU_FTRS_POWER4 | MMU_FTR_LOCKLESS_TLBIE +#define MMU_FTRS_DEFAULT_HPTE_ARCH_V2 (MMU_FTR_HPTE_TABLE | MMU_FTR_TLBIEL | MMU_FTR_16M_PAGE) +#define MMU_FTRS_POWER MMU_FTRS_DEFAULT_HPTE_ARCH_V2 +#define MMU_FTRS_PPC970 MMU_FTRS_POWER | MMU_FTR_TLBIE_CROP_VA +#define MMU_FTRS_POWER5 MMU_FTRS_POWER | MMU_FTR_LOCKLESS_TLBIE +#define MMU_FTRS_POWER6 MMU_FTRS_POWER5 | MMU_FTR_KERNEL_RO | MMU_FTR_68_BIT_VA +#define MMU_FTRS_POWER7 MMU_FTRS_POWER6 +#define MMU_FTRS_POWER8 MMU_FTRS_POWER6 +#define MMU_FTRS_POWER9 MMU_FTRS_POWER6 +#define MMU_FTRS_POWER10 MMU_FTRS_POWER6 +#define MMU_FTRS_POWER11 MMU_FTRS_POWER6 #define MMU_FTRS_CELL MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \ MMU_FTR_CI_LARGE_PAGE #define MMU_FTRS_PA6T MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \ MMU_FTR_CI_LARGE_PAGE | MMU_FTR_NO_SLBIE_B -#define MMU_FTRS_A2 MMU_FTR_TYPE_3E | MMU_FTR_USE_TLBILX | \ - MMU_FTR_USE_TLBIVAX_BCAST | \ - MMU_FTR_LOCK_BCAST_INVAL | \ - MMU_FTR_USE_TLBRSRV | \ - MMU_FTR_USE_PAIRED_MAS | \ - MMU_FTR_TLBIEL | \ - MMU_FTR_16M_PAGE -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ +#include <linux/bug.h> #include <asm/cputable.h> +#include <asm/page.h> + +typedef pte_t *pgtable_t; -#ifdef CONFIG_PPC_FSL_BOOK3E -#include <asm/percpu.h> -DECLARE_PER_CPU(int, next_tlbcam_idx); +enum { + MMU_FTRS_POSSIBLE = +#if defined(CONFIG_PPC_BOOK3S_604) + MMU_FTR_HPTE_TABLE | #endif +#ifdef CONFIG_PPC_8xx + MMU_FTR_TYPE_8xx | +#endif +#ifdef CONFIG_PPC_47x + MMU_FTR_TYPE_47x | MMU_FTR_USE_TLBIVAX_BCAST | MMU_FTR_LOCK_BCAST_INVAL | +#elif defined(CONFIG_44x) + MMU_FTR_TYPE_44x | +#endif +#ifdef CONFIG_PPC_E500 + MMU_FTR_TYPE_FSL_E | MMU_FTR_BIG_PHYS | MMU_FTR_USE_TLBILX | +#endif +#ifdef CONFIG_PPC_BOOK3S_32 + MMU_FTR_USE_HIGH_BATS | +#endif +#ifdef CONFIG_PPC_83xx + MMU_FTR_NEED_DTLB_SW_LRU | +#endif +#ifdef CONFIG_PPC_BOOK3S_64 + MMU_FTR_KERNEL_RO | +#ifdef CONFIG_PPC_64S_HASH_MMU + MMU_FTR_NO_SLBIE_B | MMU_FTR_16M_PAGE | MMU_FTR_TLBIEL | + MMU_FTR_LOCKLESS_TLBIE | MMU_FTR_CI_LARGE_PAGE | + MMU_FTR_1T_SEGMENT | MMU_FTR_TLBIE_CROP_VA | + MMU_FTR_68_BIT_VA | MMU_FTR_HPTE_TABLE | +#endif +#ifdef CONFIG_PPC_RADIX_MMU + MMU_FTR_TYPE_RADIX | + MMU_FTR_GTSE | MMU_FTR_NX_DSI | +#endif /* CONFIG_PPC_RADIX_MMU */ +#endif +#ifdef CONFIG_PPC_KUAP + MMU_FTR_KUAP | +#endif /* CONFIG_PPC_KUAP */ +#ifdef CONFIG_PPC_MEM_KEYS + MMU_FTR_PKEY | +#endif +#ifdef CONFIG_PPC_KUEP + MMU_FTR_BOOK3S_KUEP | +#endif /* CONFIG_PPC_KUAP */ + + 0, +}; + +#if defined(CONFIG_PPC_BOOK3S_604) && !defined(CONFIG_PPC_BOOK3S_603) +#define MMU_FTRS_ALWAYS MMU_FTR_HPTE_TABLE +#endif +#ifdef CONFIG_PPC_8xx +#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_8xx +#endif +#ifdef CONFIG_PPC_47x +#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_47x +#elif defined(CONFIG_44x) +#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_44x +#endif +#ifdef CONFIG_PPC_E500 +#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_FSL_E +#endif + +/* BOOK3S_64 options */ +#if defined(CONFIG_PPC_RADIX_MMU) && !defined(CONFIG_PPC_64S_HASH_MMU) +#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_RADIX +#elif !defined(CONFIG_PPC_RADIX_MMU) && defined(CONFIG_PPC_64S_HASH_MMU) +#define MMU_FTRS_ALWAYS MMU_FTR_HPTE_TABLE +#endif + +#ifndef MMU_FTRS_ALWAYS +#define MMU_FTRS_ALWAYS 0 +#endif + +static __always_inline bool early_mmu_has_feature(unsigned long feature) +{ + if (MMU_FTRS_ALWAYS & feature) + return true; + + return !!(MMU_FTRS_POSSIBLE & cur_cpu_spec->mmu_features & feature); +} + +#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECKS +#include <linux/jump_label.h> + +#define NUM_MMU_FTR_KEYS 32 + +extern struct static_key_true mmu_feature_keys[NUM_MMU_FTR_KEYS]; -static inline int mmu_has_feature(unsigned long feature) +extern void mmu_feature_keys_init(void); + +static __always_inline bool mmu_has_feature(unsigned long feature) { - return (cur_cpu_spec->mmu_features & feature); + int i; + + BUILD_BUG_ON(!__builtin_constant_p(feature)); + BUILD_BUG_ON(__builtin_popcountl(feature) > 1); + +#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECK_DEBUG + if (!static_key_feature_checks_initialized) { + printk("Warning! mmu_has_feature() used prior to jump label init!\n"); + dump_stack(); + return early_mmu_has_feature(feature); + } +#endif + + if (MMU_FTRS_ALWAYS & feature) + return true; + + if (!(MMU_FTRS_POSSIBLE & feature)) + return false; + + i = __builtin_ctzl(feature); + return static_branch_likely(&mmu_feature_keys[i]); } static inline void mmu_clear_feature(unsigned long feature) { + int i; + + i = __builtin_ctzl(feature); cur_cpu_spec->mmu_features &= ~feature; + static_branch_disable(&mmu_feature_keys[i]); } +#else -extern unsigned int __start___mmu_ftr_fixup, __stop___mmu_ftr_fixup; +static inline void mmu_feature_keys_init(void) +{ -/* MMU initialization */ -extern void early_init_mmu(void); -extern void early_init_mmu_secondary(void); +} -extern void setup_initial_memory_limit(phys_addr_t first_memblock_base, - phys_addr_t first_memblock_size); +static __always_inline bool mmu_has_feature(unsigned long feature) +{ + return early_mmu_has_feature(feature); +} + +static inline void mmu_clear_feature(unsigned long feature) +{ + cur_cpu_spec->mmu_features &= ~feature; +} +#endif /* CONFIG_JUMP_LABEL */ + +extern unsigned int __start___mmu_ftr_fixup, __stop___mmu_ftr_fixup; #ifdef CONFIG_PPC64 /* This is our real memory area size on ppc64 server, on embedded, we * make it match the size our of bolted TLB area */ extern u64 ppc64_rma_size; + +/* Cleanup function used by kexec */ +extern void mmu_cleanup_all(void); +extern void radix__mmu_cleanup_all(void); + +/* Functions for creating and updating partition table on POWER9 */ +extern void mmu_partition_table_init(void); +extern void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0, + unsigned long dw1, bool flush); #endif /* CONFIG_PPC64 */ struct mm_struct; @@ -156,7 +313,26 @@ static inline void assert_pte_locked(struct mm_struct *mm, unsigned long addr) } #endif /* !CONFIG_DEBUG_VM */ -#endif /* !__ASSEMBLY__ */ +static __always_inline bool radix_enabled(void) +{ + return mmu_has_feature(MMU_FTR_TYPE_RADIX); +} + +static __always_inline bool early_radix_enabled(void) +{ + return early_mmu_has_feature(MMU_FTR_TYPE_RADIX); +} + +static inline bool strict_kernel_rwx_enabled(void) +{ + return IS_ENABLED(CONFIG_STRICT_KERNEL_RWX) && rodata_enabled; +} + +static inline bool strict_module_rwx_enabled(void) +{ + return IS_ENABLED(CONFIG_STRICT_MODULE_RWX) && strict_kernel_rwx_enabled(); +} +#endif /* !__ASSEMBLER__ */ /* The kernel use the constants below to index in the page sizes array. * The use of fixed constants for this purpose is better for performances @@ -179,38 +355,48 @@ static inline void assert_pte_locked(struct mm_struct *mm, unsigned long addr) #define MMU_PAGE_64K 2 #define MMU_PAGE_64K_AP 3 /* "Admixed pages" (hash64 only) */ #define MMU_PAGE_256K 4 -#define MMU_PAGE_1M 5 -#define MMU_PAGE_4M 6 -#define MMU_PAGE_8M 7 -#define MMU_PAGE_16M 8 -#define MMU_PAGE_64M 9 -#define MMU_PAGE_256M 10 -#define MMU_PAGE_1G 11 -#define MMU_PAGE_16G 12 -#define MMU_PAGE_64G 13 - -#define MMU_PAGE_COUNT 14 - -#if defined(CONFIG_PPC_STD_MMU_64) -/* 64-bit classic hash table MMU */ -# include <asm/mmu-hash64.h> -#elif defined(CONFIG_PPC_STD_MMU_32) -/* 32-bit classic hash table MMU */ -# include <asm/mmu-hash32.h> -#elif defined(CONFIG_40x) -/* 40x-style software loaded TLB */ -# include <asm/mmu-40x.h> -#elif defined(CONFIG_44x) -/* 44x-style software loaded TLB */ -# include <asm/mmu-44x.h> -#elif defined(CONFIG_PPC_BOOK3E_MMU) -/* Freescale Book-E software loaded TLB or Book-3e (ISA 2.06+) MMU */ -# include <asm/mmu-book3e.h> -#elif defined (CONFIG_PPC_8xx) -/* Motorola/Freescale 8xx software loaded TLB */ -# include <asm/mmu-8xx.h> +#define MMU_PAGE_512K 5 +#define MMU_PAGE_1M 6 +#define MMU_PAGE_2M 7 +#define MMU_PAGE_4M 8 +#define MMU_PAGE_8M 9 +#define MMU_PAGE_16M 10 +#define MMU_PAGE_64M 11 +#define MMU_PAGE_256M 12 +#define MMU_PAGE_1G 13 +#define MMU_PAGE_16G 14 +#define MMU_PAGE_64G 15 + +/* + * N.B. we need to change the type of hpte_page_sizes if this gets to be > 16 + * Also we need to change he type of mm_context.low/high_slices_psize. + */ +#define MMU_PAGE_COUNT 16 + +#ifdef CONFIG_PPC_BOOK3S_64 +#include <asm/book3s/64/mmu.h> +#else /* CONFIG_PPC_BOOK3S_64 */ + +#ifndef __ASSEMBLER__ +/* MMU initialization */ +extern void early_init_mmu(void); +extern void early_init_mmu_secondary(void); +extern void setup_initial_memory_limit(phys_addr_t first_memblock_base, + phys_addr_t first_memblock_size); +static inline void mmu_early_init_devtree(void) { } + +static inline void pkey_early_init_devtree(void) {} + +extern void *abatron_pteptrs[2]; +#endif /* __ASSEMBLER__ */ #endif +#if defined(CONFIG_PPC_BOOK3S_32) +/* 32-bit classic hash table MMU */ +#include <asm/book3s/32/mmu-hash.h> +#elif defined(CONFIG_PPC_MMU_NOHASH) +#include <asm/nohash/mmu.h> +#endif #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_MMU_H_ */ |