diff options
| -rw-r--r-- | arch/arm/mm/Makefile | 1 | ||||
| -rw-r--r-- | arch/arm/mm/proc-macros.S | 15 | ||||
| -rw-r--r-- | arch/arm/mm/tlb-fa.S | 12 | ||||
| -rw-r--r-- | arch/arm/mm/tlb-v4.S | 15 | ||||
| -rw-r--r-- | arch/arm/mm/tlb-v4wb.S | 12 | ||||
| -rw-r--r-- | arch/arm/mm/tlb-v4wbi.S | 12 | ||||
| -rw-r--r-- | arch/arm/mm/tlb-v6.S | 12 | ||||
| -rw-r--r-- | arch/arm/mm/tlb-v7.S | 14 | ||||
| -rw-r--r-- | arch/arm/mm/tlb.c | 84 |
9 files changed, 119 insertions, 58 deletions
diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile index 71b858c9b10c..cc8255fdf56e 100644 --- a/arch/arm/mm/Makefile +++ b/arch/arm/mm/Makefile @@ -62,6 +62,7 @@ obj-$(CONFIG_CPU_TLB_FEROCEON) += tlb-v4wbi.o # reuse v4wbi TLB functions obj-$(CONFIG_CPU_TLB_V6) += tlb-v6.o obj-$(CONFIG_CPU_TLB_V7) += tlb-v7.o obj-$(CONFIG_CPU_TLB_FA) += tlb-fa.o +obj-y += tlb.o obj-$(CONFIG_CPU_ARM7TDMI) += proc-arm7tdmi.o obj-$(CONFIG_CPU_ARM720T) += proc-arm720.o diff --git a/arch/arm/mm/proc-macros.S b/arch/arm/mm/proc-macros.S index e43f6d716b4b..c0acfeac3e84 100644 --- a/arch/arm/mm/proc-macros.S +++ b/arch/arm/mm/proc-macros.S @@ -338,21 +338,6 @@ ENTRY(\name\()_cache_fns) .size \name\()_cache_fns, . - \name\()_cache_fns .endm -.macro define_tlb_functions name:req, flags_up:req, flags_smp - .type \name\()_tlb_fns, #object - .align 2 -ENTRY(\name\()_tlb_fns) - .long \name\()_flush_user_tlb_range - .long \name\()_flush_kern_tlb_range - .ifnb \flags_smp - ALT_SMP(.long \flags_smp ) - ALT_UP(.long \flags_up ) - .else - .long \flags_up - .endif - .size \name\()_tlb_fns, . - \name\()_tlb_fns -.endm - .macro globl_equ x, y .globl \x .equ \x, \y diff --git a/arch/arm/mm/tlb-fa.S b/arch/arm/mm/tlb-fa.S index def6161ec452..85a6fe766b21 100644 --- a/arch/arm/mm/tlb-fa.S +++ b/arch/arm/mm/tlb-fa.S @@ -15,6 +15,7 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/tlbflush.h> @@ -31,7 +32,7 @@ * - mm - mm_struct describing address space */ .align 4 -ENTRY(fa_flush_user_tlb_range) +SYM_TYPED_FUNC_START(fa_flush_user_tlb_range) vma_vm_mm ip, r2 act_mm r3 @ get current->active_mm eors r3, ip, r3 @ == mm ? @@ -46,9 +47,10 @@ ENTRY(fa_flush_user_tlb_range) blo 1b mcr p15, 0, r3, c7, c10, 4 @ data write barrier ret lr +SYM_FUNC_END(fa_flush_user_tlb_range) -ENTRY(fa_flush_kern_tlb_range) +SYM_TYPED_FUNC_START(fa_flush_kern_tlb_range) mov r3, #0 mcr p15, 0, r3, c7, c10, 4 @ drain WB bic r0, r0, #0x0ff @@ -60,8 +62,4 @@ ENTRY(fa_flush_kern_tlb_range) mcr p15, 0, r3, c7, c10, 4 @ data write barrier mcr p15, 0, r3, c7, c5, 4 @ prefetch flush (isb) ret lr - - __INITDATA - - /* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */ - define_tlb_functions fa, fa_tlb_flags +SYM_FUNC_END(fa_flush_kern_tlb_range) diff --git a/arch/arm/mm/tlb-v4.S b/arch/arm/mm/tlb-v4.S index b962b4e75158..09ff69008d94 100644 --- a/arch/arm/mm/tlb-v4.S +++ b/arch/arm/mm/tlb-v4.S @@ -11,6 +11,7 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/tlbflush.h> @@ -27,7 +28,7 @@ * - mm - mm_struct describing address space */ .align 5 -ENTRY(v4_flush_user_tlb_range) +SYM_TYPED_FUNC_START(v4_flush_user_tlb_range) vma_vm_mm ip, r2 act_mm r3 @ get current->active_mm eors r3, ip, r3 @ == mm ? @@ -40,6 +41,7 @@ ENTRY(v4_flush_user_tlb_range) cmp r0, r1 blo 1b ret lr +SYM_FUNC_END(v4_flush_user_tlb_range) /* * v4_flush_kern_tlb_range(start, end) @@ -50,10 +52,11 @@ ENTRY(v4_flush_user_tlb_range) * - start - virtual address (may not be aligned) * - end - virtual address (may not be aligned) */ +#ifdef CONFIG_CFI_CLANG +SYM_TYPED_FUNC_START(v4_flush_kern_tlb_range) + b .v4_flush_kern_tlb_range +SYM_FUNC_END(v4_flush_kern_tlb_range) +#else .globl v4_flush_kern_tlb_range .equ v4_flush_kern_tlb_range, .v4_flush_kern_tlb_range - - __INITDATA - - /* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */ - define_tlb_functions v4, v4_tlb_flags +#endif diff --git a/arch/arm/mm/tlb-v4wb.S b/arch/arm/mm/tlb-v4wb.S index 9348bba7586a..04e46c359e75 100644 --- a/arch/arm/mm/tlb-v4wb.S +++ b/arch/arm/mm/tlb-v4wb.S @@ -11,6 +11,7 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/tlbflush.h> @@ -27,7 +28,7 @@ * - mm - mm_struct describing address space */ .align 5 -ENTRY(v4wb_flush_user_tlb_range) +SYM_TYPED_FUNC_START(v4wb_flush_user_tlb_range) vma_vm_mm ip, r2 act_mm r3 @ get current->active_mm eors r3, ip, r3 @ == mm ? @@ -43,6 +44,7 @@ ENTRY(v4wb_flush_user_tlb_range) cmp r0, r1 blo 1b ret lr +SYM_FUNC_END(v4wb_flush_user_tlb_range) /* * v4_flush_kern_tlb_range(start, end) @@ -53,7 +55,7 @@ ENTRY(v4wb_flush_user_tlb_range) * - start - virtual address (may not be aligned) * - end - virtual address (may not be aligned) */ -ENTRY(v4wb_flush_kern_tlb_range) +SYM_TYPED_FUNC_START(v4wb_flush_kern_tlb_range) mov r3, #0 mcr p15, 0, r3, c7, c10, 4 @ drain WB bic r0, r0, #0x0ff @@ -64,8 +66,4 @@ ENTRY(v4wb_flush_kern_tlb_range) cmp r0, r1 blo 1b ret lr - - __INITDATA - - /* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */ - define_tlb_functions v4wb, v4wb_tlb_flags +SYM_FUNC_END(v4wb_flush_kern_tlb_range) diff --git a/arch/arm/mm/tlb-v4wbi.S b/arch/arm/mm/tlb-v4wbi.S index d4f9040a4111..502dfe5628a3 100644 --- a/arch/arm/mm/tlb-v4wbi.S +++ b/arch/arm/mm/tlb-v4wbi.S @@ -11,6 +11,7 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/tlbflush.h> @@ -26,7 +27,7 @@ * - mm - mm_struct describing address space */ .align 5 -ENTRY(v4wbi_flush_user_tlb_range) +SYM_TYPED_FUNC_START(v4wbi_flush_user_tlb_range) vma_vm_mm ip, r2 act_mm r3 @ get current->active_mm eors r3, ip, r3 @ == mm ? @@ -43,8 +44,9 @@ ENTRY(v4wbi_flush_user_tlb_range) cmp r0, r1 blo 1b ret lr +SYM_FUNC_END(v4wbi_flush_user_tlb_range) -ENTRY(v4wbi_flush_kern_tlb_range) +SYM_TYPED_FUNC_START(v4wbi_flush_kern_tlb_range) mov r3, #0 mcr p15, 0, r3, c7, c10, 4 @ drain WB bic r0, r0, #0x0ff @@ -55,8 +57,4 @@ ENTRY(v4wbi_flush_kern_tlb_range) cmp r0, r1 blo 1b ret lr - - __INITDATA - - /* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */ - define_tlb_functions v4wbi, v4wbi_tlb_flags +SYM_FUNC_END(v4wbi_flush_kern_tlb_range) diff --git a/arch/arm/mm/tlb-v6.S b/arch/arm/mm/tlb-v6.S index 1d91e49b2c2d..8256a67ac654 100644 --- a/arch/arm/mm/tlb-v6.S +++ b/arch/arm/mm/tlb-v6.S @@ -9,6 +9,7 @@ */ #include <linux/init.h> #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/asm-offsets.h> #include <asm/assembler.h> #include <asm/page.h> @@ -32,7 +33,7 @@ * - the "Invalidate single entry" instruction will invalidate * both the I and the D TLBs on Harvard-style TLBs */ -ENTRY(v6wbi_flush_user_tlb_range) +SYM_TYPED_FUNC_START(v6wbi_flush_user_tlb_range) vma_vm_mm r3, r2 @ get vma->vm_mm mov ip, #0 mmid r3, r3 @ get vm_mm->context.id @@ -56,6 +57,7 @@ ENTRY(v6wbi_flush_user_tlb_range) blo 1b mcr p15, 0, ip, c7, c10, 4 @ data synchronization barrier ret lr +SYM_FUNC_END(v6wbi_flush_user_tlb_range) /* * v6wbi_flush_kern_tlb_range(start,end) @@ -65,7 +67,7 @@ ENTRY(v6wbi_flush_user_tlb_range) * - start - start address (may not be aligned) * - end - end address (exclusive, may not be aligned) */ -ENTRY(v6wbi_flush_kern_tlb_range) +SYM_TYPED_FUNC_START(v6wbi_flush_kern_tlb_range) mov r2, #0 mcr p15, 0, r2, c7, c10, 4 @ drain write buffer mov r0, r0, lsr #PAGE_SHIFT @ align address @@ -85,8 +87,4 @@ ENTRY(v6wbi_flush_kern_tlb_range) mcr p15, 0, r2, c7, c10, 4 @ data synchronization barrier mcr p15, 0, r2, c7, c5, 4 @ prefetch flush (isb) ret lr - - __INIT - - /* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */ - define_tlb_functions v6wbi, v6wbi_tlb_flags +SYM_FUNC_END(v6wbi_flush_kern_tlb_range) diff --git a/arch/arm/mm/tlb-v7.S b/arch/arm/mm/tlb-v7.S index 35fd6d4f0d03..f1aa0764a2cc 100644 --- a/arch/arm/mm/tlb-v7.S +++ b/arch/arm/mm/tlb-v7.S @@ -10,6 +10,7 @@ */ #include <linux/init.h> #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/page.h> @@ -31,7 +32,7 @@ * - the "Invalidate single entry" instruction will invalidate * both the I and the D TLBs on Harvard-style TLBs */ -ENTRY(v7wbi_flush_user_tlb_range) +SYM_TYPED_FUNC_START(v7wbi_flush_user_tlb_range) vma_vm_mm r3, r2 @ get vma->vm_mm mmid r3, r3 @ get vm_mm->context.id dsb ish @@ -57,7 +58,7 @@ ENTRY(v7wbi_flush_user_tlb_range) blo 1b dsb ish ret lr -ENDPROC(v7wbi_flush_user_tlb_range) +SYM_FUNC_END(v7wbi_flush_user_tlb_range) /* * v7wbi_flush_kern_tlb_range(start,end) @@ -67,7 +68,7 @@ ENDPROC(v7wbi_flush_user_tlb_range) * - start - start address (may not be aligned) * - end - end address (exclusive, may not be aligned) */ -ENTRY(v7wbi_flush_kern_tlb_range) +SYM_TYPED_FUNC_START(v7wbi_flush_kern_tlb_range) dsb ish mov r0, r0, lsr #PAGE_SHIFT @ align address mov r1, r1, lsr #PAGE_SHIFT @@ -86,9 +87,4 @@ ENTRY(v7wbi_flush_kern_tlb_range) dsb ish isb ret lr -ENDPROC(v7wbi_flush_kern_tlb_range) - - __INIT - - /* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */ - define_tlb_functions v7wbi, v7wbi_tlb_flags_up, flags_smp=v7wbi_tlb_flags_smp +SYM_FUNC_END(v7wbi_flush_kern_tlb_range) diff --git a/arch/arm/mm/tlb.c b/arch/arm/mm/tlb.c new file mode 100644 index 000000000000..42359793120b --- /dev/null +++ b/arch/arm/mm/tlb.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright 2024 Google LLC +// Author: Ard Biesheuvel <ardb@google.com> + +#include <linux/types.h> +#include <asm/tlbflush.h> + +#ifdef CONFIG_CPU_TLB_V4WT +void v4_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *); +void v4_flush_kern_tlb_range(unsigned long, unsigned long); + +struct cpu_tlb_fns v4_tlb_fns __initconst = { + .flush_user_range = v4_flush_user_tlb_range, + .flush_kern_range = v4_flush_kern_tlb_range, + .tlb_flags = v4_tlb_flags, +}; +#endif + +#ifdef CONFIG_CPU_TLB_V4WB +void v4wb_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *); +void v4wb_flush_kern_tlb_range(unsigned long, unsigned long); + +struct cpu_tlb_fns v4wb_tlb_fns __initconst = { + .flush_user_range = v4wb_flush_user_tlb_range, + .flush_kern_range = v4wb_flush_kern_tlb_range, + .tlb_flags = v4wb_tlb_flags, +}; +#endif + +#if defined(CONFIG_CPU_TLB_V4WBI) || defined(CONFIG_CPU_TLB_FEROCEON) +void v4wbi_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *); +void v4wbi_flush_kern_tlb_range(unsigned long, unsigned long); + +struct cpu_tlb_fns v4wbi_tlb_fns __initconst = { + .flush_user_range = v4wbi_flush_user_tlb_range, + .flush_kern_range = v4wbi_flush_kern_tlb_range, + .tlb_flags = v4wbi_tlb_flags, +}; +#endif + +#ifdef CONFIG_CPU_TLB_V6 +void v6wbi_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *); +void v6wbi_flush_kern_tlb_range(unsigned long, unsigned long); + +struct cpu_tlb_fns v6wbi_tlb_fns __initconst = { + .flush_user_range = v6wbi_flush_user_tlb_range, + .flush_kern_range = v6wbi_flush_kern_tlb_range, + .tlb_flags = v6wbi_tlb_flags, +}; +#endif + +#ifdef CONFIG_CPU_TLB_V7 +void v7wbi_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *); +void v7wbi_flush_kern_tlb_range(unsigned long, unsigned long); + +struct cpu_tlb_fns v7wbi_tlb_fns __initconst = { + .flush_user_range = v7wbi_flush_user_tlb_range, + .flush_kern_range = v7wbi_flush_kern_tlb_range, + .tlb_flags = IS_ENABLED(CONFIG_SMP) ? v7wbi_tlb_flags_smp + : v7wbi_tlb_flags_up, +}; + +#ifdef CONFIG_SMP_ON_UP +/* This will be run-time patched so the offset better be right */ +static_assert(offsetof(struct cpu_tlb_fns, tlb_flags) == 8); + +asm(" .pushsection \".alt.smp.init\", \"a\" \n" \ + " .align 2 \n" \ + " .long v7wbi_tlb_fns + 8 - . \n" \ + " .long " __stringify(v7wbi_tlb_flags_up) " \n" \ + " .popsection \n"); +#endif +#endif + +#ifdef CONFIG_CPU_TLB_FA +void fa_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *); +void fa_flush_kern_tlb_range(unsigned long, unsigned long); + +struct cpu_tlb_fns fa_tlb_fns __initconst = { + .flush_user_range = fa_flush_user_tlb_range, + .flush_kern_range = fa_flush_kern_tlb_range, + .tlb_flags = fa_tlb_flags, +}; +#endif |