diff options
| -rw-r--r-- | arch/arm64/include/asm/ktext.h | 12 | ||||
| -rw-r--r-- | arch/arm64/mm/ktext.c | 42 | ||||
| -rw-r--r-- | arch/arm64/mm/mmu.c | 5 |
3 files changed, 58 insertions, 1 deletions
diff --git a/arch/arm64/include/asm/ktext.h b/arch/arm64/include/asm/ktext.h index def0c894ed46..3da93c0d8595 100644 --- a/arch/arm64/include/asm/ktext.h +++ b/arch/arm64/include/asm/ktext.h @@ -7,12 +7,16 @@ #include <linux/kprobes.h> +#include <asm/pgtable-types.h> + #ifdef CONFIG_REPLICATE_KTEXT void ktext_replication_init(void); void ktext_replication_write(void *addr, void *data, size_t size); void __kprobes ktext_replication_patch(u32 *tp, __le32 insn); void ktext_replication_patch_alternative(__le32 *src, int nr_inst); +void ktext_replication_set_swapper_pgd(pgd_t *pgdp, pgd_t pgd); +void ktext_replication_init_tramp(void); #else @@ -28,6 +32,14 @@ static inline void ktext_replication_patch_alternative(__le32 *src, int nr_inst) { } +static inline void ktext_replication_set_swapper_pgd(pgd_t *pgdp, pgd_t pgd) +{ +} + +static inline void ktext_replication_init_tramp(void) +{ +} + #endif #endif diff --git a/arch/arm64/mm/ktext.c b/arch/arm64/mm/ktext.c index dd7512171296..2c212fb02594 100644 --- a/arch/arm64/mm/ktext.c +++ b/arch/arm64/mm/ktext.c @@ -14,6 +14,7 @@ #include <asm/cacheflush.h> #include <asm/ktext.h> #include <asm/memory.h> +#include <asm/pgalloc.h> struct pgtables *pgtables[MAX_NUMNODES] = { [0 ... MAX_NUMNODES - 1] = &pgtable_node0, @@ -118,7 +119,7 @@ void ktext_replication_patch_alternative(__le32 *src, int nr_inst) } } -/* Allocate memory for the replicated kernel texts. */ +/* Allocate page tables and memory for the replicated kernel texts. */ void __init ktext_replication_init(void) { size_t size = _etext - _stext; @@ -149,5 +150,44 @@ void __init ktext_replication_init(void) memcpy(kernel_texts[nid], _stext, size); caches_clean_inval_pou((u64)kernel_texts[nid], (u64)kernel_texts[nid] + size); + + /* Allocate the pagetables for this node */ + pgtables[nid] = memblock_alloc_node(sizeof(*pgtables[0]), + PGD_SIZE, nid); + + /* Copy initial swapper page directory */ + memcpy(pgtables[nid]->swapper_pg_dir, swapper_pg_dir, PGD_SIZE); + } +} + +void ktext_replication_set_swapper_pgd(pgd_t *pgdp, pgd_t pgd) +{ + unsigned long idx = pgdp - swapper_pg_dir; + int nid; + + if (WARN_ON_ONCE(idx >= PTRS_PER_PGD) || + WARN_ON_ONCE(idx == pgd_index((phys_addr_t)KERNEL_START))) + return; + + for_each_node(nid) { + if (pgtables[nid]->swapper_pg_dir == swapper_pg_dir) + continue; + + WRITE_ONCE(pgtables[nid]->swapper_pg_dir[idx], pgd); + } +} + +#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 +void __init ktext_replication_init_tramp(void) +{ + int nid; + + for_each_node(nid) { + /* Nothing to do for node 0 */ + if (pgtables[nid]->tramp_pg_dir == tramp_pg_dir) + continue; + + memcpy(pgtables[nid]->tramp_pg_dir, tramp_pg_dir, PGD_SIZE); } } +#endif diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 95d360805f8a..4ffa2650afd6 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -31,6 +31,7 @@ #include <asm/fixmap.h> #include <asm/kasan.h> #include <asm/kernel-pgtable.h> +#include <asm/ktext.h> #include <asm/sections.h> #include <asm/setup.h> #include <linux/sizes.h> @@ -81,6 +82,7 @@ void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd) pgd_t *fixmap_pgdp; spin_lock(&swapper_pgdir_lock); + ktext_replication_set_swapper_pgd(pgdp, pgd); fixmap_pgdp = pgd_set_fixmap(__pa_symbol(pgdp)); WRITE_ONCE(*fixmap_pgdp, pgd); /* @@ -694,6 +696,9 @@ static int __init map_entry_trampoline(void) __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i, pa_start + i * PAGE_SIZE, PAGE_KERNEL_RO); + /* Copy trampoline page tables to other numa nodes */ + ktext_replication_init_tramp(); + return 0; } core_initcall(map_entry_trampoline); |