summaryrefslogtreecommitdiff
path: root/arch/powerpc/mm/book3s64/radix_pgtable.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/powerpc/mm/book3s64/radix_pgtable.c')
-rw-r--r--arch/powerpc/mm/book3s64/radix_pgtable.c102
1 files changed, 76 insertions, 26 deletions
diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c
index c6a4ac766b2b..311e2112d782 100644
--- a/arch/powerpc/mm/book3s64/radix_pgtable.c
+++ b/arch/powerpc/mm/book3s64/radix_pgtable.c
@@ -17,6 +17,7 @@
#include <linux/hugetlb.h>
#include <linux/string_helpers.h>
#include <linux/memory.h>
+#include <linux/kfence.h>
#include <asm/pgalloc.h>
#include <asm/mmu_context.h>
@@ -31,6 +32,7 @@
#include <asm/uaccess.h>
#include <asm/ultravisor.h>
#include <asm/set_memory.h>
+#include <asm/kfence.h>
#include <trace/events/thp.h>
@@ -204,14 +206,14 @@ static void radix__change_memory_range(unsigned long start, unsigned long end,
pudp = pud_alloc(&init_mm, p4dp, idx);
if (!pudp)
continue;
- if (pud_is_leaf(*pudp)) {
+ if (pud_leaf(*pudp)) {
ptep = (pte_t *)pudp;
goto update_the_pte;
}
pmdp = pmd_alloc(&init_mm, pudp, idx);
if (!pmdp)
continue;
- if (pmd_is_leaf(*pmdp)) {
+ if (pmd_leaf(*pmdp)) {
ptep = pmdp_ptep(pmdp);
goto update_the_pte;
}
@@ -293,7 +295,8 @@ static unsigned long next_boundary(unsigned long addr, unsigned long end)
static int __meminit create_physical_mapping(unsigned long start,
unsigned long end,
- int nid, pgprot_t _prot)
+ int nid, pgprot_t _prot,
+ unsigned long mapping_sz_limit)
{
unsigned long vaddr, addr, mapping_size = 0;
bool prev_exec, exec = false;
@@ -301,7 +304,10 @@ static int __meminit create_physical_mapping(unsigned long start,
int psize;
unsigned long max_mapping_size = memory_block_size;
- if (debug_pagealloc_enabled_or_kfence())
+ if (mapping_sz_limit < max_mapping_size)
+ max_mapping_size = mapping_sz_limit;
+
+ if (debug_pagealloc_enabled())
max_mapping_size = PAGE_SIZE;
start = ALIGN(start, PAGE_SIZE);
@@ -356,8 +362,62 @@ static int __meminit create_physical_mapping(unsigned long start,
return 0;
}
+#ifdef CONFIG_KFENCE
+static inline phys_addr_t alloc_kfence_pool(void)
+{
+ phys_addr_t kfence_pool;
+
+ /*
+ * TODO: Support to enable KFENCE after bootup depends on the ability to
+ * split page table mappings. As such support is not currently
+ * implemented for radix pagetables, support enabling KFENCE
+ * only at system startup for now.
+ *
+ * After support for splitting mappings is available on radix,
+ * alloc_kfence_pool() & map_kfence_pool() can be dropped and
+ * mapping for __kfence_pool memory can be
+ * split during arch_kfence_init_pool().
+ */
+ if (!kfence_early_init)
+ goto no_kfence;
+
+ kfence_pool = memblock_phys_alloc(KFENCE_POOL_SIZE, PAGE_SIZE);
+ if (!kfence_pool)
+ goto no_kfence;
+
+ memblock_mark_nomap(kfence_pool, KFENCE_POOL_SIZE);
+ return kfence_pool;
+
+no_kfence:
+ disable_kfence();
+ return 0;
+}
+
+static inline void map_kfence_pool(phys_addr_t kfence_pool)
+{
+ if (!kfence_pool)
+ return;
+
+ if (create_physical_mapping(kfence_pool, kfence_pool + KFENCE_POOL_SIZE,
+ -1, PAGE_KERNEL, PAGE_SIZE))
+ goto err;
+
+ memblock_clear_nomap(kfence_pool, KFENCE_POOL_SIZE);
+ __kfence_pool = __va(kfence_pool);
+ return;
+
+err:
+ memblock_phys_free(kfence_pool, KFENCE_POOL_SIZE);
+ disable_kfence();
+}
+#else
+static inline phys_addr_t alloc_kfence_pool(void) { return 0; }
+static inline void map_kfence_pool(phys_addr_t kfence_pool) { }
+#endif
+
static void __init radix_init_pgtable(void)
{
+ phys_addr_t kfence_pool;
unsigned long rts_field;
phys_addr_t start, end;
u64 i;
@@ -365,6 +425,8 @@ static void __init radix_init_pgtable(void)
/* We don't support slb for radix */
slb_set_size(0);
+ kfence_pool = alloc_kfence_pool();
+
/*
* Create the linear mapping
*/
@@ -381,9 +443,11 @@ static void __init radix_init_pgtable(void)
}
WARN_ON(create_physical_mapping(start, end,
- -1, PAGE_KERNEL));
+ -1, PAGE_KERNEL, ~0UL));
}
+ map_kfence_pool(kfence_pool);
+
if (!cpu_has_feature(CPU_FTR_HVMODE) &&
cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) {
/*
@@ -767,7 +831,7 @@ static void __meminit remove_pmd_table(pmd_t *pmd_start, unsigned long addr,
if (!pmd_present(*pmd))
continue;
- if (pmd_is_leaf(*pmd)) {
+ if (pmd_leaf(*pmd)) {
if (IS_ALIGNED(addr, PMD_SIZE) &&
IS_ALIGNED(next, PMD_SIZE)) {
if (!direct)
@@ -807,7 +871,7 @@ static void __meminit remove_pud_table(pud_t *pud_start, unsigned long addr,
if (!pud_present(*pud))
continue;
- if (pud_is_leaf(*pud)) {
+ if (pud_leaf(*pud)) {
if (!IS_ALIGNED(addr, PUD_SIZE) ||
!IS_ALIGNED(next, PUD_SIZE)) {
WARN_ONCE(1, "%s: unaligned range\n", __func__);
@@ -845,7 +909,7 @@ remove_pagetable(unsigned long start, unsigned long end, bool direct,
if (!p4d_present(*p4d))
continue;
- if (p4d_is_leaf(*p4d)) {
+ if (p4d_leaf(*p4d)) {
if (!IS_ALIGNED(addr, P4D_SIZE) ||
!IS_ALIGNED(next, P4D_SIZE)) {
WARN_ONCE(1, "%s: unaligned range\n", __func__);
@@ -875,7 +939,7 @@ int __meminit radix__create_section_mapping(unsigned long start,
}
return create_physical_mapping(__pa(start), __pa(end),
- nid, prot);
+ nid, prot, ~0UL);
}
int __meminit radix__remove_section_mapping(unsigned long start, unsigned long end)
@@ -924,7 +988,7 @@ bool vmemmap_can_optimize(struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
int __meminit vmemmap_check_pmd(pmd_t *pmdp, int node,
unsigned long addr, unsigned long next)
{
- int large = pmd_large(*pmdp);
+ int large = pmd_leaf(*pmdp);
if (large)
vmemmap_verify(pmdp_ptep(pmdp), node, addr, next);
@@ -1339,20 +1403,6 @@ void __ref radix__vmemmap_free(unsigned long start, unsigned long end,
#endif
#endif
-#if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KFENCE)
-void radix__kernel_map_pages(struct page *page, int numpages, int enable)
-{
- unsigned long addr;
-
- addr = (unsigned long)page_address(page);
-
- if (enable)
- set_memory_p(addr, numpages);
- else
- set_memory_np(addr, numpages);
-}
-#endif
-
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
@@ -1554,7 +1604,7 @@ int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
int pud_clear_huge(pud_t *pud)
{
- if (pud_is_leaf(*pud)) {
+ if (pud_leaf(*pud)) {
pud_clear(pud);
return 1;
}
@@ -1601,7 +1651,7 @@ int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)
int pmd_clear_huge(pmd_t *pmd)
{
- if (pmd_is_leaf(*pmd)) {
+ if (pmd_leaf(*pmd)) {
pmd_clear(pmd);
return 1;
}
generated by cgit (git 2.34.1) at 2025-07-23 10:09:23 +0000