diff options
Diffstat (limited to 'arch/mips/mm/init.c')
| -rw-r--r-- | arch/mips/mm/init.c | 273 |
1 files changed, 151 insertions, 122 deletions
diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c index b521d8e2d359..a673d3d68254 100644 --- a/arch/mips/mm/init.c +++ b/arch/mips/mm/init.c @@ -31,16 +31,16 @@ #include <linux/gfp.h> #include <linux/kcore.h> #include <linux/initrd.h> +#include <linux/execmem.h> #include <asm/bootinfo.h> #include <asm/cachectl.h> #include <asm/cpu.h> #include <asm/dma.h> -#include <asm/kmap_types.h> #include <asm/maar.h> #include <asm/mmu_context.h> +#include <asm/mmzone.h> #include <asm/sections.h> -#include <asm/pgtable.h> #include <asm/pgalloc.h> #include <asm/tlb.h> #include <asm/fixmap.h> @@ -59,24 +59,16 @@ EXPORT_SYMBOL(zero_page_mask); /* * Not static inline because used by IP27 special magic initialization code */ -void setup_zero_pages(void) +static void __init setup_zero_pages(void) { - unsigned int order, i; - struct page *page; + unsigned int order; if (cpu_has_vce) order = 3; else order = 0; - empty_zero_page = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); - if (!empty_zero_page) - panic("Oh boy, that early out of memory?"); - - page = virt_to_page((void *)empty_zero_page); - split_page(page, order); - for (i = 0; i < (1 << order); i++, page++) - mark_page_reserved(page); + empty_zero_page = (unsigned long)memblock_alloc_or_panic(PAGE_SIZE << order, PAGE_SIZE); zero_page_mask = ((PAGE_SIZE << order) - 1) & PAGE_MASK; } @@ -84,12 +76,13 @@ void setup_zero_pages(void) static void *__kmap_pgprot(struct page *page, unsigned long addr, pgprot_t prot) { enum fixed_addresses idx; + unsigned int old_mmid; unsigned long vaddr, flags, entrylo; unsigned long old_ctx; pte_t pte; int tlbidx; - BUG_ON(Page_dcache_dirty(page)); + BUG_ON(folio_test_dcache_dirty(page_folio(page))); preempt_disable(); pagefault_disable(); @@ -110,6 +103,10 @@ static void *__kmap_pgprot(struct page *page, unsigned long addr, pgprot_t prot) write_c0_entryhi(vaddr & (PAGE_MASK << 1)); write_c0_entrylo0(entrylo); write_c0_entrylo1(entrylo); + if (cpu_has_mmid) { + old_mmid = read_c0_memorymapid(); + write_c0_memorymapid(MMID_KERNEL_WIRED); + } #ifdef CONFIG_XPA if (cpu_has_xpa) { entrylo = (pte.pte_low & _PFNX_MASK); @@ -124,6 +121,8 @@ static void *__kmap_pgprot(struct page *page, unsigned long addr, pgprot_t prot) tlb_write_indexed(); tlbw_use_hazard(); write_c0_entryhi(old_ctx); + if (cpu_has_mmid) + write_c0_memorymapid(old_mmid); local_irq_restore(flags); return (void*) vaddr; @@ -164,11 +163,12 @@ void kunmap_coherent(void) void copy_user_highpage(struct page *to, struct page *from, unsigned long vaddr, struct vm_area_struct *vma) { + struct folio *src = page_folio(from); void *vfrom, *vto; vto = kmap_atomic(to); if (cpu_has_dc_aliases && - page_mapcount(from) && !Page_dcache_dirty(from)) { + folio_mapped(src) && !folio_test_dcache_dirty(src)) { vfrom = kmap_coherent(from, vaddr); copy_page(vto, vfrom); kunmap_coherent(); @@ -189,15 +189,17 @@ void copy_to_user_page(struct vm_area_struct *vma, struct page *page, unsigned long vaddr, void *dst, const void *src, unsigned long len) { + struct folio *folio = page_folio(page); + if (cpu_has_dc_aliases && - page_mapcount(page) && !Page_dcache_dirty(page)) { + folio_mapped(folio) && !folio_test_dcache_dirty(folio)) { void *vto = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK); memcpy(vto, src, len); kunmap_coherent(); } else { memcpy(dst, src, len); if (cpu_has_dc_aliases) - SetPageDcacheDirty(page); + folio_set_dcache_dirty(folio); } if (vma->vm_flags & VM_EXEC) flush_cache_page(vma, vaddr, page_to_pfn(page)); @@ -207,15 +209,17 @@ void copy_from_user_page(struct vm_area_struct *vma, struct page *page, unsigned long vaddr, void *dst, const void *src, unsigned long len) { + struct folio *folio = page_folio(page); + if (cpu_has_dc_aliases && - page_mapcount(page) && !Page_dcache_dirty(page)) { + folio_mapped(folio) && !folio_test_dcache_dirty(folio)) { void *vfrom = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK); memcpy(dst, vfrom, len); kunmap_coherent(); } else { memcpy(dst, src, len); if (cpu_has_dc_aliases) - SetPageDcacheDirty(page); + folio_set_dcache_dirty(folio); } } EXPORT_SYMBOL_GPL(copy_from_user_page); @@ -232,9 +236,9 @@ void __init fixrange_init(unsigned long start, unsigned long end, unsigned long vaddr; vaddr = start; - i = __pgd_offset(vaddr); - j = __pud_offset(vaddr); - k = __pmd_offset(vaddr); + i = pgd_index(vaddr); + j = pud_index(vaddr); + k = pmd_index(vaddr); pgd = pgd_base + i; for ( ; (i < PTRS_PER_PGD) && (vaddr < end); pgd++, i++) { @@ -245,6 +249,11 @@ void __init fixrange_init(unsigned long start, unsigned long end, if (pmd_none(*pmd)) { pte = (pte_t *) memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); + if (!pte) + panic("%s: Failed to allocate %lu bytes align=%lx\n", + __func__, PAGE_SIZE, + PAGE_SIZE); + set_pmd(pmd, __pmd((unsigned long)pte)); BUG_ON(pte != pte_offset_kernel(pmd, 0)); } @@ -257,37 +266,46 @@ void __init fixrange_init(unsigned long start, unsigned long end, #endif } -unsigned __weak platform_maar_init(unsigned num_pairs) +struct maar_walk_info { + struct maar_config cfg[16]; + unsigned int num_cfg; +}; + +static int maar_res_walk(unsigned long start_pfn, unsigned long nr_pages, + void *data) { - struct maar_config cfg[BOOT_MEM_MAP_MAX]; - unsigned i, num_configured, num_cfg = 0; - - for (i = 0; i < boot_mem_map.nr_map; i++) { - switch (boot_mem_map.map[i].type) { - case BOOT_MEM_RAM: - case BOOT_MEM_INIT_RAM: - break; - default: - continue; - } + struct maar_walk_info *wi = data; + struct maar_config *cfg = &wi->cfg[wi->num_cfg]; + unsigned int maar_align; - /* Round lower up */ - cfg[num_cfg].lower = boot_mem_map.map[i].addr; - cfg[num_cfg].lower = (cfg[num_cfg].lower + 0xffff) & ~0xffff; + /* MAAR registers hold physical addresses right shifted by 4 bits */ + maar_align = BIT(MIPS_MAAR_ADDR_SHIFT + 4); - /* Round upper down */ - cfg[num_cfg].upper = boot_mem_map.map[i].addr + - boot_mem_map.map[i].size; - cfg[num_cfg].upper = (cfg[num_cfg].upper & ~0xffff) - 1; + /* Fill in the MAAR config entry */ + cfg->lower = ALIGN(PFN_PHYS(start_pfn), maar_align); + cfg->upper = ALIGN_DOWN(PFN_PHYS(start_pfn + nr_pages), maar_align) - 1; + cfg->attrs = MIPS_MAAR_S; - cfg[num_cfg].attrs = MIPS_MAAR_S; - num_cfg++; - } + /* Ensure we don't overflow the cfg array */ + if (!WARN_ON(wi->num_cfg >= ARRAY_SIZE(wi->cfg))) + wi->num_cfg++; - num_configured = maar_config(cfg, num_cfg, num_pairs); - if (num_configured < num_cfg) - pr_warn("Not enough MAAR pairs (%u) for all bootmem regions (%u)\n", - num_pairs, num_cfg); + return 0; +} + + +unsigned __weak platform_maar_init(unsigned num_pairs) +{ + unsigned int num_configured; + struct maar_walk_info wi; + + wi.num_cfg = 0; + walk_system_ram_range(0, max_pfn, &wi, maar_res_walk); + + num_configured = maar_config(wi.cfg, wi.num_cfg, num_pairs); + if (num_configured < wi.num_cfg) + pr_warn("Not enough MAAR pairs (%u) for all memory regions (%u)\n", + num_pairs, wi.num_cfg); return num_configured; } @@ -337,17 +355,23 @@ void maar_init(void) write_c0_maari(i); back_to_back_c0_hazard(); upper = read_c0_maar(); +#ifdef CONFIG_XPA + upper |= (phys_addr_t)readx_c0_maar() << MIPS_MAARX_ADDR_SHIFT; +#endif write_c0_maari(i + 1); back_to_back_c0_hazard(); lower = read_c0_maar(); +#ifdef CONFIG_XPA + lower |= (phys_addr_t)readx_c0_maar() << MIPS_MAARX_ADDR_SHIFT; +#endif attr = lower & upper; lower = (lower & MIPS_MAAR_ADDR) << 4; upper = ((upper & MIPS_MAAR_ADDR) << 4) | 0xffff; pr_info(" [%d]: ", i / 2); - if (!(attr & MIPS_MAAR_VL)) { + if ((attr & MIPS_MAAR_V) != MIPS_MAAR_V) { pr_cont("disabled\n"); continue; } @@ -369,43 +393,13 @@ void maar_init(void) } } -#ifndef CONFIG_NEED_MULTIPLE_NODES -int page_is_ram(unsigned long pagenr) -{ - int i; - - for (i = 0; i < boot_mem_map.nr_map; i++) { - unsigned long addr, end; - - switch (boot_mem_map.map[i].type) { - case BOOT_MEM_RAM: - case BOOT_MEM_INIT_RAM: - break; - default: - /* not usable memory */ - continue; - } - - addr = PFN_UP(boot_mem_map.map[i].addr); - end = PFN_DOWN(boot_mem_map.map[i].addr + - boot_mem_map.map[i].size); - - if (pagenr >= addr && pagenr < end) - return 1; - } - - return 0; -} - +#ifndef CONFIG_NUMA void __init paging_init(void) { unsigned long max_zone_pfns[MAX_NR_ZONES]; pagetable_init(); -#ifdef CONFIG_HIGHMEM - kmap_init(); -#endif #ifdef CONFIG_ZONE_DMA max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN; #endif @@ -424,49 +418,23 @@ void __init paging_init(void) } #endif - free_area_init_nodes(max_zone_pfns); + free_area_init(max_zone_pfns); } #ifdef CONFIG_64BIT static struct kcore_list kcore_kseg0; #endif -static inline void mem_init_free_highmem(void) -{ -#ifdef CONFIG_HIGHMEM - unsigned long tmp; - - if (cpu_has_dc_aliases) - return; - - for (tmp = highstart_pfn; tmp < highend_pfn; tmp++) { - struct page *page = pfn_to_page(tmp); - - if (!page_is_ram(tmp)) - SetPageReserved(page); - else - free_highmem_page(page); - } -#endif -} - -void __init mem_init(void) +void __init arch_mm_preinit(void) { -#ifdef CONFIG_HIGHMEM -#ifdef CONFIG_DISCONTIGMEM -#error "CONFIG_HIGHMEM and CONFIG_DISCONTIGMEM dont work together yet" -#endif - max_mapnr = highend_pfn ? highend_pfn : max_low_pfn; -#else - max_mapnr = max_low_pfn; -#endif - high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT); + /* + * When PFN_PTE_SHIFT is greater than PAGE_SHIFT we won't have enough PTE + * bits to hold a full 32b physical address on MIPS32 systems. + */ + BUILD_BUG_ON(IS_ENABLED(CONFIG_32BIT) && (PFN_PTE_SHIFT > PAGE_SHIFT)); maar_init(); - memblock_free_all(); setup_zero_pages(); /* Setup zeroed pages. */ - mem_init_free_highmem(); - mem_init_print_info(NULL); #ifdef CONFIG_64BIT if ((unsigned long) &_text > (unsigned long) CKSEG0) @@ -476,7 +444,12 @@ void __init mem_init(void) 0x80000000 - 4, KCORE_TEXT); #endif } -#endif /* !CONFIG_NEED_MULTIPLE_NODES */ +#else /* CONFIG_NUMA */ +void __init arch_mm_preinit(void) +{ + setup_zero_pages(); /* This comes from node 0 */ +} +#endif /* !CONFIG_NUMA */ void free_init_pages(const char *what, unsigned long begin, unsigned long end) { @@ -492,15 +465,12 @@ void free_init_pages(const char *what, unsigned long begin, unsigned long end) printk(KERN_INFO "Freeing %s: %ldk freed\n", what, (end - begin) >> 10); } -#ifdef CONFIG_BLK_DEV_INITRD -void free_initrd_mem(unsigned long start, unsigned long end) +void (*free_init_pages_eva)(void *begin, void *end) = NULL; + +void __weak __init prom_free_prom_memory(void) { - free_reserved_area((void *)start, (void *)end, POISON_FREE_INITMEM, - "initrd"); + /* nothing to do */ } -#endif - -void (*free_init_pages_eva)(void *begin, void *end) = NULL; void __ref free_initmem(void) { @@ -516,6 +486,43 @@ void __ref free_initmem(void) free_initmem_default(POISON_FREE_INITMEM); } +#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA +unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; +EXPORT_SYMBOL(__per_cpu_offset); + +static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) +{ + return node_distance(cpu_to_node(from), cpu_to_node(to)); +} + +static int __init pcpu_cpu_to_node(int cpu) +{ + return cpu_to_node(cpu); +} + +void __init setup_per_cpu_areas(void) +{ + unsigned long delta; + unsigned int cpu; + int rc; + + /* + * Always reserve area for module percpu variables. That's + * what the legacy allocator did. + */ + rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, + PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, + pcpu_cpu_distance, + pcpu_cpu_to_node); + if (rc < 0) + panic("Failed to initialize percpu areas."); + + delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start; + for_each_possible_cpu(cpu) + __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu]; +} +#endif + #ifndef CONFIG_MIPS_PGD_C0_CONTEXT unsigned long pgd_current[NR_CPUS]; #endif @@ -527,7 +534,7 @@ unsigned long pgd_current[NR_CPUS]; * size, and waste space. So we place it in its own section and align * it in the linker script. */ -pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(.bss..swapper_pg_dir); +pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(".bss..swapper_pg_dir"); #ifndef __PAGETABLE_PUD_FOLDED pud_t invalid_pud_table[PTRS_PER_PUD] __page_aligned_bss; #endif @@ -537,3 +544,25 @@ EXPORT_SYMBOL_GPL(invalid_pmd_table); #endif pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss; EXPORT_SYMBOL(invalid_pte_table); + +#ifdef CONFIG_EXECMEM +#ifdef MODULES_VADDR +static struct execmem_info execmem_info __ro_after_init; + +struct execmem_info __init *execmem_arch_setup(void) +{ + execmem_info = (struct execmem_info){ + .ranges = { + [EXECMEM_DEFAULT] = { + .start = MODULES_VADDR, + .end = MODULES_END, + .pgprot = PAGE_KERNEL, + .alignment = 1, + }, + }, + }; + + return &execmem_info; +} +#endif +#endif /* CONFIG_EXECMEM */ |