diff options
Diffstat (limited to 'arch/arm/mm/nommu.c')
| -rw-r--r-- | arch/arm/mm/nommu.c | 401 |
1 files changed, 144 insertions, 257 deletions
diff --git a/arch/arm/mm/nommu.c b/arch/arm/mm/nommu.c index 1fa50100ab6a..d638cc87807e 100644 --- a/arch/arm/mm/nommu.c +++ b/arch/arm/mm/nommu.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * linux/arch/arm/mm/nommu.c * @@ -11,6 +12,7 @@ #include <linux/kernel.h> #include <asm/cacheflush.h> +#include <asm/cp15.h> #include <asm/sections.h> #include <asm/page.h> #include <asm/setup.h> @@ -18,292 +20,154 @@ #include <asm/mach/arch.h> #include <asm/cputype.h> #include <asm/mpu.h> +#include <asm/procinfo.h> +#include <asm/idmap.h> #include "mm.h" -#ifdef CONFIG_ARM_MPU -struct mpu_rgn_info mpu_rgn_info; - -/* Region number */ -static void rgnr_write(u32 v) -{ - asm("mcr p15, 0, %0, c6, c2, 0" : : "r" (v)); -} - -/* Data-side / unified region attributes */ - -/* Region access control register */ -static void dracr_write(u32 v) -{ - asm("mcr p15, 0, %0, c6, c1, 4" : : "r" (v)); -} - -/* Region size register */ -static void drsr_write(u32 v) -{ - asm("mcr p15, 0, %0, c6, c1, 2" : : "r" (v)); -} - -/* Region base address register */ -static void drbar_write(u32 v) -{ - asm("mcr p15, 0, %0, c6, c1, 0" : : "r" (v)); -} +unsigned long vectors_base; -static u32 drbar_read(void) -{ - u32 v; - asm("mrc p15, 0, %0, c6, c1, 0" : "=r" (v)); - return v; -} -/* Optional instruction-side region attributes */ - -/* I-side Region access control register */ -static void iracr_write(u32 v) -{ - asm("mcr p15, 0, %0, c6, c1, 5" : : "r" (v)); -} - -/* I-side Region size register */ -static void irsr_write(u32 v) -{ - asm("mcr p15, 0, %0, c6, c1, 3" : : "r" (v)); -} - -/* I-side Region base address register */ -static void irbar_write(u32 v) -{ - asm("mcr p15, 0, %0, c6, c1, 1" : : "r" (v)); -} +/* + * empty_zero_page is a special page that is used for + * zero-initialized data and COW. + */ +struct page *empty_zero_page; +EXPORT_SYMBOL(empty_zero_page); -static unsigned long irbar_read(void) -{ - unsigned long v; - asm("mrc p15, 0, %0, c6, c1, 1" : "=r" (v)); - return v; -} +#ifdef CONFIG_ARM_MPU +struct mpu_rgn_info mpu_rgn_info; +#endif -/* MPU initialisation functions */ -void __init sanity_check_meminfo_mpu(void) +#ifdef CONFIG_CPU_CP15 +#ifdef CONFIG_CPU_HIGH_VECTOR +unsigned long setup_vectors_base(void) { - int i; - struct membank *bank = meminfo.bank; - phys_addr_t phys_offset = PHYS_OFFSET; - phys_addr_t aligned_region_size, specified_mem_size, rounded_mem_size; - - /* Initially only use memory continuous from PHYS_OFFSET */ - if (bank_phys_start(&bank[0]) != phys_offset) - panic("First memory bank must be contiguous from PHYS_OFFSET"); - - /* Banks have already been sorted by start address */ - for (i = 1; i < meminfo.nr_banks; i++) { - if (bank[i].start <= bank_phys_end(&bank[0]) && - bank_phys_end(&bank[i]) > bank_phys_end(&bank[0])) { - bank[0].size = bank_phys_end(&bank[i]) - bank[0].start; - } else { - pr_notice("Ignoring RAM after 0x%.8lx. " - "First non-contiguous (ignored) bank start: 0x%.8lx\n", - (unsigned long)bank_phys_end(&bank[0]), - (unsigned long)bank_phys_start(&bank[i])); - break; - } - } - /* All contiguous banks are now merged in to the first bank */ - meminfo.nr_banks = 1; - specified_mem_size = bank[0].size; - - /* - * MPU has curious alignment requirements: Size must be power of 2, and - * region start must be aligned to the region size - */ - if (phys_offset != 0) - pr_info("PHYS_OFFSET != 0 => MPU Region size constrained by alignment requirements\n"); + unsigned long reg = get_cr(); - /* - * Maximum aligned region might overflow phys_addr_t if phys_offset is - * 0. Hence we keep everything below 4G until we take the smaller of - * the aligned_region_size and rounded_mem_size, one of which is - * guaranteed to be smaller than the maximum physical address. - */ - aligned_region_size = (phys_offset - 1) ^ (phys_offset); - /* Find the max power-of-two sized region that fits inside our bank */ - rounded_mem_size = (1 << __fls(bank[0].size)) - 1; - - /* The actual region size is the smaller of the two */ - aligned_region_size = aligned_region_size < rounded_mem_size - ? aligned_region_size + 1 - : rounded_mem_size + 1; - - if (aligned_region_size != specified_mem_size) - pr_warn("Truncating memory from 0x%.8lx to 0x%.8lx (MPU region constraints)", - (unsigned long)specified_mem_size, - (unsigned long)aligned_region_size); - - meminfo.bank[0].size = aligned_region_size; - pr_debug("MPU Region from 0x%.8lx size 0x%.8lx (end 0x%.8lx))\n", - (unsigned long)phys_offset, - (unsigned long)aligned_region_size, - (unsigned long)bank_phys_end(&bank[0])); - -} - -static int mpu_present(void) -{ - return ((read_cpuid_ext(CPUID_EXT_MMFR0) & MMFR0_PMSA) == MMFR0_PMSAv7); + set_cr(reg | CR_V); + return 0xffff0000; } - -static int mpu_max_regions(void) +#else /* CONFIG_CPU_HIGH_VECTOR */ +/* Write exception base address to VBAR */ +static inline void set_vbar(unsigned long val) { - /* - * We don't support a different number of I/D side regions so if we - * have separate instruction and data memory maps then return - * whichever side has a smaller number of supported regions. - */ - u32 dregions, iregions, mpuir; - mpuir = read_cpuid(CPUID_MPUIR); - - dregions = iregions = (mpuir & MPUIR_DREGION_SZMASK) >> MPUIR_DREGION; - - /* Check for separate d-side and i-side memory maps */ - if (mpuir & MPUIR_nU) - iregions = (mpuir & MPUIR_IREGION_SZMASK) >> MPUIR_IREGION; - - /* Use the smallest of the two maxima */ - return min(dregions, iregions); + asm("mcr p15, 0, %0, c12, c0, 0" : : "r" (val) : "cc"); } -static int mpu_iside_independent(void) -{ - /* MPUIR.nU specifies whether there is *not* a unified memory map */ - return read_cpuid(CPUID_MPUIR) & MPUIR_nU; -} - -static int mpu_min_region_order(void) -{ - u32 drbar_result, irbar_result; - /* We've kept a region free for this probing */ - rgnr_write(MPU_PROBE_REGION); - isb(); - /* - * As per ARM ARM, write 0xFFFFFFFC to DRBAR to find the minimum - * region order - */ - drbar_write(0xFFFFFFFC); - drbar_result = irbar_result = drbar_read(); - drbar_write(0x0); - /* If the MPU is non-unified, we use the larger of the two minima*/ - if (mpu_iside_independent()) { - irbar_write(0xFFFFFFFC); - irbar_result = irbar_read(); - irbar_write(0x0); - } - isb(); /* Ensure that MPU region operations have completed */ - /* Return whichever result is larger */ - return __ffs(max(drbar_result, irbar_result)); -} - -static int mpu_setup_region(unsigned int number, phys_addr_t start, - unsigned int size_order, unsigned int properties) +/* + * Security extensions, bits[7:4], permitted values, + * 0b0000 - not implemented, 0b0001/0b0010 - implemented + */ +static inline bool security_extensions_enabled(void) { - u32 size_data; - - /* We kept a region free for probing resolution of MPU regions*/ - if (number > mpu_max_regions() || number == MPU_PROBE_REGION) - return -ENOENT; - - if (size_order > 32) - return -ENOMEM; - - if (size_order < mpu_min_region_order()) - return -ENOMEM; - - /* Writing N to bits 5:1 (RSR_SZ) specifies region size 2^N+1 */ - size_data = ((size_order - 1) << MPU_RSR_SZ) | 1 << MPU_RSR_EN; - - dsb(); /* Ensure all previous data accesses occur with old mappings */ - rgnr_write(number); - isb(); - drbar_write(start); - dracr_write(properties); - isb(); /* Propagate properties before enabling region */ - drsr_write(size_data); - - /* Check for independent I-side registers */ - if (mpu_iside_independent()) { - irbar_write(start); - iracr_write(properties); - isb(); - irsr_write(size_data); - } - isb(); - - /* Store region info (we treat i/d side the same, so only store d) */ - mpu_rgn_info.rgns[number].dracr = properties; - mpu_rgn_info.rgns[number].drbar = start; - mpu_rgn_info.rgns[number].drsr = size_data; + /* Check CPUID Identification Scheme before ID_PFR1 read */ + if ((read_cpuid_id() & 0x000f0000) == 0x000f0000) + return cpuid_feature_extract(CPUID_EXT_PFR1, 4) || + cpuid_feature_extract(CPUID_EXT_PFR1, 20); return 0; } -/* -* Set up default MPU regions, doing nothing if there is no MPU -*/ -void __init mpu_setup(void) +unsigned long setup_vectors_base(void) { - int region_err; - if (!mpu_present()) - return; - - region_err = mpu_setup_region(MPU_RAM_REGION, PHYS_OFFSET, - ilog2(meminfo.bank[0].size), - MPU_AP_PL1RW_PL0RW | MPU_RGN_NORMAL); - if (region_err) { - panic("MPU region initialization failure! %d", region_err); - } else { - pr_info("Using ARMv7 PMSA Compliant MPU. " - "Region independence: %s, Max regions: %d\n", - mpu_iside_independent() ? "Yes" : "No", - mpu_max_regions()); + unsigned long base = 0, reg = get_cr(); + + set_cr(reg & ~CR_V); + if (security_extensions_enabled()) { + if (IS_ENABLED(CONFIG_REMAP_VECTORS_TO_RAM)) + base = CONFIG_DRAM_BASE; + set_vbar(base); + } else if (IS_ENABLED(CONFIG_REMAP_VECTORS_TO_RAM)) { + if (CONFIG_DRAM_BASE != 0) + pr_err("Security extensions not enabled, vectors cannot be remapped to RAM, vectors base will be 0x00000000\n"); } + + return base; } -#else -static void sanity_check_meminfo_mpu(void) {} -static void __init mpu_setup(void) {} -#endif /* CONFIG_ARM_MPU */ +#endif /* CONFIG_CPU_HIGH_VECTOR */ +#endif /* CONFIG_CPU_CP15 */ void __init arm_mm_memblock_reserve(void) { #ifndef CONFIG_CPU_V7M + vectors_base = IS_ENABLED(CONFIG_CPU_CP15) ? setup_vectors_base() : 0; /* * Register the exception vector page. * some architectures which the DRAM is the exception vector to trap, * alloc_page breaks with error, although it is not NULL, but "0." */ - memblock_reserve(CONFIG_VECTORS_BASE, PAGE_SIZE); + memblock_reserve(vectors_base, 2 * PAGE_SIZE); #else /* ifndef CONFIG_CPU_V7M */ /* * There is no dedicated vector page on V7-M. So nothing needs to be * reserved here. */ #endif + /* + * In any case, always ensure address 0 is never used as many things + * get very confused if 0 is returned as a legitimate address. + */ + memblock_reserve(0, 1); +} + +static void __init adjust_lowmem_bounds_mpu(void) +{ + unsigned long pmsa = read_cpuid_ext(CPUID_EXT_MMFR0) & MMFR0_PMSA; + + switch (pmsa) { + case MMFR0_PMSAv7: + pmsav7_adjust_lowmem_bounds(); + break; + case MMFR0_PMSAv8: + pmsav8_adjust_lowmem_bounds(); + break; + default: + break; + } +} + +static void __init mpu_setup(void) +{ + unsigned long pmsa = read_cpuid_ext(CPUID_EXT_MMFR0) & MMFR0_PMSA; + + switch (pmsa) { + case MMFR0_PMSAv7: + pmsav7_setup(); + break; + case MMFR0_PMSAv8: + pmsav8_setup(); + break; + default: + break; + } } -void __init sanity_check_meminfo(void) +void __init adjust_lowmem_bounds(void) { phys_addr_t end; - sanity_check_meminfo_mpu(); - end = bank_phys_end(&meminfo.bank[meminfo.nr_banks - 1]); + adjust_lowmem_bounds_mpu(); + end = memblock_end_of_DRAM(); high_memory = __va(end - 1) + 1; + memblock_set_current_limit(end); } /* * paging_init() sets up the page tables, initialises the zone memory * maps, and sets up the zero page, bad page and bad page tables. */ -void __init paging_init(struct machine_desc *mdesc) +void __init paging_init(const struct machine_desc *mdesc) { - early_trap_init((void *)CONFIG_VECTORS_BASE); + void *zero_page; + + early_trap_init((void *)vectors_base); mpu_setup(); + + /* allocate the zero page. */ + zero_page = (void *)memblock_alloc_or_panic(PAGE_SIZE, PAGE_SIZE); + bootmem_init(); + + empty_zero_page = virt_to_page(zero_page); + flush_dcache_page(empty_zero_page); } /* @@ -313,17 +177,17 @@ void setup_mm_for_reboot(void) { } -void flush_dcache_page(struct page *page) +void flush_dcache_folio(struct folio *folio) { - __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE); + __cpuc_flush_dcache_area(folio_address(folio), folio_size(folio)); } -EXPORT_SYMBOL(flush_dcache_page); +EXPORT_SYMBOL(flush_dcache_folio); -void flush_kernel_dcache_page(struct page *page) +void flush_dcache_page(struct page *page) { __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE); } -EXPORT_SYMBOL(flush_kernel_dcache_page); +EXPORT_SYMBOL(flush_dcache_page); void copy_to_user_page(struct vm_area_struct *vma, struct page *page, unsigned long uaddr, void *dst, const void *src, @@ -343,30 +207,53 @@ void __iomem *__arm_ioremap_pfn(unsigned long pfn, unsigned long offset, } EXPORT_SYMBOL(__arm_ioremap_pfn); -void __iomem *__arm_ioremap_pfn_caller(unsigned long pfn, unsigned long offset, - size_t size, unsigned int mtype, void *caller) +void __iomem *__arm_ioremap_caller(phys_addr_t phys_addr, size_t size, + unsigned int mtype, void *caller) { - return __arm_ioremap_pfn(pfn, offset, size, mtype); + return (void __iomem *)phys_addr; } -void __iomem *__arm_ioremap(phys_addr_t phys_addr, size_t size, - unsigned int mtype) +void __iomem * (*arch_ioremap_caller)(phys_addr_t, size_t, unsigned int, void *); + +void __iomem *ioremap(resource_size_t res_cookie, size_t size) { - return (void __iomem *)phys_addr; + return __arm_ioremap_caller(res_cookie, size, MT_DEVICE, + __builtin_return_address(0)); } -EXPORT_SYMBOL(__arm_ioremap); +EXPORT_SYMBOL(ioremap); -void __iomem * (*arch_ioremap_caller)(phys_addr_t, size_t, unsigned int, void *); +void __iomem *ioremap_cache(resource_size_t res_cookie, size_t size) +{ + return __arm_ioremap_caller(res_cookie, size, MT_DEVICE_CACHED, + __builtin_return_address(0)); +} +EXPORT_SYMBOL(ioremap_cache); -void __iomem *__arm_ioremap_caller(phys_addr_t phys_addr, size_t size, - unsigned int mtype, void *caller) +void __iomem *ioremap_wc(resource_size_t res_cookie, size_t size) +{ + return __arm_ioremap_caller(res_cookie, size, MT_DEVICE_WC, + __builtin_return_address(0)); +} +EXPORT_SYMBOL(ioremap_wc); + +#ifdef CONFIG_PCI + +#include <asm/mach/map.h> + +void __iomem *pci_remap_cfgspace(resource_size_t res_cookie, size_t size) { - return __arm_ioremap(phys_addr, size, mtype); + return arch_ioremap_caller(res_cookie, size, MT_UNCACHED, + __builtin_return_address(0)); } +EXPORT_SYMBOL_GPL(pci_remap_cfgspace); +#endif -void (*arch_iounmap)(volatile void __iomem *); +void *arch_memremap_wb(phys_addr_t phys_addr, size_t size, unsigned long flags) +{ + return (void *)phys_addr; +} -void __arm_iounmap(volatile void __iomem *addr) +void iounmap(volatile void __iomem *io_addr) { } -EXPORT_SYMBOL(__arm_iounmap); +EXPORT_SYMBOL(iounmap); |