From 172f3fcb17382faafc71091868370b6765da7a43 Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 26 Nov 2015 22:12:02 +0000 Subject: ARM: l2c: tauros2: fix OF-enabled non-DT boot Signed-off-by: Russell King --- arch/arm/mm/cache-tauros2.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/cache-tauros2.c b/arch/arm/mm/cache-tauros2.c index 1e373d268c04..95eb524ce556 100644 --- a/arch/arm/mm/cache-tauros2.c +++ b/arch/arm/mm/cache-tauros2.c @@ -287,16 +287,15 @@ void __init tauros2_init(unsigned int features) node = of_find_matching_node(NULL, tauros2_ids); if (!node) { pr_info("Not found marvell,tauros2-cache, disable it\n"); - return; + } else { + ret = of_property_read_u32(node, "marvell,tauros2-cache-features", &f); + if (ret) { + pr_info("Not found marvell,tauros-cache-features property, " + "disable extra features\n"); + features = 0; + } else + features = f; } - - ret = of_property_read_u32(node, "marvell,tauros2-cache-features", &f); - if (ret) { - pr_info("Not found marvell,tauros-cache-features property, " - "disable extra features\n"); - features = 0; - } else - features = f; #endif tauros2_internal_init(features); } -- cgit From 1d93ba2aaacc96bef018c5c2e12840f07372a2be Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 26 Nov 2015 22:12:26 +0000 Subject: ARM: l2c: tauros2: use descriptive definitions for register bits Use descriptive definitions for the Tauros2 register bits, and while we're here, clean up the "Tauros2: %s line fill burt8." message. Signed-off-by: Russell King --- arch/arm/mm/cache-tauros2.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/cache-tauros2.c b/arch/arm/mm/cache-tauros2.c index 95eb524ce556..88255bea65e4 100644 --- a/arch/arm/mm/cache-tauros2.c +++ b/arch/arm/mm/cache-tauros2.c @@ -22,6 +22,11 @@ #include #include +/* CP15 PJ4 Control configuration register */ +#define CCR_L2C_PREFETCH_DISABLE BIT(24) +#define CCR_L2C_ECC_ENABLE BIT(23) +#define CCR_L2C_WAY7_4_DISABLE BIT(21) +#define CCR_L2C_BURST8_ENABLE BIT(20) /* * When Tauros2 is used on a CPU that supports the v7 hierarchical @@ -182,18 +187,18 @@ static void enable_extra_feature(unsigned int features) u = read_extra_features(); if (features & CACHE_TAUROS2_PREFETCH_ON) - u &= ~0x01000000; + u &= ~CCR_L2C_PREFETCH_DISABLE; else - u |= 0x01000000; + u |= CCR_L2C_PREFETCH_DISABLE; pr_info("Tauros2: %s L2 prefetch.\n", (features & CACHE_TAUROS2_PREFETCH_ON) ? "Enabling" : "Disabling"); if (features & CACHE_TAUROS2_LINEFILL_BURST8) - u |= 0x00100000; + u |= CCR_L2C_BURST8_ENABLE; else - u &= ~0x00100000; - pr_info("Tauros2: %s line fill burt8.\n", + u &= ~CCR_L2C_BURST8_ENABLE; + pr_info("Tauros2: %s burst8 line fill.\n", (features & CACHE_TAUROS2_LINEFILL_BURST8) ? "Enabling" : "Disabling"); -- cgit From 2841029393fad551b49b6de34d44bfa9ef256441 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 11 Jan 2016 17:15:58 +0000 Subject: ARM: make virt_to_idmap() return unsigned long Make virt_to_idmap() return an unsigned long rather than phys_addr_t. Returning phys_addr_t here makes no sense, because the definition of virt_to_idmap() is that it shall return a physical address which maps identically with the virtual address. Since virtual addresses are limited to 32-bit, identity mapped physical addresses are as well. Almost all users already had an implicit narrowing cast to unsigned long so let's make this official and part of this interface. Tested-by: Grygorii Strashko Signed-off-by: Russell King --- arch/arm/mm/idmap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/idmap.c b/arch/arm/mm/idmap.c index d65909697165..bd274a05b8ff 100644 --- a/arch/arm/mm/idmap.c +++ b/arch/arm/mm/idmap.c @@ -15,7 +15,7 @@ * page tables. */ pgd_t *idmap_pgd; -phys_addr_t (*arch_virt_to_idmap) (unsigned long x); +unsigned long (*arch_virt_to_idmap)(unsigned long x); #ifdef CONFIG_ARM_LPAE static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end, -- cgit From 25362dc496edaf17f714c0fecd8b3eb79670207b Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Tue, 26 Jan 2016 01:19:36 +0100 Subject: ARM: 8501/1: mm: flip priority of CONFIG_DEBUG_RODATA The use of CONFIG_DEBUG_RODATA is generally seen as an essential part of kernel self-protection: http://www.openwall.com/lists/kernel-hardening/2015/11/30/13 Additionally, its name has grown to mean things beyond just rodata. To get ARM closer to this, we ought to rearrange the names of the configs that control how the kernel protects its memory. What was called CONFIG_ARM_KERNMEM_PERMS is realy doing the work that other architectures call CONFIG_DEBUG_RODATA. This redefines CONFIG_DEBUG_RODATA to actually do the bulk of the ROing (and NXing). In the place of the old CONFIG_DEBUG_RODATA, use CONFIG_DEBUG_ALIGN_RODATA, since that's what the option does: adds section alignment for making rodata explicitly NX, as arm does not split the page tables like arm64 does without _ALIGN_RODATA. Also adds human readable names to the sections so I could more easily debug my typos, and makes CONFIG_DEBUG_RODATA default "y" for CPU_V7. Results in /sys/kernel/debug/kernel_page_tables for each config state: # CONFIG_DEBUG_RODATA is not set # CONFIG_DEBUG_ALIGN_RODATA is not set ---[ Kernel Mapping ]--- 0x80000000-0x80900000 9M RW x SHD 0x80900000-0xa0000000 503M RW NX SHD CONFIG_DEBUG_RODATA=y CONFIG_DEBUG_ALIGN_RODATA=y ---[ Kernel Mapping ]--- 0x80000000-0x80100000 1M RW NX SHD 0x80100000-0x80700000 6M ro x SHD 0x80700000-0x80a00000 3M ro NX SHD 0x80a00000-0xa0000000 502M RW NX SHD CONFIG_DEBUG_RODATA=y # CONFIG_DEBUG_ALIGN_RODATA is not set ---[ Kernel Mapping ]--- 0x80000000-0x80100000 1M RW NX SHD 0x80100000-0x80a00000 9M ro x SHD 0x80a00000-0xa0000000 502M RW NX SHD Signed-off-by: Kees Cook Reviewed-by: Laura Abbott Signed-off-by: Russell King --- arch/arm/mm/Kconfig | 34 ++++++++++++++++++---------------- arch/arm/mm/init.c | 19 ++++++++++--------- 2 files changed, 28 insertions(+), 25 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 549f6d3aec5b..4daeda0a5b7f 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -1037,24 +1037,26 @@ config ARCH_SUPPORTS_BIG_ENDIAN This option specifies the architecture can support big endian operation. -config ARM_KERNMEM_PERMS - bool "Restrict kernel memory permissions" +config DEBUG_RODATA + bool "Make kernel text and rodata read-only" depends on MMU + default y if CPU_V7 help - If this is set, kernel memory other than kernel text (and rodata) - will be made non-executable. The tradeoff is that each region is - padded to section-size (1MiB) boundaries (because their permissions - are different and splitting the 1M pages into 4K ones causes TLB - performance problems), wasting memory. + If this is set, kernel text and rodata memory will be made + read-only, and non-text kernel memory will be made non-executable. + The tradeoff is that each region is padded to section-size (1MiB) + boundaries (because their permissions are different and splitting + the 1M pages into 4K ones causes TLB performance problems), which + can waste memory. -config DEBUG_RODATA - bool "Make kernel text and rodata read-only" - depends on ARM_KERNMEM_PERMS +config DEBUG_ALIGN_RODATA + bool "Make rodata strictly non-executable" + depends on DEBUG_RODATA default y help - If this is set, kernel text and rodata will be made read-only. This - is to help catch accidental or malicious attempts to change the - kernel's executable code. Additionally splits rodata from kernel - text so it can be made explicitly non-executable. This creates - another section-size padded region, so it can waste more memory - space while gaining the read-only protections. + If this is set, rodata will be made explicitly non-executable. This + provides protection on the rare chance that attackers might find and + use ROP gadgets that exist in the rodata section. This adds an + additional section-aligned split of rodata from kernel text so it + can be made explicitly non-executable. This padding may waste memory + space to gain the additional protection. diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 49bd08178008..53f42508025b 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -572,8 +572,9 @@ void __init mem_init(void) } } -#ifdef CONFIG_ARM_KERNMEM_PERMS +#ifdef CONFIG_DEBUG_RODATA struct section_perm { + const char *name; unsigned long start; unsigned long end; pmdval_t mask; @@ -584,6 +585,7 @@ struct section_perm { static struct section_perm nx_perms[] = { /* Make pages tables, etc before _stext RW (set NX). */ { + .name = "pre-text NX", .start = PAGE_OFFSET, .end = (unsigned long)_stext, .mask = ~PMD_SECT_XN, @@ -591,14 +593,16 @@ static struct section_perm nx_perms[] = { }, /* Make init RW (set NX). */ { + .name = "init NX", .start = (unsigned long)__init_begin, .end = (unsigned long)_sdata, .mask = ~PMD_SECT_XN, .prot = PMD_SECT_XN, }, -#ifdef CONFIG_DEBUG_RODATA +#ifdef CONFIG_DEBUG_ALIGN_RODATA /* Make rodata NX (set RO in ro_perms below). */ { + .name = "rodata NX", .start = (unsigned long)__start_rodata, .end = (unsigned long)__init_begin, .mask = ~PMD_SECT_XN, @@ -607,10 +611,10 @@ static struct section_perm nx_perms[] = { #endif }; -#ifdef CONFIG_DEBUG_RODATA static struct section_perm ro_perms[] = { /* Make kernel code and rodata RX (set RO). */ { + .name = "text/rodata RO", .start = (unsigned long)_stext, .end = (unsigned long)__init_begin, #ifdef CONFIG_ARM_LPAE @@ -623,7 +627,6 @@ static struct section_perm ro_perms[] = { #endif }, }; -#endif /* * Updates section permissions only for the current mm (sections are @@ -670,8 +673,8 @@ void set_section_perms(struct section_perm *perms, int n, bool set, for (i = 0; i < n; i++) { if (!IS_ALIGNED(perms[i].start, SECTION_SIZE) || !IS_ALIGNED(perms[i].end, SECTION_SIZE)) { - pr_err("BUG: section %lx-%lx not aligned to %lx\n", - perms[i].start, perms[i].end, + pr_err("BUG: %s section %lx-%lx not aligned to %lx\n", + perms[i].name, perms[i].start, perms[i].end, SECTION_SIZE); continue; } @@ -712,7 +715,6 @@ void fix_kernmem_perms(void) stop_machine(__fix_kernmem_perms, NULL, NULL); } -#ifdef CONFIG_DEBUG_RODATA int __mark_rodata_ro(void *unused) { update_sections_early(ro_perms, ARRAY_SIZE(ro_perms)); @@ -735,11 +737,10 @@ void set_kernel_text_ro(void) set_section_perms(ro_perms, ARRAY_SIZE(ro_perms), true, current->active_mm); } -#endif /* CONFIG_DEBUG_RODATA */ #else static inline void fix_kernmem_perms(void) { } -#endif /* CONFIG_ARM_KERNMEM_PERMS */ +#endif /* CONFIG_DEBUG_RODATA */ void free_tcmmem(void) { -- cgit From 33298ef6d8ddef57aaa1d11ed53fc08bef2f95aa Mon Sep 17 00:00:00 2001 From: Doug Anderson Date: Fri, 29 Jan 2016 23:06:08 +0100 Subject: ARM: 8505/1: dma-mapping: Optimize allocation The __iommu_alloc_buffer() is expected to be called to allocate pretty sizeable buffers. Upon simple tests of video I saw it trying to allocate 4,194,304 bytes. The function tries to allocate large chunks in order to optimize IOMMU TLB usage. The current function is very, very slow. One problem is the way it keeps trying and trying to allocate big chunks. Imagine a very fragmented memory that has 4M free but no contiguous pages at all. Further imagine allocating 4M (1024 pages). We'll do the following memory allocations: - For page 1: - Try to allocate order 10 (no retry) - Try to allocate order 9 (no retry) - ... - Try to allocate order 0 (with retry, but not needed) - For page 2: - Try to allocate order 9 (no retry) - Try to allocate order 8 (no retry) - ... - Try to allocate order 0 (with retry, but not needed) - ... - ... Total number of calls to alloc() calls for this case is: sum(int(math.log(i, 2)) + 1 for i in range(1, 1025)) => 9228 The above is obviously worse case, but given how slow alloc can be we really want to try to avoid even somewhat bad cases. I timed the old code with a device under memory pressure and it wasn't hard to see it take more than 120 seconds to allocate 4 megs of memory! (NOTE: testing was done on kernel 3.14, so possibly mainline would behave differently). A second problem is that allocating big chunks under memory pressure when we don't need them is just not a great idea anyway unless we really need them. We can make due pretty well with smaller chunks so it's probably wise to leave bigger chunks for other users once memory pressure is on. Let's adjust the allocation like this: 1. If a big chunk fails, stop trying to hard and bump down to lower order allocations. 2. Don't try useless orders. The whole point of big chunks is to optimize the TLB and it can really only make use of 2M, 1M, 64K and 4K sizes. We'll still tend to eat up a bunch of big chunks, but that might be the right answer for some users. A future patch could possibly add a new DMA_ATTR that would let the caller decide that TLB optimization isn't important and that we should use smaller chunks. Presumably this would be a sane strategy for some callers. Signed-off-by: Douglas Anderson Acked-by: Marek Szyprowski Reviewed-by: Robin Murphy Reviewed-by: Tomasz Figa Tested-by: Javier Martinez Canillas Signed-off-by: Russell King --- arch/arm/mm/dma-mapping.c | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 0eca3812527e..bc9cebfa0891 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -1122,6 +1122,9 @@ static inline void __free_iova(struct dma_iommu_mapping *mapping, spin_unlock_irqrestore(&mapping->lock, flags); } +/* We'll try 2M, 1M, 64K, and finally 4K; array must end with 0! */ +static const int iommu_order_array[] = { 9, 8, 4, 0 }; + static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, gfp_t gfp, struct dma_attrs *attrs) { @@ -1129,6 +1132,7 @@ static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, int count = size >> PAGE_SHIFT; int array_size = count * sizeof(struct page *); int i = 0; + int order_idx = 0; if (array_size <= PAGE_SIZE) pages = kzalloc(array_size, GFP_KERNEL); @@ -1162,22 +1166,24 @@ static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, while (count) { int j, order; - for (order = __fls(count); order > 0; --order) { - /* - * We do not want OOM killer to be invoked as long - * as we can fall back to single pages, so we force - * __GFP_NORETRY for orders higher than zero. - */ - pages[i] = alloc_pages(gfp | __GFP_NORETRY, order); - if (pages[i]) - break; + order = iommu_order_array[order_idx]; + + /* Drop down when we get small */ + if (__fls(count) < order) { + order_idx++; + continue; } - if (!pages[i]) { - /* - * Fall back to single page allocation. - * Might invoke OOM killer as last resort. - */ + if (order) { + /* See if it's easy to allocate a high-order chunk */ + pages[i] = alloc_pages(gfp | __GFP_NORETRY, order); + + /* Go down a notch at first sign of pressure */ + if (!pages[i]) { + order_idx++; + continue; + } + } else { pages[i] = alloc_pages(gfp, 0); if (!pages[i]) goto error; -- cgit From 14d3ae2efeed4ebcc6313fad61470803eb904126 Mon Sep 17 00:00:00 2001 From: Doug Anderson Date: Fri, 29 Jan 2016 23:08:46 +0100 Subject: ARM: 8507/1: dma-mapping: Use DMA_ATTR_ALLOC_SINGLE_PAGES hint to optimize alloc If we know that TLB efficiency will not be an issue when memory is accessed then it's not terribly important to allocate big chunks of memory. The whole point of allocating the big chunks was that it would make TLB usage efficient. As Marek Szyprowski indicated: Please note that mapping memory with larger pages significantly improves performance, especially when IOMMU has a little TLB cache. This can be easily observed when multimedia devices do processing of RGB data with 90/270 degree rotation Image rotation is distinctly an operation that needs to bounce around through memory, so it makes sense that TLB efficiency is important there. Video decoding, on the other hand, is a fairly sequential operation. During video decoding it's not expected that we'll be jumping all over memory. Decoding video is also pretty heavy and the TLB misses aren't a huge deal. Presumably most HW video acceleration users of dma-mapping will not care about huge pages and will set DMA_ATTR_ALLOC_SINGLE_PAGES. Allocating big chunks of memory is quite expensive, especially if we're doing it repeadly and memory is full. In one (out of tree) usage model it is common that arm_iommu_alloc_attrs() is called 16 times in a row, each one trying to allocate 4 MB of memory. This is called whenever the system encounters a new video, which could easily happen while the memory system is stressed out. In fact, on certain social media websites that auto-play video and have infinite scrolling, it's quite common to see not just one of these 16x4MB allocations but 2 or 3 right after another. Asking the system even to do a small amount of extra work to give us big chunks in this case is just not a good use of time. Allocating big chunks of memory is also expensive indirectly. Even if we ask the system not to do ANY extra work to allocate _our_ memory, we're still potentially eating up all big chunks in the system. Presumably there are other users in the system that aren't quite as flexible and that actually need these big chunks. By eating all the big chunks we're causing extra work for the rest of the system. We also may start making other memory allocations fail. While the system may be robust to such failures (as is the case with dwc2 USB trying to allocate buffers for Ethernet data and with WiFi trying to allocate buffers for WiFi data), it is yet another big performance hit. Signed-off-by: Douglas Anderson Acked-by: Marek Szyprowski Tested-by: Javier Martinez Canillas Signed-off-by: Russell King --- arch/arm/mm/dma-mapping.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index bc9cebfa0891..9f996a3d79f7 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -1158,6 +1158,10 @@ static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, return pages; } + /* Go straight to 4K chunks if caller says it's OK. */ + if (dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) + order_idx = ARRAY_SIZE(iommu_order_array) - 1; + /* * IOMMU can map any pages, so himem can also be used here */ -- cgit From 02afa9a87b232bca15bc30808b9310c6388ca1a8 Mon Sep 17 00:00:00 2001 From: Chris Brandt Date: Tue, 9 Feb 2016 19:34:43 +0100 Subject: ARM: 8518/1: Use correct symbols for XIP_KERNEL For an XIP build, _etext does not represent the end of the binary image that needs to stay mapped into the MODULES_VADDR area. Years ago, data came before text in the memory map. However, now that the order is text/init/data, an XIP_KERNEL needs to map up to the data location in order to keep from cutting off parts of the kernel that are needed. We only map up to the beginning of data because data has already been copied, so there's no reason to keep it around anymore. A new symbol is created to make it clear what it is we are referring to. This fixes the bug where you might lose the end of your kernel area after page table setup is complete. Signed-off-by: Chris Brandt Signed-off-by: Russell King --- arch/arm/mm/mmu.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 434d76f0b363..e4b681aafd6d 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -1253,7 +1253,7 @@ static inline void prepare_page_table(void) #ifdef CONFIG_XIP_KERNEL /* The XIP kernel is mapped in the module area -- skip over it */ - addr = ((unsigned long)_etext + PMD_SIZE - 1) & PMD_MASK; + addr = ((unsigned long)_exiprom + PMD_SIZE - 1) & PMD_MASK; #endif for ( ; addr < PAGE_OFFSET; addr += PMD_SIZE) pmd_clear(pmd_off_k(addr)); @@ -1335,7 +1335,7 @@ static void __init devicemaps_init(const struct machine_desc *mdesc) #ifdef CONFIG_XIP_KERNEL map.pfn = __phys_to_pfn(CONFIG_XIP_PHYS_ADDR & SECTION_MASK); map.virtual = MODULES_VADDR; - map.length = ((unsigned long)_etext - map.virtual + ~SECTION_MASK) & SECTION_MASK; + map.length = ((unsigned long)_exiprom - map.virtual + ~SECTION_MASK) & SECTION_MASK; map.type = MT_ROM; create_mapping(&map); #endif @@ -1426,7 +1426,11 @@ static void __init kmap_init(void) static void __init map_lowmem(void) { struct memblock_region *reg; +#ifdef CONFIG_XIP_KERNEL + phys_addr_t kernel_x_start = round_down(__pa(_sdata), SECTION_SIZE); +#else phys_addr_t kernel_x_start = round_down(__pa(_stext), SECTION_SIZE); +#endif phys_addr_t kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE); /* Map all the lowmem memory banks. */ -- cgit From 64ac2e74f0b21505606faf725cb5633d63b8b728 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Tue, 26 Jan 2016 01:20:21 +0100 Subject: ARM: 8502/1: mm: mark section-aligned portion of rodata NX When rodata is large enough that it crosses a section boundary after the kernel text, mark the rest NX. This is as close to full NX of rodata as we can get without splitting page tables or doing section alignment via CONFIG_DEBUG_ALIGN_RODATA. When the config is: CONFIG_DEBUG_RODATA=y # CONFIG_DEBUG_ALIGN_RODATA is not set Before: ---[ Kernel Mapping ]--- 0x80000000-0x80100000 1M RW NX SHD 0x80100000-0x80a00000 9M ro x SHD 0x80a00000-0xa0000000 502M RW NX SHD After: ---[ Kernel Mapping ]--- 0x80000000-0x80100000 1M RW NX SHD 0x80100000-0x80700000 6M ro x SHD 0x80700000-0x80a00000 3M ro NX SHD 0x80a00000-0xa0000000 502M RW NX SHD Signed-off-by: Kees Cook Reviewed-by: Ard Biesheuvel Signed-off-by: Russell King --- arch/arm/mm/init.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 53f42508025b..370581aeb871 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -582,6 +582,9 @@ struct section_perm { pmdval_t clear; }; +/* First section-aligned location at or after __start_rodata. */ +extern char __start_rodata_section_aligned[]; + static struct section_perm nx_perms[] = { /* Make pages tables, etc before _stext RW (set NX). */ { @@ -599,16 +602,14 @@ static struct section_perm nx_perms[] = { .mask = ~PMD_SECT_XN, .prot = PMD_SECT_XN, }, -#ifdef CONFIG_DEBUG_ALIGN_RODATA /* Make rodata NX (set RO in ro_perms below). */ { .name = "rodata NX", - .start = (unsigned long)__start_rodata, + .start = (unsigned long)__start_rodata_section_aligned, .end = (unsigned long)__init_begin, .mask = ~PMD_SECT_XN, .prot = PMD_SECT_XN, }, -#endif }; static struct section_perm ro_perms[] = { -- cgit From d78114554939aec0344b494e759d0679224562db Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Tue, 2 Feb 2016 00:14:53 +0100 Subject: ARM: 8512/1: proc-v7.S: Adjust stack address when XIP_KERNEL When XIP_KERNEL is enabled, the virt to phys address translation for RAM is not the same as the virt to phys address translation for .text. The only way to know where physical RAM is located is to use PLAT_PHYS_OFFSET. The MACRO will be useful for other places where there is a similar problem. Signed-off-by: Nicolas Pitre Signed-off-by: Chris Brandt Signed-off-by: Russell King --- arch/arm/mm/proc-v7.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S index 0f92d575a304..1595fb29ec12 100644 --- a/arch/arm/mm/proc-v7.S +++ b/arch/arm/mm/proc-v7.S @@ -487,7 +487,7 @@ __errata_finish: .align 2 __v7_setup_stack_ptr: - .word __v7_setup_stack - . + .word __v7_setup_stack - . + PHYS_OFFSET_FIXUP ENDPROC(__v7_setup) .bss -- cgit From 8ff97fa31333e8d0f4f7029798d9c7d59359b05c Mon Sep 17 00:00:00 2001 From: Russell King Date: Tue, 16 Feb 2016 17:33:56 +0000 Subject: ARM: make the physical-relative calculation more obvious The physical-relative calculation between the XIP text and data sections introduced by the previous patch was far from obvious. Let's simplify it by turning it into a macro which takes the two (virtual) addresses. This allows us to arrange the calculation in a more obvious manner - we can make it two sub-expressions which calculate the physical address for each symbol, and then takes the difference of those physical addresses. Signed-off-by: Russell King --- arch/arm/mm/proc-v7.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S index 1595fb29ec12..0f8963a7e7d9 100644 --- a/arch/arm/mm/proc-v7.S +++ b/arch/arm/mm/proc-v7.S @@ -487,7 +487,7 @@ __errata_finish: .align 2 __v7_setup_stack_ptr: - .word __v7_setup_stack - . + PHYS_OFFSET_FIXUP + .word PHYS_RELATIVE(__v7_setup_stack, .) ENDPROC(__v7_setup) .bss -- cgit From ac96680d22ed86168bcccb4aae768bddb869481d Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Fri, 19 Feb 2016 16:41:54 +0100 Subject: ARM: 8535/1: mm: DEBUG_RODATA makes no sense with XIP_KERNEL When CONFIG_DEBUG_ALIGN_RODATA is set, we get a link error: arch/arm/mm/built-in.o:(.data+0x4bc): undefined reference to `__start_rodata_section_aligned' However, this combination is useless, as XIP_KERNEL implies that all the RODATA is already marked readonly, so both CONFIG_DEBUG_RODATA and CONFIG_DEBUG_ALIGN_RODATA (which depends on the other) are not needed with XIP_KERNEL, and this patches enforces that using a Kconfig dependency. Signed-off-by: Arnd Bergmann Fixes: 25362dc496ed ("ARM: 8501/1: mm: flip priority of CONFIG_DEBUG_RODATA") Acked-by: Ard Biesheuvel Signed-off-by: Russell King --- arch/arm/mm/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 4daeda0a5b7f..55347662e5ed 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -1039,7 +1039,7 @@ config ARCH_SUPPORTS_BIG_ENDIAN config DEBUG_RODATA bool "Make kernel text and rodata read-only" - depends on MMU + depends on MMU && !XIP_KERNEL default y if CPU_V7 help If this is set, kernel text and rodata memory will be made -- cgit From 19e6e5e5392bd646c93d9e2c7b2b58c8558cb041 Mon Sep 17 00:00:00 2001 From: Rabin Vincent Date: Thu, 3 Mar 2016 15:58:00 +0100 Subject: ARM: 8547/1: dma-mapping: store buffer information Keep a list of allocated DMA buffers so that we can store metadata in alloc() which we later need in free(). Signed-off-by: Rabin Vincent Signed-off-by: Russell King --- arch/arm/mm/dma-mapping.c | 49 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 9f996a3d79f7..696f6ee259ee 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -42,6 +42,31 @@ #include "dma.h" #include "mm.h" +struct arm_dma_buffer { + struct list_head list; + void *virt; +}; + +static LIST_HEAD(arm_dma_bufs); +static DEFINE_SPINLOCK(arm_dma_bufs_lock); + +static struct arm_dma_buffer *arm_dma_buffer_find(void *virt) +{ + struct arm_dma_buffer *buf, *found = NULL; + unsigned long flags; + + spin_lock_irqsave(&arm_dma_bufs_lock, flags); + list_for_each_entry(buf, &arm_dma_bufs, list) { + if (buf->virt == virt) { + list_del(&buf->list); + found = buf; + break; + } + } + spin_unlock_irqrestore(&arm_dma_bufs_lock, flags); + return found; +} + /* * The DMA API is built upon the notion of "buffer ownership". A buffer * is either exclusively owned by the CPU (and therefore may be accessed @@ -620,6 +645,7 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, struct page *page = NULL; void *addr; bool want_vaddr; + struct arm_dma_buffer *buf; #ifdef CONFIG_DMA_API_DEBUG u64 limit = (mask + 1) & ~mask; @@ -633,6 +659,10 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, if (!mask) return NULL; + buf = kzalloc(sizeof(*buf), gfp); + if (!buf) + return NULL; + if (mask < 0xffffffffULL) gfp |= GFP_DMA; @@ -662,8 +692,18 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller, want_vaddr); - if (page) + if (page) { + unsigned long flags; + *handle = pfn_to_dma(dev, page_to_pfn(page)); + buf->virt = want_vaddr ? addr : page; + + spin_lock_irqsave(&arm_dma_bufs_lock, flags); + list_add(&buf->list, &arm_dma_bufs); + spin_unlock_irqrestore(&arm_dma_bufs_lock, flags); + } else { + kfree(buf); + } return want_vaddr ? addr : page; } @@ -742,6 +782,11 @@ static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, { struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); bool want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs); + struct arm_dma_buffer *buf; + + buf = arm_dma_buffer_find(cpu_addr); + if (WARN(!buf, "Freeing invalid buffer %p\n", cpu_addr)) + return; size = PAGE_ALIGN(size); @@ -760,6 +805,8 @@ static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, WARN_ON(irqs_disabled()); __free_from_contiguous(dev, page, cpu_addr, size, want_vaddr); } + + kfree(buf); } void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, -- cgit From b42686761219036ace45192476d8ab64d86a4ece Mon Sep 17 00:00:00 2001 From: Rabin Vincent Date: Thu, 3 Mar 2016 15:58:01 +0100 Subject: ARM: 8546/1: dma-mapping: refactor to fix coherent+cma+gfp=0 Given a device which uses arm_coherent_dma_ops and on which dev_get_cma_area(dev) returns non-NULL, the following usage of the DMA API with gfp=0 results in memory corruption and a memory leak. p = dma_alloc_coherent(dev, sz, &dma, 0); if (p) dma_free_coherent(dev, sz, p, dma); The memory leak is because the alloc allocates using __alloc_simple_buffer() but the free attempts dma_release_from_contiguous() which does not do free anything since the page is not in the CMA area. The memory corruption is because the free calls __dma_remap() on a page which is backed by only first level page tables. The apply_to_page_range() + __dma_update_pte() loop ends up interpreting the section mapping as an addresses to a second level page table and writing the new PTE to memory which is not used by page tables. We don't have access to the GFP flags used for allocation in the free function. Fix this by adding allocator backends and using this information in the free function so that we always use the correct release routine. Fixes: 21caf3a7 ("ARM: 8398/1: arm DMA: Fix allocation from CMA for coherent DMA") Signed-off-by: Rabin Vincent Signed-off-by: Russell King --- arch/arm/mm/dma-mapping.c | 165 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 128 insertions(+), 37 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 696f6ee259ee..deac58d5f1f7 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -42,9 +42,33 @@ #include "dma.h" #include "mm.h" +struct arm_dma_alloc_args { + struct device *dev; + size_t size; + gfp_t gfp; + pgprot_t prot; + const void *caller; + bool want_vaddr; +}; + +struct arm_dma_free_args { + struct device *dev; + size_t size; + void *cpu_addr; + struct page *page; + bool want_vaddr; +}; + +struct arm_dma_allocator { + void *(*alloc)(struct arm_dma_alloc_args *args, + struct page **ret_page); + void (*free)(struct arm_dma_free_args *args); +}; + struct arm_dma_buffer { struct list_head list; void *virt; + struct arm_dma_allocator *allocator; }; static LIST_HEAD(arm_dma_bufs); @@ -617,7 +641,7 @@ static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot) #define __alloc_remap_buffer(dev, size, gfp, prot, ret, c, wv) NULL #define __alloc_from_pool(size, ret_page) NULL #define __alloc_from_contiguous(dev, size, prot, ret, c, wv) NULL -#define __free_from_pool(cpu_addr, size) 0 +#define __free_from_pool(cpu_addr, size) do { } while (0) #define __free_from_contiguous(dev, page, cpu_addr, size, wv) do { } while (0) #define __dma_free_remap(cpu_addr, size) do { } while (0) @@ -635,7 +659,78 @@ static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp, return page_address(page); } +static void *simple_allocator_alloc(struct arm_dma_alloc_args *args, + struct page **ret_page) +{ + return __alloc_simple_buffer(args->dev, args->size, args->gfp, + ret_page); +} + +static void simple_allocator_free(struct arm_dma_free_args *args) +{ + __dma_free_buffer(args->page, args->size); +} + +static struct arm_dma_allocator simple_allocator = { + .alloc = simple_allocator_alloc, + .free = simple_allocator_free, +}; + +static void *cma_allocator_alloc(struct arm_dma_alloc_args *args, + struct page **ret_page) +{ + return __alloc_from_contiguous(args->dev, args->size, args->prot, + ret_page, args->caller, + args->want_vaddr); +} + +static void cma_allocator_free(struct arm_dma_free_args *args) +{ + __free_from_contiguous(args->dev, args->page, args->cpu_addr, + args->size, args->want_vaddr); +} + +static struct arm_dma_allocator cma_allocator = { + .alloc = cma_allocator_alloc, + .free = cma_allocator_free, +}; +static void *pool_allocator_alloc(struct arm_dma_alloc_args *args, + struct page **ret_page) +{ + return __alloc_from_pool(args->size, ret_page); +} + +static void pool_allocator_free(struct arm_dma_free_args *args) +{ + __free_from_pool(args->cpu_addr, args->size); +} + +static struct arm_dma_allocator pool_allocator = { + .alloc = pool_allocator_alloc, + .free = pool_allocator_free, +}; + +static void *remap_allocator_alloc(struct arm_dma_alloc_args *args, + struct page **ret_page) +{ + return __alloc_remap_buffer(args->dev, args->size, args->gfp, + args->prot, ret_page, args->caller, + args->want_vaddr); +} + +static void remap_allocator_free(struct arm_dma_free_args *args) +{ + if (args->want_vaddr) + __dma_free_remap(args->cpu_addr, args->size); + + __dma_free_buffer(args->page, args->size); +} + +static struct arm_dma_allocator remap_allocator = { + .alloc = remap_allocator_alloc, + .free = remap_allocator_free, +}; static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, pgprot_t prot, bool is_coherent, @@ -644,8 +739,16 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, u64 mask = get_coherent_dma_mask(dev); struct page *page = NULL; void *addr; - bool want_vaddr; + bool allowblock, cma; struct arm_dma_buffer *buf; + struct arm_dma_alloc_args args = { + .dev = dev, + .size = PAGE_ALIGN(size), + .gfp = gfp, + .prot = prot, + .caller = caller, + .want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs), + }; #ifdef CONFIG_DMA_API_DEBUG u64 limit = (mask + 1) & ~mask; @@ -674,29 +777,28 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, * platform; see CONFIG_HUGETLBFS. */ gfp &= ~(__GFP_COMP); + args.gfp = gfp; *handle = DMA_ERROR_CODE; - size = PAGE_ALIGN(size); - want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs); - - if (nommu()) - addr = __alloc_simple_buffer(dev, size, gfp, &page); - else if (dev_get_cma_area(dev) && (gfp & __GFP_DIRECT_RECLAIM)) - addr = __alloc_from_contiguous(dev, size, prot, &page, - caller, want_vaddr); - else if (is_coherent) - addr = __alloc_simple_buffer(dev, size, gfp, &page); - else if (!gfpflags_allow_blocking(gfp)) - addr = __alloc_from_pool(size, &page); + allowblock = gfpflags_allow_blocking(gfp); + cma = allowblock ? dev_get_cma_area(dev) : false; + + if (cma) + buf->allocator = &cma_allocator; + else if (nommu() || is_coherent) + buf->allocator = &simple_allocator; + else if (allowblock) + buf->allocator = &remap_allocator; else - addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, - caller, want_vaddr); + buf->allocator = &pool_allocator; + + addr = buf->allocator->alloc(&args, &page); if (page) { unsigned long flags; *handle = pfn_to_dma(dev, page_to_pfn(page)); - buf->virt = want_vaddr ? addr : page; + buf->virt = args.want_vaddr ? addr : page; spin_lock_irqsave(&arm_dma_bufs_lock, flags); list_add(&buf->list, &arm_dma_bufs); @@ -705,7 +807,7 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, kfree(buf); } - return want_vaddr ? addr : page; + return args.want_vaddr ? addr : page; } /* @@ -781,31 +883,20 @@ static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, bool is_coherent) { struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); - bool want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs); struct arm_dma_buffer *buf; + struct arm_dma_free_args args = { + .dev = dev, + .size = PAGE_ALIGN(size), + .cpu_addr = cpu_addr, + .page = page, + .want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs), + }; buf = arm_dma_buffer_find(cpu_addr); if (WARN(!buf, "Freeing invalid buffer %p\n", cpu_addr)) return; - size = PAGE_ALIGN(size); - - if (nommu()) { - __dma_free_buffer(page, size); - } else if (!is_coherent && __free_from_pool(cpu_addr, size)) { - return; - } else if (!dev_get_cma_area(dev)) { - if (want_vaddr && !is_coherent) - __dma_free_remap(cpu_addr, size); - __dma_free_buffer(page, size); - } else { - /* - * Non-atomic allocations cannot be freed with IRQs disabled - */ - WARN_ON(irqs_disabled()); - __free_from_contiguous(dev, page, cpu_addr, size, want_vaddr); - } - + buf->allocator->free(&args); kfree(buf); } -- cgit