From 516295e5ab4bf986865cfff856d484ec678e3b0b Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 21 Nov 2010 16:27:49 +0000 Subject: ARM: pgtable: add pud-level code Add pud_offset() et.al. between the pgd and pmd code in preparation of using pgtable-nopud.h rather than 4level-fixup.h. This incorporates a fix from Jamie Iles for uaccess_with_memcpy.c. Signed-off-by: Russell King --- arch/arm/include/asm/pgtable.h | 1 + arch/arm/lib/uaccess_with_memcpy.c | 7 ++++++- arch/arm/mm/dma-mapping.c | 11 ++++++++++- arch/arm/mm/fault-armv.c | 7 ++++++- arch/arm/mm/fault.c | 29 +++++++++++++++++++++++++---- arch/arm/mm/idmap.c | 35 +++++++++++++++++++++++++++++------ arch/arm/mm/mm.h | 2 +- arch/arm/mm/mmu.c | 22 ++++++++++++++++++---- arch/arm/mm/pgd.c | 24 ++++++++++++++++++++---- 9 files changed, 116 insertions(+), 22 deletions(-) (limited to 'arch/arm') diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h index ca567914d303..5750704e0271 100644 --- a/arch/arm/include/asm/pgtable.h +++ b/arch/arm/include/asm/pgtable.h @@ -301,6 +301,7 @@ extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; #define pgd_present(pgd) (1) #define pgd_clear(pgdp) do { } while (0) #define set_pgd(pgd,pgdp) do { } while (0) +#define set_pud(pud,pudp) do { } while (0) /* Find an entry in the second-level page table.. */ diff --git a/arch/arm/lib/uaccess_with_memcpy.c b/arch/arm/lib/uaccess_with_memcpy.c index e2d2f2cd0c4f..8b9b13649f81 100644 --- a/arch/arm/lib/uaccess_with_memcpy.c +++ b/arch/arm/lib/uaccess_with_memcpy.c @@ -27,13 +27,18 @@ pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp) pgd_t *pgd; pmd_t *pmd; pte_t *pte; + pud_t *pud; spinlock_t *ptl; pgd = pgd_offset(current->mm, addr); if (unlikely(pgd_none(*pgd) || pgd_bad(*pgd))) return 0; - pmd = pmd_offset(pgd, addr); + pud = pud_offset(pgd, addr); + if (unlikely(pud_none(*pud) || pud_bad(*pud))) + return 0; + + pmd = pmd_offset(pud, addr); if (unlikely(pmd_none(*pmd) || pmd_bad(*pmd))) return 0; diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index ac6a36142fcd..a9bdfcda23f4 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -148,6 +148,7 @@ static int __init consistent_init(void) { int ret = 0; pgd_t *pgd; + pud_t *pud; pmd_t *pmd; pte_t *pte; int i = 0; @@ -155,7 +156,15 @@ static int __init consistent_init(void) do { pgd = pgd_offset(&init_mm, base); - pmd = pmd_alloc(&init_mm, pgd, base); + + pud = pud_alloc(&init_mm, pgd, base); + if (!pud) { + printk(KERN_ERR "%s: no pud tables\n", __func__); + ret = -ENOMEM; + break; + } + + pmd = pmd_alloc(&init_mm, pud, base); if (!pmd) { printk(KERN_ERR "%s: no pmd tables\n", __func__); ret = -ENOMEM; diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index 01210dba0221..7cab79179421 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -95,6 +95,7 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address, { spinlock_t *ptl; pgd_t *pgd; + pud_t *pud; pmd_t *pmd; pte_t *pte; int ret; @@ -103,7 +104,11 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address, if (pgd_none_or_clear_bad(pgd)) return 0; - pmd = pmd_offset(pgd, address); + pud = pud_offset(pgd, address); + if (pud_none_or_clear_bad(pud)) + return 0; + + pmd = pmd_offset(pud, address); if (pmd_none_or_clear_bad(pmd)) return 0; diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index ef0e24f578ef..bc0e1d88fd3b 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -80,6 +80,7 @@ void show_pte(struct mm_struct *mm, unsigned long addr) addr, (long long)pgd_val(*pgd)); do { + pud_t *pud; pmd_t *pmd; pte_t *pte; @@ -91,7 +92,19 @@ void show_pte(struct mm_struct *mm, unsigned long addr) break; } - pmd = pmd_offset(pgd, addr); + pud = pud_offset(pgd, addr); + if (PTRS_PER_PUD != 1) + printk(", *pud=%08lx", pud_val(*pud)); + + if (pud_none(*pud)) + break; + + if (pud_bad(*pud)) { + printk("(bad)"); + break; + } + + pmd = pmd_offset(pud, addr); if (PTRS_PER_PMD != 1) printk(", *pmd=%08llx", (long long)pmd_val(*pmd)); @@ -390,6 +403,7 @@ do_translation_fault(unsigned long addr, unsigned int fsr, { unsigned int index; pgd_t *pgd, *pgd_k; + pud_t *pud, *pud_k; pmd_t *pmd, *pmd_k; if (addr < TASK_SIZE) @@ -408,12 +422,19 @@ do_translation_fault(unsigned long addr, unsigned int fsr, if (pgd_none(*pgd_k)) goto bad_area; - if (!pgd_present(*pgd)) set_pgd(pgd, *pgd_k); - pmd_k = pmd_offset(pgd_k, addr); - pmd = pmd_offset(pgd, addr); + pud = pud_offset(pgd, addr); + pud_k = pud_offset(pgd_k, addr); + + if (pud_none(*pud_k)) + goto bad_area; + if (!pud_present(*pud)) + set_pud(pud, *pud_k); + + pmd = pmd_offset(pud, addr); + pmd_k = pmd_offset(pud_k, addr); /* * On ARM one Linux PGD entry contains two hardware entries (see page diff --git a/arch/arm/mm/idmap.c b/arch/arm/mm/idmap.c index 57299446f787..2be9139a4ef3 100644 --- a/arch/arm/mm/idmap.c +++ b/arch/arm/mm/idmap.c @@ -4,10 +4,10 @@ #include #include -static void idmap_add_pmd(pgd_t *pgd, unsigned long addr, unsigned long end, +static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end, unsigned long prot) { - pmd_t *pmd = pmd_offset(pgd, addr); + pmd_t *pmd = pmd_offset(pud, addr); addr = (addr & PMD_MASK) | prot; pmd[0] = __pmd(addr); @@ -16,6 +16,18 @@ static void idmap_add_pmd(pgd_t *pgd, unsigned long addr, unsigned long end, flush_pmd_entry(pmd); } +static void idmap_add_pud(pgd_t *pgd, unsigned long addr, unsigned long end, + unsigned long prot) +{ + pud_t *pud = pud_offset(pgd, addr); + unsigned long next; + + do { + next = pud_addr_end(addr, end); + idmap_add_pmd(pud, addr, next, prot); + } while (pud++, addr = next, addr != end); +} + void identity_mapping_add(pgd_t *pgd, unsigned long addr, unsigned long end) { unsigned long prot, next; @@ -27,17 +39,28 @@ void identity_mapping_add(pgd_t *pgd, unsigned long addr, unsigned long end) pgd += pgd_index(addr); do { next = pgd_addr_end(addr, end); - idmap_add_pmd(pgd, addr, next, prot); + idmap_add_pud(pgd, addr, next, prot); } while (pgd++, addr = next, addr != end); } #ifdef CONFIG_SMP -static void idmap_del_pmd(pgd_t *pgd, unsigned long addr, unsigned long end) +static void idmap_del_pmd(pud_t *pud, unsigned long addr, unsigned long end) { - pmd_t *pmd = pmd_offset(pgd, addr); + pmd_t *pmd = pmd_offset(pud, addr); pmd_clear(pmd); } +static void idmap_del_pud(pgd_t *pgd, unsigned long addr, unsigned long end) +{ + pud_t *pud = pud_offset(pgd, addr); + unsigned long next; + + do { + next = pud_addr_end(addr, end); + idmap_del_pmd(pud, addr, next); + } while (pud++, addr = next, addr != end); +} + void identity_mapping_del(pgd_t *pgd, unsigned long addr, unsigned long end) { unsigned long next; @@ -45,7 +68,7 @@ void identity_mapping_del(pgd_t *pgd, unsigned long addr, unsigned long end) pgd += pgd_index(addr); do { next = pgd_addr_end(addr, end); - idmap_del_pmd(pgd, addr, next); + idmap_del_pud(pgd, addr, next); } while (pgd++, addr = next, addr != end); } #endif diff --git a/arch/arm/mm/mm.h b/arch/arm/mm/mm.h index 36960df5fb76..d2384106af9c 100644 --- a/arch/arm/mm/mm.h +++ b/arch/arm/mm/mm.h @@ -7,7 +7,7 @@ extern pmd_t *top_pmd; static inline pmd_t *pmd_off(pgd_t *pgd, unsigned long virt) { - return pmd_offset(pgd, virt); + return pmd_offset(pud_offset(pgd, virt), virt); } static inline pmd_t *pmd_off_k(unsigned long virt) diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 176749010935..82ef6966ae09 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -550,11 +550,11 @@ static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr, } while (pte++, addr += PAGE_SIZE, addr != end); } -static void __init alloc_init_section(pgd_t *pgd, unsigned long addr, +static void __init alloc_init_section(pud_t *pud, unsigned long addr, unsigned long end, phys_addr_t phys, const struct mem_type *type) { - pmd_t *pmd = pmd_offset(pgd, addr); + pmd_t *pmd = pmd_offset(pud, addr); /* * Try a section mapping - end, addr and phys must all be aligned @@ -583,6 +583,19 @@ static void __init alloc_init_section(pgd_t *pgd, unsigned long addr, } } +static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end, + unsigned long phys, const struct mem_type *type) +{ + pud_t *pud = pud_offset(pgd, addr); + unsigned long next; + + do { + next = pud_addr_end(addr, end); + alloc_init_section(pud, addr, next, phys, type); + phys += next - addr; + } while (pud++, addr = next, addr != end); +} + static void __init create_36bit_mapping(struct map_desc *md, const struct mem_type *type) { @@ -630,7 +643,8 @@ static void __init create_36bit_mapping(struct map_desc *md, pgd = pgd_offset_k(addr); end = addr + length; do { - pmd_t *pmd = pmd_offset(pgd, addr); + pud_t *pud = pud_offset(pgd, addr); + pmd_t *pmd = pmd_offset(pud, addr); int i; for (i = 0; i < 16; i++) @@ -696,7 +710,7 @@ static void __init create_mapping(struct map_desc *md) do { unsigned long next = pgd_addr_end(addr, end); - alloc_init_section(pgd, addr, next, phys, type); + alloc_init_pud(pgd, addr, next, phys, type); phys += next - addr; addr = next; diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c index 93292a18cf77..f7fafb1741f4 100644 --- a/arch/arm/mm/pgd.c +++ b/arch/arm/mm/pgd.c @@ -23,6 +23,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm) { pgd_t *new_pgd, *init_pgd; + pud_t *new_pud, *init_pud; pmd_t *new_pmd, *init_pmd; pte_t *new_pte, *init_pte; @@ -46,7 +47,11 @@ pgd_t *pgd_alloc(struct mm_struct *mm) * On ARM, first page must always be allocated since it * contains the machine vectors. */ - new_pmd = pmd_alloc(mm, new_pgd, 0); + new_pud = pud_alloc(mm, new_pgd, 0); + if (!new_pud) + goto no_pud; + + new_pmd = pmd_alloc(mm, new_pud, 0); if (!new_pmd) goto no_pmd; @@ -54,7 +59,8 @@ pgd_t *pgd_alloc(struct mm_struct *mm) if (!new_pte) goto no_pte; - init_pmd = pmd_offset(init_pgd, 0); + init_pud = pud_offset(init_pgd, 0); + init_pmd = pmd_offset(init_pud, 0); init_pte = pte_offset_map(init_pmd, 0); set_pte_ext(new_pte, *init_pte, 0); pte_unmap(init_pte); @@ -66,6 +72,8 @@ pgd_t *pgd_alloc(struct mm_struct *mm) no_pte: pmd_free(mm, new_pmd); no_pmd: + pud_free(mm, new_pud); +no_pud: free_pages((unsigned long)new_pgd, 2); no_pgd: return NULL; @@ -74,6 +82,7 @@ no_pgd: void pgd_free(struct mm_struct *mm, pgd_t *pgd_base) { pgd_t *pgd; + pud_t *pud; pmd_t *pmd; pgtable_t pte; @@ -84,7 +93,11 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd_base) if (pgd_none_or_clear_bad(pgd)) goto no_pgd; - pmd = pmd_offset(pgd, 0); + pud = pud_offset(pgd, 0); + if (pud_none_or_clear_bad(pud)) + goto no_pud; + + pmd = pmd_offset(pud, 0); if (pmd_none_or_clear_bad(pmd)) goto no_pmd; @@ -92,8 +105,11 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd_base) pmd_clear(pmd); pte_free(mm, pte); no_pmd: - pgd_clear(pgd); + pud_clear(pud); pmd_free(mm, pmd); +no_pud: + pgd_clear(pgd); + pud_free(mm, pud); no_pgd: free_pages((unsigned long) pgd_base, 2); } -- cgit