summaryrefslogtreecommitdiff
path: root/arch/arm/kernel/head.S
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm/kernel/head.S')
-rw-r--r--arch/arm/kernel/head.S311
1 files changed, 91 insertions, 220 deletions
diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S
index c49b39340ddb..f22c50d4bd41 100644
--- a/arch/arm/kernel/head.S
+++ b/arch/arm/kernel/head.S
@@ -10,20 +10,19 @@
*/
#include <linux/linkage.h>
#include <linux/init.h>
+#include <linux/pgtable.h>
#include <asm/assembler.h>
#include <asm/cp15.h>
#include <asm/domain.h>
#include <asm/ptrace.h>
#include <asm/asm-offsets.h>
-#include <asm/memory.h>
+#include <asm/page.h>
#include <asm/thread_info.h>
-#include <asm/pgtable.h>
#if defined(CONFIG_DEBUG_LL) && !defined(CONFIG_DEBUG_SEMIHOSTING)
#include CONFIG_DEBUG_LL_INCLUDE
#endif
-
/*
* swapper_pg_dir is the virtual address of the initial page table.
* We place the page tables 16K below KERNEL_RAM_VADDR. Therefore, we must
@@ -31,7 +30,7 @@
* the least significant 16 bits to be 0x8000, but we could probably
* relax this restriction to KERNEL_RAM_VADDR >= PAGE_OFFSET + 0x4000.
*/
-#define KERNEL_RAM_VADDR (PAGE_OFFSET + TEXT_OFFSET)
+#define KERNEL_RAM_VADDR (KERNEL_OFFSET + TEXT_OFFSET)
#if (KERNEL_RAM_VADDR & 0xffff) != 0x8000
#error KERNEL_RAM_VADDR must start at 0xXXXX8000
#endif
@@ -39,15 +38,32 @@
#ifdef CONFIG_ARM_LPAE
/* LPAE requires an additional page for the PGD */
#define PG_DIR_SIZE 0x5000
-#define PMD_ORDER 3
+#define PMD_ENTRY_ORDER 3 /* PMD entry size is 2^PMD_ENTRY_ORDER */
#else
#define PG_DIR_SIZE 0x4000
-#define PMD_ORDER 2
+#define PMD_ENTRY_ORDER 2
#endif
.globl swapper_pg_dir
.equ swapper_pg_dir, KERNEL_RAM_VADDR - PG_DIR_SIZE
+ /*
+ * This needs to be assigned at runtime when the linker symbols are
+ * resolved. These are unsigned 64bit really, but in this assembly code
+ * We store them as 32bit.
+ */
+ .pushsection .data
+ .align 2
+ .globl kernel_sec_start
+ .globl kernel_sec_end
+kernel_sec_start:
+ .long 0
+ .long 0
+kernel_sec_end:
+ .long 0
+ .long 0
+ .popsection
+
.macro pgtbl, rd, phys
add \rd, \phys, #TEXT_OFFSET
sub \rd, \rd, #PG_DIR_SIZE
@@ -103,10 +119,8 @@ ENTRY(stext)
#endif
#ifndef CONFIG_XIP_KERNEL
- adr r3, 2f
- ldmia r3, {r4, r8}
- sub r4, r3, r4 @ (PHYS_OFFSET - PAGE_OFFSET)
- add r8, r8, r4 @ PHYS_OFFSET
+ adr_l r8, _text @ __pa(_text)
+ sub r8, r8, #TEXT_OFFSET @ PHYS_OFFSET
#else
ldr r8, =PLAT_PHYS_OFFSET @ always constant in this case
#endif
@@ -158,10 +172,6 @@ ENTRY(stext)
1: b __enable_mmu
ENDPROC(stext)
.ltorg
-#ifndef CONFIG_XIP_KERNEL
-2: .long .
- .long PAGE_OFFSET
-#endif
/*
* Setup the initial page tables. We only setup the barest
@@ -224,33 +234,49 @@ __create_page_tables:
* Create identity mapping to cater for __enable_mmu.
* This identity mapping will be removed by paging_init().
*/
- adr r0, __turn_mmu_on_loc
- ldmia r0, {r3, r5, r6}
- sub r0, r0, r3 @ virt->phys offset
- add r5, r5, r0 @ phys __turn_mmu_on
- add r6, r6, r0 @ phys __turn_mmu_on_end
+ adr_l r5, __turn_mmu_on @ _pa(__turn_mmu_on)
+ adr_l r6, __turn_mmu_on_end @ _pa(__turn_mmu_on_end)
mov r5, r5, lsr #SECTION_SHIFT
mov r6, r6, lsr #SECTION_SHIFT
1: orr r3, r7, r5, lsl #SECTION_SHIFT @ flags + kernel base
- str r3, [r4, r5, lsl #PMD_ORDER] @ identity mapping
+ str r3, [r4, r5, lsl #PMD_ENTRY_ORDER] @ identity mapping
cmp r5, r6
addlo r5, r5, #1 @ next section
blo 1b
/*
- * Map our RAM from the start to the end of the kernel .bss section.
+ * The main matter: map in the kernel using section mappings, and
+ * set two variables to indicate the physical start and end of the
+ * kernel.
*/
- add r0, r4, #PAGE_OFFSET >> (SECTION_SHIFT - PMD_ORDER)
+ add r0, r4, #KERNEL_OFFSET >> (SECTION_SHIFT - PMD_ENTRY_ORDER)
ldr r6, =(_end - 1)
- orr r3, r8, r7
- add r6, r4, r6, lsr #(SECTION_SHIFT - PMD_ORDER)
-1: str r3, [r0], #1 << PMD_ORDER
+
+ /* For XIP, kernel_sec_start/kernel_sec_end are currently in RO memory */
+#ifndef CONFIG_XIP_KERNEL
+ adr_l r5, kernel_sec_start @ _pa(kernel_sec_start)
+#if defined CONFIG_CPU_ENDIAN_BE8 || defined CONFIG_CPU_ENDIAN_BE32
+ str r8, [r5, #4] @ Save physical start of kernel (BE)
+#else
+ str r8, [r5] @ Save physical start of kernel (LE)
+#endif
+#endif
+ orr r3, r8, r7 @ Add the MMU flags
+ add r6, r4, r6, lsr #(SECTION_SHIFT - PMD_ENTRY_ORDER)
+1: str r3, [r0], #1 << PMD_ENTRY_ORDER
add r3, r3, #1 << SECTION_SHIFT
cmp r0, r6
bls 1b
-
-#ifdef CONFIG_XIP_KERNEL
+#ifndef CONFIG_XIP_KERNEL
+ eor r3, r3, r7 @ Remove the MMU flags
+ adr_l r5, kernel_sec_end @ _pa(kernel_sec_end)
+#if defined CONFIG_CPU_ENDIAN_BE8 || defined CONFIG_CPU_ENDIAN_BE32
+ str r3, [r5, #4] @ Save physical end of kernel (BE)
+#else
+ str r3, [r5] @ Save physical end of kernel (LE)
+#endif
+#else
/*
* Map the kernel image separately as it is not located in RAM.
*/
@@ -258,14 +284,14 @@ __create_page_tables:
mov r3, pc
mov r3, r3, lsr #SECTION_SHIFT
orr r3, r7, r3, lsl #SECTION_SHIFT
- add r0, r4, #(XIP_START & 0xff000000) >> (SECTION_SHIFT - PMD_ORDER)
- str r3, [r0, #((XIP_START & 0x00f00000) >> SECTION_SHIFT) << PMD_ORDER]!
+ add r0, r4, #(XIP_START & 0xff000000) >> (SECTION_SHIFT - PMD_ENTRY_ORDER)
+ str r3, [r0, #((XIP_START & 0x00f00000) >> SECTION_SHIFT) << PMD_ENTRY_ORDER]!
ldr r6, =(_edata_loc - 1)
- add r0, r0, #1 << PMD_ORDER
- add r6, r4, r6, lsr #(SECTION_SHIFT - PMD_ORDER)
+ add r0, r0, #1 << PMD_ENTRY_ORDER
+ add r6, r4, r6, lsr #(SECTION_SHIFT - PMD_ENTRY_ORDER)
1: cmp r0, r6
add r3, r3, #1 << SECTION_SHIFT
- strls r3, [r0], #1 << PMD_ORDER
+ strls r3, [r0], #1 << PMD_ENTRY_ORDER
bls 1b
#endif
@@ -274,12 +300,11 @@ __create_page_tables:
* We map 2 sections in case the ATAGs/DTB crosses a section boundary.
*/
mov r0, r2, lsr #SECTION_SHIFT
- movs r0, r0, lsl #SECTION_SHIFT
- subne r3, r0, r8
- addne r3, r3, #PAGE_OFFSET
- addne r3, r4, r3, lsr #(SECTION_SHIFT - PMD_ORDER)
- orrne r6, r7, r0
- strne r6, [r3], #1 << PMD_ORDER
+ cmp r2, #0
+ ldrne r3, =FDT_FIXED_BASE >> (SECTION_SHIFT - PMD_ENTRY_ORDER)
+ addne r3, r3, r4
+ orrne r6, r7, r0, lsl #SECTION_SHIFT
+ strne r6, [r3], #1 << PMD_ENTRY_ORDER
addne r6, r6, #1 << SECTION_SHIFT
strne r6, [r3]
@@ -298,7 +323,7 @@ __create_page_tables:
addruart r7, r3, r0
mov r3, r3, lsr #SECTION_SHIFT
- mov r3, r3, lsl #PMD_ORDER
+ mov r3, r3, lsl #PMD_ENTRY_ORDER
add r0, r4, r3
mov r3, r7, lsr #SECTION_SHIFT
@@ -323,12 +348,12 @@ __create_page_tables:
ldr r7, [r10, #PROCINFO_IO_MMUFLAGS] @ io_mmuflags
#endif
-#if defined(CONFIG_ARCH_NETWINDER) || defined(CONFIG_ARCH_CATS)
+#if defined(CONFIG_ARCH_NETWINDER)
/*
* If we're using the NetWinder or CATS, we also need to map
* in the 16550-type serial port for the debug messages
*/
- add r0, r4, #0xff000000 >> (SECTION_SHIFT - PMD_ORDER)
+ add r0, r4, #0xff000000 >> (SECTION_SHIFT - PMD_ENTRY_ORDER)
orr r3, r7, #0x7c000000
str r3, [r0]
#endif
@@ -338,10 +363,10 @@ __create_page_tables:
* Similar reasons here - for debug. This is
* only for Acorn RiscPC architectures.
*/
- add r0, r4, #0x02000000 >> (SECTION_SHIFT - PMD_ORDER)
+ add r0, r4, #0x02000000 >> (SECTION_SHIFT - PMD_ENTRY_ORDER)
orr r3, r7, #0x02000000
str r3, [r0]
- add r0, r4, #0xd8000000 >> (SECTION_SHIFT - PMD_ORDER)
+ add r0, r4, #0xd8000000 >> (SECTION_SHIFT - PMD_ENTRY_ORDER)
str r3, [r0]
#endif
#endif
@@ -351,11 +376,6 @@ __create_page_tables:
ret lr
ENDPROC(__create_page_tables)
.ltorg
- .align
-__turn_mmu_on_loc:
- .long .
- .long __turn_mmu_on
- .long __turn_mmu_on_end
#if defined(CONFIG_SMP)
.text
@@ -391,10 +411,12 @@ ENTRY(secondary_startup)
/*
* Use the page tables supplied from __cpu_up.
*/
- adr r4, __secondary_data
- ldmia r4, {r5, r7, r12} @ address to jump to after
- sub lr, r4, r5 @ mmu has been enabled
- add r3, r7, lr
+#ifdef CONFIG_XIP_KERNEL
+ ldr r3, =(secondary_data + PLAT_PHYS_OFFSET - PAGE_OFFSET)
+#else
+ adr_l r3, secondary_data
+#endif
+ mov_l r12, __secondary_switched
ldrd r4, r5, [r3, #0] @ get secondary_data.pgdir
ARM_BE8(eor r4, r4, r5) @ Swap r5 and r4 in BE:
ARM_BE8(eor r5, r4, r5) @ it can be done in 3 steps
@@ -409,22 +431,21 @@ ARM_BE8(eor r4, r4, r5) @ without using a temp reg.
ENDPROC(secondary_startup)
ENDPROC(secondary_startup_arm)
- /*
- * r6 = &secondary_data
- */
ENTRY(__secondary_switched)
- ldr sp, [r7, #12] @ get secondary_data.stack
+#if defined(CONFIG_VMAP_STACK) && !defined(CONFIG_ARM_LPAE)
+ @ Before using the vmap'ed stack, we have to switch to swapper_pg_dir
+ @ as the ID map does not cover the vmalloc region.
+ mrc p15, 0, ip, c2, c0, 1 @ read TTBR1
+ mcr p15, 0, ip, c2, c0, 0 @ set TTBR0
+ instr_sync
+#endif
+ adr_l r7, secondary_data + 12 @ get secondary_data.stack
+ ldr sp, [r7]
+ ldr r0, [r7, #4] @ get secondary_data.task
mov fp, #0
b secondary_start_kernel
ENDPROC(__secondary_switched)
- .align
-
- .type __secondary_data, %object
-__secondary_data:
- .long .
- .long secondary_data
- .long __secondary_switched
#endif /* defined(CONFIG_SMP) */
@@ -539,19 +560,11 @@ ARM_BE8(rev r0, r0) @ byteswap if big endian
retne lr
__fixup_smp_on_up:
- adr r0, 1f
- ldmia r0, {r3 - r5}
- sub r3, r0, r3
- add r4, r4, r3
- add r5, r5, r3
+ adr_l r4, __smpalt_begin
+ adr_l r5, __smpalt_end
b __do_fixup_smp_on_up
ENDPROC(__fixup_smp)
- .align
-1: .word .
- .word __smpalt_begin
- .word __smpalt_end
-
.pushsection .data
.align 2
.globl smp_on_up
@@ -565,14 +578,15 @@ smp_on_up:
__do_fixup_smp_on_up:
cmp r4, r5
reths lr
- ldmia r4!, {r0, r6}
- ARM( str r6, [r0, r3] )
- THUMB( add r0, r0, r3 )
+ ldmia r4, {r0, r6}
+ ARM( str r6, [r0, r4] )
+ THUMB( add r0, r0, r4 )
+ add r4, r4, #8
#ifdef __ARMEB__
THUMB( mov r6, r6, ror #16 ) @ Convert word order for big-endian.
#endif
THUMB( strh r6, [r0], #2 ) @ For Thumb-2, store as two halfwords
- THUMB( mov r6, r6, lsr #16 ) @ to be robust against misaligned r3.
+ THUMB( mov r6, r6, lsr #16 ) @ to be robust against misaligned r0.
THUMB( strh r6, [r0] )
b __do_fixup_smp_on_up
ENDPROC(__do_fixup_smp_on_up)
@@ -581,151 +595,8 @@ ENTRY(fixup_smp)
stmfd sp!, {r4 - r6, lr}
mov r4, r0
add r5, r0, r1
- mov r3, #0
bl __do_fixup_smp_on_up
ldmfd sp!, {r4 - r6, pc}
ENDPROC(fixup_smp)
-#ifdef __ARMEB__
-#define LOW_OFFSET 0x4
-#define HIGH_OFFSET 0x0
-#else
-#define LOW_OFFSET 0x0
-#define HIGH_OFFSET 0x4
-#endif
-
-#ifdef CONFIG_ARM_PATCH_PHYS_VIRT
-
-/* __fixup_pv_table - patch the stub instructions with the delta between
- * PHYS_OFFSET and PAGE_OFFSET, which is assumed to be 16MiB aligned and
- * can be expressed by an immediate shifter operand. The stub instruction
- * has a form of '(add|sub) rd, rn, #imm'.
- */
- __HEAD
-__fixup_pv_table:
- adr r0, 1f
- ldmia r0, {r3-r7}
- mvn ip, #0
- subs r3, r0, r3 @ PHYS_OFFSET - PAGE_OFFSET
- add r4, r4, r3 @ adjust table start address
- add r5, r5, r3 @ adjust table end address
- add r6, r6, r3 @ adjust __pv_phys_pfn_offset address
- add r7, r7, r3 @ adjust __pv_offset address
- mov r0, r8, lsr #PAGE_SHIFT @ convert to PFN
- str r0, [r6] @ save computed PHYS_OFFSET to __pv_phys_pfn_offset
- strcc ip, [r7, #HIGH_OFFSET] @ save to __pv_offset high bits
- mov r6, r3, lsr #24 @ constant for add/sub instructions
- teq r3, r6, lsl #24 @ must be 16MiB aligned
-THUMB( it ne @ cross section branch )
- bne __error
- str r3, [r7, #LOW_OFFSET] @ save to __pv_offset low bits
- b __fixup_a_pv_table
-ENDPROC(__fixup_pv_table)
-
- .align
-1: .long .
- .long __pv_table_begin
- .long __pv_table_end
-2: .long __pv_phys_pfn_offset
- .long __pv_offset
-
- .text
-__fixup_a_pv_table:
- adr r0, 3f
- ldr r6, [r0]
- add r6, r6, r3
- ldr r0, [r6, #HIGH_OFFSET] @ pv_offset high word
- ldr r6, [r6, #LOW_OFFSET] @ pv_offset low word
- mov r6, r6, lsr #24
- cmn r0, #1
-#ifdef CONFIG_THUMB2_KERNEL
- moveq r0, #0x200000 @ set bit 21, mov to mvn instruction
- lsls r6, #24
- beq 2f
- clz r7, r6
- lsr r6, #24
- lsl r6, r7
- bic r6, #0x0080
- lsrs r7, #1
- orrcs r6, #0x0080
- orr r6, r6, r7, lsl #12
- orr r6, #0x4000
- b 2f
-1: add r7, r3
- ldrh ip, [r7, #2]
-ARM_BE8(rev16 ip, ip)
- tst ip, #0x4000
- and ip, #0x8f00
- orrne ip, r6 @ mask in offset bits 31-24
- orreq ip, r0 @ mask in offset bits 7-0
-ARM_BE8(rev16 ip, ip)
- strh ip, [r7, #2]
- bne 2f
- ldrh ip, [r7]
-ARM_BE8(rev16 ip, ip)
- bic ip, #0x20
- orr ip, ip, r0, lsr #16
-ARM_BE8(rev16 ip, ip)
- strh ip, [r7]
-2: cmp r4, r5
- ldrcc r7, [r4], #4 @ use branch for delay slot
- bcc 1b
- bx lr
-#else
-#ifdef CONFIG_CPU_ENDIAN_BE8
- moveq r0, #0x00004000 @ set bit 22, mov to mvn instruction
-#else
- moveq r0, #0x400000 @ set bit 22, mov to mvn instruction
-#endif
- b 2f
-1: ldr ip, [r7, r3]
-#ifdef CONFIG_CPU_ENDIAN_BE8
- @ in BE8, we load data in BE, but instructions still in LE
- bic ip, ip, #0xff000000
- tst ip, #0x000f0000 @ check the rotation field
- orrne ip, ip, r6, lsl #24 @ mask in offset bits 31-24
- biceq ip, ip, #0x00004000 @ clear bit 22
- orreq ip, ip, r0 @ mask in offset bits 7-0
-#else
- bic ip, ip, #0x000000ff
- tst ip, #0xf00 @ check the rotation field
- orrne ip, ip, r6 @ mask in offset bits 31-24
- biceq ip, ip, #0x400000 @ clear bit 22
- orreq ip, ip, r0 @ mask in offset bits 7-0
-#endif
- str ip, [r7, r3]
-2: cmp r4, r5
- ldrcc r7, [r4], #4 @ use branch for delay slot
- bcc 1b
- ret lr
-#endif
-ENDPROC(__fixup_a_pv_table)
-
- .align
-3: .long __pv_offset
-
-ENTRY(fixup_pv_table)
- stmfd sp!, {r4 - r7, lr}
- mov r3, #0 @ no offset
- mov r4, r0 @ r0 = table start
- add r5, r0, r1 @ r1 = table size
- bl __fixup_a_pv_table
- ldmfd sp!, {r4 - r7, pc}
-ENDPROC(fixup_pv_table)
-
- .data
- .align 2
- .globl __pv_phys_pfn_offset
- .type __pv_phys_pfn_offset, %object
-__pv_phys_pfn_offset:
- .word 0
- .size __pv_phys_pfn_offset, . -__pv_phys_pfn_offset
-
- .globl __pv_offset
- .type __pv_offset, %object
-__pv_offset:
- .quad 0
- .size __pv_offset, . -__pv_offset
-#endif
-
#include "head-common.S"