summaryrefslogtreecommitdiff
path: root/arch/arm/mm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm/mm')
-rw-r--r--arch/arm/mm/copypage-fa.c35
-rw-r--r--arch/arm/mm/copypage-feroceon.c98
-rw-r--r--arch/arm/mm/copypage-v4mc.c19
-rw-r--r--arch/arm/mm/copypage-v4wb.c41
-rw-r--r--arch/arm/mm/copypage-v4wt.c37
-rw-r--r--arch/arm/mm/copypage-xsc3.c79
-rw-r--r--arch/arm/mm/copypage-xscale.c79
-rw-r--r--arch/arm/mm/fault.c6
-rw-r--r--arch/arm/mm/proc-macros.S10
-rw-r--r--arch/arm/mm/proc-v7-bugs.c17
-rw-r--r--arch/arm/mm/proc-v7.S2
-rw-r--r--arch/arm/mm/pv-fixup-asm.S16
12 files changed, 213 insertions, 226 deletions
diff --git a/arch/arm/mm/copypage-fa.c b/arch/arm/mm/copypage-fa.c
index d130a5ece5d5..bf24690ec83a 100644
--- a/arch/arm/mm/copypage-fa.c
+++ b/arch/arm/mm/copypage-fa.c
@@ -17,26 +17,25 @@
/*
* Faraday optimised copy_user_page
*/
-static void __naked
-fa_copy_user_page(void *kto, const void *kfrom)
+static void fa_copy_user_page(void *kto, const void *kfrom)
{
- asm("\
- stmfd sp!, {r4, lr} @ 2\n\
- mov r2, %0 @ 1\n\
-1: ldmia r1!, {r3, r4, ip, lr} @ 4\n\
- stmia r0, {r3, r4, ip, lr} @ 4\n\
- mcr p15, 0, r0, c7, c14, 1 @ 1 clean and invalidate D line\n\
- add r0, r0, #16 @ 1\n\
- ldmia r1!, {r3, r4, ip, lr} @ 4\n\
- stmia r0, {r3, r4, ip, lr} @ 4\n\
- mcr p15, 0, r0, c7, c14, 1 @ 1 clean and invalidate D line\n\
- add r0, r0, #16 @ 1\n\
- subs r2, r2, #1 @ 1\n\
+ int tmp;
+
+ asm volatile ("\
+1: ldmia %1!, {r3, r4, ip, lr} @ 4\n\
+ stmia %0, {r3, r4, ip, lr} @ 4\n\
+ mcr p15, 0, %0, c7, c14, 1 @ 1 clean and invalidate D line\n\
+ add %0, %0, #16 @ 1\n\
+ ldmia %1!, {r3, r4, ip, lr} @ 4\n\
+ stmia %0, {r3, r4, ip, lr} @ 4\n\
+ mcr p15, 0, %0, c7, c14, 1 @ 1 clean and invalidate D line\n\
+ add %0, %0, #16 @ 1\n\
+ subs %2, %2, #1 @ 1\n\
bne 1b @ 1\n\
- mcr p15, 0, r2, c7, c10, 4 @ 1 drain WB\n\
- ldmfd sp!, {r4, pc} @ 3"
- :
- : "I" (PAGE_SIZE / 32));
+ mcr p15, 0, %2, c7, c10, 4 @ 1 drain WB"
+ : "+&r" (kto), "+&r" (kfrom), "=&r" (tmp)
+ : "2" (PAGE_SIZE / 32)
+ : "r3", "r4", "ip", "lr");
}
void fa_copy_user_highpage(struct page *to, struct page *from,
diff --git a/arch/arm/mm/copypage-feroceon.c b/arch/arm/mm/copypage-feroceon.c
index 49ee0c1a7209..cc819732d9b8 100644
--- a/arch/arm/mm/copypage-feroceon.c
+++ b/arch/arm/mm/copypage-feroceon.c
@@ -13,58 +13,56 @@
#include <linux/init.h>
#include <linux/highmem.h>
-static void __naked
-feroceon_copy_user_page(void *kto, const void *kfrom)
+static void feroceon_copy_user_page(void *kto, const void *kfrom)
{
- asm("\
- stmfd sp!, {r4-r9, lr} \n\
- mov ip, %2 \n\
-1: mov lr, r1 \n\
- ldmia r1!, {r2 - r9} \n\
- pld [lr, #32] \n\
- pld [lr, #64] \n\
- pld [lr, #96] \n\
- pld [lr, #128] \n\
- pld [lr, #160] \n\
- pld [lr, #192] \n\
- pld [lr, #224] \n\
- stmia r0, {r2 - r9} \n\
- ldmia r1!, {r2 - r9} \n\
- mcr p15, 0, r0, c7, c14, 1 @ clean and invalidate D line\n\
- add r0, r0, #32 \n\
- stmia r0, {r2 - r9} \n\
- ldmia r1!, {r2 - r9} \n\
- mcr p15, 0, r0, c7, c14, 1 @ clean and invalidate D line\n\
- add r0, r0, #32 \n\
- stmia r0, {r2 - r9} \n\
- ldmia r1!, {r2 - r9} \n\
- mcr p15, 0, r0, c7, c14, 1 @ clean and invalidate D line\n\
- add r0, r0, #32 \n\
- stmia r0, {r2 - r9} \n\
- ldmia r1!, {r2 - r9} \n\
- mcr p15, 0, r0, c7, c14, 1 @ clean and invalidate D line\n\
- add r0, r0, #32 \n\
- stmia r0, {r2 - r9} \n\
- ldmia r1!, {r2 - r9} \n\
- mcr p15, 0, r0, c7, c14, 1 @ clean and invalidate D line\n\
- add r0, r0, #32 \n\
- stmia r0, {r2 - r9} \n\
- ldmia r1!, {r2 - r9} \n\
- mcr p15, 0, r0, c7, c14, 1 @ clean and invalidate D line\n\
- add r0, r0, #32 \n\
- stmia r0, {r2 - r9} \n\
- ldmia r1!, {r2 - r9} \n\
- mcr p15, 0, r0, c7, c14, 1 @ clean and invalidate D line\n\
- add r0, r0, #32 \n\
- stmia r0, {r2 - r9} \n\
- subs ip, ip, #(32 * 8) \n\
- mcr p15, 0, r0, c7, c14, 1 @ clean and invalidate D line\n\
- add r0, r0, #32 \n\
+ int tmp;
+
+ asm volatile ("\
+1: ldmia %1!, {r2 - r7, ip, lr} \n\
+ pld [%1, #0] \n\
+ pld [%1, #32] \n\
+ pld [%1, #64] \n\
+ pld [%1, #96] \n\
+ pld [%1, #128] \n\
+ pld [%1, #160] \n\
+ pld [%1, #192] \n\
+ stmia %0, {r2 - r7, ip, lr} \n\
+ ldmia %1!, {r2 - r7, ip, lr} \n\
+ mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\
+ add %0, %0, #32 \n\
+ stmia %0, {r2 - r7, ip, lr} \n\
+ ldmia %1!, {r2 - r7, ip, lr} \n\
+ mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\
+ add %0, %0, #32 \n\
+ stmia %0, {r2 - r7, ip, lr} \n\
+ ldmia %1!, {r2 - r7, ip, lr} \n\
+ mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\
+ add %0, %0, #32 \n\
+ stmia %0, {r2 - r7, ip, lr} \n\
+ ldmia %1!, {r2 - r7, ip, lr} \n\
+ mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\
+ add %0, %0, #32 \n\
+ stmia %0, {r2 - r7, ip, lr} \n\
+ ldmia %1!, {r2 - r7, ip, lr} \n\
+ mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\
+ add %0, %0, #32 \n\
+ stmia %0, {r2 - r7, ip, lr} \n\
+ ldmia %1!, {r2 - r7, ip, lr} \n\
+ mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\
+ add %0, %0, #32 \n\
+ stmia %0, {r2 - r7, ip, lr} \n\
+ ldmia %1!, {r2 - r7, ip, lr} \n\
+ mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\
+ add %0, %0, #32 \n\
+ stmia %0, {r2 - r7, ip, lr} \n\
+ subs %2, %2, #(32 * 8) \n\
+ mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\
+ add %0, %0, #32 \n\
bne 1b \n\
- mcr p15, 0, ip, c7, c10, 4 @ drain WB\n\
- ldmfd sp!, {r4-r9, pc}"
- :
- : "r" (kto), "r" (kfrom), "I" (PAGE_SIZE));
+ mcr p15, 0, %2, c7, c10, 4 @ drain WB"
+ : "+&r" (kto), "+&r" (kfrom), "=&r" (tmp)
+ : "2" (PAGE_SIZE)
+ : "r2", "r3", "r4", "r5", "r6", "r7", "ip", "lr");
}
void feroceon_copy_user_highpage(struct page *to, struct page *from,
diff --git a/arch/arm/mm/copypage-v4mc.c b/arch/arm/mm/copypage-v4mc.c
index 0224416cba3c..b03202cddddb 100644
--- a/arch/arm/mm/copypage-v4mc.c
+++ b/arch/arm/mm/copypage-v4mc.c
@@ -40,12 +40,11 @@ static DEFINE_RAW_SPINLOCK(minicache_lock);
* instruction. If your processor does not supply this, you have to write your
* own copy_user_highpage that does the right thing.
*/
-static void __naked
-mc_copy_user_page(void *from, void *to)
+static void mc_copy_user_page(void *from, void *to)
{
- asm volatile(
- "stmfd sp!, {r4, lr} @ 2\n\
- mov r4, %2 @ 1\n\
+ int tmp;
+
+ asm volatile ("\
ldmia %0!, {r2, r3, ip, lr} @ 4\n\
1: mcr p15, 0, %1, c7, c6, 1 @ 1 invalidate D line\n\
stmia %1!, {r2, r3, ip, lr} @ 4\n\
@@ -55,13 +54,13 @@ mc_copy_user_page(void *from, void *to)
mcr p15, 0, %1, c7, c6, 1 @ 1 invalidate D line\n\
stmia %1!, {r2, r3, ip, lr} @ 4\n\
ldmia %0!, {r2, r3, ip, lr} @ 4\n\
- subs r4, r4, #1 @ 1\n\
+ subs %2, %2, #1 @ 1\n\
stmia %1!, {r2, r3, ip, lr} @ 4\n\
ldmneia %0!, {r2, r3, ip, lr} @ 4\n\
- bne 1b @ 1\n\
- ldmfd sp!, {r4, pc} @ 3"
- :
- : "r" (from), "r" (to), "I" (PAGE_SIZE / 64));
+ bne 1b @ "
+ : "+&r" (from), "+&r" (to), "=&r" (tmp)
+ : "2" (PAGE_SIZE / 64)
+ : "r2", "r3", "ip", "lr");
}
void v4_mc_copy_user_highpage(struct page *to, struct page *from,
diff --git a/arch/arm/mm/copypage-v4wb.c b/arch/arm/mm/copypage-v4wb.c
index 067d0fdd630c..cd3e165afeed 100644
--- a/arch/arm/mm/copypage-v4wb.c
+++ b/arch/arm/mm/copypage-v4wb.c
@@ -22,29 +22,28 @@
* instruction. If your processor does not supply this, you have to write your
* own copy_user_highpage that does the right thing.
*/
-static void __naked
-v4wb_copy_user_page(void *kto, const void *kfrom)
+static void v4wb_copy_user_page(void *kto, const void *kfrom)
{
- asm("\
- stmfd sp!, {r4, lr} @ 2\n\
- mov r2, %2 @ 1\n\
- ldmia r1!, {r3, r4, ip, lr} @ 4\n\
-1: mcr p15, 0, r0, c7, c6, 1 @ 1 invalidate D line\n\
- stmia r0!, {r3, r4, ip, lr} @ 4\n\
- ldmia r1!, {r3, r4, ip, lr} @ 4+1\n\
- stmia r0!, {r3, r4, ip, lr} @ 4\n\
- ldmia r1!, {r3, r4, ip, lr} @ 4\n\
- mcr p15, 0, r0, c7, c6, 1 @ 1 invalidate D line\n\
- stmia r0!, {r3, r4, ip, lr} @ 4\n\
- ldmia r1!, {r3, r4, ip, lr} @ 4\n\
- subs r2, r2, #1 @ 1\n\
- stmia r0!, {r3, r4, ip, lr} @ 4\n\
- ldmneia r1!, {r3, r4, ip, lr} @ 4\n\
+ int tmp;
+
+ asm volatile ("\
+ ldmia %1!, {r3, r4, ip, lr} @ 4\n\
+1: mcr p15, 0, %0, c7, c6, 1 @ 1 invalidate D line\n\
+ stmia %0!, {r3, r4, ip, lr} @ 4\n\
+ ldmia %1!, {r3, r4, ip, lr} @ 4+1\n\
+ stmia %0!, {r3, r4, ip, lr} @ 4\n\
+ ldmia %1!, {r3, r4, ip, lr} @ 4\n\
+ mcr p15, 0, %0, c7, c6, 1 @ 1 invalidate D line\n\
+ stmia %0!, {r3, r4, ip, lr} @ 4\n\
+ ldmia %1!, {r3, r4, ip, lr} @ 4\n\
+ subs %2, %2, #1 @ 1\n\
+ stmia %0!, {r3, r4, ip, lr} @ 4\n\
+ ldmneia %1!, {r3, r4, ip, lr} @ 4\n\
bne 1b @ 1\n\
- mcr p15, 0, r1, c7, c10, 4 @ 1 drain WB\n\
- ldmfd sp!, {r4, pc} @ 3"
- :
- : "r" (kto), "r" (kfrom), "I" (PAGE_SIZE / 64));
+ mcr p15, 0, %1, c7, c10, 4 @ 1 drain WB"
+ : "+&r" (kto), "+&r" (kfrom), "=&r" (tmp)
+ : "2" (PAGE_SIZE / 64)
+ : "r3", "r4", "ip", "lr");
}
void v4wb_copy_user_highpage(struct page *to, struct page *from,
diff --git a/arch/arm/mm/copypage-v4wt.c b/arch/arm/mm/copypage-v4wt.c
index b85c5da2e510..8614572e1296 100644
--- a/arch/arm/mm/copypage-v4wt.c
+++ b/arch/arm/mm/copypage-v4wt.c
@@ -20,27 +20,26 @@
* dirty data in the cache. However, we do have to ensure that
* subsequent reads are up to date.
*/
-static void __naked
-v4wt_copy_user_page(void *kto, const void *kfrom)
+static void v4wt_copy_user_page(void *kto, const void *kfrom)
{
- asm("\
- stmfd sp!, {r4, lr} @ 2\n\
- mov r2, %2 @ 1\n\
- ldmia r1!, {r3, r4, ip, lr} @ 4\n\
-1: stmia r0!, {r3, r4, ip, lr} @ 4\n\
- ldmia r1!, {r3, r4, ip, lr} @ 4+1\n\
- stmia r0!, {r3, r4, ip, lr} @ 4\n\
- ldmia r1!, {r3, r4, ip, lr} @ 4\n\
- stmia r0!, {r3, r4, ip, lr} @ 4\n\
- ldmia r1!, {r3, r4, ip, lr} @ 4\n\
- subs r2, r2, #1 @ 1\n\
- stmia r0!, {r3, r4, ip, lr} @ 4\n\
- ldmneia r1!, {r3, r4, ip, lr} @ 4\n\
+ int tmp;
+
+ asm volatile ("\
+ ldmia %1!, {r3, r4, ip, lr} @ 4\n\
+1: stmia %0!, {r3, r4, ip, lr} @ 4\n\
+ ldmia %1!, {r3, r4, ip, lr} @ 4+1\n\
+ stmia %0!, {r3, r4, ip, lr} @ 4\n\
+ ldmia %1!, {r3, r4, ip, lr} @ 4\n\
+ stmia %0!, {r3, r4, ip, lr} @ 4\n\
+ ldmia %1!, {r3, r4, ip, lr} @ 4\n\
+ subs %2, %2, #1 @ 1\n\
+ stmia %0!, {r3, r4, ip, lr} @ 4\n\
+ ldmneia %1!, {r3, r4, ip, lr} @ 4\n\
bne 1b @ 1\n\
- mcr p15, 0, r2, c7, c7, 0 @ flush ID cache\n\
- ldmfd sp!, {r4, pc} @ 3"
- :
- : "r" (kto), "r" (kfrom), "I" (PAGE_SIZE / 64));
+ mcr p15, 0, %2, c7, c7, 0 @ flush ID cache"
+ : "+&r" (kto), "+&r" (kfrom), "=&r" (tmp)
+ : "2" (PAGE_SIZE / 64)
+ : "r3", "r4", "ip", "lr");
}
void v4wt_copy_user_highpage(struct page *to, struct page *from,
diff --git a/arch/arm/mm/copypage-xsc3.c b/arch/arm/mm/copypage-xsc3.c
index 03a2042aced5..a08158241ad1 100644
--- a/arch/arm/mm/copypage-xsc3.c
+++ b/arch/arm/mm/copypage-xsc3.c
@@ -21,53 +21,46 @@
/*
* XSC3 optimised copy_user_highpage
- * r0 = destination
- * r1 = source
*
* The source page may have some clean entries in the cache already, but we
* can safely ignore them - break_cow() will flush them out of the cache
* if we eventually end up using our copied page.
*
*/
-static void __naked
-xsc3_mc_copy_user_page(void *kto, const void *kfrom)
+static void xsc3_mc_copy_user_page(void *kto, const void *kfrom)
{
- asm("\
- stmfd sp!, {r4, r5, lr} \n\
- mov lr, %2 \n\
- \n\
- pld [r1, #0] \n\
- pld [r1, #32] \n\
-1: pld [r1, #64] \n\
- pld [r1, #96] \n\
+ int tmp;
+
+ asm volatile ("\
+ pld [%1, #0] \n\
+ pld [%1, #32] \n\
+1: pld [%1, #64] \n\
+ pld [%1, #96] \n\
\n\
-2: ldrd r2, [r1], #8 \n\
- mov ip, r0 \n\
- ldrd r4, [r1], #8 \n\
- mcr p15, 0, ip, c7, c6, 1 @ invalidate\n\
- strd r2, [r0], #8 \n\
- ldrd r2, [r1], #8 \n\
- strd r4, [r0], #8 \n\
- ldrd r4, [r1], #8 \n\
- strd r2, [r0], #8 \n\
- strd r4, [r0], #8 \n\
- ldrd r2, [r1], #8 \n\
- mov ip, r0 \n\
- ldrd r4, [r1], #8 \n\
- mcr p15, 0, ip, c7, c6, 1 @ invalidate\n\
- strd r2, [r0], #8 \n\
- ldrd r2, [r1], #8 \n\
- subs lr, lr, #1 \n\
- strd r4, [r0], #8 \n\
- ldrd r4, [r1], #8 \n\
- strd r2, [r0], #8 \n\
- strd r4, [r0], #8 \n\
+2: ldrd r2, r3, [%1], #8 \n\
+ ldrd r4, r5, [%1], #8 \n\
+ mcr p15, 0, %0, c7, c6, 1 @ invalidate\n\
+ strd r2, r3, [%0], #8 \n\
+ ldrd r2, r3, [%1], #8 \n\
+ strd r4, r5, [%0], #8 \n\
+ ldrd r4, r5, [%1], #8 \n\
+ strd r2, r3, [%0], #8 \n\
+ strd r4, r5, [%0], #8 \n\
+ ldrd r2, r3, [%1], #8 \n\
+ ldrd r4, r5, [%1], #8 \n\
+ mcr p15, 0, %0, c7, c6, 1 @ invalidate\n\
+ strd r2, r3, [%0], #8 \n\
+ ldrd r2, r3, [%1], #8 \n\
+ subs %2, %2, #1 \n\
+ strd r4, r5, [%0], #8 \n\
+ ldrd r4, r5, [%1], #8 \n\
+ strd r2, r3, [%0], #8 \n\
+ strd r4, r5, [%0], #8 \n\
bgt 1b \n\
- beq 2b \n\
- \n\
- ldmfd sp!, {r4, r5, pc}"
- :
- : "r" (kto), "r" (kfrom), "I" (PAGE_SIZE / 64 - 1));
+ beq 2b "
+ : "+&r" (kto), "+&r" (kfrom), "=&r" (tmp)
+ : "2" (PAGE_SIZE / 64 - 1)
+ : "r2", "r3", "r4", "r5");
}
void xsc3_mc_copy_user_highpage(struct page *to, struct page *from,
@@ -85,8 +78,6 @@ void xsc3_mc_copy_user_highpage(struct page *to, struct page *from,
/*
* XScale optimised clear_user_page
- * r0 = destination
- * r1 = virtual user address of ultimate destination page
*/
void xsc3_mc_clear_user_highpage(struct page *page, unsigned long vaddr)
{
@@ -96,10 +87,10 @@ void xsc3_mc_clear_user_highpage(struct page *page, unsigned long vaddr)
mov r2, #0 \n\
mov r3, #0 \n\
1: mcr p15, 0, %0, c7, c6, 1 @ invalidate line\n\
- strd r2, [%0], #8 \n\
- strd r2, [%0], #8 \n\
- strd r2, [%0], #8 \n\
- strd r2, [%0], #8 \n\
+ strd r2, r3, [%0], #8 \n\
+ strd r2, r3, [%0], #8 \n\
+ strd r2, r3, [%0], #8 \n\
+ strd r2, r3, [%0], #8 \n\
subs r1, r1, #1 \n\
bne 1b"
: "=r" (ptr)
diff --git a/arch/arm/mm/copypage-xscale.c b/arch/arm/mm/copypage-xscale.c
index 97972379f4d6..63b921936754 100644
--- a/arch/arm/mm/copypage-xscale.c
+++ b/arch/arm/mm/copypage-xscale.c
@@ -36,52 +36,51 @@ static DEFINE_RAW_SPINLOCK(minicache_lock);
* Dcache aliasing issue. The writes will be forwarded to the write buffer,
* and merged as appropriate.
*/
-static void __naked
-mc_copy_user_page(void *from, void *to)
+static void mc_copy_user_page(void *from, void *to)
{
+ int tmp;
+
/*
* Strangely enough, best performance is achieved
* when prefetching destination as well. (NP)
*/
- asm volatile(
- "stmfd sp!, {r4, r5, lr} \n\
- mov lr, %2 \n\
- pld [r0, #0] \n\
- pld [r0, #32] \n\
- pld [r1, #0] \n\
- pld [r1, #32] \n\
-1: pld [r0, #64] \n\
- pld [r0, #96] \n\
- pld [r1, #64] \n\
- pld [r1, #96] \n\
-2: ldrd r2, [r0], #8 \n\
- ldrd r4, [r0], #8 \n\
- mov ip, r1 \n\
- strd r2, [r1], #8 \n\
- ldrd r2, [r0], #8 \n\
- strd r4, [r1], #8 \n\
- ldrd r4, [r0], #8 \n\
- strd r2, [r1], #8 \n\
- strd r4, [r1], #8 \n\
+ asm volatile ("\
+ pld [%0, #0] \n\
+ pld [%0, #32] \n\
+ pld [%1, #0] \n\
+ pld [%1, #32] \n\
+1: pld [%0, #64] \n\
+ pld [%0, #96] \n\
+ pld [%1, #64] \n\
+ pld [%1, #96] \n\
+2: ldrd r2, r3, [%0], #8 \n\
+ ldrd r4, r5, [%0], #8 \n\
+ mov ip, %1 \n\
+ strd r2, r3, [%1], #8 \n\
+ ldrd r2, r3, [%0], #8 \n\
+ strd r4, r5, [%1], #8 \n\
+ ldrd r4, r5, [%0], #8 \n\
+ strd r2, r3, [%1], #8 \n\
+ strd r4, r5, [%1], #8 \n\
mcr p15, 0, ip, c7, c10, 1 @ clean D line\n\
- ldrd r2, [r0], #8 \n\
+ ldrd r2, r3, [%0], #8 \n\
mcr p15, 0, ip, c7, c6, 1 @ invalidate D line\n\
- ldrd r4, [r0], #8 \n\
- mov ip, r1 \n\
- strd r2, [r1], #8 \n\
- ldrd r2, [r0], #8 \n\
- strd r4, [r1], #8 \n\
- ldrd r4, [r0], #8 \n\
- strd r2, [r1], #8 \n\
- strd r4, [r1], #8 \n\
+ ldrd r4, r5, [%0], #8 \n\
+ mov ip, %1 \n\
+ strd r2, r3, [%1], #8 \n\
+ ldrd r2, r3, [%0], #8 \n\
+ strd r4, r5, [%1], #8 \n\
+ ldrd r4, r5, [%0], #8 \n\
+ strd r2, r3, [%1], #8 \n\
+ strd r4, r5, [%1], #8 \n\
mcr p15, 0, ip, c7, c10, 1 @ clean D line\n\
- subs lr, lr, #1 \n\
+ subs %2, %2, #1 \n\
mcr p15, 0, ip, c7, c6, 1 @ invalidate D line\n\
bgt 1b \n\
- beq 2b \n\
- ldmfd sp!, {r4, r5, pc} "
- :
- : "r" (from), "r" (to), "I" (PAGE_SIZE / 64 - 1));
+ beq 2b "
+ : "+&r" (from), "+&r" (to), "=&r" (tmp)
+ : "2" (PAGE_SIZE / 64 - 1)
+ : "r2", "r3", "r4", "r5", "ip");
}
void xscale_mc_copy_user_highpage(struct page *to, struct page *from,
@@ -115,10 +114,10 @@ xscale_mc_clear_user_highpage(struct page *page, unsigned long vaddr)
mov r2, #0 \n\
mov r3, #0 \n\
1: mov ip, %0 \n\
- strd r2, [%0], #8 \n\
- strd r2, [%0], #8 \n\
- strd r2, [%0], #8 \n\
- strd r2, [%0], #8 \n\
+ strd r2, r3, [%0], #8 \n\
+ strd r2, r3, [%0], #8 \n\
+ strd r2, r3, [%0], #8 \n\
+ strd r2, r3, [%0], #8 \n\
mcr p15, 0, ip, c7, c10, 1 @ clean D line\n\
subs r1, r1, #1 \n\
mcr p15, 0, ip, c7, c6, 1 @ invalidate D line\n\
diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c
index f4ea4c62c613..58f69fa07df9 100644
--- a/arch/arm/mm/fault.c
+++ b/arch/arm/mm/fault.c
@@ -173,6 +173,12 @@ __do_user_fault(struct task_struct *tsk, unsigned long addr,
show_regs(regs);
}
#endif
+#ifndef CONFIG_KUSER_HELPERS
+ if ((sig == SIGSEGV) && ((addr & PAGE_MASK) == 0xffff0000))
+ printk_ratelimited(KERN_DEBUG
+ "%s: CONFIG_KUSER_HELPERS disabled at 0x%08lx\n",
+ tsk->comm, addr);
+#endif
tsk->thread.address = addr;
tsk->thread.error_code = fsr;
diff --git a/arch/arm/mm/proc-macros.S b/arch/arm/mm/proc-macros.S
index 81d0efb055c6..5461d589a1e2 100644
--- a/arch/arm/mm/proc-macros.S
+++ b/arch/arm/mm/proc-macros.S
@@ -274,6 +274,13 @@
.endm
.macro define_processor_functions name:req, dabort:req, pabort:req, nommu=0, suspend=0, bugs=0
+/*
+ * If we are building for big.Little with branch predictor hardening,
+ * we need the processor function tables to remain available after boot.
+ */
+#if defined(CONFIG_BIG_LITTLE) && defined(CONFIG_HARDEN_BRANCH_PREDICTOR)
+ .section ".rodata"
+#endif
.type \name\()_processor_functions, #object
.align 2
ENTRY(\name\()_processor_functions)
@@ -309,6 +316,9 @@ ENTRY(\name\()_processor_functions)
.endif
.size \name\()_processor_functions, . - \name\()_processor_functions
+#if defined(CONFIG_BIG_LITTLE) && defined(CONFIG_HARDEN_BRANCH_PREDICTOR)
+ .previous
+#endif
.endm
.macro define_cache_functions name:req
diff --git a/arch/arm/mm/proc-v7-bugs.c b/arch/arm/mm/proc-v7-bugs.c
index 5544b82a2e7a..9a07916af8dd 100644
--- a/arch/arm/mm/proc-v7-bugs.c
+++ b/arch/arm/mm/proc-v7-bugs.c
@@ -52,8 +52,6 @@ static void cpu_v7_spectre_init(void)
case ARM_CPU_PART_CORTEX_A17:
case ARM_CPU_PART_CORTEX_A73:
case ARM_CPU_PART_CORTEX_A75:
- if (processor.switch_mm != cpu_v7_bpiall_switch_mm)
- goto bl_error;
per_cpu(harden_branch_predictor_fn, cpu) =
harden_branch_predictor_bpiall;
spectre_v2_method = "BPIALL";
@@ -61,8 +59,6 @@ static void cpu_v7_spectre_init(void)
case ARM_CPU_PART_CORTEX_A15:
case ARM_CPU_PART_BRAHMA_B15:
- if (processor.switch_mm != cpu_v7_iciallu_switch_mm)
- goto bl_error;
per_cpu(harden_branch_predictor_fn, cpu) =
harden_branch_predictor_iciallu;
spectre_v2_method = "ICIALLU";
@@ -88,11 +84,9 @@ static void cpu_v7_spectre_init(void)
ARM_SMCCC_ARCH_WORKAROUND_1, &res);
if ((int)res.a0 != 0)
break;
- if (processor.switch_mm != cpu_v7_hvc_switch_mm && cpu)
- goto bl_error;
per_cpu(harden_branch_predictor_fn, cpu) =
call_hvc_arch_workaround_1;
- processor.switch_mm = cpu_v7_hvc_switch_mm;
+ cpu_do_switch_mm = cpu_v7_hvc_switch_mm;
spectre_v2_method = "hypervisor";
break;
@@ -101,11 +95,9 @@ static void cpu_v7_spectre_init(void)
ARM_SMCCC_ARCH_WORKAROUND_1, &res);
if ((int)res.a0 != 0)
break;
- if (processor.switch_mm != cpu_v7_smc_switch_mm && cpu)
- goto bl_error;
per_cpu(harden_branch_predictor_fn, cpu) =
call_smc_arch_workaround_1;
- processor.switch_mm = cpu_v7_smc_switch_mm;
+ cpu_do_switch_mm = cpu_v7_smc_switch_mm;
spectre_v2_method = "firmware";
break;
@@ -119,11 +111,6 @@ static void cpu_v7_spectre_init(void)
if (spectre_v2_method)
pr_info("CPU%u: Spectre v2: using %s workaround\n",
smp_processor_id(), spectre_v2_method);
- return;
-
-bl_error:
- pr_err("CPU%u: Spectre v2: incorrect context switching function, system vulnerable\n",
- cpu);
}
#else
static void cpu_v7_spectre_init(void)
diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S
index 6fe52819e014..339eb17c9808 100644
--- a/arch/arm/mm/proc-v7.S
+++ b/arch/arm/mm/proc-v7.S
@@ -112,7 +112,7 @@ ENTRY(cpu_v7_hvc_switch_mm)
hvc #0
ldmfd sp!, {r0 - r3}
b cpu_v7_switch_mm
-ENDPROC(cpu_v7_smc_switch_mm)
+ENDPROC(cpu_v7_hvc_switch_mm)
#endif
ENTRY(cpu_v7_iciallu_switch_mm)
mov r3, #0
diff --git a/arch/arm/mm/pv-fixup-asm.S b/arch/arm/mm/pv-fixup-asm.S
index 1867f3e43016..fd2ff9034d17 100644
--- a/arch/arm/mm/pv-fixup-asm.S
+++ b/arch/arm/mm/pv-fixup-asm.S
@@ -33,10 +33,10 @@ ENTRY(lpae_pgtables_remap_asm)
add r7, r2, #0x1000
add r6, r7, r6, lsr #SECTION_SHIFT - L2_ORDER
add r7, r7, #PAGE_OFFSET >> (SECTION_SHIFT - L2_ORDER)
-1: ldrd r4, [r7]
+1: ldrd r4, r5, [r7]
adds r4, r4, r0
adc r5, r5, r1
- strd r4, [r7], #1 << L2_ORDER
+ strd r4, r5, [r7], #1 << L2_ORDER
cmp r7, r6
bls 1b
@@ -44,22 +44,22 @@ ENTRY(lpae_pgtables_remap_asm)
add r7, r2, #0x1000
add r7, r7, r3, lsr #SECTION_SHIFT - L2_ORDER
bic r7, r7, #(1 << L2_ORDER) - 1
- ldrd r4, [r7]
+ ldrd r4, r5, [r7]
adds r4, r4, r0
adc r5, r5, r1
- strd r4, [r7], #1 << L2_ORDER
- ldrd r4, [r7]
+ strd r4, r5, [r7], #1 << L2_ORDER
+ ldrd r4, r5, [r7]
adds r4, r4, r0
adc r5, r5, r1
- strd r4, [r7]
+ strd r4, r5, [r7]
/* Update level 1 entries */
mov r6, #4
mov r7, r2
-2: ldrd r4, [r7]
+2: ldrd r4, r5, [r7]
adds r4, r4, r0
adc r5, r5, r1
- strd r4, [r7], #1 << L1_ORDER
+ strd r4, r5, [r7], #1 << L1_ORDER
subs r6, r6, #1
bne 2b
generated by cgit (git 2.34.1) at 2024-06-04 15:21:22 +0000