summaryrefslogtreecommitdiff
path: root/arch/parisc/kernel/entry.S
diff options
context:
space:
mode:
Diffstat (limited to 'arch/parisc/kernel/entry.S')
-rw-r--r--arch/parisc/kernel/entry.S416
1 files changed, 168 insertions, 248 deletions
diff --git a/arch/parisc/kernel/entry.S b/arch/parisc/kernel/entry.S
index b96d74496977..ab23e61a6f01 100644
--- a/arch/parisc/kernel/entry.S
+++ b/arch/parisc/kernel/entry.S
@@ -19,15 +19,16 @@
#include <asm/psw.h>
#include <asm/cache.h> /* for L1_CACHE_SHIFT */
#include <asm/assembly.h> /* for LDREG/STREG defines */
-#include <asm/pgtable.h>
#include <asm/signal.h>
#include <asm/unistd.h>
#include <asm/ldcw.h>
#include <asm/traps.h>
#include <asm/thread_info.h>
#include <asm/alternative.h>
+#include <asm/spinlock_types.h>
#include <linux/linkage.h>
+#include <linux/pgtable.h>
#ifdef CONFIG_64BIT
.level 2.0w
@@ -35,10 +36,27 @@
.level 2.0
#endif
- .import pa_tlb_lock,data
- .macro load_pa_tlb_lock reg
- mfctl %cr25,\reg
- addil L%(PAGE_SIZE << (PGD_ALLOC_ORDER - 1)),\reg
+/*
+ * We need seven instructions after a TLB insert for it to take effect.
+ * The PA8800/PA8900 processors are an exception and need 12 instructions.
+ * The RFI changes both IAOQ_Back and IAOQ_Front, so it counts as one.
+ */
+#ifdef CONFIG_64BIT
+#define NUM_PIPELINE_INSNS 12
+#else
+#define NUM_PIPELINE_INSNS 7
+#endif
+
+ /* Insert num nops */
+ .macro insert_nops num
+ .rept \num
+ nop
+ .endr
+ .endm
+
+ /* Get aligned page_table_lock address for this mm from cr28/tr4 */
+ .macro get_ptl reg
+ mfctl %cr28,\reg
.endm
/* space_to_prot macro creates a prot id from a space id */
@@ -52,30 +70,6 @@
extrd,u \spc,(64 - (SPACEID_SHIFT)),32,\prot
.endm
#endif
-
- /* Switch to virtual mapping, trashing only %r1 */
- .macro virt_map
- /* pcxt_ssm_bug */
- rsm PSW_SM_I, %r0 /* barrier for "Relied upon Translation */
- mtsp %r0, %sr4
- mtsp %r0, %sr5
- mtsp %r0, %sr6
- tovirt_r1 %r29
- load32 KERNEL_PSW, %r1
-
- rsm PSW_SM_QUIET,%r0 /* second "heavy weight" ctl op */
- mtctl %r0, %cr17 /* Clear IIASQ tail */
- mtctl %r0, %cr17 /* Clear IIASQ head */
- mtctl %r1, %ipsw
- load32 4f, %r1
- mtctl %r1, %cr18 /* Set IIAOQ tail */
- ldo 4(%r1), %r1
- mtctl %r1, %cr18 /* Set IIAOQ head */
- rfir
- nop
-4:
- .endm
-
/*
* The "get_stack" macros are responsible for determining the
* kernel stack value.
@@ -88,8 +82,8 @@
* Need to set up a kernel stack, so call the
* get_stack_use_cr30 macro to set up a pointer
* to the pt_regs structure contained within the
- * task pointer pointed to by cr30. Set the stack
- * pointer to point to the end of the task structure.
+ * task pointer pointed to by cr30. Load the stack
+ * pointer from the task structure.
*
* Note that we use shadowed registers for temps until
* we can save %r26 and %r29. %r26 is used to preserve
@@ -101,8 +95,6 @@
* or handle_interruption. %r29 is used to hold a pointer
* the register save area, and once again, it needs to
* be a non-shadowed register so that it survives the rfir.
- *
- * N.B. TASK_SZ_ALGN and PT_SZ_ALGN include space for a stack frame.
*/
.macro get_stack_use_cr30
@@ -111,12 +103,11 @@
copy %r30, %r17
mfctl %cr30, %r1
- ldo THREAD_SZ_ALGN(%r1), %r30
- mtsp %r0,%sr7
+ tophys %r1,%r9 /* task_struct */
+ LDREG TASK_STACK(%r9),%r30
+ ldo PT_SZ_ALGN(%r30),%r30
+ mtsp %r0,%sr7 /* clear sr7 after kernel stack was set! */
mtsp %r16,%sr3
- tophys %r1,%r9
- LDREG TI_TASK(%r9), %r1 /* thread_info -> task_struct */
- tophys %r1,%r9
ldo TASK_REGS(%r9),%r9
STREG %r17,PT_GR30(%r9)
STREG %r29,PT_GR29(%r9)
@@ -394,69 +385,47 @@
*/
.macro L2_ptep pmd,pte,index,va,fault
#if CONFIG_PGTABLE_LEVELS == 3
- extru \va,31-ASM_PMD_SHIFT,ASM_BITS_PER_PMD,\index
+ extru_safe \va,31-ASM_PMD_SHIFT,ASM_BITS_PER_PMD,\index
#else
-# if defined(CONFIG_64BIT)
- extrd,u \va,63-ASM_PGDIR_SHIFT,ASM_BITS_PER_PGD,\index
- #else
- # if PAGE_SIZE > 4096
- extru \va,31-ASM_PGDIR_SHIFT,32-ASM_PGDIR_SHIFT,\index
- # else
- extru \va,31-ASM_PGDIR_SHIFT,ASM_BITS_PER_PGD,\index
- # endif
-# endif
+ extru_safe \va,31-ASM_PGDIR_SHIFT,ASM_BITS_PER_PGD,\index
#endif
dep %r0,31,PAGE_SHIFT,\pmd /* clear offset */
+#if CONFIG_PGTABLE_LEVELS < 3
copy %r0,\pte
+#endif
ldw,s \index(\pmd),\pmd
bb,>=,n \pmd,_PxD_PRESENT_BIT,\fault
dep %r0,31,PxD_FLAG_SHIFT,\pmd /* clear flags */
SHLREG \pmd,PxD_VALUE_SHIFT,\pmd
- extru \va,31-PAGE_SHIFT,ASM_BITS_PER_PTE,\index
+ extru_safe \va,31-PAGE_SHIFT,ASM_BITS_PER_PTE,\index
dep %r0,31,PAGE_SHIFT,\pmd /* clear offset */
shladd \index,BITS_PER_PTE_ENTRY,\pmd,\pmd /* pmd is now pte */
.endm
- /* Look up PTE in a 3-Level scheme.
- *
- * Here we implement a Hybrid L2/L3 scheme: we allocate the
- * first pmd adjacent to the pgd. This means that we can
- * subtract a constant offset to get to it. The pmd and pgd
- * sizes are arranged so that a single pmd covers 4GB (giving
- * a full LP64 process access to 8TB) so our lookups are
- * effectively L2 for the first 4GB of the kernel (i.e. for
- * all ILP32 processes and all the kernel for machines with
- * under 4GB of memory) */
+ /* Look up PTE in a 3-Level scheme. */
.macro L3_ptep pgd,pte,index,va,fault
-#if CONFIG_PGTABLE_LEVELS == 3 /* we might have a 2-Level scheme, e.g. with 16kb page size */
+#if CONFIG_PGTABLE_LEVELS == 3
+ copy %r0,\pte
extrd,u \va,63-ASM_PGDIR_SHIFT,ASM_BITS_PER_PGD,\index
- extrd,u,*= \va,63-ASM_PGDIR_SHIFT,64-ASM_PGDIR_SHIFT,%r0
ldw,s \index(\pgd),\pgd
- extrd,u,*= \va,63-ASM_PGDIR_SHIFT,64-ASM_PGDIR_SHIFT,%r0
bb,>=,n \pgd,_PxD_PRESENT_BIT,\fault
- extrd,u,*= \va,63-ASM_PGDIR_SHIFT,64-ASM_PGDIR_SHIFT,%r0
- shld \pgd,PxD_VALUE_SHIFT,\index
- extrd,u,*= \va,63-ASM_PGDIR_SHIFT,64-ASM_PGDIR_SHIFT,%r0
- copy \index,\pgd
- extrd,u,*<> \va,63-ASM_PGDIR_SHIFT,64-ASM_PGDIR_SHIFT,%r0
- ldo ASM_PGD_PMD_OFFSET(\pgd),\pgd
+ shld \pgd,PxD_VALUE_SHIFT,\pgd
#endif
L2_ptep \pgd,\pte,\index,\va,\fault
.endm
- /* Acquire pa_tlb_lock lock and check page is present. */
- .macro tlb_lock spc,ptp,pte,tmp,tmp1,fault
-#ifdef CONFIG_SMP
+ /* Acquire page_table_lock and check page is present. */
+ .macro ptl_lock spc,ptp,pte,tmp,tmp1,fault
+#ifdef CONFIG_TLB_PTLOCK
98: cmpib,COND(=),n 0,\spc,2f
- load_pa_tlb_lock \tmp
+ get_ptl \tmp
1: LDCW 0(\tmp),\tmp1
cmpib,COND(=) 0,\tmp1,1b
nop
LDREG 0(\ptp),\pte
bb,<,n \pte,_PAGE_PRESENT_BIT,3f
- LDCW 0(\tmp),\tmp1
b \fault
- stw \spc,0(\tmp)
+ stw \tmp1,0(\tmp)
99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP)
#endif
2: LDREG 0(\ptp),\pte
@@ -464,23 +433,20 @@
3:
.endm
- /* Release pa_tlb_lock lock without reloading lock address. */
- .macro tlb_unlock0 spc,tmp,tmp1
-#ifdef CONFIG_SMP
-98: or,COND(=) %r0,\spc,%r0
- LDCW 0(\tmp),\tmp1
+ /* Release page_table_lock if for user space. We use an ordered
+ store to ensure all prior accesses are performed prior to
+ releasing the lock. Note stw may not be executed, so we
+ provide one extra nop when CONFIG_TLB_PTLOCK is defined. */
+ .macro ptl_unlock spc,tmp,tmp2
+#ifdef CONFIG_TLB_PTLOCK
+98: get_ptl \tmp
+ ldi __ARCH_SPIN_LOCK_UNLOCKED_VAL, \tmp2
or,COND(=) %r0,\spc,%r0
- stw \spc,0(\tmp)
+ stw,ma \tmp2,0(\tmp)
99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP)
-#endif
- .endm
-
- /* Release pa_tlb_lock lock. */
- .macro tlb_unlock1 spc,tmp,tmp1
-#ifdef CONFIG_SMP
-98: load_pa_tlb_lock \tmp
-99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP)
- tlb_unlock0 \spc,\tmp,\tmp1
+ insert_nops NUM_PIPELINE_INSNS - 4
+#else
+ insert_nops NUM_PIPELINE_INSNS - 1
#endif
.endm
@@ -509,13 +475,13 @@
* to a CPU TLB 4k PFN (4k => 12 bits to shift) */
#define PAGE_ADD_SHIFT (PAGE_SHIFT-12)
#define PAGE_ADD_HUGE_SHIFT (REAL_HPAGE_SHIFT-12)
+ #define PFN_START_BIT (63-ASM_PFN_PTE_SHIFT+(63-58)-PAGE_ADD_SHIFT)
/* Drop prot bits and convert to page addr for iitlbt and idtlbt */
.macro convert_for_tlb_insert20 pte,tmp
#ifdef CONFIG_HUGETLB_PAGE
copy \pte,\tmp
- extrd,u \tmp,(63-ASM_PFN_PTE_SHIFT)+(63-58)+PAGE_ADD_SHIFT,\
- 64-PAGE_SHIFT-PAGE_ADD_SHIFT,\pte
+ extrd,u \tmp,PFN_START_BIT,PFN_START_BIT+1,\pte
depdi _PAGE_SIZE_ENCODING_DEFAULT,63,\
(63-58)+PAGE_ADD_SHIFT,\pte
@@ -523,8 +489,7 @@
depdi _HUGE_PAGE_SIZE_ENCODING_DEFAULT,63,\
(63-58)+PAGE_ADD_HUGE_SHIFT,\pte
#else /* Huge pages disabled */
- extrd,u \pte,(63-ASM_PFN_PTE_SHIFT)+(63-58)+PAGE_ADD_SHIFT,\
- 64-PAGE_SHIFT-PAGE_ADD_SHIFT,\pte
+ extrd,u \pte,PFN_START_BIT,PFN_START_BIT+1,\pte
depdi _PAGE_SIZE_ENCODING_DEFAULT,63,\
(63-58)+PAGE_ADD_SHIFT,\pte
#endif
@@ -546,6 +511,10 @@
* Finally, _PAGE_READ goes in the top bit of PL1 (so we
* trigger an access rights trap in user space if the user
* tries to read an unreadable page */
+#if _PAGE_SPECIAL_BIT == _PAGE_DMB_BIT
+ /* need to drop DMB bit, as it's used as SPECIAL flag */
+ depi 0,_PAGE_SPECIAL_BIT,1,\pte
+#endif
depd \pte,8,7,\prot
/* PAGE_USER indicates the page can be read with user privileges,
@@ -576,6 +545,10 @@
* makes the tlb entry for the differently formatted pa11
* insertion instructions */
.macro make_insert_tlb_11 spc,pte,prot
+#if _PAGE_SPECIAL_BIT == _PAGE_DMB_BIT
+ /* need to drop DMB bit, as it's used as SPECIAL flag */
+ depi 0,_PAGE_SPECIAL_BIT,1,\pte
+#endif
zdep \spc,30,15,\prot
dep \pte,8,7,\prot
extru,= \pte,_PAGE_NO_CACHE_BIT,1,%r0
@@ -601,8 +574,9 @@
extrd,s \pte,63,25,\pte
.endm
- /* The alias region is an 8MB aligned 16MB to do clear and
- * copy user pages at addresses congruent with the user
+ /* The alias region is comprised of a pair of 4 MB regions
+ * aligned to 8 MB. It is used to clear/copy/flush user pages
+ * using kernel virtual addresses congruent with the user
* virtual address.
*
* To use the alias page, you set %r26 up with the to TLB
@@ -612,13 +586,8 @@
.macro do_alias spc,tmp,tmp1,va,pte,prot,fault,patype
cmpib,COND(<>),n 0,\spc,\fault
ldil L%(TMPALIAS_MAP_START),\tmp
-#if defined(CONFIG_64BIT) && (TMPALIAS_MAP_START >= 0x80000000)
- /* on LP64, ldi will sign extend into the upper 32 bits,
- * which is behaviour we don't want */
- depdi 0,31,32,\tmp
-#endif
copy \va,\tmp1
- depi 0,31,23,\tmp1
+ depi_safe 0,31,TMPALIAS_SIZE_BITS+1,\tmp1
cmpb,COND(<>),n \tmp,\tmp1,\fault
mfctl %cr19,\tmp /* iir */
/* get the opcode (first six bits) into \tmp */
@@ -651,13 +620,13 @@
* OK, it is in the temp alias region, check whether "from" or "to".
* Check "subtle" note in pacache.S re: r23/r26.
*/
-#ifdef CONFIG_64BIT
- extrd,u,*= \va,41,1,%r0
-#else
- extrw,u,= \va,9,1,%r0
-#endif
+ extrw,u,= \va,31-TMPALIAS_SIZE_BITS,1,%r0
or,COND(tr) %r23,%r0,\pte
or %r26,%r0,\pte
+
+ /* convert phys addr in \pte (from r23 or r26) to tlb insert format */
+ SHRREG \pte,PAGE_SHIFT+PAGE_ADD_SHIFT-5, \pte
+ depi_safe _PAGE_SIZE_ENCODING_DEFAULT, 31,5, \pte
.endm
@@ -769,7 +738,7 @@ ENTRY(ret_from_kernel_thread)
BL schedule_tail, %r2
nop
- LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30), %r1
+ mfctl %cr30,%r1 /* task_struct */
LDREG TASK_PT_GR25(%r1), %r26
#ifdef CONFIG_64BIT
LDREG TASK_PT_GR27(%r1), %r27
@@ -800,7 +769,6 @@ ENTRY_CFI(_switch_to)
STREG %r30, TASK_PT_KSP(%r26)
LDREG TASK_PT_KSP(%r25), %r30
- LDREG TASK_THREAD_INFO(%r25), %r25
bv %r0(%r2)
mtctl %r25,%cr30
@@ -831,17 +799,16 @@ ENDPROC_CFI(_switch_to)
.align PAGE_SIZE
ENTRY_CFI(syscall_exit_rfi)
- mfctl %cr30,%r16
- LDREG TI_TASK(%r16), %r16 /* thread_info -> task_struct */
+ mfctl %cr30,%r16 /* task_struct */
ldo TASK_REGS(%r16),%r16
/* Force iaoq to userspace, as the user has had access to our current
* context via sigcontext. Also Filter the PSW for the same reason.
*/
LDREG PT_IAOQ0(%r16),%r19
- depi 3,31,2,%r19
+ depi PRIV_USER,31,2,%r19
STREG %r19,PT_IAOQ0(%r16)
LDREG PT_IAOQ1(%r16),%r19
- depi 3,31,2,%r19
+ depi PRIV_USER,31,2,%r19
STREG %r19,PT_IAOQ1(%r16)
LDREG PT_PSW(%r16),%r19
load32 USER_PSW_MASK,%r1
@@ -877,15 +844,15 @@ ENTRY_CFI(syscall_exit_rfi)
ENTRY(intr_return)
/* check for reschedule */
mfctl %cr30,%r1
- LDREG TI_FLAGS(%r1),%r19 /* sched.h: TIF_NEED_RESCHED */
+ LDREG TASK_TI_FLAGS(%r1),%r19 /* sched.h: TIF_NEED_RESCHED */
bb,<,n %r19,31-TIF_NEED_RESCHED,intr_do_resched /* forward */
.import do_notify_resume,code
intr_check_sig:
/* As above */
mfctl %cr30,%r1
- LDREG TI_FLAGS(%r1),%r19
- ldi (_TIF_SIGPENDING|_TIF_NOTIFY_RESUME), %r20
+ LDREG TASK_TI_FLAGS(%r1),%r19
+ ldi (_TIF_USER_WORK_MASK & ~_TIF_NEED_RESCHED), %r20
and,COND(<>) %r19, %r20, %r0
b,n intr_restore /* skip past if we've nothing to do */
@@ -897,20 +864,20 @@ intr_check_sig:
* Only do signals if we are returning to user space
*/
LDREG PT_IASQ0(%r16), %r20
- cmpib,COND(=),n LINUX_GATEWAY_SPACE, %r20, intr_restore /* backward */
+ cmpib,COND(=),n LINUX_GATEWAY_SPACE, %r20, intr_restore /* forward */
LDREG PT_IASQ1(%r16), %r20
- cmpib,COND(=),n LINUX_GATEWAY_SPACE, %r20, intr_restore /* backward */
-
- /* NOTE: We need to enable interrupts if we have to deliver
- * signals. We used to do this earlier but it caused kernel
- * stack overflows. */
- ssm PSW_SM_I, %r0
+ cmpib,COND(=),n LINUX_GATEWAY_SPACE, %r20, intr_restore /* forward */
copy %r0, %r25 /* long in_syscall = 0 */
#ifdef CONFIG_64BIT
ldo -16(%r30),%r29 /* Reference param save area */
#endif
+ /* NOTE: We need to enable interrupts if we have to deliver
+ * signals. We used to do this earlier but it caused kernel
+ * stack overflows. */
+ ssm PSW_SM_I, %r0
+
BL do_notify_resume,%r2
copy %r16, %r26 /* struct pt_regs *regs */
@@ -940,14 +907,14 @@ intr_restore:
rfi
nop
-#ifndef CONFIG_PREEMPT
+#ifndef CONFIG_PREEMPTION
# define intr_do_preempt intr_restore
-#endif /* !CONFIG_PREEMPT */
+#endif /* !CONFIG_PREEMPTION */
.import schedule,code
intr_do_resched:
/* Only call schedule on return to userspace. If we're returning
- * to kernel space, we may schedule if CONFIG_PREEMPT, otherwise
+ * to kernel space, we may schedule if CONFIG_PREEMPTION, otherwise
* we jump back to intr_restore.
*/
LDREG PT_IASQ0(%r16), %r20
@@ -979,15 +946,15 @@ intr_do_resched:
* and preempt_count is 0. otherwise, we continue on
* our merry way back to the current running task.
*/
-#ifdef CONFIG_PREEMPT
+#ifdef CONFIG_PREEMPTION
.import preempt_schedule_irq,code
intr_do_preempt:
rsm PSW_SM_I, %r0 /* disable interrupts */
/* current_thread_info()->preempt_count */
mfctl %cr30, %r1
- LDREG TI_PRE_COUNT(%r1), %r19
- cmpib,COND(<>) 0, %r19, intr_restore /* if preempt_count > 0 */
+ ldw TI_PRE_COUNT(%r1), %r19
+ cmpib,<> 0, %r19, intr_restore /* if preempt_count > 0 */
nop /* prev insn branched backwards */
/* check if we interrupted a critical path */
@@ -995,11 +962,18 @@ intr_do_preempt:
bb,<,n %r20, 31 - PSW_SM_I, intr_restore
nop
+ /* ssm PSW_SM_I done later in intr_restore */
+#ifdef CONFIG_MLONGCALLS
+ ldil L%intr_restore, %r2
+ load32 preempt_schedule_irq, %r1
+ bv %r0(%r1)
+ ldo R%intr_restore(%r2), %r2
+#else
+ ldil L%intr_restore, %r1
BL preempt_schedule_irq, %r2
- nop
-
- b,n intr_restore /* ssm PSW_SM_I done by intr_restore */
-#endif /* CONFIG_PREEMPT */
+ ldo R%intr_restore(%r1), %r2
+#endif
+#endif /* CONFIG_PREEMPTION */
/*
* External interrupts.
@@ -1156,14 +1130,14 @@ dtlb_miss_20w:
L3_ptep ptp,pte,t0,va,dtlb_check_alias_20w
- tlb_lock spc,ptp,pte,t0,t1,dtlb_check_alias_20w
+ ptl_lock spc,ptp,pte,t0,t1,dtlb_check_alias_20w
update_accessed ptp,pte,t0,t1
make_insert_tlb spc,pte,prot,t1
idtlbt pte,prot
- tlb_unlock1 spc,t0,t1
+ ptl_unlock spc,t0,t1
rfir
nop
@@ -1172,6 +1146,7 @@ dtlb_check_alias_20w:
idtlbt pte,prot
+ insert_nops NUM_PIPELINE_INSNS - 1
rfir
nop
@@ -1182,14 +1157,14 @@ nadtlb_miss_20w:
L3_ptep ptp,pte,t0,va,nadtlb_check_alias_20w
- tlb_lock spc,ptp,pte,t0,t1,nadtlb_check_alias_20w
+ ptl_lock spc,ptp,pte,t0,t1,nadtlb_check_alias_20w
update_accessed ptp,pte,t0,t1
make_insert_tlb spc,pte,prot,t1
idtlbt pte,prot
- tlb_unlock1 spc,t0,t1
+ ptl_unlock spc,t0,t1
rfir
nop
@@ -1198,6 +1173,7 @@ nadtlb_check_alias_20w:
idtlbt pte,prot
+ insert_nops NUM_PIPELINE_INSNS - 1
rfir
nop
@@ -1210,7 +1186,7 @@ dtlb_miss_11:
L2_ptep ptp,pte,t0,va,dtlb_check_alias_11
- tlb_lock spc,ptp,pte,t0,t1,dtlb_check_alias_11
+ ptl_lock spc,ptp,pte,t0,t1,dtlb_check_alias_11
update_accessed ptp,pte,t0,t1
make_insert_tlb_11 spc,pte,prot
@@ -1223,7 +1199,7 @@ dtlb_miss_11:
mtsp t1, %sr1 /* Restore sr1 */
- tlb_unlock1 spc,t0,t1
+ ptl_unlock spc,t0,t1
rfir
nop
@@ -1233,6 +1209,7 @@ dtlb_check_alias_11:
idtlba pte,(va)
idtlbp prot,(va)
+ insert_nops NUM_PIPELINE_INSNS - 1
rfir
nop
@@ -1243,7 +1220,7 @@ nadtlb_miss_11:
L2_ptep ptp,pte,t0,va,nadtlb_check_alias_11
- tlb_lock spc,ptp,pte,t0,t1,nadtlb_check_alias_11
+ ptl_lock spc,ptp,pte,t0,t1,nadtlb_check_alias_11
update_accessed ptp,pte,t0,t1
make_insert_tlb_11 spc,pte,prot
@@ -1256,7 +1233,7 @@ nadtlb_miss_11:
mtsp t1, %sr1 /* Restore sr1 */
- tlb_unlock1 spc,t0,t1
+ ptl_unlock spc,t0,t1
rfir
nop
@@ -1266,6 +1243,7 @@ nadtlb_check_alias_11:
idtlba pte,(va)
idtlbp prot,(va)
+ insert_nops NUM_PIPELINE_INSNS - 1
rfir
nop
@@ -1276,7 +1254,7 @@ dtlb_miss_20:
L2_ptep ptp,pte,t0,va,dtlb_check_alias_20
- tlb_lock spc,ptp,pte,t0,t1,dtlb_check_alias_20
+ ptl_lock spc,ptp,pte,t0,t1,dtlb_check_alias_20
update_accessed ptp,pte,t0,t1
make_insert_tlb spc,pte,prot,t1
@@ -1285,7 +1263,7 @@ dtlb_miss_20:
idtlbt pte,prot
- tlb_unlock1 spc,t0,t1
+ ptl_unlock spc,t0,t1
rfir
nop
@@ -1294,6 +1272,7 @@ dtlb_check_alias_20:
idtlbt pte,prot
+ insert_nops NUM_PIPELINE_INSNS - 1
rfir
nop
@@ -1304,7 +1283,7 @@ nadtlb_miss_20:
L2_ptep ptp,pte,t0,va,nadtlb_check_alias_20
- tlb_lock spc,ptp,pte,t0,t1,nadtlb_check_alias_20
+ ptl_lock spc,ptp,pte,t0,t1,nadtlb_check_alias_20
update_accessed ptp,pte,t0,t1
make_insert_tlb spc,pte,prot,t1
@@ -1313,7 +1292,7 @@ nadtlb_miss_20:
idtlbt pte,prot
- tlb_unlock1 spc,t0,t1
+ ptl_unlock spc,t0,t1
rfir
nop
@@ -1322,6 +1301,7 @@ nadtlb_check_alias_20:
idtlbt pte,prot
+ insert_nops NUM_PIPELINE_INSNS - 1
rfir
nop
@@ -1330,74 +1310,12 @@ nadtlb_check_alias_20:
nadtlb_emulate:
/*
- * Non access misses can be caused by fdc,fic,pdc,lpa,probe and
- * probei instructions. We don't want to fault for these
- * instructions (not only does it not make sense, it can cause
- * deadlocks, since some flushes are done with the mmap
- * semaphore held). If the translation doesn't exist, we can't
- * insert a translation, so have to emulate the side effects
- * of the instruction. Since we don't insert a translation
- * we can get a lot of faults during a flush loop, so it makes
- * sense to try to do it here with minimum overhead. We only
- * emulate fdc,fic,pdc,probew,prober instructions whose base
- * and index registers are not shadowed. We defer everything
- * else to the "slow" path.
+ * Non-access misses can be caused by fdc,fic,pdc,lpa,probe and
+ * probei instructions. The kernel no longer faults doing flushes.
+ * Use of lpa and probe instructions is rare. Given the issue
+ * with shadow registers, we defer everything to the "slow" path.
*/
-
- mfctl %cr19,%r9 /* Get iir */
-
- /* PA 2.0 Arch Ref. Book pg 382 has a good description of the insn bits.
- Checks for fdc,fdce,pdc,"fic,4f",prober,probeir,probew, probeiw */
-
- /* Checks for fdc,fdce,pdc,"fic,4f" only */
- ldi 0x280,%r16
- and %r9,%r16,%r17
- cmpb,<>,n %r16,%r17,nadtlb_probe_check
- bb,>=,n %r9,26,nadtlb_nullify /* m bit not set, just nullify */
- BL get_register,%r25
- extrw,u %r9,15,5,%r8 /* Get index register # */
- cmpib,COND(=),n -1,%r1,nadtlb_fault /* have to use slow path */
- copy %r1,%r24
- BL get_register,%r25
- extrw,u %r9,10,5,%r8 /* Get base register # */
- cmpib,COND(=),n -1,%r1,nadtlb_fault /* have to use slow path */
- BL set_register,%r25
- add,l %r1,%r24,%r1 /* doesn't affect c/b bits */
-
-nadtlb_nullify:
- mfctl %ipsw,%r8
- ldil L%PSW_N,%r9
- or %r8,%r9,%r8 /* Set PSW_N */
- mtctl %r8,%ipsw
-
- rfir
- nop
-
- /*
- When there is no translation for the probe address then we
- must nullify the insn and return zero in the target register.
- This will indicate to the calling code that it does not have
- write/read privileges to this address.
-
- This should technically work for prober and probew in PA 1.1,
- and also probe,r and probe,w in PA 2.0
-
- WARNING: USE ONLY NON-SHADOW REGISTERS WITH PROBE INSN!
- THE SLOW-PATH EMULATION HAS NOT BEEN WRITTEN YET.
-
- */
-nadtlb_probe_check:
- ldi 0x80,%r16
- and %r9,%r16,%r17
- cmpb,<>,n %r16,%r17,nadtlb_fault /* Must be probe,[rw]*/
- BL get_register,%r25 /* Find the target register */
- extrw,u %r9,31,5,%r8 /* Get target register */
- cmpib,COND(=),n -1,%r1,nadtlb_fault /* have to use slow path */
- BL set_register,%r25
- copy %r0,%r1 /* Write zero to target register */
- b nadtlb_nullify /* Nullify return insn */
- nop
-
+ b,n nadtlb_fault
#ifdef CONFIG_64BIT
itlb_miss_20w:
@@ -1413,14 +1331,14 @@ itlb_miss_20w:
L3_ptep ptp,pte,t0,va,itlb_fault
- tlb_lock spc,ptp,pte,t0,t1,itlb_fault
+ ptl_lock spc,ptp,pte,t0,t1,itlb_fault
update_accessed ptp,pte,t0,t1
make_insert_tlb spc,pte,prot,t1
iitlbt pte,prot
- tlb_unlock1 spc,t0,t1
+ ptl_unlock spc,t0,t1
rfir
nop
@@ -1437,14 +1355,14 @@ naitlb_miss_20w:
L3_ptep ptp,pte,t0,va,naitlb_check_alias_20w
- tlb_lock spc,ptp,pte,t0,t1,naitlb_check_alias_20w
+ ptl_lock spc,ptp,pte,t0,t1,naitlb_check_alias_20w
update_accessed ptp,pte,t0,t1
make_insert_tlb spc,pte,prot,t1
iitlbt pte,prot
- tlb_unlock1 spc,t0,t1
+ ptl_unlock spc,t0,t1
rfir
nop
@@ -1453,6 +1371,7 @@ naitlb_check_alias_20w:
iitlbt pte,prot
+ insert_nops NUM_PIPELINE_INSNS - 1
rfir
nop
@@ -1465,7 +1384,7 @@ itlb_miss_11:
L2_ptep ptp,pte,t0,va,itlb_fault
- tlb_lock spc,ptp,pte,t0,t1,itlb_fault
+ ptl_lock spc,ptp,pte,t0,t1,itlb_fault
update_accessed ptp,pte,t0,t1
make_insert_tlb_11 spc,pte,prot
@@ -1478,7 +1397,7 @@ itlb_miss_11:
mtsp t1, %sr1 /* Restore sr1 */
- tlb_unlock1 spc,t0,t1
+ ptl_unlock spc,t0,t1
rfir
nop
@@ -1489,7 +1408,7 @@ naitlb_miss_11:
L2_ptep ptp,pte,t0,va,naitlb_check_alias_11
- tlb_lock spc,ptp,pte,t0,t1,naitlb_check_alias_11
+ ptl_lock spc,ptp,pte,t0,t1,naitlb_check_alias_11
update_accessed ptp,pte,t0,t1
make_insert_tlb_11 spc,pte,prot
@@ -1502,7 +1421,7 @@ naitlb_miss_11:
mtsp t1, %sr1 /* Restore sr1 */
- tlb_unlock1 spc,t0,t1
+ ptl_unlock spc,t0,t1
rfir
nop
@@ -1512,6 +1431,7 @@ naitlb_check_alias_11:
iitlba pte,(%sr0, va)
iitlbp prot,(%sr0, va)
+ insert_nops NUM_PIPELINE_INSNS - 1
rfir
nop
@@ -1523,7 +1443,7 @@ itlb_miss_20:
L2_ptep ptp,pte,t0,va,itlb_fault
- tlb_lock spc,ptp,pte,t0,t1,itlb_fault
+ ptl_lock spc,ptp,pte,t0,t1,itlb_fault
update_accessed ptp,pte,t0,t1
make_insert_tlb spc,pte,prot,t1
@@ -1532,7 +1452,7 @@ itlb_miss_20:
iitlbt pte,prot
- tlb_unlock1 spc,t0,t1
+ ptl_unlock spc,t0,t1
rfir
nop
@@ -1543,7 +1463,7 @@ naitlb_miss_20:
L2_ptep ptp,pte,t0,va,naitlb_check_alias_20
- tlb_lock spc,ptp,pte,t0,t1,naitlb_check_alias_20
+ ptl_lock spc,ptp,pte,t0,t1,naitlb_check_alias_20
update_accessed ptp,pte,t0,t1
make_insert_tlb spc,pte,prot,t1
@@ -1552,7 +1472,7 @@ naitlb_miss_20:
iitlbt pte,prot
- tlb_unlock1 spc,t0,t1
+ ptl_unlock spc,t0,t1
rfir
nop
@@ -1561,6 +1481,7 @@ naitlb_check_alias_20:
iitlbt pte,prot
+ insert_nops NUM_PIPELINE_INSNS - 1
rfir
nop
@@ -1575,14 +1496,14 @@ dbit_trap_20w:
L3_ptep ptp,pte,t0,va,dbit_fault
- tlb_lock spc,ptp,pte,t0,t1,dbit_fault
+ ptl_lock spc,ptp,pte,t0,t1,dbit_fault
update_dirty ptp,pte,t1
make_insert_tlb spc,pte,prot,t1
idtlbt pte,prot
- tlb_unlock0 spc,t0,t1
+ ptl_unlock spc,t0,t1
rfir
nop
#else
@@ -1595,7 +1516,7 @@ dbit_trap_11:
L2_ptep ptp,pte,t0,va,dbit_fault
- tlb_lock spc,ptp,pte,t0,t1,dbit_fault
+ ptl_lock spc,ptp,pte,t0,t1,dbit_fault
update_dirty ptp,pte,t1
make_insert_tlb_11 spc,pte,prot
@@ -1608,7 +1529,7 @@ dbit_trap_11:
mtsp t1, %sr1 /* Restore sr1 */
- tlb_unlock0 spc,t0,t1
+ ptl_unlock spc,t0,t1
rfir
nop
@@ -1619,7 +1540,7 @@ dbit_trap_20:
L2_ptep ptp,pte,t0,va,dbit_fault
- tlb_lock spc,ptp,pte,t0,t1,dbit_fault
+ ptl_lock spc,ptp,pte,t0,t1,dbit_fault
update_dirty ptp,pte,t1
make_insert_tlb spc,pte,prot,t1
@@ -1628,7 +1549,7 @@ dbit_trap_20:
idtlbt pte,prot
- tlb_unlock0 spc,t0,t1
+ ptl_unlock spc,t0,t1
rfir
nop
#endif
@@ -1721,7 +1642,7 @@ dtlb_fault:
.macro fork_like name
ENTRY_CFI(sys_\name\()_wrapper)
- LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30), %r1
+ mfctl %cr30,%r1
ldo TASK_REGS(%r1),%r1
reg_save %r1
mfctl %cr27, %r28
@@ -1741,7 +1662,7 @@ ENTRY(child_return)
BL schedule_tail, %r2
nop
finish_child_return:
- LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30), %r1
+ mfctl %cr30,%r1
ldo TASK_REGS(%r1),%r1 /* get pt regs */
LDREG PT_CR27(%r1), %r3
@@ -1752,7 +1673,7 @@ finish_child_return:
END(child_return)
ENTRY_CFI(sys_rt_sigreturn_wrapper)
- LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r26
+ mfctl %cr30,%r26
ldo TASK_REGS(%r26),%r26 /* get pt regs */
/* Don't save regs, we are going to restore them from sigcontext. */
STREG %r2, -RP_OFFSET(%r30)
@@ -1769,7 +1690,7 @@ ENTRY_CFI(sys_rt_sigreturn_wrapper)
LDREG -RP_OFFSET(%r30), %r2
/* FIXME: I think we need to restore a few more things here. */
- LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1
+ mfctl %cr30,%r1
ldo TASK_REGS(%r1),%r1 /* get pt regs */
reg_restore %r1
@@ -1788,9 +1709,7 @@ ENTRY(syscall_exit)
*/
/* save return value now */
-
mfctl %cr30, %r1
- LDREG TI_TASK(%r1),%r1
STREG %r28,TASK_PT_GR28(%r1)
/* Seems to me that dp could be wrong here, if the syscall involved
@@ -1801,14 +1720,15 @@ ENTRY(syscall_exit)
syscall_check_resched:
/* check for reschedule */
-
- LDREG TI_FLAGS-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r19 /* long */
+ mfctl %cr30,%r19
+ LDREG TASK_TI_FLAGS(%r19),%r19 /* long */
bb,<,n %r19, 31-TIF_NEED_RESCHED, syscall_do_resched /* forward */
.import do_signal,code
syscall_check_sig:
- LDREG TI_FLAGS-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r19
- ldi (_TIF_SIGPENDING|_TIF_NOTIFY_RESUME), %r26
+ mfctl %cr30,%r19
+ LDREG TASK_TI_FLAGS(%r19),%r19
+ ldi (_TIF_USER_WORK_MASK & ~_TIF_NEED_RESCHED), %r26
and,COND(<>) %r19, %r26, %r0
b,n syscall_restore /* skip past if we've nothing to do */
@@ -1818,7 +1738,7 @@ syscall_do_signal:
* consistent with all the relevant state of the process
* before the syscall. We need to verify this.
*/
- LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1
+ mfctl %cr30,%r1
ldo TASK_REGS(%r1), %r26 /* struct pt_regs *regs */
reg_save %r26
@@ -1829,18 +1749,18 @@ syscall_do_signal:
BL do_notify_resume,%r2
ldi 1, %r25 /* long in_syscall = 1 */
- LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1
+ mfctl %cr30,%r1
ldo TASK_REGS(%r1), %r20 /* reload pt_regs */
reg_restore %r20
b,n syscall_check_sig
syscall_restore:
- LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1
+ mfctl %cr30,%r1
/* Are we being ptraced? */
- ldw TASK_FLAGS(%r1),%r19
- ldi _TIF_SYSCALL_TRACE_MASK,%r2
+ LDREG TASK_TI_FLAGS(%r1),%r19
+ ldi _TIF_SINGLESTEP|_TIF_BLOCKSTEP,%r2
and,COND(=) %r19,%r2,%r0
b,n syscall_restore_rfi
@@ -1878,7 +1798,7 @@ syscall_restore:
mtsp %r1,%sr5 /* Restore sr5 */
mtsp %r1,%sr6 /* Restore sr6 */
- depi 3,31,2,%r31 /* ensure return to user mode. */
+ depi PRIV_USER,31,2,%r31 /* ensure return to user mode. */
#ifdef CONFIG_64BIT
/* decide whether to reset the wide mode bit
@@ -1954,7 +1874,7 @@ syscall_restore_rfi:
STREG %r0,TASK_PT_SR2(%r1)
LDREG TASK_PT_GR31(%r1),%r2
- depi 3,31,2,%r2 /* ensure return to user mode. */
+ depi PRIV_USER,31,2,%r2 /* ensure return to user mode. */
STREG %r2,TASK_PT_IAOQ0(%r1)
ldo 4(%r2),%r2
STREG %r2,TASK_PT_IAOQ1(%r1)
@@ -1963,10 +1883,10 @@ syscall_restore_rfi:
pt_regs_ok:
LDREG TASK_PT_IAOQ0(%r1),%r2
- depi 3,31,2,%r2 /* ensure return to user mode. */
+ depi PRIV_USER,31,2,%r2 /* ensure return to user mode. */
STREG %r2,TASK_PT_IAOQ0(%r1)
LDREG TASK_PT_IAOQ1(%r1),%r2
- depi 3,31,2,%r2
+ depi PRIV_USER,31,2,%r2
STREG %r2,TASK_PT_IAOQ1(%r1)
b intr_restore
copy %r25,%r16