diff options
Diffstat (limited to 'arch/parisc/kernel/entry.S')
| -rw-r--r-- | arch/parisc/kernel/entry.S | 435 |
1 files changed, 184 insertions, 251 deletions
diff --git a/arch/parisc/kernel/entry.S b/arch/parisc/kernel/entry.S index 4b484ec7c7da..e04c5d806c10 100644 --- a/arch/parisc/kernel/entry.S +++ b/arch/parisc/kernel/entry.S @@ -25,6 +25,7 @@ #include <asm/traps.h> #include <asm/thread_info.h> #include <asm/alternative.h> +#include <asm/spinlock_types.h> #include <linux/linkage.h> #include <linux/pgtable.h> @@ -35,10 +36,27 @@ .level 2.0 #endif - .import pa_tlb_lock,data - .macro load_pa_tlb_lock reg - mfctl %cr25,\reg - addil L%(PAGE_SIZE << (PGD_ALLOC_ORDER - 1)),\reg +/* + * We need seven instructions after a TLB insert for it to take effect. + * The PA8800/PA8900 processors are an exception and need 12 instructions. + * The RFI changes both IAOQ_Back and IAOQ_Front, so it counts as one. + */ +#ifdef CONFIG_64BIT +#define NUM_PIPELINE_INSNS 12 +#else +#define NUM_PIPELINE_INSNS 7 +#endif + + /* Insert num nops */ + .macro insert_nops num + .rept \num + nop + .endr + .endm + + /* Get aligned page_table_lock address for this mm from cr28/tr4 */ + .macro get_ptl reg + mfctl %cr28,\reg .endm /* space_to_prot macro creates a prot id from a space id */ @@ -52,30 +70,6 @@ extrd,u \spc,(64 - (SPACEID_SHIFT)),32,\prot .endm #endif - - /* Switch to virtual mapping, trashing only %r1 */ - .macro virt_map - /* pcxt_ssm_bug */ - rsm PSW_SM_I, %r0 /* barrier for "Relied upon Translation */ - mtsp %r0, %sr4 - mtsp %r0, %sr5 - mtsp %r0, %sr6 - tovirt_r1 %r29 - load32 KERNEL_PSW, %r1 - - rsm PSW_SM_QUIET,%r0 /* second "heavy weight" ctl op */ - mtctl %r0, %cr17 /* Clear IIASQ tail */ - mtctl %r0, %cr17 /* Clear IIASQ head */ - mtctl %r1, %ipsw - load32 4f, %r1 - mtctl %r1, %cr18 /* Set IIAOQ tail */ - ldo 4(%r1), %r1 - mtctl %r1, %cr18 /* Set IIAOQ head */ - rfir - nop -4: - .endm - /* * The "get_stack" macros are responsible for determining the * kernel stack value. @@ -88,8 +82,8 @@ * Need to set up a kernel stack, so call the * get_stack_use_cr30 macro to set up a pointer * to the pt_regs structure contained within the - * task pointer pointed to by cr30. Set the stack - * pointer to point to the end of the task structure. + * task pointer pointed to by cr30. Load the stack + * pointer from the task structure. * * Note that we use shadowed registers for temps until * we can save %r26 and %r29. %r26 is used to preserve @@ -101,8 +95,6 @@ * or handle_interruption. %r29 is used to hold a pointer * the register save area, and once again, it needs to * be a non-shadowed register so that it survives the rfir. - * - * N.B. TASK_SZ_ALGN and PT_SZ_ALGN include space for a stack frame. */ .macro get_stack_use_cr30 @@ -111,12 +103,11 @@ copy %r30, %r17 mfctl %cr30, %r1 - ldo THREAD_SZ_ALGN(%r1), %r30 - mtsp %r0,%sr7 + tophys %r1,%r9 /* task_struct */ + LDREG TASK_STACK(%r9),%r30 + ldo PT_SZ_ALGN(%r30),%r30 + mtsp %r0,%sr7 /* clear sr7 after kernel stack was set! */ mtsp %r16,%sr3 - tophys %r1,%r9 - LDREG TI_TASK(%r9), %r1 /* thread_info -> task_struct */ - tophys %r1,%r9 ldo TASK_REGS(%r9),%r9 STREG %r17,PT_GR30(%r9) STREG %r29,PT_GR29(%r9) @@ -394,69 +385,47 @@ */ .macro L2_ptep pmd,pte,index,va,fault #if CONFIG_PGTABLE_LEVELS == 3 - extru \va,31-ASM_PMD_SHIFT,ASM_BITS_PER_PMD,\index + extru_safe \va,31-ASM_PMD_SHIFT,ASM_BITS_PER_PMD,\index #else -# if defined(CONFIG_64BIT) - extrd,u \va,63-ASM_PGDIR_SHIFT,ASM_BITS_PER_PGD,\index - #else - # if PAGE_SIZE > 4096 - extru \va,31-ASM_PGDIR_SHIFT,32-ASM_PGDIR_SHIFT,\index - # else - extru \va,31-ASM_PGDIR_SHIFT,ASM_BITS_PER_PGD,\index - # endif -# endif + extru_safe \va,31-ASM_PGDIR_SHIFT,ASM_BITS_PER_PGD,\index #endif dep %r0,31,PAGE_SHIFT,\pmd /* clear offset */ +#if CONFIG_PGTABLE_LEVELS < 3 copy %r0,\pte +#endif ldw,s \index(\pmd),\pmd bb,>=,n \pmd,_PxD_PRESENT_BIT,\fault dep %r0,31,PxD_FLAG_SHIFT,\pmd /* clear flags */ SHLREG \pmd,PxD_VALUE_SHIFT,\pmd - extru \va,31-PAGE_SHIFT,ASM_BITS_PER_PTE,\index + extru_safe \va,31-PAGE_SHIFT,ASM_BITS_PER_PTE,\index dep %r0,31,PAGE_SHIFT,\pmd /* clear offset */ shladd \index,BITS_PER_PTE_ENTRY,\pmd,\pmd /* pmd is now pte */ .endm - /* Look up PTE in a 3-Level scheme. - * - * Here we implement a Hybrid L2/L3 scheme: we allocate the - * first pmd adjacent to the pgd. This means that we can - * subtract a constant offset to get to it. The pmd and pgd - * sizes are arranged so that a single pmd covers 4GB (giving - * a full LP64 process access to 8TB) so our lookups are - * effectively L2 for the first 4GB of the kernel (i.e. for - * all ILP32 processes and all the kernel for machines with - * under 4GB of memory) */ + /* Look up PTE in a 3-Level scheme. */ .macro L3_ptep pgd,pte,index,va,fault -#if CONFIG_PGTABLE_LEVELS == 3 /* we might have a 2-Level scheme, e.g. with 16kb page size */ +#if CONFIG_PGTABLE_LEVELS == 3 + copy %r0,\pte extrd,u \va,63-ASM_PGDIR_SHIFT,ASM_BITS_PER_PGD,\index - extrd,u,*= \va,63-ASM_PGDIR_SHIFT,64-ASM_PGDIR_SHIFT,%r0 ldw,s \index(\pgd),\pgd - extrd,u,*= \va,63-ASM_PGDIR_SHIFT,64-ASM_PGDIR_SHIFT,%r0 bb,>=,n \pgd,_PxD_PRESENT_BIT,\fault - extrd,u,*= \va,63-ASM_PGDIR_SHIFT,64-ASM_PGDIR_SHIFT,%r0 - shld \pgd,PxD_VALUE_SHIFT,\index - extrd,u,*= \va,63-ASM_PGDIR_SHIFT,64-ASM_PGDIR_SHIFT,%r0 - copy \index,\pgd - extrd,u,*<> \va,63-ASM_PGDIR_SHIFT,64-ASM_PGDIR_SHIFT,%r0 - ldo ASM_PGD_PMD_OFFSET(\pgd),\pgd + shld \pgd,PxD_VALUE_SHIFT,\pgd #endif L2_ptep \pgd,\pte,\index,\va,\fault .endm - /* Acquire pa_tlb_lock lock and check page is present. */ - .macro tlb_lock spc,ptp,pte,tmp,tmp1,fault -#ifdef CONFIG_SMP + /* Acquire page_table_lock and check page is present. */ + .macro ptl_lock spc,ptp,pte,tmp,tmp1,fault +#ifdef CONFIG_TLB_PTLOCK 98: cmpib,COND(=),n 0,\spc,2f - load_pa_tlb_lock \tmp + get_ptl \tmp 1: LDCW 0(\tmp),\tmp1 cmpib,COND(=) 0,\tmp1,1b nop LDREG 0(\ptp),\pte bb,<,n \pte,_PAGE_PRESENT_BIT,3f - LDCW 0(\tmp),\tmp1 b \fault - stw \spc,0(\tmp) + stw \tmp1,0(\tmp) 99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP) #endif 2: LDREG 0(\ptp),\pte @@ -464,23 +433,20 @@ 3: .endm - /* Release pa_tlb_lock lock without reloading lock address. */ - .macro tlb_unlock0 spc,tmp,tmp1 -#ifdef CONFIG_SMP -98: or,COND(=) %r0,\spc,%r0 - LDCW 0(\tmp),\tmp1 + /* Release page_table_lock if for user space. We use an ordered + store to ensure all prior accesses are performed prior to + releasing the lock. Note stw may not be executed, so we + provide one extra nop when CONFIG_TLB_PTLOCK is defined. */ + .macro ptl_unlock spc,tmp,tmp2 +#ifdef CONFIG_TLB_PTLOCK +98: get_ptl \tmp + ldi __ARCH_SPIN_LOCK_UNLOCKED_VAL, \tmp2 or,COND(=) %r0,\spc,%r0 - stw \spc,0(\tmp) + stw,ma \tmp2,0(\tmp) 99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP) -#endif - .endm - - /* Release pa_tlb_lock lock. */ - .macro tlb_unlock1 spc,tmp,tmp1 -#ifdef CONFIG_SMP -98: load_pa_tlb_lock \tmp -99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP) - tlb_unlock0 \spc,\tmp,\tmp1 + insert_nops NUM_PIPELINE_INSNS - 4 +#else + insert_nops NUM_PIPELINE_INSNS - 1 #endif .endm @@ -509,13 +475,13 @@ * to a CPU TLB 4k PFN (4k => 12 bits to shift) */ #define PAGE_ADD_SHIFT (PAGE_SHIFT-12) #define PAGE_ADD_HUGE_SHIFT (REAL_HPAGE_SHIFT-12) + #define PFN_START_BIT (63-ASM_PFN_PTE_SHIFT+(63-58)-PAGE_ADD_SHIFT) /* Drop prot bits and convert to page addr for iitlbt and idtlbt */ .macro convert_for_tlb_insert20 pte,tmp #ifdef CONFIG_HUGETLB_PAGE copy \pte,\tmp - extrd,u \tmp,(63-ASM_PFN_PTE_SHIFT)+(63-58)+PAGE_ADD_SHIFT,\ - 64-PAGE_SHIFT-PAGE_ADD_SHIFT,\pte + extrd,u \tmp,PFN_START_BIT,PFN_START_BIT+1,\pte depdi _PAGE_SIZE_ENCODING_DEFAULT,63,\ (63-58)+PAGE_ADD_SHIFT,\pte @@ -523,8 +489,7 @@ depdi _HUGE_PAGE_SIZE_ENCODING_DEFAULT,63,\ (63-58)+PAGE_ADD_HUGE_SHIFT,\pte #else /* Huge pages disabled */ - extrd,u \pte,(63-ASM_PFN_PTE_SHIFT)+(63-58)+PAGE_ADD_SHIFT,\ - 64-PAGE_SHIFT-PAGE_ADD_SHIFT,\pte + extrd,u \pte,PFN_START_BIT,PFN_START_BIT+1,\pte depdi _PAGE_SIZE_ENCODING_DEFAULT,63,\ (63-58)+PAGE_ADD_SHIFT,\pte #endif @@ -534,6 +499,12 @@ * this happens is quite subtle, read below */ .macro make_insert_tlb spc,pte,prot,tmp space_to_prot \spc \prot /* create prot id from space */ + +#if _PAGE_SPECIAL_BIT == _PAGE_DMB_BIT + /* need to drop DMB bit, as it's used as SPECIAL flag */ + depi 0,_PAGE_SPECIAL_BIT,1,\pte +#endif + /* The following is the real subtlety. This is depositing * T <-> _PAGE_REFTRAP * D <-> _PAGE_DIRTY @@ -550,9 +521,14 @@ /* PAGE_USER indicates the page can be read with user privileges, * so deposit X1|11 to PL1|PL2 (remember the upper bit of PL1 - * contains _PAGE_READ) */ + * contains _PAGE_READ). While the kernel can't directly write + * user pages which have _PAGE_WRITE zero, it can read pages + * which have _PAGE_READ zero (PL <= PL1). Thus, the kernel + * exception fault handler doesn't trigger when reading pages + * that aren't user read accessible */ extrd,u,*= \pte,_PAGE_USER_BIT+32,1,%r0 depdi 7,11,3,\prot + /* If we're a gateway page, drop PL2 back to zero for promotion * to kernel privilege (so we can execute the page as kernel). * Any privilege promotion page always denys read and write */ @@ -576,6 +552,10 @@ * makes the tlb entry for the differently formatted pa11 * insertion instructions */ .macro make_insert_tlb_11 spc,pte,prot +#if _PAGE_SPECIAL_BIT == _PAGE_DMB_BIT + /* need to drop DMB bit, as it's used as SPECIAL flag */ + depi 0,_PAGE_SPECIAL_BIT,1,\pte +#endif zdep \spc,30,15,\prot dep \pte,8,7,\prot extru,= \pte,_PAGE_NO_CACHE_BIT,1,%r0 @@ -601,8 +581,9 @@ extrd,s \pte,63,25,\pte .endm - /* The alias region is an 8MB aligned 16MB to do clear and - * copy user pages at addresses congruent with the user + /* The alias region is comprised of a pair of 4 MB regions + * aligned to 8 MB. It is used to clear/copy/flush user pages + * using kernel virtual addresses congruent with the user * virtual address. * * To use the alias page, you set %r26 up with the to TLB @@ -612,13 +593,8 @@ .macro do_alias spc,tmp,tmp1,va,pte,prot,fault,patype cmpib,COND(<>),n 0,\spc,\fault ldil L%(TMPALIAS_MAP_START),\tmp -#if defined(CONFIG_64BIT) && (TMPALIAS_MAP_START >= 0x80000000) - /* on LP64, ldi will sign extend into the upper 32 bits, - * which is behaviour we don't want */ - depdi 0,31,32,\tmp -#endif copy \va,\tmp1 - depi 0,31,23,\tmp1 + depi_safe 0,31,TMPALIAS_SIZE_BITS+1,\tmp1 cmpb,COND(<>),n \tmp,\tmp1,\fault mfctl %cr19,\tmp /* iir */ /* get the opcode (first six bits) into \tmp */ @@ -651,13 +627,13 @@ * OK, it is in the temp alias region, check whether "from" or "to". * Check "subtle" note in pacache.S re: r23/r26. */ -#ifdef CONFIG_64BIT - extrd,u,*= \va,41,1,%r0 -#else - extrw,u,= \va,9,1,%r0 -#endif + extrw,u,= \va,31-TMPALIAS_SIZE_BITS,1,%r0 or,COND(tr) %r23,%r0,\pte or %r26,%r0,\pte + + /* convert phys addr in \pte (from r23 or r26) to tlb insert format */ + SHRREG \pte,PAGE_SHIFT+PAGE_ADD_SHIFT-5, \pte + depi_safe _PAGE_SIZE_ENCODING_DEFAULT, 31,5, \pte .endm @@ -769,7 +745,7 @@ ENTRY(ret_from_kernel_thread) BL schedule_tail, %r2 nop - LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30), %r1 + mfctl %cr30,%r1 /* task_struct */ LDREG TASK_PT_GR25(%r1), %r26 #ifdef CONFIG_64BIT LDREG TASK_PT_GR27(%r1), %r27 @@ -800,7 +776,6 @@ ENTRY_CFI(_switch_to) STREG %r30, TASK_PT_KSP(%r26) LDREG TASK_PT_KSP(%r25), %r30 - LDREG TASK_THREAD_INFO(%r25), %r25 bv %r0(%r2) mtctl %r25,%cr30 @@ -831,17 +806,16 @@ ENDPROC_CFI(_switch_to) .align PAGE_SIZE ENTRY_CFI(syscall_exit_rfi) - mfctl %cr30,%r16 - LDREG TI_TASK(%r16), %r16 /* thread_info -> task_struct */ + mfctl %cr30,%r16 /* task_struct */ ldo TASK_REGS(%r16),%r16 /* Force iaoq to userspace, as the user has had access to our current * context via sigcontext. Also Filter the PSW for the same reason. */ LDREG PT_IAOQ0(%r16),%r19 - depi 3,31,2,%r19 + depi PRIV_USER,31,2,%r19 STREG %r19,PT_IAOQ0(%r16) LDREG PT_IAOQ1(%r16),%r19 - depi 3,31,2,%r19 + depi PRIV_USER,31,2,%r19 STREG %r19,PT_IAOQ1(%r16) LDREG PT_PSW(%r16),%r19 load32 USER_PSW_MASK,%r1 @@ -877,15 +851,15 @@ ENTRY_CFI(syscall_exit_rfi) ENTRY(intr_return) /* check for reschedule */ mfctl %cr30,%r1 - LDREG TI_FLAGS(%r1),%r19 /* sched.h: TIF_NEED_RESCHED */ + LDREG TASK_TI_FLAGS(%r1),%r19 /* sched.h: TIF_NEED_RESCHED */ bb,<,n %r19,31-TIF_NEED_RESCHED,intr_do_resched /* forward */ .import do_notify_resume,code intr_check_sig: /* As above */ mfctl %cr30,%r1 - LDREG TI_FLAGS(%r1),%r19 - ldi (_TIF_SIGPENDING|_TIF_NOTIFY_RESUME), %r20 + LDREG TASK_TI_FLAGS(%r1),%r19 + ldi (_TIF_USER_WORK_MASK & ~_TIF_NEED_RESCHED), %r20 and,COND(<>) %r19, %r20, %r0 b,n intr_restore /* skip past if we've nothing to do */ @@ -897,20 +871,20 @@ intr_check_sig: * Only do signals if we are returning to user space */ LDREG PT_IASQ0(%r16), %r20 - cmpib,COND(=),n LINUX_GATEWAY_SPACE, %r20, intr_restore /* backward */ + cmpib,COND(=),n LINUX_GATEWAY_SPACE, %r20, intr_restore /* forward */ LDREG PT_IASQ1(%r16), %r20 - cmpib,COND(=),n LINUX_GATEWAY_SPACE, %r20, intr_restore /* backward */ - - /* NOTE: We need to enable interrupts if we have to deliver - * signals. We used to do this earlier but it caused kernel - * stack overflows. */ - ssm PSW_SM_I, %r0 + cmpib,COND(=),n LINUX_GATEWAY_SPACE, %r20, intr_restore /* forward */ copy %r0, %r25 /* long in_syscall = 0 */ #ifdef CONFIG_64BIT ldo -16(%r30),%r29 /* Reference param save area */ #endif + /* NOTE: We need to enable interrupts if we have to deliver + * signals. We used to do this earlier but it caused kernel + * stack overflows. */ + ssm PSW_SM_I, %r0 + BL do_notify_resume,%r2 copy %r16, %r26 /* struct pt_regs *regs */ @@ -986,8 +960,8 @@ intr_do_preempt: /* current_thread_info()->preempt_count */ mfctl %cr30, %r1 - LDREG TI_PRE_COUNT(%r1), %r19 - cmpib,COND(<>) 0, %r19, intr_restore /* if preempt_count > 0 */ + ldw TI_PRE_COUNT(%r1), %r19 + cmpib,<> 0, %r19, intr_restore /* if preempt_count > 0 */ nop /* prev insn branched backwards */ /* check if we interrupted a critical path */ @@ -995,10 +969,17 @@ intr_do_preempt: bb,<,n %r20, 31 - PSW_SM_I, intr_restore nop + /* ssm PSW_SM_I done later in intr_restore */ +#ifdef CONFIG_MLONGCALLS + ldil L%intr_restore, %r2 + load32 preempt_schedule_irq, %r1 + bv %r0(%r1) + ldo R%intr_restore(%r2), %r2 +#else + ldil L%intr_restore, %r1 BL preempt_schedule_irq, %r2 - nop - - b,n intr_restore /* ssm PSW_SM_I done by intr_restore */ + ldo R%intr_restore(%r1), %r2 +#endif #endif /* CONFIG_PREEMPTION */ /* @@ -1077,23 +1058,26 @@ ENTRY_CFI(intr_save) /* for os_hpmc */ STREG %r16, PT_ISR(%r29) STREG %r17, PT_IOR(%r29) -#if 0 && defined(CONFIG_64BIT) - /* Revisit when we have 64-bit code above 4Gb */ - b,n intr_save2 - +#if defined(CONFIG_64BIT) skip_save_ior: /* We have a itlb miss, and when executing code above 4 Gb on ILP64, we * need to adjust iasq/iaoq here in the same way we adjusted isr/ior * above. */ - extrd,u,* %r8,PSW_W_BIT,1,%r1 - cmpib,COND(=),n 1,%r1,intr_save2 + bb,COND(>=),n %r8,PSW_W_BIT,intr_save2 LDREG PT_IASQ0(%r29), %r16 LDREG PT_IAOQ0(%r29), %r17 - /* adjust iasq/iaoq */ + /* adjust iasq0/iaoq0 */ space_adjust %r16,%r17,%r1 STREG %r16, PT_IASQ0(%r29) STREG %r17, PT_IAOQ0(%r29) + + LDREG PT_IASQ1(%r29), %r16 + LDREG PT_IAOQ1(%r29), %r17 + /* adjust iasq1/iaoq1 */ + space_adjust %r16,%r17,%r1 + STREG %r16, PT_IASQ1(%r29) + STREG %r17, PT_IAOQ1(%r29) #else skip_save_ior: #endif @@ -1156,14 +1140,14 @@ dtlb_miss_20w: L3_ptep ptp,pte,t0,va,dtlb_check_alias_20w - tlb_lock spc,ptp,pte,t0,t1,dtlb_check_alias_20w + ptl_lock spc,ptp,pte,t0,t1,dtlb_check_alias_20w update_accessed ptp,pte,t0,t1 make_insert_tlb spc,pte,prot,t1 idtlbt pte,prot - tlb_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1172,6 +1156,7 @@ dtlb_check_alias_20w: idtlbt pte,prot + insert_nops NUM_PIPELINE_INSNS - 1 rfir nop @@ -1182,14 +1167,14 @@ nadtlb_miss_20w: L3_ptep ptp,pte,t0,va,nadtlb_check_alias_20w - tlb_lock spc,ptp,pte,t0,t1,nadtlb_check_alias_20w + ptl_lock spc,ptp,pte,t0,t1,nadtlb_check_alias_20w update_accessed ptp,pte,t0,t1 make_insert_tlb spc,pte,prot,t1 idtlbt pte,prot - tlb_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1198,6 +1183,7 @@ nadtlb_check_alias_20w: idtlbt pte,prot + insert_nops NUM_PIPELINE_INSNS - 1 rfir nop @@ -1210,7 +1196,7 @@ dtlb_miss_11: L2_ptep ptp,pte,t0,va,dtlb_check_alias_11 - tlb_lock spc,ptp,pte,t0,t1,dtlb_check_alias_11 + ptl_lock spc,ptp,pte,t0,t1,dtlb_check_alias_11 update_accessed ptp,pte,t0,t1 make_insert_tlb_11 spc,pte,prot @@ -1223,7 +1209,7 @@ dtlb_miss_11: mtsp t1, %sr1 /* Restore sr1 */ - tlb_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1233,6 +1219,7 @@ dtlb_check_alias_11: idtlba pte,(va) idtlbp prot,(va) + insert_nops NUM_PIPELINE_INSNS - 1 rfir nop @@ -1243,7 +1230,7 @@ nadtlb_miss_11: L2_ptep ptp,pte,t0,va,nadtlb_check_alias_11 - tlb_lock spc,ptp,pte,t0,t1,nadtlb_check_alias_11 + ptl_lock spc,ptp,pte,t0,t1,nadtlb_check_alias_11 update_accessed ptp,pte,t0,t1 make_insert_tlb_11 spc,pte,prot @@ -1256,7 +1243,7 @@ nadtlb_miss_11: mtsp t1, %sr1 /* Restore sr1 */ - tlb_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1266,6 +1253,7 @@ nadtlb_check_alias_11: idtlba pte,(va) idtlbp prot,(va) + insert_nops NUM_PIPELINE_INSNS - 1 rfir nop @@ -1276,7 +1264,7 @@ dtlb_miss_20: L2_ptep ptp,pte,t0,va,dtlb_check_alias_20 - tlb_lock spc,ptp,pte,t0,t1,dtlb_check_alias_20 + ptl_lock spc,ptp,pte,t0,t1,dtlb_check_alias_20 update_accessed ptp,pte,t0,t1 make_insert_tlb spc,pte,prot,t1 @@ -1285,7 +1273,7 @@ dtlb_miss_20: idtlbt pte,prot - tlb_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1294,6 +1282,7 @@ dtlb_check_alias_20: idtlbt pte,prot + insert_nops NUM_PIPELINE_INSNS - 1 rfir nop @@ -1304,7 +1293,7 @@ nadtlb_miss_20: L2_ptep ptp,pte,t0,va,nadtlb_check_alias_20 - tlb_lock spc,ptp,pte,t0,t1,nadtlb_check_alias_20 + ptl_lock spc,ptp,pte,t0,t1,nadtlb_check_alias_20 update_accessed ptp,pte,t0,t1 make_insert_tlb spc,pte,prot,t1 @@ -1313,7 +1302,7 @@ nadtlb_miss_20: idtlbt pte,prot - tlb_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1322,6 +1311,7 @@ nadtlb_check_alias_20: idtlbt pte,prot + insert_nops NUM_PIPELINE_INSNS - 1 rfir nop @@ -1330,74 +1320,12 @@ nadtlb_check_alias_20: nadtlb_emulate: /* - * Non access misses can be caused by fdc,fic,pdc,lpa,probe and - * probei instructions. We don't want to fault for these - * instructions (not only does it not make sense, it can cause - * deadlocks, since some flushes are done with the mmap - * semaphore held). If the translation doesn't exist, we can't - * insert a translation, so have to emulate the side effects - * of the instruction. Since we don't insert a translation - * we can get a lot of faults during a flush loop, so it makes - * sense to try to do it here with minimum overhead. We only - * emulate fdc,fic,pdc,probew,prober instructions whose base - * and index registers are not shadowed. We defer everything - * else to the "slow" path. + * Non-access misses can be caused by fdc,fic,pdc,lpa,probe and + * probei instructions. The kernel no longer faults doing flushes. + * Use of lpa and probe instructions is rare. Given the issue + * with shadow registers, we defer everything to the "slow" path. */ - - mfctl %cr19,%r9 /* Get iir */ - - /* PA 2.0 Arch Ref. Book pg 382 has a good description of the insn bits. - Checks for fdc,fdce,pdc,"fic,4f",prober,probeir,probew, probeiw */ - - /* Checks for fdc,fdce,pdc,"fic,4f" only */ - ldi 0x280,%r16 - and %r9,%r16,%r17 - cmpb,<>,n %r16,%r17,nadtlb_probe_check - bb,>=,n %r9,26,nadtlb_nullify /* m bit not set, just nullify */ - BL get_register,%r25 - extrw,u %r9,15,5,%r8 /* Get index register # */ - cmpib,COND(=),n -1,%r1,nadtlb_fault /* have to use slow path */ - copy %r1,%r24 - BL get_register,%r25 - extrw,u %r9,10,5,%r8 /* Get base register # */ - cmpib,COND(=),n -1,%r1,nadtlb_fault /* have to use slow path */ - BL set_register,%r25 - add,l %r1,%r24,%r1 /* doesn't affect c/b bits */ - -nadtlb_nullify: - mfctl %ipsw,%r8 - ldil L%PSW_N,%r9 - or %r8,%r9,%r8 /* Set PSW_N */ - mtctl %r8,%ipsw - - rfir - nop - - /* - When there is no translation for the probe address then we - must nullify the insn and return zero in the target register. - This will indicate to the calling code that it does not have - write/read privileges to this address. - - This should technically work for prober and probew in PA 1.1, - and also probe,r and probe,w in PA 2.0 - - WARNING: USE ONLY NON-SHADOW REGISTERS WITH PROBE INSN! - THE SLOW-PATH EMULATION HAS NOT BEEN WRITTEN YET. - - */ -nadtlb_probe_check: - ldi 0x80,%r16 - and %r9,%r16,%r17 - cmpb,<>,n %r16,%r17,nadtlb_fault /* Must be probe,[rw]*/ - BL get_register,%r25 /* Find the target register */ - extrw,u %r9,31,5,%r8 /* Get target register */ - cmpib,COND(=),n -1,%r1,nadtlb_fault /* have to use slow path */ - BL set_register,%r25 - copy %r0,%r1 /* Write zero to target register */ - b nadtlb_nullify /* Nullify return insn */ - nop - + b,n nadtlb_fault #ifdef CONFIG_64BIT itlb_miss_20w: @@ -1413,14 +1341,14 @@ itlb_miss_20w: L3_ptep ptp,pte,t0,va,itlb_fault - tlb_lock spc,ptp,pte,t0,t1,itlb_fault + ptl_lock spc,ptp,pte,t0,t1,itlb_fault update_accessed ptp,pte,t0,t1 make_insert_tlb spc,pte,prot,t1 iitlbt pte,prot - tlb_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1437,14 +1365,14 @@ naitlb_miss_20w: L3_ptep ptp,pte,t0,va,naitlb_check_alias_20w - tlb_lock spc,ptp,pte,t0,t1,naitlb_check_alias_20w + ptl_lock spc,ptp,pte,t0,t1,naitlb_check_alias_20w update_accessed ptp,pte,t0,t1 make_insert_tlb spc,pte,prot,t1 iitlbt pte,prot - tlb_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1453,6 +1381,7 @@ naitlb_check_alias_20w: iitlbt pte,prot + insert_nops NUM_PIPELINE_INSNS - 1 rfir nop @@ -1465,7 +1394,7 @@ itlb_miss_11: L2_ptep ptp,pte,t0,va,itlb_fault - tlb_lock spc,ptp,pte,t0,t1,itlb_fault + ptl_lock spc,ptp,pte,t0,t1,itlb_fault update_accessed ptp,pte,t0,t1 make_insert_tlb_11 spc,pte,prot @@ -1478,7 +1407,7 @@ itlb_miss_11: mtsp t1, %sr1 /* Restore sr1 */ - tlb_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1489,7 +1418,7 @@ naitlb_miss_11: L2_ptep ptp,pte,t0,va,naitlb_check_alias_11 - tlb_lock spc,ptp,pte,t0,t1,naitlb_check_alias_11 + ptl_lock spc,ptp,pte,t0,t1,naitlb_check_alias_11 update_accessed ptp,pte,t0,t1 make_insert_tlb_11 spc,pte,prot @@ -1502,7 +1431,7 @@ naitlb_miss_11: mtsp t1, %sr1 /* Restore sr1 */ - tlb_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1512,6 +1441,7 @@ naitlb_check_alias_11: iitlba pte,(%sr0, va) iitlbp prot,(%sr0, va) + insert_nops NUM_PIPELINE_INSNS - 1 rfir nop @@ -1523,7 +1453,7 @@ itlb_miss_20: L2_ptep ptp,pte,t0,va,itlb_fault - tlb_lock spc,ptp,pte,t0,t1,itlb_fault + ptl_lock spc,ptp,pte,t0,t1,itlb_fault update_accessed ptp,pte,t0,t1 make_insert_tlb spc,pte,prot,t1 @@ -1532,7 +1462,7 @@ itlb_miss_20: iitlbt pte,prot - tlb_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1543,7 +1473,7 @@ naitlb_miss_20: L2_ptep ptp,pte,t0,va,naitlb_check_alias_20 - tlb_lock spc,ptp,pte,t0,t1,naitlb_check_alias_20 + ptl_lock spc,ptp,pte,t0,t1,naitlb_check_alias_20 update_accessed ptp,pte,t0,t1 make_insert_tlb spc,pte,prot,t1 @@ -1552,7 +1482,7 @@ naitlb_miss_20: iitlbt pte,prot - tlb_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1561,6 +1491,7 @@ naitlb_check_alias_20: iitlbt pte,prot + insert_nops NUM_PIPELINE_INSNS - 1 rfir nop @@ -1575,14 +1506,14 @@ dbit_trap_20w: L3_ptep ptp,pte,t0,va,dbit_fault - tlb_lock spc,ptp,pte,t0,t1,dbit_fault + ptl_lock spc,ptp,pte,t0,t1,dbit_fault update_dirty ptp,pte,t1 make_insert_tlb spc,pte,prot,t1 idtlbt pte,prot - tlb_unlock0 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop #else @@ -1595,7 +1526,7 @@ dbit_trap_11: L2_ptep ptp,pte,t0,va,dbit_fault - tlb_lock spc,ptp,pte,t0,t1,dbit_fault + ptl_lock spc,ptp,pte,t0,t1,dbit_fault update_dirty ptp,pte,t1 make_insert_tlb_11 spc,pte,prot @@ -1608,7 +1539,7 @@ dbit_trap_11: mtsp t1, %sr1 /* Restore sr1 */ - tlb_unlock0 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1619,7 +1550,7 @@ dbit_trap_20: L2_ptep ptp,pte,t0,va,dbit_fault - tlb_lock spc,ptp,pte,t0,t1,dbit_fault + ptl_lock spc,ptp,pte,t0,t1,dbit_fault update_dirty ptp,pte,t1 make_insert_tlb spc,pte,prot,t1 @@ -1628,7 +1559,7 @@ dbit_trap_20: idtlbt pte,prot - tlb_unlock0 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop #endif @@ -1721,7 +1652,7 @@ dtlb_fault: .macro fork_like name ENTRY_CFI(sys_\name\()_wrapper) - LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30), %r1 + mfctl %cr30,%r1 ldo TASK_REGS(%r1),%r1 reg_save %r1 mfctl %cr27, %r28 @@ -1741,7 +1672,7 @@ ENTRY(child_return) BL schedule_tail, %r2 nop finish_child_return: - LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30), %r1 + mfctl %cr30,%r1 ldo TASK_REGS(%r1),%r1 /* get pt regs */ LDREG PT_CR27(%r1), %r3 @@ -1752,7 +1683,7 @@ finish_child_return: END(child_return) ENTRY_CFI(sys_rt_sigreturn_wrapper) - LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r26 + mfctl %cr30,%r26 ldo TASK_REGS(%r26),%r26 /* get pt regs */ /* Don't save regs, we are going to restore them from sigcontext. */ STREG %r2, -RP_OFFSET(%r30) @@ -1769,7 +1700,7 @@ ENTRY_CFI(sys_rt_sigreturn_wrapper) LDREG -RP_OFFSET(%r30), %r2 /* FIXME: I think we need to restore a few more things here. */ - LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 + mfctl %cr30,%r1 ldo TASK_REGS(%r1),%r1 /* get pt regs */ reg_restore %r1 @@ -1788,9 +1719,7 @@ ENTRY(syscall_exit) */ /* save return value now */ - mfctl %cr30, %r1 - LDREG TI_TASK(%r1),%r1 STREG %r28,TASK_PT_GR28(%r1) /* Seems to me that dp could be wrong here, if the syscall involved @@ -1801,14 +1730,15 @@ ENTRY(syscall_exit) syscall_check_resched: /* check for reschedule */ - - LDREG TI_FLAGS-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r19 /* long */ + mfctl %cr30,%r19 + LDREG TASK_TI_FLAGS(%r19),%r19 /* long */ bb,<,n %r19, 31-TIF_NEED_RESCHED, syscall_do_resched /* forward */ .import do_signal,code syscall_check_sig: - LDREG TI_FLAGS-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r19 - ldi (_TIF_SIGPENDING|_TIF_NOTIFY_RESUME), %r26 + mfctl %cr30,%r19 + LDREG TASK_TI_FLAGS(%r19),%r19 + ldi (_TIF_USER_WORK_MASK & ~_TIF_NEED_RESCHED), %r26 and,COND(<>) %r19, %r26, %r0 b,n syscall_restore /* skip past if we've nothing to do */ @@ -1818,7 +1748,7 @@ syscall_do_signal: * consistent with all the relevant state of the process * before the syscall. We need to verify this. */ - LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 + mfctl %cr30,%r1 ldo TASK_REGS(%r1), %r26 /* struct pt_regs *regs */ reg_save %r26 @@ -1829,18 +1759,18 @@ syscall_do_signal: BL do_notify_resume,%r2 ldi 1, %r25 /* long in_syscall = 1 */ - LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 + mfctl %cr30,%r1 ldo TASK_REGS(%r1), %r20 /* reload pt_regs */ reg_restore %r20 b,n syscall_check_sig syscall_restore: - LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 + mfctl %cr30,%r1 /* Are we being ptraced? */ - ldw TASK_FLAGS(%r1),%r19 - ldi _TIF_SYSCALL_TRACE_MASK,%r2 + LDREG TASK_TI_FLAGS(%r1),%r19 + ldi _TIF_SINGLESTEP|_TIF_BLOCKSTEP,%r2 and,COND(=) %r19,%r2,%r0 b,n syscall_restore_rfi @@ -1878,7 +1808,7 @@ syscall_restore: mtsp %r1,%sr5 /* Restore sr5 */ mtsp %r1,%sr6 /* Restore sr6 */ - depi 3,31,2,%r31 /* ensure return to user mode. */ + depi PRIV_USER,31,2,%r31 /* ensure return to user mode. */ #ifdef CONFIG_64BIT /* decide whether to reset the wide mode bit @@ -1916,6 +1846,10 @@ syscall_restore_rfi: extru,= %r19,TIF_BLOCKSTEP_PA_BIT,1,%r0 depi -1,7,1,%r20 /* T bit */ +#ifdef CONFIG_64BIT + extru,<> %r19,TIF_32BIT_PA_BIT,1,%r0 + depi -1,4,1,%r20 /* W bit */ +#endif STREG %r20,TASK_PT_PSW(%r1) /* Always store space registers, since sr3 can be changed (e.g. fork) */ @@ -1929,7 +1863,6 @@ syscall_restore_rfi: STREG %r25,TASK_PT_IASQ0(%r1) STREG %r25,TASK_PT_IASQ1(%r1) - /* XXX W bit??? */ /* Now if old D bit is clear, it means we didn't save all registers * on syscall entry, so do that now. This only happens on TRACEME * calls, or if someone attached to us while we were on a syscall. @@ -1954,7 +1887,7 @@ syscall_restore_rfi: STREG %r0,TASK_PT_SR2(%r1) LDREG TASK_PT_GR31(%r1),%r2 - depi 3,31,2,%r2 /* ensure return to user mode. */ + depi PRIV_USER,31,2,%r2 /* ensure return to user mode. */ STREG %r2,TASK_PT_IAOQ0(%r1) ldo 4(%r2),%r2 STREG %r2,TASK_PT_IAOQ1(%r1) @@ -1963,10 +1896,10 @@ syscall_restore_rfi: pt_regs_ok: LDREG TASK_PT_IAOQ0(%r1),%r2 - depi 3,31,2,%r2 /* ensure return to user mode. */ + depi PRIV_USER,31,2,%r2 /* ensure return to user mode. */ STREG %r2,TASK_PT_IAOQ0(%r1) LDREG TASK_PT_IAOQ1(%r1),%r2 - depi 3,31,2,%r2 + depi PRIV_USER,31,2,%r2 STREG %r2,TASK_PT_IAOQ1(%r1) b intr_restore copy %r25,%r16 |