summaryrefslogtreecommitdiff
path: root/arch/mips/include/asm/r4kcache.h
diff options
context:
space:
mode:
Diffstat (limited to 'arch/mips/include/asm/r4kcache.h')
-rw-r--r--arch/mips/include/asm/r4kcache.h528
1 files changed, 58 insertions, 470 deletions
diff --git a/arch/mips/include/asm/r4kcache.h b/arch/mips/include/asm/r4kcache.h
index 7f12d7e27c94..da1cd1bbdbc5 100644
--- a/arch/mips/include/asm/r4kcache.h
+++ b/arch/mips/include/asm/r4kcache.h
@@ -15,12 +15,18 @@
#include <linux/stringify.h>
#include <asm/asm.h>
+#include <asm/asm-eva.h>
#include <asm/cacheops.h>
#include <asm/compiler.h>
#include <asm/cpu-features.h>
#include <asm/cpu-type.h>
#include <asm/mipsmtregs.h>
-#include <linux/uaccess.h> /* for uaccess_kernel() */
+#include <asm/mmzone.h>
+#include <asm/unroll.h>
+
+extern void r5k_sc_init(void);
+extern void rm7k_sc_init(void);
+extern int mips_sc_init(void);
extern void (*r4k_blast_dcache)(void);
extern void (*r4k_blast_icache)(void);
@@ -38,68 +44,27 @@ extern void (*r4k_blast_icache)(void);
*/
#define INDEX_BASE CKSEG0
-#define cache_op(op,addr) \
+#define _cache_op(insn, op, addr) \
__asm__ __volatile__( \
" .set push \n" \
" .set noreorder \n" \
" .set "MIPS_ISA_ARCH_LEVEL" \n" \
- " cache %0, %1 \n" \
+ " " insn("%0", "%1") " \n" \
" .set pop \n" \
: \
: "i" (op), "R" (*(unsigned char *)(addr)))
-#ifdef CONFIG_MIPS_MT
-
-#define __iflush_prologue \
- unsigned long redundance; \
- extern int mt_n_iflushes; \
- for (redundance = 0; redundance < mt_n_iflushes; redundance++) {
-
-#define __iflush_epilogue \
- }
-
-#define __dflush_prologue \
- unsigned long redundance; \
- extern int mt_n_dflushes; \
- for (redundance = 0; redundance < mt_n_dflushes; redundance++) {
-
-#define __dflush_epilogue \
- }
-
-#define __inv_dflush_prologue __dflush_prologue
-#define __inv_dflush_epilogue __dflush_epilogue
-#define __sflush_prologue {
-#define __sflush_epilogue }
-#define __inv_sflush_prologue __sflush_prologue
-#define __inv_sflush_epilogue __sflush_epilogue
-
-#else /* CONFIG_MIPS_MT */
-
-#define __iflush_prologue {
-#define __iflush_epilogue }
-#define __dflush_prologue {
-#define __dflush_epilogue }
-#define __inv_dflush_prologue {
-#define __inv_dflush_epilogue }
-#define __sflush_prologue {
-#define __sflush_epilogue }
-#define __inv_sflush_prologue {
-#define __inv_sflush_epilogue }
-
-#endif /* CONFIG_MIPS_MT */
+#define cache_op(op, addr) \
+ _cache_op(kernel_cache, op, addr)
static inline void flush_icache_line_indexed(unsigned long addr)
{
- __iflush_prologue
cache_op(Index_Invalidate_I, addr);
- __iflush_epilogue
}
static inline void flush_dcache_line_indexed(unsigned long addr)
{
- __dflush_prologue
cache_op(Index_Writeback_Inv_D, addr);
- __dflush_epilogue
}
static inline void flush_scache_line_indexed(unsigned long addr)
@@ -109,9 +74,8 @@ static inline void flush_scache_line_indexed(unsigned long addr)
static inline void flush_icache_line(unsigned long addr)
{
- __iflush_prologue
switch (boot_cpu_type()) {
- case CPU_LOONGSON2:
+ case CPU_LOONGSON2EF:
cache_op(Hit_Invalidate_I_Loongson2, addr);
break;
@@ -119,21 +83,16 @@ static inline void flush_icache_line(unsigned long addr)
cache_op(Hit_Invalidate_I, addr);
break;
}
- __iflush_epilogue
}
static inline void flush_dcache_line(unsigned long addr)
{
- __dflush_prologue
cache_op(Hit_Writeback_Inv_D, addr);
- __dflush_epilogue
}
static inline void invalidate_dcache_line(unsigned long addr)
{
- __dflush_prologue
cache_op(Hit_Invalidate_D, addr);
- __dflush_epilogue
}
static inline void invalidate_scache_line(unsigned long addr)
@@ -146,14 +105,17 @@ static inline void flush_scache_line(unsigned long addr)
cache_op(Hit_Writeback_Inv_SD, addr);
}
-#define protected_cache_op(op,addr) \
+#ifdef CONFIG_EVA
+
+#define protected_cache_op(op, addr) \
({ \
int __err = 0; \
__asm__ __volatile__( \
" .set push \n" \
" .set noreorder \n" \
- " .set "MIPS_ISA_ARCH_LEVEL" \n" \
- "1: cache %1, (%2) \n" \
+ " .set mips0 \n" \
+ " .set eva \n" \
+ "1: cachee %1, (%2) \n" \
"2: .insn \n" \
" .set pop \n" \
" .section .fixup,\"ax\" \n" \
@@ -161,23 +123,22 @@ static inline void flush_scache_line(unsigned long addr)
" j 2b \n" \
" .previous \n" \
" .section __ex_table,\"a\" \n" \
- " "STR(PTR)" 1b, 3b \n" \
+ " "STR(PTR_WD)" 1b, 3b \n" \
" .previous" \
: "+r" (__err) \
: "i" (op), "r" (addr), "i" (-EFAULT)); \
__err; \
})
+#else
-
-#define protected_cachee_op(op,addr) \
+#define protected_cache_op(op, addr) \
({ \
int __err = 0; \
__asm__ __volatile__( \
" .set push \n" \
" .set noreorder \n" \
- " .set mips0 \n" \
- " .set eva \n" \
- "1: cachee %1, (%2) \n" \
+ " .set "MIPS_ISA_ARCH_LEVEL" \n" \
+ "1: cache %1, (%2) \n" \
"2: .insn \n" \
" .set pop \n" \
" .section .fixup,\"ax\" \n" \
@@ -185,12 +146,13 @@ static inline void flush_scache_line(unsigned long addr)
" j 2b \n" \
" .previous \n" \
" .section __ex_table,\"a\" \n" \
- " "STR(PTR)" 1b, 3b \n" \
+ " "STR(PTR_WD)" 1b, 3b \n" \
" .previous" \
: "+r" (__err) \
: "i" (op), "r" (addr), "i" (-EFAULT)); \
__err; \
})
+#endif
/*
* The next two are for badland addresses like signal trampolines.
@@ -198,15 +160,11 @@ static inline void flush_scache_line(unsigned long addr)
static inline int protected_flush_icache_line(unsigned long addr)
{
switch (boot_cpu_type()) {
- case CPU_LOONGSON2:
+ case CPU_LOONGSON2EF:
return protected_cache_op(Hit_Invalidate_I_Loongson2, addr);
default:
-#ifdef CONFIG_EVA
- return protected_cachee_op(Hit_Invalidate_I, addr);
-#else
return protected_cache_op(Hit_Invalidate_I, addr);
-#endif
}
}
@@ -218,20 +176,12 @@ static inline int protected_flush_icache_line(unsigned long addr)
*/
static inline int protected_writeback_dcache_line(unsigned long addr)
{
-#ifdef CONFIG_EVA
- return protected_cachee_op(Hit_Writeback_Inv_D, addr);
-#else
return protected_cache_op(Hit_Writeback_Inv_D, addr);
-#endif
}
static inline int protected_writeback_scache_line(unsigned long addr)
{
-#ifdef CONFIG_EVA
- return protected_cachee_op(Hit_Writeback_Inv_SD, addr);
-#else
return protected_cache_op(Hit_Writeback_Inv_SD, addr);
-#endif
}
/*
@@ -242,338 +192,10 @@ static inline void invalidate_tcache_page(unsigned long addr)
cache_op(Page_Invalidate_T, addr);
}
-#ifndef CONFIG_CPU_MIPSR6
-#define cache16_unroll32(base,op) \
- __asm__ __volatile__( \
- " .set push \n" \
- " .set noreorder \n" \
- " .set mips3 \n" \
- " cache %1, 0x000(%0); cache %1, 0x010(%0) \n" \
- " cache %1, 0x020(%0); cache %1, 0x030(%0) \n" \
- " cache %1, 0x040(%0); cache %1, 0x050(%0) \n" \
- " cache %1, 0x060(%0); cache %1, 0x070(%0) \n" \
- " cache %1, 0x080(%0); cache %1, 0x090(%0) \n" \
- " cache %1, 0x0a0(%0); cache %1, 0x0b0(%0) \n" \
- " cache %1, 0x0c0(%0); cache %1, 0x0d0(%0) \n" \
- " cache %1, 0x0e0(%0); cache %1, 0x0f0(%0) \n" \
- " cache %1, 0x100(%0); cache %1, 0x110(%0) \n" \
- " cache %1, 0x120(%0); cache %1, 0x130(%0) \n" \
- " cache %1, 0x140(%0); cache %1, 0x150(%0) \n" \
- " cache %1, 0x160(%0); cache %1, 0x170(%0) \n" \
- " cache %1, 0x180(%0); cache %1, 0x190(%0) \n" \
- " cache %1, 0x1a0(%0); cache %1, 0x1b0(%0) \n" \
- " cache %1, 0x1c0(%0); cache %1, 0x1d0(%0) \n" \
- " cache %1, 0x1e0(%0); cache %1, 0x1f0(%0) \n" \
- " .set pop \n" \
- : \
- : "r" (base), \
- "i" (op));
-
-#define cache32_unroll32(base,op) \
- __asm__ __volatile__( \
- " .set push \n" \
- " .set noreorder \n" \
- " .set mips3 \n" \
- " cache %1, 0x000(%0); cache %1, 0x020(%0) \n" \
- " cache %1, 0x040(%0); cache %1, 0x060(%0) \n" \
- " cache %1, 0x080(%0); cache %1, 0x0a0(%0) \n" \
- " cache %1, 0x0c0(%0); cache %1, 0x0e0(%0) \n" \
- " cache %1, 0x100(%0); cache %1, 0x120(%0) \n" \
- " cache %1, 0x140(%0); cache %1, 0x160(%0) \n" \
- " cache %1, 0x180(%0); cache %1, 0x1a0(%0) \n" \
- " cache %1, 0x1c0(%0); cache %1, 0x1e0(%0) \n" \
- " cache %1, 0x200(%0); cache %1, 0x220(%0) \n" \
- " cache %1, 0x240(%0); cache %1, 0x260(%0) \n" \
- " cache %1, 0x280(%0); cache %1, 0x2a0(%0) \n" \
- " cache %1, 0x2c0(%0); cache %1, 0x2e0(%0) \n" \
- " cache %1, 0x300(%0); cache %1, 0x320(%0) \n" \
- " cache %1, 0x340(%0); cache %1, 0x360(%0) \n" \
- " cache %1, 0x380(%0); cache %1, 0x3a0(%0) \n" \
- " cache %1, 0x3c0(%0); cache %1, 0x3e0(%0) \n" \
- " .set pop \n" \
- : \
- : "r" (base), \
- "i" (op));
-
-#define cache64_unroll32(base,op) \
- __asm__ __volatile__( \
- " .set push \n" \
- " .set noreorder \n" \
- " .set mips3 \n" \
- " cache %1, 0x000(%0); cache %1, 0x040(%0) \n" \
- " cache %1, 0x080(%0); cache %1, 0x0c0(%0) \n" \
- " cache %1, 0x100(%0); cache %1, 0x140(%0) \n" \
- " cache %1, 0x180(%0); cache %1, 0x1c0(%0) \n" \
- " cache %1, 0x200(%0); cache %1, 0x240(%0) \n" \
- " cache %1, 0x280(%0); cache %1, 0x2c0(%0) \n" \
- " cache %1, 0x300(%0); cache %1, 0x340(%0) \n" \
- " cache %1, 0x380(%0); cache %1, 0x3c0(%0) \n" \
- " cache %1, 0x400(%0); cache %1, 0x440(%0) \n" \
- " cache %1, 0x480(%0); cache %1, 0x4c0(%0) \n" \
- " cache %1, 0x500(%0); cache %1, 0x540(%0) \n" \
- " cache %1, 0x580(%0); cache %1, 0x5c0(%0) \n" \
- " cache %1, 0x600(%0); cache %1, 0x640(%0) \n" \
- " cache %1, 0x680(%0); cache %1, 0x6c0(%0) \n" \
- " cache %1, 0x700(%0); cache %1, 0x740(%0) \n" \
- " cache %1, 0x780(%0); cache %1, 0x7c0(%0) \n" \
- " .set pop \n" \
- : \
- : "r" (base), \
- "i" (op));
-
-#define cache128_unroll32(base,op) \
- __asm__ __volatile__( \
- " .set push \n" \
- " .set noreorder \n" \
- " .set mips3 \n" \
- " cache %1, 0x000(%0); cache %1, 0x080(%0) \n" \
- " cache %1, 0x100(%0); cache %1, 0x180(%0) \n" \
- " cache %1, 0x200(%0); cache %1, 0x280(%0) \n" \
- " cache %1, 0x300(%0); cache %1, 0x380(%0) \n" \
- " cache %1, 0x400(%0); cache %1, 0x480(%0) \n" \
- " cache %1, 0x500(%0); cache %1, 0x580(%0) \n" \
- " cache %1, 0x600(%0); cache %1, 0x680(%0) \n" \
- " cache %1, 0x700(%0); cache %1, 0x780(%0) \n" \
- " cache %1, 0x800(%0); cache %1, 0x880(%0) \n" \
- " cache %1, 0x900(%0); cache %1, 0x980(%0) \n" \
- " cache %1, 0xa00(%0); cache %1, 0xa80(%0) \n" \
- " cache %1, 0xb00(%0); cache %1, 0xb80(%0) \n" \
- " cache %1, 0xc00(%0); cache %1, 0xc80(%0) \n" \
- " cache %1, 0xd00(%0); cache %1, 0xd80(%0) \n" \
- " cache %1, 0xe00(%0); cache %1, 0xe80(%0) \n" \
- " cache %1, 0xf00(%0); cache %1, 0xf80(%0) \n" \
- " .set pop \n" \
- : \
- : "r" (base), \
- "i" (op));
-
-#else
-/*
- * MIPS R6 changed the cache opcode and moved to a 8-bit offset field.
- * This means we now need to increment the base register before we flush
- * more cache lines
- */
-#define cache16_unroll32(base,op) \
- __asm__ __volatile__( \
- " .set push\n" \
- " .set noreorder\n" \
- " .set mips64r6\n" \
- " .set noat\n" \
- " cache %1, 0x000(%0); cache %1, 0x010(%0)\n" \
- " cache %1, 0x020(%0); cache %1, 0x030(%0)\n" \
- " cache %1, 0x040(%0); cache %1, 0x050(%0)\n" \
- " cache %1, 0x060(%0); cache %1, 0x070(%0)\n" \
- " cache %1, 0x080(%0); cache %1, 0x090(%0)\n" \
- " cache %1, 0x0a0(%0); cache %1, 0x0b0(%0)\n" \
- " cache %1, 0x0c0(%0); cache %1, 0x0d0(%0)\n" \
- " cache %1, 0x0e0(%0); cache %1, 0x0f0(%0)\n" \
- " "__stringify(LONG_ADDIU)" $1, %0, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x010($1)\n" \
- " cache %1, 0x020($1); cache %1, 0x030($1)\n" \
- " cache %1, 0x040($1); cache %1, 0x050($1)\n" \
- " cache %1, 0x060($1); cache %1, 0x070($1)\n" \
- " cache %1, 0x080($1); cache %1, 0x090($1)\n" \
- " cache %1, 0x0a0($1); cache %1, 0x0b0($1)\n" \
- " cache %1, 0x0c0($1); cache %1, 0x0d0($1)\n" \
- " cache %1, 0x0e0($1); cache %1, 0x0f0($1)\n" \
- " .set pop\n" \
- : \
- : "r" (base), \
- "i" (op));
-
-#define cache32_unroll32(base,op) \
- __asm__ __volatile__( \
- " .set push\n" \
- " .set noreorder\n" \
- " .set mips64r6\n" \
- " .set noat\n" \
- " cache %1, 0x000(%0); cache %1, 0x020(%0)\n" \
- " cache %1, 0x040(%0); cache %1, 0x060(%0)\n" \
- " cache %1, 0x080(%0); cache %1, 0x0a0(%0)\n" \
- " cache %1, 0x0c0(%0); cache %1, 0x0e0(%0)\n" \
- " "__stringify(LONG_ADDIU)" $1, %0, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x020($1)\n" \
- " cache %1, 0x040($1); cache %1, 0x060($1)\n" \
- " cache %1, 0x080($1); cache %1, 0x0a0($1)\n" \
- " cache %1, 0x0c0($1); cache %1, 0x0e0($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x020($1)\n" \
- " cache %1, 0x040($1); cache %1, 0x060($1)\n" \
- " cache %1, 0x080($1); cache %1, 0x0a0($1)\n" \
- " cache %1, 0x0c0($1); cache %1, 0x0e0($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100\n" \
- " cache %1, 0x000($1); cache %1, 0x020($1)\n" \
- " cache %1, 0x040($1); cache %1, 0x060($1)\n" \
- " cache %1, 0x080($1); cache %1, 0x0a0($1)\n" \
- " cache %1, 0x0c0($1); cache %1, 0x0e0($1)\n" \
- " .set pop\n" \
- : \
- : "r" (base), \
- "i" (op));
-
-#define cache64_unroll32(base,op) \
- __asm__ __volatile__( \
- " .set push\n" \
- " .set noreorder\n" \
- " .set mips64r6\n" \
- " .set noat\n" \
- " cache %1, 0x000(%0); cache %1, 0x040(%0)\n" \
- " cache %1, 0x080(%0); cache %1, 0x0c0(%0)\n" \
- " "__stringify(LONG_ADDIU)" $1, %0, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x040($1)\n" \
- " cache %1, 0x080($1); cache %1, 0x0c0($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x040($1)\n" \
- " cache %1, 0x080($1); cache %1, 0x0c0($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x040($1)\n" \
- " cache %1, 0x080($1); cache %1, 0x0c0($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x040($1)\n" \
- " cache %1, 0x080($1); cache %1, 0x0c0($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x040($1)\n" \
- " cache %1, 0x080($1); cache %1, 0x0c0($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x040($1)\n" \
- " cache %1, 0x080($1); cache %1, 0x0c0($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x040($1)\n" \
- " cache %1, 0x080($1); cache %1, 0x0c0($1)\n" \
- " .set pop\n" \
- : \
- : "r" (base), \
- "i" (op));
-
-#define cache128_unroll32(base,op) \
- __asm__ __volatile__( \
- " .set push\n" \
- " .set noreorder\n" \
- " .set mips64r6\n" \
- " .set noat\n" \
- " cache %1, 0x000(%0); cache %1, 0x080(%0)\n" \
- " "__stringify(LONG_ADDIU)" $1, %0, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x080($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x080($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x080($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x080($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x080($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x080($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x080($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x080($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x080($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x080($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x080($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x080($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x080($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x080($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x080($1)\n" \
- " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \
- " cache %1, 0x000($1); cache %1, 0x080($1)\n" \
- " .set pop\n" \
- : \
- : "r" (base), \
- "i" (op));
-#endif /* CONFIG_CPU_MIPSR6 */
-
-/*
- * Perform the cache operation specified by op using a user mode virtual
- * address while in kernel mode.
- */
-#define cache16_unroll32_user(base,op) \
- __asm__ __volatile__( \
- " .set push \n" \
- " .set noreorder \n" \
- " .set mips0 \n" \
- " .set eva \n" \
- " cachee %1, 0x000(%0); cachee %1, 0x010(%0) \n" \
- " cachee %1, 0x020(%0); cachee %1, 0x030(%0) \n" \
- " cachee %1, 0x040(%0); cachee %1, 0x050(%0) \n" \
- " cachee %1, 0x060(%0); cachee %1, 0x070(%0) \n" \
- " cachee %1, 0x080(%0); cachee %1, 0x090(%0) \n" \
- " cachee %1, 0x0a0(%0); cachee %1, 0x0b0(%0) \n" \
- " cachee %1, 0x0c0(%0); cachee %1, 0x0d0(%0) \n" \
- " cachee %1, 0x0e0(%0); cachee %1, 0x0f0(%0) \n" \
- " cachee %1, 0x100(%0); cachee %1, 0x110(%0) \n" \
- " cachee %1, 0x120(%0); cachee %1, 0x130(%0) \n" \
- " cachee %1, 0x140(%0); cachee %1, 0x150(%0) \n" \
- " cachee %1, 0x160(%0); cachee %1, 0x170(%0) \n" \
- " cachee %1, 0x180(%0); cachee %1, 0x190(%0) \n" \
- " cachee %1, 0x1a0(%0); cachee %1, 0x1b0(%0) \n" \
- " cachee %1, 0x1c0(%0); cachee %1, 0x1d0(%0) \n" \
- " cachee %1, 0x1e0(%0); cachee %1, 0x1f0(%0) \n" \
- " .set pop \n" \
- : \
- : "r" (base), \
- "i" (op));
-
-#define cache32_unroll32_user(base, op) \
- __asm__ __volatile__( \
- " .set push \n" \
- " .set noreorder \n" \
- " .set mips0 \n" \
- " .set eva \n" \
- " cachee %1, 0x000(%0); cachee %1, 0x020(%0) \n" \
- " cachee %1, 0x040(%0); cachee %1, 0x060(%0) \n" \
- " cachee %1, 0x080(%0); cachee %1, 0x0a0(%0) \n" \
- " cachee %1, 0x0c0(%0); cachee %1, 0x0e0(%0) \n" \
- " cachee %1, 0x100(%0); cachee %1, 0x120(%0) \n" \
- " cachee %1, 0x140(%0); cachee %1, 0x160(%0) \n" \
- " cachee %1, 0x180(%0); cachee %1, 0x1a0(%0) \n" \
- " cachee %1, 0x1c0(%0); cachee %1, 0x1e0(%0) \n" \
- " cachee %1, 0x200(%0); cachee %1, 0x220(%0) \n" \
- " cachee %1, 0x240(%0); cachee %1, 0x260(%0) \n" \
- " cachee %1, 0x280(%0); cachee %1, 0x2a0(%0) \n" \
- " cachee %1, 0x2c0(%0); cachee %1, 0x2e0(%0) \n" \
- " cachee %1, 0x300(%0); cachee %1, 0x320(%0) \n" \
- " cachee %1, 0x340(%0); cachee %1, 0x360(%0) \n" \
- " cachee %1, 0x380(%0); cachee %1, 0x3a0(%0) \n" \
- " cachee %1, 0x3c0(%0); cachee %1, 0x3e0(%0) \n" \
- " .set pop \n" \
- : \
- : "r" (base), \
- "i" (op));
-
-#define cache64_unroll32_user(base, op) \
- __asm__ __volatile__( \
- " .set push \n" \
- " .set noreorder \n" \
- " .set mips0 \n" \
- " .set eva \n" \
- " cachee %1, 0x000(%0); cachee %1, 0x040(%0) \n" \
- " cachee %1, 0x080(%0); cachee %1, 0x0c0(%0) \n" \
- " cachee %1, 0x100(%0); cachee %1, 0x140(%0) \n" \
- " cachee %1, 0x180(%0); cachee %1, 0x1c0(%0) \n" \
- " cachee %1, 0x200(%0); cachee %1, 0x240(%0) \n" \
- " cachee %1, 0x280(%0); cachee %1, 0x2c0(%0) \n" \
- " cachee %1, 0x300(%0); cachee %1, 0x340(%0) \n" \
- " cachee %1, 0x380(%0); cachee %1, 0x3c0(%0) \n" \
- " cachee %1, 0x400(%0); cachee %1, 0x440(%0) \n" \
- " cachee %1, 0x480(%0); cachee %1, 0x4c0(%0) \n" \
- " cachee %1, 0x500(%0); cachee %1, 0x540(%0) \n" \
- " cachee %1, 0x580(%0); cachee %1, 0x5c0(%0) \n" \
- " cachee %1, 0x600(%0); cachee %1, 0x640(%0) \n" \
- " cachee %1, 0x680(%0); cachee %1, 0x6c0(%0) \n" \
- " cachee %1, 0x700(%0); cachee %1, 0x740(%0) \n" \
- " cachee %1, 0x780(%0); cachee %1, 0x7c0(%0) \n" \
- " .set pop \n" \
- : \
- : "r" (base), \
- "i" (op));
+#define cache_unroll(times, insn, op, addr, lsize) do { \
+ int i = 0; \
+ unroll(times, _cache_op, insn, op, (addr) + (i++ * (lsize))); \
+} while (0)
/* build blast_xxx, blast_xxx_page, blast_xxx_page_indexed */
#define __BUILD_BLAST_CACHE(pfx, desc, indexop, hitop, lsize, extra) \
@@ -586,13 +208,10 @@ static inline void extra##blast_##pfx##cache##lsize(void) \
current_cpu_data.desc.waybit; \
unsigned long ws, addr; \
\
- __##pfx##flush_prologue \
- \
for (ws = 0; ws < ws_end; ws += ws_inc) \
for (addr = start; addr < end; addr += lsize * 32) \
- cache##lsize##_unroll32(addr|ws, indexop); \
- \
- __##pfx##flush_epilogue \
+ cache_unroll(32, kernel_cache, indexop, \
+ addr | ws, lsize); \
} \
\
static inline void extra##blast_##pfx##cache##lsize##_page(unsigned long page) \
@@ -600,14 +219,10 @@ static inline void extra##blast_##pfx##cache##lsize##_page(unsigned long page) \
unsigned long start = page; \
unsigned long end = page + PAGE_SIZE; \
\
- __##pfx##flush_prologue \
- \
do { \
- cache##lsize##_unroll32(start, hitop); \
+ cache_unroll(32, kernel_cache, hitop, start, lsize); \
start += lsize * 32; \
} while (start < end); \
- \
- __##pfx##flush_epilogue \
} \
\
static inline void extra##blast_##pfx##cache##lsize##_page_indexed(unsigned long page) \
@@ -620,13 +235,10 @@ static inline void extra##blast_##pfx##cache##lsize##_page_indexed(unsigned long
current_cpu_data.desc.waybit; \
unsigned long ws, addr; \
\
- __##pfx##flush_prologue \
- \
for (ws = 0; ws < ws_end; ws += ws_inc) \
for (addr = start; addr < end; addr += lsize * 32) \
- cache##lsize##_unroll32(addr|ws, indexop); \
- \
- __##pfx##flush_epilogue \
+ cache_unroll(32, kernel_cache, indexop, \
+ addr | ws, lsize); \
}
__BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 16, )
@@ -656,14 +268,10 @@ static inline void blast_##pfx##cache##lsize##_user_page(unsigned long page) \
unsigned long start = page; \
unsigned long end = page + PAGE_SIZE; \
\
- __##pfx##flush_prologue \
- \
do { \
- cache##lsize##_unroll32_user(start, hitop); \
+ cache_unroll(32, user_cache, hitop, start, lsize); \
start += lsize * 32; \
} while (start < end); \
- \
- __##pfx##flush_epilogue \
}
__BUILD_BLAST_USER_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D,
@@ -685,58 +293,16 @@ static inline void prot##extra##blast_##pfx##cache##_range(unsigned long start,
unsigned long addr = start & ~(lsize - 1); \
unsigned long aend = (end - 1) & ~(lsize - 1); \
\
- __##pfx##flush_prologue \
- \
while (1) { \
prot##cache_op(hitop, addr); \
if (addr == aend) \
break; \
addr += lsize; \
} \
- \
- __##pfx##flush_epilogue \
}
-#ifndef CONFIG_EVA
-
__BUILD_BLAST_CACHE_RANGE(d, dcache, Hit_Writeback_Inv_D, protected_, )
__BUILD_BLAST_CACHE_RANGE(i, icache, Hit_Invalidate_I, protected_, )
-
-#else
-
-#define __BUILD_PROT_BLAST_CACHE_RANGE(pfx, desc, hitop) \
-static inline void protected_blast_##pfx##cache##_range(unsigned long start,\
- unsigned long end) \
-{ \
- unsigned long lsize = cpu_##desc##_line_size(); \
- unsigned long addr = start & ~(lsize - 1); \
- unsigned long aend = (end - 1) & ~(lsize - 1); \
- \
- __##pfx##flush_prologue \
- \
- if (!uaccess_kernel()) { \
- while (1) { \
- protected_cachee_op(hitop, addr); \
- if (addr == aend) \
- break; \
- addr += lsize; \
- } \
- } else { \
- while (1) { \
- protected_cache_op(hitop, addr); \
- if (addr == aend) \
- break; \
- addr += lsize; \
- } \
- \
- } \
- __##pfx##flush_epilogue \
-}
-
-__BUILD_PROT_BLAST_CACHE_RANGE(d, dcache, Hit_Writeback_Inv_D)
-__BUILD_PROT_BLAST_CACHE_RANGE(i, icache, Hit_Invalidate_I)
-
-#endif
__BUILD_BLAST_CACHE_RANGE(s, scache, Hit_Writeback_Inv_SD, protected_, )
__BUILD_BLAST_CACHE_RANGE(i, icache, Hit_Invalidate_I_Loongson2, \
protected_, loongson2_)
@@ -747,4 +313,26 @@ __BUILD_BLAST_CACHE_RANGE(s, scache, Hit_Writeback_Inv_SD, , )
__BUILD_BLAST_CACHE_RANGE(inv_d, dcache, Hit_Invalidate_D, , )
__BUILD_BLAST_CACHE_RANGE(inv_s, scache, Hit_Invalidate_SD, , )
+/* Currently, this is very specific to Loongson-3 */
+#define __BUILD_BLAST_CACHE_NODE(pfx, desc, indexop, hitop, lsize) \
+static inline void blast_##pfx##cache##lsize##_node(long node) \
+{ \
+ unsigned long start = CAC_BASE | nid_to_addrbase(node); \
+ unsigned long end = start + current_cpu_data.desc.waysize; \
+ unsigned long ws_inc = 1UL << current_cpu_data.desc.waybit; \
+ unsigned long ws_end = current_cpu_data.desc.ways << \
+ current_cpu_data.desc.waybit; \
+ unsigned long ws, addr; \
+ \
+ for (ws = 0; ws < ws_end; ws += ws_inc) \
+ for (addr = start; addr < end; addr += lsize * 32) \
+ cache_unroll(32, kernel_cache, indexop, \
+ addr | ws, lsize); \
+}
+
+__BUILD_BLAST_CACHE_NODE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 16)
+__BUILD_BLAST_CACHE_NODE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 32)
+__BUILD_BLAST_CACHE_NODE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 64)
+__BUILD_BLAST_CACHE_NODE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 128)
+
#endif /* _ASM_R4KCACHE_H */
generated by cgit (git 2.34.1) at 2025-12-25 14:54:18 +0000