summaryrefslogtreecommitdiff
path: root/arch/loongarch
diff options
context:
space:
mode:
Diffstat (limited to 'arch/loongarch')
-rw-r--r--arch/loongarch/include/asm/Kbuild1
-rw-r--r--arch/loongarch/include/asm/barrier.h130
-rw-r--r--arch/loongarch/include/asm/io.h2
-rw-r--r--arch/loongarch/include/asm/qspinlock.h18
-rw-r--r--arch/loongarch/kernel/smp.c2
-rw-r--r--arch/loongarch/mm/tlbex.S6
6 files changed, 78 insertions, 81 deletions
diff --git a/arch/loongarch/include/asm/Kbuild b/arch/loongarch/include/asm/Kbuild
index 77ad8e6f0906..6b222f227342 100644
--- a/arch/loongarch/include/asm/Kbuild
+++ b/arch/loongarch/include/asm/Kbuild
@@ -5,7 +5,6 @@ generic-y += mcs_spinlock.h
generic-y += parport.h
generic-y += early_ioremap.h
generic-y += qrwlock.h
-generic-y += qspinlock.h
generic-y += rwsem.h
generic-y += segment.h
generic-y += user.h
diff --git a/arch/loongarch/include/asm/barrier.h b/arch/loongarch/include/asm/barrier.h
index cda977675854..4b663f197706 100644
--- a/arch/loongarch/include/asm/barrier.h
+++ b/arch/loongarch/include/asm/barrier.h
@@ -5,27 +5,56 @@
#ifndef __ASM_BARRIER_H
#define __ASM_BARRIER_H
-#define __sync() __asm__ __volatile__("dbar 0" : : : "memory")
+/*
+ * Hint encoding:
+ *
+ * Bit4: ordering or completion (0: completion, 1: ordering)
+ * Bit3: barrier for previous read (0: true, 1: false)
+ * Bit2: barrier for previous write (0: true, 1: false)
+ * Bit1: barrier for succeeding read (0: true, 1: false)
+ * Bit0: barrier for succeeding write (0: true, 1: false)
+ *
+ * Hint 0x700: barrier for "read after read" from the same address
+ */
+
+#define DBAR(hint) __asm__ __volatile__("dbar %0 " : : "I"(hint) : "memory")
+
+#define crwrw 0b00000
+#define cr_r_ 0b00101
+#define c_w_w 0b01010
-#define fast_wmb() __sync()
-#define fast_rmb() __sync()
-#define fast_mb() __sync()
-#define fast_iob() __sync()
-#define wbflush() __sync()
+#define orwrw 0b10000
+#define or_r_ 0b10101
+#define o_w_w 0b11010
-#define wmb() fast_wmb()
-#define rmb() fast_rmb()
-#define mb() fast_mb()
-#define iob() fast_iob()
+#define orw_w 0b10010
+#define or_rw 0b10100
-#define __smp_mb() __asm__ __volatile__("dbar 0" : : : "memory")
-#define __smp_rmb() __asm__ __volatile__("dbar 0" : : : "memory")
-#define __smp_wmb() __asm__ __volatile__("dbar 0" : : : "memory")
+#define c_sync() DBAR(crwrw)
+#define c_rsync() DBAR(cr_r_)
+#define c_wsync() DBAR(c_w_w)
+
+#define o_sync() DBAR(orwrw)
+#define o_rsync() DBAR(or_r_)
+#define o_wsync() DBAR(o_w_w)
+
+#define ldacq_mb() DBAR(or_rw)
+#define strel_mb() DBAR(orw_w)
+
+#define mb() c_sync()
+#define rmb() c_rsync()
+#define wmb() c_wsync()
+#define iob() c_sync()
+#define wbflush() c_sync()
+
+#define __smp_mb() o_sync()
+#define __smp_rmb() o_rsync()
+#define __smp_wmb() o_wsync()
#ifdef CONFIG_SMP
-#define __WEAK_LLSC_MB " dbar 0 \n"
+#define __WEAK_LLSC_MB " dbar 0x700 \n"
#else
-#define __WEAK_LLSC_MB " \n"
+#define __WEAK_LLSC_MB " \n"
#endif
#define __smp_mb__before_atomic() barrier()
@@ -59,68 +88,19 @@ static inline unsigned long array_index_mask_nospec(unsigned long index,
return mask;
}
-#define __smp_load_acquire(p) \
-({ \
- union { typeof(*p) __val; char __c[1]; } __u; \
- unsigned long __tmp = 0; \
- compiletime_assert_atomic_type(*p); \
- switch (sizeof(*p)) { \
- case 1: \
- *(__u8 *)__u.__c = *(volatile __u8 *)p; \
- __smp_mb(); \
- break; \
- case 2: \
- *(__u16 *)__u.__c = *(volatile __u16 *)p; \
- __smp_mb(); \
- break; \
- case 4: \
- __asm__ __volatile__( \
- "amor_db.w %[val], %[tmp], %[mem] \n" \
- : [val] "=&r" (*(__u32 *)__u.__c) \
- : [mem] "ZB" (*(u32 *) p), [tmp] "r" (__tmp) \
- : "memory"); \
- break; \
- case 8: \
- __asm__ __volatile__( \
- "amor_db.d %[val], %[tmp], %[mem] \n" \
- : [val] "=&r" (*(__u64 *)__u.__c) \
- : [mem] "ZB" (*(u64 *) p), [tmp] "r" (__tmp) \
- : "memory"); \
- break; \
- } \
- (typeof(*p))__u.__val; \
+#define __smp_load_acquire(p) \
+({ \
+ typeof(*p) ___p1 = READ_ONCE(*p); \
+ compiletime_assert_atomic_type(*p); \
+ ldacq_mb(); \
+ ___p1; \
})
-#define __smp_store_release(p, v) \
-do { \
- union { typeof(*p) __val; char __c[1]; } __u = \
- { .__val = (__force typeof(*p)) (v) }; \
- unsigned long __tmp; \
- compiletime_assert_atomic_type(*p); \
- switch (sizeof(*p)) { \
- case 1: \
- __smp_mb(); \
- *(volatile __u8 *)p = *(__u8 *)__u.__c; \
- break; \
- case 2: \
- __smp_mb(); \
- *(volatile __u16 *)p = *(__u16 *)__u.__c; \
- break; \
- case 4: \
- __asm__ __volatile__( \
- "amswap_db.w %[tmp], %[val], %[mem] \n" \
- : [mem] "+ZB" (*(u32 *)p), [tmp] "=&r" (__tmp) \
- : [val] "r" (*(__u32 *)__u.__c) \
- : ); \
- break; \
- case 8: \
- __asm__ __volatile__( \
- "amswap_db.d %[tmp], %[val], %[mem] \n" \
- : [mem] "+ZB" (*(u64 *)p), [tmp] "=&r" (__tmp) \
- : [val] "r" (*(__u64 *)__u.__c) \
- : ); \
- break; \
- } \
+#define __smp_store_release(p, v) \
+do { \
+ compiletime_assert_atomic_type(*p); \
+ strel_mb(); \
+ WRITE_ONCE(*p, v); \
} while (0)
#define __smp_store_mb(p, v) \
diff --git a/arch/loongarch/include/asm/io.h b/arch/loongarch/include/asm/io.h
index 545e2708fbf7..1c9410220040 100644
--- a/arch/loongarch/include/asm/io.h
+++ b/arch/loongarch/include/asm/io.h
@@ -62,7 +62,7 @@ extern pgprot_t pgprot_wc;
#define ioremap_cache(offset, size) \
ioremap_prot((offset), (size), pgprot_val(PAGE_KERNEL))
-#define mmiowb() asm volatile ("dbar 0" ::: "memory")
+#define mmiowb() wmb()
/*
* String version of I/O memory access operations.
diff --git a/arch/loongarch/include/asm/qspinlock.h b/arch/loongarch/include/asm/qspinlock.h
new file mode 100644
index 000000000000..34f43f8ad591
--- /dev/null
+++ b/arch/loongarch/include/asm/qspinlock.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_QSPINLOCK_H
+#define _ASM_QSPINLOCK_H
+
+#include <asm-generic/qspinlock_types.h>
+
+#define queued_spin_unlock queued_spin_unlock
+
+static inline void queued_spin_unlock(struct qspinlock *lock)
+{
+ compiletime_assert_atomic_type(lock->locked);
+ c_sync();
+ WRITE_ONCE(lock->locked, 0);
+}
+
+#include <asm-generic/qspinlock.h>
+
+#endif /* _ASM_QSPINLOCK_H */
diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c
index 255967ff8c36..8ea1bbcf13a7 100644
--- a/arch/loongarch/kernel/smp.c
+++ b/arch/loongarch/kernel/smp.c
@@ -115,7 +115,7 @@ static u32 ipi_read_clear(int cpu)
action = iocsr_read32(LOONGARCH_IOCSR_IPI_STATUS);
/* Clear the ipi register to clear the interrupt */
iocsr_write32(action, LOONGARCH_IOCSR_IPI_CLEAR);
- smp_mb();
+ wbflush();
return action;
}
diff --git a/arch/loongarch/mm/tlbex.S b/arch/loongarch/mm/tlbex.S
index 244e2f5aeee5..240ced55586e 100644
--- a/arch/loongarch/mm/tlbex.S
+++ b/arch/loongarch/mm/tlbex.S
@@ -184,7 +184,7 @@ tlb_huge_update_load:
ertn
nopage_tlb_load:
- dbar 0
+ dbar 0x700
csrrd ra, EXCEPTION_KS2
la_abs t0, tlb_do_page_fault_0
jr t0
@@ -333,7 +333,7 @@ tlb_huge_update_store:
ertn
nopage_tlb_store:
- dbar 0
+ dbar 0x700
csrrd ra, EXCEPTION_KS2
la_abs t0, tlb_do_page_fault_1
jr t0
@@ -480,7 +480,7 @@ tlb_huge_update_modify:
ertn
nopage_tlb_modify:
- dbar 0
+ dbar 0x700
csrrd ra, EXCEPTION_KS2
la_abs t0, tlb_do_page_fault_1
jr t0
generated by cgit (git 2.34.1) at 2025-04-01 18:12:11 +0000