diff options
Diffstat (limited to 'arch/powerpc/include/asm/barrier.h')
| -rw-r--r-- | arch/powerpc/include/asm/barrier.h | 93 |
1 files changed, 72 insertions, 21 deletions
diff --git a/arch/powerpc/include/asm/barrier.h b/arch/powerpc/include/asm/barrier.h index ae782254e731..9e9833faa4af 100644 --- a/arch/powerpc/include/asm/barrier.h +++ b/arch/powerpc/include/asm/barrier.h @@ -1,9 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu> */ #ifndef _ASM_POWERPC_BARRIER_H #define _ASM_POWERPC_BARRIER_H +#include <asm/asm-const.h> + +#ifndef __ASSEMBLER__ +#include <asm/ppc-opcode.h> +#endif + /* * Memory barrier. * The sync instruction guarantees that all memory accesses initiated @@ -15,8 +22,6 @@ * mb() prevents loads and stores being reordered across this point. * rmb() prevents loads being reordered across this point. * wmb() prevents stores being reordered across this point. - * read_barrier_depends() prevents data-dependent loads being reordered - * across this point (nop on PPC). * * *mb() variants without smp_ prefix must order all types of memory * operations with one another. sync is the only instruction sufficient @@ -30,31 +35,30 @@ * However, on CPUs that don't support lwsync, lwsync actually maps to a * heavy-weight sync, so smp_wmb() can be a lighter-weight eieio. */ -#define mb() __asm__ __volatile__ ("sync" : : : "memory") -#define rmb() __asm__ __volatile__ ("sync" : : : "memory") -#define wmb() __asm__ __volatile__ ("sync" : : : "memory") -#define read_barrier_depends() do { } while(0) +#define __mb() __asm__ __volatile__ ("sync" : : : "memory") +#define __rmb() __asm__ __volatile__ ("sync" : : : "memory") +#define __wmb() __asm__ __volatile__ ("sync" : : : "memory") -#define set_mb(var, value) do { var = value; mb(); } while (0) - -#ifdef CONFIG_SMP - -#ifdef __SUBARCH_HAS_LWSYNC +/* The sub-arch has lwsync */ +#if defined(CONFIG_PPC64) || defined(CONFIG_PPC_E500MC) # define SMPWMB LWSYNC +#elif defined(CONFIG_BOOKE) +# define SMPWMB mbar #else # define SMPWMB eieio #endif -#define smp_mb() mb() -#define smp_rmb() __asm__ __volatile__ (stringify_in_c(LWSYNC) : : :"memory") -#define smp_wmb() __asm__ __volatile__ (stringify_in_c(SMPWMB) : : :"memory") -#define smp_read_barrier_depends() read_barrier_depends() -#else -#define smp_mb() barrier() -#define smp_rmb() barrier() -#define smp_wmb() barrier() -#define smp_read_barrier_depends() do { } while(0) -#endif /* CONFIG_SMP */ +/* clang defines this macro for a builtin, which will not work with runtime patching */ +#undef __lwsync +#define __lwsync() __asm__ __volatile__ (stringify_in_c(LWSYNC) : : :"memory") +#define __dma_rmb() __lwsync() +#define __dma_wmb() __asm__ __volatile__ (stringify_in_c(SMPWMB) : : :"memory") + +#define __smp_lwsync() __lwsync() + +#define __smp_mb() __mb() +#define __smp_rmb() __lwsync() +#define __smp_wmb() __asm__ __volatile__ (stringify_in_c(SMPWMB) : : :"memory") /* * This is a barrier which prevents following instructions from being @@ -65,4 +69,51 @@ #define data_barrier(x) \ asm volatile("twi 0,%0,0; isync" : : "r" (x) : "memory"); +#define __smp_store_release(p, v) \ +do { \ + compiletime_assert_atomic_type(*p); \ + __smp_lwsync(); \ + WRITE_ONCE(*p, v); \ +} while (0) + +#define __smp_load_acquire(p) \ +({ \ + typeof(*p) ___p1 = READ_ONCE(*p); \ + compiletime_assert_atomic_type(*p); \ + __smp_lwsync(); \ + ___p1; \ +}) + +#ifdef CONFIG_PPC_BOOK3S_64 +#define NOSPEC_BARRIER_SLOT nop +#elif defined(CONFIG_PPC_E500) +#define NOSPEC_BARRIER_SLOT nop; nop +#endif + +#ifdef CONFIG_PPC_BARRIER_NOSPEC +/* + * Prevent execution of subsequent instructions until preceding branches have + * been fully resolved and are no longer executing speculatively. + */ +#define barrier_nospec_asm NOSPEC_BARRIER_FIXUP_SECTION; NOSPEC_BARRIER_SLOT + +// This also acts as a compiler barrier due to the memory clobber. +#define barrier_nospec() asm (stringify_in_c(barrier_nospec_asm) ::: "memory") + +#else /* !CONFIG_PPC_BARRIER_NOSPEC */ +#define barrier_nospec_asm +#define barrier_nospec() +#endif /* CONFIG_PPC_BARRIER_NOSPEC */ + +/* + * pmem_wmb() ensures that all stores for which the modification + * are written to persistent storage by preceding dcbfps/dcbstps + * instructions have updated persistent storage before any data + * access or data transfer caused by subsequent instructions is + * initiated. + */ +#define pmem_wmb() __asm__ __volatile__(PPC_PHWSYNC ::: "memory") + +#include <asm-generic/barrier.h> + #endif /* _ASM_POWERPC_BARRIER_H */ |