diff options
Diffstat (limited to 'arch/powerpc/include')
478 files changed, 34025 insertions, 19334 deletions
diff --git a/arch/powerpc/include/asm/8xx_immap.h b/arch/powerpc/include/asm/8xx_immap.h index bdf0563ba423..f9cac46a95cb 100644 --- a/arch/powerpc/include/asm/8xx_immap.h +++ b/arch/powerpc/include/asm/8xx_immap.h @@ -560,5 +560,7 @@ typedef struct immap { cpm8xx_t im_cpm; /* Communication processor */ } immap_t; +extern immap_t __iomem *mpc8xx_immr; + #endif /* __IMMAP_8XX__ */ #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/Kbuild b/arch/powerpc/include/asm/Kbuild index 650757c300db..2e23533b67e3 100644 --- a/arch/powerpc/include/asm/Kbuild +++ b/arch/powerpc/include/asm/Kbuild @@ -1,4 +1,8 @@ - -generic-y += clkdev.h -generic-y += rwsem.h -generic-y += trace_clock.h +# SPDX-License-Identifier: GPL-2.0 +generated-y += syscall_table_32.h +generated-y += syscall_table_64.h +generated-y += syscall_table_spu.h +generic-y += agp.h +generic-y += mcs_spinlock.h +generic-y += qrwlock.h +generic-y += early_ioremap.h diff --git a/arch/powerpc/include/asm/accounting.h b/arch/powerpc/include/asm/accounting.h new file mode 100644 index 000000000000..6d79c31700e2 --- /dev/null +++ b/arch/powerpc/include/asm/accounting.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Common time accounting prototypes and such for all ppc machines. + */ + +#ifndef __POWERPC_ACCOUNTING_H +#define __POWERPC_ACCOUNTING_H + +/* Stuff for accurate time accounting */ +struct cpu_accounting_data { + /* Accumulated cputime values to flush on ticks*/ + unsigned long utime; + unsigned long stime; +#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME + unsigned long utime_scaled; + unsigned long stime_scaled; +#endif + unsigned long gtime; + unsigned long hardirq_time; + unsigned long softirq_time; + unsigned long steal_time; + unsigned long idle_time; + /* Internal counters */ + unsigned long starttime; /* TB value snapshot */ + unsigned long starttime_user; /* TB value on exit to usermode */ +#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME + unsigned long startspurr; /* SPURR value snapshot */ + unsigned long utime_sspurr; /* ->user_time when ->startspurr set */ +#endif +}; + +#endif diff --git a/arch/powerpc/include/asm/agp.h b/arch/powerpc/include/asm/agp.h deleted file mode 100644 index 416e12c2d505..000000000000 --- a/arch/powerpc/include/asm/agp.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef _ASM_POWERPC_AGP_H -#define _ASM_POWERPC_AGP_H -#ifdef __KERNEL__ - -#include <asm/io.h> - -#define map_page_into_agp(page) -#define unmap_page_from_agp(page) -#define flush_agp_cache() mb() - -/* GATT allocation. Returns/accepts GATT kernel virtual address. */ -#define alloc_gatt_pages(order) \ - ((char *)__get_free_pages(GFP_KERNEL, (order))) -#define free_gatt_pages(table, order) \ - free_pages((unsigned long)(table), (order)) - -#endif /* __KERNEL__ */ -#endif /* _ASM_POWERPC_AGP_H */ diff --git a/arch/powerpc/include/asm/archrandom.h b/arch/powerpc/include/asm/archrandom.h new file mode 100644 index 000000000000..51b093f67528 --- /dev/null +++ b/arch/powerpc/include/asm/archrandom.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_ARCHRANDOM_H +#define _ASM_POWERPC_ARCHRANDOM_H + +static inline size_t __must_check arch_get_random_longs(unsigned long *v, size_t max_longs) +{ + return 0; +} + +size_t __must_check arch_get_random_seed_longs(unsigned long *v, size_t max_longs); + +#ifdef CONFIG_PPC_POWERNV +int pnv_get_random_long(unsigned long *v); +#endif + +#endif /* _ASM_POWERPC_ARCHRANDOM_H */ diff --git a/arch/powerpc/include/asm/asm-compat.h b/arch/powerpc/include/asm/asm-compat.h index 6e82f5f9a6fd..f48e644900a2 100644 --- a/arch/powerpc/include/asm/asm-compat.h +++ b/arch/powerpc/include/asm/asm-compat.h @@ -1,21 +1,10 @@ #ifndef _ASM_POWERPC_ASM_COMPAT_H #define _ASM_POWERPC_ASM_COMPAT_H +#include <asm/asm-const.h> #include <asm/types.h> #include <asm/ppc-opcode.h> -#ifdef __ASSEMBLY__ -# define stringify_in_c(...) __VA_ARGS__ -# define ASM_CONST(x) x -#else -/* This version of stringify will deal with commas... */ -# define __stringify_in_c(...) #__VA_ARGS__ -# define stringify_in_c(...) __stringify_in_c(__VA_ARGS__) " " -# define __ASM_CONST(x) x##UL -# define ASM_CONST(x) __ASM_CONST(x) -#endif - - #ifdef __powerpc64__ /* operations for longs and pointers */ @@ -23,15 +12,37 @@ #define PPC_STL stringify_in_c(std) #define PPC_STLU stringify_in_c(stdu) #define PPC_LCMPI stringify_in_c(cmpdi) -#define PPC_LONG stringify_in_c(.llong) +#define PPC_LCMPLI stringify_in_c(cmpldi) +#define PPC_LCMP stringify_in_c(cmpd) +#define PPC_LONG stringify_in_c(.8byte) #define PPC_LONG_ALIGN stringify_in_c(.balign 8) #define PPC_TLNEI stringify_in_c(tdnei) -#define PPC_LLARX(t, a, b, eh) PPC_LDARX(t, a, b, eh) +#define PPC_LLARX stringify_in_c(ldarx) #define PPC_STLCX stringify_in_c(stdcx.) #define PPC_CNTLZL stringify_in_c(cntlzd) #define PPC_MTOCRF(FXM, RS) MTOCRF((FXM), RS) +#define PPC_SRL stringify_in_c(srd) #define PPC_LR_STKOFF 16 #define PPC_MIN_STKFRM 112 + +#ifdef __BIG_ENDIAN__ +#define LWZX_BE stringify_in_c(lwzx) +#define LDX_BE stringify_in_c(ldx) +#define STWX_BE stringify_in_c(stwx) +#define STDX_BE stringify_in_c(stdx) +#else +#define LWZX_BE stringify_in_c(lwbrx) +#define LDX_BE stringify_in_c(ldbrx) +#define STWX_BE stringify_in_c(stwbrx) +#define STDX_BE stringify_in_c(stdbrx) +#endif + +#ifdef CONFIG_CC_IS_CLANG +#define DS_FORM_CONSTRAINT "Z<>" +#else +#define DS_FORM_CONSTRAINT "YZ<>" +#endif + #else /* 32-bit */ /* operations for longs and pointers */ @@ -39,29 +50,19 @@ #define PPC_STL stringify_in_c(stw) #define PPC_STLU stringify_in_c(stwu) #define PPC_LCMPI stringify_in_c(cmpwi) +#define PPC_LCMPLI stringify_in_c(cmplwi) +#define PPC_LCMP stringify_in_c(cmpw) #define PPC_LONG stringify_in_c(.long) #define PPC_LONG_ALIGN stringify_in_c(.balign 4) #define PPC_TLNEI stringify_in_c(twnei) -#define PPC_LLARX(t, a, b, eh) PPC_LWARX(t, a, b, eh) +#define PPC_LLARX stringify_in_c(lwarx) #define PPC_STLCX stringify_in_c(stwcx.) #define PPC_CNTLZL stringify_in_c(cntlzw) #define PPC_MTOCRF stringify_in_c(mtcrf) +#define PPC_SRL stringify_in_c(srw) #define PPC_LR_STKOFF 4 #define PPC_MIN_STKFRM 16 #endif -#ifdef __KERNEL__ -#ifdef CONFIG_IBM405_ERR77 -/* Erratum #77 on the 405 means we need a sync or dcbt before every - * stwcx. The old ATOMIC_SYNC_FIX covered some but not all of this. - */ -#define PPC405_ERR77(ra,rb) stringify_in_c(dcbt ra, rb;) -#define PPC405_ERR77_SYNC stringify_in_c(sync;) -#else -#define PPC405_ERR77(ra,rb) -#define PPC405_ERR77_SYNC -#endif -#endif - #endif /* _ASM_POWERPC_ASM_COMPAT_H */ diff --git a/arch/powerpc/include/asm/asm-const.h b/arch/powerpc/include/asm/asm-const.h new file mode 100644 index 000000000000..392bdb1f104f --- /dev/null +++ b/arch/powerpc/include/asm/asm-const.h @@ -0,0 +1,15 @@ +#ifndef _ASM_POWERPC_ASM_CONST_H +#define _ASM_POWERPC_ASM_CONST_H + +#ifdef __ASSEMBLER__ +# define stringify_in_c(...) __VA_ARGS__ +# define ASM_CONST(x) x +#else +/* This version of stringify will deal with commas... */ +# define __stringify_in_c(...) #__VA_ARGS__ +# define stringify_in_c(...) __stringify_in_c(__VA_ARGS__) " " +# define __ASM_CONST(x) x##UL +# define ASM_CONST(x) __ASM_CONST(x) +#endif + +#endif /* _ASM_POWERPC_ASM_CONST_H */ diff --git a/arch/powerpc/include/asm/asm-prototypes.h b/arch/powerpc/include/asm/asm-prototypes.h new file mode 100644 index 000000000000..274bce76f5da --- /dev/null +++ b/arch/powerpc/include/asm/asm-prototypes.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_POWERPC_ASM_PROTOTYPES_H +#define _ASM_POWERPC_ASM_PROTOTYPES_H +/* + * This file is for C prototypes of asm symbols that are EXPORTed. + * It allows the modversions logic to see their prototype and + * generate proper CRCs for them. + * + * Copyright 2016, Daniel Axtens, IBM Corporation. + */ + +#include <linux/threads.h> +#include <asm/cacheflush.h> +#include <asm/checksum.h> +#include <linux/uaccess.h> +#include <asm/epapr_hcalls.h> +#include <asm/dcr.h> +#include <asm/mmu_context.h> +#include <asm/ultravisor-api.h> + +#include <uapi/asm/ucontext.h> + +/* Ultravisor */ +#if defined(CONFIG_PPC_POWERNV) || defined(CONFIG_PPC_SVM) +long ucall_norets(unsigned long opcode, ...); +#else +static inline long ucall_norets(unsigned long opcode, ...) +{ + return U_NOT_AVAILABLE; +} +#endif + +/* OPAL */ +int64_t __opal_call(int64_t a0, int64_t a1, int64_t a2, int64_t a3, + int64_t a4, int64_t a5, int64_t a6, int64_t a7, + int64_t opcode, uint64_t msr); + +/* misc runtime */ +void enable_machine_check(void); +extern u64 __bswapdi2(u64); +extern s64 __lshrdi3(s64, int); +extern s64 __ashldi3(s64, int); +extern s64 __ashrdi3(s64, int); +extern int __cmpdi2(s64, s64); +extern int __ucmpdi2(u64, u64); + +/* tracing */ +void _mcount(void); + +/* Transaction memory related */ +void tm_enable(void); +void tm_disable(void); +void tm_abort(uint8_t cause); + +struct kvm_vcpu; +void _kvmppc_restore_tm_pr(struct kvm_vcpu *vcpu, u64 guest_msr); +void _kvmppc_save_tm_pr(struct kvm_vcpu *vcpu, u64 guest_msr); + +#ifdef CONFIG_PPC_TRANSACTIONAL_MEM +void kvmppc_save_tm_hv(struct kvm_vcpu *vcpu, u64 msr, bool preserve_nv); +void kvmppc_restore_tm_hv(struct kvm_vcpu *vcpu, u64 msr, bool preserve_nv); +#else +static inline void kvmppc_save_tm_hv(struct kvm_vcpu *vcpu, u64 msr, + bool preserve_nv) { } +static inline void kvmppc_restore_tm_hv(struct kvm_vcpu *vcpu, u64 msr, + bool preserve_nv) { } +#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ + +void kvmppc_p9_enter_guest(struct kvm_vcpu *vcpu); + +long kvmppc_h_set_dabr(struct kvm_vcpu *vcpu, unsigned long dabr); +long kvmppc_h_set_xdabr(struct kvm_vcpu *vcpu, unsigned long dabr, + unsigned long dabrx); + +#endif /* _ASM_POWERPC_ASM_PROTOTYPES_H */ diff --git a/arch/powerpc/include/asm/asm.h b/arch/powerpc/include/asm/asm.h new file mode 100644 index 000000000000..86f46b604e9a --- /dev/null +++ b/arch/powerpc/include/asm/asm.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_ASM_H +#define _ASM_POWERPC_ASM_H + +#define _ASM_PTR " .long " + +#endif /* _ASM_POWERPC_ASM_H */ diff --git a/arch/powerpc/include/asm/async_tx.h b/arch/powerpc/include/asm/async_tx.h index 8b2dc55d01ab..a14758426dd5 100644 --- a/arch/powerpc/include/asm/async_tx.h +++ b/arch/powerpc/include/asm/async_tx.h @@ -1,24 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Copyright (C) 2008-2009 DENX Software Engineering. * * Author: Yuri Tikhonov <yur@emcraft.com> - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) - * any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., 59 - * Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * - * The full GNU General Public License is included in this distribution in the - * file called COPYING. */ #ifndef _ASM_POWERPC_ASYNC_TX_H_ #define _ASM_POWERPC_ASYNC_TX_H_ diff --git a/arch/powerpc/include/asm/atomic.h b/arch/powerpc/include/asm/atomic.h index e3b1d41c89be..d1ea554c33ed 100644 --- a/arch/powerpc/include/asm/atomic.h +++ b/arch/powerpc/include/asm/atomic.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_ATOMIC_H_ #define _ASM_POWERPC_ATOMIC_H_ @@ -8,176 +9,126 @@ #ifdef __KERNEL__ #include <linux/types.h> #include <asm/cmpxchg.h> +#include <asm/barrier.h> +#include <asm/asm-const.h> +#include <asm/asm-compat.h> -#define ATOMIC_INIT(i) { (i) } - -static __inline__ int atomic_read(const atomic_t *v) -{ - int t; - - __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); - - return t; -} +/* + * Since *_return_relaxed and {cmp}xchg_relaxed are implemented with + * a "bne-" instruction at the end, so an isync is enough as a acquire barrier + * on the platform without lwsync. + */ +#define __atomic_acquire_fence() \ + __asm__ __volatile__(PPC_ACQUIRE_BARRIER "" : : : "memory") -static __inline__ void atomic_set(atomic_t *v, int i) -{ - __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); -} +#define __atomic_release_fence() \ + __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory") -static __inline__ void atomic_add(int a, atomic_t *v) +static __inline__ int arch_atomic_read(const atomic_t *v) { int t; - __asm__ __volatile__( -"1: lwarx %0,0,%3 # atomic_add\n\ - add %0,%2,%0\n" - PPC405_ERR77(0,%3) -" stwcx. %0,0,%3 \n\ - bne- 1b" - : "=&r" (t), "+m" (v->counter) - : "r" (a), "r" (&v->counter) - : "cc"); -} - -static __inline__ int atomic_add_return(int a, atomic_t *v) -{ - int t; - - __asm__ __volatile__( - PPC_ATOMIC_ENTRY_BARRIER -"1: lwarx %0,0,%2 # atomic_add_return\n\ - add %0,%1,%0\n" - PPC405_ERR77(0,%2) -" stwcx. %0,0,%2 \n\ - bne- 1b" - PPC_ATOMIC_EXIT_BARRIER - : "=&r" (t) - : "r" (a), "r" (&v->counter) - : "cc", "memory"); + /* -mprefixed can generate offsets beyond range, fall back hack */ + if (IS_ENABLED(CONFIG_PPC_KERNEL_PREFIXED)) + __asm__ __volatile__("lwz %0,0(%1)" : "=r"(t) : "b"(&v->counter)); + else + __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m<>"(v->counter)); return t; } -#define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) - -static __inline__ void atomic_sub(int a, atomic_t *v) +static __inline__ void arch_atomic_set(atomic_t *v, int i) { - int t; - - __asm__ __volatile__( -"1: lwarx %0,0,%3 # atomic_sub\n\ - subf %0,%2,%0\n" - PPC405_ERR77(0,%3) -" stwcx. %0,0,%3 \n\ - bne- 1b" - : "=&r" (t), "+m" (v->counter) - : "r" (a), "r" (&v->counter) - : "cc"); + /* -mprefixed can generate offsets beyond range, fall back hack */ + if (IS_ENABLED(CONFIG_PPC_KERNEL_PREFIXED)) + __asm__ __volatile__("stw %1,0(%2)" : "=m"(v->counter) : "r"(i), "b"(&v->counter)); + else + __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m<>"(v->counter) : "r"(i)); } -static __inline__ int atomic_sub_return(int a, atomic_t *v) -{ - int t; - - __asm__ __volatile__( - PPC_ATOMIC_ENTRY_BARRIER -"1: lwarx %0,0,%2 # atomic_sub_return\n\ - subf %0,%1,%0\n" - PPC405_ERR77(0,%2) -" stwcx. %0,0,%2 \n\ - bne- 1b" - PPC_ATOMIC_EXIT_BARRIER - : "=&r" (t) - : "r" (a), "r" (&v->counter) - : "cc", "memory"); - - return t; +#define ATOMIC_OP(op, asm_op, suffix, sign, ...) \ +static __inline__ void arch_atomic_##op(int a, atomic_t *v) \ +{ \ + int t; \ + \ + __asm__ __volatile__( \ +"1: lwarx %0,0,%3 # atomic_" #op "\n" \ + #asm_op "%I2" suffix " %0,%0,%2\n" \ +" stwcx. %0,0,%3 \n" \ +" bne- 1b\n" \ + : "=&r" (t), "+m" (v->counter) \ + : "r"#sign (a), "r" (&v->counter) \ + : "cc", ##__VA_ARGS__); \ +} \ + +#define ATOMIC_OP_RETURN_RELAXED(op, asm_op, suffix, sign, ...) \ +static inline int arch_atomic_##op##_return_relaxed(int a, atomic_t *v) \ +{ \ + int t; \ + \ + __asm__ __volatile__( \ +"1: lwarx %0,0,%3 # atomic_" #op "_return_relaxed\n" \ + #asm_op "%I2" suffix " %0,%0,%2\n" \ +" stwcx. %0,0,%3\n" \ +" bne- 1b\n" \ + : "=&r" (t), "+m" (v->counter) \ + : "r"#sign (a), "r" (&v->counter) \ + : "cc", ##__VA_ARGS__); \ + \ + return t; \ } -static __inline__ void atomic_inc(atomic_t *v) -{ - int t; - - __asm__ __volatile__( -"1: lwarx %0,0,%2 # atomic_inc\n\ - addic %0,%0,1\n" - PPC405_ERR77(0,%2) -" stwcx. %0,0,%2 \n\ - bne- 1b" - : "=&r" (t), "+m" (v->counter) - : "r" (&v->counter) - : "cc", "xer"); +#define ATOMIC_FETCH_OP_RELAXED(op, asm_op, suffix, sign, ...) \ +static inline int arch_atomic_fetch_##op##_relaxed(int a, atomic_t *v) \ +{ \ + int res, t; \ + \ + __asm__ __volatile__( \ +"1: lwarx %0,0,%4 # atomic_fetch_" #op "_relaxed\n" \ + #asm_op "%I3" suffix " %1,%0,%3\n" \ +" stwcx. %1,0,%4\n" \ +" bne- 1b\n" \ + : "=&r" (res), "=&r" (t), "+m" (v->counter) \ + : "r"#sign (a), "r" (&v->counter) \ + : "cc", ##__VA_ARGS__); \ + \ + return res; \ } -static __inline__ int atomic_inc_return(atomic_t *v) -{ - int t; - - __asm__ __volatile__( - PPC_ATOMIC_ENTRY_BARRIER -"1: lwarx %0,0,%1 # atomic_inc_return\n\ - addic %0,%0,1\n" - PPC405_ERR77(0,%1) -" stwcx. %0,0,%1 \n\ - bne- 1b" - PPC_ATOMIC_EXIT_BARRIER - : "=&r" (t) - : "r" (&v->counter) - : "cc", "xer", "memory"); - - return t; -} +#define ATOMIC_OPS(op, asm_op, suffix, sign, ...) \ + ATOMIC_OP(op, asm_op, suffix, sign, ##__VA_ARGS__) \ + ATOMIC_OP_RETURN_RELAXED(op, asm_op, suffix, sign, ##__VA_ARGS__)\ + ATOMIC_FETCH_OP_RELAXED(op, asm_op, suffix, sign, ##__VA_ARGS__) -/* - * atomic_inc_and_test - increment and test - * @v: pointer of type atomic_t - * - * Atomically increments @v by 1 - * and returns true if the result is zero, or false for all - * other cases. - */ -#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) +ATOMIC_OPS(add, add, "c", I, "xer") +ATOMIC_OPS(sub, sub, "c", I, "xer") -static __inline__ void atomic_dec(atomic_t *v) -{ - int t; +#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed +#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed - __asm__ __volatile__( -"1: lwarx %0,0,%2 # atomic_dec\n\ - addic %0,%0,-1\n" - PPC405_ERR77(0,%2)\ -" stwcx. %0,0,%2\n\ - bne- 1b" - : "=&r" (t), "+m" (v->counter) - : "r" (&v->counter) - : "cc", "xer"); -} +#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed +#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed -static __inline__ int atomic_dec_return(atomic_t *v) -{ - int t; +#undef ATOMIC_OPS +#define ATOMIC_OPS(op, asm_op, suffix, sign) \ + ATOMIC_OP(op, asm_op, suffix, sign) \ + ATOMIC_FETCH_OP_RELAXED(op, asm_op, suffix, sign) - __asm__ __volatile__( - PPC_ATOMIC_ENTRY_BARRIER -"1: lwarx %0,0,%1 # atomic_dec_return\n\ - addic %0,%0,-1\n" - PPC405_ERR77(0,%1) -" stwcx. %0,0,%1\n\ - bne- 1b" - PPC_ATOMIC_EXIT_BARRIER - : "=&r" (t) - : "r" (&v->counter) - : "cc", "xer", "memory"); +ATOMIC_OPS(and, and, ".", K) +ATOMIC_OPS(or, or, "", K) +ATOMIC_OPS(xor, xor, "", K) - return t; -} +#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed +#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed +#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed -#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) -#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) +#undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP_RELAXED +#undef ATOMIC_OP_RETURN_RELAXED +#undef ATOMIC_OP /** - * __atomic_add_unless - add unless the number is a given value + * atomic_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic_t * @a: the amount to add to v... * @u: ...unless v is equal to u. @@ -185,69 +136,35 @@ static __inline__ int atomic_dec_return(atomic_t *v) * Atomically adds @a to @v, so long as it was not @u. * Returns the old value of @v. */ -static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) +static __inline__ int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) { int t; __asm__ __volatile__ ( PPC_ATOMIC_ENTRY_BARRIER -"1: lwarx %0,0,%1 # __atomic_add_unless\n\ +"1: lwarx %0,0,%1 # atomic_fetch_add_unless\n\ cmpw 0,%0,%3 \n\ - beq- 2f \n\ - add %0,%2,%0 \n" - PPC405_ERR77(0,%2) + beq 2f \n\ + add%I2c %0,%0,%2 \n" " stwcx. %0,0,%1 \n\ bne- 1b \n" PPC_ATOMIC_EXIT_BARRIER -" subf %0,%2,%0 \n\ +" sub%I2c %0,%0,%2 \n\ 2:" : "=&r" (t) - : "r" (&v->counter), "r" (a), "r" (u) - : "cc", "memory"); + : "r" (&v->counter), "rI" (a), "r" (u) + : "cc", "memory", "xer"); return t; } - -/** - * atomic_inc_not_zero - increment unless the number is zero - * @v: pointer of type atomic_t - * - * Atomically increments @v by 1, so long as @v is non-zero. - * Returns non-zero if @v was non-zero, and zero otherwise. - */ -static __inline__ int atomic_inc_not_zero(atomic_t *v) -{ - int t1, t2; - - __asm__ __volatile__ ( - PPC_ATOMIC_ENTRY_BARRIER -"1: lwarx %0,0,%2 # atomic_inc_not_zero\n\ - cmpwi 0,%0,0\n\ - beq- 2f\n\ - addic %1,%0,1\n" - PPC405_ERR77(0,%2) -" stwcx. %1,0,%2\n\ - bne- 1b\n" - PPC_ATOMIC_EXIT_BARRIER - "\n\ -2:" - : "=&r" (t1), "=&r" (t2) - : "r" (&v->counter) - : "cc", "xer", "memory"); - - return t1; -} -#define atomic_inc_not_zero(v) atomic_inc_not_zero((v)) - -#define atomic_sub_and_test(a, v) (atomic_sub_return((a), (v)) == 0) -#define atomic_dec_and_test(v) (atomic_dec_return((v)) == 0) +#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless /* * Atomically test *v and decrement if it is greater than 0. * The function returns the old value of *v minus 1, even if * the atomic variable, v, was not decremented. */ -static __inline__ int atomic_dec_if_positive(atomic_t *v) +static __inline__ int arch_atomic_dec_if_positive(atomic_t *v) { int t; @@ -257,7 +174,6 @@ static __inline__ int atomic_dec_if_positive(atomic_t *v) cmpwi %0,1\n\ addi %0,%0,-1\n\ blt- 2f\n" - PPC405_ERR77(0,%1) " stwcx. %0,0,%1\n\ bne- 1b" PPC_ATOMIC_EXIT_BARRIER @@ -268,100 +184,120 @@ static __inline__ int atomic_dec_if_positive(atomic_t *v) return t; } -#define atomic_dec_if_positive atomic_dec_if_positive - -#define smp_mb__before_atomic_dec() smp_mb() -#define smp_mb__after_atomic_dec() smp_mb() -#define smp_mb__before_atomic_inc() smp_mb() -#define smp_mb__after_atomic_inc() smp_mb() +#define arch_atomic_dec_if_positive arch_atomic_dec_if_positive #ifdef __powerpc64__ #define ATOMIC64_INIT(i) { (i) } -static __inline__ long atomic64_read(const atomic64_t *v) +static __inline__ s64 arch_atomic64_read(const atomic64_t *v) { - long t; + s64 t; - __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); + /* -mprefixed can generate offsets beyond range, fall back hack */ + if (IS_ENABLED(CONFIG_PPC_KERNEL_PREFIXED)) + __asm__ __volatile__("ld %0,0(%1)" : "=r"(t) : "b"(&v->counter)); + else + __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : DS_FORM_CONSTRAINT (v->counter)); return t; } -static __inline__ void atomic64_set(atomic64_t *v, long i) +static __inline__ void arch_atomic64_set(atomic64_t *v, s64 i) { - __asm__ __volatile__("std%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); + /* -mprefixed can generate offsets beyond range, fall back hack */ + if (IS_ENABLED(CONFIG_PPC_KERNEL_PREFIXED)) + __asm__ __volatile__("std %1,0(%2)" : "=m"(v->counter) : "r"(i), "b"(&v->counter)); + else + __asm__ __volatile__("std%U0%X0 %1,%0" : "=" DS_FORM_CONSTRAINT (v->counter) : "r"(i)); } -static __inline__ void atomic64_add(long a, atomic64_t *v) -{ - long t; +#define ATOMIC64_OP(op, asm_op) \ +static __inline__ void arch_atomic64_##op(s64 a, atomic64_t *v) \ +{ \ + s64 t; \ + \ + __asm__ __volatile__( \ +"1: ldarx %0,0,%3 # atomic64_" #op "\n" \ + #asm_op " %0,%2,%0\n" \ +" stdcx. %0,0,%3 \n" \ +" bne- 1b\n" \ + : "=&r" (t), "+m" (v->counter) \ + : "r" (a), "r" (&v->counter) \ + : "cc"); \ +} - __asm__ __volatile__( -"1: ldarx %0,0,%3 # atomic64_add\n\ - add %0,%2,%0\n\ - stdcx. %0,0,%3 \n\ - bne- 1b" - : "=&r" (t), "+m" (v->counter) - : "r" (a), "r" (&v->counter) - : "cc"); +#define ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ +static inline s64 \ +arch_atomic64_##op##_return_relaxed(s64 a, atomic64_t *v) \ +{ \ + s64 t; \ + \ + __asm__ __volatile__( \ +"1: ldarx %0,0,%3 # atomic64_" #op "_return_relaxed\n" \ + #asm_op " %0,%2,%0\n" \ +" stdcx. %0,0,%3\n" \ +" bne- 1b\n" \ + : "=&r" (t), "+m" (v->counter) \ + : "r" (a), "r" (&v->counter) \ + : "cc"); \ + \ + return t; \ } -static __inline__ long atomic64_add_return(long a, atomic64_t *v) -{ - long t; +#define ATOMIC64_FETCH_OP_RELAXED(op, asm_op) \ +static inline s64 \ +arch_atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v) \ +{ \ + s64 res, t; \ + \ + __asm__ __volatile__( \ +"1: ldarx %0,0,%4 # atomic64_fetch_" #op "_relaxed\n" \ + #asm_op " %1,%3,%0\n" \ +" stdcx. %1,0,%4\n" \ +" bne- 1b\n" \ + : "=&r" (res), "=&r" (t), "+m" (v->counter) \ + : "r" (a), "r" (&v->counter) \ + : "cc"); \ + \ + return res; \ +} - __asm__ __volatile__( - PPC_ATOMIC_ENTRY_BARRIER -"1: ldarx %0,0,%2 # atomic64_add_return\n\ - add %0,%1,%0\n\ - stdcx. %0,0,%2 \n\ - bne- 1b" - PPC_ATOMIC_EXIT_BARRIER - : "=&r" (t) - : "r" (a), "r" (&v->counter) - : "cc", "memory"); +#define ATOMIC64_OPS(op, asm_op) \ + ATOMIC64_OP(op, asm_op) \ + ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ + ATOMIC64_FETCH_OP_RELAXED(op, asm_op) - return t; -} +ATOMIC64_OPS(add, add) +ATOMIC64_OPS(sub, subf) -#define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0) +#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed +#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed -static __inline__ void atomic64_sub(long a, atomic64_t *v) -{ - long t; +#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed +#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed - __asm__ __volatile__( -"1: ldarx %0,0,%3 # atomic64_sub\n\ - subf %0,%2,%0\n\ - stdcx. %0,0,%3 \n\ - bne- 1b" - : "=&r" (t), "+m" (v->counter) - : "r" (a), "r" (&v->counter) - : "cc"); -} +#undef ATOMIC64_OPS +#define ATOMIC64_OPS(op, asm_op) \ + ATOMIC64_OP(op, asm_op) \ + ATOMIC64_FETCH_OP_RELAXED(op, asm_op) -static __inline__ long atomic64_sub_return(long a, atomic64_t *v) -{ - long t; +ATOMIC64_OPS(and, and) +ATOMIC64_OPS(or, or) +ATOMIC64_OPS(xor, xor) - __asm__ __volatile__( - PPC_ATOMIC_ENTRY_BARRIER -"1: ldarx %0,0,%2 # atomic64_sub_return\n\ - subf %0,%1,%0\n\ - stdcx. %0,0,%2 \n\ - bne- 1b" - PPC_ATOMIC_EXIT_BARRIER - : "=&r" (t) - : "r" (a), "r" (&v->counter) - : "cc", "memory"); +#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed +#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed +#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed - return t; -} +#undef ATOPIC64_OPS +#undef ATOMIC64_FETCH_OP_RELAXED +#undef ATOMIC64_OP_RETURN_RELAXED +#undef ATOMIC64_OP -static __inline__ void atomic64_inc(atomic64_t *v) +static __inline__ void arch_atomic64_inc(atomic64_t *v) { - long t; + s64 t; __asm__ __volatile__( "1: ldarx %0,0,%2 # atomic64_inc\n\ @@ -372,38 +308,27 @@ static __inline__ void atomic64_inc(atomic64_t *v) : "r" (&v->counter) : "cc", "xer"); } +#define arch_atomic64_inc arch_atomic64_inc -static __inline__ long atomic64_inc_return(atomic64_t *v) +static __inline__ s64 arch_atomic64_inc_return_relaxed(atomic64_t *v) { - long t; + s64 t; __asm__ __volatile__( - PPC_ATOMIC_ENTRY_BARRIER -"1: ldarx %0,0,%1 # atomic64_inc_return\n\ - addic %0,%0,1\n\ - stdcx. %0,0,%1 \n\ - bne- 1b" - PPC_ATOMIC_EXIT_BARRIER - : "=&r" (t) +"1: ldarx %0,0,%2 # atomic64_inc_return_relaxed\n" +" addic %0,%0,1\n" +" stdcx. %0,0,%2\n" +" bne- 1b" + : "=&r" (t), "+m" (v->counter) : "r" (&v->counter) - : "cc", "xer", "memory"); + : "cc", "xer"); return t; } -/* - * atomic64_inc_and_test - increment and test - * @v: pointer of type atomic64_t - * - * Atomically increments @v by 1 - * and returns true if the result is zero, or false for all - * other cases. - */ -#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0) - -static __inline__ void atomic64_dec(atomic64_t *v) +static __inline__ void arch_atomic64_dec(atomic64_t *v) { - long t; + s64 t; __asm__ __volatile__( "1: ldarx %0,0,%2 # atomic64_dec\n\ @@ -414,35 +339,34 @@ static __inline__ void atomic64_dec(atomic64_t *v) : "r" (&v->counter) : "cc", "xer"); } +#define arch_atomic64_dec arch_atomic64_dec -static __inline__ long atomic64_dec_return(atomic64_t *v) +static __inline__ s64 arch_atomic64_dec_return_relaxed(atomic64_t *v) { - long t; + s64 t; __asm__ __volatile__( - PPC_ATOMIC_ENTRY_BARRIER -"1: ldarx %0,0,%1 # atomic64_dec_return\n\ - addic %0,%0,-1\n\ - stdcx. %0,0,%1\n\ - bne- 1b" - PPC_ATOMIC_EXIT_BARRIER - : "=&r" (t) +"1: ldarx %0,0,%2 # atomic64_dec_return_relaxed\n" +" addic %0,%0,-1\n" +" stdcx. %0,0,%2\n" +" bne- 1b" + : "=&r" (t), "+m" (v->counter) : "r" (&v->counter) - : "cc", "xer", "memory"); + : "cc", "xer"); return t; } -#define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0) -#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0) +#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed +#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed /* * Atomically test *v and decrement if it is greater than 0. * The function returns the old value of *v minus 1. */ -static __inline__ long atomic64_dec_if_positive(atomic64_t *v) +static __inline__ s64 arch_atomic64_dec_if_positive(atomic64_t *v) { - long t; + s64 t; __asm__ __volatile__( PPC_ATOMIC_ENTRY_BARRIER @@ -459,12 +383,10 @@ static __inline__ long atomic64_dec_if_positive(atomic64_t *v) return t; } - -#define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) -#define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) +#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive /** - * atomic64_add_unless - add unless the number is a given value + * atomic64_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic64_t * @a: the amount to add to v... * @u: ...unless v is equal to u. @@ -472,15 +394,15 @@ static __inline__ long atomic64_dec_if_positive(atomic64_t *v) * Atomically adds @a to @v, so long as it was not @u. * Returns the old value of @v. */ -static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) +static __inline__ s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) { - long t; + s64 t; __asm__ __volatile__ ( PPC_ATOMIC_ENTRY_BARRIER -"1: ldarx %0,0,%1 # __atomic_add_unless\n\ +"1: ldarx %0,0,%1 # atomic64_fetch_add_unless\n\ cmpd 0,%0,%3 \n\ - beq- 2f \n\ + beq 2f \n\ add %0,%2,%0 \n" " stdcx. %0,0,%1 \n\ bne- 1b \n" @@ -491,8 +413,9 @@ static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) : "r" (&v->counter), "r" (a), "r" (u) : "cc", "memory"); - return t != u; + return t; } +#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless /** * atomic_inc64_not_zero - increment unless the number is zero @@ -501,9 +424,9 @@ static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) * Atomically increments @v by 1, so long as @v is non-zero. * Returns non-zero if @v was non-zero, and zero otherwise. */ -static __inline__ long atomic64_inc_not_zero(atomic64_t *v) +static __inline__ int arch_atomic64_inc_not_zero(atomic64_t *v) { - long t1, t2; + s64 t1, t2; __asm__ __volatile__ ( PPC_ATOMIC_ENTRY_BARRIER @@ -520,8 +443,9 @@ static __inline__ long atomic64_inc_not_zero(atomic64_t *v) : "r" (&v->counter) : "cc", "xer", "memory"); - return t1; + return t1 != 0; } +#define arch_atomic64_inc_not_zero(v) arch_atomic64_inc_not_zero((v)) #endif /* __powerpc64__ */ diff --git a/arch/powerpc/include/asm/backlight.h b/arch/powerpc/include/asm/backlight.h index 8cf5c37c3817..061a910d7492 100644 --- a/arch/powerpc/include/asm/backlight.h +++ b/arch/powerpc/include/asm/backlight.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Routines for handling backlight control on PowerBooks * @@ -9,15 +10,14 @@ #define __ASM_POWERPC_BACKLIGHT_H #ifdef __KERNEL__ -#include <linux/fb.h> #include <linux/mutex.h> +struct backlight_device; + /* For locking instructions, see the implementation file */ extern struct backlight_device *pmac_backlight; extern struct mutex pmac_backlight_mutex; -extern int pmac_backlight_curve_lookup(struct fb_info *info, int value); - extern int pmac_has_backlight_type(const char *type); extern void pmac_backlight_key(int direction); diff --git a/arch/powerpc/include/asm/barrier.h b/arch/powerpc/include/asm/barrier.h index ae782254e731..9e9833faa4af 100644 --- a/arch/powerpc/include/asm/barrier.h +++ b/arch/powerpc/include/asm/barrier.h @@ -1,9 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu> */ #ifndef _ASM_POWERPC_BARRIER_H #define _ASM_POWERPC_BARRIER_H +#include <asm/asm-const.h> + +#ifndef __ASSEMBLER__ +#include <asm/ppc-opcode.h> +#endif + /* * Memory barrier. * The sync instruction guarantees that all memory accesses initiated @@ -15,8 +22,6 @@ * mb() prevents loads and stores being reordered across this point. * rmb() prevents loads being reordered across this point. * wmb() prevents stores being reordered across this point. - * read_barrier_depends() prevents data-dependent loads being reordered - * across this point (nop on PPC). * * *mb() variants without smp_ prefix must order all types of memory * operations with one another. sync is the only instruction sufficient @@ -30,31 +35,30 @@ * However, on CPUs that don't support lwsync, lwsync actually maps to a * heavy-weight sync, so smp_wmb() can be a lighter-weight eieio. */ -#define mb() __asm__ __volatile__ ("sync" : : : "memory") -#define rmb() __asm__ __volatile__ ("sync" : : : "memory") -#define wmb() __asm__ __volatile__ ("sync" : : : "memory") -#define read_barrier_depends() do { } while(0) +#define __mb() __asm__ __volatile__ ("sync" : : : "memory") +#define __rmb() __asm__ __volatile__ ("sync" : : : "memory") +#define __wmb() __asm__ __volatile__ ("sync" : : : "memory") -#define set_mb(var, value) do { var = value; mb(); } while (0) - -#ifdef CONFIG_SMP - -#ifdef __SUBARCH_HAS_LWSYNC +/* The sub-arch has lwsync */ +#if defined(CONFIG_PPC64) || defined(CONFIG_PPC_E500MC) # define SMPWMB LWSYNC +#elif defined(CONFIG_BOOKE) +# define SMPWMB mbar #else # define SMPWMB eieio #endif -#define smp_mb() mb() -#define smp_rmb() __asm__ __volatile__ (stringify_in_c(LWSYNC) : : :"memory") -#define smp_wmb() __asm__ __volatile__ (stringify_in_c(SMPWMB) : : :"memory") -#define smp_read_barrier_depends() read_barrier_depends() -#else -#define smp_mb() barrier() -#define smp_rmb() barrier() -#define smp_wmb() barrier() -#define smp_read_barrier_depends() do { } while(0) -#endif /* CONFIG_SMP */ +/* clang defines this macro for a builtin, which will not work with runtime patching */ +#undef __lwsync +#define __lwsync() __asm__ __volatile__ (stringify_in_c(LWSYNC) : : :"memory") +#define __dma_rmb() __lwsync() +#define __dma_wmb() __asm__ __volatile__ (stringify_in_c(SMPWMB) : : :"memory") + +#define __smp_lwsync() __lwsync() + +#define __smp_mb() __mb() +#define __smp_rmb() __lwsync() +#define __smp_wmb() __asm__ __volatile__ (stringify_in_c(SMPWMB) : : :"memory") /* * This is a barrier which prevents following instructions from being @@ -65,4 +69,51 @@ #define data_barrier(x) \ asm volatile("twi 0,%0,0; isync" : : "r" (x) : "memory"); +#define __smp_store_release(p, v) \ +do { \ + compiletime_assert_atomic_type(*p); \ + __smp_lwsync(); \ + WRITE_ONCE(*p, v); \ +} while (0) + +#define __smp_load_acquire(p) \ +({ \ + typeof(*p) ___p1 = READ_ONCE(*p); \ + compiletime_assert_atomic_type(*p); \ + __smp_lwsync(); \ + ___p1; \ +}) + +#ifdef CONFIG_PPC_BOOK3S_64 +#define NOSPEC_BARRIER_SLOT nop +#elif defined(CONFIG_PPC_E500) +#define NOSPEC_BARRIER_SLOT nop; nop +#endif + +#ifdef CONFIG_PPC_BARRIER_NOSPEC +/* + * Prevent execution of subsequent instructions until preceding branches have + * been fully resolved and are no longer executing speculatively. + */ +#define barrier_nospec_asm NOSPEC_BARRIER_FIXUP_SECTION; NOSPEC_BARRIER_SLOT + +// This also acts as a compiler barrier due to the memory clobber. +#define barrier_nospec() asm (stringify_in_c(barrier_nospec_asm) ::: "memory") + +#else /* !CONFIG_PPC_BARRIER_NOSPEC */ +#define barrier_nospec_asm +#define barrier_nospec() +#endif /* CONFIG_PPC_BARRIER_NOSPEC */ + +/* + * pmem_wmb() ensures that all stores for which the modification + * are written to persistent storage by preceding dcbfps/dcbstps + * instructions have updated persistent storage before any data + * access or data transfer caused by subsequent instructions is + * initiated. + */ +#define pmem_wmb() __asm__ __volatile__(PPC_PHWSYNC ::: "memory") + +#include <asm-generic/barrier.h> + #endif /* _ASM_POWERPC_BARRIER_H */ diff --git a/arch/powerpc/include/asm/bitops.h b/arch/powerpc/include/asm/bitops.h index 910194e9a1e2..0d0470cd5ac3 100644 --- a/arch/powerpc/include/asm/bitops.h +++ b/arch/powerpc/include/asm/bitops.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * PowerPC atomic bit operations. * @@ -14,9 +15,9 @@ * * The bitop functions are defined to work on unsigned longs, so for a * ppc64 system the bits end up numbered: - * |63..............0|127............64|191...........128|255...........196| + * |63..............0|127............64|191...........128|255...........192| * and on ppc32: - * |31.....0|63....31|95....64|127...96|159..128|191..160|223..192|255..224| + * |31.....0|63....32|95....64|127...96|159..128|191..160|223..192|255..224| * * There are a few little-endian macros used mostly for filesystem * bitmaps, these work on similar bit arrays layouts, but @@ -26,11 +27,6 @@ * The main difference is that bit 3-5 (64b) or 3-4 (32b) in the bit * number field needs to be reversed compared to the big-endian bit * fields. This can be achieved by XOR with 0x38 (64b) or 0x18 (32b). - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef _ASM_POWERPC_BITOPS_H @@ -46,52 +42,106 @@ #include <asm/asm-compat.h> #include <asm/synch.h> -/* - * clear_bit doesn't imply a memory barrier - */ -#define smp_mb__before_clear_bit() smp_mb() -#define smp_mb__after_clear_bit() smp_mb() +/* PPC bit number conversion */ +#define PPC_BITLSHIFT(be) (BITS_PER_LONG - 1 - (be)) +#define PPC_BIT(bit) (1UL << PPC_BITLSHIFT(bit)) +#define PPC_BITMASK(bs, be) ((PPC_BIT(bs) - PPC_BIT(be)) | PPC_BIT(bs)) + +/* Put a PPC bit into a "normal" bit position */ +#define PPC_BITEXTRACT(bits, ppc_bit, dst_bit) \ + ((((bits) >> PPC_BITLSHIFT(ppc_bit)) & 1) << (dst_bit)) + +#define PPC_BITLSHIFT32(be) (32 - 1 - (be)) +#define PPC_BIT32(bit) (1UL << PPC_BITLSHIFT32(bit)) +#define PPC_BITMASK32(bs, be) ((PPC_BIT32(bs) - PPC_BIT32(be))|PPC_BIT32(bs)) + +#define PPC_BITLSHIFT8(be) (8 - 1 - (be)) +#define PPC_BIT8(bit) (1UL << PPC_BITLSHIFT8(bit)) +#define PPC_BITMASK8(bs, be) ((PPC_BIT8(bs) - PPC_BIT8(be))|PPC_BIT8(bs)) + +#include <asm/barrier.h> /* Macro for generating the ***_bits() functions */ #define DEFINE_BITOP(fn, op, prefix) \ -static __inline__ void fn(unsigned long mask, \ +static inline void fn(unsigned long mask, \ volatile unsigned long *_p) \ { \ unsigned long old; \ unsigned long *p = (unsigned long *)_p; \ __asm__ __volatile__ ( \ prefix \ -"1:" PPC_LLARX(%0,0,%3,0) "\n" \ - stringify_in_c(op) "%0,%0,%2\n" \ - PPC405_ERR77(0,%3) \ +"1:" PPC_LLARX "%0,0,%3,0\n" \ + #op "%I2 %0,%0,%2\n" \ PPC_STLCX "%0,0,%3\n" \ "bne- 1b\n" \ : "=&r" (old), "+m" (*p) \ - : "r" (mask), "r" (p) \ + : "rK" (mask), "r" (p) \ : "cc", "memory"); \ } DEFINE_BITOP(set_bits, or, "") -DEFINE_BITOP(clear_bits, andc, "") -DEFINE_BITOP(clear_bits_unlock, andc, PPC_RELEASE_BARRIER) DEFINE_BITOP(change_bits, xor, "") -static __inline__ void set_bit(int nr, volatile unsigned long *addr) +static __always_inline bool is_rlwinm_mask_valid(unsigned long x) +{ + if (!x) + return false; + if (x & 1) + x = ~x; // make the mask non-wrapping + x += x & -x; // adding the low set bit results in at most one bit set + + return !(x & (x - 1)); +} + +#define DEFINE_CLROP(fn, prefix) \ +static inline void fn(unsigned long mask, volatile unsigned long *_p) \ +{ \ + unsigned long old; \ + unsigned long *p = (unsigned long *)_p; \ + \ + if (IS_ENABLED(CONFIG_PPC32) && \ + __builtin_constant_p(mask) && is_rlwinm_mask_valid(~mask)) {\ + asm volatile ( \ + prefix \ + "1:" "lwarx %0,0,%3\n" \ + "rlwinm %0,%0,0,%2\n" \ + "stwcx. %0,0,%3\n" \ + "bne- 1b\n" \ + : "=&r" (old), "+m" (*p) \ + : "n" (~mask), "r" (p) \ + : "cc", "memory"); \ + } else { \ + asm volatile ( \ + prefix \ + "1:" PPC_LLARX "%0,0,%3,0\n" \ + "andc %0,%0,%2\n" \ + PPC_STLCX "%0,0,%3\n" \ + "bne- 1b\n" \ + : "=&r" (old), "+m" (*p) \ + : "r" (mask), "r" (p) \ + : "cc", "memory"); \ + } \ +} + +DEFINE_CLROP(clear_bits, "") +DEFINE_CLROP(clear_bits_unlock, PPC_RELEASE_BARRIER) + +static inline void arch_set_bit(int nr, volatile unsigned long *addr) { set_bits(BIT_MASK(nr), addr + BIT_WORD(nr)); } -static __inline__ void clear_bit(int nr, volatile unsigned long *addr) +static inline void arch_clear_bit(int nr, volatile unsigned long *addr) { clear_bits(BIT_MASK(nr), addr + BIT_WORD(nr)); } -static __inline__ void clear_bit_unlock(int nr, volatile unsigned long *addr) +static inline void arch_clear_bit_unlock(int nr, volatile unsigned long *addr) { clear_bits_unlock(BIT_MASK(nr), addr + BIT_WORD(nr)); } -static __inline__ void change_bit(int nr, volatile unsigned long *addr) +static inline void arch_change_bit(int nr, volatile unsigned long *addr) { change_bits(BIT_MASK(nr), addr + BIT_WORD(nr)); } @@ -99,7 +149,7 @@ static __inline__ void change_bit(int nr, volatile unsigned long *addr) /* Like DEFINE_BITOP(), with changes to the arguments to 'op' and the output * operands. */ #define DEFINE_TESTOP(fn, op, prefix, postfix, eh) \ -static __inline__ unsigned long fn( \ +static inline unsigned long fn( \ unsigned long mask, \ volatile unsigned long *_p) \ { \ @@ -107,14 +157,13 @@ static __inline__ unsigned long fn( \ unsigned long *p = (unsigned long *)_p; \ __asm__ __volatile__ ( \ prefix \ -"1:" PPC_LLARX(%0,0,%3,eh) "\n" \ - stringify_in_c(op) "%1,%0,%2\n" \ - PPC405_ERR77(0,%3) \ +"1:" PPC_LLARX "%0,0,%3,%4\n" \ + #op "%I2 %1,%0,%2\n" \ PPC_STLCX "%1,0,%3\n" \ "bne- 1b\n" \ postfix \ : "=&r" (old), "=&r" (t) \ - : "r" (mask), "r" (p) \ + : "rK" (mask), "r" (p), "n" (eh) \ : "cc", "memory"); \ return (old & mask); \ } @@ -122,40 +171,90 @@ static __inline__ unsigned long fn( \ DEFINE_TESTOP(test_and_set_bits, or, PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, 0) DEFINE_TESTOP(test_and_set_bits_lock, or, "", - PPC_ACQUIRE_BARRIER, 1) -DEFINE_TESTOP(test_and_clear_bits, andc, PPC_ATOMIC_ENTRY_BARRIER, - PPC_ATOMIC_EXIT_BARRIER, 0) + PPC_ACQUIRE_BARRIER, IS_ENABLED(CONFIG_PPC64)) DEFINE_TESTOP(test_and_change_bits, xor, PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, 0) -static __inline__ int test_and_set_bit(unsigned long nr, - volatile unsigned long *addr) +static inline unsigned long test_and_clear_bits(unsigned long mask, volatile unsigned long *_p) +{ + unsigned long old, t; + unsigned long *p = (unsigned long *)_p; + + if (IS_ENABLED(CONFIG_PPC32) && + __builtin_constant_p(mask) && is_rlwinm_mask_valid(~mask)) { + asm volatile ( + PPC_ATOMIC_ENTRY_BARRIER + "1:" "lwarx %0,0,%3\n" + "rlwinm %1,%0,0,%2\n" + "stwcx. %1,0,%3\n" + "bne- 1b\n" + PPC_ATOMIC_EXIT_BARRIER + : "=&r" (old), "=&r" (t) + : "n" (~mask), "r" (p) + : "cc", "memory"); + } else { + asm volatile ( + PPC_ATOMIC_ENTRY_BARRIER + "1:" PPC_LLARX "%0,0,%3,0\n" + "andc %1,%0,%2\n" + PPC_STLCX "%1,0,%3\n" + "bne- 1b\n" + PPC_ATOMIC_EXIT_BARRIER + : "=&r" (old), "=&r" (t) + : "r" (mask), "r" (p) + : "cc", "memory"); + } + + return (old & mask); +} + +static inline int arch_test_and_set_bit(unsigned long nr, + volatile unsigned long *addr) { return test_and_set_bits(BIT_MASK(nr), addr + BIT_WORD(nr)) != 0; } -static __inline__ int test_and_set_bit_lock(unsigned long nr, - volatile unsigned long *addr) +static inline int arch_test_and_set_bit_lock(unsigned long nr, + volatile unsigned long *addr) { return test_and_set_bits_lock(BIT_MASK(nr), addr + BIT_WORD(nr)) != 0; } -static __inline__ int test_and_clear_bit(unsigned long nr, - volatile unsigned long *addr) +static inline int arch_test_and_clear_bit(unsigned long nr, + volatile unsigned long *addr) { return test_and_clear_bits(BIT_MASK(nr), addr + BIT_WORD(nr)) != 0; } -static __inline__ int test_and_change_bit(unsigned long nr, - volatile unsigned long *addr) +static inline int arch_test_and_change_bit(unsigned long nr, + volatile unsigned long *addr) { return test_and_change_bits(BIT_MASK(nr), addr + BIT_WORD(nr)) != 0; } +static inline bool arch_xor_unlock_is_negative_byte(unsigned long mask, + volatile unsigned long *p) +{ + unsigned long old, t; + + __asm__ __volatile__ ( + PPC_RELEASE_BARRIER +"1:" PPC_LLARX "%0,0,%3,0\n" + "xor %1,%0,%2\n" + PPC_STLCX "%1,0,%3\n" + "bne- 1b\n" + : "=&r" (old), "=&r" (t) + : "r" (mask), "r" (p) + : "cc", "memory"); + + return (old & BIT_MASK(7)) != 0; +} +#define arch_xor_unlock_is_negative_byte arch_xor_unlock_is_negative_byte + #include <asm-generic/bitops/non-atomic.h> -static __inline__ void __clear_bit_unlock(int nr, volatile unsigned long *addr) +static inline void arch___clear_bit_unlock(int nr, volatile unsigned long *addr) { __asm__ __volatile__(PPC_RELEASE_BARRIER "" ::: "memory"); __clear_bit(nr, addr); @@ -165,102 +264,48 @@ static __inline__ void __clear_bit_unlock(int nr, volatile unsigned long *addr) * Return the zero-based bit position (LE, not IBM bit numbering) of * the most significant 1-bit in a double word. */ -static __inline__ __attribute__((const)) -int __ilog2(unsigned long x) -{ - int lz; - - asm (PPC_CNTLZL "%0,%1" : "=r" (lz) : "r" (x)); - return BITS_PER_LONG - 1 - lz; -} - -static inline __attribute__((const)) -int __ilog2_u32(u32 n) -{ - int bit; - asm ("cntlzw %0,%1" : "=r" (bit) : "r" (n)); - return 31 - bit; -} +#define __ilog2(x) ilog2(x) -#ifdef __powerpc64__ -static inline __attribute__((const)) -int __ilog2_u64(u64 n) -{ - int bit; - asm ("cntlzd %0,%1" : "=r" (bit) : "r" (n)); - return 63 - bit; -} -#endif - -/* - * Determines the bit position of the least significant 0 bit in the - * specified double word. The returned bit position will be - * zero-based, starting from the right side (63/31 - 0). - */ -static __inline__ unsigned long ffz(unsigned long x) -{ - /* no zero exists anywhere in the 8 byte area. */ - if ((x = ~x) == 0) - return BITS_PER_LONG; - - /* - * Calculate the bit position of the least significant '1' bit in x - * (since x has been changed this will actually be the least significant - * '0' bit in * the original x). Note: (x & -x) gives us a mask that - * is the least significant * (RIGHT-most) 1-bit of the value in x. - */ - return __ilog2(x & -x); -} +#include <asm-generic/bitops/ffz.h> -static __inline__ int __ffs(unsigned long x) -{ - return __ilog2(x & -x); -} +#include <asm-generic/bitops/builtin-__ffs.h> -/* - * ffs: find first bit set. This is defined the same way as - * the libc and compiler builtin ffs routines, therefore - * differs in spirit from the above ffz (man ffs). - */ -static __inline__ int ffs(int x) -{ - unsigned long i = (unsigned long)x; - return __ilog2(i & -i) + 1; -} +#include <asm-generic/bitops/builtin-ffs.h> /* * fls: find last (most-significant) bit set. * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32. */ -static __inline__ int fls(unsigned int x) +static __always_inline __attribute_const__ int fls(unsigned int x) { int lz; - asm ("cntlzw %0,%1" : "=r" (lz) : "r" (x)); + if (__builtin_constant_p(x)) + return x ? 32 - __builtin_clz(x) : 0; + asm("cntlzw %0,%1" : "=r" (lz) : "r" (x)); return 32 - lz; } -static __inline__ unsigned long __fls(unsigned long x) -{ - return __ilog2(x); -} +#include <asm-generic/bitops/builtin-__fls.h> /* * 64-bit can do this using one cntlzd (count leading zeroes doubleword) * instruction; for 32-bit we use the generic version, which does two * 32-bit fls calls. */ -#ifdef __powerpc64__ -static __inline__ int fls64(__u64 x) +#ifdef CONFIG_PPC64 +static __always_inline __attribute_const__ int fls64(__u64 x) { int lz; - asm ("cntlzd %0,%1" : "=r" (lz) : "r" (x)); + if (__builtin_constant_p(x)) + return x ? 64 - __builtin_clzll(x) : 0; + asm("cntlzd %0,%1" : "=r" (lz) : "r" (x)); return 64 - lz; } #else #include <asm-generic/bitops/fls64.h> -#endif /* __powerpc64__ */ +#endif #ifdef CONFIG_PPC64 unsigned int __arch_hweight8(unsigned int w); @@ -272,7 +317,9 @@ unsigned long __arch_hweight64(__u64 w); #include <asm-generic/bitops/hweight.h> #endif -#include <asm-generic/bitops/find.h> +/* wrappers that deal with KASAN instrumentation */ +#include <asm-generic/bitops/instrumented-atomic.h> +#include <asm-generic/bitops/instrumented-lock.h> /* Little-endian versions */ #include <asm-generic/bitops/le.h> diff --git a/arch/powerpc/include/asm/book3s/32/kup.h b/arch/powerpc/include/asm/book3s/32/kup.h new file mode 100644 index 000000000000..873c5146e326 --- /dev/null +++ b/arch/powerpc/include/asm/book3s/32/kup.h @@ -0,0 +1,175 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_32_KUP_H +#define _ASM_POWERPC_BOOK3S_32_KUP_H + +#include <asm/bug.h> +#include <asm/book3s/32/mmu-hash.h> +#include <asm/mmu.h> +#include <asm/synch.h> + +#ifndef __ASSEMBLER__ + +#ifdef CONFIG_PPC_KUAP + +#include <linux/sched.h> + +#define KUAP_NONE (~0UL) + +static __always_inline void kuap_lock_one(unsigned long addr) +{ + mtsr(mfsr(addr) | SR_KS, addr); + isync(); /* Context sync required after mtsr() */ +} + +static __always_inline void kuap_unlock_one(unsigned long addr) +{ + mtsr(mfsr(addr) & ~SR_KS, addr); + isync(); /* Context sync required after mtsr() */ +} + +static __always_inline void uaccess_begin_32s(unsigned long addr) +{ + unsigned long tmp; + + asm volatile(ASM_MMU_FTR_IFSET( + "mfsrin %0, %1;" + "rlwinm %0, %0, 0, %2;" + "mtsrin %0, %1;" + "isync", "", %3) + : "=&r"(tmp) + : "r"(addr), "i"(~SR_KS), "i"(MMU_FTR_KUAP) + : "memory"); +} + +static __always_inline void uaccess_end_32s(unsigned long addr) +{ + unsigned long tmp; + + asm volatile(ASM_MMU_FTR_IFSET( + "mfsrin %0, %1;" + "oris %0, %0, %2;" + "mtsrin %0, %1;" + "isync", "", %3) + : "=&r"(tmp) + : "r"(addr), "i"(SR_KS >> 16), "i"(MMU_FTR_KUAP) + : "memory"); +} + +static __always_inline void __kuap_save_and_lock(struct pt_regs *regs) +{ + unsigned long kuap = current->thread.kuap; + + regs->kuap = kuap; + if (unlikely(kuap == KUAP_NONE)) + return; + + current->thread.kuap = KUAP_NONE; + kuap_lock_one(kuap); +} +#define __kuap_save_and_lock __kuap_save_and_lock + +static __always_inline void kuap_user_restore(struct pt_regs *regs) +{ +} + +static __always_inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long kuap) +{ + if (unlikely(kuap != KUAP_NONE)) { + current->thread.kuap = KUAP_NONE; + kuap_lock_one(kuap); + } + + if (likely(regs->kuap == KUAP_NONE)) + return; + + current->thread.kuap = regs->kuap; + + kuap_unlock_one(regs->kuap); +} + +static __always_inline unsigned long __kuap_get_and_assert_locked(void) +{ + unsigned long kuap = current->thread.kuap; + + WARN_ON_ONCE(IS_ENABLED(CONFIG_PPC_KUAP_DEBUG) && kuap != KUAP_NONE); + + return kuap; +} +#define __kuap_get_and_assert_locked __kuap_get_and_assert_locked + +static __always_inline void allow_user_access(void __user *to, const void __user *from, + u32 size, unsigned long dir) +{ + BUILD_BUG_ON(!__builtin_constant_p(dir)); + + if (!(dir & KUAP_WRITE)) + return; + + current->thread.kuap = (__force u32)to; + uaccess_begin_32s((__force u32)to); +} + +static __always_inline void prevent_user_access(unsigned long dir) +{ + u32 kuap = current->thread.kuap; + + BUILD_BUG_ON(!__builtin_constant_p(dir)); + + if (!(dir & KUAP_WRITE)) + return; + + current->thread.kuap = KUAP_NONE; + uaccess_end_32s(kuap); +} + +static __always_inline unsigned long prevent_user_access_return(void) +{ + unsigned long flags = current->thread.kuap; + + if (flags != KUAP_NONE) { + current->thread.kuap = KUAP_NONE; + uaccess_end_32s(flags); + } + + return flags; +} + +static __always_inline void restore_user_access(unsigned long flags) +{ + if (flags != KUAP_NONE) { + current->thread.kuap = flags; + uaccess_begin_32s(flags); + } +} + +static __always_inline bool +__bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write) +{ + unsigned long kuap = regs->kuap; + + if (!is_write) + return false; + if (kuap == KUAP_NONE) + return true; + + /* + * If faulting address doesn't match unlocked segment, change segment. + * In case of unaligned store crossing two segments, emulate store. + */ + if ((kuap ^ address) & 0xf0000000) { + if (!(kuap & 0x0fffffff) && address > kuap - 4 && fix_alignment(regs)) { + regs_add_return_ip(regs, 4); + emulate_single_step(regs); + } else { + regs->kuap = address; + } + } + + return false; +} + +#endif /* CONFIG_PPC_KUAP */ + +#endif /* __ASSEMBLER__ */ + +#endif /* _ASM_POWERPC_BOOK3S_32_KUP_H */ diff --git a/arch/powerpc/include/asm/book3s/32/mmu-hash.h b/arch/powerpc/include/asm/book3s/32/mmu-hash.h new file mode 100644 index 000000000000..8435bf3cdabf --- /dev/null +++ b/arch/powerpc/include/asm/book3s/32/mmu-hash.h @@ -0,0 +1,236 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_32_MMU_HASH_H_ +#define _ASM_POWERPC_BOOK3S_32_MMU_HASH_H_ + +/* + * 32-bit hash table MMU support + */ + +/* + * BATs + */ + +/* Block size masks */ +#define BL_128K 0x000 +#define BL_256K 0x001 +#define BL_512K 0x003 +#define BL_1M 0x007 +#define BL_2M 0x00F +#define BL_4M 0x01F +#define BL_8M 0x03F +#define BL_16M 0x07F +#define BL_32M 0x0FF +#define BL_64M 0x1FF +#define BL_128M 0x3FF +#define BL_256M 0x7FF + +/* BAT Access Protection */ +#define BPP_XX 0x00 /* No access */ +#define BPP_RX 0x01 /* Read only */ +#define BPP_RW 0x02 /* Read/write */ + +#ifndef __ASSEMBLER__ +/* Contort a phys_addr_t into the right format/bits for a BAT */ +#ifdef CONFIG_PHYS_64BIT +#define BAT_PHYS_ADDR(x) ((u32)((x & 0x00000000fffe0000ULL) | \ + ((x & 0x0000000e00000000ULL) >> 24) | \ + ((x & 0x0000000100000000ULL) >> 30))) +#define PHYS_BAT_ADDR(x) (((u64)(x) & 0x00000000fffe0000ULL) | \ + (((u64)(x) << 24) & 0x0000000e00000000ULL) | \ + (((u64)(x) << 30) & 0x0000000100000000ULL)) +#else +#define BAT_PHYS_ADDR(x) (x) +#define PHYS_BAT_ADDR(x) ((x) & 0xfffe0000) +#endif + +struct ppc_bat { + u32 batu; + u32 batl; +}; +#endif /* !__ASSEMBLER__ */ + +/* + * Hash table + */ + +/* Values for PP (assumes Ks=0, Kp=1) */ +#define PP_RWXX 0 /* Supervisor read/write, User none */ +#define PP_RWRX 1 /* Supervisor read/write, User read */ +#define PP_RWRW 2 /* Supervisor read/write, User read/write */ +#define PP_RXRX 3 /* Supervisor read, User read */ + +/* Values for Segment Registers */ +#define SR_NX 0x10000000 /* No Execute */ +#define SR_KP 0x20000000 /* User key */ +#define SR_KS 0x40000000 /* Supervisor key */ + +#ifdef __ASSEMBLER__ + +#include <asm/asm-offsets.h> + +.macro uus_addi sr reg1 reg2 imm + .if NUM_USER_SEGMENTS > \sr + addi \reg1,\reg2,\imm + .endif +.endm + +.macro uus_mtsr sr reg1 + .if NUM_USER_SEGMENTS > \sr + mtsr \sr, \reg1 + .endif +.endm + +/* + * This isync() shouldn't be necessary as the kernel is not excepted to run + * any instruction in userspace soon after the update of segments and 'rfi' + * instruction is used to return to userspace, but hash based cores + * (at least G3) seem to exhibit a random behaviour when the 'isync' is not + * there. 603 cores don't have this behaviour so don't do the 'isync' as it + * saves several CPU cycles. + */ +.macro uus_isync +#ifdef CONFIG_PPC_BOOK3S_604 +BEGIN_MMU_FTR_SECTION + isync +END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) +#endif +.endm + +.macro update_user_segments_by_4 tmp1 tmp2 tmp3 tmp4 + uus_addi 1, \tmp2, \tmp1, 0x111 + uus_addi 2, \tmp3, \tmp1, 0x222 + uus_addi 3, \tmp4, \tmp1, 0x333 + + uus_mtsr 0, \tmp1 + uus_mtsr 1, \tmp2 + uus_mtsr 2, \tmp3 + uus_mtsr 3, \tmp4 + + uus_addi 4, \tmp1, \tmp1, 0x444 + uus_addi 5, \tmp2, \tmp2, 0x444 + uus_addi 6, \tmp3, \tmp3, 0x444 + uus_addi 7, \tmp4, \tmp4, 0x444 + + uus_mtsr 4, \tmp1 + uus_mtsr 5, \tmp2 + uus_mtsr 6, \tmp3 + uus_mtsr 7, \tmp4 + + uus_addi 8, \tmp1, \tmp1, 0x444 + uus_addi 9, \tmp2, \tmp2, 0x444 + uus_addi 10, \tmp3, \tmp3, 0x444 + uus_addi 11, \tmp4, \tmp4, 0x444 + + uus_mtsr 8, \tmp1 + uus_mtsr 9, \tmp2 + uus_mtsr 10, \tmp3 + uus_mtsr 11, \tmp4 + + uus_addi 12, \tmp1, \tmp1, 0x444 + uus_addi 13, \tmp2, \tmp2, 0x444 + uus_addi 14, \tmp3, \tmp3, 0x444 + uus_addi 15, \tmp4, \tmp4, 0x444 + + uus_mtsr 12, \tmp1 + uus_mtsr 13, \tmp2 + uus_mtsr 14, \tmp3 + uus_mtsr 15, \tmp4 + + uus_isync +.endm + +#else + +/* + * This macro defines the mapping from contexts to VSIDs (virtual + * segment IDs). We use a skew on both the context and the high 4 bits + * of the 32-bit virtual address (the "effective segment ID") in order + * to spread out the entries in the MMU hash table. Note, if this + * function is changed then hash functions will have to be + * changed to correspond. + */ +#define CTX_TO_VSID(c, id) ((((c) * (897 * 16)) + (id * 0x111)) & 0xffffff) + +/* + * Hardware Page Table Entry + * Note that the xpn and x bitfields are used only by processors that + * support extended addressing; otherwise, those bits are reserved. + */ +struct hash_pte { + unsigned long v:1; /* Entry is valid */ + unsigned long vsid:24; /* Virtual segment identifier */ + unsigned long h:1; /* Hash algorithm indicator */ + unsigned long api:6; /* Abbreviated page index */ + unsigned long rpn:20; /* Real (physical) page number */ + unsigned long xpn:3; /* Real page number bits 0-2, optional */ + unsigned long r:1; /* Referenced */ + unsigned long c:1; /* Changed */ + unsigned long w:1; /* Write-thru cache mode */ + unsigned long i:1; /* Cache inhibited */ + unsigned long m:1; /* Memory coherence */ + unsigned long g:1; /* Guarded */ + unsigned long x:1; /* Real page number bit 3, optional */ + unsigned long pp:2; /* Page protection */ +}; + +typedef struct { + unsigned long id; + unsigned long sr0; + void __user *vdso; +} mm_context_t; + +#ifdef CONFIG_PPC_KUEP +#define INIT_MM_CONTEXT(mm) .context.sr0 = SR_NX +#endif + +void update_bats(void); +static inline void cleanup_cpu_mmu_context(void) { } + +/* patch sites */ +extern s32 patch__hash_page_A0, patch__hash_page_A1, patch__hash_page_A2; +extern s32 patch__hash_page_B, patch__hash_page_C; +extern s32 patch__flush_hash_A0, patch__flush_hash_A1, patch__flush_hash_A2; +extern s32 patch__flush_hash_B; + +#include <asm/reg.h> +#include <asm/task_size_32.h> + +static __always_inline void update_user_segment(u32 n, u32 val) +{ + if (n << 28 < TASK_SIZE) + mtsr(val + n * 0x111, n << 28); +} + +static __always_inline void update_user_segments(u32 val) +{ + val &= 0xf0ffffff; + + update_user_segment(0, val); + update_user_segment(1, val); + update_user_segment(2, val); + update_user_segment(3, val); + update_user_segment(4, val); + update_user_segment(5, val); + update_user_segment(6, val); + update_user_segment(7, val); + update_user_segment(8, val); + update_user_segment(9, val); + update_user_segment(10, val); + update_user_segment(11, val); + update_user_segment(12, val); + update_user_segment(13, val); + update_user_segment(14, val); + update_user_segment(15, val); +} + +int __init find_free_bat(void); +unsigned int bat_block_size(unsigned long base, unsigned long top); +#endif /* !__ASSEMBLER__ */ + +/* We happily ignore the smaller BATs on 601, we don't actually use + * those definitions on hash32 at the moment anyway + */ +#define mmu_virtual_psize MMU_PAGE_4K +#define mmu_linear_psize MMU_PAGE_256M + +#endif /* _ASM_POWERPC_BOOK3S_32_MMU_HASH_H_ */ diff --git a/arch/powerpc/include/asm/book3s/32/pgalloc.h b/arch/powerpc/include/asm/book3s/32/pgalloc.h new file mode 100644 index 000000000000..f4390704d5ba --- /dev/null +++ b/arch/powerpc/include/asm/book3s/32/pgalloc.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_32_PGALLOC_H +#define _ASM_POWERPC_BOOK3S_32_PGALLOC_H + +#include <linux/threads.h> +#include <linux/slab.h> + +static inline pgd_t *pgd_alloc(struct mm_struct *mm) +{ + pgd_t *pgd = kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), + pgtable_gfp_flags(mm, GFP_KERNEL)); + +#ifdef CONFIG_PPC_BOOK3S_603 + memcpy(pgd + USER_PTRS_PER_PGD, swapper_pg_dir + USER_PTRS_PER_PGD, + (MAX_PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); +#endif + return pgd; +} + +static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) +{ + kmem_cache_free(PGT_CACHE(PGD_INDEX_SIZE), pgd); +} + +/* + * We don't have any real pmd's, and this code never triggers because + * the pgd will always be present.. + */ +/* #define pmd_alloc_one(mm,address) ({ BUG(); ((pmd_t *)2); }) */ +#define pmd_free(mm, x) do { } while (0) +#define __pmd_free_tlb(tlb,x,a) do { } while (0) +/* #define pgd_populate(mm, pmd, pte) BUG() */ + +static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, + pte_t *pte) +{ + *pmdp = __pmd(__pa(pte) | _PMD_PRESENT); +} + +static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, + pgtable_t pte_page) +{ + *pmdp = __pmd(__pa(pte_page) | _PMD_PRESENT); +} + +static inline void pgtable_free(void *table, unsigned index_size) +{ + if (!index_size) { + pte_fragment_free((unsigned long *)table, 0); + } else { + BUG_ON(index_size > MAX_PGTABLE_INDEX_SIZE); + kmem_cache_free(PGT_CACHE(index_size), table); + } +} + +static inline void pgtable_free_tlb(struct mmu_gather *tlb, + void *table, int shift) +{ + unsigned long pgf = (unsigned long)table; + BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE); + pgf |= shift; + tlb_remove_table(tlb, (void *)pgf); +} + +static inline void __tlb_remove_table(void *_table) +{ + void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE); + unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE; + + pgtable_free(table, shift); +} + +static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table, + unsigned long address) +{ + pgtable_free_tlb(tlb, table, 0); +} +#endif /* _ASM_POWERPC_BOOK3S_32_PGALLOC_H */ diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h new file mode 100644 index 000000000000..87dcca962be7 --- /dev/null +++ b/arch/powerpc/include/asm/book3s/32/pgtable.h @@ -0,0 +1,607 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_32_PGTABLE_H +#define _ASM_POWERPC_BOOK3S_32_PGTABLE_H + +#include <asm-generic/pgtable-nopmd.h> + +/* + * The "classic" 32-bit implementation of the PowerPC MMU uses a hash + * table containing PTEs, together with a set of 16 segment registers, + * to define the virtual to physical address mapping. + * + * We use the hash table as an extended TLB, i.e. a cache of currently + * active mappings. We maintain a two-level page table tree, much + * like that used by the i386, for the sake of the Linux memory + * management code. Low-level assembler code in hash_low_32.S + * (procedure hash_page) is responsible for extracting ptes from the + * tree and putting them into the hash table when necessary, and + * updating the accessed and modified bits in the page table tree. + */ + +#define _PAGE_PRESENT 0x001 /* software: pte contains a translation */ +#define _PAGE_HASHPTE 0x002 /* hash_page has made an HPTE for this pte */ +#define _PAGE_READ 0x004 /* software: read access allowed */ +#define _PAGE_GUARDED 0x008 /* G: prohibit speculative access */ +#define _PAGE_COHERENT 0x010 /* M: enforce memory coherence (SMP systems) */ +#define _PAGE_NO_CACHE 0x020 /* I: cache inhibit */ +#define _PAGE_WRITETHRU 0x040 /* W: cache write-through */ +#define _PAGE_DIRTY 0x080 /* C: page changed */ +#define _PAGE_ACCESSED 0x100 /* R: page referenced */ +#define _PAGE_EXEC 0x200 /* software: exec allowed */ +#define _PAGE_WRITE 0x400 /* software: user write access allowed */ +#define _PAGE_SPECIAL 0x800 /* software: Special page */ + +#ifdef CONFIG_PTE_64BIT +/* We never clear the high word of the pte */ +#define _PTE_NONE_MASK (0xffffffff00000000ULL | _PAGE_HASHPTE) +#else +#define _PTE_NONE_MASK _PAGE_HASHPTE +#endif + +#define _PMD_PRESENT 0 +#define _PMD_PRESENT_MASK (PAGE_MASK) +#define _PMD_BAD (~PAGE_MASK) + +/* We borrow the _PAGE_READ bit to store the exclusive marker in swap PTEs. */ +#define _PAGE_SWP_EXCLUSIVE _PAGE_READ + +/* And here we include common definitions */ + +#define _PAGE_HPTEFLAGS _PAGE_HASHPTE + +/* + * Location of the PFN in the PTE. Most 32-bit platforms use the same + * as _PAGE_SHIFT here (ie, naturally aligned). + * Platform who don't just pre-define the value so we don't override it here. + */ +#define PTE_RPN_SHIFT (PAGE_SHIFT) + +/* + * The mask covered by the RPN must be a ULL on 32-bit platforms with + * 64-bit PTEs. + */ +#ifdef CONFIG_PTE_64BIT +#define PTE_RPN_MASK (~((1ULL << PTE_RPN_SHIFT) - 1)) +#define MAX_POSSIBLE_PHYSMEM_BITS 36 +#else +#define PTE_RPN_MASK (~((1UL << PTE_RPN_SHIFT) - 1)) +#define MAX_POSSIBLE_PHYSMEM_BITS 32 +#endif + +/* + * _PAGE_CHG_MASK masks of bits that are to be preserved across + * pgprot changes. + */ +#define _PAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HASHPTE | _PAGE_DIRTY | \ + _PAGE_ACCESSED | _PAGE_SPECIAL) + +/* + * We define 2 sets of base prot bits, one for basic pages (ie, + * cacheable kernel and user pages) and one for non cacheable + * pages. We always set _PAGE_COHERENT when SMP is enabled or + * the processor might need it for DMA coherency. + */ +#define _PAGE_BASE_NC (_PAGE_PRESENT | _PAGE_ACCESSED) +#define _PAGE_BASE (_PAGE_BASE_NC | _PAGE_COHERENT) + +#include <asm/pgtable-masks.h> + +/* Permission masks used for kernel mappings */ +#define PAGE_KERNEL __pgprot(_PAGE_BASE | _PAGE_KERNEL_RW) +#define PAGE_KERNEL_NC __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | _PAGE_NO_CACHE) +#define PAGE_KERNEL_NCG __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | _PAGE_NO_CACHE | _PAGE_GUARDED) +#define PAGE_KERNEL_X __pgprot(_PAGE_BASE | _PAGE_KERNEL_RWX) +#define PAGE_KERNEL_RO __pgprot(_PAGE_BASE | _PAGE_KERNEL_RO) +#define PAGE_KERNEL_ROX __pgprot(_PAGE_BASE | _PAGE_KERNEL_ROX) + +#define PTE_INDEX_SIZE PTE_SHIFT +#define PMD_INDEX_SIZE 0 +#define PUD_INDEX_SIZE 0 +#define PGD_INDEX_SIZE (32 - PGDIR_SHIFT) + +#define PMD_CACHE_INDEX PMD_INDEX_SIZE +#define PUD_CACHE_INDEX PUD_INDEX_SIZE + +#ifndef __ASSEMBLER__ +#define PTE_TABLE_SIZE (sizeof(pte_t) << PTE_INDEX_SIZE) +#define PMD_TABLE_SIZE 0 +#define PUD_TABLE_SIZE 0 +#define PGD_TABLE_SIZE (sizeof(pgd_t) << PGD_INDEX_SIZE) + +/* Bits to mask out from a PMD to get to the PTE page */ +#define PMD_MASKED_BITS (PTE_TABLE_SIZE - 1) +#endif /* __ASSEMBLER__ */ + +#define PTRS_PER_PTE (1 << PTE_INDEX_SIZE) +#define PTRS_PER_PGD (1 << PGD_INDEX_SIZE) + +/* + * The normal case is that PTEs are 32-bits and we have a 1-page + * 1024-entry pgdir pointing to 1-page 1024-entry PTE pages. -- paulus + * + * For any >32-bit physical address platform, we can use the following + * two level page table layout where the pgdir is 8KB and the MS 13 bits + * are an index to the second level table. The combined pgdir/pmd first + * level has 2048 entries and the second level has 512 64-bit PTE entries. + * -Matt + */ +/* PGDIR_SHIFT determines what a top-level page table entry can map */ +#define PGDIR_SHIFT (PAGE_SHIFT + PTE_INDEX_SIZE) +#define PGDIR_SIZE (1UL << PGDIR_SHIFT) +#define PGDIR_MASK (~(PGDIR_SIZE-1)) + +#define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE) + +#ifndef __ASSEMBLER__ + +int map_kernel_page(unsigned long va, phys_addr_t pa, pgprot_t prot); +void unmap_kernel_page(unsigned long va); + +#endif /* !__ASSEMBLER__ */ + +/* + * This is the bottom of the PKMAP area with HIGHMEM or an arbitrary + * value (for now) on others, from where we can start layout kernel + * virtual space that goes below PKMAP and FIXMAP + */ + +#define FIXADDR_SIZE 0 +#ifdef CONFIG_KASAN +#include <asm/kasan.h> +#define FIXADDR_TOP (KASAN_SHADOW_START - PAGE_SIZE) +#else +#define FIXADDR_TOP ((unsigned long)(-PAGE_SIZE)) +#endif + +/* + * ioremap_bot starts at that address. Early ioremaps move down from there, + * until mem_init() at which point this becomes the top of the vmalloc + * and ioremap space + */ +#ifdef CONFIG_HIGHMEM +#define IOREMAP_TOP PKMAP_BASE +#else +#define IOREMAP_TOP FIXADDR_START +#endif + +/* PPC32 shares vmalloc area with ioremap */ +#define IOREMAP_START VMALLOC_START +#define IOREMAP_END VMALLOC_END + +/* + * Just any arbitrary offset to the start of the vmalloc VM area: the + * current 16MB value just means that there will be a 64MB "hole" after the + * physical memory until the kernel virtual memory starts. That means that + * any out-of-bounds memory accesses will hopefully be caught. + * The vmalloc() routines leaves a hole of 4kB between each vmalloced + * area for the same reason. ;) + * + * We no longer map larger than phys RAM with the BATs so we don't have + * to worry about the VMALLOC_OFFSET causing problems. We do have to worry + * about clashes between our early calls to ioremap() that start growing down + * from ioremap_base being run into the VM area allocations (growing upwards + * from VMALLOC_START). For this reason we have ioremap_bot to check when + * we actually run into our mappings setup in the early boot with the VM + * system. This really does become a problem for machines with good amounts + * of RAM. -- Cort + */ +#define VMALLOC_OFFSET (0x1000000) /* 16M */ + +#define VMALLOC_START ((((long)high_memory + VMALLOC_OFFSET) & ~(VMALLOC_OFFSET-1))) + +#ifdef CONFIG_KASAN_VMALLOC +#define VMALLOC_END ALIGN_DOWN(ioremap_bot, PAGE_SIZE << KASAN_SHADOW_SCALE_SHIFT) +#else +#define VMALLOC_END ioremap_bot +#endif + +#define MODULES_END ALIGN_DOWN(PAGE_OFFSET, SZ_256M) +#define MODULES_SIZE (CONFIG_MODULES_SIZE * SZ_1M) +#define MODULES_VADDR (MODULES_END - MODULES_SIZE) + +#ifndef __ASSEMBLER__ +#include <linux/sched.h> +#include <linux/threads.h> + +/* Bits to mask out from a PGD to get to the PUD page */ +#define PGD_MASKED_BITS 0 + +#define pgd_ERROR(e) \ + pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) +/* + * Bits in a linux-style PTE. These match the bits in the + * (hardware-defined) PowerPC PTE as closely as possible. + */ + +#define pte_clear(mm, addr, ptep) \ + do { pte_update(mm, addr, ptep, ~_PAGE_HASHPTE, 0, 0); } while (0) + +#define pmd_none(pmd) (!pmd_val(pmd)) +#define pmd_bad(pmd) (pmd_val(pmd) & _PMD_BAD) +#define pmd_present(pmd) (pmd_val(pmd) & _PMD_PRESENT_MASK) +static inline void pmd_clear(pmd_t *pmdp) +{ + *pmdp = __pmd(0); +} + + +/* + * When flushing the tlb entry for a page, we also need to flush the hash + * table entry. flush_hash_pages is assembler (for speed) in hashtable.S. + */ +extern int flush_hash_pages(unsigned context, unsigned long va, + unsigned long pmdval, int count); + +/* Add an HPTE to the hash table */ +extern void add_hash_page(unsigned context, unsigned long va, + unsigned long pmdval); + +/* Flush an entry from the TLB/hash table */ +static inline void flush_hash_entry(struct mm_struct *mm, pte_t *ptep, unsigned long addr) +{ + if (mmu_has_feature(MMU_FTR_HPTE_TABLE)) { + unsigned long ptephys = __pa(ptep) & PAGE_MASK; + + flush_hash_pages(mm->context.id, addr, ptephys, 1); + } +} + +/* + * PTE updates. This function is called whenever an existing + * valid PTE is updated. This does -not- include set_pte_at() + * which nowadays only sets a new PTE. + * + * Depending on the type of MMU, we may need to use atomic updates + * and the PTE may be either 32 or 64 bit wide. In the later case, + * when using atomic updates, only the low part of the PTE is + * accessed atomically. + */ +static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, pte_t *p, + unsigned long clr, unsigned long set, int huge) +{ + pte_basic_t old; + + if (mmu_has_feature(MMU_FTR_HPTE_TABLE)) { + unsigned long tmp; + + asm volatile( +#ifndef CONFIG_PTE_64BIT + "1: lwarx %0, 0, %3\n" + " andc %1, %0, %4\n" +#else + "1: lwarx %L0, 0, %3\n" + " lwz %0, -4(%3)\n" + " andc %1, %L0, %4\n" +#endif + " or %1, %1, %5\n" + " stwcx. %1, 0, %3\n" + " bne- 1b" + : "=&r" (old), "=&r" (tmp), "=m" (*p) +#ifndef CONFIG_PTE_64BIT + : "r" (p), +#else + : "b" ((unsigned long)(p) + 4), +#endif + "r" (clr), "r" (set), "m" (*p) + : "cc" ); + } else { + old = pte_val(*p); + + *p = __pte((old & ~(pte_basic_t)clr) | set); + } + + return old; +} + +/* + * 2.6 calls this without flushing the TLB entry; this is wrong + * for our hash-based implementation, we fix that up here. + */ +#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG +static inline int __ptep_test_and_clear_young(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + unsigned long old; + old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0); + if (old & _PAGE_HASHPTE) + flush_hash_entry(mm, ptep, addr); + + return (old & _PAGE_ACCESSED) != 0; +} +#define ptep_test_and_clear_young(__vma, __addr, __ptep) \ + __ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep) + +#define __HAVE_ARCH_PTEP_GET_AND_CLEAR +static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep) +{ + return __pte(pte_update(mm, addr, ptep, ~_PAGE_HASHPTE, 0, 0)); +} + +#define __HAVE_ARCH_PTEP_SET_WRPROTECT +static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, + pte_t *ptep) +{ + pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 0); +} + +static inline void __ptep_set_access_flags(struct vm_area_struct *vma, + pte_t *ptep, pte_t entry, + unsigned long address, + int psize) +{ + unsigned long set = pte_val(entry) & + (_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC); + + pte_update(vma->vm_mm, address, ptep, 0, set, 0); + + flush_tlb_page(vma, address); +} + +#define __HAVE_ARCH_PTE_SAME +#define pte_same(A,B) (((pte_val(A) ^ pte_val(B)) & ~_PAGE_HASHPTE) == 0) + +#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT) +#define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd)) + +/* + * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that + * are !pte_none() && !pte_present(). + * + * Format of swap PTEs (32bit PTEs): + * + * 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 3 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * <----------------- offset --------------------> < type -> E H P + * + * E is the exclusive marker that is not stored in swap entries. + * _PAGE_PRESENT (P) and __PAGE_HASHPTE (H) must be 0. + * + * For 64bit PTEs, the offset is extended by 32bit. + */ +#define __swp_type(entry) ((entry).val & 0x1f) +#define __swp_offset(entry) ((entry).val >> 5) +#define __swp_entry(type, offset) ((swp_entry_t) { ((type) & 0x1f) | ((offset) << 5) }) +#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) >> 3 }) +#define __swp_entry_to_pte(x) ((pte_t) { (x).val << 3 }) + +static inline bool pte_swp_exclusive(pte_t pte) +{ + return pte_val(pte) & _PAGE_SWP_EXCLUSIVE; +} + +static inline pte_t pte_swp_mkexclusive(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_SWP_EXCLUSIVE); +} + +static inline pte_t pte_swp_clear_exclusive(pte_t pte) +{ + return __pte(pte_val(pte) & ~_PAGE_SWP_EXCLUSIVE); +} + +/* Generic accessors to PTE bits */ +static inline bool pte_read(pte_t pte) +{ + return !!(pte_val(pte) & _PAGE_READ); +} + +static inline bool pte_write(pte_t pte) +{ + return !!(pte_val(pte) & _PAGE_WRITE); +} + +static inline int pte_dirty(pte_t pte) { return !!(pte_val(pte) & _PAGE_DIRTY); } +static inline int pte_young(pte_t pte) { return !!(pte_val(pte) & _PAGE_ACCESSED); } +static inline int pte_special(pte_t pte) { return !!(pte_val(pte) & _PAGE_SPECIAL); } +static inline int pte_none(pte_t pte) { return (pte_val(pte) & ~_PTE_NONE_MASK) == 0; } +static inline bool pte_exec(pte_t pte) { return pte_val(pte) & _PAGE_EXEC; } + +static inline int pte_present(pte_t pte) +{ + return pte_val(pte) & _PAGE_PRESENT; +} + +static inline bool pte_hw_valid(pte_t pte) +{ + return pte_val(pte) & _PAGE_PRESENT; +} + +static inline bool pte_hashpte(pte_t pte) +{ + return !!(pte_val(pte) & _PAGE_HASHPTE); +} + +static inline bool pte_ci(pte_t pte) +{ + return !!(pte_val(pte) & _PAGE_NO_CACHE); +} + +/* + * We only find page table entry in the last level + * Hence no need for other accessors + */ +#define pte_access_permitted pte_access_permitted +static inline bool pte_access_permitted(pte_t pte, bool write) +{ + /* + * A read-only access is controlled by _PAGE_READ bit. + * We have _PAGE_READ set for WRITE + */ + if (!pte_present(pte) || !pte_read(pte)) + return false; + + if (write && !pte_write(pte)) + return false; + + return true; +} + +/* Conversion functions: convert a page and protection to a page entry, + * and a page entry and page directory to the page they refer to. + * + * Even if PTEs can be unsigned long long, a PFN is always an unsigned + * long for now. + */ +static inline pte_t pfn_pte(unsigned long pfn, pgprot_t pgprot) +{ + return __pte(((pte_basic_t)(pfn) << PTE_RPN_SHIFT) | + pgprot_val(pgprot)); +} + +/* Generic modifiers for PTE bits */ +static inline pte_t pte_wrprotect(pte_t pte) +{ + return __pte(pte_val(pte) & ~_PAGE_WRITE); +} + +static inline pte_t pte_exprotect(pte_t pte) +{ + return __pte(pte_val(pte) & ~_PAGE_EXEC); +} + +static inline pte_t pte_mkclean(pte_t pte) +{ + return __pte(pte_val(pte) & ~_PAGE_DIRTY); +} + +static inline pte_t pte_mkold(pte_t pte) +{ + return __pte(pte_val(pte) & ~_PAGE_ACCESSED); +} + +static inline pte_t pte_mkexec(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_EXEC); +} + +static inline pte_t pte_mkpte(pte_t pte) +{ + return pte; +} + +static inline pte_t pte_mkwrite_novma(pte_t pte) +{ + /* + * write implies read, hence set both + */ + return __pte(pte_val(pte) | _PAGE_RW); +} + +static inline pte_t pte_mkdirty(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_DIRTY); +} + +static inline pte_t pte_mkyoung(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_ACCESSED); +} + +static inline pte_t pte_mkspecial(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_SPECIAL); +} + +static inline pte_t pte_mkhuge(pte_t pte) +{ + return pte; +} + +static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) +{ + return __pte((pte_val(pte) & _PAGE_CHG_MASK) | pgprot_val(newprot)); +} + + + +/* This low level function performs the actual PTE insertion + * Setting the PTE depends on the MMU type and other factors. + * + * First case is 32-bit in UP mode with 32-bit PTEs, we need to preserve + * the _PAGE_HASHPTE bit since we may not have invalidated the previous + * translation in the hash yet (done in a subsequent flush_tlb_xxx()) + * and see we need to keep track that this PTE needs invalidating. + * + * Second case is 32-bit with 64-bit PTE. In this case, we + * can just store as long as we do the two halves in the right order + * with a barrier in between. This is possible because we take care, + * in the hash code, to pre-invalidate if the PTE was already hashed, + * which synchronizes us with any concurrent invalidation. + * In the percpu case, we fallback to the simple update preserving + * the hash bits (ie, same as the non-SMP case). + * + * Third case is 32-bit in SMP mode with 32-bit PTEs. We use the + * helper pte_update() which does an atomic update. We need to do that + * because a concurrent invalidation can clear _PAGE_HASHPTE. If it's a + * per-CPU PTE such as a kmap_atomic, we also do a simple update preserving + * the hash bits instead. + */ +static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, int percpu) +{ + if ((!IS_ENABLED(CONFIG_SMP) && !IS_ENABLED(CONFIG_PTE_64BIT)) || percpu) { + *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE) | + (pte_val(pte) & ~_PAGE_HASHPTE)); + } else if (IS_ENABLED(CONFIG_PTE_64BIT)) { + if (pte_val(*ptep) & _PAGE_HASHPTE) + flush_hash_entry(mm, ptep, addr); + + asm volatile("stw%X0 %2,%0; eieio; stw%X1 %L2,%1" : + "=m" (*ptep), "=m" (*((unsigned char *)ptep+4)) : + "r" (pte) : "memory"); + } else { + pte_update(mm, addr, ptep, ~_PAGE_HASHPTE, pte_val(pte), 0); + } +} + +/* + * Macro to mark a page protection value as "uncacheable". + */ + +#define _PAGE_CACHE_CTL (_PAGE_COHERENT | _PAGE_GUARDED | _PAGE_NO_CACHE | \ + _PAGE_WRITETHRU) + +#define pgprot_noncached pgprot_noncached +static inline pgprot_t pgprot_noncached(pgprot_t prot) +{ + return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | + _PAGE_NO_CACHE | _PAGE_GUARDED); +} + +#define pgprot_noncached_wc pgprot_noncached_wc +static inline pgprot_t pgprot_noncached_wc(pgprot_t prot) +{ + return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | + _PAGE_NO_CACHE); +} + +#define pgprot_cached pgprot_cached +static inline pgprot_t pgprot_cached(pgprot_t prot) +{ + return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | + _PAGE_COHERENT); +} + +#define pgprot_cached_wthru pgprot_cached_wthru +static inline pgprot_t pgprot_cached_wthru(pgprot_t prot) +{ + return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | + _PAGE_COHERENT | _PAGE_WRITETHRU); +} + +#define pgprot_cached_noncoherent pgprot_cached_noncoherent +static inline pgprot_t pgprot_cached_noncoherent(pgprot_t prot) +{ + return __pgprot(pgprot_val(prot) & ~_PAGE_CACHE_CTL); +} + +#define pgprot_writecombine pgprot_writecombine +static inline pgprot_t pgprot_writecombine(pgprot_t prot) +{ + return pgprot_noncached_wc(prot); +} + +#endif /* !__ASSEMBLER__ */ + +#endif /* _ASM_POWERPC_BOOK3S_32_PGTABLE_H */ diff --git a/arch/powerpc/include/asm/book3s/32/tlbflush.h b/arch/powerpc/include/asm/book3s/32/tlbflush.h new file mode 100644 index 000000000000..4be2200a3c7e --- /dev/null +++ b/arch/powerpc/include/asm/book3s/32/tlbflush.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_32_TLBFLUSH_H +#define _ASM_POWERPC_BOOK3S_32_TLBFLUSH_H + +#include <linux/build_bug.h> + +#define MMU_NO_CONTEXT (0) +/* + * TLB flushing for "classic" hash-MMU 32-bit CPUs, 6xx, 7xx, 7xxx + */ +void hash__flush_tlb_mm(struct mm_struct *mm); +void hash__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); +void hash__flush_range(struct mm_struct *mm, unsigned long start, unsigned long end); +void hash__flush_gather(struct mmu_gather *tlb); + +#ifdef CONFIG_SMP +void _tlbie(unsigned long address); +#else +static inline void _tlbie(unsigned long address) +{ + asm volatile ("tlbie %0; sync" : : "r" (address) : "memory"); +} +#endif +void _tlbia(void); + +/* + * Called at the end of a mmu_gather operation to make sure the + * TLB flush is completely done. + */ +static inline void tlb_flush(struct mmu_gather *tlb) +{ + /* 603 needs to flush the whole TLB here since it doesn't use a hash table. */ + if (mmu_has_feature(MMU_FTR_HPTE_TABLE)) + hash__flush_gather(tlb); + else + _tlbia(); +} + +static inline void flush_range(struct mm_struct *mm, unsigned long start, unsigned long end) +{ + start &= PAGE_MASK; + if (mmu_has_feature(MMU_FTR_HPTE_TABLE)) + hash__flush_range(mm, start, end); + else if (end - start <= PAGE_SIZE) + _tlbie(start); + else + _tlbia(); +} + +static inline void flush_tlb_mm(struct mm_struct *mm) +{ + if (mmu_has_feature(MMU_FTR_HPTE_TABLE)) + hash__flush_tlb_mm(mm); + else + _tlbia(); +} + +static inline void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) +{ + if (mmu_has_feature(MMU_FTR_HPTE_TABLE)) + hash__flush_tlb_page(vma, vmaddr); + else + _tlbie(vmaddr); +} + +static inline void +flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) +{ + flush_range(vma->vm_mm, start, end); +} + +static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + flush_range(&init_mm, start, end); +} + +static inline void local_flush_tlb_page(struct vm_area_struct *vma, + unsigned long vmaddr) +{ + flush_tlb_page(vma, vmaddr); +} + +static inline void local_flush_tlb_page_psize(struct mm_struct *mm, + unsigned long vmaddr, int psize) +{ + flush_range(mm, vmaddr, vmaddr); +} + +static inline void local_flush_tlb_mm(struct mm_struct *mm) +{ + flush_tlb_mm(mm); +} + +#endif /* _ASM_POWERPC_BOOK3S_32_TLBFLUSH_H */ diff --git a/arch/powerpc/include/asm/book3s/64/hash-4k.h b/arch/powerpc/include/asm/book3s/64/hash-4k.h new file mode 100644 index 000000000000..8e5bd9902bed --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/hash-4k.h @@ -0,0 +1,173 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_64_HASH_4K_H +#define _ASM_POWERPC_BOOK3S_64_HASH_4K_H + +#define H_PTE_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps: 2^9 x 4KB = 2MB +#define H_PMD_INDEX_SIZE 7 // size: 8B << 7 = 1KB, maps: 2^7 x 2MB = 256MB +#define H_PUD_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps: 2^9 x 256MB = 128GB +#define H_PGD_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps: 2^9 x 128GB = 64TB + +/* + * Each context is 512TB. But on 4k we restrict our max TASK size to 64TB + * Hence also limit max EA bits to 64TB. + */ +#define MAX_EA_BITS_PER_CONTEXT 46 + + +/* + * Our page table limit us to 64TB. For 64TB physical memory, we only need 64GB + * of vmemmap space. To better support sparse memory layout, we use 61TB + * linear map range, 1TB of vmalloc, 1TB of I/O and 1TB of vmememmap. + */ +#define REGION_SHIFT (40) +#define H_KERN_MAP_SIZE (ASM_CONST(1) << REGION_SHIFT) + +/* + * Limits the linear mapping range + */ +#define H_MAX_PHYSMEM_BITS 46 + +/* + * Define the address range of the kernel non-linear virtual area (61TB) + */ +#define H_KERN_VIRT_START ASM_CONST(0xc0003d0000000000) + +#ifndef __ASSEMBLER__ +#define H_PTE_TABLE_SIZE (sizeof(pte_t) << H_PTE_INDEX_SIZE) +#define H_PMD_TABLE_SIZE (sizeof(pmd_t) << H_PMD_INDEX_SIZE) +#define H_PUD_TABLE_SIZE (sizeof(pud_t) << H_PUD_INDEX_SIZE) +#define H_PGD_TABLE_SIZE (sizeof(pgd_t) << H_PGD_INDEX_SIZE) + +#define H_PAGE_F_GIX_SHIFT _PAGE_PA_MAX +#define H_PAGE_F_SECOND _RPAGE_PKEY_BIT0 /* HPTE is in 2ndary HPTEG */ +#define H_PAGE_F_GIX (_RPAGE_RPN43 | _RPAGE_RPN42 | _RPAGE_RPN41) +#define H_PAGE_BUSY _RPAGE_RSV1 +#define H_PAGE_HASHPTE _RPAGE_PKEY_BIT4 + +/* PTE flags to conserve for HPTE identification */ +#define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_HASHPTE | \ + H_PAGE_F_SECOND | H_PAGE_F_GIX) +/* + * Not supported by 4k linux page size + */ +#define H_PAGE_4K_PFN 0x0 +#define H_PAGE_THP_HUGE 0x0 +#define H_PAGE_COMBO 0x0 + +/* 8 bytes per each pte entry */ +#define H_PTE_FRAG_SIZE_SHIFT (H_PTE_INDEX_SIZE + 3) +#define H_PTE_FRAG_NR (PAGE_SIZE >> H_PTE_FRAG_SIZE_SHIFT) +#define H_PMD_FRAG_SIZE_SHIFT (H_PMD_INDEX_SIZE + 3) +#define H_PMD_FRAG_NR (PAGE_SIZE >> H_PMD_FRAG_SIZE_SHIFT) + +/* memory key bits, only 8 keys supported */ +#define H_PTE_PKEY_BIT4 0 +#define H_PTE_PKEY_BIT3 0 +#define H_PTE_PKEY_BIT2 _RPAGE_PKEY_BIT3 +#define H_PTE_PKEY_BIT1 _RPAGE_PKEY_BIT2 +#define H_PTE_PKEY_BIT0 _RPAGE_PKEY_BIT1 + + +/* + * On all 4K setups, remap_4k_pfn() equates to remap_pfn_range() + */ +#define remap_4k_pfn(vma, addr, pfn, prot) \ + remap_pfn_range((vma), (addr), (pfn), PAGE_SIZE, (prot)) + +/* + * With 4K page size the real_pte machinery is all nops. + */ +static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep, int offset) +{ + return (real_pte_t){pte}; +} + +#define __rpte_to_pte(r) ((r).pte) + +static inline unsigned long __rpte_to_hidx(real_pte_t rpte, unsigned long index) +{ + return pte_val(__rpte_to_pte(rpte)) >> H_PAGE_F_GIX_SHIFT; +} + +#define pte_iterate_hashed_subpages(rpte, psize, va, index, shift) \ + do { \ + index = 0; \ + shift = mmu_psize_defs[psize].shift; \ + +#define pte_iterate_hashed_end() } while(0) + +/* + * We expect this to be called only for user addresses or kernel virtual + * addresses other than the linear mapping. + */ +#define pte_pagesize_index(mm, addr, pte) MMU_PAGE_4K + +/* + * 4K PTE format is different from 64K PTE format. Saving the hash_slot is just + * a matter of returning the PTE bits that need to be modified. On 64K PTE, + * things are a little more involved and hence needs many more parameters to + * accomplish the same. However we want to abstract this out from the caller by + * keeping the prototype consistent across the two formats. + */ +static inline unsigned long pte_set_hidx(pte_t *ptep, real_pte_t rpte, + unsigned int subpg_index, unsigned long hidx, + int offset) +{ + return (hidx << H_PAGE_F_GIX_SHIFT) & + (H_PAGE_F_SECOND | H_PAGE_F_GIX); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + +static inline char *get_hpte_slot_array(pmd_t *pmdp) +{ + BUG(); + return NULL; +} + +static inline unsigned int hpte_valid(unsigned char *hpte_slot_array, int index) +{ + BUG(); + return 0; +} + +static inline unsigned int hpte_hash_index(unsigned char *hpte_slot_array, + int index) +{ + BUG(); + return 0; +} + +static inline void mark_hpte_slot_valid(unsigned char *hpte_slot_array, + unsigned int index, unsigned int hidx) +{ + BUG(); +} + +static inline int hash__pmd_trans_huge(pmd_t pmd) +{ + return 0; +} + +static inline pmd_t hash__pmd_mkhuge(pmd_t pmd) +{ + BUG(); + return pmd; +} + +extern unsigned long hash__pmd_hugepage_update(struct mm_struct *mm, + unsigned long addr, pmd_t *pmdp, + unsigned long clr, unsigned long set); +extern pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp); +extern void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, + pgtable_t pgtable); +extern pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); +extern pmd_t hash__pmdp_huge_get_and_clear(struct mm_struct *mm, + unsigned long addr, pmd_t *pmdp); +extern int hash__has_transparent_hugepage(void); +#endif + +#endif /* !__ASSEMBLER__ */ + +#endif /* _ASM_POWERPC_BOOK3S_64_HASH_4K_H */ diff --git a/arch/powerpc/include/asm/book3s/64/hash-64k.h b/arch/powerpc/include/asm/book3s/64/hash-64k.h new file mode 100644 index 000000000000..7deb3a66890b --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/hash-64k.h @@ -0,0 +1,286 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_64_HASH_64K_H +#define _ASM_POWERPC_BOOK3S_64_HASH_64K_H + +#define H_PTE_INDEX_SIZE 8 // size: 8B << 8 = 2KB, maps 2^8 x 64KB = 16MB +#define H_PMD_INDEX_SIZE 10 // size: 8B << 10 = 8KB, maps 2^10 x 16MB = 16GB +#define H_PUD_INDEX_SIZE 10 // size: 8B << 10 = 8KB, maps 2^10 x 16GB = 16TB +#define H_PGD_INDEX_SIZE 8 // size: 8B << 8 = 2KB, maps 2^8 x 16TB = 4PB + +/* + * If we store section details in page->flags we can't increase the MAX_PHYSMEM_BITS + * if we increase SECTIONS_WIDTH we will not store node details in page->flags and + * page_to_nid does a page->section->node lookup + * Hence only increase for VMEMMAP. Further depending on SPARSEMEM_EXTREME reduce + * memory requirements with large number of sections. + * 51 bits is the max physical real address on POWER9 + */ +#if defined(CONFIG_SPARSEMEM_VMEMMAP) && defined(CONFIG_SPARSEMEM_EXTREME) +#define H_MAX_PHYSMEM_BITS 51 +#else +#define H_MAX_PHYSMEM_BITS 46 +#endif + +/* + * Each context is 512TB size. SLB miss for first context/default context + * is handled in the hotpath. + */ +#define MAX_EA_BITS_PER_CONTEXT 49 +#define REGION_SHIFT MAX_EA_BITS_PER_CONTEXT + +/* + * We use one context for each MAP area. + */ +#define H_KERN_MAP_SIZE (1UL << MAX_EA_BITS_PER_CONTEXT) + +/* + * Define the address range of the kernel non-linear virtual area + * 2PB + */ +#define H_KERN_VIRT_START ASM_CONST(0xc008000000000000) + +/* + * 64k aligned address free up few of the lower bits of RPN for us + * We steal that here. For more deatils look at pte_pfn/pfn_pte() + */ +#define H_PAGE_COMBO _RPAGE_RPN0 /* this is a combo 4k page */ +#define H_PAGE_4K_PFN _RPAGE_RPN1 /* PFN is for a single 4k page */ +#define H_PAGE_BUSY _RPAGE_RSV1 /* software: PTE & hash are busy */ +#define H_PAGE_HASHPTE _RPAGE_RPN43 /* PTE has associated HPTE */ + +/* memory key bits. */ +#define H_PTE_PKEY_BIT4 _RPAGE_PKEY_BIT4 +#define H_PTE_PKEY_BIT3 _RPAGE_PKEY_BIT3 +#define H_PTE_PKEY_BIT2 _RPAGE_PKEY_BIT2 +#define H_PTE_PKEY_BIT1 _RPAGE_PKEY_BIT1 +#define H_PTE_PKEY_BIT0 _RPAGE_PKEY_BIT0 + +/* + * We need to differentiate between explicit huge page and THP huge + * page, since THP huge page also need to track real subpage details + */ +#define H_PAGE_THP_HUGE H_PAGE_4K_PFN + +/* PTE flags to conserve for HPTE identification */ +#define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_HASHPTE | H_PAGE_COMBO) +/* + * We use a 2K PTE page fragment and another 2K for storing + * real_pte_t hash index + * 8 bytes per each pte entry and another 8 bytes for storing + * slot details. + */ +#define H_PTE_FRAG_SIZE_SHIFT (H_PTE_INDEX_SIZE + 3 + 1) +#define H_PTE_FRAG_NR (PAGE_SIZE >> H_PTE_FRAG_SIZE_SHIFT) + +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE) +#define H_PMD_FRAG_SIZE_SHIFT (H_PMD_INDEX_SIZE + 3 + 1) +#else +#define H_PMD_FRAG_SIZE_SHIFT (H_PMD_INDEX_SIZE + 3) +#endif +#define H_PMD_FRAG_NR (PAGE_SIZE >> H_PMD_FRAG_SIZE_SHIFT) + +#ifndef __ASSEMBLER__ +#include <asm/errno.h> + +/* + * With 64K pages on hash table, we have a special PTE format that + * uses a second "half" of the page table to encode sub-page information + * in order to deal with 64K made of 4K HW pages. Thus we override the + * generic accessors and iterators here + */ +#define __real_pte __real_pte +static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep, int offset) +{ + real_pte_t rpte; + unsigned long *hidxp; + + rpte.pte = pte; + + /* + * Ensure that we do not read the hidx before we read the PTE. Because + * the writer side is expected to finish writing the hidx first followed + * by the PTE, by using smp_wmb(). pte_set_hash_slot() ensures that. + */ + smp_rmb(); + + hidxp = (unsigned long *)(ptep + offset); + rpte.hidx = *hidxp; + return rpte; +} + +/* + * shift the hidx representation by one-modulo-0xf; i.e hidx 0 is respresented + * as 1, 1 as 2,... , and 0xf as 0. This convention lets us represent a + * invalid hidx 0xf with a 0x0 bit value. PTEs are anyway zero'd when + * allocated. We dont have to zero them gain; thus save on the initialization. + */ +#define HIDX_UNSHIFT_BY_ONE(x) ((x + 0xfUL) & 0xfUL) /* shift backward by one */ +#define HIDX_SHIFT_BY_ONE(x) ((x + 0x1UL) & 0xfUL) /* shift forward by one */ +#define HIDX_BITS(x, index) (x << (index << 2)) +#define BITS_TO_HIDX(x, index) ((x >> (index << 2)) & 0xfUL) +#define INVALID_RPTE_HIDX 0x0UL + +static inline unsigned long __rpte_to_hidx(real_pte_t rpte, unsigned long index) +{ + return HIDX_UNSHIFT_BY_ONE(BITS_TO_HIDX(rpte.hidx, index)); +} + +/* + * Commit the hidx and return PTE bits that needs to be modified. The caller is + * expected to modify the PTE bits accordingly and commit the PTE to memory. + */ +static inline unsigned long pte_set_hidx(pte_t *ptep, real_pte_t rpte, + unsigned int subpg_index, + unsigned long hidx, int offset) +{ + unsigned long *hidxp = (unsigned long *)(ptep + offset); + + rpte.hidx &= ~HIDX_BITS(0xfUL, subpg_index); + *hidxp = rpte.hidx | HIDX_BITS(HIDX_SHIFT_BY_ONE(hidx), subpg_index); + + /* + * Anyone reading PTE must ensure hidx bits are read after reading the + * PTE by using the read-side barrier smp_rmb(). __real_pte() can be + * used for that. + */ + smp_wmb(); + + /* No PTE bits to be modified, return 0x0UL */ + return 0x0UL; +} + +#define __rpte_to_pte(r) ((r).pte) +extern bool __rpte_sub_valid(real_pte_t rpte, unsigned long index); +/* + * Trick: we set __end to va + 64k, which happens works for + * a 16M page as well as we want only one iteration + */ +#define pte_iterate_hashed_subpages(rpte, psize, vpn, index, shift) \ + do { \ + unsigned long __end = vpn + (1UL << (PAGE_SHIFT - VPN_SHIFT)); \ + unsigned __split = (psize == MMU_PAGE_4K || \ + psize == MMU_PAGE_64K_AP); \ + shift = mmu_psize_defs[psize].shift; \ + for (index = 0; vpn < __end; index++, \ + vpn += (1L << (shift - VPN_SHIFT))) { \ + if (!__split || __rpte_sub_valid(rpte, index)) + +#define pte_iterate_hashed_end() } } while(0) + +#define pte_pagesize_index(mm, addr, pte) \ + (((pte) & H_PAGE_COMBO)? MMU_PAGE_4K: MMU_PAGE_64K) + +extern int remap_pfn_range(struct vm_area_struct *, unsigned long addr, + unsigned long pfn, unsigned long size, pgprot_t); +static inline int hash__remap_4k_pfn(struct vm_area_struct *vma, unsigned long addr, + unsigned long pfn, pgprot_t prot) +{ + if (pfn > (PTE_RPN_MASK >> PAGE_SHIFT)) { + WARN(1, "remap_4k_pfn called with wrong pfn value\n"); + return -EINVAL; + } + return remap_pfn_range(vma, addr, pfn, PAGE_SIZE, + __pgprot(pgprot_val(prot) | H_PAGE_4K_PFN)); +} + +#define H_PTE_TABLE_SIZE PTE_FRAG_SIZE +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined (CONFIG_HUGETLB_PAGE) +#define H_PMD_TABLE_SIZE ((sizeof(pmd_t) << PMD_INDEX_SIZE) + \ + (sizeof(unsigned long) << PMD_INDEX_SIZE)) +#else +#define H_PMD_TABLE_SIZE (sizeof(pmd_t) << PMD_INDEX_SIZE) +#endif +#ifdef CONFIG_HUGETLB_PAGE +#define H_PUD_TABLE_SIZE ((sizeof(pud_t) << PUD_INDEX_SIZE) + \ + (sizeof(unsigned long) << PUD_INDEX_SIZE)) +#else +#define H_PUD_TABLE_SIZE (sizeof(pud_t) << PUD_INDEX_SIZE) +#endif +#define H_PGD_TABLE_SIZE (sizeof(pgd_t) << PGD_INDEX_SIZE) + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline char *get_hpte_slot_array(pmd_t *pmdp) +{ + /* + * The hpte hindex is stored in the pgtable whose address is in the + * second half of the PMD + * + * Order this load with the test for pmd_trans_huge in the caller + */ + smp_rmb(); + return *(char **)(pmdp + PTRS_PER_PMD); + + +} +/* + * The linux hugepage PMD now include the pmd entries followed by the address + * to the stashed pgtable_t. The stashed pgtable_t contains the hpte bits. + * [ 000 | 1 bit secondary | 3 bit hidx | 1 bit valid]. We use one byte per + * each HPTE entry. With 16MB hugepage and 64K HPTE we need 256 entries and + * with 4K HPTE we need 4096 entries. Both will fit in a 4K pgtable_t. + * + * The top three bits are intentionally left as zero. This memory location + * are also used as normal page PTE pointers. So if we have any pointers + * left around while we collapse a hugepage, we need to make sure + * _PAGE_PRESENT bit of that is zero when we look at them + */ +static inline unsigned int hpte_valid(unsigned char *hpte_slot_array, int index) +{ + return hpte_slot_array[index] & 0x1; +} + +static inline unsigned int hpte_hash_index(unsigned char *hpte_slot_array, + int index) +{ + return hpte_slot_array[index] >> 1; +} + +static inline void mark_hpte_slot_valid(unsigned char *hpte_slot_array, + unsigned int index, unsigned int hidx) +{ + hpte_slot_array[index] = (hidx << 1) | 0x1; +} + +/* + * + * For core kernel code by design pmd_trans_huge is never run on any hugetlbfs + * page. The hugetlbfs page table walking and mangling paths are totally + * separated form the core VM paths and they're differentiated by + * VM_HUGETLB being set on vm_flags well before any pmd_trans_huge could run. + * + * pmd_trans_huge() is defined as false at build time if + * CONFIG_TRANSPARENT_HUGEPAGE=n to optimize away code blocks at build + * time in such case. + * + * For ppc64 we need to differntiate from explicit hugepages from THP, because + * for THP we also track the subpage details at the pmd level. We don't do + * that for explicit huge pages. + * + */ +static inline int hash__pmd_trans_huge(pmd_t pmd) +{ + return !!((pmd_val(pmd) & (_PAGE_PTE | H_PAGE_THP_HUGE)) == + (_PAGE_PTE | H_PAGE_THP_HUGE)); +} + +static inline pmd_t hash__pmd_mkhuge(pmd_t pmd) +{ + return __pmd(pmd_val(pmd) | (_PAGE_PTE | H_PAGE_THP_HUGE)); +} + +extern unsigned long hash__pmd_hugepage_update(struct mm_struct *mm, + unsigned long addr, pmd_t *pmdp, + unsigned long clr, unsigned long set); +extern pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp); +extern void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, + pgtable_t pgtable); +extern pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); +extern pmd_t hash__pmdp_huge_get_and_clear(struct mm_struct *mm, + unsigned long addr, pmd_t *pmdp); +extern int hash__has_transparent_hugepage(void); +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +#endif /* __ASSEMBLER__ */ + +#endif /* _ASM_POWERPC_BOOK3S_64_HASH_64K_H */ diff --git a/arch/powerpc/include/asm/book3s/64/hash-pkey.h b/arch/powerpc/include/asm/book3s/64/hash-pkey.h new file mode 100644 index 000000000000..6c5564c4fae4 --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/hash-pkey.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_64_HASH_PKEY_H +#define _ASM_POWERPC_BOOK3S_64_HASH_PKEY_H + +/* We use key 3 for KERNEL */ +#define HASH_DEFAULT_KERNEL_KEY (HPTE_R_KEY_BIT0 | HPTE_R_KEY_BIT1) + +static inline u64 hash__vmflag_to_pte_pkey_bits(u64 vm_flags) +{ + return (((vm_flags & VM_PKEY_BIT0) ? H_PTE_PKEY_BIT0 : 0x0UL) | + ((vm_flags & VM_PKEY_BIT1) ? H_PTE_PKEY_BIT1 : 0x0UL) | + ((vm_flags & VM_PKEY_BIT2) ? H_PTE_PKEY_BIT2 : 0x0UL) | + ((vm_flags & VM_PKEY_BIT3) ? H_PTE_PKEY_BIT3 : 0x0UL) | + ((vm_flags & VM_PKEY_BIT4) ? H_PTE_PKEY_BIT4 : 0x0UL)); +} + +static inline u64 pte_to_hpte_pkey_bits(u64 pteflags, unsigned long flags) +{ + unsigned long pte_pkey; + + pte_pkey = (((pteflags & H_PTE_PKEY_BIT4) ? HPTE_R_KEY_BIT4 : 0x0UL) | + ((pteflags & H_PTE_PKEY_BIT3) ? HPTE_R_KEY_BIT3 : 0x0UL) | + ((pteflags & H_PTE_PKEY_BIT2) ? HPTE_R_KEY_BIT2 : 0x0UL) | + ((pteflags & H_PTE_PKEY_BIT1) ? HPTE_R_KEY_BIT1 : 0x0UL) | + ((pteflags & H_PTE_PKEY_BIT0) ? HPTE_R_KEY_BIT0 : 0x0UL)); + + if (mmu_has_feature(MMU_FTR_KUAP) || + mmu_has_feature(MMU_FTR_BOOK3S_KUEP)) { + if ((pte_pkey == 0) && (flags & HPTE_USE_KERNEL_KEY)) + return HASH_DEFAULT_KERNEL_KEY; + } + + return pte_pkey; +} + +static inline u16 hash__pte_to_pkey_bits(u64 pteflags) +{ + return (((pteflags & H_PTE_PKEY_BIT4) ? 0x10 : 0x0UL) | + ((pteflags & H_PTE_PKEY_BIT3) ? 0x8 : 0x0UL) | + ((pteflags & H_PTE_PKEY_BIT2) ? 0x4 : 0x0UL) | + ((pteflags & H_PTE_PKEY_BIT1) ? 0x2 : 0x0UL) | + ((pteflags & H_PTE_PKEY_BIT0) ? 0x1 : 0x0UL)); +} + +#endif diff --git a/arch/powerpc/include/asm/book3s/64/hash.h b/arch/powerpc/include/asm/book3s/64/hash.h new file mode 100644 index 000000000000..5a8cbd496731 --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/hash.h @@ -0,0 +1,300 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_64_HASH_H +#define _ASM_POWERPC_BOOK3S_64_HASH_H +#ifdef __KERNEL__ + +#include <asm/asm-const.h> +#include <asm/book3s/64/slice.h> + +/* + * Common bits between 4K and 64K pages in a linux-style PTE. + * Additional bits may be defined in pgtable-hash64-*.h + * + */ +#define H_PTE_NONE_MASK _PAGE_HPTEFLAGS + +#ifdef CONFIG_PPC_64K_PAGES +#include <asm/book3s/64/hash-64k.h> +#else +#include <asm/book3s/64/hash-4k.h> +#endif + +#define H_PTRS_PER_PTE (1 << H_PTE_INDEX_SIZE) +#define H_PTRS_PER_PMD (1 << H_PMD_INDEX_SIZE) +#define H_PTRS_PER_PUD (1 << H_PUD_INDEX_SIZE) + +/* Bits to set in a PMD/PUD/PGD entry valid bit*/ +#define HASH_PMD_VAL_BITS (0x8000000000000000UL) +#define HASH_PUD_VAL_BITS (0x8000000000000000UL) +#define HASH_PGD_VAL_BITS (0x8000000000000000UL) + +/* + * Size of EA range mapped by our pagetables. + */ +#define H_PGTABLE_EADDR_SIZE (H_PTE_INDEX_SIZE + H_PMD_INDEX_SIZE + \ + H_PUD_INDEX_SIZE + H_PGD_INDEX_SIZE + PAGE_SHIFT) +#define H_PGTABLE_RANGE (ASM_CONST(1) << H_PGTABLE_EADDR_SIZE) +/* + * Top 2 bits are ignored in page table walk. + */ +#define EA_MASK (~(0xcUL << 60)) + +/* + * We store the slot details in the second half of page table. + * Increase the pud level table so that hugetlb ptes can be stored + * at pud level. + */ +#if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_PPC_64K_PAGES) +#define H_PUD_CACHE_INDEX (H_PUD_INDEX_SIZE + 1) +#else +#define H_PUD_CACHE_INDEX (H_PUD_INDEX_SIZE) +#endif + +/* + * +------------------------------+ + * | | + * | | + * | | + * +------------------------------+ Kernel virtual map end (0xc00e000000000000) + * | | + * | | + * | 512TB/16TB of vmemmap | + * | | + * | | + * +------------------------------+ Kernel vmemmap start + * | | + * | 512TB/16TB of IO map | + * | | + * +------------------------------+ Kernel IO map start + * | | + * | 512TB/16TB of vmap | + * | | + * +------------------------------+ Kernel virt start (0xc008000000000000) + * | | + * | | + * | | + * +------------------------------+ Kernel linear (0xc.....) + */ + +#define H_VMALLOC_START H_KERN_VIRT_START +#define H_VMALLOC_SIZE H_KERN_MAP_SIZE +#define H_VMALLOC_END (H_VMALLOC_START + H_VMALLOC_SIZE) + +#define H_KERN_IO_START H_VMALLOC_END +#define H_KERN_IO_SIZE H_KERN_MAP_SIZE +#define H_KERN_IO_END (H_KERN_IO_START + H_KERN_IO_SIZE) + +#define H_VMEMMAP_START H_KERN_IO_END +#define H_VMEMMAP_SIZE H_KERN_MAP_SIZE +#define H_VMEMMAP_END (H_VMEMMAP_START + H_VMEMMAP_SIZE) + +#define NON_LINEAR_REGION_ID(ea) ((((unsigned long)ea - H_KERN_VIRT_START) >> REGION_SHIFT) + 2) + +/* + * Region IDs + */ +#define USER_REGION_ID 0 +#define LINEAR_MAP_REGION_ID 1 +#define VMALLOC_REGION_ID NON_LINEAR_REGION_ID(H_VMALLOC_START) +#define IO_REGION_ID NON_LINEAR_REGION_ID(H_KERN_IO_START) +#define VMEMMAP_REGION_ID NON_LINEAR_REGION_ID(H_VMEMMAP_START) +#define INVALID_REGION_ID (VMEMMAP_REGION_ID + 1) + +/* + * Defines the address of the vmemap area, in its own region on + * hash table CPUs. + */ + +/* PTEIDX nibble */ +#define _PTEIDX_SECONDARY 0x8 +#define _PTEIDX_GROUP_IX 0x7 + +#define H_PMD_BAD_BITS (PTE_TABLE_SIZE-1) +#define H_PUD_BAD_BITS (PMD_TABLE_SIZE-1) + +#ifndef __ASSEMBLER__ +static inline int get_region_id(unsigned long ea) +{ + int region_id; + int id = (ea >> 60UL); + + if (id == 0) + return USER_REGION_ID; + + if (id != (PAGE_OFFSET >> 60)) + return INVALID_REGION_ID; + + if (ea < H_KERN_VIRT_START) + return LINEAR_MAP_REGION_ID; + + BUILD_BUG_ON(NON_LINEAR_REGION_ID(H_VMALLOC_START) != 2); + + region_id = NON_LINEAR_REGION_ID(ea); + return region_id; +} + +static inline int hash__pmd_same(pmd_t pmd_a, pmd_t pmd_b) +{ + return (((pmd_raw(pmd_a) ^ pmd_raw(pmd_b)) & ~cpu_to_be64(_PAGE_HPTEFLAGS)) == 0); +} + +#define hash__pmd_bad(pmd) (pmd_val(pmd) & H_PMD_BAD_BITS) + +/* + * pud comparison that will work with both pte and page table pointer. + */ +static inline int hash__pud_same(pud_t pud_a, pud_t pud_b) +{ + return (((pud_raw(pud_a) ^ pud_raw(pud_b)) & ~cpu_to_be64(_PAGE_HPTEFLAGS)) == 0); +} +#define hash__pud_bad(pud) (pud_val(pud) & H_PUD_BAD_BITS) + +static inline int hash__p4d_bad(p4d_t p4d) +{ + return (p4d_val(p4d) == 0); +} +#ifdef CONFIG_STRICT_KERNEL_RWX +extern void hash__mark_rodata_ro(void); +extern void hash__mark_initmem_nx(void); +#endif + +extern void hpte_need_flush(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned long pte, int huge); +unsigned long htab_convert_pte_flags(unsigned long pteflags, unsigned long flags); +/* Atomic PTE updates */ +static inline unsigned long hash__pte_update_one(pte_t *ptep, unsigned long clr, + unsigned long set) +{ + __be64 old_be, tmp_be; + + __asm__ __volatile__( + "1: ldarx %0,0,%3 # pte_update\n\ + and. %1,%0,%6\n\ + bne- 1b \n\ + andc %1,%0,%4 \n\ + or %1,%1,%7\n\ + stdcx. %1,0,%3 \n\ + bne- 1b" + : "=&r" (old_be), "=&r" (tmp_be), "=m" (*ptep) + : "r" (ptep), "r" (cpu_to_be64(clr)), "m" (*ptep), + "r" (cpu_to_be64(H_PAGE_BUSY)), "r" (cpu_to_be64(set)) + : "cc" ); + + return be64_to_cpu(old_be); +} + +static inline unsigned long hash__pte_update(struct mm_struct *mm, + unsigned long addr, + pte_t *ptep, unsigned long clr, + unsigned long set, + int huge) +{ + unsigned long old; + + old = hash__pte_update_one(ptep, clr, set); + + if (IS_ENABLED(CONFIG_PPC_4K_PAGES) && huge) { + unsigned int psize = get_slice_psize(mm, addr); + int nb, i; + + if (psize == MMU_PAGE_16M) + nb = SZ_16M / PMD_SIZE; + else if (psize == MMU_PAGE_16G) + nb = SZ_16G / PUD_SIZE; + else + nb = 1; + + WARN_ON_ONCE(nb == 1); /* Should never happen */ + + for (i = 1; i < nb; i++) + hash__pte_update_one(ptep + i, clr, set); + } + /* huge pages use the old page table lock */ + if (!huge) + assert_pte_locked(mm, addr); + + if (old & H_PAGE_HASHPTE) + hpte_need_flush(mm, addr, ptep, old, huge); + + return old; +} + +/* Set the dirty and/or accessed bits atomically in a linux PTE, this + * function doesn't need to flush the hash entry + */ +static inline void hash__ptep_set_access_flags(pte_t *ptep, pte_t entry) +{ + __be64 old, tmp, val, mask; + + mask = cpu_to_be64(_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_READ | _PAGE_WRITE | + _PAGE_EXEC | _PAGE_SOFT_DIRTY); + + val = pte_raw(entry) & mask; + + __asm__ __volatile__( + "1: ldarx %0,0,%4\n\ + and. %1,%0,%6\n\ + bne- 1b \n\ + or %0,%3,%0\n\ + stdcx. %0,0,%4\n\ + bne- 1b" + :"=&r" (old), "=&r" (tmp), "=m" (*ptep) + :"r" (val), "r" (ptep), "m" (*ptep), "r" (cpu_to_be64(H_PAGE_BUSY)) + :"cc"); +} + +static inline int hash__pte_same(pte_t pte_a, pte_t pte_b) +{ + return (((pte_raw(pte_a) ^ pte_raw(pte_b)) & ~cpu_to_be64(_PAGE_HPTEFLAGS)) == 0); +} + +static inline int hash__pte_none(pte_t pte) +{ + return (pte_val(pte) & ~H_PTE_NONE_MASK) == 0; +} + +unsigned long pte_get_hash_gslot(unsigned long vpn, unsigned long shift, + int ssize, real_pte_t rpte, unsigned int subpg_index); + +/* This low level function performs the actual PTE insertion + * Setting the PTE depends on the MMU type and other factors. It's + * an horrible mess that I'm not going to try to clean up now but + * I'm keeping it in one place rather than spread around + */ +static inline void hash__set_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, int percpu) +{ + /* + * Anything else just stores the PTE normally. That covers all 64-bit + * cases, and 32-bit non-hash with 32-bit PTEs. + */ + *ptep = pte; +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +extern void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, unsigned long old_pmd); +#else +static inline void hpte_do_hugepage_flush(struct mm_struct *mm, + unsigned long addr, pmd_t *pmdp, + unsigned long old_pmd) +{ + WARN(1, "%s called with THP disabled\n", __func__); +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + + +int hash__map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot); +extern int __meminit hash__vmemmap_create_mapping(unsigned long start, + unsigned long page_size, + unsigned long phys); +extern void hash__vmemmap_remove_mapping(unsigned long start, + unsigned long page_size); + +int hash__create_section_mapping(unsigned long start, unsigned long end, + int nid, pgprot_t prot); +int hash__remove_section_mapping(unsigned long start, unsigned long end); + +#endif /* !__ASSEMBLER__ */ +#endif /* __KERNEL__ */ +#endif /* _ASM_POWERPC_BOOK3S_64_HASH_H */ diff --git a/arch/powerpc/include/asm/book3s/64/hugetlb.h b/arch/powerpc/include/asm/book3s/64/hugetlb.h new file mode 100644 index 000000000000..bb786694dd26 --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/hugetlb.h @@ -0,0 +1,103 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_64_HUGETLB_H +#define _ASM_POWERPC_BOOK3S_64_HUGETLB_H + +#include <asm/firmware.h> + +/* + * For radix we want generic code to handle hugetlb. But then if we want + * both hash and radix to be enabled together we need to workaround the + * limitations. + */ +void radix__flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr); +void radix__local_flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr); + +extern void radix__huge_ptep_modify_prot_commit(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t old_pte, pte_t pte); + +static inline int hstate_get_psize(struct hstate *hstate) +{ + unsigned long shift; + + shift = huge_page_shift(hstate); + if (shift == mmu_psize_defs[MMU_PAGE_2M].shift) + return MMU_PAGE_2M; + else if (shift == mmu_psize_defs[MMU_PAGE_1G].shift) + return MMU_PAGE_1G; + else if (shift == mmu_psize_defs[MMU_PAGE_16M].shift) + return MMU_PAGE_16M; + else if (shift == mmu_psize_defs[MMU_PAGE_16G].shift) + return MMU_PAGE_16G; + else { + WARN(1, "Wrong huge page shift\n"); + return mmu_virtual_psize; + } +} + +#define __HAVE_ARCH_GIGANTIC_PAGE_RUNTIME_SUPPORTED +static inline bool gigantic_page_runtime_supported(void) +{ + /* + * We used gigantic page reservation with hypervisor assist in some case. + * We cannot use runtime allocation of gigantic pages in those platforms + * This is hash translation mode LPARs. + */ + if (firmware_has_feature(FW_FEATURE_LPAR) && !radix_enabled()) + return false; + + return true; +} + +#define huge_ptep_modify_prot_start huge_ptep_modify_prot_start +extern pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep); + +#define huge_ptep_modify_prot_commit huge_ptep_modify_prot_commit +extern void huge_ptep_modify_prot_commit(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t old_pte, pte_t new_pte); + +static inline void flush_hugetlb_page(struct vm_area_struct *vma, + unsigned long vmaddr) +{ + if (radix_enabled()) + return radix__flush_hugetlb_page(vma, vmaddr); +} + +void flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr); + +static inline int check_and_get_huge_psize(int shift) +{ + int mmu_psize; + + if (shift > SLICE_HIGH_SHIFT) + return -EINVAL; + + mmu_psize = shift_to_mmu_psize(shift); + + /* + * We need to make sure that for different page sizes reported by + * firmware we only add hugetlb support for page sizes that can be + * supported by linux page table layout. + * For now we have + * Radix: 2M and 1G + * Hash: 16M and 16G + */ + if (radix_enabled()) { + if (mmu_psize != MMU_PAGE_2M && mmu_psize != MMU_PAGE_1G) + return -EINVAL; + } else { + if (mmu_psize != MMU_PAGE_16M && mmu_psize != MMU_PAGE_16G) + return -EINVAL; + } + return mmu_psize; +} + +#define arch_has_huge_bootmem_alloc arch_has_huge_bootmem_alloc + +static inline bool arch_has_huge_bootmem_alloc(void) +{ + return (firmware_has_feature(FW_FEATURE_LPAR) && !radix_enabled()); +} +#endif diff --git a/arch/powerpc/include/asm/book3s/64/kexec.h b/arch/powerpc/include/asm/book3s/64/kexec.h new file mode 100644 index 000000000000..df37a76c1e9f --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/kexec.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_POWERPC_BOOK3S_64_KEXEC_H_ +#define _ASM_POWERPC_BOOK3S_64_KEXEC_H_ + +#include <asm/plpar_wrappers.h> + +#define reset_sprs reset_sprs +static inline void reset_sprs(void) +{ + if (cpu_has_feature(CPU_FTR_ARCH_206)) { + mtspr(SPRN_AMR, 0); + mtspr(SPRN_UAMOR, 0); + } + + if (cpu_has_feature(CPU_FTR_ARCH_207S)) { + mtspr(SPRN_IAMR, 0); + if (cpu_has_feature(CPU_FTR_HVMODE)) + mtspr(SPRN_CIABR, 0); + else + plpar_set_ciabr(0); + } + + if (cpu_has_feature(CPU_FTR_ARCH_31)) { + mtspr(SPRN_DEXCR, 0); + mtspr(SPRN_HASHKEYR, 0); + } + + /* Do we need isync()? We are going via a kexec reset */ + isync(); +} + +#endif diff --git a/arch/powerpc/include/asm/book3s/64/kup.h b/arch/powerpc/include/asm/book3s/64/kup.h new file mode 100644 index 000000000000..03aec3c6c851 --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/kup.h @@ -0,0 +1,418 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_64_KUP_H +#define _ASM_POWERPC_BOOK3S_64_KUP_H + +#include <linux/const.h> +#include <asm/reg.h> + +#define AMR_KUAP_BLOCK_READ UL(0x5455555555555555) +#define AMR_KUAP_BLOCK_WRITE UL(0xa8aaaaaaaaaaaaaa) +#define AMR_KUEP_BLOCKED UL(0x5455555555555555) +#define AMR_KUAP_BLOCKED (AMR_KUAP_BLOCK_READ | AMR_KUAP_BLOCK_WRITE) + +#ifdef __ASSEMBLER__ + +.macro kuap_user_restore gpr1, gpr2 +#if defined(CONFIG_PPC_PKEY) + BEGIN_MMU_FTR_SECTION_NESTED(67) + b 100f // skip_restore_amr + END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY, 67) + /* + * AMR and IAMR are going to be different when + * returning to userspace. + */ + ld \gpr1, STACK_REGS_AMR(r1) + + /* + * If kuap feature is not enabled, do the mtspr + * only if AMR value is different. + */ + BEGIN_MMU_FTR_SECTION_NESTED(68) + mfspr \gpr2, SPRN_AMR + cmpd \gpr1, \gpr2 + beq 99f + END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_KUAP, 68) + + isync + mtspr SPRN_AMR, \gpr1 +99: + /* + * Restore IAMR only when returning to userspace + */ + ld \gpr1, STACK_REGS_IAMR(r1) + + /* + * If kuep feature is not enabled, do the mtspr + * only if IAMR value is different. + */ + BEGIN_MMU_FTR_SECTION_NESTED(69) + mfspr \gpr2, SPRN_IAMR + cmpd \gpr1, \gpr2 + beq 100f + END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_BOOK3S_KUEP, 69) + + isync + mtspr SPRN_IAMR, \gpr1 + +100: //skip_restore_amr + /* No isync required, see kuap_user_restore() */ +#endif +.endm + +.macro kuap_kernel_restore gpr1, gpr2 +#if defined(CONFIG_PPC_PKEY) + + BEGIN_MMU_FTR_SECTION_NESTED(67) + /* + * AMR is going to be mostly the same since we are + * returning to the kernel. Compare and do a mtspr. + */ + ld \gpr2, STACK_REGS_AMR(r1) + mfspr \gpr1, SPRN_AMR + cmpd \gpr1, \gpr2 + beq 100f + isync + mtspr SPRN_AMR, \gpr2 + /* + * No isync required, see kuap_restore_amr() + * No need to restore IAMR when returning to kernel space. + */ +100: + END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_KUAP, 67) +#endif +.endm + +#ifdef CONFIG_PPC_KUAP +.macro kuap_check_amr gpr1, gpr2 +#ifdef CONFIG_PPC_KUAP_DEBUG + BEGIN_MMU_FTR_SECTION_NESTED(67) + mfspr \gpr1, SPRN_AMR + /* Prevent access to userspace using any key values */ + LOAD_REG_IMMEDIATE(\gpr2, AMR_KUAP_BLOCKED) +999: tdne \gpr1, \gpr2 + EMIT_WARN_ENTRY 999b, __FILE__, __LINE__, (BUGFLAG_WARNING | BUGFLAG_ONCE) + END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_KUAP, 67) +#endif +.endm +#endif + +/* + * if (pkey) { + * + * save AMR -> stack; + * if (kuap) { + * if (AMR != BLOCKED) + * KUAP_BLOCKED -> AMR; + * } + * if (from_user) { + * save IAMR -> stack; + * if (kuep) { + * KUEP_BLOCKED ->IAMR + * } + * } + * return; + * } + * + * if (kuap) { + * if (from_kernel) { + * save AMR -> stack; + * if (AMR != BLOCKED) + * KUAP_BLOCKED -> AMR; + * } + * + * } + */ +.macro kuap_save_amr_and_lock gpr1, gpr2, use_cr, msr_pr_cr +#if defined(CONFIG_PPC_PKEY) + + /* + * if both pkey and kuap is disabled, nothing to do + */ + BEGIN_MMU_FTR_SECTION_NESTED(68) + b 100f // skip_save_amr + END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY | MMU_FTR_KUAP, 68) + + /* + * if pkey is disabled and we are entering from userspace + * don't do anything. + */ + BEGIN_MMU_FTR_SECTION_NESTED(67) + .ifnb \msr_pr_cr + /* + * Without pkey we are not changing AMR outside the kernel + * hence skip this completely. + */ + bne \msr_pr_cr, 100f // from userspace + .endif + END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY, 67) + + /* + * pkey is enabled or pkey is disabled but entering from kernel + */ + mfspr \gpr1, SPRN_AMR + std \gpr1, STACK_REGS_AMR(r1) + + /* + * update kernel AMR with AMR_KUAP_BLOCKED only + * if KUAP feature is enabled + */ + BEGIN_MMU_FTR_SECTION_NESTED(69) + LOAD_REG_IMMEDIATE(\gpr2, AMR_KUAP_BLOCKED) + cmpd \use_cr, \gpr1, \gpr2 + beq \use_cr, 102f + /* + * We don't isync here because we very recently entered via an interrupt + */ + mtspr SPRN_AMR, \gpr2 + isync +102: + END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_KUAP, 69) + + /* + * if entering from kernel we don't need save IAMR + */ + .ifnb \msr_pr_cr + beq \msr_pr_cr, 100f // from kernel space + mfspr \gpr1, SPRN_IAMR + std \gpr1, STACK_REGS_IAMR(r1) + + /* + * update kernel IAMR with AMR_KUEP_BLOCKED only + * if KUEP feature is enabled + */ + BEGIN_MMU_FTR_SECTION_NESTED(70) + LOAD_REG_IMMEDIATE(\gpr2, AMR_KUEP_BLOCKED) + mtspr SPRN_IAMR, \gpr2 + isync + END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_BOOK3S_KUEP, 70) + .endif + +100: // skip_save_amr +#endif +.endm + +#else /* !__ASSEMBLER__ */ + +#include <linux/jump_label.h> +#include <linux/sched.h> + +DECLARE_STATIC_KEY_FALSE(uaccess_flush_key); + +#ifdef CONFIG_PPC_PKEY + +extern u64 __ro_after_init default_uamor; +extern u64 __ro_after_init default_amr; +extern u64 __ro_after_init default_iamr; + +#include <asm/mmu.h> +#include <asm/ptrace.h> + +/* usage of kthread_use_mm() should inherit the + * AMR value of the operating address space. But, the AMR value is + * thread-specific and we inherit the address space and not thread + * access restrictions. Because of this ignore AMR value when accessing + * userspace via kernel thread. + */ +static __always_inline u64 current_thread_amr(void) +{ + if (current->thread.regs) + return current->thread.regs->amr; + return default_amr; +} + +static __always_inline u64 current_thread_iamr(void) +{ + if (current->thread.regs) + return current->thread.regs->iamr; + return default_iamr; +} +#endif /* CONFIG_PPC_PKEY */ + +#ifdef CONFIG_PPC_KUAP + +static __always_inline void kuap_user_restore(struct pt_regs *regs) +{ + bool restore_amr = false, restore_iamr = false; + unsigned long amr, iamr; + + if (!mmu_has_feature(MMU_FTR_PKEY)) + return; + + if (!mmu_has_feature(MMU_FTR_KUAP)) { + amr = mfspr(SPRN_AMR); + if (amr != regs->amr) + restore_amr = true; + } else { + restore_amr = true; + } + + if (!mmu_has_feature(MMU_FTR_BOOK3S_KUEP)) { + iamr = mfspr(SPRN_IAMR); + if (iamr != regs->iamr) + restore_iamr = true; + } else { + restore_iamr = true; + } + + + if (restore_amr || restore_iamr) { + isync(); + if (restore_amr) + mtspr(SPRN_AMR, regs->amr); + if (restore_iamr) + mtspr(SPRN_IAMR, regs->iamr); + } + /* + * No isync required here because we are about to rfi + * back to previous context before any user accesses + * would be made, which is a CSI. + */ +} + +static __always_inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long amr) +{ + if (likely(regs->amr == amr)) + return; + + isync(); + mtspr(SPRN_AMR, regs->amr); + /* + * No isync required here because we are about to rfi + * back to previous context before any user accesses + * would be made, which is a CSI. + * + * No need to restore IAMR when returning to kernel space. + */ +} + +static __always_inline unsigned long __kuap_get_and_assert_locked(void) +{ + unsigned long amr = mfspr(SPRN_AMR); + + if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG)) /* kuap_check_amr() */ + WARN_ON_ONCE(amr != AMR_KUAP_BLOCKED); + return amr; +} +#define __kuap_get_and_assert_locked __kuap_get_and_assert_locked + +/* __kuap_lock() not required, book3s/64 does that in ASM */ + +/* + * We support individually allowing read or write, but we don't support nesting + * because that would require an expensive read/modify write of the AMR. + */ + +static __always_inline unsigned long get_kuap(void) +{ + /* + * We return AMR_KUAP_BLOCKED when we don't support KUAP because + * prevent_user_access_return needs to return AMR_KUAP_BLOCKED to + * cause restore_user_access to do a flush. + * + * This has no effect in terms of actually blocking things on hash, + * so it doesn't break anything. + */ + if (!mmu_has_feature(MMU_FTR_KUAP)) + return AMR_KUAP_BLOCKED; + + return mfspr(SPRN_AMR); +} + +static __always_inline void set_kuap(unsigned long value) +{ + if (!mmu_has_feature(MMU_FTR_KUAP)) + return; + + /* + * ISA v3.0B says we need a CSI (Context Synchronising Instruction) both + * before and after the move to AMR. See table 6 on page 1134. + */ + isync(); + mtspr(SPRN_AMR, value); + isync(); +} + +static __always_inline bool +__bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write) +{ + /* + * For radix this will be a storage protection fault (DSISR_PROTFAULT). + * For hash this will be a key fault (DSISR_KEYFAULT) + */ + /* + * We do have exception table entry, but accessing the + * userspace results in fault. This could be because we + * didn't unlock the AMR or access is denied by userspace + * using a key value that blocks access. We are only interested + * in catching the use case of accessing without unlocking + * the AMR. Hence check for BLOCK_WRITE/READ against AMR. + */ + if (is_write) { + return (regs->amr & AMR_KUAP_BLOCK_WRITE) == AMR_KUAP_BLOCK_WRITE; + } + return (regs->amr & AMR_KUAP_BLOCK_READ) == AMR_KUAP_BLOCK_READ; +} + +static __always_inline void allow_user_access(void __user *to, const void __user *from, + unsigned long size, unsigned long dir) +{ + unsigned long thread_amr = 0; + + // This is written so we can resolve to a single case at build time + BUILD_BUG_ON(!__builtin_constant_p(dir)); + + if (mmu_has_feature(MMU_FTR_PKEY)) + thread_amr = current_thread_amr(); + + if (dir == KUAP_READ) + set_kuap(thread_amr | AMR_KUAP_BLOCK_WRITE); + else if (dir == KUAP_WRITE) + set_kuap(thread_amr | AMR_KUAP_BLOCK_READ); + else if (dir == KUAP_READ_WRITE) + set_kuap(thread_amr); + else + BUILD_BUG(); +} + +#else /* CONFIG_PPC_KUAP */ + +static __always_inline unsigned long get_kuap(void) +{ + return AMR_KUAP_BLOCKED; +} + +static __always_inline void set_kuap(unsigned long value) { } + +static __always_inline void allow_user_access(void __user *to, const void __user *from, + unsigned long size, unsigned long dir) +{ } + +#endif /* !CONFIG_PPC_KUAP */ + +static __always_inline void prevent_user_access(unsigned long dir) +{ + set_kuap(AMR_KUAP_BLOCKED); + if (static_branch_unlikely(&uaccess_flush_key)) + do_uaccess_flush(); +} + +static __always_inline unsigned long prevent_user_access_return(void) +{ + unsigned long flags = get_kuap(); + + set_kuap(AMR_KUAP_BLOCKED); + if (static_branch_unlikely(&uaccess_flush_key)) + do_uaccess_flush(); + + return flags; +} + +static __always_inline void restore_user_access(unsigned long flags) +{ + set_kuap(flags); + if (static_branch_unlikely(&uaccess_flush_key) && flags == AMR_KUAP_BLOCKED) + do_uaccess_flush(); +} +#endif /* __ASSEMBLER__ */ + +#endif /* _ASM_POWERPC_BOOK3S_64_KUP_H */ diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h new file mode 100644 index 000000000000..af12e2ba8eb8 --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h @@ -0,0 +1,884 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_POWERPC_BOOK3S_64_MMU_HASH_H_ +#define _ASM_POWERPC_BOOK3S_64_MMU_HASH_H_ +/* + * PowerPC64 memory management structures + * + * Dave Engebretsen & Mike Corrigan <{engebret|mikejc}@us.ibm.com> + * PPC64 rework. + */ + +#include <asm/page.h> +#include <asm/bug.h> +#include <asm/asm-const.h> + +/* + * This is necessary to get the definition of PGTABLE_RANGE which we + * need for various slices related matters. Note that this isn't the + * complete pgtable.h but only a portion of it. + */ +#include <asm/book3s/64/pgtable.h> +#include <asm/book3s/64/slice.h> +#include <asm/task_size_64.h> +#include <asm/cpu_has_feature.h> + +/* + * SLB + */ + +#define SLB_NUM_BOLTED 2 +#define SLB_CACHE_ENTRIES 8 +#define SLB_MIN_SIZE 32 + +/* Bits in the SLB ESID word */ +#define SLB_ESID_V ASM_CONST(0x0000000008000000) /* valid */ + +/* Bits in the SLB VSID word */ +#define SLB_VSID_SHIFT 12 +#define SLB_VSID_SHIFT_256M SLB_VSID_SHIFT +#define SLB_VSID_SHIFT_1T 24 +#define SLB_VSID_SSIZE_SHIFT 62 +#define SLB_VSID_B ASM_CONST(0xc000000000000000) +#define SLB_VSID_B_256M ASM_CONST(0x0000000000000000) +#define SLB_VSID_B_1T ASM_CONST(0x4000000000000000) +#define SLB_VSID_KS ASM_CONST(0x0000000000000800) +#define SLB_VSID_KP ASM_CONST(0x0000000000000400) +#define SLB_VSID_N ASM_CONST(0x0000000000000200) /* no-execute */ +#define SLB_VSID_L ASM_CONST(0x0000000000000100) +#define SLB_VSID_C ASM_CONST(0x0000000000000080) /* class */ +#define SLB_VSID_LP ASM_CONST(0x0000000000000030) +#define SLB_VSID_LP_00 ASM_CONST(0x0000000000000000) +#define SLB_VSID_LP_01 ASM_CONST(0x0000000000000010) +#define SLB_VSID_LP_10 ASM_CONST(0x0000000000000020) +#define SLB_VSID_LP_11 ASM_CONST(0x0000000000000030) +#define SLB_VSID_LLP (SLB_VSID_L|SLB_VSID_LP) + +#define SLB_VSID_KERNEL (SLB_VSID_KP) +#define SLB_VSID_USER (SLB_VSID_KP|SLB_VSID_KS|SLB_VSID_C) + +#define SLBIE_C (0x08000000) +#define SLBIE_SSIZE_SHIFT 25 + +/* + * Hash table + */ + +#define HPTES_PER_GROUP 8 + +#define HPTE_V_SSIZE_SHIFT 62 +#define HPTE_V_AVPN_SHIFT 7 +#define HPTE_V_COMMON_BITS ASM_CONST(0x000fffffffffffff) +#define HPTE_V_AVPN ASM_CONST(0x3fffffffffffff80) +#define HPTE_V_AVPN_3_0 ASM_CONST(0x000fffffffffff80) +#define HPTE_V_AVPN_VAL(x) (((x) & HPTE_V_AVPN) >> HPTE_V_AVPN_SHIFT) +#define HPTE_V_COMPARE(x,y) (!(((x) ^ (y)) & 0xffffffffffffff80UL)) +#define HPTE_V_BOLTED ASM_CONST(0x0000000000000010) +#define HPTE_V_LOCK ASM_CONST(0x0000000000000008) +#define HPTE_V_LARGE ASM_CONST(0x0000000000000004) +#define HPTE_V_SECONDARY ASM_CONST(0x0000000000000002) +#define HPTE_V_VALID ASM_CONST(0x0000000000000001) + +/* + * ISA 3.0 has a different HPTE format. + */ +#define HPTE_R_3_0_SSIZE_SHIFT 58 +#define HPTE_R_3_0_SSIZE_MASK (3ull << HPTE_R_3_0_SSIZE_SHIFT) +#define HPTE_R_PP0 ASM_CONST(0x8000000000000000) +#define HPTE_R_TS ASM_CONST(0x4000000000000000) +#define HPTE_R_KEY_HI ASM_CONST(0x3000000000000000) +#define HPTE_R_KEY_BIT4 ASM_CONST(0x2000000000000000) +#define HPTE_R_KEY_BIT3 ASM_CONST(0x1000000000000000) +#define HPTE_R_RPN_SHIFT 12 +#define HPTE_R_RPN ASM_CONST(0x0ffffffffffff000) +#define HPTE_R_RPN_3_0 ASM_CONST(0x01fffffffffff000) +#define HPTE_R_PP ASM_CONST(0x0000000000000003) +#define HPTE_R_PPP ASM_CONST(0x8000000000000003) +#define HPTE_R_N ASM_CONST(0x0000000000000004) +#define HPTE_R_G ASM_CONST(0x0000000000000008) +#define HPTE_R_M ASM_CONST(0x0000000000000010) +#define HPTE_R_I ASM_CONST(0x0000000000000020) +#define HPTE_R_W ASM_CONST(0x0000000000000040) +#define HPTE_R_WIMG ASM_CONST(0x0000000000000078) +#define HPTE_R_C ASM_CONST(0x0000000000000080) +#define HPTE_R_R ASM_CONST(0x0000000000000100) +#define HPTE_R_KEY_LO ASM_CONST(0x0000000000000e00) +#define HPTE_R_KEY_BIT2 ASM_CONST(0x0000000000000800) +#define HPTE_R_KEY_BIT1 ASM_CONST(0x0000000000000400) +#define HPTE_R_KEY_BIT0 ASM_CONST(0x0000000000000200) +#define HPTE_R_KEY (HPTE_R_KEY_LO | HPTE_R_KEY_HI) + +#define HPTE_V_1TB_SEG ASM_CONST(0x4000000000000000) +#define HPTE_V_VRMA_MASK ASM_CONST(0x4001ffffff000000) + +/* Values for PP (assumes Ks=0, Kp=1) */ +#define PP_RWXX 0 /* Supervisor read/write, User none */ +#define PP_RWRX 1 /* Supervisor read/write, User read */ +#define PP_RWRW 2 /* Supervisor read/write, User read/write */ +#define PP_RXRX 3 /* Supervisor read, User read */ +#define PP_RXXX (HPTE_R_PP0 | 2) /* Supervisor read, user none */ + +/* Fields for tlbiel instruction in architecture 2.06 */ +#define TLBIEL_INVAL_SEL_MASK 0xc00 /* invalidation selector */ +#define TLBIEL_INVAL_PAGE 0x000 /* invalidate a single page */ +#define TLBIEL_INVAL_SET_LPID 0x800 /* invalidate a set for current LPID */ +#define TLBIEL_INVAL_SET 0xc00 /* invalidate a set for all LPIDs */ +#define TLBIEL_INVAL_SET_MASK 0xfff000 /* set number to inval. */ +#define TLBIEL_INVAL_SET_SHIFT 12 + +#define POWER7_TLB_SETS 128 /* # sets in POWER7 TLB */ +#define POWER8_TLB_SETS 512 /* # sets in POWER8 TLB */ +#define POWER9_TLB_SETS_HASH 256 /* # sets in POWER9 TLB Hash mode */ +#define POWER9_TLB_SETS_RADIX 128 /* # sets in POWER9 TLB Radix mode */ + +#ifndef __ASSEMBLER__ + +struct mmu_hash_ops { + void (*hpte_invalidate)(unsigned long slot, + unsigned long vpn, + int bpsize, int apsize, + int ssize, int local); + long (*hpte_updatepp)(unsigned long slot, + unsigned long newpp, + unsigned long vpn, + int bpsize, int apsize, + int ssize, unsigned long flags); + void (*hpte_updateboltedpp)(unsigned long newpp, + unsigned long ea, + int psize, int ssize); + long (*hpte_insert)(unsigned long hpte_group, + unsigned long vpn, + unsigned long prpn, + unsigned long rflags, + unsigned long vflags, + int psize, int apsize, + int ssize); + long (*hpte_remove)(unsigned long hpte_group); + int (*hpte_removebolted)(unsigned long ea, + int psize, int ssize); + void (*flush_hash_range)(unsigned long number, int local); + void (*hugepage_invalidate)(unsigned long vsid, + unsigned long addr, + unsigned char *hpte_slot_array, + int psize, int ssize, int local); + int (*resize_hpt)(unsigned long shift); + /* + * Special for kexec. + * To be called in real mode with interrupts disabled. No locks are + * taken as such, concurrent access on pre POWER5 hardware could result + * in a deadlock. + * The linear mapping is destroyed as well. + */ + void (*hpte_clear_all)(void); +}; +extern struct mmu_hash_ops mmu_hash_ops; + +struct hash_pte { + __be64 v; + __be64 r; +}; + +extern struct hash_pte *htab_address; +extern unsigned long htab_size_bytes; +extern unsigned long htab_hash_mask; + + +static inline int shift_to_mmu_psize(unsigned int shift) +{ + int psize; + + for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) + if (mmu_psize_defs[psize].shift == shift) + return psize; + return -1; +} + +static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize) +{ + if (mmu_psize_defs[mmu_psize].shift) + return mmu_psize_defs[mmu_psize].shift; + BUG(); +} + +static inline unsigned int ap_to_shift(unsigned long ap) +{ + int psize; + + for (psize = 0; psize < MMU_PAGE_COUNT; psize++) { + if (mmu_psize_defs[psize].ap == ap) + return mmu_psize_defs[psize].shift; + } + + return -1; +} + +static inline unsigned long get_sllp_encoding(int psize) +{ + unsigned long sllp; + + sllp = ((mmu_psize_defs[psize].sllp & SLB_VSID_L) >> 6) | + ((mmu_psize_defs[psize].sllp & SLB_VSID_LP) >> 4); + return sllp; +} + +#endif /* __ASSEMBLER__ */ + +/* + * Segment sizes. + * These are the values used by hardware in the B field of + * SLB entries and the first dword of MMU hashtable entries. + * The B field is 2 bits; the values 2 and 3 are unused and reserved. + */ +#define MMU_SEGSIZE_256M 0 +#define MMU_SEGSIZE_1T 1 + +/* + * encode page number shift. + * in order to fit the 78 bit va in a 64 bit variable we shift the va by + * 12 bits. This enable us to address upto 76 bit va. + * For hpt hash from a va we can ignore the page size bits of va and for + * hpte encoding we ignore up to 23 bits of va. So ignoring lower 12 bits ensure + * we work in all cases including 4k page size. + */ +#define VPN_SHIFT 12 + +/* + * HPTE Large Page (LP) details + */ +#define LP_SHIFT 12 +#define LP_BITS 8 +#define LP_MASK(i) ((0xFF >> (i)) << LP_SHIFT) + +#ifndef __ASSEMBLER__ + +static inline int slb_vsid_shift(int ssize) +{ + if (ssize == MMU_SEGSIZE_256M) + return SLB_VSID_SHIFT; + return SLB_VSID_SHIFT_1T; +} + +static inline int segment_shift(int ssize) +{ + if (ssize == MMU_SEGSIZE_256M) + return SID_SHIFT; + return SID_SHIFT_1T; +} + +/* + * This array is indexed by the LP field of the HPTE second dword. + * Since this field may contain some RPN bits, some entries are + * replicated so that we get the same value irrespective of RPN. + * The top 4 bits are the page size index (MMU_PAGE_*) for the + * actual page size, the bottom 4 bits are the base page size. + */ +extern u8 hpte_page_sizes[1 << LP_BITS]; + +static inline unsigned long __hpte_page_size(unsigned long h, unsigned long l, + bool is_base_size) +{ + unsigned int i, lp; + + if (!(h & HPTE_V_LARGE)) + return 1ul << 12; + + /* Look at the 8 bit LP value */ + lp = (l >> LP_SHIFT) & ((1 << LP_BITS) - 1); + i = hpte_page_sizes[lp]; + if (!i) + return 0; + if (!is_base_size) + i >>= 4; + return 1ul << mmu_psize_defs[i & 0xf].shift; +} + +static inline unsigned long hpte_page_size(unsigned long h, unsigned long l) +{ + return __hpte_page_size(h, l, 0); +} + +static inline unsigned long hpte_base_page_size(unsigned long h, unsigned long l) +{ + return __hpte_page_size(h, l, 1); +} + +/* + * The current system page and segment sizes + */ +extern int mmu_kernel_ssize; +extern int mmu_highuser_ssize; +extern u16 mmu_slb_size; +extern unsigned long tce_alloc_start, tce_alloc_end; + +/* + * If the processor supports 64k normal pages but not 64k cache + * inhibited pages, we have to be prepared to switch processes + * to use 4k pages when they create cache-inhibited mappings. + * If this is the case, mmu_ci_restrictions will be set to 1. + */ +extern int mmu_ci_restrictions; + +/* + * This computes the AVPN and B fields of the first dword of a HPTE, + * for use when we want to match an existing PTE. The bottom 7 bits + * of the returned value are zero. + */ +static inline unsigned long hpte_encode_avpn(unsigned long vpn, int psize, + int ssize) +{ + unsigned long v; + /* + * The AVA field omits the low-order 23 bits of the 78 bits VA. + * These bits are not needed in the PTE, because the + * low-order b of these bits are part of the byte offset + * into the virtual page and, if b < 23, the high-order + * 23-b of these bits are always used in selecting the + * PTEGs to be searched + */ + v = (vpn >> (23 - VPN_SHIFT)) & ~(mmu_psize_defs[psize].avpnm); + v <<= HPTE_V_AVPN_SHIFT; + v |= ((unsigned long) ssize) << HPTE_V_SSIZE_SHIFT; + return v; +} + +/* + * ISA v3.0 defines a new HPTE format, which differs from the old + * format in having smaller AVPN and ARPN fields, and the B field + * in the second dword instead of the first. + */ +static inline unsigned long hpte_old_to_new_v(unsigned long v) +{ + /* trim AVPN, drop B */ + return v & HPTE_V_COMMON_BITS; +} + +static inline unsigned long hpte_old_to_new_r(unsigned long v, unsigned long r) +{ + /* move B field from 1st to 2nd dword, trim ARPN */ + return (r & ~HPTE_R_3_0_SSIZE_MASK) | + (((v) >> HPTE_V_SSIZE_SHIFT) << HPTE_R_3_0_SSIZE_SHIFT); +} + +static inline unsigned long hpte_new_to_old_v(unsigned long v, unsigned long r) +{ + /* insert B field */ + return (v & HPTE_V_COMMON_BITS) | + ((r & HPTE_R_3_0_SSIZE_MASK) << + (HPTE_V_SSIZE_SHIFT - HPTE_R_3_0_SSIZE_SHIFT)); +} + +static inline unsigned long hpte_new_to_old_r(unsigned long r) +{ + /* clear out B field */ + return r & ~HPTE_R_3_0_SSIZE_MASK; +} + +static inline unsigned long hpte_get_old_v(struct hash_pte *hptep) +{ + unsigned long hpte_v; + + hpte_v = be64_to_cpu(hptep->v); + if (cpu_has_feature(CPU_FTR_ARCH_300)) + hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); + return hpte_v; +} + +/* + * This function sets the AVPN and L fields of the HPTE appropriately + * using the base page size and actual page size. + */ +static inline unsigned long hpte_encode_v(unsigned long vpn, int base_psize, + int actual_psize, int ssize) +{ + unsigned long v; + v = hpte_encode_avpn(vpn, base_psize, ssize); + if (actual_psize != MMU_PAGE_4K) + v |= HPTE_V_LARGE; + return v; +} + +/* + * This function sets the ARPN, and LP fields of the HPTE appropriately + * for the page size. We assume the pa is already "clean" that is properly + * aligned for the requested page size + */ +static inline unsigned long hpte_encode_r(unsigned long pa, int base_psize, + int actual_psize) +{ + /* A 4K page needs no special encoding */ + if (actual_psize == MMU_PAGE_4K) + return pa & HPTE_R_RPN; + else { + unsigned int penc = mmu_psize_defs[base_psize].penc[actual_psize]; + unsigned int shift = mmu_psize_defs[actual_psize].shift; + return (pa & ~((1ul << shift) - 1)) | (penc << LP_SHIFT); + } +} + +/* + * Build a VPN_SHIFT bit shifted va given VSID, EA and segment size. + */ +static inline unsigned long hpt_vpn(unsigned long ea, + unsigned long vsid, int ssize) +{ + unsigned long mask; + int s_shift = segment_shift(ssize); + + mask = (1ul << (s_shift - VPN_SHIFT)) - 1; + return (vsid << (s_shift - VPN_SHIFT)) | ((ea >> VPN_SHIFT) & mask); +} + +/* + * This hashes a virtual address + */ +static inline unsigned long hpt_hash(unsigned long vpn, + unsigned int shift, int ssize) +{ + unsigned long mask; + unsigned long hash, vsid; + + /* VPN_SHIFT can be atmost 12 */ + if (ssize == MMU_SEGSIZE_256M) { + mask = (1ul << (SID_SHIFT - VPN_SHIFT)) - 1; + hash = (vpn >> (SID_SHIFT - VPN_SHIFT)) ^ + ((vpn & mask) >> (shift - VPN_SHIFT)); + } else { + mask = (1ul << (SID_SHIFT_1T - VPN_SHIFT)) - 1; + vsid = vpn >> (SID_SHIFT_1T - VPN_SHIFT); + hash = vsid ^ (vsid << 25) ^ + ((vpn & mask) >> (shift - VPN_SHIFT)) ; + } + return hash & 0x7fffffffffUL; +} + +#define HPTE_LOCAL_UPDATE 0x1 +#define HPTE_NOHPTE_UPDATE 0x2 +#define HPTE_USE_KERNEL_KEY 0x4 + +long hpte_insert_repeating(unsigned long hash, unsigned long vpn, unsigned long pa, + unsigned long rlags, unsigned long vflags, int psize, int ssize); +extern int __hash_page_4K(unsigned long ea, unsigned long access, + unsigned long vsid, pte_t *ptep, unsigned long trap, + unsigned long flags, int ssize, int subpage_prot); +extern int __hash_page_64K(unsigned long ea, unsigned long access, + unsigned long vsid, pte_t *ptep, unsigned long trap, + unsigned long flags, int ssize); +struct mm_struct; +unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap); +extern int hash_page_mm(struct mm_struct *mm, unsigned long ea, + unsigned long access, unsigned long trap, + unsigned long flags); +extern int hash_page(unsigned long ea, unsigned long access, unsigned long trap, + unsigned long dsisr); +void low_hash_fault(struct pt_regs *regs, unsigned long address, int rc); +int __hash_page(unsigned long trap, unsigned long ea, unsigned long dsisr, unsigned long msr); +int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid, + pte_t *ptep, unsigned long trap, unsigned long flags, + int ssize, unsigned int shift, unsigned int mmu_psize); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +extern int __hash_page_thp(unsigned long ea, unsigned long access, + unsigned long vsid, pmd_t *pmdp, unsigned long trap, + unsigned long flags, int ssize, unsigned int psize); +#else +static inline int __hash_page_thp(unsigned long ea, unsigned long access, + unsigned long vsid, pmd_t *pmdp, + unsigned long trap, unsigned long flags, + int ssize, unsigned int psize) +{ + BUG(); + return -1; +} +#endif +extern void hash_failure_debug(unsigned long ea, unsigned long access, + unsigned long vsid, unsigned long trap, + int ssize, int psize, int lpsize, + unsigned long pte); +extern int htab_bolt_mapping(unsigned long vstart, unsigned long vend, + unsigned long pstart, unsigned long prot, + int psize, int ssize); +int htab_remove_mapping(unsigned long vstart, unsigned long vend, + int psize, int ssize); +extern void pseries_add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages); +extern void demote_segment_4k(struct mm_struct *mm, unsigned long addr); + +extern void hash__setup_new_exec(void); + +#ifdef CONFIG_PPC_PSERIES +void hpte_init_pseries(void); +#else +static inline void hpte_init_pseries(void) { } +#endif + +extern void hpte_init_native(void); + +struct slb_entry { + u64 esid; + u64 vsid; +}; + +extern void slb_initialize(void); +void slb_flush_and_restore_bolted(void); +void slb_flush_all_realmode(void); +void __slb_restore_bolted_realmode(void); +void slb_restore_bolted_realmode(void); +void slb_save_contents(struct slb_entry *slb_ptr); +void slb_dump_contents(struct slb_entry *slb_ptr); + +extern void slb_vmalloc_update(void); + +#ifdef CONFIG_PPC_64S_HASH_MMU +void slb_set_size(u16 size); +#else +static inline void slb_set_size(u16 size) { } +#endif + +#endif /* __ASSEMBLER__ */ + +/* + * VSID allocation (256MB segment) + * + * We first generate a 37-bit "proto-VSID". Proto-VSIDs are generated + * from mmu context id and effective segment id of the address. + * + * For user processes max context id is limited to MAX_USER_CONTEXT. + * more details in get_user_context + * + * For kernel space get_kernel_context + * + * The proto-VSIDs are then scrambled into real VSIDs with the + * multiplicative hash: + * + * VSID = (proto-VSID * VSID_MULTIPLIER) % VSID_MODULUS + * + * VSID_MULTIPLIER is prime, so in particular it is + * co-prime to VSID_MODULUS, making this a 1:1 scrambling function. + * Because the modulus is 2^n-1 we can compute it efficiently without + * a divide or extra multiply (see below). The scramble function gives + * robust scattering in the hash table (at least based on some initial + * results). + * + * We use VSID 0 to indicate an invalid VSID. The means we can't use context id + * 0, because a context id of 0 and an EA of 0 gives a proto-VSID of 0, which + * will produce a VSID of 0. + * + * We also need to avoid the last segment of the last context, because that + * would give a protovsid of 0x1fffffffff. That will result in a VSID 0 + * because of the modulo operation in vsid scramble. + */ + +/* + * Max Va bits we support as of now is 68 bits. We want 19 bit + * context ID. + * Restrictions: + * GPU has restrictions of not able to access beyond 128TB + * (47 bit effective address). We also cannot do more than 20bit PID. + * For p4 and p5 which can only do 65 bit VA, we restrict our CONTEXT_BITS + * to 16 bits (ie, we can only have 2^16 pids at the same time). + */ +#define VA_BITS 68 +#define CONTEXT_BITS 19 +#define ESID_BITS (VA_BITS - (SID_SHIFT + CONTEXT_BITS)) +#define ESID_BITS_1T (VA_BITS - (SID_SHIFT_1T + CONTEXT_BITS)) + +#define ESID_BITS_MASK ((1 << ESID_BITS) - 1) +#define ESID_BITS_1T_MASK ((1 << ESID_BITS_1T) - 1) + +/* + * Now certain config support MAX_PHYSMEM more than 512TB. Hence we will need + * to use more than one context for linear mapping the kernel. + * For vmalloc and memmap, we use just one context with 512TB. With 64 byte + * struct page size, we need ony 32 TB in memmap for 2PB (51 bits (MAX_PHYSMEM_BITS)). + */ +#if (H_MAX_PHYSMEM_BITS > MAX_EA_BITS_PER_CONTEXT) +#define MAX_KERNEL_CTX_CNT (1UL << (H_MAX_PHYSMEM_BITS - MAX_EA_BITS_PER_CONTEXT)) +#else +#define MAX_KERNEL_CTX_CNT 1 +#endif + +#define MAX_VMALLOC_CTX_CNT 1 +#define MAX_IO_CTX_CNT 1 +#define MAX_VMEMMAP_CTX_CNT 1 + +/* + * 256MB segment + * The proto-VSID space has 2^(CONTEX_BITS + ESID_BITS) - 1 segments + * available for user + kernel mapping. VSID 0 is reserved as invalid, contexts + * 1-4 are used for kernel mapping. Each segment contains 2^28 bytes. Each + * context maps 2^49 bytes (512TB). + * + * We also need to avoid the last segment of the last context, because that + * would give a protovsid of 0x1fffffffff. That will result in a VSID 0 + * because of the modulo operation in vsid scramble. + * + */ +#define MAX_USER_CONTEXT ((ASM_CONST(1) << CONTEXT_BITS) - 2) + +// The + 2 accounts for INVALID_REGION and 1 more to avoid overlap with kernel +#define MIN_USER_CONTEXT (MAX_KERNEL_CTX_CNT + MAX_VMALLOC_CTX_CNT + \ + MAX_IO_CTX_CNT + MAX_VMEMMAP_CTX_CNT + 2) + +/* + * For platforms that support on 65bit VA we limit the context bits + */ +#define MAX_USER_CONTEXT_65BIT_VA ((ASM_CONST(1) << (65 - (SID_SHIFT + ESID_BITS))) - 2) + +/* + * This should be computed such that protovosid * vsid_mulitplier + * doesn't overflow 64 bits. The vsid_mutliplier should also be + * co-prime to vsid_modulus. We also need to make sure that number + * of bits in multiplied result (dividend) is less than twice the number of + * protovsid bits for our modulus optmization to work. + * + * The below table shows the current values used. + * |-------+------------+----------------------+------------+-------------------| + * | | Prime Bits | proto VSID_BITS_65VA | Total Bits | 2* prot VSID_BITS | + * |-------+------------+----------------------+------------+-------------------| + * | 1T | 24 | 25 | 49 | 50 | + * |-------+------------+----------------------+------------+-------------------| + * | 256MB | 24 | 37 | 61 | 74 | + * |-------+------------+----------------------+------------+-------------------| + * + * |-------+------------+----------------------+------------+--------------------| + * | | Prime Bits | proto VSID_BITS_68VA | Total Bits | 2* proto VSID_BITS | + * |-------+------------+----------------------+------------+--------------------| + * | 1T | 24 | 28 | 52 | 56 | + * |-------+------------+----------------------+------------+--------------------| + * | 256MB | 24 | 40 | 64 | 80 | + * |-------+------------+----------------------+------------+--------------------| + * + */ +#define VSID_MULTIPLIER_256M ASM_CONST(12538073) /* 24-bit prime */ +#define VSID_BITS_256M (VA_BITS - SID_SHIFT) +#define VSID_BITS_65_256M (65 - SID_SHIFT) +/* + * Modular multiplicative inverse of VSID_MULTIPLIER under modulo VSID_MODULUS + */ +#define VSID_MULINV_256M ASM_CONST(665548017062) + +#define VSID_MULTIPLIER_1T ASM_CONST(12538073) /* 24-bit prime */ +#define VSID_BITS_1T (VA_BITS - SID_SHIFT_1T) +#define VSID_BITS_65_1T (65 - SID_SHIFT_1T) +#define VSID_MULINV_1T ASM_CONST(209034062) + +/* 1TB VSID reserved for VRMA */ +#define VRMA_VSID 0x1ffffffUL +#define USER_VSID_RANGE (1UL << (ESID_BITS + SID_SHIFT)) + +/* 4 bits per slice and we have one slice per 1TB */ +#define SLICE_ARRAY_SIZE (H_PGTABLE_RANGE >> 41) +#define LOW_SLICE_ARRAY_SZ (BITS_PER_LONG / BITS_PER_BYTE) +#define TASK_SLICE_ARRAY_SZ(x) ((x)->hash_context->slb_addr_limit >> 41) +#ifndef __ASSEMBLER__ + +#ifdef CONFIG_PPC_SUBPAGE_PROT +/* + * For the sub-page protection option, we extend the PGD with one of + * these. Basically we have a 3-level tree, with the top level being + * the protptrs array. To optimize speed and memory consumption when + * only addresses < 4GB are being protected, pointers to the first + * four pages of sub-page protection words are stored in the low_prot + * array. + * Each page of sub-page protection words protects 1GB (4 bytes + * protects 64k). For the 3-level tree, each page of pointers then + * protects 8TB. + */ +struct subpage_prot_table { + unsigned long maxaddr; /* only addresses < this are protected */ + unsigned int **protptrs[(TASK_SIZE_USER64 >> 43)]; + unsigned int *low_prot[4]; +}; + +#define SBP_L1_BITS (PAGE_SHIFT - 2) +#define SBP_L2_BITS (PAGE_SHIFT - 3) +#define SBP_L1_COUNT (1 << SBP_L1_BITS) +#define SBP_L2_COUNT (1 << SBP_L2_BITS) +#define SBP_L2_SHIFT (PAGE_SHIFT + SBP_L1_BITS) +#define SBP_L3_SHIFT (SBP_L2_SHIFT + SBP_L2_BITS) + +extern void subpage_prot_free(struct mm_struct *mm); +#else +static inline void subpage_prot_free(struct mm_struct *mm) {} +#endif /* CONFIG_PPC_SUBPAGE_PROT */ + +/* + * One bit per slice. We have lower slices which cover 256MB segments + * upto 4G range. That gets us 16 low slices. For the rest we track slices + * in 1TB size. + */ +struct slice_mask { + u64 low_slices; + DECLARE_BITMAP(high_slices, SLICE_NUM_HIGH); +}; + +struct hash_mm_context { + u16 user_psize; /* page size index */ + + /* SLB page size encodings*/ + unsigned char low_slices_psize[LOW_SLICE_ARRAY_SZ]; + unsigned char high_slices_psize[SLICE_ARRAY_SIZE]; + unsigned long slb_addr_limit; +#ifdef CONFIG_PPC_64K_PAGES + struct slice_mask mask_64k; +#endif + struct slice_mask mask_4k; +#ifdef CONFIG_HUGETLB_PAGE + struct slice_mask mask_16m; + struct slice_mask mask_16g; +#endif + +#ifdef CONFIG_PPC_SUBPAGE_PROT + struct subpage_prot_table *spt; +#endif /* CONFIG_PPC_SUBPAGE_PROT */ +}; + +#if 0 +/* + * The code below is equivalent to this function for arguments + * < 2^VSID_BITS, which is all this should ever be called + * with. However gcc is not clever enough to compute the + * modulus (2^n-1) without a second multiply. + */ +#define vsid_scramble(protovsid, size) \ + ((((protovsid) * VSID_MULTIPLIER_##size) % VSID_MODULUS_##size)) + +/* simplified form avoiding mod operation */ +#define vsid_scramble(protovsid, size) \ + ({ \ + unsigned long x; \ + x = (protovsid) * VSID_MULTIPLIER_##size; \ + x = (x >> VSID_BITS_##size) + (x & VSID_MODULUS_##size); \ + (x + ((x+1) >> VSID_BITS_##size)) & VSID_MODULUS_##size; \ + }) + +#else /* 1 */ +static inline unsigned long vsid_scramble(unsigned long protovsid, + unsigned long vsid_multiplier, int vsid_bits) +{ + unsigned long vsid; + unsigned long vsid_modulus = ((1UL << vsid_bits) - 1); + /* + * We have same multipler for both 256 and 1T segements now + */ + vsid = protovsid * vsid_multiplier; + vsid = (vsid >> vsid_bits) + (vsid & vsid_modulus); + return (vsid + ((vsid + 1) >> vsid_bits)) & vsid_modulus; +} + +#endif /* 1 */ + +/* Returns the segment size indicator for a user address */ +static inline int user_segment_size(unsigned long addr) +{ + /* Use 1T segments if possible for addresses >= 1T */ + if (addr >= (1UL << SID_SHIFT_1T)) + return mmu_highuser_ssize; + return MMU_SEGSIZE_256M; +} + +static inline unsigned long get_vsid(unsigned long context, unsigned long ea, + int ssize) +{ + unsigned long va_bits = VA_BITS; + unsigned long vsid_bits; + unsigned long protovsid; + + /* + * Bad address. We return VSID 0 for that + */ + if ((ea & EA_MASK) >= H_PGTABLE_RANGE) + return 0; + + if (!mmu_has_feature(MMU_FTR_68_BIT_VA)) + va_bits = 65; + + if (ssize == MMU_SEGSIZE_256M) { + vsid_bits = va_bits - SID_SHIFT; + protovsid = (context << ESID_BITS) | + ((ea >> SID_SHIFT) & ESID_BITS_MASK); + return vsid_scramble(protovsid, VSID_MULTIPLIER_256M, vsid_bits); + } + /* 1T segment */ + vsid_bits = va_bits - SID_SHIFT_1T; + protovsid = (context << ESID_BITS_1T) | + ((ea >> SID_SHIFT_1T) & ESID_BITS_1T_MASK); + return vsid_scramble(protovsid, VSID_MULTIPLIER_1T, vsid_bits); +} + +/* + * For kernel space, we use context ids as + * below. Range is 512TB per context. + * + * 0x00001 - [ 0xc000000000000000 - 0xc001ffffffffffff] + * 0x00002 - [ 0xc002000000000000 - 0xc003ffffffffffff] + * 0x00003 - [ 0xc004000000000000 - 0xc005ffffffffffff] + * 0x00004 - [ 0xc006000000000000 - 0xc007ffffffffffff] + * + * vmap, IO, vmemap + * + * 0x00005 - [ 0xc008000000000000 - 0xc009ffffffffffff] + * 0x00006 - [ 0xc00a000000000000 - 0xc00bffffffffffff] + * 0x00007 - [ 0xc00c000000000000 - 0xc00dffffffffffff] + * + */ +static inline unsigned long get_kernel_context(unsigned long ea) +{ + unsigned long region_id = get_region_id(ea); + unsigned long ctx; + /* + * Depending on Kernel config, kernel region can have one context + * or more. + */ + if (region_id == LINEAR_MAP_REGION_ID) { + /* + * We already verified ea to be not beyond the addr limit. + */ + ctx = 1 + ((ea & EA_MASK) >> MAX_EA_BITS_PER_CONTEXT); + } else + ctx = region_id + MAX_KERNEL_CTX_CNT - 1; + return ctx; +} + +/* + * This is only valid for addresses >= PAGE_OFFSET + */ +static inline unsigned long get_kernel_vsid(unsigned long ea, int ssize) +{ + unsigned long context; + + if (!is_kernel_addr(ea)) + return 0; + + context = get_kernel_context(ea); + return get_vsid(context, ea, ssize); +} + +unsigned htab_shift_for_mem_size(unsigned long mem_size); + +enum slb_index { + LINEAR_INDEX = 0, /* Kernel linear map (0xc000000000000000) */ + KSTACK_INDEX = 1, /* Kernel stack map */ +}; + +#define slb_esid_mask(ssize) \ + (((ssize) == MMU_SEGSIZE_256M) ? ESID_MASK : ESID_MASK_1T) + +static inline unsigned long mk_esid_data(unsigned long ea, int ssize, + enum slb_index index) +{ + return (ea & slb_esid_mask(ssize)) | SLB_ESID_V | index; +} + +static inline unsigned long __mk_vsid_data(unsigned long vsid, int ssize, + unsigned long flags) +{ + return (vsid << slb_vsid_shift(ssize)) | flags | + ((unsigned long)ssize << SLB_VSID_SSIZE_SHIFT); +} + +static inline unsigned long mk_vsid_data(unsigned long ea, int ssize, + unsigned long flags) +{ + return __mk_vsid_data(get_kernel_vsid(ea, ssize), ssize, flags); +} + +#endif /* __ASSEMBLER__ */ +#endif /* _ASM_POWERPC_BOOK3S_64_MMU_HASH_H_ */ diff --git a/arch/powerpc/include/asm/book3s/64/mmu.h b/arch/powerpc/include/asm/book3s/64/mmu.h new file mode 100644 index 000000000000..48631365b48c --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/mmu.h @@ -0,0 +1,292 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_64_MMU_H_ +#define _ASM_POWERPC_BOOK3S_64_MMU_H_ + +#include <asm/page.h> + +#ifndef __ASSEMBLER__ +/* + * Page size definition + * + * shift : is the "PAGE_SHIFT" value for that page size + * sllp : is a bit mask with the value of SLB L || LP to be or'ed + * directly to a slbmte "vsid" value + * penc : is the HPTE encoding mask for the "LP" field: + * + */ +struct mmu_psize_def { + unsigned int shift; /* number of bits */ + int penc[MMU_PAGE_COUNT]; /* HPTE encoding */ + unsigned int tlbiel; /* tlbiel supported for that page size */ + unsigned long avpnm; /* bits to mask out in AVPN in the HPTE */ + unsigned long h_rpt_pgsize; /* H_RPT_INVALIDATE page size encoding */ + union { + unsigned long sllp; /* SLB L||LP (exact mask to use in slbmte) */ + unsigned long ap; /* Ap encoding used by PowerISA 3.0 */ + }; +}; +extern struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT]; +#endif /* __ASSEMBLER__ */ + +/* 64-bit classic hash table MMU */ +#include <asm/book3s/64/mmu-hash.h> + +#ifndef __ASSEMBLER__ +/* + * ISA 3.0 partition and process table entry format + */ +struct prtb_entry { + __be64 prtb0; + __be64 prtb1; +}; +extern struct prtb_entry *process_tb; + +struct patb_entry { + __be64 patb0; + __be64 patb1; +}; +extern struct patb_entry *partition_tb; + +/* Bits in patb0 field */ +#define PATB_HR (1UL << 63) +#define RPDB_MASK 0x0fffffffffffff00UL +#define RPDB_SHIFT (1UL << 8) +#define RTS1_SHIFT 61 /* top 2 bits of radix tree size */ +#define RTS1_MASK (3UL << RTS1_SHIFT) +#define RTS2_SHIFT 5 /* bottom 3 bits of radix tree size */ +#define RTS2_MASK (7UL << RTS2_SHIFT) +#define RPDS_MASK 0x1f /* root page dir. size field */ + +/* Bits in patb1 field */ +#define PATB_GR (1UL << 63) /* guest uses radix; must match HR */ +#define PRTS_MASK 0x1f /* process table size field */ +#define PRTB_MASK 0x0ffffffffffff000UL + +/* Number of supported LPID bits */ +extern unsigned int mmu_lpid_bits; + +/* Number of supported PID bits */ +extern unsigned int mmu_pid_bits; + +/* Base PID to allocate from */ +extern unsigned int mmu_base_pid; + +extern unsigned long __ro_after_init memory_block_size; + +#define PRTB_SIZE_SHIFT (mmu_pid_bits + 4) +#define PRTB_ENTRIES (1ul << mmu_pid_bits) + +#define PATB_SIZE_SHIFT (mmu_lpid_bits + 4) +#define PATB_ENTRIES (1ul << mmu_lpid_bits) + +typedef unsigned long mm_context_id_t; +struct spinlock; + +/* Maximum possible number of NPUs in a system. */ +#define NV_MAX_NPUS 8 + +typedef struct { + union { + /* + * We use id as the PIDR content for radix. On hash we can use + * more than one id. The extended ids are used when we start + * having address above 512TB. We allocate one extended id + * for each 512TB. The new id is then used with the 49 bit + * EA to build a new VA. We always use ESID_BITS_1T_MASK bits + * from EA and new context ids to build the new VAs. + */ + mm_context_id_t id; +#ifdef CONFIG_PPC_64S_HASH_MMU + mm_context_id_t extended_id[TASK_SIZE_USER64/TASK_CONTEXT_SIZE]; +#endif + }; + + /* Number of bits in the mm_cpumask */ + atomic_t active_cpus; + + /* Number of users of the external (Nest) MMU */ + atomic_t copros; + + /* Number of user space windows opened in process mm_context */ + atomic_t vas_windows; + +#ifdef CONFIG_PPC_64S_HASH_MMU + struct hash_mm_context *hash_context; +#endif + + void __user *vdso; + /* + * pagetable fragment support + */ + void *pte_frag; + void *pmd_frag; +#ifdef CONFIG_SPAPR_TCE_IOMMU + struct list_head iommu_group_mem_list; +#endif + +#ifdef CONFIG_PPC_MEM_KEYS + /* + * Each bit represents one protection key. + * bit set -> key allocated + * bit unset -> key available for allocation + */ + u32 pkey_allocation_map; + s16 execute_only_pkey; /* key holding execute-only protection */ +#endif +} mm_context_t; + +#ifdef CONFIG_PPC_64S_HASH_MMU +static inline u16 mm_ctx_user_psize(mm_context_t *ctx) +{ + return ctx->hash_context->user_psize; +} + +static inline void mm_ctx_set_user_psize(mm_context_t *ctx, u16 user_psize) +{ + ctx->hash_context->user_psize = user_psize; +} + +static inline unsigned char *mm_ctx_low_slices(mm_context_t *ctx) +{ + return ctx->hash_context->low_slices_psize; +} + +static inline unsigned char *mm_ctx_high_slices(mm_context_t *ctx) +{ + return ctx->hash_context->high_slices_psize; +} + +static inline unsigned long mm_ctx_slb_addr_limit(mm_context_t *ctx) +{ + return ctx->hash_context->slb_addr_limit; +} + +static inline void mm_ctx_set_slb_addr_limit(mm_context_t *ctx, unsigned long limit) +{ + ctx->hash_context->slb_addr_limit = limit; +} + +static inline struct slice_mask *slice_mask_for_size(mm_context_t *ctx, int psize) +{ +#ifdef CONFIG_PPC_64K_PAGES + if (psize == MMU_PAGE_64K) + return &ctx->hash_context->mask_64k; +#endif +#ifdef CONFIG_HUGETLB_PAGE + if (psize == MMU_PAGE_16M) + return &ctx->hash_context->mask_16m; + if (psize == MMU_PAGE_16G) + return &ctx->hash_context->mask_16g; +#endif + BUG_ON(psize != MMU_PAGE_4K); + + return &ctx->hash_context->mask_4k; +} + +#ifdef CONFIG_PPC_SUBPAGE_PROT +static inline struct subpage_prot_table *mm_ctx_subpage_prot(mm_context_t *ctx) +{ + return ctx->hash_context->spt; +} +#endif + +/* + * The current system page and segment sizes + */ +extern int mmu_virtual_psize; +extern int mmu_vmalloc_psize; +extern int mmu_io_psize; +#else /* CONFIG_PPC_64S_HASH_MMU */ +#ifdef CONFIG_PPC_64K_PAGES +#define mmu_virtual_psize MMU_PAGE_64K +#else +#define mmu_virtual_psize MMU_PAGE_4K +#endif +#endif +extern int mmu_linear_psize; +extern int mmu_vmemmap_psize; + +/* MMU initialization */ +void mmu_early_init_devtree(void); +void hash__early_init_devtree(void); +void radix__early_init_devtree(void); +#ifdef CONFIG_PPC_PKEY +void pkey_early_init_devtree(void); +#else +static inline void pkey_early_init_devtree(void) {} +#endif + +extern void hash__early_init_mmu(void); +extern void radix__early_init_mmu(void); +static inline void __init early_init_mmu(void) +{ + if (radix_enabled()) + return radix__early_init_mmu(); + return hash__early_init_mmu(); +} +extern void hash__early_init_mmu_secondary(void); +extern void radix__early_init_mmu_secondary(void); +static inline void early_init_mmu_secondary(void) +{ + if (radix_enabled()) + return radix__early_init_mmu_secondary(); + return hash__early_init_mmu_secondary(); +} + +extern void hash__setup_initial_memory_limit(phys_addr_t first_memblock_base, + phys_addr_t first_memblock_size); +static inline void setup_initial_memory_limit(phys_addr_t first_memblock_base, + phys_addr_t first_memblock_size) +{ + /* + * Hash has more strict restrictions. At this point we don't + * know which translations we will pick. Hence go with hash + * restrictions. + */ + if (!early_radix_enabled()) + hash__setup_initial_memory_limit(first_memblock_base, + first_memblock_size); +} + +#ifdef CONFIG_PPC_PSERIES +void __init radix_init_pseries(void); +#else +static inline void radix_init_pseries(void) { } +#endif + +#ifdef CONFIG_HOTPLUG_CPU +#define arch_clear_mm_cpumask_cpu(cpu, mm) \ + do { \ + if (cpumask_test_cpu(cpu, mm_cpumask(mm))) { \ + dec_mm_active_cpus(mm); \ + cpumask_clear_cpu(cpu, mm_cpumask(mm)); \ + } \ + } while (0) + +void cleanup_cpu_mmu_context(void); +#endif + +#ifdef CONFIG_PPC_64S_HASH_MMU +static inline int get_user_context(mm_context_t *ctx, unsigned long ea) +{ + int index = ea >> MAX_EA_BITS_PER_CONTEXT; + + if (likely(index < ARRAY_SIZE(ctx->extended_id))) + return ctx->extended_id[index]; + + /* should never happen */ + WARN_ON(1); + return 0; +} + +static inline unsigned long get_user_vsid(mm_context_t *ctx, + unsigned long ea, int ssize) +{ + unsigned long context = get_user_context(ctx, ea); + + return get_vsid(context, ea, ssize); +} +#endif + +#endif /* __ASSEMBLER__ */ +#endif /* _ASM_POWERPC_BOOK3S_64_MMU_H_ */ diff --git a/arch/powerpc/include/asm/book3s/64/pgalloc.h b/arch/powerpc/include/asm/book3s/64/pgalloc.h new file mode 100644 index 000000000000..dd2cff53a111 --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/pgalloc.h @@ -0,0 +1,183 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_POWERPC_BOOK3S_64_PGALLOC_H +#define _ASM_POWERPC_BOOK3S_64_PGALLOC_H +/* + */ + +#include <linux/slab.h> +#include <linux/cpumask.h> +#include <linux/kmemleak.h> +#include <linux/percpu.h> + +struct vmemmap_backing { + struct vmemmap_backing *list; + unsigned long phys; + unsigned long virt_addr; +}; +extern struct vmemmap_backing *vmemmap_list; + +extern pmd_t *pmd_fragment_alloc(struct mm_struct *, unsigned long); +extern void pmd_fragment_free(unsigned long *); +extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift); +extern void __tlb_remove_table(void *_table); +void pte_frag_destroy(void *pte_frag); + +static inline pgd_t *radix__pgd_alloc(struct mm_struct *mm) +{ +#ifdef CONFIG_PPC_64K_PAGES + return (pgd_t *)__get_free_page(pgtable_gfp_flags(mm, PGALLOC_GFP)); +#else + struct page *page; + page = alloc_pages(pgtable_gfp_flags(mm, PGALLOC_GFP | __GFP_RETRY_MAYFAIL), + 4); + if (!page) + return NULL; + return (pgd_t *) page_address(page); +#endif +} + +static inline void radix__pgd_free(struct mm_struct *mm, pgd_t *pgd) +{ +#ifdef CONFIG_PPC_64K_PAGES + free_page((unsigned long)pgd); +#else + free_pages((unsigned long)pgd, 4); +#endif +} + +static inline pgd_t *pgd_alloc(struct mm_struct *mm) +{ + pgd_t *pgd; + + if (radix_enabled()) + return radix__pgd_alloc(mm); + + pgd = kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), + pgtable_gfp_flags(mm, GFP_KERNEL)); + if (unlikely(!pgd)) + return pgd; + + /* + * Don't scan the PGD for pointers, it contains references to PUDs but + * those references are not full pointers and so can't be recognised by + * kmemleak. + */ + kmemleak_no_scan(pgd); + + /* + * With hugetlb, we don't clear the second half of the page table. + * If we share the same slab cache with the pmd or pud level table, + * we need to make sure we zero out the full table on alloc. + * With 4K we don't store slot in the second half. Hence we don't + * need to do this for 4k. + */ +#if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_PPC_64K_PAGES) && \ + (H_PGD_INDEX_SIZE == H_PUD_CACHE_INDEX) + memset(pgd, 0, PGD_TABLE_SIZE); +#endif + return pgd; +} + +static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) +{ + if (radix_enabled()) + return radix__pgd_free(mm, pgd); + kmem_cache_free(PGT_CACHE(PGD_INDEX_SIZE), pgd); +} + +static inline void p4d_populate(struct mm_struct *mm, p4d_t *pgd, pud_t *pud) +{ + *pgd = __p4d(__pgtable_ptr_val(pud) | PGD_VAL_BITS); +} + +static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) +{ + pud_t *pud; + + pud = kmem_cache_alloc(PGT_CACHE(PUD_CACHE_INDEX), + pgtable_gfp_flags(mm, GFP_KERNEL)); + /* + * Tell kmemleak to ignore the PUD, that means don't scan it for + * pointers and don't consider it a leak. PUDs are typically only + * referred to by their PGD, but kmemleak is not able to recognise those + * as pointers, leading to false leak reports. + */ + kmemleak_ignore(pud); + + return pud; +} + +static inline void __pud_free(pud_t *pud) +{ + struct page *page = virt_to_page(pud); + + /* + * Early pud pages allocated via memblock allocator + * can't be directly freed to slab. KFENCE pages have + * both reserved and slab flags set so need to be freed + * kmem_cache_free. + */ + if (PageReserved(page) && !PageSlab(page)) + free_reserved_page(page); + else + kmem_cache_free(PGT_CACHE(PUD_CACHE_INDEX), pud); +} + +static inline void pud_free(struct mm_struct *mm, pud_t *pud) +{ + return __pud_free(pud); +} + +static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) +{ + *pud = __pud(__pgtable_ptr_val(pmd) | PUD_VAL_BITS); +} + +static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, + unsigned long address) +{ + pgtable_free_tlb(tlb, pud, PUD_INDEX); +} + +static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) +{ + return pmd_fragment_alloc(mm, addr); +} + +static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) +{ + pmd_fragment_free((unsigned long *)pmd); +} + +static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, + unsigned long address) +{ + return pgtable_free_tlb(tlb, pmd, PMD_INDEX); +} + +static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, + pte_t *pte) +{ + *pmd = __pmd(__pgtable_ptr_val(pte) | PMD_VAL_BITS); +} + +static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, + pgtable_t pte_page) +{ + *pmd = __pmd(__pgtable_ptr_val(pte_page) | PMD_VAL_BITS); +} + +static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table, + unsigned long address) +{ + pgtable_free_tlb(tlb, table, PTE_INDEX); +} + +extern atomic_long_t direct_pages_count[MMU_PAGE_COUNT]; +static inline void update_page_count(int psize, long count) +{ + if (IS_ENABLED(CONFIG_PROC_FS)) + atomic_long_add(count, &direct_pages_count[psize]); +} + +#endif /* _ASM_POWERPC_BOOK3S_64_PGALLOC_H */ diff --git a/arch/powerpc/include/asm/book3s/64/pgtable-64k.h b/arch/powerpc/include/asm/book3s/64/pgtable-64k.h new file mode 100644 index 000000000000..004a03e97e58 --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/pgtable-64k.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_64_PGTABLE_64K_H +#define _ASM_POWERPC_BOOK3S_64_PGTABLE_64K_H + +#ifndef __ASSEMBLER__ +#ifdef CONFIG_HUGETLB_PAGE + +#endif /* CONFIG_HUGETLB_PAGE */ + +static inline int remap_4k_pfn(struct vm_area_struct *vma, unsigned long addr, + unsigned long pfn, pgprot_t prot) +{ + if (radix_enabled()) + BUG(); + return hash__remap_4k_pfn(vma, addr, pfn, prot); +} +#endif /* __ASSEMBLER__ */ +#endif /*_ASM_POWERPC_BOOK3S_64_PGTABLE_64K_H */ diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h new file mode 100644 index 000000000000..aac8ce30cd3b --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -0,0 +1,1385 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_64_PGTABLE_H_ +#define _ASM_POWERPC_BOOK3S_64_PGTABLE_H_ + +#include <asm-generic/pgtable-nop4d.h> + +#ifndef __ASSEMBLER__ +#include <linux/mmdebug.h> +#include <linux/bug.h> +#include <linux/sizes.h> +#endif + +/* + * Common bits between hash and Radix page table + */ + +#define _PAGE_EXEC 0x00001 /* execute permission */ +#define _PAGE_WRITE 0x00002 /* write access allowed */ +#define _PAGE_READ 0x00004 /* read access allowed */ +#define _PAGE_PRIVILEGED 0x00008 /* kernel access only */ +#define _PAGE_SAO 0x00010 /* Strong access order */ +#define _PAGE_NON_IDEMPOTENT 0x00020 /* non idempotent memory */ +#define _PAGE_TOLERANT 0x00030 /* tolerant memory, cache inhibited */ +#define _PAGE_DIRTY 0x00080 /* C: page changed */ +#define _PAGE_ACCESSED 0x00100 /* R: page referenced */ +/* + * Software bits + */ +#define _RPAGE_SW0 0x2000000000000000UL +#define _RPAGE_SW1 0x00800 +#define _RPAGE_SW2 0x00400 +#define _RPAGE_SW3 0x00200 +#define _RPAGE_RSV1 0x00040UL + +#define _RPAGE_PKEY_BIT4 0x1000000000000000UL +#define _RPAGE_PKEY_BIT3 0x0800000000000000UL +#define _RPAGE_PKEY_BIT2 0x0400000000000000UL +#define _RPAGE_PKEY_BIT1 0x0200000000000000UL +#define _RPAGE_PKEY_BIT0 0x0100000000000000UL + +#define _PAGE_PTE 0x4000000000000000UL /* distinguishes PTEs from pointers */ +#define _PAGE_PRESENT 0x8000000000000000UL /* pte contains a translation */ +/* + * We need to mark a pmd pte invalid while splitting. We can do that by clearing + * the _PAGE_PRESENT bit. But then that will be taken as a swap pte. In order to + * differentiate between two use a SW field when invalidating. + * + * We do that temporary invalidate for regular pte entry in ptep_set_access_flags + * + * This is used only when _PAGE_PRESENT is cleared. + */ +#define _PAGE_INVALID _RPAGE_SW0 + +/* + * Top and bottom bits of RPN which can be used by hash + * translation mode, because we expect them to be zero + * otherwise. + */ +#define _RPAGE_RPN0 0x01000 +#define _RPAGE_RPN1 0x02000 +#define _RPAGE_RPN43 0x0080000000000000UL +#define _RPAGE_RPN42 0x0040000000000000UL +#define _RPAGE_RPN41 0x0020000000000000UL + +/* Max physical address bit as per radix table */ +#define _RPAGE_PA_MAX 56 + +/* + * Max physical address bit we will use for now. + * + * This is mostly a hardware limitation and for now Power9 has + * a 51 bit limit. + * + * This is different from the number of physical bit required to address + * the last byte of memory. That is defined by MAX_PHYSMEM_BITS. + * MAX_PHYSMEM_BITS is a linux limitation imposed by the maximum + * number of sections we can support (SECTIONS_SHIFT). + * + * This is different from Radix page table limitation above and + * should always be less than that. The limit is done such that + * we can overload the bits between _RPAGE_PA_MAX and _PAGE_PA_MAX + * for hash linux page table specific bits. + * + * In order to be compatible with future hardware generations we keep + * some offsets and limit this for now to 53 + */ +#define _PAGE_PA_MAX 53 + +#define _PAGE_SOFT_DIRTY _RPAGE_SW3 /* software: software dirty tracking */ +#define _PAGE_SPECIAL _RPAGE_SW2 /* software: special page */ + +/* + * Drivers request for cache inhibited pte mapping using _PAGE_NO_CACHE + * Instead of fixing all of them, add an alternate define which + * maps CI pte mapping. + */ +#define _PAGE_NO_CACHE _PAGE_TOLERANT +/* + * We support _RPAGE_PA_MAX bit real address in pte. On the linux side + * we are limited by _PAGE_PA_MAX. Clear everything above _PAGE_PA_MAX + * and every thing below PAGE_SHIFT; + */ +#define PTE_RPN_MASK (((1UL << _PAGE_PA_MAX) - 1) & (PAGE_MASK)) +#define PTE_RPN_SHIFT PAGE_SHIFT +/* + * set of bits not changed in pmd_modify. Even though we have hash specific bits + * in here, on radix we expect them to be zero. + */ +#define _HPAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HPTEFLAGS | _PAGE_DIRTY | \ + _PAGE_ACCESSED | H_PAGE_THP_HUGE | _PAGE_PTE | \ + _PAGE_SOFT_DIRTY) +/* + * user access blocked by key + */ +#define _PAGE_KERNEL_RW (_PAGE_PRIVILEGED | _PAGE_RW | _PAGE_DIRTY) +#define _PAGE_KERNEL_RO (_PAGE_PRIVILEGED | _PAGE_READ) +#define _PAGE_KERNEL_ROX (_PAGE_PRIVILEGED | _PAGE_READ | _PAGE_EXEC) +#define _PAGE_KERNEL_RWX (_PAGE_PRIVILEGED | _PAGE_DIRTY | _PAGE_RW | _PAGE_EXEC) +/* + * _PAGE_CHG_MASK masks of bits that are to be preserved across + * pgprot changes + */ +#define _PAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HPTEFLAGS | _PAGE_DIRTY | \ + _PAGE_ACCESSED | _PAGE_SPECIAL | _PAGE_PTE | \ + _PAGE_SOFT_DIRTY) + +/* + * We define 2 sets of base prot bits, one for basic pages (ie, + * cacheable kernel and user pages) and one for non cacheable + * pages. We always set _PAGE_COHERENT when SMP is enabled or + * the processor might need it for DMA coherency. + */ +#define _PAGE_BASE_NC (_PAGE_PRESENT | _PAGE_ACCESSED) +#define _PAGE_BASE (_PAGE_BASE_NC) + +#include <asm/pgtable-masks.h> + +/* Permission masks used for kernel mappings */ +#define PAGE_KERNEL __pgprot(_PAGE_BASE | _PAGE_KERNEL_RW) +#define PAGE_KERNEL_NC __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | _PAGE_TOLERANT) +#define PAGE_KERNEL_NCG __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | _PAGE_NON_IDEMPOTENT) +#define PAGE_KERNEL_X __pgprot(_PAGE_BASE | _PAGE_KERNEL_RWX) +#define PAGE_KERNEL_RO __pgprot(_PAGE_BASE | _PAGE_KERNEL_RO) +#define PAGE_KERNEL_ROX __pgprot(_PAGE_BASE | _PAGE_KERNEL_ROX) + +#ifndef __ASSEMBLER__ +/* + * page table defines + */ +extern unsigned long __pte_index_size; +extern unsigned long __pmd_index_size; +extern unsigned long __pud_index_size; +extern unsigned long __pgd_index_size; +extern unsigned long __pud_cache_index; +#define PTE_INDEX_SIZE __pte_index_size +#define PMD_INDEX_SIZE __pmd_index_size +#define PUD_INDEX_SIZE __pud_index_size +#define PGD_INDEX_SIZE __pgd_index_size +/* pmd table use page table fragments */ +#define PMD_CACHE_INDEX 0 +#define PUD_CACHE_INDEX __pud_cache_index +/* + * Because of use of pte fragments and THP, size of page table + * are not always derived out of index size above. + */ +extern unsigned long __pte_table_size; +extern unsigned long __pmd_table_size; +extern unsigned long __pud_table_size; +extern unsigned long __pgd_table_size; +#define PTE_TABLE_SIZE __pte_table_size +#define PMD_TABLE_SIZE __pmd_table_size +#define PUD_TABLE_SIZE __pud_table_size +#define PGD_TABLE_SIZE __pgd_table_size + +extern unsigned long __pmd_val_bits; +extern unsigned long __pud_val_bits; +extern unsigned long __pgd_val_bits; +#define PMD_VAL_BITS __pmd_val_bits +#define PUD_VAL_BITS __pud_val_bits +#define PGD_VAL_BITS __pgd_val_bits + +extern unsigned long __pte_frag_nr; +#define PTE_FRAG_NR __pte_frag_nr +extern unsigned long __pte_frag_size_shift; +#define PTE_FRAG_SIZE_SHIFT __pte_frag_size_shift +#define PTE_FRAG_SIZE (1UL << PTE_FRAG_SIZE_SHIFT) + +extern unsigned long __pmd_frag_nr; +#define PMD_FRAG_NR __pmd_frag_nr +extern unsigned long __pmd_frag_size_shift; +#define PMD_FRAG_SIZE_SHIFT __pmd_frag_size_shift +#define PMD_FRAG_SIZE (1UL << PMD_FRAG_SIZE_SHIFT) + +#define PTRS_PER_PTE (1 << PTE_INDEX_SIZE) +#define PTRS_PER_PMD (1 << PMD_INDEX_SIZE) +#define PTRS_PER_PUD (1 << PUD_INDEX_SIZE) +#define PTRS_PER_PGD (1 << PGD_INDEX_SIZE) + +#define MAX_PTRS_PER_PTE ((H_PTRS_PER_PTE > R_PTRS_PER_PTE) ? H_PTRS_PER_PTE : R_PTRS_PER_PTE) +#define MAX_PTRS_PER_PMD ((H_PTRS_PER_PMD > R_PTRS_PER_PMD) ? H_PTRS_PER_PMD : R_PTRS_PER_PMD) +#define MAX_PTRS_PER_PUD ((H_PTRS_PER_PUD > R_PTRS_PER_PUD) ? H_PTRS_PER_PUD : R_PTRS_PER_PUD) +#define MAX_PTRS_PER_PGD (1 << (H_PGD_INDEX_SIZE > RADIX_PGD_INDEX_SIZE ? \ + H_PGD_INDEX_SIZE : RADIX_PGD_INDEX_SIZE)) + +/* PMD_SHIFT determines what a second-level page table entry can map */ +#define PMD_SHIFT (PAGE_SHIFT + PTE_INDEX_SIZE) +#define PMD_SIZE (1UL << PMD_SHIFT) +#define PMD_MASK (~(PMD_SIZE-1)) + +/* PUD_SHIFT determines what a third-level page table entry can map */ +#define PUD_SHIFT (PMD_SHIFT + PMD_INDEX_SIZE) +#define PUD_SIZE (1UL << PUD_SHIFT) +#define PUD_MASK (~(PUD_SIZE-1)) + +/* PGDIR_SHIFT determines what a fourth-level page table entry can map */ +#define PGDIR_SHIFT (PUD_SHIFT + PUD_INDEX_SIZE) +#define PGDIR_SIZE (1UL << PGDIR_SHIFT) +#define PGDIR_MASK (~(PGDIR_SIZE-1)) + +/* Bits to mask out from a PMD to get to the PTE page */ +#define PMD_MASKED_BITS 0xc0000000000000ffUL +/* Bits to mask out from a PUD to get to the PMD page */ +#define PUD_MASKED_BITS 0xc0000000000000ffUL +/* Bits to mask out from a PGD to get to the PUD page */ +#define P4D_MASKED_BITS 0xc0000000000000ffUL + +/* + * Used as an indicator for rcu callback functions + */ +enum pgtable_index { + PTE_INDEX = 0, + PMD_INDEX, + PUD_INDEX, + PGD_INDEX, + /* + * Below are used with 4k page size and hugetlb + */ + HTLB_16M_INDEX, + HTLB_16G_INDEX, +}; + +extern unsigned long __vmalloc_start; +extern unsigned long __vmalloc_end; +#define VMALLOC_START __vmalloc_start +#define VMALLOC_END __vmalloc_end + +static inline unsigned int ioremap_max_order(void) +{ + if (radix_enabled()) + return PUD_SHIFT; + return 7 + PAGE_SHIFT; /* default from linux/vmalloc.h */ +} +#define IOREMAP_MAX_ORDER ioremap_max_order() + +extern unsigned long __kernel_virt_start; +extern unsigned long __kernel_io_start; +extern unsigned long __kernel_io_end; +#define KERN_VIRT_START __kernel_virt_start +#define KERN_IO_START __kernel_io_start +#define KERN_IO_END __kernel_io_end + +extern struct page *vmemmap; +extern unsigned long pci_io_base; + +#define pmd_leaf pmd_leaf +static inline bool pmd_leaf(pmd_t pmd) +{ + return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE)); +} + +#define pud_leaf pud_leaf +static inline bool pud_leaf(pud_t pud) +{ + return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PTE)); +} + +#define pmd_leaf_size pmd_leaf_size +static inline unsigned long pmd_leaf_size(pmd_t pmd) +{ + if (IS_ENABLED(CONFIG_PPC_4K_PAGES) && !radix_enabled()) + return SZ_16M; + else + return PMD_SIZE; +} + +#define pud_leaf_size pud_leaf_size +static inline unsigned long pud_leaf_size(pud_t pud) +{ + if (IS_ENABLED(CONFIG_PPC_4K_PAGES) && !radix_enabled()) + return SZ_16G; + else + return PUD_SIZE; +} +#endif /* __ASSEMBLER__ */ + +#include <asm/book3s/64/hash.h> +#include <asm/book3s/64/radix.h> + +#if H_MAX_PHYSMEM_BITS > R_MAX_PHYSMEM_BITS +#define MAX_PHYSMEM_BITS H_MAX_PHYSMEM_BITS +#else +#define MAX_PHYSMEM_BITS R_MAX_PHYSMEM_BITS +#endif + +/* hash 4k can't share hugetlb and also doesn't support THP */ +#ifdef CONFIG_PPC_64K_PAGES +#include <asm/book3s/64/pgtable-64k.h> +#endif + +#include <asm/barrier.h> +/* + * IO space itself carved into the PIO region (ISA and PHB IO space) and + * the ioremap space + * + * ISA_IO_BASE = KERN_IO_START, 64K reserved area + * PHB_IO_BASE = ISA_IO_BASE + 64K to ISA_IO_BASE + 2G, PHB IO spaces + * IOREMAP_BASE = ISA_IO_BASE + 2G to VMALLOC_START + PGTABLE_RANGE + */ +#define FULL_IO_SIZE 0x80000000ul +#define ISA_IO_BASE (KERN_IO_START) +#define ISA_IO_END (KERN_IO_START + 0x10000ul) +#define PHB_IO_BASE (ISA_IO_END) +#define PHB_IO_END (KERN_IO_START + FULL_IO_SIZE) +#define IOREMAP_BASE (PHB_IO_END) +#define IOREMAP_START (ioremap_bot) +#define IOREMAP_END (KERN_IO_END - FIXADDR_SIZE) +#define FIXADDR_SIZE SZ_32M +#define FIXADDR_TOP (IOREMAP_END + FIXADDR_SIZE) + +#ifndef __ASSEMBLER__ + +static inline unsigned long pte_update(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned long clr, + unsigned long set, int huge) +{ + if (radix_enabled()) + return radix__pte_update(mm, addr, ptep, clr, set, huge); + return hash__pte_update(mm, addr, ptep, clr, set, huge); +} +/* + * For hash even if we have _PAGE_ACCESSED = 0, we do a pte_update. + * We currently remove entries from the hashtable regardless of whether + * the entry was young or dirty. + * + * We should be more intelligent about this but for the moment we override + * these functions and force a tlb flush unconditionally + * For radix: H_PAGE_HASHPTE should be zero. Hence we can use the same + * function for both hash and radix. + */ +static inline int __ptep_test_and_clear_young(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + unsigned long old; + + if ((pte_raw(*ptep) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0) + return 0; + old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0); + return (old & _PAGE_ACCESSED) != 0; +} + +#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG +#define ptep_test_and_clear_young(__vma, __addr, __ptep) \ +({ \ + __ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep); \ +}) + +/* + * On Book3S CPUs, clearing the accessed bit without a TLB flush + * doesn't cause data corruption. [ It could cause incorrect + * page aging and the (mistaken) reclaim of hot pages, but the + * chance of that should be relatively low. ] + * + * So as a performance optimization don't flush the TLB when + * clearing the accessed bit, it will eventually be flushed by + * a context switch or a VM operation anyway. [ In the rare + * event of it not getting flushed for a long time the delay + * shouldn't really matter because there's no real memory + * pressure for swapout to react to. ] + * + * Note: this optimisation also exists in pte_needs_flush() and + * huge_pmd_needs_flush(). + */ +#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH +#define ptep_clear_flush_young ptep_test_and_clear_young + +#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH +#define pmdp_clear_flush_young pmdp_test_and_clear_young + +static inline int pte_write(pte_t pte) +{ + return !!(pte_raw(pte) & cpu_to_be64(_PAGE_WRITE)); +} + +static inline int pte_read(pte_t pte) +{ + return !!(pte_raw(pte) & cpu_to_be64(_PAGE_READ)); +} + +#define __HAVE_ARCH_PTEP_SET_WRPROTECT +static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, + pte_t *ptep) +{ + if (pte_write(*ptep)) + pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 0); +} + +#define __HAVE_ARCH_HUGE_PTEP_SET_WRPROTECT +static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + if (pte_write(*ptep)) + pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 1); +} + +#define __HAVE_ARCH_PTEP_GET_AND_CLEAR +static inline pte_t ptep_get_and_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0, 0); + return __pte(old); +} + +#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL +static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, + unsigned long addr, + pte_t *ptep, int full) +{ + if (full && radix_enabled()) { + /* + * We know that this is a full mm pte clear and + * hence can be sure there is no parallel set_pte. + */ + return radix__ptep_get_and_clear_full(mm, addr, ptep, full); + } + return ptep_get_and_clear(mm, addr, ptep); +} + + +static inline void pte_clear(struct mm_struct *mm, unsigned long addr, + pte_t * ptep) +{ + pte_update(mm, addr, ptep, ~0UL, 0, 0); +} + +static inline int pte_dirty(pte_t pte) +{ + return !!(pte_raw(pte) & cpu_to_be64(_PAGE_DIRTY)); +} + +static inline int pte_young(pte_t pte) +{ + return !!(pte_raw(pte) & cpu_to_be64(_PAGE_ACCESSED)); +} + +static inline int pte_special(pte_t pte) +{ + return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SPECIAL)); +} + +static inline bool pte_exec(pte_t pte) +{ + return !!(pte_raw(pte) & cpu_to_be64(_PAGE_EXEC)); +} + + +#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY +static inline bool pte_soft_dirty(pte_t pte) +{ + return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SOFT_DIRTY)); +} + +static inline pte_t pte_mksoft_dirty(pte_t pte) +{ + return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_SOFT_DIRTY)); +} + +static inline pte_t pte_clear_soft_dirty(pte_t pte) +{ + return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_SOFT_DIRTY)); +} +#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ + +#ifdef CONFIG_NUMA_BALANCING +static inline int pte_protnone(pte_t pte) +{ + return (pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE | _PAGE_RWX)) == + cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE); +} +#endif /* CONFIG_NUMA_BALANCING */ + +static inline bool pte_hw_valid(pte_t pte) +{ + return (pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE)) == + cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE); +} + +static inline int pte_present(pte_t pte) +{ + /* + * A pte is considerent present if _PAGE_PRESENT is set. + * We also need to consider the pte present which is marked + * invalid during ptep_set_access_flags. Hence we look for _PAGE_INVALID + * if we find _PAGE_PRESENT cleared. + */ + + if (pte_hw_valid(pte)) + return true; + return (pte_raw(pte) & cpu_to_be64(_PAGE_INVALID | _PAGE_PTE)) == + cpu_to_be64(_PAGE_INVALID | _PAGE_PTE); +} + +#ifdef CONFIG_PPC_MEM_KEYS +extern bool arch_pte_access_permitted(u64 pte, bool write, bool execute); +#else +static inline bool arch_pte_access_permitted(u64 pte, bool write, bool execute) +{ + return true; +} +#endif /* CONFIG_PPC_MEM_KEYS */ + +static inline bool pte_user(pte_t pte) +{ + return !(pte_raw(pte) & cpu_to_be64(_PAGE_PRIVILEGED)); +} + +#define pte_access_permitted pte_access_permitted +static inline bool pte_access_permitted(pte_t pte, bool write) +{ + /* + * _PAGE_READ is needed for any access and will be cleared for + * PROT_NONE. Execute-only mapping via PROT_EXEC also returns false. + */ + if (!pte_present(pte) || !pte_user(pte) || !pte_read(pte)) + return false; + + if (write && !pte_write(pte)) + return false; + + return arch_pte_access_permitted(pte_val(pte), write, 0); +} + +/* + * Conversion functions: convert a page and protection to a page entry, + * and a page entry and page directory to the page they refer to. + * + * Even if PTEs can be unsigned long long, a PFN is always an unsigned + * long for now. + */ +static inline pte_t pfn_pte(unsigned long pfn, pgprot_t pgprot) +{ + VM_BUG_ON(pfn >> (64 - PAGE_SHIFT)); + VM_BUG_ON((pfn << PAGE_SHIFT) & ~PTE_RPN_MASK); + + return __pte(((pte_basic_t)pfn << PAGE_SHIFT) | pgprot_val(pgprot) | _PAGE_PTE); +} + +/* Generic modifiers for PTE bits */ +static inline pte_t pte_wrprotect(pte_t pte) +{ + return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_WRITE)); +} + +static inline pte_t pte_exprotect(pte_t pte) +{ + return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_EXEC)); +} + +static inline pte_t pte_mkclean(pte_t pte) +{ + return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_DIRTY)); +} + +static inline pte_t pte_mkold(pte_t pte) +{ + return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_ACCESSED)); +} + +static inline pte_t pte_mkexec(pte_t pte) +{ + return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_EXEC)); +} + +static inline pte_t pte_mkwrite_novma(pte_t pte) +{ + /* + * write implies read, hence set both + */ + return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_RW)); +} + +static inline pte_t pte_mkdirty(pte_t pte) +{ + return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_DIRTY | _PAGE_SOFT_DIRTY)); +} + +static inline pte_t pte_mkyoung(pte_t pte) +{ + return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_ACCESSED)); +} + +static inline pte_t pte_mkspecial(pte_t pte) +{ + return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_SPECIAL)); +} + +static inline pte_t pte_mkhuge(pte_t pte) +{ + return pte; +} + +static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) +{ + /* FIXME!! check whether this need to be a conditional */ + return __pte_raw((pte_raw(pte) & cpu_to_be64(_PAGE_CHG_MASK)) | + cpu_to_be64(pgprot_val(newprot))); +} + +/* Encode and de-code a swap entry */ +#define MAX_SWAPFILES_CHECK() do { \ + BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS); \ + /* \ + * Don't have overlapping bits with _PAGE_HPTEFLAGS \ + * We filter HPTEFLAGS on set_pte. \ + */ \ + BUILD_BUG_ON(_PAGE_HPTEFLAGS & SWP_TYPE_MASK); \ + BUILD_BUG_ON(_PAGE_HPTEFLAGS & _PAGE_SWP_SOFT_DIRTY); \ + BUILD_BUG_ON(_PAGE_HPTEFLAGS & _PAGE_SWP_EXCLUSIVE); \ + } while (0) + +#define SWP_TYPE_BITS 5 +#define SWP_TYPE_MASK ((1UL << SWP_TYPE_BITS) - 1) +#define __swp_type(x) ((x).val & SWP_TYPE_MASK) +#define __swp_offset(x) (((x).val & PTE_RPN_MASK) >> PAGE_SHIFT) +#define __swp_entry(type, offset) ((swp_entry_t) { \ + (type) | (((offset) << PAGE_SHIFT) & PTE_RPN_MASK)}) +/* + * swp_entry_t must be independent of pte bits. We build a swp_entry_t from + * swap type and offset we get from swap and convert that to pte to find a + * matching pte in linux page table. + * Clear bits not found in swap entries here. + */ +#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val((pte)) & ~_PAGE_PTE }) +#define __swp_entry_to_pte(x) __pte((x).val | _PAGE_PTE) +#define __pmd_to_swp_entry(pmd) (__pte_to_swp_entry(pmd_pte(pmd))) +#define __swp_entry_to_pmd(x) (pte_pmd(__swp_entry_to_pte(x))) + +#ifdef CONFIG_MEM_SOFT_DIRTY +#define _PAGE_SWP_SOFT_DIRTY _PAGE_SOFT_DIRTY +#else +#define _PAGE_SWP_SOFT_DIRTY 0UL +#endif /* CONFIG_MEM_SOFT_DIRTY */ + +#define _PAGE_SWP_EXCLUSIVE _PAGE_NON_IDEMPOTENT + +#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY +static inline pte_t pte_swp_mksoft_dirty(pte_t pte) +{ + return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_SWP_SOFT_DIRTY)); +} + +static inline bool pte_swp_soft_dirty(pte_t pte) +{ + return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SWP_SOFT_DIRTY)); +} + +static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) +{ + return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_SWP_SOFT_DIRTY)); +} +#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ + +static inline pte_t pte_swp_mkexclusive(pte_t pte) +{ + return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_SWP_EXCLUSIVE)); +} + +static inline bool pte_swp_exclusive(pte_t pte) +{ + return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SWP_EXCLUSIVE)); +} + +static inline pte_t pte_swp_clear_exclusive(pte_t pte) +{ + return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_SWP_EXCLUSIVE)); +} + +static inline bool check_pte_access(unsigned long access, unsigned long ptev) +{ + /* + * This check for _PAGE_RWX and _PAGE_PRESENT bits + */ + if (access & ~ptev) + return false; + /* + * This check for access to privilege space + */ + if ((access & _PAGE_PRIVILEGED) != (ptev & _PAGE_PRIVILEGED)) + return false; + + return true; +} +/* + * Generic functions with hash/radix callbacks + */ + +static inline void __ptep_set_access_flags(struct vm_area_struct *vma, + pte_t *ptep, pte_t entry, + unsigned long address, + int psize) +{ + if (radix_enabled()) + return radix__ptep_set_access_flags(vma, ptep, entry, + address, psize); + return hash__ptep_set_access_flags(ptep, entry); +} + +#define __HAVE_ARCH_PTE_SAME +static inline int pte_same(pte_t pte_a, pte_t pte_b) +{ + if (radix_enabled()) + return radix__pte_same(pte_a, pte_b); + return hash__pte_same(pte_a, pte_b); +} + +static inline int pte_none(pte_t pte) +{ + if (radix_enabled()) + return radix__pte_none(pte); + return hash__pte_none(pte); +} + +static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, int percpu) +{ + + VM_WARN_ON(!(pte_raw(pte) & cpu_to_be64(_PAGE_PTE))); + /* + * Keep the _PAGE_PTE added till we are sure we handle _PAGE_PTE + * in all the callers. + */ + pte = __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_PTE)); + + if (radix_enabled()) + return radix__set_pte_at(mm, addr, ptep, pte, percpu); + return hash__set_pte_at(mm, addr, ptep, pte, percpu); +} + +#define _PAGE_CACHE_CTL (_PAGE_SAO | _PAGE_NON_IDEMPOTENT | _PAGE_TOLERANT) + +#define pgprot_noncached pgprot_noncached +static inline pgprot_t pgprot_noncached(pgprot_t prot) +{ + return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | + _PAGE_NON_IDEMPOTENT); +} + +#define pgprot_noncached_wc pgprot_noncached_wc +static inline pgprot_t pgprot_noncached_wc(pgprot_t prot) +{ + return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | + _PAGE_TOLERANT); +} + +#define pgprot_cached pgprot_cached +static inline pgprot_t pgprot_cached(pgprot_t prot) +{ + return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL)); +} + +#define pgprot_writecombine pgprot_writecombine +static inline pgprot_t pgprot_writecombine(pgprot_t prot) +{ + return pgprot_noncached_wc(prot); +} +/* + * check a pte mapping have cache inhibited property + */ +static inline bool pte_ci(pte_t pte) +{ + __be64 pte_v = pte_raw(pte); + + if (((pte_v & cpu_to_be64(_PAGE_CACHE_CTL)) == cpu_to_be64(_PAGE_TOLERANT)) || + ((pte_v & cpu_to_be64(_PAGE_CACHE_CTL)) == cpu_to_be64(_PAGE_NON_IDEMPOTENT))) + return true; + return false; +} + +static inline void pmd_clear(pmd_t *pmdp) +{ + if (IS_ENABLED(CONFIG_DEBUG_VM) && !radix_enabled()) { + /* + * Don't use this if we can possibly have a hash page table + * entry mapping this. + */ + WARN_ON((pmd_val(*pmdp) & (H_PAGE_HASHPTE | _PAGE_PTE)) == (H_PAGE_HASHPTE | _PAGE_PTE)); + } + *pmdp = __pmd(0); +} + +static inline int pmd_none(pmd_t pmd) +{ + return !pmd_raw(pmd); +} + +static inline int pmd_present(pmd_t pmd) +{ + /* + * A pmd is considerent present if _PAGE_PRESENT is set. + * We also need to consider the pmd present which is marked + * invalid during a split. Hence we look for _PAGE_INVALID + * if we find _PAGE_PRESENT cleared. + */ + if (pmd_raw(pmd) & cpu_to_be64(_PAGE_PRESENT | _PAGE_INVALID)) + return true; + + return false; +} + +static inline int pmd_is_serializing(pmd_t pmd) +{ + /* + * If the pmd is undergoing a split, the _PAGE_PRESENT bit is clear + * and _PAGE_INVALID is set (see pmd_present, pmdp_invalidate). + * + * This condition may also occur when flushing a pmd while flushing + * it (see ptep_modify_prot_start), so callers must ensure this + * case is fine as well. + */ + if ((pmd_raw(pmd) & cpu_to_be64(_PAGE_PRESENT | _PAGE_INVALID)) == + cpu_to_be64(_PAGE_INVALID)) + return true; + + return false; +} + +static inline int pmd_bad(pmd_t pmd) +{ + if (radix_enabled()) + return radix__pmd_bad(pmd); + return hash__pmd_bad(pmd); +} + +static inline void pud_clear(pud_t *pudp) +{ + if (IS_ENABLED(CONFIG_DEBUG_VM) && !radix_enabled()) { + /* + * Don't use this if we can possibly have a hash page table + * entry mapping this. + */ + WARN_ON((pud_val(*pudp) & (H_PAGE_HASHPTE | _PAGE_PTE)) == (H_PAGE_HASHPTE | _PAGE_PTE)); + } + *pudp = __pud(0); +} + +static inline int pud_none(pud_t pud) +{ + return !pud_raw(pud); +} + +static inline int pud_present(pud_t pud) +{ + return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PRESENT)); +} + +extern struct page *pud_page(pud_t pud); +extern struct page *pmd_page(pmd_t pmd); +static inline pte_t pud_pte(pud_t pud) +{ + return __pte_raw(pud_raw(pud)); +} + +static inline pud_t pte_pud(pte_t pte) +{ + return __pud_raw(pte_raw(pte)); +} + +static inline pte_t *pudp_ptep(pud_t *pud) +{ + return (pte_t *)pud; +} + +#define pud_pfn(pud) pte_pfn(pud_pte(pud)) +#define pud_dirty(pud) pte_dirty(pud_pte(pud)) +#define pud_young(pud) pte_young(pud_pte(pud)) +#define pud_mkold(pud) pte_pud(pte_mkold(pud_pte(pud))) +#define pud_wrprotect(pud) pte_pud(pte_wrprotect(pud_pte(pud))) +#define pud_mkdirty(pud) pte_pud(pte_mkdirty(pud_pte(pud))) +#define pud_mkclean(pud) pte_pud(pte_mkclean(pud_pte(pud))) +#define pud_mkyoung(pud) pte_pud(pte_mkyoung(pud_pte(pud))) +#define pud_mkwrite(pud) pte_pud(pte_mkwrite_novma(pud_pte(pud))) +#define pud_write(pud) pte_write(pud_pte(pud)) + +#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY +#define pud_soft_dirty(pmd) pte_soft_dirty(pud_pte(pud)) +#define pud_mksoft_dirty(pmd) pte_pud(pte_mksoft_dirty(pud_pte(pud))) +#define pud_clear_soft_dirty(pmd) pte_pud(pte_clear_soft_dirty(pud_pte(pud))) +#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ + +static inline int pud_bad(pud_t pud) +{ + if (radix_enabled()) + return radix__pud_bad(pud); + return hash__pud_bad(pud); +} + +#define pud_access_permitted pud_access_permitted +static inline bool pud_access_permitted(pud_t pud, bool write) +{ + return pte_access_permitted(pud_pte(pud), write); +} + +#define __p4d_raw(x) ((p4d_t) { __pgd_raw(x) }) +static inline __be64 p4d_raw(p4d_t x) +{ + return pgd_raw(x.pgd); +} + +#define p4d_write(p4d) pte_write(p4d_pte(p4d)) + +static inline void p4d_clear(p4d_t *p4dp) +{ + *p4dp = __p4d(0); +} + +static inline int p4d_none(p4d_t p4d) +{ + return !p4d_raw(p4d); +} + +static inline int p4d_present(p4d_t p4d) +{ + return !!(p4d_raw(p4d) & cpu_to_be64(_PAGE_PRESENT)); +} + +static inline pte_t p4d_pte(p4d_t p4d) +{ + return __pte_raw(p4d_raw(p4d)); +} + +static inline p4d_t pte_p4d(pte_t pte) +{ + return __p4d_raw(pte_raw(pte)); +} + +static inline int p4d_bad(p4d_t p4d) +{ + if (radix_enabled()) + return radix__p4d_bad(p4d); + return hash__p4d_bad(p4d); +} + +#define p4d_access_permitted p4d_access_permitted +static inline bool p4d_access_permitted(p4d_t p4d, bool write) +{ + return pte_access_permitted(p4d_pte(p4d), write); +} + +extern struct page *p4d_page(p4d_t p4d); + +/* Pointers in the page table tree are physical addresses */ +#define __pgtable_ptr_val(ptr) __pa(ptr) + +static inline pud_t *p4d_pgtable(p4d_t p4d) +{ + return (pud_t *)__va(p4d_val(p4d) & ~P4D_MASKED_BITS); +} + +static inline pmd_t *pud_pgtable(pud_t pud) +{ + return (pmd_t *)__va(pud_val(pud) & ~PUD_MASKED_BITS); +} + +#define pmd_ERROR(e) \ + pr_err("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e)) +#define pud_ERROR(e) \ + pr_err("%s:%d: bad pud %08lx.\n", __FILE__, __LINE__, pud_val(e)) +#define pgd_ERROR(e) \ + pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) + +static inline int map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot) +{ + if (radix_enabled()) { +#if defined(CONFIG_PPC_RADIX_MMU) && defined(DEBUG_VM) + unsigned long page_size = 1 << mmu_psize_defs[mmu_io_psize].shift; + WARN((page_size != PAGE_SIZE), "I/O page size != PAGE_SIZE"); +#endif + return radix__map_kernel_page(ea, pa, prot, PAGE_SIZE); + } + return hash__map_kernel_page(ea, pa, prot); +} + +void unmap_kernel_page(unsigned long va); + +static inline int __meminit vmemmap_create_mapping(unsigned long start, + unsigned long page_size, + unsigned long phys) +{ + if (radix_enabled()) + return radix__vmemmap_create_mapping(start, page_size, phys); + return hash__vmemmap_create_mapping(start, page_size, phys); +} + +#ifdef CONFIG_MEMORY_HOTPLUG +static inline void vmemmap_remove_mapping(unsigned long start, + unsigned long page_size) +{ + if (radix_enabled()) + return radix__vmemmap_remove_mapping(start, page_size); + return hash__vmemmap_remove_mapping(start, page_size); +} +#endif + +static inline pte_t pmd_pte(pmd_t pmd) +{ + return __pte_raw(pmd_raw(pmd)); +} + +static inline pmd_t pte_pmd(pte_t pte) +{ + return __pmd_raw(pte_raw(pte)); +} + +static inline pte_t *pmdp_ptep(pmd_t *pmd) +{ + return (pte_t *)pmd; +} +#define pmd_pfn(pmd) pte_pfn(pmd_pte(pmd)) +#define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd)) +#define pmd_young(pmd) pte_young(pmd_pte(pmd)) +#define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd))) +#define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd))) +#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd))) +#define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd))) +#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd))) +#define pmd_mkwrite_novma(pmd) pte_pmd(pte_mkwrite_novma(pmd_pte(pmd))) + +#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY +#define pmd_soft_dirty(pmd) pte_soft_dirty(pmd_pte(pmd)) +#define pmd_mksoft_dirty(pmd) pte_pmd(pte_mksoft_dirty(pmd_pte(pmd))) +#define pmd_clear_soft_dirty(pmd) pte_pmd(pte_clear_soft_dirty(pmd_pte(pmd))) + +#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#define pmd_swp_mksoft_dirty(pmd) pte_pmd(pte_swp_mksoft_dirty(pmd_pte(pmd))) +#define pmd_swp_soft_dirty(pmd) pte_swp_soft_dirty(pmd_pte(pmd)) +#define pmd_swp_clear_soft_dirty(pmd) pte_pmd(pte_swp_clear_soft_dirty(pmd_pte(pmd))) +#endif +#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ + +#ifdef CONFIG_NUMA_BALANCING +static inline int pmd_protnone(pmd_t pmd) +{ + return pte_protnone(pmd_pte(pmd)); +} +#endif /* CONFIG_NUMA_BALANCING */ + +#define pmd_write(pmd) pte_write(pmd_pte(pmd)) + +#define pmd_access_permitted pmd_access_permitted +static inline bool pmd_access_permitted(pmd_t pmd, bool write) +{ + /* + * pmdp_invalidate sets this combination (which is not caught by + * !pte_present() check in pte_access_permitted), to prevent + * lock-free lookups, as part of the serialize_against_pte_lookup() + * synchronisation. + * + * This also catches the case where the PTE's hardware PRESENT bit is + * cleared while TLB is flushed, which is suboptimal but should not + * be frequent. + */ + if (pmd_is_serializing(pmd)) + return false; + + return pte_access_permitted(pmd_pte(pmd), write); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +extern pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot); +extern pud_t pfn_pud(unsigned long pfn, pgprot_t pgprot); +extern pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot); +extern pud_t pud_modify(pud_t pud, pgprot_t newprot); +extern void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd); +extern void set_pud_at(struct mm_struct *mm, unsigned long addr, + pud_t *pudp, pud_t pud); + +static inline void update_mmu_cache_pmd(struct vm_area_struct *vma, + unsigned long addr, pmd_t *pmd) +{ +} + +static inline void update_mmu_cache_pud(struct vm_area_struct *vma, + unsigned long addr, pud_t *pud) +{ +} + +extern int hash__has_transparent_hugepage(void); +static inline int has_transparent_hugepage(void) +{ + if (radix_enabled()) + return radix__has_transparent_hugepage(); + return hash__has_transparent_hugepage(); +} +#define has_transparent_hugepage has_transparent_hugepage + +static inline int has_transparent_pud_hugepage(void) +{ + if (radix_enabled()) + return radix__has_transparent_pud_hugepage(); + return 0; +} +#define has_transparent_pud_hugepage has_transparent_pud_hugepage + +static inline unsigned long +pmd_hugepage_update(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, + unsigned long clr, unsigned long set) +{ + if (radix_enabled()) + return radix__pmd_hugepage_update(mm, addr, pmdp, clr, set); + return hash__pmd_hugepage_update(mm, addr, pmdp, clr, set); +} + +static inline unsigned long +pud_hugepage_update(struct mm_struct *mm, unsigned long addr, pud_t *pudp, + unsigned long clr, unsigned long set) +{ + if (radix_enabled()) + return radix__pud_hugepage_update(mm, addr, pudp, clr, set); + BUG(); + return pud_val(*pudp); +} + +/* + * For radix we should always find H_PAGE_HASHPTE zero. Hence + * the below will work for radix too + */ +static inline int __pmdp_test_and_clear_young(struct mm_struct *mm, + unsigned long addr, pmd_t *pmdp) +{ + unsigned long old; + + if ((pmd_raw(*pmdp) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0) + return 0; + old = pmd_hugepage_update(mm, addr, pmdp, _PAGE_ACCESSED, 0); + return ((old & _PAGE_ACCESSED) != 0); +} + +static inline int __pudp_test_and_clear_young(struct mm_struct *mm, + unsigned long addr, pud_t *pudp) +{ + unsigned long old; + + if ((pud_raw(*pudp) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0) + return 0; + old = pud_hugepage_update(mm, addr, pudp, _PAGE_ACCESSED, 0); + return ((old & _PAGE_ACCESSED) != 0); +} + +#define __HAVE_ARCH_PMDP_SET_WRPROTECT +static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp) +{ + if (pmd_write(*pmdp)) + pmd_hugepage_update(mm, addr, pmdp, _PAGE_WRITE, 0); +} + +#define __HAVE_ARCH_PUDP_SET_WRPROTECT +static inline void pudp_set_wrprotect(struct mm_struct *mm, unsigned long addr, + pud_t *pudp) +{ + if (pud_write(*pudp)) + pud_hugepage_update(mm, addr, pudp, _PAGE_WRITE, 0); +} + +/* + * Only returns true for a THP. False for pmd migration entry. + * We also need to return true when we come across a pte that + * in between a thp split. While splitting THP, we mark the pmd + * invalid (pmdp_invalidate()) before we set it with pte page + * address. A pmd_trans_huge() check against a pmd entry during that time + * should return true. + * We should not call this on a hugetlb entry. We should check for HugeTLB + * entry using vma->vm_flags + * The page table walk rule is explained in Documentation/mm/transhuge.rst + */ +static inline int pmd_trans_huge(pmd_t pmd) +{ + if (!pmd_present(pmd)) + return false; + + if (radix_enabled()) + return radix__pmd_trans_huge(pmd); + return hash__pmd_trans_huge(pmd); +} + +static inline int pud_trans_huge(pud_t pud) +{ + if (!pud_present(pud)) + return false; + + if (radix_enabled()) + return radix__pud_trans_huge(pud); + return 0; +} + + +#define __HAVE_ARCH_PMD_SAME +static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b) +{ + if (radix_enabled()) + return radix__pmd_same(pmd_a, pmd_b); + return hash__pmd_same(pmd_a, pmd_b); +} + +#define pud_same pud_same +static inline int pud_same(pud_t pud_a, pud_t pud_b) +{ + if (radix_enabled()) + return radix__pud_same(pud_a, pud_b); + return hash__pud_same(pud_a, pud_b); +} + + +static inline pmd_t __pmd_mkhuge(pmd_t pmd) +{ + if (radix_enabled()) + return radix__pmd_mkhuge(pmd); + return hash__pmd_mkhuge(pmd); +} + +static inline pud_t __pud_mkhuge(pud_t pud) +{ + if (radix_enabled()) + return radix__pud_mkhuge(pud); + BUG(); + return pud; +} + +/* + * pfn_pmd return a pmd_t that can be used as pmd pte entry. + */ +static inline pmd_t pmd_mkhuge(pmd_t pmd) +{ +#ifdef CONFIG_DEBUG_VM + if (radix_enabled()) + WARN_ON((pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE)) == 0); + else + WARN_ON((pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE | H_PAGE_THP_HUGE)) != + cpu_to_be64(_PAGE_PTE | H_PAGE_THP_HUGE)); +#endif + return pmd; +} + +static inline pud_t pud_mkhuge(pud_t pud) +{ +#ifdef CONFIG_DEBUG_VM + if (radix_enabled()) + WARN_ON((pud_raw(pud) & cpu_to_be64(_PAGE_PTE)) == 0); + else + WARN_ON(1); +#endif + return pud; +} + + +#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS +extern int pmdp_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, + pmd_t entry, int dirty); +#define __HAVE_ARCH_PUDP_SET_ACCESS_FLAGS +extern int pudp_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pud_t *pudp, + pud_t entry, int dirty); + +#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG +extern int pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp); +#define __HAVE_ARCH_PUDP_TEST_AND_CLEAR_YOUNG +extern int pudp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, pud_t *pudp); + + +#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR +static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, + unsigned long addr, pmd_t *pmdp) +{ + if (radix_enabled()) + return radix__pmdp_huge_get_and_clear(mm, addr, pmdp); + return hash__pmdp_huge_get_and_clear(mm, addr, pmdp); +} + +#define __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR +static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm, + unsigned long addr, pud_t *pudp) +{ + if (radix_enabled()) + return radix__pudp_huge_get_and_clear(mm, addr, pudp); + BUG(); + return *pudp; +} + +static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp) +{ + if (radix_enabled()) + return radix__pmdp_collapse_flush(vma, address, pmdp); + return hash__pmdp_collapse_flush(vma, address, pmdp); +} +#define pmdp_collapse_flush pmdp_collapse_flush + +#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL +pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma, + unsigned long addr, + pmd_t *pmdp, int full); + +#define __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR_FULL +pud_t pudp_huge_get_and_clear_full(struct vm_area_struct *vma, + unsigned long addr, + pud_t *pudp, int full); + +#define __HAVE_ARCH_PGTABLE_DEPOSIT +static inline void pgtable_trans_huge_deposit(struct mm_struct *mm, + pmd_t *pmdp, pgtable_t pgtable) +{ + if (radix_enabled()) + return radix__pgtable_trans_huge_deposit(mm, pmdp, pgtable); + return hash__pgtable_trans_huge_deposit(mm, pmdp, pgtable); +} + +#define __HAVE_ARCH_PGTABLE_WITHDRAW +static inline pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, + pmd_t *pmdp) +{ + if (radix_enabled()) + return radix__pgtable_trans_huge_withdraw(mm, pmdp); + return hash__pgtable_trans_huge_withdraw(mm, pmdp); +} + +#define __HAVE_ARCH_PMDP_INVALIDATE +extern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, + pmd_t *pmdp); +extern pud_t pudp_invalidate(struct vm_area_struct *vma, unsigned long address, + pud_t *pudp); + +#define pmd_move_must_withdraw pmd_move_must_withdraw +struct spinlock; +extern int pmd_move_must_withdraw(struct spinlock *new_pmd_ptl, + struct spinlock *old_pmd_ptl, + struct vm_area_struct *vma); +/* + * Hash translation mode use the deposited table to store hash pte + * slot information. + */ +#define arch_needs_pgtable_deposit arch_needs_pgtable_deposit +static inline bool arch_needs_pgtable_deposit(void) +{ + if (radix_enabled()) + return false; + return true; +} +extern void serialize_against_pte_lookup(struct mm_struct *mm); + +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION +pte_t ptep_modify_prot_start(struct vm_area_struct *, unsigned long, pte_t *); +void ptep_modify_prot_commit(struct vm_area_struct *, unsigned long, + pte_t *, pte_t, pte_t); + +/* + * Returns true for a R -> RW upgrade of pte + */ +static inline bool is_pte_rw_upgrade(unsigned long old_val, unsigned long new_val) +{ + if (!(old_val & _PAGE_READ)) + return false; + + if ((!(old_val & _PAGE_WRITE)) && (new_val & _PAGE_WRITE)) + return true; + + return false; +} + +#endif /* __ASSEMBLER__ */ +#endif /* _ASM_POWERPC_BOOK3S_64_PGTABLE_H_ */ diff --git a/arch/powerpc/include/asm/book3s/64/pkeys.h b/arch/powerpc/include/asm/book3s/64/pkeys.h new file mode 100644 index 000000000000..ff911b4251d9 --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/pkeys.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ + +#ifndef _ASM_POWERPC_BOOK3S_64_PKEYS_H +#define _ASM_POWERPC_BOOK3S_64_PKEYS_H + +#include <asm/book3s/64/hash-pkey.h> + +static inline u64 vmflag_to_pte_pkey_bits(vm_flags_t vm_flags) +{ + if (!mmu_has_feature(MMU_FTR_PKEY)) + return 0x0UL; + + if (radix_enabled()) + BUG(); + return hash__vmflag_to_pte_pkey_bits(vm_flags); +} + +static inline u16 pte_to_pkey_bits(u64 pteflags) +{ + if (radix_enabled()) + BUG(); + return hash__pte_to_pkey_bits(pteflags); +} + +#endif /*_ASM_POWERPC_KEYS_H */ diff --git a/arch/powerpc/include/asm/book3s/64/radix-4k.h b/arch/powerpc/include/asm/book3s/64/radix-4k.h new file mode 100644 index 000000000000..035ceecd6d67 --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/radix-4k.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_PGTABLE_RADIX_4K_H +#define _ASM_POWERPC_PGTABLE_RADIX_4K_H + +/* + * For 4K page size supported index is 13/9/9/9 + */ +#define RADIX_PTE_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps 2^9 x 4K = 2MB +#define RADIX_PMD_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps 2^9 x 2MB = 1GB +#define RADIX_PUD_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps 2^9 x 1GB = 512GB +#define RADIX_PGD_INDEX_SIZE 13 // size: 8B << 13 = 64KB, maps 2^13 x 512GB = 4PB + +/* + * One fragment per page + */ +#define RADIX_PTE_FRAG_SIZE_SHIFT (RADIX_PTE_INDEX_SIZE + 3) +#define RADIX_PTE_FRAG_NR (PAGE_SIZE >> RADIX_PTE_FRAG_SIZE_SHIFT) + +#define RADIX_PMD_FRAG_SIZE_SHIFT (RADIX_PMD_INDEX_SIZE + 3) +#define RADIX_PMD_FRAG_NR (PAGE_SIZE >> RADIX_PMD_FRAG_SIZE_SHIFT) + +#endif /* _ASM_POWERPC_PGTABLE_RADIX_4K_H */ diff --git a/arch/powerpc/include/asm/book3s/64/radix-64k.h b/arch/powerpc/include/asm/book3s/64/radix-64k.h new file mode 100644 index 000000000000..54e33828b0fb --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/radix-64k.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_PGTABLE_RADIX_64K_H +#define _ASM_POWERPC_PGTABLE_RADIX_64K_H + +/* + * For 64K page size supported index is 13/9/9/5 + */ +#define RADIX_PTE_INDEX_SIZE 5 // size: 8B << 5 = 256B, maps 2^5 x 64K = 2MB +#define RADIX_PMD_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps 2^9 x 2MB = 1GB +#define RADIX_PUD_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps 2^9 x 1GB = 512GB +#define RADIX_PGD_INDEX_SIZE 13 // size: 8B << 13 = 64KB, maps 2^13 x 512GB = 4PB + +/* + * We use a 256 byte PTE page fragment in radix + * 8 bytes per each PTE entry. + */ +#define RADIX_PTE_FRAG_SIZE_SHIFT (RADIX_PTE_INDEX_SIZE + 3) +#define RADIX_PTE_FRAG_NR (PAGE_SIZE >> RADIX_PTE_FRAG_SIZE_SHIFT) + +#define RADIX_PMD_FRAG_SIZE_SHIFT (RADIX_PMD_INDEX_SIZE + 3) +#define RADIX_PMD_FRAG_NR (PAGE_SIZE >> RADIX_PMD_FRAG_SIZE_SHIFT) + +#endif /* _ASM_POWERPC_PGTABLE_RADIX_64K_H */ diff --git a/arch/powerpc/include/asm/book3s/64/radix.h b/arch/powerpc/include/asm/book3s/64/radix.h new file mode 100644 index 000000000000..da954e779744 --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/radix.h @@ -0,0 +1,366 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_PGTABLE_RADIX_H +#define _ASM_POWERPC_PGTABLE_RADIX_H + +#include <asm/asm-const.h> + +#ifndef __ASSEMBLER__ +#include <asm/cmpxchg.h> +#endif + +#ifdef CONFIG_PPC_64K_PAGES +#include <asm/book3s/64/radix-64k.h> +#else +#include <asm/book3s/64/radix-4k.h> +#endif + +#ifndef __ASSEMBLER__ +#include <asm/book3s/64/tlbflush-radix.h> +#include <asm/cpu_has_feature.h> +#endif + +/* An empty PTE can still have a R or C writeback */ +#define RADIX_PTE_NONE_MASK (_PAGE_DIRTY | _PAGE_ACCESSED) + +/* Bits to set in a RPMD/RPUD/RPGD */ +#define RADIX_PMD_VAL_BITS (0x8000000000000000UL | RADIX_PTE_INDEX_SIZE) +#define RADIX_PUD_VAL_BITS (0x8000000000000000UL | RADIX_PMD_INDEX_SIZE) +#define RADIX_PGD_VAL_BITS (0x8000000000000000UL | RADIX_PUD_INDEX_SIZE) + +/* Don't have anything in the reserved bits and leaf bits */ +#define RADIX_PMD_BAD_BITS 0x60000000000000e0UL +#define RADIX_PUD_BAD_BITS 0x60000000000000e0UL +#define RADIX_P4D_BAD_BITS 0x60000000000000e0UL + +#define RADIX_PMD_SHIFT (PAGE_SHIFT + RADIX_PTE_INDEX_SIZE) +#define RADIX_PUD_SHIFT (RADIX_PMD_SHIFT + RADIX_PMD_INDEX_SIZE) +#define RADIX_PGD_SHIFT (RADIX_PUD_SHIFT + RADIX_PUD_INDEX_SIZE) + +#define R_PTRS_PER_PTE (1 << RADIX_PTE_INDEX_SIZE) +#define R_PTRS_PER_PMD (1 << RADIX_PMD_INDEX_SIZE) +#define R_PTRS_PER_PUD (1 << RADIX_PUD_INDEX_SIZE) + +/* + * Size of EA range mapped by our pagetables. + */ +#define RADIX_PGTABLE_EADDR_SIZE (RADIX_PTE_INDEX_SIZE + RADIX_PMD_INDEX_SIZE + \ + RADIX_PUD_INDEX_SIZE + RADIX_PGD_INDEX_SIZE + PAGE_SHIFT) +#define RADIX_PGTABLE_RANGE (ASM_CONST(1) << RADIX_PGTABLE_EADDR_SIZE) + +/* + * We support 52 bit address space, Use top bit for kernel + * virtual mapping. Also make sure kernel fit in the top + * quadrant. + * + * +------------------+ + * +------------------+ Kernel virtual map (0xc008000000000000) + * | | + * | | + * | | + * 0b11......+------------------+ Kernel linear map (0xc....) + * | | + * | 2 quadrant | + * | | + * 0b10......+------------------+ + * | | + * | 1 quadrant | + * | | + * 0b01......+------------------+ + * | | + * | 0 quadrant | + * | | + * 0b00......+------------------+ + * + * + * 3rd quadrant expanded: + * +------------------------------+ Highest address (0xc010000000000000) + * +------------------------------+ KASAN shadow end (0xc00fc00000000000) + * | | + * | | + * +------------------------------+ Kernel vmemmap end/shadow start (0xc00e000000000000) + * | | + * | 512TB | + * | | + * +------------------------------+ Kernel IO map end/vmemap start + * | | + * | 512TB | + * | | + * +------------------------------+ Kernel vmap end/ IO map start + * | | + * | 512TB | + * | | + * +------------------------------+ Kernel virt start (0xc008000000000000) + * | | + * | | + * | | + * +------------------------------+ Kernel linear (0xc.....) + */ + +/* For the sizes of the shadow area, see kasan.h */ + +/* + * If we store section details in page->flags we can't increase the MAX_PHYSMEM_BITS + * if we increase SECTIONS_WIDTH we will not store node details in page->flags and + * page_to_nid does a page->section->node lookup + * Hence only increase for VMEMMAP. Further depending on SPARSEMEM_EXTREME reduce + * memory requirements with large number of sections. + * 51 bits is the max physical real address on POWER9 + */ + +#if defined(CONFIG_SPARSEMEM_VMEMMAP) && defined(CONFIG_SPARSEMEM_EXTREME) +#define R_MAX_PHYSMEM_BITS 51 +#else +#define R_MAX_PHYSMEM_BITS 46 +#endif + +#define RADIX_KERN_VIRT_START ASM_CONST(0xc008000000000000) +/* + * 49 = MAX_EA_BITS_PER_CONTEXT (hash specific). To make sure we pick + * the same value as hash. + */ +#define RADIX_KERN_MAP_SIZE (1UL << 49) + +#define RADIX_VMALLOC_START RADIX_KERN_VIRT_START +#define RADIX_VMALLOC_SIZE RADIX_KERN_MAP_SIZE +#define RADIX_VMALLOC_END (RADIX_VMALLOC_START + RADIX_VMALLOC_SIZE) + +#define RADIX_KERN_IO_START RADIX_VMALLOC_END +#define RADIX_KERN_IO_SIZE RADIX_KERN_MAP_SIZE +#define RADIX_KERN_IO_END (RADIX_KERN_IO_START + RADIX_KERN_IO_SIZE) + +#define RADIX_VMEMMAP_START RADIX_KERN_IO_END +#define RADIX_VMEMMAP_SIZE RADIX_KERN_MAP_SIZE +#define RADIX_VMEMMAP_END (RADIX_VMEMMAP_START + RADIX_VMEMMAP_SIZE) + +#ifndef __ASSEMBLER__ +#define RADIX_PTE_TABLE_SIZE (sizeof(pte_t) << RADIX_PTE_INDEX_SIZE) +#define RADIX_PMD_TABLE_SIZE (sizeof(pmd_t) << RADIX_PMD_INDEX_SIZE) +#define RADIX_PUD_TABLE_SIZE (sizeof(pud_t) << RADIX_PUD_INDEX_SIZE) +#define RADIX_PGD_TABLE_SIZE (sizeof(pgd_t) << RADIX_PGD_INDEX_SIZE) + +#ifdef CONFIG_STRICT_KERNEL_RWX +extern void radix__mark_rodata_ro(void); +extern void radix__mark_initmem_nx(void); +#endif + +extern void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep, + pte_t entry, unsigned long address, + int psize); + +extern void radix__ptep_modify_prot_commit(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t old_pte, pte_t pte); + +static inline unsigned long __radix_pte_update(pte_t *ptep, unsigned long clr, + unsigned long set) +{ + __be64 old_be, tmp_be; + + __asm__ __volatile__( + "1: ldarx %0,0,%3 # pte_update\n" + " andc %1,%0,%5 \n" + " or %1,%1,%4 \n" + " stdcx. %1,0,%3 \n" + " bne- 1b" + : "=&r" (old_be), "=&r" (tmp_be), "=m" (*ptep) + : "r" (ptep), "r" (cpu_to_be64(set)), "r" (cpu_to_be64(clr)) + : "cc" ); + + return be64_to_cpu(old_be); +} + +static inline unsigned long radix__pte_update(struct mm_struct *mm, + unsigned long addr, + pte_t *ptep, unsigned long clr, + unsigned long set, + int huge) +{ + unsigned long old_pte; + + old_pte = __radix_pte_update(ptep, clr, set); + if (!huge) + assert_pte_locked(mm, addr); + + return old_pte; +} + +static inline pte_t radix__ptep_get_and_clear_full(struct mm_struct *mm, + unsigned long addr, + pte_t *ptep, int full) +{ + unsigned long old_pte; + + if (full) { + old_pte = pte_val(*ptep); + *ptep = __pte(0); + } else + old_pte = radix__pte_update(mm, addr, ptep, ~0ul, 0, 0); + + return __pte(old_pte); +} + +static inline int radix__pte_same(pte_t pte_a, pte_t pte_b) +{ + return ((pte_raw(pte_a) ^ pte_raw(pte_b)) == 0); +} + +static inline int radix__pte_none(pte_t pte) +{ + return (pte_val(pte) & ~RADIX_PTE_NONE_MASK) == 0; +} + +static inline void radix__set_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, int percpu) +{ + *ptep = pte; + + /* + * The architecture suggests a ptesync after setting the pte, which + * orders the store that updates the pte with subsequent page table + * walk accesses which may load the pte. Without this it may be + * possible for a subsequent access to result in spurious fault. + * + * This is not necessary for correctness, because a spurious fault + * is tolerated by the page fault handler, and this store will + * eventually be seen. In testing, there was no noticable increase + * in user faults on POWER9. Avoiding ptesync here is a significant + * win for things like fork. If a future microarchitecture benefits + * from ptesync, it should probably go into update_mmu_cache, rather + * than set_pte_at (which is used to set ptes unrelated to faults). + * + * Spurious faults from the kernel memory are not tolerated, so there + * is a ptesync in flush_cache_vmap, and __map_kernel_page() follows + * the pte update sequence from ISA Book III 6.10 Translation Table + * Update Synchronization Requirements. + */ +} + +static inline int radix__pmd_bad(pmd_t pmd) +{ + return !!(pmd_val(pmd) & RADIX_PMD_BAD_BITS); +} + +static inline int radix__pmd_same(pmd_t pmd_a, pmd_t pmd_b) +{ + return ((pmd_raw(pmd_a) ^ pmd_raw(pmd_b)) == 0); +} + +static inline int radix__pud_bad(pud_t pud) +{ + return !!(pud_val(pud) & RADIX_PUD_BAD_BITS); +} + +static inline int radix__pud_same(pud_t pud_a, pud_t pud_b) +{ + return ((pud_raw(pud_a) ^ pud_raw(pud_b)) == 0); +} + +static inline int radix__p4d_bad(p4d_t p4d) +{ + return !!(p4d_val(p4d) & RADIX_P4D_BAD_BITS); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + +static inline int radix__pmd_trans_huge(pmd_t pmd) +{ + return (pmd_val(pmd) & _PAGE_PTE) == _PAGE_PTE; +} + +static inline pmd_t radix__pmd_mkhuge(pmd_t pmd) +{ + return __pmd(pmd_val(pmd) | _PAGE_PTE); +} + +static inline int radix__pud_trans_huge(pud_t pud) +{ + return (pud_val(pud) & _PAGE_PTE) == _PAGE_PTE; +} + +static inline pud_t radix__pud_mkhuge(pud_t pud) +{ + return __pud(pud_val(pud) | _PAGE_PTE); +} + +extern unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, unsigned long clr, + unsigned long set); +extern unsigned long radix__pud_hugepage_update(struct mm_struct *mm, unsigned long addr, + pud_t *pudp, unsigned long clr, + unsigned long set); +extern pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp); +extern void radix__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, + pgtable_t pgtable); +extern pgtable_t radix__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); +extern pmd_t radix__pmdp_huge_get_and_clear(struct mm_struct *mm, + unsigned long addr, pmd_t *pmdp); +pud_t radix__pudp_huge_get_and_clear(struct mm_struct *mm, + unsigned long addr, pud_t *pudp); + +static inline int radix__has_transparent_hugepage(void) +{ + /* For radix 2M at PMD level means thp */ + if (mmu_psize_defs[MMU_PAGE_2M].shift == PMD_SHIFT) + return 1; + return 0; +} + +static inline int radix__has_transparent_pud_hugepage(void) +{ + /* For radix 1G at PUD level means pud hugepage support */ + if (mmu_psize_defs[MMU_PAGE_1G].shift == PUD_SHIFT) + return 1; + return 0; +} +#endif + +struct vmem_altmap; +struct dev_pagemap; +extern int __meminit radix__vmemmap_create_mapping(unsigned long start, + unsigned long page_size, + unsigned long phys); +int __meminit radix__vmemmap_populate(unsigned long start, unsigned long end, + int node, struct vmem_altmap *altmap); +void __ref radix__vmemmap_free(unsigned long start, unsigned long end, + struct vmem_altmap *altmap); +extern void radix__vmemmap_remove_mapping(unsigned long start, + unsigned long page_size); + +extern int radix__map_kernel_page(unsigned long ea, unsigned long pa, + pgprot_t flags, unsigned int psz); + +static inline unsigned long radix__get_tree_size(void) +{ + unsigned long rts_field; + /* + * We support 52 bits, hence: + * bits 52 - 31 = 21, 0b10101 + * RTS encoding details + * bits 0 - 3 of rts -> bits 6 - 8 unsigned long + * bits 4 - 5 of rts -> bits 62 - 63 of unsigned long + */ + rts_field = (0x5UL << 5); /* 6 - 8 bits */ + rts_field |= (0x2UL << 61); + + return rts_field; +} + +#ifdef CONFIG_MEMORY_HOTPLUG +int radix__create_section_mapping(unsigned long start, unsigned long end, + int nid, pgprot_t prot); +int radix__remove_section_mapping(unsigned long start, unsigned long end); +#endif /* CONFIG_MEMORY_HOTPLUG */ + +#ifdef CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAP +#define vmemmap_can_optimize vmemmap_can_optimize +bool vmemmap_can_optimize(struct vmem_altmap *altmap, struct dev_pagemap *pgmap); +#endif + +#define vmemmap_populate_compound_pages vmemmap_populate_compound_pages +int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, + unsigned long start, + unsigned long end, int node, + struct dev_pagemap *pgmap); +#endif /* __ASSEMBLER__ */ +#endif diff --git a/arch/powerpc/include/asm/book3s/64/slice.h b/arch/powerpc/include/asm/book3s/64/slice.h new file mode 100644 index 000000000000..6e2f7a74cd75 --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/slice.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_64_SLICE_H +#define _ASM_POWERPC_BOOK3S_64_SLICE_H + +#ifndef __ASSEMBLER__ + +#ifdef CONFIG_PPC_64S_HASH_MMU +#ifdef CONFIG_HUGETLB_PAGE +#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA +#endif +#define HAVE_ARCH_UNMAPPED_AREA +#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN +#endif + +#define SLICE_LOW_SHIFT 28 +#define SLICE_LOW_TOP (0x100000000ul) +#define SLICE_NUM_LOW (SLICE_LOW_TOP >> SLICE_LOW_SHIFT) +#define GET_LOW_SLICE_INDEX(addr) ((addr) >> SLICE_LOW_SHIFT) + +#define SLICE_HIGH_SHIFT 40 +#define SLICE_NUM_HIGH (H_PGTABLE_RANGE >> SLICE_HIGH_SHIFT) +#define GET_HIGH_SLICE_INDEX(addr) ((addr) >> SLICE_HIGH_SHIFT) + +#define SLB_ADDR_LIMIT_DEFAULT DEFAULT_MAP_WINDOW_USER64 + +struct mm_struct; + +unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len, + unsigned long flags, unsigned int psize, + int topdown); + +unsigned int get_slice_psize(struct mm_struct *mm, unsigned long addr); + +void slice_set_range_psize(struct mm_struct *mm, unsigned long start, + unsigned long len, unsigned int psize); + +void slice_init_new_context_exec(struct mm_struct *mm); +void slice_setup_new_exec(void); + +#endif /* __ASSEMBLER__ */ + +#endif /* _ASM_POWERPC_BOOK3S_64_SLICE_H */ diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h b/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h new file mode 100644 index 000000000000..146287d9580f --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_64_TLBFLUSH_HASH_H +#define _ASM_POWERPC_BOOK3S_64_TLBFLUSH_HASH_H + +/* + * TLB flushing for 64-bit hash-MMU CPUs + */ + +#include <linux/percpu.h> +#include <asm/page.h> + +#define PPC64_TLB_BATCH_NR 192 + +struct ppc64_tlb_batch { + int active; + unsigned long index; + struct mm_struct *mm; + real_pte_t pte[PPC64_TLB_BATCH_NR]; + unsigned long vpn[PPC64_TLB_BATCH_NR]; + unsigned int psize; + int ssize; +}; +DECLARE_PER_CPU(struct ppc64_tlb_batch, ppc64_tlb_batch); + +extern void __flush_tlb_pending(struct ppc64_tlb_batch *batch); + +#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE + +static inline void arch_enter_lazy_mmu_mode(void) +{ + struct ppc64_tlb_batch *batch; + + if (radix_enabled()) + return; + /* + * apply_to_page_range can call us this preempt enabled when + * operating on kernel page tables. + */ + preempt_disable(); + batch = this_cpu_ptr(&ppc64_tlb_batch); + batch->active = 1; +} + +static inline void arch_leave_lazy_mmu_mode(void) +{ + struct ppc64_tlb_batch *batch; + + if (radix_enabled()) + return; + batch = this_cpu_ptr(&ppc64_tlb_batch); + + if (batch->index) + __flush_tlb_pending(batch); + batch->active = 0; + preempt_enable(); +} + +#define arch_flush_lazy_mmu_mode() do {} while (0) + +extern void hash__tlbiel_all(unsigned int action); + +extern void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, + int ssize, unsigned long flags); +extern void flush_hash_range(unsigned long number, int local); +extern void flush_hash_hugepage(unsigned long vsid, unsigned long addr, + pmd_t *pmdp, unsigned int psize, int ssize, + unsigned long flags); + +struct mmu_gather; +extern void hash__tlb_flush(struct mmu_gather *tlb); + +#ifdef CONFIG_PPC_64S_HASH_MMU +/* Private function for use by PCI IO mapping code */ +extern void __flush_hash_table_range(unsigned long start, unsigned long end); +void flush_hash_table_pmd_range(struct mm_struct *mm, pmd_t *pmd, unsigned long addr); +#else +static inline void __flush_hash_table_range(unsigned long start, unsigned long end) { } +#endif +#endif /* _ASM_POWERPC_BOOK3S_64_TLBFLUSH_HASH_H */ diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h new file mode 100644 index 000000000000..a38542259fab --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_TLBFLUSH_RADIX_H +#define _ASM_POWERPC_TLBFLUSH_RADIX_H + +#include <asm/hvcall.h> + +#define RIC_FLUSH_TLB 0 +#define RIC_FLUSH_PWC 1 +#define RIC_FLUSH_ALL 2 + +struct vm_area_struct; +struct mm_struct; +struct mmu_gather; + +static inline u64 psize_to_rpti_pgsize(unsigned long psize) +{ + if (psize == MMU_PAGE_4K) + return H_RPTI_PAGE_4K; + if (psize == MMU_PAGE_64K) + return H_RPTI_PAGE_64K; + if (psize == MMU_PAGE_2M) + return H_RPTI_PAGE_2M; + if (psize == MMU_PAGE_1G) + return H_RPTI_PAGE_1G; + return H_RPTI_PAGE_ALL; +} + +static inline int mmu_get_ap(int psize) +{ + return mmu_psize_defs[psize].ap; +} + +#ifdef CONFIG_PPC_RADIX_MMU +extern void radix__tlbiel_all(unsigned int action); +extern void radix__flush_tlb_lpid_page(unsigned int lpid, + unsigned long addr, + unsigned long page_size); +extern void radix__flush_pwc_lpid(unsigned int lpid); +extern void radix__flush_all_lpid(unsigned int lpid); +extern void radix__flush_all_lpid_guest(unsigned int lpid); +#else +static inline void radix__tlbiel_all(unsigned int action) { WARN_ON(1); } +static inline void radix__flush_tlb_lpid_page(unsigned int lpid, + unsigned long addr, + unsigned long page_size) +{ + WARN_ON(1); +} +static inline void radix__flush_pwc_lpid(unsigned int lpid) +{ + WARN_ON(1); +} +static inline void radix__flush_all_lpid(unsigned int lpid) +{ + WARN_ON(1); +} +static inline void radix__flush_all_lpid_guest(unsigned int lpid) +{ + WARN_ON(1); +} +#endif + +extern void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end); +extern void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start, + unsigned long end, int psize); +void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start, + unsigned long end, int psize); +extern void radix__flush_pmd_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end); +extern void radix__flush_pud_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end); +extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end); +extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end); + +extern void radix__local_flush_tlb_mm(struct mm_struct *mm); +extern void radix__local_flush_all_mm(struct mm_struct *mm); +extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); +extern void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, + int psize); +extern void radix__tlb_flush(struct mmu_gather *tlb); +#ifdef CONFIG_SMP +extern void radix__flush_tlb_mm(struct mm_struct *mm); +extern void radix__flush_all_mm(struct mm_struct *mm); +extern void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); +extern void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, + int psize); +#else +#define radix__flush_tlb_mm(mm) radix__local_flush_tlb_mm(mm) +#define radix__flush_all_mm(mm) radix__local_flush_all_mm(mm) +#define radix__flush_tlb_page(vma,addr) radix__local_flush_tlb_page(vma,addr) +#define radix__flush_tlb_page_psize(mm,addr,p) radix__local_flush_tlb_page_psize(mm,addr,p) +#endif +extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr); +extern void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr); +extern void radix__flush_tlb_all(void); + +#endif diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h new file mode 100644 index 000000000000..fd642b729775 --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h @@ -0,0 +1,225 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H +#define _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H + +#define MMU_NO_CONTEXT ~0UL + +#include <linux/mm_types.h> +#include <linux/mmu_notifier.h> +#include <asm/book3s/64/tlbflush-hash.h> +#include <asm/book3s/64/tlbflush-radix.h> + +/* TLB flush actions. Used as argument to tlbiel_all() */ +enum { + TLB_INVAL_SCOPE_GLOBAL = 0, /* invalidate all TLBs */ + TLB_INVAL_SCOPE_LPID = 1, /* invalidate TLBs for current LPID */ +}; + +static inline void tlbiel_all(void) +{ + /* + * This is used for host machine check and bootup. + * + * This uses early_radix_enabled and implementations use + * early_cpu_has_feature etc because that works early in boot + * and this is the machine check path which is not performance + * critical. + */ + if (early_radix_enabled()) + radix__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL); + else + hash__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL); +} + +static inline void tlbiel_all_lpid(bool radix) +{ + /* + * This is used for guest machine check. + */ + if (radix) + radix__tlbiel_all(TLB_INVAL_SCOPE_LPID); + else + hash__tlbiel_all(TLB_INVAL_SCOPE_LPID); +} + + +#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE +static inline void flush_pmd_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ + if (radix_enabled()) + radix__flush_pmd_tlb_range(vma, start, end); +} + +#define __HAVE_ARCH_FLUSH_PUD_TLB_RANGE +static inline void flush_pud_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ + if (radix_enabled()) + radix__flush_pud_tlb_range(vma, start, end); +} + +#define __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE +static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma, + unsigned long start, + unsigned long end) +{ + if (radix_enabled()) + radix__flush_hugetlb_tlb_range(vma, start, end); +} + +static inline void flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ + if (radix_enabled()) + radix__flush_tlb_range(vma, start, end); +} + +static inline void flush_tlb_kernel_range(unsigned long start, + unsigned long end) +{ + if (radix_enabled()) + radix__flush_tlb_kernel_range(start, end); +} + +static inline void local_flush_tlb_mm(struct mm_struct *mm) +{ + if (radix_enabled()) + radix__local_flush_tlb_mm(mm); +} + +static inline void local_flush_tlb_page(struct vm_area_struct *vma, + unsigned long vmaddr) +{ + if (radix_enabled()) + radix__local_flush_tlb_page(vma, vmaddr); +} + +static inline void local_flush_tlb_page_psize(struct mm_struct *mm, + unsigned long vmaddr, int psize) +{ + if (radix_enabled()) + radix__local_flush_tlb_page_psize(mm, vmaddr, psize); +} + +static inline void tlb_flush(struct mmu_gather *tlb) +{ + if (radix_enabled()) + radix__tlb_flush(tlb); + else + hash__tlb_flush(tlb); +} + +#ifdef CONFIG_SMP +static inline void flush_tlb_mm(struct mm_struct *mm) +{ + if (radix_enabled()) + radix__flush_tlb_mm(mm); +} + +static inline void flush_tlb_page(struct vm_area_struct *vma, + unsigned long vmaddr) +{ + if (radix_enabled()) + radix__flush_tlb_page(vma, vmaddr); +} +#else +#define flush_tlb_mm(mm) local_flush_tlb_mm(mm) +#define flush_tlb_page(vma, addr) local_flush_tlb_page(vma, addr) +#endif /* CONFIG_SMP */ + +#define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault +static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma, + unsigned long address, + pte_t *ptep) +{ + /* + * Book3S 64 does not require spurious fault flushes because the PTE + * must be re-fetched in case of an access permission problem. So the + * only reason for a spurious fault should be concurrent modification + * to the PTE, in which case the PTE will eventually be re-fetched by + * the MMU when it attempts the access again. + * + * See: Power ISA Version 3.1B, 6.10.1.2 Modifying a Translation Table + * Entry, Setting a Reference or Change Bit or Upgrading Access + * Authority (PTE Subject to Atomic Hardware Updates): + * + * "If the only change being made to a valid PTE that is subject to + * atomic hardware updates is to set the Reference or Change bit to + * 1 or to upgrade access authority, a simpler sequence suffices + * because the translation hardware will refetch the PTE if an + * access is attempted for which the only problems were reference + * and/or change bits needing to be set or insufficient access + * authority." + * + * The nest MMU in POWER9 does not perform this PTE re-fetch, but + * it avoids the spurious fault problem by flushing the TLB before + * upgrading PTE permissions, see radix__ptep_set_access_flags. + */ +} + +static inline bool __pte_flags_need_flush(unsigned long oldval, + unsigned long newval) +{ + unsigned long delta = oldval ^ newval; + + /* + * The return value of this function doesn't matter for hash, + * ptep_modify_prot_start() does a pte_update() which does or schedules + * any necessary hash table update and flush. + */ + if (!radix_enabled()) + return true; + + /* + * We do not expect kernel mappings or non-PTEs or not-present PTEs. + */ + VM_WARN_ON_ONCE(oldval & _PAGE_PRIVILEGED); + VM_WARN_ON_ONCE(newval & _PAGE_PRIVILEGED); + VM_WARN_ON_ONCE(!(oldval & _PAGE_PTE)); + VM_WARN_ON_ONCE(!(newval & _PAGE_PTE)); + VM_WARN_ON_ONCE(!(oldval & _PAGE_PRESENT)); + VM_WARN_ON_ONCE(!(newval & _PAGE_PRESENT)); + + /* + * Must flush on any change except READ, WRITE, EXEC, DIRTY, ACCESSED. + * + * In theory, some changed software bits could be tolerated, in + * practice those should rarely if ever matter. + */ + + if (delta & ~(_PAGE_RWX | _PAGE_DIRTY | _PAGE_ACCESSED)) + return true; + + /* + * If any of the above was present in old but cleared in new, flush. + * With the exception of _PAGE_ACCESSED, don't worry about flushing + * if that was cleared (see the comment in ptep_clear_flush_young()). + */ + if ((delta & ~_PAGE_ACCESSED) & oldval) + return true; + + return false; +} + +static inline bool pte_needs_flush(pte_t oldpte, pte_t newpte) +{ + return __pte_flags_need_flush(pte_val(oldpte), pte_val(newpte)); +} +#define pte_needs_flush pte_needs_flush + +static inline bool huge_pmd_needs_flush(pmd_t oldpmd, pmd_t newpmd) +{ + return __pte_flags_need_flush(pmd_val(oldpmd), pmd_val(newpmd)); +} +#define huge_pmd_needs_flush huge_pmd_needs_flush + +extern bool tlbie_capable; +extern bool tlbie_enabled; + +static inline bool cputlb_use_tlbie(void) +{ + return tlbie_enabled; +} + +#endif /* _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H */ diff --git a/arch/powerpc/include/asm/book3s/pgalloc.h b/arch/powerpc/include/asm/book3s/pgalloc.h new file mode 100644 index 000000000000..6b178ca143e7 --- /dev/null +++ b/arch/powerpc/include/asm/book3s/pgalloc.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_PGALLOC_H +#define _ASM_POWERPC_BOOK3S_PGALLOC_H + +#include <linux/mm.h> + +extern void tlb_remove_table(struct mmu_gather *tlb, void *table); + +#ifdef CONFIG_PPC64 +#include <asm/book3s/64/pgalloc.h> +#else +#include <asm/book3s/32/pgalloc.h> +#endif + +#endif /* _ASM_POWERPC_BOOK3S_PGALLOC_H */ diff --git a/arch/powerpc/include/asm/book3s/pgtable.h b/arch/powerpc/include/asm/book3s/pgtable.h new file mode 100644 index 000000000000..f42d68c6b314 --- /dev/null +++ b/arch/powerpc/include/asm/book3s/pgtable.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_PGTABLE_H +#define _ASM_POWERPC_BOOK3S_PGTABLE_H + +#ifdef CONFIG_PPC64 +#include <asm/book3s/64/pgtable.h> +#else +#include <asm/book3s/32/pgtable.h> +#endif + +#endif diff --git a/arch/powerpc/include/asm/book3s/tlbflush.h b/arch/powerpc/include/asm/book3s/tlbflush.h new file mode 100644 index 000000000000..dec11de41055 --- /dev/null +++ b/arch/powerpc/include/asm/book3s/tlbflush.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_TLBFLUSH_H +#define _ASM_POWERPC_BOOK3S_TLBFLUSH_H + +#ifdef CONFIG_PPC64 +#include <asm/book3s/64/tlbflush.h> +#else +#include <asm/book3s/32/tlbflush.h> +#endif + +#endif /* _ASM_POWERPC_BOOK3S_TLBFLUSH_H */ diff --git a/arch/powerpc/include/asm/bootx.h b/arch/powerpc/include/asm/bootx.h index dd9461003dfa..1c121f3c524f 100644 --- a/arch/powerpc/include/asm/bootx.h +++ b/arch/powerpc/include/asm/bootx.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * This file describes the structure passed from the BootX application * (for MacOS) when it is used to boot Linux. diff --git a/arch/powerpc/include/asm/bpf_perf_event.h b/arch/powerpc/include/asm/bpf_perf_event.h new file mode 100644 index 000000000000..e8a7b4ffb58c --- /dev/null +++ b/arch/powerpc/include/asm/bpf_perf_event.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BPF_PERF_EVENT_H +#define _ASM_POWERPC_BPF_PERF_EVENT_H + +#include <asm/ptrace.h> + +typedef struct user_pt_regs bpf_user_pt_regs_t; + +#endif /* _ASM_POWERPC_BPF_PERF_EVENT_H */ diff --git a/arch/powerpc/include/asm/btext.h b/arch/powerpc/include/asm/btext.h index 906f46e31006..860f8868f11e 100644 --- a/arch/powerpc/include/asm/btext.h +++ b/arch/powerpc/include/asm/btext.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Definitions for using the procedures in btext.c. * @@ -12,17 +13,22 @@ extern void btext_update_display(unsigned long phys, int width, int height, int depth, int pitch); extern void btext_setup_display(int width, int height, int depth, int pitch, unsigned long address); +#ifdef CONFIG_PPC32 extern void btext_prepare_BAT(void); +#else +static inline void btext_prepare_BAT(void) { } +#endif +extern void btext_map(void); extern void btext_unmap(void); extern void btext_drawchar(char c); extern void btext_drawstring(const char *str); -extern void btext_drawhex(unsigned long v); -extern void btext_drawtext(const char *c, unsigned int len); +void __init btext_drawhex(unsigned long v); +void __init btext_drawtext(const char *c, unsigned int len); -extern void btext_clearscreen(void); -extern void btext_flushscreen(void); -extern void btext_flushline(void); +void __init btext_clearscreen(void); +void __init btext_flushscreen(void); +void __init btext_flushline(void); #endif /* __KERNEL__ */ #endif /* __PPC_BTEXT_H */ diff --git a/arch/powerpc/include/asm/bug.h b/arch/powerpc/include/asm/bug.h index 3eb53d741070..0db48977c70c 100644 --- a/arch/powerpc/include/asm/bug.h +++ b/arch/powerpc/include/asm/bug.h @@ -1,25 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_BUG_H #define _ASM_POWERPC_BUG_H #ifdef __KERNEL__ #include <asm/asm-compat.h> -/* - * Define an illegal instr to trap on the bug. - * We don't use 0 because that marks the end of a function - * in the ELF ABI. That's "Boo Boo" in case you wonder... - */ -#define BUG_OPCODE .long 0x00b00b00 /* For asm */ -#define BUG_ILLEGAL_INSTR "0x00b00b00" /* For BUG macro */ - #ifdef CONFIG_BUG -#ifdef __ASSEMBLY__ +#ifdef __ASSEMBLER__ #include <asm/asm-offsets.h> #ifdef CONFIG_DEBUG_BUGVERBOSE .macro EMIT_BUG_ENTRY addr,file,line,flags - .section __bug_table,"a" -5001: PPC_LONG \addr, 5002f + .section __bug_table,"aw" +5001: .4byte \addr - . + .4byte 5002f - . .short \line, \flags .org 5001b+BUG_ENTRY_SIZE .previous @@ -29,33 +23,43 @@ .endm #else .macro EMIT_BUG_ENTRY addr,file,line,flags - .section __bug_table,"a" -5001: PPC_LONG \addr + .section __bug_table,"aw" +5001: .4byte \addr - . .short \flags .org 5001b+BUG_ENTRY_SIZE .previous .endm #endif /* verbose */ -#else /* !__ASSEMBLY__ */ +#else /* !__ASSEMBLER__ */ /* _EMIT_BUG_ENTRY expects args %0,%1,%2,%3 to be FILE, LINE, flags and sizeof(struct bug_entry), respectively */ #ifdef CONFIG_DEBUG_BUGVERBOSE #define _EMIT_BUG_ENTRY \ - ".section __bug_table,\"a\"\n" \ - "2:\t" PPC_LONG "1b, %0\n" \ - "\t.short %1, %2\n" \ + ".section __bug_table,\"aw\"\n" \ + "2: .4byte 1b - .\n" \ + " .4byte %0 - .\n" \ + " .short %1, %2\n" \ ".org 2b+%3\n" \ ".previous\n" #else #define _EMIT_BUG_ENTRY \ - ".section __bug_table,\"a\"\n" \ - "2:\t" PPC_LONG "1b\n" \ - "\t.short %2\n" \ + ".section __bug_table,\"aw\"\n" \ + "2: .4byte 1b - .\n" \ + " .short %2\n" \ ".org 2b+%3\n" \ ".previous\n" #endif +#define BUG_ENTRY(cond_str, insn, flags, ...) \ + __asm__ __volatile__( \ + "1: " insn "\n" \ + _EMIT_BUG_ENTRY \ + : : "i" (WARN_CONDITION_STR(cond_str) __FILE__), "i" (__LINE__), \ + "i" (flags), \ + "i" (sizeof(struct bug_entry)), \ + ##__VA_ARGS__) + /* * BUG_ON() and WARN_ON() do their best to cooperate with compile-time * optimisations. However depending on the complexity of the condition @@ -63,79 +67,68 @@ */ #define BUG() do { \ - __asm__ __volatile__( \ - "1: twi 31,0,0\n" \ - _EMIT_BUG_ENTRY \ - : : "i" (__FILE__), "i" (__LINE__), \ - "i" (0), "i" (sizeof(struct bug_entry))); \ + BUG_ENTRY("", "twi 31, 0, 0", 0); \ unreachable(); \ } while (0) +#define HAVE_ARCH_BUG + +#define __WARN_FLAGS(cond_str, flags) BUG_ENTRY(cond_str, "twi 31, 0, 0", BUGFLAG_WARNING | (flags)) +#ifdef CONFIG_PPC64 #define BUG_ON(x) do { \ if (__builtin_constant_p(x)) { \ if (x) \ BUG(); \ } else { \ - __asm__ __volatile__( \ - "1: "PPC_TLNEI" %4,0\n" \ - _EMIT_BUG_ENTRY \ - : : "i" (__FILE__), "i" (__LINE__), "i" (0), \ - "i" (sizeof(struct bug_entry)), \ - "r" ((__force long)(x))); \ + BUG_ENTRY(#x, PPC_TLNEI " %4, 0", 0, "r" ((__force long)(x))); \ } \ } while (0) -#define __WARN_TAINT(taint) do { \ - __asm__ __volatile__( \ - "1: twi 31,0,0\n" \ - _EMIT_BUG_ENTRY \ - : : "i" (__FILE__), "i" (__LINE__), \ - "i" (BUGFLAG_TAINT(taint)), \ - "i" (sizeof(struct bug_entry))); \ -} while (0) - #define WARN_ON(x) ({ \ int __ret_warn_on = !!(x); \ if (__builtin_constant_p(__ret_warn_on)) { \ if (__ret_warn_on) \ __WARN(); \ } else { \ - __asm__ __volatile__( \ - "1: "PPC_TLNEI" %4,0\n" \ - _EMIT_BUG_ENTRY \ - : : "i" (__FILE__), "i" (__LINE__), \ - "i" (BUGFLAG_TAINT(TAINT_WARN)), \ - "i" (sizeof(struct bug_entry)), \ - "r" (__ret_warn_on)); \ + BUG_ENTRY(#x, PPC_TLNEI " %4, 0", \ + BUGFLAG_WARNING | BUGFLAG_TAINT(TAINT_WARN), \ + "r" (__ret_warn_on)); \ } \ unlikely(__ret_warn_on); \ }) -#define HAVE_ARCH_BUG #define HAVE_ARCH_BUG_ON #define HAVE_ARCH_WARN_ON -#endif /* __ASSEMBLY __ */ +#endif + +#endif /* __ASSEMBLER__ */ #else -#ifdef __ASSEMBLY__ +#ifdef __ASSEMBLER__ .macro EMIT_BUG_ENTRY addr,file,line,flags .endm -#else /* !__ASSEMBLY__ */ +#else /* !__ASSEMBLER__ */ #define _EMIT_BUG_ENTRY #endif #endif /* CONFIG_BUG */ +#define EMIT_WARN_ENTRY EMIT_BUG_ENTRY + #include <asm-generic/bug.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ struct pt_regs; -extern int do_page_fault(struct pt_regs *, unsigned long, unsigned long); -extern void bad_page_fault(struct pt_regs *, unsigned long, int); +void hash__do_page_fault(struct pt_regs *); +void bad_page_fault(struct pt_regs *, int); +void emulate_single_step(struct pt_regs *regs); extern void _exception(int, struct pt_regs *, int, unsigned long); +extern void _exception_pkey(struct pt_regs *, unsigned long, int); extern void die(const char *, struct pt_regs *, long); -extern void print_backtrace(unsigned long *); - -#endif /* !__ASSEMBLY__ */ +void die_mce(const char *str, struct pt_regs *regs, long err); +extern bool die_will_crash(void); +extern void panic_flush_kmsg_start(void); +extern void panic_flush_kmsg_end(void); +#endif /* !__ASSEMBLER__ */ #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_BUG_H */ diff --git a/arch/powerpc/include/asm/bugs.h b/arch/powerpc/include/asm/bugs.h deleted file mode 100644 index 42fdb73e3068..000000000000 --- a/arch/powerpc/include/asm/bugs.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef _ASM_POWERPC_BUGS_H -#define _ASM_POWERPC_BUGS_H - -/* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ - -/* - * This file is included by 'init/main.c' to check for - * architecture-dependent bugs. - */ - -static inline void check_bugs(void) { } - -#endif /* _ASM_POWERPC_BUGS_H */ diff --git a/arch/powerpc/include/asm/cache.h b/arch/powerpc/include/asm/cache.h index 9e495c9a6a88..6796babc4d31 100644 --- a/arch/powerpc/include/asm/cache.h +++ b/arch/powerpc/include/asm/cache.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_CACHE_H #define _ASM_POWERPC_CACHE_H @@ -5,14 +6,17 @@ /* bytes per L1 cache line */ -#if defined(CONFIG_8xx) || defined(CONFIG_403GCX) +#if defined(CONFIG_PPC_8xx) #define L1_CACHE_SHIFT 4 #define MAX_COPY_PREFETCH 1 +#define IFETCH_ALIGN_SHIFT 2 #elif defined(CONFIG_PPC_E500MC) #define L1_CACHE_SHIFT 6 #define MAX_COPY_PREFETCH 4 +#define IFETCH_ALIGN_SHIFT 3 #elif defined(CONFIG_PPC32) #define MAX_COPY_PREFETCH 4 +#define IFETCH_ALIGN_SHIFT 3 /* 603 fetches 2 insn at a time */ #if defined(CONFIG_PPC_47x) #define L1_CACHE_SHIFT 7 #else @@ -20,32 +24,86 @@ #endif #else /* CONFIG_PPC64 */ #define L1_CACHE_SHIFT 7 +#define IFETCH_ALIGN_SHIFT 4 /* POWER8,9 */ #endif #define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) #define SMP_CACHE_BYTES L1_CACHE_BYTES -#if defined(__powerpc64__) && !defined(__ASSEMBLY__) +#define IFETCH_ALIGN_BYTES (1 << IFETCH_ALIGN_SHIFT) + +#ifdef CONFIG_NOT_COHERENT_CACHE +#define ARCH_DMA_MINALIGN L1_CACHE_BYTES +#endif + +#if !defined(__ASSEMBLER__) +#ifdef CONFIG_PPC64 + +struct ppc_cache_info { + u32 size; + u32 line_size; + u32 block_size; /* L1 only */ + u32 log_block_size; + u32 blocks_per_page; + u32 sets; + u32 assoc; +}; + struct ppc64_caches { - u32 dsize; /* L1 d-cache size */ - u32 dline_size; /* L1 d-cache line size */ - u32 log_dline_size; - u32 dlines_per_page; - u32 isize; /* L1 i-cache size */ - u32 iline_size; /* L1 i-cache line size */ - u32 log_iline_size; - u32 ilines_per_page; + struct ppc_cache_info l1d; + struct ppc_cache_info l1i; + struct ppc_cache_info l2; + struct ppc_cache_info l3; }; extern struct ppc64_caches ppc64_caches; -#endif /* __powerpc64__ && ! __ASSEMBLY__ */ -#if !defined(__ASSEMBLY__) +static inline u32 l1_dcache_shift(void) +{ + return ppc64_caches.l1d.log_block_size; +} + +static inline u32 l1_dcache_bytes(void) +{ + return ppc64_caches.l1d.block_size; +} + +static inline u32 l1_icache_shift(void) +{ + return ppc64_caches.l1i.log_block_size; +} + +static inline u32 l1_icache_bytes(void) +{ + return ppc64_caches.l1i.block_size; +} +#else +static inline u32 l1_dcache_shift(void) +{ + return L1_CACHE_SHIFT; +} + +static inline u32 l1_dcache_bytes(void) +{ + return L1_CACHE_BYTES; +} + +static inline u32 l1_icache_shift(void) +{ + return L1_CACHE_SHIFT; +} + +static inline u32 l1_icache_bytes(void) +{ + return L1_CACHE_BYTES; +} + +#endif -#define __read_mostly __attribute__((__section__(".data..read_mostly"))) +#define __read_mostly __section(".data..read_mostly") -#ifdef CONFIG_6xx +#ifdef CONFIG_PPC_BOOK3S_32 extern long _get_L2CR(void); extern long _get_L3CR(void); extern void _set_L2CR(unsigned long); @@ -57,9 +115,36 @@ extern void _set_L3CR(unsigned long); #define _set_L3CR(val) do { } while(0) #endif -extern void cacheable_memzero(void *p, unsigned int nb); -extern void *cacheable_memcpy(void *, const void *, unsigned int); +static inline void dcbz(void *addr) +{ + __asm__ __volatile__ ("dcbz 0, %0" : : "r"(addr) : "memory"); +} + +static inline void dcbi(void *addr) +{ + __asm__ __volatile__ ("dcbi 0, %0" : : "r"(addr) : "memory"); +} + +static inline void dcbf(void *addr) +{ + __asm__ __volatile__ ("dcbf 0, %0" : : "r"(addr) : "memory"); +} + +static inline void dcbst(void *addr) +{ + __asm__ __volatile__ ("dcbst 0, %0" : : "r"(addr) : "memory"); +} + +static inline void icbi(void *addr) +{ + asm volatile ("icbi 0, %0" : : "r"(addr) : "memory"); +} + +static inline void iccci(void *addr) +{ + asm volatile ("iccci 0, %0" : : "r"(addr) : "memory"); +} -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_CACHE_H */ diff --git a/arch/powerpc/include/asm/cacheflush.h b/arch/powerpc/include/asm/cacheflush.h index b843e35122e8..1fea42928f64 100644 --- a/arch/powerpc/include/asm/cacheflush.h +++ b/arch/powerpc/include/asm/cacheflush.h @@ -1,78 +1,136 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef _ASM_POWERPC_CACHEFLUSH_H #define _ASM_POWERPC_CACHEFLUSH_H -#ifdef __KERNEL__ - #include <linux/mm.h> #include <asm/cputable.h> +#include <asm/cpu_has_feature.h> + +/* + * This flag is used to indicate that the page pointed to by a pte is clean + * and does not require cleaning before returning it to the user. + */ +#define PG_dcache_clean PG_arch_1 +#ifdef CONFIG_PPC_BOOK3S_64 /* - * No cache flushing is required when address mappings are changed, - * because the caches on PowerPCs are physically addressed. + * Book3s has no ptesync after setting a pte, so without this ptesync it's + * possible for a kernel virtual mapping access to return a spurious fault + * if it's accessed right after the pte is set. The page fault handler does + * not expect this type of fault. flush_cache_vmap is not exactly the right + * place to put this, but it seems to work well enough. */ -#define flush_cache_all() do { } while (0) -#define flush_cache_mm(mm) do { } while (0) -#define flush_cache_dup_mm(mm) do { } while (0) -#define flush_cache_range(vma, start, end) do { } while (0) -#define flush_cache_page(vma, vmaddr, pfn) do { } while (0) -#define flush_icache_page(vma, page) do { } while (0) -#define flush_cache_vmap(start, end) do { } while (0) -#define flush_cache_vunmap(start, end) do { } while (0) +static inline void flush_cache_vmap(unsigned long start, unsigned long end) +{ + asm volatile("ptesync" ::: "memory"); +} +#define flush_cache_vmap flush_cache_vmap +#endif /* CONFIG_PPC_BOOK3S_64 */ #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 -extern void flush_dcache_page(struct page *page); -#define flush_dcache_mmap_lock(mapping) do { } while (0) -#define flush_dcache_mmap_unlock(mapping) do { } while (0) +/* + * This is called when a page has been modified by the kernel. + * It just marks the page as not i-cache clean. We do the i-cache + * flush later when the page is given to a user process, if necessary. + */ +static inline void flush_dcache_folio(struct folio *folio) +{ + if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) + return; + /* avoid an atomic op if possible */ + if (test_bit(PG_dcache_clean, &folio->flags.f)) + clear_bit(PG_dcache_clean, &folio->flags.f); +} +#define flush_dcache_folio flush_dcache_folio + +static inline void flush_dcache_page(struct page *page) +{ + flush_dcache_folio(page_folio(page)); +} + +void flush_icache_range(unsigned long start, unsigned long stop); +#define flush_icache_range flush_icache_range + +void flush_icache_user_page(struct vm_area_struct *vma, struct page *page, + unsigned long addr, int len); +#define flush_icache_user_page flush_icache_user_page -extern void __flush_disable_L1(void); +void flush_dcache_icache_folio(struct folio *folio); -extern void __flush_icache_range(unsigned long, unsigned long); -static inline void flush_icache_range(unsigned long start, unsigned long stop) +/** + * flush_dcache_range(): Write any modified data cache blocks out to memory and + * invalidate them. Does not invalidate the corresponding instruction cache + * blocks. + * + * @start: the start address + * @stop: the stop address (exclusive) + */ +static inline void flush_dcache_range(unsigned long start, unsigned long stop) { - if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) - __flush_icache_range(start, stop); + unsigned long shift = l1_dcache_shift(); + unsigned long bytes = l1_dcache_bytes(); + void *addr = (void *)(start & ~(bytes - 1)); + unsigned long size = stop - (unsigned long)addr + (bytes - 1); + unsigned long i; + + if (IS_ENABLED(CONFIG_PPC64)) + mb(); /* sync */ + + for (i = 0; i < size >> shift; i++, addr += bytes) + dcbf(addr); + mb(); /* sync */ + } -extern void flush_icache_user_range(struct vm_area_struct *vma, - struct page *page, unsigned long addr, - int len); -extern void __flush_dcache_icache(void *page_va); -extern void flush_dcache_icache_page(struct page *page); -#if defined(CONFIG_PPC32) && !defined(CONFIG_BOOKE) -extern void __flush_dcache_icache_phys(unsigned long physaddr); -#endif /* CONFIG_PPC32 && !CONFIG_BOOKE */ - -extern void flush_dcache_range(unsigned long start, unsigned long stop); -#ifdef CONFIG_PPC32 -extern void clean_dcache_range(unsigned long start, unsigned long stop); -extern void invalidate_dcache_range(unsigned long start, unsigned long stop); -#endif /* CONFIG_PPC32 */ -#ifdef CONFIG_PPC64 -extern void flush_inval_dcache_range(unsigned long start, unsigned long stop); -extern void flush_dcache_phys_range(unsigned long start, unsigned long stop); -#endif +/* + * Write any modified data cache blocks out to memory. + * Does not invalidate the corresponding cache lines (especially for + * any corresponding instruction cache). + */ +static inline void clean_dcache_range(unsigned long start, unsigned long stop) +{ + unsigned long shift = l1_dcache_shift(); + unsigned long bytes = l1_dcache_bytes(); + void *addr = (void *)(start & ~(bytes - 1)); + unsigned long size = stop - (unsigned long)addr + (bytes - 1); + unsigned long i; -#define copy_to_user_page(vma, page, vaddr, dst, src, len) \ - do { \ - memcpy(dst, src, len); \ - flush_icache_user_range(vma, page, vaddr, len); \ - } while (0) -#define copy_from_user_page(vma, page, vaddr, dst, src, len) \ - memcpy(dst, src, len) + for (i = 0; i < size >> shift; i++, addr += bytes) + dcbst(addr); + mb(); /* sync */ +} +/* + * Like above, but invalidate the D-cache. This is used by the 8xx + * to invalidate the cache so the PPC core doesn't get stale data + * from the CPM (no cache snooping here :-). + */ +static inline void invalidate_dcache_range(unsigned long start, + unsigned long stop) +{ + unsigned long shift = l1_dcache_shift(); + unsigned long bytes = l1_dcache_bytes(); + void *addr = (void *)(start & ~(bytes - 1)); + unsigned long size = stop - (unsigned long)addr + (bytes - 1); + unsigned long i; + for (i = 0; i < size >> shift; i++, addr += bytes) + dcbi(addr); + mb(); /* sync */ +} -#ifdef CONFIG_DEBUG_PAGEALLOC -/* internal debugging function */ -void kernel_map_pages(struct page *page, int numpages, int enable); +#ifdef CONFIG_44x +static inline void flush_instruction_cache(void) +{ + iccci((void *)KERNELBASE); + isync(); +} +#else +void flush_instruction_cache(void); #endif -#endif /* __KERNEL__ */ +#include <asm-generic/cacheflush.h> #endif /* _ASM_POWERPC_CACHEFLUSH_H */ diff --git a/arch/powerpc/include/asm/cell-pmu.h b/arch/powerpc/include/asm/cell-pmu.h index b4b7338ad79e..7fbefd64b4fb 100644 --- a/arch/powerpc/include/asm/cell-pmu.h +++ b/arch/powerpc/include/asm/cell-pmu.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Cell Broadband Engine Performance Monitor * @@ -6,20 +7,6 @@ * Author: * David Erb (djerb@us.ibm.com) * Kevin Corry (kevcorry@us.ibm.com) - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2, or (at your option) - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #ifndef __ASM_CELL_PMU_H__ @@ -33,36 +20,9 @@ /* Macros for the pm_control register. */ #define CBE_PM_16BIT_CTR(ctr) (1 << (24 - ((ctr) & (NR_PHYS_CTRS - 1)))) -#define CBE_PM_ENABLE_PERF_MON 0x80000000 -#define CBE_PM_STOP_AT_MAX 0x40000000 -#define CBE_PM_TRACE_MODE_GET(pm_control) (((pm_control) >> 28) & 0x3) -#define CBE_PM_TRACE_MODE_SET(mode) (((mode) & 0x3) << 28) -#define CBE_PM_TRACE_BUF_OVFLW(bit) (((bit) & 0x1) << 17) -#define CBE_PM_COUNT_MODE_SET(count) (((count) & 0x3) << 18) -#define CBE_PM_FREEZE_ALL_CTRS 0x00100000 -#define CBE_PM_ENABLE_EXT_TRACE 0x00008000 -#define CBE_PM_SPU_ADDR_TRACE_SET(msk) (((msk) & 0x3) << 9) /* Macros for the trace_address register. */ -#define CBE_PM_TRACE_BUF_FULL 0x00000800 #define CBE_PM_TRACE_BUF_EMPTY 0x00000400 -#define CBE_PM_TRACE_BUF_DATA_COUNT(ta) ((ta) & 0x3ff) -#define CBE_PM_TRACE_BUF_MAX_COUNT 0x400 - -/* Macros for the pm07_control registers. */ -#define CBE_PM_CTR_INPUT_MUX(pm07_control) (((pm07_control) >> 26) & 0x3f) -#define CBE_PM_CTR_INPUT_CONTROL 0x02000000 -#define CBE_PM_CTR_POLARITY 0x01000000 -#define CBE_PM_CTR_COUNT_CYCLES 0x00800000 -#define CBE_PM_CTR_ENABLE 0x00400000 -#define PM07_CTR_INPUT_MUX(x) (((x) & 0x3F) << 26) -#define PM07_CTR_INPUT_CONTROL(x) (((x) & 1) << 25) -#define PM07_CTR_POLARITY(x) (((x) & 1) << 24) -#define PM07_CTR_COUNT_CYCLES(x) (((x) & 1) << 23) -#define PM07_CTR_ENABLE(x) (((x) & 1) << 22) - -/* Macros for the pm_status register. */ -#define CBE_PM_CTR_OVERFLOW_INTR(ctr) (1 << (31 - ((ctr) & 7))) enum pm_reg_name { group_control, @@ -75,33 +35,4 @@ enum pm_reg_name { pm_start_stop, }; -/* Routines for reading/writing the PMU registers. */ -extern u32 cbe_read_phys_ctr(u32 cpu, u32 phys_ctr); -extern void cbe_write_phys_ctr(u32 cpu, u32 phys_ctr, u32 val); -extern u32 cbe_read_ctr(u32 cpu, u32 ctr); -extern void cbe_write_ctr(u32 cpu, u32 ctr, u32 val); - -extern u32 cbe_read_pm07_control(u32 cpu, u32 ctr); -extern void cbe_write_pm07_control(u32 cpu, u32 ctr, u32 val); -extern u32 cbe_read_pm(u32 cpu, enum pm_reg_name reg); -extern void cbe_write_pm(u32 cpu, enum pm_reg_name reg, u32 val); - -extern u32 cbe_get_ctr_size(u32 cpu, u32 phys_ctr); -extern void cbe_set_ctr_size(u32 cpu, u32 phys_ctr, u32 ctr_size); - -extern void cbe_enable_pm(u32 cpu); -extern void cbe_disable_pm(u32 cpu); - -extern void cbe_read_trace_buffer(u32 cpu, u64 *buf); - -extern void cbe_enable_pm_interrupts(u32 cpu, u32 thread, u32 mask); -extern void cbe_disable_pm_interrupts(u32 cpu); -extern u32 cbe_get_and_clear_pm_interrupts(u32 cpu); -extern void cbe_sync_irq(int node); - -#define CBE_COUNT_SUPERVISOR_MODE 0 -#define CBE_COUNT_HYPERVISOR_MODE 1 -#define CBE_COUNT_PROBLEM_MODE 2 -#define CBE_COUNT_ALL_MODES 3 - #endif /* __ASM_CELL_PMU_H__ */ diff --git a/arch/powerpc/include/asm/cell-regs.h b/arch/powerpc/include/asm/cell-regs.h index fdf64fd25950..20f7339a3d4a 100644 --- a/arch/powerpc/include/asm/cell-regs.h +++ b/arch/powerpc/include/asm/cell-regs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * cbe_regs.h * @@ -17,293 +18,6 @@ #include <asm/cell-pmu.h> -/* - * - * Some HID register definitions - * - */ - -/* CBE specific HID0 bits */ -#define HID0_CBE_THERM_WAKEUP 0x0000020000000000ul -#define HID0_CBE_SYSERR_WAKEUP 0x0000008000000000ul -#define HID0_CBE_THERM_INT_EN 0x0000000400000000ul -#define HID0_CBE_SYSERR_INT_EN 0x0000000200000000ul - -#define MAX_CBE 2 - -/* - * - * Pervasive unit register definitions - * - */ - -union spe_reg { - u64 val; - u8 spe[8]; -}; - -union ppe_spe_reg { - u64 val; - struct { - u32 ppe; - u32 spe; - }; -}; - - -struct cbe_pmd_regs { - /* Debug Bus Control */ - u64 pad_0x0000; /* 0x0000 */ - - u64 group_control; /* 0x0008 */ - - u8 pad_0x0010_0x00a8 [0x00a8 - 0x0010]; /* 0x0010 */ - - u64 debug_bus_control; /* 0x00a8 */ - - u8 pad_0x00b0_0x0100 [0x0100 - 0x00b0]; /* 0x00b0 */ - - u64 trace_aux_data; /* 0x0100 */ - u64 trace_buffer_0_63; /* 0x0108 */ - u64 trace_buffer_64_127; /* 0x0110 */ - u64 trace_address; /* 0x0118 */ - u64 ext_tr_timer; /* 0x0120 */ - - u8 pad_0x0128_0x0400 [0x0400 - 0x0128]; /* 0x0128 */ - - /* Performance Monitor */ - u64 pm_status; /* 0x0400 */ - u64 pm_control; /* 0x0408 */ - u64 pm_interval; /* 0x0410 */ - u64 pm_ctr[4]; /* 0x0418 */ - u64 pm_start_stop; /* 0x0438 */ - u64 pm07_control[8]; /* 0x0440 */ - - u8 pad_0x0480_0x0800 [0x0800 - 0x0480]; /* 0x0480 */ - - /* Thermal Sensor Registers */ - union spe_reg ts_ctsr1; /* 0x0800 */ - u64 ts_ctsr2; /* 0x0808 */ - union spe_reg ts_mtsr1; /* 0x0810 */ - u64 ts_mtsr2; /* 0x0818 */ - union spe_reg ts_itr1; /* 0x0820 */ - u64 ts_itr2; /* 0x0828 */ - u64 ts_gitr; /* 0x0830 */ - u64 ts_isr; /* 0x0838 */ - u64 ts_imr; /* 0x0840 */ - union spe_reg tm_cr1; /* 0x0848 */ - u64 tm_cr2; /* 0x0850 */ - u64 tm_simr; /* 0x0858 */ - union ppe_spe_reg tm_tpr; /* 0x0860 */ - union spe_reg tm_str1; /* 0x0868 */ - u64 tm_str2; /* 0x0870 */ - union ppe_spe_reg tm_tsr; /* 0x0878 */ - - /* Power Management */ - u64 pmcr; /* 0x0880 */ -#define CBE_PMD_PAUSE_ZERO_CONTROL 0x10000 - u64 pmsr; /* 0x0888 */ - - /* Time Base Register */ - u64 tbr; /* 0x0890 */ - - u8 pad_0x0898_0x0c00 [0x0c00 - 0x0898]; /* 0x0898 */ - - /* Fault Isolation Registers */ - u64 checkstop_fir; /* 0x0c00 */ - u64 recoverable_fir; /* 0x0c08 */ - u64 spec_att_mchk_fir; /* 0x0c10 */ - u32 fir_mode_reg; /* 0x0c18 */ - u8 pad_0x0c1c_0x0c20 [4]; /* 0x0c1c */ -#define CBE_PMD_FIR_MODE_M8 0x00800 - u64 fir_enable_mask; /* 0x0c20 */ - - u8 pad_0x0c28_0x0ca8 [0x0ca8 - 0x0c28]; /* 0x0c28 */ - u64 ras_esc_0; /* 0x0ca8 */ - u8 pad_0x0cb0_0x1000 [0x1000 - 0x0cb0]; /* 0x0cb0 */ -}; - -extern struct cbe_pmd_regs __iomem *cbe_get_pmd_regs(struct device_node *np); -extern struct cbe_pmd_regs __iomem *cbe_get_cpu_pmd_regs(int cpu); - -/* - * PMU shadow registers - * - * Many of the registers in the performance monitoring unit are write-only, - * so we need to save a copy of what we write to those registers. - * - * The actual data counters are read/write. However, writing to the counters - * only takes effect if the PMU is enabled. Otherwise the value is stored in - * a hardware latch until the next time the PMU is enabled. So we save a copy - * of the counter values if we need to read them back while the PMU is - * disabled. The counter_value_in_latch field is a bitmap indicating which - * counters currently have a value waiting to be written. - */ - -struct cbe_pmd_shadow_regs { - u32 group_control; - u32 debug_bus_control; - u32 trace_address; - u32 ext_tr_timer; - u32 pm_status; - u32 pm_control; - u32 pm_interval; - u32 pm_start_stop; - u32 pm07_control[NR_CTRS]; - - u32 pm_ctr[NR_PHYS_CTRS]; - u32 counter_value_in_latch; -}; - -extern struct cbe_pmd_shadow_regs *cbe_get_pmd_shadow_regs(struct device_node *np); -extern struct cbe_pmd_shadow_regs *cbe_get_cpu_pmd_shadow_regs(int cpu); - -/* - * - * IIC unit register definitions - * - */ - -struct cbe_iic_pending_bits { - u32 data; - u8 flags; - u8 class; - u8 source; - u8 prio; -}; - -#define CBE_IIC_IRQ_VALID 0x80 -#define CBE_IIC_IRQ_IPI 0x40 - -struct cbe_iic_thread_regs { - struct cbe_iic_pending_bits pending; - struct cbe_iic_pending_bits pending_destr; - u64 generate; - u64 prio; -}; - -struct cbe_iic_regs { - u8 pad_0x0000_0x0400[0x0400 - 0x0000]; /* 0x0000 */ - - /* IIC interrupt registers */ - struct cbe_iic_thread_regs thread[2]; /* 0x0400 */ - - u64 iic_ir; /* 0x0440 */ -#define CBE_IIC_IR_PRIO(x) (((x) & 0xf) << 12) -#define CBE_IIC_IR_DEST_NODE(x) (((x) & 0xf) << 4) -#define CBE_IIC_IR_DEST_UNIT(x) ((x) & 0xf) -#define CBE_IIC_IR_IOC_0 0x0 -#define CBE_IIC_IR_IOC_1S 0xb -#define CBE_IIC_IR_PT_0 0xe -#define CBE_IIC_IR_PT_1 0xf - - u64 iic_is; /* 0x0448 */ -#define CBE_IIC_IS_PMI 0x2 - - u8 pad_0x0450_0x0500[0x0500 - 0x0450]; /* 0x0450 */ - - /* IOC FIR */ - u64 ioc_fir_reset; /* 0x0500 */ - u64 ioc_fir_set; /* 0x0508 */ - u64 ioc_checkstop_enable; /* 0x0510 */ - u64 ioc_fir_error_mask; /* 0x0518 */ - u64 ioc_syserr_enable; /* 0x0520 */ - u64 ioc_fir; /* 0x0528 */ - - u8 pad_0x0530_0x1000[0x1000 - 0x0530]; /* 0x0530 */ -}; - -extern struct cbe_iic_regs __iomem *cbe_get_iic_regs(struct device_node *np); -extern struct cbe_iic_regs __iomem *cbe_get_cpu_iic_regs(int cpu); - - -struct cbe_mic_tm_regs { - u8 pad_0x0000_0x0040[0x0040 - 0x0000]; /* 0x0000 */ - - u64 mic_ctl_cnfg2; /* 0x0040 */ -#define CBE_MIC_ENABLE_AUX_TRC 0x8000000000000000LL -#define CBE_MIC_DISABLE_PWR_SAV_2 0x0200000000000000LL -#define CBE_MIC_DISABLE_AUX_TRC_WRAP 0x0100000000000000LL -#define CBE_MIC_ENABLE_AUX_TRC_INT 0x0080000000000000LL - - u64 pad_0x0048; /* 0x0048 */ - - u64 mic_aux_trc_base; /* 0x0050 */ - u64 mic_aux_trc_max_addr; /* 0x0058 */ - u64 mic_aux_trc_cur_addr; /* 0x0060 */ - u64 mic_aux_trc_grf_addr; /* 0x0068 */ - u64 mic_aux_trc_grf_data; /* 0x0070 */ - - u64 pad_0x0078; /* 0x0078 */ - - u64 mic_ctl_cnfg_0; /* 0x0080 */ -#define CBE_MIC_DISABLE_PWR_SAV_0 0x8000000000000000LL - - u64 pad_0x0088; /* 0x0088 */ - - u64 slow_fast_timer_0; /* 0x0090 */ - u64 slow_next_timer_0; /* 0x0098 */ - - u8 pad_0x00a0_0x00f8[0x00f8 - 0x00a0]; /* 0x00a0 */ - u64 mic_df_ecc_address_0; /* 0x00f8 */ - - u8 pad_0x0100_0x01b8[0x01b8 - 0x0100]; /* 0x0100 */ - u64 mic_df_ecc_address_1; /* 0x01b8 */ - - u64 mic_ctl_cnfg_1; /* 0x01c0 */ -#define CBE_MIC_DISABLE_PWR_SAV_1 0x8000000000000000LL - - u64 pad_0x01c8; /* 0x01c8 */ - - u64 slow_fast_timer_1; /* 0x01d0 */ - u64 slow_next_timer_1; /* 0x01d8 */ - - u8 pad_0x01e0_0x0208[0x0208 - 0x01e0]; /* 0x01e0 */ - u64 mic_exc; /* 0x0208 */ -#define CBE_MIC_EXC_BLOCK_SCRUB 0x0800000000000000ULL -#define CBE_MIC_EXC_FAST_SCRUB 0x0100000000000000ULL - - u64 mic_mnt_cfg; /* 0x0210 */ -#define CBE_MIC_MNT_CFG_CHAN_0_POP 0x0002000000000000ULL -#define CBE_MIC_MNT_CFG_CHAN_1_POP 0x0004000000000000ULL - - u64 mic_df_config; /* 0x0218 */ -#define CBE_MIC_ECC_DISABLE_0 0x4000000000000000ULL -#define CBE_MIC_ECC_REP_SINGLE_0 0x2000000000000000ULL -#define CBE_MIC_ECC_DISABLE_1 0x0080000000000000ULL -#define CBE_MIC_ECC_REP_SINGLE_1 0x0040000000000000ULL - - u8 pad_0x0220_0x0230[0x0230 - 0x0220]; /* 0x0220 */ - u64 mic_fir; /* 0x0230 */ -#define CBE_MIC_FIR_ECC_SINGLE_0_ERR 0x0200000000000000ULL -#define CBE_MIC_FIR_ECC_MULTI_0_ERR 0x0100000000000000ULL -#define CBE_MIC_FIR_ECC_SINGLE_1_ERR 0x0080000000000000ULL -#define CBE_MIC_FIR_ECC_MULTI_1_ERR 0x0040000000000000ULL -#define CBE_MIC_FIR_ECC_ERR_MASK 0xffff000000000000ULL -#define CBE_MIC_FIR_ECC_SINGLE_0_CTE 0x0000020000000000ULL -#define CBE_MIC_FIR_ECC_MULTI_0_CTE 0x0000010000000000ULL -#define CBE_MIC_FIR_ECC_SINGLE_1_CTE 0x0000008000000000ULL -#define CBE_MIC_FIR_ECC_MULTI_1_CTE 0x0000004000000000ULL -#define CBE_MIC_FIR_ECC_CTE_MASK 0x0000ffff00000000ULL -#define CBE_MIC_FIR_ECC_SINGLE_0_RESET 0x0000000002000000ULL -#define CBE_MIC_FIR_ECC_MULTI_0_RESET 0x0000000001000000ULL -#define CBE_MIC_FIR_ECC_SINGLE_1_RESET 0x0000000000800000ULL -#define CBE_MIC_FIR_ECC_MULTI_1_RESET 0x0000000000400000ULL -#define CBE_MIC_FIR_ECC_RESET_MASK 0x00000000ffff0000ULL -#define CBE_MIC_FIR_ECC_SINGLE_0_SET 0x0000000000000200ULL -#define CBE_MIC_FIR_ECC_MULTI_0_SET 0x0000000000000100ULL -#define CBE_MIC_FIR_ECC_SINGLE_1_SET 0x0000000000000080ULL -#define CBE_MIC_FIR_ECC_MULTI_1_SET 0x0000000000000040ULL -#define CBE_MIC_FIR_ECC_SET_MASK 0x000000000000ffffULL - u64 mic_fir_debug; /* 0x0238 */ - - u8 pad_0x0240_0x1000[0x1000 - 0x0240]; /* 0x0240 */ -}; - -extern struct cbe_mic_tm_regs __iomem *cbe_get_mic_tm_regs(struct device_node *np); -extern struct cbe_mic_tm_regs __iomem *cbe_get_cpu_mic_tm_regs(int cpu); - - /* Cell page table entries */ #define CBE_IOPTE_PP_W 0x8000000000000000ul /* protection: write */ #define CBE_IOPTE_PP_R 0x4000000000000000ul /* protection: read */ @@ -314,13 +28,4 @@ extern struct cbe_mic_tm_regs __iomem *cbe_get_cpu_mic_tm_regs(int cpu); #define CBE_IOPTE_H 0x0000000000000800ul /* cache hint */ #define CBE_IOPTE_IOID_Mask 0x00000000000007fful /* ioid */ -/* some utility functions to deal with SMT */ -extern u32 cbe_get_hw_thread_id(int cpu); -extern u32 cbe_cpu_to_node(int cpu); -extern u32 cbe_node_to_cpu(int node); - -/* Init this module early */ -extern void cbe_regs_init(void); - - #endif /* CBE_REGS_H */ diff --git a/arch/powerpc/include/asm/checksum.h b/arch/powerpc/include/asm/checksum.h index ce0c28495f9a..4b573a3b7e17 100644 --- a/arch/powerpc/include/asm/checksum.h +++ b/arch/powerpc/include/asm/checksum.h @@ -1,44 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _ASM_POWERPC_CHECKSUM_H #define _ASM_POWERPC_CHECKSUM_H #ifdef __KERNEL__ /* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ -/* - * This is a version of ip_compute_csum() optimized for IP headers, - * which always checksum on 4 octet boundaries. ihl is the number - * of 32-bit words and is always >= 5. - */ -extern __sum16 ip_fast_csum(const void *iph, unsigned int ihl); - -/* - * computes the checksum of the TCP/UDP pseudo-header - * returns a 16-bit checksum, already complemented - */ -extern __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, - unsigned short len, - unsigned short proto, - __wsum sum); - -/* - * computes the checksum of a memory block at buff, length len, - * and adds in "sum" (32-bit) - * - * returns a 32-bit number suitable for feeding into itself - * or csum_tcpudp_magic - * - * this function must be called with even lengths, except - * for the last fragment, which may be odd - * - * it's best to have buff aligned on a 32-bit boundary - */ -extern __wsum csum_partial(const void *buff, int len, __wsum sum); - +#include <linux/bitops.h> +#include <linux/in6.h> /* * Computes the checksum of a memory block at src, length len, * and adds in "sum" (32-bit), while copying the block to dst. @@ -49,28 +18,18 @@ extern __wsum csum_partial(const void *buff, int len, __wsum sum); * Like csum_partial, this must be called with even lengths, * except for the last fragment. */ -extern __wsum csum_partial_copy_generic(const void *src, void *dst, - int len, __wsum sum, - int *src_err, int *dst_err); +extern __wsum csum_partial_copy_generic(const void *src, void *dst, int len); -#ifdef __powerpc64__ #define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER extern __wsum csum_and_copy_from_user(const void __user *src, void *dst, - int len, __wsum sum, int *err_ptr); + int len); #define HAVE_CSUM_COPY_USER extern __wsum csum_and_copy_to_user(const void *src, void __user *dst, - int len, __wsum sum, int *err_ptr); -#else -/* - * the same as csum_partial, but copies from src to dst while it - * checksums. - */ -#define csum_partial_copy_from_user(src, dst, len, sum, errp) \ - csum_partial_copy_generic((__force const void *)(src), (dst), (len), (sum), (errp), NULL) -#endif + int len); -#define csum_partial_copy_nocheck(src, dst, len, sum) \ - csum_partial_copy_generic((src), (dst), (len), (sum), NULL, NULL) +#define _HAVE_ARCH_CSUM_AND_COPY +#define csum_partial_copy_nocheck(src, dst, len) \ + csum_partial_copy_generic((src), (dst), (len)) /* @@ -79,38 +38,36 @@ extern __wsum csum_and_copy_to_user(const void *src, void __user *dst, */ static inline __sum16 csum_fold(__wsum sum) { - unsigned int tmp; - - /* swap the two 16-bit halves of sum */ - __asm__("rlwinm %0,%1,16,0,31" : "=r" (tmp) : "r" (sum)); - /* if there is a carry from adding the two 16-bit halves, - it will carry from the lower half into the upper half, - giving us the correct sum in the upper half. */ - return (__force __sum16)(~((__force u32)sum + tmp) >> 16); + u32 tmp = (__force u32)sum; + + /* + * swap the two 16-bit halves of sum + * if there is a carry from adding the two 16-bit halves, + * it will carry from the lower half into the upper half, + * giving us the correct sum in the upper half. + */ + return (__force __sum16)(~(tmp + rol32(tmp, 16)) >> 16); } -/* - * this routine is used for miscellaneous IP-like checksums, mainly - * in icmp.c - */ -static inline __sum16 ip_compute_csum(const void *buff, int len) +static inline u32 from64to32(u64 x) { - return csum_fold(csum_partial(buff, len, 0)); + return (x + ror64(x, 32)) >> 32; } -static inline __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, - unsigned short len, - unsigned short proto, - __wsum sum) +static inline __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len, + __u8 proto, __wsum sum) { #ifdef __powerpc64__ - unsigned long s = (__force u32)sum; + u64 s = (__force u32)sum; s += (__force u32)saddr; s += (__force u32)daddr; +#ifdef __BIG_ENDIAN__ s += proto + len; - s += (s >> 32); - return (__force __wsum) s; +#else + s += (proto + len) << 8; +#endif + return (__force __wsum) from64to32(s); #else __asm__("\n\ addc %0,%0,%1 \n\ @@ -123,5 +80,142 @@ static inline __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, return sum; #endif } + +/* + * computes the checksum of the TCP/UDP pseudo-header + * returns a 16-bit checksum, already complemented + */ +static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, __u32 len, + __u8 proto, __wsum sum) +{ + return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum)); +} + +#define HAVE_ARCH_CSUM_ADD +static __always_inline __wsum csum_add(__wsum csum, __wsum addend) +{ +#ifdef __powerpc64__ + u64 res = (__force u64)csum; + + res += (__force u64)addend; + return (__force __wsum)((u32)res + (res >> 32)); +#else + if (__builtin_constant_p(csum) && csum == 0) + return addend; + if (__builtin_constant_p(addend) && addend == 0) + return csum; + + asm("addc %0,%0,%1;" + "addze %0,%0;" + : "+r" (csum) : "r" (addend) : "xer"); + return csum; +#endif +} + +#define HAVE_ARCH_CSUM_SHIFT +static __always_inline __wsum csum_shift(__wsum sum, int offset) +{ + /* rotate sum to align it with a 16b boundary */ + return (__force __wsum)rol32((__force u32)sum, (offset & 1) << 3); +} + +/* + * This is a version of ip_compute_csum() optimized for IP headers, + * which always checksum on 4 octet boundaries. ihl is the number + * of 32-bit words and is always >= 5. + */ +static inline __wsum ip_fast_csum_nofold(const void *iph, unsigned int ihl) +{ + const u32 *ptr = (const u32 *)iph + 1; +#ifdef __powerpc64__ + unsigned int i; + u64 s = *(const u32 *)iph; + + for (i = 0; i < ihl - 1; i++, ptr++) + s += *ptr; + return (__force __wsum)from64to32(s); +#else + __wsum sum, tmp; + + asm("mtctr %3;" + "addc %0,%4,%5;" + "1: lwzu %1, 4(%2);" + "adde %0,%0,%1;" + "bdnz 1b;" + "addze %0,%0;" + : "=r" (sum), "=r" (tmp), "+b" (ptr) + : "r" (ihl - 2), "r" (*(const u32 *)iph), "r" (*ptr) + : "ctr", "xer", "memory"); + + return sum; +#endif +} + +static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) +{ + return csum_fold(ip_fast_csum_nofold(iph, ihl)); +} + +/* + * computes the checksum of a memory block at buff, length len, + * and adds in "sum" (32-bit) + * + * returns a 32-bit number suitable for feeding into itself + * or csum_tcpudp_magic + * + * this function must be called with even lengths, except + * for the last fragment, which may be odd + * + * it's best to have buff aligned on a 32-bit boundary + */ +__wsum __csum_partial(const void *buff, int len, __wsum sum); + +static __always_inline __wsum csum_partial(const void *buff, int len, __wsum sum) +{ + if (__builtin_constant_p(len) && len <= 16 && (len & 1) == 0) { + if (len == 2) + sum = csum_add(sum, (__force __wsum)*(const u16 *)buff); + if (len >= 4) + sum = csum_add(sum, (__force __wsum)*(const u32 *)buff); + if (len == 6) + sum = csum_add(sum, (__force __wsum) + *(const u16 *)(buff + 4)); + if (len >= 8) + sum = csum_add(sum, (__force __wsum) + *(const u32 *)(buff + 4)); + if (len == 10) + sum = csum_add(sum, (__force __wsum) + *(const u16 *)(buff + 8)); + if (len >= 12) + sum = csum_add(sum, (__force __wsum) + *(const u32 *)(buff + 8)); + if (len == 14) + sum = csum_add(sum, (__force __wsum) + *(const u16 *)(buff + 12)); + if (len >= 16) + sum = csum_add(sum, (__force __wsum) + *(const u32 *)(buff + 12)); + } else if (__builtin_constant_p(len) && (len & 3) == 0) { + sum = csum_add(sum, ip_fast_csum_nofold(buff, len >> 2)); + } else { + sum = __csum_partial(buff, len, sum); + } + return sum; +} + +/* + * this routine is used for miscellaneous IP-like checksums, mainly + * in icmp.c + */ +static inline __sum16 ip_compute_csum(const void *buff, int len) +{ + return csum_fold(csum_partial(buff, len, 0)); +} + +#define _HAVE_ARCH_IPV6_CSUM +__sum16 csum_ipv6_magic(const struct in6_addr *saddr, + const struct in6_addr *daddr, + __u32 len, __u8 proto, __wsum sum); + #endif /* __KERNEL__ */ #endif diff --git a/arch/powerpc/include/asm/clk_interface.h b/arch/powerpc/include/asm/clk_interface.h deleted file mode 100644 index ab1882c1e176..000000000000 --- a/arch/powerpc/include/asm/clk_interface.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef __ASM_POWERPC_CLK_INTERFACE_H -#define __ASM_POWERPC_CLK_INTERFACE_H - -#include <linux/clk.h> - -struct clk_interface { - struct clk* (*clk_get) (struct device *dev, const char *id); - int (*clk_enable) (struct clk *clk); - void (*clk_disable) (struct clk *clk); - unsigned long (*clk_get_rate) (struct clk *clk); - void (*clk_put) (struct clk *clk); - long (*clk_round_rate) (struct clk *clk, unsigned long rate); - int (*clk_set_rate) (struct clk *clk, unsigned long rate); - int (*clk_set_parent) (struct clk *clk, struct clk *parent); - struct clk* (*clk_get_parent) (struct clk *clk); -}; - -extern struct clk_interface clk_functions; - -#endif /* __ASM_POWERPC_CLK_INTERFACE_H */ diff --git a/arch/powerpc/include/asm/clocksource.h b/arch/powerpc/include/asm/clocksource.h new file mode 100644 index 000000000000..0a26ef13a34a --- /dev/null +++ b/arch/powerpc/include/asm/clocksource.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_CLOCKSOURCE_H +#define _ASM_POWERPC_CLOCKSOURCE_H + +#include <asm/vdso/clocksource.h> + +#endif /* _ASM_POWERPC_CLOCKSOURCE_H */ diff --git a/arch/powerpc/include/asm/cmpxchg.h b/arch/powerpc/include/asm/cmpxchg.h index e245aab7f191..dbb50c06f0bf 100644 --- a/arch/powerpc/include/asm/cmpxchg.h +++ b/arch/powerpc/include/asm/cmpxchg.h @@ -1,42 +1,153 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_CMPXCHG_H_ #define _ASM_POWERPC_CMPXCHG_H_ #ifdef __KERNEL__ #include <linux/compiler.h> #include <asm/synch.h> -#include <asm/asm-compat.h> +#include <linux/bug.h> + +#ifdef __BIG_ENDIAN +#define BITOFF_CAL(size, off) ((sizeof(u32) - size - off) * BITS_PER_BYTE) +#else +#define BITOFF_CAL(size, off) (off * BITS_PER_BYTE) +#endif + +#define XCHG_GEN(type, sfx, cl) \ +static inline u32 __xchg_##type##sfx(volatile void *p, u32 val) \ +{ \ + unsigned int prev, prev_mask, tmp, bitoff, off; \ + \ + off = (unsigned long)p % sizeof(u32); \ + bitoff = BITOFF_CAL(sizeof(type), off); \ + p -= off; \ + val <<= bitoff; \ + prev_mask = (u32)(type)-1 << bitoff; \ + \ + __asm__ __volatile__( \ +"1: lwarx %0,0,%3\n" \ +" andc %1,%0,%5\n" \ +" or %1,%1,%4\n" \ +" stwcx. %1,0,%3\n" \ +" bne- 1b\n" \ + : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p) \ + : "r" (p), "r" (val), "r" (prev_mask) \ + : "cc", cl); \ + \ + return prev >> bitoff; \ +} + +#define CMPXCHG_GEN(type, sfx, br, br2, cl) \ +static inline \ +u32 __cmpxchg_##type##sfx(volatile void *p, u32 old, u32 new) \ +{ \ + unsigned int prev, prev_mask, tmp, bitoff, off; \ + \ + off = (unsigned long)p % sizeof(u32); \ + bitoff = BITOFF_CAL(sizeof(type), off); \ + p -= off; \ + old <<= bitoff; \ + new <<= bitoff; \ + prev_mask = (u32)(type)-1 << bitoff; \ + \ + __asm__ __volatile__( \ + br \ +"1: lwarx %0,0,%3\n" \ +" and %1,%0,%6\n" \ +" cmpw 0,%1,%4\n" \ +" bne- 2f\n" \ +" andc %1,%0,%6\n" \ +" or %1,%1,%5\n" \ +" stwcx. %1,0,%3\n" \ +" bne- 1b\n" \ + br2 \ + "\n" \ +"2:" \ + : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p) \ + : "r" (p), "r" (old), "r" (new), "r" (prev_mask) \ + : "cc", cl); \ + \ + return prev >> bitoff; \ +} /* * Atomic exchange * - * Changes the memory location '*ptr' to be val and returns + * Changes the memory location '*p' to be val and returns * the previous value stored there. */ + +#ifndef CONFIG_PPC_HAS_LBARX_LHARX +XCHG_GEN(u8, _local, "memory"); +XCHG_GEN(u8, _relaxed, "cc"); +XCHG_GEN(u16, _local, "memory"); +XCHG_GEN(u16, _relaxed, "cc"); +#else static __always_inline unsigned long -__xchg_u32(volatile void *p, unsigned long val) +__xchg_u8_local(volatile void *p, unsigned long val) { unsigned long prev; __asm__ __volatile__( - PPC_RELEASE_BARRIER -"1: lwarx %0,0,%2 \n" - PPC405_ERR77(0,%2) -" stwcx. %3,0,%2 \n\ - bne- 1b" - PPC_ACQUIRE_BARRIER - : "=&r" (prev), "+m" (*(volatile unsigned int *)p) +"1: lbarx %0,0,%2 # __xchg_u8_local\n" +" stbcx. %3,0,%2 \n" +" bne- 1b" + : "=&r" (prev), "+m" (*(volatile unsigned char *)p) + : "r" (p), "r" (val) + : "cc", "memory"); + + return prev; +} + +static __always_inline unsigned long +__xchg_u8_relaxed(u8 *p, unsigned long val) +{ + unsigned long prev; + + __asm__ __volatile__( +"1: lbarx %0,0,%2 # __xchg_u8_relaxed\n" +" stbcx. %3,0,%2\n" +" bne- 1b" + : "=&r" (prev), "+m" (*p) + : "r" (p), "r" (val) + : "cc"); + + return prev; +} + +static __always_inline unsigned long +__xchg_u16_local(volatile void *p, unsigned long val) +{ + unsigned long prev; + + __asm__ __volatile__( +"1: lharx %0,0,%2 # __xchg_u16_local\n" +" sthcx. %3,0,%2\n" +" bne- 1b" + : "=&r" (prev), "+m" (*(volatile unsigned short *)p) : "r" (p), "r" (val) : "cc", "memory"); return prev; } -/* - * Atomic exchange - * - * Changes the memory location '*ptr' to be val and returns - * the previous value stored there. - */ +static __always_inline unsigned long +__xchg_u16_relaxed(u16 *p, unsigned long val) +{ + unsigned long prev; + + __asm__ __volatile__( +"1: lharx %0,0,%2 # __xchg_u16_relaxed\n" +" sthcx. %3,0,%2\n" +" bne- 1b" + : "=&r" (prev), "+m" (*p) + : "r" (p), "r" (val) + : "cc"); + + return prev; +} +#endif + static __always_inline unsigned long __xchg_u32_local(volatile void *p, unsigned long val) { @@ -44,7 +155,6 @@ __xchg_u32_local(volatile void *p, unsigned long val) __asm__ __volatile__( "1: lwarx %0,0,%2 \n" - PPC405_ERR77(0,%2) " stwcx. %3,0,%2 \n\ bne- 1b" : "=&r" (prev), "+m" (*(volatile unsigned int *)p) @@ -54,26 +164,23 @@ __xchg_u32_local(volatile void *p, unsigned long val) return prev; } -#ifdef CONFIG_PPC64 static __always_inline unsigned long -__xchg_u64(volatile void *p, unsigned long val) +__xchg_u32_relaxed(u32 *p, unsigned long val) { unsigned long prev; __asm__ __volatile__( - PPC_RELEASE_BARRIER -"1: ldarx %0,0,%2 \n" - PPC405_ERR77(0,%2) -" stdcx. %3,0,%2 \n\ - bne- 1b" - PPC_ACQUIRE_BARRIER - : "=&r" (prev), "+m" (*(volatile unsigned long *)p) +"1: lwarx %0,0,%2\n" +" stwcx. %3,0,%2\n" +" bne- 1b" + : "=&r" (prev), "+m" (*p) : "r" (p), "r" (val) - : "cc", "memory"); + : "cc"); return prev; } +#ifdef CONFIG_PPC64 static __always_inline unsigned long __xchg_u64_local(volatile void *p, unsigned long val) { @@ -81,7 +188,6 @@ __xchg_u64_local(volatile void *p, unsigned long val) __asm__ __volatile__( "1: ldarx %0,0,%2 \n" - PPC405_ERR77(0,%2) " stdcx. %3,0,%2 \n\ bne- 1b" : "=&r" (prev), "+m" (*(volatile unsigned long *)p) @@ -90,61 +196,250 @@ __xchg_u64_local(volatile void *p, unsigned long val) return prev; } -#endif -/* - * This function doesn't exist, so you'll get a linker error - * if something tries to do an invalid xchg(). - */ -extern void __xchg_called_with_bad_pointer(void); +static __always_inline unsigned long +__xchg_u64_relaxed(u64 *p, unsigned long val) +{ + unsigned long prev; + + __asm__ __volatile__( +"1: ldarx %0,0,%2\n" +" stdcx. %3,0,%2\n" +" bne- 1b" + : "=&r" (prev), "+m" (*p) + : "r" (p), "r" (val) + : "cc"); + + return prev; +} +#endif static __always_inline unsigned long -__xchg(volatile void *ptr, unsigned long x, unsigned int size) +__xchg_local(void *ptr, unsigned long x, unsigned int size) { switch (size) { + case 1: + return __xchg_u8_local(ptr, x); + case 2: + return __xchg_u16_local(ptr, x); case 4: - return __xchg_u32(ptr, x); + return __xchg_u32_local(ptr, x); #ifdef CONFIG_PPC64 case 8: - return __xchg_u64(ptr, x); + return __xchg_u64_local(ptr, x); #endif } - __xchg_called_with_bad_pointer(); + BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local"); return x; } static __always_inline unsigned long -__xchg_local(volatile void *ptr, unsigned long x, unsigned int size) +__xchg_relaxed(void *ptr, unsigned long x, unsigned int size) { switch (size) { + case 1: + return __xchg_u8_relaxed(ptr, x); + case 2: + return __xchg_u16_relaxed(ptr, x); case 4: - return __xchg_u32_local(ptr, x); + return __xchg_u32_relaxed(ptr, x); #ifdef CONFIG_PPC64 case 8: - return __xchg_u64_local(ptr, x); + return __xchg_u64_relaxed(ptr, x); #endif } - __xchg_called_with_bad_pointer(); + BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_relaxed"); return x; } -#define xchg(ptr,x) \ - ({ \ - __typeof__(*(ptr)) _x_ = (x); \ - (__typeof__(*(ptr))) __xchg((ptr), (unsigned long)_x_, sizeof(*(ptr))); \ - }) - -#define xchg_local(ptr,x) \ +#define arch_xchg_local(ptr,x) \ ({ \ __typeof__(*(ptr)) _x_ = (x); \ (__typeof__(*(ptr))) __xchg_local((ptr), \ (unsigned long)_x_, sizeof(*(ptr))); \ }) +#define arch_xchg_relaxed(ptr, x) \ +({ \ + __typeof__(*(ptr)) _x_ = (x); \ + (__typeof__(*(ptr))) __xchg_relaxed((ptr), \ + (unsigned long)_x_, sizeof(*(ptr))); \ +}) + /* * Compare and exchange - if *p == old, set it to new, * and return the old value of *p. */ -#define __HAVE_ARCH_CMPXCHG 1 +#ifndef CONFIG_PPC_HAS_LBARX_LHARX +CMPXCHG_GEN(u8, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory"); +CMPXCHG_GEN(u8, _local, , , "memory"); +CMPXCHG_GEN(u8, _acquire, , PPC_ACQUIRE_BARRIER, "memory"); +CMPXCHG_GEN(u8, _relaxed, , , "cc"); +CMPXCHG_GEN(u16, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory"); +CMPXCHG_GEN(u16, _local, , , "memory"); +CMPXCHG_GEN(u16, _acquire, , PPC_ACQUIRE_BARRIER, "memory"); +CMPXCHG_GEN(u16, _relaxed, , , "cc"); +#else +static __always_inline unsigned long +__cmpxchg_u8(volatile unsigned char *p, unsigned long old, unsigned long new) +{ + unsigned int prev; + + __asm__ __volatile__ ( + PPC_ATOMIC_ENTRY_BARRIER +"1: lbarx %0,0,%2 # __cmpxchg_u8\n" +" cmpw 0,%0,%3\n" +" bne- 2f\n" +" stbcx. %4,0,%2\n" +" bne- 1b" + PPC_ATOMIC_EXIT_BARRIER + "\n\ +2:" + : "=&r" (prev), "+m" (*p) + : "r" (p), "r" (old), "r" (new) + : "cc", "memory"); + + return prev; +} + +static __always_inline unsigned long +__cmpxchg_u8_local(volatile unsigned char *p, unsigned long old, + unsigned long new) +{ + unsigned int prev; + + __asm__ __volatile__ ( +"1: lbarx %0,0,%2 # __cmpxchg_u8_local\n" +" cmpw 0,%0,%3\n" +" bne- 2f\n" +" stbcx. %4,0,%2\n" +" bne- 1b\n" +"2:" + : "=&r" (prev), "+m" (*p) + : "r" (p), "r" (old), "r" (new) + : "cc", "memory"); + + return prev; +} + +static __always_inline unsigned long +__cmpxchg_u8_relaxed(u8 *p, unsigned long old, unsigned long new) +{ + unsigned long prev; + + __asm__ __volatile__ ( +"1: lbarx %0,0,%2 # __cmpxchg_u8_relaxed\n" +" cmpw 0,%0,%3\n" +" bne- 2f\n" +" stbcx. %4,0,%2\n" +" bne- 1b\n" +"2:" + : "=&r" (prev), "+m" (*p) + : "r" (p), "r" (old), "r" (new) + : "cc"); + + return prev; +} + +static __always_inline unsigned long +__cmpxchg_u8_acquire(u8 *p, unsigned long old, unsigned long new) +{ + unsigned long prev; + + __asm__ __volatile__ ( +"1: lbarx %0,0,%2 # __cmpxchg_u8_acquire\n" +" cmpw 0,%0,%3\n" +" bne- 2f\n" +" stbcx. %4,0,%2\n" +" bne- 1b\n" + PPC_ACQUIRE_BARRIER +"2:" + : "=&r" (prev), "+m" (*p) + : "r" (p), "r" (old), "r" (new) + : "cc", "memory"); + + return prev; +} + +static __always_inline unsigned long +__cmpxchg_u16(volatile unsigned short *p, unsigned long old, unsigned long new) +{ + unsigned int prev; + + __asm__ __volatile__ ( + PPC_ATOMIC_ENTRY_BARRIER +"1: lharx %0,0,%2 # __cmpxchg_u16\n" +" cmpw 0,%0,%3\n" +" bne- 2f\n" +" sthcx. %4,0,%2\n" +" bne- 1b\n" + PPC_ATOMIC_EXIT_BARRIER +"2:" + : "=&r" (prev), "+m" (*p) + : "r" (p), "r" (old), "r" (new) + : "cc", "memory"); + + return prev; +} + +static __always_inline unsigned long +__cmpxchg_u16_local(volatile unsigned short *p, unsigned long old, + unsigned long new) +{ + unsigned int prev; + + __asm__ __volatile__ ( +"1: lharx %0,0,%2 # __cmpxchg_u16_local\n" +" cmpw 0,%0,%3\n" +" bne- 2f\n" +" sthcx. %4,0,%2\n" +" bne- 1b" +"2:" + : "=&r" (prev), "+m" (*p) + : "r" (p), "r" (old), "r" (new) + : "cc", "memory"); + + return prev; +} + +static __always_inline unsigned long +__cmpxchg_u16_relaxed(u16 *p, unsigned long old, unsigned long new) +{ + unsigned long prev; + + __asm__ __volatile__ ( +"1: lharx %0,0,%2 # __cmpxchg_u16_relaxed\n" +" cmpw 0,%0,%3\n" +" bne- 2f\n" +" sthcx. %4,0,%2\n" +" bne- 1b\n" +"2:" + : "=&r" (prev), "+m" (*p) + : "r" (p), "r" (old), "r" (new) + : "cc"); + + return prev; +} + +static __always_inline unsigned long +__cmpxchg_u16_acquire(u16 *p, unsigned long old, unsigned long new) +{ + unsigned long prev; + + __asm__ __volatile__ ( +"1: lharx %0,0,%2 # __cmpxchg_u16_acquire\n" +" cmpw 0,%0,%3\n" +" bne- 2f\n" +" sthcx. %4,0,%2\n" +" bne- 1b\n" + PPC_ACQUIRE_BARRIER +"2:" + : "=&r" (prev), "+m" (*p) + : "r" (p), "r" (old), "r" (new) + : "cc", "memory"); + + return prev; +} +#endif static __always_inline unsigned long __cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new) @@ -152,14 +447,13 @@ __cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new) unsigned int prev; __asm__ __volatile__ ( - PPC_RELEASE_BARRIER + PPC_ATOMIC_ENTRY_BARRIER "1: lwarx %0,0,%2 # __cmpxchg_u32\n\ cmpw 0,%0,%3\n\ bne- 2f\n" - PPC405_ERR77(0,%2) " stwcx. %4,0,%2\n\ bne- 1b" - PPC_ACQUIRE_BARRIER + PPC_ATOMIC_EXIT_BARRIER "\n\ 2:" : "=&r" (prev), "+m" (*p) @@ -179,7 +473,6 @@ __cmpxchg_u32_local(volatile unsigned int *p, unsigned long old, "1: lwarx %0,0,%2 # __cmpxchg_u32\n\ cmpw 0,%0,%3\n\ bne- 2f\n" - PPC405_ERR77(0,%2) " stwcx. %4,0,%2\n\ bne- 1b" "\n\ @@ -191,6 +484,54 @@ __cmpxchg_u32_local(volatile unsigned int *p, unsigned long old, return prev; } +static __always_inline unsigned long +__cmpxchg_u32_relaxed(u32 *p, unsigned long old, unsigned long new) +{ + unsigned long prev; + + __asm__ __volatile__ ( +"1: lwarx %0,0,%2 # __cmpxchg_u32_relaxed\n" +" cmpw 0,%0,%3\n" +" bne- 2f\n" +" stwcx. %4,0,%2\n" +" bne- 1b\n" +"2:" + : "=&r" (prev), "+m" (*p) + : "r" (p), "r" (old), "r" (new) + : "cc"); + + return prev; +} + +/* + * cmpxchg family don't have order guarantee if cmp part fails, therefore we + * can avoid superfluous barriers if we use assembly code to implement + * cmpxchg() and cmpxchg_acquire(), however we don't do the similar for + * cmpxchg_release() because that will result in putting a barrier in the + * middle of a ll/sc loop, which is probably a bad idea. For example, this + * might cause the conditional store more likely to fail. + */ +static __always_inline unsigned long +__cmpxchg_u32_acquire(u32 *p, unsigned long old, unsigned long new) +{ + unsigned long prev; + + __asm__ __volatile__ ( +"1: lwarx %0,0,%2 # __cmpxchg_u32_acquire\n" +" cmpw 0,%0,%3\n" +" bne- 2f\n" +" stwcx. %4,0,%2\n" +" bne- 1b\n" + PPC_ACQUIRE_BARRIER + "\n" +"2:" + : "=&r" (prev), "+m" (*p) + : "r" (p), "r" (old), "r" (new) + : "cc", "memory"); + + return prev; +} + #ifdef CONFIG_PPC64 static __always_inline unsigned long __cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new) @@ -198,13 +539,13 @@ __cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new) unsigned long prev; __asm__ __volatile__ ( - PPC_RELEASE_BARRIER + PPC_ATOMIC_ENTRY_BARRIER "1: ldarx %0,0,%2 # __cmpxchg_u64\n\ cmpd 0,%0,%3\n\ bne- 2f\n\ stdcx. %4,0,%2\n\ bne- 1b" - PPC_ACQUIRE_BARRIER + PPC_ATOMIC_EXIT_BARRIER "\n\ 2:" : "=&r" (prev), "+m" (*p) @@ -234,17 +575,57 @@ __cmpxchg_u64_local(volatile unsigned long *p, unsigned long old, return prev; } -#endif -/* This function doesn't exist, so you'll get a linker error - if something tries to do an invalid cmpxchg(). */ -extern void __cmpxchg_called_with_bad_pointer(void); +static __always_inline unsigned long +__cmpxchg_u64_relaxed(u64 *p, unsigned long old, unsigned long new) +{ + unsigned long prev; + + __asm__ __volatile__ ( +"1: ldarx %0,0,%2 # __cmpxchg_u64_relaxed\n" +" cmpd 0,%0,%3\n" +" bne- 2f\n" +" stdcx. %4,0,%2\n" +" bne- 1b\n" +"2:" + : "=&r" (prev), "+m" (*p) + : "r" (p), "r" (old), "r" (new) + : "cc"); + + return prev; +} + +static __always_inline unsigned long +__cmpxchg_u64_acquire(u64 *p, unsigned long old, unsigned long new) +{ + unsigned long prev; + + __asm__ __volatile__ ( +"1: ldarx %0,0,%2 # __cmpxchg_u64_acquire\n" +" cmpd 0,%0,%3\n" +" bne- 2f\n" +" stdcx. %4,0,%2\n" +" bne- 1b\n" + PPC_ACQUIRE_BARRIER + "\n" +"2:" + : "=&r" (prev), "+m" (*p) + : "r" (p), "r" (old), "r" (new) + : "cc", "memory"); + + return prev; +} +#endif static __always_inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, unsigned int size) { switch (size) { + case 1: + return __cmpxchg_u8(ptr, old, new); + case 2: + return __cmpxchg_u16(ptr, old, new); case 4: return __cmpxchg_u32(ptr, old, new); #ifdef CONFIG_PPC64 @@ -252,15 +633,19 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, return __cmpxchg_u64(ptr, old, new); #endif } - __cmpxchg_called_with_bad_pointer(); + BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg"); return old; } static __always_inline unsigned long -__cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new, +__cmpxchg_local(void *ptr, unsigned long old, unsigned long new, unsigned int size) { switch (size) { + case 1: + return __cmpxchg_u8_local(ptr, old, new); + case 2: + return __cmpxchg_u16_local(ptr, old, new); case 4: return __cmpxchg_u32_local(ptr, old, new); #ifdef CONFIG_PPC64 @@ -268,11 +653,50 @@ __cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new, return __cmpxchg_u64_local(ptr, old, new); #endif } - __cmpxchg_called_with_bad_pointer(); + BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_local"); + return old; +} + +static __always_inline unsigned long +__cmpxchg_relaxed(void *ptr, unsigned long old, unsigned long new, + unsigned int size) +{ + switch (size) { + case 1: + return __cmpxchg_u8_relaxed(ptr, old, new); + case 2: + return __cmpxchg_u16_relaxed(ptr, old, new); + case 4: + return __cmpxchg_u32_relaxed(ptr, old, new); +#ifdef CONFIG_PPC64 + case 8: + return __cmpxchg_u64_relaxed(ptr, old, new); +#endif + } + BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_relaxed"); return old; } -#define cmpxchg(ptr, o, n) \ +static __always_inline unsigned long +__cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new, + unsigned int size) +{ + switch (size) { + case 1: + return __cmpxchg_u8_acquire(ptr, old, new); + case 2: + return __cmpxchg_u16_acquire(ptr, old, new); + case 4: + return __cmpxchg_u32_acquire(ptr, old, new); +#ifdef CONFIG_PPC64 + case 8: + return __cmpxchg_u64_acquire(ptr, old, new); +#endif + } + BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_acquire"); + return old; +} +#define arch_cmpxchg(ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ @@ -281,7 +705,7 @@ __cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new, }) -#define cmpxchg_local(ptr, o, n) \ +#define arch_cmpxchg_local(ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ @@ -289,20 +713,47 @@ __cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new, (unsigned long)_n_, sizeof(*(ptr))); \ }) +#define arch_cmpxchg_relaxed(ptr, o, n) \ +({ \ + __typeof__(*(ptr)) _o_ = (o); \ + __typeof__(*(ptr)) _n_ = (n); \ + (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \ + (unsigned long)_o_, (unsigned long)_n_, \ + sizeof(*(ptr))); \ +}) + +#define arch_cmpxchg_acquire(ptr, o, n) \ +({ \ + __typeof__(*(ptr)) _o_ = (o); \ + __typeof__(*(ptr)) _n_ = (n); \ + (__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \ + (unsigned long)_o_, (unsigned long)_n_, \ + sizeof(*(ptr))); \ +}) #ifdef CONFIG_PPC64 -#define cmpxchg64(ptr, o, n) \ +#define arch_cmpxchg64(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg((ptr), (o), (n)); \ + arch_cmpxchg((ptr), (o), (n)); \ }) -#define cmpxchg64_local(ptr, o, n) \ +#define arch_cmpxchg64_local(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg_local((ptr), (o), (n)); \ + arch_cmpxchg_local((ptr), (o), (n)); \ }) +#define arch_cmpxchg64_relaxed(ptr, o, n) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ + arch_cmpxchg_relaxed((ptr), (o), (n)); \ +}) +#define arch_cmpxchg64_acquire(ptr, o, n) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ + arch_cmpxchg_acquire((ptr), (o), (n)); \ +}) #else #include <asm-generic/cmpxchg-local.h> -#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) +#define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n)) #endif #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/code-patching-asm.h b/arch/powerpc/include/asm/code-patching-asm.h new file mode 100644 index 000000000000..ed7b1448493a --- /dev/null +++ b/arch/powerpc/include/asm/code-patching-asm.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright 2018, Michael Ellerman, IBM Corporation. + */ +#ifndef _ASM_POWERPC_CODE_PATCHING_ASM_H +#define _ASM_POWERPC_CODE_PATCHING_ASM_H + +/* Define a "site" that can be patched */ +.macro patch_site label name + .pushsection ".rodata" + .balign 4 + .global \name +\name: + .4byte \label - . + .popsection +.endm + +#endif /* _ASM_POWERPC_CODE_PATCHING_ASM_H */ diff --git a/arch/powerpc/include/asm/code-patching.h b/arch/powerpc/include/asm/code-patching.h deleted file mode 100644 index a6f8c7a5cbb7..000000000000 --- a/arch/powerpc/include/asm/code-patching.h +++ /dev/null @@ -1,52 +0,0 @@ -#ifndef _ASM_POWERPC_CODE_PATCHING_H -#define _ASM_POWERPC_CODE_PATCHING_H - -/* - * Copyright 2008, Michael Ellerman, IBM Corporation. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ - -#include <asm/types.h> -#include <asm/ppc-opcode.h> - -/* Flags for create_branch: - * "b" == create_branch(addr, target, 0); - * "ba" == create_branch(addr, target, BRANCH_ABSOLUTE); - * "bl" == create_branch(addr, target, BRANCH_SET_LINK); - * "bla" == create_branch(addr, target, BRANCH_ABSOLUTE | BRANCH_SET_LINK); - */ -#define BRANCH_SET_LINK 0x1 -#define BRANCH_ABSOLUTE 0x2 - -unsigned int create_branch(const unsigned int *addr, - unsigned long target, int flags); -unsigned int create_cond_branch(const unsigned int *addr, - unsigned long target, int flags); -int patch_branch(unsigned int *addr, unsigned long target, int flags); -int patch_instruction(unsigned int *addr, unsigned int instr); - -int instr_is_relative_branch(unsigned int instr); -int instr_is_branch_to_addr(const unsigned int *instr, unsigned long addr); -unsigned long branch_target(const unsigned int *instr); -unsigned int translate_branch(const unsigned int *dest, - const unsigned int *src); - -static inline unsigned long ppc_function_entry(void *func) -{ -#ifdef CONFIG_PPC64 - /* - * On PPC64 the function pointer actually points to the function's - * descriptor. The first entry in the descriptor is the address - * of the function text. - */ - return ((func_descr_t *)func)->entry; -#else - return (unsigned long)func; -#endif -} - -#endif /* _ASM_POWERPC_CODE_PATCHING_H */ diff --git a/arch/powerpc/include/asm/compat.h b/arch/powerpc/include/asm/compat.h index 84fdf6857c31..dda4091fd012 100644 --- a/arch/powerpc/include/asm/compat.h +++ b/arch/powerpc/include/asm/compat.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_COMPAT_H #define _ASM_POWERPC_COMPAT_H #ifdef __KERNEL__ @@ -7,48 +8,20 @@ #include <linux/types.h> #include <linux/sched.h> -#define COMPAT_USER_HZ 100 -#define COMPAT_UTS_MACHINE "ppc\0\0" - -typedef u32 compat_size_t; -typedef s32 compat_ssize_t; -typedef s32 compat_time_t; -typedef s32 compat_clock_t; -typedef s32 compat_pid_t; -typedef u32 __compat_uid_t; -typedef u32 __compat_gid_t; -typedef u32 __compat_uid32_t; -typedef u32 __compat_gid32_t; -typedef u32 compat_mode_t; -typedef u32 compat_ino_t; -typedef u32 compat_dev_t; -typedef s32 compat_off_t; -typedef s64 compat_loff_t; -typedef s16 compat_nlink_t; +#define compat_ipc_pid_t compat_ipc_pid_t typedef u16 compat_ipc_pid_t; -typedef s32 compat_daddr_t; -typedef u32 compat_caddr_t; -typedef __kernel_fsid_t compat_fsid_t; -typedef s32 compat_key_t; -typedef s32 compat_timer_t; -typedef s32 compat_int_t; -typedef s32 compat_long_t; -typedef s64 compat_s64; -typedef u32 compat_uint_t; -typedef u32 compat_ulong_t; -typedef u64 compat_u64; -typedef u32 compat_uptr_t; +#define compat_ipc64_perm compat_ipc64_perm -struct compat_timespec { - compat_time_t tv_sec; - s32 tv_nsec; -}; +#include <asm-generic/compat.h> -struct compat_timeval { - compat_time_t tv_sec; - s32 tv_usec; -}; +#ifdef __BIG_ENDIAN__ +#define COMPAT_UTS_MACHINE "ppc\0\0" +#else +#define COMPAT_UTS_MACHINE "ppcle\0\0" +#endif + +typedef s16 compat_nlink_t; struct compat_stat { compat_dev_t st_dev; @@ -61,153 +34,15 @@ struct compat_stat { compat_off_t st_size; compat_off_t st_blksize; compat_off_t st_blocks; - compat_time_t st_atime; + old_time32_t st_atime; u32 st_atime_nsec; - compat_time_t st_mtime; + old_time32_t st_mtime; u32 st_mtime_nsec; - compat_time_t st_ctime; + old_time32_t st_ctime; u32 st_ctime_nsec; u32 __unused4[2]; }; -struct compat_flock { - short l_type; - short l_whence; - compat_off_t l_start; - compat_off_t l_len; - compat_pid_t l_pid; -}; - -#define F_GETLK64 12 /* using 'struct flock64' */ -#define F_SETLK64 13 -#define F_SETLKW64 14 - -struct compat_flock64 { - short l_type; - short l_whence; - compat_loff_t l_start; - compat_loff_t l_len; - compat_pid_t l_pid; -}; - -struct compat_statfs { - int f_type; - int f_bsize; - int f_blocks; - int f_bfree; - int f_bavail; - int f_files; - int f_ffree; - compat_fsid_t f_fsid; - int f_namelen; /* SunOS ignores this field. */ - int f_frsize; - int f_flags; - int f_spare[4]; -}; - -#define COMPAT_RLIM_OLD_INFINITY 0x7fffffff -#define COMPAT_RLIM_INFINITY 0xffffffff - -typedef u32 compat_old_sigset_t; - -#define _COMPAT_NSIG 64 -#define _COMPAT_NSIG_BPW 32 - -typedef u32 compat_sigset_word; - -typedef union compat_sigval { - compat_int_t sival_int; - compat_uptr_t sival_ptr; -} compat_sigval_t; - -#define SI_PAD_SIZE32 (128/sizeof(int) - 3) - -typedef struct compat_siginfo { - int si_signo; - int si_errno; - int si_code; - - union { - int _pad[SI_PAD_SIZE32]; - - /* kill() */ - struct { - compat_pid_t _pid; /* sender's pid */ - __compat_uid_t _uid; /* sender's uid */ - } _kill; - - /* POSIX.1b timers */ - struct { - compat_timer_t _tid; /* timer id */ - int _overrun; /* overrun count */ - compat_sigval_t _sigval; /* same as below */ - int _sys_private; /* not to be passed to user */ - } _timer; - - /* POSIX.1b signals */ - struct { - compat_pid_t _pid; /* sender's pid */ - __compat_uid_t _uid; /* sender's uid */ - compat_sigval_t _sigval; - } _rt; - - /* SIGCHLD */ - struct { - compat_pid_t _pid; /* which child */ - __compat_uid_t _uid; /* sender's uid */ - int _status; /* exit code */ - compat_clock_t _utime; - compat_clock_t _stime; - } _sigchld; - - /* SIGILL, SIGFPE, SIGSEGV, SIGBUS, SIGEMT */ - struct { - unsigned int _addr; /* faulting insn/memory ref. */ - } _sigfault; - - /* SIGPOLL */ - struct { - int _band; /* POLL_IN, POLL_OUT, POLL_MSG */ - int _fd; - } _sigpoll; - } _sifields; -} compat_siginfo_t; - -#define COMPAT_OFF_T_MAX 0x7fffffff -#define COMPAT_LOFF_T_MAX 0x7fffffffffffffffL - -/* - * A pointer passed in from user mode. This should not - * be used for syscall parameters, just declare them - * as pointers because the syscall entry code will have - * appropriately converted them already. - */ - -static inline void __user *compat_ptr(compat_uptr_t uptr) -{ - return (void __user *)(unsigned long)uptr; -} - -static inline compat_uptr_t ptr_to_compat(void __user *uptr) -{ - return (u32)(unsigned long)uptr; -} - -static inline void __user *arch_compat_alloc_user_space(long len) -{ - struct pt_regs *regs = current->thread.regs; - unsigned long usp = regs->gpr[1]; - - /* - * We can't access below the stack pointer in the 32bit ABI and - * can access 288 bytes in the 64bit ABI - */ - if (!is_32bit_task()) - usp -= 288; - - return (void __user *) (usp - len); -} - /* * ipc64_perm is actually 32/64bit clean but since the compat layer refers to * it we may as well define it. @@ -227,10 +62,10 @@ struct compat_ipc64_perm { struct compat_semid64_ds { struct compat_ipc64_perm sem_perm; - unsigned int __unused1; - compat_time_t sem_otime; - unsigned int __unused2; - compat_time_t sem_ctime; + unsigned int sem_otime_high; + unsigned int sem_otime; + unsigned int sem_ctime_high; + unsigned int sem_ctime; compat_ulong_t sem_nsems; compat_ulong_t __unused3; compat_ulong_t __unused4; @@ -238,12 +73,12 @@ struct compat_semid64_ds { struct compat_msqid64_ds { struct compat_ipc64_perm msg_perm; - unsigned int __unused1; - compat_time_t msg_stime; - unsigned int __unused2; - compat_time_t msg_rtime; - unsigned int __unused3; - compat_time_t msg_ctime; + unsigned int msg_stime_high; + unsigned int msg_stime; + unsigned int msg_rtime_high; + unsigned int msg_rtime; + unsigned int msg_ctime_high; + unsigned int msg_ctime; compat_ulong_t msg_cbytes; compat_ulong_t msg_qnum; compat_ulong_t msg_qbytes; @@ -255,12 +90,12 @@ struct compat_msqid64_ds { struct compat_shmid64_ds { struct compat_ipc64_perm shm_perm; - unsigned int __unused1; - compat_time_t shm_atime; - unsigned int __unused2; - compat_time_t shm_dtime; - unsigned int __unused3; - compat_time_t shm_ctime; + unsigned int shm_atime_high; + unsigned int shm_atime; + unsigned int shm_dtime_high; + unsigned int shm_dtime; + unsigned int shm_ctime_high; + unsigned int shm_ctime; unsigned int __unused4; compat_size_t shm_segsz; compat_pid_t shm_cpid; diff --git a/arch/powerpc/include/asm/context_tracking.h b/arch/powerpc/include/asm/context_tracking.h index b6f5a33b8ee2..4b63931c49e0 100644 --- a/arch/powerpc/include/asm/context_tracking.h +++ b/arch/powerpc/include/asm/context_tracking.h @@ -1,10 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_CONTEXT_TRACKING_H #define _ASM_POWERPC_CONTEXT_TRACKING_H -#ifdef CONFIG_CONTEXT_TRACKING -#define SCHEDULE_USER bl .schedule_user +#ifdef CONFIG_CONTEXT_TRACKING_USER +#define SCHEDULE_USER bl schedule_user #else -#define SCHEDULE_USER bl .schedule +#define SCHEDULE_USER bl schedule #endif #endif diff --git a/arch/powerpc/include/asm/copro.h b/arch/powerpc/include/asm/copro.h new file mode 100644 index 000000000000..81bd176203ab --- /dev/null +++ b/arch/powerpc/include/asm/copro.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright 2014 IBM Corp. + */ + +#ifndef _ASM_POWERPC_COPRO_H +#define _ASM_POWERPC_COPRO_H + +#include <linux/mm_types.h> + +struct copro_slb +{ + u64 esid, vsid; +}; + +int copro_handle_mm_fault(struct mm_struct *mm, unsigned long ea, + unsigned long dsisr, vm_fault_t *flt); + +int copro_calculate_slb(struct mm_struct *mm, u64 ea, struct copro_slb *slb); + +#endif /* _ASM_POWERPC_COPRO_H */ diff --git a/arch/powerpc/include/asm/cpm.h b/arch/powerpc/include/asm/cpm.h index 4398a6cdcf53..ce483b0f8a4d 100644 --- a/arch/powerpc/include/asm/cpm.h +++ b/arch/powerpc/include/asm/cpm.h @@ -1,212 +1 @@ -#ifndef __CPM_H -#define __CPM_H - -#include <linux/compiler.h> -#include <linux/types.h> -#include <linux/errno.h> -#include <linux/of.h> - -/* - * SPI Parameter RAM common to QE and CPM. - */ -struct spi_pram { - __be16 rbase; /* Rx Buffer descriptor base address */ - __be16 tbase; /* Tx Buffer descriptor base address */ - u8 rfcr; /* Rx function code */ - u8 tfcr; /* Tx function code */ - __be16 mrblr; /* Max receive buffer length */ - __be32 rstate; /* Internal */ - __be32 rdp; /* Internal */ - __be16 rbptr; /* Internal */ - __be16 rbc; /* Internal */ - __be32 rxtmp; /* Internal */ - __be32 tstate; /* Internal */ - __be32 tdp; /* Internal */ - __be16 tbptr; /* Internal */ - __be16 tbc; /* Internal */ - __be32 txtmp; /* Internal */ - __be32 res; /* Tx temp. */ - __be16 rpbase; /* Relocation pointer (CPM1 only) */ - __be16 res1; /* Reserved */ -}; - -/* - * USB Controller pram common to QE and CPM. - */ -struct usb_ctlr { - u8 usb_usmod; - u8 usb_usadr; - u8 usb_uscom; - u8 res1[1]; - __be16 usb_usep[4]; - u8 res2[4]; - __be16 usb_usber; - u8 res3[2]; - __be16 usb_usbmr; - u8 res4[1]; - u8 usb_usbs; - /* Fields down below are QE-only */ - __be16 usb_ussft; - u8 res5[2]; - __be16 usb_usfrn; - u8 res6[0x22]; -} __attribute__ ((packed)); - -/* - * Function code bits, usually generic to devices. - */ -#ifdef CONFIG_CPM1 -#define CPMFCR_GBL ((u_char)0x00) /* Flag doesn't exist in CPM1 */ -#define CPMFCR_TC2 ((u_char)0x00) /* Flag doesn't exist in CPM1 */ -#define CPMFCR_DTB ((u_char)0x00) /* Flag doesn't exist in CPM1 */ -#define CPMFCR_BDB ((u_char)0x00) /* Flag doesn't exist in CPM1 */ -#else -#define CPMFCR_GBL ((u_char)0x20) /* Set memory snooping */ -#define CPMFCR_TC2 ((u_char)0x04) /* Transfer code 2 value */ -#define CPMFCR_DTB ((u_char)0x02) /* Use local bus for data when set */ -#define CPMFCR_BDB ((u_char)0x01) /* Use local bus for BD when set */ -#endif -#define CPMFCR_EB ((u_char)0x10) /* Set big endian byte order */ - -/* Opcodes common to CPM1 and CPM2 -*/ -#define CPM_CR_INIT_TRX ((ushort)0x0000) -#define CPM_CR_INIT_RX ((ushort)0x0001) -#define CPM_CR_INIT_TX ((ushort)0x0002) -#define CPM_CR_HUNT_MODE ((ushort)0x0003) -#define CPM_CR_STOP_TX ((ushort)0x0004) -#define CPM_CR_GRA_STOP_TX ((ushort)0x0005) -#define CPM_CR_RESTART_TX ((ushort)0x0006) -#define CPM_CR_CLOSE_RX_BD ((ushort)0x0007) -#define CPM_CR_SET_GADDR ((ushort)0x0008) -#define CPM_CR_SET_TIMER ((ushort)0x0008) -#define CPM_CR_STOP_IDMA ((ushort)0x000b) - -/* Buffer descriptors used by many of the CPM protocols. */ -typedef struct cpm_buf_desc { - ushort cbd_sc; /* Status and Control */ - ushort cbd_datlen; /* Data length in buffer */ - uint cbd_bufaddr; /* Buffer address in host memory */ -} cbd_t; - -/* Buffer descriptor control/status used by serial - */ - -#define BD_SC_EMPTY (0x8000) /* Receive is empty */ -#define BD_SC_READY (0x8000) /* Transmit is ready */ -#define BD_SC_WRAP (0x2000) /* Last buffer descriptor */ -#define BD_SC_INTRPT (0x1000) /* Interrupt on change */ -#define BD_SC_LAST (0x0800) /* Last buffer in frame */ -#define BD_SC_TC (0x0400) /* Transmit CRC */ -#define BD_SC_CM (0x0200) /* Continuous mode */ -#define BD_SC_ID (0x0100) /* Rec'd too many idles */ -#define BD_SC_P (0x0100) /* xmt preamble */ -#define BD_SC_BR (0x0020) /* Break received */ -#define BD_SC_FR (0x0010) /* Framing error */ -#define BD_SC_PR (0x0008) /* Parity error */ -#define BD_SC_NAK (0x0004) /* NAK - did not respond */ -#define BD_SC_OV (0x0002) /* Overrun */ -#define BD_SC_UN (0x0002) /* Underrun */ -#define BD_SC_CD (0x0001) /* */ -#define BD_SC_CL (0x0001) /* Collision */ - -/* Buffer descriptor control/status used by Ethernet receive. - * Common to SCC and FCC. - */ -#define BD_ENET_RX_EMPTY (0x8000) -#define BD_ENET_RX_WRAP (0x2000) -#define BD_ENET_RX_INTR (0x1000) -#define BD_ENET_RX_LAST (0x0800) -#define BD_ENET_RX_FIRST (0x0400) -#define BD_ENET_RX_MISS (0x0100) -#define BD_ENET_RX_BC (0x0080) /* FCC Only */ -#define BD_ENET_RX_MC (0x0040) /* FCC Only */ -#define BD_ENET_RX_LG (0x0020) -#define BD_ENET_RX_NO (0x0010) -#define BD_ENET_RX_SH (0x0008) -#define BD_ENET_RX_CR (0x0004) -#define BD_ENET_RX_OV (0x0002) -#define BD_ENET_RX_CL (0x0001) -#define BD_ENET_RX_STATS (0x01ff) /* All status bits */ - -/* Buffer descriptor control/status used by Ethernet transmit. - * Common to SCC and FCC. - */ -#define BD_ENET_TX_READY (0x8000) -#define BD_ENET_TX_PAD (0x4000) -#define BD_ENET_TX_WRAP (0x2000) -#define BD_ENET_TX_INTR (0x1000) -#define BD_ENET_TX_LAST (0x0800) -#define BD_ENET_TX_TC (0x0400) -#define BD_ENET_TX_DEF (0x0200) -#define BD_ENET_TX_HB (0x0100) -#define BD_ENET_TX_LC (0x0080) -#define BD_ENET_TX_RL (0x0040) -#define BD_ENET_TX_RCMASK (0x003c) -#define BD_ENET_TX_UN (0x0002) -#define BD_ENET_TX_CSL (0x0001) -#define BD_ENET_TX_STATS (0x03ff) /* All status bits */ - -/* Buffer descriptor control/status used by Transparent mode SCC. - */ -#define BD_SCC_TX_LAST (0x0800) - -/* Buffer descriptor control/status used by I2C. - */ -#define BD_I2C_START (0x0400) - -int cpm_muram_init(void); - -#if defined(CONFIG_CPM) || defined(CONFIG_QUICC_ENGINE) -unsigned long cpm_muram_alloc(unsigned long size, unsigned long align); -int cpm_muram_free(unsigned long offset); -unsigned long cpm_muram_alloc_fixed(unsigned long offset, unsigned long size); -void __iomem *cpm_muram_addr(unsigned long offset); -unsigned long cpm_muram_offset(void __iomem *addr); -dma_addr_t cpm_muram_dma(void __iomem *addr); -#else -static inline unsigned long cpm_muram_alloc(unsigned long size, - unsigned long align) -{ - return -ENOSYS; -} - -static inline int cpm_muram_free(unsigned long offset) -{ - return -ENOSYS; -} - -static inline unsigned long cpm_muram_alloc_fixed(unsigned long offset, - unsigned long size) -{ - return -ENOSYS; -} - -static inline void __iomem *cpm_muram_addr(unsigned long offset) -{ - return NULL; -} - -static inline unsigned long cpm_muram_offset(void __iomem *addr) -{ - return -ENOSYS; -} - -static inline dma_addr_t cpm_muram_dma(void __iomem *addr) -{ - return 0; -} -#endif /* defined(CONFIG_CPM) || defined(CONFIG_QUICC_ENGINE) */ - -#ifdef CONFIG_CPM -int cpm_command(u32 command, u8 opcode); -#else -static inline int cpm_command(u32 command, u8 opcode) -{ - return -ENOSYS; -} -#endif /* CONFIG_CPM */ - -int cpm2_gpiochip_add32(struct device_node *np); - -#endif +#include <soc/fsl/cpm.h> diff --git a/arch/powerpc/include/asm/cpm1.h b/arch/powerpc/include/asm/cpm1.h index 8ee4211ca0c6..e3c6969853ef 100644 --- a/arch/powerpc/include/asm/cpm1.h +++ b/arch/powerpc/include/asm/cpm1.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * MPC8xx Communication Processor Module. * Copyright (c) 1997 Dan Malek (dmalek@jlc.net) @@ -48,11 +49,6 @@ */ extern cpm8xx_t __iomem *cpmp; /* Pointer to comm processor */ -#define cpm_dpalloc cpm_muram_alloc -#define cpm_dpfree cpm_muram_free -#define cpm_dpram_addr cpm_muram_addr -#define cpm_dpram_phys cpm_muram_dma - extern void cpm_setbrg(uint brg, uint rate); extern void __init cpm_load_patch(cpm8xx_t *cp); @@ -67,6 +63,7 @@ extern void cpm_reset(void); #define PROFF_SPI ((uint)0x0180) #define PROFF_SCC3 ((uint)0x0200) #define PROFF_SMC1 ((uint)0x0280) +#define PROFF_DSP1 ((uint)0x02c0) #define PROFF_SCC4 ((uint)0x0300) #define PROFF_SMC2 ((uint)0x0380) @@ -560,6 +557,8 @@ typedef struct risc_timer_pram { #define CPM_PIN_SECONDARY 2 #define CPM_PIN_GPIO 4 #define CPM_PIN_OPENDRAIN 8 +#define CPM_PIN_FALLEDGE 16 +#define CPM_PIN_ANYEDGE 0 enum cpm_port { CPM_PORTA, @@ -602,5 +601,7 @@ enum cpm_clk { }; int cpm1_clk_setup(enum cpm_clk_target target, int clock, int mode); +int cpm1_gpiochip_add16(struct device *dev); +int cpm1_gpiochip_add32(struct device *dev); #endif /* __CPM1__ */ diff --git a/arch/powerpc/include/asm/cpm2.h b/arch/powerpc/include/asm/cpm2.h index f42e9baf3a4e..a22acc36eb9b 100644 --- a/arch/powerpc/include/asm/cpm2.h +++ b/arch/powerpc/include/asm/cpm2.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Communication Processor Module v2. * @@ -86,10 +87,6 @@ */ extern cpm_cpm2_t __iomem *cpmp; /* Pointer to comm processor */ -#define cpm_dpalloc cpm_muram_alloc -#define cpm_dpfree cpm_muram_free -#define cpm_dpram_addr cpm_muram_addr - extern void cpm2_reset(void); /* Baud rate generators. @@ -489,7 +486,6 @@ typedef struct scc_trans { #define FCC_GFMR_TCI ((uint)0x20000000) #define FCC_GFMR_TRX ((uint)0x10000000) #define FCC_GFMR_TTX ((uint)0x08000000) -#define FCC_GFMR_TTX ((uint)0x08000000) #define FCC_GFMR_CDP ((uint)0x04000000) #define FCC_GFMR_CTSP ((uint)0x02000000) #define FCC_GFMR_CDS ((uint)0x01000000) @@ -594,7 +590,7 @@ typedef struct fcc_enet { uint fen_p256c; /* Total packets 256 < bytes <= 511 */ uint fen_p512c; /* Total packets 512 < bytes <= 1023 */ uint fen_p1024c; /* Total packets 1024 < bytes <= 1518 */ - uint fen_cambuf; /* Internal CAM buffer poiner */ + uint fen_cambuf; /* Internal CAM buffer pointer */ ushort fen_rfthr; /* Received frames threshold */ ushort fen_rfcnt; /* Received frames count */ } fcc_enet_t; @@ -1080,6 +1076,9 @@ typedef struct im_idma { #define FCC2_MEM_OFFSET FCC_MEM_OFFSET(1) #define FCC3_MEM_OFFSET FCC_MEM_OFFSET(2) +/* Pipeline Maximum Depth */ +#define MPC82XX_BCR_PLDP 0x00800000 + /* Clocks and GRG's */ enum cpm_clk_dir { @@ -1133,8 +1132,8 @@ enum cpm_clk { CPM_CLK_DUMMY }; -extern int cpm2_clk_setup(enum cpm_clk_target target, int clock, int mode); -extern int cpm2_smc_clk_setup(enum cpm_clk_target target, int clock); +int __init cpm2_clk_setup(enum cpm_clk_target target, int clock, int mode); +int __init cpm2_smc_clk_setup(enum cpm_clk_target target, int clock); #define CPM_PIN_INPUT 0 #define CPM_PIN_OUTPUT 1 @@ -1143,7 +1142,7 @@ extern int cpm2_smc_clk_setup(enum cpm_clk_target target, int clock); #define CPM_PIN_GPIO 4 #define CPM_PIN_OPENDRAIN 8 -void cpm2_set_pin(int port, int pin, int flags); +void __init cpm2_set_pin(int port, int pin, int flags); #endif /* __CPM2__ */ #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/cpu_has_feature.h b/arch/powerpc/include/asm/cpu_has_feature.h new file mode 100644 index 000000000000..604fa3b6c33d --- /dev/null +++ b/arch/powerpc/include/asm/cpu_has_feature.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_POWERPC_CPU_HAS_FEATURE_H +#define __ASM_POWERPC_CPU_HAS_FEATURE_H + +#ifndef __ASSEMBLER__ + +#include <linux/bug.h> +#include <asm/cputable.h> + +static __always_inline bool early_cpu_has_feature(unsigned long feature) +{ + return !!((CPU_FTRS_ALWAYS & feature) || + (CPU_FTRS_POSSIBLE & cur_cpu_spec->cpu_features & feature)); +} + +#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECKS +#include <linux/jump_label.h> + +#define NUM_CPU_FTR_KEYS BITS_PER_LONG + +extern struct static_key_true cpu_feature_keys[NUM_CPU_FTR_KEYS]; + +static __always_inline bool cpu_has_feature(unsigned long feature) +{ + int i; + + BUILD_BUG_ON(!__builtin_constant_p(feature)); + BUILD_BUG_ON(__builtin_popcountl(feature) > 1); + +#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECK_DEBUG + if (!static_key_feature_checks_initialized) { + printk("Warning! cpu_has_feature() used prior to jump label init!\n"); + dump_stack(); + return early_cpu_has_feature(feature); + } +#endif + + if (CPU_FTRS_ALWAYS & feature) + return true; + + if (!(CPU_FTRS_POSSIBLE & feature)) + return false; + + i = __builtin_ctzl(feature); + return static_branch_likely(&cpu_feature_keys[i]); +} +#else +static __always_inline bool cpu_has_feature(unsigned long feature) +{ + return early_cpu_has_feature(feature); +} +#endif + +#endif /* __ASSEMBLER__ */ +#endif /* __ASM_POWERPC_CPU_HAS_FEATURE_H */ diff --git a/arch/powerpc/include/asm/cpu_setup.h b/arch/powerpc/include/asm/cpu_setup.h new file mode 100644 index 000000000000..30e2fe389502 --- /dev/null +++ b/arch/powerpc/include/asm/cpu_setup.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020 IBM Corporation + */ + +#ifndef _ASM_POWERPC_CPU_SETUP_H +#define _ASM_POWERPC_CPU_SETUP_H +void __setup_cpu_power7(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_power8(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_power9(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_power10(unsigned long offset, struct cpu_spec *spec); +void __restore_cpu_power7(void); +void __restore_cpu_power8(void); +void __restore_cpu_power9(void); +void __restore_cpu_power10(void); + +void __setup_cpu_e500v1(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_e500v2(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_e500mc(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_440ep(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_440epx(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_440gx(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_440grx(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_440spe(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_440x5(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_460ex(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_460gt(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_460sx(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_apm821xx(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_603(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_604(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_750(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_750cx(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_750fx(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_7400(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_7410(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_745x(unsigned long offset, struct cpu_spec *spec); + +void __setup_cpu_ppc970(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_ppc970MP(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_pa6t(unsigned long offset, struct cpu_spec *spec); +void __restore_cpu_pa6t(void); +void __restore_cpu_ppc970(void); + +void __setup_cpu_e5500(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_e6500(unsigned long offset, struct cpu_spec *spec); +void __restore_cpu_e5500(void); +void __restore_cpu_e6500(void); +#endif /* _ASM_POWERPC_CPU_SETUP_H */ diff --git a/arch/powerpc/include/asm/cpufeature.h b/arch/powerpc/include/asm/cpufeature.h new file mode 100644 index 000000000000..2dcc66225e7f --- /dev/null +++ b/arch/powerpc/include/asm/cpufeature.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * CPU feature definitions for module loading, used by + * module_cpu_feature_match(), see asm/cputable.h for powerpc CPU features. + * + * Copyright 2016 Alastair D'Silva, IBM Corporation. + */ + +#ifndef __ASM_POWERPC_CPUFEATURE_H +#define __ASM_POWERPC_CPUFEATURE_H + +#include <asm/cputable.h> + +/* Keep these in step with powerpc/include/asm/cputable.h */ +#define MAX_CPU_FEATURES (2 * 32) + +/* + * Currently we don't have a need for any of the feature bits defined in + * cpu_user_features. When we do, they should be defined such as: + * + * #define PPC_MODULE_FEATURE_32 (ilog2(PPC_FEATURE_32)) + */ + +#define PPC_MODULE_FEATURE_VEC_CRYPTO (32 + ilog2(PPC_FEATURE2_VEC_CRYPTO)) +#define PPC_MODULE_FEATURE_P10 (32 + ilog2(PPC_FEATURE2_ARCH_3_1)) + +#define cpu_feature(x) (x) + +static inline bool cpu_have_feature(unsigned int num) +{ + if (num < 32) + return !!(cur_cpu_spec->cpu_user_features & 1UL << num); + else + return !!(cur_cpu_spec->cpu_user_features2 & 1UL << (num - 32)); +} + +#endif /* __ASM_POWERPC_CPUFEATURE_H */ diff --git a/arch/powerpc/include/asm/cpuidle.h b/arch/powerpc/include/asm/cpuidle.h new file mode 100644 index 000000000000..054cd2fcfd55 --- /dev/null +++ b/arch/powerpc/include/asm/cpuidle.h @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_CPUIDLE_H +#define _ASM_POWERPC_CPUIDLE_H + +#ifdef CONFIG_PPC_POWERNV +/* Thread state used in powernv idle state management */ +#define PNV_THREAD_RUNNING 0 +#define PNV_THREAD_NAP 1 +#define PNV_THREAD_SLEEP 2 +#define PNV_THREAD_WINKLE 3 + +/* + * Core state used in powernv idle for POWER8. + * + * The lock bit synchronizes updates to the state, as well as parts of the + * sleep/wake code (see kernel/idle_book3s.S). + * + * Bottom 8 bits track the idle state of each thread. Bit is cleared before + * the thread executes an idle instruction (nap/sleep/winkle). + * + * Then there is winkle tracking. A core does not lose complete state + * until every thread is in winkle. So the winkle count field counts the + * number of threads in winkle (small window of false positives is okay + * around the sleep/wake, so long as there are no false negatives). + * + * When the winkle count reaches 8 (the COUNT_ALL_BIT becomes set), then + * the THREAD_WINKLE_BITS are set, which indicate which threads have not + * yet woken from the winkle state. + */ +#define NR_PNV_CORE_IDLE_LOCK_BIT 28 +#define PNV_CORE_IDLE_LOCK_BIT (1ULL << NR_PNV_CORE_IDLE_LOCK_BIT) + +#define PNV_CORE_IDLE_WINKLE_COUNT_SHIFT 16 +#define PNV_CORE_IDLE_WINKLE_COUNT 0x00010000 +#define PNV_CORE_IDLE_WINKLE_COUNT_BITS 0x000F0000 +#define PNV_CORE_IDLE_THREAD_WINKLE_BITS_SHIFT 8 +#define PNV_CORE_IDLE_THREAD_WINKLE_BITS 0x0000FF00 + +#define PNV_CORE_IDLE_THREAD_BITS 0x000000FF + +/* + * ============================ NOTE ================================= + * The older firmware populates only the RL field in the psscr_val and + * sets the psscr_mask to 0xf. On such a firmware, the kernel sets the + * remaining PSSCR fields to default values as follows: + * + * - ESL and EC bits are to 1. So wakeup from any stop state will be + * at vector 0x100. + * + * - MTL and PSLL are set to the maximum allowed value as per the ISA, + * i.e. 15. + * + * - The Transition Rate, TR is set to the Maximum value 3. + */ +#define PSSCR_HV_DEFAULT_VAL (PSSCR_ESL | PSSCR_EC | \ + PSSCR_PSLL_MASK | PSSCR_TR_MASK | \ + PSSCR_MTL_MASK) + +#define PSSCR_HV_DEFAULT_MASK (PSSCR_ESL | PSSCR_EC | \ + PSSCR_PSLL_MASK | PSSCR_TR_MASK | \ + PSSCR_MTL_MASK | PSSCR_RL_MASK) +#define PSSCR_EC_SHIFT 20 +#define PSSCR_ESL_SHIFT 21 +#define GET_PSSCR_EC(x) (((x) & PSSCR_EC) >> PSSCR_EC_SHIFT) +#define GET_PSSCR_ESL(x) (((x) & PSSCR_ESL) >> PSSCR_ESL_SHIFT) +#define GET_PSSCR_RL(x) ((x) & PSSCR_RL_MASK) + +#define ERR_EC_ESL_MISMATCH -1 +#define ERR_DEEP_STATE_ESL_MISMATCH -2 + +#ifndef __ASSEMBLER__ + +#define PNV_IDLE_NAME_LEN 16 +struct pnv_idle_states_t { + char name[PNV_IDLE_NAME_LEN]; + u32 latency_ns; + u32 residency_ns; + u64 psscr_val; + u64 psscr_mask; + u32 flags; + bool valid; +}; + +extern struct pnv_idle_states_t *pnv_idle_states; +extern int nr_pnv_idle_states; + +unsigned long pnv_cpu_offline(unsigned int cpu); +int __init validate_psscr_val_mask(u64 *psscr_val, u64 *psscr_mask, u32 flags); +static inline void report_invalid_psscr_val(u64 psscr_val, int err) +{ + switch (err) { + case ERR_EC_ESL_MISMATCH: + pr_warn("Invalid psscr 0x%016llx : ESL,EC bits unequal", + psscr_val); + break; + case ERR_DEEP_STATE_ESL_MISMATCH: + pr_warn("Invalid psscr 0x%016llx : ESL cleared for deep stop-state", + psscr_val); + } +} +#endif + +#endif + +#endif diff --git a/arch/powerpc/include/asm/cputable.h b/arch/powerpc/include/asm/cputable.h index 6f3887d884d2..ec16c12296da 100644 --- a/arch/powerpc/include/asm/cputable.h +++ b/arch/powerpc/include/asm/cputable.h @@ -1,12 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __ASM_POWERPC_CPUTABLE_H #define __ASM_POWERPC_CPUTABLE_H -#include <asm/asm-compat.h> -#include <asm/feature-fixups.h> +#include <linux/types.h> #include <uapi/asm/cputable.h> +#include <asm/asm-const.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* This structure can grow, it's real size is used by head.S code * via the mkdefs mechanism. @@ -16,16 +17,6 @@ struct cpu_spec; typedef void (*cpu_setup_t)(unsigned long offset, struct cpu_spec* spec); typedef void (*cpu_restore_t)(void); -enum powerpc_oprofile_type { - PPC_OPROFILE_INVALID = 0, - PPC_OPROFILE_RS64 = 1, - PPC_OPROFILE_POWER4 = 2, - PPC_OPROFILE_G4 = 3, - PPC_OPROFILE_FSL_EMB = 4, - PPC_OPROFILE_CELL = 5, - PPC_OPROFILE_PA6T = 6, -}; - enum powerpc_pmc_type { PPC_PMC_DEFAULT = 0, PPC_PMC_IBM = 1, @@ -40,8 +31,14 @@ extern int machine_check_4xx(struct pt_regs *regs); extern int machine_check_440A(struct pt_regs *regs); extern int machine_check_e500mc(struct pt_regs *regs); extern int machine_check_e500(struct pt_regs *regs); -extern int machine_check_e200(struct pt_regs *regs); extern int machine_check_47x(struct pt_regs *regs); +int machine_check_8xx(struct pt_regs *regs); +int machine_check_83xx(struct pt_regs *regs); + +extern void cpu_down_flush_e500v2(void); +extern void cpu_down_flush_e500mc(void); +extern void cpu_down_flush_e5500(void); +extern void cpu_down_flush_e6500(void); /* NOTE WELL: Update identify_cpu() if fields are added or removed! */ struct cpu_spec { @@ -59,6 +56,9 @@ struct cpu_spec { unsigned int icache_bsize; unsigned int dcache_bsize; + /* flush caches inside the current cpu */ + void (*cpu_down_flush)(void); + /* number of performance monitor counters */ unsigned int num_pmcs; enum powerpc_pmc_type pmc_type; @@ -70,19 +70,6 @@ struct cpu_spec { /* Used to restore cpu setup on secondary processors and at resume */ cpu_restore_t cpu_restore; - /* Used by oprofile userspace to select the right counters */ - char *oprofile_cpu_type; - - /* Processor specific oprofile operations */ - enum powerpc_oprofile_type oprofile_type; - - /* Bit locations inside the mmcra change */ - unsigned long oprofile_mmcra_sihv; - unsigned long oprofile_mmcra_sipr; - - /* Bits to clear during an oprofile exception */ - unsigned long oprofile_mmcra_clear; - /* Name of processor class, for the ELF AT_PLATFORM entry */ char *platform; @@ -90,57 +77,75 @@ struct cpu_spec { * if the error is fatal, 1 if it was fully recovered and 0 to * pass up (not CPU originated) */ int (*machine_check)(struct pt_regs *regs); + + /* + * Processor specific early machine check handler which is + * called in real mode to handle SLB and TLB errors. + */ + long (*machine_check_early)(struct pt_regs *regs); }; extern struct cpu_spec *cur_cpu_spec; extern unsigned int __start___ftr_fixup, __stop___ftr_fixup; +extern void set_cur_cpu_spec(struct cpu_spec *s); extern struct cpu_spec *identify_cpu(unsigned long offset, unsigned int pvr); +extern void identify_cpu_name(unsigned int pvr); extern void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end); extern const char *powerpc_base_platform; -#endif /* __ASSEMBLY__ */ +#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECKS +extern void cpu_feature_keys_init(void); +#else +static inline void cpu_feature_keys_init(void) { } +#endif + +#endif /* __ASSEMBLER__ */ /* CPU kernel features */ -/* Retain the 32b definitions all use bottom half of word */ +/* Definitions for features that we have on both 32-bit and 64-bit chips */ #define CPU_FTR_COHERENT_ICACHE ASM_CONST(0x00000001) -#define CPU_FTR_L2CR ASM_CONST(0x00000002) -#define CPU_FTR_SPEC7450 ASM_CONST(0x00000004) -#define CPU_FTR_ALTIVEC ASM_CONST(0x00000008) -#define CPU_FTR_TAU ASM_CONST(0x00000010) -#define CPU_FTR_CAN_DOZE ASM_CONST(0x00000020) -#define CPU_FTR_USE_TB ASM_CONST(0x00000040) -#define CPU_FTR_L2CSR ASM_CONST(0x00000080) -#define CPU_FTR_601 ASM_CONST(0x00000100) -#define CPU_FTR_DBELL ASM_CONST(0x00000200) -#define CPU_FTR_CAN_NAP ASM_CONST(0x00000400) -#define CPU_FTR_L3CR ASM_CONST(0x00000800) -#define CPU_FTR_L3_DISABLE_NAP ASM_CONST(0x00001000) -#define CPU_FTR_NAP_DISABLE_L2_PR ASM_CONST(0x00002000) -#define CPU_FTR_DUAL_PLL_750FX ASM_CONST(0x00004000) -#define CPU_FTR_NO_DPM ASM_CONST(0x00008000) -#define CPU_FTR_476_DD2 ASM_CONST(0x00010000) -#define CPU_FTR_NEED_COHERENT ASM_CONST(0x00020000) -#define CPU_FTR_NO_BTIC ASM_CONST(0x00040000) -#define CPU_FTR_DEBUG_LVL_EXC ASM_CONST(0x00080000) -#define CPU_FTR_NODSISRALIGN ASM_CONST(0x00100000) -#define CPU_FTR_PPC_LE ASM_CONST(0x00200000) -#define CPU_FTR_REAL_LE ASM_CONST(0x00400000) -#define CPU_FTR_FPU_UNAVAILABLE ASM_CONST(0x00800000) -#define CPU_FTR_UNIFIED_ID_CACHE ASM_CONST(0x01000000) -#define CPU_FTR_SPE ASM_CONST(0x02000000) -#define CPU_FTR_NEED_PAIRED_STWCX ASM_CONST(0x04000000) -#define CPU_FTR_LWSYNC ASM_CONST(0x08000000) -#define CPU_FTR_NOEXECUTE ASM_CONST(0x10000000) -#define CPU_FTR_INDEXED_DCR ASM_CONST(0x20000000) -#define CPU_FTR_EMB_HV ASM_CONST(0x40000000) +#define CPU_FTR_ALTIVEC ASM_CONST(0x00000002) +#define CPU_FTR_DBELL ASM_CONST(0x00000004) +#define CPU_FTR_CAN_NAP ASM_CONST(0x00000008) +#define CPU_FTR_DEBUG_LVL_EXC ASM_CONST(0x00000010) +// ASM_CONST(0x00000020) Free +#define CPU_FTR_FPU_UNAVAILABLE ASM_CONST(0x00000040) +#define CPU_FTR_LWSYNC ASM_CONST(0x00000080) +#define CPU_FTR_NOEXECUTE ASM_CONST(0x00000100) +#define CPU_FTR_EMB_HV ASM_CONST(0x00000200) + +/* Definitions for features that only exist on 32-bit chips */ +#ifdef CONFIG_PPC32 +#define CPU_FTR_L2CR ASM_CONST(0x00002000) +#define CPU_FTR_SPEC7450 ASM_CONST(0x00004000) +#define CPU_FTR_TAU ASM_CONST(0x00008000) +#define CPU_FTR_CAN_DOZE ASM_CONST(0x00010000) +#define CPU_FTR_L3CR ASM_CONST(0x00040000) +#define CPU_FTR_L3_DISABLE_NAP ASM_CONST(0x00080000) +#define CPU_FTR_NAP_DISABLE_L2_PR ASM_CONST(0x00100000) +#define CPU_FTR_DUAL_PLL_750FX ASM_CONST(0x00200000) +#define CPU_FTR_NO_DPM ASM_CONST(0x00400000) +#define CPU_FTR_476_DD2 ASM_CONST(0x00800000) +#define CPU_FTR_NEED_COHERENT ASM_CONST(0x01000000) +#define CPU_FTR_NO_BTIC ASM_CONST(0x02000000) +#define CPU_FTR_PPC_LE ASM_CONST(0x04000000) +#define CPU_FTR_SPE ASM_CONST(0x10000000) +#define CPU_FTR_NEED_PAIRED_STWCX ASM_CONST(0x20000000) +#define CPU_FTR_INDEXED_DCR ASM_CONST(0x40000000) + +#else /* CONFIG_PPC32 */ +/* Define these to 0 for the sake of tests in common code */ +#define CPU_FTR_PPC_LE (0) +#define CPU_FTR_SPE (0) +#endif /* - * Add the 64-bit processor unique features in the top half of the word; + * Definitions for the 64-bit processor unique features; * on 32-bit, make the names available but defined to be 0. */ #ifdef __powerpc64__ @@ -149,41 +154,50 @@ extern const char *powerpc_base_platform; #define LONG_ASM_CONST(x) 0 #endif -#define CPU_FTR_HVMODE LONG_ASM_CONST(0x0000000100000000) -#define CPU_FTR_ARCH_201 LONG_ASM_CONST(0x0000000200000000) -#define CPU_FTR_ARCH_206 LONG_ASM_CONST(0x0000000400000000) -#define CPU_FTR_ARCH_207S LONG_ASM_CONST(0x0000000800000000) -#define CPU_FTR_IABR LONG_ASM_CONST(0x0000001000000000) -#define CPU_FTR_MMCRA LONG_ASM_CONST(0x0000002000000000) -#define CPU_FTR_CTRL LONG_ASM_CONST(0x0000004000000000) -#define CPU_FTR_SMT LONG_ASM_CONST(0x0000008000000000) -#define CPU_FTR_PAUSE_ZERO LONG_ASM_CONST(0x0000010000000000) -#define CPU_FTR_PURR LONG_ASM_CONST(0x0000020000000000) -#define CPU_FTR_CELL_TB_BUG LONG_ASM_CONST(0x0000040000000000) -#define CPU_FTR_SPURR LONG_ASM_CONST(0x0000080000000000) -#define CPU_FTR_DSCR LONG_ASM_CONST(0x0000100000000000) -#define CPU_FTR_VSX LONG_ASM_CONST(0x0000200000000000) -#define CPU_FTR_SAO LONG_ASM_CONST(0x0000400000000000) -#define CPU_FTR_CP_USE_DCBTZ LONG_ASM_CONST(0x0000800000000000) -#define CPU_FTR_UNALIGNED_LD_STD LONG_ASM_CONST(0x0001000000000000) -#define CPU_FTR_ASYM_SMT LONG_ASM_CONST(0x0002000000000000) -#define CPU_FTR_STCX_CHECKS_ADDRESS LONG_ASM_CONST(0x0004000000000000) -#define CPU_FTR_POPCNTB LONG_ASM_CONST(0x0008000000000000) -#define CPU_FTR_POPCNTD LONG_ASM_CONST(0x0010000000000000) -#define CPU_FTR_ICSWX LONG_ASM_CONST(0x0020000000000000) -#define CPU_FTR_VMX_COPY LONG_ASM_CONST(0x0040000000000000) -#define CPU_FTR_TM LONG_ASM_CONST(0x0080000000000000) -#define CPU_FTR_CFAR LONG_ASM_CONST(0x0100000000000000) -#define CPU_FTR_HAS_PPR LONG_ASM_CONST(0x0200000000000000) -#define CPU_FTR_DAWR LONG_ASM_CONST(0x0400000000000000) -#define CPU_FTR_DABRX LONG_ASM_CONST(0x0800000000000000) - -#ifndef __ASSEMBLY__ - -#define CPU_FTR_PPCAS_ARCH_V2 (CPU_FTR_NOEXECUTE | CPU_FTR_NODSISRALIGN) - -#define MMU_FTR_PPCAS_ARCH_V2 (MMU_FTR_SLB | MMU_FTR_TLBIEL | \ - MMU_FTR_16M_PAGE) +#define CPU_FTR_REAL_LE LONG_ASM_CONST(0x0000000000001000) +#define CPU_FTR_HVMODE LONG_ASM_CONST(0x0000000000002000) +#define CPU_FTR_ARCH_206 LONG_ASM_CONST(0x0000000000008000) +#define CPU_FTR_ARCH_207S LONG_ASM_CONST(0x0000000000010000) +#define CPU_FTR_ARCH_300 LONG_ASM_CONST(0x0000000000020000) +#define CPU_FTR_MMCRA LONG_ASM_CONST(0x0000000000040000) +#define CPU_FTR_CTRL LONG_ASM_CONST(0x0000000000080000) +#define CPU_FTR_SMT LONG_ASM_CONST(0x0000000000100000) +#define CPU_FTR_PAUSE_ZERO LONG_ASM_CONST(0x0000000000200000) +#define CPU_FTR_PURR LONG_ASM_CONST(0x0000000000400000) +#define CPU_FTR_CELL_TB_BUG LONG_ASM_CONST(0x0000000000800000) +#define CPU_FTR_SPURR LONG_ASM_CONST(0x0000000001000000) +#define CPU_FTR_DSCR LONG_ASM_CONST(0x0000000002000000) +#define CPU_FTR_VSX LONG_ASM_CONST(0x0000000004000000) +#define CPU_FTR_SAO LONG_ASM_CONST(0x0000000008000000) +#define CPU_FTR_CP_USE_DCBTZ LONG_ASM_CONST(0x0000000010000000) +#define CPU_FTR_UNALIGNED_LD_STD LONG_ASM_CONST(0x0000000020000000) +#define CPU_FTR_ASYM_SMT LONG_ASM_CONST(0x0000000040000000) +#define CPU_FTR_STCX_CHECKS_ADDRESS LONG_ASM_CONST(0x0000000080000000) +#define CPU_FTR_POPCNTB LONG_ASM_CONST(0x0000000100000000) +#define CPU_FTR_POPCNTD LONG_ASM_CONST(0x0000000200000000) +/* LONG_ASM_CONST(0x0000000400000000) Free */ +#define CPU_FTR_VMX_COPY LONG_ASM_CONST(0x0000000800000000) +#define CPU_FTR_TM LONG_ASM_CONST(0x0000001000000000) +#define CPU_FTR_CFAR LONG_ASM_CONST(0x0000002000000000) +#define CPU_FTR_HAS_PPR LONG_ASM_CONST(0x0000004000000000) +#define CPU_FTR_DAWR LONG_ASM_CONST(0x0000008000000000) +#define CPU_FTR_DABRX LONG_ASM_CONST(0x0000010000000000) +#define CPU_FTR_PMAO_BUG LONG_ASM_CONST(0x0000020000000000) +#define CPU_FTR_POWER9_DD2_1 LONG_ASM_CONST(0x0000080000000000) +#define CPU_FTR_P9_TM_HV_ASSIST LONG_ASM_CONST(0x0000100000000000) +#define CPU_FTR_P9_TM_XER_SO_BUG LONG_ASM_CONST(0x0000200000000000) +#define CPU_FTR_P9_TLBIE_STQ_BUG LONG_ASM_CONST(0x0000400000000000) +#define CPU_FTR_P9_TIDR LONG_ASM_CONST(0x0000800000000000) +#define CPU_FTR_P9_TLBIE_ERAT_BUG LONG_ASM_CONST(0x0001000000000000) +#define CPU_FTR_P9_RADIX_PREFETCH_BUG LONG_ASM_CONST(0x0002000000000000) +#define CPU_FTR_ARCH_31 LONG_ASM_CONST(0x0004000000000000) +#define CPU_FTR_DAWR1 LONG_ASM_CONST(0x0008000000000000) +#define CPU_FTR_DEXCR_NPHIE LONG_ASM_CONST(0x0010000000000000) +#define CPU_FTR_P11_PVR LONG_ASM_CONST(0x0020000000000000) + +#ifndef __ASSEMBLER__ + +#define CPU_FTR_PPCAS_ARCH_V2 (CPU_FTR_NOEXECUTE) /* We only set the altivec features if the kernel was compiled with altivec * support @@ -224,11 +238,13 @@ extern const char *powerpc_base_platform; /* We only set the TM feature if the kernel was compiled with TM supprt */ #ifdef CONFIG_PPC_TRANSACTIONAL_MEM -#define CPU_FTR_TM_COMP CPU_FTR_TM -#define PPC_FEATURE2_HTM_COMP PPC_FEATURE2_HTM +#define CPU_FTR_TM_COMP CPU_FTR_TM +#define PPC_FEATURE2_HTM_COMP PPC_FEATURE2_HTM +#define PPC_FEATURE2_HTM_NOSC_COMP PPC_FEATURE2_HTM_NOSC #else -#define CPU_FTR_TM_COMP 0 -#define PPC_FEATURE2_HTM_COMP 0 +#define CPU_FTR_TM_COMP 0 +#define PPC_FEATURE2_HTM_COMP 0 +#define PPC_FEATURE2_HTM_NOSC_COMP 0 #endif /* We need to mark all pages as being coherent if we're SMP or we have a @@ -237,7 +253,7 @@ extern const char *powerpc_base_platform; * This is also required by 52xx family. */ #if defined(CONFIG_SMP) || defined(CONFIG_MPC10X_BRIDGE) \ - || defined(CONFIG_PPC_83xx) || defined(CONFIG_8260) \ + || defined(CONFIG_PPC_83xx) || defined(CONFIG_PPC_82xx) \ || defined(CONFIG_PPC_MPC52xx) #define CPU_FTR_COMMON CPU_FTR_NEED_COHERENT #else @@ -255,26 +271,18 @@ extern const char *powerpc_base_platform; #define CPU_FTR_MAYBE_CAN_NAP 0 #endif -#define CLASSIC_PPC (!defined(CONFIG_8xx) && !defined(CONFIG_4xx) && \ - !defined(CONFIG_POWER3) && !defined(CONFIG_POWER4) && \ - !defined(CONFIG_BOOKE)) - -#define CPU_FTRS_PPC601 (CPU_FTR_COMMON | CPU_FTR_601 | \ - CPU_FTR_COHERENT_ICACHE | CPU_FTR_UNIFIED_ID_CACHE) -#define CPU_FTRS_603 (CPU_FTR_COMMON | \ - CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_USE_TB | \ - CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_PPC_LE) -#define CPU_FTRS_604 (CPU_FTR_COMMON | \ - CPU_FTR_USE_TB | CPU_FTR_PPC_LE) +#define CPU_FTRS_603 (CPU_FTR_COMMON | CPU_FTR_MAYBE_CAN_DOZE | \ + CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_PPC_LE | CPU_FTR_NOEXECUTE) +#define CPU_FTRS_604 (CPU_FTR_COMMON | CPU_FTR_PPC_LE) #define CPU_FTRS_740_NOTAU (CPU_FTR_COMMON | \ - CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_USE_TB | CPU_FTR_L2CR | \ + CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_L2CR | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_PPC_LE) #define CPU_FTRS_740 (CPU_FTR_COMMON | \ - CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_USE_TB | CPU_FTR_L2CR | \ + CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_L2CR | \ CPU_FTR_TAU | CPU_FTR_MAYBE_CAN_NAP | \ CPU_FTR_PPC_LE) #define CPU_FTRS_750 (CPU_FTR_COMMON | \ - CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_USE_TB | CPU_FTR_L2CR | \ + CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_L2CR | \ CPU_FTR_TAU | CPU_FTR_MAYBE_CAN_NAP | \ CPU_FTR_PPC_LE) #define CPU_FTRS_750CL (CPU_FTRS_750) @@ -283,125 +291,108 @@ extern const char *powerpc_base_platform; #define CPU_FTRS_750FX (CPU_FTRS_750 | CPU_FTR_DUAL_PLL_750FX) #define CPU_FTRS_750GX (CPU_FTRS_750FX) #define CPU_FTRS_7400_NOTAU (CPU_FTR_COMMON | \ - CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_USE_TB | CPU_FTR_L2CR | \ + CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_L2CR | \ CPU_FTR_ALTIVEC_COMP | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_PPC_LE) #define CPU_FTRS_7400 (CPU_FTR_COMMON | \ - CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_USE_TB | CPU_FTR_L2CR | \ + CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_L2CR | \ CPU_FTR_TAU | CPU_FTR_ALTIVEC_COMP | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_PPC_LE) #define CPU_FTRS_7450_20 (CPU_FTR_COMMON | \ - CPU_FTR_USE_TB | CPU_FTR_L2CR | CPU_FTR_ALTIVEC_COMP | \ + CPU_FTR_L2CR | CPU_FTR_ALTIVEC_COMP | \ CPU_FTR_L3CR | CPU_FTR_SPEC7450 | \ CPU_FTR_NEED_COHERENT | CPU_FTR_PPC_LE | CPU_FTR_NEED_PAIRED_STWCX) #define CPU_FTRS_7450_21 (CPU_FTR_COMMON | \ - CPU_FTR_USE_TB | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_L2CR | CPU_FTR_ALTIVEC_COMP | \ CPU_FTR_L3CR | CPU_FTR_SPEC7450 | \ CPU_FTR_NAP_DISABLE_L2_PR | CPU_FTR_L3_DISABLE_NAP | \ CPU_FTR_NEED_COHERENT | CPU_FTR_PPC_LE | CPU_FTR_NEED_PAIRED_STWCX) #define CPU_FTRS_7450_23 (CPU_FTR_COMMON | \ - CPU_FTR_USE_TB | CPU_FTR_NEED_PAIRED_STWCX | \ + CPU_FTR_NEED_PAIRED_STWCX | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_L2CR | CPU_FTR_ALTIVEC_COMP | \ CPU_FTR_L3CR | CPU_FTR_SPEC7450 | \ CPU_FTR_NAP_DISABLE_L2_PR | CPU_FTR_NEED_COHERENT | CPU_FTR_PPC_LE) #define CPU_FTRS_7455_1 (CPU_FTR_COMMON | \ - CPU_FTR_USE_TB | CPU_FTR_NEED_PAIRED_STWCX | \ + CPU_FTR_NEED_PAIRED_STWCX | \ CPU_FTR_L2CR | CPU_FTR_ALTIVEC_COMP | CPU_FTR_L3CR | \ CPU_FTR_SPEC7450 | CPU_FTR_NEED_COHERENT | CPU_FTR_PPC_LE) #define CPU_FTRS_7455_20 (CPU_FTR_COMMON | \ - CPU_FTR_USE_TB | CPU_FTR_NEED_PAIRED_STWCX | \ + CPU_FTR_NEED_PAIRED_STWCX | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_L2CR | CPU_FTR_ALTIVEC_COMP | \ CPU_FTR_L3CR | CPU_FTR_SPEC7450 | \ CPU_FTR_NAP_DISABLE_L2_PR | CPU_FTR_L3_DISABLE_NAP | \ CPU_FTR_NEED_COHERENT | CPU_FTR_PPC_LE) #define CPU_FTRS_7455 (CPU_FTR_COMMON | \ - CPU_FTR_USE_TB | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_L2CR | CPU_FTR_ALTIVEC_COMP | \ CPU_FTR_L3CR | CPU_FTR_SPEC7450 | CPU_FTR_NAP_DISABLE_L2_PR | \ CPU_FTR_NEED_COHERENT | CPU_FTR_PPC_LE | CPU_FTR_NEED_PAIRED_STWCX) #define CPU_FTRS_7447_10 (CPU_FTR_COMMON | \ - CPU_FTR_USE_TB | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_L2CR | CPU_FTR_ALTIVEC_COMP | \ CPU_FTR_L3CR | CPU_FTR_SPEC7450 | CPU_FTR_NAP_DISABLE_L2_PR | \ CPU_FTR_NEED_COHERENT | CPU_FTR_NO_BTIC | CPU_FTR_PPC_LE | \ CPU_FTR_NEED_PAIRED_STWCX) #define CPU_FTRS_7447 (CPU_FTR_COMMON | \ - CPU_FTR_USE_TB | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_L2CR | CPU_FTR_ALTIVEC_COMP | \ CPU_FTR_L3CR | CPU_FTR_SPEC7450 | CPU_FTR_NAP_DISABLE_L2_PR | \ CPU_FTR_NEED_COHERENT | CPU_FTR_PPC_LE | CPU_FTR_NEED_PAIRED_STWCX) #define CPU_FTRS_7447A (CPU_FTR_COMMON | \ - CPU_FTR_USE_TB | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_L2CR | CPU_FTR_ALTIVEC_COMP | \ CPU_FTR_SPEC7450 | CPU_FTR_NAP_DISABLE_L2_PR | \ CPU_FTR_NEED_COHERENT | CPU_FTR_PPC_LE | CPU_FTR_NEED_PAIRED_STWCX) #define CPU_FTRS_7448 (CPU_FTR_COMMON | \ - CPU_FTR_USE_TB | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_L2CR | CPU_FTR_ALTIVEC_COMP | \ CPU_FTR_SPEC7450 | CPU_FTR_NAP_DISABLE_L2_PR | \ CPU_FTR_PPC_LE | CPU_FTR_NEED_PAIRED_STWCX) -#define CPU_FTRS_82XX (CPU_FTR_COMMON | \ - CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_USE_TB) +#define CPU_FTRS_82XX (CPU_FTR_COMMON | CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_NOEXECUTE) #define CPU_FTRS_G2_LE (CPU_FTR_COMMON | CPU_FTR_MAYBE_CAN_DOZE | \ - CPU_FTR_USE_TB | CPU_FTR_MAYBE_CAN_NAP) + CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_NOEXECUTE) #define CPU_FTRS_E300 (CPU_FTR_MAYBE_CAN_DOZE | \ - CPU_FTR_USE_TB | CPU_FTR_MAYBE_CAN_NAP | \ - CPU_FTR_COMMON) + CPU_FTR_MAYBE_CAN_NAP | \ + CPU_FTR_COMMON | CPU_FTR_NOEXECUTE) #define CPU_FTRS_E300C2 (CPU_FTR_MAYBE_CAN_DOZE | \ - CPU_FTR_USE_TB | CPU_FTR_MAYBE_CAN_NAP | \ - CPU_FTR_COMMON | CPU_FTR_FPU_UNAVAILABLE) -#define CPU_FTRS_CLASSIC32 (CPU_FTR_COMMON | CPU_FTR_USE_TB) -#define CPU_FTRS_8XX (CPU_FTR_USE_TB) -#define CPU_FTRS_40X (CPU_FTR_USE_TB | CPU_FTR_NODSISRALIGN | CPU_FTR_NOEXECUTE) -#define CPU_FTRS_44X (CPU_FTR_USE_TB | CPU_FTR_NODSISRALIGN | CPU_FTR_NOEXECUTE) -#define CPU_FTRS_440x6 (CPU_FTR_USE_TB | CPU_FTR_NODSISRALIGN | CPU_FTR_NOEXECUTE | \ + CPU_FTR_MAYBE_CAN_NAP | \ + CPU_FTR_COMMON | CPU_FTR_FPU_UNAVAILABLE | CPU_FTR_NOEXECUTE) +#define CPU_FTRS_CLASSIC32 (CPU_FTR_COMMON) +#define CPU_FTRS_8XX (CPU_FTR_NOEXECUTE) +#define CPU_FTRS_44X (CPU_FTR_NOEXECUTE) +#define CPU_FTRS_440x6 (CPU_FTR_NOEXECUTE | \ CPU_FTR_INDEXED_DCR) #define CPU_FTRS_47X (CPU_FTRS_440x6) -#define CPU_FTRS_E200 (CPU_FTR_USE_TB | CPU_FTR_SPE_COMP | \ - CPU_FTR_NODSISRALIGN | CPU_FTR_COHERENT_ICACHE | \ - CPU_FTR_UNIFIED_ID_CACHE | CPU_FTR_NOEXECUTE | \ - CPU_FTR_DEBUG_LVL_EXC) -#define CPU_FTRS_E500 (CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_USE_TB | \ - CPU_FTR_SPE_COMP | CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_NODSISRALIGN | \ +#define CPU_FTRS_E500 (CPU_FTR_MAYBE_CAN_DOZE | \ + CPU_FTR_SPE_COMP | CPU_FTR_MAYBE_CAN_NAP | \ CPU_FTR_NOEXECUTE) -#define CPU_FTRS_E500_2 (CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_USE_TB | \ +#define CPU_FTRS_E500_2 (CPU_FTR_MAYBE_CAN_DOZE | \ CPU_FTR_SPE_COMP | CPU_FTR_MAYBE_CAN_NAP | \ - CPU_FTR_NODSISRALIGN | CPU_FTR_NOEXECUTE) -#define CPU_FTRS_E500MC (CPU_FTR_USE_TB | CPU_FTR_NODSISRALIGN | \ - CPU_FTR_L2CSR | CPU_FTR_LWSYNC | CPU_FTR_NOEXECUTE | \ + CPU_FTR_NOEXECUTE) +#define CPU_FTRS_E500MC ( \ + CPU_FTR_LWSYNC | CPU_FTR_NOEXECUTE | \ CPU_FTR_DBELL | CPU_FTR_DEBUG_LVL_EXC | CPU_FTR_EMB_HV) -#define CPU_FTRS_E5500 (CPU_FTR_USE_TB | CPU_FTR_NODSISRALIGN | \ - CPU_FTR_L2CSR | CPU_FTR_LWSYNC | CPU_FTR_NOEXECUTE | \ +/* + * e5500/e6500 erratum A-006958 is a timebase bug that can use the + * same workaround as CPU_FTR_CELL_TB_BUG. + */ +#define CPU_FTRS_E5500 ( \ + CPU_FTR_LWSYNC | CPU_FTR_NOEXECUTE | \ CPU_FTR_DBELL | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \ - CPU_FTR_DEBUG_LVL_EXC | CPU_FTR_EMB_HV) -#define CPU_FTRS_E6500 (CPU_FTR_USE_TB | CPU_FTR_NODSISRALIGN | \ - CPU_FTR_L2CSR | CPU_FTR_LWSYNC | CPU_FTR_NOEXECUTE | \ + CPU_FTR_DEBUG_LVL_EXC | CPU_FTR_EMB_HV | CPU_FTR_CELL_TB_BUG) +#define CPU_FTRS_E6500 ( \ + CPU_FTR_LWSYNC | CPU_FTR_NOEXECUTE | \ CPU_FTR_DBELL | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \ - CPU_FTR_DEBUG_LVL_EXC | CPU_FTR_EMB_HV | CPU_FTR_ALTIVEC_COMP) -#define CPU_FTRS_GENERIC_32 (CPU_FTR_COMMON | CPU_FTR_NODSISRALIGN) + CPU_FTR_DEBUG_LVL_EXC | CPU_FTR_EMB_HV | CPU_FTR_ALTIVEC_COMP | \ + CPU_FTR_CELL_TB_BUG | CPU_FTR_SMT) /* 64-bit CPUs */ -#define CPU_FTRS_POWER3 (CPU_FTR_USE_TB | \ - CPU_FTR_IABR | CPU_FTR_PPC_LE) -#define CPU_FTRS_RS64 (CPU_FTR_USE_TB | \ - CPU_FTR_IABR | \ - CPU_FTR_MMCRA | CPU_FTR_CTRL) -#define CPU_FTRS_POWER4 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ +#define CPU_FTRS_PPC970 (CPU_FTR_LWSYNC | \ CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ - CPU_FTR_MMCRA | CPU_FTR_CP_USE_DCBTZ | \ - CPU_FTR_STCX_CHECKS_ADDRESS) -#define CPU_FTRS_PPC970 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ - CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | CPU_FTR_ARCH_201 | \ CPU_FTR_ALTIVEC_COMP | CPU_FTR_CAN_NAP | CPU_FTR_MMCRA | \ CPU_FTR_CP_USE_DCBTZ | CPU_FTR_STCX_CHECKS_ADDRESS | \ CPU_FTR_HVMODE | CPU_FTR_DABRX) -#define CPU_FTRS_POWER5 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ +#define CPU_FTRS_POWER5 (CPU_FTR_LWSYNC | \ CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ CPU_FTR_MMCRA | CPU_FTR_SMT | \ CPU_FTR_COHERENT_ICACHE | CPU_FTR_PURR | \ CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_DABRX) -#define CPU_FTRS_POWER6 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ +#define CPU_FTRS_POWER6 (CPU_FTR_LWSYNC | \ CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ CPU_FTR_MMCRA | CPU_FTR_SMT | \ CPU_FTR_COHERENT_ICACHE | \ @@ -409,81 +400,119 @@ extern const char *powerpc_base_platform; CPU_FTR_DSCR | CPU_FTR_UNALIGNED_LD_STD | \ CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_CFAR | \ CPU_FTR_DABRX) -#define CPU_FTRS_POWER7 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ +#define CPU_FTRS_POWER7 (CPU_FTR_LWSYNC | \ CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | CPU_FTR_ARCH_206 |\ CPU_FTR_MMCRA | CPU_FTR_SMT | \ CPU_FTR_COHERENT_ICACHE | \ CPU_FTR_PURR | CPU_FTR_SPURR | CPU_FTR_REAL_LE | \ CPU_FTR_DSCR | CPU_FTR_SAO | CPU_FTR_ASYM_SMT | \ CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \ - CPU_FTR_ICSWX | CPU_FTR_CFAR | CPU_FTR_HVMODE | \ - CPU_FTR_VMX_COPY | CPU_FTR_HAS_PPR | CPU_FTR_DABRX) -#define CPU_FTRS_POWER8 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ + CPU_FTR_CFAR | CPU_FTR_HVMODE | \ + CPU_FTR_VMX_COPY | CPU_FTR_HAS_PPR | CPU_FTR_DABRX ) +#define CPU_FTRS_POWER8 (CPU_FTR_LWSYNC | \ CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | CPU_FTR_ARCH_206 |\ CPU_FTR_MMCRA | CPU_FTR_SMT | \ CPU_FTR_COHERENT_ICACHE | \ CPU_FTR_PURR | CPU_FTR_SPURR | CPU_FTR_REAL_LE | \ CPU_FTR_DSCR | CPU_FTR_SAO | \ CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \ - CPU_FTR_ICSWX | CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \ + CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \ CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_DAWR | \ - CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP) -#define CPU_FTRS_CELL (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ + CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP ) +#define CPU_FTRS_POWER8E (CPU_FTRS_POWER8 | CPU_FTR_PMAO_BUG) +#define CPU_FTRS_POWER9 (CPU_FTR_LWSYNC | \ + CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | CPU_FTR_ARCH_206 |\ + CPU_FTR_MMCRA | CPU_FTR_SMT | \ + CPU_FTR_COHERENT_ICACHE | \ + CPU_FTR_PURR | CPU_FTR_SPURR | CPU_FTR_REAL_LE | \ + CPU_FTR_DSCR | CPU_FTR_SAO | \ + CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \ + CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \ + CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_ARCH_207S | \ + CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | CPU_FTR_P9_TLBIE_STQ_BUG | \ + CPU_FTR_P9_TLBIE_ERAT_BUG | CPU_FTR_P9_TIDR) +#define CPU_FTRS_POWER9_DD2_0 (CPU_FTRS_POWER9 | CPU_FTR_P9_RADIX_PREFETCH_BUG) +#define CPU_FTRS_POWER9_DD2_1 (CPU_FTRS_POWER9 | \ + CPU_FTR_P9_RADIX_PREFETCH_BUG | \ + CPU_FTR_POWER9_DD2_1) +#define CPU_FTRS_POWER9_DD2_2 (CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD2_1 | \ + CPU_FTR_P9_TM_HV_ASSIST | \ + CPU_FTR_P9_TM_XER_SO_BUG) +#define CPU_FTRS_POWER9_DD2_3 (CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD2_1 | \ + CPU_FTR_P9_TM_HV_ASSIST | \ + CPU_FTR_P9_TM_XER_SO_BUG | \ + CPU_FTR_DAWR) +#define CPU_FTRS_POWER10 (CPU_FTR_LWSYNC | \ + CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | CPU_FTR_ARCH_206 |\ + CPU_FTR_MMCRA | CPU_FTR_SMT | \ + CPU_FTR_COHERENT_ICACHE | \ + CPU_FTR_PURR | CPU_FTR_SPURR | CPU_FTR_REAL_LE | \ + CPU_FTR_DSCR | CPU_FTR_SAO | \ + CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \ + CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \ + CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_ARCH_207S | \ + CPU_FTR_ARCH_300 | CPU_FTR_ARCH_31 | \ + CPU_FTR_DAWR | CPU_FTR_DAWR1 | \ + CPU_FTR_DEXCR_NPHIE) + +#define CPU_FTRS_POWER11 (CPU_FTRS_POWER10 | CPU_FTR_P11_PVR) + +#define CPU_FTRS_CELL (CPU_FTR_LWSYNC | \ CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ CPU_FTR_ALTIVEC_COMP | CPU_FTR_MMCRA | CPU_FTR_SMT | \ CPU_FTR_PAUSE_ZERO | CPU_FTR_CELL_TB_BUG | CPU_FTR_CP_USE_DCBTZ | \ CPU_FTR_UNALIGNED_LD_STD | CPU_FTR_DABRX) -#define CPU_FTRS_PA6T (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ +#define CPU_FTRS_PA6T (CPU_FTR_LWSYNC | \ CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_ALTIVEC_COMP | \ CPU_FTR_PURR | CPU_FTR_REAL_LE | CPU_FTR_DABRX) -#define CPU_FTRS_COMPATIBLE (CPU_FTR_USE_TB | CPU_FTR_PPCAS_ARCH_V2) +#define CPU_FTRS_COMPATIBLE (CPU_FTR_PPCAS_ARCH_V2) -#define CPU_FTRS_A2 (CPU_FTR_USE_TB | CPU_FTR_SMT | CPU_FTR_DBELL | \ - CPU_FTR_NOEXECUTE | CPU_FTR_NODSISRALIGN | \ - CPU_FTR_ICSWX | CPU_FTR_DABRX ) - -#ifdef __powerpc64__ -#ifdef CONFIG_PPC_BOOK3E -#define CPU_FTRS_POSSIBLE (CPU_FTRS_E6500 | CPU_FTRS_E5500 | CPU_FTRS_A2) +#ifdef CONFIG_PPC64 +#ifdef CONFIG_PPC_BOOK3E_64 +#define CPU_FTRS_POSSIBLE (CPU_FTRS_E6500 | CPU_FTRS_E5500) #else +#ifdef CONFIG_CPU_LITTLE_ENDIAN #define CPU_FTRS_POSSIBLE \ - (CPU_FTRS_POWER3 | CPU_FTRS_RS64 | CPU_FTRS_POWER4 | \ - CPU_FTRS_PPC970 | CPU_FTRS_POWER5 | CPU_FTRS_POWER6 | \ - CPU_FTRS_POWER7 | CPU_FTRS_POWER8 | CPU_FTRS_CELL | \ - CPU_FTRS_PA6T | CPU_FTR_VSX) + (CPU_FTRS_POWER7 | CPU_FTRS_POWER8E | CPU_FTRS_POWER8 | \ + CPU_FTR_ALTIVEC_COMP | CPU_FTR_VSX_COMP | CPU_FTRS_POWER9 | \ + CPU_FTRS_POWER9_DD2_1 | CPU_FTRS_POWER9_DD2_2 | \ + CPU_FTRS_POWER9_DD2_3 | CPU_FTRS_POWER10 | CPU_FTRS_POWER11) +#else +#define CPU_FTRS_POSSIBLE \ + (CPU_FTRS_PPC970 | CPU_FTRS_POWER5 | \ + CPU_FTRS_POWER6 | CPU_FTRS_POWER7 | CPU_FTRS_POWER8E | \ + CPU_FTRS_POWER8 | CPU_FTRS_CELL | CPU_FTRS_PA6T | \ + CPU_FTR_VSX_COMP | CPU_FTR_ALTIVEC_COMP | CPU_FTRS_POWER9 | \ + CPU_FTRS_POWER9_DD2_1 | CPU_FTRS_POWER9_DD2_2 | \ + CPU_FTRS_POWER9_DD2_3 | CPU_FTRS_POWER10 | CPU_FTRS_POWER11) +#endif /* CONFIG_CPU_LITTLE_ENDIAN */ #endif #else enum { CPU_FTRS_POSSIBLE = -#if CLASSIC_PPC - CPU_FTRS_PPC601 | CPU_FTRS_603 | CPU_FTRS_604 | CPU_FTRS_740_NOTAU | +#ifdef CONFIG_PPC_BOOK3S_604 + CPU_FTRS_604 | CPU_FTRS_740_NOTAU | CPU_FTRS_740 | CPU_FTRS_750 | CPU_FTRS_750FX1 | CPU_FTRS_750FX2 | CPU_FTRS_750FX | CPU_FTRS_750GX | CPU_FTRS_7400_NOTAU | CPU_FTRS_7400 | CPU_FTRS_7450_20 | CPU_FTRS_7450_21 | CPU_FTRS_7450_23 | CPU_FTRS_7455_1 | CPU_FTRS_7455_20 | CPU_FTRS_7455 | CPU_FTRS_7447_10 | - CPU_FTRS_7447 | CPU_FTRS_7447A | CPU_FTRS_82XX | - CPU_FTRS_G2_LE | CPU_FTRS_E300 | CPU_FTRS_E300C2 | + CPU_FTRS_7447 | CPU_FTRS_7447A | CPU_FTRS_CLASSIC32 | -#else - CPU_FTRS_GENERIC_32 | #endif -#ifdef CONFIG_8xx - CPU_FTRS_8XX | -#endif -#ifdef CONFIG_40x - CPU_FTRS_40X | +#ifdef CONFIG_PPC_BOOK3S_603 + CPU_FTRS_603 | CPU_FTRS_82XX | + CPU_FTRS_G2_LE | CPU_FTRS_E300 | CPU_FTRS_E300C2 | #endif -#ifdef CONFIG_44x - CPU_FTRS_44X | CPU_FTRS_440x6 | +#ifdef CONFIG_PPC_8xx + CPU_FTRS_8XX | #endif #ifdef CONFIG_PPC_47x CPU_FTRS_47X | CPU_FTR_476_DD2 | +#elif defined(CONFIG_44x) + CPU_FTRS_44X | CPU_FTRS_440x6 | #endif -#ifdef CONFIG_E200 - CPU_FTRS_E200 | -#endif -#ifdef CONFIG_E500 +#ifdef CONFIG_PPC_E500 CPU_FTRS_E500 | CPU_FTRS_E500_2 | #endif #ifdef CONFIG_PPC_E500MC @@ -493,44 +522,69 @@ enum { }; #endif /* __powerpc64__ */ -#ifdef __powerpc64__ -#ifdef CONFIG_PPC_BOOK3E -#define CPU_FTRS_ALWAYS (CPU_FTRS_E6500 & CPU_FTRS_E5500 & CPU_FTRS_A2) +#ifdef CONFIG_PPC64 +#ifdef CONFIG_PPC_BOOK3E_64 +#define CPU_FTRS_ALWAYS (CPU_FTRS_E6500 & CPU_FTRS_E5500) +#else + +#ifdef CONFIG_PPC_DT_CPU_FTRS +#define CPU_FTRS_DT_CPU_BASE \ + (CPU_FTR_LWSYNC | \ + CPU_FTR_FPU_UNAVAILABLE | \ + CPU_FTR_NOEXECUTE | \ + CPU_FTR_COHERENT_ICACHE | \ + CPU_FTR_STCX_CHECKS_ADDRESS | \ + CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \ + CPU_FTR_DAWR | \ + CPU_FTR_ARCH_206 | \ + CPU_FTR_ARCH_207S) +#else +#define CPU_FTRS_DT_CPU_BASE (~0ul) +#endif + +/* pseries may disable DBELL with ibm,pi-features */ +#ifdef CONFIG_CPU_LITTLE_ENDIAN +#define CPU_FTRS_ALWAYS \ + (CPU_FTRS_POSSIBLE & ~CPU_FTR_HVMODE & ~CPU_FTR_DBELL & \ + CPU_FTRS_POWER7 & CPU_FTRS_POWER8E & CPU_FTRS_POWER8 & \ + CPU_FTRS_POWER9 & CPU_FTRS_POWER9_DD2_1 & CPU_FTRS_POWER9_DD2_2 & \ + CPU_FTRS_POWER10 & CPU_FTRS_POWER11 & CPU_FTRS_DT_CPU_BASE) #else #define CPU_FTRS_ALWAYS \ - (CPU_FTRS_POWER3 & CPU_FTRS_RS64 & CPU_FTRS_POWER4 & \ - CPU_FTRS_PPC970 & CPU_FTRS_POWER5 & CPU_FTRS_POWER6 & \ - CPU_FTRS_POWER7 & CPU_FTRS_CELL & CPU_FTRS_PA6T & CPU_FTRS_POSSIBLE) + (CPU_FTRS_PPC970 & CPU_FTRS_POWER5 & \ + CPU_FTRS_POWER6 & CPU_FTRS_POWER7 & CPU_FTRS_CELL & \ + CPU_FTRS_PA6T & CPU_FTRS_POWER8 & CPU_FTRS_POWER8E & \ + ~CPU_FTR_HVMODE & ~CPU_FTR_DBELL & CPU_FTRS_POSSIBLE & \ + CPU_FTRS_POWER9 & CPU_FTRS_POWER9_DD2_1 & CPU_FTRS_POWER9_DD2_2 & \ + CPU_FTRS_POWER10 & CPU_FTRS_POWER11 & CPU_FTRS_DT_CPU_BASE) +#endif /* CONFIG_CPU_LITTLE_ENDIAN */ #endif #else enum { CPU_FTRS_ALWAYS = -#if CLASSIC_PPC - CPU_FTRS_PPC601 & CPU_FTRS_603 & CPU_FTRS_604 & CPU_FTRS_740_NOTAU & +#ifdef CONFIG_PPC_BOOK3S_604 + CPU_FTRS_604 & CPU_FTRS_740_NOTAU & CPU_FTRS_740 & CPU_FTRS_750 & CPU_FTRS_750FX1 & CPU_FTRS_750FX2 & CPU_FTRS_750FX & CPU_FTRS_750GX & CPU_FTRS_7400_NOTAU & CPU_FTRS_7400 & CPU_FTRS_7450_20 & CPU_FTRS_7450_21 & CPU_FTRS_7450_23 & CPU_FTRS_7455_1 & CPU_FTRS_7455_20 & CPU_FTRS_7455 & CPU_FTRS_7447_10 & - CPU_FTRS_7447 & CPU_FTRS_7447A & CPU_FTRS_82XX & - CPU_FTRS_G2_LE & CPU_FTRS_E300 & CPU_FTRS_E300C2 & + CPU_FTRS_7447 & CPU_FTRS_7447A & CPU_FTRS_CLASSIC32 & -#else - CPU_FTRS_GENERIC_32 & #endif -#ifdef CONFIG_8xx - CPU_FTRS_8XX & +#ifdef CONFIG_PPC_BOOK3S_603 + CPU_FTRS_603 & CPU_FTRS_82XX & + CPU_FTRS_G2_LE & CPU_FTRS_E300 & CPU_FTRS_E300C2 & #endif -#ifdef CONFIG_40x - CPU_FTRS_40X & +#ifdef CONFIG_PPC_8xx + CPU_FTRS_8XX & #endif -#ifdef CONFIG_44x +#ifdef CONFIG_PPC_47x + CPU_FTRS_47X & +#elif defined(CONFIG_44x) CPU_FTRS_44X & CPU_FTRS_440x6 & #endif -#ifdef CONFIG_E200 - CPU_FTRS_E200 & -#endif -#ifdef CONFIG_E500 +#ifdef CONFIG_PPC_E500 CPU_FTRS_E500 & CPU_FTRS_E500_2 & #endif #ifdef CONFIG_PPC_E500MC @@ -541,16 +595,13 @@ enum { }; #endif /* __powerpc64__ */ -static inline int cpu_has_feature(unsigned long feature) -{ - return (CPU_FTRS_ALWAYS & feature) || - (CPU_FTRS_POSSIBLE - & cur_cpu_spec->cpu_features - & feature); -} - -#define HBP_NUM 1 +/* + * Maximum number of hw breakpoint supported on powerpc. Number of + * breakpoints supported by actual hw might be less than this, which + * is decided at run time in nr_wp_slots(). + */ +#define HBP_NUM_MAX 2 -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* __ASM_POWERPC_CPUTABLE_H */ diff --git a/arch/powerpc/include/asm/cputhreads.h b/arch/powerpc/include/asm/cputhreads.h index ac3eedb9b74a..d06f2b20b810 100644 --- a/arch/powerpc/include/asm/cputhreads.h +++ b/arch/powerpc/include/asm/cputhreads.h @@ -1,7 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_CPUTHREADS_H #define _ASM_POWERPC_CPUTHREADS_H +#ifndef __ASSEMBLER__ #include <linux/cpumask.h> +#include <asm/cpu_has_feature.h> /* * Mapping of threads to cores @@ -18,47 +21,20 @@ #ifdef CONFIG_SMP extern int threads_per_core; +extern int threads_per_subcore; extern int threads_shift; extern cpumask_t threads_core_mask; #else #define threads_per_core 1 +#define threads_per_subcore 1 #define threads_shift 0 -#define threads_core_mask (CPU_MASK_CPU0) +#define has_big_cores 0 +#define threads_core_mask (*get_cpu_mask(0)) #endif -/* cpu_thread_mask_to_cores - Return a cpumask of one per cores - * hit by the argument - * - * @threads: a cpumask of threads - * - * This function returns a cpumask which will have one "cpu" (or thread) - * bit set for each core that has at least one thread set in the argument. - * - * This can typically be used for things like IPI for tlb invalidations - * since those need to be done only once per core/TLB - */ -static inline cpumask_t cpu_thread_mask_to_cores(const struct cpumask *threads) -{ - cpumask_t tmp, res; - int i; - - cpumask_clear(&res); - for (i = 0; i < NR_CPUS; i += threads_per_core) { - cpumask_shift_left(&tmp, &threads_core_mask, i); - if (cpumask_intersects(threads, &tmp)) - cpumask_set_cpu(i, &res); - } - return res; -} - static inline int cpu_nr_cores(void) { - return NR_CPUS >> threads_shift; -} - -static inline cpumask_t cpu_online_cores_map(void) -{ - return cpu_thread_mask_to_cores(cpu_online_mask); + return nr_cpu_ids >> threads_shift; } #ifdef CONFIG_SMP @@ -74,6 +50,11 @@ static inline int cpu_thread_in_core(int cpu) return cpu & (threads_per_core - 1); } +static inline int cpu_thread_in_subcore(int cpu) +{ + return cpu & (threads_per_subcore - 1); +} + static inline int cpu_first_thread_sibling(int cpu) { return cpu & ~(threads_per_core - 1); @@ -84,7 +65,51 @@ static inline int cpu_last_thread_sibling(int cpu) return cpu | (threads_per_core - 1); } +/* + * tlb_thread_siblings are siblings which share a TLB. This is not + * architected, is not something a hypervisor could emulate and a future + * CPU may change behaviour even in compat mode, so this should only be + * used on PowerNV, and only with care. + */ +static inline int cpu_first_tlb_thread_sibling(int cpu) +{ + if (cpu_has_feature(CPU_FTR_ARCH_300) && (threads_per_core == 8)) + return cpu & ~0x6; /* Big Core */ + else + return cpu_first_thread_sibling(cpu); +} + +static inline int cpu_last_tlb_thread_sibling(int cpu) +{ + if (cpu_has_feature(CPU_FTR_ARCH_300) && (threads_per_core == 8)) + return cpu | 0x6; /* Big Core */ + else + return cpu_last_thread_sibling(cpu); +} + +static inline int cpu_tlb_thread_sibling_step(void) +{ + if (cpu_has_feature(CPU_FTR_ARCH_300) && (threads_per_core == 8)) + return 2; /* Big Core */ + else + return 1; +} + +static inline u32 get_tensr(void) +{ +#ifdef CONFIG_BOOKE + if (cpu_has_feature(CPU_FTR_SMT)) + return mfspr(SPRN_TENSR); +#endif + return 1; +} + +void book3e_start_thread(int thread, unsigned long addr); +void book3e_stop_thread(int thread); + +#endif /* __ASSEMBLER__ */ +#define INVALID_THREAD_HWID 0x0fff #endif /* _ASM_POWERPC_CPUTHREADS_H */ diff --git a/arch/powerpc/include/asm/cputime.h b/arch/powerpc/include/asm/cputime.h index 607559ab271f..aff858ca99c0 100644 --- a/arch/powerpc/include/asm/cputime.h +++ b/arch/powerpc/include/asm/cputime.h @@ -1,13 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Definitions for measuring cputime on powerpc machines. * * Copyright (C) 2006 Paul Mackerras, IBM Corp. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * If we have CONFIG_VIRT_CPU_ACCOUNTING_NATIVE, we measure cpu time in * the same units as the timebase. Otherwise we measure cpu time * in jiffies using the generic definitions. @@ -16,220 +12,76 @@ #ifndef __POWERPC_CPUTIME_H #define __POWERPC_CPUTIME_H -#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE -#include <asm-generic/cputime.h> -#ifdef __KERNEL__ -static inline void setup_cputime_one_jiffy(void) { } -#endif -#else +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE #include <linux/types.h> #include <linux/time.h> #include <asm/div64.h> #include <asm/time.h> #include <asm/param.h> - -typedef u64 __nocast cputime_t; -typedef u64 __nocast cputime64_t; +#include <asm/firmware.h> #ifdef __KERNEL__ +#define cputime_to_nsecs(cputime) tb_to_ns(cputime) /* - * One jiffy in timebase units computed during initialization + * PPC64 uses PACA which is task independent for storing accounting data while + * PPC32 uses struct thread_info, therefore at task switch the accounting data + * has to be populated in the new task */ -extern cputime_t cputime_one_jiffy; - -/* - * Convert cputime <-> jiffies - */ -extern u64 __cputime_jiffies_factor; -DECLARE_PER_CPU(unsigned long, cputime_last_delta); -DECLARE_PER_CPU(unsigned long, cputime_scaled_last_delta); - -static inline unsigned long cputime_to_jiffies(const cputime_t ct) -{ - return mulhdu((__force u64) ct, __cputime_jiffies_factor); -} - -/* Estimate the scaled cputime by scaling the real cputime based on - * the last scaled to real ratio */ -static inline cputime_t cputime_to_scaled(const cputime_t ct) -{ - if (cpu_has_feature(CPU_FTR_SPURR) && - __get_cpu_var(cputime_last_delta)) - return (__force u64) ct * - __get_cpu_var(cputime_scaled_last_delta) / - __get_cpu_var(cputime_last_delta); - return ct; -} +#ifdef CONFIG_PPC64 +#define get_accounting(tsk) (&get_paca()->accounting) +#define raw_get_accounting(tsk) (&local_paca->accounting) -static inline cputime_t jiffies_to_cputime(const unsigned long jif) -{ - u64 ct; - unsigned long sec; - - /* have to be a little careful about overflow */ - ct = jif % HZ; - sec = jif / HZ; - if (ct) { - ct *= tb_ticks_per_sec; - do_div(ct, HZ); - } - if (sec) - ct += (cputime_t) sec * tb_ticks_per_sec; - return (__force cputime_t) ct; -} - -static inline void setup_cputime_one_jiffy(void) -{ - cputime_one_jiffy = jiffies_to_cputime(1); -} - -static inline cputime64_t jiffies64_to_cputime64(const u64 jif) -{ - u64 ct; - u64 sec; - - /* have to be a little careful about overflow */ - ct = jif % HZ; - sec = jif / HZ; - if (ct) { - ct *= tb_ticks_per_sec; - do_div(ct, HZ); - } - if (sec) - ct += (u64) sec * tb_ticks_per_sec; - return (__force cputime64_t) ct; -} - -static inline u64 cputime64_to_jiffies64(const cputime_t ct) -{ - return mulhdu((__force u64) ct, __cputime_jiffies_factor); -} - -/* - * Convert cputime <-> microseconds - */ -extern u64 __cputime_usec_factor; - -static inline unsigned long cputime_to_usecs(const cputime_t ct) -{ - return mulhdu((__force u64) ct, __cputime_usec_factor); -} - -static inline cputime_t usecs_to_cputime(const unsigned long us) -{ - u64 ct; - unsigned long sec; - - /* have to be a little careful about overflow */ - ct = us % 1000000; - sec = us / 1000000; - if (ct) { - ct *= tb_ticks_per_sec; - do_div(ct, 1000000); - } - if (sec) - ct += (cputime_t) sec * tb_ticks_per_sec; - return (__force cputime_t) ct; -} - -#define usecs_to_cputime64(us) usecs_to_cputime(us) - -/* - * Convert cputime <-> seconds - */ -extern u64 __cputime_sec_factor; - -static inline unsigned long cputime_to_secs(const cputime_t ct) -{ - return mulhdu((__force u64) ct, __cputime_sec_factor); -} - -static inline cputime_t secs_to_cputime(const unsigned long sec) -{ - return (__force cputime_t)((u64) sec * tb_ticks_per_sec); -} +#else +#define get_accounting(tsk) (&task_thread_info(tsk)->accounting) +#define raw_get_accounting(tsk) get_accounting(tsk) +#endif /* - * Convert cputime <-> timespec + * account_cpu_user_entry/exit runs "unreconciled", so can't trace, + * can't use get_paca() */ -static inline void cputime_to_timespec(const cputime_t ct, struct timespec *p) +static notrace inline void account_cpu_user_entry(void) { - u64 x = (__force u64) ct; - unsigned int frac; + unsigned long tb = mftb(); + struct cpu_accounting_data *acct = raw_get_accounting(current); - frac = do_div(x, tb_ticks_per_sec); - p->tv_sec = x; - x = (u64) frac * 1000000000; - do_div(x, tb_ticks_per_sec); - p->tv_nsec = x; + acct->utime += (tb - acct->starttime_user); + acct->starttime = tb; } -static inline cputime_t timespec_to_cputime(const struct timespec *p) +static notrace inline void account_cpu_user_exit(void) { - u64 ct; + unsigned long tb = mftb(); + struct cpu_accounting_data *acct = raw_get_accounting(current); - ct = (u64) p->tv_nsec * tb_ticks_per_sec; - do_div(ct, 1000000000); - return (__force cputime_t)(ct + (u64) p->tv_sec * tb_ticks_per_sec); + acct->stime += (tb - acct->starttime); + acct->starttime_user = tb; } -/* - * Convert cputime <-> timeval - */ -static inline void cputime_to_timeval(const cputime_t ct, struct timeval *p) +static notrace inline void account_stolen_time(void) { - u64 x = (__force u64) ct; - unsigned int frac; +#ifdef CONFIG_PPC_SPLPAR + if (firmware_has_feature(FW_FEATURE_SPLPAR)) { + struct lppaca *lp = local_paca->lppaca_ptr; - frac = do_div(x, tb_ticks_per_sec); - p->tv_sec = x; - x = (u64) frac * 1000000; - do_div(x, tb_ticks_per_sec); - p->tv_usec = x; + if (unlikely(local_paca->dtl_ridx != be64_to_cpu(lp->dtl_idx))) + pseries_accumulate_stolen_time(); + } +#endif } -static inline cputime_t timeval_to_cputime(const struct timeval *p) +#endif /* __KERNEL__ */ +#else /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ +static inline void account_cpu_user_entry(void) { - u64 ct; - - ct = (u64) p->tv_usec * tb_ticks_per_sec; - do_div(ct, 1000000); - return (__force cputime_t)(ct + (u64) p->tv_sec * tb_ticks_per_sec); } - -/* - * Convert cputime <-> clock_t (units of 1/USER_HZ seconds) - */ -extern u64 __cputime_clockt_factor; - -static inline unsigned long cputime_to_clock_t(const cputime_t ct) +static inline void account_cpu_user_exit(void) { - return mulhdu((__force u64) ct, __cputime_clockt_factor); } - -static inline cputime_t clock_t_to_cputime(const unsigned long clk) +static notrace inline void account_stolen_time(void) { - u64 ct; - unsigned long sec; - - /* have to be a little careful about overflow */ - ct = clk % USER_HZ; - sec = clk / USER_HZ; - if (ct) { - ct *= tb_ticks_per_sec; - do_div(ct, USER_HZ); - } - if (sec) - ct += (u64) sec * tb_ticks_per_sec; - return (__force cputime_t) ct; } - -#define cputime64_to_clock_t(ct) cputime_to_clock_t((cputime_t)(ct)) - -static inline void arch_vtime_task_switch(struct task_struct *tsk) { } - -#endif /* __KERNEL__ */ #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ #endif /* __POWERPC_CPUTIME_H */ diff --git a/arch/powerpc/include/asm/crash_reserve.h b/arch/powerpc/include/asm/crash_reserve.h new file mode 100644 index 000000000000..d1b570ddbf98 --- /dev/null +++ b/arch/powerpc/include/asm/crash_reserve.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_CRASH_RESERVE_H +#define _ASM_POWERPC_CRASH_RESERVE_H + +/* crash kernel regions are Page size agliged */ +#define CRASH_ALIGN PAGE_SIZE + +#ifdef CONFIG_ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION +static inline bool arch_add_crash_res_to_iomem(void) +{ + return false; +} +#define arch_add_crash_res_to_iomem arch_add_crash_res_to_iomem +#endif + +#endif /* _ASM_POWERPC_CRASH_RESERVE_H */ diff --git a/arch/powerpc/include/asm/crashdump-ppc64.h b/arch/powerpc/include/asm/crashdump-ppc64.h new file mode 100644 index 000000000000..68d9717cc5ee --- /dev/null +++ b/arch/powerpc/include/asm/crashdump-ppc64.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _ASM_POWERPC_CRASHDUMP_PPC64_H +#define _ASM_POWERPC_CRASHDUMP_PPC64_H + +/* + * Backup region - first 64KB of System RAM + * + * If ever the below macros are to be changed, please be judicious. + * The implicit assumptions are: + * - start, end & size are less than UINT32_MAX. + * - start & size are at least 8 byte aligned. + * + * For implementation details: arch/powerpc/purgatory/trampoline_64.S + */ +#define BACKUP_SRC_START 0 +#define BACKUP_SRC_END 0xffff +#define BACKUP_SRC_SIZE (BACKUP_SRC_END - BACKUP_SRC_START + 1) + +#endif /* __ASM_POWERPC_CRASHDUMP_PPC64_H */ diff --git a/arch/powerpc/include/asm/current.h b/arch/powerpc/include/asm/current.h index e2c7f06931e7..bbfb94800415 100644 --- a/arch/powerpc/include/asm/current.h +++ b/arch/powerpc/include/asm/current.h @@ -1,12 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _ASM_POWERPC_CURRENT_H #define _ASM_POWERPC_CURRENT_H #ifdef __KERNEL__ /* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ struct task_struct; @@ -19,7 +16,8 @@ static inline struct task_struct *get_current(void) { struct task_struct *task; - __asm__ __volatile__("ld %0,%1(13)" + /* get_current can be cached by the compiler, so no volatile */ + asm ("ld %0,%1(13)" : "=r" (task) : "i" (offsetof(struct paca_struct, __current))); diff --git a/arch/powerpc/include/asm/dbdma.h b/arch/powerpc/include/asm/dbdma.h index e23f07e73cb3..4785c1716b3e 100644 --- a/arch/powerpc/include/asm/dbdma.h +++ b/arch/powerpc/include/asm/dbdma.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Definitions for using the Apple Descriptor-Based DMA controller * in Power Macintosh computers. @@ -42,12 +43,12 @@ struct dbdma_regs { * DBDMA command structure. These fields are all little-endian! */ struct dbdma_cmd { - unsigned short req_count; /* requested byte transfer count */ - unsigned short command; /* command word (has bit-fields) */ - unsigned int phy_addr; /* physical data address */ - unsigned int cmd_dep; /* command-dependent field */ - unsigned short res_count; /* residual count after completion */ - unsigned short xfer_status; /* transfer status */ + __le16 req_count; /* requested byte transfer count */ + __le16 command; /* command word (has bit-fields) */ + __le32 phy_addr; /* physical data address */ + __le32 cmd_dep; /* command-dependent field */ + __le16 res_count; /* residual count after completion */ + __le16 xfer_status; /* transfer status */ }; /* DBDMA command values in command field */ diff --git a/arch/powerpc/include/asm/dbell.h b/arch/powerpc/include/asm/dbell.h index 5fa6b20eba10..0b9ef726f92c 100644 --- a/arch/powerpc/include/asm/dbell.h +++ b/arch/powerpc/include/asm/dbell.h @@ -1,11 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Copyright 2009 Freescale Semiconductor, Inc. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * provides masks and opcode images for use by code generation, emulation * and for instructions that older assemblers might not know about */ @@ -15,7 +11,10 @@ #include <linux/smp.h> #include <linux/threads.h> +#include <asm/cputhreads.h> #include <asm/ppc-opcode.h> +#include <asm/feature-fixups.h> +#include <asm/kvm_ppc.h> #define PPC_DBELL_MSG_BRDCAST (0x04000000) #define PPC_DBELL_TYPE(x) (((x) & 0xf) << (63-36)) @@ -34,33 +33,57 @@ enum ppc_dbell { #ifdef CONFIG_PPC_BOOK3S #define PPC_DBELL_MSGTYPE PPC_DBELL_SERVER -#define SPRN_DOORBELL_CPUTAG SPRN_TIR -#define PPC_DBELL_TAG_MASK 0x7f static inline void _ppc_msgsnd(u32 msg) { - if (cpu_has_feature(CPU_FTR_HVMODE)) - __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg)); - else - __asm__ __volatile__ (PPC_MSGSNDP(%0) : : "r" (msg)); + __asm__ __volatile__ (ASM_FTR_IFSET(PPC_MSGSND(%1), PPC_MSGSNDP(%1), %0) + : : "i" (CPU_FTR_HVMODE), "r" (msg)); +} + +/* sync after taking message interrupt */ +static inline void ppc_msgsync(void) +{ + /* sync is not required when taking messages from the same core */ + __asm__ __volatile__ (ASM_FTR_IFSET(PPC_MSGSYNC " ; lwsync", "", %0) + : : "i" (CPU_FTR_HVMODE|CPU_FTR_ARCH_300)); +} + +static inline void _ppc_msgclr(u32 msg) +{ + __asm__ __volatile__ (ASM_FTR_IFSET(PPC_MSGCLR(%1), PPC_MSGCLRP(%1), %0) + : : "i" (CPU_FTR_HVMODE), "r" (msg)); +} + +static inline void ppc_msgclr(enum ppc_dbell type) +{ + u32 msg = PPC_DBELL_TYPE(type); + + _ppc_msgclr(msg); } #else /* CONFIG_PPC_BOOK3S */ #define PPC_DBELL_MSGTYPE PPC_DBELL -#define SPRN_DOORBELL_CPUTAG SPRN_PIR -#define PPC_DBELL_TAG_MASK 0x3fff static inline void _ppc_msgsnd(u32 msg) { __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg)); } +/* sync after taking message interrupt */ +static inline void ppc_msgsync(void) +{ +} + #endif /* CONFIG_PPC_BOOK3S */ -extern void doorbell_cause_ipi(int cpu, unsigned long data); extern void doorbell_exception(struct pt_regs *regs); -extern void doorbell_setup_this_cpu(void); + +/* sync before sending message */ +static inline void ppc_msgsnd_sync(void) +{ + __asm__ __volatile__ ("sync" : : : "memory"); +} static inline void ppc_msgsnd(enum ppc_dbell type, u32 flags, u32 tag) { @@ -70,4 +93,63 @@ static inline void ppc_msgsnd(enum ppc_dbell type, u32 flags, u32 tag) _ppc_msgsnd(msg); } +#ifdef CONFIG_SMP + +/* + * Doorbells must only be used if CPU_FTR_DBELL is available. + * msgsnd is used in HV, and msgsndp is used in !HV. + * + * These should be used by platform code that is aware of restrictions. + * Other arch code should use ->cause_ipi. + * + * doorbell_global_ipi() sends a dbell to any target CPU. + * Must be used only by architectures that address msgsnd target + * by PIR/get_hard_smp_processor_id. + */ +static inline void doorbell_global_ipi(int cpu) +{ + u32 tag = get_hard_smp_processor_id(cpu); + + kvmppc_set_host_ipi(cpu); + /* Order previous accesses vs. msgsnd, which is treated as a store */ + ppc_msgsnd_sync(); + ppc_msgsnd(PPC_DBELL_MSGTYPE, 0, tag); +} + +/* + * doorbell_core_ipi() sends a dbell to a target CPU in the same core. + * Must be used only by architectures that address msgsnd target + * by TIR/cpu_thread_in_core. + */ +static inline void doorbell_core_ipi(int cpu) +{ + u32 tag = cpu_thread_in_core(cpu); + + kvmppc_set_host_ipi(cpu); + /* Order previous accesses vs. msgsnd, which is treated as a store */ + ppc_msgsnd_sync(); + ppc_msgsnd(PPC_DBELL_MSGTYPE, 0, tag); +} + +/* + * Attempt to cause a core doorbell if destination is on the same core. + * Returns 1 on success, 0 on failure. + */ +static inline int doorbell_try_core_ipi(int cpu) +{ + int this_cpu = get_cpu(); + int ret = 0; + + if (cpumask_test_cpu(cpu, cpu_sibling_mask(this_cpu))) { + doorbell_core_ipi(cpu); + ret = 1; + } + + put_cpu(); + + return ret; +} + +#endif /* CONFIG_SMP */ + #endif /* _ASM_POWERPC_DBELL_H */ diff --git a/arch/powerpc/include/asm/dcr-generic.h b/arch/powerpc/include/asm/dcr-generic.h deleted file mode 100644 index 35b71599ec46..000000000000 --- a/arch/powerpc/include/asm/dcr-generic.h +++ /dev/null @@ -1,49 +0,0 @@ -/* - * (c) Copyright 2006 Benjamin Herrenschmidt, IBM Corp. - * <benh@kernel.crashing.org> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See - * the GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#ifndef _ASM_POWERPC_DCR_GENERIC_H -#define _ASM_POWERPC_DCR_GENERIC_H -#ifdef __KERNEL__ -#ifndef __ASSEMBLY__ - -enum host_type_t {DCR_HOST_MMIO, DCR_HOST_NATIVE, DCR_HOST_INVALID}; - -typedef struct { - enum host_type_t type; - union { - dcr_host_mmio_t mmio; - dcr_host_native_t native; - } host; -} dcr_host_t; - -extern bool dcr_map_ok_generic(dcr_host_t host); - -extern dcr_host_t dcr_map_generic(struct device_node *dev, unsigned int dcr_n, - unsigned int dcr_c); -extern void dcr_unmap_generic(dcr_host_t host, unsigned int dcr_c); - -extern u32 dcr_read_generic(dcr_host_t host, unsigned int dcr_n); - -extern void dcr_write_generic(dcr_host_t host, unsigned int dcr_n, u32 value); - -#endif /* __ASSEMBLY__ */ -#endif /* __KERNEL__ */ -#endif /* _ASM_POWERPC_DCR_GENERIC_H */ - - diff --git a/arch/powerpc/include/asm/dcr-mmio.h b/arch/powerpc/include/asm/dcr-mmio.h deleted file mode 100644 index acd491dbd45a..000000000000 --- a/arch/powerpc/include/asm/dcr-mmio.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - * (c) Copyright 2006 Benjamin Herrenschmidt, IBM Corp. - * <benh@kernel.crashing.org> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See - * the GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#ifndef _ASM_POWERPC_DCR_MMIO_H -#define _ASM_POWERPC_DCR_MMIO_H -#ifdef __KERNEL__ - -#include <asm/io.h> - -typedef struct { - void __iomem *token; - unsigned int stride; - unsigned int base; -} dcr_host_mmio_t; - -static inline bool dcr_map_ok_mmio(dcr_host_mmio_t host) -{ - return host.token != NULL; -} - -extern dcr_host_mmio_t dcr_map_mmio(struct device_node *dev, - unsigned int dcr_n, - unsigned int dcr_c); -extern void dcr_unmap_mmio(dcr_host_mmio_t host, unsigned int dcr_c); - -static inline u32 dcr_read_mmio(dcr_host_mmio_t host, unsigned int dcr_n) -{ - return in_be32(host.token + ((host.base + dcr_n) * host.stride)); -} - -static inline void dcr_write_mmio(dcr_host_mmio_t host, - unsigned int dcr_n, - u32 value) -{ - out_be32(host.token + ((host.base + dcr_n) * host.stride), value); -} - -extern u64 of_translate_dcr_address(struct device_node *dev, - unsigned int dcr_n, - unsigned int *stride); - -#endif /* __KERNEL__ */ -#endif /* _ASM_POWERPC_DCR_MMIO_H */ - - diff --git a/arch/powerpc/include/asm/dcr-native.h b/arch/powerpc/include/asm/dcr-native.h index 7d2e6235726d..65b3fc2dc404 100644 --- a/arch/powerpc/include/asm/dcr-native.h +++ b/arch/powerpc/include/asm/dcr-native.h @@ -1,29 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * (c) Copyright 2006 Benjamin Herrenschmidt, IBM Corp. * <benh@kernel.crashing.org> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See - * the GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef _ASM_POWERPC_DCR_NATIVE_H #define _ASM_POWERPC_DCR_NATIVE_H #ifdef __KERNEL__ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/spinlock.h> #include <asm/cputable.h> +#include <asm/cpu_has_feature.h> +#include <linux/stringify.h> typedef struct { unsigned int base; @@ -31,7 +20,7 @@ typedef struct { static inline bool dcr_map_ok_native(dcr_host_native_t host) { - return 1; + return true; } #define dcr_map_native(dev, dcr_n, dcr_c) \ @@ -64,8 +53,8 @@ static inline void mtdcrx(unsigned int reg, unsigned int val) #define mfdcr(rn) \ ({unsigned int rval; \ if (__builtin_constant_p(rn) && rn < 1024) \ - asm volatile("mfdcr %0," __stringify(rn) \ - : "=r" (rval)); \ + asm volatile("mfdcr %0, %1" : "=r" (rval) \ + : "n" (rn)); \ else if (likely(cpu_has_feature(CPU_FTR_INDEXED_DCR))) \ rval = mfdcrx(rn); \ else \ @@ -75,8 +64,8 @@ static inline void mtdcrx(unsigned int reg, unsigned int val) #define mtdcr(rn, v) \ do { \ if (__builtin_constant_p(rn) && rn < 1024) \ - asm volatile("mtdcr " __stringify(rn) ",%0" \ - : : "r" (v)); \ + asm volatile("mtdcr %0, %1" \ + : : "n" (rn), "r" (v)); \ else if (likely(cpu_has_feature(CPU_FTR_INDEXED_DCR))) \ mtdcrx(rn, v); \ else \ @@ -150,6 +139,6 @@ static inline void __dcri_clrset(int base_addr, int base_data, int reg, DCRN_ ## base ## _CONFIG_DATA, \ reg, clr, set) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_DCR_NATIVE_H */ diff --git a/arch/powerpc/include/asm/dcr-regs.h b/arch/powerpc/include/asm/dcr-regs.h index 380274de429f..5c1a4973f46a 100644 --- a/arch/powerpc/include/asm/dcr-regs.h +++ b/arch/powerpc/include/asm/dcr-regs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Common DCR / SDR / CPR register definitions used on various IBM/AMCC * 4xx processors diff --git a/arch/powerpc/include/asm/dcr.h b/arch/powerpc/include/asm/dcr.h index 9d6851cfb841..3c0fac2cc2b2 100644 --- a/arch/powerpc/include/asm/dcr.h +++ b/arch/powerpc/include/asm/dcr.h @@ -1,68 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * (c) Copyright 2006 Benjamin Herrenschmidt, IBM Corp. * <benh@kernel.crashing.org> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See - * the GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef _ASM_POWERPC_DCR_H #define _ASM_POWERPC_DCR_H #ifdef __KERNEL__ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #ifdef CONFIG_PPC_DCR -#ifdef CONFIG_PPC_DCR_NATIVE #include <asm/dcr-native.h> -#endif -#ifdef CONFIG_PPC_DCR_MMIO -#include <asm/dcr-mmio.h> -#endif - - -/* Indirection layer for providing both NATIVE and MMIO support. */ - -#if defined(CONFIG_PPC_DCR_NATIVE) && defined(CONFIG_PPC_DCR_MMIO) - -#include <asm/dcr-generic.h> - -#define DCR_MAP_OK(host) dcr_map_ok_generic(host) -#define dcr_map(dev, dcr_n, dcr_c) dcr_map_generic(dev, dcr_n, dcr_c) -#define dcr_unmap(host, dcr_c) dcr_unmap_generic(host, dcr_c) -#define dcr_read(host, dcr_n) dcr_read_generic(host, dcr_n) -#define dcr_write(host, dcr_n, value) dcr_write_generic(host, dcr_n, value) - -#else - -#ifdef CONFIG_PPC_DCR_NATIVE typedef dcr_host_native_t dcr_host_t; #define DCR_MAP_OK(host) dcr_map_ok_native(host) #define dcr_map(dev, dcr_n, dcr_c) dcr_map_native(dev, dcr_n, dcr_c) #define dcr_unmap(host, dcr_c) dcr_unmap_native(host, dcr_c) #define dcr_read(host, dcr_n) dcr_read_native(host, dcr_n) #define dcr_write(host, dcr_n, value) dcr_write_native(host, dcr_n, value) -#else -typedef dcr_host_mmio_t dcr_host_t; -#define DCR_MAP_OK(host) dcr_map_ok_mmio(host) -#define dcr_map(dev, dcr_n, dcr_c) dcr_map_mmio(dev, dcr_n, dcr_c) -#define dcr_unmap(host, dcr_c) dcr_unmap_mmio(host, dcr_c) -#define dcr_read(host, dcr_n) dcr_read_mmio(host, dcr_n) -#define dcr_write(host, dcr_n, value) dcr_write_mmio(host, dcr_n, value) -#endif - -#endif /* defined(CONFIG_PPC_DCR_NATIVE) && defined(CONFIG_PPC_DCR_MMIO) */ /* * additional helpers to read the DCR * base from the device-tree @@ -73,6 +28,6 @@ extern unsigned int dcr_resource_start(const struct device_node *np, extern unsigned int dcr_resource_len(const struct device_node *np, unsigned int index); #endif /* CONFIG_PPC_DCR */ -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_DCR_H */ diff --git a/arch/powerpc/include/asm/debug.h b/arch/powerpc/include/asm/debug.h index d2516308ed1e..51c744608f37 100644 --- a/arch/powerpc/include/asm/debug.h +++ b/arch/powerpc/include/asm/debug.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu> */ @@ -8,9 +9,7 @@ struct pt_regs; -extern struct dentry *powerpc_debugfs_root; - -#if defined(CONFIG_DEBUGGER) || defined(CONFIG_KEXEC) +#if defined(CONFIG_DEBUGGER) || defined(CONFIG_KEXEC_CORE) extern int (*__debugger)(struct pt_regs *regs); extern int (*__debugger_ipi)(struct pt_regs *regs); @@ -46,14 +45,13 @@ static inline int debugger_break_match(struct pt_regs *regs) { return 0; } static inline int debugger_fault_handler(struct pt_regs *regs) { return 0; } #endif -int set_breakpoint(struct arch_hw_breakpoint *brk); +void __set_breakpoint(int nr, struct arch_hw_breakpoint *brk); +void suspend_breakpoints(void); +void restore_breakpoints(void); +bool ppc_breakpoint_available(void); #ifdef CONFIG_PPC_ADV_DEBUG_REGS extern void do_send_trap(struct pt_regs *regs, unsigned long address, - unsigned long error_code, int signal_code, int brkpt); -#else - -extern void do_break(struct pt_regs *regs, unsigned long address, - unsigned long error_code); + unsigned long error_code, int brkpt); #endif #endif /* _ASM_POWERPC_DEBUG_H */ diff --git a/arch/powerpc/include/asm/delay.h b/arch/powerpc/include/asm/delay.h index 52e4d54da2a9..51bb8c1476c7 100644 --- a/arch/powerpc/include/asm/delay.h +++ b/arch/powerpc/include/asm/delay.h @@ -1,18 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _ASM_POWERPC_DELAY_H #define _ASM_POWERPC_DELAY_H #ifdef __KERNEL__ +#include <linux/processor.h> #include <asm/time.h> /* * Copyright 1996, Paul Mackerras. * Copyright (C) 2009 Freescale Semiconductor, Inc. All rights reserved. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * PPC64 Support added by Dave Engebretsen, Todd Inglett, Mike Corrigan, * Anton Blanchard. */ @@ -57,12 +54,19 @@ extern void udelay(unsigned long usecs); ({ \ typeof(condition) __ret; \ unsigned long __loops = tb_ticks_per_usec * timeout; \ - unsigned long __start = get_tbl(); \ - while (!(__ret = (condition)) && (tb_ticks_since(__start) <= __loops)) \ - if (delay) \ + unsigned long __start = mftb(); \ + \ + if (delay) { \ + while (!(__ret = (condition)) && \ + (tb_ticks_since(__start) <= __loops)) \ udelay(delay); \ - else \ - cpu_relax(); \ + } else { \ + spin_begin(); \ + while (!(__ret = (condition)) && \ + (tb_ticks_since(__start) <= __loops)) \ + spin_cpu_relax(); \ + spin_end(); \ + } \ if (!__ret) \ __ret = (condition); \ __ret; \ diff --git a/arch/powerpc/include/asm/device.h b/arch/powerpc/include/asm/device.h index 77e97dd0c15d..a4dc27655b3e 100644 --- a/arch/powerpc/include/asm/device.h +++ b/arch/powerpc/include/asm/device.h @@ -1,13 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Arch specific extensions to struct device - * - * This file is released under the GPLv2 */ #ifndef _ASM_POWERPC_DEVICE_H #define _ASM_POWERPC_DEVICE_H -struct dma_map_ops; struct device_node; +#ifdef CONFIG_PPC64 +struct pci_dn; +struct iommu_table; +#endif /* * Arch extensions to struct device. @@ -16,20 +18,19 @@ struct device_node; * drivers/macintosh/macio_asic.c */ struct dev_archdata { - /* DMA operations on that device */ - struct dma_map_ops *dma_ops; - /* - * When an iommu is in use, dma_data is used as a ptr to the base of the - * iommu_table. Otherwise, it is a simple numerical offset. + * These two used to be a union. However, with the hybrid ops we need + * both so here we store both a DMA offset for direct mappings and + * an iommu_table for remapped DMA. */ - union { - dma_addr_t dma_offset; - void *iommu_table_base; - } dma_data; + dma_addr_t dma_offset; + +#ifdef CONFIG_PPC64 + struct iommu_table *iommu_table_base; +#endif -#ifdef CONFIG_SWIOTLB - dma_addr_t max_direct_dma_addr; +#ifdef CONFIG_PPC64 + struct pci_dn *pci_data; #endif #ifdef CONFIG_EEH struct eeh_dev *edev; @@ -37,12 +38,18 @@ struct dev_archdata { #ifdef CONFIG_FAIL_IOMMU int fail_iommu; #endif +#ifdef CONFIG_PCI_IOV + void *iov_data; +#endif }; struct pdev_archdata { u64 dma_mask; + /* + * Pointer to nvdimm_pmu structure, to handle the unregistering + * of pmu device + */ + void *priv; }; -#define ARCH_HAS_DMA_GET_REQUIRED_MASK - #endif /* _ASM_POWERPC_DEVICE_H */ diff --git a/arch/powerpc/include/asm/disassemble.h b/arch/powerpc/include/asm/disassemble.h index 9b198d1b3b2b..8d2ebc36d5e3 100644 --- a/arch/powerpc/include/asm/disassemble.h +++ b/arch/powerpc/include/asm/disassemble.h @@ -1,16 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * * Copyright IBM Corp. 2008 * @@ -42,6 +31,11 @@ static inline unsigned int get_dcrn(u32 inst) return ((inst >> 16) & 0x1f) | ((inst >> 6) & 0x3e0); } +static inline unsigned int get_tmrn(u32 inst) +{ + return ((inst >> 16) & 0x1f) | ((inst >> 6) & 0x3e0); +} + static inline unsigned int get_rt(u32 inst) { return (inst >> 21) & 0x1f; @@ -77,4 +71,47 @@ static inline unsigned int get_d(u32 inst) return inst & 0xffff; } +static inline unsigned int get_oc(u32 inst) +{ + return (inst >> 11) & 0x7fff; +} + +static inline unsigned int get_tx_or_sx(u32 inst) +{ + return (inst) & 0x1; +} + +#define IS_XFORM(inst) (get_op(inst) == 31) +#define IS_DSFORM(inst) (get_op(inst) >= 56) + +/* + * Create a DSISR value from the instruction + */ +static inline unsigned make_dsisr(unsigned instr) +{ + unsigned dsisr; + + + /* bits 6:15 --> 22:31 */ + dsisr = (instr & 0x03ff0000) >> 16; + + if (IS_XFORM(instr)) { + /* bits 29:30 --> 15:16 */ + dsisr |= (instr & 0x00000006) << 14; + /* bit 25 --> 17 */ + dsisr |= (instr & 0x00000040) << 8; + /* bits 21:24 --> 18:21 */ + dsisr |= (instr & 0x00000780) << 3; + } else { + /* bit 5 --> 17 */ + dsisr |= (instr & 0x04000000) >> 12; + /* bits 1: 4 --> 18:21 */ + dsisr |= (instr & 0x78000000) >> 17; + /* bits 30:31 --> 12:13 */ + if (IS_DSFORM(instr)) + dsisr |= (instr & 0x00000003) << 18; + } + + return dsisr; +} #endif /* __ASM_PPC_DISASSEMBLE_H__ */ diff --git a/arch/powerpc/include/asm/div64.h b/arch/powerpc/include/asm/div64.h deleted file mode 100644 index 6cd978cefb28..000000000000 --- a/arch/powerpc/include/asm/div64.h +++ /dev/null @@ -1 +0,0 @@ -#include <asm-generic/div64.h> diff --git a/arch/powerpc/include/asm/dma-direct.h b/arch/powerpc/include/asm/dma-direct.h new file mode 100644 index 000000000000..128304cbee1d --- /dev/null +++ b/arch/powerpc/include/asm/dma-direct.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ASM_POWERPC_DMA_DIRECT_H +#define ASM_POWERPC_DMA_DIRECT_H 1 + +static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) +{ + return paddr + dev->archdata.dma_offset; +} + +static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr) +{ + return daddr - dev->archdata.dma_offset; +} +#endif /* ASM_POWERPC_DMA_DIRECT_H */ diff --git a/arch/powerpc/include/asm/dma-mapping.h b/arch/powerpc/include/asm/dma-mapping.h deleted file mode 100644 index e27e9ad6818e..000000000000 --- a/arch/powerpc/include/asm/dma-mapping.h +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright (C) 2004 IBM - * - * Implements the generic device dma API for powerpc. - * the pci and vio busses - */ -#ifndef _ASM_DMA_MAPPING_H -#define _ASM_DMA_MAPPING_H -#ifdef __KERNEL__ - -#include <linux/types.h> -#include <linux/cache.h> -/* need struct page definitions */ -#include <linux/mm.h> -#include <linux/scatterlist.h> -#include <linux/dma-attrs.h> -#include <linux/dma-debug.h> -#include <asm/io.h> -#include <asm/swiotlb.h> - -#define DMA_ERROR_CODE (~(dma_addr_t)0x0) - -/* Some dma direct funcs must be visible for use in other dma_ops */ -extern void *dma_direct_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs); -extern void dma_direct_free_coherent(struct device *dev, size_t size, - void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs); -extern int dma_direct_mmap_coherent(struct device *dev, - struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t handle, - size_t size, struct dma_attrs *attrs); - -#ifdef CONFIG_NOT_COHERENT_CACHE -/* - * DMA-consistent mapping functions for PowerPCs that don't support - * cache snooping. These allocate/free a region of uncached mapped - * memory space for use with DMA devices. Alternatively, you could - * allocate the space "normally" and use the cache management functions - * to ensure it is consistent. - */ -struct device; -extern void *__dma_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t gfp); -extern void __dma_free_coherent(size_t size, void *vaddr); -extern void __dma_sync(void *vaddr, size_t size, int direction); -extern void __dma_sync_page(struct page *page, unsigned long offset, - size_t size, int direction); -extern unsigned long __dma_get_coherent_pfn(unsigned long cpu_addr); - -#else /* ! CONFIG_NOT_COHERENT_CACHE */ -/* - * Cache coherent cores. - */ - -#define __dma_alloc_coherent(dev, gfp, size, handle) NULL -#define __dma_free_coherent(size, addr) ((void)0) -#define __dma_sync(addr, size, rw) ((void)0) -#define __dma_sync_page(pg, off, sz, rw) ((void)0) - -#endif /* ! CONFIG_NOT_COHERENT_CACHE */ - -static inline unsigned long device_to_mask(struct device *dev) -{ - if (dev->dma_mask && *dev->dma_mask) - return *dev->dma_mask; - /* Assume devices without mask can take 32 bit addresses */ - return 0xfffffffful; -} - -/* - * Available generic sets of operations - */ -#ifdef CONFIG_PPC64 -extern struct dma_map_ops dma_iommu_ops; -#endif -extern struct dma_map_ops dma_direct_ops; - -static inline struct dma_map_ops *get_dma_ops(struct device *dev) -{ - /* We don't handle the NULL dev case for ISA for now. We could - * do it via an out of line call but it is not needed for now. The - * only ISA DMA device we support is the floppy and we have a hack - * in the floppy driver directly to get a device for us. - */ - if (unlikely(dev == NULL)) - return NULL; - - return dev->archdata.dma_ops; -} - -static inline void set_dma_ops(struct device *dev, struct dma_map_ops *ops) -{ - dev->archdata.dma_ops = ops; -} - -/* - * get_dma_offset() - * - * Get the dma offset on configurations where the dma address can be determined - * from the physical address by looking at a simple offset. Direct dma and - * swiotlb use this function, but it is typically not used by implementations - * with an iommu. - */ -static inline dma_addr_t get_dma_offset(struct device *dev) -{ - if (dev) - return dev->archdata.dma_data.dma_offset; - - return PCI_DRAM_OFFSET; -} - -static inline void set_dma_offset(struct device *dev, dma_addr_t off) -{ - if (dev) - dev->archdata.dma_data.dma_offset = off; -} - -/* this will be removed soon */ -#define flush_write_buffers() - -#include <asm-generic/dma-mapping-common.h> - -static inline int dma_supported(struct device *dev, u64 mask) -{ - struct dma_map_ops *dma_ops = get_dma_ops(dev); - - if (unlikely(dma_ops == NULL)) - return 0; - if (dma_ops->dma_supported == NULL) - return 1; - return dma_ops->dma_supported(dev, mask); -} - -extern int dma_set_mask(struct device *dev, u64 dma_mask); - -#define dma_alloc_coherent(d,s,h,f) dma_alloc_attrs(d,s,h,f,NULL) - -static inline void *dma_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) -{ - struct dma_map_ops *dma_ops = get_dma_ops(dev); - void *cpu_addr; - - BUG_ON(!dma_ops); - - cpu_addr = dma_ops->alloc(dev, size, dma_handle, flag, attrs); - - debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr); - - return cpu_addr; -} - -#define dma_free_coherent(d,s,c,h) dma_free_attrs(d,s,c,h,NULL) - -static inline void dma_free_attrs(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t dma_handle, - struct dma_attrs *attrs) -{ - struct dma_map_ops *dma_ops = get_dma_ops(dev); - - BUG_ON(!dma_ops); - - debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); - - dma_ops->free(dev, size, cpu_addr, dma_handle, attrs); -} - -static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - struct dma_map_ops *dma_ops = get_dma_ops(dev); - - debug_dma_mapping_error(dev, dma_addr); - if (dma_ops->mapping_error) - return dma_ops->mapping_error(dev, dma_addr); - -#ifdef CONFIG_PPC64 - return (dma_addr == DMA_ERROR_CODE); -#else - return 0; -#endif -} - -static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) -{ -#ifdef CONFIG_SWIOTLB - struct dev_archdata *sd = &dev->archdata; - - if (sd->max_direct_dma_addr && addr + size > sd->max_direct_dma_addr) - return 0; -#endif - - if (!dev->dma_mask) - return 0; - - return addr + size - 1 <= *dev->dma_mask; -} - -static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) -{ - return paddr + get_dma_offset(dev); -} - -static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr) -{ - return daddr - get_dma_offset(dev); -} - -#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) -#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) - -#define ARCH_HAS_DMA_MMAP_COHERENT - -static inline void dma_cache_sync(struct device *dev, void *vaddr, size_t size, - enum dma_data_direction direction) -{ - BUG_ON(direction == DMA_NONE); - __dma_sync(vaddr, size, (int)direction); -} - -#endif /* __KERNEL__ */ -#endif /* _ASM_DMA_MAPPING_H */ diff --git a/arch/powerpc/include/asm/dma.h b/arch/powerpc/include/asm/dma.h index a5c6d83b5f60..d97c66d9ae34 100644 --- a/arch/powerpc/include/asm/dma.h +++ b/arch/powerpc/include/asm/dma.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_DMA_H #define _ASM_POWERPC_DMA_H #ifdef __KERNEL__ @@ -150,10 +151,9 @@ #define DMA2_EXT_REG 0x4D6 #ifndef __powerpc64__ - /* in arch/ppc/kernel/setup.c -- Cort */ + /* in arch/powerpc/kernel/setup_32.c -- Cort */ extern unsigned int DMA_MODE_WRITE; extern unsigned int DMA_MODE_READ; - extern unsigned long ISA_DMA_THRESHOLD; #else #define DMA_MODE_READ 0x44 /* I/O to memory, no autoinit, increment, single mode */ #define DMA_MODE_WRITE 0x48 /* memory to I/O, no autoinit, increment, single mode */ @@ -340,11 +340,5 @@ extern int request_dma(unsigned int dmanr, const char *device_id); /* release it again */ extern void free_dma(unsigned int dmanr); -#ifdef CONFIG_PCI -extern int isa_dma_bridge_buggy; -#else -#define isa_dma_bridge_buggy (0) -#endif - #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_DMA_H */ diff --git a/arch/powerpc/include/asm/drmem.h b/arch/powerpc/include/asm/drmem.h new file mode 100644 index 000000000000..13bf6dee8e2d --- /dev/null +++ b/arch/powerpc/include/asm/drmem.h @@ -0,0 +1,125 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * drmem.h: Power specific logical memory block representation + * + * Copyright 2017 IBM Corporation + */ + +#ifndef _ASM_POWERPC_LMB_H +#define _ASM_POWERPC_LMB_H + +#include <linux/sched.h> + +struct drmem_lmb { + u64 base_addr; + u32 drc_index; + u32 aa_index; + u32 flags; +}; + +struct drmem_lmb_info { + struct drmem_lmb *lmbs; + int n_lmbs; + u64 lmb_size; +}; + +struct device_node; +struct property; + +extern struct drmem_lmb_info *drmem_info; + +static inline struct drmem_lmb *drmem_lmb_next(struct drmem_lmb *lmb, + const struct drmem_lmb *start) +{ + /* + * DLPAR code paths can take several milliseconds per element + * when interacting with firmware. Ensure that we don't + * unfairly monopolize the CPU. + */ + if (((++lmb - start) % 16) == 0) + cond_resched(); + + return lmb; +} + +#define for_each_drmem_lmb_in_range(lmb, start, end) \ + for ((lmb) = (start); (lmb) < (end); lmb = drmem_lmb_next(lmb, start)) + +#define for_each_drmem_lmb(lmb) \ + for_each_drmem_lmb_in_range((lmb), \ + &drmem_info->lmbs[0], \ + &drmem_info->lmbs[drmem_info->n_lmbs]) + +/* + * The of_drconf_cell_v1 struct defines the layout of the LMB data + * specified in the ibm,dynamic-memory device tree property. + * The property itself is a 32-bit value specifying the number of + * LMBs followed by an array of of_drconf_cell_v1 entries, one + * per LMB. + */ +struct of_drconf_cell_v1 { + __be64 base_addr; + __be32 drc_index; + __be32 reserved; + __be32 aa_index; + __be32 flags; +}; + +/* + * Version 2 of the ibm,dynamic-memory property is defined as a + * 32-bit value specifying the number of LMB sets followed by an + * array of of_drconf_cell_v2 entries, one per LMB set. + */ +struct of_drconf_cell_v2 { + u32 seq_lmbs; + u64 base_addr; + u32 drc_index; + u32 aa_index; + u32 flags; +} __packed; + +#define DRCONF_MEM_ASSIGNED 0x00000008 +#define DRCONF_MEM_AI_INVALID 0x00000040 +#define DRCONF_MEM_RESERVED 0x00000080 +#define DRCONF_MEM_HOTREMOVABLE 0x00000100 + +static inline u64 drmem_lmb_size(void) +{ + return drmem_info->lmb_size; +} + +#define DRMEM_LMB_RESERVED 0x80000000 + +static inline void drmem_mark_lmb_reserved(struct drmem_lmb *lmb) +{ + lmb->flags |= DRMEM_LMB_RESERVED; +} + +static inline void drmem_remove_lmb_reservation(struct drmem_lmb *lmb) +{ + lmb->flags &= ~DRMEM_LMB_RESERVED; +} + +static inline bool drmem_lmb_reserved(struct drmem_lmb *lmb) +{ + return lmb->flags & DRMEM_LMB_RESERVED; +} + +u64 drmem_lmb_memory_max(void); +int walk_drmem_lmbs(struct device_node *dn, void *data, + int (*func)(struct drmem_lmb *, const __be32 **, void *)); +int drmem_update_dt(void); + +#ifdef CONFIG_PPC_PSERIES +int __init +walk_drmem_lmbs_early(unsigned long node, void *data, + int (*func)(struct drmem_lmb *, const __be32 **, void *)); +void drmem_update_lmbs(struct property *prop); +#endif + +static inline void invalidate_lmb_associativity_index(struct drmem_lmb *lmb) +{ + lmb->aa_index = 0xffffffff; +} + +#endif /* _ASM_POWERPC_LMB_H */ diff --git a/arch/powerpc/include/asm/dt_cpu_ftrs.h b/arch/powerpc/include/asm/dt_cpu_ftrs.h new file mode 100644 index 000000000000..0c729e2d0e8a --- /dev/null +++ b/arch/powerpc/include/asm/dt_cpu_ftrs.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_POWERPC_DT_CPU_FTRS_H +#define __ASM_POWERPC_DT_CPU_FTRS_H + +/* + * Copyright 2017, IBM Corporation + * cpufeatures is the new way to discover CPU features with /cpus/features + * devicetree. This supersedes PVR based discovery ("cputable"), and older + * device tree feature advertisement. + */ + +#include <linux/types.h> +#include <uapi/asm/cputable.h> + +#ifdef CONFIG_PPC_DT_CPU_FTRS +bool dt_cpu_ftrs_init(void *fdt); +void dt_cpu_ftrs_scan(void); +bool dt_cpu_ftrs_in_use(void); +#else +static inline bool dt_cpu_ftrs_init(void *fdt) { return false; } +static inline void dt_cpu_ftrs_scan(void) { } +static inline bool dt_cpu_ftrs_in_use(void) { return false; } +#endif + +#endif /* __ASM_POWERPC_DT_CPU_FTRS_H */ diff --git a/arch/powerpc/include/asm/dtl.h b/arch/powerpc/include/asm/dtl.h new file mode 100644 index 000000000000..a5c21bc623cb --- /dev/null +++ b/arch/powerpc/include/asm/dtl.h @@ -0,0 +1,43 @@ +#ifndef _ASM_POWERPC_DTL_H +#define _ASM_POWERPC_DTL_H + +#include <linux/rwsem.h> +#include <asm/lppaca.h> + +/* + * Layout of entries in the hypervisor's dispatch trace log buffer. + */ +struct dtl_entry { + u8 dispatch_reason; + u8 preempt_reason; + __be16 processor_id; + __be32 enqueue_to_dispatch_time; + __be32 ready_to_enqueue_time; + __be32 waiting_to_ready_time; + __be64 timebase; + __be64 fault_addr; + __be64 srr0; + __be64 srr1; +}; + +#define DISPATCH_LOG_BYTES 4096 /* bytes per cpu */ +#define N_DISPATCH_LOG (DISPATCH_LOG_BYTES / sizeof(struct dtl_entry)) + +/* + * Dispatch trace log event enable mask: + * 0x1: voluntary virtual processor waits + * 0x2: time-slice preempts + * 0x4: virtual partition memory page faults + */ +#define DTL_LOG_CEDE 0x1 +#define DTL_LOG_PREEMPT 0x2 +#define DTL_LOG_FAULT 0x4 +#define DTL_LOG_ALL (DTL_LOG_CEDE | DTL_LOG_PREEMPT | DTL_LOG_FAULT) + +extern struct kmem_cache *dtl_cache; +extern struct rw_semaphore dtl_access_lock; + +extern void register_dtl_buffer(int cpu); +extern void alloc_dtl_buffers(unsigned long *time_limit); + +#endif /* _ASM_POWERPC_DTL_H */ diff --git a/arch/powerpc/include/asm/edac.h b/arch/powerpc/include/asm/edac.h index 6ead88bbfbb8..5571e23d253e 100644 --- a/arch/powerpc/include/asm/edac.h +++ b/arch/powerpc/include/asm/edac.h @@ -12,11 +12,11 @@ #define ASM_EDAC_H /* * ECC atomic, DMA, SMP and interrupt safe scrub function. - * Implements the per arch atomic_scrub() that EDAC use for software + * Implements the per arch edac_atomic_scrub() that EDAC use for software * ECC scrubbing. It reads memory and then writes back the original * value, allowing the hardware to detect and correct memory errors. */ -static __inline__ void atomic_scrub(void *va, u32 size) +static __inline__ void edac_atomic_scrub(void *va, u32 size) { unsigned int *virt_addr = va; unsigned int temp; diff --git a/arch/powerpc/include/asm/eeh.h b/arch/powerpc/include/asm/eeh.h index 09a8743143f3..5e34611de9ef 100644 --- a/arch/powerpc/include/asm/eeh.h +++ b/arch/powerpc/include/asm/eeh.h @@ -1,20 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Copyright (C) 2001 Dave Engebretsen & Todd Inglett IBM Corporation. * Copyright 2001-2012 IBM Corporation. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef _POWERPC_EEH_H @@ -25,17 +12,38 @@ #include <linux/list.h> #include <linux/string.h> #include <linux/time.h> +#include <linux/atomic.h> + +#include <uapi/asm/eeh.h> struct pci_dev; struct pci_bus; -struct device_node; +struct pci_dn; #ifdef CONFIG_EEH +/* EEH subsystem flags */ +#define EEH_ENABLED 0x01 /* EEH enabled */ +#define EEH_FORCE_DISABLED 0x02 /* EEH disabled */ +#define EEH_PROBE_MODE_DEV 0x04 /* From PCI device */ +#define EEH_PROBE_MODE_DEVTREE 0x08 /* From device tree */ +#define EEH_ENABLE_IO_FOR_LOG 0x20 /* Enable IO for log */ +#define EEH_EARLY_DUMP_LOG 0x40 /* Dump log immediately */ + +/* + * Delay for PE reset, all in ms + * + * PCI specification has reset hold time of 100 milliseconds. + * We have 250 milliseconds here. The PCI bus settlement time + * is specified as 1.5 seconds and we have 1.8 seconds. + */ +#define EEH_PE_RST_HOLD_TIME 250 +#define EEH_PE_RST_SETTLE_TIME 1800 + /* * The struct is used to trace PE related EEH functionality. * In theory, there will have one instance of the struct to - * be created against particular PE. In nature, PEs corelate + * be created against particular PE. In nature, PEs correlate * to each other. the struct has to reflect that hierarchy in * order to easily pick up those affected PEs when one particular * PE has EEH errors. @@ -50,30 +58,59 @@ struct device_node; #define EEH_PE_PHB (1 << 1) /* PHB PE */ #define EEH_PE_DEVICE (1 << 2) /* Device PE */ #define EEH_PE_BUS (1 << 3) /* Bus PE */ +#define EEH_PE_VF (1 << 4) /* VF PE */ #define EEH_PE_ISOLATED (1 << 0) /* Isolated PE */ #define EEH_PE_RECOVERING (1 << 1) /* Recovering PE */ -#define EEH_PE_PHB_DEAD (1 << 2) /* Dead PHB */ +#define EEH_PE_CFG_BLOCKED (1 << 2) /* Block config access */ +#define EEH_PE_RESET (1 << 3) /* PE reset in progress */ + +#define EEH_PE_KEEP (1 << 8) /* Keep PE on hotplug */ +#define EEH_PE_CFG_RESTRICTED (1 << 9) /* Block config on error */ +#define EEH_PE_REMOVED (1 << 10) /* Removed permanently */ +#define EEH_PE_PRI_BUS (1 << 11) /* Cached primary bus */ struct eeh_pe { int type; /* PE type: PHB/Bus/Device */ int state; /* PE EEH dependent mode */ - int config_addr; /* Traditional PCI address */ int addr; /* PE configuration address */ struct pci_controller *phb; /* Associated PHB */ struct pci_bus *bus; /* Top PCI bus for bus PE */ int check_count; /* Times of ignored error */ int freeze_count; /* Times of froze up */ - struct timeval tstamp; /* Time on first-time freeze */ + time64_t tstamp; /* Time on first-time freeze */ int false_positives; /* Times of reported #ff's */ + atomic_t pass_dev_cnt; /* Count of passed through devs */ struct eeh_pe *parent; /* Parent PE */ - struct list_head child_list; /* Link PE to the child list */ - struct list_head edevs; /* Link list of EEH devices */ - struct list_head child; /* Child PEs */ + void *data; /* PE auxiliary data */ + struct list_head child_list; /* List of PEs below this PE */ + struct list_head child; /* Memb. child_list/eeh_phb_pe */ + struct list_head edevs; /* List of eeh_dev in this PE */ + +#ifdef CONFIG_STACKTRACE + /* + * Saved stack trace. When we find a PE freeze in eeh_dev_check_failure + * the stack trace is saved here so we can print it in the recovery + * thread if it turns out to due to a real problem rather than + * a hot-remove. + * + * A max of 64 entries might be overkill, but it also might not be. + */ + unsigned long stack_trace[64]; + int trace_entries; +#endif /* CONFIG_STACKTRACE */ }; -#define eeh_pe_for_each_dev(pe, edev) \ - list_for_each_entry(edev, &pe->edevs, list) +#define eeh_pe_for_each_dev(pe, edev, tmp) \ + list_for_each_entry_safe(edev, tmp, &pe->edevs, entry) + +#define eeh_for_each_pe(root, pe) \ + for (pe = root; pe; pe = eeh_pe_next(pe, root)) + +static inline bool eeh_pe_passed(struct eeh_pe *pe) +{ + return pe ? !!atomic_read(&pe->pass_dev_cnt) : false; +} /* * The struct is used to trace EEH state for the associated @@ -82,24 +119,52 @@ struct eeh_pe { * another tree except the currently existing tree of PCI * buses and PCI devices */ -#define EEH_DEV_IRQ_DISABLED (1<<0) /* Interrupt disabled */ +#define EEH_DEV_BRIDGE (1 << 0) /* PCI bridge */ +#define EEH_DEV_ROOT_PORT (1 << 1) /* PCIe root port */ +#define EEH_DEV_DS_PORT (1 << 2) /* Downstream port */ +#define EEH_DEV_IRQ_DISABLED (1 << 3) /* Interrupt disabled */ +#define EEH_DEV_DISCONNECTED (1 << 4) /* Removing from PE */ + +#define EEH_DEV_NO_HANDLER (1 << 8) /* No error handler */ +#define EEH_DEV_SYSFS (1 << 9) /* Sysfs created */ +#define EEH_DEV_REMOVED (1 << 10) /* Removed permanently */ struct eeh_dev { int mode; /* EEH mode */ - int class_code; /* Class code of the device */ - int config_addr; /* Config address */ + int bdfn; /* bdfn of device (for cfg ops) */ + struct pci_controller *controller; int pe_config_addr; /* PE config address */ u32 config_space[16]; /* Saved PCI config space */ + int pcix_cap; /* Saved PCIx capability */ + int pcie_cap; /* Saved PCIe capability */ + int aer_cap; /* Saved AER capability */ + int af_cap; /* Saved AF capability */ struct eeh_pe *pe; /* Associated PE */ - struct list_head list; /* Form link list in the PE */ - struct pci_controller *phb; /* Associated PHB */ - struct device_node *dn; /* Associated device node */ + struct list_head entry; /* Membership in eeh_pe.edevs */ + struct list_head rmv_entry; /* Membership in rmv_list */ + struct pci_dn *pdn; /* Associated PCI device node */ struct pci_dev *pdev; /* Associated PCI device */ + bool in_error; /* Error flag for edev */ + + /* VF specific properties */ + struct pci_dev *physfn; /* Associated SRIOV PF */ + int vf_index; /* Index of this VF */ }; -static inline struct device_node *eeh_dev_to_of_node(struct eeh_dev *edev) +/* "fmt" must be a simple literal string */ +#define EEH_EDEV_PRINT(level, edev, fmt, ...) \ + pr_##level("PCI %04x:%02x:%02x.%x#%04x: EEH: " fmt, \ + (edev)->controller->global_number, PCI_BUSNO((edev)->bdfn), \ + PCI_SLOT((edev)->bdfn), PCI_FUNC((edev)->bdfn), \ + ((edev)->pe ? (edev)->pe_config_addr : 0xffff), ##__VA_ARGS__) +#define eeh_edev_dbg(edev, fmt, ...) EEH_EDEV_PRINT(debug, (edev), fmt, ##__VA_ARGS__) +#define eeh_edev_info(edev, fmt, ...) EEH_EDEV_PRINT(info, (edev), fmt, ##__VA_ARGS__) +#define eeh_edev_warn(edev, fmt, ...) EEH_EDEV_PRINT(warn, (edev), fmt, ##__VA_ARGS__) +#define eeh_edev_err(edev, fmt, ...) EEH_EDEV_PRINT(err, (edev), fmt, ##__VA_ARGS__) + +static inline struct pci_dn *eeh_dev_to_pdn(struct eeh_dev *edev) { - return edev ? edev->dn : NULL; + return edev ? edev->pdn : NULL; } static inline struct pci_dev *eeh_dev_to_pci_dev(struct eeh_dev *edev) @@ -107,6 +172,21 @@ static inline struct pci_dev *eeh_dev_to_pci_dev(struct eeh_dev *edev) return edev ? edev->pdev : NULL; } +static inline struct eeh_pe *eeh_dev_to_pe(struct eeh_dev* edev) +{ + return edev ? edev->pe : NULL; +} + +/* Return values from eeh_ops::next_error */ +enum { + EEH_NEXT_ERR_NONE = 0, + EEH_NEXT_ERR_INF, + EEH_NEXT_ERR_FROZEN_PE, + EEH_NEXT_ERR_FENCED_PHB, + EEH_NEXT_ERR_DEAD_PHB, + EEH_NEXT_ERR_DEAD_IOC +}; + /* * The struct is used to trace the registered EEH operation * callback functions. Actually, those operation callback @@ -118,6 +198,7 @@ static inline struct pci_dev *eeh_dev_to_pci_dev(struct eeh_dev *edev) #define EEH_OPT_ENABLE 1 /* EEH enable */ #define EEH_OPT_THAW_MMIO 2 /* MMIO enable */ #define EEH_OPT_THAW_DMA 3 /* DMA enable */ +#define EEH_OPT_FREEZE_PE 4 /* Freeze PE */ #define EEH_STATE_UNAVAILABLE (1 << 0) /* State unavailable */ #define EEH_STATE_NOT_SUPPORT (1 << 1) /* EEH not supported */ #define EEH_STATE_RESET_ACTIVE (1 << 2) /* Active reset */ @@ -133,43 +214,45 @@ static inline struct pci_dev *eeh_dev_to_pci_dev(struct eeh_dev *edev) struct eeh_ops { char *name; - int (*init)(void); - int (*post_init)(void); - void* (*of_probe)(struct device_node *dn, void *flag); - int (*dev_probe)(struct pci_dev *dev, void *flag); + struct eeh_dev *(*probe)(struct pci_dev *pdev); int (*set_option)(struct eeh_pe *pe, int option); - int (*get_pe_addr)(struct eeh_pe *pe); - int (*get_state)(struct eeh_pe *pe, int *state); + int (*get_state)(struct eeh_pe *pe, int *delay); int (*reset)(struct eeh_pe *pe, int option); - int (*wait_state)(struct eeh_pe *pe, int max_wait); int (*get_log)(struct eeh_pe *pe, int severity, char *drv_log, unsigned long len); int (*configure_bridge)(struct eeh_pe *pe); - int (*read_config)(struct device_node *dn, int where, int size, u32 *val); - int (*write_config)(struct device_node *dn, int where, int size, u32 val); + int (*err_inject)(struct eeh_pe *pe, int type, int func, + unsigned long addr, unsigned long mask); + int (*read_config)(struct eeh_dev *edev, int where, int size, u32 *val); + int (*write_config)(struct eeh_dev *edev, int where, int size, u32 val); int (*next_error)(struct eeh_pe **pe); + int (*restore_config)(struct eeh_dev *edev); + int (*notify_resume)(struct eeh_dev *edev); }; +extern int eeh_subsystem_flags; +extern u32 eeh_max_freezes; +extern bool eeh_debugfs_no_recover; extern struct eeh_ops *eeh_ops; -extern int eeh_subsystem_enabled; extern raw_spinlock_t confirm_error_lock; -extern int eeh_probe_mode; -#define EEH_PROBE_MODE_DEV (1<<0) /* From PCI device */ -#define EEH_PROBE_MODE_DEVTREE (1<<1) /* From device tree */ +static inline void eeh_add_flag(int flag) +{ + eeh_subsystem_flags |= flag; +} -static inline void eeh_probe_mode_set(int flag) +static inline void eeh_clear_flag(int flag) { - eeh_probe_mode = flag; + eeh_subsystem_flags &= ~flag; } -static inline int eeh_probe_mode_devtree(void) +static inline bool eeh_has_flag(int flag) { - return (eeh_probe_mode == EEH_PROBE_MODE_DEVTREE); + return !!(eeh_subsystem_flags & flag); } -static inline int eeh_probe_mode_dev(void) +static inline bool eeh_enabled(void) { - return (eeh_probe_mode == EEH_PROBE_MODE_DEV); + return eeh_has_flag(EEH_ENABLED) && !eeh_has_flag(EEH_FORCE_DISABLED); } static inline void eeh_serialize_lock(unsigned long *flags) @@ -182,37 +265,50 @@ static inline void eeh_serialize_unlock(unsigned long flags) raw_spin_unlock_irqrestore(&confirm_error_lock, flags); } -/* - * Max number of EEH freezes allowed before we consider the device - * to be permanently disabled. - */ -#define EEH_MAX_ALLOWED_FREEZES 5 +static inline bool eeh_state_active(int state) +{ + return (state & (EEH_STATE_MMIO_ACTIVE | EEH_STATE_DMA_ACTIVE)) + == (EEH_STATE_MMIO_ACTIVE | EEH_STATE_DMA_ACTIVE); +} -typedef void *(*eeh_traverse_func)(void *data, void *flag); +typedef void (*eeh_edev_traverse_func)(struct eeh_dev *edev, void *flag); +typedef void *(*eeh_pe_traverse_func)(struct eeh_pe *pe, void *flag); +void eeh_set_pe_aux_size(int size); int eeh_phb_pe_create(struct pci_controller *phb); +int eeh_wait_state(struct eeh_pe *pe, int max_wait); struct eeh_pe *eeh_phb_pe_get(struct pci_controller *phb); -struct eeh_pe *eeh_pe_get(struct eeh_dev *edev); -int eeh_add_to_parent_pe(struct eeh_dev *edev); -int eeh_rmv_from_parent_pe(struct eeh_dev *edev, int purge_pe); +struct eeh_pe *eeh_pe_next(struct eeh_pe *pe, struct eeh_pe *root); +struct eeh_pe *eeh_pe_get(struct pci_controller *phb, int pe_no); +int eeh_pe_tree_insert(struct eeh_dev *edev, struct eeh_pe *new_pe_parent); +int eeh_pe_tree_remove(struct eeh_dev *edev); void eeh_pe_update_time_stamp(struct eeh_pe *pe); -void *eeh_pe_dev_traverse(struct eeh_pe *root, - eeh_traverse_func fn, void *flag); +void *eeh_pe_traverse(struct eeh_pe *root, + eeh_pe_traverse_func fn, void *flag); +void eeh_pe_dev_traverse(struct eeh_pe *root, + eeh_edev_traverse_func fn, void *flag); void eeh_pe_restore_bars(struct eeh_pe *pe); +const char *eeh_pe_loc_get(struct eeh_pe *pe); struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe); -void *eeh_dev_init(struct device_node *dn, void *data); -void eeh_dev_phb_init_dynamic(struct pci_controller *phb); -int eeh_init(void); -int __init eeh_ops_register(struct eeh_ops *ops); -int __exit eeh_ops_unregister(const char *name); -unsigned long eeh_check_failure(const volatile void __iomem *token, - unsigned long val); +void eeh_show_enabled(void); +int __init eeh_init(struct eeh_ops *ops); +int eeh_check_failure(const volatile void __iomem *token); int eeh_dev_check_failure(struct eeh_dev *edev); -void eeh_addr_cache_build(void); -void eeh_add_device_tree_early(struct device_node *); -void eeh_add_device_tree_late(struct pci_bus *); -void eeh_add_sysfs_files(struct pci_bus *); -void eeh_remove_bus_device(struct pci_dev *, int); +void eeh_addr_cache_init(void); +void eeh_probe_device(struct pci_dev *pdev); +void eeh_remove_device(struct pci_dev *); +int eeh_unfreeze_pe(struct eeh_pe *pe); +int eeh_pe_reset_and_recover(struct eeh_pe *pe); +int eeh_dev_open(struct pci_dev *pdev); +void eeh_dev_release(struct pci_dev *pdev); +struct eeh_pe *eeh_iommu_group_to_pe(struct iommu_group *group); +int eeh_pe_set_option(struct eeh_pe *pe, int option); +int eeh_pe_get_state(struct eeh_pe *pe); +int eeh_pe_reset(struct eeh_pe *pe, int option, bool include_passed); +int eeh_pe_configure(struct eeh_pe *pe); +int eeh_pe_inject_err(struct eeh_pe *pe, int type, int func, + unsigned long addr, unsigned long mask); +int eeh_pe_inject_mmio_error(struct pci_dev *pdev); /** * EEH_POSSIBLE_ERROR() -- test for possible MMIO failure. @@ -220,7 +316,7 @@ void eeh_remove_bus_device(struct pci_dev *, int); * If this macro yields TRUE, the caller relays to eeh_check_failure() * which does further tests out of line. */ -#define EEH_POSSIBLE_ERROR(val, type) ((val) == (type)~0 && eeh_subsystem_enabled) +#define EEH_POSSIBLE_ERROR(val, type) ((val) == (type)~0 && eeh_enabled()) /* * Reads from a device which has been isolated by EEH will return @@ -231,39 +327,35 @@ void eeh_remove_bus_device(struct pci_dev *, int); #else /* !CONFIG_EEH */ -static inline int eeh_init(void) -{ - return 0; -} - -static inline void *eeh_dev_init(struct device_node *dn, void *data) +static inline bool eeh_enabled(void) { - return NULL; + return false; } -static inline void eeh_dev_phb_init_dynamic(struct pci_controller *phb) { } +static inline void eeh_show_enabled(void) { } -static inline unsigned long eeh_check_failure(const volatile void __iomem *token, unsigned long val) +static inline int eeh_check_failure(const volatile void __iomem *token) { - return val; + return 0; } #define eeh_dev_check_failure(x) (0) -static inline void eeh_addr_cache_build(void) { } +static inline void eeh_addr_cache_init(void) { } -static inline void eeh_add_device_tree_early(struct device_node *dn) { } +static inline void eeh_probe_device(struct pci_dev *dev) { } -static inline void eeh_add_device_tree_late(struct pci_bus *bus) { } - -static inline void eeh_add_sysfs_files(struct pci_bus *bus) { } - -static inline void eeh_remove_bus_device(struct pci_dev *dev, int purge_pe) { } +static inline void eeh_remove_device(struct pci_dev *dev) { } #define EEH_POSSIBLE_ERROR(val, type) (0) #define EEH_IO_ERROR_VALUE(size) (-1UL) +static inline int eeh_phb_pe_create(struct pci_controller *phb) { return 0; } #endif /* CONFIG_EEH */ +#if defined(CONFIG_PPC_PSERIES) && defined(CONFIG_EEH) +void pseries_eeh_init_edev_recursive(struct pci_dn *pdn); +#endif + #ifdef CONFIG_PPC64 /* * MMIO read/write operations with EEH support. @@ -272,7 +364,7 @@ static inline u8 eeh_readb(const volatile void __iomem *addr) { u8 val = in_8(addr); if (EEH_POSSIBLE_ERROR(val, u8)) - return eeh_check_failure(addr, val); + eeh_check_failure(addr); return val; } @@ -280,7 +372,7 @@ static inline u16 eeh_readw(const volatile void __iomem *addr) { u16 val = in_le16(addr); if (EEH_POSSIBLE_ERROR(val, u16)) - return eeh_check_failure(addr, val); + eeh_check_failure(addr); return val; } @@ -288,7 +380,7 @@ static inline u32 eeh_readl(const volatile void __iomem *addr) { u32 val = in_le32(addr); if (EEH_POSSIBLE_ERROR(val, u32)) - return eeh_check_failure(addr, val); + eeh_check_failure(addr); return val; } @@ -296,7 +388,7 @@ static inline u64 eeh_readq(const volatile void __iomem *addr) { u64 val = in_le64(addr); if (EEH_POSSIBLE_ERROR(val, u64)) - return eeh_check_failure(addr, val); + eeh_check_failure(addr); return val; } @@ -304,7 +396,7 @@ static inline u16 eeh_readw_be(const volatile void __iomem *addr) { u16 val = in_be16(addr); if (EEH_POSSIBLE_ERROR(val, u16)) - return eeh_check_failure(addr, val); + eeh_check_failure(addr); return val; } @@ -312,7 +404,7 @@ static inline u32 eeh_readl_be(const volatile void __iomem *addr) { u32 val = in_be32(addr); if (EEH_POSSIBLE_ERROR(val, u32)) - return eeh_check_failure(addr, val); + eeh_check_failure(addr); return val; } @@ -320,7 +412,7 @@ static inline u64 eeh_readq_be(const volatile void __iomem *addr) { u64 val = in_be64(addr); if (EEH_POSSIBLE_ERROR(val, u64)) - return eeh_check_failure(addr, val); + eeh_check_failure(addr); return val; } @@ -334,7 +426,7 @@ static inline void eeh_memcpy_fromio(void *dest, const * were copied. Check all four bytes. */ if (n >= 4 && EEH_POSSIBLE_ERROR(*((u32 *)(dest + n - 4)), u32)) - eeh_check_failure(src, *((u32 *)(dest + n - 4))); + eeh_check_failure(src); } /* in-string eeh macros */ @@ -343,7 +435,7 @@ static inline void eeh_readsb(const volatile void __iomem *addr, void * buf, { _insb(addr, buf, ns); if (EEH_POSSIBLE_ERROR((*(((u8*)buf)+ns-1)), u8)) - eeh_check_failure(addr, *(u8*)buf); + eeh_check_failure(addr); } static inline void eeh_readsw(const volatile void __iomem *addr, void * buf, @@ -351,7 +443,7 @@ static inline void eeh_readsw(const volatile void __iomem *addr, void * buf, { _insw(addr, buf, ns); if (EEH_POSSIBLE_ERROR((*(((u16*)buf)+ns-1)), u16)) - eeh_check_failure(addr, *(u16*)buf); + eeh_check_failure(addr); } static inline void eeh_readsl(const volatile void __iomem *addr, void * buf, @@ -359,9 +451,12 @@ static inline void eeh_readsl(const volatile void __iomem *addr, void * buf, { _insl(addr, buf, nl); if (EEH_POSSIBLE_ERROR((*(((u32*)buf)+nl-1)), u32)) - eeh_check_failure(addr, *(u32*)buf); + eeh_check_failure(addr); } + +void __init eeh_cache_debugfs_init(void); + #endif /* CONFIG_PPC64 */ #endif /* __KERNEL__ */ #endif /* _POWERPC_EEH_H */ diff --git a/arch/powerpc/include/asm/eeh_event.h b/arch/powerpc/include/asm/eeh_event.h index 89d5670b2eeb..dadde7d52f46 100644 --- a/arch/powerpc/include/asm/eeh_event.h +++ b/arch/powerpc/include/asm/eeh_event.h @@ -1,17 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA * * Copyright (c) 2005 Linas Vepstas <linas@linas.org> */ @@ -33,8 +21,10 @@ struct eeh_event { int eeh_event_init(void); int eeh_send_failure_event(struct eeh_pe *pe); -void eeh_remove_event(struct eeh_pe *pe); -void eeh_handle_event(struct eeh_pe *pe); +int __eeh_send_failure_event(struct eeh_pe *pe); +void eeh_remove_event(struct eeh_pe *pe, bool force); +void eeh_handle_normal_event(struct eeh_pe *pe); +void eeh_handle_special_event(void); #endif /* __KERNEL__ */ #endif /* ASM_POWERPC_EEH_EVENT_H */ diff --git a/arch/powerpc/include/asm/elf.h b/arch/powerpc/include/asm/elf.h index cc0655a702a7..bb4b94444d3e 100644 --- a/arch/powerpc/include/asm/elf.h +++ b/arch/powerpc/include/asm/elf.h @@ -1,10 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * ELF register definitions.. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef _ASM_POWERPC_ELF_H #define _ASM_POWERPC_ELF_H @@ -23,13 +19,15 @@ #define CORE_DUMP_USE_REGSET #define ELF_EXEC_PAGESIZE PAGE_SIZE -/* This is the location that an ET_DYN program is loaded if exec'ed. Typical - use of this is to invoke "./ld.so someprog" to test out a new version of - the loader. We need to make sure that it is out of the way of the program - that it will "exec", and that there is sufficient room for the brk. */ +/* + * This is the base location for PIE (ET_DYN with INTERP) loads. On + * 64-bit, this is raised to 4GB to leave the entire 32-bit address + * space open for things that want to use the area for 32-bit pointers. + */ +#define ELF_ET_DYN_BASE (is_32bit_task() ? 0x000400000UL : \ + 0x100000000UL) -extern unsigned long randomize_et_dyn(unsigned long base); -#define ELF_ET_DYN_BASE (randomize_et_dyn(0x20000000)) +#define ELF_CORE_EFLAGS (is_elf2_task() ? 2 : 0) /* * Our registers are always unsigned longs, whether we're a 32 bit @@ -55,8 +53,6 @@ static inline void ppc_elf_core_copy_regs(elf_gregset_t elf_regs, } #define ELF_CORE_COPY_REGS(gregs, regs) ppc_elf_core_copy_regs(gregs, regs); -typedef elf_vrregset_t elf_fpxregset_t; - /* ELF_HWCAP yields a mask that user programs can use to figure out what instruction set this cpu supports. This could be done in userspace, but it's not easy, and we've already done it here. */ @@ -86,6 +82,10 @@ typedef elf_vrregset_t elf_fpxregset_t; #ifdef __powerpc64__ # define SET_PERSONALITY(ex) \ do { \ + if (((ex).e_flags & 0x3) == 2) \ + set_thread_flag(TIF_ELF2ABI); \ + else \ + clear_thread_flag(TIF_ELF2ABI); \ if ((ex).e_ident[EI_CLASS] == ELFCLASS32) \ set_thread_flag(TIF_32BIT); \ else \ @@ -123,16 +123,73 @@ extern int arch_setup_additional_pages(struct linux_binprm *bprm, (0x7ff >> (PAGE_SHIFT - 12)) : \ (0x3ffff >> (PAGE_SHIFT - 12))) -extern unsigned long arch_randomize_brk(struct mm_struct *mm); -#define arch_randomize_brk arch_randomize_brk - - #ifdef CONFIG_SPU_BASE /* Notes used in ET_CORE. Note name is "SPU/<fd>/<filename>". */ #define NT_SPU 1 -#define ARCH_HAVE_EXTRA_ELF_NOTES - #endif /* CONFIG_SPU_BASE */ +#ifdef CONFIG_PPC64 + +#define get_cache_geometry(level) \ + (ppc64_caches.level.assoc << 16 | ppc64_caches.level.line_size) + +#define ARCH_DLINFO_CACHE_GEOMETRY \ + NEW_AUX_ENT(AT_L1I_CACHESIZE, ppc64_caches.l1i.size); \ + NEW_AUX_ENT(AT_L1I_CACHEGEOMETRY, get_cache_geometry(l1i)); \ + NEW_AUX_ENT(AT_L1D_CACHESIZE, ppc64_caches.l1d.size); \ + NEW_AUX_ENT(AT_L1D_CACHEGEOMETRY, get_cache_geometry(l1d)); \ + NEW_AUX_ENT(AT_L2_CACHESIZE, ppc64_caches.l2.size); \ + NEW_AUX_ENT(AT_L2_CACHEGEOMETRY, get_cache_geometry(l2)); \ + NEW_AUX_ENT(AT_L3_CACHESIZE, ppc64_caches.l3.size); \ + NEW_AUX_ENT(AT_L3_CACHEGEOMETRY, get_cache_geometry(l3)) + +#else +#define ARCH_DLINFO_CACHE_GEOMETRY +#endif + +/* + * The requirements here are: + * - keep the final alignment of sp (sp & 0xf) + * - make sure the 32-bit value at the first 16 byte aligned position of + * AUXV is greater than 16 for glibc compatibility. + * AT_IGNOREPPC is used for that. + * - for compatibility with glibc ARCH_DLINFO must always be defined on PPC, + * even if DLINFO_ARCH_ITEMS goes to zero or is undefined. + * update AT_VECTOR_SIZE_ARCH if the number of NEW_AUX_ENT entries changes + */ +#define COMMON_ARCH_DLINFO \ +do { \ + /* Handle glibc compatibility. */ \ + NEW_AUX_ENT(AT_IGNOREPPC, AT_IGNOREPPC); \ + NEW_AUX_ENT(AT_IGNOREPPC, AT_IGNOREPPC); \ + /* Cache size items */ \ + NEW_AUX_ENT(AT_DCACHEBSIZE, dcache_bsize); \ + NEW_AUX_ENT(AT_ICACHEBSIZE, icache_bsize); \ + NEW_AUX_ENT(AT_UCACHEBSIZE, 0); \ + VDSO_AUX_ENT(AT_SYSINFO_EHDR, (unsigned long)current->mm->context.vdso);\ + ARCH_DLINFO_CACHE_GEOMETRY; \ +} while (0) + +#define ARCH_DLINFO \ +do { \ + COMMON_ARCH_DLINFO; \ + NEW_AUX_ENT(AT_MINSIGSTKSZ, get_min_sigframe_size()); \ +} while (0) + +#define COMPAT_ARCH_DLINFO \ +do { \ + COMMON_ARCH_DLINFO; \ + NEW_AUX_ENT(AT_MINSIGSTKSZ, get_min_sigframe_size_compat()); \ +} while (0) + +/* Relocate the kernel image to @final_address */ +void relocate(unsigned long final_address); + +struct func_desc { + unsigned long addr; + unsigned long toc; + unsigned long env; +}; + #endif /* _ASM_POWERPC_ELF_H */ diff --git a/arch/powerpc/include/asm/elfnote.h b/arch/powerpc/include/asm/elfnote.h new file mode 100644 index 000000000000..a201b6e9ae44 --- /dev/null +++ b/arch/powerpc/include/asm/elfnote.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * PowerPC ELF notes. + * + * Copyright 2019, IBM Corporation + */ + +#ifndef __ASM_POWERPC_ELFNOTE_H__ +#define __ASM_POWERPC_ELFNOTE_H__ + +/* + * These note types should live in a SHT_NOTE segment and have + * "PowerPC" in the name field. + */ + +/* + * The capabilities supported/required by this kernel (bitmap). + * + * This type uses a bitmap as "desc" field. Each bit is described + * in arch/powerpc/kernel/note.S + */ +#define PPC_ELFNOTE_CAPABILITIES 1 + +#endif /* __ASM_POWERPC_ELFNOTE_H__ */ diff --git a/arch/powerpc/include/asm/emulated_ops.h b/arch/powerpc/include/asm/emulated_ops.h index 63f2a22e9954..800cb21000cf 100644 --- a/arch/powerpc/include/asm/emulated_ops.h +++ b/arch/powerpc/include/asm/emulated_ops.h @@ -1,18 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright 2007 Sony Corporation - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. - * If not, see <http://www.gnu.org/licenses/>. */ #ifndef _ASM_POWERPC_EMULATED_OPS_H @@ -43,11 +31,10 @@ extern struct ppc_emulated { struct ppc_emulated_entry popcntb; struct ppc_emulated_entry spe; struct ppc_emulated_entry string; + struct ppc_emulated_entry sync; struct ppc_emulated_entry unaligned; #ifdef CONFIG_MATH_EMULATION struct ppc_emulated_entry math; -#elif defined(CONFIG_8XX_MINIMAL_FPEMU) - struct ppc_emulated_entry 8xx; #endif #ifdef CONFIG_VSX struct ppc_emulated_entry vsx; @@ -55,6 +42,11 @@ extern struct ppc_emulated { #ifdef CONFIG_PPC64 struct ppc_emulated_entry mfdscr; struct ppc_emulated_entry mtdscr; + struct ppc_emulated_entry lq_stq; + struct ppc_emulated_entry lxvw4x; + struct ppc_emulated_entry lxvh8x; + struct ppc_emulated_entry lxvd2x; + struct ppc_emulated_entry lxvb16x; #endif } ppc_emulated; diff --git a/arch/powerpc/include/asm/epapr_hcalls.h b/arch/powerpc/include/asm/epapr_hcalls.h index d3d634274d2c..8fc5aaa4bbba 100644 --- a/arch/powerpc/include/asm/epapr_hcalls.h +++ b/arch/powerpc/include/asm/epapr_hcalls.h @@ -37,7 +37,7 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/* A "hypercall" is an "sc 1" instruction. This header file file provides C +/* A "hypercall" is an "sc 1" instruction. This header file provides C * wrapper functions for the ePAPR hypervisor interface. It is inteded * for use by Linux device drivers and other operating systems. * @@ -52,7 +52,7 @@ #include <uapi/asm/epapr_hcalls.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> #include <linux/errno.h> #include <asm/byteorder.h> @@ -65,7 +65,7 @@ * but the gcc inline assembly syntax does not allow us to specify registers * on the clobber list that are also on the input/output list. Therefore, * the lists of clobbered registers depends on the number of register - * parmeters ("+r" and "=r") passed to the hypercall. + * parameters ("+r" and "=r") passed to the hypercall. * * Each assembly block should use one of the HCALL_CLOBBERSx macros. As a * general rule, 'x' is the number of parameters passed to the assembly @@ -105,6 +105,12 @@ extern bool epapr_paravirt_enabled; extern u32 epapr_hypercall_start[]; +#ifdef CONFIG_EPAPR_PARAVIRT +int __init epapr_paravirt_early_init(void); +#else +static inline int epapr_paravirt_early_init(void) { return 0; } +#endif + /* * We use "uintptr_t" to define a register because it's guaranteed to be a * 32-bit integer on a 32-bit platform, and a 64-bit integer on a 64-bit @@ -240,7 +246,7 @@ static inline unsigned int ev_int_get_mask(unsigned int interrupt, * ev_int_eoi - signal the end of interrupt processing * @interrupt: the interrupt number * - * This function signals the end of processing for the the specified + * This function signals the end of processing for the specified * interrupt, which must be the interrupt currently in service. By * definition, this is also the highest-priority interrupt. * @@ -454,5 +460,116 @@ static inline unsigned int ev_idle(void) return r3; } -#endif /* !__ASSEMBLY__ */ + +#ifdef CONFIG_EPAPR_PARAVIRT +static inline unsigned long epapr_hypercall(unsigned long *in, + unsigned long *out, + unsigned long nr) +{ + register unsigned long r0 asm("r0"); + register unsigned long r3 asm("r3") = in[0]; + register unsigned long r4 asm("r4") = in[1]; + register unsigned long r5 asm("r5") = in[2]; + register unsigned long r6 asm("r6") = in[3]; + register unsigned long r7 asm("r7") = in[4]; + register unsigned long r8 asm("r8") = in[5]; + register unsigned long r9 asm("r9") = in[6]; + register unsigned long r10 asm("r10") = in[7]; + register unsigned long r11 asm("r11") = nr; + register unsigned long r12 asm("r12"); + + asm volatile("bl epapr_hypercall_start" + : "=r"(r0), "=r"(r3), "=r"(r4), "=r"(r5), "=r"(r6), + "=r"(r7), "=r"(r8), "=r"(r9), "=r"(r10), "=r"(r11), + "=r"(r12) + : "r"(r3), "r"(r4), "r"(r5), "r"(r6), "r"(r7), "r"(r8), + "r"(r9), "r"(r10), "r"(r11) + : "memory", "cc", "xer", "ctr", "lr"); + + out[0] = r4; + out[1] = r5; + out[2] = r6; + out[3] = r7; + out[4] = r8; + out[5] = r9; + out[6] = r10; + out[7] = r11; + + return r3; +} +#else +static unsigned long epapr_hypercall(unsigned long *in, + unsigned long *out, + unsigned long nr) +{ + return EV_UNIMPLEMENTED; +} +#endif + +static inline long epapr_hypercall0_1(unsigned int nr, unsigned long *r2) +{ + unsigned long in[8] = {0}; + unsigned long out[8]; + unsigned long r; + + r = epapr_hypercall(in, out, nr); + *r2 = out[0]; + + return r; +} + +static inline long epapr_hypercall0(unsigned int nr) +{ + unsigned long in[8] = {0}; + unsigned long out[8]; + + return epapr_hypercall(in, out, nr); +} + +static inline long epapr_hypercall1(unsigned int nr, unsigned long p1) +{ + unsigned long in[8] = {0}; + unsigned long out[8]; + + in[0] = p1; + return epapr_hypercall(in, out, nr); +} + +static inline long epapr_hypercall2(unsigned int nr, unsigned long p1, + unsigned long p2) +{ + unsigned long in[8] = {0}; + unsigned long out[8]; + + in[0] = p1; + in[1] = p2; + return epapr_hypercall(in, out, nr); +} + +static inline long epapr_hypercall3(unsigned int nr, unsigned long p1, + unsigned long p2, unsigned long p3) +{ + unsigned long in[8] = {0}; + unsigned long out[8]; + + in[0] = p1; + in[1] = p2; + in[2] = p3; + return epapr_hypercall(in, out, nr); +} + +static inline long epapr_hypercall4(unsigned int nr, unsigned long p1, + unsigned long p2, unsigned long p3, + unsigned long p4) +{ + unsigned long in[8] = {0}; + unsigned long out[8]; + + in[0] = p1; + in[1] = p2; + in[2] = p3; + in[3] = p4; + return epapr_hypercall(in, out, nr); +} +#endif /* !__ASSEMBLER__ */ #endif /* _EPAPR_HCALLS_H */ diff --git a/arch/powerpc/include/asm/exception-64e.h b/arch/powerpc/include/asm/exception-64e.h index 51fa43e536b9..1a83b1ff3578 100644 --- a/arch/powerpc/include/asm/exception-64e.h +++ b/arch/powerpc/include/asm/exception-64e.h @@ -1,12 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Definitions for use by exception code on Book3-E * * Copyright (C) 2008 Ben. Herrenschmidt (benh@kernel.crashing.org), IBM Corp. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef _ASM_POWERPC_EXCEPTION_64E_H #define _ASM_POWERPC_EXCEPTION_64E_H @@ -46,9 +42,8 @@ #define EX_CR (1 * 8) #define EX_R10 (2 * 8) #define EX_R11 (3 * 8) -#define EX_R13 (4 * 8) -#define EX_R14 (5 * 8) -#define EX_R15 (6 * 8) +#define EX_R14 (4 * 8) +#define EX_R15 (5 * 8) /* * The TLB miss exception uses different slots. @@ -70,14 +65,8 @@ #define EX_TLB_ESR ( 9 * 8) /* Level 0 and 2 only */ #define EX_TLB_SRR0 (10 * 8) #define EX_TLB_SRR1 (11 * 8) -#ifdef CONFIG_BOOK3E_MMU_TLB_STATS -#define EX_TLB_R8 (12 * 8) -#define EX_TLB_R9 (13 * 8) -#define EX_TLB_LR (14 * 8) -#define EX_TLB_SIZE (15 * 8) -#else -#define EX_TLB_SIZE (12 * 8) -#endif +#define EX_TLB_R7 (12 * 8) +#define EX_TLB_SIZE (13 * 8) #define START_EXCEPTION(label) \ .globl exc_##label##_book3e; \ @@ -114,8 +103,7 @@ exc_##label##_book3e: std r11,EX_TLB_R12(r12); \ mtspr SPRN_SPRG_TLB_EXFRAME,r14; \ std r15,EX_TLB_SRR1(r12); \ - std r16,EX_TLB_SRR0(r12); \ - TLB_MISS_PROLOG_STATS + std r16,EX_TLB_SRR0(r12); /* And these are the matching epilogs that restores things * @@ -147,7 +135,6 @@ exc_##label##_book3e: mtspr SPRN_SRR0,r15; \ ld r15,EX_TLB_R15(r12); \ mtspr SPRN_SRR1,r16; \ - TLB_MISS_RESTORE_STATS \ ld r16,EX_TLB_R16(r12); \ ld r12,EX_TLB_R12(r12); \ @@ -162,62 +149,24 @@ exc_##label##_book3e: addi r11,r13,PACA_EXTLB; \ TLB_MISS_RESTORE(r11) -#ifdef CONFIG_BOOK3E_MMU_TLB_STATS -#define TLB_MISS_PROLOG_STATS \ - mflr r10; \ - std r8,EX_TLB_R8(r12); \ - std r9,EX_TLB_R9(r12); \ - std r10,EX_TLB_LR(r12); -#define TLB_MISS_RESTORE_STATS \ - ld r16,EX_TLB_LR(r12); \ - ld r9,EX_TLB_R9(r12); \ - ld r8,EX_TLB_R8(r12); \ - mtlr r16; -#define TLB_MISS_PROLOG_STATS_BOLTED \ - mflr r10; \ - std r8,PACA_EXTLB+EX_TLB_R8(r13); \ - std r9,PACA_EXTLB+EX_TLB_R9(r13); \ - std r10,PACA_EXTLB+EX_TLB_LR(r13); -#define TLB_MISS_RESTORE_STATS_BOLTED \ - ld r16,PACA_EXTLB+EX_TLB_LR(r13); \ - ld r9,PACA_EXTLB+EX_TLB_R9(r13); \ - ld r8,PACA_EXTLB+EX_TLB_R8(r13); \ - mtlr r16; -#define TLB_MISS_STATS_D(name) \ - addi r9,r13,MMSTAT_DSTATS+name; \ - bl .tlb_stat_inc; -#define TLB_MISS_STATS_I(name) \ - addi r9,r13,MMSTAT_ISTATS+name; \ - bl .tlb_stat_inc; -#define TLB_MISS_STATS_X(name) \ - ld r8,PACA_EXTLB+EX_TLB_ESR(r13); \ - cmpdi cr2,r8,-1; \ - beq cr2,61f; \ - addi r9,r13,MMSTAT_DSTATS+name; \ - b 62f; \ -61: addi r9,r13,MMSTAT_ISTATS+name; \ -62: bl .tlb_stat_inc; -#define TLB_MISS_STATS_SAVE_INFO \ - std r14,EX_TLB_ESR(r12); /* save ESR */ -#define TLB_MISS_STATS_SAVE_INFO_BOLTED \ - std r14,PACA_EXTLB+EX_TLB_ESR(r13); /* save ESR */ -#else -#define TLB_MISS_PROLOG_STATS -#define TLB_MISS_RESTORE_STATS -#define TLB_MISS_PROLOG_STATS_BOLTED -#define TLB_MISS_RESTORE_STATS_BOLTED -#define TLB_MISS_STATS_D(name) -#define TLB_MISS_STATS_I(name) -#define TLB_MISS_STATS_X(name) -#define TLB_MISS_STATS_Y(name) -#define TLB_MISS_STATS_SAVE_INFO -#define TLB_MISS_STATS_SAVE_INFO_BOLTED +#ifndef __ASSEMBLER__ +extern unsigned int interrupt_base_book3e; #endif #define SET_IVOR(vector_number, vector_offset) \ - li r3,vector_offset@l; \ - ori r3,r3,interrupt_base_book3e@l; \ + LOAD_REG_ADDR(r3,interrupt_base_book3e);\ + ori r3,r3,vector_offset@l; \ mtspr SPRN_IVOR##vector_number,r3; +/* + * powerpc relies on return from interrupt/syscall being context synchronising + * (which rfi is) to support ARCH_HAS_MEMBARRIER_SYNC_CORE without additional + * synchronisation instructions. + */ +#define RFI_TO_KERNEL \ + rfi + +#define RFI_TO_USER \ + rfi #endif /* _ASM_POWERPC_EXCEPTION_64E_H */ diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index 07ca627e52c0..a9437e89f69f 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _ASM_POWERPC_EXCEPTION_H #define _ASM_POWERPC_EXCEPTION_H /* @@ -18,11 +19,6 @@ * * This file contains the low-level support and setup for the * PowerPC-64 platform, including trap and interrupt dispatch. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ /* * The following macros define the code that appears as @@ -34,504 +30,149 @@ * exception handlers (including pSeries LPAR) and iSeries LPAR * implementations as possible. */ +#include <asm/feature-fixups.h> + +/* PACA save area size in u64 units (exgen, exmc, etc) */ +#define EX_SIZE 10 +/* PACA save area offsets */ #define EX_R9 0 #define EX_R10 8 #define EX_R11 16 #define EX_R12 24 #define EX_R13 32 -#define EX_SRR0 40 -#define EX_DAR 48 -#define EX_DSISR 56 -#define EX_CCR 60 -#define EX_R3 64 -#define EX_LR 72 -#define EX_CFAR 80 -#define EX_PPR 88 /* SMT thread status register (priority) */ - -#ifdef CONFIG_RELOCATABLE -#define __EXCEPTION_RELON_PROLOG_PSERIES_1(label, h) \ - ld r12,PACAKBASE(r13); /* get high part of &label */ \ - mfspr r11,SPRN_##h##SRR0; /* save SRR0 */ \ - LOAD_HANDLER(r12,label); \ - mtlr r12; \ - mfspr r12,SPRN_##h##SRR1; /* and SRR1 */ \ - li r10,MSR_RI; \ - mtmsrd r10,1; /* Set RI (EE=0) */ \ - blr; -#else -/* If not relocatable, we can jump directly -- and save messing with LR */ -#define __EXCEPTION_RELON_PROLOG_PSERIES_1(label, h) \ - mfspr r11,SPRN_##h##SRR0; /* save SRR0 */ \ - mfspr r12,SPRN_##h##SRR1; /* and SRR1 */ \ - li r10,MSR_RI; \ - mtmsrd r10,1; /* Set RI (EE=0) */ \ - b label; -#endif -#define EXCEPTION_RELON_PROLOG_PSERIES_1(label, h) \ - __EXCEPTION_RELON_PROLOG_PSERIES_1(label, h) \ - -/* - * As EXCEPTION_PROLOG_PSERIES(), except we've already got relocation on - * so no need to rfid. Save lr in case we're CONFIG_RELOCATABLE, in which - * case EXCEPTION_RELON_PROLOG_PSERIES_1 will be using lr. - */ -#define EXCEPTION_RELON_PROLOG_PSERIES(area, label, h, extra, vec) \ - EXCEPTION_PROLOG_0(area); \ - EXCEPTION_PROLOG_1(area, extra, vec); \ - EXCEPTION_RELON_PROLOG_PSERIES_1(label, h) - -/* - * We're short on space and time in the exception prolog, so we can't - * use the normal SET_REG_IMMEDIATE macro. Normally we just need the - * low halfword of the address, but for Kdump we need the whole low - * word. - */ -#define LOAD_HANDLER(reg, label) \ - /* Handlers must be within 64K of kbase, which must be 64k aligned */ \ - ori reg,reg,(label)-_stext; /* virt addr of handler ... */ +#define EX_DAR 40 +#define EX_DSISR 48 +#define EX_CCR 52 +#define EX_CFAR 56 +#define EX_PPR 64 +#define EX_CTR 72 -/* Exception register prefixes */ -#define EXC_HV H -#define EXC_STD - -#if defined(CONFIG_RELOCATABLE) /* - * If we support interrupts with relocation on AND we're a relocatable - * kernel, we need to use LR to get to the 2nd level handler. So, save/restore - * it when required. + * maximum recursive depth of MCE exceptions */ -#define SAVE_LR(reg, area) mflr reg ; std reg,area+EX_LR(r13) -#define GET_LR(reg, area) ld reg,area+EX_LR(r13) -#define RESTORE_LR(reg, area) ld reg,area+EX_LR(r13) ; mtlr reg -#else -/* ...else LR is unused and in register. */ -#define SAVE_LR(reg, area) -#define GET_LR(reg, area) mflr reg -#define RESTORE_LR(reg, area) -#endif - -/* - * PPR save/restore macros used in exceptions_64s.S - * Used for P7 or later processors - */ -#define SAVE_PPR(area, ra, rb) \ -BEGIN_FTR_SECTION_NESTED(940) \ - ld ra,PACACURRENT(r13); \ - ld rb,area+EX_PPR(r13); /* Read PPR from paca */ \ - std rb,TASKTHREADPPR(ra); \ -END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,940) - -#define RESTORE_PPR_PACA(area, ra) \ -BEGIN_FTR_SECTION_NESTED(941) \ - ld ra,area+EX_PPR(r13); \ - mtspr SPRN_PPR,ra; \ -END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,941) +#define MAX_MCE_DEPTH 4 + +#ifdef __ASSEMBLER__ + +#define STF_ENTRY_BARRIER_SLOT \ + STF_ENTRY_BARRIER_FIXUP_SECTION; \ + nop; \ + nop; \ + nop + +#define STF_EXIT_BARRIER_SLOT \ + STF_EXIT_BARRIER_FIXUP_SECTION; \ + nop; \ + nop; \ + nop; \ + nop; \ + nop; \ + nop + +#define ENTRY_FLUSH_SLOT \ + ENTRY_FLUSH_FIXUP_SECTION; \ + nop; \ + nop; \ + nop; + +#define SCV_ENTRY_FLUSH_SLOT \ + SCV_ENTRY_FLUSH_FIXUP_SECTION; \ + nop; \ + nop; \ + nop; /* - * Increase the priority on systems where PPR save/restore is not - * implemented/ supported. + * r10 must be free to use, r13 must be paca */ -#define HMT_MEDIUM_PPR_DISCARD \ -BEGIN_FTR_SECTION_NESTED(942) \ - HMT_MEDIUM; \ -END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,0,942) /*non P7*/ +#define INTERRUPT_TO_KERNEL \ + STF_ENTRY_BARRIER_SLOT; \ + ENTRY_FLUSH_SLOT /* - * Get an SPR into a register if the CPU has the given feature + * r10, ctr must be free to use, r13 must be paca */ -#define OPT_GET_SPR(ra, spr, ftr) \ -BEGIN_FTR_SECTION_NESTED(943) \ - mfspr ra,spr; \ -END_FTR_SECTION_NESTED(ftr,ftr,943) +#define SCV_INTERRUPT_TO_KERNEL \ + STF_ENTRY_BARRIER_SLOT; \ + SCV_ENTRY_FLUSH_SLOT /* - * Save a register to the PACA if the CPU has the given feature - */ -#define OPT_SAVE_REG_TO_PACA(offset, ra, ftr) \ -BEGIN_FTR_SECTION_NESTED(943) \ - std ra,offset(r13); \ -END_FTR_SECTION_NESTED(ftr,ftr,943) - -#define EXCEPTION_PROLOG_0(area) \ - GET_PACA(r13); \ - std r9,area+EX_R9(r13); /* save r9 */ \ - OPT_GET_SPR(r9, SPRN_PPR, CPU_FTR_HAS_PPR); \ - HMT_MEDIUM; \ - std r10,area+EX_R10(r13); /* save r10 - r12 */ \ - OPT_GET_SPR(r10, SPRN_CFAR, CPU_FTR_CFAR) - -#define __EXCEPTION_PROLOG_1(area, extra, vec) \ - OPT_SAVE_REG_TO_PACA(area+EX_PPR, r9, CPU_FTR_HAS_PPR); \ - OPT_SAVE_REG_TO_PACA(area+EX_CFAR, r10, CPU_FTR_CFAR); \ - SAVE_LR(r10, area); \ - mfcr r9; \ - extra(vec); \ - std r11,area+EX_R11(r13); \ - std r12,area+EX_R12(r13); \ - GET_SCRATCH0(r10); \ - std r10,area+EX_R13(r13) -#define EXCEPTION_PROLOG_1(area, extra, vec) \ - __EXCEPTION_PROLOG_1(area, extra, vec) - -#define __EXCEPTION_PROLOG_PSERIES_1(label, h) \ - ld r12,PACAKBASE(r13); /* get high part of &label */ \ - ld r10,PACAKMSR(r13); /* get MSR value for kernel */ \ - mfspr r11,SPRN_##h##SRR0; /* save SRR0 */ \ - LOAD_HANDLER(r12,label) \ - mtspr SPRN_##h##SRR0,r12; \ - mfspr r12,SPRN_##h##SRR1; /* and SRR1 */ \ - mtspr SPRN_##h##SRR1,r10; \ - h##rfid; \ - b . /* prevent speculative execution */ -#define EXCEPTION_PROLOG_PSERIES_1(label, h) \ - __EXCEPTION_PROLOG_PSERIES_1(label, h) - -#define EXCEPTION_PROLOG_PSERIES(area, label, h, extra, vec) \ - EXCEPTION_PROLOG_0(area); \ - EXCEPTION_PROLOG_1(area, extra, vec); \ - EXCEPTION_PROLOG_PSERIES_1(label, h); - -#define __KVMTEST(n) \ - lbz r10,HSTATE_IN_GUEST(r13); \ - cmpwi r10,0; \ - bne do_kvm_##n - -#define __KVM_HANDLER(area, h, n) \ -do_kvm_##n: \ - BEGIN_FTR_SECTION_NESTED(947) \ - ld r10,area+EX_CFAR(r13); \ - std r10,HSTATE_CFAR(r13); \ - END_FTR_SECTION_NESTED(CPU_FTR_CFAR,CPU_FTR_CFAR,947); \ - ld r10,area+EX_R10(r13); \ - stw r9,HSTATE_SCRATCH1(r13); \ - ld r9,area+EX_R9(r13); \ - std r12,HSTATE_SCRATCH0(r13); \ - li r12,n; \ - b kvmppc_interrupt - -#define __KVM_HANDLER_SKIP(area, h, n) \ -do_kvm_##n: \ - cmpwi r10,KVM_GUEST_MODE_SKIP; \ - ld r10,area+EX_R10(r13); \ - beq 89f; \ - stw r9,HSTATE_SCRATCH1(r13); \ - ld r9,area+EX_R9(r13); \ - std r12,HSTATE_SCRATCH0(r13); \ - li r12,n; \ - b kvmppc_interrupt; \ -89: mtocrf 0x80,r9; \ - ld r9,area+EX_R9(r13); \ - b kvmppc_skip_##h##interrupt - -#ifdef CONFIG_KVM_BOOK3S_64_HANDLER -#define KVMTEST(n) __KVMTEST(n) -#define KVM_HANDLER(area, h, n) __KVM_HANDLER(area, h, n) -#define KVM_HANDLER_SKIP(area, h, n) __KVM_HANDLER_SKIP(area, h, n) - -#else -#define KVMTEST(n) -#define KVM_HANDLER(area, h, n) -#define KVM_HANDLER_SKIP(area, h, n) -#endif - -#ifdef CONFIG_KVM_BOOK3S_PR -#define KVMTEST_PR(n) __KVMTEST(n) -#define KVM_HANDLER_PR(area, h, n) __KVM_HANDLER(area, h, n) -#define KVM_HANDLER_PR_SKIP(area, h, n) __KVM_HANDLER_SKIP(area, h, n) - -#else -#define KVMTEST_PR(n) -#define KVM_HANDLER_PR(area, h, n) -#define KVM_HANDLER_PR_SKIP(area, h, n) -#endif - -#define NOTEST(n) - -/* - * The common exception prolog is used for all except a few exceptions - * such as a segment miss on a kernel address. We have to be prepared - * to take another exception from the point where we first touch the - * kernel stack onwards. + * Macros for annotating the expected destination of (h)rfid * - * On entry r13 points to the paca, r9-r13 are saved in the paca, - * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and - * SRR1, and relocation is on. - */ -#define EXCEPTION_PROLOG_COMMON(n, area) \ - andi. r10,r12,MSR_PR; /* See if coming from user */ \ - mr r10,r1; /* Save r1 */ \ - subi r1,r1,INT_FRAME_SIZE; /* alloc frame on kernel stack */ \ - beq- 1f; \ - ld r1,PACAKSAVE(r13); /* kernel stack to use */ \ -1: cmpdi cr1,r1,0; /* check if r1 is in userspace */ \ - blt+ cr1,3f; /* abort if it is */ \ - li r1,(n); /* will be reloaded later */ \ - sth r1,PACA_TRAP_SAVE(r13); \ - std r3,area+EX_R3(r13); \ - addi r3,r13,area; /* r3 -> where regs are saved*/ \ - RESTORE_LR(r1, area); \ - b bad_stack; \ -3: std r9,_CCR(r1); /* save CR in stackframe */ \ - std r11,_NIP(r1); /* save SRR0 in stackframe */ \ - std r12,_MSR(r1); /* save SRR1 in stackframe */ \ - std r10,0(r1); /* make stack chain pointer */ \ - std r0,GPR0(r1); /* save r0 in stackframe */ \ - std r10,GPR1(r1); /* save r1 in stackframe */ \ - beq 4f; /* if from kernel mode */ \ - ACCOUNT_CPU_USER_ENTRY(r9, r10); \ - SAVE_PPR(area, r9, r10); \ -4: std r2,GPR2(r1); /* save r2 in stackframe */ \ - SAVE_4GPRS(3, r1); /* save r3 - r6 in stackframe */ \ - SAVE_2GPRS(7, r1); /* save r7, r8 in stackframe */ \ - ld r9,area+EX_R9(r13); /* move r9, r10 to stackframe */ \ - ld r10,area+EX_R10(r13); \ - std r9,GPR9(r1); \ - std r10,GPR10(r1); \ - ld r9,area+EX_R11(r13); /* move r11 - r13 to stackframe */ \ - ld r10,area+EX_R12(r13); \ - ld r11,area+EX_R13(r13); \ - std r9,GPR11(r1); \ - std r10,GPR12(r1); \ - std r11,GPR13(r1); \ - BEGIN_FTR_SECTION_NESTED(66); \ - ld r10,area+EX_CFAR(r13); \ - std r10,ORIG_GPR3(r1); \ - END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 66); \ - GET_LR(r9,area); /* Get LR, later save to stack */ \ - ld r2,PACATOC(r13); /* get kernel TOC into r2 */ \ - std r9,_LINK(r1); \ - mfctr r10; /* save CTR in stackframe */ \ - std r10,_CTR(r1); \ - lbz r10,PACASOFTIRQEN(r13); \ - mfspr r11,SPRN_XER; /* save XER in stackframe */ \ - std r10,SOFTE(r1); \ - std r11,_XER(r1); \ - li r9,(n)+1; \ - std r9,_TRAP(r1); /* set trap number */ \ - li r10,0; \ - ld r11,exception_marker@toc(r2); \ - std r10,RESULT(r1); /* clear regs->result */ \ - std r11,STACK_FRAME_OVERHEAD-16(r1); /* mark the frame */ \ - ACCOUNT_STOLEN_TIME - -/* - * Exception vectors. - */ -#define STD_EXCEPTION_PSERIES(loc, vec, label) \ - . = loc; \ - .globl label##_pSeries; \ -label##_pSeries: \ - HMT_MEDIUM_PPR_DISCARD; \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common, \ - EXC_STD, KVMTEST_PR, vec) - -/* Version of above for when we have to branch out-of-line */ -#define STD_EXCEPTION_PSERIES_OOL(vec, label) \ - .globl label##_pSeries; \ -label##_pSeries: \ - EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_PR, vec); \ - EXCEPTION_PROLOG_PSERIES_1(label##_common, EXC_STD) - -#define STD_EXCEPTION_HV(loc, vec, label) \ - . = loc; \ - .globl label##_hv; \ -label##_hv: \ - HMT_MEDIUM_PPR_DISCARD; \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common, \ - EXC_HV, KVMTEST, vec) - -/* Version of above for when we have to branch out-of-line */ -#define STD_EXCEPTION_HV_OOL(vec, label) \ - .globl label##_hv; \ -label##_hv: \ - EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST, vec); \ - EXCEPTION_PROLOG_PSERIES_1(label##_common, EXC_HV) - -#define STD_RELON_EXCEPTION_PSERIES(loc, vec, label) \ - . = loc; \ - .globl label##_relon_pSeries; \ -label##_relon_pSeries: \ - HMT_MEDIUM_PPR_DISCARD; \ - /* No guest interrupts come through here */ \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_RELON_PROLOG_PSERIES(PACA_EXGEN, label##_common, \ - EXC_STD, NOTEST, vec) - -#define STD_RELON_EXCEPTION_PSERIES_OOL(vec, label) \ - .globl label##_relon_pSeries; \ -label##_relon_pSeries: \ - EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, vec); \ - EXCEPTION_RELON_PROLOG_PSERIES_1(label##_common, EXC_STD) - -#define STD_RELON_EXCEPTION_HV(loc, vec, label) \ - . = loc; \ - .globl label##_relon_hv; \ -label##_relon_hv: \ - HMT_MEDIUM_PPR_DISCARD; \ - /* No guest interrupts come through here */ \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_RELON_PROLOG_PSERIES(PACA_EXGEN, label##_common, \ - EXC_HV, NOTEST, vec) - -#define STD_RELON_EXCEPTION_HV_OOL(vec, label) \ - .globl label##_relon_hv; \ -label##_relon_hv: \ - EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, vec); \ - EXCEPTION_RELON_PROLOG_PSERIES_1(label##_common, EXC_HV) - -/* This associate vector numbers with bits in paca->irq_happened */ -#define SOFTEN_VALUE_0x500 PACA_IRQ_EE -#define SOFTEN_VALUE_0x502 PACA_IRQ_EE -#define SOFTEN_VALUE_0x900 PACA_IRQ_DEC -#define SOFTEN_VALUE_0x982 PACA_IRQ_DEC -#define SOFTEN_VALUE_0xa00 PACA_IRQ_DBELL -#define SOFTEN_VALUE_0xe80 PACA_IRQ_DBELL -#define SOFTEN_VALUE_0xe82 PACA_IRQ_DBELL - -#define __SOFTEN_TEST(h, vec) \ - lbz r10,PACASOFTIRQEN(r13); \ - cmpwi r10,0; \ - li r10,SOFTEN_VALUE_##vec; \ - beq masked_##h##interrupt -#define _SOFTEN_TEST(h, vec) __SOFTEN_TEST(h, vec) - -#define SOFTEN_TEST_PR(vec) \ - KVMTEST_PR(vec); \ - _SOFTEN_TEST(EXC_STD, vec) - -#define SOFTEN_TEST_HV(vec) \ - KVMTEST(vec); \ - _SOFTEN_TEST(EXC_HV, vec) - -#define SOFTEN_TEST_HV_201(vec) \ - KVMTEST(vec); \ - _SOFTEN_TEST(EXC_STD, vec) - -#define SOFTEN_NOTEST_PR(vec) _SOFTEN_TEST(EXC_STD, vec) -#define SOFTEN_NOTEST_HV(vec) _SOFTEN_TEST(EXC_HV, vec) - -#define __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra) \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_0(PACA_EXGEN); \ - __EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec); \ - EXCEPTION_PROLOG_PSERIES_1(label##_common, h); - -#define _MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra) \ - __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra) - -#define MASKABLE_EXCEPTION_PSERIES(loc, vec, label) \ - . = loc; \ - .globl label##_pSeries; \ -label##_pSeries: \ - HMT_MEDIUM_PPR_DISCARD; \ - _MASKABLE_EXCEPTION_PSERIES(vec, label, \ - EXC_STD, SOFTEN_TEST_PR) - -#define MASKABLE_EXCEPTION_HV(loc, vec, label) \ - . = loc; \ - .globl label##_hv; \ -label##_hv: \ - _MASKABLE_EXCEPTION_PSERIES(vec, label, \ - EXC_HV, SOFTEN_TEST_HV) - -#define MASKABLE_EXCEPTION_HV_OOL(vec, label) \ - .globl label##_hv; \ -label##_hv: \ - EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec); \ - EXCEPTION_PROLOG_PSERIES_1(label##_common, EXC_HV); - -#define __MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra) \ - HMT_MEDIUM_PPR_DISCARD; \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_0(PACA_EXGEN); \ - __EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec); \ - EXCEPTION_RELON_PROLOG_PSERIES_1(label##_common, h); -#define _MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra) \ - __MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra) - -#define MASKABLE_RELON_EXCEPTION_PSERIES(loc, vec, label) \ - . = loc; \ - .globl label##_relon_pSeries; \ -label##_relon_pSeries: \ - _MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, \ - EXC_STD, SOFTEN_NOTEST_PR) - -#define MASKABLE_RELON_EXCEPTION_HV(loc, vec, label) \ - . = loc; \ - .globl label##_relon_hv; \ -label##_relon_hv: \ - _MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, \ - EXC_HV, SOFTEN_NOTEST_HV) - -#define MASKABLE_RELON_EXCEPTION_HV_OOL(vec, label) \ - .globl label##_relon_hv; \ -label##_relon_hv: \ - EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_NOTEST_HV, vec); \ - EXCEPTION_PROLOG_PSERIES_1(label##_common, EXC_HV); - -/* - * Our exception common code can be passed various "additions" - * to specify the behaviour of interrupts, whether to kick the - * runlatch, etc... - */ - -/* Exception addition: Hard disable interrupts */ -#define DISABLE_INTS SOFT_DISABLE_INTS(r10,r11) - -#define ADD_NVGPRS \ - bl .save_nvgprs - -#define RUNLATCH_ON \ -BEGIN_FTR_SECTION \ - CURRENT_THREAD_INFO(r3, r1); \ - ld r4,TI_LOCAL_FLAGS(r3); \ - andi. r0,r4,_TLF_RUNLATCH; \ - beql ppc64_runlatch_on_trampoline; \ -END_FTR_SECTION_IFSET(CPU_FTR_CTRL) - -#define EXCEPTION_COMMON(trap, label, hdlr, ret, additions) \ - .align 7; \ - .globl label##_common; \ -label##_common: \ - EXCEPTION_PROLOG_COMMON(trap, PACA_EXGEN); \ - additions; \ - addi r3,r1,STACK_FRAME_OVERHEAD; \ - bl hdlr; \ - b ret - -#define STD_EXCEPTION_COMMON(trap, label, hdlr) \ - EXCEPTION_COMMON(trap, label, hdlr, ret_from_except, \ - ADD_NVGPRS;DISABLE_INTS) - -/* - * Like STD_EXCEPTION_COMMON, but for exceptions that can occur - * in the idle task and therefore need the special idle handling - * (finish nap and runlatch) - */ -#define STD_EXCEPTION_COMMON_ASYNC(trap, label, hdlr) \ - EXCEPTION_COMMON(trap, label, hdlr, ret_from_except_lite, \ - FINISH_NAP;DISABLE_INTS;RUNLATCH_ON) - -/* - * When the idle code in power4_idle puts the CPU into NAP mode, - * it has to do so in a loop, and relies on the external interrupt - * and decrementer interrupt entry code to get it out of the loop. - * It sets the _TLF_NAPPING bit in current_thread_info()->local_flags - * to signal that it is in the loop and needs help to get out. + * The nop instructions allow us to insert one or more instructions to flush the + * L1-D cache when returning to userspace or a guest. + * + * powerpc relies on return from interrupt/syscall being context synchronising + * (which hrfid, rfid, and rfscv are) to support ARCH_HAS_MEMBARRIER_SYNC_CORE + * without additional synchronisation instructions. + * + * soft-masked interrupt replay does not include a context-synchronising rfid, + * but those always return to kernel, the sync is only required when returning + * to user. */ -#ifdef CONFIG_PPC_970_NAP -#define FINISH_NAP \ -BEGIN_FTR_SECTION \ - CURRENT_THREAD_INFO(r11, r1); \ - ld r9,TI_LOCAL_FLAGS(r11); \ - andi. r10,r9,_TLF_NAPPING; \ - bnel power4_fixup_nap; \ -END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) -#else -#define FINISH_NAP -#endif +#define RFI_FLUSH_SLOT \ + RFI_FLUSH_FIXUP_SECTION; \ + nop; \ + nop; \ + nop + +#define RFI_TO_KERNEL \ + rfid + +#define RFI_TO_USER \ + STF_EXIT_BARRIER_SLOT; \ + RFI_FLUSH_SLOT; \ + rfid; \ + b rfi_flush_fallback + +#define RFI_TO_USER_OR_KERNEL \ + STF_EXIT_BARRIER_SLOT; \ + RFI_FLUSH_SLOT; \ + rfid; \ + b rfi_flush_fallback + +#define RFI_TO_GUEST \ + STF_EXIT_BARRIER_SLOT; \ + RFI_FLUSH_SLOT; \ + rfid; \ + b rfi_flush_fallback + +#define HRFI_TO_KERNEL \ + hrfid + +#define HRFI_TO_USER \ + STF_EXIT_BARRIER_SLOT; \ + RFI_FLUSH_SLOT; \ + hrfid; \ + b hrfi_flush_fallback + +#define HRFI_TO_USER_OR_KERNEL \ + STF_EXIT_BARRIER_SLOT; \ + RFI_FLUSH_SLOT; \ + hrfid; \ + b hrfi_flush_fallback + +#define HRFI_TO_GUEST \ + STF_EXIT_BARRIER_SLOT; \ + RFI_FLUSH_SLOT; \ + hrfid; \ + b hrfi_flush_fallback + +#define HRFI_TO_UNKNOWN \ + STF_EXIT_BARRIER_SLOT; \ + RFI_FLUSH_SLOT; \ + hrfid; \ + b hrfi_flush_fallback + +#define RFSCV_TO_USER \ + STF_EXIT_BARRIER_SLOT; \ + RFI_FLUSH_SLOT; \ + RFSCV; \ + b rfscv_flush_fallback + +#else /* __ASSEMBLER__ */ +/* Prototype for function defined in exceptions-64s.S */ +void do_uaccess_flush(void); +#endif /* __ASSEMBLER__ */ #endif /* _ASM_POWERPC_EXCEPTION_H */ diff --git a/arch/powerpc/include/asm/exec.h b/arch/powerpc/include/asm/exec.h index 8196e9c7d7e8..92cac4851275 100644 --- a/arch/powerpc/include/asm/exec.h +++ b/arch/powerpc/include/asm/exec.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu> */ diff --git a/arch/powerpc/include/asm/extable.h b/arch/powerpc/include/asm/extable.h new file mode 100644 index 000000000000..d483a9c24ba9 --- /dev/null +++ b/arch/powerpc/include/asm/extable.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ARCH_POWERPC_EXTABLE_H +#define _ARCH_POWERPC_EXTABLE_H + +/* + * The exception table consists of pairs of relative addresses: the first is + * the address of an instruction that is allowed to fault, and the second is + * the address at which the program should continue. No registers are + * modified, so it is entirely up to the continuation code to figure out what + * to do. + * + * All the routines below use bits of fixup code that are out of line with the + * main instruction path. This means when everything is well, we don't even + * have to jump over them. Further, they do not intrude on our cache or tlb + * entries. + */ + +#define ARCH_HAS_RELATIVE_EXTABLE + +#ifndef __ASSEMBLER__ + +struct exception_table_entry { + int insn; + int fixup; +}; + +static inline unsigned long extable_fixup(const struct exception_table_entry *x) +{ + return (unsigned long)&x->fixup + x->fixup; +} + +#endif + +/* + * Helper macro for exception table entries + */ +#define EX_TABLE(_fault, _target) \ + stringify_in_c(.section __ex_table,"a";)\ + stringify_in_c(.balign 4;) \ + stringify_in_c(.long (_fault) - . ;) \ + stringify_in_c(.long (_target) - . ;) \ + stringify_in_c(.previous) + +#endif diff --git a/arch/powerpc/include/asm/fadump-internal.h b/arch/powerpc/include/asm/fadump-internal.h new file mode 100644 index 000000000000..e83869a4eb6a --- /dev/null +++ b/arch/powerpc/include/asm/fadump-internal.h @@ -0,0 +1,196 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Firmware-Assisted Dump internal code. + * + * Copyright 2011, Mahesh Salgaonkar, IBM Corporation. + * Copyright 2019, Hari Bathini, IBM Corporation. + */ + +#ifndef _ASM_POWERPC_FADUMP_INTERNAL_H +#define _ASM_POWERPC_FADUMP_INTERNAL_H + +/* Maximum number of memory regions kernel supports */ +#define FADUMP_MAX_MEM_REGS 128 + +#ifndef CONFIG_PRESERVE_FA_DUMP + +/* The upper limit percentage for user specified boot memory size (25%) */ +#define MAX_BOOT_MEM_RATIO 4 + +#define memblock_num_regions(memblock_type) (memblock.memblock_type.cnt) + +/* FAD commands */ +#define FADUMP_REGISTER 1 +#define FADUMP_UNREGISTER 2 +#define FADUMP_INVALIDATE 3 + +/* + * Copy the ascii values for first 8 characters from a string into u64 + * variable at their respective indexes. + * e.g. + * The string "FADMPINF" will be converted into 0x4641444d50494e46 + */ +static inline u64 fadump_str_to_u64(const char *str) +{ + u64 val = 0; + int i; + + for (i = 0; i < sizeof(val); i++) + val = (*str) ? (val << 8) | *str++ : val << 8; + return val; +} + +#define FADUMP_CPU_UNKNOWN (~((u32)0)) + +/* + * The introduction of new fields in the fadump crash info header has + * led to a change in the magic key from `FADMPINF` to `FADMPSIG` for + * identifying a kernel crash from an old kernel. + * + * To prevent the need for further changes to the magic number in the + * event of future modifications to the fadump crash info header, a + * version field has been introduced to track the fadump crash info + * header version. + * + * Consider a few points before adding new members to the fadump crash info + * header structure: + * + * - Append new members; avoid adding them in between. + * - Non-primitive members should have a size member as well. + * - For every change in the fadump header, increment the + * fadump header version. This helps the updated kernel decide how to + * handle kernel dumps from older kernels. + */ +#define FADUMP_CRASH_INFO_MAGIC_OLD fadump_str_to_u64("FADMPINF") +#define FADUMP_CRASH_INFO_MAGIC fadump_str_to_u64("FADMPSIG") +#define FADUMP_HEADER_VERSION 1 + +/* fadump crash info structure */ +struct fadump_crash_info_header { + u64 magic_number; + u32 version; + u32 crashing_cpu; + u64 vmcoreinfo_raddr; + u64 vmcoreinfo_size; + u32 pt_regs_sz; + u32 cpu_mask_sz; + struct pt_regs regs; + struct cpumask cpu_mask; +}; + +struct fadump_memory_range { + u64 base; + u64 size; +}; + +/* fadump memory ranges info */ +#define RNG_NAME_SZ 16 +struct fadump_mrange_info { + char name[RNG_NAME_SZ]; + struct fadump_memory_range *mem_ranges; + u32 mem_ranges_sz; + u32 mem_range_cnt; + u32 max_mem_ranges; + bool is_static; +}; + +/* Platform specific callback functions */ +struct fadump_ops; + +/* Firmware-assisted dump configuration details. */ +struct fw_dump { + unsigned long reserve_dump_area_start; + unsigned long reserve_dump_area_size; + /* cmd line option during boot */ + unsigned long reserve_bootvar; + + unsigned long cpu_state_data_size; + u64 cpu_state_dest_vaddr; + u32 cpu_state_data_version; + u32 cpu_state_entry_size; + + unsigned long hpte_region_size; + + unsigned long boot_memory_size; + u64 boot_mem_dest_addr; + u64 boot_mem_addr[FADUMP_MAX_MEM_REGS]; + u64 boot_mem_sz[FADUMP_MAX_MEM_REGS]; + u64 boot_mem_top; + u64 boot_mem_regs_cnt; + + unsigned long fadumphdr_addr; + u64 elfcorehdr_addr; + u64 elfcorehdr_size; + unsigned long cpu_notes_buf_vaddr; + unsigned long cpu_notes_buf_size; + + unsigned long param_area; + + /* + * Maximum size supported by firmware to copy from source to + * destination address per entry. + */ + u64 max_copy_size; + u64 kernel_metadata; + + int ibm_configure_kernel_dump; + + unsigned long fadump_enabled:1; + unsigned long fadump_supported:1; + unsigned long dump_active:1; + unsigned long dump_registered:1; + unsigned long nocma:1; + unsigned long param_area_supported:1; + + struct fadump_ops *ops; +}; + +struct fadump_ops { + u64 (*fadump_init_mem_struct)(struct fw_dump *fadump_conf); + u64 (*fadump_get_metadata_size)(void); + int (*fadump_setup_metadata)(struct fw_dump *fadump_conf); + u64 (*fadump_get_bootmem_min)(void); + int (*fadump_register)(struct fw_dump *fadump_conf); + int (*fadump_unregister)(struct fw_dump *fadump_conf); + int (*fadump_invalidate)(struct fw_dump *fadump_conf); + void (*fadump_cleanup)(struct fw_dump *fadump_conf); + int (*fadump_process)(struct fw_dump *fadump_conf); + void (*fadump_region_show)(struct fw_dump *fadump_conf, + struct seq_file *m); + void (*fadump_trigger)(struct fadump_crash_info_header *fdh, + const char *msg); + int (*fadump_max_boot_mem_rgns)(void); +}; + +/* Helper functions */ +s32 __init fadump_setup_cpu_notes_buf(u32 num_cpus); +void fadump_free_cpu_notes_buf(void); +u32 *__init fadump_regs_to_elf_notes(u32 *buf, struct pt_regs *regs); +void __init fadump_update_elfcore_header(char *bufp); +bool is_fadump_reserved_mem_contiguous(void); + +#else /* !CONFIG_PRESERVE_FA_DUMP */ + +/* Firmware-assisted dump configuration details. */ +struct fw_dump { + u64 boot_mem_top; + u64 dump_active; +}; + +#endif /* CONFIG_PRESERVE_FA_DUMP */ + +#ifdef CONFIG_PPC_PSERIES +extern void rtas_fadump_dt_scan(struct fw_dump *fadump_conf, u64 node); +#else +static inline void +rtas_fadump_dt_scan(struct fw_dump *fadump_conf, u64 node) { } +#endif + +#ifdef CONFIG_PPC_POWERNV +extern void opal_fadump_dt_scan(struct fw_dump *fadump_conf, u64 node); +#else +static inline void +opal_fadump_dt_scan(struct fw_dump *fadump_conf, u64 node) { } +#endif + +#endif /* _ASM_POWERPC_FADUMP_INTERNAL_H */ diff --git a/arch/powerpc/include/asm/fadump.h b/arch/powerpc/include/asm/fadump.h index 88dbf9659185..a48f54dde4f6 100644 --- a/arch/powerpc/include/asm/fadump.h +++ b/arch/powerpc/include/asm/fadump.h @@ -1,218 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Firmware Assisted dump header file. * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * * Copyright 2011 IBM Corporation * Author: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com> */ -#ifndef __PPC64_FA_DUMP_H__ -#define __PPC64_FA_DUMP_H__ +#ifndef _ASM_POWERPC_FADUMP_H +#define _ASM_POWERPC_FADUMP_H #ifdef CONFIG_FA_DUMP -/* - * The RMA region will be saved for later dumping when kernel crashes. - * RMA is Real Mode Area, the first block of logical memory address owned - * by logical partition, containing the storage that may be accessed with - * translate off. - */ -#define RMA_START 0x0 -#define RMA_END (ppc64_rma_size) - -/* - * On some Power systems where RMO is 128MB, it still requires minimum of - * 256MB for kernel to boot successfully. When kdump infrastructure is - * configured to save vmcore over network, we run into OOM issue while - * loading modules related to network setup. Hence we need aditional 64M - * of memory to avoid OOM issue. - */ -#define MIN_BOOT_MEM (((RMA_END < (0x1UL << 28)) ? (0x1UL << 28) : RMA_END) \ - + (0x1UL << 26)) - -#define memblock_num_regions(memblock_type) (memblock.memblock_type.cnt) - -#ifndef ELF_CORE_EFLAGS -#define ELF_CORE_EFLAGS 0 -#endif - -/* Firmware provided dump sections */ -#define FADUMP_CPU_STATE_DATA 0x0001 -#define FADUMP_HPTE_REGION 0x0002 -#define FADUMP_REAL_MODE_REGION 0x0011 - -/* Dump request flag */ -#define FADUMP_REQUEST_FLAG 0x00000001 - -/* FAD commands */ -#define FADUMP_REGISTER 1 -#define FADUMP_UNREGISTER 2 -#define FADUMP_INVALIDATE 3 - -/* Dump status flag */ -#define FADUMP_ERROR_FLAG 0x2000 - -#define FADUMP_CPU_ID_MASK ((1UL << 32) - 1) - -#define CPU_UNKNOWN (~((u32)0)) - -/* Utility macros */ -#define SKIP_TO_NEXT_CPU(reg_entry) \ -({ \ - while (reg_entry->reg_id != REG_ID("CPUEND")) \ - reg_entry++; \ - reg_entry++; \ -}) - -/* Kernel Dump section info */ -struct fadump_section { - u32 request_flag; - u16 source_data_type; - u16 error_flags; - u64 source_address; - u64 source_len; - u64 bytes_dumped; - u64 destination_address; -}; - -/* ibm,configure-kernel-dump header. */ -struct fadump_section_header { - u32 dump_format_version; - u16 dump_num_sections; - u16 dump_status_flag; - u32 offset_first_dump_section; - - /* Fields for disk dump option. */ - u32 dd_block_size; - u64 dd_block_offset; - u64 dd_num_blocks; - u32 dd_offset_disk_path; - - /* Maximum time allowed to prevent an automatic dump-reboot. */ - u32 max_time_auto; -}; - -/* - * Firmware Assisted dump memory structure. This structure is required for - * registering future kernel dump with power firmware through rtas call. - * - * No disk dump option. Hence disk dump path string section is not included. - */ -struct fadump_mem_struct { - struct fadump_section_header header; - - /* Kernel dump sections */ - struct fadump_section cpu_state_data; - struct fadump_section hpte_region; - struct fadump_section rmr_region; -}; - -/* Firmware-assisted dump configuration details. */ -struct fw_dump { - unsigned long cpu_state_data_size; - unsigned long hpte_region_size; - unsigned long boot_memory_size; - unsigned long reserve_dump_area_start; - unsigned long reserve_dump_area_size; - /* cmd line option during boot */ - unsigned long reserve_bootvar; - - unsigned long fadumphdr_addr; - unsigned long cpu_notes_buf; - unsigned long cpu_notes_buf_size; - - int ibm_configure_kernel_dump; - - unsigned long fadump_enabled:1; - unsigned long fadump_supported:1; - unsigned long dump_active:1; - unsigned long dump_registered:1; -}; - -/* - * Copy the ascii values for first 8 characters from a string into u64 - * variable at their respective indexes. - * e.g. - * The string "FADMPINF" will be converted into 0x4641444d50494e46 - */ -static inline u64 str_to_u64(const char *str) -{ - u64 val = 0; - int i; - - for (i = 0; i < sizeof(val); i++) - val = (*str) ? (val << 8) | *str++ : val << 8; - return val; -} -#define STR_TO_HEX(x) str_to_u64(x) -#define REG_ID(x) str_to_u64(x) - -#define FADUMP_CRASH_INFO_MAGIC STR_TO_HEX("FADMPINF") -#define REGSAVE_AREA_MAGIC STR_TO_HEX("REGSAVE") +extern int crashing_cpu; -/* The firmware-assisted dump format. - * - * The register save area is an area in the partition's memory used to preserve - * the register contents (CPU state data) for the active CPUs during a firmware - * assisted dump. The dump format contains register save area header followed - * by register entries. Each list of registers for a CPU starts with - * "CPUSTRT" and ends with "CPUEND". - */ - -/* Register save area header. */ -struct fadump_reg_save_area_header { - u64 magic_number; - u32 version; - u32 num_cpu_offset; -}; - -/* Register entry. */ -struct fadump_reg_entry { - u64 reg_id; - u64 reg_value; -}; - -/* fadump crash info structure */ -struct fadump_crash_info_header { - u64 magic_number; - u64 elfcorehdr_addr; - u32 crashing_cpu; - struct pt_regs regs; - struct cpumask cpu_online_mask; -}; - -/* Crash memory ranges */ -#define INIT_CRASHMEM_RANGES (INIT_MEMBLOCK_REGIONS + 2) - -struct fad_crash_memory_ranges { - unsigned long long base; - unsigned long long size; -}; - -extern int early_init_dt_scan_fw_dump(unsigned long node, - const char *uname, int depth, void *data); -extern int fadump_reserve_mem(void); +extern int is_fadump_memory_area(u64 addr, ulong size); extern int setup_fadump(void); extern int is_fadump_active(void); +extern int should_fadump_crash(void); extern void crash_fadump(struct pt_regs *, const char *); extern void fadump_cleanup(void); +void fadump_setup_param_area(void); +extern void fadump_append_bootargs(void); -extern void vmcore_cleanup(void); #else /* CONFIG_FA_DUMP */ static inline int is_fadump_active(void) { return 0; } +static inline int should_fadump_crash(void) { return 0; } static inline void crash_fadump(struct pt_regs *regs, const char *str) { } +static inline void fadump_cleanup(void) { } +static inline void fadump_setup_param_area(void) { } +static inline void fadump_append_bootargs(void) { } +#endif /* !CONFIG_FA_DUMP */ + +#if defined(CONFIG_FA_DUMP) || defined(CONFIG_PRESERVE_FA_DUMP) +extern int early_init_dt_scan_fw_dump(unsigned long node, const char *uname, + int depth, void *data); +extern int fadump_reserve_mem(void); #endif + +#if defined(CONFIG_FA_DUMP) && defined(CONFIG_CMA) +void fadump_cma_init(void); +#else +static inline void fadump_cma_init(void) { } #endif + +#endif /* _ASM_POWERPC_FADUMP_H */ diff --git a/arch/powerpc/include/asm/fb.h b/arch/powerpc/include/asm/fb.h deleted file mode 100644 index 411af8d17a69..000000000000 --- a/arch/powerpc/include/asm/fb.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef _ASM_FB_H_ -#define _ASM_FB_H_ - -#include <linux/fb.h> -#include <linux/fs.h> -#include <asm/page.h> - -static inline void fb_pgprotect(struct file *file, struct vm_area_struct *vma, - unsigned long off) -{ - vma->vm_page_prot = phys_mem_access_prot(file, off >> PAGE_SHIFT, - vma->vm_end - vma->vm_start, - vma->vm_page_prot); -} - -static inline int fb_is_primary_device(struct fb_info *info) -{ - return 0; -} - -#endif /* _ASM_FB_H_ */ diff --git a/arch/powerpc/include/asm/feature-fixups.h b/arch/powerpc/include/asm/feature-fixups.h index 9a67a38bf7b9..756a6c694018 100644 --- a/arch/powerpc/include/asm/feature-fixups.h +++ b/arch/powerpc/include/asm/feature-fixups.h @@ -1,11 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef __ASM_POWERPC_FEATURE_FIXUPS_H #define __ASM_POWERPC_FEATURE_FIXUPS_H +#include <asm/asm-const.h> + /* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ /* @@ -19,11 +18,11 @@ */ #if defined(CONFIG_PPC64) && !defined(__powerpc64__) /* 64 bits kernel, 32 bits code (ie. vdso32) */ -#define FTR_ENTRY_LONG .llong +#define FTR_ENTRY_LONG .8byte #define FTR_ENTRY_OFFSET .long 0xffffffff; .long #elif defined(CONFIG_PPC64) -#define FTR_ENTRY_LONG .llong -#define FTR_ENTRY_OFFSET .llong +#define FTR_ENTRY_LONG .8byte +#define FTR_ENTRY_OFFSET .8byte #else #define FTR_ENTRY_LONG .long #define FTR_ENTRY_OFFSET .long @@ -37,6 +36,24 @@ label##2: \ .align 2; \ label##3: + +#ifndef CONFIG_CC_IS_CLANG +#define CHECK_ALT_SIZE(else_size, body_size) \ + .ifgt (else_size) - (body_size); \ + .error "Feature section else case larger than body"; \ + .endif; +#else +/* + * If we use the ifgt syntax above, clang's assembler complains about the + * expression being non-absolute when the code appears in an inline assembly + * statement. + * As a workaround use an .org directive that has no effect if the else case + * instructions are smaller than the body, but fails otherwise. + */ +#define CHECK_ALT_SIZE(else_size, body_size) \ + .org . + ((else_size) > (body_size)); +#endif + #define MAKE_FTR_SECTION_ENTRY(msk, val, label, sect) \ label##4: \ .popsection; \ @@ -49,9 +66,7 @@ label##5: \ FTR_ENTRY_OFFSET label##2b-label##5b; \ FTR_ENTRY_OFFSET label##3b-label##5b; \ FTR_ENTRY_OFFSET label##4b-label##5b; \ - .ifgt (label##4b- label##3b)-(label##2b- label##1b); \ - .error "Feature section else case larger than body"; \ - .endif; \ + CHECK_ALT_SIZE((label##4b-label##3b), (label##2b-label##1b)); \ .popsection; @@ -66,6 +81,9 @@ label##5: \ #define END_FTR_SECTION(msk, val) \ END_FTR_SECTION_NESTED(msk, val, 97) +#define END_FTR_SECTION_NESTED_IFSET(msk, label) \ + END_FTR_SECTION_NESTED((msk), (msk), label) + #define END_FTR_SECTION_IFSET(msk) END_FTR_SECTION((msk), (msk)) #define END_FTR_SECTION_IFCLR(msk) END_FTR_SECTION((msk), 0) @@ -95,6 +113,12 @@ label##5: \ #define END_MMU_FTR_SECTION(msk, val) \ END_MMU_FTR_SECTION_NESTED(msk, val, 97) +#define END_MMU_FTR_SECTION_NESTED_IFSET(msk, label) \ + END_MMU_FTR_SECTION_NESTED((msk), (msk), label) + +#define END_MMU_FTR_SECTION_NESTED_IFCLR(msk, label) \ + END_MMU_FTR_SECTION_NESTED((msk), 0, label) + #define END_MMU_FTR_SECTION_IFSET(msk) END_MMU_FTR_SECTION((msk), (msk)) #define END_MMU_FTR_SECTION_IFCLR(msk) END_MMU_FTR_SECTION((msk), 0) @@ -144,7 +168,7 @@ label##5: \ #define ALT_FW_FTR_SECTION_END_IFCLR(msk) \ ALT_FW_FTR_SECTION_END_NESTED_IFCLR(msk, 97) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define ASM_FTR_IF(section_if, section_else, msk, val) \ stringify_in_c(BEGIN_FTR_SECTION) \ @@ -172,7 +196,7 @@ label##5: \ #define ASM_MMU_FTR_IFCLR(section_if, section_else, msk) \ ASM_MMU_FTR_IF(section_if, section_else, (msk), 0) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ /* LWSYNC feature sections */ #define START_LWSYNC_SECTION(label) label##1: @@ -184,4 +208,94 @@ label##3: \ FTR_ENTRY_OFFSET label##1b-label##3b; \ .popsection; +#define STF_ENTRY_BARRIER_FIXUP_SECTION \ +953: \ + .pushsection __stf_entry_barrier_fixup,"a"; \ + .align 2; \ +954: \ + FTR_ENTRY_OFFSET 953b-954b; \ + .popsection; + +#define STF_EXIT_BARRIER_FIXUP_SECTION \ +955: \ + .pushsection __stf_exit_barrier_fixup,"a"; \ + .align 2; \ +956: \ + FTR_ENTRY_OFFSET 955b-956b; \ + .popsection; + +#define UACCESS_FLUSH_FIXUP_SECTION \ +959: \ + .pushsection __uaccess_flush_fixup,"a"; \ + .align 2; \ +960: \ + FTR_ENTRY_OFFSET 959b-960b; \ + .popsection; + +#define ENTRY_FLUSH_FIXUP_SECTION \ +957: \ + .pushsection __entry_flush_fixup,"a"; \ + .align 2; \ +958: \ + FTR_ENTRY_OFFSET 957b-958b; \ + .popsection; + +#define SCV_ENTRY_FLUSH_FIXUP_SECTION \ +957: \ + .pushsection __scv_entry_flush_fixup,"a"; \ + .align 2; \ +958: \ + FTR_ENTRY_OFFSET 957b-958b; \ + .popsection; + +#define RFI_FLUSH_FIXUP_SECTION \ +951: \ + .pushsection __rfi_flush_fixup,"a"; \ + .align 2; \ +952: \ + FTR_ENTRY_OFFSET 951b-952b; \ + .popsection; + +#define NOSPEC_BARRIER_FIXUP_SECTION \ +953: \ + .pushsection __barrier_nospec_fixup,"a"; \ + .align 2; \ +954: \ + FTR_ENTRY_OFFSET 953b-954b; \ + .popsection; + +#define START_BTB_FLUSH_SECTION \ +955: \ + +#define END_BTB_FLUSH_SECTION \ +956: \ + .pushsection __btb_flush_fixup,"a"; \ + .align 2; \ +957: \ + FTR_ENTRY_OFFSET 955b-957b; \ + FTR_ENTRY_OFFSET 956b-957b; \ + .popsection; + +#ifndef __ASSEMBLER__ +#include <linux/types.h> + +extern long stf_barrier_fallback; +extern long entry_flush_fallback; +extern long scv_entry_flush_fallback; +extern long __start___stf_entry_barrier_fixup, __stop___stf_entry_barrier_fixup; +extern long __start___stf_exit_barrier_fixup, __stop___stf_exit_barrier_fixup; +extern long __start___uaccess_flush_fixup, __stop___uaccess_flush_fixup; +extern long __start___entry_flush_fixup, __stop___entry_flush_fixup; +extern long __start___scv_entry_flush_fixup, __stop___scv_entry_flush_fixup; +extern long __start___rfi_flush_fixup, __stop___rfi_flush_fixup; +extern long __start___barrier_nospec_fixup, __stop___barrier_nospec_fixup; +extern long __start__btb_flush_fixup, __stop__btb_flush_fixup; + +extern bool static_key_feature_checks_initialized; + +void apply_feature_fixups(void); +void update_mmu_feature_fixups(unsigned long mask); +void setup_feature_keys(void); +#endif + #endif /* __ASM_POWERPC_FEATURE_FIXUPS_H */ diff --git a/arch/powerpc/include/asm/firmware.h b/arch/powerpc/include/asm/firmware.h index 681bc0314b6b..abd7c56f4d55 100644 --- a/arch/powerpc/include/asm/firmware.h +++ b/arch/powerpc/include/asm/firmware.h @@ -1,21 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Copyright (C) 2001 Ben. Herrenschmidt (benh@kernel.crashing.org) * * Modifications for ppc64: * Copyright (C) 2003 Dave Engebretsen <engebret@us.ibm.com> - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef __ASM_POWERPC_FIRMWARE_H #define __ASM_POWERPC_FIRMWARE_H #ifdef __KERNEL__ -#include <asm/asm-compat.h> -#include <asm/feature-fixups.h> +#include <asm/asm-const.h> /* firmware feature bitmask values */ @@ -38,23 +33,32 @@ #define FW_FEATURE_LLAN ASM_CONST(0x0000000000010000) #define FW_FEATURE_BULK_REMOVE ASM_CONST(0x0000000000020000) #define FW_FEATURE_XDABR ASM_CONST(0x0000000000040000) -#define FW_FEATURE_MULTITCE ASM_CONST(0x0000000000080000) +#define FW_FEATURE_PUT_TCE_IND ASM_CONST(0x0000000000080000) #define FW_FEATURE_SPLPAR ASM_CONST(0x0000000000100000) #define FW_FEATURE_LPAR ASM_CONST(0x0000000000400000) #define FW_FEATURE_PS3_LV1 ASM_CONST(0x0000000000800000) -#define FW_FEATURE_BEAT ASM_CONST(0x0000000001000000) +#define FW_FEATURE_HPT_RESIZE ASM_CONST(0x0000000001000000) #define FW_FEATURE_CMO ASM_CONST(0x0000000002000000) #define FW_FEATURE_VPHN ASM_CONST(0x0000000004000000) #define FW_FEATURE_XCMO ASM_CONST(0x0000000008000000) #define FW_FEATURE_OPAL ASM_CONST(0x0000000010000000) -#define FW_FEATURE_OPALv2 ASM_CONST(0x0000000020000000) #define FW_FEATURE_SET_MODE ASM_CONST(0x0000000040000000) #define FW_FEATURE_BEST_ENERGY ASM_CONST(0x0000000080000000) -#define FW_FEATURE_TYPE1_AFFINITY ASM_CONST(0x0000000100000000) +#define FW_FEATURE_FORM1_AFFINITY ASM_CONST(0x0000000100000000) #define FW_FEATURE_PRRN ASM_CONST(0x0000000200000000) -#define FW_FEATURE_OPALv3 ASM_CONST(0x0000000400000000) +#define FW_FEATURE_DRMEM_V2 ASM_CONST(0x0000000400000000) +#define FW_FEATURE_DRC_INFO ASM_CONST(0x0000000800000000) +#define FW_FEATURE_BLOCK_REMOVE ASM_CONST(0x0000001000000000) +#define FW_FEATURE_PAPR_SCM ASM_CONST(0x0000002000000000) +#define FW_FEATURE_ULTRAVISOR ASM_CONST(0x0000004000000000) +#define FW_FEATURE_STUFF_TCE ASM_CONST(0x0000008000000000) +#define FW_FEATURE_RPT_INVALIDATE ASM_CONST(0x0000010000000000) +#define FW_FEATURE_FORM2_AFFINITY ASM_CONST(0x0000020000000000) +#define FW_FEATURE_ENERGY_SCALE_INFO ASM_CONST(0x0000040000000000) +#define FW_FEATURE_WATCHDOG ASM_CONST(0x0000080000000000) +#define FW_FEATURE_PLPKS ASM_CONST(0x0000100000000000) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ enum { #ifdef CONFIG_PPC64 @@ -65,18 +69,22 @@ enum { FW_FEATURE_MIGRATE | FW_FEATURE_PERFMON | FW_FEATURE_CRQ | FW_FEATURE_VIO | FW_FEATURE_RDMA | FW_FEATURE_LLAN | FW_FEATURE_BULK_REMOVE | FW_FEATURE_XDABR | - FW_FEATURE_MULTITCE | FW_FEATURE_SPLPAR | FW_FEATURE_LPAR | + FW_FEATURE_PUT_TCE_IND | FW_FEATURE_STUFF_TCE | + FW_FEATURE_SPLPAR | FW_FEATURE_LPAR | FW_FEATURE_CMO | FW_FEATURE_VPHN | FW_FEATURE_XCMO | FW_FEATURE_SET_MODE | FW_FEATURE_BEST_ENERGY | - FW_FEATURE_TYPE1_AFFINITY | FW_FEATURE_PRRN, + FW_FEATURE_FORM1_AFFINITY | FW_FEATURE_PRRN | + FW_FEATURE_HPT_RESIZE | FW_FEATURE_DRMEM_V2 | + FW_FEATURE_DRC_INFO | FW_FEATURE_BLOCK_REMOVE | + FW_FEATURE_PAPR_SCM | FW_FEATURE_ULTRAVISOR | + FW_FEATURE_RPT_INVALIDATE | FW_FEATURE_FORM2_AFFINITY | + FW_FEATURE_ENERGY_SCALE_INFO | FW_FEATURE_WATCHDOG | + FW_FEATURE_PLPKS, FW_FEATURE_PSERIES_ALWAYS = 0, - FW_FEATURE_POWERNV_POSSIBLE = FW_FEATURE_OPAL | FW_FEATURE_OPALv2 | - FW_FEATURE_OPALv3, + FW_FEATURE_POWERNV_POSSIBLE = FW_FEATURE_OPAL | FW_FEATURE_ULTRAVISOR, FW_FEATURE_POWERNV_ALWAYS = 0, FW_FEATURE_PS3_POSSIBLE = FW_FEATURE_LPAR | FW_FEATURE_PS3_LV1, FW_FEATURE_PS3_ALWAYS = FW_FEATURE_LPAR | FW_FEATURE_PS3_LV1, - FW_FEATURE_CELLEB_POSSIBLE = FW_FEATURE_LPAR | FW_FEATURE_BEAT, - FW_FEATURE_CELLEB_ALWAYS = 0, FW_FEATURE_NATIVE_POSSIBLE = 0, FW_FEATURE_NATIVE_ALWAYS = 0, FW_FEATURE_POSSIBLE = @@ -89,10 +97,7 @@ enum { #ifdef CONFIG_PPC_PS3 FW_FEATURE_PS3_POSSIBLE | #endif -#ifdef CONFIG_PPC_CELLEB - FW_FEATURE_CELLEB_POSSIBLE | -#endif -#ifdef CONFIG_PPC_NATIVE +#ifdef CONFIG_PPC_HASH_MMU_NATIVE FW_FEATURE_NATIVE_ALWAYS | #endif 0, @@ -106,10 +111,7 @@ enum { #ifdef CONFIG_PPC_PS3 FW_FEATURE_PS3_ALWAYS & #endif -#ifdef CONFIG_PPC_CELLEB - FW_FEATURE_CELLEB_ALWAYS & -#endif -#ifdef CONFIG_PPC_NATIVE +#ifdef CONFIG_PPC_HASH_MMU_NATIVE FW_FEATURE_NATIVE_ALWAYS & #endif FW_FEATURE_POSSIBLE, @@ -134,9 +136,16 @@ extern void machine_check_fwnmi(void); /* This is true if we are using the firmware NMI handler (typically LPAR) */ extern int fwnmi_active; +extern int ibm_nmi_interlock_token; extern unsigned int __start___fw_ftr_fixup, __stop___fw_ftr_fixup; -#endif /* __ASSEMBLY__ */ +#ifdef CONFIG_PPC_PSERIES +void pseries_probe_fw_features(void); +#else +static inline void pseries_probe_fw_features(void) { } +#endif + +#endif /* __ASSEMBLER__ */ #endif /* __KERNEL__ */ #endif /* __ASM_POWERPC_FIRMWARE_H */ diff --git a/arch/powerpc/include/asm/fixmap.h b/arch/powerpc/include/asm/fixmap.h index 5c2c0233175e..bc5109eab5b7 100644 --- a/arch/powerpc/include/asm/fixmap.h +++ b/arch/powerpc/include/asm/fixmap.h @@ -14,16 +14,15 @@ #ifndef _ASM_FIXMAP_H #define _ASM_FIXMAP_H -#ifndef __ASSEMBLY__ -#include <linux/kernel.h> +#ifndef __ASSEMBLER__ +#include <linux/sizes.h> +#include <linux/pgtable.h> #include <asm/page.h> #ifdef CONFIG_HIGHMEM #include <linux/threads.h> -#include <asm/kmap_types.h> +#include <asm/kmap_size.h> #endif -#define FIXADDR_TOP ((unsigned long)(-PAGE_SIZE)) - /* * Here we define all the compile-time 'special' virtual * addresses. The point is to have a constant address at @@ -44,66 +43,73 @@ */ enum fixed_addresses { FIX_HOLE, +#ifdef CONFIG_PPC32 /* reserve the top 128K for early debugging purposes */ FIX_EARLY_DEBUG_TOP = FIX_HOLE, - FIX_EARLY_DEBUG_BASE = FIX_EARLY_DEBUG_TOP+((128*1024)/PAGE_SIZE)-1, + FIX_EARLY_DEBUG_BASE = FIX_EARLY_DEBUG_TOP+(ALIGN(SZ_128K, PAGE_SIZE)/PAGE_SIZE)-1, #ifdef CONFIG_HIGHMEM FIX_KMAP_BEGIN, /* reserved pte's for temporary kernel mappings */ - FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1, + FIX_KMAP_END = FIX_KMAP_BEGIN + (KM_MAX_IDX * NR_CPUS) - 1, +#endif +#ifdef CONFIG_PPC_8xx + /* For IMMR we need an aligned 512K area */ +#define FIX_IMMR_SIZE (512 * 1024 / PAGE_SIZE) + FIX_IMMR_START, + FIX_IMMR_BASE = __ALIGN_MASK(FIX_IMMR_START, FIX_IMMR_SIZE - 1) - 1 + + FIX_IMMR_SIZE, +#endif +#ifdef CONFIG_PPC_83xx + /* For IMMR we need an aligned 2M area */ +#define FIX_IMMR_SIZE (SZ_2M / PAGE_SIZE) + FIX_IMMR_START, + FIX_IMMR_BASE = __ALIGN_MASK(FIX_IMMR_START, FIX_IMMR_SIZE - 1) - 1 + + FIX_IMMR_SIZE, #endif /* FIX_PCIE_MCFG, */ - __end_of_fixed_addresses -}; +#endif /* CONFIG_PPC32 */ + __end_of_permanent_fixed_addresses, -extern void __set_fixmap (enum fixed_addresses idx, - phys_addr_t phys, pgprot_t flags); +#define NR_FIX_BTMAPS (SZ_256K / PAGE_SIZE) +#define FIX_BTMAPS_SLOTS 16 +#define TOTAL_FIX_BTMAPS (NR_FIX_BTMAPS * FIX_BTMAPS_SLOTS) -#define set_fixmap(idx, phys) \ - __set_fixmap(idx, phys, PAGE_KERNEL) -/* - * Some hardware wants to get fixmapped without caching. - */ -#define set_fixmap_nocache(idx, phys) \ - __set_fixmap(idx, phys, PAGE_KERNEL_NCG) - -#define clear_fixmap(idx) \ - __set_fixmap(idx, 0, __pgprot(0)) + FIX_BTMAP_END = __end_of_permanent_fixed_addresses, + FIX_BTMAP_BEGIN = FIX_BTMAP_END + TOTAL_FIX_BTMAPS - 1, + __end_of_fixed_addresses +}; #define __FIXADDR_SIZE (__end_of_fixed_addresses << PAGE_SHIFT) #define FIXADDR_START (FIXADDR_TOP - __FIXADDR_SIZE) -#define __fix_to_virt(x) (FIXADDR_TOP - ((x) << PAGE_SHIFT)) -#define __virt_to_fix(x) ((FIXADDR_TOP - ((x)&PAGE_MASK)) >> PAGE_SHIFT) +#define FIXMAP_ALIGNED_SIZE (ALIGN(FIXADDR_TOP, PGDIR_SIZE) - \ + ALIGN_DOWN(FIXADDR_START, PGDIR_SIZE)) +#define FIXMAP_PTE_SIZE (FIXMAP_ALIGNED_SIZE / PGDIR_SIZE * PTE_TABLE_SIZE) -extern void __this_fixmap_does_not_exist(void); +#define FIXMAP_PAGE_NOCACHE PAGE_KERNEL_NCG +#define FIXMAP_PAGE_IO PAGE_KERNEL_NCG -/* - * 'index to address' translation. If anyone tries to use the idx - * directly without tranlation, we catch the bug with a NULL-deference - * kernel oops. Illegal ranges of incoming indices are caught too. - */ -static __always_inline unsigned long fix_to_virt(const unsigned int idx) +#include <asm-generic/fixmap.h> + +static inline void __set_fixmap(enum fixed_addresses idx, + phys_addr_t phys, pgprot_t flags) { - /* - * this branch gets completely eliminated after inlining, - * except when someone tries to use fixaddr indices in an - * illegal way. (such as mixing up address types or using - * out-of-range indices). - * - * If it doesn't get removed, the linker will complain - * loudly with a reasonably clear error message.. - */ - if (idx >= __end_of_fixed_addresses) - __this_fixmap_does_not_exist(); + BUILD_BUG_ON(IS_ENABLED(CONFIG_PPC64) && __FIXADDR_SIZE > FIXADDR_SIZE); - return __fix_to_virt(idx); + if (__builtin_constant_p(idx)) + BUILD_BUG_ON(idx >= __end_of_fixed_addresses); + else if (WARN_ON(idx >= __end_of_fixed_addresses)) + return; + if (pgprot_val(flags)) + map_kernel_page(__fix_to_virt(idx), phys, flags); + else + unmap_kernel_page(__fix_to_virt(idx)); } -static inline unsigned long virt_to_fix(const unsigned long vaddr) -{ - BUG_ON(vaddr >= FIXADDR_TOP || vaddr < FIXADDR_START); - return __virt_to_fix(vaddr); -} +#define __early_set_fixmap __set_fixmap + +#ifdef CONFIG_PPC_8xx +#define VIRT_IMMR_BASE (__fix_to_virt(FIX_IMMR_BASE)) +#endif -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif diff --git a/arch/powerpc/include/asm/floppy.h b/arch/powerpc/include/asm/floppy.h index 936a904ae78c..f4dc657638b3 100644 --- a/arch/powerpc/include/asm/floppy.h +++ b/arch/powerpc/include/asm/floppy.h @@ -13,8 +13,8 @@ #include <asm/machdep.h> -#define fd_inb(port) inb_p(port) -#define fd_outb(value,port) outb_p(value,port) +#define fd_inb(base, reg) inb_p((base) + (reg)) +#define fd_outb(value, base, reg) outb_p(value, (base) + (reg)) #define fd_enable_dma() enable_dma(FLOPPY_DMA) #define fd_disable_dma() fd_ops->_disable_dma(FLOPPY_DMA) @@ -25,7 +25,6 @@ #define fd_get_dma_residue() fd_ops->_get_dma_residue(FLOPPY_DMA) #define fd_enable_irq() enable_irq(FLOPPY_IRQ) #define fd_disable_irq() disable_irq(FLOPPY_IRQ) -#define fd_cacheflush(addr,size) /* nothing */ #define fd_free_irq() free_irq(FLOPPY_IRQ, NULL); #include <linux/pci.h> @@ -62,21 +61,22 @@ static irqreturn_t floppy_hardint(int irq, void *dev_id) st = 1; for (lcount=virtual_dma_count, lptr=virtual_dma_addr; lcount; lcount--, lptr++) { - st=inb(virtual_dma_port+4) & 0xa0 ; - if (st != 0xa0) + st = inb(virtual_dma_port + FD_STATUS); + st &= STATUS_DMA | STATUS_READY; + if (st != (STATUS_DMA | STATUS_READY)) break; if (virtual_dma_mode) - outb_p(*lptr, virtual_dma_port+5); + outb_p(*lptr, virtual_dma_port + FD_DATA); else - *lptr = inb_p(virtual_dma_port+5); + *lptr = inb_p(virtual_dma_port + FD_DATA); } virtual_dma_count = lcount; virtual_dma_addr = lptr; - st = inb(virtual_dma_port+4); + st = inb(virtual_dma_port + FD_STATUS); - if (st == 0x20) + if (st == STATUS_DMA) return IRQ_HANDLED; - if (!(st & 0x20)) { + if (!(st & STATUS_DMA)) { virtual_dma_residue += virtual_dma_count; virtual_dma_count=0; doing_vdma = 0; @@ -134,17 +134,22 @@ static int hard_dma_setup(char *addr, unsigned long size, int mode, int io) int dir; doing_vdma = 0; - dir = (mode == DMA_MODE_READ) ? PCI_DMA_FROMDEVICE : PCI_DMA_TODEVICE; + dir = (mode == DMA_MODE_READ) ? DMA_FROM_DEVICE : DMA_TO_DEVICE; if (bus_addr && (addr != prev_addr || size != prev_size || dir != prev_dir)) { /* different from last time -- unmap prev */ - pci_unmap_single(isa_bridge_pcidev, bus_addr, prev_size, prev_dir); + dma_unmap_single(&isa_bridge_pcidev->dev, bus_addr, prev_size, + prev_dir); bus_addr = 0; } - if (!bus_addr) /* need to map it */ - bus_addr = pci_map_single(isa_bridge_pcidev, addr, size, dir); + if (!bus_addr) { /* need to map it */ + bus_addr = dma_map_single(&isa_bridge_pcidev->dev, addr, size, + dir); + if (dma_mapping_error(&isa_bridge_pcidev->dev, bus_addr)) + return -ENOMEM; + } /* remember this one as prev */ prev_addr = addr; @@ -152,7 +157,6 @@ static int hard_dma_setup(char *addr, unsigned long size, int mode, int io) prev_dir = dir; fd_clear_dma_ff(); - fd_cacheflush(addr, size); fd_set_dma_mode(mode); set_dma_addr(FLOPPY_DMA, bus_addr); fd_set_dma_count(size); @@ -202,11 +206,6 @@ static int FDC2 = -1; #define N_FDC 2 /* Don't change this! */ #define N_DRIVE 8 -/* - * The PowerPC has no problems with floppy DMA crossing 64k borders. - */ -#define CROSS_64KB(a,s) (0) - #define EXTRA_FLOPPY_PARAMS #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/fprobe.h b/arch/powerpc/include/asm/fprobe.h new file mode 100644 index 000000000000..d64bc28fb3d3 --- /dev/null +++ b/arch/powerpc/include/asm/fprobe.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_PPC_FPROBE_H +#define _ASM_PPC_FPROBE_H + +#include <asm-generic/fprobe.h> + +#ifdef CONFIG_64BIT +#undef FPROBE_HEADER_MSB_PATTERN +#define FPROBE_HEADER_MSB_PATTERN (PAGE_OFFSET & ~FPROBE_HEADER_MSB_MASK) +#endif + +#endif /* _ASM_PPC_FPROBE_H */ diff --git a/arch/powerpc/include/asm/fpu.h b/arch/powerpc/include/asm/fpu.h new file mode 100644 index 000000000000..ca584e4bc40f --- /dev/null +++ b/arch/powerpc/include/asm/fpu.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2023 SiFive + */ + +#ifndef _ASM_POWERPC_FPU_H +#define _ASM_POWERPC_FPU_H + +#include <linux/preempt.h> + +#include <asm/cpu_has_feature.h> +#include <asm/switch_to.h> + +#define kernel_fpu_available() (!cpu_has_feature(CPU_FTR_FPU_UNAVAILABLE)) + +static inline void kernel_fpu_begin(void) +{ + preempt_disable(); + enable_kernel_fp(); +} + +static inline void kernel_fpu_end(void) +{ + disable_kernel_fp(); + preempt_enable(); +} + +#endif /* ! _ASM_POWERPC_FPU_H */ diff --git a/arch/powerpc/include/asm/fs_pd.h b/arch/powerpc/include/asm/fs_pd.h deleted file mode 100644 index 9361cd5342cc..000000000000 --- a/arch/powerpc/include/asm/fs_pd.h +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Platform information definitions. - * - * 2006 (c) MontaVista Software, Inc. - * Vitaly Bordug <vbordug@ru.mvista.com> - * - * This file is licensed under the terms of the GNU General Public License - * version 2. This program is licensed "as is" without any warranty of any - * kind, whether express or implied. - */ - -#ifndef FS_PD_H -#define FS_PD_H -#include <sysdev/fsl_soc.h> -#include <asm/time.h> - -#ifdef CONFIG_CPM2 -#include <asm/cpm2.h> - -#if defined(CONFIG_8260) -#include <asm/mpc8260.h> -#endif - -#define cpm2_map(member) (&cpm2_immr->member) -#define cpm2_map_size(member, size) (&cpm2_immr->member) -#define cpm2_unmap(addr) do {} while(0) -#endif - -#ifdef CONFIG_8xx -#include <asm/8xx_immap.h> -#include <asm/mpc8xx.h> - -extern immap_t __iomem *mpc8xx_immr; - -#define immr_map(member) (&mpc8xx_immr->member) -#define immr_map_size(member, size) (&mpc8xx_immr->member) -#define immr_unmap(addr) do {} while (0) -#endif - -static inline int uart_baudrate(void) -{ - return get_baudrate(); -} - -static inline int uart_clock(void) -{ - return ppc_proc_freq; -} - -#endif diff --git a/arch/powerpc/include/asm/fsl_85xx_cache_sram.h b/arch/powerpc/include/asm/fsl_85xx_cache_sram.h deleted file mode 100644 index 2af2bdc37b2e..000000000000 --- a/arch/powerpc/include/asm/fsl_85xx_cache_sram.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2009 Freescale Semiconductor, Inc. - * - * Cache SRAM handling for QorIQ platform - * - * Author: Vivek Mahajan <vivek.mahajan@freescale.com> - - * This file is derived from the original work done - * by Sylvain Munaut for the Bestcomm SRAM allocator. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#ifndef __ASM_POWERPC_FSL_85XX_CACHE_SRAM_H__ -#define __ASM_POWERPC_FSL_85XX_CACHE_SRAM_H__ - -#include <asm/rheap.h> -#include <linux/spinlock.h> - -/* - * Cache-SRAM - */ - -struct mpc85xx_cache_sram { - phys_addr_t base_phys; - void *base_virt; - unsigned int size; - rh_info_t *rh; - spinlock_t lock; -}; - -extern void mpc85xx_cache_sram_free(void *ptr); -extern void *mpc85xx_cache_sram_alloc(unsigned int size, - phys_addr_t *phys, unsigned int align); - -#endif /* __AMS_POWERPC_FSL_85XX_CACHE_SRAM_H__ */ diff --git a/arch/powerpc/include/asm/fsl_gtm.h b/arch/powerpc/include/asm/fsl_gtm.h index 3b05808f9caa..6ff68765094d 100644 --- a/arch/powerpc/include/asm/fsl_gtm.h +++ b/arch/powerpc/include/asm/fsl_gtm.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Freescale General-purpose Timers Module * @@ -6,11 +7,6 @@ * Jerry Huang <Chang-Ming.Huang@freescale.com> * Copyright (c) MontaVista Software, Inc. 2008. * Anton Vorontsov <avorontsov@ru.mvista.com> - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. */ #ifndef __ASM_FSL_GTM_H diff --git a/arch/powerpc/include/asm/fsl_guts.h b/arch/powerpc/include/asm/fsl_guts.h deleted file mode 100644 index 77ced0b3d81d..000000000000 --- a/arch/powerpc/include/asm/fsl_guts.h +++ /dev/null @@ -1,189 +0,0 @@ -/** - * Freecale 85xx and 86xx Global Utilties register set - * - * Authors: Jeff Brown - * Timur Tabi <timur@freescale.com> - * - * Copyright 2004,2007,2012 Freescale Semiconductor, Inc - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - */ - -#ifndef __ASM_POWERPC_FSL_GUTS_H__ -#define __ASM_POWERPC_FSL_GUTS_H__ -#ifdef __KERNEL__ - -/** - * Global Utility Registers. - * - * Not all registers defined in this structure are available on all chips, so - * you are expected to know whether a given register actually exists on your - * chip before you access it. - * - * Also, some registers are similar on different chips but have slightly - * different names. In these cases, one name is chosen to avoid extraneous - * #ifdefs. - */ -struct ccsr_guts { - __be32 porpllsr; /* 0x.0000 - POR PLL Ratio Status Register */ - __be32 porbmsr; /* 0x.0004 - POR Boot Mode Status Register */ - __be32 porimpscr; /* 0x.0008 - POR I/O Impedance Status and Control Register */ - __be32 pordevsr; /* 0x.000c - POR I/O Device Status Register */ - __be32 pordbgmsr; /* 0x.0010 - POR Debug Mode Status Register */ - __be32 pordevsr2; /* 0x.0014 - POR device status register 2 */ - u8 res018[0x20 - 0x18]; - __be32 porcir; /* 0x.0020 - POR Configuration Information Register */ - u8 res024[0x30 - 0x24]; - __be32 gpiocr; /* 0x.0030 - GPIO Control Register */ - u8 res034[0x40 - 0x34]; - __be32 gpoutdr; /* 0x.0040 - General-Purpose Output Data Register */ - u8 res044[0x50 - 0x44]; - __be32 gpindr; /* 0x.0050 - General-Purpose Input Data Register */ - u8 res054[0x60 - 0x54]; - __be32 pmuxcr; /* 0x.0060 - Alternate Function Signal Multiplex Control */ - __be32 pmuxcr2; /* 0x.0064 - Alternate function signal multiplex control 2 */ - __be32 dmuxcr; /* 0x.0068 - DMA Mux Control Register */ - u8 res06c[0x70 - 0x6c]; - __be32 devdisr; /* 0x.0070 - Device Disable Control */ -#define CCSR_GUTS_DEVDISR_TB1 0x00001000 -#define CCSR_GUTS_DEVDISR_TB0 0x00004000 - __be32 devdisr2; /* 0x.0074 - Device Disable Control 2 */ - u8 res078[0x7c - 0x78]; - __be32 pmjcr; /* 0x.007c - 4 Power Management Jog Control Register */ - __be32 powmgtcsr; /* 0x.0080 - Power Management Status and Control Register */ - __be32 pmrccr; /* 0x.0084 - Power Management Reset Counter Configuration Register */ - __be32 pmpdccr; /* 0x.0088 - Power Management Power Down Counter Configuration Register */ - __be32 pmcdr; /* 0x.008c - 4Power management clock disable register */ - __be32 mcpsumr; /* 0x.0090 - Machine Check Summary Register */ - __be32 rstrscr; /* 0x.0094 - Reset Request Status and Control Register */ - __be32 ectrstcr; /* 0x.0098 - Exception reset control register */ - __be32 autorstsr; /* 0x.009c - Automatic reset status register */ - __be32 pvr; /* 0x.00a0 - Processor Version Register */ - __be32 svr; /* 0x.00a4 - System Version Register */ - u8 res0a8[0xb0 - 0xa8]; - __be32 rstcr; /* 0x.00b0 - Reset Control Register */ - u8 res0b4[0xc0 - 0xb4]; - __be32 iovselsr; /* 0x.00c0 - I/O voltage select status register - Called 'elbcvselcr' on 86xx SOCs */ - u8 res0c4[0x224 - 0xc4]; - __be32 iodelay1; /* 0x.0224 - IO delay control register 1 */ - __be32 iodelay2; /* 0x.0228 - IO delay control register 2 */ - u8 res22c[0x604 - 0x22c]; - __be32 pamubypenr; /* 0x.604 - PAMU bypass enable register */ - u8 res608[0x800 - 0x608]; - __be32 clkdvdr; /* 0x.0800 - Clock Divide Register */ - u8 res804[0x900 - 0x804]; - __be32 ircr; /* 0x.0900 - Infrared Control Register */ - u8 res904[0x908 - 0x904]; - __be32 dmacr; /* 0x.0908 - DMA Control Register */ - u8 res90c[0x914 - 0x90c]; - __be32 elbccr; /* 0x.0914 - eLBC Control Register */ - u8 res918[0xb20 - 0x918]; - __be32 ddr1clkdr; /* 0x.0b20 - DDR1 Clock Disable Register */ - __be32 ddr2clkdr; /* 0x.0b24 - DDR2 Clock Disable Register */ - __be32 ddrclkdr; /* 0x.0b28 - DDR Clock Disable Register */ - u8 resb2c[0xe00 - 0xb2c]; - __be32 clkocr; /* 0x.0e00 - Clock Out Select Register */ - u8 rese04[0xe10 - 0xe04]; - __be32 ddrdllcr; /* 0x.0e10 - DDR DLL Control Register */ - u8 rese14[0xe20 - 0xe14]; - __be32 lbcdllcr; /* 0x.0e20 - LBC DLL Control Register */ - __be32 cpfor; /* 0x.0e24 - L2 charge pump fuse override register */ - u8 rese28[0xf04 - 0xe28]; - __be32 srds1cr0; /* 0x.0f04 - SerDes1 Control Register 0 */ - __be32 srds1cr1; /* 0x.0f08 - SerDes1 Control Register 0 */ - u8 resf0c[0xf2c - 0xf0c]; - __be32 itcr; /* 0x.0f2c - Internal transaction control register */ - u8 resf30[0xf40 - 0xf30]; - __be32 srds2cr0; /* 0x.0f40 - SerDes2 Control Register 0 */ - __be32 srds2cr1; /* 0x.0f44 - SerDes2 Control Register 0 */ -} __attribute__ ((packed)); - - -/* Alternate function signal multiplex control */ -#define MPC85xx_PMUXCR_QE(x) (0x8000 >> (x)) - -#ifdef CONFIG_PPC_86xx - -#define CCSR_GUTS_DMACR_DEV_SSI 0 /* DMA controller/channel set to SSI */ -#define CCSR_GUTS_DMACR_DEV_IR 1 /* DMA controller/channel set to IR */ - -/* - * Set the DMACR register in the GUTS - * - * The DMACR register determines the source of initiated transfers for each - * channel on each DMA controller. Rather than have a bunch of repetitive - * macros for the bit patterns, we just have a function that calculates - * them. - * - * guts: Pointer to GUTS structure - * co: The DMA controller (0 or 1) - * ch: The channel on the DMA controller (0, 1, 2, or 3) - * device: The device to set as the source (CCSR_GUTS_DMACR_DEV_xx) - */ -static inline void guts_set_dmacr(struct ccsr_guts __iomem *guts, - unsigned int co, unsigned int ch, unsigned int device) -{ - unsigned int shift = 16 + (8 * (1 - co) + 2 * (3 - ch)); - - clrsetbits_be32(&guts->dmacr, 3 << shift, device << shift); -} - -#define CCSR_GUTS_PMUXCR_LDPSEL 0x00010000 -#define CCSR_GUTS_PMUXCR_SSI1_MASK 0x0000C000 /* Bitmask for SSI1 */ -#define CCSR_GUTS_PMUXCR_SSI1_LA 0x00000000 /* Latched address */ -#define CCSR_GUTS_PMUXCR_SSI1_HI 0x00004000 /* High impedance */ -#define CCSR_GUTS_PMUXCR_SSI1_SSI 0x00008000 /* Used for SSI1 */ -#define CCSR_GUTS_PMUXCR_SSI2_MASK 0x00003000 /* Bitmask for SSI2 */ -#define CCSR_GUTS_PMUXCR_SSI2_LA 0x00000000 /* Latched address */ -#define CCSR_GUTS_PMUXCR_SSI2_HI 0x00001000 /* High impedance */ -#define CCSR_GUTS_PMUXCR_SSI2_SSI 0x00002000 /* Used for SSI2 */ -#define CCSR_GUTS_PMUXCR_LA_22_25_LA 0x00000000 /* Latched Address */ -#define CCSR_GUTS_PMUXCR_LA_22_25_HI 0x00000400 /* High impedance */ -#define CCSR_GUTS_PMUXCR_DBGDRV 0x00000200 /* Signals not driven */ -#define CCSR_GUTS_PMUXCR_DMA2_0 0x00000008 -#define CCSR_GUTS_PMUXCR_DMA2_3 0x00000004 -#define CCSR_GUTS_PMUXCR_DMA1_0 0x00000002 -#define CCSR_GUTS_PMUXCR_DMA1_3 0x00000001 - -/* - * Set the DMA external control bits in the GUTS - * - * The DMA external control bits in the PMUXCR are only meaningful for - * channels 0 and 3. Any other channels are ignored. - * - * guts: Pointer to GUTS structure - * co: The DMA controller (0 or 1) - * ch: The channel on the DMA controller (0, 1, 2, or 3) - * value: the new value for the bit (0 or 1) - */ -static inline void guts_set_pmuxcr_dma(struct ccsr_guts __iomem *guts, - unsigned int co, unsigned int ch, unsigned int value) -{ - if ((ch == 0) || (ch == 3)) { - unsigned int shift = 2 * (co + 1) - (ch & 1) - 1; - - clrsetbits_be32(&guts->pmuxcr, 1 << shift, value << shift); - } -} - -#define CCSR_GUTS_CLKDVDR_PXCKEN 0x80000000 -#define CCSR_GUTS_CLKDVDR_SSICKEN 0x20000000 -#define CCSR_GUTS_CLKDVDR_PXCKINV 0x10000000 -#define CCSR_GUTS_CLKDVDR_PXCKDLY_SHIFT 25 -#define CCSR_GUTS_CLKDVDR_PXCKDLY_MASK 0x06000000 -#define CCSR_GUTS_CLKDVDR_PXCKDLY(x) \ - (((x) & 3) << CCSR_GUTS_CLKDVDR_PXCKDLY_SHIFT) -#define CCSR_GUTS_CLKDVDR_PXCLK_SHIFT 16 -#define CCSR_GUTS_CLKDVDR_PXCLK_MASK 0x001F0000 -#define CCSR_GUTS_CLKDVDR_PXCLK(x) (((x) & 31) << CCSR_GUTS_CLKDVDR_PXCLK_SHIFT) -#define CCSR_GUTS_CLKDVDR_SSICLK_MASK 0x000000FF -#define CCSR_GUTS_CLKDVDR_SSICLK(x) ((x) & CCSR_GUTS_CLKDVDR_SSICLK_MASK) - -#endif - -#endif -#endif diff --git a/arch/powerpc/include/asm/fsl_hcalls.h b/arch/powerpc/include/asm/fsl_hcalls.h index 3abb58394da4..b889d13547fd 100644 --- a/arch/powerpc/include/asm/fsl_hcalls.h +++ b/arch/powerpc/include/asm/fsl_hcalls.h @@ -109,7 +109,7 @@ static inline unsigned int fh_send_nmi(unsigned int vcpu_mask) #define FH_DTPROP_MAX_PROPLEN 32768 /** - * fh_partiton_get_dtprop - get a property from a guest device tree. + * fh_partition_get_dtprop - get a property from a guest device tree. * @handle: handle of partition whose device tree is to be accessed * @dtpath_addr: physical address of device tree path to access * @propname_addr: physical address of name of property diff --git a/arch/powerpc/include/asm/fsl_ifc.h b/arch/powerpc/include/asm/fsl_ifc.h deleted file mode 100644 index b8a4b9bc50b3..000000000000 --- a/arch/powerpc/include/asm/fsl_ifc.h +++ /dev/null @@ -1,836 +0,0 @@ -/* Freescale Integrated Flash Controller - * - * Copyright 2011 Freescale Semiconductor, Inc - * - * Author: Dipen Dudhat <dipen.dudhat@freescale.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#ifndef __ASM_FSL_IFC_H -#define __ASM_FSL_IFC_H - -#include <linux/compiler.h> -#include <linux/types.h> -#include <linux/io.h> - -#include <linux/of_platform.h> -#include <linux/interrupt.h> - -#define FSL_IFC_BANK_COUNT 4 - -/* - * CSPR - Chip Select Property Register - */ -#define CSPR_BA 0xFFFF0000 -#define CSPR_BA_SHIFT 16 -#define CSPR_PORT_SIZE 0x00000180 -#define CSPR_PORT_SIZE_SHIFT 7 -/* Port Size 8 bit */ -#define CSPR_PORT_SIZE_8 0x00000080 -/* Port Size 16 bit */ -#define CSPR_PORT_SIZE_16 0x00000100 -/* Port Size 32 bit */ -#define CSPR_PORT_SIZE_32 0x00000180 -/* Write Protect */ -#define CSPR_WP 0x00000040 -#define CSPR_WP_SHIFT 6 -/* Machine Select */ -#define CSPR_MSEL 0x00000006 -#define CSPR_MSEL_SHIFT 1 -/* NOR */ -#define CSPR_MSEL_NOR 0x00000000 -/* NAND */ -#define CSPR_MSEL_NAND 0x00000002 -/* GPCM */ -#define CSPR_MSEL_GPCM 0x00000004 -/* Bank Valid */ -#define CSPR_V 0x00000001 -#define CSPR_V_SHIFT 0 - -/* - * Address Mask Register - */ -#define IFC_AMASK_MASK 0xFFFF0000 -#define IFC_AMASK_SHIFT 16 -#define IFC_AMASK(n) (IFC_AMASK_MASK << \ - (__ilog2(n) - IFC_AMASK_SHIFT)) - -/* - * Chip Select Option Register IFC_NAND Machine - */ -/* Enable ECC Encoder */ -#define CSOR_NAND_ECC_ENC_EN 0x80000000 -#define CSOR_NAND_ECC_MODE_MASK 0x30000000 -/* 4 bit correction per 520 Byte sector */ -#define CSOR_NAND_ECC_MODE_4 0x00000000 -/* 8 bit correction per 528 Byte sector */ -#define CSOR_NAND_ECC_MODE_8 0x10000000 -/* Enable ECC Decoder */ -#define CSOR_NAND_ECC_DEC_EN 0x04000000 -/* Row Address Length */ -#define CSOR_NAND_RAL_MASK 0x01800000 -#define CSOR_NAND_RAL_SHIFT 20 -#define CSOR_NAND_RAL_1 0x00000000 -#define CSOR_NAND_RAL_2 0x00800000 -#define CSOR_NAND_RAL_3 0x01000000 -#define CSOR_NAND_RAL_4 0x01800000 -/* Page Size 512b, 2k, 4k */ -#define CSOR_NAND_PGS_MASK 0x00180000 -#define CSOR_NAND_PGS_SHIFT 16 -#define CSOR_NAND_PGS_512 0x00000000 -#define CSOR_NAND_PGS_2K 0x00080000 -#define CSOR_NAND_PGS_4K 0x00100000 -/* Spare region Size */ -#define CSOR_NAND_SPRZ_MASK 0x0000E000 -#define CSOR_NAND_SPRZ_SHIFT 13 -#define CSOR_NAND_SPRZ_16 0x00000000 -#define CSOR_NAND_SPRZ_64 0x00002000 -#define CSOR_NAND_SPRZ_128 0x00004000 -#define CSOR_NAND_SPRZ_210 0x00006000 -#define CSOR_NAND_SPRZ_218 0x00008000 -#define CSOR_NAND_SPRZ_224 0x0000A000 -/* Pages Per Block */ -#define CSOR_NAND_PB_MASK 0x00000700 -#define CSOR_NAND_PB_SHIFT 8 -#define CSOR_NAND_PB(n) ((__ilog2(n) - 5) << CSOR_NAND_PB_SHIFT) -/* Time for Read Enable High to Output High Impedance */ -#define CSOR_NAND_TRHZ_MASK 0x0000001C -#define CSOR_NAND_TRHZ_SHIFT 2 -#define CSOR_NAND_TRHZ_20 0x00000000 -#define CSOR_NAND_TRHZ_40 0x00000004 -#define CSOR_NAND_TRHZ_60 0x00000008 -#define CSOR_NAND_TRHZ_80 0x0000000C -#define CSOR_NAND_TRHZ_100 0x00000010 -/* Buffer control disable */ -#define CSOR_NAND_BCTLD 0x00000001 - -/* - * Chip Select Option Register - NOR Flash Mode - */ -/* Enable Address shift Mode */ -#define CSOR_NOR_ADM_SHFT_MODE_EN 0x80000000 -/* Page Read Enable from NOR device */ -#define CSOR_NOR_PGRD_EN 0x10000000 -/* AVD Toggle Enable during Burst Program */ -#define CSOR_NOR_AVD_TGL_PGM_EN 0x01000000 -/* Address Data Multiplexing Shift */ -#define CSOR_NOR_ADM_MASK 0x0003E000 -#define CSOR_NOR_ADM_SHIFT_SHIFT 13 -#define CSOR_NOR_ADM_SHIFT(n) ((n) << CSOR_NOR_ADM_SHIFT_SHIFT) -/* Type of the NOR device hooked */ -#define CSOR_NOR_NOR_MODE_AYSNC_NOR 0x00000000 -#define CSOR_NOR_NOR_MODE_AVD_NOR 0x00000020 -/* Time for Read Enable High to Output High Impedance */ -#define CSOR_NOR_TRHZ_MASK 0x0000001C -#define CSOR_NOR_TRHZ_SHIFT 2 -#define CSOR_NOR_TRHZ_20 0x00000000 -#define CSOR_NOR_TRHZ_40 0x00000004 -#define CSOR_NOR_TRHZ_60 0x00000008 -#define CSOR_NOR_TRHZ_80 0x0000000C -#define CSOR_NOR_TRHZ_100 0x00000010 -/* Buffer control disable */ -#define CSOR_NOR_BCTLD 0x00000001 - -/* - * Chip Select Option Register - GPCM Mode - */ -/* GPCM Mode - Normal */ -#define CSOR_GPCM_GPMODE_NORMAL 0x00000000 -/* GPCM Mode - GenericASIC */ -#define CSOR_GPCM_GPMODE_ASIC 0x80000000 -/* Parity Mode odd/even */ -#define CSOR_GPCM_PARITY_EVEN 0x40000000 -/* Parity Checking enable/disable */ -#define CSOR_GPCM_PAR_EN 0x20000000 -/* GPCM Timeout Count */ -#define CSOR_GPCM_GPTO_MASK 0x0F000000 -#define CSOR_GPCM_GPTO_SHIFT 24 -#define CSOR_GPCM_GPTO(n) ((__ilog2(n) - 8) << CSOR_GPCM_GPTO_SHIFT) -/* GPCM External Access Termination mode for read access */ -#define CSOR_GPCM_RGETA_EXT 0x00080000 -/* GPCM External Access Termination mode for write access */ -#define CSOR_GPCM_WGETA_EXT 0x00040000 -/* Address Data Multiplexing Shift */ -#define CSOR_GPCM_ADM_MASK 0x0003E000 -#define CSOR_GPCM_ADM_SHIFT_SHIFT 13 -#define CSOR_GPCM_ADM_SHIFT(n) ((n) << CSOR_GPCM_ADM_SHIFT_SHIFT) -/* Generic ASIC Parity error indication delay */ -#define CSOR_GPCM_GAPERRD_MASK 0x00000180 -#define CSOR_GPCM_GAPERRD_SHIFT 7 -#define CSOR_GPCM_GAPERRD(n) (((n) - 1) << CSOR_GPCM_GAPERRD_SHIFT) -/* Time for Read Enable High to Output High Impedance */ -#define CSOR_GPCM_TRHZ_MASK 0x0000001C -#define CSOR_GPCM_TRHZ_20 0x00000000 -#define CSOR_GPCM_TRHZ_40 0x00000004 -#define CSOR_GPCM_TRHZ_60 0x00000008 -#define CSOR_GPCM_TRHZ_80 0x0000000C -#define CSOR_GPCM_TRHZ_100 0x00000010 -/* Buffer control disable */ -#define CSOR_GPCM_BCTLD 0x00000001 - -/* - * Ready Busy Status Register (RB_STAT) - */ -/* CSn is READY */ -#define IFC_RB_STAT_READY_CS0 0x80000000 -#define IFC_RB_STAT_READY_CS1 0x40000000 -#define IFC_RB_STAT_READY_CS2 0x20000000 -#define IFC_RB_STAT_READY_CS3 0x10000000 - -/* - * General Control Register (GCR) - */ -#define IFC_GCR_MASK 0x8000F800 -/* reset all IFC hardware */ -#define IFC_GCR_SOFT_RST_ALL 0x80000000 -/* Turnaroud Time of external buffer */ -#define IFC_GCR_TBCTL_TRN_TIME 0x0000F800 -#define IFC_GCR_TBCTL_TRN_TIME_SHIFT 11 - -/* - * Common Event and Error Status Register (CM_EVTER_STAT) - */ -/* Chip select error */ -#define IFC_CM_EVTER_STAT_CSER 0x80000000 - -/* - * Common Event and Error Enable Register (CM_EVTER_EN) - */ -/* Chip select error checking enable */ -#define IFC_CM_EVTER_EN_CSEREN 0x80000000 - -/* - * Common Event and Error Interrupt Enable Register (CM_EVTER_INTR_EN) - */ -/* Chip select error interrupt enable */ -#define IFC_CM_EVTER_INTR_EN_CSERIREN 0x80000000 - -/* - * Common Transfer Error Attribute Register-0 (CM_ERATTR0) - */ -/* transaction type of error Read/Write */ -#define IFC_CM_ERATTR0_ERTYP_READ 0x80000000 -#define IFC_CM_ERATTR0_ERAID 0x0FF00000 -#define IFC_CM_ERATTR0_ERAID_SHIFT 20 -#define IFC_CM_ERATTR0_ESRCID 0x0000FF00 -#define IFC_CM_ERATTR0_ESRCID_SHIFT 8 - -/* - * Clock Control Register (CCR) - */ -#define IFC_CCR_MASK 0x0F0F8800 -/* Clock division ratio */ -#define IFC_CCR_CLK_DIV_MASK 0x0F000000 -#define IFC_CCR_CLK_DIV_SHIFT 24 -#define IFC_CCR_CLK_DIV(n) ((n-1) << IFC_CCR_CLK_DIV_SHIFT) -/* IFC Clock Delay */ -#define IFC_CCR_CLK_DLY_MASK 0x000F0000 -#define IFC_CCR_CLK_DLY_SHIFT 16 -#define IFC_CCR_CLK_DLY(n) ((n) << IFC_CCR_CLK_DLY_SHIFT) -/* Invert IFC clock before sending out */ -#define IFC_CCR_INV_CLK_EN 0x00008000 -/* Fedback IFC Clock */ -#define IFC_CCR_FB_IFC_CLK_SEL 0x00000800 - -/* - * Clock Status Register (CSR) - */ -/* Clk is stable */ -#define IFC_CSR_CLK_STAT_STABLE 0x80000000 - -/* - * IFC_NAND Machine Specific Registers - */ -/* - * NAND Configuration Register (NCFGR) - */ -/* Auto Boot Mode */ -#define IFC_NAND_NCFGR_BOOT 0x80000000 -/* Addressing Mode-ROW0+n/COL0 */ -#define IFC_NAND_NCFGR_ADDR_MODE_RC0 0x00000000 -/* Addressing Mode-ROW0+n/COL0+n */ -#define IFC_NAND_NCFGR_ADDR_MODE_RC1 0x00400000 -/* Number of loop iterations of FIR sequences for multi page operations */ -#define IFC_NAND_NCFGR_NUM_LOOP_MASK 0x0000F000 -#define IFC_NAND_NCFGR_NUM_LOOP_SHIFT 12 -#define IFC_NAND_NCFGR_NUM_LOOP(n) ((n) << IFC_NAND_NCFGR_NUM_LOOP_SHIFT) -/* Number of wait cycles */ -#define IFC_NAND_NCFGR_NUM_WAIT_MASK 0x000000FF -#define IFC_NAND_NCFGR_NUM_WAIT_SHIFT 0 - -/* - * NAND Flash Command Registers (NAND_FCR0/NAND_FCR1) - */ -/* General purpose FCM flash command bytes CMD0-CMD7 */ -#define IFC_NAND_FCR0_CMD0 0xFF000000 -#define IFC_NAND_FCR0_CMD0_SHIFT 24 -#define IFC_NAND_FCR0_CMD1 0x00FF0000 -#define IFC_NAND_FCR0_CMD1_SHIFT 16 -#define IFC_NAND_FCR0_CMD2 0x0000FF00 -#define IFC_NAND_FCR0_CMD2_SHIFT 8 -#define IFC_NAND_FCR0_CMD3 0x000000FF -#define IFC_NAND_FCR0_CMD3_SHIFT 0 -#define IFC_NAND_FCR1_CMD4 0xFF000000 -#define IFC_NAND_FCR1_CMD4_SHIFT 24 -#define IFC_NAND_FCR1_CMD5 0x00FF0000 -#define IFC_NAND_FCR1_CMD5_SHIFT 16 -#define IFC_NAND_FCR1_CMD6 0x0000FF00 -#define IFC_NAND_FCR1_CMD6_SHIFT 8 -#define IFC_NAND_FCR1_CMD7 0x000000FF -#define IFC_NAND_FCR1_CMD7_SHIFT 0 - -/* - * Flash ROW and COL Address Register (ROWn, COLn) - */ -/* Main/spare region locator */ -#define IFC_NAND_COL_MS 0x80000000 -/* Column Address */ -#define IFC_NAND_COL_CA_MASK 0x00000FFF - -/* - * NAND Flash Byte Count Register (NAND_BC) - */ -/* Byte Count for read/Write */ -#define IFC_NAND_BC 0x000001FF - -/* - * NAND Flash Instruction Registers (NAND_FIR0/NAND_FIR1/NAND_FIR2) - */ -/* NAND Machine specific opcodes OP0-OP14*/ -#define IFC_NAND_FIR0_OP0 0xFC000000 -#define IFC_NAND_FIR0_OP0_SHIFT 26 -#define IFC_NAND_FIR0_OP1 0x03F00000 -#define IFC_NAND_FIR0_OP1_SHIFT 20 -#define IFC_NAND_FIR0_OP2 0x000FC000 -#define IFC_NAND_FIR0_OP2_SHIFT 14 -#define IFC_NAND_FIR0_OP3 0x00003F00 -#define IFC_NAND_FIR0_OP3_SHIFT 8 -#define IFC_NAND_FIR0_OP4 0x000000FC -#define IFC_NAND_FIR0_OP4_SHIFT 2 -#define IFC_NAND_FIR1_OP5 0xFC000000 -#define IFC_NAND_FIR1_OP5_SHIFT 26 -#define IFC_NAND_FIR1_OP6 0x03F00000 -#define IFC_NAND_FIR1_OP6_SHIFT 20 -#define IFC_NAND_FIR1_OP7 0x000FC000 -#define IFC_NAND_FIR1_OP7_SHIFT 14 -#define IFC_NAND_FIR1_OP8 0x00003F00 -#define IFC_NAND_FIR1_OP8_SHIFT 8 -#define IFC_NAND_FIR1_OP9 0x000000FC -#define IFC_NAND_FIR1_OP9_SHIFT 2 -#define IFC_NAND_FIR2_OP10 0xFC000000 -#define IFC_NAND_FIR2_OP10_SHIFT 26 -#define IFC_NAND_FIR2_OP11 0x03F00000 -#define IFC_NAND_FIR2_OP11_SHIFT 20 -#define IFC_NAND_FIR2_OP12 0x000FC000 -#define IFC_NAND_FIR2_OP12_SHIFT 14 -#define IFC_NAND_FIR2_OP13 0x00003F00 -#define IFC_NAND_FIR2_OP13_SHIFT 8 -#define IFC_NAND_FIR2_OP14 0x000000FC -#define IFC_NAND_FIR2_OP14_SHIFT 2 - -/* - * Instruction opcodes to be programmed - * in FIR registers- 6bits - */ -enum ifc_nand_fir_opcodes { - IFC_FIR_OP_NOP, - IFC_FIR_OP_CA0, - IFC_FIR_OP_CA1, - IFC_FIR_OP_CA2, - IFC_FIR_OP_CA3, - IFC_FIR_OP_RA0, - IFC_FIR_OP_RA1, - IFC_FIR_OP_RA2, - IFC_FIR_OP_RA3, - IFC_FIR_OP_CMD0, - IFC_FIR_OP_CMD1, - IFC_FIR_OP_CMD2, - IFC_FIR_OP_CMD3, - IFC_FIR_OP_CMD4, - IFC_FIR_OP_CMD5, - IFC_FIR_OP_CMD6, - IFC_FIR_OP_CMD7, - IFC_FIR_OP_CW0, - IFC_FIR_OP_CW1, - IFC_FIR_OP_CW2, - IFC_FIR_OP_CW3, - IFC_FIR_OP_CW4, - IFC_FIR_OP_CW5, - IFC_FIR_OP_CW6, - IFC_FIR_OP_CW7, - IFC_FIR_OP_WBCD, - IFC_FIR_OP_RBCD, - IFC_FIR_OP_BTRD, - IFC_FIR_OP_RDSTAT, - IFC_FIR_OP_NWAIT, - IFC_FIR_OP_WFR, - IFC_FIR_OP_SBRD, - IFC_FIR_OP_UA, - IFC_FIR_OP_RB, -}; - -/* - * NAND Chip Select Register (NAND_CSEL) - */ -#define IFC_NAND_CSEL 0x0C000000 -#define IFC_NAND_CSEL_SHIFT 26 -#define IFC_NAND_CSEL_CS0 0x00000000 -#define IFC_NAND_CSEL_CS1 0x04000000 -#define IFC_NAND_CSEL_CS2 0x08000000 -#define IFC_NAND_CSEL_CS3 0x0C000000 - -/* - * NAND Operation Sequence Start (NANDSEQ_STRT) - */ -/* NAND Flash Operation Start */ -#define IFC_NAND_SEQ_STRT_FIR_STRT 0x80000000 -/* Automatic Erase */ -#define IFC_NAND_SEQ_STRT_AUTO_ERS 0x00800000 -/* Automatic Program */ -#define IFC_NAND_SEQ_STRT_AUTO_PGM 0x00100000 -/* Automatic Copyback */ -#define IFC_NAND_SEQ_STRT_AUTO_CPB 0x00020000 -/* Automatic Read Operation */ -#define IFC_NAND_SEQ_STRT_AUTO_RD 0x00004000 -/* Automatic Status Read */ -#define IFC_NAND_SEQ_STRT_AUTO_STAT_RD 0x00000800 - -/* - * NAND Event and Error Status Register (NAND_EVTER_STAT) - */ -/* Operation Complete */ -#define IFC_NAND_EVTER_STAT_OPC 0x80000000 -/* Flash Timeout Error */ -#define IFC_NAND_EVTER_STAT_FTOER 0x08000000 -/* Write Protect Error */ -#define IFC_NAND_EVTER_STAT_WPER 0x04000000 -/* ECC Error */ -#define IFC_NAND_EVTER_STAT_ECCER 0x02000000 -/* RCW Load Done */ -#define IFC_NAND_EVTER_STAT_RCW_DN 0x00008000 -/* Boot Loadr Done */ -#define IFC_NAND_EVTER_STAT_BOOT_DN 0x00004000 -/* Bad Block Indicator search select */ -#define IFC_NAND_EVTER_STAT_BBI_SRCH_SE 0x00000800 - -/* - * NAND Flash Page Read Completion Event Status Register - * (PGRDCMPL_EVT_STAT) - */ -#define PGRDCMPL_EVT_STAT_MASK 0xFFFF0000 -/* Small Page 0-15 Done */ -#define PGRDCMPL_EVT_STAT_SECTION_SP(n) (1 << (31 - (n))) -/* Large Page(2K) 0-3 Done */ -#define PGRDCMPL_EVT_STAT_LP_2K(n) (0xF << (28 - (n)*4)) -/* Large Page(4K) 0-1 Done */ -#define PGRDCMPL_EVT_STAT_LP_4K(n) (0xFF << (24 - (n)*8)) - -/* - * NAND Event and Error Enable Register (NAND_EVTER_EN) - */ -/* Operation complete event enable */ -#define IFC_NAND_EVTER_EN_OPC_EN 0x80000000 -/* Page read complete event enable */ -#define IFC_NAND_EVTER_EN_PGRDCMPL_EN 0x20000000 -/* Flash Timeout error enable */ -#define IFC_NAND_EVTER_EN_FTOER_EN 0x08000000 -/* Write Protect error enable */ -#define IFC_NAND_EVTER_EN_WPER_EN 0x04000000 -/* ECC error logging enable */ -#define IFC_NAND_EVTER_EN_ECCER_EN 0x02000000 - -/* - * NAND Event and Error Interrupt Enable Register (NAND_EVTER_INTR_EN) - */ -/* Enable interrupt for operation complete */ -#define IFC_NAND_EVTER_INTR_OPCIR_EN 0x80000000 -/* Enable interrupt for Page read complete */ -#define IFC_NAND_EVTER_INTR_PGRDCMPLIR_EN 0x20000000 -/* Enable interrupt for Flash timeout error */ -#define IFC_NAND_EVTER_INTR_FTOERIR_EN 0x08000000 -/* Enable interrupt for Write protect error */ -#define IFC_NAND_EVTER_INTR_WPERIR_EN 0x04000000 -/* Enable interrupt for ECC error*/ -#define IFC_NAND_EVTER_INTR_ECCERIR_EN 0x02000000 - -/* - * NAND Transfer Error Attribute Register-0 (NAND_ERATTR0) - */ -#define IFC_NAND_ERATTR0_MASK 0x0C080000 -/* Error on CS0-3 for NAND */ -#define IFC_NAND_ERATTR0_ERCS_CS0 0x00000000 -#define IFC_NAND_ERATTR0_ERCS_CS1 0x04000000 -#define IFC_NAND_ERATTR0_ERCS_CS2 0x08000000 -#define IFC_NAND_ERATTR0_ERCS_CS3 0x0C000000 -/* Transaction type of error Read/Write */ -#define IFC_NAND_ERATTR0_ERTTYPE_READ 0x00080000 - -/* - * NAND Flash Status Register (NAND_FSR) - */ -/* First byte of data read from read status op */ -#define IFC_NAND_NFSR_RS0 0xFF000000 -/* Second byte of data read from read status op */ -#define IFC_NAND_NFSR_RS1 0x00FF0000 - -/* - * ECC Error Status Registers (ECCSTAT0-ECCSTAT3) - */ -/* Number of ECC errors on sector n (n = 0-15) */ -#define IFC_NAND_ECCSTAT0_ERRCNT_SECTOR0_MASK 0x0F000000 -#define IFC_NAND_ECCSTAT0_ERRCNT_SECTOR0_SHIFT 24 -#define IFC_NAND_ECCSTAT0_ERRCNT_SECTOR1_MASK 0x000F0000 -#define IFC_NAND_ECCSTAT0_ERRCNT_SECTOR1_SHIFT 16 -#define IFC_NAND_ECCSTAT0_ERRCNT_SECTOR2_MASK 0x00000F00 -#define IFC_NAND_ECCSTAT0_ERRCNT_SECTOR2_SHIFT 8 -#define IFC_NAND_ECCSTAT0_ERRCNT_SECTOR3_MASK 0x0000000F -#define IFC_NAND_ECCSTAT0_ERRCNT_SECTOR3_SHIFT 0 -#define IFC_NAND_ECCSTAT1_ERRCNT_SECTOR4_MASK 0x0F000000 -#define IFC_NAND_ECCSTAT1_ERRCNT_SECTOR4_SHIFT 24 -#define IFC_NAND_ECCSTAT1_ERRCNT_SECTOR5_MASK 0x000F0000 -#define IFC_NAND_ECCSTAT1_ERRCNT_SECTOR5_SHIFT 16 -#define IFC_NAND_ECCSTAT1_ERRCNT_SECTOR6_MASK 0x00000F00 -#define IFC_NAND_ECCSTAT1_ERRCNT_SECTOR6_SHIFT 8 -#define IFC_NAND_ECCSTAT1_ERRCNT_SECTOR7_MASK 0x0000000F -#define IFC_NAND_ECCSTAT1_ERRCNT_SECTOR7_SHIFT 0 -#define IFC_NAND_ECCSTAT2_ERRCNT_SECTOR8_MASK 0x0F000000 -#define IFC_NAND_ECCSTAT2_ERRCNT_SECTOR8_SHIFT 24 -#define IFC_NAND_ECCSTAT2_ERRCNT_SECTOR9_MASK 0x000F0000 -#define IFC_NAND_ECCSTAT2_ERRCNT_SECTOR9_SHIFT 16 -#define IFC_NAND_ECCSTAT2_ERRCNT_SECTOR10_MASK 0x00000F00 -#define IFC_NAND_ECCSTAT2_ERRCNT_SECTOR10_SHIFT 8 -#define IFC_NAND_ECCSTAT2_ERRCNT_SECTOR11_MASK 0x0000000F -#define IFC_NAND_ECCSTAT2_ERRCNT_SECTOR11_SHIFT 0 -#define IFC_NAND_ECCSTAT3_ERRCNT_SECTOR12_MASK 0x0F000000 -#define IFC_NAND_ECCSTAT3_ERRCNT_SECTOR12_SHIFT 24 -#define IFC_NAND_ECCSTAT3_ERRCNT_SECTOR13_MASK 0x000F0000 -#define IFC_NAND_ECCSTAT3_ERRCNT_SECTOR13_SHIFT 16 -#define IFC_NAND_ECCSTAT3_ERRCNT_SECTOR14_MASK 0x00000F00 -#define IFC_NAND_ECCSTAT3_ERRCNT_SECTOR14_SHIFT 8 -#define IFC_NAND_ECCSTAT3_ERRCNT_SECTOR15_MASK 0x0000000F -#define IFC_NAND_ECCSTAT3_ERRCNT_SECTOR15_SHIFT 0 - -/* - * NAND Control Register (NANDCR) - */ -#define IFC_NAND_NCR_FTOCNT_MASK 0x1E000000 -#define IFC_NAND_NCR_FTOCNT_SHIFT 25 -#define IFC_NAND_NCR_FTOCNT(n) ((_ilog2(n) - 8) << IFC_NAND_NCR_FTOCNT_SHIFT) - -/* - * NAND_AUTOBOOT_TRGR - */ -/* Trigger RCW load */ -#define IFC_NAND_AUTOBOOT_TRGR_RCW_LD 0x80000000 -/* Trigget Auto Boot */ -#define IFC_NAND_AUTOBOOT_TRGR_BOOT_LD 0x20000000 - -/* - * NAND_MDR - */ -/* 1st read data byte when opcode SBRD */ -#define IFC_NAND_MDR_RDATA0 0xFF000000 -/* 2nd read data byte when opcode SBRD */ -#define IFC_NAND_MDR_RDATA1 0x00FF0000 - -/* - * NOR Machine Specific Registers - */ -/* - * NOR Event and Error Status Register (NOR_EVTER_STAT) - */ -/* NOR Command Sequence Operation Complete */ -#define IFC_NOR_EVTER_STAT_OPC_NOR 0x80000000 -/* Write Protect Error */ -#define IFC_NOR_EVTER_STAT_WPER 0x04000000 -/* Command Sequence Timeout Error */ -#define IFC_NOR_EVTER_STAT_STOER 0x01000000 - -/* - * NOR Event and Error Enable Register (NOR_EVTER_EN) - */ -/* NOR Command Seq complete event enable */ -#define IFC_NOR_EVTER_EN_OPCEN_NOR 0x80000000 -/* Write Protect Error Checking Enable */ -#define IFC_NOR_EVTER_EN_WPEREN 0x04000000 -/* Timeout Error Enable */ -#define IFC_NOR_EVTER_EN_STOEREN 0x01000000 - -/* - * NOR Event and Error Interrupt Enable Register (NOR_EVTER_INTR_EN) - */ -/* Enable interrupt for OPC complete */ -#define IFC_NOR_EVTER_INTR_OPCEN_NOR 0x80000000 -/* Enable interrupt for write protect error */ -#define IFC_NOR_EVTER_INTR_WPEREN 0x04000000 -/* Enable interrupt for timeout error */ -#define IFC_NOR_EVTER_INTR_STOEREN 0x01000000 - -/* - * NOR Transfer Error Attribute Register-0 (NOR_ERATTR0) - */ -/* Source ID for error transaction */ -#define IFC_NOR_ERATTR0_ERSRCID 0xFF000000 -/* AXI ID for error transation */ -#define IFC_NOR_ERATTR0_ERAID 0x000FF000 -/* Chip select corresponds to NOR error */ -#define IFC_NOR_ERATTR0_ERCS_CS0 0x00000000 -#define IFC_NOR_ERATTR0_ERCS_CS1 0x00000010 -#define IFC_NOR_ERATTR0_ERCS_CS2 0x00000020 -#define IFC_NOR_ERATTR0_ERCS_CS3 0x00000030 -/* Type of transaction read/write */ -#define IFC_NOR_ERATTR0_ERTYPE_READ 0x00000001 - -/* - * NOR Transfer Error Attribute Register-2 (NOR_ERATTR2) - */ -#define IFC_NOR_ERATTR2_ER_NUM_PHASE_EXP 0x000F0000 -#define IFC_NOR_ERATTR2_ER_NUM_PHASE_PER 0x00000F00 - -/* - * NOR Control Register (NORCR) - */ -#define IFC_NORCR_MASK 0x0F0F0000 -/* No. of Address/Data Phase */ -#define IFC_NORCR_NUM_PHASE_MASK 0x0F000000 -#define IFC_NORCR_NUM_PHASE_SHIFT 24 -#define IFC_NORCR_NUM_PHASE(n) ((n-1) << IFC_NORCR_NUM_PHASE_SHIFT) -/* Sequence Timeout Count */ -#define IFC_NORCR_STOCNT_MASK 0x000F0000 -#define IFC_NORCR_STOCNT_SHIFT 16 -#define IFC_NORCR_STOCNT(n) ((__ilog2(n) - 8) << IFC_NORCR_STOCNT_SHIFT) - -/* - * GPCM Machine specific registers - */ -/* - * GPCM Event and Error Status Register (GPCM_EVTER_STAT) - */ -/* Timeout error */ -#define IFC_GPCM_EVTER_STAT_TOER 0x04000000 -/* Parity error */ -#define IFC_GPCM_EVTER_STAT_PER 0x01000000 - -/* - * GPCM Event and Error Enable Register (GPCM_EVTER_EN) - */ -/* Timeout error enable */ -#define IFC_GPCM_EVTER_EN_TOER_EN 0x04000000 -/* Parity error enable */ -#define IFC_GPCM_EVTER_EN_PER_EN 0x01000000 - -/* - * GPCM Event and Error Interrupt Enable Register (GPCM_EVTER_INTR_EN) - */ -/* Enable Interrupt for timeout error */ -#define IFC_GPCM_EEIER_TOERIR_EN 0x04000000 -/* Enable Interrupt for Parity error */ -#define IFC_GPCM_EEIER_PERIR_EN 0x01000000 - -/* - * GPCM Transfer Error Attribute Register-0 (GPCM_ERATTR0) - */ -/* Source ID for error transaction */ -#define IFC_GPCM_ERATTR0_ERSRCID 0xFF000000 -/* AXI ID for error transaction */ -#define IFC_GPCM_ERATTR0_ERAID 0x000FF000 -/* Chip select corresponds to GPCM error */ -#define IFC_GPCM_ERATTR0_ERCS_CS0 0x00000000 -#define IFC_GPCM_ERATTR0_ERCS_CS1 0x00000040 -#define IFC_GPCM_ERATTR0_ERCS_CS2 0x00000080 -#define IFC_GPCM_ERATTR0_ERCS_CS3 0x000000C0 -/* Type of transaction read/Write */ -#define IFC_GPCM_ERATTR0_ERTYPE_READ 0x00000001 - -/* - * GPCM Transfer Error Attribute Register-2 (GPCM_ERATTR2) - */ -/* On which beat of address/data parity error is observed */ -#define IFC_GPCM_ERATTR2_PERR_BEAT 0x00000C00 -/* Parity Error on byte */ -#define IFC_GPCM_ERATTR2_PERR_BYTE 0x000000F0 -/* Parity Error reported in addr or data phase */ -#define IFC_GPCM_ERATTR2_PERR_DATA_PHASE 0x00000001 - -/* - * GPCM Status Register (GPCM_STAT) - */ -#define IFC_GPCM_STAT_BSY 0x80000000 /* GPCM is busy */ - -/* - * IFC Controller NAND Machine registers - */ -struct fsl_ifc_nand { - __be32 ncfgr; - u32 res1[0x4]; - __be32 nand_fcr0; - __be32 nand_fcr1; - u32 res2[0x8]; - __be32 row0; - u32 res3; - __be32 col0; - u32 res4; - __be32 row1; - u32 res5; - __be32 col1; - u32 res6; - __be32 row2; - u32 res7; - __be32 col2; - u32 res8; - __be32 row3; - u32 res9; - __be32 col3; - u32 res10[0x24]; - __be32 nand_fbcr; - u32 res11; - __be32 nand_fir0; - __be32 nand_fir1; - __be32 nand_fir2; - u32 res12[0x10]; - __be32 nand_csel; - u32 res13; - __be32 nandseq_strt; - u32 res14; - __be32 nand_evter_stat; - u32 res15; - __be32 pgrdcmpl_evt_stat; - u32 res16[0x2]; - __be32 nand_evter_en; - u32 res17[0x2]; - __be32 nand_evter_intr_en; - u32 res18[0x2]; - __be32 nand_erattr0; - __be32 nand_erattr1; - u32 res19[0x10]; - __be32 nand_fsr; - u32 res20; - __be32 nand_eccstat[4]; - u32 res21[0x20]; - __be32 nanndcr; - u32 res22[0x2]; - __be32 nand_autoboot_trgr; - u32 res23; - __be32 nand_mdr; - u32 res24[0x5C]; -}; - -/* - * IFC controller NOR Machine registers - */ -struct fsl_ifc_nor { - __be32 nor_evter_stat; - u32 res1[0x2]; - __be32 nor_evter_en; - u32 res2[0x2]; - __be32 nor_evter_intr_en; - u32 res3[0x2]; - __be32 nor_erattr0; - __be32 nor_erattr1; - __be32 nor_erattr2; - u32 res4[0x4]; - __be32 norcr; - u32 res5[0xEF]; -}; - -/* - * IFC controller GPCM Machine registers - */ -struct fsl_ifc_gpcm { - __be32 gpcm_evter_stat; - u32 res1[0x2]; - __be32 gpcm_evter_en; - u32 res2[0x2]; - __be32 gpcm_evter_intr_en; - u32 res3[0x2]; - __be32 gpcm_erattr0; - __be32 gpcm_erattr1; - __be32 gpcm_erattr2; - __be32 gpcm_stat; - u32 res4[0x1F3]; -}; - -/* - * IFC Controller Registers - */ -struct fsl_ifc_regs { - __be32 ifc_rev; - u32 res1[0x2]; - struct { - __be32 cspr_ext; - __be32 cspr; - u32 res2; - } cspr_cs[FSL_IFC_BANK_COUNT]; - u32 res3[0x19]; - struct { - __be32 amask; - u32 res4[0x2]; - } amask_cs[FSL_IFC_BANK_COUNT]; - u32 res5[0x17]; - struct { - __be32 csor_ext; - __be32 csor; - u32 res6; - } csor_cs[FSL_IFC_BANK_COUNT]; - u32 res7[0x19]; - struct { - __be32 ftim[4]; - u32 res8[0x8]; - } ftim_cs[FSL_IFC_BANK_COUNT]; - u32 res9[0x60]; - __be32 rb_stat; - u32 res10[0x2]; - __be32 ifc_gcr; - u32 res11[0x2]; - __be32 cm_evter_stat; - u32 res12[0x2]; - __be32 cm_evter_en; - u32 res13[0x2]; - __be32 cm_evter_intr_en; - u32 res14[0x2]; - __be32 cm_erattr0; - __be32 cm_erattr1; - u32 res15[0x2]; - __be32 ifc_ccr; - __be32 ifc_csr; - u32 res16[0x2EB]; - struct fsl_ifc_nand ifc_nand; - struct fsl_ifc_nor ifc_nor; - struct fsl_ifc_gpcm ifc_gpcm; -}; - -extern unsigned int convert_ifc_address(phys_addr_t addr_base); -extern int fsl_ifc_find(phys_addr_t addr_base); - -/* overview of the fsl ifc controller */ - -struct fsl_ifc_ctrl { - /* device info */ - struct device *dev; - struct fsl_ifc_regs __iomem *regs; - int irq; - int nand_irq; - spinlock_t lock; - void *nand; - - u32 nand_stat; - wait_queue_head_t nand_wait; -}; - -extern struct fsl_ifc_ctrl *fsl_ifc_ctrl_dev; - - -#endif /* __ASM_FSL_IFC_H */ diff --git a/arch/powerpc/include/asm/fsl_lbc.h b/arch/powerpc/include/asm/fsl_lbc.h index 420b45368fcf..c4af5ee716aa 100644 --- a/arch/powerpc/include/asm/fsl_lbc.h +++ b/arch/powerpc/include/asm/fsl_lbc.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* Freescale Local Bus Controller * * Copyright © 2006-2007, 2010 Freescale Semiconductor @@ -5,20 +6,6 @@ * Authors: Nick Spence <nick.spence@freescale.com>, * Scott Wood <scottwood@freescale.com> * Jack Lan <jack.lan@freescale.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef __ASM_FSL_LBC_H @@ -95,6 +82,9 @@ struct fsl_lbc_bank { #define OR_FCM_TRLX_SHIFT 2 #define OR_FCM_EHTR 0x00000002 #define OR_FCM_EHTR_SHIFT 1 + +#define OR_GPCM_AM 0xFFFF8000 +#define OR_GPCM_AM_SHIFT 15 }; struct fsl_lbc_regs { @@ -285,7 +275,7 @@ struct fsl_lbc_ctrl { /* device info */ struct device *dev; struct fsl_lbc_regs __iomem *regs; - int irq; + int irq[2]; wait_queue_head_t irq_wait; spinlock_t lock; void *nand; diff --git a/arch/powerpc/include/asm/fsl_pamu_stash.h b/arch/powerpc/include/asm/fsl_pamu_stash.h new file mode 100644 index 000000000000..c0fbadb70b5d --- /dev/null +++ b/arch/powerpc/include/asm/fsl_pamu_stash.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * + * Copyright (C) 2013 Freescale Semiconductor, Inc. + */ + +#ifndef __FSL_PAMU_STASH_H +#define __FSL_PAMU_STASH_H + +struct iommu_domain; + +/* cache stash targets */ +enum pamu_stash_target { + PAMU_ATTR_CACHE_L1 = 1, + PAMU_ATTR_CACHE_L2, + PAMU_ATTR_CACHE_L3, +}; + +int fsl_pamu_configure_l1_stash(struct iommu_domain *domain, u32 cpu); + +#endif /* __FSL_PAMU_STASH_H */ diff --git a/arch/powerpc/include/asm/fsl_pm.h b/arch/powerpc/include/asm/fsl_pm.h new file mode 100644 index 000000000000..61a4c977320f --- /dev/null +++ b/arch/powerpc/include/asm/fsl_pm.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Support Power Management + * + * Copyright 2014-2015 Freescale Semiconductor Inc. + */ +#ifndef __PPC_FSL_PM_H +#define __PPC_FSL_PM_H + +#define E500_PM_PH10 1 +#define E500_PM_PH15 2 +#define E500_PM_PH20 3 +#define E500_PM_PH30 4 +#define E500_PM_DOZE E500_PM_PH10 +#define E500_PM_NAP E500_PM_PH15 + +#define PLAT_PM_SLEEP 20 +#define PLAT_PM_LPM20 30 + +#define FSL_PM_SLEEP (1 << 0) +#define FSL_PM_DEEP_SLEEP (1 << 1) + +struct fsl_pm_ops { + /* mask pending interrupts to the RCPM from MPIC */ + void (*irq_mask)(int cpu); + + /* unmask pending interrupts to the RCPM from MPIC */ + void (*irq_unmask)(int cpu); + void (*cpu_enter_state)(int cpu, int state); + void (*cpu_exit_state)(int cpu, int state); + void (*cpu_up_prepare)(int cpu); + void (*cpu_die)(int cpu); + int (*plat_enter_sleep)(void); + void (*freeze_time_base)(bool freeze); + + /* keep the power of IP blocks during sleep/deep sleep */ + void (*set_ip_power)(bool enable, u32 mask); + + /* get platform supported power management modes */ + unsigned int (*get_pm_modes)(void); +}; + +extern const struct fsl_pm_ops *qoriq_pm_ops; + +int __init fsl_rcpm_init(void); + +#endif /* __PPC_FSL_PM_H */ diff --git a/arch/powerpc/include/asm/ftrace.h b/arch/powerpc/include/asm/ftrace.h index 169d039ed402..5984eaa75ce8 100644 --- a/arch/powerpc/include/asm/ftrace.h +++ b/arch/powerpc/include/asm/ftrace.h @@ -1,77 +1,180 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_FTRACE #define _ASM_POWERPC_FTRACE +#include <asm/types.h> + #ifdef CONFIG_FUNCTION_TRACER -#define MCOUNT_ADDR ((long)(_mcount)) +#define MCOUNT_ADDR ((unsigned long)(_mcount)) #define MCOUNT_INSN_SIZE 4 /* sizeof mcount call */ -#ifdef __ASSEMBLY__ - -/* Based off of objdump optput from glibc */ - -#define MCOUNT_SAVE_FRAME \ - stwu r1,-48(r1); \ - stw r3, 12(r1); \ - stw r4, 16(r1); \ - stw r5, 20(r1); \ - stw r6, 24(r1); \ - mflr r3; \ - lwz r4, 52(r1); \ - mfcr r5; \ - stw r7, 28(r1); \ - stw r8, 32(r1); \ - stw r9, 36(r1); \ - stw r10,40(r1); \ - stw r3, 44(r1); \ - stw r5, 8(r1) - -#define MCOUNT_RESTORE_FRAME \ - lwz r6, 8(r1); \ - lwz r0, 44(r1); \ - lwz r3, 12(r1); \ - mtctr r0; \ - lwz r4, 16(r1); \ - mtcr r6; \ - lwz r5, 20(r1); \ - lwz r6, 24(r1); \ - lwz r0, 52(r1); \ - lwz r7, 28(r1); \ - lwz r8, 32(r1); \ - mtlr r0; \ - lwz r9, 36(r1); \ - lwz r10,40(r1); \ - addi r1, r1, 48 - -#else /* !__ASSEMBLY__ */ +/* Ignore unused weak functions which will have larger offsets */ +#if defined(CONFIG_MPROFILE_KERNEL) || defined(CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY) +#define FTRACE_MCOUNT_MAX_OFFSET 16 +#elif defined(CONFIG_PPC32) +#define FTRACE_MCOUNT_MAX_OFFSET 8 +#endif + +#ifndef __ASSEMBLER__ extern void _mcount(void); -#ifdef CONFIG_DYNAMIC_FTRACE -static inline unsigned long ftrace_call_adjust(unsigned long addr) -{ - /* reloction of mcount call site is the same as the address */ - return addr; -} +unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip, + unsigned long sp); +struct module; +struct dyn_ftrace; struct dyn_arch_ftrace { - struct module *mod; +#ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE + /* pointer to the associated out-of-line stub */ + unsigned long ool_stub; +#endif }; -#endif /* CONFIG_DYNAMIC_FTRACE */ -#endif /* __ASSEMBLY__ */ +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS +#define ftrace_need_init_nop() (true) +int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec); +#define ftrace_init_nop ftrace_init_nop + +#include <linux/ftrace_regs.h> + +static __always_inline struct pt_regs *arch_ftrace_get_regs(struct ftrace_regs *fregs) +{ + /* We clear regs.msr in ftrace_call */ + return arch_ftrace_regs(fregs)->regs.msr ? &arch_ftrace_regs(fregs)->regs : NULL; +} + +#define arch_ftrace_fill_perf_regs(fregs, _regs) do { \ + (_regs)->result = 0; \ + (_regs)->nip = arch_ftrace_regs(fregs)->regs.nip; \ + (_regs)->gpr[1] = arch_ftrace_regs(fregs)->regs.gpr[1]; \ + asm volatile("mfmsr %0" : "=r" ((_regs)->msr)); \ + } while (0) + +#undef ftrace_regs_get_return_value +static __always_inline unsigned long +ftrace_regs_get_return_value(const struct ftrace_regs *fregs) +{ + return arch_ftrace_regs(fregs)->regs.gpr[3]; +} +#define ftrace_regs_get_return_value ftrace_regs_get_return_value + +#undef ftrace_regs_get_frame_pointer +static __always_inline unsigned long +ftrace_regs_get_frame_pointer(const struct ftrace_regs *fregs) +{ + return arch_ftrace_regs(fregs)->regs.gpr[1]; +} + +static __always_inline void +ftrace_regs_set_instruction_pointer(struct ftrace_regs *fregs, + unsigned long ip) +{ + regs_set_return_ip(&arch_ftrace_regs(fregs)->regs, ip); +} + +static __always_inline unsigned long +ftrace_regs_get_return_address(struct ftrace_regs *fregs) +{ + return arch_ftrace_regs(fregs)->regs.link; +} + +struct ftrace_ops; + +#define ftrace_graph_func ftrace_graph_func +void ftrace_graph_func(unsigned long ip, unsigned long parent_ip, + struct ftrace_ops *op, struct ftrace_regs *fregs); #endif +#endif /* __ASSEMBLER__ */ -#if defined(CONFIG_FTRACE_SYSCALLS) && defined(CONFIG_PPC64) && !defined(__ASSEMBLY__) +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS +#define ARCH_SUPPORTS_FTRACE_OPS 1 +#endif +#endif /* CONFIG_FUNCTION_TRACER */ + +#ifndef __ASSEMBLER__ +#ifdef CONFIG_FTRACE_SYSCALLS +/* + * Some syscall entry functions on powerpc start with "ppc_" (fork and clone, + * for instance) or ppc32_/ppc64_. We should also match the sys_ variant with + * those. + */ #define ARCH_HAS_SYSCALL_MATCH_SYM_NAME static inline bool arch_syscall_match_sym_name(const char *sym, const char *name) { - /* - * Compare the symbol name with the system call name. Skip the .sys or .SyS - * prefix from the symbol name and the sys prefix from the system call name and - * just match the rest. This is only needed on ppc64 since symbol names on - * 32bit do not start with a period so the generic function will work. - */ - return !strcmp(sym + 4, name + 3); + return !strcmp(sym, name) || + (!strncmp(sym, "__se_sys", 8) && !strcmp(sym + 5, name)) || + (!strncmp(sym, "ppc_", 4) && !strcmp(sym + 4, name + 4)) || + (!strncmp(sym, "ppc32_", 6) && !strcmp(sym + 6, name + 4)) || + (!strncmp(sym, "ppc64_", 6) && !strcmp(sym + 6, name + 4)); +} +#endif /* CONFIG_FTRACE_SYSCALLS */ + +#if defined(CONFIG_PPC64) && defined(CONFIG_FUNCTION_TRACER) +#include <asm/paca.h> + +static inline void this_cpu_disable_ftrace(void) +{ + get_paca()->ftrace_enabled = 0; +} + +static inline void this_cpu_enable_ftrace(void) +{ + get_paca()->ftrace_enabled = 1; +} + +/* Disable ftrace on this CPU if possible (may not be implemented) */ +static inline void this_cpu_set_ftrace_enabled(u8 ftrace_enabled) +{ + get_paca()->ftrace_enabled = ftrace_enabled; } -#endif /* CONFIG_FTRACE_SYSCALLS && CONFIG_PPC64 && !__ASSEMBLY__ */ + +static inline u8 this_cpu_get_ftrace_enabled(void) +{ + return get_paca()->ftrace_enabled; +} +#else /* CONFIG_PPC64 */ +static inline void this_cpu_disable_ftrace(void) { } +static inline void this_cpu_enable_ftrace(void) { } +static inline void this_cpu_set_ftrace_enabled(u8 ftrace_enabled) { } +static inline u8 this_cpu_get_ftrace_enabled(void) { return 1; } +#endif /* CONFIG_PPC64 */ + +#ifdef CONFIG_FUNCTION_TRACER +extern unsigned int ftrace_tramp_text[], ftrace_tramp_init[]; +#ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE +struct ftrace_ool_stub { +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS + struct ftrace_ops *ftrace_op; +#endif + u32 insn[4]; +} __aligned(sizeof(unsigned long)); +extern struct ftrace_ool_stub ftrace_ool_stub_text_end[], ftrace_ool_stub_text[], + ftrace_ool_stub_inittext[]; +extern unsigned int ftrace_ool_stub_text_end_count, ftrace_ool_stub_text_count, + ftrace_ool_stub_inittext_count; +#endif +void ftrace_free_init_tramp(void); +unsigned long ftrace_call_adjust(unsigned long addr); + +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS +/* + * When an ftrace registered caller is tracing a function that is also set by a + * register_ftrace_direct() call, it needs to be differentiated in the + * ftrace_caller trampoline so that the direct call can be invoked after the + * other ftrace ops. To do this, place the direct caller in the orig_gpr3 field + * of pt_regs. This tells ftrace_caller that there's a direct caller. + */ +static inline void arch_ftrace_set_direct_caller(struct ftrace_regs *fregs, unsigned long addr) +{ + struct pt_regs *regs = &arch_ftrace_regs(fregs)->regs; + + regs->orig_gpr3 = addr; +} +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ +#else +static inline void ftrace_free_init_tramp(void) { } +static inline unsigned long ftrace_call_adjust(unsigned long addr) { return addr; } +#endif +#endif /* !__ASSEMBLER__ */ #endif /* _ASM_POWERPC_FTRACE */ diff --git a/arch/powerpc/include/asm/futex.h b/arch/powerpc/include/asm/futex.h index 2a9cf845473b..b3001f8b2c1e 100644 --- a/arch/powerpc/include/asm/futex.h +++ b/arch/powerpc/include/asm/futex.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_FUTEX_H #define _ASM_POWERPC_FUTEX_H @@ -7,14 +8,12 @@ #include <linux/uaccess.h> #include <asm/errno.h> #include <asm/synch.h> -#include <asm/asm-compat.h> #define __futex_atomic_op(insn, ret, oldval, uaddr, oparg) \ __asm__ __volatile ( \ PPC_ATOMIC_ENTRY_BARRIER \ "1: lwarx %0,0,%2\n" \ insn \ - PPC405_ERR77(0, %2) \ "2: stwcx. %1,0,%2\n" \ "bne- 1b\n" \ PPC_ATOMIC_EXIT_BARRIER \ @@ -23,29 +22,20 @@ "4: li %1,%3\n" \ "b 3b\n" \ ".previous\n" \ - ".section __ex_table,\"a\"\n" \ - ".align 3\n" \ - PPC_LONG "1b,4b,2b,4b\n" \ - ".previous" \ + EX_TABLE(1b, 4b) \ + EX_TABLE(2b, 4b) \ : "=&r" (oldval), "=&r" (ret) \ : "b" (uaddr), "i" (-EFAULT), "r" (oparg) \ : "cr0", "memory") -static inline int futex_atomic_op_inuser (int encoded_op, u32 __user *uaddr) +static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, + u32 __user *uaddr) { - int op = (encoded_op >> 28) & 7; - int cmp = (encoded_op >> 24) & 15; - int oparg = (encoded_op << 8) >> 20; - int cmparg = (encoded_op << 20) >> 20; int oldval = 0, ret; - if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28)) - oparg = 1 << oparg; - if (! access_ok (VERIFY_WRITE, uaddr, sizeof(u32))) + if (!user_access_begin(uaddr, sizeof(u32))) return -EFAULT; - pagefault_disable(); - switch (op) { case FUTEX_OP_SET: __futex_atomic_op("mr %1,%4\n", ret, oldval, uaddr, oparg); @@ -65,20 +55,10 @@ static inline int futex_atomic_op_inuser (int encoded_op, u32 __user *uaddr) default: ret = -ENOSYS; } + user_access_end(); - pagefault_enable(); + *oval = oldval; - if (!ret) { - switch (cmp) { - case FUTEX_OP_CMP_EQ: ret = (oldval == cmparg); break; - case FUTEX_OP_CMP_NE: ret = (oldval != cmparg); break; - case FUTEX_OP_CMP_LT: ret = (oldval < cmparg); break; - case FUTEX_OP_CMP_GE: ret = (oldval >= cmparg); break; - case FUTEX_OP_CMP_LE: ret = (oldval <= cmparg); break; - case FUTEX_OP_CMP_GT: ret = (oldval > cmparg); break; - default: ret = -ENOSYS; - } - } return ret; } @@ -89,7 +69,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, int ret = 0; u32 prev; - if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) + if (!user_access_begin(uaddr, sizeof(u32))) return -EFAULT; __asm__ __volatile__ ( @@ -97,23 +77,23 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, "1: lwarx %1,0,%3 # futex_atomic_cmpxchg_inatomic\n\ cmpw 0,%1,%4\n\ bne- 3f\n" - PPC405_ERR77(0,%3) "2: stwcx. %5,0,%3\n\ bne- 1b\n" PPC_ATOMIC_EXIT_BARRIER "3: .section .fixup,\"ax\"\n\ 4: li %0,%6\n\ b 3b\n\ - .previous\n\ - .section __ex_table,\"a\"\n\ - .align 3\n\ - " PPC_LONG "1b,4b,2b,4b\n\ - .previous" \ + .previous\n" + EX_TABLE(1b, 4b) + EX_TABLE(2b, 4b) : "+r" (ret), "=&r" (prev), "+m" (*uaddr) : "r" (uaddr), "r" (oldval), "r" (newval), "i" (-EFAULT) : "cc", "memory"); + user_access_end(); + *uval = prev; + return ret; } diff --git a/arch/powerpc/include/asm/gpio.h b/arch/powerpc/include/asm/gpio.h deleted file mode 100644 index b3799d88ffcf..000000000000 --- a/arch/powerpc/include/asm/gpio.h +++ /dev/null @@ -1,4 +0,0 @@ -#ifndef __LINUX_GPIO_H -#warning Include linux/gpio.h instead of asm/gpio.h -#include <linux/gpio.h> -#endif diff --git a/arch/powerpc/include/asm/grackle.h b/arch/powerpc/include/asm/grackle.h index bd7812a519d4..7376e3fa1570 100644 --- a/arch/powerpc/include/asm/grackle.h +++ b/arch/powerpc/include/asm/grackle.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_GRACKLE_H #define _ASM_POWERPC_GRACKLE_H #ifdef __KERNEL__ diff --git a/arch/powerpc/include/asm/guest-state-buffer.h b/arch/powerpc/include/asm/guest-state-buffer.h new file mode 100644 index 000000000000..acd61eb36d59 --- /dev/null +++ b/arch/powerpc/include/asm/guest-state-buffer.h @@ -0,0 +1,1019 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Interface based on include/net/netlink.h + */ +#ifndef _ASM_POWERPC_GUEST_STATE_BUFFER_H +#define _ASM_POWERPC_GUEST_STATE_BUFFER_H + +#include "asm/hvcall.h" +#include <linux/gfp.h> +#include <linux/bitmap.h> +#include <asm/plpar_wrappers.h> + +/************************************************************************** + * Guest State Buffer Constants + **************************************************************************/ +/* Element without a value and any length */ +#define KVMPPC_GSID_BLANK 0x0000 +/* Size required for the L0's internal VCPU representation */ +#define KVMPPC_GSID_HOST_STATE_SIZE 0x0001 + /* Minimum size for the H_GUEST_RUN_VCPU output buffer */ +#define KVMPPC_GSID_RUN_OUTPUT_MIN_SIZE 0x0002 + /* "Logical" PVR value as defined in the PAPR */ +#define KVMPPC_GSID_LOGICAL_PVR 0x0003 + /* L0 relative timebase offset */ +#define KVMPPC_GSID_TB_OFFSET 0x0004 + /* Partition Scoped Page Table Info */ +#define KVMPPC_GSID_PARTITION_TABLE 0x0005 + /* Process Table Info */ +#define KVMPPC_GSID_PROCESS_TABLE 0x0006 + +/* Guest Management Heap Size */ +#define KVMPPC_GSID_L0_GUEST_HEAP 0x0800 + +/* Guest Management Heap Max Size */ +#define KVMPPC_GSID_L0_GUEST_HEAP_MAX 0x0801 + +/* Guest Pagetable Size */ +#define KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE 0x0802 + +/* Guest Pagetable Max Size */ +#define KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE_MAX 0x0803 + +/* Guest Pagetable Reclaim in bytes */ +#define KVMPPC_GSID_L0_GUEST_PGTABLE_RECLAIM 0x0804 + +/* H_GUEST_RUN_VCPU input buffer Info */ +#define KVMPPC_GSID_RUN_INPUT 0x0C00 +/* H_GUEST_RUN_VCPU output buffer Info */ +#define KVMPPC_GSID_RUN_OUTPUT 0x0C01 +#define KVMPPC_GSID_VPA 0x0C02 + +#define KVMPPC_GSID_GPR(x) (0x1000 + (x)) +#define KVMPPC_GSID_HDEC_EXPIRY_TB 0x1020 +#define KVMPPC_GSID_NIA 0x1021 +#define KVMPPC_GSID_MSR 0x1022 +#define KVMPPC_GSID_LR 0x1023 +#define KVMPPC_GSID_XER 0x1024 +#define KVMPPC_GSID_CTR 0x1025 +#define KVMPPC_GSID_CFAR 0x1026 +#define KVMPPC_GSID_SRR0 0x1027 +#define KVMPPC_GSID_SRR1 0x1028 +#define KVMPPC_GSID_DAR 0x1029 +#define KVMPPC_GSID_DEC_EXPIRY_TB 0x102A +#define KVMPPC_GSID_VTB 0x102B +#define KVMPPC_GSID_LPCR 0x102C +#define KVMPPC_GSID_HFSCR 0x102D +#define KVMPPC_GSID_FSCR 0x102E +#define KVMPPC_GSID_FPSCR 0x102F +#define KVMPPC_GSID_DAWR0 0x1030 +#define KVMPPC_GSID_DAWR1 0x1031 +#define KVMPPC_GSID_CIABR 0x1032 +#define KVMPPC_GSID_PURR 0x1033 +#define KVMPPC_GSID_SPURR 0x1034 +#define KVMPPC_GSID_IC 0x1035 +#define KVMPPC_GSID_SPRG0 0x1036 +#define KVMPPC_GSID_SPRG1 0x1037 +#define KVMPPC_GSID_SPRG2 0x1038 +#define KVMPPC_GSID_SPRG3 0x1039 +#define KVMPPC_GSID_PPR 0x103A +#define KVMPPC_GSID_MMCR(x) (0x103B + (x)) +#define KVMPPC_GSID_MMCRA 0x103F +#define KVMPPC_GSID_SIER(x) (0x1040 + (x)) +#define KVMPPC_GSID_BESCR 0x1043 +#define KVMPPC_GSID_EBBHR 0x1044 +#define KVMPPC_GSID_EBBRR 0x1045 +#define KVMPPC_GSID_AMR 0x1046 +#define KVMPPC_GSID_IAMR 0x1047 +#define KVMPPC_GSID_AMOR 0x1048 +#define KVMPPC_GSID_UAMOR 0x1049 +#define KVMPPC_GSID_SDAR 0x104A +#define KVMPPC_GSID_SIAR 0x104B +#define KVMPPC_GSID_DSCR 0x104C +#define KVMPPC_GSID_TAR 0x104D +#define KVMPPC_GSID_DEXCR 0x104E +#define KVMPPC_GSID_HDEXCR 0x104F +#define KVMPPC_GSID_HASHKEYR 0x1050 +#define KVMPPC_GSID_HASHPKEYR 0x1051 +#define KVMPPC_GSID_CTRL 0x1052 +#define KVMPPC_GSID_DPDES 0x1053 + +#define KVMPPC_GSID_CR 0x2000 +#define KVMPPC_GSID_PIDR 0x2001 +#define KVMPPC_GSID_DSISR 0x2002 +#define KVMPPC_GSID_VSCR 0x2003 +#define KVMPPC_GSID_VRSAVE 0x2004 +#define KVMPPC_GSID_DAWRX0 0x2005 +#define KVMPPC_GSID_DAWRX1 0x2006 +#define KVMPPC_GSID_PMC(x) (0x2007 + (x)) +#define KVMPPC_GSID_WORT 0x200D +#define KVMPPC_GSID_PSPB 0x200E + +#define KVMPPC_GSID_VSRS(x) (0x3000 + (x)) + +#define KVMPPC_GSID_HDAR 0xF000 +#define KVMPPC_GSID_HDSISR 0xF001 +#define KVMPPC_GSID_HEIR 0xF002 +#define KVMPPC_GSID_ASDR 0xF003 + +#define KVMPPC_GSE_GUESTWIDE_START KVMPPC_GSID_BLANK +#define KVMPPC_GSE_GUESTWIDE_END KVMPPC_GSID_PROCESS_TABLE +#define KVMPPC_GSE_GUESTWIDE_COUNT \ + (KVMPPC_GSE_GUESTWIDE_END - KVMPPC_GSE_GUESTWIDE_START + 1) + +#define KVMPPC_GSE_HOSTWIDE_START KVMPPC_GSID_L0_GUEST_HEAP +#define KVMPPC_GSE_HOSTWIDE_END KVMPPC_GSID_L0_GUEST_PGTABLE_RECLAIM +#define KVMPPC_GSE_HOSTWIDE_COUNT \ + (KVMPPC_GSE_HOSTWIDE_END - KVMPPC_GSE_HOSTWIDE_START + 1) + +#define KVMPPC_GSE_META_START KVMPPC_GSID_RUN_INPUT +#define KVMPPC_GSE_META_END KVMPPC_GSID_VPA +#define KVMPPC_GSE_META_COUNT (KVMPPC_GSE_META_END - KVMPPC_GSE_META_START + 1) + +#define KVMPPC_GSE_DW_REGS_START KVMPPC_GSID_GPR(0) +#define KVMPPC_GSE_DW_REGS_END KVMPPC_GSID_DPDES +#define KVMPPC_GSE_DW_REGS_COUNT \ + (KVMPPC_GSE_DW_REGS_END - KVMPPC_GSE_DW_REGS_START + 1) + +#define KVMPPC_GSE_W_REGS_START KVMPPC_GSID_CR +#define KVMPPC_GSE_W_REGS_END KVMPPC_GSID_PSPB +#define KVMPPC_GSE_W_REGS_COUNT \ + (KVMPPC_GSE_W_REGS_END - KVMPPC_GSE_W_REGS_START + 1) + +#define KVMPPC_GSE_VSRS_START KVMPPC_GSID_VSRS(0) +#define KVMPPC_GSE_VSRS_END KVMPPC_GSID_VSRS(63) +#define KVMPPC_GSE_VSRS_COUNT (KVMPPC_GSE_VSRS_END - KVMPPC_GSE_VSRS_START + 1) + +#define KVMPPC_GSE_INTR_REGS_START KVMPPC_GSID_HDAR +#define KVMPPC_GSE_INTR_REGS_END KVMPPC_GSID_ASDR +#define KVMPPC_GSE_INTR_REGS_COUNT \ + (KVMPPC_GSE_INTR_REGS_END - KVMPPC_GSE_INTR_REGS_START + 1) + +#define KVMPPC_GSE_IDEN_COUNT \ + (KVMPPC_GSE_HOSTWIDE_COUNT + \ + KVMPPC_GSE_GUESTWIDE_COUNT + KVMPPC_GSE_META_COUNT + \ + KVMPPC_GSE_DW_REGS_COUNT + KVMPPC_GSE_W_REGS_COUNT + \ + KVMPPC_GSE_VSRS_COUNT + KVMPPC_GSE_INTR_REGS_COUNT) + +/** + * Ranges of guest state buffer elements + */ +enum { + KVMPPC_GS_CLASS_GUESTWIDE = 0x01, + KVMPPC_GS_CLASS_HOSTWIDE = 0x02, + KVMPPC_GS_CLASS_META = 0x04, + KVMPPC_GS_CLASS_DWORD_REG = 0x08, + KVMPPC_GS_CLASS_WORD_REG = 0x10, + KVMPPC_GS_CLASS_VECTOR = 0x18, + KVMPPC_GS_CLASS_INTR = 0x20, +}; + +/** + * Types of guest state buffer elements + */ +enum { + KVMPPC_GSE_BE32, + KVMPPC_GSE_BE64, + KVMPPC_GSE_VEC128, + KVMPPC_GSE_PARTITION_TABLE, + KVMPPC_GSE_PROCESS_TABLE, + KVMPPC_GSE_BUFFER, + __KVMPPC_GSE_TYPE_MAX, +}; + +/** + * Flags for guest state elements + */ +enum { + KVMPPC_GS_FLAGS_WIDE = 0x01, + KVMPPC_GS_FLAGS_HOST_WIDE = 0x02, +}; + +/** + * struct kvmppc_gs_part_table - deserialized partition table information + * element + * @address: start of the partition table + * @ea_bits: number of bits in the effective address + * @gpd_size: root page directory size + */ +struct kvmppc_gs_part_table { + u64 address; + u64 ea_bits; + u64 gpd_size; +}; + +/** + * struct kvmppc_gs_proc_table - deserialized process table information element + * @address: start of the process table + * @gpd_size: process table size + */ +struct kvmppc_gs_proc_table { + u64 address; + u64 gpd_size; +}; + +/** + * struct kvmppc_gs_buff_info - deserialized meta guest state buffer information + * @address: start of the guest state buffer + * @size: size of the guest state buffer + */ +struct kvmppc_gs_buff_info { + u64 address; + u64 size; +}; + +/** + * struct kvmppc_gs_header - serialized guest state buffer header + * @nelem: count of guest state elements in the buffer + * @data: start of the stream of elements in the buffer + */ +struct kvmppc_gs_header { + __be32 nelems; + char data[]; +} __packed; + +/** + * struct kvmppc_gs_elem - serialized guest state buffer element + * @iden: Guest State ID + * @len: length of data + * @data: the guest state buffer element's value + */ +struct kvmppc_gs_elem { + __be16 iden; + __be16 len; + char data[]; +} __packed; + +/** + * struct kvmppc_gs_buff - a guest state buffer with metadata. + * @capacity: total length of the buffer + * @len: current length of the elements and header + * @guest_id: guest id associated with the buffer + * @vcpu_id: vcpu_id associated with the buffer + * @hdr: the serialised guest state buffer + */ +struct kvmppc_gs_buff { + size_t capacity; + size_t len; + unsigned long guest_id; + unsigned long vcpu_id; + struct kvmppc_gs_header *hdr; +}; + +/** + * struct kvmppc_gs_bitmap - a bitmap for element ids + * @bitmap: a bitmap large enough for all Guest State IDs + */ +struct kvmppc_gs_bitmap { + /* private: */ + DECLARE_BITMAP(bitmap, KVMPPC_GSE_IDEN_COUNT); +}; + +/** + * struct kvmppc_gs_parser - a map of element ids to locations in a buffer + * @iterator: bitmap used for iterating + * @gses: contains the pointers to elements + * + * A guest state parser is used for deserialising a guest state buffer. + * Given a buffer, it then allows looking up guest state elements using + * a guest state id. + */ +struct kvmppc_gs_parser { + /* private: */ + struct kvmppc_gs_bitmap iterator; + struct kvmppc_gs_elem *gses[KVMPPC_GSE_IDEN_COUNT]; +}; + +enum { + GSM_GUEST_WIDE = 0x1, + GSM_SEND = 0x2, + GSM_RECEIVE = 0x4, + GSM_GSB_OWNER = 0x8, +}; + +struct kvmppc_gs_msg; + +/** + * struct kvmppc_gs_msg_ops - guest state message behavior + * @get_size: maximum size required for the message data + * @fill_info: serializes to the guest state buffer format + * @refresh_info: dserializes from the guest state buffer format + */ +struct kvmppc_gs_msg_ops { + size_t (*get_size)(struct kvmppc_gs_msg *gsm); + int (*fill_info)(struct kvmppc_gs_buff *gsb, struct kvmppc_gs_msg *gsm); + int (*refresh_info)(struct kvmppc_gs_msg *gsm, + struct kvmppc_gs_buff *gsb); +}; + +/** + * struct kvmppc_gs_msg - a guest state message + * @bitmap: the guest state ids that should be included + * @ops: modify message behavior for reading and writing to buffers + * @flags: host wide, guest wide or thread wide + * @data: location where buffer data will be written to or from. + * + * A guest state message is allows flexibility in sending in receiving data + * in a guest state buffer format. + */ +struct kvmppc_gs_msg { + struct kvmppc_gs_bitmap bitmap; + struct kvmppc_gs_msg_ops *ops; + unsigned long flags; + void *data; +}; + +/************************************************************************** + * Guest State IDs + **************************************************************************/ + +u16 kvmppc_gsid_size(u16 iden); +unsigned long kvmppc_gsid_flags(u16 iden); +u64 kvmppc_gsid_mask(u16 iden); + +/************************************************************************** + * Guest State Buffers + **************************************************************************/ +struct kvmppc_gs_buff *kvmppc_gsb_new(size_t size, unsigned long guest_id, + unsigned long vcpu_id, gfp_t flags); +void kvmppc_gsb_free(struct kvmppc_gs_buff *gsb); +void *kvmppc_gsb_put(struct kvmppc_gs_buff *gsb, size_t size); +int kvmppc_gsb_send(struct kvmppc_gs_buff *gsb, unsigned long flags); +int kvmppc_gsb_recv(struct kvmppc_gs_buff *gsb, unsigned long flags); + +/** + * kvmppc_gsb_header() - the header of a guest state buffer + * @gsb: guest state buffer + * + * Returns a pointer to the buffer header. + */ +static inline struct kvmppc_gs_header * +kvmppc_gsb_header(struct kvmppc_gs_buff *gsb) +{ + return gsb->hdr; +} + +/** + * kvmppc_gsb_data() - the elements of a guest state buffer + * @gsb: guest state buffer + * + * Returns a pointer to the first element of the buffer data. + */ +static inline struct kvmppc_gs_elem *kvmppc_gsb_data(struct kvmppc_gs_buff *gsb) +{ + return (struct kvmppc_gs_elem *)kvmppc_gsb_header(gsb)->data; +} + +/** + * kvmppc_gsb_len() - the current length of a guest state buffer + * @gsb: guest state buffer + * + * Returns the length including the header of a buffer. + */ +static inline size_t kvmppc_gsb_len(struct kvmppc_gs_buff *gsb) +{ + return gsb->len; +} + +/** + * kvmppc_gsb_capacity() - the capacity of a guest state buffer + * @gsb: guest state buffer + * + * Returns the capacity of a buffer. + */ +static inline size_t kvmppc_gsb_capacity(struct kvmppc_gs_buff *gsb) +{ + return gsb->capacity; +} + +/** + * kvmppc_gsb_paddress() - the physical address of buffer + * @gsb: guest state buffer + * + * Returns the physical address of the buffer. + */ +static inline u64 kvmppc_gsb_paddress(struct kvmppc_gs_buff *gsb) +{ + return __pa(kvmppc_gsb_header(gsb)); +} + +/** + * kvmppc_gsb_nelems() - the number of elements in a buffer + * @gsb: guest state buffer + * + * Returns the number of elements in a buffer + */ +static inline u32 kvmppc_gsb_nelems(struct kvmppc_gs_buff *gsb) +{ + return be32_to_cpu(kvmppc_gsb_header(gsb)->nelems); +} + +/** + * kvmppc_gsb_reset() - empty a guest state buffer + * @gsb: guest state buffer + * + * Reset the number of elements and length of buffer to empty. + */ +static inline void kvmppc_gsb_reset(struct kvmppc_gs_buff *gsb) +{ + kvmppc_gsb_header(gsb)->nelems = cpu_to_be32(0); + gsb->len = sizeof(struct kvmppc_gs_header); +} + +/** + * kvmppc_gsb_data_len() - the length of a buffer excluding the header + * @gsb: guest state buffer + * + * Returns the length of a buffer excluding the header + */ +static inline size_t kvmppc_gsb_data_len(struct kvmppc_gs_buff *gsb) +{ + return gsb->len - sizeof(struct kvmppc_gs_header); +} + +/** + * kvmppc_gsb_data_cap() - the capacity of a buffer excluding the header + * @gsb: guest state buffer + * + * Returns the capacity of a buffer excluding the header + */ +static inline size_t kvmppc_gsb_data_cap(struct kvmppc_gs_buff *gsb) +{ + return gsb->capacity - sizeof(struct kvmppc_gs_header); +} + +/** + * kvmppc_gsb_for_each_elem - iterate over the elements in a buffer + * @i: loop counter + * @pos: set to current element + * @gsb: guest state buffer + * @rem: initialized to buffer capacity, holds bytes currently remaining in + * stream + */ +#define kvmppc_gsb_for_each_elem(i, pos, gsb, rem) \ + kvmppc_gse_for_each_elem(i, kvmppc_gsb_nelems(gsb), pos, \ + kvmppc_gsb_data(gsb), \ + kvmppc_gsb_data_cap(gsb), rem) + +/************************************************************************** + * Guest State Elements + **************************************************************************/ + +/** + * kvmppc_gse_iden() - guest state ID of element + * @gse: guest state element + * + * Return the guest state ID in host endianness. + */ +static inline u16 kvmppc_gse_iden(const struct kvmppc_gs_elem *gse) +{ + return be16_to_cpu(gse->iden); +} + +/** + * kvmppc_gse_len() - length of guest state element data + * @gse: guest state element + * + * Returns the length of guest state element data + */ +static inline u16 kvmppc_gse_len(const struct kvmppc_gs_elem *gse) +{ + return be16_to_cpu(gse->len); +} + +/** + * kvmppc_gse_total_len() - total length of guest state element + * @gse: guest state element + * + * Returns the length of the data plus the ID and size header. + */ +static inline u16 kvmppc_gse_total_len(const struct kvmppc_gs_elem *gse) +{ + return be16_to_cpu(gse->len) + sizeof(*gse); +} + +/** + * kvmppc_gse_total_size() - space needed for a given data length + * @size: data length + * + * Returns size plus the space needed for the ID and size header. + */ +static inline u16 kvmppc_gse_total_size(u16 size) +{ + return sizeof(struct kvmppc_gs_elem) + size; +} + +/** + * kvmppc_gse_data() - pointer to data of a guest state element + * @gse: guest state element + * + * Returns a pointer to the beginning of guest state element data. + */ +static inline void *kvmppc_gse_data(const struct kvmppc_gs_elem *gse) +{ + return (void *)gse->data; +} + +/** + * kvmppc_gse_ok() - checks space exists for guest state element + * @gse: guest state element + * @remaining: bytes of space remaining + * + * Returns true if the guest state element can fit in remaining space. + */ +static inline bool kvmppc_gse_ok(const struct kvmppc_gs_elem *gse, + int remaining) +{ + return remaining >= kvmppc_gse_total_len(gse); +} + +/** + * kvmppc_gse_next() - iterate to the next guest state element in a stream + * @gse: stream of guest state elements + * @remaining: length of the guest element stream + * + * Returns the next guest state element in a stream of elements. The length of + * the stream is updated in remaining. + */ +static inline struct kvmppc_gs_elem * +kvmppc_gse_next(const struct kvmppc_gs_elem *gse, int *remaining) +{ + int len = sizeof(*gse) + kvmppc_gse_len(gse); + + *remaining -= len; + return (struct kvmppc_gs_elem *)(gse->data + kvmppc_gse_len(gse)); +} + +/** + * kvmppc_gse_for_each_elem - iterate over a stream of guest state elements + * @i: loop counter + * @max: number of elements + * @pos: set to current element + * @head: head of elements + * @len: length of the stream + * @rem: initialized to len, holds bytes currently remaining elements + */ +#define kvmppc_gse_for_each_elem(i, max, pos, head, len, rem) \ + for (i = 0, pos = head, rem = len; kvmppc_gse_ok(pos, rem) && i < max; \ + pos = kvmppc_gse_next(pos, &(rem)), i++) + +int __kvmppc_gse_put(struct kvmppc_gs_buff *gsb, u16 iden, u16 size, + const void *data); +int kvmppc_gse_parse(struct kvmppc_gs_parser *gsp, struct kvmppc_gs_buff *gsb); + +/** + * kvmppc_gse_put_be32() - add a be32 guest state element to a buffer + * @gsb: guest state buffer to add element to + * @iden: guest state ID + * @val: big endian value + */ +static inline int kvmppc_gse_put_be32(struct kvmppc_gs_buff *gsb, u16 iden, + __be32 val) +{ + __be32 tmp; + + tmp = val; + return __kvmppc_gse_put(gsb, iden, sizeof(__be32), &tmp); +} + +/** + * kvmppc_gse_put_u32() - add a host endian 32bit int guest state element to a + * buffer + * @gsb: guest state buffer to add element to + * @iden: guest state ID + * @val: host endian value + */ +static inline int kvmppc_gse_put_u32(struct kvmppc_gs_buff *gsb, u16 iden, + u32 val) +{ + __be32 tmp; + + val &= kvmppc_gsid_mask(iden); + tmp = cpu_to_be32(val); + return kvmppc_gse_put_be32(gsb, iden, tmp); +} + +/** + * kvmppc_gse_put_be64() - add a be64 guest state element to a buffer + * @gsb: guest state buffer to add element to + * @iden: guest state ID + * @val: big endian value + */ +static inline int kvmppc_gse_put_be64(struct kvmppc_gs_buff *gsb, u16 iden, + __be64 val) +{ + __be64 tmp; + + tmp = val; + return __kvmppc_gse_put(gsb, iden, sizeof(__be64), &tmp); +} + +/** + * kvmppc_gse_put_u64() - add a host endian 64bit guest state element to a + * buffer + * @gsb: guest state buffer to add element to + * @iden: guest state ID + * @val: host endian value + */ +static inline int kvmppc_gse_put_u64(struct kvmppc_gs_buff *gsb, u16 iden, + u64 val) +{ + __be64 tmp; + + val &= kvmppc_gsid_mask(iden); + tmp = cpu_to_be64(val); + return kvmppc_gse_put_be64(gsb, iden, tmp); +} + +/** + * __kvmppc_gse_put_reg() - add a register type guest state element to a buffer + * @gsb: guest state buffer to add element to + * @iden: guest state ID + * @val: host endian value + * + * Adds a register type guest state element. Uses the guest state ID for + * determining the length of the guest element. If the guest state ID has + * bits that can not be set they will be cleared. + */ +static inline int __kvmppc_gse_put_reg(struct kvmppc_gs_buff *gsb, u16 iden, + u64 val) +{ + val &= kvmppc_gsid_mask(iden); + if (kvmppc_gsid_size(iden) == sizeof(u64)) + return kvmppc_gse_put_u64(gsb, iden, val); + + if (kvmppc_gsid_size(iden) == sizeof(u32)) { + u32 tmp; + + tmp = (u32)val; + if (tmp != val) + return -EINVAL; + + return kvmppc_gse_put_u32(gsb, iden, tmp); + } + return -EINVAL; +} + +/** + * kvmppc_gse_put_vector128() - add a vector guest state element to a buffer + * @gsb: guest state buffer to add element to + * @iden: guest state ID + * @val: 16 byte vector value + */ +static inline int kvmppc_gse_put_vector128(struct kvmppc_gs_buff *gsb, u16 iden, + vector128 *val) +{ + __be64 tmp[2] = { 0 }; + union { + __vector128 v; + u64 dw[2]; + } u; + + u.v = *val; + tmp[0] = cpu_to_be64(u.dw[TS_FPROFFSET]); +#ifdef CONFIG_VSX + tmp[1] = cpu_to_be64(u.dw[TS_VSRLOWOFFSET]); +#endif + return __kvmppc_gse_put(gsb, iden, sizeof(tmp), &tmp); +} + +/** + * kvmppc_gse_put_part_table() - add a partition table guest state element to a + * buffer + * @gsb: guest state buffer to add element to + * @iden: guest state ID + * @val: partition table value + */ +static inline int kvmppc_gse_put_part_table(struct kvmppc_gs_buff *gsb, + u16 iden, + struct kvmppc_gs_part_table val) +{ + __be64 tmp[3]; + + tmp[0] = cpu_to_be64(val.address); + tmp[1] = cpu_to_be64(val.ea_bits); + tmp[2] = cpu_to_be64(val.gpd_size); + return __kvmppc_gse_put(gsb, KVMPPC_GSID_PARTITION_TABLE, sizeof(tmp), + &tmp); +} + +/** + * kvmppc_gse_put_proc_table() - add a process table guest state element to a + * buffer + * @gsb: guest state buffer to add element to + * @iden: guest state ID + * @val: process table value + */ +static inline int kvmppc_gse_put_proc_table(struct kvmppc_gs_buff *gsb, + u16 iden, + struct kvmppc_gs_proc_table val) +{ + __be64 tmp[2]; + + tmp[0] = cpu_to_be64(val.address); + tmp[1] = cpu_to_be64(val.gpd_size); + return __kvmppc_gse_put(gsb, KVMPPC_GSID_PROCESS_TABLE, sizeof(tmp), + &tmp); +} + +/** + * kvmppc_gse_put_buff_info() - adds a GSB description guest state element to a + * buffer + * @gsb: guest state buffer to add element to + * @iden: guest state ID + * @val: guest state buffer description value + */ +static inline int kvmppc_gse_put_buff_info(struct kvmppc_gs_buff *gsb, u16 iden, + struct kvmppc_gs_buff_info val) +{ + __be64 tmp[2]; + + tmp[0] = cpu_to_be64(val.address); + tmp[1] = cpu_to_be64(val.size); + return __kvmppc_gse_put(gsb, iden, sizeof(tmp), &tmp); +} + +int __kvmppc_gse_put(struct kvmppc_gs_buff *gsb, u16 iden, u16 size, + const void *data); + +/** + * kvmppc_gse_get_be32() - return the data of a be32 element + * @gse: guest state element + */ +static inline __be32 kvmppc_gse_get_be32(const struct kvmppc_gs_elem *gse) +{ + if (WARN_ON(kvmppc_gse_len(gse) != sizeof(__be32))) + return 0; + return *(__be32 *)kvmppc_gse_data(gse); +} + +/** + * kvmppc_gse_get_u32() - return the data of a be32 element in host endianness + * @gse: guest state element + */ +static inline u32 kvmppc_gse_get_u32(const struct kvmppc_gs_elem *gse) +{ + return be32_to_cpu(kvmppc_gse_get_be32(gse)); +} + +/** + * kvmppc_gse_get_be64() - return the data of a be64 element + * @gse: guest state element + */ +static inline __be64 kvmppc_gse_get_be64(const struct kvmppc_gs_elem *gse) +{ + if (WARN_ON(kvmppc_gse_len(gse) != sizeof(__be64))) + return 0; + return *(__be64 *)kvmppc_gse_data(gse); +} + +/** + * kvmppc_gse_get_u64() - return the data of a be64 element in host endianness + * @gse: guest state element + */ +static inline u64 kvmppc_gse_get_u64(const struct kvmppc_gs_elem *gse) +{ + return be64_to_cpu(kvmppc_gse_get_be64(gse)); +} + +/** + * kvmppc_gse_get_vector128() - return the data of a vector element + * @gse: guest state element + */ +static inline void kvmppc_gse_get_vector128(const struct kvmppc_gs_elem *gse, + vector128 *v) +{ + union { + __vector128 v; + u64 dw[2]; + } u = { 0 }; + __be64 *src; + + if (WARN_ON(kvmppc_gse_len(gse) != sizeof(__vector128))) + *v = u.v; + + src = (__be64 *)kvmppc_gse_data(gse); + u.dw[TS_FPROFFSET] = be64_to_cpu(src[0]); +#ifdef CONFIG_VSX + u.dw[TS_VSRLOWOFFSET] = be64_to_cpu(src[1]); +#endif + *v = u.v; +} + +/************************************************************************** + * Guest State Bitmap + **************************************************************************/ + +bool kvmppc_gsbm_test(struct kvmppc_gs_bitmap *gsbm, u16 iden); +void kvmppc_gsbm_set(struct kvmppc_gs_bitmap *gsbm, u16 iden); +void kvmppc_gsbm_clear(struct kvmppc_gs_bitmap *gsbm, u16 iden); +u16 kvmppc_gsbm_next(struct kvmppc_gs_bitmap *gsbm, u16 prev); + +/** + * kvmppc_gsbm_zero - zero the entire bitmap + * @gsbm: guest state buffer bitmap + */ +static inline void kvmppc_gsbm_zero(struct kvmppc_gs_bitmap *gsbm) +{ + bitmap_zero(gsbm->bitmap, KVMPPC_GSE_IDEN_COUNT); +} + +/** + * kvmppc_gsbm_fill - fill the entire bitmap + * @gsbm: guest state buffer bitmap + */ +static inline void kvmppc_gsbm_fill(struct kvmppc_gs_bitmap *gsbm) +{ + bitmap_fill(gsbm->bitmap, KVMPPC_GSE_IDEN_COUNT); + clear_bit(0, gsbm->bitmap); +} + +/** + * kvmppc_gsbm_for_each - iterate the present guest state IDs + * @gsbm: guest state buffer bitmap + * @iden: current guest state ID + */ +#define kvmppc_gsbm_for_each(gsbm, iden) \ + for (iden = kvmppc_gsbm_next(gsbm, 0); iden != 0; \ + iden = kvmppc_gsbm_next(gsbm, iden)) + +/************************************************************************** + * Guest State Parser + **************************************************************************/ + +void kvmppc_gsp_insert(struct kvmppc_gs_parser *gsp, u16 iden, + struct kvmppc_gs_elem *gse); +struct kvmppc_gs_elem *kvmppc_gsp_lookup(struct kvmppc_gs_parser *gsp, + u16 iden); + +/** + * kvmppc_gsp_for_each - iterate the <guest state IDs, guest state element> + * pairs + * @gsp: guest state buffer bitmap + * @iden: current guest state ID + * @gse: guest state element + */ +#define kvmppc_gsp_for_each(gsp, iden, gse) \ + for (iden = kvmppc_gsbm_next(&(gsp)->iterator, 0), \ + gse = kvmppc_gsp_lookup((gsp), iden); \ + iden != 0; iden = kvmppc_gsbm_next(&(gsp)->iterator, iden), \ + gse = kvmppc_gsp_lookup((gsp), iden)) + +/************************************************************************** + * Guest State Message + **************************************************************************/ + +/** + * kvmppc_gsm_for_each - iterate the guest state IDs included in a guest state + * message + * @gsp: guest state buffer bitmap + * @iden: current guest state ID + * @gse: guest state element + */ +#define kvmppc_gsm_for_each(gsm, iden) \ + for (iden = kvmppc_gsbm_next(&gsm->bitmap, 0); iden != 0; \ + iden = kvmppc_gsbm_next(&gsm->bitmap, iden)) + +int kvmppc_gsm_init(struct kvmppc_gs_msg *mgs, struct kvmppc_gs_msg_ops *ops, + void *data, unsigned long flags); + +struct kvmppc_gs_msg *kvmppc_gsm_new(struct kvmppc_gs_msg_ops *ops, void *data, + unsigned long flags, gfp_t gfp_flags); +void kvmppc_gsm_free(struct kvmppc_gs_msg *gsm); +size_t kvmppc_gsm_size(struct kvmppc_gs_msg *gsm); +int kvmppc_gsm_fill_info(struct kvmppc_gs_msg *gsm, struct kvmppc_gs_buff *gsb); +int kvmppc_gsm_refresh_info(struct kvmppc_gs_msg *gsm, + struct kvmppc_gs_buff *gsb); + +/** + * kvmppc_gsm_include - indicate a guest state ID should be included when + * serializing + * @gsm: guest state message + * @iden: guest state ID + */ +static inline void kvmppc_gsm_include(struct kvmppc_gs_msg *gsm, u16 iden) +{ + kvmppc_gsbm_set(&gsm->bitmap, iden); +} + +/** + * kvmppc_gsm_includes - check if a guest state ID will be included when + * serializing + * @gsm: guest state message + * @iden: guest state ID + */ +static inline bool kvmppc_gsm_includes(struct kvmppc_gs_msg *gsm, u16 iden) +{ + return kvmppc_gsbm_test(&gsm->bitmap, iden); +} + +/** + * kvmppc_gsm_includes - indicate all guest state IDs should be included when + * serializing + * @gsm: guest state message + * @iden: guest state ID + */ +static inline void kvmppc_gsm_include_all(struct kvmppc_gs_msg *gsm) +{ + kvmppc_gsbm_fill(&gsm->bitmap); +} + +/** + * kvmppc_gsm_include - clear the guest state IDs that should be included when + * serializing + * @gsm: guest state message + */ +static inline void kvmppc_gsm_reset(struct kvmppc_gs_msg *gsm) +{ + kvmppc_gsbm_zero(&gsm->bitmap); +} + +/** + * kvmppc_gsb_receive_data - flexibly update values from a guest state buffer + * @gsb: guest state buffer + * @gsm: guest state message + * + * Requests updated values for the guest state values included in the guest + * state message. The guest state message will then deserialize the guest state + * buffer. + */ +static inline int kvmppc_gsb_receive_data(struct kvmppc_gs_buff *gsb, + struct kvmppc_gs_msg *gsm) +{ + int rc; + + kvmppc_gsb_reset(gsb); + rc = kvmppc_gsm_fill_info(gsm, gsb); + if (rc < 0) + return rc; + + rc = kvmppc_gsb_recv(gsb, gsm->flags); + if (rc < 0) + return rc; + + rc = kvmppc_gsm_refresh_info(gsm, gsb); + if (rc < 0) + return rc; + return 0; +} + +/** + * kvmppc_gsb_recv - receive a single guest state ID + * @gsb: guest state buffer + * @gsm: guest state message + * @iden: guest state identity + */ +static inline int kvmppc_gsb_receive_datum(struct kvmppc_gs_buff *gsb, + struct kvmppc_gs_msg *gsm, u16 iden) +{ + int rc; + + kvmppc_gsm_include(gsm, iden); + rc = kvmppc_gsb_receive_data(gsb, gsm); + if (rc < 0) + return rc; + kvmppc_gsm_reset(gsm); + return 0; +} + +/** + * kvmppc_gsb_send_data - flexibly send values from a guest state buffer + * @gsb: guest state buffer + * @gsm: guest state message + * + * Sends the guest state values included in the guest state message. + */ +static inline int kvmppc_gsb_send_data(struct kvmppc_gs_buff *gsb, + struct kvmppc_gs_msg *gsm) +{ + int rc; + + kvmppc_gsb_reset(gsb); + rc = kvmppc_gsm_fill_info(gsm, gsb); + if (rc < 0) + return rc; + rc = kvmppc_gsb_send(gsb, gsm->flags); + + return rc; +} + +/** + * kvmppc_gsb_recv - send a single guest state ID + * @gsb: guest state buffer + * @gsm: guest state message + * @iden: guest state identity + */ +static inline int kvmppc_gsb_send_datum(struct kvmppc_gs_buff *gsb, + struct kvmppc_gs_msg *gsm, u16 iden) +{ + int rc; + + kvmppc_gsm_include(gsm, iden); + rc = kvmppc_gsb_send_data(gsb, gsm); + if (rc < 0) + return rc; + kvmppc_gsm_reset(gsm); + return 0; +} + +#endif /* _ASM_POWERPC_GUEST_STATE_BUFFER_H */ diff --git a/arch/powerpc/include/asm/hardirq.h b/arch/powerpc/include/asm/hardirq.h index 3bdcfce2c42a..f133b5930ae1 100644 --- a/arch/powerpc/include/asm/hardirq.h +++ b/arch/powerpc/include/asm/hardirq.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_HARDIRQ_H #define _ASM_POWERPC_HARDIRQ_H @@ -6,10 +7,16 @@ typedef struct { unsigned int __softirq_pending; - unsigned int timer_irqs; + unsigned int timer_irqs_event; + unsigned int broadcast_irqs_event; + unsigned int timer_irqs_others; unsigned int pmu_irqs; unsigned int mce_exceptions; unsigned int spurious_irqs; + unsigned int sreset_irqs; +#ifdef CONFIG_PPC_WATCHDOG + unsigned int soft_nmi_irqs; +#endif #ifdef CONFIG_PPC_DOORBELL unsigned int doorbell_irqs; #endif @@ -18,8 +25,7 @@ typedef struct { DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat); #define __ARCH_IRQ_STAT - -#define local_softirq_pending() __get_cpu_var(irq_stat).__softirq_pending +#define __ARCH_IRQ_EXIT_IRQS_DISABLED static inline void ack_bad_irq(unsigned int irq) { diff --git a/arch/powerpc/include/asm/head-64.h b/arch/powerpc/include/asm/head-64.h new file mode 100644 index 000000000000..3966bd5810cb --- /dev/null +++ b/arch/powerpc/include/asm/head-64.h @@ -0,0 +1,172 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_HEAD_64_H +#define _ASM_POWERPC_HEAD_64_H + +#include <asm/cache.h> + +#ifdef __ASSEMBLER__ +/* + * We can't do CPP stringification and concatination directly into the section + * name for some reason, so these macros can do it for us. + */ +.macro define_ftsec name + .section ".head.text.\name\()","ax",@progbits +.endm +.macro define_data_ftsec name + .section ".head.data.\name\()","a",@progbits +.endm +.macro use_ftsec name + .section ".head.text.\name\()","ax",@progbits +.endm + +/* + * Fixed (location) sections are used by opening fixed sections and emitting + * fixed section entries into them before closing them. Multiple fixed sections + * can be open at any time. + * + * Each fixed section created in a .S file must have corresponding linkage + * directives including location, added to arch/powerpc/kernel/vmlinux.lds.S + * + * For each fixed section, code is generated into it in the order which it + * appears in the source. Fixed section entries can be placed at a fixed + * location within the section using _LOCATION postifx variants. These must + * be ordered according to their relative placements within the section. + * + * OPEN_FIXED_SECTION(section_name, start_address, end_address) + * FIXED_SECTION_ENTRY_BEGIN(section_name, label1) + * + * USE_FIXED_SECTION(section_name) + * label3: + * li r10,128 + * mv r11,r10 + + * FIXED_SECTION_ENTRY_BEGIN_LOCATION(section_name, label2, start_address, size) + * FIXED_SECTION_ENTRY_END_LOCATION(section_name, label2, start_address, size) + * CLOSE_FIXED_SECTION(section_name) + * + * ZERO_FIXED_SECTION can be used to emit zeroed data. + * + * Troubleshooting: + * - If the build dies with "Error: attempt to move .org backwards" at + * CLOSE_FIXED_SECTION() or elsewhere, there may be something + * unexpected being added there. Remove the '. = x_len' line, rebuild, and + * check what is pushing the section down. + * - If the build dies in linking, check arch/powerpc/tools/head_check.sh + * comments. + * - If the kernel crashes or hangs in very early boot, it could be linker + * stubs at the start of the main text. + */ + +#define OPEN_FIXED_SECTION(sname, start, end) \ + sname##_start = (start); \ + sname##_end = (end); \ + sname##_len = (end) - (start); \ + define_ftsec sname; \ + . = 0x0; \ +start_##sname: + +/* + * .linker_stub_catch section is used to catch linker stubs from being + * inserted in our .text section, above the start_text label (which breaks + * the ABS_ADDR calculation). See kernel/vmlinux.lds.S and tools/head_check.sh + * for more details. We would prefer to just keep a cacheline (0x80), but + * 0x100 seems to be how the linker aligns branch stub groups. + */ +#ifdef CONFIG_LD_HEAD_STUB_CATCH +#define OPEN_TEXT_SECTION(start) \ + .section ".linker_stub_catch","ax",@progbits; \ +linker_stub_catch: \ + . = 0x4; \ + text_start = (start) + 0x100; \ + .section ".text","ax",@progbits; \ + .balign 0x100; \ +start_text: +#else +#define OPEN_TEXT_SECTION(start) \ + text_start = (start); \ + .section ".text","ax",@progbits; \ + . = 0x0; \ +start_text: +#endif + +#define ZERO_FIXED_SECTION(sname, start, end) \ + sname##_start = (start); \ + sname##_end = (end); \ + sname##_len = (end) - (start); \ + define_data_ftsec sname; \ + . = 0x0; \ + . = sname##_len; + +#define USE_FIXED_SECTION(sname) \ + use_ftsec sname; + +#define USE_TEXT_SECTION() \ + .text + +#define CLOSE_FIXED_SECTION(sname) \ + USE_FIXED_SECTION(sname); \ + . = sname##_len; \ +end_##sname: + + +#define __FIXED_SECTION_ENTRY_BEGIN(sname, name, __align) \ + USE_FIXED_SECTION(sname); \ + .balign __align; \ + .global name; \ +name: + +#define FIXED_SECTION_ENTRY_BEGIN(sname, name) \ + __FIXED_SECTION_ENTRY_BEGIN(sname, name, IFETCH_ALIGN_BYTES) + +#define FIXED_SECTION_ENTRY_BEGIN_LOCATION(sname, name, start, size) \ + USE_FIXED_SECTION(sname); \ + name##_start = (start); \ + .if ((start) % (size) != 0); \ + .error "Fixed section exception vector misalignment"; \ + .endif; \ + .if ((size) != 0x20) && ((size) != 0x80) && ((size) != 0x100) && ((size) != 0x1000); \ + .error "Fixed section exception vector bad size"; \ + .endif; \ + .if (start) < sname##_start; \ + .error "Fixed section underflow"; \ + .abort; \ + .endif; \ + . = (start) - sname##_start; \ + .global name; \ +name: + +#define FIXED_SECTION_ENTRY_END_LOCATION(sname, name, start, size) \ + .if (start) + (size) > sname##_end; \ + .error "Fixed section overflow"; \ + .abort; \ + .endif; \ + .if (. - name > (start) + (size) - name##_start); \ + .error "Fixed entry overflow"; \ + .abort; \ + .endif; \ + . = ((start) + (size) - sname##_start); \ + + +/* + * These macros are used to change symbols in other fixed sections to be + * absolute or related to our current fixed section. + * + * - DEFINE_FIXED_SYMBOL / FIXED_SYMBOL_ABS_ADDR is used to find the + * absolute address of a symbol within a fixed section, from any section. + * + * - ABS_ADDR is used to find the absolute address of any symbol, from within + * a fixed section. + */ +// define label as being _in_ sname +#define DEFINE_FIXED_SYMBOL(label, sname) \ + label##_absolute = (label - start_ ## sname + sname ## _start) + +#define FIXED_SYMBOL_ABS_ADDR(label) \ + (label##_absolute) + +// find label from _within_ sname +#define ABS_ADDR(label, sname) (label - start_ ## sname + sname ## _start) + +#endif /* __ASSEMBLER__ */ + +#endif /* _ASM_POWERPC_HEAD_64_H */ diff --git a/arch/powerpc/include/asm/heathrow.h b/arch/powerpc/include/asm/heathrow.h index 93f54958a9d1..8bc5b168762e 100644 --- a/arch/powerpc/include/asm/heathrow.h +++ b/arch/powerpc/include/asm/heathrow.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_HEATHROW_H #define _ASM_POWERPC_HEATHROW_H #ifdef __KERNEL__ diff --git a/arch/powerpc/include/asm/highmem.h b/arch/powerpc/include/asm/highmem.h index caaf6e00630d..c0fcd1bbdba9 100644 --- a/arch/powerpc/include/asm/highmem.h +++ b/arch/powerpc/include/asm/highmem.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * highmem.h: virtual kernel memory mappings for high memory * @@ -23,13 +24,10 @@ #ifdef __KERNEL__ #include <linux/interrupt.h> -#include <asm/kmap_types.h> -#include <asm/tlbflush.h> +#include <asm/cacheflush.h> #include <asm/page.h> #include <asm/fixmap.h> -extern pte_t *kmap_pte; -extern pgprot_t kmap_prot; extern pte_t *pkmap_page_table; /* @@ -58,48 +56,15 @@ extern pte_t *pkmap_page_table; #define PKMAP_NR(virt) ((virt-PKMAP_BASE) >> PAGE_SHIFT) #define PKMAP_ADDR(nr) (PKMAP_BASE + ((nr) << PAGE_SHIFT)) -extern void *kmap_high(struct page *page); -extern void kunmap_high(struct page *page); -extern void *kmap_atomic_prot(struct page *page, pgprot_t prot); -extern void __kunmap_atomic(void *kvaddr); - -static inline void *kmap(struct page *page) -{ - might_sleep(); - if (!PageHighMem(page)) - return page_address(page); - return kmap_high(page); -} - -static inline void kunmap(struct page *page) -{ - BUG_ON(in_interrupt()); - if (!PageHighMem(page)) - return; - kunmap_high(page); -} - -static inline void *kmap_atomic(struct page *page) -{ - return kmap_atomic_prot(page, kmap_prot); -} - -static inline struct page *kmap_atomic_to_page(void *ptr) -{ - unsigned long idx, vaddr = (unsigned long) ptr; - pte_t *pte; - - if (vaddr < FIXADDR_START) - return virt_to_page(ptr); - - idx = virt_to_fix(vaddr); - pte = kmap_pte - (idx - FIX_KMAP_BEGIN); - return pte_page(*pte); -} - - #define flush_cache_kmaps() flush_cache_all() +#define arch_kmap_local_set_pte(mm, vaddr, ptep, ptev) \ + __set_pte_at(mm, vaddr, ptep, ptev, 1) +#define arch_kmap_local_post_map(vaddr, pteval) \ + local_flush_tlb_page(NULL, vaddr) +#define arch_kmap_local_post_unmap(vaddr) \ + local_flush_tlb_page(NULL, vaddr) + #endif /* __KERNEL__ */ #endif /* _ASM_HIGHMEM_H */ diff --git a/arch/powerpc/include/asm/hmi.h b/arch/powerpc/include/asm/hmi.h new file mode 100644 index 000000000000..155748460c5d --- /dev/null +++ b/arch/powerpc/include/asm/hmi.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Hypervisor Maintenance Interrupt header file. + * + * Copyright 2015 IBM Corporation + * Author: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com> + */ + +#ifndef __ASM_PPC64_HMI_H__ +#define __ASM_PPC64_HMI_H__ + +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE + +#define CORE_TB_RESYNC_REQ_BIT 63 +#define MAX_SUBCORE_PER_CORE 4 + +/* + * sibling_subcore_state structure is used to co-ordinate all threads + * during HMI to avoid TB corruption. This structure is allocated once + * per each core and shared by all threads on that core. + */ +struct sibling_subcore_state { + unsigned long flags; + u8 in_guest[MAX_SUBCORE_PER_CORE]; +}; + +extern void wait_for_subcore_guest_exit(void); +extern void wait_for_tb_resync(void); +#else +static inline void wait_for_subcore_guest_exit(void) { } +static inline void wait_for_tb_resync(void) { } +#endif + +struct pt_regs; +extern long hmi_handle_debugtrig(struct pt_regs *regs); + +#endif /* __ASM_PPC64_HMI_H__ */ diff --git a/arch/powerpc/include/asm/hugetlb.h b/arch/powerpc/include/asm/hugetlb.h index d750336b171d..86326587e58d 100644 --- a/arch/powerpc/include/asm/hugetlb.h +++ b/arch/powerpc/include/asm/hugetlb.h @@ -1,190 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_HUGETLB_H #define _ASM_POWERPC_HUGETLB_H #ifdef CONFIG_HUGETLB_PAGE #include <asm/page.h> -#include <asm-generic/hugetlb.h> - -extern struct kmem_cache *hugepte_cache; #ifdef CONFIG_PPC_BOOK3S_64 -/* - * This should work for other subarchs too. But right now we use the - * new format only for 64bit book3s - */ -static inline pte_t *hugepd_page(hugepd_t hpd) -{ - BUG_ON(!hugepd_ok(hpd)); - /* - * We have only four bits to encode, MMU page size - */ - BUILD_BUG_ON((MMU_PAGE_COUNT - 1) > 0xf); - return (pte_t *)(hpd.pd & ~HUGEPD_SHIFT_MASK); -} - -static inline unsigned int hugepd_mmu_psize(hugepd_t hpd) -{ - return (hpd.pd & HUGEPD_SHIFT_MASK) >> 2; -} - -static inline unsigned int hugepd_shift(hugepd_t hpd) -{ - return mmu_psize_to_shift(hugepd_mmu_psize(hpd)); -} - -#else - -static inline pte_t *hugepd_page(hugepd_t hpd) -{ - BUG_ON(!hugepd_ok(hpd)); - return (pte_t *)((hpd.pd & ~HUGEPD_SHIFT_MASK) | PD_HUGE); -} - -static inline unsigned int hugepd_shift(hugepd_t hpd) -{ - return hpd.pd & HUGEPD_SHIFT_MASK; -} - +#include <asm/book3s/64/hugetlb.h> +#elif defined(CONFIG_PPC_E500) +#include <asm/nohash/hugetlb-e500.h> +#elif defined(CONFIG_PPC_8xx) +#include <asm/nohash/32/hugetlb-8xx.h> #endif /* CONFIG_PPC_BOOK3S_64 */ +extern bool hugetlb_disabled; -static inline pte_t *hugepte_offset(hugepd_t *hpdp, unsigned long addr, - unsigned pdshift) +static inline bool hugepages_supported(void) { - /* - * On FSL BookE, we have multiple higher-level table entries that - * point to the same hugepte. Just use the first one since they're all - * identical. So for that case, idx=0. - */ - unsigned long idx = 0; - - pte_t *dir = hugepd_page(*hpdp); -#ifndef CONFIG_PPC_FSL_BOOK3E - idx = (addr & ((1UL << pdshift) - 1)) >> hugepd_shift(*hpdp); -#endif - - return dir + idx; -} + if (hugetlb_disabled) + return false; -pte_t *huge_pte_offset_and_shift(struct mm_struct *mm, - unsigned long addr, unsigned *shift); + return HPAGE_SHIFT != 0; +} +#define hugepages_supported hugepages_supported -void flush_dcache_icache_hugepage(struct page *page); +void __init hugetlbpage_init_defaultsize(void); -#if defined(CONFIG_PPC_MM_SLICES) || defined(CONFIG_PPC_SUBPAGE_PROT) -int is_hugepage_only_range(struct mm_struct *mm, unsigned long addr, +int slice_is_hugepage_only_range(struct mm_struct *mm, unsigned long addr, unsigned long len); -#else + static inline int is_hugepage_only_range(struct mm_struct *mm, unsigned long addr, unsigned long len) { + if (IS_ENABLED(CONFIG_PPC_64S_HASH_MMU) && !radix_enabled()) + return slice_is_hugepage_only_range(mm, addr, len); return 0; } -#endif - -void book3e_hugetlb_preload(struct vm_area_struct *vma, unsigned long ea, - pte_t pte); -void flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr); - -void hugetlb_free_pgd_range(struct mmu_gather *tlb, unsigned long addr, - unsigned long end, unsigned long floor, - unsigned long ceiling); - -/* - * The version of vma_mmu_pagesize() in arch/powerpc/mm/hugetlbpage.c needs - * to override the version in mm/hugetlb.c - */ -#define vma_mmu_pagesize vma_mmu_pagesize - -/* - * If the arch doesn't supply something else, assume that hugepage - * size aligned regions are ok without further preparation. - */ -static inline int prepare_hugepage_range(struct file *file, - unsigned long addr, unsigned long len) -{ - struct hstate *h = hstate_file(file); - if (len & ~huge_page_mask(h)) - return -EINVAL; - if (addr & ~huge_page_mask(h)) - return -EINVAL; - return 0; -} - -static inline void hugetlb_prefault_arch_hook(struct mm_struct *mm) -{ -} +#define is_hugepage_only_range is_hugepage_only_range +#define __HAVE_ARCH_HUGE_SET_HUGE_PTE_AT +void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, + pte_t pte, unsigned long sz); -static inline void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t pte) -{ - set_pte_at(mm, addr, ptep, pte); -} - +#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm, - unsigned long addr, pte_t *ptep) + unsigned long addr, pte_t *ptep, + unsigned long sz) { -#ifdef CONFIG_PPC64 - return __pte(pte_update(mm, addr, ptep, ~0UL, 1)); -#else - return __pte(pte_update(ptep, ~0UL, 0)); -#endif + return __pte(pte_update(mm, addr, ptep, ~0UL, 0, 1)); } -static inline void huge_ptep_clear_flush(struct vm_area_struct *vma, - unsigned long addr, pte_t *ptep) +#define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH +static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) { pte_t pte; - pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep); - flush_tlb_page(vma, addr); -} - -static inline int huge_pte_none(pte_t pte) -{ - return pte_none(pte); -} - -static inline pte_t huge_pte_wrprotect(pte_t pte) -{ - return pte_wrprotect(pte); -} - -static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma, - unsigned long addr, pte_t *ptep, - pte_t pte, int dirty) -{ -#ifdef HUGETLB_NEED_PRELOAD - /* - * The "return 1" forces a call of update_mmu_cache, which will write a - * TLB entry. Without this, platforms that don't do a write of the TLB - * entry in the TLB miss handler asm will fault ad infinitum. - */ - ptep_set_access_flags(vma, addr, ptep, pte, dirty); - return 1; -#else - return ptep_set_access_flags(vma, addr, ptep, pte, dirty); -#endif -} + unsigned long sz = huge_page_size(hstate_vma(vma)); -static inline pte_t huge_ptep_get(pte_t *ptep) -{ - return *ptep; + pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep, sz); + flush_hugetlb_page(vma, addr); + return pte; } -static inline int arch_prepare_hugepage(struct page *page) -{ - return 0; -} - -static inline void arch_release_hugepage(struct page *page) -{ -} +#define __HAVE_ARCH_HUGE_PTEP_SET_ACCESS_FLAGS +int huge_ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t pte, int dirty); -static inline void arch_clear_hugepage_flags(struct page *page) -{ -} +void gigantic_hugetlb_cma_reserve(void) __init; +#include <asm-generic/hugetlb.h> #else /* ! CONFIG_HUGETLB_PAGE */ static inline void flush_hugetlb_page(struct vm_area_struct *vma, @@ -192,25 +77,13 @@ static inline void flush_hugetlb_page(struct vm_area_struct *vma, { } -#define hugepd_shift(x) 0 -static inline pte_t *hugepte_offset(hugepd_t *hpdp, unsigned long addr, - unsigned pdshift) +static inline void __init gigantic_hugetlb_cma_reserve(void) { - return 0; } -#endif /* CONFIG_HUGETLB_PAGE */ -/* - * FSL Book3E platforms require special gpage handling - the gpages - * are reserved early in the boot process by memblock instead of via - * the .dts as on IBM platforms. - */ -#if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_PPC_FSL_BOOK3E) -extern void __init reserve_hugetlb_gpages(void); -#else -static inline void reserve_hugetlb_gpages(void) +static inline void __init hugetlbpage_init_defaultsize(void) { } -#endif +#endif /* CONFIG_HUGETLB_PAGE */ #endif /* _ASM_POWERPC_HUGETLB_H */ diff --git a/arch/powerpc/include/asm/hvcall.h b/arch/powerpc/include/asm/hvcall.h index 0c7f2bfcf134..9aef16149d92 100644 --- a/arch/powerpc/include/asm/hvcall.h +++ b/arch/powerpc/include/asm/hvcall.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_HVCALL_H #define _ASM_POWERPC_HVCALL_H #ifdef __KERNEL__ @@ -78,6 +79,7 @@ #define H_NOT_ENOUGH_RESOURCES -44 #define H_R_STATE -45 #define H_RESCINDED -46 +#define H_ABORTED -54 #define H_P2 -55 #define H_P3 -56 #define H_P4 -57 @@ -86,7 +88,9 @@ #define H_P7 -60 #define H_P8 -61 #define H_P9 -62 +#define H_NOOP -63 #define H_TOO_BIG -64 +#define H_UNSUPPORTED -67 #define H_OVERLAP -68 #define H_INTERRUPT -69 #define H_BAD_DATA -70 @@ -94,6 +98,20 @@ #define H_SG_LIST -72 #define H_OP_MODE -73 #define H_COP_HW -74 +#define H_STATE -75 +#define H_IN_USE -77 + +#define H_INVALID_ELEMENT_ID -79 +#define H_INVALID_ELEMENT_SIZE -80 +#define H_INVALID_ELEMENT_VALUE -81 +#define H_INPUT_BUFFER_NOT_DEFINED -82 +#define H_INPUT_BUFFER_TOO_SMALL -83 +#define H_OUTPUT_BUFFER_NOT_DEFINED -84 +#define H_OUTPUT_BUFFER_TOO_SMALL -85 +#define H_PARTITION_PAGE_TABLE_NOT_DEFINED -86 +#define H_GUEST_VCPU_STATE_NOT_HV_OWNED -87 + + #define H_UNSUPPORTED_FLAG_START -256 #define H_UNSUPPORTED_FLAG_END -511 #define H_MULTI_THREADS_ACTIVE -9005 @@ -152,6 +170,14 @@ #define H_VASI_RESUMED 5 #define H_VASI_COMPLETED 6 +/* VASI signal codes. Only the Cancel code is valid for H_VASI_SIGNAL. */ +#define H_VASI_SIGNAL_CANCEL 1 +#define H_VASI_SIGNAL_ABORT 2 +#define H_VASI_SIGNAL_SUSPEND 3 +#define H_VASI_SIGNAL_COMPLETE 4 +#define H_VASI_SIGNAL_ENABLE 5 +#define H_VASI_SIGNAL_FAILOVER 6 + /* Each control block has to be on a 4K boundary */ #define H_CB_ALIGNMENT 4096 @@ -234,15 +260,17 @@ #define H_CREATE_RPT 0x1A4 #define H_REMOVE_RPT 0x1A8 #define H_REGISTER_RPAGES 0x1AC -#define H_DISABLE_AND_GETC 0x1B0 +#define H_DISABLE_AND_GET 0x1B0 #define H_ERROR_DATA 0x1B4 #define H_GET_HCA_INFO 0x1B8 #define H_GET_PERF_COUNT 0x1BC #define H_MANAGE_TRACE 0x1C0 +#define H_GET_CPU_CHARACTERISTICS 0x1C8 #define H_FREE_LOGICAL_LAN_BUFFER 0x1D4 #define H_QUERY_INT_STATE 0x1E4 #define H_POLL_PENDING 0x1D8 #define H_ILLAN_ATTRIBUTES 0x244 +#define H_ADD_LOGICAL_LAN_BUFFERS 0x248 #define H_MODIFY_HEA_QP 0x250 #define H_QUERY_HEA_QP 0x254 #define H_QUERY_HEA 0x258 @@ -257,24 +285,257 @@ #define H_ADD_CONN 0x284 #define H_DEL_CONN 0x288 #define H_JOIN 0x298 +#define H_VASI_SIGNAL 0x2A0 #define H_VASI_STATE 0x2A4 +#define H_VIOCTL 0x2A8 #define H_ENABLE_CRQ 0x2B0 #define H_GET_EM_PARMS 0x2B8 #define H_SET_MPP 0x2D0 #define H_GET_MPP 0x2D4 +#define H_REG_SUB_CRQ 0x2DC #define H_HOME_NODE_ASSOCIATIVITY 0x2EC +#define H_FREE_SUB_CRQ 0x2E0 +#define H_SEND_SUB_CRQ 0x2E4 +#define H_SEND_SUB_CRQ_INDIRECT 0x2E8 #define H_BEST_ENERGY 0x2F4 #define H_XIRR_X 0x2FC #define H_RANDOM 0x300 #define H_COP 0x304 #define H_GET_MPP_X 0x314 #define H_SET_MODE 0x31C -#define MAX_HCALL_OPCODE H_SET_MODE +#define H_BLOCK_REMOVE 0x328 +#define H_CLEAR_HPT 0x358 +#define H_REQUEST_VMC 0x360 +#define H_RESIZE_HPT_PREPARE 0x36C +#define H_RESIZE_HPT_COMMIT 0x370 +#define H_REGISTER_PROC_TBL 0x37C +#define H_SIGNAL_SYS_RESET 0x380 +#define H_ALLOCATE_VAS_WINDOW 0x388 +#define H_MODIFY_VAS_WINDOW 0x38C +#define H_DEALLOCATE_VAS_WINDOW 0x390 +#define H_QUERY_VAS_WINDOW 0x394 +#define H_QUERY_VAS_CAPABILITIES 0x398 +#define H_QUERY_NX_CAPABILITIES 0x39C +#define H_GET_NX_FAULT 0x3A0 +#define H_INT_GET_SOURCE_INFO 0x3A8 +#define H_INT_SET_SOURCE_CONFIG 0x3AC +#define H_INT_GET_SOURCE_CONFIG 0x3B0 +#define H_INT_GET_QUEUE_INFO 0x3B4 +#define H_INT_SET_QUEUE_CONFIG 0x3B8 +#define H_INT_GET_QUEUE_CONFIG 0x3BC +#define H_INT_SET_OS_REPORTING_LINE 0x3C0 +#define H_INT_GET_OS_REPORTING_LINE 0x3C4 +#define H_INT_ESB 0x3C8 +#define H_INT_SYNC 0x3CC +#define H_INT_RESET 0x3D0 +#define H_SCM_READ_METADATA 0x3E4 +#define H_SCM_WRITE_METADATA 0x3E8 +#define H_SCM_BIND_MEM 0x3EC +#define H_SCM_UNBIND_MEM 0x3F0 +#define H_SCM_QUERY_BLOCK_MEM_BINDING 0x3F4 +#define H_SCM_QUERY_LOGICAL_MEM_BINDING 0x3F8 +#define H_SCM_UNBIND_ALL 0x3FC +#define H_SCM_HEALTH 0x400 +#define H_SCM_PERFORMANCE_STATS 0x418 +#define H_PKS_GET_CONFIG 0x41C +#define H_PKS_SET_PASSWORD 0x420 +#define H_PKS_GEN_PASSWORD 0x424 +#define H_PKS_WRITE_OBJECT 0x42C +#define H_PKS_GEN_KEY 0x430 +#define H_PKS_READ_OBJECT 0x434 +#define H_PKS_REMOVE_OBJECT 0x438 +#define H_PKS_CONFIRM_OBJECT_FLUSHED 0x43C +#define H_RPT_INVALIDATE 0x448 +#define H_SCM_FLUSH 0x44C +#define H_GET_ENERGY_SCALE_INFO 0x450 +#define H_PKS_SIGNED_UPDATE 0x454 +#define H_HTM 0x458 +#define H_WATCHDOG 0x45C +#define H_GUEST_GET_CAPABILITIES 0x460 +#define H_GUEST_SET_CAPABILITIES 0x464 +#define H_GUEST_CREATE 0x470 +#define H_GUEST_CREATE_VCPU 0x474 +#define H_GUEST_GET_STATE 0x478 +#define H_GUEST_SET_STATE 0x47C +#define H_GUEST_RUN_VCPU 0x480 +#define H_GUEST_COPY_MEMORY 0x484 +#define H_GUEST_DELETE 0x488 +#define MAX_HCALL_OPCODE H_GUEST_DELETE + +/* Scope args for H_SCM_UNBIND_ALL */ +#define H_UNBIND_SCOPE_ALL (0x1) +#define H_UNBIND_SCOPE_DRC (0x2) + +/* H_VIOCTL functions */ +#define H_GET_VIOA_DUMP_SIZE 0x01 +#define H_GET_VIOA_DUMP 0x02 +#define H_GET_ILLAN_NUM_VLAN_IDS 0x03 +#define H_GET_ILLAN_VLAN_ID_LIST 0x04 +#define H_GET_ILLAN_SWITCH_ID 0x05 +#define H_DISABLE_MIGRATION 0x06 +#define H_ENABLE_MIGRATION 0x07 +#define H_GET_PARTNER_INFO 0x08 +#define H_GET_PARTNER_WWPN_LIST 0x09 +#define H_DISABLE_ALL_VIO_INTS 0x0A +#define H_DISABLE_VIO_INTERRUPT 0x0B +#define H_ENABLE_VIO_INTERRUPT 0x0C +#define H_GET_SESSION_TOKEN 0x19 +#define H_SESSION_ERR_DETECTED 0x1A + /* Platform specific hcalls, used by KVM */ #define H_RTAS 0xf000 -#ifndef __ASSEMBLY__ +/* + * Platform specific hcalls, used by QEMU/SLOF. These are ignored by + * KVM and only kept here so we can identify them during tracing. + */ +#define H_LOGICAL_MEMOP 0xF001 +#define H_CAS 0XF002 +#define H_UPDATE_DT 0XF003 + +/* "Platform specific hcalls", provided by PHYP */ +#define H_GET_24X7_CATALOG_PAGE 0xF078 +#define H_GET_24X7_DATA 0xF07C +#define H_GET_PERF_COUNTER_INFO 0xF080 + +/* Platform-specific hcalls used for nested HV KVM */ +#define H_SET_PARTITION_TABLE 0xF800 +#define H_ENTER_NESTED 0xF804 +#define H_TLB_INVALIDATE 0xF808 +#define H_COPY_TOFROM_GUEST 0xF80C + +/* Flags for H_SVM_PAGE_IN */ +#define H_PAGE_IN_SHARED 0x1 + +/* Platform-specific hcalls used by the Ultravisor */ +#define H_SVM_PAGE_IN 0xEF00 +#define H_SVM_PAGE_OUT 0xEF04 +#define H_SVM_INIT_START 0xEF08 +#define H_SVM_INIT_DONE 0xEF0C +#define H_SVM_INIT_ABORT 0xEF14 + +/* Values for 2nd argument to H_SET_MODE */ +#define H_SET_MODE_RESOURCE_SET_CIABR 1 +#define H_SET_MODE_RESOURCE_SET_DAWR0 2 +#define H_SET_MODE_RESOURCE_ADDR_TRANS_MODE 3 +#define H_SET_MODE_RESOURCE_LE 4 +#define H_SET_MODE_RESOURCE_SET_DAWR1 5 + +/* Values for argument to H_SIGNAL_SYS_RESET */ +#define H_SIGNAL_SYS_RESET_ALL -1 +#define H_SIGNAL_SYS_RESET_ALL_OTHERS -2 +/* >= 0 values are CPU number */ + +/* H_GET_CPU_CHARACTERISTICS return values */ +#define H_CPU_CHAR_SPEC_BAR_ORI31 (1ull << 63) // IBM bit 0 +#define H_CPU_CHAR_BCCTRL_SERIALISED (1ull << 62) // IBM bit 1 +#define H_CPU_CHAR_L1D_FLUSH_ORI30 (1ull << 61) // IBM bit 2 +#define H_CPU_CHAR_L1D_FLUSH_TRIG2 (1ull << 60) // IBM bit 3 +#define H_CPU_CHAR_L1D_THREAD_PRIV (1ull << 59) // IBM bit 4 +#define H_CPU_CHAR_BRANCH_HINTS_HONORED (1ull << 58) // IBM bit 5 +#define H_CPU_CHAR_THREAD_RECONFIG_CTRL (1ull << 57) // IBM bit 6 +#define H_CPU_CHAR_COUNT_CACHE_DISABLED (1ull << 56) // IBM bit 7 +#define H_CPU_CHAR_BCCTR_FLUSH_ASSIST (1ull << 54) // IBM bit 9 +#define H_CPU_CHAR_BCCTR_LINK_FLUSH_ASSIST (1ull << 52) // IBM bit 11 + +#define H_CPU_BEHAV_FAVOUR_SECURITY (1ull << 63) // IBM bit 0 +#define H_CPU_BEHAV_L1D_FLUSH_PR (1ull << 62) // IBM bit 1 +#define H_CPU_BEHAV_BNDS_CHK_SPEC_BAR (1ull << 61) // IBM bit 2 +#define H_CPU_BEHAV_FAVOUR_SECURITY_H (1ull << 60) // IBM bit 3 +#define H_CPU_BEHAV_FLUSH_COUNT_CACHE (1ull << 58) // IBM bit 5 +#define H_CPU_BEHAV_FLUSH_LINK_STACK (1ull << 57) // IBM bit 6 +#define H_CPU_BEHAV_NO_L1D_FLUSH_ENTRY (1ull << 56) // IBM bit 7 +#define H_CPU_BEHAV_NO_L1D_FLUSH_UACCESS (1ull << 55) // IBM bit 8 +#define H_CPU_BEHAV_NO_STF_BARRIER (1ull << 54) // IBM bit 9 + +/* Flag values used in H_REGISTER_PROC_TBL hcall */ +#define PROC_TABLE_OP_MASK 0x18 +#define PROC_TABLE_DEREG 0x10 +#define PROC_TABLE_NEW 0x18 +#define PROC_TABLE_TYPE_MASK 0x06 +#define PROC_TABLE_HPT_SLB 0x00 +#define PROC_TABLE_HPT_PT 0x02 +#define PROC_TABLE_RADIX 0x04 +#define PROC_TABLE_GTSE 0x01 + +/* + * Defines for + * H_RPT_INVALIDATE - Invalidate RPT translation lookaside information. + */ + +/* Type of translation to invalidate (type) */ +#define H_RPTI_TYPE_NESTED 0x0001 /* Invalidate nested guest partition-scope */ +#define H_RPTI_TYPE_TLB 0x0002 /* Invalidate TLB */ +#define H_RPTI_TYPE_PWC 0x0004 /* Invalidate Page Walk Cache */ +/* Invalidate caching of Process Table Entries if H_RPTI_TYPE_NESTED is clear */ +#define H_RPTI_TYPE_PRT 0x0008 +/* Invalidate caching of Partition Table Entries if H_RPTI_TYPE_NESTED is set */ +#define H_RPTI_TYPE_PAT 0x0008 +#define H_RPTI_TYPE_ALL (H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | \ + H_RPTI_TYPE_PRT) +#define H_RPTI_TYPE_NESTED_ALL (H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | \ + H_RPTI_TYPE_PAT) + +/* Invalidation targets (target) */ +#define H_RPTI_TARGET_CMMU 0x01 /* All virtual processors in the partition */ +#define H_RPTI_TARGET_CMMU_LOCAL 0x02 /* Current virtual processor */ +/* All nest/accelerator agents in use by the partition */ +#define H_RPTI_TARGET_NMMU 0x04 + +/* Page size mask (page sizes) */ +#define H_RPTI_PAGE_4K 0x01 +#define H_RPTI_PAGE_64K 0x02 +#define H_RPTI_PAGE_2M 0x04 +#define H_RPTI_PAGE_1G 0x08 +#define H_RPTI_PAGE_ALL (-1UL) + +/* Flags for H_GUEST_{S,G}_STATE */ +#define H_GUEST_FLAGS_WIDE (1UL << (63 - 0)) +#define H_GUEST_FLAGS_HOST_WIDE (1UL << (63 - 1)) + +/* Flag values used for H_{S,G}SET_GUEST_CAPABILITIES */ +#define H_GUEST_CAP_COPY_MEM (1UL << (63 - 0)) +#define H_GUEST_CAP_POWER9 (1UL << (63 - 1)) +#define H_GUEST_CAP_POWER10 (1UL << (63 - 2)) +#define H_GUEST_CAP_POWER11 (1UL << (63 - 3)) +#define H_GUEST_CAP_BITMAP2 (1UL << (63 - 63)) + +/* + * Defines for H_HTM - Macros for hardware trace macro (HTM) function. + */ +#define H_HTM_FLAGS_HARDWARE_TARGET (1ul << 63) +#define H_HTM_FLAGS_LOGICAL_TARGET (1ul << 62) +#define H_HTM_FLAGS_PROCID_TARGET (1ul << 61) +#define H_HTM_FLAGS_NOWRAP (1ul << 60) + +#define H_HTM_OP_SHIFT (63-15) +#define H_HTM_OP(x) ((unsigned long)(x)<<H_HTM_OP_SHIFT) +#define H_HTM_OP_CAPABILITIES 0x01 +#define H_HTM_OP_STATUS 0x02 +#define H_HTM_OP_SETUP 0x03 +#define H_HTM_OP_CONFIGURE 0x04 +#define H_HTM_OP_START 0x05 +#define H_HTM_OP_STOP 0x06 +#define H_HTM_OP_DECONFIGURE 0x07 +#define H_HTM_OP_DUMP_DETAILS 0x08 +#define H_HTM_OP_DUMP_DATA 0x09 +#define H_HTM_OP_DUMP_SYSMEM_CONF 0x0a +#define H_HTM_OP_DUMP_SYSPROC_CONF 0x0b + +#define H_HTM_TYPE_SHIFT (63-31) +#define H_HTM_TYPE(x) ((unsigned long)(x)<<H_HTM_TYPE_SHIFT) +#define H_HTM_TYPE_NEST 0x01 +#define H_HTM_TYPE_CORE 0x02 +#define H_HTM_TYPE_LLAT 0x03 +#define H_HTM_TYPE_GLOBAL 0xff + +#define H_HTM_TARGET_NODE_INDEX(x) ((unsigned long)(x)<<(63-15)) +#define H_HTM_TARGET_NODAL_CHIP_INDEX(x) ((unsigned long)(x)<<(63-31)) +#define H_HTM_TARGET_CORE_INDEX_ON_CHIP(x) ((unsigned long)(x)<<(63-47)) + +#ifndef __ASSEMBLER__ +#include <linux/types.h> /** * plpar_hcall_norets: - Make a pseries hypervisor call with no return arguments @@ -286,6 +547,9 @@ */ long plpar_hcall_norets(unsigned long opcode, ...); +/* Variant which does not do hcall tracing */ +long plpar_hcall_norets_notrace(unsigned long opcode, ...); + /** * plpar_hcall: - Make a pseries hypervisor call * @opcode: The hypervisor call to make. @@ -297,7 +561,7 @@ long plpar_hcall_norets(unsigned long opcode, ...); * Used for all but the craziest of phyp interfaces (see plpar_hcall9) */ #define PLPAR_HCALL_BUFSIZE 4 -long plpar_hcall(unsigned long opcode, unsigned long *retbuf, ...); +long plpar_hcall(unsigned long opcode, unsigned long retbuf[static PLPAR_HCALL_BUFSIZE], ...); /** * plpar_hcall_raw: - Make a hypervisor call without calculating hcall stats @@ -311,7 +575,7 @@ long plpar_hcall(unsigned long opcode, unsigned long *retbuf, ...); * plpar_hcall, but plpar_hcall_raw works in real mode and does not * calculate hypervisor call statistics. */ -long plpar_hcall_raw(unsigned long opcode, unsigned long *retbuf, ...); +long plpar_hcall_raw(unsigned long opcode, unsigned long retbuf[static PLPAR_HCALL_BUFSIZE], ...); /** * plpar_hcall9: - Make a pseries hypervisor call with up to 9 return arguments @@ -322,18 +586,13 @@ long plpar_hcall_raw(unsigned long opcode, unsigned long *retbuf, ...); * PLPAR_HCALL9_BUFSIZE to size the return argument buffer. */ #define PLPAR_HCALL9_BUFSIZE 9 -long plpar_hcall9(unsigned long opcode, unsigned long *retbuf, ...); -long plpar_hcall9_raw(unsigned long opcode, unsigned long *retbuf, ...); - -/* For hcall instrumentation. One structure per-hcall, per-CPU */ -struct hcall_stats { - unsigned long num_calls; /* number of calls (on this CPU) */ - unsigned long tb_total; /* total wall time (mftb) of calls. */ - unsigned long purr_total; /* total cpu time (PURR) of calls. */ - unsigned long tb_start; - unsigned long purr_start; -}; -#define HCALL_STAT_ARRAY_SIZE ((MAX_HCALL_OPCODE >> 2) + 1) +long plpar_hcall9(unsigned long opcode, unsigned long retbuf[static PLPAR_HCALL9_BUFSIZE], ...); +long plpar_hcall9_raw(unsigned long opcode, unsigned long retbuf[static PLPAR_HCALL9_BUFSIZE], ...); + +/* pseries hcall tracing */ +extern struct static_key hcall_tracepoint_key; +void __trace_hcall_entry(unsigned long opcode, unsigned long *args); +void __trace_hcall_exit(long opcode, long retval, unsigned long *retbuf); struct hvcall_mpp_data { unsigned long entitled_mem; @@ -348,7 +607,7 @@ struct hvcall_mpp_data { unsigned long backing_mem; }; -int h_get_mpp(struct hvcall_mpp_data *); +long h_get_mpp(struct hvcall_mpp_data *mpp_data); struct hvcall_mpp_x_data { unsigned long coalesced_bytes; @@ -380,36 +639,102 @@ static inline unsigned int get_longbusy_msecs(int longbusy_rc) } } -#ifdef CONFIG_PPC_PSERIES -extern int CMO_PrPSP; -extern int CMO_SecPSP; -extern unsigned long CMO_PageSize; +struct h_cpu_char_result { + u64 character; + u64 behaviour; +}; -static inline int cmo_get_primary_psp(void) -{ - return CMO_PrPSP; -} +/* + * Register state for entering a nested guest with H_ENTER_NESTED. + * New member must be added at the end. + */ +struct hv_guest_state { + u64 version; /* version of this structure layout, must be first */ + u32 lpid; + u32 vcpu_token; + /* These registers are hypervisor privileged (at least for writing) */ + u64 lpcr; + u64 pcr; + u64 amor; + u64 dpdes; + u64 hfscr; + s64 tb_offset; + u64 dawr0; + u64 dawrx0; + u64 ciabr; + u64 hdec_expiry; + u64 purr; + u64 spurr; + u64 ic; + u64 vtb; + u64 hdar; + u64 hdsisr; + u64 heir; + u64 asdr; + /* These are OS privileged but need to be set late in guest entry */ + u64 srr0; + u64 srr1; + u64 sprg[4]; + u64 pidr; + u64 cfar; + u64 ppr; + /* Version 1 ends here */ + u64 dawr1; + u64 dawrx1; + /* Version 2 ends here */ +}; -static inline int cmo_get_secondary_psp(void) -{ - return CMO_SecPSP; -} +/* Latest version of hv_guest_state structure */ +#define HV_GUEST_STATE_VERSION 2 -static inline unsigned long cmo_get_page_size(void) +static inline int hv_guest_state_size(unsigned int version) { - return CMO_PageSize; + switch (version) { + case 1: + return offsetofend(struct hv_guest_state, ppr); + case 2: + return offsetofend(struct hv_guest_state, dawrx1); + default: + return -1; + } } -extern long pSeries_enable_reloc_on_exc(void); -extern long pSeries_disable_reloc_on_exc(void); - -#else - -#define pSeries_enable_reloc_on_exc() do {} while (0) -#define pSeries_disable_reloc_on_exc() do {} while (0) - -#endif /* CONFIG_PPC_PSERIES */ - -#endif /* __ASSEMBLY__ */ +/* + * From the document "H_GetPerformanceCounterInfo Interface" v1.07 + * + * H_GET_PERF_COUNTER_INFO argument + */ +struct hv_get_perf_counter_info_params { + __be32 counter_request; /* I */ + __be32 starting_index; /* IO */ + __be16 secondary_index; /* IO */ + __be16 returned_values; /* O */ + __be32 detail_rc; /* O, only needed when called via *_norets() */ + + /* + * O, size each of counter_value element in bytes, only set for version + * >= 0x3 + */ + __be16 cv_element_size; + + /* I, 0 (zero) for versions < 0x3 */ + __u8 counter_info_version_in; + + /* O, 0 (zero) if version < 0x3. Must be set to 0 when making hcall */ + __u8 counter_info_version_out; + __u8 reserved[0xC]; + __u8 counter_value[]; +} __packed; + +#define HGPCI_REQ_BUFFER_SIZE 4096 +#define HGPCI_MAX_DATA_BYTES \ + (HGPCI_REQ_BUFFER_SIZE - sizeof(struct hv_get_perf_counter_info_params)) + +struct hv_gpci_request_buffer { + struct hv_get_perf_counter_info_params params; + uint8_t bytes[HGPCI_MAX_DATA_BYTES]; +} __packed; + +#endif /* __ASSEMBLER__ */ #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_HVCALL_H */ diff --git a/arch/powerpc/include/asm/hvconsole.h b/arch/powerpc/include/asm/hvconsole.h index 35ea69e8121f..d841a97010a0 100644 --- a/arch/powerpc/include/asm/hvconsole.h +++ b/arch/powerpc/include/asm/hvconsole.h @@ -1,22 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * hvconsole.h * Copyright (C) 2004 Ryan S Arnold, IBM Corporation * * LPAR console support. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef _PPC64_HVCONSOLE_H @@ -34,8 +21,11 @@ * Vio firmware always attempts to fetch MAX_VIO_GET_CHARS chars. The 'count' * parm is included to conform to put_chars() function pointer template */ -extern int hvc_get_chars(uint32_t vtermno, char *buf, int count); -extern int hvc_put_chars(uint32_t vtermno, const char *buf, int count); +extern ssize_t hvc_get_chars(uint32_t vtermno, u8 *buf, size_t count); +extern ssize_t hvc_put_chars(uint32_t vtermno, const u8 *buf, size_t count); + +/* Provided by HVC VIO */ +void hvc_vio_init_early(void); #endif /* __KERNEL__ */ #endif /* _PPC64_HVCONSOLE_H */ diff --git a/arch/powerpc/include/asm/hvcserver.h b/arch/powerpc/include/asm/hvcserver.h index 67d7da3a4da4..2b20403e9fde 100644 --- a/arch/powerpc/include/asm/hvcserver.h +++ b/arch/powerpc/include/asm/hvcserver.h @@ -1,22 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * hvcserver.h * Copyright (C) 2004 Ryan S Arnold, IBM Corporation * * PPC64 virtual I/O console server support. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef _PPC64_HVCSERVER_H diff --git a/arch/powerpc/include/asm/hvsi.h b/arch/powerpc/include/asm/hvsi.h index d3f64f361814..9058edcb632b 100644 --- a/arch/powerpc/include/asm/hvsi.h +++ b/arch/powerpc/include/asm/hvsi.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _HVSI_H #define _HVSI_H @@ -25,7 +26,7 @@ struct hvsi_header { uint8_t type; uint8_t len; - uint16_t seqno; + __be16 seqno; } __attribute__((packed)); struct hvsi_data { @@ -35,24 +36,24 @@ struct hvsi_data { struct hvsi_control { struct hvsi_header hdr; - uint16_t verb; + __be16 verb; /* optional depending on verb: */ - uint32_t word; - uint32_t mask; + __be32 word; + __be32 mask; } __attribute__((packed)); struct hvsi_query { struct hvsi_header hdr; - uint16_t verb; + __be16 verb; } __attribute__((packed)); struct hvsi_query_response { struct hvsi_header hdr; - uint16_t verb; - uint16_t query_seqno; + __be16 verb; + __be16 query_seqno; union { uint8_t version; - uint32_t mctrl_word; + __be32 mctrl_word; } u; } __attribute__((packed)); @@ -63,7 +64,7 @@ struct hvsi_priv { unsigned int inbuf_len; /* data in input buffer */ unsigned char inbuf[HVSI_INBUF_SIZE]; unsigned int inbuf_cur; /* Cursor in input buffer */ - unsigned int inbuf_pktlen; /* packet lenght from cursor */ + size_t inbuf_pktlen; /* packet length from cursor */ atomic_t seqno; /* packet sequence number */ unsigned int opened:1; /* driver opened */ unsigned int established:1; /* protocol established */ @@ -71,24 +72,26 @@ struct hvsi_priv { unsigned int mctrl_update:1; /* modem control updated */ unsigned short mctrl; /* modem control */ struct tty_struct *tty; /* tty structure */ - int (*get_chars)(uint32_t termno, char *buf, int count); - int (*put_chars)(uint32_t termno, const char *buf, int count); + ssize_t (*get_chars)(uint32_t termno, u8 *buf, size_t count); + ssize_t (*put_chars)(uint32_t termno, const u8 *buf, size_t count); uint32_t termno; }; /* hvsi lib functions */ struct hvc_struct; extern void hvsilib_init(struct hvsi_priv *pv, - int (*get_chars)(uint32_t termno, char *buf, int count), - int (*put_chars)(uint32_t termno, const char *buf, - int count), + ssize_t (*get_chars)(uint32_t termno, u8 *buf, + size_t count), + ssize_t (*put_chars)(uint32_t termno, const u8 *buf, + size_t count), int termno, int is_console); extern int hvsilib_open(struct hvsi_priv *pv, struct hvc_struct *hp); extern void hvsilib_close(struct hvsi_priv *pv, struct hvc_struct *hp); extern int hvsilib_read_mctrl(struct hvsi_priv *pv); extern int hvsilib_write_mctrl(struct hvsi_priv *pv, int dtr); extern void hvsilib_establish(struct hvsi_priv *pv); -extern int hvsilib_get_chars(struct hvsi_priv *pv, char *buf, int count); -extern int hvsilib_put_chars(struct hvsi_priv *pv, const char *buf, int count); +extern ssize_t hvsilib_get_chars(struct hvsi_priv *pv, u8 *buf, size_t count); +extern ssize_t hvsilib_put_chars(struct hvsi_priv *pv, const u8 *buf, + size_t count); #endif /* _HVSI_H */ diff --git a/arch/powerpc/include/asm/hw_breakpoint.h b/arch/powerpc/include/asm/hw_breakpoint.h index eb0f4ac75c4c..66db0147d5b4 100644 --- a/arch/powerpc/include/asm/hw_breakpoint.h +++ b/arch/powerpc/include/asm/hw_breakpoint.h @@ -1,36 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * PowerPC BookIII S hardware breakpoint definitions * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * * Copyright 2010, IBM Corporation. * Author: K.Prasad <prasad@linux.vnet.ibm.com> - * */ #ifndef _PPC_BOOK3S_64_HW_BREAKPOINT_H #define _PPC_BOOK3S_64_HW_BREAKPOINT_H +#include <asm/cpu_has_feature.h> + #ifdef __KERNEL__ struct arch_hw_breakpoint { unsigned long address; u16 type; u16 len; /* length of the target data symbol */ + u16 hw_len; /* length programmed in hw */ + u8 flags; + bool perf_single_step; /* temporarily uninstalled for a perf single step */ }; -/* Note: Don't change the the first 6 bits below as they are in the same order +/* Note: Don't change the first 6 bits below as they are in the same order * as the dabr and dabrx. */ #define HW_BRK_TYPE_READ 0x01 @@ -47,21 +38,48 @@ struct arch_hw_breakpoint { #define HW_BRK_TYPE_PRIV_ALL (HW_BRK_TYPE_USER | HW_BRK_TYPE_KERNEL | \ HW_BRK_TYPE_HYP) +#define HW_BRK_FLAG_DISABLED 0x1 + +/* Minimum granularity */ +#ifdef CONFIG_PPC_8xx +#define HW_BREAKPOINT_SIZE 0x4 +#else +#define HW_BREAKPOINT_SIZE 0x8 +#endif +#define HW_BREAKPOINT_SIZE_QUADWORD 0x10 + +#define DABR_MAX_LEN 8 +#define DAWR_MAX_LEN 512 + +static inline int nr_wp_slots(void) +{ + return cpu_has_feature(CPU_FTR_DAWR1) ? 2 : 1; +} + +bool wp_check_constraints(struct pt_regs *regs, ppc_inst_t instr, + unsigned long ea, int type, int size, + struct arch_hw_breakpoint *info); + +void wp_get_instr_detail(struct pt_regs *regs, ppc_inst_t *instr, + int *type, int *size, unsigned long *ea); + #ifdef CONFIG_HAVE_HW_BREAKPOINT #include <linux/kdebug.h> #include <asm/reg.h> #include <asm/debug.h> +struct perf_event_attr; struct perf_event; struct pmu; struct perf_sample_data; - -#define HW_BREAKPOINT_ALIGN 0x7 +struct task_struct; extern int hw_breakpoint_slots(int type); extern int arch_bp_generic_fields(int type, int *gen_bp_type); -extern int arch_check_bp_in_kernelspace(struct perf_event *bp); -extern int arch_validate_hwbkpt_settings(struct perf_event *bp); +extern int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw); +extern int hw_breakpoint_arch_parse(struct perf_event *bp, + const struct perf_event_attr *attr, + struct arch_hw_breakpoint *hw); extern int hw_breakpoint_exceptions_notify(struct notifier_block *unused, unsigned long val, void *data); int arch_install_hw_breakpoint(struct perf_event *bp); @@ -74,19 +92,37 @@ extern void ptrace_triggered(struct perf_event *bp, struct perf_sample_data *data, struct pt_regs *regs); static inline void hw_breakpoint_disable(void) { - struct arch_hw_breakpoint brk; + int i; + struct arch_hw_breakpoint null_brk = {0}; - brk.address = 0; - brk.type = 0; - brk.len = 0; - set_breakpoint(&brk); + if (!ppc_breakpoint_available()) + return; + + for (i = 0; i < nr_wp_slots(); i++) + __set_breakpoint(i, &null_brk); } extern void thread_change_pc(struct task_struct *tsk, struct pt_regs *regs); +int hw_breakpoint_handler(struct die_args *args); #else /* CONFIG_HAVE_HW_BREAKPOINT */ static inline void hw_breakpoint_disable(void) { } static inline void thread_change_pc(struct task_struct *tsk, struct pt_regs *regs) { } + #endif /* CONFIG_HAVE_HW_BREAKPOINT */ + + +#ifdef CONFIG_PPC_DAWR +extern bool dawr_force_enable; +static inline bool dawr_enabled(void) +{ + return dawr_force_enable; +} +int set_dawr(int nr, struct arch_hw_breakpoint *brk); +#else +static inline bool dawr_enabled(void) { return false; } +static inline int set_dawr(int nr, struct arch_hw_breakpoint *brk) { return -1; } +#endif + #endif /* __KERNEL__ */ #endif /* _PPC_BOOK3S_64_HW_BREAKPOINT_H */ diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h index ba713f166fa5..1078ba88efaf 100644 --- a/arch/powerpc/include/asm/hw_irq.h +++ b/arch/powerpc/include/asm/hw_irq.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu> */ @@ -17,69 +18,197 @@ * PACA flags in paca->irq_happened. * * This bits are set when interrupts occur while soft-disabled - * and allow a proper replay. Additionally, PACA_IRQ_HARD_DIS - * is set whenever we manually hard disable. + * and allow a proper replay. + * + * The PACA_IRQ_HARD_DIS is set whenever we hard disable. It is almost + * always in synch with the MSR[EE] state, except: + * - A window in interrupt entry, where hardware disables MSR[EE] and that + * must be "reconciled" with the soft mask state. + * - NMI interrupts that hit in awkward places, until they fix the state. + * - When local irqs are being enabled and state is being fixed up. + * - When returning from an interrupt there are some windows where this + * can become out of synch, but gets fixed before the RFI or before + * executing the next user instruction (see arch/powerpc/kernel/interrupt.c). */ #define PACA_IRQ_HARD_DIS 0x01 #define PACA_IRQ_DBELL 0x02 #define PACA_IRQ_EE 0x04 #define PACA_IRQ_DEC 0x08 /* Or FIT */ -#define PACA_IRQ_EE_EDGE 0x10 /* BookE only */ +#define PACA_IRQ_HMI 0x10 +#define PACA_IRQ_PMI 0x20 +#define PACA_IRQ_REPLAYING 0x40 + +/* + * Some soft-masked interrupts must be hard masked until they are replayed + * (e.g., because the soft-masked handler does not clear the exception). + * Interrupt replay itself must remain hard masked too. + */ +#ifdef CONFIG_PPC_BOOK3S +#define PACA_IRQ_MUST_HARD_MASK (PACA_IRQ_EE|PACA_IRQ_PMI|PACA_IRQ_REPLAYING) +#else +#define PACA_IRQ_MUST_HARD_MASK (PACA_IRQ_EE|PACA_IRQ_REPLAYING) +#endif #endif /* CONFIG_PPC64 */ -#ifndef __ASSEMBLY__ +/* + * flags for paca->irq_soft_mask + */ +#define IRQS_ENABLED 0 +#define IRQS_DISABLED 1 /* local_irq_disable() interrupts */ +#define IRQS_PMI_DISABLED 2 +#define IRQS_ALL_DISABLED (IRQS_DISABLED | IRQS_PMI_DISABLED) + +#ifndef __ASSEMBLER__ + +static inline void __hard_irq_enable(void) +{ + if (IS_ENABLED(CONFIG_BOOKE)) + wrtee(MSR_EE); + else if (IS_ENABLED(CONFIG_PPC_8xx)) + wrtspr(SPRN_EIE); + else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) + __mtmsrd(MSR_EE | MSR_RI, 1); + else + mtmsr(mfmsr() | MSR_EE); +} + +static inline void __hard_irq_disable(void) +{ + if (IS_ENABLED(CONFIG_BOOKE)) + wrtee(0); + else if (IS_ENABLED(CONFIG_PPC_8xx)) + wrtspr(SPRN_EID); + else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) + __mtmsrd(MSR_RI, 1); + else + mtmsr(mfmsr() & ~MSR_EE); +} -extern void __replay_interrupt(unsigned int vector); +static inline void __hard_EE_RI_disable(void) +{ + if (IS_ENABLED(CONFIG_BOOKE)) + wrtee(0); + else if (IS_ENABLED(CONFIG_PPC_8xx)) + wrtspr(SPRN_NRI); + else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) + __mtmsrd(0, 1); + else + mtmsr(mfmsr() & ~(MSR_EE | MSR_RI)); +} -extern void timer_interrupt(struct pt_regs *); -extern void performance_monitor_exception(struct pt_regs *regs); -extern void WatchdogException(struct pt_regs *regs); -extern void unknown_exception(struct pt_regs *regs); +static inline void __hard_RI_enable(void) +{ + if (IS_ENABLED(CONFIG_BOOKE)) + return; + + if (IS_ENABLED(CONFIG_PPC_8xx)) + wrtspr(SPRN_EID); + else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) + __mtmsrd(MSR_RI, 1); + else + mtmsr(mfmsr() | MSR_RI); +} #ifdef CONFIG_PPC64 #include <asm/paca.h> -static inline unsigned long arch_local_save_flags(void) +static inline notrace unsigned long irq_soft_mask_return(void) { unsigned long flags; asm volatile( "lbz %0,%1(13)" : "=r" (flags) - : "i" (offsetof(struct paca_struct, soft_enabled))); + : "i" (offsetof(struct paca_struct, irq_soft_mask))); return flags; } -static inline unsigned long arch_local_irq_disable(void) +/* + * The "memory" clobber acts as both a compiler barrier + * for the critical section and as a clobber because + * we changed paca->irq_soft_mask + */ +static inline notrace void irq_soft_mask_set(unsigned long mask) { - unsigned long flags, zero; + /* + * The irq mask must always include the STD bit if any are set. + * + * and interrupts don't get replayed until the standard + * interrupt (local_irq_disable()) is unmasked. + * + * Other masks must only provide additional masking beyond + * the standard, and they are also not replayed until the + * standard interrupt becomes unmasked. + * + * This could be changed, but it will require partial + * unmasks to be replayed, among other things. For now, take + * the simple approach. + */ + if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) + WARN_ON(mask && !(mask & IRQS_DISABLED)); asm volatile( - "li %1,0; lbz %0,%2(13); stb %1,%2(13)" - : "=r" (flags), "=&r" (zero) - : "i" (offsetof(struct paca_struct, soft_enabled)) + "stb %0,%1(13)" + : + : "r" (mask), + "i" (offsetof(struct paca_struct, irq_soft_mask)) : "memory"); +} + +static inline notrace unsigned long irq_soft_mask_set_return(unsigned long mask) +{ + unsigned long flags = irq_soft_mask_return(); + + irq_soft_mask_set(mask); return flags; } +static inline notrace unsigned long irq_soft_mask_or_return(unsigned long mask) +{ + unsigned long flags = irq_soft_mask_return(); + + irq_soft_mask_set(flags | mask); + + return flags; +} + +static inline notrace unsigned long irq_soft_mask_andc_return(unsigned long mask) +{ + unsigned long flags = irq_soft_mask_return(); + + irq_soft_mask_set(flags & ~mask); + + return flags; +} + +static inline unsigned long arch_local_save_flags(void) +{ + return irq_soft_mask_return(); +} + +static inline void arch_local_irq_disable(void) +{ + irq_soft_mask_set(IRQS_DISABLED); +} + extern void arch_local_irq_restore(unsigned long); static inline void arch_local_irq_enable(void) { - arch_local_irq_restore(1); + arch_local_irq_restore(IRQS_ENABLED); } static inline unsigned long arch_local_irq_save(void) { - return arch_local_irq_disable(); + return irq_soft_mask_or_return(IRQS_DISABLED); } static inline bool arch_irqs_disabled_flags(unsigned long flags) { - return flags == 0; + return flags & IRQS_DISABLED; } static inline bool arch_irqs_disabled(void) @@ -87,50 +216,206 @@ static inline bool arch_irqs_disabled(void) return arch_irqs_disabled_flags(arch_local_save_flags()); } -#ifdef CONFIG_PPC_BOOK3E -#define __hard_irq_enable() asm volatile("wrteei 1" : : : "memory") -#define __hard_irq_disable() asm volatile("wrteei 0" : : : "memory") -#else -#define __hard_irq_enable() __mtmsrd(local_paca->kernel_msr | MSR_EE, 1) -#define __hard_irq_disable() __mtmsrd(local_paca->kernel_msr, 1) -#endif +static inline void set_pmi_irq_pending(void) +{ + /* + * Invoked from PMU callback functions to set PMI bit in the paca. + * This has to be called with irq's disabled (via hard_irq_disable()). + */ + if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) + WARN_ON_ONCE(mfmsr() & MSR_EE); + + get_paca()->irq_happened |= PACA_IRQ_PMI; +} + +static inline void clear_pmi_irq_pending(void) +{ + /* + * Invoked from PMU callback functions to clear the pending PMI bit + * in the paca. + */ + if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) + WARN_ON_ONCE(mfmsr() & MSR_EE); + + get_paca()->irq_happened &= ~PACA_IRQ_PMI; +} -#define hard_irq_disable() do { \ - u8 _was_enabled = get_paca()->soft_enabled; \ - __hard_irq_disable(); \ - get_paca()->soft_enabled = 0; \ - get_paca()->irq_happened |= PACA_IRQ_HARD_DIS; \ - if (_was_enabled) \ - trace_hardirqs_off(); \ +static inline bool pmi_irq_pending(void) +{ + /* + * Invoked from PMU callback functions to check if there is a pending + * PMI bit in the paca. + */ + if (get_paca()->irq_happened & PACA_IRQ_PMI) + return true; + + return false; +} + +#ifdef CONFIG_PPC_BOOK3S +/* + * To support disabling and enabling of irq with PMI, set of + * new powerpc_local_irq_pmu_save() and powerpc_local_irq_restore() + * functions are added. These macros are implemented using generic + * linux local_irq_* code from include/linux/irqflags.h. + */ +#define raw_local_irq_pmu_save(flags) \ + do { \ + typecheck(unsigned long, flags); \ + flags = irq_soft_mask_or_return(IRQS_DISABLED | \ + IRQS_PMI_DISABLED); \ + } while(0) + +#define raw_local_irq_pmu_restore(flags) \ + do { \ + typecheck(unsigned long, flags); \ + arch_local_irq_restore(flags); \ + } while(0) + +#ifdef CONFIG_TRACE_IRQFLAGS +#define powerpc_local_irq_pmu_save(flags) \ + do { \ + raw_local_irq_pmu_save(flags); \ + if (!raw_irqs_disabled_flags(flags)) \ + trace_hardirqs_off(); \ + } while(0) +#define powerpc_local_irq_pmu_restore(flags) \ + do { \ + if (!raw_irqs_disabled_flags(flags)) \ + trace_hardirqs_on(); \ + raw_local_irq_pmu_restore(flags); \ + } while(0) +#else +#define powerpc_local_irq_pmu_save(flags) \ + do { \ + raw_local_irq_pmu_save(flags); \ + } while(0) +#define powerpc_local_irq_pmu_restore(flags) \ + do { \ + raw_local_irq_pmu_restore(flags); \ + } while (0) +#endif /* CONFIG_TRACE_IRQFLAGS */ + +#endif /* CONFIG_PPC_BOOK3S */ + +#define hard_irq_disable() do { \ + unsigned long flags; \ + __hard_irq_disable(); \ + flags = irq_soft_mask_set_return(IRQS_ALL_DISABLED); \ + local_paca->irq_happened |= PACA_IRQ_HARD_DIS; \ + if (!arch_irqs_disabled_flags(flags)) { \ + asm volatile("std%X0 %1,%0" : "=m" (local_paca->saved_r1) \ + : "r" (current_stack_pointer)); \ + trace_hardirqs_off(); \ + } \ } while(0) +static inline bool __lazy_irq_pending(u8 irq_happened) +{ + return !!(irq_happened & ~PACA_IRQ_HARD_DIS); +} + +/* + * Check if a lazy IRQ is pending. Should be called with IRQs hard disabled. + */ static inline bool lazy_irq_pending(void) { - return !!(get_paca()->irq_happened & ~PACA_IRQ_HARD_DIS); + return __lazy_irq_pending(get_paca()->irq_happened); } /* - * This is called by asynchronous interrupts to conditionally - * re-enable hard interrupts when soft-disabled after having - * cleared the source of the interrupt + * Check if a lazy IRQ is pending, with no debugging checks. + * Should be called with IRQs hard disabled. + * For use in RI disabled code or other constrained situations. */ -static inline void may_hard_irq_enable(void) +static inline bool lazy_irq_pending_nocheck(void) { + return __lazy_irq_pending(local_paca->irq_happened); +} + +bool power_pmu_wants_prompt_pmi(void); + +/* + * This is called by asynchronous interrupts to check whether to + * conditionally re-enable hard interrupts after having cleared + * the source of the interrupt. They are kept disabled if there + * is a different soft-masked interrupt pending that requires hard + * masking. + */ +static inline bool should_hard_irq_enable(struct pt_regs *regs) +{ + if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) { + WARN_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED); + WARN_ON(!(get_paca()->irq_happened & PACA_IRQ_HARD_DIS)); + WARN_ON(mfmsr() & MSR_EE); + } + + if (!IS_ENABLED(CONFIG_PERF_EVENTS)) + return false; + /* + * If the PMU is not running, there is not much reason to enable + * MSR[EE] in irq handlers because any interrupts would just be + * soft-masked. + * + * TODO: Add test for 64e + */ + if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) { + if (!power_pmu_wants_prompt_pmi()) + return false; + /* + * If PMIs are disabled then IRQs should be disabled as well, + * so we shouldn't see this condition, check for it just in + * case because we are about to enable PMIs. + */ + if (WARN_ON_ONCE(regs->softe & IRQS_PMI_DISABLED)) + return false; + } + + if (get_paca()->irq_happened & PACA_IRQ_MUST_HARD_MASK) + return false; + + return true; +} + +/* + * Do the hard enabling, only call this if should_hard_irq_enable is true. + * This allows PMI interrupts to profile irq handlers. + */ +static inline void do_hard_irq_enable(void) +{ + /* + * Asynch interrupts come in with IRQS_ALL_DISABLED, + * PACA_IRQ_HARD_DIS, and MSR[EE]=0. + */ + if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) + irq_soft_mask_andc_return(IRQS_PMI_DISABLED); get_paca()->irq_happened &= ~PACA_IRQ_HARD_DIS; - if (!(get_paca()->irq_happened & PACA_IRQ_EE)) - __hard_irq_enable(); + __hard_irq_enable(); } static inline bool arch_irq_disabled_regs(struct pt_regs *regs) { - return !regs->softe; + return (regs->softe & IRQS_DISABLED); } extern bool prep_irq_for_idle(void); +extern bool prep_irq_for_idle_irqsoff(void); +extern void irq_set_pending_from_srr1(unsigned long srr1); + +#define fini_irq_for_idle_irqsoff() trace_hardirqs_off(); +extern void force_external_irq_replay(void); + +static inline void irq_soft_mask_regs_set_state(struct pt_regs *regs, unsigned long val) +{ + regs->softe = val; +} #else /* CONFIG_PPC64 */ -#define SET_MSR_EE(x) mtmsr(x) +static inline notrace unsigned long irq_soft_mask_return(void) +{ + return 0; +} static inline unsigned long arch_local_save_flags(void) { @@ -139,41 +424,34 @@ static inline unsigned long arch_local_save_flags(void) static inline void arch_local_irq_restore(unsigned long flags) { -#if defined(CONFIG_BOOKE) - asm volatile("wrtee %0" : : "r" (flags) : "memory"); -#else - mtmsr(flags); -#endif + if (IS_ENABLED(CONFIG_BOOKE)) + wrtee(flags); + else + mtmsr(flags); } static inline unsigned long arch_local_irq_save(void) { unsigned long flags = arch_local_save_flags(); -#ifdef CONFIG_BOOKE - asm volatile("wrteei 0" : : : "memory"); -#else - SET_MSR_EE(flags & ~MSR_EE); -#endif + + if (IS_ENABLED(CONFIG_BOOKE)) + wrtee(0); + else if (IS_ENABLED(CONFIG_PPC_8xx)) + wrtspr(SPRN_EID); + else + mtmsr(flags & ~MSR_EE); + return flags; } static inline void arch_local_irq_disable(void) { -#ifdef CONFIG_BOOKE - asm volatile("wrteei 0" : : : "memory"); -#else - arch_local_irq_save(); -#endif + __hard_irq_disable(); } static inline void arch_local_irq_enable(void) { -#ifdef CONFIG_BOOKE - asm volatile("wrteei 1" : : : "memory"); -#else - unsigned long msr = mfmsr(); - SET_MSR_EE(msr | MSR_EE); -#endif + __hard_irq_enable(); } static inline bool arch_irqs_disabled_flags(unsigned long flags) @@ -193,18 +471,51 @@ static inline bool arch_irq_disabled_regs(struct pt_regs *regs) return !(regs->msr & MSR_EE); } -static inline void may_hard_irq_enable(void) { } +static __always_inline bool should_hard_irq_enable(struct pt_regs *regs) +{ + return false; +} + +static inline void do_hard_irq_enable(void) +{ + BUILD_BUG(); +} +static inline void clear_pmi_irq_pending(void) { } +static inline void set_pmi_irq_pending(void) { } +static inline bool pmi_irq_pending(void) { return false; } + +static inline void irq_soft_mask_regs_set_state(struct pt_regs *regs, unsigned long val) +{ +} #endif /* CONFIG_PPC64 */ -#define ARCH_IRQ_INIT_FLAGS IRQ_NOREQUEST +static inline unsigned long mtmsr_isync_irqsafe(unsigned long msr) +{ +#ifdef CONFIG_PPC64 + if (arch_irqs_disabled()) { + /* + * With soft-masking, MSR[EE] can change from 1 to 0 + * asynchronously when irqs are disabled, and we don't want to + * set MSR[EE] back to 1 here if that has happened. A race-free + * way to do this is ensure EE is already 0. Another way it + * could be done is with a RESTART_TABLE handler, but that's + * probably overkill here. + */ + msr &= ~MSR_EE; + mtmsr_isync(msr); + irq_soft_mask_set(IRQS_ALL_DISABLED); + local_paca->irq_happened |= PACA_IRQ_HARD_DIS; + } else +#endif + mtmsr_isync(msr); -/* - * interrupt-retrigger: should we handle this via lost interrupts and IPIs - * or should we not care like we do now ? --BenH. - */ -struct irq_chip; + return msr; +} + + +#define ARCH_IRQ_INIT_FLAGS IRQ_NOREQUEST -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_HW_IRQ_H */ diff --git a/arch/powerpc/include/asm/hydra.h b/arch/powerpc/include/asm/hydra.h index 5b0c98bd46ab..d024447283a0 100644 --- a/arch/powerpc/include/asm/hydra.h +++ b/arch/powerpc/include/asm/hydra.h @@ -10,7 +10,7 @@ * * © Copyright 1995 Apple Computer, Inc. All rights reserved. * - * It's available online from http://www.cpu.lu/~mlan/ftp/MacTech.pdf + * It's available online from https://www.cpu.lu/~mlan/ftp/MacTech.pdf * You can obtain paper copies of this book from computer bookstores or by * writing Morgan Kaufmann Publishers, Inc., 340 Pine Street, Sixth Floor, San * Francisco, CA 94104. Reference ISBN 1-55860-393-X. @@ -89,14 +89,11 @@ extern volatile struct Hydra __iomem *Hydra; #define HYDRA_INT_EXT2 13 /* PCI IRQX */ #define HYDRA_INT_EXT3 14 /* PCI IRQY */ #define HYDRA_INT_EXT4 15 /* PCI IRQZ */ -#define HYDRA_INT_EXT5 16 /* IDE Primay/Secondary */ +#define HYDRA_INT_EXT5 16 /* IDE Primary/Secondary */ #define HYDRA_INT_EXT6 17 /* IDE Secondary */ #define HYDRA_INT_EXT7 18 /* Power Off Request */ #define HYDRA_INT_SPARE 19 -extern int hydra_init(void); -extern void macio_adb_init(void); - #endif /* __KERNEL__ */ #endif /* _ASMPPC_HYDRA_H */ diff --git a/arch/powerpc/include/asm/i8259.h b/arch/powerpc/include/asm/i8259.h index c3fdfbd5a673..75481d363cd8 100644 --- a/arch/powerpc/include/asm/i8259.h +++ b/arch/powerpc/include/asm/i8259.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_I8259_H #define _ASM_POWERPC_I8259_H #ifdef __KERNEL__ @@ -6,7 +7,7 @@ extern void i8259_init(struct device_node *node, unsigned long intack_addr); extern unsigned int i8259_irq(void); -extern struct irq_domain *i8259_get_host(void); +struct irq_domain *__init i8259_get_host(void); #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_I8259_H */ diff --git a/arch/powerpc/include/asm/ibmebus.h b/arch/powerpc/include/asm/ibmebus.h index 088f95b2e14f..46fe406f461c 100644 --- a/arch/powerpc/include/asm/ibmebus.h +++ b/arch/powerpc/include/asm/ibmebus.h @@ -46,7 +46,9 @@ #include <linux/of_device.h> #include <linux/of_platform.h> -extern struct bus_type ibmebus_bus_type; +struct platform_driver; + +extern const struct bus_type ibmebus_bus_type; int ibmebus_register_driver(struct platform_driver *drv); void ibmebus_unregister_driver(struct platform_driver *drv); diff --git a/arch/powerpc/include/asm/icswx.h b/arch/powerpc/include/asm/icswx.h new file mode 100644 index 000000000000..f6599ccb3012 --- /dev/null +++ b/arch/powerpc/include/asm/icswx.h @@ -0,0 +1,204 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * ICSWX api + * + * Copyright (C) 2015 IBM Corp. + * + * This provides the Initiate Coprocessor Store Word Indexed (ICSWX) + * instruction. This instruction is used to communicate with PowerPC + * coprocessors. This also provides definitions of the structures used + * to communicate with the coprocessor. + * + * The RFC02130: Coprocessor Architecture document is the reference for + * everything in this file unless otherwise noted. + */ +#ifndef _ARCH_POWERPC_INCLUDE_ASM_ICSWX_H_ +#define _ARCH_POWERPC_INCLUDE_ASM_ICSWX_H_ + +#include <asm/ppc-opcode.h> /* for PPC_ICSWX */ + +/* Chapter 6.5.8 Coprocessor-Completion Block (CCB) */ + +#define CCB_VALUE (0x3fffffffffffffff) +#define CCB_ADDRESS (0xfffffffffffffff8) +#define CCB_CM (0x0000000000000007) +#define CCB_CM0 (0x0000000000000004) +#define CCB_CM12 (0x0000000000000003) + +#define CCB_CM0_ALL_COMPLETIONS (0x0) +#define CCB_CM0_LAST_IN_CHAIN (0x4) +#define CCB_CM12_STORE (0x0) +#define CCB_CM12_INTERRUPT (0x1) + +#define CCB_SIZE (0x10) +#define CCB_ALIGN CCB_SIZE + +struct coprocessor_completion_block { + __be64 value; + __be64 address; +} __packed __aligned(CCB_ALIGN); + + +/* Chapter 6.5.7 Coprocessor-Status Block (CSB) */ + +#define CSB_V (0x80) +#define CSB_F (0x04) +#define CSB_CH (0x03) +#define CSB_CE_INCOMPLETE (0x80) +#define CSB_CE_TERMINATION (0x40) +#define CSB_CE_TPBC (0x20) + +#define CSB_CC_SUCCESS (0) +#define CSB_CC_INVALID_ALIGN (1) +#define CSB_CC_OPERAND_OVERLAP (2) +#define CSB_CC_DATA_LENGTH (3) +#define CSB_CC_TRANSLATION (5) +#define CSB_CC_PROTECTION (6) +#define CSB_CC_RD_EXTERNAL (7) +#define CSB_CC_INVALID_OPERAND (8) +#define CSB_CC_PRIVILEGE (9) +#define CSB_CC_INTERNAL (10) +#define CSB_CC_WR_EXTERNAL (12) +#define CSB_CC_NOSPC (13) +#define CSB_CC_EXCESSIVE_DDE (14) +#define CSB_CC_WR_TRANSLATION (15) +#define CSB_CC_WR_PROTECTION (16) +#define CSB_CC_UNKNOWN_CODE (17) +#define CSB_CC_ABORT (18) +#define CSB_CC_EXCEED_BYTE_COUNT (19) /* P9 or later */ +#define CSB_CC_TRANSPORT (20) +#define CSB_CC_INVALID_CRB (21) /* P9 or later */ +#define CSB_CC_INVALID_DDE (30) /* P9 or later */ +#define CSB_CC_SEGMENTED_DDL (31) +#define CSB_CC_PROGRESS_POINT (32) +#define CSB_CC_DDE_OVERFLOW (33) +#define CSB_CC_SESSION (34) +#define CSB_CC_PROVISION (36) +#define CSB_CC_CHAIN (37) +#define CSB_CC_SEQUENCE (38) +#define CSB_CC_HW (39) +/* P9 DD2 NX Workbook 3.2 (Table 4-36): Address translation fault */ +#define CSB_CC_FAULT_ADDRESS (250) + +#define CSB_SIZE (0x10) +#define CSB_ALIGN CSB_SIZE + +struct coprocessor_status_block { + u8 flags; + u8 cs; + u8 cc; + u8 ce; + __be32 count; + __be64 address; +} __packed __aligned(CSB_ALIGN); + + +/* Chapter 6.5.10 Data-Descriptor List (DDL) + * each list contains one or more Data-Descriptor Entries (DDE) + */ + +#define DDE_P (0x8000) + +#define DDE_SIZE (0x10) +#define DDE_ALIGN DDE_SIZE + +struct data_descriptor_entry { + __be16 flags; + u8 count; + u8 index; + __be32 length; + __be64 address; +} __packed __aligned(DDE_ALIGN); + +/* 4.3.2 NX-stamped Fault CRB */ + +#define NX_STAMP_ALIGN (0x10) + +struct nx_fault_stamp { + __be64 fault_storage_addr; + __be16 reserved; + __u8 flags; + __u8 fault_status; + __be32 pswid; +} __packed __aligned(NX_STAMP_ALIGN); + +/* Chapter 6.5.2 Coprocessor-Request Block (CRB) */ + +#define CRB_SIZE (0x80) +#define CRB_ALIGN (0x100) /* Errata: requires 256 alignment */ + +/* Coprocessor Status Block field + * ADDRESS address of CSB + * C CCB is valid + * AT 0 = addrs are virtual, 1 = addrs are phys + * M enable perf monitor + */ +#define CRB_CSB_ADDRESS (0xfffffffffffffff0) +#define CRB_CSB_C (0x0000000000000008) +#define CRB_CSB_AT (0x0000000000000002) +#define CRB_CSB_M (0x0000000000000001) + +struct coprocessor_request_block { + __be32 ccw; + __be32 flags; + __be64 csb_addr; + + struct data_descriptor_entry source; + struct data_descriptor_entry target; + + struct coprocessor_completion_block ccb; + + union { + struct nx_fault_stamp nx; + u8 reserved[16]; + } stamp; + + u8 reserved[32]; + + struct coprocessor_status_block csb; +} __aligned(128); + +/* RFC02167 Initiate Coprocessor Instructions document + * Chapter 8.2.1.1.1 RS + * Chapter 8.2.3 Coprocessor Directive + * Chapter 8.2.4 Execution + * + * The CCW must be converted to BE before passing to icswx() + */ + +#define CCW_PS (0xff000000) +#define CCW_CT (0x00ff0000) +#define CCW_CD (0x0000ffff) +#define CCW_CL (0x0000c000) + + +/* RFC02167 Initiate Coprocessor Instructions document + * Chapter 8.2.1 Initiate Coprocessor Store Word Indexed (ICSWX) + * Chapter 8.2.4.1 Condition Register 0 + */ + +#define ICSWX_INITIATED (0x8) +#define ICSWX_BUSY (0x4) +#define ICSWX_REJECTED (0x2) +#define ICSWX_XERS0 (0x1) /* undefined or set from XERSO. */ + +static inline int icswx(__be32 ccw, struct coprocessor_request_block *crb) +{ + __be64 ccw_reg = ccw; + u32 cr; + + /* NB: the same structures are used by VAS-NX */ + BUILD_BUG_ON(sizeof(*crb) != 128); + + __asm__ __volatile__( + PPC_ICSWX(%1,0,%2) "\n" + "mfcr %0\n" + : "=r" (cr) + : "r" (ccw_reg), "r" (crb) + : "cr0", "memory"); + + return (int)((cr >> 28) & 0xf); +} + + +#endif /* _ARCH_POWERPC_INCLUDE_ASM_ICSWX_H_ */ diff --git a/arch/powerpc/include/asm/ide.h b/arch/powerpc/include/asm/ide.h deleted file mode 100644 index da01b20aea59..000000000000 --- a/arch/powerpc/include/asm/ide.h +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (C) 1994-1996 Linus Torvalds & authors - * - * This file contains the powerpc architecture specific IDE code. - */ -#ifndef _ASM_POWERPC_IDE_H -#define _ASM_POWERPC_IDE_H - -#include <linux/compiler.h> -#include <asm/io.h> - -#define __ide_mm_insw(p, a, c) readsw((void __iomem *)(p), (a), (c)) -#define __ide_mm_insl(p, a, c) readsl((void __iomem *)(p), (a), (c)) -#define __ide_mm_outsw(p, a, c) writesw((void __iomem *)(p), (a), (c)) -#define __ide_mm_outsl(p, a, c) writesl((void __iomem *)(p), (a), (c)) - -#endif /* _ASM_POWERPC_IDE_H */ diff --git a/arch/powerpc/include/asm/idle.h b/arch/powerpc/include/asm/idle.h new file mode 100644 index 000000000000..00f360667391 --- /dev/null +++ b/arch/powerpc/include/asm/idle.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_POWERPC_IDLE_H +#define _ASM_POWERPC_IDLE_H +#include <asm/runlatch.h> +#include <asm/paca.h> + +#ifdef CONFIG_PPC_PSERIES +DECLARE_PER_CPU(u64, idle_spurr_cycles); +DECLARE_PER_CPU(u64, idle_entry_purr_snap); +DECLARE_PER_CPU(u64, idle_entry_spurr_snap); + +static __always_inline void snapshot_purr_idle_entry(void) +{ + *this_cpu_ptr(&idle_entry_purr_snap) = mfspr(SPRN_PURR); +} + +static __always_inline void snapshot_spurr_idle_entry(void) +{ + *this_cpu_ptr(&idle_entry_spurr_snap) = mfspr(SPRN_SPURR); +} + +static __always_inline void update_idle_purr_accounting(void) +{ + u64 wait_cycles; + u64 in_purr = *this_cpu_ptr(&idle_entry_purr_snap); + + wait_cycles = be64_to_cpu(get_lppaca()->wait_state_cycles); + wait_cycles += mfspr(SPRN_PURR) - in_purr; + get_lppaca()->wait_state_cycles = cpu_to_be64(wait_cycles); +} + +static __always_inline void update_idle_spurr_accounting(void) +{ + u64 *idle_spurr_cycles_ptr = this_cpu_ptr(&idle_spurr_cycles); + u64 in_spurr = *this_cpu_ptr(&idle_entry_spurr_snap); + + *idle_spurr_cycles_ptr += mfspr(SPRN_SPURR) - in_spurr; +} + +static __always_inline void pseries_idle_prolog(void) +{ + ppc64_runlatch_off(); + snapshot_purr_idle_entry(); + snapshot_spurr_idle_entry(); + /* + * Indicate to the HV that we are idle. Now would be + * a good time to find other work to dispatch. + */ + get_lppaca()->idle = 1; +} + +static __always_inline void pseries_idle_epilog(void) +{ + update_idle_purr_accounting(); + update_idle_spurr_accounting(); + get_lppaca()->idle = 0; + ppc64_runlatch_on(); +} + +static inline u64 read_this_idle_purr(void) +{ + /* + * If we are reading from an idle context, update the + * idle-purr cycles corresponding to the last idle period. + * Since the idle context is not yet over, take a fresh + * snapshot of the idle-purr. + */ + if (unlikely(get_lppaca()->idle == 1)) { + update_idle_purr_accounting(); + snapshot_purr_idle_entry(); + } + + return be64_to_cpu(get_lppaca()->wait_state_cycles); +} + +static inline u64 read_this_idle_spurr(void) +{ + /* + * If we are reading from an idle context, update the + * idle-spurr cycles corresponding to the last idle period. + * Since the idle context is not yet over, take a fresh + * snapshot of the idle-spurr. + */ + if (get_lppaca()->idle == 1) { + update_idle_spurr_accounting(); + snapshot_spurr_idle_entry(); + } + + return *this_cpu_ptr(&idle_spurr_cycles); +} + +#endif /* CONFIG_PPC_PSERIES */ +#endif diff --git a/arch/powerpc/include/asm/imc-pmu.h b/arch/powerpc/include/asm/imc-pmu.h new file mode 100644 index 000000000000..a656635df386 --- /dev/null +++ b/arch/powerpc/include/asm/imc-pmu.h @@ -0,0 +1,172 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef __ASM_POWERPC_IMC_PMU_H +#define __ASM_POWERPC_IMC_PMU_H + +/* + * IMC Nest Performance Monitor counter support. + * + * Copyright (C) 2017 Madhavan Srinivasan, IBM Corporation. + * (C) 2017 Anju T Sudhakar, IBM Corporation. + * (C) 2017 Hemant K Shaw, IBM Corporation. + */ + +#include <linux/perf_event.h> +#include <linux/slab.h> +#include <linux/of.h> +#include <linux/io.h> +#include <asm/opal.h> + +/* + * Compatibility macros for IMC devices + */ +#define IMC_DTB_COMPAT "ibm,opal-in-memory-counters" +#define IMC_DTB_UNIT_COMPAT "ibm,imc-counters" + + +/* + * LDBAR: Counter address and Enable/Disable macro. + * perf/imc-pmu.c has the LDBAR layout information. + */ +#define THREAD_IMC_LDBAR_MASK 0x0003ffffffffe000ULL +#define THREAD_IMC_ENABLE 0x8000000000000000ULL +#define TRACE_IMC_ENABLE 0x4000000000000000ULL + +/* + * For debugfs interface for imc-mode and imc-command + */ +#define IMC_CNTL_BLK_OFFSET 0x3FC00 +#define IMC_CNTL_BLK_CMD_OFFSET 8 +#define IMC_CNTL_BLK_MODE_OFFSET 32 + +/* + * Structure to hold memory address information for imc units. + */ +struct imc_mem_info { + u64 *vbase; + u32 id; +}; + +/* + * Place holder for nest pmu events and values. + */ +struct imc_events { + u32 value; + char *name; + char *unit; + char *scale; +}; + +/* + * Trace IMC hardware updates a 64bytes record on + * Core Performance Monitoring Counter (CPMC) + * overflow. Here is the layout for the trace imc record + * + * DW 0 : Timebase + * DW 1 : Program Counter + * DW 2 : PIDR information + * DW 3 : CPMC1 + * DW 4 : CPMC2 + * DW 5 : CPMC3 + * Dw 6 : CPMC4 + * DW 7 : Timebase + * ..... + * + * The following is the data structure to hold trace imc data. + */ +struct trace_imc_data { + __be64 tb1; + __be64 ip; + __be64 val; + __be64 cpmc1; + __be64 cpmc2; + __be64 cpmc3; + __be64 cpmc4; + __be64 tb2; +}; + +/* Event attribute array index */ +#define IMC_FORMAT_ATTR 0 +#define IMC_EVENT_ATTR 1 +#define IMC_CPUMASK_ATTR 2 +#define IMC_NULL_ATTR 3 + +/* PMU Format attribute macros */ +#define IMC_EVENT_OFFSET_MASK 0xffffffffULL + +/* + * Macro to mask bits 0:21 of first double word(which is the timebase) to + * compare with 8th double word (timebase) of trace imc record data. + */ +#define IMC_TRACE_RECORD_TB1_MASK 0x3ffffffffffULL + +/* + * Bit 0:1 in third DW of IMC trace record + * specifies the MSR[HV PR] values. + */ +#define IMC_TRACE_RECORD_VAL_HVPR(x) ((x) >> 62) + +/* + * Device tree parser code detects IMC pmu support and + * registers new IMC pmus. This structure will hold the + * pmu functions, events, counter memory information + * and attrs for each imc pmu and will be referenced at + * the time of pmu registration. + */ +struct imc_pmu { + struct pmu pmu; + struct imc_mem_info *mem_info; + struct imc_events *events; + /* + * Attribute groups for the PMU. Slot 0 used for + * format attribute, slot 1 used for cpusmask attribute, + * slot 2 used for event attribute. Slot 3 keep as + * NULL. + */ + const struct attribute_group *attr_groups[4]; + u32 counter_mem_size; + int domain; + /* + * flag to notify whether the memory is mmaped + * or allocated by kernel. + */ + bool imc_counter_mmaped; +}; + +/* + * Structure to hold id, lock and reference count for the imc events which + * are inited. + */ +struct imc_pmu_ref { + spinlock_t lock; + unsigned int id; + int refc; +}; + +/* + * In-Memory Collection Counters type. + * Data comes from Device tree. + * Three device type are supported. + */ + +enum { + IMC_TYPE_THREAD = 0x1, + IMC_TYPE_TRACE = 0x2, + IMC_TYPE_CORE = 0x4, + IMC_TYPE_CHIP = 0x10, +}; + +/* + * Domains for IMC PMUs + */ +#define IMC_DOMAIN_NEST 1 +#define IMC_DOMAIN_CORE 2 +#define IMC_DOMAIN_THREAD 3 +/* For trace-imc the domain is still thread but it operates in trace-mode */ +#define IMC_DOMAIN_TRACE 4 + +extern int init_imc_pmu(struct device_node *parent, + struct imc_pmu *pmu_ptr, int pmu_id); +extern void thread_imc_disable(void); +extern int get_max_nest_dev(void); +extern void unregister_thread_imc(void); +#endif /* __ASM_POWERPC_IMC_PMU_H */ diff --git a/arch/powerpc/include/asm/immap_cpm2.h b/arch/powerpc/include/asm/immap_cpm2.h index 7c64fda5357b..845d5b3fb212 100644 --- a/arch/powerpc/include/asm/immap_cpm2.h +++ b/arch/powerpc/include/asm/immap_cpm2.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * CPM2 Internal Memory Map * Copyright (c) 1999 Dan Malek (dmalek@jlc.net) diff --git a/arch/powerpc/include/asm/immap_qe.h b/arch/powerpc/include/asm/immap_qe.h deleted file mode 100644 index bedbff891423..000000000000 --- a/arch/powerpc/include/asm/immap_qe.h +++ /dev/null @@ -1,491 +0,0 @@ -/* - * QUICC Engine (QE) Internal Memory Map. - * The Internal Memory Map for devices with QE on them. This - * is the superset of all QE devices (8360, etc.). - - * Copyright (C) 2006. Freescale Semiconductor, Inc. All rights reserved. - * - * Authors: Shlomi Gridish <gridish@freescale.com> - * Li Yang <leoli@freescale.com> - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - */ -#ifndef _ASM_POWERPC_IMMAP_QE_H -#define _ASM_POWERPC_IMMAP_QE_H -#ifdef __KERNEL__ - -#include <linux/kernel.h> -#include <asm/io.h> - -#define QE_IMMAP_SIZE (1024 * 1024) /* 1MB from 1MB+IMMR */ - -/* QE I-RAM */ -struct qe_iram { - __be32 iadd; /* I-RAM Address Register */ - __be32 idata; /* I-RAM Data Register */ - u8 res0[0x04]; - __be32 iready; /* I-RAM Ready Register */ - u8 res1[0x70]; -} __attribute__ ((packed)); - -/* QE Interrupt Controller */ -struct qe_ic_regs { - __be32 qicr; - __be32 qivec; - __be32 qripnr; - __be32 qipnr; - __be32 qipxcc; - __be32 qipycc; - __be32 qipwcc; - __be32 qipzcc; - __be32 qimr; - __be32 qrimr; - __be32 qicnr; - u8 res0[0x4]; - __be32 qiprta; - __be32 qiprtb; - u8 res1[0x4]; - __be32 qricr; - u8 res2[0x20]; - __be32 qhivec; - u8 res3[0x1C]; -} __attribute__ ((packed)); - -/* Communications Processor */ -struct cp_qe { - __be32 cecr; /* QE command register */ - __be32 ceccr; /* QE controller configuration register */ - __be32 cecdr; /* QE command data register */ - u8 res0[0xA]; - __be16 ceter; /* QE timer event register */ - u8 res1[0x2]; - __be16 cetmr; /* QE timers mask register */ - __be32 cetscr; /* QE time-stamp timer control register */ - __be32 cetsr1; /* QE time-stamp register 1 */ - __be32 cetsr2; /* QE time-stamp register 2 */ - u8 res2[0x8]; - __be32 cevter; /* QE virtual tasks event register */ - __be32 cevtmr; /* QE virtual tasks mask register */ - __be16 cercr; /* QE RAM control register */ - u8 res3[0x2]; - u8 res4[0x24]; - __be16 ceexe1; /* QE external request 1 event register */ - u8 res5[0x2]; - __be16 ceexm1; /* QE external request 1 mask register */ - u8 res6[0x2]; - __be16 ceexe2; /* QE external request 2 event register */ - u8 res7[0x2]; - __be16 ceexm2; /* QE external request 2 mask register */ - u8 res8[0x2]; - __be16 ceexe3; /* QE external request 3 event register */ - u8 res9[0x2]; - __be16 ceexm3; /* QE external request 3 mask register */ - u8 res10[0x2]; - __be16 ceexe4; /* QE external request 4 event register */ - u8 res11[0x2]; - __be16 ceexm4; /* QE external request 4 mask register */ - u8 res12[0x3A]; - __be32 ceurnr; /* QE microcode revision number register */ - u8 res13[0x244]; -} __attribute__ ((packed)); - -/* QE Multiplexer */ -struct qe_mux { - __be32 cmxgcr; /* CMX general clock route register */ - __be32 cmxsi1cr_l; /* CMX SI1 clock route low register */ - __be32 cmxsi1cr_h; /* CMX SI1 clock route high register */ - __be32 cmxsi1syr; /* CMX SI1 SYNC route register */ - __be32 cmxucr[4]; /* CMX UCCx clock route registers */ - __be32 cmxupcr; /* CMX UPC clock route register */ - u8 res0[0x1C]; -} __attribute__ ((packed)); - -/* QE Timers */ -struct qe_timers { - u8 gtcfr1; /* Timer 1 and Timer 2 global config register*/ - u8 res0[0x3]; - u8 gtcfr2; /* Timer 3 and timer 4 global config register*/ - u8 res1[0xB]; - __be16 gtmdr1; /* Timer 1 mode register */ - __be16 gtmdr2; /* Timer 2 mode register */ - __be16 gtrfr1; /* Timer 1 reference register */ - __be16 gtrfr2; /* Timer 2 reference register */ - __be16 gtcpr1; /* Timer 1 capture register */ - __be16 gtcpr2; /* Timer 2 capture register */ - __be16 gtcnr1; /* Timer 1 counter */ - __be16 gtcnr2; /* Timer 2 counter */ - __be16 gtmdr3; /* Timer 3 mode register */ - __be16 gtmdr4; /* Timer 4 mode register */ - __be16 gtrfr3; /* Timer 3 reference register */ - __be16 gtrfr4; /* Timer 4 reference register */ - __be16 gtcpr3; /* Timer 3 capture register */ - __be16 gtcpr4; /* Timer 4 capture register */ - __be16 gtcnr3; /* Timer 3 counter */ - __be16 gtcnr4; /* Timer 4 counter */ - __be16 gtevr1; /* Timer 1 event register */ - __be16 gtevr2; /* Timer 2 event register */ - __be16 gtevr3; /* Timer 3 event register */ - __be16 gtevr4; /* Timer 4 event register */ - __be16 gtps; /* Timer 1 prescale register */ - u8 res2[0x46]; -} __attribute__ ((packed)); - -/* BRG */ -struct qe_brg { - __be32 brgc[16]; /* BRG configuration registers */ - u8 res0[0x40]; -} __attribute__ ((packed)); - -/* SPI */ -struct spi { - u8 res0[0x20]; - __be32 spmode; /* SPI mode register */ - u8 res1[0x2]; - u8 spie; /* SPI event register */ - u8 res2[0x1]; - u8 res3[0x2]; - u8 spim; /* SPI mask register */ - u8 res4[0x1]; - u8 res5[0x1]; - u8 spcom; /* SPI command register */ - u8 res6[0x2]; - __be32 spitd; /* SPI transmit data register (cpu mode) */ - __be32 spird; /* SPI receive data register (cpu mode) */ - u8 res7[0x8]; -} __attribute__ ((packed)); - -/* SI */ -struct si1 { - __be16 siamr1; /* SI1 TDMA mode register */ - __be16 sibmr1; /* SI1 TDMB mode register */ - __be16 sicmr1; /* SI1 TDMC mode register */ - __be16 sidmr1; /* SI1 TDMD mode register */ - u8 siglmr1_h; /* SI1 global mode register high */ - u8 res0[0x1]; - u8 sicmdr1_h; /* SI1 command register high */ - u8 res2[0x1]; - u8 sistr1_h; /* SI1 status register high */ - u8 res3[0x1]; - __be16 sirsr1_h; /* SI1 RAM shadow address register high */ - u8 sitarc1; /* SI1 RAM counter Tx TDMA */ - u8 sitbrc1; /* SI1 RAM counter Tx TDMB */ - u8 sitcrc1; /* SI1 RAM counter Tx TDMC */ - u8 sitdrc1; /* SI1 RAM counter Tx TDMD */ - u8 sirarc1; /* SI1 RAM counter Rx TDMA */ - u8 sirbrc1; /* SI1 RAM counter Rx TDMB */ - u8 sircrc1; /* SI1 RAM counter Rx TDMC */ - u8 sirdrc1; /* SI1 RAM counter Rx TDMD */ - u8 res4[0x8]; - __be16 siemr1; /* SI1 TDME mode register 16 bits */ - __be16 sifmr1; /* SI1 TDMF mode register 16 bits */ - __be16 sigmr1; /* SI1 TDMG mode register 16 bits */ - __be16 sihmr1; /* SI1 TDMH mode register 16 bits */ - u8 siglmg1_l; /* SI1 global mode register low 8 bits */ - u8 res5[0x1]; - u8 sicmdr1_l; /* SI1 command register low 8 bits */ - u8 res6[0x1]; - u8 sistr1_l; /* SI1 status register low 8 bits */ - u8 res7[0x1]; - __be16 sirsr1_l; /* SI1 RAM shadow address register low 16 bits*/ - u8 siterc1; /* SI1 RAM counter Tx TDME 8 bits */ - u8 sitfrc1; /* SI1 RAM counter Tx TDMF 8 bits */ - u8 sitgrc1; /* SI1 RAM counter Tx TDMG 8 bits */ - u8 sithrc1; /* SI1 RAM counter Tx TDMH 8 bits */ - u8 sirerc1; /* SI1 RAM counter Rx TDME 8 bits */ - u8 sirfrc1; /* SI1 RAM counter Rx TDMF 8 bits */ - u8 sirgrc1; /* SI1 RAM counter Rx TDMG 8 bits */ - u8 sirhrc1; /* SI1 RAM counter Rx TDMH 8 bits */ - u8 res8[0x8]; - __be32 siml1; /* SI1 multiframe limit register */ - u8 siedm1; /* SI1 extended diagnostic mode register */ - u8 res9[0xBB]; -} __attribute__ ((packed)); - -/* SI Routing Tables */ -struct sir { - u8 tx[0x400]; - u8 rx[0x400]; - u8 res0[0x800]; -} __attribute__ ((packed)); - -/* USB Controller */ -struct qe_usb_ctlr { - u8 usb_usmod; - u8 usb_usadr; - u8 usb_uscom; - u8 res1[1]; - __be16 usb_usep[4]; - u8 res2[4]; - __be16 usb_usber; - u8 res3[2]; - __be16 usb_usbmr; - u8 res4[1]; - u8 usb_usbs; - __be16 usb_ussft; - u8 res5[2]; - __be16 usb_usfrn; - u8 res6[0x22]; -} __attribute__ ((packed)); - -/* MCC */ -struct qe_mcc { - __be32 mcce; /* MCC event register */ - __be32 mccm; /* MCC mask register */ - __be32 mccf; /* MCC configuration register */ - __be32 merl; /* MCC emergency request level register */ - u8 res0[0xF0]; -} __attribute__ ((packed)); - -/* QE UCC Slow */ -struct ucc_slow { - __be32 gumr_l; /* UCCx general mode register (low) */ - __be32 gumr_h; /* UCCx general mode register (high) */ - __be16 upsmr; /* UCCx protocol-specific mode register */ - u8 res0[0x2]; - __be16 utodr; /* UCCx transmit on demand register */ - __be16 udsr; /* UCCx data synchronization register */ - __be16 ucce; /* UCCx event register */ - u8 res1[0x2]; - __be16 uccm; /* UCCx mask register */ - u8 res2[0x1]; - u8 uccs; /* UCCx status register */ - u8 res3[0x24]; - __be16 utpt; - u8 res4[0x52]; - u8 guemr; /* UCC general extended mode register */ -} __attribute__ ((packed)); - -/* QE UCC Fast */ -struct ucc_fast { - __be32 gumr; /* UCCx general mode register */ - __be32 upsmr; /* UCCx protocol-specific mode register */ - __be16 utodr; /* UCCx transmit on demand register */ - u8 res0[0x2]; - __be16 udsr; /* UCCx data synchronization register */ - u8 res1[0x2]; - __be32 ucce; /* UCCx event register */ - __be32 uccm; /* UCCx mask register */ - u8 uccs; /* UCCx status register */ - u8 res2[0x7]; - __be32 urfb; /* UCC receive FIFO base */ - __be16 urfs; /* UCC receive FIFO size */ - u8 res3[0x2]; - __be16 urfet; /* UCC receive FIFO emergency threshold */ - __be16 urfset; /* UCC receive FIFO special emergency - threshold */ - __be32 utfb; /* UCC transmit FIFO base */ - __be16 utfs; /* UCC transmit FIFO size */ - u8 res4[0x2]; - __be16 utfet; /* UCC transmit FIFO emergency threshold */ - u8 res5[0x2]; - __be16 utftt; /* UCC transmit FIFO transmit threshold */ - u8 res6[0x2]; - __be16 utpt; /* UCC transmit polling timer */ - u8 res7[0x2]; - __be32 urtry; /* UCC retry counter register */ - u8 res8[0x4C]; - u8 guemr; /* UCC general extended mode register */ -} __attribute__ ((packed)); - -struct ucc { - union { - struct ucc_slow slow; - struct ucc_fast fast; - u8 res[0x200]; /* UCC blocks are 512 bytes each */ - }; -} __attribute__ ((packed)); - -/* MultiPHY UTOPIA POS Controllers (UPC) */ -struct upc { - __be32 upgcr; /* UTOPIA/POS general configuration register */ - __be32 uplpa; /* UTOPIA/POS last PHY address */ - __be32 uphec; /* ATM HEC register */ - __be32 upuc; /* UTOPIA/POS UCC configuration */ - __be32 updc1; /* UTOPIA/POS device 1 configuration */ - __be32 updc2; /* UTOPIA/POS device 2 configuration */ - __be32 updc3; /* UTOPIA/POS device 3 configuration */ - __be32 updc4; /* UTOPIA/POS device 4 configuration */ - __be32 upstpa; /* UTOPIA/POS STPA threshold */ - u8 res0[0xC]; - __be32 updrs1_h; /* UTOPIA/POS device 1 rate select */ - __be32 updrs1_l; /* UTOPIA/POS device 1 rate select */ - __be32 updrs2_h; /* UTOPIA/POS device 2 rate select */ - __be32 updrs2_l; /* UTOPIA/POS device 2 rate select */ - __be32 updrs3_h; /* UTOPIA/POS device 3 rate select */ - __be32 updrs3_l; /* UTOPIA/POS device 3 rate select */ - __be32 updrs4_h; /* UTOPIA/POS device 4 rate select */ - __be32 updrs4_l; /* UTOPIA/POS device 4 rate select */ - __be32 updrp1; /* UTOPIA/POS device 1 receive priority low */ - __be32 updrp2; /* UTOPIA/POS device 2 receive priority low */ - __be32 updrp3; /* UTOPIA/POS device 3 receive priority low */ - __be32 updrp4; /* UTOPIA/POS device 4 receive priority low */ - __be32 upde1; /* UTOPIA/POS device 1 event */ - __be32 upde2; /* UTOPIA/POS device 2 event */ - __be32 upde3; /* UTOPIA/POS device 3 event */ - __be32 upde4; /* UTOPIA/POS device 4 event */ - __be16 uprp1; - __be16 uprp2; - __be16 uprp3; - __be16 uprp4; - u8 res1[0x8]; - __be16 uptirr1_0; /* Device 1 transmit internal rate 0 */ - __be16 uptirr1_1; /* Device 1 transmit internal rate 1 */ - __be16 uptirr1_2; /* Device 1 transmit internal rate 2 */ - __be16 uptirr1_3; /* Device 1 transmit internal rate 3 */ - __be16 uptirr2_0; /* Device 2 transmit internal rate 0 */ - __be16 uptirr2_1; /* Device 2 transmit internal rate 1 */ - __be16 uptirr2_2; /* Device 2 transmit internal rate 2 */ - __be16 uptirr2_3; /* Device 2 transmit internal rate 3 */ - __be16 uptirr3_0; /* Device 3 transmit internal rate 0 */ - __be16 uptirr3_1; /* Device 3 transmit internal rate 1 */ - __be16 uptirr3_2; /* Device 3 transmit internal rate 2 */ - __be16 uptirr3_3; /* Device 3 transmit internal rate 3 */ - __be16 uptirr4_0; /* Device 4 transmit internal rate 0 */ - __be16 uptirr4_1; /* Device 4 transmit internal rate 1 */ - __be16 uptirr4_2; /* Device 4 transmit internal rate 2 */ - __be16 uptirr4_3; /* Device 4 transmit internal rate 3 */ - __be32 uper1; /* Device 1 port enable register */ - __be32 uper2; /* Device 2 port enable register */ - __be32 uper3; /* Device 3 port enable register */ - __be32 uper4; /* Device 4 port enable register */ - u8 res2[0x150]; -} __attribute__ ((packed)); - -/* SDMA */ -struct sdma { - __be32 sdsr; /* Serial DMA status register */ - __be32 sdmr; /* Serial DMA mode register */ - __be32 sdtr1; /* SDMA system bus threshold register */ - __be32 sdtr2; /* SDMA secondary bus threshold register */ - __be32 sdhy1; /* SDMA system bus hysteresis register */ - __be32 sdhy2; /* SDMA secondary bus hysteresis register */ - __be32 sdta1; /* SDMA system bus address register */ - __be32 sdta2; /* SDMA secondary bus address register */ - __be32 sdtm1; /* SDMA system bus MSNUM register */ - __be32 sdtm2; /* SDMA secondary bus MSNUM register */ - u8 res0[0x10]; - __be32 sdaqr; /* SDMA address bus qualify register */ - __be32 sdaqmr; /* SDMA address bus qualify mask register */ - u8 res1[0x4]; - __be32 sdebcr; /* SDMA CAM entries base register */ - u8 res2[0x38]; -} __attribute__ ((packed)); - -/* Debug Space */ -struct dbg { - __be32 bpdcr; /* Breakpoint debug command register */ - __be32 bpdsr; /* Breakpoint debug status register */ - __be32 bpdmr; /* Breakpoint debug mask register */ - __be32 bprmrr0; /* Breakpoint request mode risc register 0 */ - __be32 bprmrr1; /* Breakpoint request mode risc register 1 */ - u8 res0[0x8]; - __be32 bprmtr0; /* Breakpoint request mode trb register 0 */ - __be32 bprmtr1; /* Breakpoint request mode trb register 1 */ - u8 res1[0x8]; - __be32 bprmir; /* Breakpoint request mode immediate register */ - __be32 bprmsr; /* Breakpoint request mode serial register */ - __be32 bpemr; /* Breakpoint exit mode register */ - u8 res2[0x48]; -} __attribute__ ((packed)); - -/* - * RISC Special Registers (Trap and Breakpoint). These are described in - * the QE Developer's Handbook. - */ -struct rsp { - __be32 tibcr[16]; /* Trap/instruction breakpoint control regs */ - u8 res0[64]; - __be32 ibcr0; - __be32 ibs0; - __be32 ibcnr0; - u8 res1[4]; - __be32 ibcr1; - __be32 ibs1; - __be32 ibcnr1; - __be32 npcr; - __be32 dbcr; - __be32 dbar; - __be32 dbamr; - __be32 dbsr; - __be32 dbcnr; - u8 res2[12]; - __be32 dbdr_h; - __be32 dbdr_l; - __be32 dbdmr_h; - __be32 dbdmr_l; - __be32 bsr; - __be32 bor; - __be32 bior; - u8 res3[4]; - __be32 iatr[4]; - __be32 eccr; /* Exception control configuration register */ - __be32 eicr; - u8 res4[0x100-0xf8]; -} __attribute__ ((packed)); - -struct qe_immap { - struct qe_iram iram; /* I-RAM */ - struct qe_ic_regs ic; /* Interrupt Controller */ - struct cp_qe cp; /* Communications Processor */ - struct qe_mux qmx; /* QE Multiplexer */ - struct qe_timers qet; /* QE Timers */ - struct spi spi[0x2]; /* spi */ - struct qe_mcc mcc; /* mcc */ - struct qe_brg brg; /* brg */ - struct qe_usb_ctlr usb; /* USB */ - struct si1 si1; /* SI */ - u8 res11[0x800]; - struct sir sir; /* SI Routing Tables */ - struct ucc ucc1; /* ucc1 */ - struct ucc ucc3; /* ucc3 */ - struct ucc ucc5; /* ucc5 */ - struct ucc ucc7; /* ucc7 */ - u8 res12[0x600]; - struct upc upc1; /* MultiPHY UTOPIA POS Ctrlr 1*/ - struct ucc ucc2; /* ucc2 */ - struct ucc ucc4; /* ucc4 */ - struct ucc ucc6; /* ucc6 */ - struct ucc ucc8; /* ucc8 */ - u8 res13[0x600]; - struct upc upc2; /* MultiPHY UTOPIA POS Ctrlr 2*/ - struct sdma sdma; /* SDMA */ - struct dbg dbg; /* 0x104080 - 0x1040FF - Debug Space */ - struct rsp rsp[0x2]; /* 0x104100 - 0x1042FF - RISC Special Registers - (Trap and Breakpoint) */ - u8 res14[0x300]; /* 0x104300 - 0x1045FF */ - u8 res15[0x3A00]; /* 0x104600 - 0x107FFF */ - u8 res16[0x8000]; /* 0x108000 - 0x110000 */ - u8 muram[0xC000]; /* 0x110000 - 0x11C000 - Multi-user RAM */ - u8 res17[0x24000]; /* 0x11C000 - 0x140000 */ - u8 res18[0xC0000]; /* 0x140000 - 0x200000 */ -} __attribute__ ((packed)); - -extern struct qe_immap __iomem *qe_immr; -extern phys_addr_t get_qe_base(void); - -/* - * Returns the offset within the QE address space of the given pointer. - * - * Note that the QE does not support 36-bit physical addresses, so if - * get_qe_base() returns a number above 4GB, the caller will probably fail. - */ -static inline phys_addr_t immrbar_virt_to_phys(void *address) -{ - void *q = (void *)qe_immr; - - /* Is it a MURAM address? */ - if ((address >= q) && (address < (q + QE_IMMAP_SIZE))) - return get_qe_base() + (address - q); - - /* It's an address returned by kmalloc */ - return virt_to_phys(address); -} - -#endif /* __KERNEL__ */ -#endif /* _ASM_POWERPC_IMMAP_QE_H */ diff --git a/arch/powerpc/include/asm/inst.h b/arch/powerpc/include/asm/inst.h new file mode 100644 index 000000000000..ffa82167c860 --- /dev/null +++ b/arch/powerpc/include/asm/inst.h @@ -0,0 +1,166 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_POWERPC_INST_H +#define _ASM_POWERPC_INST_H + +#include <asm/ppc-opcode.h> +#include <asm/reg.h> +#include <asm/disassemble.h> +#include <asm/uaccess.h> + +#define ___get_user_instr(gu_op, dest, ptr) \ +({ \ + long __gui_ret; \ + u32 __user *__gui_ptr = (u32 __user *)ptr; \ + ppc_inst_t __gui_inst; \ + unsigned int __prefix, __suffix; \ + \ + __chk_user_ptr(ptr); \ + __gui_ret = gu_op(__prefix, __gui_ptr); \ + if (__gui_ret == 0) { \ + if (IS_ENABLED(CONFIG_PPC64) && (__prefix >> 26) == OP_PREFIX) { \ + __gui_ret = gu_op(__suffix, __gui_ptr + 1); \ + __gui_inst = ppc_inst_prefix(__prefix, __suffix); \ + } else { \ + __gui_inst = ppc_inst(__prefix); \ + } \ + if (__gui_ret == 0) \ + (dest) = __gui_inst; \ + } \ + __gui_ret; \ +}) + +#define get_user_instr(x, ptr) ___get_user_instr(get_user, x, ptr) + +#define __get_user_instr(x, ptr) ___get_user_instr(__get_user, x, ptr) + +/* + * Instruction data type for POWER + */ + +#if defined(CONFIG_PPC64) || defined(__CHECKER__) +static inline u32 ppc_inst_val(ppc_inst_t x) +{ + return x.val; +} + +#define ppc_inst(x) ((ppc_inst_t){ .val = (x) }) + +#else +static inline u32 ppc_inst_val(ppc_inst_t x) +{ + return x; +} +#define ppc_inst(x) (x) +#endif + +static inline int ppc_inst_primary_opcode(ppc_inst_t x) +{ + return ppc_inst_val(x) >> 26; +} + +#ifdef CONFIG_PPC64 +#define ppc_inst_prefix(x, y) ((ppc_inst_t){ .val = (x), .suffix = (y) }) + +static inline u32 ppc_inst_suffix(ppc_inst_t x) +{ + return x.suffix; +} + +#else +#define ppc_inst_prefix(x, y) ((void)y, ppc_inst(x)) + +static inline u32 ppc_inst_suffix(ppc_inst_t x) +{ + return 0; +} + +#endif /* CONFIG_PPC64 */ + +static inline ppc_inst_t ppc_inst_read(const u32 *ptr) +{ + if (IS_ENABLED(CONFIG_PPC64) && (*ptr >> 26) == OP_PREFIX) + return ppc_inst_prefix(*ptr, *(ptr + 1)); + else + return ppc_inst(*ptr); +} + +static inline bool ppc_inst_prefixed(ppc_inst_t x) +{ + return IS_ENABLED(CONFIG_PPC64) && ppc_inst_primary_opcode(x) == OP_PREFIX; +} + +static inline ppc_inst_t ppc_inst_swab(ppc_inst_t x) +{ + return ppc_inst_prefix(swab32(ppc_inst_val(x)), swab32(ppc_inst_suffix(x))); +} + +static inline bool ppc_inst_equal(ppc_inst_t x, ppc_inst_t y) +{ + if (ppc_inst_val(x) != ppc_inst_val(y)) + return false; + if (!ppc_inst_prefixed(x)) + return true; + return ppc_inst_suffix(x) == ppc_inst_suffix(y); +} + +static inline int ppc_inst_len(ppc_inst_t x) +{ + return ppc_inst_prefixed(x) ? 8 : 4; +} + +/* + * Return the address of the next instruction, if the instruction @value was + * located at @location. + */ +static inline u32 *ppc_inst_next(u32 *location, u32 *value) +{ + ppc_inst_t tmp; + + tmp = ppc_inst_read(value); + + return (void *)location + ppc_inst_len(tmp); +} + +static inline unsigned long ppc_inst_as_ulong(ppc_inst_t x) +{ + if (IS_ENABLED(CONFIG_PPC32)) + return ppc_inst_val(x); + else if (IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN)) + return (u64)ppc_inst_suffix(x) << 32 | ppc_inst_val(x); + else + return (u64)ppc_inst_val(x) << 32 | ppc_inst_suffix(x); +} + +static inline void ppc_inst_write(u32 *ptr, ppc_inst_t x) +{ + if (!ppc_inst_prefixed(x)) + *ptr = ppc_inst_val(x); + else + *(u64 *)ptr = ppc_inst_as_ulong(x); +} + +static inline int __copy_inst_from_kernel_nofault(ppc_inst_t *inst, u32 *src) +{ + unsigned int val, suffix; + + __get_kernel_nofault(&val, src, u32, Efault); + if (IS_ENABLED(CONFIG_PPC64) && get_op(val) == OP_PREFIX) { + __get_kernel_nofault(&suffix, src + 1, u32, Efault); + *inst = ppc_inst_prefix(val, suffix); + } else { + *inst = ppc_inst(val); + } + return 0; +Efault: + return -EFAULT; +} + +static inline int copy_inst_from_kernel_nofault(ppc_inst_t *inst, u32 *src) +{ + if (unlikely(!is_kernel_addr((unsigned long)src))) + return -ERANGE; + + return __copy_inst_from_kernel_nofault(inst, src); +} + +#endif /* _ASM_POWERPC_INST_H */ diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h new file mode 100644 index 000000000000..eb0e4a20b818 --- /dev/null +++ b/arch/powerpc/include/asm/interrupt.h @@ -0,0 +1,680 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_POWERPC_INTERRUPT_H +#define _ASM_POWERPC_INTERRUPT_H + +/* BookE/4xx */ +#define INTERRUPT_CRITICAL_INPUT 0x100 + +/* BookE */ +#define INTERRUPT_DEBUG 0xd00 +#ifdef CONFIG_BOOKE +#define INTERRUPT_PERFMON 0x260 +#define INTERRUPT_DOORBELL 0x280 +#endif + +/* BookS/4xx/8xx */ +#define INTERRUPT_MACHINE_CHECK 0x200 + +/* BookS/8xx */ +#define INTERRUPT_SYSTEM_RESET 0x100 + +/* BookS */ +#define INTERRUPT_DATA_SEGMENT 0x380 +#define INTERRUPT_INST_SEGMENT 0x480 +#define INTERRUPT_TRACE 0xd00 +#define INTERRUPT_H_DATA_STORAGE 0xe00 +#define INTERRUPT_HMI 0xe60 +#define INTERRUPT_H_FAC_UNAVAIL 0xf80 +#ifdef CONFIG_PPC_BOOK3S +#define INTERRUPT_DOORBELL 0xa00 +#define INTERRUPT_PERFMON 0xf00 +#define INTERRUPT_ALTIVEC_UNAVAIL 0xf20 +#endif + +/* BookE/BookS/4xx/8xx */ +#define INTERRUPT_DATA_STORAGE 0x300 +#define INTERRUPT_INST_STORAGE 0x400 +#define INTERRUPT_EXTERNAL 0x500 +#define INTERRUPT_ALIGNMENT 0x600 +#define INTERRUPT_PROGRAM 0x700 +#define INTERRUPT_SYSCALL 0xc00 +#define INTERRUPT_TRACE 0xd00 + +/* BookE/BookS/44x */ +#define INTERRUPT_FP_UNAVAIL 0x800 + +/* BookE/BookS/44x/8xx */ +#define INTERRUPT_DECREMENTER 0x900 + +#ifndef INTERRUPT_PERFMON +#define INTERRUPT_PERFMON 0x0 +#endif + +/* 8xx */ +#define INTERRUPT_SOFT_EMU_8xx 0x1000 +#define INTERRUPT_INST_TLB_MISS_8xx 0x1100 +#define INTERRUPT_DATA_TLB_MISS_8xx 0x1200 +#define INTERRUPT_INST_TLB_ERROR_8xx 0x1300 +#define INTERRUPT_DATA_TLB_ERROR_8xx 0x1400 +#define INTERRUPT_DATA_BREAKPOINT_8xx 0x1c00 +#define INTERRUPT_INST_BREAKPOINT_8xx 0x1d00 + +/* 603 */ +#define INTERRUPT_INST_TLB_MISS_603 0x1000 +#define INTERRUPT_DATA_LOAD_TLB_MISS_603 0x1100 +#define INTERRUPT_DATA_STORE_TLB_MISS_603 0x1200 + +#ifndef __ASSEMBLER__ + +#include <linux/context_tracking.h> +#include <linux/hardirq.h> +#include <asm/cputime.h> +#include <asm/firmware.h> +#include <asm/ftrace.h> +#include <asm/kprobes.h> +#include <asm/runlatch.h> + +#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG +/* + * WARN/BUG is handled with a program interrupt so minimise checks here to + * avoid recursion and maximise the chance of getting the first oops handled. + */ +#define INT_SOFT_MASK_BUG_ON(regs, cond) \ +do { \ + if ((user_mode(regs) || (TRAP(regs) != INTERRUPT_PROGRAM))) \ + BUG_ON(cond); \ +} while (0) +#else +#define INT_SOFT_MASK_BUG_ON(regs, cond) +#endif + +#ifdef CONFIG_PPC_BOOK3S_64 +extern char __end_soft_masked[]; +bool search_kernel_soft_mask_table(unsigned long addr); +unsigned long search_kernel_restart_table(unsigned long addr); + +DECLARE_STATIC_KEY_FALSE(interrupt_exit_not_reentrant); + +static inline bool is_implicit_soft_masked(struct pt_regs *regs) +{ + if (user_mode(regs)) + return false; + + if (regs->nip >= (unsigned long)__end_soft_masked) + return false; + + return search_kernel_soft_mask_table(regs->nip); +} + +static inline void srr_regs_clobbered(void) +{ + local_paca->srr_valid = 0; + local_paca->hsrr_valid = 0; +} +#else +static inline unsigned long search_kernel_restart_table(unsigned long addr) +{ + return 0; +} + +static inline bool is_implicit_soft_masked(struct pt_regs *regs) +{ + return false; +} + +static inline void srr_regs_clobbered(void) +{ +} +#endif + +static inline void nap_adjust_return(struct pt_regs *regs) +{ +#ifdef CONFIG_PPC_970_NAP + if (unlikely(test_thread_local_flags(_TLF_NAPPING))) { + /* Can avoid a test-and-clear because NMIs do not call this */ + clear_thread_local_flags(_TLF_NAPPING); + regs_set_return_ip(regs, (unsigned long)power4_idle_nap_return); + } +#endif +} + +static inline void booke_restore_dbcr0(void) +{ +#ifdef CONFIG_PPC_ADV_DEBUG_REGS + unsigned long dbcr0 = current->thread.debug.dbcr0; + + if (IS_ENABLED(CONFIG_PPC32) && unlikely(dbcr0 & DBCR0_IDM)) { + mtspr(SPRN_DBSR, -1); + mtspr(SPRN_DBCR0, global_dbcr0[smp_processor_id()]); + } +#endif +} + +static inline void interrupt_enter_prepare(struct pt_regs *regs) +{ +#ifdef CONFIG_PPC64 + irq_soft_mask_set(IRQS_ALL_DISABLED); + + /* + * If the interrupt was taken with HARD_DIS clear, then enable MSR[EE]. + * Asynchronous interrupts get here with HARD_DIS set (see below), so + * this enables MSR[EE] for synchronous interrupts. IRQs remain + * soft-masked. The interrupt handler may later call + * interrupt_cond_local_irq_enable() to achieve a regular process + * context. + */ + if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) { + INT_SOFT_MASK_BUG_ON(regs, !(regs->msr & MSR_EE)); + __hard_irq_enable(); + } else { + __hard_RI_enable(); + } + /* Enable MSR[RI] early, to support kernel SLB and hash faults */ +#endif + + if (!arch_irq_disabled_regs(regs)) + trace_hardirqs_off(); + + if (user_mode(regs)) { + kuap_lock(); + CT_WARN_ON(ct_state() != CT_STATE_USER); + user_exit_irqoff(); + + account_cpu_user_entry(); + account_stolen_time(); + } else { + kuap_save_and_lock(regs); + /* + * CT_WARN_ON comes here via program_check_exception, + * so avoid recursion. + */ + if (TRAP(regs) != INTERRUPT_PROGRAM) + CT_WARN_ON(ct_state() != CT_STATE_KERNEL && + ct_state() != CT_STATE_IDLE); + INT_SOFT_MASK_BUG_ON(regs, is_implicit_soft_masked(regs)); + INT_SOFT_MASK_BUG_ON(regs, arch_irq_disabled_regs(regs) && + search_kernel_restart_table(regs->nip)); + } + INT_SOFT_MASK_BUG_ON(regs, !arch_irq_disabled_regs(regs) && + !(regs->msr & MSR_EE)); + + booke_restore_dbcr0(); +} + +/* + * Care should be taken to note that interrupt_exit_prepare and + * interrupt_async_exit_prepare do not necessarily return immediately to + * regs context (e.g., if regs is usermode, we don't necessarily return to + * user mode). Other interrupts might be taken between here and return, + * context switch / preemption may occur in the exit path after this, or a + * signal may be delivered, etc. + * + * The real interrupt exit code is platform specific, e.g., + * interrupt_exit_user_prepare / interrupt_exit_kernel_prepare for 64s. + * + * However interrupt_nmi_exit_prepare does return directly to regs, because + * NMIs do not do "exit work" or replay soft-masked interrupts. + */ +static inline void interrupt_exit_prepare(struct pt_regs *regs) +{ +} + +static inline void interrupt_async_enter_prepare(struct pt_regs *regs) +{ +#ifdef CONFIG_PPC64 + /* Ensure interrupt_enter_prepare does not enable MSR[EE] */ + local_paca->irq_happened |= PACA_IRQ_HARD_DIS; +#endif + interrupt_enter_prepare(regs); +#ifdef CONFIG_PPC_BOOK3S_64 + /* + * RI=1 is set by interrupt_enter_prepare, so this thread flags access + * has to come afterward (it can cause SLB faults). + */ + if (cpu_has_feature(CPU_FTR_CTRL) && + !test_thread_local_flags(_TLF_RUNLATCH)) + __ppc64_runlatch_on(); +#endif + irq_enter(); +} + +static inline void interrupt_async_exit_prepare(struct pt_regs *regs) +{ + /* + * Adjust at exit so the main handler sees the true NIA. This must + * come before irq_exit() because irq_exit can enable interrupts, and + * if another interrupt is taken before nap_adjust_return has run + * here, then that interrupt would return directly to idle nap return. + */ + nap_adjust_return(regs); + + irq_exit(); + interrupt_exit_prepare(regs); +} + +struct interrupt_nmi_state { +#ifdef CONFIG_PPC64 + u8 irq_soft_mask; + u8 irq_happened; + u8 ftrace_enabled; + u64 softe; +#endif +}; + +static inline bool nmi_disables_ftrace(struct pt_regs *regs) +{ + /* Allow DEC and PMI to be traced when they are soft-NMI */ + if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) { + if (TRAP(regs) == INTERRUPT_DECREMENTER) + return false; + if (TRAP(regs) == INTERRUPT_PERFMON) + return false; + } + if (IS_ENABLED(CONFIG_PPC_BOOK3E_64)) { + if (TRAP(regs) == INTERRUPT_PERFMON) + return false; + } + + return true; +} + +static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state) +{ +#ifdef CONFIG_PPC64 + state->irq_soft_mask = local_paca->irq_soft_mask; + state->irq_happened = local_paca->irq_happened; + state->softe = regs->softe; + + /* + * Set IRQS_ALL_DISABLED unconditionally so irqs_disabled() does + * the right thing, and set IRQ_HARD_DIS. We do not want to reconcile + * because that goes through irq tracing which we don't want in NMI. + */ + local_paca->irq_soft_mask = IRQS_ALL_DISABLED; + local_paca->irq_happened |= PACA_IRQ_HARD_DIS; + + if (!(regs->msr & MSR_EE) || is_implicit_soft_masked(regs)) { + /* + * Adjust regs->softe to be soft-masked if it had not been + * reconcied (e.g., interrupt entry with MSR[EE]=0 but softe + * not yet set disabled), or if it was in an implicit soft + * masked state. This makes arch_irq_disabled_regs(regs) + * behave as expected. + */ + regs->softe = IRQS_ALL_DISABLED; + } + + __hard_RI_enable(); + + /* Don't do any per-CPU operations until interrupt state is fixed */ + + if (nmi_disables_ftrace(regs)) { + state->ftrace_enabled = this_cpu_get_ftrace_enabled(); + this_cpu_set_ftrace_enabled(0); + } +#endif + + /* If data relocations are enabled, it's safe to use nmi_enter() */ + if (mfmsr() & MSR_DR) { + nmi_enter(); + return; + } + + /* + * But do not use nmi_enter() for pseries hash guest taking a real-mode + * NMI because not everything it touches is within the RMA limit. + */ + if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && + firmware_has_feature(FW_FEATURE_LPAR) && + !radix_enabled()) + return; + + /* + * Likewise, don't use it if we have some form of instrumentation (like + * KASAN shadow) that is not safe to access in real mode (even on radix) + */ + if (IS_ENABLED(CONFIG_KASAN)) + return; + + /* + * Likewise, do not use it in real mode if percpu first chunk is not + * embedded. With CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK enabled there + * are chances where percpu allocation can come from vmalloc area. + */ + if (percpu_first_chunk_is_paged) + return; + + /* Otherwise, it should be safe to call it */ + nmi_enter(); +} + +static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state) +{ + if (mfmsr() & MSR_DR) { + // nmi_exit if relocations are on + nmi_exit(); + } else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && + firmware_has_feature(FW_FEATURE_LPAR) && + !radix_enabled()) { + // no nmi_exit for a pseries hash guest taking a real mode exception + } else if (IS_ENABLED(CONFIG_KASAN)) { + // no nmi_exit for KASAN in real mode + } else if (percpu_first_chunk_is_paged) { + // no nmi_exit if percpu first chunk is not embedded + } else { + nmi_exit(); + } + + /* + * nmi does not call nap_adjust_return because nmi should not create + * new work to do (must use irq_work for that). + */ + +#ifdef CONFIG_PPC64 +#ifdef CONFIG_PPC_BOOK3S + if (arch_irq_disabled_regs(regs)) { + unsigned long rst = search_kernel_restart_table(regs->nip); + if (rst) + regs_set_return_ip(regs, rst); + } +#endif + + if (nmi_disables_ftrace(regs)) + this_cpu_set_ftrace_enabled(state->ftrace_enabled); + + /* Check we didn't change the pending interrupt mask. */ + WARN_ON_ONCE((state->irq_happened | PACA_IRQ_HARD_DIS) != local_paca->irq_happened); + regs->softe = state->softe; + local_paca->irq_happened = state->irq_happened; + local_paca->irq_soft_mask = state->irq_soft_mask; +#endif +} + +/* + * Don't use noinstr here like x86, but rather add NOKPROBE_SYMBOL to each + * function definition. The reason for this is the noinstr section is placed + * after the main text section, i.e., very far away from the interrupt entry + * asm. That creates problems with fitting linker stubs when building large + * kernels. + */ +#define interrupt_handler __visible noinline notrace __no_kcsan __no_sanitize_address + +/** + * DECLARE_INTERRUPT_HANDLER_RAW - Declare raw interrupt handler function + * @func: Function name of the entry point + * @returns: Returns a value back to asm caller + */ +#define DECLARE_INTERRUPT_HANDLER_RAW(func) \ + __visible long func(struct pt_regs *regs) + +/** + * DEFINE_INTERRUPT_HANDLER_RAW - Define raw interrupt handler function + * @func: Function name of the entry point + * @returns: Returns a value back to asm caller + * + * @func is called from ASM entry code. + * + * This is a plain function which does no tracing, reconciling, etc. + * The macro is written so it acts as function definition. Append the + * body with a pair of curly brackets. + * + * raw interrupt handlers must not enable or disable interrupts, or + * schedule, tracing and instrumentation (ftrace, lockdep, etc) would + * not be advisable either, although may be possible in a pinch, the + * trace will look odd at least. + * + * A raw handler may call one of the other interrupt handler functions + * to be converted into that interrupt context without these restrictions. + * + * On PPC64, _RAW handlers may return with fast_interrupt_return. + * + * Specific handlers may have additional restrictions. + */ +#define DEFINE_INTERRUPT_HANDLER_RAW(func) \ +static __always_inline __no_sanitize_address __no_kcsan long \ +____##func(struct pt_regs *regs); \ + \ +interrupt_handler long func(struct pt_regs *regs) \ +{ \ + long ret; \ + \ + __hard_RI_enable(); \ + \ + ret = ____##func (regs); \ + \ + return ret; \ +} \ +NOKPROBE_SYMBOL(func); \ + \ +static __always_inline __no_sanitize_address __no_kcsan long \ +____##func(struct pt_regs *regs) + +/** + * DECLARE_INTERRUPT_HANDLER - Declare synchronous interrupt handler function + * @func: Function name of the entry point + */ +#define DECLARE_INTERRUPT_HANDLER(func) \ + __visible void func(struct pt_regs *regs) + +/** + * DEFINE_INTERRUPT_HANDLER - Define synchronous interrupt handler function + * @func: Function name of the entry point + * + * @func is called from ASM entry code. + * + * The macro is written so it acts as function definition. Append the + * body with a pair of curly brackets. + */ +#define DEFINE_INTERRUPT_HANDLER(func) \ +static __always_inline void ____##func(struct pt_regs *regs); \ + \ +interrupt_handler void func(struct pt_regs *regs) \ +{ \ + interrupt_enter_prepare(regs); \ + \ + ____##func (regs); \ + \ + interrupt_exit_prepare(regs); \ +} \ +NOKPROBE_SYMBOL(func); \ + \ +static __always_inline void ____##func(struct pt_regs *regs) + +/** + * DECLARE_INTERRUPT_HANDLER_RET - Declare synchronous interrupt handler function + * @func: Function name of the entry point + * @returns: Returns a value back to asm caller + */ +#define DECLARE_INTERRUPT_HANDLER_RET(func) \ + __visible long func(struct pt_regs *regs) + +/** + * DEFINE_INTERRUPT_HANDLER_RET - Define synchronous interrupt handler function + * @func: Function name of the entry point + * @returns: Returns a value back to asm caller + * + * @func is called from ASM entry code. + * + * The macro is written so it acts as function definition. Append the + * body with a pair of curly brackets. + */ +#define DEFINE_INTERRUPT_HANDLER_RET(func) \ +static __always_inline long ____##func(struct pt_regs *regs); \ + \ +interrupt_handler long func(struct pt_regs *regs) \ +{ \ + long ret; \ + \ + interrupt_enter_prepare(regs); \ + \ + ret = ____##func (regs); \ + \ + interrupt_exit_prepare(regs); \ + \ + return ret; \ +} \ +NOKPROBE_SYMBOL(func); \ + \ +static __always_inline long ____##func(struct pt_regs *regs) + +/** + * DECLARE_INTERRUPT_HANDLER_ASYNC - Declare asynchronous interrupt handler function + * @func: Function name of the entry point + */ +#define DECLARE_INTERRUPT_HANDLER_ASYNC(func) \ + __visible void func(struct pt_regs *regs) + +/** + * DEFINE_INTERRUPT_HANDLER_ASYNC - Define asynchronous interrupt handler function + * @func: Function name of the entry point + * + * @func is called from ASM entry code. + * + * The macro is written so it acts as function definition. Append the + * body with a pair of curly brackets. + */ +#define DEFINE_INTERRUPT_HANDLER_ASYNC(func) \ +static __always_inline void ____##func(struct pt_regs *regs); \ + \ +interrupt_handler void func(struct pt_regs *regs) \ +{ \ + interrupt_async_enter_prepare(regs); \ + \ + ____##func (regs); \ + \ + interrupt_async_exit_prepare(regs); \ +} \ +NOKPROBE_SYMBOL(func); \ + \ +static __always_inline void ____##func(struct pt_regs *regs) + +/** + * DECLARE_INTERRUPT_HANDLER_NMI - Declare NMI interrupt handler function + * @func: Function name of the entry point + * @returns: Returns a value back to asm caller + */ +#define DECLARE_INTERRUPT_HANDLER_NMI(func) \ + __visible long func(struct pt_regs *regs) + +/** + * DEFINE_INTERRUPT_HANDLER_NMI - Define NMI interrupt handler function + * @func: Function name of the entry point + * @returns: Returns a value back to asm caller + * + * @func is called from ASM entry code. + * + * The macro is written so it acts as function definition. Append the + * body with a pair of curly brackets. + */ +#define DEFINE_INTERRUPT_HANDLER_NMI(func) \ +static __always_inline __no_sanitize_address __no_kcsan long \ +____##func(struct pt_regs *regs); \ + \ +interrupt_handler long func(struct pt_regs *regs) \ +{ \ + struct interrupt_nmi_state state; \ + long ret; \ + \ + interrupt_nmi_enter_prepare(regs, &state); \ + \ + ret = ____##func (regs); \ + \ + interrupt_nmi_exit_prepare(regs, &state); \ + \ + return ret; \ +} \ +NOKPROBE_SYMBOL(func); \ + \ +static __always_inline __no_sanitize_address __no_kcsan long \ +____##func(struct pt_regs *regs) + + +/* Interrupt handlers */ +/* kernel/traps.c */ +DECLARE_INTERRUPT_HANDLER_NMI(system_reset_exception); +#ifdef CONFIG_PPC_BOOK3S_64 +DECLARE_INTERRUPT_HANDLER_RAW(machine_check_early_boot); +DECLARE_INTERRUPT_HANDLER_ASYNC(machine_check_exception_async); +#endif +DECLARE_INTERRUPT_HANDLER_NMI(machine_check_exception); +DECLARE_INTERRUPT_HANDLER(SMIException); +DECLARE_INTERRUPT_HANDLER(handle_hmi_exception); +DECLARE_INTERRUPT_HANDLER(unknown_exception); +DECLARE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception); +DECLARE_INTERRUPT_HANDLER_NMI(unknown_nmi_exception); +DECLARE_INTERRUPT_HANDLER(instruction_breakpoint_exception); +DECLARE_INTERRUPT_HANDLER(RunModeException); +DECLARE_INTERRUPT_HANDLER(single_step_exception); +DECLARE_INTERRUPT_HANDLER(program_check_exception); +DECLARE_INTERRUPT_HANDLER(emulation_assist_interrupt); +DECLARE_INTERRUPT_HANDLER(alignment_exception); +DECLARE_INTERRUPT_HANDLER(StackOverflow); +DECLARE_INTERRUPT_HANDLER(stack_overflow_exception); +DECLARE_INTERRUPT_HANDLER(kernel_fp_unavailable_exception); +DECLARE_INTERRUPT_HANDLER(altivec_unavailable_exception); +DECLARE_INTERRUPT_HANDLER(vsx_unavailable_exception); +DECLARE_INTERRUPT_HANDLER(facility_unavailable_exception); +DECLARE_INTERRUPT_HANDLER(fp_unavailable_tm); +DECLARE_INTERRUPT_HANDLER(altivec_unavailable_tm); +DECLARE_INTERRUPT_HANDLER(vsx_unavailable_tm); +DECLARE_INTERRUPT_HANDLER_NMI(performance_monitor_exception_nmi); +DECLARE_INTERRUPT_HANDLER_ASYNC(performance_monitor_exception_async); +DECLARE_INTERRUPT_HANDLER_RAW(performance_monitor_exception); +DECLARE_INTERRUPT_HANDLER(DebugException); +DECLARE_INTERRUPT_HANDLER(altivec_assist_exception); +DECLARE_INTERRUPT_HANDLER(CacheLockingException); +DECLARE_INTERRUPT_HANDLER(SPEFloatingPointException); +DECLARE_INTERRUPT_HANDLER(SPEFloatingPointRoundException); +DECLARE_INTERRUPT_HANDLER_NMI(WatchdogException); +DECLARE_INTERRUPT_HANDLER(kernel_bad_stack); + +/* slb.c */ +DECLARE_INTERRUPT_HANDLER_RAW(do_slb_fault); +DECLARE_INTERRUPT_HANDLER(do_bad_segment_interrupt); + +/* hash_utils.c */ +DECLARE_INTERRUPT_HANDLER(do_hash_fault); + +/* fault.c */ +DECLARE_INTERRUPT_HANDLER(do_page_fault); +DECLARE_INTERRUPT_HANDLER(do_bad_page_fault_segv); + +/* process.c */ +DECLARE_INTERRUPT_HANDLER(do_break); + +/* time.c */ +DECLARE_INTERRUPT_HANDLER_ASYNC(timer_interrupt); + +/* mce.c */ +DECLARE_INTERRUPT_HANDLER_NMI(machine_check_early); +DECLARE_INTERRUPT_HANDLER_NMI(hmi_exception_realmode); + +DECLARE_INTERRUPT_HANDLER_ASYNC(TAUException); + +/* irq.c */ +DECLARE_INTERRUPT_HANDLER_ASYNC(do_IRQ); + +void __noreturn unrecoverable_exception(struct pt_regs *regs); + +void replay_system_reset(void); +void replay_soft_interrupts(void); + +static inline void interrupt_cond_local_irq_enable(struct pt_regs *regs) +{ + if (!arch_irq_disabled_regs(regs)) + local_irq_enable(); +} + +long system_call_exception(struct pt_regs *regs, unsigned long r0); +notrace unsigned long syscall_exit_prepare(unsigned long r3, struct pt_regs *regs, long scv); +notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs); +notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs); +#ifdef CONFIG_PPC64 +unsigned long syscall_exit_restart(unsigned long r3, struct pt_regs *regs); +unsigned long interrupt_exit_user_restart(struct pt_regs *regs); +unsigned long interrupt_exit_kernel_restart(struct pt_regs *regs); +#endif + +#endif /* __ASSEMBLER__ */ + +#endif /* _ASM_POWERPC_INTERRUPT_H */ diff --git a/arch/powerpc/include/asm/io-defs.h b/arch/powerpc/include/asm/io-defs.h index 44d7927aec69..5c2be9b54a9d 100644 --- a/arch/powerpc/include/asm/io-defs.h +++ b/arch/powerpc/include/asm/io-defs.h @@ -1,60 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* This file is meant to be include multiple times by other headers */ -/* last 2 argments are used by platforms/cell/io-workarounds.[ch] */ -DEF_PCI_AC_RET(readb, u8, (const PCI_IO_ADDR addr), (addr), mem, addr) -DEF_PCI_AC_RET(readw, u16, (const PCI_IO_ADDR addr), (addr), mem, addr) -DEF_PCI_AC_RET(readl, u32, (const PCI_IO_ADDR addr), (addr), mem, addr) -DEF_PCI_AC_RET(readw_be, u16, (const PCI_IO_ADDR addr), (addr), mem, addr) -DEF_PCI_AC_RET(readl_be, u32, (const PCI_IO_ADDR addr), (addr), mem, addr) -DEF_PCI_AC_NORET(writeb, (u8 val, PCI_IO_ADDR addr), (val, addr), mem, addr) -DEF_PCI_AC_NORET(writew, (u16 val, PCI_IO_ADDR addr), (val, addr), mem, addr) -DEF_PCI_AC_NORET(writel, (u32 val, PCI_IO_ADDR addr), (val, addr), mem, addr) -DEF_PCI_AC_NORET(writew_be, (u16 val, PCI_IO_ADDR addr), (val, addr), mem, addr) -DEF_PCI_AC_NORET(writel_be, (u32 val, PCI_IO_ADDR addr), (val, addr), mem, addr) - -#ifdef __powerpc64__ -DEF_PCI_AC_RET(readq, u64, (const PCI_IO_ADDR addr), (addr), mem, addr) -DEF_PCI_AC_RET(readq_be, u64, (const PCI_IO_ADDR addr), (addr), mem, addr) -DEF_PCI_AC_NORET(writeq, (u64 val, PCI_IO_ADDR addr), (val, addr), mem, addr) -DEF_PCI_AC_NORET(writeq_be, (u64 val, PCI_IO_ADDR addr), (val, addr), mem, addr) -#endif /* __powerpc64__ */ - -DEF_PCI_AC_RET(inb, u8, (unsigned long port), (port), pio, port) -DEF_PCI_AC_RET(inw, u16, (unsigned long port), (port), pio, port) -DEF_PCI_AC_RET(inl, u32, (unsigned long port), (port), pio, port) -DEF_PCI_AC_NORET(outb, (u8 val, unsigned long port), (val, port), pio, port) -DEF_PCI_AC_NORET(outw, (u16 val, unsigned long port), (val, port), pio, port) -DEF_PCI_AC_NORET(outl, (u32 val, unsigned long port), (val, port), pio, port) - -DEF_PCI_AC_NORET(readsb, (const PCI_IO_ADDR a, void *b, unsigned long c), - (a, b, c), mem, a) -DEF_PCI_AC_NORET(readsw, (const PCI_IO_ADDR a, void *b, unsigned long c), - (a, b, c), mem, a) -DEF_PCI_AC_NORET(readsl, (const PCI_IO_ADDR a, void *b, unsigned long c), - (a, b, c), mem, a) -DEF_PCI_AC_NORET(writesb, (PCI_IO_ADDR a, const void *b, unsigned long c), - (a, b, c), mem, a) -DEF_PCI_AC_NORET(writesw, (PCI_IO_ADDR a, const void *b, unsigned long c), - (a, b, c), mem, a) -DEF_PCI_AC_NORET(writesl, (PCI_IO_ADDR a, const void *b, unsigned long c), - (a, b, c), mem, a) - -DEF_PCI_AC_NORET(insb, (unsigned long p, void *b, unsigned long c), - (p, b, c), pio, p) -DEF_PCI_AC_NORET(insw, (unsigned long p, void *b, unsigned long c), - (p, b, c), pio, p) -DEF_PCI_AC_NORET(insl, (unsigned long p, void *b, unsigned long c), - (p, b, c), pio, p) -DEF_PCI_AC_NORET(outsb, (unsigned long p, const void *b, unsigned long c), - (p, b, c), pio, p) -DEF_PCI_AC_NORET(outsw, (unsigned long p, const void *b, unsigned long c), - (p, b, c), pio, p) -DEF_PCI_AC_NORET(outsl, (unsigned long p, const void *b, unsigned long c), - (p, b, c), pio, p) - -DEF_PCI_AC_NORET(memset_io, (PCI_IO_ADDR a, int c, unsigned long n), - (a, c, n), mem, a) -DEF_PCI_AC_NORET(memcpy_fromio, (void *d, const PCI_IO_ADDR s, unsigned long n), - (d, s, n), mem, s) -DEF_PCI_AC_NORET(memcpy_toio, (PCI_IO_ADDR d, const void *s, unsigned long n), - (d, s, n), mem, d) +DEF_PCI_AC_RET(inb, u8, (unsigned long port), (port)) +DEF_PCI_AC_RET(inw, u16, (unsigned long port), (port)) +DEF_PCI_AC_RET(inl, u32, (unsigned long port), (port)) +DEF_PCI_AC_NORET(outb, (u8 val, unsigned long port), (val, port)) +DEF_PCI_AC_NORET(outw, (u16 val, unsigned long port), (val, port)) +DEF_PCI_AC_NORET(outl, (u32 val, unsigned long port), (val, port)) +DEF_PCI_AC_NORET(insb, (unsigned long p, void *b, unsigned long c), (p, b, c)) +DEF_PCI_AC_NORET(insw, (unsigned long p, void *b, unsigned long c), (p, b, c)) +DEF_PCI_AC_NORET(insl, (unsigned long p, void *b, unsigned long c), (p, b, c)) +DEF_PCI_AC_NORET(outsb, (unsigned long p, const void *b, unsigned long c), (p, b, c)) +DEF_PCI_AC_NORET(outsw, (unsigned long p, const void *b, unsigned long c), (p, b, c)) +DEF_PCI_AC_NORET(outsl, (unsigned long p, const void *b, unsigned long c), (p, b, c)) diff --git a/arch/powerpc/include/asm/io-workarounds.h b/arch/powerpc/include/asm/io-workarounds.h deleted file mode 100644 index f96dd096ff4e..000000000000 --- a/arch/powerpc/include/asm/io-workarounds.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Support PCI IO workaround - * - * (C) Copyright 2007-2008 TOSHIBA CORPORATION - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#ifndef _IO_WORKAROUNDS_H -#define _IO_WORKAROUNDS_H - -#include <linux/io.h> -#include <asm/pci-bridge.h> - -/* Bus info */ -struct iowa_bus { - struct pci_controller *phb; - struct ppc_pci_io *ops; - void *private; -}; - -void iowa_register_bus(struct pci_controller *, struct ppc_pci_io *, - int (*)(struct iowa_bus *, void *), void *); -struct iowa_bus *iowa_mem_find_bus(const PCI_IO_ADDR); -struct iowa_bus *iowa_pio_find_bus(unsigned long); - -extern struct ppc_pci_io spiderpci_ops; -extern int spiderpci_iowa_init(struct iowa_bus *, void *); - -#define SPIDER_PCI_REG_BASE 0xd000 -#define SPIDER_PCI_REG_SIZE 0x1000 -#define SPIDER_PCI_VCI_CNTL_STAT 0x0110 -#define SPIDER_PCI_DUMMY_READ 0x0810 -#define SPIDER_PCI_DUMMY_READ_BASE 0x0814 - -#endif /* _IO_WORKAROUNDS_H */ diff --git a/arch/powerpc/include/asm/io.h b/arch/powerpc/include/asm/io.h index dd15e5e37d6d..7a89754842d6 100644 --- a/arch/powerpc/include/asm/io.h +++ b/arch/powerpc/include/asm/io.h @@ -1,14 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _ASM_POWERPC_IO_H #define _ASM_POWERPC_IO_H #ifdef __KERNEL__ -#define ARCH_HAS_IOREMAP_WC - /* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ /* Check of existence of legacy devices */ @@ -21,36 +16,28 @@ extern struct pci_dev *isa_bridge_pcidev; /* * has legacy ISA devices ? */ -#define arch_has_dev_port() (isa_bridge_pcidev != NULL) +#define arch_has_dev_port() (isa_bridge_pcidev != NULL || isa_io_special) #endif #include <linux/device.h> -#include <linux/io.h> - #include <linux/compiler.h> +#include <linux/mm.h> #include <asm/page.h> #include <asm/byteorder.h> #include <asm/synch.h> #include <asm/delay.h> +#include <asm/mmiowb.h> #include <asm/mmu.h> -#include <asm-generic/iomap.h> - -#ifdef CONFIG_PPC64 -#include <asm/paca.h> -#endif - #define SIO_CONFIG_RA 0x398 #define SIO_CONFIG_RD 0x399 -#define SLOW_DOWN_IO - /* 32 bits uses slightly different variables for the various IO * bases. Most of this file only uses _IO_BASE though which we * define properly based on the platform */ #ifndef CONFIG_PCI -#define _IO_BASE 0 +#define _IO_BASE POISON_POINTER_DELTA #define _ISA_MEM_BASE 0 #define PCI_DRAM_OFFSET 0 #elif defined(CONFIG_PPC32) @@ -69,8 +56,18 @@ extern unsigned long pci_dram_offset; extern resource_size_t isa_mem_base; -#if defined(CONFIG_PPC32) && defined(CONFIG_PPC_INDIRECT_IO) -#error CONFIG_PPC_INDIRECT_IO is not yet supported on 32 bits +/* Boolean set by platform if PIO accesses are suppored while _IO_BASE + * is not set or addresses cannot be translated to MMIO. This is typically + * set when the platform supports "special" PIO accesses via a non memory + * mapped mechanism, and allows things like the early udbg UART code to + * function. + */ +extern bool isa_io_special; + +#ifdef CONFIG_PPC32 +#ifdef CONFIG_PPC_INDIRECT_PIO +#error CONFIG_PPC_INDIRECT_PIO is not yet supported on 32 bits +#endif #endif /* @@ -83,44 +80,51 @@ extern resource_size_t isa_mem_base; * * in_8, in_le16, in_be16, in_le32, in_be32, in_le64, in_be64 * out_8, out_le16, out_be16, out_le32, out_be32, out_le64, out_be64 - * _insb, _insw_ns, _insl_ns, _outsb, _outsw_ns, _outsl_ns + * _insb, _insw, _insl, _outsb, _outsw, _outsl * * Those operate directly on a kernel virtual address. Note that the prototype * for the out_* accessors has the arguments in opposite order from the usual * linux PCI accessors. Unlike those, they take the address first and the value * next. - * - * Note: I might drop the _ns suffix on the stream operations soon as it is - * simply normal for stream operations to not swap in the first place. - * */ -#ifdef CONFIG_PPC64 -#define IO_SET_SYNC_FLAG() do { local_paca->io_sync = 1; } while(0) -#else -#define IO_SET_SYNC_FLAG() -#endif - -/* gcc 4.0 and older doesn't have 'Z' constraint */ -#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ == 0) -#define DEF_MMIO_IN_LE(name, size, insn) \ +/* -mprefixed can generate offsets beyond range, fall back hack */ +#ifdef CONFIG_PPC_KERNEL_PREFIXED +#define DEF_MMIO_IN_X(name, size, insn) \ static inline u##size name(const volatile u##size __iomem *addr) \ { \ u##size ret; \ __asm__ __volatile__("sync;"#insn" %0,0,%1;twi 0,%0,0;isync" \ - : "=r" (ret) : "r" (addr), "m" (*addr) : "memory"); \ + : "=r" (ret) : "r" (addr) : "memory"); \ return ret; \ } -#define DEF_MMIO_OUT_LE(name, size, insn) \ +#define DEF_MMIO_OUT_X(name, size, insn) \ static inline void name(volatile u##size __iomem *addr, u##size val) \ { \ - __asm__ __volatile__("sync;"#insn" %1,0,%2" \ - : "=m" (*addr) : "r" (val), "r" (addr) : "memory"); \ - IO_SET_SYNC_FLAG(); \ + __asm__ __volatile__("sync;"#insn" %1,0,%0" \ + : : "r" (addr), "r" (val) : "memory"); \ + mmiowb_set_pending(); \ } -#else /* newer gcc */ -#define DEF_MMIO_IN_LE(name, size, insn) \ + +#define DEF_MMIO_IN_D(name, size, insn) \ +static inline u##size name(const volatile u##size __iomem *addr) \ +{ \ + u##size ret; \ + __asm__ __volatile__("sync;"#insn" %0,0(%1);twi 0,%0,0;isync"\ + : "=r" (ret) : "b" (addr) : "memory"); \ + return ret; \ +} + +#define DEF_MMIO_OUT_D(name, size, insn) \ +static inline void name(volatile u##size __iomem *addr, u##size val) \ +{ \ + __asm__ __volatile__("sync;"#insn" %1,0(%0)" \ + : : "b" (addr), "r" (val) : "memory"); \ + mmiowb_set_pending(); \ +} +#else +#define DEF_MMIO_IN_X(name, size, insn) \ static inline u##size name(const volatile u##size __iomem *addr) \ { \ u##size ret; \ @@ -129,48 +133,63 @@ static inline u##size name(const volatile u##size __iomem *addr) \ return ret; \ } -#define DEF_MMIO_OUT_LE(name, size, insn) \ +#define DEF_MMIO_OUT_X(name, size, insn) \ static inline void name(volatile u##size __iomem *addr, u##size val) \ { \ __asm__ __volatile__("sync;"#insn" %1,%y0" \ : "=Z" (*addr) : "r" (val) : "memory"); \ - IO_SET_SYNC_FLAG(); \ + mmiowb_set_pending(); \ } -#endif -#define DEF_MMIO_IN_BE(name, size, insn) \ +#define DEF_MMIO_IN_D(name, size, insn) \ static inline u##size name(const volatile u##size __iomem *addr) \ { \ u##size ret; \ __asm__ __volatile__("sync;"#insn"%U1%X1 %0,%1;twi 0,%0,0;isync"\ - : "=r" (ret) : "m" (*addr) : "memory"); \ + : "=r" (ret) : "m<>" (*addr) : "memory"); \ return ret; \ } -#define DEF_MMIO_OUT_BE(name, size, insn) \ +#define DEF_MMIO_OUT_D(name, size, insn) \ static inline void name(volatile u##size __iomem *addr, u##size val) \ { \ __asm__ __volatile__("sync;"#insn"%U0%X0 %1,%0" \ - : "=m" (*addr) : "r" (val) : "memory"); \ - IO_SET_SYNC_FLAG(); \ + : "=m<>" (*addr) : "r" (val) : "memory"); \ + mmiowb_set_pending(); \ } +#endif + +DEF_MMIO_IN_D(in_8, 8, lbz); +DEF_MMIO_OUT_D(out_8, 8, stb); + +#ifdef __BIG_ENDIAN__ +DEF_MMIO_IN_D(in_be16, 16, lhz); +DEF_MMIO_IN_D(in_be32, 32, lwz); +DEF_MMIO_IN_X(in_le16, 16, lhbrx); +DEF_MMIO_IN_X(in_le32, 32, lwbrx); +DEF_MMIO_OUT_D(out_be16, 16, sth); +DEF_MMIO_OUT_D(out_be32, 32, stw); +DEF_MMIO_OUT_X(out_le16, 16, sthbrx); +DEF_MMIO_OUT_X(out_le32, 32, stwbrx); +#else +DEF_MMIO_IN_X(in_be16, 16, lhbrx); +DEF_MMIO_IN_X(in_be32, 32, lwbrx); +DEF_MMIO_IN_D(in_le16, 16, lhz); +DEF_MMIO_IN_D(in_le32, 32, lwz); -DEF_MMIO_IN_BE(in_8, 8, lbz); -DEF_MMIO_IN_BE(in_be16, 16, lhz); -DEF_MMIO_IN_BE(in_be32, 32, lwz); -DEF_MMIO_IN_LE(in_le16, 16, lhbrx); -DEF_MMIO_IN_LE(in_le32, 32, lwbrx); +DEF_MMIO_OUT_X(out_be16, 16, sthbrx); +DEF_MMIO_OUT_X(out_be32, 32, stwbrx); +DEF_MMIO_OUT_D(out_le16, 16, sth); +DEF_MMIO_OUT_D(out_le32, 32, stw); -DEF_MMIO_OUT_BE(out_8, 8, stb); -DEF_MMIO_OUT_BE(out_be16, 16, sth); -DEF_MMIO_OUT_BE(out_be32, 32, stw); -DEF_MMIO_OUT_LE(out_le16, 16, sthbrx); -DEF_MMIO_OUT_LE(out_le32, 32, stwbrx); +#endif /* __BIG_ENDIAN */ #ifdef __powerpc64__ -DEF_MMIO_OUT_BE(out_be64, 64, std); -DEF_MMIO_IN_BE(in_be64, 64, ld); + +#ifdef __BIG_ENDIAN__ +DEF_MMIO_OUT_D(out_be64, 64, std); +DEF_MMIO_IN_D(in_be64, 64, ld); /* There is no asm instructions for 64 bits reverse loads and stores */ static inline u64 in_le64(const volatile u64 __iomem *addr) @@ -182,6 +201,22 @@ static inline void out_le64(volatile u64 __iomem *addr, u64 val) { out_be64(addr, swab64(val)); } +#else +DEF_MMIO_OUT_D(out_le64, 64, std); +DEF_MMIO_IN_D(in_le64, 64, ld); + +/* There is no asm instructions for 64 bits reverse loads and stores */ +static inline u64 in_be64(const volatile u64 __iomem *addr) +{ + return swab64(in_le64(addr)); +} + +static inline void out_be64(volatile u64 __iomem *addr, u64 val) +{ + out_le64(addr, swab64(val)); +} + +#endif #endif /* __powerpc64__ */ /* @@ -189,19 +224,10 @@ static inline void out_le64(volatile u64 __iomem *addr, u64 val) */ extern void _insb(const volatile u8 __iomem *addr, void *buf, long count); extern void _outsb(volatile u8 __iomem *addr,const void *buf,long count); -extern void _insw_ns(const volatile u16 __iomem *addr, void *buf, long count); -extern void _outsw_ns(volatile u16 __iomem *addr, const void *buf, long count); -extern void _insl_ns(const volatile u32 __iomem *addr, void *buf, long count); -extern void _outsl_ns(volatile u32 __iomem *addr, const void *buf, long count); - -/* The _ns naming is historical and will be removed. For now, just #define - * the non _ns equivalent names - */ -#define _insw _insw_ns -#define _insl _insl_ns -#define _outsw _outsw_ns -#define _outsl _outsl_ns - +extern void _insw(const volatile u16 __iomem *addr, void *buf, long count); +extern void _outsw(volatile u16 __iomem *addr, const void *buf, long count); +extern void _insl(const volatile u32 __iomem *addr, void *buf, long count); +extern void _outsl(volatile u32 __iomem *addr, const void *buf, long count); /* * memset_io, memcpy_toio, memcpy_fromio base implementations are out of line @@ -222,8 +248,8 @@ extern void _memcpy_toio(volatile void __iomem *dest, const void *src, * for PowerPC is as close as possible to the x86 version of these, and thus * provides fairly heavy weight barriers for the non-raw versions * - * In addition, they support a hook mechanism when CONFIG_PPC_INDIRECT_IO - * allowing the platform to provide its own implementation of some or all + * In addition, they support a hook mechanism when CONFIG_PPC_INDIRECT_PIO + * is set allowing the platform to provide its own implementation of some * of the accessors. */ @@ -235,95 +261,96 @@ extern void _memcpy_toio(volatile void __iomem *dest, const void *src, #include <asm/eeh.h> #endif -/* Shortcut to the MMIO argument pointer */ -#define PCI_IO_ADDR volatile void __iomem * - -/* Indirect IO address tokens: - * - * When CONFIG_PPC_INDIRECT_IO is set, the platform can provide hooks - * on all IOs. (Note that this is all 64 bits only for now) - * - * To help platforms who may need to differenciate MMIO addresses in - * their hooks, a bitfield is reserved for use by the platform near the - * top of MMIO addresses (not PIO, those have to cope the hard way). - * - * This bit field is 12 bits and is at the top of the IO virtual - * addresses PCI_IO_INDIRECT_TOKEN_MASK. - * - * The kernel virtual space is thus: - * - * 0xD000000000000000 : vmalloc - * 0xD000080000000000 : PCI PHB IO space - * 0xD000080080000000 : ioremap - * 0xD0000fffffffffff : end of ioremap region - * - * Since the top 4 bits are reserved as the region ID, we use thus - * the next 12 bits and keep 4 bits available for the future if the - * virtual address space is ever to be extended. - * - * The direct IO mapping operations will then mask off those bits - * before doing the actual access, though that only happen when - * CONFIG_PPC_INDIRECT_IO is set, thus be careful when you use that - * mechanism - */ - -#ifdef CONFIG_PPC_INDIRECT_IO -#define PCI_IO_IND_TOKEN_MASK 0x0fff000000000000ul -#define PCI_IO_IND_TOKEN_SHIFT 48 -#define PCI_FIX_ADDR(addr) \ - ((PCI_IO_ADDR)(((unsigned long)(addr)) & ~PCI_IO_IND_TOKEN_MASK)) -#define PCI_GET_ADDR_TOKEN(addr) \ - (((unsigned long)(addr) & PCI_IO_IND_TOKEN_MASK) >> \ - PCI_IO_IND_TOKEN_SHIFT) -#define PCI_SET_ADDR_TOKEN(addr, token) \ -do { \ - unsigned long __a = (unsigned long)(addr); \ - __a &= ~PCI_IO_IND_TOKEN_MASK; \ - __a |= ((unsigned long)(token)) << PCI_IO_IND_TOKEN_SHIFT; \ - (addr) = (void __iomem *)__a; \ -} while(0) -#else -#define PCI_FIX_ADDR(addr) (addr) -#endif - +#define _IO_PORT(port) ((volatile void __iomem *)(_IO_BASE + (port))) +#ifdef __powerpc64__ /* - * Non ordered and non-swapping "raw" accessors + * Real mode versions of raw accessors. Those instructions are only supposed + * to be used in hypervisor real mode as per the architecture spec. */ +static inline void __raw_rm_writeb(u8 val, volatile void __iomem *paddr) +{ + __asm__ __volatile__(".machine push; \ + .machine power6; \ + stbcix %0,0,%1; \ + .machine pop;" + : : "r" (val), "r" (paddr) : "memory"); +} -static inline unsigned char __raw_readb(const volatile void __iomem *addr) +static inline void __raw_rm_writew(u16 val, volatile void __iomem *paddr) { - return *(volatile unsigned char __force *)PCI_FIX_ADDR(addr); + __asm__ __volatile__(".machine push; \ + .machine power6; \ + sthcix %0,0,%1; \ + .machine pop;" + : : "r" (val), "r" (paddr) : "memory"); } -static inline unsigned short __raw_readw(const volatile void __iomem *addr) + +static inline void __raw_rm_writel(u32 val, volatile void __iomem *paddr) { - return *(volatile unsigned short __force *)PCI_FIX_ADDR(addr); + __asm__ __volatile__(".machine push; \ + .machine power6; \ + stwcix %0,0,%1; \ + .machine pop;" + : : "r" (val), "r" (paddr) : "memory"); } -static inline unsigned int __raw_readl(const volatile void __iomem *addr) + +static inline void __raw_rm_writeq(u64 val, volatile void __iomem *paddr) { - return *(volatile unsigned int __force *)PCI_FIX_ADDR(addr); + __asm__ __volatile__(".machine push; \ + .machine power6; \ + stdcix %0,0,%1; \ + .machine pop;" + : : "r" (val), "r" (paddr) : "memory"); } -static inline void __raw_writeb(unsigned char v, volatile void __iomem *addr) + +static inline void __raw_rm_writeq_be(u64 val, volatile void __iomem *paddr) { - *(volatile unsigned char __force *)PCI_FIX_ADDR(addr) = v; + __raw_rm_writeq((__force u64)cpu_to_be64(val), paddr); } -static inline void __raw_writew(unsigned short v, volatile void __iomem *addr) + +static inline u8 __raw_rm_readb(volatile void __iomem *paddr) { - *(volatile unsigned short __force *)PCI_FIX_ADDR(addr) = v; + u8 ret; + __asm__ __volatile__(".machine push; \ + .machine power6; \ + lbzcix %0,0, %1; \ + .machine pop;" + : "=r" (ret) : "r" (paddr) : "memory"); + return ret; } -static inline void __raw_writel(unsigned int v, volatile void __iomem *addr) + +static inline u16 __raw_rm_readw(volatile void __iomem *paddr) { - *(volatile unsigned int __force *)PCI_FIX_ADDR(addr) = v; + u16 ret; + __asm__ __volatile__(".machine push; \ + .machine power6; \ + lhzcix %0,0, %1; \ + .machine pop;" + : "=r" (ret) : "r" (paddr) : "memory"); + return ret; } -#ifdef __powerpc64__ -static inline unsigned long __raw_readq(const volatile void __iomem *addr) +static inline u32 __raw_rm_readl(volatile void __iomem *paddr) { - return *(volatile unsigned long __force *)PCI_FIX_ADDR(addr); + u32 ret; + __asm__ __volatile__(".machine push; \ + .machine power6; \ + lwzcix %0,0, %1; \ + .machine pop;" + : "=r" (ret) : "r" (paddr) : "memory"); + return ret; } -static inline void __raw_writeq(unsigned long v, volatile void __iomem *addr) + +static inline u64 __raw_rm_readq(volatile void __iomem *paddr) { - *(volatile unsigned long __force *)PCI_FIX_ADDR(addr) = v; + u64 ret; + __asm__ __volatile__(".machine push; \ + .machine power6; \ + ldcix %0,0, %1; \ + .machine pop;" + : "=r" (ret) : "r" (paddr) : "memory"); + return ret; } #endif /* __powerpc64__ */ @@ -358,13 +385,10 @@ static inline unsigned int name(unsigned int port) \ "5: li %0,-1\n" \ " b 4b\n" \ ".previous\n" \ - ".section __ex_table,\"a\"\n" \ - " .align 2\n" \ - " .long 0b,5b\n" \ - " .long 1b,5b\n" \ - " .long 2b,5b\n" \ - " .long 3b,5b\n" \ - ".previous" \ + EX_TABLE(0b, 5b) \ + EX_TABLE(1b, 5b) \ + EX_TABLE(2b, 5b) \ + EX_TABLE(3b, 5b) \ : "=&r" (x) \ : "r" (port + _IO_BASE) \ : "memory"); \ @@ -379,11 +403,8 @@ static inline void name(unsigned int val, unsigned int port) \ "0:" op " %0,0,%1\n" \ "1: sync\n" \ "2:\n" \ - ".section __ex_table,\"a\"\n" \ - " .align 2\n" \ - " .long 0b,2b\n" \ - " .long 1b,2b\n" \ - ".previous" \ + EX_TABLE(0b, 2b) \ + EX_TABLE(1b, 2b) \ : : "r" (val), "r" (port + _IO_BASE) \ : "memory"); \ } @@ -412,30 +433,23 @@ __do_out_asm(_rec_outl, "stwbrx") * possible to hook directly at the toplevel PIO operation if they have to * be handled differently */ -#define __do_writeb(val, addr) out_8(PCI_FIX_ADDR(addr), val) -#define __do_writew(val, addr) out_le16(PCI_FIX_ADDR(addr), val) -#define __do_writel(val, addr) out_le32(PCI_FIX_ADDR(addr), val) -#define __do_writeq(val, addr) out_le64(PCI_FIX_ADDR(addr), val) -#define __do_writew_be(val, addr) out_be16(PCI_FIX_ADDR(addr), val) -#define __do_writel_be(val, addr) out_be32(PCI_FIX_ADDR(addr), val) -#define __do_writeq_be(val, addr) out_be64(PCI_FIX_ADDR(addr), val) #ifdef CONFIG_EEH -#define __do_readb(addr) eeh_readb(PCI_FIX_ADDR(addr)) -#define __do_readw(addr) eeh_readw(PCI_FIX_ADDR(addr)) -#define __do_readl(addr) eeh_readl(PCI_FIX_ADDR(addr)) -#define __do_readq(addr) eeh_readq(PCI_FIX_ADDR(addr)) -#define __do_readw_be(addr) eeh_readw_be(PCI_FIX_ADDR(addr)) -#define __do_readl_be(addr) eeh_readl_be(PCI_FIX_ADDR(addr)) -#define __do_readq_be(addr) eeh_readq_be(PCI_FIX_ADDR(addr)) +#define __do_readb(addr) eeh_readb(addr) +#define __do_readw(addr) eeh_readw(addr) +#define __do_readl(addr) eeh_readl(addr) +#define __do_readq(addr) eeh_readq(addr) +#define __do_readw_be(addr) eeh_readw_be(addr) +#define __do_readl_be(addr) eeh_readl_be(addr) +#define __do_readq_be(addr) eeh_readq_be(addr) #else /* CONFIG_EEH */ -#define __do_readb(addr) in_8(PCI_FIX_ADDR(addr)) -#define __do_readw(addr) in_le16(PCI_FIX_ADDR(addr)) -#define __do_readl(addr) in_le32(PCI_FIX_ADDR(addr)) -#define __do_readq(addr) in_le64(PCI_FIX_ADDR(addr)) -#define __do_readw_be(addr) in_be16(PCI_FIX_ADDR(addr)) -#define __do_readl_be(addr) in_be32(PCI_FIX_ADDR(addr)) -#define __do_readq_be(addr) in_be64(PCI_FIX_ADDR(addr)) +#define __do_readb(addr) in_8(addr) +#define __do_readw(addr) in_le16(addr) +#define __do_readl(addr) in_le32(addr) +#define __do_readq(addr) in_le64(addr) +#define __do_readw_be(addr) in_be16(addr) +#define __do_readl_be(addr) in_be32(addr) +#define __do_readq_be(addr) in_be64(addr) #endif /* !defined(CONFIG_EEH) */ #ifdef CONFIG_PPC32 @@ -446,64 +460,185 @@ __do_out_asm(_rec_outl, "stwbrx") #define __do_inw(port) _rec_inw(port) #define __do_inl(port) _rec_inl(port) #else /* CONFIG_PPC32 */ -#define __do_outb(val, port) writeb(val,(PCI_IO_ADDR)_IO_BASE+port); -#define __do_outw(val, port) writew(val,(PCI_IO_ADDR)_IO_BASE+port); -#define __do_outl(val, port) writel(val,(PCI_IO_ADDR)_IO_BASE+port); -#define __do_inb(port) readb((PCI_IO_ADDR)_IO_BASE + port); -#define __do_inw(port) readw((PCI_IO_ADDR)_IO_BASE + port); -#define __do_inl(port) readl((PCI_IO_ADDR)_IO_BASE + port); +#define __do_outb(val, port) writeb(val,_IO_PORT(port)); +#define __do_outw(val, port) writew(val,_IO_PORT(port)); +#define __do_outl(val, port) writel(val,_IO_PORT(port)); +#define __do_inb(port) readb(_IO_PORT(port)); +#define __do_inw(port) readw(_IO_PORT(port)); +#define __do_inl(port) readl(_IO_PORT(port)); #endif /* !CONFIG_PPC32 */ #ifdef CONFIG_EEH -#define __do_readsb(a, b, n) eeh_readsb(PCI_FIX_ADDR(a), (b), (n)) -#define __do_readsw(a, b, n) eeh_readsw(PCI_FIX_ADDR(a), (b), (n)) -#define __do_readsl(a, b, n) eeh_readsl(PCI_FIX_ADDR(a), (b), (n)) +#define __do_readsb(a, b, n) eeh_readsb(a, (b), (n)) +#define __do_readsw(a, b, n) eeh_readsw(a, (b), (n)) +#define __do_readsl(a, b, n) eeh_readsl(a, (b), (n)) #else /* CONFIG_EEH */ -#define __do_readsb(a, b, n) _insb(PCI_FIX_ADDR(a), (b), (n)) -#define __do_readsw(a, b, n) _insw(PCI_FIX_ADDR(a), (b), (n)) -#define __do_readsl(a, b, n) _insl(PCI_FIX_ADDR(a), (b), (n)) +#define __do_readsb(a, b, n) _insb(a, (b), (n)) +#define __do_readsw(a, b, n) _insw(a, (b), (n)) +#define __do_readsl(a, b, n) _insl(a, (b), (n)) #endif /* !CONFIG_EEH */ -#define __do_writesb(a, b, n) _outsb(PCI_FIX_ADDR(a),(b),(n)) -#define __do_writesw(a, b, n) _outsw(PCI_FIX_ADDR(a),(b),(n)) -#define __do_writesl(a, b, n) _outsl(PCI_FIX_ADDR(a),(b),(n)) - -#define __do_insb(p, b, n) readsb((PCI_IO_ADDR)_IO_BASE+(p), (b), (n)) -#define __do_insw(p, b, n) readsw((PCI_IO_ADDR)_IO_BASE+(p), (b), (n)) -#define __do_insl(p, b, n) readsl((PCI_IO_ADDR)_IO_BASE+(p), (b), (n)) -#define __do_outsb(p, b, n) writesb((PCI_IO_ADDR)_IO_BASE+(p),(b),(n)) -#define __do_outsw(p, b, n) writesw((PCI_IO_ADDR)_IO_BASE+(p),(b),(n)) -#define __do_outsl(p, b, n) writesl((PCI_IO_ADDR)_IO_BASE+(p),(b),(n)) - -#define __do_memset_io(addr, c, n) \ - _memset_io(PCI_FIX_ADDR(addr), c, n) -#define __do_memcpy_toio(dst, src, n) \ - _memcpy_toio(PCI_FIX_ADDR(dst), src, n) +#define __do_writesb(a, b, n) _outsb(a, (b), (n)) +#define __do_writesw(a, b, n) _outsw(a, (b), (n)) +#define __do_writesl(a, b, n) _outsl(a, (b), (n)) + +#define __do_insb(p, b, n) readsb(_IO_PORT(p), (b), (n)) +#define __do_insw(p, b, n) readsw(_IO_PORT(p), (b), (n)) +#define __do_insl(p, b, n) readsl(_IO_PORT(p), (b), (n)) +#define __do_outsb(p, b, n) writesb(_IO_PORT(p),(b),(n)) +#define __do_outsw(p, b, n) writesw(_IO_PORT(p),(b),(n)) +#define __do_outsl(p, b, n) writesl(_IO_PORT(p),(b),(n)) #ifdef CONFIG_EEH #define __do_memcpy_fromio(dst, src, n) \ - eeh_memcpy_fromio(dst, PCI_FIX_ADDR(src), n) + eeh_memcpy_fromio(dst, src, n) #else /* CONFIG_EEH */ #define __do_memcpy_fromio(dst, src, n) \ - _memcpy_fromio(dst,PCI_FIX_ADDR(src),n) + _memcpy_fromio(dst, src, n) #endif /* !CONFIG_EEH */ -#ifdef CONFIG_PPC_INDIRECT_PIO -#define DEF_PCI_HOOK_pio(x) x -#else -#define DEF_PCI_HOOK_pio(x) NULL -#endif +static inline u8 readb(const volatile void __iomem *addr) +{ + return __do_readb(addr); +} +#define readb readb + +static inline u16 readw(const volatile void __iomem *addr) +{ + return __do_readw(addr); +} +#define readw readw + +static inline u32 readl(const volatile void __iomem *addr) +{ + return __do_readl(addr); +} +#define readl readl + +static inline u16 readw_be(const volatile void __iomem *addr) +{ + return __do_readw_be(addr); +} + +static inline u32 readl_be(const volatile void __iomem *addr) +{ + return __do_readl_be(addr); +} + +static inline void writeb(u8 val, volatile void __iomem *addr) +{ + out_8(addr, val); +} +#define writeb writeb + +static inline void writew(u16 val, volatile void __iomem *addr) +{ + out_le16(addr, val); +} +#define writew writew + +static inline void writel(u32 val, volatile void __iomem *addr) +{ + out_le32(addr, val); +} +#define writel writel -#ifdef CONFIG_PPC_INDIRECT_MMIO -#define DEF_PCI_HOOK_mem(x) x +static inline void writew_be(u16 val, volatile void __iomem *addr) +{ + out_be16(addr, val); +} + +static inline void writel_be(u32 val, volatile void __iomem *addr) +{ + out_be32(addr, val); +} + +static inline void readsb(const volatile void __iomem *a, void *b, unsigned long c) +{ + __do_readsb(a, b, c); +} +#define readsb readsb + +static inline void readsw(const volatile void __iomem *a, void *b, unsigned long c) +{ + __do_readsw(a, b, c); +} +#define readsw readsw + +static inline void readsl(const volatile void __iomem *a, void *b, unsigned long c) +{ + __do_readsl(a, b, c); +} +#define readsl readsl + +static inline void writesb(volatile void __iomem *a, const void *b, unsigned long c) +{ + __do_writesb(a, b, c); +} +#define writesb writesb + +static inline void writesw(volatile void __iomem *a, const void *b, unsigned long c) +{ + __do_writesw(a, b, c); +} +#define writesw writesw + +static inline void writesl(volatile void __iomem *a, const void *b, unsigned long c) +{ + __do_writesl(a, b, c); +} +#define writesl writesl + +static inline void memset_io(volatile void __iomem *a, int c, unsigned long n) +{ + _memset_io(a, c, n); +} +#define memset_io memset_io + +static inline void memcpy_fromio(void *d, const volatile void __iomem *s, unsigned long n) +{ + __do_memcpy_fromio(d, s, n); +} +#define memcpy_fromio memcpy_fromio + +static inline void memcpy_toio(volatile void __iomem *d, const void *s, unsigned long n) +{ + _memcpy_toio(d, s, n); +} +#define memcpy_toio memcpy_toio + +#ifdef __powerpc64__ +static inline u64 readq(const volatile void __iomem *addr) +{ + return __do_readq(addr); +} + +static inline u64 readq_be(const volatile void __iomem *addr) +{ + return __do_readq_be(addr); +} + +static inline void writeq(u64 val, volatile void __iomem *addr) +{ + out_le64(addr, val); +} + +static inline void writeq_be(u64 val, volatile void __iomem *addr) +{ + out_be64(addr, val); +} +#endif /* __powerpc64__ */ + +#ifdef CONFIG_PPC_INDIRECT_PIO +#define DEF_PCI_HOOK(x) x #else -#define DEF_PCI_HOOK_mem(x) NULL +#define DEF_PCI_HOOK(x) NULL #endif /* Structure containing all the hooks */ extern struct ppc_pci_io { -#define DEF_PCI_AC_RET(name, ret, at, al, space, aa) ret (*name) at; -#define DEF_PCI_AC_NORET(name, at, al, space, aa) void (*name) at; +#define DEF_PCI_AC_RET(name, ret, at, al) ret (*name) at; +#define DEF_PCI_AC_NORET(name, at, al) void (*name) at; #include <asm/io-defs.h> @@ -513,18 +648,18 @@ extern struct ppc_pci_io { } ppc_pci_io; /* The inline wrappers */ -#define DEF_PCI_AC_RET(name, ret, at, al, space, aa) \ +#define DEF_PCI_AC_RET(name, ret, at, al) \ static inline ret name at \ { \ - if (DEF_PCI_HOOK_##space(ppc_pci_io.name) != NULL) \ + if (DEF_PCI_HOOK(ppc_pci_io.name) != NULL) \ return ppc_pci_io.name al; \ return __do_##name al; \ } -#define DEF_PCI_AC_NORET(name, at, al, space, aa) \ +#define DEF_PCI_AC_NORET(name, at, al) \ static inline void name at \ { \ - if (DEF_PCI_HOOK_##space(ppc_pci_io.name) != NULL) \ + if (DEF_PCI_HOOK(ppc_pci_io.name) != NULL) \ ppc_pci_io.name al; \ else \ __do_##name al; \ @@ -535,50 +670,86 @@ static inline void name at \ #undef DEF_PCI_AC_RET #undef DEF_PCI_AC_NORET -/* Some drivers check for the presence of readq & writeq with - * a #ifdef, so we make them happy here. - */ +// Signal to asm-generic/io.h that we have implemented these. +#define inb inb +#define inw inw +#define inl inl +#define outb outb +#define outw outw +#define outl outl +#define insb insb +#define insw insw +#define insl insl +#define outsb outsb +#define outsw outsw +#define outsl outsl #ifdef __powerpc64__ #define readq readq #define writeq writeq #endif /* - * Convert a physical pointer to a virtual kernel pointer for /dev/mem - * access + * We don't do relaxed operations yet, at least not with this semantic */ -#define xlate_dev_mem_ptr(p) __va(p) - +#define readb_relaxed(addr) readb(addr) +#define readw_relaxed(addr) readw(addr) +#define readl_relaxed(addr) readl(addr) +#define readq_relaxed(addr) readq(addr) +#define writeb_relaxed(v, addr) writeb(v, addr) +#define writew_relaxed(v, addr) writew(v, addr) +#define writel_relaxed(v, addr) writel(v, addr) +#define writeq_relaxed(v, addr) writeq(v, addr) + +#ifndef CONFIG_GENERIC_IOMAP /* - * Convert a virtual cached pointer to an uncached pointer + * Here comes the implementation of the IOMAP interfaces. */ -#define xlate_dev_kmem_ptr(p) p +static inline unsigned int ioread16be(const void __iomem *addr) +{ + return readw_be(addr); +} +#define ioread16be ioread16be -/* - * We don't do relaxed operations yet, at least not with this semantic - */ -#define readb_relaxed(addr) readb(addr) -#define readw_relaxed(addr) readw(addr) -#define readl_relaxed(addr) readl(addr) -#define readq_relaxed(addr) readq(addr) +static inline unsigned int ioread32be(const void __iomem *addr) +{ + return readl_be(addr); +} +#define ioread32be ioread32be -#ifdef CONFIG_PPC32 -#define mmiowb() -#else -/* - * Enforce synchronisation of stores vs. spin_unlock - * (this does it explicitly, though our implementation of spin_unlock - * does it implicitely too) - */ -static inline void mmiowb(void) +#ifdef __powerpc64__ +static inline u64 ioread64be(const void __iomem *addr) { - unsigned long tmp; + return readq_be(addr); +} +#define ioread64be ioread64be +#endif /* __powerpc64__ */ - __asm__ __volatile__("sync; li %0,0; stb %0,%1(13)" - : "=&r" (tmp) : "i" (offsetof(struct paca_struct, io_sync)) - : "memory"); +static inline void iowrite16be(u16 val, void __iomem *addr) +{ + writew_be(val, addr); } -#endif /* !CONFIG_PPC32 */ +#define iowrite16be iowrite16be + +static inline void iowrite32be(u32 val, void __iomem *addr) +{ + writel_be(val, addr); +} +#define iowrite32be iowrite32be + +#ifdef __powerpc64__ +static inline void iowrite64be(u64 val, void __iomem *addr) +{ + writeq_be(val, addr); +} +#define iowrite64be iowrite64be +#endif /* __powerpc64__ */ + +struct pci_dev; +void pci_iounmap(struct pci_dev *dev, void __iomem *addr); +#define pci_iounmap pci_iounmap +void __iomem *ioport_map(unsigned long port, unsigned int len); +#define ioport_map ioport_map +#endif static inline void iosync(void) { @@ -611,7 +782,6 @@ static inline void iosync(void) #define IO_SPACE_LIMIT ~(0UL) - /** * ioremap - map bus memory into CPU space * @address: bus address of the memory @@ -631,48 +801,45 @@ static inline void iosync(void) * * ioremap_prot allows to specify the page flags as an argument and can * also be hooked by the platform via ppc_md. * - * * ioremap_nocache is identical to ioremap - * * * ioremap_wc enables write combining * - * * iounmap undoes such a mapping and can be hooked + * * ioremap_wt enables write through * - * * __ioremap_at (and the pending __iounmap_at) are low level functions to - * create hand-made mappings for use only by the PCI code and cannot - * currently be hooked. Must be page aligned. + * * ioremap_coherent maps coherent cached memory * - * * __ioremap is the low level implementation used by ioremap and - * ioremap_prot and cannot be hooked (but can be used by a hook on one - * of the previous ones) + * * iounmap undoes such a mapping and can be hooked * * * __ioremap_caller is the same as above but takes an explicit caller * reference rather than using __builtin_return_address(0) * - * * __iounmap, is the low level implementation used by iounmap and cannot - * be hooked (but can be used by a hook on iounmap) - * */ extern void __iomem *ioremap(phys_addr_t address, unsigned long size); -extern void __iomem *ioremap_prot(phys_addr_t address, unsigned long size, - unsigned long flags); +#define ioremap ioremap +#define ioremap_prot ioremap_prot extern void __iomem *ioremap_wc(phys_addr_t address, unsigned long size); -#define ioremap_nocache(addr, size) ioremap((addr), (size)) +#define ioremap_wc ioremap_wc -extern void iounmap(volatile void __iomem *addr); +#ifdef CONFIG_PPC32 +void __iomem *ioremap_wt(phys_addr_t address, unsigned long size); +#define ioremap_wt ioremap_wt +#endif -extern void __iomem *__ioremap(phys_addr_t, unsigned long size, - unsigned long flags); -extern void __iomem *__ioremap_caller(phys_addr_t, unsigned long size, - unsigned long flags, void *caller); +void __iomem *ioremap_coherent(phys_addr_t address, unsigned long size); +#define ioremap_cache(addr, size) \ + ioremap_prot((addr), (size), PAGE_KERNEL) -extern void __iounmap(volatile void __iomem *addr); +#define iounmap iounmap -extern void __iomem * __ioremap_at(phys_addr_t pa, void *ea, - unsigned long size, unsigned long flags); -extern void __iounmap_at(void *ea, unsigned long size); +void __iomem *ioremap_phb(phys_addr_t paddr, unsigned long size); + +int early_ioremap_range(unsigned long ea, phys_addr_t pa, + unsigned long size, pgprot_t prot); + +extern void __iomem *__ioremap_caller(phys_addr_t, unsigned long size, + pgprot_t prot, void *caller); /* - * When CONFIG_PPC_INDIRECT_IO is set, we use the generic iomap implementation + * When CONFIG_PPC_INDIRECT_PIO is set, we use the generic iomap implementation * which needs some additional definitions here. They basically allow PIO * space overall to be 1GB. This will work as long as we never try to use * iomap to map MMIO below 1GB which should be fine on ppc64 @@ -684,8 +851,10 @@ extern void __iounmap_at(void *ea, unsigned long size); #define mmio_read16be(addr) readw_be(addr) #define mmio_read32be(addr) readl_be(addr) +#define mmio_read64be(addr) readq_be(addr) #define mmio_write16be(val, addr) writew_be(val, addr) #define mmio_write32be(val, addr) writel_be(val, addr) +#define mmio_write64be(val, addr) writeq_be(val, addr) #define mmio_insb(addr, dst, count) readsb(addr, dst, count) #define mmio_insw(addr, dst, count) readsw(addr, dst, count) #define mmio_insl(addr, dst, count) readsl(addr, dst, count) @@ -705,10 +874,13 @@ extern void __iounmap_at(void *ea, unsigned long size); * almost all conceivable cases a device driver should not be using * this function */ -static inline unsigned long virt_to_phys(volatile void * address) +static inline unsigned long virt_to_phys(const volatile void * address) { + WARN_ON(IS_ENABLED(CONFIG_DEBUG_VIRTUAL) && !virt_addr_valid(address)); + return __pa((unsigned long)address); } +#define virt_to_phys virt_to_phys /** * phys_to_virt - map physical address to virtual @@ -726,14 +898,10 @@ static inline void * phys_to_virt(unsigned long address) { return (void *)__va(address); } +#define phys_to_virt phys_to_virt /* - * Change "struct page" to physical address. - */ -#define page_to_phys(page) ((phys_addr_t)page_to_pfn(page) << PAGE_SHIFT) - -/* - * 32 bits still uses virt_to_bus() for it's implementation of DMA + * 32 bits still uses virt_to_bus() for its implementation of DMA * mappings se we have to keep it defined here. We also have some old * drivers (shame shame shame) that use bus_to_virt() and haven't been * fixed yet so I need to define it here. @@ -746,6 +914,7 @@ static inline unsigned long virt_to_bus(volatile void * address) return 0; return __pa(address) + PCI_DRAM_OFFSET; } +#define virt_to_bus virt_to_bus static inline void * bus_to_virt(unsigned long address) { @@ -753,8 +922,7 @@ static inline void * bus_to_virt(unsigned long address) return NULL; return __va(address - PCI_DRAM_OFFSET); } - -#define page_to_bus(page) (page_to_phys(page) + PCI_DRAM_OFFSET) +#define bus_to_virt bus_to_virt #endif /* CONFIG_PPC32 */ @@ -791,8 +959,15 @@ static inline void * bus_to_virt(unsigned long address) #define clrsetbits_8(addr, clear, set) clrsetbits(8, addr, clear, set) -void __iomem *devm_ioremap_prot(struct device *dev, resource_size_t offset, - size_t size, unsigned long flags); +#include <asm-generic/io.h> + +#ifdef __powerpc64__ +static inline void __raw_writeq_be(unsigned long v, volatile void __iomem *addr) +{ + __raw_writeq((__force unsigned long)cpu_to_be64(v), addr); +} +#define __raw_writeq_be __raw_writeq_be +#endif // __powerpc64__ #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/io_event_irq.h b/arch/powerpc/include/asm/io_event_irq.h index b1a9a1be3c21..290c7530d1b6 100644 --- a/arch/powerpc/include/asm/io_event_irq.h +++ b/arch/powerpc/include/asm/io_event_irq.h @@ -1,10 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Copyright 2010, 2011 Mark Nelson and Tseng-Hui (Frank) Lin, IBM Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef _ASM_POWERPC_IO_EVENT_IRQ_H diff --git a/arch/powerpc/include/asm/iommu.h b/arch/powerpc/include/asm/iommu.h index c34656a8925e..eafdd63cd6c4 100644 --- a/arch/powerpc/include/asm/iommu.h +++ b/arch/powerpc/include/asm/iommu.h @@ -1,21 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation * Rewrite, cleanup: * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef _ASM_IOMMU_H @@ -25,26 +12,69 @@ #include <linux/compiler.h> #include <linux/spinlock.h> #include <linux/device.h> -#include <linux/dma-mapping.h> +#include <linux/dma-map-ops.h> #include <linux/bitops.h> #include <asm/machdep.h> #include <asm/types.h> +#include <asm/pci-bridge.h> +#include <asm/asm-const.h> -#define IOMMU_PAGE_SHIFT 12 -#define IOMMU_PAGE_SIZE (ASM_CONST(1) << IOMMU_PAGE_SHIFT) -#define IOMMU_PAGE_MASK (~((1 << IOMMU_PAGE_SHIFT) - 1)) -#define IOMMU_PAGE_ALIGN(addr) _ALIGN_UP(addr, IOMMU_PAGE_SIZE) +#define IOMMU_PAGE_SHIFT_4K 12 +#define IOMMU_PAGE_SIZE_4K (ASM_CONST(1) << IOMMU_PAGE_SHIFT_4K) +#define IOMMU_PAGE_MASK_4K (~((1 << IOMMU_PAGE_SHIFT_4K) - 1)) +#define IOMMU_PAGE_ALIGN_4K(addr) ALIGN(addr, IOMMU_PAGE_SIZE_4K) + +#define IOMMU_PAGE_SIZE(tblptr) (ASM_CONST(1) << (tblptr)->it_page_shift) +#define IOMMU_PAGE_MASK(tblptr) (~((1 << (tblptr)->it_page_shift) - 1)) +#define IOMMU_PAGE_ALIGN(addr, tblptr) ALIGN(addr, IOMMU_PAGE_SIZE(tblptr)) + +#define DIRECT64_PROPNAME "linux,direct64-ddr-window-info" +#define DMA64_PROPNAME "linux,dma64-ddr-window-info" + +#define MIN_DDW_VPMEM_DMA_WINDOW SZ_2G /* Boot time flags */ extern int iommu_is_off; extern int iommu_force_on; -/* Pure 2^n version of get_order */ -static __inline__ __attribute_const__ int get_iommu_order(unsigned long size) -{ - return __ilog2((size - 1) >> IOMMU_PAGE_SHIFT) + 1; -} +struct iommu_table_ops { + /* + * When called with direction==DMA_NONE, it is equal to clear(). + * uaddr is a linear map address. + */ + int (*set)(struct iommu_table *tbl, + long index, long npages, + unsigned long uaddr, + enum dma_data_direction direction, + unsigned long attrs); +#ifdef CONFIG_IOMMU_API + /* + * Exchanges existing TCE with new TCE plus direction bits; + * returns old TCE and DMA direction mask. + * @tce is a physical address. + */ + int (*xchg_no_kill)(struct iommu_table *tbl, + long index, + unsigned long *hpa, + enum dma_data_direction *direction); + void (*tce_kill)(struct iommu_table *tbl, + unsigned long index, + unsigned long pages); + + __be64 *(*useraddrptr)(struct iommu_table *tbl, long index, bool alloc); +#endif + void (*clear)(struct iommu_table *tbl, + long index, long npages); + /* get() returns a physical address */ + unsigned long (*get)(struct iommu_table *tbl, long index); + void (*flush)(struct iommu_table *tbl); + void (*free)(struct iommu_table *tbl); +}; + +/* These are used by VIO */ +extern struct iommu_table_ops iommu_table_lpar_multi_ops; +extern struct iommu_table_ops iommu_table_pseries_ops; /* * IOMAP_MAX_ORDER defines the largest contiguous block @@ -66,6 +96,9 @@ struct iommu_pool { struct iommu_table { unsigned long it_busno; /* Bus number this table belongs to */ unsigned long it_size; /* Size of iommu table in entries */ + unsigned long it_indirect_levels; + unsigned long it_level_size; + unsigned long it_allocated_size; unsigned long it_offset; /* Offset into global table */ unsigned long it_base; /* mapped address of tce table */ unsigned long it_index; /* which iommu table this is */ @@ -76,68 +109,185 @@ struct iommu_table { struct iommu_pool large_pool; struct iommu_pool pools[IOMMU_NR_POOLS]; unsigned long *it_map; /* A simple allocation bitmap for now */ -#ifdef CONFIG_IOMMU_API - struct iommu_group *it_group; -#endif + unsigned long it_page_shift;/* table iommu page size */ + struct list_head it_group_list;/* List of iommu_table_group_link */ + __be64 *it_userspace; /* userspace view of the table */ + struct iommu_table_ops *it_ops; + struct kref it_kref; + int it_nid; + unsigned long it_reserved_start; /* Start of not-DMA-able (MMIO) area */ + unsigned long it_reserved_end; }; +#define IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry) \ + ((tbl)->it_ops->useraddrptr((tbl), (entry), false)) +#define IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry) \ + ((tbl)->it_ops->useraddrptr((tbl), (entry), true)) + +/* Pure 2^n version of get_order */ +static inline __attribute_const__ +int get_iommu_order(unsigned long size, struct iommu_table *tbl) +{ + return __ilog2((size - 1) >> tbl->it_page_shift) + 1; +} + + struct scatterlist; -static inline void set_iommu_table_base(struct device *dev, void *base) +#ifdef CONFIG_PPC64 + +static inline void set_iommu_table_base(struct device *dev, + struct iommu_table *base) { - dev->archdata.dma_data.iommu_table_base = base; + dev->archdata.iommu_table_base = base; } static inline void *get_iommu_table_base(struct device *dev) { - return dev->archdata.dma_data.iommu_table_base; + return dev->archdata.iommu_table_base; } -/* Frees table for an individual device node */ -extern void iommu_free_table(struct iommu_table *tbl, const char *node_name); +extern int dma_iommu_dma_supported(struct device *dev, u64 mask); + +extern struct iommu_table *iommu_tce_table_get(struct iommu_table *tbl); +extern int iommu_tce_table_put(struct iommu_table *tbl); /* Initializes an iommu_table based in values set in the passed-in * structure */ -extern struct iommu_table *iommu_init_table(struct iommu_table * tbl, - int nid); -extern void iommu_register_group(struct iommu_table *tbl, +extern struct iommu_table *iommu_init_table(struct iommu_table *tbl, + int nid, unsigned long res_start, unsigned long res_end); +bool iommu_table_in_use(struct iommu_table *tbl); +extern void iommu_table_reserve_pages(struct iommu_table *tbl, + unsigned long res_start, unsigned long res_end); +extern void iommu_table_clear(struct iommu_table *tbl); + +#define IOMMU_TABLE_GROUP_MAX_TABLES 2 + +struct iommu_table_group; + +struct iommu_table_group_ops { + unsigned long (*get_table_size)( + __u32 page_shift, + __u64 window_size, + __u32 levels); + long (*create_table)(struct iommu_table_group *table_group, + int num, + __u32 page_shift, + __u64 window_size, + __u32 levels, + struct iommu_table **ptbl); + long (*set_window)(struct iommu_table_group *table_group, + int num, + struct iommu_table *tblnew); + long (*unset_window)(struct iommu_table_group *table_group, + int num); + /* Switch ownership from platform code to external user (e.g. VFIO) */ + long (*take_ownership)(struct iommu_table_group *table_group, struct device *dev); + /* Switch ownership from external user (e.g. VFIO) back to core */ + void (*release_ownership)(struct iommu_table_group *table_group, struct device *dev); +}; + +struct iommu_table_group_link { + struct list_head next; + struct rcu_head rcu; + struct iommu_table_group *table_group; +}; + +struct iommu_table_group { + /* IOMMU properties */ + __u32 tce32_start; + __u32 tce32_size; + __u64 pgsizes; /* Bitmap of supported page sizes */ + __u32 max_dynamic_windows_supported; + __u32 max_levels; + + struct iommu_group *group; + struct iommu_table *tables[IOMMU_TABLE_GROUP_MAX_TABLES]; + struct iommu_table_group_ops *ops; +}; + +#ifdef CONFIG_IOMMU_API + +extern void iommu_register_group(struct iommu_table_group *table_group, int pci_domain_number, unsigned long pe_num); +extern int iommu_add_device(struct iommu_table_group *table_group, + struct device *dev); +extern long iommu_tce_xchg(struct mm_struct *mm, struct iommu_table *tbl, + unsigned long entry, unsigned long *hpa, + enum dma_data_direction *direction); +extern long iommu_tce_xchg_no_kill(struct mm_struct *mm, + struct iommu_table *tbl, + unsigned long entry, unsigned long *hpa, + enum dma_data_direction *direction); +extern void iommu_tce_kill(struct iommu_table *tbl, + unsigned long entry, unsigned long pages); +int dev_has_iommu_table(struct device *dev, void *data); -extern int iommu_map_sg(struct device *dev, struct iommu_table *tbl, - struct scatterlist *sglist, int nelems, - unsigned long mask, enum dma_data_direction direction, - struct dma_attrs *attrs); -extern void iommu_unmap_sg(struct iommu_table *tbl, struct scatterlist *sglist, - int nelems, enum dma_data_direction direction, - struct dma_attrs *attrs); +#else +static inline void iommu_register_group(struct iommu_table_group *table_group, + int pci_domain_number, + unsigned long pe_num) +{ +} + +static inline int iommu_add_device(struct iommu_table_group *table_group, + struct device *dev) +{ + return 0; +} + +static inline int dev_has_iommu_table(struct device *dev, void *data) +{ + return 0; +} +#endif /* !CONFIG_IOMMU_API */ + +u64 dma_iommu_get_required_mask(struct device *dev); +#else + +static inline void *get_iommu_table_base(struct device *dev) +{ + return NULL; +} + +static inline int dma_iommu_dma_supported(struct device *dev, u64 mask) +{ + return 0; +} + +#endif /* CONFIG_PPC64 */ + +extern int ppc_iommu_map_sg(struct device *dev, struct iommu_table *tbl, + struct scatterlist *sglist, int nelems, + unsigned long mask, + enum dma_data_direction direction, + unsigned long attrs); +extern void ppc_iommu_unmap_sg(struct iommu_table *tbl, + struct scatterlist *sglist, + int nelems, + enum dma_data_direction direction, + unsigned long attrs); extern void *iommu_alloc_coherent(struct device *dev, struct iommu_table *tbl, size_t size, dma_addr_t *dma_handle, unsigned long mask, gfp_t flag, int node); extern void iommu_free_coherent(struct iommu_table *tbl, size_t size, void *vaddr, dma_addr_t dma_handle); -extern dma_addr_t iommu_map_page(struct device *dev, struct iommu_table *tbl, - struct page *page, unsigned long offset, - size_t size, unsigned long mask, +extern dma_addr_t iommu_map_phys(struct device *dev, struct iommu_table *tbl, + phys_addr_t phys, size_t size, + unsigned long mask, enum dma_data_direction direction, - struct dma_attrs *attrs); -extern void iommu_unmap_page(struct iommu_table *tbl, dma_addr_t dma_handle, + unsigned long attrs); +extern void iommu_unmap_phys(struct iommu_table *tbl, dma_addr_t dma_handle, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs); + unsigned long attrs); -extern void iommu_init_early_pSeries(void); -extern void iommu_init_early_dart(void); +void __init iommu_init_early_pSeries(void); +extern void iommu_init_early_dart(struct pci_controller_ops *controller_ops); extern void iommu_init_early_pasemi(void); -extern void alloc_dart_table(void); #if defined(CONFIG_PPC64) && defined(CONFIG_PM) -static inline void iommu_save(void) -{ - if (ppc_md.iommu_save) - ppc_md.iommu_save(); -} - static inline void iommu_restore(void) { if (ppc_md.iommu_restore) @@ -146,25 +296,28 @@ static inline void iommu_restore(void) #endif /* The API to support IOMMU operations for VFIO */ -extern int iommu_tce_clear_param_check(struct iommu_table *tbl, - unsigned long ioba, unsigned long tce_value, - unsigned long npages); -extern int iommu_tce_put_param_check(struct iommu_table *tbl, - unsigned long ioba, unsigned long tce); -extern int iommu_tce_build(struct iommu_table *tbl, unsigned long entry, - unsigned long hwaddr, enum dma_data_direction direction); -extern unsigned long iommu_clear_tce(struct iommu_table *tbl, - unsigned long entry); -extern int iommu_clear_tces_and_put_pages(struct iommu_table *tbl, - unsigned long entry, unsigned long pages); -extern int iommu_put_tce_user_mode(struct iommu_table *tbl, - unsigned long entry, unsigned long tce); +extern int iommu_tce_check_ioba(unsigned long page_shift, + unsigned long offset, unsigned long size, + unsigned long ioba, unsigned long npages); +extern int iommu_tce_check_gpa(unsigned long page_shift, + unsigned long gpa); + +#define iommu_tce_clear_param_check(tbl, ioba, tce_value, npages) \ + (iommu_tce_check_ioba((tbl)->it_page_shift, \ + (tbl)->it_offset, (tbl)->it_size, \ + (ioba), (npages)) || (tce_value)) +#define iommu_tce_put_param_check(tbl, ioba, gpa) \ + (iommu_tce_check_ioba((tbl)->it_page_shift, \ + (tbl)->it_offset, (tbl)->it_size, \ + (ioba), 1) || \ + iommu_tce_check_gpa((tbl)->it_page_shift, (gpa))) extern void iommu_flush_tce(struct iommu_table *tbl); -extern int iommu_take_ownership(struct iommu_table *tbl); -extern void iommu_release_ownership(struct iommu_table *tbl); extern enum dma_data_direction iommu_tce_direction(unsigned long tce); +extern unsigned long iommu_direction_to_tce_perm(enum dma_data_direction dir); + +extern const struct dma_map_ops dma_iommu_ops; #endif /* __KERNEL__ */ #endif /* _ASM_IOMMU_H */ diff --git a/arch/powerpc/include/asm/ipic.h b/arch/powerpc/include/asm/ipic.h index fb59829983b8..b47ca7dc7199 100644 --- a/arch/powerpc/include/asm/ipic.h +++ b/arch/powerpc/include/asm/ipic.h @@ -1,14 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * IPIC external definitions and structure. * * Maintainer: Kumar Gala <galak@kernel.crashing.org> * * Copyright 2005 Freescale Semiconductor, Inc - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. */ #ifdef __KERNEL__ #ifndef __ASM_IPIC_H__ @@ -69,11 +65,7 @@ enum ipic_mcp_irq { IPIC_MCP_MU = 7, }; -extern int ipic_set_priority(unsigned int irq, unsigned int priority); -extern void ipic_set_highest_priority(unsigned int irq); -extern void ipic_set_default_priority(void); -extern void ipic_enable_mcp(enum ipic_mcp_irq mcp_irq); -extern void ipic_disable_mcp(enum ipic_mcp_irq mcp_irq); +void __init ipic_set_default_priority(void); extern u32 ipic_get_mcp_status(void); extern void ipic_clear_mcp_status(u32 mask); diff --git a/arch/powerpc/include/asm/irq.h b/arch/powerpc/include/asm/irq.h index 0e40843a1c6e..aa3751960ffd 100644 --- a/arch/powerpc/include/asm/irq.h +++ b/arch/powerpc/include/asm/irq.h @@ -1,15 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifdef __KERNEL__ #ifndef _ASM_POWERPC_IRQ_H #define _ASM_POWERPC_IRQ_H /* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ -#include <linux/irqdomain.h> #include <linux/threads.h> #include <linux/list.h> #include <linux/radix-tree.h> @@ -20,22 +16,14 @@ extern atomic_t ppc_n_lost_interrupts; -/* This number is used when no interrupt has been assigned */ -#define NO_IRQ (0) - /* Total number of virq in the platform */ #define NR_IRQS CONFIG_NR_IRQS -/* Same thing, used by the generic IRQ code */ -#define NR_IRQS_LEGACY NUM_ISA_INTERRUPTS +/* Number of irqs reserved for a legacy isa controller */ +#define NR_IRQS_LEGACY 16 extern irq_hw_number_t virq_to_hw(unsigned int virq); -/** - * irq_early_init - Init irq remapping subsystem - */ -extern void irq_early_init(void); - static __inline__ int irq_canonicalize(int irq) { return irq; @@ -43,37 +31,33 @@ static __inline__ int irq_canonicalize(int irq) extern int distribute_irqs; -struct irqaction; struct pt_regs; -#define __ARCH_HAS_DO_SOFTIRQ - -#if defined(CONFIG_BOOKE) || defined(CONFIG_40x) +#ifdef CONFIG_BOOKE /* * Per-cpu stacks for handling critical, debug and machine check * level interrupts. */ -extern struct thread_info *critirq_ctx[NR_CPUS]; -extern struct thread_info *dbgirq_ctx[NR_CPUS]; -extern struct thread_info *mcheckirq_ctx[NR_CPUS]; -extern void exc_lvl_ctx_init(void); -#else -#define exc_lvl_ctx_init() +extern void *critirq_ctx[NR_CPUS]; +extern void *dbgirq_ctx[NR_CPUS]; +extern void *mcheckirq_ctx[NR_CPUS]; #endif /* * Per-cpu stacks for handling hard and soft interrupts. */ -extern struct thread_info *hardirq_ctx[NR_CPUS]; -extern struct thread_info *softirq_ctx[NR_CPUS]; +extern void *hardirq_ctx[NR_CPUS]; +extern void *softirq_ctx[NR_CPUS]; -extern void irq_ctx_init(void); -extern void call_do_softirq(struct thread_info *tp); -extern int call_handle_irq(int irq, void *p1, - struct thread_info *tp, void *func); -extern void do_IRQ(struct pt_regs *regs); +void __do_IRQ(struct pt_regs *regs); int irq_choose_cpu(const struct cpumask *mask); +#if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_NMI_IPI) +extern void arch_trigger_cpumask_backtrace(const cpumask_t *mask, + int exclude_cpu); +#define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace +#endif + #endif /* _ASM_IRQ_H */ #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/irq_regs.h b/arch/powerpc/include/asm/irq_regs.h deleted file mode 100644 index ba94b51a0a70..000000000000 --- a/arch/powerpc/include/asm/irq_regs.h +++ /dev/null @@ -1,2 +0,0 @@ -#include <asm-generic/irq_regs.h> - diff --git a/arch/powerpc/include/asm/irq_work.h b/arch/powerpc/include/asm/irq_work.h new file mode 100644 index 000000000000..c6d3078bd8c3 --- /dev/null +++ b/arch/powerpc/include/asm/irq_work.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_IRQ_WORK_H +#define _ASM_POWERPC_IRQ_WORK_H + +static inline bool arch_irq_work_has_interrupt(void) +{ + return true; +} + +#endif /* _ASM_POWERPC_IRQ_WORK_H */ diff --git a/arch/powerpc/include/asm/irqflags.h b/arch/powerpc/include/asm/irqflags.h index 6f9b6e23dc5a..1351fb40fe74 100644 --- a/arch/powerpc/include/asm/irqflags.h +++ b/arch/powerpc/include/asm/irqflags.h @@ -1,70 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * IRQ flags handling */ #ifndef _ASM_IRQFLAGS_H #define _ASM_IRQFLAGS_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * Get definitions for arch_local_save_flags(x), etc. */ #include <asm/hw_irq.h> -#else -#ifdef CONFIG_TRACE_IRQFLAGS -#ifdef CONFIG_IRQSOFF_TRACER -/* - * Since the ftrace irqsoff latency trace checks CALLER_ADDR1, - * which is the stack frame here, we need to force a stack frame - * in case we came from user space. - */ -#define TRACE_WITH_FRAME_BUFFER(func) \ - mflr r0; \ - stdu r1, -32(r1); \ - std r0, 16(r1); \ - stdu r1, -32(r1); \ - bl func; \ - ld r1, 0(r1); \ - ld r1, 0(r1); -#else -#define TRACE_WITH_FRAME_BUFFER(func) \ - bl func; -#endif - -/* - * Most of the CPU's IRQ-state tracing is done from assembly code; we - * have to call a C function so call a wrapper that saves all the - * C-clobbered registers. - */ -#define TRACE_ENABLE_INTS TRACE_WITH_FRAME_BUFFER(.trace_hardirqs_on) -#define TRACE_DISABLE_INTS TRACE_WITH_FRAME_BUFFER(.trace_hardirqs_off) - -/* - * This is used by assembly code to soft-disable interrupts - */ -#define SOFT_DISABLE_INTS(__rA, __rB) \ - lbz __rA,PACASOFTIRQEN(r13); \ - lbz __rB,PACAIRQHAPPENED(r13); \ - cmpwi cr0,__rA,0; \ - li __rA,0; \ - ori __rB,__rB,PACA_IRQ_HARD_DIS; \ - stb __rB,PACAIRQHAPPENED(r13); \ - beq 44f; \ - stb __rA,PACASOFTIRQEN(r13); \ - TRACE_DISABLE_INTS; \ -44: - -#else -#define TRACE_ENABLE_INTS -#define TRACE_DISABLE_INTS - -#define SOFT_DISABLE_INTS(__rA, __rB) \ - lbz __rA,PACAIRQHAPPENED(r13); \ - li __rB,0; \ - ori __rA,__rA,PACA_IRQ_HARD_DIS; \ - stb __rB,PACASOFTIRQEN(r13); \ - stb __rA,PACAIRQHAPPENED(r13) -#endif #endif #endif diff --git a/arch/powerpc/include/asm/isa-bridge.h b/arch/powerpc/include/asm/isa-bridge.h new file mode 100644 index 000000000000..47295894bf91 --- /dev/null +++ b/arch/powerpc/include/asm/isa-bridge.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ISA_BRIDGE_H +#define __ISA_BRIDGE_H + +#ifdef CONFIG_PPC64 + +extern void isa_bridge_find_early(struct pci_controller *hose); +extern void isa_bridge_init_non_pci(struct device_node *np); + +static inline int isa_vaddr_is_ioport(void __iomem *address) +{ + /* Check if address hits the reserved legacy IO range */ + unsigned long ea = (unsigned long)address; + return ea >= ISA_IO_BASE && ea < ISA_IO_END; +} + +#else + +static inline int isa_vaddr_is_ioport(void __iomem *address) +{ + /* No specific ISA handling on ppc32 at this stage, it + * all goes through PCI + */ + return 0; +} + +#endif + +#endif /* __ISA_BRIDGE_H */ + diff --git a/arch/powerpc/include/asm/jump_label.h b/arch/powerpc/include/asm/jump_label.h index ae098c438f00..d4eaba459a0e 100644 --- a/arch/powerpc/include/asm/jump_label.h +++ b/arch/powerpc/include/asm/jump_label.h @@ -1,45 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _ASM_POWERPC_JUMP_LABEL_H #define _ASM_POWERPC_JUMP_LABEL_H /* * Copyright 2010 Michael Ellerman, IBM Corp. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ +#ifndef __ASSEMBLER__ #include <linux/types.h> #include <asm/feature-fixups.h> +#include <asm/asm-const.h> #define JUMP_ENTRY_TYPE stringify_in_c(FTR_ENTRY_LONG) #define JUMP_LABEL_NOP_SIZE 4 -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm goto("1:\n\t" - "nop\n\t" + "nop # arch_static_branch\n\t" ".pushsection __jump_table, \"aw\"\n\t" - JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t" + ".long 1b - ., %l[l_yes] - .\n\t" + JUMP_ENTRY_TYPE "%c0 - .\n\t" ".popsection \n\t" - : : "i" (key) : : l_yes); + : : "i" (&((char *)key)[branch]) : : l_yes); + + return false; +l_yes: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm goto("1:\n\t" + "b %l[l_yes] # arch_static_branch_jump\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".long 1b - ., %l[l_yes] - .\n\t" + JUMP_ENTRY_TYPE "%c0 - .\n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); + return false; l_yes: return true; } -#ifdef CONFIG_PPC64 -typedef u64 jump_label_t; #else -typedef u32 jump_label_t; +#define ARCH_STATIC_BRANCH(LABEL, KEY) \ +1098: nop; \ + .pushsection __jump_table, "aw"; \ + .long 1098b - ., LABEL - .; \ + FTR_ENTRY_LONG KEY - .; \ + .popsection #endif -struct jump_entry { - jump_label_t code; - jump_label_t target; - jump_label_t key; -}; - #endif /* _ASM_POWERPC_JUMP_LABEL_H */ diff --git a/arch/powerpc/include/asm/kasan.h b/arch/powerpc/include/asm/kasan.h new file mode 100644 index 000000000000..045804a86f98 --- /dev/null +++ b/arch/powerpc/include/asm/kasan.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_KASAN_H +#define __ASM_KASAN_H + +#if defined(CONFIG_KASAN) && !defined(CONFIG_CC_HAS_KASAN_MEMINTRINSIC_PREFIX) +#define _GLOBAL_KASAN(fn) _GLOBAL(__##fn) +#define _GLOBAL_TOC_KASAN(fn) _GLOBAL_TOC(__##fn) +#define EXPORT_SYMBOL_KASAN(fn) EXPORT_SYMBOL(__##fn) +#else +#define _GLOBAL_KASAN(fn) _GLOBAL(fn) +#define _GLOBAL_TOC_KASAN(fn) _GLOBAL_TOC(fn) +#define EXPORT_SYMBOL_KASAN(fn) +#endif + +#ifndef __ASSEMBLER__ + +#include <asm/page.h> +#include <linux/sizes.h> + +#define KASAN_SHADOW_SCALE_SHIFT 3 + +#if defined(CONFIG_EXECMEM) && defined(CONFIG_PPC32) +#define KASAN_KERN_START ALIGN_DOWN(PAGE_OFFSET - SZ_256M, SZ_256M) +#else +#define KASAN_KERN_START PAGE_OFFSET +#endif + +#define KASAN_SHADOW_START (KASAN_SHADOW_OFFSET + \ + (KASAN_KERN_START >> KASAN_SHADOW_SCALE_SHIFT)) + +#define KASAN_SHADOW_OFFSET ASM_CONST(CONFIG_KASAN_SHADOW_OFFSET) + +#ifdef CONFIG_PPC32 +#define KASAN_SHADOW_END (-(-KASAN_SHADOW_START >> KASAN_SHADOW_SCALE_SHIFT)) +#elif defined(CONFIG_PPC_BOOK3S_64) +/* + * The shadow ends before the highest accessible address + * because we don't need a shadow for the shadow. Instead: + * c00e000000000000 << 3 + a80e000000000000 = c00fc00000000000 + */ +#define KASAN_SHADOW_END 0xc00fc00000000000UL + +#else + +/* + * The shadow ends before the highest accessible address + * because we don't need a shadow for the shadow. + * But it doesn't hurt to have a shadow for the shadow, + * keep shadow end aligned eases things. + */ +#define KASAN_SHADOW_END 0xc000200000000000UL + +#endif + +#ifdef CONFIG_KASAN + +void kasan_early_init(void); +void kasan_mmu_init(void); +void kasan_init(void); +void kasan_late_init(void); +#else +static inline void kasan_init(void) { } +static inline void kasan_mmu_init(void) { } +static inline void kasan_late_init(void) { } +#endif + +void kasan_update_early_region(unsigned long k_start, unsigned long k_end, pte_t pte); +int kasan_init_shadow_page_tables(unsigned long k_start, unsigned long k_end); +int kasan_init_region(void *start, size_t size); + +#endif /* __ASSEMBLER__ */ +#endif diff --git a/arch/powerpc/include/asm/kdebug.h b/arch/powerpc/include/asm/kdebug.h index ae6d206728af..0f7c1ef37d0d 100644 --- a/arch/powerpc/include/asm/kdebug.h +++ b/arch/powerpc/include/asm/kdebug.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_KDEBUG_H #define _ASM_POWERPC_KDEBUG_H #ifdef __KERNEL__ diff --git a/arch/powerpc/include/asm/kdump.h b/arch/powerpc/include/asm/kdump.h index c9776202d7ec..802644178f43 100644 --- a/arch/powerpc/include/asm/kdump.h +++ b/arch/powerpc/include/asm/kdump.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _PPC64_KDUMP_H #define _PPC64_KDUMP_H @@ -30,7 +31,7 @@ #endif /* CONFIG_CRASH_DUMP */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #if defined(CONFIG_CRASH_DUMP) && !defined(CONFIG_NONSTATIC_KERNEL) extern void reserve_kdump_trampoline(void); @@ -41,6 +42,6 @@ static inline void reserve_kdump_trampoline(void) { ; } static inline void setup_kdump_trampoline(void) { ; } #endif -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __PPC64_KDUMP_H */ diff --git a/arch/powerpc/include/asm/kexec.h b/arch/powerpc/include/asm/kexec.h index 16d7e33d35e9..bd4a6c42a5f3 100644 --- a/arch/powerpc/include/asm/kexec.h +++ b/arch/powerpc/include/asm/kexec.h @@ -1,8 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_KEXEC_H #define _ASM_POWERPC_KEXEC_H #ifdef __KERNEL__ -#if defined(CONFIG_FSL_BOOKE) || defined(CONFIG_44x) +#if defined(CONFIG_PPC_85xx) || defined(CONFIG_44x) /* * On FSL-BookE we setup a 1:1 mapping which covers the first 2GiB of memory @@ -48,13 +49,80 @@ #define KEXEC_STATE_IRQS_OFF 1 #define KEXEC_STATE_REAL_MODE 2 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm/reg.h> typedef void (*crash_shutdown_t)(void); -#ifdef CONFIG_KEXEC +#ifdef CONFIG_KEXEC_CORE +struct kimage; +struct pt_regs; + +extern void kexec_smp_wait(void); /* get and clear naca physid, wait for + master to copy new code to 0 */ +extern void default_machine_kexec(struct kimage *image); + +void relocate_new_kernel(unsigned long indirection_page, unsigned long reboot_code_buffer, + unsigned long start_address) __noreturn; +void kexec_copy_flush(struct kimage *image); + +#ifdef CONFIG_KEXEC_FILE +extern const struct kexec_file_ops kexec_elf64_ops; + +#define ARCH_HAS_KIMAGE_ARCH + +struct kimage_arch { + struct crash_mem *exclude_ranges; + + unsigned long backup_start; + void *backup_buf; + void *fdt; +}; + +char *setup_kdump_cmdline(struct kimage *image, char *cmdline, + unsigned long cmdline_len); +int setup_purgatory(struct kimage *image, const void *slave_code, + const void *fdt, unsigned long kernel_load_addr, + unsigned long fdt_load_addr); + +#ifdef CONFIG_PPC64 +struct kexec_buf; + +int arch_kexec_kernel_image_probe(struct kimage *image, void *buf, unsigned long buf_len); +#define arch_kexec_kernel_image_probe arch_kexec_kernel_image_probe +int arch_kimage_file_post_load_cleanup(struct kimage *image); +#define arch_kimage_file_post_load_cleanup arch_kimage_file_post_load_cleanup + +int arch_check_excluded_range(struct kimage *image, unsigned long start, + unsigned long end); +#define arch_check_excluded_range arch_check_excluded_range + + +int load_crashdump_segments_ppc64(struct kimage *image, + struct kexec_buf *kbuf); +int setup_purgatory_ppc64(struct kimage *image, const void *slave_code, + const void *fdt, unsigned long kernel_load_addr, + unsigned long fdt_load_addr); +unsigned int kexec_extra_fdt_size_ppc64(struct kimage *image, struct crash_mem *rmem); +int setup_new_fdt_ppc64(const struct kimage *image, void *fdt, struct crash_mem *rmem); +#endif /* CONFIG_PPC64 */ + +#endif /* CONFIG_KEXEC_FILE */ + +#endif /* CONFIG_KEXEC_CORE */ + +#ifdef CONFIG_CRASH_RESERVE +int __init overlaps_crashkernel(unsigned long start, unsigned long size); +extern void arch_reserve_crashkernel(void); +extern void kdump_cma_reserve(void); +#else +static inline void arch_reserve_crashkernel(void) {} +static inline int overlaps_crashkernel(unsigned long start, unsigned long size) { return 0; } +static inline void kdump_cma_reserve(void) { } +#endif + +#if defined(CONFIG_CRASH_DUMP) /* * This function is responsible for capturing register states if coming * via panic or invoking dump using sysrq-trigger. @@ -68,34 +136,43 @@ static inline void crash_setup_regs(struct pt_regs *newregs, ppc_save_regs(newregs); } -extern void kexec_smp_wait(void); /* get and clear naca physid, wait for - master to copy new code to 0 */ +#ifdef CONFIG_CRASH_HOTPLUG +void arch_crash_handle_hotplug_event(struct kimage *image, void *arg); +#define arch_crash_handle_hotplug_event arch_crash_handle_hotplug_event + +int arch_crash_hotplug_support(struct kimage *image, unsigned long kexec_flags); +#define arch_crash_hotplug_support arch_crash_hotplug_support + +unsigned int arch_crash_get_elfcorehdr_size(void); +#define crash_get_elfcorehdr_size arch_crash_get_elfcorehdr_size +#endif /* CONFIG_CRASH_HOTPLUG */ + extern int crashing_cpu; extern void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *)); +extern void crash_ipi_callback(struct pt_regs *regs); +extern int crash_wake_offline; -struct kimage; -struct pt_regs; -extern void default_machine_kexec(struct kimage *image); -extern int default_machine_kexec_prepare(struct kimage *image); -extern void default_machine_crash_shutdown(struct pt_regs *regs); extern int crash_shutdown_register(crash_shutdown_t handler); extern int crash_shutdown_unregister(crash_shutdown_t handler); +extern void default_machine_crash_shutdown(struct pt_regs *regs); -extern void machine_kexec_simple(struct kimage *image); +extern void crash_kexec_prepare(void); extern void crash_kexec_secondary(struct pt_regs *regs); -extern int overlaps_crashkernel(unsigned long start, unsigned long size); -extern void reserve_crashkernel(void); -extern void machine_kexec_mask_interrupts(void); - -#else /* !CONFIG_KEXEC */ -static inline void crash_kexec_secondary(struct pt_regs *regs) { } -static inline int overlaps_crashkernel(unsigned long start, unsigned long size) +static inline bool kdump_in_progress(void) { - return 0; + return crashing_cpu >= 0; } -static inline void reserve_crashkernel(void) { ; } +bool is_kdump_kernel(void); +#define is_kdump_kernel is_kdump_kernel +#if defined(CONFIG_PPC_RTAS) +void crash_free_reserved_phys_range(unsigned long begin, unsigned long end); +#define crash_free_reserved_phys_range crash_free_reserved_phys_range +#endif /* CONFIG_PPC_RTAS */ + +#else /* !CONFIG_CRASH_DUMP */ +static inline void crash_kexec_secondary(struct pt_regs *regs) { } static inline int crash_shutdown_register(crash_shutdown_t handler) { @@ -107,7 +184,34 @@ static inline int crash_shutdown_unregister(crash_shutdown_t handler) return 0; } -#endif /* CONFIG_KEXEC */ -#endif /* ! __ASSEMBLY__ */ +static inline bool kdump_in_progress(void) +{ + return false; +} + +static inline void crash_ipi_callback(struct pt_regs *regs) { } + +static inline void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *)) +{ +} + +#endif /* CONFIG_CRASH_DUMP */ + +#if defined(CONFIG_KEXEC_FILE) || defined(CONFIG_CRASH_DUMP) +int update_cpus_node(void *fdt); +#endif + +#ifdef CONFIG_PPC_BOOK3S_64 +#include <asm/book3s/64/kexec.h> +#endif + +#ifndef reset_sprs +#define reset_sprs reset_sprs +static inline void reset_sprs(void) +{ +} +#endif + +#endif /* ! __ASSEMBLER__ */ #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_KEXEC_H */ diff --git a/arch/powerpc/include/asm/kexec_ranges.h b/arch/powerpc/include/asm/kexec_ranges.h new file mode 100644 index 000000000000..14055896cbcb --- /dev/null +++ b/arch/powerpc/include/asm/kexec_ranges.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _ASM_POWERPC_KEXEC_RANGES_H +#define _ASM_POWERPC_KEXEC_RANGES_H + +#define MEM_RANGE_CHUNK_SZ 2048 /* Memory ranges size chunk */ + +void sort_memory_ranges(struct crash_mem *mrngs, bool merge); +struct crash_mem *realloc_mem_ranges(struct crash_mem **mem_ranges); +int add_mem_range(struct crash_mem **mem_ranges, u64 base, u64 size); +int remove_mem_range(struct crash_mem **mem_ranges, u64 base, u64 size); +int get_exclude_memory_ranges(struct crash_mem **mem_ranges); +int get_reserved_memory_ranges(struct crash_mem **mem_ranges); +int get_crash_memory_ranges(struct crash_mem **mem_ranges); +int get_usable_memory_ranges(struct crash_mem **mem_ranges); +#endif /* _ASM_POWERPC_KEXEC_RANGES_H */ diff --git a/arch/powerpc/include/asm/keylargo.h b/arch/powerpc/include/asm/keylargo.h index 2156315d8a90..debdf548009d 100644 --- a/arch/powerpc/include/asm/keylargo.h +++ b/arch/powerpc/include/asm/keylargo.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_KEYLARGO_H #define _ASM_POWERPC_KEYLARGO_H #ifdef __KERNEL__ diff --git a/arch/powerpc/include/asm/kfence.h b/arch/powerpc/include/asm/kfence.h new file mode 100644 index 000000000000..1f7cab58ab2c --- /dev/null +++ b/arch/powerpc/include/asm/kfence.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * powerpc KFENCE support. + * + * Copyright (C) 2020 CS GROUP France + */ + +#ifndef __ASM_POWERPC_KFENCE_H +#define __ASM_POWERPC_KFENCE_H + +#include <linux/mm.h> +#include <asm/pgtable.h> + +#ifdef CONFIG_PPC64_ELF_ABI_V1 +#define ARCH_FUNC_PREFIX "." +#endif + +extern bool kfence_early_init; +extern bool kfence_disabled; + +static inline void disable_kfence(void) +{ + kfence_disabled = true; +} + +static inline bool arch_kfence_init_pool(void) +{ + return !kfence_disabled; +} + +static inline bool kfence_early_init_enabled(void) +{ + return IS_ENABLED(CONFIG_KFENCE) && kfence_early_init; +} + +#ifdef CONFIG_PPC64 +static inline bool kfence_protect_page(unsigned long addr, bool protect) +{ + struct page *page = virt_to_page((void *)addr); + + __kernel_map_pages(page, 1, !protect); + + return true; +} +#else +static inline bool kfence_protect_page(unsigned long addr, bool protect) +{ + pte_t *kpte = virt_to_kpte(addr); + + if (protect) { + pte_update(&init_mm, addr, kpte, _PAGE_PRESENT, 0, 0); + flush_tlb_kernel_range(addr, addr + PAGE_SIZE); + } else { + pte_update(&init_mm, addr, kpte, 0, _PAGE_PRESENT, 0); + } + + return true; +} +#endif + +#endif /* __ASM_POWERPC_KFENCE_H */ diff --git a/arch/powerpc/include/asm/kgdb.h b/arch/powerpc/include/asm/kgdb.h index 9db24e77b9f4..f39531903325 100644 --- a/arch/powerpc/include/asm/kgdb.h +++ b/arch/powerpc/include/asm/kgdb.h @@ -21,14 +21,17 @@ #ifndef __POWERPC_KGDB_H__ #define __POWERPC_KGDB_H__ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define BREAK_INSTR_SIZE 4 #define BUFMAX ((NUMREGBYTES * 2) + 512) #define OUTBUFMAX ((NUMREGBYTES * 2) + 512) + +#define BREAK_INSTR 0x7d821008 /* twge r2, r2 */ + static inline void arch_kgdb_breakpoint(void) { - asm(".long 0x7d821008"); /* twge r2, r2 */ + asm(stringify_in_c(.long BREAK_INSTR)); } #define CACHE_FLUSH_IS_SAFE 1 #define DBG_MAX_REG_NUM 70 @@ -49,7 +52,7 @@ static inline void arch_kgdb_breakpoint(void) /* On non-E500 family PPC32 we determine the size by picking the last * register we need, but on E500 we skip sections so we list what we * need to store, and add it up. */ -#ifndef CONFIG_E500 +#ifndef CONFIG_PPC_E500 #define MAXREG (PT_FPSCR+1) #else /* 32 GPRs (8 bytes), nip, msr, ccr, link, ctr, xer, acc (8 bytes), spefscr*/ @@ -59,6 +62,6 @@ static inline void arch_kgdb_breakpoint(void) /* CR/LR, R1, R2, R13-R31 inclusive. */ #define NUMCRITREGBYTES (23 * sizeof(int)) #endif /* 32/64 */ -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ #endif /* !__POWERPC_KGDB_H__ */ #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/kmap_types.h b/arch/powerpc/include/asm/kmap_types.h deleted file mode 100644 index 5acabbd7ac6f..000000000000 --- a/arch/powerpc/include/asm/kmap_types.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef _ASM_POWERPC_KMAP_TYPES_H -#define _ASM_POWERPC_KMAP_TYPES_H - -#ifdef __KERNEL__ - -/* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ - -#define KM_TYPE_NR 16 - -#endif /* __KERNEL__ */ -#endif /* _ASM_POWERPC_KMAP_TYPES_H */ diff --git a/arch/powerpc/include/asm/kprobes.h b/arch/powerpc/include/asm/kprobes.h index 7b6feab6fd26..dfe2e5ad3b21 100644 --- a/arch/powerpc/include/asm/kprobes.h +++ b/arch/powerpc/include/asm/kprobes.h @@ -1,23 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _ASM_POWERPC_KPROBES_H #define _ASM_POWERPC_KPROBES_H + +#include <asm-generic/kprobes.h> + #ifdef __KERNEL__ /* * Kernel Probes (KProbes) * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * * Copyright (C) IBM Corporation, 2002, 2004 * * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel @@ -29,50 +19,39 @@ #include <linux/types.h> #include <linux/ptrace.h> #include <linux/percpu.h> +#include <linux/module.h> #include <asm/probes.h> +#include <asm/text-patching.h> +#ifdef CONFIG_KPROBES #define __ARCH_WANT_KPROBES_INSN_SLOT struct pt_regs; struct kprobe; -typedef ppc_opcode_t kprobe_opcode_t; -#define MAX_INSN_SIZE 1 +typedef u32 kprobe_opcode_t; -#ifdef CONFIG_PPC64 -/* - * 64bit powerpc uses function descriptors. - * Handle cases where: - * - User passes a <.symbol> or <module:.symbol> - * - User passes a <symbol> or <module:symbol> - * - User passes a non-existent symbol, kallsyms_lookup_name - * returns 0. Don't deref the NULL pointer in that case - */ -#define kprobe_lookup_name(name, addr) \ -{ \ - addr = (kprobe_opcode_t *)kallsyms_lookup_name(name); \ - if (addr) { \ - char *colon; \ - if ((colon = strchr(name, ':')) != NULL) { \ - colon++; \ - if (*colon != '\0' && *colon != '.') \ - addr = *(kprobe_opcode_t **)addr; \ - } else if (name[0] != '.') \ - addr = *(kprobe_opcode_t **)addr; \ - } else { \ - char dot_name[KSYM_NAME_LEN]; \ - dot_name[0] = '.'; \ - dot_name[1] = '\0'; \ - strncat(dot_name, name, KSYM_NAME_LEN - 2); \ - addr = (kprobe_opcode_t *)kallsyms_lookup_name(dot_name); \ - } \ -} -#endif +extern kprobe_opcode_t optinsn_slot; + +/* Optinsn template address */ +extern kprobe_opcode_t optprobe_template_entry[]; +extern kprobe_opcode_t optprobe_template_op_address[]; +extern kprobe_opcode_t optprobe_template_call_handler[]; +extern kprobe_opcode_t optprobe_template_insn[]; +extern kprobe_opcode_t optprobe_template_call_emulate[]; +extern kprobe_opcode_t optprobe_template_ret[]; +extern kprobe_opcode_t optprobe_template_end[]; + +/* Fixed instruction size for powerpc */ +#define MAX_INSN_SIZE 2 +#define MAX_OPTIMIZED_LENGTH sizeof(kprobe_opcode_t) /* 4 bytes */ +#define MAX_OPTINSN_SIZE (optprobe_template_end - optprobe_template_entry) +#define RELATIVEJUMP_SIZE sizeof(kprobe_opcode_t) /* 4 bytes */ #define flush_insn_slot(p) do { } while (0) #define kretprobe_blacklist_size 0 -void kretprobe_trampoline(void); +void __kretprobe_trampoline(void); extern void arch_remove_kprobe(struct kprobe *p); /* Architecture specific copy of original instruction */ @@ -96,12 +75,21 @@ struct prev_kprobe { struct kprobe_ctlblk { unsigned long kprobe_status; unsigned long kprobe_saved_msr; - struct pt_regs jprobe_saved_regs; struct prev_kprobe prev_kprobe; }; -extern int kprobe_exceptions_notify(struct notifier_block *self, - unsigned long val, void *data); +struct arch_optimized_insn { + kprobe_opcode_t copied_insn[1]; + /* detour buffer */ + kprobe_opcode_t *insn; +}; + extern int kprobe_fault_handler(struct pt_regs *regs, int trapnr); +extern int kprobe_handler(struct pt_regs *regs); +extern int kprobe_post_handler(struct pt_regs *regs); +#else +static inline int kprobe_handler(struct pt_regs *regs) { return 0; } +static inline int kprobe_post_handler(struct pt_regs *regs) { return 0; } +#endif /* CONFIG_KPROBES */ #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_KPROBES_H */ diff --git a/arch/powerpc/include/asm/kup.h b/arch/powerpc/include/asm/kup.h new file mode 100644 index 000000000000..dab63b82a8d4 --- /dev/null +++ b/arch/powerpc/include/asm/kup.h @@ -0,0 +1,186 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_KUP_H_ +#define _ASM_POWERPC_KUP_H_ + +#define KUAP_READ 1 +#define KUAP_WRITE 2 +#define KUAP_READ_WRITE (KUAP_READ | KUAP_WRITE) + +#ifndef __ASSEMBLER__ +#include <linux/types.h> + +static __always_inline bool kuap_is_disabled(void); +#endif + +#ifdef CONFIG_PPC_BOOK3S_64 +#include <asm/book3s/64/kup.h> +#endif + +#ifdef CONFIG_PPC_8xx +#include <asm/nohash/32/kup-8xx.h> +#endif + +#ifdef CONFIG_BOOKE +#include <asm/nohash/kup-booke.h> +#endif + +#ifdef CONFIG_PPC_BOOK3S_32 +#include <asm/book3s/32/kup.h> +#endif + +#ifdef __ASSEMBLER__ +#ifndef CONFIG_PPC_KUAP +.macro kuap_check_amr gpr1, gpr2 +.endm + +#endif + +#else /* !__ASSEMBLER__ */ + +extern bool disable_kuep; +extern bool disable_kuap; + +#include <linux/pgtable.h> + +void setup_kup(void); +void setup_kuep(bool disabled); + +#ifdef CONFIG_PPC_KUAP +void setup_kuap(bool disabled); + +static __always_inline bool kuap_is_disabled(void) +{ + return !mmu_has_feature(MMU_FTR_KUAP); +} +#else +static inline void setup_kuap(bool disabled) { } + +static __always_inline bool kuap_is_disabled(void) { return true; } + +static __always_inline bool +__bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write) +{ + return false; +} + +static __always_inline void kuap_user_restore(struct pt_regs *regs) { } +static __always_inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long amr) { } + +/* + * book3s/64/kup-radix.h defines these functions for the !KUAP case to flush + * the L1D cache after user accesses. Only include the empty stubs for other + * platforms. + */ +#ifndef CONFIG_PPC_BOOK3S_64 +static __always_inline void allow_user_access(void __user *to, const void __user *from, + unsigned long size, unsigned long dir) { } +static __always_inline void prevent_user_access(unsigned long dir) { } +static __always_inline unsigned long prevent_user_access_return(void) { return 0UL; } +static __always_inline void restore_user_access(unsigned long flags) { } +#endif /* CONFIG_PPC_BOOK3S_64 */ +#endif /* CONFIG_PPC_KUAP */ + +static __always_inline bool +bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write) +{ + if (kuap_is_disabled()) + return false; + + return __bad_kuap_fault(regs, address, is_write); +} + +static __always_inline void kuap_lock(void) +{ +#ifdef __kuap_lock + if (kuap_is_disabled()) + return; + + __kuap_lock(); +#endif +} + +static __always_inline void kuap_save_and_lock(struct pt_regs *regs) +{ +#ifdef __kuap_save_and_lock + if (kuap_is_disabled()) + return; + + __kuap_save_and_lock(regs); +#endif +} + +static __always_inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long amr) +{ + if (kuap_is_disabled()) + return; + + __kuap_kernel_restore(regs, amr); +} + +static __always_inline unsigned long kuap_get_and_assert_locked(void) +{ +#ifdef __kuap_get_and_assert_locked + if (!kuap_is_disabled()) + return __kuap_get_and_assert_locked(); +#endif + return 0; +} + +static __always_inline void kuap_assert_locked(void) +{ + if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG)) + kuap_get_and_assert_locked(); +} + +static __always_inline void allow_read_from_user(const void __user *from, unsigned long size) +{ + barrier_nospec(); + allow_user_access(NULL, from, size, KUAP_READ); +} + +static __always_inline void allow_write_to_user(void __user *to, unsigned long size) +{ + allow_user_access(to, NULL, size, KUAP_WRITE); +} + +static __always_inline void allow_read_write_user(void __user *to, const void __user *from, + unsigned long size) +{ + barrier_nospec(); + allow_user_access(to, from, size, KUAP_READ_WRITE); +} + +static __always_inline void prevent_read_from_user(const void __user *from, unsigned long size) +{ + prevent_user_access(KUAP_READ); +} + +static __always_inline void prevent_write_to_user(void __user *to, unsigned long size) +{ + prevent_user_access(KUAP_WRITE); +} + +static __always_inline void prevent_read_write_user(void __user *to, const void __user *from, + unsigned long size) +{ + prevent_user_access(KUAP_READ_WRITE); +} + +static __always_inline void prevent_current_access_user(void) +{ + prevent_user_access(KUAP_READ_WRITE); +} + +static __always_inline void prevent_current_read_from_user(void) +{ + prevent_user_access(KUAP_READ); +} + +static __always_inline void prevent_current_write_to_user(void) +{ + prevent_user_access(KUAP_WRITE); +} + +#endif /* !__ASSEMBLER__ */ + +#endif /* _ASM_POWERPC_KUAP_H_ */ diff --git a/arch/powerpc/include/asm/kvm_44x.h b/arch/powerpc/include/asm/kvm_44x.h deleted file mode 100644 index a0e57618ff33..000000000000 --- a/arch/powerpc/include/asm/kvm_44x.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * - * Copyright IBM Corp. 2008 - * - * Authors: Hollis Blanchard <hollisb@us.ibm.com> - */ - -#ifndef __ASM_44X_H__ -#define __ASM_44X_H__ - -#include <linux/kvm_host.h> - -#define PPC44x_TLB_SIZE 64 - -/* If the guest is expecting it, this can be as large as we like; we'd just - * need to find some way of advertising it. */ -#define KVM44x_GUEST_TLB_SIZE 64 - -struct kvmppc_44x_tlbe { - u32 tid; /* Only the low 8 bits are used. */ - u32 word0; - u32 word1; - u32 word2; -}; - -struct kvmppc_44x_shadow_ref { - struct page *page; - u16 gtlb_index; - u8 writeable; - u8 tid; -}; - -struct kvmppc_vcpu_44x { - /* Unmodified copy of the guest's TLB. */ - struct kvmppc_44x_tlbe guest_tlb[KVM44x_GUEST_TLB_SIZE]; - - /* References to guest pages in the hardware TLB. */ - struct kvmppc_44x_shadow_ref shadow_refs[PPC44x_TLB_SIZE]; - - /* State of the shadow TLB at guest context switch time. */ - struct kvmppc_44x_tlbe shadow_tlb[PPC44x_TLB_SIZE]; - u8 shadow_tlb_mod[PPC44x_TLB_SIZE]; - - struct kvm_vcpu vcpu; -}; - -static inline struct kvmppc_vcpu_44x *to_44x(struct kvm_vcpu *vcpu) -{ - return container_of(vcpu, struct kvmppc_vcpu_44x, vcpu); -} - -void kvmppc_44x_tlb_put(struct kvm_vcpu *vcpu); -void kvmppc_44x_tlb_load(struct kvm_vcpu *vcpu); - -#endif /* __ASM_44X_H__ */ diff --git a/arch/powerpc/include/asm/kvm_asm.h b/arch/powerpc/include/asm/kvm_asm.h index 851bac7afa4b..f9af8df09077 100644 --- a/arch/powerpc/include/asm/kvm_asm.h +++ b/arch/powerpc/include/asm/kvm_asm.h @@ -1,16 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * * Copyright IBM Corp. 2008 * @@ -20,7 +9,7 @@ #ifndef __POWERPC_KVM_ASM_H__ #define __POWERPC_KVM_ASM_H__ -#ifdef __ASSEMBLY__ +#ifdef __ASSEMBLER__ #ifdef CONFIG_64BIT #define PPC_STD(sreg, offset, areg) std sreg, (offset)(areg) #define PPC_LD(treg, offset, areg) ld treg, (offset)(areg) @@ -33,7 +22,6 @@ /* IVPR must be 64KiB-aligned. */ #define VCPU_SIZE_ORDER 4 #define VCPU_SIZE_LOG (VCPU_SIZE_ORDER + 12) -#define VCPU_TLB_PGSZ PPC44x_TLB_64K #define VCPU_SIZE_BYTES (1<<VCPU_SIZE_LOG) #define BOOKE_INTERRUPT_CRITICAL 0 @@ -54,17 +42,17 @@ #define BOOKE_INTERRUPT_DEBUG 15 /* E500 */ -#define BOOKE_INTERRUPT_SPE_ALTIVEC_UNAVAIL 32 -#define BOOKE_INTERRUPT_SPE_FP_DATA_ALTIVEC_ASSIST 33 -/* - * TODO: Unify 32-bit and 64-bit kernel exception handlers to use same defines - */ -#define BOOKE_INTERRUPT_SPE_UNAVAIL BOOKE_INTERRUPT_SPE_ALTIVEC_UNAVAIL -#define BOOKE_INTERRUPT_SPE_FP_DATA BOOKE_INTERRUPT_SPE_FP_DATA_ALTIVEC_ASSIST -#define BOOKE_INTERRUPT_ALTIVEC_UNAVAIL BOOKE_INTERRUPT_SPE_ALTIVEC_UNAVAIL -#define BOOKE_INTERRUPT_ALTIVEC_ASSIST \ - BOOKE_INTERRUPT_SPE_FP_DATA_ALTIVEC_ASSIST +#ifdef CONFIG_SPE_POSSIBLE +#define BOOKE_INTERRUPT_SPE_UNAVAIL 32 +#define BOOKE_INTERRUPT_SPE_FP_DATA 33 #define BOOKE_INTERRUPT_SPE_FP_ROUND 34 +#endif + +#ifdef CONFIG_PPC_E500MC +#define BOOKE_INTERRUPT_ALTIVEC_UNAVAIL 32 +#define BOOKE_INTERRUPT_ALTIVEC_ASSIST 33 +#endif + #define BOOKE_INTERRUPT_PERFORMANCE_MONITOR 35 #define BOOKE_INTERRUPT_DOORBELL 36 #define BOOKE_INTERRUPT_DOORBELL_CRITICAL 37 @@ -74,6 +62,7 @@ #define BOOKE_INTERRUPT_GUEST_DBELL_CRIT 39 #define BOOKE_INTERRUPT_HV_SYSCALL 40 #define BOOKE_INTERRUPT_HV_PRIV 41 +#define BOOKE_INTERRUPT_LRAT_ERROR 42 /* book3s */ @@ -84,21 +73,38 @@ #define BOOK3S_INTERRUPT_INST_STORAGE 0x400 #define BOOK3S_INTERRUPT_INST_SEGMENT 0x480 #define BOOK3S_INTERRUPT_EXTERNAL 0x500 -#define BOOK3S_INTERRUPT_EXTERNAL_LEVEL 0x501 #define BOOK3S_INTERRUPT_EXTERNAL_HV 0x502 #define BOOK3S_INTERRUPT_ALIGNMENT 0x600 #define BOOK3S_INTERRUPT_PROGRAM 0x700 #define BOOK3S_INTERRUPT_FP_UNAVAIL 0x800 #define BOOK3S_INTERRUPT_DECREMENTER 0x900 #define BOOK3S_INTERRUPT_HV_DECREMENTER 0x980 +#define BOOK3S_INTERRUPT_NESTED_HV_DECREMENTER 0x1980 +#define BOOK3S_INTERRUPT_DOORBELL 0xa00 #define BOOK3S_INTERRUPT_SYSCALL 0xc00 #define BOOK3S_INTERRUPT_TRACE 0xd00 #define BOOK3S_INTERRUPT_H_DATA_STORAGE 0xe00 #define BOOK3S_INTERRUPT_H_INST_STORAGE 0xe20 #define BOOK3S_INTERRUPT_H_EMUL_ASSIST 0xe40 +#define BOOK3S_INTERRUPT_HMI 0xe60 +#define BOOK3S_INTERRUPT_H_DOORBELL 0xe80 +#define BOOK3S_INTERRUPT_H_VIRT 0xea0 #define BOOK3S_INTERRUPT_PERFMON 0xf00 #define BOOK3S_INTERRUPT_ALTIVEC 0xf20 #define BOOK3S_INTERRUPT_VSX 0xf40 +#define BOOK3S_INTERRUPT_FAC_UNAVAIL 0xf60 +#define BOOK3S_INTERRUPT_H_FAC_UNAVAIL 0xf80 + +/* book3s_hv */ + +#define BOOK3S_INTERRUPT_HV_SOFTPATCH 0x1500 + +/* + * Special trap used to indicate to host that this is a + * passthrough interrupt that could not be handled + * completely in the guest. + */ +#define BOOK3S_INTERRUPT_HV_RM_HARD 0x5555 #define BOOK3S_IRQPRIO_SYSTEM_RESET 0 #define BOOK3S_IRQPRIO_DATA_SEGMENT 1 @@ -110,23 +116,27 @@ #define BOOK3S_IRQPRIO_FP_UNAVAIL 7 #define BOOK3S_IRQPRIO_ALTIVEC 8 #define BOOK3S_IRQPRIO_VSX 9 -#define BOOK3S_IRQPRIO_SYSCALL 10 -#define BOOK3S_IRQPRIO_MACHINE_CHECK 11 -#define BOOK3S_IRQPRIO_DEBUG 12 -#define BOOK3S_IRQPRIO_EXTERNAL 13 -#define BOOK3S_IRQPRIO_DECREMENTER 14 -#define BOOK3S_IRQPRIO_PERFORMANCE_MONITOR 15 -#define BOOK3S_IRQPRIO_EXTERNAL_LEVEL 16 +#define BOOK3S_IRQPRIO_FAC_UNAVAIL 10 +#define BOOK3S_IRQPRIO_SYSCALL 11 +#define BOOK3S_IRQPRIO_MACHINE_CHECK 12 +#define BOOK3S_IRQPRIO_DEBUG 13 +#define BOOK3S_IRQPRIO_EXTERNAL 14 +#define BOOK3S_IRQPRIO_DECREMENTER 15 +#define BOOK3S_IRQPRIO_PERFORMANCE_MONITOR 16 #define BOOK3S_IRQPRIO_MAX 17 #define BOOK3S_HFLAG_DCBZ32 0x1 #define BOOK3S_HFLAG_SLB 0x2 #define BOOK3S_HFLAG_PAIRED_SINGLE 0x4 #define BOOK3S_HFLAG_NATIVE_PS 0x8 +#define BOOK3S_HFLAG_MULTI_PGSIZE 0x10 +#define BOOK3S_HFLAG_NEW_TLBIE 0x20 +#define BOOK3S_HFLAG_SPLIT_HACK 0x40 #define RESUME_FLAG_NV (1<<0) /* Reload guest nonvolatile state? */ #define RESUME_FLAG_HOST (1<<1) /* Resume host? */ #define RESUME_FLAG_ARCH1 (1<<2) +#define RESUME_FLAG_ARCH2 (1<<3) #define RESUME_GUEST 0 #define RESUME_GUEST_NV RESUME_FLAG_NV @@ -136,7 +146,13 @@ #define KVM_GUEST_MODE_NONE 0 #define KVM_GUEST_MODE_GUEST 1 #define KVM_GUEST_MODE_SKIP 2 +#define KVM_GUEST_MODE_GUEST_HV 3 +#define KVM_GUEST_MODE_HOST_HV 4 +#define KVM_GUEST_MODE_HV_P9 5 /* ISA >= v3.0 path */ #define KVM_INST_FETCH_FAILED -1 +/* Extract PO and XOP opcode fields */ +#define PO_XOP_OPCODE_MASK 0xfc0007fe + #endif /* __POWERPC_KVM_ASM_H__ */ diff --git a/arch/powerpc/include/asm/kvm_book3s.h b/arch/powerpc/include/asm/kvm_book3s.h index 08891d07aeb6..e1ff291ba891 100644 --- a/arch/powerpc/include/asm/kvm_book3s.h +++ b/arch/powerpc/include/asm/kvm_book3s.h @@ -1,16 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * * Copyright SUSE Linux Products GmbH 2009 * @@ -23,6 +12,7 @@ #include <linux/types.h> #include <linux/kvm_host.h> #include <asm/kvm_book3s_asm.h> +#include <asm/guest-state-buffer.h> struct kvmppc_bat { u64 raw; @@ -58,16 +48,56 @@ struct hpte_cache { struct hlist_node list_pte_long; struct hlist_node list_vpte; struct hlist_node list_vpte_long; +#ifdef CONFIG_PPC_BOOK3S_64 + struct hlist_node list_vpte_64k; +#endif struct rcu_head rcu_head; u64 host_vpn; u64 pfn; ulong slot; struct kvmppc_pte pte; + int pagesize; +}; + +/* + * Struct for a virtual core. + * Note: entry_exit_map combines a bitmap of threads that have entered + * in the bottom 8 bits and a bitmap of threads that have exited in the + * next 8 bits. This is so that we can atomically set the entry bit + * iff the exit map is 0 without taking a lock. + */ +struct kvmppc_vcore { + int n_runnable; + int num_threads; + int entry_exit_map; + int napping_threads; + int first_vcpuid; + u16 pcpu; + u16 last_cpu; + u8 vcore_state; + u8 in_guest; + struct kvm_vcpu *runnable_threads[MAX_SMT_THREADS]; + struct list_head preempt_list; + spinlock_t lock; + struct rcuwait wait; + spinlock_t stoltb_lock; /* protects stolen_tb and preempt_tb */ + u64 stolen_tb; + u64 preempt_tb; + struct kvm_vcpu *runner; + struct kvm *kvm; + u64 tb_offset; /* guest timebase - host timebase */ + u64 tb_offset_applied; /* timebase offset currently in force */ + ulong lpcr; + u32 arch_compat; + ulong pcr; + ulong dpdes; /* doorbell state (POWER8) */ + ulong vtb; /* virtual timebase */ + ulong conferring_threads; + unsigned int halt_poll_ns; + atomic_t online_count; }; struct kvmppc_vcpu_book3s { - struct kvm_vcpu vcpu; - struct kvmppc_book3s_shadow_vcpu *shadow_vcpu; struct kvmppc_sid_map sid_map[SID_MAP_NUM]; struct { u64 esid; @@ -81,8 +111,7 @@ struct kvmppc_vcpu_book3s { u64 sdr1; u64 hior; u64 msr_mask; - u64 purr_offset; - u64 spurr_offset; + u64 vtb; #ifdef CONFIG_PPC_BOOK3S_32 u32 vsid_pool[VSID_POOL_SIZE]; u32 vsid_next; @@ -99,16 +128,16 @@ struct kvmppc_vcpu_book3s { struct hlist_head hpte_hash_pte_long[HPTEG_HASH_NUM_PTE_LONG]; struct hlist_head hpte_hash_vpte[HPTEG_HASH_NUM_VPTE]; struct hlist_head hpte_hash_vpte_long[HPTEG_HASH_NUM_VPTE_LONG]; +#ifdef CONFIG_PPC_BOOK3S_64 + struct hlist_head hpte_hash_vpte_64k[HPTEG_HASH_NUM_VPTE_64K]; +#endif int hpte_cache_count; spinlock_t mmu_lock; }; -#define CONTEXT_HOST 0 -#define CONTEXT_GUEST 1 -#define CONTEXT_GUEST_END 2 - -#define VSID_REAL 0x0fffffffffc00000ULL -#define VSID_BAT 0x0fffffffffb00000ULL +#define VSID_REAL 0x07ffffffffc00000ULL +#define VSID_BAT 0x07ffffffffb00000ULL +#define VSID_64K 0x0800000000000000ULL #define VSID_1T 0x1000000000000000ULL #define VSID_REAL_DR 0x2000000000000000ULL #define VSID_REAL_IR 0x4000000000000000ULL @@ -118,344 +147,495 @@ extern void kvmppc_mmu_pte_flush(struct kvm_vcpu *vcpu, ulong ea, ulong ea_mask) extern void kvmppc_mmu_pte_vflush(struct kvm_vcpu *vcpu, u64 vp, u64 vp_mask); extern void kvmppc_mmu_pte_pflush(struct kvm_vcpu *vcpu, ulong pa_start, ulong pa_end); extern void kvmppc_set_msr(struct kvm_vcpu *vcpu, u64 new_msr); -extern void kvmppc_set_pvr(struct kvm_vcpu *vcpu, u32 pvr); extern void kvmppc_mmu_book3s_64_init(struct kvm_vcpu *vcpu); extern void kvmppc_mmu_book3s_32_init(struct kvm_vcpu *vcpu); extern void kvmppc_mmu_book3s_hv_init(struct kvm_vcpu *vcpu); -extern int kvmppc_mmu_map_page(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte); +extern int kvmppc_mmu_map_page(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte, + bool iswrite); +extern void kvmppc_mmu_unmap_page(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte); extern int kvmppc_mmu_map_segment(struct kvm_vcpu *vcpu, ulong eaddr); extern void kvmppc_mmu_flush_segment(struct kvm_vcpu *vcpu, ulong eaddr, ulong seg_size); extern void kvmppc_mmu_flush_segments(struct kvm_vcpu *vcpu); -extern int kvmppc_book3s_hv_page_fault(struct kvm_run *run, - struct kvm_vcpu *vcpu, unsigned long addr, - unsigned long status); +extern int kvmppc_book3s_hv_page_fault(struct kvm_vcpu *vcpu, + unsigned long addr, unsigned long status); extern long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v, unsigned long valid); +extern int kvmppc_hv_emulate_mmio(struct kvm_vcpu *vcpu, + unsigned long gpa, gva_t ea, int is_store); extern void kvmppc_mmu_hpte_cache_map(struct kvm_vcpu *vcpu, struct hpte_cache *pte); extern struct hpte_cache *kvmppc_mmu_hpte_cache_next(struct kvm_vcpu *vcpu); +extern void kvmppc_mmu_hpte_cache_free(struct hpte_cache *pte); extern void kvmppc_mmu_hpte_destroy(struct kvm_vcpu *vcpu); extern int kvmppc_mmu_hpte_init(struct kvm_vcpu *vcpu); extern void kvmppc_mmu_invalidate_pte(struct kvm_vcpu *vcpu, struct hpte_cache *pte); extern int kvmppc_mmu_hpte_sysinit(void); extern void kvmppc_mmu_hpte_sysexit(void); extern int kvmppc_mmu_hv_init(void); +extern int kvmppc_book3s_hcall_implemented(struct kvm *kvm, unsigned long hc); + +extern int kvmppc_book3s_radix_page_fault(struct kvm_vcpu *vcpu, + unsigned long ea, unsigned long dsisr); +extern unsigned long __kvmhv_copy_tofrom_guest_radix(int lpid, int pid, + gva_t eaddr, void *to, void *from, + unsigned long n); +extern long kvmhv_copy_from_guest_radix(struct kvm_vcpu *vcpu, gva_t eaddr, + void *to, unsigned long n); +extern long kvmhv_copy_to_guest_radix(struct kvm_vcpu *vcpu, gva_t eaddr, + void *from, unsigned long n); +extern int kvmppc_mmu_walk_radix_tree(struct kvm_vcpu *vcpu, gva_t eaddr, + struct kvmppc_pte *gpte, u64 root, + u64 *pte_ret_p); +extern int kvmppc_mmu_radix_translate_table(struct kvm_vcpu *vcpu, gva_t eaddr, + struct kvmppc_pte *gpte, u64 table, + int table_index, u64 *pte_ret_p); +extern int kvmppc_mmu_radix_xlate(struct kvm_vcpu *vcpu, gva_t eaddr, + struct kvmppc_pte *gpte, bool data, bool iswrite); +extern void kvmppc_radix_tlbie_page(struct kvm *kvm, unsigned long addr, + unsigned int pshift, u64 lpid); +extern void kvmppc_unmap_pte(struct kvm *kvm, pte_t *pte, unsigned long gpa, + unsigned int shift, + const struct kvm_memory_slot *memslot, + u64 lpid); +extern bool kvmppc_hv_handle_set_rc(struct kvm *kvm, bool nested, + bool writing, unsigned long gpa, + u64 lpid); +extern int kvmppc_book3s_instantiate_page(struct kvm_vcpu *vcpu, + unsigned long gpa, + struct kvm_memory_slot *memslot, + bool writing, + pte_t *inserted_pte, unsigned int *levelp); +extern int kvmppc_init_vm_radix(struct kvm *kvm); +extern void kvmppc_free_radix(struct kvm *kvm); +extern void kvmppc_free_pgtable_radix(struct kvm *kvm, pgd_t *pgd, + u64 lpid); +extern int kvmppc_radix_init(void); +extern void kvmppc_radix_exit(void); +extern void kvm_unmap_radix(struct kvm *kvm, struct kvm_memory_slot *memslot, + unsigned long gfn); +extern bool kvm_age_radix(struct kvm *kvm, struct kvm_memory_slot *memslot, + unsigned long gfn); +extern bool kvm_test_age_radix(struct kvm *kvm, struct kvm_memory_slot *memslot, + unsigned long gfn); +extern long kvmppc_hv_get_dirty_log_radix(struct kvm *kvm, + struct kvm_memory_slot *memslot, unsigned long *map); +extern void kvmppc_radix_flush_memslot(struct kvm *kvm, + const struct kvm_memory_slot *memslot); +extern int kvmhv_get_rmmu_info(struct kvm *kvm, struct kvm_ppc_rmmu_info *info); +/* XXX remove this export when load_last_inst() is generic */ extern int kvmppc_ld(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr, bool data); -extern int kvmppc_st(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr, bool data); extern void kvmppc_book3s_queue_irqprio(struct kvm_vcpu *vcpu, unsigned int vec); extern void kvmppc_book3s_dequeue_irqprio(struct kvm_vcpu *vcpu, unsigned int vec); extern void kvmppc_inject_interrupt(struct kvm_vcpu *vcpu, int vec, u64 flags); +extern void kvmppc_trigger_fac_interrupt(struct kvm_vcpu *vcpu, ulong fac); extern void kvmppc_set_bat(struct kvm_vcpu *vcpu, struct kvmppc_bat *bat, bool upper, u32 val); extern void kvmppc_giveup_ext(struct kvm_vcpu *vcpu, ulong msr); -extern int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu); -extern pfn_t kvmppc_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn); +extern int kvmppc_emulate_paired_single(struct kvm_vcpu *vcpu); +extern kvm_pfn_t kvmppc_gpa_to_pfn(struct kvm_vcpu *vcpu, gpa_t gpa, + bool writing, bool *writable, struct page **page); extern void kvmppc_add_revmap_chain(struct kvm *kvm, struct revmap_entry *rev, unsigned long *rmap, long pte_index, int realmode); -extern void kvmppc_invalidate_hpte(struct kvm *kvm, unsigned long *hptep, +extern void kvmppc_update_dirty_map(const struct kvm_memory_slot *memslot, + unsigned long gfn, unsigned long psize); +extern void kvmppc_invalidate_hpte(struct kvm *kvm, __be64 *hptep, unsigned long pte_index); -void kvmppc_clear_ref_hpte(struct kvm *kvm, unsigned long *hptep, +void kvmppc_clear_ref_hpte(struct kvm *kvm, __be64 *hptep, unsigned long pte_index); extern void *kvmppc_pin_guest_page(struct kvm *kvm, unsigned long addr, unsigned long *nb_ret); extern void kvmppc_unpin_guest_page(struct kvm *kvm, void *addr, unsigned long gpa, bool dirty); -extern long kvmppc_virtmode_h_enter(struct kvm_vcpu *vcpu, unsigned long flags, - long pte_index, unsigned long pteh, unsigned long ptel); extern long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags, long pte_index, unsigned long pteh, unsigned long ptel, pgd_t *pgdir, bool realmode, unsigned long *idx_ret); extern long kvmppc_do_h_remove(struct kvm *kvm, unsigned long flags, unsigned long pte_index, unsigned long avpn, unsigned long *hpret); -extern long kvmppc_hv_get_dirty_log(struct kvm *kvm, +extern long kvmppc_hv_get_dirty_log_hpt(struct kvm *kvm, struct kvm_memory_slot *memslot, unsigned long *map); +extern void kvmppc_harvest_vpa_dirty(struct kvmppc_vpa *vpa, + struct kvm_memory_slot *memslot, + unsigned long *map); +extern unsigned long kvmppc_filter_lpcr_hv(struct kvm *kvm, + unsigned long lpcr); +extern void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, + unsigned long mask); +extern void kvmppc_set_fscr(struct kvm_vcpu *vcpu, u64 fscr); + +extern int kvmhv_p9_tm_emulation_early(struct kvm_vcpu *vcpu); +extern int kvmhv_p9_tm_emulation(struct kvm_vcpu *vcpu); +extern void kvmhv_emulate_tm_rollback(struct kvm_vcpu *vcpu); extern void kvmppc_entry_trampoline(void); extern void kvmppc_hv_entry_trampoline(void); -extern void kvmppc_load_up_fpu(void); -extern void kvmppc_load_up_altivec(void); -extern void kvmppc_load_up_vsx(void); extern u32 kvmppc_alignment_dsisr(struct kvm_vcpu *vcpu, unsigned int inst); extern ulong kvmppc_alignment_dar(struct kvm_vcpu *vcpu, unsigned int inst); extern int kvmppc_h_pr(struct kvm_vcpu *vcpu, unsigned long cmd); +extern void kvmppc_pr_init_default_hcalls(struct kvm *kvm); +extern int kvmppc_hcall_impl_pr(unsigned long cmd); +extern int kvmppc_hcall_impl_hv_realmode(unsigned long cmd); +extern void kvmppc_copy_to_svcpu(struct kvm_vcpu *vcpu); +extern void kvmppc_copy_from_svcpu(struct kvm_vcpu *vcpu); + +long kvmppc_read_intr(void); +void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr); +void kvmppc_inject_interrupt_hv(struct kvm_vcpu *vcpu, int vec, u64 srr1_flags); + +#ifdef CONFIG_PPC_TRANSACTIONAL_MEM +void kvmppc_save_tm_pr(struct kvm_vcpu *vcpu); +void kvmppc_restore_tm_pr(struct kvm_vcpu *vcpu); +void kvmppc_save_tm_sprs(struct kvm_vcpu *vcpu); +void kvmppc_restore_tm_sprs(struct kvm_vcpu *vcpu); +#else +static inline void kvmppc_save_tm_pr(struct kvm_vcpu *vcpu) {} +static inline void kvmppc_restore_tm_pr(struct kvm_vcpu *vcpu) {} +static inline void kvmppc_save_tm_sprs(struct kvm_vcpu *vcpu) {} +static inline void kvmppc_restore_tm_sprs(struct kvm_vcpu *vcpu) {} +#endif -static inline struct kvmppc_vcpu_book3s *to_book3s(struct kvm_vcpu *vcpu) +extern unsigned long nested_capabilities; +long kvmhv_nested_init(void); +void kvmhv_nested_exit(void); +void kvmhv_vm_nested_init(struct kvm *kvm); +long kvmhv_set_partition_table(struct kvm_vcpu *vcpu); +long kvmhv_copy_tofrom_guest_nested(struct kvm_vcpu *vcpu); +void kvmhv_flush_lpid(u64 lpid); +void kvmhv_set_ptbl_entry(u64 lpid, u64 dw0, u64 dw1); +void kvmhv_release_all_nested(struct kvm *kvm); +long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu); +long kvmhv_do_nested_tlbie(struct kvm_vcpu *vcpu); +long do_h_rpt_invalidate_pat(struct kvm_vcpu *vcpu, unsigned long lpid, + unsigned long type, unsigned long pg_sizes, + unsigned long start, unsigned long end); +int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, + u64 time_limit, unsigned long lpcr); +void kvmhv_save_hv_regs(struct kvm_vcpu *vcpu, struct hv_guest_state *hr); +void kvmhv_restore_hv_return_state(struct kvm_vcpu *vcpu, + struct hv_guest_state *hr); +long int kvmhv_nested_page_fault(struct kvm_vcpu *vcpu); + +void kvmppc_giveup_fac(struct kvm_vcpu *vcpu, ulong fac); + + +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE + +extern struct static_key_false __kvmhv_is_nestedv2; + +static inline bool kvmhv_is_nestedv2(void) { - return container_of(vcpu, struct kvmppc_vcpu_book3s, vcpu); + return static_branch_unlikely(&__kvmhv_is_nestedv2); } -extern void kvm_return_point(void); +static inline bool kvmhv_is_nestedv1(void) +{ + return !static_branch_likely(&__kvmhv_is_nestedv2); +} -/* Also add subarch specific defines */ +#else + +static inline bool kvmhv_is_nestedv2(void) +{ + return false; +} + +static inline bool kvmhv_is_nestedv1(void) +{ + return false; +} -#ifdef CONFIG_KVM_BOOK3S_32_HANDLER -#include <asm/kvm_book3s_32.h> -#endif -#ifdef CONFIG_KVM_BOOK3S_64_HANDLER -#include <asm/kvm_book3s_64.h> #endif -#ifdef CONFIG_KVM_BOOK3S_PR +int __kvmhv_nestedv2_reload_ptregs(struct kvm_vcpu *vcpu, struct pt_regs *regs); +int __kvmhv_nestedv2_mark_dirty_ptregs(struct kvm_vcpu *vcpu, struct pt_regs *regs); +int __kvmhv_nestedv2_mark_dirty(struct kvm_vcpu *vcpu, u16 iden); +int __kvmhv_nestedv2_cached_reload(struct kvm_vcpu *vcpu, u16 iden); -static inline unsigned long kvmppc_interrupt_offset(struct kvm_vcpu *vcpu) +static inline int kvmhv_nestedv2_reload_ptregs(struct kvm_vcpu *vcpu, + struct pt_regs *regs) { - return to_book3s(vcpu)->hior; + if (kvmhv_is_nestedv2()) + return __kvmhv_nestedv2_reload_ptregs(vcpu, regs); + return 0; +} +static inline int kvmhv_nestedv2_mark_dirty_ptregs(struct kvm_vcpu *vcpu, + struct pt_regs *regs) +{ + if (kvmhv_is_nestedv2()) + return __kvmhv_nestedv2_mark_dirty_ptregs(vcpu, regs); + return 0; } -static inline void kvmppc_update_int_pending(struct kvm_vcpu *vcpu, - unsigned long pending_now, unsigned long old_pending) +static inline int kvmhv_nestedv2_mark_dirty(struct kvm_vcpu *vcpu, u16 iden) { - if (pending_now) - vcpu->arch.shared->int_pending = 1; - else if (old_pending) - vcpu->arch.shared->int_pending = 0; + if (kvmhv_is_nestedv2()) + return __kvmhv_nestedv2_mark_dirty(vcpu, iden); + return 0; } +static inline int kvmhv_nestedv2_cached_reload(struct kvm_vcpu *vcpu, u16 iden) +{ + if (kvmhv_is_nestedv2()) + return __kvmhv_nestedv2_cached_reload(vcpu, iden); + return 0; +} + +extern int kvm_irq_bypass; + +static inline struct kvmppc_vcpu_book3s *to_book3s(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.book3s; +} + +/* Also add subarch specific defines */ + +#ifdef CONFIG_KVM_BOOK3S_32_HANDLER +#include <asm/kvm_book3s_32.h> +#endif +#ifdef CONFIG_KVM_BOOK3S_64_HANDLER +#include <asm/kvm_book3s_64.h> +#endif + static inline void kvmppc_set_gpr(struct kvm_vcpu *vcpu, int num, ulong val) { - if ( num < 14 ) { - struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); - svcpu->gpr[num] = val; - svcpu_put(svcpu); - to_book3s(vcpu)->shadow_vcpu->gpr[num] = val; - } else - vcpu->arch.gpr[num] = val; + vcpu->arch.regs.gpr[num] = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_GPR(num)); } static inline ulong kvmppc_get_gpr(struct kvm_vcpu *vcpu, int num) { - if ( num < 14 ) { - struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); - ulong r = svcpu->gpr[num]; - svcpu_put(svcpu); - return r; - } else - return vcpu->arch.gpr[num]; + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_GPR(num)) < 0); + return vcpu->arch.regs.gpr[num]; } static inline void kvmppc_set_cr(struct kvm_vcpu *vcpu, u32 val) { - struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); - svcpu->cr = val; - svcpu_put(svcpu); - to_book3s(vcpu)->shadow_vcpu->cr = val; + vcpu->arch.regs.ccr = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_CR); } static inline u32 kvmppc_get_cr(struct kvm_vcpu *vcpu) { - struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); - u32 r; - r = svcpu->cr; - svcpu_put(svcpu); - return r; + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_CR) < 0); + return vcpu->arch.regs.ccr; } -static inline void kvmppc_set_xer(struct kvm_vcpu *vcpu, u32 val) +static inline void kvmppc_set_xer(struct kvm_vcpu *vcpu, ulong val) { - struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); - svcpu->xer = val; - to_book3s(vcpu)->shadow_vcpu->xer = val; - svcpu_put(svcpu); + vcpu->arch.regs.xer = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_XER); } -static inline u32 kvmppc_get_xer(struct kvm_vcpu *vcpu) +static inline ulong kvmppc_get_xer(struct kvm_vcpu *vcpu) { - struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); - u32 r; - r = svcpu->xer; - svcpu_put(svcpu); - return r; + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_XER) < 0); + return vcpu->arch.regs.xer; } static inline void kvmppc_set_ctr(struct kvm_vcpu *vcpu, ulong val) { - struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); - svcpu->ctr = val; - svcpu_put(svcpu); + vcpu->arch.regs.ctr = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_CTR); } static inline ulong kvmppc_get_ctr(struct kvm_vcpu *vcpu) { - struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); - ulong r; - r = svcpu->ctr; - svcpu_put(svcpu); - return r; + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_CTR) < 0); + return vcpu->arch.regs.ctr; } static inline void kvmppc_set_lr(struct kvm_vcpu *vcpu, ulong val) { - struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); - svcpu->lr = val; - svcpu_put(svcpu); + vcpu->arch.regs.link = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_LR); } static inline ulong kvmppc_get_lr(struct kvm_vcpu *vcpu) { - struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); - ulong r; - r = svcpu->lr; - svcpu_put(svcpu); - return r; + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_LR) < 0); + return vcpu->arch.regs.link; } static inline void kvmppc_set_pc(struct kvm_vcpu *vcpu, ulong val) { - struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); - svcpu->pc = val; - svcpu_put(svcpu); + vcpu->arch.regs.nip = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_NIA); } static inline ulong kvmppc_get_pc(struct kvm_vcpu *vcpu) { - struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); - ulong r; - r = svcpu->pc; - svcpu_put(svcpu); - return r; + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_NIA) < 0); + return vcpu->arch.regs.nip; } -static inline u32 kvmppc_get_last_inst(struct kvm_vcpu *vcpu) +static inline u64 kvmppc_get_msr(struct kvm_vcpu *vcpu); +static inline bool kvmppc_need_byteswap(struct kvm_vcpu *vcpu) { - ulong pc = kvmppc_get_pc(vcpu); - struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); - u32 r; - - /* Load the instruction manually if it failed to do so in the - * exit path */ - if (svcpu->last_inst == KVM_INST_FETCH_FAILED) - kvmppc_ld(vcpu, &pc, sizeof(u32), &svcpu->last_inst, false); - - r = svcpu->last_inst; - svcpu_put(svcpu); - return r; + return (kvmppc_get_msr(vcpu) & MSR_LE) != (MSR_KERNEL & MSR_LE); } static inline ulong kvmppc_get_fault_dar(struct kvm_vcpu *vcpu) { - struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); - ulong r; - r = svcpu->fault_dar; - svcpu_put(svcpu); - return r; + return vcpu->arch.fault_dar; } -static inline bool kvmppc_critical_section(struct kvm_vcpu *vcpu) +static inline u64 kvmppc_get_fpr(struct kvm_vcpu *vcpu, int i) { - ulong crit_raw = vcpu->arch.shared->critical; - ulong crit_r1 = kvmppc_get_gpr(vcpu, 1); - bool crit; - - /* Truncate crit indicators in 32 bit mode */ - if (!(vcpu->arch.shared->msr & MSR_SF)) { - crit_raw &= 0xffffffff; - crit_r1 &= 0xffffffff; - } - - /* Critical section when crit == r1 */ - crit = (crit_raw == crit_r1); - /* ... and we're in supervisor mode */ - crit = crit && !(vcpu->arch.shared->msr & MSR_PR); - - return crit; + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_VSRS(i)) < 0); + return vcpu->arch.fp.fpr[i][TS_FPROFFSET]; } -#else /* CONFIG_KVM_BOOK3S_PR */ -static inline unsigned long kvmppc_interrupt_offset(struct kvm_vcpu *vcpu) +static inline void kvmppc_set_fpr(struct kvm_vcpu *vcpu, int i, u64 val) { - return 0; + vcpu->arch.fp.fpr[i][TS_FPROFFSET] = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_VSRS(i)); } -static inline void kvmppc_update_int_pending(struct kvm_vcpu *vcpu, - unsigned long pending_now, unsigned long old_pending) +static inline u64 kvmppc_get_fpscr(struct kvm_vcpu *vcpu) { + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_FPSCR) < 0); + return vcpu->arch.fp.fpscr; } -static inline void kvmppc_set_gpr(struct kvm_vcpu *vcpu, int num, ulong val) +static inline void kvmppc_set_fpscr(struct kvm_vcpu *vcpu, u64 val) { - vcpu->arch.gpr[num] = val; + vcpu->arch.fp.fpscr = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_FPSCR); } -static inline ulong kvmppc_get_gpr(struct kvm_vcpu *vcpu, int num) + +static inline u64 kvmppc_get_vsx_fpr(struct kvm_vcpu *vcpu, int i, int j) { - return vcpu->arch.gpr[num]; + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_VSRS(i)) < 0); + return vcpu->arch.fp.fpr[i][j]; } -static inline void kvmppc_set_cr(struct kvm_vcpu *vcpu, u32 val) +static inline void kvmppc_set_vsx_fpr(struct kvm_vcpu *vcpu, int i, int j, + u64 val) { - vcpu->arch.cr = val; + vcpu->arch.fp.fpr[i][j] = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_VSRS(i)); } -static inline u32 kvmppc_get_cr(struct kvm_vcpu *vcpu) +#ifdef CONFIG_ALTIVEC +static inline void kvmppc_get_vsx_vr(struct kvm_vcpu *vcpu, int i, vector128 *v) { - return vcpu->arch.cr; + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_VSRS(32 + i)) < 0); + *v = vcpu->arch.vr.vr[i]; } -static inline void kvmppc_set_xer(struct kvm_vcpu *vcpu, u32 val) +static inline void kvmppc_set_vsx_vr(struct kvm_vcpu *vcpu, int i, + vector128 *val) { - vcpu->arch.xer = val; + vcpu->arch.vr.vr[i] = *val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_VSRS(32 + i)); } -static inline u32 kvmppc_get_xer(struct kvm_vcpu *vcpu) +static inline u32 kvmppc_get_vscr(struct kvm_vcpu *vcpu) { - return vcpu->arch.xer; + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_VSCR) < 0); + return vcpu->arch.vr.vscr.u[3]; } -static inline void kvmppc_set_ctr(struct kvm_vcpu *vcpu, ulong val) +static inline void kvmppc_set_vscr(struct kvm_vcpu *vcpu, u32 val) { - vcpu->arch.ctr = val; + vcpu->arch.vr.vscr.u[3] = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_VSCR); } +#endif -static inline ulong kvmppc_get_ctr(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.ctr; +#define KVMPPC_BOOK3S_VCPU_ACCESSOR_SET(reg, size, iden) \ +static inline void kvmppc_set_##reg(struct kvm_vcpu *vcpu, u##size val) \ +{ \ + \ + vcpu->arch.reg = val; \ + kvmhv_nestedv2_mark_dirty(vcpu, iden); \ } -static inline void kvmppc_set_lr(struct kvm_vcpu *vcpu, ulong val) -{ - vcpu->arch.lr = val; +#define KVMPPC_BOOK3S_VCPU_ACCESSOR_GET(reg, size, iden) \ +static inline u##size kvmppc_get_##reg(struct kvm_vcpu *vcpu) \ +{ \ + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, iden) < 0); \ + return vcpu->arch.reg; \ } -static inline ulong kvmppc_get_lr(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.lr; +#define KVMPPC_BOOK3S_VCPU_ACCESSOR(reg, size, iden) \ + KVMPPC_BOOK3S_VCPU_ACCESSOR_SET(reg, size, iden) \ + KVMPPC_BOOK3S_VCPU_ACCESSOR_GET(reg, size, iden) \ + +KVMPPC_BOOK3S_VCPU_ACCESSOR(pid, 32, KVMPPC_GSID_PIDR) +KVMPPC_BOOK3S_VCPU_ACCESSOR(tar, 64, KVMPPC_GSID_TAR) +KVMPPC_BOOK3S_VCPU_ACCESSOR(ebbhr, 64, KVMPPC_GSID_EBBHR) +KVMPPC_BOOK3S_VCPU_ACCESSOR(ebbrr, 64, KVMPPC_GSID_EBBRR) +KVMPPC_BOOK3S_VCPU_ACCESSOR(bescr, 64, KVMPPC_GSID_BESCR) +KVMPPC_BOOK3S_VCPU_ACCESSOR(ic, 64, KVMPPC_GSID_IC) +KVMPPC_BOOK3S_VCPU_ACCESSOR(vrsave, 64, KVMPPC_GSID_VRSAVE) + + +#define KVMPPC_BOOK3S_VCORE_ACCESSOR_SET(reg, size, iden) \ +static inline void kvmppc_set_##reg(struct kvm_vcpu *vcpu, u##size val) \ +{ \ + vcpu->arch.vcore->reg = val; \ + kvmhv_nestedv2_mark_dirty(vcpu, iden); \ } -static inline void kvmppc_set_pc(struct kvm_vcpu *vcpu, ulong val) -{ - vcpu->arch.pc = val; +#define KVMPPC_BOOK3S_VCORE_ACCESSOR_GET(reg, size, iden) \ +static inline u##size kvmppc_get_##reg(struct kvm_vcpu *vcpu) \ +{ \ + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, iden) < 0); \ + return vcpu->arch.vcore->reg; \ } -static inline ulong kvmppc_get_pc(struct kvm_vcpu *vcpu) +#define KVMPPC_BOOK3S_VCORE_ACCESSOR(reg, size, iden) \ + KVMPPC_BOOK3S_VCORE_ACCESSOR_SET(reg, size, iden) \ + KVMPPC_BOOK3S_VCORE_ACCESSOR_GET(reg, size, iden) \ + + +KVMPPC_BOOK3S_VCORE_ACCESSOR(vtb, 64, KVMPPC_GSID_VTB) +KVMPPC_BOOK3S_VCORE_ACCESSOR(dpdes, 64, KVMPPC_GSID_DPDES) +KVMPPC_BOOK3S_VCORE_ACCESSOR_GET(arch_compat, 32, KVMPPC_GSID_LOGICAL_PVR) +KVMPPC_BOOK3S_VCORE_ACCESSOR_GET(lpcr, 64, KVMPPC_GSID_LPCR) +KVMPPC_BOOK3S_VCORE_ACCESSOR_SET(tb_offset, 64, KVMPPC_GSID_TB_OFFSET) + +static inline u64 kvmppc_get_tb_offset(struct kvm_vcpu *vcpu) { - return vcpu->arch.pc; + return vcpu->arch.vcore->tb_offset; } -static inline u32 kvmppc_get_last_inst(struct kvm_vcpu *vcpu) +static inline u64 kvmppc_get_dec_expires(struct kvm_vcpu *vcpu) { - ulong pc = kvmppc_get_pc(vcpu); + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_DEC_EXPIRY_TB) < 0); + return vcpu->arch.dec_expires; +} - /* Load the instruction manually if it failed to do so in the - * exit path */ - if (vcpu->arch.last_inst == KVM_INST_FETCH_FAILED) - kvmppc_ld(vcpu, &pc, sizeof(u32), &vcpu->arch.last_inst, false); +static inline void kvmppc_set_dec_expires(struct kvm_vcpu *vcpu, u64 val) +{ + vcpu->arch.dec_expires = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_DEC_EXPIRY_TB); +} - return vcpu->arch.last_inst; +/* Expiry time of vcpu DEC relative to host TB */ +static inline u64 kvmppc_dec_expires_host_tb(struct kvm_vcpu *vcpu) +{ + return kvmppc_get_dec_expires(vcpu) - kvmppc_get_tb_offset(vcpu); } -static inline ulong kvmppc_get_fault_dar(struct kvm_vcpu *vcpu) +static inline bool is_kvmppc_resume_guest(int r) { - return vcpu->arch.fault_dar; + return (r == RESUME_GUEST || r == RESUME_GUEST_NV); } -static inline bool kvmppc_critical_section(struct kvm_vcpu *vcpu) +static inline bool is_kvmppc_hv_enabled(struct kvm *kvm); +static inline bool kvmppc_supports_magic_page(struct kvm_vcpu *vcpu) { - return false; + /* Only PR KVM supports the magic page */ + return !is_kvmppc_hv_enabled(vcpu->kvm); } -#endif + +extern int kvmppc_h_logical_ci_load(struct kvm_vcpu *vcpu); +extern int kvmppc_h_logical_ci_store(struct kvm_vcpu *vcpu); /* Magic register values loaded into r3 and r4 before the 'sc' assembly * instruction for the OSI hypercalls */ @@ -466,7 +646,54 @@ static inline bool kvmppc_critical_section(struct kvm_vcpu *vcpu) /* TO = 31 for unconditional trap */ #define INS_TW 0x7fe00008 -/* LPIDs we support with this build -- runtime limit may be lower */ -#define KVMPPC_NR_LPIDS (LPID_RSVD + 1) +#define SPLIT_HACK_MASK 0xff000000 +#define SPLIT_HACK_OFFS 0xfb000000 + +/* + * This packs a VCPU ID from the [0..KVM_MAX_VCPU_IDS) space down to the + * [0..KVM_MAX_VCPUS) space, using knowledge of the guest's core stride + * (but not its actual threading mode, which is not available) to avoid + * collisions. + * + * The implementation leaves VCPU IDs from the range [0..KVM_MAX_VCPUS) (block + * 0) unchanged: if the guest is filling each VCORE completely then it will be + * using consecutive IDs and it will fill the space without any packing. + * + * For higher VCPU IDs, the packed ID is based on the VCPU ID modulo + * KVM_MAX_VCPUS (effectively masking off the top bits) and then an offset is + * added to avoid collisions. + * + * VCPU IDs in the range [KVM_MAX_VCPUS..(KVM_MAX_VCPUS*2)) (block 1) are only + * possible if the guest is leaving at least 1/2 of each VCORE empty, so IDs + * can be safely packed into the second half of each VCORE by adding an offset + * of (stride / 2). + * + * Similarly, if VCPU IDs in the range [(KVM_MAX_VCPUS*2)..(KVM_MAX_VCPUS*4)) + * (blocks 2 and 3) are seen, the guest must be leaving at least 3/4 of each + * VCORE empty so packed IDs can be offset by (stride / 4) and (stride * 3 / 4). + * + * Finally, VCPU IDs from blocks 5..7 will only be seen if the guest is using a + * stride of 8 and 1 thread per core so the remaining offsets of 1, 5, 3 and 7 + * must be free to use. + * + * (The offsets for each block are stored in block_offsets[], indexed by the + * block number if the stride is 8. For cases where the guest's stride is less + * than 8, we can re-use the block_offsets array by multiplying the block + * number by (MAX_SMT_THREADS / stride) to reach the correct entry.) + */ +static inline u32 kvmppc_pack_vcpu_id(struct kvm *kvm, u32 id) +{ + const int block_offsets[MAX_SMT_THREADS] = {0, 4, 2, 6, 1, 5, 3, 7}; + int stride = kvm->arch.emul_smt_mode; + int block = (id / KVM_MAX_VCPUS) * (MAX_SMT_THREADS / stride); + u32 packed_id; + + if (WARN_ONCE(block >= MAX_SMT_THREADS, "VCPU ID too large to pack")) + return 0; + packed_id = (id % KVM_MAX_VCPUS) + block_offsets[block]; + if (WARN_ONCE(packed_id >= KVM_MAX_VCPUS, "VCPU ID packing failed")) + return 0; + return packed_id; +} #endif /* __ASM_KVM_BOOK3S_H__ */ diff --git a/arch/powerpc/include/asm/kvm_book3s_32.h b/arch/powerpc/include/asm/kvm_book3s_32.h index ce0ef6ce8f86..e9d2e8463105 100644 --- a/arch/powerpc/include/asm/kvm_book3s_32.h +++ b/arch/powerpc/include/asm/kvm_book3s_32.h @@ -1,16 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * * Copyright SUSE Linux Products GmbH 2010 * @@ -22,7 +11,7 @@ static inline struct kvmppc_book3s_shadow_vcpu *svcpu_get(struct kvm_vcpu *vcpu) { - return to_book3s(vcpu)->shadow_vcpu; + return vcpu->arch.shadow_vcpu; } static inline void svcpu_put(struct kvmppc_book3s_shadow_vcpu *svcpu) diff --git a/arch/powerpc/include/asm/kvm_book3s_64.h b/arch/powerpc/include/asm/kvm_book3s_64.h index a1ecb14e4442..b936e174eefd 100644 --- a/arch/powerpc/include/asm/kvm_book3s_64.h +++ b/arch/powerpc/include/asm/kvm_book3s_64.h @@ -1,16 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * * Copyright SUSE Linux Products GmbH 2010 * @@ -20,7 +9,106 @@ #ifndef __ASM_KVM_BOOK3S_64_H__ #define __ASM_KVM_BOOK3S_64_H__ -#ifdef CONFIG_KVM_BOOK3S_PR +#include <linux/string.h> +#include <asm/bitops.h> +#include <asm/book3s/64/mmu-hash.h> +#include <asm/cpu_has_feature.h> +#include <asm/ppc-opcode.h> +#include <asm/pte-walk.h> + +/* + * Structure for a nested guest, that is, for a guest that is managed by + * one of our guests. + */ +struct kvm_nested_guest { + struct kvm *l1_host; /* L1 VM that owns this nested guest */ + int l1_lpid; /* lpid L1 guest thinks this guest is */ + int shadow_lpid; /* real lpid of this nested guest */ + pgd_t *shadow_pgtable; /* our page table for this guest */ + u64 l1_gr_to_hr; /* L1's addr of part'n-scoped table */ + u64 process_table; /* process table entry for this guest */ + long refcnt; /* number of pointers to this struct */ + struct mutex tlb_lock; /* serialize page faults and tlbies */ + struct kvm_nested_guest *next; + cpumask_t need_tlb_flush; + short prev_cpu[NR_CPUS]; + u8 radix; /* is this nested guest radix */ +}; + +/* + * We define a nested rmap entry as a single 64-bit quantity + * 0xFFF0000000000000 12-bit lpid field + * 0x000FFFFFFFFFF000 40-bit guest 4k page frame number + * 0x0000000000000001 1-bit single entry flag + */ +#define RMAP_NESTED_LPID_MASK 0xFFF0000000000000UL +#define RMAP_NESTED_LPID_SHIFT (52) +#define RMAP_NESTED_GPA_MASK 0x000FFFFFFFFFF000UL +#define RMAP_NESTED_IS_SINGLE_ENTRY 0x0000000000000001UL + +/* Structure for a nested guest rmap entry */ +struct rmap_nested { + struct llist_node list; + u64 rmap; +}; + +/* + * for_each_nest_rmap_safe - iterate over the list of nested rmap entries + * safe against removal of the list entry or NULL list + * @pos: a (struct rmap_nested *) to use as a loop cursor + * @node: pointer to the first entry + * NOTE: this can be NULL + * @rmapp: an (unsigned long *) in which to return the rmap entries on each + * iteration + * NOTE: this must point to already allocated memory + * + * The nested_rmap is a llist of (struct rmap_nested) entries pointed to by the + * rmap entry in the memslot. The list is always terminated by a "single entry" + * stored in the list element of the final entry of the llist. If there is ONLY + * a single entry then this is itself in the rmap entry of the memslot, not a + * llist head pointer. + * + * Note that the iterator below assumes that a nested rmap entry is always + * non-zero. This is true for our usage because the LPID field is always + * non-zero (zero is reserved for the host). + * + * This should be used to iterate over the list of rmap_nested entries with + * processing done on the u64 rmap value given by each iteration. This is safe + * against removal of list entries and it is always safe to call free on (pos). + * + * e.g. + * struct rmap_nested *cursor; + * struct llist_node *first; + * unsigned long rmap; + * for_each_nest_rmap_safe(cursor, first, &rmap) { + * do_something(rmap); + * free(cursor); + * } + */ +#define for_each_nest_rmap_safe(pos, node, rmapp) \ + for ((pos) = llist_entry((node), typeof(*(pos)), list); \ + (node) && \ + (*(rmapp) = ((RMAP_NESTED_IS_SINGLE_ENTRY & ((u64) (node))) ? \ + ((u64) (node)) : ((pos)->rmap))) && \ + (((node) = ((RMAP_NESTED_IS_SINGLE_ENTRY & ((u64) (node))) ? \ + ((struct llist_node *) ((pos) = NULL)) : \ + (pos)->list.next)), true); \ + (pos) = llist_entry((node), typeof(*(pos)), list)) + +struct kvm_nested_guest *kvmhv_get_nested(struct kvm *kvm, int l1_lpid, + bool create); +void kvmhv_put_nested(struct kvm_nested_guest *gp); +int kvmhv_nested_next_lpid(struct kvm *kvm, int lpid); + +/* Encoding of first parameter for H_TLB_INVALIDATE */ +#define H_TLBIE_P1_ENC(ric, prs, r) (___PPC_RIC(ric) | ___PPC_PRS(prs) | \ + ___PPC_R(r)) + +/* Power architecture requires HPT is at least 256kiB, at most 64TiB */ +#define PPC_MIN_HPT_ORDER 18 +#define PPC_MAX_HPT_ORDER 46 + +#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE static inline struct kvmppc_book3s_shadow_vcpu *svcpu_get(struct kvm_vcpu *vcpu) { preempt_disable(); @@ -33,14 +121,37 @@ static inline void svcpu_put(struct kvmppc_book3s_shadow_vcpu *svcpu) } #endif -#define SPAPR_TCE_SHIFT 12 +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE + +static inline bool kvm_is_radix(struct kvm *kvm) +{ + return kvm->arch.radix; +} + +static inline bool kvmhv_vcpu_is_radix(struct kvm_vcpu *vcpu) +{ + bool radix; + + if (vcpu->arch.nested) + radix = vcpu->arch.nested->radix; + else + radix = kvm_is_radix(vcpu->kvm); + + return radix; +} + +unsigned long kvmppc_msr_hard_disable_set_facilities(struct kvm_vcpu *vcpu, unsigned long msr); + +int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcr, u64 *tb); -#ifdef CONFIG_KVM_BOOK3S_64_HV #define KVM_DEFAULT_HPT_ORDER 24 /* 16MB HPT by default */ -extern int kvm_hpt_order; /* order of preallocated HPTs */ #endif -#define VRMA_VSID 0x1ffffffUL /* 1TB VSID reserved for VRMA */ +/* + * Invalid HDSISR value which is used to indicate when HW has not set the reg. + * Used to work around an errata. + */ +#define HDSISR_CANARY 0x7fff /* * We use a lock bit in HPTE dword 0 to synchronize updates and @@ -59,69 +170,210 @@ extern int kvm_hpt_order; /* order of preallocated HPTs */ /* These bits are reserved in the guest view of the HPTE */ #define HPTE_GR_RESERVED HPTE_GR_MODIFIED -static inline long try_lock_hpte(unsigned long *hpte, unsigned long bits) +static inline long try_lock_hpte(__be64 *hpte, unsigned long bits) { unsigned long tmp, old; + __be64 be_lockbit, be_bits; + + /* + * We load/store in native endian, but the HTAB is in big endian. If + * we byte swap all data we apply on the PTE we're implicitly correct + * again. + */ + be_lockbit = cpu_to_be64(HPTE_V_HVLOCK); + be_bits = cpu_to_be64(bits); asm volatile(" ldarx %0,0,%2\n" " and. %1,%0,%3\n" " bne 2f\n" - " ori %0,%0,%4\n" + " or %0,%0,%4\n" " stdcx. %0,0,%2\n" " beq+ 2f\n" " mr %1,%3\n" "2: isync" : "=&r" (tmp), "=&r" (old) - : "r" (hpte), "r" (bits), "i" (HPTE_V_HVLOCK) + : "r" (hpte), "r" (be_bits), "r" (be_lockbit) : "cc", "memory"); return old == 0; } +static inline void unlock_hpte(__be64 *hpte, unsigned long hpte_v) +{ + hpte_v &= ~HPTE_V_HVLOCK; + asm volatile(PPC_RELEASE_BARRIER "" : : : "memory"); + hpte[0] = cpu_to_be64(hpte_v); +} + +/* Without barrier */ +static inline void __unlock_hpte(__be64 *hpte, unsigned long hpte_v) +{ + hpte_v &= ~HPTE_V_HVLOCK; + hpte[0] = cpu_to_be64(hpte_v); +} + +/* + * These functions encode knowledge of the POWER7/8/9 hardware + * interpretations of the HPTE LP (large page size) field. + */ +static inline int kvmppc_hpte_page_shifts(unsigned long h, unsigned long l) +{ + unsigned int lphi; + + if (!(h & HPTE_V_LARGE)) + return 12; /* 4kB */ + lphi = (l >> 16) & 0xf; + switch ((l >> 12) & 0xf) { + case 0: + return !lphi ? 24 : 0; /* 16MB */ + break; + case 1: + return 16; /* 64kB */ + break; + case 3: + return !lphi ? 34 : 0; /* 16GB */ + break; + case 7: + return (16 << 8) + 12; /* 64kB in 4kB */ + break; + case 8: + if (!lphi) + return (24 << 8) + 16; /* 16MB in 64kkB */ + if (lphi == 3) + return (24 << 8) + 12; /* 16MB in 4kB */ + break; + } + return 0; +} + +static inline int kvmppc_hpte_base_page_shift(unsigned long h, unsigned long l) +{ + return kvmppc_hpte_page_shifts(h, l) & 0xff; +} + +static inline int kvmppc_hpte_actual_page_shift(unsigned long h, unsigned long l) +{ + int tmp = kvmppc_hpte_page_shifts(h, l); + + if (tmp >= 0x100) + tmp >>= 8; + return tmp; +} + +static inline unsigned long kvmppc_actual_pgsz(unsigned long v, unsigned long r) +{ + int shift = kvmppc_hpte_actual_page_shift(v, r); + + if (shift) + return 1ul << shift; + return 0; +} + +static inline int kvmppc_pgsize_lp_encoding(int base_shift, int actual_shift) +{ + switch (base_shift) { + case 12: + switch (actual_shift) { + case 12: + return 0; + case 16: + return 7; + case 24: + return 0x38; + } + break; + case 16: + switch (actual_shift) { + case 16: + return 1; + case 24: + return 8; + } + break; + case 24: + return 0; + } + return -1; +} + static inline unsigned long compute_tlbie_rb(unsigned long v, unsigned long r, unsigned long pte_index) { - unsigned long rb, va_low; + int a_pgshift, b_pgshift; + unsigned long rb = 0, va_low, sllp; + + b_pgshift = a_pgshift = kvmppc_hpte_page_shifts(v, r); + if (a_pgshift >= 0x100) { + b_pgshift &= 0xff; + a_pgshift >>= 8; + } + /* + * Ignore the top 14 bits of va + * v have top two bits covering segment size, hence move + * by 16 bits, Also clear the lower HPTE_V_AVPN_SHIFT (7) bits. + * AVA field in v also have the lower 23 bits ignored. + * For base page size 4K we need 14 .. 65 bits (so need to + * collect extra 11 bits) + * For others we need 14..14+i + */ + /* This covers 14..54 bits of va*/ rb = (v & ~0x7fUL) << 16; /* AVA field */ + + /* + * AVA in v had cleared lower 23 bits. We need to derive + * that from pteg index + */ va_low = pte_index >> 3; if (v & HPTE_V_SECONDARY) va_low = ~va_low; - /* xor vsid from AVA */ + /* + * get the vpn bits from va_low using reverse of hashing. + * In v we have va with 23 bits dropped and then left shifted + * HPTE_V_AVPN_SHIFT (7) bits. Now to find vsid we need + * right shift it with (SID_SHIFT - (23 - 7)) + */ if (!(v & HPTE_V_1TB_SEG)) - va_low ^= v >> 12; + va_low ^= v >> (SID_SHIFT - 16); else - va_low ^= v >> 24; + va_low ^= v >> (SID_SHIFT_1T - 16); va_low &= 0x7ff; - if (v & HPTE_V_LARGE) { - rb |= 1; /* L field */ - if (cpu_has_feature(CPU_FTR_ARCH_206) && - (r & 0xff000)) { - /* non-16MB large page, must be 64k */ - /* (masks depend on page size) */ - rb |= 0x1000; /* page encoding in LP field */ - rb |= (va_low & 0x7f) << 16; /* 7b of VA in AVA/LP field */ - rb |= (va_low & 0xfe); /* AVAL field (P7 doesn't seem to care) */ + + if (b_pgshift <= 12) { + if (a_pgshift > 12) { + sllp = (a_pgshift == 16) ? 5 : 4; + rb |= sllp << 5; /* AP field */ } + rb |= (va_low & 0x7ff) << 12; /* remaining 11 bits of AVA */ } else { - /* 4kB page */ - rb |= (va_low & 0x7ff) << 12; /* remaining 11b of VA */ + int aval_shift; + /* + * remaining bits of AVA/LP fields + * Also contain the rr bits of LP + */ + rb |= (va_low << b_pgshift) & 0x7ff000; + /* + * Now clear not needed LP bits based on actual psize + */ + rb &= ~((1ul << a_pgshift) - 1); + /* + * AVAL field 58..77 - base_page_shift bits of va + * we have space for 58..64 bits, Missing bits should + * be zero filled. +1 is to take care of L bit shift + */ + aval_shift = 64 - (77 - b_pgshift) + 1; + rb |= ((va_low << aval_shift) & 0xfe); + + rb |= 1; /* L field */ + rb |= r & 0xff000 & ((1ul << a_pgshift) - 1); /* LP field */ } - rb |= (v >> 54) & 0x300; /* B field */ + /* + * This sets both bits of the B field in the PTE. 0b1x values are + * reserved, but those will have been filtered by kvmppc_do_h_enter. + */ + rb |= (v >> HPTE_V_SSIZE_SHIFT) << 8; /* B field */ return rb; } -static inline unsigned long hpte_page_size(unsigned long h, unsigned long l) -{ - /* only handle 4k, 64k and 16M pages for now */ - if (!(h & HPTE_V_LARGE)) - return 1ul << 12; /* 4k page */ - if ((l & 0xf000) == 0x1000 && cpu_has_feature(CPU_FTR_ARCH_206)) - return 1ul << 16; /* 64k page */ - if ((l & 0xff000) == 0) - return 1ul << 24; /* 16M page */ - return 0; /* error */ -} - static inline unsigned long hpte_rpn(unsigned long ptel, unsigned long psize) { return ((ptel & HPTE_R_RPN) & ~(psize - 1)) >> PAGE_SHIFT; @@ -143,78 +395,65 @@ static inline unsigned long hpte_make_readonly(unsigned long ptel) return ptel; } -static inline int hpte_cache_flags_ok(unsigned long ptel, unsigned long io_type) +static inline bool hpte_cache_flags_ok(unsigned long hptel, bool is_ci) { - unsigned int wimg = ptel & HPTE_R_WIMG; + unsigned int wimg = hptel & HPTE_R_WIMG; /* Handle SAO */ if (wimg == (HPTE_R_W | HPTE_R_I | HPTE_R_M) && cpu_has_feature(CPU_FTR_ARCH_206)) wimg = HPTE_R_M; - if (!io_type) + if (!is_ci) return wimg == HPTE_R_M; - - return (wimg & (HPTE_R_W | HPTE_R_I)) == io_type; + /* + * if host is mapped cache inhibited, make sure hptel also have + * cache inhibited. + */ + if (wimg & HPTE_R_W) /* FIXME!! is this ok for all guest. ? */ + return false; + return !!(wimg & HPTE_R_I); } /* * If it's present and writable, atomically set dirty and referenced bits and - * return the PTE, otherwise return 0. If we find a transparent hugepage - * and if it is marked splitting we return 0; + * return the PTE, otherwise return 0. */ -static inline pte_t kvmppc_read_update_linux_pte(pte_t *ptep, int writing, - unsigned int hugepage) +static inline pte_t kvmppc_read_update_linux_pte(pte_t *ptep, int writing) { pte_t old_pte, new_pte = __pte(0); while (1) { - old_pte = pte_val(*ptep); /* - * wait until _PAGE_BUSY is clear then set it atomically + * Make sure we don't reload from ptep + */ + old_pte = READ_ONCE(*ptep); + /* + * wait until H_PAGE_BUSY is clear then set it atomically */ - if (unlikely(old_pte & _PAGE_BUSY)) { + if (unlikely(pte_val(old_pte) & H_PAGE_BUSY)) { cpu_relax(); continue; } -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - /* If hugepage and is trans splitting return None */ - if (unlikely(hugepage && - pmd_trans_splitting(pte_pmd(old_pte)))) - return __pte(0); -#endif /* If pte is not present return None */ - if (unlikely(!(old_pte & _PAGE_PRESENT))) + if (unlikely(!pte_present(old_pte))) return __pte(0); new_pte = pte_mkyoung(old_pte); if (writing && pte_write(old_pte)) new_pte = pte_mkdirty(new_pte); - if (old_pte == __cmpxchg_u64((unsigned long *)ptep, old_pte, - new_pte)) + if (pte_xchg(ptep, old_pte, new_pte)) break; } return new_pte; } - -/* Return HPTE cache control bits corresponding to Linux pte bits */ -static inline unsigned long hpte_cache_bits(unsigned long pte_val) -{ -#if _PAGE_NO_CACHE == HPTE_R_I && _PAGE_WRITETHRU == HPTE_R_W - return pte_val & (HPTE_R_W | HPTE_R_I); -#else - return ((pte_val & _PAGE_NO_CACHE) ? HPTE_R_I : 0) + - ((pte_val & _PAGE_WRITETHRU) ? HPTE_R_W : 0); -#endif -} - static inline bool hpte_read_permission(unsigned long pp, unsigned long key) { if (key) return PP_RWRX <= pp && pp <= PP_RXRX; - return 1; + return true; } static inline bool hpte_write_permission(unsigned long pp, unsigned long key) @@ -252,7 +491,7 @@ static inline bool slot_is_aligned(struct kvm_memory_slot *memslot, unsigned long mask = (pagesize >> PAGE_SHIFT) - 1; if (pagesize <= PAGE_SIZE) - return 1; + return true; return !(memslot->base_gfn & mask) && !(memslot->npages & mask); } @@ -278,7 +517,7 @@ static inline int is_vrma_hpte(unsigned long hpte_v) (HPTE_V_1TB_SEG | (VRMA_VSID << (40 - 16))); } -#ifdef CONFIG_KVM_BOOK3S_64_HV +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE /* * Note modification of an HPTE; set the HPTE modified bit * if anyone is interested. @@ -289,6 +528,173 @@ static inline void note_hpte_modification(struct kvm *kvm, if (atomic_read(&kvm->arch.hpte_mod_interest)) rev->guest_rpte |= HPTE_GR_MODIFIED; } -#endif /* CONFIG_KVM_BOOK3S_64_HV */ + +/* + * Like kvm_memslots(), but for use in real mode when we can't do + * any RCU stuff (since the secondary threads are offline from the + * kernel's point of view), and we can't print anything. + * Thus we use rcu_dereference_raw() rather than rcu_dereference_check(). + */ +static inline struct kvm_memslots *kvm_memslots_raw(struct kvm *kvm) +{ + return rcu_dereference_raw_check(kvm->memslots[0]); +} + +extern void kvmppc_mmu_debugfs_init(struct kvm *kvm); +extern void kvmhv_radix_debugfs_init(struct kvm *kvm); + +extern void kvmhv_rm_send_ipi(int cpu); + +static inline unsigned long kvmppc_hpt_npte(struct kvm_hpt_info *hpt) +{ + /* HPTEs are 2**4 bytes long */ + return 1UL << (hpt->order - 4); +} + +static inline unsigned long kvmppc_hpt_mask(struct kvm_hpt_info *hpt) +{ + /* 128 (2**7) bytes in each HPTEG */ + return (1UL << (hpt->order - 7)) - 1; +} + +/* Set bits in a dirty bitmap, which is in LE format */ +static inline void set_dirty_bits(unsigned long *map, unsigned long i, + unsigned long npages) +{ + + if (npages >= 8) + memset((char *)map + i / 8, 0xff, npages / 8); + else + for (; npages; ++i, --npages) + __set_bit_le(i, map); +} + +static inline void set_dirty_bits_atomic(unsigned long *map, unsigned long i, + unsigned long npages) +{ + if (npages >= 8) + memset((char *)map + i / 8, 0xff, npages / 8); + else + for (; npages; ++i, --npages) + set_bit_le(i, map); +} + +static inline u64 sanitize_msr(u64 msr) +{ + msr &= ~MSR_HV; + msr |= MSR_ME; + return msr; +} + +#ifdef CONFIG_PPC_TRANSACTIONAL_MEM +static inline void copy_from_checkpoint(struct kvm_vcpu *vcpu) +{ + vcpu->arch.regs.ccr = vcpu->arch.cr_tm; + vcpu->arch.regs.xer = vcpu->arch.xer_tm; + vcpu->arch.regs.link = vcpu->arch.lr_tm; + vcpu->arch.regs.ctr = vcpu->arch.ctr_tm; + vcpu->arch.amr = vcpu->arch.amr_tm; + vcpu->arch.ppr = vcpu->arch.ppr_tm; + vcpu->arch.dscr = vcpu->arch.dscr_tm; + vcpu->arch.tar = vcpu->arch.tar_tm; + memcpy(vcpu->arch.regs.gpr, vcpu->arch.gpr_tm, + sizeof(vcpu->arch.regs.gpr)); + vcpu->arch.fp = vcpu->arch.fp_tm; + vcpu->arch.vr = vcpu->arch.vr_tm; + vcpu->arch.vrsave = vcpu->arch.vrsave_tm; +} + +static inline void copy_to_checkpoint(struct kvm_vcpu *vcpu) +{ + vcpu->arch.cr_tm = vcpu->arch.regs.ccr; + vcpu->arch.xer_tm = vcpu->arch.regs.xer; + vcpu->arch.lr_tm = vcpu->arch.regs.link; + vcpu->arch.ctr_tm = vcpu->arch.regs.ctr; + vcpu->arch.amr_tm = vcpu->arch.amr; + vcpu->arch.ppr_tm = vcpu->arch.ppr; + vcpu->arch.dscr_tm = vcpu->arch.dscr; + vcpu->arch.tar_tm = vcpu->arch.tar; + memcpy(vcpu->arch.gpr_tm, vcpu->arch.regs.gpr, + sizeof(vcpu->arch.regs.gpr)); + vcpu->arch.fp_tm = vcpu->arch.fp; + vcpu->arch.vr_tm = vcpu->arch.vr; + vcpu->arch.vrsave_tm = vcpu->arch.vrsave; +} +#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ + +extern int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte, + unsigned long gpa, unsigned int level, + unsigned long mmu_seq, u64 lpid, + unsigned long *rmapp, struct rmap_nested **n_rmap); +extern void kvmhv_insert_nest_rmap(struct kvm *kvm, unsigned long *rmapp, + struct rmap_nested **n_rmap); +extern void kvmhv_update_nest_rmap_rc_list(struct kvm *kvm, unsigned long *rmapp, + unsigned long clr, unsigned long set, + unsigned long hpa, unsigned long nbytes); +extern void kvmhv_remove_nest_rmap_range(struct kvm *kvm, + const struct kvm_memory_slot *memslot, + unsigned long gpa, unsigned long hpa, + unsigned long nbytes); + +static inline pte_t * +find_kvm_secondary_pte_unlocked(struct kvm *kvm, unsigned long ea, + unsigned *hshift) +{ + pte_t *pte; + + pte = __find_linux_pte(kvm->arch.pgtable, ea, NULL, hshift); + return pte; +} + +static inline pte_t *find_kvm_secondary_pte(struct kvm *kvm, unsigned long ea, + unsigned *hshift) +{ + pte_t *pte; + + VM_WARN(!spin_is_locked(&kvm->mmu_lock), + "%s called with kvm mmu_lock not held \n", __func__); + pte = __find_linux_pte(kvm->arch.pgtable, ea, NULL, hshift); + + return pte; +} + +static inline pte_t *find_kvm_host_pte(struct kvm *kvm, unsigned long mmu_seq, + unsigned long ea, unsigned *hshift) +{ + pte_t *pte; + + VM_WARN(!spin_is_locked(&kvm->mmu_lock), + "%s called with kvm mmu_lock not held \n", __func__); + + if (mmu_invalidate_retry(kvm, mmu_seq)) + return NULL; + + pte = __find_linux_pte(kvm->mm->pgd, ea, NULL, hshift); + + return pte; +} + +extern pte_t *find_kvm_nested_guest_pte(struct kvm *kvm, unsigned long lpid, + unsigned long ea, unsigned *hshift); + +int kvmhv_nestedv2_vcpu_create(struct kvm_vcpu *vcpu, struct kvmhv_nestedv2_io *io); +void kvmhv_nestedv2_vcpu_free(struct kvm_vcpu *vcpu, struct kvmhv_nestedv2_io *io); +int kvmhv_nestedv2_flush_vcpu(struct kvm_vcpu *vcpu, u64 time_limit); +int kvmhv_nestedv2_set_ptbl_entry(unsigned long lpid, u64 dw0, u64 dw1); +int kvmhv_nestedv2_parse_output(struct kvm_vcpu *vcpu); +int kvmhv_nestedv2_set_vpa(struct kvm_vcpu *vcpu, unsigned long vpa); + +int kvmhv_counters_tracepoint_regfunc(void); +void kvmhv_counters_tracepoint_unregfunc(void); +int kvmhv_get_l2_counters_status(void); +void kvmhv_set_l2_counters_status(int cpu, bool status); +u64 kvmhv_get_l1_to_l2_cs_time(void); +u64 kvmhv_get_l2_to_l1_cs_time(void); +u64 kvmhv_get_l2_runtime_agg(void); +u64 kvmhv_get_l1_to_l2_cs_time_vcpu(void); +u64 kvmhv_get_l2_to_l1_cs_time_vcpu(void); +u64 kvmhv_get_l2_runtime_agg_vcpu(void); + +#endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ #endif /* __ASM_KVM_BOOK3S_64_H__ */ diff --git a/arch/powerpc/include/asm/kvm_book3s_asm.h b/arch/powerpc/include/asm/kvm_book3s_asm.h index 9039d3c97eec..3435fe144908 100644 --- a/arch/powerpc/include/asm/kvm_book3s_asm.h +++ b/arch/powerpc/include/asm/kvm_book3s_asm.h @@ -1,16 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * * Copyright SUSE Linux Products GmbH 2009 * @@ -25,7 +14,13 @@ #define XICS_MFRR 0xc #define XICS_IPI 2 /* interrupt source # for IPIs */ -#ifdef __ASSEMBLY__ +/* Maximum number of threads per physical core */ +#define MAX_SMT_THREADS 8 + +/* Maximum number of subcores per physical core */ +#define MAX_SUBCORES 4 + +#ifdef __ASSEMBLER__ #ifdef CONFIG_KVM_BOOK3S_HANDLER @@ -63,7 +58,20 @@ kvmppc_resume_\intno: #endif /* CONFIG_KVM_BOOK3S_HANDLER */ -#else /*__ASSEMBLY__ */ +#else /*__ASSEMBLER__ */ + +struct kvmppc_vcore; + +/* Struct used for coordinating micro-threading (split-core) mode changes */ +struct kvm_split_mode { + unsigned long rpr; + unsigned long pmmar; + unsigned long ldbar; + u8 subcore_size; + u8 do_nap; + u8 napped[MAX_SMT_THREADS]; + struct kvmppc_vcore *vc[MAX_SUBCORES]; +}; /* * This struct goes in the PACA on 64-bit processors. It is used @@ -79,43 +87,52 @@ struct kvmppc_host_state { ulong vmhandler; ulong scratch0; ulong scratch1; + ulong scratch2; u8 in_guest; u8 restore_hid5; u8 napping; -#ifdef CONFIG_KVM_BOOK3S_64_HV +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE u8 hwthread_req; u8 hwthread_state; u8 host_ipi; + u8 ptid; /* thread number within subcore when split */ + u8 fake_suspend; struct kvm_vcpu *kvm_vcpu; struct kvmppc_vcore *kvm_vcore; - unsigned long xics_phys; + void __iomem *xics_phys; + void __iomem *xive_tima_phys; + void __iomem *xive_tima_virt; u32 saved_xirr; u64 dabr; - u64 host_mmcr[3]; + u64 host_mmcr[7]; /* MMCR 0,1,A, SIAR, SDAR, MMCR2, SIER */ u32 host_pmc[8]; u64 host_purr; u64 host_spurr; u64 host_dscr; u64 dec_expires; + struct kvm_split_mode *kvm_split_mode; #endif #ifdef CONFIG_PPC_BOOK3S_64 u64 cfar; + u64 ppr; + u64 host_fscr; #endif }; struct kvmppc_book3s_shadow_vcpu { + bool in_use; ulong gpr[14]; u32 cr; - u32 xer; - - u32 fault_dsisr; - u32 last_inst; + ulong xer; ulong ctr; ulong lr; ulong pc; + ulong shadow_srr1; ulong fault_dar; + u32 fault_dsisr; + u32 last_inst; #ifdef CONFIG_PPC_BOOK3S_32 u32 sr[16]; /* Guest SRs */ @@ -129,14 +146,15 @@ struct kvmppc_book3s_shadow_vcpu { u64 esid; u64 vsid; } slb[64]; /* guest SLB */ + u64 shadow_fscr; #endif }; -#endif /*__ASSEMBLY__ */ +#endif /*__ASSEMBLER__ */ /* Values for kvm_state */ #define KVM_HWTHREAD_IN_KERNEL 0 -#define KVM_HWTHREAD_IN_NAP 1 +#define KVM_HWTHREAD_IN_IDLE 1 #define KVM_HWTHREAD_IN_KVM 2 #endif /* __ASM_KVM_BOOK3S_ASM_H__ */ diff --git a/arch/powerpc/include/asm/kvm_book3s_uvmem.h b/arch/powerpc/include/asm/kvm_book3s_uvmem.h new file mode 100644 index 000000000000..0a6319448cb6 --- /dev/null +++ b/arch/powerpc/include/asm/kvm_book3s_uvmem.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_KVM_BOOK3S_UVMEM_H__ +#define __ASM_KVM_BOOK3S_UVMEM_H__ + +#ifdef CONFIG_PPC_UV +int kvmppc_uvmem_init(void); +void kvmppc_uvmem_free(void); +bool kvmppc_uvmem_available(void); +int kvmppc_uvmem_slot_init(struct kvm *kvm, const struct kvm_memory_slot *slot); +void kvmppc_uvmem_slot_free(struct kvm *kvm, + const struct kvm_memory_slot *slot); +unsigned long kvmppc_h_svm_page_in(struct kvm *kvm, + unsigned long gra, + unsigned long flags, + unsigned long page_shift); +unsigned long kvmppc_h_svm_page_out(struct kvm *kvm, + unsigned long gra, + unsigned long flags, + unsigned long page_shift); +unsigned long kvmppc_h_svm_init_start(struct kvm *kvm); +unsigned long kvmppc_h_svm_init_done(struct kvm *kvm); +int kvmppc_send_page_to_uv(struct kvm *kvm, unsigned long gfn); +unsigned long kvmppc_h_svm_init_abort(struct kvm *kvm); +void kvmppc_uvmem_drop_pages(const struct kvm_memory_slot *free, + struct kvm *kvm, bool skip_page_out); +int kvmppc_uvmem_memslot_create(struct kvm *kvm, + const struct kvm_memory_slot *new); +void kvmppc_uvmem_memslot_delete(struct kvm *kvm, + const struct kvm_memory_slot *old); +#else +static inline int kvmppc_uvmem_init(void) +{ + return 0; +} + +static inline void kvmppc_uvmem_free(void) { } + +static inline bool kvmppc_uvmem_available(void) +{ + return false; +} + +static inline int +kvmppc_uvmem_slot_init(struct kvm *kvm, const struct kvm_memory_slot *slot) +{ + return 0; +} + +static inline void +kvmppc_uvmem_slot_free(struct kvm *kvm, const struct kvm_memory_slot *slot) { } + +static inline unsigned long +kvmppc_h_svm_page_in(struct kvm *kvm, unsigned long gra, + unsigned long flags, unsigned long page_shift) +{ + return H_UNSUPPORTED; +} + +static inline unsigned long +kvmppc_h_svm_page_out(struct kvm *kvm, unsigned long gra, + unsigned long flags, unsigned long page_shift) +{ + return H_UNSUPPORTED; +} + +static inline unsigned long kvmppc_h_svm_init_start(struct kvm *kvm) +{ + return H_UNSUPPORTED; +} + +static inline unsigned long kvmppc_h_svm_init_done(struct kvm *kvm) +{ + return H_UNSUPPORTED; +} + +static inline unsigned long kvmppc_h_svm_init_abort(struct kvm *kvm) +{ + return H_UNSUPPORTED; +} + +static inline int kvmppc_send_page_to_uv(struct kvm *kvm, unsigned long gfn) +{ + return -EFAULT; +} + +static inline void +kvmppc_uvmem_drop_pages(const struct kvm_memory_slot *free, + struct kvm *kvm, bool skip_page_out) { } + +static inline int kvmppc_uvmem_memslot_create(struct kvm *kvm, + const struct kvm_memory_slot *new) +{ + return H_UNSUPPORTED; +} + +static inline void kvmppc_uvmem_memslot_delete(struct kvm *kvm, + const struct kvm_memory_slot *old) { } + +#endif /* CONFIG_PPC_UV */ +#endif /* __ASM_KVM_BOOK3S_UVMEM_H__ */ diff --git a/arch/powerpc/include/asm/kvm_booke.h b/arch/powerpc/include/asm/kvm_booke.h index d3c1eb34c986..7c3291aa8922 100644 --- a/arch/powerpc/include/asm/kvm_booke.h +++ b/arch/powerpc/include/asm/kvm_booke.h @@ -1,16 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * * Copyright SUSE Linux Products GmbH 2010 * @@ -23,83 +12,107 @@ #include <linux/types.h> #include <linux/kvm_host.h> -/* LPIDs we support with this build -- runtime limit may be lower */ +/* + * Number of available lpids. Only the low-order 6 bits of LPID rgister are + * implemented on e500mc+ cores. + */ #define KVMPPC_NR_LPIDS 64 -#define KVMPPC_INST_EHPRIV 0x7c00021c +#define KVMPPC_INST_EHPRIV 0x7c00021c +#define EHPRIV_OC_SHIFT 11 +/* "ehpriv 1" : ehpriv with OC = 1 is used for debug emulation */ +#define EHPRIV_OC_DEBUG 1 static inline void kvmppc_set_gpr(struct kvm_vcpu *vcpu, int num, ulong val) { - vcpu->arch.gpr[num] = val; + vcpu->arch.regs.gpr[num] = val; } static inline ulong kvmppc_get_gpr(struct kvm_vcpu *vcpu, int num) { - return vcpu->arch.gpr[num]; + return vcpu->arch.regs.gpr[num]; } static inline void kvmppc_set_cr(struct kvm_vcpu *vcpu, u32 val) { - vcpu->arch.cr = val; + vcpu->arch.regs.ccr = val; } static inline u32 kvmppc_get_cr(struct kvm_vcpu *vcpu) { - return vcpu->arch.cr; + return vcpu->arch.regs.ccr; } -static inline void kvmppc_set_xer(struct kvm_vcpu *vcpu, u32 val) +static inline void kvmppc_set_xer(struct kvm_vcpu *vcpu, ulong val) { - vcpu->arch.xer = val; + vcpu->arch.regs.xer = val; } -static inline u32 kvmppc_get_xer(struct kvm_vcpu *vcpu) +static inline ulong kvmppc_get_xer(struct kvm_vcpu *vcpu) { - return vcpu->arch.xer; + return vcpu->arch.regs.xer; } -static inline u32 kvmppc_get_last_inst(struct kvm_vcpu *vcpu) +static inline bool kvmppc_need_byteswap(struct kvm_vcpu *vcpu) { - return vcpu->arch.last_inst; + /* XXX Would need to check TLB entry */ + return false; } static inline void kvmppc_set_ctr(struct kvm_vcpu *vcpu, ulong val) { - vcpu->arch.ctr = val; + vcpu->arch.regs.ctr = val; } static inline ulong kvmppc_get_ctr(struct kvm_vcpu *vcpu) { - return vcpu->arch.ctr; + return vcpu->arch.regs.ctr; } static inline void kvmppc_set_lr(struct kvm_vcpu *vcpu, ulong val) { - vcpu->arch.lr = val; + vcpu->arch.regs.link = val; } static inline ulong kvmppc_get_lr(struct kvm_vcpu *vcpu) { - return vcpu->arch.lr; + return vcpu->arch.regs.link; } static inline void kvmppc_set_pc(struct kvm_vcpu *vcpu, ulong val) { - vcpu->arch.pc = val; + vcpu->arch.regs.nip = val; } static inline ulong kvmppc_get_pc(struct kvm_vcpu *vcpu) { - return vcpu->arch.pc; + return vcpu->arch.regs.nip; +} + +static inline void kvmppc_set_fpr(struct kvm_vcpu *vcpu, int i, u64 val) +{ + vcpu->arch.fp.fpr[i][TS_FPROFFSET] = val; +} + +static inline u64 kvmppc_get_fpr(struct kvm_vcpu *vcpu, int i) +{ + return vcpu->arch.fp.fpr[i][TS_FPROFFSET]; } +#ifdef CONFIG_BOOKE static inline ulong kvmppc_get_fault_dar(struct kvm_vcpu *vcpu) { return vcpu->arch.fault_dear; } +#endif -static inline ulong kvmppc_get_msr(struct kvm_vcpu *vcpu) +static inline bool kvmppc_supports_magic_page(struct kvm_vcpu *vcpu) { - return vcpu->arch.shared->msr; + /* Magic page is only supported on e500v2 */ +#ifdef CONFIG_KVM_E500V2 + return true; +#else + return false; +#endif } #endif /* __ASM_KVM_BOOKE_H__ */ diff --git a/arch/powerpc/include/asm/kvm_booke_hv_asm.h b/arch/powerpc/include/asm/kvm_booke_hv_asm.h index 3a79f5325712..3acf2995d364 100644 --- a/arch/powerpc/include/asm/kvm_booke_hv_asm.h +++ b/arch/powerpc/include/asm/kvm_booke_hv_asm.h @@ -1,15 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright 2010-2011 Freescale Semiconductor, Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, as - * published by the Free Software Foundation. */ #ifndef ASM_KVM_BOOKE_HV_ASM_H #define ASM_KVM_BOOKE_HV_ASM_H -#ifdef __ASSEMBLY__ +#include <asm/feature-fixups.h> + +#ifdef __ASSEMBLER__ /* * All exceptions from guest state must go through KVM @@ -36,26 +35,21 @@ * *(r8 + GPR11) = saved r11 * * 64-bit host - * Expected inputs (GEN/GDBELL/DBG/MC exception types): + * Expected inputs (GEN/GDBELL/DBG/CRIT/MC exception types): * r10 = saved CR * r13 = PACA_POINTER * *(r13 + PACA_EX##type + EX_R10) = saved r10 * *(r13 + PACA_EX##type + EX_R11) = saved r11 * SPRN_SPRG_##type##_SCRATCH = saved r13 * - * Expected inputs (CRIT exception type): - * r10 = saved CR - * r13 = PACA_POINTER - * *(r13 + PACA_EX##type + EX_R10) = saved r10 - * *(r13 + PACA_EX##type + EX_R11) = saved r11 - * *(r13 + PACA_EX##type + EX_R13) = saved r13 - * * Expected inputs (TLB exception type): * r10 = saved CR + * r12 = extlb pointer * r13 = PACA_POINTER - * *(r13 + PACA_EX##type + EX_TLB_R10) = saved r10 - * *(r13 + PACA_EX##type + EX_TLB_R11) = saved r11 - * SPRN_SPRG_GEN_SCRATCH = saved r13 + * *(r12 + EX_TLB_R10) = saved r10 + * *(r12 + EX_TLB_R11) = saved r11 + * *(r12 + EX_TLB_R13) = saved r13 + * SPRN_SPRG_GEN_SCRATCH = saved r12 * * Only the bolted version of TLB miss exception handlers is supported now. */ @@ -70,5 +64,5 @@ END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV) #endif .endm -#endif /*__ASSEMBLY__ */ +#endif /*__ASSEMBLER__ */ #endif /* ASM_KVM_BOOKE_HV_ASM_H */ diff --git a/arch/powerpc/include/asm/kvm_fpu.h b/arch/powerpc/include/asm/kvm_fpu.h index 92daae132492..25df316b7ebf 100644 --- a/arch/powerpc/include/asm/kvm_fpu.h +++ b/arch/powerpc/include/asm/kvm_fpu.h @@ -1,16 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * * Copyright Novell Inc. 2010 * diff --git a/arch/powerpc/include/asm/kvm_guest.h b/arch/powerpc/include/asm/kvm_guest.h new file mode 100644 index 000000000000..68e499abdb24 --- /dev/null +++ b/arch/powerpc/include/asm/kvm_guest.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 IBM Corporation + */ + +#ifndef _ASM_POWERPC_KVM_GUEST_H_ +#define _ASM_POWERPC_KVM_GUEST_H_ + +#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_KVM_GUEST) +#include <linux/jump_label.h> + +DECLARE_STATIC_KEY_FALSE(kvm_guest); + +static inline bool is_kvm_guest(void) +{ + return static_branch_unlikely(&kvm_guest); +} + +int __init check_kvm_guest(void); +#else +static inline bool is_kvm_guest(void) { return false; } +static inline int check_kvm_guest(void) { return 0; } +#endif + +#endif /* _ASM_POWERPC_KVM_GUEST_H_ */ diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index af326cde7cb6..2d139c807577 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h @@ -1,16 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * * Copyright IBM Corp. 2007 * @@ -34,101 +23,113 @@ #include <asm/processor.h> #include <asm/page.h> #include <asm/cacheflush.h> +#include <asm/hvcall.h> +#include <asm/mce.h> +#include <asm/guest-state-buffer.h> + +#define __KVM_HAVE_ARCH_VCPU_DEBUGFS #define KVM_MAX_VCPUS NR_CPUS #define KVM_MAX_VCORES NR_CPUS -#define KVM_USER_MEM_SLOTS 32 -#define KVM_MEM_SLOTS_NUM KVM_USER_MEM_SLOTS -#ifdef CONFIG_KVM_MMIO -#define KVM_COALESCED_MMIO_PAGE_OFFSET 1 -#endif +#include <asm/cputhreads.h> -/* These values are internal and can be increased later */ -#define KVM_NR_IRQCHIPS 1 -#define KVM_IRQCHIP_NUM_PINS 256 +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE +#include <asm/kvm_book3s_asm.h> /* for MAX_SMT_THREADS */ +#define KVM_MAX_VCPU_IDS (MAX_SMT_THREADS * KVM_MAX_VCORES) -#if !defined(CONFIG_KVM_440) -#include <linux/mmu_notifier.h> +/* + * Limit the nested partition table to 4096 entries (because that's what + * hardware supports). Both guest and host use this value. + */ +#define KVM_MAX_NESTED_GUESTS_SHIFT 12 -#define KVM_ARCH_WANT_MMU_NOTIFIER +#else +#define KVM_MAX_VCPU_IDS KVM_MAX_VCPUS +#endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ -struct kvm; -extern int kvm_unmap_hva(struct kvm *kvm, unsigned long hva); -extern int kvm_unmap_hva_range(struct kvm *kvm, - unsigned long start, unsigned long end); -extern int kvm_age_hva(struct kvm *kvm, unsigned long hva); -extern int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); -extern void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); +#define __KVM_HAVE_ARCH_INTC_INITIALIZED -#endif +#define KVM_HALT_POLL_NS_DEFAULT 10000 /* 10 us */ -/* We don't currently support large pages. */ -#define KVM_HPAGE_GFN_SHIFT(x) 0 -#define KVM_NR_PAGE_SIZES 1 -#define KVM_PAGES_PER_HPAGE(x) (1UL<<31) +/* These values are internal and can be increased later */ +#define KVM_NR_IRQCHIPS 1 +#define KVM_IRQCHIP_NUM_PINS 256 + +/* PPC-specific vcpu->requests bit members */ +#define KVM_REQ_WATCHDOG KVM_ARCH_REQ(0) +#define KVM_REQ_EPR_EXIT KVM_ARCH_REQ(1) +#define KVM_REQ_PENDING_TIMER KVM_ARCH_REQ(2) + +#include <linux/mmu_notifier.h> #define HPTEG_CACHE_NUM (1 << 15) #define HPTEG_HASH_BITS_PTE 13 #define HPTEG_HASH_BITS_PTE_LONG 12 #define HPTEG_HASH_BITS_VPTE 13 #define HPTEG_HASH_BITS_VPTE_LONG 5 +#define HPTEG_HASH_BITS_VPTE_64K 11 #define HPTEG_HASH_NUM_PTE (1 << HPTEG_HASH_BITS_PTE) #define HPTEG_HASH_NUM_PTE_LONG (1 << HPTEG_HASH_BITS_PTE_LONG) #define HPTEG_HASH_NUM_VPTE (1 << HPTEG_HASH_BITS_VPTE) #define HPTEG_HASH_NUM_VPTE_LONG (1 << HPTEG_HASH_BITS_VPTE_LONG) +#define HPTEG_HASH_NUM_VPTE_64K (1 << HPTEG_HASH_BITS_VPTE_64K) /* Physical Address Mask - allowed range of real mode RAM access */ #define KVM_PAM 0x0fffffffffffffffULL -struct kvm; -struct kvm_run; -struct kvm_vcpu; - struct lppaca; struct slb_shadow; struct dtl_entry; +struct kvmppc_vcpu_book3s; +struct kvmppc_book3s_shadow_vcpu; +struct kvm_nested_guest; + struct kvm_vm_stat { - u32 remote_tlb_flush; + struct kvm_vm_stat_generic generic; + u64 num_2M_pages; + u64 num_1G_pages; }; struct kvm_vcpu_stat { - u32 sum_exits; - u32 mmio_exits; - u32 dcr_exits; - u32 signal_exits; - u32 light_exits; + struct kvm_vcpu_stat_generic generic; + u64 sum_exits; + u64 mmio_exits; + u64 signal_exits; + u64 light_exits; /* Account for special types of light exits: */ - u32 itlb_real_miss_exits; - u32 itlb_virt_miss_exits; - u32 dtlb_real_miss_exits; - u32 dtlb_virt_miss_exits; - u32 syscall_exits; - u32 isi_exits; - u32 dsi_exits; - u32 emulated_inst_exits; - u32 dec_exits; - u32 ext_intr_exits; - u32 halt_wakeup; - u32 dbell_exits; - u32 gdbell_exits; + u64 itlb_real_miss_exits; + u64 itlb_virt_miss_exits; + u64 dtlb_real_miss_exits; + u64 dtlb_virt_miss_exits; + u64 syscall_exits; + u64 isi_exits; + u64 dsi_exits; + u64 emulated_inst_exits; + u64 dec_exits; + u64 ext_intr_exits; + u64 halt_successful_wait; + u64 dbell_exits; + u64 gdbell_exits; + u64 ld; + u64 st; #ifdef CONFIG_PPC_BOOK3S - u32 pf_storage; - u32 pf_instruc; - u32 sp_storage; - u32 sp_instruc; - u32 queue_intr; - u32 ld; - u32 ld_slow; - u32 st; - u32 st_slow; + u64 pf_storage; + u64 pf_instruc; + u64 sp_storage; + u64 sp_instruc; + u64 queue_intr; + u64 ld_slow; + u64 st_slow; #endif + u64 pthru_all; + u64 pthru_host; + u64 pthru_bad_aff; }; enum kvm_exit_types { MMIO_EXITS, - DCR_EXITS, SIGNAL_EXITS, ITLB_REAL_MISS_EXITS, ITLB_VIRT_MISS_EXITS, @@ -148,6 +149,7 @@ enum kvm_exit_types { EMULATED_TLBWE_EXITS, EMULATED_RFI_EXITS, EMULATED_RFCI_EXITS, + EMULATED_RFDI_EXITS, DEC_EXITS, EXT_INTR_EXITS, HALT_WAKEUP, @@ -175,26 +177,40 @@ struct kvmppc_pginfo { atomic_t refcnt; }; +struct kvmppc_spapr_tce_iommu_table { + struct rcu_head rcu; + struct list_head next; + struct iommu_table *tbl; + struct kref kref; +}; + +#define TCES_PER_PAGE (PAGE_SIZE / sizeof(u64)) + struct kvmppc_spapr_tce_table { struct list_head list; struct kvm *kvm; u64 liobn; - u32 window_size; - struct page *pages[0]; -}; - -struct kvmppc_linear_info { - void *base_virt; - unsigned long base_pfn; - unsigned long npages; - struct list_head list; - atomic_t use_count; - int type; + struct rcu_head rcu; + u32 page_shift; + u64 offset; /* in pages */ + u64 size; /* window size in pages */ + struct list_head iommu_tables; + struct mutex alloc_lock; + struct page *pages[]; }; /* XICS components, defined in book3s_xics.c */ struct kvmppc_xics; struct kvmppc_icp; +extern struct kvm_device_ops kvm_xics_ops; + +/* XIVE components, defined in book3s_xive.c */ +struct kvmppc_xive; +struct kvmppc_xive_vcpu; +extern struct kvm_device_ops kvm_xive_ops; +extern struct kvm_device_ops kvm_xive_native_ops; + +struct kvmppc_passthru_irqmap; /* * The reverse mapping array has one entry for each HPTE, @@ -210,105 +226,137 @@ struct revmap_entry { }; /* - * We use the top bit of each memslot->arch.rmap entry as a lock bit, - * and bit 32 as a present flag. The bottom 32 bits are the - * index in the guest HPT of a HPTE that points to the page. + * The rmap array of size number of guest pages is allocated for each memslot. + * This array is used to store usage specific information about the guest page. + * Below are the encodings of the various possible usage types. */ -#define KVMPPC_RMAP_LOCK_BIT 63 +/* Free bits which can be used to define a new usage */ +#define KVMPPC_RMAP_TYPE_MASK 0xff00000000000000 +#define KVMPPC_RMAP_NESTED 0xc000000000000000 /* Nested rmap array */ +#define KVMPPC_RMAP_HPT 0x0100000000000000 /* HPT guest */ + +/* + * rmap usage definition for a hash page table (hpt) guest: + * 0x0000080000000000 Lock bit + * 0x0000018000000000 RC bits + * 0x0000000100000000 Present bit + * 0x00000000ffffffff HPT index bits + * The bottom 32 bits are the index in the guest HPT of a HPTE that points to + * the page. + */ +#define KVMPPC_RMAP_LOCK_BIT 43 #define KVMPPC_RMAP_RC_SHIFT 32 #define KVMPPC_RMAP_REFERENCED (HPTE_R_R << KVMPPC_RMAP_RC_SHIFT) -#define KVMPPC_RMAP_CHANGED (HPTE_R_C << KVMPPC_RMAP_RC_SHIFT) #define KVMPPC_RMAP_PRESENT 0x100000000ul #define KVMPPC_RMAP_INDEX 0xfffffffful -/* Low-order bits in memslot->arch.slot_phys[] */ -#define KVMPPC_PAGE_ORDER_MASK 0x1f -#define KVMPPC_PAGE_NO_CACHE HPTE_R_I /* 0x20 */ -#define KVMPPC_PAGE_WRITETHRU HPTE_R_W /* 0x40 */ -#define KVMPPC_GOT_PAGE 0x80 - struct kvm_arch_memory_slot { -#ifdef CONFIG_KVM_BOOK3S_64_HV +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE unsigned long *rmap; - unsigned long *slot_phys; -#endif /* CONFIG_KVM_BOOK3S_64_HV */ +#endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ +}; + +struct kvm_hpt_info { + /* Host virtual (linear mapping) address of guest HPT */ + unsigned long virt; + /* Array of reverse mapping entries for each guest HPTE */ + struct revmap_entry *rev; + /* Guest HPT size is 2**(order) bytes */ + u32 order; + /* 1 if HPT allocated with CMA, 0 otherwise */ + int cma; }; +struct kvm_resize_hpt; + +/* Flag values for kvm_arch.secure_guest */ +#define KVMPPC_SECURE_INIT_START 0x1 /* H_SVM_INIT_START has been called */ +#define KVMPPC_SECURE_INIT_DONE 0x2 /* H_SVM_INIT_DONE completed */ +#define KVMPPC_SECURE_INIT_ABORT 0x4 /* H_SVM_INIT_ABORT issued */ + struct kvm_arch { - unsigned int lpid; -#ifdef CONFIG_KVM_BOOK3S_64_HV - unsigned long hpt_virt; - struct revmap_entry *revmap; + u64 lpid; + unsigned int smt_mode; /* # vcpus per virtual core */ + unsigned int emul_smt_mode; /* emualted SMT mode, on P9 */ +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE + unsigned int tlb_sets; + struct kvm_hpt_info hpt; + atomic64_t mmio_update; unsigned int host_lpid; unsigned long host_lpcr; unsigned long sdr1; unsigned long host_sdr1; - int tlbie_lock; unsigned long lpcr; - unsigned long rmor; - struct kvmppc_linear_info *rma; unsigned long vrma_slb_v; - int rma_setup_done; - int using_mmu_notifiers; - u32 hpt_order; + int mmu_ready; atomic_t vcpus_running; u32 online_vcores; - unsigned long hpt_npte; - unsigned long hpt_mask; atomic_t hpte_mod_interest; - spinlock_t slot_phys_lock; cpumask_t need_tlb_flush; - struct kvmppc_vcore *vcores[KVM_MAX_VCORES]; - struct kvmppc_linear_info *hpt_li; -#endif /* CONFIG_KVM_BOOK3S_64_HV */ + u8 radix; + u8 fwnmi_enabled; + u8 secure_guest; + u8 svm_enabled; + bool nested_enable; + bool dawr1_enabled; + pgd_t *pgtable; + u64 process_table; + struct kvm_resize_hpt *resize_hpt; /* protected by kvm->lock */ +#endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ +#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE + struct mutex hpt_mutex; +#endif #ifdef CONFIG_PPC_BOOK3S_64 struct list_head spapr_tce_tables; struct list_head rtas_tokens; + struct mutex rtas_token_lock; + DECLARE_BITMAP(enabled_hcalls, MAX_HCALL_OPCODE/4 + 1); #endif #ifdef CONFIG_KVM_MPIC struct openpic *mpic; #endif #ifdef CONFIG_KVM_XICS struct kvmppc_xics *xics; + struct kvmppc_xics *xics_device; + struct kvmppc_xive *xive; /* Current XIVE device in use */ + struct { + struct kvmppc_xive *native; + struct kvmppc_xive *xics_on_xive; + } xive_devices; + struct kvmppc_passthru_irqmap *pimap; +#endif + struct kvmppc_ops *kvm_ops; +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE + struct mutex uvmem_lock; + struct list_head uvmem_pfns; + struct mutex mmu_setup_lock; /* nests inside vcpu mutexes */ + u64 l1_ptcr; + struct idr kvm_nested_guest_idr; + /* This array can grow quite large, keep it at the end */ + struct kvmppc_vcore *vcores[KVM_MAX_VCORES]; #endif }; -/* - * Struct for a virtual core. - * Note: entry_exit_count combines an entry count in the bottom 8 bits - * and an exit count in the next 8 bits. This is so that we can - * atomically increment the entry count iff the exit count is 0 - * without taking the lock. - */ -struct kvmppc_vcore { - int n_runnable; - int n_busy; - int num_threads; - int entry_exit_count; - int n_woken; - int nap_count; - int napping_threads; - u16 pcpu; - u16 last_cpu; - u8 vcore_state; - u8 in_guest; - struct list_head runnable_threads; - spinlock_t lock; - wait_queue_head_t wq; - u64 stolen_tb; - u64 preempt_tb; - struct kvm_vcpu *runner; -}; +#define VCORE_ENTRY_MAP(vc) ((vc)->entry_exit_map & 0xff) +#define VCORE_EXIT_MAP(vc) ((vc)->entry_exit_map >> 8) +#define VCORE_IS_EXITING(vc) (VCORE_EXIT_MAP(vc) != 0) -#define VCORE_ENTRY_COUNT(vc) ((vc)->entry_exit_count & 0xff) -#define VCORE_EXIT_COUNT(vc) ((vc)->entry_exit_count >> 8) +/* This bit is used when a vcore exit is triggered from outside the vcore */ +#define VCORE_EXIT_REQ 0x10000 -/* Values for vcore_state */ +/* + * Values for vcore_state. + * Note that these are arranged such that lower values + * (< VCORE_SLEEPING) don't require stolen time accounting + * on load/unload, and higher values do. + */ #define VCORE_INACTIVE 0 -#define VCORE_SLEEPING 1 -#define VCORE_STARTING 2 -#define VCORE_RUNNING 3 -#define VCORE_EXITING 4 +#define VCORE_PREEMPT 1 +#define VCORE_PIGGYBACK 2 +#define VCORE_SLEEPING 3 +#define VCORE_RUNNING 4 +#define VCORE_EXITING 5 +#define VCORE_POLLING 6 /* * Struct used to manage memory for a virtual processor area @@ -332,6 +380,10 @@ struct kvmppc_pte { bool may_read : 1; bool may_write : 1; bool may_execute : 1; + unsigned long wimg; + unsigned long rc; + u8 page_size; /* MMU_PAGE_xxx */ + u8 page_shift; }; struct kvmppc_mmu { @@ -339,13 +391,14 @@ struct kvmppc_mmu { void (*slbmte)(struct kvm_vcpu *vcpu, u64 rb, u64 rs); u64 (*slbmfee)(struct kvm_vcpu *vcpu, u64 slb_nr); u64 (*slbmfev)(struct kvm_vcpu *vcpu, u64 slb_nr); + int (*slbfee)(struct kvm_vcpu *vcpu, gva_t eaddr, ulong *ret_slb); void (*slbie)(struct kvm_vcpu *vcpu, u64 slb_nr); void (*slbia)(struct kvm_vcpu *vcpu); /* book3s */ void (*mtsrin)(struct kvm_vcpu *vcpu, u32 srnum, ulong value); u32 (*mfsrin)(struct kvm_vcpu *vcpu, u32 srnum); - int (*xlate)(struct kvm_vcpu *vcpu, gva_t eaddr, struct kvmppc_pte *pte, bool data); - void (*reset_msr)(struct kvm_vcpu *vcpu); + int (*xlate)(struct kvm_vcpu *vcpu, gva_t eaddr, + struct kvmppc_pte *pte, bool data, bool iswrite); void (*tlbie)(struct kvm_vcpu *vcpu, ulong addr, bool large); int (*esid_to_vsid)(struct kvm_vcpu *vcpu, ulong esid, u64 *vsid); u64 (*ea_to_vp)(struct kvm_vcpu *vcpu, gva_t eaddr, bool data); @@ -364,9 +417,32 @@ struct kvmppc_slb { bool large : 1; /* PTEs are 16MB */ bool tb : 1; /* 1TB segment */ bool class : 1; + u8 base_page_size; /* MMU_PAGE_xxx */ }; -# ifdef CONFIG_PPC_FSL_BOOK3E +/* Struct used to accumulate timing information in HV real mode code */ +struct kvmhv_tb_accumulator { + u64 seqcount; /* used to synchronize access, also count * 2 */ + u64 tb_total; /* total time in timebase ticks */ + u64 tb_min; /* min time */ + u64 tb_max; /* max time */ +}; + +#ifdef CONFIG_PPC_BOOK3S_64 +struct kvmppc_irq_map { + u32 r_hwirq; + u32 v_hwirq; + struct irq_desc *desc; +}; + +#define KVMPPC_PIRQ_MAPPED 1024 +struct kvmppc_passthru_irqmap { + int n_mapped; + struct kvmppc_irq_map mapped[KVMPPC_PIRQ_MAPPED]; +}; +#endif + +# ifdef CONFIG_PPC_E500 #define KVMPPC_BOOKE_IAC_NUM 2 #define KVMPPC_BOOKE_DAC_NUM 2 # else @@ -381,23 +457,74 @@ struct kvmppc_slb { #define KVMPPC_EPR_USER 1 /* exit to userspace to fill EPR */ #define KVMPPC_EPR_KERNEL 2 /* in-kernel irqchip */ -struct kvmppc_booke_debug_reg { - u32 dbcr0; - u32 dbcr1; - u32 dbcr2; -#ifdef CONFIG_KVM_E500MC - u32 dbcr4; -#endif - u64 iac[KVMPPC_BOOKE_MAX_IAC]; - u64 dac[KVMPPC_BOOKE_MAX_DAC]; -}; - #define KVMPPC_IRQ_DEFAULT 0 #define KVMPPC_IRQ_MPIC 1 -#define KVMPPC_IRQ_XICS 2 +#define KVMPPC_IRQ_XICS 2 /* Includes a XIVE option */ +#define KVMPPC_IRQ_XIVE 3 /* XIVE native exploitation mode */ + +#define MMIO_HPTE_CACHE_SIZE 4 + +struct mmio_hpte_cache_entry { + unsigned long hpte_v; + unsigned long hpte_r; + unsigned long rpte; + unsigned long pte_index; + unsigned long eaddr; + unsigned long slb_v; + long mmio_update; + unsigned int slb_base_pshift; +}; + +struct mmio_hpte_cache { + struct mmio_hpte_cache_entry entry[MMIO_HPTE_CACHE_SIZE]; + unsigned int index; +}; + +#define KVMPPC_VSX_COPY_NONE 0 +#define KVMPPC_VSX_COPY_WORD 1 +#define KVMPPC_VSX_COPY_DWORD 2 +#define KVMPPC_VSX_COPY_DWORD_LOAD_DUMP 3 +#define KVMPPC_VSX_COPY_WORD_LOAD_DUMP 4 + +#define KVMPPC_VMX_COPY_BYTE 8 +#define KVMPPC_VMX_COPY_HWORD 9 +#define KVMPPC_VMX_COPY_WORD 10 +#define KVMPPC_VMX_COPY_DWORD 11 struct openpic; +/* W0 and W1 of a XIVE thread management context */ +union xive_tma_w01 { + struct { + u8 nsr; + u8 cppr; + u8 ipb; + u8 lsmfb; + u8 ack; + u8 inc; + u8 age; + u8 pipr; + }; + __be64 w01; +}; + + /* Nestedv2 H_GUEST_RUN_VCPU configuration */ +struct kvmhv_nestedv2_config { + struct kvmppc_gs_buff_info vcpu_run_output_cfg; + struct kvmppc_gs_buff_info vcpu_run_input_cfg; + u64 vcpu_run_output_size; +}; + + /* Nestedv2 L1<->L0 communication state */ +struct kvmhv_nestedv2_io { + struct kvmhv_nestedv2_config cfg; + struct kvmppc_gs_buff *vcpu_run_output; + struct kvmppc_gs_buff *vcpu_run_input; + struct kvmppc_gs_msg *vcpu_message; + struct kvmppc_gs_msg *vcore_message; + struct kvmppc_gs_bitmap valids; +}; + struct kvm_vcpu_arch { ulong host_stack; u32 host_pid; @@ -406,12 +533,19 @@ struct kvm_vcpu_arch { int slb_max; /* 1 + index of last valid entry in slb[] */ int slb_nr; /* total number of entries in SLB */ struct kvmppc_mmu mmu; + struct kvmppc_vcpu_book3s *book3s; +#endif +#ifdef CONFIG_PPC_BOOK3S_32 + struct kvmppc_book3s_shadow_vcpu *shadow_vcpu; #endif - ulong gpr[32]; + /* + * This is passed along to the HV via H_ENTER_NESTED. Align to + * prevent it crossing a real 4K page. + */ + struct pt_regs regs __aligned(512); - u64 fpr[32]; - u64 fpscr; + struct thread_fp_state fp; #ifdef CONFIG_SPE ulong evr[32]; @@ -420,12 +554,7 @@ struct kvm_vcpu_arch { u64 acc; #endif #ifdef CONFIG_ALTIVEC - vector128 vr[32]; - vector128 vscr; -#endif - -#ifdef CONFIG_VSX - u64 vsr[64]; + struct thread_vr_state vr; #endif #ifdef CONFIG_KVM_BOOKE_HV @@ -449,24 +578,52 @@ struct kvm_vcpu_arch { u32 qpr[32]; #endif - ulong pc; - ulong ctr; - ulong lr; - - ulong xer; - u32 cr; +#ifdef CONFIG_PPC_BOOK3S + ulong tar; +#endif #ifdef CONFIG_PPC_BOOK3S ulong hflags; ulong guest_owned_ext; ulong purr; ulong spurr; + ulong ic; ulong dscr; ulong amr; ulong uamor; + ulong iamr; u32 ctrl; + u32 dabrx; ulong dabr; + ulong dawr0; + ulong dawrx0; + ulong dawr1; + ulong dawrx1; + ulong dexcr; + ulong hashkeyr; + ulong hashpkeyr; + ulong ciabr; ulong cfar; + ulong ppr; + u32 pspb; + u8 load_ebb; +#ifdef CONFIG_PPC_TRANSACTIONAL_MEM + u8 load_tm; +#endif + ulong fscr; + ulong shadow_fscr; + ulong ebbhr; + ulong ebbrr; + ulong bescr; + ulong csigr; + ulong tacr; + ulong tcscr; + ulong acop; + ulong wort; + ulong tid; + ulong psscr; + ulong hfscr; + ulong shadow_srr1; #endif u32 vrsave; /* also USPRG0 */ u32 mmucr; @@ -479,12 +636,14 @@ struct kvm_vcpu_arch { ulong mcsrr0; ulong mcsrr1; ulong mcsr; - u32 dec; + ulong dec; #ifdef CONFIG_BOOKE u32 decar; #endif - u32 tbl; - u32 tbu; + /* Time base value when we entered the guest */ + u64 entry_tb; + u64 entry_vtb; + u64 entry_ic; u32 tcr; ulong tsr; /* we need to perform set/clr_bits() which requires ulong */ u32 ivor[64]; @@ -500,8 +659,36 @@ struct kvm_vcpu_arch { u32 ccr1; u32 dbsr; - u64 mmcr[3]; + u64 mmcr[4]; /* MMCR0, MMCR1, MMCR2, MMCR3 */ + u64 mmcra; + u64 mmcrs; u32 pmc[8]; + u32 spmc[2]; + u64 siar; + u64 sdar; + u64 sier[3]; +#ifdef CONFIG_PPC_TRANSACTIONAL_MEM + u64 tfhar; + u64 texasr; + u64 tfiar; + u64 orig_texasr; + + u32 cr_tm; + u64 xer_tm; + u64 lr_tm; + u64 ctr_tm; + u64 amr_tm; + u64 ppr_tm; + u64 dscr_tm; + u64 tar_tm; + + ulong gpr_tm[32]; + + struct thread_fp_state fp_tm; + + struct thread_vr_state vr_tm; + u32 vrsave_tm; /* also USPRG0 */ +#endif #ifdef CONFIG_KVM_EXIT_TIMING struct mutex exit_timing_lock; @@ -514,12 +701,18 @@ struct kvm_vcpu_arch { u64 timing_min_duration[__NUMBER_OF_KVM_EXIT_TYPES]; u64 timing_max_duration[__NUMBER_OF_KVM_EXIT_TYPES]; u64 timing_last_exit; - struct dentry *debugfs_exit_timing; #endif #ifdef CONFIG_PPC_BOOK3S ulong fault_dar; u32 fault_dsisr; + unsigned long intr_msr; + /* + * POWER9 and later: fault_gpa contains the guest real address of page + * fault for a radix guest, or segment descriptor (equivalent to result + * from slbmfev of SLB entry that translated the EA) for hash guests. + */ + ulong fault_gpa; #endif #ifdef CONFIG_BOOKE @@ -534,17 +727,35 @@ struct kvm_vcpu_arch { u32 mmucfg; u32 eptcfg; u32 epr; + u64 sprg9; + u32 pwrmgtcr0; u32 crit_save; - struct kvmppc_booke_debug_reg dbg_reg; + /* guest debug registers*/ + struct debug_reg dbg_reg; #endif gpa_t paddr_accessed; gva_t vaddr_accessed; + pgd_t *pgdir; - u8 io_gpr; /* GPR used as IO source/target */ - u8 mmio_is_bigendian; + u16 io_gpr; /* GPR used as IO source/target */ + u8 mmio_host_swabbed; u8 mmio_sign_extend; - u8 dcr_needed; - u8 dcr_is_write; + /* conversion between single and double precision */ + u8 mmio_sp64_extend; + /* + * Number of simulations for vsx. + * If we use 2*8bytes to simulate 1*16bytes, + * then the number should be 2 and + * mmio_copy_type=KVMPPC_VSX_COPY_DWORD. + * If we use 4*4bytes to simulate 1*16bytes, + * the number should be 4 and + * mmio_vsx_copy_type=KVMPPC_VSX_COPY_WORD. + */ + u8 mmio_vsx_copy_nums; + u8 mmio_vsx_offset; + u8 mmio_vmx_copy_nums; + u8 mmio_vmx_offset; + u8 mmio_copy_type; u8 osi_needed; u8 osi_enabled; u8 papr_enabled; @@ -554,49 +765,65 @@ struct kvm_vcpu_arch { u8 hcall_needed; u8 epr_flags; /* KVMPPC_EPR_xxx */ u8 epr_needed; + u8 external_oneshot; /* clear external irq after delivery */ u32 cpr0_cfgaddr; /* holds the last set cpr0_cfgaddr */ struct hrtimer dec_timer; - struct tasklet_struct tasklet; u64 dec_jiffies; - u64 dec_expires; + u64 dec_expires; /* Relative to guest timebase. */ unsigned long pending_exceptions; u8 ceded; u8 prodded; - u32 last_inst; + u8 doorbell_request; + u8 irq_pending; /* Used by XIVE to signal pending guest irqs */ + unsigned long last_inst; - wait_queue_head_t *wqp; + struct rcuwait wait; + struct rcuwait *waitp; struct kvmppc_vcore *vcore; int ret; int trap; int state; int ptid; + int thread_cpu; + int prev_cpu; bool timer_running; wait_queue_head_t cpu_run; + struct machine_check_event mce_evt; /* Valid if trap == 0x200 */ struct kvm_vcpu_arch_shared *shared; +#if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_KVM_BOOK3S_PR_POSSIBLE) + bool shared_big_endian; +#endif unsigned long magic_page_pa; /* phys addr to map the magic page to */ unsigned long magic_page_ea; /* effect. addr to map the magic page to */ + bool disable_kernel_nx; int irq_type; /* one of KVM_IRQ_* */ int irq_cpu_id; struct openpic *mpic; /* KVM_IRQ_MPIC */ #ifdef CONFIG_KVM_XICS struct kvmppc_icp *icp; /* XICS presentation controller */ + struct kvmppc_xive_vcpu *xive_vcpu; /* XIVE virtual CPU data */ + __be32 xive_cam_word; /* Cooked W2 in proper endian with valid bit */ + u8 xive_pushed; /* Is the VP pushed on the physical CPU ? */ + u8 xive_esc_on; /* Is the escalation irq enabled ? */ + union xive_tma_w01 xive_saved_state; /* W0..1 of XIVE thread state */ + u64 xive_esc_raddr; /* Escalation interrupt ESB real addr */ + u64 xive_esc_vaddr; /* Escalation interrupt ESB virt addr */ #endif -#ifdef CONFIG_KVM_BOOK3S_64_HV +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE struct kvm_vcpu_arch_shared shregs; + struct mmio_hpte_cache mmio_cache; unsigned long pgfault_addr; long pgfault_index; unsigned long pgfault_hpte[2]; + struct mmio_hpte_cache_entry *pgfault_cache; - struct list_head run_list; struct task_struct *run_task; - struct kvm_run *kvm_run; - pgd_t *pgdir; spinlock_t vpa_update_lock; struct kvmppc_vpa vpa; @@ -609,23 +836,75 @@ struct kvm_vcpu_arch { spinlock_t tbacct_lock; u64 busy_stolen; u64 busy_preempt; + + u64 emul_inst; + + u32 online; + + u64 hfscr_permitted; /* A mask of permitted HFSCR facilities */ + + /* For support of nested guests */ + struct kvm_nested_guest *nested; + u64 nested_hfscr; /* HFSCR that the L1 requested for the nested guest */ + u32 nested_vcpu_id; + gpa_t nested_io_gpr; + /* For nested APIv2 guests*/ + struct kvmhv_nestedv2_io nestedv2_io; +#endif + +#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING + struct kvmhv_tb_accumulator *cur_activity; /* What we're timing */ + u64 cur_tb_start; /* when it started */ +#ifdef CONFIG_KVM_BOOK3S_HV_P9_TIMING + struct kvmhv_tb_accumulator vcpu_entry; + struct kvmhv_tb_accumulator vcpu_exit; + struct kvmhv_tb_accumulator in_guest; + struct kvmhv_tb_accumulator hcall; + struct kvmhv_tb_accumulator pg_fault; + struct kvmhv_tb_accumulator guest_entry; + struct kvmhv_tb_accumulator guest_exit; +#else + struct kvmhv_tb_accumulator rm_entry; /* real-mode entry code */ + struct kvmhv_tb_accumulator rm_intr; /* real-mode intr handling */ + struct kvmhv_tb_accumulator rm_exit; /* real-mode exit code */ + struct kvmhv_tb_accumulator guest_time; /* guest execution */ + struct kvmhv_tb_accumulator cede_time; /* time napping inside guest */ +#endif +#endif /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */ +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE + u64 l1_to_l2_cs; + u64 l2_to_l1_cs; + u64 l2_runtime_agg; #endif }; +#define VCPU_FPR(vcpu, i) (vcpu)->arch.fp.fpr[i][TS_FPROFFSET] +#define VCPU_VSX_FPR(vcpu, i, j) ((vcpu)->arch.fp.fpr[i][j]) +#define VCPU_VSX_VR(vcpu, i) ((vcpu)->arch.vr.vr[i]) + /* Values for vcpu->arch.state */ #define KVMPPC_VCPU_NOTREADY 0 #define KVMPPC_VCPU_RUNNABLE 1 #define KVMPPC_VCPU_BUSY_IN_HOST 2 /* Values for vcpu->arch.io_gpr */ -#define KVM_MMIO_REG_MASK 0x001f -#define KVM_MMIO_REG_EXT_MASK 0xffe0 +#define KVM_MMIO_REG_MASK 0x003f +#define KVM_MMIO_REG_EXT_MASK 0xffc0 #define KVM_MMIO_REG_GPR 0x0000 -#define KVM_MMIO_REG_FPR 0x0020 -#define KVM_MMIO_REG_QPR 0x0040 -#define KVM_MMIO_REG_FQPR 0x0060 +#define KVM_MMIO_REG_FPR 0x0040 +#define KVM_MMIO_REG_QPR 0x0080 +#define KVM_MMIO_REG_FQPR 0x00c0 +#define KVM_MMIO_REG_VSX 0x0100 +#define KVM_MMIO_REG_VMX 0x0180 +#define KVM_MMIO_REG_NESTED_GPR 0xffc0 + #define __KVM_HAVE_ARCH_WQP #define __KVM_HAVE_CREATE_DEVICE +static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} +static inline void kvm_arch_flush_shadow_all(struct kvm *kvm) {} +static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} +static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} + #endif /* __POWERPC_KVM_HOST_H__ */ diff --git a/arch/powerpc/include/asm/kvm_para.h b/arch/powerpc/include/asm/kvm_para.h index 2b119654b4c1..abe1b5e82547 100644 --- a/arch/powerpc/include/asm/kvm_para.h +++ b/arch/powerpc/include/asm/kvm_para.h @@ -1,16 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * * Copyright IBM Corp. 2008 * @@ -19,113 +8,15 @@ #ifndef __POWERPC_KVM_PARA_H__ #define __POWERPC_KVM_PARA_H__ -#include <uapi/asm/kvm_para.h> - -#ifdef CONFIG_KVM_GUEST - -#include <linux/of.h> - -static inline int kvm_para_available(void) -{ - struct device_node *hyper_node; - - hyper_node = of_find_node_by_path("/hypervisor"); - if (!hyper_node) - return 0; - - if (!of_device_is_compatible(hyper_node, "linux,kvm")) - return 0; - - return 1; -} +#include <asm/kvm_guest.h> -extern unsigned long kvm_hypercall(unsigned long *in, - unsigned long *out, - unsigned long nr); - -#else +#include <uapi/asm/kvm_para.h> static inline int kvm_para_available(void) { - return 0; + return IS_ENABLED(CONFIG_KVM_GUEST) && is_kvm_guest(); } -static unsigned long kvm_hypercall(unsigned long *in, - unsigned long *out, - unsigned long nr) -{ - return EV_UNIMPLEMENTED; -} - -#endif - -static inline long kvm_hypercall0_1(unsigned int nr, unsigned long *r2) -{ - unsigned long in[8]; - unsigned long out[8]; - unsigned long r; - - r = kvm_hypercall(in, out, KVM_HCALL_TOKEN(nr)); - *r2 = out[0]; - - return r; -} - -static inline long kvm_hypercall0(unsigned int nr) -{ - unsigned long in[8]; - unsigned long out[8]; - - return kvm_hypercall(in, out, KVM_HCALL_TOKEN(nr)); -} - -static inline long kvm_hypercall1(unsigned int nr, unsigned long p1) -{ - unsigned long in[8]; - unsigned long out[8]; - - in[0] = p1; - return kvm_hypercall(in, out, KVM_HCALL_TOKEN(nr)); -} - -static inline long kvm_hypercall2(unsigned int nr, unsigned long p1, - unsigned long p2) -{ - unsigned long in[8]; - unsigned long out[8]; - - in[0] = p1; - in[1] = p2; - return kvm_hypercall(in, out, KVM_HCALL_TOKEN(nr)); -} - -static inline long kvm_hypercall3(unsigned int nr, unsigned long p1, - unsigned long p2, unsigned long p3) -{ - unsigned long in[8]; - unsigned long out[8]; - - in[0] = p1; - in[1] = p2; - in[2] = p3; - return kvm_hypercall(in, out, KVM_HCALL_TOKEN(nr)); -} - -static inline long kvm_hypercall4(unsigned int nr, unsigned long p1, - unsigned long p2, unsigned long p3, - unsigned long p4) -{ - unsigned long in[8]; - unsigned long out[8]; - - in[0] = p1; - in[1] = p2; - in[2] = p3; - in[3] = p4; - return kvm_hypercall(in, out, KVM_HCALL_TOKEN(nr)); -} - - static inline unsigned int kvm_arch_para_features(void) { unsigned long r; @@ -133,12 +24,17 @@ static inline unsigned int kvm_arch_para_features(void) if (!kvm_para_available()) return 0; - if(kvm_hypercall0_1(KVM_HC_FEATURES, &r)) + if(epapr_hypercall0_1(KVM_HCALL_TOKEN(KVM_HC_FEATURES), &r)) return 0; return r; } +static inline unsigned int kvm_arch_para_hints(void) +{ + return 0; +} + static inline bool kvm_check_and_clear_guest_paused(void) { return false; diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h index a5287fe03d77..0953f2daa466 100644 --- a/arch/powerpc/include/asm/kvm_ppc.h +++ b/arch/powerpc/include/asm/kvm_ppc.h @@ -1,16 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * * Copyright IBM Corp. 2008 * @@ -36,37 +25,79 @@ #endif #ifdef CONFIG_KVM_BOOK3S_64_HANDLER #include <asm/paca.h> +#include <asm/xive.h> +#include <asm/cpu_has_feature.h> #endif +#include <asm/inst.h> + +/* + * KVMPPC_INST_SW_BREAKPOINT is debug Instruction + * for supporting software breakpoint. + */ +#define KVMPPC_INST_SW_BREAKPOINT 0x00dddd00 enum emulation_result { EMULATE_DONE, /* no further processing */ EMULATE_DO_MMIO, /* kvm_run filled with MMIO request */ - EMULATE_DO_DCR, /* kvm_run filled with DCR request */ EMULATE_FAIL, /* can't emulate this instruction */ EMULATE_AGAIN, /* something went wrong. go again */ EMULATE_EXIT_USER, /* emulation requires exit to user-space */ }; -extern int kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu); -extern int __kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu); +enum instruction_fetch_type { + INST_GENERIC, + INST_SC, /* system call */ +}; + +enum xlate_instdata { + XLATE_INST, /* translate instruction address */ + XLATE_DATA /* translate data address */ +}; + +enum xlate_readwrite { + XLATE_READ, /* check for read permissions */ + XLATE_WRITE /* check for write permissions */ +}; + +extern int kvmppc_vcpu_run(struct kvm_vcpu *vcpu); +extern int __kvmppc_vcpu_run(struct kvm_vcpu *vcpu); extern void kvmppc_handler_highmem(void); extern void kvmppc_dump_vcpu(struct kvm_vcpu *vcpu); -extern int kvmppc_handle_load(struct kvm_run *run, struct kvm_vcpu *vcpu, +extern int kvmppc_handle_load(struct kvm_vcpu *vcpu, unsigned int rt, unsigned int bytes, - int is_bigendian); -extern int kvmppc_handle_loads(struct kvm_run *run, struct kvm_vcpu *vcpu, + int is_default_endian); +extern int kvmppc_handle_loads(struct kvm_vcpu *vcpu, unsigned int rt, unsigned int bytes, - int is_bigendian); -extern int kvmppc_handle_store(struct kvm_run *run, struct kvm_vcpu *vcpu, - u64 val, unsigned int bytes, int is_bigendian); - -extern int kvmppc_emulate_instruction(struct kvm_run *run, - struct kvm_vcpu *vcpu); -extern int kvmppc_emulate_mmio(struct kvm_run *run, struct kvm_vcpu *vcpu); + int is_default_endian); +extern int kvmppc_handle_vsx_load(struct kvm_vcpu *vcpu, + unsigned int rt, unsigned int bytes, + int is_default_endian, int mmio_sign_extend); +extern int kvmppc_handle_vmx_load(struct kvm_vcpu *vcpu, + unsigned int rt, unsigned int bytes, int is_default_endian); +extern int kvmppc_handle_vmx_store(struct kvm_vcpu *vcpu, + unsigned int rs, unsigned int bytes, int is_default_endian); +extern int kvmppc_handle_store(struct kvm_vcpu *vcpu, + u64 val, unsigned int bytes, + int is_default_endian); +extern int kvmppc_handle_vsx_store(struct kvm_vcpu *vcpu, + int rs, unsigned int bytes, + int is_default_endian); + +extern int kvmppc_load_last_inst(struct kvm_vcpu *vcpu, + enum instruction_fetch_type type, + unsigned long *inst); + +extern int kvmppc_ld(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr, + bool data); +extern int kvmppc_st(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr, + bool data); +extern int kvmppc_emulate_instruction(struct kvm_vcpu *vcpu); +extern int kvmppc_emulate_loadstore(struct kvm_vcpu *vcpu); +extern int kvmppc_emulate_mmio(struct kvm_vcpu *vcpu); extern void kvmppc_emulate_dec(struct kvm_vcpu *vcpu); extern u32 kvmppc_get_dec(struct kvm_vcpu *vcpu, u64 tb); -extern void kvmppc_decrementer_func(unsigned long data); +extern void kvmppc_decrementer_func(struct kvm_vcpu *vcpu); extern int kvmppc_sanity_check(struct kvm_vcpu *vcpu); extern int kvmppc_subarch_vcpu_init(struct kvm_vcpu *vcpu); extern void kvmppc_subarch_vcpu_uninit(struct kvm_vcpu *vcpu); @@ -75,22 +106,20 @@ extern void kvmppc_subarch_vcpu_uninit(struct kvm_vcpu *vcpu); extern void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 gvaddr, gpa_t gpaddr, unsigned int gtlb_idx); -extern void kvmppc_mmu_priv_switch(struct kvm_vcpu *vcpu, int usermode); extern void kvmppc_mmu_switch_pid(struct kvm_vcpu *vcpu, u32 pid); -extern void kvmppc_mmu_destroy(struct kvm_vcpu *vcpu); -extern int kvmppc_mmu_init(struct kvm_vcpu *vcpu); extern int kvmppc_mmu_dtlb_index(struct kvm_vcpu *vcpu, gva_t eaddr); extern int kvmppc_mmu_itlb_index(struct kvm_vcpu *vcpu, gva_t eaddr); extern gpa_t kvmppc_mmu_xlate(struct kvm_vcpu *vcpu, unsigned int gtlb_index, gva_t eaddr); extern void kvmppc_mmu_dtlb_miss(struct kvm_vcpu *vcpu); extern void kvmppc_mmu_itlb_miss(struct kvm_vcpu *vcpu); +extern int kvmppc_xlate(struct kvm_vcpu *vcpu, ulong eaddr, + enum xlate_instdata xlid, enum xlate_readwrite xlrw, + struct kvmppc_pte *pte); -extern struct kvm_vcpu *kvmppc_core_vcpu_create(struct kvm *kvm, - unsigned int id); +extern int kvmppc_core_vcpu_create(struct kvm_vcpu *vcpu); extern void kvmppc_core_vcpu_free(struct kvm_vcpu *vcpu); extern int kvmppc_core_vcpu_setup(struct kvm_vcpu *vcpu); -extern int kvmppc_core_check_processor_compat(void); extern int kvmppc_core_vcpu_translate(struct kvm_vcpu *vcpu, struct kvm_translation *tr); @@ -99,60 +128,89 @@ extern void kvmppc_core_vcpu_put(struct kvm_vcpu *vcpu); extern int kvmppc_core_prepare_to_enter(struct kvm_vcpu *vcpu); extern int kvmppc_core_pending_dec(struct kvm_vcpu *vcpu); -extern void kvmppc_core_queue_program(struct kvm_vcpu *vcpu, ulong flags); + +extern void kvmppc_core_queue_machine_check(struct kvm_vcpu *vcpu, + ulong srr1_flags); +extern void kvmppc_core_queue_syscall(struct kvm_vcpu *vcpu); +extern void kvmppc_core_queue_program(struct kvm_vcpu *vcpu, + ulong srr1_flags); +extern void kvmppc_core_queue_fpunavail(struct kvm_vcpu *vcpu, + ulong srr1_flags); +extern void kvmppc_core_queue_vec_unavail(struct kvm_vcpu *vcpu, + ulong srr1_flags); +extern void kvmppc_core_queue_vsx_unavail(struct kvm_vcpu *vcpu, + ulong srr1_flags); extern void kvmppc_core_queue_dec(struct kvm_vcpu *vcpu); extern void kvmppc_core_dequeue_dec(struct kvm_vcpu *vcpu); extern void kvmppc_core_queue_external(struct kvm_vcpu *vcpu, struct kvm_interrupt *irq); extern void kvmppc_core_dequeue_external(struct kvm_vcpu *vcpu); -extern void kvmppc_core_flush_tlb(struct kvm_vcpu *vcpu); +extern void kvmppc_core_queue_dtlb_miss(struct kvm_vcpu *vcpu, + ulong dear_flags, + ulong esr_flags); +extern void kvmppc_core_queue_data_storage(struct kvm_vcpu *vcpu, + ulong srr1_flags, + ulong dar, + ulong dsisr); +extern void kvmppc_core_queue_itlb_miss(struct kvm_vcpu *vcpu); +extern void kvmppc_core_queue_inst_storage(struct kvm_vcpu *vcpu, + ulong srr1_flags); -extern int kvmppc_core_emulate_op(struct kvm_run *run, struct kvm_vcpu *vcpu, - unsigned int op, int *advance); -extern int kvmppc_core_emulate_mtspr(struct kvm_vcpu *vcpu, int sprn, - ulong val); -extern int kvmppc_core_emulate_mfspr(struct kvm_vcpu *vcpu, int sprn, - ulong *val); +extern void kvmppc_core_flush_tlb(struct kvm_vcpu *vcpu); extern int kvmppc_core_check_requests(struct kvm_vcpu *vcpu); extern int kvmppc_booke_init(void); extern void kvmppc_booke_exit(void); -extern void kvmppc_core_destroy_mmu(struct kvm_vcpu *vcpu); extern int kvmppc_kvm_pv(struct kvm_vcpu *vcpu); extern void kvmppc_map_magic(struct kvm_vcpu *vcpu); -extern long kvmppc_alloc_hpt(struct kvm *kvm, u32 *htab_orderp); -extern long kvmppc_alloc_reset_hpt(struct kvm *kvm, u32 *htab_orderp); -extern void kvmppc_free_hpt(struct kvm *kvm); -extern long kvmppc_prepare_vrma(struct kvm *kvm, - struct kvm_userspace_memory_region *mem); +extern int kvmppc_allocate_hpt(struct kvm_hpt_info *info, u32 order); +extern void kvmppc_set_hpt(struct kvm *kvm, struct kvm_hpt_info *info); +extern int kvmppc_alloc_reset_hpt(struct kvm *kvm, int order); +extern void kvmppc_free_hpt(struct kvm_hpt_info *info); +extern void kvmppc_rmap_reset(struct kvm *kvm); extern void kvmppc_map_vrma(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, unsigned long porder); extern int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu); - -extern long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm, - struct kvm_create_spapr_tce *args); +extern long kvm_spapr_tce_attach_iommu_group(struct kvm *kvm, int tablefd, + struct iommu_group *grp); +extern void kvm_spapr_tce_release_iommu_group(struct kvm *kvm, + struct iommu_group *grp); +extern int kvmppc_switch_mmu_to_hpt(struct kvm *kvm); +extern int kvmppc_switch_mmu_to_radix(struct kvm *kvm); +extern void kvmppc_setup_partition_table(struct kvm *kvm); + +extern int kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm, + struct kvm_create_spapr_tce_64 *args); +#define kvmppc_ioba_validate(stt, ioba, npages) \ + (iommu_tce_check_ioba((stt)->page_shift, (stt)->offset, \ + (stt)->size, (ioba), (npages)) ? \ + H_PARAMETER : H_SUCCESS) extern long kvmppc_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn, unsigned long ioba, unsigned long tce); -extern long kvm_vm_ioctl_allocate_rma(struct kvm *kvm, - struct kvm_allocate_rma *rma); -extern struct kvmppc_linear_info *kvm_alloc_rma(void); -extern void kvm_release_rma(struct kvmppc_linear_info *ri); -extern struct kvmppc_linear_info *kvm_alloc_hpt(void); -extern void kvm_release_hpt(struct kvmppc_linear_info *li); +extern long kvmppc_h_put_tce_indirect(struct kvm_vcpu *vcpu, + unsigned long liobn, unsigned long ioba, + unsigned long tce_list, unsigned long npages); +extern long kvmppc_h_stuff_tce(struct kvm_vcpu *vcpu, + unsigned long liobn, unsigned long ioba, + unsigned long tce_value, unsigned long npages); +extern long kvmppc_h_get_tce(struct kvm_vcpu *vcpu, unsigned long liobn, + unsigned long ioba); +extern struct page *kvm_alloc_hpt_cma(unsigned long nr_pages); +extern void kvm_free_hpt_cma(struct page *page, unsigned long nr_pages); extern int kvmppc_core_init_vm(struct kvm *kvm); extern void kvmppc_core_destroy_vm(struct kvm *kvm); -extern void kvmppc_core_free_memslot(struct kvm_memory_slot *free, - struct kvm_memory_slot *dont); -extern int kvmppc_core_create_memslot(struct kvm_memory_slot *slot, - unsigned long npages); +extern void kvmppc_core_free_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot); extern int kvmppc_core_prepare_memory_region(struct kvm *kvm, - struct kvm_memory_slot *memslot, - struct kvm_userspace_memory_region *mem); + const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, + enum kvm_mr_change change); extern void kvmppc_core_commit_memory_region(struct kvm *kvm, - struct kvm_userspace_memory_region *mem, - const struct kvm_memory_slot *old); + struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change); extern int kvm_vm_ioctl_get_smmu_info(struct kvm *kvm, struct kvm_ppc_smmu_info *info); extern void kvmppc_core_flush_memslot(struct kvm *kvm, @@ -164,12 +222,17 @@ extern void kvmppc_bookehv_exit(void); extern int kvmppc_prepare_to_enter(struct kvm_vcpu *vcpu); extern int kvm_vm_ioctl_get_htab_fd(struct kvm *kvm, struct kvm_get_htab_fd *); +extern int kvm_vm_ioctl_resize_hpt_prepare(struct kvm *kvm, + struct kvm_ppc_resize_hpt *rhpt); +extern int kvm_vm_ioctl_resize_hpt_commit(struct kvm *kvm, + struct kvm_ppc_resize_hpt *rhpt); int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, struct kvm_interrupt *irq); extern int kvm_vm_ioctl_rtas_define_token(struct kvm *kvm, void __user *argp); extern int kvmppc_rtas_hcall(struct kvm_vcpu *vcpu); extern void kvmppc_rtas_tokens_free(struct kvm *kvm); + extern int kvmppc_xics_set_xive(struct kvm *kvm, u32 irq, u32 server, u32 priority); extern int kvmppc_xics_get_xive(struct kvm *kvm, u32 irq, u32 *server, @@ -177,6 +240,135 @@ extern int kvmppc_xics_get_xive(struct kvm *kvm, u32 irq, u32 *server, extern int kvmppc_xics_int_on(struct kvm *kvm, u32 irq); extern int kvmppc_xics_int_off(struct kvm *kvm, u32 irq); +void kvmppc_core_dequeue_debug(struct kvm_vcpu *vcpu); +void kvmppc_core_queue_debug(struct kvm_vcpu *vcpu); + +union kvmppc_one_reg { + u32 wval; + u64 dval; + vector128 vval; + u64 vsxval[2]; + u32 vsx32val[4]; + u16 vsx16val[8]; + u8 vsx8val[16]; + struct { + u64 addr; + u64 length; + } vpaval; + u64 xive_timaval[2]; +}; + +struct kvmppc_ops { + struct module *owner; + int (*get_sregs)(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs); + int (*set_sregs)(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs); + int (*get_one_reg)(struct kvm_vcpu *vcpu, u64 id, + union kvmppc_one_reg *val); + int (*set_one_reg)(struct kvm_vcpu *vcpu, u64 id, + union kvmppc_one_reg *val); + void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu); + void (*vcpu_put)(struct kvm_vcpu *vcpu); + void (*inject_interrupt)(struct kvm_vcpu *vcpu, int vec, u64 srr1_flags); + void (*set_msr)(struct kvm_vcpu *vcpu, u64 msr); + int (*vcpu_run)(struct kvm_vcpu *vcpu); + int (*vcpu_create)(struct kvm_vcpu *vcpu); + void (*vcpu_free)(struct kvm_vcpu *vcpu); + int (*check_requests)(struct kvm_vcpu *vcpu); + int (*get_dirty_log)(struct kvm *kvm, struct kvm_dirty_log *log); + void (*flush_memslot)(struct kvm *kvm, struct kvm_memory_slot *memslot); + int (*prepare_memory_region)(struct kvm *kvm, + const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, + enum kvm_mr_change change); + void (*commit_memory_region)(struct kvm *kvm, + struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change); + bool (*unmap_gfn_range)(struct kvm *kvm, struct kvm_gfn_range *range); + bool (*age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range); + bool (*test_age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range); + void (*free_memslot)(struct kvm_memory_slot *slot); + int (*init_vm)(struct kvm *kvm); + void (*destroy_vm)(struct kvm *kvm); + int (*get_smmu_info)(struct kvm *kvm, struct kvm_ppc_smmu_info *info); + int (*emulate_op)(struct kvm_vcpu *vcpu, + unsigned int inst, int *advance); + int (*emulate_mtspr)(struct kvm_vcpu *vcpu, int sprn, ulong spr_val); + int (*emulate_mfspr)(struct kvm_vcpu *vcpu, int sprn, ulong *spr_val); + void (*fast_vcpu_kick)(struct kvm_vcpu *vcpu); + int (*arch_vm_ioctl)(struct file *filp, unsigned int ioctl, + unsigned long arg); + int (*hcall_implemented)(unsigned long hcall); + int (*irq_bypass_add_producer)(struct irq_bypass_consumer *, + struct irq_bypass_producer *); + void (*irq_bypass_del_producer)(struct irq_bypass_consumer *, + struct irq_bypass_producer *); + int (*configure_mmu)(struct kvm *kvm, struct kvm_ppc_mmuv3_cfg *cfg); + int (*get_rmmu_info)(struct kvm *kvm, struct kvm_ppc_rmmu_info *info); + int (*set_smt_mode)(struct kvm *kvm, unsigned long mode, + unsigned long flags); + void (*giveup_ext)(struct kvm_vcpu *vcpu, ulong msr); + int (*enable_nested)(struct kvm *kvm); + int (*load_from_eaddr)(struct kvm_vcpu *vcpu, ulong *eaddr, void *ptr, + int size); + int (*store_to_eaddr)(struct kvm_vcpu *vcpu, ulong *eaddr, void *ptr, + int size); + int (*enable_svm)(struct kvm *kvm); + int (*svm_off)(struct kvm *kvm); + int (*enable_dawr1)(struct kvm *kvm); + bool (*hash_v3_possible)(void); + int (*create_vm_debugfs)(struct kvm *kvm); + int (*create_vcpu_debugfs)(struct kvm_vcpu *vcpu, struct dentry *debugfs_dentry); +}; + +extern struct kvmppc_ops *kvmppc_hv_ops; +extern struct kvmppc_ops *kvmppc_pr_ops; + +static inline int kvmppc_get_last_inst(struct kvm_vcpu *vcpu, + enum instruction_fetch_type type, ppc_inst_t *inst) +{ + int ret = EMULATE_DONE; + u32 fetched_inst; + + /* Load the instruction manually if it failed to do so in the + * exit path */ + if (vcpu->arch.last_inst == KVM_INST_FETCH_FAILED) + ret = kvmppc_load_last_inst(vcpu, type, &vcpu->arch.last_inst); + + /* Write fetch_failed unswapped if the fetch failed */ + if (ret != EMULATE_DONE) { + *inst = ppc_inst(KVM_INST_FETCH_FAILED); + return ret; + } + +#ifdef CONFIG_PPC64 + /* Is this a prefixed instruction? */ + if ((vcpu->arch.last_inst >> 32) != 0) { + u32 prefix = vcpu->arch.last_inst >> 32; + u32 suffix = vcpu->arch.last_inst; + if (kvmppc_need_byteswap(vcpu)) { + prefix = swab32(prefix); + suffix = swab32(suffix); + } + *inst = ppc_inst_prefix(prefix, suffix); + return EMULATE_DONE; + } +#endif + + fetched_inst = kvmppc_need_byteswap(vcpu) ? + swab32(vcpu->arch.last_inst) : + vcpu->arch.last_inst; + *inst = ppc_inst(fetched_inst); + return EMULATE_DONE; +} + +static inline bool is_kvmppc_hv_enabled(struct kvm *kvm) +{ + return kvm->arch.kvm_ops == kvmppc_hv_ops; +} + +extern int kvmppc_hwrng_present(void); + /* * Cuts out inst bits with ordering according to spec. * That means the leftmost bit is zero. All given bits are included. @@ -210,17 +402,6 @@ static inline u32 kvmppc_set_field(u64 inst, int msb, int lsb, int value) return r; } -union kvmppc_one_reg { - u32 wval; - u64 dval; - vector128 vval; - u64 vsxval[2]; - struct { - u64 addr; - u64 length; - } vpaval; -}; - #define one_reg_size(id) \ (1ul << (((id) & KVM_REG_SIZE_MASK) >> KVM_REG_SIZE_SHIFT)) @@ -245,10 +426,10 @@ union kvmppc_one_reg { __v; \ }) -void kvmppc_core_get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs); +int kvmppc_core_get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs); int kvmppc_core_set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs); -void kvmppc_get_sregs_ivor(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs); +int kvmppc_get_sregs_ivor(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs); int kvmppc_set_sregs_ivor(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs); int kvm_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu, struct kvm_one_reg *reg); @@ -260,34 +441,147 @@ void kvmppc_set_pid(struct kvm_vcpu *vcpu, u32 pid); struct openpic; -#ifdef CONFIG_KVM_BOOK3S_64_HV +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE +extern void kvm_cma_reserve(void) __init; static inline void kvmppc_set_xics_phys(int cpu, unsigned long addr) { - paca[cpu].kvm_hstate.xics_phys = addr; + paca_ptrs[cpu]->kvm_hstate.xics_phys = (void __iomem *)addr; +} + +static inline void kvmppc_set_xive_tima(int cpu, + unsigned long phys_addr, + void __iomem *virt_addr) +{ + paca_ptrs[cpu]->kvm_hstate.xive_tima_phys = (void __iomem *)phys_addr; + paca_ptrs[cpu]->kvm_hstate.xive_tima_virt = virt_addr; } static inline u32 kvmppc_get_xics_latch(void) { - u32 xirr = get_paca()->kvm_hstate.saved_xirr; + u32 xirr; + xirr = get_paca()->kvm_hstate.saved_xirr; get_paca()->kvm_hstate.saved_xirr = 0; - return xirr; } -static inline void kvmppc_set_host_ipi(int cpu, u8 host_ipi) +/* + * To avoid the need to unnecessarily exit fully to the host kernel, an IPI to + * a CPU thread that's running/napping inside of a guest is by default regarded + * as a request to wake the CPU (if needed) and continue execution within the + * guest, potentially to process new state like externally-generated + * interrupts or IPIs sent from within the guest itself (e.g. H_PROD/H_IPI). + * + * To force an exit to the host kernel, kvmppc_set_host_ipi() must be called + * prior to issuing the IPI to set the corresponding 'host_ipi' flag in the + * target CPU's PACA. To avoid unnecessary exits to the host, this flag should + * be immediately cleared via kvmppc_clear_host_ipi() by the IPI handler on + * the receiving side prior to processing the IPI work. + * + * NOTE: + * + * We currently issue an smp_mb() at the beginning of kvmppc_set_host_ipi(). + * This is to guard against sequences such as the following: + * + * CPU + * X: smp_muxed_ipi_set_message(): + * X: smp_mb() + * X: message[RESCHEDULE] = 1 + * X: doorbell_global_ipi(42): + * X: kvmppc_set_host_ipi(42) + * X: ppc_msgsnd_sync()/smp_mb() + * X: ppc_msgsnd() -> 42 + * 42: doorbell_exception(): // from CPU X + * 42: ppc_msgsync() + * 105: smp_muxed_ipi_set_message(): + * 105: smb_mb() + * // STORE DEFERRED DUE TO RE-ORDERING + * --105: message[CALL_FUNCTION] = 1 + * | 105: doorbell_global_ipi(42): + * | 105: kvmppc_set_host_ipi(42) + * | 42: kvmppc_clear_host_ipi(42) + * | 42: smp_ipi_demux_relaxed() + * | 42: // returns to executing guest + * | // RE-ORDERED STORE COMPLETES + * ->105: message[CALL_FUNCTION] = 1 + * 105: ppc_msgsnd_sync()/smp_mb() + * 105: ppc_msgsnd() -> 42 + * 42: local_paca->kvm_hstate.host_ipi == 0 // IPI ignored + * 105: // hangs waiting on 42 to process messages/call_single_queue + * + * We also issue an smp_mb() at the end of kvmppc_clear_host_ipi(). This is + * to guard against sequences such as the following (as well as to create + * a read-side pairing with the barrier in kvmppc_set_host_ipi()): + * + * CPU + * X: smp_muxed_ipi_set_message(): + * X: smp_mb() + * X: message[RESCHEDULE] = 1 + * X: doorbell_global_ipi(42): + * X: kvmppc_set_host_ipi(42) + * X: ppc_msgsnd_sync()/smp_mb() + * X: ppc_msgsnd() -> 42 + * 42: doorbell_exception(): // from CPU X + * 42: ppc_msgsync() + * // STORE DEFERRED DUE TO RE-ORDERING + * -- 42: kvmppc_clear_host_ipi(42) + * | 42: smp_ipi_demux_relaxed() + * | 105: smp_muxed_ipi_set_message(): + * | 105: smb_mb() + * | 105: message[CALL_FUNCTION] = 1 + * | 105: doorbell_global_ipi(42): + * | 105: kvmppc_set_host_ipi(42) + * | // RE-ORDERED STORE COMPLETES + * -> 42: kvmppc_clear_host_ipi(42) + * 42: // returns to executing guest + * 105: ppc_msgsnd_sync()/smp_mb() + * 105: ppc_msgsnd() -> 42 + * 42: local_paca->kvm_hstate.host_ipi == 0 // IPI ignored + * 105: // hangs waiting on 42 to process messages/call_single_queue + */ +static inline void kvmppc_set_host_ipi(int cpu) +{ + /* + * order stores of IPI messages vs. setting of host_ipi flag + * + * pairs with the barrier in kvmppc_clear_host_ipi() + */ + smp_mb(); + WRITE_ONCE(paca_ptrs[cpu]->kvm_hstate.host_ipi, 1); +} + +static inline void kvmppc_clear_host_ipi(int cpu) +{ + WRITE_ONCE(paca_ptrs[cpu]->kvm_hstate.host_ipi, 0); + /* + * order clearing of host_ipi flag vs. processing of IPI messages + * + * pairs with the barrier in kvmppc_set_host_ipi() + */ + smp_mb(); +} + +static inline void kvmppc_fast_vcpu_kick(struct kvm_vcpu *vcpu) { - paca[cpu].kvm_hstate.host_ipi = host_ipi; + vcpu->kvm->arch.kvm_ops->fast_vcpu_kick(vcpu); } -extern void kvmppc_fast_vcpu_kick(struct kvm_vcpu *vcpu); -extern void kvm_linear_init(void); +extern void kvm_hv_vm_activated(void); +extern void kvm_hv_vm_deactivated(void); +extern bool kvm_hv_mode_active(void); + +extern void kvmppc_check_need_tlb_flush(struct kvm *kvm, int pcpu); #else +static inline void __init kvm_cma_reserve(void) +{} + static inline void kvmppc_set_xics_phys(int cpu, unsigned long addr) {} -static inline void kvm_linear_init(void) +static inline void kvmppc_set_xive_tima(int cpu, + unsigned long phys_addr, + void __iomem *virt_addr) {} static inline u32 kvmppc_get_xics_latch(void) @@ -295,13 +589,67 @@ static inline u32 kvmppc_get_xics_latch(void) return 0; } -static inline void kvmppc_set_host_ipi(int cpu, u8 host_ipi) +static inline void kvmppc_set_host_ipi(int cpu) +{} + +static inline void kvmppc_clear_host_ipi(int cpu) {} static inline void kvmppc_fast_vcpu_kick(struct kvm_vcpu *vcpu) { kvm_vcpu_kick(vcpu); } + +static inline bool kvm_hv_mode_active(void) { return false; } + +#endif + +#ifdef CONFIG_PPC_PSERIES +static inline bool kvmhv_on_pseries(void) +{ + return !cpu_has_feature(CPU_FTR_HVMODE); +} +#else +static inline bool kvmhv_on_pseries(void) +{ + return false; +} + +#endif + +#ifndef CONFIG_PPC_BOOK3S + +static inline bool kvmhv_is_nestedv2(void) +{ + return false; +} + +static inline bool kvmhv_is_nestedv1(void) +{ + return false; +} + +static inline int kvmhv_nestedv2_reload_ptregs(struct kvm_vcpu *vcpu, + struct pt_regs *regs) +{ + return 0; +} +static inline int kvmhv_nestedv2_mark_dirty_ptregs(struct kvm_vcpu *vcpu, + struct pt_regs *regs) +{ + return 0; +} + +static inline int kvmhv_nestedv2_mark_dirty(struct kvm_vcpu *vcpu, u16 iden) +{ + return 0; +} + +static inline int kvmhv_nestedv2_cached_reload(struct kvm_vcpu *vcpu, u16 iden) +{ + return 0; +} + #endif #ifdef CONFIG_KVM_XICS @@ -309,27 +657,231 @@ static inline int kvmppc_xics_enabled(struct kvm_vcpu *vcpu) { return vcpu->arch.irq_type == KVMPPC_IRQ_XICS; } + +static inline struct kvmppc_passthru_irqmap *kvmppc_get_passthru_irqmap( + struct kvm *kvm) +{ + if (kvm && kvm_irq_bypass) + return kvm->arch.pimap; + return NULL; +} + +extern void kvmppc_alloc_host_rm_ops(void); +extern void kvmppc_free_host_rm_ops(void); +extern void kvmppc_free_pimap(struct kvm *kvm); +extern int kvmppc_xics_rm_complete(struct kvm_vcpu *vcpu, u32 hcall); extern void kvmppc_xics_free_icp(struct kvm_vcpu *vcpu); -extern int kvmppc_xics_create_icp(struct kvm_vcpu *vcpu, unsigned long server); -extern int kvm_vm_ioctl_xics_irq(struct kvm *kvm, struct kvm_irq_level *args); extern int kvmppc_xics_hcall(struct kvm_vcpu *vcpu, u32 cmd); +extern int kvmppc_xive_xics_hcall(struct kvm_vcpu *vcpu, u32 req); extern u64 kvmppc_xics_get_icp(struct kvm_vcpu *vcpu); extern int kvmppc_xics_set_icp(struct kvm_vcpu *vcpu, u64 icpval); extern int kvmppc_xics_connect_vcpu(struct kvm_device *dev, struct kvm_vcpu *vcpu, u32 cpu); +extern void kvmppc_xics_ipi_action(void); +extern void kvmppc_xics_set_mapped(struct kvm *kvm, unsigned long guest_irq, + unsigned long host_irq); +extern void kvmppc_xics_clr_mapped(struct kvm *kvm, unsigned long guest_irq, + unsigned long host_irq); +extern long kvmppc_deliver_irq_passthru(struct kvm_vcpu *vcpu, __be32 xirr, + struct kvmppc_irq_map *irq_map, + struct kvmppc_passthru_irqmap *pimap, + bool *again); + +extern int kvmppc_xics_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, + int level, bool line_status); + +extern int h_ipi_redirect; #else +static inline struct kvmppc_passthru_irqmap *kvmppc_get_passthru_irqmap( + struct kvm *kvm) + { return NULL; } +static inline void kvmppc_alloc_host_rm_ops(void) {} +static inline void kvmppc_free_host_rm_ops(void) {} +static inline void kvmppc_free_pimap(struct kvm *kvm) {} +static inline int kvmppc_xics_rm_complete(struct kvm_vcpu *vcpu, u32 hcall) + { return 0; } static inline int kvmppc_xics_enabled(struct kvm_vcpu *vcpu) { return 0; } static inline void kvmppc_xics_free_icp(struct kvm_vcpu *vcpu) { } -static inline int kvmppc_xics_create_icp(struct kvm_vcpu *vcpu, - unsigned long server) - { return -EINVAL; } -static inline int kvm_vm_ioctl_xics_irq(struct kvm *kvm, - struct kvm_irq_level *args) - { return -ENOTTY; } static inline int kvmppc_xics_hcall(struct kvm_vcpu *vcpu, u32 cmd) { return 0; } +static inline int kvmppc_xive_xics_hcall(struct kvm_vcpu *vcpu, u32 req) + { return 0; } +#endif + +#ifdef CONFIG_KVM_XIVE +/* + * Below the first "xive" is the "eXternal Interrupt Virtualization Engine" + * ie. P9 new interrupt controller, while the second "xive" is the legacy + * "eXternal Interrupt Vector Entry" which is the configuration of an + * interrupt on the "xics" interrupt controller on P8 and earlier. Those + * two function consume or produce a legacy "XIVE" state from the + * new "XIVE" interrupt controller. + */ +extern int kvmppc_xive_set_xive(struct kvm *kvm, u32 irq, u32 server, + u32 priority); +extern int kvmppc_xive_get_xive(struct kvm *kvm, u32 irq, u32 *server, + u32 *priority); +extern int kvmppc_xive_int_on(struct kvm *kvm, u32 irq); +extern int kvmppc_xive_int_off(struct kvm *kvm, u32 irq); + +extern int kvmppc_xive_connect_vcpu(struct kvm_device *dev, + struct kvm_vcpu *vcpu, u32 cpu); +extern void kvmppc_xive_cleanup_vcpu(struct kvm_vcpu *vcpu); +extern int kvmppc_xive_set_mapped(struct kvm *kvm, unsigned long guest_irq, + unsigned long host_irq); +extern int kvmppc_xive_clr_mapped(struct kvm *kvm, unsigned long guest_irq, + unsigned long host_irq); +extern u64 kvmppc_xive_get_icp(struct kvm_vcpu *vcpu); +extern int kvmppc_xive_set_icp(struct kvm_vcpu *vcpu, u64 icpval); + +extern int kvmppc_xive_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, + int level, bool line_status); +extern void kvmppc_xive_push_vcpu(struct kvm_vcpu *vcpu); +extern void kvmppc_xive_pull_vcpu(struct kvm_vcpu *vcpu); +extern bool kvmppc_xive_rearm_escalation(struct kvm_vcpu *vcpu); + +static inline int kvmppc_xive_enabled(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.irq_type == KVMPPC_IRQ_XIVE; +} + +extern int kvmppc_xive_native_connect_vcpu(struct kvm_device *dev, + struct kvm_vcpu *vcpu, u32 cpu); +extern void kvmppc_xive_native_cleanup_vcpu(struct kvm_vcpu *vcpu); +extern int kvmppc_xive_native_get_vp(struct kvm_vcpu *vcpu, + union kvmppc_one_reg *val); +extern int kvmppc_xive_native_set_vp(struct kvm_vcpu *vcpu, + union kvmppc_one_reg *val); +extern bool kvmppc_xive_native_supported(void); + +#else +static inline int kvmppc_xive_set_xive(struct kvm *kvm, u32 irq, u32 server, + u32 priority) { return -1; } +static inline int kvmppc_xive_get_xive(struct kvm *kvm, u32 irq, u32 *server, + u32 *priority) { return -1; } +static inline int kvmppc_xive_int_on(struct kvm *kvm, u32 irq) { return -1; } +static inline int kvmppc_xive_int_off(struct kvm *kvm, u32 irq) { return -1; } + +static inline int kvmppc_xive_connect_vcpu(struct kvm_device *dev, + struct kvm_vcpu *vcpu, u32 cpu) { return -EBUSY; } +static inline void kvmppc_xive_cleanup_vcpu(struct kvm_vcpu *vcpu) { } +static inline int kvmppc_xive_set_mapped(struct kvm *kvm, unsigned long guest_irq, + struct irq_desc *host_desc) { return -ENODEV; } +static inline int kvmppc_xive_clr_mapped(struct kvm *kvm, unsigned long guest_irq, + struct irq_desc *host_desc) { return -ENODEV; } +static inline u64 kvmppc_xive_get_icp(struct kvm_vcpu *vcpu) { return 0; } +static inline int kvmppc_xive_set_icp(struct kvm_vcpu *vcpu, u64 icpval) { return -ENOENT; } + +static inline int kvmppc_xive_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, + int level, bool line_status) { return -ENODEV; } +static inline void kvmppc_xive_push_vcpu(struct kvm_vcpu *vcpu) { } +static inline void kvmppc_xive_pull_vcpu(struct kvm_vcpu *vcpu) { } +static inline bool kvmppc_xive_rearm_escalation(struct kvm_vcpu *vcpu) { return true; } + +static inline int kvmppc_xive_enabled(struct kvm_vcpu *vcpu) + { return 0; } +static inline int kvmppc_xive_native_connect_vcpu(struct kvm_device *dev, + struct kvm_vcpu *vcpu, u32 cpu) { return -EBUSY; } +static inline void kvmppc_xive_native_cleanup_vcpu(struct kvm_vcpu *vcpu) { } +static inline int kvmppc_xive_native_get_vp(struct kvm_vcpu *vcpu, + union kvmppc_one_reg *val) +{ return 0; } +static inline int kvmppc_xive_native_set_vp(struct kvm_vcpu *vcpu, + union kvmppc_one_reg *val) +{ return -ENOENT; } + +#endif /* CONFIG_KVM_XIVE */ + +#if defined(CONFIG_PPC_POWERNV) && defined(CONFIG_KVM_BOOK3S_64_HANDLER) +static inline bool xics_on_xive(void) +{ + return xive_enabled() && cpu_has_feature(CPU_FTR_HVMODE); +} +#else +static inline bool xics_on_xive(void) +{ + return false; +} +#endif + +/* + * Prototypes for functions called only from assembler code. + * Having prototypes reduces sparse errors. + */ +long kvmppc_rm_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn, + unsigned long ioba, unsigned long tce); +long kvmppc_rm_h_put_tce_indirect(struct kvm_vcpu *vcpu, + unsigned long liobn, unsigned long ioba, + unsigned long tce_list, unsigned long npages); +long kvmppc_rm_h_stuff_tce(struct kvm_vcpu *vcpu, + unsigned long liobn, unsigned long ioba, + unsigned long tce_value, unsigned long npages); +long int kvmppc_rm_h_confer(struct kvm_vcpu *vcpu, int target, + unsigned int yield_count); +long kvmppc_rm_h_random(struct kvm_vcpu *vcpu); +void kvmhv_commence_exit(int trap); +void kvmppc_realmode_machine_check(struct kvm_vcpu *vcpu); +void kvmppc_subcore_enter_guest(void); +void kvmppc_subcore_exit_guest(void); +long kvmppc_realmode_hmi_handler(void); +long kvmppc_p9_realmode_hmi_handler(struct kvm_vcpu *vcpu); +long kvmppc_h_enter(struct kvm_vcpu *vcpu, unsigned long flags, + long pte_index, unsigned long pteh, unsigned long ptel); +long kvmppc_h_remove(struct kvm_vcpu *vcpu, unsigned long flags, + unsigned long pte_index, unsigned long avpn); +long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu); +long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags, + unsigned long pte_index, unsigned long avpn); +long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags, + unsigned long pte_index); +long kvmppc_h_clear_ref(struct kvm_vcpu *vcpu, unsigned long flags, + unsigned long pte_index); +long kvmppc_h_clear_mod(struct kvm_vcpu *vcpu, unsigned long flags, + unsigned long pte_index); +long kvmppc_rm_h_page_init(struct kvm_vcpu *vcpu, unsigned long flags, + unsigned long dest, unsigned long src); +long kvmppc_hpte_hv_fault(struct kvm_vcpu *vcpu, unsigned long addr, + unsigned long slb_v, unsigned int status, bool data); +void kvmppc_guest_entry_inject_int(struct kvm_vcpu *vcpu); + +/* + * Host-side operations we want to set up while running in real + * mode in the guest operating on the xics. + * Currently only VCPU wakeup is supported. + */ + +union kvmppc_rm_state { + unsigned long raw; + struct { + u32 in_host; + u32 rm_action; + }; +}; + +struct kvmppc_host_rm_core { + union kvmppc_rm_state rm_state; + void *rm_data; + char pad[112]; +}; + +struct kvmppc_host_rm_ops { + struct kvmppc_host_rm_core *rm_core; + void (*vcpu_kick)(struct kvm_vcpu *vcpu); +}; + +extern struct kvmppc_host_rm_ops *kvmppc_host_rm_ops_hv; + +static inline unsigned long kvmppc_get_epr(struct kvm_vcpu *vcpu) +{ +#ifdef CONFIG_KVM_BOOKE_HV + return mfspr(SPRN_GEPR); +#elif defined(CONFIG_BOOKE) + return vcpu->arch.epr; +#else + return 0; #endif +} static inline void kvmppc_set_epr(struct kvm_vcpu *vcpu, u32 epr) { @@ -372,13 +924,12 @@ int kvm_vcpu_ioctl_dirty_tlb(struct kvm_vcpu *vcpu, struct kvm_dirty_tlb *cfg); long kvmppc_alloc_lpid(void); -void kvmppc_claim_lpid(long lpid); void kvmppc_free_lpid(long lpid); void kvmppc_init_lpid(unsigned long nr_lpids); -static inline void kvmppc_mmu_flush_icache(pfn_t pfn) +static inline void kvmppc_mmu_flush_icache(kvm_pfn_t pfn) { - struct page *page; + struct folio *folio; /* * We can only access pages that the kernel maps * as memory. Bail out for unmapped ones. @@ -387,24 +938,160 @@ static inline void kvmppc_mmu_flush_icache(pfn_t pfn) return; /* Clear i-cache for new pages */ - page = pfn_to_page(pfn); - if (!test_bit(PG_arch_1, &page->flags)) { - flush_dcache_icache_page(page); - set_bit(PG_arch_1, &page->flags); + folio = page_folio(pfn_to_page(pfn)); + if (!test_bit(PG_dcache_clean, &folio->flags.f)) { + flush_dcache_icache_folio(folio); + set_bit(PG_dcache_clean, &folio->flags.f); } } -/* Please call after prepare_to_enter. This function puts the lazy ee state - back to normal mode, without actually enabling interrupts. */ -static inline void kvmppc_lazy_ee_enable(void) +/* + * Shared struct helpers. The shared struct can be little or big endian, + * depending on the guest endianness. So expose helpers to all of them. + */ +static inline bool kvmppc_shared_big_endian(struct kvm_vcpu *vcpu) +{ +#if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_KVM_BOOK3S_PR_POSSIBLE) + /* Only Book3S_64 PR supports bi-endian for now */ + return vcpu->arch.shared_big_endian; +#elif defined(CONFIG_PPC_BOOK3S_64) && defined(__LITTLE_ENDIAN__) + /* Book3s_64 HV on little endian is always little endian */ + return false; +#else + return true; +#endif +} + +#define KVMPPC_BOOKE_HV_SPRNG_ACCESSOR_GET(reg, bookehv_spr) \ +static inline ulong kvmppc_get_##reg(struct kvm_vcpu *vcpu) \ +{ \ + return mfspr(bookehv_spr); \ +} \ + +#define KVMPPC_BOOKE_HV_SPRNG_ACCESSOR_SET(reg, bookehv_spr) \ +static inline void kvmppc_set_##reg(struct kvm_vcpu *vcpu, ulong val) \ +{ \ + mtspr(bookehv_spr, val); \ +} \ + +#define KVMPPC_VCPU_SHARED_REGS_ACCESSOR_GET(reg, size, iden) \ +static inline u##size kvmppc_get_##reg(struct kvm_vcpu *vcpu) \ +{ \ + if (iden) \ + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, iden) < 0); \ + if (kvmppc_shared_big_endian(vcpu)) \ + return be##size##_to_cpu((__be##size __force)vcpu->arch.shared->reg); \ + else \ + return le##size##_to_cpu((__le##size __force)vcpu->arch.shared->reg); \ +} \ + +#define KVMPPC_VCPU_SHARED_REGS_ACCESSOR_SET(reg, size, iden) \ +static inline void kvmppc_set_##reg(struct kvm_vcpu *vcpu, u##size val) \ +{ \ + if (kvmppc_shared_big_endian(vcpu)) \ + vcpu->arch.shared->reg = (u##size __force)cpu_to_be##size(val); \ + else \ + vcpu->arch.shared->reg = (u##size __force)cpu_to_le##size(val); \ + \ + if (iden) \ + kvmhv_nestedv2_mark_dirty(vcpu, iden); \ +} \ + +#define KVMPPC_VCPU_SHARED_REGS_ACCESSOR(reg, size, iden) \ + KVMPPC_VCPU_SHARED_REGS_ACCESSOR_GET(reg, size, iden) \ + KVMPPC_VCPU_SHARED_REGS_ACCESSOR_SET(reg, size, iden) \ + +#define KVMPPC_BOOKE_HV_SPRNG_ACCESSOR(reg, bookehv_spr) \ + KVMPPC_BOOKE_HV_SPRNG_ACCESSOR_GET(reg, bookehv_spr) \ + KVMPPC_BOOKE_HV_SPRNG_ACCESSOR_SET(reg, bookehv_spr) \ + +#ifdef CONFIG_KVM_BOOKE_HV + +#define KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(reg, size, bookehv_spr, iden) \ + KVMPPC_BOOKE_HV_SPRNG_ACCESSOR(reg, bookehv_spr) \ + +#else + +#define KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(reg, size, bookehv_spr, iden) \ + KVMPPC_VCPU_SHARED_REGS_ACCESSOR(reg, size, iden) \ + +#endif + +KVMPPC_VCPU_SHARED_REGS_ACCESSOR(critical, 64, 0) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(sprg0, 64, SPRN_GSPRG0, KVMPPC_GSID_SPRG0) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(sprg1, 64, SPRN_GSPRG1, KVMPPC_GSID_SPRG1) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(sprg2, 64, SPRN_GSPRG2, KVMPPC_GSID_SPRG2) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(sprg3, 64, SPRN_GSPRG3, KVMPPC_GSID_SPRG3) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(srr0, 64, SPRN_GSRR0, KVMPPC_GSID_SRR0) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(srr1, 64, SPRN_GSRR1, KVMPPC_GSID_SRR1) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(dar, 64, SPRN_GDEAR, KVMPPC_GSID_DAR) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(esr, 64, SPRN_GESR, 0) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR_GET(msr, 64, KVMPPC_GSID_MSR) +static inline void kvmppc_set_msr_fast(struct kvm_vcpu *vcpu, u64 val) +{ + if (kvmppc_shared_big_endian(vcpu)) + vcpu->arch.shared->msr = cpu_to_be64(val); + else + vcpu->arch.shared->msr = cpu_to_le64(val); + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_MSR); +} +KVMPPC_VCPU_SHARED_REGS_ACCESSOR(dsisr, 32, KVMPPC_GSID_DSISR) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR(int_pending, 32, 0) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR(sprg4, 64, 0) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR(sprg5, 64, 0) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR(sprg6, 64, 0) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR(sprg7, 64, 0) + +static inline u32 kvmppc_get_sr(struct kvm_vcpu *vcpu, int nr) +{ + if (kvmppc_shared_big_endian(vcpu)) + return be32_to_cpu(vcpu->arch.shared->sr[nr]); + else + return le32_to_cpu(vcpu->arch.shared->sr[nr]); +} + +static inline void kvmppc_set_sr(struct kvm_vcpu *vcpu, int nr, u32 val) +{ + if (kvmppc_shared_big_endian(vcpu)) + vcpu->arch.shared->sr[nr] = cpu_to_be32(val); + else + vcpu->arch.shared->sr[nr] = cpu_to_le32(val); +} + +/* + * Please call after prepare_to_enter. This function puts the lazy ee and irq + * disabled tracking state back to normal mode, without actually enabling + * interrupts. + */ +static inline void kvmppc_fix_ee_before_entry(void) { + trace_hardirqs_on(); + #ifdef CONFIG_PPC64 + /* + * To avoid races, the caller must have gone directly from having + * interrupts fully-enabled to hard-disabled. + */ + WARN_ON(local_paca->irq_happened != PACA_IRQ_HARD_DIS); + /* Only need to enable IRQs by hard enabling them after this */ local_paca->irq_happened = 0; - local_paca->soft_enabled = 1; + irq_soft_mask_set(IRQS_ENABLED); +#endif +} + +static inline void kvmppc_fix_ee_after_exit(void) +{ +#ifdef CONFIG_PPC64 + /* Only need to enable IRQs by hard enabling them after this */ + local_paca->irq_happened = PACA_IRQ_HARD_DIS; + irq_soft_mask_set(IRQS_ALL_DISABLED); #endif + + trace_hardirqs_off(); } + static inline ulong kvmppc_get_ea_indexed(struct kvm_vcpu *vcpu, int ra, int rb) { ulong ea; @@ -420,7 +1107,7 @@ static inline ulong kvmppc_get_ea_indexed(struct kvm_vcpu *vcpu, int ra, int rb) msr_64bit = MSR_SF; #endif - if (!(vcpu->arch.shared->msr & msr_64bit)) + if (!(kvmppc_get_msr(vcpu) & msr_64bit)) ea = (uint32_t)ea; return ea; diff --git a/arch/powerpc/include/asm/kvm_types.h b/arch/powerpc/include/asm/kvm_types.h new file mode 100644 index 000000000000..5d4bffea7d47 --- /dev/null +++ b/arch/powerpc/include/asm/kvm_types.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_PPC_KVM_TYPES_H +#define _ASM_PPC_KVM_TYPES_H + +#if IS_MODULE(CONFIG_KVM_BOOK3S_64_PR) && IS_MODULE(CONFIG_KVM_BOOK3S_64_HV) +#define KVM_SUB_MODULES kvm-pr,kvm-hv +#elif IS_MODULE(CONFIG_KVM_BOOK3S_64_PR) +#define KVM_SUB_MODULES kvm-pr +#elif IS_MODULE(CONFIG_KVM_BOOK3S_64_HV) +#define KVM_SUB_MODULES kvm-hv +#else +#undef KVM_SUB_MODULES +#endif + +#endif diff --git a/arch/powerpc/include/asm/libata-portmap.h b/arch/powerpc/include/asm/libata-portmap.h index 4d8518049f4d..7c602da62560 100644 --- a/arch/powerpc/include/asm/libata-portmap.h +++ b/arch/powerpc/include/asm/libata-portmap.h @@ -1,12 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __ASM_POWERPC_LIBATA_PORTMAP_H #define __ASM_POWERPC_LIBATA_PORTMAP_H -#define ATA_PRIMARY_CMD 0x1F0 -#define ATA_PRIMARY_CTL 0x3F6 #define ATA_PRIMARY_IRQ(dev) pci_get_legacy_ide_irq(dev, 0) -#define ATA_SECONDARY_CMD 0x170 -#define ATA_SECONDARY_CTL 0x376 #define ATA_SECONDARY_IRQ(dev) pci_get_legacy_ide_irq(dev, 1) #endif diff --git a/arch/powerpc/include/asm/linkage.h b/arch/powerpc/include/asm/linkage.h index b36f650a13ff..b71b9582e754 100644 --- a/arch/powerpc/include/asm/linkage.h +++ b/arch/powerpc/include/asm/linkage.h @@ -1,7 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_LINKAGE_H #define _ASM_POWERPC_LINKAGE_H -#ifdef CONFIG_PPC64 +#include <asm/types.h> + +#ifdef CONFIG_PPC64_ELF_ABI_V1 #define cond_syscall(x) \ asm ("\t.weak " #x "\n\t.set " #x ", sys_ni_syscall\n" \ "\t.weak ." #x "\n\t.set ." #x ", .sys_ni_syscall\n") diff --git a/arch/powerpc/include/asm/livepatch.h b/arch/powerpc/include/asm/livepatch.h new file mode 100644 index 000000000000..d044a1fd4f44 --- /dev/null +++ b/arch/powerpc/include/asm/livepatch.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * livepatch.h - powerpc-specific Kernel Live Patching Core + * + * Copyright (C) 2015-2016, SUSE, IBM Corp. + */ +#ifndef _ASM_POWERPC_LIVEPATCH_H +#define _ASM_POWERPC_LIVEPATCH_H + +#include <linux/sched.h> +#include <linux/sched/task_stack.h> + +#ifdef CONFIG_LIVEPATCH_64 +static inline void klp_init_thread_info(struct task_struct *p) +{ + /* + 1 to account for STACK_END_MAGIC */ + task_thread_info(p)->livepatch_sp = end_of_stack(p) + 1; +} +#else +static inline void klp_init_thread_info(struct task_struct *p) { } +#endif + +#endif /* _ASM_POWERPC_LIVEPATCH_H */ diff --git a/arch/powerpc/include/asm/local.h b/arch/powerpc/include/asm/local.h index b8da91363864..ec6ced6d7ced 100644 --- a/arch/powerpc/include/asm/local.h +++ b/arch/powerpc/include/asm/local.h @@ -1,76 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ARCH_POWERPC_LOCAL_H #define _ARCH_POWERPC_LOCAL_H +#ifdef CONFIG_PPC_BOOK3S_64 + #include <linux/percpu.h> #include <linux/atomic.h> +#include <linux/irqflags.h> + +#include <asm/hw_irq.h> typedef struct { - atomic_long_t a; + long v; } local_t; -#define LOCAL_INIT(i) { ATOMIC_LONG_INIT(i) } - -#define local_read(l) atomic_long_read(&(l)->a) -#define local_set(l,i) atomic_long_set(&(l)->a, (i)) - -#define local_add(i,l) atomic_long_add((i),(&(l)->a)) -#define local_sub(i,l) atomic_long_sub((i),(&(l)->a)) -#define local_inc(l) atomic_long_inc(&(l)->a) -#define local_dec(l) atomic_long_dec(&(l)->a) +#define LOCAL_INIT(i) { (i) } -static __inline__ long local_add_return(long a, local_t *l) +static __inline__ long local_read(const local_t *l) { - long t; - - __asm__ __volatile__( -"1:" PPC_LLARX(%0,0,%2,0) " # local_add_return\n\ - add %0,%1,%0\n" - PPC405_ERR77(0,%2) - PPC_STLCX "%0,0,%2 \n\ - bne- 1b" - : "=&r" (t) - : "r" (a), "r" (&(l->a.counter)) - : "cc", "memory"); - - return t; + return READ_ONCE(l->v); } -#define local_add_negative(a, l) (local_add_return((a), (l)) < 0) - -static __inline__ long local_sub_return(long a, local_t *l) +static __inline__ void local_set(local_t *l, long i) { - long t; + WRITE_ONCE(l->v, i); +} - __asm__ __volatile__( -"1:" PPC_LLARX(%0,0,%2,0) " # local_sub_return\n\ - subf %0,%1,%0\n" - PPC405_ERR77(0,%2) - PPC_STLCX "%0,0,%2 \n\ - bne- 1b" - : "=&r" (t) - : "r" (a), "r" (&(l->a.counter)) - : "cc", "memory"); +#define LOCAL_OP(op, c_op) \ +static __inline__ void local_##op(long i, local_t *l) \ +{ \ + unsigned long flags; \ + \ + powerpc_local_irq_pmu_save(flags); \ + l->v c_op i; \ + powerpc_local_irq_pmu_restore(flags); \ +} - return t; +#define LOCAL_OP_RETURN(op, c_op) \ +static __inline__ long local_##op##_return(long a, local_t *l) \ +{ \ + long t; \ + unsigned long flags; \ + \ + powerpc_local_irq_pmu_save(flags); \ + t = (l->v c_op a); \ + powerpc_local_irq_pmu_restore(flags); \ + \ + return t; \ } -static __inline__ long local_inc_return(local_t *l) -{ - long t; +#define LOCAL_OPS(op, c_op) \ + LOCAL_OP(op, c_op) \ + LOCAL_OP_RETURN(op, c_op) - __asm__ __volatile__( -"1:" PPC_LLARX(%0,0,%1,0) " # local_inc_return\n\ - addic %0,%0,1\n" - PPC405_ERR77(0,%1) - PPC_STLCX "%0,0,%1 \n\ - bne- 1b" - : "=&r" (t) - : "r" (&(l->a.counter)) - : "cc", "xer", "memory"); +LOCAL_OPS(add, +=) +LOCAL_OPS(sub, -=) - return t; -} +#define local_add_negative(a, l) (local_add_return((a), (l)) < 0) +#define local_inc_return(l) local_add_return(1LL, l) +#define local_inc(l) local_inc_return(l) /* * local_inc_and_test - increment and test @@ -80,96 +69,91 @@ static __inline__ long local_inc_return(local_t *l) * and returns true if the result is zero, or false for all * other cases. */ -#define local_inc_and_test(l) (local_inc_return(l) == 0) +#define local_inc_and_test(l) (local_inc_return(l) == 0) -static __inline__ long local_dec_return(local_t *l) +#define local_dec_return(l) local_sub_return(1LL, l) +#define local_dec(l) local_dec_return(l) +#define local_sub_and_test(a, l) (local_sub_return((a), (l)) == 0) +#define local_dec_and_test(l) (local_dec_return((l)) == 0) + +static __inline__ long local_cmpxchg(local_t *l, long o, long n) { long t; + unsigned long flags; - __asm__ __volatile__( -"1:" PPC_LLARX(%0,0,%1,0) " # local_dec_return\n\ - addic %0,%0,-1\n" - PPC405_ERR77(0,%1) - PPC_STLCX "%0,0,%1\n\ - bne- 1b" - : "=&r" (t) - : "r" (&(l->a.counter)) - : "cc", "xer", "memory"); + powerpc_local_irq_pmu_save(flags); + t = l->v; + if (t == o) + l->v = n; + powerpc_local_irq_pmu_restore(flags); return t; } -#define local_cmpxchg(l, o, n) \ - (cmpxchg_local(&((l)->a.counter), (o), (n))) -#define local_xchg(l, n) (xchg_local(&((l)->a.counter), (n))) - -/** - * local_add_unless - add unless the number is a given value - * @l: pointer of type local_t - * @a: the amount to add to v... - * @u: ...unless v is equal to u. - * - * Atomically adds @a to @l, so long as it was not @u. - * Returns non-zero if @l was not @u, and zero otherwise. - */ -static __inline__ int local_add_unless(local_t *l, long a, long u) +static __inline__ bool local_try_cmpxchg(local_t *l, long *po, long n) { - long t; + long o = *po, r; + + r = local_cmpxchg(l, o, n); + if (unlikely(r != o)) + *po = r; - __asm__ __volatile__ ( -"1:" PPC_LLARX(%0,0,%1,0) " # local_add_unless\n\ - cmpw 0,%0,%3 \n\ - beq- 2f \n\ - add %0,%2,%0 \n" - PPC405_ERR77(0,%2) - PPC_STLCX "%0,0,%1 \n\ - bne- 1b \n" -" subf %0,%2,%0 \n\ -2:" - : "=&r" (t) - : "r" (&(l->a.counter)), "r" (a), "r" (u) - : "cc", "memory"); - - return t != u; + return likely(r == o); } -#define local_inc_not_zero(l) local_add_unless((l), 1, 0) +static __inline__ long local_xchg(local_t *l, long n) +{ + long t; + unsigned long flags; -#define local_sub_and_test(a, l) (local_sub_return((a), (l)) == 0) -#define local_dec_and_test(l) (local_dec_return((l)) == 0) + powerpc_local_irq_pmu_save(flags); + t = l->v; + l->v = n; + powerpc_local_irq_pmu_restore(flags); -/* - * Atomically test *l and decrement if it is greater than 0. - * The function returns the old value of *l minus 1. + return t; +} + +/** + * local_add_unless - add unless the number is already a given value + * @l: pointer of type local_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @l, if @v was not already @u. + * Returns true if the addition was done. */ -static __inline__ long local_dec_if_positive(local_t *l) +static __inline__ bool local_add_unless(local_t *l, long a, long u) { - long t; + unsigned long flags; + bool ret = false; - __asm__ __volatile__( -"1:" PPC_LLARX(%0,0,%1,0) " # local_dec_if_positive\n\ - cmpwi %0,1\n\ - addi %0,%0,-1\n\ - blt- 2f\n" - PPC405_ERR77(0,%1) - PPC_STLCX "%0,0,%1\n\ - bne- 1b" - "\n\ -2:" : "=&b" (t) - : "r" (&(l->a.counter)) - : "cc", "memory"); + powerpc_local_irq_pmu_save(flags); + if (l->v != u) { + l->v += a; + ret = true; + } + powerpc_local_irq_pmu_restore(flags); - return t; + return ret; } +#define local_inc_not_zero(l) local_add_unless((l), 1, 0) + /* Use these for per-cpu local_t variables: on some archs they are * much more efficient than these naive implementations. Note they take * a variable, not an address. */ -#define __local_inc(l) ((l)->a.counter++) -#define __local_dec(l) ((l)->a.counter++) -#define __local_add(i,l) ((l)->a.counter+=(i)) -#define __local_sub(i,l) ((l)->a.counter-=(i)) +#define __local_inc(l) ((l)->v++) +#define __local_dec(l) ((l)->v++) +#define __local_add(i,l) ((l)->v+=(i)) +#define __local_sub(i,l) ((l)->v-=(i)) + +#else /* CONFIG_PPC64 */ + +#include <asm-generic/local.h> + +#endif /* CONFIG_PPC64 */ #endif /* _ARCH_POWERPC_LOCAL_H */ diff --git a/arch/powerpc/include/asm/local64.h b/arch/powerpc/include/asm/local64.h deleted file mode 100644 index 36c93b5cc239..000000000000 --- a/arch/powerpc/include/asm/local64.h +++ /dev/null @@ -1 +0,0 @@ -#include <asm-generic/local64.h> diff --git a/arch/powerpc/include/asm/lppaca.h b/arch/powerpc/include/asm/lppaca.h index 9b12f88d4adb..f40a646bee3c 100644 --- a/arch/powerpc/include/asm/lppaca.h +++ b/arch/powerpc/include/asm/lppaca.h @@ -1,23 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * lppaca.h * Copyright (C) 2001 Mike Corrigan IBM Corporation - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef _ASM_POWERPC_LPPACA_H #define _ASM_POWERPC_LPPACA_H + #ifdef __KERNEL__ /* @@ -34,29 +22,31 @@ #include <linux/threads.h> #include <asm/types.h> #include <asm/mmu.h> +#include <asm/firmware.h> +#include <asm/paca.h> /* - * We only have to have statically allocated lppaca structs on - * legacy iSeries, which supports at most 64 cpus. - */ -#define NR_LPPACAS 1 - -/* - * The Hypervisor barfs if the lppaca crosses a page boundary. A 1k - * alignment is sufficient to prevent this + * The lppaca is the "virtual processor area" registered with the hypervisor, + * H_REGISTER_VPA etc. + * + * According to PAPR, the structure is 640 bytes long, must be L1 cache line + * aligned, and must not cross a 4kB boundary. Its size field must be at + * least 640 bytes (but may be more). + * + * Pre-v4.14 KVM hypervisors reject the VPA if its size field is smaller than + * 1kB, so we dynamically allocate 1kB and advertise size as 1kB, but keep + * this structure as the canonical 640 byte size. */ struct lppaca { /* cacheline 1 contains read-only data */ - u32 desc; /* Eye catcher 0xD397D781 */ - u16 size; /* Size of this struct */ - u16 reserved1; - u16 reserved2:14; - u8 shared_proc:1; /* Shared processor indicator */ - u8 secondary_thread:1; /* Secondary thread indicator */ + __be32 desc; /* Eye catcher 0xD397D781 */ + __be16 size; /* Size of this struct */ + u8 reserved1[3]; + u8 __old_status; /* Old status, including shared proc */ u8 reserved3[14]; - volatile u32 dyn_hw_node_id; /* Dynamic hardware node id */ - volatile u32 dyn_hw_proc_id; /* Dynamic hardware proc id */ + volatile __be32 dyn_hw_node_id; /* Dynamic hardware node id */ + volatile __be32 dyn_hw_proc_id; /* Dynamic hardware proc id */ u8 reserved4[56]; volatile u8 vphn_assoc_counts[8]; /* Virtual processor home node */ /* associativity change counters */ @@ -72,10 +62,11 @@ struct lppaca { u8 donate_dedicated_cpu; /* Donate dedicated CPU cycles */ u8 fpregs_in_use; u8 pmcregs_in_use; - u8 reserved8[28]; - u64 wait_state_cycles; /* Wait cycles for this proc */ + u8 l2_counters_enable; /* Enable usage of counters for KVM guest */ + u8 reserved8[27]; + __be64 wait_state_cycles; /* Wait cycles for this proc */ u8 reserved9[28]; - u16 slb_count; /* # of SLBs to maintain */ + __be16 slb_count; /* # of SLBs to maintain */ u8 idle; /* Indicate OS is idle */ u8 vmxregs_in_use; @@ -86,26 +77,56 @@ struct lppaca { * the processor is yielded (either because of an OS yield or a * hypervisor preempt). An even value implies that the processor is * currently executing. - * NOTE: This value will ALWAYS be zero for dedicated processors and - * will NEVER be zero for shared processors (ie, initialized to a 1). + * NOTE: Even dedicated processor partitions can yield so this + * field cannot be used to determine if we are shared or dedicated. */ - volatile u32 yield_count; - volatile u32 dispersion_count; /* dispatch changed physical cpu */ - volatile u64 cmo_faults; /* CMO page fault count */ - volatile u64 cmo_fault_time; /* CMO page fault time */ - u8 reserved10[104]; + volatile __be32 yield_count; + volatile __be32 dispersion_count; /* dispatch changed physical cpu */ + volatile __be64 cmo_faults; /* CMO page fault count */ + volatile __be64 cmo_fault_time; /* CMO page fault time */ + u8 reserved10[64]; /* [S]PURR expropriated/donated */ + volatile __be64 enqueue_dispatch_tb; /* Total TB enqueue->dispatch */ + volatile __be64 ready_enqueue_tb; /* Total TB ready->enqueue */ + volatile __be64 wait_ready_tb; /* Total TB wait->ready */ + u8 reserved11[16]; /* cacheline 4-5 */ - u32 page_ins; /* CMO Hint - # page ins by OS */ - u8 reserved11[148]; - volatile u64 dtl_idx; /* Dispatch Trace Log head index */ - u8 reserved12[96]; -} __attribute__((__aligned__(0x400))); + __be32 page_ins; /* CMO Hint - # page ins by OS */ + u8 reserved12[28]; + volatile __be64 l1_to_l2_cs_tb; + volatile __be64 l2_to_l1_cs_tb; + volatile __be64 l2_runtime_tb; + u8 reserved13[96]; + volatile __be64 dtl_idx; /* Dispatch Trace Log head index */ + u8 reserved14[96]; +} ____cacheline_aligned; + +#define lppaca_of(cpu) (*paca_ptrs[cpu]->lppaca_ptr) + +/* + * We are using a non architected field to determine if a partition is + * shared or dedicated. This currently works on both KVM and PHYP, but + * we will have to transition to something better. + */ +#define LPPACA_OLD_SHARED_PROC 2 + +#ifdef CONFIG_PPC_PSERIES +/* + * All CPUs should have the same shared proc value, so directly access the PACA + * to avoid false positives from DEBUG_PREEMPT. + */ +static inline bool lppaca_shared_proc(void) +{ + struct lppaca *l = local_paca->lppaca_ptr; -extern struct lppaca lppaca[]; + if (!firmware_has_feature(FW_FEATURE_SPLPAR)) + return false; + return !!(l->__old_status & LPPACA_OLD_SHARED_PROC); +} -#define lppaca_of(cpu) (*paca[cpu].lppaca_ptr) +#define get_lppaca() (get_paca()->lppaca_ptr) +#endif /* * SLB shadow buffer structure as defined in the PAPR. The save_area @@ -113,46 +134,15 @@ extern struct lppaca lppaca[]; * ESID is stored in the lower 64bits, then the VSID. */ struct slb_shadow { - u32 persistent; /* Number of persistent SLBs */ - u32 buffer_length; /* Total shadow buffer length */ - u64 reserved; + __be32 persistent; /* Number of persistent SLBs */ + __be32 buffer_length; /* Total shadow buffer length */ + __be64 reserved; struct { - u64 esid; - u64 vsid; + __be64 esid; + __be64 vsid; } save_area[SLB_NUM_BOLTED]; } ____cacheline_aligned; -extern struct slb_shadow slb_shadow[]; - -/* - * Layout of entries in the hypervisor's dispatch trace log buffer. - */ -struct dtl_entry { - u8 dispatch_reason; - u8 preempt_reason; - u16 processor_id; - u32 enqueue_to_dispatch_time; - u32 ready_to_enqueue_time; - u32 waiting_to_ready_time; - u64 timebase; - u64 fault_addr; - u64 srr0; - u64 srr1; -}; - -#define DISPATCH_LOG_BYTES 4096 /* bytes per cpu */ -#define N_DISPATCH_LOG (DISPATCH_LOG_BYTES / sizeof(struct dtl_entry)) - -extern struct kmem_cache *dtl_cache; - -/* - * When CONFIG_VIRT_CPU_ACCOUNTING_NATIVE = y, the cpu accounting code controls - * reading from the dispatch trace log. If other code wants to consume - * DTL entries, it can set this pointer to a function that will get - * called once for each DTL entry that gets processed. - */ -extern void (*dtl_consumer)(struct dtl_entry *entry, u64 index); - #endif /* CONFIG_PPC_BOOK3S */ #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_LPPACA_H */ diff --git a/arch/powerpc/include/asm/lv1call.h b/arch/powerpc/include/asm/lv1call.h index f5117674bf92..ae70120953a8 100644 --- a/arch/powerpc/include/asm/lv1call.h +++ b/arch/powerpc/include/asm/lv1call.h @@ -1,28 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * PS3 hvcall interface. * * Copyright (C) 2006 Sony Computer Entertainment Inc. * Copyright 2006 Sony Corp. * Copyright 2003, 2004 (c) MontaVista Software, Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #if !defined(_ASM_POWERPC_LV1CALL_H) #define _ASM_POWERPC_LV1CALL_H -#if !defined(__ASSEMBLY__) +#if !defined(__ASSEMBLER__) #include <linux/types.h> #include <linux/export.h> @@ -223,7 +211,7 @@ {return _lv1_##name(LV1_##in##_IN_##out##_OUT_ARGS);} #endif -#endif /* !defined(__ASSEMBLY__) */ +#endif /* !defined(__ASSEMBLER__) */ /* lv1 call table */ diff --git a/arch/powerpc/include/asm/machdep.h b/arch/powerpc/include/asm/machdep.h index 8b480901165a..3298eec123a3 100644 --- a/arch/powerpc/include/asm/machdep.h +++ b/arch/powerpc/include/asm/machdep.h @@ -1,144 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _ASM_POWERPC_MACHDEP_H #define _ASM_POWERPC_MACHDEP_H #ifdef __KERNEL__ -/* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ - -#include <linux/seq_file.h> +#include <linux/compiler.h> #include <linux/init.h> -#include <linux/dma-mapping.h> #include <linux/export.h> +#include <linux/time64.h> -#include <asm/setup.h> - -/* We export this macro for external modules like Alsa to know if - * ppc_md.feature_call is implemented or not - */ -#define CONFIG_PPC_HAS_FEATURE_CALLS +#include <asm/page.h> struct pt_regs; -struct pci_bus; +struct pci_bus; +struct device; struct device_node; struct iommu_table; struct rtc_time; struct file; +struct pci_dev; struct pci_controller; struct kimage; struct pci_host_bridge; +struct seq_file; struct machdep_calls { - char *name; + const char *name; + const char *compatible; + const char * const *compatibles; #ifdef CONFIG_PPC64 - void (*hpte_invalidate)(unsigned long slot, - unsigned long vpn, - int bpsize, int apsize, - int ssize, int local); - long (*hpte_updatepp)(unsigned long slot, - unsigned long newpp, - unsigned long vpn, - int bpsize, int apsize, - int ssize, int local); - void (*hpte_updateboltedpp)(unsigned long newpp, - unsigned long ea, - int psize, int ssize); - long (*hpte_insert)(unsigned long hpte_group, - unsigned long vpn, - unsigned long prpn, - unsigned long rflags, - unsigned long vflags, - int psize, int apsize, - int ssize); - long (*hpte_remove)(unsigned long hpte_group); - void (*hpte_removebolted)(unsigned long ea, - int psize, int ssize); - void (*flush_hash_range)(unsigned long number, int local); - void (*hugepage_invalidate)(struct mm_struct *mm, - unsigned char *hpte_slot_array, - unsigned long addr, int psize); - - /* special for kexec, to be called in real mode, linear mapping is - * destroyed as well */ - void (*hpte_clear_all)(void); - - int (*tce_build)(struct iommu_table *tbl, - long index, - long npages, - unsigned long uaddr, - enum dma_data_direction direction, - struct dma_attrs *attrs); - void (*tce_free)(struct iommu_table *tbl, - long index, - long npages); - unsigned long (*tce_get)(struct iommu_table *tbl, - long index); - void (*tce_flush)(struct iommu_table *tbl); - - void __iomem * (*ioremap)(phys_addr_t addr, unsigned long size, - unsigned long flags, void *caller); - void (*iounmap)(volatile void __iomem *token); - #ifdef CONFIG_PM - void (*iommu_save)(void); void (*iommu_restore)(void); #endif +#ifdef CONFIG_MEMORY_HOTPLUG + unsigned long (*memory_block_size)(void); +#endif #endif /* CONFIG_PPC64 */ - void (*pci_dma_dev_setup)(struct pci_dev *dev); - void (*pci_dma_bus_setup)(struct pci_bus *bus); - - /* Platform set_dma_mask and dma_get_required_mask overrides */ - int (*dma_set_mask)(struct device *dev, u64 dma_mask); - u64 (*dma_get_required_mask)(struct device *dev); + void (*dma_set_mask)(struct device *dev, u64 dma_mask); int (*probe)(void); void (*setup_arch)(void); /* Optional, may be NULL */ - void (*init_early)(void); /* Optional, may be NULL. */ void (*show_cpuinfo)(struct seq_file *m); - void (*show_percpuinfo)(struct seq_file *m, int i); + /* Returns the current operating frequency of "cpu" in Hz */ + unsigned long (*get_proc_freq)(unsigned int cpu); void (*init_IRQ)(void); - /* Return an irq, or NO_IRQ to indicate there are none pending. */ + /* Return an irq, or 0 to indicate there are none pending. */ unsigned int (*get_irq)(void); /* PCI stuff */ - /* Called after scanning the bus, before allocating resources */ + /* Called after allocating resources */ void (*pcibios_fixup)(void); - int (*pci_probe_mode)(struct pci_bus *); void (*pci_irq_fixup)(struct pci_dev *dev); int (*pcibios_root_bridge_prepare)(struct pci_host_bridge *bridge); + /* finds all the pci_controllers present at boot */ + void (*discover_phbs)(void); + /* To setup PHBs when using automatic OF platform driver for PCI */ int (*pci_setup_phb)(struct pci_controller *host); -#ifdef CONFIG_PCI_MSI - int (*msi_check_device)(struct pci_dev* dev, - int nvec, int type); - int (*setup_msi_irqs)(struct pci_dev *dev, - int nvec, int type); - void (*teardown_msi_irqs)(struct pci_dev *dev); -#endif - - void (*restart)(char *cmd); - void (*power_off)(void); - void (*halt)(void); + void __noreturn (*restart)(char *cmd); + void __noreturn (*halt)(void); void (*panic)(char *str); - void (*cpu_die)(void); long (*time_init)(void); /* Optional, may be NULL */ int (*set_rtc_time)(struct rtc_time *); void (*get_rtc_time)(struct rtc_time *); - unsigned long (*get_boot_time)(void); - unsigned char (*rtc_read_val)(int addr); - void (*rtc_write_val)(int addr, unsigned char val); + time64_t (*get_boot_time)(void); void (*calibrate_decr)(void); @@ -150,13 +83,23 @@ struct machdep_calls { unsigned char (*nvram_read_val)(int addr); void (*nvram_write_val)(int addr, unsigned char val); ssize_t (*nvram_write)(char *buf, size_t count, loff_t *index); - ssize_t (*nvram_read)(char *buf, size_t count, loff_t *index); - ssize_t (*nvram_size)(void); + ssize_t (*nvram_read)(char *buf, size_t count, loff_t *index); + ssize_t (*nvram_size)(void); void (*nvram_sync)(void); /* Exception handlers */ int (*system_reset_exception)(struct pt_regs *regs); int (*machine_check_exception)(struct pt_regs *regs); + int (*handle_hmi_exception)(struct pt_regs *regs); + + /* Early exception handlers called in realmode */ + int (*hmi_exception_early)(struct pt_regs *regs); + long (*machine_check_early)(struct pt_regs *regs); + + /* Called during machine check exception to retrive fixup address. */ + bool (*mce_check_early_recovery)(struct pt_regs *regs); + + void (*machine_check_log_err)(void); /* Motherboard/chipset features. This is a kind of general purpose * hook used to control some machine specific features (like reset @@ -164,12 +107,11 @@ struct machdep_calls { */ long (*feature_call)(unsigned int feature, ...); - /* Get legacy PCI/IDE interrupt mapping */ + /* Get legacy PCI/IDE interrupt mapping */ int (*pci_get_legacy_ide_irq)(struct pci_dev *dev, int channel); - + /* Get access protection for /dev/mem */ - pgprot_t (*phys_mem_access_prot)(struct file *file, - unsigned long pfn, + pgprot_t (*phys_mem_access_prot)(unsigned long pfn, unsigned long size, pgprot_t vma_prot); @@ -183,12 +125,12 @@ struct machdep_calls { platform, called once per cpu. */ void (*enable_pmcs)(void); - /* Set DABR for this platform, leave empty for default implemenation */ + /* Set DABR for this platform, leave empty for default implementation */ int (*set_dabr)(unsigned long dabr, unsigned long dabrx); - /* Set DAWR for this platform, leave empty for default implemenation */ - int (*set_dawr)(unsigned long dawr, + /* Set DAWR for this platform, leave empty for default implementation */ + int (*set_dawr)(int nr, unsigned long dawr, unsigned long dawrx); #ifdef CONFIG_PPC32 /* XXX for now */ @@ -196,8 +138,6 @@ struct machdep_calls { May be NULL. */ void (*init)(void); - void (*kgdb_map_scc)(void); - /* * optional PCI "hooks" */ @@ -216,37 +156,38 @@ struct machdep_calls { /* Called for each PCI bus in the system when it's probed */ void (*pcibios_fixup_bus)(struct pci_bus *); - /* Called when pci_enable_device() is called. Returns 0 to - * allow assignment/enabling of the device. */ - int (*pcibios_enable_device_hook)(struct pci_dev *); - /* Called after scan and before resource survey */ void (*pcibios_fixup_phb)(struct pci_controller *hose); - /* Called during PCI resource reassignment */ - resource_size_t (*pcibios_window_alignment)(struct pci_bus *, unsigned long type); + /* + * Called after device has been added to bus and + * before sysfs has been created. + */ + void (*pcibios_bus_add_device)(struct pci_dev *pdev); + + resource_size_t (*pcibios_default_alignment)(void); + +#ifdef CONFIG_PCI_IOV + void (*pcibios_fixup_sriov)(struct pci_dev *pdev); + resource_size_t (*pcibios_iov_resource_alignment)(struct pci_dev *, int resno); + int (*pcibios_sriov_enable)(struct pci_dev *pdev, u16 num_vfs); + int (*pcibios_sriov_disable)(struct pci_dev *pdev); +#endif /* CONFIG_PCI_IOV */ /* Called to shutdown machine specific hardware not already controlled * by other drivers. */ void (*machine_shutdown)(void); -#ifdef CONFIG_KEXEC +#ifdef CONFIG_KEXEC_CORE void (*kexec_cpu_down)(int crash_shutdown, int secondary); - /* Called to do what every setup is needed on image and the - * reboot code buffer. Returns 0 on success. - * Provide your own (maybe dummy) implementation if your platform - * claims to support kexec. - */ - int (*machine_kexec_prepare)(struct kimage *image); - /* Called to perform the _real_ kexec. * Do NOT allocate memory or fail here. We are past the point of * no return. */ void (*machine_kexec)(struct kimage *image); -#endif /* CONFIG_KEXEC */ +#endif /* CONFIG_KEXEC_CORE */ #ifdef CONFIG_SUSPEND /* These are called to disable and enable, respectively, IRQs when @@ -257,19 +198,18 @@ struct machdep_calls { void (*suspend_disable_irqs)(void); void (*suspend_enable_irqs)(void); #endif - int (*suspend_disable_cpu)(void); #ifdef CONFIG_ARCH_CPU_PROBE_RELEASE ssize_t (*cpu_probe)(const char *, size_t); ssize_t (*cpu_release)(const char *, size_t); #endif + + int (*get_random_seed)(unsigned long *v); }; extern void e500_idle(void); extern void power4_idle(void); -extern void power7_idle(void); extern void ppc6xx_idle(void); -extern void book3e_idle(void); /* * ppc_md contains a copy of the machine description structure for the @@ -279,48 +219,24 @@ extern void book3e_idle(void); extern struct machdep_calls ppc_md; extern struct machdep_calls *machine_id; -#define __machine_desc __attribute__ ((__section__ (".machine.desc"))) +#define __machine_desc __section(".machine.desc") #define define_machine(name) \ extern struct machdep_calls mach_##name; \ EXPORT_SYMBOL(mach_##name); \ struct machdep_calls mach_##name __machine_desc = -#define machine_is(name) \ - ({ \ - extern struct machdep_calls mach_##name \ - __attribute__((weak)); \ - machine_id == &mach_##name; \ - }) - -extern void probe_machine(void); - -extern char cmd_line[COMMAND_LINE_SIZE]; +static inline bool __machine_is(const struct machdep_calls *md) +{ + WARN_ON(!machine_id); // complain if used before probe_machine() + return machine_id == md; +} -#ifdef CONFIG_PPC_PMAC -/* - * Power macintoshes have either a CUDA, PMU or SMU controlling - * system reset, power, NVRAM, RTC. - */ -typedef enum sys_ctrler_kind { - SYS_CTRLER_UNKNOWN = 0, - SYS_CTRLER_CUDA = 1, - SYS_CTRLER_PMU = 2, - SYS_CTRLER_SMU = 3, -} sys_ctrler_t; -extern sys_ctrler_t sys_ctrler; - -#endif /* CONFIG_PPC_PMAC */ - - -/* Functions to produce codes on the leds. - * The SRC code should be unique for the message category and should - * be limited to the lower 24 bits (the upper 8 are set by these funcs), - * and (for boot & dump) should be sorted numerically in the order - * the events occur. - */ -/* Print a boot progress message. */ -void ppc64_boot_msg(unsigned int src, const char *msg); +#define machine_is(name) \ + ({ \ + extern struct machdep_calls mach_##name __weak; \ + __machine_is(&mach_##name); \ + }) static inline void log_error(char *buf, unsigned int err_type, int fatal) { @@ -335,6 +251,7 @@ static inline void log_error(char *buf, unsigned int err_type, int fatal) } \ __define_initcall(__machine_initcall_##mach##_##fn, id); +#define machine_early_initcall(mach, fn) __define_machine_initcall(mach, fn, early) #define machine_core_initcall(mach, fn) __define_machine_initcall(mach, fn, 1) #define machine_core_initcall_sync(mach, fn) __define_machine_initcall(mach, fn, 1s) #define machine_postcore_initcall(mach, fn) __define_machine_initcall(mach, fn, 2) diff --git a/arch/powerpc/include/asm/macio.h b/arch/powerpc/include/asm/macio.h index 27af7f8bbb8d..9203ff6acbf6 100644 --- a/arch/powerpc/include/asm/macio.h +++ b/arch/powerpc/include/asm/macio.h @@ -1,10 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __MACIO_ASIC_H__ #define __MACIO_ASIC_H__ #ifdef __KERNEL__ -#include <linux/of_device.h> +#include <linux/of.h> +#include <linux/platform_device.h> -extern struct bus_type macio_bus_type; +extern const struct bus_type macio_bus_type; /* MacIO device driver is defined later */ struct macio_driver; @@ -124,7 +126,7 @@ static inline struct pci_dev *macio_get_pci_dev(struct macio_dev *mdev) struct macio_driver { int (*probe)(struct macio_dev* dev, const struct of_device_id *match); - int (*remove)(struct macio_dev* dev); + void (*remove)(struct macio_dev *dev); int (*suspend)(struct macio_dev* dev, pm_message_t state); int (*resume)(struct macio_dev* dev); diff --git a/arch/powerpc/include/asm/mc146818rtc.h b/arch/powerpc/include/asm/mc146818rtc.h index f2741c8b59a1..d9e4ecd41009 100644 --- a/arch/powerpc/include/asm/mc146818rtc.h +++ b/arch/powerpc/include/asm/mc146818rtc.h @@ -1,13 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _ASM_POWERPC_MC146818RTC_H #define _ASM_POWERPC_MC146818RTC_H /* * Machine dependent access functions for RTC registers. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifdef __KERNEL__ diff --git a/arch/powerpc/include/asm/mce.h b/arch/powerpc/include/asm/mce.h new file mode 100644 index 000000000000..c9f0936bd3c9 --- /dev/null +++ b/arch/powerpc/include/asm/mce.h @@ -0,0 +1,268 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Machine check exception header file. + * + * Copyright 2013 IBM Corporation + * Author: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com> + */ + +#ifndef __ASM_PPC64_MCE_H__ +#define __ASM_PPC64_MCE_H__ + +#include <linux/bitops.h> + +enum MCE_Version { + MCE_V1 = 1, +}; + +enum MCE_Severity { + MCE_SEV_NO_ERROR = 0, + MCE_SEV_WARNING = 1, + MCE_SEV_SEVERE = 2, + MCE_SEV_FATAL = 3, +}; + +enum MCE_Disposition { + MCE_DISPOSITION_RECOVERED = 0, + MCE_DISPOSITION_NOT_RECOVERED = 1, +}; + +enum MCE_Initiator { + MCE_INITIATOR_UNKNOWN = 0, + MCE_INITIATOR_CPU = 1, + MCE_INITIATOR_PCI = 2, + MCE_INITIATOR_ISA = 3, + MCE_INITIATOR_MEMORY= 4, + MCE_INITIATOR_POWERMGM = 5, +}; + +enum MCE_ErrorType { + MCE_ERROR_TYPE_UNKNOWN = 0, + MCE_ERROR_TYPE_UE = 1, + MCE_ERROR_TYPE_SLB = 2, + MCE_ERROR_TYPE_ERAT = 3, + MCE_ERROR_TYPE_TLB = 4, + MCE_ERROR_TYPE_USER = 5, + MCE_ERROR_TYPE_RA = 6, + MCE_ERROR_TYPE_LINK = 7, + MCE_ERROR_TYPE_DCACHE = 8, + MCE_ERROR_TYPE_ICACHE = 9, +}; + +enum MCE_ErrorClass { + MCE_ECLASS_UNKNOWN = 0, + MCE_ECLASS_HARDWARE, + MCE_ECLASS_HARD_INDETERMINATE, + MCE_ECLASS_SOFTWARE, + MCE_ECLASS_SOFT_INDETERMINATE, +}; + +enum MCE_UeErrorType { + MCE_UE_ERROR_INDETERMINATE = 0, + MCE_UE_ERROR_IFETCH = 1, + MCE_UE_ERROR_PAGE_TABLE_WALK_IFETCH = 2, + MCE_UE_ERROR_LOAD_STORE = 3, + MCE_UE_ERROR_PAGE_TABLE_WALK_LOAD_STORE = 4, +}; + +enum MCE_SlbErrorType { + MCE_SLB_ERROR_INDETERMINATE = 0, + MCE_SLB_ERROR_PARITY = 1, + MCE_SLB_ERROR_MULTIHIT = 2, +}; + +enum MCE_EratErrorType { + MCE_ERAT_ERROR_INDETERMINATE = 0, + MCE_ERAT_ERROR_PARITY = 1, + MCE_ERAT_ERROR_MULTIHIT = 2, +}; + +enum MCE_TlbErrorType { + MCE_TLB_ERROR_INDETERMINATE = 0, + MCE_TLB_ERROR_PARITY = 1, + MCE_TLB_ERROR_MULTIHIT = 2, +}; + +enum MCE_UserErrorType { + MCE_USER_ERROR_INDETERMINATE = 0, + MCE_USER_ERROR_TLBIE = 1, + MCE_USER_ERROR_SCV = 2, +}; + +enum MCE_RaErrorType { + MCE_RA_ERROR_INDETERMINATE = 0, + MCE_RA_ERROR_IFETCH = 1, + MCE_RA_ERROR_IFETCH_FOREIGN = 2, + MCE_RA_ERROR_PAGE_TABLE_WALK_IFETCH = 3, + MCE_RA_ERROR_PAGE_TABLE_WALK_IFETCH_FOREIGN = 4, + MCE_RA_ERROR_LOAD = 5, + MCE_RA_ERROR_STORE = 6, + MCE_RA_ERROR_PAGE_TABLE_WALK_LOAD_STORE = 7, + MCE_RA_ERROR_PAGE_TABLE_WALK_LOAD_STORE_FOREIGN = 8, + MCE_RA_ERROR_LOAD_STORE_FOREIGN = 9, +}; + +enum MCE_LinkErrorType { + MCE_LINK_ERROR_INDETERMINATE = 0, + MCE_LINK_ERROR_IFETCH_TIMEOUT = 1, + MCE_LINK_ERROR_PAGE_TABLE_WALK_IFETCH_TIMEOUT = 2, + MCE_LINK_ERROR_LOAD_TIMEOUT = 3, + MCE_LINK_ERROR_STORE_TIMEOUT = 4, + MCE_LINK_ERROR_PAGE_TABLE_WALK_LOAD_STORE_TIMEOUT = 5, +}; + +struct machine_check_event { + enum MCE_Version version:8; + u8 in_use; + enum MCE_Severity severity:8; + enum MCE_Initiator initiator:8; + enum MCE_ErrorType error_type:8; + enum MCE_ErrorClass error_class:8; + enum MCE_Disposition disposition:8; + bool sync_error; + u16 cpu; + u64 gpr3; + u64 srr0; + u64 srr1; + union { + struct { + enum MCE_UeErrorType ue_error_type:8; + u8 effective_address_provided; + u8 physical_address_provided; + u8 ignore_event; + u8 reserved_1[4]; + u64 effective_address; + u64 physical_address; + u8 reserved_2[8]; + } ue_error; + + struct { + enum MCE_SlbErrorType slb_error_type:8; + u8 effective_address_provided; + u8 reserved_1[6]; + u64 effective_address; + u8 reserved_2[16]; + } slb_error; + + struct { + enum MCE_EratErrorType erat_error_type:8; + u8 effective_address_provided; + u8 reserved_1[6]; + u64 effective_address; + u8 reserved_2[16]; + } erat_error; + + struct { + enum MCE_TlbErrorType tlb_error_type:8; + u8 effective_address_provided; + u8 reserved_1[6]; + u64 effective_address; + u8 reserved_2[16]; + } tlb_error; + + struct { + enum MCE_UserErrorType user_error_type:8; + u8 effective_address_provided; + u8 reserved_1[6]; + u64 effective_address; + u8 reserved_2[16]; + } user_error; + + struct { + enum MCE_RaErrorType ra_error_type:8; + u8 effective_address_provided; + u8 reserved_1[6]; + u64 effective_address; + u8 reserved_2[16]; + } ra_error; + + struct { + enum MCE_LinkErrorType link_error_type:8; + u8 effective_address_provided; + u8 reserved_1[6]; + u64 effective_address; + u8 reserved_2[16]; + } link_error; + } u; +}; + +struct mce_error_info { + enum MCE_ErrorType error_type:8; + union { + enum MCE_UeErrorType ue_error_type:8; + enum MCE_SlbErrorType slb_error_type:8; + enum MCE_EratErrorType erat_error_type:8; + enum MCE_TlbErrorType tlb_error_type:8; + enum MCE_UserErrorType user_error_type:8; + enum MCE_RaErrorType ra_error_type:8; + enum MCE_LinkErrorType link_error_type:8; + } u; + enum MCE_Severity severity:8; + enum MCE_Initiator initiator:8; + enum MCE_ErrorClass error_class:8; + bool sync_error; + bool ignore_event; +}; + +#define MAX_MC_EVT 10 + +struct mce_info { + int mce_nest_count; + struct machine_check_event mce_event[MAX_MC_EVT]; + /* Queue for delayed MCE events. */ + int mce_queue_count; + struct machine_check_event mce_event_queue[MAX_MC_EVT]; + /* Queue for delayed MCE UE events. */ + int mce_ue_count; + struct machine_check_event mce_ue_event_queue[MAX_MC_EVT]; +}; + +/* Release flags for get_mce_event() */ +#define MCE_EVENT_RELEASE true +#define MCE_EVENT_DONTRELEASE false + +struct pt_regs; +struct notifier_block; + +extern void save_mce_event(struct pt_regs *regs, long handled, + struct mce_error_info *mce_err, uint64_t nip, + uint64_t addr, uint64_t phys_addr); +extern int get_mce_event(struct machine_check_event *mce, bool release); +extern void release_mce_event(void); +extern void machine_check_queue_event(void); +extern void machine_check_print_event_info(struct machine_check_event *evt, + bool user_mode, bool in_guest); +unsigned long addr_to_pfn(struct pt_regs *regs, unsigned long addr); +extern void mce_common_process_ue(struct pt_regs *regs, + struct mce_error_info *mce_err); +void mce_irq_work_queue(void); +int mce_register_notifier(struct notifier_block *nb); +int mce_unregister_notifier(struct notifier_block *nb); + +#ifdef CONFIG_PPC_BOOK3S_64 +void mce_run_irq_context_handlers(void); +#else +static inline void mce_run_irq_context_handlers(void) { }; +#endif /* CONFIG_PPC_BOOK3S_64 */ + +#ifdef CONFIG_PPC_BOOK3S_64 +void set_mce_pending_irq_work(void); +void clear_mce_pending_irq_work(void); +#endif /* CONFIG_PPC_BOOK3S_64 */ + +#ifdef CONFIG_PPC_BOOK3S_64 +void flush_and_reload_slb(void); +void flush_erat(void); +long __machine_check_early_realmode_p7(struct pt_regs *regs); +long __machine_check_early_realmode_p8(struct pt_regs *regs); +long __machine_check_early_realmode_p9(struct pt_regs *regs); +long __machine_check_early_realmode_p10(struct pt_regs *regs); +#endif /* CONFIG_PPC_BOOK3S_64 */ + +#ifdef CONFIG_PPC_BOOK3S_64 +void mce_init(void); +#else +static inline void mce_init(void) { }; +#endif /* CONFIG_PPC_BOOK3S_64 */ + +#endif /* __ASM_PPC64_MCE_H__ */ diff --git a/arch/powerpc/include/asm/mediabay.h b/arch/powerpc/include/asm/mediabay.h index 11037a4133ee..230fda4707b8 100644 --- a/arch/powerpc/include/asm/mediabay.h +++ b/arch/powerpc/include/asm/mediabay.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * mediabay.h: definitions for using the media bay * on PowerBook 3400 and similar computers. diff --git a/arch/powerpc/include/asm/mem_encrypt.h b/arch/powerpc/include/asm/mem_encrypt.h new file mode 100644 index 000000000000..e355ca46fad9 --- /dev/null +++ b/arch/powerpc/include/asm/mem_encrypt.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * SVM helper functions + * + * Copyright 2018 IBM Corporation + */ + +#ifndef _ASM_POWERPC_MEM_ENCRYPT_H +#define _ASM_POWERPC_MEM_ENCRYPT_H + +#include <asm/svm.h> +#include <linux/types.h> + +struct device; + +static inline bool force_dma_unencrypted(struct device *dev) +{ + return is_secure_guest(); +} + +int set_memory_encrypted(unsigned long addr, int numpages); +int set_memory_decrypted(unsigned long addr, int numpages); + +#endif /* _ASM_POWERPC_MEM_ENCRYPT_H */ diff --git a/arch/powerpc/include/asm/membarrier.h b/arch/powerpc/include/asm/membarrier.h new file mode 100644 index 000000000000..de7f79157918 --- /dev/null +++ b/arch/powerpc/include/asm/membarrier.h @@ -0,0 +1,28 @@ +#ifndef _ASM_POWERPC_MEMBARRIER_H +#define _ASM_POWERPC_MEMBARRIER_H + +static inline void membarrier_arch_switch_mm(struct mm_struct *prev, + struct mm_struct *next, + struct task_struct *tsk) +{ + /* + * Only need the full barrier when switching between processes. + * Barrier when switching from kernel to userspace is not + * required here, given that it is implied by mmdrop(). Barrier + * when switching from userspace to kernel is not needed after + * store to rq->curr. + */ + if (IS_ENABLED(CONFIG_SMP) && + likely(!(atomic_read(&next->membarrier_state) & + (MEMBARRIER_STATE_PRIVATE_EXPEDITED | + MEMBARRIER_STATE_GLOBAL_EXPEDITED)) || !prev)) + return; + + /* + * The membarrier system call requires a full memory barrier + * after storing to rq->curr, before going back to user-space. + */ + smp_mb(); +} + +#endif /* _ASM_POWERPC_MEMBARRIER_H */ diff --git a/arch/powerpc/include/asm/mman.h b/arch/powerpc/include/asm/mman.h index 8565c254151a..912f78a956a1 100644 --- a/arch/powerpc/include/asm/mman.h +++ b/arch/powerpc/include/asm/mman.h @@ -1,44 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef _ASM_POWERPC_MMAN_H #define _ASM_POWERPC_MMAN_H #include <uapi/asm/mman.h> -#ifdef CONFIG_PPC64 +#if defined(CONFIG_PPC64) && !defined(BUILD_VDSO) #include <asm/cputable.h> #include <linux/mm.h> +#include <linux/pkeys.h> +#include <asm/cpu_has_feature.h> +#include <asm/firmware.h> -/* - * This file is included by linux/mman.h, so we can't use cacl_vm_prot_bits() - * here. How important is the optimization? - */ -static inline unsigned long arch_calc_vm_prot_bits(unsigned long prot) -{ - return (prot & PROT_SAO) ? VM_SAO : 0; -} -#define arch_calc_vm_prot_bits(prot) arch_calc_vm_prot_bits(prot) - -static inline pgprot_t arch_vm_get_page_prot(unsigned long vm_flags) +static inline vm_flags_t arch_calc_vm_prot_bits(unsigned long prot, + unsigned long pkey) { - return (vm_flags & VM_SAO) ? __pgprot(_PAGE_SAO) : __pgprot(0); +#ifdef CONFIG_PPC_MEM_KEYS + return (((prot & PROT_SAO) ? VM_SAO : 0) | pkey_to_vmflag_bits(pkey)); +#else + return ((prot & PROT_SAO) ? VM_SAO : 0); +#endif } -#define arch_vm_get_page_prot(vm_flags) arch_vm_get_page_prot(vm_flags) +#define arch_calc_vm_prot_bits(prot, pkey) arch_calc_vm_prot_bits(prot, pkey) -static inline int arch_validate_prot(unsigned long prot) +static inline bool arch_validate_prot(unsigned long prot, unsigned long addr) { if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC | PROT_SEM | PROT_SAO)) - return 0; - if ((prot & PROT_SAO) && !cpu_has_feature(CPU_FTR_SAO)) - return 0; - return 1; + return false; + if (prot & PROT_SAO) { + if (!cpu_has_feature(CPU_FTR_SAO)) + return false; + if (firmware_has_feature(FW_FEATURE_LPAR) && + !IS_ENABLED(CONFIG_PPC_PROT_SAO_LPAR)) + return false; + } + return true; } -#define arch_validate_prot(prot) arch_validate_prot(prot) +#define arch_validate_prot arch_validate_prot #endif /* CONFIG_PPC64 */ #endif /* _ASM_POWERPC_MMAN_H */ diff --git a/arch/powerpc/include/asm/mmiowb.h b/arch/powerpc/include/asm/mmiowb.h new file mode 100644 index 000000000000..74a00127eb20 --- /dev/null +++ b/arch/powerpc/include/asm/mmiowb.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_MMIOWB_H +#define _ASM_POWERPC_MMIOWB_H + +#ifdef CONFIG_MMIOWB + +#include <linux/compiler.h> +#include <asm/barrier.h> +#include <asm/paca.h> + +#define arch_mmiowb_state() (&local_paca->mmiowb_state) +#define mmiowb() mb() + +#endif /* CONFIG_MMIOWB */ + +#include <asm-generic/mmiowb.h> + +#endif /* _ASM_POWERPC_MMIOWB_H */ diff --git a/arch/powerpc/include/asm/mmu-40x.h b/arch/powerpc/include/asm/mmu-40x.h deleted file mode 100644 index 34916865eaef..000000000000 --- a/arch/powerpc/include/asm/mmu-40x.h +++ /dev/null @@ -1,67 +0,0 @@ -#ifndef _ASM_POWERPC_MMU_40X_H_ -#define _ASM_POWERPC_MMU_40X_H_ - -/* - * PPC40x support - */ - -#define PPC40X_TLB_SIZE 64 - -/* - * TLB entries are defined by a "high" tag portion and a "low" data - * portion. On all architectures, the data portion is 32-bits. - * - * TLB entries are managed entirely under software control by reading, - * writing, and searchoing using the 4xx-specific tlbre, tlbwr, and tlbsx - * instructions. - */ - -#define TLB_LO 1 -#define TLB_HI 0 - -#define TLB_DATA TLB_LO -#define TLB_TAG TLB_HI - -/* Tag portion */ - -#define TLB_EPN_MASK 0xFFFFFC00 /* Effective Page Number */ -#define TLB_PAGESZ_MASK 0x00000380 -#define TLB_PAGESZ(x) (((x) & 0x7) << 7) -#define PAGESZ_1K 0 -#define PAGESZ_4K 1 -#define PAGESZ_16K 2 -#define PAGESZ_64K 3 -#define PAGESZ_256K 4 -#define PAGESZ_1M 5 -#define PAGESZ_4M 6 -#define PAGESZ_16M 7 -#define TLB_VALID 0x00000040 /* Entry is valid */ - -/* Data portion */ - -#define TLB_RPN_MASK 0xFFFFFC00 /* Real Page Number */ -#define TLB_PERM_MASK 0x00000300 -#define TLB_EX 0x00000200 /* Instruction execution allowed */ -#define TLB_WR 0x00000100 /* Writes permitted */ -#define TLB_ZSEL_MASK 0x000000F0 -#define TLB_ZSEL(x) (((x) & 0xF) << 4) -#define TLB_ATTR_MASK 0x0000000F -#define TLB_W 0x00000008 /* Caching is write-through */ -#define TLB_I 0x00000004 /* Caching is inhibited */ -#define TLB_M 0x00000002 /* Memory is coherent */ -#define TLB_G 0x00000001 /* Memory is guarded from prefetch */ - -#ifndef __ASSEMBLY__ - -typedef struct { - unsigned int id; - unsigned int active; - unsigned long vdso_base; -} mm_context_t; - -#endif /* !__ASSEMBLY__ */ - -#define mmu_virtual_psize MMU_PAGE_4K -#define mmu_linear_psize MMU_PAGE_256M - -#endif /* _ASM_POWERPC_MMU_40X_H_ */ diff --git a/arch/powerpc/include/asm/mmu-hash32.h b/arch/powerpc/include/asm/mmu-hash32.h deleted file mode 100644 index 16f513e5cbd7..000000000000 --- a/arch/powerpc/include/asm/mmu-hash32.h +++ /dev/null @@ -1,93 +0,0 @@ -#ifndef _ASM_POWERPC_MMU_HASH32_H_ -#define _ASM_POWERPC_MMU_HASH32_H_ -/* - * 32-bit hash table MMU support - */ - -/* - * BATs - */ - -/* Block size masks */ -#define BL_128K 0x000 -#define BL_256K 0x001 -#define BL_512K 0x003 -#define BL_1M 0x007 -#define BL_2M 0x00F -#define BL_4M 0x01F -#define BL_8M 0x03F -#define BL_16M 0x07F -#define BL_32M 0x0FF -#define BL_64M 0x1FF -#define BL_128M 0x3FF -#define BL_256M 0x7FF - -/* BAT Access Protection */ -#define BPP_XX 0x00 /* No access */ -#define BPP_RX 0x01 /* Read only */ -#define BPP_RW 0x02 /* Read/write */ - -#ifndef __ASSEMBLY__ -/* Contort a phys_addr_t into the right format/bits for a BAT */ -#ifdef CONFIG_PHYS_64BIT -#define BAT_PHYS_ADDR(x) ((u32)((x & 0x00000000fffe0000ULL) | \ - ((x & 0x0000000e00000000ULL) >> 24) | \ - ((x & 0x0000000100000000ULL) >> 30))) -#else -#define BAT_PHYS_ADDR(x) (x) -#endif - -struct ppc_bat { - u32 batu; - u32 batl; -}; -#endif /* !__ASSEMBLY__ */ - -/* - * Hash table - */ - -/* Values for PP (assumes Ks=0, Kp=1) */ -#define PP_RWXX 0 /* Supervisor read/write, User none */ -#define PP_RWRX 1 /* Supervisor read/write, User read */ -#define PP_RWRW 2 /* Supervisor read/write, User read/write */ -#define PP_RXRX 3 /* Supervisor read, User read */ - -#ifndef __ASSEMBLY__ - -/* - * Hardware Page Table Entry - * Note that the xpn and x bitfields are used only by processors that - * support extended addressing; otherwise, those bits are reserved. - */ -struct hash_pte { - unsigned long v:1; /* Entry is valid */ - unsigned long vsid:24; /* Virtual segment identifier */ - unsigned long h:1; /* Hash algorithm indicator */ - unsigned long api:6; /* Abbreviated page index */ - unsigned long rpn:20; /* Real (physical) page number */ - unsigned long xpn:3; /* Real page number bits 0-2, optional */ - unsigned long r:1; /* Referenced */ - unsigned long c:1; /* Changed */ - unsigned long w:1; /* Write-thru cache mode */ - unsigned long i:1; /* Cache inhibited */ - unsigned long m:1; /* Memory coherence */ - unsigned long g:1; /* Guarded */ - unsigned long x:1; /* Real page number bit 3, optional */ - unsigned long pp:2; /* Page protection */ -}; - -typedef struct { - unsigned long id; - unsigned long vdso_base; -} mm_context_t; - -#endif /* !__ASSEMBLY__ */ - -/* We happily ignore the smaller BATs on 601, we don't actually use - * those definitions on hash32 at the moment anyway - */ -#define mmu_virtual_psize MMU_PAGE_4K -#define mmu_linear_psize MMU_PAGE_256M - -#endif /* _ASM_POWERPC_MMU_HASH32_H_ */ diff --git a/arch/powerpc/include/asm/mmu-hash64.h b/arch/powerpc/include/asm/mmu-hash64.h deleted file mode 100644 index c4cf01197273..000000000000 --- a/arch/powerpc/include/asm/mmu-hash64.h +++ /dev/null @@ -1,612 +0,0 @@ -#ifndef _ASM_POWERPC_MMU_HASH64_H_ -#define _ASM_POWERPC_MMU_HASH64_H_ -/* - * PowerPC64 memory management structures - * - * Dave Engebretsen & Mike Corrigan <{engebret|mikejc}@us.ibm.com> - * PPC64 rework. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ - -#include <asm/asm-compat.h> -#include <asm/page.h> - -/* - * This is necessary to get the definition of PGTABLE_RANGE which we - * need for various slices related matters. Note that this isn't the - * complete pgtable.h but only a portion of it. - */ -#include <asm/pgtable-ppc64.h> -#include <asm/bug.h> - -/* - * Segment table - */ - -#define STE_ESID_V 0x80 -#define STE_ESID_KS 0x20 -#define STE_ESID_KP 0x10 -#define STE_ESID_N 0x08 - -#define STE_VSID_SHIFT 12 - -/* Location of cpu0's segment table */ -#define STAB0_PAGE 0x8 -#define STAB0_OFFSET (STAB0_PAGE << 12) -#define STAB0_PHYS_ADDR (STAB0_OFFSET + PHYSICAL_START) - -#ifndef __ASSEMBLY__ -extern char initial_stab[]; -#endif /* ! __ASSEMBLY */ - -/* - * SLB - */ - -#define SLB_NUM_BOLTED 3 -#define SLB_CACHE_ENTRIES 8 -#define SLB_MIN_SIZE 32 - -/* Bits in the SLB ESID word */ -#define SLB_ESID_V ASM_CONST(0x0000000008000000) /* valid */ - -/* Bits in the SLB VSID word */ -#define SLB_VSID_SHIFT 12 -#define SLB_VSID_SHIFT_1T 24 -#define SLB_VSID_SSIZE_SHIFT 62 -#define SLB_VSID_B ASM_CONST(0xc000000000000000) -#define SLB_VSID_B_256M ASM_CONST(0x0000000000000000) -#define SLB_VSID_B_1T ASM_CONST(0x4000000000000000) -#define SLB_VSID_KS ASM_CONST(0x0000000000000800) -#define SLB_VSID_KP ASM_CONST(0x0000000000000400) -#define SLB_VSID_N ASM_CONST(0x0000000000000200) /* no-execute */ -#define SLB_VSID_L ASM_CONST(0x0000000000000100) -#define SLB_VSID_C ASM_CONST(0x0000000000000080) /* class */ -#define SLB_VSID_LP ASM_CONST(0x0000000000000030) -#define SLB_VSID_LP_00 ASM_CONST(0x0000000000000000) -#define SLB_VSID_LP_01 ASM_CONST(0x0000000000000010) -#define SLB_VSID_LP_10 ASM_CONST(0x0000000000000020) -#define SLB_VSID_LP_11 ASM_CONST(0x0000000000000030) -#define SLB_VSID_LLP (SLB_VSID_L|SLB_VSID_LP) - -#define SLB_VSID_KERNEL (SLB_VSID_KP) -#define SLB_VSID_USER (SLB_VSID_KP|SLB_VSID_KS|SLB_VSID_C) - -#define SLBIE_C (0x08000000) -#define SLBIE_SSIZE_SHIFT 25 - -/* - * Hash table - */ - -#define HPTES_PER_GROUP 8 - -#define HPTE_V_SSIZE_SHIFT 62 -#define HPTE_V_AVPN_SHIFT 7 -#define HPTE_V_AVPN ASM_CONST(0x3fffffffffffff80) -#define HPTE_V_AVPN_VAL(x) (((x) & HPTE_V_AVPN) >> HPTE_V_AVPN_SHIFT) -#define HPTE_V_COMPARE(x,y) (!(((x) ^ (y)) & 0xffffffffffffff80UL)) -#define HPTE_V_BOLTED ASM_CONST(0x0000000000000010) -#define HPTE_V_LOCK ASM_CONST(0x0000000000000008) -#define HPTE_V_LARGE ASM_CONST(0x0000000000000004) -#define HPTE_V_SECONDARY ASM_CONST(0x0000000000000002) -#define HPTE_V_VALID ASM_CONST(0x0000000000000001) - -#define HPTE_R_PP0 ASM_CONST(0x8000000000000000) -#define HPTE_R_TS ASM_CONST(0x4000000000000000) -#define HPTE_R_KEY_HI ASM_CONST(0x3000000000000000) -#define HPTE_R_RPN_SHIFT 12 -#define HPTE_R_RPN ASM_CONST(0x0ffffffffffff000) -#define HPTE_R_PP ASM_CONST(0x0000000000000003) -#define HPTE_R_N ASM_CONST(0x0000000000000004) -#define HPTE_R_G ASM_CONST(0x0000000000000008) -#define HPTE_R_M ASM_CONST(0x0000000000000010) -#define HPTE_R_I ASM_CONST(0x0000000000000020) -#define HPTE_R_W ASM_CONST(0x0000000000000040) -#define HPTE_R_WIMG ASM_CONST(0x0000000000000078) -#define HPTE_R_C ASM_CONST(0x0000000000000080) -#define HPTE_R_R ASM_CONST(0x0000000000000100) -#define HPTE_R_KEY_LO ASM_CONST(0x0000000000000e00) - -#define HPTE_V_1TB_SEG ASM_CONST(0x4000000000000000) -#define HPTE_V_VRMA_MASK ASM_CONST(0x4001ffffff000000) - -/* Values for PP (assumes Ks=0, Kp=1) */ -#define PP_RWXX 0 /* Supervisor read/write, User none */ -#define PP_RWRX 1 /* Supervisor read/write, User read */ -#define PP_RWRW 2 /* Supervisor read/write, User read/write */ -#define PP_RXRX 3 /* Supervisor read, User read */ -#define PP_RXXX (HPTE_R_PP0 | 2) /* Supervisor read, user none */ - -/* Fields for tlbiel instruction in architecture 2.06 */ -#define TLBIEL_INVAL_SEL_MASK 0xc00 /* invalidation selector */ -#define TLBIEL_INVAL_PAGE 0x000 /* invalidate a single page */ -#define TLBIEL_INVAL_SET_LPID 0x800 /* invalidate a set for current LPID */ -#define TLBIEL_INVAL_SET 0xc00 /* invalidate a set for all LPIDs */ -#define TLBIEL_INVAL_SET_MASK 0xfff000 /* set number to inval. */ -#define TLBIEL_INVAL_SET_SHIFT 12 - -#define POWER7_TLB_SETS 128 /* # sets in POWER7 TLB */ - -#ifndef __ASSEMBLY__ - -struct hash_pte { - unsigned long v; - unsigned long r; -}; - -extern struct hash_pte *htab_address; -extern unsigned long htab_size_bytes; -extern unsigned long htab_hash_mask; - -/* - * Page size definition - * - * shift : is the "PAGE_SHIFT" value for that page size - * sllp : is a bit mask with the value of SLB L || LP to be or'ed - * directly to a slbmte "vsid" value - * penc : is the HPTE encoding mask for the "LP" field: - * - */ -struct mmu_psize_def -{ - unsigned int shift; /* number of bits */ - int penc[MMU_PAGE_COUNT]; /* HPTE encoding */ - unsigned int tlbiel; /* tlbiel supported for that page size */ - unsigned long avpnm; /* bits to mask out in AVPN in the HPTE */ - unsigned long sllp; /* SLB L||LP (exact mask to use in slbmte) */ -}; -extern struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT]; - -static inline int shift_to_mmu_psize(unsigned int shift) -{ - int psize; - - for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) - if (mmu_psize_defs[psize].shift == shift) - return psize; - return -1; -} - -static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize) -{ - if (mmu_psize_defs[mmu_psize].shift) - return mmu_psize_defs[mmu_psize].shift; - BUG(); -} - -#endif /* __ASSEMBLY__ */ - -/* - * Segment sizes. - * These are the values used by hardware in the B field of - * SLB entries and the first dword of MMU hashtable entries. - * The B field is 2 bits; the values 2 and 3 are unused and reserved. - */ -#define MMU_SEGSIZE_256M 0 -#define MMU_SEGSIZE_1T 1 - -/* - * encode page number shift. - * in order to fit the 78 bit va in a 64 bit variable we shift the va by - * 12 bits. This enable us to address upto 76 bit va. - * For hpt hash from a va we can ignore the page size bits of va and for - * hpte encoding we ignore up to 23 bits of va. So ignoring lower 12 bits ensure - * we work in all cases including 4k page size. - */ -#define VPN_SHIFT 12 - -/* - * HPTE Large Page (LP) details - */ -#define LP_SHIFT 12 -#define LP_BITS 8 -#define LP_MASK(i) ((0xFF >> (i)) << LP_SHIFT) - -#ifndef __ASSEMBLY__ - -static inline int segment_shift(int ssize) -{ - if (ssize == MMU_SEGSIZE_256M) - return SID_SHIFT; - return SID_SHIFT_1T; -} - -/* - * The current system page and segment sizes - */ -extern int mmu_linear_psize; -extern int mmu_virtual_psize; -extern int mmu_vmalloc_psize; -extern int mmu_vmemmap_psize; -extern int mmu_io_psize; -extern int mmu_kernel_ssize; -extern int mmu_highuser_ssize; -extern u16 mmu_slb_size; -extern unsigned long tce_alloc_start, tce_alloc_end; - -/* - * If the processor supports 64k normal pages but not 64k cache - * inhibited pages, we have to be prepared to switch processes - * to use 4k pages when they create cache-inhibited mappings. - * If this is the case, mmu_ci_restrictions will be set to 1. - */ -extern int mmu_ci_restrictions; - -/* - * This computes the AVPN and B fields of the first dword of a HPTE, - * for use when we want to match an existing PTE. The bottom 7 bits - * of the returned value are zero. - */ -static inline unsigned long hpte_encode_avpn(unsigned long vpn, int psize, - int ssize) -{ - unsigned long v; - /* - * The AVA field omits the low-order 23 bits of the 78 bits VA. - * These bits are not needed in the PTE, because the - * low-order b of these bits are part of the byte offset - * into the virtual page and, if b < 23, the high-order - * 23-b of these bits are always used in selecting the - * PTEGs to be searched - */ - v = (vpn >> (23 - VPN_SHIFT)) & ~(mmu_psize_defs[psize].avpnm); - v <<= HPTE_V_AVPN_SHIFT; - v |= ((unsigned long) ssize) << HPTE_V_SSIZE_SHIFT; - return v; -} - -/* - * This function sets the AVPN and L fields of the HPTE appropriately - * using the base page size and actual page size. - */ -static inline unsigned long hpte_encode_v(unsigned long vpn, int base_psize, - int actual_psize, int ssize) -{ - unsigned long v; - v = hpte_encode_avpn(vpn, base_psize, ssize); - if (actual_psize != MMU_PAGE_4K) - v |= HPTE_V_LARGE; - return v; -} - -/* - * This function sets the ARPN, and LP fields of the HPTE appropriately - * for the page size. We assume the pa is already "clean" that is properly - * aligned for the requested page size - */ -static inline unsigned long hpte_encode_r(unsigned long pa, int base_psize, - int actual_psize) -{ - /* A 4K page needs no special encoding */ - if (actual_psize == MMU_PAGE_4K) - return pa & HPTE_R_RPN; - else { - unsigned int penc = mmu_psize_defs[base_psize].penc[actual_psize]; - unsigned int shift = mmu_psize_defs[actual_psize].shift; - return (pa & ~((1ul << shift) - 1)) | (penc << LP_SHIFT); - } -} - -/* - * Build a VPN_SHIFT bit shifted va given VSID, EA and segment size. - */ -static inline unsigned long hpt_vpn(unsigned long ea, - unsigned long vsid, int ssize) -{ - unsigned long mask; - int s_shift = segment_shift(ssize); - - mask = (1ul << (s_shift - VPN_SHIFT)) - 1; - return (vsid << (s_shift - VPN_SHIFT)) | ((ea >> VPN_SHIFT) & mask); -} - -/* - * This hashes a virtual address - */ -static inline unsigned long hpt_hash(unsigned long vpn, - unsigned int shift, int ssize) -{ - int mask; - unsigned long hash, vsid; - - /* VPN_SHIFT can be atmost 12 */ - if (ssize == MMU_SEGSIZE_256M) { - mask = (1ul << (SID_SHIFT - VPN_SHIFT)) - 1; - hash = (vpn >> (SID_SHIFT - VPN_SHIFT)) ^ - ((vpn & mask) >> (shift - VPN_SHIFT)); - } else { - mask = (1ul << (SID_SHIFT_1T - VPN_SHIFT)) - 1; - vsid = vpn >> (SID_SHIFT_1T - VPN_SHIFT); - hash = vsid ^ (vsid << 25) ^ - ((vpn & mask) >> (shift - VPN_SHIFT)) ; - } - return hash & 0x7fffffffffUL; -} - -extern int __hash_page_4K(unsigned long ea, unsigned long access, - unsigned long vsid, pte_t *ptep, unsigned long trap, - unsigned int local, int ssize, int subpage_prot); -extern int __hash_page_64K(unsigned long ea, unsigned long access, - unsigned long vsid, pte_t *ptep, unsigned long trap, - unsigned int local, int ssize); -struct mm_struct; -unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap); -extern int hash_page(unsigned long ea, unsigned long access, unsigned long trap); -int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid, - pte_t *ptep, unsigned long trap, int local, int ssize, - unsigned int shift, unsigned int mmu_psize); -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -extern int __hash_page_thp(unsigned long ea, unsigned long access, - unsigned long vsid, pmd_t *pmdp, unsigned long trap, - int local, int ssize, unsigned int psize); -#else -static inline int __hash_page_thp(unsigned long ea, unsigned long access, - unsigned long vsid, pmd_t *pmdp, - unsigned long trap, int local, - int ssize, unsigned int psize) -{ - BUG(); - return -1; -} -#endif -extern void hash_failure_debug(unsigned long ea, unsigned long access, - unsigned long vsid, unsigned long trap, - int ssize, int psize, int lpsize, - unsigned long pte); -extern int htab_bolt_mapping(unsigned long vstart, unsigned long vend, - unsigned long pstart, unsigned long prot, - int psize, int ssize); -extern void add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages); -extern void demote_segment_4k(struct mm_struct *mm, unsigned long addr); - -extern void hpte_init_native(void); -extern void hpte_init_lpar(void); -extern void hpte_init_beat(void); -extern void hpte_init_beat_v3(void); - -extern void stabs_alloc(void); -extern void slb_initialize(void); -extern void slb_flush_and_rebolt(void); -extern void stab_initialize(unsigned long stab); - -extern void slb_vmalloc_update(void); -extern void slb_set_size(u16 size); -#endif /* __ASSEMBLY__ */ - -/* - * VSID allocation (256MB segment) - * - * We first generate a 37-bit "proto-VSID". Proto-VSIDs are generated - * from mmu context id and effective segment id of the address. - * - * For user processes max context id is limited to ((1ul << 19) - 5) - * for kernel space, we use the top 4 context ids to map address as below - * NOTE: each context only support 64TB now. - * 0x7fffc - [ 0xc000000000000000 - 0xc0003fffffffffff ] - * 0x7fffd - [ 0xd000000000000000 - 0xd0003fffffffffff ] - * 0x7fffe - [ 0xe000000000000000 - 0xe0003fffffffffff ] - * 0x7ffff - [ 0xf000000000000000 - 0xf0003fffffffffff ] - * - * The proto-VSIDs are then scrambled into real VSIDs with the - * multiplicative hash: - * - * VSID = (proto-VSID * VSID_MULTIPLIER) % VSID_MODULUS - * - * VSID_MULTIPLIER is prime, so in particular it is - * co-prime to VSID_MODULUS, making this a 1:1 scrambling function. - * Because the modulus is 2^n-1 we can compute it efficiently without - * a divide or extra multiply (see below). The scramble function gives - * robust scattering in the hash table (at least based on some initial - * results). - * - * We also consider VSID 0 special. We use VSID 0 for slb entries mapping - * bad address. This enables us to consolidate bad address handling in - * hash_page. - * - * We also need to avoid the last segment of the last context, because that - * would give a protovsid of 0x1fffffffff. That will result in a VSID 0 - * because of the modulo operation in vsid scramble. But the vmemmap - * (which is what uses region 0xf) will never be close to 64TB in size - * (it's 56 bytes per page of system memory). - */ - -#define CONTEXT_BITS 19 -#define ESID_BITS 18 -#define ESID_BITS_1T 6 - -/* - * 256MB segment - * The proto-VSID space has 2^(CONTEX_BITS + ESID_BITS) - 1 segments - * available for user + kernel mapping. The top 4 contexts are used for - * kernel mapping. Each segment contains 2^28 bytes. Each - * context maps 2^46 bytes (64TB) so we can support 2^19-1 contexts - * (19 == 37 + 28 - 46). - */ -#define MAX_USER_CONTEXT ((ASM_CONST(1) << CONTEXT_BITS) - 5) - -/* - * This should be computed such that protovosid * vsid_mulitplier - * doesn't overflow 64 bits. It should also be co-prime to vsid_modulus - */ -#define VSID_MULTIPLIER_256M ASM_CONST(12538073) /* 24-bit prime */ -#define VSID_BITS_256M (CONTEXT_BITS + ESID_BITS) -#define VSID_MODULUS_256M ((1UL<<VSID_BITS_256M)-1) - -#define VSID_MULTIPLIER_1T ASM_CONST(12538073) /* 24-bit prime */ -#define VSID_BITS_1T (CONTEXT_BITS + ESID_BITS_1T) -#define VSID_MODULUS_1T ((1UL<<VSID_BITS_1T)-1) - - -#define USER_VSID_RANGE (1UL << (ESID_BITS + SID_SHIFT)) - -/* - * This macro generates asm code to compute the VSID scramble - * function. Used in slb_allocate() and do_stab_bolted. The function - * computed is: (protovsid*VSID_MULTIPLIER) % VSID_MODULUS - * - * rt = register continaing the proto-VSID and into which the - * VSID will be stored - * rx = scratch register (clobbered) - * - * - rt and rx must be different registers - * - The answer will end up in the low VSID_BITS bits of rt. The higher - * bits may contain other garbage, so you may need to mask the - * result. - */ -#define ASM_VSID_SCRAMBLE(rt, rx, size) \ - lis rx,VSID_MULTIPLIER_##size@h; \ - ori rx,rx,VSID_MULTIPLIER_##size@l; \ - mulld rt,rt,rx; /* rt = rt * MULTIPLIER */ \ - \ - srdi rx,rt,VSID_BITS_##size; \ - clrldi rt,rt,(64-VSID_BITS_##size); \ - add rt,rt,rx; /* add high and low bits */ \ - /* NOTE: explanation based on VSID_BITS_##size = 36 \ - * Now, r3 == VSID (mod 2^36-1), and lies between 0 and \ - * 2^36-1+2^28-1. That in particular means that if r3 >= \ - * 2^36-1, then r3+1 has the 2^36 bit set. So, if r3+1 has \ - * the bit clear, r3 already has the answer we want, if it \ - * doesn't, the answer is the low 36 bits of r3+1. So in all \ - * cases the answer is the low 36 bits of (r3 + ((r3+1) >> 36))*/\ - addi rx,rt,1; \ - srdi rx,rx,VSID_BITS_##size; /* extract 2^VSID_BITS bit */ \ - add rt,rt,rx - -/* 4 bits per slice and we have one slice per 1TB */ -#define SLICE_ARRAY_SIZE (PGTABLE_RANGE >> 41) - -#ifndef __ASSEMBLY__ - -#ifdef CONFIG_PPC_SUBPAGE_PROT -/* - * For the sub-page protection option, we extend the PGD with one of - * these. Basically we have a 3-level tree, with the top level being - * the protptrs array. To optimize speed and memory consumption when - * only addresses < 4GB are being protected, pointers to the first - * four pages of sub-page protection words are stored in the low_prot - * array. - * Each page of sub-page protection words protects 1GB (4 bytes - * protects 64k). For the 3-level tree, each page of pointers then - * protects 8TB. - */ -struct subpage_prot_table { - unsigned long maxaddr; /* only addresses < this are protected */ - unsigned int **protptrs[2]; - unsigned int *low_prot[4]; -}; - -#define SBP_L1_BITS (PAGE_SHIFT - 2) -#define SBP_L2_BITS (PAGE_SHIFT - 3) -#define SBP_L1_COUNT (1 << SBP_L1_BITS) -#define SBP_L2_COUNT (1 << SBP_L2_BITS) -#define SBP_L2_SHIFT (PAGE_SHIFT + SBP_L1_BITS) -#define SBP_L3_SHIFT (SBP_L2_SHIFT + SBP_L2_BITS) - -extern void subpage_prot_free(struct mm_struct *mm); -extern void subpage_prot_init_new_context(struct mm_struct *mm); -#else -static inline void subpage_prot_free(struct mm_struct *mm) {} -static inline void subpage_prot_init_new_context(struct mm_struct *mm) { } -#endif /* CONFIG_PPC_SUBPAGE_PROT */ - -typedef unsigned long mm_context_id_t; -struct spinlock; - -typedef struct { - mm_context_id_t id; - u16 user_psize; /* page size index */ - -#ifdef CONFIG_PPC_MM_SLICES - u64 low_slices_psize; /* SLB page size encodings */ - unsigned char high_slices_psize[SLICE_ARRAY_SIZE]; -#else - u16 sllp; /* SLB page size encoding */ -#endif - unsigned long vdso_base; -#ifdef CONFIG_PPC_SUBPAGE_PROT - struct subpage_prot_table spt; -#endif /* CONFIG_PPC_SUBPAGE_PROT */ -#ifdef CONFIG_PPC_ICSWX - struct spinlock *cop_lockp; /* guard acop and cop_pid */ - unsigned long acop; /* mask of enabled coprocessor types */ - unsigned int cop_pid; /* pid value used with coprocessors */ -#endif /* CONFIG_PPC_ICSWX */ -#ifdef CONFIG_PPC_64K_PAGES - /* for 4K PTE fragment support */ - void *pte_frag; -#endif -} mm_context_t; - - -#if 0 -/* - * The code below is equivalent to this function for arguments - * < 2^VSID_BITS, which is all this should ever be called - * with. However gcc is not clever enough to compute the - * modulus (2^n-1) without a second multiply. - */ -#define vsid_scramble(protovsid, size) \ - ((((protovsid) * VSID_MULTIPLIER_##size) % VSID_MODULUS_##size)) - -#else /* 1 */ -#define vsid_scramble(protovsid, size) \ - ({ \ - unsigned long x; \ - x = (protovsid) * VSID_MULTIPLIER_##size; \ - x = (x >> VSID_BITS_##size) + (x & VSID_MODULUS_##size); \ - (x + ((x+1) >> VSID_BITS_##size)) & VSID_MODULUS_##size; \ - }) -#endif /* 1 */ - -/* Returns the segment size indicator for a user address */ -static inline int user_segment_size(unsigned long addr) -{ - /* Use 1T segments if possible for addresses >= 1T */ - if (addr >= (1UL << SID_SHIFT_1T)) - return mmu_highuser_ssize; - return MMU_SEGSIZE_256M; -} - -static inline unsigned long get_vsid(unsigned long context, unsigned long ea, - int ssize) -{ - /* - * Bad address. We return VSID 0 for that - */ - if ((ea & ~REGION_MASK) >= PGTABLE_RANGE) - return 0; - - if (ssize == MMU_SEGSIZE_256M) - return vsid_scramble((context << ESID_BITS) - | (ea >> SID_SHIFT), 256M); - return vsid_scramble((context << ESID_BITS_1T) - | (ea >> SID_SHIFT_1T), 1T); -} - -/* - * This is only valid for addresses >= PAGE_OFFSET - * - * For kernel space, we use the top 4 context ids to map address as below - * 0x7fffc - [ 0xc000000000000000 - 0xc0003fffffffffff ] - * 0x7fffd - [ 0xd000000000000000 - 0xd0003fffffffffff ] - * 0x7fffe - [ 0xe000000000000000 - 0xe0003fffffffffff ] - * 0x7ffff - [ 0xf000000000000000 - 0xf0003fffffffffff ] - */ -static inline unsigned long get_kernel_vsid(unsigned long ea, int ssize) -{ - unsigned long context; - - /* - * kernel take the top 4 context from the available range - */ - context = (MAX_USER_CONTEXT) + ((ea >> 60) - 0xc) + 1; - return get_vsid(context, ea, ssize); -} -#endif /* __ASSEMBLY__ */ - -#endif /* _ASM_POWERPC_MMU_HASH64_H_ */ diff --git a/arch/powerpc/include/asm/mmu.h b/arch/powerpc/include/asm/mmu.h index 691fd8aca939..5f9c5d436e17 100644 --- a/arch/powerpc/include/asm/mmu.h +++ b/arch/powerpc/include/asm/mmu.h @@ -1,36 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_MMU_H_ #define _ASM_POWERPC_MMU_H_ #ifdef __KERNEL__ #include <linux/types.h> -#include <asm/asm-compat.h> -#include <asm/feature-fixups.h> +#include <asm/asm-const.h> /* * MMU features bit definitions */ /* - * First half is MMU families + * MMU families */ #define MMU_FTR_HPTE_TABLE ASM_CONST(0x00000001) #define MMU_FTR_TYPE_8xx ASM_CONST(0x00000002) -#define MMU_FTR_TYPE_40x ASM_CONST(0x00000004) #define MMU_FTR_TYPE_44x ASM_CONST(0x00000008) #define MMU_FTR_TYPE_FSL_E ASM_CONST(0x00000010) -#define MMU_FTR_TYPE_3E ASM_CONST(0x00000020) -#define MMU_FTR_TYPE_47x ASM_CONST(0x00000040) +#define MMU_FTR_TYPE_47x ASM_CONST(0x00000020) + +/* Radix page table supported and enabled */ +#define MMU_FTR_TYPE_RADIX ASM_CONST(0x00000040) + +/* + * Individual features below. + */ + +/* + * Supports KUAP feature + * key 0 controlling userspace addresses on radix + * Key 3 on hash + */ +#define MMU_FTR_KUAP ASM_CONST(0x00000200) /* - * This is individual features + * Supports KUEP feature + * key 0 controlling userspace addresses on radix + * Key 3 on hash */ +#define MMU_FTR_BOOK3S_KUEP ASM_CONST(0x00000400) + +/* + * Support for memory protection keys. + */ +#define MMU_FTR_PKEY ASM_CONST(0x00000800) + +/* Guest Translation Shootdown Enable */ +#define MMU_FTR_GTSE ASM_CONST(0x00001000) + +/* + * Support for 68 bit VA space. We added that from ISA 2.05 + */ +#define MMU_FTR_68_BIT_VA ASM_CONST(0x00002000) +/* + * Kernel read only support. + * We added the ppp value 0b110 in ISA 2.04. + */ +#define MMU_FTR_KERNEL_RO ASM_CONST(0x00004000) + +/* + * We need to clear top 16bits of va (from the remaining 64 bits )in + * tlbie* instructions + */ +#define MMU_FTR_TLBIE_CROP_VA ASM_CONST(0x00008000) /* Enable use of high BAT registers */ #define MMU_FTR_USE_HIGH_BATS ASM_CONST(0x00010000) /* Enable >32-bit physical addresses on 32-bit processor, only used - * by CONFIG_6xx currently as BookE supports that from day 1 + * by CONFIG_PPC_BOOK3S_32 currently as BookE supports that from day 1 */ #define MMU_FTR_BIG_PHYS ASM_CONST(0x00020000) @@ -56,18 +95,9 @@ */ #define MMU_FTR_NEED_DTLB_SW_LRU ASM_CONST(0x00200000) -/* Enable use of TLB reservation. Processor should support tlbsrx. - * instruction and MAS0[WQ]. - */ -#define MMU_FTR_USE_TLBRSRV ASM_CONST(0x00800000) - -/* Use paired MAS registers (MAS7||MAS3, etc.) - */ -#define MMU_FTR_USE_PAIRED_MAS ASM_CONST(0x01000000) - -/* MMU is SLB-based +/* Doesn't support the B bit (1T segment) in SLBIE */ -#define MMU_FTR_SLB ASM_CONST(0x02000000) +#define MMU_FTR_NO_SLBIE_B ASM_CONST(0x02000000) /* Support 16M large pages */ @@ -89,62 +119,189 @@ */ #define MMU_FTR_1T_SEGMENT ASM_CONST(0x40000000) -/* Doesn't support the B bit (1T segment) in SLBIE - */ -#define MMU_FTR_NO_SLBIE_B ASM_CONST(0x80000000) +// NX paste RMA reject in DSI +#define MMU_FTR_NX_DSI ASM_CONST(0x80000000) /* MMU feature bit sets for various CPUs */ -#define MMU_FTRS_DEFAULT_HPTE_ARCH_V2 \ - MMU_FTR_HPTE_TABLE | MMU_FTR_PPCAS_ARCH_V2 -#define MMU_FTRS_POWER4 MMU_FTRS_DEFAULT_HPTE_ARCH_V2 -#define MMU_FTRS_PPC970 MMU_FTRS_POWER4 -#define MMU_FTRS_POWER5 MMU_FTRS_POWER4 | MMU_FTR_LOCKLESS_TLBIE -#define MMU_FTRS_POWER6 MMU_FTRS_POWER4 | MMU_FTR_LOCKLESS_TLBIE -#define MMU_FTRS_POWER7 MMU_FTRS_POWER4 | MMU_FTR_LOCKLESS_TLBIE -#define MMU_FTRS_POWER8 MMU_FTRS_POWER4 | MMU_FTR_LOCKLESS_TLBIE +#define MMU_FTRS_DEFAULT_HPTE_ARCH_V2 (MMU_FTR_HPTE_TABLE | MMU_FTR_TLBIEL | MMU_FTR_16M_PAGE) +#define MMU_FTRS_POWER MMU_FTRS_DEFAULT_HPTE_ARCH_V2 +#define MMU_FTRS_PPC970 MMU_FTRS_POWER | MMU_FTR_TLBIE_CROP_VA +#define MMU_FTRS_POWER5 MMU_FTRS_POWER | MMU_FTR_LOCKLESS_TLBIE +#define MMU_FTRS_POWER6 MMU_FTRS_POWER5 | MMU_FTR_KERNEL_RO | MMU_FTR_68_BIT_VA +#define MMU_FTRS_POWER7 MMU_FTRS_POWER6 +#define MMU_FTRS_POWER8 MMU_FTRS_POWER6 +#define MMU_FTRS_POWER9 MMU_FTRS_POWER6 +#define MMU_FTRS_POWER10 MMU_FTRS_POWER6 +#define MMU_FTRS_POWER11 MMU_FTRS_POWER6 #define MMU_FTRS_CELL MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \ MMU_FTR_CI_LARGE_PAGE #define MMU_FTRS_PA6T MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \ MMU_FTR_CI_LARGE_PAGE | MMU_FTR_NO_SLBIE_B -#define MMU_FTRS_A2 MMU_FTR_TYPE_3E | MMU_FTR_USE_TLBILX | \ - MMU_FTR_USE_TLBIVAX_BCAST | \ - MMU_FTR_LOCK_BCAST_INVAL | \ - MMU_FTR_USE_TLBRSRV | \ - MMU_FTR_USE_PAIRED_MAS | \ - MMU_FTR_TLBIEL | \ - MMU_FTR_16M_PAGE -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ +#include <linux/bug.h> #include <asm/cputable.h> +#include <asm/page.h> + +typedef pte_t *pgtable_t; -#ifdef CONFIG_PPC_FSL_BOOK3E -#include <asm/percpu.h> -DECLARE_PER_CPU(int, next_tlbcam_idx); +enum { + MMU_FTRS_POSSIBLE = +#if defined(CONFIG_PPC_BOOK3S_604) + MMU_FTR_HPTE_TABLE | #endif +#ifdef CONFIG_PPC_8xx + MMU_FTR_TYPE_8xx | +#endif +#ifdef CONFIG_PPC_47x + MMU_FTR_TYPE_47x | MMU_FTR_USE_TLBIVAX_BCAST | MMU_FTR_LOCK_BCAST_INVAL | +#elif defined(CONFIG_44x) + MMU_FTR_TYPE_44x | +#endif +#ifdef CONFIG_PPC_E500 + MMU_FTR_TYPE_FSL_E | MMU_FTR_BIG_PHYS | MMU_FTR_USE_TLBILX | +#endif +#ifdef CONFIG_PPC_BOOK3S_32 + MMU_FTR_USE_HIGH_BATS | +#endif +#ifdef CONFIG_PPC_83xx + MMU_FTR_NEED_DTLB_SW_LRU | +#endif +#ifdef CONFIG_PPC_BOOK3S_64 + MMU_FTR_KERNEL_RO | +#ifdef CONFIG_PPC_64S_HASH_MMU + MMU_FTR_NO_SLBIE_B | MMU_FTR_16M_PAGE | MMU_FTR_TLBIEL | + MMU_FTR_LOCKLESS_TLBIE | MMU_FTR_CI_LARGE_PAGE | + MMU_FTR_1T_SEGMENT | MMU_FTR_TLBIE_CROP_VA | + MMU_FTR_68_BIT_VA | MMU_FTR_HPTE_TABLE | +#endif +#ifdef CONFIG_PPC_RADIX_MMU + MMU_FTR_TYPE_RADIX | + MMU_FTR_GTSE | MMU_FTR_NX_DSI | +#endif /* CONFIG_PPC_RADIX_MMU */ +#endif +#ifdef CONFIG_PPC_KUAP + MMU_FTR_KUAP | +#endif /* CONFIG_PPC_KUAP */ +#ifdef CONFIG_PPC_MEM_KEYS + MMU_FTR_PKEY | +#endif +#ifdef CONFIG_PPC_KUEP + MMU_FTR_BOOK3S_KUEP | +#endif /* CONFIG_PPC_KUAP */ + + 0, +}; + +#if defined(CONFIG_PPC_BOOK3S_604) && !defined(CONFIG_PPC_BOOK3S_603) +#define MMU_FTRS_ALWAYS MMU_FTR_HPTE_TABLE +#endif +#ifdef CONFIG_PPC_8xx +#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_8xx +#endif +#ifdef CONFIG_PPC_47x +#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_47x +#elif defined(CONFIG_44x) +#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_44x +#endif +#ifdef CONFIG_PPC_E500 +#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_FSL_E +#endif + +/* BOOK3S_64 options */ +#if defined(CONFIG_PPC_RADIX_MMU) && !defined(CONFIG_PPC_64S_HASH_MMU) +#define MMU_FTRS_ALWAYS MMU_FTR_TYPE_RADIX +#elif !defined(CONFIG_PPC_RADIX_MMU) && defined(CONFIG_PPC_64S_HASH_MMU) +#define MMU_FTRS_ALWAYS MMU_FTR_HPTE_TABLE +#endif + +#ifndef MMU_FTRS_ALWAYS +#define MMU_FTRS_ALWAYS 0 +#endif + +static __always_inline bool early_mmu_has_feature(unsigned long feature) +{ + if (MMU_FTRS_ALWAYS & feature) + return true; + + return !!(MMU_FTRS_POSSIBLE & cur_cpu_spec->mmu_features & feature); +} + +#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECKS +#include <linux/jump_label.h> + +#define NUM_MMU_FTR_KEYS 32 + +extern struct static_key_true mmu_feature_keys[NUM_MMU_FTR_KEYS]; -static inline int mmu_has_feature(unsigned long feature) +extern void mmu_feature_keys_init(void); + +static __always_inline bool mmu_has_feature(unsigned long feature) { - return (cur_cpu_spec->mmu_features & feature); + int i; + + BUILD_BUG_ON(!__builtin_constant_p(feature)); + BUILD_BUG_ON(__builtin_popcountl(feature) > 1); + +#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECK_DEBUG + if (!static_key_feature_checks_initialized) { + printk("Warning! mmu_has_feature() used prior to jump label init!\n"); + dump_stack(); + return early_mmu_has_feature(feature); + } +#endif + + if (MMU_FTRS_ALWAYS & feature) + return true; + + if (!(MMU_FTRS_POSSIBLE & feature)) + return false; + + i = __builtin_ctzl(feature); + return static_branch_likely(&mmu_feature_keys[i]); } static inline void mmu_clear_feature(unsigned long feature) { + int i; + + i = __builtin_ctzl(feature); cur_cpu_spec->mmu_features &= ~feature; + static_branch_disable(&mmu_feature_keys[i]); } +#else -extern unsigned int __start___mmu_ftr_fixup, __stop___mmu_ftr_fixup; +static inline void mmu_feature_keys_init(void) +{ -/* MMU initialization */ -extern void early_init_mmu(void); -extern void early_init_mmu_secondary(void); +} -extern void setup_initial_memory_limit(phys_addr_t first_memblock_base, - phys_addr_t first_memblock_size); +static __always_inline bool mmu_has_feature(unsigned long feature) +{ + return early_mmu_has_feature(feature); +} + +static inline void mmu_clear_feature(unsigned long feature) +{ + cur_cpu_spec->mmu_features &= ~feature; +} +#endif /* CONFIG_JUMP_LABEL */ + +extern unsigned int __start___mmu_ftr_fixup, __stop___mmu_ftr_fixup; #ifdef CONFIG_PPC64 /* This is our real memory area size on ppc64 server, on embedded, we * make it match the size our of bolted TLB area */ extern u64 ppc64_rma_size; + +/* Cleanup function used by kexec */ +extern void mmu_cleanup_all(void); +extern void radix__mmu_cleanup_all(void); + +/* Functions for creating and updating partition table on POWER9 */ +extern void mmu_partition_table_init(void); +extern void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0, + unsigned long dw1, bool flush); #endif /* CONFIG_PPC64 */ struct mm_struct; @@ -156,7 +313,26 @@ static inline void assert_pte_locked(struct mm_struct *mm, unsigned long addr) } #endif /* !CONFIG_DEBUG_VM */ -#endif /* !__ASSEMBLY__ */ +static __always_inline bool radix_enabled(void) +{ + return mmu_has_feature(MMU_FTR_TYPE_RADIX); +} + +static __always_inline bool early_radix_enabled(void) +{ + return early_mmu_has_feature(MMU_FTR_TYPE_RADIX); +} + +static inline bool strict_kernel_rwx_enabled(void) +{ + return IS_ENABLED(CONFIG_STRICT_KERNEL_RWX) && rodata_enabled; +} + +static inline bool strict_module_rwx_enabled(void) +{ + return IS_ENABLED(CONFIG_STRICT_MODULE_RWX) && strict_kernel_rwx_enabled(); +} +#endif /* !__ASSEMBLER__ */ /* The kernel use the constants below to index in the page sizes array. * The use of fixed constants for this purpose is better for performances @@ -179,38 +355,48 @@ static inline void assert_pte_locked(struct mm_struct *mm, unsigned long addr) #define MMU_PAGE_64K 2 #define MMU_PAGE_64K_AP 3 /* "Admixed pages" (hash64 only) */ #define MMU_PAGE_256K 4 -#define MMU_PAGE_1M 5 -#define MMU_PAGE_4M 6 -#define MMU_PAGE_8M 7 -#define MMU_PAGE_16M 8 -#define MMU_PAGE_64M 9 -#define MMU_PAGE_256M 10 -#define MMU_PAGE_1G 11 -#define MMU_PAGE_16G 12 -#define MMU_PAGE_64G 13 - -#define MMU_PAGE_COUNT 14 - -#if defined(CONFIG_PPC_STD_MMU_64) -/* 64-bit classic hash table MMU */ -# include <asm/mmu-hash64.h> -#elif defined(CONFIG_PPC_STD_MMU_32) -/* 32-bit classic hash table MMU */ -# include <asm/mmu-hash32.h> -#elif defined(CONFIG_40x) -/* 40x-style software loaded TLB */ -# include <asm/mmu-40x.h> -#elif defined(CONFIG_44x) -/* 44x-style software loaded TLB */ -# include <asm/mmu-44x.h> -#elif defined(CONFIG_PPC_BOOK3E_MMU) -/* Freescale Book-E software loaded TLB or Book-3e (ISA 2.06+) MMU */ -# include <asm/mmu-book3e.h> -#elif defined (CONFIG_PPC_8xx) -/* Motorola/Freescale 8xx software loaded TLB */ -# include <asm/mmu-8xx.h> +#define MMU_PAGE_512K 5 +#define MMU_PAGE_1M 6 +#define MMU_PAGE_2M 7 +#define MMU_PAGE_4M 8 +#define MMU_PAGE_8M 9 +#define MMU_PAGE_16M 10 +#define MMU_PAGE_64M 11 +#define MMU_PAGE_256M 12 +#define MMU_PAGE_1G 13 +#define MMU_PAGE_16G 14 +#define MMU_PAGE_64G 15 + +/* + * N.B. we need to change the type of hpte_page_sizes if this gets to be > 16 + * Also we need to change he type of mm_context.low/high_slices_psize. + */ +#define MMU_PAGE_COUNT 16 + +#ifdef CONFIG_PPC_BOOK3S_64 +#include <asm/book3s/64/mmu.h> +#else /* CONFIG_PPC_BOOK3S_64 */ + +#ifndef __ASSEMBLER__ +/* MMU initialization */ +extern void early_init_mmu(void); +extern void early_init_mmu_secondary(void); +extern void setup_initial_memory_limit(phys_addr_t first_memblock_base, + phys_addr_t first_memblock_size); +static inline void mmu_early_init_devtree(void) { } + +static inline void pkey_early_init_devtree(void) {} + +extern void *abatron_pteptrs[2]; +#endif /* __ASSEMBLER__ */ #endif +#if defined(CONFIG_PPC_BOOK3S_32) +/* 32-bit classic hash table MMU */ +#include <asm/book3s/32/mmu-hash.h> +#elif defined(CONFIG_PPC_MMU_NOHASH) +#include <asm/nohash/mmu.h> +#endif #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_MMU_H_ */ diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h index b467530e2485..a157ab513347 100644 --- a/arch/powerpc/include/asm/mmu_context.h +++ b/arch/powerpc/include/asm/mmu_context.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __ASM_POWERPC_MMU_CONTEXT_H #define __ASM_POWERPC_MMU_CONTEXT_H #ifdef __KERNEL__ @@ -8,110 +9,284 @@ #include <linux/spinlock.h> #include <asm/mmu.h> #include <asm/cputable.h> -#include <asm-generic/mm_hooks.h> #include <asm/cputhreads.h> /* * Most if the context management is out of line */ +#define init_new_context init_new_context extern int init_new_context(struct task_struct *tsk, struct mm_struct *mm); +#define destroy_context destroy_context extern void destroy_context(struct mm_struct *mm); +#ifdef CONFIG_SPAPR_TCE_IOMMU +struct mm_iommu_table_group_mem_t; -extern void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next); -extern void switch_stab(struct task_struct *tsk, struct mm_struct *mm); +extern bool mm_iommu_preregistered(struct mm_struct *mm); +extern long mm_iommu_new(struct mm_struct *mm, + unsigned long ua, unsigned long entries, + struct mm_iommu_table_group_mem_t **pmem); +extern long mm_iommu_newdev(struct mm_struct *mm, unsigned long ua, + unsigned long entries, unsigned long dev_hpa, + struct mm_iommu_table_group_mem_t **pmem); +extern long mm_iommu_put(struct mm_struct *mm, + struct mm_iommu_table_group_mem_t *mem); +extern void mm_iommu_init(struct mm_struct *mm); +extern struct mm_iommu_table_group_mem_t *mm_iommu_lookup(struct mm_struct *mm, + unsigned long ua, unsigned long size); +extern struct mm_iommu_table_group_mem_t *mm_iommu_get(struct mm_struct *mm, + unsigned long ua, unsigned long entries); +extern long mm_iommu_ua_to_hpa(struct mm_iommu_table_group_mem_t *mem, + unsigned long ua, unsigned int pageshift, unsigned long *hpa); +extern bool mm_iommu_is_devmem(struct mm_struct *mm, unsigned long hpa, + unsigned int pageshift, unsigned long *size); +extern long mm_iommu_mapped_inc(struct mm_iommu_table_group_mem_t *mem); +extern void mm_iommu_mapped_dec(struct mm_iommu_table_group_mem_t *mem); +#else +static inline bool mm_iommu_is_devmem(struct mm_struct *mm, unsigned long hpa, + unsigned int pageshift, unsigned long *size) +{ + return false; +} +static inline void mm_iommu_init(struct mm_struct *mm) { } +#endif extern void switch_slb(struct task_struct *tsk, struct mm_struct *mm); -extern void set_context(unsigned long id, pgd_t *pgd); #ifdef CONFIG_PPC_BOOK3S_64 -extern int __init_new_context(void); +extern void radix__switch_mmu_context(struct mm_struct *prev, + struct mm_struct *next); +static inline void switch_mmu_context(struct mm_struct *prev, + struct mm_struct *next, + struct task_struct *tsk) +{ + if (radix_enabled()) + return radix__switch_mmu_context(prev, next); + return switch_slb(tsk, next); +} + +extern int hash__alloc_context_id(void); +void __init hash__reserve_context_id(int id); extern void __destroy_context(int context_id); static inline void mmu_context_init(void) { } + +#ifdef CONFIG_PPC_64S_HASH_MMU +static inline int alloc_extended_context(struct mm_struct *mm, + unsigned long ea) +{ + int context_id; + + int index = ea >> MAX_EA_BITS_PER_CONTEXT; + + context_id = hash__alloc_context_id(); + if (context_id < 0) + return context_id; + + VM_WARN_ON(mm->context.extended_id[index]); + mm->context.extended_id[index] = context_id; + return context_id; +} + +static inline bool need_extra_context(struct mm_struct *mm, unsigned long ea) +{ + int context_id; + + context_id = get_user_context(&mm->context, ea); + if (!context_id) + return true; + return false; +} +#endif + #else +extern void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next, + struct task_struct *tsk); extern unsigned long __init_new_context(void); extern void __destroy_context(unsigned long context_id); extern void mmu_context_init(void); +static inline int alloc_extended_context(struct mm_struct *mm, + unsigned long ea) +{ + /* non book3s_64 should never find this called */ + WARN_ON(1); + return -ENOMEM; +} + +static inline bool need_extra_context(struct mm_struct *mm, unsigned long ea) +{ + return false; +} #endif -extern void switch_cop(struct mm_struct *next); -extern int use_cop(unsigned long acop, struct mm_struct *mm); -extern void drop_cop(unsigned long acop, struct mm_struct *mm); +#ifdef CONFIG_PPC_BOOK3S_64 +static inline void inc_mm_active_cpus(struct mm_struct *mm) +{ + atomic_inc(&mm->context.active_cpus); +} + +static inline void dec_mm_active_cpus(struct mm_struct *mm) +{ + VM_WARN_ON_ONCE(atomic_read(&mm->context.active_cpus) <= 0); + atomic_dec(&mm->context.active_cpus); +} + +static inline void mm_context_add_copro(struct mm_struct *mm) +{ + /* + * If any copro is in use, increment the active CPU count + * in order to force TLB invalidations to be global as to + * propagate to the Nest MMU. + */ + if (atomic_inc_return(&mm->context.copros) == 1) + inc_mm_active_cpus(mm); +} + +static inline void mm_context_remove_copro(struct mm_struct *mm) +{ + int c; + + /* + * When removing the last copro, we need to broadcast a global + * flush of the full mm, as the next TLBI may be local and the + * nMMU and/or PSL need to be cleaned up. + * + * Both the 'copros' and 'active_cpus' counts are looked at in + * radix__flush_all_mm() to determine the scope (local/global) + * of the TLBIs, so we need to flush first before decrementing + * 'copros'. If this API is used by several callers for the + * same context, it can lead to over-flushing. It's hopefully + * not common enough to be a problem. + * + * Skip on hash, as we don't know how to do the proper flush + * for the time being. Invalidations will remain global if + * used on hash. Note that we can't drop 'copros' either, as + * it could make some invalidations local with no flush + * in-between. + */ + if (radix_enabled()) { + radix__flush_all_mm(mm); + + c = atomic_dec_if_positive(&mm->context.copros); + /* Detect imbalance between add and remove */ + WARN_ON(c < 0); + + if (c == 0) + dec_mm_active_cpus(mm); + } +} /* - * switch_mm is the entry point called from the architecture independent - * code in kernel/sched/core.c + * vas_windows counter shows number of open windows in the mm + * context. During context switch, use this counter to clear the + * foreign real address mapping (CP_ABORT) for the thread / process + * that intend to use COPY/PASTE. When a process closes all windows, + * disable CP_ABORT which is expensive to run. + * + * For user context, register a copro so that TLBIs are seen by the + * nest MMU. mm_context_add/remove_vas_window() are used only for user + * space windows. */ -static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, - struct task_struct *tsk) +static inline void mm_context_add_vas_window(struct mm_struct *mm) +{ + atomic_inc(&mm->context.vas_windows); + mm_context_add_copro(mm); +} + +static inline void mm_context_remove_vas_window(struct mm_struct *mm) { - /* Mark this context has been used on the new CPU */ - cpumask_set_cpu(smp_processor_id(), mm_cpumask(next)); + int v; - /* 32-bit keeps track of the current PGDIR in the thread struct */ -#ifdef CONFIG_PPC32 - tsk->thread.pgdir = next->pgd; -#endif /* CONFIG_PPC32 */ + mm_context_remove_copro(mm); + v = atomic_dec_if_positive(&mm->context.vas_windows); - /* 64-bit Book3E keeps track of current PGD in the PACA */ -#ifdef CONFIG_PPC_BOOK3E_64 - get_paca()->pgd = next->pgd; + /* Detect imbalance between add and remove */ + WARN_ON(v < 0); +} +#else +static inline void inc_mm_active_cpus(struct mm_struct *mm) { } +static inline void dec_mm_active_cpus(struct mm_struct *mm) { } +static inline void mm_context_add_copro(struct mm_struct *mm) { } +static inline void mm_context_remove_copro(struct mm_struct *mm) { } #endif - /* Nothing else to do if we aren't actually switching */ - if (prev == next) - return; - -#ifdef CONFIG_PPC_ICSWX - /* Switch coprocessor context only if prev or next uses a coprocessor */ - if (prev->context.acop || next->context.acop) - switch_cop(next); -#endif /* CONFIG_PPC_ICSWX */ - - /* We must stop all altivec streams before changing the HW - * context - */ -#ifdef CONFIG_ALTIVEC - if (cpu_has_feature(CPU_FTR_ALTIVEC)) - asm volatile ("dssall"); -#endif /* CONFIG_ALTIVEC */ - /* The actual HW switching method differs between the various - * sub architectures. - */ -#ifdef CONFIG_PPC_STD_MMU_64 - if (mmu_has_feature(MMU_FTR_SLB)) - switch_slb(tsk, next); - else - switch_stab(tsk, next); +#if defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE) && defined(CONFIG_PPC_RADIX_MMU) +void do_h_rpt_invalidate_prt(unsigned long pid, unsigned long lpid, + unsigned long type, unsigned long pg_sizes, + unsigned long start, unsigned long end); #else - /* Out of line for now */ - switch_mmu_context(prev, next); +static inline void do_h_rpt_invalidate_prt(unsigned long pid, + unsigned long lpid, + unsigned long type, + unsigned long pg_sizes, + unsigned long start, + unsigned long end) { } #endif -} +extern void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, + struct task_struct *tsk); + +static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, + struct task_struct *tsk) +{ + unsigned long flags; -#define deactivate_mm(tsk,mm) do { } while (0) + local_irq_save(flags); + switch_mm_irqs_off(prev, next, tsk); + local_irq_restore(flags); +} +#define switch_mm_irqs_off switch_mm_irqs_off /* * After we have set current->mm to a new value, this activates * the context for the new mm so we see the new mappings. */ +#define activate_mm activate_mm static inline void activate_mm(struct mm_struct *prev, struct mm_struct *next) { - unsigned long flags; - - local_irq_save(flags); - switch_mm(prev, next, current); - local_irq_restore(flags); + switch_mm_irqs_off(prev, next, current); } /* We don't currently use enter_lazy_tlb() for anything */ +#ifdef CONFIG_PPC_BOOK3E_64 +#define enter_lazy_tlb enter_lazy_tlb static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) { /* 64-bit Book3E keeps track of current PGD in the PACA */ -#ifdef CONFIG_PPC_BOOK3E_64 get_paca()->pgd = NULL; +} #endif + +extern void arch_exit_mmap(struct mm_struct *mm); + +#ifdef CONFIG_PPC_MEM_KEYS +bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write, + bool execute, bool foreign); +void arch_dup_pkeys(struct mm_struct *oldmm, struct mm_struct *mm); +#else /* CONFIG_PPC_MEM_KEYS */ +static inline bool arch_vma_access_permitted(struct vm_area_struct *vma, + bool write, bool execute, bool foreign) +{ + /* by default, allow everything */ + return true; +} + +#define pkey_mm_init(mm) +#define arch_dup_pkeys(oldmm, mm) + +static inline u64 pte_to_hpte_pkey_bits(u64 pteflags, unsigned long flags) +{ + return 0x0UL; } +#endif /* CONFIG_PPC_MEM_KEYS */ + +static inline int arch_dup_mmap(struct mm_struct *oldmm, + struct mm_struct *mm) +{ + arch_dup_pkeys(oldmm, mm); + return 0; +} + +#include <asm-generic/mmu_context.h> + #endif /* __KERNEL__ */ #endif /* __ASM_POWERPC_MMU_CONTEXT_H */ diff --git a/arch/powerpc/include/asm/mmzone.h b/arch/powerpc/include/asm/mmzone.h index 7b589178be46..049152f8d597 100644 --- a/arch/powerpc/include/asm/mmzone.h +++ b/arch/powerpc/include/asm/mmzone.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Written by Kanoj Sarcar (kanoj@sgi.com) Aug 99 * @@ -17,13 +18,7 @@ * flags field of the struct page */ -#ifdef CONFIG_NEED_MULTIPLE_NODES - -extern struct pglist_data *node_data[]; -/* - * Return a pointer to the node data for node n. - */ -#define NODE_DATA(nid) (node_data[nid]) +#ifdef CONFIG_NUMA /* * Following are specific to this numa platform. @@ -34,13 +29,14 @@ extern cpumask_var_t node_to_cpumask_map[]; #ifdef CONFIG_MEMORY_HOTPLUG extern unsigned long max_pfn; u64 memory_hotplug_max(void); +u64 hot_add_drconf_memory_max(void); #else #define memory_hotplug_max() memblock_end_of_DRAM() #endif #else #define memory_hotplug_max() memblock_end_of_DRAM() -#endif /* CONFIG_NEED_MULTIPLE_NODES */ +#endif /* CONFIG_NUMA */ #endif /* __KERNEL__ */ #endif /* _ASM_MMZONE_H_ */ diff --git a/arch/powerpc/include/asm/module.h b/arch/powerpc/include/asm/module.h index c1df590ec444..864e22deaa2c 100644 --- a/arch/powerpc/include/asm/module.h +++ b/arch/powerpc/include/asm/module.h @@ -1,25 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _ASM_POWERPC_MODULE_H #define _ASM_POWERPC_MODULE_H #ifdef __KERNEL__ -/* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ - #include <linux/list.h> #include <asm/bug.h> #include <asm-generic/module.h> - #ifndef __powerpc64__ /* * Thanks to Paul M for explaining this. * * PPC can only do rel jumps += 32MB, and often the kernel and other - * modules are furthur away than this. So, we jump to a table of + * modules are further away than this. So, we jump to a table of * trampolines attached to the module (the Procedure Linkage Table) * whenever that happens. */ @@ -34,35 +27,48 @@ struct ppc_plt_entry { struct mod_arch_specific { #ifdef __powerpc64__ unsigned int stubs_section; /* Index of stubs section in module */ + unsigned int stub_count; /* Number of stubs used */ +#ifdef CONFIG_PPC_KERNEL_PCREL + unsigned int got_section; /* What section is the GOT? */ + unsigned int pcpu_section; /* .data..percpu section */ +#else unsigned int toc_section; /* What section is the TOC? */ -#ifdef CONFIG_DYNAMIC_FTRACE - unsigned long toc; - unsigned long tramp; + bool toc_fixed; /* Have we fixed up .TOC.? */ #endif +#ifdef CONFIG_PPC64_ELF_ABI_V1 + /* For module function descriptor dereference */ + unsigned long start_opd; + unsigned long end_opd; +#endif #else /* powerpc64 */ /* Indices of PLT sections within module. */ unsigned int core_plt_section; unsigned int init_plt_section; +#endif /* powerpc64 */ + #ifdef CONFIG_DYNAMIC_FTRACE unsigned long tramp; + unsigned long tramp_regs; +#ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE + struct ftrace_ool_stub *ool_stubs; + unsigned int ool_stub_count; + unsigned int ool_stub_index; +#endif #endif -#endif /* powerpc64 */ - - /* List of BUG addresses, source line numbers and filenames */ - struct list_head bug_list; - struct bug_entry *bug_table; - unsigned int num_bugs; }; /* * Select ELF headers. - * Make empty section for module_frob_arch_sections to expand. + * Make empty sections for module_frob_arch_sections to expand. */ #ifdef __powerpc64__ # ifdef MODULE asm(".section .stubs,\"ax\",@nobits; .align 3; .previous"); +# ifdef CONFIG_PPC_KERNEL_PCREL + asm(".section .mygot,\"a\",@nobits; .align 3; .previous"); +# endif # endif #else # ifdef MODULE @@ -72,20 +78,15 @@ struct mod_arch_specific { #endif #ifdef CONFIG_DYNAMIC_FTRACE -# ifdef MODULE - asm(".section .ftrace.tramp,\"ax\",@nobits; .align 3; .previous"); -# endif /* MODULE */ +int module_trampoline_target(struct module *mod, unsigned long trampoline, + unsigned long *target); +int module_finalize_ftrace(struct module *mod, const Elf_Shdr *sechdrs); +#else +static inline int module_finalize_ftrace(struct module *mod, const Elf_Shdr *sechdrs) +{ + return 0; +} #endif - -struct exception_table_entry; -void sort_ex_table(struct exception_table_entry *start, - struct exception_table_entry *finish); - -#ifdef CONFIG_MODVERSIONS -#define ARCH_RELOCATES_KCRCTAB - -extern const unsigned long reloc_start[]; -#endif #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_MODULE_H */ diff --git a/arch/powerpc/include/asm/module.lds.h b/arch/powerpc/include/asm/module.lds.h new file mode 100644 index 000000000000..cea5dc124be4 --- /dev/null +++ b/arch/powerpc/include/asm/module.lds.h @@ -0,0 +1,8 @@ +/* Force alignment of .toc section. */ +SECTIONS +{ + .toc 0 : ALIGN(256) + { + *(.got .toc) + } +} diff --git a/arch/powerpc/include/asm/mpc5121.h b/arch/powerpc/include/asm/mpc5121.h index 8ae133eaf9fa..9ae49e743b34 100644 --- a/arch/powerpc/include/asm/mpc5121.h +++ b/arch/powerpc/include/asm/mpc5121.h @@ -1,8 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * MPC5121 Prototypes and definitions - * - * This file is licensed under the terms of the GNU General Public - * License version 2. */ #ifndef __ASM_POWERPC_MPC5121_H__ @@ -32,26 +30,17 @@ struct mpc512x_ccm { u32 scfr2; /* System Clock Frequency Register 2 */ u32 scfr2s; /* System Clock Frequency Shadow Register 2 */ u32 bcr; /* Bread Crumb Register */ - u32 p0ccr; /* PSC0 Clock Control Register */ - u32 p1ccr; /* PSC1 CCR */ - u32 p2ccr; /* PSC2 CCR */ - u32 p3ccr; /* PSC3 CCR */ - u32 p4ccr; /* PSC4 CCR */ - u32 p5ccr; /* PSC5 CCR */ - u32 p6ccr; /* PSC6 CCR */ - u32 p7ccr; /* PSC7 CCR */ - u32 p8ccr; /* PSC8 CCR */ - u32 p9ccr; /* PSC9 CCR */ - u32 p10ccr; /* PSC10 CCR */ - u32 p11ccr; /* PSC11 CCR */ + u32 psc_ccr[12]; /* PSC Clock Control Registers */ u32 spccr; /* SPDIF Clock Control Register */ u32 cccr; /* CFM Clock Control Register */ u32 dccr; /* DIU Clock Control Register */ - u32 m1ccr; /* MSCAN1 CCR */ - u32 m2ccr; /* MSCAN2 CCR */ - u32 m3ccr; /* MSCAN3 CCR */ - u32 m4ccr; /* MSCAN4 CCR */ - u8 res[0x98]; /* Reserved */ + u32 mscan_ccr[4]; /* MSCAN Clock Control Registers */ + u32 out_ccr[4]; /* OUT CLK Configure Registers */ + u32 rsv0[2]; /* Reserved */ + u32 scfr3; /* System Clock Frequency Register 3 */ + u32 rsv1[3]; /* Reserved */ + u32 spll_lock_cnt; /* System PLL Lock Counter */ + u8 res[0x6c]; /* Reserved */ }; /* @@ -69,4 +58,63 @@ struct mpc512x_lpc { int mpc512x_cs_config(unsigned int cs, u32 val); +/* + * SCLPC Module (LPB FIFO) + */ +struct mpc512x_lpbfifo { + u32 pkt_size; /* SCLPC Packet Size Register */ + u32 start_addr; /* SCLPC Start Address Register */ + u32 ctrl; /* SCLPC Control Register */ + u32 enable; /* SCLPC Enable Register */ + u32 reserved1; + u32 status; /* SCLPC Status Register */ + u32 bytes_done; /* SCLPC Bytes Done Register */ + u32 emb_sc; /* EMB Share Counter Register */ + u32 emb_pc; /* EMB Pause Control Register */ + u32 reserved2[7]; + u32 data_word; /* LPC RX/TX FIFO Data Word Register */ + u32 fifo_status; /* LPC RX/TX FIFO Status Register */ + u32 fifo_ctrl; /* LPC RX/TX FIFO Control Register */ + u32 fifo_alarm; /* LPC RX/TX FIFO Alarm Register */ +}; + +#define MPC512X_SCLPC_START (1 << 31) +#define MPC512X_SCLPC_CS(x) (((x) & 0x7) << 24) +#define MPC512X_SCLPC_FLUSH (1 << 17) +#define MPC512X_SCLPC_READ (1 << 16) +#define MPC512X_SCLPC_DAI (1 << 8) +#define MPC512X_SCLPC_BPT(x) ((x) & 0x3f) +#define MPC512X_SCLPC_RESET (1 << 24) +#define MPC512X_SCLPC_FIFO_RESET (1 << 16) +#define MPC512X_SCLPC_ABORT_INT_ENABLE (1 << 9) +#define MPC512X_SCLPC_NORM_INT_ENABLE (1 << 8) +#define MPC512X_SCLPC_ENABLE (1 << 0) +#define MPC512X_SCLPC_SUCCESS (1 << 24) +#define MPC512X_SCLPC_FIFO_CTRL(x) (((x) & 0x7) << 24) +#define MPC512X_SCLPC_FIFO_ALARM(x) ((x) & 0x3ff) + +enum lpb_dev_portsize { + LPB_DEV_PORTSIZE_UNDEFINED = 0, + LPB_DEV_PORTSIZE_1_BYTE = 1, + LPB_DEV_PORTSIZE_2_BYTES = 2, + LPB_DEV_PORTSIZE_4_BYTES = 4, + LPB_DEV_PORTSIZE_8_BYTES = 8 +}; + +enum mpc512x_lpbfifo_req_dir { + MPC512X_LPBFIFO_REQ_DIR_READ, + MPC512X_LPBFIFO_REQ_DIR_WRITE +}; + +struct mpc512x_lpbfifo_request { + phys_addr_t dev_phys_addr; /* physical address of some device on LPB */ + void *ram_virt_addr; /* virtual address of some region in RAM */ + u32 size; + enum lpb_dev_portsize portsize; + enum mpc512x_lpbfifo_req_dir dir; + void (*callback)(struct mpc512x_lpbfifo_request *); +}; + +int mpc512x_lpbfifo_submit(struct mpc512x_lpbfifo_request *req); + #endif /* __ASM_POWERPC_MPC5121_H__ */ diff --git a/arch/powerpc/include/asm/mpc52xx.h b/arch/powerpc/include/asm/mpc52xx.h index 0acc7c7c28d1..d7ffbd06797d 100644 --- a/arch/powerpc/include/asm/mpc52xx.h +++ b/arch/powerpc/include/asm/mpc52xx.h @@ -13,11 +13,10 @@ #ifndef __ASM_POWERPC_MPC52xx_H__ #define __ASM_POWERPC_MPC52xx_H__ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm/types.h> -#include <asm/prom.h> #include <asm/mpc5xxx.h> -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #include <linux/suspend.h> @@ -31,7 +30,7 @@ /* Structures mapping of some unit register set */ /* ======================================================================== */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* Memory Mapping Control */ struct mpc52xx_mmap_ctl { @@ -259,14 +258,16 @@ struct mpc52xx_intr { u32 per_error; /* INTR + 0x38 */ }; -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ /* ========================================================================= */ /* Prototypes for MPC52xx sysdev */ /* ========================================================================= */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ + +struct device_node; /* mpc52xx_common.c */ extern void mpc5200_setup_xlb_arbiter(void); @@ -274,8 +275,7 @@ extern void mpc52xx_declare_of_platform_devices(void); extern int mpc5200_psc_ac97_gpio_reset(int psc_number); extern void mpc52xx_map_common_devices(void); extern int mpc52xx_set_psc_clkdiv(int psc_id, int clkdiv); -extern unsigned int mpc52xx_get_xtal_freq(struct device_node *node); -extern void mpc52xx_restart(char *cmd); +extern void __noreturn mpc52xx_restart(char *cmd); /* mpc52xx_gpt.c */ struct mpc52xx_gpt_priv; @@ -285,47 +285,6 @@ extern int mpc52xx_gpt_start_timer(struct mpc52xx_gpt_priv *gpt, u64 period, extern u64 mpc52xx_gpt_timer_period(struct mpc52xx_gpt_priv *gpt); extern int mpc52xx_gpt_stop_timer(struct mpc52xx_gpt_priv *gpt); -/* mpc52xx_lpbfifo.c */ -#define MPC52XX_LPBFIFO_FLAG_READ (0) -#define MPC52XX_LPBFIFO_FLAG_WRITE (1<<0) -#define MPC52XX_LPBFIFO_FLAG_NO_INCREMENT (1<<1) -#define MPC52XX_LPBFIFO_FLAG_NO_DMA (1<<2) -#define MPC52XX_LPBFIFO_FLAG_POLL_DMA (1<<3) - -struct mpc52xx_lpbfifo_request { - struct list_head list; - - /* localplus bus address */ - unsigned int cs; - size_t offset; - - /* Memory address */ - void *data; - phys_addr_t data_phys; - - /* Details of transfer */ - size_t size; - size_t pos; /* current position of transfer */ - int flags; - int defer_xfer_start; - - /* What to do when finished */ - void (*callback)(struct mpc52xx_lpbfifo_request *); - - void *priv; /* Driver private data */ - - /* statistics */ - int irq_count; - int irq_ticks; - u8 last_byte; - int buffer_not_done_cnt; -}; - -extern int mpc52xx_lpbfifo_submit(struct mpc52xx_lpbfifo_request *req); -extern void mpc52xx_lpbfifo_abort(struct mpc52xx_lpbfifo_request *req); -extern void mpc52xx_lpbfifo_poll(void); -extern int mpc52xx_lpbfifo_start_xfer(struct mpc52xx_lpbfifo_request *req); - /* mpc52xx_pic.c */ extern void mpc52xx_init_irq(void); extern unsigned int mpc52xx_get_irq(void); @@ -338,7 +297,7 @@ extern void __init mpc52xx_setup_pci(void); static inline void mpc52xx_setup_pci(void) { } #endif -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #ifdef CONFIG_PM struct mpc52xx_suspend { @@ -350,14 +309,14 @@ extern struct mpc52xx_suspend mpc52xx_suspend; extern int __init mpc52xx_pm_init(void); extern int mpc52xx_set_wakeup_gpio(u8 pin, u8 level); -#ifdef CONFIG_PPC_LITE5200 -extern int __init lite5200_pm_init(void); - /* lite5200 calls mpc5200 suspend functions, so here they are */ extern int mpc52xx_pm_prepare(void); extern int mpc52xx_pm_enter(suspend_state_t); extern void mpc52xx_pm_finish(void); extern char saved_sram[0x4000]; /* reuse buffer from mpc52xx suspend */ + +#ifdef CONFIG_PPC_LITE5200 +int __init lite5200_pm_init(void); #endif #endif /* CONFIG_PM */ diff --git a/arch/powerpc/include/asm/mpc52xx_psc.h b/arch/powerpc/include/asm/mpc52xx_psc.h index d0ece257d310..ec995b289280 100644 --- a/arch/powerpc/include/asm/mpc52xx_psc.h +++ b/arch/powerpc/include/asm/mpc52xx_psc.h @@ -150,7 +150,10 @@ /* Structure of the hardware registers */ struct mpc52xx_psc { - u8 mode; /* PSC + 0x00 */ + union { + u8 mode; /* PSC + 0x00 */ + u8 mr2; + }; u8 reserved0[3]; union { /* PSC + 0x04 */ u16 status; @@ -258,8 +261,6 @@ struct mpc52xx_psc_fifo { #define MPC512x_PSC_FIFO_FULL 0x2 #define MPC512x_PSC_FIFO_ALARM 0x4 #define MPC512x_PSC_FIFO_URERR 0x8 -#define MPC512x_PSC_FIFO_ORERR 0x01 -#define MPC512x_PSC_FIFO_MEMERROR 0x02 struct mpc512x_psc_fifo { u32 reserved1[10]; diff --git a/arch/powerpc/include/asm/mpc5xxx.h b/arch/powerpc/include/asm/mpc5xxx.h index 5ce9c5fa434a..44db26380435 100644 --- a/arch/powerpc/include/asm/mpc5xxx.h +++ b/arch/powerpc/include/asm/mpc5xxx.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved. * @@ -5,18 +6,19 @@ * * Description: * MPC5xxx Prototypes and definitions - * - * This is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * */ #ifndef __ASM_POWERPC_MPC5xxx_H__ #define __ASM_POWERPC_MPC5xxx_H__ -extern unsigned long mpc5xxx_get_bus_frequency(struct device_node *node); +#include <linux/property.h> + +unsigned long mpc5xxx_fwnode_get_bus_frequency(struct fwnode_handle *fwnode); + +static inline unsigned long mpc5xxx_get_bus_frequency(struct device *dev) +{ + return mpc5xxx_fwnode_get_bus_frequency(dev_fwnode(dev)); +} #endif /* __ASM_POWERPC_MPC5xxx_H__ */ diff --git a/arch/powerpc/include/asm/mpc6xx.h b/arch/powerpc/include/asm/mpc6xx.h index effc2291beb2..6ed9f4ccc7b9 100644 --- a/arch/powerpc/include/asm/mpc6xx.h +++ b/arch/powerpc/include/asm/mpc6xx.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __ASM_POWERPC_MPC6xx_H #define __ASM_POWERPC_MPC6xx_H diff --git a/arch/powerpc/include/asm/mpc8260.h b/arch/powerpc/include/asm/mpc8260.h deleted file mode 100644 index 03317e1e6185..000000000000 --- a/arch/powerpc/include/asm/mpc8260.h +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Since there are many different boards and no standard configuration, - * we have a unique include file for each. Rather than change every - * file that has to include MPC8260 configuration, they all include - * this one and the configuration switching is done here. - */ -#ifdef __KERNEL__ -#ifndef __ASM_POWERPC_MPC8260_H__ -#define __ASM_POWERPC_MPC8260_H__ - -#define MPC82XX_BCR_PLDP 0x00800000 /* Pipeline Maximum Depth */ - -#ifdef CONFIG_8260 - -#if defined(CONFIG_PQ2ADS) || defined (CONFIG_PQ2FADS) -#include <platforms/82xx/pq2ads.h> -#endif - -#ifdef CONFIG_PCI_8260 -#include <platforms/82xx/m82xx_pci.h> -#endif - -#endif /* CONFIG_8260 */ -#endif /* !__ASM_POWERPC_MPC8260_H__ */ -#endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/mpc85xx.h b/arch/powerpc/include/asm/mpc85xx.h new file mode 100644 index 000000000000..21aabc323015 --- /dev/null +++ b/arch/powerpc/include/asm/mpc85xx.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * MPC85xx cpu type detection + * + * Copyright 2011-2012 Freescale Semiconductor, Inc. + */ + +#ifndef __ASM_PPC_MPC85XX_H +#define __ASM_PPC_MPC85XX_H + +#define SVR_REV(svr) ((svr) & 0xFF) /* SOC design resision */ +#define SVR_MAJ(svr) (((svr) >> 4) & 0xF) /* Major revision field*/ +#define SVR_MIN(svr) (((svr) >> 0) & 0xF) /* Minor revision field*/ + +/* Some parts define SVR[0:23] as the SOC version */ +#define SVR_SOC_VER(svr) (((svr) >> 8) & 0xFFF7FF) /* SOC Version fields */ + +#define SVR_8533 0x803400 +#define SVR_8535 0x803701 +#define SVR_8536 0x803700 +#define SVR_8540 0x803000 +#define SVR_8541 0x807200 +#define SVR_8543 0x803200 +#define SVR_8544 0x803401 +#define SVR_8545 0x803102 +#define SVR_8547 0x803101 +#define SVR_8548 0x803100 +#define SVR_8555 0x807100 +#define SVR_8560 0x807000 +#define SVR_8567 0x807501 +#define SVR_8568 0x807500 +#define SVR_8569 0x808000 +#define SVR_8572 0x80E000 +#define SVR_P1010 0x80F100 +#define SVR_P1011 0x80E500 +#define SVR_P1012 0x80E501 +#define SVR_P1013 0x80E700 +#define SVR_P1014 0x80F101 +#define SVR_P1017 0x80F700 +#define SVR_P1020 0x80E400 +#define SVR_P1021 0x80E401 +#define SVR_P1022 0x80E600 +#define SVR_P1023 0x80F600 +#define SVR_P1024 0x80E402 +#define SVR_P1025 0x80E403 +#define SVR_P2010 0x80E300 +#define SVR_P2020 0x80E200 +#define SVR_P2040 0x821000 +#define SVR_P2041 0x821001 +#define SVR_P3041 0x821103 +#define SVR_P4040 0x820100 +#define SVR_P4080 0x820000 +#define SVR_P5010 0x822100 +#define SVR_P5020 0x822000 +#define SVR_P5021 0X820500 +#define SVR_P5040 0x820400 +#define SVR_T4240 0x824000 +#define SVR_T4120 0x824001 +#define SVR_T4160 0x824100 +#define SVR_T4080 0x824102 +#define SVR_C291 0x850000 +#define SVR_C292 0x850020 +#define SVR_C293 0x850030 +#define SVR_B4860 0X868000 +#define SVR_G4860 0x868001 +#define SVR_G4060 0x868003 +#define SVR_B4440 0x868100 +#define SVR_G4440 0x868101 +#define SVR_B4420 0x868102 +#define SVR_B4220 0x868103 +#define SVR_T1040 0x852000 +#define SVR_T1041 0x852001 +#define SVR_T1042 0x852002 +#define SVR_T1020 0x852100 +#define SVR_T1021 0x852101 +#define SVR_T1022 0x852102 +#define SVR_T2080 0x853000 +#define SVR_T2081 0x853100 + +#define SVR_8610 0x80A000 +#define SVR_8641 0x809000 +#define SVR_8641D 0x809001 + +#define SVR_9130 0x860001 +#define SVR_9131 0x860000 +#define SVR_9132 0x861000 +#define SVR_9232 0x861400 + +#define SVR_Unknown 0xFFFFFF + +#endif diff --git a/arch/powerpc/include/asm/mpc8xx.h b/arch/powerpc/include/asm/mpc8xx.h deleted file mode 100644 index 98f3c4f17328..000000000000 --- a/arch/powerpc/include/asm/mpc8xx.h +++ /dev/null @@ -1,12 +0,0 @@ -/* This is the single file included by all MPC8xx build options. - * Since there are many different boards and no standard configuration, - * we have a unique include file for each. Rather than change every - * file that has to include MPC8xx configuration, they all include - * this one and the configuration switching is done here. - */ -#ifndef __CONFIG_8xx_DEFS -#define __CONFIG_8xx_DEFS - -extern struct mpc8xx_pcmcia_ops m8xx_pcmcia_ops; - -#endif /* __CONFIG_8xx_DEFS */ diff --git a/arch/powerpc/include/asm/mpic.h b/arch/powerpc/include/asm/mpic.h index 4a1ac9fbf186..0c03a98986cd 100644 --- a/arch/powerpc/include/asm/mpic.h +++ b/arch/powerpc/include/asm/mpic.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_MPIC_H #define _ASM_POWERPC_MPIC_H #ifdef __KERNEL__ @@ -34,10 +35,6 @@ #define MPIC_GREG_GCONF_BASE_MASK 0x000fffff #define MPIC_GREG_GCONF_MCK 0x08000000 #define MPIC_GREG_GLOBAL_CONF_1 0x00030 -#define MPIC_GREG_GLOBAL_CONF_1_SIE 0x08000000 -#define MPIC_GREG_GLOBAL_CONF_1_CLK_RATIO_MASK 0x70000000 -#define MPIC_GREG_GLOBAL_CONF_1_CLK_RATIO(r) \ - (((r) << 28) & MPIC_GREG_GLOBAL_CONF_1_CLK_RATIO_MASK) #define MPIC_GREG_VENDOR_0 0x00040 #define MPIC_GREG_VENDOR_1 0x00050 #define MPIC_GREG_VENDOR_2 0x00060 @@ -339,7 +336,7 @@ struct mpic #endif }; -extern struct bus_type mpic_subsys; +extern const struct bus_type mpic_subsys; /* * MPIC flags (passed to mpic_alloc) @@ -396,7 +393,14 @@ extern struct bus_type mpic_subsys; #define MPIC_REGSET_TSI108 MPIC_REGSET(1) /* Tsi108/109 PIC */ /* Get the version of primary MPIC */ +#ifdef CONFIG_MPIC extern u32 fsl_mpic_primary_get_version(void); +#else +static inline u32 fsl_mpic_primary_get_version(void) +{ + return 0; +} +#endif /* Allocate the controller structure and setup the linux irq descs * for the range if interrupts passed in. No HW initialization is @@ -468,7 +472,7 @@ extern int mpic_cpu_get_priority(void); extern void mpic_cpu_set_priority(int prio); /* Request IPIs on primary mpic */ -extern void mpic_request_ipis(void); +void __init mpic_request_ipis(void); /* Send a message (IPI) to a given target (cpu number or MSG_*) */ void smp_mpic_message_pass(int target, int msg); @@ -489,11 +493,5 @@ extern unsigned int mpic_get_coreint_irq(void); /* Fetch Machine Check interrupt from primary mpic */ extern unsigned int mpic_get_mcirq(void); -/* Set the EPIC clock ratio */ -void mpic_set_clk_ratio(struct mpic *mpic, u32 clock_ratio); - -/* Enable/Disable EPIC serial interrupt mode */ -void mpic_set_serial_int(struct mpic *mpic, int enable); - #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_MPIC_H */ diff --git a/arch/powerpc/include/asm/mpic_msgr.h b/arch/powerpc/include/asm/mpic_msgr.h index d4f471fb1031..cd25eeced208 100644 --- a/arch/powerpc/include/asm/mpic_msgr.h +++ b/arch/powerpc/include/asm/mpic_msgr.h @@ -1,11 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright 2011-2012, Meador Inge, Mentor Graphics Corporation. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; version 2 of the - * License. - * */ #ifndef _ASM_MPIC_MSGR_H @@ -122,9 +117,9 @@ static inline void mpic_msgr_set_destination(struct mpic_msgr *msgr, * @msgr: the message register whose IRQ is to be returned * * Returns the IRQ number associated with the given message register. - * NO_IRQ is returned if this message register is not capable of - * receiving interrupts. What message register can and cannot receive - * interrupts is specified in the device tree for the system. + * 0 is returned if this message register is not capable of receiving + * interrupts. What message register can and cannot receive interrupts is + * specified in the device tree for the system. */ static inline int mpic_msgr_get_irq(struct mpic_msgr *msgr) { diff --git a/arch/powerpc/include/asm/mpic_timer.h b/arch/powerpc/include/asm/mpic_timer.h index 0e23cd4ac8aa..d33e4149be17 100644 --- a/arch/powerpc/include/asm/mpic_timer.h +++ b/arch/powerpc/include/asm/mpic_timer.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * arch/powerpc/include/asm/mpic_timer.h * @@ -7,11 +8,6 @@ * * Author: Wang Dongsheng <Dongsheng.Wang@freescale.com> * Li Yang <leoli@freescale.com> - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. */ #ifndef __MPIC_TIMER__ @@ -29,17 +25,17 @@ struct mpic_timer { #ifdef CONFIG_MPIC_TIMER struct mpic_timer *mpic_request_timer(irq_handler_t fn, void *dev, - const struct timeval *time); + time64_t time); void mpic_start_timer(struct mpic_timer *handle); void mpic_stop_timer(struct mpic_timer *handle); -void mpic_get_remain_time(struct mpic_timer *handle, struct timeval *time); +void mpic_get_remain_time(struct mpic_timer *handle, time64_t *time); void mpic_free_timer(struct mpic_timer *handle); #else struct mpic_timer *mpic_request_timer(irq_handler_t fn, void *dev, - const struct timeval *time) { return NULL; } + time64_t time) { return NULL; } void mpic_start_timer(struct mpic_timer *handle) { } void mpic_stop_timer(struct mpic_timer *handle) { } -void mpic_get_remain_time(struct mpic_timer *handle, struct timeval *time) { } +void mpic_get_remain_time(struct mpic_timer *handle, time64_t *time) { } void mpic_free_timer(struct mpic_timer *handle) { } #endif diff --git a/arch/powerpc/include/asm/msi_bitmap.h b/arch/powerpc/include/asm/msi_bitmap.h index 97ac3f46ae0d..55c2f7db9cbd 100644 --- a/arch/powerpc/include/asm/msi_bitmap.h +++ b/arch/powerpc/include/asm/msi_bitmap.h @@ -1,14 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ #ifndef _POWERPC_SYSDEV_MSI_BITMAP_H #define _POWERPC_SYSDEV_MSI_BITMAP_H /* * Copyright 2008, Michael Ellerman, IBM Corporation. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; version 2 of the - * License. - * */ #include <linux/of.h> @@ -19,6 +14,7 @@ struct msi_bitmap { unsigned long *bitmap; spinlock_t lock; unsigned int irq_count; + bool bitmap_from_slab; }; int msi_bitmap_alloc_hwirqs(struct msi_bitmap *bmp, int num); diff --git a/arch/powerpc/include/asm/mutex.h b/arch/powerpc/include/asm/mutex.h deleted file mode 100644 index 127ab23e1f6c..000000000000 --- a/arch/powerpc/include/asm/mutex.h +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Optimised mutex implementation of include/asm-generic/mutex-dec.h algorithm - */ -#ifndef _ASM_POWERPC_MUTEX_H -#define _ASM_POWERPC_MUTEX_H - -static inline int __mutex_cmpxchg_lock(atomic_t *v, int old, int new) -{ - int t; - - __asm__ __volatile__ ( -"1: lwarx %0,0,%1 # mutex trylock\n\ - cmpw 0,%0,%2\n\ - bne- 2f\n" - PPC405_ERR77(0,%1) -" stwcx. %3,0,%1\n\ - bne- 1b" - PPC_ACQUIRE_BARRIER - "\n\ -2:" - : "=&r" (t) - : "r" (&v->counter), "r" (old), "r" (new) - : "cc", "memory"); - - return t; -} - -static inline int __mutex_dec_return_lock(atomic_t *v) -{ - int t; - - __asm__ __volatile__( -"1: lwarx %0,0,%1 # mutex lock\n\ - addic %0,%0,-1\n" - PPC405_ERR77(0,%1) -" stwcx. %0,0,%1\n\ - bne- 1b" - PPC_ACQUIRE_BARRIER - : "=&r" (t) - : "r" (&v->counter) - : "cc", "memory"); - - return t; -} - -static inline int __mutex_inc_return_unlock(atomic_t *v) -{ - int t; - - __asm__ __volatile__( - PPC_RELEASE_BARRIER -"1: lwarx %0,0,%1 # mutex unlock\n\ - addic %0,%0,1\n" - PPC405_ERR77(0,%1) -" stwcx. %0,0,%1 \n\ - bne- 1b" - : "=&r" (t) - : "r" (&v->counter) - : "cc", "memory"); - - return t; -} - -/** - * __mutex_fastpath_lock - try to take the lock by moving the count - * from 1 to a 0 value - * @count: pointer of type atomic_t - * @fail_fn: function to call if the original value was not 1 - * - * Change the count from 1 to a value lower than 1, and call <fail_fn> if - * it wasn't 1 originally. This function MUST leave the value lower than - * 1 even when the "1" assertion wasn't true. - */ -static inline void -__mutex_fastpath_lock(atomic_t *count, void (*fail_fn)(atomic_t *)) -{ - if (unlikely(__mutex_dec_return_lock(count) < 0)) - fail_fn(count); -} - -/** - * __mutex_fastpath_lock_retval - try to take the lock by moving the count - * from 1 to a 0 value - * @count: pointer of type atomic_t - * - * Change the count from 1 to a value lower than 1. This function returns 0 - * if the fastpath succeeds, or -1 otherwise. - */ -static inline int -__mutex_fastpath_lock_retval(atomic_t *count) -{ - if (unlikely(__mutex_dec_return_lock(count) < 0)) - return -1; - return 0; -} - -/** - * __mutex_fastpath_unlock - try to promote the count from 0 to 1 - * @count: pointer of type atomic_t - * @fail_fn: function to call if the original value was not 0 - * - * Try to promote the count from 0 to 1. If it wasn't 0, call <fail_fn>. - * In the failure case, this function is allowed to either set the value to - * 1, or to set it to a value lower than 1. - */ -static inline void -__mutex_fastpath_unlock(atomic_t *count, void (*fail_fn)(atomic_t *)) -{ - if (unlikely(__mutex_inc_return_unlock(count) <= 0)) - fail_fn(count); -} - -#define __mutex_slowpath_needs_to_unlock() 1 - -/** - * __mutex_fastpath_trylock - try to acquire the mutex, without waiting - * - * @count: pointer of type atomic_t - * @fail_fn: fallback function - * - * Change the count from 1 to 0, and return 1 (success), or if the count - * was not 1, then return 0 (failure). - */ -static inline int -__mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *)) -{ - if (likely(__mutex_cmpxchg_lock(count, 1, 0) == 1)) - return 1; - return 0; -} - -#endif diff --git a/arch/powerpc/include/asm/nmi.h b/arch/powerpc/include/asm/nmi.h new file mode 100644 index 000000000000..49a75340c3e0 --- /dev/null +++ b/arch/powerpc/include/asm/nmi.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_NMI_H +#define _ASM_NMI_H + +#ifdef CONFIG_PPC_WATCHDOG +long soft_nmi_interrupt(struct pt_regs *regs); +void watchdog_hardlockup_set_timeout_pct(u64 pct); +#else +static inline void watchdog_hardlockup_set_timeout_pct(u64 pct) {} +#endif + +extern void hv_nmi_check_nonrecoverable(struct pt_regs *regs); + +#endif /* _ASM_NMI_H */ diff --git a/arch/powerpc/include/asm/nohash/32/hugetlb-8xx.h b/arch/powerpc/include/asm/nohash/32/hugetlb-8xx.h new file mode 100644 index 000000000000..014799557f60 --- /dev/null +++ b/arch/powerpc/include/asm/nohash/32/hugetlb-8xx.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_NOHASH_32_HUGETLB_8XX_H +#define _ASM_POWERPC_NOHASH_32_HUGETLB_8XX_H + +#define PAGE_SHIFT_8M 23 + +static inline void flush_hugetlb_page(struct vm_area_struct *vma, + unsigned long vmaddr) +{ + flush_tlb_page(vma, vmaddr); +} + +static inline int check_and_get_huge_psize(int shift) +{ + return shift_to_mmu_psize(shift); +} + +#define __HAVE_ARCH_HUGE_SET_HUGE_PTE_AT +void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, + pte_t pte, unsigned long sz); + +#define __HAVE_ARCH_HUGE_PTEP_GET +static inline pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep) +{ + if (ptep_is_8m_pmdp(mm, addr, ptep)) + ptep = pte_offset_kernel((pmd_t *)ptep, ALIGN_DOWN(addr, SZ_8M)); + return ptep_get(ptep); +} + +#define __HAVE_ARCH_HUGE_PTE_CLEAR +static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned long sz) +{ + pte_update(mm, addr, ptep, ~0UL, 0, 1); +} + +#define __HAVE_ARCH_HUGE_PTEP_SET_WRPROTECT +static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + unsigned long clr = ~pte_val(pte_wrprotect(__pte(~0))); + unsigned long set = pte_val(pte_wrprotect(__pte(0))); + + pte_update(mm, addr, ptep, clr, set, 1); +} + +#ifdef CONFIG_PPC_4K_PAGES +static inline pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags) +{ + size_t size = 1UL << shift; + + if (size == SZ_16K) + return __pte(pte_val(entry) | _PAGE_SPS); + else + return __pte(pte_val(entry) | _PAGE_SPS | _PAGE_HUGE); +} +#define arch_make_huge_pte arch_make_huge_pte +#endif + +#endif /* _ASM_POWERPC_NOHASH_32_HUGETLB_8XX_H */ diff --git a/arch/powerpc/include/asm/nohash/32/kup-8xx.h b/arch/powerpc/include/asm/nohash/32/kup-8xx.h new file mode 100644 index 000000000000..08486b15b207 --- /dev/null +++ b/arch/powerpc/include/asm/nohash/32/kup-8xx.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_KUP_8XX_H_ +#define _ASM_POWERPC_KUP_8XX_H_ + +#include <asm/bug.h> +#include <asm/mmu.h> + +#ifdef CONFIG_PPC_KUAP + +#ifndef __ASSEMBLER__ + +#include <asm/reg.h> + +static __always_inline void __kuap_save_and_lock(struct pt_regs *regs) +{ + regs->kuap = mfspr(SPRN_MD_AP); + mtspr(SPRN_MD_AP, MD_APG_KUAP); +} +#define __kuap_save_and_lock __kuap_save_and_lock + +static __always_inline void kuap_user_restore(struct pt_regs *regs) +{ +} + +static __always_inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long kuap) +{ + mtspr(SPRN_MD_AP, regs->kuap); +} + +#ifdef CONFIG_PPC_KUAP_DEBUG +static __always_inline unsigned long __kuap_get_and_assert_locked(void) +{ + WARN_ON_ONCE(mfspr(SPRN_MD_AP) >> 16 != MD_APG_KUAP >> 16); + + return 0; +} +#define __kuap_get_and_assert_locked __kuap_get_and_assert_locked +#endif + +static __always_inline void uaccess_begin_8xx(unsigned long val) +{ + asm(ASM_MMU_FTR_IFSET("mtspr %0, %1", "", %2) : : + "i"(SPRN_MD_AP), "r"(val), "i"(MMU_FTR_KUAP) : "memory"); +} + +static __always_inline void uaccess_end_8xx(void) +{ + asm(ASM_MMU_FTR_IFSET("mtspr %0, %1", "", %2) : : + "i"(SPRN_MD_AP), "r"(MD_APG_KUAP), "i"(MMU_FTR_KUAP) : "memory"); +} + +static __always_inline void allow_user_access(void __user *to, const void __user *from, + unsigned long size, unsigned long dir) +{ + uaccess_begin_8xx(MD_APG_INIT); +} + +static __always_inline void prevent_user_access(unsigned long dir) +{ + uaccess_end_8xx(); +} + +static __always_inline unsigned long prevent_user_access_return(void) +{ + unsigned long flags; + + flags = mfspr(SPRN_MD_AP); + + uaccess_end_8xx(); + + return flags; +} + +static __always_inline void restore_user_access(unsigned long flags) +{ + uaccess_begin_8xx(flags); +} + +static __always_inline bool +__bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write) +{ + return !((regs->kuap ^ MD_APG_KUAP) & 0xff000000); +} + +#endif /* !__ASSEMBLER__ */ + +#endif /* CONFIG_PPC_KUAP */ + +#endif /* _ASM_POWERPC_KUP_8XX_H_ */ diff --git a/arch/powerpc/include/asm/mmu-44x.h b/arch/powerpc/include/asm/nohash/32/mmu-44x.h index bf52d704fc47..c3d192194324 100644 --- a/arch/powerpc/include/asm/mmu-44x.h +++ b/arch/powerpc/include/asm/nohash/32/mmu-44x.h @@ -1,10 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_MMU_44X_H_ #define _ASM_POWERPC_MMU_44X_H_ /* * PPC440 support */ -#include <asm/page.h> +#include <asm/asm-const.h> #define PPC44x_MMUCR_TID 0x000000ff #define PPC44x_MMUCR_STS 0x00010000 @@ -99,7 +100,7 @@ #define PPC47x_TLB2_S_RW (PPC47x_TLB2_SW | PPC47x_TLB2_SR) #define PPC47x_TLB2_IMG (PPC47x_TLB2_I | PPC47x_TLB2_M | PPC47x_TLB2_G) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ extern unsigned int tlb_44x_hwater; extern unsigned int tlb_44x_index; @@ -107,10 +108,13 @@ extern unsigned int tlb_44x_index; typedef struct { unsigned int id; unsigned int active; - unsigned long vdso_base; + void __user *vdso; } mm_context_t; -#endif /* !__ASSEMBLY__ */ +/* patch sites */ +extern s32 patch__tlb_44x_hwater_D, patch__tlb_44x_hwater_I; + +#endif /* !__ASSEMBLER__ */ #ifndef CONFIG_PPC_EARLY_DEBUG_44x #define PPC44x_EARLY_TLBS 1 @@ -123,19 +127,19 @@ typedef struct { /* Size of the TLBs used for pinning in lowmem */ #define PPC_PIN_SIZE (1 << 28) /* 256M */ -#if (PAGE_SHIFT == 12) +#if defined(CONFIG_PPC_4K_PAGES) #define PPC44x_TLBE_SIZE PPC44x_TLB_4K #define PPC47x_TLBE_SIZE PPC47x_TLB0_4K #define mmu_virtual_psize MMU_PAGE_4K -#elif (PAGE_SHIFT == 14) +#elif defined(CONFIG_PPC_16K_PAGES) #define PPC44x_TLBE_SIZE PPC44x_TLB_16K #define PPC47x_TLBE_SIZE PPC47x_TLB0_16K #define mmu_virtual_psize MMU_PAGE_16K -#elif (PAGE_SHIFT == 16) +#elif defined(CONFIG_PPC_64K_PAGES) #define PPC44x_TLBE_SIZE PPC44x_TLB_64K #define PPC47x_TLBE_SIZE PPC47x_TLB0_64K #define mmu_virtual_psize MMU_PAGE_64K -#elif (PAGE_SHIFT == 18) +#elif defined(CONFIG_PPC_256K_PAGES) #define PPC44x_TLBE_SIZE PPC44x_TLB_256K #define mmu_virtual_psize MMU_PAGE_256K #else diff --git a/arch/powerpc/include/asm/mmu-8xx.h b/arch/powerpc/include/asm/nohash/32/mmu-8xx.h index 3d11d3ce79ec..f19115db8072 100644 --- a/arch/powerpc/include/asm/mmu-8xx.h +++ b/arch/powerpc/include/asm/nohash/32/mmu-8xx.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_MMU_8XX_H_ #define _ASM_POWERPC_MMU_8XX_H_ /* @@ -18,7 +19,6 @@ #define MI_RSV4I 0x08000000 /* Reserve 4 TLB entries */ #define MI_PPCS 0x02000000 /* Use MI_RPN prob/priv state */ #define MI_IDXMASK 0x00001f00 /* TLB index to be loaded */ -#define MI_RESETVAL 0x00000000 /* Value of register at reset */ /* These are the Ks and Kp from the PowerPC books. For proper operation, * Ks = 0, Kp = 1. @@ -27,6 +27,23 @@ #define MI_Ks 0x80000000 /* Should not be set */ #define MI_Kp 0x40000000 /* Should always be set */ +/* + * All pages' PP data bits are set to either 001 or 011 by copying _PAGE_EXEC + * into bit 21 in the ITLBmiss handler (bit 21 is the middle bit), which means + * respectively NA for All or X for Supervisor and no access for User. + * Then we use the APG to say whether accesses are according to Page rules or + * "all Supervisor" rules (Access to all) + * _PAGE_ACCESSED is also managed via APG. When _PAGE_ACCESSED is not set, say + * "all User" rules, that will lead to NA for all. + * Therefore, we define 4 APG groups. lsb is _PAGE_ACCESSED + * 0 => Kernel => 11 (all accesses performed according as user iaw page definition) + * 1 => Kernel+Accessed => 01 (all accesses performed according to page definition) + * 2 => User => 11 (all accesses performed according as user iaw page definition) + * 3 => User+Accessed => 10 (all accesses performed according to swaped page definition) for KUEP + * 4-15 => Not Used + */ +#define MI_APG_INIT 0xde000000 + /* The effective page number register. When read, contains the information * about the last instruction TLB miss. When MI_RPN is written, bits in * this register are used to create the TLB entry. @@ -56,6 +73,7 @@ * additional information from the MI_EPN, and MI_TWC registers. */ #define SPRN_MI_RPN 790 +#define MI_SPS16K 0x00000008 /* Small page size (0 = 4k, 1 = 16k) */ /* Define an RPN value for mapping kernel memory to large virtual * pages for boot initialization. This has real page number of 0, @@ -73,7 +91,6 @@ #define MD_TWAM 0x04000000 /* Use 4K page hardware assist */ #define MD_PPCS 0x02000000 /* Use MI_RPN prob/priv state */ #define MD_IDXMASK 0x00001f00 /* TLB index to be loaded */ -#define MD_RESETVAL 0x04000000 /* Value of register at reset */ #define SPRN_M_CASID 793 /* Address space ID (context) to match */ #define MC_ASIDMASK 0x0000000f /* Bits used for ASID value */ @@ -86,6 +103,10 @@ #define MD_Ks 0x80000000 /* Should not be set */ #define MD_Kp 0x40000000 /* Should always be set */ +/* See explanation above at the definition of MI_APG_INIT */ +#define MD_APG_INIT 0xdc000000 +#define MD_APG_KUAP 0xde000000 + /* The effective page number register. When read, contains the information * about the last instruction TLB miss. When MD_RPN is written, bits in * this register are used to create the TLB entry. @@ -129,21 +150,121 @@ * additional information from the MD_EPN, and MD_TWC registers. */ #define SPRN_MD_RPN 798 +#define MD_SPS16K 0x00000008 /* Small page size (0 = 4k, 1 = 16k) */ /* This is a temporary storage register that could be used to save * a processor working register during a tablewalk. */ #define SPRN_M_TW 799 -#ifndef __ASSEMBLY__ +#if defined(CONFIG_PPC_4K_PAGES) +#define mmu_virtual_psize MMU_PAGE_4K +#elif defined(CONFIG_PPC_16K_PAGES) +#define mmu_virtual_psize MMU_PAGE_16K +#define PTE_FRAG_NR 4 +#define PTE_FRAG_SIZE_SHIFT 12 +#define PTE_FRAG_SIZE (1UL << 12) +#else +#error "Unsupported PAGE_SIZE" +#endif + +#define mmu_linear_psize MMU_PAGE_8M + +#define MODULES_END PAGE_OFFSET +#define MODULES_SIZE (CONFIG_MODULES_SIZE * SZ_1M) +#define MODULES_VADDR (MODULES_END - MODULES_SIZE) + +#ifndef __ASSEMBLER__ + +#include <linux/mmdebug.h> +#include <linux/sizes.h> + +void mmu_pin_tlb(unsigned long top, bool readonly); + typedef struct { unsigned int id; unsigned int active; - unsigned long vdso_base; + void __user *vdso; + void *pte_frag; } mm_context_t; -#endif /* !__ASSEMBLY__ */ -#define mmu_virtual_psize MMU_PAGE_4K -#define mmu_linear_psize MMU_PAGE_8M +#define PHYS_IMMR_BASE (mfspr(SPRN_IMMR) & 0xfff80000) + +/* + * Page size definitions for 8xx + * + * shift : is the "PAGE_SHIFT" value for that page size + * + */ +struct mmu_psize_def { + unsigned int shift; /* number of bits */ +}; + +extern struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT]; + +static inline int shift_to_mmu_psize(unsigned int shift) +{ + int psize; + + for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) + if (mmu_psize_defs[psize].shift == shift) + return psize; + return -1; +} + +static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize) +{ + if (mmu_psize_defs[mmu_psize].shift) + return mmu_psize_defs[mmu_psize].shift; + BUG(); +} + +static inline bool arch_vmap_try_size(unsigned long addr, unsigned long end, u64 pfn, + unsigned int max_page_shift, unsigned long size) +{ + if (end - addr < size) + return false; + + if ((1UL << max_page_shift) < size) + return false; + + if (!IS_ALIGNED(addr, size)) + return false; + + if (!IS_ALIGNED(PFN_PHYS(pfn), size)) + return false; + + return true; +} + +static inline unsigned long arch_vmap_pte_range_map_size(unsigned long addr, unsigned long end, + u64 pfn, unsigned int max_page_shift) +{ + if (arch_vmap_try_size(addr, end, pfn, max_page_shift, SZ_512K)) + return SZ_512K; + if (PAGE_SIZE == SZ_16K) + return SZ_16K; + if (arch_vmap_try_size(addr, end, pfn, max_page_shift, SZ_16K)) + return SZ_16K; + return PAGE_SIZE; +} +#define arch_vmap_pte_range_map_size arch_vmap_pte_range_map_size + +static inline int arch_vmap_pte_supported_shift(unsigned long size) +{ + if (size >= SZ_512K) + return 19; + else if (size >= SZ_16K) + return 14; + else + return PAGE_SHIFT; +} +#define arch_vmap_pte_supported_shift arch_vmap_pte_supported_shift + +/* patch sites */ +extern s32 patch__itlbmiss_exit_1, patch__dtlbmiss_exit_1; +extern s32 patch__itlbmiss_perf, patch__dtlbmiss_perf; + +#endif /* !__ASSEMBLER__ */ #endif /* _ASM_POWERPC_MMU_8XX_H_ */ diff --git a/arch/powerpc/include/asm/nohash/32/pgalloc.h b/arch/powerpc/include/asm/nohash/32/pgalloc.h new file mode 100644 index 000000000000..11eac371e7e0 --- /dev/null +++ b/arch/powerpc/include/asm/nohash/32/pgalloc.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_PGALLOC_32_H +#define _ASM_POWERPC_PGALLOC_32_H + +#include <linux/threads.h> +#include <linux/slab.h> + +/* + * We don't have any real pmd's, and this code never triggers because + * the pgd will always be present.. + */ +/* #define pmd_alloc_one(mm,address) ({ BUG(); ((pmd_t *)2); }) */ +#define pmd_free(mm, x) do { } while (0) +#define __pmd_free_tlb(tlb,x,a) do { } while (0) +/* #define pgd_populate(mm, pmd, pte) BUG() */ + +static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, + pte_t *pte) +{ + if (IS_ENABLED(CONFIG_BOOKE)) + *pmdp = __pmd((unsigned long)pte | _PMD_PRESENT); + else + *pmdp = __pmd(__pa(pte) | _PMD_PRESENT); +} + +static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, + pgtable_t pte_page) +{ + if (IS_ENABLED(CONFIG_BOOKE)) + *pmdp = __pmd((unsigned long)pte_page | _PMD_PRESENT); + else + *pmdp = __pmd(__pa(pte_page) | _PMD_USER | _PMD_PRESENT); +} + +#endif /* _ASM_POWERPC_PGALLOC_32_H */ diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h new file mode 100644 index 000000000000..2d71e4b7cd09 --- /dev/null +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h @@ -0,0 +1,204 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_NOHASH_32_PGTABLE_H +#define _ASM_POWERPC_NOHASH_32_PGTABLE_H + +#include <asm-generic/pgtable-nopmd.h> + +#ifndef __ASSEMBLER__ +#include <linux/sched.h> +#include <linux/threads.h> +#include <asm/mmu.h> /* For sub-arch specific PPC_PIN_SIZE */ + +#endif /* __ASSEMBLER__ */ + +#define PTE_INDEX_SIZE PTE_SHIFT +#define PMD_INDEX_SIZE 0 +#define PUD_INDEX_SIZE 0 +#define PGD_INDEX_SIZE (32 - PGDIR_SHIFT) + +#define PMD_CACHE_INDEX PMD_INDEX_SIZE +#define PUD_CACHE_INDEX PUD_INDEX_SIZE + +#ifndef __ASSEMBLER__ +#define PTE_TABLE_SIZE (sizeof(pte_t) << PTE_INDEX_SIZE) +#define PMD_TABLE_SIZE 0 +#define PUD_TABLE_SIZE 0 +#define PGD_TABLE_SIZE (sizeof(pgd_t) << PGD_INDEX_SIZE) + +#define PMD_MASKED_BITS (PTE_TABLE_SIZE - 1) +#endif /* __ASSEMBLER__ */ + +#define PTRS_PER_PTE (1 << PTE_INDEX_SIZE) +#define PTRS_PER_PGD (1 << PGD_INDEX_SIZE) + +/* + * The normal case is that PTEs are 32-bits and we have a 1-page + * 1024-entry pgdir pointing to 1-page 1024-entry PTE pages. -- paulus + * + * For any >32-bit physical address platform, we can use the following + * two level page table layout where the pgdir is 8KB and the MS 13 bits + * are an index to the second level table. The combined pgdir/pmd first + * level has 2048 entries and the second level has 512 64-bit PTE entries. + * -Matt + */ +/* PGDIR_SHIFT determines what a top-level page table entry can map */ +#define PGDIR_SHIFT (PAGE_SHIFT + PTE_INDEX_SIZE) +#define PGDIR_SIZE (1UL << PGDIR_SHIFT) +#define PGDIR_MASK (~(PGDIR_SIZE-1)) + +/* Bits to mask out from a PGD to get to the PUD page */ +#define PGD_MASKED_BITS 0 + +#define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE) + +#define pgd_ERROR(e) \ + pr_err("%s:%d: bad pgd %08llx.\n", __FILE__, __LINE__, (unsigned long long)pgd_val(e)) + +/* + * This is the bottom of the PKMAP area with HIGHMEM or an arbitrary + * value (for now) on others, from where we can start layout kernel + * virtual space that goes below PKMAP and FIXMAP + */ + +#define FIXADDR_SIZE 0 +#ifdef CONFIG_KASAN +#include <asm/kasan.h> +#define FIXADDR_TOP (KASAN_SHADOW_START - PAGE_SIZE) +#else +#define FIXADDR_TOP ((unsigned long)(-PAGE_SIZE)) +#endif + +/* + * ioremap_bot starts at that address. Early ioremaps move down from there, + * until mem_init() at which point this becomes the top of the vmalloc + * and ioremap space + */ +#ifdef CONFIG_HIGHMEM +#define IOREMAP_TOP PKMAP_BASE +#else +#define IOREMAP_TOP FIXADDR_START +#endif + +/* PPC32 shares vmalloc area with ioremap */ +#define IOREMAP_START VMALLOC_START +#define IOREMAP_END VMALLOC_END + +/* + * Just any arbitrary offset to the start of the vmalloc VM area: the + * current 16MB value just means that there will be a 64MB "hole" after the + * physical memory until the kernel virtual memory starts. That means that + * any out-of-bounds memory accesses will hopefully be caught. + * The vmalloc() routines leaves a hole of 4kB between each vmalloced + * area for the same reason. ;) + * + * We no longer map larger than phys RAM with the BATs so we don't have + * to worry about the VMALLOC_OFFSET causing problems. We do have to worry + * about clashes between our early calls to ioremap() that start growing down + * from IOREMAP_TOP being run into the VM area allocations (growing upwards + * from VMALLOC_START). For this reason we have ioremap_bot to check when + * we actually run into our mappings setup in the early boot with the VM + * system. This really does become a problem for machines with good amounts + * of RAM. -- Cort + */ +#define VMALLOC_OFFSET (0x1000000) /* 16M */ +#ifdef PPC_PIN_SIZE +#define VMALLOC_START (((ALIGN((long)high_memory, PPC_PIN_SIZE) + VMALLOC_OFFSET) & ~(VMALLOC_OFFSET-1))) +#else +#define VMALLOC_START ((((long)high_memory + VMALLOC_OFFSET) & ~(VMALLOC_OFFSET-1))) +#endif + +#ifdef CONFIG_KASAN_VMALLOC +#define VMALLOC_END ALIGN_DOWN(ioremap_bot, PAGE_SIZE << KASAN_SHADOW_SCALE_SHIFT) +#else +#define VMALLOC_END ioremap_bot +#endif + +/* + * Bits in a linux-style PTE. These match the bits in the + * (hardware-defined) PowerPC PTE as closely as possible. + */ + +#if defined(CONFIG_44x) +#include <asm/nohash/32/pte-44x.h> +#elif defined(CONFIG_PPC_85xx) && defined(CONFIG_PTE_64BIT) +#include <asm/nohash/pte-e500.h> +#elif defined(CONFIG_PPC_85xx) +#include <asm/nohash/32/pte-85xx.h> +#elif defined(CONFIG_PPC_8xx) +#include <asm/nohash/32/pte-8xx.h> +#endif + +/* + * Location of the PFN in the PTE. Most 32-bit platforms use the same + * as _PAGE_SHIFT here (ie, naturally aligned). + * Platform who don't just pre-define the value so we don't override it here. + */ +#ifndef PTE_RPN_SHIFT +#define PTE_RPN_SHIFT (PAGE_SHIFT) +#endif + +/* + * The mask covered by the RPN must be a ULL on 32-bit platforms with + * 64-bit PTEs. + */ +#ifdef CONFIG_PTE_64BIT +#define PTE_RPN_MASK (~((1ULL << PTE_RPN_SHIFT) - 1)) +#define MAX_POSSIBLE_PHYSMEM_BITS 36 +#else +#define PTE_RPN_MASK (~((1UL << PTE_RPN_SHIFT) - 1)) +#define MAX_POSSIBLE_PHYSMEM_BITS 32 +#endif + +#ifndef __ASSEMBLER__ + +#define pmd_none(pmd) (!pmd_val(pmd)) +#define pmd_bad(pmd) (pmd_val(pmd) & _PMD_BAD) +#define pmd_present(pmd) (pmd_val(pmd) & _PMD_PRESENT_MASK) +static inline void pmd_clear(pmd_t *pmdp) +{ + *pmdp = __pmd(0); +} + +/* + * Note that on Book E processors, the pmd contains the kernel virtual + * (lowmem) address of the pte page. The physical address is less useful + * because everything runs with translation enabled (even the TLB miss + * handler). On everything else the pmd contains the physical address + * of the pte page. -- paulus + */ +#ifndef CONFIG_BOOKE +#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT) +#else +#define pmd_page_vaddr(pmd) \ + ((const void *)((unsigned long)pmd_val(pmd) & ~(PTE_TABLE_SIZE - 1))) +#define pmd_pfn(pmd) (__pa(pmd_val(pmd)) >> PAGE_SHIFT) +#endif + +#define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd)) + +/* + * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that + * are !pte_none() && !pte_present(). + * + * Format of swap PTEs (32bit PTEs): + * + * 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 3 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * <------------------ offset -------------------> < type -> E 0 0 + * + * E is the exclusive marker that is not stored in swap entries. + * + * For 64bit PTEs, the offset is extended by 32bit. + */ +#define __swp_type(entry) ((entry).val & 0x1f) +#define __swp_offset(entry) ((entry).val >> 5) +#define __swp_entry(type, offset) ((swp_entry_t) { ((type) & 0x1f) | ((offset) << 5) }) +#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) >> 3 }) +#define __swp_entry_to_pte(x) ((pte_t) { (x).val << 3 }) + +/* We borrow LSB 2 to store the exclusive marker in swap PTEs. */ +#define _PAGE_SWP_EXCLUSIVE 0x000004 + +#endif /* !__ASSEMBLER__ */ + +#endif /* __ASM_POWERPC_NOHASH_32_PGTABLE_H */ diff --git a/arch/powerpc/include/asm/pte-44x.h b/arch/powerpc/include/asm/nohash/32/pte-44x.h index 4192b9bad901..da0469928273 100644 --- a/arch/powerpc/include/asm/pte-44x.h +++ b/arch/powerpc/include/asm/nohash/32/pte-44x.h @@ -1,5 +1,6 @@ -#ifndef _ASM_POWERPC_PTE_44x_H -#define _ASM_POWERPC_PTE_44x_H +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_NOHASH_32_PTE_44x_H +#define _ASM_POWERPC_NOHASH_32_PTE_44x_H #ifdef __KERNEL__ /* @@ -32,7 +33,7 @@ * - - - - - - U0 U1 U2 U3 W I M G E - UX UW UR SX SW SR * * Newer 440 cores (440x6 as used on AMCC 460EX/460GT) have additional - * TLB2 storage attibute fields. Those are: + * TLB2 storage attribute fields. Those are: * * TLB2: * 0...10 11 12 13 14 15 16...31 @@ -44,9 +45,6 @@ * - PRESENT *must* be in the bottom three bits because swap cache * entries use the top 29 bits for TLB2. * - * - FILE *must* be in the bottom three bits because swap cache - * entries use the top 29 bits for TLB2. - * * - CACHE COHERENT bit (M) has no effect on original PPC440 cores, * because it doesn't support SMP. However, some later 460 variants * have -some- form of SMP support and so I keep the bit there for @@ -58,31 +56,19 @@ * above bits. Note that the bit values are CPU specific, not architecture * specific. * - * The kernel PTE entry holds an arch-dependent swp_entry structure under - * certain situations. In other words, in such situations some portion of - * the PTE bits are used as a swp_entry. In the PPC implementation, the - * 3-24th LSB are shared with swp_entry, however the 0-2nd three LSB still - * hold protection values. That means the three protection bits are - * reserved for both PTE and SWAP entry at the most significant three - * LSBs. - * - * There are three protection bits available for SWAP entry: - * _PAGE_PRESENT - * _PAGE_FILE - * _PAGE_HASHPTE (if HW has) - * - * So those three bits have to be inside of 0-2nd LSB of PTE. - * + * The kernel PTE entry can be an ordinary PTE mapping a page or a special swap + * PTE. In case of a swap PTE, LSB 2-24 are used to store information regarding + * the swap entry. However LSB 0-1 still hold protection values, for example, + * to distinguish swap PTEs from ordinary PTEs, and must be used with care. */ #define _PAGE_PRESENT 0x00000001 /* S: PTE valid */ -#define _PAGE_RW 0x00000002 /* S: Write permission */ -#define _PAGE_FILE 0x00000004 /* S: nonlinear file mapping */ +#define _PAGE_WRITE 0x00000002 /* S: Write permission */ #define _PAGE_EXEC 0x00000004 /* H: Execute permission */ -#define _PAGE_ACCESSED 0x00000008 /* S: Page referenced */ +#define _PAGE_READ 0x00000008 /* S: Read permission */ #define _PAGE_DIRTY 0x00000010 /* S: Page dirty */ #define _PAGE_SPECIAL 0x00000020 /* S: Special page */ -#define _PAGE_USER 0x00000040 /* S: User page */ +#define _PAGE_ACCESSED 0x00000040 /* S: Page referenced */ #define _PAGE_ENDIAN 0x00000080 /* H: E bit */ #define _PAGE_GUARDED 0x00000100 /* H: G bit */ #define _PAGE_COHERENT 0x00000200 /* H: M bit */ @@ -93,10 +79,25 @@ #define _PMD_PRESENT 0 #define _PMD_PRESENT_MASK (PAGE_MASK) #define _PMD_BAD (~PAGE_MASK) +#define _PMD_USER 0 /* ERPN in a PTE never gets cleared, ignore it */ #define _PTE_NONE_MASK 0xffffffff00000000ULL +/* + * We define 2 sets of base prot bits, one for basic pages (ie, + * cacheable kernel and user pages) and one for non cacheable + * pages. We always set _PAGE_COHERENT when SMP is enabled or + * the processor might need it for DMA coherency. + */ +#define _PAGE_BASE_NC (_PAGE_PRESENT | _PAGE_ACCESSED) +#if defined(CONFIG_SMP) +#define _PAGE_BASE (_PAGE_BASE_NC | _PAGE_COHERENT) +#else +#define _PAGE_BASE (_PAGE_BASE_NC) +#endif + +#include <asm/pgtable-masks.h> #endif /* __KERNEL__ */ -#endif /* _ASM_POWERPC_PTE_44x_H */ +#endif /* _ASM_POWERPC_NOHASH_32_PTE_44x_H */ diff --git a/arch/powerpc/include/asm/nohash/32/pte-85xx.h b/arch/powerpc/include/asm/nohash/32/pte-85xx.h new file mode 100644 index 000000000000..14d64b4f3f14 --- /dev/null +++ b/arch/powerpc/include/asm/nohash/32/pte-85xx.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_NOHASH_32_PTE_85xx_H +#define _ASM_POWERPC_NOHASH_32_PTE_85xx_H +#ifdef __KERNEL__ + +/* PTE bit definitions for Freescale BookE SW loaded TLB MMU based + * processors + * + MMU Assist Register 3: + + 32 33 34 35 36 ... 50 51 52 53 54 55 56 57 58 59 60 61 62 63 + RPN...................... 0 0 U0 U1 U2 U3 UX SX UW SW UR SR + + - PRESENT *must* be in the bottom two bits because swap PTEs use + the top 30 bits. + +*/ + +/* Definitions for FSL Book-E Cores */ +#define _PAGE_READ 0x00001 /* H: Read permission (SR) */ +#define _PAGE_PRESENT 0x00002 /* S: PTE contains a translation */ +#define _PAGE_WRITE 0x00004 /* S: Write permission (SW) */ +#define _PAGE_DIRTY 0x00008 /* S: Page dirty */ +#define _PAGE_EXEC 0x00010 /* H: SX permission */ +#define _PAGE_ACCESSED 0x00020 /* S: Page referenced */ + +#define _PAGE_ENDIAN 0x00040 /* H: E bit */ +#define _PAGE_GUARDED 0x00080 /* H: G bit */ +#define _PAGE_COHERENT 0x00100 /* H: M bit */ +#define _PAGE_NO_CACHE 0x00200 /* H: I bit */ +#define _PAGE_WRITETHRU 0x00400 /* H: W bit */ +#define _PAGE_SPECIAL 0x00800 /* S: Special page */ + +#define _PMD_PRESENT 0 +#define _PMD_PRESENT_MASK (PAGE_MASK) +#define _PMD_BAD (~PAGE_MASK) +#define _PMD_USER 0 + +#define _PTE_NONE_MASK 0 + +#define PTE_WIMGE_SHIFT (6) + +/* + * We define 2 sets of base prot bits, one for basic pages (ie, + * cacheable kernel and user pages) and one for non cacheable + * pages. We always set _PAGE_COHERENT when SMP is enabled or + * the processor might need it for DMA coherency. + */ +#define _PAGE_BASE_NC (_PAGE_PRESENT | _PAGE_ACCESSED) +#if defined(CONFIG_SMP) || defined(CONFIG_PPC_E500MC) +#define _PAGE_BASE (_PAGE_BASE_NC | _PAGE_COHERENT) +#else +#define _PAGE_BASE (_PAGE_BASE_NC) +#endif + +#include <asm/pgtable-masks.h> + +#endif /* __KERNEL__ */ +#endif /* _ASM_POWERPC_NOHASH_32_PTE_FSL_85xx_H */ diff --git a/arch/powerpc/include/asm/nohash/32/pte-8xx.h b/arch/powerpc/include/asm/nohash/32/pte-8xx.h new file mode 100644 index 000000000000..e2ea8ba9f8ca --- /dev/null +++ b/arch/powerpc/include/asm/nohash/32/pte-8xx.h @@ -0,0 +1,241 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_NOHASH_32_PTE_8xx_H +#define _ASM_POWERPC_NOHASH_32_PTE_8xx_H +#ifdef __KERNEL__ + +/* + * The PowerPC MPC8xx uses a TLB with hardware assisted, software tablewalk. + * We also use the two level tables, but we can put the real bits in them + * needed for the TLB and tablewalk. These definitions require Mx_CTR.PPM = 0, + * Mx_CTR.PPCS = 0, and MD_CTR.TWAM = 1. The level 2 descriptor has + * additional page protection (when Mx_CTR.PPCS = 1) that allows TLB hit + * based upon user/super access. The TLB does not have accessed nor write + * protect. We assume that if the TLB get loaded with an entry it is + * accessed, and overload the changed bit for write protect. We use + * two bits in the software pte that are supposed to be set to zero in + * the TLB entry (24 and 25) for these indicators. Although the level 1 + * descriptor contains the guarded and writethrough/copyback bits, we can + * set these at the page level since they get copied from the Mx_TWC + * register when the TLB entry is loaded. We will use bit 27 for guard, since + * that is where it exists in the MD_TWC, and bit 26 for writethrough. + * These will get masked from the level 2 descriptor at TLB load time, and + * copied to the MD_TWC before it gets loaded. + * Large page sizes added. We currently support two sizes, 4K and 8M. + * This also allows a TLB hander optimization because we can directly + * load the PMD into MD_TWC. The 8M pages are only used for kernel + * mapping of well known areas. The PMD (PGD) entries contain control + * flags in addition to the address, so care must be taken that the + * software no longer assumes these are only pointers. + */ + +/* Definitions for 8xx embedded chips. */ +#define _PAGE_PRESENT 0x0001 /* V: Page is valid */ +#define _PAGE_NO_CACHE 0x0002 /* CI: cache inhibit */ +#define _PAGE_SH 0x0004 /* SH: No ASID (context) compare */ +#define _PAGE_SPS 0x0008 /* SPS: Small Page Size (1 if 16k, 512k or 8M)*/ +#define _PAGE_DIRTY 0x0100 /* C: page changed */ + +/* These 4 software bits must be masked out when the L2 entry is loaded + * into the TLB. + */ +#define _PAGE_GUARDED 0x0010 /* Copied to L1 G entry in DTLB */ +#define _PAGE_ACCESSED 0x0020 /* Copied to L1 APG 1 entry in I/DTLB */ +#define _PAGE_EXEC 0x0040 /* Copied to PP (bit 21) in ITLB */ +#define _PAGE_SPECIAL 0x0080 /* SW entry */ + +#define _PAGE_NA 0x0200 /* Supervisor NA, User no access */ +#define _PAGE_RO 0x0600 /* Supervisor RO, User no access */ + +#define _PAGE_HUGE 0x0800 /* Copied to L1 PS bit 29 */ + +#define _PAGE_NAX (_PAGE_NA | _PAGE_EXEC) +#define _PAGE_ROX (_PAGE_RO | _PAGE_EXEC) +#define _PAGE_RW 0 +#define _PAGE_RWX _PAGE_EXEC + +/* cache related flags non existing on 8xx */ +#define _PAGE_COHERENT 0 +#define _PAGE_WRITETHRU 0 + +#define _PAGE_KERNEL_RO (_PAGE_SH | _PAGE_RO) +#define _PAGE_KERNEL_ROX (_PAGE_SH | _PAGE_RO | _PAGE_EXEC) +#define _PAGE_KERNEL_RW (_PAGE_SH | _PAGE_DIRTY) +#define _PAGE_KERNEL_RWX (_PAGE_SH | _PAGE_DIRTY | _PAGE_EXEC) + +#define _PMD_PRESENT 0x0001 +#define _PMD_PRESENT_MASK _PMD_PRESENT +#define _PMD_BAD 0x0f90 +#define _PMD_PAGE_MASK 0x000c +#define _PMD_PAGE_8M 0x000c +#define _PMD_PAGE_512K 0x0004 +#define _PMD_ACCESSED 0x0020 /* APG 1 */ +#define _PMD_USER 0x0040 /* APG 2 */ + +#define _PTE_NONE_MASK 0 + +#ifdef CONFIG_PPC_16K_PAGES +#define _PAGE_BASE_NC (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_SPS) +#else +#define _PAGE_BASE_NC (_PAGE_PRESENT | _PAGE_ACCESSED) +#endif + +#define _PAGE_BASE (_PAGE_BASE_NC) + +#include <asm/pgtable-masks.h> + +#ifndef __ASSEMBLER__ +static inline pte_t pte_wrprotect(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_RO); +} + +#define pte_wrprotect pte_wrprotect + +static inline int pte_read(pte_t pte) +{ + return (pte_val(pte) & _PAGE_RO) != _PAGE_NA; +} + +#define pte_read pte_read + +static inline int pte_write(pte_t pte) +{ + return !(pte_val(pte) & _PAGE_RO); +} + +#define pte_write pte_write + +static inline pte_t pte_mkwrite_novma(pte_t pte) +{ + return __pte(pte_val(pte) & ~_PAGE_RO); +} + +#define pte_mkwrite_novma pte_mkwrite_novma + +static inline pte_t pte_mkhuge(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_SPS | _PAGE_HUGE); +} + +#define pte_mkhuge pte_mkhuge + +static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, pte_t *ptep, + unsigned long clr, unsigned long set, int huge); + +static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) +{ + pte_update(mm, addr, ptep, 0, _PAGE_RO, 0); +} +#define ptep_set_wrprotect ptep_set_wrprotect + +static inline void __ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep, + pte_t entry, unsigned long address, int psize) +{ + unsigned long set = pte_val(entry) & (_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_EXEC); + unsigned long clr = ~pte_val(entry) & _PAGE_RO; + int huge = psize > mmu_virtual_psize ? 1 : 0; + + pte_update(vma->vm_mm, address, ptep, clr, set, huge); + + flush_tlb_page(vma, address); +} +#define __ptep_set_access_flags __ptep_set_access_flags + +static inline unsigned long __pte_leaf_size(pmd_t pmd, pte_t pte) +{ + pte_basic_t val = pte_val(pte); + + if (pmd_val(pmd) & _PMD_PAGE_8M) + return SZ_8M; + if (val & _PAGE_HUGE) + return SZ_512K; + if (val & _PAGE_SPS) + return SZ_16K; + return SZ_4K; +} + +#define __pte_leaf_size __pte_leaf_size + +/* + * On the 8xx, the page tables are a bit special. For 16k pages, we have + * 4 identical entries. For 512k pages, we have 128 entries as if it was + * 4k pages, but they are flagged as 512k pages for the hardware. + * For 8M pages, we have 1024 entries as if it was 4M pages (PMD_SIZE) + * but they are flagged as 8M pages for the hardware. + * For 4k pages, we have a single entry in the table. + */ +static pmd_t *pmd_off(struct mm_struct *mm, unsigned long addr); +static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address); + +static inline bool ptep_is_8m_pmdp(struct mm_struct *mm, unsigned long addr, pte_t *ptep) +{ + return (pmd_t *)ptep == pmd_off(mm, ALIGN_DOWN(addr, SZ_8M)); +} + +static inline int number_of_cells_per_pte(pmd_t *pmd, pte_basic_t val, int huge) +{ + if (!huge) + return PAGE_SIZE / SZ_4K; + else if ((pmd_val(*pmd) & _PMD_PAGE_MASK) == _PMD_PAGE_8M) + return SZ_4M / SZ_4K; + else if (IS_ENABLED(CONFIG_PPC_4K_PAGES) && !(val & _PAGE_HUGE)) + return SZ_16K / SZ_4K; + else + return SZ_512K / SZ_4K; +} + +static inline pte_basic_t __pte_update(struct mm_struct *mm, unsigned long addr, pte_t *p, + unsigned long clr, unsigned long set, int huge) +{ + pte_basic_t *entry = (pte_basic_t *)p; + pte_basic_t old = pte_val(*p); + pte_basic_t new = (old & ~(pte_basic_t)clr) | set; + int num, i; + pmd_t *pmd = pmd_off(mm, addr); + + num = number_of_cells_per_pte(pmd, new, huge); + + for (i = 0; i < num; i += PAGE_SIZE / SZ_4K, new += PAGE_SIZE) { + *entry++ = new; + if (IS_ENABLED(CONFIG_PPC_16K_PAGES)) { + *entry++ = new; + *entry++ = new; + *entry++ = new; + } + } + + return old; +} + +static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, pte_t *ptep, + unsigned long clr, unsigned long set, int huge) +{ + pte_basic_t old; + + if (huge && ptep_is_8m_pmdp(mm, addr, ptep)) { + pmd_t *pmdp = (pmd_t *)ptep; + + old = __pte_update(mm, addr, pte_offset_kernel(pmdp, 0), clr, set, huge); + __pte_update(mm, addr, pte_offset_kernel(pmdp + 1, 0), clr, set, huge); + } else { + old = __pte_update(mm, addr, ptep, clr, set, huge); + } + return old; +} +#define pte_update pte_update + +#ifdef CONFIG_PPC_16K_PAGES +#define ptep_get ptep_get +static inline pte_t ptep_get(pte_t *ptep) +{ + pte_basic_t val = READ_ONCE(ptep->pte); + pte_t pte = {val, val, val, val}; + + return pte; +} +#endif /* CONFIG_PPC_16K_PAGES */ + +#endif + +#endif /* __KERNEL__ */ +#endif /* _ASM_POWERPC_NOHASH_32_PTE_8xx_H */ diff --git a/arch/powerpc/include/asm/nohash/64/pgalloc.h b/arch/powerpc/include/asm/nohash/64/pgalloc.h new file mode 100644 index 000000000000..e50b211becb3 --- /dev/null +++ b/arch/powerpc/include/asm/nohash/64/pgalloc.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_POWERPC_PGALLOC_64_H +#define _ASM_POWERPC_PGALLOC_64_H +/* + */ + +#include <linux/slab.h> +#include <linux/cpumask.h> +#include <linux/percpu.h> + +struct vmemmap_backing { + struct vmemmap_backing *list; + unsigned long phys; + unsigned long virt_addr; +}; +extern struct vmemmap_backing *vmemmap_list; + +static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) +{ + p4d_set(p4d, (unsigned long)pud); +} + +static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) +{ + return kmem_cache_alloc(PGT_CACHE(PUD_INDEX_SIZE), + pgtable_gfp_flags(mm, GFP_KERNEL)); +} + +static inline void pud_free(struct mm_struct *mm, pud_t *pud) +{ + kmem_cache_free(PGT_CACHE(PUD_INDEX_SIZE), pud); +} + +static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) +{ + pud_set(pud, (unsigned long)pmd); +} + +static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, + pte_t *pte) +{ + pmd_set(pmd, (unsigned long)pte); +} + +static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, + pgtable_t pte_page) +{ + pmd_set(pmd, (unsigned long)pte_page); +} + +static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) +{ + return kmem_cache_alloc(PGT_CACHE(PMD_CACHE_INDEX), + pgtable_gfp_flags(mm, GFP_KERNEL)); +} + +static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) +{ + kmem_cache_free(PGT_CACHE(PMD_CACHE_INDEX), pmd); +} + +#define __pmd_free_tlb(tlb, pmd, addr) \ + pgtable_free_tlb(tlb, pmd, PMD_CACHE_INDEX) +#define __pud_free_tlb(tlb, pud, addr) \ + pgtable_free_tlb(tlb, pud, PUD_INDEX_SIZE) + +#endif /* _ASM_POWERPC_PGALLOC_64_H */ diff --git a/arch/powerpc/include/asm/pgtable-ppc64-4k.h b/arch/powerpc/include/asm/nohash/64/pgtable-4k.h index 12798c9d4b4b..fb6fa1d4e074 100644 --- a/arch/powerpc/include/asm/pgtable-ppc64-4k.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable-4k.h @@ -1,5 +1,9 @@ -#ifndef _ASM_POWERPC_PGTABLE_PPC64_4K_H -#define _ASM_POWERPC_PGTABLE_PPC64_4K_H +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_NOHASH_64_PGTABLE_4K_H +#define _ASM_POWERPC_NOHASH_64_PGTABLE_4K_H + +#include <asm-generic/pgtable-nop4d.h> + /* * Entries per page directory level. The PTE level must use a 64b record * for each page table entry. The PMD and PGD level use a 32b record for @@ -10,12 +14,12 @@ #define PUD_INDEX_SIZE 9 #define PGD_INDEX_SIZE 9 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define PTE_TABLE_SIZE (sizeof(pte_t) << PTE_INDEX_SIZE) #define PMD_TABLE_SIZE (sizeof(pmd_t) << PMD_INDEX_SIZE) #define PUD_TABLE_SIZE (sizeof(pud_t) << PUD_INDEX_SIZE) #define PGD_TABLE_SIZE (sizeof(pgd_t) << PGD_INDEX_SIZE) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #define PTRS_PER_PTE (1 << PTE_INDEX_SIZE) #define PTRS_PER_PMD (1 << PMD_INDEX_SIZE) @@ -27,9 +31,6 @@ #define PMD_SIZE (1UL << PMD_SHIFT) #define PMD_MASK (~(PMD_SIZE-1)) -/* With 4k base page size, hugepage PTEs go at the PMD level */ -#define MIN_HUGEPTE_SHIFT PMD_SHIFT - /* PUD_SHIFT determines what a third-level page table entry can map */ #define PUD_SHIFT (PMD_SHIFT + PMD_INDEX_SIZE) #define PUD_SIZE (1UL << PUD_SHIFT) @@ -44,31 +45,49 @@ #define PMD_MASKED_BITS 0 /* Bits to mask out from a PUD to get to the PMD page */ #define PUD_MASKED_BITS 0 -/* Bits to mask out from a PGD to get to the PUD page */ -#define PGD_MASKED_BITS 0 +/* Bits to mask out from a P4D to get to the PUD page */ +#define P4D_MASKED_BITS 0 /* * 4-level page tables related bits */ -#define pgd_none(pgd) (!pgd_val(pgd)) -#define pgd_bad(pgd) (pgd_val(pgd) == 0) -#define pgd_present(pgd) (pgd_val(pgd) != 0) -#define pgd_clear(pgdp) (pgd_val(*(pgdp)) = 0) -#define pgd_page_vaddr(pgd) (pgd_val(pgd) & ~PGD_MASKED_BITS) -#define pgd_page(pgd) virt_to_page(pgd_page_vaddr(pgd)) +#define p4d_none(p4d) (!p4d_val(p4d)) +#define p4d_bad(p4d) (p4d_val(p4d) == 0) +#define p4d_present(p4d) (p4d_val(p4d) != 0) + +#ifndef __ASSEMBLER__ + +static inline pud_t *p4d_pgtable(p4d_t p4d) +{ + return (pud_t *) (p4d_val(p4d) & ~P4D_MASKED_BITS); +} + +static inline void p4d_clear(p4d_t *p4dp) +{ + *p4dp = __p4d(0); +} + +static inline pte_t p4d_pte(p4d_t p4d) +{ + return __pte(p4d_val(p4d)); +} + +static inline p4d_t pte_p4d(pte_t pte) +{ + return __p4d(pte_val(pte)); +} +extern struct page *p4d_page(p4d_t p4d); -#define pud_offset(pgdp, addr) \ - (((pud_t *) pgd_page_vaddr(*(pgdp))) + \ - (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))) +#endif /* !__ASSEMBLER__ */ #define pud_ERROR(e) \ - printk("%s:%d: bad pud %08lx.\n", __FILE__, __LINE__, pud_val(e)) + pr_err("%s:%d: bad pud %08lx.\n", __FILE__, __LINE__, pud_val(e)) /* * On all 4K setups, remap_4k_pfn() equates to remap_pfn_range() */ #define remap_4k_pfn(vma, addr, pfn, prot) \ remap_pfn_range((vma), (addr), (pfn), PAGE_SIZE, (prot)) -#endif /* _ASM_POWERPC_PGTABLE_PPC64_4K_H */ +#endif /* _ _ASM_POWERPC_NOHASH_64_PGTABLE_4K_H */ diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h new file mode 100644 index 000000000000..2deb955b7bc8 --- /dev/null +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h @@ -0,0 +1,214 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_NOHASH_64_PGTABLE_H +#define _ASM_POWERPC_NOHASH_64_PGTABLE_H +/* + * This file contains the functions and defines necessary to modify and use + * the ppc64 non-hashed page table. + */ + +#include <linux/sizes.h> + +#include <asm/nohash/64/pgtable-4k.h> +#include <asm/barrier.h> +#include <asm/asm-const.h> + +/* + * Size of EA range mapped by our pagetables. + */ +#define PGTABLE_EADDR_SIZE (PTE_INDEX_SIZE + PMD_INDEX_SIZE + \ + PUD_INDEX_SIZE + PGD_INDEX_SIZE + PAGE_SHIFT) +#define PGTABLE_RANGE (ASM_CONST(1) << PGTABLE_EADDR_SIZE) + +#define PMD_CACHE_INDEX PMD_INDEX_SIZE +#define PUD_CACHE_INDEX PUD_INDEX_SIZE + +/* + * Define the address range of the kernel non-linear virtual area + */ +#define KERN_VIRT_START ASM_CONST(0xc000100000000000) +#define KERN_VIRT_SIZE ASM_CONST(0x0000100000000000) + +/* + * The vmalloc space starts at the beginning of that region, and + * occupies a quarter of it on Book3E + * (we keep a quarter for the virtual memmap) + */ +#define VMALLOC_START KERN_VIRT_START +#define VMALLOC_SIZE (KERN_VIRT_SIZE >> 2) +#define VMALLOC_END (VMALLOC_START + VMALLOC_SIZE) + +/* + * The third quarter of the kernel virtual space is used for IO mappings, + * it's itself carved into the PIO region (ISA and PHB IO space) and + * the ioremap space + * + * ISA_IO_BASE = KERN_IO_START, 64K reserved area + * PHB_IO_BASE = ISA_IO_BASE + 64K to ISA_IO_BASE + 2G, PHB IO spaces + * IOREMAP_BASE = ISA_IO_BASE + 2G to KERN_IO_START + KERN_IO_SIZE + */ +#define KERN_IO_START (KERN_VIRT_START + (KERN_VIRT_SIZE >> 1)) +#define KERN_IO_SIZE (KERN_VIRT_SIZE >> 2) +#define FULL_IO_SIZE 0x80000000ul +#define ISA_IO_BASE (KERN_IO_START) +#define ISA_IO_END (KERN_IO_START + 0x10000ul) +#define PHB_IO_BASE (ISA_IO_END) +#define PHB_IO_END (KERN_IO_START + FULL_IO_SIZE) +#define IOREMAP_BASE (PHB_IO_END) +#define IOREMAP_START (ioremap_bot) +#define IOREMAP_END (KERN_IO_START + KERN_IO_SIZE - FIXADDR_SIZE) +#define FIXADDR_SIZE SZ_32M +#define FIXADDR_TOP (IOREMAP_END + FIXADDR_SIZE) + +/* + * Defines the address of the vmemap area, in its own region on + * after the vmalloc space on Book3E + */ +#define VMEMMAP_BASE VMALLOC_END +#define VMEMMAP_END KERN_IO_START +#define vmemmap ((struct page *)VMEMMAP_BASE) + + +/* + * Include the PTE bits definitions + */ +#include <asm/nohash/pte-e500.h> + +#define PTE_RPN_MASK (~((1UL << PTE_RPN_SHIFT) - 1)) + +#define H_PAGE_4K_PFN 0 + +#ifndef __ASSEMBLER__ +/* pte_clear moved to later in this file */ + +#define PMD_BAD_BITS (PTE_TABLE_SIZE-1) +#define PUD_BAD_BITS (PMD_TABLE_SIZE-1) + +static inline void pmd_set(pmd_t *pmdp, unsigned long val) +{ + *pmdp = __pmd(val); +} + +static inline void pmd_clear(pmd_t *pmdp) +{ + *pmdp = __pmd(0); +} + +static inline pte_t pmd_pte(pmd_t pmd) +{ + return __pte(pmd_val(pmd)); +} + +#define pmd_none(pmd) (!pmd_val(pmd)) +#define pmd_bad(pmd) (!is_kernel_addr(pmd_val(pmd)) \ + || (pmd_val(pmd) & PMD_BAD_BITS)) +#define pmd_present(pmd) (!pmd_none(pmd)) +#define pmd_page_vaddr(pmd) ((const void *)(pmd_val(pmd) & ~PMD_MASKED_BITS)) +extern struct page *pmd_page(pmd_t pmd); +#define pmd_pfn(pmd) (page_to_pfn(pmd_page(pmd))) + +static inline void pud_set(pud_t *pudp, unsigned long val) +{ + *pudp = __pud(val); +} + +static inline void pud_clear(pud_t *pudp) +{ + *pudp = __pud(0); +} + +#define pud_none(pud) (!pud_val(pud)) +#define pud_bad(pud) (!is_kernel_addr(pud_val(pud)) \ + || (pud_val(pud) & PUD_BAD_BITS)) +#define pud_present(pud) (pud_val(pud) != 0) + +static inline pmd_t *pud_pgtable(pud_t pud) +{ + return (pmd_t *)(pud_val(pud) & ~PUD_MASKED_BITS); +} + +extern struct page *pud_page(pud_t pud); + +static inline pte_t pud_pte(pud_t pud) +{ + return __pte(pud_val(pud)); +} + +static inline pud_t pte_pud(pte_t pte) +{ + return __pud(pte_val(pte)); +} +#define pud_write(pud) pte_write(pud_pte(pud)) +#define p4d_write(pgd) pte_write(p4d_pte(p4d)) + +static inline void p4d_set(p4d_t *p4dp, unsigned long val) +{ + *p4dp = __p4d(val); +} + +#define __HAVE_ARCH_HUGE_PTEP_SET_WRPROTECT +static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 1); +} + +#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH +#define ptep_clear_flush_young(__vma, __address, __ptep) \ +({ \ + int __young = ptep_test_and_clear_young(__vma, __address, __ptep);\ + __young; \ +}) + +#define pmd_ERROR(e) \ + pr_err("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e)) +#define pgd_ERROR(e) \ + pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) + +/* + * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that + * are !pte_none() && !pte_present(). + * + * Format of swap PTEs: + * + * 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 3 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * <-------------------------- offset ---------------------------- + * + * 3 3 3 3 3 3 3 3 4 4 4 4 4 4 4 4 4 4 5 5 5 5 5 5 5 5 5 5 6 6 6 6 + * 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 + * --------------> <----------- zero ------------> E < type -> 0 0 + * + * E is the exclusive marker that is not stored in swap entries. + */ +#define MAX_SWAPFILES_CHECK() do { \ + BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS); \ + } while (0) + +#define SWP_TYPE_BITS 5 +#define __swp_type(x) (((x).val >> 2) \ + & ((1UL << SWP_TYPE_BITS) - 1)) +#define __swp_offset(x) ((x).val >> PTE_RPN_SHIFT) +#define __swp_entry(type, offset) ((swp_entry_t) { \ + (((type) & 0x1f) << 2) \ + | ((offset) << PTE_RPN_SHIFT) }) + +#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val((pte)) }) +#define __swp_entry_to_pte(x) __pte((x).val) + +/* We borrow MSB 56 (LSB 7) to store the exclusive marker in swap PTEs. */ +#define _PAGE_SWP_EXCLUSIVE 0x80 + +extern int __meminit vmemmap_create_mapping(unsigned long start, + unsigned long page_size, + unsigned long phys); +extern void vmemmap_remove_mapping(unsigned long start, + unsigned long page_size); +void __patch_exception(int exc, unsigned long addr); +#define patch_exception(exc, name) do { \ + extern unsigned int name; \ + __patch_exception((exc), (unsigned long)&name); \ +} while (0) + +#endif /* __ASSEMBLER__ */ + +#endif /* _ASM_POWERPC_NOHASH_64_PGTABLE_H */ diff --git a/arch/powerpc/include/asm/nohash/hugetlb-e500.h b/arch/powerpc/include/asm/nohash/hugetlb-e500.h new file mode 100644 index 000000000000..cab0e1f1eea0 --- /dev/null +++ b/arch/powerpc/include/asm/nohash/hugetlb-e500.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_NOHASH_HUGETLB_E500_H +#define _ASM_POWERPC_NOHASH_HUGETLB_E500_H + +void flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr); + +static inline int check_and_get_huge_psize(int shift) +{ + if (shift & 1) /* Not a power of 4 */ + return -EINVAL; + + return shift_to_mmu_psize(shift); +} + +static inline pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags) +{ + unsigned int tsize = shift - _PAGE_PSIZE_SHIFT_OFFSET; + pte_basic_t val = (tsize << _PAGE_PSIZE_SHIFT) & _PAGE_PSIZE_MSK; + + return __pte((pte_val(entry) & ~(pte_basic_t)_PAGE_PSIZE_MSK) | val); +} +#define arch_make_huge_pte arch_make_huge_pte + +#endif /* _ASM_POWERPC_NOHASH_HUGETLB_E500_H */ diff --git a/arch/powerpc/include/asm/nohash/kup-booke.h b/arch/powerpc/include/asm/nohash/kup-booke.h new file mode 100644 index 000000000000..d6bbb6d78bbe --- /dev/null +++ b/arch/powerpc/include/asm/nohash/kup-booke.h @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_KUP_BOOKE_H_ +#define _ASM_POWERPC_KUP_BOOKE_H_ + +#include <asm/bug.h> +#include <asm/mmu.h> + +#ifdef CONFIG_PPC_KUAP + +#ifdef __ASSEMBLER__ + +.macro kuap_check_amr gpr1, gpr2 +.endm + +#else + +#include <linux/sched.h> + +#include <asm/reg.h> + +static __always_inline void __kuap_lock(void) +{ + mtspr(SPRN_PID, 0); + isync(); +} +#define __kuap_lock __kuap_lock + +static __always_inline void __kuap_save_and_lock(struct pt_regs *regs) +{ + regs->kuap = mfspr(SPRN_PID); + mtspr(SPRN_PID, 0); + isync(); +} +#define __kuap_save_and_lock __kuap_save_and_lock + +static __always_inline void kuap_user_restore(struct pt_regs *regs) +{ + if (kuap_is_disabled()) + return; + + mtspr(SPRN_PID, current->thread.pid); + + /* Context synchronisation is performed by rfi */ +} + +static __always_inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long kuap) +{ + if (regs->kuap) + mtspr(SPRN_PID, current->thread.pid); + + /* Context synchronisation is performed by rfi */ +} + +#ifdef CONFIG_PPC_KUAP_DEBUG +static __always_inline unsigned long __kuap_get_and_assert_locked(void) +{ + WARN_ON_ONCE(mfspr(SPRN_PID)); + + return 0; +} +#define __kuap_get_and_assert_locked __kuap_get_and_assert_locked +#endif + +static __always_inline void uaccess_begin_booke(unsigned long val) +{ + asm(ASM_MMU_FTR_IFSET("mtspr %0, %1; isync", "", %2) : : + "i"(SPRN_PID), "r"(val), "i"(MMU_FTR_KUAP) : "memory"); +} + +static __always_inline void uaccess_end_booke(void) +{ + asm(ASM_MMU_FTR_IFSET("mtspr %0, %1; isync", "", %2) : : + "i"(SPRN_PID), "r"(0), "i"(MMU_FTR_KUAP) : "memory"); +} + +static __always_inline void allow_user_access(void __user *to, const void __user *from, + unsigned long size, unsigned long dir) +{ + uaccess_begin_booke(current->thread.pid); +} + +static __always_inline void prevent_user_access(unsigned long dir) +{ + uaccess_end_booke(); +} + +static __always_inline unsigned long prevent_user_access_return(void) +{ + unsigned long flags = mfspr(SPRN_PID); + + uaccess_end_booke(); + + return flags; +} + +static __always_inline void restore_user_access(unsigned long flags) +{ + if (flags) + uaccess_begin_booke(current->thread.pid); +} + +static __always_inline bool +__bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write) +{ + return !regs->kuap; +} + +#endif /* !__ASSEMBLER__ */ + +#endif /* CONFIG_PPC_KUAP */ + +#endif /* _ASM_POWERPC_KUP_BOOKE_H_ */ diff --git a/arch/powerpc/include/asm/mmu-book3e.h b/arch/powerpc/include/asm/nohash/mmu-e500.h index 936db360790a..2fad5ff426a0 100644 --- a/arch/powerpc/include/asm/mmu-book3e.h +++ b/arch/powerpc/include/asm/nohash/mmu-e500.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_MMU_BOOK3E_H_ #define _ASM_POWERPC_MMU_BOOK3E_H_ /* @@ -40,7 +41,11 @@ /* MAS registers bit definitions */ -#define MAS0_TLBSEL(x) (((x) << 28) & 0x30000000) +#define MAS0_TLBSEL_MASK 0x30000000 +#define MAS0_TLBSEL_SHIFT 28 +#define MAS0_TLBSEL(x) (((x) << MAS0_TLBSEL_SHIFT) & MAS0_TLBSEL_MASK) +#define MAS0_GET_TLBSEL(mas0) (((mas0) & MAS0_TLBSEL_MASK) >> \ + MAS0_TLBSEL_SHIFT) #define MAS0_ESEL_MASK 0x0FFF0000 #define MAS0_ESEL_SHIFT 16 #define MAS0_ESEL(x) (((x) << MAS0_ESEL_SHIFT) & MAS0_ESEL_MASK) @@ -58,6 +63,7 @@ #define MAS1_TSIZE_MASK 0x00000f80 #define MAS1_TSIZE_SHIFT 7 #define MAS1_TSIZE(x) (((x) << MAS1_TSIZE_SHIFT) & MAS1_TSIZE_MASK) +#define MAS1_GET_TSIZE(mas1) (((mas1) & MAS1_TSIZE_MASK) >> MAS1_TSIZE_SHIFT) #define MAS2_EPN (~0xFFFUL) #define MAS2_X0 0x00000040 @@ -69,7 +75,6 @@ #define MAS2_E 0x00000001 #define MAS2_WIMGE_MASK 0x0000001f #define MAS2_EPN_MASK(size) (~0 << (size + 10)) -#define MAS2_VAL(addr, size, flags) ((addr) & MAS2_EPN_MASK(size) | (flags)) #define MAS3_RPN 0xFFFFF000 #define MAS3_U0 0x00000200 @@ -86,6 +91,7 @@ #define MAS3_SPSIZE 0x0000003e #define MAS3_SPSIZE_SHIFT 1 +#define MAS4_TLBSEL_MASK MAS0_TLBSEL_MASK #define MAS4_TLBSELD(x) MAS0_TLBSEL(x) #define MAS4_INDD 0x00008000 /* Default IND */ #define MAS4_TSIZED(x) MAS1_TSIZE(x) @@ -214,7 +220,17 @@ #define TLBILX_T_CLASS2 6 #define TLBILX_T_CLASS3 7 -#ifndef __ASSEMBLY__ +/* + * The mapping only needs to be cache-coherent on SMP, except on + * Freescale e500mc derivatives where it's also needed for coherent DMA. + */ +#if defined(CONFIG_SMP) || defined(CONFIG_PPC_E500MC) +#define MAS2_M_IF_NEEDED MAS2_M +#else +#define MAS2_M_IF_NEEDED 0 +#endif + +#ifndef __ASSEMBLER__ #include <asm/bug.h> extern unsigned int tlbcam_index; @@ -222,33 +238,17 @@ extern unsigned int tlbcam_index; typedef struct { unsigned int id; unsigned int active; - unsigned long vdso_base; -#ifdef CONFIG_PPC_ICSWX - struct spinlock *cop_lockp; /* guard cop related stuff */ - unsigned long acop; /* mask of enabled coprocessor types */ -#endif /* CONFIG_PPC_ICSWX */ -#ifdef CONFIG_PPC_MM_SLICES - u64 low_slices_psize; /* SLB page size encodings */ - u64 high_slices_psize; /* 4 bits per slice for now */ - u16 user_psize; /* page size index */ -#endif -#ifdef CONFIG_PPC_64K_PAGES - /* for 4K PTE fragment support */ - void *pte_frag; -#endif + void __user *vdso; } mm_context_t; /* Page size definitions, common between 32 and 64-bit * * shift : is the "PAGE_SHIFT" value for that page size - * penc : is the pte encoding mask * */ struct mmu_psize_def { unsigned int shift; /* number of bits */ - unsigned int enc; /* PTE encoding */ - unsigned int ind; /* Corresponding indirect page size shift */ unsigned int flags; #define MMU_PAGE_SIZE_DIRECT 0x1 /* Supported as a direct size */ #define MMU_PAGE_SIZE_INDIRECT 0x2 /* Supported as an indirect size */ @@ -277,8 +277,6 @@ static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize) */ #if defined(CONFIG_PPC_4K_PAGES) #define mmu_virtual_psize MMU_PAGE_4K -#elif defined(CONFIG_PPC_64K_PAGES) -#define mmu_virtual_psize MMU_PAGE_64K #else #error Unsupported page size #endif @@ -286,8 +284,23 @@ static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize) extern int mmu_linear_psize; extern int mmu_vmemmap_psize; +struct tlb_core_data { + /* + * Per-core spinlock for e6500 TLB handlers (no tlbsrx.) + * Must be the first struct element. + */ + u8 lock; + + /* For software way selection, as on Freescale TLB1 */ + u8 esel_next, esel_max, esel_first; +}; + #ifdef CONFIG_PPC64 extern unsigned long linear_map_top; +extern int book3e_htw_mode; + +#define PPC_HTW_NONE 0 +#define PPC_HTW_E6500 1 /* * 64-bit booke platforms don't load the tlb in the tlb miss handler code. @@ -295,8 +308,16 @@ extern unsigned long linear_map_top; * return 1, indicating that the tlb requires preloading. */ #define HUGETLB_NEED_PRELOAD + +#define mmu_cleanup_all NULL + +#define MAX_PHYSMEM_BITS 44 + #endif -#endif /* !__ASSEMBLY__ */ +#include <asm/percpu.h> +DECLARE_PER_CPU(int, next_tlbcam_idx); + +#endif /* !__ASSEMBLER__ */ #endif /* _ASM_POWERPC_MMU_BOOK3E_H_ */ diff --git a/arch/powerpc/include/asm/nohash/mmu.h b/arch/powerpc/include/asm/nohash/mmu.h new file mode 100644 index 000000000000..4cc795044103 --- /dev/null +++ b/arch/powerpc/include/asm/nohash/mmu.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_NOHASH_MMU_H_ +#define _ASM_POWERPC_NOHASH_MMU_H_ + +#if defined(CONFIG_44x) +/* 44x-style software loaded TLB */ +#include <asm/nohash/32/mmu-44x.h> +#elif defined(CONFIG_PPC_E500) +/* Freescale Book-E software loaded TLB or Book-3e (ISA 2.06+) MMU */ +#include <asm/nohash/mmu-e500.h> +#elif defined (CONFIG_PPC_8xx) +/* Motorola/Freescale 8xx software loaded TLB */ +#include <asm/nohash/32/mmu-8xx.h> +#endif + +#endif /* _ASM_POWERPC_NOHASH_MMU_H_ */ diff --git a/arch/powerpc/include/asm/nohash/pgalloc.h b/arch/powerpc/include/asm/nohash/pgalloc.h new file mode 100644 index 000000000000..4ef780b291bc --- /dev/null +++ b/arch/powerpc/include/asm/nohash/pgalloc.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_NOHASH_PGALLOC_H +#define _ASM_POWERPC_NOHASH_PGALLOC_H + +#include <linux/mm.h> +#include <linux/slab.h> + +extern void tlb_remove_table(struct mmu_gather *tlb, void *table); +#ifdef CONFIG_PPC64 +extern void tlb_flush_pgtable(struct mmu_gather *tlb, unsigned long address); +#else +/* 44x etc which is BOOKE not BOOK3E */ +static inline void tlb_flush_pgtable(struct mmu_gather *tlb, + unsigned long address) +{ + +} +#endif /* !CONFIG_PPC_BOOK3E_64 */ + +static inline pgd_t *pgd_alloc(struct mm_struct *mm) +{ + pgd_t *pgd = kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), + pgtable_gfp_flags(mm, GFP_KERNEL)); + +#ifdef CONFIG_PPC_8xx + memcpy(pgd + USER_PTRS_PER_PGD, swapper_pg_dir + USER_PTRS_PER_PGD, + (MAX_PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); +#endif + return pgd; +} + +static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) +{ + kmem_cache_free(PGT_CACHE(PGD_INDEX_SIZE), pgd); +} + +#ifdef CONFIG_PPC64 +#include <asm/nohash/64/pgalloc.h> +#else +#include <asm/nohash/32/pgalloc.h> +#endif + +static inline void pgtable_free(void *table, int shift) +{ + if (!shift) { + pte_fragment_free((unsigned long *)table, 0); + } else { + BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE); + kmem_cache_free(PGT_CACHE(shift), table); + } +} + +static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift) +{ + unsigned long pgf = (unsigned long)table; + + BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE); + pgf |= shift; + tlb_remove_table(tlb, (void *)pgf); +} + +static inline void __tlb_remove_table(void *_table) +{ + void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE); + unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE; + + pgtable_free(table, shift); +} + +static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table, + unsigned long address) +{ + tlb_flush_pgtable(tlb, address); + pgtable_free_tlb(tlb, table, 0); +} +#endif /* _ASM_POWERPC_NOHASH_PGALLOC_H */ diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h new file mode 100644 index 000000000000..5af168b7f292 --- /dev/null +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -0,0 +1,377 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_NOHASH_PGTABLE_H +#define _ASM_POWERPC_NOHASH_PGTABLE_H + +#ifndef __ASSEMBLER__ +static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, pte_t *p, + unsigned long clr, unsigned long set, int huge); +#endif + +#if defined(CONFIG_PPC64) +#include <asm/nohash/64/pgtable.h> +#else +#include <asm/nohash/32/pgtable.h> +#endif + +/* + * _PAGE_CHG_MASK masks of bits that are to be preserved across + * pgprot changes. + */ +#define _PAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_SPECIAL) + +/* Permission masks used for kernel mappings */ +#define PAGE_KERNEL __pgprot(_PAGE_BASE | _PAGE_KERNEL_RW) +#define PAGE_KERNEL_NC __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | _PAGE_NO_CACHE) +#define PAGE_KERNEL_NCG __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | _PAGE_NO_CACHE | _PAGE_GUARDED) +#define PAGE_KERNEL_X __pgprot(_PAGE_BASE | _PAGE_KERNEL_RWX) +#define PAGE_KERNEL_RO __pgprot(_PAGE_BASE | _PAGE_KERNEL_RO) +#define PAGE_KERNEL_ROX __pgprot(_PAGE_BASE | _PAGE_KERNEL_ROX) + +#ifndef __ASSEMBLER__ + +extern int icache_44x_need_flush; + +#ifndef pte_huge_size +static inline unsigned long pte_huge_size(pte_t pte) +{ + return PAGE_SIZE; +} +#endif + +/* + * PTE updates. This function is called whenever an existing + * valid PTE is updated. This does -not- include set_pte_at() + * which nowadays only sets a new PTE. + * + * Depending on the type of MMU, we may need to use atomic updates + * and the PTE may be either 32 or 64 bit wide. In the later case, + * when using atomic updates, only the low part of the PTE is + * accessed atomically. + * + * In addition, on 44x, we also maintain a global flag indicating + * that an executable user mapping was modified, which is needed + * to properly flush the virtually tagged instruction cache of + * those implementations. + */ +#ifndef pte_update +static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, pte_t *p, + unsigned long clr, unsigned long set, int huge) +{ + pte_basic_t old = pte_val(*p); + pte_basic_t new = (old & ~(pte_basic_t)clr) | set; + unsigned long sz; + unsigned long pdsize; + int i; + + if (new == old) + return old; + + if (huge) + sz = pte_huge_size(__pte(old)); + else + sz = PAGE_SIZE; + + if (sz < PMD_SIZE) + pdsize = PAGE_SIZE; + else if (sz < PUD_SIZE) + pdsize = PMD_SIZE; + else if (sz < P4D_SIZE) + pdsize = PUD_SIZE; + else if (sz < PGDIR_SIZE) + pdsize = P4D_SIZE; + else + pdsize = PGDIR_SIZE; + + for (i = 0; i < sz / pdsize; i++, p++) { + *p = __pte(new); + if (new) + new += (unsigned long long)(pdsize / PAGE_SIZE) << PTE_RPN_SHIFT; + } + + if (IS_ENABLED(CONFIG_44x) && !is_kernel_addr(addr) && (old & _PAGE_EXEC)) + icache_44x_need_flush = 1; + + /* huge pages use the old page table lock */ + if (!huge) + assert_pte_locked(mm, addr); + + return old; +} +#endif + +static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + unsigned long old; + + old = pte_update(vma->vm_mm, addr, ptep, _PAGE_ACCESSED, 0, 0); + + return (old & _PAGE_ACCESSED) != 0; +} +#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG + +#ifndef ptep_set_wrprotect +static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, + pte_t *ptep) +{ + pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 0); +} +#endif +#define __HAVE_ARCH_PTEP_SET_WRPROTECT + +static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep) +{ + return __pte(pte_update(mm, addr, ptep, ~0UL, 0, 0)); +} +#define __HAVE_ARCH_PTEP_GET_AND_CLEAR + +static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) +{ + pte_update(mm, addr, ptep, ~0UL, 0, 0); +} + +/* Set the dirty and/or accessed bits atomically in a linux PTE */ +#ifndef __ptep_set_access_flags +static inline void __ptep_set_access_flags(struct vm_area_struct *vma, + pte_t *ptep, pte_t entry, + unsigned long address, + int psize) +{ + unsigned long set = pte_val(entry) & + (_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC); + int huge = psize > mmu_virtual_psize ? 1 : 0; + + pte_update(vma->vm_mm, address, ptep, 0, set, huge); + + flush_tlb_page(vma, address); +} +#endif + +/* Generic accessors to PTE bits */ +#ifndef pte_mkwrite_novma +static inline pte_t pte_mkwrite_novma(pte_t pte) +{ + /* + * write implies read, hence set both + */ + return __pte(pte_val(pte) | _PAGE_RW); +} +#endif + +static inline pte_t pte_mkdirty(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_DIRTY); +} + +static inline pte_t pte_mkyoung(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_ACCESSED); +} + +#ifndef pte_wrprotect +static inline pte_t pte_wrprotect(pte_t pte) +{ + return __pte(pte_val(pte) & ~_PAGE_WRITE); +} +#endif + +#ifndef pte_mkexec +static inline pte_t pte_mkexec(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_EXEC); +} +#endif + +#ifndef pte_write +static inline int pte_write(pte_t pte) +{ + return pte_val(pte) & _PAGE_WRITE; +} +#endif +static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_DIRTY; } +static inline int pte_special(pte_t pte) { return pte_val(pte) & _PAGE_SPECIAL; } +static inline int pte_none(pte_t pte) { return (pte_val(pte) & ~_PTE_NONE_MASK) == 0; } +static inline bool pte_hashpte(pte_t pte) { return false; } +static inline bool pte_ci(pte_t pte) { return pte_val(pte) & _PAGE_NO_CACHE; } +static inline bool pte_exec(pte_t pte) { return pte_val(pte) & _PAGE_EXEC; } + +static inline int pte_present(pte_t pte) +{ + return pte_val(pte) & _PAGE_PRESENT; +} + +static inline bool pte_hw_valid(pte_t pte) +{ + return pte_val(pte) & _PAGE_PRESENT; +} + +static inline int pte_young(pte_t pte) +{ + return pte_val(pte) & _PAGE_ACCESSED; +} + +/* + * Don't just check for any non zero bits in __PAGE_READ, since for book3e + * and PTE_64BIT, PAGE_KERNEL_X contains _PAGE_BAP_SR which is also in + * _PAGE_READ. Need to explicitly match _PAGE_BAP_UR bit in that case too. + */ +#ifndef pte_read +static inline bool pte_read(pte_t pte) +{ + return (pte_val(pte) & _PAGE_READ) == _PAGE_READ; +} +#endif + +/* + * We only find page table entry in the last level + * Hence no need for other accessors + */ +#define pte_access_permitted pte_access_permitted +static inline bool pte_access_permitted(pte_t pte, bool write) +{ + /* + * A read-only access is controlled by _PAGE_READ bit. + * We have _PAGE_READ set for WRITE + */ + if (!pte_present(pte) || !pte_read(pte)) + return false; + + if (write && !pte_write(pte)) + return false; + + return true; +} + +/* Conversion functions: convert a page and protection to a page entry, + * and a page entry and page directory to the page they refer to. + * + * Even if PTEs can be unsigned long long, a PFN is always an unsigned + * long for now. + */ +static inline pte_t pfn_pte(unsigned long pfn, pgprot_t pgprot) { + return __pte(((pte_basic_t)(pfn) << PTE_RPN_SHIFT) | + pgprot_val(pgprot)); } + +/* Generic modifiers for PTE bits */ +static inline pte_t pte_exprotect(pte_t pte) +{ + return __pte(pte_val(pte) & ~_PAGE_EXEC); +} + +static inline pte_t pte_mkclean(pte_t pte) +{ + return __pte(pte_val(pte) & ~_PAGE_DIRTY); +} + +static inline pte_t pte_mkold(pte_t pte) +{ + return __pte(pte_val(pte) & ~_PAGE_ACCESSED); +} + +static inline pte_t pte_mkspecial(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_SPECIAL); +} + +#ifndef pte_mkhuge +static inline pte_t pte_mkhuge(pte_t pte) +{ + return __pte(pte_val(pte)); +} +#endif + +static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) +{ + return __pte((pte_val(pte) & _PAGE_CHG_MASK) | pgprot_val(newprot)); +} + +static inline bool pte_swp_exclusive(pte_t pte) +{ + return pte_val(pte) & _PAGE_SWP_EXCLUSIVE; +} + +static inline pte_t pte_swp_mkexclusive(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_SWP_EXCLUSIVE); +} + +static inline pte_t pte_swp_clear_exclusive(pte_t pte) +{ + return __pte(pte_val(pte) & ~_PAGE_SWP_EXCLUSIVE); +} + +/* This low level function performs the actual PTE insertion + * Setting the PTE depends on the MMU type and other factors. It's + * an horrible mess that I'm not going to try to clean up now but + * I'm keeping it in one place rather than spread around + */ +static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, int percpu) +{ + /* Second case is 32-bit with 64-bit PTE. In this case, we + * can just store as long as we do the two halves in the right order + * with a barrier in between. + * In the percpu case, we also fallback to the simple update + */ + if (IS_ENABLED(CONFIG_PPC32) && IS_ENABLED(CONFIG_PTE_64BIT) && !percpu) { + __asm__ __volatile__("\ + stw%X0 %2,%0\n\ + mbar\n\ + stw%X1 %L2,%1" + : "=m" (*ptep), "=m" (*((unsigned char *)ptep+4)) + : "r" (pte) : "memory"); + return; + } + /* Anything else just stores the PTE normally. That covers all 64-bit + * cases, and 32-bit non-hash with 32-bit PTEs. + */ +#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PPC_16K_PAGES) + ptep->pte3 = ptep->pte2 = ptep->pte1 = ptep->pte = pte_val(pte); +#else + *ptep = pte; +#endif + + /* + * With hardware tablewalk, a sync is needed to ensure that + * subsequent accesses see the PTE we just wrote. Unlike userspace + * mappings, we can't tolerate spurious faults, so make sure + * the new PTE will be seen the first time. + */ + if (IS_ENABLED(CONFIG_PPC_BOOK3E_64) && is_kernel_addr(addr)) + mb(); +} + +/* + * Macro to mark a page protection value as "uncacheable". + */ + +#define _PAGE_CACHE_CTL (_PAGE_COHERENT | _PAGE_GUARDED | _PAGE_NO_CACHE | \ + _PAGE_WRITETHRU) + +#define pgprot_noncached(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ + _PAGE_NO_CACHE | _PAGE_GUARDED)) + +#define pgprot_noncached_wc(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ + _PAGE_NO_CACHE)) + +#define pgprot_cached(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ + _PAGE_COHERENT)) + +#if _PAGE_WRITETHRU != 0 +#define pgprot_cached_wthru(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ + _PAGE_COHERENT | _PAGE_WRITETHRU)) +#else +#define pgprot_cached_wthru(prot) pgprot_noncached(prot) +#endif + +#define pgprot_cached_noncoherent(prot) \ + (__pgprot(pgprot_val(prot) & ~_PAGE_CACHE_CTL)) + +#define pgprot_writecombine pgprot_noncached_wc + +int map_kernel_page(unsigned long va, phys_addr_t pa, pgprot_t prot); +void unmap_kernel_page(unsigned long va); + +#endif /* __ASSEMBLER__ */ +#endif diff --git a/arch/powerpc/include/asm/nohash/pte-e500.h b/arch/powerpc/include/asm/nohash/pte-e500.h new file mode 100644 index 000000000000..b61efc3ee904 --- /dev/null +++ b/arch/powerpc/include/asm/nohash/pte-e500.h @@ -0,0 +1,140 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_NOHASH_PTE_E500_H +#define _ASM_POWERPC_NOHASH_PTE_E500_H +#ifdef __KERNEL__ + +/* PTE bit definitions for processors compliant to the Book3E + * architecture 2.06 or later. The position of the PTE bits + * matches the HW definition of the optional Embedded Page Table + * category. + */ + +/* Architected bits */ +#define _PAGE_PRESENT 0x000001 /* software: pte contains a translation */ +#define _PAGE_SW1 0x000002 +#define _PAGE_BAP_SR 0x000004 +#define _PAGE_BAP_UR 0x000008 +#define _PAGE_BAP_SW 0x000010 +#define _PAGE_BAP_UW 0x000020 +#define _PAGE_BAP_SX 0x000040 +#define _PAGE_BAP_UX 0x000080 +#define _PAGE_PSIZE_MSK 0x000f00 +#define _PAGE_TSIZE_4K 0x000100 +#define _PAGE_DIRTY 0x001000 /* C: page changed */ +#define _PAGE_SW0 0x002000 +#define _PAGE_U3 0x004000 +#define _PAGE_U2 0x008000 +#define _PAGE_U1 0x010000 +#define _PAGE_U0 0x020000 +#define _PAGE_ACCESSED 0x040000 +#define _PAGE_ENDIAN 0x080000 +#define _PAGE_GUARDED 0x100000 +#define _PAGE_COHERENT 0x200000 /* M: enforce memory coherence */ +#define _PAGE_NO_CACHE 0x400000 /* I: cache inhibit */ +#define _PAGE_WRITETHRU 0x800000 /* W: cache write-through */ + +#define _PAGE_PSIZE_SHIFT 7 +#define _PAGE_PSIZE_SHIFT_OFFSET 10 + +/* "Higher level" linux bit combinations */ +#define _PAGE_EXEC (_PAGE_BAP_SX | _PAGE_BAP_UX) /* .. and was cache cleaned */ +#define _PAGE_READ (_PAGE_BAP_SR | _PAGE_BAP_UR) /* User read permission */ +#define _PAGE_WRITE (_PAGE_BAP_SW | _PAGE_BAP_UW) /* User write permission */ + +#define _PAGE_KERNEL_RW (_PAGE_BAP_SW | _PAGE_BAP_SR | _PAGE_DIRTY) +#define _PAGE_KERNEL_RO (_PAGE_BAP_SR) +#define _PAGE_KERNEL_RWX (_PAGE_BAP_SW | _PAGE_BAP_SR | _PAGE_DIRTY | _PAGE_BAP_SX) +#define _PAGE_KERNEL_ROX (_PAGE_BAP_SR | _PAGE_BAP_SX) + +#define _PAGE_NA 0 +#define _PAGE_NAX _PAGE_BAP_UX +#define _PAGE_RO _PAGE_READ +#define _PAGE_ROX (_PAGE_READ | _PAGE_BAP_UX) +#define _PAGE_RW (_PAGE_READ | _PAGE_WRITE) +#define _PAGE_RWX (_PAGE_READ | _PAGE_WRITE | _PAGE_BAP_UX) + +#define _PAGE_SPECIAL _PAGE_SW0 + +#define PTE_RPN_SHIFT (24) + +#define PTE_WIMGE_SHIFT (19) +#define PTE_BAP_SHIFT (2) + +/* On 32-bit, we never clear the top part of the PTE */ +#ifdef CONFIG_PPC32 +#define _PTE_NONE_MASK 0xffffffff00000000ULL +#define _PMD_PRESENT 0 +#define _PMD_PRESENT_MASK (PAGE_MASK) +#define _PMD_BAD (~PAGE_MASK) +#define _PMD_USER 0 +#else +#define _PTE_NONE_MASK 0 +#endif + +/* + * We define 2 sets of base prot bits, one for basic pages (ie, + * cacheable kernel and user pages) and one for non cacheable + * pages. We always set _PAGE_COHERENT when SMP is enabled or + * the processor might need it for DMA coherency. + */ +#define _PAGE_BASE_NC (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_TSIZE_4K) +#if defined(CONFIG_SMP) +#define _PAGE_BASE (_PAGE_BASE_NC | _PAGE_COHERENT) +#else +#define _PAGE_BASE (_PAGE_BASE_NC) +#endif + +#include <asm/pgtable-masks.h> + +#ifndef __ASSEMBLER__ +static inline pte_t pte_mkexec(pte_t pte) +{ + return __pte((pte_val(pte) & ~_PAGE_BAP_SX) | _PAGE_BAP_UX); +} +#define pte_mkexec pte_mkexec + +static inline unsigned long pte_huge_size(pte_t pte) +{ + pte_basic_t val = pte_val(pte); + + return 1UL << (((val & _PAGE_PSIZE_MSK) >> _PAGE_PSIZE_SHIFT) + _PAGE_PSIZE_SHIFT_OFFSET); +} +#define pte_huge_size pte_huge_size + +static inline int pmd_leaf(pmd_t pmd) +{ + if (IS_ENABLED(CONFIG_PPC64)) + return (long)pmd_val(pmd) > 0; + else + return pmd_val(pmd) & _PAGE_PSIZE_MSK; +} +#define pmd_leaf pmd_leaf + +static inline unsigned long pmd_leaf_size(pmd_t pmd) +{ + return pte_huge_size(__pte(pmd_val(pmd))); +} +#define pmd_leaf_size pmd_leaf_size + +#ifdef CONFIG_PPC64 +static inline int pud_leaf(pud_t pud) +{ + if (IS_ENABLED(CONFIG_PPC64)) + return (long)pud_val(pud) > 0; + else + return pud_val(pud) & _PAGE_PSIZE_MSK; +} +#define pud_leaf pud_leaf + +static inline unsigned long pud_leaf_size(pud_t pud) +{ + return pte_huge_size(__pte(pud_val(pud))); +} +#define pud_leaf_size pud_leaf_size + +#endif + +#endif /* __ASSEMBLER__ */ + +#endif /* __KERNEL__ */ +#endif /* _ASM_POWERPC_NOHASH_PTE_E500_H */ diff --git a/arch/powerpc/include/asm/nohash/tlbflush.h b/arch/powerpc/include/asm/nohash/tlbflush.h new file mode 100644 index 000000000000..9a2cf83ea4f1 --- /dev/null +++ b/arch/powerpc/include/asm/nohash/tlbflush.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_NOHASH_TLBFLUSH_H +#define _ASM_POWERPC_NOHASH_TLBFLUSH_H + +/* + * TLB flushing: + * + * - flush_tlb_mm(mm) flushes the specified mm context TLB's + * - flush_tlb_page(vma, vmaddr) flushes one page + * - local_flush_tlb_mm(mm, full) flushes the specified mm context on + * the local processor + * - local_flush_tlb_page(vma, vmaddr) flushes one page on the local processor + * - flush_tlb_range(vma, start, end) flushes a range of pages + * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages + * + */ + +/* + * TLB flushing for software loaded TLB chips + * + * TODO: (CONFIG_PPC_85xx) determine if flush_tlb_range & + * flush_tlb_kernel_range are best implemented as tlbia vs + * specific tlbie's + */ + +struct vm_area_struct; +struct mm_struct; + +#define MMU_NO_CONTEXT ((unsigned int)-1) + +extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end); + +#ifdef CONFIG_PPC_8xx +static inline void local_flush_tlb_mm(struct mm_struct *mm) +{ + unsigned int pid = READ_ONCE(mm->context.id); + + if (pid != MMU_NO_CONTEXT) + asm volatile ("sync; tlbia; isync" : : : "memory"); +} + +static inline void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) +{ + asm volatile ("tlbie %0; sync" : : "r" (vmaddr) : "memory"); +} + +static inline void local_flush_tlb_page_psize(struct mm_struct *mm, + unsigned long vmaddr, int psize) +{ + asm volatile ("tlbie %0; sync" : : "r" (vmaddr) : "memory"); +} + +static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + start &= PAGE_MASK; + + if (end - start <= PAGE_SIZE) + asm volatile ("tlbie %0; sync" : : "r" (start) : "memory"); + else + asm volatile ("sync; tlbia; isync" : : : "memory"); +} +#else +extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); +extern void local_flush_tlb_mm(struct mm_struct *mm); +extern void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); +void local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, int psize); + +extern void __local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, + int tsize, int ind); +#endif + +#ifdef CONFIG_SMP +extern void flush_tlb_mm(struct mm_struct *mm); +extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); +extern void __flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, + int tsize, int ind); +#else +#define flush_tlb_mm(mm) local_flush_tlb_mm(mm) +#define flush_tlb_page(vma,addr) local_flush_tlb_page(vma,addr) +#define __flush_tlb_page(mm,addr,p,i) __local_flush_tlb_page(mm,addr,p,i) +#endif + +#endif /* _ASM_POWERPC_NOHASH_TLBFLUSH_H */ diff --git a/arch/powerpc/include/asm/nvram.h b/arch/powerpc/include/asm/nvram.h index b0fe0fe4e626..eda7fac3500e 100644 --- a/arch/powerpc/include/asm/nvram.h +++ b/arch/powerpc/include/asm/nvram.h @@ -1,20 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * NVRAM definitions and access functions. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef _ASM_POWERPC_NVRAM_H #define _ASM_POWERPC_NVRAM_H - +#include <linux/types.h> #include <linux/errno.h> #include <linux/list.h> #include <uapi/asm/nvram.h> +/* + * Set oops header version to distinguish between old and new format header. + * lnx,oops-log partition max size is 4000, header version > 4000 will + * help in identifying new header. + */ +#define OOPS_HDR_VERSION 5000 + +struct err_log_info { + __be32 error_type; + __be32 seq_num; +}; + +struct nvram_os_partition { + const char *name; + int req_size; /* desired size, in bytes */ + int min_size; /* minimum acceptable size (0 means req_size) */ + long size; /* size of data portion (excluding err_log_info) */ + long index; /* offset of data portion of partition */ + bool os_partition; /* partition initialized by OS, not FW */ +}; + +struct oops_log_info { + __be16 version; + __be16 report_length; + __be64 timestamp; +} __attribute__((packed)); + +extern struct nvram_os_partition oops_log_partition; + #ifdef CONFIG_PPC_PSERIES +extern struct nvram_os_partition rtas_log_partition; + extern int nvram_write_error_log(char * buff, int length, unsigned int err_type, unsigned int err_seq); extern int nvram_read_error_log(char * buff, int length, @@ -47,13 +74,21 @@ extern int pmac_get_partition(int partition); extern u8 pmac_xpram_read(int xpaddr); extern void pmac_xpram_write(int xpaddr, u8 data); -/* Synchronize NVRAM */ -extern void nvram_sync(void); +/* Initialize NVRAM OS partition */ +extern int __init nvram_init_os_partition(struct nvram_os_partition *part); + +/* Initialize NVRAM oops partition */ +extern void __init nvram_init_oops_partition(int rtas_partition_exists); + +/* Read a NVRAM partition */ +extern int nvram_read_partition(struct nvram_os_partition *part, char *buff, + int length, unsigned int *err_type, + unsigned int *error_log_cnt); -/* Determine NVRAM size */ -extern ssize_t nvram_get_size(void); +/* Write to NVRAM OS partition */ +extern int nvram_write_os_partition(struct nvram_os_partition *part, + char *buff, int length, + unsigned int err_type, + unsigned int error_log_cnt); -/* Normal access to NVRAM */ -extern unsigned char nvram_read_byte(int i); -extern void nvram_write_byte(unsigned char c, int i); #endif /* _ASM_POWERPC_NVRAM_H */ diff --git a/arch/powerpc/include/asm/ohare.h b/arch/powerpc/include/asm/ohare.h index 0d030f9dea24..da3371fc348c 100644 --- a/arch/powerpc/include/asm/ohare.h +++ b/arch/powerpc/include/asm/ohare.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_OHARE_H #define _ASM_POWERPC_OHARE_H #ifdef __KERNEL__ diff --git a/arch/powerpc/include/asm/opal-api.h b/arch/powerpc/include/asm/opal-api.h new file mode 100644 index 000000000000..d3eaa3425797 --- /dev/null +++ b/arch/powerpc/include/asm/opal-api.h @@ -0,0 +1,1188 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * OPAL API definitions. + * + * Copyright 2011-2015 IBM Corp. + */ + +#ifndef __OPAL_API_H +#define __OPAL_API_H + +/****** OPAL APIs ******/ + +/* Return codes */ +#define OPAL_SUCCESS 0 +#define OPAL_PARAMETER -1 +#define OPAL_BUSY -2 +#define OPAL_PARTIAL -3 +#define OPAL_CONSTRAINED -4 +#define OPAL_CLOSED -5 +#define OPAL_HARDWARE -6 +#define OPAL_UNSUPPORTED -7 +#define OPAL_PERMISSION -8 +#define OPAL_NO_MEM -9 +#define OPAL_RESOURCE -10 +#define OPAL_INTERNAL_ERROR -11 +#define OPAL_BUSY_EVENT -12 +#define OPAL_HARDWARE_FROZEN -13 +#define OPAL_WRONG_STATE -14 +#define OPAL_ASYNC_COMPLETION -15 +#define OPAL_EMPTY -16 +#define OPAL_I2C_TIMEOUT -17 +#define OPAL_I2C_INVALID_CMD -18 +#define OPAL_I2C_LBUS_PARITY -19 +#define OPAL_I2C_BKEND_OVERRUN -20 +#define OPAL_I2C_BKEND_ACCESS -21 +#define OPAL_I2C_ARBT_LOST -22 +#define OPAL_I2C_NACK_RCVD -23 +#define OPAL_I2C_STOP_ERR -24 +#define OPAL_XIVE_PROVISIONING -31 +#define OPAL_XIVE_FREE_ACTIVE -32 +#define OPAL_TIMEOUT -33 + +/* API Tokens (in r0) */ +#define OPAL_INVALID_CALL -1 +#define OPAL_TEST 0 +#define OPAL_CONSOLE_WRITE 1 +#define OPAL_CONSOLE_READ 2 +#define OPAL_RTC_READ 3 +#define OPAL_RTC_WRITE 4 +#define OPAL_CEC_POWER_DOWN 5 +#define OPAL_CEC_REBOOT 6 +#define OPAL_READ_NVRAM 7 +#define OPAL_WRITE_NVRAM 8 +#define OPAL_HANDLE_INTERRUPT 9 +#define OPAL_POLL_EVENTS 10 +#define OPAL_PCI_SET_HUB_TCE_MEMORY 11 +#define OPAL_PCI_SET_PHB_TCE_MEMORY 12 +#define OPAL_PCI_CONFIG_READ_BYTE 13 +#define OPAL_PCI_CONFIG_READ_HALF_WORD 14 +#define OPAL_PCI_CONFIG_READ_WORD 15 +#define OPAL_PCI_CONFIG_WRITE_BYTE 16 +#define OPAL_PCI_CONFIG_WRITE_HALF_WORD 17 +#define OPAL_PCI_CONFIG_WRITE_WORD 18 +#define OPAL_SET_XIVE 19 +#define OPAL_GET_XIVE 20 +#define OPAL_GET_COMPLETION_TOKEN_STATUS 21 /* obsolete */ +#define OPAL_REGISTER_OPAL_EXCEPTION_HANDLER 22 +#define OPAL_PCI_EEH_FREEZE_STATUS 23 +#define OPAL_PCI_SHPC 24 +#define OPAL_CONSOLE_WRITE_BUFFER_SPACE 25 +#define OPAL_PCI_EEH_FREEZE_CLEAR 26 +#define OPAL_PCI_PHB_MMIO_ENABLE 27 +#define OPAL_PCI_SET_PHB_MEM_WINDOW 28 +#define OPAL_PCI_MAP_PE_MMIO_WINDOW 29 +#define OPAL_PCI_SET_PHB_TABLE_MEMORY 30 +#define OPAL_PCI_SET_PE 31 +#define OPAL_PCI_SET_PELTV 32 +#define OPAL_PCI_SET_MVE 33 +#define OPAL_PCI_SET_MVE_ENABLE 34 +#define OPAL_PCI_GET_XIVE_REISSUE 35 +#define OPAL_PCI_SET_XIVE_REISSUE 36 +#define OPAL_PCI_SET_XIVE_PE 37 +#define OPAL_GET_XIVE_SOURCE 38 +#define OPAL_GET_MSI_32 39 +#define OPAL_GET_MSI_64 40 +#define OPAL_START_CPU 41 +#define OPAL_QUERY_CPU_STATUS 42 +#define OPAL_WRITE_OPPANEL 43 /* unimplemented */ +#define OPAL_PCI_MAP_PE_DMA_WINDOW 44 +#define OPAL_PCI_MAP_PE_DMA_WINDOW_REAL 45 +#define OPAL_PCI_RESET 49 +#define OPAL_PCI_GET_HUB_DIAG_DATA 50 +#define OPAL_PCI_GET_PHB_DIAG_DATA 51 +#define OPAL_PCI_FENCE_PHB 52 +#define OPAL_PCI_REINIT 53 +#define OPAL_PCI_MASK_PE_ERROR 54 +#define OPAL_SET_SLOT_LED_STATUS 55 +#define OPAL_GET_EPOW_STATUS 56 +#define OPAL_SET_SYSTEM_ATTENTION_LED 57 +#define OPAL_RESERVED1 58 +#define OPAL_RESERVED2 59 +#define OPAL_PCI_NEXT_ERROR 60 +#define OPAL_PCI_EEH_FREEZE_STATUS2 61 +#define OPAL_PCI_POLL 62 +#define OPAL_PCI_MSI_EOI 63 +#define OPAL_PCI_GET_PHB_DIAG_DATA2 64 +#define OPAL_XSCOM_READ 65 +#define OPAL_XSCOM_WRITE 66 +#define OPAL_LPC_READ 67 +#define OPAL_LPC_WRITE 68 +#define OPAL_RETURN_CPU 69 +#define OPAL_REINIT_CPUS 70 +#define OPAL_ELOG_READ 71 +#define OPAL_ELOG_WRITE 72 +#define OPAL_ELOG_ACK 73 +#define OPAL_ELOG_RESEND 74 +#define OPAL_ELOG_SIZE 75 +#define OPAL_FLASH_VALIDATE 76 +#define OPAL_FLASH_MANAGE 77 +#define OPAL_FLASH_UPDATE 78 +#define OPAL_RESYNC_TIMEBASE 79 +#define OPAL_CHECK_TOKEN 80 +#define OPAL_DUMP_INIT 81 +#define OPAL_DUMP_INFO 82 +#define OPAL_DUMP_READ 83 +#define OPAL_DUMP_ACK 84 +#define OPAL_GET_MSG 85 +#define OPAL_CHECK_ASYNC_COMPLETION 86 +#define OPAL_SYNC_HOST_REBOOT 87 +#define OPAL_SENSOR_READ 88 +#define OPAL_GET_PARAM 89 +#define OPAL_SET_PARAM 90 +#define OPAL_DUMP_RESEND 91 +#define OPAL_ELOG_SEND 92 /* Deprecated */ +#define OPAL_PCI_SET_PHB_CAPI_MODE 93 +#define OPAL_DUMP_INFO2 94 +#define OPAL_WRITE_OPPANEL_ASYNC 95 +#define OPAL_PCI_ERR_INJECT 96 +#define OPAL_PCI_EEH_FREEZE_SET 97 +#define OPAL_HANDLE_HMI 98 +#define OPAL_CONFIG_CPU_IDLE_STATE 99 +#define OPAL_SLW_SET_REG 100 +#define OPAL_REGISTER_DUMP_REGION 101 +#define OPAL_UNREGISTER_DUMP_REGION 102 +#define OPAL_WRITE_TPO 103 +#define OPAL_READ_TPO 104 +#define OPAL_GET_DPO_STATUS 105 +#define OPAL_OLD_I2C_REQUEST 106 /* Deprecated */ +#define OPAL_IPMI_SEND 107 +#define OPAL_IPMI_RECV 108 +#define OPAL_I2C_REQUEST 109 +#define OPAL_FLASH_READ 110 +#define OPAL_FLASH_WRITE 111 +#define OPAL_FLASH_ERASE 112 +#define OPAL_PRD_MSG 113 +#define OPAL_LEDS_GET_INDICATOR 114 +#define OPAL_LEDS_SET_INDICATOR 115 +#define OPAL_CEC_REBOOT2 116 +#define OPAL_CONSOLE_FLUSH 117 +#define OPAL_GET_DEVICE_TREE 118 +#define OPAL_PCI_GET_PRESENCE_STATE 119 +#define OPAL_PCI_GET_POWER_STATE 120 +#define OPAL_PCI_SET_POWER_STATE 121 +#define OPAL_INT_GET_XIRR 122 +#define OPAL_INT_SET_CPPR 123 +#define OPAL_INT_EOI 124 +#define OPAL_INT_SET_MFRR 125 +#define OPAL_PCI_TCE_KILL 126 +#define OPAL_NMMU_SET_PTCR 127 +#define OPAL_XIVE_RESET 128 +#define OPAL_XIVE_GET_IRQ_INFO 129 +#define OPAL_XIVE_GET_IRQ_CONFIG 130 +#define OPAL_XIVE_SET_IRQ_CONFIG 131 +#define OPAL_XIVE_GET_QUEUE_INFO 132 +#define OPAL_XIVE_SET_QUEUE_INFO 133 +#define OPAL_XIVE_DONATE_PAGE 134 +#define OPAL_XIVE_ALLOCATE_VP_BLOCK 135 +#define OPAL_XIVE_FREE_VP_BLOCK 136 +#define OPAL_XIVE_GET_VP_INFO 137 +#define OPAL_XIVE_SET_VP_INFO 138 +#define OPAL_XIVE_ALLOCATE_IRQ 139 +#define OPAL_XIVE_FREE_IRQ 140 +#define OPAL_XIVE_SYNC 141 +#define OPAL_XIVE_DUMP 142 +#define OPAL_XIVE_GET_QUEUE_STATE 143 +#define OPAL_XIVE_SET_QUEUE_STATE 144 +#define OPAL_SIGNAL_SYSTEM_RESET 145 +#define OPAL_NPU_INIT_CONTEXT 146 +#define OPAL_NPU_DESTROY_CONTEXT 147 +#define OPAL_NPU_MAP_LPAR 148 +#define OPAL_IMC_COUNTERS_INIT 149 +#define OPAL_IMC_COUNTERS_START 150 +#define OPAL_IMC_COUNTERS_STOP 151 +#define OPAL_GET_POWERCAP 152 +#define OPAL_SET_POWERCAP 153 +#define OPAL_GET_POWER_SHIFT_RATIO 154 +#define OPAL_SET_POWER_SHIFT_RATIO 155 +#define OPAL_SENSOR_GROUP_CLEAR 156 +#define OPAL_PCI_SET_P2P 157 +#define OPAL_QUIESCE 158 +#define OPAL_NPU_SPA_SETUP 159 +#define OPAL_NPU_SPA_CLEAR_CACHE 160 +#define OPAL_NPU_TL_SET 161 +#define OPAL_SENSOR_READ_U64 162 +#define OPAL_SENSOR_GROUP_ENABLE 163 +#define OPAL_PCI_GET_PBCQ_TUNNEL_BAR 164 +#define OPAL_PCI_SET_PBCQ_TUNNEL_BAR 165 +#define OPAL_HANDLE_HMI2 166 +#define OPAL_NX_COPROC_INIT 167 +#define OPAL_XIVE_GET_VP_STATE 170 +#define OPAL_MPIPL_UPDATE 173 +#define OPAL_MPIPL_REGISTER_TAG 174 +#define OPAL_MPIPL_QUERY_TAG 175 +#define OPAL_SECVAR_GET 176 +#define OPAL_SECVAR_GET_NEXT 177 +#define OPAL_SECVAR_ENQUEUE_UPDATE 178 +#define OPAL_LAST 178 + +#define QUIESCE_HOLD 1 /* Spin all calls at entry */ +#define QUIESCE_REJECT 2 /* Fail all calls with OPAL_BUSY */ +#define QUIESCE_LOCK_BREAK 3 /* Set to ignore locks. */ +#define QUIESCE_RESUME 4 /* Un-quiesce */ +#define QUIESCE_RESUME_FAST_REBOOT 5 /* Un-quiesce, fast reboot */ + +/* Device tree flags */ + +/* + * Flags set in power-mgmt nodes in device tree describing + * idle states that are supported in the platform. + */ + +#define OPAL_PM_TIMEBASE_STOP 0x00000002 +#define OPAL_PM_LOSE_HYP_CONTEXT 0x00002000 +#define OPAL_PM_LOSE_FULL_CONTEXT 0x00004000 +#define OPAL_PM_NAP_ENABLED 0x00010000 +#define OPAL_PM_SLEEP_ENABLED 0x00020000 +#define OPAL_PM_WINKLE_ENABLED 0x00040000 +#define OPAL_PM_SLEEP_ENABLED_ER1 0x00080000 /* with workaround */ +#define OPAL_PM_STOP_INST_FAST 0x00100000 +#define OPAL_PM_STOP_INST_DEEP 0x00200000 + +/* + * OPAL_CONFIG_CPU_IDLE_STATE parameters + */ +#define OPAL_CONFIG_IDLE_FASTSLEEP 1 +#define OPAL_CONFIG_IDLE_UNDO 0 +#define OPAL_CONFIG_IDLE_APPLY 1 + +#ifndef __ASSEMBLER__ + +/* Other enums */ +enum OpalFreezeState { + OPAL_EEH_STOPPED_NOT_FROZEN = 0, + OPAL_EEH_STOPPED_MMIO_FREEZE = 1, + OPAL_EEH_STOPPED_DMA_FREEZE = 2, + OPAL_EEH_STOPPED_MMIO_DMA_FREEZE = 3, + OPAL_EEH_STOPPED_RESET = 4, + OPAL_EEH_STOPPED_TEMP_UNAVAIL = 5, + OPAL_EEH_STOPPED_PERM_UNAVAIL = 6 +}; + +enum OpalEehFreezeActionToken { + OPAL_EEH_ACTION_CLEAR_FREEZE_MMIO = 1, + OPAL_EEH_ACTION_CLEAR_FREEZE_DMA = 2, + OPAL_EEH_ACTION_CLEAR_FREEZE_ALL = 3, + + OPAL_EEH_ACTION_SET_FREEZE_MMIO = 1, + OPAL_EEH_ACTION_SET_FREEZE_DMA = 2, + OPAL_EEH_ACTION_SET_FREEZE_ALL = 3 +}; + +enum OpalPciStatusToken { + OPAL_EEH_NO_ERROR = 0, + OPAL_EEH_IOC_ERROR = 1, + OPAL_EEH_PHB_ERROR = 2, + OPAL_EEH_PE_ERROR = 3, + OPAL_EEH_PE_MMIO_ERROR = 4, + OPAL_EEH_PE_DMA_ERROR = 5 +}; + +enum OpalPciErrorSeverity { + OPAL_EEH_SEV_NO_ERROR = 0, + OPAL_EEH_SEV_IOC_DEAD = 1, + OPAL_EEH_SEV_PHB_DEAD = 2, + OPAL_EEH_SEV_PHB_FENCED = 3, + OPAL_EEH_SEV_PE_ER = 4, + OPAL_EEH_SEV_INF = 5 +}; + +enum OpalErrinjectType { + OPAL_ERR_INJECT_TYPE_IOA_BUS_ERR = 0, + OPAL_ERR_INJECT_TYPE_IOA_BUS_ERR64 = 1, +}; + +enum OpalErrinjectFunc { + /* IOA bus specific errors */ + OPAL_ERR_INJECT_FUNC_IOA_LD_MEM_ADDR = 0, + OPAL_ERR_INJECT_FUNC_IOA_LD_MEM_DATA = 1, + OPAL_ERR_INJECT_FUNC_IOA_LD_IO_ADDR = 2, + OPAL_ERR_INJECT_FUNC_IOA_LD_IO_DATA = 3, + OPAL_ERR_INJECT_FUNC_IOA_LD_CFG_ADDR = 4, + OPAL_ERR_INJECT_FUNC_IOA_LD_CFG_DATA = 5, + OPAL_ERR_INJECT_FUNC_IOA_ST_MEM_ADDR = 6, + OPAL_ERR_INJECT_FUNC_IOA_ST_MEM_DATA = 7, + OPAL_ERR_INJECT_FUNC_IOA_ST_IO_ADDR = 8, + OPAL_ERR_INJECT_FUNC_IOA_ST_IO_DATA = 9, + OPAL_ERR_INJECT_FUNC_IOA_ST_CFG_ADDR = 10, + OPAL_ERR_INJECT_FUNC_IOA_ST_CFG_DATA = 11, + OPAL_ERR_INJECT_FUNC_IOA_DMA_RD_ADDR = 12, + OPAL_ERR_INJECT_FUNC_IOA_DMA_RD_DATA = 13, + OPAL_ERR_INJECT_FUNC_IOA_DMA_RD_MASTER = 14, + OPAL_ERR_INJECT_FUNC_IOA_DMA_RD_TARGET = 15, + OPAL_ERR_INJECT_FUNC_IOA_DMA_WR_ADDR = 16, + OPAL_ERR_INJECT_FUNC_IOA_DMA_WR_DATA = 17, + OPAL_ERR_INJECT_FUNC_IOA_DMA_WR_MASTER = 18, + OPAL_ERR_INJECT_FUNC_IOA_DMA_WR_TARGET = 19, +}; + +enum OpalMmioWindowType { + OPAL_M32_WINDOW_TYPE = 1, + OPAL_M64_WINDOW_TYPE = 2, + OPAL_IO_WINDOW_TYPE = 3 +}; + +enum OpalExceptionHandler { + OPAL_MACHINE_CHECK_HANDLER = 1, + OPAL_HYPERVISOR_MAINTENANCE_HANDLER = 2, + OPAL_SOFTPATCH_HANDLER = 3 +}; + +enum OpalPendingState { + OPAL_EVENT_OPAL_INTERNAL = 0x1, + OPAL_EVENT_NVRAM = 0x2, + OPAL_EVENT_RTC = 0x4, + OPAL_EVENT_CONSOLE_OUTPUT = 0x8, + OPAL_EVENT_CONSOLE_INPUT = 0x10, + OPAL_EVENT_ERROR_LOG_AVAIL = 0x20, + OPAL_EVENT_ERROR_LOG = 0x40, + OPAL_EVENT_EPOW = 0x80, + OPAL_EVENT_LED_STATUS = 0x100, + OPAL_EVENT_PCI_ERROR = 0x200, + OPAL_EVENT_DUMP_AVAIL = 0x400, + OPAL_EVENT_MSG_PENDING = 0x800, +}; + +enum OpalThreadStatus { + OPAL_THREAD_INACTIVE = 0x0, + OPAL_THREAD_STARTED = 0x1, + OPAL_THREAD_UNAVAILABLE = 0x2 /* opal-v3 */ +}; + +enum OpalPciBusCompare { + OpalPciBusAny = 0, /* Any bus number match */ + OpalPciBus3Bits = 2, /* Match top 3 bits of bus number */ + OpalPciBus4Bits = 3, /* Match top 4 bits of bus number */ + OpalPciBus5Bits = 4, /* Match top 5 bits of bus number */ + OpalPciBus6Bits = 5, /* Match top 6 bits of bus number */ + OpalPciBus7Bits = 6, /* Match top 7 bits of bus number */ + OpalPciBusAll = 7, /* Match bus number exactly */ +}; + +enum OpalDeviceCompare { + OPAL_IGNORE_RID_DEVICE_NUMBER = 0, + OPAL_COMPARE_RID_DEVICE_NUMBER = 1 +}; + +enum OpalFuncCompare { + OPAL_IGNORE_RID_FUNCTION_NUMBER = 0, + OPAL_COMPARE_RID_FUNCTION_NUMBER = 1 +}; + +enum OpalPeAction { + OPAL_UNMAP_PE = 0, + OPAL_MAP_PE = 1 +}; + +enum OpalPeltvAction { + OPAL_REMOVE_PE_FROM_DOMAIN = 0, + OPAL_ADD_PE_TO_DOMAIN = 1 +}; + +enum OpalMveEnableAction { + OPAL_DISABLE_MVE = 0, + OPAL_ENABLE_MVE = 1 +}; + +enum OpalM64Action { + OPAL_DISABLE_M64 = 0, + OPAL_ENABLE_M64_SPLIT = 1, + OPAL_ENABLE_M64_NON_SPLIT = 2 +}; + +enum OpalPciResetScope { + OPAL_RESET_PHB_COMPLETE = 1, + OPAL_RESET_PCI_LINK = 2, + OPAL_RESET_PHB_ERROR = 3, + OPAL_RESET_PCI_HOT = 4, + OPAL_RESET_PCI_FUNDAMENTAL = 5, + OPAL_RESET_PCI_IODA_TABLE = 6 +}; + +enum OpalPciReinitScope { + /* + * Note: we chose values that do not overlap + * OpalPciResetScope as OPAL v2 used the same + * enum for both + */ + OPAL_REINIT_PCI_DEV = 1000 +}; + +enum OpalPciResetState { + OPAL_DEASSERT_RESET = 0, + OPAL_ASSERT_RESET = 1 +}; + +enum OpalPciSlotPresence { + OPAL_PCI_SLOT_EMPTY = 0, + OPAL_PCI_SLOT_PRESENT = 1 +}; + +enum OpalPciSlotPower { + OPAL_PCI_SLOT_POWER_OFF = 0, + OPAL_PCI_SLOT_POWER_ON = 1, + OPAL_PCI_SLOT_OFFLINE = 2, + OPAL_PCI_SLOT_ONLINE = 3 +}; + +enum OpalSlotLedType { + OPAL_SLOT_LED_TYPE_ID = 0, /* IDENTIFY LED */ + OPAL_SLOT_LED_TYPE_FAULT = 1, /* FAULT LED */ + OPAL_SLOT_LED_TYPE_ATTN = 2, /* System Attention LED */ + OPAL_SLOT_LED_TYPE_MAX = 3 +}; + +enum OpalSlotLedState { + OPAL_SLOT_LED_STATE_OFF = 0, /* LED is OFF */ + OPAL_SLOT_LED_STATE_ON = 1 /* LED is ON */ +}; + +/* + * Address cycle types for LPC accesses. These also correspond + * to the content of the first cell of the "reg" property for + * device nodes on the LPC bus + */ +enum OpalLPCAddressType { + OPAL_LPC_MEM = 0, + OPAL_LPC_IO = 1, + OPAL_LPC_FW = 2, +}; + +enum opal_msg_type { + OPAL_MSG_ASYNC_COMP = 0, /* params[0] = token, params[1] = rc, + * additional params function-specific + */ + OPAL_MSG_MEM_ERR = 1, + OPAL_MSG_EPOW = 2, + OPAL_MSG_SHUTDOWN = 3, /* params[0] = 1 reboot, 0 shutdown */ + OPAL_MSG_HMI_EVT = 4, + OPAL_MSG_DPO = 5, + OPAL_MSG_PRD = 6, + OPAL_MSG_OCC = 7, + OPAL_MSG_PRD2 = 8, + OPAL_MSG_TYPE_MAX, +}; + +struct opal_msg { + __be32 msg_type; + __be32 reserved; + __be64 params[8]; +}; + +/* System parameter permission */ +enum OpalSysparamPerm { + OPAL_SYSPARAM_READ = 0x1, + OPAL_SYSPARAM_WRITE = 0x2, + OPAL_SYSPARAM_RW = (OPAL_SYSPARAM_READ | OPAL_SYSPARAM_WRITE), +}; + +enum { + OPAL_IPMI_MSG_FORMAT_VERSION_1 = 1, +}; + +struct opal_ipmi_msg { + uint8_t version; + uint8_t netfn; + uint8_t cmd; + uint8_t data[]; +}; + +/* FSP memory errors handling */ +enum OpalMemErr_Version { + OpalMemErr_V1 = 1, +}; + +enum OpalMemErrType { + OPAL_MEM_ERR_TYPE_RESILIENCE = 0, + OPAL_MEM_ERR_TYPE_DYN_DALLOC, +}; + +/* Memory Reilience error type */ +enum OpalMemErr_ResilErrType { + OPAL_MEM_RESILIENCE_CE = 0, + OPAL_MEM_RESILIENCE_UE, + OPAL_MEM_RESILIENCE_UE_SCRUB, +}; + +/* Dynamic Memory Deallocation type */ +enum OpalMemErr_DynErrType { + OPAL_MEM_DYNAMIC_DEALLOC = 0, +}; + +struct OpalMemoryErrorData { + enum OpalMemErr_Version version:8; /* 0x00 */ + enum OpalMemErrType type:8; /* 0x01 */ + __be16 flags; /* 0x02 */ + uint8_t reserved_1[4]; /* 0x04 */ + + union { + /* Memory Resilience corrected/uncorrected error info */ + struct { + enum OpalMemErr_ResilErrType resil_err_type:8; + uint8_t reserved_1[7]; + __be64 physical_address_start; + __be64 physical_address_end; + } resilience; + /* Dynamic memory deallocation error info */ + struct { + enum OpalMemErr_DynErrType dyn_err_type:8; + uint8_t reserved_1[7]; + __be64 physical_address_start; + __be64 physical_address_end; + } dyn_dealloc; + } u; +}; + +/* HMI interrupt event */ +enum OpalHMI_Version { + OpalHMIEvt_V1 = 1, + OpalHMIEvt_V2 = 2, +}; + +enum OpalHMI_Severity { + OpalHMI_SEV_NO_ERROR = 0, + OpalHMI_SEV_WARNING = 1, + OpalHMI_SEV_ERROR_SYNC = 2, + OpalHMI_SEV_FATAL = 3, +}; + +enum OpalHMI_Disposition { + OpalHMI_DISPOSITION_RECOVERED = 0, + OpalHMI_DISPOSITION_NOT_RECOVERED = 1, +}; + +enum OpalHMI_ErrType { + OpalHMI_ERROR_MALFUNC_ALERT = 0, + OpalHMI_ERROR_PROC_RECOV_DONE, + OpalHMI_ERROR_PROC_RECOV_DONE_AGAIN, + OpalHMI_ERROR_PROC_RECOV_MASKED, + OpalHMI_ERROR_TFAC, + OpalHMI_ERROR_TFMR_PARITY, + OpalHMI_ERROR_HA_OVERFLOW_WARN, + OpalHMI_ERROR_XSCOM_FAIL, + OpalHMI_ERROR_XSCOM_DONE, + OpalHMI_ERROR_SCOM_FIR, + OpalHMI_ERROR_DEBUG_TRIG_FIR, + OpalHMI_ERROR_HYP_RESOURCE, + OpalHMI_ERROR_CAPP_RECOVERY, +}; + +enum OpalHMI_XstopType { + CHECKSTOP_TYPE_UNKNOWN = 0, + CHECKSTOP_TYPE_CORE = 1, + CHECKSTOP_TYPE_NX = 2, + CHECKSTOP_TYPE_NPU = 3 +}; + +enum OpalHMI_CoreXstopReason { + CORE_CHECKSTOP_IFU_REGFILE = 0x00000001, + CORE_CHECKSTOP_IFU_LOGIC = 0x00000002, + CORE_CHECKSTOP_PC_DURING_RECOV = 0x00000004, + CORE_CHECKSTOP_ISU_REGFILE = 0x00000008, + CORE_CHECKSTOP_ISU_LOGIC = 0x00000010, + CORE_CHECKSTOP_FXU_LOGIC = 0x00000020, + CORE_CHECKSTOP_VSU_LOGIC = 0x00000040, + CORE_CHECKSTOP_PC_RECOV_IN_MAINT_MODE = 0x00000080, + CORE_CHECKSTOP_LSU_REGFILE = 0x00000100, + CORE_CHECKSTOP_PC_FWD_PROGRESS = 0x00000200, + CORE_CHECKSTOP_LSU_LOGIC = 0x00000400, + CORE_CHECKSTOP_PC_LOGIC = 0x00000800, + CORE_CHECKSTOP_PC_HYP_RESOURCE = 0x00001000, + CORE_CHECKSTOP_PC_HANG_RECOV_FAILED = 0x00002000, + CORE_CHECKSTOP_PC_AMBI_HANG_DETECTED = 0x00004000, + CORE_CHECKSTOP_PC_DEBUG_TRIG_ERR_INJ = 0x00008000, + CORE_CHECKSTOP_PC_SPRD_HYP_ERR_INJ = 0x00010000, +}; + +enum OpalHMI_NestAccelXstopReason { + NX_CHECKSTOP_SHM_INVAL_STATE_ERR = 0x00000001, + NX_CHECKSTOP_DMA_INVAL_STATE_ERR_1 = 0x00000002, + NX_CHECKSTOP_DMA_INVAL_STATE_ERR_2 = 0x00000004, + NX_CHECKSTOP_DMA_CH0_INVAL_STATE_ERR = 0x00000008, + NX_CHECKSTOP_DMA_CH1_INVAL_STATE_ERR = 0x00000010, + NX_CHECKSTOP_DMA_CH2_INVAL_STATE_ERR = 0x00000020, + NX_CHECKSTOP_DMA_CH3_INVAL_STATE_ERR = 0x00000040, + NX_CHECKSTOP_DMA_CH4_INVAL_STATE_ERR = 0x00000080, + NX_CHECKSTOP_DMA_CH5_INVAL_STATE_ERR = 0x00000100, + NX_CHECKSTOP_DMA_CH6_INVAL_STATE_ERR = 0x00000200, + NX_CHECKSTOP_DMA_CH7_INVAL_STATE_ERR = 0x00000400, + NX_CHECKSTOP_DMA_CRB_UE = 0x00000800, + NX_CHECKSTOP_DMA_CRB_SUE = 0x00001000, + NX_CHECKSTOP_PBI_ISN_UE = 0x00002000, +}; + +struct OpalHMIEvent { + uint8_t version; /* 0x00 */ + uint8_t severity; /* 0x01 */ + uint8_t type; /* 0x02 */ + uint8_t disposition; /* 0x03 */ + uint8_t reserved_1[4]; /* 0x04 */ + + __be64 hmer; + /* TFMR register. Valid only for TFAC and TFMR_PARITY error type. */ + __be64 tfmr; + + /* version 2 and later */ + union { + /* + * checkstop info (Core/NX). + * Valid for OpalHMI_ERROR_MALFUNC_ALERT. + */ + struct { + uint8_t xstop_type; /* enum OpalHMI_XstopType */ + uint8_t reserved_1[3]; + __be32 xstop_reason; + union { + __be32 pir; /* for CHECKSTOP_TYPE_CORE */ + __be32 chip_id; /* for CHECKSTOP_TYPE_NX */ + } u; + } xstop_error; + } u; +}; + +/* OPAL_HANDLE_HMI2 out_flags */ +enum { + OPAL_HMI_FLAGS_TB_RESYNC = (1ull << 0), /* Timebase has been resynced */ + OPAL_HMI_FLAGS_DEC_LOST = (1ull << 1), /* DEC lost, needs to be reprogrammed */ + OPAL_HMI_FLAGS_HDEC_LOST = (1ull << 2), /* HDEC lost, needs to be reprogrammed */ + OPAL_HMI_FLAGS_TOD_TB_FAIL = (1ull << 3), /* TOD/TB recovery failed. */ + OPAL_HMI_FLAGS_NEW_EVENT = (1ull << 63), /* An event has been created */ +}; + +enum { + OPAL_P7IOC_DIAG_TYPE_NONE = 0, + OPAL_P7IOC_DIAG_TYPE_RGC = 1, + OPAL_P7IOC_DIAG_TYPE_BI = 2, + OPAL_P7IOC_DIAG_TYPE_CI = 3, + OPAL_P7IOC_DIAG_TYPE_MISC = 4, + OPAL_P7IOC_DIAG_TYPE_I2C = 5, + OPAL_P7IOC_DIAG_TYPE_LAST = 6 +}; + +struct OpalIoP7IOCErrorData { + __be16 type; + + /* GEM */ + __be64 gemXfir; + __be64 gemRfir; + __be64 gemRirqfir; + __be64 gemMask; + __be64 gemRwof; + + /* LEM */ + __be64 lemFir; + __be64 lemErrMask; + __be64 lemAction0; + __be64 lemAction1; + __be64 lemWof; + + union { + struct OpalIoP7IOCRgcErrorData { + __be64 rgcStatus; /* 3E1C10 */ + __be64 rgcLdcp; /* 3E1C18 */ + }rgc; + struct OpalIoP7IOCBiErrorData { + __be64 biLdcp0; /* 3C0100, 3C0118 */ + __be64 biLdcp1; /* 3C0108, 3C0120 */ + __be64 biLdcp2; /* 3C0110, 3C0128 */ + __be64 biFenceStatus; /* 3C0130, 3C0130 */ + + uint8_t biDownbound; /* BI Downbound or Upbound */ + }bi; + struct OpalIoP7IOCCiErrorData { + __be64 ciPortStatus; /* 3Dn008 */ + __be64 ciPortLdcp; /* 3Dn010 */ + + uint8_t ciPort; /* Index of CI port: 0/1 */ + }ci; + }; +}; + +/** + * This structure defines the overlay which will be used to store PHB error + * data upon request. + */ +enum { + OPAL_PHB_ERROR_DATA_VERSION_1 = 1, +}; + +enum { + OPAL_PHB_ERROR_DATA_TYPE_P7IOC = 1, + OPAL_PHB_ERROR_DATA_TYPE_PHB3 = 2, + OPAL_PHB_ERROR_DATA_TYPE_PHB4 = 3 +}; + +enum { + OPAL_P7IOC_NUM_PEST_REGS = 128, + OPAL_PHB3_NUM_PEST_REGS = 256, + OPAL_PHB4_NUM_PEST_REGS = 512 +}; + +struct OpalIoPhbErrorCommon { + __be32 version; + __be32 ioType; + __be32 len; +}; + +struct OpalIoP7IOCPhbErrorData { + struct OpalIoPhbErrorCommon common; + + __be32 brdgCtl; + + // P7IOC utl regs + __be32 portStatusReg; + __be32 rootCmplxStatus; + __be32 busAgentStatus; + + // P7IOC cfg regs + __be32 deviceStatus; + __be32 slotStatus; + __be32 linkStatus; + __be32 devCmdStatus; + __be32 devSecStatus; + + // cfg AER regs + __be32 rootErrorStatus; + __be32 uncorrErrorStatus; + __be32 corrErrorStatus; + __be32 tlpHdr1; + __be32 tlpHdr2; + __be32 tlpHdr3; + __be32 tlpHdr4; + __be32 sourceId; + + __be32 rsv3; + + // Record data about the call to allocate a buffer. + __be64 errorClass; + __be64 correlator; + + //P7IOC MMIO Error Regs + __be64 p7iocPlssr; // n120 + __be64 p7iocCsr; // n110 + __be64 lemFir; // nC00 + __be64 lemErrorMask; // nC18 + __be64 lemWOF; // nC40 + __be64 phbErrorStatus; // nC80 + __be64 phbFirstErrorStatus; // nC88 + __be64 phbErrorLog0; // nCC0 + __be64 phbErrorLog1; // nCC8 + __be64 mmioErrorStatus; // nD00 + __be64 mmioFirstErrorStatus; // nD08 + __be64 mmioErrorLog0; // nD40 + __be64 mmioErrorLog1; // nD48 + __be64 dma0ErrorStatus; // nD80 + __be64 dma0FirstErrorStatus; // nD88 + __be64 dma0ErrorLog0; // nDC0 + __be64 dma0ErrorLog1; // nDC8 + __be64 dma1ErrorStatus; // nE00 + __be64 dma1FirstErrorStatus; // nE08 + __be64 dma1ErrorLog0; // nE40 + __be64 dma1ErrorLog1; // nE48 + __be64 pestA[OPAL_P7IOC_NUM_PEST_REGS]; + __be64 pestB[OPAL_P7IOC_NUM_PEST_REGS]; +}; + +struct OpalIoPhb3ErrorData { + struct OpalIoPhbErrorCommon common; + + __be32 brdgCtl; + + /* PHB3 UTL regs */ + __be32 portStatusReg; + __be32 rootCmplxStatus; + __be32 busAgentStatus; + + /* PHB3 cfg regs */ + __be32 deviceStatus; + __be32 slotStatus; + __be32 linkStatus; + __be32 devCmdStatus; + __be32 devSecStatus; + + /* cfg AER regs */ + __be32 rootErrorStatus; + __be32 uncorrErrorStatus; + __be32 corrErrorStatus; + __be32 tlpHdr1; + __be32 tlpHdr2; + __be32 tlpHdr3; + __be32 tlpHdr4; + __be32 sourceId; + + __be32 rsv3; + + /* Record data about the call to allocate a buffer */ + __be64 errorClass; + __be64 correlator; + + /* PHB3 MMIO Error Regs */ + __be64 nFir; /* 000 */ + __be64 nFirMask; /* 003 */ + __be64 nFirWOF; /* 008 */ + __be64 phbPlssr; /* 120 */ + __be64 phbCsr; /* 110 */ + __be64 lemFir; /* C00 */ + __be64 lemErrorMask; /* C18 */ + __be64 lemWOF; /* C40 */ + __be64 phbErrorStatus; /* C80 */ + __be64 phbFirstErrorStatus; /* C88 */ + __be64 phbErrorLog0; /* CC0 */ + __be64 phbErrorLog1; /* CC8 */ + __be64 mmioErrorStatus; /* D00 */ + __be64 mmioFirstErrorStatus; /* D08 */ + __be64 mmioErrorLog0; /* D40 */ + __be64 mmioErrorLog1; /* D48 */ + __be64 dma0ErrorStatus; /* D80 */ + __be64 dma0FirstErrorStatus; /* D88 */ + __be64 dma0ErrorLog0; /* DC0 */ + __be64 dma0ErrorLog1; /* DC8 */ + __be64 dma1ErrorStatus; /* E00 */ + __be64 dma1FirstErrorStatus; /* E08 */ + __be64 dma1ErrorLog0; /* E40 */ + __be64 dma1ErrorLog1; /* E48 */ + __be64 pestA[OPAL_PHB3_NUM_PEST_REGS]; + __be64 pestB[OPAL_PHB3_NUM_PEST_REGS]; +}; + +struct OpalIoPhb4ErrorData { + struct OpalIoPhbErrorCommon common; + + __be32 brdgCtl; + + /* PHB4 cfg regs */ + __be32 deviceStatus; + __be32 slotStatus; + __be32 linkStatus; + __be32 devCmdStatus; + __be32 devSecStatus; + + /* cfg AER regs */ + __be32 rootErrorStatus; + __be32 uncorrErrorStatus; + __be32 corrErrorStatus; + __be32 tlpHdr1; + __be32 tlpHdr2; + __be32 tlpHdr3; + __be32 tlpHdr4; + __be32 sourceId; + + /* PHB4 ETU Error Regs */ + __be64 nFir; /* 000 */ + __be64 nFirMask; /* 003 */ + __be64 nFirWOF; /* 008 */ + __be64 phbPlssr; /* 120 */ + __be64 phbCsr; /* 110 */ + __be64 lemFir; /* C00 */ + __be64 lemErrorMask; /* C18 */ + __be64 lemWOF; /* C40 */ + __be64 phbErrorStatus; /* C80 */ + __be64 phbFirstErrorStatus; /* C88 */ + __be64 phbErrorLog0; /* CC0 */ + __be64 phbErrorLog1; /* CC8 */ + __be64 phbTxeErrorStatus; /* D00 */ + __be64 phbTxeFirstErrorStatus; /* D08 */ + __be64 phbTxeErrorLog0; /* D40 */ + __be64 phbTxeErrorLog1; /* D48 */ + __be64 phbRxeArbErrorStatus; /* D80 */ + __be64 phbRxeArbFirstErrorStatus; /* D88 */ + __be64 phbRxeArbErrorLog0; /* DC0 */ + __be64 phbRxeArbErrorLog1; /* DC8 */ + __be64 phbRxeMrgErrorStatus; /* E00 */ + __be64 phbRxeMrgFirstErrorStatus; /* E08 */ + __be64 phbRxeMrgErrorLog0; /* E40 */ + __be64 phbRxeMrgErrorLog1; /* E48 */ + __be64 phbRxeTceErrorStatus; /* E80 */ + __be64 phbRxeTceFirstErrorStatus; /* E88 */ + __be64 phbRxeTceErrorLog0; /* EC0 */ + __be64 phbRxeTceErrorLog1; /* EC8 */ + + /* PHB4 REGB Error Regs */ + __be64 phbPblErrorStatus; /* 1900 */ + __be64 phbPblFirstErrorStatus; /* 1908 */ + __be64 phbPblErrorLog0; /* 1940 */ + __be64 phbPblErrorLog1; /* 1948 */ + __be64 phbPcieDlpErrorLog1; /* 1AA0 */ + __be64 phbPcieDlpErrorLog2; /* 1AA8 */ + __be64 phbPcieDlpErrorStatus; /* 1AB0 */ + __be64 phbRegbErrorStatus; /* 1C00 */ + __be64 phbRegbFirstErrorStatus; /* 1C08 */ + __be64 phbRegbErrorLog0; /* 1C40 */ + __be64 phbRegbErrorLog1; /* 1C48 */ + + __be64 pestA[OPAL_PHB4_NUM_PEST_REGS]; + __be64 pestB[OPAL_PHB4_NUM_PEST_REGS]; +}; + +enum { + OPAL_REINIT_CPUS_HILE_BE = (1 << 0), + OPAL_REINIT_CPUS_HILE_LE = (1 << 1), + + /* These two define the base MMU mode of the host on P9 + * + * On P9 Nimbus DD2.0 and Cumlus (and later), KVM can still + * create hash guests in "radix" mode with care (full core + * switch only). + */ + OPAL_REINIT_CPUS_MMU_HASH = (1 << 2), + OPAL_REINIT_CPUS_MMU_RADIX = (1 << 3), + + OPAL_REINIT_CPUS_TM_SUSPEND_DISABLED = (1 << 4), +}; + +typedef struct oppanel_line { + __be64 line; + __be64 line_len; +} oppanel_line_t; + +enum opal_prd_msg_type { + OPAL_PRD_MSG_TYPE_INIT = 0, /* HBRT --> OPAL */ + OPAL_PRD_MSG_TYPE_FINI, /* HBRT/kernel --> OPAL */ + OPAL_PRD_MSG_TYPE_ATTN, /* HBRT <-- OPAL */ + OPAL_PRD_MSG_TYPE_ATTN_ACK, /* HBRT --> OPAL */ + OPAL_PRD_MSG_TYPE_OCC_ERROR, /* HBRT <-- OPAL */ + OPAL_PRD_MSG_TYPE_OCC_RESET, /* HBRT <-- OPAL */ +}; + +struct opal_prd_msg_header { + uint8_t type; + uint8_t pad[1]; + __be16 size; +}; + +struct opal_prd_msg; + +#define OCC_RESET 0 +#define OCC_LOAD 1 +#define OCC_THROTTLE 2 +#define OCC_MAX_THROTTLE_STATUS 5 + +struct opal_occ_msg { + __be64 type; + __be64 chip; + __be64 throttle_status; +}; + +/* + * SG entries + * + * WARNING: The current implementation requires each entry + * to represent a block that is 4k aligned *and* each block + * size except the last one in the list to be as well. + */ +struct opal_sg_entry { + __be64 data; + __be64 length; +}; + +/* + * Candidate image SG list. + * + * length = VER | length + */ +struct opal_sg_list { + __be64 length; + __be64 next; + struct opal_sg_entry entry[]; +}; + +/* + * Dump region ID range usable by the OS + */ +#define OPAL_DUMP_REGION_HOST_START 0x80 +#define OPAL_DUMP_REGION_LOG_BUF 0x80 +#define OPAL_DUMP_REGION_HOST_END 0xFF + +/* CAPI modes for PHB */ +enum { + OPAL_PHB_CAPI_MODE_PCIE = 0, + OPAL_PHB_CAPI_MODE_CAPI = 1, + OPAL_PHB_CAPI_MODE_SNOOP_OFF = 2, + OPAL_PHB_CAPI_MODE_SNOOP_ON = 3, + OPAL_PHB_CAPI_MODE_DMA = 4, + OPAL_PHB_CAPI_MODE_DMA_TVT1 = 5, +}; + +/* OPAL I2C request */ +struct opal_i2c_request { + uint8_t type; +#define OPAL_I2C_RAW_READ 0 +#define OPAL_I2C_RAW_WRITE 1 +#define OPAL_I2C_SM_READ 2 +#define OPAL_I2C_SM_WRITE 3 + uint8_t flags; +#define OPAL_I2C_ADDR_10 0x01 /* Not supported yet */ + uint8_t subaddr_sz; /* Max 4 */ + uint8_t reserved; + __be16 addr; /* 7 or 10 bit address */ + __be16 reserved2; + __be32 subaddr; /* Sub-address if any */ + __be32 size; /* Data size */ + __be64 buffer_ra; /* Buffer real address */ +}; + +/* + * EPOW status sharing (OPAL and the host) + * + * The host will pass on OPAL, a buffer of length OPAL_SYSEPOW_MAX + * with individual elements being 16 bits wide to fetch the system + * wide EPOW status. Each element in the buffer will contain the + * EPOW status in its bit representation for a particular EPOW sub + * class as defined here. So multiple detailed EPOW status bits + * specific for any sub class can be represented in a single buffer + * element as its bit representation. + */ + +/* System EPOW type */ +enum OpalSysEpow { + OPAL_SYSEPOW_POWER = 0, /* Power EPOW */ + OPAL_SYSEPOW_TEMP = 1, /* Temperature EPOW */ + OPAL_SYSEPOW_COOLING = 2, /* Cooling EPOW */ + OPAL_SYSEPOW_MAX = 3, /* Max EPOW categories */ +}; + +/* Power EPOW */ +enum OpalSysPower { + OPAL_SYSPOWER_UPS = 0x0001, /* System on UPS power */ + OPAL_SYSPOWER_CHNG = 0x0002, /* System power config change */ + OPAL_SYSPOWER_FAIL = 0x0004, /* System impending power failure */ + OPAL_SYSPOWER_INCL = 0x0008, /* System incomplete power */ +}; + +/* Temperature EPOW */ +enum OpalSysTemp { + OPAL_SYSTEMP_AMB = 0x0001, /* System over ambient temperature */ + OPAL_SYSTEMP_INT = 0x0002, /* System over internal temperature */ + OPAL_SYSTEMP_HMD = 0x0004, /* System over ambient humidity */ +}; + +/* Cooling EPOW */ +enum OpalSysCooling { + OPAL_SYSCOOL_INSF = 0x0001, /* System insufficient cooling */ +}; + +/* Argument to OPAL_CEC_REBOOT2() */ +enum { + OPAL_REBOOT_NORMAL = 0, + OPAL_REBOOT_PLATFORM_ERROR = 1, + OPAL_REBOOT_FULL_IPL = 2, + OPAL_REBOOT_MPIPL = 3, + OPAL_REBOOT_FAST = 4, +}; + +/* Argument to OPAL_PCI_TCE_KILL */ +enum { + OPAL_PCI_TCE_KILL_PAGES, + OPAL_PCI_TCE_KILL_PE, + OPAL_PCI_TCE_KILL_ALL, +}; + +/* The xive operation mode indicates the active "API" and + * corresponds to the "mode" parameter of the opal_xive_reset() + * call + */ +enum { + OPAL_XIVE_MODE_EMU = 0, + OPAL_XIVE_MODE_EXPL = 1, +}; + +/* Flags for OPAL_XIVE_GET_IRQ_INFO */ +enum { + OPAL_XIVE_IRQ_TRIGGER_PAGE = 0x00000001, + OPAL_XIVE_IRQ_STORE_EOI = 0x00000002, + OPAL_XIVE_IRQ_LSI = 0x00000004, + OPAL_XIVE_IRQ_SHIFT_BUG = 0x00000008, /* P9 DD1.0 workaround */ + OPAL_XIVE_IRQ_MASK_VIA_FW = 0x00000010, /* P9 DD1.0 workaround */ + OPAL_XIVE_IRQ_EOI_VIA_FW = 0x00000020, /* P9 DD1.0 workaround */ + OPAL_XIVE_IRQ_STORE_EOI2 = 0x00000040, +}; + +/* Flags for OPAL_XIVE_GET/SET_QUEUE_INFO */ +enum { + OPAL_XIVE_EQ_ENABLED = 0x00000001, + OPAL_XIVE_EQ_ALWAYS_NOTIFY = 0x00000002, + OPAL_XIVE_EQ_ESCALATE = 0x00000004, +}; + +/* Flags for OPAL_XIVE_GET/SET_VP_INFO */ +enum { + OPAL_XIVE_VP_ENABLED = 0x00000001, + OPAL_XIVE_VP_SINGLE_ESCALATION = 0x00000002, +}; + +/* "Any chip" replacement for chip ID for allocation functions */ +enum { + OPAL_XIVE_ANY_CHIP = 0xffffffff, +}; + +/* Xive sync options */ +enum { + /* This bits are cumulative, arg is a girq */ + XIVE_SYNC_EAS = 0x00000001, /* Sync irq source */ + XIVE_SYNC_QUEUE = 0x00000002, /* Sync irq target */ +}; + +/* Dump options */ +enum { + XIVE_DUMP_TM_HYP = 0, + XIVE_DUMP_TM_POOL = 1, + XIVE_DUMP_TM_OS = 2, + XIVE_DUMP_TM_USER = 3, + XIVE_DUMP_VP = 4, + XIVE_DUMP_EMU_STATE = 5, +}; + +/* "type" argument options for OPAL_IMC_COUNTERS_* calls */ +enum { + OPAL_IMC_COUNTERS_NEST = 1, + OPAL_IMC_COUNTERS_CORE = 2, + OPAL_IMC_COUNTERS_TRACE = 3, +}; + + +/* PCI p2p descriptor */ +#define OPAL_PCI_P2P_ENABLE 0x1 +#define OPAL_PCI_P2P_LOAD 0x2 +#define OPAL_PCI_P2P_STORE 0x4 + +/* MPIPL update operations */ +enum opal_mpipl_ops { + OPAL_MPIPL_ADD_RANGE = 0, + OPAL_MPIPL_REMOVE_RANGE = 1, + OPAL_MPIPL_REMOVE_ALL = 2, + OPAL_MPIPL_FREE_PRESERVED_MEMORY = 3, +}; + +/* Tag will point to various metadata area. Kernel will + * use tag to get metadata value. + */ +enum opal_mpipl_tags { + OPAL_MPIPL_TAG_CPU = 0, + OPAL_MPIPL_TAG_OPAL = 1, + OPAL_MPIPL_TAG_KERNEL = 2, + OPAL_MPIPL_TAG_BOOT_MEM = 3, +}; + +/* Preserved memory details */ +struct opal_mpipl_region { + __be64 src; + __be64 dest; + __be64 size; +}; + +/* Structure version */ +#define OPAL_MPIPL_VERSION 0x01 + +struct opal_mpipl_fadump { + u8 version; + u8 reserved[7]; + __be32 crashing_pir; /* OPAL crashing CPU PIR */ + __be32 cpu_data_version; + __be32 cpu_data_size; + __be32 region_cnt; + struct opal_mpipl_region region[]; +} __packed; + +#endif /* __ASSEMBLER__ */ + +#endif /* __OPAL_API_H */ diff --git a/arch/powerpc/include/asm/opal.h b/arch/powerpc/include/asm/opal.h index 029fe85722aa..0a398265ba04 100644 --- a/arch/powerpc/include/asm/opal.h +++ b/arch/powerpc/include/asm/opal.h @@ -1,542 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * PowerNV OPAL definitions. * * Copyright 2011 IBM Corp. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ -#ifndef __OPAL_H -#define __OPAL_H +#ifndef _ASM_POWERPC_OPAL_H +#define _ASM_POWERPC_OPAL_H -/****** Takeover interface ********/ +#include <asm/opal-api.h> -/* PAPR H-Call used to querty the HAL existence and/or instanciate - * it from within pHyp (tech preview only). - * - * This is exclusively used in prom_init.c - */ +#ifndef __ASSEMBLER__ -#ifndef __ASSEMBLY__ - -struct opal_takeover_args { - u64 k_image; /* r4 */ - u64 k_size; /* r5 */ - u64 k_entry; /* r6 */ - u64 k_entry2; /* r7 */ - u64 hal_addr; /* r8 */ - u64 rd_image; /* r9 */ - u64 rd_size; /* r10 */ - u64 rd_loc; /* r11 */ -}; - -extern long opal_query_takeover(u64 *hal_size, u64 *hal_align); - -extern long opal_do_takeover(struct opal_takeover_args *args); - -struct rtas_args; -extern int opal_enter_rtas(struct rtas_args *args, - unsigned long data, - unsigned long entry); - -#endif /* __ASSEMBLY__ */ - -/****** OPAL APIs ******/ - -/* Return codes */ -#define OPAL_SUCCESS 0 -#define OPAL_PARAMETER -1 -#define OPAL_BUSY -2 -#define OPAL_PARTIAL -3 -#define OPAL_CONSTRAINED -4 -#define OPAL_CLOSED -5 -#define OPAL_HARDWARE -6 -#define OPAL_UNSUPPORTED -7 -#define OPAL_PERMISSION -8 -#define OPAL_NO_MEM -9 -#define OPAL_RESOURCE -10 -#define OPAL_INTERNAL_ERROR -11 -#define OPAL_BUSY_EVENT -12 -#define OPAL_HARDWARE_FROZEN -13 - -/* API Tokens (in r0) */ -#define OPAL_CONSOLE_WRITE 1 -#define OPAL_CONSOLE_READ 2 -#define OPAL_RTC_READ 3 -#define OPAL_RTC_WRITE 4 -#define OPAL_CEC_POWER_DOWN 5 -#define OPAL_CEC_REBOOT 6 -#define OPAL_READ_NVRAM 7 -#define OPAL_WRITE_NVRAM 8 -#define OPAL_HANDLE_INTERRUPT 9 -#define OPAL_POLL_EVENTS 10 -#define OPAL_PCI_SET_HUB_TCE_MEMORY 11 -#define OPAL_PCI_SET_PHB_TCE_MEMORY 12 -#define OPAL_PCI_CONFIG_READ_BYTE 13 -#define OPAL_PCI_CONFIG_READ_HALF_WORD 14 -#define OPAL_PCI_CONFIG_READ_WORD 15 -#define OPAL_PCI_CONFIG_WRITE_BYTE 16 -#define OPAL_PCI_CONFIG_WRITE_HALF_WORD 17 -#define OPAL_PCI_CONFIG_WRITE_WORD 18 -#define OPAL_SET_XIVE 19 -#define OPAL_GET_XIVE 20 -#define OPAL_GET_COMPLETION_TOKEN_STATUS 21 /* obsolete */ -#define OPAL_REGISTER_OPAL_EXCEPTION_HANDLER 22 -#define OPAL_PCI_EEH_FREEZE_STATUS 23 -#define OPAL_PCI_SHPC 24 -#define OPAL_CONSOLE_WRITE_BUFFER_SPACE 25 -#define OPAL_PCI_EEH_FREEZE_CLEAR 26 -#define OPAL_PCI_PHB_MMIO_ENABLE 27 -#define OPAL_PCI_SET_PHB_MEM_WINDOW 28 -#define OPAL_PCI_MAP_PE_MMIO_WINDOW 29 -#define OPAL_PCI_SET_PHB_TABLE_MEMORY 30 -#define OPAL_PCI_SET_PE 31 -#define OPAL_PCI_SET_PELTV 32 -#define OPAL_PCI_SET_MVE 33 -#define OPAL_PCI_SET_MVE_ENABLE 34 -#define OPAL_PCI_GET_XIVE_REISSUE 35 -#define OPAL_PCI_SET_XIVE_REISSUE 36 -#define OPAL_PCI_SET_XIVE_PE 37 -#define OPAL_GET_XIVE_SOURCE 38 -#define OPAL_GET_MSI_32 39 -#define OPAL_GET_MSI_64 40 -#define OPAL_START_CPU 41 -#define OPAL_QUERY_CPU_STATUS 42 -#define OPAL_WRITE_OPPANEL 43 -#define OPAL_PCI_MAP_PE_DMA_WINDOW 44 -#define OPAL_PCI_MAP_PE_DMA_WINDOW_REAL 45 -#define OPAL_PCI_RESET 49 -#define OPAL_PCI_GET_HUB_DIAG_DATA 50 -#define OPAL_PCI_GET_PHB_DIAG_DATA 51 -#define OPAL_PCI_FENCE_PHB 52 -#define OPAL_PCI_REINIT 53 -#define OPAL_PCI_MASK_PE_ERROR 54 -#define OPAL_SET_SLOT_LED_STATUS 55 -#define OPAL_GET_EPOW_STATUS 56 -#define OPAL_SET_SYSTEM_ATTENTION_LED 57 -#define OPAL_RESERVED1 58 -#define OPAL_RESERVED2 59 -#define OPAL_PCI_NEXT_ERROR 60 -#define OPAL_PCI_EEH_FREEZE_STATUS2 61 -#define OPAL_PCI_POLL 62 -#define OPAL_PCI_MSI_EOI 63 -#define OPAL_PCI_GET_PHB_DIAG_DATA2 64 - -#ifndef __ASSEMBLY__ - -/* Other enums */ -enum OpalVendorApiTokens { - OPAL_START_VENDOR_API_RANGE = 1000, OPAL_END_VENDOR_API_RANGE = 1999 -}; - -enum OpalFreezeState { - OPAL_EEH_STOPPED_NOT_FROZEN = 0, - OPAL_EEH_STOPPED_MMIO_FREEZE = 1, - OPAL_EEH_STOPPED_DMA_FREEZE = 2, - OPAL_EEH_STOPPED_MMIO_DMA_FREEZE = 3, - OPAL_EEH_STOPPED_RESET = 4, - OPAL_EEH_STOPPED_TEMP_UNAVAIL = 5, - OPAL_EEH_STOPPED_PERM_UNAVAIL = 6 -}; - -enum OpalEehFreezeActionToken { - OPAL_EEH_ACTION_CLEAR_FREEZE_MMIO = 1, - OPAL_EEH_ACTION_CLEAR_FREEZE_DMA = 2, - OPAL_EEH_ACTION_CLEAR_FREEZE_ALL = 3 -}; - -enum OpalPciStatusToken { - OPAL_EEH_NO_ERROR = 0, - OPAL_EEH_IOC_ERROR = 1, - OPAL_EEH_PHB_ERROR = 2, - OPAL_EEH_PE_ERROR = 3, - OPAL_EEH_PE_MMIO_ERROR = 4, - OPAL_EEH_PE_DMA_ERROR = 5 -}; - -enum OpalPciErrorSeverity { - OPAL_EEH_SEV_NO_ERROR = 0, - OPAL_EEH_SEV_IOC_DEAD = 1, - OPAL_EEH_SEV_PHB_DEAD = 2, - OPAL_EEH_SEV_PHB_FENCED = 3, - OPAL_EEH_SEV_PE_ER = 4, - OPAL_EEH_SEV_INF = 5 -}; - -enum OpalShpcAction { - OPAL_SHPC_GET_LINK_STATE = 0, - OPAL_SHPC_GET_SLOT_STATE = 1 -}; - -enum OpalShpcLinkState { - OPAL_SHPC_LINK_DOWN = 0, - OPAL_SHPC_LINK_UP = 1 -}; - -enum OpalMmioWindowType { - OPAL_M32_WINDOW_TYPE = 1, - OPAL_M64_WINDOW_TYPE = 2, - OPAL_IO_WINDOW_TYPE = 3 -}; - -enum OpalShpcSlotState { - OPAL_SHPC_DEV_NOT_PRESENT = 0, - OPAL_SHPC_DEV_PRESENT = 1 -}; - -enum OpalExceptionHandler { - OPAL_MACHINE_CHECK_HANDLER = 1, - OPAL_HYPERVISOR_MAINTENANCE_HANDLER = 2, - OPAL_SOFTPATCH_HANDLER = 3 -}; - -enum OpalPendingState { - OPAL_EVENT_OPAL_INTERNAL = 0x1, - OPAL_EVENT_NVRAM = 0x2, - OPAL_EVENT_RTC = 0x4, - OPAL_EVENT_CONSOLE_OUTPUT = 0x8, - OPAL_EVENT_CONSOLE_INPUT = 0x10, - OPAL_EVENT_ERROR_LOG_AVAIL = 0x20, - OPAL_EVENT_ERROR_LOG = 0x40, - OPAL_EVENT_EPOW = 0x80, - OPAL_EVENT_LED_STATUS = 0x100, - OPAL_EVENT_PCI_ERROR = 0x200 -}; - -/* Machine check related definitions */ -enum OpalMCE_Version { - OpalMCE_V1 = 1, -}; - -enum OpalMCE_Severity { - OpalMCE_SEV_NO_ERROR = 0, - OpalMCE_SEV_WARNING = 1, - OpalMCE_SEV_ERROR_SYNC = 2, - OpalMCE_SEV_FATAL = 3, -}; - -enum OpalMCE_Disposition { - OpalMCE_DISPOSITION_RECOVERED = 0, - OpalMCE_DISPOSITION_NOT_RECOVERED = 1, -}; - -enum OpalMCE_Initiator { - OpalMCE_INITIATOR_UNKNOWN = 0, - OpalMCE_INITIATOR_CPU = 1, -}; - -enum OpalMCE_ErrorType { - OpalMCE_ERROR_TYPE_UNKNOWN = 0, - OpalMCE_ERROR_TYPE_UE = 1, - OpalMCE_ERROR_TYPE_SLB = 2, - OpalMCE_ERROR_TYPE_ERAT = 3, - OpalMCE_ERROR_TYPE_TLB = 4, -}; - -enum OpalMCE_UeErrorType { - OpalMCE_UE_ERROR_INDETERMINATE = 0, - OpalMCE_UE_ERROR_IFETCH = 1, - OpalMCE_UE_ERROR_PAGE_TABLE_WALK_IFETCH = 2, - OpalMCE_UE_ERROR_LOAD_STORE = 3, - OpalMCE_UE_ERROR_PAGE_TABLE_WALK_LOAD_STORE = 4, -}; - -enum OpalMCE_SlbErrorType { - OpalMCE_SLB_ERROR_INDETERMINATE = 0, - OpalMCE_SLB_ERROR_PARITY = 1, - OpalMCE_SLB_ERROR_MULTIHIT = 2, -}; - -enum OpalMCE_EratErrorType { - OpalMCE_ERAT_ERROR_INDETERMINATE = 0, - OpalMCE_ERAT_ERROR_PARITY = 1, - OpalMCE_ERAT_ERROR_MULTIHIT = 2, -}; - -enum OpalMCE_TlbErrorType { - OpalMCE_TLB_ERROR_INDETERMINATE = 0, - OpalMCE_TLB_ERROR_PARITY = 1, - OpalMCE_TLB_ERROR_MULTIHIT = 2, -}; - -enum OpalThreadStatus { - OPAL_THREAD_INACTIVE = 0x0, - OPAL_THREAD_STARTED = 0x1, - OPAL_THREAD_UNAVAILABLE = 0x2 /* opal-v3 */ -}; - -enum OpalPciBusCompare { - OpalPciBusAny = 0, /* Any bus number match */ - OpalPciBus3Bits = 2, /* Match top 3 bits of bus number */ - OpalPciBus4Bits = 3, /* Match top 4 bits of bus number */ - OpalPciBus5Bits = 4, /* Match top 5 bits of bus number */ - OpalPciBus6Bits = 5, /* Match top 6 bits of bus number */ - OpalPciBus7Bits = 6, /* Match top 7 bits of bus number */ - OpalPciBusAll = 7, /* Match bus number exactly */ -}; - -enum OpalDeviceCompare { - OPAL_IGNORE_RID_DEVICE_NUMBER = 0, - OPAL_COMPARE_RID_DEVICE_NUMBER = 1 -}; - -enum OpalFuncCompare { - OPAL_IGNORE_RID_FUNCTION_NUMBER = 0, - OPAL_COMPARE_RID_FUNCTION_NUMBER = 1 -}; - -enum OpalPeAction { - OPAL_UNMAP_PE = 0, - OPAL_MAP_PE = 1 -}; - -enum OpalPeltvAction { - OPAL_REMOVE_PE_FROM_DOMAIN = 0, - OPAL_ADD_PE_TO_DOMAIN = 1 -}; - -enum OpalMveEnableAction { - OPAL_DISABLE_MVE = 0, - OPAL_ENABLE_MVE = 1 -}; - -enum OpalPciResetAndReinitScope { - OPAL_PHB_COMPLETE = 1, OPAL_PCI_LINK = 2, OPAL_PHB_ERROR = 3, - OPAL_PCI_HOT_RESET = 4, OPAL_PCI_FUNDAMENTAL_RESET = 5, - OPAL_PCI_IODA_TABLE_RESET = 6, -}; - -enum OpalPciResetState { - OPAL_DEASSERT_RESET = 0, - OPAL_ASSERT_RESET = 1 -}; - -enum OpalPciMaskAction { - OPAL_UNMASK_ERROR_TYPE = 0, - OPAL_MASK_ERROR_TYPE = 1 -}; - -enum OpalSlotLedType { - OPAL_SLOT_LED_ID_TYPE = 0, - OPAL_SLOT_LED_FAULT_TYPE = 1 -}; - -enum OpalLedAction { - OPAL_TURN_OFF_LED = 0, - OPAL_TURN_ON_LED = 1, - OPAL_QUERY_LED_STATE_AFTER_BUSY = 2 -}; - -enum OpalEpowStatus { - OPAL_EPOW_NONE = 0, - OPAL_EPOW_UPS = 1, - OPAL_EPOW_OVER_AMBIENT_TEMP = 2, - OPAL_EPOW_OVER_INTERNAL_TEMP = 3 -}; - -struct opal_machine_check_event { - enum OpalMCE_Version version:8; /* 0x00 */ - uint8_t in_use; /* 0x01 */ - enum OpalMCE_Severity severity:8; /* 0x02 */ - enum OpalMCE_Initiator initiator:8; /* 0x03 */ - enum OpalMCE_ErrorType error_type:8; /* 0x04 */ - enum OpalMCE_Disposition disposition:8; /* 0x05 */ - uint8_t reserved_1[2]; /* 0x06 */ - uint64_t gpr3; /* 0x08 */ - uint64_t srr0; /* 0x10 */ - uint64_t srr1; /* 0x18 */ - union { /* 0x20 */ - struct { - enum OpalMCE_UeErrorType ue_error_type:8; - uint8_t effective_address_provided; - uint8_t physical_address_provided; - uint8_t reserved_1[5]; - uint64_t effective_address; - uint64_t physical_address; - uint8_t reserved_2[8]; - } ue_error; - - struct { - enum OpalMCE_SlbErrorType slb_error_type:8; - uint8_t effective_address_provided; - uint8_t reserved_1[6]; - uint64_t effective_address; - uint8_t reserved_2[16]; - } slb_error; - - struct { - enum OpalMCE_EratErrorType erat_error_type:8; - uint8_t effective_address_provided; - uint8_t reserved_1[6]; - uint64_t effective_address; - uint8_t reserved_2[16]; - } erat_error; - - struct { - enum OpalMCE_TlbErrorType tlb_error_type:8; - uint8_t effective_address_provided; - uint8_t reserved_1[6]; - uint64_t effective_address; - uint8_t reserved_2[16]; - } tlb_error; - } u; -}; - -enum { - OPAL_P7IOC_DIAG_TYPE_NONE = 0, - OPAL_P7IOC_DIAG_TYPE_RGC = 1, - OPAL_P7IOC_DIAG_TYPE_BI = 2, - OPAL_P7IOC_DIAG_TYPE_CI = 3, - OPAL_P7IOC_DIAG_TYPE_MISC = 4, - OPAL_P7IOC_DIAG_TYPE_I2C = 5, - OPAL_P7IOC_DIAG_TYPE_LAST = 6 -}; - -struct OpalIoP7IOCErrorData { - uint16_t type; - - /* GEM */ - uint64_t gemXfir; - uint64_t gemRfir; - uint64_t gemRirqfir; - uint64_t gemMask; - uint64_t gemRwof; - - /* LEM */ - uint64_t lemFir; - uint64_t lemErrMask; - uint64_t lemAction0; - uint64_t lemAction1; - uint64_t lemWof; - - union { - struct OpalIoP7IOCRgcErrorData { - uint64_t rgcStatus; /* 3E1C10 */ - uint64_t rgcLdcp; /* 3E1C18 */ - }rgc; - struct OpalIoP7IOCBiErrorData { - uint64_t biLdcp0; /* 3C0100, 3C0118 */ - uint64_t biLdcp1; /* 3C0108, 3C0120 */ - uint64_t biLdcp2; /* 3C0110, 3C0128 */ - uint64_t biFenceStatus; /* 3C0130, 3C0130 */ - - uint8_t biDownbound; /* BI Downbound or Upbound */ - }bi; - struct OpalIoP7IOCCiErrorData { - uint64_t ciPortStatus; /* 3Dn008 */ - uint64_t ciPortLdcp; /* 3Dn010 */ - - uint8_t ciPort; /* Index of CI port: 0/1 */ - }ci; - }; -}; - -/** - * This structure defines the overlay which will be used to store PHB error - * data upon request. - */ -enum { - OPAL_PHB_ERROR_DATA_VERSION_1 = 1, -}; - -enum { - OPAL_PHB_ERROR_DATA_TYPE_P7IOC = 1, -}; - -enum { - OPAL_P7IOC_NUM_PEST_REGS = 128, -}; - -struct OpalIoPhbErrorCommon { - uint32_t version; - uint32_t ioType; - uint32_t len; -}; - -struct OpalIoP7IOCPhbErrorData { - struct OpalIoPhbErrorCommon common; - - uint32_t brdgCtl; - - // P7IOC utl regs - uint32_t portStatusReg; - uint32_t rootCmplxStatus; - uint32_t busAgentStatus; - - // P7IOC cfg regs - uint32_t deviceStatus; - uint32_t slotStatus; - uint32_t linkStatus; - uint32_t devCmdStatus; - uint32_t devSecStatus; - - // cfg AER regs - uint32_t rootErrorStatus; - uint32_t uncorrErrorStatus; - uint32_t corrErrorStatus; - uint32_t tlpHdr1; - uint32_t tlpHdr2; - uint32_t tlpHdr3; - uint32_t tlpHdr4; - uint32_t sourceId; - - uint32_t rsv3; - - // Record data about the call to allocate a buffer. - uint64_t errorClass; - uint64_t correlator; - - //P7IOC MMIO Error Regs - uint64_t p7iocPlssr; // n120 - uint64_t p7iocCsr; // n110 - uint64_t lemFir; // nC00 - uint64_t lemErrorMask; // nC18 - uint64_t lemWOF; // nC40 - uint64_t phbErrorStatus; // nC80 - uint64_t phbFirstErrorStatus; // nC88 - uint64_t phbErrorLog0; // nCC0 - uint64_t phbErrorLog1; // nCC8 - uint64_t mmioErrorStatus; // nD00 - uint64_t mmioFirstErrorStatus; // nD08 - uint64_t mmioErrorLog0; // nD40 - uint64_t mmioErrorLog1; // nD48 - uint64_t dma0ErrorStatus; // nD80 - uint64_t dma0FirstErrorStatus; // nD88 - uint64_t dma0ErrorLog0; // nDC0 - uint64_t dma0ErrorLog1; // nDC8 - uint64_t dma1ErrorStatus; // nE00 - uint64_t dma1FirstErrorStatus; // nE08 - uint64_t dma1ErrorLog0; // nE40 - uint64_t dma1ErrorLog1; // nE48 - uint64_t pestA[OPAL_P7IOC_NUM_PEST_REGS]; - uint64_t pestB[OPAL_P7IOC_NUM_PEST_REGS]; -}; - -typedef struct oppanel_line { - const char * line; - uint64_t line_len; -} oppanel_line_t; +#include <linux/notifier.h> + +/* We calculate number of sg entries based on PAGE_SIZE */ +#define SG_ENTRIES_PER_NODE ((PAGE_SIZE - 16) / sizeof(struct opal_sg_entry)) + +/* Default time to sleep or delay between OPAL_BUSY/OPAL_BUSY_EVENT loops */ +#define OPAL_BUSY_DELAY_MS 10 + +/* /sys/firmware/opal */ +extern struct kobject *opal_kobj; + +/* /ibm,opal */ +extern struct device_node *opal_node; /* API functions */ -int64_t opal_console_write(int64_t term_number, int64_t *length, +int64_t opal_invalid_call(void); +int64_t opal_npu_map_lpar(uint64_t phb_id, uint64_t bdf, uint64_t lparid, + uint64_t lpcr); +int64_t opal_npu_spa_setup(uint64_t phb_id, uint32_t bdfn, + uint64_t addr, uint64_t PE_mask); +int64_t opal_npu_spa_clear_cache(uint64_t phb_id, uint32_t bdfn, + uint64_t PE_handle); +int64_t opal_npu_tl_set(uint64_t phb_id, uint32_t bdfn, long cap, + uint64_t rate_phys, uint32_t size); + +int64_t opal_console_write(int64_t term_number, __be64 *length, const uint8_t *buffer); -int64_t opal_console_read(int64_t term_number, int64_t *length, +int64_t opal_console_read(int64_t term_number, __be64 *length, uint8_t *buffer); int64_t opal_console_write_buffer_space(int64_t term_number, - int64_t *length); -int64_t opal_rtc_read(uint32_t *year_month_day, - uint64_t *hour_minute_second_millisecond); + __be64 *length); +int64_t opal_console_flush(int64_t term_number); +int64_t opal_rtc_read(__be32 *year_month_day, + __be64 *hour_minute_second_millisecond); int64_t opal_rtc_write(uint32_t year_month_day, uint64_t hour_minute_second_millisecond); +int64_t opal_tpo_read(uint64_t token, __be32 *year_mon_day, __be32 *hour_min); +int64_t opal_tpo_write(uint64_t token, uint32_t year_mon_day, + uint32_t hour_min); int64_t opal_cec_power_down(uint64_t request); int64_t opal_cec_reboot(void); +int64_t opal_cec_reboot2(uint32_t reboot_type, const char *diag); int64_t opal_read_nvram(uint64_t buffer, uint64_t size, uint64_t offset); int64_t opal_write_nvram(uint64_t buffer, uint64_t size, uint64_t offset); -int64_t opal_handle_interrupt(uint64_t isn, uint64_t *outstanding_event_mask); -int64_t opal_poll_events(uint64_t *outstanding_event_mask); +int64_t opal_handle_interrupt(uint64_t isn, __be64 *outstanding_event_mask); +int64_t opal_poll_events(__be64 *outstanding_event_mask); int64_t opal_pci_set_hub_tce_memory(uint64_t hub_id, uint64_t tce_mem_addr, uint64_t tce_mem_size); int64_t opal_pci_set_phb_tce_memory(uint64_t phb_id, uint64_t tce_mem_addr, @@ -544,9 +65,9 @@ int64_t opal_pci_set_phb_tce_memory(uint64_t phb_id, uint64_t tce_mem_addr, int64_t opal_pci_config_read_byte(uint64_t phb_id, uint64_t bus_dev_func, uint64_t offset, uint8_t *data); int64_t opal_pci_config_read_half_word(uint64_t phb_id, uint64_t bus_dev_func, - uint64_t offset, uint16_t *data); + uint64_t offset, __be16 *data); int64_t opal_pci_config_read_word(uint64_t phb_id, uint64_t bus_dev_func, - uint64_t offset, uint32_t *data); + uint64_t offset, __be32 *data); int64_t opal_pci_config_write_byte(uint64_t phb_id, uint64_t bus_dev_func, uint64_t offset, uint8_t data); int64_t opal_pci_config_write_half_word(uint64_t phb_id, uint64_t bus_dev_func, @@ -554,16 +75,20 @@ int64_t opal_pci_config_write_half_word(uint64_t phb_id, uint64_t bus_dev_func, int64_t opal_pci_config_write_word(uint64_t phb_id, uint64_t bus_dev_func, uint64_t offset, uint32_t data); int64_t opal_set_xive(uint32_t isn, uint16_t server, uint8_t priority); -int64_t opal_get_xive(uint32_t isn, uint16_t *server, uint8_t *priority); +int64_t opal_get_xive(uint32_t isn, __be16 *server, uint8_t *priority); int64_t opal_register_exception_handler(uint64_t opal_exception, uint64_t handler_address, uint64_t glue_cache_line); int64_t opal_pci_eeh_freeze_status(uint64_t phb_id, uint64_t pe_number, uint8_t *freeze_state, - uint16_t *pci_error_type, - uint64_t *phb_status); + __be16 *pci_error_type, + __be64 *phb_status); int64_t opal_pci_eeh_freeze_clear(uint64_t phb_id, uint64_t pe_number, uint64_t eeh_action_token); +int64_t opal_pci_eeh_freeze_set(uint64_t phb_id, uint64_t pe_number, + uint64_t eeh_action_token); +int64_t opal_pci_err_inject(uint64_t phb_id, uint32_t pe_no, uint32_t type, + uint32_t func, uint64_t addr, uint64_t mask); int64_t opal_pci_shpc(uint64_t phb_id, uint64_t shpc_action, uint8_t *state); @@ -574,7 +99,7 @@ int64_t opal_pci_set_phb_mem_window(uint64_t phb_id, uint16_t window_type, uint16_t window_num, uint64_t starting_real_address, uint64_t starting_pci_address, - uint16_t segment_size); + uint64_t size); int64_t opal_pci_map_pe_mmio_window(uint64_t phb_id, uint16_t pe_number, uint16_t window_type, uint16_t window_num, uint16_t segment_num); @@ -587,9 +112,6 @@ int64_t opal_pci_set_pe(uint64_t phb_id, uint64_t pe_number, uint64_t bus_dev_fu uint8_t pe_action); int64_t opal_pci_set_peltv(uint64_t phb_id, uint32_t parent_pe, uint32_t child_pe, uint8_t state); -int64_t opal_pci_set_mve(uint64_t phb_id, uint32_t mve_number, uint32_t pe_number); -int64_t opal_pci_set_mve_enable(uint64_t phb_id, uint32_t mve_number, - uint32_t state); int64_t opal_pci_get_xive_reissue(uint64_t phb_id, uint32_t xive_number, uint8_t *p_bit, uint8_t *q_bit); int64_t opal_pci_set_xive_reissue(uint64_t phb_id, uint32_t xive_number, @@ -598,13 +120,13 @@ int64_t opal_pci_msi_eoi(uint64_t phb_id, uint32_t hw_irq); int64_t opal_pci_set_xive_pe(uint64_t phb_id, uint32_t pe_number, uint32_t xive_num); int64_t opal_get_xive_source(uint64_t phb_id, uint32_t xive_num, - int32_t *interrupt_source_number); + __be32 *interrupt_source_number); int64_t opal_get_msi_32(uint64_t phb_id, uint32_t mve_number, uint32_t xive_num, - uint8_t msi_range, uint32_t *msi_address, - uint32_t *message_data); + uint8_t msi_range, __be32 *msi_address, + __be32 *message_data); int64_t opal_get_msi_64(uint64_t phb_id, uint32_t mve_number, uint32_t xive_num, uint8_t msi_range, - uint64_t *msi_address, uint32_t *message_data); + __be64 *msi_address, __be32 *message_data); int64_t opal_start_cpu(uint64_t thread_number, uint64_t start_address); int64_t opal_query_cpu_status(uint64_t thread_number, uint8_t *thread_status); int64_t opal_write_oppanel(oppanel_line_t *lines, uint64_t num_lines); @@ -614,7 +136,7 @@ int64_t opal_pci_map_pe_dma_window(uint64_t phb_id, uint16_t pe_number, uint16_t int64_t opal_pci_map_pe_dma_window_real(uint64_t phb_id, uint16_t pe_number, uint16_t dma_window_number, uint64_t pci_start_addr, uint64_t pci_mem_size); -int64_t opal_pci_reset(uint64_t phb_id, uint8_t reset_scope, uint8_t assert_state); +int64_t opal_pci_reset(uint64_t id, uint8_t reset_scope, uint8_t assert_state); int64_t opal_pci_get_hub_diag_data(uint64_t hub_id, void *diag_buffer, uint64_t diag_buffer_len); @@ -623,47 +145,251 @@ int64_t opal_pci_get_phb_diag_data(uint64_t phb_id, void *diag_buffer, int64_t opal_pci_get_phb_diag_data2(uint64_t phb_id, void *diag_buffer, uint64_t diag_buffer_len); int64_t opal_pci_fence_phb(uint64_t phb_id); -int64_t opal_pci_reinit(uint64_t phb_id, uint8_t reinit_scope); +int64_t opal_pci_reinit(uint64_t phb_id, uint64_t reinit_scope, uint64_t data); int64_t opal_pci_mask_pe_error(uint64_t phb_id, uint16_t pe_number, uint8_t error_type, uint8_t mask_action); int64_t opal_set_slot_led_status(uint64_t phb_id, uint64_t slot_id, uint8_t led_type, uint8_t led_action); -int64_t opal_get_epow_status(uint64_t *status); +int64_t opal_get_epow_status(__be16 *epow_status, __be16 *num_epow_classes); +int64_t opal_get_dpo_status(__be64 *dpo_timeout); int64_t opal_set_system_attention_led(uint8_t led_action); -int64_t opal_pci_next_error(uint64_t phb_id, uint64_t *first_frozen_pe, - uint16_t *pci_error_type, uint16_t *severity); -int64_t opal_pci_poll(uint64_t phb_id); - -/* Internal functions */ -extern int early_init_dt_scan_opal(unsigned long node, const char *uname, int depth, void *data); - -extern int opal_get_chars(uint32_t vtermno, char *buf, int count); -extern int opal_put_chars(uint32_t vtermno, const char *buf, int total_len); - -extern void hvc_opal_init_early(void); +int64_t opal_pci_next_error(uint64_t phb_id, __be64 *first_frozen_pe, + __be16 *pci_error_type, __be16 *severity); +int64_t opal_pci_poll(uint64_t id); +int64_t opal_return_cpu(void); +int64_t opal_check_token(uint64_t token); +int64_t opal_reinit_cpus(uint64_t flags); + +int64_t opal_xscom_read(uint32_t gcid, uint64_t pcb_addr, __be64 *val); +int64_t opal_xscom_write(uint32_t gcid, uint64_t pcb_addr, uint64_t val); + +int64_t opal_lpc_write(uint32_t chip_id, enum OpalLPCAddressType addr_type, + uint32_t addr, uint32_t data, uint32_t sz); +int64_t opal_lpc_read(uint32_t chip_id, enum OpalLPCAddressType addr_type, + uint32_t addr, __be32 *data, uint32_t sz); + +int64_t opal_read_elog(uint64_t buffer, uint64_t size, uint64_t log_id); +int64_t opal_get_elog_size(__be64 *log_id, __be64 *size, __be64 *elog_type); +int64_t opal_write_elog(uint64_t buffer, uint64_t size, uint64_t offset); +int64_t opal_send_ack_elog(uint64_t log_id); +void opal_resend_pending_logs(void); + +int64_t opal_validate_flash(uint64_t buffer, uint32_t *size, uint32_t *result); +int64_t opal_manage_flash(uint8_t op); +int64_t opal_update_flash(uint64_t blk_list); +int64_t opal_dump_init(uint8_t dump_type); +int64_t opal_dump_info(__be32 *dump_id, __be32 *dump_size); +int64_t opal_dump_info2(__be32 *dump_id, __be32 *dump_size, __be32 *dump_type); +int64_t opal_dump_read(uint32_t dump_id, uint64_t buffer); +int64_t opal_dump_ack(uint32_t dump_id); +int64_t opal_dump_resend_notification(void); + +int64_t opal_get_msg(uint64_t buffer, uint64_t size); +int64_t opal_write_oppanel_async(uint64_t token, oppanel_line_t *lines, + uint64_t num_lines); +int64_t opal_check_completion(uint64_t buffer, uint64_t size, uint64_t token); +int64_t opal_sync_host_reboot(void); +int64_t opal_get_param(uint64_t token, uint32_t param_id, uint64_t buffer, + uint64_t length); +int64_t opal_set_param(uint64_t token, uint32_t param_id, uint64_t buffer, + uint64_t length); +int64_t opal_sensor_read(uint32_t sensor_hndl, int token, __be32 *sensor_data); +int64_t opal_sensor_read_u64(u32 sensor_hndl, int token, __be64 *sensor_data); +int64_t opal_handle_hmi(void); +int64_t opal_handle_hmi2(__be64 *out_flags); +int64_t opal_register_dump_region(uint32_t id, uint64_t start, uint64_t end); +int64_t opal_unregister_dump_region(uint32_t id); +int64_t opal_slw_set_reg(uint64_t cpu_pir, uint64_t sprn, uint64_t val); +int64_t opal_config_cpu_idle_state(uint64_t state, uint64_t flag); +int64_t opal_pci_set_phb_cxl_mode(uint64_t phb_id, uint64_t mode, uint64_t pe_number); +int64_t opal_pci_get_pbcq_tunnel_bar(uint64_t phb_id, uint64_t *addr); +int64_t opal_pci_set_pbcq_tunnel_bar(uint64_t phb_id, uint64_t addr); +int64_t opal_ipmi_send(uint64_t interface, struct opal_ipmi_msg *msg, + uint64_t msg_len); +int64_t opal_ipmi_recv(uint64_t interface, struct opal_ipmi_msg *msg, + uint64_t *msg_len); +int64_t opal_i2c_request(uint64_t async_token, uint32_t bus_id, + struct opal_i2c_request *oreq); +int64_t opal_prd_msg(struct opal_prd_msg *msg); +int64_t opal_leds_get_ind(char *loc_code, __be64 *led_mask, + __be64 *led_value, __be64 *max_led_type); +int64_t opal_leds_set_ind(uint64_t token, char *loc_code, const u64 led_mask, + const u64 led_value, __be64 *max_led_type); + +int64_t opal_flash_read(uint64_t id, uint64_t offset, uint64_t buf, + uint64_t size, uint64_t token); +int64_t opal_flash_write(uint64_t id, uint64_t offset, uint64_t buf, + uint64_t size, uint64_t token); +int64_t opal_flash_erase(uint64_t id, uint64_t offset, uint64_t size, + uint64_t token); +int64_t opal_get_device_tree(uint32_t phandle, uint64_t buf, uint64_t len); +int64_t opal_pci_get_presence_state(uint64_t id, uint64_t data); +int64_t opal_pci_get_power_state(uint64_t id, uint64_t data); +int64_t opal_pci_set_power_state(uint64_t async_token, uint64_t id, + uint64_t data); +int64_t opal_pci_poll2(uint64_t id, uint64_t data); + +int64_t opal_int_get_xirr(__be32 *out_xirr, bool just_poll); +int64_t opal_int_set_cppr(uint8_t cppr); +int64_t opal_int_eoi(uint32_t xirr); +int64_t opal_int_set_mfrr(uint32_t cpu, uint8_t mfrr); +int64_t opal_pci_tce_kill(uint64_t phb_id, uint32_t kill_type, + uint32_t pe_num, uint32_t tce_size, + uint64_t dma_addr, uint32_t npages); +int64_t opal_nmmu_set_ptcr(uint64_t chip_id, uint64_t ptcr); +int64_t opal_xive_reset(uint64_t version); +int64_t opal_xive_get_irq_info(uint32_t girq, + __be64 *out_flags, + __be64 *out_eoi_page, + __be64 *out_trig_page, + __be32 *out_esb_shift, + __be32 *out_src_chip); +int64_t opal_xive_get_irq_config(uint32_t girq, __be64 *out_vp, + uint8_t *out_prio, __be32 *out_lirq); +int64_t opal_xive_set_irq_config(uint32_t girq, uint64_t vp, uint8_t prio, + uint32_t lirq); +int64_t opal_xive_get_queue_info(uint64_t vp, uint32_t prio, + __be64 *out_qpage, + __be64 *out_qsize, + __be64 *out_qeoi_page, + __be32 *out_escalate_irq, + __be64 *out_qflags); +int64_t opal_xive_set_queue_info(uint64_t vp, uint32_t prio, + uint64_t qpage, + uint64_t qsize, + uint64_t qflags); +int64_t opal_xive_donate_page(uint32_t chip_id, uint64_t addr); +int64_t opal_xive_alloc_vp_block(uint32_t alloc_order); +int64_t opal_xive_free_vp_block(uint64_t vp); +int64_t opal_xive_get_vp_info(uint64_t vp, + __be64 *out_flags, + __be64 *out_cam_value, + __be64 *out_report_cl_pair, + __be32 *out_chip_id); +int64_t opal_xive_set_vp_info(uint64_t vp, + uint64_t flags, + uint64_t report_cl_pair); +int64_t opal_xive_allocate_irq_raw(uint32_t chip_id); +int64_t opal_xive_free_irq(uint32_t girq); +int64_t opal_xive_sync(uint32_t type, uint32_t id); +int64_t opal_xive_dump(uint32_t type, uint32_t id); +int64_t opal_xive_get_queue_state(uint64_t vp, uint32_t prio, + __be32 *out_qtoggle, + __be32 *out_qindex); +int64_t opal_xive_set_queue_state(uint64_t vp, uint32_t prio, + uint32_t qtoggle, + uint32_t qindex); +int64_t opal_xive_get_vp_state(uint64_t vp, __be64 *out_w01); + +int64_t opal_imc_counters_init(uint32_t type, uint64_t address, + uint64_t cpu_pir); +int64_t opal_imc_counters_start(uint32_t type, uint64_t cpu_pir); +int64_t opal_imc_counters_stop(uint32_t type, uint64_t cpu_pir); + +int opal_get_powercap(u32 handle, int token, u32 *pcap); +int opal_set_powercap(u32 handle, int token, u32 pcap); +int opal_get_power_shift_ratio(u32 handle, int token, u32 *psr); +int opal_set_power_shift_ratio(u32 handle, int token, u32 psr); +int opal_sensor_group_clear(u32 group_hndl, int token); +int opal_sensor_group_enable(u32 group_hndl, int token, bool enable); +int opal_nx_coproc_init(uint32_t chip_id, uint32_t ct); + +int opal_secvar_get(const char *key, uint64_t key_len, u8 *data, + uint64_t *data_size); +int opal_secvar_get_next(const char *key, uint64_t *key_len, + uint64_t key_buf_size); +int opal_secvar_enqueue_update(const char *key, uint64_t key_len, u8 *data, + uint64_t data_size); + +s64 opal_mpipl_update(enum opal_mpipl_ops op, u64 src, u64 dest, u64 size); +s64 opal_mpipl_register_tag(enum opal_mpipl_tags tag, u64 addr); +s64 opal_mpipl_query_tag(enum opal_mpipl_tags tag, __be64 *addr); + +s64 opal_signal_system_reset(s32 cpu); +s64 opal_quiesce(u64 shutdown_type, s32 cpu); /* Internal functions */ extern int early_init_dt_scan_opal(unsigned long node, const char *uname, int depth, void *data); +extern int early_init_dt_scan_recoverable_ranges(unsigned long node, + const char *uname, int depth, void *data); +void __init opal_configure_cores(void); + +extern ssize_t opal_get_chars(uint32_t vtermno, u8 *buf, size_t count); +extern ssize_t opal_put_chars(uint32_t vtermno, const u8 *buf, + size_t total_len); +extern ssize_t opal_put_chars_atomic(uint32_t vtermno, const u8 *buf, + size_t total_len); +extern int opal_flush_chars(uint32_t vtermno, bool wait); +extern int opal_flush_console(uint32_t vtermno); -extern int opal_notifier_register(struct notifier_block *nb); -extern void opal_notifier_enable(void); -extern void opal_notifier_disable(void); -extern void opal_notifier_update_evt(uint64_t evt_mask, uint64_t evt_val); +extern void hvc_opal_init_early(void); -extern int opal_get_chars(uint32_t vtermno, char *buf, int count); -extern int opal_put_chars(uint32_t vtermno, const char *buf, int total_len); +extern int opal_message_notifier_register(enum opal_msg_type msg_type, + struct notifier_block *nb); +extern int opal_message_notifier_unregister(enum opal_msg_type msg_type, + struct notifier_block *nb); -extern void hvc_opal_init_early(void); +extern int opal_async_get_token_interruptible(void); +extern int opal_async_release_token(int token); +extern int opal_async_wait_response(uint64_t token, struct opal_msg *msg); +extern int opal_async_wait_response_interruptible(uint64_t token, + struct opal_msg *msg); +extern int opal_get_sensor_data(u32 sensor_hndl, u32 *sensor_data); +extern int opal_get_sensor_data_u64(u32 sensor_hndl, u64 *sensor_data); +extern int sensor_group_enable(u32 grp_hndl, bool enable); struct rtc_time; -extern int opal_set_rtc_time(struct rtc_time *tm); -extern void opal_get_rtc_time(struct rtc_time *tm); -extern unsigned long opal_get_boot_time(void); +extern time64_t opal_get_boot_time(void); extern void opal_nvram_init(void); +extern void opal_flash_update_init(void); +extern void opal_flash_update_print_message(void); +extern int opal_elog_init(void); +extern void opal_platform_dump_init(void); +extern void opal_sys_param_init(void); +extern void opal_msglog_init(void); +extern void opal_msglog_sysfs_init(void); +extern int opal_async_comp_init(void); +extern int opal_sensor_init(void); +extern int opal_hmi_handler_init(void); +extern int opal_event_init(void); +int opal_power_control_init(void); extern int opal_machine_check(struct pt_regs *regs); +extern bool opal_mce_check_early_recovery(struct pt_regs *regs); +extern int opal_hmi_exception_early(struct pt_regs *regs); +extern int opal_hmi_exception_early2(struct pt_regs *regs); +extern int opal_handle_hmi_exception(struct pt_regs *regs); extern void opal_shutdown(void); +extern int opal_resync_timebase(void); + +extern void opal_lpc_init(void); + +extern void opal_kmsg_init(void); + +extern int opal_event_request(unsigned int opal_event_nr); + +struct opal_sg_list *opal_vmalloc_to_sg_list(void *vmalloc_addr, + unsigned long vmalloc_size); +void opal_free_sg_list(struct opal_sg_list *sg); + +extern int opal_error_code(int rc); + +ssize_t opal_msglog_copy(char *to, loff_t pos, size_t count); + +static inline int opal_get_async_rc(struct opal_msg msg) +{ + if (msg.msg_type != OPAL_MSG_ASYNC_COMP) + return OPAL_PARAMETER; + else + return be64_to_cpu(msg.params[1]); +} + +void opal_wake_poller(void); + +void opal_powercap_init(void); +void opal_psr_init(void); +void opal_sensor_groups_init(void); -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ -#endif /* __OPAL_H */ +#endif /* _ASM_POWERPC_OPAL_H */ diff --git a/arch/powerpc/include/asm/oprofile_impl.h b/arch/powerpc/include/asm/oprofile_impl.h deleted file mode 100644 index d697b08994c9..000000000000 --- a/arch/powerpc/include/asm/oprofile_impl.h +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (C) 2004 Anton Blanchard <anton@au.ibm.com>, IBM - * - * Based on alpha version. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ - -#ifndef _ASM_POWERPC_OPROFILE_IMPL_H -#define _ASM_POWERPC_OPROFILE_IMPL_H -#ifdef __KERNEL__ - -#define OP_MAX_COUNTER 8 - -/* Per-counter configuration as set via oprofilefs. */ -struct op_counter_config { - unsigned long enabled; - unsigned long event; - unsigned long count; - /* Classic doesn't support per-counter user/kernel selection */ - unsigned long kernel; - unsigned long user; - unsigned long unit_mask; -}; - -/* System-wide configuration as set via oprofilefs. */ -struct op_system_config { -#ifdef CONFIG_PPC64 - unsigned long mmcr0; - unsigned long mmcr1; - unsigned long mmcra; -#ifdef CONFIG_OPROFILE_CELL - /* Register for oprofile user tool to check cell kernel profiling - * support. - */ - unsigned long cell_support; -#endif -#endif - unsigned long enable_kernel; - unsigned long enable_user; -}; - -/* Per-arch configuration */ -struct op_powerpc_model { - int (*reg_setup) (struct op_counter_config *, - struct op_system_config *, - int num_counters); - int (*cpu_setup) (struct op_counter_config *); - int (*start) (struct op_counter_config *); - int (*global_start) (struct op_counter_config *); - void (*stop) (void); - void (*global_stop) (void); - int (*sync_start)(void); - int (*sync_stop)(void); - void (*handle_interrupt) (struct pt_regs *, - struct op_counter_config *); - int num_counters; -}; - -extern struct op_powerpc_model op_model_fsl_emb; -extern struct op_powerpc_model op_model_rs64; -extern struct op_powerpc_model op_model_power4; -extern struct op_powerpc_model op_model_7450; -extern struct op_powerpc_model op_model_cell; -extern struct op_powerpc_model op_model_pa6t; - - -/* All the classic PPC parts use these */ -static inline unsigned int classic_ctr_read(unsigned int i) -{ - switch(i) { - case 0: - return mfspr(SPRN_PMC1); - case 1: - return mfspr(SPRN_PMC2); - case 2: - return mfspr(SPRN_PMC3); - case 3: - return mfspr(SPRN_PMC4); - case 4: - return mfspr(SPRN_PMC5); - case 5: - return mfspr(SPRN_PMC6); - -/* No PPC32 chip has more than 6 so far */ -#ifdef CONFIG_PPC64 - case 6: - return mfspr(SPRN_PMC7); - case 7: - return mfspr(SPRN_PMC8); -#endif - default: - return 0; - } -} - -static inline void classic_ctr_write(unsigned int i, unsigned int val) -{ - switch(i) { - case 0: - mtspr(SPRN_PMC1, val); - break; - case 1: - mtspr(SPRN_PMC2, val); - break; - case 2: - mtspr(SPRN_PMC3, val); - break; - case 3: - mtspr(SPRN_PMC4, val); - break; - case 4: - mtspr(SPRN_PMC5, val); - break; - case 5: - mtspr(SPRN_PMC6, val); - break; - -/* No PPC32 chip has more than 6, yet */ -#ifdef CONFIG_PPC64 - case 6: - mtspr(SPRN_PMC7, val); - break; - case 7: - mtspr(SPRN_PMC8, val); - break; -#endif - default: - break; - } -} - - -extern void op_powerpc_backtrace(struct pt_regs * const regs, unsigned int depth); - -#endif /* __KERNEL__ */ -#endif /* _ASM_POWERPC_OPROFILE_IMPL_H */ diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h index 77c91e74b612..1d58da946739 100644 --- a/arch/powerpc/include/asm/paca.h +++ b/arch/powerpc/include/asm/paca.h @@ -1,14 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * This control block defines the PACA which defines the processor * specific data for each logical processor on the system. * There are some pointers defined that are utilized by PLIC. * * C 2001 PPC 64 Team, IBM Corp - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef _ASM_POWERPC_PACA_H #define _ASM_POWERPC_PACA_H @@ -16,15 +12,26 @@ #ifdef CONFIG_PPC64 -#include <linux/init.h> +#include <linux/cache.h> +#include <linux/string.h> #include <asm/types.h> -#include <asm/lppaca.h> #include <asm/mmu.h> #include <asm/page.h> +#ifdef CONFIG_PPC_BOOK3E_64 #include <asm/exception-64e.h> +#else +#include <asm/exception-64s.h> +#endif #ifdef CONFIG_KVM_BOOK3S_64_HANDLER #include <asm/kvm_book3s_asm.h> #endif +#include <asm/accounting.h> +#include <asm/hmi.h> +#include <asm/cpuidle.h> +#include <asm/atomic.h> +#include <asm/mce.h> + +#include <asm-generic/mmiowb_types.h> register struct paca_struct *local_paca asm("r13"); @@ -39,11 +46,11 @@ extern unsigned int debug_smp_processor_id(void); /* from linux/smp.h */ #define get_paca() local_paca #endif -#define get_lppaca() (get_paca()->lppaca_ptr) #define get_slb_shadow() (get_paca()->slb_shadow_ptr) struct task_struct; -struct opal_machine_check_event; +struct rtas_args; +struct lppaca; /* * Defines the layout of the paca. @@ -52,7 +59,7 @@ struct opal_machine_check_event; * processor. */ struct paca_struct { -#ifdef CONFIG_PPC_BOOK3S +#ifdef CONFIG_PPC_PSERIES /* * Because hw_cpu_id, unlike other paca fields, is accessed * routinely from other CPUs (from the IRQ code), we stick to @@ -61,55 +68,77 @@ struct paca_struct { */ struct lppaca *lppaca_ptr; /* Pointer to LpPaca for PLIC */ -#endif /* CONFIG_PPC_BOOK3S */ +#endif /* CONFIG_PPC_PSERIES */ + /* * MAGIC: the spinlock functions in arch/powerpc/lib/locks.c * load lock_token and paca_index with a single lwz * instruction. They must travel together and be properly * aligned. */ +#ifdef __BIG_ENDIAN__ u16 lock_token; /* Constant 0x8000, used in locks */ u16 paca_index; /* Logical processor number */ +#else + u16 paca_index; /* Logical processor number */ + u16 lock_token; /* Constant 0x8000, used in locks */ +#endif +#ifndef CONFIG_PPC_KERNEL_PCREL u64 kernel_toc; /* Kernel TOC address */ +#endif u64 kernelbase; /* Base address of kernel */ u64 kernel_msr; /* MSR while running in kernel */ -#ifdef CONFIG_PPC_STD_MMU_64 - u64 stab_real; /* Absolute address of segment table */ - u64 stab_addr; /* Virtual address of segment table */ -#endif /* CONFIG_PPC_STD_MMU_64 */ void *emergency_sp; /* pointer to emergency stack */ u64 data_offset; /* per cpu data offset */ s16 hw_cpu_id; /* Physical processor number */ u8 cpu_start; /* At startup, processor spins until */ /* this becomes non-zero. */ u8 kexec_state; /* set when kexec down has irqs off */ -#ifdef CONFIG_PPC_STD_MMU_64 +#ifdef CONFIG_PPC_BOOK3S_64 +#ifdef CONFIG_PPC_64S_HASH_MMU struct slb_shadow *slb_shadow_ptr; +#endif struct dtl_entry *dispatch_log; struct dtl_entry *dispatch_log_end; +#endif + u64 dscr_default; /* per-CPU default DSCR */ +#ifdef CONFIG_PPC_BOOK3S_64 /* * Now, starting in cacheline 2, the exception save areas */ /* used for most interrupts/exceptions */ - u64 exgen[12] __attribute__((aligned(0x80))); - u64 exmc[12]; /* used for machine checks */ - u64 exslb[12]; /* used for SLB/segment table misses - * on the linear mapping */ + u64 exgen[EX_SIZE] __attribute__((aligned(0x80))); + +#ifdef CONFIG_PPC_64S_HASH_MMU /* SLB related definitions */ u16 vmalloc_sllp; - u16 slb_cache_ptr; + u8 slb_cache_ptr; + u8 stab_rr; /* stab/slb round-robin counter */ +#ifdef CONFIG_DEBUG_VM + u8 in_kernel_slb_handler; +#endif + u32 slb_used_bitmap; /* Bitmaps for first 32 SLB entries. */ + u32 slb_kern_bitmap; u32 slb_cache[SLB_CACHE_ENTRIES]; -#endif /* CONFIG_PPC_STD_MMU_64 */ +#endif +#endif /* CONFIG_PPC_BOOK3S_64 */ -#ifdef CONFIG_PPC_BOOK3E - u64 exgen[8] __attribute__((aligned(0x80))); +#ifdef CONFIG_PPC_BOOK3E_64 + u64 exgen[8] __aligned(0x40); /* Keep pgd in the same cacheline as the start of extlb */ - pgd_t *pgd __attribute__((aligned(0x80))); /* Current PGD */ + pgd_t *pgd __aligned(0x40); /* Current PGD */ pgd_t *kernel_pgd; /* Kernel PGD */ - /* We can have up to 3 levels of reentrancy in the TLB miss handler */ - u64 extlb[3][EX_TLB_SIZE / sizeof(u64)]; + + /* Shared by all threads of a core -- points to tcd of first thread */ + struct tlb_core_data *tcd_ptr; + + /* + * We can have up to 3 levels of reentrancy in the TLB miss handler, + * in each of four exception levels (normal, crit, mcheck, debug). + */ + u64 extlb[12][EX_TLB_SIZE / sizeof(u64)]; u64 exmc[8]; /* used for machine checks */ u64 excrit[8]; /* used for crit interrupts */ u64 exdbg[8]; /* used for debug interrupts */ @@ -118,67 +147,151 @@ struct paca_struct { void *mc_kstack; void *crit_kstack; void *dbg_kstack; -#endif /* CONFIG_PPC_BOOK3E */ - mm_context_t context; + struct tlb_core_data tcd; +#endif /* CONFIG_PPC_BOOK3E_64 */ + +#ifdef CONFIG_PPC_64S_HASH_MMU + unsigned char mm_ctx_low_slices_psize[BITS_PER_LONG / BITS_PER_BYTE]; + unsigned char mm_ctx_high_slices_psize[SLICE_ARRAY_SIZE]; +#endif /* * then miscellaneous read-write fields */ struct task_struct *__current; /* Pointer to current */ u64 kstack; /* Saved Kernel stack addr */ - u64 stab_rr; /* stab/slb round-robin counter */ - u64 saved_r1; /* r1 save for RTAS calls or PM */ + u64 saved_r1; /* r1 save for RTAS calls or PM or EE=0 */ u64 saved_msr; /* MSR saved here by enter_rtas */ + u64 exit_save_r1; /* Syscall/interrupt R1 save */ +#ifdef CONFIG_PPC_BOOK3E_64 u16 trap_save; /* Used when bad stack is encountered */ - u8 soft_enabled; /* irq soft-enable flag */ +#endif +#ifdef CONFIG_PPC_BOOK3S_64 + u8 hsrr_valid; /* HSRRs set for HRFID */ + u8 srr_valid; /* SRRs set for RFID */ +#endif + u8 irq_soft_mask; /* mask for irq soft masking */ u8 irq_happened; /* irq happened while soft-disabled */ - u8 io_sync; /* writel() needs spin_unlock sync */ u8 irq_work_pending; /* IRQ_WORK interrupt while soft-disable */ - u8 nap_state_lost; /* NV GPR values lost in power7_idle */ - u64 sprg3; /* Saved user-visible sprg */ +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE + u8 pmcregs_in_use; /* pseries puts this in lppaca */ +#endif + u64 sprg_vdso; /* Saved user-visible sprg */ #ifdef CONFIG_PPC_TRANSACTIONAL_MEM u64 tm_scratch; /* TM scratch area for reclaim */ #endif #ifdef CONFIG_PPC_POWERNV - /* Pointer to OPAL machine check event structure set by the - * early exception handler for use by high level C handler + /* PowerNV idle fields */ + /* PNV_CORE_IDLE_* bits, all siblings work on thread 0 paca */ + unsigned long idle_lock; /* A value of 1 means acquired */ + unsigned long idle_state; + union { + /* P7/P8 specific fields */ + struct { + /* PNV_THREAD_RUNNING/NAP/SLEEP */ + u8 thread_idle_state; + /* Mask to denote subcore sibling threads */ + u8 subcore_sibling_mask; + }; + + /* P9 specific fields */ + struct { +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE + /* The PSSCR value that the kernel requested before going to stop */ + u64 requested_psscr; + /* Flag to request this thread not to stop */ + atomic_t dont_stop; +#endif + }; + }; +#endif + +#ifdef CONFIG_PPC_BOOK3S_64 + /* Non-maskable exceptions that are not performance critical */ + u64 exnmi[EX_SIZE]; /* used for system reset (nmi) */ + u64 exmc[EX_SIZE]; /* used for machine checks */ + /* Exclusive stacks for system reset and machine check exception. */ + void *nmi_emergency_sp; + void *mc_emergency_sp; + + u16 in_nmi; /* In nmi handler */ + + /* + * Flag to check whether we are in machine check early handler + * and already using emergency stack. */ - struct opal_machine_check_event *opal_mc_evt; + u16 in_mce; + u8 hmi_event_available; /* HMI event is available */ + u8 hmi_p9_special_emu; /* HMI P9 special emulation */ + u32 hmi_irqs; /* HMI irq stat */ #endif + u8 ftrace_enabled; /* Hard disable ftrace */ /* Stuff for accurate time accounting */ - u64 user_time; /* accumulated usermode TB ticks */ - u64 system_time; /* accumulated system TB ticks */ - u64 user_time_scaled; /* accumulated usermode SPURR ticks */ - u64 starttime; /* TB value snapshot */ - u64 starttime_user; /* TB value on exit to usermode */ - u64 startspurr; /* SPURR value snapshot */ - u64 utime_sspurr; /* ->user_time when ->startspurr set */ - u64 stolen_time; /* TB ticks taken by hypervisor */ + struct cpu_accounting_data accounting; u64 dtl_ridx; /* read index in dispatch log */ struct dtl_entry *dtl_curr; /* pointer corresponding to dtl_ridx */ #ifdef CONFIG_KVM_BOOK3S_HANDLER -#ifdef CONFIG_KVM_BOOK3S_PR +#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE /* We use this to store guest state in */ struct kvmppc_book3s_shadow_vcpu shadow_vcpu; #endif struct kvmppc_host_state kvm_hstate; +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE + /* + * Bitmap for sibling subcore status. See kvm/book3s_hv_ras.c for + * more details + */ + struct sibling_subcore_state *sibling_subcore_state; +#endif +#endif +#ifdef CONFIG_PPC_BOOK3S_64 + /* + * rfi fallback flush must be in its own cacheline to prevent + * other paca data leaking into the L1d + */ + u64 exrfi[EX_SIZE] __aligned(0x80); + void *rfi_flush_fallback_area; + u64 l1d_flush_size; +#endif +#ifdef CONFIG_PPC_PSERIES + u8 *mce_data_buf; /* buffer to hold per cpu rtas errlog */ +#endif /* CONFIG_PPC_PSERIES */ + +#ifdef CONFIG_PPC_BOOK3S_64 +#ifdef CONFIG_PPC_64S_HASH_MMU + /* Capture SLB related old contents in MCE handler. */ + struct slb_entry *mce_faulty_slbs; + u16 slb_save_cache_ptr; +#endif +#endif /* CONFIG_PPC_BOOK3S_64 */ +#ifdef CONFIG_STACKPROTECTOR + unsigned long canary; +#endif +#ifdef CONFIG_MMIOWB + struct mmiowb_state mmiowb_state; #endif -}; +#ifdef CONFIG_PPC_BOOK3S_64 + struct mce_info *mce_info; + u8 mce_pending_irq_work; +#endif /* CONFIG_PPC_BOOK3S_64 */ +} ____cacheline_aligned; -extern struct paca_struct *paca; +extern void copy_mm_to_paca(struct mm_struct *mm); +extern struct paca_struct **paca_ptrs; extern void initialise_paca(struct paca_struct *new_paca, int cpu); extern void setup_paca(struct paca_struct *new_paca); -extern void allocate_pacas(void); +extern void allocate_paca_ptrs(void); +extern void allocate_paca(int cpu); extern void free_unused_pacas(void); #else /* CONFIG_PPC64 */ -static inline void allocate_pacas(void) { }; -static inline void free_unused_pacas(void) { }; +static inline void allocate_paca(int cpu) { } +static inline void free_unused_pacas(void) { } #endif /* CONFIG_PPC64 */ diff --git a/arch/powerpc/include/asm/page.h b/arch/powerpc/include/asm/page.h index 988c812aab5b..b28fbb1d57eb 100644 --- a/arch/powerpc/include/asm/page.h +++ b/arch/powerpc/include/asm/page.h @@ -1,46 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _ASM_POWERPC_PAGE_H #define _ASM_POWERPC_PAGE_H /* * Copyright (C) 2001,2005 IBM Corporation. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> +#include <linux/kernel.h> +#include <linux/bug.h> #else #include <asm/types.h> #endif -#include <asm/asm-compat.h> -#include <asm/kdump.h> +#include <asm/asm-const.h> /* * On regular PPC32 page size is 4K (but we support 4K/16K/64K/256K pages - * on PPC44x). For PPC64 we support either 4K or 64K software + * on PPC44x and 4K/16K on 8xx). For PPC64 we support either 4K or 64K software * page size. When using 64K pages however, whether we are really supporting * 64K pages in HW or not is irrelevant to those definitions. */ -#if defined(CONFIG_PPC_256K_PAGES) -#define PAGE_SHIFT 18 -#elif defined(CONFIG_PPC_64K_PAGES) -#define PAGE_SHIFT 16 -#elif defined(CONFIG_PPC_16K_PAGES) -#define PAGE_SHIFT 14 -#else -#define PAGE_SHIFT 12 -#endif +#include <vdso/page.h> -#define PAGE_SIZE (ASM_CONST(1) << PAGE_SHIFT) - -#ifndef __ASSEMBLY__ -#ifdef CONFIG_HUGETLB_PAGE -extern unsigned int HPAGE_SHIFT; -#else +#ifndef __ASSEMBLER__ +#ifndef CONFIG_HUGETLB_PAGE #define HPAGE_SHIFT PAGE_SHIFT +#elif defined(CONFIG_PPC_BOOK3S_64) +extern unsigned int hpage_shift; +#define HPAGE_SHIFT hpage_shift +#elif defined(CONFIG_PPC_8xx) +#define HPAGE_SHIFT 19 /* 512k pages */ +#elif defined(CONFIG_PPC_E500) +#define HPAGE_SHIFT 22 /* 4M pages */ #endif #define HPAGE_SIZE ((1UL) << HPAGE_SHIFT) #define HPAGE_MASK (~(HPAGE_SIZE - 1)) @@ -48,16 +40,6 @@ extern unsigned int HPAGE_SHIFT; #define HUGE_MAX_HSTATE (MMU_PAGE_COUNT-1) #endif -/* We do define AT_SYSINFO_EHDR but don't use the gate mechanism */ -#define __HAVE_ARCH_GATE_AREA 1 - -/* - * Subtle: (1 << PAGE_SHIFT) is an int, not an unsigned long. So if we - * assign PAGE_MASK to a larger type it gets extended the way we want - * (i.e. with 1s in the high bits) - */ -#define PAGE_MASK (~((1 << PAGE_SHIFT) - 1)) - /* * KERNELBASE is the virtual address of the start of the kernel, it's often * the same as PAGE_OFFSET, but _might not be_. @@ -78,7 +60,7 @@ extern unsigned int HPAGE_SHIFT; * * Also, KERNELBASE >= PAGE_OFFSET and PHYSICAL_START >= MEMORY_START * - * There are two was to determine a physical address from a virtual one: + * There are two ways to determine a physical address from a virtual one: * va = pa + PAGE_OFFSET - MEMORY_START * va = pa + KERNELBASE - PHYSICAL_START * @@ -93,16 +75,16 @@ extern unsigned int HPAGE_SHIFT; #define LOAD_OFFSET ASM_CONST((CONFIG_KERNEL_START-CONFIG_PHYSICAL_START)) #if defined(CONFIG_NONSTATIC_KERNEL) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ extern phys_addr_t memstart_addr; extern phys_addr_t kernstart_addr; -#ifdef CONFIG_RELOCATABLE_PPC32 +#if defined(CONFIG_RELOCATABLE) && defined(CONFIG_PPC32) extern long long virt_phys_offset; #endif -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #define PHYSICAL_START kernstart_addr #else /* !CONFIG_NONSTATIC_KERNEL */ @@ -110,12 +92,13 @@ extern long long virt_phys_offset; #endif /* See Description below for VIRT_PHYS_OFFSET */ -#ifdef CONFIG_RELOCATABLE_PPC32 +#if defined(CONFIG_PPC32) && defined(CONFIG_BOOKE) +#ifdef CONFIG_RELOCATABLE #define VIRT_PHYS_OFFSET virt_phys_offset #else #define VIRT_PHYS_OFFSET (KERNELBASE - PHYSICAL_START) #endif - +#endif #ifdef CONFIG_PPC64 #define MEMORY_START 0UL @@ -127,21 +110,16 @@ extern long long virt_phys_offset; #ifdef CONFIG_FLATMEM #define ARCH_PFN_OFFSET ((unsigned long)(MEMORY_START >> PAGE_SHIFT)) -#define pfn_valid(pfn) ((pfn) >= ARCH_PFN_OFFSET && (pfn) < max_mapnr) #endif -#define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT) -#define pfn_to_kaddr(pfn) __va((pfn) << PAGE_SHIFT) -#define virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT) - /* * On Book-E parts we need __va to parse the device tree and we can't * determine MEMORY_START until then. However we can determine PHYSICAL_START * from information at hand (program counter, TLB lookup). * - * On BookE with RELOCATABLE (RELOCATABLE_PPC32) + * On BookE with RELOCATABLE && PPC32 * - * With RELOCATABLE_PPC32, we support loading the kernel at any physical + * With RELOCATABLE && PPC32, we support loading the kernel at any physical * address without any restriction on the page alignment. * * We find the runtime address of _stext and relocate ourselves based on @@ -207,24 +185,63 @@ extern long long virt_phys_offset; * On non-Book-E PPC64 PAGE_OFFSET and MEMORY_START are constants so use * the other definitions for __va & __pa. */ -#ifdef CONFIG_BOOKE +#if defined(CONFIG_PPC32) && defined(CONFIG_BOOKE) #define __va(x) ((void *)(unsigned long)((phys_addr_t)(x) + VIRT_PHYS_OFFSET)) -#define __pa(x) ((unsigned long)(x) - VIRT_PHYS_OFFSET) +#define __pa(x) ((phys_addr_t)(unsigned long)(x) - VIRT_PHYS_OFFSET) #else +#ifdef CONFIG_PPC64 + +#define VIRTUAL_WARN_ON(x) WARN_ON(IS_ENABLED(CONFIG_DEBUG_VIRTUAL) && (x)) + +/* + * gcc miscompiles (unsigned long)(&static_var) - PAGE_OFFSET + * with -mcmodel=medium, so we use & and | instead of - and + on 64-bit. + * This also results in better code generation. + */ +#define __va(x) \ +({ \ + VIRTUAL_WARN_ON((unsigned long)(x) >= PAGE_OFFSET); \ + (void *)(unsigned long)((phys_addr_t)(x) | PAGE_OFFSET); \ +}) + +#define __pa(x) \ +({ \ + VIRTUAL_WARN_ON((unsigned long)(x) < PAGE_OFFSET); \ + (unsigned long)(x) & 0x0fffffffffffffffUL; \ +}) + +#else /* 32-bit, non book E */ #define __va(x) ((void *)(unsigned long)((phys_addr_t)(x) + PAGE_OFFSET - MEMORY_START)) #define __pa(x) ((unsigned long)(x) - PAGE_OFFSET + MEMORY_START) #endif +#endif + +#ifndef __ASSEMBLER__ +static inline unsigned long virt_to_pfn(const void *kaddr) +{ + return __pa(kaddr) >> PAGE_SHIFT; +} + +static inline const void *pfn_to_kaddr(unsigned long pfn) +{ + return __va(pfn << PAGE_SHIFT); +} +#endif + +#define virt_to_page(kaddr) pfn_to_page(virt_to_pfn(kaddr)) +#define virt_addr_valid(vaddr) ({ \ + unsigned long _addr = (unsigned long)vaddr; \ + _addr >= PAGE_OFFSET && _addr < (unsigned long)high_memory && \ + pfn_valid(virt_to_pfn((void *)_addr)); \ +}) /* * Unfortunately the PLT is in the BSS in the PPC32 ELF ABI, * and needs to be executable. This means the whole heap ends * up being executable. */ -#define VM_DATA_DEFAULT_FLAGS32 (VM_READ | VM_WRITE | VM_EXEC | \ - VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) - -#define VM_DATA_DEFAULT_FLAGS64 (VM_READ | VM_WRITE | \ - VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) +#define VM_DATA_DEFAULT_FLAGS32 VM_DATA_FLAGS_TSK_EXEC +#define VM_DATA_DEFAULT_FLAGS64 VM_DATA_FLAGS_NON_EXEC #ifdef __powerpc64__ #include <asm/page_64.h> @@ -232,158 +249,30 @@ extern long long virt_phys_offset; #include <asm/page_32.h> #endif -/* align addr on a size boundary - adjust address up/down if needed */ -#define _ALIGN_UP(addr,size) (((addr)+((size)-1))&(~((size)-1))) -#define _ALIGN_DOWN(addr,size) ((addr)&(~((size)-1))) - -/* align addr on a size boundary - adjust address up if needed */ -#define _ALIGN(addr,size) _ALIGN_UP(addr,size) - /* * Don't compare things with KERNELBASE or PAGE_OFFSET to test for * "kernelness", use is_kernel_addr() - it should do what you want. */ #ifdef CONFIG_PPC_BOOK3E_64 #define is_kernel_addr(x) ((x) >= 0x8000000000000000ul) -#else +#elif defined(CONFIG_PPC_BOOK3S_64) #define is_kernel_addr(x) ((x) >= PAGE_OFFSET) -#endif - -#ifndef CONFIG_PPC_BOOK3S_64 -/* - * Use the top bit of the higher-level page table entries to indicate whether - * the entries we point to contain hugepages. This works because we know that - * the page tables live in kernel space. If we ever decide to support having - * page tables at arbitrary addresses, this breaks and will have to change. - */ -#ifdef CONFIG_PPC64 -#define PD_HUGE 0x8000000000000000 #else -#define PD_HUGE 0x80000000 +#define is_kernel_addr(x) ((x) >= TASK_SIZE) #endif -#endif /* CONFIG_PPC_BOOK3S_64 */ - -/* - * Some number of bits at the level of the page table that points to - * a hugepte are used to encode the size. This masks those bits. - */ -#define HUGEPD_SHIFT_MASK 0x3f -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ -#undef STRICT_MM_TYPECHECKS - -#ifdef STRICT_MM_TYPECHECKS -/* These are used to make use of C type-checking. */ - -/* PTE level */ -typedef struct { pte_basic_t pte; } pte_t; -#define pte_val(x) ((x).pte) -#define __pte(x) ((pte_t) { (x) }) - -/* 64k pages additionally define a bigger "real PTE" type that gathers - * the "second half" part of the PTE for pseudo 64k pages - */ -#if defined(CONFIG_PPC_64K_PAGES) && defined(CONFIG_PPC_STD_MMU_64) -typedef struct { pte_t pte; unsigned long hidx; } real_pte_t; -#else -typedef struct { pte_t pte; } real_pte_t; -#endif - -/* PMD level */ -#ifdef CONFIG_PPC64 -typedef struct { unsigned long pmd; } pmd_t; -#define pmd_val(x) ((x).pmd) -#define __pmd(x) ((pmd_t) { (x) }) - -/* PUD level exusts only on 4k pages */ -#ifndef CONFIG_PPC_64K_PAGES -typedef struct { unsigned long pud; } pud_t; -#define pud_val(x) ((x).pud) -#define __pud(x) ((pud_t) { (x) }) -#endif /* !CONFIG_PPC_64K_PAGES */ -#endif /* CONFIG_PPC64 */ - -/* PGD level */ -typedef struct { unsigned long pgd; } pgd_t; -#define pgd_val(x) ((x).pgd) -#define __pgd(x) ((pgd_t) { (x) }) - -/* Page protection bits */ -typedef struct { unsigned long pgprot; } pgprot_t; -#define pgprot_val(x) ((x).pgprot) -#define __pgprot(x) ((pgprot_t) { (x) }) - -#else - -/* - * .. while these make it easier on the compiler - */ - -typedef pte_basic_t pte_t; -#define pte_val(x) (x) -#define __pte(x) (x) - -#if defined(CONFIG_PPC_64K_PAGES) && defined(CONFIG_PPC_STD_MMU_64) -typedef struct { pte_t pte; unsigned long hidx; } real_pte_t; -#else -typedef pte_t real_pte_t; -#endif - - -#ifdef CONFIG_PPC64 -typedef unsigned long pmd_t; -#define pmd_val(x) (x) -#define __pmd(x) (x) - -#ifndef CONFIG_PPC_64K_PAGES -typedef unsigned long pud_t; -#define pud_val(x) (x) -#define __pud(x) (x) -#endif /* !CONFIG_PPC_64K_PAGES */ -#endif /* CONFIG_PPC64 */ - -typedef unsigned long pgd_t; -#define pgd_val(x) (x) -#define pgprot_val(x) (x) - -typedef unsigned long pgprot_t; -#define __pgd(x) (x) -#define __pgprot(x) (x) - -#endif - -typedef struct { signed long pd; } hugepd_t; - -#ifdef CONFIG_HUGETLB_PAGE #ifdef CONFIG_PPC_BOOK3S_64 -static inline int hugepd_ok(hugepd_t hpd) -{ - /* - * hugepd pointer, bottom two bits == 00 and next 4 bits - * indicate size of table - */ - return (((hpd.pd & 0x3) == 0x0) && ((hpd.pd & HUGEPD_SHIFT_MASK) != 0)); -} +#include <asm/pgtable-be-types.h> #else -static inline int hugepd_ok(hugepd_t hpd) -{ - return (hpd.pd > 0); -} +#include <asm/pgtable-types.h> #endif -#define is_hugepd(pdep) (hugepd_ok(*((hugepd_t *)(pdep)))) -int pgd_huge(pgd_t pgd); -#else /* CONFIG_HUGETLB_PAGE */ -#define is_hugepd(pdep) 0 -#define pgd_huge(pgd) 0 -#endif /* CONFIG_HUGETLB_PAGE */ - struct page; extern void clear_user_page(void *page, unsigned long vaddr, struct page *pg); extern void copy_user_page(void *to, void *from, unsigned long vaddr, struct page *p); -extern int page_is_ram(unsigned long pfn); extern int devmem_is_allowed(unsigned long pfn); #ifdef CONFIG_PPC_SMLPAR @@ -393,13 +282,14 @@ void arch_free_page(struct page *page, int order); struct vm_area_struct; -#ifdef CONFIG_PPC_64K_PAGES -typedef pte_t *pgtable_t; -#else -typedef struct page *pgtable_t; -#endif +extern unsigned long kernstart_virt_addr; + +static inline unsigned long kaslr_offset(void) +{ + return kernstart_virt_addr - KERNELBASE; +} #include <asm-generic/memory_model.h> -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _ASM_POWERPC_PAGE_H */ diff --git a/arch/powerpc/include/asm/page_32.h b/arch/powerpc/include/asm/page_32.h index 68d73b2a7bfc..25482405a811 100644 --- a/arch/powerpc/include/asm/page_32.h +++ b/arch/powerpc/include/asm/page_32.h @@ -1,6 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_PAGE_32_H #define _ASM_POWERPC_PAGE_32_H +#include <asm/cache.h> + #if defined(CONFIG_PHYSICAL_ALIGN) && (CONFIG_PHYSICAL_START != 0) #if (CONFIG_PHYSICAL_START % CONFIG_PHYSICAL_ALIGN) != 0 #error "CONFIG_PHYSICAL_START must be a multiple of CONFIG_PHYSICAL_ALIGN" @@ -9,23 +12,14 @@ #define VM_DATA_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS32 -#ifdef CONFIG_NOT_COHERENT_CACHE -#define ARCH_DMA_MINALIGN L1_CACHE_BYTES -#endif - -#ifdef CONFIG_PTE_64BIT -#define PTE_FLAGS_OFFSET 4 /* offset of PTE flags, in bytes */ -#else -#define PTE_FLAGS_OFFSET 0 -#endif - -#ifdef CONFIG_PPC_256K_PAGES +#if defined(CONFIG_PPC_256K_PAGES) || \ + (defined(CONFIG_PPC_8xx) && defined(CONFIG_PPC_16K_PAGES)) #define PTE_SHIFT (PAGE_SHIFT - PTE_T_LOG2 - 2) /* 1/4 of a page */ #else #define PTE_SHIFT (PAGE_SHIFT - PTE_T_LOG2) /* full page */ #endif -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * The basic type of a PTE - 64 bits for those CPUs with > 32 bit * physical addressing. @@ -36,9 +30,22 @@ typedef unsigned long long pte_basic_t; typedef unsigned long pte_basic_t; #endif -struct page; -extern void clear_pages(void *page, int order); -static inline void clear_page(void *page) { clear_pages(page, 0); } +#include <asm/bug.h> + +/* + * Clear page using the dcbz instruction, which doesn't cause any + * memory traffic (except to write out any cache lines which get + * displaced). This only works on cacheable memory. + */ +static inline void clear_page(void *addr) +{ + unsigned int i; + + WARN_ON((unsigned long)addr & (L1_CACHE_BYTES - 1)); + + for (i = 0; i < PAGE_SIZE / L1_CACHE_BYTES; i++, addr += L1_CACHE_BYTES) + dcbz(addr); +} extern void copy_page(void *to, void *from); #include <asm-generic/getorder.h> @@ -46,6 +53,6 @@ extern void copy_page(void *to, void *from); #define PGD_T_LOG2 (__builtin_ffs(sizeof(pgd_t)) - 1) #define PTE_T_LOG2 (__builtin_ffs(sizeof(pte_t)) - 1) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _ASM_POWERPC_PAGE_32_H */ diff --git a/arch/powerpc/include/asm/page_64.h b/arch/powerpc/include/asm/page_64.h index 88693cef4f3d..0f564a06bf68 100644 --- a/arch/powerpc/include/asm/page_64.h +++ b/arch/powerpc/include/asm/page_64.h @@ -1,15 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _ASM_POWERPC_PAGE_64_H #define _ASM_POWERPC_PAGE_64_H /* * Copyright (C) 2001 PPC64 Team, IBM Corp - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ +#include <asm/asm-const.h> + /* * We always define HW_PAGE_SHIFT to 12 as use of 64K pages remains Linux * specific, every notion of page number shared with the firmware, TCEs, @@ -37,25 +35,45 @@ #define ESID_MASK_1T 0xffffff0000000000UL #define GET_ESID_1T(x) (((x) >> SID_SHIFT_1T) & SID_MASK_1T) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm/cache.h> typedef unsigned long pte_basic_t; -static __inline__ void clear_page(void *addr) +static inline void clear_page(void *addr) { - unsigned long lines, line_size; - - line_size = ppc64_caches.dline_size; - lines = ppc64_caches.dlines_per_page; - - __asm__ __volatile__( + unsigned long iterations; + unsigned long onex, twox, fourx, eightx; + + iterations = ppc64_caches.l1d.blocks_per_page / 8; + + /* + * Some verisions of gcc use multiply instructions to + * calculate the offsets so lets give it a hand to + * do better. + */ + onex = ppc64_caches.l1d.block_size; + twox = onex << 1; + fourx = onex << 2; + eightx = onex << 3; + + asm volatile( "mtctr %1 # clear_page\n\ -1: dcbz 0,%0\n\ - add %0,%0,%3\n\ + .balign 16\n\ +1: dcbz 0,%0\n\ + dcbz %3,%0\n\ + dcbz %4,%0\n\ + dcbz %5,%0\n\ + dcbz %6,%0\n\ + dcbz %7,%0\n\ + dcbz %8,%0\n\ + dcbz %9,%0\n\ + add %0,%0,%10\n\ bdnz+ 1b" - : "=r" (addr) - : "r" (lines), "0" (addr), "r" (line_size) + : "=&r" (addr) + : "r" (iterations), "0" (addr), "b" (onex), "b" (twox), + "b" (twox+onex), "b" (fourx), "b" (fourx+onex), + "b" (twox+fourx), "b" (eightx-onex), "r" (eightx) : "ctr", "memory"); } @@ -64,84 +82,7 @@ extern void copy_page(void *to, void *from); /* Log 2 of page table size */ extern u64 ppc64_pft_size; -#endif /* __ASSEMBLY__ */ - -#ifdef CONFIG_PPC_MM_SLICES - -#define SLICE_LOW_SHIFT 28 -#define SLICE_HIGH_SHIFT 40 - -#define SLICE_LOW_TOP (0x100000000ul) -#define SLICE_NUM_LOW (SLICE_LOW_TOP >> SLICE_LOW_SHIFT) -#define SLICE_NUM_HIGH (PGTABLE_RANGE >> SLICE_HIGH_SHIFT) - -#define GET_LOW_SLICE_INDEX(addr) ((addr) >> SLICE_LOW_SHIFT) -#define GET_HIGH_SLICE_INDEX(addr) ((addr) >> SLICE_HIGH_SHIFT) - -/* - * 1 bit per slice and we have one slice per 1TB - * Right now we support only 64TB. - * IF we change this we will have to change the type - * of high_slices - */ -#define SLICE_MASK_SIZE 8 - -#ifndef __ASSEMBLY__ - -struct slice_mask { - u16 low_slices; - u64 high_slices; -}; - -struct mm_struct; - -extern unsigned long slice_get_unmapped_area(unsigned long addr, - unsigned long len, - unsigned long flags, - unsigned int psize, - int topdown); - -extern unsigned int get_slice_psize(struct mm_struct *mm, - unsigned long addr); - -extern void slice_init_context(struct mm_struct *mm, unsigned int psize); -extern void slice_set_user_psize(struct mm_struct *mm, unsigned int psize); -extern void slice_set_range_psize(struct mm_struct *mm, unsigned long start, - unsigned long len, unsigned int psize); - -#define slice_mm_new_context(mm) ((mm)->context.id == MMU_NO_CONTEXT) - -#endif /* __ASSEMBLY__ */ -#else -#define slice_init() -#ifdef CONFIG_PPC_STD_MMU_64 -#define get_slice_psize(mm, addr) ((mm)->context.user_psize) -#define slice_set_user_psize(mm, psize) \ -do { \ - (mm)->context.user_psize = (psize); \ - (mm)->context.sllp = SLB_VSID_USER | mmu_psize_defs[(psize)].sllp; \ -} while (0) -#else /* CONFIG_PPC_STD_MMU_64 */ -#ifdef CONFIG_PPC_64K_PAGES -#define get_slice_psize(mm, addr) MMU_PAGE_64K -#else /* CONFIG_PPC_64K_PAGES */ -#define get_slice_psize(mm, addr) MMU_PAGE_4K -#endif /* !CONFIG_PPC_64K_PAGES */ -#define slice_set_user_psize(mm, psize) do { BUG(); } while(0) -#endif /* !CONFIG_PPC_STD_MMU_64 */ - -#define slice_set_range_psize(mm, start, len, psize) \ - slice_set_user_psize((mm), (psize)) -#define slice_mm_new_context(mm) 1 -#endif /* CONFIG_PPC_MM_SLICES */ - -#ifdef CONFIG_HUGETLB_PAGE - -#ifdef CONFIG_PPC_MM_SLICES -#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA -#endif - -#endif /* !CONFIG_HUGETLB_PAGE */ +#endif /* __ASSEMBLER__ */ #define VM_DATA_DEFAULT_FLAGS \ (is_32bit_task() ? \ @@ -153,11 +94,8 @@ do { \ * stack by default, so in the absence of a PT_GNU_STACK program header * we turn execute permission off. */ -#define VM_STACK_DEFAULT_FLAGS32 (VM_READ | VM_WRITE | VM_EXEC | \ - VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) - -#define VM_STACK_DEFAULT_FLAGS64 (VM_READ | VM_WRITE | \ - VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) +#define VM_STACK_DEFAULT_FLAGS32 VM_DATA_FLAGS_EXEC +#define VM_STACK_DEFAULT_FLAGS64 VM_DATA_FLAGS_NON_EXEC #define VM_STACK_DEFAULT_FLAGS \ (is_32bit_task() ? \ diff --git a/arch/powerpc/include/asm/papr-sysparm.h b/arch/powerpc/include/asm/papr-sysparm.h new file mode 100644 index 000000000000..a3b5a0d05db6 --- /dev/null +++ b/arch/powerpc/include/asm/papr-sysparm.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _ASM_POWERPC_PAPR_SYSPARM_H +#define _ASM_POWERPC_PAPR_SYSPARM_H + +#include <uapi/asm/papr-sysparm.h> + +typedef struct { + u32 token; +} papr_sysparm_t; + +#define mk_papr_sysparm(x_) ((papr_sysparm_t){ .token = x_, }) + +/* + * Derived from the "Defined Parameters" table in PAPR 7.3.16 System + * Parameters Option. Where the spec says "characteristics", we use + * "attrs" in the symbolic names to keep them from getting too + * unwieldy. + */ +#define PAPR_SYSPARM_SHARED_PROC_LPAR_ATTRS mk_papr_sysparm(20) +#define PAPR_SYSPARM_PROC_MODULE_INFO mk_papr_sysparm(43) +#define PAPR_SYSPARM_COOP_MEM_OVERCOMMIT_ATTRS mk_papr_sysparm(44) +#define PAPR_SYSPARM_TLB_BLOCK_INVALIDATE_ATTRS mk_papr_sysparm(50) +#define PAPR_SYSPARM_LPAR_NAME mk_papr_sysparm(55) +#define PAPR_SYSPARM_HVPIPE_ENABLE mk_papr_sysparm(64) + +/** + * struct papr_sysparm_buf - RTAS work area layout for system parameter functions. + * + * This is the memory layout of the buffers passed to/from + * ibm,get-system-parameter and ibm,set-system-parameter. It is + * distinct from the papr_sysparm_io_block structure that is passed + * between user space and the kernel. + */ +struct papr_sysparm_buf { + __be16 len; + u8 val[PAPR_SYSPARM_MAX_OUTPUT]; +}; + +struct papr_sysparm_buf *papr_sysparm_buf_alloc(void); +void papr_sysparm_buf_free(struct papr_sysparm_buf *buf); +int papr_sysparm_set(papr_sysparm_t param, const struct papr_sysparm_buf *buf); +int papr_sysparm_get(papr_sysparm_t param, struct papr_sysparm_buf *buf); + +#endif /* _ASM_POWERPC_PAPR_SYSPARM_H */ diff --git a/arch/powerpc/include/asm/paravirt.h b/arch/powerpc/include/asm/paravirt.h new file mode 100644 index 000000000000..b78b82d66057 --- /dev/null +++ b/arch/powerpc/include/asm/paravirt.h @@ -0,0 +1,223 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_POWERPC_PARAVIRT_H +#define _ASM_POWERPC_PARAVIRT_H + +#include <linux/jump_label.h> +#include <asm/smp.h> +#ifdef CONFIG_PPC64 +#include <asm/paca.h> +#include <asm/lppaca.h> +#include <asm/hvcall.h> +#endif + +#ifdef CONFIG_PPC_SPLPAR +#include <linux/smp.h> +#include <asm/kvm_guest.h> +#include <asm/cputhreads.h> + +DECLARE_STATIC_KEY_FALSE(shared_processor); + +static inline bool is_shared_processor(void) +{ + return static_branch_unlikely(&shared_processor); +} + +#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING +extern struct static_key paravirt_steal_enabled; +extern struct static_key paravirt_steal_rq_enabled; + +u64 pseries_paravirt_steal_clock(int cpu); + +static inline u64 paravirt_steal_clock(int cpu) +{ + return pseries_paravirt_steal_clock(cpu); +} +#endif + +/* If bit 0 is set, the cpu has been ceded, conferred, or preempted */ +static inline u32 yield_count_of(int cpu) +{ + __be32 yield_count = READ_ONCE(lppaca_of(cpu).yield_count); + return be32_to_cpu(yield_count); +} + +/* + * Spinlock code confers and prods, so don't trace the hcalls because the + * tracing code takes spinlocks which can cause recursion deadlocks. + * + * These calls are made while the lock is not held: the lock slowpath yields if + * it can not acquire the lock, and unlock slow path might prod if a waiter has + * yielded). So this may not be a problem for simple spin locks because the + * tracing does not technically recurse on the lock, but we avoid it anyway. + * + * However the queued spin lock contended path is more strictly ordered: the + * H_CONFER hcall is made after the task has queued itself on the lock, so then + * recursing on that lock will cause the task to then queue up again behind the + * first instance (or worse: queued spinlocks use tricks that assume a context + * never waits on more than one spinlock, so such recursion may cause random + * corruption in the lock code). + */ +static inline void yield_to_preempted(int cpu, u32 yield_count) +{ + plpar_hcall_norets_notrace(H_CONFER, get_hard_smp_processor_id(cpu), yield_count); +} + +static inline void prod_cpu(int cpu) +{ + plpar_hcall_norets_notrace(H_PROD, get_hard_smp_processor_id(cpu)); +} + +static inline void yield_to_any(void) +{ + plpar_hcall_norets_notrace(H_CONFER, -1, 0); +} + +static inline bool is_vcpu_idle(int vcpu) +{ + return lppaca_of(vcpu).idle; +} + +static inline bool vcpu_is_dispatched(int vcpu) +{ + /* + * This is the yield_count. An "odd" value (low bit on) means that + * the processor is yielded (either because of an OS yield or a + * hypervisor preempt). An even value implies that the processor is + * currently executing. + */ + return (!(yield_count_of(vcpu) & 1)); +} +#else +static inline bool is_shared_processor(void) +{ + return false; +} + +static inline u32 yield_count_of(int cpu) +{ + return 0; +} + +extern void ___bad_yield_to_preempted(void); +static inline void yield_to_preempted(int cpu, u32 yield_count) +{ + ___bad_yield_to_preempted(); /* This would be a bug */ +} + +extern void ___bad_yield_to_any(void); +static inline void yield_to_any(void) +{ + ___bad_yield_to_any(); /* This would be a bug */ +} + +extern void ___bad_prod_cpu(void); +static inline void prod_cpu(int cpu) +{ + ___bad_prod_cpu(); /* This would be a bug */ +} + +static inline bool is_vcpu_idle(int vcpu) +{ + return false; +} +static inline bool vcpu_is_dispatched(int vcpu) +{ + return true; +} +#endif + +#define vcpu_is_preempted vcpu_is_preempted +static inline bool vcpu_is_preempted(int cpu) +{ + /* + * The dispatch/yield bit alone is an imperfect indicator of + * whether the hypervisor has dispatched @cpu to run on a physical + * processor. When it is clear, @cpu is definitely not preempted. + * But when it is set, it means only that it *might* be, subject to + * other conditions. So we check other properties of the VM and + * @cpu first, resorting to the yield count last. + */ + + /* + * Hypervisor preemption isn't possible in dedicated processor + * mode by definition. + */ + if (!is_shared_processor()) + return false; + + /* + * If the hypervisor has dispatched the target CPU on a physical + * processor, then the target CPU is definitely not preempted. + */ + if (vcpu_is_dispatched(cpu)) + return false; + + /* + * if the target CPU is not dispatched and the guest OS + * has not marked the CPU idle, then it is hypervisor preempted. + */ + if (!is_vcpu_idle(cpu)) + return true; + +#ifdef CONFIG_PPC_SPLPAR + if (!is_kvm_guest()) { + int first_cpu, i; + + /* + * The result of vcpu_is_preempted() is used in a + * speculative way, and is always subject to invalidation + * by events internal and external to Linux. While we can + * be called in preemptable context (in the Linux sense), + * we're not accessing per-cpu resources in a way that can + * race destructively with Linux scheduler preemption and + * migration, and callers can tolerate the potential for + * error introduced by sampling the CPU index without + * pinning the task to it. So it is permissible to use + * raw_smp_processor_id() here to defeat the preempt debug + * warnings that can arise from using smp_processor_id() + * in arbitrary contexts. + */ + first_cpu = cpu_first_thread_sibling(raw_smp_processor_id()); + + /* + * The PowerVM hypervisor dispatches VMs on a whole core + * basis. So we know that a thread sibling of the executing CPU + * cannot have been preempted by the hypervisor, even if it + * has called H_CONFER, which will set the yield bit. + */ + if (cpu_first_thread_sibling(cpu) == first_cpu) + return false; + + /* + * The specific target CPU was marked by guest OS as idle, but + * then also check all other cpus in the core for PowerVM + * because it does core scheduling and one of the vcpu + * of the core getting preempted by hypervisor implies + * other vcpus can also be considered preempted. + */ + first_cpu = cpu_first_thread_sibling(cpu); + for (i = first_cpu; i < first_cpu + threads_per_core; i++) { + if (i == cpu) + continue; + if (vcpu_is_dispatched(i)) + return false; + if (!is_vcpu_idle(i)) + return true; + } + } +#endif + + /* + * None of the threads in target CPU's core are running but none of + * them were preempted too. Hence assume the target CPU to be + * non-preempted. + */ + return false; +} + +static inline bool pv_is_native_spin_unlock(void) +{ + return !is_shared_processor(); +} + +#endif /* _ASM_POWERPC_PARAVIRT_H */ diff --git a/arch/powerpc/include/asm/paravirt_api_clock.h b/arch/powerpc/include/asm/paravirt_api_clock.h new file mode 100644 index 000000000000..d25ca7ac57c7 --- /dev/null +++ b/arch/powerpc/include/asm/paravirt_api_clock.h @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include <asm/paravirt.h> diff --git a/arch/powerpc/include/asm/parport.h b/arch/powerpc/include/asm/parport.h index a452968b29ea..42cc321ed754 100644 --- a/arch/powerpc/include/asm/parport.h +++ b/arch/powerpc/include/asm/parport.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * parport.h: platform-specific PC-style parport initialisation * @@ -10,7 +11,7 @@ #define _ASM_POWERPC_PARPORT_H #ifdef __KERNEL__ -#include <asm/prom.h> +#include <linux/of_irq.h> static int parport_pc_find_nonpci_ports (int autoirq, int autodma) { @@ -28,7 +29,7 @@ static int parport_pc_find_nonpci_ports (int autoirq, int autodma) io1 = prop[1]; io2 = prop[2]; virq = irq_of_parse_and_map(np, 0); - if (virq == NO_IRQ) + if (!virq) continue; if (parport_pc_probe_port(io1, io2, virq, autodma, NULL, 0) diff --git a/arch/powerpc/include/asm/pasemi_dma.h b/arch/powerpc/include/asm/pasemi_dma.h index eafa5a5f56de..712a0b32120f 100644 --- a/arch/powerpc/include/asm/pasemi_dma.h +++ b/arch/powerpc/include/asm/pasemi_dma.h @@ -1,22 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (C) 2006-2008 PA Semi, Inc * * Hardware register layout and descriptor formats for the on-board * DMA engine on PA Semi PWRficient. Used by ethernet, function and security * drivers. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef ASM_PASEMI_DMA_H diff --git a/arch/powerpc/include/asm/pci-bridge.h b/arch/powerpc/include/asm/pci-bridge.h index 2c1d8cb9b265..1dae53130782 100644 --- a/arch/powerpc/include/asm/pci-bridge.h +++ b/arch/powerpc/include/asm/pci-bridge.h @@ -1,20 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _ASM_POWERPC_PCI_BRIDGE_H #define _ASM_POWERPC_PCI_BRIDGE_H #ifdef __KERNEL__ /* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #include <linux/pci.h> #include <linux/list.h> #include <linux/ioport.h> -#include <asm-generic/pci-bridge.h> +#include <linux/numa.h> +#include <linux/iommu.h> struct device_node; /* + * PCI controller operations + */ +struct pci_controller_ops { + void (*dma_dev_setup)(struct pci_dev *pdev); + void (*dma_bus_setup)(struct pci_bus *bus); + bool (*iommu_bypass_supported)(struct pci_dev *pdev, + u64 mask); + + int (*probe_mode)(struct pci_bus *bus); + + /* Called when pci_enable_device() is called. Returns true to + * allow assignment/enabling of the device. */ + bool (*enable_device_hook)(struct pci_dev *pdev); + + void (*disable_device)(struct pci_dev *pdev); + + void (*release_device)(struct pci_dev *pdev); + + /* Called during PCI resource reassignment */ + resource_size_t (*window_alignment)(struct pci_bus *bus, + unsigned long type); + void (*setup_bridge)(struct pci_bus *bus, + unsigned long type); + void (*reset_secondary_bus)(struct pci_dev *pdev); + +#ifdef CONFIG_PCI_MSI + int (*setup_msi_irqs)(struct pci_dev *pdev, + int nvec, int type); + void (*teardown_msi_irqs)(struct pci_dev *pdev); +#endif + + void (*shutdown)(struct pci_controller *hose); + + struct iommu_group *(*device_group)(struct pci_controller *hose, + struct pci_dev *pdev); +}; + +/* * Structure of a PCI controller (host bridge) */ struct pci_controller { @@ -34,7 +70,7 @@ struct pci_controller { void __iomem *io_base_virt; #ifdef CONFIG_PPC64 - void *io_base_alloc; + void __iomem *io_base_alloc; #endif resource_size_t io_base_phys; resource_size_t pci_io_size; @@ -46,6 +82,7 @@ struct pci_controller { resource_size_t isa_mem_phys; resource_size_t isa_mem_size; + struct pci_controller_ops controller_ops; struct pci_ops *ops; unsigned int __iomem *cfg_addr; void __iomem *cfg_data; @@ -89,9 +126,16 @@ struct pci_controller { #ifdef CONFIG_PPC64 unsigned long buid; + struct pci_dn *pci_data; #endif /* CONFIG_PPC64 */ void *private_data; + + /* IRQ domain hierarchy */ + struct irq_domain *dev_domain; + + /* iommu_ops support */ + struct iommu_device iommu; }; /* These are used for config access before all the PCI probing @@ -119,6 +163,10 @@ extern void setup_indirect_pci(struct pci_controller* hose, extern int indirect_read_config(struct pci_bus *bus, unsigned int devfn, int offset, int len, u32 *val); +extern int __indirect_read_config(struct pci_controller *hose, + unsigned char bus_number, unsigned int devfn, + int offset, int len, u32 *val); + extern int indirect_write_config(struct pci_bus *bus, unsigned int devfn, int offset, int len, u32 val); @@ -127,19 +175,17 @@ static inline struct pci_controller *pci_bus_to_host(const struct pci_bus *bus) return bus->sysdata; } -#ifndef CONFIG_PPC64 - +#ifdef CONFIG_PPC_PMAC extern int pci_device_from_OF_node(struct device_node *node, u8 *bus, u8 *devfn); -extern void pci_create_OF_bus_map(void); +#endif +#ifndef CONFIG_PPC64 -static inline int isa_vaddr_is_ioport(void __iomem *address) -{ - /* No specific ISA handling on ppc32 at this stage, it - * all goes through PCI - */ - return 0; -} +#ifdef CONFIG_PPC_PCI_OF_BUS_MAP +extern void pci_create_OF_bus_map(void); +#else +static inline void pci_create_OF_bus_map(void) {} +#endif #else /* CONFIG_PPC64 */ @@ -150,80 +196,73 @@ static inline int isa_vaddr_is_ioport(void __iomem *address) struct iommu_table; struct pci_dn { + int flags; +#define PCI_DN_FLAG_IOV_VF 0x01 +#define PCI_DN_FLAG_DEAD 0x02 /* Device has been hot-removed */ + int busno; /* pci bus number */ int devfn; /* pci device and function number */ + int vendor_id; /* Vendor ID */ + int device_id; /* Device ID */ + int class_code; /* Device class code */ + struct pci_dn *parent; struct pci_controller *phb; /* for pci devices */ - struct iommu_table *iommu_table; /* for phb's or bridges */ - struct device_node *node; /* back-pointer to the device_node */ + struct iommu_table_group *table_group; /* for phb's or bridges */ int pci_ext_config_space; /* for pci devices */ - - int force_32bit_msi:1; - - struct pci_dev *pcidev; /* back-pointer to the pci device */ #ifdef CONFIG_EEH struct eeh_dev *edev; /* eeh device */ #endif -#define IODA_INVALID_PE (-1) -#ifdef CONFIG_PPC_POWERNV - int pe_number; -#endif +#define IODA_INVALID_PE 0xFFFFFFFF + unsigned int pe_number; +#ifdef CONFIG_PCI_IOV + u16 vfs_expanded; /* number of VFs IOV BAR expanded */ + u16 num_vfs; /* number of VFs enabled*/ + unsigned int *pe_num_map; /* PE# for the first VF PE or array */ + bool m64_single_mode; /* Use M64 BAR in Single Mode */ +#define IODA_INVALID_M64 (-1) + int (*m64_map)[PCI_SRIOV_NUM_BARS]; /* Only used on powernv */ + int last_allow_rc; /* Only used on pseries */ +#endif /* CONFIG_PCI_IOV */ + int mps; /* Maximum Payload Size */ + struct list_head child_list; + struct list_head list; + struct resource holes[PCI_SRIOV_NUM_BARS]; }; /* Get the pointer to a device_node's pci_dn */ #define PCI_DN(dn) ((struct pci_dn *) (dn)->data) +extern struct pci_dn *pci_get_pdn_by_devfn(struct pci_bus *bus, + int devfn); extern struct pci_dn *pci_get_pdn(struct pci_dev *pdev); +extern struct pci_dn *pci_add_device_node_info(struct pci_controller *hose, + struct device_node *dn); +extern void pci_remove_device_node_info(struct device_node *dn); -extern void * update_dn_pci_info(struct device_node *dn, void *data); - -static inline int pci_device_from_OF_node(struct device_node *np, - u8 *bus, u8 *devfn) -{ - if (!PCI_DN(np)) - return -ENODEV; - *bus = PCI_DN(np)->busno; - *devfn = PCI_DN(np)->devfn; - return 0; -} +#ifdef CONFIG_PCI_IOV +struct pci_dn *add_sriov_vf_pdns(struct pci_dev *pdev); +void remove_sriov_vf_pdns(struct pci_dev *pdev); +#endif #if defined(CONFIG_EEH) -static inline struct eeh_dev *of_node_to_eeh_dev(struct device_node *dn) +static inline struct eeh_dev *pdn_to_eeh_dev(struct pci_dn *pdn) { - /* - * For those OF nodes whose parent isn't PCI bridge, they - * don't have PCI_DN actually. So we have to skip them for - * any EEH operations. - */ - if (!dn || !PCI_DN(dn)) - return NULL; - - return PCI_DN(dn)->edev; + return pdn ? pdn->edev : NULL; } #else -#define of_node_to_eeh_dev(x) (NULL) +#define pdn_to_eeh_dev(x) (NULL) #endif /** Find the bus corresponding to the indicated device node */ -extern struct pci_bus *pcibios_find_pci_bus(struct device_node *dn); +extern struct pci_bus *pci_find_bus_by_node(struct device_node *dn); /** Remove all of the PCI devices under this bus */ -extern void __pcibios_remove_pci_devices(struct pci_bus *bus, int purge_pe); -extern void pcibios_remove_pci_devices(struct pci_bus *bus); +extern void pci_hp_remove_devices(struct pci_bus *bus); /** Discover new pci devices under this bus, and add them */ -extern void pcibios_add_pci_devices(struct pci_bus *bus); - - -extern void isa_bridge_find_early(struct pci_controller *hose); - -static inline int isa_vaddr_is_ioport(void __iomem *address) -{ - /* Check if address hits the reserved legacy IO range */ - unsigned long ea = (unsigned long)address; - return ea >= ISA_IO_BASE && ea < ISA_IO_END; -} +extern void pci_hp_add_devices(struct pci_bus *bus); extern int pcibios_unmap_io_space(struct pci_bus *bus); extern int pcibios_map_io_space(struct pci_bus *bus); @@ -231,7 +270,7 @@ extern int pcibios_map_io_space(struct pci_bus *bus); #ifdef CONFIG_NUMA #define PHB_SET_NODE(PHB, NODE) ((PHB)->node = (NODE)) #else -#define PHB_SET_NODE(PHB, NODE) ((PHB)->node = -1) +#define PHB_SET_NODE(PHB, NODE) ((PHB)->node = NUMA_NO_NODE) #endif #endif /* CONFIG_PPC64 */ @@ -240,6 +279,8 @@ extern int pcibios_map_io_space(struct pci_bus *bus); extern struct pci_controller *pci_find_hose_for_OF_device( struct device_node* node); +extern struct pci_controller *pci_find_controller_for_domain(int domain_nr); + /* Fill up host controller resources from the OF node */ extern void pci_process_bridge_OF_ranges(struct pci_controller *hose, struct device_node *dev, int primary); @@ -247,6 +288,7 @@ extern void pci_process_bridge_OF_ranges(struct pci_controller *hose, /* Allocate & free a PCI host bridge structure */ extern struct pci_controller *pcibios_alloc_controller(struct device_node *dev); extern void pcibios_free_controller(struct pci_controller *phb); +extern void pcibios_free_controller_deferred(struct pci_host_bridge *bridge); #ifdef CONFIG_PCI extern int pcibios_vaddr_is_ioport(void __iomem *address); diff --git a/arch/powerpc/include/asm/pci.h b/arch/powerpc/include/asm/pci.h index 6653f2743c4e..46a9c4491ed0 100644 --- a/arch/powerpc/include/asm/pci.h +++ b/arch/powerpc/include/asm/pci.h @@ -1,28 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef __ASM_POWERPC_PCI_H #define __ASM_POWERPC_PCI_H #ifdef __KERNEL__ /* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #include <linux/types.h> #include <linux/slab.h> #include <linux/string.h> -#include <linux/dma-mapping.h> +#include <linux/dma-map-ops.h> +#include <linux/scatterlist.h> #include <asm/machdep.h> -#include <asm/scatterlist.h> #include <asm/io.h> -#include <asm/prom.h> #include <asm/pci-bridge.h> -#include <asm-generic/pci-dma-compat.h> - -/* Return values for ppc_md.pci_probe_mode function */ +/* Return values for pci_controller_ops.probe_mode function */ #define PCI_PROBE_NONE -1 /* Don't look at this bus at all */ #define PCI_PROBE_NORMAL 0 /* Do normal PCI probing */ #define PCI_PROBE_DEVTREE 1 /* Instantiate from device tree */ @@ -30,8 +24,6 @@ #define PCIBIOS_MIN_IO 0x1000 #define PCIBIOS_MIN_MEM 0x10000000 -struct pci_dev; - /* Values for the `which' argument to sys_pciconfig_iobase syscall. */ #define IOBASE_BRIDGE_NUMBER 0 #define IOBASE_MEMORY 1 @@ -46,12 +38,6 @@ struct pci_dev; #define pcibios_assign_all_busses() \ (pci_has_flag(PCI_REASSIGN_ALL_BUS)) -static inline void pcibios_penalize_isa_irq(int irq, int active) -{ - /* We don't do dynamic PCI IRQ allocation */ -} - -#define HAVE_ARCH_PCI_GET_LEGACY_IDE_IRQ static inline int pci_get_legacy_ide_irq(struct pci_dev *dev, int channel) { if (ppc_md.pci_get_legacy_ide_irq) @@ -60,11 +46,9 @@ static inline int pci_get_legacy_ide_irq(struct pci_dev *dev, int channel) } #ifdef CONFIG_PCI -extern void set_pci_dma_ops(struct dma_map_ops *dma_ops); -extern struct dma_map_ops *get_pci_dma_ops(void); +void __init set_pci_dma_ops(const struct dma_map_ops *dma_ops); #else /* CONFIG_PCI */ #define set_pci_dma_ops(d) -#define get_pci_dma_ops() NULL #endif #ifdef CONFIG_PPC64 @@ -76,36 +60,6 @@ extern struct dma_map_ops *get_pci_dma_ops(void); */ #define PCI_DISABLE_MWI -#ifdef CONFIG_PCI -static inline void pci_dma_burst_advice(struct pci_dev *pdev, - enum pci_dma_burst_strategy *strat, - unsigned long *strategy_parameter) -{ - unsigned long cacheline_size; - u8 byte; - - pci_read_config_byte(pdev, PCI_CACHE_LINE_SIZE, &byte); - if (byte == 0) - cacheline_size = 1024; - else - cacheline_size = (int) byte * 4; - - *strat = PCI_DMA_BURST_MULTIPLE; - *strategy_parameter = cacheline_size; -} -#endif - -#else /* 32-bit */ - -#ifdef CONFIG_PCI -static inline void pci_dma_burst_advice(struct pci_dev *pdev, - enum pci_dma_burst_strategy *strat, - unsigned long *strategy_parameter) -{ - *strat = PCI_DMA_BURST_INFINITY; - *strategy_parameter = ~0UL; -} -#endif #endif /* CONFIG_PPC64 */ extern int pci_domain_nr(struct pci_bus *bus); @@ -113,18 +67,13 @@ extern int pci_domain_nr(struct pci_bus *bus); /* Decide whether to display the domain number in /proc */ extern int pci_proc_domain(struct pci_bus *bus); -/* MSI arch hooks */ -#define arch_setup_msi_irqs arch_setup_msi_irqs -#define arch_teardown_msi_irqs arch_teardown_msi_irqs -#define arch_msi_check_device arch_msi_check_device - struct vm_area_struct; -/* Map a range of PCI memory or I/O space for a device into user space */ -int pci_mmap_page_range(struct pci_dev *pdev, struct vm_area_struct *vma, - enum pci_mmap_state mmap_state, int write_combine); -/* Tell drivers/pci/proc.c that we have pci_mmap_page_range() */ -#define HAVE_PCI_MMAP 1 +/* Tell PCI code what kind of PCI resource mappings we support */ +#define HAVE_PCI_MMAP 1 +#define ARCH_GENERIC_PCI_MMAP_RESOURCE 1 +#define arch_can_pci_mmap_io() 1 +#define arch_can_pci_mmap_wc() 1 extern int pci_legacy_read(struct pci_bus *bus, loff_t port, u32 *val, size_t count); @@ -133,27 +82,10 @@ extern int pci_legacy_write(struct pci_bus *bus, loff_t port, u32 val, extern int pci_mmap_legacy_page_range(struct pci_bus *bus, struct vm_area_struct *vma, enum pci_mmap_state mmap_state); - +extern void pci_adjust_legacy_attr(struct pci_bus *bus, + enum pci_mmap_state mmap_type); #define HAVE_PCI_LEGACY 1 -#ifdef CONFIG_PPC64 - -/* The PCI address space does not equal the physical memory address - * space (we have an IOMMU). The IDE and SCSI device layers use - * this boolean for bounce buffer decisions. - */ -#define PCI_DMA_BUS_IS_PHYS (0) - -#else /* 32-bit */ - -/* The PCI address space does equal the physical memory - * address space (no IOMMU). The IDE and SCSI device layers use - * this boolean for bounce buffer decisions. - */ -#define PCI_DMA_BUS_IS_PHYS (1) - -#endif /* CONFIG_PPC64 */ - extern void pcibios_claim_one_bus(struct pci_bus *b); extern void pcibios_finish_adding_to_bus(struct pci_bus *bus); @@ -166,27 +98,22 @@ extern int remove_phb_dynamic(struct pci_controller *phb); extern struct pci_dev *of_create_pci_dev(struct device_node *node, struct pci_bus *bus, int devfn); +extern unsigned int pci_parse_of_flags(u32 addr0, int bridge); + extern void of_scan_pci_bridge(struct pci_dev *dev); extern void of_scan_bus(struct device_node *node, struct pci_bus *bus); extern void of_rescan_bus(struct device_node *node, struct pci_bus *bus); -struct file; -extern pgprot_t pci_phys_mem_access_prot(struct file *file, - unsigned long pfn, +extern pgprot_t pci_phys_mem_access_prot(unsigned long pfn, unsigned long size, pgprot_t prot); -#define HAVE_ARCH_PCI_RESOURCE_TO_USER -extern void pci_resource_to_user(const struct pci_dev *dev, int bar, - const struct resource *rsrc, - resource_size_t *start, resource_size_t *end); - extern resource_size_t pcibios_io_space_offset(struct pci_controller *hose); -extern void pcibios_setup_bus_devices(struct pci_bus *bus); extern void pcibios_setup_bus_self(struct pci_bus *bus); extern void pcibios_setup_phb_io_space(struct pci_controller *hose); extern void pcibios_scan_phb(struct pci_controller *hose); #endif /* __KERNEL__ */ + #endif /* __ASM_POWERPC_PCI_H */ diff --git a/arch/powerpc/include/asm/percpu.h b/arch/powerpc/include/asm/percpu.h index 2cedefddba37..ecf5ac70cfae 100644 --- a/arch/powerpc/include/asm/percpu.h +++ b/arch/powerpc/include/asm/percpu.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_PERCPU_H_ #define _ASM_POWERPC_PERCPU_H_ #ifdef __powerpc64__ @@ -9,13 +10,23 @@ #ifdef CONFIG_SMP -#include <asm/paca.h> - #define __my_cpu_offset local_paca->data_offset #endif /* CONFIG_SMP */ #endif /* __powerpc64__ */ +#if defined(CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK) && defined(CONFIG_SMP) +#include <linux/jump_label.h> +DECLARE_STATIC_KEY_FALSE(__percpu_first_chunk_is_paged); + +#define percpu_first_chunk_is_paged \ + (static_key_enabled(&__percpu_first_chunk_is_paged.key)) +#else +#define percpu_first_chunk_is_paged false +#endif + #include <asm-generic/percpu.h> +#include <asm/paca.h> + #endif /* _ASM_POWERPC_PERCPU_H_ */ diff --git a/arch/powerpc/include/asm/perf_event.h b/arch/powerpc/include/asm/perf_event.h index 0bb23725b1e7..164e910bf654 100644 --- a/arch/powerpc/include/asm/perf_event.h +++ b/arch/powerpc/include/asm/perf_event.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Performance event support - hardware-specific disambiguation * @@ -7,15 +8,13 @@ * devices other than the core which provide their own performance counters. * * Copyright 2010 Freescale Semiconductor, Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifdef CONFIG_PPC_PERF_CTRS #include <asm/perf_event_server.h> +#else +static inline bool is_sier_available(void) { return false; } +static inline unsigned long get_pmcs_ext_regs(int idx) { return 0; } #endif #ifdef CONFIG_FSL_EMB_PERF_EVENT @@ -26,6 +25,8 @@ #include <asm/ptrace.h> #include <asm/reg.h> +#define perf_arch_bpf_user_pt_regs(regs) ®s->user_regs + /* * Overload regs->result to specify whether we should use the MSR (result * is zero) or the SIAR (result is non zero). @@ -34,7 +35,14 @@ do { \ (regs)->result = 0; \ (regs)->nip = __ip; \ - (regs)->gpr[1] = *(unsigned long *)__get_SP(); \ + (regs)->gpr[1] = current_stack_frame(); \ asm volatile("mfmsr %0" : "=r" ((regs)->msr)); \ } while (0) + +/* To support perf_regs sier update */ +extern bool is_sier_available(void); +extern unsigned long get_pmcs_ext_regs(int idx); +/* To define perf extended regs mask value */ +extern u64 PERF_REG_EXTENDED_MASK; +#define PERF_REG_EXTENDED_MASK PERF_REG_EXTENDED_MASK #endif diff --git a/arch/powerpc/include/asm/perf_event_fsl_emb.h b/arch/powerpc/include/asm/perf_event_fsl_emb.h index 718a9fa94e68..c4d9ceb03e8f 100644 --- a/arch/powerpc/include/asm/perf_event_fsl_emb.h +++ b/arch/powerpc/include/asm/perf_event_fsl_emb.h @@ -1,19 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Performance event support - Freescale embedded specific definitions. * * Copyright 2008-2009 Paul Mackerras, IBM Corporation. * Copyright 2010 Freescale Semiconductor, Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #include <linux/types.h> #include <asm/hw_irq.h> -#define MAX_HWEVENTS 4 +#define MAX_HWEVENTS 6 /* event flags */ #define FSL_EMB_EVENT_VALID 1 diff --git a/arch/powerpc/include/asm/perf_event_server.h b/arch/powerpc/include/asm/perf_event_server.h index 2dd7bfc459be..af0f46e2373b 100644 --- a/arch/powerpc/include/asm/perf_event_server.h +++ b/arch/powerpc/include/asm/perf_event_server.h @@ -1,22 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Performance event support - PowerPC classic/server specific definitions. * * Copyright 2008-2009 Paul Mackerras, IBM Corporation. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #include <linux/types.h> #include <asm/hw_irq.h> #include <linux/device.h> +#include <uapi/asm/perf_event.h> +/* Update perf_event_print_debug() if this changes */ #define MAX_HWEVENTS 8 #define MAX_EVENT_ALTERNATIVES 8 #define MAX_LIMITED_HWCOUNTERS 2 +struct perf_event; + +struct mmcr_regs { + unsigned long mmcr0; + unsigned long mmcr1; + unsigned long mmcr2; + unsigned long mmcra; + unsigned long mmcr3; +}; /* * This struct provides the constants and functions needed to * describe the PMU on a particular POWER-family CPU. @@ -28,25 +35,44 @@ struct power_pmu { unsigned long add_fields; unsigned long test_adder; int (*compute_mmcr)(u64 events[], int n_ev, - unsigned int hwc[], unsigned long mmcr[]); + unsigned int hwc[], struct mmcr_regs *mmcr, + struct perf_event *pevents[], u32 flags); int (*get_constraint)(u64 event_id, unsigned long *mskp, - unsigned long *valp); + unsigned long *valp, u64 event_config1); int (*get_alternatives)(u64 event_id, unsigned int flags, u64 alt[]); + void (*get_mem_data_src)(union perf_mem_data_src *dsrc, + u32 flags, struct pt_regs *regs); + void (*get_mem_weight)(u64 *weight, u64 type); + unsigned long group_constraint_mask; + unsigned long group_constraint_val; u64 (*bhrb_filter_map)(u64 branch_sample_type); void (*config_bhrb)(u64 pmu_bhrb_filter); - void (*disable_pmc)(unsigned int pmc, unsigned long mmcr[]); + void (*disable_pmc)(unsigned int pmc, struct mmcr_regs *mmcr); int (*limited_pmc_event)(u64 event_id); u32 flags; const struct attribute_group **attr_groups; int n_generic; int *generic_events; - int (*cache_events)[PERF_COUNT_HW_CACHE_MAX] + u64 (*cache_events)[PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_RESULT_MAX]; + int n_blacklist_ev; + int *blacklist_ev; /* BHRB entries in the PMU */ int bhrb_nr; + /* + * set this flag with `PERF_PMU_CAP_EXTENDED_REGS` if + * the pmu supports extended perf regs capability + */ + int capabilities; + /* + * Function to check event code for values which are + * reserved. Function takes struct perf_event as input, + * since event code could be spread in attr.config* + */ + int (*check_attr_config)(struct perf_event *ev); }; /* @@ -59,8 +85,12 @@ struct power_pmu { #define PPMU_SIAR_VALID 0x00000010 /* Processor has SIAR Valid bit */ #define PPMU_HAS_SSLOT 0x00000020 /* Has sampled slot in MMCRA */ #define PPMU_HAS_SIER 0x00000040 /* Has SIER */ -#define PPMU_BHRB 0x00000080 /* has BHRB feature enabled */ -#define PPMU_EBB 0x00000100 /* supports event based branch */ +#define PPMU_ARCH_207S 0x00000080 /* PMC is architecture v2.07S */ +#define PPMU_NO_SIAR 0x00000100 /* Do not use SIAR */ +#define PPMU_ARCH_31 0x00000200 /* Has MMCR3, SIER2 and SIER3 */ +#define PPMU_P10_DD1 0x00000400 /* Is power10 DD1 processor version */ +#define PPMU_P10 0x00000800 /* For power10 pmu */ +#define PPMU_HAS_ATTR_CONFIG1 0x00001000 /* Using config1 attribute */ /* * Values for flags to get_alternatives() @@ -69,16 +99,11 @@ struct power_pmu { #define PPMU_LIMITED_PMC_REQD 2 /* have to put this on a limited PMC */ #define PPMU_ONLY_COUNT_RUN 4 /* only counting in run state */ -/* - * We use the event config bit 63 as a flag to request EBB. - */ -#define EVENT_CONFIG_EBB_SHIFT 63 - -extern int register_power_pmu(struct power_pmu *); +int __init register_power_pmu(struct power_pmu *pmu); struct pt_regs; -extern unsigned long perf_misc_flags(struct pt_regs *regs); -extern unsigned long perf_instruction_pointer(struct pt_regs *regs); +extern unsigned long perf_arch_misc_flags(struct pt_regs *regs); +extern unsigned long perf_arch_instruction_pointer(struct pt_regs *regs); extern unsigned long int read_bhrb(int n); /* @@ -86,7 +111,7 @@ extern unsigned long int read_bhrb(int n); * if we have hardware PMU support. */ #ifdef CONFIG_PPC_PERF_CTRS -#define perf_misc_flags(regs) perf_misc_flags(regs) +#define perf_arch_misc_flags(regs) perf_arch_misc_flags(regs) #endif /* @@ -137,16 +162,24 @@ extern ssize_t power_events_sysfs_show(struct device *dev, * event 'cpu-cycles' can have two entries in sysfs: 'cpu-cycles' and * 'PM_CYC' where the latter is the name by which the event is known in * POWER CPU specification. + * + * Similarly, some hardware and cache events use the same event code. Eg. + * on POWER8, both "cache-references" and "L1-dcache-loads" events refer + * to the same event, PM_LD_REF_L1. The suffix, allows us to have two + * sysfs objects for the same event and thus two entries/aliases in sysfs. */ #define EVENT_VAR(_id, _suffix) event_attr_##_id##_suffix #define EVENT_PTR(_id, _suffix) &EVENT_VAR(_id, _suffix).attr.attr #define EVENT_ATTR(_name, _id, _suffix) \ - PMU_EVENT_ATTR(_name, EVENT_VAR(_id, _suffix), PME_PM_##_id, \ + PMU_EVENT_ATTR(_name, EVENT_VAR(_id, _suffix), _id, \ power_events_sysfs_show) #define GENERIC_EVENT_ATTR(_name, _id) EVENT_ATTR(_name, _id, _g) #define GENERIC_EVENT_PTR(_id) EVENT_PTR(_id, _g) -#define POWER_EVENT_ATTR(_name, _id) EVENT_ATTR(PM_##_name, _id, _p) +#define CACHE_EVENT_ATTR(_name, _id) EVENT_ATTR(_name, _id, _c) +#define CACHE_EVENT_PTR(_id) EVENT_PTR(_id, _c) + +#define POWER_EVENT_ATTR(_name, _id) EVENT_ATTR(_name, _id, _p) #define POWER_EVENT_PTR(_id) EVENT_PTR(_id, _p) diff --git a/arch/powerpc/include/asm/pgalloc-32.h b/arch/powerpc/include/asm/pgalloc-32.h deleted file mode 100644 index 27b2386f738a..000000000000 --- a/arch/powerpc/include/asm/pgalloc-32.h +++ /dev/null @@ -1,93 +0,0 @@ -#ifndef _ASM_POWERPC_PGALLOC_32_H -#define _ASM_POWERPC_PGALLOC_32_H - -#include <linux/threads.h> - -/* For 32-bit, all levels of page tables are just drawn from get_free_page() */ -#define MAX_PGTABLE_INDEX_SIZE 0 - -extern void __bad_pte(pmd_t *pmd); - -extern pgd_t *pgd_alloc(struct mm_struct *mm); -extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); - -/* - * We don't have any real pmd's, and this code never triggers because - * the pgd will always be present.. - */ -/* #define pmd_alloc_one(mm,address) ({ BUG(); ((pmd_t *)2); }) */ -#define pmd_free(mm, x) do { } while (0) -#define __pmd_free_tlb(tlb,x,a) do { } while (0) -/* #define pgd_populate(mm, pmd, pte) BUG() */ - -#ifndef CONFIG_BOOKE -#define pmd_populate_kernel(mm, pmd, pte) \ - (pmd_val(*(pmd)) = __pa(pte) | _PMD_PRESENT) -#define pmd_populate(mm, pmd, pte) \ - (pmd_val(*(pmd)) = (page_to_pfn(pte) << PAGE_SHIFT) | _PMD_PRESENT) -#define pmd_pgtable(pmd) pmd_page(pmd) -#else -#define pmd_populate_kernel(mm, pmd, pte) \ - (pmd_val(*(pmd)) = (unsigned long)pte | _PMD_PRESENT) -#define pmd_populate(mm, pmd, pte) \ - (pmd_val(*(pmd)) = (unsigned long)lowmem_page_address(pte) | _PMD_PRESENT) -#define pmd_pgtable(pmd) pmd_page(pmd) -#endif - -extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long addr); -extern pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addr); - -static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) -{ - free_page((unsigned long)pte); -} - -static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage) -{ - pgtable_page_dtor(ptepage); - __free_page(ptepage); -} - -static inline void pgtable_free(void *table, unsigned index_size) -{ - BUG_ON(index_size); /* 32-bit doesn't use this */ - free_page((unsigned long)table); -} - -#define check_pgt_cache() do { } while (0) - -#ifdef CONFIG_SMP -static inline void pgtable_free_tlb(struct mmu_gather *tlb, - void *table, int shift) -{ - unsigned long pgf = (unsigned long)table; - BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE); - pgf |= shift; - tlb_remove_table(tlb, (void *)pgf); -} - -static inline void __tlb_remove_table(void *_table) -{ - void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE); - unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE; - - pgtable_free(table, shift); -} -#else -static inline void pgtable_free_tlb(struct mmu_gather *tlb, - void *table, int shift) -{ - pgtable_free(table, shift); -} -#endif - -static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table, - unsigned long address) -{ - struct page *page = page_address(table); - - tlb_flush_pgtable(tlb, address); - pgtable_page_dtor(page); - pgtable_free_tlb(tlb, page, 0); -} -#endif /* _ASM_POWERPC_PGALLOC_32_H */ diff --git a/arch/powerpc/include/asm/pgalloc-64.h b/arch/powerpc/include/asm/pgalloc-64.h deleted file mode 100644 index f65e27b09bd3..000000000000 --- a/arch/powerpc/include/asm/pgalloc-64.h +++ /dev/null @@ -1,243 +0,0 @@ -#ifndef _ASM_POWERPC_PGALLOC_64_H -#define _ASM_POWERPC_PGALLOC_64_H -/* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ - -#include <linux/slab.h> -#include <linux/cpumask.h> -#include <linux/percpu.h> - -struct vmemmap_backing { - struct vmemmap_backing *list; - unsigned long phys; - unsigned long virt_addr; -}; - -/* - * Functions that deal with pagetables that could be at any level of - * the table need to be passed an "index_size" so they know how to - * handle allocation. For PTE pages (which are linked to a struct - * page for now, and drawn from the main get_free_pages() pool), the - * allocation size will be (2^index_size * sizeof(pointer)) and - * allocations are drawn from the kmem_cache in PGT_CACHE(index_size). - * - * The maximum index size needs to be big enough to allow any - * pagetable sizes we need, but small enough to fit in the low bits of - * any page table pointer. In other words all pagetables, even tiny - * ones, must be aligned to allow at least enough low 0 bits to - * contain this value. This value is also used as a mask, so it must - * be one less than a power of two. - */ -#define MAX_PGTABLE_INDEX_SIZE 0xf - -extern struct kmem_cache *pgtable_cache[]; -#define PGT_CACHE(shift) ({ \ - BUG_ON(!(shift)); \ - pgtable_cache[(shift) - 1]; \ - }) - -static inline pgd_t *pgd_alloc(struct mm_struct *mm) -{ - return kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), GFP_KERNEL); -} - -static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) -{ - kmem_cache_free(PGT_CACHE(PGD_INDEX_SIZE), pgd); -} - -#ifndef CONFIG_PPC_64K_PAGES - -#define pgd_populate(MM, PGD, PUD) pgd_set(PGD, PUD) - -static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) -{ - return kmem_cache_alloc(PGT_CACHE(PUD_INDEX_SIZE), - GFP_KERNEL|__GFP_REPEAT); -} - -static inline void pud_free(struct mm_struct *mm, pud_t *pud) -{ - kmem_cache_free(PGT_CACHE(PUD_INDEX_SIZE), pud); -} - -static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) -{ - pud_set(pud, (unsigned long)pmd); -} - -#define pmd_populate(mm, pmd, pte_page) \ - pmd_populate_kernel(mm, pmd, page_address(pte_page)) -#define pmd_populate_kernel(mm, pmd, pte) pmd_set(pmd, (unsigned long)(pte)) -#define pmd_pgtable(pmd) pmd_page(pmd) - -static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, - unsigned long address) -{ - return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO); -} - -static inline pgtable_t pte_alloc_one(struct mm_struct *mm, - unsigned long address) -{ - struct page *page; - pte_t *pte; - - pte = pte_alloc_one_kernel(mm, address); - if (!pte) - return NULL; - page = virt_to_page(pte); - pgtable_page_ctor(page); - return page; -} - -static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) -{ - free_page((unsigned long)pte); -} - -static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage) -{ - pgtable_page_dtor(ptepage); - __free_page(ptepage); -} - -static inline void pgtable_free(void *table, unsigned index_size) -{ - if (!index_size) - free_page((unsigned long)table); - else { - BUG_ON(index_size > MAX_PGTABLE_INDEX_SIZE); - kmem_cache_free(PGT_CACHE(index_size), table); - } -} - -#ifdef CONFIG_SMP -static inline void pgtable_free_tlb(struct mmu_gather *tlb, - void *table, int shift) -{ - unsigned long pgf = (unsigned long)table; - BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE); - pgf |= shift; - tlb_remove_table(tlb, (void *)pgf); -} - -static inline void __tlb_remove_table(void *_table) -{ - void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE); - unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE; - - pgtable_free(table, shift); -} -#else /* !CONFIG_SMP */ -static inline void pgtable_free_tlb(struct mmu_gather *tlb, - void *table, int shift) -{ - pgtable_free(table, shift); -} -#endif /* CONFIG_SMP */ - -static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table, - unsigned long address) -{ - struct page *page = page_address(table); - - tlb_flush_pgtable(tlb, address); - pgtable_page_dtor(page); - pgtable_free_tlb(tlb, page, 0); -} - -#else /* if CONFIG_PPC_64K_PAGES */ -/* - * we support 16 fragments per PTE page. - */ -#define PTE_FRAG_NR 16 -/* - * We use a 2K PTE page fragment and another 2K for storing - * real_pte_t hash index - */ -#define PTE_FRAG_SIZE_SHIFT 12 -#define PTE_FRAG_SIZE (2 * PTRS_PER_PTE * sizeof(pte_t)) - -extern pte_t *page_table_alloc(struct mm_struct *, unsigned long, int); -extern void page_table_free(struct mm_struct *, unsigned long *, int); -extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift); -#ifdef CONFIG_SMP -extern void __tlb_remove_table(void *_table); -#endif - -#define pud_populate(mm, pud, pmd) pud_set(pud, (unsigned long)pmd) - -static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, - pte_t *pte) -{ - pmd_set(pmd, (unsigned long)pte); -} - -static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, - pgtable_t pte_page) -{ - pmd_set(pmd, (unsigned long)pte_page); -} - -static inline pgtable_t pmd_pgtable(pmd_t pmd) -{ - return (pgtable_t)(pmd_val(pmd) & ~PMD_MASKED_BITS); -} - -static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, - unsigned long address) -{ - return (pte_t *)page_table_alloc(mm, address, 1); -} - -static inline pgtable_t pte_alloc_one(struct mm_struct *mm, - unsigned long address) -{ - return (pgtable_t)page_table_alloc(mm, address, 0); -} - -static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) -{ - page_table_free(mm, (unsigned long *)pte, 1); -} - -static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage) -{ - page_table_free(mm, (unsigned long *)ptepage, 0); -} - -static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table, - unsigned long address) -{ - tlb_flush_pgtable(tlb, address); - pgtable_free_tlb(tlb, table, 0); -} -#endif /* CONFIG_PPC_64K_PAGES */ - -static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) -{ - return kmem_cache_alloc(PGT_CACHE(PMD_CACHE_INDEX), - GFP_KERNEL|__GFP_REPEAT); -} - -static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) -{ - kmem_cache_free(PGT_CACHE(PMD_CACHE_INDEX), pmd); -} - -#define __pmd_free_tlb(tlb, pmd, addr) \ - pgtable_free_tlb(tlb, pmd, PMD_CACHE_INDEX) -#ifndef CONFIG_PPC_64K_PAGES -#define __pud_free_tlb(tlb, pud, addr) \ - pgtable_free_tlb(tlb, pud, PUD_INDEX_SIZE) - -#endif /* CONFIG_PPC_64K_PAGES */ - -#define check_pgt_cache() do { } while (0) - -#endif /* _ASM_POWERPC_PGALLOC_64_H */ diff --git a/arch/powerpc/include/asm/pgalloc.h b/arch/powerpc/include/asm/pgalloc.h index e9a9f60e596d..3a971e2a8c73 100644 --- a/arch/powerpc/include/asm/pgalloc.h +++ b/arch/powerpc/include/asm/pgalloc.h @@ -1,24 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_PGALLOC_H #define _ASM_POWERPC_PGALLOC_H -#ifdef __KERNEL__ #include <linux/mm.h> -#include <asm-generic/tlb.h> -#ifdef CONFIG_PPC_BOOK3E -extern void tlb_flush_pgtable(struct mmu_gather *tlb, unsigned long address); -#else /* CONFIG_PPC_BOOK3E */ -static inline void tlb_flush_pgtable(struct mmu_gather *tlb, - unsigned long address) +#ifndef MODULE +static inline gfp_t pgtable_gfp_flags(struct mm_struct *mm, gfp_t gfp) { + if (unlikely(mm == &init_mm)) + return gfp; + return gfp | __GFP_ACCOUNT; } -#endif /* !CONFIG_PPC_BOOK3E */ +#else /* !MODULE */ +static inline gfp_t pgtable_gfp_flags(struct mm_struct *mm, gfp_t gfp) +{ + return gfp | __GFP_ACCOUNT; +} +#endif /* MODULE */ + +#define PGALLOC_GFP (GFP_KERNEL | __GFP_ZERO) + +pte_t *pte_fragment_alloc(struct mm_struct *mm, int kernel); + +static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm) +{ + return (pte_t *)pte_fragment_alloc(mm, 1); +} + +static inline pgtable_t pte_alloc_one(struct mm_struct *mm) +{ + return (pgtable_t)pte_fragment_alloc(mm, 0); +} + +void pte_frag_destroy(void *pte_frag); +void pte_fragment_free(unsigned long *table, int kernel); + +static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) +{ + pte_fragment_free((unsigned long *)pte, 1); +} + +static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage) +{ + pte_fragment_free((unsigned long *)ptepage, 0); +} + +/* arch use pte_free_defer() implementation in arch/powerpc/mm/pgtable-frag.c */ +#define pte_free_defer pte_free_defer +void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable); + +/* + * Functions that deal with pagetables that could be at any level of + * the table need to be passed an "index_size" so they know how to + * handle allocation. For PTE pages, the allocation size will be + * (2^index_size * sizeof(pointer)) and allocations are drawn from + * the kmem_cache in PGT_CACHE(index_size). + * + * The maximum index size needs to be big enough to allow any + * pagetable sizes we need, but small enough to fit in the low bits of + * any page table pointer. In other words all pagetables, even tiny + * ones, must be aligned to allow at least enough low 0 bits to + * contain this value. This value is also used as a mask, so it must + * be one less than a power of two. + */ +#define MAX_PGTABLE_INDEX_SIZE 0xf + +extern struct kmem_cache *pgtable_cache[]; +#define PGT_CACHE(shift) pgtable_cache[shift] -#ifdef CONFIG_PPC64 -#include <asm/pgalloc-64.h> +#ifdef CONFIG_PPC_BOOK3S +#include <asm/book3s/pgalloc.h> #else -#include <asm/pgalloc-32.h> +#include <asm/nohash/pgalloc.h> #endif -#endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_PGALLOC_H */ diff --git a/arch/powerpc/include/asm/pgtable-be-types.h b/arch/powerpc/include/asm/pgtable-be-types.h new file mode 100644 index 000000000000..6bd8f89b25dc --- /dev/null +++ b/arch/powerpc/include/asm/pgtable-be-types.h @@ -0,0 +1,104 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_PGTABLE_BE_TYPES_H +#define _ASM_POWERPC_PGTABLE_BE_TYPES_H + +#include <asm/cmpxchg.h> + +/* PTE level */ +typedef struct { __be64 pte; } pte_t; +#define __pte(x) ((pte_t) { cpu_to_be64(x) }) +#define __pte_raw(x) ((pte_t) { (x) }) +static inline unsigned long pte_val(pte_t x) +{ + return be64_to_cpu(x.pte); +} + +static inline __be64 pte_raw(pte_t x) +{ + return x.pte; +} + +/* PMD level */ +#ifdef CONFIG_PPC64 +typedef struct { __be64 pmd; } pmd_t; +#define __pmd(x) ((pmd_t) { cpu_to_be64(x) }) +#define __pmd_raw(x) ((pmd_t) { (x) }) +static inline unsigned long pmd_val(pmd_t x) +{ + return be64_to_cpu(x.pmd); +} + +static inline __be64 pmd_raw(pmd_t x) +{ + return x.pmd; +} + +/* 64 bit always use 4 level table. */ +typedef struct { __be64 pud; } pud_t; +#define __pud(x) ((pud_t) { cpu_to_be64(x) }) +#define __pud_raw(x) ((pud_t) { (x) }) +static inline unsigned long pud_val(pud_t x) +{ + return be64_to_cpu(x.pud); +} + +static inline __be64 pud_raw(pud_t x) +{ + return x.pud; +} + +#endif /* CONFIG_PPC64 */ + +/* PGD level */ +typedef struct { __be64 pgd; } pgd_t; +#define __pgd(x) ((pgd_t) { cpu_to_be64(x) }) +#define __pgd_raw(x) ((pgd_t) { (x) }) +static inline unsigned long pgd_val(pgd_t x) +{ + return be64_to_cpu(x.pgd); +} + +static inline __be64 pgd_raw(pgd_t x) +{ + return x.pgd; +} + +/* Page protection bits */ +typedef struct { unsigned long pgprot; } pgprot_t; +#define pgprot_val(x) ((x).pgprot) +#define __pgprot(x) ((pgprot_t) { (x) }) + +/* + * With hash config 64k pages additionally define a bigger "real PTE" type that + * gathers the "second half" part of the PTE for pseudo 64k pages + */ +#ifdef CONFIG_PPC_64K_PAGES +typedef struct { pte_t pte; unsigned long hidx; } real_pte_t; +#else +typedef struct { pte_t pte; } real_pte_t; +#endif + +static inline bool pte_xchg(pte_t *ptep, pte_t old, pte_t new) +{ + unsigned long *p = (unsigned long *)ptep; + __be64 prev; + + /* See comment in switch_mm_irqs_off() */ + prev = (__force __be64)__cmpxchg_u64(p, (__force unsigned long)pte_raw(old), + (__force unsigned long)pte_raw(new)); + + return pte_raw(old) == prev; +} + +static inline bool pmd_xchg(pmd_t *pmdp, pmd_t old, pmd_t new) +{ + unsigned long *p = (unsigned long *)pmdp; + __be64 prev; + + prev = (__force __be64)__cmpxchg_u64(p, (__force unsigned long)pmd_raw(old), + (__force unsigned long)pmd_raw(new)); + + return pmd_raw(old) == prev; +} + +#endif /* _ASM_POWERPC_PGTABLE_BE_TYPES_H */ diff --git a/arch/powerpc/include/asm/pgtable-masks.h b/arch/powerpc/include/asm/pgtable-masks.h new file mode 100644 index 000000000000..6e8e2db26a5a --- /dev/null +++ b/arch/powerpc/include/asm/pgtable-masks.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_PGTABLE_MASKS_H +#define _ASM_POWERPC_PGTABLE_MASKS_H + +#ifndef _PAGE_NA +#define _PAGE_NA 0 +#define _PAGE_NAX _PAGE_EXEC +#define _PAGE_RO _PAGE_READ +#define _PAGE_ROX (_PAGE_READ | _PAGE_EXEC) +#define _PAGE_RW (_PAGE_READ | _PAGE_WRITE) +#define _PAGE_RWX (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC) +#endif + +/* Permission flags for kernel mappings */ +#ifndef _PAGE_KERNEL_RO +#define _PAGE_KERNEL_RO _PAGE_RO +#define _PAGE_KERNEL_ROX _PAGE_ROX +#define _PAGE_KERNEL_RW (_PAGE_RW | _PAGE_DIRTY) +#define _PAGE_KERNEL_RWX (_PAGE_RWX | _PAGE_DIRTY) +#endif + +/* Permission masks used to generate the __P and __S table */ +#define PAGE_NONE __pgprot(_PAGE_BASE | _PAGE_NA) +#define PAGE_EXECONLY_X __pgprot(_PAGE_BASE | _PAGE_NAX) +#define PAGE_SHARED __pgprot(_PAGE_BASE | _PAGE_RW) +#define PAGE_SHARED_X __pgprot(_PAGE_BASE | _PAGE_RWX) +#define PAGE_COPY __pgprot(_PAGE_BASE | _PAGE_RO) +#define PAGE_COPY_X __pgprot(_PAGE_BASE | _PAGE_ROX) +#define PAGE_READONLY __pgprot(_PAGE_BASE | _PAGE_RO) +#define PAGE_READONLY_X __pgprot(_PAGE_BASE | _PAGE_ROX) + +#endif /* _ASM_POWERPC_PGTABLE_MASKS_H */ diff --git a/arch/powerpc/include/asm/pgtable-ppc32.h b/arch/powerpc/include/asm/pgtable-ppc32.h deleted file mode 100644 index 47edde8c3556..000000000000 --- a/arch/powerpc/include/asm/pgtable-ppc32.h +++ /dev/null @@ -1,341 +0,0 @@ -#ifndef _ASM_POWERPC_PGTABLE_PPC32_H -#define _ASM_POWERPC_PGTABLE_PPC32_H - -#include <asm-generic/pgtable-nopmd.h> - -#ifndef __ASSEMBLY__ -#include <linux/sched.h> -#include <linux/threads.h> -#include <asm/io.h> /* For sub-arch specific PPC_PIN_SIZE */ - -extern unsigned long va_to_phys(unsigned long address); -extern pte_t *va_to_pte(unsigned long address); -extern unsigned long ioremap_bot; - -#ifdef CONFIG_44x -extern int icache_44x_need_flush; -#endif - -#endif /* __ASSEMBLY__ */ - -/* - * The normal case is that PTEs are 32-bits and we have a 1-page - * 1024-entry pgdir pointing to 1-page 1024-entry PTE pages. -- paulus - * - * For any >32-bit physical address platform, we can use the following - * two level page table layout where the pgdir is 8KB and the MS 13 bits - * are an index to the second level table. The combined pgdir/pmd first - * level has 2048 entries and the second level has 512 64-bit PTE entries. - * -Matt - */ -/* PGDIR_SHIFT determines what a top-level page table entry can map */ -#define PGDIR_SHIFT (PAGE_SHIFT + PTE_SHIFT) -#define PGDIR_SIZE (1UL << PGDIR_SHIFT) -#define PGDIR_MASK (~(PGDIR_SIZE-1)) - -/* - * entries per page directory level: our page-table tree is two-level, so - * we don't really have any PMD directory. - */ -#ifndef __ASSEMBLY__ -#define PTE_TABLE_SIZE (sizeof(pte_t) << PTE_SHIFT) -#define PGD_TABLE_SIZE (sizeof(pgd_t) << (32 - PGDIR_SHIFT)) -#endif /* __ASSEMBLY__ */ - -#define PTRS_PER_PTE (1 << PTE_SHIFT) -#define PTRS_PER_PMD 1 -#define PTRS_PER_PGD (1 << (32 - PGDIR_SHIFT)) - -#define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE) -#define FIRST_USER_ADDRESS 0 - -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %llx.\n", __FILE__, __LINE__, \ - (unsigned long long)pte_val(e)) -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) - -/* - * This is the bottom of the PKMAP area with HIGHMEM or an arbitrary - * value (for now) on others, from where we can start layout kernel - * virtual space that goes below PKMAP and FIXMAP - */ -#ifdef CONFIG_HIGHMEM -#define KVIRT_TOP PKMAP_BASE -#else -#define KVIRT_TOP (0xfe000000UL) /* for now, could be FIXMAP_BASE ? */ -#endif - -/* - * ioremap_bot starts at that address. Early ioremaps move down from there, - * until mem_init() at which point this becomes the top of the vmalloc - * and ioremap space - */ -#ifdef CONFIG_NOT_COHERENT_CACHE -#define IOREMAP_TOP ((KVIRT_TOP - CONFIG_CONSISTENT_SIZE) & PAGE_MASK) -#else -#define IOREMAP_TOP KVIRT_TOP -#endif - -/* - * Just any arbitrary offset to the start of the vmalloc VM area: the - * current 16MB value just means that there will be a 64MB "hole" after the - * physical memory until the kernel virtual memory starts. That means that - * any out-of-bounds memory accesses will hopefully be caught. - * The vmalloc() routines leaves a hole of 4kB between each vmalloced - * area for the same reason. ;) - * - * We no longer map larger than phys RAM with the BATs so we don't have - * to worry about the VMALLOC_OFFSET causing problems. We do have to worry - * about clashes between our early calls to ioremap() that start growing down - * from ioremap_base being run into the VM area allocations (growing upwards - * from VMALLOC_START). For this reason we have ioremap_bot to check when - * we actually run into our mappings setup in the early boot with the VM - * system. This really does become a problem for machines with good amounts - * of RAM. -- Cort - */ -#define VMALLOC_OFFSET (0x1000000) /* 16M */ -#ifdef PPC_PIN_SIZE -#define VMALLOC_START (((_ALIGN((long)high_memory, PPC_PIN_SIZE) + VMALLOC_OFFSET) & ~(VMALLOC_OFFSET-1))) -#else -#define VMALLOC_START ((((long)high_memory + VMALLOC_OFFSET) & ~(VMALLOC_OFFSET-1))) -#endif -#define VMALLOC_END ioremap_bot - -/* - * Bits in a linux-style PTE. These match the bits in the - * (hardware-defined) PowerPC PTE as closely as possible. - */ - -#if defined(CONFIG_40x) -#include <asm/pte-40x.h> -#elif defined(CONFIG_44x) -#include <asm/pte-44x.h> -#elif defined(CONFIG_FSL_BOOKE) && defined(CONFIG_PTE_64BIT) -#include <asm/pte-book3e.h> -#elif defined(CONFIG_FSL_BOOKE) -#include <asm/pte-fsl-booke.h> -#elif defined(CONFIG_8xx) -#include <asm/pte-8xx.h> -#else /* CONFIG_6xx */ -#include <asm/pte-hash32.h> -#endif - -/* And here we include common definitions */ -#include <asm/pte-common.h> - -#ifndef __ASSEMBLY__ - -#define pte_clear(mm, addr, ptep) \ - do { pte_update(ptep, ~_PAGE_HASHPTE, 0); } while (0) - -#define pmd_none(pmd) (!pmd_val(pmd)) -#define pmd_bad(pmd) (pmd_val(pmd) & _PMD_BAD) -#define pmd_present(pmd) (pmd_val(pmd) & _PMD_PRESENT_MASK) -#define pmd_clear(pmdp) do { pmd_val(*(pmdp)) = 0; } while (0) - -/* - * When flushing the tlb entry for a page, we also need to flush the hash - * table entry. flush_hash_pages is assembler (for speed) in hashtable.S. - */ -extern int flush_hash_pages(unsigned context, unsigned long va, - unsigned long pmdval, int count); - -/* Add an HPTE to the hash table */ -extern void add_hash_page(unsigned context, unsigned long va, - unsigned long pmdval); - -/* Flush an entry from the TLB/hash table */ -extern void flush_hash_entry(struct mm_struct *mm, pte_t *ptep, - unsigned long address); - -/* - * PTE updates. This function is called whenever an existing - * valid PTE is updated. This does -not- include set_pte_at() - * which nowadays only sets a new PTE. - * - * Depending on the type of MMU, we may need to use atomic updates - * and the PTE may be either 32 or 64 bit wide. In the later case, - * when using atomic updates, only the low part of the PTE is - * accessed atomically. - * - * In addition, on 44x, we also maintain a global flag indicating - * that an executable user mapping was modified, which is needed - * to properly flush the virtually tagged instruction cache of - * those implementations. - */ -#ifndef CONFIG_PTE_64BIT -static inline unsigned long pte_update(pte_t *p, - unsigned long clr, - unsigned long set) -{ -#ifdef PTE_ATOMIC_UPDATES - unsigned long old, tmp; - - __asm__ __volatile__("\ -1: lwarx %0,0,%3\n\ - andc %1,%0,%4\n\ - or %1,%1,%5\n" - PPC405_ERR77(0,%3) -" stwcx. %1,0,%3\n\ - bne- 1b" - : "=&r" (old), "=&r" (tmp), "=m" (*p) - : "r" (p), "r" (clr), "r" (set), "m" (*p) - : "cc" ); -#else /* PTE_ATOMIC_UPDATES */ - unsigned long old = pte_val(*p); - *p = __pte((old & ~clr) | set); -#endif /* !PTE_ATOMIC_UPDATES */ - -#ifdef CONFIG_44x - if ((old & _PAGE_USER) && (old & _PAGE_EXEC)) - icache_44x_need_flush = 1; -#endif - return old; -} -#else /* CONFIG_PTE_64BIT */ -static inline unsigned long long pte_update(pte_t *p, - unsigned long clr, - unsigned long set) -{ -#ifdef PTE_ATOMIC_UPDATES - unsigned long long old; - unsigned long tmp; - - __asm__ __volatile__("\ -1: lwarx %L0,0,%4\n\ - lwzx %0,0,%3\n\ - andc %1,%L0,%5\n\ - or %1,%1,%6\n" - PPC405_ERR77(0,%3) -" stwcx. %1,0,%4\n\ - bne- 1b" - : "=&r" (old), "=&r" (tmp), "=m" (*p) - : "r" (p), "r" ((unsigned long)(p) + 4), "r" (clr), "r" (set), "m" (*p) - : "cc" ); -#else /* PTE_ATOMIC_UPDATES */ - unsigned long long old = pte_val(*p); - *p = __pte((old & ~(unsigned long long)clr) | set); -#endif /* !PTE_ATOMIC_UPDATES */ - -#ifdef CONFIG_44x - if ((old & _PAGE_USER) && (old & _PAGE_EXEC)) - icache_44x_need_flush = 1; -#endif - return old; -} -#endif /* CONFIG_PTE_64BIT */ - -/* - * 2.6 calls this without flushing the TLB entry; this is wrong - * for our hash-based implementation, we fix that up here. - */ -#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG -static inline int __ptep_test_and_clear_young(unsigned int context, unsigned long addr, pte_t *ptep) -{ - unsigned long old; - old = pte_update(ptep, _PAGE_ACCESSED, 0); -#if _PAGE_HASHPTE != 0 - if (old & _PAGE_HASHPTE) { - unsigned long ptephys = __pa(ptep) & PAGE_MASK; - flush_hash_pages(context, addr, ptephys, 1); - } -#endif - return (old & _PAGE_ACCESSED) != 0; -} -#define ptep_test_and_clear_young(__vma, __addr, __ptep) \ - __ptep_test_and_clear_young((__vma)->vm_mm->context.id, __addr, __ptep) - -#define __HAVE_ARCH_PTEP_GET_AND_CLEAR -static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, - pte_t *ptep) -{ - return __pte(pte_update(ptep, ~_PAGE_HASHPTE, 0)); -} - -#define __HAVE_ARCH_PTEP_SET_WRPROTECT -static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, - pte_t *ptep) -{ - pte_update(ptep, (_PAGE_RW | _PAGE_HWWRITE), 0); -} -static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, - unsigned long addr, pte_t *ptep) -{ - ptep_set_wrprotect(mm, addr, ptep); -} - - -static inline void __ptep_set_access_flags(pte_t *ptep, pte_t entry) -{ - unsigned long bits = pte_val(entry) & - (_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC); - pte_update(ptep, 0, bits); -} - -#define __HAVE_ARCH_PTE_SAME -#define pte_same(A,B) (((pte_val(A) ^ pte_val(B)) & ~_PAGE_HASHPTE) == 0) - -/* - * Note that on Book E processors, the pmd contains the kernel virtual - * (lowmem) address of the pte page. The physical address is less useful - * because everything runs with translation enabled (even the TLB miss - * handler). On everything else the pmd contains the physical address - * of the pte page. -- paulus - */ -#ifndef CONFIG_BOOKE -#define pmd_page_vaddr(pmd) \ - ((unsigned long) __va(pmd_val(pmd) & PAGE_MASK)) -#define pmd_page(pmd) \ - pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT) -#else -#define pmd_page_vaddr(pmd) \ - ((unsigned long) (pmd_val(pmd) & PAGE_MASK)) -#define pmd_page(pmd) \ - pfn_to_page((__pa(pmd_val(pmd)) >> PAGE_SHIFT)) -#endif - -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) - -/* to find an entry in a page-table-directory */ -#define pgd_index(address) ((address) >> PGDIR_SHIFT) -#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) - -/* Find an entry in the third-level page table.. */ -#define pte_index(address) \ - (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) -#define pte_offset_kernel(dir, addr) \ - ((pte_t *) pmd_page_vaddr(*(dir)) + pte_index(addr)) -#define pte_offset_map(dir, addr) \ - ((pte_t *) kmap_atomic(pmd_page(*(dir))) + pte_index(addr)) -#define pte_unmap(pte) kunmap_atomic(pte) - -/* - * Encode and decode a swap entry. - * Note that the bits we use in a PTE for representing a swap entry - * must not include the _PAGE_PRESENT bit, the _PAGE_FILE bit, or the - *_PAGE_HASHPTE bit (if used). -- paulus - */ -#define __swp_type(entry) ((entry).val & 0x1f) -#define __swp_offset(entry) ((entry).val >> 5) -#define __swp_entry(type, offset) ((swp_entry_t) { (type) | ((offset) << 5) }) -#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) >> 3 }) -#define __swp_entry_to_pte(x) ((pte_t) { (x).val << 3 }) - -/* Encode and decode a nonlinear file mapping entry */ -#define PTE_FILE_MAX_BITS 29 -#define pte_to_pgoff(pte) (pte_val(pte) >> 3) -#define pgoff_to_pte(off) ((pte_t) { ((off) << 3) | _PAGE_FILE }) - -/* - * No page table caches to initialise - */ -#define pgtable_cache_init() do { } while (0) - -extern int get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep, - pmd_t **pmdp); - -#endif /* !__ASSEMBLY__ */ - -#endif /* _ASM_POWERPC_PGTABLE_PPC32_H */ diff --git a/arch/powerpc/include/asm/pgtable-ppc64-64k.h b/arch/powerpc/include/asm/pgtable-ppc64-64k.h deleted file mode 100644 index a56b82fb0609..000000000000 --- a/arch/powerpc/include/asm/pgtable-ppc64-64k.h +++ /dev/null @@ -1,41 +0,0 @@ -#ifndef _ASM_POWERPC_PGTABLE_PPC64_64K_H -#define _ASM_POWERPC_PGTABLE_PPC64_64K_H - -#include <asm-generic/pgtable-nopud.h> - - -#define PTE_INDEX_SIZE 8 -#define PMD_INDEX_SIZE 10 -#define PUD_INDEX_SIZE 0 -#define PGD_INDEX_SIZE 12 - -#ifndef __ASSEMBLY__ -#define PTE_TABLE_SIZE (sizeof(real_pte_t) << PTE_INDEX_SIZE) -#define PMD_TABLE_SIZE (sizeof(pmd_t) << PMD_INDEX_SIZE) -#define PGD_TABLE_SIZE (sizeof(pgd_t) << PGD_INDEX_SIZE) -#endif /* __ASSEMBLY__ */ - -#define PTRS_PER_PTE (1 << PTE_INDEX_SIZE) -#define PTRS_PER_PMD (1 << PMD_INDEX_SIZE) -#define PTRS_PER_PGD (1 << PGD_INDEX_SIZE) - -/* With 4k base page size, hugepage PTEs go at the PMD level */ -#define MIN_HUGEPTE_SHIFT PAGE_SHIFT - -/* PMD_SHIFT determines what a second-level page table entry can map */ -#define PMD_SHIFT (PAGE_SHIFT + PTE_INDEX_SIZE) -#define PMD_SIZE (1UL << PMD_SHIFT) -#define PMD_MASK (~(PMD_SIZE-1)) - -/* PGDIR_SHIFT determines what a third-level page table entry can map */ -#define PGDIR_SHIFT (PMD_SHIFT + PMD_INDEX_SIZE) -#define PGDIR_SIZE (1UL << PGDIR_SHIFT) -#define PGDIR_MASK (~(PGDIR_SIZE-1)) - -/* Bits to mask out from a PMD to get to the PTE page */ -/* PMDs point to PTE table fragments which are 4K aligned. */ -#define PMD_MASKED_BITS 0xfff -/* Bits to mask out from a PGD/PUD to get to the PMD page */ -#define PUD_MASKED_BITS 0x1ff - -#endif /* _ASM_POWERPC_PGTABLE_PPC64_64K_H */ diff --git a/arch/powerpc/include/asm/pgtable-ppc64.h b/arch/powerpc/include/asm/pgtable-ppc64.h deleted file mode 100644 index 46db09414a10..000000000000 --- a/arch/powerpc/include/asm/pgtable-ppc64.h +++ /dev/null @@ -1,560 +0,0 @@ -#ifndef _ASM_POWERPC_PGTABLE_PPC64_H_ -#define _ASM_POWERPC_PGTABLE_PPC64_H_ -/* - * This file contains the functions and defines necessary to modify and use - * the ppc64 hashed page table. - */ - -#ifdef CONFIG_PPC_64K_PAGES -#include <asm/pgtable-ppc64-64k.h> -#else -#include <asm/pgtable-ppc64-4k.h> -#endif -#include <asm/barrier.h> - -#define FIRST_USER_ADDRESS 0 - -/* - * Size of EA range mapped by our pagetables. - */ -#define PGTABLE_EADDR_SIZE (PTE_INDEX_SIZE + PMD_INDEX_SIZE + \ - PUD_INDEX_SIZE + PGD_INDEX_SIZE + PAGE_SHIFT) -#define PGTABLE_RANGE (ASM_CONST(1) << PGTABLE_EADDR_SIZE) - -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -#define PMD_CACHE_INDEX (PMD_INDEX_SIZE + 1) -#else -#define PMD_CACHE_INDEX PMD_INDEX_SIZE -#endif -/* - * Define the address range of the kernel non-linear virtual area - */ - -#ifdef CONFIG_PPC_BOOK3E -#define KERN_VIRT_START ASM_CONST(0x8000000000000000) -#else -#define KERN_VIRT_START ASM_CONST(0xD000000000000000) -#endif -#define KERN_VIRT_SIZE ASM_CONST(0x0000100000000000) - -/* - * The vmalloc space starts at the beginning of that region, and - * occupies half of it on hash CPUs and a quarter of it on Book3E - * (we keep a quarter for the virtual memmap) - */ -#define VMALLOC_START KERN_VIRT_START -#ifdef CONFIG_PPC_BOOK3E -#define VMALLOC_SIZE (KERN_VIRT_SIZE >> 2) -#else -#define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1) -#endif -#define VMALLOC_END (VMALLOC_START + VMALLOC_SIZE) - -/* - * The second half of the kernel virtual space is used for IO mappings, - * it's itself carved into the PIO region (ISA and PHB IO space) and - * the ioremap space - * - * ISA_IO_BASE = KERN_IO_START, 64K reserved area - * PHB_IO_BASE = ISA_IO_BASE + 64K to ISA_IO_BASE + 2G, PHB IO spaces - * IOREMAP_BASE = ISA_IO_BASE + 2G to VMALLOC_START + PGTABLE_RANGE - */ -#define KERN_IO_START (KERN_VIRT_START + (KERN_VIRT_SIZE >> 1)) -#define FULL_IO_SIZE 0x80000000ul -#define ISA_IO_BASE (KERN_IO_START) -#define ISA_IO_END (KERN_IO_START + 0x10000ul) -#define PHB_IO_BASE (ISA_IO_END) -#define PHB_IO_END (KERN_IO_START + FULL_IO_SIZE) -#define IOREMAP_BASE (PHB_IO_END) -#define IOREMAP_END (KERN_VIRT_START + KERN_VIRT_SIZE) - - -/* - * Region IDs - */ -#define REGION_SHIFT 60UL -#define REGION_MASK (0xfUL << REGION_SHIFT) -#define REGION_ID(ea) (((unsigned long)(ea)) >> REGION_SHIFT) - -#define VMALLOC_REGION_ID (REGION_ID(VMALLOC_START)) -#define KERNEL_REGION_ID (REGION_ID(PAGE_OFFSET)) -#define VMEMMAP_REGION_ID (0xfUL) /* Server only */ -#define USER_REGION_ID (0UL) - -/* - * Defines the address of the vmemap area, in its own region on - * hash table CPUs and after the vmalloc space on Book3E - */ -#ifdef CONFIG_PPC_BOOK3E -#define VMEMMAP_BASE VMALLOC_END -#define VMEMMAP_END KERN_IO_START -#else -#define VMEMMAP_BASE (VMEMMAP_REGION_ID << REGION_SHIFT) -#endif -#define vmemmap ((struct page *)VMEMMAP_BASE) - - -/* - * Include the PTE bits definitions - */ -#ifdef CONFIG_PPC_BOOK3S -#include <asm/pte-hash64.h> -#else -#include <asm/pte-book3e.h> -#endif -#include <asm/pte-common.h> - -#ifdef CONFIG_PPC_MM_SLICES -#define HAVE_ARCH_UNMAPPED_AREA -#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN -#endif /* CONFIG_PPC_MM_SLICES */ - -#ifndef __ASSEMBLY__ - -/* - * This is the default implementation of various PTE accessors, it's - * used in all cases except Book3S with 64K pages where we have a - * concept of sub-pages - */ -#ifndef __real_pte - -#ifdef STRICT_MM_TYPECHECKS -#define __real_pte(e,p) ((real_pte_t){(e)}) -#define __rpte_to_pte(r) ((r).pte) -#else -#define __real_pte(e,p) (e) -#define __rpte_to_pte(r) (__pte(r)) -#endif -#define __rpte_to_hidx(r,index) (pte_val(__rpte_to_pte(r)) >> 12) - -#define pte_iterate_hashed_subpages(rpte, psize, va, index, shift) \ - do { \ - index = 0; \ - shift = mmu_psize_defs[psize].shift; \ - -#define pte_iterate_hashed_end() } while(0) - -#ifdef CONFIG_PPC_HAS_HASH_64K -#define pte_pagesize_index(mm, addr, pte) get_slice_psize(mm, addr) -#else -#define pte_pagesize_index(mm, addr, pte) MMU_PAGE_4K -#endif - -#endif /* __real_pte */ - - -/* pte_clear moved to later in this file */ - -#define PMD_BAD_BITS (PTE_TABLE_SIZE-1) -#define PUD_BAD_BITS (PMD_TABLE_SIZE-1) - -#define pmd_set(pmdp, pmdval) (pmd_val(*(pmdp)) = (pmdval)) -#define pmd_none(pmd) (!pmd_val(pmd)) -#define pmd_bad(pmd) (!is_kernel_addr(pmd_val(pmd)) \ - || (pmd_val(pmd) & PMD_BAD_BITS)) -#define pmd_present(pmd) (pmd_val(pmd) != 0) -#define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0) -#define pmd_page_vaddr(pmd) (pmd_val(pmd) & ~PMD_MASKED_BITS) -extern struct page *pmd_page(pmd_t pmd); - -#define pud_set(pudp, pudval) (pud_val(*(pudp)) = (pudval)) -#define pud_none(pud) (!pud_val(pud)) -#define pud_bad(pud) (!is_kernel_addr(pud_val(pud)) \ - || (pud_val(pud) & PUD_BAD_BITS)) -#define pud_present(pud) (pud_val(pud) != 0) -#define pud_clear(pudp) (pud_val(*(pudp)) = 0) -#define pud_page_vaddr(pud) (pud_val(pud) & ~PUD_MASKED_BITS) -#define pud_page(pud) virt_to_page(pud_page_vaddr(pud)) - -#define pgd_set(pgdp, pudp) ({pgd_val(*(pgdp)) = (unsigned long)(pudp);}) - -/* - * Find an entry in a page-table-directory. We combine the address region - * (the high order N bits) and the pgd portion of the address. - */ -#define pgd_index(address) (((address) >> (PGDIR_SHIFT)) & (PTRS_PER_PGD - 1)) - -#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) - -#define pmd_offset(pudp,addr) \ - (((pmd_t *) pud_page_vaddr(*(pudp))) + (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))) - -#define pte_offset_kernel(dir,addr) \ - (((pte_t *) pmd_page_vaddr(*(dir))) + (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))) - -#define pte_offset_map(dir,addr) pte_offset_kernel((dir), (addr)) -#define pte_unmap(pte) do { } while(0) - -/* to find an entry in a kernel page-table-directory */ -/* This now only contains the vmalloc pages */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) -extern void hpte_need_flush(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, unsigned long pte, int huge); - -/* Atomic PTE updates */ -static inline unsigned long pte_update(struct mm_struct *mm, - unsigned long addr, - pte_t *ptep, unsigned long clr, - int huge) -{ -#ifdef PTE_ATOMIC_UPDATES - unsigned long old, tmp; - - __asm__ __volatile__( - "1: ldarx %0,0,%3 # pte_update\n\ - andi. %1,%0,%6\n\ - bne- 1b \n\ - andc %1,%0,%4 \n\ - stdcx. %1,0,%3 \n\ - bne- 1b" - : "=&r" (old), "=&r" (tmp), "=m" (*ptep) - : "r" (ptep), "r" (clr), "m" (*ptep), "i" (_PAGE_BUSY) - : "cc" ); -#else - unsigned long old = pte_val(*ptep); - *ptep = __pte(old & ~clr); -#endif - /* huge pages use the old page table lock */ - if (!huge) - assert_pte_locked(mm, addr); - -#ifdef CONFIG_PPC_STD_MMU_64 - if (old & _PAGE_HASHPTE) - hpte_need_flush(mm, addr, ptep, old, huge); -#endif - - return old; -} - -static inline int __ptep_test_and_clear_young(struct mm_struct *mm, - unsigned long addr, pte_t *ptep) -{ - unsigned long old; - - if ((pte_val(*ptep) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0) - return 0; - old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0); - return (old & _PAGE_ACCESSED) != 0; -} -#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG -#define ptep_test_and_clear_young(__vma, __addr, __ptep) \ -({ \ - int __r; \ - __r = __ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep); \ - __r; \ -}) - -#define __HAVE_ARCH_PTEP_SET_WRPROTECT -static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, - pte_t *ptep) -{ - - if ((pte_val(*ptep) & _PAGE_RW) == 0) - return; - - pte_update(mm, addr, ptep, _PAGE_RW, 0); -} - -static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, - unsigned long addr, pte_t *ptep) -{ - if ((pte_val(*ptep) & _PAGE_RW) == 0) - return; - - pte_update(mm, addr, ptep, _PAGE_RW, 1); -} - -/* - * We currently remove entries from the hashtable regardless of whether - * the entry was young or dirty. The generic routines only flush if the - * entry was young or dirty which is not good enough. - * - * We should be more intelligent about this but for the moment we override - * these functions and force a tlb flush unconditionally - */ -#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH -#define ptep_clear_flush_young(__vma, __address, __ptep) \ -({ \ - int __young = __ptep_test_and_clear_young((__vma)->vm_mm, __address, \ - __ptep); \ - __young; \ -}) - -#define __HAVE_ARCH_PTEP_GET_AND_CLEAR -static inline pte_t ptep_get_and_clear(struct mm_struct *mm, - unsigned long addr, pte_t *ptep) -{ - unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0); - return __pte(old); -} - -static inline void pte_clear(struct mm_struct *mm, unsigned long addr, - pte_t * ptep) -{ - pte_update(mm, addr, ptep, ~0UL, 0); -} - - -/* Set the dirty and/or accessed bits atomically in a linux PTE, this - * function doesn't need to flush the hash entry - */ -static inline void __ptep_set_access_flags(pte_t *ptep, pte_t entry) -{ - unsigned long bits = pte_val(entry) & - (_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC); - -#ifdef PTE_ATOMIC_UPDATES - unsigned long old, tmp; - - __asm__ __volatile__( - "1: ldarx %0,0,%4\n\ - andi. %1,%0,%6\n\ - bne- 1b \n\ - or %0,%3,%0\n\ - stdcx. %0,0,%4\n\ - bne- 1b" - :"=&r" (old), "=&r" (tmp), "=m" (*ptep) - :"r" (bits), "r" (ptep), "m" (*ptep), "i" (_PAGE_BUSY) - :"cc"); -#else - unsigned long old = pte_val(*ptep); - *ptep = __pte(old | bits); -#endif -} - -#define __HAVE_ARCH_PTE_SAME -#define pte_same(A,B) (((pte_val(A) ^ pte_val(B)) & ~_PAGE_HPTEFLAGS) == 0) - -#define pte_ERROR(e) \ - printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) -#define pmd_ERROR(e) \ - printk("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e)) -#define pgd_ERROR(e) \ - printk("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) - -/* Encode and de-code a swap entry */ -#define __swp_type(entry) (((entry).val >> 1) & 0x3f) -#define __swp_offset(entry) ((entry).val >> 8) -#define __swp_entry(type, offset) ((swp_entry_t){((type)<< 1)|((offset)<<8)}) -#define __pte_to_swp_entry(pte) ((swp_entry_t){pte_val(pte) >> PTE_RPN_SHIFT}) -#define __swp_entry_to_pte(x) ((pte_t) { (x).val << PTE_RPN_SHIFT }) -#define pte_to_pgoff(pte) (pte_val(pte) >> PTE_RPN_SHIFT) -#define pgoff_to_pte(off) ((pte_t) {((off) << PTE_RPN_SHIFT)|_PAGE_FILE}) -#define PTE_FILE_MAX_BITS (BITS_PER_LONG - PTE_RPN_SHIFT) - -void pgtable_cache_add(unsigned shift, void (*ctor)(void *)); -void pgtable_cache_init(void); -#endif /* __ASSEMBLY__ */ - -/* - * THP pages can't be special. So use the _PAGE_SPECIAL - */ -#define _PAGE_SPLITTING _PAGE_SPECIAL - -/* - * We need to differentiate between explicit huge page and THP huge - * page, since THP huge page also need to track real subpage details - */ -#define _PAGE_THP_HUGE _PAGE_4K_PFN - -/* - * set of bits not changed in pmd_modify. - */ -#define _HPAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HPTEFLAGS | \ - _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_SPLITTING | \ - _PAGE_THP_HUGE) - -#ifndef __ASSEMBLY__ -/* - * The linux hugepage PMD now include the pmd entries followed by the address - * to the stashed pgtable_t. The stashed pgtable_t contains the hpte bits. - * [ 1 bit secondary | 3 bit hidx | 1 bit valid | 000]. We use one byte per - * each HPTE entry. With 16MB hugepage and 64K HPTE we need 256 entries and - * with 4K HPTE we need 4096 entries. Both will fit in a 4K pgtable_t. - * - * The last three bits are intentionally left to zero. This memory location - * are also used as normal page PTE pointers. So if we have any pointers - * left around while we collapse a hugepage, we need to make sure - * _PAGE_PRESENT and _PAGE_FILE bits of that are zero when we look at them - */ -static inline unsigned int hpte_valid(unsigned char *hpte_slot_array, int index) -{ - return (hpte_slot_array[index] >> 3) & 0x1; -} - -static inline unsigned int hpte_hash_index(unsigned char *hpte_slot_array, - int index) -{ - return hpte_slot_array[index] >> 4; -} - -static inline void mark_hpte_slot_valid(unsigned char *hpte_slot_array, - unsigned int index, unsigned int hidx) -{ - hpte_slot_array[index] = hidx << 4 | 0x1 << 3; -} - -static inline char *get_hpte_slot_array(pmd_t *pmdp) -{ - /* - * The hpte hindex is stored in the pgtable whose address is in the - * second half of the PMD - * - * Order this load with the test for pmd_trans_huge in the caller - */ - smp_rmb(); - return *(char **)(pmdp + PTRS_PER_PMD); - - -} - -extern void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr, - pmd_t *pmdp); -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -extern pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot); -extern pmd_t mk_pmd(struct page *page, pgprot_t pgprot); -extern pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot); -extern void set_pmd_at(struct mm_struct *mm, unsigned long addr, - pmd_t *pmdp, pmd_t pmd); -extern void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr, - pmd_t *pmd); - -static inline int pmd_trans_huge(pmd_t pmd) -{ - /* - * leaf pte for huge page, bottom two bits != 00 - */ - return (pmd_val(pmd) & 0x3) && (pmd_val(pmd) & _PAGE_THP_HUGE); -} - -static inline int pmd_large(pmd_t pmd) -{ - /* - * leaf pte for huge page, bottom two bits != 00 - */ - if (pmd_trans_huge(pmd)) - return pmd_val(pmd) & _PAGE_PRESENT; - return 0; -} - -static inline int pmd_trans_splitting(pmd_t pmd) -{ - if (pmd_trans_huge(pmd)) - return pmd_val(pmd) & _PAGE_SPLITTING; - return 0; -} - -extern int has_transparent_hugepage(void); -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ - -static inline pte_t pmd_pte(pmd_t pmd) -{ - return __pte(pmd_val(pmd)); -} - -static inline pmd_t pte_pmd(pte_t pte) -{ - return __pmd(pte_val(pte)); -} - -static inline pte_t *pmdp_ptep(pmd_t *pmd) -{ - return (pte_t *)pmd; -} - -#define pmd_pfn(pmd) pte_pfn(pmd_pte(pmd)) -#define pmd_young(pmd) pte_young(pmd_pte(pmd)) -#define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd))) -#define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd))) -#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd))) -#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd))) -#define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd))) - -#define __HAVE_ARCH_PMD_WRITE -#define pmd_write(pmd) pte_write(pmd_pte(pmd)) - -static inline pmd_t pmd_mkhuge(pmd_t pmd) -{ - /* Do nothing, mk_pmd() does this part. */ - return pmd; -} - -static inline pmd_t pmd_mknotpresent(pmd_t pmd) -{ - pmd_val(pmd) &= ~_PAGE_PRESENT; - return pmd; -} - -static inline pmd_t pmd_mksplitting(pmd_t pmd) -{ - pmd_val(pmd) |= _PAGE_SPLITTING; - return pmd; -} - -#define __HAVE_ARCH_PMD_SAME -static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b) -{ - return (((pmd_val(pmd_a) ^ pmd_val(pmd_b)) & ~_PAGE_HPTEFLAGS) == 0); -} - -#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS -extern int pmdp_set_access_flags(struct vm_area_struct *vma, - unsigned long address, pmd_t *pmdp, - pmd_t entry, int dirty); - -extern unsigned long pmd_hugepage_update(struct mm_struct *mm, - unsigned long addr, - pmd_t *pmdp, unsigned long clr); - -static inline int __pmdp_test_and_clear_young(struct mm_struct *mm, - unsigned long addr, pmd_t *pmdp) -{ - unsigned long old; - - if ((pmd_val(*pmdp) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0) - return 0; - old = pmd_hugepage_update(mm, addr, pmdp, _PAGE_ACCESSED); - return ((old & _PAGE_ACCESSED) != 0); -} - -#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG -extern int pmdp_test_and_clear_young(struct vm_area_struct *vma, - unsigned long address, pmd_t *pmdp); -#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH -extern int pmdp_clear_flush_young(struct vm_area_struct *vma, - unsigned long address, pmd_t *pmdp); - -#define __HAVE_ARCH_PMDP_GET_AND_CLEAR -extern pmd_t pmdp_get_and_clear(struct mm_struct *mm, - unsigned long addr, pmd_t *pmdp); - -#define __HAVE_ARCH_PMDP_CLEAR_FLUSH -extern pmd_t pmdp_clear_flush(struct vm_area_struct *vma, unsigned long address, - pmd_t *pmdp); - -#define __HAVE_ARCH_PMDP_SET_WRPROTECT -static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr, - pmd_t *pmdp) -{ - - if ((pmd_val(*pmdp) & _PAGE_RW) == 0) - return; - - pmd_hugepage_update(mm, addr, pmdp, _PAGE_RW); -} - -#define __HAVE_ARCH_PMDP_SPLITTING_FLUSH -extern void pmdp_splitting_flush(struct vm_area_struct *vma, - unsigned long address, pmd_t *pmdp); - -#define __HAVE_ARCH_PGTABLE_DEPOSIT -extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, - pgtable_t pgtable); -#define __HAVE_ARCH_PGTABLE_WITHDRAW -extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); - -#define __HAVE_ARCH_PMDP_INVALIDATE -extern void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, - pmd_t *pmdp); -#endif /* __ASSEMBLY__ */ -#endif /* _ASM_POWERPC_PGTABLE_PPC64_H_ */ diff --git a/arch/powerpc/include/asm/pgtable-types.h b/arch/powerpc/include/asm/pgtable-types.h new file mode 100644 index 000000000000..f3086e39e7d2 --- /dev/null +++ b/arch/powerpc/include/asm/pgtable-types.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_PGTABLE_TYPES_H +#define _ASM_POWERPC_PGTABLE_TYPES_H + +#if defined(__CHECKER__) || !defined(CONFIG_PPC32) +#define STRICT_MM_TYPECHECKS +#endif + +/* PTE level */ +#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PPC_16K_PAGES) +typedef struct { pte_basic_t pte, pte1, pte2, pte3; } pte_t; +#elif defined(STRICT_MM_TYPECHECKS) +typedef struct { pte_basic_t pte; } pte_t; +#else +typedef pte_basic_t pte_t; +#endif + +#if defined(STRICT_MM_TYPECHECKS) || \ + (defined(CONFIG_PPC_8xx) && defined(CONFIG_PPC_16K_PAGES)) +#define __pte(x) ((pte_t) { (x) }) +static inline pte_basic_t pte_val(pte_t x) +{ + return x.pte; +} +#else +#define __pte(x) ((pte_t)(x)) +static inline pte_basic_t pte_val(pte_t x) +{ + return x; +} +#endif + +/* PMD level */ +#ifdef CONFIG_PPC64 +typedef struct { unsigned long pmd; } pmd_t; +#define __pmd(x) ((pmd_t) { (x) }) +static inline unsigned long pmd_val(pmd_t x) +{ + return x.pmd; +} + +/* 64 bit always use 4 level table. */ +typedef struct { unsigned long pud; } pud_t; +#define __pud(x) ((pud_t) { (x) }) +static inline unsigned long pud_val(pud_t x) +{ + return x.pud; +} +#endif /* CONFIG_PPC64 */ + +/* PGD level */ +#if defined(CONFIG_PPC_85xx) && defined(CONFIG_PTE_64BIT) +typedef struct { unsigned long long pgd; } pgd_t; + +static inline unsigned long long pgd_val(pgd_t x) +{ + return x.pgd; +} +#else +typedef struct { unsigned long pgd; } pgd_t; + +static inline unsigned long pgd_val(pgd_t x) +{ + return x.pgd; +} +#endif +#define __pgd(x) ((pgd_t) { (x) }) + +/* Page protection bits */ +typedef struct { unsigned long pgprot; } pgprot_t; +#define pgprot_val(x) ((x).pgprot) +#define __pgprot(x) ((pgprot_t) { (x) }) + +/* + * With hash config 64k pages additionally define a bigger "real PTE" type that + * gathers the "second half" part of the PTE for pseudo 64k pages + */ +#ifdef CONFIG_PPC_64K_PAGES +typedef struct { pte_t pte; unsigned long hidx; } real_pte_t; +#else +typedef struct { pte_t pte; } real_pte_t; +#endif + +#ifdef CONFIG_PPC_BOOK3S_64 +#include <asm/cmpxchg.h> + +static inline bool pte_xchg(pte_t *ptep, pte_t old, pte_t new) +{ + unsigned long *p = (unsigned long *)ptep; + + /* See comment in switch_mm_irqs_off() */ + return pte_val(old) == __cmpxchg_u64(p, pte_val(old), pte_val(new)); +} +#endif + +#endif /* _ASM_POWERPC_PGTABLE_TYPES_H */ diff --git a/arch/powerpc/include/asm/pgtable.h b/arch/powerpc/include/asm/pgtable.h index 7d6eacf249cf..17fd7ff6e535 100644 --- a/arch/powerpc/include/asm/pgtable.h +++ b/arch/powerpc/include/asm/pgtable.h @@ -1,185 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_PGTABLE_H #define _ASM_POWERPC_PGTABLE_H -#ifdef __KERNEL__ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ +#include <linux/mmdebug.h> +#include <linux/mmzone.h> #include <asm/processor.h> /* For TASK_SIZE */ #include <asm/mmu.h> #include <asm/page.h> +#include <asm/tlbflush.h> struct mm_struct; -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ -#if defined(CONFIG_PPC64) -# include <asm/pgtable-ppc64.h> +#ifdef CONFIG_PPC_BOOK3S +#include <asm/book3s/pgtable.h> #else -# include <asm/pgtable-ppc32.h> -#endif +#include <asm/nohash/pgtable.h> +#endif /* !CONFIG_PPC_BOOK3S */ -/* - * We save the slot number & secondary bit in the second half of the - * PTE page. We use the 8 bytes per each pte entry. - */ -#define PTE_PAGE_HIDX_OFFSET (PTRS_PER_PTE * 8) +/* Make modules code happy. We don't set RO yet */ +#define PAGE_KERNEL_EXEC PAGE_KERNEL_X -#ifndef __ASSEMBLY__ +/* Advertise special mapping type for AGP */ +#define PAGE_AGP (PAGE_KERNEL_NC) +#define HAVE_PAGE_AGP -#include <asm/tlbflush.h> +#ifndef __ASSEMBLER__ -/* Generic accessors to PTE bits */ -static inline int pte_write(pte_t pte) { return pte_val(pte) & _PAGE_RW; } -static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_DIRTY; } -static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; } -static inline int pte_file(pte_t pte) { return pte_val(pte) & _PAGE_FILE; } -static inline int pte_special(pte_t pte) { return pte_val(pte) & _PAGE_SPECIAL; } -static inline int pte_present(pte_t pte) { return pte_val(pte) & _PAGE_PRESENT; } -static inline int pte_none(pte_t pte) { return (pte_val(pte) & ~_PTE_NONE_MASK) == 0; } -static inline pgprot_t pte_pgprot(pte_t pte) { return __pgprot(pte_val(pte) & PAGE_PROT_BITS); } - -/* Conversion functions: convert a page and protection to a page entry, - * and a page entry and page directory to the page they refer to. - * - * Even if PTEs can be unsigned long long, a PFN is always an unsigned - * long for now. - */ -static inline pte_t pfn_pte(unsigned long pfn, pgprot_t pgprot) { - return __pte(((pte_basic_t)(pfn) << PTE_RPN_SHIFT) | - pgprot_val(pgprot)); } -static inline unsigned long pte_pfn(pte_t pte) { - return pte_val(pte) >> PTE_RPN_SHIFT; } +#define PFN_PTE_SHIFT PTE_RPN_SHIFT -/* Keep these as a macros to avoid include dependency mess */ -#define pte_page(x) pfn_to_page(pte_pfn(x)) -#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) - -/* Generic modifiers for PTE bits */ -static inline pte_t pte_wrprotect(pte_t pte) { - pte_val(pte) &= ~(_PAGE_RW | _PAGE_HWWRITE); return pte; } -static inline pte_t pte_mkclean(pte_t pte) { - pte_val(pte) &= ~(_PAGE_DIRTY | _PAGE_HWWRITE); return pte; } -static inline pte_t pte_mkold(pte_t pte) { - pte_val(pte) &= ~_PAGE_ACCESSED; return pte; } -static inline pte_t pte_mkwrite(pte_t pte) { - pte_val(pte) |= _PAGE_RW; return pte; } -static inline pte_t pte_mkdirty(pte_t pte) { - pte_val(pte) |= _PAGE_DIRTY; return pte; } -static inline pte_t pte_mkyoung(pte_t pte) { - pte_val(pte) |= _PAGE_ACCESSED; return pte; } -static inline pte_t pte_mkspecial(pte_t pte) { - pte_val(pte) |= _PAGE_SPECIAL; return pte; } -static inline pte_t pte_mkhuge(pte_t pte) { - return pte; } -static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) -{ - pte_val(pte) = (pte_val(pte) & _PAGE_CHG_MASK) | pgprot_val(newprot); - return pte; -} +void set_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, + pte_t pte, unsigned int nr); +#define set_ptes set_ptes +#define update_mmu_cache(vma, addr, ptep) \ + update_mmu_cache_range(NULL, vma, addr, ptep, 1) +#ifndef MAX_PTRS_PER_PGD +#define MAX_PTRS_PER_PGD PTRS_PER_PGD +#endif -/* Insert a PTE, top-level function is out of line. It uses an inline - * low level function in the respective pgtable-* files - */ -extern void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, - pte_t pte); +/* Keep this as a macro to avoid include dependency mess */ +#define pte_page(x) pfn_to_page(pte_pfn(x)) -/* This low level function performs the actual PTE insertion - * Setting the PTE depends on the MMU type and other factors. It's - * an horrible mess that I'm not going to try to clean up now but - * I'm keeping it in one place rather than spread around - */ -static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t pte, int percpu) +static inline unsigned long pte_pfn(pte_t pte) { -#if defined(CONFIG_PPC_STD_MMU_32) && defined(CONFIG_SMP) && !defined(CONFIG_PTE_64BIT) - /* First case is 32-bit Hash MMU in SMP mode with 32-bit PTEs. We use the - * helper pte_update() which does an atomic update. We need to do that - * because a concurrent invalidation can clear _PAGE_HASHPTE. If it's a - * per-CPU PTE such as a kmap_atomic, we do a simple update preserving - * the hash bits instead (ie, same as the non-SMP case) - */ - if (percpu) - *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE) - | (pte_val(pte) & ~_PAGE_HASHPTE)); - else - pte_update(ptep, ~_PAGE_HASHPTE, pte_val(pte)); - -#elif defined(CONFIG_PPC32) && defined(CONFIG_PTE_64BIT) - /* Second case is 32-bit with 64-bit PTE. In this case, we - * can just store as long as we do the two halves in the right order - * with a barrier in between. This is possible because we take care, - * in the hash code, to pre-invalidate if the PTE was already hashed, - * which synchronizes us with any concurrent invalidation. - * In the percpu case, we also fallback to the simple update preserving - * the hash bits - */ - if (percpu) { - *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE) - | (pte_val(pte) & ~_PAGE_HASHPTE)); - return; - } -#if _PAGE_HASHPTE != 0 - if (pte_val(*ptep) & _PAGE_HASHPTE) - flush_hash_entry(mm, ptep, addr); -#endif - __asm__ __volatile__("\ - stw%U0%X0 %2,%0\n\ - eieio\n\ - stw%U0%X0 %L2,%1" - : "=m" (*ptep), "=m" (*((unsigned char *)ptep+4)) - : "r" (pte) : "memory"); - -#elif defined(CONFIG_PPC_STD_MMU_32) - /* Third case is 32-bit hash table in UP mode, we need to preserve - * the _PAGE_HASHPTE bit since we may not have invalidated the previous - * translation in the hash yet (done in a subsequent flush_tlb_xxx()) - * and see we need to keep track that this PTE needs invalidating - */ - *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE) - | (pte_val(pte) & ~_PAGE_HASHPTE)); - -#else - /* Anything else just stores the PTE normally. That covers all 64-bit - * cases, and 32-bit non-hash with 32-bit PTEs. - */ - *ptep = pte; -#endif + return (pte_val(pte) & PTE_RPN_MASK) >> PTE_RPN_SHIFT; } - -#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS -extern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, - pte_t *ptep, pte_t entry, int dirty); - /* - * Macro to mark a page protection value as "uncacheable". + * Select all bits except the pfn */ +#define pte_pgprot pte_pgprot +static inline pgprot_t pte_pgprot(pte_t pte) +{ + unsigned long pte_flags; -#define _PAGE_CACHE_CTL (_PAGE_COHERENT | _PAGE_GUARDED | _PAGE_NO_CACHE | \ - _PAGE_WRITETHRU) - -#define pgprot_noncached(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ - _PAGE_NO_CACHE | _PAGE_GUARDED)) - -#define pgprot_noncached_wc(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ - _PAGE_NO_CACHE)) - -#define pgprot_cached(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ - _PAGE_COHERENT)) - -#define pgprot_cached_wthru(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ - _PAGE_COHERENT | _PAGE_WRITETHRU)) - -#define pgprot_cached_noncoherent(prot) \ - (__pgprot(pgprot_val(prot) & ~_PAGE_CACHE_CTL)) - -#define pgprot_writecombine pgprot_noncached_wc + pte_flags = pte_val(pte) & ~PTE_RPN_MASK; + return __pgprot(pte_flags); +} -struct file; -extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, - unsigned long size, pgprot_t vma_prot); -#define __HAVE_PHYS_MEM_ACCESS_PROT +static inline pgprot_t pgprot_nx(pgprot_t prot) +{ + return pte_pgprot(pte_exprotect(__pte(pgprot_val(prot)))); +} +#define pgprot_nx pgprot_nx +#ifndef pmd_page_vaddr +static inline const void *pmd_page_vaddr(pmd_t pmd) +{ + return __va(pmd_val(pmd) & ~PMD_MASKED_BITS); +} +#define pmd_page_vaddr pmd_page_vaddr +#endif /* * ZERO_PAGE is a global shared page that is always zero: used * for zero-mapped memory areas etc.. @@ -190,16 +84,43 @@ extern unsigned long empty_zero_page[]; extern pgd_t swapper_pg_dir[]; extern void paging_init(void); +void poking_init(void); -/* - * kern_addr_valid is intended to indicate whether an address is a valid - * kernel address. Most 32-bit archs define it as always true (like this) - * but most 64-bit archs actually perform a test. What should we do here? - */ -#define kern_addr_valid(addr) (1) +extern unsigned long ioremap_bot; +extern const pgprot_t protection_map[16]; -#include <asm-generic/pgtable.h> +/* can we use this in kvm */ +unsigned long vmalloc_to_phys(void *vmalloc_addr); +void pgtable_cache_add(unsigned int shift); + +#ifdef CONFIG_PPC32 +void __init *early_alloc_pgtable(unsigned long size); +#endif +pte_t *early_pte_alloc_kernel(pmd_t *pmdp, unsigned long va); + +#if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_PPC32) +void mark_initmem_nx(void); +#else +static inline void mark_initmem_nx(void) { } +#endif + +#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS +int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, + pte_t *ptep, pte_t entry, int dirty); + +pgprot_t __phys_mem_access_prot(unsigned long pfn, unsigned long size, + pgprot_t vma_prot); + +struct file; +static inline pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, + unsigned long size, pgprot_t vma_prot) +{ + return __phys_mem_access_prot(pfn, size, vma_prot); +} +#define __HAVE_PHYS_MEM_ACCESS_PROT + +void __update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep); /* * This gets called at the end of handling a page fault, when @@ -210,20 +131,77 @@ extern void paging_init(void); * corresponding HPTE into the hash table ahead of time, instead of * waiting for the inevitable extra hash-table miss exception. */ -extern void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t *); +static inline void update_mmu_cache_range(struct vm_fault *vmf, + struct vm_area_struct *vma, unsigned long address, + pte_t *ptep, unsigned int nr) +{ + if ((mmu_has_feature(MMU_FTR_HPTE_TABLE) && !radix_enabled()) || + (IS_ENABLED(CONFIG_PPC_E500) && IS_ENABLED(CONFIG_HUGETLB_PAGE))) + __update_mmu_cache(vma, address, ptep); +} + +/* + * When used, PTE_FRAG_NR is defined in subarch pgtable.h + * so we are sure it is included when arriving here. + */ +#ifdef PTE_FRAG_NR +static inline void *pte_frag_get(mm_context_t *ctx) +{ + return ctx->pte_frag; +} + +static inline void pte_frag_set(mm_context_t *ctx, void *p) +{ + ctx->pte_frag = p; +} +#else +#define PTE_FRAG_NR 1 +#define PTE_FRAG_SIZE_SHIFT PAGE_SHIFT +#define PTE_FRAG_SIZE (1UL << PTE_FRAG_SIZE_SHIFT) -extern int gup_hugepd(hugepd_t *hugepd, unsigned pdshift, unsigned long addr, - unsigned long end, int write, struct page **pages, int *nr); +static inline void *pte_frag_get(mm_context_t *ctx) +{ + return NULL; +} -extern int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, - unsigned long end, int write, struct page **pages, int *nr); -#ifndef CONFIG_TRANSPARENT_HUGEPAGE -#define pmd_large(pmd) 0 -#define has_transparent_hugepage() 0 +static inline void pte_frag_set(mm_context_t *ctx, void *p) +{ +} #endif -pte_t *find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea, - unsigned *shift); -#endif /* __ASSEMBLY__ */ -#endif /* __KERNEL__ */ +#define pmd_pgtable pmd_pgtable +static inline pgtable_t pmd_pgtable(pmd_t pmd) +{ + return (pgtable_t)pmd_page_vaddr(pmd); +} + +#ifdef CONFIG_PPC64 +int __meminit vmemmap_populated(unsigned long vmemmap_addr, int vmemmap_map_size); +bool altmap_cross_boundary(struct vmem_altmap *altmap, unsigned long start, + unsigned long page_size); +/* + * mm/memory_hotplug.c:mhp_supports_memmap_on_memory goes into details + * some of the restrictions. We don't check for PMD_SIZE because our + * vmemmap allocation code can fallback correctly. The pageblock + * alignment requirement is met using altmap->reserve blocks. + */ +#define arch_supports_memmap_on_memory arch_supports_memmap_on_memory +static inline bool arch_supports_memmap_on_memory(unsigned long vmemmap_size) +{ + if (!radix_enabled()) + return false; + /* + * With 4K page size and 2M PMD_SIZE, we can align + * things better with memory block size value + * starting from 128MB. Hence align things with PMD_SIZE. + */ + if (IS_ENABLED(CONFIG_PPC_4K_PAGES)) + return IS_ALIGNED(vmemmap_size, PMD_SIZE); + return true; +} + +#endif /* CONFIG_PPC64 */ + +#endif /* __ASSEMBLER__ */ + #endif /* _ASM_POWERPC_PGTABLE_H */ diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h new file mode 100644 index 000000000000..28e752138996 --- /dev/null +++ b/arch/powerpc/include/asm/pkeys.h @@ -0,0 +1,172 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * PowerPC Memory Protection Keys management + * + * Copyright 2017, Ram Pai, IBM Corporation. + */ + +#ifndef _ASM_POWERPC_KEYS_H +#define _ASM_POWERPC_KEYS_H + +#include <linux/jump_label.h> +#include <asm/firmware.h> + +extern int num_pkey; +extern u32 reserved_allocation_mask; /* bits set for reserved keys */ + +#define ARCH_VM_PKEY_FLAGS (VM_PKEY_BIT0 | VM_PKEY_BIT1 | VM_PKEY_BIT2 | \ + VM_PKEY_BIT3 | VM_PKEY_BIT4) + +/* Override any generic PKEY permission defines */ +#define PKEY_DISABLE_EXECUTE 0x4 +#define PKEY_ACCESS_MASK (PKEY_DISABLE_ACCESS | \ + PKEY_DISABLE_WRITE | \ + PKEY_DISABLE_EXECUTE) + +#ifdef CONFIG_PPC_BOOK3S_64 +#include <asm/book3s/64/pkeys.h> +#else +#error "Not supported" +#endif + + +static inline vm_flags_t pkey_to_vmflag_bits(u16 pkey) +{ + return (((vm_flags_t)pkey << VM_PKEY_SHIFT) & ARCH_VM_PKEY_FLAGS); +} + +static inline int vma_pkey(struct vm_area_struct *vma) +{ + if (!mmu_has_feature(MMU_FTR_PKEY)) + return 0; + return (vma->vm_flags & ARCH_VM_PKEY_FLAGS) >> VM_PKEY_SHIFT; +} + +static inline int arch_max_pkey(void) +{ + return num_pkey; +} + +#define pkey_alloc_mask(pkey) (0x1 << pkey) + +#define mm_pkey_allocation_map(mm) (mm->context.pkey_allocation_map) + +#define __mm_pkey_allocated(mm, pkey) { \ + mm_pkey_allocation_map(mm) |= pkey_alloc_mask(pkey); \ +} + +#define __mm_pkey_free(mm, pkey) { \ + mm_pkey_allocation_map(mm) &= ~pkey_alloc_mask(pkey); \ +} + +#define __mm_pkey_is_allocated(mm, pkey) \ + (mm_pkey_allocation_map(mm) & pkey_alloc_mask(pkey)) + +#define __mm_pkey_is_reserved(pkey) (reserved_allocation_mask & \ + pkey_alloc_mask(pkey)) + +static inline bool mm_pkey_is_allocated(struct mm_struct *mm, int pkey) +{ + if (pkey < 0 || pkey >= arch_max_pkey()) + return false; + + /* Reserved keys are never allocated. */ + if (__mm_pkey_is_reserved(pkey)) + return false; + + return __mm_pkey_is_allocated(mm, pkey); +} + +/* + * Returns a positive, 5-bit key on success, or -1 on failure. + * Relies on the mmap_lock to protect against concurrency in mm_pkey_alloc() and + * mm_pkey_free(). + */ +static inline int mm_pkey_alloc(struct mm_struct *mm) +{ + /* + * Note: this is the one and only place we make sure that the pkey is + * valid as far as the hardware is concerned. The rest of the kernel + * trusts that only good, valid pkeys come out of here. + */ + u32 all_pkeys_mask = (u32)(~(0x0)); + int ret; + + if (!mmu_has_feature(MMU_FTR_PKEY)) + return -1; + /* + * Are we out of pkeys? We must handle this specially because ffz() + * behavior is undefined if there are no zeros. + */ + if (mm_pkey_allocation_map(mm) == all_pkeys_mask) + return -1; + + ret = ffz((u32)mm_pkey_allocation_map(mm)); + __mm_pkey_allocated(mm, ret); + + return ret; +} + +static inline int mm_pkey_free(struct mm_struct *mm, int pkey) +{ + if (!mmu_has_feature(MMU_FTR_PKEY)) + return -1; + + if (!mm_pkey_is_allocated(mm, pkey)) + return -EINVAL; + + __mm_pkey_free(mm, pkey); + + return 0; +} + +/* + * Try to dedicate one of the protection keys to be used as an + * execute-only protection key. + */ +extern int execute_only_pkey(struct mm_struct *mm); +extern int __arch_override_mprotect_pkey(struct vm_area_struct *vma, + int prot, int pkey); +static inline int arch_override_mprotect_pkey(struct vm_area_struct *vma, + int prot, int pkey) +{ + if (!mmu_has_feature(MMU_FTR_PKEY)) + return 0; + + /* + * Is this an mprotect_pkey() call? If so, never override the value that + * came from the user. + */ + if (pkey != -1) + return pkey; + + return __arch_override_mprotect_pkey(vma, prot, pkey); +} + +extern int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey, + unsigned long init_val); +static inline int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, + unsigned long init_val) +{ + if (!mmu_has_feature(MMU_FTR_PKEY)) + return -EINVAL; + + /* + * userspace should not change pkey-0 permissions. + * pkey-0 is associated with every page in the kernel. + * If userspace denies any permission on pkey-0, the + * kernel cannot operate. + */ + if (pkey == 0) + return init_val ? -EINVAL : 0; + + return __arch_set_user_pkey_access(tsk, pkey, init_val); +} + +static inline bool arch_pkeys_enabled(void) +{ + return mmu_has_feature(MMU_FTR_PKEY); +} + +extern void pkey_mm_init(struct mm_struct *mm); +#endif /*_ASM_POWERPC_KEYS_H */ diff --git a/arch/powerpc/include/asm/plpar_wrappers.h b/arch/powerpc/include/asm/plpar_wrappers.h new file mode 100644 index 000000000000..f2b6cc4341bb --- /dev/null +++ b/arch/powerpc/include/asm/plpar_wrappers.h @@ -0,0 +1,676 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_PLPAR_WRAPPERS_H +#define _ASM_POWERPC_PLPAR_WRAPPERS_H + +#ifdef CONFIG_PPC_PSERIES + +#include <linux/string.h> +#include <linux/irqflags.h> +#include <linux/delay.h> + +#include <asm/hvcall.h> +#include <asm/paca.h> +#include <asm/lppaca.h> +#include <asm/page.h> + +static inline long poll_pending(void) +{ + return plpar_hcall_norets(H_POLL_PENDING); +} + +static inline long cede_processor(void) +{ + /* + * We cannot call tracepoints inside RCU idle regions which + * means we must not trace H_CEDE. + */ + return plpar_hcall_norets_notrace(H_CEDE); +} + +static inline long vpa_call(unsigned long flags, unsigned long cpu, + unsigned long vpa) +{ + flags = flags << H_VPA_FUNC_SHIFT; + + return plpar_hcall_norets(H_REGISTER_VPA, flags, cpu, vpa); +} + +static inline long unregister_vpa(unsigned long cpu) +{ + return vpa_call(H_VPA_DEREG_VPA, cpu, 0); +} + +static inline long register_vpa(unsigned long cpu, unsigned long vpa) +{ + return vpa_call(H_VPA_REG_VPA, cpu, vpa); +} + +static inline long unregister_slb_shadow(unsigned long cpu) +{ + return vpa_call(H_VPA_DEREG_SLB, cpu, 0); +} + +static inline long register_slb_shadow(unsigned long cpu, unsigned long vpa) +{ + return vpa_call(H_VPA_REG_SLB, cpu, vpa); +} + +static inline long unregister_dtl(unsigned long cpu) +{ + return vpa_call(H_VPA_DEREG_DTL, cpu, 0); +} + +static inline long register_dtl(unsigned long cpu, unsigned long vpa) +{ + return vpa_call(H_VPA_REG_DTL, cpu, vpa); +} + +/* + * Invokes H_HTM hcall with parameters passed from htm_hcall_wrapper. + * flags: Set to hardwareTarget. + * target: Specifies target using node index, nodal chip index and core index. + * operation : action to perform ie configure, start, stop, deconfigure, trace + * based on the HTM type. + * param1, param2, param3: parameters for each action. + */ +static inline long htm_call(unsigned long flags, unsigned long target, + unsigned long operation, unsigned long param1, + unsigned long param2, unsigned long param3) +{ + return plpar_hcall_norets(H_HTM, flags, target, operation, + param1, param2, param3); +} + +static inline long htm_hcall_wrapper(unsigned long flags, unsigned long nodeindex, + unsigned long nodalchipindex, unsigned long coreindexonchip, + unsigned long type, unsigned long htm_op, unsigned long param1, unsigned long param2, + unsigned long param3) +{ + return htm_call(H_HTM_FLAGS_HARDWARE_TARGET | flags, + H_HTM_TARGET_NODE_INDEX(nodeindex) | + H_HTM_TARGET_NODAL_CHIP_INDEX(nodalchipindex) | + H_HTM_TARGET_CORE_INDEX_ON_CHIP(coreindexonchip), + H_HTM_OP(htm_op) | H_HTM_TYPE(type), + param1, param2, param3); +} + +extern void vpa_init(int cpu); + +static inline long plpar_pte_enter(unsigned long flags, + unsigned long hpte_group, unsigned long hpte_v, + unsigned long hpte_r, unsigned long *slot) +{ + long rc; + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + + rc = plpar_hcall(H_ENTER, retbuf, flags, hpte_group, hpte_v, hpte_r); + + *slot = retbuf[0]; + + return rc; +} + +static inline long plpar_pte_remove(unsigned long flags, unsigned long ptex, + unsigned long avpn, unsigned long *old_pteh_ret, + unsigned long *old_ptel_ret) +{ + long rc; + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + + rc = plpar_hcall(H_REMOVE, retbuf, flags, ptex, avpn); + + *old_pteh_ret = retbuf[0]; + *old_ptel_ret = retbuf[1]; + + return rc; +} + +/* plpar_pte_remove_raw can be called in real mode. It calls plpar_hcall_raw */ +static inline long plpar_pte_remove_raw(unsigned long flags, unsigned long ptex, + unsigned long avpn, unsigned long *old_pteh_ret, + unsigned long *old_ptel_ret) +{ + long rc; + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + + rc = plpar_hcall_raw(H_REMOVE, retbuf, flags, ptex, avpn); + + *old_pteh_ret = retbuf[0]; + *old_ptel_ret = retbuf[1]; + + return rc; +} + +static inline long plpar_pte_read(unsigned long flags, unsigned long ptex, + unsigned long *old_pteh_ret, unsigned long *old_ptel_ret) +{ + long rc; + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + + rc = plpar_hcall(H_READ, retbuf, flags, ptex); + + *old_pteh_ret = retbuf[0]; + *old_ptel_ret = retbuf[1]; + + return rc; +} + +/* plpar_pte_read_raw can be called in real mode. It calls plpar_hcall_raw */ +static inline long plpar_pte_read_raw(unsigned long flags, unsigned long ptex, + unsigned long *old_pteh_ret, unsigned long *old_ptel_ret) +{ + long rc; + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + + rc = plpar_hcall_raw(H_READ, retbuf, flags, ptex); + + *old_pteh_ret = retbuf[0]; + *old_ptel_ret = retbuf[1]; + + return rc; +} + +/* + * ptes must be 8*sizeof(unsigned long) + */ +static inline long plpar_pte_read_4(unsigned long flags, unsigned long ptex, + unsigned long *ptes) + +{ + long rc; + unsigned long retbuf[PLPAR_HCALL9_BUFSIZE]; + + rc = plpar_hcall9(H_READ, retbuf, flags | H_READ_4, ptex); + + memcpy(ptes, retbuf, 8*sizeof(unsigned long)); + + return rc; +} + +/* + * plpar_pte_read_4_raw can be called in real mode. + * ptes must be 8*sizeof(unsigned long) + */ +static inline long plpar_pte_read_4_raw(unsigned long flags, unsigned long ptex, + unsigned long *ptes) + +{ + long rc; + unsigned long retbuf[PLPAR_HCALL9_BUFSIZE]; + + rc = plpar_hcall9_raw(H_READ, retbuf, flags | H_READ_4, ptex); + + memcpy(ptes, retbuf, 8*sizeof(unsigned long)); + + return rc; +} + +static inline long plpar_pte_protect(unsigned long flags, unsigned long ptex, + unsigned long avpn) +{ + return plpar_hcall_norets(H_PROTECT, flags, ptex, avpn); +} + +static inline long plpar_resize_hpt_prepare(unsigned long flags, + unsigned long shift) +{ + return plpar_hcall_norets(H_RESIZE_HPT_PREPARE, flags, shift); +} + +static inline long plpar_resize_hpt_commit(unsigned long flags, + unsigned long shift) +{ + return plpar_hcall_norets(H_RESIZE_HPT_COMMIT, flags, shift); +} + +static inline long plpar_tce_get(unsigned long liobn, unsigned long ioba, + unsigned long *tce_ret) +{ + long rc; + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + + rc = plpar_hcall(H_GET_TCE, retbuf, liobn, ioba); + + *tce_ret = retbuf[0]; + + return rc; +} + +static inline long plpar_tce_put(unsigned long liobn, unsigned long ioba, + unsigned long tceval) +{ + return plpar_hcall_norets(H_PUT_TCE, liobn, ioba, tceval); +} + +static inline long plpar_tce_put_indirect(unsigned long liobn, + unsigned long ioba, unsigned long page, unsigned long count) +{ + return plpar_hcall_norets(H_PUT_TCE_INDIRECT, liobn, ioba, page, count); +} + +static inline long plpar_tce_stuff(unsigned long liobn, unsigned long ioba, + unsigned long tceval, unsigned long count) +{ + return plpar_hcall_norets(H_STUFF_TCE, liobn, ioba, tceval, count); +} + +/* Set various resource mode parameters */ +static inline long plpar_set_mode(unsigned long mflags, unsigned long resource, + unsigned long value1, unsigned long value2) +{ + return plpar_hcall_norets(H_SET_MODE, mflags, resource, value1, value2); +} + +/* + * Enable relocation on exceptions on this partition + * + * Note: this call has a partition wide scope and can take a while to complete. + * If it returns H_LONG_BUSY_* it should be retried periodically until it + * returns H_SUCCESS. + */ +static inline long enable_reloc_on_exceptions(void) +{ + /* mflags = 3: Exceptions at 0xC000000000004000 */ + return plpar_set_mode(3, H_SET_MODE_RESOURCE_ADDR_TRANS_MODE, 0, 0); +} + +/* + * Disable relocation on exceptions on this partition + * + * Note: this call has a partition wide scope and can take a while to complete. + * If it returns H_LONG_BUSY_* it should be retried periodically until it + * returns H_SUCCESS. + */ +static inline long disable_reloc_on_exceptions(void) { + return plpar_set_mode(0, H_SET_MODE_RESOURCE_ADDR_TRANS_MODE, 0, 0); +} + +/* + * Take exceptions in big endian mode on this partition + * + * Note: this call has a partition wide scope and can take a while to complete. + * If it returns H_LONG_BUSY_* it should be retried periodically until it + * returns H_SUCCESS. + */ +static inline long enable_big_endian_exceptions(void) +{ + /* mflags = 0: big endian exceptions */ + return plpar_set_mode(0, H_SET_MODE_RESOURCE_LE, 0, 0); +} + +/* + * Take exceptions in little endian mode on this partition + * + * Note: this call has a partition wide scope and can take a while to complete. + * If it returns H_LONG_BUSY_* it should be retried periodically until it + * returns H_SUCCESS. + */ +static inline long enable_little_endian_exceptions(void) +{ + /* mflags = 1: little endian exceptions */ + return plpar_set_mode(1, H_SET_MODE_RESOURCE_LE, 0, 0); +} + +static inline long plpar_set_ciabr(unsigned long ciabr) +{ + return plpar_set_mode(0, H_SET_MODE_RESOURCE_SET_CIABR, ciabr, 0); +} + +static inline long plpar_set_watchpoint0(unsigned long dawr0, unsigned long dawrx0) +{ + return plpar_set_mode(0, H_SET_MODE_RESOURCE_SET_DAWR0, dawr0, dawrx0); +} + +static inline long plpar_set_watchpoint1(unsigned long dawr1, unsigned long dawrx1) +{ + return plpar_set_mode(0, H_SET_MODE_RESOURCE_SET_DAWR1, dawr1, dawrx1); +} + +static inline long plpar_signal_sys_reset(long cpu) +{ + return plpar_hcall_norets(H_SIGNAL_SYS_RESET, cpu); +} + +static inline long plpar_get_cpu_characteristics(struct h_cpu_char_result *p) +{ + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + long rc; + + rc = plpar_hcall(H_GET_CPU_CHARACTERISTICS, retbuf); + if (rc == H_SUCCESS) { + p->character = retbuf[0]; + p->behaviour = retbuf[1]; + } + + return rc; +} + +static inline long plpar_guest_create(unsigned long flags, unsigned long *guest_id) +{ + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + unsigned long token; + long rc; + + token = -1UL; + do { + rc = plpar_hcall(H_GUEST_CREATE, retbuf, flags, token); + if (rc == H_SUCCESS) + *guest_id = retbuf[0]; + + if (rc == H_BUSY) { + token = retbuf[0]; + cond_resched(); + } + + if (H_IS_LONG_BUSY(rc)) { + token = retbuf[0]; + msleep(get_longbusy_msecs(rc)); + rc = H_BUSY; + } + + } while (rc == H_BUSY); + + return rc; +} + +static inline long plpar_guest_create_vcpu(unsigned long flags, + unsigned long guest_id, + unsigned long vcpu_id) +{ + long rc; + + do { + rc = plpar_hcall_norets(H_GUEST_CREATE_VCPU, 0, guest_id, vcpu_id); + + if (rc == H_BUSY) + cond_resched(); + + if (H_IS_LONG_BUSY(rc)) { + msleep(get_longbusy_msecs(rc)); + rc = H_BUSY; + } + + } while (rc == H_BUSY); + + return rc; +} + +static inline long plpar_guest_set_state(unsigned long flags, + unsigned long guest_id, + unsigned long vcpu_id, + unsigned long data_buffer, + unsigned long data_size, + unsigned long *failed_index) +{ + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + long rc; + + while (true) { + rc = plpar_hcall(H_GUEST_SET_STATE, retbuf, flags, guest_id, + vcpu_id, data_buffer, data_size); + + if (rc == H_BUSY) { + cpu_relax(); + continue; + } + + if (H_IS_LONG_BUSY(rc)) { + mdelay(get_longbusy_msecs(rc)); + continue; + } + + if (rc == H_INVALID_ELEMENT_ID) + *failed_index = retbuf[0]; + else if (rc == H_INVALID_ELEMENT_SIZE) + *failed_index = retbuf[0]; + else if (rc == H_INVALID_ELEMENT_VALUE) + *failed_index = retbuf[0]; + + break; + } + + return rc; +} + +static inline long plpar_guest_get_state(unsigned long flags, + unsigned long guest_id, + unsigned long vcpu_id, + unsigned long data_buffer, + unsigned long data_size, + unsigned long *failed_index) +{ + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + long rc; + + while (true) { + rc = plpar_hcall(H_GUEST_GET_STATE, retbuf, flags, guest_id, + vcpu_id, data_buffer, data_size); + + if (rc == H_BUSY) { + cpu_relax(); + continue; + } + + if (H_IS_LONG_BUSY(rc)) { + mdelay(get_longbusy_msecs(rc)); + continue; + } + + if (rc == H_INVALID_ELEMENT_ID) + *failed_index = retbuf[0]; + else if (rc == H_INVALID_ELEMENT_SIZE) + *failed_index = retbuf[0]; + else if (rc == H_INVALID_ELEMENT_VALUE) + *failed_index = retbuf[0]; + + break; + } + + return rc; +} + +static inline long plpar_guest_run_vcpu(unsigned long flags, unsigned long guest_id, + unsigned long vcpu_id, int *trap, + unsigned long *failed_index) +{ + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + long rc; + + rc = plpar_hcall(H_GUEST_RUN_VCPU, retbuf, flags, guest_id, vcpu_id); + if (rc == H_SUCCESS) + *trap = retbuf[0]; + else if (rc == H_INVALID_ELEMENT_ID) + *failed_index = retbuf[0]; + else if (rc == H_INVALID_ELEMENT_SIZE) + *failed_index = retbuf[0]; + else if (rc == H_INVALID_ELEMENT_VALUE) + *failed_index = retbuf[0]; + + return rc; +} + +static inline long plpar_guest_delete(unsigned long flags, u64 guest_id) +{ + long rc; + + do { + rc = plpar_hcall_norets(H_GUEST_DELETE, flags, guest_id); + if (rc == H_BUSY) + cond_resched(); + + if (H_IS_LONG_BUSY(rc)) { + msleep(get_longbusy_msecs(rc)); + rc = H_BUSY; + } + + } while (rc == H_BUSY); + + return rc; +} + +static inline long plpar_guest_set_capabilities(unsigned long flags, + unsigned long capabilities) +{ + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + long rc; + + do { + rc = plpar_hcall(H_GUEST_SET_CAPABILITIES, retbuf, flags, capabilities); + if (rc == H_BUSY) + cond_resched(); + + if (H_IS_LONG_BUSY(rc)) { + msleep(get_longbusy_msecs(rc)); + rc = H_BUSY; + } + } while (rc == H_BUSY); + + return rc; +} + +static inline long plpar_guest_get_capabilities(unsigned long flags, + unsigned long *capabilities) +{ + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + long rc; + + do { + rc = plpar_hcall(H_GUEST_GET_CAPABILITIES, retbuf, flags); + if (rc == H_BUSY) + cond_resched(); + + if (H_IS_LONG_BUSY(rc)) { + msleep(get_longbusy_msecs(rc)); + rc = H_BUSY; + } + } while (rc == H_BUSY); + + if (rc == H_SUCCESS) + *capabilities = retbuf[0]; + + return rc; +} + +/* + * Wrapper to H_RPT_INVALIDATE hcall that handles return values appropriately + * + * - Returns H_SUCCESS on success + * - For H_BUSY return value, we retry the hcall. + * - For any other hcall failures, attempt a full flush once before + * resorting to BUG(). + * + * Note: This hcall is expected to fail only very rarely. The correct + * error recovery of killing the process/guest will be eventually + * needed. + */ +static inline long pseries_rpt_invalidate(u64 pid, u64 target, u64 type, + u64 page_sizes, u64 start, u64 end) +{ + long rc; + unsigned long all; + + while (true) { + rc = plpar_hcall_norets(H_RPT_INVALIDATE, pid, target, type, + page_sizes, start, end); + if (rc == H_BUSY) { + cpu_relax(); + continue; + } else if (rc == H_SUCCESS) + return rc; + + /* Flush request failed, try with a full flush once */ + if (type & H_RPTI_TYPE_NESTED) + all = H_RPTI_TYPE_NESTED | H_RPTI_TYPE_NESTED_ALL; + else + all = H_RPTI_TYPE_ALL; +retry: + rc = plpar_hcall_norets(H_RPT_INVALIDATE, pid, target, + all, page_sizes, 0, -1UL); + if (rc == H_BUSY) { + cpu_relax(); + goto retry; + } else if (rc == H_SUCCESS) + return rc; + + BUG(); + } +} + +#else /* !CONFIG_PPC_PSERIES */ + +static inline long plpar_set_ciabr(unsigned long ciabr) +{ + return 0; +} + +static inline long plpar_pte_read_4(unsigned long flags, unsigned long ptex, + unsigned long *ptes) +{ + return 0; +} + +static inline long pseries_rpt_invalidate(u64 pid, u64 target, u64 type, + u64 page_sizes, u64 start, u64 end) +{ + return 0; +} + +static inline long plpar_guest_create_vcpu(unsigned long flags, + unsigned long guest_id, + unsigned long vcpu_id) +{ + return 0; +} + +static inline long plpar_guest_get_state(unsigned long flags, + unsigned long guest_id, + unsigned long vcpu_id, + unsigned long data_buffer, + unsigned long data_size, + unsigned long *failed_index) +{ + return 0; +} + +static inline long plpar_guest_set_state(unsigned long flags, + unsigned long guest_id, + unsigned long vcpu_id, + unsigned long data_buffer, + unsigned long data_size, + unsigned long *failed_index) +{ + return 0; +} + +static inline long plpar_guest_run_vcpu(unsigned long flags, unsigned long guest_id, + unsigned long vcpu_id, int *trap, + unsigned long *failed_index) +{ + return 0; +} + +static inline long plpar_guest_create(unsigned long flags, unsigned long *guest_id) +{ + return 0; +} + +static inline long plpar_guest_delete(unsigned long flags, u64 guest_id) +{ + return 0; +} + +static inline long plpar_guest_get_capabilities(unsigned long flags, + unsigned long *capabilities) +{ + return 0; +} + +static inline long plpar_guest_set_capabilities(unsigned long flags, + unsigned long capabilities) +{ + return 0; +} + +#endif /* CONFIG_PPC_PSERIES */ + +#endif /* _ASM_POWERPC_PLPAR_WRAPPERS_H */ diff --git a/arch/powerpc/include/asm/plpks.h b/arch/powerpc/include/asm/plpks.h new file mode 100644 index 000000000000..7a84069759b0 --- /dev/null +++ b/arch/powerpc/include/asm/plpks.h @@ -0,0 +1,194 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2022 IBM Corporation + * Author: Nayna Jain <nayna@linux.ibm.com> + * + * Platform keystore for pseries LPAR(PLPKS). + */ + +#ifndef _ASM_POWERPC_PLPKS_H +#define _ASM_POWERPC_PLPKS_H + +#ifdef CONFIG_PSERIES_PLPKS + +#include <linux/types.h> +#include <linux/list.h> + +// Object policy flags from supported_policies +#define PLPKS_OSSECBOOTAUDIT PPC_BIT32(1) // OS secure boot must be audit/enforce +#define PLPKS_OSSECBOOTENFORCE PPC_BIT32(2) // OS secure boot must be enforce +#define PLPKS_PWSET PPC_BIT32(3) // No access without password set +#define PLPKS_WORLDREADABLE PPC_BIT32(4) // Readable without authentication +#define PLPKS_IMMUTABLE PPC_BIT32(5) // Once written, object cannot be removed +#define PLPKS_TRANSIENT PPC_BIT32(6) // Object does not persist through reboot +#define PLPKS_SIGNEDUPDATE PPC_BIT32(7) // Object can only be modified by signed updates +#define PLPKS_HVPROVISIONED PPC_BIT32(28) // Hypervisor has provisioned this object + +// Signature algorithm flags from signed_update_algorithms +#define PLPKS_ALG_RSA2048 PPC_BIT(0) +#define PLPKS_ALG_RSA4096 PPC_BIT(1) + +// Object label OS metadata flags +#define PLPKS_VAR_LINUX 0x02 +#define PLPKS_VAR_COMMON 0x04 + +// Flags for which consumer owns an object is owned by +#define PLPKS_FW_OWNER 0x1 +#define PLPKS_BOOTLOADER_OWNER 0x2 +#define PLPKS_OS_OWNER 0x3 + +// Flags for label metadata fields +#define PLPKS_LABEL_VERSION 0 +#define PLPKS_MAX_LABEL_ATTR_SIZE 16 +#define PLPKS_MAX_NAME_SIZE 239 +#define PLPKS_MAX_DATA_SIZE 4000 + +// Timeouts for PLPKS operations +#define PLPKS_MAX_TIMEOUT (5 * USEC_PER_SEC) +#define PLPKS_FLUSH_SLEEP 10000 // usec + +struct plpks_var { + char *component; + u8 *name; + u8 *data; + u32 policy; + u16 namelen; + u16 datalen; + u8 os; +}; + +struct plpks_var_name { + u8 *name; + u16 namelen; +}; + +struct plpks_var_name_list { + u32 varcount; + struct plpks_var_name varlist[]; +}; + +/** + * Updates the authenticated variable. It expects NULL as the component. + */ +int plpks_signed_update_var(struct plpks_var *var, u64 flags); + +/** + * Writes the specified var and its data to PKS. + * Any caller of PKS driver should present a valid component type for + * their variable. + */ +int plpks_write_var(struct plpks_var var); + +/** + * Removes the specified var and its data from PKS. + */ +int plpks_remove_var(char *component, u8 varos, + struct plpks_var_name vname); + +/** + * Returns the data for the specified os variable. + * + * Caller must allocate a buffer in var->data with length in var->datalen. + * If no buffer is provided, var->datalen will be populated with the object's + * size. + */ +int plpks_read_os_var(struct plpks_var *var); + +/** + * Returns the data for the specified firmware variable. + * + * Caller must allocate a buffer in var->data with length in var->datalen. + * If no buffer is provided, var->datalen will be populated with the object's + * size. + */ +int plpks_read_fw_var(struct plpks_var *var); + +/** + * Returns the data for the specified bootloader variable. + * + * Caller must allocate a buffer in var->data with length in var->datalen. + * If no buffer is provided, var->datalen will be populated with the object's + * size. + */ +int plpks_read_bootloader_var(struct plpks_var *var); + +/** + * Returns if PKS is available on this LPAR. + */ +bool plpks_is_available(void); + +/** + * Returns version of the Platform KeyStore. + */ +u8 plpks_get_version(void); + +/** + * Returns hypervisor storage overhead per object, not including the size of + * the object or label. Only valid for config version >= 2 + */ +u16 plpks_get_objoverhead(void); + +/** + * Returns maximum password size. Must be >= 32 bytes + */ +u16 plpks_get_maxpwsize(void); + +/** + * Returns maximum object size supported by Platform KeyStore. + */ +u16 plpks_get_maxobjectsize(void); + +/** + * Returns maximum object label size supported by Platform KeyStore. + */ +u16 plpks_get_maxobjectlabelsize(void); + +/** + * Returns total size of the configured Platform KeyStore. + */ +u32 plpks_get_totalsize(void); + +/** + * Returns used space from the total size of the Platform KeyStore. + */ +u32 plpks_get_usedspace(void); + +/** + * Returns bitmask of policies supported by the hypervisor. + */ +u32 plpks_get_supportedpolicies(void); + +/** + * Returns maximum byte size of a single object supported by the hypervisor. + * Only valid for config version >= 3 + */ +u32 plpks_get_maxlargeobjectsize(void); + +/** + * Returns bitmask of signature algorithms supported for signed updates. + * Only valid for config version >= 3 + */ +u64 plpks_get_signedupdatealgorithms(void); + +/** + * Returns the length of the PLPKS password in bytes. + */ +u16 plpks_get_passwordlen(void); + +/** + * Called in early init to retrieve and clear the PLPKS password from the DT. + */ +void plpks_early_init_devtree(void); + +/** + * Populates the FDT with the PLPKS password to prepare for kexec. + */ +int plpks_populate_fdt(void *fdt); +#else // CONFIG_PSERIES_PLPKS +static inline bool plpks_is_available(void) { return false; } +static inline u16 plpks_get_passwordlen(void) { BUILD_BUG(); } +static inline void plpks_early_init_devtree(void) { } +static inline int plpks_populate_fdt(void *fdt) { BUILD_BUG(); } +#endif // CONFIG_PSERIES_PLPKS + +#endif // _ASM_POWERPC_PLPKS_H diff --git a/arch/powerpc/include/asm/pmac_feature.h b/arch/powerpc/include/asm/pmac_feature.h index 10902c9375d0..420e2878ae67 100644 --- a/arch/powerpc/include/asm/pmac_feature.h +++ b/arch/powerpc/include/asm/pmac_feature.h @@ -46,7 +46,7 @@ /* PowerSurge are the first generation of PCI Pmacs. This include * all of the Grand-Central based machines. We currently don't - * differenciate most of them. + * differentiate most of them. */ #define PMAC_TYPE_PSURGE 0x10 /* PowerSurge */ #define PMAC_TYPE_ANS 0x11 /* Apple Network Server */ @@ -192,7 +192,7 @@ static inline long pmac_call_feature(int selector, struct device_node* node, /* PMAC_FTR_BMAC_ENABLE (struct device_node* node, 0, int value) * enable/disable the bmac (ethernet) cell of a mac-io ASIC, also drive - * it's reset line + * its reset line */ #define PMAC_FTR_BMAC_ENABLE PMAC_FTR_DEF(6) @@ -210,7 +210,7 @@ static inline long pmac_call_feature(int selector, struct device_node* node, /* PMAC_FTR_SOUND_CHIP_ENABLE (struct device_node* node, 0, int value) * enable/disable the sound chip, whatever it is and provided it can - * acually be controlled + * actually be controlled */ #define PMAC_FTR_SOUND_CHIP_ENABLE PMAC_FTR_DEF(9) @@ -401,5 +401,17 @@ extern u32 __iomem *uninorth_base; */ extern int pmac_get_uninorth_variant(void); +/* + * Power macintoshes have either a CUDA, PMU or SMU controlling + * system reset, power, NVRAM, RTC. + */ +typedef enum sys_ctrler_kind { + SYS_CTRLER_UNKNOWN = 0, + SYS_CTRLER_CUDA = 1, + SYS_CTRLER_PMU = 2, + SYS_CTRLER_SMU = 3, +} sys_ctrler_t; +extern sys_ctrler_t sys_ctrler; + #endif /* __ASM_POWERPC_PMAC_FEATURE_H */ #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/pmac_low_i2c.h b/arch/powerpc/include/asm/pmac_low_i2c.h index 01d71826d92f..21bd7297c87f 100644 --- a/arch/powerpc/include/asm/pmac_low_i2c.h +++ b/arch/powerpc/include/asm/pmac_low_i2c.h @@ -1,13 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * include/asm-ppc/pmac_low_i2c.h * * Copyright (C) 2003 Ben. Herrenschmidt (benh@kernel.crashing.org) - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * */ #ifndef __PMAC_LOW_I2C_H__ #define __PMAC_LOW_I2C_H__ diff --git a/arch/powerpc/include/asm/pmac_pfunc.h b/arch/powerpc/include/asm/pmac_pfunc.h index 1330d6a58c57..cee4e9f5b8cf 100644 --- a/arch/powerpc/include/asm/pmac_pfunc.h +++ b/arch/powerpc/include/asm/pmac_pfunc.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __PMAC_PFUNC_H__ #define __PMAC_PFUNC_H__ @@ -244,6 +245,7 @@ extern void pmf_put_function(struct pmf_function *func); extern int pmf_call_one(struct pmf_function *func, struct pmf_args *args); +int pmac_pfunc_base_install(void); /* Suspend/resume code called by via-pmu directly for now */ extern void pmac_pfunc_base_suspend(void); diff --git a/arch/powerpc/include/asm/pmc.h b/arch/powerpc/include/asm/pmc.h index 5a9ede4962cb..3c09109e708e 100644 --- a/arch/powerpc/include/asm/pmc.h +++ b/arch/powerpc/include/asm/pmc.h @@ -1,20 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * pmc.h * Copyright (C) 2004 David Gibson, IBM Corporation - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef _POWERPC_PMC_H #define _POWERPC_PMC_H @@ -31,12 +18,29 @@ void ppc_enable_pmcs(void); #ifdef CONFIG_PPC_BOOK3S_64 #include <asm/lppaca.h> +#include <asm/firmware.h> static inline void ppc_set_pmu_inuse(int inuse) { - get_lppaca()->pmcregs_in_use = inuse; +#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE) + if (firmware_has_feature(FW_FEATURE_LPAR)) { +#ifdef CONFIG_PPC_PSERIES + get_lppaca()->pmcregs_in_use = inuse; +#endif + } +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE + get_paca()->pmcregs_in_use = inuse; +#endif +#endif } +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE +static inline int ppc_get_pmu_inuse(void) +{ + return get_paca()->pmcregs_in_use; +} +#endif + extern void power4_enable_pmcs(void); #else /* CONFIG_PPC64 */ diff --git a/arch/powerpc/include/asm/pmi.h b/arch/powerpc/include/asm/pmi.h deleted file mode 100644 index b4e91fbf5081..000000000000 --- a/arch/powerpc/include/asm/pmi.h +++ /dev/null @@ -1,66 +0,0 @@ -#ifndef _POWERPC_PMI_H -#define _POWERPC_PMI_H - -/* - * Definitions for talking with PMI device on PowerPC - * - * PMI (Platform Management Interrupt) is a way to communicate - * with the BMC (Baseboard Management Controller) via interrupts. - * Unlike IPMI it is bidirectional and has a low latency. - * - * (C) Copyright IBM Deutschland Entwicklung GmbH 2005 - * - * Author: Christian Krafft <krafft@de.ibm.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2, or (at your option) - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#ifdef __KERNEL__ - -#define PMI_TYPE_FREQ_CHANGE 0x01 -#define PMI_TYPE_POWER_BUTTON 0x02 -#define PMI_READ_TYPE 0 -#define PMI_READ_DATA0 1 -#define PMI_READ_DATA1 2 -#define PMI_READ_DATA2 3 -#define PMI_WRITE_TYPE 4 -#define PMI_WRITE_DATA0 5 -#define PMI_WRITE_DATA1 6 -#define PMI_WRITE_DATA2 7 - -#define PMI_ACK 0x80 - -#define PMI_TIMEOUT 100 - -typedef struct { - u8 type; - u8 data0; - u8 data1; - u8 data2; -} pmi_message_t; - -struct pmi_handler { - struct list_head node; - u8 type; - void (*handle_pmi_message) (pmi_message_t); -}; - -int pmi_register_handler(struct pmi_handler *); -void pmi_unregister_handler(struct pmi_handler *); - -int pmi_send_message(pmi_message_t); - -#endif /* __KERNEL__ */ -#endif /* _POWERPC_PMI_H */ diff --git a/arch/powerpc/include/asm/pnv-ocxl.h b/arch/powerpc/include/asm/pnv-ocxl.h new file mode 100644 index 000000000000..9acd1fbf1197 --- /dev/null +++ b/arch/powerpc/include/asm/pnv-ocxl.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +// Copyright 2017 IBM Corp. +#ifndef _ASM_PNV_OCXL_H +#define _ASM_PNV_OCXL_H + +#include <linux/bitfield.h> +#include <linux/pci.h> + +#define PNV_OCXL_TL_MAX_TEMPLATE 63 +#define PNV_OCXL_TL_BITS_PER_RATE 4 +#define PNV_OCXL_TL_RATE_BUF_SIZE ((PNV_OCXL_TL_MAX_TEMPLATE+1) * PNV_OCXL_TL_BITS_PER_RATE / 8) + +#define PNV_OCXL_ATSD_TIMEOUT 1 + +/* TLB Management Instructions */ +#define PNV_OCXL_ATSD_LNCH 0x00 +/* Radix Invalidate */ +#define PNV_OCXL_ATSD_LNCH_R PPC_BIT(0) +/* Radix Invalidation Control + * 0b00 Just invalidate TLB. + * 0b01 Invalidate just Page Walk Cache. + * 0b10 Invalidate TLB, Page Walk Cache, and any + * caching of Partition and Process Table Entries. + */ +#define PNV_OCXL_ATSD_LNCH_RIC PPC_BITMASK(1, 2) +/* Number and Page Size of translations to be invalidated */ +#define PNV_OCXL_ATSD_LNCH_LP PPC_BITMASK(3, 10) +/* Invalidation Criteria + * 0b00 Invalidate just the target VA. + * 0b01 Invalidate matching PID. + */ +#define PNV_OCXL_ATSD_LNCH_IS PPC_BITMASK(11, 12) +/* 0b1: Process Scope, 0b0: Partition Scope */ +#define PNV_OCXL_ATSD_LNCH_PRS PPC_BIT(13) +/* Invalidation Flag */ +#define PNV_OCXL_ATSD_LNCH_B PPC_BIT(14) +/* Actual Page Size to be invalidated + * 000 4KB + * 101 64KB + * 001 2MB + * 010 1GB + */ +#define PNV_OCXL_ATSD_LNCH_AP PPC_BITMASK(15, 17) +/* Defines the large page select + * L=0b0 for 4KB pages + * L=0b1 for large pages) + */ +#define PNV_OCXL_ATSD_LNCH_L PPC_BIT(18) +/* Process ID */ +#define PNV_OCXL_ATSD_LNCH_PID PPC_BITMASK(19, 38) +/* NoFlush – Assumed to be 0b0 */ +#define PNV_OCXL_ATSD_LNCH_F PPC_BIT(39) +#define PNV_OCXL_ATSD_LNCH_OCAPI_SLBI PPC_BIT(40) +#define PNV_OCXL_ATSD_LNCH_OCAPI_SINGLETON PPC_BIT(41) +#define PNV_OCXL_ATSD_AVA 0x08 +#define PNV_OCXL_ATSD_AVA_AVA PPC_BITMASK(0, 51) +#define PNV_OCXL_ATSD_STAT 0x10 + +int pnv_ocxl_get_actag(struct pci_dev *dev, u16 *base, u16 *enabled, u16 *supported); +int pnv_ocxl_get_pasid_count(struct pci_dev *dev, int *count); + +int pnv_ocxl_get_tl_cap(struct pci_dev *dev, long *cap, + char *rate_buf, int rate_buf_size); +int pnv_ocxl_set_tl_conf(struct pci_dev *dev, long cap, + uint64_t rate_buf_phys, int rate_buf_size); + +int pnv_ocxl_get_xsl_irq(struct pci_dev *dev, int *hwirq); +void pnv_ocxl_unmap_xsl_regs(void __iomem *dsisr, void __iomem *dar, + void __iomem *tfc, void __iomem *pe_handle); +int pnv_ocxl_map_xsl_regs(struct pci_dev *dev, void __iomem **dsisr, + void __iomem **dar, void __iomem **tfc, + void __iomem **pe_handle); + +int pnv_ocxl_spa_setup(struct pci_dev *dev, void *spa_mem, int PE_mask, void **platform_data); +void pnv_ocxl_spa_release(void *platform_data); +int pnv_ocxl_spa_remove_pe_from_cache(void *platform_data, int pe_handle); + +int pnv_ocxl_map_lpar(struct pci_dev *dev, uint64_t lparid, + uint64_t lpcr, void __iomem **arva); +void pnv_ocxl_unmap_lpar(void __iomem *arva); +void pnv_ocxl_tlb_invalidate(void __iomem *arva, + unsigned long pid, + unsigned long addr, + unsigned long page_size); +#endif /* _ASM_PNV_OCXL_H */ diff --git a/arch/powerpc/include/asm/pnv-pci.h b/arch/powerpc/include/asm/pnv-pci.h new file mode 100644 index 000000000000..7e9a479951a3 --- /dev/null +++ b/arch/powerpc/include/asm/pnv-pci.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright 2014 IBM Corp. + */ + +#ifndef _ASM_PNV_PCI_H +#define _ASM_PNV_PCI_H + +#include <linux/pci.h> +#include <linux/pci_hotplug.h> +#include <linux/irq.h> +#include <linux/of.h> +#include <asm/opal-api.h> + +#define PCI_SLOT_ID_PREFIX (1UL << 63) +#define PCI_SLOT_ID(phb_id, bdfn) \ + (PCI_SLOT_ID_PREFIX | ((uint64_t)(bdfn) << 16) | (phb_id)) +#define PCI_PHB_SLOT_ID(phb_id) (phb_id) + +extern int pnv_pci_get_slot_id(struct device_node *np, uint64_t *id); +extern int pnv_pci_get_device_tree(uint32_t phandle, void *buf, uint64_t len); +extern int pnv_pci_get_presence_state(uint64_t id, uint8_t *state); +extern int pnv_pci_get_power_state(uint64_t id, uint8_t *state); +extern int pnv_pci_set_power_state(uint64_t id, uint8_t state, + struct opal_msg *msg); + +int64_t pnv_opal_pci_msi_eoi(struct irq_data *d); +bool is_pnv_opal_msi(struct irq_chip *chip); + +struct pnv_php_slot { + struct hotplug_slot slot; + uint64_t id; + char *name; + int slot_no; + unsigned int flags; +#define PNV_PHP_FLAG_BROKEN_PDC 0x1 + struct kref kref; +#define PNV_PHP_STATE_INITIALIZED 0 +#define PNV_PHP_STATE_REGISTERED 1 +#define PNV_PHP_STATE_POPULATED 2 +#define PNV_PHP_STATE_OFFLINE 3 + int state; + int irq; + struct workqueue_struct *wq; + struct device_node *dn; + struct pci_dev *pdev; + struct pci_bus *bus; + bool power_state_check; + u8 attention_state; + void *fdt; + void *dt; + struct of_changeset ocs; + struct pnv_php_slot *parent; + struct list_head children; + struct list_head link; +}; +extern struct pnv_php_slot *pnv_php_find_slot(struct device_node *dn); +extern int pnv_php_set_slot_power_state(struct hotplug_slot *slot, + uint8_t state); + +#endif diff --git a/arch/powerpc/include/asm/powernv.h b/arch/powerpc/include/asm/powernv.h new file mode 100644 index 000000000000..e1a858718716 --- /dev/null +++ b/arch/powerpc/include/asm/powernv.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright 2017 IBM Corp. + */ + +#ifndef _ASM_POWERNV_H +#define _ASM_POWERNV_H + +#ifdef CONFIG_PPC_POWERNV +extern void powernv_set_nmmu_ptcr(unsigned long ptcr); + +void pnv_program_cpu_hotplug_lpcr(unsigned int cpu, u64 lpcr_val); + +void pnv_tm_init(void); +#else +static inline void powernv_set_nmmu_ptcr(unsigned long ptcr) { } + +static inline void pnv_tm_init(void) { } +#endif + +#endif /* _ASM_POWERNV_H */ diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h index eccfc161e58e..55ca49d18319 100644 --- a/arch/powerpc/include/asm/ppc-opcode.h +++ b/arch/powerpc/include/asm/ppc-opcode.h @@ -1,19 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Copyright 2009 Freescale Semiconductor, Inc. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * provides masks and opcode images for use by code generation, emulation * and for instructions that older assemblers might not know about */ #ifndef _ASM_POWERPC_PPC_OPCODE_H #define _ASM_POWERPC_PPC_OPCODE_H -#include <linux/stringify.h> -#include <asm/asm-compat.h> +#include <asm/asm-const.h> #define __REG_R0 0 #define __REG_R1 1 @@ -81,121 +76,257 @@ #define __REGA0_R30 30 #define __REGA0_R31 31 +/* For use with PPC_RAW_() macros */ +#define _R0 0 +#define _R1 1 +#define _R2 2 +#define _R3 3 +#define _R4 4 +#define _R5 5 +#define _R6 6 +#define _R7 7 +#define _R8 8 +#define _R9 9 +#define _R10 10 +#define _R11 11 +#define _R12 12 +#define _R13 13 +#define _R14 14 +#define _R15 15 +#define _R16 16 +#define _R17 17 +#define _R18 18 +#define _R19 19 +#define _R20 20 +#define _R21 21 +#define _R22 22 +#define _R23 23 +#define _R24 24 +#define _R25 25 +#define _R26 26 +#define _R27 27 +#define _R28 28 +#define _R29 29 +#define _R30 30 +#define _R31 31 + +#define IMM_L(i) ((uintptr_t)(i) & 0xffff) +#define IMM_DS(i) ((uintptr_t)(i) & 0xfffc) +#define IMM_DQ(i) ((uintptr_t)(i) & 0xfff0) +#define IMM_D0(i) (((uintptr_t)(i) >> 16) & 0x3ffff) +#define IMM_D1(i) IMM_L(i) + +/* + * 16-bit immediate helper macros: HA() is for use with sign-extending instrs + * (e.g. LD, ADDI). If the bottom 16 bits is "-ve", add another bit into the + * top half to negate the effect (i.e. 0xffff + 1 = 0x(1)0000). + * + * XXX: should these mask out possible sign bits? + */ +#define IMM_H(i) ((uintptr_t)(i)>>16) +#define IMM_HA(i) (((uintptr_t)(i)>>16) + \ + (((uintptr_t)(i) & 0x8000) >> 15)) + +/* + * 18-bit immediate helper for prefix 18-bit upper immediate si0 field. + */ +#define IMM_H18(i) (((uintptr_t)(i)>>16) & 0x3ffff) + + +/* opcode and xopcode for instructions */ +#define OP_PREFIX 1 +#define OP_TRAP_64 2 +#define OP_TRAP 3 +#define OP_SC 17 +#define OP_19 19 +#define OP_31 31 +#define OP_LWZ 32 +#define OP_LWZU 33 +#define OP_LBZ 34 +#define OP_LBZU 35 +#define OP_STW 36 +#define OP_STWU 37 +#define OP_STB 38 +#define OP_STBU 39 +#define OP_LHZ 40 +#define OP_LHZU 41 +#define OP_LHA 42 +#define OP_LHAU 43 +#define OP_STH 44 +#define OP_STHU 45 +#define OP_LMW 46 +#define OP_STMW 47 +#define OP_LFS 48 +#define OP_LFSU 49 +#define OP_LFD 50 +#define OP_LFDU 51 +#define OP_STFS 52 +#define OP_STFSU 53 +#define OP_STFD 54 +#define OP_STFDU 55 +#define OP_LQ 56 +#define OP_LD 58 +#define OP_STD 62 + +#define OP_19_XOP_RFID 18 +#define OP_19_XOP_RFMCI 38 +#define OP_19_XOP_RFDI 39 +#define OP_19_XOP_RFI 50 +#define OP_19_XOP_RFCI 51 +#define OP_19_XOP_RFSCV 82 +#define OP_19_XOP_HRFID 274 +#define OP_19_XOP_URFID 306 +#define OP_19_XOP_STOP 370 +#define OP_19_XOP_DOZE 402 +#define OP_19_XOP_NAP 434 +#define OP_19_XOP_SLEEP 466 +#define OP_19_XOP_RVWINKLE 498 + +#define OP_31_XOP_TRAP 4 +#define OP_31_XOP_LDX 21 +#define OP_31_XOP_LWZX 23 +#define OP_31_XOP_LDUX 53 +#define OP_31_XOP_DCBST 54 +#define OP_31_XOP_LWZUX 55 +#define OP_31_XOP_TRAP_64 68 +#define OP_31_XOP_DCBF 86 +#define OP_31_XOP_LBZX 87 +#define OP_31_XOP_STDX 149 +#define OP_31_XOP_STWX 151 +#define OP_31_XOP_STDUX 181 +#define OP_31_XOP_STWUX 183 +#define OP_31_XOP_STBX 215 +#define OP_31_XOP_LBZUX 119 +#define OP_31_XOP_STBUX 247 +#define OP_31_XOP_LHZX 279 +#define OP_31_XOP_LHZUX 311 +#define OP_31_XOP_MSGSNDP 142 +#define OP_31_XOP_MSGCLRP 174 +#define OP_31_XOP_MTMSR 146 +#define OP_31_XOP_MTMSRD 178 +#define OP_31_XOP_TLBIE 306 +#define OP_31_XOP_MFSPR 339 +#define OP_31_XOP_LWAX 341 +#define OP_31_XOP_LHAX 343 +#define OP_31_XOP_LWAUX 373 +#define OP_31_XOP_LHAUX 375 +#define OP_31_XOP_STHX 407 +#define OP_31_XOP_STHUX 439 +#define OP_31_XOP_MTSPR 467 +#define OP_31_XOP_DCBI 470 +#define OP_31_XOP_LDBRX 532 +#define OP_31_XOP_LWBRX 534 +#define OP_31_XOP_TLBSYNC 566 +#define OP_31_XOP_STDBRX 660 +#define OP_31_XOP_STWBRX 662 +#define OP_31_XOP_STFSX 663 +#define OP_31_XOP_STFSUX 695 +#define OP_31_XOP_STFDX 727 +#define OP_31_XOP_HASHCHK 754 +#define OP_31_XOP_STFDUX 759 +#define OP_31_XOP_LHBRX 790 +#define OP_31_XOP_LFIWAX 855 +#define OP_31_XOP_LFIWZX 887 +#define OP_31_XOP_STHBRX 918 +#define OP_31_XOP_STFIWX 983 + +/* VSX Scalar Load Instructions */ +#define OP_31_XOP_LXSDX 588 +#define OP_31_XOP_LXSSPX 524 +#define OP_31_XOP_LXSIWAX 76 +#define OP_31_XOP_LXSIWZX 12 + +/* VSX Scalar Store Instructions */ +#define OP_31_XOP_STXSDX 716 +#define OP_31_XOP_STXSSPX 652 +#define OP_31_XOP_STXSIWX 140 + +/* VSX Vector Load Instructions */ +#define OP_31_XOP_LXVD2X 844 +#define OP_31_XOP_LXVW4X 780 + +/* VSX Vector Load and Splat Instruction */ +#define OP_31_XOP_LXVDSX 332 + +/* VSX Vector Store Instructions */ +#define OP_31_XOP_STXVD2X 972 +#define OP_31_XOP_STXVW4X 908 + +#define OP_31_XOP_LFSX 535 +#define OP_31_XOP_LFSUX 567 +#define OP_31_XOP_LFDX 599 +#define OP_31_XOP_LFDUX 631 + +/* VMX Vector Load Instructions */ +#define OP_31_XOP_LVX 103 + +/* VMX Vector Store Instructions */ +#define OP_31_XOP_STVX 231 + /* sorted alphabetically */ -#define PPC_INST_BHRBE 0x7c00025c -#define PPC_INST_CLRBHRB 0x7c00035c +#define PPC_INST_BCCTR_FLUSH 0x4c400420 +#define PPC_INST_COPY 0x7c20060c #define PPC_INST_DCBA 0x7c0005ec #define PPC_INST_DCBA_MASK 0xfc0007fe -#define PPC_INST_DCBAL 0x7c2005ec -#define PPC_INST_DCBZL 0x7c2007ec -#define PPC_INST_ICBT 0x7c00002c +#define PPC_INST_DSSALL 0x7e00066c #define PPC_INST_ISEL 0x7c00001e #define PPC_INST_ISEL_MASK 0xfc00003e -#define PPC_INST_LDARX 0x7c0000a8 #define PPC_INST_LSWI 0x7c0004aa #define PPC_INST_LSWX 0x7c00042a -#define PPC_INST_LWARX 0x7c000028 #define PPC_INST_LWSYNC 0x7c2004ac -#define PPC_INST_LXVD2X 0x7c000698 +#define PPC_INST_SYNC 0x7c0004ac +#define PPC_INST_SYNC_MASK 0xfc0007fe #define PPC_INST_MCRXR 0x7c000400 #define PPC_INST_MCRXR_MASK 0xfc0007fe #define PPC_INST_MFSPR_PVR 0x7c1f42a6 -#define PPC_INST_MFSPR_PVR_MASK 0xfc1fffff -#define PPC_INST_MSGSND 0x7c00019c -#define PPC_INST_MSGSNDP 0x7c00011c -#define PPC_INST_NOP 0x60000000 +#define PPC_INST_MFSPR_PVR_MASK 0xfc1ffffe +#define PPC_INST_MTMSRD 0x7c000164 +#define PPC_INST_PASTE 0x7c20070d +#define PPC_INST_PASTE_MASK 0xfc2007ff #define PPC_INST_POPCNTB 0x7c0000f4 #define PPC_INST_POPCNTB_MASK 0xfc0007fe -#define PPC_INST_POPCNTD 0x7c0003f4 -#define PPC_INST_POPCNTW 0x7c0002f4 -#define PPC_INST_RFCI 0x4c000066 -#define PPC_INST_RFDI 0x4c00004e -#define PPC_INST_RFMCI 0x4c00004c +#define PPC_INST_RFEBB 0x4c000124 +#define PPC_INST_RFID 0x4c000024 #define PPC_INST_MFSPR_DSCR 0x7c1102a6 -#define PPC_INST_MFSPR_DSCR_MASK 0xfc1fffff +#define PPC_INST_MFSPR_DSCR_MASK 0xfc1ffffe #define PPC_INST_MTSPR_DSCR 0x7c1103a6 -#define PPC_INST_MTSPR_DSCR_MASK 0xfc1fffff +#define PPC_INST_MTSPR_DSCR_MASK 0xfc1ffffe #define PPC_INST_MFSPR_DSCR_USER 0x7c0302a6 -#define PPC_INST_MFSPR_DSCR_USER_MASK 0xfc1fffff +#define PPC_INST_MFSPR_DSCR_USER_MASK 0xfc1ffffe #define PPC_INST_MTSPR_DSCR_USER 0x7c0303a6 -#define PPC_INST_MTSPR_DSCR_USER_MASK 0xfc1fffff -#define PPC_INST_SLBFEE 0x7c0007a7 - +#define PPC_INST_MTSPR_DSCR_USER_MASK 0xfc1ffffe #define PPC_INST_STRING 0x7c00042a #define PPC_INST_STRING_MASK 0xfc0007fe #define PPC_INST_STRING_GEN_MASK 0xfc00067e - #define PPC_INST_STSWI 0x7c0005aa #define PPC_INST_STSWX 0x7c00052a -#define PPC_INST_STXVD2X 0x7c000798 -#define PPC_INST_TLBIE 0x7c000264 -#define PPC_INST_TLBILX 0x7c000024 -#define PPC_INST_WAIT 0x7c00007c -#define PPC_INST_TLBIVAX 0x7c000624 -#define PPC_INST_TLBSRX_DOT 0x7c0006a5 -#define PPC_INST_XXLOR 0xf0000510 -#define PPC_INST_XVCPSGNDP 0xf0000780 #define PPC_INST_TRECHKPT 0x7c0007dd #define PPC_INST_TRECLAIM 0x7c00075d -#define PPC_INST_TABORT 0x7c00071d - -#define PPC_INST_NAP 0x4c000364 -#define PPC_INST_SLEEP 0x4c0003a4 - -/* A2 specific instructions */ -#define PPC_INST_ERATWE 0x7c0001a6 -#define PPC_INST_ERATRE 0x7c000166 -#define PPC_INST_ERATILX 0x7c000066 -#define PPC_INST_ERATIVAX 0x7c000666 -#define PPC_INST_ERATSX 0x7c000126 -#define PPC_INST_ERATSX_DOT 0x7c000127 - -/* Misc instructions for BPF compiler */ -#define PPC_INST_LD 0xe8000000 -#define PPC_INST_LHZ 0xa0000000 -#define PPC_INST_LWZ 0x80000000 -#define PPC_INST_STD 0xf8000000 -#define PPC_INST_STDU 0xf8000001 -#define PPC_INST_MFLR 0x7c0802a6 -#define PPC_INST_MTLR 0x7c0803a6 -#define PPC_INST_CMPWI 0x2c000000 -#define PPC_INST_CMPDI 0x2c200000 -#define PPC_INST_CMPLW 0x7c000040 -#define PPC_INST_CMPLWI 0x28000000 -#define PPC_INST_ADDI 0x38000000 -#define PPC_INST_ADDIS 0x3c000000 -#define PPC_INST_ADD 0x7c000214 -#define PPC_INST_SUB 0x7c000050 -#define PPC_INST_BLR 0x4e800020 -#define PPC_INST_BLRL 0x4e800021 -#define PPC_INST_MULLW 0x7c0001d6 -#define PPC_INST_MULHWU 0x7c000016 -#define PPC_INST_MULLI 0x1c000000 -#define PPC_INST_DIVWU 0x7c0003d6 -#define PPC_INST_RLWINM 0x54000000 -#define PPC_INST_RLDICR 0x78000004 -#define PPC_INST_SLW 0x7c000030 -#define PPC_INST_SRW 0x7c000430 -#define PPC_INST_AND 0x7c000038 -#define PPC_INST_ANDDOT 0x7c000039 -#define PPC_INST_OR 0x7c000378 -#define PPC_INST_XOR 0x7c000278 -#define PPC_INST_ANDI 0x70000000 -#define PPC_INST_ORI 0x60000000 -#define PPC_INST_ORIS 0x64000000 -#define PPC_INST_XORI 0x68000000 -#define PPC_INST_XORIS 0x6c000000 -#define PPC_INST_NEG 0x7c0000d0 -#define PPC_INST_BRANCH 0x48000000 +#define PPC_INST_TSR 0x7c0005dd #define PPC_INST_BRANCH_COND 0x40800000 -#define PPC_INST_LBZCIX 0x7c0006aa -#define PPC_INST_STBCIX 0x7c0007aa + +/* Prefixes */ +#define PPC_INST_LFS 0xc0000000 +#define PPC_INST_STFS 0xd0000000 +#define PPC_INST_LFD 0xc8000000 +#define PPC_INST_STFD 0xd8000000 +#define PPC_PREFIX_MLS 0x06000000 +#define PPC_PREFIX_8LS 0x04000000 + +/* Prefixed instructions */ +#define PPC_INST_PADDI 0x38000000 +#define PPC_INST_PLD 0xe4000000 +#define PPC_INST_PSTD 0xf4000000 /* macros to insert fields into opcodes */ #define ___PPC_RA(a) (((a) & 0x1f) << 16) #define ___PPC_RB(b) (((b) & 0x1f) << 11) +#define ___PPC_RC(c) (((c) & 0x1f) << 6) #define ___PPC_RS(s) (((s) & 0x1f) << 21) #define ___PPC_RT(t) ___PPC_RS(t) +#define ___PPC_R(r) (((r) & 0x1) << 16) +#define ___PPC_PRS(prs) (((prs) & 0x1) << 17) +#define ___PPC_RIC(ric) (((ric) & 0x3) << 18) #define __PPC_RA(a) ___PPC_RA(__REG_##a) #define __PPC_RA0(a) ___PPC_RA(__REGA0_##a) #define __PPC_RB(b) ___PPC_RB(__REG_##b) @@ -205,115 +336,388 @@ #define __PPC_XB(b) ((((b) & 0x1f) << 11) | (((b) & 0x20) >> 4)) #define __PPC_XS(s) ((((s) & 0x1f) << 21) | (((s) & 0x20) >> 5)) #define __PPC_XT(s) __PPC_XS(s) +#define __PPC_XSP(s) ((((s) & 0x1e) | (((s) >> 5) & 0x1)) << 21) +#define __PPC_XTP(s) __PPC_XSP(s) #define __PPC_T_TLB(t) (((t) & 0x3) << 21) +#define __PPC_PL(p) (((p) & 0x3) << 16) #define __PPC_WC(w) (((w) & 0x3) << 21) #define __PPC_WS(w) (((w) & 0x1f) << 11) #define __PPC_SH(s) __PPC_WS(s) -#define __PPC_MB(s) (((s) & 0x1f) << 6) +#define __PPC_SH64(s) (__PPC_SH(s) | (((s) & 0x20) >> 4)) +#define __PPC_MB(s) ___PPC_RC(s) #define __PPC_ME(s) (((s) & 0x1f) << 1) +#define __PPC_MB64(s) (__PPC_MB(s) | ((s) & 0x20)) +#define __PPC_ME64(s) __PPC_MB64(s) #define __PPC_BI(s) (((s) & 0x1f) << 16) #define __PPC_CT(t) (((t) & 0x0f) << 21) +#define __PPC_SPR(r) ((((r) & 0x1f) << 16) | ((((r) >> 5) & 0x1f) << 11)) +#define __PPC_RC21 (0x1 << 10) +#define __PPC_PRFX_R(r) (((r) & 0x1) << 20) +#define __PPC_EH(eh) (((eh) & 0x1) << 0) /* - * Only use the larx hint bit on 64bit CPUs. e500v1/v2 based CPUs will treat a - * larx with EH set as an illegal instruction. + * Both low and high 16 bits are added as SIGNED additions, so if low 16 bits + * has high bit set, high 16 bits must be adjusted. These macros do that (stolen + * from binutils). */ -#ifdef CONFIG_PPC64 -#define __PPC_EH(eh) (((eh) & 0x1) << 0) +#define PPC_LO(v) ((v) & 0xffff) +#define PPC_HI(v) (((v) >> 16) & 0xffff) +#define PPC_HA(v) PPC_HI((v) + 0x8000) +#define PPC_HIGHER(v) (((v) >> 32) & 0xffff) +#define PPC_HIGHEST(v) (((v) >> 48) & 0xffff) + +/* LI Field */ +#define PPC_LI_MASK 0x03fffffc +#define PPC_LI(v) ((v) & PPC_LI_MASK) + +/* Base instruction encoding */ +#define PPC_RAW_CP_ABORT (0x7c00068c) +#define PPC_RAW_COPY(a, b) (PPC_INST_COPY | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_DARN(t, l) (0x7c0005e6 | ___PPC_RT(t) | (((l) & 0x3) << 16)) +#define PPC_RAW_DCBAL(a, b) (0x7c2005ec | __PPC_RA(a) | __PPC_RB(b)) +#define PPC_RAW_DCBZL(a, b) (0x7c2007ec | __PPC_RA(a) | __PPC_RB(b)) +#define PPC_RAW_LQARX(t, a, b, eh) (0x7c000228 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | __PPC_EH(eh)) +#define PPC_RAW_LDARX(t, a, b, eh) (0x7c0000a8 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | __PPC_EH(eh)) +#define PPC_RAW_LWARX(t, a, b, eh) (0x7c000028 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | __PPC_EH(eh)) +#define PPC_RAW_PHWSYNC (0x7c8004ac) +#define PPC_RAW_PLWSYNC (0x7ca004ac) +#define PPC_RAW_STQCX(t, a, b) (0x7c00016d | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_MADDHD(t, a, b, c) (0x10000030 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | ___PPC_RC(c)) +#define PPC_RAW_MADDHDU(t, a, b, c) (0x10000031 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | ___PPC_RC(c)) +#define PPC_RAW_MADDLD(t, a, b, c) (0x10000033 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | ___PPC_RC(c)) +#define PPC_RAW_MSGSND(b) (0x7c00019c | ___PPC_RB(b)) +#define PPC_RAW_MSGSYNC (0x7c0006ec) +#define PPC_RAW_MSGCLR(b) (0x7c0001dc | ___PPC_RB(b)) +#define PPC_RAW_MSGSNDP(b) (0x7c00011c | ___PPC_RB(b)) +#define PPC_RAW_MSGCLRP(b) (0x7c00015c | ___PPC_RB(b)) +#define PPC_RAW_PASTE(a, b) (0x7c20070d | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_POPCNTB(a, s) (PPC_INST_POPCNTB | __PPC_RA(a) | __PPC_RS(s)) +#define PPC_RAW_POPCNTD(a, s) (0x7c0003f4 | __PPC_RA(a) | __PPC_RS(s)) +#define PPC_RAW_POPCNTW(a, s) (0x7c0002f4 | __PPC_RA(a) | __PPC_RS(s)) +#define PPC_RAW_RFCI (0x4c000066) +#define PPC_RAW_RFDI (0x4c00004e) +#define PPC_RAW_RFMCI (0x4c00004c) +#define PPC_RAW_TLBILX_LPID (0x7c000024) +#define PPC_RAW_TLBILX(t, a, b) (0x7c000024 | __PPC_T_TLB(t) | __PPC_RA0(a) | __PPC_RB(b)) +#define PPC_RAW_WAIT_v203 (0x7c00007c) +#define PPC_RAW_WAIT(w, p) (0x7c00003c | __PPC_WC(w) | __PPC_PL(p)) +#define PPC_RAW_TLBIE(lp, a) (0x7c000264 | ___PPC_RB(a) | ___PPC_RS(lp)) +#define PPC_RAW_TLBIE_5(rb, rs, ric, prs, r) \ + (0x7c000264 | ___PPC_RB(rb) | ___PPC_RS(rs) | ___PPC_RIC(ric) | ___PPC_PRS(prs) | ___PPC_R(r)) +#define PPC_RAW_TLBIEL(rb, rs, ric, prs, r) \ + (0x7c000224 | ___PPC_RB(rb) | ___PPC_RS(rs) | ___PPC_RIC(ric) | ___PPC_PRS(prs) | ___PPC_R(r)) +#define PPC_RAW_TLBIEL_v205(rb, l) (0x7c000224 | ___PPC_RB(rb) | (l << 21)) +#define PPC_RAW_TLBSRX_DOT(a, b) (0x7c0006a5 | __PPC_RA0(a) | __PPC_RB(b)) +#define PPC_RAW_TLBIVAX(a, b) (0x7c000624 | __PPC_RA0(a) | __PPC_RB(b)) +#define PPC_RAW_ERATWE(s, a, w) (0x7c0001a6 | __PPC_RS(s) | __PPC_RA(a) | __PPC_WS(w)) +#define PPC_RAW_ERATRE(s, a, w) (0x7c000166 | __PPC_RS(s) | __PPC_RA(a) | __PPC_WS(w)) +#define PPC_RAW_ERATILX(t, a, b) (0x7c000066 | __PPC_T_TLB(t) | __PPC_RA0(a) | __PPC_RB(b)) +#define PPC_RAW_ERATIVAX(s, a, b) (0x7c000666 | __PPC_RS(s) | __PPC_RA0(a) | __PPC_RB(b)) +#define PPC_RAW_ERATSX(t, a, w) (0x7c000126 | __PPC_RS(t) | __PPC_RA0(a) | __PPC_RB(b)) +#define PPC_RAW_ERATSX_DOT(t, a, w) (0x7c000127 | __PPC_RS(t) | __PPC_RA0(a) | __PPC_RB(b)) +#define PPC_RAW_SLBFEE_DOT(t, b) (0x7c0007a7 | __PPC_RT(t) | __PPC_RB(b)) +#define __PPC_RAW_SLBFEE_DOT(t, b) (0x7c0007a7 | ___PPC_RT(t) | ___PPC_RB(b)) +#define PPC_RAW_ICBT(c, a, b) (0x7c00002c | __PPC_CT(c) | __PPC_RA0(a) | __PPC_RB(b)) +#define PPC_RAW_LBZCIX(t, a, b) (0x7c0006aa | __PPC_RT(t) | __PPC_RA(a) | __PPC_RB(b)) +#define PPC_RAW_STBCIX(s, a, b) (0x7c0007aa | __PPC_RS(s) | __PPC_RA(a) | __PPC_RB(b)) +#define PPC_RAW_DCBFPS(a, b) (0x7c0000ac | ___PPC_RA(a) | ___PPC_RB(b) | (4 << 21)) +#define PPC_RAW_DCBSTPS(a, b) (0x7c0000ac | ___PPC_RA(a) | ___PPC_RB(b) | (6 << 21)) +#define PPC_RAW_SC() (0x44000002) +#define PPC_RAW_SYNC() (0x7c0004ac) +#define PPC_RAW_ISYNC() (0x4c00012c) +#define PPC_RAW_LWSYNC() (0x7c2004ac) + +/* + * Define what the VSX XX1 form instructions will look like, then add + * the 128 bit load store instructions based on that. + */ +#define VSX_XX1(s, a, b) (__PPC_XS(s) | __PPC_RA(a) | __PPC_RB(b)) +#define VSX_XX3(t, a, b) (__PPC_XT(t) | __PPC_XA(a) | __PPC_XB(b)) +#define PPC_RAW_STXVD2X(s, a, b) (0x7c000798 | VSX_XX1((s), a, b)) +#define PPC_RAW_LXVD2X(s, a, b) (0x7c000698 | VSX_XX1((s), a, b)) +#define PPC_RAW_MFVRD(a, t) (0x7c000066 | VSX_XX1((t) + 32, a, R0)) +#define PPC_RAW_MTVRD(t, a) (0x7c000166 | VSX_XX1((t) + 32, a, R0)) +#define PPC_RAW_VPMSUMW(t, a, b) (0x10000488 | VSX_XX3((t), a, b)) +#define PPC_RAW_VPMSUMD(t, a, b) (0x100004c8 | VSX_XX3((t), a, b)) +#define PPC_RAW_XXLOR(t, a, b) (0xf0000490 | VSX_XX3((t), a, b)) +#define PPC_RAW_XXSWAPD(t, a) (0xf0000250 | VSX_XX3((t), a, a)) +#define PPC_RAW_XVCPSGNDP(t, a, b) ((0xf0000780 | VSX_XX3((t), (a), (b)))) +#define PPC_RAW_VPERMXOR(vrt, vra, vrb, vrc) \ + ((0x1000002d | ___PPC_RT(vrt) | ___PPC_RA(vra) | ___PPC_RB(vrb) | (((vrc) & 0x1f) << 6))) +#define PPC_RAW_LXVP(xtp, a, i) (0x18000000 | __PPC_XTP(xtp) | ___PPC_RA(a) | IMM_DQ(i)) +#define PPC_RAW_STXVP(xsp, a, i) (0x18000001 | __PPC_XSP(xsp) | ___PPC_RA(a) | IMM_DQ(i)) +#define PPC_RAW_LXVPX(xtp, a, b) (0x7c00029a | __PPC_XTP(xtp) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_STXVPX(xsp, a, b) (0x7c00039a | __PPC_XSP(xsp) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_PLXVP_P(xtp, i, a, pr) (PPC_PREFIX_8LS | __PPC_PRFX_R(pr) | IMM_D0(i)) +#define PPC_RAW_PLXVP_S(xtp, i, a, pr) (0xe8000000 | __PPC_XTP(xtp) | ___PPC_RA(a) | IMM_D1(i)) +#define PPC_RAW_PSTXVP_P(xsp, i, a, pr) (PPC_PREFIX_8LS | __PPC_PRFX_R(pr) | IMM_D0(i)) +#define PPC_RAW_PSTXVP_S(xsp, i, a, pr) (0xf8000000 | __PPC_XSP(xsp) | ___PPC_RA(a) | IMM_D1(i)) +#define PPC_RAW_NAP (0x4c000364) +#define PPC_RAW_SLEEP (0x4c0003a4) +#define PPC_RAW_WINKLE (0x4c0003e4) +#define PPC_RAW_STOP (0x4c0002e4) +#define PPC_RAW_CLRBHRB (0x7c00035c) +#define PPC_RAW_MFBHRBE(r, n) (0x7c00025c | __PPC_RT(r) | (((n) & 0x3ff) << 11)) +#define PPC_RAW_TRECHKPT (PPC_INST_TRECHKPT) +#define PPC_RAW_TRECLAIM(r) (PPC_INST_TRECLAIM | __PPC_RA(r)) +#define PPC_RAW_TABORT(r) (0x7c00071d | __PPC_RA(r)) +#define TMRN(x) ((((x) & 0x1f) << 16) | (((x) & 0x3e0) << 6)) +#define PPC_RAW_MTTMR(tmr, r) (0x7c0003dc | TMRN(tmr) | ___PPC_RS(r)) +#define PPC_RAW_MFTMR(tmr, r) (0x7c0002dc | TMRN(tmr) | ___PPC_RT(r)) +#define PPC_RAW_ICSWX(s, a, b) (0x7c00032d | ___PPC_RS(s) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_ICSWEPX(s, a, b) (0x7c00076d | ___PPC_RS(s) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_SLBIA(IH) (0x7c0003e4 | (((IH) & 0x7) << 21)) +#define PPC_RAW_VCMPEQUD_RC(vrt, vra, vrb) \ + (0x100000c7 | ___PPC_RT(vrt) | ___PPC_RA(vra) | ___PPC_RB(vrb) | __PPC_RC21) +#define PPC_RAW_VCMPEQUB_RC(vrt, vra, vrb) \ + (0x10000006 | ___PPC_RT(vrt) | ___PPC_RA(vra) | ___PPC_RB(vrb) | __PPC_RC21) +#define PPC_RAW_LD(r, base, i) (0xe8000000 | ___PPC_RT(r) | ___PPC_RA(base) | IMM_DS(i)) +#define PPC_RAW_LWA(r, base, i) (0xe8000002 | ___PPC_RT(r) | ___PPC_RA(base) | IMM_DS(i)) +#define PPC_RAW_LWZ(r, base, i) (0x80000000 | ___PPC_RT(r) | ___PPC_RA(base) | IMM_L(i)) +#define PPC_RAW_LWZX(t, a, b) (0x7c00002e | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_STD(r, base, i) (0xf8000000 | ___PPC_RS(r) | ___PPC_RA(base) | IMM_DS(i)) +#define PPC_RAW_STDCX(s, a, b) (0x7c0001ad | ___PPC_RS(s) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_LFSX(t, a, b) (0x7c00042e | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_STFSX(s, a, b) (0x7c00052e | ___PPC_RS(s) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_LFDX(t, a, b) (0x7c0004ae | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_STFDX(s, a, b) (0x7c0005ae | ___PPC_RS(s) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_LVX(t, a, b) (0x7c0000ce | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_STVX(s, a, b) (0x7c0001ce | ___PPC_RS(s) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_ADDE(t, a, b) (0x7c000114 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_ADDZE(t, a) (0x7c000194 | ___PPC_RT(t) | ___PPC_RA(a)) +#define PPC_RAW_ADDME(t, a) (0x7c0001d4 | ___PPC_RT(t) | ___PPC_RA(a)) +#define PPC_RAW_ADD(t, a, b) (0x7c000214 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_ADD_DOT(t, a, b) (0x7c000214 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | 0x1) +#define PPC_RAW_ADDC(t, a, b) (0x7c000014 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_ADDC_DOT(t, a, b) (0x7c000014 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | 0x1) +#define PPC_RAW_NOP() PPC_RAW_ORI(0, 0, 0) +#define PPC_RAW_BLR() (0x4e800020) +#define PPC_RAW_BLRL() (0x4e800021) +#define PPC_RAW_MTLR(r) (0x7c0803a6 | ___PPC_RT(r)) +#define PPC_RAW_MFLR(t) (0x7c0802a6 | ___PPC_RT(t)) +#define PPC_RAW_BCTR() (0x4e800420) +#define PPC_RAW_BCTRL() (0x4e800421) +#define PPC_RAW_MTCTR(r) (0x7c0903a6 | ___PPC_RT(r)) +#define PPC_RAW_ADDI(d, a, i) (0x38000000 | ___PPC_RT(d) | ___PPC_RA(a) | IMM_L(i)) +#define PPC_RAW_LI(r, i) PPC_RAW_ADDI(r, 0, i) +#define PPC_RAW_ADDIS(d, a, i) (0x3c000000 | ___PPC_RT(d) | ___PPC_RA(a) | IMM_L(i)) +#define PPC_RAW_ADDIC(d, a, i) (0x30000000 | ___PPC_RT(d) | ___PPC_RA(a) | IMM_L(i)) +#define PPC_RAW_ADDIC_DOT(d, a, i) (0x34000000 | ___PPC_RT(d) | ___PPC_RA(a) | IMM_L(i)) +#define PPC_RAW_LIS(r, i) PPC_RAW_ADDIS(r, 0, i) +#define PPC_RAW_STDX(r, base, b) (0x7c00012a | ___PPC_RS(r) | ___PPC_RA(base) | ___PPC_RB(b)) +#define PPC_RAW_STDU(r, base, i) (0xf8000001 | ___PPC_RS(r) | ___PPC_RA(base) | ((i) & 0xfffc)) +#define PPC_RAW_STW(r, base, i) (0x90000000 | ___PPC_RS(r) | ___PPC_RA(base) | IMM_L(i)) +#define PPC_RAW_STWU(r, base, i) (0x94000000 | ___PPC_RS(r) | ___PPC_RA(base) | IMM_L(i)) +#define PPC_RAW_STH(r, base, i) (0xb0000000 | ___PPC_RS(r) | ___PPC_RA(base) | IMM_L(i)) +#define PPC_RAW_STB(r, base, i) (0x98000000 | ___PPC_RS(r) | ___PPC_RA(base) | IMM_L(i)) +#define PPC_RAW_LBZ(r, base, i) (0x88000000 | ___PPC_RT(r) | ___PPC_RA(base) | IMM_L(i)) +#define PPC_RAW_LDX(r, base, b) (0x7c00002a | ___PPC_RT(r) | ___PPC_RA(base) | ___PPC_RB(b)) +#define PPC_RAW_LHA(r, base, i) (0xa8000000 | ___PPC_RT(r) | ___PPC_RA(base) | IMM_L(i)) +#define PPC_RAW_LHZ(r, base, i) (0xa0000000 | ___PPC_RT(r) | ___PPC_RA(base) | IMM_L(i)) +#define PPC_RAW_LHBRX(r, base, b) (0x7c00062c | ___PPC_RT(r) | ___PPC_RA(base) | ___PPC_RB(b)) +#define PPC_RAW_LWBRX(r, base, b) (0x7c00042c | ___PPC_RT(r) | ___PPC_RA(base) | ___PPC_RB(b)) +#define PPC_RAW_LDBRX(r, base, b) (0x7c000428 | ___PPC_RT(r) | ___PPC_RA(base) | ___PPC_RB(b)) +#define PPC_RAW_STWCX(s, a, b) (0x7c00012d | ___PPC_RS(s) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_CMPWI(a, i) (0x2c000000 | ___PPC_RA(a) | IMM_L(i)) +#define PPC_RAW_CMPDI(a, i) (0x2c200000 | ___PPC_RA(a) | IMM_L(i)) +#define PPC_RAW_CMPW(a, b) (0x7c000000 | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_CMPD(a, b) (0x7c200000 | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_CMPLWI(a, i) (0x28000000 | ___PPC_RA(a) | IMM_L(i)) +#define PPC_RAW_CMPLDI(a, i) (0x28200000 | ___PPC_RA(a) | IMM_L(i)) +#define PPC_RAW_CMPLW(a, b) (0x7c000040 | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_CMPLD(a, b) (0x7c200040 | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_SUB(d, a, b) (0x7c000050 | ___PPC_RT(d) | ___PPC_RB(a) | ___PPC_RA(b)) +#define PPC_RAW_SUBFC(d, a, b) (0x7c000010 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_SUBFE(d, a, b) (0x7c000110 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_SUBFIC(d, a, i) (0x20000000 | ___PPC_RT(d) | ___PPC_RA(a) | IMM_L(i)) +#define PPC_RAW_SUBFZE(d, a) (0x7c000190 | ___PPC_RT(d) | ___PPC_RA(a)) +#define PPC_RAW_MULD(d, a, b) (0x7c0001d2 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_MULW(d, a, b) (0x7c0001d6 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_MULHWU(d, a, b) (0x7c000016 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_MULI(d, a, i) (0x1c000000 | ___PPC_RT(d) | ___PPC_RA(a) | IMM_L(i)) +#define PPC_RAW_DIVW(d, a, b) (0x7c0003d6 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_DIVWU(d, a, b) (0x7c000396 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_DIVD(d, a, b) (0x7c0003d2 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_DIVDU(d, a, b) (0x7c000392 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_DIVDE(t, a, b) (0x7c000352 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_DIVDE_DOT(t, a, b) (0x7c000352 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | 0x1) +#define PPC_RAW_DIVDEU(t, a, b) (0x7c000312 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_DIVDEU_DOT(t, a, b) (0x7c000312 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | 0x1) +#define PPC_RAW_AND(d, a, b) (0x7c000038 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(b)) +#define PPC_RAW_ANDI(d, a, i) (0x70000000 | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i)) +#define PPC_RAW_ANDIS(d, a, i) (0x74000000 | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i)) +#define PPC_RAW_AND_DOT(d, a, b) (0x7c000039 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(b)) +#define PPC_RAW_OR(d, a, b) (0x7c000378 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(b)) +#define PPC_RAW_MR(d, a) PPC_RAW_OR(d, a, a) +#define PPC_RAW_ORI(d, a, i) (0x60000000 | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i)) +#define PPC_RAW_ORIS(d, a, i) (0x64000000 | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i)) +#define PPC_RAW_NOR(d, a, b) (0x7c0000f8 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(b)) +#define PPC_RAW_XOR(d, a, b) (0x7c000278 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(b)) +#define PPC_RAW_XORI(d, a, i) (0x68000000 | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i)) +#define PPC_RAW_XORIS(d, a, i) (0x6c000000 | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i)) +#define PPC_RAW_EXTSB(d, a) (0x7c000774 | ___PPC_RA(d) | ___PPC_RS(a)) +#define PPC_RAW_EXTSH(d, a) (0x7c000734 | ___PPC_RA(d) | ___PPC_RS(a)) +#define PPC_RAW_EXTSW(d, a) (0x7c0007b4 | ___PPC_RA(d) | ___PPC_RS(a)) +#define PPC_RAW_SLW(d, a, s) (0x7c000030 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(s)) +#define PPC_RAW_SLD(d, a, s) (0x7c000036 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(s)) +#define PPC_RAW_SRW(d, a, s) (0x7c000430 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(s)) +#define PPC_RAW_SRAW(d, a, s) (0x7c000630 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(s)) +#define PPC_RAW_SRAWI(d, a, i) (0x7c000670 | ___PPC_RA(d) | ___PPC_RS(a) | __PPC_SH(i)) +#define PPC_RAW_SRD(d, a, s) (0x7c000436 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(s)) +#define PPC_RAW_SRAD(d, a, s) (0x7c000634 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(s)) +#define PPC_RAW_SRADI(d, a, i) (0x7c000674 | ___PPC_RA(d) | ___PPC_RS(a) | __PPC_SH64(i)) +#define PPC_RAW_RLWINM(d, a, i, mb, me) (0x54000000 | ___PPC_RA(d) | ___PPC_RS(a) | __PPC_SH(i) | __PPC_MB(mb) | __PPC_ME(me)) +#define PPC_RAW_RLWINM_DOT(d, a, i, mb, me) \ + (0x54000001 | ___PPC_RA(d) | ___PPC_RS(a) | __PPC_SH(i) | __PPC_MB(mb) | __PPC_ME(me)) +#define PPC_RAW_RLWIMI(d, a, i, mb, me) (0x50000000 | ___PPC_RA(d) | ___PPC_RS(a) | __PPC_SH(i) | __PPC_MB(mb) | __PPC_ME(me)) +#define PPC_RAW_RLDICL(d, a, i, mb) (0x78000000 | ___PPC_RA(d) | ___PPC_RS(a) | __PPC_SH64(i) | __PPC_MB64(mb)) +#define PPC_RAW_RLDICL_DOT(d, a, i, mb) (0x78000000 | ___PPC_RA(d) | ___PPC_RS(a) | __PPC_SH64(i) | __PPC_MB64(mb) | 0x1) +#define PPC_RAW_RLDICR(d, a, i, me) (0x78000004 | ___PPC_RA(d) | ___PPC_RS(a) | __PPC_SH64(i) | __PPC_ME64(me)) + +/* slwi = rlwinm Rx, Ry, n, 0, 31-n */ +#define PPC_RAW_SLWI(d, a, i) PPC_RAW_RLWINM(d, a, i, 0, 31-(i)) +/* srwi = rlwinm Rx, Ry, 32-n, n, 31 */ +#define PPC_RAW_SRWI(d, a, i) PPC_RAW_RLWINM(d, a, 32-(i), i, 31) +/* sldi = rldicr Rx, Ry, n, 63-n */ +#define PPC_RAW_SLDI(d, a, i) PPC_RAW_RLDICR(d, a, i, 63-(i)) +/* sldi = rldicl Rx, Ry, 64-n, n */ +#define PPC_RAW_SRDI(d, a, i) PPC_RAW_RLDICL(d, a, 64-(i), i) + +#define PPC_RAW_NEG(d, a) (0x7c0000d0 | ___PPC_RT(d) | ___PPC_RA(a)) + +#define PPC_RAW_MFSPR(d, spr) (0x7c0002a6 | ___PPC_RT(d) | __PPC_SPR(spr)) +#define PPC_RAW_MTSPR(spr, d) (0x7c0003a6 | ___PPC_RS(d) | __PPC_SPR(spr)) +#define PPC_RAW_EIEIO() (0x7c0006ac) + +/* bcl 20,31,$+4 */ +#define PPC_RAW_BCL4() (0x429f0005) +#define PPC_RAW_BRANCH(offset) (0x48000000 | PPC_LI(offset)) +#define PPC_RAW_BL(offset) (0x48000001 | PPC_LI(offset)) +#define PPC_RAW_TW(t0, a, b) (0x7c000008 | ___PPC_RS(t0) | ___PPC_RA(a) | ___PPC_RB(b)) +#define PPC_RAW_TRAP() PPC_RAW_TW(31, 0, 0) +#define PPC_RAW_SETB(t, bfa) (0x7c000100 | ___PPC_RT(t) | ___PPC_RA((bfa) << 2)) + +#ifdef CONFIG_PPC32 +#define PPC_RAW_STL PPC_RAW_STW +#define PPC_RAW_STLU PPC_RAW_STWU +#define PPC_RAW_LL PPC_RAW_LWZ +#define PPC_RAW_CMPLI PPC_RAW_CMPWI #else -#define __PPC_EH(eh) 0 +#define PPC_RAW_STL PPC_RAW_STD +#define PPC_RAW_STLU PPC_RAW_STDU +#define PPC_RAW_LL PPC_RAW_LD +#define PPC_RAW_CMPLI PPC_RAW_CMPDI #endif /* Deal with instructions that older assemblers aren't aware of */ -#define PPC_DCBAL(a, b) stringify_in_c(.long PPC_INST_DCBAL | \ - __PPC_RA(a) | __PPC_RB(b)) -#define PPC_DCBZL(a, b) stringify_in_c(.long PPC_INST_DCBZL | \ - __PPC_RA(a) | __PPC_RB(b)) -#define PPC_LDARX(t, a, b, eh) stringify_in_c(.long PPC_INST_LDARX | \ - ___PPC_RT(t) | ___PPC_RA(a) | \ - ___PPC_RB(b) | __PPC_EH(eh)) -#define PPC_LWARX(t, a, b, eh) stringify_in_c(.long PPC_INST_LWARX | \ - ___PPC_RT(t) | ___PPC_RA(a) | \ - ___PPC_RB(b) | __PPC_EH(eh)) -#define PPC_MSGSND(b) stringify_in_c(.long PPC_INST_MSGSND | \ - ___PPC_RB(b)) -#define PPC_MSGSNDP(b) stringify_in_c(.long PPC_INST_MSGSNDP | \ - ___PPC_RB(b)) -#define PPC_POPCNTB(a, s) stringify_in_c(.long PPC_INST_POPCNTB | \ - __PPC_RA(a) | __PPC_RS(s)) -#define PPC_POPCNTD(a, s) stringify_in_c(.long PPC_INST_POPCNTD | \ - __PPC_RA(a) | __PPC_RS(s)) -#define PPC_POPCNTW(a, s) stringify_in_c(.long PPC_INST_POPCNTW | \ - __PPC_RA(a) | __PPC_RS(s)) -#define PPC_RFCI stringify_in_c(.long PPC_INST_RFCI) -#define PPC_RFDI stringify_in_c(.long PPC_INST_RFDI) -#define PPC_RFMCI stringify_in_c(.long PPC_INST_RFMCI) -#define PPC_TLBILX(t, a, b) stringify_in_c(.long PPC_INST_TLBILX | \ - __PPC_T_TLB(t) | __PPC_RA0(a) | __PPC_RB(b)) +#define PPC_BCCTR_FLUSH stringify_in_c(.long PPC_INST_BCCTR_FLUSH) +#define PPC_CP_ABORT stringify_in_c(.long PPC_RAW_CP_ABORT) +#define PPC_COPY(a, b) stringify_in_c(.long PPC_RAW_COPY(a, b)) +#define PPC_DARN(t, l) stringify_in_c(.long PPC_RAW_DARN(t, l)) +#define PPC_DCBAL(a, b) stringify_in_c(.long PPC_RAW_DCBAL(a, b)) +#define PPC_DCBZL(a, b) stringify_in_c(.long PPC_RAW_DCBZL(a, b)) +#define PPC_DIVDE(t, a, b) stringify_in_c(.long PPC_RAW_DIVDE(t, a, b)) +#define PPC_DIVDEU(t, a, b) stringify_in_c(.long PPC_RAW_DIVDEU(t, a, b)) +#define PPC_DSSALL stringify_in_c(.long PPC_INST_DSSALL) +#define PPC_LQARX(t, a, b, eh) stringify_in_c(.long PPC_RAW_LQARX(t, a, b, eh)) +#define PPC_STQCX(t, a, b) stringify_in_c(.long PPC_RAW_STQCX(t, a, b)) +#define PPC_MADDHD(t, a, b, c) stringify_in_c(.long PPC_RAW_MADDHD(t, a, b, c)) +#define PPC_MADDHDU(t, a, b, c) stringify_in_c(.long PPC_RAW_MADDHDU(t, a, b, c)) +#define PPC_MADDLD(t, a, b, c) stringify_in_c(.long PPC_RAW_MADDLD(t, a, b, c)) +#define PPC_MSGSND(b) stringify_in_c(.long PPC_RAW_MSGSND(b)) +#define PPC_MSGSYNC stringify_in_c(.long PPC_RAW_MSGSYNC) +#define PPC_MSGCLR(b) stringify_in_c(.long PPC_RAW_MSGCLR(b)) +#define PPC_MSGSNDP(b) stringify_in_c(.long PPC_RAW_MSGSNDP(b)) +#define PPC_MSGCLRP(b) stringify_in_c(.long PPC_RAW_MSGCLRP(b)) +#define PPC_PASTE(a, b) stringify_in_c(.long PPC_RAW_PASTE(a, b)) +#define PPC_POPCNTB(a, s) stringify_in_c(.long PPC_RAW_POPCNTB(a, s)) +#define PPC_POPCNTD(a, s) stringify_in_c(.long PPC_RAW_POPCNTD(a, s)) +#define PPC_POPCNTW(a, s) stringify_in_c(.long PPC_RAW_POPCNTW(a, s)) +#define PPC_RFCI stringify_in_c(.long PPC_RAW_RFCI) +#define PPC_RFDI stringify_in_c(.long PPC_RAW_RFDI) +#define PPC_RFMCI stringify_in_c(.long PPC_RAW_RFMCI) +#define PPC_TLBILX(t, a, b) stringify_in_c(.long PPC_RAW_TLBILX(t, a, b)) #define PPC_TLBILX_ALL(a, b) PPC_TLBILX(0, a, b) #define PPC_TLBILX_PID(a, b) PPC_TLBILX(1, a, b) +#define PPC_TLBILX_LPID stringify_in_c(.long PPC_RAW_TLBILX_LPID) #define PPC_TLBILX_VA(a, b) PPC_TLBILX(3, a, b) -#define PPC_WAIT(w) stringify_in_c(.long PPC_INST_WAIT | \ - __PPC_WC(w)) -#define PPC_TLBIE(lp,a) stringify_in_c(.long PPC_INST_TLBIE | \ - ___PPC_RB(a) | ___PPC_RS(lp)) -#define PPC_TLBSRX_DOT(a,b) stringify_in_c(.long PPC_INST_TLBSRX_DOT | \ - __PPC_RA0(a) | __PPC_RB(b)) -#define PPC_TLBIVAX(a,b) stringify_in_c(.long PPC_INST_TLBIVAX | \ - __PPC_RA0(a) | __PPC_RB(b)) - -#define PPC_ERATWE(s, a, w) stringify_in_c(.long PPC_INST_ERATWE | \ - __PPC_RS(s) | __PPC_RA(a) | __PPC_WS(w)) -#define PPC_ERATRE(s, a, w) stringify_in_c(.long PPC_INST_ERATRE | \ - __PPC_RS(s) | __PPC_RA(a) | __PPC_WS(w)) -#define PPC_ERATILX(t, a, b) stringify_in_c(.long PPC_INST_ERATILX | \ - __PPC_T_TLB(t) | __PPC_RA0(a) | \ - __PPC_RB(b)) -#define PPC_ERATIVAX(s, a, b) stringify_in_c(.long PPC_INST_ERATIVAX | \ - __PPC_RS(s) | __PPC_RA0(a) | __PPC_RB(b)) -#define PPC_ERATSX(t, a, w) stringify_in_c(.long PPC_INST_ERATSX | \ - __PPC_RS(t) | __PPC_RA0(a) | __PPC_RB(b)) -#define PPC_ERATSX_DOT(t, a, w) stringify_in_c(.long PPC_INST_ERATSX_DOT | \ - __PPC_RS(t) | __PPC_RA0(a) | __PPC_RB(b)) -#define PPC_SLBFEE_DOT(t, b) stringify_in_c(.long PPC_INST_SLBFEE | \ - __PPC_RT(t) | __PPC_RB(b)) -#define PPC_ICBT(c,a,b) stringify_in_c(.long PPC_INST_ICBT | \ - __PPC_CT(c) | __PPC_RA0(a) | __PPC_RB(b)) +#define PPC_WAIT_v203 stringify_in_c(.long PPC_RAW_WAIT_v203) +#define PPC_WAIT(w, p) stringify_in_c(.long PPC_RAW_WAIT(w, p)) +#define PPC_TLBIE(lp, a) stringify_in_c(.long PPC_RAW_TLBIE(lp, a)) +#define PPC_TLBIE_5(rb, rs, ric, prs, r) \ + stringify_in_c(.long PPC_RAW_TLBIE_5(rb, rs, ric, prs, r)) +#define PPC_TLBIEL(rb,rs,ric,prs,r) \ + stringify_in_c(.long PPC_RAW_TLBIEL(rb, rs, ric, prs, r)) +#define PPC_TLBIEL_v205(rb, l) stringify_in_c(.long PPC_RAW_TLBIEL_v205(rb, l)) +#define PPC_TLBSRX_DOT(a, b) stringify_in_c(.long PPC_RAW_TLBSRX_DOT(a, b)) +#define PPC_TLBIVAX(a, b) stringify_in_c(.long PPC_RAW_TLBIVAX(a, b)) + +#define PPC_ERATWE(s, a, w) stringify_in_c(.long PPC_RAW_ERATWE(s, a, w)) +#define PPC_ERATRE(s, a, w) stringify_in_c(.long PPC_RAW_ERATRE(a, a, w)) +#define PPC_ERATILX(t, a, b) stringify_in_c(.long PPC_RAW_ERATILX(t, a, b)) +#define PPC_ERATIVAX(s, a, b) stringify_in_c(.long PPC_RAW_ERATIVAX(s, a, b)) +#define PPC_ERATSX(t, a, w) stringify_in_c(.long PPC_RAW_ERATSX(t, a, w)) +#define PPC_ERATSX_DOT(t, a, w) stringify_in_c(.long PPC_RAW_ERATSX_DOT(t, a, w)) +#define PPC_SLBFEE_DOT(t, b) stringify_in_c(.long PPC_RAW_SLBFEE_DOT(t, b)) +#define __PPC_SLBFEE_DOT(t, b) stringify_in_c(.long __PPC_RAW_SLBFEE_DOT(t, b)) +#define PPC_ICBT(c, a, b) stringify_in_c(.long PPC_RAW_ICBT(c, a, b)) /* PASemi instructions */ -#define LBZCIX(t,a,b) stringify_in_c(.long PPC_INST_LBZCIX | \ - __PPC_RT(t) | __PPC_RA(a) | __PPC_RB(b)) -#define STBCIX(s,a,b) stringify_in_c(.long PPC_INST_STBCIX | \ - __PPC_RS(s) | __PPC_RA(a) | __PPC_RB(b)) +#define LBZCIX(t, a, b) stringify_in_c(.long PPC_RAW_LBZCIX(t, a, b)) +#define STBCIX(s, a, b) stringify_in_c(.long PPC_RAW_STBCIX(s, a, b)) +#define PPC_DCBFPS(a, b) stringify_in_c(.long PPC_RAW_DCBFPS(a, b)) +#define PPC_DCBSTPS(a, b) stringify_in_c(.long PPC_RAW_DCBSTPS(a, b)) +#define PPC_PHWSYNC stringify_in_c(.long PPC_RAW_PHWSYNC) +#define PPC_PLWSYNC stringify_in_c(.long PPC_RAW_PLWSYNC) +#define STXVD2X(s, a, b) stringify_in_c(.long PPC_RAW_STXVD2X(s, a, b)) +#define LXVD2X(s, a, b) stringify_in_c(.long PPC_RAW_LXVD2X(s, a, b)) +#define MFVRD(a, t) stringify_in_c(.long PPC_RAW_MFVRD(a, t)) +#define MTVRD(t, a) stringify_in_c(.long PPC_RAW_MTVRD(t, a)) +#define VPMSUMW(t, a, b) stringify_in_c(.long PPC_RAW_VPMSUMW(t, a, b)) +#define VPMSUMD(t, a, b) stringify_in_c(.long PPC_RAW_VPMSUMD(t, a, b)) +#define XXLOR(t, a, b) stringify_in_c(.long PPC_RAW_XXLOR(t, a, b)) +#define XXSWAPD(t, a) stringify_in_c(.long PPC_RAW_XXSWAPD(t, a)) +#define XVCPSGNDP(t, a, b) stringify_in_c(.long (PPC_RAW_XVCPSGNDP(t, a, b))) -/* - * Define what the VSX XX1 form instructions will look like, then add - * the 128 bit load store instructions based on that. - */ -#define VSX_XX1(s, a, b) (__PPC_XS(s) | __PPC_RA(a) | __PPC_RB(b)) -#define VSX_XX3(t, a, b) (__PPC_XT(t) | __PPC_XA(a) | __PPC_XB(b)) -#define STXVD2X(s, a, b) stringify_in_c(.long PPC_INST_STXVD2X | \ - VSX_XX1((s), a, b)) -#define LXVD2X(s, a, b) stringify_in_c(.long PPC_INST_LXVD2X | \ - VSX_XX1((s), a, b)) -#define XXLOR(t, a, b) stringify_in_c(.long PPC_INST_XXLOR | \ - VSX_XX3((t), a, b)) -#define XVCPSGNDP(t, a, b) stringify_in_c(.long (PPC_INST_XVCPSGNDP | \ - VSX_XX3((t), (a), (b)))) - -#define PPC_NAP stringify_in_c(.long PPC_INST_NAP) -#define PPC_SLEEP stringify_in_c(.long PPC_INST_SLEEP) +#define VPERMXOR(vrt, vra, vrb, vrc) \ + stringify_in_c(.long (PPC_RAW_VPERMXOR(vrt, vra, vrb, vrc))) + +#define PPC_NAP stringify_in_c(.long PPC_RAW_NAP) +#define PPC_SLEEP stringify_in_c(.long PPC_RAW_SLEEP) +#define PPC_WINKLE stringify_in_c(.long PPC_RAW_WINKLE) + +#define PPC_STOP stringify_in_c(.long PPC_RAW_STOP) /* BHRB instructions */ -#define PPC_CLRBHRB stringify_in_c(.long PPC_INST_CLRBHRB) -#define PPC_MFBHRBE(r, n) stringify_in_c(.long PPC_INST_BHRBE | \ - __PPC_RT(r) | \ - (((n) & 0x3ff) << 11)) +#define PPC_CLRBHRB stringify_in_c(.long PPC_RAW_CLRBHRB) +#define PPC_MFBHRBE(r, n) stringify_in_c(.long PPC_RAW_MFBHRBE(r, n)) /* Transactional memory instructions */ -#define TRECHKPT stringify_in_c(.long PPC_INST_TRECHKPT) -#define TRECLAIM(r) stringify_in_c(.long PPC_INST_TRECLAIM \ - | __PPC_RA(r)) -#define TABORT(r) stringify_in_c(.long PPC_INST_TABORT \ - | __PPC_RA(r)) +#define TRECHKPT stringify_in_c(.long PPC_RAW_TRECHKPT) +#define TRECLAIM(r) stringify_in_c(.long PPC_RAW_TRECLAIM(r)) +#define TABORT(r) stringify_in_c(.long PPC_RAW_TABORT(r)) + +/* book3e thread control instructions */ +#define MTTMR(tmr, r) stringify_in_c(.long PPC_RAW_MTTMR(tmr, r)) +#define MFTMR(tmr, r) stringify_in_c(.long PPC_RAW_MFTMR(tmr, r)) + +/* Coprocessor instructions */ +#define PPC_ICSWX(s, a, b) stringify_in_c(.long PPC_RAW_ICSWX(s, a, b)) +#define PPC_ICSWEPX(s, a, b) stringify_in_c(.long PPC_RAW_ICSWEPX(s, a, b)) + +#define PPC_SLBIA(IH) stringify_in_c(.long PPC_RAW_SLBIA(IH)) + +/* + * These may only be used on ISA v3.0 or later (aka. CPU_FTR_ARCH_300, radix + * implies CPU_FTR_ARCH_300). USER/GUEST invalidates may only be used by radix + * mode (on HPT these would also invalidate various SLBEs which may not be + * desired). + */ +#define PPC_ISA_3_0_INVALIDATE_ERAT PPC_SLBIA(7) +#define PPC_RADIX_INVALIDATE_ERAT_USER PPC_SLBIA(3) +#define PPC_RADIX_INVALIDATE_ERAT_GUEST PPC_SLBIA(6) + +#define VCMPEQUD_RC(vrt, vra, vrb) stringify_in_c(.long PPC_RAW_VCMPEQUD_RC(vrt, vra, vrb)) + +#define VCMPEQUB_RC(vrt, vra, vrb) stringify_in_c(.long PPC_RAW_VCMPEQUB_RC(vrt, vra, vrb)) #endif /* _ASM_POWERPC_PPC_OPCODE_H */ diff --git a/arch/powerpc/include/asm/ppc-pci.h b/arch/powerpc/include/asm/ppc-pci.h index ed57fa7920c8..a8b7e8682f5b 100644 --- a/arch/powerpc/include/asm/ppc-pci.h +++ b/arch/powerpc/include/asm/ppc-pci.h @@ -1,10 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * c 2001 PPC 64 Team, IBM Corp - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef _ASM_POWERPC_PPC_PCI_H #define _ASM_POWERPC_PPC_PCI_H @@ -17,14 +13,8 @@ extern unsigned long isa_io_base; -extern void pci_setup_phb_io(struct pci_controller *hose, int primary); -extern void pci_setup_phb_io_dynamic(struct pci_controller *hose, int primary); - - extern struct list_head hose_list; -extern void find_and_init_phbs(void); - extern struct pci_dev *isa_bridge_pcidev; /* may be NULL if no ISA bus */ /** Bus Unit ID macros; get low and hi 32-bits of the 64-bit BUID */ @@ -33,18 +23,31 @@ extern struct pci_dev *isa_bridge_pcidev; /* may be NULL if no ISA bus */ /* PCI device_node operations */ struct device_node; -typedef void *(*traverse_func)(struct device_node *me, void *data); -void *traverse_pci_devices(struct device_node *start, traverse_func pre, - void *data); +struct pci_dn; -extern void pci_devs_phb_init(void); +void *pci_traverse_device_nodes(struct device_node *start, + void *(*fn)(struct device_node *, void *), + void *data); extern void pci_devs_phb_init_dynamic(struct pci_controller *phb); +#if defined(CONFIG_IOMMU_API) && (defined(CONFIG_PPC_PSERIES) || \ + defined(CONFIG_PPC_POWERNV)) +extern void ppc_iommu_register_device(struct pci_controller *phb); +extern void ppc_iommu_unregister_device(struct pci_controller *phb); +#else +static inline void ppc_iommu_register_device(struct pci_controller *phb) { } +static inline void ppc_iommu_unregister_device(struct pci_controller *phb) { } +#endif + + /* From rtas_pci.h */ extern void init_pci_config_tokens (void); extern unsigned long get_phb_buid (struct device_node *); extern int rtas_setup_phb(struct pci_controller *phb); +int rtas_pci_dn_read_config(struct pci_dn *pdn, int where, int size, u32 *val); +int rtas_pci_dn_write_config(struct pci_dn *pdn, int where, int size, u32 val); + #ifdef CONFIG_EEH void eeh_addr_cache_insert_dev(struct pci_dev *dev); @@ -52,32 +55,32 @@ void eeh_addr_cache_rmv_dev(struct pci_dev *dev); struct eeh_dev *eeh_addr_cache_get_dev(unsigned long addr); void eeh_slot_error_detail(struct eeh_pe *pe, int severity); int eeh_pci_enable(struct eeh_pe *pe, int function); -int eeh_reset_pe(struct eeh_pe *); +int eeh_pe_reset_full(struct eeh_pe *pe, bool include_passed); void eeh_save_bars(struct eeh_dev *edev); -int rtas_write_config(struct pci_dn *, int where, int size, u32 val); -int rtas_read_config(struct pci_dn *, int where, int size, u32 *val); void eeh_pe_state_mark(struct eeh_pe *pe, int state); -void eeh_pe_state_clear(struct eeh_pe *pe, int state); +void eeh_pe_mark_isolated(struct eeh_pe *pe); +void eeh_pe_state_clear(struct eeh_pe *pe, int state, bool include_passed); +void eeh_pe_state_mark_with_cfg(struct eeh_pe *pe, int state); +void eeh_pe_dev_mode_mark(struct eeh_pe *pe, int mode); void eeh_sysfs_add_device(struct pci_dev *pdev); void eeh_sysfs_remove_device(struct pci_dev *pdev); -static inline const char *eeh_pci_name(struct pci_dev *pdev) -{ - return pdev ? pci_name(pdev) : "<null>"; -} +#endif /* CONFIG_EEH */ -static inline const char *eeh_driver_name(struct pci_dev *pdev) -{ - return (pdev && pdev->driver) ? pdev->driver->name : "<null>"; -} +#ifdef CONFIG_FSL_ULI1575 +void __init uli_init(void); +#endif /* CONFIG_FSL_ULI1575 */ -#endif /* CONFIG_EEH */ +#define PCI_BUSNO(bdfn) ((bdfn >> 8) & 0xff) #else /* CONFIG_PCI */ -static inline void find_and_init_phbs(void) { } static inline void init_pci_config_tokens(void) { } #endif /* !CONFIG_PCI */ +#if !defined(CONFIG_PCI) || !defined(CONFIG_FSL_ULI1575) +static inline void __init uli_init(void) {} +#endif /* !defined(CONFIG_PCI) || !defined(CONFIG_FSL_ULI1575) */ + #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_PPC_PCI_H */ diff --git a/arch/powerpc/include/asm/ppc4xx.h b/arch/powerpc/include/asm/ppc4xx.h index 033039a80c42..b37119e48543 100644 --- a/arch/powerpc/include/asm/ppc4xx.h +++ b/arch/powerpc/include/asm/ppc4xx.h @@ -1,18 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * PPC4xx Prototypes and definitions * * Copyright 2008 DENX Software Engineering, Stefan Roese <sr@denx.de> - * - * This is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * */ #ifndef __ASM_POWERPC_PPC4xx_H__ #define __ASM_POWERPC_PPC4xx_H__ -extern void ppc4xx_reset_system(char *cmd); +extern void __noreturn ppc4xx_reset_system(char *cmd); #endif /* __ASM_POWERPC_PPC4xx_H__ */ diff --git a/arch/powerpc/include/asm/ppc4xx_ocm.h b/arch/powerpc/include/asm/ppc4xx_ocm.h deleted file mode 100644 index 6ce904605538..000000000000 --- a/arch/powerpc/include/asm/ppc4xx_ocm.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * PowerPC 4xx OCM memory allocation support - * - * (C) Copyright 2009, Applied Micro Circuits Corporation - * Victor Gallardo (vgallardo@amcc.com) - * - * See file CREDITS for list of people who contributed to this - * project. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation; either version 2 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, - * MA 02111-1307 USA - */ - -#ifndef __ASM_POWERPC_PPC4XX_OCM_H__ -#define __ASM_POWERPC_PPC4XX_OCM_H__ - -#define PPC4XX_OCM_NON_CACHED 0 -#define PPC4XX_OCM_CACHED 1 - -#if defined(CONFIG_PPC4xx_OCM) - -void *ppc4xx_ocm_alloc(phys_addr_t *phys, int size, int align, - int flags, const char *owner); -void ppc4xx_ocm_free(const void *virt); - -#else - -#define ppc4xx_ocm_alloc(phys, size, align, flags, owner) NULL -#define ppc4xx_ocm_free(addr) ((void)0) - -#endif /* CONFIG_PPC4xx_OCM */ - -#endif /* __ASM_POWERPC_PPC4XX_OCM_H__ */ diff --git a/arch/powerpc/include/asm/ppc_asm.h b/arch/powerpc/include/asm/ppc_asm.h index 2f1b6c5f8174..46947c82a712 100644 --- a/arch/powerpc/include/asm/ppc_asm.h +++ b/arch/powerpc/include/asm/ppc_asm.h @@ -4,234 +4,145 @@ #ifndef _ASM_POWERPC_PPC_ASM_H #define _ASM_POWERPC_PPC_ASM_H -#include <linux/init.h> #include <linux/stringify.h> #include <asm/asm-compat.h> #include <asm/processor.h> #include <asm/ppc-opcode.h> #include <asm/firmware.h> +#include <asm/feature-fixups.h> +#include <asm/extable.h> -#ifndef __ASSEMBLY__ -#error __FILE__ should only be used in assembler files -#else +#ifdef __ASSEMBLER__ #define SZL (BITS_PER_LONG/8) /* - * Stuff for accurate CPU time accounting. - * These macros handle transitions between user and system state - * in exception entry and exit and accumulate time to the - * user_time and system_time fields in the paca. + * This expands to a sequence of operations with reg incrementing from + * start to end inclusive, of this form: + * + * op reg, (offset + (width * reg))(base) + * + * Note that offset is not the offset of the first operation unless start + * is zero (or width is zero). */ +.macro OP_REGS op, width, start, end, base, offset + .Lreg=\start + .rept (\end - \start + 1) + \op .Lreg, \offset + \width * .Lreg(\base) + .Lreg=.Lreg+1 + .endr +.endm -#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE -#define ACCOUNT_CPU_USER_ENTRY(ra, rb) -#define ACCOUNT_CPU_USER_EXIT(ra, rb) -#define ACCOUNT_STOLEN_TIME -#else -#define ACCOUNT_CPU_USER_ENTRY(ra, rb) \ - MFTB(ra); /* get timebase */ \ - ld rb,PACA_STARTTIME_USER(r13); \ - std ra,PACA_STARTTIME(r13); \ - subf rb,rb,ra; /* subtract start value */ \ - ld ra,PACA_USER_TIME(r13); \ - add ra,ra,rb; /* add on to user time */ \ - std ra,PACA_USER_TIME(r13); \ - -#define ACCOUNT_CPU_USER_EXIT(ra, rb) \ - MFTB(ra); /* get timebase */ \ - ld rb,PACA_STARTTIME(r13); \ - std ra,PACA_STARTTIME_USER(r13); \ - subf rb,rb,ra; /* subtract start value */ \ - ld ra,PACA_SYSTEM_TIME(r13); \ - add ra,ra,rb; /* add on to system time */ \ - std ra,PACA_SYSTEM_TIME(r13) - -#ifdef CONFIG_PPC_SPLPAR -#define ACCOUNT_STOLEN_TIME \ -BEGIN_FW_FTR_SECTION; \ - beq 33f; \ - /* from user - see if there are any DTL entries to process */ \ - ld r10,PACALPPACAPTR(r13); /* get ptr to VPA */ \ - ld r11,PACA_DTL_RIDX(r13); /* get log read index */ \ - ld r10,LPPACA_DTLIDX(r10); /* get log write index */ \ - cmpd cr1,r11,r10; \ - beq+ cr1,33f; \ - bl .accumulate_stolen_time; \ - ld r12,_MSR(r1); \ - andi. r10,r12,MSR_PR; /* Restore cr0 (coming from user) */ \ -33: \ -END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR) - -#else /* CONFIG_PPC_SPLPAR */ -#define ACCOUNT_STOLEN_TIME - -#endif /* CONFIG_PPC_SPLPAR */ - -#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ +/* + * This expands to a sequence of register clears for regs start to end + * inclusive, of the form: + * + * li rN, 0 + */ +.macro ZEROIZE_REGS start, end + .Lreg=\start + .rept (\end - \start + 1) + li .Lreg, 0 + .Lreg=.Lreg+1 + .endr +.endm /* * Macros for storing registers into and loading registers from * exception frames. */ #ifdef __powerpc64__ -#define SAVE_GPR(n, base) std n,GPR0+8*(n)(base) -#define REST_GPR(n, base) ld n,GPR0+8*(n)(base) -#define SAVE_NVGPRS(base) SAVE_8GPRS(14, base); SAVE_10GPRS(22, base) -#define REST_NVGPRS(base) REST_8GPRS(14, base); REST_10GPRS(22, base) +#define SAVE_GPRS(start, end, base) OP_REGS std, 8, start, end, base, GPR0 +#define REST_GPRS(start, end, base) OP_REGS ld, 8, start, end, base, GPR0 +#define SAVE_NVGPRS(base) SAVE_GPRS(14, 31, base) +#define REST_NVGPRS(base) REST_GPRS(14, 31, base) #else -#define SAVE_GPR(n, base) stw n,GPR0+4*(n)(base) -#define REST_GPR(n, base) lwz n,GPR0+4*(n)(base) -#define SAVE_NVGPRS(base) SAVE_GPR(13, base); SAVE_8GPRS(14, base); \ - SAVE_10GPRS(22, base) -#define REST_NVGPRS(base) REST_GPR(13, base); REST_8GPRS(14, base); \ - REST_10GPRS(22, base) +#define SAVE_GPRS(start, end, base) OP_REGS stw, 4, start, end, base, GPR0 +#define REST_GPRS(start, end, base) OP_REGS lwz, 4, start, end, base, GPR0 +#define SAVE_NVGPRS(base) SAVE_GPRS(13, 31, base) +#define REST_NVGPRS(base) REST_GPRS(13, 31, base) #endif -#define SAVE_2GPRS(n, base) SAVE_GPR(n, base); SAVE_GPR(n+1, base) -#define SAVE_4GPRS(n, base) SAVE_2GPRS(n, base); SAVE_2GPRS(n+2, base) -#define SAVE_8GPRS(n, base) SAVE_4GPRS(n, base); SAVE_4GPRS(n+4, base) -#define SAVE_10GPRS(n, base) SAVE_8GPRS(n, base); SAVE_2GPRS(n+8, base) -#define REST_2GPRS(n, base) REST_GPR(n, base); REST_GPR(n+1, base) -#define REST_4GPRS(n, base) REST_2GPRS(n, base); REST_2GPRS(n+2, base) -#define REST_8GPRS(n, base) REST_4GPRS(n, base); REST_4GPRS(n+4, base) -#define REST_10GPRS(n, base) REST_8GPRS(n, base); REST_2GPRS(n+8, base) - -#define SAVE_FPR(n, base) stfd n,THREAD_FPR0+8*TS_FPRWIDTH*(n)(base) +#define ZEROIZE_GPRS(start, end) ZEROIZE_REGS start, end +#ifdef __powerpc64__ +#define ZEROIZE_NVGPRS() ZEROIZE_GPRS(14, 31) +#else +#define ZEROIZE_NVGPRS() ZEROIZE_GPRS(13, 31) +#endif +#define ZEROIZE_GPR(n) ZEROIZE_GPRS(n, n) + +#define SAVE_GPR(n, base) SAVE_GPRS(n, n, base) +#define REST_GPR(n, base) REST_GPRS(n, n, base) + +/* macros for handling user register sanitisation */ +#ifdef CONFIG_INTERRUPT_SANITIZE_REGISTERS +#define SANITIZE_SYSCALL_GPRS() ZEROIZE_GPR(0); \ + ZEROIZE_GPRS(5, 12); \ + ZEROIZE_NVGPRS() +#define SANITIZE_GPR(n) ZEROIZE_GPR(n) +#define SANITIZE_GPRS(start, end) ZEROIZE_GPRS(start, end) +#define SANITIZE_NVGPRS() ZEROIZE_NVGPRS() +#define SANITIZE_RESTORE_NVGPRS() REST_NVGPRS(r1) +#define HANDLER_RESTORE_NVGPRS() +#else +#define SANITIZE_SYSCALL_GPRS() +#define SANITIZE_GPR(n) +#define SANITIZE_GPRS(start, end) +#define SANITIZE_NVGPRS() +#define SANITIZE_RESTORE_NVGPRS() +#define HANDLER_RESTORE_NVGPRS() REST_NVGPRS(r1) +#endif /* CONFIG_INTERRUPT_SANITIZE_REGISTERS */ + +#define SAVE_FPR(n, base) stfd n,8*TS_FPRWIDTH*(n)(base) #define SAVE_2FPRS(n, base) SAVE_FPR(n, base); SAVE_FPR(n+1, base) #define SAVE_4FPRS(n, base) SAVE_2FPRS(n, base); SAVE_2FPRS(n+2, base) #define SAVE_8FPRS(n, base) SAVE_4FPRS(n, base); SAVE_4FPRS(n+4, base) #define SAVE_16FPRS(n, base) SAVE_8FPRS(n, base); SAVE_8FPRS(n+8, base) #define SAVE_32FPRS(n, base) SAVE_16FPRS(n, base); SAVE_16FPRS(n+16, base) -#define REST_FPR(n, base) lfd n,THREAD_FPR0+8*TS_FPRWIDTH*(n)(base) +#define REST_FPR(n, base) lfd n,8*TS_FPRWIDTH*(n)(base) #define REST_2FPRS(n, base) REST_FPR(n, base); REST_FPR(n+1, base) #define REST_4FPRS(n, base) REST_2FPRS(n, base); REST_2FPRS(n+2, base) #define REST_8FPRS(n, base) REST_4FPRS(n, base); REST_4FPRS(n+4, base) #define REST_16FPRS(n, base) REST_8FPRS(n, base); REST_8FPRS(n+8, base) #define REST_32FPRS(n, base) REST_16FPRS(n, base); REST_16FPRS(n+16, base) -#define SAVE_VR(n,b,base) li b,THREAD_VR0+(16*(n)); stvx n,base,b +#define SAVE_VR(n,b,base) li b,16*(n); stvx n,base,b #define SAVE_2VRS(n,b,base) SAVE_VR(n,b,base); SAVE_VR(n+1,b,base) #define SAVE_4VRS(n,b,base) SAVE_2VRS(n,b,base); SAVE_2VRS(n+2,b,base) #define SAVE_8VRS(n,b,base) SAVE_4VRS(n,b,base); SAVE_4VRS(n+4,b,base) #define SAVE_16VRS(n,b,base) SAVE_8VRS(n,b,base); SAVE_8VRS(n+8,b,base) #define SAVE_32VRS(n,b,base) SAVE_16VRS(n,b,base); SAVE_16VRS(n+16,b,base) -#define REST_VR(n,b,base) li b,THREAD_VR0+(16*(n)); lvx n,base,b +#define REST_VR(n,b,base) li b,16*(n); lvx n,base,b #define REST_2VRS(n,b,base) REST_VR(n,b,base); REST_VR(n+1,b,base) #define REST_4VRS(n,b,base) REST_2VRS(n,b,base); REST_2VRS(n+2,b,base) #define REST_8VRS(n,b,base) REST_4VRS(n,b,base); REST_4VRS(n+4,b,base) #define REST_16VRS(n,b,base) REST_8VRS(n,b,base); REST_8VRS(n+8,b,base) #define REST_32VRS(n,b,base) REST_16VRS(n,b,base); REST_16VRS(n+16,b,base) -/* Save/restore FPRs, VRs and VSRs from their checkpointed backups in - * thread_struct: - */ -#define SAVE_FPR_TRANSACT(n, base) stfd n,THREAD_TRANSACT_FPR0+ \ - 8*TS_FPRWIDTH*(n)(base) -#define SAVE_2FPRS_TRANSACT(n, base) SAVE_FPR_TRANSACT(n, base); \ - SAVE_FPR_TRANSACT(n+1, base) -#define SAVE_4FPRS_TRANSACT(n, base) SAVE_2FPRS_TRANSACT(n, base); \ - SAVE_2FPRS_TRANSACT(n+2, base) -#define SAVE_8FPRS_TRANSACT(n, base) SAVE_4FPRS_TRANSACT(n, base); \ - SAVE_4FPRS_TRANSACT(n+4, base) -#define SAVE_16FPRS_TRANSACT(n, base) SAVE_8FPRS_TRANSACT(n, base); \ - SAVE_8FPRS_TRANSACT(n+8, base) -#define SAVE_32FPRS_TRANSACT(n, base) SAVE_16FPRS_TRANSACT(n, base); \ - SAVE_16FPRS_TRANSACT(n+16, base) - -#define REST_FPR_TRANSACT(n, base) lfd n,THREAD_TRANSACT_FPR0+ \ - 8*TS_FPRWIDTH*(n)(base) -#define REST_2FPRS_TRANSACT(n, base) REST_FPR_TRANSACT(n, base); \ - REST_FPR_TRANSACT(n+1, base) -#define REST_4FPRS_TRANSACT(n, base) REST_2FPRS_TRANSACT(n, base); \ - REST_2FPRS_TRANSACT(n+2, base) -#define REST_8FPRS_TRANSACT(n, base) REST_4FPRS_TRANSACT(n, base); \ - REST_4FPRS_TRANSACT(n+4, base) -#define REST_16FPRS_TRANSACT(n, base) REST_8FPRS_TRANSACT(n, base); \ - REST_8FPRS_TRANSACT(n+8, base) -#define REST_32FPRS_TRANSACT(n, base) REST_16FPRS_TRANSACT(n, base); \ - REST_16FPRS_TRANSACT(n+16, base) - - -#define SAVE_VR_TRANSACT(n,b,base) li b,THREAD_TRANSACT_VR0+(16*(n)); \ - stvx n,b,base -#define SAVE_2VRS_TRANSACT(n,b,base) SAVE_VR_TRANSACT(n,b,base); \ - SAVE_VR_TRANSACT(n+1,b,base) -#define SAVE_4VRS_TRANSACT(n,b,base) SAVE_2VRS_TRANSACT(n,b,base); \ - SAVE_2VRS_TRANSACT(n+2,b,base) -#define SAVE_8VRS_TRANSACT(n,b,base) SAVE_4VRS_TRANSACT(n,b,base); \ - SAVE_4VRS_TRANSACT(n+4,b,base) -#define SAVE_16VRS_TRANSACT(n,b,base) SAVE_8VRS_TRANSACT(n,b,base); \ - SAVE_8VRS_TRANSACT(n+8,b,base) -#define SAVE_32VRS_TRANSACT(n,b,base) SAVE_16VRS_TRANSACT(n,b,base); \ - SAVE_16VRS_TRANSACT(n+16,b,base) - -#define REST_VR_TRANSACT(n,b,base) li b,THREAD_TRANSACT_VR0+(16*(n)); \ - lvx n,b,base -#define REST_2VRS_TRANSACT(n,b,base) REST_VR_TRANSACT(n,b,base); \ - REST_VR_TRANSACT(n+1,b,base) -#define REST_4VRS_TRANSACT(n,b,base) REST_2VRS_TRANSACT(n,b,base); \ - REST_2VRS_TRANSACT(n+2,b,base) -#define REST_8VRS_TRANSACT(n,b,base) REST_4VRS_TRANSACT(n,b,base); \ - REST_4VRS_TRANSACT(n+4,b,base) -#define REST_16VRS_TRANSACT(n,b,base) REST_8VRS_TRANSACT(n,b,base); \ - REST_8VRS_TRANSACT(n+8,b,base) -#define REST_32VRS_TRANSACT(n,b,base) REST_16VRS_TRANSACT(n,b,base); \ - REST_16VRS_TRANSACT(n+16,b,base) - - -#define SAVE_VSR_TRANSACT(n,b,base) li b,THREAD_TRANSACT_VSR0+(16*(n)); \ - STXVD2X(n,R##base,R##b) -#define SAVE_2VSRS_TRANSACT(n,b,base) SAVE_VSR_TRANSACT(n,b,base); \ - SAVE_VSR_TRANSACT(n+1,b,base) -#define SAVE_4VSRS_TRANSACT(n,b,base) SAVE_2VSRS_TRANSACT(n,b,base); \ - SAVE_2VSRS_TRANSACT(n+2,b,base) -#define SAVE_8VSRS_TRANSACT(n,b,base) SAVE_4VSRS_TRANSACT(n,b,base); \ - SAVE_4VSRS_TRANSACT(n+4,b,base) -#define SAVE_16VSRS_TRANSACT(n,b,base) SAVE_8VSRS_TRANSACT(n,b,base); \ - SAVE_8VSRS_TRANSACT(n+8,b,base) -#define SAVE_32VSRS_TRANSACT(n,b,base) SAVE_16VSRS_TRANSACT(n,b,base); \ - SAVE_16VSRS_TRANSACT(n+16,b,base) - -#define REST_VSR_TRANSACT(n,b,base) li b,THREAD_TRANSACT_VSR0+(16*(n)); \ - LXVD2X(n,R##base,R##b) -#define REST_2VSRS_TRANSACT(n,b,base) REST_VSR_TRANSACT(n,b,base); \ - REST_VSR_TRANSACT(n+1,b,base) -#define REST_4VSRS_TRANSACT(n,b,base) REST_2VSRS_TRANSACT(n,b,base); \ - REST_2VSRS_TRANSACT(n+2,b,base) -#define REST_8VSRS_TRANSACT(n,b,base) REST_4VSRS_TRANSACT(n,b,base); \ - REST_4VSRS_TRANSACT(n+4,b,base) -#define REST_16VSRS_TRANSACT(n,b,base) REST_8VSRS_TRANSACT(n,b,base); \ - REST_8VSRS_TRANSACT(n+8,b,base) -#define REST_32VSRS_TRANSACT(n,b,base) REST_16VSRS_TRANSACT(n,b,base); \ - REST_16VSRS_TRANSACT(n+16,b,base) +#ifdef __BIG_ENDIAN__ +#define STXVD2X_ROT(n,b,base) STXVD2X(n,b,base) +#define LXVD2X_ROT(n,b,base) LXVD2X(n,b,base) +#else +#define STXVD2X_ROT(n,b,base) XXSWAPD(n,n); \ + STXVD2X(n,b,base); \ + XXSWAPD(n,n) +#define LXVD2X_ROT(n,b,base) LXVD2X(n,b,base); \ + XXSWAPD(n,n) +#endif /* Save the lower 32 VSRs in the thread VSR region */ -#define SAVE_VSR(n,b,base) li b,THREAD_VSR0+(16*(n)); STXVD2X(n,R##base,R##b) +#define SAVE_VSR(n,b,base) li b,16*(n); STXVD2X_ROT(n,R##base,R##b) #define SAVE_2VSRS(n,b,base) SAVE_VSR(n,b,base); SAVE_VSR(n+1,b,base) #define SAVE_4VSRS(n,b,base) SAVE_2VSRS(n,b,base); SAVE_2VSRS(n+2,b,base) #define SAVE_8VSRS(n,b,base) SAVE_4VSRS(n,b,base); SAVE_4VSRS(n+4,b,base) #define SAVE_16VSRS(n,b,base) SAVE_8VSRS(n,b,base); SAVE_8VSRS(n+8,b,base) #define SAVE_32VSRS(n,b,base) SAVE_16VSRS(n,b,base); SAVE_16VSRS(n+16,b,base) -#define REST_VSR(n,b,base) li b,THREAD_VSR0+(16*(n)); LXVD2X(n,R##base,R##b) +#define REST_VSR(n,b,base) li b,16*(n); LXVD2X_ROT(n,R##base,R##b) #define REST_2VSRS(n,b,base) REST_VSR(n,b,base); REST_VSR(n+1,b,base) #define REST_4VSRS(n,b,base) REST_2VSRS(n,b,base); REST_2VSRS(n+2,b,base) #define REST_8VSRS(n,b,base) REST_4VSRS(n,b,base); REST_4VSRS(n+4,b,base) #define REST_16VSRS(n,b,base) REST_8VSRS(n,b,base); REST_8VSRS(n+8,b,base) #define REST_32VSRS(n,b,base) REST_16VSRS(n,b,base); REST_16VSRS(n+16,b,base) -/* Save the upper 32 VSRs (32-63) in the thread VSX region (0-31) */ -#define SAVE_VSRU(n,b,base) li b,THREAD_VR0+(16*(n)); STXVD2X(n+32,R##base,R##b) -#define SAVE_2VSRSU(n,b,base) SAVE_VSRU(n,b,base); SAVE_VSRU(n+1,b,base) -#define SAVE_4VSRSU(n,b,base) SAVE_2VSRSU(n,b,base); SAVE_2VSRSU(n+2,b,base) -#define SAVE_8VSRSU(n,b,base) SAVE_4VSRSU(n,b,base); SAVE_4VSRSU(n+4,b,base) -#define SAVE_16VSRSU(n,b,base) SAVE_8VSRSU(n,b,base); SAVE_8VSRSU(n+8,b,base) -#define SAVE_32VSRSU(n,b,base) SAVE_16VSRSU(n,b,base); SAVE_16VSRSU(n+16,b,base) -#define REST_VSRU(n,b,base) li b,THREAD_VR0+(16*(n)); LXVD2X(n+32,R##base,R##b) -#define REST_2VSRSU(n,b,base) REST_VSRU(n,b,base); REST_VSRU(n+1,b,base) -#define REST_4VSRSU(n,b,base) REST_2VSRSU(n,b,base); REST_2VSRSU(n+2,b,base) -#define REST_8VSRSU(n,b,base) REST_4VSRSU(n,b,base); REST_4VSRSU(n+4,b,base) -#define REST_16VSRSU(n,b,base) REST_8VSRSU(n,b,base); REST_8VSRSU(n+8,b,base) -#define REST_32VSRSU(n,b,base) REST_16VSRSU(n,b,base); REST_16VSRSU(n+16,b,base) /* * b = base register for addressing, o = base offset from register of 1st EVR @@ -268,103 +179,117 @@ END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR) #define VCPU_GPR(n) __VCPU_GPR(__REG_##n) #ifdef __KERNEL__ -#ifdef CONFIG_PPC64 + +/* + * Used to name C functions called from asm + */ +#if defined(__powerpc64__) && defined(CONFIG_PPC_KERNEL_PCREL) +#define CFUNC(name) name@notoc +#else +#define CFUNC(name) name +#endif + +/* + * We use __powerpc64__ here because we want the compat VDSO to use the 32-bit + * version below in the else case of the ifdef. + */ +#ifdef __powerpc64__ #define STACKFRAMESIZE 256 #define __STK_REG(i) (112 + ((i)-14)*8) #define STK_REG(i) __STK_REG(__REG_##i) -#define __STK_PARAM(i) (48 + ((i)-3)*8) +#ifdef CONFIG_PPC64_ELF_ABI_V2 +#define STK_GOT 24 +#define STK_PARAM_AREA 32 +#else +#define STK_GOT 40 +#define STK_PARAM_AREA 48 +#endif + +#define __STK_PARAM(i) (STK_PARAM_AREA + ((i)-3)*8) #define STK_PARAM(i) __STK_PARAM(__REG_##i) -#define XGLUE(a,b) a##b -#define GLUE(a,b) XGLUE(a,b) +#ifdef CONFIG_PPC64_ELF_ABI_V2 #define _GLOBAL(name) \ - .section ".text"; \ .align 2 ; \ + .type name,@function; \ .globl name; \ - .globl GLUE(.,name); \ - .section ".opd","aw"; \ -name: \ - .quad GLUE(.,name); \ - .quad .TOC.@tocbase; \ - .quad 0; \ - .previous; \ - .type GLUE(.,name),@function; \ -GLUE(.,name): +name: -#define _INIT_GLOBAL(name) \ - __REF; \ +#ifdef CONFIG_PPC_KERNEL_PCREL +#define _GLOBAL_TOC _GLOBAL +#else +#define _GLOBAL_TOC(name) \ .align 2 ; \ + .type name,@function; \ .globl name; \ - .globl GLUE(.,name); \ - .section ".opd","aw"; \ name: \ - .quad GLUE(.,name); \ - .quad .TOC.@tocbase; \ - .quad 0; \ - .previous; \ - .type GLUE(.,name),@function; \ -GLUE(.,name): +0: addis r2,r12,(.TOC.-0b)@ha; \ + addi r2,r2,(.TOC.-0b)@l; \ + .localentry name,.-name +#endif + +#define DOTSYM(a) a + +#else -#define _KPROBE(name) \ - .section ".kprobes.text","a"; \ +#define XGLUE(a,b) a##b +#define GLUE(a,b) XGLUE(a,b) + +#define _GLOBAL(name) \ .align 2 ; \ .globl name; \ .globl GLUE(.,name); \ - .section ".opd","aw"; \ + .pushsection ".opd","aw"; \ name: \ .quad GLUE(.,name); \ .quad .TOC.@tocbase; \ .quad 0; \ - .previous; \ + .popsection; \ .type GLUE(.,name),@function; \ GLUE(.,name): -#define _STATIC(name) \ - .section ".text"; \ - .align 2 ; \ - .section ".opd","aw"; \ -name: \ - .quad GLUE(.,name); \ - .quad .TOC.@tocbase; \ - .quad 0; \ - .previous; \ - .type GLUE(.,name),@function; \ -GLUE(.,name): +#define _GLOBAL_TOC(name) _GLOBAL(name) -#define _INIT_STATIC(name) \ - __REF; \ - .align 2 ; \ - .section ".opd","aw"; \ -name: \ - .quad GLUE(.,name); \ - .quad .TOC.@tocbase; \ - .quad 0; \ - .previous; \ - .type GLUE(.,name),@function; \ -GLUE(.,name): +#define DOTSYM(a) GLUE(.,a) -#else /* 32-bit */ +#endif -#define _ENTRY(n) \ - .globl n; \ -n: +#else /* 32-bit */ #define _GLOBAL(n) \ - .text; \ - .stabs __stringify(n:F-1),N_FUN,0,0,n;\ .globl n; \ n: -#define _KPROBE(n) \ - .section ".kprobes.text","a"; \ - .globl n; \ -n: +#define _GLOBAL_TOC(name) _GLOBAL(name) +#define DOTSYM(a) a + +#endif + +/* + * __kprobes (the C annotation) puts the symbol into the .kprobes.text + * section, which gets emitted at the end of regular text. + * + * _ASM_NOKPROBE_SYMBOL and NOKPROBE_SYMBOL just adds the symbol to + * a blacklist. The former is for core kprobe functions/data, the + * latter is for those that incdentially must be excluded from probing + * and allows them to be linked at more optimal location within text. + */ +#ifdef CONFIG_KPROBES +#define _ASM_NOKPROBE_SYMBOL(entry) \ + .pushsection "_kprobe_blacklist","aw"; \ + PPC_LONG (entry) ; \ + .popsection +#else +#define _ASM_NOKPROBE_SYMBOL(entry) #endif +#define FUNC_START(name) _GLOBAL(name) +#define FUNC_END(name) + /* * LOAD_REG_IMMEDIATE(rn, expr) * Loads the value of the constant expression 'expr' into register 'rn' @@ -379,6 +304,11 @@ n: * you want to access various offsets within it). On ppc32 this is * identical to LOAD_REG_IMMEDIATE. * + * LOAD_REG_ADDR_PIC(rn, name) + * Loads the address of label 'name' into register 'run'. Use this when + * the kernel doesn't run at the linked or relocated address. Please + * note that this macro will clobber the lr register. + * * LOAD_REG_ADDRBASE(rn, name) * ADDROFF(name) * LOAD_REG_ADDRBASE loads part of the address of label 'name' into @@ -389,16 +319,88 @@ n: * LOAD_REG_ADDRBASE(rX, name) * ld rY,ADDROFF(name)(rX) */ + +/* Be careful, this will clobber the lr register. */ +#define LOAD_REG_ADDR_PIC(reg, name) \ + bcl 20,31,$+4; \ +0: mflr reg; \ + addis reg,reg,(name - 0b)@ha; \ + addi reg,reg,(name - 0b)@l; + +#if defined(__powerpc64__) && defined(HAVE_AS_ATHIGH) +#define __AS_ATHIGH high +#else +#define __AS_ATHIGH h +#endif + +.macro __LOAD_REG_IMMEDIATE_32 r, x + .if (\x) >= 0x8000 || (\x) < -0x8000 + lis \r, (\x)@__AS_ATHIGH + .if (\x) & 0xffff != 0 + ori \r, \r, (\x)@l + .endif + .else + li \r, (\x)@l + .endif +.endm + +.macro __LOAD_REG_IMMEDIATE r, x + .if (\x) >= 0x80000000 || (\x) < -0x80000000 + __LOAD_REG_IMMEDIATE_32 \r, (\x) >> 32 + sldi \r, \r, 32 + .if (\x) & 0xffff0000 != 0 + oris \r, \r, (\x)@__AS_ATHIGH + .endif + .if (\x) & 0xffff != 0 + ori \r, \r, (\x)@l + .endif + .else + __LOAD_REG_IMMEDIATE_32 \r, \x + .endif +.endm + #ifdef __powerpc64__ -#define LOAD_REG_IMMEDIATE(reg,expr) \ - lis reg,(expr)@highest; \ - ori reg,reg,(expr)@higher; \ - rldicr reg,reg,32,31; \ - oris reg,reg,(expr)@h; \ - ori reg,reg,(expr)@l; +#ifdef CONFIG_PPC_KERNEL_PCREL +#define __LOAD_PACA_TOC(reg) \ + li reg,-1 +#else +#define __LOAD_PACA_TOC(reg) \ + ld reg,PACATOC(r13) +#endif + +#define LOAD_PACA_TOC() \ + __LOAD_PACA_TOC(r2) + +#define LOAD_REG_IMMEDIATE(reg, expr) __LOAD_REG_IMMEDIATE reg, expr + +#define LOAD_REG_IMMEDIATE_SYM(reg, tmp, expr) \ + lis tmp, (expr)@highest; \ + lis reg, (expr)@__AS_ATHIGH; \ + ori tmp, tmp, (expr)@higher; \ + ori reg, reg, (expr)@l; \ + rldimi reg, tmp, 32, 0 + +#ifdef CONFIG_PPC_KERNEL_PCREL #define LOAD_REG_ADDR(reg,name) \ - ld reg,name@got(r2) + pla reg,name@pcrel + +#else +#define LOAD_REG_ADDR(reg,name) \ + addis reg,r2,name@toc@ha; \ + addi reg,reg,name@toc@l +#endif + +#ifdef CONFIG_PPC_BOOK3E_64 +/* + * This is used in register-constrained interrupt handlers. Not to be used + * by BOOK3S. ld complains with "got/toc optimization is not supported" if r2 + * is not used for the TOC offset, so use @got(tocreg). If the interrupt + * handlers saved r2 instead, LOAD_REG_ADDR could be used. + */ +#define LOAD_REG_ADDR_ALTTOC(reg,tocreg,name) \ + ld reg,name@got(tocreg) +#endif #define LOAD_REG_ADDRBASE(reg,name) LOAD_REG_ADDR(reg,name) #define ADDROFF(name) 0 @@ -406,13 +408,24 @@ n: /* offsets for stack frame layout */ #define LRSAVE 16 +/* + * GCC stack frames follow a different pattern on 32 vs 64. This can be used + * to make asm frames be consistent with C. + */ +#define PPC_CREATE_STACK_FRAME(size) \ + mflr r0; \ + std r0,16(r1); \ + stdu r1,-(size)(r1) + #else /* 32-bit */ -#define LOAD_REG_IMMEDIATE(reg,expr) \ +#define LOAD_REG_IMMEDIATE(reg, expr) __LOAD_REG_IMMEDIATE_32 reg, expr + +#define LOAD_REG_IMMEDIATE_SYM(reg,expr) \ lis reg,(expr)@ha; \ addi reg,reg,(expr)@l; -#define LOAD_REG_ADDR(reg,name) LOAD_REG_IMMEDIATE(reg, name) +#define LOAD_REG_ADDR(reg,name) LOAD_REG_IMMEDIATE_SYM(reg, name) #define LOAD_REG_ADDRBASE(reg, name) lis reg,name@ha #define ADDROFF(name) name@l @@ -420,49 +433,37 @@ n: /* offsets for stack frame layout */ #define LRSAVE 4 -#endif +#define PPC_CREATE_STACK_FRAME(size) \ + stwu r1,-(size)(r1); \ + mflr r0; \ + stw r0,(size+4)(r1) -/* various errata or part fixups */ -#ifdef CONFIG_PPC601_SYNC_FIX -#define SYNC \ -BEGIN_FTR_SECTION \ - sync; \ - isync; \ -END_FTR_SECTION_IFSET(CPU_FTR_601) -#define SYNC_601 \ -BEGIN_FTR_SECTION \ - sync; \ -END_FTR_SECTION_IFSET(CPU_FTR_601) -#define ISYNC_601 \ -BEGIN_FTR_SECTION \ - isync; \ -END_FTR_SECTION_IFSET(CPU_FTR_601) -#else -#define SYNC -#define SYNC_601 -#define ISYNC_601 #endif -#ifdef CONFIG_PPC_CELL +/* various errata or part fixups */ +#if defined(CONFIG_PPC_CELL) || defined(CONFIG_PPC_E500) #define MFTB(dest) \ -90: mftb dest; \ +90: mfspr dest, SPRN_TBRL; \ BEGIN_FTR_SECTION_NESTED(96); \ cmpwi dest,0; \ beq- 90b; \ END_FTR_SECTION_NESTED(CPU_FTR_CELL_TB_BUG, CPU_FTR_CELL_TB_BUG, 96) #else -#define MFTB(dest) mftb dest +#define MFTB(dest) MFTBL(dest) +#endif + +#ifdef CONFIG_PPC_8xx +#define MFTBL(dest) mftb dest +#define MFTBU(dest) mftbu dest +#else +#define MFTBL(dest) mfspr dest, SPRN_TBRL +#define MFTBU(dest) mfspr dest, SPRN_TBRU #endif #ifndef CONFIG_SMP #define TLBSYNC -#else /* CONFIG_SMP */ -/* tlbsync is not implemented on 601 */ -#define TLBSYNC \ -BEGIN_FTR_SECTION \ - tlbsync; \ - sync; \ -END_FTR_SECTION_IFCLR(CPU_FTR_601) +#else +#define TLBSYNC tlbsync; sync #endif #ifdef CONFIG_PPC64 @@ -472,31 +473,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_601) FTR_SECTION_ELSE_NESTED(848); \ mtocrf (FXM), RS; \ ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_NOEXECUTE, 848) - -/* - * PPR restore macros used in entry_64.S - * Used for P7 or later processors - */ -#define HMT_MEDIUM_LOW_HAS_PPR \ -BEGIN_FTR_SECTION_NESTED(944) \ - HMT_MEDIUM_LOW; \ -END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,944) - -#define SET_DEFAULT_THREAD_PPR(ra, rb) \ -BEGIN_FTR_SECTION_NESTED(945) \ - lis ra,INIT_PPR@highest; /* default ppr=3 */ \ - ld rb,PACACURRENT(r13); \ - sldi ra,ra,32; /* 11- 13 bits are used for ppr */ \ - std ra,TASKTHREADPPR(rb); \ -END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,945) - -#define RESTORE_PPR(ra, rb) \ -BEGIN_FTR_SECTION_NESTED(946) \ - ld ra,PACACURRENT(r13); \ - ld rb,TASKTHREADPPR(ra); \ - mtspr SPRN_PPR,rb; /* Restore PPR */ \ -END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,946) - #endif /* @@ -506,12 +482,15 @@ END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,946) * and they must be used. */ -#if !defined(CONFIG_4xx) && !defined(CONFIG_8xx) +#if !defined(CONFIG_44x) && !defined(CONFIG_PPC_8xx) #define tlbia \ li r4,1024; \ mtctr r4; \ lis r4,KERNELBASE@h; \ + .machine push; \ + .machine "power4"; \ 0: tlbie r4; \ + .machine pop; \ addi r4,r4,0x1000; \ bdnz 0b #endif @@ -525,14 +504,29 @@ END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,946) /* The following stops all load and store data streams associated with stream * ID (ie. streams created explicitly). The embedded and server mnemonics for - * dcbt are different so we use machine "power4" here explicitly. + * dcbt are different so this must only be used for server. */ -#define DCBT_STOP_ALL_STREAM_IDS(scratch) \ -.machine push ; \ -.machine "power4" ; \ - lis scratch,0x60000000@h; \ - dcbt r0,scratch,0b01010; \ -.machine pop +#define DCBT_BOOK3S_STOP_ALL_STREAM_IDS(scratch) \ + lis scratch,0x60000000@h; \ + .machine push; \ + .machine power4; \ + dcbt 0,scratch,0b01010; \ + .machine pop; + +#define DCBT_SETUP_STREAMS(from, from_parms, to, to_parms, scratch) \ + lis scratch,0x8000; /* GO=1 */ \ + clrldi scratch,scratch,32; \ + .machine push; \ + .machine power4; \ + /* setup read stream 0 */ \ + dcbt 0,from,0b01000; /* addr from */ \ + dcbt 0,from_parms,0b01010; /* length and depth from */ \ + /* setup write stream 1 */ \ + dcbtst 0,to,0b01000; /* addr to */ \ + dcbtst 0,to_parms,0b01010; /* length and depth to */ \ + eieio; \ + dcbt 0,scratch,0b01010; /* all streams GO */ \ + .machine pop; /* * toreal/fromreal/tophys/tovirt macros. 32-bit BookE makes them @@ -570,42 +564,19 @@ END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,946) ori rd,rd,((KERNELBASE>>48)&0xFFFF);\ rotldi rd,rd,48 #else -/* - * On APUS (Amiga PowerPC cpu upgrade board), we don't know the - * physical base address of RAM at compile time. - */ #define toreal(rd) tophys(rd,rd) #define fromreal(rd) tovirt(rd,rd) -#define tophys(rd,rs) \ -0: addis rd,rs,-PAGE_OFFSET@h; \ - .section ".vtop_fixup","aw"; \ - .align 1; \ - .long 0b; \ - .previous - -#define tovirt(rd,rs) \ -0: addis rd,rs,PAGE_OFFSET@h; \ - .section ".ptov_fixup","aw"; \ - .align 1; \ - .long 0b; \ - .previous +#define tophys(rd, rs) addis rd, rs, -PAGE_OFFSET@h +#define tovirt(rd, rs) addis rd, rs, PAGE_OFFSET@h #endif #ifdef CONFIG_PPC_BOOK3S_64 -#define RFI rfid #define MTMSRD(r) mtmsrd r #define MTMSR_EERI(reg) mtmsrd reg,1 #else -#define FIX_SRR1(ra, rb) -#ifndef CONFIG_40x -#define RFI rfi -#else -#define RFI rfi; b . /* Prevent prefetch past rfi */ -#endif #define MTMSRD(r) mtmsr r #define MTMSR_EERI(reg) mtmsr reg -#define CLR_TOP32(r) #endif #endif /* __KERNEL__ */ @@ -703,105 +674,105 @@ END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,946) /* AltiVec Registers (VPRs) */ -#define vr0 0 -#define vr1 1 -#define vr2 2 -#define vr3 3 -#define vr4 4 -#define vr5 5 -#define vr6 6 -#define vr7 7 -#define vr8 8 -#define vr9 9 -#define vr10 10 -#define vr11 11 -#define vr12 12 -#define vr13 13 -#define vr14 14 -#define vr15 15 -#define vr16 16 -#define vr17 17 -#define vr18 18 -#define vr19 19 -#define vr20 20 -#define vr21 21 -#define vr22 22 -#define vr23 23 -#define vr24 24 -#define vr25 25 -#define vr26 26 -#define vr27 27 -#define vr28 28 -#define vr29 29 -#define vr30 30 -#define vr31 31 +#define v0 0 +#define v1 1 +#define v2 2 +#define v3 3 +#define v4 4 +#define v5 5 +#define v6 6 +#define v7 7 +#define v8 8 +#define v9 9 +#define v10 10 +#define v11 11 +#define v12 12 +#define v13 13 +#define v14 14 +#define v15 15 +#define v16 16 +#define v17 17 +#define v18 18 +#define v19 19 +#define v20 20 +#define v21 21 +#define v22 22 +#define v23 23 +#define v24 24 +#define v25 25 +#define v26 26 +#define v27 27 +#define v28 28 +#define v29 29 +#define v30 30 +#define v31 31 /* VSX Registers (VSRs) */ -#define vsr0 0 -#define vsr1 1 -#define vsr2 2 -#define vsr3 3 -#define vsr4 4 -#define vsr5 5 -#define vsr6 6 -#define vsr7 7 -#define vsr8 8 -#define vsr9 9 -#define vsr10 10 -#define vsr11 11 -#define vsr12 12 -#define vsr13 13 -#define vsr14 14 -#define vsr15 15 -#define vsr16 16 -#define vsr17 17 -#define vsr18 18 -#define vsr19 19 -#define vsr20 20 -#define vsr21 21 -#define vsr22 22 -#define vsr23 23 -#define vsr24 24 -#define vsr25 25 -#define vsr26 26 -#define vsr27 27 -#define vsr28 28 -#define vsr29 29 -#define vsr30 30 -#define vsr31 31 -#define vsr32 32 -#define vsr33 33 -#define vsr34 34 -#define vsr35 35 -#define vsr36 36 -#define vsr37 37 -#define vsr38 38 -#define vsr39 39 -#define vsr40 40 -#define vsr41 41 -#define vsr42 42 -#define vsr43 43 -#define vsr44 44 -#define vsr45 45 -#define vsr46 46 -#define vsr47 47 -#define vsr48 48 -#define vsr49 49 -#define vsr50 50 -#define vsr51 51 -#define vsr52 52 -#define vsr53 53 -#define vsr54 54 -#define vsr55 55 -#define vsr56 56 -#define vsr57 57 -#define vsr58 58 -#define vsr59 59 -#define vsr60 60 -#define vsr61 61 -#define vsr62 62 -#define vsr63 63 +#define vs0 0 +#define vs1 1 +#define vs2 2 +#define vs3 3 +#define vs4 4 +#define vs5 5 +#define vs6 6 +#define vs7 7 +#define vs8 8 +#define vs9 9 +#define vs10 10 +#define vs11 11 +#define vs12 12 +#define vs13 13 +#define vs14 14 +#define vs15 15 +#define vs16 16 +#define vs17 17 +#define vs18 18 +#define vs19 19 +#define vs20 20 +#define vs21 21 +#define vs22 22 +#define vs23 23 +#define vs24 24 +#define vs25 25 +#define vs26 26 +#define vs27 27 +#define vs28 28 +#define vs29 29 +#define vs30 30 +#define vs31 31 +#define vs32 32 +#define vs33 33 +#define vs34 34 +#define vs35 35 +#define vs36 36 +#define vs37 37 +#define vs38 38 +#define vs39 39 +#define vs40 40 +#define vs41 41 +#define vs42 42 +#define vs43 43 +#define vs44 44 +#define vs45 45 +#define vs46 46 +#define vs47 47 +#define vs48 48 +#define vs49 49 +#define vs50 50 +#define vs51 51 +#define vs52 52 +#define vs53 53 +#define vs54 54 +#define vs55 55 +#define vs56 56 +#define vs57 57 +#define vs58 58 +#define vs59 59 +#define vs60 60 +#define vs61 61 +#define vs62 62 +#define vs63 63 /* SPE Registers (EVPRs) */ @@ -838,12 +809,98 @@ END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,946) #define evr30 30 #define evr31 31 -/* some stab codes */ -#define N_FUN 36 -#define N_RSYM 64 -#define N_SLINE 68 -#define N_SO 100 +#define RFSCV .long 0x4c0000a4 + +/* + * Create an endian fixup trampoline + * + * This starts with a "tdi 0,0,0x48" instruction which is + * essentially a "trap never", and thus akin to a nop. + * + * The opcode for this instruction read with the wrong endian + * however results in a b . + 8 + * + * So essentially we use that trick to execute the following + * trampoline in "reverse endian" if we are running with the + * MSR_LE bit set the "wrong" way for whatever endianness the + * kernel is built for. + */ -#endif /* __ASSEMBLY__ */ +#ifdef CONFIG_PPC_BOOK3E_64 +#define FIXUP_ENDIAN +#else +/* + * This version may be used in HV or non-HV context. + * MSR[EE] must be disabled. + */ +#define FIXUP_ENDIAN \ + tdi 0,0,0x48; /* Reverse endian of b . + 8 */ \ + b 191f; /* Skip trampoline if endian is good */ \ + .long 0xa600607d; /* mfmsr r11 */ \ + .long 0x01006b69; /* xori r11,r11,1 */ \ + .long 0x00004039; /* li r10,0 */ \ + .long 0x6401417d; /* mtmsrd r10,1 */ \ + .long 0x05009f42; /* bcl 20,31,$+4 */ \ + .long 0xa602487d; /* mflr r10 */ \ + .long 0x14004a39; /* addi r10,r10,20 */ \ + .long 0xa6035a7d; /* mtsrr0 r10 */ \ + .long 0xa6037b7d; /* mtsrr1 r11 */ \ + .long 0x2400004c; /* rfid */ \ +191: + +/* + * This version that may only be used with MSR[HV]=1 + * - Does not clear MSR[RI], so more robust. + * - Slightly smaller and faster. + */ +#define FIXUP_ENDIAN_HV \ + tdi 0,0,0x48; /* Reverse endian of b . + 8 */ \ + b 191f; /* Skip trampoline if endian is good */ \ + .long 0xa600607d; /* mfmsr r11 */ \ + .long 0x01006b69; /* xori r11,r11,1 */ \ + .long 0x05009f42; /* bcl 20,31,$+4 */ \ + .long 0xa602487d; /* mflr r10 */ \ + .long 0x14004a39; /* addi r10,r10,20 */ \ + .long 0xa64b5a7d; /* mthsrr0 r10 */ \ + .long 0xa64b7b7d; /* mthsrr1 r11 */ \ + .long 0x2402004c; /* hrfid */ \ +191: + +#endif /* !CONFIG_PPC_BOOK3E_64 */ + +#endif /* __ASSEMBLER__ */ + +#define SOFT_MASK_TABLE(_start, _end) \ + stringify_in_c(.section __soft_mask_table,"a";)\ + stringify_in_c(.balign 8;) \ + stringify_in_c(.llong (_start);) \ + stringify_in_c(.llong (_end);) \ + stringify_in_c(.previous) + +#define RESTART_TABLE(_start, _end, _target) \ + stringify_in_c(.section __restart_table,"a";)\ + stringify_in_c(.balign 8;) \ + stringify_in_c(.llong (_start);) \ + stringify_in_c(.llong (_end);) \ + stringify_in_c(.llong (_target);) \ + stringify_in_c(.previous) + +#ifdef CONFIG_PPC_E500 +#define BTB_FLUSH(reg) \ + lis reg,BUCSR_INIT@h; \ + ori reg,reg,BUCSR_INIT@l; \ + mtspr SPRN_BUCSR,reg; \ + isync; +#else +#define BTB_FLUSH(reg) +#endif /* CONFIG_PPC_E500 */ + +#if defined(CONFIG_PPC64_ELF_ABI_V1) +#define STACK_FRAME_PARAMS 48 +#elif defined(CONFIG_PPC64_ELF_ABI_V2) +#define STACK_FRAME_PARAMS 32 +#elif defined(CONFIG_PPC32) +#define STACK_FRAME_PARAMS 8 +#endif #endif /* _ASM_POWERPC_PPC_ASM_H */ diff --git a/arch/powerpc/include/asm/preempt.h b/arch/powerpc/include/asm/preempt.h new file mode 100644 index 000000000000..000e2b9681f3 --- /dev/null +++ b/arch/powerpc/include/asm/preempt.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_POWERPC_PREEMPT_H +#define __ASM_POWERPC_PREEMPT_H + +#include <asm-generic/preempt.h> + +#if defined(CONFIG_PREEMPT_DYNAMIC) +#include <linux/jump_label.h> +DECLARE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); +#define need_irq_preemption() \ + (static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched)) +#else +#define need_irq_preemption() (IS_ENABLED(CONFIG_PREEMPTION)) +#endif + +#endif /* __ASM_POWERPC_PREEMPT_H */ diff --git a/arch/powerpc/include/asm/probes.h b/arch/powerpc/include/asm/probes.h index 3421637cfd7b..e77a2ed7d938 100644 --- a/arch/powerpc/include/asm/probes.h +++ b/arch/powerpc/include/asm/probes.h @@ -1,29 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _ASM_POWERPC_PROBES_H #define _ASM_POWERPC_PROBES_H #ifdef __KERNEL__ /* * Definitions common to probes files * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * * Copyright IBM Corporation, 2012 */ #include <linux/types.h> +#include <asm/disassemble.h> +#include <asm/ppc-opcode.h> -typedef u32 ppc_opcode_t; -#define BREAKPOINT_INSTRUCTION 0x7fe00008 /* trap */ +#define BREAKPOINT_INSTRUCTION PPC_RAW_TRAP() /* trap */ /* Trap definitions per ISA */ #define IS_TW(instr) (((instr) & 0xfc0007fe) == 0x7c000008) @@ -44,17 +32,52 @@ typedef u32 ppc_opcode_t; #define MSR_SINGLESTEP (MSR_SE) #endif +static inline bool can_single_step(u32 inst) +{ + switch (get_op(inst)) { + case OP_TRAP_64: return false; + case OP_TRAP: return false; + case OP_SC: return false; + case OP_19: + switch (get_xop(inst)) { + case OP_19_XOP_RFID: return false; + case OP_19_XOP_RFMCI: return false; + case OP_19_XOP_RFDI: return false; + case OP_19_XOP_RFI: return false; + case OP_19_XOP_RFCI: return false; + case OP_19_XOP_RFSCV: return false; + case OP_19_XOP_HRFID: return false; + case OP_19_XOP_URFID: return false; + case OP_19_XOP_STOP: return false; + case OP_19_XOP_DOZE: return false; + case OP_19_XOP_NAP: return false; + case OP_19_XOP_SLEEP: return false; + case OP_19_XOP_RVWINKLE: return false; + } + break; + case OP_31: + switch (get_xop(inst)) { + case OP_31_XOP_TRAP: return false; + case OP_31_XOP_TRAP_64: return false; + case OP_31_XOP_MTMSR: return false; + case OP_31_XOP_MTMSRD: return false; + } + break; + } + return true; +} + /* Enable single stepping for the current task */ static inline void enable_single_step(struct pt_regs *regs) { - regs->msr |= MSR_SINGLESTEP; + regs_set_return_msr(regs, regs->msr | MSR_SINGLESTEP); #ifdef CONFIG_PPC_ADV_DEBUG_REGS /* * We turn off Critical Input Exception(CE) to ensure that the single * step will be for the instruction we have the probe on; if we don't, * it is possible we'd get the single step reported for CE. */ - regs->msr &= ~MSR_CE; + regs_set_return_msr(regs, regs->msr & ~MSR_CE); mtspr(SPRN_DBCR0, mfspr(SPRN_DBCR0) | DBCR0_IC | DBCR0_IDM); #ifdef CONFIG_PPC_47x isync(); diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h index 47a35b08b963..f156bdb43e2b 100644 --- a/arch/powerpc/include/asm/processor.h +++ b/arch/powerpc/include/asm/processor.h @@ -1,38 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _ASM_POWERPC_PROCESSOR_H #define _ASM_POWERPC_PROCESSOR_H /* * Copyright (C) 2001 PPC 64 Team, IBM Corp - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ +#include <vdso/processor.h> + #include <asm/reg.h> #ifdef CONFIG_VSX #define TS_FPRWIDTH 2 + +#ifdef __BIG_ENDIAN__ +#define TS_FPROFFSET 0 +#define TS_VSRLOWOFFSET 1 +#else +#define TS_FPROFFSET 1 +#define TS_VSRLOWOFFSET 0 +#endif + #else #define TS_FPRWIDTH 1 +#define TS_FPROFFSET 0 #endif #ifdef CONFIG_PPC64 /* Default SMT priority is set to 3. Use 11- 13bits to save priority. */ #define PPR_PRIORITY 3 -#ifdef __ASSEMBLY__ -#define INIT_PPR (PPR_PRIORITY << 50) +#ifdef __ASSEMBLER__ +#define DEFAULT_PPR (PPR_PRIORITY << 50) #else -#define INIT_PPR ((u64)PPR_PRIORITY << 50) -#endif /* __ASSEMBLY__ */ +#define DEFAULT_PPR ((u64)PPR_PRIORITY << 50) +#endif /* __ASSEMBLER__ */ #endif /* CONFIG_PPC64 */ -#ifndef __ASSEMBLY__ -#include <linux/compiler.h> -#include <linux/cache.h> +#ifndef __ASSEMBLER__ +#include <linux/types.h> +#include <linux/thread_info.h> #include <asm/ptrace.h> -#include <asm/types.h> #include <asm/hw_breakpoint.h> /* We do _not_ want to define new machine types at all, those must die @@ -58,111 +65,33 @@ extern int _chrp_type; #endif /* defined(__KERNEL__) && defined(CONFIG_PPC32) */ -/* - * Default implementation of macro that returns current - * instruction pointer ("program counter"). - */ -#define current_text_addr() ({ __label__ _l; _l: &&_l;}) - -/* Macros for adjusting thread priority (hardware multi-threading) */ -#define HMT_very_low() asm volatile("or 31,31,31 # very low priority") -#define HMT_low() asm volatile("or 1,1,1 # low priority") -#define HMT_medium_low() asm volatile("or 6,6,6 # medium low priority") -#define HMT_medium() asm volatile("or 2,2,2 # medium priority") -#define HMT_medium_high() asm volatile("or 5,5,5 # medium high priority") -#define HMT_high() asm volatile("or 3,3,3 # high priority") - #ifdef __KERNEL__ -struct task_struct; -void start_thread(struct pt_regs *regs, unsigned long fdptr, unsigned long sp); -void release_thread(struct task_struct *); - -/* Lazy FPU handling on uni-processor */ -extern struct task_struct *last_task_used_math; -extern struct task_struct *last_task_used_altivec; -extern struct task_struct *last_task_used_vsx; -extern struct task_struct *last_task_used_spe; - -#ifdef CONFIG_PPC32 - -#if CONFIG_TASK_SIZE > CONFIG_KERNEL_START -#error User TASK_SIZE overlaps with KERNEL_START address -#endif -#define TASK_SIZE (CONFIG_TASK_SIZE) - -/* This decides where the kernel will search for a free chunk of vm - * space during mmap's. - */ -#define TASK_UNMAPPED_BASE (TASK_SIZE / 8 * 3) -#endif - #ifdef CONFIG_PPC64 -/* 64-bit user address space is 46-bits (64TB user VM) */ -#define TASK_SIZE_USER64 (0x0000400000000000UL) - -/* - * 32-bit user address space is 4GB - 1 page - * (this 1 page is needed so referencing of 0xFFFFFFFF generates EFAULT - */ -#define TASK_SIZE_USER32 (0x0000000100000000UL - (1*PAGE_SIZE)) - -#define TASK_SIZE_OF(tsk) (test_tsk_thread_flag(tsk, TIF_32BIT) ? \ - TASK_SIZE_USER32 : TASK_SIZE_USER64) -#define TASK_SIZE TASK_SIZE_OF(current) - -/* This decides where the kernel will search for a free chunk of vm - * space during mmap's. - */ -#define TASK_UNMAPPED_BASE_USER32 (PAGE_ALIGN(TASK_SIZE_USER32 / 4)) -#define TASK_UNMAPPED_BASE_USER64 (PAGE_ALIGN(TASK_SIZE_USER64 / 4)) - -#define TASK_UNMAPPED_BASE ((is_32bit_task()) ? \ - TASK_UNMAPPED_BASE_USER32 : TASK_UNMAPPED_BASE_USER64 ) +#include <asm/task_size_64.h> +#else +#include <asm/task_size_32.h> #endif -#ifdef __powerpc64__ - -#define STACK_TOP_USER64 TASK_SIZE_USER64 -#define STACK_TOP_USER32 TASK_SIZE_USER32 - -#define STACK_TOP (is_32bit_task() ? \ - STACK_TOP_USER32 : STACK_TOP_USER64) - -#define STACK_TOP_MAX STACK_TOP_USER64 - -#else /* __powerpc64__ */ - -#define STACK_TOP TASK_SIZE -#define STACK_TOP_MAX STACK_TOP - -#endif /* __powerpc64__ */ +struct task_struct; +void start_thread(struct pt_regs *regs, unsigned long fdptr, unsigned long sp); -typedef struct { - unsigned long seg; -} mm_segment_t; +#define TS_FPR(i) fp_state.fpr[i][TS_FPROFFSET] +#define TS_CKFPR(i) ckfp_state.fpr[i][TS_FPROFFSET] -#define TS_FPROFFSET 0 -#define TS_VSRLOWOFFSET 1 -#define TS_FPR(i) fpr[i][TS_FPROFFSET] -#define TS_TRANS_FPR(i) transact_fpr[i][TS_FPROFFSET] +/* FP and VSX 0-31 register set */ +struct thread_fp_state { + u64 fpr[32][TS_FPRWIDTH] __attribute__((aligned(16))); + u64 fpscr; /* Floating point status */ +}; -struct thread_struct { - unsigned long ksp; /* Kernel stack pointer */ - unsigned long ksp_limit; /* if ksp <= ksp_limit stack overflow */ +/* Complete AltiVec register set including VSCR */ +struct thread_vr_state { + vector128 vr[32] __attribute__((aligned(16))); + vector128 vscr __attribute__((aligned(16))); +}; -#ifdef CONFIG_PPC64 - unsigned long ksp_vsid; -#endif - struct pt_regs *regs; /* Pointer to saved register state */ - mm_segment_t fs; /* for get_fs() validation */ -#ifdef CONFIG_BOOKE - /* BookE base exception scratch space; align on cacheline */ - unsigned long normsave[8] ____cacheline_aligned; -#endif -#ifdef CONFIG_PPC32 - void *pgdir; /* root of page-table tree */ -#endif +struct debug_reg { #ifdef CONFIG_PPC_ADV_DEBUG_REGS /* * The following help to manage the use of Debug Control Registers @@ -199,74 +128,100 @@ struct thread_struct { unsigned long dvc2; #endif #endif - /* FP and VSX 0-31 register set */ - double fpr[32][TS_FPRWIDTH] __attribute__((aligned(16))); - struct { +}; + +struct thread_struct { + unsigned long ksp; /* Kernel stack pointer */ - unsigned int pad; - unsigned int val; /* Floating point status */ - } fpscr; +#ifdef CONFIG_PPC64 + unsigned long ksp_vsid; +#endif + struct pt_regs *regs; /* Pointer to saved register state */ +#ifdef CONFIG_BOOKE + /* BookE base exception scratch space; align on cacheline */ + unsigned long normsave[8] ____cacheline_aligned; +#endif +#ifdef CONFIG_PPC32 + void *pgdir; /* root of page-table tree */ +#ifdef CONFIG_PPC_RTAS + unsigned long rtas_sp; /* stack pointer for when in RTAS */ +#endif +#if defined(CONFIG_PPC_BOOK3S_32) && defined(CONFIG_PPC_KUAP) + unsigned long kuap; /* opened segments for user access */ +#endif + unsigned long srr0; + unsigned long srr1; + unsigned long dar; + unsigned long dsisr; +#ifdef CONFIG_PPC_BOOK3S_32 + unsigned long r0, r3, r4, r5, r6, r8, r9, r11; + unsigned long lr, ctr; + unsigned long sr0; +#endif +#endif /* CONFIG_PPC32 */ +#if defined(CONFIG_BOOKE) && defined(CONFIG_PPC_KUAP) + unsigned long pid; /* value written in PID reg. at interrupt exit */ +#endif + /* Debug Registers */ + struct debug_reg debug; +#ifdef CONFIG_PPC_FPU_REGS + struct thread_fp_state fp_state; + struct thread_fp_state *fp_save_area; +#endif int fpexc_mode; /* floating-point exception mode */ unsigned int align_ctl; /* alignment handling control */ -#ifdef CONFIG_PPC64 - unsigned long start_tb; /* Start purr when proc switched in */ - unsigned long accum_tb; /* Total accumilated purr for process */ #ifdef CONFIG_HAVE_HW_BREAKPOINT - struct perf_event *ptrace_bps[HBP_NUM]; - /* - * Helps identify source of single-step exception and subsequent - * hw-breakpoint enablement - */ - struct perf_event *last_hit_ubp; + struct perf_event *ptrace_bps[HBP_NUM_MAX]; #endif /* CONFIG_HAVE_HW_BREAKPOINT */ -#endif - struct arch_hw_breakpoint hw_brk; /* info on the hardware breakpoint */ + struct arch_hw_breakpoint hw_brk[HBP_NUM_MAX]; /* hardware breakpoint info */ unsigned long trap_nr; /* last trap # on this thread */ + u8 load_slb; /* Ages out SLB preload cache entries */ + u8 load_fp; #ifdef CONFIG_ALTIVEC - /* Complete AltiVec register set */ - vector128 vr[32] __attribute__((aligned(16))); - /* AltiVec status */ - vector128 vscr __attribute__((aligned(16))); + u8 load_vec; + struct thread_vr_state vr_state; + struct thread_vr_state *vr_save_area; unsigned long vrsave; int used_vr; /* set if process has used altivec */ #endif /* CONFIG_ALTIVEC */ #ifdef CONFIG_VSX /* VSR status */ - int used_vsr; /* set if process has used altivec */ + int used_vsr; /* set if process has used VSX */ #endif /* CONFIG_VSX */ #ifdef CONFIG_SPE - unsigned long evr[32]; /* upper 32-bits of SPE regs */ - u64 acc; /* Accumulator */ + struct_group(spe, + unsigned long evr[32]; /* upper 32-bits of SPE regs */ + u64 acc; /* Accumulator */ + ); unsigned long spefscr; /* SPE & eFP status */ + unsigned long spefscr_last; /* SPEFSCR value on last prctl + call or trap return */ int used_spe; /* set if process has used spe */ #endif /* CONFIG_SPE */ #ifdef CONFIG_PPC_TRANSACTIONAL_MEM + u8 load_tm; u64 tm_tfhar; /* Transaction fail handler addr */ u64 tm_texasr; /* Transaction exception & summary */ u64 tm_tfiar; /* Transaction fail instr address reg */ - unsigned long tm_orig_msr; /* Thread's MSR on ctx switch */ struct pt_regs ckpt_regs; /* Checkpointed registers */ + unsigned long tm_tar; + unsigned long tm_ppr; + unsigned long tm_dscr; + unsigned long tm_amr; + /* - * Transactional FP and VSX 0-31 register set. - * NOTE: the sense of these is the opposite of the integer ckpt_regs! + * Checkpointed FP and VSX 0-31 register set. * * When a transaction is active/signalled/scheduled etc., *regs is the * most recent set of/speculated GPRs with ckpt_regs being the older * checkpointed regs to which we roll back if transaction aborts. * - * However, fpr[] is the checkpointed 'base state' of FP regs, and - * transact_fpr[] is the new set of transactional values. - * VRs work the same way. + * These are analogous to how ckpt_regs and pt_regs work */ - double transact_fpr[32][TS_FPRWIDTH]; - struct { - unsigned int pad; - unsigned int val; /* Floating point status */ - } transact_fpscr; - vector128 transact_vr[32] __attribute__((aligned(16))); - vector128 transact_vscr __attribute__((aligned(16))); - unsigned long transact_vrsave; + struct thread_fp_state ckfp_state; /* Checkpointed FP state */ + struct thread_vr_state ckvr_state; /* Checkpointed VR state */ + unsigned long ckvrsave; /* Checkpointed VRSAVE */ #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ #ifdef CONFIG_KVM_BOOK3S_32_HANDLER void* kvm_shadow_vcpu; /* KVM internal data */ @@ -276,8 +231,18 @@ struct thread_struct { #endif #ifdef CONFIG_PPC64 unsigned long dscr; + unsigned long fscr; + /* + * This member element dscr_inherit indicates that the process + * has explicitly attempted and changed the DSCR register value + * for itself. Hence kernel wont use the default CPU DSCR value + * contained in the PACA structure anymore during process context + * switch. Once this variable is set, this behaviour will also be + * inherited to all the children of this process from that point + * onwards. + */ int dscr_inherit; - unsigned long ppr; /* used to save/restore SMT priority */ + unsigned long tidr; #endif #ifdef CONFIG_PPC_BOOK3S_64 unsigned long tar; @@ -289,53 +254,63 @@ struct thread_struct { unsigned long sier; unsigned long mmcr2; unsigned mmcr0; + unsigned used_ebb; + unsigned long mmcr3; + unsigned long sier2; + unsigned long sier3; + unsigned long hashkeyr; + unsigned long dexcr; + unsigned long dexcr_onexec; /* Reset value to load on exec */ #endif }; #define ARCH_MIN_TASKALIGN 16 #define INIT_SP (sizeof(init_stack) + (unsigned long) &init_stack) -#define INIT_SP_LIMIT \ - (_ALIGN_UP(sizeof(init_thread_info), 16) + (unsigned long) &init_stack) +#define INIT_SP_LIMIT ((unsigned long)&init_stack) #ifdef CONFIG_SPE -#define SPEFSCR_INIT .spefscr = SPEFSCR_FINVE | SPEFSCR_FDBZE | SPEFSCR_FUNFE | SPEFSCR_FOVFE, +#define SPEFSCR_INIT \ + .spefscr = SPEFSCR_FINVE | SPEFSCR_FDBZE | SPEFSCR_FUNFE | SPEFSCR_FOVFE, \ + .spefscr_last = SPEFSCR_FINVE | SPEFSCR_FDBZE | SPEFSCR_FUNFE | SPEFSCR_FOVFE, #else #define SPEFSCR_INIT #endif -#ifdef CONFIG_PPC32 +#ifdef CONFIG_PPC_BOOK3S_32 +#define SR0_INIT .sr0 = IS_ENABLED(CONFIG_PPC_KUEP) ? SR_NX : 0, +#else +#define SR0_INIT +#endif + +#if defined(CONFIG_PPC_BOOK3S_32) && defined(CONFIG_PPC_KUAP) +#define INIT_THREAD { \ + .ksp = INIT_SP, \ + .pgdir = swapper_pg_dir, \ + .kuap = ~0UL, /* KUAP_NONE */ \ + .fpexc_mode = MSR_FE0 | MSR_FE1, \ + SPEFSCR_INIT \ + SR0_INIT \ +} +#elif defined(CONFIG_PPC32) #define INIT_THREAD { \ .ksp = INIT_SP, \ - .ksp_limit = INIT_SP_LIMIT, \ - .fs = KERNEL_DS, \ .pgdir = swapper_pg_dir, \ .fpexc_mode = MSR_FE0 | MSR_FE1, \ SPEFSCR_INIT \ + SR0_INIT \ } #else #define INIT_THREAD { \ .ksp = INIT_SP, \ - .ksp_limit = INIT_SP_LIMIT, \ - .regs = (struct pt_regs *)INIT_SP - 1, /* XXX bogus, I think */ \ - .fs = KERNEL_DS, \ - .fpr = {{0}}, \ - .fpscr = { .val = 0, }, \ .fpexc_mode = 0, \ - .ppr = INIT_PPR, \ } #endif -/* - * Return saved PC of a blocked thread. For now, this is the "user" PC - */ -#define thread_saved_pc(tsk) \ - ((tsk)->thread.regs? (tsk)->thread.regs->nip: 0) +#define task_pt_regs(tsk) ((tsk)->thread.regs) -#define task_pt_regs(tsk) ((struct pt_regs *)(tsk)->thread.regs) - -unsigned long get_wchan(struct task_struct *p); +unsigned long __get_wchan(struct task_struct *p); #define KSTK_EIP(tsk) ((tsk)->thread.regs? (tsk)->thread.regs->nip: 0) #define KSTK_ESP(tsk) ((tsk)->thread.regs? (tsk)->thread.regs->gpr[1]: 0) @@ -359,6 +334,21 @@ extern int set_endian(struct task_struct *tsk, unsigned int val); extern int get_unalign_ctl(struct task_struct *tsk, unsigned long adr); extern int set_unalign_ctl(struct task_struct *tsk, unsigned int val); +#ifdef CONFIG_PPC_BOOK3S_64 + +#define PPC_GET_DEXCR_ASPECT(tsk, asp) get_dexcr_prctl((tsk), (asp)) +#define PPC_SET_DEXCR_ASPECT(tsk, asp, val) set_dexcr_prctl((tsk), (asp), (val)) + +int get_dexcr_prctl(struct task_struct *tsk, unsigned long asp); +int set_dexcr_prctl(struct task_struct *tsk, unsigned long asp, unsigned long val); + +#endif + +extern void load_fp_state(struct thread_fp_state *fp); +extern void store_fp_state(struct thread_fp_state *fp); +extern void load_vr_state(struct thread_vr_state *vr); +extern void store_vr_state(struct thread_vr_state *vr); + static inline unsigned int __unpack_fe01(unsigned long msr_bits) { return ((msr_bits & MSR_FE0) >> 10) | ((msr_bits & MSR_FE1) >> 8); @@ -370,21 +360,45 @@ static inline unsigned long __pack_fe01(unsigned int fpmode) } #ifdef CONFIG_PPC64 -#define cpu_relax() do { HMT_low(); HMT_medium(); barrier(); } while (0) -#else -#define cpu_relax() barrier() + +#define spin_begin() \ + asm volatile(ASM_FTR_IFCLR( \ + "or 1,1,1", /* HMT_LOW */ \ + "nop", /* v3.1 uses pause_short in cpu_relax instead */ \ + %0) :: "i" (CPU_FTR_ARCH_31) : "memory") + +#define spin_cpu_relax() \ + asm volatile(ASM_FTR_IFCLR( \ + "nop", /* Before v3.1 use priority nops in spin_begin/end */ \ + PPC_WAIT(2, 0), /* aka pause_short */ \ + %0) :: "i" (CPU_FTR_ARCH_31) : "memory") + +#define spin_end() \ + asm volatile(ASM_FTR_IFCLR( \ + "or 2,2,2", /* HMT_MEDIUM */ \ + "nop", \ + %0) :: "i" (CPU_FTR_ARCH_31) : "memory") + #endif -/* Check that a certain kernel stack pointer is valid in task_struct p */ -int validate_sp(unsigned long sp, struct task_struct *p, - unsigned long nbytes); +/* + * Check that a certain kernel stack pointer is a valid (minimum sized) + * stack frame in task_struct p. + */ +int validate_sp(unsigned long sp, struct task_struct *p); + +/* + * validate the stack frame of a particular minimum size, used for when we are + * looking at a certain object in the stack beyond the minimum. + */ +int validate_sp_size(unsigned long sp, struct task_struct *p, + unsigned long nbytes); /* * Prefetch macros. */ #define ARCH_HAS_PREFETCH #define ARCH_HAS_PREFETCHW -#define ARCH_HAS_SPINLOCK_PREFETCH static inline void prefetch(const void *x) { @@ -402,43 +416,27 @@ static inline void prefetchw(const void *x) __asm__ __volatile__ ("dcbtst 0,%0" : : "r" (x)); } -#define spin_lock_prefetch(x) prefetchw(x) - -#define HAVE_ARCH_PICK_MMAP_LAYOUT - -#ifdef CONFIG_PPC64 -static inline unsigned long get_clean_sp(unsigned long sp, int is_32) -{ - if (is_32) - return sp & 0x0ffffffffUL; - return sp; -} -#else -static inline unsigned long get_clean_sp(unsigned long sp, int is_32) -{ - return sp; -} +/* asm stubs */ +extern unsigned long isa300_idle_stop_noloss(unsigned long psscr_val); +extern unsigned long isa300_idle_stop_mayloss(unsigned long psscr_val); +extern unsigned long isa206_idle_insn_mayloss(unsigned long type); +#ifdef CONFIG_PPC_970_NAP +extern void power4_idle_nap(void); +void power4_idle_nap_return(void); #endif extern unsigned long cpuidle_disable; enum idle_boot_override {IDLE_NO_OVERRIDE = 0, IDLE_POWERSAVE_OFF}; extern int powersave_nap; /* set if nap mode can be used in idle loop */ -extern void power7_nap(void); -#ifdef CONFIG_PSERIES_IDLE -extern void update_smt_snooze_delay(int cpu, int residency); -#else -static inline void update_smt_snooze_delay(int cpu, int residency) {} -#endif +extern void power7_idle_type(unsigned long type); +extern void arch300_idle_type(unsigned long stop_psscr_val, + unsigned long stop_psscr_mask); +void pnv_power9_force_smt4_catch(void); +void pnv_power9_force_smt4_release(void); -extern void flush_instruction_cache(void); -extern void hard_reset_now(void); -extern void poweroff_now(void); extern int fix_alignment(struct pt_regs *); -extern void cvt_fd(float *from, double *to); -extern void cvt_df(double *from, float *to); -extern void _nmask_and_or_msr(unsigned long nmask, unsigned long or_val); #ifdef CONFIG_PPC64 /* @@ -451,6 +449,16 @@ extern void _nmask_and_or_msr(unsigned long nmask, unsigned long or_val); #define NET_IP_ALIGN 0 #endif +int do_mathemu(struct pt_regs *regs); +int do_spe_mathemu(struct pt_regs *regs); +int speround_handler(struct pt_regs *regs); + +/* VMX copying */ +int enter_vmx_usercopy(void); +int exit_vmx_usercopy(void); +int enter_vmx_ops(void); +void *exit_vmx_ops(void *dest); + #endif /* __KERNEL__ */ -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _ASM_POWERPC_PROCESSOR_H */ diff --git a/arch/powerpc/include/asm/prom.h b/arch/powerpc/include/asm/prom.h index bc2da154f68b..f679a11a7e7f 100644 --- a/arch/powerpc/include/asm/prom.h +++ b/arch/powerpc/include/asm/prom.h @@ -1,4 +1,4 @@ -#include <linux/of.h> /* linux/of.h gets to determine #include ordering */ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _POWERPC_PROM_H #define _POWERPC_PROM_H #ifdef __KERNEL__ @@ -10,69 +10,84 @@ * Copyright (C) 1996-2005 Paul Mackerras. * * Updates for PPC64 by Peter Bergner & David Engebretsen, IBM Corp. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #include <linux/types.h> -#include <asm/irq.h> -#include <linux/atomic.h> +#include <asm/firmware.h> + +struct device_node; +struct property; + +#define MIN_RMA 768 /* Minimum RMA (in MB) for CAS negotiation */ -#define HAVE_ARCH_DEVTREE_FIXUPS +#define OF_DT_BEGIN_NODE 0x1 /* Start of node, full name */ +#define OF_DT_END_NODE 0x2 /* End node */ +#define OF_DT_PROP 0x3 /* Property: name off, size, + * content */ +#define OF_DT_NOP 0x4 /* nop */ +#define OF_DT_END 0x9 + +#define OF_DT_VERSION 0x10 /* - * OF address retreival & translation + * This is what gets passed to the kernel by prom_init or kexec + * + * The dt struct contains the device tree structure, full pathes and + * property contents. The dt strings contain a separate block with just + * the strings for the property names, and is fully page aligned and + * self contained in a page, so that it can be kept around by the kernel, + * each property name appears only once in this page (cheap compression) + * + * the mem_rsvmap contains a map of reserved ranges of physical memory, + * passing it here instead of in the device-tree itself greatly simplifies + * the job of everybody. It's just a list of u64 pairs (base/size) that + * ends when size is 0 */ +struct boot_param_header { + __be32 magic; /* magic word OF_DT_HEADER */ + __be32 totalsize; /* total size of DT block */ + __be32 off_dt_struct; /* offset to structure */ + __be32 off_dt_strings; /* offset to strings */ + __be32 off_mem_rsvmap; /* offset to memory reserve map */ + __be32 version; /* format version */ + __be32 last_comp_version; /* last compatible version */ + /* version 2 fields below */ + __be32 boot_cpuid_phys; /* Physical CPU id we're booting on */ + /* version 3 fields below */ + __be32 dt_strings_size; /* size of the DT strings block */ + /* version 17 fields below */ + __be32 dt_struct_size; /* size of the DT structure block */ +}; -/* Translate a DMA address from device space to CPU space */ -extern u64 of_translate_dma_address(struct device_node *dev, - const __be32 *in_addr); - -#ifdef CONFIG_PCI -extern unsigned long pci_address_to_pio(phys_addr_t address); -#define pci_address_to_pio pci_address_to_pio -#endif /* CONFIG_PCI */ +/* + * OF address retreival & translation + */ /* Parse the ibm,dma-window property of an OF node into the busno, phys and * size parameters. */ -void of_parse_dma_window(struct device_node *dn, const void *dma_window_prop, - unsigned long *busno, unsigned long *phys, unsigned long *size); - -extern void kdump_move_device_tree(void); - -/* CPU OF node matching */ -struct device_node *of_get_cpu_node(int cpu, unsigned int *thread); - -/* cache lookup */ -struct device_node *of_find_next_cache_node(struct device_node *np); - -#ifdef CONFIG_NUMA -extern int of_node_to_nid(struct device_node *device); -#else -static inline int of_node_to_nid(struct device_node *device) { return 0; } -#endif -#define of_node_to_nid of_node_to_nid +void of_parse_dma_window(struct device_node *dn, const __be32 *dma_window, + unsigned long *busno, unsigned long *phys, + unsigned long *size); extern void of_instantiate_rtc(void); -/* The of_drconf_cell struct defines the layout of the LMB array - * specified in the device tree property - * ibm,dynamic-reconfiguration-memory/ibm,dynamic-memory - */ -struct of_drconf_cell { - u64 base_addr; - u32 drc_index; - u32 reserved; - u32 aa_index; - u32 flags; +extern int of_get_ibm_chip_id(struct device_node *np); + +struct of_drc_info { + char *drc_type; + char *drc_name_prefix; + u32 drc_index_start; + u32 drc_name_suffix_start; + u32 num_sequential_elems; + u32 sequential_inc; + u32 drc_power_domain; + u32 last_drc_index; }; -#define DRCONF_MEM_ASSIGNED 0x00000008 -#define DRCONF_MEM_AI_INVALID 0x00000040 -#define DRCONF_MEM_RESERVED 0x00000080 +extern int of_read_drc_info_cell(struct property **prop, + const __be32 **curval, struct of_drc_info *data); + +extern unsigned int boot_cpu_node_count; /* * There are two methods for telling firmware what our capabilities are. @@ -99,6 +114,9 @@ struct of_drconf_cell { #define OV1_PPC_2_06 0x02 /* set if we support PowerPC 2.06 */ #define OV1_PPC_2_07 0x01 /* set if we support PowerPC 2.07 */ +#define OV1_PPC_3_00 0x80 /* set if we support PowerPC 3.00 */ +#define OV1_PPC_3_1 0x40 /* set if we support PowerPC 3.1 */ + /* Option vector 2: Open Firmware options supported */ #define OV2_REAL_MODE 0x20 /* set if we want OF in real mode */ @@ -127,30 +145,36 @@ struct of_drconf_cell { #define OV5_MSI 0x0201 /* PCIe/MSI support */ #define OV5_CMO 0x0480 /* Cooperative Memory Overcommitment */ #define OV5_XCMO 0x0440 /* Page Coalescing */ -#define OV5_TYPE1_AFFINITY 0x0580 /* Type 1 NUMA affinity */ +#define OV5_FORM1_AFFINITY 0x0580 /* FORM1 NUMA affinity */ #define OV5_PRRN 0x0540 /* Platform Resource Reassignment */ -#define OV5_PFO_HW_RNG 0x0E80 /* PFO Random Number Generator */ -#define OV5_PFO_HW_842 0x0E40 /* PFO Compression Accelerator */ -#define OV5_PFO_HW_ENCR 0x0E20 /* PFO Encryption Accelerator */ -#define OV5_SUB_PROCESSORS 0x0F01 /* 1,2,or 4 Sub-Processors supported */ +#define OV5_FORM2_AFFINITY 0x0520 /* Form2 NUMA affinity */ +#define OV5_HP_EVT 0x0604 /* Hot Plug Event support */ +#define OV5_RESIZE_HPT 0x0601 /* Hash Page Table resizing */ +#define OV5_PFO_HW_RNG 0x1180 /* PFO Random Number Generator */ +#define OV5_PFO_HW_842 0x1140 /* PFO Compression Accelerator */ +#define OV5_PFO_HW_ENCR 0x1120 /* PFO Encryption Accelerator */ +#define OV5_SUB_PROCESSORS 0x1501 /* 1,2,or 4 Sub-Processors supported */ +#define OV5_DRMEM_V2 0x1680 /* ibm,dynamic-reconfiguration-v2 */ +#define OV5_XIVE_SUPPORT 0x17C0 /* XIVE Exploitation Support Mask */ +#define OV5_XIVE_LEGACY 0x1700 /* XIVE legacy mode Only */ +#define OV5_XIVE_EXPLOIT 0x1740 /* XIVE exploitation mode Only */ +#define OV5_XIVE_EITHER 0x1780 /* XIVE legacy or exploitation mode */ +/* MMU Base Architecture */ +#define OV5_MMU_SUPPORT 0x18C0 /* MMU Mode Support Mask */ +#define OV5_MMU_HASH 0x1800 /* Hash MMU Only */ +#define OV5_MMU_RADIX 0x1840 /* Radix MMU Only */ +#define OV5_MMU_EITHER 0x1880 /* Hash or Radix Supported */ +#define OV5_MMU_DYNAMIC 0x18C0 /* Hash or Radix Can Switch Later */ +#define OV5_NMMU 0x1820 /* Nest MMU Available */ +/* Hash Table Extensions */ +#define OV5_HASH_SEG_TBL 0x1980 /* In Memory Segment Tables Available */ +#define OV5_HASH_GTSE 0x1940 /* Guest Translation Shoot Down Avail */ +/* Radix Table Extensions */ +#define OV5_RADIX_GTSE 0x1A40 /* Guest Translation Shoot Down Avail */ +#define OV5_DRC_INFO 0x1640 /* Redef Prop Structures: drc-info */ /* Option Vector 6: IBM PAPR hints */ #define OV6_LINUX 0x02 /* Linux is our OS */ -/* - * The architecture vector has an array of PVR mask/value pairs, - * followed by # option vectors - 1, followed by the option vectors. - */ -extern unsigned char ibm_architecture_vec[]; - -/* These includes are put at the bottom because they may contain things - * that are overridden by this file. Ideally they shouldn't be included - * by this file, but there are a bunch of .c files that currently depend - * on it. Eventually they will be cleaned up. */ -#include <linux/of_fdt.h> -#include <linux/of_address.h> -#include <linux/of_irq.h> -#include <linux/platform_device.h> - #endif /* __KERNEL__ */ #endif /* _POWERPC_PROM_H */ diff --git a/arch/powerpc/include/asm/ps3.h b/arch/powerpc/include/asm/ps3.h index 678a7c1d9cb8..987e23a2bd28 100644 --- a/arch/powerpc/include/asm/ps3.h +++ b/arch/powerpc/include/asm/ps3.h @@ -1,27 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * PS3 platform declarations. * * Copyright (C) 2006 Sony Computer Entertainment Inc. * Copyright 2006 Sony Corp. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #if !defined(_ASM_POWERPC_PS3_H) #define _ASM_POWERPC_PS3_H -#include <linux/init.h> #include <linux/types.h> #include <linux/device.h> #include <asm/cell-pmu.h> @@ -84,6 +71,7 @@ struct ps3_dma_region_ops; * @bus_addr: The 'translated' bus address of the region. * @len: The length in bytes of the region. * @offset: The offset from the start of memory of the region. + * @dma_mask: Device dma_mask. * @ioid: The IOID of the device who owns this region * @chunk_list: Opaque variable used by the ioc page manager. * @region_ops: struct ps3_dma_region_ops - dma region operations @@ -98,6 +86,7 @@ struct ps3_dma_region { enum ps3_dma_region_type region_type; unsigned long len; unsigned long offset; + u64 dma_mask; /* driver variables (set by ps3_dma_region_create) */ unsigned long bus_addr; @@ -245,7 +234,7 @@ enum lv1_result { static inline const char* ps3_result(int result) { -#if defined(DEBUG) || defined(PS3_VERBOSE_RESULT) +#if defined(DEBUG) || defined(PS3_VERBOSE_RESULT) || defined(CONFIG_PS3_VERBOSE_RESULT) switch (result) { case LV1_SUCCESS: return "LV1_SUCCESS (0)"; @@ -391,8 +380,8 @@ struct ps3_system_bus_driver { enum ps3_match_sub_id match_sub_id; struct device_driver core; int (*probe)(struct ps3_system_bus_device *); - int (*remove)(struct ps3_system_bus_device *); - int (*shutdown)(struct ps3_system_bus_device *); + void (*remove)(struct ps3_system_bus_device *); + void (*shutdown)(struct ps3_system_bus_device *); /* int (*suspend)(struct ps3_system_bus_device *, pm_message_t); */ /* int (*resume)(struct ps3_system_bus_device *); */ }; @@ -401,13 +390,9 @@ int ps3_system_bus_device_register(struct ps3_system_bus_device *dev); int ps3_system_bus_driver_register(struct ps3_system_bus_driver *drv); void ps3_system_bus_driver_unregister(struct ps3_system_bus_driver *drv); -static inline struct ps3_system_bus_driver *ps3_drv_to_system_bus_drv( - struct device_driver *_drv) -{ - return container_of(_drv, struct ps3_system_bus_driver, core); -} +#define ps3_drv_to_system_bus_drv(_drv) container_of_const(_drv, struct ps3_system_bus_driver, core) static inline struct ps3_system_bus_device *ps3_dev_to_system_bus_dev( - struct device *_dev) + const struct device *_dev) { return container_of(_dev, struct ps3_system_bus_device, core); } @@ -436,10 +421,6 @@ static inline void *ps3_system_bus_get_drvdata( return dev_get_drvdata(&dev->core); } -/* These two need global scope for get_dma_ops(). */ - -extern struct bus_type ps3_system_bus_type; - /* system manager */ struct ps3_sys_manager_ops { @@ -527,4 +508,12 @@ void ps3_sync_irq(int node); u32 ps3_get_hw_thread_id(int cpu); u64 ps3_get_spe_id(void *arg); +void ps3_early_mm_init(void); + +#ifdef CONFIG_PPC_EARLY_DEBUG_PS3GELIC +void udbg_shutdown_ps3gelic(void); +#else +static inline void udbg_shutdown_ps3gelic(void) {} +#endif + #endif diff --git a/arch/powerpc/include/asm/ps3av.h b/arch/powerpc/include/asm/ps3av.h index 0427b0b53d2d..c8b0f2ffcd35 100644 --- a/arch/powerpc/include/asm/ps3av.h +++ b/arch/powerpc/include/asm/ps3av.h @@ -1,21 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * PS3 AV backend support. * * Copyright (C) 2007 Sony Computer Entertainment Inc. * Copyright 2007 Sony Corp. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef _ASM_POWERPC_PS3AV_H_ @@ -104,7 +92,7 @@ #define PS3AV_CMD_AV_INPUTLEN_16 0x02 #define PS3AV_CMD_AV_INPUTLEN_20 0x0a #define PS3AV_CMD_AV_INPUTLEN_24 0x0b -/* alayout */ +/* av_layout */ #define PS3AV_CMD_AV_LAYOUT_32 (1 << 0) #define PS3AV_CMD_AV_LAYOUT_44 (1 << 1) #define PS3AV_CMD_AV_LAYOUT_48 (1 << 2) @@ -738,6 +726,4 @@ extern int ps3av_video_mode2res(u32, u32 *, u32 *); extern int ps3av_video_mute(int); extern int ps3av_audio_mute(int); extern int ps3av_audio_mute_analog(int); -extern int ps3av_dev_open(void); -extern int ps3av_dev_close(void); #endif /* _ASM_POWERPC_PS3AV_H_ */ diff --git a/arch/powerpc/include/asm/ps3gpu.h b/arch/powerpc/include/asm/ps3gpu.h index b2b89591907c..9645c30471b5 100644 --- a/arch/powerpc/include/asm/ps3gpu.h +++ b/arch/powerpc/include/asm/ps3gpu.h @@ -1,20 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * PS3 GPU declarations. * * Copyright 2009 Sony Corporation - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. - * If not, see <http://www.gnu.org/licenses/>. */ #ifndef _ASM_POWERPC_PS3GPU_H diff --git a/arch/powerpc/include/asm/ps3stor.h b/arch/powerpc/include/asm/ps3stor.h index 6fcaf714fa50..1d8279014f22 100644 --- a/arch/powerpc/include/asm/ps3stor.h +++ b/arch/powerpc/include/asm/ps3stor.h @@ -1,21 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * PS3 Storage Devices * * Copyright (C) 2007 Sony Computer Entertainment Inc. * Copyright 2007 Sony Corp. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published - * by the Free Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ #ifndef _ASM_POWERPC_PS3STOR_H_ @@ -51,7 +39,7 @@ struct ps3_storage_device { unsigned int num_regions; unsigned long accessible_regions; unsigned int region_idx; /* first accessible region */ - struct ps3_storage_region regions[0]; /* Must be last */ + struct ps3_storage_region regions[]; /* Must be last */ }; static inline struct ps3_storage_device *to_ps3_storage_device(struct device *dev) diff --git a/arch/powerpc/include/asm/pte-40x.h b/arch/powerpc/include/asm/pte-40x.h deleted file mode 100644 index ec0b0b0d1df9..000000000000 --- a/arch/powerpc/include/asm/pte-40x.h +++ /dev/null @@ -1,65 +0,0 @@ -#ifndef _ASM_POWERPC_PTE_40x_H -#define _ASM_POWERPC_PTE_40x_H -#ifdef __KERNEL__ - -/* - * At present, all PowerPC 400-class processors share a similar TLB - * architecture. The instruction and data sides share a unified, - * 64-entry, fully-associative TLB which is maintained totally under - * software control. In addition, the instruction side has a - * hardware-managed, 4-entry, fully-associative TLB which serves as a - * first level to the shared TLB. These two TLBs are known as the UTLB - * and ITLB, respectively (see "mmu.h" for definitions). - * - * There are several potential gotchas here. The 40x hardware TLBLO - * field looks like this: - * - * 0 1 2 3 4 ... 18 19 20 21 22 23 24 25 26 27 28 29 30 31 - * RPN..................... 0 0 EX WR ZSEL....... W I M G - * - * Where possible we make the Linux PTE bits match up with this - * - * - bits 20 and 21 must be cleared, because we use 4k pages (40x can - * support down to 1k pages), this is done in the TLBMiss exception - * handler. - * - We use only zones 0 (for kernel pages) and 1 (for user pages) - * of the 16 available. Bit 24-26 of the TLB are cleared in the TLB - * miss handler. Bit 27 is PAGE_USER, thus selecting the correct - * zone. - * - PRESENT *must* be in the bottom two bits because swap cache - * entries use the top 30 bits. Because 40x doesn't support SMP - * anyway, M is irrelevant so we borrow it for PAGE_PRESENT. Bit 30 - * is cleared in the TLB miss handler before the TLB entry is loaded. - * - All other bits of the PTE are loaded into TLBLO without - * modification, leaving us only the bits 20, 21, 24, 25, 26, 30 for - * software PTE bits. We actually use use bits 21, 24, 25, and - * 30 respectively for the software bits: ACCESSED, DIRTY, RW, and - * PRESENT. - */ - -#define _PAGE_GUARDED 0x001 /* G: page is guarded from prefetch */ -#define _PAGE_FILE 0x001 /* when !present: nonlinear file mapping */ -#define _PAGE_PRESENT 0x002 /* software: PTE contains a translation */ -#define _PAGE_NO_CACHE 0x004 /* I: caching is inhibited */ -#define _PAGE_WRITETHRU 0x008 /* W: caching is write-through */ -#define _PAGE_USER 0x010 /* matches one of the zone permission bits */ -#define _PAGE_SPECIAL 0x020 /* software: Special page */ -#define _PAGE_RW 0x040 /* software: Writes permitted */ -#define _PAGE_DIRTY 0x080 /* software: dirty page */ -#define _PAGE_HWWRITE 0x100 /* hardware: Dirty & RW, set in exception */ -#define _PAGE_EXEC 0x200 /* hardware: EX permission */ -#define _PAGE_ACCESSED 0x400 /* software: R: page referenced */ - -#define _PMD_PRESENT 0x400 /* PMD points to page of PTEs */ -#define _PMD_BAD 0x802 -#define _PMD_SIZE 0x0e0 /* size field, != 0 for large-page PMD entry */ -#define _PMD_SIZE_4M 0x0c0 -#define _PMD_SIZE_16M 0x0e0 - -#define PMD_PAGE_SIZE(pmdval) (1024 << (((pmdval) & _PMD_SIZE) >> 4)) - -/* Until my rework is finished, 40x still needs atomic PTE updates */ -#define PTE_ATOMIC_UPDATES 1 - -#endif /* __KERNEL__ */ -#endif /* _ASM_POWERPC_PTE_40x_H */ diff --git a/arch/powerpc/include/asm/pte-8xx.h b/arch/powerpc/include/asm/pte-8xx.h deleted file mode 100644 index d44826e4ff97..000000000000 --- a/arch/powerpc/include/asm/pte-8xx.h +++ /dev/null @@ -1,67 +0,0 @@ -#ifndef _ASM_POWERPC_PTE_8xx_H -#define _ASM_POWERPC_PTE_8xx_H -#ifdef __KERNEL__ - -/* - * The PowerPC MPC8xx uses a TLB with hardware assisted, software tablewalk. - * We also use the two level tables, but we can put the real bits in them - * needed for the TLB and tablewalk. These definitions require Mx_CTR.PPM = 0, - * Mx_CTR.PPCS = 0, and MD_CTR.TWAM = 1. The level 2 descriptor has - * additional page protection (when Mx_CTR.PPCS = 1) that allows TLB hit - * based upon user/super access. The TLB does not have accessed nor write - * protect. We assume that if the TLB get loaded with an entry it is - * accessed, and overload the changed bit for write protect. We use - * two bits in the software pte that are supposed to be set to zero in - * the TLB entry (24 and 25) for these indicators. Although the level 1 - * descriptor contains the guarded and writethrough/copyback bits, we can - * set these at the page level since they get copied from the Mx_TWC - * register when the TLB entry is loaded. We will use bit 27 for guard, since - * that is where it exists in the MD_TWC, and bit 26 for writethrough. - * These will get masked from the level 2 descriptor at TLB load time, and - * copied to the MD_TWC before it gets loaded. - * Large page sizes added. We currently support two sizes, 4K and 8M. - * This also allows a TLB hander optimization because we can directly - * load the PMD into MD_TWC. The 8M pages are only used for kernel - * mapping of well known areas. The PMD (PGD) entries contain control - * flags in addition to the address, so care must be taken that the - * software no longer assumes these are only pointers. - */ - -/* Definitions for 8xx embedded chips. */ -#define _PAGE_PRESENT 0x0001 /* Page is valid */ -#define _PAGE_FILE 0x0002 /* when !present: nonlinear file mapping */ -#define _PAGE_NO_CACHE 0x0002 /* I: cache inhibit */ -#define _PAGE_SHARED 0x0004 /* No ASID (context) compare */ -#define _PAGE_SPECIAL 0x0008 /* SW entry, forced to 0 by the TLB miss */ -#define _PAGE_DIRTY 0x0100 /* C: page changed */ - -/* These 4 software bits must be masked out when the entry is loaded - * into the TLB, 1 SW bit left(0x0080). - */ -#define _PAGE_GUARDED 0x0010 /* software: guarded access */ -#define _PAGE_ACCESSED 0x0020 /* software: page referenced */ -#define _PAGE_WRITETHRU 0x0040 /* software: caching is write through */ - -/* Setting any bits in the nibble with the follow two controls will - * require a TLB exception handler change. It is assumed unused bits - * are always zero. - */ -#define _PAGE_RW 0x0400 /* lsb PP bits, inverted in HW */ -#define _PAGE_USER 0x0800 /* msb PP bits */ - -#define _PMD_PRESENT 0x0001 -#define _PMD_BAD 0x0ff0 -#define _PMD_PAGE_MASK 0x000c -#define _PMD_PAGE_8M 0x000c - -#define _PTE_NONE_MASK _PAGE_ACCESSED - -/* Until my rework is finished, 8xx still needs atomic PTE updates */ -#define PTE_ATOMIC_UPDATES 1 - -/* We need to add _PAGE_SHARED to kernel pages */ -#define _PAGE_KERNEL_RO (_PAGE_SHARED) -#define _PAGE_KERNEL_RW (_PAGE_DIRTY | _PAGE_RW | _PAGE_HWWRITE) - -#endif /* __KERNEL__ */ -#endif /* _ASM_POWERPC_PTE_8xx_H */ diff --git a/arch/powerpc/include/asm/pte-book3e.h b/arch/powerpc/include/asm/pte-book3e.h deleted file mode 100644 index 0156702ba24e..000000000000 --- a/arch/powerpc/include/asm/pte-book3e.h +++ /dev/null @@ -1,87 +0,0 @@ -#ifndef _ASM_POWERPC_PTE_BOOK3E_H -#define _ASM_POWERPC_PTE_BOOK3E_H -#ifdef __KERNEL__ - -/* PTE bit definitions for processors compliant to the Book3E - * architecture 2.06 or later. The position of the PTE bits - * matches the HW definition of the optional Embedded Page Table - * category. - */ - -/* Architected bits */ -#define _PAGE_PRESENT 0x000001 /* software: pte contains a translation */ -#define _PAGE_FILE 0x000002 /* (!present only) software: pte holds file offset */ -#define _PAGE_SW1 0x000002 -#define _PAGE_BAP_SR 0x000004 -#define _PAGE_BAP_UR 0x000008 -#define _PAGE_BAP_SW 0x000010 -#define _PAGE_BAP_UW 0x000020 -#define _PAGE_BAP_SX 0x000040 -#define _PAGE_BAP_UX 0x000080 -#define _PAGE_PSIZE_MSK 0x000f00 -#define _PAGE_PSIZE_4K 0x000200 -#define _PAGE_PSIZE_8K 0x000300 -#define _PAGE_PSIZE_16K 0x000400 -#define _PAGE_PSIZE_32K 0x000500 -#define _PAGE_PSIZE_64K 0x000600 -#define _PAGE_PSIZE_128K 0x000700 -#define _PAGE_PSIZE_256K 0x000800 -#define _PAGE_PSIZE_512K 0x000900 -#define _PAGE_PSIZE_1M 0x000a00 -#define _PAGE_PSIZE_2M 0x000b00 -#define _PAGE_PSIZE_4M 0x000c00 -#define _PAGE_PSIZE_8M 0x000d00 -#define _PAGE_PSIZE_16M 0x000e00 -#define _PAGE_PSIZE_32M 0x000f00 -#define _PAGE_DIRTY 0x001000 /* C: page changed */ -#define _PAGE_SW0 0x002000 -#define _PAGE_U3 0x004000 -#define _PAGE_U2 0x008000 -#define _PAGE_U1 0x010000 -#define _PAGE_U0 0x020000 -#define _PAGE_ACCESSED 0x040000 -#define _PAGE_LENDIAN 0x080000 -#define _PAGE_GUARDED 0x100000 -#define _PAGE_COHERENT 0x200000 /* M: enforce memory coherence */ -#define _PAGE_NO_CACHE 0x400000 /* I: cache inhibit */ -#define _PAGE_WRITETHRU 0x800000 /* W: cache write-through */ - -/* "Higher level" linux bit combinations */ -#define _PAGE_EXEC _PAGE_BAP_UX /* .. and was cache cleaned */ -#define _PAGE_RW (_PAGE_BAP_SW | _PAGE_BAP_UW) /* User write permission */ -#define _PAGE_KERNEL_RW (_PAGE_BAP_SW | _PAGE_BAP_SR | _PAGE_DIRTY) -#define _PAGE_KERNEL_RO (_PAGE_BAP_SR) -#define _PAGE_KERNEL_RWX (_PAGE_BAP_SW | _PAGE_BAP_SR | _PAGE_DIRTY | _PAGE_BAP_SX) -#define _PAGE_KERNEL_ROX (_PAGE_BAP_SR | _PAGE_BAP_SX) -#define _PAGE_USER (_PAGE_BAP_UR | _PAGE_BAP_SR) /* Can be read */ - -#define _PAGE_HASHPTE 0 -#define _PAGE_BUSY 0 - -#define _PAGE_SPECIAL _PAGE_SW0 - -/* Flags to be preserved on PTE modifications */ -#define _PAGE_HPTEFLAGS _PAGE_BUSY - -/* Base page size */ -#ifdef CONFIG_PPC_64K_PAGES -#define _PAGE_PSIZE _PAGE_PSIZE_64K -#define PTE_RPN_SHIFT (28) -#else -#define _PAGE_PSIZE _PAGE_PSIZE_4K -#define PTE_RPN_SHIFT (24) -#endif - -#define PTE_WIMGE_SHIFT (19) -#define PTE_BAP_SHIFT (2) - -/* On 32-bit, we never clear the top part of the PTE */ -#ifdef CONFIG_PPC32 -#define _PTE_NONE_MASK 0xffffffff00000000ULL -#define _PMD_PRESENT 0 -#define _PMD_PRESENT_MASK (PAGE_MASK) -#define _PMD_BAD (~PAGE_MASK) -#endif - -#endif /* __KERNEL__ */ -#endif /* _ASM_POWERPC_PTE_FSL_BOOKE_H */ diff --git a/arch/powerpc/include/asm/pte-common.h b/arch/powerpc/include/asm/pte-common.h deleted file mode 100644 index 8d1569c29042..000000000000 --- a/arch/powerpc/include/asm/pte-common.h +++ /dev/null @@ -1,187 +0,0 @@ -/* Included from asm/pgtable-*.h only ! */ - -/* - * Some bits are only used on some cpu families... Make sure that all - * the undefined gets a sensible default - */ -#ifndef _PAGE_HASHPTE -#define _PAGE_HASHPTE 0 -#endif -#ifndef _PAGE_SHARED -#define _PAGE_SHARED 0 -#endif -#ifndef _PAGE_HWWRITE -#define _PAGE_HWWRITE 0 -#endif -#ifndef _PAGE_EXEC -#define _PAGE_EXEC 0 -#endif -#ifndef _PAGE_ENDIAN -#define _PAGE_ENDIAN 0 -#endif -#ifndef _PAGE_COHERENT -#define _PAGE_COHERENT 0 -#endif -#ifndef _PAGE_WRITETHRU -#define _PAGE_WRITETHRU 0 -#endif -#ifndef _PAGE_4K_PFN -#define _PAGE_4K_PFN 0 -#endif -#ifndef _PAGE_SAO -#define _PAGE_SAO 0 -#endif -#ifndef _PAGE_PSIZE -#define _PAGE_PSIZE 0 -#endif -#ifndef _PMD_PRESENT_MASK -#define _PMD_PRESENT_MASK _PMD_PRESENT -#endif -#ifndef _PMD_SIZE -#define _PMD_SIZE 0 -#define PMD_PAGE_SIZE(pmd) bad_call_to_PMD_PAGE_SIZE() -#endif -#ifndef _PAGE_KERNEL_RO -#define _PAGE_KERNEL_RO 0 -#endif -#ifndef _PAGE_KERNEL_ROX -#define _PAGE_KERNEL_ROX (_PAGE_EXEC) -#endif -#ifndef _PAGE_KERNEL_RW -#define _PAGE_KERNEL_RW (_PAGE_DIRTY | _PAGE_RW | _PAGE_HWWRITE) -#endif -#ifndef _PAGE_KERNEL_RWX -#define _PAGE_KERNEL_RWX (_PAGE_DIRTY | _PAGE_RW | _PAGE_HWWRITE | _PAGE_EXEC) -#endif -#ifndef _PAGE_HPTEFLAGS -#define _PAGE_HPTEFLAGS _PAGE_HASHPTE -#endif -#ifndef _PTE_NONE_MASK -#define _PTE_NONE_MASK _PAGE_HPTEFLAGS -#endif - -/* Make sure we get a link error if PMD_PAGE_SIZE is ever called on a - * kernel without large page PMD support - */ -#ifndef __ASSEMBLY__ -extern unsigned long bad_call_to_PMD_PAGE_SIZE(void); -#endif /* __ASSEMBLY__ */ - -/* Location of the PFN in the PTE. Most 32-bit platforms use the same - * as _PAGE_SHIFT here (ie, naturally aligned). - * Platform who don't just pre-define the value so we don't override it here - */ -#ifndef PTE_RPN_SHIFT -#define PTE_RPN_SHIFT (PAGE_SHIFT) -#endif - -/* The mask convered by the RPN must be a ULL on 32-bit platforms with - * 64-bit PTEs - */ -#if defined(CONFIG_PPC32) && defined(CONFIG_PTE_64BIT) -#define PTE_RPN_MAX (1ULL << (64 - PTE_RPN_SHIFT)) -#define PTE_RPN_MASK (~((1ULL<<PTE_RPN_SHIFT)-1)) -#else -#define PTE_RPN_MAX (1UL << (32 - PTE_RPN_SHIFT)) -#define PTE_RPN_MASK (~((1UL<<PTE_RPN_SHIFT)-1)) -#endif - -/* _PAGE_CHG_MASK masks of bits that are to be preserved across - * pgprot changes - */ -#define _PAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HPTEFLAGS | _PAGE_DIRTY | \ - _PAGE_ACCESSED | _PAGE_SPECIAL) - -/* Mask of bits returned by pte_pgprot() */ -#define PAGE_PROT_BITS (_PAGE_GUARDED | _PAGE_COHERENT | _PAGE_NO_CACHE | \ - _PAGE_WRITETHRU | _PAGE_ENDIAN | _PAGE_4K_PFN | \ - _PAGE_USER | _PAGE_ACCESSED | \ - _PAGE_RW | _PAGE_HWWRITE | _PAGE_DIRTY | _PAGE_EXEC) - -/* - * We define 2 sets of base prot bits, one for basic pages (ie, - * cacheable kernel and user pages) and one for non cacheable - * pages. We always set _PAGE_COHERENT when SMP is enabled or - * the processor might need it for DMA coherency. - */ -#define _PAGE_BASE_NC (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_PSIZE) -#if defined(CONFIG_SMP) || defined(CONFIG_PPC_STD_MMU) -#define _PAGE_BASE (_PAGE_BASE_NC | _PAGE_COHERENT) -#else -#define _PAGE_BASE (_PAGE_BASE_NC) -#endif - -/* Permission masks used to generate the __P and __S table, - * - * Note:__pgprot is defined in arch/powerpc/include/asm/page.h - * - * Write permissions imply read permissions for now (we could make write-only - * pages on BookE but we don't bother for now). Execute permission control is - * possible on platforms that define _PAGE_EXEC - * - * Note due to the way vm flags are laid out, the bits are XWR - */ -#define PAGE_NONE __pgprot(_PAGE_BASE) -#define PAGE_SHARED __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_RW) -#define PAGE_SHARED_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_RW | _PAGE_EXEC) -#define PAGE_COPY __pgprot(_PAGE_BASE | _PAGE_USER) -#define PAGE_COPY_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_EXEC) -#define PAGE_READONLY __pgprot(_PAGE_BASE | _PAGE_USER) -#define PAGE_READONLY_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_EXEC) - -#define __P000 PAGE_NONE -#define __P001 PAGE_READONLY -#define __P010 PAGE_COPY -#define __P011 PAGE_COPY -#define __P100 PAGE_READONLY_X -#define __P101 PAGE_READONLY_X -#define __P110 PAGE_COPY_X -#define __P111 PAGE_COPY_X - -#define __S000 PAGE_NONE -#define __S001 PAGE_READONLY -#define __S010 PAGE_SHARED -#define __S011 PAGE_SHARED -#define __S100 PAGE_READONLY_X -#define __S101 PAGE_READONLY_X -#define __S110 PAGE_SHARED_X -#define __S111 PAGE_SHARED_X - -/* Permission masks used for kernel mappings */ -#define PAGE_KERNEL __pgprot(_PAGE_BASE | _PAGE_KERNEL_RW) -#define PAGE_KERNEL_NC __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | \ - _PAGE_NO_CACHE) -#define PAGE_KERNEL_NCG __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | \ - _PAGE_NO_CACHE | _PAGE_GUARDED) -#define PAGE_KERNEL_X __pgprot(_PAGE_BASE | _PAGE_KERNEL_RWX) -#define PAGE_KERNEL_RO __pgprot(_PAGE_BASE | _PAGE_KERNEL_RO) -#define PAGE_KERNEL_ROX __pgprot(_PAGE_BASE | _PAGE_KERNEL_ROX) - -/* Protection used for kernel text. We want the debuggers to be able to - * set breakpoints anywhere, so don't write protect the kernel text - * on platforms where such control is possible. - */ -#if defined(CONFIG_KGDB) || defined(CONFIG_XMON) || defined(CONFIG_BDI_SWITCH) ||\ - defined(CONFIG_KPROBES) || defined(CONFIG_DYNAMIC_FTRACE) -#define PAGE_KERNEL_TEXT PAGE_KERNEL_X -#else -#define PAGE_KERNEL_TEXT PAGE_KERNEL_ROX -#endif - -/* Make modules code happy. We don't set RO yet */ -#define PAGE_KERNEL_EXEC PAGE_KERNEL_X - -/* - * Don't just check for any non zero bits in __PAGE_USER, since for book3e - * and PTE_64BIT, PAGE_KERNEL_X contains _PAGE_BAP_SR which is also in - * _PAGE_USER. Need to explicitly match _PAGE_BAP_UR bit in that case too. - */ -#define pte_user(val) ((val & _PAGE_USER) == _PAGE_USER) - -/* Advertise special mapping type for AGP */ -#define PAGE_AGP (PAGE_KERNEL_NC) -#define HAVE_PAGE_AGP - -/* Advertise support for _PAGE_SPECIAL */ -#define __HAVE_ARCH_PTE_SPECIAL - diff --git a/arch/powerpc/include/asm/pte-fsl-booke.h b/arch/powerpc/include/asm/pte-fsl-booke.h deleted file mode 100644 index 2c12be5f677a..000000000000 --- a/arch/powerpc/include/asm/pte-fsl-booke.h +++ /dev/null @@ -1,41 +0,0 @@ -#ifndef _ASM_POWERPC_PTE_FSL_BOOKE_H -#define _ASM_POWERPC_PTE_FSL_BOOKE_H -#ifdef __KERNEL__ - -/* PTE bit definitions for Freescale BookE SW loaded TLB MMU based - * processors - * - MMU Assist Register 3: - - 32 33 34 35 36 ... 50 51 52 53 54 55 56 57 58 59 60 61 62 63 - RPN...................... 0 0 U0 U1 U2 U3 UX SX UW SW UR SR - - - PRESENT *must* be in the bottom three bits because swap cache - entries use the top 29 bits. - - - FILE *must* be in the bottom three bits because swap cache - entries use the top 29 bits. -*/ - -/* Definitions for FSL Book-E Cores */ -#define _PAGE_PRESENT 0x00001 /* S: PTE contains a translation */ -#define _PAGE_USER 0x00002 /* S: User page (maps to UR) */ -#define _PAGE_FILE 0x00002 /* S: when !present: nonlinear file mapping */ -#define _PAGE_RW 0x00004 /* S: Write permission (SW) */ -#define _PAGE_DIRTY 0x00008 /* S: Page dirty */ -#define _PAGE_EXEC 0x00010 /* H: SX permission */ -#define _PAGE_ACCESSED 0x00020 /* S: Page referenced */ - -#define _PAGE_ENDIAN 0x00040 /* H: E bit */ -#define _PAGE_GUARDED 0x00080 /* H: G bit */ -#define _PAGE_COHERENT 0x00100 /* H: M bit */ -#define _PAGE_NO_CACHE 0x00200 /* H: I bit */ -#define _PAGE_WRITETHRU 0x00400 /* H: W bit */ -#define _PAGE_SPECIAL 0x00800 /* S: Special page */ - -#define _PMD_PRESENT 0 -#define _PMD_PRESENT_MASK (PAGE_MASK) -#define _PMD_BAD (~PAGE_MASK) - -#endif /* __KERNEL__ */ -#endif /* _ASM_POWERPC_PTE_FSL_BOOKE_H */ diff --git a/arch/powerpc/include/asm/pte-hash32.h b/arch/powerpc/include/asm/pte-hash32.h deleted file mode 100644 index 4aad4132d0a8..000000000000 --- a/arch/powerpc/include/asm/pte-hash32.h +++ /dev/null @@ -1,47 +0,0 @@ -#ifndef _ASM_POWERPC_PTE_HASH32_H -#define _ASM_POWERPC_PTE_HASH32_H -#ifdef __KERNEL__ - -/* - * The "classic" 32-bit implementation of the PowerPC MMU uses a hash - * table containing PTEs, together with a set of 16 segment registers, - * to define the virtual to physical address mapping. - * - * We use the hash table as an extended TLB, i.e. a cache of currently - * active mappings. We maintain a two-level page table tree, much - * like that used by the i386, for the sake of the Linux memory - * management code. Low-level assembler code in hash_low_32.S - * (procedure hash_page) is responsible for extracting ptes from the - * tree and putting them into the hash table when necessary, and - * updating the accessed and modified bits in the page table tree. - */ - -#define _PAGE_PRESENT 0x001 /* software: pte contains a translation */ -#define _PAGE_HASHPTE 0x002 /* hash_page has made an HPTE for this pte */ -#define _PAGE_FILE 0x004 /* when !present: nonlinear file mapping */ -#define _PAGE_USER 0x004 /* usermode access allowed */ -#define _PAGE_GUARDED 0x008 /* G: prohibit speculative access */ -#define _PAGE_COHERENT 0x010 /* M: enforce memory coherence (SMP systems) */ -#define _PAGE_NO_CACHE 0x020 /* I: cache inhibit */ -#define _PAGE_WRITETHRU 0x040 /* W: cache write-through */ -#define _PAGE_DIRTY 0x080 /* C: page changed */ -#define _PAGE_ACCESSED 0x100 /* R: page referenced */ -#define _PAGE_RW 0x400 /* software: user write access allowed */ -#define _PAGE_SPECIAL 0x800 /* software: Special page */ - -#ifdef CONFIG_PTE_64BIT -/* We never clear the high word of the pte */ -#define _PTE_NONE_MASK (0xffffffff00000000ULL | _PAGE_HASHPTE) -#else -#define _PTE_NONE_MASK _PAGE_HASHPTE -#endif - -#define _PMD_PRESENT 0 -#define _PMD_PRESENT_MASK (PAGE_MASK) -#define _PMD_BAD (~PAGE_MASK) - -/* Hash table based platforms need atomic updates of the linux PTE */ -#define PTE_ATOMIC_UPDATES 1 - -#endif /* __KERNEL__ */ -#endif /* _ASM_POWERPC_PTE_HASH32_H */ diff --git a/arch/powerpc/include/asm/pte-hash64-4k.h b/arch/powerpc/include/asm/pte-hash64-4k.h deleted file mode 100644 index c134e809aac3..000000000000 --- a/arch/powerpc/include/asm/pte-hash64-4k.h +++ /dev/null @@ -1,17 +0,0 @@ -/* To be include by pgtable-hash64.h only */ - -/* PTE bits */ -#define _PAGE_HASHPTE 0x0400 /* software: pte has an associated HPTE */ -#define _PAGE_SECONDARY 0x8000 /* software: HPTE is in secondary group */ -#define _PAGE_GROUP_IX 0x7000 /* software: HPTE index within group */ -#define _PAGE_F_SECOND _PAGE_SECONDARY -#define _PAGE_F_GIX _PAGE_GROUP_IX -#define _PAGE_SPECIAL 0x10000 /* software: special page */ - -/* PTE flags to conserve for HPTE identification */ -#define _PAGE_HPTEFLAGS (_PAGE_BUSY | _PAGE_HASHPTE | \ - _PAGE_SECONDARY | _PAGE_GROUP_IX) - -/* shift to put page number into pte */ -#define PTE_RPN_SHIFT (17) - diff --git a/arch/powerpc/include/asm/pte-hash64-64k.h b/arch/powerpc/include/asm/pte-hash64-64k.h deleted file mode 100644 index d836d945068d..000000000000 --- a/arch/powerpc/include/asm/pte-hash64-64k.h +++ /dev/null @@ -1,81 +0,0 @@ -/* To be include by pgtable-hash64.h only */ - -/* Additional PTE bits (don't change without checking asm in hash_low.S) */ -#define _PAGE_SPECIAL 0x00000400 /* software: special page */ -#define _PAGE_HPTE_SUB 0x0ffff000 /* combo only: sub pages HPTE bits */ -#define _PAGE_HPTE_SUB0 0x08000000 /* combo only: first sub page */ -#define _PAGE_COMBO 0x10000000 /* this is a combo 4k page */ -#define _PAGE_4K_PFN 0x20000000 /* PFN is for a single 4k page */ - -/* For 64K page, we don't have a separate _PAGE_HASHPTE bit. Instead, - * we set that to be the whole sub-bits mask. The C code will only - * test this, so a multi-bit mask will work. For combo pages, this - * is equivalent as effectively, the old _PAGE_HASHPTE was an OR of - * all the sub bits. For real 64k pages, we now have the assembly set - * _PAGE_HPTE_SUB0 in addition to setting the HIDX bits which overlap - * that mask. This is fine as long as the HIDX bits are never set on - * a PTE that isn't hashed, which is the case today. - * - * A little nit is for the huge page C code, which does the hashing - * in C, we need to provide which bit to use. - */ -#define _PAGE_HASHPTE _PAGE_HPTE_SUB - -/* Note the full page bits must be in the same location as for normal - * 4k pages as the same assembly will be used to insert 64K pages - * whether the kernel has CONFIG_PPC_64K_PAGES or not - */ -#define _PAGE_F_SECOND 0x00008000 /* full page: hidx bits */ -#define _PAGE_F_GIX 0x00007000 /* full page: hidx bits */ - -/* PTE flags to conserve for HPTE identification */ -#define _PAGE_HPTEFLAGS (_PAGE_BUSY | _PAGE_HASHPTE | _PAGE_COMBO) - -/* Shift to put page number into pte. - * - * That gives us a max RPN of 34 bits, which means a max of 50 bits - * of addressable physical space, or 46 bits for the special 4k PFNs. - */ -#define PTE_RPN_SHIFT (30) - -#ifndef __ASSEMBLY__ - -/* - * With 64K pages on hash table, we have a special PTE format that - * uses a second "half" of the page table to encode sub-page information - * in order to deal with 64K made of 4K HW pages. Thus we override the - * generic accessors and iterators here - */ -#define __real_pte(e,p) ((real_pte_t) { \ - (e), (pte_val(e) & _PAGE_COMBO) ? \ - (pte_val(*((p) + PTRS_PER_PTE))) : 0 }) -#define __rpte_to_hidx(r,index) ((pte_val((r).pte) & _PAGE_COMBO) ? \ - (((r).hidx >> ((index)<<2)) & 0xf) : ((pte_val((r).pte) >> 12) & 0xf)) -#define __rpte_to_pte(r) ((r).pte) -#define __rpte_sub_valid(rpte, index) \ - (pte_val(rpte.pte) & (_PAGE_HPTE_SUB0 >> (index))) - -/* Trick: we set __end to va + 64k, which happens works for - * a 16M page as well as we want only one iteration - */ -#define pte_iterate_hashed_subpages(rpte, psize, vpn, index, shift) \ - do { \ - unsigned long __end = vpn + (1UL << (PAGE_SHIFT - VPN_SHIFT)); \ - unsigned __split = (psize == MMU_PAGE_4K || \ - psize == MMU_PAGE_64K_AP); \ - shift = mmu_psize_defs[psize].shift; \ - for (index = 0; vpn < __end; index++, \ - vpn += (1L << (shift - VPN_SHIFT))) { \ - if (!__split || __rpte_sub_valid(rpte, index)) \ - do { - -#define pte_iterate_hashed_end() } while(0); } } while(0) - -#define pte_pagesize_index(mm, addr, pte) \ - (((pte) & _PAGE_COMBO)? MMU_PAGE_4K: MMU_PAGE_64K) - -#define remap_4k_pfn(vma, addr, pfn, prot) \ - remap_pfn_range((vma), (addr), (pfn), PAGE_SIZE, \ - __pgprot(pgprot_val((prot)) | _PAGE_4K_PFN)) - -#endif /* __ASSEMBLY__ */ diff --git a/arch/powerpc/include/asm/pte-hash64.h b/arch/powerpc/include/asm/pte-hash64.h deleted file mode 100644 index 0419eeb53274..000000000000 --- a/arch/powerpc/include/asm/pte-hash64.h +++ /dev/null @@ -1,54 +0,0 @@ -#ifndef _ASM_POWERPC_PTE_HASH64_H -#define _ASM_POWERPC_PTE_HASH64_H -#ifdef __KERNEL__ - -/* - * Common bits between 4K and 64K pages in a linux-style PTE. - * These match the bits in the (hardware-defined) PowerPC PTE as closely - * as possible. Additional bits may be defined in pgtable-hash64-*.h - * - * Note: We only support user read/write permissions. Supervisor always - * have full read/write to pages above PAGE_OFFSET (pages below that - * always use the user access permissions). - * - * We could create separate kernel read-only if we used the 3 PP bits - * combinations that newer processors provide but we currently don't. - */ -#define _PAGE_PRESENT 0x0001 /* software: pte contains a translation */ -#define _PAGE_USER 0x0002 /* matches one of the PP bits */ -#define _PAGE_FILE 0x0002 /* (!present only) software: pte holds file offset */ -#define _PAGE_EXEC 0x0004 /* No execute on POWER4 and newer (we invert) */ -#define _PAGE_GUARDED 0x0008 -#define _PAGE_COHERENT 0x0010 /* M: enforce memory coherence (SMP systems) */ -#define _PAGE_NO_CACHE 0x0020 /* I: cache inhibit */ -#define _PAGE_WRITETHRU 0x0040 /* W: cache write-through */ -#define _PAGE_DIRTY 0x0080 /* C: page changed */ -#define _PAGE_ACCESSED 0x0100 /* R: page referenced */ -#define _PAGE_RW 0x0200 /* software: user write access allowed */ -#define _PAGE_BUSY 0x0800 /* software: PTE & hash are busy */ - -/* No separate kernel read-only */ -#define _PAGE_KERNEL_RW (_PAGE_RW | _PAGE_DIRTY) /* user access blocked by key */ -#define _PAGE_KERNEL_RO _PAGE_KERNEL_RW - -/* Strong Access Ordering */ -#define _PAGE_SAO (_PAGE_WRITETHRU | _PAGE_NO_CACHE | _PAGE_COHERENT) - -/* No page size encoding in the linux PTE */ -#define _PAGE_PSIZE 0 - -/* PTEIDX nibble */ -#define _PTEIDX_SECONDARY 0x8 -#define _PTEIDX_GROUP_IX 0x7 - -/* Hash table based platforms need atomic updates of the linux PTE */ -#define PTE_ATOMIC_UPDATES 1 - -#ifdef CONFIG_PPC_64K_PAGES -#include <asm/pte-hash64-64k.h> -#else -#include <asm/pte-hash64-4k.h> -#endif - -#endif /* __KERNEL__ */ -#endif /* _ASM_POWERPC_PTE_HASH64_H */ diff --git a/arch/powerpc/include/asm/pte-walk.h b/arch/powerpc/include/asm/pte-walk.h new file mode 100644 index 000000000000..73c22c579a79 --- /dev/null +++ b/arch/powerpc/include/asm/pte-walk.h @@ -0,0 +1,63 @@ +#ifndef _ASM_POWERPC_PTE_WALK_H +#define _ASM_POWERPC_PTE_WALK_H + +#include <linux/sched.h> + +/* Don't use this directly */ +extern pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea, + bool *is_thp, unsigned *hshift); + +static inline pte_t *find_linux_pte(pgd_t *pgdir, unsigned long ea, + bool *is_thp, unsigned *hshift) +{ + pte_t *pte; + + VM_WARN(!arch_irqs_disabled(), "%s called with irq enabled\n", __func__); + pte = __find_linux_pte(pgdir, ea, is_thp, hshift); + +#if defined(CONFIG_DEBUG_VM) && \ + !(defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)) + /* + * We should not find huge page if these configs are not enabled. + */ + if (hshift) + WARN_ON(*hshift); +#endif + return pte; +} + +static inline pte_t *find_init_mm_pte(unsigned long ea, unsigned *hshift) +{ + pgd_t *pgdir = init_mm.pgd; + return __find_linux_pte(pgdir, ea, NULL, hshift); +} + +/* + * Convert a kernel vmap virtual address (vmalloc or ioremap space) to a + * physical address, without taking locks. This can be used in real-mode. + */ +static inline phys_addr_t ppc_find_vmap_phys(unsigned long addr) +{ + pte_t *ptep; + phys_addr_t pa; + int hugepage_shift; + + /* + * init_mm does not free page tables, and does not do THP. It may + * have huge pages from huge vmalloc / ioremap etc. + */ + ptep = find_init_mm_pte(addr, &hugepage_shift); + if (WARN_ON(!ptep)) + return 0; + + pa = PFN_PHYS(pte_pfn(*ptep)); + + if (!hugepage_shift) + hugepage_shift = PAGE_SHIFT; + + pa |= addr & ((1ul << hugepage_shift) - 1); + + return pa; +} + +#endif /* _ASM_POWERPC_PTE_WALK_H */ diff --git a/arch/powerpc/include/asm/ptrace.h b/arch/powerpc/include/asm/ptrace.h index becc08e6a65c..94aa1de2b06e 100644 --- a/arch/powerpc/include/asm/ptrace.h +++ b/arch/powerpc/include/asm/ptrace.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Copyright (C) 2001 PPC64 Team, IBM Corp * @@ -14,26 +15,131 @@ * * Note that the offsets of the fields in this struct correspond with * the PT_* values below. This simplifies arch/powerpc/kernel/ptrace.c. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef _ASM_POWERPC_PTRACE_H #define _ASM_POWERPC_PTRACE_H +#include <linux/err.h> #include <uapi/asm/ptrace.h> +#include <asm/asm-const.h> +#include <asm/reg.h> +#ifndef __ASSEMBLER__ +struct pt_regs +{ + union { + struct user_pt_regs user_regs; + struct { + unsigned long gpr[32]; + unsigned long nip; + unsigned long msr; + unsigned long orig_gpr3; + unsigned long ctr; + unsigned long link; + unsigned long xer; + unsigned long ccr; +#ifdef CONFIG_PPC64 + unsigned long softe; +#else + unsigned long mq; +#endif + unsigned long trap; + union { + unsigned long dar; + unsigned long dear; + }; + union { + unsigned long dsisr; + unsigned long esr; + }; + unsigned long result; + }; + }; +#if defined(CONFIG_PPC64) || defined(CONFIG_PPC_KUAP) + union { + struct { +#ifdef CONFIG_PPC64 + unsigned long ppr; + unsigned long exit_result; +#endif + union { +#ifdef CONFIG_PPC_KUAP + unsigned long kuap; +#endif +#ifdef CONFIG_PPC_PKEY + unsigned long amr; +#endif + }; +#ifdef CONFIG_PPC_PKEY + unsigned long iamr; +#endif + }; + unsigned long __pad[4]; /* Maintain 16 byte interrupt stack alignment */ + }; +#endif +#if defined(CONFIG_PPC32) && defined(CONFIG_BOOKE) + struct { /* Must be a multiple of 16 bytes */ + unsigned long mas0; + unsigned long mas1; + unsigned long mas2; + unsigned long mas3; + unsigned long mas6; + unsigned long mas7; + unsigned long srr0; + unsigned long srr1; + unsigned long csrr0; + unsigned long csrr1; + unsigned long dsrr0; + unsigned long dsrr1; + }; +#endif +}; +#endif + + +// Always displays as "REGS" in memory dumps +#ifdef CONFIG_CPU_BIG_ENDIAN +#define STACK_FRAME_REGS_MARKER ASM_CONST(0x52454753) +#else +#define STACK_FRAME_REGS_MARKER ASM_CONST(0x53474552) +#endif #ifdef __powerpc64__ -#define STACK_FRAME_OVERHEAD 112 /* size of minimum stack frame */ +/* + * Size of redzone that userspace is allowed to use below the stack + * pointer. This is 288 in the 64-bit big-endian ELF ABI, and 512 in + * the new ELFv2 little-endian ABI, so we allow the larger amount. + * + * For kernel code we allow a 288-byte redzone, in order to conserve + * kernel stack space; gcc currently only uses 288 bytes, and will + * hopefully allow explicit control of the redzone size in future. + */ +#define USER_REDZONE_SIZE 512 +#define KERNEL_REDZONE_SIZE 288 + #define STACK_FRAME_LR_SAVE 2 /* Location of LR in stack frame */ -#define STACK_FRAME_REGS_MARKER ASM_CONST(0x7265677368657265) -#define STACK_INT_FRAME_SIZE (sizeof(struct pt_regs) + \ - STACK_FRAME_OVERHEAD + 288) -#define STACK_FRAME_MARKER 12 + +#ifdef CONFIG_PPC64_ELF_ABI_V2 +#define STACK_FRAME_MIN_SIZE 32 +#define STACK_USER_INT_FRAME_SIZE (sizeof(struct pt_regs) + STACK_FRAME_MIN_SIZE + 16) +#define STACK_INT_FRAME_REGS (STACK_FRAME_MIN_SIZE + 16) +#define STACK_INT_FRAME_MARKER STACK_FRAME_MIN_SIZE +#define STACK_SWITCH_FRAME_SIZE (sizeof(struct pt_regs) + STACK_FRAME_MIN_SIZE + 16) +#define STACK_SWITCH_FRAME_REGS (STACK_FRAME_MIN_SIZE + 16) +#else +/* + * The ELFv1 ABI specifies 48 bytes plus a minimum 64 byte parameter save + * area. This parameter area is not used by calls to C from interrupt entry, + * so the second from last one of those is used for the frame marker. + */ +#define STACK_FRAME_MIN_SIZE 112 +#define STACK_USER_INT_FRAME_SIZE (sizeof(struct pt_regs) + STACK_FRAME_MIN_SIZE) +#define STACK_INT_FRAME_REGS STACK_FRAME_MIN_SIZE +#define STACK_INT_FRAME_MARKER (STACK_FRAME_MIN_SIZE - 16) +#define STACK_SWITCH_FRAME_SIZE (sizeof(struct pt_regs) + STACK_FRAME_MIN_SIZE) +#define STACK_SWITCH_FRAME_REGS STACK_FRAME_MIN_SIZE +#endif /* Size of dummy stack frame allocated when calling signal handler. */ #define __SIGNAL_FRAMESIZE 128 @@ -41,93 +147,192 @@ #else /* __powerpc64__ */ -#define STACK_FRAME_OVERHEAD 16 /* size of minimum stack frame */ +#define USER_REDZONE_SIZE 0 +#define KERNEL_REDZONE_SIZE 0 +#define STACK_FRAME_MIN_SIZE 16 #define STACK_FRAME_LR_SAVE 1 /* Location of LR in stack frame */ -#define STACK_FRAME_REGS_MARKER ASM_CONST(0x72656773) -#define STACK_INT_FRAME_SIZE (sizeof(struct pt_regs) + STACK_FRAME_OVERHEAD) -#define STACK_FRAME_MARKER 2 +#define STACK_USER_INT_FRAME_SIZE (sizeof(struct pt_regs) + STACK_FRAME_MIN_SIZE) +#define STACK_INT_FRAME_REGS STACK_FRAME_MIN_SIZE +#define STACK_INT_FRAME_MARKER (STACK_FRAME_MIN_SIZE - 8) +#define STACK_SWITCH_FRAME_SIZE (sizeof(struct pt_regs) + STACK_FRAME_MIN_SIZE) +#define STACK_SWITCH_FRAME_REGS STACK_FRAME_MIN_SIZE /* Size of stack frame allocated when calling signal handler. */ #define __SIGNAL_FRAMESIZE 64 #endif /* __powerpc64__ */ -#ifndef __ASSEMBLY__ +#define STACK_INT_FRAME_SIZE (KERNEL_REDZONE_SIZE + STACK_USER_INT_FRAME_SIZE) +#define STACK_INT_FRAME_MARKER_LONGS (STACK_INT_FRAME_MARKER/sizeof(long)) -#define GET_IP(regs) ((regs)->nip) -#define GET_USP(regs) ((regs)->gpr[1]) -#define GET_FP(regs) (0) -#define SET_FP(regs, val) +#ifndef __ASSEMBLER__ +#include <asm/paca.h> #ifdef CONFIG_SMP extern unsigned long profile_pc(struct pt_regs *regs); -#define profile_pc profile_pc +#else +#define profile_pc(regs) instruction_pointer(regs) #endif -#include <asm-generic/ptrace.h> +long do_syscall_trace_enter(struct pt_regs *regs); +void do_syscall_trace_leave(struct pt_regs *regs); -#define kernel_stack_pointer(regs) ((regs)->gpr[1]) -static inline int is_syscall_success(struct pt_regs *regs) +static inline void set_return_regs_changed(void) { - return !(regs->ccr & 0x10000000); +#ifdef CONFIG_PPC_BOOK3S_64 + WRITE_ONCE(local_paca->hsrr_valid, 0); + WRITE_ONCE(local_paca->srr_valid, 0); +#endif } -static inline long regs_return_value(struct pt_regs *regs) +static inline void regs_set_return_ip(struct pt_regs *regs, unsigned long ip) { - if (is_syscall_success(regs)) - return regs->gpr[3]; - else - return -regs->gpr[3]; + regs->nip = ip; + set_return_regs_changed(); +} + +static inline void regs_set_return_msr(struct pt_regs *regs, unsigned long msr) +{ + regs->msr = msr; + set_return_regs_changed(); +} + +static inline void regs_add_return_ip(struct pt_regs *regs, long offset) +{ + regs_set_return_ip(regs, regs->nip + offset); +} + +static inline unsigned long instruction_pointer(struct pt_regs *regs) +{ + return regs->nip; +} + +static inline void instruction_pointer_set(struct pt_regs *regs, + unsigned long val) +{ + regs_set_return_ip(regs, val); +} + +static inline unsigned long user_stack_pointer(struct pt_regs *regs) +{ + return regs->gpr[1]; +} + +static inline unsigned long frame_pointer(struct pt_regs *regs) +{ + return 0; } -#ifdef __powerpc64__ -#define user_mode(regs) ((((regs)->msr) >> MSR_PR_LG) & 0x1) -#else #define user_mode(regs) (((regs)->msr & MSR_PR) != 0) -#endif #define force_successful_syscall_return() \ do { \ set_thread_flag(TIF_NOERROR); \ } while(0) -struct task_struct; -extern int ptrace_get_reg(struct task_struct *task, int regno, - unsigned long *data); -extern int ptrace_put_reg(struct task_struct *task, int regno, - unsigned long data); - #define current_pt_regs() \ - ((struct pt_regs *)((unsigned long)current_thread_info() + THREAD_SIZE) - 1) + ((struct pt_regs *)((unsigned long)task_stack_page(current) + THREAD_SIZE) - 1) + +/* + * The 4 low bits (0xf) are available as flags to overload the trap word, + * because interrupt vectors have minimum alignment of 0x10. TRAP_FLAGS_MASK + * must cover the bits used as flags, including bit 0 which is used as the + * "norestart" bit. + */ +#ifdef __powerpc64__ +#define TRAP_FLAGS_MASK 0x1 +#else /* - * We use the least-significant bit of the trap field to indicate - * whether we have saved the full set of registers, or only a - * partial set. A 1 there means the partial set. - * On 4xx we use the next bit to indicate whether the exception + * On 4xx we use bit 1 in the trap word to indicate whether the exception * is a critical exception (1 means it is). */ -#define FULL_REGS(regs) (((regs)->trap & 1) == 0) -#ifndef __powerpc64__ +#define TRAP_FLAGS_MASK 0xf #define IS_CRITICAL_EXC(regs) (((regs)->trap & 2) != 0) #define IS_MCHECK_EXC(regs) (((regs)->trap & 4) != 0) #define IS_DEBUG_EXC(regs) (((regs)->trap & 8) != 0) -#endif /* ! __powerpc64__ */ -#define TRAP(regs) ((regs)->trap & ~0xF) -#ifdef __powerpc64__ -#define NV_REG_POISON 0xdeadbeefdeadbeefUL -#define CHECK_FULL_REGS(regs) BUG_ON(regs->trap & 1) -#else -#define NV_REG_POISON 0xdeadbeef -#define CHECK_FULL_REGS(regs) \ -do { \ - if ((regs)->trap & 1) \ - printk(KERN_CRIT "%s: partial register set\n", __func__); \ -} while (0) #endif /* __powerpc64__ */ +#define TRAP(regs) ((regs)->trap & ~TRAP_FLAGS_MASK) + +static __always_inline void set_trap(struct pt_regs *regs, unsigned long val) +{ + regs->trap = (regs->trap & TRAP_FLAGS_MASK) | (val & ~TRAP_FLAGS_MASK); +} + +static inline bool trap_is_scv(struct pt_regs *regs) +{ + return (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && TRAP(regs) == 0x3000); +} + +static inline bool trap_is_unsupported_scv(struct pt_regs *regs) +{ + return IS_ENABLED(CONFIG_PPC_BOOK3S_64) && TRAP(regs) == 0x7ff0; +} + +static inline bool trap_is_syscall(struct pt_regs *regs) +{ + return (trap_is_scv(regs) || TRAP(regs) == 0xc00); +} + +static inline bool trap_norestart(struct pt_regs *regs) +{ + return regs->trap & 0x1; +} + +static __always_inline void set_trap_norestart(struct pt_regs *regs) +{ + regs->trap |= 0x1; +} + +#define kernel_stack_pointer(regs) ((regs)->gpr[1]) +static inline int is_syscall_success(struct pt_regs *regs) +{ + if (trap_is_scv(regs)) + return !IS_ERR_VALUE((unsigned long)regs->gpr[3]); + else + return !(regs->ccr & 0x10000000); +} + +static inline long regs_return_value(struct pt_regs *regs) +{ + if (trap_is_scv(regs)) + return regs->gpr[3]; + + if (is_syscall_success(regs)) + return regs->gpr[3]; + else + return -regs->gpr[3]; +} + +static inline void regs_set_return_value(struct pt_regs *regs, unsigned long rc) +{ + regs->gpr[3] = rc; +} + +static inline bool cpu_has_msr_ri(void) +{ + return !IS_ENABLED(CONFIG_BOOKE); +} + +static inline bool regs_is_unrecoverable(struct pt_regs *regs) +{ + return unlikely(cpu_has_msr_ri() && !(regs->msr & MSR_RI)); +} + +static inline void regs_set_recoverable(struct pt_regs *regs) +{ + if (cpu_has_msr_ri()) + regs_set_return_msr(regs, regs->msr | MSR_RI); +} + +static inline void regs_set_unrecoverable(struct pt_regs *regs) +{ + if (cpu_has_msr_ri()) + regs_set_return_msr(regs, regs->msr & ~MSR_RI); +} #define arch_has_single_step() (1) -#define arch_has_block_step() (!cpu_has_feature(CPU_FTR_601)) -#define ARCH_HAS_USER_SINGLE_STEP_INFO +#define arch_has_block_step() (true) +#define ARCH_HAS_USER_SINGLE_STEP_REPORT /* * kprobe-based event tracer support @@ -192,9 +397,28 @@ static inline unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, return 0; } -#endif /* __ASSEMBLY__ */ +/** + * regs_get_kernel_argument() - get Nth function argument in kernel + * @regs: pt_regs of that context + * @n: function argument number (start from 0) + * + * We support up to 8 arguments and assume they are sent in through the GPRs. + * This will fail for fp/vector arguments, but those aren't usually found in + * kernel code. This is expected to be called from kprobes or ftrace with regs. + */ +static inline unsigned long regs_get_kernel_argument(struct pt_regs *regs, unsigned int n) +{ +#define NR_REG_ARGUMENTS 8 + if (n < NR_REG_ARGUMENTS) + return regs_get_register(regs, offsetof(struct pt_regs, gpr[3 + n])); + return 0; +} + +#endif /* __ASSEMBLER__ */ #ifndef __powerpc64__ +/* We need PT_SOFTE defined at all time to avoid #ifdefs */ +#define PT_SOFTE PT_MQ #else /* __powerpc64__ */ #define PT_FPSCR32 (PT_FPR0 + 2*32 + 1) /* each FP reg occupies 2 32-bit userspace slots */ #define PT_VR0_32 164 /* each Vector reg occupies 4 slots in 32-bit */ diff --git a/arch/powerpc/include/asm/qe.h b/arch/powerpc/include/asm/qe.h deleted file mode 100644 index 32b9bfa0c9bd..000000000000 --- a/arch/powerpc/include/asm/qe.h +++ /dev/null @@ -1,740 +0,0 @@ -/* - * Copyright (C) 2006 Freescale Semiconductor, Inc. All rights reserved. - * - * Authors: Shlomi Gridish <gridish@freescale.com> - * Li Yang <leoli@freescale.com> - * - * Description: - * QUICC Engine (QE) external definitions and structure. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - */ -#ifndef _ASM_POWERPC_QE_H -#define _ASM_POWERPC_QE_H -#ifdef __KERNEL__ - -#include <linux/spinlock.h> -#include <linux/errno.h> -#include <linux/err.h> -#include <asm/cpm.h> -#include <asm/immap_qe.h> - -#define QE_NUM_OF_SNUM 256 /* There are 256 serial number in QE */ -#define QE_NUM_OF_BRGS 16 -#define QE_NUM_OF_PORTS 1024 - -/* Memory partitions -*/ -#define MEM_PART_SYSTEM 0 -#define MEM_PART_SECONDARY 1 -#define MEM_PART_MURAM 2 - -/* Clocks and BRGs */ -enum qe_clock { - QE_CLK_NONE = 0, - QE_BRG1, /* Baud Rate Generator 1 */ - QE_BRG2, /* Baud Rate Generator 2 */ - QE_BRG3, /* Baud Rate Generator 3 */ - QE_BRG4, /* Baud Rate Generator 4 */ - QE_BRG5, /* Baud Rate Generator 5 */ - QE_BRG6, /* Baud Rate Generator 6 */ - QE_BRG7, /* Baud Rate Generator 7 */ - QE_BRG8, /* Baud Rate Generator 8 */ - QE_BRG9, /* Baud Rate Generator 9 */ - QE_BRG10, /* Baud Rate Generator 10 */ - QE_BRG11, /* Baud Rate Generator 11 */ - QE_BRG12, /* Baud Rate Generator 12 */ - QE_BRG13, /* Baud Rate Generator 13 */ - QE_BRG14, /* Baud Rate Generator 14 */ - QE_BRG15, /* Baud Rate Generator 15 */ - QE_BRG16, /* Baud Rate Generator 16 */ - QE_CLK1, /* Clock 1 */ - QE_CLK2, /* Clock 2 */ - QE_CLK3, /* Clock 3 */ - QE_CLK4, /* Clock 4 */ - QE_CLK5, /* Clock 5 */ - QE_CLK6, /* Clock 6 */ - QE_CLK7, /* Clock 7 */ - QE_CLK8, /* Clock 8 */ - QE_CLK9, /* Clock 9 */ - QE_CLK10, /* Clock 10 */ - QE_CLK11, /* Clock 11 */ - QE_CLK12, /* Clock 12 */ - QE_CLK13, /* Clock 13 */ - QE_CLK14, /* Clock 14 */ - QE_CLK15, /* Clock 15 */ - QE_CLK16, /* Clock 16 */ - QE_CLK17, /* Clock 17 */ - QE_CLK18, /* Clock 18 */ - QE_CLK19, /* Clock 19 */ - QE_CLK20, /* Clock 20 */ - QE_CLK21, /* Clock 21 */ - QE_CLK22, /* Clock 22 */ - QE_CLK23, /* Clock 23 */ - QE_CLK24, /* Clock 24 */ - QE_CLK_DUMMY -}; - -static inline bool qe_clock_is_brg(enum qe_clock clk) -{ - return clk >= QE_BRG1 && clk <= QE_BRG16; -} - -extern spinlock_t cmxgcr_lock; - -/* Export QE common operations */ -#ifdef CONFIG_QUICC_ENGINE -extern void qe_reset(void); -#else -static inline void qe_reset(void) {} -#endif - -/* QE PIO */ -#define QE_PIO_PINS 32 - -struct qe_pio_regs { - __be32 cpodr; /* Open drain register */ - __be32 cpdata; /* Data register */ - __be32 cpdir1; /* Direction register */ - __be32 cpdir2; /* Direction register */ - __be32 cppar1; /* Pin assignment register */ - __be32 cppar2; /* Pin assignment register */ -#ifdef CONFIG_PPC_85xx - u8 pad[8]; -#endif -}; - -#define QE_PIO_DIR_IN 2 -#define QE_PIO_DIR_OUT 1 -extern void __par_io_config_pin(struct qe_pio_regs __iomem *par_io, u8 pin, - int dir, int open_drain, int assignment, - int has_irq); -#ifdef CONFIG_QUICC_ENGINE -extern int par_io_init(struct device_node *np); -extern int par_io_of_config(struct device_node *np); -extern int par_io_config_pin(u8 port, u8 pin, int dir, int open_drain, - int assignment, int has_irq); -extern int par_io_data_set(u8 port, u8 pin, u8 val); -#else -static inline int par_io_init(struct device_node *np) { return -ENOSYS; } -static inline int par_io_of_config(struct device_node *np) { return -ENOSYS; } -static inline int par_io_config_pin(u8 port, u8 pin, int dir, int open_drain, - int assignment, int has_irq) { return -ENOSYS; } -static inline int par_io_data_set(u8 port, u8 pin, u8 val) { return -ENOSYS; } -#endif /* CONFIG_QUICC_ENGINE */ - -/* - * Pin multiplexing functions. - */ -struct qe_pin; -#ifdef CONFIG_QE_GPIO -extern struct qe_pin *qe_pin_request(struct device_node *np, int index); -extern void qe_pin_free(struct qe_pin *qe_pin); -extern void qe_pin_set_gpio(struct qe_pin *qe_pin); -extern void qe_pin_set_dedicated(struct qe_pin *pin); -#else -static inline struct qe_pin *qe_pin_request(struct device_node *np, int index) -{ - return ERR_PTR(-ENOSYS); -} -static inline void qe_pin_free(struct qe_pin *qe_pin) {} -static inline void qe_pin_set_gpio(struct qe_pin *qe_pin) {} -static inline void qe_pin_set_dedicated(struct qe_pin *pin) {} -#endif /* CONFIG_QE_GPIO */ - -#ifdef CONFIG_QUICC_ENGINE -int qe_issue_cmd(u32 cmd, u32 device, u8 mcn_protocol, u32 cmd_input); -#else -static inline int qe_issue_cmd(u32 cmd, u32 device, u8 mcn_protocol, - u32 cmd_input) -{ - return -ENOSYS; -} -#endif /* CONFIG_QUICC_ENGINE */ - -/* QE internal API */ -enum qe_clock qe_clock_source(const char *source); -unsigned int qe_get_brg_clk(void); -int qe_setbrg(enum qe_clock brg, unsigned int rate, unsigned int multiplier); -int qe_get_snum(void); -void qe_put_snum(u8 snum); -unsigned int qe_get_num_of_risc(void); -unsigned int qe_get_num_of_snums(void); - -static inline int qe_alive_during_sleep(void) -{ - /* - * MPC8568E reference manual says: - * - * "...power down sequence waits for all I/O interfaces to become idle. - * In some applications this may happen eventually without actively - * shutting down interfaces, but most likely, software will have to - * take steps to shut down the eTSEC, QUICC Engine Block, and PCI - * interfaces before issuing the command (either the write to the core - * MSR[WE] as described above or writing to POWMGTCSR) to put the - * device into sleep state." - * - * MPC8569E reference manual has a similar paragraph. - */ -#ifdef CONFIG_PPC_85xx - return 0; -#else - return 1; -#endif -} - -/* we actually use cpm_muram implementation, define this for convenience */ -#define qe_muram_init cpm_muram_init -#define qe_muram_alloc cpm_muram_alloc -#define qe_muram_alloc_fixed cpm_muram_alloc_fixed -#define qe_muram_free cpm_muram_free -#define qe_muram_addr cpm_muram_addr -#define qe_muram_offset cpm_muram_offset - -/* Structure that defines QE firmware binary files. - * - * See Documentation/powerpc/qe_firmware.txt for a description of these - * fields. - */ -struct qe_firmware { - struct qe_header { - __be32 length; /* Length of the entire structure, in bytes */ - u8 magic[3]; /* Set to { 'Q', 'E', 'F' } */ - u8 version; /* Version of this layout. First ver is '1' */ - } header; - u8 id[62]; /* Null-terminated identifier string */ - u8 split; /* 0 = shared I-RAM, 1 = split I-RAM */ - u8 count; /* Number of microcode[] structures */ - struct { - __be16 model; /* The SOC model */ - u8 major; /* The SOC revision major */ - u8 minor; /* The SOC revision minor */ - } __attribute__ ((packed)) soc; - u8 padding[4]; /* Reserved, for alignment */ - __be64 extended_modes; /* Extended modes */ - __be32 vtraps[8]; /* Virtual trap addresses */ - u8 reserved[4]; /* Reserved, for future expansion */ - struct qe_microcode { - u8 id[32]; /* Null-terminated identifier */ - __be32 traps[16]; /* Trap addresses, 0 == ignore */ - __be32 eccr; /* The value for the ECCR register */ - __be32 iram_offset; /* Offset into I-RAM for the code */ - __be32 count; /* Number of 32-bit words of the code */ - __be32 code_offset; /* Offset of the actual microcode */ - u8 major; /* The microcode version major */ - u8 minor; /* The microcode version minor */ - u8 revision; /* The microcode version revision */ - u8 padding; /* Reserved, for alignment */ - u8 reserved[4]; /* Reserved, for future expansion */ - } __attribute__ ((packed)) microcode[1]; - /* All microcode binaries should be located here */ - /* CRC32 should be located here, after the microcode binaries */ -} __attribute__ ((packed)); - -struct qe_firmware_info { - char id[64]; /* Firmware name */ - u32 vtraps[8]; /* Virtual trap addresses */ - u64 extended_modes; /* Extended modes */ -}; - -#ifdef CONFIG_QUICC_ENGINE -/* Upload a firmware to the QE */ -int qe_upload_firmware(const struct qe_firmware *firmware); -#else -static inline int qe_upload_firmware(const struct qe_firmware *firmware) -{ - return -ENOSYS; -} -#endif /* CONFIG_QUICC_ENGINE */ - -/* Obtain information on the uploaded firmware */ -struct qe_firmware_info *qe_get_firmware_info(void); - -/* QE USB */ -int qe_usb_clock_set(enum qe_clock clk, int rate); - -/* Buffer descriptors */ -struct qe_bd { - __be16 status; - __be16 length; - __be32 buf; -} __attribute__ ((packed)); - -#define BD_STATUS_MASK 0xffff0000 -#define BD_LENGTH_MASK 0x0000ffff - -/* Alignment */ -#define QE_INTR_TABLE_ALIGN 16 /* ??? */ -#define QE_ALIGNMENT_OF_BD 8 -#define QE_ALIGNMENT_OF_PRAM 64 - -/* RISC allocation */ -#define QE_RISC_ALLOCATION_RISC1 0x1 /* RISC 1 */ -#define QE_RISC_ALLOCATION_RISC2 0x2 /* RISC 2 */ -#define QE_RISC_ALLOCATION_RISC3 0x4 /* RISC 3 */ -#define QE_RISC_ALLOCATION_RISC4 0x8 /* RISC 4 */ -#define QE_RISC_ALLOCATION_RISC1_AND_RISC2 (QE_RISC_ALLOCATION_RISC1 | \ - QE_RISC_ALLOCATION_RISC2) -#define QE_RISC_ALLOCATION_FOUR_RISCS (QE_RISC_ALLOCATION_RISC1 | \ - QE_RISC_ALLOCATION_RISC2 | \ - QE_RISC_ALLOCATION_RISC3 | \ - QE_RISC_ALLOCATION_RISC4) - -/* QE extended filtering Table Lookup Key Size */ -enum qe_fltr_tbl_lookup_key_size { - QE_FLTR_TABLE_LOOKUP_KEY_SIZE_8_BYTES - = 0x3f, /* LookupKey parsed by the Generate LookupKey - CMD is truncated to 8 bytes */ - QE_FLTR_TABLE_LOOKUP_KEY_SIZE_16_BYTES - = 0x5f, /* LookupKey parsed by the Generate LookupKey - CMD is truncated to 16 bytes */ -}; - -/* QE FLTR extended filtering Largest External Table Lookup Key Size */ -enum qe_fltr_largest_external_tbl_lookup_key_size { - QE_FLTR_LARGEST_EXTERNAL_TABLE_LOOKUP_KEY_SIZE_NONE - = 0x0,/* not used */ - QE_FLTR_LARGEST_EXTERNAL_TABLE_LOOKUP_KEY_SIZE_8_BYTES - = QE_FLTR_TABLE_LOOKUP_KEY_SIZE_8_BYTES, /* 8 bytes */ - QE_FLTR_LARGEST_EXTERNAL_TABLE_LOOKUP_KEY_SIZE_16_BYTES - = QE_FLTR_TABLE_LOOKUP_KEY_SIZE_16_BYTES, /* 16 bytes */ -}; - -/* structure representing QE parameter RAM */ -struct qe_timer_tables { - u16 tm_base; /* QE timer table base adr */ - u16 tm_ptr; /* QE timer table pointer */ - u16 r_tmr; /* QE timer mode register */ - u16 r_tmv; /* QE timer valid register */ - u32 tm_cmd; /* QE timer cmd register */ - u32 tm_cnt; /* QE timer internal cnt */ -} __attribute__ ((packed)); - -#define QE_FLTR_TAD_SIZE 8 - -/* QE extended filtering Termination Action Descriptor (TAD) */ -struct qe_fltr_tad { - u8 serialized[QE_FLTR_TAD_SIZE]; -} __attribute__ ((packed)); - -/* Communication Direction */ -enum comm_dir { - COMM_DIR_NONE = 0, - COMM_DIR_RX = 1, - COMM_DIR_TX = 2, - COMM_DIR_RX_AND_TX = 3 -}; - -/* QE CMXUCR Registers. - * There are two UCCs represented in each of the four CMXUCR registers. - * These values are for the UCC in the LSBs - */ -#define QE_CMXUCR_MII_ENET_MNG 0x00007000 -#define QE_CMXUCR_MII_ENET_MNG_SHIFT 12 -#define QE_CMXUCR_GRANT 0x00008000 -#define QE_CMXUCR_TSA 0x00004000 -#define QE_CMXUCR_BKPT 0x00000100 -#define QE_CMXUCR_TX_CLK_SRC_MASK 0x0000000F - -/* QE CMXGCR Registers. -*/ -#define QE_CMXGCR_MII_ENET_MNG 0x00007000 -#define QE_CMXGCR_MII_ENET_MNG_SHIFT 12 -#define QE_CMXGCR_USBCS 0x0000000f -#define QE_CMXGCR_USBCS_CLK3 0x1 -#define QE_CMXGCR_USBCS_CLK5 0x2 -#define QE_CMXGCR_USBCS_CLK7 0x3 -#define QE_CMXGCR_USBCS_CLK9 0x4 -#define QE_CMXGCR_USBCS_CLK13 0x5 -#define QE_CMXGCR_USBCS_CLK17 0x6 -#define QE_CMXGCR_USBCS_CLK19 0x7 -#define QE_CMXGCR_USBCS_CLK21 0x8 -#define QE_CMXGCR_USBCS_BRG9 0x9 -#define QE_CMXGCR_USBCS_BRG10 0xa - -/* QE CECR Commands. -*/ -#define QE_CR_FLG 0x00010000 -#define QE_RESET 0x80000000 -#define QE_INIT_TX_RX 0x00000000 -#define QE_INIT_RX 0x00000001 -#define QE_INIT_TX 0x00000002 -#define QE_ENTER_HUNT_MODE 0x00000003 -#define QE_STOP_TX 0x00000004 -#define QE_GRACEFUL_STOP_TX 0x00000005 -#define QE_RESTART_TX 0x00000006 -#define QE_CLOSE_RX_BD 0x00000007 -#define QE_SWITCH_COMMAND 0x00000007 -#define QE_SET_GROUP_ADDRESS 0x00000008 -#define QE_START_IDMA 0x00000009 -#define QE_MCC_STOP_RX 0x00000009 -#define QE_ATM_TRANSMIT 0x0000000a -#define QE_HPAC_CLEAR_ALL 0x0000000b -#define QE_GRACEFUL_STOP_RX 0x0000001a -#define QE_RESTART_RX 0x0000001b -#define QE_HPAC_SET_PRIORITY 0x0000010b -#define QE_HPAC_STOP_TX 0x0000020b -#define QE_HPAC_STOP_RX 0x0000030b -#define QE_HPAC_GRACEFUL_STOP_TX 0x0000040b -#define QE_HPAC_GRACEFUL_STOP_RX 0x0000050b -#define QE_HPAC_START_TX 0x0000060b -#define QE_HPAC_START_RX 0x0000070b -#define QE_USB_STOP_TX 0x0000000a -#define QE_USB_RESTART_TX 0x0000000c -#define QE_QMC_STOP_TX 0x0000000c -#define QE_QMC_STOP_RX 0x0000000d -#define QE_SS7_SU_FIL_RESET 0x0000000e -/* jonathbr added from here down for 83xx */ -#define QE_RESET_BCS 0x0000000a -#define QE_MCC_INIT_TX_RX_16 0x00000003 -#define QE_MCC_STOP_TX 0x00000004 -#define QE_MCC_INIT_TX_1 0x00000005 -#define QE_MCC_INIT_RX_1 0x00000006 -#define QE_MCC_RESET 0x00000007 -#define QE_SET_TIMER 0x00000008 -#define QE_RANDOM_NUMBER 0x0000000c -#define QE_ATM_MULTI_THREAD_INIT 0x00000011 -#define QE_ASSIGN_PAGE 0x00000012 -#define QE_ADD_REMOVE_HASH_ENTRY 0x00000013 -#define QE_START_FLOW_CONTROL 0x00000014 -#define QE_STOP_FLOW_CONTROL 0x00000015 -#define QE_ASSIGN_PAGE_TO_DEVICE 0x00000016 - -#define QE_ASSIGN_RISC 0x00000010 -#define QE_CR_MCN_NORMAL_SHIFT 6 -#define QE_CR_MCN_USB_SHIFT 4 -#define QE_CR_MCN_RISC_ASSIGN_SHIFT 8 -#define QE_CR_SNUM_SHIFT 17 - -/* QE CECR Sub Block - sub block of QE command. -*/ -#define QE_CR_SUBBLOCK_INVALID 0x00000000 -#define QE_CR_SUBBLOCK_USB 0x03200000 -#define QE_CR_SUBBLOCK_UCCFAST1 0x02000000 -#define QE_CR_SUBBLOCK_UCCFAST2 0x02200000 -#define QE_CR_SUBBLOCK_UCCFAST3 0x02400000 -#define QE_CR_SUBBLOCK_UCCFAST4 0x02600000 -#define QE_CR_SUBBLOCK_UCCFAST5 0x02800000 -#define QE_CR_SUBBLOCK_UCCFAST6 0x02a00000 -#define QE_CR_SUBBLOCK_UCCFAST7 0x02c00000 -#define QE_CR_SUBBLOCK_UCCFAST8 0x02e00000 -#define QE_CR_SUBBLOCK_UCCSLOW1 0x00000000 -#define QE_CR_SUBBLOCK_UCCSLOW2 0x00200000 -#define QE_CR_SUBBLOCK_UCCSLOW3 0x00400000 -#define QE_CR_SUBBLOCK_UCCSLOW4 0x00600000 -#define QE_CR_SUBBLOCK_UCCSLOW5 0x00800000 -#define QE_CR_SUBBLOCK_UCCSLOW6 0x00a00000 -#define QE_CR_SUBBLOCK_UCCSLOW7 0x00c00000 -#define QE_CR_SUBBLOCK_UCCSLOW8 0x00e00000 -#define QE_CR_SUBBLOCK_MCC1 0x03800000 -#define QE_CR_SUBBLOCK_MCC2 0x03a00000 -#define QE_CR_SUBBLOCK_MCC3 0x03000000 -#define QE_CR_SUBBLOCK_IDMA1 0x02800000 -#define QE_CR_SUBBLOCK_IDMA2 0x02a00000 -#define QE_CR_SUBBLOCK_IDMA3 0x02c00000 -#define QE_CR_SUBBLOCK_IDMA4 0x02e00000 -#define QE_CR_SUBBLOCK_HPAC 0x01e00000 -#define QE_CR_SUBBLOCK_SPI1 0x01400000 -#define QE_CR_SUBBLOCK_SPI2 0x01600000 -#define QE_CR_SUBBLOCK_RAND 0x01c00000 -#define QE_CR_SUBBLOCK_TIMER 0x01e00000 -#define QE_CR_SUBBLOCK_GENERAL 0x03c00000 - -/* QE CECR Protocol - For non-MCC, specifies mode for QE CECR command */ -#define QE_CR_PROTOCOL_UNSPECIFIED 0x00 /* For all other protocols */ -#define QE_CR_PROTOCOL_HDLC_TRANSPARENT 0x00 -#define QE_CR_PROTOCOL_QMC 0x02 -#define QE_CR_PROTOCOL_UART 0x04 -#define QE_CR_PROTOCOL_ATM_POS 0x0A -#define QE_CR_PROTOCOL_ETHERNET 0x0C -#define QE_CR_PROTOCOL_L2_SWITCH 0x0D - -/* BRG configuration register */ -#define QE_BRGC_ENABLE 0x00010000 -#define QE_BRGC_DIVISOR_SHIFT 1 -#define QE_BRGC_DIVISOR_MAX 0xFFF -#define QE_BRGC_DIV16 1 - -/* QE Timers registers */ -#define QE_GTCFR1_PCAS 0x80 -#define QE_GTCFR1_STP2 0x20 -#define QE_GTCFR1_RST2 0x10 -#define QE_GTCFR1_GM2 0x08 -#define QE_GTCFR1_GM1 0x04 -#define QE_GTCFR1_STP1 0x02 -#define QE_GTCFR1_RST1 0x01 - -/* SDMA registers */ -#define QE_SDSR_BER1 0x02000000 -#define QE_SDSR_BER2 0x01000000 - -#define QE_SDMR_GLB_1_MSK 0x80000000 -#define QE_SDMR_ADR_SEL 0x20000000 -#define QE_SDMR_BER1_MSK 0x02000000 -#define QE_SDMR_BER2_MSK 0x01000000 -#define QE_SDMR_EB1_MSK 0x00800000 -#define QE_SDMR_ER1_MSK 0x00080000 -#define QE_SDMR_ER2_MSK 0x00040000 -#define QE_SDMR_CEN_MASK 0x0000E000 -#define QE_SDMR_SBER_1 0x00000200 -#define QE_SDMR_SBER_2 0x00000200 -#define QE_SDMR_EB1_PR_MASK 0x000000C0 -#define QE_SDMR_ER1_PR 0x00000008 - -#define QE_SDMR_CEN_SHIFT 13 -#define QE_SDMR_EB1_PR_SHIFT 6 - -#define QE_SDTM_MSNUM_SHIFT 24 - -#define QE_SDEBCR_BA_MASK 0x01FFFFFF - -/* Communication Processor */ -#define QE_CP_CERCR_MEE 0x8000 /* Multi-user RAM ECC enable */ -#define QE_CP_CERCR_IEE 0x4000 /* Instruction RAM ECC enable */ -#define QE_CP_CERCR_CIR 0x0800 /* Common instruction RAM */ - -/* I-RAM */ -#define QE_IRAM_IADD_AIE 0x80000000 /* Auto Increment Enable */ -#define QE_IRAM_IADD_BADDR 0x00080000 /* Base Address */ -#define QE_IRAM_READY 0x80000000 /* Ready */ - -/* UPC */ -#define UPGCR_PROTOCOL 0x80000000 /* protocol ul2 or pl2 */ -#define UPGCR_TMS 0x40000000 /* Transmit master/slave mode */ -#define UPGCR_RMS 0x20000000 /* Receive master/slave mode */ -#define UPGCR_ADDR 0x10000000 /* Master MPHY Addr multiplexing */ -#define UPGCR_DIAG 0x01000000 /* Diagnostic mode */ - -/* UCC GUEMR register */ -#define UCC_GUEMR_MODE_MASK_RX 0x02 -#define UCC_GUEMR_MODE_FAST_RX 0x02 -#define UCC_GUEMR_MODE_SLOW_RX 0x00 -#define UCC_GUEMR_MODE_MASK_TX 0x01 -#define UCC_GUEMR_MODE_FAST_TX 0x01 -#define UCC_GUEMR_MODE_SLOW_TX 0x00 -#define UCC_GUEMR_MODE_MASK (UCC_GUEMR_MODE_MASK_RX | UCC_GUEMR_MODE_MASK_TX) -#define UCC_GUEMR_SET_RESERVED3 0x10 /* Bit 3 in the guemr is reserved but - must be set 1 */ - -/* structure representing UCC SLOW parameter RAM */ -struct ucc_slow_pram { - __be16 rbase; /* RX BD base address */ - __be16 tbase; /* TX BD base address */ - u8 rbmr; /* RX bus mode register (same as CPM's RFCR) */ - u8 tbmr; /* TX bus mode register (same as CPM's TFCR) */ - __be16 mrblr; /* Rx buffer length */ - __be32 rstate; /* Rx internal state */ - __be32 rptr; /* Rx internal data pointer */ - __be16 rbptr; /* rb BD Pointer */ - __be16 rcount; /* Rx internal byte count */ - __be32 rtemp; /* Rx temp */ - __be32 tstate; /* Tx internal state */ - __be32 tptr; /* Tx internal data pointer */ - __be16 tbptr; /* Tx BD pointer */ - __be16 tcount; /* Tx byte count */ - __be32 ttemp; /* Tx temp */ - __be32 rcrc; /* temp receive CRC */ - __be32 tcrc; /* temp transmit CRC */ -} __attribute__ ((packed)); - -/* General UCC SLOW Mode Register (GUMRH & GUMRL) */ -#define UCC_SLOW_GUMR_H_SAM_QMC 0x00000000 -#define UCC_SLOW_GUMR_H_SAM_SATM 0x00008000 -#define UCC_SLOW_GUMR_H_REVD 0x00002000 -#define UCC_SLOW_GUMR_H_TRX 0x00001000 -#define UCC_SLOW_GUMR_H_TTX 0x00000800 -#define UCC_SLOW_GUMR_H_CDP 0x00000400 -#define UCC_SLOW_GUMR_H_CTSP 0x00000200 -#define UCC_SLOW_GUMR_H_CDS 0x00000100 -#define UCC_SLOW_GUMR_H_CTSS 0x00000080 -#define UCC_SLOW_GUMR_H_TFL 0x00000040 -#define UCC_SLOW_GUMR_H_RFW 0x00000020 -#define UCC_SLOW_GUMR_H_TXSY 0x00000010 -#define UCC_SLOW_GUMR_H_4SYNC 0x00000004 -#define UCC_SLOW_GUMR_H_8SYNC 0x00000008 -#define UCC_SLOW_GUMR_H_16SYNC 0x0000000c -#define UCC_SLOW_GUMR_H_RTSM 0x00000002 -#define UCC_SLOW_GUMR_H_RSYN 0x00000001 - -#define UCC_SLOW_GUMR_L_TCI 0x10000000 -#define UCC_SLOW_GUMR_L_RINV 0x02000000 -#define UCC_SLOW_GUMR_L_TINV 0x01000000 -#define UCC_SLOW_GUMR_L_TEND 0x00040000 -#define UCC_SLOW_GUMR_L_TDCR_MASK 0x00030000 -#define UCC_SLOW_GUMR_L_TDCR_32 0x00030000 -#define UCC_SLOW_GUMR_L_TDCR_16 0x00020000 -#define UCC_SLOW_GUMR_L_TDCR_8 0x00010000 -#define UCC_SLOW_GUMR_L_TDCR_1 0x00000000 -#define UCC_SLOW_GUMR_L_RDCR_MASK 0x0000c000 -#define UCC_SLOW_GUMR_L_RDCR_32 0x0000c000 -#define UCC_SLOW_GUMR_L_RDCR_16 0x00008000 -#define UCC_SLOW_GUMR_L_RDCR_8 0x00004000 -#define UCC_SLOW_GUMR_L_RDCR_1 0x00000000 -#define UCC_SLOW_GUMR_L_RENC_NRZI 0x00000800 -#define UCC_SLOW_GUMR_L_RENC_NRZ 0x00000000 -#define UCC_SLOW_GUMR_L_TENC_NRZI 0x00000100 -#define UCC_SLOW_GUMR_L_TENC_NRZ 0x00000000 -#define UCC_SLOW_GUMR_L_DIAG_MASK 0x000000c0 -#define UCC_SLOW_GUMR_L_DIAG_LE 0x000000c0 -#define UCC_SLOW_GUMR_L_DIAG_ECHO 0x00000080 -#define UCC_SLOW_GUMR_L_DIAG_LOOP 0x00000040 -#define UCC_SLOW_GUMR_L_DIAG_NORM 0x00000000 -#define UCC_SLOW_GUMR_L_ENR 0x00000020 -#define UCC_SLOW_GUMR_L_ENT 0x00000010 -#define UCC_SLOW_GUMR_L_MODE_MASK 0x0000000F -#define UCC_SLOW_GUMR_L_MODE_BISYNC 0x00000008 -#define UCC_SLOW_GUMR_L_MODE_AHDLC 0x00000006 -#define UCC_SLOW_GUMR_L_MODE_UART 0x00000004 -#define UCC_SLOW_GUMR_L_MODE_QMC 0x00000002 - -/* General UCC FAST Mode Register */ -#define UCC_FAST_GUMR_TCI 0x20000000 -#define UCC_FAST_GUMR_TRX 0x10000000 -#define UCC_FAST_GUMR_TTX 0x08000000 -#define UCC_FAST_GUMR_CDP 0x04000000 -#define UCC_FAST_GUMR_CTSP 0x02000000 -#define UCC_FAST_GUMR_CDS 0x01000000 -#define UCC_FAST_GUMR_CTSS 0x00800000 -#define UCC_FAST_GUMR_TXSY 0x00020000 -#define UCC_FAST_GUMR_RSYN 0x00010000 -#define UCC_FAST_GUMR_RTSM 0x00002000 -#define UCC_FAST_GUMR_REVD 0x00000400 -#define UCC_FAST_GUMR_ENR 0x00000020 -#define UCC_FAST_GUMR_ENT 0x00000010 - -/* UART Slow UCC Event Register (UCCE) */ -#define UCC_UART_UCCE_AB 0x0200 -#define UCC_UART_UCCE_IDLE 0x0100 -#define UCC_UART_UCCE_GRA 0x0080 -#define UCC_UART_UCCE_BRKE 0x0040 -#define UCC_UART_UCCE_BRKS 0x0020 -#define UCC_UART_UCCE_CCR 0x0008 -#define UCC_UART_UCCE_BSY 0x0004 -#define UCC_UART_UCCE_TX 0x0002 -#define UCC_UART_UCCE_RX 0x0001 - -/* HDLC Slow UCC Event Register (UCCE) */ -#define UCC_HDLC_UCCE_GLR 0x1000 -#define UCC_HDLC_UCCE_GLT 0x0800 -#define UCC_HDLC_UCCE_IDLE 0x0100 -#define UCC_HDLC_UCCE_BRKE 0x0040 -#define UCC_HDLC_UCCE_BRKS 0x0020 -#define UCC_HDLC_UCCE_TXE 0x0010 -#define UCC_HDLC_UCCE_RXF 0x0008 -#define UCC_HDLC_UCCE_BSY 0x0004 -#define UCC_HDLC_UCCE_TXB 0x0002 -#define UCC_HDLC_UCCE_RXB 0x0001 - -/* BISYNC Slow UCC Event Register (UCCE) */ -#define UCC_BISYNC_UCCE_GRA 0x0080 -#define UCC_BISYNC_UCCE_TXE 0x0010 -#define UCC_BISYNC_UCCE_RCH 0x0008 -#define UCC_BISYNC_UCCE_BSY 0x0004 -#define UCC_BISYNC_UCCE_TXB 0x0002 -#define UCC_BISYNC_UCCE_RXB 0x0001 - -/* Gigabit Ethernet Fast UCC Event Register (UCCE) */ -#define UCC_GETH_UCCE_MPD 0x80000000 -#define UCC_GETH_UCCE_SCAR 0x40000000 -#define UCC_GETH_UCCE_GRA 0x20000000 -#define UCC_GETH_UCCE_CBPR 0x10000000 -#define UCC_GETH_UCCE_BSY 0x08000000 -#define UCC_GETH_UCCE_RXC 0x04000000 -#define UCC_GETH_UCCE_TXC 0x02000000 -#define UCC_GETH_UCCE_TXE 0x01000000 -#define UCC_GETH_UCCE_TXB7 0x00800000 -#define UCC_GETH_UCCE_TXB6 0x00400000 -#define UCC_GETH_UCCE_TXB5 0x00200000 -#define UCC_GETH_UCCE_TXB4 0x00100000 -#define UCC_GETH_UCCE_TXB3 0x00080000 -#define UCC_GETH_UCCE_TXB2 0x00040000 -#define UCC_GETH_UCCE_TXB1 0x00020000 -#define UCC_GETH_UCCE_TXB0 0x00010000 -#define UCC_GETH_UCCE_RXB7 0x00008000 -#define UCC_GETH_UCCE_RXB6 0x00004000 -#define UCC_GETH_UCCE_RXB5 0x00002000 -#define UCC_GETH_UCCE_RXB4 0x00001000 -#define UCC_GETH_UCCE_RXB3 0x00000800 -#define UCC_GETH_UCCE_RXB2 0x00000400 -#define UCC_GETH_UCCE_RXB1 0x00000200 -#define UCC_GETH_UCCE_RXB0 0x00000100 -#define UCC_GETH_UCCE_RXF7 0x00000080 -#define UCC_GETH_UCCE_RXF6 0x00000040 -#define UCC_GETH_UCCE_RXF5 0x00000020 -#define UCC_GETH_UCCE_RXF4 0x00000010 -#define UCC_GETH_UCCE_RXF3 0x00000008 -#define UCC_GETH_UCCE_RXF2 0x00000004 -#define UCC_GETH_UCCE_RXF1 0x00000002 -#define UCC_GETH_UCCE_RXF0 0x00000001 - -/* UCC Protocol Specific Mode Register (UPSMR), when used for UART */ -#define UCC_UART_UPSMR_FLC 0x8000 -#define UCC_UART_UPSMR_SL 0x4000 -#define UCC_UART_UPSMR_CL_MASK 0x3000 -#define UCC_UART_UPSMR_CL_8 0x3000 -#define UCC_UART_UPSMR_CL_7 0x2000 -#define UCC_UART_UPSMR_CL_6 0x1000 -#define UCC_UART_UPSMR_CL_5 0x0000 -#define UCC_UART_UPSMR_UM_MASK 0x0c00 -#define UCC_UART_UPSMR_UM_NORMAL 0x0000 -#define UCC_UART_UPSMR_UM_MAN_MULTI 0x0400 -#define UCC_UART_UPSMR_UM_AUTO_MULTI 0x0c00 -#define UCC_UART_UPSMR_FRZ 0x0200 -#define UCC_UART_UPSMR_RZS 0x0100 -#define UCC_UART_UPSMR_SYN 0x0080 -#define UCC_UART_UPSMR_DRT 0x0040 -#define UCC_UART_UPSMR_PEN 0x0010 -#define UCC_UART_UPSMR_RPM_MASK 0x000c -#define UCC_UART_UPSMR_RPM_ODD 0x0000 -#define UCC_UART_UPSMR_RPM_LOW 0x0004 -#define UCC_UART_UPSMR_RPM_EVEN 0x0008 -#define UCC_UART_UPSMR_RPM_HIGH 0x000C -#define UCC_UART_UPSMR_TPM_MASK 0x0003 -#define UCC_UART_UPSMR_TPM_ODD 0x0000 -#define UCC_UART_UPSMR_TPM_LOW 0x0001 -#define UCC_UART_UPSMR_TPM_EVEN 0x0002 -#define UCC_UART_UPSMR_TPM_HIGH 0x0003 - -/* UCC Protocol Specific Mode Register (UPSMR), when used for Ethernet */ -#define UCC_GETH_UPSMR_FTFE 0x80000000 -#define UCC_GETH_UPSMR_PTPE 0x40000000 -#define UCC_GETH_UPSMR_ECM 0x04000000 -#define UCC_GETH_UPSMR_HSE 0x02000000 -#define UCC_GETH_UPSMR_PRO 0x00400000 -#define UCC_GETH_UPSMR_CAP 0x00200000 -#define UCC_GETH_UPSMR_RSH 0x00100000 -#define UCC_GETH_UPSMR_RPM 0x00080000 -#define UCC_GETH_UPSMR_R10M 0x00040000 -#define UCC_GETH_UPSMR_RLPB 0x00020000 -#define UCC_GETH_UPSMR_TBIM 0x00010000 -#define UCC_GETH_UPSMR_RES1 0x00002000 -#define UCC_GETH_UPSMR_RMM 0x00001000 -#define UCC_GETH_UPSMR_CAM 0x00000400 -#define UCC_GETH_UPSMR_BRO 0x00000200 -#define UCC_GETH_UPSMR_SMM 0x00000080 -#define UCC_GETH_UPSMR_SGMM 0x00000020 - -/* UCC Transmit On Demand Register (UTODR) */ -#define UCC_SLOW_TOD 0x8000 -#define UCC_FAST_TOD 0x8000 - -/* UCC Bus Mode Register masks */ -/* Not to be confused with the Bundle Mode Register */ -#define UCC_BMR_GBL 0x20 -#define UCC_BMR_BO_BE 0x10 -#define UCC_BMR_CETM 0x04 -#define UCC_BMR_DTB 0x02 -#define UCC_BMR_BDB 0x01 - -/* Function code masks */ -#define FC_GBL 0x20 -#define FC_DTB_LCL 0x02 -#define UCC_FAST_FUNCTION_CODE_GBL 0x20 -#define UCC_FAST_FUNCTION_CODE_DTB_LCL 0x02 -#define UCC_FAST_FUNCTION_CODE_BDB_LCL 0x01 - -#endif /* __KERNEL__ */ -#endif /* _ASM_POWERPC_QE_H */ diff --git a/arch/powerpc/include/asm/qe_ic.h b/arch/powerpc/include/asm/qe_ic.h deleted file mode 100644 index 25784cc959a0..000000000000 --- a/arch/powerpc/include/asm/qe_ic.h +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright (C) 2006 Freescale Semiconductor, Inc. All rights reserved. - * - * Authors: Shlomi Gridish <gridish@freescale.com> - * Li Yang <leoli@freescale.com> - * - * Description: - * QE IC external definitions and structure. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - */ -#ifndef _ASM_POWERPC_QE_IC_H -#define _ASM_POWERPC_QE_IC_H - -#include <linux/irq.h> - -struct device_node; -struct qe_ic; - -#define NUM_OF_QE_IC_GROUPS 6 - -/* Flags when we init the QE IC */ -#define QE_IC_SPREADMODE_GRP_W 0x00000001 -#define QE_IC_SPREADMODE_GRP_X 0x00000002 -#define QE_IC_SPREADMODE_GRP_Y 0x00000004 -#define QE_IC_SPREADMODE_GRP_Z 0x00000008 -#define QE_IC_SPREADMODE_GRP_RISCA 0x00000010 -#define QE_IC_SPREADMODE_GRP_RISCB 0x00000020 - -#define QE_IC_LOW_SIGNAL 0x00000100 -#define QE_IC_HIGH_SIGNAL 0x00000200 - -#define QE_IC_GRP_W_PRI0_DEST_SIGNAL_HIGH 0x00001000 -#define QE_IC_GRP_W_PRI1_DEST_SIGNAL_HIGH 0x00002000 -#define QE_IC_GRP_X_PRI0_DEST_SIGNAL_HIGH 0x00004000 -#define QE_IC_GRP_X_PRI1_DEST_SIGNAL_HIGH 0x00008000 -#define QE_IC_GRP_Y_PRI0_DEST_SIGNAL_HIGH 0x00010000 -#define QE_IC_GRP_Y_PRI1_DEST_SIGNAL_HIGH 0x00020000 -#define QE_IC_GRP_Z_PRI0_DEST_SIGNAL_HIGH 0x00040000 -#define QE_IC_GRP_Z_PRI1_DEST_SIGNAL_HIGH 0x00080000 -#define QE_IC_GRP_RISCA_PRI0_DEST_SIGNAL_HIGH 0x00100000 -#define QE_IC_GRP_RISCA_PRI1_DEST_SIGNAL_HIGH 0x00200000 -#define QE_IC_GRP_RISCB_PRI0_DEST_SIGNAL_HIGH 0x00400000 -#define QE_IC_GRP_RISCB_PRI1_DEST_SIGNAL_HIGH 0x00800000 -#define QE_IC_GRP_W_DEST_SIGNAL_SHIFT (12) - -/* QE interrupt sources groups */ -enum qe_ic_grp_id { - QE_IC_GRP_W = 0, /* QE interrupt controller group W */ - QE_IC_GRP_X, /* QE interrupt controller group X */ - QE_IC_GRP_Y, /* QE interrupt controller group Y */ - QE_IC_GRP_Z, /* QE interrupt controller group Z */ - QE_IC_GRP_RISCA, /* QE interrupt controller RISC group A */ - QE_IC_GRP_RISCB /* QE interrupt controller RISC group B */ -}; - -#ifdef CONFIG_QUICC_ENGINE -void qe_ic_init(struct device_node *node, unsigned int flags, - void (*low_handler)(unsigned int irq, struct irq_desc *desc), - void (*high_handler)(unsigned int irq, struct irq_desc *desc)); -unsigned int qe_ic_get_low_irq(struct qe_ic *qe_ic); -unsigned int qe_ic_get_high_irq(struct qe_ic *qe_ic); -#else -static inline void qe_ic_init(struct device_node *node, unsigned int flags, - void (*low_handler)(unsigned int irq, struct irq_desc *desc), - void (*high_handler)(unsigned int irq, struct irq_desc *desc)) -{} -static inline unsigned int qe_ic_get_low_irq(struct qe_ic *qe_ic) -{ return 0; } -static inline unsigned int qe_ic_get_high_irq(struct qe_ic *qe_ic) -{ return 0; } -#endif /* CONFIG_QUICC_ENGINE */ - -void qe_ic_set_highest_priority(unsigned int virq, int high); -int qe_ic_set_priority(unsigned int virq, unsigned int priority); -int qe_ic_set_high_priority(unsigned int virq, unsigned int priority, int high); - -static inline void qe_ic_cascade_low_ipic(unsigned int irq, - struct irq_desc *desc) -{ - struct qe_ic *qe_ic = irq_desc_get_handler_data(desc); - unsigned int cascade_irq = qe_ic_get_low_irq(qe_ic); - - if (cascade_irq != NO_IRQ) - generic_handle_irq(cascade_irq); -} - -static inline void qe_ic_cascade_high_ipic(unsigned int irq, - struct irq_desc *desc) -{ - struct qe_ic *qe_ic = irq_desc_get_handler_data(desc); - unsigned int cascade_irq = qe_ic_get_high_irq(qe_ic); - - if (cascade_irq != NO_IRQ) - generic_handle_irq(cascade_irq); -} - -static inline void qe_ic_cascade_low_mpic(unsigned int irq, - struct irq_desc *desc) -{ - struct qe_ic *qe_ic = irq_desc_get_handler_data(desc); - unsigned int cascade_irq = qe_ic_get_low_irq(qe_ic); - struct irq_chip *chip = irq_desc_get_chip(desc); - - if (cascade_irq != NO_IRQ) - generic_handle_irq(cascade_irq); - - chip->irq_eoi(&desc->irq_data); -} - -static inline void qe_ic_cascade_high_mpic(unsigned int irq, - struct irq_desc *desc) -{ - struct qe_ic *qe_ic = irq_desc_get_handler_data(desc); - unsigned int cascade_irq = qe_ic_get_high_irq(qe_ic); - struct irq_chip *chip = irq_desc_get_chip(desc); - - if (cascade_irq != NO_IRQ) - generic_handle_irq(cascade_irq); - - chip->irq_eoi(&desc->irq_data); -} - -static inline void qe_ic_cascade_muxed_mpic(unsigned int irq, - struct irq_desc *desc) -{ - struct qe_ic *qe_ic = irq_desc_get_handler_data(desc); - unsigned int cascade_irq; - struct irq_chip *chip = irq_desc_get_chip(desc); - - cascade_irq = qe_ic_get_high_irq(qe_ic); - if (cascade_irq == NO_IRQ) - cascade_irq = qe_ic_get_low_irq(qe_ic); - - if (cascade_irq != NO_IRQ) - generic_handle_irq(cascade_irq); - - chip->irq_eoi(&desc->irq_data); -} - -#endif /* _ASM_POWERPC_QE_IC_H */ diff --git a/arch/powerpc/include/asm/qspinlock.h b/arch/powerpc/include/asm/qspinlock.h new file mode 100644 index 000000000000..28a53fb69b38 --- /dev/null +++ b/arch/powerpc/include/asm/qspinlock.h @@ -0,0 +1,174 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_QSPINLOCK_H +#define _ASM_POWERPC_QSPINLOCK_H + +#include <linux/compiler.h> +#include <asm/qspinlock_types.h> +#include <asm/paravirt.h> + +#ifdef CONFIG_PPC64 +/* + * Use the EH=1 hint for accesses that result in the lock being acquired. + * The hardware is supposed to optimise this pattern by holding the lock + * cacheline longer, and releasing when a store to the same memory (the + * unlock) is performed. + */ +#define _Q_SPIN_EH_HINT 1 +#else +#define _Q_SPIN_EH_HINT 0 +#endif + +/* + * The trylock itself may steal. This makes trylocks slightly stronger, and + * makes locks slightly more efficient when stealing. + * + * This is compile-time, so if true then there may always be stealers, so the + * nosteal paths become unused. + */ +#define _Q_SPIN_TRY_LOCK_STEAL 1 + +/* + * Put a speculation barrier after testing the lock/node and finding it + * busy. Try to prevent pointless speculation in slow paths. + * + * Slows down the lockstorm microbenchmark with no stealing, where locking + * is purely FIFO through the queue. May have more benefit in real workload + * where speculating into the wrong place could have a greater cost. + */ +#define _Q_SPIN_SPEC_BARRIER 0 + +#ifdef CONFIG_PPC64 +/* + * Execute a miso instruction after passing the MCS lock ownership to the + * queue head. Miso is intended to make stores visible to other CPUs sooner. + * + * This seems to make the lockstorm microbenchmark nospin test go slightly + * faster on POWER10, but disable for now. + */ +#define _Q_SPIN_MISO 0 +#else +#define _Q_SPIN_MISO 0 +#endif + +#ifdef CONFIG_PPC64 +/* + * This executes miso after an unlock of the lock word, having ownership + * pass to the next CPU sooner. This will slow the uncontended path to some + * degree. Not evidence it helps yet. + */ +#define _Q_SPIN_MISO_UNLOCK 0 +#else +#define _Q_SPIN_MISO_UNLOCK 0 +#endif + +/* + * Seems to slow down lockstorm microbenchmark, suspect queue node just + * has to become shared again right afterwards when its waiter spins on + * the lock field. + */ +#define _Q_SPIN_PREFETCH_NEXT 0 + +static __always_inline int queued_spin_is_locked(struct qspinlock *lock) +{ + return READ_ONCE(lock->val); +} + +static __always_inline int queued_spin_value_unlocked(struct qspinlock lock) +{ + return !lock.val; +} + +static __always_inline int queued_spin_is_contended(struct qspinlock *lock) +{ + return !!(READ_ONCE(lock->val) & _Q_TAIL_CPU_MASK); +} + +static __always_inline u32 queued_spin_encode_locked_val(void) +{ + /* XXX: make this use lock value in paca like simple spinlocks? */ + return _Q_LOCKED_VAL | (smp_processor_id() << _Q_OWNER_CPU_OFFSET); +} + +static __always_inline int __queued_spin_trylock_nosteal(struct qspinlock *lock) +{ + u32 new = queued_spin_encode_locked_val(); + u32 prev; + + /* Trylock succeeds only when unlocked and no queued nodes */ + asm volatile( +"1: lwarx %0,0,%1,%3 # __queued_spin_trylock_nosteal \n" +" cmpwi 0,%0,0 \n" +" bne- 2f \n" +" stwcx. %2,0,%1 \n" +" bne- 1b \n" +"\t" PPC_ACQUIRE_BARRIER " \n" +"2: \n" + : "=&r" (prev) + : "r" (&lock->val), "r" (new), + "i" (_Q_SPIN_EH_HINT) + : "cr0", "memory"); + + return likely(prev == 0); +} + +static __always_inline int __queued_spin_trylock_steal(struct qspinlock *lock) +{ + u32 new = queued_spin_encode_locked_val(); + u32 prev, tmp; + + /* Trylock may get ahead of queued nodes if it finds unlocked */ + asm volatile( +"1: lwarx %0,0,%2,%5 # __queued_spin_trylock_steal \n" +" andc. %1,%0,%4 \n" +" bne- 2f \n" +" and %1,%0,%4 \n" +" or %1,%1,%3 \n" +" stwcx. %1,0,%2 \n" +" bne- 1b \n" +"\t" PPC_ACQUIRE_BARRIER " \n" +"2: \n" + : "=&r" (prev), "=&r" (tmp) + : "r" (&lock->val), "r" (new), "r" (_Q_TAIL_CPU_MASK), + "i" (_Q_SPIN_EH_HINT) + : "cr0", "memory"); + + return likely(!(prev & ~_Q_TAIL_CPU_MASK)); +} + +static __always_inline int queued_spin_trylock(struct qspinlock *lock) +{ + if (!_Q_SPIN_TRY_LOCK_STEAL) + return __queued_spin_trylock_nosteal(lock); + else + return __queued_spin_trylock_steal(lock); +} + +void queued_spin_lock_slowpath(struct qspinlock *lock); + +static __always_inline void queued_spin_lock(struct qspinlock *lock) +{ + if (!queued_spin_trylock(lock)) + queued_spin_lock_slowpath(lock); +} + +static inline void queued_spin_unlock(struct qspinlock *lock) +{ + smp_store_release(&lock->locked, 0); + if (_Q_SPIN_MISO_UNLOCK) + asm volatile("miso" ::: "memory"); +} + +#define arch_spin_is_locked(l) queued_spin_is_locked(l) +#define arch_spin_is_contended(l) queued_spin_is_contended(l) +#define arch_spin_value_unlocked(l) queued_spin_value_unlocked(l) +#define arch_spin_lock(l) queued_spin_lock(l) +#define arch_spin_trylock(l) queued_spin_trylock(l) +#define arch_spin_unlock(l) queued_spin_unlock(l) + +#ifdef CONFIG_PARAVIRT_SPINLOCKS +void pv_spinlocks_init(void); +#else +static inline void pv_spinlocks_init(void) { } +#endif + +#endif /* _ASM_POWERPC_QSPINLOCK_H */ diff --git a/arch/powerpc/include/asm/qspinlock_types.h b/arch/powerpc/include/asm/qspinlock_types.h new file mode 100644 index 000000000000..4766a7aa03cb --- /dev/null +++ b/arch/powerpc/include/asm/qspinlock_types.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_POWERPC_QSPINLOCK_TYPES_H +#define _ASM_POWERPC_QSPINLOCK_TYPES_H + +#include <linux/types.h> +#include <asm/byteorder.h> + +typedef struct qspinlock { + union { + u32 val; + +#ifdef __LITTLE_ENDIAN + struct { + u16 locked; + u8 reserved[2]; + }; +#else + struct { + u8 reserved[2]; + u16 locked; + }; +#endif + }; +} arch_spinlock_t; + +#define __ARCH_SPIN_LOCK_UNLOCKED { { .val = 0 } } + +/* + * Bitfields in the lock word: + * + * 0: locked bit + * 1-14: lock holder cpu + * 15: lock owner or queuer vcpus observed to be preempted bit + * 16: must queue bit + * 17-31: tail cpu (+1) + */ +#define _Q_SET_MASK(type) (((1U << _Q_ ## type ## _BITS) - 1)\ + << _Q_ ## type ## _OFFSET) +/* 0x00000001 */ +#define _Q_LOCKED_OFFSET 0 +#define _Q_LOCKED_BITS 1 +#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET) + +/* 0x00007ffe */ +#define _Q_OWNER_CPU_OFFSET 1 +#define _Q_OWNER_CPU_BITS 14 +#define _Q_OWNER_CPU_MASK _Q_SET_MASK(OWNER_CPU) + +#if CONFIG_NR_CPUS > (1U << _Q_OWNER_CPU_BITS) +#error "qspinlock does not support such large CONFIG_NR_CPUS" +#endif + +/* 0x00008000 */ +#define _Q_SLEEPY_OFFSET 15 +#define _Q_SLEEPY_BITS 1 +#define _Q_SLEEPY_VAL (1U << _Q_SLEEPY_OFFSET) + +/* 0x00010000 */ +#define _Q_MUST_Q_OFFSET 16 +#define _Q_MUST_Q_BITS 1 +#define _Q_MUST_Q_VAL (1U << _Q_MUST_Q_OFFSET) + +/* 0xfffe0000 */ +#define _Q_TAIL_CPU_OFFSET 17 +#define _Q_TAIL_CPU_BITS 15 +#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU) + +#if CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS) +#error "qspinlock does not support such large CONFIG_NR_CPUS" +#endif + +#endif /* _ASM_POWERPC_QSPINLOCK_TYPES_H */ diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h index 5d7d9c2a5473..3fe186635432 100644 --- a/arch/powerpc/include/asm/reg.h +++ b/arch/powerpc/include/asm/reg.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Contains the definition of registers common to all PowerPC variants. * If a register definition has been changed in a different PowerPC @@ -11,23 +12,23 @@ #ifdef __KERNEL__ #include <linux/stringify.h> +#include <linux/const.h> #include <asm/cputable.h> +#include <asm/asm-const.h> +#include <asm/feature-fixups.h> /* Pickup Book E specific registers. */ -#if defined(CONFIG_BOOKE) || defined(CONFIG_40x) +#ifdef CONFIG_BOOKE #include <asm/reg_booke.h> -#endif /* CONFIG_BOOKE || CONFIG_40x */ +#endif #ifdef CONFIG_FSL_EMB_PERFMON #include <asm/reg_fsl_emb.h> #endif -#ifdef CONFIG_8xx #include <asm/reg_8xx.h> -#endif /* CONFIG_8xx */ #define MSR_SF_LG 63 /* Enable 64 bit mode */ -#define MSR_ISF_LG 61 /* Interrupt 64b mode valid on 630 */ #define MSR_HV_LG 60 /* Hypervisor state */ #define MSR_TS_T_LG 34 /* Trans Mem state: Transactional */ #define MSR_TS_S_LG 33 /* Trans Mem state: Suspended */ @@ -35,6 +36,7 @@ #define MSR_TM_LG 32 /* Trans Mem Available */ #define MSR_VEC_LG 25 /* Enable AltiVec */ #define MSR_VSX_LG 23 /* Enable VSX */ +#define MSR_S_LG 22 /* Secure state */ #define MSR_POW_LG 18 /* Enable Power Management */ #define MSR_WE_LG 18 /* Wait State Enable */ #define MSR_TGPR_LG 17 /* TLB Update registers in use */ @@ -58,7 +60,7 @@ #define MSR_RI_LG 1 /* Recoverable Exception */ #define MSR_LE_LG 0 /* Little Endian */ -#ifdef __ASSEMBLY__ +#ifdef __ASSEMBLER__ #define __MASK(X) (1<<(X)) #else #define __MASK(X) (1UL<<(X)) @@ -66,13 +68,21 @@ #ifdef CONFIG_PPC64 #define MSR_SF __MASK(MSR_SF_LG) /* Enable 64 bit mode */ -#define MSR_ISF __MASK(MSR_ISF_LG) /* Interrupt 64b mode valid on 630 */ #define MSR_HV __MASK(MSR_HV_LG) /* Hypervisor state */ +#define MSR_S __MASK(MSR_S_LG) /* Secure state */ #else /* so tests for these bits fail on 32-bit */ #define MSR_SF 0 -#define MSR_ISF 0 #define MSR_HV 0 +#define MSR_S 0 +#endif + +/* + * To be used in shared book E/book S, this avoids needing to worry about + * book S/book E in shared code + */ +#ifndef MSR_SPE +#define MSR_SPE 0 #endif #define MSR_VEC __MASK(MSR_VEC_LG) /* Enable AltiVec */ @@ -107,19 +117,32 @@ #define MSR_TS_S __MASK(MSR_TS_S_LG) /* Transaction Suspended */ #define MSR_TS_T __MASK(MSR_TS_T_LG) /* Transaction Transactional */ #define MSR_TS_MASK (MSR_TS_T | MSR_TS_S) /* Transaction State bits */ -#define MSR_TM_ACTIVE(x) (((x) & MSR_TS_MASK) != 0) /* Transaction active? */ +#define MSR_TM_RESV(x) (((x) & MSR_TS_MASK) == MSR_TS_MASK) /* Reserved */ #define MSR_TM_TRANSACTIONAL(x) (((x) & MSR_TS_MASK) == MSR_TS_T) #define MSR_TM_SUSPENDED(x) (((x) & MSR_TS_MASK) == MSR_TS_S) +#ifdef CONFIG_PPC_TRANSACTIONAL_MEM +#define MSR_TM_ACTIVE(x) (((x) & MSR_TS_MASK) != 0) /* Transaction active? */ +#else +#define MSR_TM_ACTIVE(x) ((void)(x), 0) +#endif + #if defined(CONFIG_PPC_BOOK3S_64) #define MSR_64BIT MSR_SF /* Server variant */ -#define MSR_ MSR_ME | MSR_RI | MSR_IR | MSR_DR | MSR_ISF |MSR_HV -#define MSR_KERNEL MSR_ | MSR_64BIT -#define MSR_USER32 MSR_ | MSR_PR | MSR_EE -#define MSR_USER64 MSR_USER32 | MSR_64BIT -#elif defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_8xx) +#define __MSR (MSR_ME | MSR_RI | MSR_IR | MSR_DR | MSR_HV) +#ifdef __BIG_ENDIAN__ +#define MSR_ __MSR +#define MSR_IDLE (MSR_ME | MSR_SF | MSR_HV) +#else +#define MSR_ (__MSR | MSR_LE) +#define MSR_IDLE (MSR_ME | MSR_SF | MSR_HV | MSR_LE) +#endif +#define MSR_KERNEL (MSR_ | MSR_64BIT) +#define MSR_USER32 (MSR_ | MSR_PR | MSR_EE) +#define MSR_USER64 (MSR_USER32 | MSR_64BIT) +#elif defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_PPC_8xx) /* Default MSR for kernel mode. */ #define MSR_KERNEL (MSR_ME|MSR_RI|MSR_IR|MSR_DR) #define MSR_USER (MSR_KERNEL|MSR_PR|MSR_EE) @@ -129,6 +152,26 @@ #define MSR_64BIT 0 #endif +/* Condition Register related */ +#define CR0_SHIFT 28 +#define CR0_MASK 0xF +#define CR0_TBEGIN_FAILURE (0x2 << 28) /* 0b0010 */ + + +/* Power Management - Processor Stop Status and Control Register Fields */ +#define PSSCR_RL_MASK 0x0000000F /* Requested Level */ +#define PSSCR_MTL_MASK 0x000000F0 /* Maximum Transition Level */ +#define PSSCR_TR_MASK 0x00000300 /* Transition State */ +#define PSSCR_PSLL_MASK 0x000F0000 /* Power-Saving Level Limit */ +#define PSSCR_EC 0x00100000 /* Exit Criterion */ +#define PSSCR_ESL 0x00200000 /* Enable State Loss */ +#define PSSCR_SD 0x00400000 /* Status Disable */ +#define PSSCR_PLS 0xf000000000000000 /* Power-saving Level Status */ +#define PSSCR_PLS_SHIFT 60 +#define PSSCR_GUEST_VIS 0xf0000000000003ffUL /* Guest-visible PSSCR fields */ +#define PSSCR_FAKE_SUSPEND 0x00000400 /* Fake-suspend bit (P9 DD2.2) */ +#define PSSCR_FAKE_SUSPEND_LG 10 /* Fake-suspend bit position */ + /* Floating Point Status and Control Register (FPSCR) Fields */ #define FPSCR_FX 0x80000000 /* FPU exception summary */ #define FPSCR_FEX 0x40000000 /* FPU enabled exception summary */ @@ -190,14 +233,10 @@ /* Special Purpose Registers (SPRNs)*/ -#ifdef CONFIG_40x -#define SPRN_PID 0x3B1 /* Process ID */ -#else #define SPRN_PID 0x030 /* Process ID */ #ifdef CONFIG_BOOKE #define SPRN_PID0 SPRN_PID/* Process ID Register 0 */ #endif -#endif #define SPRN_CTR 0x009 /* Count Register */ #define SPRN_DSCR 0x11 @@ -209,7 +248,28 @@ #define SPRN_TFIAR 0x81 /* Transaction Failure Inst Addr */ #define SPRN_TEXASR 0x82 /* Transaction EXception & Summary */ #define SPRN_TEXASRU 0x83 /* '' '' '' Upper 32 */ + +#define TEXASR_FC_LG (63 - 7) /* Failure Code */ +#define TEXASR_AB_LG (63 - 31) /* Abort */ +#define TEXASR_SU_LG (63 - 32) /* Suspend */ +#define TEXASR_HV_LG (63 - 34) /* Hypervisor state*/ +#define TEXASR_PR_LG (63 - 35) /* Privilege level */ +#define TEXASR_FS_LG (63 - 36) /* failure summary */ +#define TEXASR_EX_LG (63 - 37) /* TFIAR exact bit */ +#define TEXASR_ROT_LG (63 - 38) /* ROT bit */ + +#define TEXASR_ABORT __MASK(TEXASR_AB_LG) /* terminated by tabort or treclaim */ +#define TEXASR_SUSP __MASK(TEXASR_SU_LG) /* tx failed in suspended state */ +#define TEXASR_HV __MASK(TEXASR_HV_LG) /* MSR[HV] when failure occurred */ +#define TEXASR_PR __MASK(TEXASR_PR_LG) /* MSR[PR] when failure occurred */ +#define TEXASR_FS __MASK(TEXASR_FS_LG) /* TEXASR Failure Summary */ +#define TEXASR_EXACT __MASK(TEXASR_EX_LG) /* TFIAR value is exact */ +#define TEXASR_ROT __MASK(TEXASR_ROT_LG) +#define TEXASR_FC (ASM_CONST(0xFF) << TEXASR_FC_LG) + #define SPRN_TFHAR 0x80 /* Transaction Failure Handler Addr */ + +#define SPRN_TIDR 144 /* Thread ID register */ #define SPRN_CTRLF 0x088 #define SPRN_CTRLT 0x098 #define CTRL_CT 0xc0000000 /* current thread */ @@ -217,32 +277,98 @@ #define CTRL_CT1 0x40000000 /* thread 1 */ #define CTRL_TE 0x00c00000 /* thread enable */ #define CTRL_RUNLATCH 0x1 -#define SPRN_DAWR 0xB4 -#define SPRN_DAWRX 0xBC -#define DAWRX_USER (1UL << 0) -#define DAWRX_KERNEL (1UL << 1) -#define DAWRX_HYP (1UL << 2) +#define SPRN_DAWR0 0xB4 +#define SPRN_DAWR1 0xB5 +#define SPRN_RPR 0xBA /* Relative Priority Register */ +#define SPRN_CIABR 0xBB +#define CIABR_PRIV 0x3 +#define CIABR_PRIV_USER 1 +#define CIABR_PRIV_SUPER 2 +#define CIABR_PRIV_HYPER 3 +#define SPRN_DAWRX0 0xBC +#define SPRN_DAWRX1 0xBD +#define DAWRX_USER __MASK(0) +#define DAWRX_KERNEL __MASK(1) +#define DAWRX_HYP __MASK(2) +#define DAWRX_WTI __MASK(3) +#define DAWRX_WT __MASK(4) +#define DAWRX_DR __MASK(5) +#define DAWRX_DW __MASK(6) #define SPRN_DABR 0x3F5 /* Data Address Breakpoint Register */ #define SPRN_DABR2 0x13D /* e300 */ #define SPRN_DABRX 0x3F7 /* Data Address Breakpoint Register Extension */ -#define DABRX_USER (1UL << 0) -#define DABRX_KERNEL (1UL << 1) -#define DABRX_HYP (1UL << 2) -#define DABRX_BTI (1UL << 3) +#define DABRX_USER __MASK(0) +#define DABRX_KERNEL __MASK(1) +#define DABRX_HYP __MASK(2) +#define DABRX_BTI __MASK(3) #define DABRX_ALL (DABRX_BTI | DABRX_HYP | DABRX_KERNEL | DABRX_USER) #define SPRN_DAR 0x013 /* Data Address Register */ #define SPRN_DBCR 0x136 /* e300 Data Breakpoint Control Reg */ #define SPRN_DSISR 0x012 /* Data Storage Interrupt Status Register */ -#define DSISR_NOHPTE 0x40000000 /* no translation found */ -#define DSISR_PROTFAULT 0x08000000 /* protection fault */ -#define DSISR_ISSTORE 0x02000000 /* access was a store */ -#define DSISR_DABRMATCH 0x00400000 /* hit data breakpoint */ -#define DSISR_NOSEGMENT 0x00200000 /* STAB/SLB miss */ -#define DSISR_KEYFAULT 0x00200000 /* Key fault */ +#define DSISR_BAD_DIRECT_ST 0x80000000 /* Obsolete: Direct store error */ +#define DSISR_NOHPTE 0x40000000 /* no translation found */ +#define DSISR_ATTR_CONFLICT 0x20000000 /* P9: Process vs. Partition attr */ +#define DSISR_NOEXEC_OR_G 0x10000000 /* Alias of SRR1 bit, see below */ +#define DSISR_PROTFAULT 0x08000000 /* protection fault */ +#define DSISR_BADACCESS 0x04000000 /* bad access to CI or G */ +#define DSISR_ISSTORE 0x02000000 /* access was a store */ +#define DSISR_DABRMATCH 0x00400000 /* hit data breakpoint */ +#define DSISR_NOSEGMENT 0x00200000 /* STAB miss (unsupported) */ +#define DSISR_KEYFAULT 0x00200000 /* Storage Key fault */ +#define DSISR_BAD_EXT_CTRL 0x00100000 /* Obsolete: External ctrl error */ +#define DSISR_UNSUPP_MMU 0x00080000 /* P9: Unsupported MMU config */ +#define DSISR_SET_RC 0x00040000 /* P9: Failed setting of R/C bits */ +#define DSISR_PRTABLE_FAULT 0x00020000 /* P9: Fault on process table */ +#define DSISR_ICSWX_NO_CT 0x00004000 /* P7: icswx unavailable cp type */ +#define DSISR_BAD_COPYPASTE 0x00000008 /* P9: Copy/Paste on wrong memtype */ +#define DSISR_BAD_AMO 0x00000004 /* P9: Incorrect AMO opcode */ +#define DSISR_BAD_CI_LDST 0x00000002 /* P8: Bad HV CI load/store */ + +/* + * DSISR_NOEXEC_OR_G doesn't actually exist. This bit is always + * 0 on DSIs. However, on ISIs, the corresponding bit in SRR1 + * indicates an attempt at executing from a no-execute PTE + * or segment or from a guarded page. + * + * We add a definition here for completeness as we alias + * DSISR and SRR1 in do_page_fault. + */ + +/* + * DSISR bits that are treated as a fault. Any bit set + * here will skip hash_page, and cause do_page_fault to + * trigger a SIGBUS or SIGSEGV: + */ +#define DSISR_BAD_FAULT_32S (DSISR_BAD_DIRECT_ST | \ + DSISR_BADACCESS | \ + DSISR_BAD_EXT_CTRL) +#define DSISR_BAD_FAULT_64S (DSISR_BAD_FAULT_32S | \ + DSISR_ATTR_CONFLICT | \ + DSISR_UNSUPP_MMU | \ + DSISR_PRTABLE_FAULT | \ + DSISR_ICSWX_NO_CT | \ + DSISR_BAD_COPYPASTE | \ + DSISR_BAD_AMO | \ + DSISR_BAD_CI_LDST) +/* + * These bits are equivalent in SRR1 and DSISR for 0x400 + * instruction access interrupts on Book3S + */ +#define DSISR_SRR1_MATCH_32S (DSISR_NOHPTE | \ + DSISR_NOEXEC_OR_G | \ + DSISR_PROTFAULT) +#define DSISR_SRR1_MATCH_64S (DSISR_SRR1_MATCH_32S | \ + DSISR_KEYFAULT | \ + DSISR_UNSUPP_MMU | \ + DSISR_SET_RC | \ + DSISR_PRTABLE_FAULT) + #define SPRN_TBRL 0x10C /* Time Base Read Lower Register (user, R/O) */ #define SPRN_TBRU 0x10D /* Time Base Read Upper Register (user, R/O) */ +#define SPRN_CIR 0x11B /* Chip Information Register (hyper, R/0) */ #define SPRN_TBWL 0x11C /* Time Base Lower Register (super, R/W) */ #define SPRN_TBWU 0x11D /* Time Base Upper Register (super, R/W) */ +#define SPRN_TBU40 0x11E /* Timebase upper 40 bits (hyper, R/W) */ #define SPRN_SPURR 0x134 /* Scaled PURR */ #define SPRN_HSPRG0 0x130 /* Hypervisor Scratch 0 */ #define SPRN_HSPRG1 0x131 /* Hypervisor Scratch 1 */ @@ -252,56 +378,126 @@ #define SPRN_HIOR 0x137 /* 970 Hypervisor interrupt offset */ #define SPRN_RMOR 0x138 /* Real mode offset register */ #define SPRN_HRMOR 0x139 /* Real mode offset register */ -#define SPRN_HSRR0 0x13A /* Hypervisor Save/Restore 0 */ -#define SPRN_HSRR1 0x13B /* Hypervisor Save/Restore 1 */ +#define SPRN_HDEXCR_RO 0x1C7 /* Hypervisor DEXCR (non-privileged, readonly) */ +#define SPRN_HASHKEYR 0x1D4 /* Non-privileged hashst/hashchk key register */ +#define SPRN_HDEXCR 0x1D7 /* Hypervisor dynamic execution control register */ +#define SPRN_DEXCR_RO 0x32C /* DEXCR (non-privileged, readonly) */ +#define SPRN_ASDR 0x330 /* Access segment descriptor register */ +#define SPRN_DEXCR 0x33C /* Dynamic execution control register */ +#define DEXCR_PR_SBHE 0x80000000UL /* 0: Speculative Branch Hint Enable */ +#define DEXCR_PR_IBRTPD 0x10000000UL /* 3: Indirect Branch Recurrent Target Prediction Disable */ +#define DEXCR_PR_SRAPD 0x08000000UL /* 4: Subroutine Return Address Prediction Disable */ +#define DEXCR_PR_NPHIE 0x04000000UL /* 5: Non-Privileged Hash Instruction Enable */ +#define DEXCR_INIT DEXCR_PR_NPHIE /* Fixed DEXCR value to initialise all CPUs with */ +#define SPRN_IC 0x350 /* Virtual Instruction Count */ +#define SPRN_VTB 0x351 /* Virtual Time Base */ +#define SPRN_LDBAR 0x352 /* LD Base Address Register */ +#define SPRN_PMICR 0x354 /* Power Management Idle Control Reg */ +#define SPRN_PMSR 0x355 /* Power Management Status Reg */ +#define SPRN_PMMAR 0x356 /* Power Management Memory Activity Register */ +#define SPRN_PSSCR 0x357 /* Processor Stop Status and Control Register (ISA 3.0) */ +#define SPRN_PSSCR_PR 0x337 /* PSSCR ISA 3.0, privileged mode access */ +#define SPRN_TRIG2 0x372 +#define SPRN_PMCR 0x374 /* Power Management Control Register */ +#define SPRN_RWMR 0x375 /* Region-Weighting Mode Register */ + +/* HFSCR and FSCR bit numbers are the same */ +#define FSCR_PREFIX_LG 13 /* Enable Prefix Instructions */ +#define FSCR_SCV_LG 12 /* Enable System Call Vectored */ +#define FSCR_MSGP_LG 10 /* Enable MSGP */ +#define FSCR_TAR_LG 8 /* Enable Target Address Register */ +#define FSCR_EBB_LG 7 /* Enable Event Based Branching */ +#define FSCR_TM_LG 5 /* Enable Transactional Memory */ +#define FSCR_BHRB_LG 4 /* Enable Branch History Rolling Buffer*/ +#define FSCR_PM_LG 3 /* Enable prob/priv access to PMU SPRs */ +#define FSCR_DSCR_LG 2 /* Enable Data Stream Control Register */ +#define FSCR_VECVSX_LG 1 /* Enable VMX/VSX */ +#define FSCR_FP_LG 0 /* Enable Floating Point */ #define SPRN_FSCR 0x099 /* Facility Status & Control Register */ -#define FSCR_TAR (1 << (63-55)) /* Enable Target Address Register */ -#define FSCR_EBB (1 << (63-56)) /* Enable Event Based Branching */ -#define FSCR_DSCR (1 << (63-61)) /* Enable Data Stream Control Register */ +#define FSCR_PREFIX __MASK(FSCR_PREFIX_LG) +#define FSCR_SCV __MASK(FSCR_SCV_LG) +#define FSCR_TAR __MASK(FSCR_TAR_LG) +#define FSCR_EBB __MASK(FSCR_EBB_LG) +#define FSCR_DSCR __MASK(FSCR_DSCR_LG) +#define FSCR_INTR_CAUSE (ASM_CONST(0xFF) << 56) /* interrupt cause */ #define SPRN_HFSCR 0xbe /* HV=1 Facility Status & Control Register */ -#define HFSCR_TAR (1 << (63-55)) /* Enable Target Address Register */ -#define HFSCR_EBB (1 << (63-56)) /* Enable Event Based Branching */ -#define HFSCR_TM (1 << (63-58)) /* Enable Transactional Memory */ -#define HFSCR_PM (1 << (63-60)) /* Enable prob/priv access to PMU SPRs */ -#define HFSCR_BHRB (1 << (63-59)) /* Enable Branch History Rolling Buffer*/ -#define HFSCR_DSCR (1 << (63-61)) /* Enable Data Stream Control Register */ -#define HFSCR_VECVSX (1 << (63-62)) /* Enable VMX/VSX */ -#define HFSCR_FP (1 << (63-63)) /* Enable Floating Point */ +#define HFSCR_PREFIX __MASK(FSCR_PREFIX_LG) +#define HFSCR_MSGP __MASK(FSCR_MSGP_LG) +#define HFSCR_TAR __MASK(FSCR_TAR_LG) +#define HFSCR_EBB __MASK(FSCR_EBB_LG) +#define HFSCR_TM __MASK(FSCR_TM_LG) +#define HFSCR_PM __MASK(FSCR_PM_LG) +#define HFSCR_BHRB __MASK(FSCR_BHRB_LG) +#define HFSCR_DSCR __MASK(FSCR_DSCR_LG) +#define HFSCR_VECVSX __MASK(FSCR_VECVSX_LG) +#define HFSCR_FP __MASK(FSCR_FP_LG) +#define HFSCR_INTR_CAUSE FSCR_INTR_CAUSE #define SPRN_TAR 0x32f /* Target Address Register */ #define SPRN_LPCR 0x13E /* LPAR Control Register */ -#define LPCR_VPM0 (1ul << (63-0)) -#define LPCR_VPM1 (1ul << (63-1)) -#define LPCR_ISL (1ul << (63-2)) -#define LPCR_VC_SH (63-2) -#define LPCR_DPFD_SH (63-11) -#define LPCR_VRMASD (0x1ful << (63-16)) -#define LPCR_VRMA_L (1ul << (63-12)) -#define LPCR_VRMA_LP0 (1ul << (63-15)) -#define LPCR_VRMA_LP1 (1ul << (63-16)) -#define LPCR_VRMASD_SH (63-16) -#define LPCR_RMLS 0x1C000000 /* impl dependent rmo limit sel */ -#define LPCR_RMLS_SH (63-37) -#define LPCR_ILE 0x02000000 /* !HV irqs set MSR:LE */ -#define LPCR_AIL_0 0x00000000 /* MMU off exception offset 0x0 */ -#define LPCR_AIL_3 0x01800000 /* MMU on exception offset 0xc00...4xxx */ -#define LPCR_PECE 0x00007000 /* powersave exit cause enable */ -#define LPCR_PECE0 0x00004000 /* ext. exceptions can cause exit */ -#define LPCR_PECE1 0x00002000 /* decrementer can cause exit */ -#define LPCR_PECE2 0x00001000 /* machine check etc can cause exit */ -#define LPCR_MER 0x00000800 /* Mediated External Exception */ -#define LPCR_MER_SH 11 -#define LPCR_LPES 0x0000000c -#define LPCR_LPES0 0x00000008 /* LPAR Env selector 0 */ -#define LPCR_LPES1 0x00000004 /* LPAR Env selector 1 */ -#define LPCR_LPES_SH 2 -#define LPCR_RMI 0x00000002 /* real mode is cache inhibit */ -#define LPCR_HDICE 0x00000001 /* Hyp Decr enable (HV,PR,EE) */ +#define LPCR_VPM0 ASM_CONST(0x8000000000000000) +#define LPCR_VPM1 ASM_CONST(0x4000000000000000) +#define LPCR_ISL ASM_CONST(0x2000000000000000) +#define LPCR_VC_SH 61 +#define LPCR_DPFD_SH 52 +#define LPCR_DPFD (ASM_CONST(7) << LPCR_DPFD_SH) +#define LPCR_VRMASD_SH 47 +#define LPCR_VRMASD (ASM_CONST(0x1f) << LPCR_VRMASD_SH) +#define LPCR_VRMA_L ASM_CONST(0x0008000000000000) +#define LPCR_VRMA_LP0 ASM_CONST(0x0001000000000000) +#define LPCR_VRMA_LP1 ASM_CONST(0x0000800000000000) +#define LPCR_RMLS 0x1C000000 /* Implementation dependent RMO limit sel */ +#define LPCR_RMLS_SH 26 +#define LPCR_HAIL ASM_CONST(0x0000000004000000) /* HV AIL (ISAv3.1) */ +#define LPCR_ILE ASM_CONST(0x0000000002000000) /* !HV irqs set MSR:LE */ +#define LPCR_AIL ASM_CONST(0x0000000001800000) /* Alternate interrupt location */ +#define LPCR_AIL_0 ASM_CONST(0x0000000000000000) /* MMU off exception offset 0x0 */ +#define LPCR_AIL_3 ASM_CONST(0x0000000001800000) /* MMU on exception offset 0xc00...4xxx */ +#define LPCR_ONL ASM_CONST(0x0000000000040000) /* online - PURR/SPURR count */ +#define LPCR_LD ASM_CONST(0x0000000000020000) /* large decremeter */ +#define LPCR_PECE ASM_CONST(0x000000000001f000) /* powersave exit cause enable */ +#define LPCR_PECEDP ASM_CONST(0x0000000000010000) /* directed priv dbells cause exit */ +#define LPCR_PECEDH ASM_CONST(0x0000000000008000) /* directed hyp dbells cause exit */ +#define LPCR_PECE0 ASM_CONST(0x0000000000004000) /* ext. exceptions can cause exit */ +#define LPCR_PECE1 ASM_CONST(0x0000000000002000) /* decrementer can cause exit */ +#define LPCR_PECE2 ASM_CONST(0x0000000000001000) /* machine check etc can cause exit */ +#define LPCR_PECE_HVEE ASM_CONST(0x0000400000000000) /* P9 Wakeup on HV interrupts */ +#define LPCR_MER ASM_CONST(0x0000000000000800) /* Mediated External Exception */ +#define LPCR_MER_SH 11 +#define LPCR_GTSE ASM_CONST(0x0000000000000400) /* Guest Translation Shootdown Enable */ +#define LPCR_TC ASM_CONST(0x0000000000000200) /* Translation control */ +#define LPCR_HEIC ASM_CONST(0x0000000000000010) /* Hypervisor External Interrupt Control */ +#define LPCR_LPES 0x0000000c +#define LPCR_LPES0 ASM_CONST(0x0000000000000008) /* LPAR Env selector 0 */ +#define LPCR_LPES1 ASM_CONST(0x0000000000000004) /* LPAR Env selector 1 */ +#define LPCR_LPES_SH 2 +#define LPCR_RMI ASM_CONST(0x0000000000000002) /* real mode is cache inhibit */ +#define LPCR_HVICE ASM_CONST(0x0000000000000002) /* P9: HV interrupt enable */ +#define LPCR_HDICE ASM_CONST(0x0000000000000001) /* Hyp Decr enable (HV,PR,EE) */ +#define LPCR_UPRT ASM_CONST(0x0000000000400000) /* Use Process Table (ISA 3) */ +#define LPCR_HR ASM_CONST(0x0000000000100000) #ifndef SPRN_LPID #define SPRN_LPID 0x13F /* Logical Partition Identifier */ #endif -#define LPID_RSVD 0x3ff /* Reserved LPID for partn switching */ -#define SPRN_HMER 0x150 /* Hardware m? error recovery */ -#define SPRN_HMEER 0x151 /* Hardware m? enable error recovery */ +#define SPRN_HMER 0x150 /* Hypervisor maintenance exception reg */ +#define HMER_DEBUG_TRIG (1ul << (63 - 17)) /* Debug trigger */ +#define SPRN_HMEER 0x151 /* Hyp maintenance exception enable reg */ +#define SPRN_PCR 0x152 /* Processor compatibility register */ +#define PCR_VEC_DIS (__MASK(63-0)) /* Vec. disable (bit NA since POWER8) */ +#define PCR_VSX_DIS (__MASK(63-1)) /* VSX disable (bit NA since POWER8) */ +#define PCR_TM_DIS (__MASK(63-2)) /* Trans. memory disable (POWER8) */ +#define PCR_MMA_DIS (__MASK(63-3)) /* Matrix-Multiply Accelerator */ +#define PCR_HIGH_BITS (PCR_MMA_DIS | PCR_VEC_DIS | PCR_VSX_DIS | PCR_TM_DIS) +/* + * These bits are used in the function kvmppc_set_arch_compat() to specify and + * determine both the compatibility level which we want to emulate and the + * compatibility level which the host is capable of emulating. + */ +#define PCR_ARCH_300 0x10 /* Architecture 3.00 */ +#define PCR_ARCH_207 0x8 /* Architecture 2.07 */ +#define PCR_ARCH_206 0x4 /* Architecture 2.06 */ +#define PCR_ARCH_205 0x2 /* Architecture 2.05 */ +#define PCR_LOW_BITS (PCR_ARCH_207 | PCR_ARCH_206 | PCR_ARCH_205 | PCR_ARCH_300) +#define PCR_MASK ~(PCR_HIGH_BITS | PCR_LOW_BITS) /* PCR Reserved Bits */ #define SPRN_HEIR 0x153 /* Hypervisor Emulated Instruction Register */ #define SPRN_TLBINDEXR 0x154 /* P7 TLB control register */ #define SPRN_TLBVPNR 0x155 /* P7 TLB control register */ @@ -324,9 +520,12 @@ #define SPRN_DBAT7L 0x23F /* Data BAT 7 Lower Register */ #define SPRN_DBAT7U 0x23E /* Data BAT 7 Upper Register */ #define SPRN_PPR 0x380 /* SMT Thread status Register */ +#define SPRN_TSCR 0x399 /* Thread Switch Control Register */ #define SPRN_DEC 0x016 /* Decrement Register */ -#define SPRN_DER 0x095 /* Debug Enable Regsiter */ +#define SPRN_PIT 0x3DB /* Programmable Interval Timer (BOOKE) */ + +#define SPRN_DER 0x095 /* Debug Enable Register */ #define DER_RSTE 0x40000000 /* Reset Interrupt */ #define DER_CHSTPE 0x20000000 /* Check Stop */ #define DER_MCIE 0x10000000 /* Machine Check Interrupt */ @@ -347,9 +546,11 @@ #define DER_EBRKE 0x00000002 /* External Breakpoint Interrupt */ #define DER_DPIE 0x00000001 /* Dev. Port Nonmaskable Request */ #define SPRN_DMISS 0x3D0 /* Data TLB Miss Register */ +#define SPRN_DHDES 0x0B1 /* Directed Hyp. Doorbell Exc. State */ +#define SPRN_DPDES 0x0B0 /* Directed Priv. Doorbell Exc. State */ #define SPRN_EAR 0x11A /* External Address Register */ #define SPRN_HASH1 0x3D2 /* Primary Hash Address Register */ -#define SPRN_HASH2 0x3D3 /* Secondary Hash Address Resgister */ +#define SPRN_HASH2 0x3D3 /* Secondary Hash Address Register */ #define SPRN_HID0 0x3F0 /* Hardware Implementation Register 0 */ #define HID0_HDICE_SH (63 - 23) /* 970 HDEC interrupt enable */ #define HID0_EMCP (1<<31) /* Enable Machine Check pin */ @@ -388,9 +589,18 @@ #define HID0_BTCD (1<<1) /* Branch target cache disable */ #define HID0_NOPDST (1<<1) /* No-op dst, dstt, etc. instr. */ #define HID0_NOPTI (1<<0) /* No-op dcbt and dcbst instr. */ +/* POWER8 HID0 bits */ +#define HID0_POWER8_4LPARMODE __MASK(61) +#define HID0_POWER8_2LPARMODE __MASK(57) +#define HID0_POWER8_1TO2LPAR __MASK(52) +#define HID0_POWER8_1TO4LPAR __MASK(51) +#define HID0_POWER8_DYNLPARDIS __MASK(48) + +/* POWER9 HID0 bits */ +#define HID0_POWER9_RADIX __MASK(63 - 8) #define SPRN_HID1 0x3F1 /* Hardware Implementation Register 1 */ -#ifdef CONFIG_6xx +#ifdef CONFIG_PPC_BOOK3S_32 #define HID1_EMCP (1<<31) /* 7450 Machine Check Pin Enable */ #define HID1_DFS (1<<22) /* 7447A Dynamic Frequency Scaling */ #define HID1_PC0 (1<<16) /* 7450 PLL_CFG[0] */ @@ -401,16 +611,20 @@ #define HID1_ABE (1<<10) /* 7450 Address Broadcast Enable */ #define HID1_PS (1<<16) /* 750FX PLL selection */ #endif -#define SPRN_HID2 0x3F8 /* Hardware Implementation Register 2 */ +#define SPRN_HID2_750FX 0x3F8 /* IBM 750FX HID2 Register */ #define SPRN_HID2_GEKKO 0x398 /* Gekko HID2 Register */ +#define SPRN_HID2_G2_LE 0x3F3 /* G2_LE HID2 Register */ +#define HID2_G2_LE_HBE (1<<18) /* High BAT Enable (G2_LE) */ #define SPRN_IABR 0x3F2 /* Instruction Address Breakpoint Register */ #define SPRN_IABR2 0x3FA /* 83xx */ #define SPRN_IBCR 0x135 /* 83xx Insn Breakpoint Control Reg */ +#define SPRN_IAMR 0x03D /* Instr. Authority Mask Reg */ #define SPRN_HID4 0x3F4 /* 970 HID4 */ #define HID4_LPES0 (1ul << (63-0)) /* LPAR env. sel. bit 0 */ #define HID4_RMLS2_SH (63 - 2) /* Real mode limit bottom 2 bits */ #define HID4_LPID5_SH (63 - 6) /* partition ID bottom 4 bits */ #define HID4_RMOR_SH (63 - 22) /* real mode offset (16 bits) */ +#define HID4_RMOR (0xFFFFul << HID4_RMOR_SH) #define HID4_LPES1 (1 << (63-57)) /* LPAR env. sel. bit 1 */ #define HID4_RMLS0_SH (63 - 58) /* Real mode limit top bit */ #define HID4_LPID1_SH 0 /* partition ID top 2 bits */ @@ -448,13 +662,15 @@ #define SPRN_IBAT7U 0x236 /* Instruction BAT 7 Upper Register */ #define SPRN_ICMP 0x3D5 /* Instruction TLB Compare Register */ #define SPRN_ICTC 0x3FB /* Instruction Cache Throttling Control Reg */ +#ifndef SPRN_ICTRL #define SPRN_ICTRL 0x3F3 /* 1011 7450 icache and interrupt ctrl */ +#endif #define ICTRL_EICE 0x08000000 /* enable icache parity errs */ #define ICTRL_EDC 0x04000000 /* enable dcache parity errs */ #define ICTRL_EICP 0x00000100 /* enable icache par. check */ #define SPRN_IMISS 0x3D4 /* Instruction TLB Miss Register */ #define SPRN_IMMR 0x27E /* Internal Memory Map Register */ -#define SPRN_L2CR 0x3F9 /* Level 2 Cache Control Regsiter */ +#define SPRN_L2CR 0x3F9 /* Level 2 Cache Control Register */ #define SPRN_L2CR2 0x3f8 #define L2CR_L2E 0x80000000 /* L2 enable */ #define L2CR_L2PE 0x40000000 /* L2 parity enable */ @@ -489,7 +705,7 @@ #define L2CR_L2DO_745x 0x00010000 /* L2 data only (745x) */ #define L2CR_L2REP_745x 0x00001000 /* L2 repl. algorithm (745x) */ #define L2CR_L2HWF_745x 0x00000800 /* L2 hardware flush (745x) */ -#define SPRN_L3CR 0x3FA /* Level 3 Cache Control Regsiter */ +#define SPRN_L3CR 0x3FA /* Level 3 Cache Control Register */ #define L3CR_L3E 0x80000000 /* L3 enable */ #define L3CR_L3PE 0x40000000 /* L3 data parity enable */ #define L3CR_L3APE 0x20000000 /* L3 addr parity enable */ @@ -519,6 +735,8 @@ #define SPRN_PIR 0x3FF /* Processor Identification Register */ #endif #define SPRN_TIR 0x1BE /* Thread Identification Register */ +#define SPRN_PTCR 0x1D0 /* Partition table control Register */ +#define SPRN_PSPB 0x09F /* Problem State Priority Boost reg */ #define SPRN_PTEHI 0x3D5 /* 981 7450 PTE HI word (S/W TLB load) */ #define SPRN_PTELO 0x3D6 /* 982 7450 PTE LO word (S/W TLB load) */ #define SPRN_PURR 0x135 /* Processor Utilization of Resources Reg */ @@ -534,36 +752,62 @@ #define SPRN_SPRG3 0x113 /* Special Purpose Register General 3 */ #define SPRN_USPRG3 0x103 /* SPRG3 userspace read */ #define SPRN_SPRG4 0x114 /* Special Purpose Register General 4 */ +#define SPRN_USPRG4 0x104 /* SPRG4 userspace read */ #define SPRN_SPRG5 0x115 /* Special Purpose Register General 5 */ +#define SPRN_USPRG5 0x105 /* SPRG5 userspace read */ #define SPRN_SPRG6 0x116 /* Special Purpose Register General 6 */ +#define SPRN_USPRG6 0x106 /* SPRG6 userspace read */ #define SPRN_SPRG7 0x117 /* Special Purpose Register General 7 */ +#define SPRN_USPRG7 0x107 /* SPRG7 userspace read */ #define SPRN_SRR0 0x01A /* Save/Restore Register 0 */ #define SPRN_SRR1 0x01B /* Save/Restore Register 1 */ + +#ifdef CONFIG_PPC_BOOK3S +/* + * Bits loaded from MSR upon interrupt. + * PPC (64-bit) bits 33-36,42-47 are interrupt dependent, the others are + * loaded from MSR. The exception is that SRESET and MCE do not always load + * bit 62 (RI) from MSR. Don't use PPC_BITMASK for this because 32-bit uses + * it. + */ +#define SRR1_MSR_BITS (~0x783f0000UL) +#endif + #define SRR1_ISI_NOPT 0x40000000 /* ISI: Not found in hash */ -#define SRR1_ISI_N_OR_G 0x10000000 /* ISI: Access is no-exec or G */ +#define SRR1_ISI_N_G_OR_CIP 0x10000000 /* ISI: Access is no-exec or G or CI for a prefixed instruction */ #define SRR1_ISI_PROT 0x08000000 /* ISI: Other protection fault */ #define SRR1_WAKEMASK 0x00380000 /* reason for wakeup */ +#define SRR1_WAKEMASK_P8 0x003c0000 /* reason for wakeup on POWER8 and 9 */ +#define SRR1_WAKEMCE_RESVD 0x003c0000 /* Unused/reserved value used by MCE wakeup to indicate cause to idle wakeup handler */ #define SRR1_WAKESYSERR 0x00300000 /* System error */ #define SRR1_WAKEEE 0x00200000 /* External interrupt */ +#define SRR1_WAKEHVI 0x00240000 /* Hypervisor Virtualization Interrupt (P9) */ #define SRR1_WAKEMT 0x00280000 /* mtctrl */ #define SRR1_WAKEHMI 0x00280000 /* Hypervisor maintenance */ #define SRR1_WAKEDEC 0x00180000 /* Decrementer interrupt */ +#define SRR1_WAKEDBELL 0x00140000 /* Privileged doorbell on P8 */ #define SRR1_WAKETHERM 0x00100000 /* Thermal management interrupt */ #define SRR1_WAKERESET 0x00100000 /* System reset */ +#define SRR1_WAKEHDBELL 0x000c0000 /* Hypervisor doorbell on P8 */ #define SRR1_WAKESTATE 0x00030000 /* Powersave exit mask [46:47] */ -#define SRR1_WS_DEEPEST 0x00030000 /* Some resources not maintained, - * may not be recoverable */ -#define SRR1_WS_DEEPER 0x00020000 /* Some resources not maintained */ -#define SRR1_WS_DEEP 0x00010000 /* All resources maintained */ +#define SRR1_WS_HVLOSS 0x00030000 /* HV resources not maintained */ +#define SRR1_WS_GPRLOSS 0x00020000 /* GPRs not maintained */ +#define SRR1_WS_NOLOSS 0x00010000 /* All resources maintained */ +#define SRR1_PROGTM 0x00200000 /* TM Bad Thing */ #define SRR1_PROGFPE 0x00100000 /* Floating Point Enabled */ #define SRR1_PROGILL 0x00080000 /* Illegal instruction */ #define SRR1_PROGPRIV 0x00040000 /* Privileged instruction */ #define SRR1_PROGTRAP 0x00020000 /* Trap */ #define SRR1_PROGADDR 0x00010000 /* SRR0 contains subsequent addr */ +#define SRR1_MCE_MCP 0x00080000 /* Machine check signal caused interrupt */ +#define SRR1_BOUNDARY 0x10000000 /* Prefixed instruction crosses 64-byte boundary */ +#define SRR1_PREFIXED 0x20000000 /* Exception caused by prefixed instruction */ + #define SPRN_HSRR0 0x13A /* Save/Restore Register 0 */ #define SPRN_HSRR1 0x13B /* Save/Restore Register 1 */ #define HSRR1_DENORM 0x00100000 /* Denorm exception */ +#define HSRR1_HISI_WRITE 0x00010000 /* HISI bcs couldn't update mem */ #define SPRN_TBCTL 0x35f /* PA6T Timebase control register */ #define TBCTL_FREEZE 0x0000000000000000ull /* Freeze all tbs */ @@ -579,7 +823,7 @@ #define THRM1_TIN (1 << 31) #define THRM1_TIV (1 << 30) #define THRM1_THRES(x) ((x&0x7f)<<23) -#define THRM3_SITV(x) ((x&0x3fff)<<1) +#define THRM3_SITV(x) ((x & 0x1fff) << 1) #define THRM1_TID (1<<2) #define THRM1_TIE (1<<1) #define THRM1_V (1<<0) @@ -618,16 +862,21 @@ #define MMCR0_PROBLEM_DISABLE MMCR0_FCP #define MMCR0_FCM1 0x10000000UL /* freeze counters while MSR mark = 1 */ #define MMCR0_FCM0 0x08000000UL /* freeze counters while MSR mark = 0 */ -#define MMCR0_PMXE 0x04000000UL /* performance monitor exception enable */ -#define MMCR0_FCECE 0x02000000UL /* freeze ctrs on enabled cond or event */ +#define MMCR0_PMXE ASM_CONST(0x04000000) /* perf mon exception enable */ +#define MMCR0_FCECE ASM_CONST(0x02000000) /* freeze ctrs on enabled cond or event */ #define MMCR0_TBEE 0x00400000UL /* time base exception enable */ +#define MMCR0_BHRBA 0x00200000UL /* BHRB Access allowed in userspace */ #define MMCR0_EBE 0x00100000UL /* Event based branch enable */ #define MMCR0_PMCC 0x000c0000UL /* PMC control */ +#define MMCR0_PMCCEXT ASM_CONST(0x00000200) /* PMCCEXT control */ #define MMCR0_PMCC_U6 0x00080000UL /* PMC1-6 are R/W by user (PR) */ #define MMCR0_PMC1CE 0x00008000UL /* PMC1 count enable*/ -#define MMCR0_PMCjCE 0x00004000UL /* PMCj count enable*/ +#define MMCR0_PMCjCE ASM_CONST(0x00004000) /* PMCj count enable*/ #define MMCR0_TRIGGER 0x00002000UL /* TRIGGER enable */ -#define MMCR0_PMAO 0x00000080UL /* performance monitor alert has occurred, set to 0 after handling exception */ +#define MMCR0_PMAO_SYNC ASM_CONST(0x00000800) /* PMU intr is synchronous */ +#define MMCR0_C56RUN ASM_CONST(0x00000100) /* PMC5/6 count when RUN=0 */ +/* performance monitor alert has occurred, set to 0 after handling exception */ +#define MMCR0_PMAO ASM_CONST(0x00000080) #define MMCR0_SHRFC 0x00000040UL /* SHRre freeze conditions between threads */ #define MMCR0_FC56 0x00000010UL /* freeze counters 5 and 6 */ #define MMCR0_FCTI 0x00000008UL /* freeze counters in tags inactive mode */ @@ -635,7 +884,10 @@ #define MMCR0_FCWAIT 0x00000002UL /* freeze counter in WAIT state */ #define MMCR0_FCHV 0x00000001UL /* freeze conditions in hypervisor mode */ #define SPRN_MMCR1 798 -#define SPRN_MMCR2 769 +#define SPRN_MMCR2 785 +#define SPRN_MMCR3 754 +#define SPRN_UMMCR2 769 +#define SPRN_UMMCR3 738 #define SPRN_MMCRA 0x312 #define MMCRA_SDSYNC 0x80000000UL /* SDAR synced with SIAR */ #define MMCRA_SDAR_DCACHE_MISS 0x40000000UL @@ -645,6 +897,7 @@ #define MMCRA_SLOT 0x07000000UL /* SLOT bits (37-39) */ #define MMCRA_SLOT_SHIFT 24 #define MMCRA_SAMPLE_ENABLE 0x00000001UL /* enable sampling */ +#define MMCRA_BHRB_DISABLE _UL(0x2000000000) // BHRB disable bit for ISA v3.1 #define POWER6_MMCRA_SDSYNC 0x0000080000000000ULL /* SDAR/SIAR synced */ #define POWER6_MMCRA_SIHV 0x0000040000000000ULL #define POWER6_MMCRA_SIPR 0x0000020000000000ULL @@ -660,6 +913,9 @@ #define SPRN_EBBHR 804 /* Event based branch handler register */ #define SPRN_EBBRR 805 /* Event based branch return register */ #define SPRN_BESCR 806 /* Branch event status and control register */ +#define BESCR_GE 0x8000000000000000ULL /* Global Enable */ +#define SPRN_WORT 895 /* Workload optimization register - thread */ +#define SPRN_WORC 863 /* Workload optimization register - core */ #define SPRN_PMC1 787 #define SPRN_PMC2 788 @@ -669,13 +925,22 @@ #define SPRN_PMC6 792 #define SPRN_PMC7 793 #define SPRN_PMC8 794 -#define SPRN_SIAR 780 -#define SPRN_SDAR 781 #define SPRN_SIER 784 #define SIER_SIPR 0x2000000 /* Sampled MSR_PR */ #define SIER_SIHV 0x1000000 /* Sampled MSR_HV */ #define SIER_SIAR_VALID 0x0400000 /* SIAR contents valid */ #define SIER_SDAR_VALID 0x0200000 /* SDAR contents valid */ +#define SPRN_SIER2 752 +#define SPRN_SIER3 753 +#define SPRN_USIER2 736 +#define SPRN_USIER3 737 +#define SPRN_SIAR 796 +#define SPRN_SDAR 797 +#define SPRN_TACR 888 +#define SPRN_TCSCR 889 +#define SPRN_CSIGR 890 +#define SPRN_SPMC1 892 +#define SPRN_SPMC2 893 /* When EBB is enabled, some of MMCR0/MMCR2/SIER are user accessible */ #define MMCR0_USER_MASK (MMCR0_FC | MMCR0_PMXE | MMCR0_PMAO) @@ -830,16 +1095,15 @@ * 64-bit embedded * - SPRG0 generic exception scratch * - SPRG2 TLB exception stack - * - SPRG3 critical exception scratch and - * CPU and NUMA node for VDSO getcpu (user visible) + * - SPRG3 critical exception scratch (user visible, sorry!) * - SPRG4 unused (user visible) * - SPRG6 TLB miss scratch (user visible, sorry !) - * - SPRG7 critical exception scratch + * - SPRG7 CPU and NUMA node for VDSO getcpu (user visible) * - SPRG8 machine check exception scratch * - SPRG9 debug exception scratch * * All 32-bit: - * - SPRG3 current thread_info pointer + * - SPRG3 current thread_struct physical addr pointer * (virtual on BookE, physical on others) * * 32-bit classic: @@ -848,15 +1112,6 @@ * - SPRG2 indicator that we are in RTAS * - SPRG4 (603 only) pseudo TLB LRU data * - * 32-bit 40x: - * - SPRG0 scratch for exception vectors - * - SPRG1 scratch for exception vectors - * - SPRG2 scratch for exception vectors - * - SPRG4 scratch for exception vectors (not 403) - * - SPRG5 scratch for exception vectors (not 403) - * - SPRG6 scratch for exception vectors (not 403) - * - SPRG7 scratch for exception vectors (not 403) - * * 32-bit 440 and FSL BookE: * - SPRG0 scratch for exception vectors * - SPRG1 scratch for exception vectors (*) @@ -872,13 +1127,10 @@ * readable variant for reads, which can avoid a fault * with KVM type virtualization. * - * (*) Under KVM, the host SPRG1 is used to point to - * the current VCPU data structure - * * 32-bit 8xx: * - SPRG0 scratch for exception vectors * - SPRG1 scratch for exception vectors - * - SPRG2 apparently unused but initialized + * - SPRG2 scratch for exception vectors * */ #ifdef CONFIG_PPC64 @@ -891,6 +1143,8 @@ #define SPRN_SPRG_SCRATCH0 SPRN_SPRG2 #define SPRN_SPRG_HPACA SPRN_HSPRG0 #define SPRN_SPRG_HSCRATCH0 SPRN_HSPRG1 +#define SPRN_SPRG_VDSO_READ SPRN_USPRG3 +#define SPRN_SPRG_VDSO_WRITE SPRN_SPRG3 #define GET_PACA(rX) \ BEGIN_FTR_SECTION_NESTED(66); \ @@ -934,6 +1188,8 @@ #define SPRN_SPRG_TLB_SCRATCH SPRN_SPRG6 #define SPRN_SPRG_GEN_SCRATCH SPRN_SPRG0 #define SPRN_SPRG_GDBELL_SCRATCH SPRN_SPRG_GEN_SCRATCH +#define SPRN_SPRG_VDSO_READ SPRN_USPRG7 +#define SPRN_SPRG_VDSO_WRITE SPRN_SPRG7 #define SET_PACA(rX) mtspr SPRN_SPRG_PACA,rX #define GET_PACA(rX) mfspr rX,SPRN_SPRG_PACA @@ -943,18 +1199,8 @@ #ifdef CONFIG_PPC_BOOK3S_32 #define SPRN_SPRG_SCRATCH0 SPRN_SPRG0 #define SPRN_SPRG_SCRATCH1 SPRN_SPRG1 -#define SPRN_SPRG_RTAS SPRN_SPRG2 -#define SPRN_SPRG_603_LRU SPRN_SPRG4 -#endif - -#ifdef CONFIG_40x -#define SPRN_SPRG_SCRATCH0 SPRN_SPRG0 -#define SPRN_SPRG_SCRATCH1 SPRN_SPRG1 #define SPRN_SPRG_SCRATCH2 SPRN_SPRG2 -#define SPRN_SPRG_SCRATCH3 SPRN_SPRG4 -#define SPRN_SPRG_SCRATCH4 SPRN_SPRG5 -#define SPRN_SPRG_SCRATCH5 SPRN_SPRG6 -#define SPRN_SPRG_SCRATCH6 SPRN_SPRG7 +#define SPRN_SPRG_603_LRU SPRN_SPRG4 #endif #ifdef CONFIG_BOOKE @@ -972,18 +1218,14 @@ #define SPRN_SPRG_WSCRATCH_MC SPRN_SPRG1 #define SPRN_SPRG_RSCRATCH4 SPRN_SPRG7R #define SPRN_SPRG_WSCRATCH4 SPRN_SPRG7W -#ifdef CONFIG_E200 -#define SPRN_SPRG_RSCRATCH_DBG SPRN_SPRG6R -#define SPRN_SPRG_WSCRATCH_DBG SPRN_SPRG6W -#else #define SPRN_SPRG_RSCRATCH_DBG SPRN_SPRG9 #define SPRN_SPRG_WSCRATCH_DBG SPRN_SPRG9 #endif -#endif -#ifdef CONFIG_8xx +#ifdef CONFIG_PPC_8xx #define SPRN_SPRG_SCRATCH0 SPRN_SPRG0 #define SPRN_SPRG_SCRATCH1 SPRN_SPRG1 +#define SPRN_SPRG_SCRATCH2 SPRN_SPRG2 #endif @@ -1053,16 +1295,23 @@ #define PVR_8560 0x80200000 #define PVR_VER_E500V1 0x8020 #define PVR_VER_E500V2 0x8021 +#define PVR_VER_E500MC 0x8023 +#define PVR_VER_E5500 0x8024 +#define PVR_VER_E6500 0x8040 +#define PVR_VER_7450 0x8000 +#define PVR_VER_7455 0x8001 +#define PVR_VER_7447 0x8002 +#define PVR_VER_7447A 0x8003 +#define PVR_VER_7448 0x8004 + /* * For the 8xx processors, all of them report the same PVR family for * the PowerPC core. The various versions of these processors must be * differentiated by the version number in the Communication Processor * Module (CPM). */ -#define PVR_821 0x00500000 -#define PVR_823 PVR_821 -#define PVR_850 PVR_821 -#define PVR_860 PVR_821 +#define PVR_8xx 0x00500000 + #define PVR_8240 0x00810100 #define PVR_8245 0x80811014 #define PVR_8260 PVR_8240 @@ -1088,78 +1337,112 @@ #define PVR_970MP 0x0044 #define PVR_970GX 0x0045 #define PVR_POWER7p 0x004A -#define PVR_POWER8 0x004B +#define PVR_POWER8E 0x004B +#define PVR_POWER8NVL 0x004C +#define PVR_POWER8 0x004D +#define PVR_HX_C2000 0x0066 +#define PVR_POWER9 0x004E +#define PVR_POWER10 0x0080 +#define PVR_POWER11 0x0082 #define PVR_BE 0x0070 #define PVR_PA6T 0x0090 +/* "Logical" PVR values defined in PAPR, representing architecture levels */ +#define PVR_ARCH_204 0x0f000001 +#define PVR_ARCH_205 0x0f000002 +#define PVR_ARCH_206 0x0f000003 +#define PVR_ARCH_206p 0x0f100003 +#define PVR_ARCH_207 0x0f000004 +#define PVR_ARCH_300 0x0f000005 +#define PVR_ARCH_31 0x0f000006 +#define PVR_ARCH_31_P11 0x0f000007 + /* Macros for setting and retrieving special purpose registers */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ + +#if defined(CONFIG_PPC64) || defined(__CHECKER__) +typedef struct { + u32 val; +#ifdef CONFIG_PPC64 + u32 suffix; +#endif +} __packed ppc_inst_t; +#else +typedef u32 ppc_inst_t; +#endif + #define mfmsr() ({unsigned long rval; \ asm volatile("mfmsr %0" : "=r" (rval) : \ : "memory"); rval;}) #ifdef CONFIG_PPC_BOOK3S_64 #define __mtmsrd(v, l) asm volatile("mtmsrd %0," __stringify(l) \ : : "r" (v) : "memory") -#define mtmsrd(v) __mtmsrd((v), 0) -#define mtmsr(v) mtmsrd(v) +#define mtmsr(v) __mtmsrd((v), 0) +#define __MTMSR "mtmsrd" #else #define mtmsr(v) asm volatile("mtmsr %0" : \ : "r" ((unsigned long)(v)) \ : "memory") +#define __mtmsrd(v, l) BUILD_BUG() +#define __MTMSR "mtmsr" #endif +static inline void mtmsr_isync(unsigned long val) +{ + asm volatile(__MTMSR " %0; " ASM_FTR_IFCLR("isync", "nop", %1) : : + "r" (val), "i" (CPU_FTR_ARCH_206) : "memory"); +} + #define mfspr(rn) ({unsigned long rval; \ asm volatile("mfspr %0," __stringify(rn) \ : "=r" (rval)); rval;}) #define mtspr(rn, v) asm volatile("mtspr " __stringify(rn) ",%0" : \ : "r" ((unsigned long)(v)) \ : "memory") - -#ifdef __powerpc64__ -#ifdef CONFIG_PPC_CELL -#define mftb() ({unsigned long rval; \ - asm volatile( \ - "90: mftb %0;\n" \ - "97: cmpwi %0,0;\n" \ - " beq- 90b;\n" \ - "99:\n" \ - ".section __ftr_fixup,\"a\"\n" \ - ".align 3\n" \ - "98:\n" \ - " .llong %1\n" \ - " .llong %1\n" \ - " .llong 97b-98b\n" \ - " .llong 99b-98b\n" \ - " .llong 0\n" \ - " .llong 0\n" \ - ".previous" \ - : "=r" (rval) : "i" (CPU_FTR_CELL_TB_BUG)); rval;}) -#else -#define mftb() ({unsigned long rval; \ - asm volatile("mftb %0" : "=r" (rval)); rval;}) -#endif /* !CONFIG_PPC_CELL */ - -#else /* __powerpc64__ */ - -#define mftbl() ({unsigned long rval; \ - asm volatile("mftbl %0" : "=r" (rval)); rval;}) -#define mftbu() ({unsigned long rval; \ - asm volatile("mftbu %0" : "=r" (rval)); rval;}) -#endif /* !__powerpc64__ */ - -#define mttbl(v) asm volatile("mttbl %0":: "r"(v)) -#define mttbu(v) asm volatile("mttbu %0":: "r"(v)) +#define wrtspr(rn) asm volatile("mtspr " __stringify(rn) ",2" : : : "memory") + +static inline void wrtee(unsigned long val) +{ + if (__builtin_constant_p(val)) + asm volatile("wrteei %0" : : "i" ((val & MSR_EE) ? 1 : 0) : "memory"); + else + asm volatile("wrtee %0" : : "r" (val) : "memory"); +} + +extern unsigned long msr_check_and_set(unsigned long bits); +extern bool strict_msr_control; +extern void __msr_check_and_clear(unsigned long bits); +static inline void msr_check_and_clear(unsigned long bits) +{ + if (strict_msr_control) + __msr_check_and_clear(bits); +} #ifdef CONFIG_PPC32 -#define mfsrin(v) ({unsigned int rval; \ - asm volatile("mfsrin %0,%1" : "=r" (rval) : "r" (v)); \ - rval;}) +static inline u32 mfsr(u32 idx) +{ + u32 val; + + if (__builtin_constant_p(idx)) + asm volatile("mfsr %0, %1" : "=r" (val): "i" (idx >> 28)); + else + asm volatile("mfsrin %0, %1" : "=r" (val): "r" (idx)); + + return val; +} + +static inline void mtsr(u32 val, u32 idx) +{ + if (__builtin_constant_p(idx)) + asm volatile("mtsr %1, %0" : : "r" (val), "i" (idx >> 28)); + else + asm volatile("mtsrin %0, %1" : : "r" (val), "r" (idx)); +} #endif -#define proc_trap() asm volatile("trap") +extern unsigned long current_stack_frame(void); -#define __get_SP() ({unsigned long sp; \ - asm volatile("mr %0,1": "=r" (sp)); sp;}) +register unsigned long current_stack_pointer asm("r1"); extern unsigned long scom970_read(unsigned int address); extern void scom970_write(unsigned int address, unsigned long value); @@ -1167,7 +1450,6 @@ extern void scom970_write(unsigned int address, unsigned long value); struct pt_regs; extern void ppc_save_regs(struct pt_regs *regs); - -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_REG_H */ diff --git a/arch/powerpc/include/asm/reg_8xx.h b/arch/powerpc/include/asm/reg_8xx.h index e8ea346b21d3..299ee7be0f67 100644 --- a/arch/powerpc/include/asm/reg_8xx.h +++ b/arch/powerpc/include/asm/reg_8xx.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Contains register definitions common to PowerPC 8xx CPUs. Notice */ @@ -14,6 +15,46 @@ #define SPRN_DC_ADR 569 /* Address needed for some commands */ #define SPRN_DC_DAT 570 /* Read-only data register */ +/* Misc Debug */ +#define SPRN_DPDR 630 +#define SPRN_MI_CAM 816 +#define SPRN_MI_RAM0 817 +#define SPRN_MI_RAM1 818 +#define SPRN_MD_CAM 824 +#define SPRN_MD_RAM0 825 +#define SPRN_MD_RAM1 826 + +/* Special MSR manipulation registers */ +#define SPRN_EIE 80 /* External interrupt enable (EE=1, RI=1) */ +#define SPRN_EID 81 /* External interrupt disable (EE=0, RI=1) */ +#define SPRN_NRI 82 /* Non recoverable interrupt (EE=0, RI=0) */ + +/* Debug registers */ +#define SPRN_CMPA 144 +#define SPRN_COUNTA 150 +#define SPRN_CMPE 152 +#define SPRN_CMPF 153 +#define SPRN_LCTRL1 156 +#define LCTRL1_CTE_GT 0xc0000000 +#define LCTRL1_CTF_LT 0x14000000 +#define LCTRL1_CRWE_RW 0x00000000 +#define LCTRL1_CRWE_RO 0x00040000 +#define LCTRL1_CRWE_WO 0x000c0000 +#define LCTRL1_CRWF_RW 0x00000000 +#define LCTRL1_CRWF_RO 0x00010000 +#define LCTRL1_CRWF_WO 0x00030000 +#define SPRN_LCTRL2 157 +#define LCTRL2_LW0EN 0x80000000 +#define LCTRL2_LW0LA_E 0x00000000 +#define LCTRL2_LW0LA_F 0x04000000 +#define LCTRL2_LW0LA_EandF 0x08000000 +#define LCTRL2_LW0LADC 0x02000000 +#define LCTRL2_SLW0EN 0x00000002 +#ifdef CONFIG_PPC_8xx +#define SPRN_ICTRL 158 +#endif +#define SPRN_BAR 159 + /* Commands. Only the first few are available to the instruction cache. */ #define IDC_ENABLE 0x02000000 /* Cache enable */ diff --git a/arch/powerpc/include/asm/reg_a2.h b/arch/powerpc/include/asm/reg_a2.h deleted file mode 100644 index 3d52a1132f3d..000000000000 --- a/arch/powerpc/include/asm/reg_a2.h +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Register definitions specific to the A2 core - * - * Copyright (C) 2008 Ben. Herrenschmidt (benh@kernel.crashing.org), IBM Corp. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ - -#ifndef __ASM_POWERPC_REG_A2_H__ -#define __ASM_POWERPC_REG_A2_H__ - -#define SPRN_TENSR 0x1b5 -#define SPRN_TENS 0x1b6 /* Thread ENable Set */ -#define SPRN_TENC 0x1b7 /* Thread ENable Clear */ - -#define SPRN_A2_CCR0 0x3f0 /* Core Configuration Register 0 */ -#define SPRN_A2_CCR1 0x3f1 /* Core Configuration Register 1 */ -#define SPRN_A2_CCR2 0x3f2 /* Core Configuration Register 2 */ -#define SPRN_MMUCR0 0x3fc /* MMU Control Register 0 */ -#define SPRN_MMUCR1 0x3fd /* MMU Control Register 1 */ -#define SPRN_MMUCR2 0x3fe /* MMU Control Register 2 */ -#define SPRN_MMUCR3 0x3ff /* MMU Control Register 3 */ - -#define SPRN_IAR 0x372 - -#define SPRN_IUCR0 0x3f3 -#define IUCR0_ICBI_ACK 0x1000 - -#define SPRN_XUCR0 0x3f6 /* Execution Unit Config Register 0 */ - -#define A2_IERAT_SIZE 16 -#define A2_DERAT_SIZE 32 - -/* A2 MMUCR0 bits */ -#define MMUCR0_ECL 0x80000000 /* Extended Class for TLB fills */ -#define MMUCR0_TID_NZ 0x40000000 /* TID is non-zero */ -#define MMUCR0_TS 0x10000000 /* Translation space for TLB fills */ -#define MMUCR0_TGS 0x20000000 /* Guest space for TLB fills */ -#define MMUCR0_TLBSEL 0x0c000000 /* TLB or ERAT target for TLB fills */ -#define MMUCR0_TLBSEL_U 0x00000000 /* TLBSEL = UTLB */ -#define MMUCR0_TLBSEL_I 0x08000000 /* TLBSEL = I-ERAT */ -#define MMUCR0_TLBSEL_D 0x0c000000 /* TLBSEL = D-ERAT */ -#define MMUCR0_LOCKSRSH 0x02000000 /* Use TLB lock on tlbsx. */ -#define MMUCR0_TID_MASK 0x000000ff /* TID field */ - -/* A2 MMUCR1 bits */ -#define MMUCR1_IRRE 0x80000000 /* I-ERAT round robin enable */ -#define MMUCR1_DRRE 0x40000000 /* D-ERAT round robin enable */ -#define MMUCR1_REE 0x20000000 /* Reference Exception Enable*/ -#define MMUCR1_CEE 0x10000000 /* Change exception enable */ -#define MMUCR1_CSINV_ALL 0x00000000 /* Inval ERAT on all CS evts */ -#define MMUCR1_CSINV_NISYNC 0x04000000 /* Inval ERAT on all ex isync*/ -#define MMUCR1_CSINV_NEVER 0x0c000000 /* Don't inval ERAT on CS */ -#define MMUCR1_ICTID 0x00080000 /* IERAT class field as TID */ -#define MMUCR1_ITTID 0x00040000 /* IERAT thdid field as TID */ -#define MMUCR1_DCTID 0x00020000 /* DERAT class field as TID */ -#define MMUCR1_DTTID 0x00010000 /* DERAT thdid field as TID */ -#define MMUCR1_DCCD 0x00008000 /* DERAT class ignore */ -#define MMUCR1_TLBWE_BINV 0x00004000 /* back invalidate on tlbwe */ - -/* A2 MMUCR2 bits */ -#define MMUCR2_PSSEL_SHIFT 4 - -/* A2 MMUCR3 bits */ -#define MMUCR3_THID 0x0000000f /* Thread ID */ - -/* *** ERAT TLB bits definitions */ -#define TLB0_EPN_MASK ASM_CONST(0xfffffffffffff000) -#define TLB0_CLASS_MASK ASM_CONST(0x0000000000000c00) -#define TLB0_CLASS_00 ASM_CONST(0x0000000000000000) -#define TLB0_CLASS_01 ASM_CONST(0x0000000000000400) -#define TLB0_CLASS_10 ASM_CONST(0x0000000000000800) -#define TLB0_CLASS_11 ASM_CONST(0x0000000000000c00) -#define TLB0_V ASM_CONST(0x0000000000000200) -#define TLB0_X ASM_CONST(0x0000000000000100) -#define TLB0_SIZE_MASK ASM_CONST(0x00000000000000f0) -#define TLB0_SIZE_4K ASM_CONST(0x0000000000000010) -#define TLB0_SIZE_64K ASM_CONST(0x0000000000000030) -#define TLB0_SIZE_1M ASM_CONST(0x0000000000000050) -#define TLB0_SIZE_16M ASM_CONST(0x0000000000000070) -#define TLB0_SIZE_1G ASM_CONST(0x00000000000000a0) -#define TLB0_THDID_MASK ASM_CONST(0x000000000000000f) -#define TLB0_THDID_0 ASM_CONST(0x0000000000000001) -#define TLB0_THDID_1 ASM_CONST(0x0000000000000002) -#define TLB0_THDID_2 ASM_CONST(0x0000000000000004) -#define TLB0_THDID_3 ASM_CONST(0x0000000000000008) -#define TLB0_THDID_ALL ASM_CONST(0x000000000000000f) - -#define TLB1_RESVATTR ASM_CONST(0x00f0000000000000) -#define TLB1_U0 ASM_CONST(0x0008000000000000) -#define TLB1_U1 ASM_CONST(0x0004000000000000) -#define TLB1_U2 ASM_CONST(0x0002000000000000) -#define TLB1_U3 ASM_CONST(0x0001000000000000) -#define TLB1_R ASM_CONST(0x0000800000000000) -#define TLB1_C ASM_CONST(0x0000400000000000) -#define TLB1_RPN_MASK ASM_CONST(0x000003fffffff000) -#define TLB1_W ASM_CONST(0x0000000000000800) -#define TLB1_I ASM_CONST(0x0000000000000400) -#define TLB1_M ASM_CONST(0x0000000000000200) -#define TLB1_G ASM_CONST(0x0000000000000100) -#define TLB1_E ASM_CONST(0x0000000000000080) -#define TLB1_VF ASM_CONST(0x0000000000000040) -#define TLB1_UX ASM_CONST(0x0000000000000020) -#define TLB1_SX ASM_CONST(0x0000000000000010) -#define TLB1_UW ASM_CONST(0x0000000000000008) -#define TLB1_SW ASM_CONST(0x0000000000000004) -#define TLB1_UR ASM_CONST(0x0000000000000002) -#define TLB1_SR ASM_CONST(0x0000000000000001) - -#ifdef CONFIG_PPC_EARLY_DEBUG_WSP -#define WSP_UART_PHYS 0xffc000c000 -/* This needs to be careful chosen to hit a !0 congruence class - * in the TLB since we bolt it in way 3, which is already occupied - * by our linear mapping primary bolted entry in CC 0. - */ -#define WSP_UART_VIRT 0xf000000000001000 -#endif - -/* A2 erativax attributes definitions */ -#define ERATIVAX_RS_IS_ALL 0x000 -#define ERATIVAX_RS_IS_TID 0x040 -#define ERATIVAX_RS_IS_CLASS 0x080 -#define ERATIVAX_RS_IS_FULLMATCH 0x0c0 -#define ERATIVAX_CLASS_00 0x000 -#define ERATIVAX_CLASS_01 0x010 -#define ERATIVAX_CLASS_10 0x020 -#define ERATIVAX_CLASS_11 0x030 -#define ERATIVAX_PSIZE_4K (TLB_PSIZE_4K >> 1) -#define ERATIVAX_PSIZE_64K (TLB_PSIZE_64K >> 1) -#define ERATIVAX_PSIZE_1M (TLB_PSIZE_1M >> 1) -#define ERATIVAX_PSIZE_16M (TLB_PSIZE_16M >> 1) -#define ERATIVAX_PSIZE_1G (TLB_PSIZE_1G >> 1) - -/* A2 eratilx attributes definitions */ -#define ERATILX_T_ALL 0 -#define ERATILX_T_TID 1 -#define ERATILX_T_TGS 2 -#define ERATILX_T_FULLMATCH 3 -#define ERATILX_T_CLASS0 4 -#define ERATILX_T_CLASS1 5 -#define ERATILX_T_CLASS2 6 -#define ERATILX_T_CLASS3 7 - -/* XUCR0 bits */ -#define XUCR0_TRACE_UM_T0 0x40000000 /* Thread 0 */ -#define XUCR0_TRACE_UM_T1 0x20000000 /* Thread 1 */ -#define XUCR0_TRACE_UM_T2 0x10000000 /* Thread 2 */ -#define XUCR0_TRACE_UM_T3 0x08000000 /* Thread 3 */ - -/* A2 CCR0 register */ -#define A2_CCR0_PME_DISABLED 0x00000000 -#define A2_CCR0_PME_SLEEP 0x40000000 -#define A2_CCR0_PME_RVW 0x80000000 -#define A2_CCR0_PME_DISABLED2 0xc0000000 - -/* A2 CCR2 register */ -#define A2_CCR2_ERAT_ONLY_MODE 0x00000001 -#define A2_CCR2_ENABLE_ICSWX 0x00000002 -#define A2_CCR2_ENABLE_PC 0x20000000 -#define A2_CCR2_ENABLE_TRACE 0x40000000 - -#endif /* __ASM_POWERPC_REG_A2_H__ */ diff --git a/arch/powerpc/include/asm/reg_booke.h b/arch/powerpc/include/asm/reg_booke.h index b417de3cc2c4..56f9d3b1de85 100644 --- a/arch/powerpc/include/asm/reg_booke.h +++ b/arch/powerpc/include/asm/reg_booke.h @@ -1,13 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Contains register definitions common to the Book E PowerPC - * specification. Notice that while the IBM-40x series of CPUs - * are not true Book E PowerPCs, they borrowed a number of features - * before Book E was finalized, and are included here as well. Unfortunately, - * they sometimes used different locations than true Book E CPUs did. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 2 - * as published by the Free Software Foundation. + * specification. * * Copyright 2009-2010 Freescale Semiconductor, Inc. */ @@ -15,27 +9,36 @@ #ifndef __ASM_POWERPC_REG_BOOKE_H__ #define __ASM_POWERPC_REG_BOOKE_H__ +#include <asm/ppc-opcode.h> + /* Machine State Register (MSR) Fields */ -#define MSR_GS (1<<28) /* Guest state */ -#define MSR_UCLE (1<<26) /* User-mode cache lock enable */ -#define MSR_SPE (1<<25) /* Enable SPE */ -#define MSR_DWE (1<<10) /* Debug Wait Enable */ -#define MSR_UBLE (1<<10) /* BTB lock enable (e500) */ -#define MSR_IS MSR_IR /* Instruction Space */ -#define MSR_DS MSR_DR /* Data Space */ -#define MSR_PMM (1<<2) /* Performance monitor mark bit */ -#define MSR_CM (1<<31) /* Computation Mode (0=32-bit, 1=64-bit) */ +#define MSR_GS_LG 28 /* Guest state */ +#define MSR_UCLE_LG 26 /* User-mode cache lock enable */ +#define MSR_SPE_LG 25 /* Enable SPE */ +#define MSR_DWE_LG 10 /* Debug Wait Enable */ +#define MSR_UBLE_LG 10 /* BTB lock enable (e500) */ +#define MSR_IS_LG MSR_IR_LG /* Instruction Space */ +#define MSR_DS_LG MSR_DR_LG /* Data Space */ +#define MSR_PMM_LG 2 /* Performance monitor mark bit */ +#define MSR_CM_LG 31 /* Computation Mode (0=32-bit, 1=64-bit) */ + +#define MSR_GS __MASK(MSR_GS_LG) +#define MSR_UCLE __MASK(MSR_UCLE_LG) +#define MSR_SPE __MASK(MSR_SPE_LG) +#define MSR_DWE __MASK(MSR_DWE_LG) +#define MSR_UBLE __MASK(MSR_UBLE_LG) +#define MSR_IS __MASK(MSR_IS_LG) +#define MSR_DS __MASK(MSR_DS_LG) +#define MSR_PMM __MASK(MSR_PMM_LG) +#define MSR_CM __MASK(MSR_CM_LG) #if defined(CONFIG_PPC_BOOK3E_64) #define MSR_64BIT MSR_CM -#define MSR_ MSR_ME | MSR_CE -#define MSR_KERNEL MSR_ | MSR_64BIT -#define MSR_USER32 MSR_ | MSR_PR | MSR_EE -#define MSR_USER64 MSR_USER32 | MSR_64BIT -#elif defined (CONFIG_40x) -#define MSR_KERNEL (MSR_ME|MSR_RI|MSR_IR|MSR_DR|MSR_CE) -#define MSR_USER (MSR_KERNEL|MSR_PR|MSR_EE) +#define MSR_ (MSR_ME | MSR_RI | MSR_CE) +#define MSR_KERNEL (MSR_ | MSR_64BIT) +#define MSR_USER32 (MSR_ | MSR_PR | MSR_EE) +#define MSR_USER64 (MSR_USER32 | MSR_64BIT) #else #define MSR_KERNEL (MSR_ME|MSR_RI|MSR_CE) #define MSR_USER (MSR_KERNEL|MSR_PR|MSR_EE) @@ -101,6 +104,7 @@ #define SPRN_IVOR39 0x1B1 /* Interrupt Vector Offset Register 39 */ #define SPRN_IVOR40 0x1B2 /* Interrupt Vector Offset Register 40 */ #define SPRN_IVOR41 0x1B3 /* Interrupt Vector Offset Register 41 */ +#define SPRN_IVOR42 0x1B4 /* Interrupt Vector Offset Register 42 */ #define SPRN_GIVOR2 0x1B8 /* Guest IVOR2 */ #define SPRN_GIVOR3 0x1B9 /* Guest IVOR3 */ #define SPRN_GIVOR4 0x1BA /* Guest IVOR4 */ @@ -147,7 +151,6 @@ #define SPRN_TLB3CFG 0x2B3 /* TLB 3 Config Register */ #define SPRN_EPR 0x2BE /* External Proxy Register */ #define SPRN_CCR1 0x378 /* Core Configuration Register 1 */ -#define SPRN_ZPR 0x3B0 /* Zone Protection Register (40x) */ #define SPRN_MAS7 0x3B0 /* MMU Assist Register 7 */ #define SPRN_MMUCR 0x3B2 /* MMU Control Register */ #define SPRN_CCR0 0x3B3 /* Core Configuration Register 0 */ @@ -156,7 +159,6 @@ #define SPRN_SGR 0x3B9 /* Storage Guarded Register */ #define SPRN_DCWR 0x3BA /* Data Cache Write-thru Register */ #define SPRN_SLER 0x3BB /* Little-endian real mode */ -#define SPRN_SU0R 0x3BC /* "User 0" real mode (40x) */ #define SPRN_DCMP 0x3D1 /* Data TLB Compare Register */ #define SPRN_ICDBDR 0x3D3 /* Instruction Cache Debug Data Register */ #define SPRN_EVPR 0x3D6 /* Exception Vector Prefix Register */ @@ -164,19 +166,17 @@ #define SPRN_L1CSR1 0x3F3 /* L1 Cache Control and Status Register 1 */ #define SPRN_MMUCSR0 0x3F4 /* MMU Control and Status Register 0 */ #define SPRN_MMUCFG 0x3F7 /* MMU Configuration Register */ -#define SPRN_PIT 0x3DB /* Programmable Interval Timer */ #define SPRN_BUCSR 0x3F5 /* Branch Unit Control and Status */ #define SPRN_L2CSR0 0x3F9 /* L2 Data Cache Control and Status Register 0 */ #define SPRN_L2CSR1 0x3FA /* L2 Data Cache Control and Status Register 1 */ #define SPRN_DCCR 0x3FA /* Data Cache Cacheability Register */ #define SPRN_ICCR 0x3FB /* Instruction Cache Cacheability Register */ +#define SPRN_PWRMGTCR0 0x3FB /* Power management control register 0 */ #define SPRN_SVR 0x3FF /* System Version Register */ /* - * SPRs which have conflicting definitions on true Book E versus classic, - * or IBM 40x. + * SPRs which have conflicting definitions on true Book E versus classic. */ -#ifdef CONFIG_BOOKE #define SPRN_CSRR0 0x03A /* Critical Save and Restore Register 0 */ #define SPRN_CSRR1 0x03B /* Critical Save and Restore Register 1 */ #define SPRN_DEAR 0x03D /* Data Error Address Register */ @@ -191,31 +191,20 @@ #define SPRN_DAC2 0x13D /* Data Address Compare 2 */ #define SPRN_TSR 0x150 /* Timer Status Register */ #define SPRN_TCR 0x154 /* Timer Control Register */ -#endif /* Book E */ -#ifdef CONFIG_40x -#define SPRN_DBCR1 0x3BD /* Debug Control Register 1 */ -#define SPRN_ESR 0x3D4 /* Exception Syndrome Register */ -#define SPRN_DEAR 0x3D5 /* Data Error Address Register */ -#define SPRN_TSR 0x3D8 /* Timer Status Register */ -#define SPRN_TCR 0x3DA /* Timer Control Register */ -#define SPRN_SRR2 0x3DE /* Save/Restore Register 2 */ -#define SPRN_SRR3 0x3DF /* Save/Restore Register 3 */ -#define SPRN_DBSR 0x3F0 /* Debug Status Register */ -#define SPRN_DBCR0 0x3F2 /* Debug Control Register 0 */ -#define SPRN_DAC1 0x3F6 /* Data Address Compare 1 */ -#define SPRN_DAC2 0x3F7 /* Data Address Compare 2 */ -#define SPRN_CSRR0 SPRN_SRR2 /* Critical Save and Restore Register 0 */ -#define SPRN_CSRR1 SPRN_SRR3 /* Critical Save and Restore Register 1 */ -#endif - -#ifdef CONFIG_PPC_ICSWX #define SPRN_HACOP 0x15F /* Hypervisor Available Coprocessor Register */ -#endif /* Bit definitions for CCR1. */ #define CCR1_DPC 0x00000100 /* Disable L1 I-Cache/D-Cache parity checking */ #define CCR1_TCS 0x00000080 /* Timer Clock Select */ +/* Bit definitions for PWRMGTCR0. */ +#define PWRMGTCR0_PW20_WAIT (1 << 14) /* PW20 state enable bit */ +#define PWRMGTCR0_PW20_ENT_SHIFT 8 +#define PWRMGTCR0_PW20_ENT 0x3F00 +#define PWRMGTCR0_AV_IDLE_PD_EN (1 << 22) /* Altivec idle enable */ +#define PWRMGTCR0_AV_IDLE_CNT_SHIFT 16 +#define PWRMGTCR0_AV_IDLE_CNT 0x3F0000 + /* Bit definitions for the MCSR. */ #define MCSR_MCS 0x80000000 /* Machine Check Summary */ #define MCSR_IB 0x40000000 /* Instruction PLB Error */ @@ -231,7 +220,7 @@ #define PPC47x_MCSR_FPR 0x00800000 /* FPR parity error */ #define PPC47x_MCSR_IPR 0x00400000 /* Imprecise Machine Check Exception */ -#ifdef CONFIG_E500 +#ifdef CONFIG_PPC_E500 /* All e500 */ #define MCSR_MCP 0x80000000UL /* Machine Check Input Pin */ #define MCSR_ICPERR 0x40000000UL /* I-Cache Parity Error */ @@ -250,7 +239,7 @@ /* e500mc */ #define MCSR_DCPERR_MC 0x20000000UL /* D-Cache Parity Error */ -#define MCSR_L2MMU_MHIT 0x04000000UL /* Hit on multiple TLB entries */ +#define MCSR_L2MMU_MHIT 0x08000000UL /* Hit on multiple TLB entries */ #define MCSR_NMI 0x00100000UL /* Non-Maskable Interrupt */ #define MCSR_MAV 0x00080000UL /* MCAR address valid */ #define MCSR_MEA 0x00040000UL /* MCAR is effective address */ @@ -266,20 +255,8 @@ #define MSRP_PMMP 0x00000004 /* Protect MSR[PMM] */ #endif -#ifdef CONFIG_E200 -#define MCSR_MCP 0x80000000UL /* Machine Check Input Pin */ -#define MCSR_CP_PERR 0x20000000UL /* Cache Push Parity Error */ -#define MCSR_CPERR 0x10000000UL /* Cache Parity Error */ -#define MCSR_EXCP_ERR 0x08000000UL /* ISI, ITLB, or Bus Error on 1st insn - fetch for an exception handler */ -#define MCSR_BUS_IRERR 0x00000010UL /* Read Bus Error on instruction fetch*/ -#define MCSR_BUS_DRERR 0x00000008UL /* Read Bus Error on data load */ -#define MCSR_BUS_WRERR 0x00000004UL /* Write Bus Error on buffered - store or cache line push */ -#endif - /* Bit definitions for the HID1 */ -#ifdef CONFIG_E500 +#ifdef CONFIG_PPC_E500 /* e500v1/v2 */ #define HID1_PLL_CFG_MASK 0xfc000000 /* PLL_CFG input pins */ #define HID1_RFXE 0x00020000 /* Read fault exception enable */ @@ -293,10 +270,8 @@ #endif /* Bit definitions for the DBSR. */ -/* - * DBSR bits which have conflicting definitions on true Book E versus IBM 40x. - */ -#ifdef CONFIG_BOOKE +#define DBSR_IDE 0x80000000 /* Imprecise Debug Event */ +#define DBSR_MRR 0x30000000 /* Most Recent Reset */ #define DBSR_IC 0x08000000 /* Instruction Completion */ #define DBSR_BT 0x04000000 /* Branch Taken */ #define DBSR_IRPT 0x02000000 /* Exception Debug Event */ @@ -314,21 +289,6 @@ #define DBSR_CRET 0x00000020 /* Critical Return Debug Event */ #define DBSR_IAC12ATS 0x00000002 /* Instr Address Compare 1/2 Toggle */ #define DBSR_IAC34ATS 0x00000001 /* Instr Address Compare 3/4 Toggle */ -#endif -#ifdef CONFIG_40x -#define DBSR_IC 0x80000000 /* Instruction Completion */ -#define DBSR_BT 0x40000000 /* Branch taken */ -#define DBSR_IRPT 0x20000000 /* Exception Debug Event */ -#define DBSR_TIE 0x10000000 /* Trap Instruction debug Event */ -#define DBSR_IAC1 0x04000000 /* Instruction Address Compare 1 Event */ -#define DBSR_IAC2 0x02000000 /* Instruction Address Compare 2 Event */ -#define DBSR_IAC3 0x00080000 /* Instruction Address Compare 3 Event */ -#define DBSR_IAC4 0x00040000 /* Instruction Address Compare 4 Event */ -#define DBSR_DAC1R 0x01000000 /* Data Address Compare 1 Read Event */ -#define DBSR_DAC1W 0x00800000 /* Data Address Compare 1 Write Event */ -#define DBSR_DAC2R 0x00400000 /* Data Address Compare 2 Read Event */ -#define DBSR_DAC2W 0x00200000 /* Data Address Compare 2 Write Event */ -#endif /* Bit definitions related to the ESR. */ #define ESR_MCI 0x80000000 /* Machine Check - Instruction */ @@ -350,69 +310,6 @@ #define ESR_SPV 0x00000080 /* Signal Processing operation */ /* Bit definitions related to the DBCR0. */ -#if defined(CONFIG_40x) -#define DBCR0_EDM 0x80000000 /* External Debug Mode */ -#define DBCR0_IDM 0x40000000 /* Internal Debug Mode */ -#define DBCR0_RST 0x30000000 /* all the bits in the RST field */ -#define DBCR0_RST_SYSTEM 0x30000000 /* System Reset */ -#define DBCR0_RST_CHIP 0x20000000 /* Chip Reset */ -#define DBCR0_RST_CORE 0x10000000 /* Core Reset */ -#define DBCR0_RST_NONE 0x00000000 /* No Reset */ -#define DBCR0_IC 0x08000000 /* Instruction Completion */ -#define DBCR0_ICMP DBCR0_IC -#define DBCR0_BT 0x04000000 /* Branch Taken */ -#define DBCR0_BRT DBCR0_BT -#define DBCR0_EDE 0x02000000 /* Exception Debug Event */ -#define DBCR0_IRPT DBCR0_EDE -#define DBCR0_TDE 0x01000000 /* TRAP Debug Event */ -#define DBCR0_IA1 0x00800000 /* Instr Addr compare 1 enable */ -#define DBCR0_IAC1 DBCR0_IA1 -#define DBCR0_IA2 0x00400000 /* Instr Addr compare 2 enable */ -#define DBCR0_IAC2 DBCR0_IA2 -#define DBCR0_IA12 0x00200000 /* Instr Addr 1-2 range enable */ -#define DBCR0_IA12X 0x00100000 /* Instr Addr 1-2 range eXclusive */ -#define DBCR0_IA3 0x00080000 /* Instr Addr compare 3 enable */ -#define DBCR0_IAC3 DBCR0_IA3 -#define DBCR0_IA4 0x00040000 /* Instr Addr compare 4 enable */ -#define DBCR0_IAC4 DBCR0_IA4 -#define DBCR0_IA34 0x00020000 /* Instr Addr 3-4 range Enable */ -#define DBCR0_IA34X 0x00010000 /* Instr Addr 3-4 range eXclusive */ -#define DBCR0_IA12T 0x00008000 /* Instr Addr 1-2 range Toggle */ -#define DBCR0_IA34T 0x00004000 /* Instr Addr 3-4 range Toggle */ -#define DBCR0_FT 0x00000001 /* Freeze Timers on debug event */ - -#define dbcr_iac_range(task) ((task)->thread.dbcr0) -#define DBCR_IAC12I DBCR0_IA12 /* Range Inclusive */ -#define DBCR_IAC12X (DBCR0_IA12 | DBCR0_IA12X) /* Range Exclusive */ -#define DBCR_IAC12MODE (DBCR0_IA12 | DBCR0_IA12X) /* IAC 1-2 Mode Bits */ -#define DBCR_IAC34I DBCR0_IA34 /* Range Inclusive */ -#define DBCR_IAC34X (DBCR0_IA34 | DBCR0_IA34X) /* Range Exclusive */ -#define DBCR_IAC34MODE (DBCR0_IA34 | DBCR0_IA34X) /* IAC 3-4 Mode Bits */ - -/* Bit definitions related to the DBCR1. */ -#define DBCR1_DAC1R 0x80000000 /* DAC1 Read Debug Event */ -#define DBCR1_DAC2R 0x40000000 /* DAC2 Read Debug Event */ -#define DBCR1_DAC1W 0x20000000 /* DAC1 Write Debug Event */ -#define DBCR1_DAC2W 0x10000000 /* DAC2 Write Debug Event */ - -#define dbcr_dac(task) ((task)->thread.dbcr1) -#define DBCR_DAC1R DBCR1_DAC1R -#define DBCR_DAC1W DBCR1_DAC1W -#define DBCR_DAC2R DBCR1_DAC2R -#define DBCR_DAC2W DBCR1_DAC2W - -/* - * Are there any active Debug Events represented in the - * Debug Control Registers? - */ -#define DBCR0_ACTIVE_EVENTS (DBCR0_ICMP | DBCR0_IAC1 | DBCR0_IAC2 | \ - DBCR0_IAC3 | DBCR0_IAC4) -#define DBCR1_ACTIVE_EVENTS (DBCR1_DAC1R | DBCR1_DAC2R | \ - DBCR1_DAC1W | DBCR1_DAC2W) -#define DBCR_ACTIVE_EVENTS(dbcr0, dbcr1) (((dbcr0) & DBCR0_ACTIVE_EVENTS) || \ - ((dbcr1) & DBCR1_ACTIVE_EVENTS)) - -#elif defined(CONFIG_BOOKE) #define DBCR0_EDM 0x80000000 /* External Debug Mode */ #define DBCR0_IDM 0x40000000 /* Internal Debug Mode */ #define DBCR0_RST 0x30000000 /* all the bits in the RST field */ @@ -441,7 +338,7 @@ #define DBCR0_CRET 0x00000020 /* Critical Return Debug Event */ #define DBCR0_FT 0x00000001 /* Freeze Timers on debug event */ -#define dbcr_dac(task) ((task)->thread.dbcr0) +#define dbcr_dac(task) ((task)->thread.debug.dbcr0) #define DBCR_DAC1R DBCR0_DAC1R #define DBCR_DAC1W DBCR0_DAC1W #define DBCR_DAC2R DBCR0_DAC2R @@ -475,7 +372,7 @@ #define DBCR1_IAC34MX 0x000000C0 /* Instr Addr 3-4 range eXclusive */ #define DBCR1_IAC34AT 0x00000001 /* Instr Addr 3-4 range Toggle */ -#define dbcr_iac_range(task) ((task)->thread.dbcr1) +#define dbcr_iac_range(task) ((task)->thread.debug.dbcr1) #define DBCR_IAC12I DBCR1_IAC12M /* Range Inclusive */ #define DBCR_IAC12X DBCR1_IAC12MX /* Range Exclusive */ #define DBCR_IAC12MODE DBCR1_IAC12MX /* IAC 1-2 Mode Bits */ @@ -513,7 +410,6 @@ #define DBCR_ACTIVE_EVENTS(dbcr0, dbcr1) (((dbcr0) & DBCR0_ACTIVE_EVENTS) || \ ((dbcr1) & DBCR1_ACTIVE_EVENTS)) -#endif /* #elif defined(CONFIG_BOOKE) */ /* Bit definitions related to the TCR. */ #define TCR_WP(x) (((x)&0x3)<<30) /* WDT Period */ @@ -540,7 +436,7 @@ #define TCR_FIE 0x00800000 /* FIT Interrupt Enable */ #define TCR_ARE 0x00400000 /* Auto Reload Enable */ -#ifdef CONFIG_E500 +#ifdef CONFIG_PPC_E500 #define TCR_GET_WP(tcr) ((((tcr) & 0xC0000000) >> 30) | \ (((tcr) & 0x1E0000) >> 15)) #else @@ -573,6 +469,7 @@ /* Bit definitions for L1CSR0. */ #define L1CSR0_CPE 0x00010000 /* Data Cache Parity Enable */ +#define L1CSR0_CUL 0x00000400 /* Data Cache Unable to Lock */ #define L1CSR0_CLFC 0x00000100 /* Cache Lock Bits Flash Clear */ #define L1CSR0_DCFI 0x00000002 /* Data Cache Flash Invalidate */ #define L1CSR0_CFI 0x00000002 /* Cache Flash Invalidate */ @@ -587,6 +484,13 @@ /* Bit definitions for L1CSR2. */ #define L1CSR2_DCWS 0x40000000 /* Data Cache write shadow */ +/* Bit definitions for BUCSR. */ +#define BUCSR_STAC_EN 0x01000000 /* Segment Target Address Cache */ +#define BUCSR_LS_EN 0x00400000 /* Link Stack */ +#define BUCSR_BBFI 0x00000200 /* Branch Buffer flash invalidate */ +#define BUCSR_BPEN 0x00000001 /* Branch prediction enable */ +#define BUCSR_INIT (BUCSR_STAC_EN | BUCSR_LS_EN | BUCSR_BBFI | BUCSR_BPEN) + /* Bit definitions for L2CSR0. */ #define L2CSR0_L2E 0x80000000 /* L2 Cache Enable */ #define L2CSR0_L2PE 0x40000000 /* L2 Cache Parity/ECC Enable */ @@ -637,60 +541,6 @@ #define EPC_EPID 0x00003fff #define EPC_EPID_SHIFT 0 -/* - * The IBM-403 is an even more odd special case, as it is much - * older than the IBM-405 series. We put these down here incase someone - * wishes to support these machines again. - */ -#ifdef CONFIG_403GCX -/* Special Purpose Registers (SPRNs)*/ -#define SPRN_TBHU 0x3CC /* Time Base High User-mode */ -#define SPRN_TBLU 0x3CD /* Time Base Low User-mode */ -#define SPRN_CDBCR 0x3D7 /* Cache Debug Control Register */ -#define SPRN_TBHI 0x3DC /* Time Base High */ -#define SPRN_TBLO 0x3DD /* Time Base Low */ -#define SPRN_DBCR 0x3F2 /* Debug Control Regsiter */ -#define SPRN_PBL1 0x3FC /* Protection Bound Lower 1 */ -#define SPRN_PBL2 0x3FE /* Protection Bound Lower 2 */ -#define SPRN_PBU1 0x3FD /* Protection Bound Upper 1 */ -#define SPRN_PBU2 0x3FF /* Protection Bound Upper 2 */ - - -/* Bit definitions for the DBCR. */ -#define DBCR_EDM DBCR0_EDM -#define DBCR_IDM DBCR0_IDM -#define DBCR_RST(x) (((x) & 0x3) << 28) -#define DBCR_RST_NONE 0 -#define DBCR_RST_CORE 1 -#define DBCR_RST_CHIP 2 -#define DBCR_RST_SYSTEM 3 -#define DBCR_IC DBCR0_IC /* Instruction Completion Debug Evnt */ -#define DBCR_BT DBCR0_BT /* Branch Taken Debug Event */ -#define DBCR_EDE DBCR0_EDE /* Exception Debug Event */ -#define DBCR_TDE DBCR0_TDE /* TRAP Debug Event */ -#define DBCR_FER 0x00F80000 /* First Events Remaining Mask */ -#define DBCR_FT 0x00040000 /* Freeze Timers on Debug Event */ -#define DBCR_IA1 0x00020000 /* Instr. Addr. Compare 1 Enable */ -#define DBCR_IA2 0x00010000 /* Instr. Addr. Compare 2 Enable */ -#define DBCR_D1R 0x00008000 /* Data Addr. Compare 1 Read Enable */ -#define DBCR_D1W 0x00004000 /* Data Addr. Compare 1 Write Enable */ -#define DBCR_D1S(x) (((x) & 0x3) << 12) /* Data Adrr. Compare 1 Size */ -#define DAC_BYTE 0 -#define DAC_HALF 1 -#define DAC_WORD 2 -#define DAC_QUAD 3 -#define DBCR_D2R 0x00000800 /* Data Addr. Compare 2 Read Enable */ -#define DBCR_D2W 0x00000400 /* Data Addr. Compare 2 Write Enable */ -#define DBCR_D2S(x) (((x) & 0x3) << 8) /* Data Addr. Compare 2 Size */ -#define DBCR_SBT 0x00000040 /* Second Branch Taken Debug Event */ -#define DBCR_SED 0x00000020 /* Second Exception Debug Event */ -#define DBCR_STD 0x00000010 /* Second Trap Debug Event */ -#define DBCR_SIA 0x00000008 /* Second IAC Enable */ -#define DBCR_SDA 0x00000004 /* Second DAC Enable */ -#define DBCR_JOI 0x00000002 /* JTAG Serial Outbound Int. Enable */ -#define DBCR_JII 0x00000001 /* JTAG Serial Inbound Int. Enable */ -#endif /* 403GCX */ - /* Some 476 specific registers */ #define SPRN_SSPCR 830 #define SPRN_USPCR 831 @@ -710,5 +560,32 @@ #define MMUBE1_VBE4 0x00000002 #define MMUBE1_VBE5 0x00000001 +#define TMRN_TMCFG0 16 /* Thread Management Configuration Register 0 */ +#define TMRN_TMCFG0_NPRIBITS 0x003f0000 /* Bits of thread priority */ +#define TMRN_TMCFG0_NPRIBITS_SHIFT 16 +#define TMRN_TMCFG0_NATHRD 0x00003f00 /* Number of active threads */ +#define TMRN_TMCFG0_NATHRD_SHIFT 8 +#define TMRN_TMCFG0_NTHRD 0x0000003f /* Number of threads */ +#define TMRN_IMSR0 0x120 /* Initial MSR Register 0 (e6500) */ +#define TMRN_IMSR1 0x121 /* Initial MSR Register 1 (e6500) */ +#define TMRN_INIA0 0x140 /* Next Instruction Address Register 0 */ +#define TMRN_INIA1 0x141 /* Next Instruction Address Register 1 */ +#define SPRN_TENSR 0x1b5 /* Thread Enable Status Register */ +#define SPRN_TENS 0x1b6 /* Thread Enable Set Register */ +#define SPRN_TENC 0x1b7 /* Thread Enable Clear Register */ + +#define TEN_THREAD(x) (1 << (x)) + +#ifndef __ASSEMBLER__ +#define mftmr(rn) ({unsigned long rval; \ + asm volatile(MFTMR(rn, %0) : "=r" (rval)); rval;}) +#define mttmr(rn, v) asm volatile(MTTMR(rn, %0) : \ + : "r" ((unsigned long)(v)) \ + : "memory") + +extern unsigned long global_dbcr0[]; + +#endif /* !__ASSEMBLER__ */ + #endif /* __ASM_POWERPC_REG_BOOKE_H__ */ #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/reg_fsl_emb.h b/arch/powerpc/include/asm/reg_fsl_emb.h index 77bb71cfd991..ec459c3d9498 100644 --- a/arch/powerpc/include/asm/reg_fsl_emb.h +++ b/arch/powerpc/include/asm/reg_fsl_emb.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Contains register definitions for the Freescale Embedded Performance * Monitor. @@ -6,23 +7,46 @@ #ifndef __ASM_POWERPC_REG_FSL_EMB_H__ #define __ASM_POWERPC_REG_FSL_EMB_H__ -#ifndef __ASSEMBLY__ +#include <linux/stringify.h> + +#ifndef __ASSEMBLER__ /* Performance Monitor Registers */ -#define mfpmr(rn) ({unsigned int rval; \ - asm volatile("mfpmr %0," __stringify(rn) \ - : "=r" (rval)); rval;}) -#define mtpmr(rn, v) asm volatile("mtpmr " __stringify(rn) ",%0" : : "r" (v)) -#endif /* __ASSEMBLY__ */ +static __always_inline unsigned int mfpmr(unsigned int rn) +{ + unsigned int rval; + + asm (".machine push; " + ".machine e300; " + "mfpmr %[rval], %[rn];" + ".machine pop;" + : [rval] "=r" (rval) : [rn] "i" (rn)); + + return rval; +} + +static __always_inline void mtpmr(unsigned int rn, unsigned int val) +{ + asm (".machine push; " + ".machine e300; " + "mtpmr %[rn], %[val];" + ".machine pop;" + : [val] "=r" (val) : [rn] "i" (rn)); +} +#endif /* __ASSEMBLER__ */ /* Freescale Book E Performance Monitor APU Registers */ #define PMRN_PMC0 0x010 /* Performance Monitor Counter 0 */ #define PMRN_PMC1 0x011 /* Performance Monitor Counter 1 */ -#define PMRN_PMC2 0x012 /* Performance Monitor Counter 1 */ -#define PMRN_PMC3 0x013 /* Performance Monitor Counter 1 */ +#define PMRN_PMC2 0x012 /* Performance Monitor Counter 2 */ +#define PMRN_PMC3 0x013 /* Performance Monitor Counter 3 */ +#define PMRN_PMC4 0x014 /* Performance Monitor Counter 4 */ +#define PMRN_PMC5 0x015 /* Performance Monitor Counter 5 */ #define PMRN_PMLCA0 0x090 /* PM Local Control A0 */ #define PMRN_PMLCA1 0x091 /* PM Local Control A1 */ #define PMRN_PMLCA2 0x092 /* PM Local Control A2 */ #define PMRN_PMLCA3 0x093 /* PM Local Control A3 */ +#define PMRN_PMLCA4 0x094 /* PM Local Control A4 */ +#define PMRN_PMLCA5 0x095 /* PM Local Control A5 */ #define PMLCA_FC 0x80000000 /* Freeze Counter */ #define PMLCA_FCS 0x40000000 /* Freeze in Supervisor */ @@ -30,14 +54,18 @@ #define PMLCA_FCM1 0x10000000 /* Freeze when PMM==1 */ #define PMLCA_FCM0 0x08000000 /* Freeze when PMM==0 */ #define PMLCA_CE 0x04000000 /* Condition Enable */ +#define PMLCA_FGCS1 0x00000002 /* Freeze in guest state */ +#define PMLCA_FGCS0 0x00000001 /* Freeze in hypervisor state */ -#define PMLCA_EVENT_MASK 0x00ff0000 /* Event field */ +#define PMLCA_EVENT_MASK 0x01ff0000 /* Event field */ #define PMLCA_EVENT_SHIFT 16 #define PMRN_PMLCB0 0x110 /* PM Local Control B0 */ #define PMRN_PMLCB1 0x111 /* PM Local Control B1 */ #define PMRN_PMLCB2 0x112 /* PM Local Control B2 */ #define PMRN_PMLCB3 0x113 /* PM Local Control B3 */ +#define PMRN_PMLCB4 0x114 /* PM Local Control B4 */ +#define PMRN_PMLCB5 0x115 /* PM Local Control B5 */ #define PMLCB_THRESHMUL_MASK 0x0700 /* Threshold Multiple Field */ #define PMLCB_THRESHMUL_SHIFT 8 @@ -55,16 +83,22 @@ #define PMRN_UPMC0 0x000 /* User Performance Monitor Counter 0 */ #define PMRN_UPMC1 0x001 /* User Performance Monitor Counter 1 */ -#define PMRN_UPMC2 0x002 /* User Performance Monitor Counter 1 */ -#define PMRN_UPMC3 0x003 /* User Performance Monitor Counter 1 */ +#define PMRN_UPMC2 0x002 /* User Performance Monitor Counter 2 */ +#define PMRN_UPMC3 0x003 /* User Performance Monitor Counter 3 */ +#define PMRN_UPMC4 0x004 /* User Performance Monitor Counter 4 */ +#define PMRN_UPMC5 0x005 /* User Performance Monitor Counter 5 */ #define PMRN_UPMLCA0 0x080 /* User PM Local Control A0 */ #define PMRN_UPMLCA1 0x081 /* User PM Local Control A1 */ #define PMRN_UPMLCA2 0x082 /* User PM Local Control A2 */ #define PMRN_UPMLCA3 0x083 /* User PM Local Control A3 */ +#define PMRN_UPMLCA4 0x084 /* User PM Local Control A4 */ +#define PMRN_UPMLCA5 0x085 /* User PM Local Control A5 */ #define PMRN_UPMLCB0 0x100 /* User PM Local Control B0 */ #define PMRN_UPMLCB1 0x101 /* User PM Local Control B1 */ #define PMRN_UPMLCB2 0x102 /* User PM Local Control B2 */ #define PMRN_UPMLCB3 0x103 /* User PM Local Control B3 */ +#define PMRN_UPMLCB4 0x104 /* User PM Local Control B4 */ +#define PMRN_UPMLCB5 0x105 /* User PM Local Control B5 */ #define PMRN_UPMGC0 0x180 /* User PM Global Control 0 */ diff --git a/arch/powerpc/include/asm/rheap.h b/arch/powerpc/include/asm/rheap.h index 172381769cfc..8e83703d6736 100644 --- a/arch/powerpc/include/asm/rheap.h +++ b/arch/powerpc/include/asm/rheap.h @@ -83,6 +83,9 @@ extern int rh_get_stats(rh_info_t * info, int what, int max_stats, /* Simple dump of remote heap info */ extern void rh_dump(rh_info_t * info); +/* Simple dump of remote info block */ +void rh_dump_blk(rh_info_t *info, rh_block_t *blk); + /* Set owner of taken block */ extern int rh_set_owner(rh_info_t * info, unsigned long start, const char *owner); diff --git a/arch/powerpc/include/asm/rio.h b/arch/powerpc/include/asm/rio.h index b1d2deceeedb..0e57cda2a64c 100644 --- a/arch/powerpc/include/asm/rio.h +++ b/arch/powerpc/include/asm/rio.h @@ -1,19 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * RapidIO architecture support * * Copyright 2005 MontaVista Software, Inc. * Matt Porter <mporter@kernel.crashing.org> - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. */ #ifndef ASM_PPC_RIO_H #define ASM_PPC_RIO_H -extern void platform_rio_init(void); #ifdef CONFIG_FSL_RIO extern int fsl_rio_mcheck_exception(struct pt_regs *); #else diff --git a/arch/powerpc/include/asm/rtas-types.h b/arch/powerpc/include/asm/rtas-types.h new file mode 100644 index 000000000000..9d5b16803cbb --- /dev/null +++ b/arch/powerpc/include/asm/rtas-types.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_POWERPC_RTAS_TYPES_H +#define _ASM_POWERPC_RTAS_TYPES_H + +#include <linux/compiler_attributes.h> + +typedef __be32 rtas_arg_t; + +struct rtas_args { + __be32 token; + __be32 nargs; + __be32 nret; + rtas_arg_t args[16]; + rtas_arg_t *rets; /* Pointer to return values in args[]. */ +} __aligned(8); + +struct rtas_t { + unsigned long entry; /* physical address pointer */ + unsigned long base; /* physical address pointer */ + unsigned long size; + struct device_node *dev; /* virtual address pointer */ +}; + +struct rtas_error_log { + /* Byte 0 */ + u8 byte0; /* Architectural version */ + + /* Byte 1 */ + u8 byte1; + /* XXXXXXXX + * XXX 3: Severity level of error + * XX 2: Degree of recovery + * X 1: Extended log present? + * XX 2: Reserved + */ + + /* Byte 2 */ + u8 byte2; + /* XXXXXXXX + * XXXX 4: Initiator of event + * XXXX 4: Target of failed operation + */ + u8 byte3; /* General event or error*/ + __be32 extended_log_length; /* length in bytes */ + unsigned char buffer[1]; /* Start of extended log */ + /* Variable length. */ +}; + +/* RTAS general extended event log, Version 6. The extended log starts + * from "buffer" field of struct rtas_error_log defined above. + */ +struct rtas_ext_event_log_v6 { + /* Byte 0 */ + u8 byte0; + /* XXXXXXXX + * X 1: Log valid + * X 1: Unrecoverable error + * X 1: Recoverable (correctable or successfully retried) + * X 1: Bypassed unrecoverable error (degraded operation) + * X 1: Predictive error + * X 1: "New" log (always 1 for data returned from RTAS) + * X 1: Big Endian + * X 1: Reserved + */ + + /* Byte 1 */ + u8 byte1; /* reserved */ + + /* Byte 2 */ + u8 byte2; + /* XXXXXXXX + * X 1: Set to 1 (indicating log is in PowerPC format) + * XXX 3: Reserved + * XXXX 4: Log format used for bytes 12-2047 + */ + + /* Byte 3 */ + u8 byte3; /* reserved */ + /* Byte 4-11 */ + u8 reserved[8]; /* reserved */ + /* Byte 12-15 */ + __be32 company_id; /* Company ID of the company */ + /* that defines the format for */ + /* the vendor specific log type */ + /* Byte 16-end of log */ + u8 vendor_log[1]; /* Start of vendor specific log */ + /* Variable length. */ +}; + +/* Vendor specific Platform Event Log Format, Version 6, section header */ +struct pseries_errorlog { + __be16 id; /* 0x00 2-byte ASCII section ID */ + __be16 length; /* 0x02 Section length in bytes */ + u8 version; /* 0x04 Section version */ + u8 subtype; /* 0x05 Section subtype */ + __be16 creator_component; /* 0x06 Creator component ID */ + u8 data[]; /* 0x08 Start of section data */ +}; + +/* RTAS pseries hotplug errorlog section */ +struct pseries_hp_errorlog { + u8 resource; + u8 action; + u8 id_type; + u8 reserved; + union { + __be32 drc_index; + __be32 drc_count; + struct { __be32 count, index; } ic; + char drc_name[1]; + } _drc_u; +}; + +#endif /* _ASM_POWERPC_RTAS_TYPES_H */ diff --git a/arch/powerpc/include/asm/rtas-work-area.h b/arch/powerpc/include/asm/rtas-work-area.h new file mode 100644 index 000000000000..251a395dbd2e --- /dev/null +++ b/arch/powerpc/include/asm/rtas-work-area.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _ASM_POWERPC_RTAS_WORK_AREA_H +#define _ASM_POWERPC_RTAS_WORK_AREA_H + +#include <linux/build_bug.h> +#include <linux/sizes.h> +#include <linux/types.h> + +#include <asm/page.h> + +/** + * struct rtas_work_area - RTAS work area descriptor. + * + * Descriptor for a "work area" in PAPR terminology that satisfies + * RTAS addressing requirements. + */ +struct rtas_work_area { + /* private: Use the APIs provided below. */ + char *buf; + size_t size; +}; + +enum { + /* Maximum allocation size, enforced at build time. */ + RTAS_WORK_AREA_MAX_ALLOC_SZ = SZ_128K, +}; + +/** + * rtas_work_area_alloc() - Acquire a work area of the requested size. + * @size_: Allocation size. Must be compile-time constant and not more + * than %RTAS_WORK_AREA_MAX_ALLOC_SZ. + * + * Allocate a buffer suitable for passing to RTAS functions that have + * a memory address parameter, often (but not always) referred to as a + * "work area" in PAPR. Although callers are allowed to block while + * holding a work area, the amount of memory reserved for this purpose + * is limited, and allocations should be short-lived. A good guideline + * is to release any allocated work area before returning from a + * system call. + * + * This function does not fail. It blocks until the allocation + * succeeds. To prevent deadlocks, callers are discouraged from + * allocating more than one work area simultaneously in a single task + * context. + * + * Context: This function may sleep. + * Return: A &struct rtas_work_area descriptor for the allocated work area. + */ +#define rtas_work_area_alloc(size_) ({ \ + static_assert(__builtin_constant_p(size_)); \ + static_assert((size_) > 0); \ + static_assert((size_) <= RTAS_WORK_AREA_MAX_ALLOC_SZ); \ + __rtas_work_area_alloc(size_); \ +}) + +/* + * Do not call __rtas_work_area_alloc() directly. Use + * rtas_work_area_alloc(). + */ +struct rtas_work_area *__rtas_work_area_alloc(size_t size); + +/** + * rtas_work_area_free() - Release a work area. + * @area: Work area descriptor as returned from rtas_work_area_alloc(). + * + * Return a work area buffer to the pool. + */ +void rtas_work_area_free(struct rtas_work_area *area); + +static inline char *rtas_work_area_raw_buf(const struct rtas_work_area *area) +{ + return area->buf; +} + +static inline size_t rtas_work_area_size(const struct rtas_work_area *area) +{ + return area->size; +} + +static inline phys_addr_t rtas_work_area_phys(const struct rtas_work_area *area) +{ + return __pa(area->buf); +} + +/* + * Early setup for the work area allocator. Call from + * rtas_initialize() only. + */ + +#ifdef CONFIG_PPC_PSERIES +void rtas_work_area_reserve_arena(phys_addr_t limit); +#else /* CONFIG_PPC_PSERIES */ +static inline void rtas_work_area_reserve_arena(phys_addr_t limit) {} +#endif /* CONFIG_PPC_PSERIES */ + +#endif /* _ASM_POWERPC_RTAS_WORK_AREA_H */ diff --git a/arch/powerpc/include/asm/rtas.h b/arch/powerpc/include/asm/rtas.h index c7a8bfc9f6f5..d046bbd5017d 100644 --- a/arch/powerpc/include/asm/rtas.h +++ b/arch/powerpc/include/asm/rtas.h @@ -1,81 +1,244 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _POWERPC_RTAS_H #define _POWERPC_RTAS_H #ifdef __KERNEL__ +#include <linux/mutex.h> #include <linux/spinlock.h> #include <asm/page.h> +#include <asm/rtas-types.h> +#include <linux/time.h> +#include <linux/cpumask.h> /* * Definitions for talking to the RTAS on CHRP machines. * * Copyright (C) 2001 Peter Bergner * Copyright (C) 2001 PPC 64 Team, IBM Corp - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ +enum rtas_function_index { + RTAS_FNIDX__CHECK_EXCEPTION, + RTAS_FNIDX__DISPLAY_CHARACTER, + RTAS_FNIDX__EVENT_SCAN, + RTAS_FNIDX__FREEZE_TIME_BASE, + RTAS_FNIDX__GET_POWER_LEVEL, + RTAS_FNIDX__GET_SENSOR_STATE, + RTAS_FNIDX__GET_TERM_CHAR, + RTAS_FNIDX__GET_TIME_OF_DAY, + RTAS_FNIDX__IBM_ACTIVATE_FIRMWARE, + RTAS_FNIDX__IBM_CBE_START_PTCAL, + RTAS_FNIDX__IBM_CBE_STOP_PTCAL, + RTAS_FNIDX__IBM_CHANGE_MSI, + RTAS_FNIDX__IBM_CLOSE_ERRINJCT, + RTAS_FNIDX__IBM_CONFIGURE_BRIDGE, + RTAS_FNIDX__IBM_CONFIGURE_CONNECTOR, + RTAS_FNIDX__IBM_CONFIGURE_KERNEL_DUMP, + RTAS_FNIDX__IBM_CONFIGURE_PE, + RTAS_FNIDX__IBM_CREATE_PE_DMA_WINDOW, + RTAS_FNIDX__IBM_DISPLAY_MESSAGE, + RTAS_FNIDX__IBM_ERRINJCT, + RTAS_FNIDX__IBM_EXTI2C, + RTAS_FNIDX__IBM_GET_CONFIG_ADDR_INFO, + RTAS_FNIDX__IBM_GET_CONFIG_ADDR_INFO2, + RTAS_FNIDX__IBM_GET_DYNAMIC_SENSOR_STATE, + RTAS_FNIDX__IBM_GET_INDICES, + RTAS_FNIDX__IBM_GET_RIO_TOPOLOGY, + RTAS_FNIDX__IBM_GET_SYSTEM_PARAMETER, + RTAS_FNIDX__IBM_GET_VPD, + RTAS_FNIDX__IBM_GET_XIVE, + RTAS_FNIDX__IBM_INT_OFF, + RTAS_FNIDX__IBM_INT_ON, + RTAS_FNIDX__IBM_IO_QUIESCE_ACK, + RTAS_FNIDX__IBM_LPAR_PERFTOOLS, + RTAS_FNIDX__IBM_MANAGE_FLASH_IMAGE, + RTAS_FNIDX__IBM_MANAGE_STORAGE_PRESERVATION, + RTAS_FNIDX__IBM_NMI_INTERLOCK, + RTAS_FNIDX__IBM_NMI_REGISTER, + RTAS_FNIDX__IBM_OPEN_ERRINJCT, + RTAS_FNIDX__IBM_OPEN_SRIOV_ALLOW_UNFREEZE, + RTAS_FNIDX__IBM_OPEN_SRIOV_MAP_PE_NUMBER, + RTAS_FNIDX__IBM_OS_TERM, + RTAS_FNIDX__IBM_PARTNER_CONTROL, + RTAS_FNIDX__IBM_PHYSICAL_ATTESTATION, + RTAS_FNIDX__IBM_PLATFORM_DUMP, + RTAS_FNIDX__IBM_POWER_OFF_UPS, + RTAS_FNIDX__IBM_QUERY_INTERRUPT_SOURCE_NUMBER, + RTAS_FNIDX__IBM_QUERY_PE_DMA_WINDOW, + RTAS_FNIDX__IBM_READ_PCI_CONFIG, + RTAS_FNIDX__IBM_READ_SLOT_RESET_STATE, + RTAS_FNIDX__IBM_READ_SLOT_RESET_STATE2, + RTAS_FNIDX__IBM_RECEIVE_HVPIPE_MSG, + RTAS_FNIDX__IBM_REMOVE_PE_DMA_WINDOW, + RTAS_FNIDX__IBM_RESET_PE_DMA_WINDOW, + RTAS_FNIDX__IBM_SCAN_LOG_DUMP, + RTAS_FNIDX__IBM_SEND_HVPIPE_MSG, + RTAS_FNIDX__IBM_SET_DYNAMIC_INDICATOR, + RTAS_FNIDX__IBM_SET_EEH_OPTION, + RTAS_FNIDX__IBM_SET_SLOT_RESET, + RTAS_FNIDX__IBM_SET_SYSTEM_PARAMETER, + RTAS_FNIDX__IBM_SET_XIVE, + RTAS_FNIDX__IBM_SLOT_ERROR_DETAIL, + RTAS_FNIDX__IBM_SUSPEND_ME, + RTAS_FNIDX__IBM_TUNE_DMA_PARMS, + RTAS_FNIDX__IBM_UPDATE_FLASH_64_AND_REBOOT, + RTAS_FNIDX__IBM_UPDATE_NODES, + RTAS_FNIDX__IBM_UPDATE_PROPERTIES, + RTAS_FNIDX__IBM_VALIDATE_FLASH_IMAGE, + RTAS_FNIDX__IBM_WRITE_PCI_CONFIG, + RTAS_FNIDX__NVRAM_FETCH, + RTAS_FNIDX__NVRAM_STORE, + RTAS_FNIDX__POWER_OFF, + RTAS_FNIDX__PUT_TERM_CHAR, + RTAS_FNIDX__QUERY_CPU_STOPPED_STATE, + RTAS_FNIDX__READ_PCI_CONFIG, + RTAS_FNIDX__RTAS_LAST_ERROR, + RTAS_FNIDX__SET_INDICATOR, + RTAS_FNIDX__SET_POWER_LEVEL, + RTAS_FNIDX__SET_TIME_FOR_POWER_ON, + RTAS_FNIDX__SET_TIME_OF_DAY, + RTAS_FNIDX__START_CPU, + RTAS_FNIDX__STOP_SELF, + RTAS_FNIDX__SYSTEM_REBOOT, + RTAS_FNIDX__THAW_TIME_BASE, + RTAS_FNIDX__WRITE_PCI_CONFIG, +}; + +/* + * Opaque handle for client code to refer to RTAS functions. All valid + * function handles are build-time constants prefixed with RTAS_FN_. + */ +typedef struct { + const enum rtas_function_index index; +} rtas_fn_handle_t; + + +#define rtas_fn_handle(x_) ((const rtas_fn_handle_t) { .index = x_, }) + +#define RTAS_FN_CHECK_EXCEPTION rtas_fn_handle(RTAS_FNIDX__CHECK_EXCEPTION) +#define RTAS_FN_DISPLAY_CHARACTER rtas_fn_handle(RTAS_FNIDX__DISPLAY_CHARACTER) +#define RTAS_FN_EVENT_SCAN rtas_fn_handle(RTAS_FNIDX__EVENT_SCAN) +#define RTAS_FN_FREEZE_TIME_BASE rtas_fn_handle(RTAS_FNIDX__FREEZE_TIME_BASE) +#define RTAS_FN_GET_POWER_LEVEL rtas_fn_handle(RTAS_FNIDX__GET_POWER_LEVEL) +#define RTAS_FN_GET_SENSOR_STATE rtas_fn_handle(RTAS_FNIDX__GET_SENSOR_STATE) +#define RTAS_FN_GET_TERM_CHAR rtas_fn_handle(RTAS_FNIDX__GET_TERM_CHAR) +#define RTAS_FN_GET_TIME_OF_DAY rtas_fn_handle(RTAS_FNIDX__GET_TIME_OF_DAY) +#define RTAS_FN_IBM_ACTIVATE_FIRMWARE rtas_fn_handle(RTAS_FNIDX__IBM_ACTIVATE_FIRMWARE) +#define RTAS_FN_IBM_CBE_START_PTCAL rtas_fn_handle(RTAS_FNIDX__IBM_CBE_START_PTCAL) +#define RTAS_FN_IBM_CBE_STOP_PTCAL rtas_fn_handle(RTAS_FNIDX__IBM_CBE_STOP_PTCAL) +#define RTAS_FN_IBM_CHANGE_MSI rtas_fn_handle(RTAS_FNIDX__IBM_CHANGE_MSI) +#define RTAS_FN_IBM_CLOSE_ERRINJCT rtas_fn_handle(RTAS_FNIDX__IBM_CLOSE_ERRINJCT) +#define RTAS_FN_IBM_CONFIGURE_BRIDGE rtas_fn_handle(RTAS_FNIDX__IBM_CONFIGURE_BRIDGE) +#define RTAS_FN_IBM_CONFIGURE_CONNECTOR rtas_fn_handle(RTAS_FNIDX__IBM_CONFIGURE_CONNECTOR) +#define RTAS_FN_IBM_CONFIGURE_KERNEL_DUMP rtas_fn_handle(RTAS_FNIDX__IBM_CONFIGURE_KERNEL_DUMP) +#define RTAS_FN_IBM_CONFIGURE_PE rtas_fn_handle(RTAS_FNIDX__IBM_CONFIGURE_PE) +#define RTAS_FN_IBM_CREATE_PE_DMA_WINDOW rtas_fn_handle(RTAS_FNIDX__IBM_CREATE_PE_DMA_WINDOW) +#define RTAS_FN_IBM_DISPLAY_MESSAGE rtas_fn_handle(RTAS_FNIDX__IBM_DISPLAY_MESSAGE) +#define RTAS_FN_IBM_ERRINJCT rtas_fn_handle(RTAS_FNIDX__IBM_ERRINJCT) +#define RTAS_FN_IBM_EXTI2C rtas_fn_handle(RTAS_FNIDX__IBM_EXTI2C) +#define RTAS_FN_IBM_GET_CONFIG_ADDR_INFO rtas_fn_handle(RTAS_FNIDX__IBM_GET_CONFIG_ADDR_INFO) +#define RTAS_FN_IBM_GET_CONFIG_ADDR_INFO2 rtas_fn_handle(RTAS_FNIDX__IBM_GET_CONFIG_ADDR_INFO2) +#define RTAS_FN_IBM_GET_DYNAMIC_SENSOR_STATE rtas_fn_handle(RTAS_FNIDX__IBM_GET_DYNAMIC_SENSOR_STATE) +#define RTAS_FN_IBM_GET_INDICES rtas_fn_handle(RTAS_FNIDX__IBM_GET_INDICES) +#define RTAS_FN_IBM_GET_RIO_TOPOLOGY rtas_fn_handle(RTAS_FNIDX__IBM_GET_RIO_TOPOLOGY) +#define RTAS_FN_IBM_GET_SYSTEM_PARAMETER rtas_fn_handle(RTAS_FNIDX__IBM_GET_SYSTEM_PARAMETER) +#define RTAS_FN_IBM_GET_VPD rtas_fn_handle(RTAS_FNIDX__IBM_GET_VPD) +#define RTAS_FN_IBM_GET_XIVE rtas_fn_handle(RTAS_FNIDX__IBM_GET_XIVE) +#define RTAS_FN_IBM_INT_OFF rtas_fn_handle(RTAS_FNIDX__IBM_INT_OFF) +#define RTAS_FN_IBM_INT_ON rtas_fn_handle(RTAS_FNIDX__IBM_INT_ON) +#define RTAS_FN_IBM_IO_QUIESCE_ACK rtas_fn_handle(RTAS_FNIDX__IBM_IO_QUIESCE_ACK) +#define RTAS_FN_IBM_LPAR_PERFTOOLS rtas_fn_handle(RTAS_FNIDX__IBM_LPAR_PERFTOOLS) +#define RTAS_FN_IBM_MANAGE_FLASH_IMAGE rtas_fn_handle(RTAS_FNIDX__IBM_MANAGE_FLASH_IMAGE) +#define RTAS_FN_IBM_MANAGE_STORAGE_PRESERVATION rtas_fn_handle(RTAS_FNIDX__IBM_MANAGE_STORAGE_PRESERVATION) +#define RTAS_FN_IBM_NMI_INTERLOCK rtas_fn_handle(RTAS_FNIDX__IBM_NMI_INTERLOCK) +#define RTAS_FN_IBM_NMI_REGISTER rtas_fn_handle(RTAS_FNIDX__IBM_NMI_REGISTER) +#define RTAS_FN_IBM_OPEN_ERRINJCT rtas_fn_handle(RTAS_FNIDX__IBM_OPEN_ERRINJCT) +#define RTAS_FN_IBM_OPEN_SRIOV_ALLOW_UNFREEZE rtas_fn_handle(RTAS_FNIDX__IBM_OPEN_SRIOV_ALLOW_UNFREEZE) +#define RTAS_FN_IBM_OPEN_SRIOV_MAP_PE_NUMBER rtas_fn_handle(RTAS_FNIDX__IBM_OPEN_SRIOV_MAP_PE_NUMBER) +#define RTAS_FN_IBM_OS_TERM rtas_fn_handle(RTAS_FNIDX__IBM_OS_TERM) +#define RTAS_FN_IBM_PARTNER_CONTROL rtas_fn_handle(RTAS_FNIDX__IBM_PARTNER_CONTROL) +#define RTAS_FN_IBM_PHYSICAL_ATTESTATION rtas_fn_handle(RTAS_FNIDX__IBM_PHYSICAL_ATTESTATION) +#define RTAS_FN_IBM_PLATFORM_DUMP rtas_fn_handle(RTAS_FNIDX__IBM_PLATFORM_DUMP) +#define RTAS_FN_IBM_POWER_OFF_UPS rtas_fn_handle(RTAS_FNIDX__IBM_POWER_OFF_UPS) +#define RTAS_FN_IBM_QUERY_INTERRUPT_SOURCE_NUMBER rtas_fn_handle(RTAS_FNIDX__IBM_QUERY_INTERRUPT_SOURCE_NUMBER) +#define RTAS_FN_IBM_QUERY_PE_DMA_WINDOW rtas_fn_handle(RTAS_FNIDX__IBM_QUERY_PE_DMA_WINDOW) +#define RTAS_FN_IBM_READ_PCI_CONFIG rtas_fn_handle(RTAS_FNIDX__IBM_READ_PCI_CONFIG) +#define RTAS_FN_IBM_READ_SLOT_RESET_STATE rtas_fn_handle(RTAS_FNIDX__IBM_READ_SLOT_RESET_STATE) +#define RTAS_FN_IBM_READ_SLOT_RESET_STATE2 rtas_fn_handle(RTAS_FNIDX__IBM_READ_SLOT_RESET_STATE2) +#define RTAS_FN_IBM_RECEIVE_HVPIPE_MSG rtas_fn_handle(RTAS_FNIDX__IBM_RECEIVE_HVPIPE_MSG) +#define RTAS_FN_IBM_REMOVE_PE_DMA_WINDOW rtas_fn_handle(RTAS_FNIDX__IBM_REMOVE_PE_DMA_WINDOW) +#define RTAS_FN_IBM_RESET_PE_DMA_WINDOW rtas_fn_handle(RTAS_FNIDX__IBM_RESET_PE_DMA_WINDOW) +#define RTAS_FN_IBM_SCAN_LOG_DUMP rtas_fn_handle(RTAS_FNIDX__IBM_SCAN_LOG_DUMP) +#define RTAS_FN_IBM_SEND_HVPIPE_MSG rtas_fn_handle(RTAS_FNIDX__IBM_SEND_HVPIPE_MSG) +#define RTAS_FN_IBM_SET_DYNAMIC_INDICATOR rtas_fn_handle(RTAS_FNIDX__IBM_SET_DYNAMIC_INDICATOR) +#define RTAS_FN_IBM_SET_EEH_OPTION rtas_fn_handle(RTAS_FNIDX__IBM_SET_EEH_OPTION) +#define RTAS_FN_IBM_SET_SLOT_RESET rtas_fn_handle(RTAS_FNIDX__IBM_SET_SLOT_RESET) +#define RTAS_FN_IBM_SET_SYSTEM_PARAMETER rtas_fn_handle(RTAS_FNIDX__IBM_SET_SYSTEM_PARAMETER) +#define RTAS_FN_IBM_SET_XIVE rtas_fn_handle(RTAS_FNIDX__IBM_SET_XIVE) +#define RTAS_FN_IBM_SLOT_ERROR_DETAIL rtas_fn_handle(RTAS_FNIDX__IBM_SLOT_ERROR_DETAIL) +#define RTAS_FN_IBM_SUSPEND_ME rtas_fn_handle(RTAS_FNIDX__IBM_SUSPEND_ME) +#define RTAS_FN_IBM_TUNE_DMA_PARMS rtas_fn_handle(RTAS_FNIDX__IBM_TUNE_DMA_PARMS) +#define RTAS_FN_IBM_UPDATE_FLASH_64_AND_REBOOT rtas_fn_handle(RTAS_FNIDX__IBM_UPDATE_FLASH_64_AND_REBOOT) +#define RTAS_FN_IBM_UPDATE_NODES rtas_fn_handle(RTAS_FNIDX__IBM_UPDATE_NODES) +#define RTAS_FN_IBM_UPDATE_PROPERTIES rtas_fn_handle(RTAS_FNIDX__IBM_UPDATE_PROPERTIES) +#define RTAS_FN_IBM_VALIDATE_FLASH_IMAGE rtas_fn_handle(RTAS_FNIDX__IBM_VALIDATE_FLASH_IMAGE) +#define RTAS_FN_IBM_WRITE_PCI_CONFIG rtas_fn_handle(RTAS_FNIDX__IBM_WRITE_PCI_CONFIG) +#define RTAS_FN_NVRAM_FETCH rtas_fn_handle(RTAS_FNIDX__NVRAM_FETCH) +#define RTAS_FN_NVRAM_STORE rtas_fn_handle(RTAS_FNIDX__NVRAM_STORE) +#define RTAS_FN_POWER_OFF rtas_fn_handle(RTAS_FNIDX__POWER_OFF) +#define RTAS_FN_PUT_TERM_CHAR rtas_fn_handle(RTAS_FNIDX__PUT_TERM_CHAR) +#define RTAS_FN_QUERY_CPU_STOPPED_STATE rtas_fn_handle(RTAS_FNIDX__QUERY_CPU_STOPPED_STATE) +#define RTAS_FN_READ_PCI_CONFIG rtas_fn_handle(RTAS_FNIDX__READ_PCI_CONFIG) +#define RTAS_FN_RTAS_LAST_ERROR rtas_fn_handle(RTAS_FNIDX__RTAS_LAST_ERROR) +#define RTAS_FN_SET_INDICATOR rtas_fn_handle(RTAS_FNIDX__SET_INDICATOR) +#define RTAS_FN_SET_POWER_LEVEL rtas_fn_handle(RTAS_FNIDX__SET_POWER_LEVEL) +#define RTAS_FN_SET_TIME_FOR_POWER_ON rtas_fn_handle(RTAS_FNIDX__SET_TIME_FOR_POWER_ON) +#define RTAS_FN_SET_TIME_OF_DAY rtas_fn_handle(RTAS_FNIDX__SET_TIME_OF_DAY) +#define RTAS_FN_START_CPU rtas_fn_handle(RTAS_FNIDX__START_CPU) +#define RTAS_FN_STOP_SELF rtas_fn_handle(RTAS_FNIDX__STOP_SELF) +#define RTAS_FN_SYSTEM_REBOOT rtas_fn_handle(RTAS_FNIDX__SYSTEM_REBOOT) +#define RTAS_FN_THAW_TIME_BASE rtas_fn_handle(RTAS_FNIDX__THAW_TIME_BASE) +#define RTAS_FN_WRITE_PCI_CONFIG rtas_fn_handle(RTAS_FNIDX__WRITE_PCI_CONFIG) + #define RTAS_UNKNOWN_SERVICE (-1) #define RTAS_INSTANTIATE_MAX (1ULL<<30) /* Don't instantiate rtas at/above this value */ -/* Buffer size for ppc_rtas system call. */ -#define RTAS_RMOBUF_MAX (64 * 1024) - -/* RTAS return status codes */ -#define RTAS_NOT_SUSPENDABLE -9004 -#define RTAS_BUSY -2 /* RTAS Busy */ -#define RTAS_EXTENDED_DELAY_MIN 9900 -#define RTAS_EXTENDED_DELAY_MAX 9905 +/* Memory set aside for sys_rtas to use with calls that need a work area. */ +#define RTAS_USER_REGION_SIZE (64 * 1024) /* - * In general to call RTAS use rtas_token("string") to lookup - * an RTAS token for the given string (e.g. "event-scan"). - * To actually perform the call use - * ret = rtas_call(token, n_in, n_out, ...) - * Where n_in is the number of input parameters and - * n_out is the number of output parameters - * - * If the "string" is invalid on this system, RTAS_UNKNOWN_SERVICE - * will be returned as a token. rtas_call() does look for this - * token and error out gracefully so rtas_call(rtas_token("str"), ...) - * may be safely used for one-shot calls to RTAS. + * Common RTAS function return values, derived from the table "RTAS + * Status Word Values" in PAPR+ v2.13 7.2.8: "Return Codes". If a + * function can return a value in this table then generally it has the + * meaning listed here. More extended commentary in the documentation + * for rtas_call(). * + * RTAS functions may use negative and positive numbers not in this + * set for function-specific error and success conditions, + * respectively. */ - -typedef u32 rtas_arg_t; - -struct rtas_args { - u32 token; - u32 nargs; - u32 nret; - rtas_arg_t args[16]; - rtas_arg_t *rets; /* Pointer to return values in args[]. */ -}; - -struct rtas_t { - unsigned long entry; /* physical address pointer */ - unsigned long base; /* physical address pointer */ - unsigned long size; - arch_spinlock_t lock; - struct rtas_args args; - struct device_node *dev; /* virtual address pointer */ -}; - -struct rtas_suspend_me_data { - atomic_t working; /* number of cpus accessing this struct */ - atomic_t done; - int token; /* ibm,suspend-me */ - atomic_t error; - struct completion *complete; /* wait on this until working == 0 */ -}; +#define RTAS_SUCCESS 0 /* Success. */ +#define RTAS_HARDWARE_ERROR -1 /* Hardware or other unspecified error. */ +#define RTAS_BUSY -2 /* Retry immediately. */ +#define RTAS_INVALID_PARAMETER -3 /* Invalid indicator/domain/sensor etc. */ +#define RTAS_FUNC_NOT_SUPPORTED -5 /* Function not supported */ +#define RTAS_UNEXPECTED_STATE_CHANGE -7 /* Seems limited to EEH and slot reset. */ +#define RTAS_EXTENDED_DELAY_MIN 9900 /* Retry after delaying for ~1ms. */ +#define RTAS_EXTENDED_DELAY_MAX 9905 /* Retry after delaying for ~100s. */ +#define RTAS_ML_ISOLATION_ERROR -9000 /* Multi-level isolation error. */ + +/* statuses specific to ibm,suspend-me */ +#define RTAS_SUSPEND_ABORTED 9000 /* Suspension aborted */ +#define RTAS_NOT_SUSPENDABLE -9004 /* Partition not suspendable */ +#define RTAS_THREADS_ACTIVE -9005 /* Multiple processor threads active */ +#define RTAS_OUTSTANDING_COPROC -9006 /* Outstanding coprocessor operations */ /* RTAS event classes */ #define RTAS_INTERNAL_ERROR 0x80000000 /* set bit 0 */ #define RTAS_EPOW_WARNING 0x40000000 /* set bit 1 */ #define RTAS_HOTPLUG_EVENTS 0x10000000 /* set bit 3 */ #define RTAS_IO_EVENTS 0x08000000 /* set bit 4 */ +#define RTAS_HVPIPE_MSG_EVENTS 0x04000000 /* set bit 5 */ #define RTAS_EVENT_SCAN_ALL_EVENTS 0xffffffff /* RTAS event severity */ @@ -124,7 +287,9 @@ struct rtas_suspend_me_data { #define RTAS_TYPE_INFO 0xE2 #define RTAS_TYPE_DEALLOC 0xE3 #define RTAS_TYPE_DUMP 0xE4 -/* I don't add PowerMGM events right now, this is a different topic */ +#define RTAS_TYPE_HOTPLUG 0xE5 +#define RTAS_TYPE_HVPIPE 0xE6 +/* I don't add PowerMGM events right now, this is a different topic */ #define RTAS_TYPE_PMGM_POWER_SW_ON 0x60 #define RTAS_TYPE_PMGM_POWER_SW_OFF 0x61 #define RTAS_TYPE_PMGM_LID_OPEN 0x62 @@ -149,61 +314,56 @@ struct rtas_suspend_me_data { /* RTAS check-exception vector offset */ #define RTAS_VECTOR_EXTERNAL_INTERRUPT 0x500 -struct rtas_error_log { - unsigned long version:8; /* Architectural version */ - unsigned long severity:3; /* Severity level of error */ - unsigned long disposition:2; /* Degree of recovery */ - unsigned long extended:1; /* extended log present? */ - unsigned long /* reserved */ :2; /* Reserved for future use */ - unsigned long initiator:4; /* Initiator of event */ - unsigned long target:4; /* Target of failed operation */ - unsigned long type:8; /* General event or error*/ - unsigned long extended_log_length:32; /* length in bytes */ - unsigned char buffer[1]; /* Start of extended log */ - /* Variable length. */ -}; +static inline uint8_t rtas_error_severity(const struct rtas_error_log *elog) +{ + return (elog->byte1 & 0xE0) >> 5; +} + +static inline uint8_t rtas_error_disposition(const struct rtas_error_log *elog) +{ + return (elog->byte1 & 0x18) >> 3; +} + +static inline +void rtas_set_disposition_recovered(struct rtas_error_log *elog) +{ + elog->byte1 &= ~0x18; + elog->byte1 |= (RTAS_DISP_FULLY_RECOVERED << 3); +} + +static inline uint8_t rtas_error_extended(const struct rtas_error_log *elog) +{ + return (elog->byte1 & 0x04) >> 2; +} + +static inline uint8_t rtas_error_initiator(const struct rtas_error_log *elog) +{ + return (elog->byte2 & 0xf0) >> 4; +} + +#define rtas_error_type(x) ((x)->byte3) + +static inline +uint32_t rtas_error_extended_log_length(const struct rtas_error_log *elog) +{ + return be32_to_cpu(elog->extended_log_length); +} #define RTAS_V6EXT_LOG_FORMAT_EVENT_LOG 14 #define RTAS_V6EXT_COMPANY_ID_IBM (('I' << 24) | ('B' << 16) | ('M' << 8)) -/* RTAS general extended event log, Version 6. The extended log starts - * from "buffer" field of struct rtas_error_log defined above. - */ -struct rtas_ext_event_log_v6 { - /* Byte 0 */ - uint32_t log_valid:1; /* 1:Log valid */ - uint32_t unrecoverable_error:1; /* 1:Unrecoverable error */ - uint32_t recoverable_error:1; /* 1:recoverable (correctable */ - /* or successfully retried) */ - uint32_t degraded_operation:1; /* 1:Unrecoverable err, bypassed*/ - /* - degraded operation (e.g. */ - /* CPU or mem taken off-line) */ - uint32_t predictive_error:1; - uint32_t new_log:1; /* 1:"New" log (Always 1 for */ - /* data returned from RTAS */ - uint32_t big_endian:1; /* 1: Big endian */ - uint32_t :1; /* reserved */ - /* Byte 1 */ - uint32_t :8; /* reserved */ - /* Byte 2 */ - uint32_t powerpc_format:1; /* Set to 1 (indicating log is */ - /* in PowerPC format */ - uint32_t :3; /* reserved */ - uint32_t log_format:4; /* Log format indicator. Define */ - /* format used for byte 12-2047 */ - /* Byte 3 */ - uint32_t :8; /* reserved */ - /* Byte 4-11 */ - uint8_t reserved[8]; /* reserved */ - /* Byte 12-15 */ - uint32_t company_id; /* Company ID of the company */ - /* that defines the format for */ - /* the vendor specific log type */ - /* Byte 16-end of log */ - uint8_t vendor_log[1]; /* Start of vendor specific log */ - /* Variable length. */ -}; +static +inline uint8_t rtas_ext_event_log_format(struct rtas_ext_event_log_v6 *ext_log) +{ + return ext_log->byte2 & 0x0F; +} + +static +inline uint32_t rtas_ext_event_company_id(struct rtas_ext_event_log_v6 *ext_log) +{ + return be32_to_cpu(ext_log->company_id); +} /* pSeries event log format */ @@ -221,19 +381,39 @@ struct rtas_ext_event_log_v6 { #define PSERIES_ELOG_SECT_ID_HMC_ID (('H' << 8) | 'M') #define PSERIES_ELOG_SECT_ID_EPOW (('E' << 8) | 'P') #define PSERIES_ELOG_SECT_ID_IO_EVENT (('I' << 8) | 'E') +#define PSERIES_ELOG_SECT_ID_HVPIPE_EVENT (('P' << 8) | 'E') #define PSERIES_ELOG_SECT_ID_MANUFACT_INFO (('M' << 8) | 'I') #define PSERIES_ELOG_SECT_ID_CALL_HOME (('C' << 8) | 'H') #define PSERIES_ELOG_SECT_ID_USER_DEF (('U' << 8) | 'D') +#define PSERIES_ELOG_SECT_ID_HOTPLUG (('H' << 8) | 'P') +#define PSERIES_ELOG_SECT_ID_MCE (('M' << 8) | 'C') -/* Vendor specific Platform Event Log Format, Version 6, section header */ -struct pseries_errorlog { - uint16_t id; /* 0x00 2-byte ASCII section ID */ - uint16_t length; /* 0x02 Section length in bytes */ - uint8_t version; /* 0x04 Section version */ - uint8_t subtype; /* 0x05 Section subtype */ - uint16_t creator_component; /* 0x06 Creator component ID */ - uint8_t data[]; /* 0x08 Start of section data */ -}; +static +inline uint16_t pseries_errorlog_id(struct pseries_errorlog *sect) +{ + return be16_to_cpu(sect->id); +} + +static +inline uint16_t pseries_errorlog_length(struct pseries_errorlog *sect) +{ + return be16_to_cpu(sect->length); +} + +#define PSERIES_HP_ELOG_RESOURCE_CPU 1 +#define PSERIES_HP_ELOG_RESOURCE_MEM 2 +#define PSERIES_HP_ELOG_RESOURCE_SLOT 3 +#define PSERIES_HP_ELOG_RESOURCE_PHB 4 +#define PSERIES_HP_ELOG_RESOURCE_PMEM 6 +#define PSERIES_HP_ELOG_RESOURCE_DT 7 + +#define PSERIES_HP_ELOG_ACTION_ADD 1 +#define PSERIES_HP_ELOG_ACTION_REMOVE 2 + +#define PSERIES_HP_ELOG_ID_DRC_NAME 1 +#define PSERIES_HP_ELOG_ID_DRC_INDEX 2 +#define PSERIES_HP_ELOG_ID_DRC_COUNT 3 +#define PSERIES_HP_ELOG_ID_DRC_IC 4 struct pseries_errorlog *get_pseries_errorlog(struct rtas_error_log *log, uint16_t section_id); @@ -246,54 +426,64 @@ extern void (*rtas_flash_term_hook)(int); extern struct rtas_t rtas; -extern void enter_rtas(unsigned long); -extern int rtas_token(const char *service); -extern int rtas_service_present(const char *service); -extern int rtas_call(int token, int, int, int *, ...); -extern void rtas_restart(char *cmd); -extern void rtas_power_off(void); -extern void rtas_halt(void); -extern void rtas_os_term(char *str); -extern int rtas_get_sensor(int sensor, int index, int *state); -extern int rtas_get_power_level(int powerdomain, int *level); -extern int rtas_set_power_level(int powerdomain, int level, int *setlevel); -extern bool rtas_indicator_present(int token, int *maxindex); -extern int rtas_set_indicator(int indicator, int index, int new_value); -extern int rtas_set_indicator_fast(int indicator, int index, int new_value); -extern void rtas_progress(char *s, unsigned short hex); -extern void rtas_initialize(void); -extern int rtas_suspend_cpu(struct rtas_suspend_me_data *data); -extern int rtas_suspend_last_cpu(struct rtas_suspend_me_data *data); -extern int rtas_online_cpus_mask(cpumask_var_t cpus); -extern int rtas_offline_cpus_mask(cpumask_var_t cpus); -extern int rtas_ibm_suspend_me(struct rtas_args *); +s32 rtas_function_token(const rtas_fn_handle_t handle); +static inline bool rtas_function_implemented(const rtas_fn_handle_t handle) +{ + return rtas_function_token(handle) != RTAS_UNKNOWN_SERVICE; +} +int rtas_token(const char *service); +int rtas_call(int token, int nargs, int nret, int *outputs, ...); +void rtas_call_unlocked(struct rtas_args *args, int token, int nargs, + int nret, ...); +void __noreturn rtas_restart(char *cmd); +void rtas_power_off(void); +void __noreturn rtas_halt(void); +void rtas_os_term(char *str); +void rtas_activate_firmware(void); +int rtas_get_sensor(int sensor, int index, int *state); +int rtas_get_sensor_fast(int sensor, int index, int *state); +int rtas_get_power_level(int powerdomain, int *level); +int rtas_set_power_level(int powerdomain, int level, int *setlevel); +bool rtas_indicator_present(int token, int *maxindex); +int rtas_set_indicator(int indicator, int index, int new_value); +int rtas_set_indicator_fast(int indicator, int index, int new_value); +void rtas_progress(char *s, unsigned short hex); +int rtas_ibm_suspend_me(int *fw_status); +int rtas_error_rc(int rtas_rc); struct rtc_time; -extern unsigned long rtas_get_boot_time(void); -extern void rtas_get_rtc_time(struct rtc_time *rtc_time); -extern int rtas_set_rtc_time(struct rtc_time *rtc_time); +time64_t rtas_get_boot_time(void); +void rtas_get_rtc_time(struct rtc_time *rtc_time); +int rtas_set_rtc_time(struct rtc_time *rtc_time); -extern unsigned int rtas_busy_delay_time(int status); -extern unsigned int rtas_busy_delay(int status); +unsigned int rtas_busy_delay_time(int status); +bool rtas_busy_delay(int status); -extern int early_init_dt_scan_rtas(unsigned long node, - const char *uname, int depth, void *data); +int early_init_dt_scan_rtas(unsigned long node, const char *uname, int depth, void *data); -extern void pSeries_log_error(char *buf, unsigned int err_type, int fatal); +void pSeries_log_error(char *buf, unsigned int err_type, int fatal); #ifdef CONFIG_PPC_PSERIES -extern int pseries_devicetree_update(s32 scope); +extern time64_t last_rtas_event; +int clobbering_unread_rtas_event(void); +int rtas_syscall_dispatch_ibm_suspend_me(u64 handle); +#else +static inline int clobbering_unread_rtas_event(void) { return 0; } +static inline int rtas_syscall_dispatch_ibm_suspend_me(u64 handle) +{ + return -EINVAL; +} #endif #ifdef CONFIG_PPC_RTAS_DAEMON -extern void rtas_cancel_event_scan(void); +void rtas_cancel_event_scan(void); #else static inline void rtas_cancel_event_scan(void) { } #endif /* Error types logged. */ #define ERR_FLAG_ALREADY_LOGGED 0x0 -#define ERR_FLAG_BOOT 0x1 /* log was pulled from NVRAM on boot */ +#define ERR_FLAG_BOOT 0x1 /* log was pulled from NVRAM on boot */ #define ERR_TYPE_RTAS_LOG 0x2 /* from rtas event-scan */ #define ERR_TYPE_KERNEL_PANIC 0x4 /* from die()/panic() */ #define ERR_TYPE_KERNEL_PANIC_GZ 0x8 /* ditto, compressed */ @@ -303,7 +493,7 @@ static inline void rtas_cancel_event_scan(void) { } (ERR_TYPE_RTAS_LOG | ERR_TYPE_KERNEL_PANIC | ERR_TYPE_KERNEL_PANIC_GZ) #define RTAS_DEBUG KERN_DEBUG "RTAS: " - + #define RTAS_ERROR_LOG_MAX 2048 /* @@ -311,7 +501,7 @@ static inline void rtas_cancel_event_scan(void) { } * for all rtas calls that require an error buffer argument. * This includes 'check-exception' and 'rtas-last-error'. */ -extern int rtas_get_error_log_max(void); +int rtas_get_error_log_max(void); /* Event Scan Parameters */ #define EVENT_SCAN_ALL_EVENTS 0xf0000000 @@ -332,6 +522,13 @@ extern char rtas_data_buf[RTAS_DATA_BUF_SIZE]; /* RMO buffer reserved for user-space RTAS use */ extern unsigned long rtas_rmo_buf; +extern struct mutex rtas_ibm_get_vpd_lock; +extern struct mutex rtas_ibm_get_indices_lock; +extern struct mutex rtas_ibm_set_dynamic_indicator_lock; +extern struct mutex rtas_ibm_get_dynamic_sensor_state_lock; +extern struct mutex rtas_ibm_physical_attestation_lock; +extern struct mutex rtas_ibm_send_hvpipe_msg_lock; + #define GLOBAL_INTERRUPT_QUEUE 9005 /** @@ -350,27 +547,33 @@ static inline u32 rtas_config_addr(int busno, int devfn, int reg) (devfn << 8) | (reg & 0xff); } -extern void rtas_give_timebase(void); -extern void rtas_take_timebase(void); +void rtas_give_timebase(void); +void rtas_take_timebase(void); #ifdef CONFIG_PPC_RTAS static inline int page_is_rtas_user_buf(unsigned long pfn) { unsigned long paddr = (pfn << PAGE_SHIFT); - if (paddr >= rtas_rmo_buf && paddr < (rtas_rmo_buf + RTAS_RMOBUF_MAX)) + if (paddr >= rtas_rmo_buf && paddr < (rtas_rmo_buf + RTAS_USER_REGION_SIZE)) return 1; return 0; } /* Not the best place to put pSeries_coalesce_init, will be fixed when we * move some of the rtas suspend-me stuff to pseries */ -extern void pSeries_coalesce_init(void); +void pSeries_coalesce_init(void); +void rtas_initialize(void); #else static inline int page_is_rtas_user_buf(unsigned long pfn) { return 0;} static inline void pSeries_coalesce_init(void) { } +static inline void rtas_initialize(void) { } #endif -extern int call_rtas(const char *, int, int, unsigned long *, ...); +#ifdef CONFIG_HV_PERF_CTRS +void read_24x7_sys_info(void); +#else +static inline void read_24x7_sys_info(void) { } +#endif #endif /* __KERNEL__ */ #endif /* _POWERPC_RTAS_H */ diff --git a/arch/powerpc/include/asm/rtc.h b/arch/powerpc/include/asm/rtc.h deleted file mode 100644 index f5802926b6c0..000000000000 --- a/arch/powerpc/include/asm/rtc.h +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Real-time clock definitions and interfaces - * - * Author: Tom Rini <trini@mvista.com> - * - * 2002 (c) MontaVista, Software, Inc. This file is licensed under - * the terms of the GNU General Public License version 2. This program - * is licensed "as is" without any warranty of any kind, whether express - * or implied. - * - * Based on: - * include/asm-m68k/rtc.h - * - * Copyright Richard Zidlicky - * implementation details for genrtc/q40rtc driver - * - * And the old drivers/macintosh/rtc.c which was heavily based on: - * Linux/SPARC Real Time Clock Driver - * Copyright (C) 1996 Thomas K. Dyas (tdyas@eden.rutgers.edu) - * - * With additional work by Paul Mackerras and Franz Sirl. - */ - -#ifndef __ASM_POWERPC_RTC_H__ -#define __ASM_POWERPC_RTC_H__ - -#ifdef __KERNEL__ - -#include <linux/rtc.h> - -#include <asm/machdep.h> -#include <asm/time.h> - -#define RTC_PIE 0x40 /* periodic interrupt enable */ -#define RTC_AIE 0x20 /* alarm interrupt enable */ -#define RTC_UIE 0x10 /* update-finished interrupt enable */ - -/* some dummy definitions */ -#define RTC_BATT_BAD 0x100 /* battery bad */ -#define RTC_SQWE 0x08 /* enable square-wave output */ -#define RTC_DM_BINARY 0x04 /* all time/date values are BCD if clear */ -#define RTC_24H 0x02 /* 24 hour mode - else hours bit 7 means pm */ -#define RTC_DST_EN 0x01 /* auto switch DST - works f. USA only */ - -static inline unsigned int get_rtc_time(struct rtc_time *time) -{ - if (ppc_md.get_rtc_time) - ppc_md.get_rtc_time(time); - return RTC_24H; -} - -/* Set the current date and time in the real time clock. */ -static inline int set_rtc_time(struct rtc_time *time) -{ - if (ppc_md.set_rtc_time) - return ppc_md.set_rtc_time(time); - return -EINVAL; -} - -static inline unsigned int get_rtc_ss(void) -{ - struct rtc_time h; - - get_rtc_time(&h); - return h.tm_sec; -} - -static inline int get_rtc_pll(struct rtc_pll_info *pll) -{ - return -EINVAL; -} -static inline int set_rtc_pll(struct rtc_pll_info *pll) -{ - return -EINVAL; -} - -#endif /* __KERNEL__ */ -#endif /* __ASM_POWERPC_RTC_H__ */ diff --git a/arch/powerpc/include/asm/runlatch.h b/arch/powerpc/include/asm/runlatch.h index 54e9b963876e..ceb66d761fe1 100644 --- a/arch/powerpc/include/asm/runlatch.h +++ b/arch/powerpc/include/asm/runlatch.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu> */ @@ -18,10 +19,9 @@ extern void __ppc64_runlatch_off(void); do { \ if (cpu_has_feature(CPU_FTR_CTRL) && \ test_thread_local_flags(_TLF_RUNLATCH)) { \ - unsigned long msr = mfmsr(); \ __hard_irq_disable(); \ __ppc64_runlatch_off(); \ - if (msr & MSR_EE) \ + if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) \ __hard_irq_enable(); \ } \ } while (0) @@ -30,10 +30,9 @@ extern void __ppc64_runlatch_off(void); do { \ if (cpu_has_feature(CPU_FTR_CTRL) && \ !test_thread_local_flags(_TLF_RUNLATCH)) { \ - unsigned long msr = mfmsr(); \ __hard_irq_disable(); \ __ppc64_runlatch_on(); \ - if (msr & MSR_EE) \ + if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) \ __hard_irq_enable(); \ } \ } while (0) diff --git a/arch/powerpc/include/asm/scatterlist.h b/arch/powerpc/include/asm/scatterlist.h deleted file mode 100644 index de1f620bd5c9..000000000000 --- a/arch/powerpc/include/asm/scatterlist.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef _ASM_POWERPC_SCATTERLIST_H -#define _ASM_POWERPC_SCATTERLIST_H -/* - * Copyright (C) 2001 PPC64 Team, IBM Corp - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ - -#include <asm/dma.h> -#include <asm-generic/scatterlist.h> - -#define ARCH_HAS_SG_CHAIN - -#endif /* _ASM_POWERPC_SCATTERLIST_H */ diff --git a/arch/powerpc/include/asm/scom.h b/arch/powerpc/include/asm/scom.h deleted file mode 100644 index 0cabfd7bc2d1..000000000000 --- a/arch/powerpc/include/asm/scom.h +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright 2010 Benjamin Herrenschmidt, IBM Corp - * <benh@kernel.crashing.org> - * and David Gibson, IBM Corporation. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See - * the GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#ifndef _ASM_POWERPC_SCOM_H -#define _ASM_POWERPC_SCOM_H - -#ifdef __KERNEL__ -#ifndef __ASSEMBLY__ -#ifdef CONFIG_PPC_SCOM - -/* - * The SCOM bus is a sideband bus used for accessing various internal - * registers of the processor or the chipset. The implementation details - * differ between processors and platforms, and the access method as - * well. - * - * This API allows to "map" ranges of SCOM register numbers associated - * with a given SCOM controller. The later must be represented by a - * device node, though some implementations might support NULL if there - * is no possible ambiguity - * - * Then, scom_read/scom_write can be used to accesses registers inside - * that range. The argument passed is a register number relative to - * the beginning of the range mapped. - */ - -typedef void *scom_map_t; - -/* Value for an invalid SCOM map */ -#define SCOM_MAP_INVALID (NULL) - -/* The scom_controller data structure is what the platform passes - * to the core code in scom_init, it provides the actual implementation - * of all the SCOM functions - */ -struct scom_controller { - scom_map_t (*map)(struct device_node *ctrl_dev, u64 reg, u64 count); - void (*unmap)(scom_map_t map); - - u64 (*read)(scom_map_t map, u32 reg); - void (*write)(scom_map_t map, u32 reg, u64 value); -}; - -extern const struct scom_controller *scom_controller; - -/** - * scom_init - Initialize the SCOM backend, called by the platform - * @controller: The platform SCOM controller - */ -static inline void scom_init(const struct scom_controller *controller) -{ - scom_controller = controller; -} - -/** - * scom_map_ok - Test is a SCOM mapping is successful - * @map: The result of scom_map to test - */ -static inline int scom_map_ok(scom_map_t map) -{ - return map != SCOM_MAP_INVALID; -} - -/** - * scom_map - Map a block of SCOM registers - * @ctrl_dev: Device node of the SCOM controller - * some implementations allow NULL here - * @reg: first SCOM register to map - * @count: Number of SCOM registers to map - */ - -static inline scom_map_t scom_map(struct device_node *ctrl_dev, - u64 reg, u64 count) -{ - return scom_controller->map(ctrl_dev, reg, count); -} - -/** - * scom_find_parent - Find the SCOM controller for a device - * @dev: OF node of the device - * - * This is not meant for general usage, but in combination with - * scom_map() allows to map registers not represented by the - * device own scom-reg property. Useful for applying HW workarounds - * on things not properly represented in the device-tree for example. - */ -struct device_node *scom_find_parent(struct device_node *dev); - - -/** - * scom_map_device - Map a device's block of SCOM registers - * @dev: OF node of the device - * @index: Register bank index (index in "scom-reg" property) - * - * This function will use the device-tree binding for SCOM which - * is to follow "scom-parent" properties until it finds a node with - * a "scom-controller" property to find the controller. It will then - * use the "scom-reg" property which is made of reg/count pairs, - * each of them having a size defined by the controller's #scom-cells - * property - */ -extern scom_map_t scom_map_device(struct device_node *dev, int index); - - -/** - * scom_unmap - Unmap a block of SCOM registers - * @map: Result of scom_map is to be unmapped - */ -static inline void scom_unmap(scom_map_t map) -{ - if (scom_map_ok(map)) - scom_controller->unmap(map); -} - -/** - * scom_read - Read a SCOM register - * @map: Result of scom_map - * @reg: Register index within that map - */ -static inline u64 scom_read(scom_map_t map, u32 reg) -{ - return scom_controller->read(map, reg); -} - -/** - * scom_write - Write to a SCOM register - * @map: Result of scom_map - * @reg: Register index within that map - * @value: Value to write - */ -static inline void scom_write(scom_map_t map, u32 reg, u64 value) -{ - scom_controller->write(map, reg, value); -} - -#endif /* CONFIG_PPC_SCOM */ -#endif /* __ASSEMBLY__ */ -#endif /* __KERNEL__ */ -#endif /* _ASM_POWERPC_SCOM_H */ diff --git a/arch/powerpc/include/asm/seccomp.h b/arch/powerpc/include/asm/seccomp.h new file mode 100644 index 000000000000..ac2033f134f0 --- /dev/null +++ b/arch/powerpc/include/asm/seccomp.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_SECCOMP_H +#define _ASM_POWERPC_SECCOMP_H + +#include <linux/unistd.h> + +#define __NR_seccomp_sigreturn_32 __NR_sigreturn + +#include <asm-generic/seccomp.h> + +#ifdef __LITTLE_ENDIAN__ +#define __SECCOMP_ARCH_LE __AUDIT_ARCH_LE +#define __SECCOMP_ARCH_LE_NAME "le" +#else +#define __SECCOMP_ARCH_LE 0 +#define __SECCOMP_ARCH_LE_NAME +#endif + +#ifdef CONFIG_PPC64 +# define SECCOMP_ARCH_NATIVE (AUDIT_ARCH_PPC64 | __SECCOMP_ARCH_LE) +# define SECCOMP_ARCH_NATIVE_NR NR_syscalls +# define SECCOMP_ARCH_NATIVE_NAME "ppc64" __SECCOMP_ARCH_LE_NAME +# ifdef CONFIG_COMPAT +# define SECCOMP_ARCH_COMPAT (AUDIT_ARCH_PPC | __SECCOMP_ARCH_LE) +# define SECCOMP_ARCH_COMPAT_NR NR_syscalls +# define SECCOMP_ARCH_COMPAT_NAME "ppc" __SECCOMP_ARCH_LE_NAME +# endif +#else /* !CONFIG_PPC64 */ +# define SECCOMP_ARCH_NATIVE (AUDIT_ARCH_PPC | __SECCOMP_ARCH_LE) +# define SECCOMP_ARCH_NATIVE_NR NR_syscalls +# define SECCOMP_ARCH_NATIVE_NAME "ppc" __SECCOMP_ARCH_LE_NAME +#endif + +#endif /* _ASM_POWERPC_SECCOMP_H */ diff --git a/arch/powerpc/include/asm/sections.h b/arch/powerpc/include/asm/sections.h index 4ee06fe15de4..f43f3a6b0051 100644 --- a/arch/powerpc/include/asm/sections.h +++ b/arch/powerpc/include/asm/sections.h @@ -1,24 +1,72 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_SECTIONS_H #define _ASM_POWERPC_SECTIONS_H #ifdef __KERNEL__ #include <linux/elf.h> #include <linux/uaccess.h> + +#ifdef CONFIG_HAVE_FUNCTION_DESCRIPTORS +typedef struct func_desc func_desc_t; +#endif + #include <asm-generic/sections.h> +extern char __head_end[]; +extern char __srwx_boundary[]; +extern char __exittext_begin[], __exittext_end[]; + +/* Patch sites */ +extern s32 patch__call_flush_branch_caches1; +extern s32 patch__call_flush_branch_caches2; +extern s32 patch__call_flush_branch_caches3; +extern s32 patch__flush_count_cache_return; +extern s32 patch__flush_link_stack_return; +extern s32 patch__call_kvm_flush_link_stack; +extern s32 patch__call_kvm_flush_link_stack_p9; +extern s32 patch__memset_nocache, patch__memcpy_nocache; + +extern long flush_branch_caches; +extern long kvm_flush_link_stack; + #ifdef __powerpc64__ +extern char __start_interrupts[]; extern char __end_interrupts[]; -extern char __prom_init_toc_start[]; -extern char __prom_init_toc_end[]; +#ifdef CONFIG_PPC_POWERNV +extern char start_real_trampolines[]; +extern char end_real_trampolines[]; +extern char start_virt_trampolines[]; +extern char end_virt_trampolines[]; +#endif + +/* + * This assumes the kernel is never compiled -mcmodel=small or + * the total .toc is always less than 64k. + */ +static inline unsigned long kernel_toc_addr(void) +{ +#ifdef CONFIG_PPC_KERNEL_PCREL + BUILD_BUG(); + return -1UL; +#else + unsigned long toc_ptr; + + asm volatile("mr %0, 2" : "=r" (toc_ptr)); + return toc_ptr; +#endif +} -static inline int in_kernel_text(unsigned long addr) +static inline int overlaps_interrupt_vector_text(unsigned long start, + unsigned long end) { - if (addr >= (unsigned long)_stext && addr < (unsigned long)__init_end) - return 1; + unsigned long real_start, real_end; + real_start = __start_interrupts - _stext; + real_end = __end_interrupts - _stext; - return 0; + return start < (unsigned long)__va(real_end) && + (unsigned long)__va(real_start) < end; } static inline int overlaps_kernel_text(unsigned long start, unsigned long end) @@ -27,17 +75,8 @@ static inline int overlaps_kernel_text(unsigned long start, unsigned long end) (unsigned long)_stext < end; } -#undef dereference_function_descriptor -static inline void *dereference_function_descriptor(void *ptr) -{ - struct ppc64_opd_entry *desc = ptr; - void *p; - - if (!probe_kernel_address(&desc->funcaddr, p)) - ptr = p; - return ptr; -} - +#else +static inline unsigned long kernel_toc_addr(void) { BUILD_BUG(); return -1UL; } #endif #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/secure_boot.h b/arch/powerpc/include/asm/secure_boot.h new file mode 100644 index 000000000000..a2ff556916c6 --- /dev/null +++ b/arch/powerpc/include/asm/secure_boot.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Secure boot definitions + * + * Copyright (C) 2019 IBM Corporation + * Author: Nayna Jain + */ +#ifndef _ASM_POWER_SECURE_BOOT_H +#define _ASM_POWER_SECURE_BOOT_H + +#ifdef CONFIG_PPC_SECURE_BOOT + +bool is_ppc_secureboot_enabled(void); +bool is_ppc_trustedboot_enabled(void); + +#else + +static inline bool is_ppc_secureboot_enabled(void) +{ + return false; +} + +static inline bool is_ppc_trustedboot_enabled(void) +{ + return false; +} + +#endif +#endif diff --git a/arch/powerpc/include/asm/security_features.h b/arch/powerpc/include/asm/security_features.h new file mode 100644 index 000000000000..27574f218b37 --- /dev/null +++ b/arch/powerpc/include/asm/security_features.h @@ -0,0 +1,113 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Security related feature bit definitions. + * + * Copyright 2018, Michael Ellerman, IBM Corporation. + */ + +#ifndef _ASM_POWERPC_SECURITY_FEATURES_H +#define _ASM_POWERPC_SECURITY_FEATURES_H + + +extern u64 powerpc_security_features; +extern bool rfi_flush; + +/* These are bit flags */ +enum stf_barrier_type { + STF_BARRIER_NONE = 0x1, + STF_BARRIER_FALLBACK = 0x2, + STF_BARRIER_EIEIO = 0x4, + STF_BARRIER_SYNC_ORI = 0x8, +}; + +void setup_stf_barrier(void); +void do_stf_barrier_fixups(enum stf_barrier_type types); +void setup_count_cache_flush(void); + +static inline void security_ftr_set(u64 feature) +{ + powerpc_security_features |= feature; +} + +static inline void security_ftr_clear(u64 feature) +{ + powerpc_security_features &= ~feature; +} + +static inline bool security_ftr_enabled(u64 feature) +{ + return !!(powerpc_security_features & feature); +} + +#ifdef CONFIG_PPC_BOOK3S_64 +enum stf_barrier_type stf_barrier_type_get(void); +#else +static inline enum stf_barrier_type stf_barrier_type_get(void) { return STF_BARRIER_NONE; } +#endif + +// Features indicating support for Spectre/Meltdown mitigations + +// The L1-D cache can be flushed with ori r30,r30,0 +#define SEC_FTR_L1D_FLUSH_ORI30 0x0000000000000001ull + +// The L1-D cache can be flushed with mtspr 882,r0 (aka SPRN_TRIG2) +#define SEC_FTR_L1D_FLUSH_TRIG2 0x0000000000000002ull + +// ori r31,r31,0 acts as a speculation barrier +#define SEC_FTR_SPEC_BAR_ORI31 0x0000000000000004ull + +// Speculation past bctr is disabled +#define SEC_FTR_BCCTRL_SERIALISED 0x0000000000000008ull + +// Entries in L1-D are private to a SMT thread +#define SEC_FTR_L1D_THREAD_PRIV 0x0000000000000010ull + +// Indirect branch prediction cache disabled +#define SEC_FTR_COUNT_CACHE_DISABLED 0x0000000000000020ull + +// bcctr 2,0,0 triggers a hardware assisted count cache flush +#define SEC_FTR_BCCTR_FLUSH_ASSIST 0x0000000000000800ull + +// bcctr 2,0,0 triggers a hardware assisted link stack flush +#define SEC_FTR_BCCTR_LINK_FLUSH_ASSIST 0x0000000000002000ull + +// Features indicating need for Spectre/Meltdown mitigations + +// The L1-D cache should be flushed on MSR[HV] 1->0 transition (hypervisor to guest) +#define SEC_FTR_L1D_FLUSH_HV 0x0000000000000040ull + +// The L1-D cache should be flushed on MSR[PR] 0->1 transition (kernel to userspace) +#define SEC_FTR_L1D_FLUSH_PR 0x0000000000000080ull + +// A speculation barrier should be used for bounds checks (Spectre variant 1) +#define SEC_FTR_BNDS_CHK_SPEC_BAR 0x0000000000000100ull + +// Firmware configuration indicates user favours security over performance +#define SEC_FTR_FAVOUR_SECURITY 0x0000000000000200ull + +// Software required to flush count cache on context switch +#define SEC_FTR_FLUSH_COUNT_CACHE 0x0000000000000400ull + +// Software required to flush link stack on context switch +#define SEC_FTR_FLUSH_LINK_STACK 0x0000000000001000ull + +// The L1-D cache should be flushed when entering the kernel +#define SEC_FTR_L1D_FLUSH_ENTRY 0x0000000000004000ull + +// The L1-D cache should be flushed after user accesses from the kernel +#define SEC_FTR_L1D_FLUSH_UACCESS 0x0000000000008000ull + +// The STF flush should be executed on privilege state switch +#define SEC_FTR_STF_BARRIER 0x0000000000010000ull + +// Features enabled by default +#define SEC_FTR_DEFAULT \ + (SEC_FTR_L1D_FLUSH_HV | \ + SEC_FTR_L1D_FLUSH_PR | \ + SEC_FTR_BNDS_CHK_SPEC_BAR | \ + SEC_FTR_L1D_FLUSH_ENTRY | \ + SEC_FTR_L1D_FLUSH_UACCESS | \ + SEC_FTR_STF_BARRIER | \ + SEC_FTR_FAVOUR_SECURITY) + +#endif /* _ASM_POWERPC_SECURITY_FEATURES_H */ diff --git a/arch/powerpc/include/asm/secvar.h b/arch/powerpc/include/asm/secvar.h new file mode 100644 index 000000000000..4828e0ab7e3c --- /dev/null +++ b/arch/powerpc/include/asm/secvar.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2019 IBM Corporation + * Author: Nayna Jain + * + * PowerPC secure variable operations. + */ +#ifndef SECVAR_OPS_H +#define SECVAR_OPS_H + +#include <linux/types.h> +#include <linux/errno.h> +#include <linux/sysfs.h> + +extern const struct secvar_operations *secvar_ops; + +struct secvar_operations { + int (*get)(const char *key, u64 key_len, u8 *data, u64 *data_size); + int (*get_next)(const char *key, u64 *key_len, u64 keybufsize); + int (*set)(const char *key, u64 key_len, u8 *data, u64 data_size); + ssize_t (*format)(char *buf, size_t bufsize); + int (*max_size)(u64 *max_size); + const struct attribute **config_attrs; + + // NULL-terminated array of fixed variable names + // Only used if get_next() isn't provided + const char * const *var_names; +}; + +#ifdef CONFIG_PPC_SECURE_BOOT + +int set_secvar_ops(const struct secvar_operations *ops); + +#else + +static inline int set_secvar_ops(const struct secvar_operations *ops) { return 0; } + +#endif + +#endif diff --git a/arch/powerpc/include/asm/serial.h b/arch/powerpc/include/asm/serial.h index 3e8589b43cb2..cd6c18d0e66e 100644 --- a/arch/powerpc/include/asm/serial.h +++ b/arch/powerpc/include/asm/serial.h @@ -1,8 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef _ASM_POWERPC_SERIAL_H #define _ASM_POWERPC_SERIAL_H diff --git a/arch/powerpc/include/asm/set_memory.h b/arch/powerpc/include/asm/set_memory.h new file mode 100644 index 000000000000..9c8d5747755d --- /dev/null +++ b/arch/powerpc/include/asm/set_memory.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_SET_MEMORY_H +#define _ASM_POWERPC_SET_MEMORY_H + +#define SET_MEMORY_RO 0 +#define SET_MEMORY_RW 1 +#define SET_MEMORY_NX 2 +#define SET_MEMORY_X 3 +#define SET_MEMORY_NP 4 /* Set memory non present */ +#define SET_MEMORY_P 5 /* Set memory present */ +#define SET_MEMORY_ROX 6 + +int change_memory_attr(unsigned long addr, int numpages, long action); + +static inline int __must_check set_memory_ro(unsigned long addr, int numpages) +{ + return change_memory_attr(addr, numpages, SET_MEMORY_RO); +} + +static inline int __must_check set_memory_rw(unsigned long addr, int numpages) +{ + return change_memory_attr(addr, numpages, SET_MEMORY_RW); +} + +static inline int __must_check set_memory_nx(unsigned long addr, int numpages) +{ + return change_memory_attr(addr, numpages, SET_MEMORY_NX); +} + +static inline int __must_check set_memory_x(unsigned long addr, int numpages) +{ + return change_memory_attr(addr, numpages, SET_MEMORY_X); +} + +static inline int __must_check set_memory_np(unsigned long addr, int numpages) +{ + return change_memory_attr(addr, numpages, SET_MEMORY_NP); +} + +static inline int __must_check set_memory_p(unsigned long addr, int numpages) +{ + return change_memory_attr(addr, numpages, SET_MEMORY_P); +} + +static inline int __must_check set_memory_rox(unsigned long addr, int numpages) +{ + return change_memory_attr(addr, numpages, SET_MEMORY_ROX); +} +#define set_memory_rox set_memory_rox + +#endif diff --git a/arch/powerpc/include/asm/setjmp.h b/arch/powerpc/include/asm/setjmp.h index 279d03a1eec6..f798e80e4106 100644 --- a/arch/powerpc/include/asm/setjmp.h +++ b/arch/powerpc/include/asm/setjmp.h @@ -1,18 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Copyright © 2008 Michael Neuling IBM Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * */ #ifndef _ASM_POWERPC_SETJMP_H #define _ASM_POWERPC_SETJMP_H #define JMP_BUF_LEN 23 -extern long setjmp(long *); -extern void longjmp(long *, long); +typedef long jmp_buf[JMP_BUF_LEN]; + +extern int setjmp(jmp_buf env) __attribute__((returns_twice)); +extern void longjmp(jmp_buf env, int val) __attribute__((noreturn)); #endif /* _ASM_POWERPC_SETJMP_H */ diff --git a/arch/powerpc/include/asm/setup.h b/arch/powerpc/include/asm/setup.h index d3ca85529b8b..50a92b24628d 100644 --- a/arch/powerpc/include/asm/setup.h +++ b/arch/powerpc/include/asm/setup.h @@ -1,20 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_SETUP_H #define _ASM_POWERPC_SETUP_H #include <uapi/asm/setup.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ extern void ppc_printk_progress(char *s, unsigned short hex); -extern unsigned int rtas_data; -extern int mem_init_done; /* set on boot once kmalloc can be called */ -extern int init_bootmem_done; /* set once bootmem is available */ extern unsigned long long memory_limit; -extern unsigned long klimit; -extern void *zalloc_maybe_bootmem(size_t size, gfp_t mask); struct device_node; -extern void note_scsi_host(struct device_node *, void *); /* Used in very early kernel initialization. */ extern unsigned long reloc_offset(void); @@ -23,7 +18,78 @@ extern void reloc_got2(unsigned long); #define PTRRELOC(x) ((typeof(x)) add_reloc_offset((unsigned long)(x))) -#endif /* !__ASSEMBLY__ */ +void check_for_initrd(void); +void mem_topology_setup(void); +void initmem_init(void); +void setup_panic(void); +#define ARCH_PANIC_TIMEOUT 180 + +#ifdef CONFIG_PPC_PSERIES +extern bool pseries_reloc_on_exception(void); +extern bool pseries_enable_reloc_on_exc(void); +extern void pseries_disable_reloc_on_exc(void); +extern void pseries_big_endian_exceptions(void); +void __init pseries_little_endian_exceptions(void); +#else +static inline bool pseries_reloc_on_exception(void) { return false; } +static inline bool pseries_enable_reloc_on_exc(void) { return false; } +static inline void pseries_disable_reloc_on_exc(void) {} +static inline void pseries_big_endian_exceptions(void) {} +static inline void pseries_little_endian_exceptions(void) {} +#endif /* CONFIG_PPC_PSERIES */ + +void rfi_flush_enable(bool enable); + +/* These are bit flags */ +enum l1d_flush_type { + L1D_FLUSH_NONE = 0x1, + L1D_FLUSH_FALLBACK = 0x2, + L1D_FLUSH_ORI = 0x4, + L1D_FLUSH_MTTRIG = 0x8, +}; + +void setup_rfi_flush(enum l1d_flush_type, bool enable); +void setup_entry_flush(bool enable); +void setup_uaccess_flush(bool enable); +void do_rfi_flush_fixups(enum l1d_flush_type types); +#ifdef CONFIG_PPC_BARRIER_NOSPEC +void __init setup_barrier_nospec(void); +#else +static inline void setup_barrier_nospec(void) { } +#endif +void do_uaccess_flush_fixups(enum l1d_flush_type types); +void do_entry_flush_fixups(enum l1d_flush_type types); +void do_barrier_nospec_fixups(bool enable); +extern bool barrier_nospec_enabled; + +#ifdef CONFIG_PPC_BARRIER_NOSPEC +void do_barrier_nospec_fixups_range(bool enable, void *start, void *end); +#else +static inline void do_barrier_nospec_fixups_range(bool enable, void *start, void *end) { } +#endif + +#ifdef CONFIG_PPC_E500 +void __init setup_spectre_v2(void); +#else +static inline void setup_spectre_v2(void) {} +#endif +void __init do_btb_flush_fixups(void); + +#ifdef CONFIG_PPC32 +unsigned long __init early_init(unsigned long dt_ptr); +void __init machine_init(u64 dt_ptr); +#endif +void __init early_setup(unsigned long dt_ptr); +void early_setup_secondary(void); + +/* prom_init (OpenFirmware) */ +unsigned long __init prom_init(unsigned long r3, unsigned long r4, + unsigned long pp, unsigned long r6, + unsigned long r7, unsigned long kbase); + +extern struct seq_buf ppc_hw_desc; + +#endif /* !__ASSEMBLER__ */ #endif /* _ASM_POWERPC_SETUP_H */ diff --git a/arch/powerpc/include/asm/sfp-machine.h b/arch/powerpc/include/asm/sfp-machine.h index 3a7a67a0d006..8b957aabb826 100644 --- a/arch/powerpc/include/asm/sfp-machine.h +++ b/arch/powerpc/include/asm/sfp-machine.h @@ -125,7 +125,7 @@ #define FP_EX_DIVZERO (1 << (31 - 5)) #define FP_EX_INEXACT (1 << (31 - 6)) -#define __FPU_FPSCR (current->thread.fpscr.val) +#define __FPU_FPSCR (current->thread.fp_state.fpscr) /* We only actually write to the destination register * if exceptions signalled (if any) will not trap. @@ -213,30 +213,18 @@ * respectively. The result is placed in HIGH_SUM and LOW_SUM. Overflow * (i.e. carry out) is not stored anywhere, and is lost. */ -#define add_ssaaaa(sh, sl, ah, al, bh, bl) \ +#define add_ssaaaa(sh, sl, ah, al, bh, bl) \ do { \ if (__builtin_constant_p (bh) && (bh) == 0) \ - __asm__ ("{a%I4|add%I4c} %1,%3,%4\n\t{aze|addze} %0,%2" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ - : "%r" ((USItype)(ah)), \ - "%r" ((USItype)(al)), \ - "rI" ((USItype)(bl))); \ - else if (__builtin_constant_p (bh) && (bh) ==~(USItype) 0) \ - __asm__ ("{a%I4|add%I4c} %1,%3,%4\n\t{ame|addme} %0,%2" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ - : "%r" ((USItype)(ah)), \ - "%r" ((USItype)(al)), \ - "rI" ((USItype)(bl))); \ + __asm__ ("add%I4c %1,%3,%4\n\taddze %0,%2" \ + : "=r" (sh), "=&r" (sl) : "r" (ah), "%r" (al), "rI" (bl));\ + else if (__builtin_constant_p (bh) && (bh) == ~(USItype) 0) \ + __asm__ ("add%I4c %1,%3,%4\n\taddme %0,%2" \ + : "=r" (sh), "=&r" (sl) : "r" (ah), "%r" (al), "rI" (bl));\ else \ - __asm__ ("{a%I5|add%I5c} %1,%4,%5\n\t{ae|adde} %0,%2,%3" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ - : "%r" ((USItype)(ah)), \ - "r" ((USItype)(bh)), \ - "%r" ((USItype)(al)), \ - "rI" ((USItype)(bl))); \ + __asm__ ("add%I5c %1,%4,%5\n\tadde %0,%2,%3" \ + : "=r" (sh), "=&r" (sl) \ + : "%r" (ah), "r" (bh), "%r" (al), "rI" (bl)); \ } while (0) /* sub_ddmmss is used in op-2.h and udivmodti4.c and should be equivalent to @@ -248,44 +236,24 @@ * and LOW_DIFFERENCE. Overflow (i.e. carry out) is not stored anywhere, * and is lost. */ -#define sub_ddmmss(sh, sl, ah, al, bh, bl) \ +#define sub_ddmmss(sh, sl, ah, al, bh, bl) \ do { \ if (__builtin_constant_p (ah) && (ah) == 0) \ - __asm__ ("{sf%I3|subf%I3c} %1,%4,%3\n\t{sfze|subfze} %0,%2" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ - : "r" ((USItype)(bh)), \ - "rI" ((USItype)(al)), \ - "r" ((USItype)(bl))); \ - else if (__builtin_constant_p (ah) && (ah) ==~(USItype) 0) \ - __asm__ ("{sf%I3|subf%I3c} %1,%4,%3\n\t{sfme|subfme} %0,%2" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ - : "r" ((USItype)(bh)), \ - "rI" ((USItype)(al)), \ - "r" ((USItype)(bl))); \ + __asm__ ("subf%I3c %1,%4,%3\n\tsubfze %0,%2" \ + : "=r" (sh), "=&r" (sl) : "r" (bh), "rI" (al), "r" (bl));\ + else if (__builtin_constant_p (ah) && (ah) == ~(USItype) 0) \ + __asm__ ("subf%I3c %1,%4,%3\n\tsubfme %0,%2" \ + : "=r" (sh), "=&r" (sl) : "r" (bh), "rI" (al), "r" (bl));\ else if (__builtin_constant_p (bh) && (bh) == 0) \ - __asm__ ("{sf%I3|subf%I3c} %1,%4,%3\n\t{ame|addme} %0,%2" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ - : "r" ((USItype)(ah)), \ - "rI" ((USItype)(al)), \ - "r" ((USItype)(bl))); \ - else if (__builtin_constant_p (bh) && (bh) ==~(USItype) 0) \ - __asm__ ("{sf%I3|subf%I3c} %1,%4,%3\n\t{aze|addze} %0,%2" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ - : "r" ((USItype)(ah)), \ - "rI" ((USItype)(al)), \ - "r" ((USItype)(bl))); \ + __asm__ ("subf%I3c %1,%4,%3\n\taddme %0,%2" \ + : "=r" (sh), "=&r" (sl) : "r" (ah), "rI" (al), "r" (bl));\ + else if (__builtin_constant_p (bh) && (bh) == ~(USItype) 0) \ + __asm__ ("subf%I3c %1,%4,%3\n\taddze %0,%2" \ + : "=r" (sh), "=&r" (sl) : "r" (ah), "rI" (al), "r" (bl));\ else \ - __asm__ ("{sf%I4|subf%I4c} %1,%5,%4\n\t{sfe|subfe} %0,%3,%2" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ - : "r" ((USItype)(ah)), \ - "r" ((USItype)(bh)), \ - "rI" ((USItype)(al)), \ - "r" ((USItype)(bl))); \ + __asm__ ("subf%I4c %1,%5,%4\n\tsubfe %0,%3,%2" \ + : "=r" (sh), "=&r" (sl) \ + : "r" (ah), "r" (bh), "rI" (al), "r" (bl)); \ } while (0) /* asm fragments for mul and div */ @@ -294,13 +262,10 @@ * UWtype integers MULTIPLER and MULTIPLICAND, and generates a two UWtype * word product in HIGH_PROD and LOW_PROD. */ -#define umul_ppmm(ph, pl, m0, m1) \ +#define umul_ppmm(ph, pl, m0, m1) \ do { \ USItype __m0 = (m0), __m1 = (m1); \ - __asm__ ("mulhwu %0,%1,%2" \ - : "=r" ((USItype)(ph)) \ - : "%r" (__m0), \ - "r" (__m1)); \ + __asm__ ("mulhwu %0,%1,%2" : "=r" (ph) : "%r" (m0), "r" (m1)); \ (pl) = __m0 * __m1; \ } while (0) @@ -312,9 +277,10 @@ * significant bit of DENOMINATOR must be 1, then the pre-processor symbol * UDIV_NEEDS_NORMALIZATION is defined to 1. */ -#define udiv_qrnnd(q, r, n1, n0, d) \ +#define udiv_qrnnd(q, r, n1, n0, d) \ do { \ - UWtype __d1, __d0, __q1, __q0, __r1, __r0, __m; \ + UWtype __d1, __d0, __q1, __q0; \ + UWtype __r1, __r0, __m; \ __d1 = __ll_highpart (d); \ __d0 = __ll_lowpart (d); \ \ @@ -325,7 +291,7 @@ if (__r1 < __m) \ { \ __q1--, __r1 += (d); \ - if (__r1 >= (d)) /* we didn't get carry when adding to __r1 */ \ + if (__r1 >= (d)) /* i.e. we didn't get carry when adding to __r1 */\ if (__r1 < __m) \ __q1--, __r1 += (d); \ } \ diff --git a/arch/powerpc/include/asm/shmparam.h b/arch/powerpc/include/asm/shmparam.h index 5cda42a6d39e..bc0968839565 100644 --- a/arch/powerpc/include/asm/shmparam.h +++ b/arch/powerpc/include/asm/shmparam.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_SHMPARAM_H #define _ASM_POWERPC_SHMPARAM_H diff --git a/arch/powerpc/include/asm/signal.h b/arch/powerpc/include/asm/signal.h index 9322c28aebd2..922d43700fb4 100644 --- a/arch/powerpc/include/asm/signal.h +++ b/arch/powerpc/include/asm/signal.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_SIGNAL_H #define _ASM_POWERPC_SIGNAL_H @@ -5,6 +6,12 @@ #include <uapi/asm/signal.h> #include <uapi/asm/ptrace.h> -extern unsigned long get_tm_stackpointer(struct pt_regs *regs); +struct pt_regs; +void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags); + +unsigned long get_min_sigframe_size_32(void); +unsigned long get_min_sigframe_size_64(void); +unsigned long get_min_sigframe_size(void); +unsigned long get_min_sigframe_size_compat(void); #endif /* _ASM_POWERPC_SIGNAL_H */ diff --git a/arch/powerpc/include/asm/simple_spinlock.h b/arch/powerpc/include/asm/simple_spinlock.h new file mode 100644 index 000000000000..4dd12dcb9ef8 --- /dev/null +++ b/arch/powerpc/include/asm/simple_spinlock.h @@ -0,0 +1,268 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_POWERPC_SIMPLE_SPINLOCK_H +#define _ASM_POWERPC_SIMPLE_SPINLOCK_H + +/* + * Simple spin lock operations. + * + * Copyright (C) 2001-2004 Paul Mackerras <paulus@au.ibm.com>, IBM + * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM + * Copyright (C) 2002 Dave Engebretsen <engebret@us.ibm.com>, IBM + * Rework to support virtual processors + * + * Type of int is used as a full 64b word is not necessary. + * + * (the type definitions are in asm/simple_spinlock_types.h) + */ +#include <linux/irqflags.h> +#include <linux/kcsan-checks.h> +#include <asm/paravirt.h> +#include <asm/paca.h> +#include <asm/synch.h> +#include <asm/ppc-opcode.h> + +#ifdef CONFIG_PPC64 +/* use 0x800000yy when locked, where yy == CPU number */ +#ifdef __BIG_ENDIAN__ +#define LOCK_TOKEN (*(u32 *)(&get_paca()->lock_token)) +#else +#define LOCK_TOKEN (*(u32 *)(&get_paca()->paca_index)) +#endif +#else +#define LOCK_TOKEN 1 +#endif + +static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock) +{ + return lock.slock == 0; +} + +static inline int arch_spin_is_locked(arch_spinlock_t *lock) +{ + return !arch_spin_value_unlocked(READ_ONCE(*lock)); +} + +/* + * This returns the old value in the lock, so we succeeded + * in getting the lock if the return value is 0. + */ +static inline unsigned long __arch_spin_trylock(arch_spinlock_t *lock) +{ + unsigned long tmp, token; + unsigned int eh = IS_ENABLED(CONFIG_PPC64); + + token = LOCK_TOKEN; + __asm__ __volatile__( +"1: lwarx %0,0,%2,%[eh]\n\ + cmpwi 0,%0,0\n\ + bne- 2f\n\ + stwcx. %1,0,%2\n\ + bne- 1b\n" + PPC_ACQUIRE_BARRIER +"2:" + : "=&r" (tmp) + : "r" (token), "r" (&lock->slock), [eh] "n" (eh) + : "cr0", "memory"); + + return tmp; +} + +static inline int arch_spin_trylock(arch_spinlock_t *lock) +{ + return __arch_spin_trylock(lock) == 0; +} + +/* + * On a system with shared processors (that is, where a physical + * processor is multiplexed between several virtual processors), + * there is no point spinning on a lock if the holder of the lock + * isn't currently scheduled on a physical processor. Instead + * we detect this situation and ask the hypervisor to give the + * rest of our timeslice to the lock holder. + * + * So that we can tell which virtual processor is holding a lock, + * we put 0x80000000 | smp_processor_id() in the lock when it is + * held. Conveniently, we have a word in the paca that holds this + * value. + */ + +#if defined(CONFIG_PPC_SPLPAR) +/* We only yield to the hypervisor if we are in shared processor mode */ +void splpar_spin_yield(arch_spinlock_t *lock); +void splpar_rw_yield(arch_rwlock_t *lock); +#else /* SPLPAR */ +static inline void splpar_spin_yield(arch_spinlock_t *lock) {} +static inline void splpar_rw_yield(arch_rwlock_t *lock) {} +#endif + +static inline void spin_yield(arch_spinlock_t *lock) +{ + if (is_shared_processor()) + splpar_spin_yield(lock); + else + barrier(); +} + +static inline void rw_yield(arch_rwlock_t *lock) +{ + if (is_shared_processor()) + splpar_rw_yield(lock); + else + barrier(); +} + +static inline void arch_spin_lock(arch_spinlock_t *lock) +{ + while (1) { + if (likely(__arch_spin_trylock(lock) == 0)) + break; + do { + HMT_low(); + if (is_shared_processor()) + splpar_spin_yield(lock); + } while (unlikely(lock->slock != 0)); + HMT_medium(); + } +} + +static inline void arch_spin_unlock(arch_spinlock_t *lock) +{ + kcsan_mb(); + __asm__ __volatile__("# arch_spin_unlock\n\t" + PPC_RELEASE_BARRIER: : :"memory"); + lock->slock = 0; +} + +/* + * Read-write spinlocks, allowing multiple readers + * but only one writer. + * + * NOTE! it is quite common to have readers in interrupts + * but no interrupt writers. For those circumstances we + * can "mix" irq-safe locks - any writer needs to get a + * irq-safe write-lock, but readers can get non-irqsafe + * read-locks. + */ + +#ifdef CONFIG_PPC64 +#define __DO_SIGN_EXTEND "extsw %0,%0\n" +#define WRLOCK_TOKEN LOCK_TOKEN /* it's negative */ +#else +#define __DO_SIGN_EXTEND +#define WRLOCK_TOKEN (-1) +#endif + +/* + * This returns the old value in the lock + 1, + * so we got a read lock if the return value is > 0. + */ +static inline long __arch_read_trylock(arch_rwlock_t *rw) +{ + long tmp; + unsigned int eh = IS_ENABLED(CONFIG_PPC64); + + __asm__ __volatile__( +"1: lwarx %0,0,%1,%[eh]\n" + __DO_SIGN_EXTEND +" addic. %0,%0,1\n\ + ble- 2f\n" +" stwcx. %0,0,%1\n\ + bne- 1b\n" + PPC_ACQUIRE_BARRIER +"2:" : "=&r" (tmp) + : "r" (&rw->lock), [eh] "n" (eh) + : "cr0", "xer", "memory"); + + return tmp; +} + +/* + * This returns the old value in the lock, + * so we got the write lock if the return value is 0. + */ +static inline long __arch_write_trylock(arch_rwlock_t *rw) +{ + long tmp, token; + unsigned int eh = IS_ENABLED(CONFIG_PPC64); + + token = WRLOCK_TOKEN; + __asm__ __volatile__( +"1: lwarx %0,0,%2,%[eh]\n\ + cmpwi 0,%0,0\n\ + bne- 2f\n" +" stwcx. %1,0,%2\n\ + bne- 1b\n" + PPC_ACQUIRE_BARRIER +"2:" : "=&r" (tmp) + : "r" (token), "r" (&rw->lock), [eh] "n" (eh) + : "cr0", "memory"); + + return tmp; +} + +static inline void arch_read_lock(arch_rwlock_t *rw) +{ + while (1) { + if (likely(__arch_read_trylock(rw) > 0)) + break; + do { + HMT_low(); + if (is_shared_processor()) + splpar_rw_yield(rw); + } while (unlikely(rw->lock < 0)); + HMT_medium(); + } +} + +static inline void arch_write_lock(arch_rwlock_t *rw) +{ + while (1) { + if (likely(__arch_write_trylock(rw) == 0)) + break; + do { + HMT_low(); + if (is_shared_processor()) + splpar_rw_yield(rw); + } while (unlikely(rw->lock != 0)); + HMT_medium(); + } +} + +static inline int arch_read_trylock(arch_rwlock_t *rw) +{ + return __arch_read_trylock(rw) > 0; +} + +static inline int arch_write_trylock(arch_rwlock_t *rw) +{ + return __arch_write_trylock(rw) == 0; +} + +static inline void arch_read_unlock(arch_rwlock_t *rw) +{ + long tmp; + + __asm__ __volatile__( + "# read_unlock\n\t" + PPC_RELEASE_BARRIER +"1: lwarx %0,0,%1\n\ + addic %0,%0,-1\n" +" stwcx. %0,0,%1\n\ + bne- 1b" + : "=&r"(tmp) + : "r"(&rw->lock) + : "cr0", "xer", "memory"); +} + +static inline void arch_write_unlock(arch_rwlock_t *rw) +{ + __asm__ __volatile__("# write_unlock\n\t" + PPC_RELEASE_BARRIER: : :"memory"); + rw->lock = 0; +} + +#define arch_spin_relax(lock) spin_yield(lock) +#define arch_read_relax(lock) rw_yield(lock) +#define arch_write_relax(lock) rw_yield(lock) + +#endif /* _ASM_POWERPC_SIMPLE_SPINLOCK_H */ diff --git a/arch/powerpc/include/asm/simple_spinlock_types.h b/arch/powerpc/include/asm/simple_spinlock_types.h new file mode 100644 index 000000000000..391fc19f7272 --- /dev/null +++ b/arch/powerpc/include/asm/simple_spinlock_types.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_SIMPLE_SPINLOCK_TYPES_H +#define _ASM_POWERPC_SIMPLE_SPINLOCK_TYPES_H + +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H +# error "Please do not include this file directly." +#endif + +typedef struct { + volatile unsigned int slock; +} arch_spinlock_t; + +#define __ARCH_SPIN_LOCK_UNLOCKED { 0 } + +typedef struct { + volatile signed int lock; +} arch_rwlock_t; + +#define __ARCH_RW_LOCK_UNLOCKED { 0 } + +#endif /* _ASM_POWERPC_SIMPLE_SPINLOCK_TYPES_H */ diff --git a/arch/powerpc/include/asm/smp.h b/arch/powerpc/include/asm/smp.h index ffbaabebcdca..e41b9ea42122 100644 --- a/arch/powerpc/include/asm/smp.h +++ b/arch/powerpc/include/asm/smp.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * smp.h: PowerPC-specific SMP code. * @@ -6,11 +7,6 @@ * * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) * Copyright (C) 1996-2001 Cort Dougan <cort@fsmlabs.com> - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef _ASM_POWERPC_SMP_H @@ -22,7 +18,7 @@ #include <linux/kernel.h> #include <linux/irqreturn.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #ifdef CONFIG_PPC64 #include <asm/paca.h> @@ -30,19 +26,30 @@ #include <asm/percpu.h> extern int boot_cpuid; +extern int boot_cpu_hwid; /* PPC64 only */ +extern int boot_core_hwid; extern int spinning_secondaries; +extern u32 *cpu_to_phys_id; +extern bool coregroup_enabled; + +extern int cpu_to_chip_id(int cpu); +extern int *chip_id_lookup_table; -extern void cpu_die(void); +DECLARE_PER_CPU(cpumask_var_t, thread_group_l1_cache_map); +DECLARE_PER_CPU(cpumask_var_t, thread_group_l2_cache_map); +DECLARE_PER_CPU(cpumask_var_t, thread_group_l3_cache_map); #ifdef CONFIG_SMP struct smp_ops_t { void (*message_pass)(int cpu, int msg); #ifdef CONFIG_PPC_SMP_MUXED_IPI - void (*cause_ipi)(int cpu, unsigned long data); + void (*cause_ipi)(int cpu); #endif - int (*probe)(void); + int (*cause_nmi_ipi)(int cpu); + void (*probe)(void); int (*kick_cpu)(int nr); + int (*prepare_cpu)(int nr); void (*setup_cpu)(int nr); void (*bringup_done)(void); void (*take_timebase)(void); @@ -50,31 +57,32 @@ struct smp_ops_t { int (*cpu_disable)(void); void (*cpu_die)(unsigned int nr); int (*cpu_bootable)(unsigned int nr); +#ifdef CONFIG_HOTPLUG_CPU + void (*cpu_offline_self)(void); +#endif }; +extern struct task_struct *secondary_current; + +void start_secondary(void *unused); +extern int smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us); +extern int smp_send_safe_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us); extern void smp_send_debugger_break(void); -extern void start_secondary_resume(void); +extern void __noreturn start_secondary_resume(void); extern void smp_generic_give_timebase(void); extern void smp_generic_take_timebase(void); DECLARE_PER_CPU(unsigned int, cpu_pvr); #ifdef CONFIG_HOTPLUG_CPU -extern void migrate_irqs(void); int generic_cpu_disable(void); void generic_cpu_die(unsigned int cpu); -void generic_mach_cpu_die(void); void generic_set_cpu_dead(unsigned int cpu); void generic_set_cpu_up(unsigned int cpu); int generic_check_cpu_restart(unsigned int cpu); - -extern void inhibit_secondary_onlining(void); -extern void uninhibit_secondary_onlining(void); - -#else /* HOTPLUG_CPU */ -static inline void inhibit_secondary_onlining(void) {} -static inline void uninhibit_secondary_onlining(void) {} - +int is_cpu_dead(unsigned int cpu); +#else +#define generic_set_cpu_up(i) do { } while (0) #endif #ifdef CONFIG_PPC64 @@ -84,7 +92,7 @@ static inline void uninhibit_secondary_onlining(void) {} /* 32-bit */ extern int smp_hw_index[]; -#define raw_smp_processor_id() (current_thread_info()->cpu) +#define raw_smp_processor_id() (current_thread_info()->cpu) #define hard_smp_processor_id() (smp_hw_index[smp_processor_id()]) static inline int get_hard_smp_processor_id(int cpu) @@ -99,7 +107,9 @@ static inline void set_hard_smp_processor_id(int cpu, int phys) #endif DECLARE_PER_CPU(cpumask_var_t, cpu_sibling_map); +DECLARE_PER_CPU(cpumask_var_t, cpu_l2_cache_map); DECLARE_PER_CPU(cpumask_var_t, cpu_core_map); +DECLARE_PER_CPU(cpumask_var_t, cpu_smallcore_map); static inline struct cpumask *cpu_sibling_mask(int cpu) { @@ -111,29 +121,65 @@ static inline struct cpumask *cpu_core_mask(int cpu) return per_cpu(cpu_core_map, cpu); } +static inline struct cpumask *cpu_l2_cache_mask(int cpu) +{ + return per_cpu(cpu_l2_cache_map, cpu); +} + +static inline struct cpumask *cpu_smallcore_mask(int cpu) +{ + return per_cpu(cpu_smallcore_map, cpu); +} + extern int cpu_to_core_id(int cpu); +extern bool has_big_cores; +extern bool thread_group_shares_l2; +extern bool thread_group_shares_l3; + +#define cpu_smt_mask cpu_smt_mask +#ifdef CONFIG_SCHED_SMT +static inline const struct cpumask *cpu_smt_mask(int cpu) +{ + if (has_big_cores) + return per_cpu(cpu_smallcore_map, cpu); + + return per_cpu(cpu_sibling_map, cpu); +} +#endif /* CONFIG_SCHED_SMT */ + /* Since OpenPIC has only 4 IPIs, we use slightly different message numbers. * * Make sure this matches openpic_request_IPIs in open_pic.c, or what shows up * in /proc/interrupts will be wrong!!! --Troy */ -#define PPC_MSG_CALL_FUNCTION 0 -#define PPC_MSG_RESCHEDULE 1 -#define PPC_MSG_CALL_FUNC_SINGLE 2 -#define PPC_MSG_DEBUGGER_BREAK 3 +#define PPC_MSG_CALL_FUNCTION 0 +#define PPC_MSG_RESCHEDULE 1 +#define PPC_MSG_TICK_BROADCAST 2 +#define PPC_MSG_NMI_IPI 3 + +/* This is only used by the powernv kernel */ +#define PPC_MSG_RM_HOST_ACTION 4 + +#define NMI_IPI_ALL_OTHERS -2 + +#ifdef CONFIG_NMI_IPI +extern int smp_handle_nmi_ipi(struct pt_regs *regs); +#else +static inline int smp_handle_nmi_ipi(struct pt_regs *regs) { return 0; } +#endif /* for irq controllers that have dedicated ipis per message (4) */ extern int smp_request_message_ipi(int virq, int message); extern const char *smp_ipi_name[]; /* for irq controllers with only a single ipi */ -extern void smp_muxed_ipi_set_data(int cpu, unsigned long data); extern void smp_muxed_ipi_message_pass(int cpu, int msg); +extern void smp_muxed_ipi_set_message(int cpu, int msg); extern irqreturn_t smp_ipi_demux(void); +extern irqreturn_t smp_ipi_demux_relaxed(void); void smp_init_pSeries(void); void smp_init_cell(void); -void smp_init_celleb(void); void smp_setup_cpu_maps(void); extern int __cpu_disable(void); @@ -143,24 +189,34 @@ extern void __cpu_die(unsigned int cpu); /* for UP */ #define hard_smp_processor_id() get_hard_smp_processor_id(0) #define smp_setup_cpu_maps() -static inline void inhibit_secondary_onlining(void) {} -static inline void uninhibit_secondary_onlining(void) {} +#define thread_group_shares_l2 0 +#define thread_group_shares_l3 0 +static inline const struct cpumask *cpu_sibling_mask(int cpu) +{ + return cpumask_of(cpu); +} + +static inline const struct cpumask *cpu_smallcore_mask(int cpu) +{ + return cpumask_of(cpu); +} +static inline const struct cpumask *cpu_l2_cache_mask(int cpu) +{ + return cpumask_of(cpu); +} #endif /* CONFIG_SMP */ #ifdef CONFIG_PPC64 static inline int get_hard_smp_processor_id(int cpu) { - return paca[cpu].hw_cpu_id; + return paca_ptrs[cpu]->hw_cpu_id; } static inline void set_hard_smp_processor_id(int cpu, int phys) { - paca[cpu].hw_cpu_id = phys; + paca_ptrs[cpu]->hw_cpu_id = phys; } - -extern void smp_release_cpus(void); - #else /* 32-bit */ #ifndef CONFIG_SMP @@ -177,11 +233,19 @@ static inline void set_hard_smp_processor_id(int cpu, int phys) #endif /* !CONFIG_SMP */ #endif /* !CONFIG_PPC64 */ +#if defined(CONFIG_PPC64) && (defined(CONFIG_SMP) || defined(CONFIG_KEXEC_CORE)) +extern void smp_release_cpus(void); +#else +static inline void smp_release_cpus(void) { } +#endif + extern int smt_enabled_at_boot; -extern int smp_mpic_probe(void); +extern void smp_mpic_probe(void); extern void smp_mpic_setup_cpu(int cpu); extern int smp_generic_kick_cpu(int nr); +extern int smp_generic_cpu_bootable(unsigned int nr); + extern void smp_generic_give_timebase(void); extern void smp_generic_take_timebase(void); @@ -196,13 +260,13 @@ extern void arch_send_call_function_ipi_mask(const struct cpumask *mask); * 64-bit but defining them all here doesn't harm */ extern void generic_secondary_smp_init(void); -extern void generic_secondary_thread_init(void); extern unsigned long __secondary_hold_spinloop; extern unsigned long __secondary_hold_acknowledge; extern char __secondary_hold; +extern unsigned int booting_thread_hwid; extern void __early_start(void); -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_SMP_H) */ diff --git a/arch/powerpc/include/asm/smu.h b/arch/powerpc/include/asm/smu.h index 6e909f3e6a46..2ac6ab903023 100644 --- a/arch/powerpc/include/asm/smu.h +++ b/arch/powerpc/include/asm/smu.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SMU_H #define _SMU_H @@ -107,7 +108,7 @@ /* * i2c commands * - * To issue an i2c command, first is to send a parameter block to the + * To issue an i2c command, first is to send a parameter block to * the SMU. This is a command of type 0x9a with 9 bytes of header * eventually followed by data for a write: * @@ -154,7 +155,7 @@ * * The Darwin I2C driver is less subtle though. On any non-success status * from the response command, it waits 5ms and tries again up to 20 times, - * it doesn't differenciate between fatal errors or "busy" status. + * it doesn't differentiate between fatal errors or "busy" status. * * This driver provides an asynchronous paramblock based i2c command * interface to be used either directly by low level code or by a higher @@ -185,7 +186,7 @@ * x = processor mask * y = op. point index * z = processor freq. step index - * I haven't yet decyphered result codes + * I haven't yet deciphered result codes * */ #define SMU_CMD_POWER_COMMAND 0xaa @@ -455,7 +456,7 @@ extern void smu_poll(void); /* * Init routine, presence check.... */ -extern int smu_init(void); +int __init smu_init(void); extern int smu_present(void); struct platform_device; extern struct platform_device *smu_get_ofdev(void); @@ -471,14 +472,7 @@ extern int smu_get_rtc_time(struct rtc_time *time, int spinwait); extern int smu_set_rtc_time(struct rtc_time *time, int spinwait); /* - * SMU command buffer absolute address, exported by pmac_setup, - * this is allocated very early during boot. - */ -extern unsigned long smu_cmdbuf_abs; - - -/* - * Kenrel asynchronous i2c interface + * Kernel asynchronous i2c interface */ #define SMU_I2C_READ_MAX 0x1d diff --git a/arch/powerpc/include/asm/sparsemem.h b/arch/powerpc/include/asm/sparsemem.h index f6fc0ee813d7..d072866842e4 100644 --- a/arch/powerpc/include/asm/sparsemem.h +++ b/arch/powerpc/include/asm/sparsemem.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_SPARSEMEM_H #define _ASM_POWERPC_SPARSEMEM_H 1 #ifdef __KERNEL__ @@ -5,19 +6,17 @@ #ifdef CONFIG_SPARSEMEM /* * SECTION_SIZE_BITS 2^N: how big each section will be - * MAX_PHYSADDR_BITS 2^N: how much physical address space we have * MAX_PHYSMEM_BITS 2^N: how much memory we can have in that space */ #define SECTION_SIZE_BITS 24 -#define MAX_PHYSADDR_BITS 46 -#define MAX_PHYSMEM_BITS 46 - #endif /* CONFIG_SPARSEMEM */ #ifdef CONFIG_MEMORY_HOTPLUG -extern int create_section_mapping(unsigned long start, unsigned long end); extern int remove_section_mapping(unsigned long start, unsigned long end); +extern int memory_add_physaddr_to_nid(u64 start); +#define memory_add_physaddr_to_nid memory_add_physaddr_to_nid + #ifdef CONFIG_NUMA extern int hot_add_scn_to_nid(unsigned long scn_addr); #else @@ -27,6 +26,5 @@ static inline int hot_add_scn_to_nid(unsigned long scn_addr) } #endif /* CONFIG_NUMA */ #endif /* CONFIG_MEMORY_HOTPLUG */ - #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_SPARSEMEM_H */ diff --git a/arch/powerpc/include/asm/spinlock.h b/arch/powerpc/include/asm/spinlock.h index 5b23f910ee57..7dafca8e3f02 100644 --- a/arch/powerpc/include/asm/spinlock.h +++ b/arch/powerpc/include/asm/spinlock.h @@ -1,299 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef __ASM_SPINLOCK_H #define __ASM_SPINLOCK_H #ifdef __KERNEL__ -/* - * Simple spin lock operations. - * - * Copyright (C) 2001-2004 Paul Mackerras <paulus@au.ibm.com>, IBM - * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM - * Copyright (C) 2002 Dave Engebretsen <engebret@us.ibm.com>, IBM - * Rework to support virtual processors - * - * Type of int is used as a full 64b word is not necessary. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * - * (the type definitions are in asm/spinlock_types.h) - */ -#include <linux/irqflags.h> -#ifdef CONFIG_PPC64 -#include <asm/paca.h> -#include <asm/hvcall.h> -#endif -#include <asm/asm-compat.h> -#include <asm/synch.h> -#include <asm/ppc-opcode.h> - -#define arch_spin_is_locked(x) ((x)->slock != 0) - -#ifdef CONFIG_PPC64 -/* use 0x800000yy when locked, where yy == CPU number */ -#define LOCK_TOKEN (*(u32 *)(&get_paca()->lock_token)) -#else -#define LOCK_TOKEN 1 -#endif - -#if defined(CONFIG_PPC64) && defined(CONFIG_SMP) -#define CLEAR_IO_SYNC (get_paca()->io_sync = 0) -#define SYNC_IO do { \ - if (unlikely(get_paca()->io_sync)) { \ - mb(); \ - get_paca()->io_sync = 0; \ - } \ - } while (0) +#ifdef CONFIG_PPC_QUEUED_SPINLOCKS +#include <asm/qspinlock.h> +#include <asm/qrwlock.h> #else -#define CLEAR_IO_SYNC -#define SYNC_IO +#include <asm/simple_spinlock.h> #endif -/* - * This returns the old value in the lock, so we succeeded - * in getting the lock if the return value is 0. - */ -static inline unsigned long __arch_spin_trylock(arch_spinlock_t *lock) -{ - unsigned long tmp, token; - - token = LOCK_TOKEN; - __asm__ __volatile__( -"1: " PPC_LWARX(%0,0,%2,1) "\n\ - cmpwi 0,%0,0\n\ - bne- 2f\n\ - stwcx. %1,0,%2\n\ - bne- 1b\n" - PPC_ACQUIRE_BARRIER -"2:" - : "=&r" (tmp) - : "r" (token), "r" (&lock->slock) - : "cr0", "memory"); - - return tmp; -} - -static inline int arch_spin_trylock(arch_spinlock_t *lock) -{ - CLEAR_IO_SYNC; - return __arch_spin_trylock(lock) == 0; -} - -/* - * On a system with shared processors (that is, where a physical - * processor is multiplexed between several virtual processors), - * there is no point spinning on a lock if the holder of the lock - * isn't currently scheduled on a physical processor. Instead - * we detect this situation and ask the hypervisor to give the - * rest of our timeslice to the lock holder. - * - * So that we can tell which virtual processor is holding a lock, - * we put 0x80000000 | smp_processor_id() in the lock when it is - * held. Conveniently, we have a word in the paca that holds this - * value. - */ +/* See include/linux/spinlock.h */ +#define smp_mb__after_spinlock() smp_mb() -#if defined(CONFIG_PPC_SPLPAR) -/* We only yield to the hypervisor if we are in shared processor mode */ -#define SHARED_PROCESSOR (local_paca->lppaca_ptr->shared_proc) -extern void __spin_yield(arch_spinlock_t *lock); -extern void __rw_yield(arch_rwlock_t *lock); -#else /* SPLPAR */ -#define __spin_yield(x) barrier() -#define __rw_yield(x) barrier() -#define SHARED_PROCESSOR 0 +#ifndef CONFIG_PPC_QUEUED_SPINLOCKS +static inline void pv_spinlocks_init(void) { } #endif -static inline void arch_spin_lock(arch_spinlock_t *lock) -{ - CLEAR_IO_SYNC; - while (1) { - if (likely(__arch_spin_trylock(lock) == 0)) - break; - do { - HMT_low(); - if (SHARED_PROCESSOR) - __spin_yield(lock); - } while (unlikely(lock->slock != 0)); - HMT_medium(); - } -} - -static inline -void arch_spin_lock_flags(arch_spinlock_t *lock, unsigned long flags) -{ - unsigned long flags_dis; - - CLEAR_IO_SYNC; - while (1) { - if (likely(__arch_spin_trylock(lock) == 0)) - break; - local_save_flags(flags_dis); - local_irq_restore(flags); - do { - HMT_low(); - if (SHARED_PROCESSOR) - __spin_yield(lock); - } while (unlikely(lock->slock != 0)); - HMT_medium(); - local_irq_restore(flags_dis); - } -} - -static inline void arch_spin_unlock(arch_spinlock_t *lock) -{ - SYNC_IO; - __asm__ __volatile__("# arch_spin_unlock\n\t" - PPC_RELEASE_BARRIER: : :"memory"); - lock->slock = 0; -} - -#ifdef CONFIG_PPC64 -extern void arch_spin_unlock_wait(arch_spinlock_t *lock); -#else -#define arch_spin_unlock_wait(lock) \ - do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0) -#endif - -/* - * Read-write spinlocks, allowing multiple readers - * but only one writer. - * - * NOTE! it is quite common to have readers in interrupts - * but no interrupt writers. For those circumstances we - * can "mix" irq-safe locks - any writer needs to get a - * irq-safe write-lock, but readers can get non-irqsafe - * read-locks. - */ - -#define arch_read_can_lock(rw) ((rw)->lock >= 0) -#define arch_write_can_lock(rw) (!(rw)->lock) - -#ifdef CONFIG_PPC64 -#define __DO_SIGN_EXTEND "extsw %0,%0\n" -#define WRLOCK_TOKEN LOCK_TOKEN /* it's negative */ -#else -#define __DO_SIGN_EXTEND -#define WRLOCK_TOKEN (-1) -#endif - -/* - * This returns the old value in the lock + 1, - * so we got a read lock if the return value is > 0. - */ -static inline long __arch_read_trylock(arch_rwlock_t *rw) -{ - long tmp; - - __asm__ __volatile__( -"1: " PPC_LWARX(%0,0,%1,1) "\n" - __DO_SIGN_EXTEND -" addic. %0,%0,1\n\ - ble- 2f\n" - PPC405_ERR77(0,%1) -" stwcx. %0,0,%1\n\ - bne- 1b\n" - PPC_ACQUIRE_BARRIER -"2:" : "=&r" (tmp) - : "r" (&rw->lock) - : "cr0", "xer", "memory"); - - return tmp; -} - -/* - * This returns the old value in the lock, - * so we got the write lock if the return value is 0. - */ -static inline long __arch_write_trylock(arch_rwlock_t *rw) -{ - long tmp, token; - - token = WRLOCK_TOKEN; - __asm__ __volatile__( -"1: " PPC_LWARX(%0,0,%2,1) "\n\ - cmpwi 0,%0,0\n\ - bne- 2f\n" - PPC405_ERR77(0,%1) -" stwcx. %1,0,%2\n\ - bne- 1b\n" - PPC_ACQUIRE_BARRIER -"2:" : "=&r" (tmp) - : "r" (token), "r" (&rw->lock) - : "cr0", "memory"); - - return tmp; -} - -static inline void arch_read_lock(arch_rwlock_t *rw) -{ - while (1) { - if (likely(__arch_read_trylock(rw) > 0)) - break; - do { - HMT_low(); - if (SHARED_PROCESSOR) - __rw_yield(rw); - } while (unlikely(rw->lock < 0)); - HMT_medium(); - } -} - -static inline void arch_write_lock(arch_rwlock_t *rw) -{ - while (1) { - if (likely(__arch_write_trylock(rw) == 0)) - break; - do { - HMT_low(); - if (SHARED_PROCESSOR) - __rw_yield(rw); - } while (unlikely(rw->lock != 0)); - HMT_medium(); - } -} - -static inline int arch_read_trylock(arch_rwlock_t *rw) -{ - return __arch_read_trylock(rw) > 0; -} - -static inline int arch_write_trylock(arch_rwlock_t *rw) -{ - return __arch_write_trylock(rw) == 0; -} - -static inline void arch_read_unlock(arch_rwlock_t *rw) -{ - long tmp; - - __asm__ __volatile__( - "# read_unlock\n\t" - PPC_RELEASE_BARRIER -"1: lwarx %0,0,%1\n\ - addic %0,%0,-1\n" - PPC405_ERR77(0,%1) -" stwcx. %0,0,%1\n\ - bne- 1b" - : "=&r"(tmp) - : "r"(&rw->lock) - : "cr0", "xer", "memory"); -} - -static inline void arch_write_unlock(arch_rwlock_t *rw) -{ - __asm__ __volatile__("# write_unlock\n\t" - PPC_RELEASE_BARRIER: : :"memory"); - rw->lock = 0; -} - -#define arch_read_lock_flags(lock, flags) arch_read_lock(lock) -#define arch_write_lock_flags(lock, flags) arch_write_lock(lock) - -#define arch_spin_relax(lock) __spin_yield(lock) -#define arch_read_relax(lock) __rw_yield(lock) -#define arch_write_relax(lock) __rw_yield(lock) - #endif /* __KERNEL__ */ #endif /* __ASM_SPINLOCK_H */ diff --git a/arch/powerpc/include/asm/spinlock_types.h b/arch/powerpc/include/asm/spinlock_types.h index 2351adc4fdc4..569765fa16bc 100644 --- a/arch/powerpc/include/asm/spinlock_types.h +++ b/arch/powerpc/include/asm/spinlock_types.h @@ -1,20 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_SPINLOCK_TYPES_H #define _ASM_POWERPC_SPINLOCK_TYPES_H -#ifndef __LINUX_SPINLOCK_TYPES_H -# error "please don't include this file directly" +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H +# error "Please do not include this file directly." #endif -typedef struct { - volatile unsigned int slock; -} arch_spinlock_t; - -#define __ARCH_SPIN_LOCK_UNLOCKED { 0 } - -typedef struct { - volatile signed int lock; -} arch_rwlock_t; - -#define __ARCH_RW_LOCK_UNLOCKED { 0 } +#ifdef CONFIG_PPC_QUEUED_SPINLOCKS +#include <asm/qspinlock_types.h> +#include <asm-generic/qrwlock_types.h> +#else +#include <asm/simple_spinlock_types.h> +#endif #endif diff --git a/arch/powerpc/include/asm/spu.h b/arch/powerpc/include/asm/spu.h index 93f280e23279..96ad4510c895 100644 --- a/arch/powerpc/include/asm/spu.h +++ b/arch/powerpc/include/asm/spu.h @@ -1,23 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * SPU core / file system interface and HW structures * * (C) Copyright IBM Deutschland Entwicklung GmbH 2005 * * Author: Arnd Bergmann <arndb@de.ibm.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2, or (at your option) - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #ifndef _SPU_H @@ -27,6 +14,8 @@ #include <linux/workqueue.h> #include <linux/device.h> #include <linux/mutex.h> +#include <asm/reg.h> +#include <asm/copro.h> #define LS_SIZE (256 * 1024) #define LS_ADDR_MASK (LS_SIZE - 1) @@ -212,20 +201,6 @@ int spu_64k_pages_available(void); struct mm_struct; extern void spu_flush_all_slbs(struct mm_struct *mm); -/* This interface allows a profiler (e.g., OProfile) to store a ref - * to spu context information that it creates. This caching technique - * avoids the need to recreate this information after a save/restore operation. - * - * Assumes the caller has already incremented the ref count to - * profile_info; then spu_context_destroy must call kref_put - * on prof_info_kref. - */ -void spu_set_profile_private_kref(struct spu_context *ctx, - struct kref *prof_info_kref, - void ( * prof_info_release) (struct kref *kref)); - -void *spu_get_profile_private_kref(struct spu_context *ctx); - /* system callbacks from the SPU */ struct spu_syscall_block { u64 nr_ret; @@ -235,6 +210,7 @@ extern long spu_sys_callback(struct spu_syscall_block *s); /* syscalls implemented in spufs */ struct file; +struct coredump_params; struct spufs_calls { long (*create_thread)(const char __user *name, unsigned int flags, umode_t mode, @@ -242,7 +218,7 @@ struct spufs_calls { long (*spu_run)(struct file *filp, __u32 __user *unpc, __u32 __user *ustatus); int (*coredump_extra_notes_size)(void); - int (*coredump_extra_notes_write)(struct file *file, loff_t *foffset); + int (*coredump_extra_notes_write)(struct coredump_params *cprm); void (*notify_spus_active)(void); struct module *owner; }; @@ -273,30 +249,8 @@ void unregister_spu_syscalls(struct spufs_calls *calls); int spu_add_dev_attr(struct device_attribute *attr); void spu_remove_dev_attr(struct device_attribute *attr); -int spu_add_dev_attr_group(struct attribute_group *attrs); -void spu_remove_dev_attr_group(struct attribute_group *attrs); - -int spu_handle_mm_fault(struct mm_struct *mm, unsigned long ea, - unsigned long dsisr, unsigned *flt); - -/* - * Notifier blocks: - * - * oprofile can get notified when a context switch is performed - * on an spe. The notifer function that gets called is passed - * a pointer to the SPU structure as well as the object-id that - * identifies the binary running on that SPU now. - * - * For a context save, the object-id that is passed is zero, - * identifying that the kernel will run from that moment on. - * - * For a context restore, the object-id is the value written - * to object-id spufs file from user space and the notifer - * function can assume that spu->ctx is valid. - */ -struct notifier_block; -int spu_switch_event_register(struct notifier_block * n); -int spu_switch_event_unregister(struct notifier_block * n); +int spu_add_dev_attr_group(const struct attribute_group *attrs); +void spu_remove_dev_attr_group(const struct attribute_group *attrs); extern void notify_spus_active(void); extern void do_notify_spus_active(void); diff --git a/arch/powerpc/include/asm/spu_csa.h b/arch/powerpc/include/asm/spu_csa.h index a40fd491250c..1b3271a03392 100644 --- a/arch/powerpc/include/asm/spu_csa.h +++ b/arch/powerpc/include/asm/spu_csa.h @@ -1,23 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * spu_csa.h: Definitions for SPU context save area (CSA). * * (C) Copyright IBM 2005 * * Author: Mark Nutter <mnutter@us.ibm.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2, or (at your option) - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #ifndef _SPU_CSA_H_ @@ -56,7 +43,7 @@ #define SPU_DECR_STATUS_RUNNING 0x1 #define SPU_DECR_STATUS_WRAPPED 0x2 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /** * spu_reg128 - generic 128-bit register definition. */ @@ -241,12 +228,6 @@ struct spu_priv2_collapsed { */ struct spu_state { struct spu_lscsa *lscsa; -#ifdef CONFIG_SPU_FS_64K_LS - int use_big_pages; - /* One struct page per 64k page */ -#define SPU_LSCSA_NUM_BIG_PAGES (sizeof(struct spu_lscsa) / 0x10000) - struct page *lscsa_pages[SPU_LSCSA_NUM_BIG_PAGES]; -#endif struct spu_problem_collapsed prob; struct spu_priv1_collapsed priv1; struct spu_priv2_collapsed priv2; @@ -262,5 +243,5 @@ struct spu_state { #endif /* !__SPU__ */ #endif /* __KERNEL__ */ -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* _SPU_CSA_H_ */ diff --git a/arch/powerpc/include/asm/spu_info.h b/arch/powerpc/include/asm/spu_info.h index 7146b78e40f1..732431034a63 100644 --- a/arch/powerpc/include/asm/spu_info.h +++ b/arch/powerpc/include/asm/spu_info.h @@ -1,23 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * SPU info structures * * (C) Copyright 2006 IBM Corp. * * Author: Dwayne Grant McConnell <decimal@us.ibm.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2, or (at your option) - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #ifndef _SPU_INFO_H #define _SPU_INFO_H diff --git a/arch/powerpc/include/asm/spu_priv1.h b/arch/powerpc/include/asm/spu_priv1.h index d8f5c60f61c1..66b111fa1cd1 100644 --- a/arch/powerpc/include/asm/spu_priv1.h +++ b/arch/powerpc/include/asm/spu_priv1.h @@ -1,20 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Defines an spu hypervisor abstraction layer. * * Copyright 2006 Sony Corp. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #if !defined(_SPU_PRIV1_H) @@ -227,9 +215,6 @@ spu_disable_spu (struct spu_context *ctx) * and only intended to be used by the platform setup code. */ -extern const struct spu_priv1_ops spu_priv1_mmio_ops; -extern const struct spu_priv1_ops spu_priv1_beat_ops; - extern const struct spu_management_ops spu_management_of_ops; #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/sstep.h b/arch/powerpc/include/asm/sstep.h index f593b0f9b627..e3d0e714ff28 100644 --- a/arch/powerpc/include/asm/sstep.h +++ b/arch/powerpc/include/asm/sstep.h @@ -1,11 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Copyright (C) 2004 Paul Mackerras <paulus@au.ibm.com>, IBM - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ +#include <asm/inst.h> struct pt_regs; @@ -16,12 +13,164 @@ struct pt_regs; * we don't allow putting a breakpoint on an mtmsrd instruction. * Similarly we don't allow breakpoints on rfid instructions. * These macros tell us if an instruction is a mtmsrd or rfid. - * Note that IS_MTMSRD returns true for both an mtmsr (32-bit) - * and an mtmsrd (64-bit). + * Note that these return true for both mtmsr/rfi (32-bit) + * and mtmsrd/rfid (64-bit). + */ +#define IS_MTMSRD(instr) ((ppc_inst_val(instr) & 0xfc0007be) == 0x7c000124) +#define IS_RFID(instr) ((ppc_inst_val(instr) & 0xfc0007be) == 0x4c000024) + +enum instruction_type { + COMPUTE, /* arith/logical/CR op, etc. */ + LOAD, /* load and store types need to be contiguous */ + LOAD_MULTI, + LOAD_FP, + LOAD_VMX, + LOAD_VSX, + STORE, + STORE_MULTI, + STORE_FP, + STORE_VMX, + STORE_VSX, + LARX, + STCX, + BRANCH, + MFSPR, + MTSPR, + CACHEOP, + BARRIER, + SYSCALL, + SYSCALL_VECTORED_0, + MFMSR, + MTMSR, + RFI, + INTERRUPT, + UNKNOWN +}; + +#define INSTR_TYPE_MASK 0x1f + +#define OP_IS_LOAD(type) ((LOAD <= (type) && (type) <= LOAD_VSX) || (type) == LARX) +#define OP_IS_STORE(type) ((STORE <= (type) && (type) <= STORE_VSX) || (type) == STCX) +#define OP_IS_LOAD_STORE(type) (LOAD <= (type) && (type) <= STCX) + +/* Compute flags, ORed in with type */ +#define SETREG 0x20 +#define SETCC 0x40 +#define SETXER 0x80 + +/* Branch flags, ORed in with type */ +#define SETLK 0x20 +#define BRTAKEN 0x40 +#define DECCTR 0x80 + +/* Load/store flags, ORed in with type */ +#define SIGNEXT 0x20 +#define UPDATE 0x40 /* matches bit in opcode 31 instructions */ +#define BYTEREV 0x80 +#define FPCONV 0x100 + +/* Barrier type field, ORed in with type */ +#define BARRIER_MASK 0xe0 +#define BARRIER_SYNC 0x00 +#define BARRIER_ISYNC 0x20 +#define BARRIER_EIEIO 0x40 +#define BARRIER_LWSYNC 0x60 +#define BARRIER_PTESYNC 0x80 + +/* Cacheop values, ORed in with type */ +#define CACHEOP_MASK 0x700 +#define DCBST 0 +#define DCBF 0x100 +#define DCBTST 0x200 +#define DCBT 0x300 +#define ICBI 0x400 +#define DCBZ 0x500 + +/* VSX flags values */ +#define VSX_FPCONV 1 /* do floating point SP/DP conversion */ +#define VSX_SPLAT 2 /* store loaded value into all elements */ +#define VSX_LDLEFT 4 /* load VSX register from left */ +#define VSX_CHECK_VEC 8 /* check MSR_VEC not MSR_VSX for reg >= 32 */ + +/* Prefixed flag, ORed in with type */ +#define PREFIXED 0x800 + +/* Size field in type word */ +#define SIZE(n) ((n) << 12) +#define GETSIZE(w) ((w) >> 12) + +#define GETTYPE(t) ((t) & INSTR_TYPE_MASK) +#define GETLENGTH(t) (((t) & PREFIXED) ? 8 : 4) + +#define MKOP(t, f, s) ((t) | (f) | SIZE(s)) + +/* Prefix instruction operands */ +#define GET_PREFIX_RA(i) (((i) >> 16) & 0x1f) +#define GET_PREFIX_R(i) ((i) & (1ul << 20)) + +extern s32 patch__exec_instr; + +struct instruction_op { + int type; + int reg; + unsigned long val; + /* For LOAD/STORE/LARX/STCX */ + unsigned long ea; + int update_reg; + /* For MFSPR */ + int spr; + u32 ccval; + u32 xerval; + u8 element_size; /* for VSX/VMX loads/stores */ + u8 vsx_flags; +}; + +union vsx_reg { + u8 b[16]; + u16 h[8]; + u32 w[4]; + unsigned long d[2]; + float fp[4]; + double dp[2]; + __vector128 v; +}; + +/* + * Decode an instruction, and return information about it in *op + * without changing *regs. + * + * Return value is 1 if the instruction can be emulated just by + * updating *regs with the information in *op, -1 if we need the + * GPRs but *regs doesn't contain the full register set, or 0 + * otherwise. + */ +extern int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, + ppc_inst_t instr); + +/* + * Emulate an instruction that can be executed just by updating + * fields in *regs. + */ +void emulate_update_regs(struct pt_regs *reg, struct instruction_op *op); + +/* + * Emulate instructions that cause a transfer of control, + * arithmetic/logical instructions, loads and stores, + * cache operations and barriers. + * + * Returns 1 if the instruction was emulated successfully, + * 0 if it could not be emulated, or -1 for an instruction that + * should not be emulated (rfid, mtmsrd clearing MSR_RI, etc.). + */ +int emulate_step(struct pt_regs *regs, ppc_inst_t instr); + +/* + * Emulate a load or store instruction by reading/writing the + * memory of the current process. FP/VMX/VSX registers are assumed + * to hold live values if the appropriate enable bit in regs->msr is + * set; otherwise this will use the saved values in the thread struct + * for user-mode accesses. */ -#define IS_MTMSRD(instr) (((instr) & 0xfc0007be) == 0x7c000124) -#define IS_RFID(instr) (((instr) & 0xfc0007fe) == 0x4c000024) -#define IS_RFI(instr) (((instr) & 0xfc0007fe) == 0x4c000064) +extern int emulate_loadstore(struct pt_regs *regs, struct instruction_op *op); -/* Emulate instructions that cause a transfer of control. */ -extern int emulate_step(struct pt_regs *regs, unsigned int instr); +extern int emulate_dcbz(unsigned long ea, struct pt_regs *regs); diff --git a/arch/powerpc/include/asm/stackprotector.h b/arch/powerpc/include/asm/stackprotector.h new file mode 100644 index 000000000000..283c34647856 --- /dev/null +++ b/arch/powerpc/include/asm/stackprotector.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * GCC stack protector support. + * + */ + +#ifndef _ASM_STACKPROTECTOR_H +#define _ASM_STACKPROTECTOR_H + +#include <asm/reg.h> +#include <asm/current.h> +#include <asm/paca.h> + +/* + * Initialize the stackprotector canary value. + * + * NOTE: this must only be called from functions that never return, + * and it must always be inlined. + */ +static __always_inline void boot_init_stack_canary(void) +{ + unsigned long canary = get_random_canary(); + + current->stack_canary = canary; +#ifdef CONFIG_PPC64 + get_paca()->canary = canary; +#endif +} + +#endif /* _ASM_STACKPROTECTOR_H */ diff --git a/arch/powerpc/include/asm/stacktrace.h b/arch/powerpc/include/asm/stacktrace.h new file mode 100644 index 000000000000..6149b53b3bc8 --- /dev/null +++ b/arch/powerpc/include/asm/stacktrace.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Stack trace functions. + * + * Copyright 2018, Murilo Opsfelder Araujo, IBM Corporation. + */ + +#ifndef _ASM_POWERPC_STACKTRACE_H +#define _ASM_POWERPC_STACKTRACE_H + +void show_user_instructions(struct pt_regs *regs); + +#endif /* _ASM_POWERPC_STACKTRACE_H */ diff --git a/arch/powerpc/include/asm/static_call.h b/arch/powerpc/include/asm/static_call.h new file mode 100644 index 000000000000..e3d5d3823dac --- /dev/null +++ b/arch/powerpc/include/asm/static_call.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_STATIC_CALL_H +#define _ASM_POWERPC_STATIC_CALL_H + +#define __PPC_SCT(name, inst) \ + asm(".pushsection .text, \"ax\" \n" \ + ".align 5 \n" \ + ".globl " STATIC_CALL_TRAMP_STR(name) " \n" \ + STATIC_CALL_TRAMP_STR(name) ": \n" \ + inst " \n" \ + " lis 12,2f@ha \n" \ + " lwz 12,2f@l(12) \n" \ + " mtctr 12 \n" \ + " bctr \n" \ + "1: li 3, 0 \n" \ + " blr \n" \ + "2: .long 0 \n" \ + ".type " STATIC_CALL_TRAMP_STR(name) ", @function \n" \ + ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \ + ".popsection \n") + +#define PPC_SCT_RET0 20 /* Offset of label 1 */ +#define PPC_SCT_DATA 28 /* Offset of label 2 */ + +#define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func) __PPC_SCT(name, "b " #func) +#define ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) __PPC_SCT(name, "blr") +#define ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name) __PPC_SCT(name, "b .+20") + +#define CALL_INSN_SIZE 4 + +#endif /* _ASM_POWERPC_STATIC_CALL_H */ diff --git a/arch/powerpc/include/asm/string.h b/arch/powerpc/include/asm/string.h index e40010abcaf1..60ba22770f51 100644 --- a/arch/powerpc/include/asm/string.h +++ b/arch/powerpc/include/asm/string.h @@ -1,19 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_STRING_H #define _ASM_POWERPC_STRING_H #ifdef __KERNEL__ -#define __HAVE_ARCH_STRCPY +#ifndef CONFIG_KASAN #define __HAVE_ARCH_STRNCPY -#define __HAVE_ARCH_STRLEN -#define __HAVE_ARCH_STRCMP #define __HAVE_ARCH_STRNCMP -#define __HAVE_ARCH_STRCAT +#define __HAVE_ARCH_MEMCHR +#define __HAVE_ARCH_MEMCMP +#define __HAVE_ARCH_MEMSET16 +#endif + #define __HAVE_ARCH_MEMSET #define __HAVE_ARCH_MEMCPY #define __HAVE_ARCH_MEMMOVE -#define __HAVE_ARCH_MEMCMP -#define __HAVE_ARCH_MEMCHR +#define __HAVE_ARCH_MEMCPY_FLUSHCACHE extern char * strcpy(char *,const char *); extern char * strncpy(char *,const char *, __kernel_size_t); @@ -26,7 +28,65 @@ extern void * memcpy(void *,const void *,__kernel_size_t); extern void * memmove(void *,const void *,__kernel_size_t); extern int memcmp(const void *,const void *,__kernel_size_t); extern void * memchr(const void *,int,__kernel_size_t); +void memcpy_flushcache(void *dest, const void *src, size_t size); + +#ifdef CONFIG_KASAN +/* __mem variants are used by KASAN to implement instrumented meminstrinsics. */ +#ifdef CONFIG_CC_HAS_KASAN_MEMINTRINSIC_PREFIX +#define __memset memset +#define __memcpy memcpy +#define __memmove memmove +#else /* CONFIG_CC_HAS_KASAN_MEMINTRINSIC_PREFIX */ +void *__memset(void *s, int c, __kernel_size_t count); +void *__memcpy(void *to, const void *from, __kernel_size_t n); +void *__memmove(void *to, const void *from, __kernel_size_t n); +#ifndef __SANITIZE_ADDRESS__ +/* + * For files that are not instrumented (e.g. mm/slub.c) we + * should use not instrumented version of mem* functions. + */ +#define memcpy(dst, src, len) __memcpy(dst, src, len) +#define memmove(dst, src, len) __memmove(dst, src, len) +#define memset(s, c, n) __memset(s, c, n) + +#ifndef __NO_FORTIFY +#define __NO_FORTIFY /* FORTIFY_SOURCE uses __builtin_memcpy, etc. */ +#endif +#endif /* !__SANITIZE_ADDRESS__ */ +#endif /* CONFIG_CC_HAS_KASAN_MEMINTRINSIC_PREFIX */ +#endif /* CONFIG_KASAN */ + +#ifdef CONFIG_PPC64 +#ifndef CONFIG_KASAN +#define __HAVE_ARCH_MEMSET32 +#define __HAVE_ARCH_MEMSET64 + +extern void *__memset16(uint16_t *, uint16_t v, __kernel_size_t); +extern void *__memset32(uint32_t *, uint32_t v, __kernel_size_t); +extern void *__memset64(uint64_t *, uint64_t v, __kernel_size_t); + +static inline void *memset16(uint16_t *p, uint16_t v, __kernel_size_t n) +{ + return __memset16(p, v, n * 2); +} + +static inline void *memset32(uint32_t *p, uint32_t v, __kernel_size_t n) +{ + return __memset32(p, v, n * 4); +} + +static inline void *memset64(uint64_t *p, uint64_t v, __kernel_size_t n) +{ + return __memset64(p, v, n * 8); +} +#endif +#else +#ifndef CONFIG_KASAN +#define __HAVE_ARCH_STRLEN +#endif +extern void *memset16(uint16_t *, uint16_t, __kernel_size_t); +#endif #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_STRING_H */ diff --git a/arch/powerpc/include/asm/svm.h b/arch/powerpc/include/asm/svm.h new file mode 100644 index 000000000000..a02bd54b8948 --- /dev/null +++ b/arch/powerpc/include/asm/svm.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * SVM helper functions + * + * Copyright 2018 Anshuman Khandual, IBM Corporation. + */ + +#ifndef _ASM_POWERPC_SVM_H +#define _ASM_POWERPC_SVM_H + +#ifdef CONFIG_PPC_SVM + +#include <asm/reg.h> + +static inline bool is_secure_guest(void) +{ + return mfmsr() & MSR_S; +} + +void dtl_cache_ctor(void *addr); +#define get_dtl_cache_ctor() (is_secure_guest() ? dtl_cache_ctor : NULL) + +#else /* CONFIG_PPC_SVM */ + +static inline bool is_secure_guest(void) +{ + return false; +} + +#define get_dtl_cache_ctor() NULL + +#endif /* CONFIG_PPC_SVM */ +#endif /* _ASM_POWERPC_SVM_H */ diff --git a/arch/powerpc/include/asm/swab.h b/arch/powerpc/include/asm/swab.h index b9bd1ca944d0..f4cfdc1246d3 100644 --- a/arch/powerpc/include/asm/swab.h +++ b/arch/powerpc/include/asm/swab.h @@ -1,81 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef _ASM_POWERPC_SWAB_H #define _ASM_POWERPC_SWAB_H #include <uapi/asm/swab.h> -#ifdef __GNUC__ -#ifndef __powerpc64__ -#endif /* __powerpc64__ */ - -static __inline__ __u16 ld_le16(const volatile __u16 *addr) -{ - __u16 val; - - __asm__ __volatile__ ("lhbrx %0,0,%1" : "=r" (val) : "r" (addr), "m" (*addr)); - return val; -} -#define __arch_swab16p ld_le16 - -static __inline__ void st_le16(volatile __u16 *addr, const __u16 val) -{ - __asm__ __volatile__ ("sthbrx %1,0,%2" : "=m" (*addr) : "r" (val), "r" (addr)); -} - -static inline void __arch_swab16s(__u16 *addr) -{ - st_le16(addr, *addr); -} -#define __arch_swab16s __arch_swab16s - -static __inline__ __u32 ld_le32(const volatile __u32 *addr) -{ - __u32 val; - - __asm__ __volatile__ ("lwbrx %0,0,%1" : "=r" (val) : "r" (addr), "m" (*addr)); - return val; -} -#define __arch_swab32p ld_le32 - -static __inline__ void st_le32(volatile __u32 *addr, const __u32 val) -{ - __asm__ __volatile__ ("stwbrx %1,0,%2" : "=m" (*addr) : "r" (val), "r" (addr)); -} - -static inline void __arch_swab32s(__u32 *addr) -{ - st_le32(addr, *addr); -} -#define __arch_swab32s __arch_swab32s - -static inline __attribute_const__ __u16 __arch_swab16(__u16 value) -{ - __u16 result; - - __asm__("rlwimi %0,%1,8,16,23" - : "=r" (result) - : "r" (value), "0" (value >> 8)); - return result; -} -#define __arch_swab16 __arch_swab16 - -static inline __attribute_const__ __u32 __arch_swab32(__u32 value) -{ - __u32 result; - - __asm__("rlwimi %0,%1,24,16,23\n\t" - "rlwimi %0,%1,8,8,15\n\t" - "rlwimi %0,%1,24,0,7" - : "=r" (result) - : "r" (value), "0" (value >> 24)); - return result; -} -#define __arch_swab32 __arch_swab32 - -#endif /* __GNUC__ */ #endif /* _ASM_POWERPC_SWAB_H */ diff --git a/arch/powerpc/include/asm/swiotlb.h b/arch/powerpc/include/asm/swiotlb.h index de99d6e29430..4203b5e0a88e 100644 --- a/arch/powerpc/include/asm/swiotlb.h +++ b/arch/powerpc/include/asm/swiotlb.h @@ -1,11 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Copyright (C) 2009 Becky Bruce, Freescale Semiconductor - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * */ #ifndef __ASM_SWIOTLB_H @@ -13,14 +8,8 @@ #include <linux/swiotlb.h> -extern struct dma_map_ops swiotlb_dma_ops; - -static inline void dma_mark_clean(void *addr, size_t size) {} - extern unsigned int ppc_swiotlb_enable; -int __init swiotlb_setup_bus_notifier(void); - -extern void pci_dma_dev_setup_swiotlb(struct pci_dev *pdev); +extern unsigned int ppc_swiotlb_flags; #ifdef CONFIG_SWIOTLB void swiotlb_detect_4g(void); diff --git a/arch/powerpc/include/asm/switch_to.h b/arch/powerpc/include/asm/switch_to.h index 49a13e0ef234..fc933807ddc8 100644 --- a/arch/powerpc/include/asm/switch_to.h +++ b/arch/powerpc/include/asm/switch_to.h @@ -1,9 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu> */ #ifndef _ASM_POWERPC_SWITCH_TO_H #define _ASM_POWERPC_SWITCH_TO_H +#include <linux/sched.h> +#include <asm/reg.h> + struct thread_struct; struct task_struct; struct pt_regs; @@ -12,65 +16,105 @@ extern struct task_struct *__switch_to(struct task_struct *, struct task_struct *); #define switch_to(prev, next, last) ((last) = __switch_to((prev), (next))) -struct thread_struct; extern struct task_struct *_switch(struct thread_struct *prev, struct thread_struct *next); -extern void giveup_fpu(struct task_struct *); -extern void load_up_fpu(void); -extern void disable_kernel_fp(void); -extern void enable_kernel_fp(void); -extern void flush_fp_to_thread(struct task_struct *); -extern void enable_kernel_altivec(void); -extern void load_up_altivec(struct task_struct *); +extern void switch_booke_debug_regs(struct debug_reg *new_debug); + extern int emulate_altivec(struct pt_regs *); -extern void __giveup_vsx(struct task_struct *); -extern void giveup_vsx(struct task_struct *); -extern void enable_kernel_spe(void); -extern void giveup_spe(struct task_struct *); -extern void load_up_spe(struct task_struct *); -#ifndef CONFIG_SMP -extern void discard_lazy_cpu_state(void); +#ifdef CONFIG_PPC_BOOK3S_64 +void restore_math(struct pt_regs *regs); #else -static inline void discard_lazy_cpu_state(void) +static inline void restore_math(struct pt_regs *regs) { } #endif +void restore_tm_state(struct pt_regs *regs); + +extern void flush_all_to_thread(struct task_struct *); +extern void giveup_all(struct task_struct *); + +#ifdef CONFIG_PPC_FPU +extern void enable_kernel_fp(void); +extern void flush_fp_to_thread(struct task_struct *); +extern void giveup_fpu(struct task_struct *); +extern void save_fpu(struct task_struct *); +static inline void disable_kernel_fp(void) +{ + msr_check_and_clear(MSR_FP); +} +#else +static inline void save_fpu(struct task_struct *t) { } +static inline void flush_fp_to_thread(struct task_struct *t) { } +static inline void enable_kernel_fp(void) +{ + BUILD_BUG(); +} +#endif + #ifdef CONFIG_ALTIVEC +extern void enable_kernel_altivec(void); extern void flush_altivec_to_thread(struct task_struct *); extern void giveup_altivec(struct task_struct *); -extern void giveup_altivec_notask(void); +extern void save_altivec(struct task_struct *); +static inline void disable_kernel_altivec(void) +{ + msr_check_and_clear(MSR_VEC); +} #else -static inline void flush_altivec_to_thread(struct task_struct *t) +static inline void save_altivec(struct task_struct *t) { } +static inline void __giveup_altivec(struct task_struct *t) { } +static inline void enable_kernel_altivec(void) { + BUILD_BUG(); } -static inline void giveup_altivec(struct task_struct *t) + +static inline void disable_kernel_altivec(void) { + BUILD_BUG(); } #endif #ifdef CONFIG_VSX +extern void enable_kernel_vsx(void); extern void flush_vsx_to_thread(struct task_struct *); +static inline void disable_kernel_vsx(void) +{ + msr_check_and_clear(MSR_FP|MSR_VEC|MSR_VSX); +} #else -static inline void flush_vsx_to_thread(struct task_struct *t) +static inline void enable_kernel_vsx(void) { + BUILD_BUG(); +} + +static inline void disable_kernel_vsx(void) +{ + BUILD_BUG(); } #endif #ifdef CONFIG_SPE +extern void enable_kernel_spe(void); extern void flush_spe_to_thread(struct task_struct *); -#else -static inline void flush_spe_to_thread(struct task_struct *t) +extern void giveup_spe(struct task_struct *); +extern void __giveup_spe(struct task_struct *); +static inline void disable_kernel_spe(void) { + msr_check_and_clear(MSR_SPE); } +#else +static inline void __giveup_spe(struct task_struct *t) { } #endif static inline void clear_task_ebb(struct task_struct *t) { #ifdef CONFIG_PPC_BOOK3S_64 /* EBB perf events are not inherited, so clear all EBB state. */ + t->thread.ebbrr = 0; + t->thread.ebbhr = 0; t->thread.bescr = 0; t->thread.mmcr2 = 0; t->thread.mmcr0 = 0; @@ -81,4 +125,9 @@ static inline void clear_task_ebb(struct task_struct *t) #endif } +void kvmppc_save_user_regs(void); +void kvmppc_save_current_sprs(void); + +extern int set_thread_tidr(struct task_struct *t); + #endif /* _ASM_POWERPC_SWITCH_TO_H */ diff --git a/arch/powerpc/include/asm/synch.h b/arch/powerpc/include/asm/synch.h index e682a7143edb..0d3ccb34adfb 100644 --- a/arch/powerpc/include/asm/synch.h +++ b/arch/powerpc/include/asm/synch.h @@ -1,34 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_SYNCH_H #define _ASM_POWERPC_SYNCH_H #ifdef __KERNEL__ -#include <linux/stringify.h> +#include <asm/cputable.h> #include <asm/feature-fixups.h> +#include <asm/ppc-opcode.h> -#if defined(__powerpc64__) || defined(CONFIG_PPC_E500MC) -#define __SUBARCH_HAS_LWSYNC -#endif - -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ extern unsigned int __start___lwsync_fixup, __stop___lwsync_fixup; extern void do_lwsync_fixups(unsigned long value, void *fixup_start, void *fixup_end); -extern void do_final_fixups(void); static inline void eieio(void) { - __asm__ __volatile__ ("eieio" : : : "memory"); + if (IS_ENABLED(CONFIG_BOOKE)) + __asm__ __volatile__ ("mbar" : : : "memory"); + else + __asm__ __volatile__ ("eieio" : : : "memory"); } static inline void isync(void) { __asm__ __volatile__ ("isync" : : : "memory"); } -#endif /* __ASSEMBLY__ */ + +static inline void ppc_after_tlbiel_barrier(void) +{ + asm volatile("ptesync": : :"memory"); + /* + * POWER9, POWER10 need a cp_abort after tlbiel to ensure the copy is + * invalidated correctly. If this is not done, the paste can take data + * from the physical address that was translated at copy time. + * + * POWER9 in practice does not need this, because address spaces with + * accelerators mapped will use tlbie (which does invalidate the copy) + * to invalidate translations. It's not possible to limit POWER10 this + * way due to local copy-paste. + */ + asm volatile(ASM_FTR_IFSET(PPC_CP_ABORT, "", %0) : : "i" (CPU_FTR_ARCH_31) : "memory"); +} +#endif /* __ASSEMBLER__ */ #if defined(__powerpc64__) # define LWSYNC lwsync -#elif defined(CONFIG_E500) +#elif defined(CONFIG_PPC_E500) # define LWSYNC \ START_LWSYNC_SECTION(96); \ sync; \ @@ -44,7 +60,7 @@ static inline void isync(void) MAKE_LWSYNC_SECTION_ENTRY(97, __lwsync_fixup); #define PPC_ACQUIRE_BARRIER "\n" stringify_in_c(__PPC_ACQUIRE_BARRIER) #define PPC_RELEASE_BARRIER stringify_in_c(LWSYNC) "\n" -#define PPC_ATOMIC_ENTRY_BARRIER "\n" stringify_in_c(LWSYNC) "\n" +#define PPC_ATOMIC_ENTRY_BARRIER "\n" stringify_in_c(sync) "\n" #define PPC_ATOMIC_EXIT_BARRIER "\n" stringify_in_c(sync) "\n" #else #define PPC_ACQUIRE_BARRIER diff --git a/arch/powerpc/include/asm/syscall.h b/arch/powerpc/include/asm/syscall.h index b54b2add07be..4b3c52ed6e9d 100644 --- a/arch/powerpc/include/asm/syscall.h +++ b/arch/powerpc/include/asm/syscall.h @@ -1,29 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Access to user system call parameters and results * * Copyright (C) 2008 Red Hat, Inc. All rights reserved. * - * This copyrighted material is made available to anyone wishing to use, - * modify, copy, or redistribute it subject to the terms and conditions - * of the GNU General Public License v.2. - * * See asm-generic/syscall.h for descriptions of what we must do here. */ #ifndef _ASM_SYSCALL_H #define _ASM_SYSCALL_H 1 +#include <uapi/linux/audit.h> #include <linux/sched.h> +#include <linux/thread_info.h> + +#ifdef CONFIG_ARCH_HAS_SYSCALL_WRAPPER +typedef long (*syscall_fn)(const struct pt_regs *); +#else +typedef long (*syscall_fn)(unsigned long, unsigned long, unsigned long, + unsigned long, unsigned long, unsigned long); +#endif /* ftrace syscalls requires exporting the sys_call_table */ -#ifdef CONFIG_FTRACE_SYSCALLS -extern const unsigned long *sys_call_table; -#endif /* CONFIG_FTRACE_SYSCALLS */ +extern const syscall_fn sys_call_table[]; +extern const syscall_fn compat_sys_call_table[]; + +static inline int syscall_get_nr(struct task_struct *task, struct pt_regs *regs) +{ + /* + * Note that we are returning an int here. That means 0xffffffff, ie. + * 32-bit negative 1, will be interpreted as -1 on a 64-bit kernel. + * This is important for seccomp so that compat tasks can set r0 = -1 + * to reject the syscall. + */ + if (trap_is_syscall(regs)) + return regs->gpr[0]; + else + return -1; +} -static inline long syscall_get_nr(struct task_struct *task, - struct pt_regs *regs) +static inline void syscall_set_nr(struct task_struct *task, struct pt_regs *regs, int nr) { - return TRAP(regs) == 0xc00 ? regs->gpr[0] : -1L; + /* + * Unlike syscall_get_nr(), syscall_set_nr() can be called only when + * the target task is stopped for tracing on entering syscall, so + * there is no need to have the same check syscall_get_nr() has. + */ + regs->gpr[0] = nr; } static inline void syscall_rollback(struct task_struct *task, @@ -35,7 +58,17 @@ static inline void syscall_rollback(struct task_struct *task, static inline long syscall_get_error(struct task_struct *task, struct pt_regs *regs) { - return (regs->ccr & 0x10000000) ? -regs->gpr[3] : 0; + if (trap_is_scv(regs)) { + unsigned long error = regs->gpr[3]; + + return IS_ERR_VALUE(error) ? error : 0; + } else { + /* + * If the system call failed, + * regs->gpr[3] contains a positive ERRORCODE. + */ + return (regs->ccr & 0x10000000UL) ? -regs->gpr[3] : 0; + } } static inline long syscall_get_return_value(struct task_struct *task, @@ -48,42 +81,62 @@ static inline void syscall_set_return_value(struct task_struct *task, struct pt_regs *regs, int error, long val) { - if (error) { - regs->ccr |= 0x10000000L; - regs->gpr[3] = -error; + if (trap_is_scv(regs)) { + regs->gpr[3] = (long) error ?: val; } else { - regs->ccr &= ~0x10000000L; - regs->gpr[3] = val; + /* + * In the general case it's not obvious that we must deal with + * CCR here, as the syscall exit path will also do that for us. + * However there are some places, eg. the signal code, which + * check ccr to decide if the value in r3 is actually an error. + */ + if (error) { + regs->ccr |= 0x10000000L; + regs->gpr[3] = error; + } else { + regs->ccr &= ~0x10000000L; + regs->gpr[3] = val; + } } } static inline void syscall_get_arguments(struct task_struct *task, struct pt_regs *regs, - unsigned int i, unsigned int n, unsigned long *args) { - BUG_ON(i + n > 6); -#ifdef CONFIG_PPC64 - if (test_tsk_thread_flag(task, TIF_32BIT)) { - /* - * Zero-extend 32-bit argument values. The high bits are - * garbage ignored by the actual syscall dispatch. - */ - while (n-- > 0) - args[n] = (u32) regs->gpr[3 + i + n]; - return; + unsigned long val, mask = -1UL; + unsigned int n = 6; + + if (is_tsk_32bit_task(task)) + mask = 0xffffffff; + + while (n--) { + if (n == 0) + val = regs->orig_gpr3; + else + val = regs->gpr[3 + n]; + + args[n] = val & mask; } -#endif - memcpy(args, ®s->gpr[3 + i], n * sizeof(args[0])); } static inline void syscall_set_arguments(struct task_struct *task, struct pt_regs *regs, - unsigned int i, unsigned int n, const unsigned long *args) { - BUG_ON(i + n > 6); - memcpy(®s->gpr[3 + i], args, n * sizeof(args[0])); + memcpy(®s->gpr[3], args, 6 * sizeof(args[0])); + + /* Also copy the first argument into orig_gpr3 */ + regs->orig_gpr3 = args[0]; } +static inline int syscall_get_arch(struct task_struct *task) +{ + if (is_tsk_32bit_task(task)) + return AUDIT_ARCH_PPC; + else if (IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN)) + return AUDIT_ARCH_PPC64LE; + else + return AUDIT_ARCH_PPC64; +} #endif /* _ASM_SYSCALL_H */ diff --git a/arch/powerpc/include/asm/syscall_wrapper.h b/arch/powerpc/include/asm/syscall_wrapper.h new file mode 100644 index 000000000000..67486c67e8a2 --- /dev/null +++ b/arch/powerpc/include/asm/syscall_wrapper.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * syscall_wrapper.h - powerpc specific wrappers to syscall definitions + * + * Based on arch/{x86,arm64}/include/asm/syscall_wrapper.h + */ + +#ifndef __ASM_POWERPC_SYSCALL_WRAPPER_H +#define __ASM_POWERPC_SYSCALL_WRAPPER_H + +struct pt_regs; + +#define SC_POWERPC_REGS_TO_ARGS(x, ...) \ + __MAP(x,__SC_ARGS \ + ,,regs->gpr[3],,regs->gpr[4],,regs->gpr[5] \ + ,,regs->gpr[6],,regs->gpr[7],,regs->gpr[8]) + +#define __SYSCALL_DEFINEx(x, name, ...) \ + long sys##name(const struct pt_regs *regs); \ + ALLOW_ERROR_INJECTION(sys##name, ERRNO); \ + static long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)); \ + static inline long __do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)); \ + long sys##name(const struct pt_regs *regs) \ + { \ + return __se_sys##name(SC_POWERPC_REGS_TO_ARGS(x,__VA_ARGS__)); \ + } \ + static long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)) \ + { \ + long ret = __do_sys##name(__MAP(x,__SC_CAST,__VA_ARGS__)); \ + __MAP(x,__SC_TEST,__VA_ARGS__); \ + __PROTECT(x, ret,__MAP(x,__SC_ARGS,__VA_ARGS__)); \ + return ret; \ + } \ + static inline long __do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)) + +#define SYSCALL_DEFINE0(sname) \ + SYSCALL_METADATA(_##sname, 0); \ + long sys_##sname(const struct pt_regs *__unused); \ + ALLOW_ERROR_INJECTION(sys_##sname, ERRNO); \ + long sys_##sname(const struct pt_regs *__unused) + +#define COND_SYSCALL(name) \ + long sys_##name(const struct pt_regs *regs); \ + long __weak sys_##name(const struct pt_regs *regs) \ + { \ + return sys_ni_syscall(); \ + } + +#endif // __ASM_POWERPC_SYSCALL_WRAPPER_H diff --git a/arch/powerpc/include/asm/syscalls.h b/arch/powerpc/include/asm/syscalls.h index 23be8f1e7e64..6d51b007b59e 100644 --- a/arch/powerpc/include/asm/syscalls.h +++ b/arch/powerpc/include/asm/syscalls.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __ASM_POWERPC_SYSCALLS_H #define __ASM_POWERPC_SYSCALLS_H #ifdef __KERNEL__ @@ -5,17 +6,156 @@ #include <linux/compiler.h> #include <linux/linkage.h> #include <linux/types.h> +#include <linux/compat.h> + +#include <asm/syscall.h> +#ifdef CONFIG_PPC64 +#include <asm/syscalls_32.h> +#endif +#include <asm/unistd.h> +#include <asm/ucontext.h> + +#ifndef CONFIG_ARCH_HAS_SYSCALL_WRAPPER +long sys_ni_syscall(void); +#else +long sys_ni_syscall(const struct pt_regs *regs); +#endif struct rtas_args; -asmlinkage unsigned long sys_mmap(unsigned long addr, size_t len, - unsigned long prot, unsigned long flags, - unsigned long fd, off_t offset); -asmlinkage unsigned long sys_mmap2(unsigned long addr, size_t len, - unsigned long prot, unsigned long flags, - unsigned long fd, unsigned long pgoff); -asmlinkage long ppc64_personality(unsigned long personality); -asmlinkage int ppc_rtas(struct rtas_args __user *uargs); +/* + * long long munging: + * The 32 bit ABI passes long longs in an odd even register pair. + * High and low parts are swapped depending on endian mode, + * so define a macro (similar to mips linux32) to handle that. + */ +#ifdef __LITTLE_ENDIAN__ +#define merge_64(low, high) (((u64)high << 32) | low) +#else +#define merge_64(high, low) (((u64)high << 32) | low) +#endif + +/* + * PowerPC architecture-specific syscalls + */ + +#ifndef CONFIG_ARCH_HAS_SYSCALL_WRAPPER + +long sys_rtas(struct rtas_args __user *uargs); + +#ifdef CONFIG_PPC64 +long sys_ppc64_personality(unsigned long personality); +#ifdef CONFIG_COMPAT +long compat_sys_ppc64_personality(unsigned long personality); +#endif /* CONFIG_COMPAT */ +#endif /* CONFIG_PPC64 */ + +long sys_swapcontext(struct ucontext __user *old_ctx, + struct ucontext __user *new_ctx, long ctx_size); +long sys_mmap(unsigned long addr, size_t len, + unsigned long prot, unsigned long flags, + unsigned long fd, off_t offset); +long sys_mmap2(unsigned long addr, size_t len, + unsigned long prot, unsigned long flags, + unsigned long fd, unsigned long pgoff); +long sys_switch_endian(void); + +#ifdef CONFIG_PPC32 +long sys_sigreturn(void); +long sys_debug_setcontext(struct ucontext __user *ctx, int ndbg, + struct sig_dbg_op __user *dbg); +#endif + +long sys_rt_sigreturn(void); + +long sys_subpage_prot(unsigned long addr, + unsigned long len, u32 __user *map); + +#ifdef CONFIG_COMPAT +long compat_sys_swapcontext(struct ucontext32 __user *old_ctx, + struct ucontext32 __user *new_ctx, + int ctx_size); +long compat_sys_old_getrlimit(unsigned int resource, + struct compat_rlimit __user *rlim); +long compat_sys_sigreturn(void); +long compat_sys_rt_sigreturn(void); +#endif /* CONFIG_COMPAT */ + +/* + * Architecture specific signatures required by long long munging: + * The 32 bit ABI passes long longs in an odd even register pair. + * The following signatures provide a machine long parameter for + * each register that will be supplied. The implementation is + * responsible for combining parameter pairs. + */ + +#ifdef CONFIG_PPC32 +long sys_ppc_pread64(unsigned int fd, + char __user *ubuf, compat_size_t count, + u32 reg6, u32 pos1, u32 pos2); +long sys_ppc_pwrite64(unsigned int fd, + const char __user *ubuf, compat_size_t count, + u32 reg6, u32 pos1, u32 pos2); +long sys_ppc_readahead(int fd, u32 r4, + u32 offset1, u32 offset2, u32 count); +long sys_ppc_truncate64(const char __user *path, u32 reg4, + unsigned long len1, unsigned long len2); +long sys_ppc_ftruncate64(unsigned int fd, u32 reg4, + unsigned long len1, unsigned long len2); +long sys_ppc32_fadvise64(int fd, u32 unused, u32 offset1, u32 offset2, + size_t len, int advice); +long sys_ppc_sync_file_range2(int fd, unsigned int flags, + unsigned int offset1, + unsigned int offset2, + unsigned int nbytes1, + unsigned int nbytes2); +long sys_ppc_fallocate(int fd, int mode, u32 offset1, u32 offset2, + u32 len1, u32 len2); +#endif +#ifdef CONFIG_COMPAT +long compat_sys_mmap2(unsigned long addr, size_t len, + unsigned long prot, unsigned long flags, + unsigned long fd, unsigned long pgoff); +long compat_sys_ppc_pread64(unsigned int fd, + char __user *ubuf, compat_size_t count, + u32 reg6, u32 pos1, u32 pos2); +long compat_sys_ppc_pwrite64(unsigned int fd, + const char __user *ubuf, compat_size_t count, + u32 reg6, u32 pos1, u32 pos2); +long compat_sys_ppc_readahead(int fd, u32 r4, + u32 offset1, u32 offset2, u32 count); +long compat_sys_ppc_truncate64(const char __user *path, u32 reg4, + unsigned long len1, unsigned long len2); +long compat_sys_ppc_ftruncate64(unsigned int fd, u32 reg4, + unsigned long len1, unsigned long len2); +long compat_sys_ppc32_fadvise64(int fd, u32 unused, u32 offset1, u32 offset2, + size_t len, int advice); +long compat_sys_ppc_sync_file_range2(int fd, unsigned int flags, + unsigned int offset1, + unsigned int offset2, + unsigned int nbytes1, + unsigned int nbytes2); +#endif /* CONFIG_COMPAT */ + +#if defined(CONFIG_PPC32) || defined(CONFIG_COMPAT) +long sys_ppc_fadvise64_64(int fd, int advice, + u32 offset_high, u32 offset_low, + u32 len_high, u32 len_low); +#endif + +#else + +#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, native) +#define __SYSCALL(nr, entry) \ + long entry(const struct pt_regs *regs); + +#ifdef CONFIG_PPC64 +#include <asm/syscall_table_64.h> +#else +#include <asm/syscall_table_32.h> +#endif /* CONFIG_PPC64 */ + +#endif /* CONFIG_ARCH_HAS_SYSCALL_WRAPPER */ #endif /* __KERNEL__ */ #endif /* __ASM_POWERPC_SYSCALLS_H */ diff --git a/arch/powerpc/include/asm/syscalls_32.h b/arch/powerpc/include/asm/syscalls_32.h new file mode 100644 index 000000000000..749255568be9 --- /dev/null +++ b/arch/powerpc/include/asm/syscalls_32.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_POWERPC_SYSCALLS_32_H +#define _ASM_POWERPC_SYSCALLS_32_H + +#include <linux/compat.h> +#include <asm/siginfo.h> +#include <asm/signal.h> + +/* + * Data types and macros for providing 32b PowerPC support. + */ + +/* These are here to support 32-bit syscalls on a 64-bit kernel. */ + +struct pt_regs32 { + unsigned int gpr[32]; + unsigned int nip; + unsigned int msr; + unsigned int orig_gpr3; /* Used for restarting system calls */ + unsigned int ctr; + unsigned int link; + unsigned int xer; + unsigned int ccr; + unsigned int mq; /* 601 only (not used at present) */ + unsigned int trap; /* Reason for being here */ + unsigned int dar; /* Fault registers */ + unsigned int dsisr; + unsigned int result; /* Result of a system call */ +}; + +struct sigcontext32 { + unsigned int _unused[4]; + int signal; + compat_uptr_t handler; + unsigned int oldmask; + compat_uptr_t regs; /* 4 byte pointer to the pt_regs32 structure. */ +}; + +struct mcontext32 { + elf_gregset_t32 mc_gregs; + elf_fpregset_t mc_fregs; + unsigned int mc_pad[2]; + elf_vrregset_t32 mc_vregs __attribute__((__aligned__(16))); + elf_vsrreghalf_t32 mc_vsregs __attribute__((__aligned__(16))); +}; + +struct ucontext32 { + unsigned int uc_flags; + unsigned int uc_link; + compat_stack_t uc_stack; + int uc_pad[7]; + compat_uptr_t uc_regs; /* points to uc_mcontext field */ + compat_sigset_t uc_sigmask; /* mask last for extensibility */ + /* glibc has 1024-bit signal masks, ours are 64-bit */ + int uc_maskext[30]; + int uc_pad2[3]; + struct mcontext32 uc_mcontext; +}; + +#endif // _ASM_POWERPC_SYSCALLS_32_H diff --git a/arch/powerpc/include/asm/systbl.h b/arch/powerpc/include/asm/systbl.h deleted file mode 100644 index 43523fe0d8b4..000000000000 --- a/arch/powerpc/include/asm/systbl.h +++ /dev/null @@ -1,361 +0,0 @@ -/* - * List of powerpc syscalls. For the meaning of the _SPU suffix see - * arch/powerpc/platforms/cell/spu_callbacks.c - */ - -SYSCALL(restart_syscall) -SYSCALL(exit) -PPC_SYS(fork) -SYSCALL_SPU(read) -SYSCALL_SPU(write) -COMPAT_SYS_SPU(open) -SYSCALL_SPU(close) -SYSCALL_SPU(waitpid) -SYSCALL_SPU(creat) -SYSCALL_SPU(link) -SYSCALL_SPU(unlink) -COMPAT_SYS(execve) -SYSCALL_SPU(chdir) -COMPAT_SYS_SPU(time) -SYSCALL_SPU(mknod) -SYSCALL_SPU(chmod) -SYSCALL_SPU(lchown) -SYSCALL(ni_syscall) -OLDSYS(stat) -COMPAT_SYS_SPU(lseek) -SYSCALL_SPU(getpid) -COMPAT_SYS(mount) -SYSX(sys_ni_syscall,sys_oldumount,sys_oldumount) -SYSCALL_SPU(setuid) -SYSCALL_SPU(getuid) -COMPAT_SYS_SPU(stime) -COMPAT_SYS(ptrace) -SYSCALL_SPU(alarm) -OLDSYS(fstat) -SYSCALL(pause) -COMPAT_SYS(utime) -SYSCALL(ni_syscall) -SYSCALL(ni_syscall) -SYSCALL_SPU(access) -SYSCALL_SPU(nice) -SYSCALL(ni_syscall) -SYSCALL_SPU(sync) -SYSCALL_SPU(kill) -SYSCALL_SPU(rename) -SYSCALL_SPU(mkdir) -SYSCALL_SPU(rmdir) -SYSCALL_SPU(dup) -SYSCALL_SPU(pipe) -COMPAT_SYS_SPU(times) -SYSCALL(ni_syscall) -SYSCALL_SPU(brk) -SYSCALL_SPU(setgid) -SYSCALL_SPU(getgid) -SYSCALL(signal) -SYSCALL_SPU(geteuid) -SYSCALL_SPU(getegid) -SYSCALL(acct) -SYSCALL(umount) -SYSCALL(ni_syscall) -COMPAT_SYS_SPU(ioctl) -COMPAT_SYS_SPU(fcntl) -SYSCALL(ni_syscall) -SYSCALL_SPU(setpgid) -SYSCALL(ni_syscall) -SYSX(sys_ni_syscall,sys_olduname, sys_olduname) -SYSCALL_SPU(umask) -SYSCALL_SPU(chroot) -COMPAT_SYS(ustat) -SYSCALL_SPU(dup2) -SYSCALL_SPU(getppid) -SYSCALL_SPU(getpgrp) -SYSCALL_SPU(setsid) -SYS32ONLY(sigaction) -SYSCALL_SPU(sgetmask) -SYSCALL_SPU(ssetmask) -SYSCALL_SPU(setreuid) -SYSCALL_SPU(setregid) -#define compat_sys_sigsuspend sys_sigsuspend -SYS32ONLY(sigsuspend) -COMPAT_SYS(sigpending) -SYSCALL_SPU(sethostname) -COMPAT_SYS_SPU(setrlimit) -COMPAT_SYS(old_getrlimit) -COMPAT_SYS_SPU(getrusage) -COMPAT_SYS_SPU(gettimeofday) -COMPAT_SYS_SPU(settimeofday) -SYSCALL_SPU(getgroups) -SYSCALL_SPU(setgroups) -SYSX(sys_ni_syscall,sys_ni_syscall,ppc_select) -SYSCALL_SPU(symlink) -OLDSYS(lstat) -SYSCALL_SPU(readlink) -SYSCALL(uselib) -SYSCALL(swapon) -SYSCALL(reboot) -SYSX(sys_ni_syscall,compat_sys_old_readdir,sys_old_readdir) -SYSCALL_SPU(mmap) -SYSCALL_SPU(munmap) -COMPAT_SYS_SPU(truncate) -COMPAT_SYS_SPU(ftruncate) -SYSCALL_SPU(fchmod) -SYSCALL_SPU(fchown) -SYSCALL_SPU(getpriority) -SYSCALL_SPU(setpriority) -SYSCALL(ni_syscall) -COMPAT_SYS(statfs) -COMPAT_SYS(fstatfs) -SYSCALL(ni_syscall) -COMPAT_SYS_SPU(socketcall) -SYSCALL_SPU(syslog) -COMPAT_SYS_SPU(setitimer) -COMPAT_SYS_SPU(getitimer) -COMPAT_SYS_SPU(newstat) -COMPAT_SYS_SPU(newlstat) -COMPAT_SYS_SPU(newfstat) -SYSX(sys_ni_syscall,sys_uname,sys_uname) -SYSCALL(ni_syscall) -SYSCALL_SPU(vhangup) -SYSCALL(ni_syscall) -SYSCALL(ni_syscall) -COMPAT_SYS_SPU(wait4) -SYSCALL(swapoff) -COMPAT_SYS_SPU(sysinfo) -COMPAT_SYS(ipc) -SYSCALL_SPU(fsync) -SYS32ONLY(sigreturn) -PPC_SYS(clone) -SYSCALL_SPU(setdomainname) -SYSCALL_SPU(newuname) -SYSCALL(ni_syscall) -COMPAT_SYS_SPU(adjtimex) -SYSCALL_SPU(mprotect) -SYSX(sys_ni_syscall,compat_sys_sigprocmask,sys_sigprocmask) -SYSCALL(ni_syscall) -SYSCALL(init_module) -SYSCALL(delete_module) -SYSCALL(ni_syscall) -SYSCALL(quotactl) -SYSCALL_SPU(getpgid) -SYSCALL_SPU(fchdir) -SYSCALL_SPU(bdflush) -SYSCALL_SPU(sysfs) -SYSX_SPU(ppc64_personality,ppc64_personality,sys_personality) -SYSCALL(ni_syscall) -SYSCALL_SPU(setfsuid) -SYSCALL_SPU(setfsgid) -SYSCALL_SPU(llseek) -COMPAT_SYS_SPU(getdents) -SYSX_SPU(sys_select,ppc32_select,sys_select) -SYSCALL_SPU(flock) -SYSCALL_SPU(msync) -COMPAT_SYS_SPU(readv) -COMPAT_SYS_SPU(writev) -SYSCALL_SPU(getsid) -SYSCALL_SPU(fdatasync) -COMPAT_SYS(sysctl) -SYSCALL_SPU(mlock) -SYSCALL_SPU(munlock) -SYSCALL_SPU(mlockall) -SYSCALL_SPU(munlockall) -SYSCALL_SPU(sched_setparam) -SYSCALL_SPU(sched_getparam) -SYSCALL_SPU(sched_setscheduler) -SYSCALL_SPU(sched_getscheduler) -SYSCALL_SPU(sched_yield) -SYSCALL_SPU(sched_get_priority_max) -SYSCALL_SPU(sched_get_priority_min) -COMPAT_SYS_SPU(sched_rr_get_interval) -COMPAT_SYS_SPU(nanosleep) -SYSCALL_SPU(mremap) -SYSCALL_SPU(setresuid) -SYSCALL_SPU(getresuid) -SYSCALL(ni_syscall) -SYSCALL_SPU(poll) -SYSCALL(ni_syscall) -SYSCALL_SPU(setresgid) -SYSCALL_SPU(getresgid) -SYSCALL_SPU(prctl) -COMPAT_SYS(rt_sigreturn) -COMPAT_SYS(rt_sigaction) -COMPAT_SYS(rt_sigprocmask) -COMPAT_SYS(rt_sigpending) -COMPAT_SYS(rt_sigtimedwait) -COMPAT_SYS(rt_sigqueueinfo) -COMPAT_SYS(rt_sigsuspend) -COMPAT_SYS_SPU(pread64) -COMPAT_SYS_SPU(pwrite64) -SYSCALL_SPU(chown) -SYSCALL_SPU(getcwd) -SYSCALL_SPU(capget) -SYSCALL_SPU(capset) -COMPAT_SYS(sigaltstack) -COMPAT_SYS_SPU(sendfile) -SYSCALL(ni_syscall) -SYSCALL(ni_syscall) -PPC_SYS(vfork) -COMPAT_SYS_SPU(getrlimit) -COMPAT_SYS_SPU(readahead) -SYS32ONLY(mmap2) -SYS32ONLY(truncate64) -SYS32ONLY(ftruncate64) -SYSX(sys_ni_syscall,sys_stat64,sys_stat64) -SYSX(sys_ni_syscall,sys_lstat64,sys_lstat64) -SYSX(sys_ni_syscall,sys_fstat64,sys_fstat64) -SYSCALL(pciconfig_read) -SYSCALL(pciconfig_write) -SYSCALL(pciconfig_iobase) -SYSCALL(ni_syscall) -SYSCALL_SPU(getdents64) -SYSCALL_SPU(pivot_root) -SYSX(sys_ni_syscall,compat_sys_fcntl64,sys_fcntl64) -SYSCALL_SPU(madvise) -SYSCALL_SPU(mincore) -SYSCALL_SPU(gettid) -SYSCALL_SPU(tkill) -SYSCALL_SPU(setxattr) -SYSCALL_SPU(lsetxattr) -SYSCALL_SPU(fsetxattr) -SYSCALL_SPU(getxattr) -SYSCALL_SPU(lgetxattr) -SYSCALL_SPU(fgetxattr) -SYSCALL_SPU(listxattr) -SYSCALL_SPU(llistxattr) -SYSCALL_SPU(flistxattr) -SYSCALL_SPU(removexattr) -SYSCALL_SPU(lremovexattr) -SYSCALL_SPU(fremovexattr) -COMPAT_SYS_SPU(futex) -COMPAT_SYS_SPU(sched_setaffinity) -COMPAT_SYS_SPU(sched_getaffinity) -SYSCALL(ni_syscall) -SYSCALL(ni_syscall) -SYS32ONLY(sendfile64) -COMPAT_SYS_SPU(io_setup) -SYSCALL_SPU(io_destroy) -COMPAT_SYS_SPU(io_getevents) -COMPAT_SYS_SPU(io_submit) -SYSCALL_SPU(io_cancel) -SYSCALL(set_tid_address) -SYSX_SPU(sys_fadvise64,ppc32_fadvise64,sys_fadvise64) -SYSCALL(exit_group) -COMPAT_SYS(lookup_dcookie) -SYSCALL_SPU(epoll_create) -SYSCALL_SPU(epoll_ctl) -SYSCALL_SPU(epoll_wait) -SYSCALL_SPU(remap_file_pages) -SYSX_SPU(sys_timer_create,compat_sys_timer_create,sys_timer_create) -COMPAT_SYS_SPU(timer_settime) -COMPAT_SYS_SPU(timer_gettime) -SYSCALL_SPU(timer_getoverrun) -SYSCALL_SPU(timer_delete) -COMPAT_SYS_SPU(clock_settime) -COMPAT_SYS_SPU(clock_gettime) -COMPAT_SYS_SPU(clock_getres) -COMPAT_SYS_SPU(clock_nanosleep) -SYSX(ppc64_swapcontext,ppc32_swapcontext,ppc_swapcontext) -SYSCALL_SPU(tgkill) -COMPAT_SYS_SPU(utimes) -COMPAT_SYS_SPU(statfs64) -COMPAT_SYS_SPU(fstatfs64) -SYSX(sys_ni_syscall, ppc_fadvise64_64, ppc_fadvise64_64) -PPC_SYS_SPU(rtas) -OLDSYS(debug_setcontext) -SYSCALL(ni_syscall) -COMPAT_SYS(migrate_pages) -COMPAT_SYS(mbind) -COMPAT_SYS(get_mempolicy) -COMPAT_SYS(set_mempolicy) -COMPAT_SYS(mq_open) -SYSCALL(mq_unlink) -COMPAT_SYS(mq_timedsend) -COMPAT_SYS(mq_timedreceive) -COMPAT_SYS(mq_notify) -COMPAT_SYS(mq_getsetattr) -COMPAT_SYS(kexec_load) -SYSCALL(add_key) -SYSCALL(request_key) -COMPAT_SYS(keyctl) -COMPAT_SYS(waitid) -SYSCALL(ioprio_set) -SYSCALL(ioprio_get) -SYSCALL(inotify_init) -SYSCALL(inotify_add_watch) -SYSCALL(inotify_rm_watch) -SYSCALL(spu_run) -SYSCALL(spu_create) -COMPAT_SYS(pselect6) -COMPAT_SYS(ppoll) -SYSCALL_SPU(unshare) -SYSCALL_SPU(splice) -SYSCALL_SPU(tee) -COMPAT_SYS_SPU(vmsplice) -COMPAT_SYS_SPU(openat) -SYSCALL_SPU(mkdirat) -SYSCALL_SPU(mknodat) -SYSCALL_SPU(fchownat) -COMPAT_SYS_SPU(futimesat) -SYSX_SPU(sys_newfstatat, sys_fstatat64, sys_fstatat64) -SYSCALL_SPU(unlinkat) -SYSCALL_SPU(renameat) -SYSCALL_SPU(linkat) -SYSCALL_SPU(symlinkat) -SYSCALL_SPU(readlinkat) -SYSCALL_SPU(fchmodat) -SYSCALL_SPU(faccessat) -COMPAT_SYS_SPU(get_robust_list) -COMPAT_SYS_SPU(set_robust_list) -COMPAT_SYS_SPU(move_pages) -SYSCALL_SPU(getcpu) -COMPAT_SYS(epoll_pwait) -COMPAT_SYS_SPU(utimensat) -COMPAT_SYS_SPU(signalfd) -SYSCALL_SPU(timerfd_create) -SYSCALL_SPU(eventfd) -COMPAT_SYS_SPU(sync_file_range2) -COMPAT_SYS(fallocate) -SYSCALL(subpage_prot) -COMPAT_SYS_SPU(timerfd_settime) -COMPAT_SYS_SPU(timerfd_gettime) -COMPAT_SYS_SPU(signalfd4) -SYSCALL_SPU(eventfd2) -SYSCALL_SPU(epoll_create1) -SYSCALL_SPU(dup3) -SYSCALL_SPU(pipe2) -SYSCALL(inotify_init1) -SYSCALL_SPU(perf_event_open) -COMPAT_SYS_SPU(preadv) -COMPAT_SYS_SPU(pwritev) -COMPAT_SYS(rt_tgsigqueueinfo) -SYSCALL(fanotify_init) -COMPAT_SYS(fanotify_mark) -SYSCALL_SPU(prlimit64) -SYSCALL_SPU(socket) -SYSCALL_SPU(bind) -SYSCALL_SPU(connect) -SYSCALL_SPU(listen) -SYSCALL_SPU(accept) -SYSCALL_SPU(getsockname) -SYSCALL_SPU(getpeername) -SYSCALL_SPU(socketpair) -SYSCALL_SPU(send) -SYSCALL_SPU(sendto) -COMPAT_SYS_SPU(recv) -COMPAT_SYS_SPU(recvfrom) -SYSCALL_SPU(shutdown) -COMPAT_SYS_SPU(setsockopt) -COMPAT_SYS_SPU(getsockopt) -COMPAT_SYS_SPU(sendmsg) -COMPAT_SYS_SPU(recvmsg) -COMPAT_SYS_SPU(recvmmsg) -SYSCALL_SPU(accept4) -SYSCALL_SPU(name_to_handle_at) -COMPAT_SYS_SPU(open_by_handle_at) -COMPAT_SYS_SPU(clock_adjtime) -SYSCALL_SPU(syncfs) -COMPAT_SYS_SPU(sendmmsg) -SYSCALL_SPU(setns) -COMPAT_SYS(process_vm_readv) -COMPAT_SYS(process_vm_writev) -SYSCALL(finit_module) -SYSCALL(ni_syscall) /* sys_kcmp */ diff --git a/arch/powerpc/include/asm/systemcfg.h b/arch/powerpc/include/asm/systemcfg.h new file mode 100644 index 000000000000..2f9b1d6a5c98 --- /dev/null +++ b/arch/powerpc/include/asm/systemcfg.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _SYSTEMCFG_H +#define _SYSTEMCFG_H + +/* + * Copyright (C) 2002 Peter Bergner <bergner@vnet.ibm.com>, IBM + * Copyright (C) 2005 Benjamin Herrenschmidy <benh@kernel.crashing.org>, + * IBM Corp. + */ + +#ifdef CONFIG_PPC64 + +/* + * If the major version changes we are incompatible. + * Minor version changes are a hint. + */ +#define SYSTEMCFG_MAJOR 1 +#define SYSTEMCFG_MINOR 1 + +#include <linux/types.h> + +struct systemcfg { + __u8 eye_catcher[16]; /* Eyecatcher: SYSTEMCFG:PPC64 0x00 */ + struct { /* Systemcfg version numbers */ + __u32 major; /* Major number 0x10 */ + __u32 minor; /* Minor number 0x14 */ + } version; + + /* Note about the platform flags: it now only contains the lpar + * bit. The actual platform number is dead and buried + */ + __u32 platform; /* Platform flags 0x18 */ + __u32 processor; /* Processor type 0x1C */ + __u64 processorCount; /* # of physical processors 0x20 */ + __u64 physicalMemorySize; /* Size of real memory(B) 0x28 */ + __u64 tb_orig_stamp; /* (NU) Timebase at boot 0x30 */ + __u64 tb_ticks_per_sec; /* Timebase tics / sec 0x38 */ + __u64 tb_to_xs; /* (NU) Inverse of TB to 2^20 0x40 */ + __u64 stamp_xsec; /* (NU) 0x48 */ + __u64 tb_update_count; /* (NU) Timebase atomicity ctr 0x50 */ + __u32 tz_minuteswest; /* (NU) Min. west of Greenwich 0x58 */ + __u32 tz_dsttime; /* (NU) Type of dst correction 0x5C */ + __u32 dcache_size; /* L1 d-cache size 0x60 */ + __u32 dcache_line_size; /* L1 d-cache line size 0x64 */ + __u32 icache_size; /* L1 i-cache size 0x68 */ + __u32 icache_line_size; /* L1 i-cache line size 0x6C */ +}; + +extern struct systemcfg *systemcfg; + +#endif /* CONFIG_PPC64 */ +#endif /* _SYSTEMCFG_H */ diff --git a/arch/powerpc/include/asm/task_size_32.h b/arch/powerpc/include/asm/task_size_32.h new file mode 100644 index 000000000000..de7290ee770f --- /dev/null +++ b/arch/powerpc/include/asm/task_size_32.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_TASK_SIZE_32_H +#define _ASM_POWERPC_TASK_SIZE_32_H + +#if CONFIG_TASK_SIZE > CONFIG_KERNEL_START +#error User TASK_SIZE overlaps with KERNEL_START address +#endif + +#define TASK_SIZE (CONFIG_TASK_SIZE) + +/* + * This decides where the kernel will search for a free chunk of vm space during + * mmap's. + */ +#define TASK_UNMAPPED_BASE (TASK_SIZE / 8 * 3) + +#define DEFAULT_MAP_WINDOW TASK_SIZE +#define STACK_TOP TASK_SIZE +#define STACK_TOP_MAX STACK_TOP + +#endif /* _ASM_POWERPC_TASK_SIZE_32_H */ diff --git a/arch/powerpc/include/asm/task_size_64.h b/arch/powerpc/include/asm/task_size_64.h new file mode 100644 index 000000000000..5a709951c901 --- /dev/null +++ b/arch/powerpc/include/asm/task_size_64.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_TASK_SIZE_64_H +#define _ASM_POWERPC_TASK_SIZE_64_H + +/* + * 64-bit user address space can have multiple limits + * For now supported values are: + */ +#define TASK_SIZE_64TB (0x0000400000000000UL) +#define TASK_SIZE_128TB (0x0000800000000000UL) +#define TASK_SIZE_512TB (0x0002000000000000UL) +#define TASK_SIZE_1PB (0x0004000000000000UL) +#define TASK_SIZE_2PB (0x0008000000000000UL) + +/* + * With 52 bits in the address we can support up to 4PB of range. + */ +#define TASK_SIZE_4PB (0x0010000000000000UL) + +/* + * For now 512TB is only supported with book3s and 64K linux page size. + */ +#ifdef CONFIG_PPC_64K_PAGES +/* + * Max value currently used: + */ +#define TASK_SIZE_USER64 TASK_SIZE_4PB +#define DEFAULT_MAP_WINDOW_USER64 TASK_SIZE_128TB +#define TASK_CONTEXT_SIZE TASK_SIZE_512TB +#else +#define TASK_SIZE_USER64 TASK_SIZE_64TB +#define DEFAULT_MAP_WINDOW_USER64 TASK_SIZE_64TB + +/* + * We don't need to allocate extended context ids for 4K page size, because we + * limit the max effective address on this config to 64TB. + */ +#define TASK_CONTEXT_SIZE TASK_SIZE_64TB +#endif + +/* + * 32-bit user address space is 4GB - 1 page + * (this 1 page is needed so referencing of 0xFFFFFFFF generates EFAULT + */ +#define TASK_SIZE_USER32 (0x0000000100000000UL - (1 * PAGE_SIZE)) + +#define TASK_SIZE (is_32bit_task() ? TASK_SIZE_USER32 : TASK_SIZE_USER64) + +#define TASK_UNMAPPED_BASE_USER32 (PAGE_ALIGN(TASK_SIZE_USER32 / 4)) +#define TASK_UNMAPPED_BASE_USER64 (PAGE_ALIGN(DEFAULT_MAP_WINDOW_USER64 / 4)) + +/* + * This decides where the kernel will search for a free chunk of vm space during + * mmap's. + */ +#define TASK_UNMAPPED_BASE \ + ((is_32bit_task()) ? TASK_UNMAPPED_BASE_USER32 : TASK_UNMAPPED_BASE_USER64) + +/* + * Initial task size value for user applications. For book3s 64 we start + * with 128TB and conditionally enable upto 512TB + */ +#ifdef CONFIG_PPC_BOOK3S_64 +#define DEFAULT_MAP_WINDOW \ + ((is_32bit_task()) ? TASK_SIZE_USER32 : DEFAULT_MAP_WINDOW_USER64) +#else +#define DEFAULT_MAP_WINDOW TASK_SIZE +#endif + +#define STACK_TOP_USER64 DEFAULT_MAP_WINDOW_USER64 +#define STACK_TOP_USER32 TASK_SIZE_USER32 +#define STACK_TOP_MAX TASK_SIZE_USER64 +#define STACK_TOP (is_32bit_task() ? STACK_TOP_USER32 : STACK_TOP_USER64) + +#define arch_get_mmap_base(addr, base) \ + (((addr) > DEFAULT_MAP_WINDOW) ? (base) + TASK_SIZE - DEFAULT_MAP_WINDOW : (base)) + +#define arch_get_mmap_end(addr, len, flags) \ + (((addr) > DEFAULT_MAP_WINDOW) || \ + (((flags) & MAP_FIXED) && ((addr) + (len) > DEFAULT_MAP_WINDOW)) ? TASK_SIZE : \ + DEFAULT_MAP_WINDOW) + +#endif /* _ASM_POWERPC_TASK_SIZE_64_H */ diff --git a/arch/powerpc/include/asm/tce.h b/arch/powerpc/include/asm/tce.h index 743f36b38e5d..0c34d2756d92 100644 --- a/arch/powerpc/include/asm/tce.h +++ b/arch/powerpc/include/asm/tce.h @@ -1,21 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation * Rewrite, cleanup: * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef _ASM_POWERPC_TCE_H @@ -31,19 +18,8 @@ */ #define TCE_VB 0 #define TCE_PCI 1 -#define TCE_PCI_SWINV_CREATE 2 -#define TCE_PCI_SWINV_FREE 4 -#define TCE_PCI_SWINV_PAIR 8 - -/* TCE page size is 4096 bytes (1 << 12) */ - -#define TCE_SHIFT 12 -#define TCE_PAGE_SIZE (1 << TCE_SHIFT) #define TCE_ENTRY_SIZE 8 /* each TCE is 64 bits */ - -#define TCE_RPN_MASK 0xfffffffffful /* 40-bit RPN (4K pages) */ -#define TCE_RPN_SHIFT 12 #define TCE_VALID 0x800 /* TCE valid */ #define TCE_ALLIO 0x400 /* TCE valid for all lpars */ #define TCE_PCI_WRITE 0x2 /* write from PCI allowed */ diff --git a/arch/powerpc/include/asm/termios.h b/arch/powerpc/include/asm/termios.h deleted file mode 100644 index b8353e2032d0..000000000000 --- a/arch/powerpc/include/asm/termios.h +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Liberally adapted from alpha/termios.h. In particular, the c_cc[] - * fields have been reordered so that termio & termios share the - * common subset in the same order (for brain dead programs that don't - * know or care about the differences). - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ -#ifndef _ASM_POWERPC_TERMIOS_H -#define _ASM_POWERPC_TERMIOS_H - -#include <uapi/asm/termios.h> - -/* ^C ^\ del ^U ^D 1 0 0 0 0 ^W ^R ^Z ^Q ^S ^V ^U */ -#define INIT_C_CC "\003\034\177\025\004\001\000\000\000\000\027\022\032\021\023\026\025" - -#include <asm-generic/termios-base.h> - -#endif /* _ASM_POWERPC_TERMIOS_H */ diff --git a/arch/powerpc/include/asm/text-patching.h b/arch/powerpc/include/asm/text-patching.h new file mode 100644 index 000000000000..e7f14720f630 --- /dev/null +++ b/arch/powerpc/include/asm/text-patching.h @@ -0,0 +1,275 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_POWERPC_CODE_PATCHING_H +#define _ASM_POWERPC_CODE_PATCHING_H + +/* + * Copyright 2008, Michael Ellerman, IBM Corporation. + */ + +#include <asm/types.h> +#include <asm/ppc-opcode.h> +#include <linux/string.h> +#include <linux/kallsyms.h> +#include <asm/asm-compat.h> +#include <asm/inst.h> + +/* Flags for create_branch: + * "b" == create_branch(addr, target, 0); + * "ba" == create_branch(addr, target, BRANCH_ABSOLUTE); + * "bl" == create_branch(addr, target, BRANCH_SET_LINK); + * "bla" == create_branch(addr, target, BRANCH_ABSOLUTE | BRANCH_SET_LINK); + */ +#define BRANCH_SET_LINK 0x1 +#define BRANCH_ABSOLUTE 0x2 + +/* + * Powerpc branch instruction is : + * + * 0 6 30 31 + * +---------+----------------+---+---+ + * | opcode | LI |AA |LK | + * +---------+----------------+---+---+ + * Where AA = 0 and LK = 0 + * + * LI is a signed 24 bits integer. The real branch offset is computed + * by: imm32 = SignExtend(LI:'0b00', 32); + * + * So the maximum forward branch should be: + * (0x007fffff << 2) = 0x01fffffc = 0x1fffffc + * The maximum backward branch should be: + * (0xff800000 << 2) = 0xfe000000 = -0x2000000 + */ +static inline bool is_offset_in_branch_range(long offset) +{ + return (offset >= -0x2000000 && offset <= 0x1fffffc && !(offset & 0x3)); +} + +static inline bool is_offset_in_cond_branch_range(long offset) +{ + return offset >= -0x8000 && offset <= 0x7fff && !(offset & 0x3); +} + +static inline int create_branch(ppc_inst_t *instr, const u32 *addr, + unsigned long target, int flags) +{ + long offset; + + *instr = ppc_inst(0); + offset = target; + if (! (flags & BRANCH_ABSOLUTE)) + offset = offset - (unsigned long)addr; + + /* Check we can represent the target in the instruction format */ + if (!is_offset_in_branch_range(offset)) + return 1; + + /* Mask out the flags and target, so they don't step on each other. */ + *instr = ppc_inst(0x48000000 | (flags & 0x3) | (offset & 0x03FFFFFC)); + + return 0; +} + +int create_cond_branch(ppc_inst_t *instr, const u32 *addr, + unsigned long target, int flags); +int patch_branch(u32 *addr, unsigned long target, int flags); +int patch_instruction(u32 *addr, ppc_inst_t instr); +int raw_patch_instruction(u32 *addr, ppc_inst_t instr); +int patch_instructions(u32 *addr, u32 *code, size_t len, bool repeat_instr); + +/* + * The data patching functions patch_uint() and patch_ulong(), etc., must be + * called on aligned addresses. + * + * The instruction patching functions patch_instruction() and similar must be + * called on addresses satisfying instruction alignment requirements. + */ + +#ifdef CONFIG_PPC64 + +int patch_uint(void *addr, unsigned int val); +int patch_ulong(void *addr, unsigned long val); + +#define patch_u64 patch_ulong + +#else + +static inline int patch_uint(void *addr, unsigned int val) +{ + if (!IS_ALIGNED((unsigned long)addr, sizeof(unsigned int))) + return -EINVAL; + + return patch_instruction(addr, ppc_inst(val)); +} + +static inline int patch_ulong(void *addr, unsigned long val) +{ + if (!IS_ALIGNED((unsigned long)addr, sizeof(unsigned long))) + return -EINVAL; + + return patch_instruction(addr, ppc_inst(val)); +} + +#endif + +#define patch_u32 patch_uint + +static inline unsigned long patch_site_addr(s32 *site) +{ + return (unsigned long)site + *site; +} + +static inline int patch_instruction_site(s32 *site, ppc_inst_t instr) +{ + return patch_instruction((u32 *)patch_site_addr(site), instr); +} + +static inline int patch_branch_site(s32 *site, unsigned long target, int flags) +{ + return patch_branch((u32 *)patch_site_addr(site), target, flags); +} + +static inline int modify_instruction(unsigned int *addr, unsigned int clr, + unsigned int set) +{ + return patch_instruction(addr, ppc_inst((*addr & ~clr) | set)); +} + +static inline int modify_instruction_site(s32 *site, unsigned int clr, unsigned int set) +{ + return modify_instruction((unsigned int *)patch_site_addr(site), clr, set); +} + +static inline unsigned int branch_opcode(ppc_inst_t instr) +{ + return ppc_inst_primary_opcode(instr) & 0x3F; +} + +static inline int instr_is_branch_iform(ppc_inst_t instr) +{ + return branch_opcode(instr) == 18; +} + +static inline int instr_is_branch_bform(ppc_inst_t instr) +{ + return branch_opcode(instr) == 16; +} + +int instr_is_relative_branch(ppc_inst_t instr); +int instr_is_relative_link_branch(ppc_inst_t instr); +unsigned long branch_target(const u32 *instr); +int translate_branch(ppc_inst_t *instr, const u32 *dest, const u32 *src); +bool is_conditional_branch(ppc_inst_t instr); + +#define OP_RT_RA_MASK 0xffff0000UL +#define LIS_R2 (PPC_RAW_LIS(_R2, 0)) +#define ADDIS_R2_R12 (PPC_RAW_ADDIS(_R2, _R12, 0)) +#define ADDI_R2_R2 (PPC_RAW_ADDI(_R2, _R2, 0)) + + +static inline unsigned long ppc_function_entry(void *func) +{ +#ifdef CONFIG_PPC64_ELF_ABI_V2 + u32 *insn = func; + + /* + * A PPC64 ABIv2 function may have a local and a global entry + * point. We need to use the local entry point when patching + * functions, so identify and step over the global entry point + * sequence. + * + * The global entry point sequence is always of the form: + * + * addis r2,r12,XXXX + * addi r2,r2,XXXX + * + * A linker optimisation may convert the addis to lis: + * + * lis r2,XXXX + * addi r2,r2,XXXX + */ + if ((((*insn & OP_RT_RA_MASK) == ADDIS_R2_R12) || + ((*insn & OP_RT_RA_MASK) == LIS_R2)) && + ((*(insn+1) & OP_RT_RA_MASK) == ADDI_R2_R2)) + return (unsigned long)(insn + 2); + else + return (unsigned long)func; +#elif defined(CONFIG_PPC64_ELF_ABI_V1) + /* + * On PPC64 ABIv1 the function pointer actually points to the + * function's descriptor. The first entry in the descriptor is the + * address of the function text. + */ + return ((struct func_desc *)func)->addr; +#else + return (unsigned long)func; +#endif +} + +static inline unsigned long ppc_global_function_entry(void *func) +{ +#ifdef CONFIG_PPC64_ELF_ABI_V2 + /* PPC64 ABIv2 the global entry point is at the address */ + return (unsigned long)func; +#else + /* All other cases there is no change vs ppc_function_entry() */ + return ppc_function_entry(func); +#endif +} + +/* + * Wrapper around kallsyms_lookup() to return function entry address: + * - For ABIv1, we lookup the dot variant. + * - For ABIv2, we return the local entry point. + */ +static inline unsigned long ppc_kallsyms_lookup_name(const char *name) +{ + unsigned long addr; +#ifdef CONFIG_PPC64_ELF_ABI_V1 + /* check for dot variant */ + char dot_name[1 + KSYM_NAME_LEN]; + bool dot_appended = false; + + if (strnlen(name, KSYM_NAME_LEN) >= KSYM_NAME_LEN) + return 0; + + if (name[0] != '.') { + dot_name[0] = '.'; + dot_name[1] = '\0'; + strlcat(dot_name, name, sizeof(dot_name)); + dot_appended = true; + } else { + dot_name[0] = '\0'; + strlcat(dot_name, name, sizeof(dot_name)); + } + addr = kallsyms_lookup_name(dot_name); + if (!addr && dot_appended) + /* Let's try the original non-dot symbol lookup */ + addr = kallsyms_lookup_name(name); +#elif defined(CONFIG_PPC64_ELF_ABI_V2) + addr = kallsyms_lookup_name(name); + if (addr) + addr = ppc_function_entry((void *)addr); +#else + addr = kallsyms_lookup_name(name); +#endif + return addr; +} + +/* + * Some instruction encodings commonly used in dynamic ftracing + * and function live patching. + */ + +/* This must match the definition of STK_GOT in <asm/ppc_asm.h> */ +#ifdef CONFIG_PPC64_ELF_ABI_V2 +#define R2_STACK_OFFSET 24 +#else +#define R2_STACK_OFFSET 40 +#endif + +#define PPC_INST_LD_TOC PPC_RAW_LD(_R2, _R1, R2_STACK_OFFSET) + +/* usually preceded by a mflr r0 */ +#define PPC_INST_STD_LR PPC_RAW_STD(_R0, _R1, PPC_LR_STKOFF) + +#endif /* _ASM_POWERPC_CODE_PATCHING_H */ diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h index ba7b1973866e..b0f200aba2b3 100644 --- a/arch/powerpc/include/asm/thread_info.h +++ b/arch/powerpc/include/asm/thread_info.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* thread_info.h: PowerPC low-level thread information * adapted from the i386 version by Paul Mackerras * @@ -8,43 +9,64 @@ #ifndef _ASM_POWERPC_THREAD_INFO_H #define _ASM_POWERPC_THREAD_INFO_H +#include <asm/asm-const.h> +#include <asm/page.h> + #ifdef __KERNEL__ -/* We have 8k stacks on ppc32 and 16k on ppc64 */ +#if defined(CONFIG_KASAN) && CONFIG_THREAD_SHIFT < 15 +#define MIN_THREAD_SHIFT (CONFIG_THREAD_SHIFT + 1) +#else +#define MIN_THREAD_SHIFT CONFIG_THREAD_SHIFT +#endif -#if defined(CONFIG_PPC64) -#define THREAD_SHIFT 14 -#elif defined(CONFIG_PPC_256K_PAGES) -#define THREAD_SHIFT 15 +#if defined(CONFIG_VMAP_STACK) && MIN_THREAD_SHIFT < PAGE_SHIFT +#define THREAD_SHIFT PAGE_SHIFT #else -#define THREAD_SHIFT 13 +#define THREAD_SHIFT MIN_THREAD_SHIFT #endif #define THREAD_SIZE (1 << THREAD_SHIFT) -#ifdef CONFIG_PPC64 -#define CURRENT_THREAD_INFO(dest, sp) clrrdi dest, sp, THREAD_SHIFT +/* + * By aligning VMAP'd stacks to 2 * THREAD_SIZE, we can detect overflow by + * checking sp & (1 << THREAD_SHIFT), which we can do cheaply in the entry + * assembly. + */ +#ifdef CONFIG_VMAP_STACK +#define THREAD_ALIGN_SHIFT (THREAD_SHIFT + 1) #else -#define CURRENT_THREAD_INFO(dest, sp) rlwinm dest, sp, 0, 0, 31-THREAD_SHIFT +#define THREAD_ALIGN_SHIFT THREAD_SHIFT #endif -#ifndef __ASSEMBLY__ +#define THREAD_ALIGN (1 << THREAD_ALIGN_SHIFT) + +#ifndef __ASSEMBLER__ #include <linux/cache.h> #include <asm/processor.h> -#include <asm/page.h> -#include <linux/stringify.h> +#include <asm/accounting.h> +#include <asm/ppc_asm.h> +#define SLB_PRELOAD_NR 16U /* * low level task data. */ struct thread_info { - struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ - int cpu; /* cpu we're on */ int preempt_count; /* 0 => preemptable, <0 => BUG */ - struct restart_block restart_block; +#ifdef CONFIG_SMP + unsigned int cpu; +#endif unsigned long local_flags; /* private flags for thread */ +#ifdef CONFIG_LIVEPATCH_64 + unsigned long *livepatch_sp; +#endif +#if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC32) + struct cpu_accounting_data accounting; +#endif + unsigned char slb_preload_nr; + unsigned char slb_preload_tail; + u32 slb_preload_esid[SLB_PRELOAD_NR]; /* low level flags - has atomic operations done on it */ unsigned long flags ____cacheline_aligned_in_smp; @@ -55,34 +77,19 @@ struct thread_info { */ #define INIT_THREAD_INFO(tsk) \ { \ - .task = &tsk, \ - .exec_domain = &default_exec_domain, \ - .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ - .restart_block = { \ - .fn = do_no_restart_syscall, \ - }, \ .flags = 0, \ } -#define init_thread_info (init_thread_union.thread_info) -#define init_stack (init_thread_union.stack) - #define THREAD_SIZE_ORDER (THREAD_SHIFT - PAGE_SHIFT) /* how to get the thread information struct from C */ -static inline struct thread_info *current_thread_info(void) -{ - register unsigned long sp asm("r1"); - - /* gcc4, at least, is smart enough to turn this into a single - * rlwinm for ppc32 and clrrdi for ppc64 */ - return (struct thread_info *)(sp & ~(THREAD_SIZE-1)); -} +extern int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src); -#endif /* __ASSEMBLY__ */ +void arch_setup_new_exec(void); +#define arch_setup_new_exec arch_setup_new_exec -#define PREEMPT_ACTIVE 0x10000000 +#endif /* __ASSEMBLER__ */ /* * thread information flag bit numbers @@ -90,14 +97,13 @@ static inline struct thread_info *current_thread_info(void) #define TIF_SYSCALL_TRACE 0 /* syscall trace active */ #define TIF_SIGPENDING 1 /* signal pending */ #define TIF_NEED_RESCHED 2 /* rescheduling necessary */ -#define TIF_POLLING_NRFLAG 3 /* true if poll_idle() is polling - TIF_NEED_RESCHED */ -#define TIF_32BIT 4 /* 32 bit binary */ -#define TIF_PERFMON_WORK 5 /* work for pfm_handle_work() */ -#define TIF_PERFMON_CTXSW 6 /* perfmon needs ctxsw calls */ +#define TIF_NOTIFY_SIGNAL 3 /* signal notifications exist */ +#define TIF_SYSCALL_EMU 4 /* syscall emulation active */ +#define TIF_RESTORE_TM 5 /* need to restore TM FP/VEC/VSX */ +#define TIF_PATCH_PENDING 6 /* pending live patching update */ #define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */ #define TIF_SINGLESTEP 8 /* singlestepping active */ -#define TIF_NOHZ 9 /* in adaptive nohz mode */ +#define TIF_NEED_RESCHED_LAZY 9 /* Scheduler driven lazy preemption */ #define TIF_SECCOMP 10 /* secure computing */ #define TIF_RESTOREALL 11 /* Restore all regs (implies NOERROR) */ #define TIF_NOERROR 12 /* Force successful syscall return */ @@ -107,15 +113,22 @@ static inline struct thread_info *current_thread_info(void) #define TIF_EMULATE_STACK_STORE 16 /* Is an instruction emulation for stack store? */ #define TIF_MEMDIE 17 /* is terminating due to OOM killer */ +#if defined(CONFIG_PPC64) +#define TIF_ELF2ABI 18 /* function descriptors must die! */ +#endif +#define TIF_POLLING_NRFLAG 19 /* true if poll_idle() is polling TIF_NEED_RESCHED */ +#define TIF_32BIT 20 /* 32 bit binary */ /* as above, but as bit values */ #define _TIF_SYSCALL_TRACE (1<<TIF_SYSCALL_TRACE) #define _TIF_SIGPENDING (1<<TIF_SIGPENDING) #define _TIF_NEED_RESCHED (1<<TIF_NEED_RESCHED) +#define _TIF_NEED_RESCHED_LAZY (1<<TIF_NEED_RESCHED_LAZY) +#define _TIF_NOTIFY_SIGNAL (1<<TIF_NOTIFY_SIGNAL) #define _TIF_POLLING_NRFLAG (1<<TIF_POLLING_NRFLAG) #define _TIF_32BIT (1<<TIF_32BIT) -#define _TIF_PERFMON_WORK (1<<TIF_PERFMON_WORK) -#define _TIF_PERFMON_CTXSW (1<<TIF_PERFMON_CTXSW) +#define _TIF_RESTORE_TM (1<<TIF_RESTORE_TM) +#define _TIF_PATCH_PENDING (1<<TIF_PATCH_PENDING) #define _TIF_SYSCALL_AUDIT (1<<TIF_SYSCALL_AUDIT) #define _TIF_SINGLESTEP (1<<TIF_SINGLESTEP) #define _TIF_SECCOMP (1<<TIF_SECCOMP) @@ -125,52 +138,36 @@ static inline struct thread_info *current_thread_info(void) #define _TIF_UPROBE (1<<TIF_UPROBE) #define _TIF_SYSCALL_TRACEPOINT (1<<TIF_SYSCALL_TRACEPOINT) #define _TIF_EMULATE_STACK_STORE (1<<TIF_EMULATE_STACK_STORE) -#define _TIF_NOHZ (1<<TIF_NOHZ) -#define _TIF_SYSCALL_T_OR_A (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ +#define _TIF_SYSCALL_EMU (1<<TIF_SYSCALL_EMU) +#define _TIF_SYSCALL_DOTRACE (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ _TIF_SECCOMP | _TIF_SYSCALL_TRACEPOINT | \ - _TIF_NOHZ) + _TIF_SYSCALL_EMU) #define _TIF_USER_WORK_MASK (_TIF_SIGPENDING | _TIF_NEED_RESCHED | \ - _TIF_NOTIFY_RESUME | _TIF_UPROBE) + _TIF_NEED_RESCHED_LAZY | _TIF_NOTIFY_RESUME | \ + _TIF_UPROBE | _TIF_RESTORE_TM | \ + _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL) + #define _TIF_PERSYSCALL_MASK (_TIF_RESTOREALL|_TIF_NOERROR) /* Bits in local_flags */ /* Don't move TLF_NAPPING without adjusting the code in entry_32.S */ #define TLF_NAPPING 0 /* idle thread enabled NAP mode */ #define TLF_SLEEPING 1 /* suspend code enabled SLEEP mode */ -#define TLF_RESTORE_SIGMASK 2 /* Restore signal mask in do_signal */ #define TLF_LAZY_MMU 3 /* tlb_batch is active */ #define TLF_RUNLATCH 4 /* Is the runlatch enabled? */ #define _TLF_NAPPING (1 << TLF_NAPPING) #define _TLF_SLEEPING (1 << TLF_SLEEPING) -#define _TLF_RESTORE_SIGMASK (1 << TLF_RESTORE_SIGMASK) #define _TLF_LAZY_MMU (1 << TLF_LAZY_MMU) #define _TLF_RUNLATCH (1 << TLF_RUNLATCH) -#ifndef __ASSEMBLY__ -#define HAVE_SET_RESTORE_SIGMASK 1 -static inline void set_restore_sigmask(void) -{ - struct thread_info *ti = current_thread_info(); - ti->local_flags |= _TLF_RESTORE_SIGMASK; - WARN_ON(!test_bit(TIF_SIGPENDING, &ti->flags)); -} -static inline void clear_restore_sigmask(void) -{ - current_thread_info()->local_flags &= ~_TLF_RESTORE_SIGMASK; -} -static inline bool test_restore_sigmask(void) -{ - return current_thread_info()->local_flags & _TLF_RESTORE_SIGMASK; -} -static inline bool test_and_clear_restore_sigmask(void) +#ifndef __ASSEMBLER__ + +static inline void clear_thread_local_flags(unsigned int flags) { struct thread_info *ti = current_thread_info(); - if (!(ti->local_flags & _TLF_RESTORE_SIGMASK)) - return false; - ti->local_flags &= ~_TLF_RESTORE_SIGMASK; - return true; + ti->local_flags &= ~flags; } static inline bool test_thread_local_flags(unsigned int flags) @@ -179,13 +176,64 @@ static inline bool test_thread_local_flags(unsigned int flags) return (ti->local_flags & flags) != 0; } -#ifdef CONFIG_PPC64 +#ifdef CONFIG_COMPAT #define is_32bit_task() (test_thread_flag(TIF_32BIT)) +#define is_tsk_32bit_task(tsk) (test_tsk_thread_flag(tsk, TIF_32BIT)) +#define clear_tsk_compat_task(tsk) (clear_tsk_thread_flag(p, TIF_32BIT)) #else -#define is_32bit_task() (1) +#define is_32bit_task() (IS_ENABLED(CONFIG_PPC32)) +#define is_tsk_32bit_task(tsk) (IS_ENABLED(CONFIG_PPC32)) +#define clear_tsk_compat_task(tsk) do { } while (0) +#endif + +#if defined(CONFIG_PPC64) +#define is_elf2_task() (test_thread_flag(TIF_ELF2ABI)) +#else +#define is_elf2_task() (0) +#endif + +/* + * Walks up the stack frames to make sure that the specified object is + * entirely contained by a single stack frame. + * + * Returns: + * GOOD_FRAME if within a frame + * BAD_STACK if placed across a frame boundary (or outside stack) + */ +static inline int arch_within_stack_frames(const void * const stack, + const void * const stackend, + const void *obj, unsigned long len) +{ + const void *params; + const void *frame; + + params = *(const void * const *)current_stack_pointer + STACK_FRAME_PARAMS; + frame = **(const void * const * const *)current_stack_pointer; + + /* + * low -----------------------------------------------------------> high + * [backchain][metadata][params][local vars][saved registers][backchain] + * ^------------------------------------^ + * | allows copies only in this region | + * | | + * params frame + * The metadata region contains the saved LR, CR etc. + */ + while (stack <= frame && frame < stackend) { + if (obj + len <= frame) + return obj >= params ? GOOD_FRAME : BAD_STACK; + params = frame + STACK_FRAME_PARAMS; + frame = *(const void * const *)frame; + } + + return BAD_STACK; +} + +#ifdef CONFIG_PPC32 +extern void *emergency_ctx[]; #endif -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/time.h b/arch/powerpc/include/asm/time.h index c1f267694acb..7991ab1d4cb8 100644 --- a/arch/powerpc/include/asm/time.h +++ b/arch/powerpc/include/asm/time.h @@ -1,13 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Common time prototypes and such for all ppc machines. * * Written by Cort Dougan (cort@cs.nmt.edu) to merge * Paul Mackerras' version and mine for PReP and Pmac. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef __POWERPC_TIME_H @@ -18,20 +14,24 @@ #include <linux/percpu.h> #include <asm/processor.h> +#include <asm/cpu_has_feature.h> +#include <asm/vdso/timebase.h> /* time.c */ +extern u64 decrementer_max; + extern unsigned long tb_ticks_per_jiffy; extern unsigned long tb_ticks_per_usec; extern unsigned long tb_ticks_per_sec; extern struct clock_event_device decrementer_clockevent; +extern u64 decrementer_max; -struct rtc_time; -extern void to_tm(int tim, struct rtc_time * tm); -extern void GregorianDay(struct rtc_time *tm); extern void generic_calibrate_decr(void); -extern void set_dec_cpu6(unsigned int val); +#ifdef CONFIG_PPC_SPLPAR +extern u64 get_boot_tb(void); +#endif /* Some sane defaults: 125 MHz timebase, 1GHz processor */ extern unsigned long ppc_proc_freq; @@ -39,98 +39,19 @@ extern unsigned long ppc_proc_freq; extern unsigned long ppc_tb_freq; #define DEFAULT_TB_FREQ 125000000UL +extern bool tb_invalid; + struct div_result { u64 result_high; u64 result_low; }; -/* Accessor functions for the timebase (RTC on 601) registers. */ -/* If one day CONFIG_POWER is added just define __USE_RTC as 1 */ -#ifdef CONFIG_6xx -#define __USE_RTC() (!cpu_has_feature(CPU_FTR_USE_TB)) -#else -#define __USE_RTC() 0 -#endif - -#ifdef CONFIG_PPC64 - -/* For compatibility, get_tbl() is defined as get_tb() on ppc64 */ -#define get_tbl get_tb - -#else - -static inline unsigned long get_tbl(void) +static inline u64 get_vtb(void) { -#if defined(CONFIG_403GCX) - unsigned long tbl; - asm volatile("mfspr %0, 0x3dd" : "=r" (tbl)); - return tbl; -#else - return mftbl(); -#endif -} + if (cpu_has_feature(CPU_FTR_ARCH_207S)) + return mfspr(SPRN_VTB); -static inline unsigned int get_tbu(void) -{ -#ifdef CONFIG_403GCX - unsigned int tbu; - asm volatile("mfspr %0, 0x3dc" : "=r" (tbu)); - return tbu; -#else - return mftbu(); -#endif -} -#endif /* !CONFIG_PPC64 */ - -static inline unsigned int get_rtcl(void) -{ - unsigned int rtcl; - - asm volatile("mfrtcl %0" : "=r" (rtcl)); - return rtcl; -} - -static inline u64 get_rtc(void) -{ - unsigned int hi, lo, hi2; - - do { - asm volatile("mfrtcu %0; mfrtcl %1; mfrtcu %2" - : "=r" (hi), "=r" (lo), "=r" (hi2)); - } while (hi2 != hi); - return (u64)hi * 1000000000 + lo; -} - -#ifdef CONFIG_PPC64 -static inline u64 get_tb(void) -{ - return mftb(); -} -#else /* CONFIG_PPC64 */ -static inline u64 get_tb(void) -{ - unsigned int tbhi, tblo, tbhi2; - - do { - tbhi = get_tbu(); - tblo = get_tbl(); - tbhi2 = get_tbu(); - } while (tbhi != tbhi2); - - return ((u64)tbhi << 32) | tblo; -} -#endif /* !CONFIG_PPC64 */ - -static inline u64 get_tb_or_rtc(void) -{ - return __USE_RTC() ? get_rtc() : get_tb(); -} - -static inline void set_tb(unsigned int upper, unsigned int lower) -{ - mtspr(SPRN_TBWL, 0); - mtspr(SPRN_TBWU, upper); - mtspr(SPRN_TBWL, lower); + return 0; } /* Accessor functions for the decrementer register. @@ -139,13 +60,9 @@ static inline void set_tb(unsigned int upper, unsigned int lower) * in auto-reload mode. The problem is PIT stops counting when it * hits zero. If it would wrap, we could use it just like a decrementer. */ -static inline unsigned int get_dec(void) +static inline u64 get_dec(void) { -#if defined(CONFIG_40x) - return (mfspr(SPRN_PIT)); -#else - return (mfspr(SPRN_DEC)); -#endif + return mfspr(SPRN_DEC); } /* @@ -153,27 +70,17 @@ static inline unsigned int get_dec(void) * in when the decrementer generates its interrupt: on the 1 to 0 * transition for Book E/4xx, but on the 0 to -1 transition for others. */ -static inline void set_dec(int val) +static inline void set_dec(u64 val) { -#if defined(CONFIG_40x) - mtspr(SPRN_PIT, val); -#elif defined(CONFIG_8xx_CPU6) - set_dec_cpu6(val - 1); -#else -#ifndef CONFIG_BOOKE - --val; -#endif - mtspr(SPRN_DEC, val); -#endif /* not 40x or 8xx_CPU6 */ + if (IS_ENABLED(CONFIG_BOOKE)) + mtspr(SPRN_DEC, val); + else + mtspr(SPRN_DEC, val - 1); } static inline unsigned long tb_ticks_since(unsigned long tstamp) { - if (__USE_RTC()) { - int delta = get_rtcl() - (unsigned int) tstamp; - return delta < 0 ? delta + 1000000000 : delta; - } - return get_tbl() - tstamp; + return mftb() - tstamp; } #define mulhwu(x,y) \ @@ -183,23 +90,31 @@ static inline unsigned long tb_ticks_since(unsigned long tstamp) #define mulhdu(x,y) \ ({unsigned long z; asm ("mulhdu %0,%1,%2" : "=r" (z) : "r" (x), "r" (y)); z;}) #else -extern u64 mulhdu(u64, u64); +#define mulhdu(x, y) mul_u64_u64_shr(x, y, 64) #endif -extern void div128_by_32(u64 dividend_high, u64 dividend_low, - unsigned divisor, struct div_result *dr); +extern void secondary_cpu_time_init(void); +extern void __init time_init(void); -/* Used to store Processor Utilization register (purr) values */ +DECLARE_PER_CPU(u64, decrementers_next_tb); -struct cpu_usage { - u64 current_tb; /* Holds the current purr register values */ -}; +static inline u64 timer_get_next_tb(void) +{ + return __this_cpu_read(decrementers_next_tb); +} + +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE +void timer_rearm_host_dec(u64 now); +#endif -DECLARE_PER_CPU(struct cpu_usage, cpu_usage_array); +/* Convert timebase ticks to nanoseconds */ +unsigned long long tb_to_ns(unsigned long long tb_ticks); -extern void secondary_cpu_time_init(void); +void timer_broadcast_interrupt(void); -DECLARE_PER_CPU(u64, decrementers_next_tb); +/* SPLPAR and VIRT_CPU_ACCOUNTING_NATIVE */ +void pseries_accumulate_stolen_time(void); +u64 pseries_calculate_stolen_time(u64 stop_tb); #endif /* __KERNEL__ */ #endif /* __POWERPC_TIME_H */ diff --git a/arch/powerpc/include/asm/timex.h b/arch/powerpc/include/asm/timex.h index c55e14f7ef44..14b4489de52c 100644 --- a/arch/powerpc/include/asm/timex.h +++ b/arch/powerpc/include/asm/timex.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_TIMEX_H #define _ASM_POWERPC_TIMEX_H @@ -8,7 +9,7 @@ */ #include <asm/cputable.h> -#include <asm/reg.h> +#include <asm/vdso/timebase.h> #define CLOCK_TICK_RATE 1024000 /* Underlying HZ */ @@ -16,35 +17,9 @@ typedef unsigned long cycles_t; static inline cycles_t get_cycles(void) { -#ifdef __powerpc64__ return mftb(); -#else - cycles_t ret; - - /* - * For the "cycle" counter we use the timebase lower half. - * Currently only used on SMP. - */ - - ret = 0; - - __asm__ __volatile__( - "97: mftb %0\n" - "99:\n" - ".section __ftr_fixup,\"a\"\n" - ".align 2\n" - "98:\n" - " .long %1\n" - " .long 0\n" - " .long 97b-98b\n" - " .long 99b-98b\n" - " .long 0\n" - " .long 0\n" - ".previous" - : "=r" (ret) : "i" (CPU_FTR_601)); - return ret; -#endif } +#define get_cycles get_cycles #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_TIMEX_H */ diff --git a/arch/powerpc/include/asm/tlb.h b/arch/powerpc/include/asm/tlb.h index e2b428b0f7ba..2058e8d3e013 100644 --- a/arch/powerpc/include/asm/tlb.h +++ b/arch/powerpc/include/asm/tlb.h @@ -1,23 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * TLB shootdown specifics for powerpc * * Copyright (C) 2002 Anton Blanchard, IBM Corp. * Copyright (C) 2002 Paul Mackerras, IBM Corp. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef _ASM_POWERPC_TLB_H #define _ASM_POWERPC_TLB_H #ifdef __KERNEL__ #ifndef __powerpc64__ -#include <asm/pgtable.h> +#include <linux/pgtable.h> #endif -#include <asm/pgalloc.h> -#include <asm/tlbflush.h> #ifndef __powerpc64__ #include <asm/page.h> #include <asm/mmu.h> @@ -25,25 +19,76 @@ #include <linux/pagemap.h> -#define tlb_start_vma(tlb, vma) do { } while (0) -#define tlb_end_vma(tlb, vma) do { } while (0) +static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, + unsigned long address); +#define __tlb_remove_tlb_entry __tlb_remove_tlb_entry +#define tlb_flush tlb_flush extern void tlb_flush(struct mmu_gather *tlb); +/* + * book3s: + * Hash does not use the linux page-tables, so we can avoid + * the TLB invalidate for page-table freeing, Radix otoh does use the + * page-tables and needs the TLBI. + * + * nohash: + * We still do TLB invalidate in the __pte_free_tlb routine before we + * add the page table pages to mmu gather table batch. + */ +#define tlb_needs_table_invalidate() radix_enabled() +#define __HAVE_ARCH_TLB_REMOVE_TABLE /* Get the generic bits... */ #include <asm-generic/tlb.h> -extern void flush_hash_entry(struct mm_struct *mm, pte_t *ptep, - unsigned long address); - static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long address) { -#ifdef CONFIG_PPC_STD_MMU_32 +#ifdef CONFIG_PPC_BOOK3S_32 if (pte_val(*ptep) & _PAGE_HASHPTE) flush_hash_entry(tlb->mm, ptep, address); #endif } +#ifdef CONFIG_SMP +static inline int mm_is_core_local(struct mm_struct *mm) +{ + return cpumask_subset(mm_cpumask(mm), + topology_sibling_cpumask(smp_processor_id())); +} + +#ifdef CONFIG_PPC_BOOK3S_64 +static inline int mm_is_thread_local(struct mm_struct *mm) +{ + if (atomic_read(&mm->context.active_cpus) > 1) + return false; + return cpumask_test_cpu(smp_processor_id(), mm_cpumask(mm)); +} +#else /* CONFIG_PPC_BOOK3S_64 */ +static inline int mm_is_thread_local(struct mm_struct *mm) +{ + return cpumask_equal(mm_cpumask(mm), + cpumask_of(smp_processor_id())); +} +#endif /* !CONFIG_PPC_BOOK3S_64 */ + +#else /* CONFIG_SMP */ +static inline int mm_is_core_local(struct mm_struct *mm) +{ + return 1; +} + +static inline int mm_is_thread_local(struct mm_struct *mm) +{ + return 1; +} +#endif + +#define arch_supports_page_table_move arch_supports_page_table_move +static inline bool arch_supports_page_table_move(void) +{ + return radix_enabled(); +} + #endif /* __KERNEL__ */ #endif /* __ASM_POWERPC_TLB_H */ diff --git a/arch/powerpc/include/asm/tlbflush.h b/arch/powerpc/include/asm/tlbflush.h index 2def01ed0cb2..61fba43bf8b2 100644 --- a/arch/powerpc/include/asm/tlbflush.h +++ b/arch/powerpc/include/asm/tlbflush.h @@ -1,175 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_TLBFLUSH_H #define _ASM_POWERPC_TLBFLUSH_H -/* - * TLB flushing: - * - * - flush_tlb_mm(mm) flushes the specified mm context TLB's - * - flush_tlb_page(vma, vmaddr) flushes one page - * - local_flush_tlb_mm(mm, full) flushes the specified mm context on - * the local processor - * - local_flush_tlb_page(vma, vmaddr) flushes one page on the local processor - * - flush_tlb_page_nohash(vma, vmaddr) flushes one page if SW loaded TLB - * - flush_tlb_range(vma, start, end) flushes a range of pages - * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ -#ifdef __KERNEL__ - -#ifdef CONFIG_PPC_MMU_NOHASH -/* - * TLB flushing for software loaded TLB chips - * - * TODO: (CONFIG_FSL_BOOKE) determine if flush_tlb_range & - * flush_tlb_kernel_range are best implemented as tlbia vs - * specific tlbie's - */ - -struct vm_area_struct; -struct mm_struct; - -#define MMU_NO_CONTEXT ((unsigned int)-1) - -extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, - unsigned long end); -extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); - -extern void local_flush_tlb_mm(struct mm_struct *mm); -extern void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); - -extern void __local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, - int tsize, int ind); - -#ifdef CONFIG_SMP -extern void flush_tlb_mm(struct mm_struct *mm); -extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); -extern void __flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, - int tsize, int ind); -#else -#define flush_tlb_mm(mm) local_flush_tlb_mm(mm) -#define flush_tlb_page(vma,addr) local_flush_tlb_page(vma,addr) -#define __flush_tlb_page(mm,addr,p,i) __local_flush_tlb_page(mm,addr,p,i) -#endif -#define flush_tlb_page_nohash(vma,addr) flush_tlb_page(vma,addr) - -#elif defined(CONFIG_PPC_STD_MMU_32) - -/* - * TLB flushing for "classic" hash-MMU 32-bit CPUs, 6xx, 7xx, 7xxx - */ -extern void flush_tlb_mm(struct mm_struct *mm); -extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); -extern void flush_tlb_page_nohash(struct vm_area_struct *vma, unsigned long addr); -extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, - unsigned long end); -extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); -static inline void local_flush_tlb_page(struct vm_area_struct *vma, - unsigned long vmaddr) -{ - flush_tlb_page(vma, vmaddr); -} -static inline void local_flush_tlb_mm(struct mm_struct *mm) -{ - flush_tlb_mm(mm); -} - -#elif defined(CONFIG_PPC_STD_MMU_64) - -#define MMU_NO_CONTEXT 0 - -/* - * TLB flushing for 64-bit hash-MMU CPUs - */ - -#include <linux/percpu.h> -#include <asm/page.h> - -#define PPC64_TLB_BATCH_NR 192 - -struct ppc64_tlb_batch { - int active; - unsigned long index; - struct mm_struct *mm; - real_pte_t pte[PPC64_TLB_BATCH_NR]; - unsigned long vpn[PPC64_TLB_BATCH_NR]; - unsigned int psize; - int ssize; -}; -DECLARE_PER_CPU(struct ppc64_tlb_batch, ppc64_tlb_batch); - -extern void __flush_tlb_pending(struct ppc64_tlb_batch *batch); - -#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE - -static inline void arch_enter_lazy_mmu_mode(void) -{ - struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch); - - batch->active = 1; -} - -static inline void arch_leave_lazy_mmu_mode(void) -{ - struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch); - - if (batch->index) - __flush_tlb_pending(batch); - batch->active = 0; -} - -#define arch_flush_lazy_mmu_mode() do {} while (0) - - -extern void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, - int ssize, int local); -extern void flush_hash_range(unsigned long number, int local); - - -static inline void local_flush_tlb_mm(struct mm_struct *mm) -{ -} - -static inline void flush_tlb_mm(struct mm_struct *mm) -{ -} - -static inline void local_flush_tlb_page(struct vm_area_struct *vma, - unsigned long vmaddr) -{ -} - -static inline void flush_tlb_page(struct vm_area_struct *vma, - unsigned long vmaddr) -{ -} - -static inline void flush_tlb_page_nohash(struct vm_area_struct *vma, - unsigned long vmaddr) -{ -} - -static inline void flush_tlb_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end) -{ -} - -static inline void flush_tlb_kernel_range(unsigned long start, - unsigned long end) -{ -} - -/* Private function for use by PCI IO mapping code */ -extern void __flush_hash_table_range(struct mm_struct *mm, unsigned long start, - unsigned long end); -extern void flush_tlb_pmd_range(struct mm_struct *mm, pmd_t *pmd, - unsigned long addr); +#ifdef CONFIG_PPC_BOOK3S +#include <asm/book3s/tlbflush.h> #else -#error Unsupported MMU type -#endif +#include <asm/nohash/tlbflush.h> +#endif /* !CONFIG_PPC_BOOK3S */ -#endif /*__KERNEL__ */ #endif /* _ASM_POWERPC_TLBFLUSH_H */ diff --git a/arch/powerpc/include/asm/tm.h b/arch/powerpc/include/asm/tm.h index 9dfbc34bdbf5..d700affba448 100644 --- a/arch/powerpc/include/asm/tm.h +++ b/arch/powerpc/include/asm/tm.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Transactional memory support routines to reclaim and recheckpoint * transactional process state. @@ -7,16 +8,15 @@ #include <uapi/asm/tm.h> -#ifdef CONFIG_PPC_TRANSACTIONAL_MEM -extern void do_load_up_transact_fpu(struct thread_struct *thread); -extern void do_load_up_transact_altivec(struct thread_struct *thread); -#endif +#ifndef __ASSEMBLER__ -extern void tm_enable(void); extern void tm_reclaim(struct thread_struct *thread, - unsigned long orig_msr, uint8_t cause); -extern void tm_recheckpoint(struct thread_struct *thread, - unsigned long orig_msr); -extern void tm_abort(uint8_t cause); + uint8_t cause); +extern void tm_reclaim_current(uint8_t cause); +extern void tm_recheckpoint(struct thread_struct *thread); extern void tm_save_sprs(struct thread_struct *thread); extern void tm_restore_sprs(struct thread_struct *thread); + +extern bool tm_suspend_disabled; + +#endif /* __ASSEMBLER__ */ diff --git a/arch/powerpc/include/asm/topology.h b/arch/powerpc/include/asm/topology.h index 161ab662843b..66ed5fe1b718 100644 --- a/arch/powerpc/include/asm/topology.h +++ b/arch/powerpc/include/asm/topology.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_TOPOLOGY_H #define _ASM_POWERPC_TOPOLOGY_H #ifdef __KERNEL__ @@ -5,28 +6,18 @@ struct device; struct device_node; +struct drmem_lmb; #ifdef CONFIG_NUMA /* - * Before going off node we want the VM to try and reclaim from the local - * node. It does this if the remote distance is larger than RECLAIM_DISTANCE. - * With the default REMOTE_DISTANCE of 20 and the default RECLAIM_DISTANCE of - * 20, we never reclaim and go off node straight away. - * - * To fix this we choose a smaller value of RECLAIM_DISTANCE. + * If zone_reclaim_mode is enabled, a RECLAIM_DISTANCE of 10 will mean that + * all zones on all nodes will be eligible for zone_reclaim(). */ #define RECLAIM_DISTANCE 10 #include <asm/mmzone.h> -static inline int cpu_to_node(int cpu) -{ - return numa_cpu_lookup_table[cpu]; -} - -#define parent_node(node) (node) - #define cpumask_of_node(node) ((node) == -1 ? \ cpu_all_mask : \ node_to_cpumask_map[node]) @@ -45,6 +36,7 @@ static inline int pcibus_to_node(struct pci_bus *bus) cpu_all_mask : \ cpumask_of_node(pcibus_to_node(bus))) +int cpu_relative_distance(__be32 *cpu1_assoc, __be32 *cpu2_assoc); extern int __node_distance(int, int); #define node_distance(a, b) __node_distance(a, b) @@ -53,8 +45,36 @@ extern void __init dump_numa_cpu_topology(void); extern int sysfs_add_device_to_node(struct device *dev, int nid); extern void sysfs_remove_device_from_node(struct device *dev, int nid); +static inline void update_numa_cpu_lookup_table(unsigned int cpu, int node) +{ + numa_cpu_lookup_table[cpu] = node; +} + +static inline int early_cpu_to_node(int cpu) +{ + int nid; + + nid = numa_cpu_lookup_table[cpu]; + + /* + * Fall back to node 0 if nid is unset (it should be, except bugs). + * This allows callers to safely do NODE_DATA(early_cpu_to_node(cpu)). + */ + return (nid < 0) ? 0 : nid; +} + +int of_drconf_to_nid_single(struct drmem_lmb *lmb); +void update_numa_distance(struct device_node *node); + +extern void map_cpu_to_node(int cpu, int node); +#ifdef CONFIG_HOTPLUG_CPU +extern void unmap_cpu_from_node(unsigned long cpu); +#endif /* CONFIG_HOTPLUG_CPU */ + #else +static inline int early_cpu_to_node(int cpu) { return 0; } + static inline void dump_numa_cpu_topology(void) {} static inline int sysfs_add_device_to_node(struct device *dev, int nid) @@ -66,40 +86,95 @@ static inline void sysfs_remove_device_from_node(struct device *dev, int nid) { } -#endif /* CONFIG_NUMA */ -#if defined(CONFIG_NUMA) && defined(CONFIG_PPC_SPLPAR) -extern int start_topology_update(void); -extern int stop_topology_update(void); -extern int prrn_is_enabled(void); -#else -static inline int start_topology_update(void) +static inline void update_numa_cpu_lookup_table(unsigned int cpu, int node) {} + +static inline int cpu_relative_distance(__be32 *cpu1_assoc, __be32 *cpu2_assoc) { return 0; } -static inline int stop_topology_update(void) + +static inline int of_drconf_to_nid_single(struct drmem_lmb *lmb) { - return 0; + return first_online_node; } -static inline int prrn_is_enabled(void) + +static inline void update_numa_distance(struct device_node *node) {} + +#ifdef CONFIG_SMP +static inline void map_cpu_to_node(int cpu, int node) {} +#ifdef CONFIG_HOTPLUG_CPU +static inline void unmap_cpu_from_node(unsigned long cpu) {} +#endif /* CONFIG_HOTPLUG_CPU */ +#endif /* CONFIG_SMP */ + +#endif /* CONFIG_NUMA */ + +#if defined(CONFIG_NUMA) && defined(CONFIG_PPC_SPLPAR) +void find_and_update_cpu_nid(int cpu); +extern int cpu_to_coregroup_id(int cpu); +#else +static inline void find_and_update_cpu_nid(int cpu) {} +static inline int cpu_to_coregroup_id(int cpu) { +#ifdef CONFIG_SMP + return cpu_to_core_id(cpu); +#else return 0; +#endif } + #endif /* CONFIG_NUMA && CONFIG_PPC_SPLPAR */ #include <asm-generic/topology.h> #ifdef CONFIG_SMP #include <asm/cputable.h> -#define smt_capable() (cpu_has_feature(CPU_FTR_SMT)) + +struct cpumask *cpu_coregroup_mask(int cpu); +const struct cpumask *cpu_die_mask(int cpu); +int cpu_die_id(int cpu); #ifdef CONFIG_PPC64 #include <asm/smp.h> -#define topology_thread_cpumask(cpu) (per_cpu(cpu_sibling_map, cpu)) -#define topology_core_cpumask(cpu) (per_cpu(cpu_core_map, cpu)) -#define topology_core_id(cpu) (cpu_to_core_id(cpu)) +#define topology_physical_package_id(cpu) (cpu_to_chip_id(cpu)) +#define topology_sibling_cpumask(cpu) (per_cpu(cpu_sibling_map, cpu)) +#define topology_core_cpumask(cpu) (per_cpu(cpu_core_map, cpu)) +#define topology_core_id(cpu) (cpu_to_core_id(cpu)) +#define topology_die_id(cpu) (cpu_die_id(cpu)) +#define topology_die_cpumask(cpu) (cpu_die_mask(cpu)) + +#endif #endif + +#ifdef CONFIG_HOTPLUG_SMT +#include <linux/cpu_smt.h> +#include <linux/cpumask.h> +#include <asm/cputhreads.h> + +static inline bool topology_is_primary_thread(unsigned int cpu) +{ + return cpu == cpu_first_thread_sibling(cpu); +} +#define topology_is_primary_thread topology_is_primary_thread + +static inline bool topology_smt_thread_allowed(unsigned int cpu) +{ + return cpu_thread_in_core(cpu) < cpu_smt_num_threads; +} + +#define topology_is_core_online topology_is_core_online +static inline bool topology_is_core_online(unsigned int cpu) +{ + int i, first_cpu = cpu_first_thread_sibling(cpu); + + for (i = first_cpu; i < first_cpu + threads_per_core; ++i) { + if (cpu_online(i)) + return true; + } + return false; +} #endif #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/trace.h b/arch/powerpc/include/asm/trace.h index 5712f06905a9..a7b69b25296b 100644 --- a/arch/powerpc/include/asm/trace.h +++ b/arch/powerpc/include/asm/trace.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #undef TRACE_SYSTEM #define TRACE_SYSTEM powerpc @@ -53,16 +54,34 @@ DEFINE_EVENT(ppc64_interrupt_class, timer_interrupt_exit, TP_ARGS(regs) ); +#ifdef CONFIG_PPC_DOORBELL +DEFINE_EVENT(ppc64_interrupt_class, doorbell_entry, + + TP_PROTO(struct pt_regs *regs), + + TP_ARGS(regs) +); + +DEFINE_EVENT(ppc64_interrupt_class, doorbell_exit, + + TP_PROTO(struct pt_regs *regs), + + TP_ARGS(regs) +); +#endif + #ifdef CONFIG_PPC_PSERIES -extern void hcall_tracepoint_regfunc(void); +extern int hcall_tracepoint_regfunc(void); extern void hcall_tracepoint_unregfunc(void); -TRACE_EVENT_FN(hcall_entry, +TRACE_EVENT_FN_COND(hcall_entry, TP_PROTO(unsigned long opcode, unsigned long *args), TP_ARGS(opcode, args), + TP_CONDITION(cpu_online(raw_smp_processor_id())), + TP_STRUCT__entry( __field(unsigned long, opcode) ), @@ -76,13 +95,162 @@ TRACE_EVENT_FN(hcall_entry, hcall_tracepoint_regfunc, hcall_tracepoint_unregfunc ); -TRACE_EVENT_FN(hcall_exit, +TRACE_EVENT_FN_COND(hcall_exit, - TP_PROTO(unsigned long opcode, unsigned long retval, - unsigned long *retbuf), + TP_PROTO(unsigned long opcode, long retval, unsigned long *retbuf), TP_ARGS(opcode, retval, retbuf), + TP_CONDITION(cpu_online(raw_smp_processor_id())), + + TP_STRUCT__entry( + __field(unsigned long, opcode) + __field(long, retval) + ), + + TP_fast_assign( + __entry->opcode = opcode; + __entry->retval = retval; + ), + + TP_printk("opcode=%lu retval=%ld", __entry->opcode, __entry->retval), + + hcall_tracepoint_regfunc, hcall_tracepoint_unregfunc +); +#endif + +#ifdef CONFIG_PPC_RTAS + +#include <asm/rtas-types.h> + +TRACE_EVENT(rtas_input, + + TP_PROTO(struct rtas_args *rtas_args, const char *name), + + TP_ARGS(rtas_args, name), + + TP_STRUCT__entry( + __field(__u32, nargs) + __string(name, name) + __dynamic_array(__u32, inputs, be32_to_cpu(rtas_args->nargs)) + ), + + TP_fast_assign( + __entry->nargs = be32_to_cpu(rtas_args->nargs); + __assign_str(name); + be32_to_cpu_array(__get_dynamic_array(inputs), rtas_args->args, __entry->nargs); + ), + + TP_printk("%s arguments: %s", __get_str(name), + __print_array(__get_dynamic_array(inputs), __entry->nargs, 4) + ) +); + +TRACE_EVENT(rtas_output, + + TP_PROTO(struct rtas_args *rtas_args, const char *name), + + TP_ARGS(rtas_args, name), + + TP_STRUCT__entry( + __field(__u32, nr_other) + __field(__s32, status) + __string(name, name) + __dynamic_array(__u32, other_outputs, be32_to_cpu(rtas_args->nret) - 1) + ), + + TP_fast_assign( + __entry->nr_other = be32_to_cpu(rtas_args->nret) - 1; + __entry->status = be32_to_cpu(rtas_args->rets[0]); + __assign_str(name); + be32_to_cpu_array(__get_dynamic_array(other_outputs), + &rtas_args->rets[1], __entry->nr_other); + ), + + TP_printk("%s status: %d, other outputs: %s", __get_str(name), __entry->status, + __print_array(__get_dynamic_array(other_outputs), + __entry->nr_other, 4) + ) +); + +DECLARE_EVENT_CLASS(rtas_parameter_block, + + TP_PROTO(struct rtas_args *rtas_args), + + TP_ARGS(rtas_args), + + TP_STRUCT__entry( + __field(u32, token) + __field(u32, nargs) + __field(u32, nret) + __array(__u32, params, 16) + ), + + TP_fast_assign( + __entry->token = be32_to_cpu(rtas_args->token); + __entry->nargs = be32_to_cpu(rtas_args->nargs); + __entry->nret = be32_to_cpu(rtas_args->nret); + be32_to_cpu_array(__entry->params, rtas_args->args, ARRAY_SIZE(rtas_args->args)); + ), + + TP_printk("token=%u nargs=%u nret=%u params:" + " [0]=0x%08x [1]=0x%08x [2]=0x%08x [3]=0x%08x" + " [4]=0x%08x [5]=0x%08x [6]=0x%08x [7]=0x%08x" + " [8]=0x%08x [9]=0x%08x [10]=0x%08x [11]=0x%08x" + " [12]=0x%08x [13]=0x%08x [14]=0x%08x [15]=0x%08x", + __entry->token, __entry->nargs, __entry->nret, + __entry->params[0], __entry->params[1], __entry->params[2], __entry->params[3], + __entry->params[4], __entry->params[5], __entry->params[6], __entry->params[7], + __entry->params[8], __entry->params[9], __entry->params[10], __entry->params[11], + __entry->params[12], __entry->params[13], __entry->params[14], __entry->params[15] + ) +); + +DEFINE_EVENT(rtas_parameter_block, rtas_ll_entry, + + TP_PROTO(struct rtas_args *rtas_args), + + TP_ARGS(rtas_args) +); + +DEFINE_EVENT(rtas_parameter_block, rtas_ll_exit, + + TP_PROTO(struct rtas_args *rtas_args), + + TP_ARGS(rtas_args) +); + +#endif /* CONFIG_PPC_RTAS */ + +#ifdef CONFIG_PPC_POWERNV +extern int opal_tracepoint_regfunc(void); +extern void opal_tracepoint_unregfunc(void); + +TRACE_EVENT_FN(opal_entry, + + TP_PROTO(unsigned long opcode, unsigned long *args), + + TP_ARGS(opcode, args), + + TP_STRUCT__entry( + __field(unsigned long, opcode) + ), + + TP_fast_assign( + __entry->opcode = opcode; + ), + + TP_printk("opcode=%lu", __entry->opcode), + + opal_tracepoint_regfunc, opal_tracepoint_unregfunc +); + +TRACE_EVENT_FN(opal_exit, + + TP_PROTO(unsigned long opcode, unsigned long retval), + + TP_ARGS(opcode, retval), + TP_STRUCT__entry( __field(unsigned long, opcode) __field(unsigned long, retval) @@ -95,10 +263,79 @@ TRACE_EVENT_FN(hcall_exit, TP_printk("opcode=%lu retval=%lu", __entry->opcode, __entry->retval), - hcall_tracepoint_regfunc, hcall_tracepoint_unregfunc + opal_tracepoint_regfunc, opal_tracepoint_unregfunc +); +#endif + +#ifdef CONFIG_PPC_64S_HASH_MMU +TRACE_EVENT(hash_fault, + + TP_PROTO(unsigned long addr, unsigned long access, unsigned long trap), + TP_ARGS(addr, access, trap), + TP_STRUCT__entry( + __field(unsigned long, addr) + __field(unsigned long, access) + __field(unsigned long, trap) + ), + + TP_fast_assign( + __entry->addr = addr; + __entry->access = access; + __entry->trap = trap; + ), + + TP_printk("hash fault with addr 0x%lx and access = 0x%lx trap = 0x%lx", + __entry->addr, __entry->access, __entry->trap) ); #endif +TRACE_EVENT(tlbie, + + TP_PROTO(unsigned long lpid, unsigned long local, unsigned long rb, + unsigned long rs, unsigned long ric, unsigned long prs, + unsigned long r), + TP_ARGS(lpid, local, rb, rs, ric, prs, r), + TP_STRUCT__entry( + __field(unsigned long, lpid) + __field(unsigned long, local) + __field(unsigned long, rb) + __field(unsigned long, rs) + __field(unsigned long, ric) + __field(unsigned long, prs) + __field(unsigned long, r) + ), + + TP_fast_assign( + __entry->lpid = lpid; + __entry->local = local; + __entry->rb = rb; + __entry->rs = rs; + __entry->ric = ric; + __entry->prs = prs; + __entry->r = r; + ), + + TP_printk("lpid=%ld, local=%ld, rb=0x%lx, rs=0x%lx, ric=0x%lx, " + "prs=0x%lx, r=0x%lx", __entry->lpid, __entry->local, + __entry->rb, __entry->rs, __entry->ric, __entry->prs, + __entry->r) +); + +TRACE_EVENT(tlbia, + + TP_PROTO(unsigned long id), + TP_ARGS(id), + TP_STRUCT__entry( + __field(unsigned long, id) + ), + + TP_fast_assign( + __entry->id = id; + ), + + TP_printk("ctx.id=0x%lx", __entry->id) +); + #endif /* _TRACE_POWERPC_H */ #undef TRACE_INCLUDE_PATH diff --git a/arch/powerpc/include/asm/trace_clock.h b/arch/powerpc/include/asm/trace_clock.h new file mode 100644 index 000000000000..ef70c2f7974d --- /dev/null +++ b/arch/powerpc/include/asm/trace_clock.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * + * Copyright (C) 2015 Naveen N. Rao, IBM Corporation + */ + +#ifndef _ASM_PPC_TRACE_CLOCK_H +#define _ASM_PPC_TRACE_CLOCK_H + +#include <linux/compiler.h> +#include <linux/types.h> + +extern u64 notrace trace_clock_ppc_tb(void); + +#define ARCH_TRACE_CLOCKS { trace_clock_ppc_tb, "ppc-tb", 0 }, + +#endif /* _ASM_PPC_TRACE_CLOCK_H */ diff --git a/arch/powerpc/include/asm/tsi108.h b/arch/powerpc/include/asm/tsi108.h index f8b60793b7a9..8a2b6427d300 100644 --- a/arch/powerpc/include/asm/tsi108.h +++ b/arch/powerpc/include/asm/tsi108.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * common routine and memory layout for Tundra TSI108(Grendel) host bridge * memory controller. @@ -6,11 +7,6 @@ * Alex Bounine (alexandreb@tundra.com) * * Copyright 2004-2006 Freescale Semiconductor, Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef __PPC_KERNEL_TSI108_H @@ -77,17 +73,13 @@ * nodes if your board uses the Broadcom PHYs */ #define TSI108_PHY_MV88E 0 /* Marvel 88Exxxx PHY */ -#define TSI108_PHY_BCM54XX 1 /* Broardcom BCM54xx PHY */ +#define TSI108_PHY_BCM54XX 1 /* Broadcom BCM54xx PHY */ /* Global variables */ extern u32 tsi108_pci_cfg_base; /* Exported functions */ -extern int tsi108_bridge_init(struct pci_controller *hose, uint phys_csr_base); -extern unsigned long tsi108_get_mem_size(void); -extern unsigned long tsi108_get_cpu_clk(void); -extern unsigned long tsi108_get_sdc_clk(void); extern int tsi108_direct_write_config(struct pci_bus *bus, unsigned int devfn, int offset, int len, u32 val); extern int tsi108_direct_read_config(struct pci_bus *bus, unsigned int devfn, diff --git a/arch/powerpc/include/asm/tsi108_irq.h b/arch/powerpc/include/asm/tsi108_irq.h index 6ed93979fbe4..df602ca4cc52 100644 --- a/arch/powerpc/include/asm/tsi108_irq.h +++ b/arch/powerpc/include/asm/tsi108_irq.h @@ -1,24 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * (C) Copyright 2005 Tundra Semiconductor Corp. * Alex Bounine, <alexandreb at tundra.com). * * See file CREDITS for list of people who contributed to this * project. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation; either version 2 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, - * MA 02111-1307 USA */ /* diff --git a/arch/powerpc/include/asm/tsi108_pci.h b/arch/powerpc/include/asm/tsi108_pci.h index 5653d7cc3e24..fb6f62669154 100644 --- a/arch/powerpc/include/asm/tsi108_pci.h +++ b/arch/powerpc/include/asm/tsi108_pci.h @@ -1,21 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Copyright 2007 IBM Corp - * - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation; either version 2 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, - * MA 02111-1307 USA */ #ifndef _ASM_POWERPC_TSI108_PCI_H @@ -39,7 +24,7 @@ extern int tsi108_setup_pci(struct device_node *dev, u32 cfg_phys, int primary); extern void tsi108_pci_int_init(struct device_node *node); -extern void tsi108_irq_cascade(unsigned int irq, struct irq_desc *desc); +extern void tsi108_irq_cascade(struct irq_desc *desc); extern void tsi108_clear_pci_cfg_error(void); #endif /* _ASM_POWERPC_TSI108_PCI_H */ diff --git a/arch/powerpc/include/asm/types.h b/arch/powerpc/include/asm/types.h index bfb6ded38ffa..55d7ba6d910b 100644 --- a/arch/powerpc/include/asm/types.h +++ b/arch/powerpc/include/asm/types.h @@ -1,30 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * This file is never included by application software unless * explicitly requested (e.g., via linux/types.h) in which case the * application is Linux specific so (user-) name space pollution is * not a major issue. However, for interoperability, libraries still * need to be careful to avoid a name clashes. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef _ASM_POWERPC_TYPES_H #define _ASM_POWERPC_TYPES_H #include <uapi/asm/types.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ typedef __vector128 vector128; -typedef struct { - unsigned long entry; - unsigned long toc; - unsigned long env; -} func_descr_t; - -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _ASM_POWERPC_TYPES_H */ diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index 9485b43a7c00..784a00e681fa 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -1,84 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ARCH_POWERPC_UACCESS_H #define _ARCH_POWERPC_UACCESS_H -#ifdef __KERNEL__ -#ifndef __ASSEMBLY__ - -#include <linux/sched.h> -#include <linux/errno.h> -#include <asm/asm-compat.h> #include <asm/processor.h> #include <asm/page.h> +#include <asm/extable.h> +#include <asm/kup.h> +#include <asm/asm-compat.h> -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - -/* - * The fs value determines whether argument validity checking should be - * performed or not. If get_fs() == USER_DS, checking is performed, with - * get_fs() == KERNEL_DS, checking is bypassed. - * - * For historical reasons, these macros are grossly misnamed. - * - * The fs/ds values are now the highest legal address in the "segment". - * This simplifies the checking in the routines below. - */ - -#define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) - -#define KERNEL_DS MAKE_MM_SEG(~0UL) #ifdef __powerpc64__ /* We use TASK_SIZE_USER64 as TASK_SIZE is not constant */ -#define USER_DS MAKE_MM_SEG(TASK_SIZE_USER64 - 1) -#else -#define USER_DS MAKE_MM_SEG(TASK_SIZE - 1) -#endif - -#define get_ds() (KERNEL_DS) -#define get_fs() (current->thread.fs) -#define set_fs(val) (current->thread.fs = (val)) - -#define segment_eq(a, b) ((a).seg == (b).seg) - -#define user_addr_max() (get_fs().seg) - -#ifdef __powerpc64__ -/* - * This check is sufficient because there is a large enough - * gap between user addresses and the kernel addresses - */ -#define __access_ok(addr, size, segment) \ - (((addr) <= (segment).seg) && ((size) <= (segment).seg)) - -#else - -#define __access_ok(addr, size, segment) \ - (((addr) <= (segment).seg) && \ - (((size) == 0) || (((size) - 1) <= ((segment).seg - (addr))))) - +#define TASK_SIZE_MAX TASK_SIZE_USER64 #endif -#define access_ok(type, addr, size) \ - (__chk_user_ptr(addr), \ - __access_ok((__force unsigned long)(addr), (size), get_fs())) - -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ - -struct exception_table_entry { - unsigned long insn; - unsigned long fixup; -}; +#include <asm-generic/access_ok.h> /* * These are the main single-value transfer routines. They automatically @@ -99,128 +34,212 @@ struct exception_table_entry { * exception handling means that it's no longer "just"...) * */ -#define get_user(x, ptr) \ - __get_user_check((x), (ptr), sizeof(*(ptr))) -#define put_user(x, ptr) \ - __put_user_check((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) - -#define __get_user(x, ptr) \ - __get_user_nocheck((x), (ptr), sizeof(*(ptr))) -#define __put_user(x, ptr) \ - __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) - -#define __get_user_inatomic(x, ptr) \ - __get_user_nosleep((x), (ptr), sizeof(*(ptr))) -#define __put_user_inatomic(x, ptr) \ - __put_user_nosleep((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) - -#define __get_user_unaligned __get_user -#define __put_user_unaligned __put_user +#define __put_user(x, ptr) \ +({ \ + long __pu_err; \ + __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ + __typeof__(*(ptr)) __pu_val = (__typeof__(*(ptr)))(x); \ + __typeof__(sizeof(*(ptr))) __pu_size = sizeof(*(ptr)); \ + \ + might_fault(); \ + do { \ + __label__ __pu_failed; \ + \ + allow_write_to_user(__pu_addr, __pu_size); \ + __put_user_size_goto(__pu_val, __pu_addr, __pu_size, __pu_failed); \ + prevent_write_to_user(__pu_addr, __pu_size); \ + __pu_err = 0; \ + break; \ + \ +__pu_failed: \ + prevent_write_to_user(__pu_addr, __pu_size); \ + __pu_err = -EFAULT; \ + } while (0); \ + \ + __pu_err; \ +}) -extern long __put_user_bad(void); +#define put_user(x, ptr) \ +({ \ + __typeof__(*(ptr)) __user *_pu_addr = (ptr); \ + \ + access_ok(_pu_addr, sizeof(*(ptr))) ? \ + __put_user(x, _pu_addr) : -EFAULT; \ +}) /* * We don't tell gcc that we are accessing memory, but this is OK * because we do not write to any memory gcc knows about, so there * are no aliasing issues. */ -#define __put_user_asm(x, addr, err, op) \ - __asm__ __volatile__( \ - "1: " op " %1,0(%2) # put_user\n" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: li %0,%3\n" \ - " b 2b\n" \ - ".previous\n" \ - ".section __ex_table,\"a\"\n" \ - PPC_LONG_ALIGN "\n" \ - PPC_LONG "1b,3b\n" \ - ".previous" \ - : "=r" (err) \ - : "r" (x), "b" (addr), "i" (-EFAULT), "0" (err)) +/* -mprefixed can generate offsets beyond range, fall back hack */ +#ifdef CONFIG_PPC_KERNEL_PREFIXED +#define __put_user_asm_goto(x, addr, label, op) \ + asm goto( \ + "1: " op " %0,0(%1) # put_user\n" \ + EX_TABLE(1b, %l2) \ + : \ + : "r" (x), "b" (addr) \ + : \ + : label) +#else +#define __put_user_asm_goto(x, addr, label, op) \ + asm goto( \ + "1: " op "%U1%X1 %0,%1 # put_user\n" \ + EX_TABLE(1b, %l2) \ + : \ + : "r" (x), "m<>" (*addr) \ + : \ + : label) +#endif #ifdef __powerpc64__ -#define __put_user_asm2(x, ptr, retval) \ - __put_user_asm(x, ptr, retval, "std") +#ifdef CONFIG_PPC_KERNEL_PREFIXED +#define __put_user_asm2_goto(x, ptr, label) \ + __put_user_asm_goto(x, ptr, label, "std") +#else +#define __put_user_asm2_goto(x, addr, label) \ + asm goto ("1: std%U1%X1 %0,%1 # put_user\n" \ + EX_TABLE(1b, %l2) \ + : \ + : "r" (x), DS_FORM_CONSTRAINT (*addr) \ + : \ + : label) +#endif // CONFIG_PPC_KERNEL_PREFIXED #else /* __powerpc64__ */ -#define __put_user_asm2(x, addr, err) \ - __asm__ __volatile__( \ - "1: stw %1,0(%2)\n" \ - "2: stw %1+1,4(%2)\n" \ - "3:\n" \ - ".section .fixup,\"ax\"\n" \ - "4: li %0,%3\n" \ - " b 3b\n" \ - ".previous\n" \ - ".section __ex_table,\"a\"\n" \ - PPC_LONG_ALIGN "\n" \ - PPC_LONG "1b,4b\n" \ - PPC_LONG "2b,4b\n" \ - ".previous" \ - : "=r" (err) \ - : "r" (x), "b" (addr), "i" (-EFAULT), "0" (err)) +#define __put_user_asm2_goto(x, addr, label) \ + asm goto( \ + "1: stw%X1 %0, %1\n" \ + "2: stw%X1 %L0, %L1\n" \ + EX_TABLE(1b, %l2) \ + EX_TABLE(2b, %l2) \ + : \ + : "r" (x), "m" (*addr) \ + : \ + : label) #endif /* __powerpc64__ */ -#define __put_user_size(x, ptr, size, retval) \ +#define __put_user_size_goto(x, ptr, size, label) \ do { \ - retval = 0; \ + __typeof__(*(ptr)) __user *__pus_addr = (ptr); \ + \ switch (size) { \ - case 1: __put_user_asm(x, ptr, retval, "stb"); break; \ - case 2: __put_user_asm(x, ptr, retval, "sth"); break; \ - case 4: __put_user_asm(x, ptr, retval, "stw"); break; \ - case 8: __put_user_asm2(x, ptr, retval); break; \ - default: __put_user_bad(); \ + case 1: __put_user_asm_goto(x, __pus_addr, label, "stb"); break; \ + case 2: __put_user_asm_goto(x, __pus_addr, label, "sth"); break; \ + case 4: __put_user_asm_goto(x, __pus_addr, label, "stw"); break; \ + case 8: __put_user_asm2_goto(x, __pus_addr, label); break; \ + default: BUILD_BUG(); \ } \ } while (0) -#define __put_user_nocheck(x, ptr, size) \ -({ \ - long __pu_err; \ - __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ - if (!is_kernel_addr((unsigned long)__pu_addr)) \ - might_fault(); \ - __chk_user_ptr(ptr); \ - __put_user_size((x), __pu_addr, (size), __pu_err); \ - __pu_err; \ -}) +/* + * This does an atomic 128 byte aligned load from userspace. + * Upto caller to do enable_kernel_vmx() before calling! + */ +#define __get_user_atomic_128_aligned(kaddr, uaddr, err) \ + __asm__ __volatile__( \ + ".machine push\n" \ + ".machine altivec\n" \ + "1: lvx 0,0,%1 # get user\n" \ + " stvx 0,0,%2 # put kernel\n" \ + ".machine pop\n" \ + "2:\n" \ + ".section .fixup,\"ax\"\n" \ + "3: li %0,%3\n" \ + " b 2b\n" \ + ".previous\n" \ + EX_TABLE(1b, 3b) \ + : "=r" (err) \ + : "b" (uaddr), "b" (kaddr), "i" (-EFAULT), "0" (err)) + +#ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT + +/* -mprefixed can generate offsets beyond range, fall back hack */ +#ifdef CONFIG_PPC_KERNEL_PREFIXED +#define __get_user_asm_goto(x, addr, label, op) \ + asm_goto_output( \ + "1: "op" %0,0(%1) # get_user\n" \ + EX_TABLE(1b, %l2) \ + : "=r" (x) \ + : "b" (addr) \ + : \ + : label) +#else +#define __get_user_asm_goto(x, addr, label, op) \ + asm_goto_output( \ + "1: "op"%U1%X1 %0, %1 # get_user\n" \ + EX_TABLE(1b, %l2) \ + : "=r" (x) \ + : "m<>" (*addr) \ + : \ + : label) +#endif -#define __put_user_check(x, ptr, size) \ -({ \ - long __pu_err = -EFAULT; \ - __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ - might_fault(); \ - if (access_ok(VERIFY_WRITE, __pu_addr, size)) \ - __put_user_size((x), __pu_addr, (size), __pu_err); \ - __pu_err; \ -}) +#ifdef __powerpc64__ +#ifdef CONFIG_PPC_KERNEL_PREFIXED +#define __get_user_asm2_goto(x, addr, label) \ + __get_user_asm_goto(x, addr, label, "ld") +#else +#define __get_user_asm2_goto(x, addr, label) \ + asm_goto_output( \ + "1: ld%U1%X1 %0, %1 # get_user\n" \ + EX_TABLE(1b, %l2) \ + : "=r" (x) \ + : DS_FORM_CONSTRAINT (*addr) \ + : \ + : label) +#endif // CONFIG_PPC_KERNEL_PREFIXED +#else /* __powerpc64__ */ +#define __get_user_asm2_goto(x, addr, label) \ + asm_goto_output( \ + "1: lwz%X1 %0, %1\n" \ + "2: lwz%X1 %L0, %L1\n" \ + EX_TABLE(1b, %l2) \ + EX_TABLE(2b, %l2) \ + : "=&r" (x) \ + : "m" (*addr) \ + : \ + : label) +#endif /* __powerpc64__ */ -#define __put_user_nosleep(x, ptr, size) \ -({ \ - long __pu_err; \ - __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ - __chk_user_ptr(ptr); \ - __put_user_size((x), __pu_addr, (size), __pu_err); \ - __pu_err; \ -}) +#define __get_user_size_goto(x, ptr, size, label) \ +do { \ + BUILD_BUG_ON(size > sizeof(x)); \ + switch (size) { \ + case 1: __get_user_asm_goto(x, (u8 __user *)ptr, label, "lbz"); break; \ + case 2: __get_user_asm_goto(x, (u16 __user *)ptr, label, "lhz"); break; \ + case 4: __get_user_asm_goto(x, (u32 __user *)ptr, label, "lwz"); break; \ + case 8: __get_user_asm2_goto(x, (u64 __user *)ptr, label); break; \ + default: x = 0; BUILD_BUG(); \ + } \ +} while (0) +#define __get_user_size_allowed(x, ptr, size, retval) \ +do { \ + __label__ __gus_failed; \ + \ + __get_user_size_goto(x, ptr, size, __gus_failed); \ + retval = 0; \ + break; \ +__gus_failed: \ + x = 0; \ + retval = -EFAULT; \ +} while (0) -extern long __get_user_bad(void); +#else /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ #define __get_user_asm(x, addr, err, op) \ __asm__ __volatile__( \ - "1: "op" %1,0(%2) # get_user\n" \ + "1: "op"%U2%X2 %1, %2 # get_user\n" \ "2:\n" \ ".section .fixup,\"ax\"\n" \ "3: li %0,%3\n" \ " li %1,0\n" \ " b 2b\n" \ ".previous\n" \ - ".section __ex_table,\"a\"\n" \ - PPC_LONG_ALIGN "\n" \ - PPC_LONG "1b,3b\n" \ - ".previous" \ + EX_TABLE(1b, 3b) \ : "=r" (err), "=r" (x) \ - : "b" (addr), "i" (-EFAULT), "0" (err)) + : "m<>" (*addr), "i" (-EFAULT), "0" (err)) #ifdef __powerpc64__ #define __get_user_asm2(x, addr, err) \ @@ -228,8 +247,8 @@ extern long __get_user_bad(void); #else /* __powerpc64__ */ #define __get_user_asm2(x, addr, err) \ __asm__ __volatile__( \ - "1: lwz %1,0(%2)\n" \ - "2: lwz %1+1,4(%2)\n" \ + "1: lwz%X2 %1, %2\n" \ + "2: lwz%X2 %L1, %L2\n" \ "3:\n" \ ".section .fixup,\"ax\"\n" \ "4: li %0,%3\n" \ @@ -237,218 +256,260 @@ extern long __get_user_bad(void); " li %1+1,0\n" \ " b 3b\n" \ ".previous\n" \ - ".section __ex_table,\"a\"\n" \ - PPC_LONG_ALIGN "\n" \ - PPC_LONG "1b,4b\n" \ - PPC_LONG "2b,4b\n" \ - ".previous" \ + EX_TABLE(1b, 4b) \ + EX_TABLE(2b, 4b) \ : "=r" (err), "=&r" (x) \ - : "b" (addr), "i" (-EFAULT), "0" (err)) + : "m" (*addr), "i" (-EFAULT), "0" (err)) #endif /* __powerpc64__ */ -#define __get_user_size(x, ptr, size, retval) \ +#define __get_user_size_allowed(x, ptr, size, retval) \ do { \ retval = 0; \ - __chk_user_ptr(ptr); \ - if (size > sizeof(x)) \ - (x) = __get_user_bad(); \ + BUILD_BUG_ON(size > sizeof(x)); \ switch (size) { \ - case 1: __get_user_asm(x, ptr, retval, "lbz"); break; \ - case 2: __get_user_asm(x, ptr, retval, "lhz"); break; \ - case 4: __get_user_asm(x, ptr, retval, "lwz"); break; \ - case 8: __get_user_asm2(x, ptr, retval); break; \ - default: (x) = __get_user_bad(); \ + case 1: __get_user_asm(x, (u8 __user *)ptr, retval, "lbz"); break; \ + case 2: __get_user_asm(x, (u16 __user *)ptr, retval, "lhz"); break; \ + case 4: __get_user_asm(x, (u32 __user *)ptr, retval, "lwz"); break; \ + case 8: __get_user_asm2(x, (u64 __user *)ptr, retval); break; \ + default: x = 0; BUILD_BUG(); \ } \ } while (0) -#define __get_user_nocheck(x, ptr, size) \ -({ \ - long __gu_err; \ - unsigned long __gu_val; \ - const __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ - __chk_user_ptr(ptr); \ - if (!is_kernel_addr((unsigned long)__gu_addr)) \ - might_fault(); \ - __get_user_size(__gu_val, __gu_addr, (size), __gu_err); \ - (x) = (__typeof__(*(ptr)))__gu_val; \ - __gu_err; \ -}) +#define __get_user_size_goto(x, ptr, size, label) \ +do { \ + long __gus_retval; \ + \ + __get_user_size_allowed(x, ptr, size, __gus_retval); \ + if (__gus_retval) \ + goto label; \ +} while (0) -#ifndef __powerpc64__ -#define __get_user64_nocheck(x, ptr, size) \ -({ \ - long __gu_err; \ - long long __gu_val; \ - const __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ - __chk_user_ptr(ptr); \ - if (!is_kernel_addr((unsigned long)__gu_addr)) \ - might_fault(); \ - __get_user_size(__gu_val, __gu_addr, (size), __gu_err); \ - (x) = (__typeof__(*(ptr)))__gu_val; \ - __gu_err; \ -}) -#endif /* __powerpc64__ */ +#endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ -#define __get_user_check(x, ptr, size) \ -({ \ - long __gu_err = -EFAULT; \ - unsigned long __gu_val = 0; \ - const __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ - might_fault(); \ - if (access_ok(VERIFY_READ, __gu_addr, (size))) \ - __get_user_size(__gu_val, __gu_addr, (size), __gu_err); \ - (x) = (__typeof__(*(ptr)))__gu_val; \ - __gu_err; \ -}) +/* + * This is a type: either unsigned long, if the argument fits into + * that type, or otherwise unsigned long long. + */ +#define __long_type(x) \ + __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL)) -#define __get_user_nosleep(x, ptr, size) \ +#define __get_user(x, ptr) \ ({ \ long __gu_err; \ - unsigned long __gu_val; \ - const __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ - __chk_user_ptr(ptr); \ - __get_user_size(__gu_val, __gu_addr, (size), __gu_err); \ + __long_type(*(ptr)) __gu_val; \ + __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ + __typeof__(sizeof(*(ptr))) __gu_size = sizeof(*(ptr)); \ + \ + might_fault(); \ + allow_read_from_user(__gu_addr, __gu_size); \ + __get_user_size_allowed(__gu_val, __gu_addr, __gu_size, __gu_err); \ + prevent_read_from_user(__gu_addr, __gu_size); \ (x) = (__typeof__(*(ptr)))__gu_val; \ + \ __gu_err; \ }) +#define get_user(x, ptr) \ +({ \ + __typeof__(*(ptr)) __user *_gu_addr = (ptr); \ + \ + access_ok(_gu_addr, sizeof(*(ptr))) ? \ + __get_user(x, _gu_addr) : \ + ((x) = (__force __typeof__(*(ptr)))0, -EFAULT); \ +}) /* more complex routines */ extern unsigned long __copy_tofrom_user(void __user *to, const void __user *from, unsigned long size); -#ifndef __powerpc64__ +#ifdef __powerpc64__ +static inline unsigned long +raw_copy_in_user(void __user *to, const void __user *from, unsigned long n) +{ + unsigned long ret; + + allow_read_write_user(to, from, n); + ret = __copy_tofrom_user(to, from, n); + prevent_read_write_user(to, from, n); + return ret; +} +#endif /* __powerpc64__ */ -static inline unsigned long copy_from_user(void *to, +static inline unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n) { - unsigned long over; - - if (access_ok(VERIFY_READ, from, n)) - return __copy_tofrom_user((__force void __user *)to, from, n); - if ((unsigned long)from < TASK_SIZE) { - over = (unsigned long)from + n - TASK_SIZE; - return __copy_tofrom_user((__force void __user *)to, from, - n - over) + over; - } - return n; + unsigned long ret; + + allow_read_from_user(from, n); + ret = __copy_tofrom_user((__force void __user *)to, from, n); + prevent_read_from_user(from, n); + return ret; } -static inline unsigned long copy_to_user(void __user *to, - const void *from, unsigned long n) +static inline unsigned long +raw_copy_to_user(void __user *to, const void *from, unsigned long n) { - unsigned long over; - - if (access_ok(VERIFY_WRITE, to, n)) - return __copy_tofrom_user(to, (__force void __user *)from, n); - if ((unsigned long)to < TASK_SIZE) { - over = (unsigned long)to + n - TASK_SIZE; - return __copy_tofrom_user(to, (__force void __user *)from, - n - over) + over; - } - return n; + unsigned long ret; + + allow_write_to_user(to, n); + ret = __copy_tofrom_user(to, (__force const void __user *)from, n); + prevent_write_to_user(to, n); + return ret; } -#else /* __powerpc64__ */ +unsigned long __arch_clear_user(void __user *addr, unsigned long size); -#define __copy_in_user(to, from, size) \ - __copy_tofrom_user((to), (from), (size)) +static inline unsigned long __clear_user(void __user *addr, unsigned long size) +{ + unsigned long ret; -extern unsigned long copy_from_user(void *to, const void __user *from, - unsigned long n); -extern unsigned long copy_to_user(void __user *to, const void *from, - unsigned long n); -extern unsigned long copy_in_user(void __user *to, const void __user *from, - unsigned long n); + might_fault(); + allow_write_to_user(addr, size); + ret = __arch_clear_user(addr, size); + prevent_write_to_user(addr, size); + return ret; +} -#endif /* __powerpc64__ */ +static inline unsigned long clear_user(void __user *addr, unsigned long size) +{ + return likely(access_ok(addr, size)) ? __clear_user(addr, size) : size; +} -static inline unsigned long __copy_from_user_inatomic(void *to, - const void __user *from, unsigned long n) +extern long strncpy_from_user(char *dst, const char __user *src, long count); +extern __must_check long strnlen_user(const char __user *str, long n); + +#ifdef CONFIG_ARCH_HAS_COPY_MC +unsigned long __must_check +copy_mc_generic(void *to, const void *from, unsigned long size); + +static inline unsigned long __must_check +copy_mc_to_kernel(void *to, const void *from, unsigned long size) { - if (__builtin_constant_p(n) && (n <= 8)) { - unsigned long ret = 1; - - switch (n) { - case 1: - __get_user_size(*(u8 *)to, from, 1, ret); - break; - case 2: - __get_user_size(*(u16 *)to, from, 2, ret); - break; - case 4: - __get_user_size(*(u32 *)to, from, 4, ret); - break; - case 8: - __get_user_size(*(u64 *)to, from, 8, ret); - break; - } - if (ret == 0) - return 0; - } - return __copy_tofrom_user((__force void __user *)to, from, n); + return copy_mc_generic(to, from, size); } +#define copy_mc_to_kernel copy_mc_to_kernel -static inline unsigned long __copy_to_user_inatomic(void __user *to, - const void *from, unsigned long n) +static inline unsigned long __must_check +copy_mc_to_user(void __user *to, const void *from, unsigned long n) { - if (__builtin_constant_p(n) && (n <= 8)) { - unsigned long ret = 1; - - switch (n) { - case 1: - __put_user_size(*(u8 *)from, (u8 __user *)to, 1, ret); - break; - case 2: - __put_user_size(*(u16 *)from, (u16 __user *)to, 2, ret); - break; - case 4: - __put_user_size(*(u32 *)from, (u32 __user *)to, 4, ret); - break; - case 8: - __put_user_size(*(u64 *)from, (u64 __user *)to, 8, ret); - break; + if (check_copy_size(from, n, true)) { + if (access_ok(to, n)) { + allow_write_to_user(to, n); + n = copy_mc_generic((void __force *)to, from, n); + prevent_write_to_user(to, n); } - if (ret == 0) - return 0; } - return __copy_tofrom_user(to, (__force const void __user *)from, n); + + return n; } +#endif + +extern long __copy_from_user_flushcache(void *dst, const void __user *src, + unsigned size); -static inline unsigned long __copy_from_user(void *to, - const void __user *from, unsigned long size) +static __must_check __always_inline bool user_access_begin(const void __user *ptr, size_t len) { + if (unlikely(!access_ok(ptr, len))) + return false; + might_fault(); - return __copy_from_user_inatomic(to, from, size); + + allow_read_write_user((void __user *)ptr, ptr, len); + return true; } +#define user_access_begin user_access_begin +#define user_access_end prevent_current_access_user +#define user_access_save prevent_user_access_return +#define user_access_restore restore_user_access -static inline unsigned long __copy_to_user(void __user *to, - const void *from, unsigned long size) +static __must_check __always_inline bool +user_read_access_begin(const void __user *ptr, size_t len) { + if (unlikely(!access_ok(ptr, len))) + return false; + might_fault(); - return __copy_to_user_inatomic(to, from, size); -} -extern unsigned long __clear_user(void __user *addr, unsigned long size); + allow_read_from_user(ptr, len); + return true; +} +#define user_read_access_begin user_read_access_begin +#define user_read_access_end prevent_current_read_from_user -static inline unsigned long clear_user(void __user *addr, unsigned long size) +static __must_check __always_inline bool +user_write_access_begin(const void __user *ptr, size_t len) { + if (unlikely(!access_ok(ptr, len))) + return false; + might_fault(); - if (likely(access_ok(VERIFY_WRITE, addr, size))) - return __clear_user(addr, size); - if ((unsigned long)addr < TASK_SIZE) { - unsigned long over = (unsigned long)addr + size - TASK_SIZE; - return __clear_user(addr, size - over) + over; - } - return size; + + allow_write_to_user((void __user *)ptr, len); + return true; } +#define user_write_access_begin user_write_access_begin +#define user_write_access_end prevent_current_write_to_user + +#define arch_unsafe_get_user(x, p, e) do { \ + __long_type(*(p)) __gu_val; \ + __typeof__(*(p)) __user *__gu_addr = (p); \ + \ + __get_user_size_goto(__gu_val, __gu_addr, sizeof(*(p)), e); \ + (x) = (__typeof__(*(p)))__gu_val; \ +} while (0) -extern long strncpy_from_user(char *dst, const char __user *src, long count); -extern __must_check long strlen_user(const char __user *str); -extern __must_check long strnlen_user(const char __user *str, long n); +#define arch_unsafe_put_user(x, p, e) \ + __put_user_size_goto((__typeof__(*(p)))(x), (p), sizeof(*(p)), e) + +#define unsafe_copy_from_user(d, s, l, e) \ +do { \ + u8 *_dst = (u8 *)(d); \ + const u8 __user *_src = (const u8 __user *)(s); \ + size_t _len = (l); \ + int _i; \ + \ + for (_i = 0; _i < (_len & ~(sizeof(u64) - 1)); _i += sizeof(u64)) \ + unsafe_get_user(*(u64 *)(_dst + _i), (u64 __user *)(_src + _i), e); \ + if (_len & 4) { \ + unsafe_get_user(*(u32 *)(_dst + _i), (u32 __user *)(_src + _i), e); \ + _i += 4; \ + } \ + if (_len & 2) { \ + unsafe_get_user(*(u16 *)(_dst + _i), (u16 __user *)(_src + _i), e); \ + _i += 2; \ + } \ + if (_len & 1) \ + unsafe_get_user(*(u8 *)(_dst + _i), (u8 __user *)(_src + _i), e); \ +} while (0) + +#define unsafe_copy_to_user(d, s, l, e) \ +do { \ + u8 __user *_dst = (u8 __user *)(d); \ + const u8 *_src = (const u8 *)(s); \ + size_t _len = (l); \ + int _i; \ + \ + for (_i = 0; _i < (_len & ~(sizeof(u64) - 1)); _i += sizeof(u64)) \ + unsafe_put_user(*(u64 *)(_src + _i), (u64 __user *)(_dst + _i), e); \ + if (_len & 4) { \ + unsafe_put_user(*(u32*)(_src + _i), (u32 __user *)(_dst + _i), e); \ + _i += 4; \ + } \ + if (_len & 2) { \ + unsafe_put_user(*(u16*)(_src + _i), (u16 __user *)(_dst + _i), e); \ + _i += 2; \ + } \ + if (_len & 1) \ + unsafe_put_user(*(u8*)(_src + _i), (u8 __user *)(_dst + _i), e); \ +} while (0) + +#define arch_get_kernel_nofault(dst, src, type, err_label) \ + __get_user_size_goto(*((type *)(dst)), \ + (__force type __user *)(src), sizeof(type), err_label) -#endif /* __ASSEMBLY__ */ -#endif /* __KERNEL__ */ +#define arch_put_kernel_nofault(dst, src, type, err_label) \ + __put_user_size_goto(*((type *)(src)), \ + (__force type __user *)(dst), sizeof(type), err_label) #endif /* _ARCH_POWERPC_UACCESS_H */ diff --git a/arch/powerpc/include/asm/ucc.h b/arch/powerpc/include/asm/ucc.h deleted file mode 100644 index 6927ac26516e..000000000000 --- a/arch/powerpc/include/asm/ucc.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (C) 2006 Freescale Semiconductor, Inc. All rights reserved. - * - * Authors: Shlomi Gridish <gridish@freescale.com> - * Li Yang <leoli@freescale.com> - * - * Description: - * Internal header file for UCC unit routines. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - */ -#ifndef __UCC_H__ -#define __UCC_H__ - -#include <asm/immap_qe.h> -#include <asm/qe.h> - -#define STATISTICS - -#define UCC_MAX_NUM 8 - -/* Slow or fast type for UCCs. -*/ -enum ucc_speed_type { - UCC_SPEED_TYPE_FAST = UCC_GUEMR_MODE_FAST_RX | UCC_GUEMR_MODE_FAST_TX, - UCC_SPEED_TYPE_SLOW = UCC_GUEMR_MODE_SLOW_RX | UCC_GUEMR_MODE_SLOW_TX -}; - -/* ucc_set_type - * Sets UCC to slow or fast mode. - * - * ucc_num - (In) number of UCC (0-7). - * speed - (In) slow or fast mode for UCC. - */ -int ucc_set_type(unsigned int ucc_num, enum ucc_speed_type speed); - -int ucc_set_qe_mux_mii_mng(unsigned int ucc_num); - -int ucc_set_qe_mux_rxtx(unsigned int ucc_num, enum qe_clock clock, - enum comm_dir mode); - -int ucc_mux_set_grant_tsa_bkpt(unsigned int ucc_num, int set, u32 mask); - -/* QE MUX clock routing for UCC -*/ -static inline int ucc_set_qe_mux_grant(unsigned int ucc_num, int set) -{ - return ucc_mux_set_grant_tsa_bkpt(ucc_num, set, QE_CMXUCR_GRANT); -} - -static inline int ucc_set_qe_mux_tsa(unsigned int ucc_num, int set) -{ - return ucc_mux_set_grant_tsa_bkpt(ucc_num, set, QE_CMXUCR_TSA); -} - -static inline int ucc_set_qe_mux_bkpt(unsigned int ucc_num, int set) -{ - return ucc_mux_set_grant_tsa_bkpt(ucc_num, set, QE_CMXUCR_BKPT); -} - -#endif /* __UCC_H__ */ diff --git a/arch/powerpc/include/asm/ucc_fast.h b/arch/powerpc/include/asm/ucc_fast.h deleted file mode 100644 index 72ea9bab07df..000000000000 --- a/arch/powerpc/include/asm/ucc_fast.h +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Internal header file for UCC FAST unit routines. - * - * Copyright (C) 2006 Freescale Semiconductor, Inc. All rights reserved. - * - * Authors: Shlomi Gridish <gridish@freescale.com> - * Li Yang <leoli@freescale.com> - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - */ -#ifndef __UCC_FAST_H__ -#define __UCC_FAST_H__ - -#include <linux/kernel.h> - -#include <asm/immap_qe.h> -#include <asm/qe.h> - -#include <asm/ucc.h> - -/* Receive BD's status */ -#define R_E 0x80000000 /* buffer empty */ -#define R_W 0x20000000 /* wrap bit */ -#define R_I 0x10000000 /* interrupt on reception */ -#define R_L 0x08000000 /* last */ -#define R_F 0x04000000 /* first */ - -/* transmit BD's status */ -#define T_R 0x80000000 /* ready bit */ -#define T_W 0x20000000 /* wrap bit */ -#define T_I 0x10000000 /* interrupt on completion */ -#define T_L 0x08000000 /* last */ - -/* Rx Data buffer must be 4 bytes aligned in most cases */ -#define UCC_FAST_RX_ALIGN 4 -#define UCC_FAST_MRBLR_ALIGNMENT 4 -#define UCC_FAST_VIRT_FIFO_REGS_ALIGNMENT 8 - -/* Sizes */ -#define UCC_FAST_URFS_MIN_VAL 0x88 -#define UCC_FAST_RECEIVE_VIRTUAL_FIFO_SIZE_FUDGE_FACTOR 8 - -/* ucc_fast_channel_protocol_mode - UCC FAST mode */ -enum ucc_fast_channel_protocol_mode { - UCC_FAST_PROTOCOL_MODE_HDLC = 0x00000000, - UCC_FAST_PROTOCOL_MODE_RESERVED01 = 0x00000001, - UCC_FAST_PROTOCOL_MODE_RESERVED_QMC = 0x00000002, - UCC_FAST_PROTOCOL_MODE_RESERVED02 = 0x00000003, - UCC_FAST_PROTOCOL_MODE_RESERVED_UART = 0x00000004, - UCC_FAST_PROTOCOL_MODE_RESERVED03 = 0x00000005, - UCC_FAST_PROTOCOL_MODE_RESERVED_EX_MAC_1 = 0x00000006, - UCC_FAST_PROTOCOL_MODE_RESERVED_EX_MAC_2 = 0x00000007, - UCC_FAST_PROTOCOL_MODE_RESERVED_BISYNC = 0x00000008, - UCC_FAST_PROTOCOL_MODE_RESERVED04 = 0x00000009, - UCC_FAST_PROTOCOL_MODE_ATM = 0x0000000A, - UCC_FAST_PROTOCOL_MODE_RESERVED05 = 0x0000000B, - UCC_FAST_PROTOCOL_MODE_ETHERNET = 0x0000000C, - UCC_FAST_PROTOCOL_MODE_RESERVED06 = 0x0000000D, - UCC_FAST_PROTOCOL_MODE_POS = 0x0000000E, - UCC_FAST_PROTOCOL_MODE_RESERVED07 = 0x0000000F -}; - -/* ucc_fast_transparent_txrx - UCC Fast Transparent TX & RX */ -enum ucc_fast_transparent_txrx { - UCC_FAST_GUMR_TRANSPARENT_TTX_TRX_NORMAL = 0x00000000, - UCC_FAST_GUMR_TRANSPARENT_TTX_TRX_TRANSPARENT = 0x18000000 -}; - -/* UCC fast diagnostic mode */ -enum ucc_fast_diag_mode { - UCC_FAST_DIAGNOSTIC_NORMAL = 0x0, - UCC_FAST_DIAGNOSTIC_LOCAL_LOOP_BACK = 0x40000000, - UCC_FAST_DIAGNOSTIC_AUTO_ECHO = 0x80000000, - UCC_FAST_DIAGNOSTIC_LOOP_BACK_AND_ECHO = 0xC0000000 -}; - -/* UCC fast Sync length (transparent mode only) */ -enum ucc_fast_sync_len { - UCC_FAST_SYNC_LEN_NOT_USED = 0x0, - UCC_FAST_SYNC_LEN_AUTOMATIC = 0x00004000, - UCC_FAST_SYNC_LEN_8_BIT = 0x00008000, - UCC_FAST_SYNC_LEN_16_BIT = 0x0000C000 -}; - -/* UCC fast RTS mode */ -enum ucc_fast_ready_to_send { - UCC_FAST_SEND_IDLES_BETWEEN_FRAMES = 0x00000000, - UCC_FAST_SEND_FLAGS_BETWEEN_FRAMES = 0x00002000 -}; - -/* UCC fast receiver decoding mode */ -enum ucc_fast_rx_decoding_method { - UCC_FAST_RX_ENCODING_NRZ = 0x00000000, - UCC_FAST_RX_ENCODING_NRZI = 0x00000800, - UCC_FAST_RX_ENCODING_RESERVED0 = 0x00001000, - UCC_FAST_RX_ENCODING_RESERVED1 = 0x00001800 -}; - -/* UCC fast transmitter encoding mode */ -enum ucc_fast_tx_encoding_method { - UCC_FAST_TX_ENCODING_NRZ = 0x00000000, - UCC_FAST_TX_ENCODING_NRZI = 0x00000100, - UCC_FAST_TX_ENCODING_RESERVED0 = 0x00000200, - UCC_FAST_TX_ENCODING_RESERVED1 = 0x00000300 -}; - -/* UCC fast CRC length */ -enum ucc_fast_transparent_tcrc { - UCC_FAST_16_BIT_CRC = 0x00000000, - UCC_FAST_CRC_RESERVED0 = 0x00000040, - UCC_FAST_32_BIT_CRC = 0x00000080, - UCC_FAST_CRC_RESERVED1 = 0x000000C0 -}; - -/* Fast UCC initialization structure */ -struct ucc_fast_info { - int ucc_num; - enum qe_clock rx_clock; - enum qe_clock tx_clock; - u32 regs; - int irq; - u32 uccm_mask; - int bd_mem_part; - int brkpt_support; - int grant_support; - int tsa; - int cdp; - int cds; - int ctsp; - int ctss; - int tci; - int txsy; - int rtsm; - int revd; - int rsyn; - u16 max_rx_buf_length; - u16 urfs; - u16 urfet; - u16 urfset; - u16 utfs; - u16 utfet; - u16 utftt; - u16 ufpt; - enum ucc_fast_channel_protocol_mode mode; - enum ucc_fast_transparent_txrx ttx_trx; - enum ucc_fast_tx_encoding_method tenc; - enum ucc_fast_rx_decoding_method renc; - enum ucc_fast_transparent_tcrc tcrc; - enum ucc_fast_sync_len synl; -}; - -struct ucc_fast_private { - struct ucc_fast_info *uf_info; - struct ucc_fast __iomem *uf_regs; /* a pointer to the UCC regs. */ - u32 __iomem *p_ucce; /* a pointer to the event register in memory. */ - u32 __iomem *p_uccm; /* a pointer to the mask register in memory. */ -#ifdef CONFIG_UGETH_TX_ON_DEMAND - u16 __iomem *p_utodr; /* pointer to the transmit on demand register */ -#endif - int enabled_tx; /* Whether channel is enabled for Tx (ENT) */ - int enabled_rx; /* Whether channel is enabled for Rx (ENR) */ - int stopped_tx; /* Whether channel has been stopped for Tx - (STOP_TX, etc.) */ - int stopped_rx; /* Whether channel has been stopped for Rx */ - u32 ucc_fast_tx_virtual_fifo_base_offset;/* pointer to base of Tx - virtual fifo */ - u32 ucc_fast_rx_virtual_fifo_base_offset;/* pointer to base of Rx - virtual fifo */ -#ifdef STATISTICS - u32 tx_frames; /* Transmitted frames counter. */ - u32 rx_frames; /* Received frames counter (only frames - passed to application). */ - u32 tx_discarded; /* Discarded tx frames counter (frames that - were discarded by the driver due to errors). - */ - u32 rx_discarded; /* Discarded rx frames counter (frames that - were discarded by the driver due to errors). - */ -#endif /* STATISTICS */ - u16 mrblr; /* maximum receive buffer length */ -}; - -/* ucc_fast_init - * Initializes Fast UCC according to user provided parameters. - * - * uf_info - (In) pointer to the fast UCC info structure. - * uccf_ret - (Out) pointer to the fast UCC structure. - */ -int ucc_fast_init(struct ucc_fast_info * uf_info, struct ucc_fast_private ** uccf_ret); - -/* ucc_fast_free - * Frees all resources for fast UCC. - * - * uccf - (In) pointer to the fast UCC structure. - */ -void ucc_fast_free(struct ucc_fast_private * uccf); - -/* ucc_fast_enable - * Enables a fast UCC port. - * This routine enables Tx and/or Rx through the General UCC Mode Register. - * - * uccf - (In) pointer to the fast UCC structure. - * mode - (In) TX, RX, or both. - */ -void ucc_fast_enable(struct ucc_fast_private * uccf, enum comm_dir mode); - -/* ucc_fast_disable - * Disables a fast UCC port. - * This routine disables Tx and/or Rx through the General UCC Mode Register. - * - * uccf - (In) pointer to the fast UCC structure. - * mode - (In) TX, RX, or both. - */ -void ucc_fast_disable(struct ucc_fast_private * uccf, enum comm_dir mode); - -/* ucc_fast_irq - * Handles interrupts on fast UCC. - * Called from the general interrupt routine to handle interrupts on fast UCC. - * - * uccf - (In) pointer to the fast UCC structure. - */ -void ucc_fast_irq(struct ucc_fast_private * uccf); - -/* ucc_fast_transmit_on_demand - * Immediately forces a poll of the transmitter for data to be sent. - * Typically, the hardware performs a periodic poll for data that the - * transmit routine has set up to be transmitted. In cases where - * this polling cycle is not soon enough, this optional routine can - * be invoked to force a poll right away, instead. Proper use for - * each transmission for which this functionality is desired is to - * call the transmit routine and then this routine right after. - * - * uccf - (In) pointer to the fast UCC structure. - */ -void ucc_fast_transmit_on_demand(struct ucc_fast_private * uccf); - -u32 ucc_fast_get_qe_cr_subblock(int uccf_num); - -void ucc_fast_dump_regs(struct ucc_fast_private * uccf); - -#endif /* __UCC_FAST_H__ */ diff --git a/arch/powerpc/include/asm/ucc_slow.h b/arch/powerpc/include/asm/ucc_slow.h deleted file mode 100644 index c44131e68e11..000000000000 --- a/arch/powerpc/include/asm/ucc_slow.h +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Copyright (C) 2006 Freescale Semiconductor, Inc. All rights reserved. - * - * Authors: Shlomi Gridish <gridish@freescale.com> - * Li Yang <leoli@freescale.com> - * - * Description: - * Internal header file for UCC SLOW unit routines. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - */ -#ifndef __UCC_SLOW_H__ -#define __UCC_SLOW_H__ - -#include <linux/kernel.h> - -#include <asm/immap_qe.h> -#include <asm/qe.h> - -#include <asm/ucc.h> - -/* transmit BD's status */ -#define T_R 0x80000000 /* ready bit */ -#define T_PAD 0x40000000 /* add pads to short frames */ -#define T_W 0x20000000 /* wrap bit */ -#define T_I 0x10000000 /* interrupt on completion */ -#define T_L 0x08000000 /* last */ - -#define T_A 0x04000000 /* Address - the data transmitted as address - chars */ -#define T_TC 0x04000000 /* transmit CRC */ -#define T_CM 0x02000000 /* continuous mode */ -#define T_DEF 0x02000000 /* collision on previous attempt to transmit */ -#define T_P 0x01000000 /* Preamble - send Preamble sequence before - data */ -#define T_HB 0x01000000 /* heartbeat */ -#define T_NS 0x00800000 /* No Stop */ -#define T_LC 0x00800000 /* late collision */ -#define T_RL 0x00400000 /* retransmission limit */ -#define T_UN 0x00020000 /* underrun */ -#define T_CT 0x00010000 /* CTS lost */ -#define T_CSL 0x00010000 /* carrier sense lost */ -#define T_RC 0x003c0000 /* retry count */ - -/* Receive BD's status */ -#define R_E 0x80000000 /* buffer empty */ -#define R_W 0x20000000 /* wrap bit */ -#define R_I 0x10000000 /* interrupt on reception */ -#define R_L 0x08000000 /* last */ -#define R_C 0x08000000 /* the last byte in this buffer is a cntl - char */ -#define R_F 0x04000000 /* first */ -#define R_A 0x04000000 /* the first byte in this buffer is address - byte */ -#define R_CM 0x02000000 /* continuous mode */ -#define R_ID 0x01000000 /* buffer close on reception of idles */ -#define R_M 0x01000000 /* Frame received because of promiscuous - mode */ -#define R_AM 0x00800000 /* Address match */ -#define R_DE 0x00800000 /* Address match */ -#define R_LG 0x00200000 /* Break received */ -#define R_BR 0x00200000 /* Frame length violation */ -#define R_NO 0x00100000 /* Rx Non Octet Aligned Packet */ -#define R_FR 0x00100000 /* Framing Error (no stop bit) character - received */ -#define R_PR 0x00080000 /* Parity Error character received */ -#define R_AB 0x00080000 /* Frame Aborted */ -#define R_SH 0x00080000 /* frame is too short */ -#define R_CR 0x00040000 /* CRC Error */ -#define R_OV 0x00020000 /* Overrun */ -#define R_CD 0x00010000 /* CD lost */ -#define R_CL 0x00010000 /* this frame is closed because of a - collision */ - -/* Rx Data buffer must be 4 bytes aligned in most cases.*/ -#define UCC_SLOW_RX_ALIGN 4 -#define UCC_SLOW_MRBLR_ALIGNMENT 4 -#define UCC_SLOW_PRAM_SIZE 0x100 -#define ALIGNMENT_OF_UCC_SLOW_PRAM 64 - -/* UCC Slow Channel Protocol Mode */ -enum ucc_slow_channel_protocol_mode { - UCC_SLOW_CHANNEL_PROTOCOL_MODE_QMC = 0x00000002, - UCC_SLOW_CHANNEL_PROTOCOL_MODE_UART = 0x00000004, - UCC_SLOW_CHANNEL_PROTOCOL_MODE_BISYNC = 0x00000008, -}; - -/* UCC Slow Transparent Transmit CRC (TCRC) */ -enum ucc_slow_transparent_tcrc { - /* 16-bit CCITT CRC (HDLC). (X16 + X12 + X5 + 1) */ - UCC_SLOW_TRANSPARENT_TCRC_CCITT_CRC16 = 0x00000000, - /* CRC16 (BISYNC). (X16 + X15 + X2 + 1) */ - UCC_SLOW_TRANSPARENT_TCRC_CRC16 = 0x00004000, - /* 32-bit CCITT CRC (Ethernet and HDLC) */ - UCC_SLOW_TRANSPARENT_TCRC_CCITT_CRC32 = 0x00008000, -}; - -/* UCC Slow oversampling rate for transmitter (TDCR) */ -enum ucc_slow_tx_oversampling_rate { - /* 1x clock mode */ - UCC_SLOW_OVERSAMPLING_RATE_TX_TDCR_1 = 0x00000000, - /* 8x clock mode */ - UCC_SLOW_OVERSAMPLING_RATE_TX_TDCR_8 = 0x00010000, - /* 16x clock mode */ - UCC_SLOW_OVERSAMPLING_RATE_TX_TDCR_16 = 0x00020000, - /* 32x clock mode */ - UCC_SLOW_OVERSAMPLING_RATE_TX_TDCR_32 = 0x00030000, -}; - -/* UCC Slow Oversampling rate for receiver (RDCR) -*/ -enum ucc_slow_rx_oversampling_rate { - /* 1x clock mode */ - UCC_SLOW_OVERSAMPLING_RATE_RX_RDCR_1 = 0x00000000, - /* 8x clock mode */ - UCC_SLOW_OVERSAMPLING_RATE_RX_RDCR_8 = 0x00004000, - /* 16x clock mode */ - UCC_SLOW_OVERSAMPLING_RATE_RX_RDCR_16 = 0x00008000, - /* 32x clock mode */ - UCC_SLOW_OVERSAMPLING_RATE_RX_RDCR_32 = 0x0000c000, -}; - -/* UCC Slow Transmitter encoding method (TENC) -*/ -enum ucc_slow_tx_encoding_method { - UCC_SLOW_TRANSMITTER_ENCODING_METHOD_TENC_NRZ = 0x00000000, - UCC_SLOW_TRANSMITTER_ENCODING_METHOD_TENC_NRZI = 0x00000100 -}; - -/* UCC Slow Receiver decoding method (RENC) -*/ -enum ucc_slow_rx_decoding_method { - UCC_SLOW_RECEIVER_DECODING_METHOD_RENC_NRZ = 0x00000000, - UCC_SLOW_RECEIVER_DECODING_METHOD_RENC_NRZI = 0x00000800 -}; - -/* UCC Slow Diagnostic mode (DIAG) -*/ -enum ucc_slow_diag_mode { - UCC_SLOW_DIAG_MODE_NORMAL = 0x00000000, - UCC_SLOW_DIAG_MODE_LOOPBACK = 0x00000040, - UCC_SLOW_DIAG_MODE_ECHO = 0x00000080, - UCC_SLOW_DIAG_MODE_LOOPBACK_ECHO = 0x000000c0 -}; - -struct ucc_slow_info { - int ucc_num; - int protocol; /* QE_CR_PROTOCOL_xxx */ - enum qe_clock rx_clock; - enum qe_clock tx_clock; - phys_addr_t regs; - int irq; - u16 uccm_mask; - int data_mem_part; - int init_tx; - int init_rx; - u32 tx_bd_ring_len; - u32 rx_bd_ring_len; - int rx_interrupts; - int brkpt_support; - int grant_support; - int tsa; - int cdp; - int cds; - int ctsp; - int ctss; - int rinv; - int tinv; - int rtsm; - int rfw; - int tci; - int tend; - int tfl; - int txsy; - u16 max_rx_buf_length; - enum ucc_slow_transparent_tcrc tcrc; - enum ucc_slow_channel_protocol_mode mode; - enum ucc_slow_diag_mode diag; - enum ucc_slow_tx_oversampling_rate tdcr; - enum ucc_slow_rx_oversampling_rate rdcr; - enum ucc_slow_tx_encoding_method tenc; - enum ucc_slow_rx_decoding_method renc; -}; - -struct ucc_slow_private { - struct ucc_slow_info *us_info; - struct ucc_slow __iomem *us_regs; /* Ptr to memory map of UCC regs */ - struct ucc_slow_pram *us_pram; /* a pointer to the parameter RAM */ - u32 us_pram_offset; - int enabled_tx; /* Whether channel is enabled for Tx (ENT) */ - int enabled_rx; /* Whether channel is enabled for Rx (ENR) */ - int stopped_tx; /* Whether channel has been stopped for Tx - (STOP_TX, etc.) */ - int stopped_rx; /* Whether channel has been stopped for Rx */ - struct list_head confQ; /* frames passed to chip waiting for tx */ - u32 first_tx_bd_mask; /* mask is used in Tx routine to save status - and length for first BD in a frame */ - u32 tx_base_offset; /* first BD in Tx BD table offset (In MURAM) */ - u32 rx_base_offset; /* first BD in Rx BD table offset (In MURAM) */ - struct qe_bd *confBd; /* next BD for confirm after Tx */ - struct qe_bd *tx_bd; /* next BD for new Tx request */ - struct qe_bd *rx_bd; /* next BD to collect after Rx */ - void *p_rx_frame; /* accumulating receive frame */ - u16 *p_ucce; /* a pointer to the event register in memory. - */ - u16 *p_uccm; /* a pointer to the mask register in memory */ - u16 saved_uccm; /* a saved mask for the RX Interrupt bits */ -#ifdef STATISTICS - u32 tx_frames; /* Transmitted frames counters */ - u32 rx_frames; /* Received frames counters (only frames - passed to application) */ - u32 rx_discarded; /* Discarded frames counters (frames that - were discarded by the driver due to - errors) */ -#endif /* STATISTICS */ -}; - -/* ucc_slow_init - * Initializes Slow UCC according to provided parameters. - * - * us_info - (In) pointer to the slow UCC info structure. - * uccs_ret - (Out) pointer to the slow UCC structure. - */ -int ucc_slow_init(struct ucc_slow_info * us_info, struct ucc_slow_private ** uccs_ret); - -/* ucc_slow_free - * Frees all resources for slow UCC. - * - * uccs - (In) pointer to the slow UCC structure. - */ -void ucc_slow_free(struct ucc_slow_private * uccs); - -/* ucc_slow_enable - * Enables a fast UCC port. - * This routine enables Tx and/or Rx through the General UCC Mode Register. - * - * uccs - (In) pointer to the slow UCC structure. - * mode - (In) TX, RX, or both. - */ -void ucc_slow_enable(struct ucc_slow_private * uccs, enum comm_dir mode); - -/* ucc_slow_disable - * Disables a fast UCC port. - * This routine disables Tx and/or Rx through the General UCC Mode Register. - * - * uccs - (In) pointer to the slow UCC structure. - * mode - (In) TX, RX, or both. - */ -void ucc_slow_disable(struct ucc_slow_private * uccs, enum comm_dir mode); - -/* ucc_slow_poll_transmitter_now - * Immediately forces a poll of the transmitter for data to be sent. - * Typically, the hardware performs a periodic poll for data that the - * transmit routine has set up to be transmitted. In cases where - * this polling cycle is not soon enough, this optional routine can - * be invoked to force a poll right away, instead. Proper use for - * each transmission for which this functionality is desired is to - * call the transmit routine and then this routine right after. - * - * uccs - (In) pointer to the slow UCC structure. - */ -void ucc_slow_poll_transmitter_now(struct ucc_slow_private * uccs); - -/* ucc_slow_graceful_stop_tx - * Smoothly stops transmission on a specified slow UCC. - * - * uccs - (In) pointer to the slow UCC structure. - */ -void ucc_slow_graceful_stop_tx(struct ucc_slow_private * uccs); - -/* ucc_slow_stop_tx - * Stops transmission on a specified slow UCC. - * - * uccs - (In) pointer to the slow UCC structure. - */ -void ucc_slow_stop_tx(struct ucc_slow_private * uccs); - -/* ucc_slow_restart_tx - * Restarts transmitting on a specified slow UCC. - * - * uccs - (In) pointer to the slow UCC structure. - */ -void ucc_slow_restart_tx(struct ucc_slow_private *uccs); - -u32 ucc_slow_get_qe_cr_subblock(int uccs_num); - -#endif /* __UCC_SLOW_H__ */ diff --git a/arch/powerpc/include/asm/udbg.h b/arch/powerpc/include/asm/udbg.h index dc590919f8eb..a8681b12864f 100644 --- a/arch/powerpc/include/asm/udbg.h +++ b/arch/powerpc/include/asm/udbg.h @@ -1,10 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * (c) 2001, 2006 IBM Corporation. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef _ASM_POWERPC_UDBG_H @@ -19,44 +15,42 @@ extern void (*udbg_flush)(void); extern int (*udbg_getc)(void); extern int (*udbg_getc_poll)(void); -extern void udbg_puts(const char *s); -extern int udbg_write(const char *s, int n); +void udbg_puts(const char *s); +int udbg_write(const char *s, int n); -extern void register_early_udbg_console(void); -extern void udbg_printf(const char *fmt, ...) +void register_early_udbg_console(void); +void udbg_printf(const char *fmt, ...) __attribute__ ((format (printf, 1, 2))); -extern void udbg_progress(char *s, unsigned short hex); +void udbg_progress(char *s, unsigned short hex); -extern void udbg_init_uart(void __iomem *comport, unsigned int speed, - unsigned int clock); -extern unsigned int udbg_probe_uart_speed(void __iomem *comport, - unsigned int clock); +void __init udbg_uart_init_mmio(void __iomem *addr, unsigned int stride); +void __init udbg_uart_init_pio(unsigned long port, unsigned int stride); + +void __init udbg_uart_setup(unsigned int speed, unsigned int clock); +unsigned int __init udbg_probe_uart_speed(unsigned int clock); struct device_node; -extern void udbg_scc_init(int force_scc); -extern int udbg_adb_init(int force_btext); -extern void udbg_adb_init_early(void); - -extern void __init udbg_early_init(void); -extern void __init udbg_init_debug_lpar(void); -extern void __init udbg_init_debug_lpar_hvsi(void); -extern void __init udbg_init_pmac_realmode(void); -extern void __init udbg_init_maple_realmode(void); -extern void __init udbg_init_pas_realmode(void); -extern void __init udbg_init_rtas_panel(void); -extern void __init udbg_init_rtas_console(void); -extern void __init udbg_init_debug_beat(void); -extern void __init udbg_init_btext(void); -extern void __init udbg_init_44x_as1(void); -extern void __init udbg_init_40x_realmode(void); -extern void __init udbg_init_cpm(void); -extern void __init udbg_init_usbgecko(void); -extern void __init udbg_init_wsp(void); -extern void __init udbg_init_memcons(void); -extern void __init udbg_init_ehv_bc(void); -extern void __init udbg_init_ps3gelic(void); -extern void __init udbg_init_debug_opal_raw(void); -extern void __init udbg_init_debug_opal_hvsi(void); +void __init udbg_scc_init(int force_scc); +int udbg_adb_init(int force_btext); +void udbg_adb_init_early(void); + +void __init udbg_early_init(void); +void __init udbg_init_debug_lpar(void); +void __init udbg_init_debug_lpar_hvsi(void); +void __init udbg_init_pmac_realmode(void); +void __init udbg_init_pas_realmode(void); +void __init udbg_init_rtas_panel(void); +void __init udbg_init_rtas_console(void); +void __init udbg_init_btext(void); +void __init udbg_init_44x_as1(void); +void __init udbg_init_cpm(void); +void __init udbg_init_usbgecko(void); +void __init udbg_init_memcons(void); +void __init udbg_init_ehv_bc(void); +void __init udbg_init_ps3gelic(void); +void __init udbg_init_debug_opal_raw(void); +void __init udbg_init_debug_opal_hvsi(void); +void __init udbg_init_debug_16550(void); #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_UDBG_H */ diff --git a/arch/powerpc/include/asm/uic.h b/arch/powerpc/include/asm/uic.h index 597edfcae3d6..7b7bd15b1c5c 100644 --- a/arch/powerpc/include/asm/uic.h +++ b/arch/powerpc/include/asm/uic.h @@ -1,13 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * IBM PPC4xx UIC external definitions and structure. * * Maintainer: David Gibson <dwg@au1.ibm.com> * Copyright 2007 IBM Corporation. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. */ #ifndef _ASM_POWERPC_UIC_H #define _ASM_POWERPC_UIC_H diff --git a/arch/powerpc/include/asm/ultravisor-api.h b/arch/powerpc/include/asm/ultravisor-api.h new file mode 100644 index 000000000000..b66f6db7be6c --- /dev/null +++ b/arch/powerpc/include/asm/ultravisor-api.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Ultravisor API. + * + * Copyright 2019, IBM Corporation. + * + */ +#ifndef _ASM_POWERPC_ULTRAVISOR_API_H +#define _ASM_POWERPC_ULTRAVISOR_API_H + +#include <asm/hvcall.h> + +/* Return codes */ +#define U_BUSY H_BUSY +#define U_FUNCTION H_FUNCTION +#define U_NOT_AVAILABLE H_NOT_AVAILABLE +#define U_P2 H_P2 +#define U_P3 H_P3 +#define U_P4 H_P4 +#define U_P5 H_P5 +#define U_PARAMETER H_PARAMETER +#define U_PERMISSION H_PERMISSION +#define U_SUCCESS H_SUCCESS + +/* opcodes */ +#define UV_WRITE_PATE 0xF104 +#define UV_RETURN 0xF11C +#define UV_ESM 0xF110 +#define UV_REGISTER_MEM_SLOT 0xF120 +#define UV_UNREGISTER_MEM_SLOT 0xF124 +#define UV_PAGE_IN 0xF128 +#define UV_PAGE_OUT 0xF12C +#define UV_SHARE_PAGE 0xF130 +#define UV_UNSHARE_PAGE 0xF134 +#define UV_UNSHARE_ALL_PAGES 0xF140 +#define UV_PAGE_INVAL 0xF138 +#define UV_SVM_TERMINATE 0xF13C + +#endif /* _ASM_POWERPC_ULTRAVISOR_API_H */ diff --git a/arch/powerpc/include/asm/ultravisor.h b/arch/powerpc/include/asm/ultravisor.h new file mode 100644 index 000000000000..790b0e63681f --- /dev/null +++ b/arch/powerpc/include/asm/ultravisor.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Ultravisor definitions + * + * Copyright 2019, IBM Corporation. + * + */ +#ifndef _ASM_POWERPC_ULTRAVISOR_H +#define _ASM_POWERPC_ULTRAVISOR_H + +#include <asm/asm-prototypes.h> +#include <asm/ultravisor-api.h> +#include <asm/firmware.h> + +int early_init_dt_scan_ultravisor(unsigned long node, const char *uname, + int depth, void *data); + +/* + * In ultravisor enabled systems, PTCR becomes ultravisor privileged only for + * writing and an attempt to write to it will cause a Hypervisor Emulation + * Assistance interrupt. + */ +static inline void set_ptcr_when_no_uv(u64 val) +{ + if (!firmware_has_feature(FW_FEATURE_ULTRAVISOR)) + mtspr(SPRN_PTCR, val); +} + +static inline int uv_register_pate(u64 lpid, u64 dw0, u64 dw1) +{ + return ucall_norets(UV_WRITE_PATE, lpid, dw0, dw1); +} + +static inline int uv_share_page(u64 pfn, u64 npages) +{ + return ucall_norets(UV_SHARE_PAGE, pfn, npages); +} + +static inline int uv_unshare_page(u64 pfn, u64 npages) +{ + return ucall_norets(UV_UNSHARE_PAGE, pfn, npages); +} + +static inline int uv_unshare_all_pages(void) +{ + return ucall_norets(UV_UNSHARE_ALL_PAGES); +} + +static inline int uv_page_in(u64 lpid, u64 src_ra, u64 dst_gpa, u64 flags, + u64 page_shift) +{ + return ucall_norets(UV_PAGE_IN, lpid, src_ra, dst_gpa, flags, + page_shift); +} + +static inline int uv_page_out(u64 lpid, u64 dst_ra, u64 src_gpa, u64 flags, + u64 page_shift) +{ + return ucall_norets(UV_PAGE_OUT, lpid, dst_ra, src_gpa, flags, + page_shift); +} + +static inline int uv_register_mem_slot(u64 lpid, u64 start_gpa, u64 size, + u64 flags, u64 slotid) +{ + return ucall_norets(UV_REGISTER_MEM_SLOT, lpid, start_gpa, + size, flags, slotid); +} + +static inline int uv_unregister_mem_slot(u64 lpid, u64 slotid) +{ + return ucall_norets(UV_UNREGISTER_MEM_SLOT, lpid, slotid); +} + +static inline int uv_page_inval(u64 lpid, u64 gpa, u64 page_shift) +{ + return ucall_norets(UV_PAGE_INVAL, lpid, gpa, page_shift); +} + +static inline int uv_svm_terminate(u64 lpid) +{ + return ucall_norets(UV_SVM_TERMINATE, lpid); +} + +#endif /* _ASM_POWERPC_ULTRAVISOR_H */ diff --git a/arch/powerpc/include/asm/unaligned.h b/arch/powerpc/include/asm/unaligned.h deleted file mode 100644 index 5f1b1e3c2137..000000000000 --- a/arch/powerpc/include/asm/unaligned.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef _ASM_POWERPC_UNALIGNED_H -#define _ASM_POWERPC_UNALIGNED_H - -#ifdef __KERNEL__ - -/* - * The PowerPC can do unaligned accesses itself in big endian mode. - */ -#include <linux/unaligned/access_ok.h> -#include <linux/unaligned/generic.h> - -#define get_unaligned __get_unaligned_be -#define put_unaligned __put_unaligned_be - -#endif /* __KERNEL__ */ -#endif /* _ASM_POWERPC_UNALIGNED_H */ diff --git a/arch/powerpc/include/asm/uninorth.h b/arch/powerpc/include/asm/uninorth.h index d12b11d7641e..6949b5daa37d 100644 --- a/arch/powerpc/include/asm/uninorth.h +++ b/arch/powerpc/include/asm/uninorth.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * uninorth.h: definitions for using the "UniNorth" host bridge chip * from Apple. This chip is used on "Core99" machines @@ -132,7 +133,7 @@ /* This one _might_ return the CPU number of the CPU reading it; * the bootROM decides whether to boot or to sleep/spinloop depending - * on this register beeing 0 or not + * on this register being 0 or not */ #define UNI_N_CPU_NUMBER 0x0050 @@ -143,7 +144,7 @@ #define UNI_N_HWINIT_STATE_SLEEPING 0x01 #define UNI_N_HWINIT_STATE_RUNNING 0x02 /* This last bit appear to be used by the bootROM to know the second - * CPU has started and will enter it's sleep loop with IP=0 + * CPU has started and will enter its sleep loop with IP=0 */ #define UNI_N_HWINIT_STATE_CPU1_FLAG 0x10000000 diff --git a/arch/powerpc/include/asm/unistd.h b/arch/powerpc/include/asm/unistd.h index 3ca819f541bf..b873fbb6d712 100644 --- a/arch/powerpc/include/asm/unistd.h +++ b/arch/powerpc/include/asm/unistd.h @@ -1,38 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * This file contains the system call numbers. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef _ASM_POWERPC_UNISTD_H_ #define _ASM_POWERPC_UNISTD_H_ #include <uapi/asm/unistd.h> - -#define __NR_syscalls 355 - -#define __NR__exit __NR_exit #define NR_syscalls __NR_syscalls -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> #include <linux/compiler.h> #include <linux/linkage.h> +#define __ARCH_WANT_NEW_STAT #define __ARCH_WANT_OLD_READDIR #define __ARCH_WANT_STAT64 #define __ARCH_WANT_SYS_ALARM #define __ARCH_WANT_SYS_GETHOSTNAME #define __ARCH_WANT_SYS_IPC #define __ARCH_WANT_SYS_PAUSE -#define __ARCH_WANT_SYS_SGETMASK #define __ARCH_WANT_SYS_SIGNAL -#define __ARCH_WANT_SYS_TIME -#define __ARCH_WANT_SYS_UTIME +#define __ARCH_WANT_SYS_TIME32 +#define __ARCH_WANT_SYS_UTIME32 #define __ARCH_WANT_SYS_WAITPID #define __ARCH_WANT_SYS_SOCKETCALL #define __ARCH_WANT_SYS_FADVISE64 @@ -46,15 +38,19 @@ #define __ARCH_WANT_SYS_SIGPROCMASK #ifdef CONFIG_PPC32 #define __ARCH_WANT_OLD_STAT +#define __ARCH_WANT_SYS_OLD_SELECT #endif #ifdef CONFIG_PPC64 -#define __ARCH_WANT_COMPAT_SYS_TIME +#define __ARCH_WANT_SYS_TIME +#define __ARCH_WANT_SYS_UTIME #define __ARCH_WANT_SYS_NEWFSTATAT +#define __ARCH_WANT_COMPAT_STAT +#define __ARCH_WANT_COMPAT_FALLOCATE #define __ARCH_WANT_COMPAT_SYS_SENDFILE #endif #define __ARCH_WANT_SYS_FORK #define __ARCH_WANT_SYS_VFORK #define __ARCH_WANT_SYS_CLONE -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _ASM_POWERPC_UNISTD_H_ */ diff --git a/arch/powerpc/include/asm/uprobes.h b/arch/powerpc/include/asm/uprobes.h index 23016020915e..4fea116d3d37 100644 --- a/arch/powerpc/include/asm/uprobes.h +++ b/arch/powerpc/include/asm/uprobes.h @@ -1,22 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _ASM_UPROBES_H #define _ASM_UPROBES_H /* * User-space Probes (UProbes) for powerpc * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * * Copyright IBM Corporation, 2007-2012 * * Adapted from the x86 port by Ananth N Mavinakayanahalli <ananth@in.ibm.com> @@ -25,9 +12,9 @@ #include <linux/notifier.h> #include <asm/probes.h> -typedef ppc_opcode_t uprobe_opcode_t; +typedef u32 uprobe_opcode_t; -#define MAX_UINSN_BYTES 4 +#define MAX_UINSN_BYTES 8 #define UPROBE_XOL_SLOT_BYTES (MAX_UINSN_BYTES) /* The following alias is needed for reference from arch-agnostic code */ @@ -36,8 +23,8 @@ typedef ppc_opcode_t uprobe_opcode_t; struct arch_uprobe { union { - u8 insn[MAX_UINSN_BYTES]; - u32 ainsn; + u32 insn[2]; + u32 ixol[2]; }; }; @@ -45,11 +32,4 @@ struct arch_uprobe_task { unsigned long saved_trap_nr; }; -extern int arch_uprobe_analyze_insn(struct arch_uprobe *aup, struct mm_struct *mm, unsigned long addr); -extern int arch_uprobe_pre_xol(struct arch_uprobe *aup, struct pt_regs *regs); -extern int arch_uprobe_post_xol(struct arch_uprobe *aup, struct pt_regs *regs); -extern bool arch_uprobe_xol_was_trapped(struct task_struct *tsk); -extern int arch_uprobe_exception_notify(struct notifier_block *self, unsigned long val, void *data); -extern void arch_uprobe_abort_xol(struct arch_uprobe *aup, struct pt_regs *regs); -extern unsigned long arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr, struct pt_regs *regs); #endif /* _ASM_UPROBES_H */ diff --git a/arch/powerpc/include/asm/user.h b/arch/powerpc/include/asm/user.h index 3fd4545dd74e..7fae7e597ba4 100644 --- a/arch/powerpc/include/asm/user.h +++ b/arch/powerpc/include/asm/user.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_USER_H #define _ASM_POWERPC_USER_H @@ -30,7 +31,7 @@ * to write an integer number of pages. */ struct user { - struct pt_regs regs; /* entire machine state */ + struct user_pt_regs regs; /* entire machine state */ size_t u_tsize; /* text size (pages) */ size_t u_dsize; /* data size (pages) */ size_t u_ssize; /* stack size (pages) */ @@ -43,9 +44,4 @@ struct user { char u_comm[32]; /* user command name */ }; -#define NBPG PAGE_SIZE -#define UPAGES 1 -#define HOST_TEXT_START_ADDR (u.start_code) -#define HOST_DATA_START_ADDR (u.start_data) -#define HOST_STACK_END_ADDR (u.start_stack + u.u_ssize * NBPG) #endif /* _ASM_POWERPC_USER_H */ diff --git a/arch/powerpc/include/asm/vas.h b/arch/powerpc/include/asm/vas.h new file mode 100644 index 000000000000..c36f71e01c0f --- /dev/null +++ b/arch/powerpc/include/asm/vas.h @@ -0,0 +1,294 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright 2016-17 IBM Corp. + */ + +#ifndef _ASM_POWERPC_VAS_H +#define _ASM_POWERPC_VAS_H +#include <linux/sched/mm.h> +#include <linux/mmu_context.h> +#include <asm/icswx.h> +#include <uapi/asm/vas-api.h> + +/* + * Min and max FIFO sizes are based on Version 1.05 Section 3.1.4.25 + * (Local FIFO Size Register) of the VAS workbook. + */ +#define VAS_RX_FIFO_SIZE_MIN (1 << 10) /* 1KB */ +#define VAS_RX_FIFO_SIZE_MAX (8 << 20) /* 8MB */ + +/* + * Threshold Control Mode: Have paste operation fail if the number of + * requests in receive FIFO exceeds a threshold. + * + * NOTE: No special error code yet if paste is rejected because of these + * limits. So users can't distinguish between this and other errors. + */ +#define VAS_THRESH_DISABLED 0 +#define VAS_THRESH_FIFO_GT_HALF_FULL 1 +#define VAS_THRESH_FIFO_GT_QTR_FULL 2 +#define VAS_THRESH_FIFO_GT_EIGHTH_FULL 3 + +/* + * VAS window Linux status bits + */ +#define VAS_WIN_ACTIVE 0x0 /* Used in platform independent */ + /* vas mmap() */ +/* Window is closed in the hypervisor due to lost credit */ +#define VAS_WIN_NO_CRED_CLOSE 0x00000001 +/* Window is closed due to migration */ +#define VAS_WIN_MIGRATE_CLOSE 0x00000002 + +/* + * Get/Set bit fields + */ +#define GET_FIELD(m, v) (((v) & (m)) >> MASK_LSH(m)) +#define MASK_LSH(m) (__builtin_ffsl(m) - 1) +#define SET_FIELD(m, v, val) \ + (((v) & ~(m)) | ((((typeof(v))(val)) << MASK_LSH(m)) & (m))) + +/* + * Co-processor Engine type. + */ +enum vas_cop_type { + VAS_COP_TYPE_FAULT, + VAS_COP_TYPE_842, + VAS_COP_TYPE_842_HIPRI, + VAS_COP_TYPE_GZIP, + VAS_COP_TYPE_GZIP_HIPRI, + VAS_COP_TYPE_FTW, + VAS_COP_TYPE_MAX, +}; + +/* + * User space VAS windows are opened by tasks and take references + * to pid and mm until windows are closed. + * Stores pid, mm, and tgid for each window. + */ +struct vas_user_win_ref { + struct pid *pid; /* PID of owner */ + struct pid *tgid; /* Thread group ID of owner */ + struct mm_struct *mm; /* Linux process mm_struct */ + struct mutex mmap_mutex; /* protects paste address mmap() */ + /* with DLPAR close/open windows */ + struct vm_area_struct *vma; /* Save VMA and used in DLPAR ops */ +}; + +/* + * Common VAS window struct on PowerNV and PowerVM + */ +struct vas_window { + u32 winid; + u32 wcreds_max; /* Window credits */ + u32 status; /* Window status used in OS */ + enum vas_cop_type cop; + struct vas_user_win_ref task_ref; + char *dbgname; + struct dentry *dbgdir; +}; + +/* + * User space window operations used for powernv and powerVM + */ +struct vas_user_win_ops { + struct vas_window * (*open_win)(int vas_id, u64 flags, + enum vas_cop_type); + u64 (*paste_addr)(struct vas_window *); + int (*close_win)(struct vas_window *); +}; + +static inline void put_vas_user_win_ref(struct vas_user_win_ref *ref) +{ + /* Drop references to pid, tgid, and mm */ + put_pid(ref->pid); + put_pid(ref->tgid); + if (ref->mm) + mmdrop(ref->mm); +} + +static inline void vas_user_win_add_mm_context(struct vas_user_win_ref *ref) +{ + mm_context_add_vas_window(ref->mm); + /* + * Even a process that has no foreign real address mapping can + * use an unpaired COPY instruction (to no real effect). Issue + * CP_ABORT to clear any pending COPY and prevent a covert + * channel. + * + * __switch_to() will issue CP_ABORT on future context switches + * if process / thread has any open VAS window (Use + * current->mm->context.vas_windows). + */ + asm volatile(PPC_CP_ABORT); +} + +/* + * Receive window attributes specified by the (in-kernel) owner of window. + */ +struct vas_rx_win_attr { + u64 rx_fifo; + int rx_fifo_size; + int wcreds_max; + + bool pin_win; + bool rej_no_credit; + bool tx_wcred_mode; + bool rx_wcred_mode; + bool tx_win_ord_mode; + bool rx_win_ord_mode; + bool data_stamp; + bool nx_win; + bool fault_win; + bool user_win; + bool notify_disable; + bool intr_disable; + bool notify_early; + + int lnotify_lpid; + int lnotify_pid; + int lnotify_tid; + u32 pswid; + + int tc_mode; +}; + +/* + * Window attributes specified by the in-kernel owner of a send window. + */ +struct vas_tx_win_attr { + enum vas_cop_type cop; + int wcreds_max; + int lpid; + int pidr; /* hardware PID (from SPRN_PID) */ + int pswid; + int rsvd_txbuf_count; + int tc_mode; + + bool user_win; + bool pin_win; + bool rej_no_credit; + bool rsvd_txbuf_enable; + bool tx_wcred_mode; + bool rx_wcred_mode; + bool tx_win_ord_mode; + bool rx_win_ord_mode; +}; + +#ifdef CONFIG_PPC_POWERNV +/* + * Helper to map a chip id to VAS id. + * For POWER9, this is a 1:1 mapping. In the future this maybe a 1:N + * mapping in which case, we will need to update this helper. + * + * Return the VAS id or -1 if no matching vasid is found. + */ +int chip_to_vas_id(int chipid); + +/* + * Helper to initialize receive window attributes to defaults for an + * NX window. + */ +void vas_init_rx_win_attr(struct vas_rx_win_attr *rxattr, enum vas_cop_type cop); + +/* + * Open a VAS receive window for the instance of VAS identified by @vasid + * Use @attr to initialize the attributes of the window. + * + * Return a handle to the window or ERR_PTR() on error. + */ +struct vas_window *vas_rx_win_open(int vasid, enum vas_cop_type cop, + struct vas_rx_win_attr *attr); + +/* + * Helper to initialize send window attributes to defaults for an NX window. + */ +extern void vas_init_tx_win_attr(struct vas_tx_win_attr *txattr, + enum vas_cop_type cop); + +/* + * Open a VAS send window for the instance of VAS identified by @vasid + * and the co-processor type @cop. Use @attr to initialize attributes + * of the window. + * + * Note: The instance of VAS must already have an open receive window for + * the coprocessor type @cop. + * + * Return a handle to the send window or ERR_PTR() on error. + */ +struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop, + struct vas_tx_win_attr *attr); + +/* + * Close the send or receive window identified by @win. For receive windows + * return -EAGAIN if there are active send windows attached to this receive + * window. + */ +int vas_win_close(struct vas_window *win); + +/* + * Copy the co-processor request block (CRB) @crb into the local L2 cache. + */ +int vas_copy_crb(void *crb, int offset); + +/* + * Paste a previously copied CRB (see vas_copy_crb()) from the L2 cache to + * the hardware address associated with the window @win. @re is expected/ + * assumed to be true for NX windows. + */ +int vas_paste_crb(struct vas_window *win, int offset, bool re); + +int vas_register_api_powernv(struct module *mod, enum vas_cop_type cop_type, + const char *name); +void vas_unregister_api_powernv(void); +#endif + +#ifdef CONFIG_PPC_PSERIES + +/* VAS Capabilities */ +#define VAS_GZIP_QOS_FEAT 0x1 +#define VAS_GZIP_DEF_FEAT 0x2 +#define VAS_GZIP_QOS_FEAT_BIT PPC_BIT(VAS_GZIP_QOS_FEAT) /* Bit 1 */ +#define VAS_GZIP_DEF_FEAT_BIT PPC_BIT(VAS_GZIP_DEF_FEAT) /* Bit 2 */ + +/* NX Capabilities */ +#define VAS_NX_GZIP_FEAT 0x1 +#define VAS_NX_GZIP_FEAT_BIT PPC_BIT(VAS_NX_GZIP_FEAT) /* Bit 1 */ + +/* + * These structs are used to retrieve overall VAS capabilities that + * the hypervisor provides. + */ +struct hv_vas_all_caps { + __be64 descriptor; + __be64 feat_type; +} __packed __aligned(0x1000); + +struct vas_all_caps { + u64 descriptor; + u64 feat_type; +}; + +int h_query_vas_capabilities(const u64 hcall, u8 query_type, u64 result); +int vas_register_api_pseries(struct module *mod, + enum vas_cop_type cop_type, const char *name); +void vas_unregister_api_pseries(void); +#endif + +/* + * Register / unregister coprocessor type to VAS API which will be exported + * to user space. Applications can use this API to open / close window + * which can be used to send / receive requests directly to cooprcessor. + * + * Only NX GZIP coprocessor type is supported now, but this API can be + * used for others in future. + */ +int vas_register_coproc_api(struct module *mod, enum vas_cop_type cop_type, + const char *name, + const struct vas_user_win_ops *vops); +void vas_unregister_coproc_api(void); + +int get_vas_user_win_ref(struct vas_user_win_ref *task_ref); +void vas_update_csb(struct coprocessor_request_block *crb, + struct vas_user_win_ref *task_ref); +void vas_dump_crb(struct coprocessor_request_block *crb); +#endif /* __ASM_POWERPC_VAS_H */ diff --git a/arch/powerpc/include/asm/vdso.h b/arch/powerpc/include/asm/vdso.h index 0d9cecddf8a4..07af32576072 100644 --- a/arch/powerpc/include/asm/vdso.h +++ b/arch/powerpc/include/asm/vdso.h @@ -1,61 +1,38 @@ -#ifndef __PPC64_VDSO_H__ -#define __PPC64_VDSO_H__ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_VDSO_H +#define _ASM_POWERPC_VDSO_H -#ifdef __KERNEL__ - -/* Default link addresses for the vDSOs */ -#define VDSO32_LBASE 0x100000 -#define VDSO64_LBASE 0x100000 +#define VDSO_VERSION_STRING LINUX_2.6.15 +#define __VDSO_PAGES 4 -/* Default map addresses for 32bit vDSO */ -#define VDSO32_MBASE VDSO32_LBASE +#ifndef __ASSEMBLER__ -#define VDSO_VERSION_STRING LINUX_2.6.15 +#ifdef CONFIG_PPC64 +#include <generated/vdso64-offsets.h> +#endif -/* Define if 64 bits VDSO has procedure descriptors */ -#undef VDS64_HAS_DESCRIPTORS +#ifdef CONFIG_VDSO32 +#include <generated/vdso32-offsets.h> +#endif -#ifndef __ASSEMBLY__ +#define VDSO64_SYMBOL(base, name) ((unsigned long)(base) + (vdso64_offset_##name)) -/* Offsets relative to thread->vdso_base */ -extern unsigned long vdso64_rt_sigtramp; -extern unsigned long vdso32_sigtramp; -extern unsigned long vdso32_rt_sigtramp; +#define VDSO32_SYMBOL(base, name) ((unsigned long)(base) + (vdso32_offset_##name)) int vdso_getcpu_init(void); -#else /* __ASSEMBLY__ */ +#else /* __ASSEMBLER__ */ #ifdef __VDSO64__ -#ifdef VDS64_HAS_DESCRIPTORS -#define V_FUNCTION_BEGIN(name) \ - .globl name; \ - .section ".opd","a"; \ - .align 3; \ - name: \ - .quad .name,.TOC.@tocbase,0; \ - .previous; \ - .globl .name; \ - .type .name,@function; \ - .name: \ - -#define V_FUNCTION_END(name) \ - .size .name,.-.name; - -#define V_LOCAL_FUNC(name) (.name) - -#else /* VDS64_HAS_DESCRIPTORS */ - #define V_FUNCTION_BEGIN(name) \ .globl name; \ + .type name,@function; \ name: \ #define V_FUNCTION_END(name) \ .size name,.-name; #define V_LOCAL_FUNC(name) (name) - -#endif /* VDS64_HAS_DESCRIPTORS */ #endif /* __VDSO64__ */ #ifdef __VDSO32__ @@ -72,8 +49,6 @@ int vdso_getcpu_init(void); #endif /* __VDSO32__ */ -#endif /* __ASSEMBLY__ */ - -#endif /* __KERNEL__ */ +#endif /* __ASSEMBLER__ */ -#endif /* __PPC64_VDSO_H__ */ +#endif /* _ASM_POWERPC_VDSO_H */ diff --git a/arch/powerpc/include/asm/vdso/arch_data.h b/arch/powerpc/include/asm/vdso/arch_data.h new file mode 100644 index 000000000000..c240a6b87518 --- /dev/null +++ b/arch/powerpc/include/asm/vdso/arch_data.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2002 Peter Bergner <bergner@vnet.ibm.com>, IBM + * Copyright (C) 2005 Benjamin Herrenschmidy <benh@kernel.crashing.org>, + * IBM Corp. + */ +#ifndef _ASM_POWERPC_VDSO_ARCH_DATA_H +#define _ASM_POWERPC_VDSO_ARCH_DATA_H + +#include <linux/unistd.h> +#include <linux/types.h> + +#define SYSCALL_MAP_SIZE ((NR_syscalls + 31) / 32) + +#ifdef CONFIG_PPC64 + +struct vdso_arch_data { + __u64 tb_ticks_per_sec; /* Timebase tics / sec */ + __u32 dcache_block_size; /* L1 d-cache block size */ + __u32 icache_block_size; /* L1 i-cache block size */ + __u32 dcache_log_block_size; /* L1 d-cache log block size */ + __u32 icache_log_block_size; /* L1 i-cache log block size */ + __u32 syscall_map[SYSCALL_MAP_SIZE]; /* Map of syscalls */ + __u32 compat_syscall_map[SYSCALL_MAP_SIZE]; /* Map of compat syscalls */ +}; + +#else /* CONFIG_PPC64 */ + +struct vdso_arch_data { + __u64 tb_ticks_per_sec; /* Timebase tics / sec */ + __u32 syscall_map[SYSCALL_MAP_SIZE]; /* Map of syscalls */ + __u32 compat_syscall_map[0]; /* No compat syscalls on PPC32 */ +}; + +#endif /* CONFIG_PPC64 */ + +#endif /* _ASM_POWERPC_VDSO_ARCH_DATA_H */ diff --git a/arch/powerpc/include/asm/vdso/clocksource.h b/arch/powerpc/include/asm/vdso/clocksource.h new file mode 100644 index 000000000000..c1ba56b82ee5 --- /dev/null +++ b/arch/powerpc/include/asm/vdso/clocksource.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_VDSO_CLOCKSOURCE_H +#define _ASM_POWERPC_VDSO_CLOCKSOURCE_H + +#define VDSO_ARCH_CLOCKMODES VDSO_CLOCKMODE_ARCHTIMER + +#endif diff --git a/arch/powerpc/include/asm/vdso/getrandom.h b/arch/powerpc/include/asm/vdso/getrandom.h new file mode 100644 index 000000000000..4c24976061f4 --- /dev/null +++ b/arch/powerpc/include/asm/vdso/getrandom.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2024 Christophe Leroy <christophe.leroy@csgroup.eu>, CS GROUP France + */ +#ifndef _ASM_POWERPC_VDSO_GETRANDOM_H +#define _ASM_POWERPC_VDSO_GETRANDOM_H + +#ifndef __ASSEMBLER__ + +#include <asm/vdso_datapage.h> + +static __always_inline int do_syscall_3(const unsigned long _r0, const unsigned long _r3, + const unsigned long _r4, const unsigned long _r5) +{ + register long r0 asm("r0") = _r0; + register unsigned long r3 asm("r3") = _r3; + register unsigned long r4 asm("r4") = _r4; + register unsigned long r5 asm("r5") = _r5; + register int ret asm ("r3"); + + asm volatile( + " sc\n" + " bns+ 1f\n" + " neg %0, %0\n" + "1:\n" + : "=r" (ret), "+r" (r4), "+r" (r5), "+r" (r0) + : "r" (r3) + : "memory", "r6", "r7", "r8", "r9", "r10", "r11", "r12", "cr0", "ctr"); + + return ret; +} + +/** + * getrandom_syscall - Invoke the getrandom() syscall. + * @buffer: Destination buffer to fill with random bytes. + * @len: Size of @buffer in bytes. + * @flags: Zero or more GRND_* flags. + * Returns: The number of bytes written to @buffer, or a negative value indicating an error. + */ +static __always_inline ssize_t getrandom_syscall(void *buffer, size_t len, unsigned int flags) +{ + return do_syscall_3(__NR_getrandom, (unsigned long)buffer, + (unsigned long)len, (unsigned long)flags); +} + +static __always_inline const struct vdso_rng_data *__arch_get_vdso_u_rng_data(void) +{ + struct vdso_rng_data *data; + + asm ( + " bcl 20, 31, .+4 ;" + "0: mflr %0 ;" + " addis %0, %0, (vdso_u_rng_data - 0b)@ha ;" + " addi %0, %0, (vdso_u_rng_data - 0b)@l ;" + : "=r" (data) : : "lr" + ); + + return data; +} +#define __arch_get_vdso_u_rng_data __arch_get_vdso_u_rng_data + +ssize_t __c_kernel_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state, + size_t opaque_len); + +#endif /* !__ASSEMBLER__ */ + +#endif /* _ASM_POWERPC_VDSO_GETRANDOM_H */ diff --git a/arch/powerpc/include/asm/vdso/gettimeofday.h b/arch/powerpc/include/asm/vdso/gettimeofday.h new file mode 100644 index 000000000000..ab3df12c8d94 --- /dev/null +++ b/arch/powerpc/include/asm/vdso/gettimeofday.h @@ -0,0 +1,146 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_VDSO_GETTIMEOFDAY_H +#define _ASM_POWERPC_VDSO_GETTIMEOFDAY_H + +#ifndef __ASSEMBLER__ + +#include <asm/vdso/timebase.h> +#include <asm/barrier.h> +#include <asm/unistd.h> +#include <uapi/linux/time.h> + +#define VDSO_HAS_CLOCK_GETRES 1 + +#define VDSO_HAS_TIME 1 + +/* + * powerpc specific delta calculation. + * + * This variant removes the masking of the subtraction because the + * clocksource mask of all VDSO capable clocksources on powerpc is U64_MAX + * which would result in a pointless operation. The compiler cannot + * optimize it away as the mask comes from the vdso data and is not compile + * time constant. + */ +#define VDSO_DELTA_NOMASK 1 + +static __always_inline int do_syscall_2(const unsigned long _r0, const unsigned long _r3, + const unsigned long _r4) +{ + register long r0 asm("r0") = _r0; + register unsigned long r3 asm("r3") = _r3; + register unsigned long r4 asm("r4") = _r4; + register int ret asm ("r3"); + + asm volatile( + " sc\n" + " bns+ 1f\n" + " neg %0, %0\n" + "1:\n" + : "=r" (ret), "+r" (r4), "+r" (r0) + : "r" (r3) + : "memory", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12", "cr0", "ctr"); + + return ret; +} + +static __always_inline +int gettimeofday_fallback(struct __kernel_old_timeval *_tv, struct timezone *_tz) +{ + return do_syscall_2(__NR_gettimeofday, (unsigned long)_tv, (unsigned long)_tz); +} + +#ifdef __powerpc64__ + +static __always_inline +int clock_gettime_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) +{ + return do_syscall_2(__NR_clock_gettime, _clkid, (unsigned long)_ts); +} + +static __always_inline +int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) +{ + return do_syscall_2(__NR_clock_getres, _clkid, (unsigned long)_ts); +} + +#else + +#define BUILD_VDSO32 1 + +static __always_inline +int clock_gettime_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) +{ + return do_syscall_2(__NR_clock_gettime64, _clkid, (unsigned long)_ts); +} + +static __always_inline +int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) +{ + return do_syscall_2(__NR_clock_getres_time64, _clkid, (unsigned long)_ts); +} + +static __always_inline +int clock_gettime32_fallback(clockid_t _clkid, struct old_timespec32 *_ts) +{ + return do_syscall_2(__NR_clock_gettime, _clkid, (unsigned long)_ts); +} + +static __always_inline +int clock_getres32_fallback(clockid_t _clkid, struct old_timespec32 *_ts) +{ + return do_syscall_2(__NR_clock_getres, _clkid, (unsigned long)_ts); +} +#endif + +static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, + const struct vdso_time_data *vd) +{ + return get_tb(); +} + +static inline bool vdso_clocksource_ok(const struct vdso_clock *vc) +{ + return true; +} +#define vdso_clocksource_ok vdso_clocksource_ok + +#ifndef __powerpc64__ +static __always_inline u64 vdso_shift_ns(u64 ns, unsigned long shift) +{ + u32 hi = ns >> 32; + u32 lo = ns; + + lo >>= shift; + lo |= hi << (32 - shift); + hi >>= shift; + + if (likely(hi == 0)) + return lo; + + return ((u64)hi << 32) | lo; +} +#define vdso_shift_ns vdso_shift_ns +#endif + +#ifdef __powerpc64__ +int __c_kernel_clock_gettime(clockid_t clock, struct __kernel_timespec *ts, + const struct vdso_time_data *vd); +int __c_kernel_clock_getres(clockid_t clock_id, struct __kernel_timespec *res, + const struct vdso_time_data *vd); +#else +int __c_kernel_clock_gettime(clockid_t clock, struct old_timespec32 *ts, + const struct vdso_time_data *vd); +int __c_kernel_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts, + const struct vdso_time_data *vd); +int __c_kernel_clock_getres(clockid_t clock_id, struct old_timespec32 *res, + const struct vdso_time_data *vd); +#endif +int __c_kernel_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz, + const struct vdso_time_data *vd); +__kernel_old_time_t __c_kernel_time(__kernel_old_time_t *time, + const struct vdso_time_data *vd); + +#endif /* __ASSEMBLER__ */ + +#endif /* _ASM_POWERPC_VDSO_GETTIMEOFDAY_H */ diff --git a/arch/powerpc/include/asm/vdso/processor.h b/arch/powerpc/include/asm/vdso/processor.h new file mode 100644 index 000000000000..c1f3d7aaf3ee --- /dev/null +++ b/arch/powerpc/include/asm/vdso/processor.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _ASM_POWERPC_VDSO_PROCESSOR_H +#define _ASM_POWERPC_VDSO_PROCESSOR_H + +#ifndef __ASSEMBLER__ + +/* Macros for adjusting thread priority (hardware multi-threading) */ +#ifdef CONFIG_PPC64 +#define HMT_very_low() asm volatile("or 31, 31, 31 # very low priority") +#define HMT_low() asm volatile("or 1, 1, 1 # low priority") +#define HMT_medium_low() asm volatile("or 6, 6, 6 # medium low priority") +#define HMT_medium() asm volatile("or 2, 2, 2 # medium priority") +#define HMT_medium_high() asm volatile("or 5, 5, 5 # medium high priority") +#define HMT_high() asm volatile("or 3, 3, 3 # high priority") +#else +#define HMT_very_low() +#define HMT_low() +#define HMT_medium_low() +#define HMT_medium() +#define HMT_medium_high() +#define HMT_high() +#endif + +#ifdef CONFIG_PPC64 +#define cpu_relax() \ + asm volatile(ASM_FTR_IFCLR( \ + /* Pre-POWER10 uses low ; medium priority nops */ \ + "or 1,1,1 ; or 2,2,2", \ + /* POWER10 onward uses pause_short (wait 2,0) */ \ + PPC_WAIT(2, 0), \ + %0) :: "i" (CPU_FTR_ARCH_31) : "memory") +#else +#define cpu_relax() barrier() +#endif + +#endif /* __ASSEMBLER__ */ + +#endif /* _ASM_POWERPC_VDSO_PROCESSOR_H */ diff --git a/arch/powerpc/include/asm/vdso/timebase.h b/arch/powerpc/include/asm/vdso/timebase.h new file mode 100644 index 000000000000..e9245f86a46c --- /dev/null +++ b/arch/powerpc/include/asm/vdso/timebase.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Common timebase prototypes and such for all ppc machines. + */ + +#ifndef _ASM_POWERPC_VDSO_TIMEBASE_H +#define _ASM_POWERPC_VDSO_TIMEBASE_H + +#include <asm/reg.h> + +/* + * We use __powerpc64__ here because we want the compat VDSO to use the 32-bit + * version below in the else case of the ifdef. + */ +#if defined(__powerpc64__) && (defined(CONFIG_PPC_CELL) || defined(CONFIG_PPC_E500)) +#define mftb() ({unsigned long rval; \ + asm volatile( \ + "90: mfspr %0, %2;\n" \ + ASM_FTR_IFSET( \ + "97: cmpwi %0,0;\n" \ + " beq- 90b;\n", "", %1) \ + : "=r" (rval) \ + : "i" (CPU_FTR_CELL_TB_BUG), "i" (SPRN_TBRL) : "cr0"); \ + rval;}) +#elif defined(CONFIG_PPC_8xx) +#define mftb() ({unsigned long rval; \ + asm volatile("mftbl %0" : "=r" (rval)); rval;}) +#else +#define mftb() ({unsigned long rval; \ + asm volatile("mfspr %0, %1" : \ + "=r" (rval) : "i" (SPRN_TBRL)); rval;}) +#endif /* !CONFIG_PPC_CELL */ + +#if defined(CONFIG_PPC_8xx) +#define mftbu() ({unsigned long rval; \ + asm volatile("mftbu %0" : "=r" (rval)); rval;}) +#else +#define mftbu() ({unsigned long rval; \ + asm volatile("mfspr %0, %1" : "=r" (rval) : \ + "i" (SPRN_TBRU)); rval;}) +#endif + +#define mttbl(v) asm volatile("mttbl %0":: "r"(v)) +#define mttbu(v) asm volatile("mttbu %0":: "r"(v)) + +static __always_inline u64 get_tb(void) +{ + unsigned int tbhi, tblo, tbhi2; + + /* + * We use __powerpc64__ here not CONFIG_PPC64 because we want the compat + * VDSO to use the 32-bit compatible version in the while loop below. + */ + if (__is_defined(__powerpc64__)) + return mftb(); + + do { + tbhi = mftbu(); + tblo = mftb(); + tbhi2 = mftbu(); + } while (tbhi != tbhi2); + + return ((u64)tbhi << 32) | tblo; +} + +static inline void set_tb(unsigned int upper, unsigned int lower) +{ + mtspr(SPRN_TBWL, 0); + mtspr(SPRN_TBWU, upper); + mtspr(SPRN_TBWL, lower); +} + +#endif /* _ASM_POWERPC_VDSO_TIMEBASE_H */ diff --git a/arch/powerpc/include/asm/vdso/vsyscall.h b/arch/powerpc/include/asm/vdso/vsyscall.h new file mode 100644 index 000000000000..bee18e8660a0 --- /dev/null +++ b/arch/powerpc/include/asm/vdso/vsyscall.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_VDSO_VSYSCALL_H +#define _ASM_POWERPC_VDSO_VSYSCALL_H + +#ifndef __ASSEMBLER__ + +#include <asm/vdso_datapage.h> + +/* The asm-generic header needs to be included after the definitions above */ +#include <asm-generic/vdso/vsyscall.h> + +#endif /* !__ASSEMBLER__ */ + +#endif /* _ASM_POWERPC_VDSO_VSYSCALL_H */ diff --git a/arch/powerpc/include/asm/vdso_datapage.h b/arch/powerpc/include/asm/vdso_datapage.h index b73a8199f161..441264af0e36 100644 --- a/arch/powerpc/include/asm/vdso_datapage.h +++ b/arch/powerpc/include/asm/vdso_datapage.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _VDSO_DATAPAGE_H #define _VDSO_DATAPAGE_H #ifdef __KERNEL__ @@ -6,119 +7,23 @@ * Copyright (C) 2002 Peter Bergner <bergner@vnet.ibm.com>, IBM * Copyright (C) 2005 Benjamin Herrenschmidy <benh@kernel.crashing.org>, * IBM Corp. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ +#ifndef __ASSEMBLER__ -/* - * Note about this structure: - * - * This structure was historically called systemcfg and exposed to - * userland via /proc/ppc64/systemcfg. Unfortunately, this became an - * ABI issue as some proprietary software started relying on being able - * to mmap() it, thus we have to keep the base layout at least for a - * few kernel versions. - * - * However, since ppc32 doesn't suffer from this backward handicap, - * a simpler version of the data structure is used there with only the - * fields actually used by the vDSO. - * - */ - -/* - * If the major version changes we are incompatible. - * Minor version changes are a hint. - */ -#define SYSTEMCFG_MAJOR 1 -#define SYSTEMCFG_MINOR 1 - -#ifndef __ASSEMBLY__ - -#include <linux/unistd.h> -#include <linux/time.h> - -#define SYSCALL_MAP_SIZE ((__NR_syscalls + 31) / 32) - -/* - * So here is the ppc64 backward compatible version - */ - -#ifdef CONFIG_PPC64 - -struct vdso_data { - __u8 eye_catcher[16]; /* Eyecatcher: SYSTEMCFG:PPC64 0x00 */ - struct { /* Systemcfg version numbers */ - __u32 major; /* Major number 0x10 */ - __u32 minor; /* Minor number 0x14 */ - } version; - - /* Note about the platform flags: it now only contains the lpar - * bit. The actual platform number is dead and buried - */ - __u32 platform; /* Platform flags 0x18 */ - __u32 processor; /* Processor type 0x1C */ - __u64 processorCount; /* # of physical processors 0x20 */ - __u64 physicalMemorySize; /* Size of real memory(B) 0x28 */ - __u64 tb_orig_stamp; /* Timebase at boot 0x30 */ - __u64 tb_ticks_per_sec; /* Timebase tics / sec 0x38 */ - __u64 tb_to_xs; /* Inverse of TB to 2^20 0x40 */ - __u64 stamp_xsec; /* 0x48 */ - __u64 tb_update_count; /* Timebase atomicity ctr 0x50 */ - __u32 tz_minuteswest; /* Minutes west of Greenwich 0x58 */ - __u32 tz_dsttime; /* Type of dst correction 0x5C */ - __u32 dcache_size; /* L1 d-cache size 0x60 */ - __u32 dcache_line_size; /* L1 d-cache line size 0x64 */ - __u32 icache_size; /* L1 i-cache size 0x68 */ - __u32 icache_line_size; /* L1 i-cache line size 0x6C */ - - /* those additional ones don't have to be located anywhere - * special as they were not part of the original systemcfg - */ - __u32 dcache_block_size; /* L1 d-cache block size */ - __u32 icache_block_size; /* L1 i-cache block size */ - __u32 dcache_log_block_size; /* L1 d-cache log block size */ - __u32 icache_log_block_size; /* L1 i-cache log block size */ - __s32 wtom_clock_sec; /* Wall to monotonic clock */ - __s32 wtom_clock_nsec; - struct timespec stamp_xtime; /* xtime as at tb_orig_stamp */ - __u32 stamp_sec_fraction; /* fractional seconds of stamp_xtime */ - __u32 syscall_map_64[SYSCALL_MAP_SIZE]; /* map of syscalls */ - __u32 syscall_map_32[SYSCALL_MAP_SIZE]; /* map of syscalls */ -}; - -#else /* CONFIG_PPC64 */ - -/* - * And here is the simpler 32 bits version - */ -struct vdso_data { - __u64 tb_orig_stamp; /* Timebase at boot 0x30 */ - __u64 tb_ticks_per_sec; /* Timebase tics / sec 0x38 */ - __u64 tb_to_xs; /* Inverse of TB to 2^20 0x40 */ - __u64 stamp_xsec; /* 0x48 */ - __u32 tb_update_count; /* Timebase atomicity ctr 0x50 */ - __u32 tz_minuteswest; /* Minutes west of Greenwich 0x58 */ - __u32 tz_dsttime; /* Type of dst correction 0x5C */ - __s32 wtom_clock_sec; /* Wall to monotonic clock */ - __s32 wtom_clock_nsec; - struct timespec stamp_xtime; /* xtime as at tb_orig_stamp */ - __u32 stamp_sec_fraction; /* fractional seconds of stamp_xtime */ - __u32 syscall_map_32[SYSCALL_MAP_SIZE]; /* map of syscalls */ - __u32 dcache_block_size; /* L1 d-cache block size */ - __u32 icache_block_size; /* L1 i-cache block size */ - __u32 dcache_log_block_size; /* L1 d-cache log block size */ - __u32 icache_log_block_size; /* L1 i-cache log block size */ -}; +#include <vdso/datapage.h> -#endif /* CONFIG_PPC64 */ +#else /* __ASSEMBLER__ */ -extern struct vdso_data *vdso_data; +.macro get_datapage ptr symbol + bcl 20, 31, .+4 +999: + mflr \ptr + addis \ptr, \ptr, (\symbol - 999b)@ha + addi \ptr, \ptr, (\symbol - 999b)@l +.endm -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __KERNEL__ */ #endif /* _SYSTEMCFG_H */ diff --git a/arch/powerpc/include/asm/vermagic.h b/arch/powerpc/include/asm/vermagic.h new file mode 100644 index 000000000000..6f250fe506bd --- /dev/null +++ b/arch/powerpc/include/asm/vermagic.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_VERMAGIC_H +#define _ASM_VERMAGIC_H + +#ifdef CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY +#define MODULE_ARCH_VERMAGIC_FTRACE "patchable-function-entry " +#elif defined(CONFIG_MPROFILE_KERNEL) +#define MODULE_ARCH_VERMAGIC_FTRACE "mprofile-kernel " +#else +#define MODULE_ARCH_VERMAGIC_FTRACE "" +#endif + +#ifdef CONFIG_RELOCATABLE +#define MODULE_ARCH_VERMAGIC_RELOCATABLE "relocatable " +#else +#define MODULE_ARCH_VERMAGIC_RELOCATABLE "" +#endif + +#define MODULE_ARCH_VERMAGIC \ + MODULE_ARCH_VERMAGIC_FTRACE MODULE_ARCH_VERMAGIC_RELOCATABLE + +#endif /* _ASM_VERMAGIC_H */ diff --git a/arch/powerpc/include/asm/vga.h b/arch/powerpc/include/asm/vga.h index a2eac409c1ec..f2dc40e1c52a 100644 --- a/arch/powerpc/include/asm/vga.h +++ b/arch/powerpc/include/asm/vga.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_VGA_H_ #define _ASM_POWERPC_VGA_H_ @@ -25,25 +26,26 @@ static inline void scr_writew(u16 val, volatile u16 *addr) { - st_le16(addr, val); + *addr = cpu_to_le16(val); } static inline u16 scr_readw(volatile const u16 *addr) { - return ld_le16(addr); + return le16_to_cpu(*addr); } -#define VT_BUF_HAVE_MEMCPYW -#define scr_memcpyw memcpy +#define VT_BUF_HAVE_MEMSETW +static inline void scr_memsetw(u16 *s, u16 v, unsigned int n) +{ + memset16(s, cpu_to_le16(v), n / 2); +} #endif /* !CONFIG_VGA_CONSOLE && !CONFIG_MDA_CONSOLE */ -extern unsigned long vgacon_remap_base; - #ifdef __powerpc64__ #define VGA_MAP_MEM(x,s) ((unsigned long) ioremap((x), s)) #else -#define VGA_MAP_MEM(x,s) (x + vgacon_remap_base) +#define VGA_MAP_MEM(x,s) (x) #endif #define vga_readb(x) (*(x)) diff --git a/arch/powerpc/include/asm/video.h b/arch/powerpc/include/asm/video.h new file mode 100644 index 000000000000..e1770114ffc3 --- /dev/null +++ b/arch/powerpc/include/asm/video.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_VIDEO_H_ +#define _ASM_VIDEO_H_ + +#include <asm/page.h> + +static inline pgprot_t pgprot_framebuffer(pgprot_t prot, + unsigned long vm_start, unsigned long vm_end, + unsigned long offset) +{ + return __phys_mem_access_prot(PHYS_PFN(offset), vm_end - vm_start, prot); +} +#define pgprot_framebuffer pgprot_framebuffer + +#include <asm-generic/video.h> + +#endif /* _ASM_VIDEO_H_ */ diff --git a/arch/powerpc/include/asm/vio.h b/arch/powerpc/include/asm/vio.h index 68d0cc998b1b..7c444150c5ad 100644 --- a/arch/powerpc/include/asm/vio.h +++ b/arch/powerpc/include/asm/vio.h @@ -1,28 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * IBM PowerPC Virtual I/O Infrastructure Support. * * Copyright (c) 2003 IBM Corp. * Dave Engebretsen engebret@us.ibm.com * Santiago Leon santil@us.ibm.com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef _ASM_POWERPC_VIO_H #define _ASM_POWERPC_VIO_H #ifdef __KERNEL__ -#include <linux/init.h> #include <linux/errno.h> #include <linux/device.h> #include <linux/dma-mapping.h> #include <linux/mod_devicetable.h> +#include <linux/scatterlist.h> #include <asm/hvcall.h> -#include <asm/scatterlist.h> /* * Architecture-specific constants for drivers to @@ -44,7 +39,7 @@ */ #define VIO_CMO_MIN_ENT 1562624 -extern struct bus_type vio_bus_type; +extern const struct bus_type vio_bus_type; struct iommu_table; @@ -118,7 +113,8 @@ struct vio_driver { const char *name; const struct vio_device_id *id_table; int (*probe)(struct vio_dev *dev, const struct vio_device_id *id); - int (*remove)(struct vio_dev *dev); + void (*remove)(struct vio_dev *dev); + void (*shutdown)(struct vio_dev *dev); /* A driver must have a get_desired_dma() function to * be loaded in a CMO environment if it uses DMA. */ @@ -160,15 +156,8 @@ static inline int vio_enable_interrupts(struct vio_dev *dev) } #endif -static inline struct vio_driver *to_vio_driver(struct device_driver *drv) -{ - return container_of(drv, struct vio_driver, driver); -} - -static inline struct vio_dev *to_vio_dev(struct device *dev) -{ - return container_of(dev, struct vio_dev, dev); -} +#define to_vio_driver(__drv) container_of_const(__drv, struct vio_driver, driver) +#define to_vio_dev(__dev) container_of_const(__dev, struct vio_dev, dev) #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_VIO_H */ diff --git a/arch/powerpc/include/asm/vmalloc.h b/arch/powerpc/include/asm/vmalloc.h new file mode 100644 index 000000000000..59ed89890c90 --- /dev/null +++ b/arch/powerpc/include/asm/vmalloc.h @@ -0,0 +1,24 @@ +#ifndef _ASM_POWERPC_VMALLOC_H +#define _ASM_POWERPC_VMALLOC_H + +#include <asm/mmu.h> +#include <asm/page.h> + +#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP + +#define arch_vmap_pud_supported arch_vmap_pud_supported +static __always_inline bool arch_vmap_pud_supported(pgprot_t prot) +{ + /* HPT does not cope with large pages in the vmalloc area */ + return radix_enabled(); +} + +#define arch_vmap_pmd_supported arch_vmap_pmd_supported +static __always_inline bool arch_vmap_pmd_supported(pgprot_t prot) +{ + return radix_enabled(); +} + +#endif + +#endif /* _ASM_POWERPC_VMALLOC_H */ diff --git a/arch/powerpc/include/asm/vphn.h b/arch/powerpc/include/asm/vphn.h new file mode 100644 index 000000000000..8c2f795eea68 --- /dev/null +++ b/arch/powerpc/include/asm/vphn.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_POWERPC_VPHN_H +#define _ASM_POWERPC_VPHN_H + +/* The H_HOME_NODE_ASSOCIATIVITY h_call returns 6 64-bit registers. */ +#define VPHN_REGISTER_COUNT 6 + +/* + * 6 64-bit registers unpacked into up to 24 be32 associativity values. To + * form the complete property we have to add the length in the first cell. + */ +#define VPHN_ASSOC_BUFSIZE (VPHN_REGISTER_COUNT*sizeof(u64)/sizeof(u16) + 1) + +/* + * The H_HOME_NODE_ASSOCIATIVITY hcall takes two values for flags: + * 1 for retrieving associativity information for a guest cpu + * 2 for retrieving associativity information for a host/hypervisor cpu + */ +#define VPHN_FLAG_VCPU 1 +#define VPHN_FLAG_PCPU 2 + +long hcall_vphn(unsigned long cpu, u64 flags, __be32 *associativity); + +#endif // _ASM_POWERPC_VPHN_H diff --git a/arch/powerpc/include/asm/word-at-a-time.h b/arch/powerpc/include/asm/word-at-a-time.h index d0b6d4ac6dda..54653a863414 100644 --- a/arch/powerpc/include/asm/word-at-a-time.h +++ b/arch/powerpc/include/asm/word-at-a-time.h @@ -4,9 +4,12 @@ /* * Word-at-a-time interfaces for PowerPC. */ - -#include <linux/kernel.h> +#include <linux/bitops.h> +#include <linux/wordpart.h> #include <asm/asm-compat.h> +#include <asm/extable.h> + +#ifdef __BIG_ENDIAN__ struct word_at_a_time { const unsigned long high_bits, low_bits; @@ -31,11 +34,173 @@ static inline long find_zero(unsigned long mask) return leading_zero_bits >> 3; } -static inline bool has_zero(unsigned long val, unsigned long *data, const struct word_at_a_time *c) +static inline unsigned long has_zero(unsigned long val, unsigned long *data, const struct word_at_a_time *c) { unsigned long rhs = val | c->low_bits; *data = rhs; return (val + c->high_bits) & ~rhs; } +static inline unsigned long zero_bytemask(unsigned long mask) +{ + return ~1ul << __fls(mask); +} + +#else + +#ifdef CONFIG_64BIT + +/* unused */ +struct word_at_a_time { +}; + +#define WORD_AT_A_TIME_CONSTANTS { } + +/* This will give us 0xff for a NULL char and 0x00 elsewhere */ +static inline unsigned long has_zero(unsigned long a, unsigned long *bits, const struct word_at_a_time *c) +{ + unsigned long ret; + unsigned long zero = 0; + + asm("cmpb %0,%1,%2" : "=r" (ret) : "r" (a), "r" (zero)); + *bits = ret; + + return ret; +} + +static inline unsigned long prep_zero_mask(unsigned long a, unsigned long bits, const struct word_at_a_time *c) +{ + return bits; +} + +/* Alan Modra's little-endian strlen tail for 64-bit */ +static inline unsigned long create_zero_mask(unsigned long bits) +{ + unsigned long leading_zero_bits; + long trailing_zero_bit_mask; + + asm("addi %1,%2,-1\n\t" + "andc %1,%1,%2\n\t" + "popcntd %0,%1" + : "=r" (leading_zero_bits), "=&r" (trailing_zero_bit_mask) + : "b" (bits)); + + return leading_zero_bits; +} + +static inline unsigned long find_zero(unsigned long mask) +{ + return mask >> 3; +} + +/* This assumes that we never ask for an all 1s bitmask */ +static inline unsigned long zero_bytemask(unsigned long mask) +{ + return (1UL << mask) - 1; +} + +#else /* 32-bit case */ + +struct word_at_a_time { + const unsigned long one_bits, high_bits; +}; + +#define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x01), REPEAT_BYTE(0x80) } + +/* + * This is largely generic for little-endian machines, but the + * optimal byte mask counting is probably going to be something + * that is architecture-specific. If you have a reliably fast + * bit count instruction, that might be better than the multiply + * and shift, for example. + */ + +/* Carl Chatfield / Jan Achrenius G+ version for 32-bit */ +static inline long count_masked_bytes(long mask) +{ + /* (000000 0000ff 00ffff ffffff) -> ( 1 1 2 3 ) */ + long a = (0x0ff0001+mask) >> 23; + /* Fix the 1 for 00 case */ + return a & mask; +} + +static inline unsigned long create_zero_mask(unsigned long bits) +{ + bits = (bits - 1) & ~bits; + return bits >> 7; +} + +static inline unsigned long find_zero(unsigned long mask) +{ + return count_masked_bytes(mask); +} + +/* Return nonzero if it has a zero */ +static inline unsigned long has_zero(unsigned long a, unsigned long *bits, const struct word_at_a_time *c) +{ + unsigned long mask = ((a - c->one_bits) & ~a) & c->high_bits; + *bits = mask; + return mask; +} + +static inline unsigned long prep_zero_mask(unsigned long a, unsigned long bits, const struct word_at_a_time *c) +{ + return bits; +} + +/* The mask we created is directly usable as a bytemask */ +#define zero_bytemask(mask) (mask) + +#endif /* CONFIG_64BIT */ + +#endif /* __BIG_ENDIAN__ */ + +/* + * We use load_unaligned_zero() in a selftest, which builds a userspace + * program. Some linker scripts seem to discard the .fixup section, so allow + * the test code to use a different section name. + */ +#ifndef FIXUP_SECTION +#define FIXUP_SECTION ".fixup" +#endif + +static inline unsigned long load_unaligned_zeropad(const void *addr) +{ + unsigned long ret, offset, tmp; + + asm( + "1: " PPC_LL "%[ret], 0(%[addr])\n" + "2:\n" + ".section " FIXUP_SECTION ",\"ax\"\n" + "3: " +#ifdef __powerpc64__ + "clrrdi %[tmp], %[addr], 3\n\t" + "clrlsldi %[offset], %[addr], 61, 3\n\t" + "ld %[ret], 0(%[tmp])\n\t" +#ifdef __BIG_ENDIAN__ + "sld %[ret], %[ret], %[offset]\n\t" +#else + "srd %[ret], %[ret], %[offset]\n\t" +#endif +#else + "clrrwi %[tmp], %[addr], 2\n\t" + "clrlslwi %[offset], %[addr], 30, 3\n\t" + "lwz %[ret], 0(%[tmp])\n\t" +#ifdef __BIG_ENDIAN__ + "slw %[ret], %[ret], %[offset]\n\t" +#else + "srw %[ret], %[ret], %[offset]\n\t" +#endif +#endif + "b 2b\n" + ".previous\n" + EX_TABLE(1b, 3b) + : [tmp] "=&b" (tmp), [offset] "=&r" (offset), [ret] "=&r" (ret) + : [addr] "b" (addr), "m" (*(unsigned long *)addr)); + + return ret; +} + +#undef FIXUP_SECTION + #endif /* _ASM_WORD_AT_A_TIME_H */ diff --git a/arch/powerpc/include/asm/wsp.h b/arch/powerpc/include/asm/wsp.h deleted file mode 100644 index c7dc83088a33..000000000000 --- a/arch/powerpc/include/asm/wsp.h +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright 2011 Michael Ellerman, IBM Corp. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ -#ifndef __ASM_POWERPC_WSP_H -#define __ASM_POWERPC_WSP_H - -extern int wsp_get_chip_id(struct device_node *dn); - -#endif /* __ASM_POWERPC_WSP_H */ diff --git a/arch/powerpc/include/asm/xics.h b/arch/powerpc/include/asm/xics.h index 282d43a0c855..60ef312dab05 100644 --- a/arch/powerpc/include/asm/xics.h +++ b/arch/powerpc/include/asm/xics.h @@ -1,5 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* - * Common definitions accross all variants of ICP and ICS interrupt + * Common definitions across all variants of ICP and ICS interrupt * controllers. */ @@ -29,17 +30,25 @@ /* Native ICP */ #ifdef CONFIG_PPC_ICP_NATIVE extern int icp_native_init(void); +extern void icp_native_flush_interrupt(void); #else static inline int icp_native_init(void) { return -ENODEV; } #endif /* PAPR ICP */ #ifdef CONFIG_PPC_ICP_HV -extern int icp_hv_init(void); +int __init icp_hv_init(void); #else static inline int icp_hv_init(void) { return -ENODEV; } #endif +#ifdef CONFIG_PPC_POWERNV +int __init icp_opal_init(void); +extern void icp_opal_flush_interrupt(void); +#else +static inline int icp_opal_init(void) { return -ENODEV; } +#endif + /* ICP ops */ struct icp_ops { unsigned int (*get_irq)(void); @@ -48,15 +57,19 @@ struct icp_ops { void (*teardown_cpu)(void); void (*flush_ipi)(void); #ifdef CONFIG_SMP - void (*cause_ipi)(int cpu, unsigned long data); + void (*cause_ipi)(int cpu); irq_handler_t ipi_action; #endif }; extern const struct icp_ops *icp_ops; +#ifdef CONFIG_PPC_ICS_NATIVE /* Native ICS */ extern int ics_native_init(void); +#else +static inline int ics_native_init(void) { return -ENODEV; } +#endif /* RTAS ICS */ #ifdef CONFIG_PPC_ICS_RTAS @@ -75,10 +88,11 @@ static inline int ics_opal_init(void) { return -ENODEV; } /* ICS instance, hooked up to chip_data of an irq */ struct ics { struct list_head link; - int (*map)(struct ics *ics, unsigned int virq); + int (*check)(struct ics *ics, unsigned int hwirq); void (*mask_unknown)(struct ics *ics, unsigned long vec); long (*get_server)(struct ics *ics, unsigned long vec); int (*host_match)(struct ics *ics, struct device_node *node); + struct irq_chip *chip; char data[]; }; @@ -97,7 +111,7 @@ DECLARE_PER_CPU(struct xics_cppr, xics_cppr); static inline void xics_push_cppr(unsigned int vec) { - struct xics_cppr *os_cppr = &__get_cpu_var(xics_cppr); + struct xics_cppr *os_cppr = this_cpu_ptr(&xics_cppr); if (WARN_ON(os_cppr->index >= MAX_NUM_PRIORITIES - 1)) return; @@ -110,7 +124,7 @@ static inline void xics_push_cppr(unsigned int vec) static inline unsigned char xics_pop_cppr(void) { - struct xics_cppr *os_cppr = &__get_cpu_var(xics_cppr); + struct xics_cppr *os_cppr = this_cpu_ptr(&xics_cppr); if (WARN_ON(os_cppr->index < 1)) return LOWEST_PRIORITY; @@ -120,7 +134,7 @@ static inline unsigned char xics_pop_cppr(void) static inline void xics_set_base_cppr(unsigned char cppr) { - struct xics_cppr *os_cppr = &__get_cpu_var(xics_cppr); + struct xics_cppr *os_cppr = this_cpu_ptr(&xics_cppr); /* we only really want to set the priority when there's * just one cppr value on the stack @@ -132,7 +146,7 @@ static inline void xics_set_base_cppr(unsigned char cppr) static inline unsigned char xics_cppr_top(void) { - struct xics_cppr *os_cppr = &__get_cpu_var(xics_cppr); + struct xics_cppr *os_cppr = this_cpu_ptr(&xics_cppr); return os_cppr->stack[os_cppr->index]; } @@ -144,13 +158,14 @@ extern void xics_setup_cpu(void); extern void xics_update_irq_servers(void); extern void xics_set_cpu_giq(unsigned int gserver, unsigned int join); extern void xics_mask_unknown_vec(unsigned int vec); -extern irqreturn_t xics_ipi_dispatch(int cpu); -extern int xics_smp_probe(void); +extern void xics_smp_probe(void); extern void xics_register_ics(struct ics *ics); extern void xics_teardown_cpu(void); extern void xics_kexec_teardown_cpu(int secondary); extern void xics_migrate_irqs_away(void); extern void icp_native_eoi(struct irq_data *d); +extern int xics_set_irq_type(struct irq_data *d, unsigned int flow_type); +extern int xics_retrigger(struct irq_data *data); #ifdef CONFIG_SMP extern int xics_get_irq_server(unsigned int virq, const struct cpumask *cpumask, unsigned int strict_check); diff --git a/arch/powerpc/include/asm/xilinx_intc.h b/arch/powerpc/include/asm/xilinx_intc.h deleted file mode 100644 index 343612f8fece..000000000000 --- a/arch/powerpc/include/asm/xilinx_intc.h +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Xilinx intc external definitions - * - * Copyright 2007 Secret Lab Technologies Ltd. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - */ -#ifndef _ASM_POWERPC_XILINX_INTC_H -#define _ASM_POWERPC_XILINX_INTC_H - -#ifdef __KERNEL__ - -extern void __init xilinx_intc_init_tree(void); -extern unsigned int xilinx_intc_get_irq(void); - -#endif /* __KERNEL__ */ -#endif /* _ASM_POWERPC_XILINX_INTC_H */ diff --git a/arch/powerpc/include/asm/xilinx_pci.h b/arch/powerpc/include/asm/xilinx_pci.h deleted file mode 100644 index 7a8275caf6af..000000000000 --- a/arch/powerpc/include/asm/xilinx_pci.h +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Xilinx pci external definitions - * - * Copyright 2009 Roderick Colenbrander - * Copyright 2009 Secret Lab Technologies Ltd. - * - * This file is licensed under the terms of the GNU General Public License - * version 2. This program is licensed "as is" without any warranty of any - * kind, whether express or implied. - */ - -#ifndef INCLUDE_XILINX_PCI -#define INCLUDE_XILINX_PCI - -#ifdef CONFIG_XILINX_PCI -extern void __init xilinx_pci_init(void); -#else -static inline void __init xilinx_pci_init(void) { return; } -#endif - -#endif /* INCLUDE_XILINX_PCI */ diff --git a/arch/powerpc/include/asm/xive-regs.h b/arch/powerpc/include/asm/xive-regs.h new file mode 100644 index 000000000000..cf8bb6ac4463 --- /dev/null +++ b/arch/powerpc/include/asm/xive-regs.h @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright 2016,2017 IBM Corporation. + */ +#ifndef _ASM_POWERPC_XIVE_REGS_H +#define _ASM_POWERPC_XIVE_REGS_H + +/* + * "magic" Event State Buffer (ESB) MMIO offsets. + * + * Each interrupt source has a 2-bit state machine called ESB + * which can be controlled by MMIO. It's made of 2 bits, P and + * Q. P indicates that an interrupt is pending (has been sent + * to a queue and is waiting for an EOI). Q indicates that the + * interrupt has been triggered while pending. + * + * This acts as a coalescing mechanism in order to guarantee + * that a given interrupt only occurs at most once in a queue. + * + * When doing an EOI, the Q bit will indicate if the interrupt + * needs to be re-triggered. + * + * The following offsets into the ESB MMIO allow to read or + * manipulate the PQ bits. They must be used with an 8-bytes + * load instruction. They all return the previous state of the + * interrupt (atomically). + * + * Additionally, some ESB pages support doing an EOI via a + * store at 0 and some ESBs support doing a trigger via a + * separate trigger page. + */ +#define XIVE_ESB_STORE_EOI 0x400 /* Store */ +#define XIVE_ESB_LOAD_EOI 0x000 /* Load */ +#define XIVE_ESB_GET 0x800 /* Load */ +#define XIVE_ESB_SET_PQ_00 0xc00 /* Load */ +#define XIVE_ESB_SET_PQ_01 0xd00 /* Load */ +#define XIVE_ESB_SET_PQ_10 0xe00 /* Load */ +#define XIVE_ESB_SET_PQ_11 0xf00 /* Load */ + +/* + * Load-after-store ordering + * + * Adding this offset to the load address will enforce + * load-after-store ordering. This is required to use StoreEOI. + */ +#define XIVE_ESB_LD_ST_MO 0x40 /* Load-after-store ordering */ + +#define XIVE_ESB_VAL_P 0x2 +#define XIVE_ESB_VAL_Q 0x1 +#define XIVE_ESB_INVALID 0xFF + +/* + * Thread Management (aka "TM") registers + */ + +/* TM register offsets */ +#define TM_QW0_USER 0x000 /* All rings */ +#define TM_QW1_OS 0x010 /* Ring 0..2 */ +#define TM_QW2_HV_POOL 0x020 /* Ring 0..1 */ +#define TM_QW3_HV_PHYS 0x030 /* Ring 0..1 */ + +/* Byte offsets inside a QW QW0 QW1 QW2 QW3 */ +#define TM_NSR 0x0 /* + + - + */ +#define TM_CPPR 0x1 /* - + - + */ +#define TM_IPB 0x2 /* - + + + */ +#define TM_LSMFB 0x3 /* - + + + */ +#define TM_ACK_CNT 0x4 /* - + - - */ +#define TM_INC 0x5 /* - + - + */ +#define TM_AGE 0x6 /* - + - + */ +#define TM_PIPR 0x7 /* - + - + */ + +#define TM_WORD0 0x0 +#define TM_WORD1 0x4 + +/* + * QW word 2 contains the valid bit at the top and other fields + * depending on the QW. + */ +#define TM_WORD2 0x8 +#define TM_QW0W2_VU PPC_BIT32(0) +#define TM_QW0W2_LOGIC_SERV PPC_BITMASK32(1,31) // XX 2,31 ? +#define TM_QW1W2_VO PPC_BIT32(0) +#define TM_QW1W2_HO PPC_BIT32(1) /* P10 XIVE2 */ +#define TM_QW1W2_OS_CAM PPC_BITMASK32(8,31) +#define TM_QW2W2_VP PPC_BIT32(0) +#define TM_QW2W2_HP PPC_BIT32(1) /* P10 XIVE2 */ +#define TM_QW2W2_POOL_CAM PPC_BITMASK32(8,31) +#define TM_QW3W2_VT PPC_BIT32(0) +#define TM_QW3W2_HT PPC_BIT32(1) /* P10 XIVE2 */ +#define TM_QW3W2_LP PPC_BIT32(6) +#define TM_QW3W2_LE PPC_BIT32(7) +#define TM_QW3W2_T PPC_BIT32(31) + +/* + * In addition to normal loads to "peek" and writes (only when invalid) + * using 4 and 8 bytes accesses, the above registers support these + * "special" byte operations: + * + * - Byte load from QW0[NSR] - User level NSR (EBB) + * - Byte store to QW0[NSR] - User level NSR (EBB) + * - Byte load/store to QW1[CPPR] and QW3[CPPR] - CPPR access + * - Byte load from QW3[TM_WORD2] - Read VT||00000||LP||LE on thrd 0 + * otherwise VT||0000000 + * - Byte store to QW3[TM_WORD2] - Set VT bit (and LP/LE if present) + * + * Then we have all these "special" CI ops at these offset that trigger + * all sorts of side effects: + */ +#define TM_SPC_ACK_EBB 0x800 /* Load8 ack EBB to reg*/ +#define TM_SPC_ACK_OS_REG 0x810 /* Load16 ack OS irq to reg */ +#define TM_SPC_PUSH_USR_CTX 0x808 /* Store32 Push/Validate user context */ +#define TM_SPC_PULL_USR_CTX 0x808 /* Load32 Pull/Invalidate user context */ +#define TM_SPC_SET_OS_PENDING 0x812 /* Store8 Set OS irq pending bit */ +#define TM_SPC_PULL_OS_CTX 0x818 /* Load32/Load64 Pull/Invalidate OS context to reg */ +#define TM_SPC_PULL_POOL_CTX 0x828 /* Load32/Load64 Pull/Invalidate Pool context to reg*/ +#define TM_SPC_ACK_HV_REG 0x830 /* Load16 ack HV irq to reg */ +#define TM_SPC_PULL_USR_CTX_OL 0xc08 /* Store8 Pull/Inval usr ctx to odd line */ +#define TM_SPC_ACK_OS_EL 0xc10 /* Store8 ack OS irq to even line */ +#define TM_SPC_ACK_HV_POOL_EL 0xc20 /* Store8 ack HV evt pool to even line */ +#define TM_SPC_ACK_HV_EL 0xc30 /* Store8 ack HV irq to even line */ +/* XXX more... */ + +/* NSR fields for the various QW ack types */ +#define TM_QW0_NSR_EB PPC_BIT8(0) +#define TM_QW1_NSR_EO PPC_BIT8(0) +#define TM_QW3_NSR_HE PPC_BITMASK8(0,1) +#define TM_QW3_NSR_HE_NONE 0 +#define TM_QW3_NSR_HE_POOL 1 +#define TM_QW3_NSR_HE_PHYS 2 +#define TM_QW3_NSR_HE_LSI 3 +#define TM_QW3_NSR_I PPC_BIT8(2) +#define TM_QW3_NSR_GRP_LVL PPC_BIT8(3,7) + +#endif /* _ASM_POWERPC_XIVE_REGS_H */ diff --git a/arch/powerpc/include/asm/xive.h b/arch/powerpc/include/asm/xive.h new file mode 100644 index 000000000000..efb0f5effcc6 --- /dev/null +++ b/arch/powerpc/include/asm/xive.h @@ -0,0 +1,168 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright 2016,2017 IBM Corporation. + */ +#ifndef _ASM_POWERPC_XIVE_H +#define _ASM_POWERPC_XIVE_H + +#include <asm/opal-api.h> + +#define XIVE_INVALID_VP 0xffffffff + +#ifdef CONFIG_PPC_XIVE + +/* + * Thread Interrupt Management Area (TIMA) + * + * This is a global MMIO region divided in 4 pages of varying access + * permissions, providing access to per-cpu interrupt management + * functions. It always identifies the CPU doing the access based + * on the PowerBus initiator ID, thus we always access via the + * same offset regardless of where the code is executing + */ +extern void __iomem *xive_tima; +extern unsigned long xive_tima_os; + +/* + * Offset in the TM area of our current execution level (provided by + * the backend) + */ +extern u32 xive_tima_offset; + +/* + * Per-irq data (irq_get_handler_data for normal IRQs), IPIs + * have it stored in the xive_cpu structure. We also cache + * for normal interrupts the current target CPU. + * + * This structure is setup by the backend for each interrupt. + */ +struct xive_irq_data { + u64 flags; + u64 eoi_page; + void __iomem *eoi_mmio; + u64 trig_page; + void __iomem *trig_mmio; + u32 esb_shift; + int src_chip; + u32 hw_irq; + + /* Setup/used by frontend */ + int target; + /* + * saved_p means that there is a queue entry for this interrupt + * in some CPU's queue (not including guest vcpu queues), even + * if P is not set in the source ESB. + * stale_p means that there is no queue entry for this interrupt + * in some CPU's queue, even if P is set in the source ESB. + */ + bool saved_p; + bool stale_p; +}; +#define XIVE_IRQ_FLAG_STORE_EOI 0x01 +#define XIVE_IRQ_FLAG_LSI 0x02 +/* #define XIVE_IRQ_FLAG_SHIFT_BUG 0x04 */ /* P9 DD1.0 workaround */ +/* #define XIVE_IRQ_FLAG_MASK_FW 0x08 */ /* P9 DD1.0 workaround */ +/* #define XIVE_IRQ_FLAG_EOI_FW 0x10 */ /* P9 DD1.0 workaround */ +#define XIVE_IRQ_FLAG_H_INT_ESB 0x20 + +/* Special flag set by KVM for excalation interrupts */ +#define XIVE_IRQ_FLAG_NO_EOI 0x80 + +#define XIVE_INVALID_CHIP_ID -1 + +/* A queue tracking structure in a CPU */ +struct xive_q { + __be32 *qpage; + u32 msk; + u32 idx; + u32 toggle; + u64 eoi_phys; + u32 esc_irq; + atomic_t count; + atomic_t pending_count; + u64 guest_qaddr; + u32 guest_qshift; +}; + +/* Global enable flags for the XIVE support */ +extern bool __xive_enabled; + +static inline bool xive_enabled(void) { return __xive_enabled; } + +bool xive_spapr_init(void); +bool xive_native_init(void); +void xive_smp_probe(void); +int xive_smp_prepare_cpu(unsigned int cpu); +void xive_smp_setup_cpu(void); +void xive_smp_disable_cpu(void); +void xive_teardown_cpu(void); +void xive_shutdown(void); +void xive_flush_interrupt(void); + +/* xmon hook */ +void xmon_xive_do_dump(int cpu); +int xmon_xive_get_irq_config(u32 hw_irq, struct irq_data *d); +void xmon_xive_get_irq_all(void); + +/* APIs used by KVM */ +u32 xive_native_default_eq_shift(void); +u32 xive_native_alloc_vp_block(u32 max_vcpus); +void xive_native_free_vp_block(u32 vp_base); +int xive_native_populate_irq_data(u32 hw_irq, + struct xive_irq_data *data); +void xive_cleanup_irq_data(struct xive_irq_data *xd); +void xive_native_free_irq(u32 irq); +int xive_native_configure_irq(u32 hw_irq, u32 target, u8 prio, u32 sw_irq); + +int xive_native_configure_queue(u32 vp_id, struct xive_q *q, u8 prio, + __be32 *qpage, u32 order, bool can_escalate); +void xive_native_disable_queue(u32 vp_id, struct xive_q *q, u8 prio); + +void xive_native_sync_source(u32 hw_irq); +void xive_native_sync_queue(u32 hw_irq); +bool is_xive_irq(struct irq_chip *chip); +int xive_native_enable_vp(u32 vp_id, bool single_escalation); +int xive_native_disable_vp(u32 vp_id); +int xive_native_get_vp_info(u32 vp_id, u32 *out_cam_id, u32 *out_chip_id); +bool xive_native_has_single_escalation(void); +bool xive_native_has_save_restore(void); + +int xive_native_get_queue_info(u32 vp_id, uint32_t prio, + u64 *out_qpage, + u64 *out_qsize, + u64 *out_qeoi_page, + u32 *out_escalate_irq, + u64 *out_qflags); + +int xive_native_get_queue_state(u32 vp_id, uint32_t prio, u32 *qtoggle, + u32 *qindex); +int xive_native_set_queue_state(u32 vp_id, uint32_t prio, u32 qtoggle, + u32 qindex); +int xive_native_get_vp_state(u32 vp_id, u64 *out_state); +bool xive_native_has_queue_state_support(void); +extern u32 xive_native_alloc_irq_on_chip(u32 chip_id); + +static inline u32 xive_native_alloc_irq(void) +{ + return xive_native_alloc_irq_on_chip(OPAL_XIVE_ANY_CHIP); +} + +#else + +static inline bool xive_enabled(void) { return false; } + +static inline bool xive_spapr_init(void) { return false; } +static inline bool xive_native_init(void) { return false; } +static inline void xive_smp_probe(void) { } +static inline int xive_smp_prepare_cpu(unsigned int cpu) { return -EINVAL; } +static inline void xive_smp_setup_cpu(void) { } +static inline void xive_smp_disable_cpu(void) { } +static inline void xive_shutdown(void) { } +static inline void xive_flush_interrupt(void) { } + +static inline u32 xive_native_alloc_vp_block(u32 max_vcpus) { return XIVE_INVALID_VP; } +static inline void xive_native_free_vp_block(u32 vp_base) { } + +#endif + +#endif /* _ASM_POWERPC_XIVE_H */ diff --git a/arch/powerpc/include/asm/xmon.h b/arch/powerpc/include/asm/xmon.h index 5eb8e599e5cc..535cdb1e411a 100644 --- a/arch/powerpc/include/asm/xmon.h +++ b/arch/powerpc/include/asm/xmon.h @@ -1,13 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef __ASM_POWERPC_XMON_H #define __ASM_POWERPC_XMON_H /* * Copyrignt (C) 2006 IBM Corp - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifdef __KERNEL__ @@ -16,18 +12,18 @@ #ifdef CONFIG_XMON extern void xmon_setup(void); -extern void xmon_register_spus(struct list_head *list); struct pt_regs; extern int xmon(struct pt_regs *excp); extern irqreturn_t xmon_irq(int, void *); #else -static inline void xmon_setup(void) { }; -static inline void xmon_register_spus(struct list_head *list) { }; +static inline void xmon_setup(void) { } #endif #if defined(CONFIG_XMON) && defined(CONFIG_SMP) extern int cpus_are_in_xmon(void); #endif +extern __printf(1, 2) void xmon_printf(const char *format, ...); + #endif /* __KERNEL __ */ #endif /* __ASM_POWERPC_XMON_H */ diff --git a/arch/powerpc/include/asm/xor.h b/arch/powerpc/include/asm/xor.h index c82eb12a5b18..37d05c11d09c 100644 --- a/arch/powerpc/include/asm/xor.h +++ b/arch/powerpc/include/asm/xor.h @@ -1 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * + * Copyright (C) IBM Corporation, 2012 + * + * Author: Anton Blanchard <anton@au.ibm.com> + */ +#ifndef _ASM_POWERPC_XOR_H +#define _ASM_POWERPC_XOR_H + +#ifdef CONFIG_ALTIVEC + +#include <asm/cputable.h> +#include <asm/cpu_has_feature.h> +#include <asm/xor_altivec.h> + +static struct xor_block_template xor_block_altivec = { + .name = "altivec", + .do_2 = xor_altivec_2, + .do_3 = xor_altivec_3, + .do_4 = xor_altivec_4, + .do_5 = xor_altivec_5, +}; + +#define XOR_SPEED_ALTIVEC() \ + do { \ + if (cpu_has_feature(CPU_FTR_ALTIVEC)) \ + xor_speed(&xor_block_altivec); \ + } while (0) +#else +#define XOR_SPEED_ALTIVEC() +#endif + +/* Also try the generic routines. */ #include <asm-generic/xor.h> + +#undef XOR_TRY_TEMPLATES +#define XOR_TRY_TEMPLATES \ +do { \ + xor_speed(&xor_block_8regs); \ + xor_speed(&xor_block_8regs_p); \ + xor_speed(&xor_block_32regs); \ + xor_speed(&xor_block_32regs_p); \ + XOR_SPEED_ALTIVEC(); \ +} while (0) + +#endif /* _ASM_POWERPC_XOR_H */ diff --git a/arch/powerpc/include/asm/xor_altivec.h b/arch/powerpc/include/asm/xor_altivec.h new file mode 100644 index 000000000000..294620a25f80 --- /dev/null +++ b/arch/powerpc/include/asm/xor_altivec.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_XOR_ALTIVEC_H +#define _ASM_POWERPC_XOR_ALTIVEC_H + +#ifdef CONFIG_ALTIVEC +void xor_altivec_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2); +void xor_altivec_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3); +void xor_altivec_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4); +void xor_altivec_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5); + +#endif +#endif /* _ASM_POWERPC_XOR_ALTIVEC_H */ diff --git a/arch/powerpc/include/uapi/asm/Kbuild b/arch/powerpc/include/uapi/asm/Kbuild index 5182c8622b54..353b70b1998f 100644 --- a/arch/powerpc/include/uapi/asm/Kbuild +++ b/arch/powerpc/include/uapi/asm/Kbuild @@ -1,46 +1,3 @@ -# UAPI Header export list -include include/uapi/asm-generic/Kbuild.asm - -header-y += auxvec.h -header-y += bitsperlong.h -header-y += bootx.h -header-y += byteorder.h -header-y += cputable.h -header-y += elf.h -header-y += epapr_hcalls.h -header-y += errno.h -header-y += fcntl.h -header-y += ioctl.h -header-y += ioctls.h -header-y += ipcbuf.h -header-y += kvm.h -header-y += kvm_para.h -header-y += linkage.h -header-y += mman.h -header-y += msgbuf.h -header-y += nvram.h -header-y += param.h -header-y += poll.h -header-y += posix_types.h -header-y += ps3fb.h -header-y += ptrace.h -header-y += resource.h -header-y += seccomp.h -header-y += sembuf.h -header-y += setup.h -header-y += shmbuf.h -header-y += sigcontext.h -header-y += siginfo.h -header-y += signal.h -header-y += socket.h -header-y += sockios.h -header-y += spu_info.h -header-y += stat.h -header-y += statfs.h -header-y += swab.h -header-y += termbits.h -header-y += termios.h -header-y += tm.h -header-y += types.h -header-y += ucontext.h -header-y += unistd.h +# SPDX-License-Identifier: GPL-2.0 +generated-y += unistd_32.h +generated-y += unistd_64.h diff --git a/arch/powerpc/include/uapi/asm/auxvec.h b/arch/powerpc/include/uapi/asm/auxvec.h index ce17d2c9eb4e..aa7c16215453 100644 --- a/arch/powerpc/include/uapi/asm/auxvec.h +++ b/arch/powerpc/include/uapi/asm/auxvec.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _ASM_POWERPC_AUXVEC_H #define _ASM_POWERPC_AUXVEC_H @@ -16,6 +17,39 @@ */ #define AT_SYSINFO_EHDR 33 -#define AT_VECTOR_SIZE_ARCH 6 /* entries in ARCH_DLINFO */ +/* + * AT_*CACHEBSIZE above represent the cache *block* size which is + * the size that is affected by the cache management instructions. + * + * It doesn't nececssarily matches the cache *line* size which is + * more of a performance tuning hint. Additionally the latter can + * be different for the different cache levels. + * + * The set of entries below represent more extensive information + * about the caches, in the form of two entry per cache type, + * one entry containing the cache size in bytes, and the other + * containing the cache line size in bytes in the bottom 16 bits + * and the cache associativity in the next 16 bits. + * + * The associativity is such that if N is the 16-bit value, the + * cache is N way set associative. A value if 0xffff means fully + * associative, a value of 1 means directly mapped. + * + * For all these fields, a value of 0 means that the information + * is not known. + */ + +#define AT_L1I_CACHESIZE 40 +#define AT_L1I_CACHEGEOMETRY 41 +#define AT_L1D_CACHESIZE 42 +#define AT_L1D_CACHEGEOMETRY 43 +#define AT_L2_CACHESIZE 44 +#define AT_L2_CACHEGEOMETRY 45 +#define AT_L3_CACHESIZE 46 +#define AT_L3_CACHEGEOMETRY 47 + +#define AT_MINSIGSTKSZ 51 /* stack needed for signal delivery */ + +#define AT_VECTOR_SIZE_ARCH 15 /* entries in ARCH_DLINFO */ #endif diff --git a/arch/powerpc/include/uapi/asm/bitsperlong.h b/arch/powerpc/include/uapi/asm/bitsperlong.h index 5f1659032c40..46ece3ecff31 100644 --- a/arch/powerpc/include/uapi/asm/bitsperlong.h +++ b/arch/powerpc/include/uapi/asm/bitsperlong.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef __ASM_POWERPC_BITSPERLONG_H #define __ASM_POWERPC_BITSPERLONG_H diff --git a/arch/powerpc/include/uapi/asm/bootx.h b/arch/powerpc/include/uapi/asm/bootx.h index 6e51cf0708a1..1b8c121071d9 100644 --- a/arch/powerpc/include/uapi/asm/bootx.h +++ b/arch/powerpc/include/uapi/asm/bootx.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * This file describes the structure passed from the BootX application * (for MacOS) when it is used to boot Linux. @@ -107,7 +108,7 @@ typedef struct boot_infos /* ALL BELOW NEW (vers. 4) */ /* This defines the physical memory. Valid with BOOT_ARCH_NUBUS flag - (non-PCI) only. On PCI, memory is contiguous and it's size is in the + (non-PCI) only. On PCI, memory is contiguous and its size is in the device-tree. */ boot_info_map_entry_t physMemoryMap[MAX_MEM_MAP_SIZE]; /* Where the phys memory is */ diff --git a/arch/powerpc/include/uapi/asm/byteorder.h b/arch/powerpc/include/uapi/asm/byteorder.h index aa6cc4fac965..8ef66f7d9db9 100644 --- a/arch/powerpc/include/uapi/asm/byteorder.h +++ b/arch/powerpc/include/uapi/asm/byteorder.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ #ifndef _ASM_POWERPC_BYTEORDER_H #define _ASM_POWERPC_BYTEORDER_H @@ -7,6 +8,10 @@ * as published by the Free Software Foundation; either version * 2 of the License, or (at your option) any later version. */ +#ifdef __LITTLE_ENDIAN__ +#include <linux/byteorder/little_endian.h> +#else #include <linux/byteorder/big_endian.h> +#endif #endif /* _ASM_POWERPC_BYTEORDER_H */ diff --git a/arch/powerpc/include/uapi/asm/cputable.h b/arch/powerpc/include/uapi/asm/cputable.h index 5b7657959faa..731b97dc2d15 100644 --- a/arch/powerpc/include/uapi/asm/cputable.h +++ b/arch/powerpc/include/uapi/asm/cputable.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _UAPI__ASM_POWERPC_CPUTABLE_H #define _UAPI__ASM_POWERPC_CPUTABLE_H @@ -31,6 +32,7 @@ #define PPC_FEATURE_PSERIES_PERFMON_COMPAT \ 0x00000040 +/* Reserved - do not use 0x00000004 */ #define PPC_FEATURE_TRUE_LE 0x00000002 #define PPC_FEATURE_PPC_LE 0x00000001 @@ -41,5 +43,21 @@ #define PPC_FEATURE2_EBB 0x10000000 #define PPC_FEATURE2_ISEL 0x08000000 #define PPC_FEATURE2_TAR 0x04000000 +#define PPC_FEATURE2_VEC_CRYPTO 0x02000000 +#define PPC_FEATURE2_HTM_NOSC 0x01000000 +#define PPC_FEATURE2_ARCH_3_00 0x00800000 /* ISA 3.00 */ +#define PPC_FEATURE2_HAS_IEEE128 0x00400000 /* VSX IEEE Binary Float 128-bit */ +#define PPC_FEATURE2_DARN 0x00200000 /* darn random number insn */ +#define PPC_FEATURE2_SCV 0x00100000 /* scv syscall */ +#define PPC_FEATURE2_HTM_NO_SUSPEND 0x00080000 /* TM w/out suspended state */ +#define PPC_FEATURE2_ARCH_3_1 0x00040000 /* ISA 3.1 */ +#define PPC_FEATURE2_MMA 0x00020000 /* Matrix Multiply Assist */ + +/* + * IMPORTANT! + * All future PPC_FEATURE definitions should be allocated in cooperation with + * OPAL / skiboot firmware, in accordance with the ibm,powerpc-cpu-features + * device tree binding. + */ #endif /* _UAPI__ASM_POWERPC_CPUTABLE_H */ diff --git a/arch/powerpc/include/uapi/asm/eeh.h b/arch/powerpc/include/uapi/asm/eeh.h new file mode 100644 index 000000000000..3b5c47ff3fc4 --- /dev/null +++ b/arch/powerpc/include/uapi/asm/eeh.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Copyright IBM Corp. 2015 + * + * Authors: Gavin Shan <gwshan@linux.vnet.ibm.com> + */ + +#ifndef _ASM_POWERPC_EEH_H +#define _ASM_POWERPC_EEH_H + +/* PE states */ +#define EEH_PE_STATE_NORMAL 0 /* Normal state */ +#define EEH_PE_STATE_RESET 1 /* PE reset asserted */ +#define EEH_PE_STATE_STOPPED_IO_DMA 2 /* Frozen PE */ +#define EEH_PE_STATE_STOPPED_DMA 4 /* Stopped DMA only */ +#define EEH_PE_STATE_UNAVAIL 5 /* Unavailable */ + +/* EEH error types and functions */ +#define EEH_ERR_TYPE_32 0 /* 32-bits error */ +#define EEH_ERR_TYPE_64 1 /* 64-bits error */ +#define EEH_ERR_FUNC_MIN 0 +#define EEH_ERR_FUNC_LD_MEM_ADDR 0 /* Memory load */ +#define EEH_ERR_FUNC_LD_MEM_DATA 1 +#define EEH_ERR_FUNC_LD_IO_ADDR 2 /* IO load */ +#define EEH_ERR_FUNC_LD_IO_DATA 3 +#define EEH_ERR_FUNC_LD_CFG_ADDR 4 /* Config load */ +#define EEH_ERR_FUNC_LD_CFG_DATA 5 +#define EEH_ERR_FUNC_ST_MEM_ADDR 6 /* Memory store */ +#define EEH_ERR_FUNC_ST_MEM_DATA 7 +#define EEH_ERR_FUNC_ST_IO_ADDR 8 /* IO store */ +#define EEH_ERR_FUNC_ST_IO_DATA 9 +#define EEH_ERR_FUNC_ST_CFG_ADDR 10 /* Config store */ +#define EEH_ERR_FUNC_ST_CFG_DATA 11 +#define EEH_ERR_FUNC_DMA_RD_ADDR 12 /* DMA read */ +#define EEH_ERR_FUNC_DMA_RD_DATA 13 +#define EEH_ERR_FUNC_DMA_RD_MASTER 14 +#define EEH_ERR_FUNC_DMA_RD_TARGET 15 +#define EEH_ERR_FUNC_DMA_WR_ADDR 16 /* DMA write */ +#define EEH_ERR_FUNC_DMA_WR_DATA 17 +#define EEH_ERR_FUNC_DMA_WR_MASTER 18 +#define EEH_ERR_FUNC_DMA_WR_TARGET 19 +#define EEH_ERR_FUNC_MAX 19 + +#endif /* _ASM_POWERPC_EEH_H */ diff --git a/arch/powerpc/include/uapi/asm/elf.h b/arch/powerpc/include/uapi/asm/elf.h index 05b8d560cfba..a5377f494fa3 100644 --- a/arch/powerpc/include/uapi/asm/elf.h +++ b/arch/powerpc/include/uapi/asm/elf.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ /* * ELF register definitions.. * @@ -91,6 +92,14 @@ #define ELF_NGREG 48 /* includes nip, msr, lr, etc. */ #define ELF_NFPREG 33 /* includes fpscr */ +#define ELF_NVMX 34 /* includes all vector registers */ +#define ELF_NVSX 32 /* includes all VSX registers */ +#define ELF_NTMSPRREG 3 /* include tfhar, tfiar, texasr */ +#define ELF_NEBB 3 /* includes ebbrr, ebbhr, bescr */ +#define ELF_NPMU 5 /* includes siar, sdar, sier, mmcr2, mmcr0 */ +#define ELF_NPKEY 3 /* includes amr, iamr, uamor */ +#define ELF_NDEXCR 2 /* includes dexcr, hdexcr */ +#define ELF_NHASHKEYR 1 /* includes hashkeyr */ typedef unsigned long elf_greg_t64; typedef elf_greg_t64 elf_gregset_t64[ELF_NGREG]; @@ -107,26 +116,25 @@ typedef elf_gregset_t32 compat_elf_gregset_t; # define ELF_NVRREG 34 /* includes vscr & vrsave in split vectors */ # define ELF_NVSRHALFREG 32 /* Half the vsx registers */ # define ELF_GREG_TYPE elf_greg_t64 +# define ELF_ARCH EM_PPC64 +# define ELF_CLASS ELFCLASS64 +typedef elf_greg_t64 elf_greg_t; +typedef elf_gregset_t64 elf_gregset_t; #else # define ELF_NEVRREG 34 /* includes acc (as 2) */ # define ELF_NVRREG 33 /* includes vscr */ # define ELF_GREG_TYPE elf_greg_t32 # define ELF_ARCH EM_PPC # define ELF_CLASS ELFCLASS32 -# define ELF_DATA ELFDATA2MSB +typedef elf_greg_t32 elf_greg_t; +typedef elf_gregset_t32 elf_gregset_t; #endif /* __powerpc64__ */ -#ifndef ELF_ARCH -# define ELF_ARCH EM_PPC64 -# define ELF_CLASS ELFCLASS64 -# define ELF_DATA ELFDATA2MSB - typedef elf_greg_t64 elf_greg_t; - typedef elf_gregset_t64 elf_gregset_t; +#ifdef __BIG_ENDIAN__ +#define ELF_DATA ELFDATA2MSB #else - /* Assumption: ELF_ARCH == EM_PPC and ELF_CLASS == ELFCLASS32 */ - typedef elf_greg_t32 elf_greg_t; - typedef elf_gregset_t32 elf_gregset_t; -#endif /* ELF_ARCH */ +#define ELF_DATA ELFDATA2LSB +#endif /* Floating point registers */ typedef double elf_fpreg_t; @@ -158,29 +166,6 @@ typedef elf_vrreg_t elf_vrregset_t32[ELF_NVRREG32]; typedef elf_fpreg_t elf_vsrreghalf_t32[ELF_NVSRHALFREG]; #endif - -/* - * The requirements here are: - * - keep the final alignment of sp (sp & 0xf) - * - make sure the 32-bit value at the first 16 byte aligned position of - * AUXV is greater than 16 for glibc compatibility. - * AT_IGNOREPPC is used for that. - * - for compatibility with glibc ARCH_DLINFO must always be defined on PPC, - * even if DLINFO_ARCH_ITEMS goes to zero or is undefined. - * update AT_VECTOR_SIZE_ARCH if the number of NEW_AUX_ENT entries changes - */ -#define ARCH_DLINFO \ -do { \ - /* Handle glibc compatibility. */ \ - NEW_AUX_ENT(AT_IGNOREPPC, AT_IGNOREPPC); \ - NEW_AUX_ENT(AT_IGNOREPPC, AT_IGNOREPPC); \ - /* Cache size items */ \ - NEW_AUX_ENT(AT_DCACHEBSIZE, dcache_bsize); \ - NEW_AUX_ENT(AT_ICACHEBSIZE, icache_bsize); \ - NEW_AUX_ENT(AT_UCACHEBSIZE, ucache_bsize); \ - VDSO_AUX_ENT(AT_SYSINFO_EHDR, current->mm->context.vdso_base); \ -} while (0) - /* PowerPC64 relocations defined by the ABIs */ #define R_PPC64_NONE R_PPC_NONE #define R_PPC64_ADDR32 R_PPC_ADDR32 /* 32bit absolute address. */ @@ -292,16 +277,22 @@ do { \ #define R_PPC64_DTPREL16_HIGHERA 104 /* half16 (sym+add)@dtprel@highera */ #define R_PPC64_DTPREL16_HIGHEST 105 /* half16 (sym+add)@dtprel@highest */ #define R_PPC64_DTPREL16_HIGHESTA 106 /* half16 (sym+add)@dtprel@highesta */ +#define R_PPC64_TLSGD 107 +#define R_PPC64_TLSLD 108 +#define R_PPC64_TOCSAVE 109 -/* Keep this the last entry. */ -#define R_PPC64_NUM 107 +#define R_PPC64_REL24_NOTOC 116 +#define R_PPC64_ENTRY 118 + +#define R_PPC64_PCREL34 132 +#define R_PPC64_GOT_PCREL34 133 -/* There's actually a third entry here, but it's unused */ -struct ppc64_opd_entry -{ - unsigned long funcaddr; - unsigned long r2; -}; +#define R_PPC64_REL16 249 +#define R_PPC64_REL16_LO 250 +#define R_PPC64_REL16_HI 251 +#define R_PPC64_REL16_HA 252 +/* Keep this the last entry. */ +#define R_PPC64_NUM 253 #endif /* _UAPI_ASM_POWERPC_ELF_H */ diff --git a/arch/powerpc/include/uapi/asm/epapr_hcalls.h b/arch/powerpc/include/uapi/asm/epapr_hcalls.h index 7f9c74b46704..90a0ee6d0bb3 100644 --- a/arch/powerpc/include/uapi/asm/epapr_hcalls.h +++ b/arch/powerpc/include/uapi/asm/epapr_hcalls.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: ((GPL-2.0+ WITH Linux-syscall-note) OR BSD-3-Clause) */ /* * ePAPR hcall interface * @@ -78,7 +79,7 @@ #define EV_SUCCESS 0 #define EV_EPERM 1 /* Operation not permitted */ #define EV_ENOENT 2 /* Entry Not Found */ -#define EV_EIO 3 /* I/O error occured */ +#define EV_EIO 3 /* I/O error occurred */ #define EV_EAGAIN 4 /* The operation had insufficient * resources to complete and should be * retried @@ -89,7 +90,7 @@ #define EV_ENODEV 7 /* No such device */ #define EV_EINVAL 8 /* An argument supplied to the hcall was out of range or invalid */ -#define EV_INTERNAL 9 /* An internal error occured */ +#define EV_INTERNAL 9 /* An internal error occurred */ #define EV_CONFIG 10 /* A configuration error was detected */ #define EV_INVALID_STATE 11 /* The object is in an invalid state */ #define EV_UNIMPLEMENTED 12 /* Unimplemented hypercall */ diff --git a/arch/powerpc/include/uapi/asm/errno.h b/arch/powerpc/include/uapi/asm/errno.h index 8c145fd17d86..4ba87de32be0 100644 --- a/arch/powerpc/include/uapi/asm/errno.h +++ b/arch/powerpc/include/uapi/asm/errno.h @@ -1,11 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _ASM_POWERPC_ERRNO_H #define _ASM_POWERPC_ERRNO_H +#undef EDEADLOCK #include <asm-generic/errno.h> #undef EDEADLOCK #define EDEADLOCK 58 /* File locking deadlock error */ -#define _LAST_ERRNO 516 - #endif /* _ASM_POWERPC_ERRNO_H */ diff --git a/arch/powerpc/include/uapi/asm/fcntl.h b/arch/powerpc/include/uapi/asm/fcntl.h index ce5c4516d404..65ce08322a89 100644 --- a/arch/powerpc/include/uapi/asm/fcntl.h +++ b/arch/powerpc/include/uapi/asm/fcntl.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _ASM_FCNTL_H #define _ASM_FCNTL_H diff --git a/arch/powerpc/include/uapi/asm/ioctl.h b/arch/powerpc/include/uapi/asm/ioctl.h index 57d68304218b..d623af4b9cd6 100644 --- a/arch/powerpc/include/uapi/asm/ioctl.h +++ b/arch/powerpc/include/uapi/asm/ioctl.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _ASM_POWERPC_IOCTL_H #define _ASM_POWERPC_IOCTL_H diff --git a/arch/powerpc/include/uapi/asm/ioctls.h b/arch/powerpc/include/uapi/asm/ioctls.h index 49a25796a61a..b5211e413829 100644 --- a/arch/powerpc/include/uapi/asm/ioctls.h +++ b/arch/powerpc/include/uapi/asm/ioctls.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _ASM_POWERPC_IOCTLS_H #define _ASM_POWERPC_IOCTLS_H @@ -22,10 +23,10 @@ #define TCSETSW _IOW('t', 21, struct termios) #define TCSETSF _IOW('t', 22, struct termios) -#define TCGETA _IOR('t', 23, struct termio) -#define TCSETA _IOW('t', 24, struct termio) -#define TCSETAW _IOW('t', 25, struct termio) -#define TCSETAF _IOW('t', 28, struct termio) +#define TCGETA 0x40147417 /* _IOR('t', 23, struct termio) */ +#define TCSETA 0x80147418 /* _IOW('t', 24, struct termio) */ +#define TCSETAW 0x80147419 /* _IOW('t', 25, struct termio) */ +#define TCSETAF 0x8014741c /* _IOW('t', 28, struct termio) */ #define TCSBRK _IO('t', 29) #define TCXONC _IO('t', 30) @@ -100,6 +101,9 @@ #define TIOCGPKT _IOR('T', 0x38, int) /* Get packet mode state */ #define TIOCGPTLCK _IOR('T', 0x39, int) /* Get Pty lock state */ #define TIOCGEXCL _IOR('T', 0x40, int) /* Get exclusive mode state */ +#define TIOCGPTPEER _IO('T', 0x41) /* Safely open the slave */ +#define TIOCGISO7816 _IOR('T', 0x42, struct serial_iso7816) +#define TIOCSISO7816 _IOWR('T', 0x43, struct serial_iso7816) #define TIOCSERCONFIG 0x5453 #define TIOCSERGWILD 0x5454 diff --git a/arch/powerpc/include/uapi/asm/ipcbuf.h b/arch/powerpc/include/uapi/asm/ipcbuf.h index 2c3e1d94db1d..21e1e0ec0ba2 100644 --- a/arch/powerpc/include/uapi/asm/ipcbuf.h +++ b/arch/powerpc/include/uapi/asm/ipcbuf.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ #ifndef _ASM_POWERPC_IPCBUF_H #define _ASM_POWERPC_IPCBUF_H diff --git a/arch/powerpc/include/uapi/asm/kvm.h b/arch/powerpc/include/uapi/asm/kvm.h index 0fb1a6e9ff90..077c5437f521 100644 --- a/arch/powerpc/include/uapi/asm/kvm.h +++ b/arch/powerpc/include/uapi/asm/kvm.h @@ -1,17 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * * Copyright IBM Corp. 2007 * * Authors: Hollis Blanchard <hollisb@us.ibm.com> @@ -28,6 +16,9 @@ #define __KVM_HAVE_IRQCHIP #define __KVM_HAVE_IRQ_LINE +/* Not always available, but if it is, this is the correct offset. */ +#define KVM_COALESCED_MMIO_PAGE_OFFSET 1 + struct kvm_regs { __u64 pc; __u64 cr; @@ -56,6 +47,12 @@ struct kvm_regs { #define KVM_SREGS_E_FSL_PIDn (1 << 0) /* PID1/PID2 */ +/* flags for kvm_run.flags */ +#define KVM_RUN_PPC_NMI_DISP_MASK (3 << 0) +#define KVM_RUN_PPC_NMI_DISP_FULLY_RECOV (1 << 0) +#define KVM_RUN_PPC_NMI_DISP_LIMITED_RECOV (2 << 0) +#define KVM_RUN_PPC_NMI_DISP_NOT_RECOV (3 << 0) + /* * Feature bits indicate which sections of the sregs struct are valid, * both in KVM_GET_SREGS and KVM_SET_SREGS. On KVM_SET_SREGS, registers @@ -269,7 +266,24 @@ struct kvm_fpu { __u64 fpr[32]; }; +/* + * Defines for h/w breakpoint, watchpoint (read, write or both) and + * software breakpoint. + * These are used as "type" in KVM_SET_GUEST_DEBUG ioctl and "status" + * for KVM_DEBUG_EXIT. + */ +#define KVMPPC_DEBUG_NONE 0x0 +#define KVMPPC_DEBUG_BREAKPOINT (1UL << 1) +#define KVMPPC_DEBUG_WATCH_WRITE (1UL << 2) +#define KVMPPC_DEBUG_WATCH_READ (1UL << 3) struct kvm_debug_exit_arch { + __u64 address; + /* + * exiting to userspace because of h/w breakpoint, watchpoint + * (read, write or both) and software breakpoint. + */ + __u32 status; + __u32 reserved; }; /* for KVM_SET_GUEST_DEBUG */ @@ -281,10 +295,6 @@ struct kvm_guest_debug_arch { * Type denotes h/w breakpoint, read watchpoint, write * watchpoint or watchpoint (both read and write). */ -#define KVMPPC_DEBUG_NONE 0x0 -#define KVMPPC_DEBUG_BREAKPOINT (1UL << 1) -#define KVMPPC_DEBUG_WATCH_WRITE (1UL << 2) -#define KVMPPC_DEBUG_WATCH_READ (1UL << 3) __u32 type; __u32 reserved; } bp[16]; @@ -319,6 +329,15 @@ struct kvm_create_spapr_tce { __u32 window_size; }; +/* for KVM_CAP_SPAPR_TCE_64 */ +struct kvm_create_spapr_tce_64 { + __u64 liobn; + __u32 page_shift; + __u32 flags; + __u64 offset; /* in pages */ + __u64 size; /* in pages */ +}; + /* for KVM_ALLOCATE_RMA */ struct kvm_allocate_rma { __u64 rma_size; @@ -390,6 +409,53 @@ struct kvm_get_htab_header { __u16 n_invalid; }; +/* For KVM_PPC_CONFIGURE_V3_MMU */ +struct kvm_ppc_mmuv3_cfg { + __u64 flags; + __u64 process_table; /* second doubleword of partition table entry */ +}; + +/* Flag values for KVM_PPC_CONFIGURE_V3_MMU */ +#define KVM_PPC_MMUV3_RADIX 1 /* 1 = radix mode, 0 = HPT */ +#define KVM_PPC_MMUV3_GTSE 2 /* global translation shootdown enb. */ + +/* For KVM_PPC_GET_RMMU_INFO */ +struct kvm_ppc_rmmu_info { + struct kvm_ppc_radix_geom { + __u8 page_shift; + __u8 level_bits[4]; + __u8 pad[3]; + } geometries[8]; + __u32 ap_encodings[8]; +}; + +/* For KVM_PPC_GET_CPU_CHAR */ +struct kvm_ppc_cpu_char { + __u64 character; /* characteristics of the CPU */ + __u64 behaviour; /* recommended software behaviour */ + __u64 character_mask; /* valid bits in character */ + __u64 behaviour_mask; /* valid bits in behaviour */ +}; + +/* + * Values for character and character_mask. + * These are identical to the values used by H_GET_CPU_CHARACTERISTICS. + */ +#define KVM_PPC_CPU_CHAR_SPEC_BAR_ORI31 (1ULL << 63) +#define KVM_PPC_CPU_CHAR_BCCTRL_SERIALISED (1ULL << 62) +#define KVM_PPC_CPU_CHAR_L1D_FLUSH_ORI30 (1ULL << 61) +#define KVM_PPC_CPU_CHAR_L1D_FLUSH_TRIG2 (1ULL << 60) +#define KVM_PPC_CPU_CHAR_L1D_THREAD_PRIV (1ULL << 59) +#define KVM_PPC_CPU_CHAR_BR_HINT_HONOURED (1ULL << 58) +#define KVM_PPC_CPU_CHAR_MTTRIG_THR_RECONF (1ULL << 57) +#define KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS (1ULL << 56) +#define KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST (1ull << 54) + +#define KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY (1ULL << 63) +#define KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR (1ULL << 62) +#define KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR (1ULL << 61) +#define KVM_PPC_CPU_BEHAV_FLUSH_COUNT_CACHE (1ull << 58) + /* Per-vcpu XICS interrupt controller state */ #define KVM_REG_PPC_ICP_STATE (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x8c) @@ -402,6 +468,8 @@ struct kvm_get_htab_header { #define KVM_REG_PPC_ICP_PPRI_SHIFT 16 /* pending irq priority */ #define KVM_REG_PPC_ICP_PPRI_MASK 0xff +#define KVM_REG_PPC_VP_STATE (KVM_REG_PPC | KVM_REG_SIZE_U128 | 0x8d) + /* Device control API: PPC-specific devices */ #define KVM_DEV_MPIC_GRP_MISC 1 #define KVM_DEV_MPIC_BASE_ADDR 0 /* 64-bit */ @@ -429,6 +497,11 @@ struct kvm_get_htab_header { #define KVM_REG_PPC_MMCR0 (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x10) #define KVM_REG_PPC_MMCR1 (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x11) #define KVM_REG_PPC_MMCRA (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x12) +#define KVM_REG_PPC_MMCR2 (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x13) +#define KVM_REG_PPC_MMCRS (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x14) +#define KVM_REG_PPC_SIAR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x15) +#define KVM_REG_PPC_SDAR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x16) +#define KVM_REG_PPC_SIER (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x17) #define KVM_REG_PPC_PMC1 (KVM_REG_PPC | KVM_REG_SIZE_U32 | 0x18) #define KVM_REG_PPC_PMC2 (KVM_REG_PPC | KVM_REG_SIZE_U32 | 0x19) @@ -457,6 +530,11 @@ struct kvm_get_htab_header { /* FP and vector status/control registers */ #define KVM_REG_PPC_FPSCR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x80) +/* + * VSCR register is documented as a 32-bit register in the ISA, but it can + * only be accesses via a vector register. Expose VSCR as a 32-bit register + * even though the kernel represents it as a 128-bit vector. + */ #define KVM_REG_PPC_VSCR (KVM_REG_PPC | KVM_REG_SIZE_U32 | 0x81) /* Virtual processor areas */ @@ -499,8 +577,94 @@ struct kvm_get_htab_header { #define KVM_REG_PPC_TLB3PS (KVM_REG_PPC | KVM_REG_SIZE_U32 | 0x9a) #define KVM_REG_PPC_EPTCFG (KVM_REG_PPC | KVM_REG_SIZE_U32 | 0x9b) +/* Timebase offset */ +#define KVM_REG_PPC_TB_OFFSET (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x9c) + +/* POWER8 registers */ +#define KVM_REG_PPC_SPMC1 (KVM_REG_PPC | KVM_REG_SIZE_U32 | 0x9d) +#define KVM_REG_PPC_SPMC2 (KVM_REG_PPC | KVM_REG_SIZE_U32 | 0x9e) +#define KVM_REG_PPC_IAMR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x9f) +#define KVM_REG_PPC_TFHAR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xa0) +#define KVM_REG_PPC_TFIAR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xa1) +#define KVM_REG_PPC_TEXASR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xa2) +#define KVM_REG_PPC_FSCR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xa3) +#define KVM_REG_PPC_PSPB (KVM_REG_PPC | KVM_REG_SIZE_U32 | 0xa4) +#define KVM_REG_PPC_EBBHR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xa5) +#define KVM_REG_PPC_EBBRR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xa6) +#define KVM_REG_PPC_BESCR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xa7) +#define KVM_REG_PPC_TAR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xa8) +#define KVM_REG_PPC_DPDES (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xa9) +#define KVM_REG_PPC_DAWR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xaa) +#define KVM_REG_PPC_DAWRX (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xab) +#define KVM_REG_PPC_CIABR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xac) +#define KVM_REG_PPC_IC (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xad) +#define KVM_REG_PPC_VTB (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xae) +#define KVM_REG_PPC_CSIGR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xaf) +#define KVM_REG_PPC_TACR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xb0) +#define KVM_REG_PPC_TCSCR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xb1) +#define KVM_REG_PPC_PID (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xb2) +#define KVM_REG_PPC_ACOP (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xb3) + +#define KVM_REG_PPC_VRSAVE (KVM_REG_PPC | KVM_REG_SIZE_U32 | 0xb4) +#define KVM_REG_PPC_LPCR (KVM_REG_PPC | KVM_REG_SIZE_U32 | 0xb5) +#define KVM_REG_PPC_LPCR_64 (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xb5) +#define KVM_REG_PPC_PPR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xb6) + +/* Architecture compatibility level */ +#define KVM_REG_PPC_ARCH_COMPAT (KVM_REG_PPC | KVM_REG_SIZE_U32 | 0xb7) + +#define KVM_REG_PPC_DABRX (KVM_REG_PPC | KVM_REG_SIZE_U32 | 0xb8) +#define KVM_REG_PPC_WORT (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xb9) +#define KVM_REG_PPC_SPRG9 (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xba) +#define KVM_REG_PPC_DBSR (KVM_REG_PPC | KVM_REG_SIZE_U32 | 0xbb) + +/* POWER9 registers */ +#define KVM_REG_PPC_TIDR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xbc) +#define KVM_REG_PPC_PSSCR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xbd) + +#define KVM_REG_PPC_DEC_EXPIRY (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xbe) +#define KVM_REG_PPC_ONLINE (KVM_REG_PPC | KVM_REG_SIZE_U32 | 0xbf) +#define KVM_REG_PPC_PTCR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xc0) + +/* POWER10 registers */ +#define KVM_REG_PPC_MMCR3 (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xc1) +#define KVM_REG_PPC_SIER2 (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xc2) +#define KVM_REG_PPC_SIER3 (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xc3) +#define KVM_REG_PPC_DAWR1 (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xc4) +#define KVM_REG_PPC_DAWRX1 (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xc5) +#define KVM_REG_PPC_DEXCR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xc6) +#define KVM_REG_PPC_HASHKEYR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xc7) +#define KVM_REG_PPC_HASHPKEYR (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xc8) + +/* Transactional Memory checkpointed state: + * This is all GPRs, all VSX regs and a subset of SPRs + */ +#define KVM_REG_PPC_TM (KVM_REG_PPC | 0x80000000) +/* TM GPRs */ +#define KVM_REG_PPC_TM_GPR0 (KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0) +#define KVM_REG_PPC_TM_GPR(n) (KVM_REG_PPC_TM_GPR0 + (n)) +#define KVM_REG_PPC_TM_GPR31 (KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x1f) +/* TM VSX */ +#define KVM_REG_PPC_TM_VSR0 (KVM_REG_PPC_TM | KVM_REG_SIZE_U128 | 0x20) +#define KVM_REG_PPC_TM_VSR(n) (KVM_REG_PPC_TM_VSR0 + (n)) +#define KVM_REG_PPC_TM_VSR63 (KVM_REG_PPC_TM | KVM_REG_SIZE_U128 | 0x5f) +/* TM SPRS */ +#define KVM_REG_PPC_TM_CR (KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x60) +#define KVM_REG_PPC_TM_LR (KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x61) +#define KVM_REG_PPC_TM_CTR (KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x62) +#define KVM_REG_PPC_TM_FPSCR (KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x63) +#define KVM_REG_PPC_TM_AMR (KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x64) +#define KVM_REG_PPC_TM_PPR (KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x65) +#define KVM_REG_PPC_TM_VRSAVE (KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x66) +#define KVM_REG_PPC_TM_VSCR (KVM_REG_PPC_TM | KVM_REG_SIZE_U32 | 0x67) +#define KVM_REG_PPC_TM_DSCR (KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x68) +#define KVM_REG_PPC_TM_TAR (KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x69) +#define KVM_REG_PPC_TM_XER (KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x6a) + /* PPC64 eXternal Interrupt Controller Specification */ #define KVM_DEV_XICS_GRP_SOURCES 1 /* 64-bit source attributes */ +#define KVM_DEV_XICS_GRP_CTRL 2 +#define KVM_DEV_XICS_NR_SERVERS 1 /* Layout of 64-bit source attribute values */ #define KVM_XICS_DESTINATION_SHIFT 0 @@ -510,5 +674,96 @@ struct kvm_get_htab_header { #define KVM_XICS_LEVEL_SENSITIVE (1ULL << 40) #define KVM_XICS_MASKED (1ULL << 41) #define KVM_XICS_PENDING (1ULL << 42) +#define KVM_XICS_PRESENTED (1ULL << 43) +#define KVM_XICS_QUEUED (1ULL << 44) + +/* POWER9 XIVE Native Interrupt Controller */ +#define KVM_DEV_XIVE_GRP_CTRL 1 +#define KVM_DEV_XIVE_RESET 1 +#define KVM_DEV_XIVE_EQ_SYNC 2 +#define KVM_DEV_XIVE_NR_SERVERS 3 +#define KVM_DEV_XIVE_GRP_SOURCE 2 /* 64-bit source identifier */ +#define KVM_DEV_XIVE_GRP_SOURCE_CONFIG 3 /* 64-bit source identifier */ +#define KVM_DEV_XIVE_GRP_EQ_CONFIG 4 /* 64-bit EQ identifier */ +#define KVM_DEV_XIVE_GRP_SOURCE_SYNC 5 /* 64-bit source identifier */ + +/* Layout of 64-bit XIVE source attribute values */ +#define KVM_XIVE_LEVEL_SENSITIVE (1ULL << 0) +#define KVM_XIVE_LEVEL_ASSERTED (1ULL << 1) + +/* Layout of 64-bit XIVE source configuration attribute values */ +#define KVM_XIVE_SOURCE_PRIORITY_SHIFT 0 +#define KVM_XIVE_SOURCE_PRIORITY_MASK 0x7 +#define KVM_XIVE_SOURCE_SERVER_SHIFT 3 +#define KVM_XIVE_SOURCE_SERVER_MASK 0xfffffff8ULL +#define KVM_XIVE_SOURCE_MASKED_SHIFT 32 +#define KVM_XIVE_SOURCE_MASKED_MASK 0x100000000ULL +#define KVM_XIVE_SOURCE_EISN_SHIFT 33 +#define KVM_XIVE_SOURCE_EISN_MASK 0xfffffffe00000000ULL + +/* Layout of 64-bit EQ identifier */ +#define KVM_XIVE_EQ_PRIORITY_SHIFT 0 +#define KVM_XIVE_EQ_PRIORITY_MASK 0x7 +#define KVM_XIVE_EQ_SERVER_SHIFT 3 +#define KVM_XIVE_EQ_SERVER_MASK 0xfffffff8ULL + +/* Layout of EQ configuration values (64 bytes) */ +struct kvm_ppc_xive_eq { + __u32 flags; + __u32 qshift; + __u64 qaddr; + __u32 qtoggle; + __u32 qindex; + __u8 pad[40]; +}; + +#define KVM_XIVE_EQ_ALWAYS_NOTIFY 0x00000001 + +#define KVM_XIVE_TIMA_PAGE_OFFSET 0 +#define KVM_XIVE_ESB_PAGE_OFFSET 4 + +/* for KVM_PPC_GET_PVINFO */ + +#define KVM_PPC_PVINFO_FLAGS_EV_IDLE (1<<0) + +struct kvm_ppc_pvinfo { + /* out */ + __u32 flags; + __u32 hcall[4]; + __u8 pad[108]; +}; + +/* for KVM_PPC_GET_SMMU_INFO */ +#define KVM_PPC_PAGE_SIZES_MAX_SZ 8 + +struct kvm_ppc_one_page_size { + __u32 page_shift; /* Page shift (or 0) */ + __u32 pte_enc; /* Encoding in the HPTE (>>12) */ +}; + +struct kvm_ppc_one_seg_page_size { + __u32 page_shift; /* Base page shift of segment (or 0) */ + __u32 slb_enc; /* SLB encoding for BookS */ + struct kvm_ppc_one_page_size enc[KVM_PPC_PAGE_SIZES_MAX_SZ]; +}; + +#define KVM_PPC_PAGE_SIZES_REAL 0x00000001 +#define KVM_PPC_1T_SEGMENTS 0x00000002 +#define KVM_PPC_NO_HASH 0x00000004 + +struct kvm_ppc_smmu_info { + __u64 flags; + __u32 slb_size; + __u16 data_keys; /* # storage keys supported for data */ + __u16 instr_keys; /* # storage keys supported for instructions */ + struct kvm_ppc_one_seg_page_size sps[KVM_PPC_PAGE_SIZES_MAX_SZ]; +}; + +/* for KVM_PPC_RESIZE_HPT_{PREPARE,COMMIT} */ +struct kvm_ppc_resize_hpt { + __u64 flags; + __u32 shift; + __u32 pad; +}; #endif /* __LINUX_KVM_POWERPC_H */ diff --git a/arch/powerpc/include/uapi/asm/kvm_para.h b/arch/powerpc/include/uapi/asm/kvm_para.h index e3af3286a068..ac596064d4c7 100644 --- a/arch/powerpc/include/uapi/asm/kvm_para.h +++ b/arch/powerpc/include/uapi/asm/kvm_para.h @@ -1,17 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * * Copyright IBM Corp. 2008 * * Authors: Hollis Blanchard <hollisb@us.ibm.com> @@ -30,7 +18,7 @@ * Struct fields are always 32 or 64 bit aligned, depending on them being 32 * or 64 bit wide respectively. * - * See Documentation/virtual/kvm/ppc-pv.txt + * See Documentation/virt/kvm/ppc-pv.rst */ struct kvm_vcpu_arch_shared { __u64 scratch1; @@ -82,10 +70,16 @@ struct kvm_vcpu_arch_shared { #define KVM_FEATURE_MAGIC_PAGE 1 +/* Magic page flags from host to guest */ + #define KVM_MAGIC_FEAT_SR (1 << 0) /* MASn, ESR, PIR, and high SPRGs */ #define KVM_MAGIC_FEAT_MAS0_TO_SPRG7 (1 << 1) +/* Magic page flags from guest to host */ + +#define MAGIC_PAGE_FLAG_NOT_MAPPED_NX (1 << 0) + #endif /* _UAPI__POWERPC_KVM_PARA_H__ */ diff --git a/arch/powerpc/include/uapi/asm/mman.h b/arch/powerpc/include/uapi/asm/mman.h index 6ea26df0a73c..c0c737215b00 100644 --- a/arch/powerpc/include/uapi/asm/mman.h +++ b/arch/powerpc/include/uapi/asm/mman.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ /* * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -20,12 +21,15 @@ #define MAP_DENYWRITE 0x0800 /* ETXTBSY */ #define MAP_EXECUTABLE 0x1000 /* mark it as an executable */ + #define MCL_CURRENT 0x2000 /* lock all currently mapped pages */ #define MCL_FUTURE 0x4000 /* lock all additions to address space */ - -#define MAP_POPULATE 0x8000 /* populate (prefault) pagetables */ -#define MAP_NONBLOCK 0x10000 /* do not block on IO */ -#define MAP_STACK 0x20000 /* give out an address that is best suited for process/thread stacks */ -#define MAP_HUGETLB 0x40000 /* create a huge page mapping */ - +#define MCL_ONFAULT 0x8000 /* lock all pages that are faulted in */ + +/* Override any generic PKEY permission defines */ +#define PKEY_DISABLE_EXECUTE 0x4 +#undef PKEY_ACCESS_MASK +#define PKEY_ACCESS_MASK (PKEY_DISABLE_ACCESS |\ + PKEY_DISABLE_WRITE |\ + PKEY_DISABLE_EXECUTE) #endif /* _UAPI_ASM_POWERPC_MMAN_H */ diff --git a/arch/powerpc/include/uapi/asm/msgbuf.h b/arch/powerpc/include/uapi/asm/msgbuf.h index dd76743c7537..7919b2ba41b5 100644 --- a/arch/powerpc/include/uapi/asm/msgbuf.h +++ b/arch/powerpc/include/uapi/asm/msgbuf.h @@ -1,6 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _ASM_POWERPC_MSGBUF_H #define _ASM_POWERPC_MSGBUF_H +#include <asm/ipcbuf.h> + /* * The msqid64_ds structure for the PowerPC architecture. * Note extra padding because this structure is passed back and forth @@ -9,18 +12,18 @@ struct msqid64_ds { struct ipc64_perm msg_perm; -#ifndef __powerpc64__ - unsigned int __unused1; -#endif - __kernel_time_t msg_stime; /* last msgsnd time */ -#ifndef __powerpc64__ - unsigned int __unused2; -#endif - __kernel_time_t msg_rtime; /* last msgrcv time */ -#ifndef __powerpc64__ - unsigned int __unused3; +#ifdef __powerpc64__ + long msg_stime; /* last msgsnd time */ + long msg_rtime; /* last msgrcv time */ + long msg_ctime; /* last change time */ +#else + unsigned long msg_stime_high; + unsigned long msg_stime; /* last msgsnd time */ + unsigned long msg_rtime_high; + unsigned long msg_rtime; /* last msgrcv time */ + unsigned long msg_ctime_high; + unsigned long msg_ctime; /* last change time */ #endif - __kernel_time_t msg_ctime; /* last change time */ unsigned long msg_cbytes; /* current number of bytes on queue */ unsigned long msg_qnum; /* number of messages in queue */ unsigned long msg_qbytes; /* max number of bytes on queue */ diff --git a/arch/powerpc/include/uapi/asm/nvram.h b/arch/powerpc/include/uapi/asm/nvram.h index 608bdc8aedd1..c92c7f056a91 100644 --- a/arch/powerpc/include/uapi/asm/nvram.h +++ b/arch/powerpc/include/uapi/asm/nvram.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ /* * NVRAM definitions and access functions. * diff --git a/arch/powerpc/include/uapi/asm/opal-prd.h b/arch/powerpc/include/uapi/asm/opal-prd.h new file mode 100644 index 000000000000..11abcf0192ca --- /dev/null +++ b/arch/powerpc/include/uapi/asm/opal-prd.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ +/* + * OPAL Runtime Diagnostics interface driver + * Supported on POWERNV platform + * + * (C) Copyright IBM 2015 + * + * Author: Vaidyanathan Srinivasan <svaidy at linux.vnet.ibm.com> + * Author: Jeremy Kerr <jk@ozlabs.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _UAPI_ASM_POWERPC_OPAL_PRD_H_ +#define _UAPI_ASM_POWERPC_OPAL_PRD_H_ + +#include <linux/types.h> + +/** + * The version of the kernel interface of the PRD system. This describes the + * interface available for the /dev/opal-prd device. The actual PRD message + * layout and content is private to the firmware <--> userspace interface, so + * is not covered by this versioning. + * + * Future interface versions are backwards-compatible; if a later kernel + * version is encountered, functionality provided in earlier versions + * will work. + */ +#define OPAL_PRD_KERNEL_VERSION 1 + +#define OPAL_PRD_GET_INFO _IOR('o', 0x01, struct opal_prd_info) +#define OPAL_PRD_SCOM_READ _IOR('o', 0x02, struct opal_prd_scom) +#define OPAL_PRD_SCOM_WRITE _IOW('o', 0x03, struct opal_prd_scom) + +#ifndef __ASSEMBLER__ + +struct opal_prd_info { + __u64 version; + __u64 reserved[3]; +}; + +struct opal_prd_scom { + __u64 chip; + __u64 addr; + __u64 data; + __s64 rc; +}; + +#endif /* __ASSEMBLER__ */ + +#endif /* _UAPI_ASM_POWERPC_OPAL_PRD_H */ diff --git a/arch/powerpc/include/uapi/asm/papr-hvpipe.h b/arch/powerpc/include/uapi/asm/papr-hvpipe.h new file mode 100644 index 000000000000..f8794139d06a --- /dev/null +++ b/arch/powerpc/include/uapi/asm/papr-hvpipe.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_PAPR_HVPIPE_H_ +#define _UAPI_PAPR_HVPIPE_H_ + +#include <linux/types.h> +#include <asm/ioctl.h> +#include <asm/papr-miscdev.h> + +/* + * This header is included in payload between OS and the user + * space. + * flags: OS notifies the user space whether the hvpipe is + * closed or the buffer has the payload. + */ +struct papr_hvpipe_hdr { + __u8 version; + __u8 reserved[3]; + __u32 flags; + __u8 reserved2[40]; +}; + +/* + * ioctl for /dev/papr-hvpipe + */ +#define PAPR_HVPIPE_IOC_CREATE_HANDLE _IOW(PAPR_MISCDEV_IOC_ID, 9, __u32) + +/* + * hvpipe_hdr flags used for read() + */ +#define HVPIPE_MSG_AVAILABLE 0x01 /* Payload is available */ +#define HVPIPE_LOST_CONNECTION 0x02 /* Pipe connection is closed/unavailable */ + +#endif /* _UAPI_PAPR_HVPIPE_H_ */ diff --git a/arch/powerpc/include/uapi/asm/papr-indices.h b/arch/powerpc/include/uapi/asm/papr-indices.h new file mode 100644 index 000000000000..c2999d89d52a --- /dev/null +++ b/arch/powerpc/include/uapi/asm/papr-indices.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_PAPR_INDICES_H_ +#define _UAPI_PAPR_INDICES_H_ + +#include <linux/types.h> +#include <asm/ioctl.h> +#include <asm/papr-miscdev.h> + +#define LOC_CODE_SIZE 80 +#define RTAS_GET_INDICES_BUF_SIZE SZ_4K + +struct papr_indices_io_block { + union { + struct { + __u8 is_sensor; /* 0 for indicator and 1 for sensor */ + __u32 indice_type; + } indices; + struct { + __u32 token; /* Sensor or indicator token */ + __u32 state; /* get / set state */ + /* + * PAPR+ 12.3.2.4 Converged Location Code Rules - Length + * Restrictions. 79 characters plus null. + */ + char location_code_str[LOC_CODE_SIZE]; /* location code */ + } dynamic_param; + }; +}; + +/* + * ioctls for /dev/papr-indices. + * PAPR_INDICES_IOC_GET: Returns a get-indices handle fd to read data + * PAPR_DYNAMIC_SENSOR_IOC_GET: Gets the state of the input sensor + * PAPR_DYNAMIC_INDICATOR_IOC_SET: Sets the new state for the input indicator + */ +#define PAPR_INDICES_IOC_GET _IOW(PAPR_MISCDEV_IOC_ID, 3, struct papr_indices_io_block) +#define PAPR_DYNAMIC_SENSOR_IOC_GET _IOWR(PAPR_MISCDEV_IOC_ID, 4, struct papr_indices_io_block) +#define PAPR_DYNAMIC_INDICATOR_IOC_SET _IOW(PAPR_MISCDEV_IOC_ID, 5, struct papr_indices_io_block) + + +#endif /* _UAPI_PAPR_INDICES_H_ */ diff --git a/arch/powerpc/include/uapi/asm/papr-miscdev.h b/arch/powerpc/include/uapi/asm/papr-miscdev.h new file mode 100644 index 000000000000..49a2a270b7f3 --- /dev/null +++ b/arch/powerpc/include/uapi/asm/papr-miscdev.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_PAPR_MISCDEV_H_ +#define _UAPI_PAPR_MISCDEV_H_ + +enum { + PAPR_MISCDEV_IOC_ID = 0xb2, +}; + +#endif /* _UAPI_PAPR_MISCDEV_H_ */ diff --git a/arch/powerpc/include/uapi/asm/papr-physical-attestation.h b/arch/powerpc/include/uapi/asm/papr-physical-attestation.h new file mode 100644 index 000000000000..ea746837bb9a --- /dev/null +++ b/arch/powerpc/include/uapi/asm/papr-physical-attestation.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_PAPR_PHYSICAL_ATTESTATION_H_ +#define _UAPI_PAPR_PHYSICAL_ATTESTATION_H_ + +#include <linux/types.h> +#include <asm/ioctl.h> +#include <asm/papr-miscdev.h> + +#define PAPR_PHYATTEST_MAX_INPUT 4084 /* Max 4K buffer: 4K-12 */ + +/* + * Defined in PAPR 2.13+ 21.6 Attestation Command Structures. + * User space pass this struct and the max size should be 4K. + */ +struct papr_phy_attest_io_block { + __u8 version; + __u8 command; + __u8 TCG_major_ver; + __u8 TCG_minor_ver; + __be32 length; + __be32 correlator; + __u8 payload[PAPR_PHYATTEST_MAX_INPUT]; +}; + +/* + * ioctl for /dev/papr-physical-attestation. Returns a attestation + * command fd handle + */ +#define PAPR_PHY_ATTEST_IOC_HANDLE _IOW(PAPR_MISCDEV_IOC_ID, 8, struct papr_phy_attest_io_block) + +#endif /* _UAPI_PAPR_PHYSICAL_ATTESTATION_H_ */ diff --git a/arch/powerpc/include/uapi/asm/papr-platform-dump.h b/arch/powerpc/include/uapi/asm/papr-platform-dump.h new file mode 100644 index 000000000000..8a1c060e89a9 --- /dev/null +++ b/arch/powerpc/include/uapi/asm/papr-platform-dump.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_PAPR_PLATFORM_DUMP_H_ +#define _UAPI_PAPR_PLATFORM_DUMP_H_ + +#include <linux/types.h> +#include <asm/ioctl.h> +#include <asm/papr-miscdev.h> + +/* + * ioctl for /dev/papr-platform-dump. Returns a platform-dump handle fd + * corresponding to dump tag. + */ +#define PAPR_PLATFORM_DUMP_IOC_CREATE_HANDLE _IOW(PAPR_MISCDEV_IOC_ID, 6, __u64) +#define PAPR_PLATFORM_DUMP_IOC_INVALIDATE _IOW(PAPR_MISCDEV_IOC_ID, 7, __u64) + +#endif /* _UAPI_PAPR_PLATFORM_DUMP_H_ */ diff --git a/arch/powerpc/include/uapi/asm/papr-sysparm.h b/arch/powerpc/include/uapi/asm/papr-sysparm.h new file mode 100644 index 000000000000..f733467b1534 --- /dev/null +++ b/arch/powerpc/include/uapi/asm/papr-sysparm.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_PAPR_SYSPARM_H_ +#define _UAPI_PAPR_SYSPARM_H_ + +#include <linux/types.h> +#include <asm/ioctl.h> +#include <asm/papr-miscdev.h> + +enum { + PAPR_SYSPARM_MAX_INPUT = 1024, + PAPR_SYSPARM_MAX_OUTPUT = 4000, +}; + +struct papr_sysparm_io_block { + __u32 parameter; + __u16 length; + __u8 data[PAPR_SYSPARM_MAX_OUTPUT]; +}; + +/** + * PAPR_SYSPARM_IOC_GET - Retrieve the value of a PAPR system parameter. + * + * Uses _IOWR because of one corner case: Retrieving the value of the + * "OS Service Entitlement Status" parameter (60) requires the caller + * to supply input data (a date string) in the buffer passed to + * firmware. So the @length and @data of the incoming + * papr_sysparm_io_block are always used to initialize the work area + * supplied to ibm,get-system-parameter. No other parameters are known + * to parameterize the result this way, and callers are encouraged + * (but not required) to zero-initialize @length and @data in the + * common case. + * + * On error the contents of the ioblock are indeterminate. + * + * Return: + * 0: Success; @length is the length of valid data in @data, not to exceed @PAPR_SYSPARM_MAX_OUTPUT. + * -EIO: Platform error. (-1) + * -EINVAL: Incorrect data length or format. (-9999) + * -EPERM: The calling partition is not allowed to access this parameter. (-9002) + * -EOPNOTSUPP: Parameter not supported on this platform (-3) + */ +#define PAPR_SYSPARM_IOC_GET _IOWR(PAPR_MISCDEV_IOC_ID, 1, struct papr_sysparm_io_block) + +/** + * PAPR_SYSPARM_IOC_SET - Update the value of a PAPR system parameter. + * + * The contents of the ioblock are unchanged regardless of success. + * + * Return: + * 0: Success; the parameter has been updated. + * -EIO: Platform error. (-1) + * -EINVAL: Incorrect data length or format. (-9999) + * -EPERM: The calling partition is not allowed to access this parameter. (-9002) + * -EOPNOTSUPP: Parameter not supported on this platform (-3) + */ +#define PAPR_SYSPARM_IOC_SET _IOW(PAPR_MISCDEV_IOC_ID, 2, struct papr_sysparm_io_block) + +#endif /* _UAPI_PAPR_SYSPARM_H_ */ diff --git a/arch/powerpc/include/uapi/asm/papr-vpd.h b/arch/powerpc/include/uapi/asm/papr-vpd.h new file mode 100644 index 000000000000..1c88e87cb420 --- /dev/null +++ b/arch/powerpc/include/uapi/asm/papr-vpd.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_PAPR_VPD_H_ +#define _UAPI_PAPR_VPD_H_ + +#include <asm/ioctl.h> +#include <asm/papr-miscdev.h> + +struct papr_location_code { + /* + * PAPR+ v2.13 12.3.2.4 Converged Location Code Rules - Length + * Restrictions. 79 characters plus nul. + */ + char str[80]; +}; + +/* + * ioctl for /dev/papr-vpd. Returns a VPD handle fd corresponding to + * the location code. + */ +#define PAPR_VPD_IOC_CREATE_HANDLE _IOW(PAPR_MISCDEV_IOC_ID, 0, struct papr_location_code) + +#endif /* _UAPI_PAPR_VPD_H_ */ diff --git a/arch/powerpc/include/uapi/asm/param.h b/arch/powerpc/include/uapi/asm/param.h deleted file mode 100644 index 965d45427975..000000000000 --- a/arch/powerpc/include/uapi/asm/param.h +++ /dev/null @@ -1 +0,0 @@ -#include <asm-generic/param.h> diff --git a/arch/powerpc/include/uapi/asm/perf_event.h b/arch/powerpc/include/uapi/asm/perf_event.h new file mode 100644 index 000000000000..ce488e48db44 --- /dev/null +++ b/arch/powerpc/include/uapi/asm/perf_event.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Copyright 2013 Michael Ellerman, IBM Corp. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 of the + * License. + */ + +#ifndef _UAPI_ASM_POWERPC_PERF_EVENT_H +#define _UAPI_ASM_POWERPC_PERF_EVENT_H + +/* + * We use bit 63 of perf_event_attr.config as a flag to request EBB. + */ +#define PERF_EVENT_CONFIG_EBB_SHIFT 63 + +#endif /* _UAPI_ASM_POWERPC_PERF_EVENT_H */ diff --git a/arch/powerpc/include/uapi/asm/perf_regs.h b/arch/powerpc/include/uapi/asm/perf_regs.h new file mode 100644 index 000000000000..749a2e3af89e --- /dev/null +++ b/arch/powerpc/include/uapi/asm/perf_regs.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_POWERPC_PERF_REGS_H +#define _UAPI_ASM_POWERPC_PERF_REGS_H + +enum perf_event_powerpc_regs { + PERF_REG_POWERPC_R0, + PERF_REG_POWERPC_R1, + PERF_REG_POWERPC_R2, + PERF_REG_POWERPC_R3, + PERF_REG_POWERPC_R4, + PERF_REG_POWERPC_R5, + PERF_REG_POWERPC_R6, + PERF_REG_POWERPC_R7, + PERF_REG_POWERPC_R8, + PERF_REG_POWERPC_R9, + PERF_REG_POWERPC_R10, + PERF_REG_POWERPC_R11, + PERF_REG_POWERPC_R12, + PERF_REG_POWERPC_R13, + PERF_REG_POWERPC_R14, + PERF_REG_POWERPC_R15, + PERF_REG_POWERPC_R16, + PERF_REG_POWERPC_R17, + PERF_REG_POWERPC_R18, + PERF_REG_POWERPC_R19, + PERF_REG_POWERPC_R20, + PERF_REG_POWERPC_R21, + PERF_REG_POWERPC_R22, + PERF_REG_POWERPC_R23, + PERF_REG_POWERPC_R24, + PERF_REG_POWERPC_R25, + PERF_REG_POWERPC_R26, + PERF_REG_POWERPC_R27, + PERF_REG_POWERPC_R28, + PERF_REG_POWERPC_R29, + PERF_REG_POWERPC_R30, + PERF_REG_POWERPC_R31, + PERF_REG_POWERPC_NIP, + PERF_REG_POWERPC_MSR, + PERF_REG_POWERPC_ORIG_R3, + PERF_REG_POWERPC_CTR, + PERF_REG_POWERPC_LINK, + PERF_REG_POWERPC_XER, + PERF_REG_POWERPC_CCR, + PERF_REG_POWERPC_SOFTE, + PERF_REG_POWERPC_TRAP, + PERF_REG_POWERPC_DAR, + PERF_REG_POWERPC_DSISR, + PERF_REG_POWERPC_SIER, + PERF_REG_POWERPC_MMCRA, + /* Extended registers */ + PERF_REG_POWERPC_MMCR0, + PERF_REG_POWERPC_MMCR1, + PERF_REG_POWERPC_MMCR2, + PERF_REG_POWERPC_MMCR3, + PERF_REG_POWERPC_SIER2, + PERF_REG_POWERPC_SIER3, + PERF_REG_POWERPC_PMC1, + PERF_REG_POWERPC_PMC2, + PERF_REG_POWERPC_PMC3, + PERF_REG_POWERPC_PMC4, + PERF_REG_POWERPC_PMC5, + PERF_REG_POWERPC_PMC6, + PERF_REG_POWERPC_SDAR, + PERF_REG_POWERPC_SIAR, + /* Max mask value for interrupt regs w/o extended regs */ + PERF_REG_POWERPC_MAX = PERF_REG_POWERPC_MMCRA + 1, + /* Max mask value for interrupt regs including extended regs */ + PERF_REG_EXTENDED_MAX = PERF_REG_POWERPC_SIAR + 1, +}; + +#define PERF_REG_PMU_MASK ((1ULL << PERF_REG_POWERPC_MAX) - 1) + +/* + * PERF_REG_EXTENDED_MASK value for CPU_FTR_ARCH_300 + * includes 11 SPRS from MMCR0 to SIAR excluding the + * unsupported SPRS MMCR3, SIER2 and SIER3. + */ +#define PERF_REG_PMU_MASK_300 \ + ((1ULL << PERF_REG_POWERPC_MMCR0) | (1ULL << PERF_REG_POWERPC_MMCR1) | \ + (1ULL << PERF_REG_POWERPC_MMCR2) | (1ULL << PERF_REG_POWERPC_PMC1) | \ + (1ULL << PERF_REG_POWERPC_PMC2) | (1ULL << PERF_REG_POWERPC_PMC3) | \ + (1ULL << PERF_REG_POWERPC_PMC4) | (1ULL << PERF_REG_POWERPC_PMC5) | \ + (1ULL << PERF_REG_POWERPC_PMC6) | (1ULL << PERF_REG_POWERPC_SDAR) | \ + (1ULL << PERF_REG_POWERPC_SIAR)) + +/* + * PERF_REG_EXTENDED_MASK value for CPU_FTR_ARCH_31 + * includes 14 SPRs from MMCR0 to SIAR. + */ +#define PERF_REG_PMU_MASK_31 \ + (PERF_REG_PMU_MASK_300 | (1ULL << PERF_REG_POWERPC_MMCR3) | \ + (1ULL << PERF_REG_POWERPC_SIER2) | (1ULL << PERF_REG_POWERPC_SIER3)) + +#endif /* _UAPI_ASM_POWERPC_PERF_REGS_H */ diff --git a/arch/powerpc/include/uapi/asm/poll.h b/arch/powerpc/include/uapi/asm/poll.h deleted file mode 100644 index c98509d3149e..000000000000 --- a/arch/powerpc/include/uapi/asm/poll.h +++ /dev/null @@ -1 +0,0 @@ -#include <asm-generic/poll.h> diff --git a/arch/powerpc/include/uapi/asm/posix_types.h b/arch/powerpc/include/uapi/asm/posix_types.h index 2958c5b97b2d..9c0342312544 100644 --- a/arch/powerpc/include/uapi/asm/posix_types.h +++ b/arch/powerpc/include/uapi/asm/posix_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _ASM_POWERPC_POSIX_TYPES_H #define _ASM_POWERPC_POSIX_TYPES_H @@ -11,11 +12,6 @@ typedef unsigned long __kernel_old_dev_t; #define __kernel_old_dev_t __kernel_old_dev_t #else -typedef unsigned int __kernel_size_t; -typedef int __kernel_ssize_t; -typedef long __kernel_ptrdiff_t; -#define __kernel_size_t __kernel_size_t - typedef short __kernel_ipc_pid_t; #define __kernel_ipc_pid_t __kernel_ipc_pid_t #endif diff --git a/arch/powerpc/include/uapi/asm/ps3fb.h b/arch/powerpc/include/uapi/asm/ps3fb.h index e7233a849680..b1c6b0cd9e80 100644 --- a/arch/powerpc/include/uapi/asm/ps3fb.h +++ b/arch/powerpc/include/uapi/asm/ps3fb.h @@ -1,19 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * Copyright (C) 2006 Sony Computer Entertainment Inc. * Copyright 2006, 2007 Sony Corporation - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published - * by the Free Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ #ifndef _ASM_POWERPC_PS3FB_H_ diff --git a/arch/powerpc/include/uapi/asm/ptrace.h b/arch/powerpc/include/uapi/asm/ptrace.h index 77d2ed35b111..01e630149d48 100644 --- a/arch/powerpc/include/uapi/asm/ptrace.h +++ b/arch/powerpc/include/uapi/asm/ptrace.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ /* * Copyright (C) 2001 PPC64 Team, IBM Corp * @@ -26,9 +27,14 @@ #include <linux/types.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ -struct pt_regs { +#ifdef __KERNEL__ +struct user_pt_regs +#else +struct pt_regs +#endif +{ unsigned long gpr[32]; unsigned long nip; unsigned long msr; @@ -51,7 +57,7 @@ struct pt_regs { unsigned long result; /* Result of a system call */ }; -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ /* @@ -136,7 +142,7 @@ struct pt_regs { #endif /* __powerpc64__ */ /* - * Get/set all the altivec registers vr0..vr31, vscr, vrsave, in one go. + * Get/set all the altivec registers v0..v31, vscr, vrsave, in one go. * The transfer totals 34 quadword. Quadwords 0-31 contain the * corresponding vector registers. Quadword 32 contains the vscr as the * last word (offset 12) within that quadword. Quadword 33 contains the @@ -159,6 +165,10 @@ struct pt_regs { #define PTRACE_GETVSRREGS 0x1b #define PTRACE_SETVSRREGS 0x1c +/* Syscall emulation defines */ +#define PTRACE_SYSEMU 0x1d +#define PTRACE_SYSEMU_SINGLESTEP 0x1e + /* * Get or set a debug register. The first 16 are DABR registers and the * second 16 are IABR registers. @@ -190,7 +200,7 @@ struct pt_regs { #define PPC_PTRACE_SETHWDEBUG 0x88 #define PPC_PTRACE_DELHWDEBUG 0x87 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ struct ppc_debug_info { __u32 version; /* Only version 1 exists to date */ @@ -202,7 +212,7 @@ struct ppc_debug_info { __u64 features; }; -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ /* * features will have bits indication whether there is support for: @@ -212,8 +222,9 @@ struct ppc_debug_info { #define PPC_DEBUG_FEATURE_DATA_BP_RANGE 0x0000000000000004 #define PPC_DEBUG_FEATURE_DATA_BP_MASK 0x0000000000000008 #define PPC_DEBUG_FEATURE_DATA_BP_DAWR 0x0000000000000010 +#define PPC_DEBUG_FEATURE_DATA_BP_ARCH_31 0x0000000000000020 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ struct ppc_hw_breakpoint { __u32 version; /* currently, version must be 1 */ @@ -225,7 +236,7 @@ struct ppc_hw_breakpoint { __u64 condition_value; /* contents of the DVC register */ }; -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ /* * Trigger Type diff --git a/arch/powerpc/include/uapi/asm/resource.h b/arch/powerpc/include/uapi/asm/resource.h deleted file mode 100644 index 04bc4db8921b..000000000000 --- a/arch/powerpc/include/uapi/asm/resource.h +++ /dev/null @@ -1 +0,0 @@ -#include <asm-generic/resource.h> diff --git a/arch/powerpc/include/uapi/asm/seccomp.h b/arch/powerpc/include/uapi/asm/seccomp.h deleted file mode 100644 index 00c1d9133cfe..000000000000 --- a/arch/powerpc/include/uapi/asm/seccomp.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef _ASM_POWERPC_SECCOMP_H -#define _ASM_POWERPC_SECCOMP_H - -#include <linux/unistd.h> - -#define __NR_seccomp_read __NR_read -#define __NR_seccomp_write __NR_write -#define __NR_seccomp_exit __NR_exit -#define __NR_seccomp_sigreturn __NR_rt_sigreturn - -#define __NR_seccomp_read_32 __NR_read -#define __NR_seccomp_write_32 __NR_write -#define __NR_seccomp_exit_32 __NR_exit -#define __NR_seccomp_sigreturn_32 __NR_sigreturn - -#endif /* _ASM_POWERPC_SECCOMP_H */ diff --git a/arch/powerpc/include/uapi/asm/sembuf.h b/arch/powerpc/include/uapi/asm/sembuf.h index 99a41938ae3d..85e96ccb5f0f 100644 --- a/arch/powerpc/include/uapi/asm/sembuf.h +++ b/arch/powerpc/include/uapi/asm/sembuf.h @@ -1,6 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ #ifndef _ASM_POWERPC_SEMBUF_H #define _ASM_POWERPC_SEMBUF_H +#include <asm/ipcbuf.h> + /* * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -14,20 +17,20 @@ * between kernel and user space. * * Pad space is left for: - * - 64-bit time_t to solve y2038 problem - * - 2 miscellaneous 32-bit values + * - 2 miscellaneous 32/64-bit values */ struct semid64_ds { struct ipc64_perm sem_perm; /* permissions .. see ipc.h */ #ifndef __powerpc64__ - unsigned long __unused1; -#endif - __kernel_time_t sem_otime; /* last semop time */ -#ifndef __powerpc64__ - unsigned long __unused2; + unsigned long sem_otime_high; + unsigned long sem_otime; /* last semop time */ + unsigned long sem_ctime_high; + unsigned long sem_ctime; /* last change time */ +#else + long sem_otime; /* last semop time */ + long sem_ctime; /* last change time */ #endif - __kernel_time_t sem_ctime; /* last change time */ unsigned long sem_nsems; /* no. of semaphores in array */ unsigned long __unused3; unsigned long __unused4; diff --git a/arch/powerpc/include/uapi/asm/setup.h b/arch/powerpc/include/uapi/asm/setup.h index 552df83f1a49..c54940b09d06 100644 --- a/arch/powerpc/include/uapi/asm/setup.h +++ b/arch/powerpc/include/uapi/asm/setup.h @@ -1 +1,7 @@ -#include <asm-generic/setup.h> +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_POWERPC_SETUP_H +#define _UAPI_ASM_POWERPC_SETUP_H + +#define COMMAND_LINE_SIZE 2048 + +#endif /* _UAPI_ASM_POWERPC_SETUP_H */ diff --git a/arch/powerpc/include/uapi/asm/shmbuf.h b/arch/powerpc/include/uapi/asm/shmbuf.h index 8efa39698b6c..439a3a02ba64 100644 --- a/arch/powerpc/include/uapi/asm/shmbuf.h +++ b/arch/powerpc/include/uapi/asm/shmbuf.h @@ -1,6 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ #ifndef _ASM_POWERPC_SHMBUF_H #define _ASM_POWERPC_SHMBUF_H +#include <asm/ipcbuf.h> +#include <asm/posix_types.h> + /* * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -15,28 +19,25 @@ * between kernel and user space. * * Pad space is left for: - * - 64-bit time_t to solve y2038 problem * - 2 miscellaneous 32-bit values */ struct shmid64_ds { struct ipc64_perm shm_perm; /* operation perms */ -#ifndef __powerpc64__ - unsigned long __unused1; -#endif - __kernel_time_t shm_atime; /* last attach time */ -#ifndef __powerpc64__ - unsigned long __unused2; -#endif - __kernel_time_t shm_dtime; /* last detach time */ -#ifndef __powerpc64__ - unsigned long __unused3; -#endif - __kernel_time_t shm_ctime; /* last change time */ -#ifndef __powerpc64__ +#ifdef __powerpc64__ + long shm_atime; /* last attach time */ + long shm_dtime; /* last detach time */ + long shm_ctime; /* last change time */ +#else + unsigned long shm_atime_high; + unsigned long shm_atime; /* last attach time */ + unsigned long shm_dtime_high; + unsigned long shm_dtime; /* last detach time */ + unsigned long shm_ctime_high; + unsigned long shm_ctime; /* last change time */ unsigned long __unused4; #endif - size_t shm_segsz; /* size of segment (bytes) */ + __kernel_size_t shm_segsz; /* size of segment (bytes) */ __kernel_pid_t shm_cpid; /* pid of creator */ __kernel_pid_t shm_lpid; /* pid of last operator */ unsigned long shm_nattch; /* no. of current attaches */ diff --git a/arch/powerpc/include/uapi/asm/sigcontext.h b/arch/powerpc/include/uapi/asm/sigcontext.h index 9c1f24fd5d11..630aeda56d59 100644 --- a/arch/powerpc/include/uapi/asm/sigcontext.h +++ b/arch/powerpc/include/uapi/asm/sigcontext.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ #ifndef _ASM_POWERPC_SIGCONTEXT_H #define _ASM_POWERPC_SIGCONTEXT_H @@ -21,14 +22,18 @@ struct sigcontext { #endif unsigned long handler; unsigned long oldmask; - struct pt_regs __user *regs; +#ifdef __KERNEL__ + struct user_pt_regs __user *regs; +#else + struct pt_regs *regs; +#endif #ifdef __powerpc64__ elf_gregset_t gp_regs; elf_fpregset_t fp_regs; /* * To maintain compatibility with current implementations the sigcontext is * extended by appending a pointer (v_regs) to a quadword type (elf_vrreg_t) - * followed by an unstructured (vmx_reserve) field of 69 doublewords. This + * followed by an unstructured (vmx_reserve) field of 101 doublewords. This * allows the array of vector registers to be quadword aligned independent of * the alignment of the containing sigcontext or ucontext. It is the * responsibility of the code setting the sigcontext to set this pointer to @@ -80,7 +85,7 @@ struct sigcontext { * registers and vscr/vrsave. */ elf_vrreg_t __user *v_regs; - long vmx_reserve[ELF_NVRREG+ELF_NVRREG+32+1]; + long vmx_reserve[ELF_NVRREG + ELF_NVRREG + 1 + 32]; #endif }; diff --git a/arch/powerpc/include/uapi/asm/siginfo.h b/arch/powerpc/include/uapi/asm/siginfo.h deleted file mode 100644 index ccce3ef5cd86..000000000000 --- a/arch/powerpc/include/uapi/asm/siginfo.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef _ASM_POWERPC_SIGINFO_H -#define _ASM_POWERPC_SIGINFO_H - -/* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ - -#ifdef __powerpc64__ -# define __ARCH_SI_PREAMBLE_SIZE (4 * sizeof(int)) -#endif - -#include <asm-generic/siginfo.h> - -#undef NSIGTRAP -#define NSIGTRAP 4 - -#endif /* _ASM_POWERPC_SIGINFO_H */ diff --git a/arch/powerpc/include/uapi/asm/signal.h b/arch/powerpc/include/uapi/asm/signal.h index 6c69ee94fd8d..a5dfe84f50ab 100644 --- a/arch/powerpc/include/uapi/asm/signal.h +++ b/arch/powerpc/include/uapi/asm/signal.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _UAPI_ASM_POWERPC_SIGNAL_H #define _UAPI_ASM_POWERPC_SIGNAL_H @@ -59,34 +60,15 @@ typedef struct { #define SIGRTMIN 32 #define SIGRTMAX _NSIG -/* - * SA_FLAGS values: - * - * SA_ONSTACK is not currently supported, but will allow sigaltstack(2). - * SA_RESTART flag to get restarting signals (which were the default long ago) - * SA_NOCLDSTOP flag to turn off SIGCHLD when children stop. - * SA_RESETHAND clears the handler when the signal is delivered. - * SA_NOCLDWAIT flag on SIGCHLD to inhibit zombies. - * SA_NODEFER prevents the current signal from being masked in the handler. - * - * SA_ONESHOT and SA_NOMASK are the historical Linux names for the Single - * Unix names RESETHAND and NODEFER respectively. - */ -#define SA_NOCLDSTOP 0x00000001U -#define SA_NOCLDWAIT 0x00000002U -#define SA_SIGINFO 0x00000004U -#define SA_ONSTACK 0x08000000U -#define SA_RESTART 0x10000000U -#define SA_NODEFER 0x40000000U -#define SA_RESETHAND 0x80000000U - -#define SA_NOMASK SA_NODEFER -#define SA_ONESHOT SA_RESETHAND - #define SA_RESTORER 0x04000000U +#ifdef __powerpc64__ +#define MINSIGSTKSZ 8192 +#define SIGSTKSZ 32768 +#else #define MINSIGSTKSZ 2048 #define SIGSTKSZ 8192 +#endif #include <asm-generic/signal-defs.h> @@ -109,7 +91,7 @@ struct sigaction { typedef struct sigaltstack { void __user *ss_sp; int ss_flags; - size_t ss_size; + __kernel_size_t ss_size; } stack_t; diff --git a/arch/powerpc/include/uapi/asm/socket.h b/arch/powerpc/include/uapi/asm/socket.h index a6d74467c9ed..12aa0c43e775 100644 --- a/arch/powerpc/include/uapi/asm/socket.h +++ b/arch/powerpc/include/uapi/asm/socket.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ #ifndef _ASM_POWERPC_SOCKET_H #define _ASM_POWERPC_SOCKET_H @@ -8,79 +9,13 @@ * 2 of the License, or (at your option) any later version. */ -#include <asm/sockios.h> - -/* For setsockopt(2) */ -#define SOL_SOCKET 1 - -#define SO_DEBUG 1 -#define SO_REUSEADDR 2 -#define SO_TYPE 3 -#define SO_ERROR 4 -#define SO_DONTROUTE 5 -#define SO_BROADCAST 6 -#define SO_SNDBUF 7 -#define SO_RCVBUF 8 -#define SO_SNDBUFFORCE 32 -#define SO_RCVBUFFORCE 33 -#define SO_KEEPALIVE 9 -#define SO_OOBINLINE 10 -#define SO_NO_CHECK 11 -#define SO_PRIORITY 12 -#define SO_LINGER 13 -#define SO_BSDCOMPAT 14 -#define SO_REUSEPORT 15 #define SO_RCVLOWAT 16 #define SO_SNDLOWAT 17 -#define SO_RCVTIMEO 18 -#define SO_SNDTIMEO 19 +#define SO_RCVTIMEO_OLD 18 +#define SO_SNDTIMEO_OLD 19 #define SO_PASSCRED 20 #define SO_PEERCRED 21 -/* Security levels - as per NRL IPv6 - don't actually do anything */ -#define SO_SECURITY_AUTHENTICATION 22 -#define SO_SECURITY_ENCRYPTION_TRANSPORT 23 -#define SO_SECURITY_ENCRYPTION_NETWORK 24 - -#define SO_BINDTODEVICE 25 - -/* Socket filtering */ -#define SO_ATTACH_FILTER 26 -#define SO_DETACH_FILTER 27 -#define SO_GET_FILTER SO_ATTACH_FILTER - -#define SO_PEERNAME 28 -#define SO_TIMESTAMP 29 -#define SCM_TIMESTAMP SO_TIMESTAMP - -#define SO_ACCEPTCONN 30 - -#define SO_PEERSEC 31 -#define SO_PASSSEC 34 -#define SO_TIMESTAMPNS 35 -#define SCM_TIMESTAMPNS SO_TIMESTAMPNS - -#define SO_MARK 36 - -#define SO_TIMESTAMPING 37 -#define SCM_TIMESTAMPING SO_TIMESTAMPING - -#define SO_PROTOCOL 38 -#define SO_DOMAIN 39 - -#define SO_RXQ_OVFL 40 - -#define SO_WIFI_STATUS 41 -#define SCM_WIFI_STATUS SO_WIFI_STATUS -#define SO_PEEK_OFF 42 - -/* Instruct lower device to use last 4-bytes of skb data as FCS */ -#define SO_NOFCS 43 - -#define SO_LOCK_FILTER 44 - -#define SO_SELECT_ERR_QUEUE 45 - -#define SO_BUSY_POLL 46 +#include <asm-generic/socket.h> #endif /* _ASM_POWERPC_SOCKET_H */ diff --git a/arch/powerpc/include/uapi/asm/sockios.h b/arch/powerpc/include/uapi/asm/sockios.h deleted file mode 100644 index 55cef7675a31..000000000000 --- a/arch/powerpc/include/uapi/asm/sockios.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef _ASM_POWERPC_SOCKIOS_H -#define _ASM_POWERPC_SOCKIOS_H - -/* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ - -/* Socket-level I/O control calls. */ -#define FIOSETOWN 0x8901 -#define SIOCSPGRP 0x8902 -#define FIOGETOWN 0x8903 -#define SIOCGPGRP 0x8904 -#define SIOCATMARK 0x8905 -#define SIOCGSTAMP 0x8906 /* Get stamp (timeval) */ -#define SIOCGSTAMPNS 0x8907 /* Get stamp (timespec) */ - -#endif /* _ASM_POWERPC_SOCKIOS_H */ diff --git a/arch/powerpc/include/uapi/asm/spu_info.h b/arch/powerpc/include/uapi/asm/spu_info.h index ed071bf97707..45f97150587b 100644 --- a/arch/powerpc/include/uapi/asm/spu_info.h +++ b/arch/powerpc/include/uapi/asm/spu_info.h @@ -1,23 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ /* * SPU info structures * * (C) Copyright 2006 IBM Corp. * * Author: Dwayne Grant McConnell <decimal@us.ibm.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2, or (at your option) - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #ifndef _UAPI_SPU_INFO_H diff --git a/arch/powerpc/include/uapi/asm/stat.h b/arch/powerpc/include/uapi/asm/stat.h index 84880b80cc1c..d50901664239 100644 --- a/arch/powerpc/include/uapi/asm/stat.h +++ b/arch/powerpc/include/uapi/asm/stat.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ #ifndef _ASM_POWERPC_STAT_H #define _ASM_POWERPC_STAT_H /* @@ -28,18 +29,18 @@ struct __old_kernel_stat { struct stat { unsigned long st_dev; - ino_t st_ino; + __kernel_ino_t st_ino; #ifdef __powerpc64__ unsigned long st_nlink; - mode_t st_mode; + __kernel_mode_t st_mode; #else - mode_t st_mode; + __kernel_mode_t st_mode; unsigned short st_nlink; #endif - uid_t st_uid; - gid_t st_gid; + __kernel_uid32_t st_uid; + __kernel_gid32_t st_gid; unsigned long st_rdev; - off_t st_size; + long st_size; unsigned long st_blksize; unsigned long st_blocks; unsigned long st_atime; diff --git a/arch/powerpc/include/uapi/asm/statfs.h b/arch/powerpc/include/uapi/asm/statfs.h deleted file mode 100644 index 5244834583a4..000000000000 --- a/arch/powerpc/include/uapi/asm/statfs.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef _ASM_POWERPC_STATFS_H -#define _ASM_POWERPC_STATFS_H - -#include <asm-generic/statfs.h> - -#endif diff --git a/arch/powerpc/include/uapi/asm/swab.h b/arch/powerpc/include/uapi/asm/swab.h index b6c368aa5c05..17b16c44d20c 100644 --- a/arch/powerpc/include/uapi/asm/swab.h +++ b/arch/powerpc/include/uapi/asm/swab.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ /* * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License diff --git a/arch/powerpc/include/uapi/asm/termbits.h b/arch/powerpc/include/uapi/asm/termbits.h index 549d700e18f2..21dc86dcb2f1 100644 --- a/arch/powerpc/include/uapi/asm/termbits.h +++ b/arch/powerpc/include/uapi/asm/termbits.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ #ifndef _ASM_POWERPC_TERMBITS_H #define _ASM_POWERPC_TERMBITS_H @@ -8,8 +9,8 @@ * 2 of the License, or (at your option) any later version. */ -typedef unsigned char cc_t; -typedef unsigned int speed_t; +#include <asm-generic/termbits-common.h> + typedef unsigned int tcflag_t; /* @@ -63,115 +64,72 @@ struct ktermios { #define VDISCARD 16 /* c_iflag bits */ -#define IGNBRK 0000001 -#define BRKINT 0000002 -#define IGNPAR 0000004 -#define PARMRK 0000010 -#define INPCK 0000020 -#define ISTRIP 0000040 -#define INLCR 0000100 -#define IGNCR 0000200 -#define ICRNL 0000400 -#define IXON 0001000 -#define IXOFF 0002000 -#define IXANY 0004000 -#define IUCLC 0010000 -#define IMAXBEL 0020000 -#define IUTF8 0040000 +#define IXON 0x0200 +#define IXOFF 0x0400 +#define IUCLC 0x1000 +#define IMAXBEL 0x2000 +#define IUTF8 0x4000 /* c_oflag bits */ -#define OPOST 0000001 -#define ONLCR 0000002 -#define OLCUC 0000004 - -#define OCRNL 0000010 -#define ONOCR 0000020 -#define ONLRET 0000040 - -#define OFILL 00000100 -#define OFDEL 00000200 -#define NLDLY 00001400 -#define NL0 00000000 -#define NL1 00000400 -#define NL2 00001000 -#define NL3 00001400 -#define TABDLY 00006000 -#define TAB0 00000000 -#define TAB1 00002000 -#define TAB2 00004000 -#define TAB3 00006000 -#define XTABS 00006000 /* required by POSIX to == TAB3 */ -#define CRDLY 00030000 -#define CR0 00000000 -#define CR1 00010000 -#define CR2 00020000 -#define CR3 00030000 -#define FFDLY 00040000 -#define FF0 00000000 -#define FF1 00040000 -#define BSDLY 00100000 -#define BS0 00000000 -#define BS1 00100000 -#define VTDLY 00200000 -#define VT0 00000000 -#define VT1 00200000 +#define ONLCR 0x00002 +#define OLCUC 0x00004 +#define NLDLY 0x00300 +#define NL0 0x00000 +#define NL1 0x00100 +#define NL2 0x00200 +#define NL3 0x00300 +#define TABDLY 0x00c00 +#define TAB0 0x00000 +#define TAB1 0x00400 +#define TAB2 0x00800 +#define TAB3 0x00c00 +#define XTABS 0x00c00 /* required by POSIX to == TAB3 */ +#define CRDLY 0x03000 +#define CR0 0x00000 +#define CR1 0x01000 +#define CR2 0x02000 +#define CR3 0x03000 +#define FFDLY 0x04000 +#define FF0 0x00000 +#define FF1 0x04000 +#define BSDLY 0x08000 +#define BS0 0x00000 +#define BS1 0x08000 +#define VTDLY 0x10000 +#define VT0 0x00000 +#define VT1 0x10000 /* c_cflag bit meaning */ -#define CBAUD 0000377 -#define B0 0000000 /* hang up */ -#define B50 0000001 -#define B75 0000002 -#define B110 0000003 -#define B134 0000004 -#define B150 0000005 -#define B200 0000006 -#define B300 0000007 -#define B600 0000010 -#define B1200 0000011 -#define B1800 0000012 -#define B2400 0000013 -#define B4800 0000014 -#define B9600 0000015 -#define B19200 0000016 -#define B38400 0000017 -#define EXTA B19200 -#define EXTB B38400 -#define CBAUDEX 0000000 -#define B57600 00020 -#define B115200 00021 -#define B230400 00022 -#define B460800 00023 -#define B500000 00024 -#define B576000 00025 -#define B921600 00026 -#define B1000000 00027 -#define B1152000 00030 -#define B1500000 00031 -#define B2000000 00032 -#define B2500000 00033 -#define B3000000 00034 -#define B3500000 00035 -#define B4000000 00036 -#define BOTHER 00037 - -#define CIBAUD 077600000 -#define IBSHIFT 16 /* Shift from CBAUD to CIBAUD */ - -#define CSIZE 00001400 -#define CS5 00000000 -#define CS6 00000400 -#define CS7 00001000 -#define CS8 00001400 - -#define CSTOPB 00002000 -#define CREAD 00004000 -#define PARENB 00010000 -#define PARODD 00020000 -#define HUPCL 00040000 - -#define CLOCAL 00100000 -#define CMSPAR 010000000000 /* mark or space (stick) parity */ -#define CRTSCTS 020000000000 /* flow control */ +#define CBAUD 0x000000ff +#define CBAUDEX 0x00000000 +#define BOTHER 0x0000001f +#define B57600 0x00000010 +#define B115200 0x00000011 +#define B230400 0x00000012 +#define B460800 0x00000013 +#define B500000 0x00000014 +#define B576000 0x00000015 +#define B921600 0x00000016 +#define B1000000 0x00000017 +#define B1152000 0x00000018 +#define B1500000 0x00000019 +#define B2000000 0x0000001a +#define B2500000 0x0000001b +#define B3000000 0x0000001c +#define B3500000 0x0000001d +#define B4000000 0x0000001e +#define CSIZE 0x00000300 +#define CS5 0x00000000 +#define CS6 0x00000100 +#define CS7 0x00000200 +#define CS8 0x00000300 +#define CSTOPB 0x00000400 +#define CREAD 0x00000800 +#define PARENB 0x00001000 +#define PARODD 0x00002000 +#define HUPCL 0x00004000 +#define CLOCAL 0x00008000 +#define CIBAUD 0x00ff0000 /* c_lflag bits */ #define ISIG 0x00000080 @@ -191,17 +149,6 @@ struct ktermios { #define IEXTEN 0x00000400 #define EXTPROC 0x10000000 -/* Values for the ACTION argument to `tcflow'. */ -#define TCOOFF 0 -#define TCOON 1 -#define TCIOFF 2 -#define TCION 3 - -/* Values for the QUEUE_SELECTOR argument to `tcflush'. */ -#define TCIFLUSH 0 -#define TCOFLUSH 1 -#define TCIOFLUSH 2 - /* Values for the OPTIONAL_ACTIONS argument to `tcsetattr'. */ #define TCSANOW 0 #define TCSADRAIN 1 diff --git a/arch/powerpc/include/uapi/asm/termios.h b/arch/powerpc/include/uapi/asm/termios.h index 6cca5cdfec04..5d07fc89bcb6 100644 --- a/arch/powerpc/include/uapi/asm/termios.h +++ b/arch/powerpc/include/uapi/asm/termios.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ /* * Liberally adapted from alpha/termios.h. In particular, the c_cc[] * fields have been reordered so that termio & termios share the diff --git a/arch/powerpc/include/uapi/asm/tm.h b/arch/powerpc/include/uapi/asm/tm.h index 85059a00f560..e1bf0e2fac43 100644 --- a/arch/powerpc/include/uapi/asm/tm.h +++ b/arch/powerpc/include/uapi/asm/tm.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _ASM_POWERPC_TM_H #define _ASM_POWERPC_TM_H @@ -6,10 +7,12 @@ * the failure is persistent. PAPR saves 0xff-0xe0 for the hypervisor. */ #define TM_CAUSE_PERSISTENT 0x01 +#define TM_CAUSE_KVM_RESCHED 0xe0 /* From PAPR */ +#define TM_CAUSE_KVM_FAC_UNAV 0xe2 /* From PAPR */ #define TM_CAUSE_RESCHED 0xde #define TM_CAUSE_TLBI 0xdc #define TM_CAUSE_FAC_UNAV 0xda -#define TM_CAUSE_SYSCALL 0xd8 /* future use */ +#define TM_CAUSE_SYSCALL 0xd8 #define TM_CAUSE_MISC 0xd6 /* future use */ #define TM_CAUSE_SIGNAL 0xd4 #define TM_CAUSE_ALIGNMENT 0xd2 diff --git a/arch/powerpc/include/uapi/asm/types.h b/arch/powerpc/include/uapi/asm/types.h index 4b8ab990a3c1..9dbf55e38ea5 100644 --- a/arch/powerpc/include/uapi/asm/types.h +++ b/arch/powerpc/include/uapi/asm/types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ /* * This file is never included by application software unless * explicitly requested (e.g., via linux/types.h) in which case the @@ -27,14 +28,14 @@ # include <asm-generic/int-ll64.h> #endif -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ typedef struct { __u32 u[4]; } __attribute__((aligned(16))) __vector128; -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _UAPI_ASM_POWERPC_TYPES_H */ diff --git a/arch/powerpc/include/uapi/asm/ucontext.h b/arch/powerpc/include/uapi/asm/ucontext.h index d9a4ddf0cc86..6f14a96d4985 100644 --- a/arch/powerpc/include/uapi/asm/ucontext.h +++ b/arch/powerpc/include/uapi/asm/ucontext.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _ASM_POWERPC_UCONTEXT_H #define _ASM_POWERPC_UCONTEXT_H diff --git a/arch/powerpc/include/uapi/asm/unistd.h b/arch/powerpc/include/uapi/asm/unistd.h index 74cb4d72d673..5f84e3dc98d0 100644 --- a/arch/powerpc/include/uapi/asm/unistd.h +++ b/arch/powerpc/include/uapi/asm/unistd.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ /* * This file contains the system call numbers. * @@ -9,374 +10,10 @@ #ifndef _UAPI_ASM_POWERPC_UNISTD_H_ #define _UAPI_ASM_POWERPC_UNISTD_H_ - -#define __NR_restart_syscall 0 -#define __NR_exit 1 -#define __NR_fork 2 -#define __NR_read 3 -#define __NR_write 4 -#define __NR_open 5 -#define __NR_close 6 -#define __NR_waitpid 7 -#define __NR_creat 8 -#define __NR_link 9 -#define __NR_unlink 10 -#define __NR_execve 11 -#define __NR_chdir 12 -#define __NR_time 13 -#define __NR_mknod 14 -#define __NR_chmod 15 -#define __NR_lchown 16 -#define __NR_break 17 -#define __NR_oldstat 18 -#define __NR_lseek 19 -#define __NR_getpid 20 -#define __NR_mount 21 -#define __NR_umount 22 -#define __NR_setuid 23 -#define __NR_getuid 24 -#define __NR_stime 25 -#define __NR_ptrace 26 -#define __NR_alarm 27 -#define __NR_oldfstat 28 -#define __NR_pause 29 -#define __NR_utime 30 -#define __NR_stty 31 -#define __NR_gtty 32 -#define __NR_access 33 -#define __NR_nice 34 -#define __NR_ftime 35 -#define __NR_sync 36 -#define __NR_kill 37 -#define __NR_rename 38 -#define __NR_mkdir 39 -#define __NR_rmdir 40 -#define __NR_dup 41 -#define __NR_pipe 42 -#define __NR_times 43 -#define __NR_prof 44 -#define __NR_brk 45 -#define __NR_setgid 46 -#define __NR_getgid 47 -#define __NR_signal 48 -#define __NR_geteuid 49 -#define __NR_getegid 50 -#define __NR_acct 51 -#define __NR_umount2 52 -#define __NR_lock 53 -#define __NR_ioctl 54 -#define __NR_fcntl 55 -#define __NR_mpx 56 -#define __NR_setpgid 57 -#define __NR_ulimit 58 -#define __NR_oldolduname 59 -#define __NR_umask 60 -#define __NR_chroot 61 -#define __NR_ustat 62 -#define __NR_dup2 63 -#define __NR_getppid 64 -#define __NR_getpgrp 65 -#define __NR_setsid 66 -#define __NR_sigaction 67 -#define __NR_sgetmask 68 -#define __NR_ssetmask 69 -#define __NR_setreuid 70 -#define __NR_setregid 71 -#define __NR_sigsuspend 72 -#define __NR_sigpending 73 -#define __NR_sethostname 74 -#define __NR_setrlimit 75 -#define __NR_getrlimit 76 -#define __NR_getrusage 77 -#define __NR_gettimeofday 78 -#define __NR_settimeofday 79 -#define __NR_getgroups 80 -#define __NR_setgroups 81 -#define __NR_select 82 -#define __NR_symlink 83 -#define __NR_oldlstat 84 -#define __NR_readlink 85 -#define __NR_uselib 86 -#define __NR_swapon 87 -#define __NR_reboot 88 -#define __NR_readdir 89 -#define __NR_mmap 90 -#define __NR_munmap 91 -#define __NR_truncate 92 -#define __NR_ftruncate 93 -#define __NR_fchmod 94 -#define __NR_fchown 95 -#define __NR_getpriority 96 -#define __NR_setpriority 97 -#define __NR_profil 98 -#define __NR_statfs 99 -#define __NR_fstatfs 100 -#define __NR_ioperm 101 -#define __NR_socketcall 102 -#define __NR_syslog 103 -#define __NR_setitimer 104 -#define __NR_getitimer 105 -#define __NR_stat 106 -#define __NR_lstat 107 -#define __NR_fstat 108 -#define __NR_olduname 109 -#define __NR_iopl 110 -#define __NR_vhangup 111 -#define __NR_idle 112 -#define __NR_vm86 113 -#define __NR_wait4 114 -#define __NR_swapoff 115 -#define __NR_sysinfo 116 -#define __NR_ipc 117 -#define __NR_fsync 118 -#define __NR_sigreturn 119 -#define __NR_clone 120 -#define __NR_setdomainname 121 -#define __NR_uname 122 -#define __NR_modify_ldt 123 -#define __NR_adjtimex 124 -#define __NR_mprotect 125 -#define __NR_sigprocmask 126 -#define __NR_create_module 127 -#define __NR_init_module 128 -#define __NR_delete_module 129 -#define __NR_get_kernel_syms 130 -#define __NR_quotactl 131 -#define __NR_getpgid 132 -#define __NR_fchdir 133 -#define __NR_bdflush 134 -#define __NR_sysfs 135 -#define __NR_personality 136 -#define __NR_afs_syscall 137 /* Syscall for Andrew File System */ -#define __NR_setfsuid 138 -#define __NR_setfsgid 139 -#define __NR__llseek 140 -#define __NR_getdents 141 -#define __NR__newselect 142 -#define __NR_flock 143 -#define __NR_msync 144 -#define __NR_readv 145 -#define __NR_writev 146 -#define __NR_getsid 147 -#define __NR_fdatasync 148 -#define __NR__sysctl 149 -#define __NR_mlock 150 -#define __NR_munlock 151 -#define __NR_mlockall 152 -#define __NR_munlockall 153 -#define __NR_sched_setparam 154 -#define __NR_sched_getparam 155 -#define __NR_sched_setscheduler 156 -#define __NR_sched_getscheduler 157 -#define __NR_sched_yield 158 -#define __NR_sched_get_priority_max 159 -#define __NR_sched_get_priority_min 160 -#define __NR_sched_rr_get_interval 161 -#define __NR_nanosleep 162 -#define __NR_mremap 163 -#define __NR_setresuid 164 -#define __NR_getresuid 165 -#define __NR_query_module 166 -#define __NR_poll 167 -#define __NR_nfsservctl 168 -#define __NR_setresgid 169 -#define __NR_getresgid 170 -#define __NR_prctl 171 -#define __NR_rt_sigreturn 172 -#define __NR_rt_sigaction 173 -#define __NR_rt_sigprocmask 174 -#define __NR_rt_sigpending 175 -#define __NR_rt_sigtimedwait 176 -#define __NR_rt_sigqueueinfo 177 -#define __NR_rt_sigsuspend 178 -#define __NR_pread64 179 -#define __NR_pwrite64 180 -#define __NR_chown 181 -#define __NR_getcwd 182 -#define __NR_capget 183 -#define __NR_capset 184 -#define __NR_sigaltstack 185 -#define __NR_sendfile 186 -#define __NR_getpmsg 187 /* some people actually want streams */ -#define __NR_putpmsg 188 /* some people actually want streams */ -#define __NR_vfork 189 -#define __NR_ugetrlimit 190 /* SuS compliant getrlimit */ -#define __NR_readahead 191 -#ifndef __powerpc64__ /* these are 32-bit only */ -#define __NR_mmap2 192 -#define __NR_truncate64 193 -#define __NR_ftruncate64 194 -#define __NR_stat64 195 -#define __NR_lstat64 196 -#define __NR_fstat64 197 -#endif -#define __NR_pciconfig_read 198 -#define __NR_pciconfig_write 199 -#define __NR_pciconfig_iobase 200 -#define __NR_multiplexer 201 -#define __NR_getdents64 202 -#define __NR_pivot_root 203 -#ifndef __powerpc64__ -#define __NR_fcntl64 204 -#endif -#define __NR_madvise 205 -#define __NR_mincore 206 -#define __NR_gettid 207 -#define __NR_tkill 208 -#define __NR_setxattr 209 -#define __NR_lsetxattr 210 -#define __NR_fsetxattr 211 -#define __NR_getxattr 212 -#define __NR_lgetxattr 213 -#define __NR_fgetxattr 214 -#define __NR_listxattr 215 -#define __NR_llistxattr 216 -#define __NR_flistxattr 217 -#define __NR_removexattr 218 -#define __NR_lremovexattr 219 -#define __NR_fremovexattr 220 -#define __NR_futex 221 -#define __NR_sched_setaffinity 222 -#define __NR_sched_getaffinity 223 -/* 224 currently unused */ -#define __NR_tuxcall 225 #ifndef __powerpc64__ -#define __NR_sendfile64 226 -#endif -#define __NR_io_setup 227 -#define __NR_io_destroy 228 -#define __NR_io_getevents 229 -#define __NR_io_submit 230 -#define __NR_io_cancel 231 -#define __NR_set_tid_address 232 -#define __NR_fadvise64 233 -#define __NR_exit_group 234 -#define __NR_lookup_dcookie 235 -#define __NR_epoll_create 236 -#define __NR_epoll_ctl 237 -#define __NR_epoll_wait 238 -#define __NR_remap_file_pages 239 -#define __NR_timer_create 240 -#define __NR_timer_settime 241 -#define __NR_timer_gettime 242 -#define __NR_timer_getoverrun 243 -#define __NR_timer_delete 244 -#define __NR_clock_settime 245 -#define __NR_clock_gettime 246 -#define __NR_clock_getres 247 -#define __NR_clock_nanosleep 248 -#define __NR_swapcontext 249 -#define __NR_tgkill 250 -#define __NR_utimes 251 -#define __NR_statfs64 252 -#define __NR_fstatfs64 253 -#ifndef __powerpc64__ -#define __NR_fadvise64_64 254 -#endif -#define __NR_rtas 255 -#define __NR_sys_debug_setcontext 256 -/* Number 257 is reserved for vserver */ -#define __NR_migrate_pages 258 -#define __NR_mbind 259 -#define __NR_get_mempolicy 260 -#define __NR_set_mempolicy 261 -#define __NR_mq_open 262 -#define __NR_mq_unlink 263 -#define __NR_mq_timedsend 264 -#define __NR_mq_timedreceive 265 -#define __NR_mq_notify 266 -#define __NR_mq_getsetattr 267 -#define __NR_kexec_load 268 -#define __NR_add_key 269 -#define __NR_request_key 270 -#define __NR_keyctl 271 -#define __NR_waitid 272 -#define __NR_ioprio_set 273 -#define __NR_ioprio_get 274 -#define __NR_inotify_init 275 -#define __NR_inotify_add_watch 276 -#define __NR_inotify_rm_watch 277 -#define __NR_spu_run 278 -#define __NR_spu_create 279 -#define __NR_pselect6 280 -#define __NR_ppoll 281 -#define __NR_unshare 282 -#define __NR_splice 283 -#define __NR_tee 284 -#define __NR_vmsplice 285 -#define __NR_openat 286 -#define __NR_mkdirat 287 -#define __NR_mknodat 288 -#define __NR_fchownat 289 -#define __NR_futimesat 290 -#ifdef __powerpc64__ -#define __NR_newfstatat 291 +#include <asm/unistd_32.h> #else -#define __NR_fstatat64 291 +#include <asm/unistd_64.h> #endif -#define __NR_unlinkat 292 -#define __NR_renameat 293 -#define __NR_linkat 294 -#define __NR_symlinkat 295 -#define __NR_readlinkat 296 -#define __NR_fchmodat 297 -#define __NR_faccessat 298 -#define __NR_get_robust_list 299 -#define __NR_set_robust_list 300 -#define __NR_move_pages 301 -#define __NR_getcpu 302 -#define __NR_epoll_pwait 303 -#define __NR_utimensat 304 -#define __NR_signalfd 305 -#define __NR_timerfd_create 306 -#define __NR_eventfd 307 -#define __NR_sync_file_range2 308 -#define __NR_fallocate 309 -#define __NR_subpage_prot 310 -#define __NR_timerfd_settime 311 -#define __NR_timerfd_gettime 312 -#define __NR_signalfd4 313 -#define __NR_eventfd2 314 -#define __NR_epoll_create1 315 -#define __NR_dup3 316 -#define __NR_pipe2 317 -#define __NR_inotify_init1 318 -#define __NR_perf_event_open 319 -#define __NR_preadv 320 -#define __NR_pwritev 321 -#define __NR_rt_tgsigqueueinfo 322 -#define __NR_fanotify_init 323 -#define __NR_fanotify_mark 324 -#define __NR_prlimit64 325 -#define __NR_socket 326 -#define __NR_bind 327 -#define __NR_connect 328 -#define __NR_listen 329 -#define __NR_accept 330 -#define __NR_getsockname 331 -#define __NR_getpeername 332 -#define __NR_socketpair 333 -#define __NR_send 334 -#define __NR_sendto 335 -#define __NR_recv 336 -#define __NR_recvfrom 337 -#define __NR_shutdown 338 -#define __NR_setsockopt 339 -#define __NR_getsockopt 340 -#define __NR_sendmsg 341 -#define __NR_recvmsg 342 -#define __NR_recvmmsg 343 -#define __NR_accept4 344 -#define __NR_name_to_handle_at 345 -#define __NR_open_by_handle_at 346 -#define __NR_clock_adjtime 347 -#define __NR_syncfs 348 -#define __NR_sendmmsg 349 -#define __NR_setns 350 -#define __NR_process_vm_readv 351 -#define __NR_process_vm_writev 352 -#define __NR_finit_module 353 -#define __NR_kcmp 354 - #endif /* _UAPI_ASM_POWERPC_UNISTD_H_ */ diff --git a/arch/powerpc/include/uapi/asm/vas-api.h b/arch/powerpc/include/uapi/asm/vas-api.h new file mode 100644 index 000000000000..7c81301ecdba --- /dev/null +++ b/arch/powerpc/include/uapi/asm/vas-api.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ +/* + * Copyright 2019 IBM Corp. + */ + +#ifndef _UAPI_MISC_VAS_H +#define _UAPI_MISC_VAS_H + +#include <linux/types.h> + +#include <asm/ioctl.h> + +#define VAS_MAGIC 'v' +#define VAS_TX_WIN_OPEN _IOW(VAS_MAGIC, 0x20, struct vas_tx_win_open_attr) + +/* Flags to VAS TX open window ioctl */ +/* To allocate a window with QoS credit, otherwise use default credit */ +#define VAS_TX_WIN_FLAG_QOS_CREDIT 0x0000000000000001 + +struct vas_tx_win_open_attr { + __u32 version; + __s16 vas_id; /* specific instance of vas or -1 for default */ + __u16 reserved1; + __u64 flags; + __u64 reserved2[6]; +}; + +#endif /* _UAPI_MISC_VAS_H */ |