diff options
| -rw-r--r-- | arch/powerpc/include/asm/interrupt.h | 52 | ||||
| -rw-r--r-- | arch/powerpc/kernel/smp.c | 22 | ||||
| -rw-r--r-- | arch/powerpc/kernel/traps.c | 6 | ||||
| -rw-r--r-- | arch/powerpc/lib/Makefile | 3 | ||||
| -rw-r--r-- | arch/powerpc/platforms/powernv/smp.c | 2 |
5 files changed, 59 insertions, 26 deletions
diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h index f964ef5c57d8..b14f54d789d2 100644 --- a/arch/powerpc/include/asm/interrupt.h +++ b/arch/powerpc/include/asm/interrupt.h @@ -324,22 +324,46 @@ static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct inte } #endif + /* If data relocations are enabled, it's safe to use nmi_enter() */ + if (mfmsr() & MSR_DR) { + nmi_enter(); + return; + } + /* - * Do not use nmi_enter() for pseries hash guest taking a real-mode + * But do not use nmi_enter() for pseries hash guest taking a real-mode * NMI because not everything it touches is within the RMA limit. */ - if (!IS_ENABLED(CONFIG_PPC_BOOK3S_64) || - !firmware_has_feature(FW_FEATURE_LPAR) || - radix_enabled() || (mfmsr() & MSR_DR)) - nmi_enter(); + if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && + firmware_has_feature(FW_FEATURE_LPAR) && + !radix_enabled()) + return; + + /* + * Likewise, don't use it if we have some form of instrumentation (like + * KASAN shadow) that is not safe to access in real mode (even on radix) + */ + if (IS_ENABLED(CONFIG_KASAN)) + return; + + /* Otherwise, it should be safe to call it */ + nmi_enter(); } static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state) { - if (!IS_ENABLED(CONFIG_PPC_BOOK3S_64) || - !firmware_has_feature(FW_FEATURE_LPAR) || - radix_enabled() || (mfmsr() & MSR_DR)) + if (mfmsr() & MSR_DR) { + // nmi_exit if relocations are on nmi_exit(); + } else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && + firmware_has_feature(FW_FEATURE_LPAR) && + !radix_enabled()) { + // no nmi_exit for a pseries hash guest taking a real mode exception + } else if (IS_ENABLED(CONFIG_KASAN)) { + // no nmi_exit for KASAN in real mode + } else { + nmi_exit(); + } /* * nmi does not call nap_adjust_return because nmi should not create @@ -407,7 +431,8 @@ static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct inter * Specific handlers may have additional restrictions. */ #define DEFINE_INTERRUPT_HANDLER_RAW(func) \ -static __always_inline long ____##func(struct pt_regs *regs); \ +static __always_inline __no_sanitize_address __no_kcsan long \ +____##func(struct pt_regs *regs); \ \ interrupt_handler long func(struct pt_regs *regs) \ { \ @@ -421,7 +446,8 @@ interrupt_handler long func(struct pt_regs *regs) \ } \ NOKPROBE_SYMBOL(func); \ \ -static __always_inline long ____##func(struct pt_regs *regs) +static __always_inline __no_sanitize_address __no_kcsan long \ +____##func(struct pt_regs *regs) /** * DECLARE_INTERRUPT_HANDLER - Declare synchronous interrupt handler function @@ -541,7 +567,8 @@ static __always_inline void ____##func(struct pt_regs *regs) * body with a pair of curly brackets. */ #define DEFINE_INTERRUPT_HANDLER_NMI(func) \ -static __always_inline long ____##func(struct pt_regs *regs); \ +static __always_inline __no_sanitize_address __no_kcsan long \ +____##func(struct pt_regs *regs); \ \ interrupt_handler long func(struct pt_regs *regs) \ { \ @@ -558,7 +585,8 @@ interrupt_handler long func(struct pt_regs *regs) \ } \ NOKPROBE_SYMBOL(func); \ \ -static __always_inline long ____##func(struct pt_regs *regs) +static __always_inline __no_sanitize_address __no_kcsan long \ +____##func(struct pt_regs *regs) /* Interrupt handlers */ diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 4c4511b6a75d..4335efcb3184 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -411,32 +411,32 @@ static struct cpumask nmi_ipi_pending_mask; static bool nmi_ipi_busy = false; static void (*nmi_ipi_function)(struct pt_regs *) = NULL; -static void nmi_ipi_lock_start(unsigned long *flags) +noinstr static void nmi_ipi_lock_start(unsigned long *flags) { raw_local_irq_save(*flags); hard_irq_disable(); - while (atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1) { + while (arch_atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1) { raw_local_irq_restore(*flags); - spin_until_cond(atomic_read(&__nmi_ipi_lock) == 0); + spin_until_cond(arch_atomic_read(&__nmi_ipi_lock) == 0); raw_local_irq_save(*flags); hard_irq_disable(); } } -static void nmi_ipi_lock(void) +noinstr static void nmi_ipi_lock(void) { - while (atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1) - spin_until_cond(atomic_read(&__nmi_ipi_lock) == 0); + while (arch_atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1) + spin_until_cond(arch_atomic_read(&__nmi_ipi_lock) == 0); } -static void nmi_ipi_unlock(void) +noinstr static void nmi_ipi_unlock(void) { smp_mb(); - WARN_ON(atomic_read(&__nmi_ipi_lock) != 1); - atomic_set(&__nmi_ipi_lock, 0); + WARN_ON(arch_atomic_read(&__nmi_ipi_lock) != 1); + arch_atomic_set(&__nmi_ipi_lock, 0); } -static void nmi_ipi_unlock_end(unsigned long *flags) +noinstr static void nmi_ipi_unlock_end(unsigned long *flags) { nmi_ipi_unlock(); raw_local_irq_restore(*flags); @@ -445,7 +445,7 @@ static void nmi_ipi_unlock_end(unsigned long *flags) /* * Platform NMI handler calls this to ack */ -int smp_handle_nmi_ipi(struct pt_regs *regs) +noinstr int smp_handle_nmi_ipi(struct pt_regs *regs) { void (*fn)(struct pt_regs *) = NULL; unsigned long flags; diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c index a08bb7cefdc5..3aaa50e5c72f 100644 --- a/arch/powerpc/kernel/traps.c +++ b/arch/powerpc/kernel/traps.c @@ -393,7 +393,7 @@ void _exception(int signr, struct pt_regs *regs, int code, unsigned long addr) * Builds that do not support KVM could take this second option to increase * the recoverability of NMIs. */ -void hv_nmi_check_nonrecoverable(struct pt_regs *regs) +noinstr void hv_nmi_check_nonrecoverable(struct pt_regs *regs) { #ifdef CONFIG_PPC_POWERNV unsigned long kbase = (unsigned long)_stext; @@ -433,7 +433,9 @@ void hv_nmi_check_nonrecoverable(struct pt_regs *regs) return; nonrecoverable: - regs_set_unrecoverable(regs); + regs->msr &= ~MSR_RI; + local_paca->hsrr_valid = 0; + local_paca->srr_valid = 0; #endif } DEFINE_INTERRUPT_HANDLER_NMI(system_reset_exception) diff --git a/arch/powerpc/lib/Makefile b/arch/powerpc/lib/Makefile index 5d1881d2e39a..8560c912186d 100644 --- a/arch/powerpc/lib/Makefile +++ b/arch/powerpc/lib/Makefile @@ -13,6 +13,9 @@ CFLAGS_REMOVE_feature-fixups.o = $(CC_FLAGS_FTRACE) KASAN_SANITIZE_code-patching.o := n KASAN_SANITIZE_feature-fixups.o := n +# restart_table.o contains functions called in the NMI interrupt path +# which can be in real mode. Disable KASAN. +KASAN_SANITIZE_restart_table.o := n ifdef CONFIG_KASAN CFLAGS_code-patching.o += -DDISABLE_BRANCH_PROFILING diff --git a/arch/powerpc/platforms/powernv/smp.c b/arch/powerpc/platforms/powernv/smp.c index cbb67813cd5d..9e1a25398f98 100644 --- a/arch/powerpc/platforms/powernv/smp.c +++ b/arch/powerpc/platforms/powernv/smp.c @@ -345,7 +345,7 @@ static void __init pnv_smp_probe(void) } } -static int pnv_system_reset_exception(struct pt_regs *regs) +noinstr static int pnv_system_reset_exception(struct pt_regs *regs) { if (smp_handle_nmi_ipi(regs)) return 1; |