summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/arm64/include/asm/cpufeature.h3
-rw-r--r--arch/arm64/include/asm/exception.h7
-rw-r--r--arch/arm64/include/asm/spectre.h2
-rw-r--r--arch/arm64/include/asm/traps.h19
-rw-r--r--arch/arm64/kernel/armv8_deprecated.c567
-rw-r--r--arch/arm64/kernel/cpufeature.c23
-rw-r--r--arch/arm64/kernel/entry-common.c8
-rw-r--r--arch/arm64/kernel/proton-pack.c26
-rw-r--r--arch/arm64/kernel/traps.c93
9 files changed, 345 insertions, 403 deletions
diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index f73f11b55042..03d1c9d7af82 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -832,7 +832,8 @@ static inline bool system_supports_tlb_range(void)
cpus_have_const_cap(ARM64_HAS_TLB_RANGE);
}
-extern int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt);
+int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt);
+bool try_emulate_mrs(struct pt_regs *regs, u32 isn);
static inline u32 id_aa64mmfr0_parange_to_phys_shift(int parange)
{
diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h
index 19713d0f013b..92963f98afec 100644
--- a/arch/arm64/include/asm/exception.h
+++ b/arch/arm64/include/asm/exception.h
@@ -58,7 +58,8 @@ asmlinkage void call_on_irq_stack(struct pt_regs *regs,
asmlinkage void asm_exit_to_user_mode(struct pt_regs *regs);
void do_mem_abort(unsigned long far, unsigned long esr, struct pt_regs *regs);
-void do_undefinstr(struct pt_regs *regs, unsigned long esr);
+void do_el0_undef(struct pt_regs *regs, unsigned long esr);
+void do_el1_undef(struct pt_regs *regs, unsigned long esr);
void do_el0_bti(struct pt_regs *regs);
void do_el1_bti(struct pt_regs *regs, unsigned long esr);
void do_debug_exception(unsigned long addr_if_watchpoint, unsigned long esr,
@@ -67,10 +68,10 @@ void do_fpsimd_acc(unsigned long esr, struct pt_regs *regs);
void do_sve_acc(unsigned long esr, struct pt_regs *regs);
void do_sme_acc(unsigned long esr, struct pt_regs *regs);
void do_fpsimd_exc(unsigned long esr, struct pt_regs *regs);
-void do_sysinstr(unsigned long esr, struct pt_regs *regs);
+void do_el0_sys(unsigned long esr, struct pt_regs *regs);
void do_sp_pc_abort(unsigned long addr, unsigned long esr, struct pt_regs *regs);
void bad_el0_sync(struct pt_regs *regs, int reason, unsigned long esr);
-void do_cp15instr(unsigned long esr, struct pt_regs *regs);
+void do_el0_cp15(unsigned long esr, struct pt_regs *regs);
int do_compat_alignment_fixup(unsigned long addr, struct pt_regs *regs);
void do_el0_svc(struct pt_regs *regs);
void do_el0_svc_compat(struct pt_regs *regs);
diff --git a/arch/arm64/include/asm/spectre.h b/arch/arm64/include/asm/spectre.h
index aa3d3607d5c8..db7b371b367c 100644
--- a/arch/arm64/include/asm/spectre.h
+++ b/arch/arm64/include/asm/spectre.h
@@ -26,6 +26,7 @@ enum mitigation_state {
SPECTRE_VULNERABLE,
};
+struct pt_regs;
struct task_struct;
/*
@@ -98,5 +99,6 @@ enum mitigation_state arm64_get_spectre_bhb_state(void);
bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, int scope);
u8 spectre_bhb_loop_affected(int scope);
void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *__unused);
+bool try_emulate_el1_ssbs(struct pt_regs *regs, u32 instr);
#endif /* __ASSEMBLY__ */
#endif /* __ASM_SPECTRE_H */
diff --git a/arch/arm64/include/asm/traps.h b/arch/arm64/include/asm/traps.h
index 6e5826470bea..1f361e2da516 100644
--- a/arch/arm64/include/asm/traps.h
+++ b/arch/arm64/include/asm/traps.h
@@ -13,17 +13,16 @@
struct pt_regs;
-struct undef_hook {
- struct list_head node;
- u32 instr_mask;
- u32 instr_val;
- u64 pstate_mask;
- u64 pstate_val;
- int (*fn)(struct pt_regs *regs, u32 instr);
-};
+#ifdef CONFIG_ARMV8_DEPRECATED
+bool try_emulate_armv8_deprecated(struct pt_regs *regs, u32 insn);
+#else
+static inline bool
+try_emulate_armv8_deprecated(struct pt_regs *regs, u32 insn)
+{
+ return false;
+}
+#endif /* CONFIG_ARMV8_DEPRECATED */
-void register_undef_hook(struct undef_hook *hook);
-void unregister_undef_hook(struct undef_hook *hook);
void force_signal_inject(int signal, int code, unsigned long address, unsigned long err);
void arm64_notify_segfault(unsigned long addr);
void arm64_force_sig_fault(int signo, int code, unsigned long far, const char *str);
diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c
index fb0e7c7b2e20..8a9052cf3013 100644
--- a/arch/arm64/kernel/armv8_deprecated.c
+++ b/arch/arm64/kernel/armv8_deprecated.c
@@ -17,7 +17,6 @@
#include <asm/sysreg.h>
#include <asm/system_misc.h>
#include <asm/traps.h>
-#include <asm/kprobes.h>
#define CREATE_TRACE_POINTS
#include "trace-events-emulation.h"
@@ -39,226 +38,46 @@ enum insn_emulation_mode {
enum legacy_insn_status {
INSN_DEPRECATED,
INSN_OBSOLETE,
-};
-
-struct insn_emulation_ops {
- const char *name;
- enum legacy_insn_status status;
- struct undef_hook *hooks;
- int (*set_hw_mode)(bool enable);
+ INSN_UNAVAILABLE,
};
struct insn_emulation {
- struct list_head node;
- struct insn_emulation_ops *ops;
+ const char *name;
+ enum legacy_insn_status status;
+ bool (*try_emulate)(struct pt_regs *regs,
+ u32 insn);
+ int (*set_hw_mode)(bool enable);
+
int current_mode;
int min;
int max;
-};
-
-static LIST_HEAD(insn_emulation);
-static int nr_insn_emulated __initdata;
-static DEFINE_RAW_SPINLOCK(insn_emulation_lock);
-static DEFINE_MUTEX(insn_emulation_mutex);
-
-static void register_emulation_hooks(struct insn_emulation_ops *ops)
-{
- struct undef_hook *hook;
-
- BUG_ON(!ops->hooks);
-
- for (hook = ops->hooks; hook->instr_mask; hook++)
- register_undef_hook(hook);
-
- pr_notice("Registered %s emulation handler\n", ops->name);
-}
-
-static void remove_emulation_hooks(struct insn_emulation_ops *ops)
-{
- struct undef_hook *hook;
-
- BUG_ON(!ops->hooks);
-
- for (hook = ops->hooks; hook->instr_mask; hook++)
- unregister_undef_hook(hook);
-
- pr_notice("Removed %s emulation handler\n", ops->name);
-}
-
-static void enable_insn_hw_mode(void *data)
-{
- struct insn_emulation *insn = (struct insn_emulation *)data;
- if (insn->ops->set_hw_mode)
- insn->ops->set_hw_mode(true);
-}
-
-static void disable_insn_hw_mode(void *data)
-{
- struct insn_emulation *insn = (struct insn_emulation *)data;
- if (insn->ops->set_hw_mode)
- insn->ops->set_hw_mode(false);
-}
-
-/* Run set_hw_mode(mode) on all active CPUs */
-static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable)
-{
- if (!insn->ops->set_hw_mode)
- return -EINVAL;
- if (enable)
- on_each_cpu(enable_insn_hw_mode, (void *)insn, true);
- else
- on_each_cpu(disable_insn_hw_mode, (void *)insn, true);
- return 0;
-}
-
-/*
- * Run set_hw_mode for all insns on a starting CPU.
- * Returns:
- * 0 - If all the hooks ran successfully.
- * -EINVAL - At least one hook is not supported by the CPU.
- */
-static int run_all_insn_set_hw_mode(unsigned int cpu)
-{
- int rc = 0;
- unsigned long flags;
- struct insn_emulation *insn;
-
- raw_spin_lock_irqsave(&insn_emulation_lock, flags);
- list_for_each_entry(insn, &insn_emulation, node) {
- bool enable = (insn->current_mode == INSN_HW);
- if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(enable)) {
- pr_warn("CPU[%u] cannot support the emulation of %s",
- cpu, insn->ops->name);
- rc = -EINVAL;
- }
- }
- raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
- return rc;
-}
-
-static int update_insn_emulation_mode(struct insn_emulation *insn,
- enum insn_emulation_mode prev)
-{
- int ret = 0;
-
- switch (prev) {
- case INSN_UNDEF: /* Nothing to be done */
- break;
- case INSN_EMULATE:
- remove_emulation_hooks(insn->ops);
- break;
- case INSN_HW:
- if (!run_all_cpu_set_hw_mode(insn, false))
- pr_notice("Disabled %s support\n", insn->ops->name);
- break;
- }
-
- switch (insn->current_mode) {
- case INSN_UNDEF:
- break;
- case INSN_EMULATE:
- register_emulation_hooks(insn->ops);
- break;
- case INSN_HW:
- ret = run_all_cpu_set_hw_mode(insn, true);
- if (!ret)
- pr_notice("Enabled %s support\n", insn->ops->name);
- break;
- }
- return ret;
-}
-
-static void __init register_insn_emulation(struct insn_emulation_ops *ops)
-{
- unsigned long flags;
- struct insn_emulation *insn;
-
- insn = kzalloc(sizeof(*insn), GFP_KERNEL);
- if (!insn)
- return;
-
- insn->ops = ops;
- insn->min = INSN_UNDEF;
-
- switch (ops->status) {
- case INSN_DEPRECATED:
- insn->current_mode = INSN_EMULATE;
- /* Disable the HW mode if it was turned on at early boot time */
- run_all_cpu_set_hw_mode(insn, false);
- insn->max = INSN_HW;
- break;
- case INSN_OBSOLETE:
- insn->current_mode = INSN_UNDEF;
- insn->max = INSN_EMULATE;
- break;
- }
-
- raw_spin_lock_irqsave(&insn_emulation_lock, flags);
- list_add(&insn->node, &insn_emulation);
- nr_insn_emulated++;
- raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
-
- /* Register any handlers if required */
- update_insn_emulation_mode(insn, INSN_UNDEF);
-}
-
-static int emulation_proc_handler(struct ctl_table *table, int write,
- void *buffer, size_t *lenp,
- loff_t *ppos)
-{
- int ret = 0;
- struct insn_emulation *insn = container_of(table->data, struct insn_emulation, current_mode);
- enum insn_emulation_mode prev_mode = insn->current_mode;
-
- mutex_lock(&insn_emulation_mutex);
- ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+ /*
+ * sysctl for this emulation + a sentinal entry.
+ */
+ struct ctl_table sysctl[2];
+};
- if (ret || !write || prev_mode == insn->current_mode)
- goto ret;
+#define ARM_OPCODE_CONDTEST_FAIL 0
+#define ARM_OPCODE_CONDTEST_PASS 1
+#define ARM_OPCODE_CONDTEST_UNCOND 2
- ret = update_insn_emulation_mode(insn, prev_mode);
- if (ret) {
- /* Mode change failed, revert to previous mode. */
- insn->current_mode = prev_mode;
- update_insn_emulation_mode(insn, INSN_UNDEF);
- }
-ret:
- mutex_unlock(&insn_emulation_mutex);
- return ret;
-}
+#define ARM_OPCODE_CONDITION_UNCOND 0xf
-static void __init register_insn_emulation_sysctl(void)
+static unsigned int __maybe_unused aarch32_check_condition(u32 opcode, u32 psr)
{
- unsigned long flags;
- int i = 0;
- struct insn_emulation *insn;
- struct ctl_table *insns_sysctl, *sysctl;
-
- insns_sysctl = kcalloc(nr_insn_emulated + 1, sizeof(*sysctl),
- GFP_KERNEL);
- if (!insns_sysctl)
- return;
-
- raw_spin_lock_irqsave(&insn_emulation_lock, flags);
- list_for_each_entry(insn, &insn_emulation, node) {
- sysctl = &insns_sysctl[i];
-
- sysctl->mode = 0644;
- sysctl->maxlen = sizeof(int);
+ u32 cc_bits = opcode >> 28;
- sysctl->procname = insn->ops->name;
- sysctl->data = &insn->current_mode;
- sysctl->extra1 = &insn->min;
- sysctl->extra2 = &insn->max;
- sysctl->proc_handler = emulation_proc_handler;
- i++;
+ if (cc_bits != ARM_OPCODE_CONDITION_UNCOND) {
+ if ((*aarch32_opcode_cond_checks[cc_bits])(psr))
+ return ARM_OPCODE_CONDTEST_PASS;
+ else
+ return ARM_OPCODE_CONDTEST_FAIL;
}
- raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
-
- register_sysctl("abi", insns_sysctl);
+ return ARM_OPCODE_CONDTEST_UNCOND;
}
+#ifdef CONFIG_SWP_EMULATION
/*
* Implement emulation of the SWP/SWPB instructions using load-exclusive and
* store-exclusive.
@@ -339,25 +158,6 @@ static int emulate_swpX(unsigned int address, unsigned int *data,
return res;
}
-#define ARM_OPCODE_CONDTEST_FAIL 0
-#define ARM_OPCODE_CONDTEST_PASS 1
-#define ARM_OPCODE_CONDTEST_UNCOND 2
-
-#define ARM_OPCODE_CONDITION_UNCOND 0xf
-
-static unsigned int __kprobes aarch32_check_condition(u32 opcode, u32 psr)
-{
- u32 cc_bits = opcode >> 28;
-
- if (cc_bits != ARM_OPCODE_CONDITION_UNCOND) {
- if ((*aarch32_opcode_cond_checks[cc_bits])(psr))
- return ARM_OPCODE_CONDTEST_PASS;
- else
- return ARM_OPCODE_CONDTEST_FAIL;
- }
- return ARM_OPCODE_CONDTEST_UNCOND;
-}
-
/*
* swp_handler logs the id of calling process, dissects the instruction, sanity
* checks the memory location, calls emulate_swpX for the actual operation and
@@ -430,28 +230,27 @@ fault:
return 0;
}
-/*
- * Only emulate SWP/SWPB executed in ARM state/User mode.
- * The kernel must be SWP free and SWP{B} does not exist in Thumb.
- */
-static struct undef_hook swp_hooks[] = {
- {
- .instr_mask = 0x0fb00ff0,
- .instr_val = 0x01000090,
- .pstate_mask = PSR_AA32_MODE_MASK,
- .pstate_val = PSR_AA32_MODE_USR,
- .fn = swp_handler
- },
- { }
-};
+static bool try_emulate_swp(struct pt_regs *regs, u32 insn)
+{
+ /* SWP{B} only exists in ARM state and does not exist in Thumb */
+ if (!compat_user_mode(regs) || compat_thumb_mode(regs))
+ return false;
-static struct insn_emulation_ops swp_ops = {
+ if ((insn & 0x0fb00ff0) != 0x01000090)
+ return false;
+
+ return swp_handler(regs, insn) == 0;
+}
+
+static struct insn_emulation insn_swp = {
.name = "swp",
.status = INSN_OBSOLETE,
- .hooks = swp_hooks,
+ .try_emulate = try_emulate_swp,
.set_hw_mode = NULL,
};
+#endif /* CONFIG_SWP_EMULATION */
+#ifdef CONFIG_CP15_BARRIER_EMULATION
static int cp15barrier_handler(struct pt_regs *regs, u32 instr)
{
perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
@@ -514,31 +313,29 @@ static int cp15_barrier_set_hw_mode(bool enable)
return 0;
}
-static struct undef_hook cp15_barrier_hooks[] = {
- {
- .instr_mask = 0x0fff0fdf,
- .instr_val = 0x0e070f9a,
- .pstate_mask = PSR_AA32_MODE_MASK,
- .pstate_val = PSR_AA32_MODE_USR,
- .fn = cp15barrier_handler,
- },
- {
- .instr_mask = 0x0fff0fff,
- .instr_val = 0x0e070f95,
- .pstate_mask = PSR_AA32_MODE_MASK,
- .pstate_val = PSR_AA32_MODE_USR,
- .fn = cp15barrier_handler,
- },
- { }
-};
+static bool try_emulate_cp15_barrier(struct pt_regs *regs, u32 insn)
+{
+ if (!compat_user_mode(regs) || compat_thumb_mode(regs))
+ return false;
+
+ if ((insn & 0x0fff0fdf) == 0x0e070f9a)
+ return cp15barrier_handler(regs, insn) == 0;
+
+ if ((insn & 0x0fff0fff) == 0x0e070f95)
+ return cp15barrier_handler(regs, insn) == 0;
-static struct insn_emulation_ops cp15_barrier_ops = {
+ return false;
+}
+
+static struct insn_emulation insn_cp15_barrier = {
.name = "cp15_barrier",
.status = INSN_DEPRECATED,
- .hooks = cp15_barrier_hooks,
+ .try_emulate = try_emulate_cp15_barrier,
.set_hw_mode = cp15_barrier_set_hw_mode,
};
+#endif /* CONFIG_CP15_BARRIER_EMULATION */
+#ifdef CONFIG_SETEND_EMULATION
static int setend_set_hw_mode(bool enable)
{
if (!cpu_supports_mixed_endian_el0())
@@ -586,31 +383,218 @@ static int t16_setend_handler(struct pt_regs *regs, u32 instr)
return rc;
}
-static struct undef_hook setend_hooks[] = {
- {
- .instr_mask = 0xfffffdff,
- .instr_val = 0xf1010000,
- .pstate_mask = PSR_AA32_MODE_MASK,
- .pstate_val = PSR_AA32_MODE_USR,
- .fn = a32_setend_handler,
- },
- {
- /* Thumb mode */
- .instr_mask = 0xfffffff7,
- .instr_val = 0x0000b650,
- .pstate_mask = (PSR_AA32_T_BIT | PSR_AA32_MODE_MASK),
- .pstate_val = (PSR_AA32_T_BIT | PSR_AA32_MODE_USR),
- .fn = t16_setend_handler,
- },
- {}
-};
+static bool try_emulate_setend(struct pt_regs *regs, u32 insn)
+{
+ if (compat_thumb_mode(regs) &&
+ (insn & 0xfffffff7) == 0x0000b650)
+ return t16_setend_handler(regs, insn) == 0;
+
+ if (compat_user_mode(regs) &&
+ (insn & 0xfffffdff) == 0xf1010000)
+ return a32_setend_handler(regs, insn) == 0;
-static struct insn_emulation_ops setend_ops = {
+ return false;
+}
+
+static struct insn_emulation insn_setend = {
.name = "setend",
.status = INSN_DEPRECATED,
- .hooks = setend_hooks,
+ .try_emulate = try_emulate_setend,
.set_hw_mode = setend_set_hw_mode,
};
+#endif /* CONFIG_SETEND_EMULATION */
+
+static struct insn_emulation *insn_emulations[] = {
+#ifdef CONFIG_SWP_EMULATION
+ &insn_swp,
+#endif
+#ifdef CONFIG_CP15_BARRIER_EMULATION
+ &insn_cp15_barrier,
+#endif
+#ifdef CONFIG_SETEND_EMULATION
+ &insn_setend,
+#endif
+};
+
+static DEFINE_MUTEX(insn_emulation_mutex);
+
+static void enable_insn_hw_mode(void *data)
+{
+ struct insn_emulation *insn = (struct insn_emulation *)data;
+ if (insn->set_hw_mode)
+ insn->set_hw_mode(true);
+}
+
+static void disable_insn_hw_mode(void *data)
+{
+ struct insn_emulation *insn = (struct insn_emulation *)data;
+ if (insn->set_hw_mode)
+ insn->set_hw_mode(false);
+}
+
+/* Run set_hw_mode(mode) on all active CPUs */
+static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable)
+{
+ if (!insn->set_hw_mode)
+ return -EINVAL;
+ if (enable)
+ on_each_cpu(enable_insn_hw_mode, (void *)insn, true);
+ else
+ on_each_cpu(disable_insn_hw_mode, (void *)insn, true);
+ return 0;
+}
+
+/*
+ * Run set_hw_mode for all insns on a starting CPU.
+ * Returns:
+ * 0 - If all the hooks ran successfully.
+ * -EINVAL - At least one hook is not supported by the CPU.
+ */
+static int run_all_insn_set_hw_mode(unsigned int cpu)
+{
+ int rc = 0;
+ unsigned long flags;
+
+ /*
+ * Disable IRQs to serialize against an IPI from
+ * run_all_cpu_set_hw_mode(), ensuring the HW is programmed to the most
+ * recent enablement state if the two race with one another.
+ */
+ local_irq_save(flags);
+ for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
+ struct insn_emulation *insn = insn_emulations[i];
+ bool enable = READ_ONCE(insn->current_mode) == INSN_HW;
+ if (insn->set_hw_mode && insn->set_hw_mode(enable)) {
+ pr_warn("CPU[%u] cannot support the emulation of %s",
+ cpu, insn->name);
+ rc = -EINVAL;
+ }
+ }
+ local_irq_restore(flags);
+
+ return rc;
+}
+
+static int update_insn_emulation_mode(struct insn_emulation *insn,
+ enum insn_emulation_mode prev)
+{
+ int ret = 0;
+
+ switch (prev) {
+ case INSN_UNDEF: /* Nothing to be done */
+ break;
+ case INSN_EMULATE:
+ break;
+ case INSN_HW:
+ if (!run_all_cpu_set_hw_mode(insn, false))
+ pr_notice("Disabled %s support\n", insn->name);
+ break;
+ }
+
+ switch (insn->current_mode) {
+ case INSN_UNDEF:
+ break;
+ case INSN_EMULATE:
+ break;
+ case INSN_HW:
+ ret = run_all_cpu_set_hw_mode(insn, true);
+ if (!ret)
+ pr_notice("Enabled %s support\n", insn->name);
+ break;
+ }
+
+ return ret;
+}
+
+static int emulation_proc_handler(struct ctl_table *table, int write,
+ void *buffer, size_t *lenp,
+ loff_t *ppos)
+{
+ int ret = 0;
+ struct insn_emulation *insn = container_of(table->data, struct insn_emulation, current_mode);
+ enum insn_emulation_mode prev_mode = insn->current_mode;
+
+ mutex_lock(&insn_emulation_mutex);
+ ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+
+ if (ret || !write || prev_mode == insn->current_mode)
+ goto ret;
+
+ ret = update_insn_emulation_mode(insn, prev_mode);
+ if (ret) {
+ /* Mode change failed, revert to previous mode. */
+ WRITE_ONCE(insn->current_mode, prev_mode);
+ update_insn_emulation_mode(insn, INSN_UNDEF);
+ }
+ret:
+ mutex_unlock(&insn_emulation_mutex);
+ return ret;
+}
+
+static void __init register_insn_emulation(struct insn_emulation *insn)
+{
+ struct ctl_table *sysctl;
+
+ insn->min = INSN_UNDEF;
+
+ switch (insn->status) {
+ case INSN_DEPRECATED:
+ insn->current_mode = INSN_EMULATE;
+ /* Disable the HW mode if it was turned on at early boot time */
+ run_all_cpu_set_hw_mode(insn, false);
+ insn->max = INSN_HW;
+ break;
+ case INSN_OBSOLETE:
+ insn->current_mode = INSN_UNDEF;
+ insn->max = INSN_EMULATE;
+ break;
+ case INSN_UNAVAILABLE:
+ insn->current_mode = INSN_UNDEF;
+ insn->max = INSN_UNDEF;
+ break;
+ }
+
+ /* Program the HW if required */
+ update_insn_emulation_mode(insn, INSN_UNDEF);
+
+ if (insn->status != INSN_UNAVAILABLE) {
+ sysctl = &insn->sysctl[0];
+
+ sysctl->mode = 0644;
+ sysctl->maxlen = sizeof(int);
+
+ sysctl->procname = insn->name;
+ sysctl->data = &insn->current_mode;
+ sysctl->extra1 = &insn->min;
+ sysctl->extra2 = &insn->max;
+ sysctl->proc_handler = emulation_proc_handler;
+
+ register_sysctl("abi", sysctl);
+ }
+}
+
+bool try_emulate_armv8_deprecated(struct pt_regs *regs, u32 insn)
+{
+ for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
+ struct insn_emulation *ie = insn_emulations[i];
+
+ if (ie->status == INSN_UNAVAILABLE)
+ continue;
+
+ /*
+ * A trap may race with the mode being changed
+ * INSN_EMULATE<->INSN_HW. Try to emulate the instruction to
+ * avoid a spurious UNDEF.
+ */
+ if (READ_ONCE(ie->current_mode) == INSN_UNDEF)
+ continue;
+
+ if (ie->try_emulate(regs, insn))
+ return true;
+ }
+
+ return false;
+}
/*
* Invoked as core_initcall, which guarantees that the instruction
@@ -618,24 +602,25 @@ static struct insn_emulation_ops setend_ops = {
*/
static int __init armv8_deprecated_init(void)
{
- if (IS_ENABLED(CONFIG_SWP_EMULATION))
- register_insn_emulation(&swp_ops);
+#ifdef CONFIG_SETEND_EMULATION
+ if (!system_supports_mixed_endian_el0()) {
+ insn_setend.status = INSN_UNAVAILABLE;
+ pr_info("setend instruction emulation is not supported on this system\n");
+ }
- if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION))
- register_insn_emulation(&cp15_barrier_ops);
+#endif
+ for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
+ struct insn_emulation *ie = insn_emulations[i];
- if (IS_ENABLED(CONFIG_SETEND_EMULATION)) {
- if (system_supports_mixed_endian_el0())
- register_insn_emulation(&setend_ops);
- else
- pr_info("setend instruction emulation is not supported on this system\n");
+ if (ie->status == INSN_UNAVAILABLE)
+ continue;
+
+ register_insn_emulation(ie);
}
cpuhp_setup_state_nocalls(CPUHP_AP_ARM64_ISNDEP_STARTING,
"arm64/isndep:starting",
run_all_insn_set_hw_mode, NULL);
- register_insn_emulation_sysctl();
-
return 0;
}
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index ff74d78145bd..7e76e1fda2a1 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -3457,35 +3457,22 @@ int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt)
return rc;
}
-static int emulate_mrs(struct pt_regs *regs, u32 insn)
+bool try_emulate_mrs(struct pt_regs *regs, u32 insn)
{
u32 sys_reg, rt;
+ if (compat_user_mode(regs) || !aarch64_insn_is_mrs(insn))
+ return false;
+
/*
* sys_reg values are defined as used in mrs/msr instruction.
* shift the imm value to get the encoding.
*/
sys_reg = (u32)aarch64_insn_decode_immediate(AARCH64_INSN_IMM_16, insn) << 5;
rt = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn);
- return do_emulate_mrs(regs, sys_reg, rt);
+ return do_emulate_mrs(regs, sys_reg, rt) == 0;
}
-static struct undef_hook mrs_hook = {
- .instr_mask = 0xffff0000,
- .instr_val = 0xd5380000,
- .pstate_mask = PSR_AA32_MODE_MASK,
- .pstate_val = PSR_MODE_EL0t,
- .fn = emulate_mrs,
-};
-
-static int __init enable_mrs_emulation(void)
-{
- register_undef_hook(&mrs_hook);
- return 0;
-}
-
-core_initcall(enable_mrs_emulation);
-
enum mitigation_state arm64_get_meltdown_state(void)
{
if (__meltdown_safe)
diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index 9a553b185e63..cce1167199e3 100644
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -384,7 +384,7 @@ static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr)
{
enter_from_kernel_mode(regs);
local_daif_inherit(regs);
- do_undefinstr(regs, esr);
+ do_el1_undef(regs, esr);
local_daif_mask();
exit_to_kernel_mode(regs);
}
@@ -570,7 +570,7 @@ static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)
{
enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
- do_sysinstr(esr, regs);
+ do_el0_sys(esr, regs);
exit_to_user_mode(regs);
}
@@ -599,7 +599,7 @@ static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr)
{
enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
- do_undefinstr(regs, esr);
+ do_el0_undef(regs, esr);
exit_to_user_mode(regs);
}
@@ -762,7 +762,7 @@ static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)
{
enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
- do_cp15instr(esr, regs);
+ do_el0_cp15(esr, regs);
exit_to_user_mode(regs);
}
diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c
index bfce41c2a53b..fca9cc6f5581 100644
--- a/arch/arm64/kernel/proton-pack.c
+++ b/arch/arm64/kernel/proton-pack.c
@@ -521,10 +521,13 @@ bool has_spectre_v4(const struct arm64_cpu_capabilities *cap, int scope)
return state != SPECTRE_UNAFFECTED;
}
-static int ssbs_emulation_handler(struct pt_regs *regs, u32 instr)
+bool try_emulate_el1_ssbs(struct pt_regs *regs, u32 instr)
{
- if (user_mode(regs))
- return 1;
+ const u32 instr_mask = ~(1U << PSTATE_Imm_shift);
+ const u32 instr_val = 0xd500401f | PSTATE_SSBS;
+
+ if ((instr & instr_mask) != instr_val)
+ return false;
if (instr & BIT(PSTATE_Imm_shift))
regs->pstate |= PSR_SSBS_BIT;
@@ -532,19 +535,11 @@ static int ssbs_emulation_handler(struct pt_regs *regs, u32 instr)
regs->pstate &= ~PSR_SSBS_BIT;
arm64_skip_faulting_instruction(regs, 4);
- return 0;
+ return true;
}
-static struct undef_hook ssbs_emulation_hook = {
- .instr_mask = ~(1U << PSTATE_Imm_shift),
- .instr_val = 0xd500401f | PSTATE_SSBS,
- .fn = ssbs_emulation_handler,
-};
-
static enum mitigation_state spectre_v4_enable_hw_mitigation(void)
{
- static bool undef_hook_registered = false;
- static DEFINE_RAW_SPINLOCK(hook_lock);
enum mitigation_state state;
/*
@@ -555,13 +550,6 @@ static enum mitigation_state spectre_v4_enable_hw_mitigation(void)
if (state != SPECTRE_MITIGATED || !this_cpu_has_cap(ARM64_SSBS))
return state;
- raw_spin_lock(&hook_lock);
- if (!undef_hook_registered) {
- register_undef_hook(&ssbs_emulation_hook);
- undef_hook_registered = true;
- }
- raw_spin_unlock(&hook_lock);
-
if (spectre_v4_mitigations_off()) {
sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS);
set_pstate_ssbs(1);
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index 23d281ed7621..4c0caa589e12 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -373,51 +373,22 @@ void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size)
regs->pstate &= ~PSR_BTYPE_MASK;
}
-static LIST_HEAD(undef_hook);
-static DEFINE_RAW_SPINLOCK(undef_lock);
-
-void register_undef_hook(struct undef_hook *hook)
+static int user_insn_read(struct pt_regs *regs, u32 *insnp)
{
- unsigned long flags;
-
- raw_spin_lock_irqsave(&undef_lock, flags);
- list_add(&hook->node, &undef_hook);
- raw_spin_unlock_irqrestore(&undef_lock, flags);
-}
-
-void unregister_undef_hook(struct undef_hook *hook)
-{
- unsigned long flags;
-
- raw_spin_lock_irqsave(&undef_lock, flags);
- list_del(&hook->node);
- raw_spin_unlock_irqrestore(&undef_lock, flags);
-}
-
-static int call_undef_hook(struct pt_regs *regs)
-{
- struct undef_hook *hook;
- unsigned long flags;
u32 instr;
- int (*fn)(struct pt_regs *regs, u32 instr) = NULL;
unsigned long pc = instruction_pointer(regs);
- if (!user_mode(regs)) {
- __le32 instr_le;
- if (get_kernel_nofault(instr_le, (__le32 *)pc))
- goto exit;
- instr = le32_to_cpu(instr_le);
- } else if (compat_thumb_mode(regs)) {
+ if (compat_thumb_mode(regs)) {
/* 16-bit Thumb instruction */
__le16 instr_le;
if (get_user(instr_le, (__le16 __user *)pc))
- goto exit;
+ return -EFAULT;
instr = le16_to_cpu(instr_le);
if (aarch32_insn_is_wide(instr)) {
u32 instr2;
if (get_user(instr_le, (__le16 __user *)(pc + 2)))
- goto exit;
+ return -EFAULT;
instr2 = le16_to_cpu(instr_le);
instr = (instr << 16) | instr2;
}
@@ -425,19 +396,12 @@ static int call_undef_hook(struct pt_regs *regs)
/* 32-bit ARM instruction */
__le32 instr_le;
if (get_user(instr_le, (__le32 __user *)pc))
- goto exit;
+ return -EFAULT;
instr = le32_to_cpu(instr_le);
}
- raw_spin_lock_irqsave(&undef_lock, flags);
- list_for_each_entry(hook, &undef_hook, node)
- if ((instr & hook->instr_mask) == hook->instr_val &&
- (regs->pstate & hook->pstate_mask) == hook->pstate_val)
- fn = hook->fn;
-
- raw_spin_unlock_irqrestore(&undef_lock, flags);
-exit:
- return fn ? fn(regs, instr) : 1;
+ *insnp = instr;
+ return 0;
}
void force_signal_inject(int signal, int code, unsigned long address, unsigned long err)
@@ -486,21 +450,40 @@ void arm64_notify_segfault(unsigned long addr)
force_signal_inject(SIGSEGV, code, addr, 0);
}
-void do_undefinstr(struct pt_regs *regs, unsigned long esr)
+void do_el0_undef(struct pt_regs *regs, unsigned long esr)
{
+ u32 insn;
+
/* check for AArch32 breakpoint instructions */
if (!aarch32_break_handler(regs))
return;
- if (call_undef_hook(regs) == 0)
+ if (user_insn_read(regs, &insn))
+ goto out_err;
+
+ if (try_emulate_mrs(regs, insn))
return;
- if (!user_mode(regs))
- die("Oops - Undefined instruction", regs, esr);
+ if (try_emulate_armv8_deprecated(regs, insn))
+ return;
+out_err:
force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
}
-NOKPROBE_SYMBOL(do_undefinstr);
+
+void do_el1_undef(struct pt_regs *regs, unsigned long esr)
+{
+ u32 insn;
+
+ if (aarch64_insn_read((void *)regs->pc, &insn))
+ goto out_err;
+
+ if (try_emulate_el1_ssbs(regs, insn))
+ return;
+
+out_err:
+ die("Oops - Undefined instruction", regs, esr);
+}
void do_el0_bti(struct pt_regs *regs)
{
@@ -511,7 +494,6 @@ void do_el1_bti(struct pt_regs *regs, unsigned long esr)
{
die("Oops - BTI", regs, esr);
}
-NOKPROBE_SYMBOL(do_el1_bti);
void do_el0_fpac(struct pt_regs *regs, unsigned long esr)
{
@@ -526,7 +508,6 @@ void do_el1_fpac(struct pt_regs *regs, unsigned long esr)
*/
die("Oops - FPAC", regs, esr);
}
-NOKPROBE_SYMBOL(do_el1_fpac)
#define __user_cache_maint(insn, address, res) \
if (address >= TASK_SIZE_MAX) { \
@@ -748,7 +729,7 @@ static const struct sys64_hook cp15_64_hooks[] = {
{},
};
-void do_cp15instr(unsigned long esr, struct pt_regs *regs)
+void do_el0_cp15(unsigned long esr, struct pt_regs *regs)
{
const struct sys64_hook *hook, *hook_base;
@@ -769,7 +750,7 @@ void do_cp15instr(unsigned long esr, struct pt_regs *regs)
hook_base = cp15_64_hooks;
break;
default:
- do_undefinstr(regs, esr);
+ do_el0_undef(regs, esr);
return;
}
@@ -784,12 +765,11 @@ void do_cp15instr(unsigned long esr, struct pt_regs *regs)
* EL0. Fall back to our usual undefined instruction handler
* so that we handle these consistently.
*/
- do_undefinstr(regs, esr);
+ do_el0_undef(regs, esr);
}
-NOKPROBE_SYMBOL(do_cp15instr);
#endif
-void do_sysinstr(unsigned long esr, struct pt_regs *regs)
+void do_el0_sys(unsigned long esr, struct pt_regs *regs)
{
const struct sys64_hook *hook;
@@ -804,9 +784,8 @@ void do_sysinstr(unsigned long esr, struct pt_regs *regs)
* back to our usual undefined instruction handler so that we handle
* these consistently.
*/
- do_undefinstr(regs, esr);
+ do_el0_undef(regs, esr);
}
-NOKPROBE_SYMBOL(do_sysinstr);
static const char *esr_class_str[] = {
[0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC",
generated by cgit (git 2.34.1) at 2024-05-28 12:45:08 +0000