summaryrefslogtreecommitdiff
path: root/arch/arm64/kernel/mte.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm64/kernel/mte.c')
-rw-r--r--arch/arm64/kernel/mte.c67
1 files changed, 61 insertions, 6 deletions
diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c
index e5e801bc5312..f418ebc65f95 100644
--- a/arch/arm64/kernel/mte.c
+++ b/arch/arm64/kernel/mte.c
@@ -26,9 +26,12 @@
static DEFINE_PER_CPU_READ_MOSTLY(u64, mte_tcf_preferred);
#ifdef CONFIG_KASAN_HW_TAGS
-/* Whether the MTE asynchronous mode is enabled. */
-DEFINE_STATIC_KEY_FALSE(mte_async_mode);
-EXPORT_SYMBOL_GPL(mte_async_mode);
+/*
+ * The asynchronous and asymmetric MTE modes have the same behavior for
+ * store operations. This flag is set when either of these modes is enabled.
+ */
+DEFINE_STATIC_KEY_FALSE(mte_async_or_asymm_mode);
+EXPORT_SYMBOL_GPL(mte_async_or_asymm_mode);
#endif
static void mte_sync_page_tags(struct page *page, pte_t old_pte,
@@ -116,7 +119,7 @@ void mte_enable_kernel_sync(void)
* Make sure we enter this function when no PE has set
* async mode previously.
*/
- WARN_ONCE(system_uses_mte_async_mode(),
+ WARN_ONCE(system_uses_mte_async_or_asymm_mode(),
"MTE async mode enabled system wide!");
__mte_enable_kernel("synchronous", SCTLR_ELx_TCF_SYNC);
@@ -134,8 +137,34 @@ void mte_enable_kernel_async(void)
* mode in between sync and async, this strategy needs
* to be reviewed.
*/
- if (!system_uses_mte_async_mode())
- static_branch_enable(&mte_async_mode);
+ if (!system_uses_mte_async_or_asymm_mode())
+ static_branch_enable(&mte_async_or_asymm_mode);
+}
+
+void mte_enable_kernel_asymm(void)
+{
+ if (cpus_have_cap(ARM64_MTE_ASYMM)) {
+ __mte_enable_kernel("asymmetric", SCTLR_ELx_TCF_ASYMM);
+
+ /*
+ * MTE asymm mode behaves as async mode for store
+ * operations. The mode is set system wide by the
+ * first PE that executes this function.
+ *
+ * Note: If in future KASAN acquires a runtime switching
+ * mode in between sync and async, this strategy needs
+ * to be reviewed.
+ */
+ if (!system_uses_mte_async_or_asymm_mode())
+ static_branch_enable(&mte_async_or_asymm_mode);
+ } else {
+ /*
+ * If the CPU does not support MTE asymmetric mode the
+ * kernel falls back on synchronous mode which is the
+ * default for kasan=on.
+ */
+ mte_enable_kernel_sync();
+ }
}
#endif
@@ -179,6 +208,30 @@ static void mte_update_sctlr_user(struct task_struct *task)
task->thread.sctlr_user = sctlr;
}
+static void mte_update_gcr_excl(struct task_struct *task)
+{
+ /*
+ * SYS_GCR_EL1 will be set to current->thread.mte_ctrl value by
+ * mte_set_user_gcr() in kernel_exit, but only if KASAN is enabled.
+ */
+ if (kasan_hw_tags_enabled())
+ return;
+
+ write_sysreg_s(
+ ((task->thread.mte_ctrl >> MTE_CTRL_GCR_USER_EXCL_SHIFT) &
+ SYS_GCR_EL1_EXCL_MASK) | SYS_GCR_EL1_RRND,
+ SYS_GCR_EL1);
+}
+
+void __init kasan_hw_tags_enable(struct alt_instr *alt, __le32 *origptr,
+ __le32 *updptr, int nr_inst)
+{
+ BUG_ON(nr_inst != 1); /* Branch -> NOP */
+
+ if (kasan_hw_tags_enabled())
+ *updptr = cpu_to_le32(aarch64_insn_gen_nop());
+}
+
void mte_thread_init_user(void)
{
if (!system_supports_mte())
@@ -198,6 +251,7 @@ void mte_thread_switch(struct task_struct *next)
return;
mte_update_sctlr_user(next);
+ mte_update_gcr_excl(next);
/*
* Check if an async tag exception occurred at EL1.
@@ -243,6 +297,7 @@ long set_mte_ctrl(struct task_struct *task, unsigned long arg)
if (task == current) {
preempt_disable();
mte_update_sctlr_user(task);
+ mte_update_gcr_excl(task);
update_sctlr_el1(task->thread.sctlr_user);
preempt_enable();
}