// SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (C) 2023 SiFive * Author: Andy Chiu */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static bool riscv_v_implicit_uacc = IS_ENABLED(CONFIG_RISCV_ISA_V_DEFAULT_ENABLE); unsigned long riscv_v_vsize __read_mostly; EXPORT_SYMBOL_GPL(riscv_v_vsize); int riscv_v_setup_vsize(void) { unsigned long this_vsize; /* There are 32 vector registers with vlenb length. */ riscv_v_enable(); this_vsize = csr_read(CSR_VLENB) * 32; riscv_v_disable(); if (!riscv_v_vsize) { riscv_v_vsize = this_vsize; return 0; } if (riscv_v_vsize != this_vsize) { WARN(1, "RISCV_ISA_V only supports one vlenb on SMP systems"); return -EOPNOTSUPP; } return 0; } static bool insn_is_vector(u32 insn_buf) { u32 opcode = insn_buf & __INSN_OPCODE_MASK; u32 width, csr; /* * All V-related instructions, including CSR operations are 4-Byte. So, * do not handle if the instruction length is not 4-Byte. */ if (unlikely(GET_INSN_LENGTH(insn_buf) != 4)) return false; switch (opcode) { case RVV_OPCODE_VECTOR: return true; case RVV_OPCODE_VL: case RVV_OPCODE_VS: width = RVV_EXRACT_VL_VS_WIDTH(insn_buf); if (width == RVV_VL_VS_WIDTH_8 || width == RVV_VL_VS_WIDTH_16 || width == RVV_VL_VS_WIDTH_32 || width == RVV_VL_VS_WIDTH_64) return true; break; case RVG_OPCODE_SYSTEM: csr = RVG_EXTRACT_SYSTEM_CSR(insn_buf); if ((csr >= CSR_VSTART && csr <= CSR_VCSR) || (csr >= CSR_VL && csr <= CSR_VLENB)) return true; } return false; } static int riscv_v_thread_zalloc(void) { void *datap; datap = kzalloc(riscv_v_vsize, GFP_KERNEL); if (!datap) return -ENOMEM; current->thread.vstate.datap = datap; memset(¤t->thread.vstate, 0, offsetof(struct __riscv_v_ext_state, datap)); return 0; } #define VSTATE_CTRL_GET_CUR(x) ((x) & PR_RISCV_V_VSTATE_CTRL_CUR_MASK) #define VSTATE_CTRL_GET_NEXT(x) (((x) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) >> 2) #define VSTATE_CTRL_MAKE_NEXT(x) (((x) << 2) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) #define VSTATE_CTRL_GET_INHERIT(x) (!!((x) & PR_RISCV_V_VSTATE_CTRL_INHERIT)) static inline int riscv_v_ctrl_get_cur(struct task_struct *tsk) { return VSTATE_CTRL_GET_CUR(tsk->thread.vstate_ctrl); } static inline int riscv_v_ctrl_get_next(struct task_struct *tsk) { return VSTATE_CTRL_GET_NEXT(tsk->thread.vstate_ctrl); } static inline bool riscv_v_ctrl_test_inherit(struct task_struct *tsk) { return VSTATE_CTRL_GET_INHERIT(tsk->thread.vstate_ctrl); } static inline void riscv_v_ctrl_set(struct task_struct *tsk, int cur, int nxt, bool inherit) { unsigned long ctrl; ctrl = cur & PR_RISCV_V_VSTATE_CTRL_CUR_MASK; ctrl |= VSTATE_CTRL_MAKE_NEXT(nxt); if (inherit) ctrl |= PR_RISCV_V_VSTATE_CTRL_INHERIT; tsk->thread.vstate_ctrl = ctrl; } bool riscv_v_vstate_ctrl_user_allowed(void) { return riscv_v_ctrl_get_cur(current) == PR_RISCV_V_VSTATE_CTRL_ON; } EXPORT_SYMBOL_GPL(riscv_v_vstate_ctrl_user_allowed); bool riscv_v_first_use_handler(struct pt_regs *regs) { u32 __user *epc = (u32 __user *)regs->epc; u32 insn = (u32)regs->badaddr; /* Do not handle if V is not supported, or disabled */ if (!(ELF_HWCAP & COMPAT_HWCAP_ISA_V)) return false; /* If V has been enabled then it is not the first-use trap */ if (riscv_v_vstate_query(regs)) return false; /* Get the instruction */ if (!insn) { if (__get_user(insn, epc)) return false; } /* Filter out non-V instructions */ if (!insn_is_vector(insn)) return false; /* Sanity check. datap should be null by the time of the first-use trap */ WARN_ON(current->thread.vstate.datap); /* * Now we sure that this is a V instruction. And it executes in the * context where VS has been off. So, try to allocate the user's V * context and resume execution. */ if (riscv_v_thread_zalloc()) { force_sig(SIGBUS); return true; } riscv_v_vstate_on(regs); riscv_v_vstate_restore(current, regs); return true; } void riscv_v_vstate_ctrl_init(struct task_struct *tsk) { bool inherit; int cur, next; if (!has_vector()) return; next = riscv_v_ctrl_get_next(tsk); if (!next) { if (READ_ONCE(riscv_v_implicit_uacc)) cur = PR_RISCV_V_VSTATE_CTRL_ON; else cur = PR_RISCV_V_VSTATE_CTRL_OFF; } else { cur = next; } /* Clear next mask if inherit-bit is not set */ inherit = riscv_v_ctrl_test_inherit(tsk); if (!inherit) next = PR_RISCV_V_VSTATE_CTRL_DEFAULT; riscv_v_ctrl_set(tsk, cur, next, inherit); } long riscv_v_vstate_ctrl_get_current(void) { if (!has_vector()) return -EINVAL; return current->thread.vstate_ctrl & PR_RISCV_V_VSTATE_CTRL_MASK; } long riscv_v_vstate_ctrl_set_current(unsigned long arg) { bool inherit; int cur, next; if (!has_vector()) return -EINVAL; if (arg & ~PR_RISCV_V_VSTATE_CTRL_MASK) return -EINVAL; cur = VSTATE_CTRL_GET_CUR(arg); switch (cur) { case PR_RISCV_V_VSTATE_CTRL_OFF: /* Do not allow user to turn off V if current is not off */ if (riscv_v_ctrl_get_cur(current) != PR_RISCV_V_VSTATE_CTRL_OFF) return -EPERM; break; case PR_RISCV_V_VSTATE_CTRL_ON: break; case PR_RISCV_V_VSTATE_CTRL_DEFAULT: cur = riscv_v_ctrl_get_cur(current); break; default: return -EINVAL; } next = VSTATE_CTRL_GET_NEXT(arg); inherit = VSTATE_CTRL_GET_INHERIT(arg); switch (next) { case PR_RISCV_V_VSTATE_CTRL_DEFAULT: case PR_RISCV_V_VSTATE_CTRL_OFF: case PR_RISCV_V_VSTATE_CTRL_ON: riscv_v_ctrl_set(current, cur, next, inherit); return 0; } return -EINVAL; } #ifdef CONFIG_SYSCTL static struct ctl_table riscv_v_default_vstate_table[] = { { .procname = "riscv_v_default_allow", .data = &riscv_v_implicit_uacc, .maxlen = sizeof(riscv_v_implicit_uacc), .mode = 0644, .proc_handler = proc_dobool, }, }; static int __init riscv_v_sysctl_init(void) { if (has_vector()) if (!register_sysctl("abi", riscv_v_default_vstate_table)) return -EINVAL; return 0; } #else /* ! CONFIG_SYSCTL */ static int __init riscv_v_sysctl_init(void) { return 0; } #endif /* ! CONFIG_SYSCTL */ static int riscv_v_init(void) { return riscv_v_sysctl_init(); } core_initcall(riscv_v_init);