summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/arm64/include/asm/asm-extable.h24
-rw-r--r--arch/arm64/include/asm/futex.h25
-rw-r--r--arch/arm64/include/asm/uaccess.h19
-rw-r--r--arch/arm64/kernel/armv8_deprecated.c12
-rw-r--r--arch/arm64/kernel/traps.c9
-rw-r--r--arch/arm64/mm/extable.c17
6 files changed, 58 insertions, 48 deletions
diff --git a/arch/arm64/include/asm/asm-extable.h b/arch/arm64/include/asm/asm-extable.h
index 5ee748edaef1..11209da19c62 100644
--- a/arch/arm64/include/asm/asm-extable.h
+++ b/arch/arm64/include/asm/asm-extable.h
@@ -5,6 +5,7 @@
#define EX_TYPE_NONE 0
#define EX_TYPE_FIXUP 1
#define EX_TYPE_BPF 2
+#define EX_TYPE_UACCESS_ERR_ZERO 3
#ifdef __ASSEMBLY__
@@ -37,8 +38,11 @@
#else /* __ASSEMBLY__ */
+#include <linux/bits.h>
#include <linux/stringify.h>
+#include <asm/gpr-num.h>
+
#define __ASM_EXTABLE_RAW(insn, fixup, type, data) \
".pushsection __ex_table, \"a\"\n" \
".align 2\n" \
@@ -51,6 +55,26 @@
#define _ASM_EXTABLE(insn, fixup) \
__ASM_EXTABLE_RAW(#insn, #fixup, __stringify(EX_TYPE_FIXUP), "0")
+#define EX_DATA_REG_ERR_SHIFT 0
+#define EX_DATA_REG_ERR GENMASK(4, 0)
+#define EX_DATA_REG_ZERO_SHIFT 5
+#define EX_DATA_REG_ZERO GENMASK(9, 5)
+
+#define EX_DATA_REG(reg, gpr) \
+ "((.L__gpr_num_" #gpr ") << " __stringify(EX_DATA_REG_##reg##_SHIFT) ")"
+
+#define _ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, err, zero) \
+ __DEFINE_ASM_GPR_NUMS \
+ __ASM_EXTABLE_RAW(#insn, #fixup, \
+ __stringify(EX_TYPE_UACCESS_ERR_ZERO), \
+ "(" \
+ EX_DATA_REG(ERR, err) " | " \
+ EX_DATA_REG(ZERO, zero) \
+ ")")
+
+#define _ASM_EXTABLE_UACCESS_ERR(insn, fixup, err) \
+ _ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, err, wzr)
+
#endif /* __ASSEMBLY__ */
#endif /* __ASM_ASM_EXTABLE_H */
diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h
index 8e41faa37c69..bc06691d2062 100644
--- a/arch/arm64/include/asm/futex.h
+++ b/arch/arm64/include/asm/futex.h
@@ -25,19 +25,14 @@ do { \
" cbz %w0, 3f\n" \
" sub %w4, %w4, %w0\n" \
" cbnz %w4, 1b\n" \
-" mov %w0, %w7\n" \
+" mov %w0, %w6\n" \
"3:\n" \
" dmb ish\n" \
-" .pushsection .fixup,\"ax\"\n" \
-" .align 2\n" \
-"4: mov %w0, %w6\n" \
-" b 3b\n" \
-" .popsection\n" \
- _ASM_EXTABLE(1b, 4b) \
- _ASM_EXTABLE(2b, 4b) \
+ _ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w0) \
+ _ASM_EXTABLE_UACCESS_ERR(2b, 3b, %w0) \
: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp), \
"+r" (loops) \
- : "r" (oparg), "Ir" (-EFAULT), "Ir" (-EAGAIN) \
+ : "r" (oparg), "Ir" (-EAGAIN) \
: "memory"); \
uaccess_disable_privileged(); \
} while (0)
@@ -105,18 +100,14 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr,
" cbz %w3, 3f\n"
" sub %w4, %w4, %w3\n"
" cbnz %w4, 1b\n"
-" mov %w0, %w8\n"
+" mov %w0, %w7\n"
"3:\n"
" dmb ish\n"
"4:\n"
-" .pushsection .fixup,\"ax\"\n"
-"5: mov %w0, %w7\n"
-" b 4b\n"
-" .popsection\n"
- _ASM_EXTABLE(1b, 5b)
- _ASM_EXTABLE(2b, 5b)
+ _ASM_EXTABLE_UACCESS_ERR(1b, 4b, %w0)
+ _ASM_EXTABLE_UACCESS_ERR(2b, 4b, %w0)
: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp), "+r" (loops)
- : "r" (oldval), "r" (newval), "Ir" (-EFAULT), "Ir" (-EAGAIN)
+ : "r" (oldval), "r" (newval), "Ir" (-EAGAIN)
: "memory");
uaccess_disable_privileged();
diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index 759019523b85..9bc218991c5a 100644
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -255,15 +255,9 @@ static inline void __user *__uaccess_mask_ptr(const void __user *ptr)
asm volatile( \
"1: " load " " reg "1, [%2]\n" \
"2:\n" \
- " .section .fixup, \"ax\"\n" \
- " .align 2\n" \
- "3: mov %w0, %3\n" \
- " mov %1, #0\n" \
- " b 2b\n" \
- " .previous\n" \
- _ASM_EXTABLE(1b, 3b) \
+ _ASM_EXTABLE_UACCESS_ERR_ZERO(1b, 2b, %w0, %w1) \
: "+r" (err), "=&r" (x) \
- : "r" (addr), "i" (-EFAULT))
+ : "r" (addr))
#define __raw_get_mem(ldr, x, ptr, err) \
do { \
@@ -332,14 +326,9 @@ do { \
asm volatile( \
"1: " store " " reg "1, [%2]\n" \
"2:\n" \
- " .section .fixup,\"ax\"\n" \
- " .align 2\n" \
- "3: mov %w0, %3\n" \
- " b 2b\n" \
- " .previous\n" \
- _ASM_EXTABLE(1b, 3b) \
+ _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w0) \
: "+r" (err) \
- : "r" (x), "r" (addr), "i" (-EFAULT))
+ : "r" (x), "r" (addr))
#define __raw_put_mem(str, x, ptr, err) \
do { \
diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c
index 0e86e8b9cedd..6875a16b09d2 100644
--- a/arch/arm64/kernel/armv8_deprecated.c
+++ b/arch/arm64/kernel/armv8_deprecated.c
@@ -279,7 +279,7 @@ static void __init register_insn_emulation_sysctl(void)
do { \
uaccess_enable_privileged(); \
__asm__ __volatile__( \
- " mov %w3, %w7\n" \
+ " mov %w3, %w6\n" \
"0: ldxr"B" %w2, [%4]\n" \
"1: stxr"B" %w0, %w1, [%4]\n" \
" cbz %w0, 2f\n" \
@@ -290,16 +290,10 @@ do { \
"2:\n" \
" mov %w1, %w2\n" \
"3:\n" \
- " .pushsection .fixup,\"ax\"\n" \
- " .align 2\n" \
- "4: mov %w0, %w6\n" \
- " b 3b\n" \
- " .popsection" \
- _ASM_EXTABLE(0b, 4b) \
- _ASM_EXTABLE(1b, 4b) \
+ _ASM_EXTABLE_UACCESS_ERR(0b, 3b, %w0) \
+ _ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w0) \
: "=&r" (res), "+r" (data), "=&r" (temp), "=&r" (temp2) \
: "r" ((unsigned long)addr), "i" (-EAGAIN), \
- "i" (-EFAULT), \
"i" (__SWP_LL_SC_LOOPS) \
: "memory"); \
uaccess_disable_privileged(); \
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index b03e383d944a..268a81a3006e 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -527,14 +527,9 @@ NOKPROBE_SYMBOL(do_ptrauth_fault);
"1: " insn ", %1\n" \
" mov %w0, #0\n" \
"2:\n" \
- " .pushsection .fixup,\"ax\"\n" \
- " .align 2\n" \
- "3: mov %w0, %w2\n" \
- " b 2b\n" \
- " .popsection\n" \
- _ASM_EXTABLE(1b, 3b) \
+ _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w0) \
: "=r" (res) \
- : "r" (address), "i" (-EFAULT)); \
+ : "r" (address)); \
uaccess_ttbr0_disable(); \
}
diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c
index c2951b963335..bbbc95313f2e 100644
--- a/arch/arm64/mm/extable.c
+++ b/arch/arm64/mm/extable.c
@@ -3,10 +3,12 @@
* Based on arch/arm/mm/extable.c
*/
+#include <linux/bitfield.h>
#include <linux/extable.h>
#include <linux/uaccess.h>
#include <asm/asm-extable.h>
+#include <asm/ptrace.h>
typedef bool (*ex_handler_t)(const struct exception_table_entry *,
struct pt_regs *);
@@ -24,6 +26,19 @@ static bool ex_handler_fixup(const struct exception_table_entry *ex,
return true;
}
+static bool ex_handler_uaccess_err_zero(const struct exception_table_entry *ex,
+ struct pt_regs *regs)
+{
+ int reg_err = FIELD_GET(EX_DATA_REG_ERR, ex->data);
+ int reg_zero = FIELD_GET(EX_DATA_REG_ZERO, ex->data);
+
+ pt_regs_write_reg(regs, reg_err, -EFAULT);
+ pt_regs_write_reg(regs, reg_zero, 0);
+
+ regs->pc = get_ex_fixup(ex);
+ return true;
+}
+
bool fixup_exception(struct pt_regs *regs)
{
const struct exception_table_entry *ex;
@@ -37,6 +52,8 @@ bool fixup_exception(struct pt_regs *regs)
return ex_handler_fixup(ex, regs);
case EX_TYPE_BPF:
return ex_handler_bpf(ex, regs);
+ case EX_TYPE_UACCESS_ERR_ZERO:
+ return ex_handler_uaccess_err_zero(ex, regs);
}
BUG();
generated by cgit (git 2.34.1) at 2024-06-08 08:59:27 +0000