summaryrefslogtreecommitdiff
path: root/arch
diff options
context:
space:
mode:
Diffstat (limited to 'arch')
-rw-r--r--arch/alpha/Kconfig1
-rw-r--r--arch/alpha/include/asm/extable.h55
-rw-r--r--arch/alpha/include/asm/futex.h16
-rw-r--r--arch/alpha/include/asm/uaccess.h296
-rw-r--r--arch/alpha/kernel/osf_sys.c2
-rw-r--r--arch/alpha/kernel/traps.c152
-rw-r--r--arch/alpha/lib/clear_user.S66
-rw-r--r--arch/alpha/lib/copy_user.S82
-rw-r--r--arch/alpha/lib/csum_partial_copy.c10
-rw-r--r--arch/alpha/lib/ev6-clear_user.S84
-rw-r--r--arch/alpha/lib/ev6-copy_user.S104
-rw-r--r--arch/arc/Kconfig1
-rw-r--r--arch/arc/include/asm/uaccess.h19
-rw-r--r--arch/arc/mm/extable.c14
-rw-r--r--arch/arm/Kconfig1
-rw-r--r--arch/arm/include/asm/Kbuild1
-rw-r--r--arch/arm/include/asm/uaccess.h80
-rw-r--r--arch/arm64/Kconfig1
-rw-r--r--arch/arm64/include/asm/extable.h25
-rw-r--r--arch/arm64/include/asm/uaccess.h78
-rw-r--r--arch/arm64/kernel/arm64ksyms.c2
-rw-r--r--arch/arm64/lib/copy_in_user.S4
-rw-r--r--arch/avr32/Kconfig1
-rw-r--r--arch/avr32/include/asm/Kbuild1
-rw-r--r--arch/avr32/include/asm/uaccess.h33
-rw-r--r--arch/avr32/kernel/avr32_ksyms.c2
-rw-r--r--arch/avr32/lib/copy_user.S15
-rw-r--r--arch/blackfin/Kconfig1
-rw-r--r--arch/blackfin/include/asm/Kbuild1
-rw-r--r--arch/blackfin/include/asm/uaccess.h43
-rw-r--r--arch/c6x/Kconfig1
-rw-r--r--arch/c6x/include/asm/uaccess.h16
-rw-r--r--arch/cris/Kconfig1
-rw-r--r--arch/cris/arch-v10/lib/usercopy.c31
-rw-r--r--arch/cris/arch-v32/lib/usercopy.c30
-rw-r--r--arch/cris/include/arch-v10/arch/uaccess.h46
-rw-r--r--arch/cris/include/arch-v32/arch/uaccess.h54
-rw-r--r--arch/cris/include/asm/Kbuild1
-rw-r--r--arch/cris/include/asm/uaccess.h67
-rw-r--r--arch/frv/Kconfig1
-rw-r--r--arch/frv/include/asm/Kbuild1
-rw-r--r--arch/frv/include/asm/uaccess.h80
-rw-r--r--arch/frv/kernel/traps.c7
-rw-r--r--arch/frv/mm/extable.c27
-rw-r--r--arch/frv/mm/fault.c6
-rw-r--r--arch/h8300/Kconfig1
-rw-r--r--arch/h8300/include/asm/uaccess.h10
-rw-r--r--arch/hexagon/Kconfig1
-rw-r--r--arch/hexagon/include/asm/uaccess.h23
-rw-r--r--arch/hexagon/kernel/hexagon_ksyms.c4
-rw-r--r--arch/hexagon/mm/copy_from_user.S2
-rw-r--r--arch/hexagon/mm/copy_to_user.S2
-rw-r--r--arch/ia64/Kconfig1
-rw-r--r--arch/ia64/include/asm/extable.h11
-rw-r--r--arch/ia64/include/asm/uaccess.h97
-rw-r--r--arch/ia64/lib/memcpy_mck.S13
-rw-r--r--arch/ia64/mm/extable.c5
-rw-r--r--arch/m32r/Kconfig1
-rw-r--r--arch/m32r/include/asm/Kbuild1
-rw-r--r--arch/m32r/include/asm/uaccess.h184
-rw-r--r--arch/m32r/kernel/m32r_ksyms.c2
-rw-r--r--arch/m32r/lib/usercopy.c21
-rw-r--r--arch/m68k/Kconfig1
-rw-r--r--arch/m68k/include/asm/Kbuild1
-rw-r--r--arch/m68k/include/asm/processor.h10
-rw-r--r--arch/m68k/include/asm/uaccess.h1
-rw-r--r--arch/m68k/include/asm/uaccess_mm.h96
-rw-r--r--arch/m68k/include/asm/uaccess_no.h39
-rw-r--r--arch/m68k/kernel/signal.c2
-rw-r--r--arch/m68k/kernel/traps.c9
-rw-r--r--arch/m68k/lib/uaccess.c12
-rw-r--r--arch/m68k/mm/fault.c2
-rw-r--r--arch/metag/Kconfig1
-rw-r--r--arch/metag/include/asm/Kbuild1
-rw-r--r--arch/metag/include/asm/uaccess.h57
-rw-r--r--arch/metag/lib/usercopy.c318
-rw-r--r--arch/microblaze/Kconfig1
-rw-r--r--arch/microblaze/include/asm/Kbuild1
-rw-r--r--arch/microblaze/include/asm/uaccess.h53
-rw-r--r--arch/mips/Kconfig1
-rw-r--r--arch/mips/cavium-octeon/octeon-memcpy.S31
-rw-r--r--arch/mips/include/asm/uaccess.h442
-rw-r--r--arch/mips/kernel/mips-r2-to-r6-emul.c24
-rw-r--r--arch/mips/kernel/syscall.c2
-rw-r--r--arch/mips/lib/memcpy.S49
-rw-r--r--arch/mips/oprofile/backtrace.c2
-rw-r--r--arch/mn10300/Kconfig1
-rw-r--r--arch/mn10300/include/asm/Kbuild1
-rw-r--r--arch/mn10300/include/asm/uaccess.h182
-rw-r--r--arch/mn10300/kernel/mn10300_ksyms.c2
-rw-r--r--arch/mn10300/lib/usercopy.c18
-rw-r--r--arch/nios2/Kconfig1
-rw-r--r--arch/nios2/include/asm/Kbuild1
-rw-r--r--arch/nios2/include/asm/uaccess.h50
-rw-r--r--arch/nios2/mm/uaccess.c16
-rw-r--r--arch/openrisc/Kconfig1
-rw-r--r--arch/openrisc/include/asm/Kbuild1
-rw-r--r--arch/openrisc/include/asm/uaccess.h46
-rw-r--r--arch/powerpc/Kconfig1
-rw-r--r--arch/powerpc/include/asm/extable.h29
-rw-r--r--arch/powerpc/include/asm/uaccess.h85
-rw-r--r--arch/powerpc/lib/Makefile2
-rw-r--r--arch/powerpc/lib/copy_32.S14
-rw-r--r--arch/powerpc/lib/copyuser_64.S35
-rw-r--r--arch/powerpc/lib/usercopy_64.c41
-rw-r--r--arch/s390/Kconfig1
-rw-r--r--arch/s390/include/asm/extable.h28
-rw-r--r--arch/s390/include/asm/uaccess.h146
-rw-r--r--arch/s390/lib/uaccess.c68
-rw-r--r--arch/score/Kconfig1
-rw-r--r--arch/score/include/asm/Kbuild1
-rw-r--r--arch/score/include/asm/extable.h11
-rw-r--r--arch/score/include/asm/uaccess.h54
-rw-r--r--arch/sh/Kconfig1
-rw-r--r--arch/sh/include/asm/extable.h10
-rw-r--r--arch/sh/include/asm/uaccess.h59
-rw-r--r--arch/sparc/Kconfig1
-rw-r--r--arch/sparc/include/asm/uaccess_32.h31
-rw-r--r--arch/sparc/include/asm/uaccess_64.h33
-rw-r--r--arch/sparc/kernel/head_32.S7
-rw-r--r--arch/sparc/lib/GENcopy_from_user.S2
-rw-r--r--arch/sparc/lib/GENcopy_to_user.S2
-rw-r--r--arch/sparc/lib/GENpatch.S4
-rw-r--r--arch/sparc/lib/NG2copy_from_user.S2
-rw-r--r--arch/sparc/lib/NG2copy_to_user.S2
-rw-r--r--arch/sparc/lib/NG2patch.S4
-rw-r--r--arch/sparc/lib/NG4copy_from_user.S2
-rw-r--r--arch/sparc/lib/NG4copy_to_user.S2
-rw-r--r--arch/sparc/lib/NG4patch.S4
-rw-r--r--arch/sparc/lib/NGcopy_from_user.S2
-rw-r--r--arch/sparc/lib/NGcopy_to_user.S2
-rw-r--r--arch/sparc/lib/NGpatch.S4
-rw-r--r--arch/sparc/lib/U1copy_from_user.S4
-rw-r--r--arch/sparc/lib/U1copy_to_user.S4
-rw-r--r--arch/sparc/lib/U3copy_to_user.S2
-rw-r--r--arch/sparc/lib/U3patch.S4
-rw-r--r--arch/sparc/lib/copy_in_user.S6
-rw-r--r--arch/sparc/lib/copy_user.S16
-rw-r--r--arch/tile/Kconfig1
-rw-r--r--arch/tile/include/asm/Kbuild1
-rw-r--r--arch/tile/include/asm/uaccess.h162
-rw-r--r--arch/tile/lib/exports.c7
-rw-r--r--arch/tile/lib/memcpy_32.S41
-rw-r--r--arch/tile/lib/memcpy_user_64.c15
-rw-r--r--arch/um/Kconfig.common1
-rw-r--r--arch/um/include/asm/uaccess.h8
-rw-r--r--arch/um/kernel/skas/uaccess.c8
-rw-r--r--arch/unicore32/Kconfig1
-rw-r--r--arch/unicore32/include/asm/uaccess.h6
-rw-r--r--arch/unicore32/kernel/ksyms.c4
-rw-r--r--arch/unicore32/lib/copy_from_user.S16
-rw-r--r--arch/unicore32/lib/copy_to_user.S6
-rw-r--r--arch/x86/Kconfig1
-rw-r--r--arch/x86/include/asm/uaccess.h65
-rw-r--r--arch/x86/include/asm/uaccess_32.h125
-rw-r--r--arch/x86/include/asm/uaccess_64.h127
-rw-r--r--arch/x86/lib/usercopy.c54
-rw-r--r--arch/x86/lib/usercopy_32.c288
-rw-r--r--arch/x86/lib/usercopy_64.c13
-rw-r--r--arch/xtensa/Kconfig1
-rw-r--r--arch/xtensa/include/asm/Kbuild1
-rw-r--r--arch/xtensa/include/asm/uaccess.h59
-rw-r--r--arch/xtensa/lib/usercopy.S116
163 files changed, 1237 insertions, 4032 deletions
diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig
index 0e49d39ea74a..1be5f61dc630 100644
--- a/arch/alpha/Kconfig
+++ b/arch/alpha/Kconfig
@@ -26,6 +26,7 @@ config ALPHA
select ODD_RT_SIGACTION
select OLD_SIGSUSPEND
select CPU_NO_EFFICIENT_FFS if !ALPHA_EV67
+ select ARCH_HAS_RAW_COPY_USER
help
The Alpha is a 64-bit general-purpose processor designed and
marketed by the Digital Equipment Corporation of blessed memory,
diff --git a/arch/alpha/include/asm/extable.h b/arch/alpha/include/asm/extable.h
new file mode 100644
index 000000000000..048e209e524c
--- /dev/null
+++ b/arch/alpha/include/asm/extable.h
@@ -0,0 +1,55 @@
+#ifndef _ASM_EXTABLE_H
+#define _ASM_EXTABLE_H
+
+/*
+ * About the exception table:
+ *
+ * - insn is a 32-bit pc-relative offset from the faulting insn.
+ * - nextinsn is a 16-bit offset off of the faulting instruction
+ * (not off of the *next* instruction as branches are).
+ * - errreg is the register in which to place -EFAULT.
+ * - valreg is the final target register for the load sequence
+ * and will be zeroed.
+ *
+ * Either errreg or valreg may be $31, in which case nothing happens.
+ *
+ * The exception fixup information "just so happens" to be arranged
+ * as in a MEM format instruction. This lets us emit our three
+ * values like so:
+ *
+ * lda valreg, nextinsn(errreg)
+ *
+ */
+
+struct exception_table_entry
+{
+ signed int insn;
+ union exception_fixup {
+ unsigned unit;
+ struct {
+ signed int nextinsn : 16;
+ unsigned int errreg : 5;
+ unsigned int valreg : 5;
+ } bits;
+ } fixup;
+};
+
+/* Returns the new pc */
+#define fixup_exception(map_reg, _fixup, pc) \
+({ \
+ if ((_fixup)->fixup.bits.valreg != 31) \
+ map_reg((_fixup)->fixup.bits.valreg) = 0; \
+ if ((_fixup)->fixup.bits.errreg != 31) \
+ map_reg((_fixup)->fixup.bits.errreg) = -EFAULT; \
+ (pc) + (_fixup)->fixup.bits.nextinsn; \
+})
+
+#define ARCH_HAS_RELATIVE_EXTABLE
+
+#define swap_ex_entry_fixup(a, b, tmp, delta) \
+ do { \
+ (a)->fixup.unit = (b)->fixup.unit; \
+ (b)->fixup.unit = (tmp).fixup.unit; \
+ } while (0)
+
+#endif
diff --git a/arch/alpha/include/asm/futex.h b/arch/alpha/include/asm/futex.h
index f939794363ac..fb01dfb760c2 100644
--- a/arch/alpha/include/asm/futex.h
+++ b/arch/alpha/include/asm/futex.h
@@ -19,12 +19,8 @@
"3: .subsection 2\n" \
"4: br 1b\n" \
" .previous\n" \
- " .section __ex_table,\"a\"\n" \
- " .long 1b-.\n" \
- " lda $31,3b-1b(%1)\n" \
- " .long 2b-.\n" \
- " lda $31,3b-2b(%1)\n" \
- " .previous\n" \
+ EXC(1b,3b,%1,$31) \
+ EXC(2b,3b,%1,$31) \
: "=&r" (oldval), "=&r"(ret) \
: "r" (uaddr), "r"(oparg) \
: "memory")
@@ -101,12 +97,8 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
"3: .subsection 2\n"
"4: br 1b\n"
" .previous\n"
- " .section __ex_table,\"a\"\n"
- " .long 1b-.\n"
- " lda $31,3b-1b(%0)\n"
- " .long 2b-.\n"
- " lda $31,3b-2b(%0)\n"
- " .previous\n"
+ EXC(1b,3b,%0,$31)
+ EXC(2b,3b,%0,$31)
: "+r"(ret), "=&r"(prev), "=&r"(cmp)
: "r"(uaddr), "r"((long)(int)oldval), "r"(newval)
: "memory");
diff --git a/arch/alpha/include/asm/uaccess.h b/arch/alpha/include/asm/uaccess.h
index b2f62b98e290..7b82dc9a8556 100644
--- a/arch/alpha/include/asm/uaccess.h
+++ b/arch/alpha/include/asm/uaccess.h
@@ -32,13 +32,13 @@
* - AND "addr+size" doesn't have any high-bits set
* - OR we are in kernel mode.
*/
-#define __access_ok(addr, size, segment) \
- (((segment).seg & (addr | size | (addr+size))) == 0)
+#define __access_ok(addr, size) \
+ ((get_fs().seg & (addr | size | (addr+size))) == 0)
-#define access_ok(type, addr, size) \
-({ \
- __chk_user_ptr(addr); \
- __access_ok(((unsigned long)(addr)), (size), get_fs()); \
+#define access_ok(type, addr, size) \
+({ \
+ __chk_user_ptr(addr); \
+ __access_ok(((unsigned long)(addr)), (size)); \
})
/*
@@ -54,9 +54,9 @@
* (b) require any knowledge of processes at this stage
*/
#define put_user(x, ptr) \
- __put_user_check((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)), get_fs())
+ __put_user_check((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
#define get_user(x, ptr) \
- __get_user_check((x), (ptr), sizeof(*(ptr)), get_fs())
+ __get_user_check((x), (ptr), sizeof(*(ptr)))
/*
* The "__xxx" versions do not do address space checking, useful when
@@ -74,6 +74,11 @@
* more extensive comments with fixup_inline_exception below for
* more information.
*/
+#define EXC(label,cont,res,err) \
+ ".section __ex_table,\"a\"\n" \
+ " .long "#label"-.\n" \
+ " lda "#res","#cont"-"#label"("#err")\n" \
+ ".previous\n"
extern void __get_user_unknown(void);
@@ -93,23 +98,23 @@ extern void __get_user_unknown(void);
__gu_err; \
})
-#define __get_user_check(x, ptr, size, segment) \
-({ \
- long __gu_err = -EFAULT; \
- unsigned long __gu_val = 0; \
- const __typeof__(*(ptr)) __user *__gu_addr = (ptr); \
- if (__access_ok((unsigned long)__gu_addr, size, segment)) { \
- __gu_err = 0; \
- switch (size) { \
- case 1: __get_user_8(__gu_addr); break; \
- case 2: __get_user_16(__gu_addr); break; \
- case 4: __get_user_32(__gu_addr); break; \
- case 8: __get_user_64(__gu_addr); break; \
- default: __get_user_unknown(); break; \
- } \
- } \
- (x) = (__force __typeof__(*(ptr))) __gu_val; \
- __gu_err; \
+#define __get_user_check(x, ptr, size) \
+({ \
+ long __gu_err = -EFAULT; \
+ unsigned long __gu_val = 0; \
+ const __typeof__(*(ptr)) __user *__gu_addr = (ptr); \
+ if (__access_ok((unsigned long)__gu_addr, size)) { \
+ __gu_err = 0; \
+ switch (size) { \
+ case 1: __get_user_8(__gu_addr); break; \
+ case 2: __get_user_16(__gu_addr); break; \
+ case 4: __get_user_32(__gu_addr); break; \
+ case 8: __get_user_64(__gu_addr); break; \
+ default: __get_user_unknown(); break; \
+ } \
+ } \
+ (x) = (__force __typeof__(*(ptr))) __gu_val; \
+ __gu_err; \
})
struct __large_struct { unsigned long buf[100]; };
@@ -118,20 +123,14 @@ struct __large_struct { unsigned long buf[100]; };
#define __get_user_64(addr) \
__asm__("1: ldq %0,%2\n" \
"2:\n" \
- ".section __ex_table,\"a\"\n" \
- " .long 1b - .\n" \
- " lda %0, 2b-1b(%1)\n" \
- ".previous" \
+ EXC(1b,2b,%0,%1) \
: "=r"(__gu_val), "=r"(__gu_err) \
: "m"(__m(addr)), "1"(__gu_err))
#define __get_user_32(addr) \
__asm__("1: ldl %0,%2\n" \
"2:\n" \
- ".section __ex_table,\"a\"\n" \
- " .long 1b - .\n" \
- " lda %0, 2b-1b(%1)\n" \
- ".previous" \
+ EXC(1b,2b,%0,%1) \
: "=r"(__gu_val), "=r"(__gu_err) \
: "m"(__m(addr)), "1"(__gu_err))
@@ -141,20 +140,14 @@ struct __large_struct { unsigned long buf[100]; };
#define __get_user_16(addr) \
__asm__("1: ldwu %0,%2\n" \
"2:\n" \
- ".section __ex_table,\"a\"\n" \
- " .long 1b - .\n" \
- " lda %0, 2b-1b(%1)\n" \
- ".previous" \
+ EXC(1b,2b,%0,%1) \
: "=r"(__gu_val), "=r"(__gu_err) \
: "m"(__m(addr)), "1"(__gu_err))
#define __get_user_8(addr) \
__asm__("1: ldbu %0,%2\n" \
"2:\n" \
- ".section __ex_table,\"a\"\n" \
- " .long 1b - .\n" \
- " lda %0, 2b-1b(%1)\n" \
- ".previous" \
+ EXC(1b,2b,%0,%1) \
: "=r"(__gu_val), "=r"(__gu_err) \
: "m"(__m(addr)), "1"(__gu_err))
#else
@@ -170,12 +163,8 @@ struct __large_struct { unsigned long buf[100]; };
" extwh %1,%3,%1\n" \
" or %0,%1,%0\n" \
"3:\n" \
- ".section __ex_table,\"a\"\n" \
- " .long 1b - .\n" \
- " lda %0, 3b-1b(%2)\n" \
- " .long 2b - .\n" \
- " lda %0, 3b-2b(%2)\n" \
- ".previous" \
+ EXC(1b,3b,%0,%2) \
+ EXC(2b,3b,%0,%2) \
: "=&r"(__gu_val), "=&r"(__gu_tmp), "=r"(__gu_err) \
: "r"(addr), "2"(__gu_err)); \
}
@@ -184,10 +173,7 @@ struct __large_struct { unsigned long buf[100]; };
__asm__("1: ldq_u %0,0(%2)\n" \
" extbl %0,%2,%0\n" \
"2:\n" \
- ".section __ex_table,\"a\"\n" \
- " .long 1b - .\n" \
- " lda %0, 2b-1b(%1)\n" \
- ".previous" \
+ EXC(1b,2b,%0,%1) \
: "=&r"(__gu_val), "=r"(__gu_err) \
: "r"(addr), "1"(__gu_err))
#endif
@@ -208,21 +194,21 @@ extern void __put_user_unknown(void);
__pu_err; \
})
-#define __put_user_check(x, ptr, size, segment) \
-({ \
- long __pu_err = -EFAULT; \
- __typeof__(*(ptr)) __user *__pu_addr = (ptr); \
- if (__access_ok((unsigned long)__pu_addr, size, segment)) { \
- __pu_err = 0; \
- switch (size) { \
- case 1: __put_user_8(x, __pu_addr); break; \
- case 2: __put_user_16(x, __pu_addr); break; \
- case 4: __put_user_32(x, __pu_addr); break; \
- case 8: __put_user_64(x, __pu_addr); break; \
- default: __put_user_unknown(); break; \
- } \
- } \
- __pu_err; \
+#define __put_user_check(x, ptr, size) \
+({ \
+ long __pu_err = -EFAULT; \
+ __typeof__(*(ptr)) __user *__pu_addr = (ptr); \
+ if (__access_ok((unsigned long)__pu_addr, size)) { \
+ __pu_err = 0; \
+ switch (size) { \
+ case 1: __put_user_8(x, __pu_addr); break; \
+ case 2: __put_user_16(x, __pu_addr); break; \
+ case 4: __put_user_32(x, __pu_addr); break; \
+ case 8: __put_user_64(x, __pu_addr); break; \
+ default: __put_user_unknown(); break; \
+ } \
+ } \
+ __pu_err; \
})
/*
@@ -233,20 +219,14 @@ extern void __put_user_unknown(void);
#define __put_user_64(x, addr) \
__asm__ __volatile__("1: stq %r2,%1\n" \
"2:\n" \
- ".section __ex_table,\"a\"\n" \
- " .long 1b - .\n" \
- " lda $31,2b-1b(%0)\n" \
- ".previous" \
+ EXC(1b,2b,$31,%0) \
: "=r"(__pu_err) \
: "m" (__m(addr)), "rJ" (x), "0"(__pu_err))
#define __put_user_32(x, addr) \
__asm__ __volatile__("1: stl %r2,%1\n" \
"2:\n" \
- ".section __ex_table,\"a\"\n" \
- " .long 1b - .\n" \
- " lda $31,2b-1b(%0)\n" \
- ".previous" \
+ EXC(1b,2b,$31,%0) \
: "=r"(__pu_err) \
: "m"(__m(addr)), "rJ"(x), "0"(__pu_err))
@@ -256,20 +236,14 @@ __asm__ __volatile__("1: stl %r2,%1\n" \
#define __put_user_16(x, addr) \
__asm__ __volatile__("1: stw %r2,%1\n" \
"2:\n" \
- ".section __ex_table,\"a\"\n" \
- " .long 1b - .\n" \
- " lda $31,2b-1b(%0)\n" \
- ".previous" \
+ EXC(1b,2b,$31,%0) \
: "=r"(__pu_err) \
: "m"(__m(addr)), "rJ"(x), "0"(__pu_err))
#define __put_user_8(x, addr) \
__asm__ __volatile__("1: stb %r2,%1\n" \
"2:\n" \
- ".section __ex_table,\"a\"\n" \
- " .long 1b - .\n" \
- " lda $31,2b-1b(%0)\n" \
- ".previous" \
+ EXC(1b,2b,$31,%0) \
: "=r"(__pu_err) \
: "m"(__m(addr)), "rJ"(x), "0"(__pu_err))
#else
@@ -291,16 +265,10 @@ __asm__ __volatile__("1: stb %r2,%1\n" \
"3: stq_u %2,1(%5)\n" \
"4: stq_u %1,0(%5)\n" \
"5:\n" \
- ".section __ex_table,\"a\"\n" \
- " .long 1b - .\n" \
- " lda $31, 5b-1b(%0)\n" \
- " .long 2b - .\n" \
- " lda $31, 5b-2b(%0)\n" \
- " .long 3b - .\n" \
- " lda $31, 5b-3b(%0)\n" \
- " .long 4b - .\n" \
- " lda $31, 5b-4b(%0)\n" \
- ".previous" \
+ EXC(1b,5b,$31,%0) \
+ EXC(2b,5b,$31,%0) \
+ EXC(3b,5b,$31,%0) \
+ EXC(4b,5b,$31,%0) \
: "=r"(__pu_err), "=&r"(__pu_tmp1), \
"=&r"(__pu_tmp2), "=&r"(__pu_tmp3), \
"=&r"(__pu_tmp4) \
@@ -317,12 +285,8 @@ __asm__ __volatile__("1: stb %r2,%1\n" \
" or %1,%2,%1\n" \
"2: stq_u %1,0(%4)\n" \
"3:\n" \
- ".section __ex_table,\"a\"\n" \
- " .long 1b - .\n" \
- " lda $31, 3b-1b(%0)\n" \
- " .long 2b - .\n" \
- " lda $31, 3b-2b(%0)\n" \
- ".previous" \
+ EXC(1b,3b,$31,%0) \
+ EXC(2b,3b,$31,%0) \
: "=r"(__pu_err), \
"=&r"(__pu_tmp1), "=&r"(__pu_tmp2) \
: "r"((unsigned long)(x)), "r"(addr), "0"(__pu_err)); \
@@ -334,96 +298,30 @@ __asm__ __volatile__("1: stb %r2,%1\n" \
* Complex access routines
*/
-/* This little bit of silliness is to get the GP loaded for a function
- that ordinarily wouldn't. Otherwise we could have it done by the macro
- directly, which can be optimized the linker. */
-#ifdef MODULE
-#define __module_address(sym) "r"(sym),
-#define __module_call(ra, arg, sym) "jsr $" #ra ",(%" #arg ")," #sym
-#else
-#define __module_address(sym)
-#define __module_call(ra, arg, sym) "bsr $" #ra "," #sym " !samegp"
-#endif
-
-extern void __copy_user(void);
+extern long __copy_user(void *to, const void *from, long len);
-extern inline long
-__copy_tofrom_user_nocheck(void *to, const void *from, long len)
+static inline unsigned long
+raw_copy_from_user(void *to, const void __user *from, unsigned long len)
{
- register void * __cu_to __asm__("$6") = to;
- register const void * __cu_from __asm__("$7") = from;
- register long __cu_len __asm__("$0") = len;
-
- __asm__ __volatile__(
- __module_call(28, 3, __copy_user)
- : "=r" (__cu_len), "=r" (__cu_from), "=r" (__cu_to)
- : __module_address(__copy_user)
- "0" (__cu_len), "1" (__cu_from), "2" (__cu_to)
- : "$1", "$2", "$3", "$4", "$5", "$28", "memory");
-
- return __cu_len;
+ return __copy_user(to, (__force const void *)from, len);
}
-#define __copy_to_user(to, from, n) \
-({ \
- __chk_user_ptr(to); \
- __copy_tofrom_user_nocheck((__force void *)(to), (from), (n)); \
-})
-#define __copy_from_user(to, from, n) \
-({ \
- __chk_user_ptr(from); \
- __copy_tofrom_user_nocheck((to), (__force void *)(from), (n)); \
-})
-
-#define __copy_to_user_inatomic __copy_to_user
-#define __copy_from_user_inatomic __copy_from_user
-
-extern inline long
-copy_to_user(void __user *to, const void *from, long n)
+static inline unsigned long
+raw_copy_to_user(void __user *to, const void *from, unsigned long len)
{
- if (likely(__access_ok((unsigned long)to, n, get_fs())))
- n = __copy_tofrom_user_nocheck((__force void *)to, from, n);
- return n;
+ return __copy_user((__force void *)to, from, len);
}
-extern inline long
-copy_from_user(void *to, const void __user *from, long n)
-{
- long res = n;
- if (likely(__access_ok((unsigned long)from, n, get_fs())))
- res = __copy_from_user_inatomic(to, from, n);
- if (unlikely(res))
- memset(to + (n - res), 0, res);
- return res;
-}
-
-extern void __do_clear_user(void);
-
-extern inline long
-__clear_user(void __user *to, long len)
-{
- register void __user * __cl_to __asm__("$6") = to;
- register long __cl_len __asm__("$0") = len;
- __asm__ __volatile__(
- __module_call(28, 2, __do_clear_user)
- : "=r"(__cl_len), "=r"(__cl_to)
- : __module_address(__do_clear_user)
- "0"(__cl_len), "1"(__cl_to)
- : "$1", "$2", "$3", "$4", "$5", "$28", "memory");
- return __cl_len;
-}
+extern long __clear_user(void __user *to, long len);
extern inline long
clear_user(void __user *to, long len)
{
- if (__access_ok((unsigned long)to, len, get_fs()))
+ if (__access_ok((unsigned long)to, len))
len = __clear_user(to, len);
return len;
}
-#undef __module_address
-#undef __module_call
-
#define user_addr_max() \
(uaccess_kernel() ? ~0UL : TASK_SIZE)
@@ -431,56 +329,6 @@ extern long strncpy_from_user(char *dest, const char __user *src, long count);
extern __must_check long strlen_user(const char __user *str);
extern __must_check long strnlen_user(const char __user *str, long n);
-/*
- * About the exception table:
- *
- * - insn is a 32-bit pc-relative offset from the faulting insn.
- * - nextinsn is a 16-bit offset off of the faulting instruction
- * (not off of the *next* instruction as branches are).
- * - errreg is the register in which to place -EFAULT.
- * - valreg is the final target register for the load sequence
- * and will be zeroed.
- *
- * Either errreg or valreg may be $31, in which case nothing happens.
- *
- * The exception fixup information "just so happens" to be arranged
- * as in a MEM format instruction. This lets us emit our three
- * values like so:
- *
- * lda valreg, nextinsn(errreg)
- *
- */
-
-struct exception_table_entry
-{
- signed int insn;
- union exception_fixup {
- unsigned unit;
- struct {
- signed int nextinsn : 16;
- unsigned int errreg : 5;
- unsigned int valreg : 5;
- } bits;
- } fixup;
-};
-
-/* Returns the new pc */
-#define fixup_exception(map_reg, _fixup, pc) \
-({ \
- if ((_fixup)->fixup.bits.valreg != 31) \
- map_reg((_fixup)->fixup.bits.valreg) = 0; \
- if ((_fixup)->fixup.bits.errreg != 31) \
- map_reg((_fixup)->fixup.bits.errreg) = -EFAULT; \
- (pc) + (_fixup)->fixup.bits.nextinsn; \
-})
-
-#define ARCH_HAS_RELATIVE_EXTABLE
-
-#define swap_ex_entry_fixup(a, b, tmp, delta) \
- do { \
- (a)->fixup.unit = (b)->fixup.unit; \
- (b)->fixup.unit = (tmp).fixup.unit; \
- } while (0)
-
+#include <asm/extable.h>
#endif /* __ALPHA_UACCESS_H */
diff --git a/arch/alpha/kernel/osf_sys.c b/arch/alpha/kernel/osf_sys.c
index 0b961093ca5c..6d76e528ab8f 100644
--- a/arch/alpha/kernel/osf_sys.c
+++ b/arch/alpha/kernel/osf_sys.c
@@ -1290,7 +1290,7 @@ SYSCALL_DEFINE1(old_adjtimex, struct timex32 __user *, txc_p)
/* copy relevant bits of struct timex. */
if (copy_from_user(&txc, txc_p, offsetof(struct timex32, time)) ||
copy_from_user(&txc.tick, &txc_p->tick, sizeof(struct timex32) -
- offsetof(struct timex32, time)))
+ offsetof(struct timex32, tick)))
return -EFAULT;
ret = do_adjtimex(&txc);
diff --git a/arch/alpha/kernel/traps.c b/arch/alpha/kernel/traps.c
index b137390e87e7..65bb102d985b 100644
--- a/arch/alpha/kernel/traps.c
+++ b/arch/alpha/kernel/traps.c
@@ -482,12 +482,8 @@ do_entUna(void * va, unsigned long opcode, unsigned long reg,
" extwl %1,%3,%1\n"
" extwh %2,%3,%2\n"
"3:\n"
- ".section __ex_table,\"a\"\n"
- " .long 1b - .\n"
- " lda %1,3b-1b(%0)\n"
- " .long 2b - .\n"
- " lda %2,3b-2b(%0)\n"
- ".previous"
+ EXC(1b,3b,%1,%0)
+ EXC(2b,3b,%2,%0)
: "=r"(error), "=&r"(tmp1), "=&r"(tmp2)
: "r"(va), "0"(0));
if (error)
@@ -502,12 +498,8 @@ do_entUna(void * va, unsigned long opcode, unsigned long reg,
" extll %1,%3,%1\n"
" extlh %2,%3,%2\n"
"3:\n"
- ".section __ex_table,\"a\"\n"
- " .long 1b - .\n"
- " lda %1,3b-1b(%0)\n"
- " .long 2b - .\n"
- " lda %2,3b-2b(%0)\n"
- ".previous"
+ EXC(1b,3b,%1,%0)
+ EXC(2b,3b,%2,%0)
: "=r"(error), "=&r"(tmp1), "=&r"(tmp2)
: "r"(va), "0"(0));
if (error)
@@ -522,12 +514,8 @@ do_entUna(void * va, unsigned long opcode, unsigned long reg,
" extql %1,%3,%1\n"
" extqh %2,%3,%2\n"
"3:\n"
- ".section __ex_table,\"a\"\n"
- " .long 1b - .\n"
- " lda %1,3b-1b(%0)\n"
- " .long 2b - .\n"
- " lda %2,3b-2b(%0)\n"
- ".previous"
+ EXC(1b,3b,%1,%0)
+ EXC(2b,3b,%2,%0)
: "=r"(error), "=&r"(tmp1), "=&r"(tmp2)
: "r"(va), "0"(0));
if (error)
@@ -551,16 +539,10 @@ do_entUna(void * va, unsigned long opcode, unsigned long reg,
"3: stq_u %2,1(%5)\n"
"4: stq_u %1,0(%5)\n"
"5:\n"
- ".section __ex_table,\"a\"\n"
- " .long 1b - .\n"
- " lda %2,5b-1b(%0)\n"
- " .long 2b - .\n"
- " lda %1,5b-2b(%0)\n"
- " .long 3b - .\n"
- " lda $31,5b-3b(%0)\n"
- " .long 4b - .\n"
- " lda $31,5b-4b(%0)\n"
- ".previous"
+ EXC(1b,5b,%2,%0)
+ EXC(2b,5b,%1,%0)
+ EXC(3b,5b,$31,%0)
+ EXC(4b,5b,$31,%0)
: "=r"(error), "=&r"(tmp1), "=&r"(tmp2),
"=&r"(tmp3), "=&r"(tmp4)
: "r"(va), "r"(una_reg(reg)), "0"(0));
@@ -581,16 +563,10 @@ do_entUna(void * va, unsigned long opcode, unsigned long reg,
"3: stq_u %2,3(%5)\n"
"4: stq_u %1,0(%5)\n"
"5:\n"
- ".section __ex_table,\"a\"\n"
- " .long 1b - .\n"
- " lda %2,5b-1b(%0)\n"
- " .long 2b - .\n"
- " lda %1,5b-2b(%0)\n"
- " .long 3b - .\n"
- " lda $31,5b-3b(%0)\n"
- " .long 4b - .\n"
- " lda $31,5b-4b(%0)\n"
- ".previous"
+ EXC(1b,5b,%2,%0)
+ EXC(2b,5b,%1,%0)
+ EXC(3b,5b,$31,%0)
+ EXC(4b,5b,$31,%0)
: "=r"(error), "=&r"(tmp1), "=&r"(tmp2),
"=&r"(tmp3), "=&r"(tmp4)
: "r"(va), "r"(una_reg(reg)), "0"(0));
@@ -611,16 +587,10 @@ do_entUna(void * va, unsigned long opcode, unsigned long reg,
"3: stq_u %2,7(%5)\n"
"4: stq_u %1,0(%5)\n"
"5:\n"
- ".section __ex_table,\"a\"\n\t"
- " .long 1b - .\n"
- " lda %2,5b-1b(%0)\n"
- " .long 2b - .\n"
- " lda %1,5b-2b(%0)\n"
- " .long 3b - .\n"
- " lda $31,5b-3b(%0)\n"
- " .long 4b - .\n"
- " lda $31,5b-4b(%0)\n"
- ".previous"
+ EXC(1b,5b,%2,%0)
+ EXC(2b,5b,%1,%0)
+ EXC(3b,5b,$31,%0)
+ EXC(4b,5b,$31,%0)
: "=r"(error), "=&r"(tmp1), "=&r"(tmp2),
"=&r"(tmp3), "=&r"(tmp4)
: "r"(va), "r"(una_reg(reg)), "0"(0));
@@ -802,7 +772,7 @@ do_entUnaUser(void __user * va, unsigned long opcode,
/* Don't bother reading ds in the access check since we already
know that this came from the user. Also rely on the fact that
the page at TASK_SIZE is unmapped and so can't be touched anyway. */
- if (!__access_ok((unsigned long)va, 0, USER_DS))
+ if ((unsigned long)va >= TASK_SIZE)
goto give_sigsegv;
++unaligned[1].count;
@@ -835,12 +805,8 @@ do_entUnaUser(void __user * va, unsigned long opcode,
" extwl %1,%3,%1\n"
" extwh %2,%3,%2\n"
"3:\n"
- ".section __ex_table,\"a\"\n"
- " .long 1b - .\n"
- " lda %1,3b-1b(%0)\n"
- " .long 2b - .\n"
- " lda %2,3b-2b(%0)\n"
- ".previous"
+ EXC(1b,3b,%1,%0)
+ EXC(2b,3b,%2,%0)
: "=r"(error), "=&r"(tmp1), "=&r"(tmp2)
: "r"(va), "0"(0));
if (error)
@@ -855,12 +821,8 @@ do_entUnaUser(void __user * va, unsigned long opcode,
" extll %1,%3,%1\n"
" extlh %2,%3,%2\n"
"3:\n"
- ".section __ex_table,\"a\"\n"
- " .long 1b - .\n"
- " lda %1,3b-1b(%0)\n"
- " .long 2b - .\n"
- " lda %2,3b-2b(%0)\n"
- ".previous"
+ EXC(1b,3b,%1,%0)
+ EXC(2b,3b,%2,%0)
: "=r"(error), "=&r"(tmp1), "=&r"(tmp2)
: "r"(va), "0"(0));
if (error)
@@ -875,12 +837,8 @@ do_entUnaUser(void __user * va, unsigned long opcode,
" extql %1,%3,%1\n"
" extqh %2,%3,%2\n"
"3:\n"
- ".section __ex_table,\"a\"\n"
- " .long 1b - .\n"
- " lda %1,3b-1b(%0)\n"
- " .long 2b - .\n"
- " lda %2,3b-2b(%0)\n"
- ".previous"
+ EXC(1b,3b,%1,%0)
+ EXC(2b,3b,%2,%0)
: "=r"(error), "=&r"(tmp1), "=&r"(tmp2)
: "r"(va), "0"(0));
if (error)
@@ -895,12 +853,8 @@ do_entUnaUser(void __user * va, unsigned long opcode,
" extll %1,%3,%1\n"
" extlh %2,%3,%2\n"
"3:\n"
- ".section __ex_table,\"a\"\n"
- " .long 1b - .\n"
- " lda %1,3b-1b(%0)\n"
- " .long 2b - .\n"
- " lda %2,3b-2b(%0)\n"
- ".previous"
+ EXC(1b,3b,%1,%0)
+ EXC(2b,3b,%2,%0)
: "=r"(error), "=&r"(tmp1), "=&r"(tmp2)
: "r"(va), "0"(0));
if (error)
@@ -915,12 +869,8 @@ do_entUnaUser(void __user * va, unsigned long opcode,
" extql %1,%3,%1\n"
" extqh %2,%3,%2\n"
"3:\n"
- ".section __ex_table,\"a\"\n"
- " .long 1b - .\n"
- " lda %1,3b-1b(%0)\n"
- " .long 2b - .\n"
- " lda %2,3b-2b(%0)\n"
- ".previous"
+ EXC(1b,3b,%1,%0)
+ EXC(2b,3b,%2,%0)
: "=r"(error), "=&r"(tmp1), "=&r"(tmp2)
: "r"(va), "0"(0));
if (error)
@@ -944,16 +894,10 @@ do_entUnaUser(void __user * va, unsigned long opcode,
"3: stq_u %2,1(%5)\n"
"4: stq_u %1,0(%5)\n"
"5:\n"
- ".section __ex_table,\"a\"\n"
- " .long 1b - .\n"
- " lda %2,5b-1b(%0)\n"
- " .long 2b - .\n"
- " lda %1,5b-2b(%0)\n"
- " .long 3b - .\n"
- " lda $31,5b-3b(%0)\n"
- " .long 4b - .\n"
- " lda $31,5b-4b(%0)\n"
- ".previous"
+ EXC(1b,5b,%2,%0)
+ EXC(2b,5b,%1,%0)
+ EXC(3b,5b,$31,%0)
+ EXC(4b,5b,$31,%0)
: "=r"(error), "=&r"(tmp1), "=&r"(tmp2),
"=&r"(tmp3), "=&r"(tmp4)
: "r"(va), "r"(*reg_addr), "0"(0));
@@ -978,16 +922,10 @@ do_entUnaUser(void __user * va, unsigned long opcode,
"3: stq_u %2,3(%5)\n"
"4: stq_u %1,0(%5)\n"
"5:\n"
- ".section __ex_table,\"a\"\n"
- " .long 1b - .\n"
- " lda %2,5b-1b(%0)\n"
- " .long 2b - .\n"
- " lda %1,5b-2b(%0)\n"
- " .long 3b - .\n"
- " lda $31,5b-3b(%0)\n"
- " .long 4b - .\n"
- " lda $31,5b-4b(%0)\n"
- ".previous"
+ EXC(1b,5b,%2,%0)
+ EXC(2b,5b,%1,%0)
+ EXC(3b,5b,$31,%0)
+ EXC(4b,5b,$31,%0)
: "=r"(error), "=&r"(tmp1), "=&r"(tmp2),
"=&r"(tmp3), "=&r"(tmp4)
: "r"(va), "r"(*reg_addr), "0"(0));
@@ -1012,16 +950,10 @@ do_entUnaUser(void __user * va, unsigned long opcode,
"3: stq_u %2,7(%5)\n"
"4: stq_u %1,0(%5)\n"
"5:\n"
- ".section __ex_table,\"a\"\n\t"
- " .long 1b - .\n"
- " lda %2,5b-1b(%0)\n"
- " .long 2b - .\n"
- " lda %1,5b-2b(%0)\n"
- " .long 3b - .\n"
- " lda $31,5b-3b(%0)\n"
- " .long 4b - .\n"
- " lda $31,5b-4b(%0)\n"
- ".previous"
+ EXC(1b,5b,%2,%0)
+ EXC(2b,5b,%1,%0)
+ EXC(3b,5b,$31,%0)
+ EXC(4b,5b,$31,%0)
: "=r"(error), "=&r"(tmp1), "=&r"(tmp2),
"=&r"(tmp3), "=&r"(tmp4)
: "r"(va), "r"(*reg_addr), "0"(0));
@@ -1047,7 +979,7 @@ give_sigsegv:
/* We need to replicate some of the logic in mm/fault.c,
since we don't have access to the fault code in the
exception handling return path. */
- if (!__access_ok((unsigned long)va, 0, USER_DS))
+ if ((unsigned long)va >= TASK_SIZE)
info.si_code = SEGV_ACCERR;
else {
struct mm_struct *mm = current->mm;
diff --git a/arch/alpha/lib/clear_user.S b/arch/alpha/lib/clear_user.S
index bf5b931866ba..006f469fef73 100644
--- a/arch/alpha/lib/clear_user.S
+++ b/arch/alpha/lib/clear_user.S
@@ -8,21 +8,6 @@
* right "bytes left to zero" value (and that it is updated only _after_
* a successful copy). There is also some rather minor exception setup
* stuff.
- *
- * NOTE! This is not directly C-callable, because the calling semantics
- * are different:
- *
- * Inputs:
- * length in $0
- * destination address in $6
- * exception pointer in $7
- * return address in $28 (exceptions expect it there)
- *
- * Outputs:
- * bytes left to copy in $0
- *
- * Clobbers:
- * $1,$2,$3,$4,$5,$6
*/
#include <asm/export.h>
@@ -38,62 +23,63 @@
.set noreorder
.align 4
- .globl __do_clear_user
- .ent __do_clear_user
- .frame $30, 0, $28
+ .globl __clear_user
+ .ent __clear_user
+ .frame $30, 0, $26
.prologue 0
$loop:
and $1, 3, $4 # e0 :
beq $4, 1f # .. e1 :
-0: EX( stq_u $31, 0($6) ) # e0 : zero one word
+0: EX( stq_u $31, 0($16) ) # e0 : zero one word
subq $0, 8, $0 # .. e1 :
subq $4, 1, $4 # e0 :
- addq $6, 8, $6 # .. e1 :
+ addq $16, 8, $16 # .. e1 :
bne $4, 0b # e1 :
unop # :
1: bic $1, 3, $1 # e0 :
beq $1, $tail # .. e1 :
-2: EX( stq_u $31, 0($6) ) # e0 : zero four words
+2: EX( stq_u $31, 0($16) ) # e0 : zero four words
subq $0, 8, $0 # .. e1 :
- EX( stq_u $31, 8($6) ) # e0 :
+ EX( stq_u $31, 8($16) ) # e0 :
subq $0, 8, $0 # .. e1 :
- EX( stq_u $31, 16($6) ) # e0 :
+ EX( stq_u $31, 16($16) ) # e0 :
subq $0, 8, $0 # .. e1 :
- EX( stq_u $31, 24($6) ) # e0 :
+ EX( stq_u $31, 24($16) ) # e0 :
subq $0, 8, $0 # .. e1 :
subq $1, 4, $1 # e0 :
- addq $6, 32, $6 # .. e1 :
+ addq $16, 32, $16 # .. e1 :
bne $1, 2b # e1 :
$tail:
bne $2, 1f # e1 : is there a tail to do?
- ret $31, ($28), 1 # .. e1 :
+ ret $31, ($26), 1 # .. e1 :
-1: EX( ldq_u $5, 0($6) ) # e0 :
+1: EX( ldq_u $5, 0($16) ) # e0 :
clr $0 # .. e1 :
nop # e1 :
mskqh $5, $0, $5 # e0 :
- EX( stq_u $5, 0($6) ) # e0 :
- ret $31, ($28), 1 # .. e1 :
+ EX( stq_u $5, 0($16) ) # e0 :
+ ret $31, ($26), 1 # .. e1 :
-__do_clear_user:
- and $6, 7, $4 # e0 : find dest misalignment
+__clear_user:
+ and $17, $17, $0
+ and $16, 7, $4 # e0 : find dest misalignment
beq $0, $zerolength # .. e1 :
addq $0, $4, $1 # e0 : bias counter
and $1, 7, $2 # e1 : number of bytes in tail
srl $1, 3, $1 # e0 :
beq $4, $loop # .. e1 :
- EX( ldq_u $5, 0($6) ) # e0 : load dst word to mask back in
+ EX( ldq_u $5, 0($16) ) # e0 : load dst word to mask back in
beq $1, $oneword # .. e1 : sub-word store?
- mskql $5, $6, $5 # e0 : take care of misaligned head
- addq $6, 8, $6 # .. e1 :
- EX( stq_u $5, -8($6) ) # e0 :
+ mskql $5, $16, $5 # e0 : take care of misaligned head
+ addq $16, 8, $16 # .. e1 :
+ EX( stq_u $5, -8($16) ) # e0 :
addq $0, $4, $0 # .. e1 : bytes left -= 8 - misalignment
subq $1, 1, $1 # e0 :
subq $0, 8, $0 # .. e1 :
@@ -101,15 +87,15 @@ __do_clear_user:
unop # :
$oneword:
- mskql $5, $6, $4 # e0 :
+ mskql $5, $16, $4 # e0 :
mskqh $5, $2, $5 # e0 :
or $5, $4, $5 # e1 :
- EX( stq_u $5, 0($6) ) # e0 :
+ EX( stq_u $5, 0($16) ) # e0 :
clr $0 # .. e1 :
$zerolength:
$exception:
- ret $31, ($28), 1 # .. e1 :
+ ret $31, ($26), 1 # .. e1 :
- .end __do_clear_user
- EXPORT_SYMBOL(__do_clear_user)
+ .end __clear_user
+ EXPORT_SYMBOL(__clear_user)
diff --git a/arch/alpha/lib/copy_user.S b/arch/alpha/lib/copy_user.S
index 509f62b65311..159f1b7e6e49 100644
--- a/arch/alpha/lib/copy_user.S
+++ b/arch/alpha/lib/copy_user.S
@@ -9,21 +9,6 @@
* contains the right "bytes left to copy" value (and that it is updated
* only _after_ a successful copy). There is also some rather minor
* exception setup stuff..
- *
- * NOTE! This is not directly C-callable, because the calling semantics are
- * different:
- *
- * Inputs:
- * length in $0
- * destination address in $6
- * source address in $7
- * return address in $28
- *
- * Outputs:
- * bytes left to copy in $0
- *
- * Clobbers:
- * $1,$2,$3,$4,$5,$6,$7
*/
#include <asm/export.h>
@@ -49,58 +34,59 @@
.ent __copy_user
__copy_user:
.prologue 0
- and $6,7,$3
+ and $18,$18,$0
+ and $16,7,$3
beq $0,$35
beq $3,$36
subq $3,8,$3
.align 4
$37:
- EXI( ldq_u $1,0($7) )
- EXO( ldq_u $2,0($6) )
- extbl $1,$7,$1
- mskbl $2,$6,$2
- insbl $1,$6,$1
+ EXI( ldq_u $1,0($17) )
+ EXO( ldq_u $2,0($16) )
+ extbl $1,$17,$1
+ mskbl $2,$16,$2
+ insbl $1,$16,$1
addq $3,1,$3
bis $1,$2,$1
- EXO( stq_u $1,0($6) )
+ EXO( stq_u $1,0($16) )
subq $0,1,$0
- addq $6,1,$6
- addq $7,1,$7
+ addq $16,1,$16
+ addq $17,1,$17
beq $0,$41
bne $3,$37
$36:
- and $7,7,$1
+ and $17,7,$1
bic $0,7,$4
beq $1,$43
beq $4,$48
- EXI( ldq_u $3,0($7) )
+ EXI( ldq_u $3,0($17) )
.align 4
$50:
- EXI( ldq_u $2,8($7) )
+ EXI( ldq_u $2,8($17) )
subq $4,8,$4
- extql $3,$7,$3
- extqh $2,$7,$1
+ extql $3,$17,$3
+ extqh $2,$17,$1
bis $3,$1,$1
- EXO( stq $1,0($6) )
- addq $7,8,$7
+ EXO( stq $1,0($16) )
+ addq $17,8,$17
subq $0,8,$0
- addq $6,8,$6
+ addq $16,8,$16
bis $2,$2,$3
bne $4,$50
$48:
beq $0,$41
.align 4
$57:
- EXI( ldq_u $1,0($7) )
- EXO( ldq_u $2,0($6) )
- extbl $1,$7,$1
- mskbl $2,$6,$2
- insbl $1,$6,$1
+ EXI( ldq_u $1,0($17) )
+ EXO( ldq_u $2,0($16) )
+ extbl $1,$17,$1
+ mskbl $2,$16,$2
+ insbl $1,$16,$1
bis $1,$2,$1
- EXO( stq_u $1,0($6) )
+ EXO( stq_u $1,0($16) )
subq $0,1,$0
- addq $6,1,$6
- addq $7,1,$7
+ addq $16,1,$16
+ addq $17,1,$17
bne $0,$57
br $31,$41
.align 4
@@ -108,27 +94,27 @@ $43:
beq $4,$65
.align 4
$66:
- EXI( ldq $1,0($7) )
+ EXI( ldq $1,0($17) )
subq $4,8,$4
- EXO( stq $1,0($6) )
- addq $7,8,$7
+ EXO( stq $1,0($16) )
+ addq $17,8,$17
subq $0,8,$0
- addq $6,8,$6
+ addq $16,8,$16
bne $4,$66
$65:
beq $0,$41
- EXI( ldq $2,0($7) )
- EXO( ldq $1,0($6) )
+ EXI( ldq $2,0($17) )
+ EXO( ldq $1,0($16) )
mskql $2,$0,$2
mskqh $1,$0,$1
bis $2,$1,$2
- EXO( stq $2,0($6) )
+ EXO( stq $2,0($16) )
bis $31,$31,$0
$41:
$35:
$exitin:
$exitout:
- ret $31,($28),1
+ ret $31,($26),1
.end __copy_user
EXPORT_SYMBOL(__copy_user)
diff --git a/arch/alpha/lib/csum_partial_copy.c b/arch/alpha/lib/csum_partial_copy.c
index 5dfb7975895f..ab42afba1720 100644
--- a/arch/alpha/lib/csum_partial_copy.c
+++ b/arch/alpha/lib/csum_partial_copy.c
@@ -45,10 +45,7 @@ __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
__asm__ __volatile__( \
"1: ldq_u %0,%2\n" \
"2:\n" \
- ".section __ex_table,\"a\"\n" \
- " .long 1b - .\n" \
- " lda %0,2b-1b(%1)\n" \
- ".previous" \
+ EXC(1b,2b,%0,%1) \
: "=r"(x), "=r"(__guu_err) \
: "m"(__m(ptr)), "1"(0)); \
__guu_err; \
@@ -60,10 +57,7 @@ __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
__asm__ __volatile__( \
"1: stq_u %2,%1\n" \
"2:\n" \
- ".section __ex_table,\"a\"\n" \
- " .long 1b - ." \
- " lda $31,2b-1b(%0)\n" \
- ".previous" \
+ EXC(1b,2b,$31,%0) \
: "=r"(__puu_err) \
: "m"(__m(addr)), "rJ"(x), "0"(0)); \
__puu_err; \
diff --git a/arch/alpha/lib/ev6-clear_user.S b/arch/alpha/lib/ev6-clear_user.S
index 05bef6b50598..e179e4757ef8 100644
--- a/arch/alpha/lib/ev6-clear_user.S
+++ b/arch/alpha/lib/ev6-clear_user.S
@@ -9,21 +9,6 @@
* a successful copy). There is also some rather minor exception setup
* stuff.
*
- * NOTE! This is not directly C-callable, because the calling semantics
- * are different:
- *
- * Inputs:
- * length in $0
- * destination address in $6
- * exception pointer in $7
- * return address in $28 (exceptions expect it there)
- *
- * Outputs:
- * bytes left to copy in $0
- *
- * Clobbers:
- * $1,$2,$3,$4,$5,$6
- *
* Much of the information about 21264 scheduling/coding comes from:
* Compiler Writer's Guide for the Alpha 21264
* abbreviated as 'CWG' in other comments here
@@ -56,14 +41,15 @@
.set noreorder
.align 4
- .globl __do_clear_user
- .ent __do_clear_user
- .frame $30, 0, $28
+ .globl __clear_user
+ .ent __clear_user
+ .frame $30, 0, $26
.prologue 0
# Pipeline info : Slotting & Comments
-__do_clear_user:
- and $6, 7, $4 # .. E .. .. : find dest head misalignment
+__clear_user:
+ and $17, $17, $0
+ and $16, 7, $4 # .. E .. .. : find dest head misalignment
beq $0, $zerolength # U .. .. .. : U L U L
addq $0, $4, $1 # .. .. .. E : bias counter
@@ -75,14 +61,14 @@ __do_clear_user:
/*
* Head is not aligned. Write (8 - $4) bytes to head of destination
- * This means $6 is known to be misaligned
+ * This means $16 is known to be misaligned
*/
- EX( ldq_u $5, 0($6) ) # .. .. .. L : load dst word to mask back in
+ EX( ldq_u $5, 0($16) ) # .. .. .. L : load dst word to mask back in
beq $1, $onebyte # .. .. U .. : sub-word store?
- mskql $5, $6, $5 # .. U .. .. : take care of misaligned head
- addq $6, 8, $6 # E .. .. .. : L U U L
+ mskql $5, $16, $5 # .. U .. .. : take care of misaligned head
+ addq $16, 8, $16 # E .. .. .. : L U U L
- EX( stq_u $5, -8($6) ) # .. .. .. L :
+ EX( stq_u $5, -8($16) ) # .. .. .. L :
subq $1, 1, $1 # .. .. E .. :
addq $0, $4, $0 # .. E .. .. : bytes left -= 8 - misalignment
subq $0, 8, $0 # E .. .. .. : U L U L
@@ -93,11 +79,11 @@ __do_clear_user:
* values upon initial entry to the loop
* $1 is number of quadwords to clear (zero is a valid value)
* $2 is number of trailing bytes (0..7) ($2 never used...)
- * $6 is known to be aligned 0mod8
+ * $16 is known to be aligned 0mod8
*/
$headalign:
subq $1, 16, $4 # .. .. .. E : If < 16, we can not use the huge loop
- and $6, 0x3f, $2 # .. .. E .. : Forward work for huge loop
+ and $16, 0x3f, $2 # .. .. E .. : Forward work for huge loop
subq $2, 0x40, $3 # .. E .. .. : bias counter (huge loop)
blt $4, $trailquad # U .. .. .. : U L U L
@@ -114,21 +100,21 @@ $headalign:
beq $3, $bigalign # U .. .. .. : U L U L : Aligned 0mod64
$alignmod64:
- EX( stq_u $31, 0($6) ) # .. .. .. L
+ EX( stq_u $31, 0($16) ) # .. .. .. L
addq $3, 8, $3 # .. .. E ..
subq $0, 8, $0 # .. E .. ..
nop # E .. .. .. : U L U L
nop # .. .. .. E
subq $1, 1, $1 # .. .. E ..
- addq $6, 8, $6 # .. E .. ..
+ addq $16, 8, $16 # .. E .. ..
blt $3, $alignmod64 # U .. .. .. : U L U L
$bigalign:
/*
* $0 is the number of bytes left
* $1 is the number of quads left
- * $6 is aligned 0mod64
+ * $16 is aligned 0mod64
* we know that we'll be taking a minimum of one trip through
* CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle
* We are _not_ going to update $0 after every single store. That
@@ -145,39 +131,39 @@ $bigalign:
nop # E :
nop # E :
nop # E :
- bis $6,$6,$3 # E : U L U L : Initial wh64 address is dest
+ bis $16,$16,$3 # E : U L U L : Initial wh64 address is dest
/* This might actually help for the current trip... */
$do_wh64:
wh64 ($3) # .. .. .. L1 : memory subsystem hint
subq $1, 16, $4 # .. .. E .. : Forward calculation - repeat the loop?
- EX( stq_u $31, 0($6) ) # .. L .. ..
+ EX( stq_u $31, 0($16) ) # .. L .. ..
subq $0, 8, $0 # E .. .. .. : U L U L
- addq $6, 128, $3 # E : Target address of wh64
- EX( stq_u $31, 8($6) ) # L :
- EX( stq_u $31, 16($6) ) # L :
+ addq $16, 128, $3 # E : Target address of wh64
+ EX( stq_u $31, 8($16) ) # L :
+ EX( stq_u $31, 16($16) ) # L :
subq $0, 16, $0 # E : U L L U
nop # E :
- EX( stq_u $31, 24($6) ) # L :
- EX( stq_u $31, 32($6) ) # L :
+ EX( stq_u $31, 24($16) ) # L :
+ EX( stq_u $31, 32($16) ) # L :
subq $0, 168, $5 # E : U L L U : two trips through the loop left?
/* 168 = 192 - 24, since we've already completed some stores */
subq $0, 16, $0 # E :
- EX( stq_u $31, 40($6) ) # L :
- EX( stq_u $31, 48($6) ) # L :
- cmovlt $5, $6, $3 # E : U L L U : Latency 2, extra mapping cycle
+ EX( stq_u $31, 40($16) ) # L :
+ EX( stq_u $31, 48($16) ) # L :
+ cmovlt $5, $16, $3 # E : U L L U : Latency 2, extra mapping cycle
subq $1, 8, $1 # E :
subq $0, 16, $0 # E :
- EX( stq_u $31, 56($6) ) # L :
+ EX( stq_u $31, 56($16) ) # L :
nop # E : U L U L
nop # E :
subq $0, 8, $0 # E :
- addq $6, 64, $6 # E :
+ addq $16, 64, $16 # E :
bge $4, $do_wh64 # U : U L U L
$trailquad:
@@ -190,14 +176,14 @@ $trailquad:
beq $1, $trailbytes # U .. .. .. : U L U L : Only 0..7 bytes to go
$onequad:
- EX( stq_u $31, 0($6) ) # .. .. .. L
+ EX( stq_u $31, 0($16) ) # .. .. .. L
subq $1, 1, $1 # .. .. E ..
subq $0, 8, $0 # .. E .. ..
nop # E .. .. .. : U L U L
nop # .. .. .. E
nop # .. .. E ..
- addq $6, 8, $6 # .. E .. ..
+ addq $16, 8, $16 # .. E .. ..
bgt $1, $onequad # U .. .. .. : U L U L
# We have an unknown number of bytes left to go.
@@ -211,9 +197,9 @@ $trailbytes:
# so we will use $0 as the loop counter
# We know for a fact that $0 > 0 zero due to previous context
$onebyte:
- EX( stb $31, 0($6) ) # .. .. .. L
+ EX( stb $31, 0($16) ) # .. .. .. L
subq $0, 1, $0 # .. .. E .. :
- addq $6, 1, $6 # .. E .. .. :
+ addq $16, 1, $16 # .. E .. .. :
bgt $0, $onebyte # U .. .. .. : U L U L
$zerolength:
@@ -221,6 +207,6 @@ $exception: # Destination for exception recovery(?)
nop # .. .. .. E :
nop # .. .. E .. :
nop # .. E .. .. :
- ret $31, ($28), 1 # L0 .. .. .. : L U L U
- .end __do_clear_user
- EXPORT_SYMBOL(__do_clear_user)
+ ret $31, ($26), 1 # L0 .. .. .. : L U L U
+ .end __clear_user
+ EXPORT_SYMBOL(__clear_user)
diff --git a/arch/alpha/lib/ev6-copy_user.S b/arch/alpha/lib/ev6-copy_user.S
index be720b518af9..35e6710d0700 100644
--- a/arch/alpha/lib/ev6-copy_user.S
+++ b/arch/alpha/lib/ev6-copy_user.S
@@ -12,21 +12,6 @@
* only _after_ a successful copy). There is also some rather minor
* exception setup stuff..
*
- * NOTE! This is not directly C-callable, because the calling semantics are
- * different:
- *
- * Inputs:
- * length in $0
- * destination address in $6
- * source address in $7
- * return address in $28
- *
- * Outputs:
- * bytes left to copy in $0
- *
- * Clobbers:
- * $1,$2,$3,$4,$5,$6,$7
- *
* Much of the information about 21264 scheduling/coding comes from:
* Compiler Writer's Guide for the Alpha 21264
* abbreviated as 'CWG' in other comments here
@@ -60,10 +45,11 @@
# Pipeline info: Slotting & Comments
__copy_user:
.prologue 0
- subq $0, 32, $1 # .. E .. .. : Is this going to be a small copy?
+ andq $18, $18, $0
+ subq $18, 32, $1 # .. E .. .. : Is this going to be a small copy?
beq $0, $zerolength # U .. .. .. : U L U L
- and $6,7,$3 # .. .. .. E : is leading dest misalignment
+ and $16,7,$3 # .. .. .. E : is leading dest misalignment
ble $1, $onebyteloop # .. .. U .. : 1st branch : small amount of data
beq $3, $destaligned # .. U .. .. : 2nd (one cycle fetcher stall)
subq $3, 8, $3 # E .. .. .. : L U U L : trip counter
@@ -73,17 +59,17 @@ __copy_user:
* We know we have at least one trip through this loop
*/
$aligndest:
- EXI( ldbu $1,0($7) ) # .. .. .. L : Keep loads separate from stores
- addq $6,1,$6 # .. .. E .. : Section 3.8 in the CWG
+ EXI( ldbu $1,0($17) ) # .. .. .. L : Keep loads separate from stores
+ addq $16,1,$16 # .. .. E .. : Section 3.8 in the CWG
addq $3,1,$3 # .. E .. .. :
nop # E .. .. .. : U L U L
/*
- * the -1 is to compensate for the inc($6) done in a previous quadpack
+ * the -1 is to compensate for the inc($16) done in a previous quadpack
* which allows us zero dependencies within either quadpack in the loop
*/
- EXO( stb $1,-1($6) ) # .. .. .. L :
- addq $7,1,$7 # .. .. E .. : Section 3.8 in the CWG
+ EXO( stb $1,-1($16) ) # .. .. .. L :
+ addq $17,1,$17 # .. .. E .. : Section 3.8 in the CWG
subq $0,1,$0 # .. E .. .. :
bne $3, $aligndest # U .. .. .. : U L U L
@@ -92,29 +78,29 @@ $aligndest:
* If we arrived via branch, we have a minimum of 32 bytes
*/
$destaligned:
- and $7,7,$1 # .. .. .. E : Check _current_ source alignment
+ and $17,7,$1 # .. .. .. E : Check _current_ source alignment
bic $0,7,$4 # .. .. E .. : number bytes as a quadword loop
- EXI( ldq_u $3,0($7) ) # .. L .. .. : Forward fetch for fallthrough code
+ EXI( ldq_u $3,0($17) ) # .. L .. .. : Forward fetch for fallthrough code
beq $1,$quadaligned # U .. .. .. : U L U L
/*
- * In the worst case, we've just executed an ldq_u here from 0($7)
+ * In the worst case, we've just executed an ldq_u here from 0($17)
* and we'll repeat it once if we take the branch
*/
/* Misaligned quadword loop - not unrolled. Leave it that way. */
$misquad:
- EXI( ldq_u $2,8($7) ) # .. .. .. L :
+ EXI( ldq_u $2,8($17) ) # .. .. .. L :
subq $4,8,$4 # .. .. E .. :
- extql $3,$7,$3 # .. U .. .. :
- extqh $2,$7,$1 # U .. .. .. : U U L L
+ extql $3,$17,$3 # .. U .. .. :
+ extqh $2,$17,$1 # U .. .. .. : U U L L
bis $3,$1,$1 # .. .. .. E :
- EXO( stq $1,0($6) ) # .. .. L .. :
- addq $7,8,$7 # .. E .. .. :
+ EXO( stq $1,0($16) ) # .. .. L .. :
+ addq $17,8,$17 # .. E .. .. :
subq $0,8,$0 # E .. .. .. : U L L U
- addq $6,8,$6 # .. .. .. E :
+ addq $16,8,$16 # .. .. .. E :
bis $2,$2,$3 # .. .. E .. :
nop # .. E .. .. :
bne $4,$misquad # U .. .. .. : U L U L
@@ -125,8 +111,8 @@ $misquad:
beq $0,$zerolength # U .. .. .. : U L U L
/* We know we have at least one trip through the byte loop */
- EXI ( ldbu $2,0($7) ) # .. .. .. L : No loads in the same quad
- addq $6,1,$6 # .. .. E .. : as the store (Section 3.8 in CWG)
+ EXI ( ldbu $2,0($17) ) # .. .. .. L : No loads in the same quad
+ addq $16,1,$16 # .. .. E .. : as the store (Section 3.8 in CWG)
nop # .. E .. .. :
br $31, $dirtyentry # L0 .. .. .. : L U U L
/* Do the trailing byte loop load, then hop into the store part of the loop */
@@ -136,8 +122,8 @@ $misquad:
* Based upon the usage context, it's worth the effort to unroll this loop
* $0 - number of bytes to be moved
* $4 - number of bytes to move as quadwords
- * $6 is current destination address
- * $7 is current source address
+ * $16 is current destination address
+ * $17 is current source address
*/
$quadaligned:
subq $4, 32, $2 # .. .. .. E : do not unroll for small stuff
@@ -155,29 +141,29 @@ $quadaligned:
* instruction memory hint instruction).
*/
$unroll4:
- EXI( ldq $1,0($7) ) # .. .. .. L
- EXI( ldq $2,8($7) ) # .. .. L ..
+ EXI( ldq $1,0($17) ) # .. .. .. L
+ EXI( ldq $2,8($17) ) # .. .. L ..
subq $4,32,$4 # .. E .. ..
nop # E .. .. .. : U U L L
- addq $7,16,$7 # .. .. .. E
- EXO( stq $1,0($6) ) # .. .. L ..
- EXO( stq $2,8($6) ) # .. L .. ..
+ addq $17,16,$17 # .. .. .. E
+ EXO( stq $1,0($16) ) # .. .. L ..
+ EXO( stq $2,8($16) ) # .. L .. ..
subq $0,16,$0 # E .. .. .. : U L L U
- addq $6,16,$6 # .. .. .. E
- EXI( ldq $1,0($7) ) # .. .. L ..
- EXI( ldq $2,8($7) ) # .. L .. ..
+ addq $16,16,$16 # .. .. .. E
+ EXI( ldq $1,0($17) ) # .. .. L ..
+ EXI( ldq $2,8($17) ) # .. L .. ..
subq $4, 32, $3 # E .. .. .. : U U L L : is there enough for another trip?
- EXO( stq $1,0($6) ) # .. .. .. L
- EXO( stq $2,8($6) ) # .. .. L ..
+ EXO( stq $1,0($16) ) # .. .. .. L
+ EXO( stq $2,8($16) ) # .. .. L ..
subq $0,16,$0 # .. E .. ..
- addq $7,16,$7 # E .. .. .. : U L L U
+ addq $17,16,$17 # E .. .. .. : U L L U
nop # .. .. .. E
nop # .. .. E ..
- addq $6,16,$6 # .. E .. ..
+ addq $16,16,$16 # .. E .. ..
bgt $3,$unroll4 # U .. .. .. : U L U L
nop
@@ -186,14 +172,14 @@ $unroll4:
beq $4, $noquads
$onequad:
- EXI( ldq $1,0($7) )
+ EXI( ldq $1,0($17) )
subq $4,8,$4
- addq $7,8,$7
+ addq $17,8,$17
nop
- EXO( stq $1,0($6) )
+ EXO( stq $1,0($16) )
subq $0,8,$0
- addq $6,8,$6
+ addq $16,8,$16
bne $4,$onequad
$noquads:
@@ -207,23 +193,23 @@ $noquads:
* There's no point in doing a lot of complex alignment calculations to try to
* to quadword stuff for a small amount of data.
* $0 - remaining number of bytes left to copy
- * $6 - current dest addr
- * $7 - current source addr
+ * $16 - current dest addr
+ * $17 - current source addr
*/
$onebyteloop:
- EXI ( ldbu $2,0($7) ) # .. .. .. L : No loads in the same quad
- addq $6,1,$6 # .. .. E .. : as the store (Section 3.8 in CWG)
+ EXI ( ldbu $2,0($17) ) # .. .. .. L : No loads in the same quad
+ addq $16,1,$16 # .. .. E .. : as the store (Section 3.8 in CWG)
nop # .. E .. .. :
nop # E .. .. .. : U L U L
$dirtyentry:
/*
- * the -1 is to compensate for the inc($6) done in a previous quadpack
+ * the -1 is to compensate for the inc($16) done in a previous quadpack
* which allows us zero dependencies within either quadpack in the loop
*/
- EXO ( stb $2,-1($6) ) # .. .. .. L :
- addq $7,1,$7 # .. .. E .. : quadpack as the load
+ EXO ( stb $2,-1($16) ) # .. .. .. L :
+ addq $17,1,$17 # .. .. E .. : quadpack as the load
subq $0,1,$0 # .. E .. .. : change count _after_ copy
bgt $0,$onebyteloop # U .. .. .. : U L U L
@@ -233,7 +219,7 @@ $exitout: # Destination for exception recovery(?)
nop # .. .. .. E
nop # .. .. E ..
nop # .. E .. ..
- ret $31,($28),1 # L0 .. .. .. : L U L U
+ ret $31,($26),1 # L0 .. .. .. : L U L U
.end __copy_user
EXPORT_SYMBOL(__copy_user)
diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig
index c9f30f4763ab..7e213ff4f01f 100644
--- a/arch/arc/Kconfig
+++ b/arch/arc/Kconfig
@@ -44,6 +44,7 @@ config ARC
select HAVE_GENERIC_DMA_COHERENT
select HAVE_KERNEL_GZIP
select HAVE_KERNEL_LZMA
+ select ARCH_HAS_RAW_COPY_USER
config MIGHT_HAVE_PCI
bool
diff --git a/arch/arc/include/asm/uaccess.h b/arch/arc/include/asm/uaccess.h
index ffd14e630aca..f35974ee7264 100644
--- a/arch/arc/include/asm/uaccess.h
+++ b/arch/arc/include/asm/uaccess.h
@@ -168,7 +168,7 @@
static inline unsigned long
-__arc_copy_from_user(void *to, const void __user *from, unsigned long n)
+raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
long res = 0;
char val;
@@ -394,11 +394,8 @@ __arc_copy_from_user(void *to, const void __user *from, unsigned long n)
return res;
}
-extern unsigned long slowpath_copy_to_user(void __user *to, const void *from,
- unsigned long n);
-
static inline unsigned long
-__arc_copy_to_user(void __user *to, const void *from, unsigned long n)
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
long res = 0;
char val;
@@ -724,24 +721,20 @@ static inline long __arc_strnlen_user(const char __user *s, long n)
}
#ifndef CONFIG_CC_OPTIMIZE_FOR_SIZE
-#define __copy_from_user(t, f, n) __arc_copy_from_user(t, f, n)
-#define __copy_to_user(t, f, n) __arc_copy_to_user(t, f, n)
+
+#define INLINE_COPY_TO_USER
+#define INLINE_COPY_FROM_USER
+
#define __clear_user(d, n) __arc_clear_user(d, n)
#define __strncpy_from_user(d, s, n) __arc_strncpy_from_user(d, s, n)
#define __strnlen_user(s, n) __arc_strnlen_user(s, n)
#else
-extern long arc_copy_from_user_noinline(void *to, const void __user * from,
- unsigned long n);
-extern long arc_copy_to_user_noinline(void __user *to, const void *from,
- unsigned long n);
extern unsigned long arc_clear_user_noinline(void __user *to,
unsigned long n);
extern long arc_strncpy_from_user_noinline (char *dst, const char __user *src,
long count);
extern long arc_strnlen_user_noinline(const char __user *src, long n);
-#define __copy_from_user(t, f, n) arc_copy_from_user_noinline(t, f, n)
-#define __copy_to_user(t, f, n) arc_copy_to_user_noinline(t, f, n)
#define __clear_user(d, n) arc_clear_user_noinline(d, n)
#define __strncpy_from_user(d, s, n) arc_strncpy_from_user_noinline(d, s, n)
#define __strnlen_user(s, n) arc_strnlen_user_noinline(s, n)
diff --git a/arch/arc/mm/extable.c b/arch/arc/mm/extable.c
index c86906b41bfe..72125a34e780 100644
--- a/arch/arc/mm/extable.c
+++ b/arch/arc/mm/extable.c
@@ -28,20 +28,6 @@ int fixup_exception(struct pt_regs *regs)
#ifdef CONFIG_CC_OPTIMIZE_FOR_SIZE
-long arc_copy_from_user_noinline(void *to, const void __user *from,
- unsigned long n)
-{
- return __arc_copy_from_user(to, from, n);
-}
-EXPORT_SYMBOL(arc_copy_from_user_noinline);
-
-long arc_copy_to_user_noinline(void __user *to, const void *from,
- unsigned long n)
-{
- return __arc_copy_to_user(to, from, n);
-}
-EXPORT_SYMBOL(arc_copy_to_user_noinline);
-
unsigned long arc_clear_user_noinline(void __user *to,
unsigned long n)
{
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 0d4e71b42c77..6fab7f34739c 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -96,6 +96,7 @@ config ARM
select PERF_USE_VMALLOC
select RTC_LIB
select SYS_SUPPORTS_APM_EMULATION
+ select ARCH_HAS_RAW_COPY_USER
# Above selects are sorted alphabetically; please add new ones
# according to that. Thanks.
help
diff --git a/arch/arm/include/asm/Kbuild b/arch/arm/include/asm/Kbuild
index b14e8c7d71bd..3a36d99ff836 100644
--- a/arch/arm/include/asm/Kbuild
+++ b/arch/arm/include/asm/Kbuild
@@ -7,6 +7,7 @@ generic-y += early_ioremap.h
generic-y += emergency-restart.h
generic-y += errno.h
generic-y += exec.h
+generic-y += extable.h
generic-y += ioctl.h
generic-y += ipcbuf.h
generic-y += irq_regs.h
diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h
index b63527359d52..2577405d082d 100644
--- a/arch/arm/include/asm/uaccess.h
+++ b/arch/arm/include/asm/uaccess.h
@@ -24,25 +24,7 @@
#define __put_user_unaligned __put_user
#endif
-/*
- * The exception table consists of pairs of addresses: the first is the
- * address of an instruction that is allowed to fault, and the second is
- * the address at which the program should continue. No registers are
- * modified, so it is entirely up to the continuation code to figure out
- * what to do.
- *
- * All the routines below use bits of fixup code that are out of line
- * with the main instruction path. This means when everything is well,
- * we don't even have to jump over them. Further, they do not intrude
- * on our cache or tlb entries.
- */
-
-struct exception_table_entry
-{
- unsigned long insn, fixup;
-};
-
-extern int fixup_exception(struct pt_regs *regs);
+#include <asm/extable.h>
/*
* These two functions allow hooking accesses to userspace to increase
@@ -473,7 +455,7 @@ extern unsigned long __must_check
arm_copy_from_user(void *to, const void __user *from, unsigned long n);
static inline unsigned long __must_check
-__arch_copy_from_user(void *to, const void __user *from, unsigned long n)
+raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
unsigned int __ua_flags;
@@ -489,7 +471,7 @@ extern unsigned long __must_check
__copy_to_user_std(void __user *to, const void *from, unsigned long n);
static inline unsigned long __must_check
-__arch_copy_to_user(void __user *to, const void *from, unsigned long n)
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
#ifndef CONFIG_UACCESS_WITH_MEMCPY
unsigned int __ua_flags;
@@ -517,54 +499,22 @@ __clear_user(void __user *addr, unsigned long n)
}
#else
-#define __arch_copy_from_user(to, from, n) \
- (memcpy(to, (void __force *)from, n), 0)
-#define __arch_copy_to_user(to, from, n) \
- (memcpy((void __force *)to, from, n), 0)
-#define __clear_user(addr, n) (memset((void __force *)addr, 0, n), 0)
-#endif
-
-static inline unsigned long __must_check
-__copy_from_user(void *to, const void __user *from, unsigned long n)
-{
- check_object_size(to, n, false);
- return __arch_copy_from_user(to, from, n);
-}
-
-static inline unsigned long __must_check
-copy_from_user(void *to, const void __user *from, unsigned long n)
+static inline unsigned long
+raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
- unsigned long res = n;
-
- check_object_size(to, n, false);
-
- if (likely(access_ok(VERIFY_READ, from, n)))
- res = __arch_copy_from_user(to, from, n);
- if (unlikely(res))
- memset(to + (n - res), 0, res);
- return res;
-}
-
-static inline unsigned long __must_check
-__copy_to_user(void __user *to, const void *from, unsigned long n)
-{
- check_object_size(from, n, true);
-
- return __arch_copy_to_user(to, from, n);
+ memcpy(to, (const void __force *)from, n);
+ return 0;
}
-
-static inline unsigned long __must_check
-copy_to_user(void __user *to, const void *from, unsigned long n)
+static inline unsigned long
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
- check_object_size(from, n, true);
-
- if (access_ok(VERIFY_WRITE, to, n))
- n = __arch_copy_to_user(to, from, n);
- return n;
+ memcpy((void __force *)to, from, n);
+ return 0;
}
-
-#define __copy_to_user_inatomic __copy_to_user
-#define __copy_from_user_inatomic __copy_from_user
+#define __clear_user(addr, n) (memset((void __force *)addr, 0, n), 0)
+#endif
+#define INLINE_COPY_TO_USER
+#define INLINE_COPY_FROM_USER
static inline unsigned long __must_check clear_user(void __user *to, unsigned long n)
{
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 3741859765cf..3c833ff3303c 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -115,6 +115,7 @@ config ARM64
select SPARSE_IRQ
select SYSCTL_EXCEPTION_TRACE
select THREAD_INFO_IN_TASK
+ select ARCH_HAS_RAW_COPY_USER
help
ARM 64-bit (AArch64) Linux support.
diff --git a/arch/arm64/include/asm/extable.h b/arch/arm64/include/asm/extable.h
new file mode 100644
index 000000000000..42f50f15a44c
--- /dev/null
+++ b/arch/arm64/include/asm/extable.h
@@ -0,0 +1,25 @@
+#ifndef __ASM_EXTABLE_H
+#define __ASM_EXTABLE_H
+
+/*
+ * The exception table consists of pairs of relative offsets: the first
+ * is the relative offset to an instruction that is allowed to fault,
+ * and the second is the relative offset at which the program should
+ * continue. No registers are modified, so it is entirely up to the
+ * continuation code to figure out what to do.
+ *
+ * All the routines below use bits of fixup code that are out of line
+ * with the main instruction path. This means when everything is well,
+ * we don't even have to jump over them. Further, they do not intrude
+ * on our cache or tlb entries.
+ */
+
+struct exception_table_entry
+{
+ int insn, fixup;
+};
+
+#define ARCH_HAS_RELATIVE_EXTABLE
+
+extern int fixup_exception(struct pt_regs *regs);
+#endif
diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index 7c514e10a08e..ba497172610d 100644
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -33,28 +33,7 @@
#include <asm/ptrace.h>
#include <asm/memory.h>
#include <asm/compiler.h>
-
-/*
- * The exception table consists of pairs of relative offsets: the first
- * is the relative offset to an instruction that is allowed to fault,
- * and the second is the relative offset at which the program should
- * continue. No registers are modified, so it is entirely up to the
- * continuation code to figure out what to do.
- *
- * All the routines below use bits of fixup code that are out of line
- * with the main instruction path. This means when everything is well,
- * we don't even have to jump over them. Further, they do not intrude
- * on our cache or tlb entries.
- */
-
-struct exception_table_entry
-{
- int insn, fixup;
-};
-
-#define ARCH_HAS_RELATIVE_EXTABLE
-
-extern int fixup_exception(struct pt_regs *regs);
+#include <asm/extable.h>
#define KERNEL_DS (-1UL)
#define get_ds() (KERNEL_DS)
@@ -352,58 +331,13 @@ do { \
})
extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n);
+#define raw_copy_from_user __arch_copy_from_user
extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n);
-extern unsigned long __must_check __copy_in_user(void __user *to, const void __user *from, unsigned long n);
+#define raw_copy_to_user __arch_copy_to_user
+extern unsigned long __must_check raw_copy_in_user(void __user *to, const void __user *from, unsigned long n);
extern unsigned long __must_check __clear_user(void __user *addr, unsigned long n);
-
-static inline unsigned long __must_check __copy_from_user(void *to, const void __user *from, unsigned long n)
-{
- kasan_check_write(to, n);
- check_object_size(to, n, false);
- return __arch_copy_from_user(to, from, n);
-}
-
-static inline unsigned long __must_check __copy_to_user(void __user *to, const void *from, unsigned long n)
-{
- kasan_check_read(from, n);
- check_object_size(from, n, true);
- return __arch_copy_to_user(to, from, n);
-}
-
-static inline unsigned long __must_check copy_from_user(void *to, const void __user *from, unsigned long n)
-{
- unsigned long res = n;
- kasan_check_write(to, n);
- check_object_size(to, n, false);
-
- if (access_ok(VERIFY_READ, from, n)) {
- res = __arch_copy_from_user(to, from, n);
- }
- if (unlikely(res))
- memset(to + (n - res), 0, res);
- return res;
-}
-
-static inline unsigned long __must_check copy_to_user(void __user *to, const void *from, unsigned long n)
-{
- kasan_check_read(from, n);
- check_object_size(from, n, true);
-
- if (access_ok(VERIFY_WRITE, to, n)) {
- n = __arch_copy_to_user(to, from, n);
- }
- return n;
-}
-
-static inline unsigned long __must_check copy_in_user(void __user *to, const void __user *from, unsigned long n)
-{
- if (access_ok(VERIFY_READ, from, n) && access_ok(VERIFY_WRITE, to, n))
- n = __copy_in_user(to, from, n);
- return n;
-}
-
-#define __copy_to_user_inatomic __copy_to_user
-#define __copy_from_user_inatomic __copy_from_user
+#define INLINE_COPY_TO_USER
+#define INLINE_COPY_FROM_USER
static inline unsigned long __must_check clear_user(void __user *to, unsigned long n)
{
diff --git a/arch/arm64/kernel/arm64ksyms.c b/arch/arm64/kernel/arm64ksyms.c
index e9c4dc9e0ada..67368c7329c0 100644
--- a/arch/arm64/kernel/arm64ksyms.c
+++ b/arch/arm64/kernel/arm64ksyms.c
@@ -38,7 +38,7 @@ EXPORT_SYMBOL(clear_page);
EXPORT_SYMBOL(__arch_copy_from_user);
EXPORT_SYMBOL(__arch_copy_to_user);
EXPORT_SYMBOL(__clear_user);
-EXPORT_SYMBOL(__copy_in_user);
+EXPORT_SYMBOL(raw_copy_in_user);
/* physical memory */
EXPORT_SYMBOL(memstart_addr);
diff --git a/arch/arm64/lib/copy_in_user.S b/arch/arm64/lib/copy_in_user.S
index 47184c3a97da..b24a830419ad 100644
--- a/arch/arm64/lib/copy_in_user.S
+++ b/arch/arm64/lib/copy_in_user.S
@@ -64,14 +64,14 @@
.endm
end .req x5
-ENTRY(__copy_in_user)
+ENTRY(raw_copy_in_user)
uaccess_enable_not_uao x3, x4
add end, x0, x2
#include "copy_template.S"
uaccess_disable_not_uao x3
mov x0, #0
ret
-ENDPROC(__copy_in_user)
+ENDPROC(raw_copy_in_user)
.section .fixup,"ax"
.align 2
diff --git a/arch/avr32/Kconfig b/arch/avr32/Kconfig
index 7e75d45e20cd..8c349f2a9ebb 100644
--- a/arch/avr32/Kconfig
+++ b/arch/avr32/Kconfig
@@ -19,6 +19,7 @@ config AVR32
select HAVE_MOD_ARCH_SPECIFIC
select MODULES_USE_ELF_RELA
select HAVE_NMI
+ select ARCH_HAS_RAW_COPY_USER
help
AVR32 is a high-performance 32-bit RISC microprocessor core,
designed for cost-sensitive embedded applications, with particular
diff --git a/arch/avr32/include/asm/Kbuild b/arch/avr32/include/asm/Kbuild
index 3d7ef2c17a7c..ff7d7393b25c 100644
--- a/arch/avr32/include/asm/Kbuild
+++ b/arch/avr32/include/asm/Kbuild
@@ -5,6 +5,7 @@ generic-y += device.h
generic-y += div64.h
generic-y += emergency-restart.h
generic-y += exec.h
+generic-y += extable.h
generic-y += futex.h
generic-y += irq_regs.h
generic-y += irq_work.h
diff --git a/arch/avr32/include/asm/uaccess.h b/arch/avr32/include/asm/uaccess.h
index 7ca5cb33369b..619d50e843a5 100644
--- a/arch/avr32/include/asm/uaccess.h
+++ b/arch/avr32/include/asm/uaccess.h
@@ -66,34 +66,18 @@ static inline void set_fs(mm_segment_t s)
extern __kernel_size_t __copy_user(void *to, const void *from,
__kernel_size_t n);
-extern __kernel_size_t copy_to_user(void __user *to, const void *from,
- __kernel_size_t n);
-extern __kernel_size_t ___copy_from_user(void *to, const void __user *from,
- __kernel_size_t n);
-
-static inline __kernel_size_t __copy_to_user(void __user *to, const void *from,
- __kernel_size_t n)
+static inline unsigned long
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
return __copy_user((void __force *)to, from, n);
}
-static inline __kernel_size_t __copy_from_user(void *to,
- const void __user *from,
- __kernel_size_t n)
+static inline unsigned long
+raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
return __copy_user(to, (const void __force *)from, n);
}
-static inline __kernel_size_t copy_from_user(void *to,
- const void __user *from,
- __kernel_size_t n)
-{
- size_t res = ___copy_from_user(to, from, n);
- if (unlikely(res))
- memset(to + (n - res), 0, res);
- return res;
-}
-
-#define __copy_to_user_inatomic __copy_to_user
-#define __copy_from_user_inatomic __copy_from_user
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
/*
* put_user: - Write a simple value into user space.
@@ -323,9 +307,6 @@ extern long __strnlen_user(const char __user *__s, long __n);
#define strlen_user(s) strnlen_user(s, ~0UL >> 1)
-struct exception_table_entry
-{
- unsigned long insn, fixup;
-};
+#include <asm/extable.h>
#endif /* __ASM_AVR32_UACCESS_H */
diff --git a/arch/avr32/kernel/avr32_ksyms.c b/arch/avr32/kernel/avr32_ksyms.c
index 0d05fd095468..0cdae8eb0220 100644
--- a/arch/avr32/kernel/avr32_ksyms.c
+++ b/arch/avr32/kernel/avr32_ksyms.c
@@ -36,8 +36,6 @@ EXPORT_SYMBOL(copy_page);
/*
* Userspace access stuff.
*/
-EXPORT_SYMBOL(___copy_from_user);
-EXPORT_SYMBOL(copy_to_user);
EXPORT_SYMBOL(__copy_user);
EXPORT_SYMBOL(strncpy_from_user);
EXPORT_SYMBOL(__strncpy_from_user);
diff --git a/arch/avr32/lib/copy_user.S b/arch/avr32/lib/copy_user.S
index 075373471da1..f43275796a0c 100644
--- a/arch/avr32/lib/copy_user.S
+++ b/arch/avr32/lib/copy_user.S
@@ -23,21 +23,6 @@
*/
.text
.align 1
- .global ___copy_from_user
- .type ___copy_from_user, @function
-___copy_from_user:
- branch_if_kernel r8, __copy_user
- ret_if_privileged r8, r11, r10, r10
- rjmp __copy_user
- .size ___copy_from_user, . - ___copy_from_user
-
- .global copy_to_user
- .type copy_to_user, @function
-copy_to_user:
- branch_if_kernel r8, __copy_user
- ret_if_privileged r8, r12, r10, r10
- .size copy_to_user, . - copy_to_user
-
.global __copy_user
.type __copy_user, @function
__copy_user:
diff --git a/arch/blackfin/Kconfig b/arch/blackfin/Kconfig
index 3c1bd640042a..919dad1436f7 100644
--- a/arch/blackfin/Kconfig
+++ b/arch/blackfin/Kconfig
@@ -41,6 +41,7 @@ config BLACKFIN
select MODULES_USE_ELF_RELA
select HAVE_DEBUG_STACKOVERFLOW
select HAVE_NMI
+ select ARCH_HAS_RAW_COPY_USER
config GENERIC_CSUM
def_bool y
diff --git a/arch/blackfin/include/asm/Kbuild b/arch/blackfin/include/asm/Kbuild
index 625db8ac815e..dc4ef9ac1e83 100644
--- a/arch/blackfin/include/asm/Kbuild
+++ b/arch/blackfin/include/asm/Kbuild
@@ -7,6 +7,7 @@ generic-y += device.h
generic-y += div64.h
generic-y += emergency-restart.h
generic-y += errno.h
+generic-y += extable.h
generic-y += fb.h
generic-y += futex.h
generic-y += hw_irq.h
diff --git a/arch/blackfin/include/asm/uaccess.h b/arch/blackfin/include/asm/uaccess.h
index c9fedc3be30c..f54a34f31cea 100644
--- a/arch/blackfin/include/asm/uaccess.h
+++ b/arch/blackfin/include/asm/uaccess.h
@@ -42,22 +42,7 @@ static inline int _access_ok(unsigned long addr, unsigned long size) { return 1;
extern int _access_ok(unsigned long addr, unsigned long size);
#endif
-/*
- * The exception table consists of pairs of addresses: the first is the
- * address of an instruction that is allowed to fault, and the second is
- * the address at which the program should continue. No registers are
- * modified, so it is entirely up to the continuation code to figure out
- * what to do.
- *
- * All the routines below use bits of fixup code that are out of line
- * with the main instruction path. This means when everything is well,
- * we don't even have to jump over them. Further, they do not intrude
- * on our cache or tlb entries.
- */
-
-struct exception_table_entry {
- unsigned long insn, fixup;
-};
+#include <asm/extable.h>
/*
* These are the main single-value transfer routines. They automatically
@@ -159,41 +144,23 @@ static inline int bad_user_access_length(void)
: "a" (__ptr(ptr))); \
})
-#define __copy_to_user_inatomic __copy_to_user
-#define __copy_from_user_inatomic __copy_from_user
-
static inline unsigned long __must_check
-__copy_from_user(void *to, const void __user *from, unsigned long n)
+raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
memcpy(to, (const void __force *)from, n);
return 0;
}
static inline unsigned long __must_check
-__copy_to_user(void __user *to, const void *from, unsigned long n)
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
memcpy((void __force *)to, from, n);
SSYNC();
return 0;
}
-static inline unsigned long __must_check
-copy_from_user(void *to, const void __user *from, unsigned long n)
-{
- if (likely(access_ok(VERIFY_READ, from, n)))
- return __copy_from_user(to, from, n);
- memset(to, 0, n);
- return n;
-}
-
-static inline unsigned long __must_check
-copy_to_user(void __user *to, const void *from, unsigned long n)
-{
- if (likely(access_ok(VERIFY_WRITE, to, n)))
- return __copy_to_user(to, from, n);
- return n;
-}
-
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
/*
* Copy a null terminated string from userspace.
*/
diff --git a/arch/c6x/Kconfig b/arch/c6x/Kconfig
index 5aa8ea8bad2d..3c7bd9a29f90 100644
--- a/arch/c6x/Kconfig
+++ b/arch/c6x/Kconfig
@@ -18,6 +18,7 @@ config C6X
select GENERIC_CLOCKEVENTS
select MODULES_USE_ELF_RELA
select ARCH_NO_COHERENT_DMA_MMAP
+ select ARCH_HAS_RAW_COPY_USER
config MMU
def_bool n
diff --git a/arch/c6x/include/asm/uaccess.h b/arch/c6x/include/asm/uaccess.h
index 79fb6f439c5c..ba6756879f00 100644
--- a/arch/c6x/include/asm/uaccess.h
+++ b/arch/c6x/include/asm/uaccess.h
@@ -13,17 +13,11 @@
#include <linux/compiler.h>
#include <linux/string.h>
-#ifdef CONFIG_ACCESS_CHECK
-#define __access_ok _access_ok
-#endif
-
/*
- * __copy_from_user/copy_to_user are based on ones in asm-generic/uaccess.h
- *
* C6X supports unaligned 32 and 64 bit loads and stores.
*/
-static inline __must_check long __copy_from_user(void *to,
- const void __user *from, unsigned long n)
+static inline __must_check unsigned long
+raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
u32 tmp32;
u64 tmp64;
@@ -58,8 +52,8 @@ static inline __must_check long __copy_from_user(void *to,
return 0;
}
-static inline __must_check long __copy_to_user(void __user *to,
- const void *from, unsigned long n)
+static inline __must_check unsigned long
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
u32 tmp32;
u64 tmp64;
@@ -93,6 +87,8 @@ static inline __must_check long __copy_to_user(void __user *to,
memcpy((void __force *)to, from, n);
return 0;
}
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
extern int _access_ok(unsigned long addr, unsigned long size);
#ifdef CONFIG_ACCESS_CHECK
diff --git a/arch/cris/Kconfig b/arch/cris/Kconfig
index 71b758dc3a96..36f94c45e3f9 100644
--- a/arch/cris/Kconfig
+++ b/arch/cris/Kconfig
@@ -71,6 +71,7 @@ config CRIS
select GENERIC_SCHED_CLOCK if ETRAX_ARCH_V32
select HAVE_DEBUG_BUGVERBOSE if ETRAX_ARCH_V32
select HAVE_NMI
+ select ARCH_HAS_RAW_COPY_USER
config HZ
int
diff --git a/arch/cris/arch-v10/lib/usercopy.c b/arch/cris/arch-v10/lib/usercopy.c
index 1ba7cc000dfc..48fa37fe0f9b 100644
--- a/arch/cris/arch-v10/lib/usercopy.c
+++ b/arch/cris/arch-v10/lib/usercopy.c
@@ -188,11 +188,10 @@ unsigned long __copy_user(void __user *pdst, const void *psrc, unsigned long pn)
}
EXPORT_SYMBOL(__copy_user);
-/* Copy from user to kernel, zeroing the bytes that were inaccessible in
- userland. The return-value is the number of bytes that were
+/* Copy from user to kernel. The return-value is the number of bytes that were
inaccessible. */
-unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc,
+unsigned long __copy_user_in(void *pdst, const void __user *psrc,
unsigned long pn)
{
/* We want the parameters put in special registers.
@@ -217,19 +216,17 @@ unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc,
{
__asm_copy_from_user_1 (dst, src, retn);
n--;
+ if (retn)
+ goto exception;
}
if (((unsigned long) src & 2) && n >= 2)
{
__asm_copy_from_user_2 (dst, src, retn);
n -= 2;
+ if (retn)
+ goto exception;
}
-
- /* We only need one check after the unalignment-adjustments, because
- if both adjustments were done, either both or neither reference
- had an exception. */
- if (retn != 0)
- goto copy_exception_bytes;
}
/* Decide which copying method to use. */
@@ -328,7 +325,7 @@ unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc,
n -= 4;
if (retn)
- goto copy_exception_bytes;
+ goto exception;
}
/* If we get here, there were no memory read faults. */
@@ -356,20 +353,10 @@ unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc,
bytes. */
return retn;
-copy_exception_bytes:
- /* We already have "retn" bytes cleared, and need to clear the
- remaining "n" bytes. A non-optimized simple byte-for-byte in-line
- memset is preferred here, since this isn't speed-critical code and
- we'd rather have this a leaf-function than calling memset. */
- {
- char *endp;
- for (endp = dst + n; dst < endp; dst++)
- *dst = 0;
- }
-
+exception:
return retn + n;
}
-EXPORT_SYMBOL(__copy_user_zeroing);
+EXPORT_SYMBOL(__copy_user_in);
/* Zero userspace. */
unsigned long __do_clear_user(void __user *pto, unsigned long pn)
diff --git a/arch/cris/arch-v32/lib/usercopy.c b/arch/cris/arch-v32/lib/usercopy.c
index 05e58dab800d..20b608026913 100644
--- a/arch/cris/arch-v32/lib/usercopy.c
+++ b/arch/cris/arch-v32/lib/usercopy.c
@@ -156,10 +156,9 @@ unsigned long __copy_user(void __user *pdst, const void *psrc, unsigned long pn)
}
EXPORT_SYMBOL(__copy_user);
-/* Copy from user to kernel, zeroing the bytes that were inaccessible in
- userland. The return-value is the number of bytes that were
+/* Copy from user to kernel. The return-value is the number of bytes that were
inaccessible. */
-unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc,
+unsigned long __copy_user_in(void *pdst, const void __user *psrc,
unsigned long pn)
{
/* We want the parameters put in special registers.
@@ -184,19 +183,18 @@ unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc,
{
__asm_copy_from_user_1 (dst, src, retn);
n--;
+ if (retn != 0)
+ goto exception;
}
if (((unsigned long) src & 2) && n >= 2)
{
__asm_copy_from_user_2 (dst, src, retn);
n -= 2;
+ if (retn != 0)
+ goto exception;
}
- /* We only need one check after the unalignment-adjustments, because
- if both adjustments were done, either both or neither reference
- had an exception. */
- if (retn != 0)
- goto copy_exception_bytes;
}
/* Movem is dirt cheap. The overheap is low enough to always use the
@@ -279,7 +277,7 @@ unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc,
n -= 4;
if (retn)
- goto copy_exception_bytes;
+ goto exception;
}
/* If we get here, there were no memory read faults. */
@@ -307,20 +305,10 @@ unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc,
bytes. */
return retn;
-copy_exception_bytes:
- /* We already have "retn" bytes cleared, and need to clear the
- remaining "n" bytes. A non-optimized simple byte-for-byte in-line
- memset is preferred here, since this isn't speed-critical code and
- we'd rather have this a leaf-function than calling memset. */
- {
- char *endp;
- for (endp = dst + n; dst < endp; dst++)
- *dst = 0;
- }
-
+exception:
return retn + n;
}
-EXPORT_SYMBOL(__copy_user_zeroing);
+EXPORT_SYMBOL(__copy_user_in);
/* Zero userspace. */
unsigned long __do_clear_user(void __user *pto, unsigned long pn)
diff --git a/arch/cris/include/arch-v10/arch/uaccess.h b/arch/cris/include/arch-v10/arch/uaccess.h
index 65b02d9b605a..5477c98c2281 100644
--- a/arch/cris/include/arch-v10/arch/uaccess.h
+++ b/arch/cris/include/arch-v10/arch/uaccess.h
@@ -172,16 +172,14 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
__asm_copy_user_cont(to, from, ret, \
" move.b [%1+],$r9\n" \
"2: move.b $r9,[%0+]\n", \
- "3: addq 1,%2\n" \
- " clear.b [%0+]\n", \
+ "3: addq 1,%2\n", \
" .dword 2b,3b\n")
#define __asm_copy_from_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
__asm_copy_user_cont(to, from, ret, \
" move.w [%1+],$r9\n" \
"2: move.w $r9,[%0+]\n" COPY, \
- "3: addq 2,%2\n" \
- " clear.w [%0+]\n" FIXUP, \
+ "3: addq 2,%2\n" FIXUP, \
" .dword 2b,3b\n" TENTRY)
#define __asm_copy_from_user_2(to, from, ret) \
@@ -191,16 +189,14 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
__asm_copy_from_user_2x_cont(to, from, ret, \
" move.b [%1+],$r9\n" \
"4: move.b $r9,[%0+]\n", \
- "5: addq 1,%2\n" \
- " clear.b [%0+]\n", \
+ "5: addq 1,%2\n", \
" .dword 4b,5b\n")
#define __asm_copy_from_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
__asm_copy_user_cont(to, from, ret, \
" move.d [%1+],$r9\n" \
"2: move.d $r9,[%0+]\n" COPY, \
- "3: addq 4,%2\n" \
- " clear.d [%0+]\n" FIXUP, \
+ "3: addq 4,%2\n" FIXUP, \
" .dword 2b,3b\n" TENTRY)
#define __asm_copy_from_user_4(to, from, ret) \
@@ -210,8 +206,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
__asm_copy_from_user_4x_cont(to, from, ret, \
" move.b [%1+],$r9\n" \
"4: move.b $r9,[%0+]\n", \
- "5: addq 1,%2\n" \
- " clear.b [%0+]\n", \
+ "5: addq 1,%2\n", \
" .dword 4b,5b\n")
#define __asm_copy_from_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
@@ -219,7 +214,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
" move.w [%1+],$r9\n" \
"4: move.w $r9,[%0+]\n" COPY, \
"5: addq 2,%2\n" \
- " clear.w [%0+]\n" FIXUP, \
+ FIXUP, \
" .dword 4b,5b\n" TENTRY)
#define __asm_copy_from_user_6(to, from, ret) \
@@ -229,8 +224,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
__asm_copy_from_user_6x_cont(to, from, ret, \
" move.b [%1+],$r9\n" \
"6: move.b $r9,[%0+]\n", \
- "7: addq 1,%2\n" \
- " clear.b [%0+]\n", \
+ "7: addq 1,%2\n", \
" .dword 6b,7b\n")
#define __asm_copy_from_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
@@ -238,7 +232,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
" move.d [%1+],$r9\n" \
"4: move.d $r9,[%0+]\n" COPY, \
"5: addq 4,%2\n" \
- " clear.d [%0+]\n" FIXUP, \
+ FIXUP, \
" .dword 4b,5b\n" TENTRY)
#define __asm_copy_from_user_8(to, from, ret) \
@@ -248,8 +242,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
__asm_copy_from_user_8x_cont(to, from, ret, \
" move.b [%1+],$r9\n" \
"6: move.b $r9,[%0+]\n", \
- "7: addq 1,%2\n" \
- " clear.b [%0+]\n", \
+ "7: addq 1,%2\n", \
" .dword 6b,7b\n")
#define __asm_copy_from_user_10x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
@@ -257,7 +250,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
" move.w [%1+],$r9\n" \
"6: move.w $r9,[%0+]\n" COPY, \
"7: addq 2,%2\n" \
- " clear.w [%0+]\n" FIXUP, \
+ FIXUP, \
" .dword 6b,7b\n" TENTRY)
#define __asm_copy_from_user_10(to, from, ret) \
@@ -267,8 +260,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
__asm_copy_from_user_10x_cont(to, from, ret, \
" move.b [%1+],$r9\n" \
"8: move.b $r9,[%0+]\n", \
- "9: addq 1,%2\n" \
- " clear.b [%0+]\n", \
+ "9: addq 1,%2\n", \
" .dword 8b,9b\n")
#define __asm_copy_from_user_12x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
@@ -276,7 +268,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
" move.d [%1+],$r9\n" \
"6: move.d $r9,[%0+]\n" COPY, \
"7: addq 4,%2\n" \
- " clear.d [%0+]\n" FIXUP, \
+ FIXUP, \
" .dword 6b,7b\n" TENTRY)
#define __asm_copy_from_user_12(to, from, ret) \
@@ -286,8 +278,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
__asm_copy_from_user_12x_cont(to, from, ret, \
" move.b [%1+],$r9\n" \
"8: move.b $r9,[%0+]\n", \
- "9: addq 1,%2\n" \
- " clear.b [%0+]\n", \
+ "9: addq 1,%2\n", \
" .dword 8b,9b\n")
#define __asm_copy_from_user_14x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
@@ -295,7 +286,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
" move.w [%1+],$r9\n" \
"8: move.w $r9,[%0+]\n" COPY, \
"9: addq 2,%2\n" \
- " clear.w [%0+]\n" FIXUP, \
+ FIXUP, \
" .dword 8b,9b\n" TENTRY)
#define __asm_copy_from_user_14(to, from, ret) \
@@ -305,8 +296,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
__asm_copy_from_user_14x_cont(to, from, ret, \
" move.b [%1+],$r9\n" \
"10: move.b $r9,[%0+]\n", \
- "11: addq 1,%2\n" \
- " clear.b [%0+]\n", \
+ "11: addq 1,%2\n", \
" .dword 10b,11b\n")
#define __asm_copy_from_user_16x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
@@ -314,7 +304,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
" move.d [%1+],$r9\n" \
"8: move.d $r9,[%0+]\n" COPY, \
"9: addq 4,%2\n" \
- " clear.d [%0+]\n" FIXUP, \
+ FIXUP, \
" .dword 8b,9b\n" TENTRY)
#define __asm_copy_from_user_16(to, from, ret) \
@@ -325,7 +315,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
" move.d [%1+],$r9\n" \
"10: move.d $r9,[%0+]\n" COPY, \
"11: addq 4,%2\n" \
- " clear.d [%0+]\n" FIXUP, \
+ FIXUP, \
" .dword 10b,11b\n" TENTRY)
#define __asm_copy_from_user_20(to, from, ret) \
@@ -336,7 +326,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
" move.d [%1+],$r9\n" \
"12: move.d $r9,[%0+]\n" COPY, \
"13: addq 4,%2\n" \
- " clear.d [%0+]\n" FIXUP, \
+ FIXUP, \
" .dword 12b,13b\n" TENTRY)
#define __asm_copy_from_user_24(to, from, ret) \
diff --git a/arch/cris/include/arch-v32/arch/uaccess.h b/arch/cris/include/arch-v32/arch/uaccess.h
index 3196019706cb..dc2ce090f624 100644
--- a/arch/cris/include/arch-v32/arch/uaccess.h
+++ b/arch/cris/include/arch-v32/arch/uaccess.h
@@ -178,8 +178,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
"2: move.b [%1+],$acr\n" \
" move.b $acr,[%0+]\n", \
"3: addq 1,%2\n" \
- " jump 1b\n" \
- " clear.b [%0+]\n", \
+ " jump 1b\n", \
" .dword 2b,3b\n")
#define __asm_copy_from_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
@@ -189,8 +188,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
" move.w $acr,[%0+]\n", \
FIXUP \
"3: addq 2,%2\n" \
- " jump 1b\n" \
- " clear.w [%0+]\n", \
+ " jump 1b\n", \
TENTRY \
" .dword 2b,3b\n")
@@ -201,8 +199,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
__asm_copy_from_user_2x_cont(to, from, ret, \
"4: move.b [%1+],$acr\n" \
" move.b $acr,[%0+]\n", \
- "5: addq 1,%2\n" \
- " clear.b [%0+]\n", \
+ "5: addq 1,%2\n", \
" .dword 4b,5b\n")
#define __asm_copy_from_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
@@ -212,8 +209,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
" move.d $acr,[%0+]\n", \
FIXUP \
"3: addq 4,%2\n" \
- " jump 1b\n" \
- " clear.d [%0+]\n", \
+ " jump 1b\n", \
TENTRY \
" .dword 2b,3b\n")
@@ -224,8 +220,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
__asm_copy_from_user_4x_cont(to, from, ret, \
"4: move.b [%1+],$acr\n" \
" move.b $acr,[%0+]\n", \
- "5: addq 1,%2\n" \
- " clear.b [%0+]\n", \
+ "5: addq 1,%2\n", \
" .dword 4b,5b\n")
#define __asm_copy_from_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
@@ -234,8 +229,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
"4: move.w [%1+],$acr\n" \
" move.w $acr,[%0+]\n", \
FIXUP \
- "5: addq 2,%2\n" \
- " clear.w [%0+]\n", \
+ "5: addq 2,%2\n", \
TENTRY \
" .dword 4b,5b\n")
@@ -246,8 +240,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
__asm_copy_from_user_6x_cont(to, from, ret, \
"6: move.b [%1+],$acr\n" \
" move.b $acr,[%0+]\n", \
- "7: addq 1,%2\n" \
- " clear.b [%0+]\n", \
+ "7: addq 1,%2\n", \
" .dword 6b,7b\n")
#define __asm_copy_from_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
@@ -256,8 +249,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
"4: move.d [%1+],$acr\n" \
" move.d $acr,[%0+]\n", \
FIXUP \
- "5: addq 4,%2\n" \
- " clear.d [%0+]\n", \
+ "5: addq 4,%2\n", \
TENTRY \
" .dword 4b,5b\n")
@@ -268,8 +260,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
__asm_copy_from_user_8x_cont(to, from, ret, \
"6: move.b [%1+],$acr\n" \
" move.b $acr,[%0+]\n", \
- "7: addq 1,%2\n" \
- " clear.b [%0+]\n", \
+ "7: addq 1,%2\n", \
" .dword 6b,7b\n")
#define __asm_copy_from_user_10x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
@@ -278,8 +269,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
"6: move.w [%1+],$acr\n" \
" move.w $acr,[%0+]\n", \
FIXUP \
- "7: addq 2,%2\n" \
- " clear.w [%0+]\n", \
+ "7: addq 2,%2\n", \
TENTRY \
" .dword 6b,7b\n")
@@ -290,8 +280,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
__asm_copy_from_user_10x_cont(to, from, ret, \
"8: move.b [%1+],$acr\n" \
" move.b $acr,[%0+]\n", \
- "9: addq 1,%2\n" \
- " clear.b [%0+]\n", \
+ "9: addq 1,%2\n", \
" .dword 8b,9b\n")
#define __asm_copy_from_user_12x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
@@ -300,8 +289,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
"6: move.d [%1+],$acr\n" \
" move.d $acr,[%0+]\n", \
FIXUP \
- "7: addq 4,%2\n" \
- " clear.d [%0+]\n", \
+ "7: addq 4,%2\n", \
TENTRY \
" .dword 6b,7b\n")
@@ -312,8 +300,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
__asm_copy_from_user_12x_cont(to, from, ret, \
"8: move.b [%1+],$acr\n" \
" move.b $acr,[%0+]\n", \
- "9: addq 1,%2\n" \
- " clear.b [%0+]\n", \
+ "9: addq 1,%2\n", \
" .dword 8b,9b\n")
#define __asm_copy_from_user_14x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
@@ -322,8 +309,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
"8: move.w [%1+],$acr\n" \
" move.w $acr,[%0+]\n", \
FIXUP \
- "9: addq 2,%2\n" \
- " clear.w [%0+]\n", \
+ "9: addq 2,%2\n", \
TENTRY \
" .dword 8b,9b\n")
@@ -334,8 +320,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
__asm_copy_from_user_14x_cont(to, from, ret, \
"10: move.b [%1+],$acr\n" \
" move.b $acr,[%0+]\n", \
- "11: addq 1,%2\n" \
- " clear.b [%0+]\n", \
+ "11: addq 1,%2\n", \
" .dword 10b,11b\n")
#define __asm_copy_from_user_16x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
@@ -344,8 +329,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
"8: move.d [%1+],$acr\n" \
" move.d $acr,[%0+]\n", \
FIXUP \
- "9: addq 4,%2\n" \
- " clear.d [%0+]\n", \
+ "9: addq 4,%2\n", \
TENTRY \
" .dword 8b,9b\n")
@@ -358,8 +342,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
"10: move.d [%1+],$acr\n" \
" move.d $acr,[%0+]\n", \
FIXUP \
- "11: addq 4,%2\n" \
- " clear.d [%0+]\n", \
+ "11: addq 4,%2\n", \
TENTRY \
" .dword 10b,11b\n")
@@ -372,8 +355,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count)
"12: move.d [%1+],$acr\n" \
" move.d $acr,[%0+]\n", \
FIXUP \
- "13: addq 4,%2\n" \
- " clear.d [%0+]\n", \
+ "13: addq 4,%2\n", \
TENTRY \
" .dword 12b,13b\n")
diff --git a/arch/cris/include/asm/Kbuild b/arch/cris/include/asm/Kbuild
index 0f5132b08896..2890099992a9 100644
--- a/arch/cris/include/asm/Kbuild
+++ b/arch/cris/include/asm/Kbuild
@@ -9,6 +9,7 @@ generic-y += device.h
generic-y += div64.h
generic-y += errno.h
generic-y += exec.h
+generic-y += extable.h
generic-y += emergency-restart.h
generic-y += fcntl.h
generic-y += futex.h
diff --git a/arch/cris/include/asm/uaccess.h b/arch/cris/include/asm/uaccess.h
index 5f5b8f53d2d7..0d473aec3066 100644
--- a/arch/cris/include/asm/uaccess.h
+++ b/arch/cris/include/asm/uaccess.h
@@ -50,23 +50,7 @@
#define access_ok(type, addr, size) __access_ok((unsigned long)(addr), (size))
#include <arch/uaccess.h>
-
-/*
- * The exception table consists of pairs of addresses: the first is the
- * address of an instruction that is allowed to fault, and the second is
- * the address at which the program should continue. No registers are
- * modified, so it is entirely up to the continuation code to figure out
- * what to do.
- *
- * All the routines below use bits of fixup code that are out of line
- * with the main instruction path. This means when everything is well,
- * we don't even have to jump over them. Further, they do not intrude
- * on our cache or tlb entries.
- */
-
-struct exception_table_entry {
- unsigned long insn, fixup;
-};
+#include <asm/extable.h>
/*
* These are the main single-value transfer routines. They automatically
@@ -185,7 +169,7 @@ extern long __get_user_bad(void);
live in lib/usercopy.c */
extern unsigned long __copy_user(void __user *to, const void *from, unsigned long n);
-extern unsigned long __copy_user_zeroing(void *to, const void __user *from, unsigned long n);
+extern unsigned long __copy_user_in(void *to, const void __user *from, unsigned long n);
extern unsigned long __do_clear_user(void __user *to, unsigned long n);
static inline long
@@ -252,7 +236,7 @@ __constant_copy_from_user(void *to, const void __user *from, unsigned long n)
else if (n == 24)
__asm_copy_from_user_24(to, from, ret);
else
- ret = __copy_user_zeroing(to, from, n);
+ ret = __copy_user_in(to, from, n);
return ret;
}
@@ -352,62 +336,33 @@ static inline size_t clear_user(void __user *to, size_t n)
return __do_clear_user(to, n);
}
-static inline size_t copy_from_user(void *to, const void __user *from, size_t n)
+static inline unsigned long
+raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
- if (unlikely(!access_ok(VERIFY_READ, from, n))) {
- memset(to, 0, n);
- return n;
- }
if (__builtin_constant_p(n))
return __constant_copy_from_user(to, from, n);
else
- return __copy_user_zeroing(to, from, n);
+ return __copy_user_in(to, from, n);
}
-static inline size_t copy_to_user(void __user *to, const void *from, size_t n)
+static inline unsigned long
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
- if (unlikely(!access_ok(VERIFY_WRITE, to, n)))
- return n;
if (__builtin_constant_p(n))
return __constant_copy_to_user(to, from, n);
else
return __copy_user(to, from, n);
}
-/* We let the __ versions of copy_from/to_user inline, because they're often
- * used in fast paths and have only a small space overhead.
- */
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
static inline unsigned long
-__generic_copy_from_user_nocheck(void *to, const void __user *from,
- unsigned long n)
-{
- return __copy_user_zeroing(to, from, n);
-}
-
-static inline unsigned long
-__generic_copy_to_user_nocheck(void __user *to, const void *from,
- unsigned long n)
-{
- return __copy_user(to, from, n);
-}
-
-static inline unsigned long
-__generic_clear_user_nocheck(void __user *to, unsigned long n)
+__clear_user(void __user *to, unsigned long n)
{
return __do_clear_user(to, n);
}
-/* without checking */
-
-#define __copy_to_user(to, from, n) \
- __generic_copy_to_user_nocheck((to), (from), (n))
-#define __copy_from_user(to, from, n) \
- __generic_copy_from_user_nocheck((to), (from), (n))
-#define __copy_to_user_inatomic __copy_to_user
-#define __copy_from_user_inatomic __copy_from_user
-#define __clear_user(to, n) __generic_clear_user_nocheck((to), (n))
-
#define strlen_user(str) strnlen_user((str), 0x7ffffffe)
#endif /* _CRIS_UACCESS_H */
diff --git a/arch/frv/Kconfig b/arch/frv/Kconfig
index eefd9a4ed156..e489fef111dd 100644
--- a/arch/frv/Kconfig
+++ b/arch/frv/Kconfig
@@ -16,6 +16,7 @@ config FRV
select OLD_SIGACTION
select HAVE_DEBUG_STACKOVERFLOW
select ARCH_NO_COHERENT_DMA_MMAP
+ select ARCH_HAS_RAW_COPY_USER
config ZONE_DMA
bool
diff --git a/arch/frv/include/asm/Kbuild b/arch/frv/include/asm/Kbuild
index c33b46715f65..cce3bc3603ea 100644
--- a/arch/frv/include/asm/Kbuild
+++ b/arch/frv/include/asm/Kbuild
@@ -1,6 +1,7 @@
generic-y += clkdev.h
generic-y += exec.h
+generic-y += extable.h
generic-y += irq_work.h
generic-y += mcs_spinlock.h
generic-y += mm-arch-hooks.h
diff --git a/arch/frv/include/asm/uaccess.h b/arch/frv/include/asm/uaccess.h
index 55b3a69c6c53..e4e33b4cd3ae 100644
--- a/arch/frv/include/asm/uaccess.h
+++ b/arch/frv/include/asm/uaccess.h
@@ -18,6 +18,7 @@
#include <linux/mm.h>
#include <asm/segment.h>
#include <asm/sections.h>
+#include <asm/extable.h>
#define __ptr(x) ((unsigned long __force *)(x))
@@ -59,26 +60,6 @@ static inline int ___range_ok(unsigned long addr, unsigned long size)
#define access_ok(type,addr,size) (__range_ok((void __user *)(addr), (size)) == 0)
#define __access_ok(addr,size) (__range_ok((addr), (size)) == 0)
-/*
- * The exception table consists of pairs of addresses: the first is the
- * address of an instruction that is allowed to fault, and the second is
- * the address at which the program should continue. No registers are
- * modified, so it is entirely up to the continuation code to figure out
- * what to do.
- *
- * All the routines below use bits of fixup code that are out of line
- * with the main instruction path. This means when everything is well,
- * we don't even have to jump over them. Further, they do not intrude
- * on our cache or tlb entries.
- */
-struct exception_table_entry
-{
- unsigned long insn, fixup;
-};
-
-/* Returns 0 if exception not found and fixup otherwise. */
-extern unsigned long search_exception_table(unsigned long);
-
/*
* These are the main single-value transfer routines. They automatically
@@ -252,61 +233,50 @@ do { \
/*
*
*/
+
#define ____force(x) (__force void *)(void __user *)(x)
#ifdef CONFIG_MMU
extern long __memset_user(void *dst, unsigned long count);
extern long __memcpy_user(void *dst, const void *src, unsigned long count);
#define __clear_user(dst,count) __memset_user(____force(dst), (count))
-#define __copy_from_user_inatomic(to, from, n) __memcpy_user((to), ____force(from), (n))
-#define __copy_to_user_inatomic(to, from, n) __memcpy_user(____force(to), (from), (n))
#else
#define __clear_user(dst,count) (memset(____force(dst), 0, (count)), 0)
-#define __copy_from_user_inatomic(to, from, n) (memcpy((to), ____force(from), (n)), 0)
-#define __copy_to_user_inatomic(to, from, n) (memcpy(____force(to), (from), (n)), 0)
#endif
-static inline unsigned long __must_check
-clear_user(void __user *to, unsigned long n)
-{
- if (likely(__access_ok(to, n)))
- n = __clear_user(to, n);
- return n;
-}
-
-static inline unsigned long __must_check
-__copy_to_user(void __user *to, const void *from, unsigned long n)
-{
- might_fault();
- return __copy_to_user_inatomic(to, from, n);
-}
-
static inline unsigned long
-__copy_from_user(void *to, const void __user *from, unsigned long n)
+raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
- might_fault();
- return __copy_from_user_inatomic(to, from, n);
+#ifdef CONFIG_MMU
+ return __memcpy_user(to, (__force const void *)from, n);
+#else
+ memcpy(to, (__force const void *)from, n);
+ return 0;
+#endif
}
-static inline long copy_from_user(void *to, const void __user *from, unsigned long n)
+static inline unsigned long
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
- unsigned long ret = n;
-
- if (likely(__access_ok(from, n)))
- ret = __copy_from_user(to, from, n);
-
- if (unlikely(ret != 0))
- memset(to + (n - ret), 0, ret);
-
- return ret;
+#ifdef CONFIG_MMU
+ return __memcpy_user((__force void *)to, from, n);
+#else
+ memcpy((__force void *)to, from, n);
+ return 0;
+#endif
}
+#define INLINE_COPY_TO_USER
+#define INLINE_COPY_FROM_USER
-static inline long copy_to_user(void __user *to, const void *from, unsigned long n)
+static inline unsigned long __must_check
+clear_user(void __user *to, unsigned long n)
{
- return likely(__access_ok(to, n)) ? __copy_to_user(to, from, n) : n;
+ if (likely(__access_ok(to, n)))
+ n = __clear_user(to, n);
+ return n;
}
extern long strncpy_from_user(char *dst, const char __user *src, long count);
@@ -314,6 +284,4 @@ extern long strnlen_user(const char __user *src, long count);
#define strlen_user(str) strnlen_user(str, 32767)
-extern unsigned long search_exception_table(unsigned long addr);
-
#endif /* _ASM_UACCESS_H */
diff --git a/arch/frv/kernel/traps.c b/arch/frv/kernel/traps.c
index ce29991e4219..fb08ebe0dab4 100644
--- a/arch/frv/kernel/traps.c
+++ b/arch/frv/kernel/traps.c
@@ -360,13 +360,8 @@ asmlinkage void memory_access_exception(unsigned long esr0,
siginfo_t info;
#ifdef CONFIG_MMU
- unsigned long fixup;
-
- fixup = search_exception_table(__frame->pc);
- if (fixup) {
- __frame->pc = fixup;
+ if (fixup_exception(__frame))
return;
- }
#endif
die_if_kernel("-- Memory Access Exception --\n"
diff --git a/arch/frv/mm/extable.c b/arch/frv/mm/extable.c
index a0e8b3e03e4c..9198ddd16092 100644
--- a/arch/frv/mm/extable.c
+++ b/arch/frv/mm/extable.c
@@ -10,40 +10,39 @@ extern const void __memset_end, __memset_user_error_lr, __memset_user_error_hand
extern const void __memcpy_end, __memcpy_user_error_lr, __memcpy_user_error_handler;
extern spinlock_t modlist_lock;
-
-/*****************************************************************************/
-/*
- * see if there's a fixup handler available to deal with a kernel fault
- */
-unsigned long search_exception_table(unsigned long pc)
+int fixup_exception(struct pt_regs *regs)
{
const struct exception_table_entry *extab;
+ unsigned long pc = regs->pc;
/* determine if the fault lay during a memcpy_user or a memset_user */
- if (__frame->lr == (unsigned long) &__memset_user_error_lr &&
+ if (regs->lr == (unsigned long) &__memset_user_error_lr &&
(unsigned long) &memset <= pc && pc < (unsigned long) &__memset_end
) {
/* the fault occurred in a protected memset
* - we search for the return address (in LR) instead of the program counter
* - it was probably during a clear_user()
*/
- return (unsigned long) &__memset_user_error_handler;
+ regs->pc = (unsigned long) &__memset_user_error_handler;
+ return 1;
}
- if (__frame->lr == (unsigned long) &__memcpy_user_error_lr &&
+ if (regs->lr == (unsigned long) &__memcpy_user_error_lr &&
(unsigned long) &memcpy <= pc && pc < (unsigned long) &__memcpy_end
) {
/* the fault occurred in a protected memset
* - we search for the return address (in LR) instead of the program counter
* - it was probably during a copy_to/from_user()
*/
- return (unsigned long) &__memcpy_user_error_handler;
+ regs->pc = (unsigned long) &__memcpy_user_error_handler;
+ return 1;
}
extab = search_exception_tables(pc);
- if (extab)
- return extab->fixup;
+ if (extab) {
+ regs->pc = extab->fixup;
+ return 1;
+ }
return 0;
-
-} /* end search_exception_table() */
+}
diff --git a/arch/frv/mm/fault.c b/arch/frv/mm/fault.c
index 614a46c413d2..179e79e220e5 100644
--- a/arch/frv/mm/fault.c
+++ b/arch/frv/mm/fault.c
@@ -33,7 +33,7 @@ asmlinkage void do_page_fault(int datammu, unsigned long esr0, unsigned long ear
{
struct vm_area_struct *vma;
struct mm_struct *mm;
- unsigned long _pme, lrai, lrad, fixup;
+ unsigned long _pme, lrai, lrad;
unsigned long flags = 0;
siginfo_t info;
pgd_t *pge;
@@ -201,10 +201,8 @@ asmlinkage void do_page_fault(int datammu, unsigned long esr0, unsigned long ear
no_context:
/* are we prepared to handle this kernel fault? */
- if ((fixup = search_exception_table(__frame->pc)) != 0) {
- __frame->pc = fixup;
+ if (fixup_exception(__frame))
return;
- }
/*
* Oops. The kernel tried to access some bad page. We'll have to
diff --git a/arch/h8300/Kconfig b/arch/h8300/Kconfig
index 3ae852507e57..473883417004 100644
--- a/arch/h8300/Kconfig
+++ b/arch/h8300/Kconfig
@@ -22,6 +22,7 @@ config H8300
select HAVE_ARCH_KGDB
select HAVE_ARCH_HASH
select CPU_NO_EFFICIENT_FFS
+ select ARCH_HAS_RAW_COPY_USER
config RWSEM_GENERIC_SPINLOCK
def_bool y
diff --git a/arch/h8300/include/asm/uaccess.h b/arch/h8300/include/asm/uaccess.h
index c156a416a279..6f6144a240ce 100644
--- a/arch/h8300/include/asm/uaccess.h
+++ b/arch/h8300/include/asm/uaccess.h
@@ -3,8 +3,8 @@
#include <linux/string.h>
-static inline __must_check long __copy_from_user(void *to,
- const void __user * from, unsigned long n)
+static inline __must_check unsigned long
+raw_copy_from_user(void *to, const void __user * from, unsigned long n)
{
if (__builtin_constant_p(n)) {
switch(n) {
@@ -24,8 +24,8 @@ static inline __must_check long __copy_from_user(void *to,
return 0;
}
-static inline __must_check long __copy_to_user(void __user *to,
- const void *from, unsigned long n)
+static inline __must_check unsigned long
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
if (__builtin_constant_p(n)) {
switch(n) {
@@ -46,6 +46,8 @@ static inline __must_check long __copy_to_user(void __user *to,
memcpy((void __force *)to, from, n);
return 0;
}
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
#include <asm-generic/uaccess.h>
diff --git a/arch/hexagon/Kconfig b/arch/hexagon/Kconfig
index 1941e4baaee6..0c536a83ea71 100644
--- a/arch/hexagon/Kconfig
+++ b/arch/hexagon/Kconfig
@@ -26,6 +26,7 @@ config HEXAGON
select GENERIC_CLOCKEVENTS_BROADCAST
select MODULES_USE_ELF_RELA
select GENERIC_CPU_DEVICES
+ select ARCH_HAS_RAW_COPY_USER
---help---
Qualcomm Hexagon is a processor architecture designed for high
performance and low power across a wide variety of applications.
diff --git a/arch/hexagon/include/asm/uaccess.h b/arch/hexagon/include/asm/uaccess.h
index 3a7f818e5ef7..458b69886b34 100644
--- a/arch/hexagon/include/asm/uaccess.h
+++ b/arch/hexagon/include/asm/uaccess.h
@@ -65,19 +65,12 @@
*/
/* Assembly somewhat optimized copy routines */
-unsigned long __copy_from_user_hexagon(void *to, const void __user *from,
+unsigned long raw_copy_from_user(void *to, const void __user *from,
unsigned long n);
-unsigned long __copy_to_user_hexagon(void __user *to, const void *from,
+unsigned long raw_copy_to_user(void __user *to, const void *from,
unsigned long n);
-
-#define __copy_from_user(to, from, n) __copy_from_user_hexagon(to, from, n)
-#define __copy_to_user(to, from, n) __copy_to_user_hexagon(to, from, n)
-
-/*
- * XXX todo: some additonal performance gain is possible by
- * implementing __copy_to/from_user_inatomic, which is much
- * like __copy_to/from_user, but performs slightly less checking.
- */
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
__kernel_size_t __clear_user_hexagon(void __user *dest, unsigned long count);
#define __clear_user(a, s) __clear_user_hexagon((a), (s))
@@ -104,10 +97,14 @@ static inline long hexagon_strncpy_from_user(char *dst, const char __user *src,
return -EFAULT;
if (res > n) {
- copy_from_user(dst, src, n);
+ long left = raw_copy_from_user(dst, src, n);
+ if (unlikely(left))
+ memset(dst + (n - left), 0, left);
return n;
} else {
- copy_from_user(dst, src, res);
+ long left = raw_copy_from_user(dst, src, res);
+ if (unlikely(left))
+ memset(dst + (res - left), 0, left);
return res-1;
}
}
diff --git a/arch/hexagon/kernel/hexagon_ksyms.c b/arch/hexagon/kernel/hexagon_ksyms.c
index af9dec4c28eb..00bcad9cbd8f 100644
--- a/arch/hexagon/kernel/hexagon_ksyms.c
+++ b/arch/hexagon/kernel/hexagon_ksyms.c
@@ -25,8 +25,8 @@
/* Additional functions */
EXPORT_SYMBOL(__clear_user_hexagon);
-EXPORT_SYMBOL(__copy_from_user_hexagon);
-EXPORT_SYMBOL(__copy_to_user_hexagon);
+EXPORT_SYMBOL(raw_copy_from_user);
+EXPORT_SYMBOL(raw_copy_to_user);
EXPORT_SYMBOL(__iounmap);
EXPORT_SYMBOL(__strnlen_user);
EXPORT_SYMBOL(__vmgetie);
diff --git a/arch/hexagon/mm/copy_from_user.S b/arch/hexagon/mm/copy_from_user.S
index 7fc94f3e6642..7da066fbd16f 100644
--- a/arch/hexagon/mm/copy_from_user.S
+++ b/arch/hexagon/mm/copy_from_user.S
@@ -44,7 +44,7 @@
#define bytes r2
#define loopcount r5
-#define FUNCNAME __copy_from_user_hexagon
+#define FUNCNAME raw_copy_from_user
#include "copy_user_template.S"
/* LOAD FAULTS from COPY_FROM_USER */
diff --git a/arch/hexagon/mm/copy_to_user.S b/arch/hexagon/mm/copy_to_user.S
index 0cfbcc09d1d9..a7b7f8db21df 100644
--- a/arch/hexagon/mm/copy_to_user.S
+++ b/arch/hexagon/mm/copy_to_user.S
@@ -43,7 +43,7 @@
#define bytes r2
#define loopcount r5
-#define FUNCNAME __copy_to_user_hexagon
+#define FUNCNAME raw_copy_to_user
#include "copy_user_template.S"
/* STORE FAULTS from COPY_TO_USER */
diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig
index 18ca6a9ce566..0ed0e44856b2 100644
--- a/arch/ia64/Kconfig
+++ b/arch/ia64/Kconfig
@@ -53,6 +53,7 @@ config IA64
select ARCH_USE_CMPXCHG_LOCKREF
select HAVE_ARCH_AUDITSYSCALL
select HAVE_ARCH_HARDENED_USERCOPY
+ select ARCH_HAS_RAW_COPY_USER
default y
help
The Itanium Processor Family is Intel's 64-bit successor to
diff --git a/arch/ia64/include/asm/extable.h b/arch/ia64/include/asm/extable.h
new file mode 100644
index 000000000000..20376e71eab4
--- /dev/null
+++ b/arch/ia64/include/asm/extable.h
@@ -0,0 +1,11 @@
+#ifndef _ASM_IA64_EXTABLE_H
+#define _ASM_IA64_EXTABLE_H
+
+#define ARCH_HAS_RELATIVE_EXTABLE
+
+struct exception_table_entry {
+ int insn; /* location-relative address of insn this fixup is for */
+ int fixup; /* location-relative continuation addr.; if bit 2 is set, r9 is set to 0 */
+};
+
+#endif
diff --git a/arch/ia64/include/asm/uaccess.h b/arch/ia64/include/asm/uaccess.h
index d471d1a1afd0..82a7646c4416 100644
--- a/arch/ia64/include/asm/uaccess.h
+++ b/arch/ia64/include/asm/uaccess.h
@@ -39,6 +39,7 @@
#include <asm/intrinsics.h>
#include <asm/pgtable.h>
#include <asm/io.h>
+#include <asm/extable.h>
/*
* For historical reasons, the following macros are grossly misnamed:
@@ -58,14 +59,14 @@
* address TASK_SIZE is never valid. We also need to make sure that the address doesn't
* point inside the virtually mapped linear page table.
*/
-#define __access_ok(addr, size, segment) \
-({ \
- __chk_user_ptr(addr); \
- (likely((unsigned long) (addr) <= (segment).seg) \
- && ((segment).seg == KERNEL_DS.seg \
- || likely(REGION_OFFSET((unsigned long) (addr)) < RGN_MAP_LIMIT))); \
-})
-#define access_ok(type, addr, size) __access_ok((addr), (size), get_fs())
+static inline int __access_ok(const void __user *p, unsigned long size)
+{
+ unsigned long addr = (unsigned long)p;
+ unsigned long seg = get_fs().seg;
+ return likely(addr <= seg) &&
+ (seg == KERNEL_DS.seg || likely(REGION_OFFSET(addr) < RGN_MAP_LIMIT));
+}
+#define access_ok(type, addr, size) __access_ok((addr), (size))
/*
* These are the main single-value transfer routines. They automatically
@@ -75,8 +76,8 @@
* (a) re-use the arguments for side effects (sizeof/typeof is ok)
* (b) require any knowledge of processes at this stage
*/
-#define put_user(x, ptr) __put_user_check((__typeof__(*(ptr))) (x), (ptr), sizeof(*(ptr)), get_fs())
-#define get_user(x, ptr) __get_user_check((x), (ptr), sizeof(*(ptr)), get_fs())
+#define put_user(x, ptr) __put_user_check((__typeof__(*(ptr))) (x), (ptr), sizeof(*(ptr)))
+#define get_user(x, ptr) __get_user_check((x), (ptr), sizeof(*(ptr)))
/*
* The "__xxx" versions do not do address space checking, useful when
@@ -179,13 +180,13 @@ extern void __get_user_unknown (void);
* could clobber r8 and r9 (among others). Thus, be careful not to evaluate it while
* using r8/r9.
*/
-#define __do_get_user(check, x, ptr, size, segment) \
+#define __do_get_user(check, x, ptr, size) \
({ \
const __typeof__(*(ptr)) __user *__gu_ptr = (ptr); \
__typeof__ (size) __gu_size = (size); \
long __gu_err = -EFAULT; \
unsigned long __gu_val = 0; \
- if (!check || __access_ok(__gu_ptr, size, segment)) \
+ if (!check || __access_ok(__gu_ptr, size)) \
switch (__gu_size) { \
case 1: __get_user_size(__gu_val, __gu_ptr, 1, __gu_err); break; \
case 2: __get_user_size(__gu_val, __gu_ptr, 2, __gu_err); break; \
@@ -197,8 +198,8 @@ extern void __get_user_unknown (void);
__gu_err; \
})
-#define __get_user_nocheck(x, ptr, size) __do_get_user(0, x, ptr, size, KERNEL_DS)
-#define __get_user_check(x, ptr, size, segment) __do_get_user(1, x, ptr, size, segment)
+#define __get_user_nocheck(x, ptr, size) __do_get_user(0, x, ptr, size)
+#define __get_user_check(x, ptr, size) __do_get_user(1, x, ptr, size)
extern void __put_user_unknown (void);
@@ -206,14 +207,14 @@ extern void __put_user_unknown (void);
* Evaluating arguments X, PTR, SIZE, and SEGMENT may involve subroutine-calls, which
* could clobber r8 (among others). Thus, be careful not to evaluate them while using r8.
*/
-#define __do_put_user(check, x, ptr, size, segment) \
+#define __do_put_user(check, x, ptr, size) \
({ \
__typeof__ (x) __pu_x = (x); \
__typeof__ (*(ptr)) __user *__pu_ptr = (ptr); \
__typeof__ (size) __pu_size = (size); \
long __pu_err = -EFAULT; \
\
- if (!check || __access_ok(__pu_ptr, __pu_size, segment)) \
+ if (!check || __access_ok(__pu_ptr, __pu_size)) \
switch (__pu_size) { \
case 1: __put_user_size(__pu_x, __pu_ptr, 1, __pu_err); break; \
case 2: __put_user_size(__pu_x, __pu_ptr, 2, __pu_err); break; \
@@ -224,8 +225,8 @@ extern void __put_user_unknown (void);
__pu_err; \
})
-#define __put_user_nocheck(x, ptr, size) __do_put_user(0, x, ptr, size, KERNEL_DS)
-#define __put_user_check(x, ptr, size, segment) __do_put_user(1, x, ptr, size, segment)
+#define __put_user_nocheck(x, ptr, size) __do_put_user(0, x, ptr, size)
+#define __put_user_check(x, ptr, size) __do_put_user(1, x, ptr, size)
/*
* Complex access routines
@@ -234,56 +235,19 @@ extern unsigned long __must_check __copy_user (void __user *to, const void __use
unsigned long count);
static inline unsigned long
-__copy_to_user (void __user *to, const void *from, unsigned long count)
+raw_copy_to_user(void __user *to, const void *from, unsigned long count)
{
- check_object_size(from, count, true);
-
return __copy_user(to, (__force void __user *) from, count);
}
static inline unsigned long
-__copy_from_user (void *to, const void __user *from, unsigned long count)
+raw_copy_from_user(void *to, const void __user *from, unsigned long count)
{
- check_object_size(to, count, false);
-
return __copy_user((__force void __user *) to, from, count);
}
-#define __copy_to_user_inatomic __copy_to_user
-#define __copy_from_user_inatomic __copy_from_user
-#define copy_to_user(to, from, n) \
-({ \
- void __user *__cu_to = (to); \
- const void *__cu_from = (from); \
- long __cu_len = (n); \
- \
- if (__access_ok(__cu_to, __cu_len, get_fs())) { \
- check_object_size(__cu_from, __cu_len, true); \
- __cu_len = __copy_user(__cu_to, (__force void __user *) __cu_from, __cu_len); \
- } \
- __cu_len; \
-})
-
-static inline unsigned long
-copy_from_user(void *to, const void __user *from, unsigned long n)
-{
- check_object_size(to, n, false);
- if (likely(__access_ok(from, n, get_fs())))
- n = __copy_user((__force void __user *) to, from, n);
- else
- memset(to, 0, n);
- return n;
-}
-
-#define __copy_in_user(to, from, size) __copy_user((to), (from), (size))
-
-static inline unsigned long
-copy_in_user (void __user *to, const void __user *from, unsigned long n)
-{
- if (likely(access_ok(VERIFY_READ, from, n) && access_ok(VERIFY_WRITE, to, n)))
- n = __copy_user(to, from, n);
- return n;
-}
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
extern unsigned long __do_clear_user (void __user *, unsigned long);
@@ -292,7 +256,7 @@ extern unsigned long __do_clear_user (void __user *, unsigned long);
#define clear_user(to, n) \
({ \
unsigned long __cu_len = (n); \
- if (__access_ok(to, __cu_len, get_fs())) \
+ if (__access_ok(to, __cu_len)) \
__cu_len = __do_clear_user(to, __cu_len); \
__cu_len; \
})
@@ -308,7 +272,7 @@ extern long __must_check __strncpy_from_user (char *to, const char __user *from,
({ \
const char __user * __sfu_from = (from); \
long __sfu_ret = -EFAULT; \
- if (__access_ok(__sfu_from, 0, get_fs())) \
+ if (__access_ok(__sfu_from, 0)) \
__sfu_ret = __strncpy_from_user((to), __sfu_from, (n)); \
__sfu_ret; \
})
@@ -320,7 +284,7 @@ extern unsigned long __strlen_user (const char __user *);
({ \
const char __user *__su_str = (str); \
unsigned long __su_ret = 0; \
- if (__access_ok(__su_str, 0, get_fs())) \
+ if (__access_ok(__su_str, 0)) \
__su_ret = __strlen_user(__su_str); \
__su_ret; \
})
@@ -336,18 +300,11 @@ extern unsigned long __strnlen_user (const char __user *, long);
({ \
const char __user *__su_str = (str); \
unsigned long __su_ret = 0; \
- if (__access_ok(__su_str, 0, get_fs())) \
+ if (__access_ok(__su_str, 0)) \
__su_ret = __strnlen_user(__su_str, len); \
__su_ret; \
})
-#define ARCH_HAS_RELATIVE_EXTABLE
-
-struct exception_table_entry {
- int insn; /* location-relative address of insn this fixup is for */
- int fixup; /* location-relative continuation addr.; if bit 2 is set, r9 is set to 0 */
-};
-
#define ARCH_HAS_TRANSLATE_MEM_PTR 1
static __inline__ void *
xlate_dev_mem_ptr(phys_addr_t p)
diff --git a/arch/ia64/lib/memcpy_mck.S b/arch/ia64/lib/memcpy_mck.S
index b264b6a7967b..bbbadc478f5b 100644
--- a/arch/ia64/lib/memcpy_mck.S
+++ b/arch/ia64/lib/memcpy_mck.S
@@ -556,9 +556,6 @@ EK(.ex_handler, (p17) st8 [dst1]=r39,8); \
#define D r22
#define F r28
-#define memset_arg0 r32
-#define memset_arg2 r33
-
#define saved_retval loc0
#define saved_rtlink loc1
#define saved_pfs_stack loc2
@@ -622,7 +619,7 @@ EK(.ex_handler, (p17) st8 [dst1]=r39,8); \
* (faulting_addr - orig_dst) -> len to faulting st address
* B = (cur_dst - orig_dst) -> len copied so far
* C = A - B -> len need to be copied
- * D = orig_len - A -> len need to be zeroed
+ * D = orig_len - A -> len need to be left along
*/
(p6) sub A = F, saved_in0
(p7) sub A = F, saved_in1
@@ -638,9 +635,6 @@ EK(.ex_handler, (p17) st8 [dst1]=r39,8); \
sub D = saved_in2, A
;;
cmp.gt p8,p0=C,r0 // more than 1 byte?
- add memset_arg0=saved_in0, A
-(p6) mov memset_arg2=0 // copy_to_user should not call memset
-(p7) mov memset_arg2=D // copy_from_user need to have kbuf zeroed
mov r8=0
mov saved_retval = D
mov saved_rtlink = b0
@@ -652,11 +646,6 @@ EK(.ex_handler, (p17) st8 [dst1]=r39,8); \
;;
add saved_retval=saved_retval,r8 // above might return non-zero value
- cmp.gt p8,p0=memset_arg2,r0 // more than 1 byte?
- mov out0=memset_arg0 // *s
- mov out1=r0 // c
- mov out2=memset_arg2 // n
-(p8) br.call.sptk.few b0=memset
;;
mov retval=saved_retval
diff --git a/arch/ia64/mm/extable.c b/arch/ia64/mm/extable.c
index 4edb816aba9a..10dd4a66e167 100644
--- a/arch/ia64/mm/extable.c
+++ b/arch/ia64/mm/extable.c
@@ -5,7 +5,10 @@
* David Mosberger-Tang <davidm@hpl.hp.com>
*/
-#include <linux/uaccess.h>
+#include <asm/ptrace.h>
+#include <asm/extable.h>
+#include <asm/errno.h>
+#include <asm/processor.h>
void
ia64_handle_exception (struct pt_regs *regs, const struct exception_table_entry *e)
diff --git a/arch/m32r/Kconfig b/arch/m32r/Kconfig
index 95474460b367..b3e82bdd6db0 100644
--- a/arch/m32r/Kconfig
+++ b/arch/m32r/Kconfig
@@ -19,6 +19,7 @@ config M32R
select HAVE_DEBUG_STACKOVERFLOW
select CPU_NO_EFFICIENT_FFS
select DMA_NOOP_OPS
+ select ARCH_HAS_RAW_COPY_USER
config SBUS
bool
diff --git a/arch/m32r/include/asm/Kbuild b/arch/m32r/include/asm/Kbuild
index deb298777df2..c000ffac8586 100644
--- a/arch/m32r/include/asm/Kbuild
+++ b/arch/m32r/include/asm/Kbuild
@@ -2,6 +2,7 @@
generic-y += clkdev.h
generic-y += current.h
generic-y += exec.h
+generic-y += extable.h
generic-y += irq_work.h
generic-y += kvm_para.h
generic-y += mcs_spinlock.h
diff --git a/arch/m32r/include/asm/uaccess.h b/arch/m32r/include/asm/uaccess.h
index 96b0efdb5f22..07be349c00ad 100644
--- a/arch/m32r/include/asm/uaccess.h
+++ b/arch/m32r/include/asm/uaccess.h
@@ -13,6 +13,7 @@
*/
#include <asm/page.h>
#include <asm/setup.h>
+#include <linux/prefetch.h>
/*
* The fs value determines whether argument validity checking should be
@@ -109,25 +110,7 @@ static inline int access_ok(int type, const void *addr, unsigned long size)
}
#endif /* CONFIG_MMU */
-/*
- * The exception table consists of pairs of addresses: the first is the
- * address of an instruction that is allowed to fault, and the second is
- * the address at which the program should continue. No registers are
- * modified, so it is entirely up to the continuation code to figure out
- * what to do.
- *
- * All the routines below use bits of fixup code that are out of line
- * with the main instruction path. This means when everything is well,
- * we don't even have to jump over them. Further, they do not intrude
- * on our cache or tlb entries.
- */
-
-struct exception_table_entry
-{
- unsigned long insn, fixup;
-};
-
-extern int fixup_exception(struct pt_regs *regs);
+#include <asm/extable.h>
/*
* These are the main single-value transfer routines. They automatically
@@ -478,174 +461,25 @@ do { \
: "r14", "memory"); \
} while (0)
-#define __copy_user_zeroing(to, from, size) \
-do { \
- unsigned long __dst, __src, __c; \
- __asm__ __volatile__ ( \
- " mv r14, %0\n" \
- " or r14, %1\n" \
- " beq %0, %1, 9f\n" \
- " beqz %2, 9f\n" \
- " and3 r14, r14, #3\n" \
- " bnez r14, 2f\n" \
- " and3 %2, %2, #3\n" \
- " beqz %3, 2f\n" \
- " addi %0, #-4 ; word_copy \n" \
- " .fillinsn\n" \
- "0: ld r14, @%1+\n" \
- " addi %3, #-1\n" \
- " .fillinsn\n" \
- "1: st r14, @+%0\n" \
- " bnez %3, 0b\n" \
- " beqz %2, 9f\n" \
- " addi %0, #4\n" \
- " .fillinsn\n" \
- "2: ldb r14, @%1 ; byte_copy \n" \
- " .fillinsn\n" \
- "3: stb r14, @%0\n" \
- " addi %1, #1\n" \
- " addi %2, #-1\n" \
- " addi %0, #1\n" \
- " bnez %2, 2b\n" \
- " .fillinsn\n" \
- "9:\n" \
- ".section .fixup,\"ax\"\n" \
- " .balign 4\n" \
- "5: addi %3, #1\n" \
- " addi %1, #-4\n" \
- " .fillinsn\n" \
- "6: slli %3, #2\n" \
- " add %2, %3\n" \
- " addi %0, #4\n" \
- " .fillinsn\n" \
- "7: ldi r14, #0 ; store zero \n" \
- " .fillinsn\n" \
- "8: addi %2, #-1\n" \
- " stb r14, @%0 ; ACE? \n" \
- " addi %0, #1\n" \
- " bnez %2, 8b\n" \
- " seth r14, #high(9b)\n" \
- " or3 r14, r14, #low(9b)\n" \
- " jmp r14\n" \
- ".previous\n" \
- ".section __ex_table,\"a\"\n" \
- " .balign 4\n" \
- " .long 0b,6b\n" \
- " .long 1b,5b\n" \
- " .long 2b,7b\n" \
- " .long 3b,7b\n" \
- ".previous\n" \
- : "=&r" (__dst), "=&r" (__src), "=&r" (size), \
- "=&r" (__c) \
- : "0" (to), "1" (from), "2" (size), "3" (size / 4) \
- : "r14", "memory"); \
-} while (0)
-
-
/* We let the __ versions of copy_from/to_user inline, because they're often
* used in fast paths and have only a small space overhead.
*/
-static inline unsigned long __generic_copy_from_user_nocheck(void *to,
- const void __user *from, unsigned long n)
+static inline unsigned long
+raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
- __copy_user_zeroing(to, from, n);
+ prefetchw(to);
+ __copy_user(to, from, n);
return n;
}
-static inline unsigned long __generic_copy_to_user_nocheck(void __user *to,
- const void *from, unsigned long n)
+static inline unsigned long
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
+ prefetch(from);
__copy_user(to, from, n);
return n;
}
-unsigned long __generic_copy_to_user(void __user *, const void *, unsigned long);
-unsigned long __generic_copy_from_user(void *, const void __user *, unsigned long);
-
-/**
- * __copy_to_user: - Copy a block of data into user space, with less checking.
- * @to: Destination address, in user space.
- * @from: Source address, in kernel space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from kernel space to user space. Caller must check
- * the specified block with access_ok() before calling this function.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- */
-#define __copy_to_user(to, from, n) \
- __generic_copy_to_user_nocheck((to), (from), (n))
-
-#define __copy_to_user_inatomic __copy_to_user
-#define __copy_from_user_inatomic __copy_from_user
-
-/**
- * copy_to_user: - Copy a block of data into user space.
- * @to: Destination address, in user space.
- * @from: Source address, in kernel space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from kernel space to user space.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- */
-#define copy_to_user(to, from, n) \
-({ \
- might_fault(); \
- __generic_copy_to_user((to), (from), (n)); \
-})
-
-/**
- * __copy_from_user: - Copy a block of data from user space, with less checking. * @to: Destination address, in kernel space.
- * @from: Source address, in user space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from user space to kernel space. Caller must check
- * the specified block with access_ok() before calling this function.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- *
- * If some data could not be copied, this function will pad the copied
- * data to the requested size using zero bytes.
- */
-#define __copy_from_user(to, from, n) \
- __generic_copy_from_user_nocheck((to), (from), (n))
-
-/**
- * copy_from_user: - Copy a block of data from user space.
- * @to: Destination address, in kernel space.
- * @from: Source address, in user space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from user space to kernel space.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- *
- * If some data could not be copied, this function will pad the copied
- * data to the requested size using zero bytes.
- */
-#define copy_from_user(to, from, n) \
-({ \
- might_fault(); \
- __generic_copy_from_user((to), (from), (n)); \
-})
-
long __must_check strncpy_from_user(char *dst, const char __user *src,
long count);
long __must_check __strncpy_from_user(char *dst,
diff --git a/arch/m32r/kernel/m32r_ksyms.c b/arch/m32r/kernel/m32r_ksyms.c
index d763f0bd2106..a4d43b5cc102 100644
--- a/arch/m32r/kernel/m32r_ksyms.c
+++ b/arch/m32r/kernel/m32r_ksyms.c
@@ -26,8 +26,6 @@ EXPORT_SYMBOL(strncpy_from_user);
EXPORT_SYMBOL(__strncpy_from_user);
EXPORT_SYMBOL(clear_user);
EXPORT_SYMBOL(__clear_user);
-EXPORT_SYMBOL(__generic_copy_from_user);
-EXPORT_SYMBOL(__generic_copy_to_user);
EXPORT_SYMBOL(strnlen_user);
#ifdef CONFIG_SMP
diff --git a/arch/m32r/lib/usercopy.c b/arch/m32r/lib/usercopy.c
index fd03f2731f20..b3ef2c899f96 100644
--- a/arch/m32r/lib/usercopy.c
+++ b/arch/m32r/lib/usercopy.c
@@ -11,27 +11,6 @@
#include <linux/thread_info.h>
#include <linux/uaccess.h>
-unsigned long
-__generic_copy_to_user(void __user *to, const void *from, unsigned long n)
-{
- prefetch(from);
- if (access_ok(VERIFY_WRITE, to, n))
- __copy_user(to,from,n);
- return n;
-}
-
-unsigned long
-__generic_copy_from_user(void *to, const void __user *from, unsigned long n)
-{
- prefetchw(to);
- if (access_ok(VERIFY_READ, from, n))
- __copy_user_zeroing(to,from,n);
- else
- memset(to, 0, n);
- return n;
-}
-
-
/*
* Copy a null terminated string from userspace.
*/
diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig
index d140206d5d29..7d345758ea16 100644
--- a/arch/m68k/Kconfig
+++ b/arch/m68k/Kconfig
@@ -22,6 +22,7 @@ config M68K
select MODULES_USE_ELF_RELA
select OLD_SIGSUSPEND3
select OLD_SIGACTION
+ select ARCH_HAS_RAW_COPY_USER
config RWSEM_GENERIC_SPINLOCK
bool
diff --git a/arch/m68k/include/asm/Kbuild b/arch/m68k/include/asm/Kbuild
index d4f9ccbfa85c..82005d2ff717 100644
--- a/arch/m68k/include/asm/Kbuild
+++ b/arch/m68k/include/asm/Kbuild
@@ -5,6 +5,7 @@ generic-y += device.h
generic-y += emergency-restart.h
generic-y += errno.h
generic-y += exec.h
+generic-y += extable.h
generic-y += futex.h
generic-y += hw_irq.h
generic-y += ioctl.h
diff --git a/arch/m68k/include/asm/processor.h b/arch/m68k/include/asm/processor.h
index f5f790c31bf8..77239e81379b 100644
--- a/arch/m68k/include/asm/processor.h
+++ b/arch/m68k/include/asm/processor.h
@@ -122,16 +122,6 @@ static inline void start_thread(struct pt_regs * regs, unsigned long pc,
wrusp(usp);
}
-#ifdef CONFIG_MMU
-extern int handle_kernel_fault(struct pt_regs *regs);
-#else
-static inline int handle_kernel_fault(struct pt_regs *regs)
-{
- /* Any fault in kernel is fatal on non-mmu */
- return 0;
-}
-#endif
-
/* Forward declaration, a strange C thing */
struct task_struct;
diff --git a/arch/m68k/include/asm/uaccess.h b/arch/m68k/include/asm/uaccess.h
index 3fadc4a93d97..67b3481d6020 100644
--- a/arch/m68k/include/asm/uaccess.h
+++ b/arch/m68k/include/asm/uaccess.h
@@ -4,6 +4,7 @@
#include <asm/uaccess_mm.h>
#endif
+#include <asm/extable.h>
#ifdef CONFIG_CPU_HAS_NO_UNALIGNED
#include <asm-generic/uaccess-unaligned.h>
#else
diff --git a/arch/m68k/include/asm/uaccess_mm.h b/arch/m68k/include/asm/uaccess_mm.h
index 14054a4e4216..ef856ffeffdf 100644
--- a/arch/m68k/include/asm/uaccess_mm.h
+++ b/arch/m68k/include/asm/uaccess_mm.h
@@ -31,24 +31,6 @@ static inline int access_ok(int type, const void __user *addr,
#define MOVES "move"
#endif
-/*
- * The exception table consists of pairs of addresses: the first is the
- * address of an instruction that is allowed to fault, and the second is
- * the address at which the program should continue. No registers are
- * modified, so it is entirely up to the continuation code to figure out
- * what to do.
- *
- * All the routines below use bits of fixup code that are out of line
- * with the main instruction path. This means when everything is well,
- * we don't even have to jump over them. Further, they do not intrude
- * on our cache or tlb entries.
- */
-
-struct exception_table_entry
-{
- unsigned long insn, fixup;
-};
-
extern int __put_user_bad(void);
extern int __get_user_bad(void);
@@ -197,39 +179,55 @@ asm volatile ("\n" \
unsigned long __generic_copy_from_user(void *to, const void __user *from, unsigned long n);
unsigned long __generic_copy_to_user(void __user *to, const void *from, unsigned long n);
-#define __constant_copy_from_user_asm(res, to, from, tmp, n, s1, s2, s3)\
+#define __suffix0
+#define __suffix1 b
+#define __suffix2 w
+#define __suffix4 l
+
+#define ____constant_copy_from_user_asm(res, to, from, tmp, n1, n2, n3, s1, s2, s3)\
asm volatile ("\n" \
"1: "MOVES"."#s1" (%2)+,%3\n" \
" move."#s1" %3,(%1)+\n" \
+ " .ifnc \""#s2"\",\"\"\n" \
"2: "MOVES"."#s2" (%2)+,%3\n" \
" move."#s2" %3,(%1)+\n" \
" .ifnc \""#s3"\",\"\"\n" \
"3: "MOVES"."#s3" (%2)+,%3\n" \
" move."#s3" %3,(%1)+\n" \
" .endif\n" \
+ " .endif\n" \
"4:\n" \
" .section __ex_table,\"a\"\n" \
" .align 4\n" \
" .long 1b,10f\n" \
+ " .ifnc \""#s2"\",\"\"\n" \
" .long 2b,20f\n" \
" .ifnc \""#s3"\",\"\"\n" \
" .long 3b,30f\n" \
" .endif\n" \
+ " .endif\n" \
" .previous\n" \
"\n" \
" .section .fixup,\"ax\"\n" \
" .even\n" \
- "10: clr."#s1" (%1)+\n" \
- "20: clr."#s2" (%1)+\n" \
+ "10: addq.l #"#n1",%0\n" \
+ " .ifnc \""#s2"\",\"\"\n" \
+ "20: addq.l #"#n2",%0\n" \
" .ifnc \""#s3"\",\"\"\n" \
- "30: clr."#s3" (%1)+\n" \
+ "30: addq.l #"#n3",%0\n" \
+ " .endif\n" \
" .endif\n" \
- " moveq.l #"#n",%0\n" \
" jra 4b\n" \
" .previous\n" \
: "+d" (res), "+&a" (to), "+a" (from), "=&d" (tmp) \
: : "memory")
+#define ___constant_copy_from_user_asm(res, to, from, tmp, n1, n2, n3, s1, s2, s3)\
+ ____constant_copy_from_user_asm(res, to, from, tmp, n1, n2, n3, s1, s2, s3)
+#define __constant_copy_from_user_asm(res, to, from, tmp, n1, n2, n3) \
+ ___constant_copy_from_user_asm(res, to, from, tmp, n1, n2, n3, \
+ __suffix##n1, __suffix##n2, __suffix##n3)
+
static __always_inline unsigned long
__constant_copy_from_user(void *to, const void __user *from, unsigned long n)
{
@@ -237,37 +235,37 @@ __constant_copy_from_user(void *to, const void __user *from, unsigned long n)
switch (n) {
case 1:
- __get_user_asm(res, *(u8 *)to, (u8 __user *)from, u8, b, d, 1);
+ __constant_copy_from_user_asm(res, to, from, tmp, 1, 0, 0);
break;
case 2:
- __get_user_asm(res, *(u16 *)to, (u16 __user *)from, u16, w, r, 2);
+ __constant_copy_from_user_asm(res, to, from, tmp, 2, 0, 0);
break;
case 3:
- __constant_copy_from_user_asm(res, to, from, tmp, 3, w, b,);
+ __constant_copy_from_user_asm(res, to, from, tmp, 2, 1, 0);
break;
case 4:
- __get_user_asm(res, *(u32 *)to, (u32 __user *)from, u32, l, r, 4);
+ __constant_copy_from_user_asm(res, to, from, tmp, 4, 0, 0);
break;
case 5:
- __constant_copy_from_user_asm(res, to, from, tmp, 5, l, b,);
+ __constant_copy_from_user_asm(res, to, from, tmp, 4, 1, 0);
break;
case 6:
- __constant_copy_from_user_asm(res, to, from, tmp, 6, l, w,);
+ __constant_copy_from_user_asm(res, to, from, tmp, 4, 2, 0);
break;
case 7:
- __constant_copy_from_user_asm(res, to, from, tmp, 7, l, w, b);
+ __constant_copy_from_user_asm(res, to, from, tmp, 4, 2, 1);
break;
case 8:
- __constant_copy_from_user_asm(res, to, from, tmp, 8, l, l,);
+ __constant_copy_from_user_asm(res, to, from, tmp, 4, 4, 0);
break;
case 9:
- __constant_copy_from_user_asm(res, to, from, tmp, 9, l, l, b);
+ __constant_copy_from_user_asm(res, to, from, tmp, 4, 4, 1);
break;
case 10:
- __constant_copy_from_user_asm(res, to, from, tmp, 10, l, l, w);
+ __constant_copy_from_user_asm(res, to, from, tmp, 4, 4, 2);
break;
case 12:
- __constant_copy_from_user_asm(res, to, from, tmp, 12, l, l, l);
+ __constant_copy_from_user_asm(res, to, from, tmp, 4, 4, 4);
break;
default:
/* we limit the inlined version to 3 moves */
@@ -358,21 +356,23 @@ __constant_copy_to_user(void __user *to, const void *from, unsigned long n)
return res;
}
-#define __copy_from_user(to, from, n) \
-(__builtin_constant_p(n) ? \
- __constant_copy_from_user(to, from, n) : \
- __generic_copy_from_user(to, from, n))
-
-#define __copy_to_user(to, from, n) \
-(__builtin_constant_p(n) ? \
- __constant_copy_to_user(to, from, n) : \
- __generic_copy_to_user(to, from, n))
-
-#define __copy_to_user_inatomic __copy_to_user
-#define __copy_from_user_inatomic __copy_from_user
+static inline unsigned long
+raw_copy_from_user(void *to, const void __user *from, unsigned long n)
+{
+ if (__builtin_constant_p(n))
+ return __constant_copy_from_user(to, from, n);
+ return __generic_copy_from_user(to, from, n);
+}
-#define copy_from_user(to, from, n) __copy_from_user(to, from, n)
-#define copy_to_user(to, from, n) __copy_to_user(to, from, n)
+static inline unsigned long
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
+{
+ if (__builtin_constant_p(n))
+ return __constant_copy_to_user(to, from, n);
+ return __generic_copy_to_user(to, from, n);
+}
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
#define user_addr_max() \
(uaccess_kernel() ? ~0UL : TASK_SIZE)
diff --git a/arch/m68k/include/asm/uaccess_no.h b/arch/m68k/include/asm/uaccess_no.h
index e77ce66c14d5..e482c3899ff1 100644
--- a/arch/m68k/include/asm/uaccess_no.h
+++ b/arch/m68k/include/asm/uaccess_no.h
@@ -23,25 +23,6 @@ static inline int _access_ok(unsigned long addr, unsigned long size)
}
/*
- * The exception table consists of pairs of addresses: the first is the
- * address of an instruction that is allowed to fault, and the second is
- * the address at which the program should continue. No registers are
- * modified, so it is entirely up to the continuation code to figure out
- * what to do.
- *
- * All the routines below use bits of fixup code that are out of line
- * with the main instruction path. This means when everything is well,
- * we don't even have to jump over them. Further, they do not intrude
- * on our cache or tlb entries.
- */
-
-struct exception_table_entry
-{
- unsigned long insn, fixup;
-};
-
-
-/*
* These are the main single-value transfer routines. They automatically
* use the right size if we just have the right pointer type.
*/
@@ -120,13 +101,21 @@ extern int __get_user_bad(void);
: "=d" (x) \
: "m" (*__ptr(ptr)))
-#define copy_from_user(to, from, n) (memcpy(to, from, n), 0)
-#define copy_to_user(to, from, n) (memcpy(to, from, n), 0)
+static inline unsigned long
+raw_copy_from_user(void *to, const void __user *from, unsigned long n)
+{
+ memcpy(to, (__force const void *)from, n);
+ return 0;
+}
-#define __copy_from_user(to, from, n) copy_from_user(to, from, n)
-#define __copy_to_user(to, from, n) copy_to_user(to, from, n)
-#define __copy_to_user_inatomic __copy_to_user
-#define __copy_from_user_inatomic __copy_from_user
+static inline unsigned long
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
+{
+ memcpy((__force void *)to, from, n);
+ return 0;
+}
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
/*
* Copy a null terminated string from userspace.
diff --git a/arch/m68k/kernel/signal.c b/arch/m68k/kernel/signal.c
index 093b7c42fb85..6f945bb5ffbd 100644
--- a/arch/m68k/kernel/signal.c
+++ b/arch/m68k/kernel/signal.c
@@ -88,7 +88,7 @@ static inline int frame_extra_sizes(int f)
return frame_size_change[f];
}
-int handle_kernel_fault(struct pt_regs *regs)
+int fixup_exception(struct pt_regs *regs)
{
const struct exception_table_entry *fixup;
struct pt_regs *tregs;
diff --git a/arch/m68k/kernel/traps.c b/arch/m68k/kernel/traps.c
index a926d2c88898..c1cc4e99aa94 100644
--- a/arch/m68k/kernel/traps.c
+++ b/arch/m68k/kernel/traps.c
@@ -1016,8 +1016,13 @@ asmlinkage void trap_c(struct frame *fp)
/* traced a trapping instruction on a 68020/30,
* real exception will be executed afterwards.
*/
- } else if (!handle_kernel_fault(&fp->ptregs))
- bad_super_trap(fp);
+ return;
+ }
+#ifdef CONFIG_MMU
+ if (fixup_exception(&fp->ptregs))
+ return;
+#endif
+ bad_super_trap(fp);
return;
}
diff --git a/arch/m68k/lib/uaccess.c b/arch/m68k/lib/uaccess.c
index a76b73abaf64..7646e461aa62 100644
--- a/arch/m68k/lib/uaccess.c
+++ b/arch/m68k/lib/uaccess.c
@@ -30,19 +30,13 @@ unsigned long __generic_copy_from_user(void *to, const void __user *from,
"6:\n"
" .section .fixup,\"ax\"\n"
" .even\n"
- "10: move.l %0,%3\n"
- "7: clr.l (%2)+\n"
- " subq.l #1,%3\n"
- " jne 7b\n"
- " lsl.l #2,%0\n"
+ "10: lsl.l #2,%0\n"
" btst #1,%5\n"
" jeq 8f\n"
- "30: clr.w (%2)+\n"
- " addq.l #2,%0\n"
+ "30: addq.l #2,%0\n"
"8: btst #0,%5\n"
" jeq 6b\n"
- "50: clr.b (%2)+\n"
- " addq.l #1,%0\n"
+ "50: addq.l #1,%0\n"
" jra 6b\n"
" .previous\n"
"\n"
diff --git a/arch/m68k/mm/fault.c b/arch/m68k/mm/fault.c
index bd66a0b20c6b..2795e4ca09d7 100644
--- a/arch/m68k/mm/fault.c
+++ b/arch/m68k/mm/fault.c
@@ -32,7 +32,7 @@ int send_fault_sig(struct pt_regs *regs)
force_sig_info(siginfo.si_signo,
&siginfo, current);
} else {
- if (handle_kernel_fault(regs))
+ if (fixup_exception(regs))
return -1;
//if (siginfo.si_signo == SIGBUS)
diff --git a/arch/metag/Kconfig b/arch/metag/Kconfig
index 5b7a45d99cfb..ecce0c5ec8e8 100644
--- a/arch/metag/Kconfig
+++ b/arch/metag/Kconfig
@@ -1,5 +1,6 @@
config METAG
def_bool y
+ select ARCH_HAS_RAW_COPY_USER
select EMBEDDED
select GENERIC_ATOMIC64
select GENERIC_CLOCKEVENTS
diff --git a/arch/metag/include/asm/Kbuild b/arch/metag/include/asm/Kbuild
index f9b9df5d6de9..8f940553a579 100644
--- a/arch/metag/include/asm/Kbuild
+++ b/arch/metag/include/asm/Kbuild
@@ -8,6 +8,7 @@ generic-y += dma.h
generic-y += emergency-restart.h
generic-y += errno.h
generic-y += exec.h
+generic-y += extable.h
generic-y += fb.h
generic-y += fcntl.h
generic-y += futex.h
diff --git a/arch/metag/include/asm/uaccess.h b/arch/metag/include/asm/uaccess.h
index 83de7554d2b3..5ebc2850690e 100644
--- a/arch/metag/include/asm/uaccess.h
+++ b/arch/metag/include/asm/uaccess.h
@@ -47,28 +47,7 @@ static inline int __access_ok(unsigned long addr, unsigned long size)
#define access_ok(type, addr, size) __access_ok((unsigned long)(addr), \
(unsigned long)(size))
-static inline int verify_area(int type, const void *addr, unsigned long size)
-{
- return access_ok(type, addr, size) ? 0 : -EFAULT;
-}
-
-/*
- * The exception table consists of pairs of addresses: the first is the
- * address of an instruction that is allowed to fault, and the second is
- * the address at which the program should continue. No registers are
- * modified, so it is entirely up to the continuation code to figure out
- * what to do.
- *
- * All the routines below use bits of fixup code that are out of line
- * with the main instruction path. This means when everything is well,
- * we don't even have to jump over them. Further, they do not intrude
- * on our cache or tlb entries.
- */
-struct exception_table_entry {
- unsigned long insn, fixup;
-};
-
-extern int fixup_exception(struct pt_regs *regs);
+#include <asm/extable.h>
/*
* These are the main single-value transfer routines. They automatically
@@ -193,36 +172,10 @@ extern long __must_check strnlen_user(const char __user *src, long count);
#define strlen_user(str) strnlen_user(str, 32767)
-extern unsigned long __must_check __copy_user_zeroing(void *to,
- const void __user *from,
- unsigned long n);
-
-static inline unsigned long
-copy_from_user(void *to, const void __user *from, unsigned long n)
-{
- if (likely(access_ok(VERIFY_READ, from, n)))
- return __copy_user_zeroing(to, from, n);
- memset(to, 0, n);
- return n;
-}
-
-#define __copy_from_user(to, from, n) __copy_user_zeroing(to, from, n)
-#define __copy_from_user_inatomic __copy_from_user
-
-extern unsigned long __must_check __copy_user(void __user *to,
- const void *from,
- unsigned long n);
-
-static inline unsigned long copy_to_user(void __user *to, const void *from,
- unsigned long n)
-{
- if (access_ok(VERIFY_WRITE, to, n))
- return __copy_user(to, from, n);
- return n;
-}
-
-#define __copy_to_user(to, from, n) __copy_user(to, from, n)
-#define __copy_to_user_inatomic __copy_to_user
+extern unsigned long raw_copy_from_user(void *to, const void __user *from,
+ unsigned long n);
+extern unsigned long raw_copy_to_user(void __user *to, const void *from,
+ unsigned long n);
/*
* Zero Userspace
diff --git a/arch/metag/lib/usercopy.c b/arch/metag/lib/usercopy.c
index b3ebfe9c8e88..e8a4ea83cabb 100644
--- a/arch/metag/lib/usercopy.c
+++ b/arch/metag/lib/usercopy.c
@@ -29,7 +29,6 @@
COPY \
"1:\n" \
" .section .fixup,\"ax\"\n" \
- " MOV D1Ar1,#0\n" \
FIXUP \
" MOVT D1Ar1,#HI(1b)\n" \
" JUMP D1Ar1,#LO(1b)\n" \
@@ -260,27 +259,31 @@
"MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
"22:\n" \
"MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
- "SUB %3, %3, #32\n" \
"23:\n" \
- "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
+ "SUB %3, %3, #32\n" \
"24:\n" \
+ "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
+ "25:\n" \
"MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
+ "26:\n" \
"SUB %3, %3, #32\n" \
"DCACHE [%1+#-64], D0Ar6\n" \
"BR $Lloop"id"\n" \
\
"MOV RAPF, %1\n" \
- "25:\n" \
+ "27:\n" \
"MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
- "26:\n" \
+ "28:\n" \
"MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
+ "29:\n" \
"SUB %3, %3, #32\n" \
- "27:\n" \
+ "30:\n" \
"MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
- "28:\n" \
+ "31:\n" \
"MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
+ "32:\n" \
"SUB %0, %0, #8\n" \
- "29:\n" \
+ "33:\n" \
"SETL [%0++], D0.7, D1.7\n" \
"SUB %3, %3, #32\n" \
"1:" \
@@ -312,11 +315,15 @@
" .long 26b,3b\n" \
" .long 27b,3b\n" \
" .long 28b,3b\n" \
- " .long 29b,4b\n" \
+ " .long 29b,3b\n" \
+ " .long 30b,3b\n" \
+ " .long 31b,3b\n" \
+ " .long 32b,3b\n" \
+ " .long 33b,4b\n" \
" .previous\n" \
: "=r" (to), "=r" (from), "=r" (ret), "=d" (n) \
: "0" (to), "1" (from), "2" (ret), "3" (n) \
- : "D1Ar1", "D0Ar2", "memory")
+ : "D1Ar1", "D0Ar2", "cc", "memory")
/* rewind 'to' and 'from' pointers when a fault occurs
*
@@ -342,7 +349,7 @@
#define __asm_copy_to_user_64bit_rapf_loop(to, from, ret, n, id)\
__asm_copy_user_64bit_rapf_loop(to, from, ret, n, id, \
"LSR D0Ar2, D0Ar2, #8\n" \
- "AND D0Ar2, D0Ar2, #0x7\n" \
+ "ANDS D0Ar2, D0Ar2, #0x7\n" \
"ADDZ D0Ar2, D0Ar2, #4\n" \
"SUB D0Ar2, D0Ar2, #1\n" \
"MOV D1Ar1, #4\n" \
@@ -403,47 +410,55 @@
"MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
"22:\n" \
"MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
- "SUB %3, %3, #16\n" \
"23:\n" \
- "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
- "24:\n" \
- "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
"SUB %3, %3, #16\n" \
- "25:\n" \
+ "24:\n" \
"MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
- "26:\n" \
+ "25:\n" \
"MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
+ "26:\n" \
"SUB %3, %3, #16\n" \
"27:\n" \
"MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
"28:\n" \
"MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
+ "29:\n" \
+ "SUB %3, %3, #16\n" \
+ "30:\n" \
+ "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
+ "31:\n" \
+ "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
+ "32:\n" \
"SUB %3, %3, #16\n" \
"DCACHE [%1+#-64], D0Ar6\n" \
"BR $Lloop"id"\n" \
\
"MOV RAPF, %1\n" \
- "29:\n" \
+ "33:\n" \
"MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
- "30:\n" \
+ "34:\n" \
"MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
+ "35:\n" \
"SUB %3, %3, #16\n" \
- "31:\n" \
+ "36:\n" \
"MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
- "32:\n" \
+ "37:\n" \
"MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
+ "38:\n" \
"SUB %3, %3, #16\n" \
- "33:\n" \
+ "39:\n" \
"MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
- "34:\n" \
+ "40:\n" \
"MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
+ "41:\n" \
"SUB %3, %3, #16\n" \
- "35:\n" \
+ "42:\n" \
"MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
- "36:\n" \
+ "43:\n" \
"MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
+ "44:\n" \
"SUB %0, %0, #4\n" \
- "37:\n" \
+ "45:\n" \
"SETD [%0++], D0.7\n" \
"SUB %3, %3, #16\n" \
"1:" \
@@ -483,11 +498,19 @@
" .long 34b,3b\n" \
" .long 35b,3b\n" \
" .long 36b,3b\n" \
- " .long 37b,4b\n" \
+ " .long 37b,3b\n" \
+ " .long 38b,3b\n" \
+ " .long 39b,3b\n" \
+ " .long 40b,3b\n" \
+ " .long 41b,3b\n" \
+ " .long 42b,3b\n" \
+ " .long 43b,3b\n" \
+ " .long 44b,3b\n" \
+ " .long 45b,4b\n" \
" .previous\n" \
: "=r" (to), "=r" (from), "=r" (ret), "=d" (n) \
: "0" (to), "1" (from), "2" (ret), "3" (n) \
- : "D1Ar1", "D0Ar2", "memory")
+ : "D1Ar1", "D0Ar2", "cc", "memory")
/* rewind 'to' and 'from' pointers when a fault occurs
*
@@ -513,7 +536,7 @@
#define __asm_copy_to_user_32bit_rapf_loop(to, from, ret, n, id)\
__asm_copy_user_32bit_rapf_loop(to, from, ret, n, id, \
"LSR D0Ar2, D0Ar2, #8\n" \
- "AND D0Ar2, D0Ar2, #0x7\n" \
+ "ANDS D0Ar2, D0Ar2, #0x7\n" \
"ADDZ D0Ar2, D0Ar2, #4\n" \
"SUB D0Ar2, D0Ar2, #1\n" \
"MOV D1Ar1, #4\n" \
@@ -525,8 +548,8 @@
"SUB %1, %1, D0Ar2\n" \
"SUB %3, %3, D1Ar1\n")
-unsigned long __copy_user(void __user *pdst, const void *psrc,
- unsigned long n)
+unsigned long raw_copy_to_user(void __user *pdst, const void *psrc,
+ unsigned long n)
{
register char __user *dst asm ("A0.2") = pdst;
register const char *src asm ("A1.2") = psrc;
@@ -538,23 +561,31 @@ unsigned long __copy_user(void __user *pdst, const void *psrc,
if ((unsigned long) src & 1) {
__asm_copy_to_user_1(dst, src, retn);
n--;
+ if (retn)
+ return retn + n;
}
if ((unsigned long) dst & 1) {
/* Worst case - byte copy */
while (n > 0) {
__asm_copy_to_user_1(dst, src, retn);
n--;
+ if (retn)
+ return retn + n;
}
}
if (((unsigned long) src & 2) && n >= 2) {
__asm_copy_to_user_2(dst, src, retn);
n -= 2;
+ if (retn)
+ return retn + n;
}
if ((unsigned long) dst & 2) {
/* Second worst case - word copy */
while (n >= 2) {
__asm_copy_to_user_2(dst, src, retn);
n -= 2;
+ if (retn)
+ return retn + n;
}
}
@@ -569,6 +600,8 @@ unsigned long __copy_user(void __user *pdst, const void *psrc,
while (n >= 8) {
__asm_copy_to_user_8x64(dst, src, retn);
n -= 8;
+ if (retn)
+ return retn + n;
}
}
if (n >= RAPF_MIN_BUF_SIZE) {
@@ -581,6 +614,8 @@ unsigned long __copy_user(void __user *pdst, const void *psrc,
while (n >= 8) {
__asm_copy_to_user_8x64(dst, src, retn);
n -= 8;
+ if (retn)
+ return retn + n;
}
}
#endif
@@ -588,11 +623,15 @@ unsigned long __copy_user(void __user *pdst, const void *psrc,
while (n >= 16) {
__asm_copy_to_user_16(dst, src, retn);
n -= 16;
+ if (retn)
+ return retn + n;
}
while (n >= 4) {
__asm_copy_to_user_4(dst, src, retn);
n -= 4;
+ if (retn)
+ return retn + n;
}
switch (n) {
@@ -609,24 +648,26 @@ unsigned long __copy_user(void __user *pdst, const void *psrc,
break;
}
+ /*
+ * If we get here, retn correctly reflects the number of failing
+ * bytes.
+ */
return retn;
}
-EXPORT_SYMBOL(__copy_user);
+EXPORT_SYMBOL(raw_copy_to_user);
#define __asm_copy_from_user_1(to, from, ret) \
__asm_copy_user_cont(to, from, ret, \
" GETB D1Ar1,[%1++]\n" \
"2: SETB [%0++],D1Ar1\n", \
- "3: ADD %2,%2,#1\n" \
- " SETB [%0++],D1Ar1\n", \
+ "3: ADD %2,%2,#1\n", \
" .long 2b,3b\n")
#define __asm_copy_from_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
__asm_copy_user_cont(to, from, ret, \
" GETW D1Ar1,[%1++]\n" \
"2: SETW [%0++],D1Ar1\n" COPY, \
- "3: ADD %2,%2,#2\n" \
- " SETW [%0++],D1Ar1\n" FIXUP, \
+ "3: ADD %2,%2,#2\n" FIXUP, \
" .long 2b,3b\n" TENTRY)
#define __asm_copy_from_user_2(to, from, ret) \
@@ -636,145 +677,26 @@ EXPORT_SYMBOL(__copy_user);
__asm_copy_from_user_2x_cont(to, from, ret, \
" GETB D1Ar1,[%1++]\n" \
"4: SETB [%0++],D1Ar1\n", \
- "5: ADD %2,%2,#1\n" \
- " SETB [%0++],D1Ar1\n", \
+ "5: ADD %2,%2,#1\n", \
" .long 4b,5b\n")
#define __asm_copy_from_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
__asm_copy_user_cont(to, from, ret, \
" GETD D1Ar1,[%1++]\n" \
"2: SETD [%0++],D1Ar1\n" COPY, \
- "3: ADD %2,%2,#4\n" \
- " SETD [%0++],D1Ar1\n" FIXUP, \
+ "3: ADD %2,%2,#4\n" FIXUP, \
" .long 2b,3b\n" TENTRY)
#define __asm_copy_from_user_4(to, from, ret) \
__asm_copy_from_user_4x_cont(to, from, ret, "", "", "")
-#define __asm_copy_from_user_5(to, from, ret) \
- __asm_copy_from_user_4x_cont(to, from, ret, \
- " GETB D1Ar1,[%1++]\n" \
- "4: SETB [%0++],D1Ar1\n", \
- "5: ADD %2,%2,#1\n" \
- " SETB [%0++],D1Ar1\n", \
- " .long 4b,5b\n")
-
-#define __asm_copy_from_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
- __asm_copy_from_user_4x_cont(to, from, ret, \
- " GETW D1Ar1,[%1++]\n" \
- "4: SETW [%0++],D1Ar1\n" COPY, \
- "5: ADD %2,%2,#2\n" \
- " SETW [%0++],D1Ar1\n" FIXUP, \
- " .long 4b,5b\n" TENTRY)
-
-#define __asm_copy_from_user_6(to, from, ret) \
- __asm_copy_from_user_6x_cont(to, from, ret, "", "", "")
-
-#define __asm_copy_from_user_7(to, from, ret) \
- __asm_copy_from_user_6x_cont(to, from, ret, \
- " GETB D1Ar1,[%1++]\n" \
- "6: SETB [%0++],D1Ar1\n", \
- "7: ADD %2,%2,#1\n" \
- " SETB [%0++],D1Ar1\n", \
- " .long 6b,7b\n")
-
-#define __asm_copy_from_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
- __asm_copy_from_user_4x_cont(to, from, ret, \
- " GETD D1Ar1,[%1++]\n" \
- "4: SETD [%0++],D1Ar1\n" COPY, \
- "5: ADD %2,%2,#4\n" \
- " SETD [%0++],D1Ar1\n" FIXUP, \
- " .long 4b,5b\n" TENTRY)
-
-#define __asm_copy_from_user_8(to, from, ret) \
- __asm_copy_from_user_8x_cont(to, from, ret, "", "", "")
-
-#define __asm_copy_from_user_9(to, from, ret) \
- __asm_copy_from_user_8x_cont(to, from, ret, \
- " GETB D1Ar1,[%1++]\n" \
- "6: SETB [%0++],D1Ar1\n", \
- "7: ADD %2,%2,#1\n" \
- " SETB [%0++],D1Ar1\n", \
- " .long 6b,7b\n")
-
-#define __asm_copy_from_user_10x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
- __asm_copy_from_user_8x_cont(to, from, ret, \
- " GETW D1Ar1,[%1++]\n" \
- "6: SETW [%0++],D1Ar1\n" COPY, \
- "7: ADD %2,%2,#2\n" \
- " SETW [%0++],D1Ar1\n" FIXUP, \
- " .long 6b,7b\n" TENTRY)
-
-#define __asm_copy_from_user_10(to, from, ret) \
- __asm_copy_from_user_10x_cont(to, from, ret, "", "", "")
-
-#define __asm_copy_from_user_11(to, from, ret) \
- __asm_copy_from_user_10x_cont(to, from, ret, \
- " GETB D1Ar1,[%1++]\n" \
- "8: SETB [%0++],D1Ar1\n", \
- "9: ADD %2,%2,#1\n" \
- " SETB [%0++],D1Ar1\n", \
- " .long 8b,9b\n")
-
-#define __asm_copy_from_user_12x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
- __asm_copy_from_user_8x_cont(to, from, ret, \
- " GETD D1Ar1,[%1++]\n" \
- "6: SETD [%0++],D1Ar1\n" COPY, \
- "7: ADD %2,%2,#4\n" \
- " SETD [%0++],D1Ar1\n" FIXUP, \
- " .long 6b,7b\n" TENTRY)
-
-#define __asm_copy_from_user_12(to, from, ret) \
- __asm_copy_from_user_12x_cont(to, from, ret, "", "", "")
-
-#define __asm_copy_from_user_13(to, from, ret) \
- __asm_copy_from_user_12x_cont(to, from, ret, \
- " GETB D1Ar1,[%1++]\n" \
- "8: SETB [%0++],D1Ar1\n", \
- "9: ADD %2,%2,#1\n" \
- " SETB [%0++],D1Ar1\n", \
- " .long 8b,9b\n")
-
-#define __asm_copy_from_user_14x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
- __asm_copy_from_user_12x_cont(to, from, ret, \
- " GETW D1Ar1,[%1++]\n" \
- "8: SETW [%0++],D1Ar1\n" COPY, \
- "9: ADD %2,%2,#2\n" \
- " SETW [%0++],D1Ar1\n" FIXUP, \
- " .long 8b,9b\n" TENTRY)
-
-#define __asm_copy_from_user_14(to, from, ret) \
- __asm_copy_from_user_14x_cont(to, from, ret, "", "", "")
-
-#define __asm_copy_from_user_15(to, from, ret) \
- __asm_copy_from_user_14x_cont(to, from, ret, \
- " GETB D1Ar1,[%1++]\n" \
- "10: SETB [%0++],D1Ar1\n", \
- "11: ADD %2,%2,#1\n" \
- " SETB [%0++],D1Ar1\n", \
- " .long 10b,11b\n")
-
-#define __asm_copy_from_user_16x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
- __asm_copy_from_user_12x_cont(to, from, ret, \
- " GETD D1Ar1,[%1++]\n" \
- "8: SETD [%0++],D1Ar1\n" COPY, \
- "9: ADD %2,%2,#4\n" \
- " SETD [%0++],D1Ar1\n" FIXUP, \
- " .long 8b,9b\n" TENTRY)
-
-#define __asm_copy_from_user_16(to, from, ret) \
- __asm_copy_from_user_16x_cont(to, from, ret, "", "", "")
-
#define __asm_copy_from_user_8x64(to, from, ret) \
asm volatile ( \
" GETL D0Ar2,D1Ar1,[%1++]\n" \
"2: SETL [%0++],D0Ar2,D1Ar1\n" \
"1:\n" \
" .section .fixup,\"ax\"\n" \
- " MOV D1Ar1,#0\n" \
- " MOV D0Ar2,#0\n" \
"3: ADD %2,%2,#8\n" \
- " SETL [%0++],D0Ar2,D1Ar1\n" \
" MOVT D0Ar2,#HI(1b)\n" \
" JUMP D0Ar2,#LO(1b)\n" \
" .previous\n" \
@@ -789,36 +711,57 @@ EXPORT_SYMBOL(__copy_user);
*
* Rationale:
* A fault occurs while reading from user buffer, which is the
- * source. Since the fault is at a single address, we only
- * need to rewind by 8 bytes.
+ * source.
* Since we don't write to kernel buffer until we read first,
* the kernel buffer is at the right state and needn't be
- * corrected.
+ * corrected, but the source must be rewound to the beginning of
+ * the block, which is LSM_STEP*8 bytes.
+ * LSM_STEP is bits 10:8 in TXSTATUS which is already read
+ * and stored in D0Ar2
+ *
+ * NOTE: If a fault occurs at the last operation in M{G,S}ETL
+ * LSM_STEP will be 0. ie: we do 4 writes in our case, if
+ * a fault happens at the 4th write, LSM_STEP will be 0
+ * instead of 4. The code copes with that.
*/
#define __asm_copy_from_user_64bit_rapf_loop(to, from, ret, n, id) \
__asm_copy_user_64bit_rapf_loop(to, from, ret, n, id, \
- "SUB %1, %1, #8\n")
+ "LSR D0Ar2, D0Ar2, #5\n" \
+ "ANDS D0Ar2, D0Ar2, #0x38\n" \
+ "ADDZ D0Ar2, D0Ar2, #32\n" \
+ "SUB %1, %1, D0Ar2\n")
/* rewind 'from' pointer when a fault occurs
*
* Rationale:
* A fault occurs while reading from user buffer, which is the
- * source. Since the fault is at a single address, we only
- * need to rewind by 4 bytes.
+ * source.
* Since we don't write to kernel buffer until we read first,
* the kernel buffer is at the right state and needn't be
- * corrected.
+ * corrected, but the source must be rewound to the beginning of
+ * the block, which is LSM_STEP*4 bytes.
+ * LSM_STEP is bits 10:8 in TXSTATUS which is already read
+ * and stored in D0Ar2
+ *
+ * NOTE: If a fault occurs at the last operation in M{G,S}ETL
+ * LSM_STEP will be 0. ie: we do 4 writes in our case, if
+ * a fault happens at the 4th write, LSM_STEP will be 0
+ * instead of 4. The code copes with that.
*/
#define __asm_copy_from_user_32bit_rapf_loop(to, from, ret, n, id) \
__asm_copy_user_32bit_rapf_loop(to, from, ret, n, id, \
- "SUB %1, %1, #4\n")
+ "LSR D0Ar2, D0Ar2, #6\n" \
+ "ANDS D0Ar2, D0Ar2, #0x1c\n" \
+ "ADDZ D0Ar2, D0Ar2, #16\n" \
+ "SUB %1, %1, D0Ar2\n")
-/* Copy from user to kernel, zeroing the bytes that were inaccessible in
- userland. The return-value is the number of bytes that were
- inaccessible. */
-unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc,
- unsigned long n)
+/*
+ * Copy from user to kernel. The return-value is the number of bytes that were
+ * inaccessible.
+ */
+unsigned long raw_copy_from_user(void *pdst, const void __user *psrc,
+ unsigned long n)
{
register char *dst asm ("A0.2") = pdst;
register const char __user *src asm ("A1.2") = psrc;
@@ -830,6 +773,8 @@ unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc,
if ((unsigned long) src & 1) {
__asm_copy_from_user_1(dst, src, retn);
n--;
+ if (retn)
+ return retn + n;
}
if ((unsigned long) dst & 1) {
/* Worst case - byte copy */
@@ -837,12 +782,14 @@ unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc,
__asm_copy_from_user_1(dst, src, retn);
n--;
if (retn)
- goto copy_exception_bytes;
+ return retn + n;
}
}
if (((unsigned long) src & 2) && n >= 2) {
__asm_copy_from_user_2(dst, src, retn);
n -= 2;
+ if (retn)
+ return retn + n;
}
if ((unsigned long) dst & 2) {
/* Second worst case - word copy */
@@ -850,16 +797,10 @@ unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc,
__asm_copy_from_user_2(dst, src, retn);
n -= 2;
if (retn)
- goto copy_exception_bytes;
+ return retn + n;
}
}
- /* We only need one check after the unalignment-adjustments,
- because if both adjustments were done, either both or
- neither reference had an exception. */
- if (retn != 0)
- goto copy_exception_bytes;
-
#ifdef USE_RAPF
/* 64 bit copy loop */
if (!(((unsigned long) src | (unsigned long) dst) & 7)) {
@@ -872,7 +813,7 @@ unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc,
__asm_copy_from_user_8x64(dst, src, retn);
n -= 8;
if (retn)
- goto copy_exception_bytes;
+ return retn + n;
}
}
@@ -888,7 +829,7 @@ unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc,
__asm_copy_from_user_8x64(dst, src, retn);
n -= 8;
if (retn)
- goto copy_exception_bytes;
+ return retn + n;
}
}
#endif
@@ -898,7 +839,7 @@ unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc,
n -= 4;
if (retn)
- goto copy_exception_bytes;
+ return retn + n;
}
/* If we get here, there were no memory read faults. */
@@ -924,21 +865,8 @@ unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc,
/* If we get here, retn correctly reflects the number of failing
bytes. */
return retn;
-
- copy_exception_bytes:
- /* We already have "retn" bytes cleared, and need to clear the
- remaining "n" bytes. A non-optimized simple byte-for-byte in-line
- memset is preferred here, since this isn't speed-critical code and
- we'd rather have this a leaf-function than calling memset. */
- {
- char *endp;
- for (endp = dst + n; dst < endp; dst++)
- *dst = 0;
- }
-
- return retn + n;
}
-EXPORT_SYMBOL(__copy_user_zeroing);
+EXPORT_SYMBOL(raw_copy_from_user);
#define __asm_clear_8x64(to, ret) \
asm volatile ( \
diff --git a/arch/microblaze/Kconfig b/arch/microblaze/Kconfig
index 85885a501dce..1aff3658a104 100644
--- a/arch/microblaze/Kconfig
+++ b/arch/microblaze/Kconfig
@@ -34,6 +34,7 @@ config MICROBLAZE
select TRACING_SUPPORT
select VIRT_TO_BUS
select CPU_NO_EFFICIENT_FFS
+ select ARCH_HAS_RAW_COPY_USER
config SWAP
def_bool n
diff --git a/arch/microblaze/include/asm/Kbuild b/arch/microblaze/include/asm/Kbuild
index 1732ec13b211..56830ff65333 100644
--- a/arch/microblaze/include/asm/Kbuild
+++ b/arch/microblaze/include/asm/Kbuild
@@ -3,6 +3,7 @@ generic-y += barrier.h
generic-y += clkdev.h
generic-y += device.h
generic-y += exec.h
+generic-y += extable.h
generic-y += irq_work.h
generic-y += mcs_spinlock.h
generic-y += mm-arch-hooks.h
diff --git a/arch/microblaze/include/asm/uaccess.h b/arch/microblaze/include/asm/uaccess.h
index b132cd301e66..38f2c9ccef10 100644
--- a/arch/microblaze/include/asm/uaccess.h
+++ b/arch/microblaze/include/asm/uaccess.h
@@ -17,6 +17,7 @@
#include <asm/mmu.h>
#include <asm/page.h>
#include <asm/pgtable.h>
+#include <asm/extable.h>
#include <linux/string.h>
/*
@@ -47,22 +48,6 @@
# define segment_eq(a, b) ((a).seg == (b).seg)
-/*
- * The exception table consists of pairs of addresses: the first is the
- * address of an instruction that is allowed to fault, and the second is
- * the address at which the program should continue. No registers are
- * modified, so it is entirely up to the continuation code to figure out
- * what to do.
- *
- * All the routines below use bits of fixup code that are out of line
- * with the main instruction path. This means when everything is well,
- * we don't even have to jump over them. Further, they do not intrude
- * on our cache or tlb entries.
- */
-struct exception_table_entry {
- unsigned long insn, fixup;
-};
-
#ifndef CONFIG_MMU
/* Check against bounds of physical memory */
@@ -351,39 +336,19 @@ extern long __user_bad(void);
__gu_err; \
})
-
-/* copy_to_from_user */
-#define __copy_from_user(to, from, n) \
- __copy_tofrom_user((__force void __user *)(to), \
- (void __user *)(from), (n))
-#define __copy_from_user_inatomic(to, from, n) \
- __copy_from_user((to), (from), (n))
-
-static inline long copy_from_user(void *to,
- const void __user *from, unsigned long n)
+static inline unsigned long
+raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
- unsigned long res = n;
- might_fault();
- if (likely(access_ok(VERIFY_READ, from, n)))
- res = __copy_from_user(to, from, n);
- if (unlikely(res))
- memset(to + (n - res), 0, res);
- return res;
+ return __copy_tofrom_user((__force void __user *)to, from, n);
}
-#define __copy_to_user(to, from, n) \
- __copy_tofrom_user((void __user *)(to), \
- (__force const void __user *)(from), (n))
-#define __copy_to_user_inatomic(to, from, n) __copy_to_user((to), (from), (n))
-
-static inline long copy_to_user(void __user *to,
- const void *from, unsigned long n)
+static inline unsigned long
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
- might_fault();
- if (access_ok(VERIFY_WRITE, to, n))
- return __copy_to_user(to, from, n);
- return n;
+ return __copy_tofrom_user(to, (__force const void __user *)from, n);
}
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
/*
* Copy a null terminated string from userspace.
diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index a008a9f03072..aff5633bfe4d 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -69,6 +69,7 @@ config MIPS
select HAVE_EXIT_THREAD
select HAVE_REGS_AND_STACK_ACCESS_API
select HAVE_ARCH_HARDENED_USERCOPY
+ select ARCH_HAS_RAW_COPY_USER
menu "Machine selection"
diff --git a/arch/mips/cavium-octeon/octeon-memcpy.S b/arch/mips/cavium-octeon/octeon-memcpy.S
index cfd97f6448bb..0a7c9834b81c 100644
--- a/arch/mips/cavium-octeon/octeon-memcpy.S
+++ b/arch/mips/cavium-octeon/octeon-memcpy.S
@@ -140,15 +140,6 @@
.set noat
/*
- * t7 is used as a flag to note inatomic mode.
- */
-LEAF(__copy_user_inatomic)
-EXPORT_SYMBOL(__copy_user_inatomic)
- b __copy_user_common
- li t7, 1
- END(__copy_user_inatomic)
-
-/*
* A combined memcpy/__copy_user
* __copy_user sets len to 0 for success; else to an upper bound of
* the number of uncopied bytes.
@@ -161,8 +152,6 @@ EXPORT_SYMBOL(memcpy)
__memcpy:
FEXPORT(__copy_user)
EXPORT_SYMBOL(__copy_user)
- li t7, 0 /* not inatomic */
-__copy_user_common:
/*
* Note: dst & src may be unaligned, len may be 0
* Temps
@@ -414,25 +403,7 @@ l_exc:
LOAD t0, TI_TASK($28)
LOAD t0, THREAD_BUADDR(t0) # t0 is just past last good address
SUB len, AT, t0 # len number of uncopied bytes
- bnez t7, 2f /* Skip the zeroing out part if inatomic */
- /*
- * Here's where we rely on src and dst being incremented in tandem,
- * See (3) above.
- * dst += (fault addr - src) to put dst at first byte to clear
- */
- ADD dst, t0 # compute start address in a1
- SUB dst, src
- /*
- * Clear len bytes starting at dst. Can't call __bzero because it
- * might modify len. An inefficient loop for these rare times...
- */
- beqz len, done
- SUB src, len, 1
-1: sb zero, 0(dst)
- ADD dst, dst, 1
- bnez src, 1b
- SUB src, src, 1
-2: jr ra
+ jr ra
nop
diff --git a/arch/mips/include/asm/uaccess.h b/arch/mips/include/asm/uaccess.h
index 70ca8eee166a..99e629a590a5 100644
--- a/arch/mips/include/asm/uaccess.h
+++ b/arch/mips/include/asm/uaccess.h
@@ -128,23 +128,14 @@ static inline bool eva_kernel_access(void)
* this function, memory access functions may still return -EFAULT.
*/
-#define __access_mask get_fs().seg
-
-#define __access_ok(addr, size, mask) \
-({ \
- unsigned long __addr = (unsigned long) (addr); \
- unsigned long __size = size; \
- unsigned long __mask = mask; \
- unsigned long __ok; \
- \
- __chk_user_ptr(addr); \
- __ok = (signed long)(__mask & (__addr | (__addr + __size) | \
- __ua_size(__size))); \
- __ok == 0; \
-})
+static inline int __access_ok(const void __user *p, unsigned long size)
+{
+ unsigned long addr = (unsigned long)p;
+ return (get_fs().seg & (addr | (addr + size) | __ua_size(size))) == 0;
+}
#define access_ok(type, addr, size) \
- likely(__access_ok((addr), (size), __access_mask))
+ likely(__access_ok((addr), (size)))
/*
* put_user: - Write a simple value into user space.
@@ -806,157 +797,7 @@ extern void __put_user_unaligned_unknown(void);
extern size_t __copy_user(void *__to, const void *__from, size_t __n);
-#ifndef CONFIG_EVA
-#define __invoke_copy_to_user(to, from, n) \
-({ \
- register void __user *__cu_to_r __asm__("$4"); \
- register const void *__cu_from_r __asm__("$5"); \
- register long __cu_len_r __asm__("$6"); \
- \
- __cu_to_r = (to); \
- __cu_from_r = (from); \
- __cu_len_r = (n); \
- __asm__ __volatile__( \
- __MODULE_JAL(__copy_user) \
- : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \
- : \
- : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \
- DADDI_SCRATCH, "memory"); \
- __cu_len_r; \
-})
-
-#define __invoke_copy_to_kernel(to, from, n) \
- __invoke_copy_to_user(to, from, n)
-
-#endif
-
-/*
- * __copy_to_user: - Copy a block of data into user space, with less checking.
- * @to: Destination address, in user space.
- * @from: Source address, in kernel space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from kernel space to user space. Caller must check
- * the specified block with access_ok() before calling this function.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- */
-#define __copy_to_user(to, from, n) \
-({ \
- void __user *__cu_to; \
- const void *__cu_from; \
- long __cu_len; \
- \
- __cu_to = (to); \
- __cu_from = (from); \
- __cu_len = (n); \
- \
- check_object_size(__cu_from, __cu_len, true); \
- might_fault(); \
- \
- if (eva_kernel_access()) \
- __cu_len = __invoke_copy_to_kernel(__cu_to, __cu_from, \
- __cu_len); \
- else \
- __cu_len = __invoke_copy_to_user(__cu_to, __cu_from, \
- __cu_len); \
- __cu_len; \
-})
-
-extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n);
-
-#define __copy_to_user_inatomic(to, from, n) \
-({ \
- void __user *__cu_to; \
- const void *__cu_from; \
- long __cu_len; \
- \
- __cu_to = (to); \
- __cu_from = (from); \
- __cu_len = (n); \
- \
- check_object_size(__cu_from, __cu_len, true); \
- \
- if (eva_kernel_access()) \
- __cu_len = __invoke_copy_to_kernel(__cu_to, __cu_from, \
- __cu_len); \
- else \
- __cu_len = __invoke_copy_to_user(__cu_to, __cu_from, \
- __cu_len); \
- __cu_len; \
-})
-
-#define __copy_from_user_inatomic(to, from, n) \
-({ \
- void *__cu_to; \
- const void __user *__cu_from; \
- long __cu_len; \
- \
- __cu_to = (to); \
- __cu_from = (from); \
- __cu_len = (n); \
- \
- check_object_size(__cu_to, __cu_len, false); \
- \
- if (eva_kernel_access()) \
- __cu_len = __invoke_copy_from_kernel_inatomic(__cu_to, \
- __cu_from,\
- __cu_len);\
- else \
- __cu_len = __invoke_copy_from_user_inatomic(__cu_to, \
- __cu_from, \
- __cu_len); \
- __cu_len; \
-})
-
-/*
- * copy_to_user: - Copy a block of data into user space.
- * @to: Destination address, in user space.
- * @from: Source address, in kernel space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from kernel space to user space.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- */
-#define copy_to_user(to, from, n) \
-({ \
- void __user *__cu_to; \
- const void *__cu_from; \
- long __cu_len; \
- \
- __cu_to = (to); \
- __cu_from = (from); \
- __cu_len = (n); \
- \
- check_object_size(__cu_from, __cu_len, true); \
- \
- if (eva_kernel_access()) { \
- __cu_len = __invoke_copy_to_kernel(__cu_to, \
- __cu_from, \
- __cu_len); \
- } else { \
- if (access_ok(VERIFY_WRITE, __cu_to, __cu_len)) { \
- might_fault(); \
- __cu_len = __invoke_copy_to_user(__cu_to, \
- __cu_from, \
- __cu_len); \
- } \
- } \
- __cu_len; \
-})
-
-#ifndef CONFIG_EVA
-
-#define __invoke_copy_from_user(to, from, n) \
+#define __invoke_copy_from(func, to, from, n) \
({ \
register void *__cu_to_r __asm__("$4"); \
register const void __user *__cu_from_r __asm__("$5"); \
@@ -967,7 +808,7 @@ extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n);
__cu_len_r = (n); \
__asm__ __volatile__( \
".set\tnoreorder\n\t" \
- __MODULE_JAL(__copy_user) \
+ __MODULE_JAL(func) \
".set\tnoat\n\t" \
__UA_ADDU "\t$1, %1, %2\n\t" \
".set\tat\n\t" \
@@ -979,33 +820,17 @@ extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n);
__cu_len_r; \
})
-#define __invoke_copy_from_kernel(to, from, n) \
- __invoke_copy_from_user(to, from, n)
-
-/* For userland <-> userland operations */
-#define ___invoke_copy_in_user(to, from, n) \
- __invoke_copy_from_user(to, from, n)
-
-/* For kernel <-> kernel operations */
-#define ___invoke_copy_in_kernel(to, from, n) \
- __invoke_copy_from_user(to, from, n)
-
-#define __invoke_copy_from_user_inatomic(to, from, n) \
+#define __invoke_copy_to(func, to, from, n) \
({ \
- register void *__cu_to_r __asm__("$4"); \
- register const void __user *__cu_from_r __asm__("$5"); \
+ register void __user *__cu_to_r __asm__("$4"); \
+ register const void *__cu_from_r __asm__("$5"); \
register long __cu_len_r __asm__("$6"); \
\
__cu_to_r = (to); \
__cu_from_r = (from); \
__cu_len_r = (n); \
__asm__ __volatile__( \
- ".set\tnoreorder\n\t" \
- __MODULE_JAL(__copy_user_inatomic) \
- ".set\tnoat\n\t" \
- __UA_ADDU "\t$1, %1, %2\n\t" \
- ".set\tat\n\t" \
- ".set\treorder" \
+ __MODULE_JAL(func) \
: "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \
: \
: "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \
@@ -1013,228 +838,79 @@ extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n);
__cu_len_r; \
})
-#define __invoke_copy_from_kernel_inatomic(to, from, n) \
- __invoke_copy_from_user_inatomic(to, from, n) \
+#define __invoke_copy_from_kernel(to, from, n) \
+ __invoke_copy_from(__copy_user, to, from, n)
+
+#define __invoke_copy_to_kernel(to, from, n) \
+ __invoke_copy_to(__copy_user, to, from, n)
+
+#define ___invoke_copy_in_kernel(to, from, n) \
+ __invoke_copy_from(__copy_user, to, from, n)
+
+#ifndef CONFIG_EVA
+#define __invoke_copy_from_user(to, from, n) \
+ __invoke_copy_from(__copy_user, to, from, n)
+
+#define __invoke_copy_to_user(to, from, n) \
+ __invoke_copy_to(__copy_user, to, from, n)
+
+#define ___invoke_copy_in_user(to, from, n) \
+ __invoke_copy_from(__copy_user, to, from, n)
#else
/* EVA specific functions */
-extern size_t __copy_user_inatomic_eva(void *__to, const void *__from,
- size_t __n);
extern size_t __copy_from_user_eva(void *__to, const void *__from,
size_t __n);
extern size_t __copy_to_user_eva(void *__to, const void *__from,
size_t __n);
extern size_t __copy_in_user_eva(void *__to, const void *__from, size_t __n);
-#define __invoke_copy_from_user_eva_generic(to, from, n, func_ptr) \
-({ \
- register void *__cu_to_r __asm__("$4"); \
- register const void __user *__cu_from_r __asm__("$5"); \
- register long __cu_len_r __asm__("$6"); \
- \
- __cu_to_r = (to); \
- __cu_from_r = (from); \
- __cu_len_r = (n); \
- __asm__ __volatile__( \
- ".set\tnoreorder\n\t" \
- __MODULE_JAL(func_ptr) \
- ".set\tnoat\n\t" \
- __UA_ADDU "\t$1, %1, %2\n\t" \
- ".set\tat\n\t" \
- ".set\treorder" \
- : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \
- : \
- : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \
- DADDI_SCRATCH, "memory"); \
- __cu_len_r; \
-})
-
-#define __invoke_copy_to_user_eva_generic(to, from, n, func_ptr) \
-({ \
- register void *__cu_to_r __asm__("$4"); \
- register const void __user *__cu_from_r __asm__("$5"); \
- register long __cu_len_r __asm__("$6"); \
- \
- __cu_to_r = (to); \
- __cu_from_r = (from); \
- __cu_len_r = (n); \
- __asm__ __volatile__( \
- __MODULE_JAL(func_ptr) \
- : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \
- : \
- : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \
- DADDI_SCRATCH, "memory"); \
- __cu_len_r; \
-})
-
/*
* Source or destination address is in userland. We need to go through
* the TLB
*/
#define __invoke_copy_from_user(to, from, n) \
- __invoke_copy_from_user_eva_generic(to, from, n, __copy_from_user_eva)
-
-#define __invoke_copy_from_user_inatomic(to, from, n) \
- __invoke_copy_from_user_eva_generic(to, from, n, \
- __copy_user_inatomic_eva)
+ __invoke_copy_from(__copy_from_user_eva, to, from, n)
#define __invoke_copy_to_user(to, from, n) \
- __invoke_copy_to_user_eva_generic(to, from, n, __copy_to_user_eva)
+ __invoke_copy_to(__copy_to_user_eva, to, from, n)
#define ___invoke_copy_in_user(to, from, n) \
- __invoke_copy_from_user_eva_generic(to, from, n, __copy_in_user_eva)
-
-/*
- * Source or destination address in the kernel. We are not going through
- * the TLB
- */
-#define __invoke_copy_from_kernel(to, from, n) \
- __invoke_copy_from_user_eva_generic(to, from, n, __copy_user)
-
-#define __invoke_copy_from_kernel_inatomic(to, from, n) \
- __invoke_copy_from_user_eva_generic(to, from, n, __copy_user_inatomic)
-
-#define __invoke_copy_to_kernel(to, from, n) \
- __invoke_copy_to_user_eva_generic(to, from, n, __copy_user)
-
-#define ___invoke_copy_in_kernel(to, from, n) \
- __invoke_copy_from_user_eva_generic(to, from, n, __copy_user)
+ __invoke_copy_from(__copy_in_user_eva, to, from, n)
#endif /* CONFIG_EVA */
-/*
- * __copy_from_user: - Copy a block of data from user space, with less checking.
- * @to: Destination address, in kernel space.
- * @from: Source address, in user space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from user space to kernel space. Caller must check
- * the specified block with access_ok() before calling this function.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- *
- * If some data could not be copied, this function will pad the copied
- * data to the requested size using zero bytes.
- */
-#define __copy_from_user(to, from, n) \
-({ \
- void *__cu_to; \
- const void __user *__cu_from; \
- long __cu_len; \
- \
- __cu_to = (to); \
- __cu_from = (from); \
- __cu_len = (n); \
- \
- check_object_size(__cu_to, __cu_len, false); \
- \
- if (eva_kernel_access()) { \
- __cu_len = __invoke_copy_from_kernel(__cu_to, \
- __cu_from, \
- __cu_len); \
- } else { \
- might_fault(); \
- __cu_len = __invoke_copy_from_user(__cu_to, __cu_from, \
- __cu_len); \
- } \
- __cu_len; \
-})
+static inline unsigned long
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
+{
+ if (eva_kernel_access())
+ return __invoke_copy_to_kernel(to, from, n);
+ else
+ return __invoke_copy_to_user(to, from, n);
+}
-/*
- * copy_from_user: - Copy a block of data from user space.
- * @to: Destination address, in kernel space.
- * @from: Source address, in user space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from user space to kernel space.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- *
- * If some data could not be copied, this function will pad the copied
- * data to the requested size using zero bytes.
- */
-#define copy_from_user(to, from, n) \
-({ \
- void *__cu_to; \
- const void __user *__cu_from; \
- long __cu_len; \
- \
- __cu_to = (to); \
- __cu_from = (from); \
- __cu_len = (n); \
- \
- check_object_size(__cu_to, __cu_len, false); \
- \
- if (eva_kernel_access()) { \
- __cu_len = __invoke_copy_from_kernel(__cu_to, \
- __cu_from, \
- __cu_len); \
- } else { \
- if (access_ok(VERIFY_READ, __cu_from, __cu_len)) { \
- might_fault(); \
- __cu_len = __invoke_copy_from_user(__cu_to, \
- __cu_from, \
- __cu_len); \
- } else { \
- memset(__cu_to, 0, __cu_len); \
- } \
- } \
- __cu_len; \
-})
+static inline unsigned long
+raw_copy_from_user(void *to, const void __user *from, unsigned long n)
+{
+ if (eva_kernel_access())
+ return __invoke_copy_from_kernel(to, from, n);
+ else
+ return __invoke_copy_from_user(to, from, n);
+}
-#define __copy_in_user(to, from, n) \
-({ \
- void __user *__cu_to; \
- const void __user *__cu_from; \
- long __cu_len; \
- \
- __cu_to = (to); \
- __cu_from = (from); \
- __cu_len = (n); \
- if (eva_kernel_access()) { \
- __cu_len = ___invoke_copy_in_kernel(__cu_to, __cu_from, \
- __cu_len); \
- } else { \
- might_fault(); \
- __cu_len = ___invoke_copy_in_user(__cu_to, __cu_from, \
- __cu_len); \
- } \
- __cu_len; \
-})
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
-#define copy_in_user(to, from, n) \
-({ \
- void __user *__cu_to; \
- const void __user *__cu_from; \
- long __cu_len; \
- \
- __cu_to = (to); \
- __cu_from = (from); \
- __cu_len = (n); \
- if (eva_kernel_access()) { \
- __cu_len = ___invoke_copy_in_kernel(__cu_to,__cu_from, \
- __cu_len); \
- } else { \
- if (likely(access_ok(VERIFY_READ, __cu_from, __cu_len) &&\
- access_ok(VERIFY_WRITE, __cu_to, __cu_len))) {\
- might_fault(); \
- __cu_len = ___invoke_copy_in_user(__cu_to, \
- __cu_from, \
- __cu_len); \
- } \
- } \
- __cu_len; \
-})
+static inline unsigned long
+raw_copy_in_user(void __user*to, const void __user *from, unsigned long n)
+{
+ if (eva_kernel_access())
+ return ___invoke_copy_in_kernel(to, from, n);
+ else
+ return ___invoke_copy_in_user(to, from, n);
+}
extern __kernel_size_t __bzero_kernel(void __user *addr, __kernel_size_t size);
extern __kernel_size_t __bzero(void __user *addr, __kernel_size_t size);
diff --git a/arch/mips/kernel/mips-r2-to-r6-emul.c b/arch/mips/kernel/mips-r2-to-r6-emul.c
index d8f1cf1ec370..550e7d03090a 100644
--- a/arch/mips/kernel/mips-r2-to-r6-emul.c
+++ b/arch/mips/kernel/mips-r2-to-r6-emul.c
@@ -1200,7 +1200,7 @@ fpu_emul:
case lwl_op:
rt = regs->regs[MIPSInst_RT(inst)];
vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst);
- if (!access_ok(VERIFY_READ, vaddr, 4)) {
+ if (!access_ok(VERIFY_READ, (void __user *)vaddr, 4)) {
current->thread.cp0_baduaddr = vaddr;
err = SIGSEGV;
break;
@@ -1273,7 +1273,7 @@ fpu_emul:
case lwr_op:
rt = regs->regs[MIPSInst_RT(inst)];
vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst);
- if (!access_ok(VERIFY_READ, vaddr, 4)) {
+ if (!access_ok(VERIFY_READ, (void __user *)vaddr, 4)) {
current->thread.cp0_baduaddr = vaddr;
err = SIGSEGV;
break;
@@ -1347,7 +1347,7 @@ fpu_emul:
case swl_op:
rt = regs->regs[MIPSInst_RT(inst)];
vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst);
- if (!access_ok(VERIFY_WRITE, vaddr, 4)) {
+ if (!access_ok(VERIFY_WRITE, (void __user *)vaddr, 4)) {
current->thread.cp0_baduaddr = vaddr;
err = SIGSEGV;
break;
@@ -1417,7 +1417,7 @@ fpu_emul:
case swr_op:
rt = regs->regs[MIPSInst_RT(inst)];
vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst);
- if (!access_ok(VERIFY_WRITE, vaddr, 4)) {
+ if (!access_ok(VERIFY_WRITE, (void __user *)vaddr, 4)) {
current->thread.cp0_baduaddr = vaddr;
err = SIGSEGV;
break;
@@ -1492,7 +1492,7 @@ fpu_emul:
rt = regs->regs[MIPSInst_RT(inst)];
vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst);
- if (!access_ok(VERIFY_READ, vaddr, 8)) {
+ if (!access_ok(VERIFY_READ, (void __user *)vaddr, 8)) {
current->thread.cp0_baduaddr = vaddr;
err = SIGSEGV;
break;
@@ -1611,7 +1611,7 @@ fpu_emul:
rt = regs->regs[MIPSInst_RT(inst)];
vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst);
- if (!access_ok(VERIFY_READ, vaddr, 8)) {
+ if (!access_ok(VERIFY_READ, (void __user *)vaddr, 8)) {
current->thread.cp0_baduaddr = vaddr;
err = SIGSEGV;
break;
@@ -1730,7 +1730,7 @@ fpu_emul:
rt = regs->regs[MIPSInst_RT(inst)];
vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst);
- if (!access_ok(VERIFY_WRITE, vaddr, 8)) {
+ if (!access_ok(VERIFY_WRITE, (void __user *)vaddr, 8)) {
current->thread.cp0_baduaddr = vaddr;
err = SIGSEGV;
break;
@@ -1848,7 +1848,7 @@ fpu_emul:
rt = regs->regs[MIPSInst_RT(inst)];
vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst);
- if (!access_ok(VERIFY_WRITE, vaddr, 8)) {
+ if (!access_ok(VERIFY_WRITE, (void __user *)vaddr, 8)) {
current->thread.cp0_baduaddr = vaddr;
err = SIGSEGV;
break;
@@ -1965,7 +1965,7 @@ fpu_emul:
err = SIGBUS;
break;
}
- if (!access_ok(VERIFY_READ, vaddr, 4)) {
+ if (!access_ok(VERIFY_READ, (void __user *)vaddr, 4)) {
current->thread.cp0_baduaddr = vaddr;
err = SIGBUS;
break;
@@ -2021,7 +2021,7 @@ fpu_emul:
err = SIGBUS;
break;
}
- if (!access_ok(VERIFY_WRITE, vaddr, 4)) {
+ if (!access_ok(VERIFY_WRITE, (void __user *)vaddr, 4)) {
current->thread.cp0_baduaddr = vaddr;
err = SIGBUS;
break;
@@ -2084,7 +2084,7 @@ fpu_emul:
err = SIGBUS;
break;
}
- if (!access_ok(VERIFY_READ, vaddr, 8)) {
+ if (!access_ok(VERIFY_READ, (void __user *)vaddr, 8)) {
current->thread.cp0_baduaddr = vaddr;
err = SIGBUS;
break;
@@ -2145,7 +2145,7 @@ fpu_emul:
err = SIGBUS;
break;
}
- if (!access_ok(VERIFY_WRITE, vaddr, 8)) {
+ if (!access_ok(VERIFY_WRITE, (void __user *)vaddr, 8)) {
current->thread.cp0_baduaddr = vaddr;
err = SIGBUS;
break;
diff --git a/arch/mips/kernel/syscall.c b/arch/mips/kernel/syscall.c
index f1d17ece4181..1dfa7f5796c7 100644
--- a/arch/mips/kernel/syscall.c
+++ b/arch/mips/kernel/syscall.c
@@ -98,7 +98,7 @@ static inline int mips_atomic_set(unsigned long addr, unsigned long new)
if (unlikely(addr & 3))
return -EINVAL;
- if (unlikely(!access_ok(VERIFY_WRITE, addr, 4)))
+ if (unlikely(!access_ok(VERIFY_WRITE, (const void __user *)addr, 4)))
return -EINVAL;
if (cpu_has_llsc && R10000_LLSC_WAR) {
diff --git a/arch/mips/lib/memcpy.S b/arch/mips/lib/memcpy.S
index c3031f18c572..3114a2ed1f4e 100644
--- a/arch/mips/lib/memcpy.S
+++ b/arch/mips/lib/memcpy.S
@@ -562,39 +562,9 @@
LOADK t0, THREAD_BUADDR(t0) # t0 is just past last good address
nop
SUB len, AT, t0 # len number of uncopied bytes
- bnez t6, .Ldone\@ /* Skip the zeroing part if inatomic */
- /*
- * Here's where we rely on src and dst being incremented in tandem,
- * See (3) above.
- * dst += (fault addr - src) to put dst at first byte to clear
- */
- ADD dst, t0 # compute start address in a1
- SUB dst, src
- /*
- * Clear len bytes starting at dst. Can't call __bzero because it
- * might modify len. An inefficient loop for these rare times...
- */
- .set reorder /* DADDI_WAR */
- SUB src, len, 1
- beqz len, .Ldone\@
- .set noreorder
-1: sb zero, 0(dst)
- ADD dst, dst, 1
-#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
- bnez src, 1b
- SUB src, src, 1
-#else
- .set push
- .set noat
- li v1, 1
- bnez src, 1b
- SUB src, src, v1
- .set pop
-#endif
jr ra
nop
-
#define SEXC(n) \
.set reorder; /* DADDI_WAR */ \
.Ls_exc_p ## n ## u\@: \
@@ -673,15 +643,6 @@ LEAF(__rmemcpy) /* a0=dst a1=src a2=len */
END(__rmemcpy)
/*
- * t6 is used as a flag to note inatomic mode.
- */
-LEAF(__copy_user_inatomic)
-EXPORT_SYMBOL(__copy_user_inatomic)
- b __copy_user_common
- li t6, 1
- END(__copy_user_inatomic)
-
-/*
* A combined memcpy/__copy_user
* __copy_user sets len to 0 for success; else to an upper bound of
* the number of uncopied bytes.
@@ -694,8 +655,6 @@ EXPORT_SYMBOL(memcpy)
.L__memcpy:
FEXPORT(__copy_user)
EXPORT_SYMBOL(__copy_user)
- li t6, 0 /* not inatomic */
-__copy_user_common:
/* Legacy Mode, user <-> user */
__BUILD_COPY_USER LEGACY_MODE USEROP USEROP
@@ -708,20 +667,12 @@ __copy_user_common:
* space
*/
-LEAF(__copy_user_inatomic_eva)
-EXPORT_SYMBOL(__copy_user_inatomic_eva)
- b __copy_from_user_common
- li t6, 1
- END(__copy_user_inatomic_eva)
-
/*
* __copy_from_user (EVA)
*/
LEAF(__copy_from_user_eva)
EXPORT_SYMBOL(__copy_from_user_eva)
- li t6, 0 /* not inatomic */
-__copy_from_user_common:
__BUILD_COPY_USER EVA_MODE USEROP KERNELOP
END(__copy_from_user_eva)
diff --git a/arch/mips/oprofile/backtrace.c b/arch/mips/oprofile/backtrace.c
index 5e645c9a3162..16ace558cd9d 100644
--- a/arch/mips/oprofile/backtrace.c
+++ b/arch/mips/oprofile/backtrace.c
@@ -18,7 +18,7 @@ struct stackframe {
static inline int get_mem(unsigned long addr, unsigned long *result)
{
unsigned long *address = (unsigned long *) addr;
- if (!access_ok(VERIFY_READ, addr, sizeof(unsigned long)))
+ if (!access_ok(VERIFY_READ, address, sizeof(unsigned long)))
return -1;
if (__copy_from_user_inatomic(result, address, sizeof(unsigned long)))
return -3;
diff --git a/arch/mn10300/Kconfig b/arch/mn10300/Kconfig
index 38e3494bfb63..a96f3dcb0119 100644
--- a/arch/mn10300/Kconfig
+++ b/arch/mn10300/Kconfig
@@ -16,6 +16,7 @@ config MN10300
select OLD_SIGACTION
select HAVE_DEBUG_STACKOVERFLOW
select ARCH_NO_COHERENT_DMA_MMAP
+ select ARCH_HAS_RAW_COPY_USER
config AM33_2
def_bool n
diff --git a/arch/mn10300/include/asm/Kbuild b/arch/mn10300/include/asm/Kbuild
index 97f64c723a0c..ed810e7206e8 100644
--- a/arch/mn10300/include/asm/Kbuild
+++ b/arch/mn10300/include/asm/Kbuild
@@ -2,6 +2,7 @@
generic-y += barrier.h
generic-y += clkdev.h
generic-y += exec.h
+generic-y += extable.h
generic-y += irq_work.h
generic-y += mcs_spinlock.h
generic-y += mm-arch-hooks.h
diff --git a/arch/mn10300/include/asm/uaccess.h b/arch/mn10300/include/asm/uaccess.h
index 2da7b0fed4aa..c6966474827f 100644
--- a/arch/mn10300/include/asm/uaccess.h
+++ b/arch/mn10300/include/asm/uaccess.h
@@ -66,26 +66,7 @@ static inline int ___range_ok(unsigned long addr, unsigned int size)
#define access_ok(type, addr, size) (__range_ok((addr), (size)) == 0)
#define __access_ok(addr, size) (__range_ok((addr), (size)) == 0)
-/*
- * The exception table consists of pairs of addresses: the first is the
- * address of an instruction that is allowed to fault, and the second is
- * the address at which the program should continue. No registers are
- * modified, so it is entirely up to the continuation code to figure out
- * what to do.
- *
- * All the routines below use bits of fixup code that are out of line
- * with the main instruction path. This means when everything is well,
- * we don't even have to jump over them. Further, they do not intrude
- * on our cache or tlb entries.
- */
-
-struct exception_table_entry
-{
- unsigned long insn, fixup;
-};
-
-/* Returns 0 if exception not found and fixup otherwise. */
-extern int fixup_exception(struct pt_regs *regs);
+#include <asm/extable.h>
#define put_user(x, ptr) __put_user_check((x), (ptr), sizeof(*(ptr)))
#define get_user(x, ptr) __get_user_check((x), (ptr), sizeof(*(ptr)))
@@ -294,170 +275,19 @@ do { \
} \
} while (0)
-#define __copy_user_zeroing(to, from, size) \
-do { \
- if (size) { \
- void *__to = to; \
- const void *__from = from; \
- int w; \
- asm volatile( \
- "0: movbu (%0),%3;\n" \
- "1: movbu %3,(%1);\n" \
- " inc %0;\n" \
- " inc %1;\n" \
- " add -1,%2;\n" \
- " bne 0b;\n" \
- "2:\n" \
- " .section .fixup,\"ax\"\n" \
- "3:\n" \
- " mov %2,%0\n" \
- " clr %3\n" \
- "4: movbu %3,(%1);\n" \
- " inc %1;\n" \
- " add -1,%2;\n" \
- " bne 4b;\n" \
- " mov %0,%2\n" \
- " jmp 2b\n" \
- " .previous\n" \
- " .section __ex_table,\"a\"\n" \
- " .balign 4\n" \
- " .long 0b,3b\n" \
- " .long 1b,3b\n" \
- " .previous\n" \
- : "=a"(__from), "=a"(__to), "=r"(size), "=&r"(w)\
- : "0"(__from), "1"(__to), "2"(size) \
- : "cc", "memory"); \
- } \
-} while (0)
-
-/* We let the __ versions of copy_from/to_user inline, because they're often
- * used in fast paths and have only a small space overhead.
- */
-static inline
-unsigned long __generic_copy_from_user_nocheck(void *to, const void *from,
- unsigned long n)
-{
- __copy_user_zeroing(to, from, n);
- return n;
-}
-
-static inline
-unsigned long __generic_copy_to_user_nocheck(void *to, const void *from,
- unsigned long n)
+static inline unsigned long
+raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
__copy_user(to, from, n);
return n;
}
-
-#if 0
-#error "don't use - these macros don't increment to & from pointers"
-/* Optimize just a little bit when we know the size of the move. */
-#define __constant_copy_user(to, from, size) \
-do { \
- asm volatile( \
- " mov %0,a0;\n" \
- "0: movbu (%1),d3;\n" \
- "1: movbu d3,(%2);\n" \
- " add -1,a0;\n" \
- " bne 0b;\n" \
- "2:;" \
- ".section .fixup,\"ax\"\n" \
- "3: jmp 2b\n" \
- ".previous\n" \
- ".section __ex_table,\"a\"\n" \
- " .balign 4\n" \
- " .long 0b,3b\n" \
- " .long 1b,3b\n" \
- ".previous" \
- : \
- : "d"(size), "d"(to), "d"(from) \
- : "d3", "a0"); \
-} while (0)
-
-/* Optimize just a little bit when we know the size of the move. */
-#define __constant_copy_user_zeroing(to, from, size) \
-do { \
- asm volatile( \
- " mov %0,a0;\n" \
- "0: movbu (%1),d3;\n" \
- "1: movbu d3,(%2);\n" \
- " add -1,a0;\n" \
- " bne 0b;\n" \
- "2:;" \
- ".section .fixup,\"ax\"\n" \
- "3: jmp 2b\n" \
- ".previous\n" \
- ".section __ex_table,\"a\"\n" \
- " .balign 4\n" \
- " .long 0b,3b\n" \
- " .long 1b,3b\n" \
- ".previous" \
- : \
- : "d"(size), "d"(to), "d"(from) \
- : "d3", "a0"); \
-} while (0)
-
-static inline
-unsigned long __constant_copy_to_user(void *to, const void *from,
- unsigned long n)
-{
- if (access_ok(VERIFY_WRITE, to, n))
- __constant_copy_user(to, from, n);
- return n;
-}
-
-static inline
-unsigned long __constant_copy_from_user(void *to, const void *from,
- unsigned long n)
+static inline unsigned long
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
- if (access_ok(VERIFY_READ, from, n))
- __constant_copy_user_zeroing(to, from, n);
- return n;
-}
-
-static inline
-unsigned long __constant_copy_to_user_nocheck(void *to, const void *from,
- unsigned long n)
-{
- __constant_copy_user(to, from, n);
- return n;
-}
-
-static inline
-unsigned long __constant_copy_from_user_nocheck(void *to, const void *from,
- unsigned long n)
-{
- __constant_copy_user_zeroing(to, from, n);
+ __copy_user(to, from, n);
return n;
}
-#endif
-
-extern unsigned long __generic_copy_to_user(void __user *, const void *,
- unsigned long);
-extern unsigned long __generic_copy_from_user(void *, const void __user *,
- unsigned long);
-
-#define __copy_to_user_inatomic(to, from, n) \
- __generic_copy_to_user_nocheck((to), (from), (n))
-#define __copy_from_user_inatomic(to, from, n) \
- __generic_copy_from_user_nocheck((to), (from), (n))
-
-#define __copy_to_user(to, from, n) \
-({ \
- might_fault(); \
- __copy_to_user_inatomic((to), (from), (n)); \
-})
-
-#define __copy_from_user(to, from, n) \
-({ \
- might_fault(); \
- __copy_from_user_inatomic((to), (from), (n)); \
-})
-
-
-#define copy_to_user(to, from, n) __generic_copy_to_user((to), (from), (n))
-#define copy_from_user(to, from, n) __generic_copy_from_user((to), (from), (n))
extern long strncpy_from_user(char *dst, const char __user *src, long count);
extern long __strncpy_from_user(char *dst, const char __user *src, long count);
diff --git a/arch/mn10300/kernel/mn10300_ksyms.c b/arch/mn10300/kernel/mn10300_ksyms.c
index ec6c4f8f93a6..5e9f919635f0 100644
--- a/arch/mn10300/kernel/mn10300_ksyms.c
+++ b/arch/mn10300/kernel/mn10300_ksyms.c
@@ -26,8 +26,6 @@ EXPORT_SYMBOL(strncpy_from_user);
EXPORT_SYMBOL(__strncpy_from_user);
EXPORT_SYMBOL(clear_user);
EXPORT_SYMBOL(__clear_user);
-EXPORT_SYMBOL(__generic_copy_from_user);
-EXPORT_SYMBOL(__generic_copy_to_user);
EXPORT_SYMBOL(strnlen_user);
extern u64 __ashrdi3(u64, unsigned);
diff --git a/arch/mn10300/lib/usercopy.c b/arch/mn10300/lib/usercopy.c
index ce8899e5e171..cece1799cc32 100644
--- a/arch/mn10300/lib/usercopy.c
+++ b/arch/mn10300/lib/usercopy.c
@@ -11,24 +11,6 @@
*/
#include <linux/uaccess.h>
-unsigned long
-__generic_copy_to_user(void *to, const void *from, unsigned long n)
-{
- if (access_ok(VERIFY_WRITE, to, n))
- __copy_user(to, from, n);
- return n;
-}
-
-unsigned long
-__generic_copy_from_user(void *to, const void *from, unsigned long n)
-{
- if (access_ok(VERIFY_READ, from, n))
- __copy_user_zeroing(to, from, n);
- else
- memset(to, 0, n);
- return n;
-}
-
/*
* Copy a null terminated string from userspace.
*/
diff --git a/arch/nios2/Kconfig b/arch/nios2/Kconfig
index 51a56c8b04b4..45b4727e3136 100644
--- a/arch/nios2/Kconfig
+++ b/arch/nios2/Kconfig
@@ -16,6 +16,7 @@ config NIOS2
select SPARSE_IRQ
select USB_ARCH_HAS_HCD if USB_SUPPORT
select CPU_NO_EFFICIENT_FFS
+ select ARCH_HAS_RAW_COPY_USER
config GENERIC_CSUM
def_bool y
diff --git a/arch/nios2/include/asm/Kbuild b/arch/nios2/include/asm/Kbuild
index aaa3c218b56c..87e70f2b463f 100644
--- a/arch/nios2/include/asm/Kbuild
+++ b/arch/nios2/include/asm/Kbuild
@@ -13,6 +13,7 @@ generic-y += dma.h
generic-y += emergency-restart.h
generic-y += errno.h
generic-y += exec.h
+generic-y += extable.h
generic-y += fb.h
generic-y += fcntl.h
generic-y += ftrace.h
diff --git a/arch/nios2/include/asm/uaccess.h b/arch/nios2/include/asm/uaccess.h
index 198bbf15f644..727bd9504899 100644
--- a/arch/nios2/include/asm/uaccess.h
+++ b/arch/nios2/include/asm/uaccess.h
@@ -17,24 +17,7 @@
#include <asm/page.h>
-/*
- * The exception table consists of pairs of addresses: the first is the
- * address of an instruction that is allowed to fault, and the second is
- * the address at which the program should continue. No registers are
- * modified, so it is entirely up to the continuation code to figure out
- * what to do.
- *
- * All the routines below use bits of fixup code that are out of line
- * with the main instruction path. This means when everything is well,
- * we don't even have to jump over them. Further, they do not intrude
- * on our cache or tlb entries.
- */
-struct exception_table_entry {
- unsigned long insn;
- unsigned long fixup;
-};
-
-extern int fixup_exception(struct pt_regs *regs);
+#include <asm/extable.h>
/*
* Segment stuff
@@ -90,36 +73,17 @@ static inline unsigned long __must_check clear_user(void __user *to,
return __clear_user(to, n);
}
-extern long __copy_from_user(void *to, const void __user *from,
- unsigned long n);
-extern long __copy_to_user(void __user *to, const void *from, unsigned long n);
-
-static inline long copy_from_user(void *to, const void __user *from,
- unsigned long n)
-{
- unsigned long res = n;
- if (access_ok(VERIFY_READ, from, n))
- res = __copy_from_user(to, from, n);
- if (unlikely(res))
- memset(to + (n - res), 0, res);
- return res;
-}
-
-static inline long copy_to_user(void __user *to, const void *from,
- unsigned long n)
-{
- if (!access_ok(VERIFY_WRITE, to, n))
- return n;
- return __copy_to_user(to, from, n);
-}
+extern unsigned long
+raw_copy_from_user(void *to, const void __user *from, unsigned long n);
+extern unsigned long
+raw_copy_to_user(void __user *to, const void *from, unsigned long n);
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
extern long strncpy_from_user(char *__to, const char __user *__from,
long __len);
extern long strnlen_user(const char __user *s, long n);
-#define __copy_from_user_inatomic __copy_from_user
-#define __copy_to_user_inatomic __copy_to_user
-
/* Optimized macros */
#define __get_user_asm(val, insn, addr, err) \
{ \
diff --git a/arch/nios2/mm/uaccess.c b/arch/nios2/mm/uaccess.c
index 7663e156ff4f..804983317766 100644
--- a/arch/nios2/mm/uaccess.c
+++ b/arch/nios2/mm/uaccess.c
@@ -10,9 +10,9 @@
#include <linux/export.h>
#include <linux/uaccess.h>
-asm(".global __copy_from_user\n"
- " .type __copy_from_user, @function\n"
- "__copy_from_user:\n"
+asm(".global raw_copy_from_user\n"
+ " .type raw_copy_from_user, @function\n"
+ "raw_copy_from_user:\n"
" movi r2,7\n"
" mov r3,r4\n"
" bge r2,r6,1f\n"
@@ -65,12 +65,12 @@ asm(".global __copy_from_user\n"
".word 7b,13b\n"
".previous\n"
);
-EXPORT_SYMBOL(__copy_from_user);
+EXPORT_SYMBOL(raw_copy_from_user);
asm(
- " .global __copy_to_user\n"
- " .type __copy_to_user, @function\n"
- "__copy_to_user:\n"
+ " .global raw_copy_to_user\n"
+ " .type raw_copy_to_user, @function\n"
+ "raw_copy_to_user:\n"
" movi r2,7\n"
" mov r3,r4\n"
" bge r2,r6,1f\n"
@@ -127,7 +127,7 @@ asm(
".word 11b,13b\n"
".word 12b,13b\n"
".previous\n");
-EXPORT_SYMBOL(__copy_to_user);
+EXPORT_SYMBOL(raw_copy_to_user);
long strncpy_from_user(char *__to, const char __user *__from, long __len)
{
diff --git a/arch/openrisc/Kconfig b/arch/openrisc/Kconfig
index 1e95920b0737..38954181fa96 100644
--- a/arch/openrisc/Kconfig
+++ b/arch/openrisc/Kconfig
@@ -28,6 +28,7 @@ config OPENRISC
select OR1K_PIC
select CPU_NO_EFFICIENT_FFS if !OPENRISC_HAVE_INST_FF1
select NO_BOOTMEM
+ select ARCH_HAS_RAW_COPY_USER
config MMU
def_bool y
diff --git a/arch/openrisc/include/asm/Kbuild b/arch/openrisc/include/asm/Kbuild
index fb01873a5aad..df8e2f7bc7dd 100644
--- a/arch/openrisc/include/asm/Kbuild
+++ b/arch/openrisc/include/asm/Kbuild
@@ -16,6 +16,7 @@ generic-y += dma.h
generic-y += emergency-restart.h
generic-y += errno.h
generic-y += exec.h
+generic-y += extable.h
generic-y += fb.h
generic-y += fcntl.h
generic-y += ftrace.h
diff --git a/arch/openrisc/include/asm/uaccess.h b/arch/openrisc/include/asm/uaccess.h
index d6e7483b9d17..a557a7cd0232 100644
--- a/arch/openrisc/include/asm/uaccess.h
+++ b/arch/openrisc/include/asm/uaccess.h
@@ -25,6 +25,7 @@
#include <linux/prefetch.h>
#include <linux/string.h>
#include <asm/page.h>
+#include <asm/extable.h>
/*
* The fs value determines whether argument validity checking should be
@@ -61,23 +62,6 @@
__range_ok((unsigned long)addr, (unsigned long)size)
/*
- * The exception table consists of pairs of addresses: the first is the
- * address of an instruction that is allowed to fault, and the second is
- * the address at which the program should continue. No registers are
- * modified, so it is entirely up to the continuation code to figure out
- * what to do.
- *
- * All the routines below use bits of fixup code that are out of line
- * with the main instruction path. This means when everything is well,
- * we don't even have to jump over them. Further, they do not intrude
- * on our cache or tlb entries.
- */
-
-struct exception_table_entry {
- unsigned long insn, fixup;
-};
-
-/*
* These are the main single-value transfer routines. They automatically
* use the right size if we just have the right pointer type.
*
@@ -252,34 +236,18 @@ do { \
extern unsigned long __must_check
__copy_tofrom_user(void *to, const void *from, unsigned long size);
-
-#define __copy_from_user(to, from, size) \
- __copy_tofrom_user(to, from, size)
-#define __copy_to_user(to, from, size) \
- __copy_tofrom_user(to, from, size)
-
-#define __copy_to_user_inatomic __copy_to_user
-#define __copy_from_user_inatomic __copy_from_user
-
static inline unsigned long
-copy_from_user(void *to, const void *from, unsigned long n)
+raw_copy_from_user(void *to, const void __user *from, unsigned long size)
{
- unsigned long res = n;
-
- if (likely(access_ok(VERIFY_READ, from, n)))
- res = __copy_tofrom_user(to, from, n);
- if (unlikely(res))
- memset(to + (n - res), 0, res);
- return res;
+ return __copy_tofrom_user(to, (__force const void *)from, size);
}
-
static inline unsigned long
-copy_to_user(void *to, const void *from, unsigned long n)
+raw_copy_to_user(void *to, const void __user *from, unsigned long size)
{
- if (likely(access_ok(VERIFY_WRITE, to, n)))
- n = __copy_tofrom_user(to, from, n);
- return n;
+ return __copy_tofrom_user((__force void *)to, from, size);
}
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
extern unsigned long __clear_user(void *addr, unsigned long size);
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 97a8bc8a095c..839f08887269 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -87,6 +87,7 @@ config PPC
select ARCH_HAS_DMA_SET_COHERENT_MASK
select ARCH_HAS_ELF_RANDOMIZE
select ARCH_HAS_GCOV_PROFILE_ALL
+ select ARCH_HAS_RAW_COPY_USER
select ARCH_HAS_SCALED_CPUTIME if VIRT_CPU_ACCOUNTING_NATIVE
select ARCH_HAS_SG_CHAIN
select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
diff --git a/arch/powerpc/include/asm/extable.h b/arch/powerpc/include/asm/extable.h
new file mode 100644
index 000000000000..07cc45cd86d9
--- /dev/null
+++ b/arch/powerpc/include/asm/extable.h
@@ -0,0 +1,29 @@
+#ifndef _ARCH_POWERPC_EXTABLE_H
+#define _ARCH_POWERPC_EXTABLE_H
+
+/*
+ * The exception table consists of pairs of relative addresses: the first is
+ * the address of an instruction that is allowed to fault, and the second is
+ * the address at which the program should continue. No registers are
+ * modified, so it is entirely up to the continuation code to figure out what
+ * to do.
+ *
+ * All the routines below use bits of fixup code that are out of line with the
+ * main instruction path. This means when everything is well, we don't even
+ * have to jump over them. Further, they do not intrude on our cache or tlb
+ * entries.
+ */
+
+#define ARCH_HAS_RELATIVE_EXTABLE
+
+struct exception_table_entry {
+ int insn;
+ int fixup;
+};
+
+static inline unsigned long extable_fixup(const struct exception_table_entry *x)
+{
+ return (unsigned long)&x->fixup + x->fixup;
+}
+
+#endif
diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h
index 3904040a3542..5c0d8a8cdae5 100644
--- a/arch/powerpc/include/asm/uaccess.h
+++ b/arch/powerpc/include/asm/uaccess.h
@@ -5,6 +5,7 @@
#include <asm/ppc_asm.h>
#include <asm/processor.h>
#include <asm/page.h>
+#include <asm/extable.h>
/*
* The fs value determines whether argument validity checking should be
@@ -56,31 +57,6 @@
__access_ok((__force unsigned long)(addr), (size), get_fs()))
/*
- * The exception table consists of pairs of relative addresses: the first is
- * the address of an instruction that is allowed to fault, and the second is
- * the address at which the program should continue. No registers are
- * modified, so it is entirely up to the continuation code to figure out what
- * to do.
- *
- * All the routines below use bits of fixup code that are out of line with the
- * main instruction path. This means when everything is well, we don't even
- * have to jump over them. Further, they do not intrude on our cache or tlb
- * entries.
- */
-
-#define ARCH_HAS_RELATIVE_EXTABLE
-
-struct exception_table_entry {
- int insn;
- int fixup;
-};
-
-static inline unsigned long extable_fixup(const struct exception_table_entry *x)
-{
- return (unsigned long)&x->fixup + x->fixup;
-}
-
-/*
* These are the main single-value transfer routines. They automatically
* use the right size if we just have the right pointer type.
*
@@ -293,42 +269,19 @@ extern unsigned long __copy_tofrom_user(void __user *to,
#ifndef __powerpc64__
-static inline unsigned long copy_from_user(void *to,
- const void __user *from, unsigned long n)
-{
- if (likely(access_ok(VERIFY_READ, from, n))) {
- check_object_size(to, n, false);
- return __copy_tofrom_user((__force void __user *)to, from, n);
- }
- memset(to, 0, n);
- return n;
-}
-
-static inline unsigned long copy_to_user(void __user *to,
- const void *from, unsigned long n)
-{
- if (access_ok(VERIFY_WRITE, to, n)) {
- check_object_size(from, n, true);
- return __copy_tofrom_user(to, (__force void __user *)from, n);
- }
- return n;
-}
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
#else /* __powerpc64__ */
-#define __copy_in_user(to, from, size) \
- __copy_tofrom_user((to), (from), (size))
-
-extern unsigned long copy_from_user(void *to, const void __user *from,
- unsigned long n);
-extern unsigned long copy_to_user(void __user *to, const void *from,
- unsigned long n);
-extern unsigned long copy_in_user(void __user *to, const void __user *from,
- unsigned long n);
-
+static inline unsigned long
+raw_copy_in_user(void __user *to, const void __user *from, unsigned long n)
+{
+ return __copy_tofrom_user(to, from, n);
+}
#endif /* __powerpc64__ */
-static inline unsigned long __copy_from_user_inatomic(void *to,
+static inline unsigned long raw_copy_from_user(void *to,
const void __user *from, unsigned long n)
{
if (__builtin_constant_p(n) && (n <= 8)) {
@@ -352,12 +305,10 @@ static inline unsigned long __copy_from_user_inatomic(void *to,
return 0;
}
- check_object_size(to, n, false);
-
return __copy_tofrom_user((__force void __user *)to, from, n);
}
-static inline unsigned long __copy_to_user_inatomic(void __user *to,
+static inline unsigned long raw_copy_to_user(void __user *to,
const void *from, unsigned long n)
{
if (__builtin_constant_p(n) && (n <= 8)) {
@@ -381,25 +332,9 @@ static inline unsigned long __copy_to_user_inatomic(void __user *to,
return 0;
}
- check_object_size(from, n, true);
-
return __copy_tofrom_user(to, (__force const void __user *)from, n);
}
-static inline unsigned long __copy_from_user(void *to,
- const void __user *from, unsigned long size)
-{
- might_fault();
- return __copy_from_user_inatomic(to, from, size);
-}
-
-static inline unsigned long __copy_to_user(void __user *to,
- const void *from, unsigned long size)
-{
- might_fault();
- return __copy_to_user_inatomic(to, from, size);
-}
-
extern unsigned long __clear_user(void __user *addr, unsigned long size);
static inline unsigned long clear_user(void __user *addr, unsigned long size)
diff --git a/arch/powerpc/lib/Makefile b/arch/powerpc/lib/Makefile
index 2b5e09020cfe..ed7dfce331e0 100644
--- a/arch/powerpc/lib/Makefile
+++ b/arch/powerpc/lib/Makefile
@@ -14,7 +14,7 @@ obj-y += string.o alloc.o crtsavres.o code-patching.o \
obj-$(CONFIG_PPC32) += div64.o copy_32.o
-obj64-y += copypage_64.o copyuser_64.o usercopy_64.o mem_64.o hweight_64.o \
+obj64-y += copypage_64.o copyuser_64.o mem_64.o hweight_64.o \
copyuser_power7.o string_64.o copypage_power7.o memcpy_power7.o \
memcpy_64.o memcmp_64.o
diff --git a/arch/powerpc/lib/copy_32.S b/arch/powerpc/lib/copy_32.S
index ff0d894d7ff9..8aedbb5f4b86 100644
--- a/arch/powerpc/lib/copy_32.S
+++ b/arch/powerpc/lib/copy_32.S
@@ -477,18 +477,6 @@ _GLOBAL(__copy_tofrom_user)
bdnz 130b
/* then clear out the destination: r3 bytes starting at 4(r6) */
132: mfctr r3
- srwi. r0,r3,2
- li r9,0
- mtctr r0
- beq 113f
-112: stwu r9,4(r6)
- bdnz 112b
-113: andi. r0,r3,3
- mtctr r0
- beq 120f
-114: stb r9,4(r6)
- addi r6,r6,1
- bdnz 114b
120: blr
EX_TABLE(30b,108b)
@@ -497,7 +485,5 @@ _GLOBAL(__copy_tofrom_user)
EX_TABLE(41b,111b)
EX_TABLE(130b,132b)
EX_TABLE(131b,120b)
- EX_TABLE(112b,120b)
- EX_TABLE(114b,120b)
EXPORT_SYMBOL(__copy_tofrom_user)
diff --git a/arch/powerpc/lib/copyuser_64.S b/arch/powerpc/lib/copyuser_64.S
index aee6e24e81ab..08da06e1bd72 100644
--- a/arch/powerpc/lib/copyuser_64.S
+++ b/arch/powerpc/lib/copyuser_64.S
@@ -319,32 +319,9 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD)
blr
/*
- * here we have trapped again, need to clear ctr bytes starting at r3
+ * here we have trapped again, amount remaining is in ctr.
*/
-143: mfctr r5
- li r0,0
- mr r4,r3
- mr r3,r5 /* return the number of bytes not copied */
-1: andi. r9,r4,7
- beq 3f
-90: stb r0,0(r4)
- addic. r5,r5,-1
- addi r4,r4,1
- bne 1b
- blr
-3: cmpldi cr1,r5,8
- srdi r9,r5,3
- andi. r5,r5,7
- blt cr1,93f
- mtctr r9
-91: std r0,0(r4)
- addi r4,r4,8
- bdnz 91b
-93: beqlr
- mtctr r5
-92: stb r0,0(r4)
- addi r4,r4,1
- bdnz 92b
+143: mfctr r3
blr
/*
@@ -389,10 +366,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD)
ld r5,-8(r1)
add r6,r6,r5
subf r3,r3,r6 /* #bytes not copied */
-190:
-191:
-192:
- blr /* #bytes not copied in r3 */
+ blr
EX_TABLE(20b,120b)
EX_TABLE(220b,320b)
@@ -451,9 +425,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD)
EX_TABLE(88b,188b)
EX_TABLE(43b,143b)
EX_TABLE(89b,189b)
- EX_TABLE(90b,190b)
- EX_TABLE(91b,191b)
- EX_TABLE(92b,192b)
/*
* Routine to copy a whole page of data, optimized for POWER4.
diff --git a/arch/powerpc/lib/usercopy_64.c b/arch/powerpc/lib/usercopy_64.c
deleted file mode 100644
index 9bd3a3dad78d..000000000000
--- a/arch/powerpc/lib/usercopy_64.c
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Functions which are too large to be inlined.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- */
-#include <linux/module.h>
-#include <linux/uaccess.h>
-
-unsigned long copy_from_user(void *to, const void __user *from, unsigned long n)
-{
- if (likely(access_ok(VERIFY_READ, from, n)))
- n = __copy_from_user(to, from, n);
- else
- memset(to, 0, n);
- return n;
-}
-
-unsigned long copy_to_user(void __user *to, const void *from, unsigned long n)
-{
- if (likely(access_ok(VERIFY_WRITE, to, n)))
- n = __copy_to_user(to, from, n);
- return n;
-}
-
-unsigned long copy_in_user(void __user *to, const void __user *from,
- unsigned long n)
-{
- might_sleep();
- if (likely(access_ok(VERIFY_READ, from, n) &&
- access_ok(VERIFY_WRITE, to, n)))
- n =__copy_tofrom_user(to, from, n);
- return n;
-}
-
-EXPORT_SYMBOL(copy_from_user);
-EXPORT_SYMBOL(copy_to_user);
-EXPORT_SYMBOL(copy_in_user);
-
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index a2dcef0aacc7..0724dbf2d8f8 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -178,6 +178,7 @@ config S390
select ARCH_HAS_SCALED_CPUTIME
select VIRT_TO_BUS
select HAVE_NMI
+ select ARCH_HAS_RAW_COPY_USER
config SCHED_OMIT_FRAME_POINTER
diff --git a/arch/s390/include/asm/extable.h b/arch/s390/include/asm/extable.h
new file mode 100644
index 000000000000..16cfe2d62eeb
--- /dev/null
+++ b/arch/s390/include/asm/extable.h
@@ -0,0 +1,28 @@
+#ifndef __S390_EXTABLE_H
+#define __S390_EXTABLE_H
+/*
+ * The exception table consists of pairs of addresses: the first is the
+ * address of an instruction that is allowed to fault, and the second is
+ * the address at which the program should continue. No registers are
+ * modified, so it is entirely up to the continuation code to figure out
+ * what to do.
+ *
+ * All the routines below use bits of fixup code that are out of line
+ * with the main instruction path. This means when everything is well,
+ * we don't even have to jump over them. Further, they do not intrude
+ * on our cache or tlb entries.
+ */
+
+struct exception_table_entry
+{
+ int insn, fixup;
+};
+
+static inline unsigned long extable_fixup(const struct exception_table_entry *x)
+{
+ return (unsigned long)&x->fixup + x->fixup;
+}
+
+#define ARCH_HAS_RELATIVE_EXTABLE
+
+#endif
diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h
index 7228ed8da67d..72d69b9585c2 100644
--- a/arch/s390/include/asm/uaccess.h
+++ b/arch/s390/include/asm/uaccess.h
@@ -14,6 +14,7 @@
*/
#include <asm/processor.h>
#include <asm/ctl_reg.h>
+#include <asm/extable.h>
/*
@@ -59,72 +60,14 @@ static inline int __range_ok(unsigned long addr, unsigned long size)
#define access_ok(type, addr, size) __access_ok(addr, size)
-/*
- * The exception table consists of pairs of addresses: the first is the
- * address of an instruction that is allowed to fault, and the second is
- * the address at which the program should continue. No registers are
- * modified, so it is entirely up to the continuation code to figure out
- * what to do.
- *
- * All the routines below use bits of fixup code that are out of line
- * with the main instruction path. This means when everything is well,
- * we don't even have to jump over them. Further, they do not intrude
- * on our cache or tlb entries.
- */
-
-struct exception_table_entry
-{
- int insn, fixup;
-};
-
-static inline unsigned long extable_fixup(const struct exception_table_entry *x)
-{
- return (unsigned long)&x->fixup + x->fixup;
-}
-
-#define ARCH_HAS_RELATIVE_EXTABLE
-
-/**
- * __copy_from_user: - Copy a block of data from user space, with less checking.
- * @to: Destination address, in kernel space.
- * @from: Source address, in user space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from user space to kernel space. Caller must check
- * the specified block with access_ok() before calling this function.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- *
- * If some data could not be copied, this function will pad the copied
- * data to the requested size using zero bytes.
- */
-unsigned long __must_check __copy_from_user(void *to, const void __user *from,
- unsigned long n);
+unsigned long __must_check
+raw_copy_from_user(void *to, const void __user *from, unsigned long n);
-/**
- * __copy_to_user: - Copy a block of data into user space, with less checking.
- * @to: Destination address, in user space.
- * @from: Source address, in kernel space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from kernel space to user space. Caller must check
- * the specified block with access_ok() before calling this function.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- */
-unsigned long __must_check __copy_to_user(void __user *to, const void *from,
- unsigned long n);
+unsigned long __must_check
+raw_copy_to_user(void __user *to, const void *from, unsigned long n);
-#define __copy_to_user_inatomic __copy_to_user
-#define __copy_from_user_inatomic __copy_from_user
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
#ifdef CONFIG_HAVE_MARCH_Z10_FEATURES
@@ -213,13 +156,13 @@ static inline int __get_user_fn(void *x, const void __user *ptr, unsigned long s
static inline int __put_user_fn(void *x, void __user *ptr, unsigned long size)
{
- size = __copy_to_user(ptr, x, size);
+ size = raw_copy_to_user(ptr, x, size);
return size ? -EFAULT : 0;
}
static inline int __get_user_fn(void *x, const void __user *ptr, unsigned long size)
{
- size = __copy_from_user(x, ptr, size);
+ size = raw_copy_from_user(x, ptr, size);
return size ? -EFAULT : 0;
}
@@ -309,77 +252,8 @@ int __get_user_bad(void) __attribute__((noreturn));
#define __put_user_unaligned __put_user
#define __get_user_unaligned __get_user
-extern void __compiletime_error("usercopy buffer size is too small")
-__bad_copy_user(void);
-
-static inline void copy_user_overflow(int size, unsigned long count)
-{
- WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count);
-}
-
-/**
- * copy_to_user: - Copy a block of data into user space.
- * @to: Destination address, in user space.
- * @from: Source address, in kernel space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from kernel space to user space.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- */
-static inline unsigned long __must_check
-copy_to_user(void __user *to, const void *from, unsigned long n)
-{
- might_fault();
- return __copy_to_user(to, from, n);
-}
-
-/**
- * copy_from_user: - Copy a block of data from user space.
- * @to: Destination address, in kernel space.
- * @from: Source address, in user space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from user space to kernel space.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- *
- * If some data could not be copied, this function will pad the copied
- * data to the requested size using zero bytes.
- */
-static inline unsigned long __must_check
-copy_from_user(void *to, const void __user *from, unsigned long n)
-{
- unsigned int sz = __compiletime_object_size(to);
-
- might_fault();
- if (unlikely(sz != -1 && sz < n)) {
- if (!__builtin_constant_p(n))
- copy_user_overflow(sz, n);
- else
- __bad_copy_user();
- return n;
- }
- return __copy_from_user(to, from, n);
-}
-
unsigned long __must_check
-__copy_in_user(void __user *to, const void __user *from, unsigned long n);
-
-static inline unsigned long __must_check
-copy_in_user(void __user *to, const void __user *from, unsigned long n)
-{
- might_fault();
- return __copy_in_user(to, from, n);
-}
+raw_copy_in_user(void __user *to, const void __user *from, unsigned long n);
/*
* Copy a null terminated string from userspace.
diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c
index f481fcde067b..1e5bb2b86c42 100644
--- a/arch/s390/lib/uaccess.c
+++ b/arch/s390/lib/uaccess.c
@@ -26,7 +26,7 @@ static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr
tmp1 = -4096UL;
asm volatile(
"0: .insn ss,0xc80000000000,0(%0,%2),0(%1),0\n"
- "9: jz 7f\n"
+ "6: jz 4f\n"
"1: algr %0,%3\n"
" slgr %1,%3\n"
" slgr %2,%3\n"
@@ -35,23 +35,13 @@ static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr
" nr %4,%3\n" /* %4 = (ptr + 4095) & -4096 */
" slgr %4,%1\n"
" clgr %0,%4\n" /* copy crosses next page boundary? */
- " jnh 4f\n"
+ " jnh 5f\n"
"3: .insn ss,0xc80000000000,0(%4,%2),0(%1),0\n"
- "10:slgr %0,%4\n"
- " algr %2,%4\n"
- "4: lghi %4,-1\n"
- " algr %4,%0\n" /* copy remaining size, subtract 1 */
- " bras %3,6f\n" /* memset loop */
- " xc 0(1,%2),0(%2)\n"
- "5: xc 0(256,%2),0(%2)\n"
- " la %2,256(%2)\n"
- "6: aghi %4,-256\n"
- " jnm 5b\n"
- " ex %4,0(%3)\n"
- " j 8f\n"
- "7: slgr %0,%0\n"
- "8:\n"
- EX_TABLE(0b,2b) EX_TABLE(3b,4b) EX_TABLE(9b,2b) EX_TABLE(10b,4b)
+ "7: slgr %0,%4\n"
+ " j 5f\n"
+ "4: slgr %0,%0\n"
+ "5:\n"
+ EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b)
: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
: "d" (reg0) : "cc", "memory");
return size;
@@ -67,49 +57,38 @@ static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr,
asm volatile(
" sacf 0\n"
"0: mvcp 0(%0,%2),0(%1),%3\n"
- "10:jz 8f\n"
+ "7: jz 5f\n"
"1: algr %0,%3\n"
" la %1,256(%1)\n"
" la %2,256(%2)\n"
"2: mvcp 0(%0,%2),0(%1),%3\n"
- "11:jnz 1b\n"
- " j 8f\n"
+ "8: jnz 1b\n"
+ " j 5f\n"
"3: la %4,255(%1)\n" /* %4 = ptr + 255 */
" lghi %3,-4096\n"
" nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */
" slgr %4,%1\n"
" clgr %0,%4\n" /* copy crosses next page boundary? */
- " jnh 5f\n"
+ " jnh 6f\n"
"4: mvcp 0(%4,%2),0(%1),%3\n"
- "12:slgr %0,%4\n"
- " algr %2,%4\n"
- "5: lghi %4,-1\n"
- " algr %4,%0\n" /* copy remaining size, subtract 1 */
- " bras %3,7f\n" /* memset loop */
- " xc 0(1,%2),0(%2)\n"
- "6: xc 0(256,%2),0(%2)\n"
- " la %2,256(%2)\n"
- "7: aghi %4,-256\n"
- " jnm 6b\n"
- " ex %4,0(%3)\n"
- " j 9f\n"
- "8: slgr %0,%0\n"
- "9: sacf 768\n"
- EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,5b)
- EX_TABLE(10b,3b) EX_TABLE(11b,3b) EX_TABLE(12b,5b)
+ "9: slgr %0,%4\n"
+ " j 6f\n"
+ "5: slgr %0,%0\n"
+ "6: sacf 768\n"
+ EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b)
+ EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b)
: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
: : "cc", "memory");
return size;
}
-unsigned long __copy_from_user(void *to, const void __user *from, unsigned long n)
+unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
- check_object_size(to, n, false);
if (static_branch_likely(&have_mvcos))
return copy_from_user_mvcos(to, from, n);
return copy_from_user_mvcp(to, from, n);
}
-EXPORT_SYMBOL(__copy_from_user);
+EXPORT_SYMBOL(raw_copy_from_user);
static inline unsigned long copy_to_user_mvcos(void __user *ptr, const void *x,
unsigned long size)
@@ -176,14 +155,13 @@ static inline unsigned long copy_to_user_mvcs(void __user *ptr, const void *x,
return size;
}
-unsigned long __copy_to_user(void __user *to, const void *from, unsigned long n)
+unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
- check_object_size(from, n, true);
if (static_branch_likely(&have_mvcos))
return copy_to_user_mvcos(to, from, n);
return copy_to_user_mvcs(to, from, n);
}
-EXPORT_SYMBOL(__copy_to_user);
+EXPORT_SYMBOL(raw_copy_to_user);
static inline unsigned long copy_in_user_mvcos(void __user *to, const void __user *from,
unsigned long size)
@@ -240,13 +218,13 @@ static inline unsigned long copy_in_user_mvc(void __user *to, const void __user
return size;
}
-unsigned long __copy_in_user(void __user *to, const void __user *from, unsigned long n)
+unsigned long raw_copy_in_user(void __user *to, const void __user *from, unsigned long n)
{
if (static_branch_likely(&have_mvcos))
return copy_in_user_mvcos(to, from, n);
return copy_in_user_mvc(to, from, n);
}
-EXPORT_SYMBOL(__copy_in_user);
+EXPORT_SYMBOL(raw_copy_in_user);
static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size)
{
diff --git a/arch/score/Kconfig b/arch/score/Kconfig
index 507d63181389..4d241cfedc3b 100644
--- a/arch/score/Kconfig
+++ b/arch/score/Kconfig
@@ -15,6 +15,7 @@ config SCORE
select MODULES_USE_ELF_REL
select CLONE_BACKWARDS
select CPU_NO_EFFICIENT_FFS
+ select ARCH_HAS_RAW_COPY_USER
choice
prompt "System type"
diff --git a/arch/score/include/asm/Kbuild b/arch/score/include/asm/Kbuild
index 926943a49ea5..e3a8d0f96652 100644
--- a/arch/score/include/asm/Kbuild
+++ b/arch/score/include/asm/Kbuild
@@ -4,6 +4,7 @@ header-y +=
generic-y += barrier.h
generic-y += clkdev.h
generic-y += current.h
+generic-y += extable.h
generic-y += irq_work.h
generic-y += mcs_spinlock.h
generic-y += mm-arch-hooks.h
diff --git a/arch/score/include/asm/extable.h b/arch/score/include/asm/extable.h
deleted file mode 100644
index c4423ccf830d..000000000000
--- a/arch/score/include/asm/extable.h
+++ /dev/null
@@ -1,11 +0,0 @@
-#ifndef _ASM_SCORE_EXTABLE_H
-#define _ASM_SCORE_EXTABLE_H
-
-struct exception_table_entry {
- unsigned long insn;
- unsigned long fixup;
-};
-
-struct pt_regs;
-extern int fixup_exception(struct pt_regs *regs);
-#endif
diff --git a/arch/score/include/asm/uaccess.h b/arch/score/include/asm/uaccess.h
index 7a6c6982420a..916e5dbf0bfd 100644
--- a/arch/score/include/asm/uaccess.h
+++ b/arch/score/include/asm/uaccess.h
@@ -295,61 +295,19 @@ extern void __put_user_unknown(void);
extern int __copy_tofrom_user(void *to, const void *from, unsigned long len);
static inline unsigned long
-copy_from_user(void *to, const void *from, unsigned long len)
+raw_copy_from_user(void *to, const void __user *from, unsigned long len)
{
- unsigned long res = len;
-
- if (likely(access_ok(VERIFY_READ, from, len)))
- res = __copy_tofrom_user(to, from, len);
-
- if (unlikely(res))
- memset(to + (len - res), 0, res);
-
- return res;
+ return __copy_tofrom_user(to, (__force const void *)from, len);
}
static inline unsigned long
-copy_to_user(void *to, const void *from, unsigned long len)
+raw_copy_to_user(void __user *to, const void *from, unsigned long len)
{
- if (likely(access_ok(VERIFY_WRITE, to, len)))
- len = __copy_tofrom_user(to, from, len);
-
- return len;
+ return __copy_tofrom_user((__force void *)to, from, len);
}
-static inline unsigned long
-__copy_from_user(void *to, const void *from, unsigned long len)
-{
- unsigned long left = __copy_tofrom_user(to, from, len);
- if (unlikely(left))
- memset(to + (len - left), 0, left);
- return left;
-}
-
-#define __copy_to_user(to, from, len) \
- __copy_tofrom_user((to), (from), (len))
-
-static inline unsigned long
-__copy_to_user_inatomic(void *to, const void *from, unsigned long len)
-{
- return __copy_to_user(to, from, len);
-}
-
-static inline unsigned long
-__copy_from_user_inatomic(void *to, const void *from, unsigned long len)
-{
- return __copy_tofrom_user(to, from, len);
-}
-
-#define __copy_in_user(to, from, len) __copy_tofrom_user(to, from, len)
-
-static inline unsigned long
-copy_in_user(void *to, const void *from, unsigned long len)
-{
- if (access_ok(VERIFY_READ, from, len) &&
- access_ok(VERFITY_WRITE, to, len))
- return __copy_tofrom_user(to, from, len);
-}
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
/*
* __clear_user: - Zero a block of memory in user space, with less checking.
diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig
index ee086958b2b2..c689645eb8b7 100644
--- a/arch/sh/Kconfig
+++ b/arch/sh/Kconfig
@@ -48,6 +48,7 @@ config SUPERH
select HAVE_ARCH_AUDITSYSCALL
select HAVE_FUTEX_CMPXCHG if FUTEX
select HAVE_NMI
+ select ARCH_HAS_RAW_COPY_USER
help
The SuperH is a RISC processor targeted for use in embedded systems
and consumer electronics; it was also used in the Sega Dreamcast
diff --git a/arch/sh/include/asm/extable.h b/arch/sh/include/asm/extable.h
new file mode 100644
index 000000000000..df2ee2fcb8d3
--- /dev/null
+++ b/arch/sh/include/asm/extable.h
@@ -0,0 +1,10 @@
+#ifndef __ASM_SH_EXTABLE_H
+#define __ASM_SH_EXTABLE_H
+
+#include <asm-generic/extable.h>
+
+#if defined(CONFIG_SUPERH64) && defined(CONFIG_MMU)
+#define ARCH_HAS_SEARCH_EXTABLE
+#endif
+
+#endif
diff --git a/arch/sh/include/asm/uaccess.h b/arch/sh/include/asm/uaccess.h
index 89a28dfbabfa..2722b61b2283 100644
--- a/arch/sh/include/asm/uaccess.h
+++ b/arch/sh/include/asm/uaccess.h
@@ -2,6 +2,7 @@
#define __ASM_SH_UACCESS_H
#include <asm/segment.h>
+#include <asm/extable.h>
#define __addr_ok(addr) \
((unsigned long __force)(addr) < current_thread_info()->addr_limit.seg)
@@ -107,19 +108,18 @@ extern __must_check long strnlen_user(const char __user *str, long n);
__kernel_size_t __copy_user(void *to, const void *from, __kernel_size_t n);
static __always_inline unsigned long
-__copy_from_user(void *to, const void __user *from, unsigned long n)
+raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
return __copy_user(to, (__force void *)from, n);
}
static __always_inline unsigned long __must_check
-__copy_to_user(void __user *to, const void *from, unsigned long n)
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
return __copy_user((__force void *)to, from, n);
}
-
-#define __copy_to_user_inatomic __copy_to_user
-#define __copy_from_user_inatomic __copy_from_user
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
/*
* Clear the area and return remaining number of bytes
@@ -139,55 +139,6 @@ __kernel_size_t __clear_user(void *addr, __kernel_size_t size);
__cl_size; \
})
-static inline unsigned long
-copy_from_user(void *to, const void __user *from, unsigned long n)
-{
- unsigned long __copy_from = (unsigned long) from;
- __kernel_size_t __copy_size = (__kernel_size_t) n;
-
- if (__copy_size && __access_ok(__copy_from, __copy_size))
- __copy_size = __copy_user(to, from, __copy_size);
-
- if (unlikely(__copy_size))
- memset(to + (n - __copy_size), 0, __copy_size);
-
- return __copy_size;
-}
-
-static inline unsigned long
-copy_to_user(void __user *to, const void *from, unsigned long n)
-{
- unsigned long __copy_to = (unsigned long) to;
- __kernel_size_t __copy_size = (__kernel_size_t) n;
-
- if (__copy_size && __access_ok(__copy_to, __copy_size))
- return __copy_user(to, from, __copy_size);
-
- return __copy_size;
-}
-
-/*
- * The exception table consists of pairs of addresses: the first is the
- * address of an instruction that is allowed to fault, and the second is
- * the address at which the program should continue. No registers are
- * modified, so it is entirely up to the continuation code to figure out
- * what to do.
- *
- * All the routines below use bits of fixup code that are out of line
- * with the main instruction path. This means when everything is well,
- * we don't even have to jump over them. Further, they do not intrude
- * on our cache or tlb entries.
- */
-struct exception_table_entry {
- unsigned long insn, fixup;
-};
-
-#if defined(CONFIG_SUPERH64) && defined(CONFIG_MMU)
-#define ARCH_HAS_SEARCH_EXTABLE
-#endif
-
-int fixup_exception(struct pt_regs *regs);
-
extern void *set_exception_table_vec(unsigned int vec, void *handler);
static inline void *set_exception_table_evt(unsigned int evt, void *handler)
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index 68ac5c7cd982..68353048f24e 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -45,6 +45,7 @@ config SPARC
select HAVE_ARCH_HARDENED_USERCOPY
select PROVE_LOCKING_SMALL if PROVE_LOCKING
select ARCH_WANT_RELAX_ORDER
+ select ARCH_HAS_RAW_COPY_USER
config SPARC32
def_bool !64BIT
diff --git a/arch/sparc/include/asm/uaccess_32.h b/arch/sparc/include/asm/uaccess_32.h
index a59a1e81986d..12ebee2d97c7 100644
--- a/arch/sparc/include/asm/uaccess_32.h
+++ b/arch/sparc/include/asm/uaccess_32.h
@@ -71,8 +71,6 @@ struct exception_table_entry
/* Returns 0 if exception not found and fixup otherwise. */
unsigned long search_extables_range(unsigned long addr, unsigned long *g2);
-void __ret_efault(void);
-
/* Uh, these should become the main single-value transfer routines..
* They automatically use the right size if we just have the right
* pointer type..
@@ -237,39 +235,18 @@ int __get_user_bad(void);
unsigned long __copy_user(void __user *to, const void __user *from, unsigned long size);
-static inline unsigned long copy_to_user(void __user *to, const void *from, unsigned long n)
-{
- if (n && __access_ok((unsigned long) to, n)) {
- check_object_size(from, n, true);
- return __copy_user(to, (__force void __user *) from, n);
- } else
- return n;
-}
-
-static inline unsigned long __copy_to_user(void __user *to, const void *from, unsigned long n)
+static inline unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
- check_object_size(from, n, true);
return __copy_user(to, (__force void __user *) from, n);
}
-static inline unsigned long copy_from_user(void *to, const void __user *from, unsigned long n)
-{
- if (n && __access_ok((unsigned long) from, n)) {
- check_object_size(to, n, false);
- return __copy_user((__force void __user *) to, from, n);
- } else {
- memset(to, 0, n);
- return n;
- }
-}
-
-static inline unsigned long __copy_from_user(void *to, const void __user *from, unsigned long n)
+static inline unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
return __copy_user((__force void __user *) to, from, n);
}
-#define __copy_to_user_inatomic __copy_to_user
-#define __copy_from_user_inatomic __copy_from_user
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
static inline unsigned long __clear_user(void __user *addr, unsigned long size)
{
diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h
index 7afb4f64553f..6096d671aa63 100644
--- a/arch/sparc/include/asm/uaccess_64.h
+++ b/arch/sparc/include/asm/uaccess_64.h
@@ -176,39 +176,19 @@ __asm__ __volatile__( \
int __get_user_bad(void);
-unsigned long __must_check ___copy_from_user(void *to,
+unsigned long __must_check raw_copy_from_user(void *to,
const void __user *from,
unsigned long size);
-static inline unsigned long __must_check
-copy_from_user(void *to, const void __user *from, unsigned long size)
-{
- check_object_size(to, size, false);
-
- return ___copy_from_user(to, from, size);
-}
-#define __copy_from_user copy_from_user
-unsigned long __must_check ___copy_to_user(void __user *to,
+unsigned long __must_check raw_copy_to_user(void __user *to,
const void *from,
unsigned long size);
-static inline unsigned long __must_check
-copy_to_user(void __user *to, const void *from, unsigned long size)
-{
- check_object_size(from, size, true);
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
- return ___copy_to_user(to, from, size);
-}
-#define __copy_to_user copy_to_user
-
-unsigned long __must_check ___copy_in_user(void __user *to,
+unsigned long __must_check raw_copy_in_user(void __user *to,
const void __user *from,
unsigned long size);
-static inline unsigned long __must_check
-copy_in_user(void __user *to, void __user *from, unsigned long size)
-{
- return ___copy_in_user(to, from, size);
-}
-#define __copy_in_user copy_in_user
unsigned long __must_check __clear_user(void __user *, unsigned long);
@@ -217,9 +197,6 @@ unsigned long __must_check __clear_user(void __user *, unsigned long);
__must_check long strlen_user(const char __user *str);
__must_check long strnlen_user(const char __user *str, long n);
-#define __copy_to_user_inatomic __copy_to_user
-#define __copy_from_user_inatomic __copy_from_user
-
struct pt_regs;
unsigned long compute_effective_address(struct pt_regs *,
unsigned int insn,
diff --git a/arch/sparc/kernel/head_32.S b/arch/sparc/kernel/head_32.S
index 7bb317b87dde..7274e43ff9be 100644
--- a/arch/sparc/kernel/head_32.S
+++ b/arch/sparc/kernel/head_32.S
@@ -809,10 +809,3 @@ lvl14_save:
.word 0
.word 0
.word t_irq14
-
- .section ".fixup",#alloc,#execinstr
- .globl __ret_efault
-__ret_efault:
- ret
- restore %g0, -EFAULT, %o0
-EXPORT_SYMBOL(__ret_efault)
diff --git a/arch/sparc/lib/GENcopy_from_user.S b/arch/sparc/lib/GENcopy_from_user.S
index 69a439fa2fc1..8aa16ef113f2 100644
--- a/arch/sparc/lib/GENcopy_from_user.S
+++ b/arch/sparc/lib/GENcopy_from_user.S
@@ -23,7 +23,7 @@
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
- bne,pn %icc, ___copy_in_user; \
+ bne,pn %icc, raw_copy_in_user; \
nop
#endif
diff --git a/arch/sparc/lib/GENcopy_to_user.S b/arch/sparc/lib/GENcopy_to_user.S
index 9947427ce354..311c8fa5e98e 100644
--- a/arch/sparc/lib/GENcopy_to_user.S
+++ b/arch/sparc/lib/GENcopy_to_user.S
@@ -27,7 +27,7 @@
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
- bne,pn %icc, ___copy_in_user; \
+ bne,pn %icc, raw_copy_in_user; \
nop
#endif
diff --git a/arch/sparc/lib/GENpatch.S b/arch/sparc/lib/GENpatch.S
index fab9e89f16bd..95e2f1f9e477 100644
--- a/arch/sparc/lib/GENpatch.S
+++ b/arch/sparc/lib/GENpatch.S
@@ -26,8 +26,8 @@
.type generic_patch_copyops,#function
generic_patch_copyops:
GEN_DO_PATCH(memcpy, GENmemcpy)
- GEN_DO_PATCH(___copy_from_user, GENcopy_from_user)
- GEN_DO_PATCH(___copy_to_user, GENcopy_to_user)
+ GEN_DO_PATCH(raw_copy_from_user, GENcopy_from_user)
+ GEN_DO_PATCH(raw_copy_to_user, GENcopy_to_user)
retl
nop
.size generic_patch_copyops,.-generic_patch_copyops
diff --git a/arch/sparc/lib/NG2copy_from_user.S b/arch/sparc/lib/NG2copy_from_user.S
index b79a6998d87c..0d8a018118c2 100644
--- a/arch/sparc/lib/NG2copy_from_user.S
+++ b/arch/sparc/lib/NG2copy_from_user.S
@@ -36,7 +36,7 @@
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
- bne,pn %icc, ___copy_in_user; \
+ bne,pn %icc, raw_copy_in_user; \
nop
#endif
diff --git a/arch/sparc/lib/NG2copy_to_user.S b/arch/sparc/lib/NG2copy_to_user.S
index dcec55f254ab..a7a0ea0d8a0b 100644
--- a/arch/sparc/lib/NG2copy_to_user.S
+++ b/arch/sparc/lib/NG2copy_to_user.S
@@ -45,7 +45,7 @@
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
- bne,pn %icc, ___copy_in_user; \
+ bne,pn %icc, raw_copy_in_user; \
nop
#endif
diff --git a/arch/sparc/lib/NG2patch.S b/arch/sparc/lib/NG2patch.S
index 28c36f06a6d1..56ccc19adde8 100644
--- a/arch/sparc/lib/NG2patch.S
+++ b/arch/sparc/lib/NG2patch.S
@@ -26,8 +26,8 @@
.type niagara2_patch_copyops,#function
niagara2_patch_copyops:
NG_DO_PATCH(memcpy, NG2memcpy)
- NG_DO_PATCH(___copy_from_user, NG2copy_from_user)
- NG_DO_PATCH(___copy_to_user, NG2copy_to_user)
+ NG_DO_PATCH(raw_copy_from_user, NG2copy_from_user)
+ NG_DO_PATCH(raw_copy_to_user, NG2copy_to_user)
retl
nop
.size niagara2_patch_copyops,.-niagara2_patch_copyops
diff --git a/arch/sparc/lib/NG4copy_from_user.S b/arch/sparc/lib/NG4copy_from_user.S
index 16a286c1a528..5bb506bd61fa 100644
--- a/arch/sparc/lib/NG4copy_from_user.S
+++ b/arch/sparc/lib/NG4copy_from_user.S
@@ -31,7 +31,7 @@
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
- bne,pn %icc, ___copy_in_user; \
+ bne,pn %icc, raw_copy_in_user; \
nop
#endif
diff --git a/arch/sparc/lib/NG4copy_to_user.S b/arch/sparc/lib/NG4copy_to_user.S
index 6b0276ffc858..a82d4d45fc1c 100644
--- a/arch/sparc/lib/NG4copy_to_user.S
+++ b/arch/sparc/lib/NG4copy_to_user.S
@@ -40,7 +40,7 @@
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
- bne,pn %icc, ___copy_in_user; \
+ bne,pn %icc, raw_copy_in_user; \
nop
#endif
diff --git a/arch/sparc/lib/NG4patch.S b/arch/sparc/lib/NG4patch.S
index a114cbcf2a48..3cc0f8cc95df 100644
--- a/arch/sparc/lib/NG4patch.S
+++ b/arch/sparc/lib/NG4patch.S
@@ -26,8 +26,8 @@
.type niagara4_patch_copyops,#function
niagara4_patch_copyops:
NG_DO_PATCH(memcpy, NG4memcpy)
- NG_DO_PATCH(___copy_from_user, NG4copy_from_user)
- NG_DO_PATCH(___copy_to_user, NG4copy_to_user)
+ NG_DO_PATCH(raw_copy_from_user, NG4copy_from_user)
+ NG_DO_PATCH(raw_copy_to_user, NG4copy_to_user)
retl
nop
.size niagara4_patch_copyops,.-niagara4_patch_copyops
diff --git a/arch/sparc/lib/NGcopy_from_user.S b/arch/sparc/lib/NGcopy_from_user.S
index 9cd42fcbc781..2333b6f3e824 100644
--- a/arch/sparc/lib/NGcopy_from_user.S
+++ b/arch/sparc/lib/NGcopy_from_user.S
@@ -25,7 +25,7 @@
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
- bne,pn %icc, ___copy_in_user; \
+ bne,pn %icc, raw_copy_in_user; \
nop
#endif
diff --git a/arch/sparc/lib/NGcopy_to_user.S b/arch/sparc/lib/NGcopy_to_user.S
index 5c358afd464e..07ba20bc4ea4 100644
--- a/arch/sparc/lib/NGcopy_to_user.S
+++ b/arch/sparc/lib/NGcopy_to_user.S
@@ -28,7 +28,7 @@
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
- bne,pn %icc, ___copy_in_user; \
+ bne,pn %icc, raw_copy_in_user; \
nop
#endif
diff --git a/arch/sparc/lib/NGpatch.S b/arch/sparc/lib/NGpatch.S
index 3b0674fc3366..62ccda7e7b38 100644
--- a/arch/sparc/lib/NGpatch.S
+++ b/arch/sparc/lib/NGpatch.S
@@ -26,8 +26,8 @@
.type niagara_patch_copyops,#function
niagara_patch_copyops:
NG_DO_PATCH(memcpy, NGmemcpy)
- NG_DO_PATCH(___copy_from_user, NGcopy_from_user)
- NG_DO_PATCH(___copy_to_user, NGcopy_to_user)
+ NG_DO_PATCH(raw_copy_from_user, NGcopy_from_user)
+ NG_DO_PATCH(raw_copy_to_user, NGcopy_to_user)
retl
nop
.size niagara_patch_copyops,.-niagara_patch_copyops
diff --git a/arch/sparc/lib/U1copy_from_user.S b/arch/sparc/lib/U1copy_from_user.S
index bb6ff73229e3..9a6e68a9bf4a 100644
--- a/arch/sparc/lib/U1copy_from_user.S
+++ b/arch/sparc/lib/U1copy_from_user.S
@@ -19,7 +19,7 @@
.text; \
.align 4;
-#define FUNC_NAME ___copy_from_user
+#define FUNC_NAME raw_copy_from_user
#define LOAD(type,addr,dest) type##a [addr] %asi, dest
#define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_AIUS, dest
#define EX_RETVAL(x) 0
@@ -31,7 +31,7 @@
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
- bne,pn %icc, ___copy_in_user; \
+ bne,pn %icc, raw_copy_in_user; \
nop; \
#include "U1memcpy.S"
diff --git a/arch/sparc/lib/U1copy_to_user.S b/arch/sparc/lib/U1copy_to_user.S
index ed92ce739558..d7b28491eddf 100644
--- a/arch/sparc/lib/U1copy_to_user.S
+++ b/arch/sparc/lib/U1copy_to_user.S
@@ -19,7 +19,7 @@
.text; \
.align 4;
-#define FUNC_NAME ___copy_to_user
+#define FUNC_NAME raw_copy_to_user
#define STORE(type,src,addr) type##a src, [addr] ASI_AIUS
#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_AIUS
#define EX_RETVAL(x) 0
@@ -31,7 +31,7 @@
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
- bne,pn %icc, ___copy_in_user; \
+ bne,pn %icc, raw_copy_in_user; \
nop; \
#include "U1memcpy.S"
diff --git a/arch/sparc/lib/U3copy_to_user.S b/arch/sparc/lib/U3copy_to_user.S
index c4ee858e352a..f48fb87fe9f2 100644
--- a/arch/sparc/lib/U3copy_to_user.S
+++ b/arch/sparc/lib/U3copy_to_user.S
@@ -31,7 +31,7 @@
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
- bne,pn %icc, ___copy_in_user; \
+ bne,pn %icc, raw_copy_in_user; \
nop; \
#include "U3memcpy.S"
diff --git a/arch/sparc/lib/U3patch.S b/arch/sparc/lib/U3patch.S
index ecc302619a6e..91cd6539b6e1 100644
--- a/arch/sparc/lib/U3patch.S
+++ b/arch/sparc/lib/U3patch.S
@@ -26,8 +26,8 @@
.type cheetah_patch_copyops,#function
cheetah_patch_copyops:
ULTRA3_DO_PATCH(memcpy, U3memcpy)
- ULTRA3_DO_PATCH(___copy_from_user, U3copy_from_user)
- ULTRA3_DO_PATCH(___copy_to_user, U3copy_to_user)
+ ULTRA3_DO_PATCH(raw_copy_from_user, U3copy_from_user)
+ ULTRA3_DO_PATCH(raw_copy_to_user, U3copy_to_user)
retl
nop
.size cheetah_patch_copyops,.-cheetah_patch_copyops
diff --git a/arch/sparc/lib/copy_in_user.S b/arch/sparc/lib/copy_in_user.S
index 0252b218de45..1b73bb80aeb0 100644
--- a/arch/sparc/lib/copy_in_user.S
+++ b/arch/sparc/lib/copy_in_user.S
@@ -44,7 +44,7 @@ __retl_o2_plus_1:
* to copy register windows around during thread cloning.
*/
-ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
+ENTRY(raw_copy_in_user) /* %o0=dst, %o1=src, %o2=len */
cmp %o2, 0
be,pn %XCC, 85f
or %o0, %o1, %o3
@@ -105,5 +105,5 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
add %o0, 1, %o0
retl
clr %o0
-ENDPROC(___copy_in_user)
-EXPORT_SYMBOL(___copy_in_user)
+ENDPROC(raw_copy_in_user)
+EXPORT_SYMBOL(raw_copy_in_user)
diff --git a/arch/sparc/lib/copy_user.S b/arch/sparc/lib/copy_user.S
index cea644dc67a6..bc243ee807cc 100644
--- a/arch/sparc/lib/copy_user.S
+++ b/arch/sparc/lib/copy_user.S
@@ -364,21 +364,7 @@ short_aligned_end:
97:
mov %o2, %g3
fixupretl:
- sethi %hi(PAGE_OFFSET), %g1
- cmp %o0, %g1
- blu 1f
- cmp %o1, %g1
- bgeu 1f
- ld [%g6 + TI_PREEMPT], %g1
- cmp %g1, 0
- bne 1f
- nop
- save %sp, -64, %sp
- mov %i0, %o0
- call __bzero
- mov %g3, %o1
- restore
-1: retl
+ retl
mov %g3, %o0
/* exception routine sets %g2 to (broken_insn - first_insn)>>2 */
diff --git a/arch/tile/Kconfig b/arch/tile/Kconfig
index 4583c0320059..27808d86eb57 100644
--- a/arch/tile/Kconfig
+++ b/arch/tile/Kconfig
@@ -33,6 +33,7 @@ config TILE
select USER_STACKTRACE_SUPPORT
select USE_PMC if PERF_EVENTS
select VIRT_TO_BUS
+ select ARCH_HAS_RAW_COPY_USER
config MMU
def_bool y
diff --git a/arch/tile/include/asm/Kbuild b/arch/tile/include/asm/Kbuild
index aa48b6eaff2d..24c44e93804d 100644
--- a/arch/tile/include/asm/Kbuild
+++ b/arch/tile/include/asm/Kbuild
@@ -7,6 +7,7 @@ generic-y += clkdev.h
generic-y += emergency-restart.h
generic-y += errno.h
generic-y += exec.h
+generic-y += extable.h
generic-y += fb.h
generic-y += fcntl.h
generic-y += hw_irq.h
diff --git a/arch/tile/include/asm/uaccess.h b/arch/tile/include/asm/uaccess.h
index 14ea3d1ca2c7..a803f6bb4d92 100644
--- a/arch/tile/include/asm/uaccess.h
+++ b/arch/tile/include/asm/uaccess.h
@@ -98,24 +98,7 @@ int __range_ok(unsigned long addr, unsigned long size);
likely(__range_ok((unsigned long)(addr), (size)) == 0); \
})
-/*
- * The exception table consists of pairs of addresses: the first is the
- * address of an instruction that is allowed to fault, and the second is
- * the address at which the program should continue. No registers are
- * modified, so it is entirely up to the continuation code to figure out
- * what to do.
- *
- * All the routines below use bits of fixup code that are out of line
- * with the main instruction path. This means when everything is well,
- * we don't even have to jump over them. Further, they do not intrude
- * on our cache or tlb entries.
- */
-
-struct exception_table_entry {
- unsigned long insn, fixup;
-};
-
-extern int fixup_exception(struct pt_regs *regs);
+#include <asm/extable.h>
/*
* This is a type: either unsigned long, if the argument fits into
@@ -330,145 +313,16 @@ extern int __put_user_bad(void)
((x) = 0, -EFAULT); \
})
-/**
- * __copy_to_user() - copy data into user space, with less checking.
- * @to: Destination address, in user space.
- * @from: Source address, in kernel space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from kernel space to user space. Caller must check
- * the specified block with access_ok() before calling this function.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- *
- * An alternate version - __copy_to_user_inatomic() - is designed
- * to be called from atomic context, typically bracketed by calls
- * to pagefault_disable() and pagefault_enable().
- */
-extern unsigned long __must_check __copy_to_user_inatomic(
- void __user *to, const void *from, unsigned long n);
-
-static inline unsigned long __must_check
-__copy_to_user(void __user *to, const void *from, unsigned long n)
-{
- might_fault();
- return __copy_to_user_inatomic(to, from, n);
-}
-
-static inline unsigned long __must_check
-copy_to_user(void __user *to, const void *from, unsigned long n)
-{
- if (access_ok(VERIFY_WRITE, to, n))
- n = __copy_to_user(to, from, n);
- return n;
-}
-
-/**
- * __copy_from_user() - copy data from user space, with less checking.
- * @to: Destination address, in kernel space.
- * @from: Source address, in user space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from user space to kernel space. Caller must check
- * the specified block with access_ok() before calling this function.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- *
- * If some data could not be copied, this function will pad the copied
- * data to the requested size using zero bytes.
- *
- * An alternate version - __copy_from_user_inatomic() - is designed
- * to be called from atomic context, typically bracketed by calls
- * to pagefault_disable() and pagefault_enable(). This version
- * does *NOT* pad with zeros.
- */
-extern unsigned long __must_check __copy_from_user_inatomic(
- void *to, const void __user *from, unsigned long n);
-extern unsigned long __must_check __copy_from_user_zeroing(
- void *to, const void __user *from, unsigned long n);
-
-static inline unsigned long __must_check
-__copy_from_user(void *to, const void __user *from, unsigned long n)
-{
- might_fault();
- return __copy_from_user_zeroing(to, from, n);
-}
-
-static inline unsigned long __must_check
-_copy_from_user(void *to, const void __user *from, unsigned long n)
-{
- if (access_ok(VERIFY_READ, from, n))
- n = __copy_from_user(to, from, n);
- else
- memset(to, 0, n);
- return n;
-}
-
-extern void __compiletime_error("usercopy buffer size is too small")
-__bad_copy_user(void);
-
-static inline void copy_user_overflow(int size, unsigned long count)
-{
- WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count);
-}
-
-static inline unsigned long __must_check copy_from_user(void *to,
- const void __user *from,
- unsigned long n)
-{
- int sz = __compiletime_object_size(to);
-
- if (likely(sz == -1 || sz >= n))
- n = _copy_from_user(to, from, n);
- else if (!__builtin_constant_p(n))
- copy_user_overflow(sz, n);
- else
- __bad_copy_user();
-
- return n;
-}
+extern unsigned long __must_check
+raw_copy_to_user(void __user *to, const void *from, unsigned long n);
+extern unsigned long __must_check
+raw_copy_from_user(void *to, const void __user *from, unsigned long n);
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
#ifdef __tilegx__
-/**
- * __copy_in_user() - copy data within user space, with less checking.
- * @to: Destination address, in user space.
- * @from: Source address, in user space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from user space to user space. Caller must check
- * the specified blocks with access_ok() before calling this function.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- */
-extern unsigned long __copy_in_user_inatomic(
+extern unsigned long raw_copy_in_user(
void __user *to, const void __user *from, unsigned long n);
-
-static inline unsigned long __must_check
-__copy_in_user(void __user *to, const void __user *from, unsigned long n)
-{
- might_fault();
- return __copy_in_user_inatomic(to, from, n);
-}
-
-static inline unsigned long __must_check
-copy_in_user(void __user *to, const void __user *from, unsigned long n)
-{
- if (access_ok(VERIFY_WRITE, to, n) && access_ok(VERIFY_READ, from, n))
- n = __copy_in_user(to, from, n);
- return n;
-}
#endif
diff --git a/arch/tile/lib/exports.c b/arch/tile/lib/exports.c
index c5369fe643c7..ecce8e177e3f 100644
--- a/arch/tile/lib/exports.c
+++ b/arch/tile/lib/exports.c
@@ -38,11 +38,10 @@ EXPORT_SYMBOL(__mcount);
/* arch/tile/lib/, various memcpy files */
EXPORT_SYMBOL(memcpy);
-EXPORT_SYMBOL(__copy_to_user_inatomic);
-EXPORT_SYMBOL(__copy_from_user_inatomic);
-EXPORT_SYMBOL(__copy_from_user_zeroing);
+EXPORT_SYMBOL(raw_copy_to_user);
+EXPORT_SYMBOL(raw_copy_from_user);
#ifdef __tilegx__
-EXPORT_SYMBOL(__copy_in_user_inatomic);
+EXPORT_SYMBOL(raw_copy_in_user);
#endif
/* hypervisor glue */
diff --git a/arch/tile/lib/memcpy_32.S b/arch/tile/lib/memcpy_32.S
index a2771ae5da53..270f1267cd18 100644
--- a/arch/tile/lib/memcpy_32.S
+++ b/arch/tile/lib/memcpy_32.S
@@ -24,7 +24,6 @@
#define IS_MEMCPY 0
#define IS_COPY_FROM_USER 1
-#define IS_COPY_FROM_USER_ZEROING 2
#define IS_COPY_TO_USER -1
.section .text.memcpy_common, "ax"
@@ -42,40 +41,31 @@
9
-/* __copy_from_user_inatomic takes the kernel target address in r0,
+/* raw_copy_from_user takes the kernel target address in r0,
* the user source in r1, and the bytes to copy in r2.
* It returns the number of uncopiable bytes (hopefully zero) in r0.
*/
-ENTRY(__copy_from_user_inatomic)
-.type __copy_from_user_inatomic, @function
- FEEDBACK_ENTER_EXPLICIT(__copy_from_user_inatomic, \
+ENTRY(raw_copy_from_user)
+.type raw_copy_from_user, @function
+ FEEDBACK_ENTER_EXPLICIT(raw_copy_from_user, \
.text.memcpy_common, \
- .Lend_memcpy_common - __copy_from_user_inatomic)
+ .Lend_memcpy_common - raw_copy_from_user)
{ movei r29, IS_COPY_FROM_USER; j memcpy_common }
- .size __copy_from_user_inatomic, . - __copy_from_user_inatomic
+ .size raw_copy_from_user, . - raw_copy_from_user
-/* __copy_from_user_zeroing is like __copy_from_user_inatomic, but
- * any uncopiable bytes are zeroed in the target.
- */
-ENTRY(__copy_from_user_zeroing)
-.type __copy_from_user_zeroing, @function
- FEEDBACK_REENTER(__copy_from_user_inatomic)
- { movei r29, IS_COPY_FROM_USER_ZEROING; j memcpy_common }
- .size __copy_from_user_zeroing, . - __copy_from_user_zeroing
-
-/* __copy_to_user_inatomic takes the user target address in r0,
+/* raw_copy_to_user takes the user target address in r0,
* the kernel source in r1, and the bytes to copy in r2.
* It returns the number of uncopiable bytes (hopefully zero) in r0.
*/
-ENTRY(__copy_to_user_inatomic)
-.type __copy_to_user_inatomic, @function
- FEEDBACK_REENTER(__copy_from_user_inatomic)
+ENTRY(raw_copy_to_user)
+.type raw_copy_to_user, @function
+ FEEDBACK_REENTER(raw_copy_from_user)
{ movei r29, IS_COPY_TO_USER; j memcpy_common }
- .size __copy_to_user_inatomic, . - __copy_to_user_inatomic
+ .size raw_copy_to_user, . - raw_copy_to_user
ENTRY(memcpy)
.type memcpy, @function
- FEEDBACK_REENTER(__copy_from_user_inatomic)
+ FEEDBACK_REENTER(raw_copy_from_user)
{ movei r29, IS_MEMCPY }
.size memcpy, . - memcpy
/* Fall through */
@@ -520,12 +510,7 @@ copy_from_user_fixup_loop:
{ bnzt r2, copy_from_user_fixup_loop }
.Lcopy_from_user_fixup_zero_remainder:
- { bbs r29, 2f } /* low bit set means IS_COPY_FROM_USER */
- /* byte-at-a-time loop faulted, so zero the rest. */
- { move r3, r2; bz r2, 2f /* should be impossible, but handle it. */ }
-1: { sb r0, zero; addi r0, r0, 1; addi r3, r3, -1 }
- { bnzt r3, 1b }
-2: move lr, r27
+ move lr, r27
{ move r0, r2; jrp lr }
copy_to_user_fixup_loop:
diff --git a/arch/tile/lib/memcpy_user_64.c b/arch/tile/lib/memcpy_user_64.c
index 97bbb6060b25..a3fea9fd973e 100644
--- a/arch/tile/lib/memcpy_user_64.c
+++ b/arch/tile/lib/memcpy_user_64.c
@@ -51,7 +51,7 @@
__v; \
})
-#define USERCOPY_FUNC __copy_to_user_inatomic
+#define USERCOPY_FUNC raw_copy_to_user
#define ST1(p, v) _ST((p), st1, (v))
#define ST2(p, v) _ST((p), st2, (v))
#define ST4(p, v) _ST((p), st4, (v))
@@ -62,7 +62,7 @@
#define LD8 LD
#include "memcpy_64.c"
-#define USERCOPY_FUNC __copy_from_user_inatomic
+#define USERCOPY_FUNC raw_copy_from_user
#define ST1 ST
#define ST2 ST
#define ST4 ST
@@ -73,7 +73,7 @@
#define LD8(p) _LD((p), ld)
#include "memcpy_64.c"
-#define USERCOPY_FUNC __copy_in_user_inatomic
+#define USERCOPY_FUNC raw_copy_in_user
#define ST1(p, v) _ST((p), st1, (v))
#define ST2(p, v) _ST((p), st2, (v))
#define ST4(p, v) _ST((p), st4, (v))
@@ -83,12 +83,3 @@
#define LD4(p) _LD((p), ld4u)
#define LD8(p) _LD((p), ld)
#include "memcpy_64.c"
-
-unsigned long __copy_from_user_zeroing(void *to, const void __user *from,
- unsigned long n)
-{
- unsigned long rc = __copy_from_user_inatomic(to, from, n);
- if (unlikely(rc))
- memset(to + n - rc, 0, rc);
- return rc;
-}
diff --git a/arch/um/Kconfig.common b/arch/um/Kconfig.common
index fd443852103c..e13f763849b1 100644
--- a/arch/um/Kconfig.common
+++ b/arch/um/Kconfig.common
@@ -13,6 +13,7 @@ config UML
select GENERIC_CLOCKEVENTS
select HAVE_GCC_PLUGINS
select TTY # Needed for line.c
+ select ARCH_HAS_RAW_COPY_USER
config MMU
bool
diff --git a/arch/um/include/asm/uaccess.h b/arch/um/include/asm/uaccess.h
index 9396c8a4e920..cc00fc50768f 100644
--- a/arch/um/include/asm/uaccess.h
+++ b/arch/um/include/asm/uaccess.h
@@ -21,8 +21,8 @@
#define __addr_range_nowrap(addr, size) \
((unsigned long) (addr) <= ((unsigned long) (addr) + (size)))
-extern long __copy_from_user(void *to, const void __user *from, unsigned long n);
-extern long __copy_to_user(void __user *to, const void *from, unsigned long n);
+extern unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n);
+extern unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n);
extern long __strncpy_from_user(char *dst, const char __user *src, long count);
extern long __strnlen_user(const void __user *str, long len);
extern unsigned long __clear_user(void __user *mem, unsigned long len);
@@ -33,8 +33,8 @@ static inline int __access_ok(unsigned long addr, unsigned long size);
#define __clear_user __clear_user
#define __strnlen_user __strnlen_user
#define __strncpy_from_user __strncpy_from_user
-#define __copy_to_user_inatomic __copy_to_user
-#define __copy_from_user_inatomic __copy_from_user
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
#include <asm-generic/uaccess.h>
diff --git a/arch/um/kernel/skas/uaccess.c b/arch/um/kernel/skas/uaccess.c
index 22c9f79db8e6..d450797a3a7c 100644
--- a/arch/um/kernel/skas/uaccess.c
+++ b/arch/um/kernel/skas/uaccess.c
@@ -139,7 +139,7 @@ static int copy_chunk_from_user(unsigned long from, int len, void *arg)
return 0;
}
-long __copy_from_user(void *to, const void __user *from, unsigned long n)
+unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
if (uaccess_kernel()) {
memcpy(to, (__force void*)from, n);
@@ -148,7 +148,7 @@ long __copy_from_user(void *to, const void __user *from, unsigned long n)
return buffer_op((unsigned long) from, n, 0, copy_chunk_from_user, &to);
}
-EXPORT_SYMBOL(__copy_from_user);
+EXPORT_SYMBOL(raw_copy_from_user);
static int copy_chunk_to_user(unsigned long to, int len, void *arg)
{
@@ -159,7 +159,7 @@ static int copy_chunk_to_user(unsigned long to, int len, void *arg)
return 0;
}
-long __copy_to_user(void __user *to, const void *from, unsigned long n)
+unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
if (uaccess_kernel()) {
memcpy((__force void *) to, from, n);
@@ -168,7 +168,7 @@ long __copy_to_user(void __user *to, const void *from, unsigned long n)
return buffer_op((unsigned long) to, n, 1, copy_chunk_to_user, &from);
}
-EXPORT_SYMBOL(__copy_to_user);
+EXPORT_SYMBOL(raw_copy_to_user);
static int strncpy_chunk_from_user(unsigned long from, int len, void *arg)
{
diff --git a/arch/unicore32/Kconfig b/arch/unicore32/Kconfig
index 0769066929c6..319519eaed24 100644
--- a/arch/unicore32/Kconfig
+++ b/arch/unicore32/Kconfig
@@ -18,6 +18,7 @@ config UNICORE32
select ARCH_WANT_FRAME_POINTERS
select GENERIC_IOMAP
select MODULES_USE_ELF_REL
+ select ARCH_HAS_RAW_COPY_USER
help
UniCore-32 is 32-bit Instruction Set Architecture,
including a series of low-power-consumption RISC chip
diff --git a/arch/unicore32/include/asm/uaccess.h b/arch/unicore32/include/asm/uaccess.h
index b2bcab69dfee..1d55f2f83759 100644
--- a/arch/unicore32/include/asm/uaccess.h
+++ b/arch/unicore32/include/asm/uaccess.h
@@ -24,15 +24,17 @@
#define __access_ok(addr, size) (__kernel_ok || __user_ok((addr), (size)))
extern unsigned long __must_check
-__copy_from_user(void *to, const void __user *from, unsigned long n);
+raw_copy_from_user(void *to, const void __user *from, unsigned long n);
extern unsigned long __must_check
-__copy_to_user(void __user *to, const void *from, unsigned long n);
+raw_copy_to_user(void __user *to, const void *from, unsigned long n);
extern unsigned long __must_check
__clear_user(void __user *addr, unsigned long n);
extern unsigned long __must_check
__strncpy_from_user(char *to, const char __user *from, unsigned long count);
extern unsigned long
__strnlen_user(const char __user *s, long n);
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
#include <asm-generic/uaccess.h>
diff --git a/arch/unicore32/kernel/ksyms.c b/arch/unicore32/kernel/ksyms.c
index 0323528a80fd..dcc72ee1fcdb 100644
--- a/arch/unicore32/kernel/ksyms.c
+++ b/arch/unicore32/kernel/ksyms.c
@@ -46,8 +46,8 @@ EXPORT_SYMBOL(__strncpy_from_user);
EXPORT_SYMBOL(copy_page);
-EXPORT_SYMBOL(__copy_from_user);
-EXPORT_SYMBOL(__copy_to_user);
+EXPORT_SYMBOL(raw_copy_from_user);
+EXPORT_SYMBOL(raw_copy_to_user);
EXPORT_SYMBOL(__clear_user);
EXPORT_SYMBOL(__ashldi3);
diff --git a/arch/unicore32/lib/copy_from_user.S b/arch/unicore32/lib/copy_from_user.S
index ab0767ea5dbd..5f80fcbe8631 100644
--- a/arch/unicore32/lib/copy_from_user.S
+++ b/arch/unicore32/lib/copy_from_user.S
@@ -16,7 +16,7 @@
/*
* Prototype:
*
- * size_t __copy_from_user(void *to, const void *from, size_t n)
+ * size_t raw_copy_from_user(void *to, const void *from, size_t n)
*
* Purpose:
*
@@ -87,22 +87,18 @@
.text
-ENTRY(__copy_from_user)
+ENTRY(raw_copy_from_user)
#include "copy_template.S"
-ENDPROC(__copy_from_user)
+ENDPROC(raw_copy_from_user)
.pushsection .fixup,"ax"
.align 0
copy_abort_preamble
- ldm.w (r1, r2), [sp]+
- sub r3, r0, r1
- rsub r2, r3, r2
- stw r2, [sp]
- mov r1, #0
- b.l memset
- ldw.w r0, [sp]+, #4
+ ldm.w (r1, r2, r3), [sp]+
+ sub r0, r0, r1
+ rsub r0, r0, r2
copy_abort_end
.popsection
diff --git a/arch/unicore32/lib/copy_to_user.S b/arch/unicore32/lib/copy_to_user.S
index 6e22151c840d..857c6816ffe7 100644
--- a/arch/unicore32/lib/copy_to_user.S
+++ b/arch/unicore32/lib/copy_to_user.S
@@ -16,7 +16,7 @@
/*
* Prototype:
*
- * size_t __copy_to_user(void *to, const void *from, size_t n)
+ * size_t raw_copy_to_user(void *to, const void *from, size_t n)
*
* Purpose:
*
@@ -79,11 +79,11 @@
.text
-WEAK(__copy_to_user)
+WEAK(raw_copy_to_user)
#include "copy_template.S"
-ENDPROC(__copy_to_user)
+ENDPROC(raw_copy_to_user)
.pushsection .fixup,"ax"
.align 0
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index cc98d5a294ee..5f59fc388063 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -175,6 +175,7 @@ config X86
select USER_STACKTRACE_SUPPORT
select VIRT_TO_BUS
select X86_FEATURE_NAMES if PROC_FS
+ select ARCH_HAS_RAW_COPY_USER
config INSTRUCTION_DECODER
def_bool y
diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index 0522d88a7f90..68766b276d9e 100644
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -379,6 +379,18 @@ do { \
: "=r" (err), ltype(x) \
: "m" (__m(addr)), "i" (errret), "0" (err))
+#define __get_user_asm_nozero(x, addr, err, itype, rtype, ltype, errret) \
+ asm volatile("\n" \
+ "1: mov"itype" %2,%"rtype"1\n" \
+ "2:\n" \
+ ".section .fixup,\"ax\"\n" \
+ "3: mov %3,%0\n" \
+ " jmp 2b\n" \
+ ".previous\n" \
+ _ASM_EXTABLE(1b, 3b) \
+ : "=r" (err), ltype(x) \
+ : "m" (__m(addr)), "i" (errret), "0" (err))
+
/*
* This doesn't do __uaccess_begin/end - the exception handling
* around it must do that.
@@ -670,59 +682,6 @@ extern struct movsl_mask {
# include <asm/uaccess_64.h>
#endif
-unsigned long __must_check _copy_from_user(void *to, const void __user *from,
- unsigned n);
-unsigned long __must_check _copy_to_user(void __user *to, const void *from,
- unsigned n);
-
-extern void __compiletime_error("usercopy buffer size is too small")
-__bad_copy_user(void);
-
-static inline void copy_user_overflow(int size, unsigned long count)
-{
- WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count);
-}
-
-static __always_inline unsigned long __must_check
-copy_from_user(void *to, const void __user *from, unsigned long n)
-{
- int sz = __compiletime_object_size(to);
-
- might_fault();
-
- kasan_check_write(to, n);
-
- if (likely(sz < 0 || sz >= n)) {
- check_object_size(to, n, false);
- n = _copy_from_user(to, from, n);
- } else if (!__builtin_constant_p(n))
- copy_user_overflow(sz, n);
- else
- __bad_copy_user();
-
- return n;
-}
-
-static __always_inline unsigned long __must_check
-copy_to_user(void __user *to, const void *from, unsigned long n)
-{
- int sz = __compiletime_object_size(from);
-
- kasan_check_read(from, n);
-
- might_fault();
-
- if (likely(sz < 0 || sz >= n)) {
- check_object_size(from, n, true);
- n = _copy_to_user(to, from, n);
- } else if (!__builtin_constant_p(n))
- copy_user_overflow(sz, n);
- else
- __bad_copy_user();
-
- return n;
-}
-
/*
* We rely on the nested NMI work to allow atomic faults from the NMI path; the
* nested NMI paths are careful to preserve CR2.
diff --git a/arch/x86/include/asm/uaccess_32.h b/arch/x86/include/asm/uaccess_32.h
index 5268ecceea96..aeda9bb8af50 100644
--- a/arch/x86/include/asm/uaccess_32.h
+++ b/arch/x86/include/asm/uaccess_32.h
@@ -8,143 +8,48 @@
#include <asm/asm.h>
#include <asm/page.h>
-unsigned long __must_check __copy_to_user_ll
- (void __user *to, const void *from, unsigned long n);
-unsigned long __must_check __copy_from_user_ll
- (void *to, const void __user *from, unsigned long n);
-unsigned long __must_check __copy_from_user_ll_nozero
- (void *to, const void __user *from, unsigned long n);
-unsigned long __must_check __copy_from_user_ll_nocache
- (void *to, const void __user *from, unsigned long n);
+unsigned long __must_check __copy_user_ll
+ (void *to, const void *from, unsigned long n);
unsigned long __must_check __copy_from_user_ll_nocache_nozero
(void *to, const void __user *from, unsigned long n);
-/**
- * __copy_to_user_inatomic: - Copy a block of data into user space, with less checking.
- * @to: Destination address, in user space.
- * @from: Source address, in kernel space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only.
- *
- * Copy data from kernel space to user space. Caller must check
- * the specified block with access_ok() before calling this function.
- * The caller should also make sure he pins the user space address
- * so that we don't result in page fault and sleep.
- */
-static __always_inline unsigned long __must_check
-__copy_to_user_inatomic(void __user *to, const void *from, unsigned long n)
-{
- check_object_size(from, n, true);
- return __copy_to_user_ll(to, from, n);
-}
-
-/**
- * __copy_to_user: - Copy a block of data into user space, with less checking.
- * @to: Destination address, in user space.
- * @from: Source address, in kernel space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from kernel space to user space. Caller must check
- * the specified block with access_ok() before calling this function.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- */
static __always_inline unsigned long __must_check
-__copy_to_user(void __user *to, const void *from, unsigned long n)
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
- might_fault();
- return __copy_to_user_inatomic(to, from, n);
+ return __copy_user_ll((__force void *)to, from, n);
}
static __always_inline unsigned long
-__copy_from_user_inatomic(void *to, const void __user *from, unsigned long n)
-{
- return __copy_from_user_ll_nozero(to, from, n);
-}
-
-/**
- * __copy_from_user: - Copy a block of data from user space, with less checking.
- * @to: Destination address, in kernel space.
- * @from: Source address, in user space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from user space to kernel space. Caller must check
- * the specified block with access_ok() before calling this function.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- *
- * If some data could not be copied, this function will pad the copied
- * data to the requested size using zero bytes.
- *
- * An alternate version - __copy_from_user_inatomic() - may be called from
- * atomic context and will fail rather than sleep. In this case the
- * uncopied bytes will *NOT* be padded with zeros. See fs/filemap.h
- * for explanation of why this is needed.
- */
-static __always_inline unsigned long
-__copy_from_user(void *to, const void __user *from, unsigned long n)
-{
- might_fault();
- check_object_size(to, n, false);
- if (__builtin_constant_p(n)) {
- unsigned long ret;
-
- switch (n) {
- case 1:
- __uaccess_begin();
- __get_user_size(*(u8 *)to, from, 1, ret, 1);
- __uaccess_end();
- return ret;
- case 2:
- __uaccess_begin();
- __get_user_size(*(u16 *)to, from, 2, ret, 2);
- __uaccess_end();
- return ret;
- case 4:
- __uaccess_begin();
- __get_user_size(*(u32 *)to, from, 4, ret, 4);
- __uaccess_end();
- return ret;
- }
- }
- return __copy_from_user_ll(to, from, n);
-}
-
-static __always_inline unsigned long __copy_from_user_nocache(void *to,
- const void __user *from, unsigned long n)
+raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
- might_fault();
if (__builtin_constant_p(n)) {
unsigned long ret;
switch (n) {
case 1:
+ ret = 0;
__uaccess_begin();
- __get_user_size(*(u8 *)to, from, 1, ret, 1);
+ __get_user_asm_nozero(*(u8 *)to, from, ret,
+ "b", "b", "=q", 1);
__uaccess_end();
return ret;
case 2:
+ ret = 0;
__uaccess_begin();
- __get_user_size(*(u16 *)to, from, 2, ret, 2);
+ __get_user_asm_nozero(*(u16 *)to, from, ret,
+ "w", "w", "=r", 2);
__uaccess_end();
return ret;
case 4:
+ ret = 0;
__uaccess_begin();
- __get_user_size(*(u32 *)to, from, 4, ret, 4);
+ __get_user_asm_nozero(*(u32 *)to, from, ret,
+ "l", "k", "=r", 4);
__uaccess_end();
return ret;
}
}
- return __copy_from_user_ll_nocache(to, from, n);
+ return __copy_user_ll(to, (__force const void *)from, n);
}
static __always_inline unsigned long
diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index 8ddadd93639e..c5504b9a472e 100644
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -45,58 +45,54 @@ copy_user_generic(void *to, const void *from, unsigned len)
return ret;
}
-__must_check unsigned long
-copy_in_user(void __user *to, const void __user *from, unsigned len);
-
-static __always_inline __must_check
-int __copy_from_user_nocheck(void *dst, const void __user *src, unsigned size)
+static __always_inline __must_check unsigned long
+raw_copy_from_user(void *dst, const void __user *src, unsigned long size)
{
int ret = 0;
- check_object_size(dst, size, false);
if (!__builtin_constant_p(size))
return copy_user_generic(dst, (__force void *)src, size);
switch (size) {
case 1:
__uaccess_begin();
- __get_user_asm(*(u8 *)dst, (u8 __user *)src,
+ __get_user_asm_nozero(*(u8 *)dst, (u8 __user *)src,
ret, "b", "b", "=q", 1);
__uaccess_end();
return ret;
case 2:
__uaccess_begin();
- __get_user_asm(*(u16 *)dst, (u16 __user *)src,
+ __get_user_asm_nozero(*(u16 *)dst, (u16 __user *)src,
ret, "w", "w", "=r", 2);
__uaccess_end();
return ret;
case 4:
__uaccess_begin();
- __get_user_asm(*(u32 *)dst, (u32 __user *)src,
+ __get_user_asm_nozero(*(u32 *)dst, (u32 __user *)src,
ret, "l", "k", "=r", 4);
__uaccess_end();
return ret;
case 8:
__uaccess_begin();
- __get_user_asm(*(u64 *)dst, (u64 __user *)src,
+ __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src,
ret, "q", "", "=r", 8);
__uaccess_end();
return ret;
case 10:
__uaccess_begin();
- __get_user_asm(*(u64 *)dst, (u64 __user *)src,
+ __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src,
ret, "q", "", "=r", 10);
if (likely(!ret))
- __get_user_asm(*(u16 *)(8 + (char *)dst),
+ __get_user_asm_nozero(*(u16 *)(8 + (char *)dst),
(u16 __user *)(8 + (char __user *)src),
ret, "w", "w", "=r", 2);
__uaccess_end();
return ret;
case 16:
__uaccess_begin();
- __get_user_asm(*(u64 *)dst, (u64 __user *)src,
+ __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src,
ret, "q", "", "=r", 16);
if (likely(!ret))
- __get_user_asm(*(u64 *)(8 + (char *)dst),
+ __get_user_asm_nozero(*(u64 *)(8 + (char *)dst),
(u64 __user *)(8 + (char __user *)src),
ret, "q", "", "=r", 8);
__uaccess_end();
@@ -106,20 +102,11 @@ int __copy_from_user_nocheck(void *dst, const void __user *src, unsigned size)
}
}
-static __always_inline __must_check
-int __copy_from_user(void *dst, const void __user *src, unsigned size)
-{
- might_fault();
- kasan_check_write(dst, size);
- return __copy_from_user_nocheck(dst, src, size);
-}
-
-static __always_inline __must_check
-int __copy_to_user_nocheck(void __user *dst, const void *src, unsigned size)
+static __always_inline __must_check unsigned long
+raw_copy_to_user(void __user *dst, const void *src, unsigned long size)
{
int ret = 0;
- check_object_size(src, size, true);
if (!__builtin_constant_p(size))
return copy_user_generic((__force void *)dst, src, size);
switch (size) {
@@ -175,100 +162,16 @@ int __copy_to_user_nocheck(void __user *dst, const void *src, unsigned size)
}
static __always_inline __must_check
-int __copy_to_user(void __user *dst, const void *src, unsigned size)
-{
- might_fault();
- kasan_check_read(src, size);
- return __copy_to_user_nocheck(dst, src, size);
-}
-
-static __always_inline __must_check
-int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
-{
- int ret = 0;
-
- might_fault();
- if (!__builtin_constant_p(size))
- return copy_user_generic((__force void *)dst,
- (__force void *)src, size);
- switch (size) {
- case 1: {
- u8 tmp;
- __uaccess_begin();
- __get_user_asm(tmp, (u8 __user *)src,
- ret, "b", "b", "=q", 1);
- if (likely(!ret))
- __put_user_asm(tmp, (u8 __user *)dst,
- ret, "b", "b", "iq", 1);
- __uaccess_end();
- return ret;
- }
- case 2: {
- u16 tmp;
- __uaccess_begin();
- __get_user_asm(tmp, (u16 __user *)src,
- ret, "w", "w", "=r", 2);
- if (likely(!ret))
- __put_user_asm(tmp, (u16 __user *)dst,
- ret, "w", "w", "ir", 2);
- __uaccess_end();
- return ret;
- }
-
- case 4: {
- u32 tmp;
- __uaccess_begin();
- __get_user_asm(tmp, (u32 __user *)src,
- ret, "l", "k", "=r", 4);
- if (likely(!ret))
- __put_user_asm(tmp, (u32 __user *)dst,
- ret, "l", "k", "ir", 4);
- __uaccess_end();
- return ret;
- }
- case 8: {
- u64 tmp;
- __uaccess_begin();
- __get_user_asm(tmp, (u64 __user *)src,
- ret, "q", "", "=r", 8);
- if (likely(!ret))
- __put_user_asm(tmp, (u64 __user *)dst,
- ret, "q", "", "er", 8);
- __uaccess_end();
- return ret;
- }
- default:
- return copy_user_generic((__force void *)dst,
- (__force void *)src, size);
- }
-}
-
-static __must_check __always_inline int
-__copy_from_user_inatomic(void *dst, const void __user *src, unsigned size)
-{
- kasan_check_write(dst, size);
- return __copy_from_user_nocheck(dst, src, size);
-}
-
-static __must_check __always_inline int
-__copy_to_user_inatomic(void __user *dst, const void *src, unsigned size)
+unsigned long raw_copy_in_user(void __user *dst, const void __user *src, unsigned long size)
{
- kasan_check_read(src, size);
- return __copy_to_user_nocheck(dst, src, size);
+ return copy_user_generic((__force void *)dst,
+ (__force void *)src, size);
}
extern long __copy_user_nocache(void *dst, const void __user *src,
unsigned size, int zerorest);
static inline int
-__copy_from_user_nocache(void *dst, const void __user *src, unsigned size)
-{
- might_fault();
- kasan_check_write(dst, size);
- return __copy_user_nocache(dst, src, size, 1);
-}
-
-static inline int
__copy_from_user_inatomic_nocache(void *dst, const void __user *src,
unsigned size)
{
diff --git a/arch/x86/lib/usercopy.c b/arch/x86/lib/usercopy.c
index c074799bddae..c8c6ad0d58b8 100644
--- a/arch/x86/lib/usercopy.c
+++ b/arch/x86/lib/usercopy.c
@@ -4,12 +4,9 @@
* For licencing details see kernel-base/COPYING
*/
-#include <linux/highmem.h>
+#include <linux/uaccess.h>
#include <linux/export.h>
-#include <asm/word-at-a-time.h>
-#include <linux/sched.h>
-
/*
* We rely on the nested NMI work to allow atomic faults from the NMI path; the
* nested NMI paths are careful to preserve CR2.
@@ -34,52 +31,3 @@ copy_from_user_nmi(void *to, const void __user *from, unsigned long n)
return ret;
}
EXPORT_SYMBOL_GPL(copy_from_user_nmi);
-
-/**
- * copy_to_user: - Copy a block of data into user space.
- * @to: Destination address, in user space.
- * @from: Source address, in kernel space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from kernel space to user space.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- */
-unsigned long _copy_to_user(void __user *to, const void *from, unsigned n)
-{
- if (access_ok(VERIFY_WRITE, to, n))
- n = __copy_to_user(to, from, n);
- return n;
-}
-EXPORT_SYMBOL(_copy_to_user);
-
-/**
- * copy_from_user: - Copy a block of data from user space.
- * @to: Destination address, in kernel space.
- * @from: Source address, in user space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Copy data from user space to kernel space.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- *
- * If some data could not be copied, this function will pad the copied
- * data to the requested size using zero bytes.
- */
-unsigned long _copy_from_user(void *to, const void __user *from, unsigned n)
-{
- if (access_ok(VERIFY_READ, from, n))
- n = __copy_from_user(to, from, n);
- else
- memset(to, 0, n);
- return n;
-}
-EXPORT_SYMBOL(_copy_from_user);
diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c
index 1f65ff6540f0..bd057a4ffe6e 100644
--- a/arch/x86/lib/usercopy_32.c
+++ b/arch/x86/lib/usercopy_32.c
@@ -5,12 +5,7 @@
* Copyright 1997 Andi Kleen <ak@muc.de>
* Copyright 1997 Linus Torvalds
*/
-#include <linux/mm.h>
-#include <linux/highmem.h>
-#include <linux/blkdev.h>
#include <linux/export.h>
-#include <linux/backing-dev.h>
-#include <linux/interrupt.h>
#include <linux/uaccess.h>
#include <asm/mmx.h>
#include <asm/asm.h>
@@ -201,197 +196,6 @@ __copy_user_intel(void __user *to, const void *from, unsigned long size)
return size;
}
-static unsigned long
-__copy_user_zeroing_intel(void *to, const void __user *from, unsigned long size)
-{
- int d0, d1;
- __asm__ __volatile__(
- " .align 2,0x90\n"
- "0: movl 32(%4), %%eax\n"
- " cmpl $67, %0\n"
- " jbe 2f\n"
- "1: movl 64(%4), %%eax\n"
- " .align 2,0x90\n"
- "2: movl 0(%4), %%eax\n"
- "21: movl 4(%4), %%edx\n"
- " movl %%eax, 0(%3)\n"
- " movl %%edx, 4(%3)\n"
- "3: movl 8(%4), %%eax\n"
- "31: movl 12(%4),%%edx\n"
- " movl %%eax, 8(%3)\n"
- " movl %%edx, 12(%3)\n"
- "4: movl 16(%4), %%eax\n"
- "41: movl 20(%4), %%edx\n"
- " movl %%eax, 16(%3)\n"
- " movl %%edx, 20(%3)\n"
- "10: movl 24(%4), %%eax\n"
- "51: movl 28(%4), %%edx\n"
- " movl %%eax, 24(%3)\n"
- " movl %%edx, 28(%3)\n"
- "11: movl 32(%4), %%eax\n"
- "61: movl 36(%4), %%edx\n"
- " movl %%eax, 32(%3)\n"
- " movl %%edx, 36(%3)\n"
- "12: movl 40(%4), %%eax\n"
- "71: movl 44(%4), %%edx\n"
- " movl %%eax, 40(%3)\n"
- " movl %%edx, 44(%3)\n"
- "13: movl 48(%4), %%eax\n"
- "81: movl 52(%4), %%edx\n"
- " movl %%eax, 48(%3)\n"
- " movl %%edx, 52(%3)\n"
- "14: movl 56(%4), %%eax\n"
- "91: movl 60(%4), %%edx\n"
- " movl %%eax, 56(%3)\n"
- " movl %%edx, 60(%3)\n"
- " addl $-64, %0\n"
- " addl $64, %4\n"
- " addl $64, %3\n"
- " cmpl $63, %0\n"
- " ja 0b\n"
- "5: movl %0, %%eax\n"
- " shrl $2, %0\n"
- " andl $3, %%eax\n"
- " cld\n"
- "6: rep; movsl\n"
- " movl %%eax,%0\n"
- "7: rep; movsb\n"
- "8:\n"
- ".section .fixup,\"ax\"\n"
- "9: lea 0(%%eax,%0,4),%0\n"
- "16: pushl %0\n"
- " pushl %%eax\n"
- " xorl %%eax,%%eax\n"
- " rep; stosb\n"
- " popl %%eax\n"
- " popl %0\n"
- " jmp 8b\n"
- ".previous\n"
- _ASM_EXTABLE(0b,16b)
- _ASM_EXTABLE(1b,16b)
- _ASM_EXTABLE(2b,16b)
- _ASM_EXTABLE(21b,16b)
- _ASM_EXTABLE(3b,16b)
- _ASM_EXTABLE(31b,16b)
- _ASM_EXTABLE(4b,16b)
- _ASM_EXTABLE(41b,16b)
- _ASM_EXTABLE(10b,16b)
- _ASM_EXTABLE(51b,16b)
- _ASM_EXTABLE(11b,16b)
- _ASM_EXTABLE(61b,16b)
- _ASM_EXTABLE(12b,16b)
- _ASM_EXTABLE(71b,16b)
- _ASM_EXTABLE(13b,16b)
- _ASM_EXTABLE(81b,16b)
- _ASM_EXTABLE(14b,16b)
- _ASM_EXTABLE(91b,16b)
- _ASM_EXTABLE(6b,9b)
- _ASM_EXTABLE(7b,16b)
- : "=&c"(size), "=&D" (d0), "=&S" (d1)
- : "1"(to), "2"(from), "0"(size)
- : "eax", "edx", "memory");
- return size;
-}
-
-/*
- * Non Temporal Hint version of __copy_user_zeroing_intel. It is cache aware.
- * hyoshiok@miraclelinux.com
- */
-
-static unsigned long __copy_user_zeroing_intel_nocache(void *to,
- const void __user *from, unsigned long size)
-{
- int d0, d1;
-
- __asm__ __volatile__(
- " .align 2,0x90\n"
- "0: movl 32(%4), %%eax\n"
- " cmpl $67, %0\n"
- " jbe 2f\n"
- "1: movl 64(%4), %%eax\n"
- " .align 2,0x90\n"
- "2: movl 0(%4), %%eax\n"
- "21: movl 4(%4), %%edx\n"
- " movnti %%eax, 0(%3)\n"
- " movnti %%edx, 4(%3)\n"
- "3: movl 8(%4), %%eax\n"
- "31: movl 12(%4),%%edx\n"
- " movnti %%eax, 8(%3)\n"
- " movnti %%edx, 12(%3)\n"
- "4: movl 16(%4), %%eax\n"
- "41: movl 20(%4), %%edx\n"
- " movnti %%eax, 16(%3)\n"
- " movnti %%edx, 20(%3)\n"
- "10: movl 24(%4), %%eax\n"
- "51: movl 28(%4), %%edx\n"
- " movnti %%eax, 24(%3)\n"
- " movnti %%edx, 28(%3)\n"
- "11: movl 32(%4), %%eax\n"
- "61: movl 36(%4), %%edx\n"
- " movnti %%eax, 32(%3)\n"
- " movnti %%edx, 36(%3)\n"
- "12: movl 40(%4), %%eax\n"
- "71: movl 44(%4), %%edx\n"
- " movnti %%eax, 40(%3)\n"
- " movnti %%edx, 44(%3)\n"
- "13: movl 48(%4), %%eax\n"
- "81: movl 52(%4), %%edx\n"
- " movnti %%eax, 48(%3)\n"
- " movnti %%edx, 52(%3)\n"
- "14: movl 56(%4), %%eax\n"
- "91: movl 60(%4), %%edx\n"
- " movnti %%eax, 56(%3)\n"
- " movnti %%edx, 60(%3)\n"
- " addl $-64, %0\n"
- " addl $64, %4\n"
- " addl $64, %3\n"
- " cmpl $63, %0\n"
- " ja 0b\n"
- " sfence \n"
- "5: movl %0, %%eax\n"
- " shrl $2, %0\n"
- " andl $3, %%eax\n"
- " cld\n"
- "6: rep; movsl\n"
- " movl %%eax,%0\n"
- "7: rep; movsb\n"
- "8:\n"
- ".section .fixup,\"ax\"\n"
- "9: lea 0(%%eax,%0,4),%0\n"
- "16: pushl %0\n"
- " pushl %%eax\n"
- " xorl %%eax,%%eax\n"
- " rep; stosb\n"
- " popl %%eax\n"
- " popl %0\n"
- " jmp 8b\n"
- ".previous\n"
- _ASM_EXTABLE(0b,16b)
- _ASM_EXTABLE(1b,16b)
- _ASM_EXTABLE(2b,16b)
- _ASM_EXTABLE(21b,16b)
- _ASM_EXTABLE(3b,16b)
- _ASM_EXTABLE(31b,16b)
- _ASM_EXTABLE(4b,16b)
- _ASM_EXTABLE(41b,16b)
- _ASM_EXTABLE(10b,16b)
- _ASM_EXTABLE(51b,16b)
- _ASM_EXTABLE(11b,16b)
- _ASM_EXTABLE(61b,16b)
- _ASM_EXTABLE(12b,16b)
- _ASM_EXTABLE(71b,16b)
- _ASM_EXTABLE(13b,16b)
- _ASM_EXTABLE(81b,16b)
- _ASM_EXTABLE(14b,16b)
- _ASM_EXTABLE(91b,16b)
- _ASM_EXTABLE(6b,9b)
- _ASM_EXTABLE(7b,16b)
- : "=&c"(size), "=&D" (d0), "=&S" (d1)
- : "1"(to), "2"(from), "0"(size)
- : "eax", "edx", "memory");
- return size;
-}
-
static unsigned long __copy_user_intel_nocache(void *to,
const void __user *from, unsigned long size)
{
@@ -486,12 +290,8 @@ static unsigned long __copy_user_intel_nocache(void *to,
* Leave these declared but undefined. They should not be any references to
* them
*/
-unsigned long __copy_user_zeroing_intel(void *to, const void __user *from,
- unsigned long size);
unsigned long __copy_user_intel(void __user *to, const void *from,
unsigned long size);
-unsigned long __copy_user_zeroing_intel_nocache(void *to,
- const void __user *from, unsigned long size);
#endif /* CONFIG_X86_INTEL_USERCOPY */
/* Generic arbitrary sized copy. */
@@ -528,47 +328,7 @@ do { \
: "memory"); \
} while (0)
-#define __copy_user_zeroing(to, from, size) \
-do { \
- int __d0, __d1, __d2; \
- __asm__ __volatile__( \
- " cmp $7,%0\n" \
- " jbe 1f\n" \
- " movl %1,%0\n" \
- " negl %0\n" \
- " andl $7,%0\n" \
- " subl %0,%3\n" \
- "4: rep; movsb\n" \
- " movl %3,%0\n" \
- " shrl $2,%0\n" \
- " andl $3,%3\n" \
- " .align 2,0x90\n" \
- "0: rep; movsl\n" \
- " movl %3,%0\n" \
- "1: rep; movsb\n" \
- "2:\n" \
- ".section .fixup,\"ax\"\n" \
- "5: addl %3,%0\n" \
- " jmp 6f\n" \
- "3: lea 0(%3,%0,4),%0\n" \
- "6: pushl %0\n" \
- " pushl %%eax\n" \
- " xorl %%eax,%%eax\n" \
- " rep; stosb\n" \
- " popl %%eax\n" \
- " popl %0\n" \
- " jmp 2b\n" \
- ".previous\n" \
- _ASM_EXTABLE(4b,5b) \
- _ASM_EXTABLE(0b,3b) \
- _ASM_EXTABLE(1b,6b) \
- : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \
- : "3"(size), "0"(size), "1"(to), "2"(from) \
- : "memory"); \
-} while (0)
-
-unsigned long __copy_to_user_ll(void __user *to, const void *from,
- unsigned long n)
+unsigned long __copy_user_ll(void *to, const void *from, unsigned long n)
{
stac();
if (movsl_is_ok(to, from, n))
@@ -578,51 +338,7 @@ unsigned long __copy_to_user_ll(void __user *to, const void *from,
clac();
return n;
}
-EXPORT_SYMBOL(__copy_to_user_ll);
-
-unsigned long __copy_from_user_ll(void *to, const void __user *from,
- unsigned long n)
-{
- stac();
- if (movsl_is_ok(to, from, n))
- __copy_user_zeroing(to, from, n);
- else
- n = __copy_user_zeroing_intel(to, from, n);
- clac();
- return n;
-}
-EXPORT_SYMBOL(__copy_from_user_ll);
-
-unsigned long __copy_from_user_ll_nozero(void *to, const void __user *from,
- unsigned long n)
-{
- stac();
- if (movsl_is_ok(to, from, n))
- __copy_user(to, from, n);
- else
- n = __copy_user_intel((void __user *)to,
- (const void *)from, n);
- clac();
- return n;
-}
-EXPORT_SYMBOL(__copy_from_user_ll_nozero);
-
-unsigned long __copy_from_user_ll_nocache(void *to, const void __user *from,
- unsigned long n)
-{
- stac();
-#ifdef CONFIG_X86_INTEL_USERCOPY
- if (n > 64 && static_cpu_has(X86_FEATURE_XMM2))
- n = __copy_user_zeroing_intel_nocache(to, from, n);
- else
- __copy_user_zeroing(to, from, n);
-#else
- __copy_user_zeroing(to, from, n);
-#endif
- clac();
- return n;
-}
-EXPORT_SYMBOL(__copy_from_user_ll_nocache);
+EXPORT_SYMBOL(__copy_user_ll);
unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from,
unsigned long n)
diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c
index 69873589c0ba..3b7c40a2e3e1 100644
--- a/arch/x86/lib/usercopy_64.c
+++ b/arch/x86/lib/usercopy_64.c
@@ -54,15 +54,6 @@ unsigned long clear_user(void __user *to, unsigned long n)
}
EXPORT_SYMBOL(clear_user);
-unsigned long copy_in_user(void __user *to, const void __user *from, unsigned len)
-{
- if (access_ok(VERIFY_WRITE, to, len) && access_ok(VERIFY_READ, from, len)) {
- return copy_user_generic((__force void *)to, (__force void *)from, len);
- }
- return len;
-}
-EXPORT_SYMBOL(copy_in_user);
-
/*
* Try to copy last bytes and clear the rest if needed.
* Since protection fault in copy_from/to_user is not a normal situation,
@@ -80,9 +71,5 @@ copy_user_handle_tail(char *to, char *from, unsigned len)
break;
}
clac();
-
- /* If the destination is a kernel buffer, we always clear the end */
- if (!__addr_ok(to))
- memset(to, 0, len);
return len;
}
diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig
index f4126cf997a4..043d37d45919 100644
--- a/arch/xtensa/Kconfig
+++ b/arch/xtensa/Kconfig
@@ -29,6 +29,7 @@ config XTENSA
select NO_BOOTMEM
select PERF_USE_VMALLOC
select VIRT_TO_BUS
+ select ARCH_HAS_RAW_COPY_USER
help
Xtensa processors are 32-bit RISC machines designed by Tensilica
primarily for embedded systems. These processors are both
diff --git a/arch/xtensa/include/asm/Kbuild b/arch/xtensa/include/asm/Kbuild
index f41408c53fe1..cc23e9ecc6bb 100644
--- a/arch/xtensa/include/asm/Kbuild
+++ b/arch/xtensa/include/asm/Kbuild
@@ -6,6 +6,7 @@ generic-y += dma-contiguous.h
generic-y += emergency-restart.h
generic-y += errno.h
generic-y += exec.h
+generic-y += extable.h
generic-y += fcntl.h
generic-y += hardirq.h
generic-y += ioctl.h
diff --git a/arch/xtensa/include/asm/uaccess.h b/arch/xtensa/include/asm/uaccess.h
index 26512692e28f..2e7bac0d4b2c 100644
--- a/arch/xtensa/include/asm/uaccess.h
+++ b/arch/xtensa/include/asm/uaccess.h
@@ -18,6 +18,7 @@
#include <linux/prefetch.h>
#include <asm/types.h>
+#include <asm/extable.h>
/*
* The fs value determines whether argument validity checking should
@@ -233,60 +234,22 @@ __asm__ __volatile__( \
* Copy to/from user space
*/
-/*
- * We use a generic, arbitrary-sized copy subroutine. The Xtensa
- * architecture would cause heavy code bloat if we tried to inline
- * these functions and provide __constant_copy_* equivalents like the
- * i386 versions. __xtensa_copy_user is quite efficient. See the
- * .fixup section of __xtensa_copy_user for a discussion on the
- * X_zeroing equivalents for Xtensa.
- */
-
extern unsigned __xtensa_copy_user(void *to, const void *from, unsigned n);
-#define __copy_user(to, from, size) __xtensa_copy_user(to, from, size)
-
-
-static inline unsigned long
-__generic_copy_from_user_nocheck(void *to, const void *from, unsigned long n)
-{
- return __copy_user(to, from, n);
-}
static inline unsigned long
-__generic_copy_to_user_nocheck(void *to, const void *from, unsigned long n)
+raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
- return __copy_user(to, from, n);
+ prefetchw(to);
+ return __xtensa_copy_user(to, (__force const void *)from, n);
}
-
static inline unsigned long
-__generic_copy_to_user(void *to, const void *from, unsigned long n)
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
prefetch(from);
- if (access_ok(VERIFY_WRITE, to, n))
- return __copy_user(to, from, n);
- return n;
+ return __xtensa_copy_user((__force void *)to, from, n);
}
-
-static inline unsigned long
-__generic_copy_from_user(void *to, const void *from, unsigned long n)
-{
- prefetchw(to);
- if (access_ok(VERIFY_READ, from, n))
- return __copy_user(to, from, n);
- else
- memset(to, 0, n);
- return n;
-}
-
-#define copy_to_user(to, from, n) __generic_copy_to_user((to), (from), (n))
-#define copy_from_user(to, from, n) __generic_copy_from_user((to), (from), (n))
-#define __copy_to_user(to, from, n) \
- __generic_copy_to_user_nocheck((to), (from), (n))
-#define __copy_from_user(to, from, n) \
- __generic_copy_from_user_nocheck((to), (from), (n))
-#define __copy_to_user_inatomic __copy_to_user
-#define __copy_from_user_inatomic __copy_from_user
-
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
/*
* We need to return the number of bytes not cleared. Our memset()
@@ -342,10 +305,4 @@ static inline long strnlen_user(const char *str, long len)
return __strnlen_user(str, len);
}
-
-struct exception_table_entry
-{
- unsigned long insn, fixup;
-};
-
#endif /* _XTENSA_UACCESS_H */
diff --git a/arch/xtensa/lib/usercopy.S b/arch/xtensa/lib/usercopy.S
index 7ea4dd68893e..d9cd766bde3e 100644
--- a/arch/xtensa/lib/usercopy.S
+++ b/arch/xtensa/lib/usercopy.S
@@ -102,9 +102,9 @@ __xtensa_copy_user:
bltui a4, 7, .Lbytecopy # do short copies byte by byte
# copy 1 byte
- EX(l8ui, a6, a3, 0, l_fixup)
+ EX(l8ui, a6, a3, 0, fixup)
addi a3, a3, 1
- EX(s8i, a6, a5, 0, s_fixup)
+ EX(s8i, a6, a5, 0, fixup)
addi a5, a5, 1
addi a4, a4, -1
bbci.l a5, 1, .Ldstaligned # if dst is now aligned, then
@@ -112,11 +112,11 @@ __xtensa_copy_user:
.Ldst2mod4: # dst 16-bit aligned
# copy 2 bytes
bltui a4, 6, .Lbytecopy # do short copies byte by byte
- EX(l8ui, a6, a3, 0, l_fixup)
- EX(l8ui, a7, a3, 1, l_fixup)
+ EX(l8ui, a6, a3, 0, fixup)
+ EX(l8ui, a7, a3, 1, fixup)
addi a3, a3, 2
- EX(s8i, a6, a5, 0, s_fixup)
- EX(s8i, a7, a5, 1, s_fixup)
+ EX(s8i, a6, a5, 0, fixup)
+ EX(s8i, a7, a5, 1, fixup)
addi a5, a5, 2
addi a4, a4, -2
j .Ldstaligned # dst is now aligned, return to main algorithm
@@ -135,9 +135,9 @@ __xtensa_copy_user:
add a7, a3, a4 # a7 = end address for source
#endif /* !XCHAL_HAVE_LOOPS */
.Lnextbyte:
- EX(l8ui, a6, a3, 0, l_fixup)
+ EX(l8ui, a6, a3, 0, fixup)
addi a3, a3, 1
- EX(s8i, a6, a5, 0, s_fixup)
+ EX(s8i, a6, a5, 0, fixup)
addi a5, a5, 1
#if !XCHAL_HAVE_LOOPS
blt a3, a7, .Lnextbyte
@@ -161,15 +161,15 @@ __xtensa_copy_user:
add a8, a8, a3 # a8 = end of last 16B source chunk
#endif /* !XCHAL_HAVE_LOOPS */
.Loop1:
- EX(l32i, a6, a3, 0, l_fixup)
- EX(l32i, a7, a3, 4, l_fixup)
- EX(s32i, a6, a5, 0, s_fixup)
- EX(l32i, a6, a3, 8, l_fixup)
- EX(s32i, a7, a5, 4, s_fixup)
- EX(l32i, a7, a3, 12, l_fixup)
- EX(s32i, a6, a5, 8, s_fixup)
+ EX(l32i, a6, a3, 0, fixup)
+ EX(l32i, a7, a3, 4, fixup)
+ EX(s32i, a6, a5, 0, fixup)
+ EX(l32i, a6, a3, 8, fixup)
+ EX(s32i, a7, a5, 4, fixup)
+ EX(l32i, a7, a3, 12, fixup)
+ EX(s32i, a6, a5, 8, fixup)
addi a3, a3, 16
- EX(s32i, a7, a5, 12, s_fixup)
+ EX(s32i, a7, a5, 12, fixup)
addi a5, a5, 16
#if !XCHAL_HAVE_LOOPS
blt a3, a8, .Loop1
@@ -177,31 +177,31 @@ __xtensa_copy_user:
.Loop1done:
bbci.l a4, 3, .L2
# copy 8 bytes
- EX(l32i, a6, a3, 0, l_fixup)
- EX(l32i, a7, a3, 4, l_fixup)
+ EX(l32i, a6, a3, 0, fixup)
+ EX(l32i, a7, a3, 4, fixup)
addi a3, a3, 8
- EX(s32i, a6, a5, 0, s_fixup)
- EX(s32i, a7, a5, 4, s_fixup)
+ EX(s32i, a6, a5, 0, fixup)
+ EX(s32i, a7, a5, 4, fixup)
addi a5, a5, 8
.L2:
bbci.l a4, 2, .L3
# copy 4 bytes
- EX(l32i, a6, a3, 0, l_fixup)
+ EX(l32i, a6, a3, 0, fixup)
addi a3, a3, 4
- EX(s32i, a6, a5, 0, s_fixup)
+ EX(s32i, a6, a5, 0, fixup)
addi a5, a5, 4
.L3:
bbci.l a4, 1, .L4
# copy 2 bytes
- EX(l16ui, a6, a3, 0, l_fixup)
+ EX(l16ui, a6, a3, 0, fixup)
addi a3, a3, 2
- EX(s16i, a6, a5, 0, s_fixup)
+ EX(s16i, a6, a5, 0, fixup)
addi a5, a5, 2
.L4:
bbci.l a4, 0, .L5
# copy 1 byte
- EX(l8ui, a6, a3, 0, l_fixup)
- EX(s8i, a6, a5, 0, s_fixup)
+ EX(l8ui, a6, a3, 0, fixup)
+ EX(s8i, a6, a5, 0, fixup)
.L5:
movi a2, 0 # return success for len bytes copied
retw
@@ -217,7 +217,7 @@ __xtensa_copy_user:
# copy 16 bytes per iteration for word-aligned dst and unaligned src
and a10, a3, a8 # save unalignment offset for below
sub a3, a3, a10 # align a3 (to avoid sim warnings only; not needed for hardware)
- EX(l32i, a6, a3, 0, l_fixup) # load first word
+ EX(l32i, a6, a3, 0, fixup) # load first word
#if XCHAL_HAVE_LOOPS
loopnez a7, .Loop2done
#else /* !XCHAL_HAVE_LOOPS */
@@ -226,19 +226,19 @@ __xtensa_copy_user:
add a12, a12, a3 # a12 = end of last 16B source chunk
#endif /* !XCHAL_HAVE_LOOPS */
.Loop2:
- EX(l32i, a7, a3, 4, l_fixup)
- EX(l32i, a8, a3, 8, l_fixup)
+ EX(l32i, a7, a3, 4, fixup)
+ EX(l32i, a8, a3, 8, fixup)
ALIGN( a6, a6, a7)
- EX(s32i, a6, a5, 0, s_fixup)
- EX(l32i, a9, a3, 12, l_fixup)
+ EX(s32i, a6, a5, 0, fixup)
+ EX(l32i, a9, a3, 12, fixup)
ALIGN( a7, a7, a8)
- EX(s32i, a7, a5, 4, s_fixup)
- EX(l32i, a6, a3, 16, l_fixup)
+ EX(s32i, a7, a5, 4, fixup)
+ EX(l32i, a6, a3, 16, fixup)
ALIGN( a8, a8, a9)
- EX(s32i, a8, a5, 8, s_fixup)
+ EX(s32i, a8, a5, 8, fixup)
addi a3, a3, 16
ALIGN( a9, a9, a6)
- EX(s32i, a9, a5, 12, s_fixup)
+ EX(s32i, a9, a5, 12, fixup)
addi a5, a5, 16
#if !XCHAL_HAVE_LOOPS
blt a3, a12, .Loop2
@@ -246,39 +246,39 @@ __xtensa_copy_user:
.Loop2done:
bbci.l a4, 3, .L12
# copy 8 bytes
- EX(l32i, a7, a3, 4, l_fixup)
- EX(l32i, a8, a3, 8, l_fixup)
+ EX(l32i, a7, a3, 4, fixup)
+ EX(l32i, a8, a3, 8, fixup)
ALIGN( a6, a6, a7)
- EX(s32i, a6, a5, 0, s_fixup)
+ EX(s32i, a6, a5, 0, fixup)
addi a3, a3, 8
ALIGN( a7, a7, a8)
- EX(s32i, a7, a5, 4, s_fixup)
+ EX(s32i, a7, a5, 4, fixup)
addi a5, a5, 8
mov a6, a8
.L12:
bbci.l a4, 2, .L13
# copy 4 bytes
- EX(l32i, a7, a3, 4, l_fixup)
+ EX(l32i, a7, a3, 4, fixup)
addi a3, a3, 4
ALIGN( a6, a6, a7)
- EX(s32i, a6, a5, 0, s_fixup)
+ EX(s32i, a6, a5, 0, fixup)
addi a5, a5, 4
mov a6, a7
.L13:
add a3, a3, a10 # readjust a3 with correct misalignment
bbci.l a4, 1, .L14
# copy 2 bytes
- EX(l8ui, a6, a3, 0, l_fixup)
- EX(l8ui, a7, a3, 1, l_fixup)
+ EX(l8ui, a6, a3, 0, fixup)
+ EX(l8ui, a7, a3, 1, fixup)
addi a3, a3, 2
- EX(s8i, a6, a5, 0, s_fixup)
- EX(s8i, a7, a5, 1, s_fixup)
+ EX(s8i, a6, a5, 0, fixup)
+ EX(s8i, a7, a5, 1, fixup)
addi a5, a5, 2
.L14:
bbci.l a4, 0, .L15
# copy 1 byte
- EX(l8ui, a6, a3, 0, l_fixup)
- EX(s8i, a6, a5, 0, s_fixup)
+ EX(l8ui, a6, a3, 0, fixup)
+ EX(s8i, a6, a5, 0, fixup)
.L15:
movi a2, 0 # return success for len bytes copied
retw
@@ -291,30 +291,10 @@ __xtensa_copy_user:
* bytes_copied = a5 - a2
* retval = bytes_not_copied = original len - bytes_copied
* retval = a11 - (a5 - a2)
- *
- * Clearing the remaining pieces of kernel memory plugs security
- * holes. This functionality is the equivalent of the *_zeroing
- * functions that some architectures provide.
*/
-.Lmemset:
- .word memset
-s_fixup:
+fixup:
sub a2, a5, a2 /* a2 <-- bytes copied */
sub a2, a11, a2 /* a2 <-- bytes not copied */
retw
-
-l_fixup:
- sub a2, a5, a2 /* a2 <-- bytes copied */
- sub a2, a11, a2 /* a2 <-- bytes not copied == return value */
-
- /* void *memset(void *s, int c, size_t n); */
- mov a6, a5 /* s */
- movi a7, 0 /* c */
- mov a8, a2 /* n */
- l32r a4, .Lmemset
- callx4 a4
- /* Ignore memset return value in a6. */
- /* a2 still contains bytes not copied. */
- retw