summaryrefslogtreecommitdiff
path: root/arch/csky/lib
diff options
context:
space:
mode:
Diffstat (limited to 'arch/csky/lib')
-rw-r--r--arch/csky/lib/Makefile4
-rw-r--r--arch/csky/lib/delay.c2
-rw-r--r--arch/csky/lib/error-inject.c10
-rw-r--r--arch/csky/lib/string.c134
-rw-r--r--arch/csky/lib/usercopy.c390
5 files changed, 320 insertions, 220 deletions
diff --git a/arch/csky/lib/Makefile b/arch/csky/lib/Makefile
index 078e2d5f32e1..d0ce6e2d7ab2 100644
--- a/arch/csky/lib/Makefile
+++ b/arch/csky/lib/Makefile
@@ -1,2 +1,6 @@
# SPDX-License-Identifier: GPL-2.0-only
lib-y := usercopy.o delay.o
+obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
+ifneq ($(CONFIG_HAVE_EFFICIENT_UNALIGNED_STRING_OPS), y)
+lib-y += string.o
+endif
diff --git a/arch/csky/lib/delay.c b/arch/csky/lib/delay.c
index 22570b0790d6..f5db317313bb 100644
--- a/arch/csky/lib/delay.c
+++ b/arch/csky/lib/delay.c
@@ -5,7 +5,7 @@
#include <linux/init.h>
#include <linux/delay.h>
-void __delay(unsigned long loops)
+void __aligned(8) __delay(unsigned long loops)
{
asm volatile (
"mov r0, r0\n"
diff --git a/arch/csky/lib/error-inject.c b/arch/csky/lib/error-inject.c
new file mode 100644
index 000000000000..c15fb36fe067
--- /dev/null
+++ b/arch/csky/lib/error-inject.c
@@ -0,0 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/error-injection.h>
+#include <linux/kprobes.h>
+
+void override_function_with_return(struct pt_regs *regs)
+{
+ instruction_pointer_set(regs, regs->lr);
+}
+NOKPROBE_SYMBOL(override_function_with_return);
diff --git a/arch/csky/lib/string.c b/arch/csky/lib/string.c
new file mode 100644
index 000000000000..d65626fcaeac
--- /dev/null
+++ b/arch/csky/lib/string.c
@@ -0,0 +1,134 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * String functions optimized for hardware which doesn't
+ * handle unaligned memory accesses efficiently.
+ *
+ * Copyright (C) 2021 Matteo Croce
+ */
+
+#include <linux/types.h>
+#include <linux/module.h>
+
+/* Minimum size for a word copy to be convenient */
+#define BYTES_LONG sizeof(long)
+#define WORD_MASK (BYTES_LONG - 1)
+#define MIN_THRESHOLD (BYTES_LONG * 2)
+
+/* convenience union to avoid cast between different pointer types */
+union types {
+ u8 *as_u8;
+ unsigned long *as_ulong;
+ uintptr_t as_uptr;
+};
+
+union const_types {
+ const u8 *as_u8;
+ unsigned long *as_ulong;
+ uintptr_t as_uptr;
+};
+
+void *memcpy(void *dest, const void *src, size_t count)
+{
+ union const_types s = { .as_u8 = src };
+ union types d = { .as_u8 = dest };
+ int distance = 0;
+
+ if (count < MIN_THRESHOLD)
+ goto copy_remainder;
+
+ /* Copy a byte at time until destination is aligned. */
+ for (; d.as_uptr & WORD_MASK; count--)
+ *d.as_u8++ = *s.as_u8++;
+
+ distance = s.as_uptr & WORD_MASK;
+
+ if (distance) {
+ unsigned long last, next;
+
+ /*
+ * s is distance bytes ahead of d, and d just reached
+ * the alignment boundary. Move s backward to word align it
+ * and shift data to compensate for distance, in order to do
+ * word-by-word copy.
+ */
+ s.as_u8 -= distance;
+
+ next = s.as_ulong[0];
+ for (; count >= BYTES_LONG; count -= BYTES_LONG) {
+ last = next;
+ next = s.as_ulong[1];
+
+ d.as_ulong[0] = last >> (distance * 8) |
+ next << ((BYTES_LONG - distance) * 8);
+
+ d.as_ulong++;
+ s.as_ulong++;
+ }
+
+ /* Restore s with the original offset. */
+ s.as_u8 += distance;
+ } else {
+ /*
+ * If the source and dest lower bits are the same, do a simple
+ * 32/64 bit wide copy.
+ */
+ for (; count >= BYTES_LONG; count -= BYTES_LONG)
+ *d.as_ulong++ = *s.as_ulong++;
+ }
+
+copy_remainder:
+ while (count--)
+ *d.as_u8++ = *s.as_u8++;
+
+ return dest;
+}
+EXPORT_SYMBOL(memcpy);
+
+/*
+ * Simply check if the buffer overlaps an call memcpy() in case,
+ * otherwise do a simple one byte at time backward copy.
+ */
+void *memmove(void *dest, const void *src, size_t count)
+{
+ if (dest < src || src + count <= dest)
+ return memcpy(dest, src, count);
+
+ if (dest > src) {
+ const char *s = src + count;
+ char *tmp = dest + count;
+
+ while (count--)
+ *--tmp = *--s;
+ }
+ return dest;
+}
+EXPORT_SYMBOL(memmove);
+
+void *memset(void *s, int c, size_t count)
+{
+ union types dest = { .as_u8 = s };
+
+ if (count >= MIN_THRESHOLD) {
+ unsigned long cu = (unsigned long)c;
+
+ /* Compose an ulong with 'c' repeated 4/8 times */
+ cu |= cu << 8;
+ cu |= cu << 16;
+ /* Suppress warning on 32 bit machines */
+ cu |= (cu << 16) << 16;
+
+ for (; count && dest.as_uptr & WORD_MASK; count--)
+ *dest.as_u8++ = c;
+
+ /* Copy using the largest size allowed */
+ for (; count >= BYTES_LONG; count -= BYTES_LONG)
+ *dest.as_ulong++ = cu;
+ }
+
+ /* copy the remainder */
+ while (count--)
+ *dest.as_u8++ = c;
+
+ return s;
+}
+EXPORT_SYMBOL(memset);
diff --git a/arch/csky/lib/usercopy.c b/arch/csky/lib/usercopy.c
index 647a23986fb5..3c01c54421ca 100644
--- a/arch/csky/lib/usercopy.c
+++ b/arch/csky/lib/usercopy.c
@@ -7,10 +7,70 @@
unsigned long raw_copy_from_user(void *to, const void *from,
unsigned long n)
{
- if (access_ok(from, n))
- __copy_user_zeroing(to, from, n);
- else
- memset(to, 0, n);
+ int tmp, nsave;
+
+ __asm__ __volatile__(
+ "0: cmpnei %1, 0 \n"
+ " bf 7f \n"
+ " mov %3, %1 \n"
+ " or %3, %2 \n"
+ " andi %3, 3 \n"
+ " cmpnei %3, 0 \n"
+ " bf 1f \n"
+ " br 5f \n"
+ "1: cmplti %0, 16 \n"
+ " bt 3f \n"
+ "2: ldw %3, (%2, 0) \n"
+ "10: ldw %4, (%2, 4) \n"
+ " stw %3, (%1, 0) \n"
+ " stw %4, (%1, 4) \n"
+ "11: ldw %3, (%2, 8) \n"
+ "12: ldw %4, (%2, 12) \n"
+ " stw %3, (%1, 8) \n"
+ " stw %4, (%1, 12) \n"
+ " addi %2, 16 \n"
+ " addi %1, 16 \n"
+ " subi %0, 16 \n"
+ " br 1b \n"
+ "3: cmplti %0, 4 \n"
+ " bt 5f \n"
+ "4: ldw %3, (%2, 0) \n"
+ " stw %3, (%1, 0) \n"
+ " addi %2, 4 \n"
+ " addi %1, 4 \n"
+ " subi %0, 4 \n"
+ " br 3b \n"
+ "5: cmpnei %0, 0 \n"
+ " bf 7f \n"
+ "6: ldb %3, (%2, 0) \n"
+ " stb %3, (%1, 0) \n"
+ " addi %2, 1 \n"
+ " addi %1, 1 \n"
+ " subi %0, 1 \n"
+ " br 5b \n"
+ "8: stw %3, (%1, 0) \n"
+ " subi %0, 4 \n"
+ " bf 7f \n"
+ "9: subi %0, 8 \n"
+ " bf 7f \n"
+ "13: stw %3, (%1, 8) \n"
+ " subi %0, 12 \n"
+ " bf 7f \n"
+ ".section __ex_table, \"a\" \n"
+ ".align 2 \n"
+ ".long 2b, 7f \n"
+ ".long 4b, 7f \n"
+ ".long 6b, 7f \n"
+ ".long 10b, 8b \n"
+ ".long 11b, 9b \n"
+ ".long 12b,13b \n"
+ ".previous \n"
+ "7: \n"
+ : "=r"(n), "=r"(to), "=r"(from), "=r"(nsave),
+ "=r"(tmp)
+ : "0"(n), "1"(to), "2"(from)
+ : "memory");
+
return n;
}
EXPORT_SYMBOL(raw_copy_from_user);
@@ -18,229 +78,69 @@ EXPORT_SYMBOL(raw_copy_from_user);
unsigned long raw_copy_to_user(void *to, const void *from,
unsigned long n)
{
- if (access_ok(to, n))
- __copy_user(to, from, n);
- return n;
-}
-EXPORT_SYMBOL(raw_copy_to_user);
-
-
-/*
- * copy a null terminated string from userspace.
- */
-#define __do_strncpy_from_user(dst, src, count, res) \
-do { \
- int tmp; \
- long faultres; \
- asm volatile( \
- " cmpnei %3, 0 \n" \
- " bf 4f \n" \
- "1: cmpnei %1, 0 \n" \
- " bf 5f \n" \
- "2: ldb %4, (%3, 0) \n" \
- " stb %4, (%2, 0) \n" \
- " cmpnei %4, 0 \n" \
- " bf 3f \n" \
- " addi %3, 1 \n" \
- " addi %2, 1 \n" \
- " subi %1, 1 \n" \
- " br 1b \n" \
- "3: subu %0, %1 \n" \
- " br 5f \n" \
- "4: mov %0, %5 \n" \
- " br 5f \n" \
- ".section __ex_table, \"a\" \n" \
- ".align 2 \n" \
- ".long 2b, 4b \n" \
- ".previous \n" \
- "5: \n" \
- : "=r"(res), "=r"(count), "=r"(dst), \
- "=r"(src), "=r"(tmp), "=r"(faultres) \
- : "5"(-EFAULT), "0"(count), "1"(count), \
- "2"(dst), "3"(src) \
- : "memory", "cc"); \
-} while (0)
-
-/*
- * __strncpy_from_user: - Copy a NUL terminated string from userspace,
- * with less checking.
- * @dst: Destination address, in kernel space. This buffer must be at
- * least @count bytes long.
- * @src: Source address, in user space.
- * @count: Maximum number of bytes to copy, including the trailing NUL.
- *
- * Copies a NUL-terminated string from userspace to kernel space.
- * Caller must check the specified block with access_ok() before calling
- * this function.
- *
- * On success, returns the length of the string (not including the trailing
- * NUL).
- *
- * If access to userspace fails, returns -EFAULT (some data may have been
- * copied).
- *
- * If @count is smaller than the length of the string, copies @count bytes
- * and returns @count.
- */
-long __strncpy_from_user(char *dst, const char *src, long count)
-{
- long res;
-
- __do_strncpy_from_user(dst, src, count, res);
- return res;
-}
-EXPORT_SYMBOL(__strncpy_from_user);
-
-/*
- * strncpy_from_user: - Copy a NUL terminated string from userspace.
- * @dst: Destination address, in kernel space. This buffer must be at
- * least @count bytes long.
- * @src: Source address, in user space.
- * @count: Maximum number of bytes to copy, including the trailing NUL.
- *
- * Copies a NUL-terminated string from userspace to kernel space.
- *
- * On success, returns the length of the string (not including the trailing
- * NUL).
- *
- * If access to userspace fails, returns -EFAULT (some data may have been
- * copied).
- *
- * If @count is smaller than the length of the string, copies @count bytes
- * and returns @count.
- */
-long strncpy_from_user(char *dst, const char *src, long count)
-{
- long res = -EFAULT;
-
- if (access_ok(src, 1))
- __do_strncpy_from_user(dst, src, count, res);
- return res;
-}
-EXPORT_SYMBOL(strncpy_from_user);
-
-/*
- * strlen_user: - Get the size of a string in user space.
- * @str: The string to measure.
- * @n: The maximum valid length
- *
- * Get the size of a NUL-terminated string in user space.
- *
- * Returns the size of the string INCLUDING the terminating NUL.
- * On exception, returns 0.
- * If the string is too long, returns a value greater than @n.
- */
-long strnlen_user(const char *s, long n)
-{
- unsigned long res, tmp;
-
- if (s == NULL)
- return 0;
-
- asm volatile(
- " cmpnei %1, 0 \n"
- " bf 3f \n"
- "1: cmpnei %0, 0 \n"
- " bf 3f \n"
- "2: ldb %3, (%1, 0) \n"
+ int w0, w1, w2, w3;
+
+ __asm__ __volatile__(
+ "0: cmpnei %1, 0 \n"
+ " bf 8f \n"
+ " mov %3, %1 \n"
+ " or %3, %2 \n"
+ " andi %3, 3 \n"
" cmpnei %3, 0 \n"
- " bf 3f \n"
- " subi %0, 1 \n"
- " addi %1, 1 \n"
+ " bf 1f \n"
+ " br 5f \n"
+ "1: cmplti %0, 16 \n" /* 4W */
+ " bt 3f \n"
+ " ldw %3, (%2, 0) \n"
+ " ldw %4, (%2, 4) \n"
+ " ldw %5, (%2, 8) \n"
+ " ldw %6, (%2, 12) \n"
+ "2: stw %3, (%1, 0) \n"
+ "9: stw %4, (%1, 4) \n"
+ "10: stw %5, (%1, 8) \n"
+ "11: stw %6, (%1, 12) \n"
+ " addi %2, 16 \n"
+ " addi %1, 16 \n"
+ " subi %0, 16 \n"
" br 1b \n"
- "3: subu %2, %0 \n"
+ "3: cmplti %0, 4 \n" /* 1W */
+ " bt 5f \n"
+ " ldw %3, (%2, 0) \n"
+ "4: stw %3, (%1, 0) \n"
+ " addi %2, 4 \n"
+ " addi %1, 4 \n"
+ " subi %0, 4 \n"
+ " br 3b \n"
+ "5: cmpnei %0, 0 \n" /* 1B */
+ " bf 13f \n"
+ " ldb %3, (%2, 0) \n"
+ "6: stb %3, (%1, 0) \n"
" addi %2, 1 \n"
- " br 5f \n"
- "4: movi %0, 0 \n"
- " br 5f \n"
+ " addi %1, 1 \n"
+ " subi %0, 1 \n"
+ " br 5b \n"
+ "7: subi %0, 4 \n"
+ "8: subi %0, 4 \n"
+ "12: subi %0, 4 \n"
+ " br 13f \n"
".section __ex_table, \"a\" \n"
".align 2 \n"
- ".long 2b, 4b \n"
+ ".long 2b, 13f \n"
+ ".long 4b, 13f \n"
+ ".long 6b, 13f \n"
+ ".long 9b, 12b \n"
+ ".long 10b, 8b \n"
+ ".long 11b, 7b \n"
".previous \n"
- "5: \n"
- : "=r"(n), "=r"(s), "=r"(res), "=r"(tmp)
- : "0"(n), "1"(s), "2"(n)
- : "memory", "cc");
+ "13: \n"
+ : "=r"(n), "=r"(to), "=r"(from), "=r"(w0),
+ "=r"(w1), "=r"(w2), "=r"(w3)
+ : "0"(n), "1"(to), "2"(from)
+ : "memory");
- return res;
-}
-EXPORT_SYMBOL(strnlen_user);
-
-#define __do_clear_user(addr, size) \
-do { \
- int __d0, zvalue, tmp; \
- \
- asm volatile( \
- "0: cmpnei %1, 0 \n" \
- " bf 7f \n" \
- " mov %3, %1 \n" \
- " andi %3, 3 \n" \
- " cmpnei %3, 0 \n" \
- " bf 1f \n" \
- " br 5f \n" \
- "1: cmplti %0, 32 \n" /* 4W */ \
- " bt 3f \n" \
- "8: stw %2, (%1, 0) \n" \
- "10: stw %2, (%1, 4) \n" \
- "11: stw %2, (%1, 8) \n" \
- "12: stw %2, (%1, 12) \n" \
- "13: stw %2, (%1, 16) \n" \
- "14: stw %2, (%1, 20) \n" \
- "15: stw %2, (%1, 24) \n" \
- "16: stw %2, (%1, 28) \n" \
- " addi %1, 32 \n" \
- " subi %0, 32 \n" \
- " br 1b \n" \
- "3: cmplti %0, 4 \n" /* 1W */ \
- " bt 5f \n" \
- "4: stw %2, (%1, 0) \n" \
- " addi %1, 4 \n" \
- " subi %0, 4 \n" \
- " br 3b \n" \
- "5: cmpnei %0, 0 \n" /* 1B */ \
- "9: bf 7f \n" \
- "6: stb %2, (%1, 0) \n" \
- " addi %1, 1 \n" \
- " subi %0, 1 \n" \
- " br 5b \n" \
- ".section __ex_table,\"a\" \n" \
- ".align 2 \n" \
- ".long 8b, 9b \n" \
- ".long 10b, 9b \n" \
- ".long 11b, 9b \n" \
- ".long 12b, 9b \n" \
- ".long 13b, 9b \n" \
- ".long 14b, 9b \n" \
- ".long 15b, 9b \n" \
- ".long 16b, 9b \n" \
- ".long 4b, 9b \n" \
- ".long 6b, 9b \n" \
- ".previous \n" \
- "7: \n" \
- : "=r"(size), "=r" (__d0), \
- "=r"(zvalue), "=r"(tmp) \
- : "0"(size), "1"(addr), "2"(0) \
- : "memory", "cc"); \
-} while (0)
-
-/*
- * clear_user: - Zero a block of memory in user space.
- * @to: Destination address, in user space.
- * @n: Number of bytes to zero.
- *
- * Zero a block of memory in user space.
- *
- * Returns number of bytes that could not be cleared.
- * On success, this will be zero.
- */
-unsigned long
-clear_user(void __user *to, unsigned long n)
-{
- if (access_ok(to, n))
- __do_clear_user(to, n);
return n;
}
-EXPORT_SYMBOL(clear_user);
+EXPORT_SYMBOL(raw_copy_to_user);
/*
* __clear_user: - Zero a block of memory in user space, with less checking.
@@ -256,7 +156,59 @@ EXPORT_SYMBOL(clear_user);
unsigned long
__clear_user(void __user *to, unsigned long n)
{
- __do_clear_user(to, n);
+ int data, value, tmp;
+
+ __asm__ __volatile__(
+ "0: cmpnei %1, 0 \n"
+ " bf 7f \n"
+ " mov %3, %1 \n"
+ " andi %3, 3 \n"
+ " cmpnei %3, 0 \n"
+ " bf 1f \n"
+ " br 5f \n"
+ "1: cmplti %0, 32 \n" /* 4W */
+ " bt 3f \n"
+ "8: stw %2, (%1, 0) \n"
+ "10: stw %2, (%1, 4) \n"
+ "11: stw %2, (%1, 8) \n"
+ "12: stw %2, (%1, 12) \n"
+ "13: stw %2, (%1, 16) \n"
+ "14: stw %2, (%1, 20) \n"
+ "15: stw %2, (%1, 24) \n"
+ "16: stw %2, (%1, 28) \n"
+ " addi %1, 32 \n"
+ " subi %0, 32 \n"
+ " br 1b \n"
+ "3: cmplti %0, 4 \n" /* 1W */
+ " bt 5f \n"
+ "4: stw %2, (%1, 0) \n"
+ " addi %1, 4 \n"
+ " subi %0, 4 \n"
+ " br 3b \n"
+ "5: cmpnei %0, 0 \n" /* 1B */
+ "9: bf 7f \n"
+ "6: stb %2, (%1, 0) \n"
+ " addi %1, 1 \n"
+ " subi %0, 1 \n"
+ " br 5b \n"
+ ".section __ex_table,\"a\" \n"
+ ".align 2 \n"
+ ".long 8b, 9b \n"
+ ".long 10b, 9b \n"
+ ".long 11b, 9b \n"
+ ".long 12b, 9b \n"
+ ".long 13b, 9b \n"
+ ".long 14b, 9b \n"
+ ".long 15b, 9b \n"
+ ".long 16b, 9b \n"
+ ".long 4b, 9b \n"
+ ".long 6b, 9b \n"
+ ".previous \n"
+ "7: \n"
+ : "=r"(n), "=r" (data), "=r"(value), "=r"(tmp)
+ : "0"(n), "1"(to), "2"(0)
+ : "memory");
+
return n;
}
EXPORT_SYMBOL(__clear_user);
generated by cgit (git 2.34.1) at 2025-07-23 10:52:58 +0000