1 files changed, 177 insertions, 54 deletions

diff --git a/mm/maccess.c b/mm/maccess.c
index f3416632e5a4..486559d68858 100644
--- a/mm/maccess.c
+++ b/mm/maccess.c
@@ -1,48 +1,144 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
- * Access kernel memory without faulting.
+ * Access kernel or user memory without faulting.
  */
 #include <linux/export.h>
 #include <linux/mm.h>
 #include <linux/uaccess.h>
+#include <asm/tlb.h>
+
+bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src,
+		size_t size)
+{
+	return true;
+}
+
+/*
+ * The below only uses kmsan_check_memory() to ensure uninitialized kernel
+ * memory isn't leaked.
+ */
+#define copy_from_kernel_nofault_loop(dst, src, len, type, err_label)	\
+	while (len >= sizeof(type)) {					\
+		__get_kernel_nofault(dst, src, type, err_label);	\
+		kmsan_check_memory(src, sizeof(type));			\
+		dst += sizeof(type);					\
+		src += sizeof(type);					\
+		len -= sizeof(type);					\
+	}
+
+long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
+{
+	unsigned long align = 0;
+
+	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
+		align = (unsigned long)dst | (unsigned long)src;
+
+	if (!copy_from_kernel_nofault_allowed(src, size))
+		return -ERANGE;
+
+	pagefault_disable();
+	if (!(align & 7))
+		copy_from_kernel_nofault_loop(dst, src, size, u64, Efault);
+	if (!(align & 3))
+		copy_from_kernel_nofault_loop(dst, src, size, u32, Efault);
+	if (!(align & 1))
+		copy_from_kernel_nofault_loop(dst, src, size, u16, Efault);
+	copy_from_kernel_nofault_loop(dst, src, size, u8, Efault);
+	pagefault_enable();
+	return 0;
+Efault:
+	pagefault_enable();
+	return -EFAULT;
+}
+EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
+
+#define copy_to_kernel_nofault_loop(dst, src, len, type, err_label)	\
+	while (len >= sizeof(type)) {					\
+		__put_kernel_nofault(dst, src, type, err_label);	\
+		instrument_write(dst, sizeof(type));			\
+		dst += sizeof(type);					\
+		src += sizeof(type);					\
+		len -= sizeof(type);					\
+	}
+
+long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
+{
+	unsigned long align = 0;
+
+	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
+		align = (unsigned long)dst | (unsigned long)src;
+
+	pagefault_disable();
+	if (!(align & 7))
+		copy_to_kernel_nofault_loop(dst, src, size, u64, Efault);
+	if (!(align & 3))
+		copy_to_kernel_nofault_loop(dst, src, size, u32, Efault);
+	if (!(align & 1))
+		copy_to_kernel_nofault_loop(dst, src, size, u16, Efault);
+	copy_to_kernel_nofault_loop(dst, src, size, u8, Efault);
+	pagefault_enable();
+	return 0;
+Efault:
+	pagefault_enable();
+	return -EFAULT;
+}
+
+long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
+{
+	const void *src = unsafe_addr;
+
+	if (unlikely(count <= 0))
+		return 0;
+	if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
+		return -ERANGE;
+
+	pagefault_disable();
+	do {
+		__get_kernel_nofault(dst, src, u8, Efault);
+		dst++;
+		src++;
+	} while (dst[-1] && src - unsafe_addr < count);
+	pagefault_enable();
+
+	dst[-1] = '\0';
+	return src - unsafe_addr;
+Efault:
+	pagefault_enable();
+	dst[0] = '\0';
+	return -EFAULT;
+}
 
 /**
- * probe_kernel_read(): safely attempt to read from a location
+ * copy_from_user_nofault(): safely attempt to read from a user-space location
  * @dst: pointer to the buffer that shall take the data
- * @src: address to read from
+ * @src: address to read from. This must be a user address.
  * @size: size of the data chunk
  *
- * Safely read from address @src to the buffer at @dst.  If a kernel fault
+ * Safely read from user address @src to the buffer at @dst. If a kernel fault
  * happens, handle that and return -EFAULT.
- *
- * We ensure that the copy_from_user is executed in atomic context so that
- * do_page_fault() doesn't attempt to take mmap_sem.  This makes
- * probe_kernel_read() suitable for use within regions where the caller
- * already holds mmap_sem, or other locks which nest inside mmap_sem.
  */
+long copy_from_user_nofault(void *dst, const void __user *src, size_t size)
+{
+	long ret = -EFAULT;
 
-long __weak probe_kernel_read(void *dst, const void *src, size_t size)
-    __attribute__((alias("__probe_kernel_read")));
+	if (!__access_ok(src, size))
+		return ret;
 
-long __probe_kernel_read(void *dst, const void *src, size_t size)
-{
-	long ret;
-	mm_segment_t old_fs = get_fs();
+	if (!nmi_uaccess_okay())
+		return ret;
 
-	set_fs(KERNEL_DS);
 	pagefault_disable();
-	current->kernel_uaccess_faults_ok++;
-	ret = __copy_from_user_inatomic(dst,
-			(__force const void __user *)src, size);
-	current->kernel_uaccess_faults_ok--;
+	ret = __copy_from_user_inatomic(dst, src, size);
 	pagefault_enable();
-	set_fs(old_fs);
 
-	return ret ? -EFAULT : 0;
+	if (ret)
+		return -EFAULT;
+	return 0;
 }
-EXPORT_SYMBOL_GPL(probe_kernel_read);
+EXPORT_SYMBOL_GPL(copy_from_user_nofault);
 
 /**
- * probe_kernel_write(): safely attempt to write to a location
+ * copy_to_user_nofault(): safely attempt to write to a user-space location
  * @dst: address to write to
  * @src: pointer to the data that shall be written
  * @size: size of the data chunk
@@ -50,34 +146,31 @@ EXPORT_SYMBOL_GPL(probe_kernel_read);
  * Safely write to address @dst from the buffer at @src.  If a kernel fault
  * happens, handle that and return -EFAULT.
  */
-long __weak probe_kernel_write(void *dst, const void *src, size_t size)
-    __attribute__((alias("__probe_kernel_write")));
-
-long __probe_kernel_write(void *dst, const void *src, size_t size)
+long copy_to_user_nofault(void __user *dst, const void *src, size_t size)
 {
-	long ret;
-	mm_segment_t old_fs = get_fs();
+	long ret = -EFAULT;
 
-	set_fs(KERNEL_DS);
-	pagefault_disable();
-	current->kernel_uaccess_faults_ok++;
-	ret = __copy_to_user_inatomic((__force void __user *)dst, src, size);
-	current->kernel_uaccess_faults_ok--;
-	pagefault_enable();
-	set_fs(old_fs);
+	if (access_ok(dst, size)) {
+		pagefault_disable();
+		ret = __copy_to_user_inatomic(dst, src, size);
+		pagefault_enable();
+	}
 
-	return ret ? -EFAULT : 0;
+	if (ret)
+		return -EFAULT;
+	return 0;
 }
-EXPORT_SYMBOL_GPL(probe_kernel_write);
+EXPORT_SYMBOL_GPL(copy_to_user_nofault);
 
 /**
- * strncpy_from_unsafe: - Copy a NUL terminated string from unsafe address.
+ * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user
+ *				address.
  * @dst:   Destination address, in kernel space.  This buffer must be at
  *         least @count bytes long.
- * @unsafe_addr: Unsafe address.
+ * @unsafe_addr: Unsafe user address.
  * @count: Maximum number of bytes to copy, including the trailing NUL.
  *
- * Copies a NUL-terminated string from unsafe address to kernel buffer.
+ * Copies a NUL-terminated string from unsafe user address to kernel buffer.
  *
  * On success, returns the length of the string INCLUDING the trailing NUL.
  *
@@ -87,27 +180,57 @@ EXPORT_SYMBOL_GPL(probe_kernel_write);
  * If @count is smaller than the length of the string, copies @count-1 bytes,
  * sets the last byte of @dst buffer to NUL and returns @count.
  */
-long strncpy_from_unsafe(char *dst, const void *unsafe_addr, long count)
+long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr,
+			      long count)
 {
-	mm_segment_t old_fs = get_fs();
-	const void *src = unsafe_addr;
 	long ret;
 
 	if (unlikely(count <= 0))
 		return 0;
 
-	set_fs(KERNEL_DS);
 	pagefault_disable();
-	current->kernel_uaccess_faults_ok++;
+	ret = strncpy_from_user(dst, unsafe_addr, count);
+	pagefault_enable();
 
-	do {
-		ret = __get_user(*dst++, (const char __user __force *)src++);
-	} while (dst[-1] && ret == 0 && src - unsafe_addr < count);
+	if (ret >= count) {
+		ret = count;
+		dst[ret - 1] = '\0';
+	} else if (ret >= 0) {
+		ret++;
+	}
 
-	current->kernel_uaccess_faults_ok--;
-	dst[-1] = '\0';
+	return ret;
+}
+
+/**
+ * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL.
+ * @unsafe_addr: The string to measure.
+ * @count: Maximum count (including NUL)
+ *
+ * Get the size of a NUL-terminated string in user space without pagefault.
+ *
+ * Returns the size of the string INCLUDING the terminating NUL.
+ *
+ * If the string is too long, returns a number larger than @count. User
+ * has to check the return value against "> count".
+ * On exception (or invalid count), returns 0.
+ *
+ * Unlike strnlen_user, this can be used from IRQ handler etc. because
+ * it disables pagefaults.
+ */
+long strnlen_user_nofault(const void __user *unsafe_addr, long count)
+{
+	int ret;
+
+	pagefault_disable();
+	ret = strnlen_user(unsafe_addr, count);
 	pagefault_enable();
-	set_fs(old_fs);
 
-	return ret ? -EFAULT : src - unsafe_addr;
+	return ret;
+}
+
+void __copy_overflow(int size, unsigned long count)
+{
+	WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count);
 }
+EXPORT_SYMBOL(__copy_overflow);