summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--include/linux/mm.h3
-rw-r--r--mm/hugetlb.c2
-rw-r--r--mm/memory.c13
-rw-r--r--mm/userfaultfd.c2
4 files changed, 14 insertions, 6 deletions
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 6bc7f2c1a6d1..c3e2be2b3296 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2426,7 +2426,8 @@ extern void copy_user_huge_page(struct page *dst, struct page *src,
unsigned int pages_per_huge_page);
extern long copy_huge_page_from_user(struct page *dst_page,
const void __user *usr_src,
- unsigned int pages_per_huge_page);
+ unsigned int pages_per_huge_page,
+ bool allow_pagefault);
#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
extern struct page_ext_operations debug_guardpage_ops;
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index dec628b26f59..5d20af921a30 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3973,7 +3973,7 @@ int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm,
ret = copy_huge_page_from_user(page,
(const void __user *) src_addr,
- pages_per_huge_page(h));
+ pages_per_huge_page(h), false);
/* fallback to copy_from_user outside mmap_sem */
if (unlikely(ret)) {
diff --git a/mm/memory.c b/mm/memory.c
index 4ade940d105c..d7676a68c80a 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -4155,7 +4155,8 @@ void copy_user_huge_page(struct page *dst, struct page *src,
long copy_huge_page_from_user(struct page *dst_page,
const void __user *usr_src,
- unsigned int pages_per_huge_page)
+ unsigned int pages_per_huge_page,
+ bool allow_pagefault)
{
void *src = (void *)usr_src;
void *page_kaddr;
@@ -4163,11 +4164,17 @@ long copy_huge_page_from_user(struct page *dst_page,
unsigned long ret_val = pages_per_huge_page * PAGE_SIZE;
for (i = 0; i < pages_per_huge_page; i++) {
- page_kaddr = kmap_atomic(dst_page + i);
+ if (allow_pagefault)
+ page_kaddr = kmap(dst_page + i);
+ else
+ page_kaddr = kmap_atomic(dst_page + i);
rc = copy_from_user(page_kaddr,
(const void __user *)(src + i * PAGE_SIZE),
PAGE_SIZE);
- kunmap_atomic(page_kaddr);
+ if (allow_pagefault)
+ kunmap(dst_page + i);
+ else
+ kunmap_atomic(page_kaddr);
ret_val -= (PAGE_SIZE - rc);
if (rc)
diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
index ef0495bfd17a..09976745be23 100644
--- a/mm/userfaultfd.c
+++ b/mm/userfaultfd.c
@@ -274,7 +274,7 @@ retry:
err = copy_huge_page_from_user(page,
(const void __user *)src_addr,
- pages_per_huge_page(h));
+ pages_per_huge_page(h), true);
if (unlikely(err)) {
err = -EFAULT;
goto out;