summaryrefslogtreecommitdiff
path: root/lib/iov_iter.c
diff options
context:
space:
mode:
Diffstat (limited to 'lib/iov_iter.c')
-rw-r--r--lib/iov_iter.c150
1 files changed, 121 insertions, 29 deletions
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index f2bd21b93dfc..25f572303801 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -1,4 +1,5 @@
#include <linux/export.h>
+#include <linux/bvec.h>
#include <linux/uio.h>
#include <linux/pagemap.h>
#include <linux/slab.h>
@@ -72,19 +73,21 @@
}
#define iterate_all_kinds(i, n, v, I, B, K) { \
- size_t skip = i->iov_offset; \
- if (unlikely(i->type & ITER_BVEC)) { \
- struct bio_vec v; \
- struct bvec_iter __bi; \
- iterate_bvec(i, n, v, __bi, skip, (B)) \
- } else if (unlikely(i->type & ITER_KVEC)) { \
- const struct kvec *kvec; \
- struct kvec v; \
- iterate_kvec(i, n, v, kvec, skip, (K)) \
- } else { \
- const struct iovec *iov; \
- struct iovec v; \
- iterate_iovec(i, n, v, iov, skip, (I)) \
+ if (likely(n)) { \
+ size_t skip = i->iov_offset; \
+ if (unlikely(i->type & ITER_BVEC)) { \
+ struct bio_vec v; \
+ struct bvec_iter __bi; \
+ iterate_bvec(i, n, v, __bi, skip, (B)) \
+ } else if (unlikely(i->type & ITER_KVEC)) { \
+ const struct kvec *kvec; \
+ struct kvec v; \
+ iterate_kvec(i, n, v, kvec, skip, (K)) \
+ } else { \
+ const struct iovec *iov; \
+ struct iovec v; \
+ iterate_iovec(i, n, v, iov, skip, (I)) \
+ } \
} \
}
@@ -568,6 +571,31 @@ size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
}
EXPORT_SYMBOL(copy_from_iter);
+bool copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
+{
+ char *to = addr;
+ if (unlikely(i->type & ITER_PIPE)) {
+ WARN_ON(1);
+ return false;
+ }
+ if (unlikely(i->count < bytes))
+ return false;
+
+ iterate_all_kinds(i, bytes, v, ({
+ if (__copy_from_user((to += v.iov_len) - v.iov_len,
+ v.iov_base, v.iov_len))
+ return false;
+ 0;}),
+ memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
+ v.bv_offset, v.bv_len),
+ memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
+ )
+
+ iov_iter_advance(i, bytes);
+ return true;
+}
+EXPORT_SYMBOL(copy_from_iter_full);
+
size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
{
char *to = addr;
@@ -587,6 +615,30 @@ size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
}
EXPORT_SYMBOL(copy_from_iter_nocache);
+bool copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
+{
+ char *to = addr;
+ if (unlikely(i->type & ITER_PIPE)) {
+ WARN_ON(1);
+ return false;
+ }
+ if (unlikely(i->count < bytes))
+ return false;
+ iterate_all_kinds(i, bytes, v, ({
+ if (__copy_from_user_nocache((to += v.iov_len) - v.iov_len,
+ v.iov_base, v.iov_len))
+ return false;
+ 0;}),
+ memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
+ v.bv_offset, v.bv_len),
+ memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
+ )
+
+ iov_iter_advance(i, bytes);
+ return true;
+}
+EXPORT_SYMBOL(copy_from_iter_full_nocache);
+
size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
struct iov_iter *i)
{
@@ -787,11 +839,8 @@ unsigned long iov_iter_alignment(const struct iov_iter *i)
unsigned long res = 0;
size_t size = i->count;
- if (!size)
- return 0;
-
if (unlikely(i->type & ITER_PIPE)) {
- if (i->iov_offset && allocated(&i->pipe->bufs[i->idx]))
+ if (size && i->iov_offset && allocated(&i->pipe->bufs[i->idx]))
return size | i->iov_offset;
return size;
}
@@ -806,10 +855,8 @@ EXPORT_SYMBOL(iov_iter_alignment);
unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
{
- unsigned long res = 0;
+ unsigned long res = 0;
size_t size = i->count;
- if (!size)
- return 0;
if (unlikely(i->type & ITER_PIPE)) {
WARN_ON(1);
@@ -824,7 +871,7 @@ unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
(res |= (!res ? 0 : (unsigned long)v.iov_base) |
(size != v.iov_len ? size : 0))
);
- return res;
+ return res;
}
EXPORT_SYMBOL(iov_iter_gap_alignment);
@@ -858,6 +905,9 @@ static ssize_t pipe_get_pages(struct iov_iter *i,
size_t capacity;
int idx;
+ if (!maxsize)
+ return 0;
+
if (!sanity(i))
return -EFAULT;
@@ -876,9 +926,6 @@ ssize_t iov_iter_get_pages(struct iov_iter *i,
if (maxsize > i->count)
maxsize = i->count;
- if (!maxsize)
- return 0;
-
if (unlikely(i->type & ITER_PIPE))
return pipe_get_pages(i, pages, maxsize, maxpages, start);
iterate_all_kinds(i, maxsize, v, ({
@@ -925,6 +972,9 @@ static ssize_t pipe_get_pages_alloc(struct iov_iter *i,
int idx;
int npages;
+ if (!maxsize)
+ return 0;
+
if (!sanity(i))
return -EFAULT;
@@ -956,9 +1006,6 @@ ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
if (maxsize > i->count)
maxsize = i->count;
- if (!maxsize)
- return 0;
-
if (unlikely(i->type & ITER_PIPE))
return pipe_get_pages_alloc(i, pages, maxsize, start);
iterate_all_kinds(i, maxsize, v, ({
@@ -1008,7 +1055,7 @@ size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
}
iterate_and_advance(i, bytes, v, ({
int err = 0;
- next = csum_and_copy_from_user(v.iov_base,
+ next = csum_and_copy_from_user(v.iov_base,
(to += v.iov_len) - v.iov_len,
v.iov_len, 0, &err);
if (!err) {
@@ -1037,6 +1084,51 @@ size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
}
EXPORT_SYMBOL(csum_and_copy_from_iter);
+bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum,
+ struct iov_iter *i)
+{
+ char *to = addr;
+ __wsum sum, next;
+ size_t off = 0;
+ sum = *csum;
+ if (unlikely(i->type & ITER_PIPE)) {
+ WARN_ON(1);
+ return false;
+ }
+ if (unlikely(i->count < bytes))
+ return false;
+ iterate_all_kinds(i, bytes, v, ({
+ int err = 0;
+ next = csum_and_copy_from_user(v.iov_base,
+ (to += v.iov_len) - v.iov_len,
+ v.iov_len, 0, &err);
+ if (err)
+ return false;
+ sum = csum_block_add(sum, next, off);
+ off += v.iov_len;
+ 0;
+ }), ({
+ char *p = kmap_atomic(v.bv_page);
+ next = csum_partial_copy_nocheck(p + v.bv_offset,
+ (to += v.bv_len) - v.bv_len,
+ v.bv_len, 0);
+ kunmap_atomic(p);
+ sum = csum_block_add(sum, next, off);
+ off += v.bv_len;
+ }),({
+ next = csum_partial_copy_nocheck(v.iov_base,
+ (to += v.iov_len) - v.iov_len,
+ v.iov_len, 0);
+ sum = csum_block_add(sum, next, off);
+ off += v.iov_len;
+ })
+ )
+ *csum = sum;
+ iov_iter_advance(i, bytes);
+ return true;
+}
+EXPORT_SYMBOL(csum_and_copy_from_iter_full);
+
size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,
struct iov_iter *i)
{
@@ -1051,7 +1143,7 @@ size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,
iterate_and_advance(i, bytes, v, ({
int err = 0;
next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
- v.iov_base,
+ v.iov_base,
v.iov_len, 0, &err);
if (!err) {
sum = csum_block_add(sum, next, off);
generated by cgit (git 2.34.1) at 2025-06-06 23:22:42 +0000