From 076098e51bd502402a7c166b5c2bb59a196e84e1 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 23 Sep 2017 16:08:57 -0400 Subject: bio_map_user_iov(): switch to iov_iter_get_pages()/iov_iter_advance() ... and to hell with iov_for_each() nonsense Signed-off-by: Al Viro --- block/bio.c | 57 ++++++++++++++++++++++++--------------------------------- 1 file changed, 24 insertions(+), 33 deletions(-) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index 101c2a9b5481..5de54dedab66 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1328,7 +1328,7 @@ struct bio *bio_map_user_iov(struct request_queue *q, struct page **pages; struct bio *bio; int cur_page = 0; - int ret, offset; + int ret; struct iov_iter i; struct iovec iov; struct bio_vec *bvec; @@ -1365,43 +1365,32 @@ struct bio *bio_map_user_iov(struct request_queue *q, if (!pages) goto out; - iov_for_each(iov, i, *iter) { - unsigned long uaddr = (unsigned long) iov.iov_base; - unsigned long len = iov.iov_len; - unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; - unsigned long start = uaddr >> PAGE_SHIFT; - const int local_nr_pages = end - start; - const int page_limit = cur_page + local_nr_pages; - - ret = get_user_pages_fast(uaddr, local_nr_pages, - (iter->type & WRITE) != WRITE, - &pages[cur_page]); - if (unlikely(ret < local_nr_pages)) { - for (j = cur_page; j < page_limit; j++) { - if (!pages[j]) - break; - put_page(pages[j]); - } - ret = -EFAULT; + i = *iter; + while (iov_iter_count(&i)) { + ssize_t bytes; + size_t offs, added = 0; + int npages; + + bytes = iov_iter_get_pages(&i, pages + cur_page, LONG_MAX, + nr_pages - cur_page, &offs); + if (unlikely(bytes <= 0)) { + ret = bytes ? bytes : -EFAULT; goto out_unmap; } - offset = offset_in_page(uaddr); - for (j = cur_page; j < page_limit; j++) { - unsigned int bytes = PAGE_SIZE - offset; + npages = DIV_ROUND_UP(offs + bytes, PAGE_SIZE); + + for (j = cur_page; j < cur_page + npages; j++) { + unsigned int n = PAGE_SIZE - offs; unsigned short prev_bi_vcnt = bio->bi_vcnt; - if (len <= 0) - break; - - if (bytes > len) - bytes = len; + if (n > bytes) + n = bytes; /* * sorry... */ - if (bio_add_pc_page(q, bio, pages[j], bytes, offset) < - bytes) + if (bio_add_pc_page(q, bio, pages[j], n, offs) < n) break; /* @@ -1411,16 +1400,18 @@ struct bio *bio_map_user_iov(struct request_queue *q, if (bio->bi_vcnt == prev_bi_vcnt) put_page(pages[j]); - len -= bytes; - offset = 0; + added += n; + bytes -= n; + offs = 0; } + iov_iter_advance(&i, added); - cur_page = j; /* * release the pages we didn't map into the bio, if any */ - while (j < page_limit) + while (j < cur_page + npages) put_page(pages[j++]); + cur_page = j; } kfree(pages); -- cgit From 629e42bcc3d0bc04b4e0e40ef3f831507a4693bd Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 23 Sep 2017 16:13:10 -0400 Subject: ... and with iov_iter_get_pages_alloc() it becomes even simpler Signed-off-by: Al Viro --- block/bio.c | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index 5de54dedab66..5dbc5e90d716 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1325,9 +1325,7 @@ struct bio *bio_map_user_iov(struct request_queue *q, { int j; int nr_pages = 0; - struct page **pages; struct bio *bio; - int cur_page = 0; int ret; struct iov_iter i; struct iovec iov; @@ -1360,19 +1358,14 @@ struct bio *bio_map_user_iov(struct request_queue *q, if (!bio) return ERR_PTR(-ENOMEM); - ret = -ENOMEM; - pages = kcalloc(nr_pages, sizeof(struct page *), gfp_mask); - if (!pages) - goto out; - i = *iter; while (iov_iter_count(&i)) { + struct page **pages; ssize_t bytes; size_t offs, added = 0; int npages; - bytes = iov_iter_get_pages(&i, pages + cur_page, LONG_MAX, - nr_pages - cur_page, &offs); + bytes = iov_iter_get_pages_alloc(&i, &pages, LONG_MAX, &offs); if (unlikely(bytes <= 0)) { ret = bytes ? bytes : -EFAULT; goto out_unmap; @@ -1380,7 +1373,7 @@ struct bio *bio_map_user_iov(struct request_queue *q, npages = DIV_ROUND_UP(offs + bytes, PAGE_SIZE); - for (j = cur_page; j < cur_page + npages; j++) { + for (j = 0; j < npages; j++) { unsigned int n = PAGE_SIZE - offs; unsigned short prev_bi_vcnt = bio->bi_vcnt; @@ -1409,13 +1402,11 @@ struct bio *bio_map_user_iov(struct request_queue *q, /* * release the pages we didn't map into the bio, if any */ - while (j < cur_page + npages) + while (j < npages) put_page(pages[j++]); - cur_page = j; + kvfree(pages); } - kfree(pages); - bio_set_flag(bio, BIO_USER_MAPPED); /* @@ -1431,8 +1422,6 @@ struct bio *bio_map_user_iov(struct request_queue *q, bio_for_each_segment_all(bvec, bio, j) { put_page(bvec->bv_page); } - out: - kfree(pages); bio_put(bio); return ERR_PTR(ret); } -- cgit From e2e115d18b76467274d8f818f8828ba168f9c80b Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 23 Sep 2017 16:16:06 -0400 Subject: don't rely upon subsequent bio_add_pc_page() calls failing ... they might actually succeed in some cases (when we are at the queue-imposed segments limit, the next page is not mergable with the last one we'd got in, but the first page covered by the next iovec *is* mergable). Make sure that once it's failed, we are done with that bio. Signed-off-by: Al Viro --- block/bio.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index 5dbc5e90d716..fe40948d62af 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1380,10 +1380,7 @@ struct bio *bio_map_user_iov(struct request_queue *q, if (n > bytes) n = bytes; - /* - * sorry... - */ - if (bio_add_pc_page(q, bio, pages[j], n, offs) < n) + if (!bio_add_pc_page(q, bio, pages[j], n, offs)) break; /* @@ -1405,6 +1402,9 @@ struct bio *bio_map_user_iov(struct request_queue *q, while (j < npages) put_page(pages[j++]); kvfree(pages); + /* couldn't stuff something into bio? */ + if (bytes) + break; } bio_set_flag(bio, BIO_USER_MAPPED); -- cgit From 98f0bc99055da5797fae0c35d3d18261d59df9ac Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 23 Sep 2017 16:23:18 -0400 Subject: bio_map_user_iov(): move alignment check into the main loop Signed-off-by: Al Viro --- block/bio.c | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index fe40948d62af..d851f68727f1 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1344,11 +1344,6 @@ struct bio *bio_map_user_iov(struct request_queue *q, return ERR_PTR(-EINVAL); nr_pages += end - start; - /* - * buffer must be aligned to at least logical block size for now - */ - if (uaddr & queue_dma_alignment(q)) - return ERR_PTR(-EINVAL); } if (!nr_pages) @@ -1373,29 +1368,34 @@ struct bio *bio_map_user_iov(struct request_queue *q, npages = DIV_ROUND_UP(offs + bytes, PAGE_SIZE); - for (j = 0; j < npages; j++) { - unsigned int n = PAGE_SIZE - offs; - unsigned short prev_bi_vcnt = bio->bi_vcnt; - - if (n > bytes) - n = bytes; - - if (!bio_add_pc_page(q, bio, pages[j], n, offs)) - break; - - /* - * check if vector was merged with previous - * drop page reference if needed - */ - if (bio->bi_vcnt == prev_bi_vcnt) - put_page(pages[j]); - - added += n; - bytes -= n; - offs = 0; + if (unlikely(offs & queue_dma_alignment(q))) { + ret = -EINVAL; + j = 0; + } else { + for (j = 0; j < npages; j++) { + struct page *page = pages[j]; + unsigned int n = PAGE_SIZE - offs; + unsigned short prev_bi_vcnt = bio->bi_vcnt; + + if (n > bytes) + n = bytes; + + if (!bio_add_pc_page(q, bio, page, n, offs)) + break; + + /* + * check if vector was merged with previous + * drop page reference if needed + */ + if (bio->bi_vcnt == prev_bi_vcnt) + put_page(page); + + added += n; + bytes -= n; + offs = 0; + } + iov_iter_advance(&i, added); } - iov_iter_advance(&i, added); - /* * release the pages we didn't map into the bio, if any */ -- cgit From b282cc766958af161249b7b04d50e3eae12a2a1c Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 23 Sep 2017 16:24:59 -0400 Subject: bio_map_user_iov(): get rid of the iov_for_each() Use iov_iter_npages() Signed-off-by: Al Viro --- block/bio.c | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index d851f68727f1..d1ca7eecc8aa 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1324,32 +1324,15 @@ struct bio *bio_map_user_iov(struct request_queue *q, gfp_t gfp_mask) { int j; - int nr_pages = 0; struct bio *bio; int ret; struct iov_iter i; - struct iovec iov; struct bio_vec *bvec; - iov_for_each(iov, i, *iter) { - unsigned long uaddr = (unsigned long) iov.iov_base; - unsigned long len = iov.iov_len; - unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; - unsigned long start = uaddr >> PAGE_SHIFT; - - /* - * Overflow, abort - */ - if (end < start) - return ERR_PTR(-EINVAL); - - nr_pages += end - start; - } - - if (!nr_pages) + if (!iov_iter_count(iter)) return ERR_PTR(-EINVAL); - bio = bio_kmalloc(gfp_mask, nr_pages); + bio = bio_kmalloc(gfp_mask, iov_iter_npages(iter, BIO_MAX_PAGES)); if (!bio) return ERR_PTR(-ENOMEM); -- cgit From e81cef5d3001501350b4e596b4bd6dfd26187afa Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 24 Sep 2017 09:25:39 -0400 Subject: blk_rq_map_user_iov(): move iov_iter_advance() down ... into bio_{map,copy}_user_iov() Signed-off-by: Al Viro --- block/bio.c | 6 ++++-- block/blk-map.c | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index d1ca7eecc8aa..cd1282db03cb 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1195,7 +1195,7 @@ int bio_uncopy_user(struct bio *bio) */ struct bio *bio_copy_user_iov(struct request_queue *q, struct rq_map_data *map_data, - const struct iov_iter *iter, + struct iov_iter *iter, gfp_t gfp_mask) { struct bio_map_data *bmd; @@ -1298,6 +1298,7 @@ struct bio *bio_copy_user_iov(struct request_queue *q, if (ret) goto cleanup; } + iov_iter_advance(iter, bio->bi_iter.bi_size); bio->bi_private = bmd; return bio; @@ -1320,7 +1321,7 @@ out_bmd: * device. Returns an error pointer in case of error. */ struct bio *bio_map_user_iov(struct request_queue *q, - const struct iov_iter *iter, + struct iov_iter *iter, gfp_t gfp_mask) { int j; @@ -1399,6 +1400,7 @@ struct bio *bio_map_user_iov(struct request_queue *q, * reference to it */ bio_get(bio); + iov_iter_advance(iter, bio->bi_iter.bi_size); return bio; out_unmap: diff --git a/block/blk-map.c b/block/blk-map.c index 2547016aa7aa..891eea11f68e 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -69,7 +69,6 @@ static int __blk_rq_map_user_iov(struct request *rq, if (map_data && map_data->null_mapped) bio_set_flag(bio, BIO_NULL_MAPPED); - iov_iter_advance(iter, bio->bi_iter.bi_size); if (map_data) map_data->offset += bio->bi_iter.bi_size; -- cgit From 2884d0be878eb5cbbc6d983c6054feef3b9aa86d Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 24 Sep 2017 12:09:21 -0400 Subject: move more stuff down into bio_copy_user_iov() Signed-off-by: Al Viro --- block/bio.c | 5 +++++ block/blk-map.c | 6 ------ 2 files changed, 5 insertions(+), 6 deletions(-) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index cd1282db03cb..02457c2d4379 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1289,6 +1289,9 @@ struct bio *bio_copy_user_iov(struct request_queue *q, if (ret) goto cleanup; + if (map_data) + map_data->offset += bio->bi_iter.bi_size; + /* * success */ @@ -1301,6 +1304,8 @@ struct bio *bio_copy_user_iov(struct request_queue *q, iov_iter_advance(iter, bio->bi_iter.bi_size); bio->bi_private = bmd; + if (map_data && map_data->null_mapped) + bio_set_flag(bio, BIO_NULL_MAPPED); return bio; cleanup: if (!map_data) diff --git a/block/blk-map.c b/block/blk-map.c index 891eea11f68e..c872d62b62fb 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -66,12 +66,6 @@ static int __blk_rq_map_user_iov(struct request *rq, bio->bi_opf &= ~REQ_OP_MASK; bio->bi_opf |= req_op(rq); - if (map_data && map_data->null_mapped) - bio_set_flag(bio, BIO_NULL_MAPPED); - - if (map_data) - map_data->offset += bio->bi_iter.bi_size; - orig_bio = bio; /* -- cgit From 98a09d6106660ec6e69ff2a6fa14039bd504412b Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 24 Sep 2017 12:14:36 -0400 Subject: bio_copy_from_iter(): get rid of copying iov_iter we want the one passed to it advanced, anyway Signed-off-by: Al Viro --- block/bio.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index 02457c2d4379..f5002b44c963 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1088,7 +1088,7 @@ static struct bio_map_data *bio_alloc_map_data(unsigned int iov_count, * Copy all pages from iov_iter to bio. * Returns 0 on success, or error on failure. */ -static int bio_copy_from_iter(struct bio *bio, struct iov_iter iter) +static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter) { int i; struct bio_vec *bvec; @@ -1099,9 +1099,9 @@ static int bio_copy_from_iter(struct bio *bio, struct iov_iter iter) ret = copy_page_from_iter(bvec->bv_page, bvec->bv_offset, bvec->bv_len, - &iter); + iter); - if (!iov_iter_count(&iter)) + if (!iov_iter_count(iter)) break; if (ret < bvec->bv_len) @@ -1297,11 +1297,12 @@ struct bio *bio_copy_user_iov(struct request_queue *q, */ if (((iter->type & WRITE) && (!map_data || !map_data->null_mapped)) || (map_data && map_data->from_user)) { - ret = bio_copy_from_iter(bio, *iter); + ret = bio_copy_from_iter(bio, iter); if (ret) goto cleanup; + } else { + iov_iter_advance(iter, bio->bi_iter.bi_size); } - iov_iter_advance(iter, bio->bi_iter.bi_size); bio->bi_private = bmd; if (map_data && map_data->null_mapped) -- cgit From 0a0f151364f5bf836ad1d4de6113adb103a6628c Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 24 Sep 2017 12:30:17 -0400 Subject: bio_map_user_iov(): get rid of copying iov_iter we do want *iter advanced Signed-off-by: Al Viro --- block/bio.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index f5002b44c963..28f66e2edc53 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1333,7 +1333,6 @@ struct bio *bio_map_user_iov(struct request_queue *q, int j; struct bio *bio; int ret; - struct iov_iter i; struct bio_vec *bvec; if (!iov_iter_count(iter)) @@ -1343,14 +1342,13 @@ struct bio *bio_map_user_iov(struct request_queue *q, if (!bio) return ERR_PTR(-ENOMEM); - i = *iter; - while (iov_iter_count(&i)) { + while (iov_iter_count(iter)) { struct page **pages; ssize_t bytes; size_t offs, added = 0; int npages; - bytes = iov_iter_get_pages_alloc(&i, &pages, LONG_MAX, &offs); + bytes = iov_iter_get_pages_alloc(iter, &pages, LONG_MAX, &offs); if (unlikely(bytes <= 0)) { ret = bytes ? bytes : -EFAULT; goto out_unmap; @@ -1384,7 +1382,7 @@ struct bio *bio_map_user_iov(struct request_queue *q, bytes -= n; offs = 0; } - iov_iter_advance(&i, added); + iov_iter_advance(iter, added); } /* * release the pages we didn't map into the bio, if any @@ -1406,7 +1404,6 @@ struct bio *bio_map_user_iov(struct request_queue *q, * reference to it */ bio_get(bio); - iov_iter_advance(iter, bio->bi_iter.bi_size); return bio; out_unmap: -- cgit From d16d44ebb016792285ec1b9566dbd9d022ce70f9 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 24 Sep 2017 13:09:18 -0400 Subject: bio_copy_user_iov(): saner bio size calculation it's a bounce buffer; we don't *care* how badly is the real source/destination fragmented, all that matters is the total size. Signed-off-by: Al Viro --- block/bio.c | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index 28f66e2edc53..e87f70cd528e 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1201,33 +1201,11 @@ struct bio *bio_copy_user_iov(struct request_queue *q, struct bio_map_data *bmd; struct page *page; struct bio *bio; - int i, ret; - int nr_pages = 0; + int i = 0, ret; + int nr_pages; unsigned int len = iter->count; unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0; - for (i = 0; i < iter->nr_segs; i++) { - unsigned long uaddr; - unsigned long end; - unsigned long start; - - uaddr = (unsigned long) iter->iov[i].iov_base; - end = (uaddr + iter->iov[i].iov_len + PAGE_SIZE - 1) - >> PAGE_SHIFT; - start = uaddr >> PAGE_SHIFT; - - /* - * Overflow, abort - */ - if (end < start) - return ERR_PTR(-EINVAL); - - nr_pages += end - start; - } - - if (offset) - nr_pages++; - bmd = bio_alloc_map_data(iter->nr_segs, gfp_mask); if (!bmd) return ERR_PTR(-ENOMEM); @@ -1242,6 +1220,10 @@ struct bio *bio_copy_user_iov(struct request_queue *q, bmd->iter = *iter; bmd->iter.iov = bmd->iov; + nr_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE); + if (nr_pages > BIO_MAX_PAGES) + nr_pages = BIO_MAX_PAGES; + ret = -ENOMEM; bio = bio_kmalloc(gfp_mask, nr_pages); if (!bio) -- cgit From 0e5b935d43f385ab23d2e38e7134b1abb0e7907e Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 24 Sep 2017 13:14:35 -0400 Subject: bio_alloc_map_data(): do bmd->iter setup right there just need to copy it iter instead of iter->nr_segs Signed-off-by: Al Viro --- block/bio.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index e87f70cd528e..ad34cdb99ad2 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1070,14 +1070,21 @@ struct bio_map_data { struct iovec iov[]; }; -static struct bio_map_data *bio_alloc_map_data(unsigned int iov_count, +static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data, gfp_t gfp_mask) { - if (iov_count > UIO_MAXIOV) + struct bio_map_data *bmd; + if (data->nr_segs > UIO_MAXIOV) return NULL; - return kmalloc(sizeof(struct bio_map_data) + - sizeof(struct iovec) * iov_count, gfp_mask); + bmd = kmalloc(sizeof(struct bio_map_data) + + sizeof(struct iovec) * data->nr_segs, gfp_mask); + if (!bmd) + return NULL; + memcpy(bmd->iov, data->iov, sizeof(struct iovec) * data->nr_segs); + bmd->iter = *data; + bmd->iter.iov = bmd->iov; + return bmd; } /** @@ -1206,7 +1213,7 @@ struct bio *bio_copy_user_iov(struct request_queue *q, unsigned int len = iter->count; unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0; - bmd = bio_alloc_map_data(iter->nr_segs, gfp_mask); + bmd = bio_alloc_map_data(iter, gfp_mask); if (!bmd) return ERR_PTR(-ENOMEM); @@ -1216,9 +1223,6 @@ struct bio *bio_copy_user_iov(struct request_queue *q, * shortlived one. */ bmd->is_our_pages = map_data ? 0 : 1; - memcpy(bmd->iov, iter->iov, sizeof(struct iovec) * iter->nr_segs); - bmd->iter = *iter; - bmd->iter.iov = bmd->iov; nr_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE); if (nr_pages > BIO_MAX_PAGES) -- cgit