1 files changed, 33 insertions, 52 deletions

diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c
index 7409490259d1..c37668790577 100644
--- a/drivers/md/dm-io.c
+++ b/drivers/md/dm-io.c
@@ -347,7 +347,7 @@ static void do_region(const blk_opf_t opf, unsigned int region,
 			break;
 		default:
 			num_bvecs = bio_max_segs(dm_sector_div_up(remaining,
-						(PAGE_SIZE >> SECTOR_SHIFT)));
+						(PAGE_SIZE >> SECTOR_SHIFT)) + 1);
 		}
 
 		bio = bio_alloc_bioset(where->bdev, num_bvecs, opf, GFP_NOIO,
@@ -379,21 +379,19 @@ static void do_region(const blk_opf_t opf, unsigned int region,
 
 		atomic_inc(&io->count);
 		submit_bio(bio);
+		WARN_ON_ONCE(opf & REQ_ATOMIC && remaining);
 	} while (remaining);
 }
 
 static void dispatch_io(blk_opf_t opf, unsigned int num_regions,
 			struct dm_io_region *where, struct dpages *dp,
-			struct io *io, int sync, unsigned short ioprio)
+			struct io *io, unsigned short ioprio)
 {
 	int i;
 	struct dpages old_pages = *dp;
 
 	BUG_ON(num_regions > DM_IO_MAX_REGIONS);
 
-	if (sync)
-		opf |= REQ_SYNC;
-
 	/*
 	 * For multiple regions we need to be careful to rewind
 	 * the dp object for each call to do_region.
@@ -411,6 +409,26 @@ static void dispatch_io(blk_opf_t opf, unsigned int num_regions,
 	dec_count(io, 0, 0);
 }
 
+static void async_io(struct dm_io_client *client, unsigned int num_regions,
+		     struct dm_io_region *where, blk_opf_t opf,
+		     struct dpages *dp, io_notify_fn fn, void *context,
+		     unsigned short ioprio)
+{
+	struct io *io;
+
+	io = mempool_alloc(&client->pool, GFP_NOIO);
+	io->error_bits = 0;
+	atomic_set(&io->count, 1); /* see dispatch_io() */
+	io->client = client;
+	io->callback = fn;
+	io->context = context;
+
+	io->vma_invalidate_address = dp->vma_invalidate_address;
+	io->vma_invalidate_size = dp->vma_invalidate_size;
+
+	dispatch_io(opf, num_regions, where, dp, io, ioprio);
+}
+
 struct sync_io {
 	unsigned long error_bits;
 	struct completion wait;
@@ -428,27 +446,12 @@ static int sync_io(struct dm_io_client *client, unsigned int num_regions,
 		   struct dm_io_region *where, blk_opf_t opf, struct dpages *dp,
 		   unsigned long *error_bits, unsigned short ioprio)
 {
-	struct io *io;
 	struct sync_io sio;
 
-	if (num_regions > 1 && !op_is_write(opf)) {
-		WARN_ON(1);
-		return -EIO;
-	}
-
 	init_completion(&sio.wait);
 
-	io = mempool_alloc(&client->pool, GFP_NOIO);
-	io->error_bits = 0;
-	atomic_set(&io->count, 1); /* see dispatch_io() */
-	io->client = client;
-	io->callback = sync_io_complete;
-	io->context = &sio;
-
-	io->vma_invalidate_address = dp->vma_invalidate_address;
-	io->vma_invalidate_size = dp->vma_invalidate_size;
-
-	dispatch_io(opf, num_regions, where, dp, io, 1, ioprio);
+	async_io(client, num_regions, where, opf | REQ_SYNC, dp,
+		 sync_io_complete, &sio, ioprio);
 
 	wait_for_completion_io(&sio.wait);
 
@@ -458,33 +461,6 @@ static int sync_io(struct dm_io_client *client, unsigned int num_regions,
 	return sio.error_bits ? -EIO : 0;
 }
 
-static int async_io(struct dm_io_client *client, unsigned int num_regions,
-		    struct dm_io_region *where, blk_opf_t opf,
-		    struct dpages *dp, io_notify_fn fn, void *context,
-		    unsigned short ioprio)
-{
-	struct io *io;
-
-	if (num_regions > 1 && !op_is_write(opf)) {
-		WARN_ON(1);
-		fn(1, context);
-		return -EIO;
-	}
-
-	io = mempool_alloc(&client->pool, GFP_NOIO);
-	io->error_bits = 0;
-	atomic_set(&io->count, 1); /* see dispatch_io() */
-	io->client = client;
-	io->callback = fn;
-	io->context = context;
-
-	io->vma_invalidate_address = dp->vma_invalidate_address;
-	io->vma_invalidate_size = dp->vma_invalidate_size;
-
-	dispatch_io(opf, num_regions, where, dp, io, 0, ioprio);
-	return 0;
-}
-
 static int dp_init(struct dm_io_request *io_req, struct dpages *dp,
 		   unsigned long size)
 {
@@ -529,6 +505,11 @@ int dm_io(struct dm_io_request *io_req, unsigned int num_regions,
 	int r;
 	struct dpages dp;
 
+	if (num_regions > 1 && !op_is_write(io_req->bi_opf)) {
+		WARN_ON(1);
+		return -EIO;
+	}
+
 	r = dp_init(io_req, &dp, (unsigned long)where->count << SECTOR_SHIFT);
 	if (r)
 		return r;
@@ -537,9 +518,9 @@ int dm_io(struct dm_io_request *io_req, unsigned int num_regions,
 		return sync_io(io_req->client, num_regions, where,
 			       io_req->bi_opf, &dp, sync_error_bits, ioprio);
 
-	return async_io(io_req->client, num_regions, where,
-			io_req->bi_opf, &dp, io_req->notify.fn,
-			io_req->notify.context, ioprio);
+	async_io(io_req->client, num_regions, where, io_req->bi_opf, &dp,
+		 io_req->notify.fn, io_req->notify.context, ioprio);
+	return 0;
 }
 EXPORT_SYMBOL(dm_io);