From 4e49ea4a3d276365bf7396c9b77b4d1d5923835a Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sun, 5 Jun 2016 14:31:41 -0500 Subject: block/fs/drivers: remove rw argument from submit_bio This has callers of submit_bio/submit_bio_wait set the bio->bi_rw instead of passing it in. This makes that use the same as generic_make_request and how we set the other bio fields. Signed-off-by: Mike Christie Fixed up fs/ext4/crypto.c Signed-off-by: Jens Axboe --- block/bio.c | 7 +++---- block/blk-core.c | 11 ++++------- block/blk-flush.c | 3 ++- block/blk-lib.c | 20 +++++++++++--------- 4 files changed, 20 insertions(+), 21 deletions(-) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index 0e4aa42bc30d..fc779eba0b95 100644 --- a/block/bio.c +++ b/block/bio.c @@ -854,21 +854,20 @@ static void submit_bio_wait_endio(struct bio *bio) /** * submit_bio_wait - submit a bio, and wait until it completes - * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead) * @bio: The &struct bio which describes the I/O * * Simple wrapper around submit_bio(). Returns 0 on success, or the error from * bio_endio() on failure. */ -int submit_bio_wait(int rw, struct bio *bio) +int submit_bio_wait(struct bio *bio) { struct submit_bio_ret ret; - rw |= REQ_SYNC; init_completion(&ret.event); bio->bi_private = &ret; bio->bi_end_io = submit_bio_wait_endio; - submit_bio(rw, bio); + bio->bi_rw |= REQ_SYNC; + submit_bio(bio); wait_for_completion_io(&ret.event); return ret.error; diff --git a/block/blk-core.c b/block/blk-core.c index 2475b1c72773..e95340789592 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -2094,7 +2094,6 @@ EXPORT_SYMBOL(generic_make_request); /** * submit_bio - submit a bio to the block device layer for I/O - * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead) * @bio: The &struct bio which describes the I/O * * submit_bio() is very similar in purpose to generic_make_request(), and @@ -2102,10 +2101,8 @@ EXPORT_SYMBOL(generic_make_request); * interfaces; @bio must be presetup and ready for I/O. * */ -blk_qc_t submit_bio(int rw, struct bio *bio) +blk_qc_t submit_bio(struct bio *bio) { - bio->bi_rw |= rw; - /* * If it's a regular read/write or a barrier with data attached, * go through the normal accounting stuff before submission. @@ -2113,12 +2110,12 @@ blk_qc_t submit_bio(int rw, struct bio *bio) if (bio_has_data(bio)) { unsigned int count; - if (unlikely(rw & REQ_WRITE_SAME)) + if (unlikely(bio->bi_rw & REQ_WRITE_SAME)) count = bdev_logical_block_size(bio->bi_bdev) >> 9; else count = bio_sectors(bio); - if (rw & WRITE) { + if (bio->bi_rw & WRITE) { count_vm_events(PGPGOUT, count); } else { task_io_account_read(bio->bi_iter.bi_size); @@ -2129,7 +2126,7 @@ blk_qc_t submit_bio(int rw, struct bio *bio) char b[BDEVNAME_SIZE]; printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n", current->comm, task_pid_nr(current), - (rw & WRITE) ? "WRITE" : "READ", + (bio->bi_rw & WRITE) ? "WRITE" : "READ", (unsigned long long)bio->bi_iter.bi_sector, bdevname(bio->bi_bdev, b), count); diff --git a/block/blk-flush.c b/block/blk-flush.c index b1c91d229e5e..3af4a5ad46f5 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -485,8 +485,9 @@ int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, bio = bio_alloc(gfp_mask, 0); bio->bi_bdev = bdev; + bio->bi_rw = WRITE_FLUSH; - ret = submit_bio_wait(WRITE_FLUSH, bio); + ret = submit_bio_wait(bio); /* * The driver must store the error location in ->bi_sector, if diff --git a/block/blk-lib.c b/block/blk-lib.c index 23d7f301a196..1f6dec5e0975 100644 --- a/block/blk-lib.c +++ b/block/blk-lib.c @@ -9,14 +9,14 @@ #include "blk.h" -static struct bio *next_bio(struct bio *bio, int rw, unsigned int nr_pages, +static struct bio *next_bio(struct bio *bio, unsigned int nr_pages, gfp_t gfp) { struct bio *new = bio_alloc(gfp, nr_pages); if (bio) { bio_chain(bio, new); - submit_bio(rw, bio); + submit_bio(bio); } return new; @@ -62,9 +62,10 @@ int __blkdev_issue_discard(struct block_device *bdev, sector_t sector, req_sects = end_sect - sector; } - bio = next_bio(bio, type, 1, gfp_mask); + bio = next_bio(bio, 1, gfp_mask); bio->bi_iter.bi_sector = sector; bio->bi_bdev = bdev; + bio->bi_rw = type; bio->bi_iter.bi_size = req_sects << 9; nr_sects -= req_sects; @@ -110,7 +111,7 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, type, &bio); if (!ret && bio) { - ret = submit_bio_wait(type, bio); + ret = submit_bio_wait(bio); if (ret == -EOPNOTSUPP) ret = 0; } @@ -147,13 +148,14 @@ int blkdev_issue_write_same(struct block_device *bdev, sector_t sector, max_write_same_sectors = UINT_MAX >> 9; while (nr_sects) { - bio = next_bio(bio, REQ_WRITE | REQ_WRITE_SAME, 1, gfp_mask); + bio = next_bio(bio, 1, gfp_mask); bio->bi_iter.bi_sector = sector; bio->bi_bdev = bdev; bio->bi_vcnt = 1; bio->bi_io_vec->bv_page = page; bio->bi_io_vec->bv_offset = 0; bio->bi_io_vec->bv_len = bdev_logical_block_size(bdev); + bio->bi_rw = REQ_WRITE | REQ_WRITE_SAME; if (nr_sects > max_write_same_sectors) { bio->bi_iter.bi_size = max_write_same_sectors << 9; @@ -166,7 +168,7 @@ int blkdev_issue_write_same(struct block_device *bdev, sector_t sector, } if (bio) - ret = submit_bio_wait(REQ_WRITE | REQ_WRITE_SAME, bio); + ret = submit_bio_wait(bio); return ret != -EOPNOTSUPP ? ret : 0; } EXPORT_SYMBOL(blkdev_issue_write_same); @@ -190,11 +192,11 @@ static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, unsigned int sz; while (nr_sects != 0) { - bio = next_bio(bio, WRITE, - min(nr_sects, (sector_t)BIO_MAX_PAGES), + bio = next_bio(bio, min(nr_sects, (sector_t)BIO_MAX_PAGES), gfp_mask); bio->bi_iter.bi_sector = sector; bio->bi_bdev = bdev; + bio->bi_rw = REQ_WRITE; while (nr_sects != 0) { sz = min((sector_t) PAGE_SIZE >> 9 , nr_sects); @@ -207,7 +209,7 @@ static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, } if (bio) - return submit_bio_wait(WRITE, bio); + return submit_bio_wait(bio); return 0; } -- cgit From a8ebb056a8aeb58aafef0af241a6b3ac34ac86bd Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sun, 5 Jun 2016 14:31:45 -0500 Subject: block, drivers, cgroup: use op_is_write helper instead of checking for REQ_WRITE We currently set REQ_WRITE/WRITE for all non READ IOs like discard, flush, writesame, etc. In the next patches where we no longer set up the op as a bitmap, we will not be able to detect a operation direction like writesame by testing if REQ_WRITE is set. This patch converts the drivers and cgroup to use the op_is_write helper. This should just cover the simple cases. I did dm, md and bcache in their own patches because they were more involved. Signed-off-by: Mike Christie Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/blk-core.c | 4 ++-- block/blk-merge.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index e95340789592..e8e5865a53e2 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -2115,7 +2115,7 @@ blk_qc_t submit_bio(struct bio *bio) else count = bio_sectors(bio); - if (bio->bi_rw & WRITE) { + if (op_is_write(bio_op(bio))) { count_vm_events(PGPGOUT, count); } else { task_io_account_read(bio->bi_iter.bi_size); @@ -2126,7 +2126,7 @@ blk_qc_t submit_bio(struct bio *bio) char b[BDEVNAME_SIZE]; printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n", current->comm, task_pid_nr(current), - (bio->bi_rw & WRITE) ? "WRITE" : "READ", + op_is_write(bio_op(bio)) ? "WRITE" : "READ", (unsigned long long)bio->bi_iter.bi_sector, bdevname(bio->bi_bdev, b), count); diff --git a/block/blk-merge.c b/block/blk-merge.c index 261353166dcf..b198070c5bc4 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -439,7 +439,7 @@ int blk_rq_map_sg(struct request_queue *q, struct request *rq, } if (q->dma_drain_size && q->dma_drain_needed(rq)) { - if (rq->cmd_flags & REQ_WRITE) + if (op_is_write(req_op(rq))) memset(q->dma_drain_buffer, 0, q->dma_drain_size); sg_unmark_end(sg); -- cgit From 95fe6c1a209ef89d9f94dd04a0ad72be1487d5d5 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sun, 5 Jun 2016 14:31:48 -0500 Subject: block, fs, mm, drivers: use bio set/get op accessors This patch converts the simple bi_rw use cases in the block, drivers, mm and fs code to set/get the bio operation using bio_set_op_attrs/bio_op These should be simple one or two liner cases, so I just did them in one patch. The next patches handle the more complicated cases in a module per patch. Signed-off-by: Mike Christie Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/bio.c | 13 ++++++------- block/blk-core.c | 6 +++--- block/blk-flush.c | 2 +- block/blk-lib.c | 4 ++-- block/blk-map.c | 2 +- block/blk-merge.c | 12 ++++++------ 6 files changed, 19 insertions(+), 20 deletions(-) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index fc779eba0b95..848cd351513b 100644 --- a/block/bio.c +++ b/block/bio.c @@ -656,16 +656,15 @@ struct bio *bio_clone_bioset(struct bio *bio_src, gfp_t gfp_mask, bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs); if (!bio) return NULL; - bio->bi_bdev = bio_src->bi_bdev; bio->bi_rw = bio_src->bi_rw; bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector; bio->bi_iter.bi_size = bio_src->bi_iter.bi_size; - if (bio->bi_rw & REQ_DISCARD) + if (bio_op(bio) == REQ_OP_DISCARD) goto integrity_clone; - if (bio->bi_rw & REQ_WRITE_SAME) { + if (bio_op(bio) == REQ_OP_WRITE_SAME) { bio->bi_io_vec[bio->bi_vcnt++] = bio_src->bi_io_vec[0]; goto integrity_clone; } @@ -1166,7 +1165,7 @@ struct bio *bio_copy_user_iov(struct request_queue *q, goto out_bmd; if (iter->type & WRITE) - bio->bi_rw |= REQ_WRITE; + bio_set_op_attrs(bio, REQ_OP_WRITE, 0); ret = 0; @@ -1336,7 +1335,7 @@ struct bio *bio_map_user_iov(struct request_queue *q, * set data direction, and check if mapped pages need bouncing */ if (iter->type & WRITE) - bio->bi_rw |= REQ_WRITE; + bio_set_op_attrs(bio, REQ_OP_WRITE, 0); bio_set_flag(bio, BIO_USER_MAPPED); @@ -1529,7 +1528,7 @@ struct bio *bio_copy_kern(struct request_queue *q, void *data, unsigned int len, bio->bi_private = data; } else { bio->bi_end_io = bio_copy_kern_endio; - bio->bi_rw |= REQ_WRITE; + bio_set_op_attrs(bio, REQ_OP_WRITE, 0); } return bio; @@ -1784,7 +1783,7 @@ struct bio *bio_split(struct bio *bio, int sectors, * Discards need a mutable bio_vec to accommodate the payload * required by the DSM TRIM and UNMAP commands. */ - if (bio->bi_rw & REQ_DISCARD) + if (bio_op(bio) == REQ_OP_DISCARD) split = bio_clone_bioset(bio, gfp, bs); else split = bio_clone_fast(bio, gfp, bs); diff --git a/block/blk-core.c b/block/blk-core.c index e8e5865a53e2..7e943dce781c 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -1973,14 +1973,14 @@ generic_make_request_checks(struct bio *bio) } } - if ((bio->bi_rw & REQ_DISCARD) && + if ((bio_op(bio) == REQ_OP_DISCARD) && (!blk_queue_discard(q) || ((bio->bi_rw & REQ_SECURE) && !blk_queue_secdiscard(q)))) { err = -EOPNOTSUPP; goto end_io; } - if (bio->bi_rw & REQ_WRITE_SAME && !bdev_write_same(bio->bi_bdev)) { + if (bio_op(bio) == REQ_OP_WRITE_SAME && !bdev_write_same(bio->bi_bdev)) { err = -EOPNOTSUPP; goto end_io; } @@ -2110,7 +2110,7 @@ blk_qc_t submit_bio(struct bio *bio) if (bio_has_data(bio)) { unsigned int count; - if (unlikely(bio->bi_rw & REQ_WRITE_SAME)) + if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME)) count = bdev_logical_block_size(bio->bi_bdev) >> 9; else count = bio_sectors(bio); diff --git a/block/blk-flush.c b/block/blk-flush.c index 3af4a5ad46f5..9fd1f63a6348 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -485,7 +485,7 @@ int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, bio = bio_alloc(gfp_mask, 0); bio->bi_bdev = bdev; - bio->bi_rw = WRITE_FLUSH; + bio_set_op_attrs(bio, REQ_OP_WRITE, WRITE_FLUSH); ret = submit_bio_wait(bio); diff --git a/block/blk-lib.c b/block/blk-lib.c index 1f6dec5e0975..c614eaa7b853 100644 --- a/block/blk-lib.c +++ b/block/blk-lib.c @@ -155,7 +155,7 @@ int blkdev_issue_write_same(struct block_device *bdev, sector_t sector, bio->bi_io_vec->bv_page = page; bio->bi_io_vec->bv_offset = 0; bio->bi_io_vec->bv_len = bdev_logical_block_size(bdev); - bio->bi_rw = REQ_WRITE | REQ_WRITE_SAME; + bio_set_op_attrs(bio, REQ_OP_WRITE_SAME, 0); if (nr_sects > max_write_same_sectors) { bio->bi_iter.bi_size = max_write_same_sectors << 9; @@ -196,7 +196,7 @@ static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, gfp_mask); bio->bi_iter.bi_sector = sector; bio->bi_bdev = bdev; - bio->bi_rw = REQ_WRITE; + bio_set_op_attrs(bio, REQ_OP_WRITE, 0); while (nr_sects != 0) { sz = min((sector_t) PAGE_SIZE >> 9 , nr_sects); diff --git a/block/blk-map.c b/block/blk-map.c index b9f88b7751fb..61733a660c3a 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -224,7 +224,7 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, return PTR_ERR(bio); if (!reading) - bio->bi_rw |= REQ_WRITE; + bio_set_op_attrs(bio, REQ_OP_WRITE, 0); if (do_copy) rq->cmd_flags |= REQ_COPY_USER; diff --git a/block/blk-merge.c b/block/blk-merge.c index b198070c5bc4..5a03f967557a 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -172,9 +172,9 @@ void blk_queue_split(struct request_queue *q, struct bio **bio, struct bio *split, *res; unsigned nsegs; - if ((*bio)->bi_rw & REQ_DISCARD) + if (bio_op(*bio) == REQ_OP_DISCARD) split = blk_bio_discard_split(q, *bio, bs, &nsegs); - else if ((*bio)->bi_rw & REQ_WRITE_SAME) + else if (bio_op(*bio) == REQ_OP_WRITE_SAME) split = blk_bio_write_same_split(q, *bio, bs, &nsegs); else split = blk_bio_segment_split(q, *bio, q->bio_split, &nsegs); @@ -213,10 +213,10 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q, * This should probably be returning 0, but blk_add_request_payload() * (Christoph!!!!) */ - if (bio->bi_rw & REQ_DISCARD) + if (bio_op(bio) == REQ_OP_DISCARD) return 1; - if (bio->bi_rw & REQ_WRITE_SAME) + if (bio_op(bio) == REQ_OP_WRITE_SAME) return 1; fbio = bio; @@ -385,7 +385,7 @@ static int __blk_bios_map_sg(struct request_queue *q, struct bio *bio, nsegs = 0; cluster = blk_queue_cluster(q); - if (bio->bi_rw & REQ_DISCARD) { + if (bio_op(bio) == REQ_OP_DISCARD) { /* * This is a hack - drivers should be neither modifying the * biovec, nor relying on bi_vcnt - but because of @@ -400,7 +400,7 @@ static int __blk_bios_map_sg(struct request_queue *q, struct bio *bio, return 0; } - if (bio->bi_rw & REQ_WRITE_SAME) { + if (bio_op(bio) == REQ_OP_WRITE_SAME) { single_segment: *sg = sglist; bvec = bio_iovec(bio); -- cgit From 469e3216e20a3946a292ff0414ab86de408d465e Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sun, 5 Jun 2016 14:31:49 -0500 Subject: block discard: use bio set op accessor This converts the block issue discard helper and users to use the bio_set_op_attrs accessor and only pass in the operation flags like REQ_SEQURE. Signed-off-by: Mike Christie Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/blk-lib.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'block') diff --git a/block/blk-lib.c b/block/blk-lib.c index c614eaa7b853..ff2a7f04af4d 100644 --- a/block/blk-lib.c +++ b/block/blk-lib.c @@ -23,7 +23,8 @@ static struct bio *next_bio(struct bio *bio, unsigned int nr_pages, } int __blkdev_issue_discard(struct block_device *bdev, sector_t sector, - sector_t nr_sects, gfp_t gfp_mask, int type, struct bio **biop) + sector_t nr_sects, gfp_t gfp_mask, int op_flags, + struct bio **biop) { struct request_queue *q = bdev_get_queue(bdev); struct bio *bio = *biop; @@ -34,7 +35,7 @@ int __blkdev_issue_discard(struct block_device *bdev, sector_t sector, return -ENXIO; if (!blk_queue_discard(q)) return -EOPNOTSUPP; - if ((type & REQ_SECURE) && !blk_queue_secdiscard(q)) + if ((op_flags & REQ_SECURE) && !blk_queue_secdiscard(q)) return -EOPNOTSUPP; /* Zero-sector (unknown) and one-sector granularities are the same. */ @@ -65,7 +66,7 @@ int __blkdev_issue_discard(struct block_device *bdev, sector_t sector, bio = next_bio(bio, 1, gfp_mask); bio->bi_iter.bi_sector = sector; bio->bi_bdev = bdev; - bio->bi_rw = type; + bio_set_op_attrs(bio, REQ_OP_DISCARD, op_flags); bio->bi_iter.bi_size = req_sects << 9; nr_sects -= req_sects; @@ -99,16 +100,16 @@ EXPORT_SYMBOL(__blkdev_issue_discard); int blkdev_issue_discard(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask, unsigned long flags) { - int type = REQ_WRITE | REQ_DISCARD; + int op_flags = 0; struct bio *bio = NULL; struct blk_plug plug; int ret; if (flags & BLKDEV_DISCARD_SECURE) - type |= REQ_SECURE; + op_flags |= REQ_SECURE; blk_start_plug(&plug); - ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, type, + ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, op_flags, &bio); if (!ret && bio) { ret = submit_bio_wait(bio); -- cgit From 4993b77d3ff1d3a887804af2434b7b3dfac4c210 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sun, 5 Jun 2016 14:32:10 -0500 Subject: block: copy bio op to request op The bio users should now always be setting up the bio op. This patch has the block layer copy that to the request. Signed-off-by: Mike Christie Signed-off-by: Jens Axboe --- block/blk-core.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index 7e943dce781c..3c45254ccb01 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -2976,8 +2976,7 @@ EXPORT_SYMBOL_GPL(__blk_end_request_err); void blk_rq_bio_prep(struct request_queue *q, struct request *rq, struct bio *bio) { - /* Bit 0 (R/W) is identical in rq->cmd_flags and bio->bi_rw */ - rq->cmd_flags |= bio->bi_rw & REQ_WRITE; + req_set_op(rq, bio_op(bio)); if (bio_has_data(bio)) rq->nr_phys_segments = bio_phys_segments(q, bio); @@ -3062,7 +3061,8 @@ EXPORT_SYMBOL_GPL(blk_rq_unprep_clone); static void __blk_rq_prep_clone(struct request *dst, struct request *src) { dst->cpu = src->cpu; - dst->cmd_flags |= (src->cmd_flags & REQ_CLONE_MASK) | REQ_NOMERGE; + req_set_op_attrs(dst, req_op(src), + (src->cmd_flags & REQ_CLONE_MASK) | REQ_NOMERGE); dst->cmd_type = src->cmd_type; dst->__sector = blk_rq_pos(src); dst->__data_len = blk_rq_bytes(src); -- cgit From e6a40b096e284ee11374807eaaab6fd21a3fbabb Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sun, 5 Jun 2016 14:32:11 -0500 Subject: block: prepare request creation/destruction code to use REQ_OPs This patch prepares *_get_request/*_put_request and freed_request, to use separate variables for the operation and flags. In the next patches the struct request users will be converted like was done for bios where the op and flags are set separately. Signed-off-by: Mike Christie Reviewed-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/blk-core.c | 54 +++++++++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 25 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index 3c45254ccb01..a68dc0709299 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -959,10 +959,10 @@ static void __freed_request(struct request_list *rl, int sync) * A request has just been released. Account for it, update the full and * congestion status, wake up any waiters. Called under q->queue_lock. */ -static void freed_request(struct request_list *rl, unsigned int flags) +static void freed_request(struct request_list *rl, int op, unsigned int flags) { struct request_queue *q = rl->q; - int sync = rw_is_sync(flags); + int sync = rw_is_sync(op | flags); q->nr_rqs[sync]--; rl->count[sync]--; @@ -1054,7 +1054,8 @@ static struct io_context *rq_ioc(struct bio *bio) /** * __get_request - get a free request * @rl: request list to allocate from - * @rw_flags: RW and SYNC flags + * @op: REQ_OP_READ/REQ_OP_WRITE + * @op_flags: rq_flag_bits * @bio: bio to allocate request for (can be %NULL) * @gfp_mask: allocation mask * @@ -1065,21 +1066,22 @@ static struct io_context *rq_ioc(struct bio *bio) * Returns ERR_PTR on failure, with @q->queue_lock held. * Returns request pointer on success, with @q->queue_lock *not held*. */ -static struct request *__get_request(struct request_list *rl, int rw_flags, - struct bio *bio, gfp_t gfp_mask) +static struct request *__get_request(struct request_list *rl, int op, + int op_flags, struct bio *bio, + gfp_t gfp_mask) { struct request_queue *q = rl->q; struct request *rq; struct elevator_type *et = q->elevator->type; struct io_context *ioc = rq_ioc(bio); struct io_cq *icq = NULL; - const bool is_sync = rw_is_sync(rw_flags) != 0; + const bool is_sync = rw_is_sync(op | op_flags) != 0; int may_queue; if (unlikely(blk_queue_dying(q))) return ERR_PTR(-ENODEV); - may_queue = elv_may_queue(q, rw_flags); + may_queue = elv_may_queue(q, op | op_flags); if (may_queue == ELV_MQUEUE_NO) goto rq_starved; @@ -1123,7 +1125,7 @@ static struct request *__get_request(struct request_list *rl, int rw_flags, /* * Decide whether the new request will be managed by elevator. If - * so, mark @rw_flags and increment elvpriv. Non-zero elvpriv will + * so, mark @op_flags and increment elvpriv. Non-zero elvpriv will * prevent the current elevator from being destroyed until the new * request is freed. This guarantees icq's won't be destroyed and * makes creating new ones safe. @@ -1132,14 +1134,14 @@ static struct request *__get_request(struct request_list *rl, int rw_flags, * it will be created after releasing queue_lock. */ if (blk_rq_should_init_elevator(bio) && !blk_queue_bypass(q)) { - rw_flags |= REQ_ELVPRIV; + op_flags |= REQ_ELVPRIV; q->nr_rqs_elvpriv++; if (et->icq_cache && ioc) icq = ioc_lookup_icq(ioc, q); } if (blk_queue_io_stat(q)) - rw_flags |= REQ_IO_STAT; + op_flags |= REQ_IO_STAT; spin_unlock_irq(q->queue_lock); /* allocate and init request */ @@ -1149,10 +1151,10 @@ static struct request *__get_request(struct request_list *rl, int rw_flags, blk_rq_init(q, rq); blk_rq_set_rl(rq, rl); - rq->cmd_flags = rw_flags | REQ_ALLOCED; + req_set_op_attrs(rq, op, op_flags | REQ_ALLOCED); /* init elvpriv */ - if (rw_flags & REQ_ELVPRIV) { + if (op_flags & REQ_ELVPRIV) { if (unlikely(et->icq_cache && !icq)) { if (ioc) icq = ioc_create_icq(ioc, q, gfp_mask); @@ -1178,7 +1180,7 @@ out: if (ioc_batching(q, ioc)) ioc->nr_batch_requests--; - trace_block_getrq(q, bio, rw_flags & 1); + trace_block_getrq(q, bio, op); return rq; fail_elvpriv: @@ -1208,7 +1210,7 @@ fail_alloc: * queue, but this is pretty rare. */ spin_lock_irq(q->queue_lock); - freed_request(rl, rw_flags); + freed_request(rl, op, op_flags); /* * in the very unlikely event that allocation failed and no @@ -1226,7 +1228,8 @@ rq_starved: /** * get_request - get a free request * @q: request_queue to allocate request from - * @rw_flags: RW and SYNC flags + * @op: REQ_OP_READ/REQ_OP_WRITE + * @op_flags: rq_flag_bits * @bio: bio to allocate request for (can be %NULL) * @gfp_mask: allocation mask * @@ -1237,17 +1240,18 @@ rq_starved: * Returns ERR_PTR on failure, with @q->queue_lock held. * Returns request pointer on success, with @q->queue_lock *not held*. */ -static struct request *get_request(struct request_queue *q, int rw_flags, - struct bio *bio, gfp_t gfp_mask) +static struct request *get_request(struct request_queue *q, int op, + int op_flags, struct bio *bio, + gfp_t gfp_mask) { - const bool is_sync = rw_is_sync(rw_flags) != 0; + const bool is_sync = rw_is_sync(op | op_flags) != 0; DEFINE_WAIT(wait); struct request_list *rl; struct request *rq; rl = blk_get_rl(q, bio); /* transferred to @rq on success */ retry: - rq = __get_request(rl, rw_flags, bio, gfp_mask); + rq = __get_request(rl, op, op_flags, bio, gfp_mask); if (!IS_ERR(rq)) return rq; @@ -1260,7 +1264,7 @@ retry: prepare_to_wait_exclusive(&rl->wait[is_sync], &wait, TASK_UNINTERRUPTIBLE); - trace_block_sleeprq(q, bio, rw_flags & 1); + trace_block_sleeprq(q, bio, op); spin_unlock_irq(q->queue_lock); io_schedule(); @@ -1289,7 +1293,7 @@ static struct request *blk_old_get_request(struct request_queue *q, int rw, create_io_context(gfp_mask, q->node); spin_lock_irq(q->queue_lock); - rq = get_request(q, rw, NULL, gfp_mask); + rq = get_request(q, rw, 0, NULL, gfp_mask); if (IS_ERR(rq)) spin_unlock_irq(q->queue_lock); /* q->queue_lock is unlocked at this point */ @@ -1491,13 +1495,14 @@ void __blk_put_request(struct request_queue *q, struct request *req) */ if (req->cmd_flags & REQ_ALLOCED) { unsigned int flags = req->cmd_flags; + int op = req_op(req); struct request_list *rl = blk_rq_rl(req); BUG_ON(!list_empty(&req->queuelist)); BUG_ON(ELV_ON_HASH(req)); blk_free_request(rl, req); - freed_request(rl, flags); + freed_request(rl, op, flags); blk_put_rl(rl); } } @@ -1712,7 +1717,7 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio) { const bool sync = !!(bio->bi_rw & REQ_SYNC); struct blk_plug *plug; - int el_ret, rw_flags, where = ELEVATOR_INSERT_SORT; + int el_ret, rw_flags = 0, where = ELEVATOR_INSERT_SORT; struct request *req; unsigned int request_count = 0; @@ -1772,7 +1777,6 @@ get_rq: * but we need to set it earlier to expose the sync flag to the * rq allocator and io schedulers. */ - rw_flags = bio_data_dir(bio); if (sync) rw_flags |= REQ_SYNC; @@ -1780,7 +1784,7 @@ get_rq: * Grab a free request. This is might sleep but can not fail. * Returns with the queue unlocked. */ - req = get_request(q, rw_flags, bio, GFP_NOIO); + req = get_request(q, bio_data_dir(bio), rw_flags, bio, GFP_NOIO); if (IS_ERR(req)) { bio->bi_error = PTR_ERR(req); bio_endio(bio); -- cgit From cc6e3b10920b425f0b34d4ff75c63d930aaf14ce Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sun, 5 Jun 2016 14:32:12 -0500 Subject: block: prepare mq request creation to use REQ_OPs This patch modifies the blk mq request creation code to use separate variables for the operation and flags, because in the the next patches the struct request users will be converted like was done for bios. Signed-off-by: Mike Christie Reviewed-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/blk-mq.c | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) (limited to 'block') diff --git a/block/blk-mq.c b/block/blk-mq.c index 29cbc1b5fbdb..3393f29faa9e 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -159,16 +159,17 @@ bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx) EXPORT_SYMBOL(blk_mq_can_queue); static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx, - struct request *rq, unsigned int rw_flags) + struct request *rq, int op, + unsigned int op_flags) { if (blk_queue_io_stat(q)) - rw_flags |= REQ_IO_STAT; + op_flags |= REQ_IO_STAT; INIT_LIST_HEAD(&rq->queuelist); /* csd/requeue_work/fifo_time is initialized before use */ rq->q = q; rq->mq_ctx = ctx; - rq->cmd_flags |= rw_flags; + req_set_op_attrs(rq, op, op_flags); /* do not touch atomic flags, it needs atomic ops against the timer */ rq->cpu = -1; INIT_HLIST_NODE(&rq->hash); @@ -203,11 +204,11 @@ static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx, rq->end_io_data = NULL; rq->next_rq = NULL; - ctx->rq_dispatched[rw_is_sync(rw_flags)]++; + ctx->rq_dispatched[rw_is_sync(op | op_flags)]++; } static struct request * -__blk_mq_alloc_request(struct blk_mq_alloc_data *data, int rw) +__blk_mq_alloc_request(struct blk_mq_alloc_data *data, int op, int op_flags) { struct request *rq; unsigned int tag; @@ -222,7 +223,7 @@ __blk_mq_alloc_request(struct blk_mq_alloc_data *data, int rw) } rq->tag = tag; - blk_mq_rq_ctx_init(data->q, data->ctx, rq, rw); + blk_mq_rq_ctx_init(data->q, data->ctx, rq, op, op_flags); return rq; } @@ -246,7 +247,7 @@ struct request *blk_mq_alloc_request(struct request_queue *q, int rw, hctx = q->mq_ops->map_queue(q, ctx->cpu); blk_mq_set_alloc_data(&alloc_data, q, flags, ctx, hctx); - rq = __blk_mq_alloc_request(&alloc_data, rw); + rq = __blk_mq_alloc_request(&alloc_data, rw, 0); if (!rq && !(flags & BLK_MQ_REQ_NOWAIT)) { __blk_mq_run_hw_queue(hctx); blk_mq_put_ctx(ctx); @@ -254,7 +255,7 @@ struct request *blk_mq_alloc_request(struct request_queue *q, int rw, ctx = blk_mq_get_ctx(q); hctx = q->mq_ops->map_queue(q, ctx->cpu); blk_mq_set_alloc_data(&alloc_data, q, flags, ctx, hctx); - rq = __blk_mq_alloc_request(&alloc_data, rw); + rq = __blk_mq_alloc_request(&alloc_data, rw, 0); ctx = alloc_data.ctx; } blk_mq_put_ctx(ctx); @@ -1169,7 +1170,8 @@ static struct request *blk_mq_map_request(struct request_queue *q, struct blk_mq_hw_ctx *hctx; struct blk_mq_ctx *ctx; struct request *rq; - int rw = bio_data_dir(bio); + int op = bio_data_dir(bio); + int op_flags = 0; struct blk_mq_alloc_data alloc_data; blk_queue_enter_live(q); @@ -1177,20 +1179,20 @@ static struct request *blk_mq_map_request(struct request_queue *q, hctx = q->mq_ops->map_queue(q, ctx->cpu); if (rw_is_sync(bio->bi_rw)) - rw |= REQ_SYNC; + op_flags |= REQ_SYNC; - trace_block_getrq(q, bio, rw); + trace_block_getrq(q, bio, op); blk_mq_set_alloc_data(&alloc_data, q, BLK_MQ_REQ_NOWAIT, ctx, hctx); - rq = __blk_mq_alloc_request(&alloc_data, rw); + rq = __blk_mq_alloc_request(&alloc_data, op, op_flags); if (unlikely(!rq)) { __blk_mq_run_hw_queue(hctx); blk_mq_put_ctx(ctx); - trace_block_sleeprq(q, bio, rw); + trace_block_sleeprq(q, bio, op); ctx = blk_mq_get_ctx(q); hctx = q->mq_ops->map_queue(q, ctx->cpu); blk_mq_set_alloc_data(&alloc_data, q, 0, ctx, hctx); - rq = __blk_mq_alloc_request(&alloc_data, rw); + rq = __blk_mq_alloc_request(&alloc_data, op, op_flags); ctx = alloc_data.ctx; hctx = alloc_data.hctx; } -- cgit From ba568ea0a2ef9a193ca24874228474ec7ae2ff98 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sun, 5 Jun 2016 14:32:13 -0500 Subject: block: prepare elevator to use REQ_OPs. This patch converts the elevator code to use separate variables for the operation and flags, and to check req_op for the REQ_OP. Signed-off-by: Mike Christie Reviewed-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/blk-core.c | 2 +- block/cfq-iosched.c | 4 ++-- block/elevator.c | 7 +++---- 3 files changed, 6 insertions(+), 7 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index a68dc0709299..090e55d7cad7 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -1081,7 +1081,7 @@ static struct request *__get_request(struct request_list *rl, int op, if (unlikely(blk_queue_dying(q))) return ERR_PTR(-ENODEV); - may_queue = elv_may_queue(q, op | op_flags); + may_queue = elv_may_queue(q, op, op_flags); if (may_queue == ELV_MQUEUE_NO) goto rq_starved; diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 4a349787bc62..3fcc5986c01d 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -4285,7 +4285,7 @@ static inline int __cfq_may_queue(struct cfq_queue *cfqq) return ELV_MQUEUE_MAY; } -static int cfq_may_queue(struct request_queue *q, int rw) +static int cfq_may_queue(struct request_queue *q, int op, int op_flags) { struct cfq_data *cfqd = q->elevator->elevator_data; struct task_struct *tsk = current; @@ -4302,7 +4302,7 @@ static int cfq_may_queue(struct request_queue *q, int rw) if (!cic) return ELV_MQUEUE_MAY; - cfqq = cic_to_cfqq(cic, rw_is_sync(rw)); + cfqq = cic_to_cfqq(cic, rw_is_sync(op | op_flags)); if (cfqq) { cfq_init_prio_data(cfqq, cic); diff --git a/block/elevator.c b/block/elevator.c index c3555c9c672f..ea9319db50d7 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -366,8 +366,7 @@ void elv_dispatch_sort(struct request_queue *q, struct request *rq) list_for_each_prev(entry, &q->queue_head) { struct request *pos = list_entry_rq(entry); - if ((rq->cmd_flags & REQ_DISCARD) != - (pos->cmd_flags & REQ_DISCARD)) + if ((req_op(rq) == REQ_OP_DISCARD) != (req_op(pos) == REQ_OP_DISCARD)) break; if (rq_data_dir(rq) != rq_data_dir(pos)) break; @@ -717,12 +716,12 @@ void elv_put_request(struct request_queue *q, struct request *rq) e->type->ops.elevator_put_req_fn(rq); } -int elv_may_queue(struct request_queue *q, int rw) +int elv_may_queue(struct request_queue *q, int op, int op_flags) { struct elevator_queue *e = q->elevator; if (e->type->ops.elevator_may_queue_fn) - return e->type->ops.elevator_may_queue_fn(q, rw); + return e->type->ops.elevator_may_queue_fn(q, op, op_flags); return ELV_MQUEUE_MAY; } -- cgit From 63a4cc24867de73626e16767ce616c50dc5438d3 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sun, 5 Jun 2016 14:32:14 -0500 Subject: blkg_rwstat: separate op from flags The bio and request operation and flags are going to be separate definitions, so we cannot pass them in as a bitmap. This patch converts the blkg_rwstat code and its caller, cfq, to pass in the values separately. Signed-off-by: Mike Christie Reviewed-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/cfq-iosched.c | 49 +++++++++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 20 deletions(-) (limited to 'block') diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 3fcc5986c01d..3dafdbad8f9c 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -667,9 +667,10 @@ static inline void cfqg_put(struct cfq_group *cfqg) } while (0) static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg, - struct cfq_group *curr_cfqg, int rw) + struct cfq_group *curr_cfqg, int op, + int op_flags) { - blkg_rwstat_add(&cfqg->stats.queued, rw, 1); + blkg_rwstat_add(&cfqg->stats.queued, op, op_flags, 1); cfqg_stats_end_empty_time(&cfqg->stats); cfqg_stats_set_start_group_wait_time(cfqg, curr_cfqg); } @@ -683,26 +684,30 @@ static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg, #endif } -static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int rw) +static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int op, + int op_flags) { - blkg_rwstat_add(&cfqg->stats.queued, rw, -1); + blkg_rwstat_add(&cfqg->stats.queued, op, op_flags, -1); } -static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int rw) +static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int op, + int op_flags) { - blkg_rwstat_add(&cfqg->stats.merged, rw, 1); + blkg_rwstat_add(&cfqg->stats.merged, op, op_flags, 1); } static inline void cfqg_stats_update_completion(struct cfq_group *cfqg, - uint64_t start_time, uint64_t io_start_time, int rw) + uint64_t start_time, uint64_t io_start_time, int op, + int op_flags) { struct cfqg_stats *stats = &cfqg->stats; unsigned long long now = sched_clock(); if (time_after64(now, io_start_time)) - blkg_rwstat_add(&stats->service_time, rw, now - io_start_time); + blkg_rwstat_add(&stats->service_time, op, op_flags, + now - io_start_time); if (time_after64(io_start_time, start_time)) - blkg_rwstat_add(&stats->wait_time, rw, + blkg_rwstat_add(&stats->wait_time, op, op_flags, io_start_time - start_time); } @@ -781,13 +786,16 @@ static inline void cfqg_put(struct cfq_group *cfqg) { } #define cfq_log_cfqg(cfqd, cfqg, fmt, args...) do {} while (0) static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg, - struct cfq_group *curr_cfqg, int rw) { } + struct cfq_group *curr_cfqg, int op, int op_flags) { } static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg, unsigned long time, unsigned long unaccounted_time) { } -static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int rw) { } -static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int rw) { } +static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int op, + int op_flags) { } +static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int op, + int op_flags) { } static inline void cfqg_stats_update_completion(struct cfq_group *cfqg, - uint64_t start_time, uint64_t io_start_time, int rw) { } + uint64_t start_time, uint64_t io_start_time, int op, + int op_flags) { } #endif /* CONFIG_CFQ_GROUP_IOSCHED */ @@ -2461,10 +2469,10 @@ static void cfq_reposition_rq_rb(struct cfq_queue *cfqq, struct request *rq) { elv_rb_del(&cfqq->sort_list, rq); cfqq->queued[rq_is_sync(rq)]--; - cfqg_stats_update_io_remove(RQ_CFQG(rq), rq->cmd_flags); + cfqg_stats_update_io_remove(RQ_CFQG(rq), req_op(rq), rq->cmd_flags); cfq_add_rq_rb(rq); cfqg_stats_update_io_add(RQ_CFQG(rq), cfqq->cfqd->serving_group, - rq->cmd_flags); + req_op(rq), rq->cmd_flags); } static struct request * @@ -2517,7 +2525,7 @@ static void cfq_remove_request(struct request *rq) cfq_del_rq_rb(rq); cfqq->cfqd->rq_queued--; - cfqg_stats_update_io_remove(RQ_CFQG(rq), rq->cmd_flags); + cfqg_stats_update_io_remove(RQ_CFQG(rq), req_op(rq), rq->cmd_flags); if (rq->cmd_flags & REQ_PRIO) { WARN_ON(!cfqq->prio_pending); cfqq->prio_pending--; @@ -2552,7 +2560,7 @@ static void cfq_merged_request(struct request_queue *q, struct request *req, static void cfq_bio_merged(struct request_queue *q, struct request *req, struct bio *bio) { - cfqg_stats_update_io_merged(RQ_CFQG(req), bio->bi_rw); + cfqg_stats_update_io_merged(RQ_CFQG(req), bio_op(bio), bio->bi_rw); } static void @@ -2575,7 +2583,7 @@ cfq_merged_requests(struct request_queue *q, struct request *rq, if (cfqq->next_rq == next) cfqq->next_rq = rq; cfq_remove_request(next); - cfqg_stats_update_io_merged(RQ_CFQG(rq), next->cmd_flags); + cfqg_stats_update_io_merged(RQ_CFQG(rq), req_op(next), next->cmd_flags); cfqq = RQ_CFQQ(next); /* @@ -4108,7 +4116,7 @@ static void cfq_insert_request(struct request_queue *q, struct request *rq) rq->fifo_time = jiffies + cfqd->cfq_fifo_expire[rq_is_sync(rq)]; list_add_tail(&rq->queuelist, &cfqq->fifo); cfq_add_rq_rb(rq); - cfqg_stats_update_io_add(RQ_CFQG(rq), cfqd->serving_group, + cfqg_stats_update_io_add(RQ_CFQG(rq), cfqd->serving_group, req_op(rq), rq->cmd_flags); cfq_rq_enqueued(cfqd, cfqq, rq); } @@ -4206,7 +4214,8 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq) cfqq->dispatched--; (RQ_CFQG(rq))->dispatched--; cfqg_stats_update_completion(cfqq->cfqg, rq_start_time_ns(rq), - rq_io_start_time_ns(rq), rq->cmd_flags); + rq_io_start_time_ns(rq), req_op(rq), + rq->cmd_flags); cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]--; -- cgit From 8fe0d473f5477e9916d3ac581a226acfe83142be Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sun, 5 Jun 2016 14:32:15 -0500 Subject: block: convert merge/insert code to check for REQ_OPs. This patch converts the block layer merging code to use separate variables for the operation and flags, and to check req_op for the REQ_OP. Signed-off-by: Mike Christie Reviewed-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/blk-core.c | 2 +- block/blk-merge.c | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index 090e55d7cad7..1333bb764b28 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -2161,7 +2161,7 @@ EXPORT_SYMBOL(submit_bio); static int blk_cloned_rq_check_limits(struct request_queue *q, struct request *rq) { - if (blk_rq_sectors(rq) > blk_queue_get_max_sectors(q, rq->cmd_flags)) { + if (blk_rq_sectors(rq) > blk_queue_get_max_sectors(q, req_op(rq))) { printk(KERN_ERR "%s: over max size limit.\n", __func__); return -EIO; } diff --git a/block/blk-merge.c b/block/blk-merge.c index 5a03f967557a..c265348b75d1 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -649,7 +649,8 @@ static int attempt_merge(struct request_queue *q, struct request *req, if (!rq_mergeable(req) || !rq_mergeable(next)) return 0; - if (!blk_check_merge_flags(req->cmd_flags, next->cmd_flags)) + if (!blk_check_merge_flags(req->cmd_flags, req_op(req), next->cmd_flags, + req_op(next))) return 0; /* @@ -663,7 +664,7 @@ static int attempt_merge(struct request_queue *q, struct request *req, || req_no_special_merge(next)) return 0; - if (req->cmd_flags & REQ_WRITE_SAME && + if (req_op(req) == REQ_OP_WRITE_SAME && !blk_write_same_mergeable(req->bio, next->bio)) return 0; @@ -751,7 +752,8 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio) if (!rq_mergeable(rq) || !bio_mergeable(bio)) return false; - if (!blk_check_merge_flags(rq->cmd_flags, bio->bi_rw)) + if (!blk_check_merge_flags(rq->cmd_flags, req_op(rq), bio->bi_rw, + bio_op(bio))) return false; /* different data direction or already started, don't merge */ @@ -767,7 +769,7 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio) return false; /* must be using the same buffer */ - if (rq->cmd_flags & REQ_WRITE_SAME && + if (req_op(rq) == REQ_OP_WRITE_SAME && !blk_write_same_mergeable(rq->bio, bio)) return false; -- cgit From d9d8c5c489f4969667a05727e9c2c4f78cffef1a Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sun, 5 Jun 2016 14:32:16 -0500 Subject: block: convert is_sync helpers to use REQ_OPs. This patch converts the is_sync helpers to use separate variables for the operation and flags. Signed-off-by: Mike Christie Reviewed-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/blk-core.c | 6 +++--- block/blk-mq.c | 8 ++++---- block/cfq-iosched.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index 1333bb764b28..f9f422837cea 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -962,7 +962,7 @@ static void __freed_request(struct request_list *rl, int sync) static void freed_request(struct request_list *rl, int op, unsigned int flags) { struct request_queue *q = rl->q; - int sync = rw_is_sync(op | flags); + int sync = rw_is_sync(op, flags); q->nr_rqs[sync]--; rl->count[sync]--; @@ -1075,7 +1075,7 @@ static struct request *__get_request(struct request_list *rl, int op, struct elevator_type *et = q->elevator->type; struct io_context *ioc = rq_ioc(bio); struct io_cq *icq = NULL; - const bool is_sync = rw_is_sync(op | op_flags) != 0; + const bool is_sync = rw_is_sync(op, op_flags) != 0; int may_queue; if (unlikely(blk_queue_dying(q))) @@ -1244,7 +1244,7 @@ static struct request *get_request(struct request_queue *q, int op, int op_flags, struct bio *bio, gfp_t gfp_mask) { - const bool is_sync = rw_is_sync(op | op_flags) != 0; + const bool is_sync = rw_is_sync(op, op_flags) != 0; DEFINE_WAIT(wait); struct request_list *rl; struct request *rq; diff --git a/block/blk-mq.c b/block/blk-mq.c index 3393f29faa9e..29bcd9c07a34 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -204,7 +204,7 @@ static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx, rq->end_io_data = NULL; rq->next_rq = NULL; - ctx->rq_dispatched[rw_is_sync(op | op_flags)]++; + ctx->rq_dispatched[rw_is_sync(op, op_flags)]++; } static struct request * @@ -1178,7 +1178,7 @@ static struct request *blk_mq_map_request(struct request_queue *q, ctx = blk_mq_get_ctx(q); hctx = q->mq_ops->map_queue(q, ctx->cpu); - if (rw_is_sync(bio->bi_rw)) + if (rw_is_sync(bio_op(bio), bio->bi_rw)) op_flags |= REQ_SYNC; trace_block_getrq(q, bio, op); @@ -1246,7 +1246,7 @@ static int blk_mq_direct_issue_request(struct request *rq, blk_qc_t *cookie) */ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) { - const int is_sync = rw_is_sync(bio->bi_rw); + const int is_sync = rw_is_sync(bio_op(bio), bio->bi_rw); const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA); struct blk_map_ctx data; struct request *rq; @@ -1343,7 +1343,7 @@ done: */ static blk_qc_t blk_sq_make_request(struct request_queue *q, struct bio *bio) { - const int is_sync = rw_is_sync(bio->bi_rw); + const int is_sync = rw_is_sync(bio_op(bio), bio->bi_rw); const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA); struct blk_plug *plug; unsigned int request_count = 0; diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 3dafdbad8f9c..b1154861f4c9 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -4311,7 +4311,7 @@ static int cfq_may_queue(struct request_queue *q, int op, int op_flags) if (!cic) return ELV_MQUEUE_MAY; - cfqq = cic_to_cfqq(cic, rw_is_sync(op | op_flags)); + cfqq = cic_to_cfqq(cic, rw_is_sync(op, op_flags)); if (cfqq) { cfq_init_prio_data(cfqq, cic); -- cgit From 6296b9604fcebc2dd8d6ec396de80b2da84d9700 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sun, 5 Jun 2016 14:32:21 -0500 Subject: block, drivers, fs: shrink bi_rw from long to int We don't need bi_rw to be so large on 64 bit archs, so reduce it to unsigned int. Signed-off-by: Mike Christie Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/blk-core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index f9f422837cea..c7d66c23a708 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -1853,7 +1853,7 @@ static void handle_bad_sector(struct bio *bio) char b[BDEVNAME_SIZE]; printk(KERN_INFO "attempt to access beyond end of device\n"); - printk(KERN_INFO "%s: rw=%ld, want=%Lu, limit=%Lu\n", + printk(KERN_INFO "%s: rw=%d, want=%Lu, limit=%Lu\n", bdevname(bio->bi_bdev, b), bio->bi_rw, (unsigned long long)bio_end_sector(bio), -- cgit From 3a5e02ced11e22ecd9da3d6710afe15bcfee1d10 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sun, 5 Jun 2016 14:32:23 -0500 Subject: block, drivers: add REQ_OP_FLUSH operation This adds a REQ_OP_FLUSH operation that is sent to request_fn based drivers by the block layer's flush code, instead of sending requests with the request->cmd_flags REQ_FLUSH bit set. Signed-off-by: Mike Christie Reviewed-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/blk-flush.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'block') diff --git a/block/blk-flush.c b/block/blk-flush.c index 9fd1f63a6348..21f0d5b0d2ca 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -29,7 +29,7 @@ * The actual execution of flush is double buffered. Whenever a request * needs to execute PRE or POSTFLUSH, it queues at * fq->flush_queue[fq->flush_pending_idx]. Once certain criteria are met, a - * flush is issued and the pending_idx is toggled. When the flush + * REQ_OP_FLUSH is issued and the pending_idx is toggled. When the flush * completes, all the requests which were pending are proceeded to the next * step. This allows arbitrary merging of different types of FLUSH/FUA * requests. @@ -330,7 +330,7 @@ static bool blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq) } flush_rq->cmd_type = REQ_TYPE_FS; - flush_rq->cmd_flags = WRITE_FLUSH | REQ_FLUSH_SEQ; + req_set_op_attrs(flush_rq, REQ_OP_FLUSH, WRITE_FLUSH | REQ_FLUSH_SEQ); flush_rq->rq_disk = first_rq->rq_disk; flush_rq->end_io = flush_end_io; -- cgit From 28a8f0d317bf225ff15008f5dd66ae16242dd843 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Sun, 5 Jun 2016 14:32:25 -0500 Subject: block, drivers, fs: rename REQ_FLUSH to REQ_PREFLUSH To avoid confusion between REQ_OP_FLUSH, which is handled by request_fn drivers, and upper layers requesting the block layer perform a flush sequence along with possibly a WRITE, this patch renames REQ_FLUSH to REQ_PREFLUSH. Signed-off-by: Mike Christie Reviewed-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/blk-core.c | 12 ++++++------ block/blk-flush.c | 16 ++++++++-------- block/blk-mq.c | 4 ++-- 3 files changed, 16 insertions(+), 16 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index c7d66c23a708..32a283eb7274 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -1029,7 +1029,7 @@ static bool blk_rq_should_init_elevator(struct bio *bio) * Flush requests do not use the elevator so skip initialization. * This allows a request to share the flush and elevator data. */ - if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) + if (bio->bi_rw & (REQ_PREFLUSH | REQ_FUA)) return false; return true; @@ -1736,7 +1736,7 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio) return BLK_QC_T_NONE; } - if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) { + if (bio->bi_rw & (REQ_PREFLUSH | REQ_FUA)) { spin_lock_irq(q->queue_lock); where = ELEVATOR_INSERT_FLUSH; goto get_rq; @@ -1968,9 +1968,9 @@ generic_make_request_checks(struct bio *bio) * drivers without flush support don't have to worry * about them. */ - if ((bio->bi_rw & (REQ_FLUSH | REQ_FUA)) && + if ((bio->bi_rw & (REQ_PREFLUSH | REQ_FUA)) && !test_bit(QUEUE_FLAG_WC, &q->queue_flags)) { - bio->bi_rw &= ~(REQ_FLUSH | REQ_FUA); + bio->bi_rw &= ~(REQ_PREFLUSH | REQ_FUA); if (!nr_sectors) { err = 0; goto end_io; @@ -2217,7 +2217,7 @@ int blk_insert_cloned_request(struct request_queue *q, struct request *rq) */ BUG_ON(blk_queued_rq(rq)); - if (rq->cmd_flags & (REQ_FLUSH|REQ_FUA)) + if (rq->cmd_flags & (REQ_PREFLUSH | REQ_FUA)) where = ELEVATOR_INSERT_FLUSH; add_acct_request(q, rq, where); @@ -3311,7 +3311,7 @@ void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule) /* * rq is already accounted, so use raw insert */ - if (rq->cmd_flags & (REQ_FLUSH | REQ_FUA)) + if (rq->cmd_flags & (REQ_PREFLUSH | REQ_FUA)) __elv_add_request(q, rq, ELEVATOR_INSERT_FLUSH); else __elv_add_request(q, rq, ELEVATOR_INSERT_SORT_MERGE); diff --git a/block/blk-flush.c b/block/blk-flush.c index 21f0d5b0d2ca..d308def812db 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -10,8 +10,8 @@ * optional steps - PREFLUSH, DATA and POSTFLUSH - according to the request * properties and hardware capability. * - * If a request doesn't have data, only REQ_FLUSH makes sense, which - * indicates a simple flush request. If there is data, REQ_FLUSH indicates + * If a request doesn't have data, only REQ_PREFLUSH makes sense, which + * indicates a simple flush request. If there is data, REQ_PREFLUSH indicates * that the device cache should be flushed before the data is executed, and * REQ_FUA means that the data must be on non-volatile media on request * completion. @@ -20,11 +20,11 @@ * difference. The requests are either completed immediately if there's no * data or executed as normal requests otherwise. * - * If the device has writeback cache and supports FUA, REQ_FLUSH is + * If the device has writeback cache and supports FUA, REQ_PREFLUSH is * translated to PREFLUSH but REQ_FUA is passed down directly with DATA. * - * If the device has writeback cache and doesn't support FUA, REQ_FLUSH is - * translated to PREFLUSH and REQ_FUA to POSTFLUSH. + * If the device has writeback cache and doesn't support FUA, REQ_PREFLUSH + * is translated to PREFLUSH and REQ_FUA to POSTFLUSH. * * The actual execution of flush is double buffered. Whenever a request * needs to execute PRE or POSTFLUSH, it queues at @@ -103,7 +103,7 @@ static unsigned int blk_flush_policy(unsigned long fflags, struct request *rq) policy |= REQ_FSEQ_DATA; if (fflags & (1UL << QUEUE_FLAG_WC)) { - if (rq->cmd_flags & REQ_FLUSH) + if (rq->cmd_flags & REQ_PREFLUSH) policy |= REQ_FSEQ_PREFLUSH; if (!(fflags & (1UL << QUEUE_FLAG_FUA)) && (rq->cmd_flags & REQ_FUA)) @@ -391,9 +391,9 @@ void blk_insert_flush(struct request *rq) /* * @policy now records what operations need to be done. Adjust - * REQ_FLUSH and FUA for the driver. + * REQ_PREFLUSH and FUA for the driver. */ - rq->cmd_flags &= ~REQ_FLUSH; + rq->cmd_flags &= ~REQ_PREFLUSH; if (!(fflags & (1UL << QUEUE_FLAG_FUA))) rq->cmd_flags &= ~REQ_FUA; diff --git a/block/blk-mq.c b/block/blk-mq.c index 29bcd9c07a34..13f460368759 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1247,7 +1247,7 @@ static int blk_mq_direct_issue_request(struct request *rq, blk_qc_t *cookie) static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) { const int is_sync = rw_is_sync(bio_op(bio), bio->bi_rw); - const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA); + const int is_flush_fua = bio->bi_rw & (REQ_PREFLUSH | REQ_FUA); struct blk_map_ctx data; struct request *rq; unsigned int request_count = 0; @@ -1344,7 +1344,7 @@ done: static blk_qc_t blk_sq_make_request(struct request_queue *q, struct bio *bio) { const int is_sync = rw_is_sync(bio_op(bio), bio->bi_rw); - const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA); + const int is_flush_fua = bio->bi_rw & (REQ_PREFLUSH | REQ_FUA); struct blk_plug *plug; unsigned int request_count = 0; struct blk_map_ctx data; -- cgit From 9a7f38c42c2b92391d9dabaf9f51df7cfe5608e4 Mon Sep 17 00:00:00 2001 From: Jeff Moyer Date: Wed, 8 Jun 2016 08:55:34 -0600 Subject: cfq-iosched: Convert from jiffies to nanoseconds Convert all time-keeping in CFQ IO scheduler from jiffies to nanoseconds so that we can later make the intervals more fine-grained than jiffies. One jiffie is several miliseconds and even for today's rotating disks that is a noticeable amount of time and thus we leave disk unnecessarily idle. Signed-off-by: Jeff Moyer Signed-off-by: Jan Kara Signed-off-by: Jens Axboe --- block/cfq-iosched.c | 273 ++++++++++++++++++++++++++-------------------------- 1 file changed, 136 insertions(+), 137 deletions(-) (limited to 'block') diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index b1154861f4c9..9c2e82c1ea88 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include @@ -22,28 +22,28 @@ */ /* max queue in one round of service */ static const int cfq_quantum = 8; -static const int cfq_fifo_expire[2] = { HZ / 4, HZ / 8 }; +static const u64 cfq_fifo_expire[2] = { NSEC_PER_SEC / 4, NSEC_PER_SEC / 8 }; /* maximum backwards seek, in KiB */ static const int cfq_back_max = 16 * 1024; /* penalty of a backwards seek */ static const int cfq_back_penalty = 2; -static const int cfq_slice_sync = HZ / 10; -static int cfq_slice_async = HZ / 25; +static const u64 cfq_slice_sync = NSEC_PER_SEC / 10; +static u64 cfq_slice_async = NSEC_PER_SEC / 25; static const int cfq_slice_async_rq = 2; -static int cfq_slice_idle = HZ / 125; -static int cfq_group_idle = HZ / 125; -static const int cfq_target_latency = HZ * 3/10; /* 300 ms */ +static u64 cfq_slice_idle = NSEC_PER_SEC / 125; +static u64 cfq_group_idle = NSEC_PER_SEC / 125; +static const u64 cfq_target_latency = (u64)NSEC_PER_SEC * 3/10; /* 300 ms */ static const int cfq_hist_divisor = 4; /* * offset from end of service tree */ -#define CFQ_IDLE_DELAY (HZ / 5) +#define CFQ_IDLE_DELAY (NSEC_PER_SEC / 5) /* * below this threshold, we consider thinktime immediate */ -#define CFQ_MIN_TT (2) +#define CFQ_MIN_TT (2 * NSEC_PER_SEC / HZ) #define CFQ_SLICE_SCALE (5) #define CFQ_HW_QUEUE_MIN (5) @@ -73,11 +73,11 @@ static struct kmem_cache *cfq_pool; #define CFQ_WEIGHT_LEGACY_MAX 1000 struct cfq_ttime { - unsigned long last_end_request; + u64 last_end_request; - unsigned long ttime_total; + u64 ttime_total; + u64 ttime_mean; unsigned long ttime_samples; - unsigned long ttime_mean; }; /* @@ -94,7 +94,7 @@ struct cfq_rb_root { struct cfq_ttime ttime; }; #define CFQ_RB_ROOT (struct cfq_rb_root) { .rb = RB_ROOT, \ - .ttime = {.last_end_request = jiffies,},} + .ttime = {.last_end_request = ktime_get_ns(),},} /* * Per process-grouping structure @@ -109,7 +109,7 @@ struct cfq_queue { /* service_tree member */ struct rb_node rb_node; /* service_tree key */ - unsigned long rb_key; + u64 rb_key; /* prio tree member */ struct rb_node p_node; /* prio tree root we belong to, if any */ @@ -126,13 +126,13 @@ struct cfq_queue { struct list_head fifo; /* time when queue got scheduled in to dispatch first request. */ - unsigned long dispatch_start; - unsigned int allocated_slice; - unsigned int slice_dispatch; + u64 dispatch_start; + u64 allocated_slice; + u64 slice_dispatch; /* time when first request from queue completed and slice started. */ - unsigned long slice_start; - unsigned long slice_end; - long slice_resid; + u64 slice_start; + u64 slice_end; + u64 slice_resid; /* pending priority requests */ int prio_pending; @@ -290,7 +290,7 @@ struct cfq_group { struct cfq_rb_root service_trees[2][3]; struct cfq_rb_root service_tree_idle; - unsigned long saved_wl_slice; + u64 saved_wl_slice; enum wl_type_t saved_wl_type; enum wl_class_t saved_wl_class; @@ -329,7 +329,7 @@ struct cfq_data { */ enum wl_class_t serving_wl_class; enum wl_type_t serving_wl_type; - unsigned long workload_expires; + u64 workload_expires; struct cfq_group *serving_group; /* @@ -374,22 +374,22 @@ struct cfq_data { * tunables, see top of file */ unsigned int cfq_quantum; - unsigned int cfq_fifo_expire[2]; unsigned int cfq_back_penalty; unsigned int cfq_back_max; - unsigned int cfq_slice[2]; unsigned int cfq_slice_async_rq; - unsigned int cfq_slice_idle; - unsigned int cfq_group_idle; unsigned int cfq_latency; - unsigned int cfq_target_latency; + u64 cfq_fifo_expire[2]; + u64 cfq_slice[2]; + u64 cfq_slice_idle; + u64 cfq_group_idle; + u64 cfq_target_latency; /* * Fallback dummy cfqq for extreme OOM conditions */ struct cfq_queue oom_cfqq; - unsigned long last_delayed_sync; + u64 last_delayed_sync; }; static struct cfq_group *cfq_get_next_cfqg(struct cfq_data *cfqd); @@ -676,7 +676,7 @@ static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg, } static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg, - unsigned long time, unsigned long unaccounted_time) + uint64_t time, unsigned long unaccounted_time) { blkg_stat_add(&cfqg->stats.time, time); #ifdef CONFIG_DEBUG_BLK_CGROUP @@ -788,7 +788,7 @@ static inline void cfqg_put(struct cfq_group *cfqg) { } static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg, struct cfq_group *curr_cfqg, int op, int op_flags) { } static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg, - unsigned long time, unsigned long unaccounted_time) { } + uint64_t time, unsigned long unaccounted_time) { } static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int op, int op_flags) { } static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int op, @@ -815,7 +815,7 @@ static inline void cfqg_stats_update_completion(struct cfq_group *cfqg, static inline bool cfq_io_thinktime_big(struct cfq_data *cfqd, struct cfq_ttime *ttime, bool group_idle) { - unsigned long slice; + u64 slice; if (!sample_valid(ttime->ttime_samples)) return false; if (group_idle) @@ -938,17 +938,18 @@ static inline void cfq_schedule_dispatch(struct cfq_data *cfqd) * if a queue is marked sync and has sync io queued. A sync queue with async * io only, should not get full sync slice length. */ -static inline int cfq_prio_slice(struct cfq_data *cfqd, bool sync, +static inline u64 cfq_prio_slice(struct cfq_data *cfqd, bool sync, unsigned short prio) { - const int base_slice = cfqd->cfq_slice[sync]; + u64 base_slice = cfqd->cfq_slice[sync]; + u64 slice = div_u64(base_slice, CFQ_SLICE_SCALE); WARN_ON(prio >= IOPRIO_BE_NR); - return base_slice + (base_slice/CFQ_SLICE_SCALE * (4 - prio)); + return base_slice + (slice * (4 - prio)); } -static inline int +static inline u64 cfq_prio_to_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) { return cfq_prio_slice(cfqd, cfq_cfqq_sync(cfqq), cfqq->ioprio); @@ -966,15 +967,14 @@ cfq_prio_to_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) * * The result is also in fixed point w/ CFQ_SERVICE_SHIFT. */ -static inline u64 cfqg_scale_charge(unsigned long charge, +static inline u64 cfqg_scale_charge(u64 charge, unsigned int vfraction) { u64 c = charge << CFQ_SERVICE_SHIFT; /* make it fixed point */ /* charge / vfraction */ c <<= CFQ_SERVICE_SHIFT; - do_div(c, vfraction); - return c; + return div_u64(c, vfraction); } static inline u64 max_vdisktime(u64 min_vdisktime, u64 vdisktime) @@ -1027,16 +1027,16 @@ static inline unsigned cfq_group_get_avg_queues(struct cfq_data *cfqd, return cfqg->busy_queues_avg[rt]; } -static inline unsigned +static inline u64 cfq_group_slice(struct cfq_data *cfqd, struct cfq_group *cfqg) { return cfqd->cfq_target_latency * cfqg->vfraction >> CFQ_SERVICE_SHIFT; } -static inline unsigned +static inline u64 cfq_scaled_cfqq_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) { - unsigned slice = cfq_prio_to_slice(cfqd, cfqq); + u64 slice = cfq_prio_to_slice(cfqd, cfqq); if (cfqd->cfq_latency) { /* * interested queues (we consider only the ones with the same @@ -1044,20 +1044,22 @@ cfq_scaled_cfqq_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) */ unsigned iq = cfq_group_get_avg_queues(cfqd, cfqq->cfqg, cfq_class_rt(cfqq)); - unsigned sync_slice = cfqd->cfq_slice[1]; - unsigned expect_latency = sync_slice * iq; - unsigned group_slice = cfq_group_slice(cfqd, cfqq->cfqg); + u64 sync_slice = cfqd->cfq_slice[1]; + u64 expect_latency = sync_slice * iq; + u64 group_slice = cfq_group_slice(cfqd, cfqq->cfqg); if (expect_latency > group_slice) { - unsigned base_low_slice = 2 * cfqd->cfq_slice_idle; + u64 base_low_slice = 2 * cfqd->cfq_slice_idle; + u64 low_slice; + /* scale low_slice according to IO priority * and sync vs async */ - unsigned low_slice = - min(slice, base_low_slice * slice / sync_slice); + low_slice = div64_u64(base_low_slice*slice, sync_slice); + low_slice = min(slice, low_slice); /* the adapted slice value is scaled to fit all iqs * into the target latency */ - slice = max(slice * group_slice / expect_latency, - low_slice); + slice = div64_u64(slice*group_slice, expect_latency); + slice = max(slice, low_slice); } } return slice; @@ -1066,12 +1068,13 @@ cfq_scaled_cfqq_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) static inline void cfq_set_prio_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) { - unsigned slice = cfq_scaled_cfqq_slice(cfqd, cfqq); + u64 slice = cfq_scaled_cfqq_slice(cfqd, cfqq); + u64 now = ktime_get_ns(); - cfqq->slice_start = jiffies; - cfqq->slice_end = jiffies + slice; + cfqq->slice_start = now; + cfqq->slice_end = now + slice; cfqq->allocated_slice = slice; - cfq_log_cfqq(cfqd, cfqq, "set_slice=%lu", cfqq->slice_end - jiffies); + cfq_log_cfqq(cfqd, cfqq, "set_slice=%llu", cfqq->slice_end - now); } /* @@ -1083,7 +1086,7 @@ static inline bool cfq_slice_used(struct cfq_queue *cfqq) { if (cfq_cfqq_slice_new(cfqq)) return false; - if (time_before(jiffies, cfqq->slice_end)) + if (ktime_get_ns() < cfqq->slice_end) return false; return true; @@ -1249,8 +1252,8 @@ cfq_find_next_rq(struct cfq_data *cfqd, struct cfq_queue *cfqq, return cfq_choose_req(cfqd, next, prev, blk_rq_pos(last)); } -static unsigned long cfq_slice_offset(struct cfq_data *cfqd, - struct cfq_queue *cfqq) +static u64 cfq_slice_offset(struct cfq_data *cfqd, + struct cfq_queue *cfqq) { /* * just an approximation, should be ok. @@ -1443,31 +1446,31 @@ cfq_group_notify_queue_del(struct cfq_data *cfqd, struct cfq_group *cfqg) cfqg_stats_update_dequeue(cfqg); } -static inline unsigned int cfq_cfqq_slice_usage(struct cfq_queue *cfqq, - unsigned int *unaccounted_time) +static inline u64 cfq_cfqq_slice_usage(struct cfq_queue *cfqq, + u64 *unaccounted_time) { - unsigned int slice_used; + u64 slice_used; + u64 now = ktime_get_ns(); /* * Queue got expired before even a single request completed or * got expired immediately after first request completion. */ - if (!cfqq->slice_start || cfqq->slice_start == jiffies) { + if (!cfqq->slice_start || cfqq->slice_start == now) { /* * Also charge the seek time incurred to the group, otherwise * if there are mutiple queues in the group, each can dispatch * a single request on seeky media and cause lots of seek time * and group will never know it. */ - slice_used = max_t(unsigned, (jiffies - cfqq->dispatch_start), - 1); + slice_used = max_t(u64, (now - cfqq->dispatch_start), 1); } else { - slice_used = jiffies - cfqq->slice_start; + slice_used = now - cfqq->slice_start; if (slice_used > cfqq->allocated_slice) { *unaccounted_time = slice_used - cfqq->allocated_slice; slice_used = cfqq->allocated_slice; } - if (time_after(cfqq->slice_start, cfqq->dispatch_start)) + if (cfqq->slice_start > cfqq->dispatch_start) *unaccounted_time += cfqq->slice_start - cfqq->dispatch_start; } @@ -1479,10 +1482,11 @@ static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg, struct cfq_queue *cfqq) { struct cfq_rb_root *st = &cfqd->grp_service_tree; - unsigned int used_sl, charge, unaccounted_sl = 0; + u64 used_sl, charge, unaccounted_sl = 0; int nr_sync = cfqg->nr_cfqq - cfqg_busy_async_queues(cfqd, cfqg) - cfqg->service_tree_idle.count; unsigned int vfr; + u64 now = ktime_get_ns(); BUG_ON(nr_sync < 0); used_sl = charge = cfq_cfqq_slice_usage(cfqq, &unaccounted_sl); @@ -1504,9 +1508,8 @@ static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg, cfq_group_service_tree_add(st, cfqg); /* This group is being expired. Save the context */ - if (time_after(cfqd->workload_expires, jiffies)) { - cfqg->saved_wl_slice = cfqd->workload_expires - - jiffies; + if (cfqd->workload_expires > now) { + cfqg->saved_wl_slice = cfqd->workload_expires - now; cfqg->saved_wl_type = cfqd->serving_wl_type; cfqg->saved_wl_class = cfqd->serving_wl_class; } else @@ -1515,7 +1518,7 @@ static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg, cfq_log_cfqg(cfqd, cfqg, "served: vt=%llu min_vt=%llu", cfqg->vdisktime, st->min_vdisktime); cfq_log_cfqq(cfqq->cfqd, cfqq, - "sl_used=%u disp=%u charge=%u iops=%u sect=%lu", + "sl_used=%llu disp=%llu charge=%llu iops=%u sect=%lu", used_sl, cfqq->slice_dispatch, charge, iops_mode(cfqd), cfqq->nr_sectors); cfqg_stats_update_timeslice_used(cfqg, used_sl, unaccounted_sl); @@ -1538,7 +1541,7 @@ static void cfq_init_cfqg_base(struct cfq_group *cfqg) *st = CFQ_RB_ROOT; RB_CLEAR_NODE(&cfqg->rb_node); - cfqg->ttime.last_end_request = jiffies; + cfqg->ttime.last_end_request = ktime_get_ns(); } #ifdef CONFIG_CFQ_GROUP_IOSCHED @@ -2221,10 +2224,11 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq, { struct rb_node **p, *parent; struct cfq_queue *__cfqq; - unsigned long rb_key; + u64 rb_key; struct cfq_rb_root *st; int left; int new_cfqq = 1; + u64 now = ktime_get_ns(); st = st_for(cfqq->cfqg, cfqq_class(cfqq), cfqq_type(cfqq)); if (cfq_class_idle(cfqq)) { @@ -2234,7 +2238,7 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq, __cfqq = rb_entry(parent, struct cfq_queue, rb_node); rb_key += __cfqq->rb_key; } else - rb_key += jiffies; + rb_key += now; } else if (!add_front) { /* * Get our rb key offset. Subtract any residual slice @@ -2242,13 +2246,13 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq, * count indicates slice overrun, and this should position * the next service time further away in the tree. */ - rb_key = cfq_slice_offset(cfqd, cfqq) + jiffies; + rb_key = cfq_slice_offset(cfqd, cfqq) + now; rb_key -= cfqq->slice_resid; cfqq->slice_resid = 0; } else { - rb_key = -HZ; + rb_key = -NSEC_PER_SEC; __cfqq = cfq_rb_first(st); - rb_key += __cfqq ? __cfqq->rb_key : jiffies; + rb_key += __cfqq ? __cfqq->rb_key : now; } if (!RB_EMPTY_NODE(&cfqq->rb_node)) { @@ -2274,7 +2278,7 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq, /* * sort by key, that represents service time. */ - if (time_before(rb_key, __cfqq->rb_key)) + if (rb_key < __cfqq->rb_key) p = &parent->rb_left; else { p = &parent->rb_right; @@ -2574,7 +2578,7 @@ cfq_merged_requests(struct request_queue *q, struct request *rq, * reposition in fifo if next is older than rq */ if (!list_empty(&rq->queuelist) && !list_empty(&next->queuelist) && - time_before(next->fifo_time, rq->fifo_time) && + next->fifo_time < rq->fifo_time && cfqq == RQ_CFQQ(next)) { list_move(&rq->queuelist, &next->queuelist); rq->fifo_time = next->fifo_time; @@ -2635,7 +2639,7 @@ static void __cfq_set_active_queue(struct cfq_data *cfqd, cfqd->serving_wl_class, cfqd->serving_wl_type); cfqg_stats_update_avg_queue_size(cfqq->cfqg); cfqq->slice_start = 0; - cfqq->dispatch_start = jiffies; + cfqq->dispatch_start = ktime_get_ns(); cfqq->allocated_slice = 0; cfqq->slice_end = 0; cfqq->slice_dispatch = 0; @@ -2684,8 +2688,8 @@ __cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq, if (cfq_cfqq_slice_new(cfqq)) cfqq->slice_resid = cfq_scaled_cfqq_slice(cfqd, cfqq); else - cfqq->slice_resid = cfqq->slice_end - jiffies; - cfq_log_cfqq(cfqd, cfqq, "resid=%ld", cfqq->slice_resid); + cfqq->slice_resid = cfqq->slice_end - ktime_get_ns(); + cfq_log_cfqq(cfqd, cfqq, "resid=%llu", cfqq->slice_resid); } cfq_group_served(cfqd, cfqq->cfqg, cfqq); @@ -2919,7 +2923,8 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd) struct cfq_queue *cfqq = cfqd->active_queue; struct cfq_rb_root *st = cfqq->service_tree; struct cfq_io_cq *cic; - unsigned long sl, group_idle = 0; + u64 sl, group_idle = 0; + u64 now = ktime_get_ns(); /* * SSD device without seek penalty, disable idling. But only do so @@ -2962,8 +2967,8 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd) * time slice. */ if (sample_valid(cic->ttime.ttime_samples) && - (cfqq->slice_end - jiffies < cic->ttime.ttime_mean)) { - cfq_log_cfqq(cfqd, cfqq, "Not idling. think_time:%lu", + (cfqq->slice_end - now < cic->ttime.ttime_mean)) { + cfq_log_cfqq(cfqd, cfqq, "Not idling. think_time:%llu", cic->ttime.ttime_mean); return; } @@ -2984,9 +2989,9 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd) else sl = cfqd->cfq_slice_idle; - mod_timer(&cfqd->idle_slice_timer, jiffies + sl); + mod_timer(&cfqd->idle_slice_timer, now + sl); cfqg_stats_set_start_idle_time(cfqq->cfqg); - cfq_log_cfqq(cfqd, cfqq, "arm_idle: %lu group_idle: %d", sl, + cfq_log_cfqq(cfqd, cfqq, "arm_idle: %llu group_idle: %d", sl, group_idle ? 1 : 0); } @@ -3026,7 +3031,7 @@ static struct request *cfq_check_fifo(struct cfq_queue *cfqq) return NULL; rq = rq_entry_fifo(cfqq->fifo.next); - if (time_before(jiffies, rq->fifo_time)) + if (ktime_get_ns() < rq->fifo_time) rq = NULL; cfq_log_cfqq(cfqq->cfqd, cfqq, "fifo=%p", rq); @@ -3104,14 +3109,14 @@ static enum wl_type_t cfq_choose_wl_type(struct cfq_data *cfqd, struct cfq_queue *queue; int i; bool key_valid = false; - unsigned long lowest_key = 0; + u64 lowest_key = 0; enum wl_type_t cur_best = SYNC_NOIDLE_WORKLOAD; for (i = 0; i <= SYNC_WORKLOAD; ++i) { /* select the one with lowest rb_key */ queue = cfq_rb_first(st_for(cfqg, wl_class, i)); if (queue && - (!key_valid || time_before(queue->rb_key, lowest_key))) { + (!key_valid || queue->rb_key < lowest_key)) { lowest_key = queue->rb_key; cur_best = i; key_valid = true; @@ -3124,11 +3129,12 @@ static enum wl_type_t cfq_choose_wl_type(struct cfq_data *cfqd, static void choose_wl_class_and_type(struct cfq_data *cfqd, struct cfq_group *cfqg) { - unsigned slice; + u64 slice; unsigned count; struct cfq_rb_root *st; - unsigned group_slice; + u64 group_slice; enum wl_class_t original_class = cfqd->serving_wl_class; + u64 now = ktime_get_ns(); /* Choose next priority. RT > BE > IDLE */ if (cfq_group_busy_queues_wl(RT_WORKLOAD, cfqd, cfqg)) @@ -3137,7 +3143,7 @@ choose_wl_class_and_type(struct cfq_data *cfqd, struct cfq_group *cfqg) cfqd->serving_wl_class = BE_WORKLOAD; else { cfqd->serving_wl_class = IDLE_WORKLOAD; - cfqd->workload_expires = jiffies + 1; + cfqd->workload_expires = now + jiffies_to_nsecs(1); return; } @@ -3155,7 +3161,7 @@ choose_wl_class_and_type(struct cfq_data *cfqd, struct cfq_group *cfqg) /* * check workload expiration, and that we still have other queues ready */ - if (count && !time_after(jiffies, cfqd->workload_expires)) + if (count && !(now > cfqd->workload_expires)) return; new_workload: @@ -3172,13 +3178,13 @@ new_workload: */ group_slice = cfq_group_slice(cfqd, cfqg); - slice = group_slice * count / + slice = div_u64(group_slice * count, max_t(unsigned, cfqg->busy_queues_avg[cfqd->serving_wl_class], cfq_group_busy_queues_wl(cfqd->serving_wl_class, cfqd, - cfqg)); + cfqg))); if (cfqd->serving_wl_type == ASYNC_WORKLOAD) { - unsigned int tmp; + u64 tmp; /* * Async queues are currently system wide. Just taking @@ -3189,19 +3195,19 @@ new_workload: */ tmp = cfqd->cfq_target_latency * cfqg_busy_async_queues(cfqd, cfqg); - tmp = tmp/cfqd->busy_queues; - slice = min_t(unsigned, slice, tmp); + tmp = div_u64(tmp, cfqd->busy_queues); + slice = min_t(u64, slice, tmp); /* async workload slice is scaled down according to * the sync/async slice ratio. */ - slice = slice * cfqd->cfq_slice[0] / cfqd->cfq_slice[1]; + slice = div64_u64(slice*cfqd->cfq_slice[0], cfqd->cfq_slice[1]); } else /* sync workload slice is at least 2 * cfq_slice_idle */ slice = max(slice, 2 * cfqd->cfq_slice_idle); - slice = max_t(unsigned, slice, CFQ_MIN_TT); - cfq_log(cfqd, "workload slice:%d", slice); - cfqd->workload_expires = jiffies + slice; + slice = max_t(u64, slice, CFQ_MIN_TT); + cfq_log(cfqd, "workload slice:%llu", slice); + cfqd->workload_expires = now + slice; } static struct cfq_group *cfq_get_next_cfqg(struct cfq_data *cfqd) @@ -3219,16 +3225,17 @@ static struct cfq_group *cfq_get_next_cfqg(struct cfq_data *cfqd) static void cfq_choose_cfqg(struct cfq_data *cfqd) { struct cfq_group *cfqg = cfq_get_next_cfqg(cfqd); + u64 now = ktime_get_ns(); cfqd->serving_group = cfqg; /* Restore the workload type data */ if (cfqg->saved_wl_slice) { - cfqd->workload_expires = jiffies + cfqg->saved_wl_slice; + cfqd->workload_expires = now + cfqg->saved_wl_slice; cfqd->serving_wl_type = cfqg->saved_wl_type; cfqd->serving_wl_class = cfqg->saved_wl_class; } else - cfqd->workload_expires = jiffies - 1; + cfqd->workload_expires = now - 1; choose_wl_class_and_type(cfqd, cfqg); } @@ -3240,6 +3247,7 @@ static void cfq_choose_cfqg(struct cfq_data *cfqd) static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd) { struct cfq_queue *cfqq, *new_cfqq = NULL; + u64 now = ktime_get_ns(); cfqq = cfqd->active_queue; if (!cfqq) @@ -3311,7 +3319,7 @@ static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd) **/ if (CFQQ_SEEKY(cfqq) && cfq_cfqq_idle_window(cfqq) && (cfq_cfqq_slice_new(cfqq) || - (cfqq->slice_end - jiffies > jiffies - cfqq->slice_start))) { + (cfqq->slice_end - now > now - cfqq->slice_start))) { cfq_clear_cfqq_deep(cfqq); cfq_clear_cfqq_idle_window(cfqq); } @@ -3389,11 +3397,12 @@ static int cfq_forced_dispatch(struct cfq_data *cfqd) static inline bool cfq_slice_used_soon(struct cfq_data *cfqd, struct cfq_queue *cfqq) { + u64 now = ktime_get_ns(); + /* the queue hasn't finished any request, can't estimate */ if (cfq_cfqq_slice_new(cfqq)) return true; - if (time_after(jiffies + cfqd->cfq_slice_idle * cfqq->dispatched, - cfqq->slice_end)) + if (now + cfqd->cfq_slice_idle * cfqq->dispatched > cfqq->slice_end) return true; return false; @@ -3468,10 +3477,10 @@ static bool cfq_may_dispatch(struct cfq_data *cfqd, struct cfq_queue *cfqq) * based on the last sync IO we serviced */ if (!cfq_cfqq_sync(cfqq) && cfqd->cfq_latency) { - unsigned long last_sync = jiffies - cfqd->last_delayed_sync; + u64 last_sync = ktime_get_ns() - cfqd->last_delayed_sync; unsigned int depth; - depth = last_sync / cfqd->cfq_slice[1]; + depth = div64_u64(last_sync, cfqd->cfq_slice[1]); if (!depth && !cfqq->dispatched) depth = 1; if (depth < max_dispatch) @@ -3554,7 +3563,7 @@ static int cfq_dispatch_requests(struct request_queue *q, int force) if (cfqd->busy_queues > 1 && ((!cfq_cfqq_sync(cfqq) && cfqq->slice_dispatch >= cfq_prio_to_maxrq(cfqd, cfqq)) || cfq_class_idle(cfqq))) { - cfqq->slice_end = jiffies + 1; + cfqq->slice_end = ktime_get_ns() + 1; cfq_slice_expired(cfqd, 0); } @@ -3632,7 +3641,7 @@ static void cfq_init_icq(struct io_cq *icq) { struct cfq_io_cq *cic = icq_to_cic(icq); - cic->ttime.last_end_request = jiffies; + cic->ttime.last_end_request = ktime_get_ns(); } static void cfq_exit_icq(struct io_cq *icq) @@ -3853,14 +3862,15 @@ out: } static void -__cfq_update_io_thinktime(struct cfq_ttime *ttime, unsigned long slice_idle) +__cfq_update_io_thinktime(struct cfq_ttime *ttime, u64 slice_idle) { - unsigned long elapsed = jiffies - ttime->last_end_request; + u64 elapsed = ktime_get_ns() - ttime->last_end_request; elapsed = min(elapsed, 2UL * slice_idle); ttime->ttime_samples = (7*ttime->ttime_samples + 256) / 8; - ttime->ttime_total = (7*ttime->ttime_total + 256*elapsed) / 8; - ttime->ttime_mean = (ttime->ttime_total + 128) / ttime->ttime_samples; + ttime->ttime_total = div_u64(7*ttime->ttime_total + 256*elapsed, 8); + ttime->ttime_mean = div64_ul(ttime->ttime_total + 128, + ttime->ttime_samples); } static void @@ -4113,7 +4123,7 @@ static void cfq_insert_request(struct request_queue *q, struct request *rq) cfq_log_cfqq(cfqd, cfqq, "insert_request"); cfq_init_prio_data(cfqq, RQ_CIC(rq)); - rq->fifo_time = jiffies + cfqd->cfq_fifo_expire[rq_is_sync(rq)]; + rq->fifo_time = ktime_get_ns() + cfqd->cfq_fifo_expire[rq_is_sync(rq)]; list_add_tail(&rq->queuelist, &cfqq->fifo); cfq_add_rq_rb(rq); cfqg_stats_update_io_add(RQ_CFQG(rq), cfqd->serving_group, req_op(rq), @@ -4161,6 +4171,7 @@ static void cfq_update_hw_tag(struct cfq_data *cfqd) static bool cfq_should_wait_busy(struct cfq_data *cfqd, struct cfq_queue *cfqq) { struct cfq_io_cq *cic = cfqd->active_cic; + u64 now = ktime_get_ns(); /* If the queue already has requests, don't wait */ if (!RB_EMPTY_ROOT(&cfqq->sort_list)) @@ -4179,7 +4190,7 @@ static bool cfq_should_wait_busy(struct cfq_data *cfqd, struct cfq_queue *cfqq) /* if slice left is less than think time, wait busy */ if (cic && sample_valid(cic->ttime.ttime_samples) - && (cfqq->slice_end - jiffies < cic->ttime.ttime_mean)) + && (cfqq->slice_end - now < cic->ttime.ttime_mean)) return true; /* @@ -4189,7 +4200,7 @@ static bool cfq_should_wait_busy(struct cfq_data *cfqd, struct cfq_queue *cfqq) * case where think time is less than a jiffy, mark the queue wait * busy if only 1 jiffy is left in the slice. */ - if (cfqq->slice_end - jiffies == 1) + if (cfqq->slice_end - now <= jiffies_to_nsecs(1)) return true; return false; @@ -4200,9 +4211,8 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq) struct cfq_queue *cfqq = RQ_CFQQ(rq); struct cfq_data *cfqd = cfqq->cfqd; const int sync = rq_is_sync(rq); - unsigned long now; + u64 now = ktime_get_ns(); - now = jiffies; cfq_log_cfqq(cfqd, cfqq, "complete rqnoidle %d", !!(rq->cmd_flags & REQ_NOIDLE)); @@ -4231,7 +4241,7 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq) cfqq_type(cfqq)); st->ttime.last_end_request = now; - if (!time_after(rq->start_time + cfqd->cfq_fifo_expire[1], now)) + if (!(rq->start_time + cfqd->cfq_fifo_expire[1] > now)) cfqd->last_delayed_sync = now; } @@ -4256,10 +4266,10 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq) * the queue. */ if (cfq_should_wait_busy(cfqd, cfqq)) { - unsigned long extend_sl = cfqd->cfq_slice_idle; + u64 extend_sl = cfqd->cfq_slice_idle; if (!cfqd->cfq_slice_idle) extend_sl = cfqd->cfq_group_idle; - cfqq->slice_end = jiffies + extend_sl; + cfqq->slice_end = now + extend_sl; cfq_mark_cfqq_wait_busy(cfqq); cfq_log_cfqq(cfqd, cfqq, "will busy wait"); } @@ -4618,7 +4628,7 @@ static int cfq_init_queue(struct request_queue *q, struct elevator_type *e) * we optimistically start assuming sync ops weren't delayed in last * second, in order to have larger depth for async operations. */ - cfqd->last_delayed_sync = jiffies - HZ; + cfqd->last_delayed_sync = ktime_get_ns() - NSEC_PER_SEC; return 0; out_free: @@ -4661,9 +4671,9 @@ cfq_var_store(unsigned int *var, const char *page, size_t count) static ssize_t __FUNC(struct elevator_queue *e, char *page) \ { \ struct cfq_data *cfqd = e->elevator_data; \ - unsigned int __data = __VAR; \ + u64 __data = __VAR; \ if (__CONV) \ - __data = jiffies_to_msecs(__data); \ + __data = div_u64(__data, NSEC_PER_MSEC); \ return cfq_var_show(__data, (page)); \ } SHOW_FUNCTION(cfq_quantum_show, cfqd->cfq_quantum, 0); @@ -4691,7 +4701,7 @@ static ssize_t __FUNC(struct elevator_queue *e, const char *page, size_t count) else if (__data > (MAX)) \ __data = (MAX); \ if (__CONV) \ - *(__PTR) = msecs_to_jiffies(__data); \ + *(__PTR) = (u64)__data * NSEC_PER_MSEC; \ else \ *(__PTR) = __data; \ return ret; \ @@ -4785,18 +4795,7 @@ static int __init cfq_init(void) { int ret; - /* - * could be 0 on HZ < 1000 setups - */ - if (!cfq_slice_async) - cfq_slice_async = 1; - if (!cfq_slice_idle) - cfq_slice_idle = 1; - #ifdef CONFIG_CFQ_GROUP_IOSCHED - if (!cfq_group_idle) - cfq_group_idle = 1; - ret = blkcg_policy_register(&blkcg_policy_cfq); if (ret) return ret; -- cgit From d2d481d04f70325e7ed45cbd6e72c15e745f6ede Mon Sep 17 00:00:00 2001 From: Jeff Moyer Date: Wed, 8 Jun 2016 15:11:38 +0200 Subject: cfq-iosched: Expose microsecond interfaces Expose interfaces to tune time slices of CFQ IO scheduler in microseconds. Signed-off-by: Jeff Moyer Signed-off-by: Jan Kara Signed-off-by: Jens Axboe --- block/cfq-iosched.c | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) (limited to 'block') diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 9c2e82c1ea88..2db7b1450dd9 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -4690,6 +4690,21 @@ SHOW_FUNCTION(cfq_low_latency_show, cfqd->cfq_latency, 0); SHOW_FUNCTION(cfq_target_latency_show, cfqd->cfq_target_latency, 1); #undef SHOW_FUNCTION +#define USEC_SHOW_FUNCTION(__FUNC, __VAR) \ +static ssize_t __FUNC(struct elevator_queue *e, char *page) \ +{ \ + struct cfq_data *cfqd = e->elevator_data; \ + u64 __data = __VAR; \ + __data = div_u64(__data, NSEC_PER_USEC); \ + return cfq_var_show(__data, (page)); \ +} +USEC_SHOW_FUNCTION(cfq_slice_idle_us_show, cfqd->cfq_slice_idle); +USEC_SHOW_FUNCTION(cfq_group_idle_us_show, cfqd->cfq_group_idle); +USEC_SHOW_FUNCTION(cfq_slice_sync_us_show, cfqd->cfq_slice[1]); +USEC_SHOW_FUNCTION(cfq_slice_async_us_show, cfqd->cfq_slice[0]); +USEC_SHOW_FUNCTION(cfq_target_latency_us_show, cfqd->cfq_target_latency); +#undef USEC_SHOW_FUNCTION + #define STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, __CONV) \ static ssize_t __FUNC(struct elevator_queue *e, const char *page, size_t count) \ { \ @@ -4724,6 +4739,26 @@ STORE_FUNCTION(cfq_low_latency_store, &cfqd->cfq_latency, 0, 1, 0); STORE_FUNCTION(cfq_target_latency_store, &cfqd->cfq_target_latency, 1, UINT_MAX, 1); #undef STORE_FUNCTION +#define USEC_STORE_FUNCTION(__FUNC, __PTR, MIN, MAX) \ +static ssize_t __FUNC(struct elevator_queue *e, const char *page, size_t count) \ +{ \ + struct cfq_data *cfqd = e->elevator_data; \ + unsigned int __data; \ + int ret = cfq_var_store(&__data, (page), count); \ + if (__data < (MIN)) \ + __data = (MIN); \ + else if (__data > (MAX)) \ + __data = (MAX); \ + *(__PTR) = (u64)__data * NSEC_PER_USEC; \ + return ret; \ +} +USEC_STORE_FUNCTION(cfq_slice_idle_us_store, &cfqd->cfq_slice_idle, 0, UINT_MAX); +USEC_STORE_FUNCTION(cfq_group_idle_us_store, &cfqd->cfq_group_idle, 0, UINT_MAX); +USEC_STORE_FUNCTION(cfq_slice_sync_us_store, &cfqd->cfq_slice[1], 1, UINT_MAX); +USEC_STORE_FUNCTION(cfq_slice_async_us_store, &cfqd->cfq_slice[0], 1, UINT_MAX); +USEC_STORE_FUNCTION(cfq_target_latency_us_store, &cfqd->cfq_target_latency, 1, UINT_MAX); +#undef USEC_STORE_FUNCTION + #define CFQ_ATTR(name) \ __ATTR(name, S_IRUGO|S_IWUSR, cfq_##name##_show, cfq_##name##_store) @@ -4734,12 +4769,17 @@ static struct elv_fs_entry cfq_attrs[] = { CFQ_ATTR(back_seek_max), CFQ_ATTR(back_seek_penalty), CFQ_ATTR(slice_sync), + CFQ_ATTR(slice_sync_us), CFQ_ATTR(slice_async), + CFQ_ATTR(slice_async_us), CFQ_ATTR(slice_async_rq), CFQ_ATTR(slice_idle), + CFQ_ATTR(slice_idle_us), CFQ_ATTR(group_idle), + CFQ_ATTR(group_idle_us), CFQ_ATTR(low_latency), CFQ_ATTR(target_latency), + CFQ_ATTR(target_latency_us), __ATTR_NULL }; -- cgit From 9114832581645e48622152a4ef21b88920c0167b Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 8 Jun 2016 15:11:39 +0200 Subject: cfq-iosched: Convert to use highres timers Reviewed-by: Jeff Moyer Signed-off-by: Jan Kara Signed-off-by: Jens Axboe --- block/cfq-iosched.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) (limited to 'block') diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 2db7b1450dd9..4e5978426ee7 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -362,7 +362,7 @@ struct cfq_data { /* * idle window management */ - struct timer_list idle_slice_timer; + struct hrtimer idle_slice_timer; struct work_struct unplug_work; struct cfq_queue *active_queue; @@ -2627,7 +2627,7 @@ static int cfq_allow_merge(struct request_queue *q, struct request *rq, static inline void cfq_del_timer(struct cfq_data *cfqd, struct cfq_queue *cfqq) { - del_timer(&cfqd->idle_slice_timer); + hrtimer_try_to_cancel(&cfqd->idle_slice_timer); cfqg_stats_update_idle_time(cfqq->cfqg); } @@ -2989,7 +2989,8 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd) else sl = cfqd->cfq_slice_idle; - mod_timer(&cfqd->idle_slice_timer, now + sl); + hrtimer_start(&cfqd->idle_slice_timer, ns_to_ktime(sl), + HRTIMER_MODE_REL); cfqg_stats_set_start_idle_time(cfqq->cfqg); cfq_log_cfqq(cfqd, cfqq, "arm_idle: %llu group_idle: %d", sl, group_idle ? 1 : 0); @@ -3308,7 +3309,7 @@ static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd) * flight or is idling for a new request, allow either of these * conditions to happen (or time out) before selecting a new queue. */ - if (timer_pending(&cfqd->idle_slice_timer)) { + if (hrtimer_active(&cfqd->idle_slice_timer)) { cfqq = NULL; goto keep_queue; } @@ -4454,9 +4455,10 @@ static void cfq_kick_queue(struct work_struct *work) /* * Timer running if the active_queue is currently idling inside its time slice */ -static void cfq_idle_slice_timer(unsigned long data) +static enum hrtimer_restart cfq_idle_slice_timer(struct hrtimer *timer) { - struct cfq_data *cfqd = (struct cfq_data *) data; + struct cfq_data *cfqd = container_of(timer, struct cfq_data, + idle_slice_timer); struct cfq_queue *cfqq; unsigned long flags; int timed_out = 1; @@ -4505,11 +4507,12 @@ out_kick: cfq_schedule_dispatch(cfqd); out_cont: spin_unlock_irqrestore(cfqd->queue->queue_lock, flags); + return HRTIMER_NORESTART; } static void cfq_shutdown_timer_wq(struct cfq_data *cfqd) { - del_timer_sync(&cfqd->idle_slice_timer); + hrtimer_cancel(&cfqd->idle_slice_timer); cancel_work_sync(&cfqd->unplug_work); } @@ -4605,9 +4608,9 @@ static int cfq_init_queue(struct request_queue *q, struct elevator_type *e) cfqg_put(cfqd->root_group); spin_unlock_irq(q->queue_lock); - init_timer(&cfqd->idle_slice_timer); + hrtimer_init(&cfqd->idle_slice_timer, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); cfqd->idle_slice_timer.function = cfq_idle_slice_timer; - cfqd->idle_slice_timer.data = (unsigned long) cfqd; INIT_WORK(&cfqd->unplug_work, cfq_kick_queue); -- cgit From 52b9c330c6a8a4b5a1819bdaddf4ec76ab571e81 Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Wed, 8 Jun 2016 18:22:20 -0700 Subject: blk-mq: actually hook up defer list when running requests If ->queue_rq() returns BLK_MQ_RQ_QUEUE_OK, we use continue and skip over the rest of the loop body. However, dptr is assigned later in the loop body, and the BLK_MQ_RQ_QUEUE_OK case is exactly the case that we'd want it for. NVMe isn't actually using BLK_MQ_F_DEFER_ISSUE yet, nor is any other in-tree driver, but if the code's going to be there, it might as well work. Fixes: 74c450521dd8 ("blk-mq: add a 'list' parameter to ->queue_rq()") Signed-off-by: Omar Sandoval Signed-off-by: Jens Axboe --- block/blk-mq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'block') diff --git a/block/blk-mq.c b/block/blk-mq.c index 13f460368759..bc7166d27340 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -785,7 +785,7 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) switch (ret) { case BLK_MQ_RQ_QUEUE_OK: queued++; - continue; + break; case BLK_MQ_RQ_QUEUE_BUSY: list_add(&rq->queuelist, &rq_list); __blk_mq_requeue_request(rq); -- cgit From b8269db456186ecc13469135c64d215883c410f6 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 9 Jun 2016 15:47:29 -0600 Subject: cfq-iosched: temporarily boost queue priority for idle classes If we're queuing REQ_PRIO IO and the task is running at an idle IO class, then temporarily boost the priority. This prevents livelocks due to priority inversion, when a low priority task is holding file system resources while attempting to do IO. An example of that is shown below. An ioniced idle task is holding the directory mutex, while a normal priority task is trying to do a directory lookup. [478381.198925] ------------[ cut here ]------------ [478381.200315] INFO: task ionice:1168369 blocked for more than 120 seconds. [478381.201324] Not tainted 4.0.9-38_fbk5_hotfix1_2936_g85409c6 #1 [478381.202278] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [478381.203462] ionice D ffff8803692736a8 0 1168369 1 0x00000080 [478381.203466] ffff8803692736a8 ffff880399c21300 ffff880276adcc00 ffff880369273698 [478381.204589] ffff880369273fd8 0000000000000000 7fffffffffffffff 0000000000000002 [478381.205752] ffffffff8177d5e0 ffff8803692736c8 ffffffff8177cea7 0000000000000000 [478381.206874] Call Trace: [478381.207253] [] ? bit_wait_io_timeout+0x80/0x80 [478381.208175] [] schedule+0x37/0x90 [478381.208932] [] schedule_timeout+0x1dc/0x250 [478381.209805] [] ? __blk_run_queue+0x37/0x50 [478381.210706] [] ? ktime_get+0x45/0xb0 [478381.211489] [] io_schedule_timeout+0xa7/0x110 [478381.212402] [] ? prepare_to_wait+0x5b/0x90 [478381.213280] [] bit_wait_io+0x36/0x50 [478381.214063] [] __wait_on_bit+0x65/0x90 [478381.214961] [] ? bit_wait_io_timeout+0x80/0x80 [478381.215872] [] out_of_line_wait_on_bit+0x7c/0x90 [478381.216806] [] ? wake_atomic_t_function+0x40/0x40 [478381.217773] [] __wait_on_buffer+0x2a/0x30 [478381.218641] [] ext4_bread+0x57/0x70 [478381.219425] [] __ext4_read_dirblock+0x3c/0x380 [478381.220467] [] ext4_dx_find_entry+0x7d/0x170 [478381.221357] [] ? find_get_entry+0x1e/0xa0 [478381.222208] [] ext4_find_entry+0x484/0x510 [478381.223090] [] ext4_lookup+0x52/0x160 [478381.223882] [] lookup_real+0x1d/0x60 [478381.224675] [] __lookup_hash+0x38/0x50 [478381.225697] [] lookup_slow+0x45/0xab [478381.226941] [] link_path_walk+0x7ae/0x820 [478381.227880] [] path_init+0xc2/0x430 [478381.228677] [] ? security_file_alloc+0x16/0x20 [478381.229776] [] path_openat+0x77/0x620 [478381.230767] [] ? page_add_file_rmap+0x2e/0x70 [478381.232019] [] do_filp_open+0x43/0xa0 [478381.233016] [] ? creds_are_invalid+0x29/0x70 [478381.234072] [] do_open_execat+0x70/0x170 [478381.235039] [] do_execveat_common.isra.36+0x1b8/0x6e0 [478381.236051] [] do_execve+0x2c/0x30 [478381.236809] [] ? getname+0x12/0x20 [478381.237564] [] SyS_execve+0x2e/0x40 [478381.238338] [] stub_execve+0x6d/0xa0 [478381.239126] ------------[ cut here ]------------ [478381.239915] ------------[ cut here ]------------ [478381.240606] INFO: task python2.7:1168375 blocked for more than 120 seconds. [478381.242673] Not tainted 4.0.9-38_fbk5_hotfix1_2936_g85409c6 #1 [478381.243653] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [478381.244902] python2.7 D ffff88005cf8fb98 0 1168375 1168248 0x00000080 [478381.244904] ffff88005cf8fb98 ffff88016c1f0980 ffffffff81c134c0 ffff88016c1f11a0 [478381.246023] ffff88005cf8ffd8 ffff880466cd0cbc ffff88016c1f0980 00000000ffffffff [478381.247138] ffff880466cd0cc0 ffff88005cf8fbb8 ffffffff8177cea7 ffff88005cf8fcc8 [478381.248252] Call Trace: [478381.248630] [] schedule+0x37/0x90 [478381.249382] [] schedule_preempt_disabled+0xe/0x10 [478381.250465] [] __mutex_lock_slowpath+0x92/0x100 [478381.251409] [] mutex_lock+0x1b/0x2f [478381.252199] [] lookup_slow+0x36/0xab [478381.253023] [] link_path_walk+0x7ae/0x820 [478381.253877] [] ? try_charge+0xc1/0x700 [478381.254690] [] path_init+0xc2/0x430 [478381.255525] [] ? security_file_alloc+0x16/0x20 [478381.256450] [] path_openat+0x77/0x620 [478381.257256] [] ? lru_cache_add_active_or_unevictable+0x2b/0xa0 [478381.258390] [] ? handle_mm_fault+0x13f3/0x1720 [478381.259309] [] do_filp_open+0x43/0xa0 [478381.260139] [] ? __alloc_fd+0x42/0x120 [478381.260962] [] do_sys_open+0x13c/0x230 [478381.261779] [] ? syscall_trace_enter_phase1+0x113/0x170 [478381.262851] [] SyS_open+0x22/0x30 [478381.263598] [] system_call_fastpath+0x12/0x17 [478381.264551] ------------[ cut here ]------------ [478381.265377] ------------[ cut here ]------------ Signed-off-by: Jens Axboe Reviewed-by: Jeff Moyer --- block/blk-core.c | 5 +++++ block/cfq-iosched.c | 22 +++++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index 32a283eb7274..3cfd67d006fb 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -1780,6 +1780,11 @@ get_rq: if (sync) rw_flags |= REQ_SYNC; + /* + * Add in META/PRIO flags, if set, before we get to the IO scheduler + */ + rw_flags |= (bio->bi_rw & (REQ_META | REQ_PRIO)); + /* * Grab a free request. This is might sleep but can not fail. * Returns with the queue unlocked. diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 4e5978426ee7..d227ad633242 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -141,7 +141,7 @@ struct cfq_queue { /* io prio of this group */ unsigned short ioprio, org_ioprio; - unsigned short ioprio_class; + unsigned short ioprio_class, org_ioprio_class; pid_t pid; @@ -3700,6 +3700,7 @@ static void cfq_init_prio_data(struct cfq_queue *cfqq, struct cfq_io_cq *cic) * elevate the priority of this queue */ cfqq->org_ioprio = cfqq->ioprio; + cfqq->org_ioprio_class = cfqq->ioprio_class; cfq_clear_cfqq_prio_changed(cfqq); } @@ -4295,6 +4296,24 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq) cfq_schedule_dispatch(cfqd); } +static void cfqq_boost_on_prio(struct cfq_queue *cfqq, int op_flags) +{ + /* + * If REQ_PRIO is set, boost class and prio level, if it's below + * BE/NORM. If prio is not set, restore the potentially boosted + * class/prio level. + */ + if (!(op_flags & REQ_PRIO)) { + cfqq->ioprio_class = cfqq->org_ioprio_class; + cfqq->ioprio = cfqq->org_ioprio; + } else { + if (cfq_class_idle(cfqq)) + cfqq->ioprio_class = IOPRIO_CLASS_BE; + if (cfqq->ioprio > IOPRIO_NORM) + cfqq->ioprio = IOPRIO_NORM; + } +} + static inline int __cfq_may_queue(struct cfq_queue *cfqq) { if (cfq_cfqq_wait_request(cfqq) && !cfq_cfqq_must_alloc_slice(cfqq)) { @@ -4325,6 +4344,7 @@ static int cfq_may_queue(struct request_queue *q, int op, int op_flags) cfqq = cic_to_cfqq(cic, rw_is_sync(op, op_flags)); if (cfqq) { cfq_init_prio_data(cfqq, cic); + cfqq_boost_on_prio(cfqq, op_flags); return __cfq_may_queue(cfqq); } -- cgit From aa8d15bfe4ed23b97bf3990e2901844efc6347ec Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Tue, 14 Jun 2016 17:03:13 +0200 Subject: block/partition-generic.c: Remove a set-but-not-used variable A value is assigned to the variable 'info' but that value is never used. Hence remove the variable 'info'. Signed-off-by: Bart Van Assche Signed-off-by: Jens Axboe --- block/partition-generic.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'block') diff --git a/block/partition-generic.c b/block/partition-generic.c index d7eb77e1e3a8..71d9ed9df8da 100644 --- a/block/partition-generic.c +++ b/block/partition-generic.c @@ -495,7 +495,6 @@ rescan: /* add partitions */ for (p = 1; p < state->limit; p++) { sector_t size, from; - struct partition_meta_info *info = NULL; size = state->parts[p].size; if (!size) @@ -530,8 +529,6 @@ rescan: } } - if (state->parts[p].has_info) - info = &state->parts[p].info; part = add_partition(disk, p, from, size, state->parts[p].flags, &state->parts[p].info); -- cgit From 1179a5a0851be6753b43601c00602dcef99e68f7 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Tue, 14 Jun 2016 17:03:45 +0200 Subject: block/bio-integrity.c: Add #include "blk.h" This patch avoids that building with W=1 C=2 triggers the following warning: block/bio-integrity.c:35:6: warning: symbol 'blk_flush_integrity' was not declared. Should it be static? Signed-off-by: Bart Van Assche Signed-off-by: Jens Axboe --- block/bio-integrity.c | 1 + 1 file changed, 1 insertion(+) (limited to 'block') diff --git a/block/bio-integrity.c b/block/bio-integrity.c index 711e4d8de6fa..15d37b1cd500 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -26,6 +26,7 @@ #include #include #include +#include "blk.h" #define BIP_INLINE_VECS 4 -- cgit From e1f3b9412edb22774cebf47e353b6ccc1b779cfe Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Tue, 14 Jun 2016 17:04:32 +0200 Subject: block/blk-cgroup.c: Declare local symbols static Detected by sparse. Signed-off-by: Bart Van Assche Cc: Tejun Heo Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'block') diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 66e6f1aae02e..dd38e5ced4a3 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -905,7 +905,7 @@ static int blkcg_print_stat(struct seq_file *sf, void *v) return 0; } -struct cftype blkcg_files[] = { +static struct cftype blkcg_files[] = { { .name = "stat", .flags = CFTYPE_NOT_ON_ROOT, @@ -914,7 +914,7 @@ struct cftype blkcg_files[] = { { } /* terminate */ }; -struct cftype blkcg_legacy_files[] = { +static struct cftype blkcg_legacy_files[] = { { .name = "reset_stats", .write_u64 = blkcg_reset_stats, -- cgit From 9828c2c6c1048c61034a8b94e6376aeff6d2284f Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 28 Jun 2016 09:03:59 +0200 Subject: block: Convert fifo_time from ulong to u64 Currently rq->fifo_time is unsigned long but CFQ stores nanosecond timestamp in it which would overflow on 32-bit archs. Convert it to u64 to avoid the overflow. Since the rq->fifo_time is unioned with struct call_single_data(), this does not change the size of struct request in any way. We have to slightly fixup block/deadline-iosched.c so that comparison happens in the right types. Fixes: 9a7f38c42c2b92391d9dabaf9f51df7cfe5608e4 Signed-off-by: Jan Kara Signed-off-by: Jens Axboe --- block/deadline-iosched.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'block') diff --git a/block/deadline-iosched.c b/block/deadline-iosched.c index d0dd7882d8c7..26a9d3c8057a 100644 --- a/block/deadline-iosched.c +++ b/block/deadline-iosched.c @@ -173,7 +173,8 @@ deadline_merged_requests(struct request_queue *q, struct request *req, * and move into next position (next will be deleted) in fifo */ if (!list_empty(&req->queuelist) && !list_empty(&next->queuelist)) { - if (time_before(next->fifo_time, req->fifo_time)) { + if (time_before((unsigned long)next->fifo_time, + (unsigned long)req->fifo_time)) { list_move(&req->queuelist, &next->queuelist); req->fifo_time = next->fifo_time; } @@ -227,7 +228,7 @@ static inline int deadline_check_fifo(struct deadline_data *dd, int ddir) /* * rq is expired! */ - if (time_after_eq(jiffies, rq->fifo_time)) + if (time_after_eq(jiffies, (unsigned long)rq->fifo_time)) return 1; return 0; -- cgit From 93fdf1478aaba6c397ba54f4cc534bf5019831b4 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 28 Jun 2016 09:04:00 +0200 Subject: cfq-iosched: Convert slice_resid from u64 to s64 slice_resid can be both positive and negative. Commit 9a7f38c42c2b (cfq-iosched: Convert from jiffies to nanoseconds) converted it from long to u64. Although this did not introduce any functional regression (the operations just overflow and the result was fine), it is certainly wrong and could cause issues in future. So convert the type to more appropriate s64. Fixes: 9a7f38c42c2b92391d9dabaf9f51df7cfe5608e4 Signed-off-by: Jan Kara Signed-off-by: Jens Axboe --- block/cfq-iosched.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'block') diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index d227ad633242..c1adbd01d0fa 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -132,7 +132,7 @@ struct cfq_queue { /* time when first request from queue completed and slice started. */ u64 slice_start; u64 slice_end; - u64 slice_resid; + s64 slice_resid; /* pending priority requests */ int prio_pending; @@ -2689,7 +2689,7 @@ __cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq, cfqq->slice_resid = cfq_scaled_cfqq_slice(cfqd, cfqq); else cfqq->slice_resid = cfqq->slice_end - ktime_get_ns(); - cfq_log_cfqq(cfqd, cfqq, "resid=%llu", cfqq->slice_resid); + cfq_log_cfqq(cfqd, cfqq, "resid=%lld", cfqq->slice_resid); } cfq_group_served(cfqd, cfqq->cfqg, cfqq); -- cgit From 149321a611d5d41cebcf5f813a3bf45b3afe66ad Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 28 Jun 2016 09:04:01 +0200 Subject: cfq-iosched: Fix regression in bonnie++ rewrite performance Commit 9a7f38c42c2 (cfq-iosched: Convert from jiffies to nanoseconds) broke the condition for detecting starved sync IO in cfq_completed_request() because rq->start_time remained in jiffies but we compared it with nanosecond values. This manifested as a regression in bonnie++ rewrite performance because we always ended up considering sync IO starved and thus never increased async IO queue depth. Since rq->start_time is used in a lot of places, converting it to ns values would be non-trivial. So just revert the condition in CFQ to use comparison with jiffies. This will lead to suboptimal results if cfq_fifo_expire[1] will ever come close to 1 jiffie but so far we are relatively far from that with the storage used with CFQ (the default value is 128 ms). Fixes: 9a7f38c42c2b92391d9dabaf9f51df7cfe5608e4 Signed-off-by: Jan Kara Signed-off-by: Jens Axboe --- block/cfq-iosched.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'block') diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index c1adbd01d0fa..2d03004dc77c 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -4243,7 +4243,16 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq) cfqq_type(cfqq)); st->ttime.last_end_request = now; - if (!(rq->start_time + cfqd->cfq_fifo_expire[1] > now)) + /* + * We have to do this check in jiffies since start_time is in + * jiffies and it is not trivial to convert to ns. If + * cfq_fifo_expire[1] ever comes close to 1 jiffie, this test + * will become problematic but so far we are fine (the default + * is 128 ms). + */ + if (!time_after(rq->start_time + + nsecs_to_jiffies(cfqd->cfq_fifo_expire[1]), + jiffies)) cfqd->last_delayed_sync = now; } -- cgit From 0b31c10c667b774e6d373c8f1146c93cff21a0cd Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 28 Jun 2016 09:04:02 +0200 Subject: cfq-iosched: Charge at least 1 jiffie instead of 1 ns Commit 9a7f38c42c2b (cfq-iosched: Convert from jiffies to nanoseconds) could result in charging just 1 ns to a cgroup submitting IO instead of 1 jiffie we always charged before. It is arguable what is the right amount to change but for now lets retain the old behavior of always charging at least one jiffie. Fixes: 9a7f38c42c2b92391d9dabaf9f51df7cfe5608e4 Signed-off-by: Jan Kara Signed-off-by: Jens Axboe --- block/cfq-iosched.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'block') diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 2d03004dc77c..892afd6d40e2 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -1463,7 +1463,8 @@ static inline u64 cfq_cfqq_slice_usage(struct cfq_queue *cfqq, * a single request on seeky media and cause lots of seek time * and group will never know it. */ - slice_used = max_t(u64, (now - cfqq->dispatch_start), 1); + slice_used = max_t(u64, (now - cfqq->dispatch_start), + jiffies_to_nsecs(1)); } else { slice_used = now - cfqq->slice_start; if (slice_used > cfqq->allocated_slice) { -- cgit From 9c8747167e204b3fe0ff143e1642c921e382e1d6 Mon Sep 17 00:00:00 2001 From: Gabriel Krisman Bertazi Date: Tue, 5 Jul 2016 13:38:32 -0300 Subject: block: atari: Return early for unsupported sector size For 4K LBA or very large disks, atari_partition can easily get tricked into thinking it has found an Atari partition table. Depending on the data in the disk, it ends up creating partitions with awkward lengths. We saw logs like this while playing with fio. [5.625867] nvme2n1: AHDI p2 [5.625872] nvme2n1: p2 size 2910030523 extends beyond EOD, truncated People has had issues with misinterpreted AHDI partition tables for a long time, see this BSD thread from 1995, for example. https://mail-index.netbsd.org/port-atari/1995/11/19/0001.html Since the atari partition, according to the spec, doesn't even support sector sizes with more than 512, a quick sanity check is reasonable to just bail out early, before even attempting to read sector 0. Signed-off-by: Gabriel Krisman Bertazi Signed-off-by: Jens Axboe --- block/partitions/atari.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'block') diff --git a/block/partitions/atari.c b/block/partitions/atari.c index 9875b05e80a2..ff1fb93712c1 100644 --- a/block/partitions/atari.c +++ b/block/partitions/atari.c @@ -42,6 +42,13 @@ int atari_partition(struct parsed_partitions *state) int part_fmt = 0; /* 0:unknown, 1:AHDI, 2:ICD/Supra */ #endif + /* + * ATARI partition scheme supports 512 lba only. If this is not + * the case, bail early to avoid miscalculating hd_size. + */ + if (bdev_logical_block_size(state->bdev) != 512) + return 0; + rs = read_part_sector(state, 0, §); if (!rs) return -1; -- cgit From ea6ca600eb90350259cee5514e97b91ad1bc1aa5 Mon Sep 17 00:00:00 2001 From: Yigal Korman Date: Thu, 23 Jun 2016 17:05:51 -0400 Subject: block: expose QUEUE_FLAG_DAX in sysfs Provides the ability to identify DAX enabled devices in userspace. Signed-off-by: Yigal Korman Signed-off-by: Toshi Kani Acked-by: Dan Williams Signed-off-by: Mike Snitzer Signed-off-by: Jens Axboe --- block/blk-sysfs.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'block') diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index 99205965f559..f87a7e747d36 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -379,6 +379,11 @@ static ssize_t queue_wc_store(struct request_queue *q, const char *page, return count; } +static ssize_t queue_dax_show(struct request_queue *q, char *page) +{ + return queue_var_show(blk_queue_dax(q), page); +} + static struct queue_sysfs_entry queue_requests_entry = { .attr = {.name = "nr_requests", .mode = S_IRUGO | S_IWUSR }, .show = queue_requests_show, @@ -516,6 +521,11 @@ static struct queue_sysfs_entry queue_wc_entry = { .store = queue_wc_store, }; +static struct queue_sysfs_entry queue_dax_entry = { + .attr = {.name = "dax", .mode = S_IRUGO }, + .show = queue_dax_show, +}; + static struct attribute *default_attrs[] = { &queue_requests_entry.attr, &queue_ra_entry.attr, @@ -542,6 +552,7 @@ static struct attribute *default_attrs[] = { &queue_random_entry.attr, &queue_poll_entry.attr, &queue_wc_entry.attr, + &queue_dax_entry.attr, NULL, }; -- cgit From 68bdf1ac2aa6318198adfe12407173bc60248b4e Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Tue, 19 Jul 2016 08:18:06 -0700 Subject: block: Fix spelling in a source code comment Signed-off-by: Bart Van Assche Cc: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-exec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'block') diff --git a/block/blk-exec.c b/block/blk-exec.c index 3fec8a29d0fa..7ea04325d02f 100644 --- a/block/blk-exec.c +++ b/block/blk-exec.c @@ -62,7 +62,7 @@ void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk, /* * don't check dying flag for MQ because the request won't - * be resued after dying flag is set + * be reused after dying flag is set */ if (q->mq_ops) { blk_mq_insert_request(rq, at_head, true, false); -- cgit From 72ef799b3f14f4cb4c56ba3af6e6bdcbae6df368 Mon Sep 17 00:00:00 2001 From: Tahsin Erdogan Date: Thu, 7 Jul 2016 11:48:22 -0700 Subject: block: do not merge requests without consulting with io scheduler Before merging a bio into an existing request, io scheduler is called to get its approval first. However, the requests that come from a plug flush may get merged by block layer without consulting with io scheduler. In case of CFQ, this can cause fairness problems. For instance, if a request gets merged into a low weight cgroup's request, high weight cgroup now will depend on low weight cgroup to get scheduled. If high weigt cgroup needs that io request to complete before submitting more requests, then it will also lose its timeslice. Following script demonstrates the problem. Group g1 has a low weight, g2 and g3 have equal high weights but g2's requests are adjacent to g1's requests so they are subject to merging. Due to these merges, g2 gets poor disk time allocation. cat > cfq-merge-repro.sh << "EOF" #!/bin/bash set -e IO_ROOT=/mnt-cgroup/io mkdir -p $IO_ROOT if ! mount | grep -qw $IO_ROOT; then mount -t cgroup none -oblkio $IO_ROOT fi cd $IO_ROOT for i in g1 g2 g3; do if [ -d $i ]; then rmdir $i fi done mkdir g1 && echo 10 > g1/blkio.weight mkdir g2 && echo 495 > g2/blkio.weight mkdir g3 && echo 495 > g3/blkio.weight RUNTIME=10 (echo $BASHPID > g1/cgroup.procs && fio --readonly --name name1 --filename /dev/sdb \ --rw read --size 64k --bs 64k --time_based \ --runtime=$RUNTIME --offset=0k &> /dev/null)& (echo $BASHPID > g2/cgroup.procs && fio --readonly --name name1 --filename /dev/sdb \ --rw read --size 64k --bs 64k --time_based \ --runtime=$RUNTIME --offset=64k &> /dev/null)& (echo $BASHPID > g3/cgroup.procs && fio --readonly --name name1 --filename /dev/sdb \ --rw read --size 64k --bs 64k --time_based \ --runtime=$RUNTIME --offset=256k &> /dev/null)& sleep $((RUNTIME+1)) for i in g1 g2 g3; do echo ---- $i ---- cat $i/blkio.time done EOF # ./cfq-merge-repro.sh ---- g1 ---- 8:16 162 ---- g2 ---- 8:16 165 ---- g3 ---- 8:16 686 After applying the patch: # ./cfq-merge-repro.sh ---- g1 ---- 8:16 90 ---- g2 ---- 8:16 445 ---- g3 ---- 8:16 471 Signed-off-by: Tahsin Erdogan Signed-off-by: Jens Axboe --- block/blk-merge.c | 6 ++++++ block/cfq-iosched.c | 15 +++++++++++---- block/deadline-iosched.c | 2 +- block/elevator.c | 22 +++++++++++----------- 4 files changed, 29 insertions(+), 16 deletions(-) (limited to 'block') diff --git a/block/blk-merge.c b/block/blk-merge.c index c265348b75d1..e7c2fbc3f656 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -744,6 +744,12 @@ int attempt_front_merge(struct request_queue *q, struct request *rq) int blk_attempt_req_merge(struct request_queue *q, struct request *rq, struct request *next) { + struct elevator_queue *e = q->elevator; + + if (e->type->ops.elevator_allow_rq_merge_fn) + if (!e->type->ops.elevator_allow_rq_merge_fn(q, rq, next)) + return 0; + return attempt_merge(q, rq, next); } diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 892afd6d40e2..acabba198de9 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -2544,7 +2544,7 @@ static int cfq_merge(struct request_queue *q, struct request **req, struct request *__rq; __rq = cfq_find_rq_fmerge(cfqd, bio); - if (__rq && elv_rq_merge_ok(__rq, bio)) { + if (__rq && elv_bio_merge_ok(__rq, bio)) { *req = __rq; return ELEVATOR_FRONT_MERGE; } @@ -2601,8 +2601,8 @@ cfq_merged_requests(struct request_queue *q, struct request *rq, cfq_del_cfqq_rr(cfqd, cfqq); } -static int cfq_allow_merge(struct request_queue *q, struct request *rq, - struct bio *bio) +static int cfq_allow_bio_merge(struct request_queue *q, struct request *rq, + struct bio *bio) { struct cfq_data *cfqd = q->elevator->elevator_data; struct cfq_io_cq *cic; @@ -2626,6 +2626,12 @@ static int cfq_allow_merge(struct request_queue *q, struct request *rq, return cfqq == RQ_CFQQ(rq); } +static int cfq_allow_rq_merge(struct request_queue *q, struct request *rq, + struct request *next) +{ + return RQ_CFQQ(rq) == RQ_CFQQ(next); +} + static inline void cfq_del_timer(struct cfq_data *cfqd, struct cfq_queue *cfqq) { hrtimer_try_to_cancel(&cfqd->idle_slice_timer); @@ -4821,7 +4827,8 @@ static struct elevator_type iosched_cfq = { .elevator_merge_fn = cfq_merge, .elevator_merged_fn = cfq_merged_request, .elevator_merge_req_fn = cfq_merged_requests, - .elevator_allow_merge_fn = cfq_allow_merge, + .elevator_allow_bio_merge_fn = cfq_allow_bio_merge, + .elevator_allow_rq_merge_fn = cfq_allow_rq_merge, .elevator_bio_merged_fn = cfq_bio_merged, .elevator_dispatch_fn = cfq_dispatch_requests, .elevator_add_req_fn = cfq_insert_request, diff --git a/block/deadline-iosched.c b/block/deadline-iosched.c index 26a9d3c8057a..55e0bb6d7da7 100644 --- a/block/deadline-iosched.c +++ b/block/deadline-iosched.c @@ -137,7 +137,7 @@ deadline_merge(struct request_queue *q, struct request **req, struct bio *bio) if (__rq) { BUG_ON(sector != blk_rq_pos(__rq)); - if (elv_rq_merge_ok(__rq, bio)) { + if (elv_bio_merge_ok(__rq, bio)) { ret = ELEVATOR_FRONT_MERGE; goto out; } diff --git a/block/elevator.c b/block/elevator.c index ea9319db50d7..7096c22041e7 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -53,13 +53,13 @@ static LIST_HEAD(elv_list); * Query io scheduler to see if the current process issuing bio may be * merged with rq. */ -static int elv_iosched_allow_merge(struct request *rq, struct bio *bio) +static int elv_iosched_allow_bio_merge(struct request *rq, struct bio *bio) { struct request_queue *q = rq->q; struct elevator_queue *e = q->elevator; - if (e->type->ops.elevator_allow_merge_fn) - return e->type->ops.elevator_allow_merge_fn(q, rq, bio); + if (e->type->ops.elevator_allow_bio_merge_fn) + return e->type->ops.elevator_allow_bio_merge_fn(q, rq, bio); return 1; } @@ -67,17 +67,17 @@ static int elv_iosched_allow_merge(struct request *rq, struct bio *bio) /* * can we safely merge with this request? */ -bool elv_rq_merge_ok(struct request *rq, struct bio *bio) +bool elv_bio_merge_ok(struct request *rq, struct bio *bio) { if (!blk_rq_merge_ok(rq, bio)) - return 0; + return false; - if (!elv_iosched_allow_merge(rq, bio)) - return 0; + if (!elv_iosched_allow_bio_merge(rq, bio)) + return false; - return 1; + return true; } -EXPORT_SYMBOL(elv_rq_merge_ok); +EXPORT_SYMBOL(elv_bio_merge_ok); static struct elevator_type *elevator_find(const char *name) { @@ -425,7 +425,7 @@ int elv_merge(struct request_queue *q, struct request **req, struct bio *bio) /* * First try one-hit cache. */ - if (q->last_merge && elv_rq_merge_ok(q->last_merge, bio)) { + if (q->last_merge && elv_bio_merge_ok(q->last_merge, bio)) { ret = blk_try_merge(q->last_merge, bio); if (ret != ELEVATOR_NO_MERGE) { *req = q->last_merge; @@ -440,7 +440,7 @@ int elv_merge(struct request_queue *q, struct request **req, struct bio *bio) * See if our hash lookup can find a potential backmerge. */ __rq = elv_rqhash_find(q, bio->bi_iter.bi_sector); - if (__rq && elv_rq_merge_ok(__rq, bio)) { + if (__rq && elv_bio_merge_ok(__rq, bio)) { *req = __rq; return ELEVATOR_BACK_MERGE; } -- cgit From 17007f3994cdb4643355c73f54f0adad006cf59e Mon Sep 17 00:00:00 2001 From: Damien Le Moal Date: Wed, 20 Jul 2016 21:40:47 -0600 Subject: block: Fix front merge check For a front merge, the maximum number of sectors of the request must be checked against the front merge BIO sector, not the current sector of the request. Signed-off-by: Damien Le Moal Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/blk-merge.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'block') diff --git a/block/blk-merge.c b/block/blk-merge.c index e7c2fbc3f656..5e4d93edeaf7 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -500,7 +500,7 @@ int ll_back_merge_fn(struct request_queue *q, struct request *req, integrity_req_gap_back_merge(req, bio)) return 0; if (blk_rq_sectors(req) + bio_sectors(bio) > - blk_rq_get_max_sectors(req)) { + blk_rq_get_max_sectors(req, blk_rq_pos(req))) { req->cmd_flags |= REQ_NOMERGE; if (req == q->last_merge) q->last_merge = NULL; @@ -524,7 +524,7 @@ int ll_front_merge_fn(struct request_queue *q, struct request *req, integrity_req_gap_front_merge(req, bio)) return 0; if (blk_rq_sectors(req) + bio_sectors(bio) > - blk_rq_get_max_sectors(req)) { + blk_rq_get_max_sectors(req, bio->bi_iter.bi_sector)) { req->cmd_flags |= REQ_NOMERGE; if (req == q->last_merge) q->last_merge = NULL; @@ -570,7 +570,7 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req, * Will it become too large? */ if ((blk_rq_sectors(req) + blk_rq_sectors(next)) > - blk_rq_get_max_sectors(req)) + blk_rq_get_max_sectors(req, blk_rq_pos(req))) return 0; total_phys_segments = req->nr_phys_segments + next->nr_phys_segments; -- cgit