diff options
Diffstat (limited to 'drivers/block/drbd/drbd_req.c')
| -rw-r--r-- | drivers/block/drbd/drbd_req.c | 119 |
1 files changed, 66 insertions, 53 deletions
diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c index 13beb98a7c5a..d15826f6ee81 100644 --- a/drivers/block/drbd/drbd_req.c +++ b/drivers/block/drbd/drbd_req.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-or-later +// SPDX-License-Identifier: GPL-2.0-only /* drbd_req.c @@ -30,12 +30,7 @@ static struct drbd_request *drbd_req_new(struct drbd_device *device, struct bio return NULL; memset(req, 0, sizeof(*req)); - req->private_bio = bio_clone_fast(bio_src, GFP_NOIO, &drbd_io_bio_set); - req->private_bio->bi_private = req; - req->private_bio->bi_end_io = drbd_request_endio; - req->rq_state = (bio_data_dir(bio_src) == WRITE ? RQ_WRITE : 0) - | (bio_op(bio_src) == REQ_OP_WRITE_SAME ? RQ_WSAME : 0) | (bio_op(bio_src) == REQ_OP_WRITE_ZEROES ? RQ_ZEROES : 0) | (bio_op(bio_src) == REQ_OP_DISCARD ? RQ_UNMAP : 0); req->device = device; @@ -127,12 +122,13 @@ void drbd_req_destroy(struct kref *kref) * before it even was submitted or sent. * In that case we do not want to touch the bitmap at all. */ + struct drbd_peer_device *peer_device = first_peer_device(device); if ((s & (RQ_POSTPONED|RQ_LOCAL_MASK|RQ_NET_MASK)) != RQ_POSTPONED) { if (!(s & RQ_NET_OK) || !(s & RQ_LOCAL_OK)) - drbd_set_out_of_sync(device, req->i.sector, req->i.size); + drbd_set_out_of_sync(peer_device, req->i.sector, req->i.size); if ((s & RQ_NET_OK) && (s & RQ_LOCAL_OK) && (s & RQ_NET_SIS)) - drbd_set_in_sync(device, req->i.sector, req->i.size); + drbd_set_in_sync(peer_device, req->i.sector, req->i.size); } /* one might be tempted to move the drbd_al_complete_io @@ -149,7 +145,7 @@ void drbd_req_destroy(struct kref *kref) if (get_ldev_if_state(device, D_FAILED)) { drbd_al_complete_io(device, &req->i); put_ldev(device); - } else if (__ratelimit(&drbd_ratelimit_state)) { + } else if (drbd_ratelimit()) { drbd_warn(device, "Should have called drbd_al_complete_io(, %llu, %u), " "but my Disk seems to have failed :(\n", (unsigned long long) req->i.sector, req->i.size); @@ -180,7 +176,8 @@ void start_new_tl_epoch(struct drbd_connection *connection) void complete_master_bio(struct drbd_device *device, struct bio_and_error *m) { - m->bio->bi_status = errno_to_blk_status(m->error); + if (unlikely(m->error)) + m->bio->bi_status = errno_to_blk_status(m->error); bio_endio(m->bio); dec_ap_bio(device); } @@ -332,17 +329,21 @@ static void set_if_null_req_next(struct drbd_peer_device *peer_device, struct dr static void advance_conn_req_next(struct drbd_peer_device *peer_device, struct drbd_request *req) { struct drbd_connection *connection = peer_device ? peer_device->connection : NULL; + struct drbd_request *iter = req; if (!connection) return; if (connection->req_next != req) return; - list_for_each_entry_continue(req, &connection->transfer_log, tl_requests) { - const unsigned s = req->rq_state; - if (s & RQ_NET_QUEUED) + + req = NULL; + list_for_each_entry_continue(iter, &connection->transfer_log, tl_requests) { + const unsigned int s = iter->rq_state; + + if (s & RQ_NET_QUEUED) { + req = iter; break; + } } - if (&req->tl_requests == &connection->transfer_log) - req = NULL; connection->req_next = req; } @@ -358,17 +359,21 @@ static void set_if_null_req_ack_pending(struct drbd_peer_device *peer_device, st static void advance_conn_req_ack_pending(struct drbd_peer_device *peer_device, struct drbd_request *req) { struct drbd_connection *connection = peer_device ? peer_device->connection : NULL; + struct drbd_request *iter = req; if (!connection) return; if (connection->req_ack_pending != req) return; - list_for_each_entry_continue(req, &connection->transfer_log, tl_requests) { - const unsigned s = req->rq_state; - if ((s & RQ_NET_SENT) && (s & RQ_NET_PENDING)) + + req = NULL; + list_for_each_entry_continue(iter, &connection->transfer_log, tl_requests) { + const unsigned int s = iter->rq_state; + + if ((s & RQ_NET_SENT) && (s & RQ_NET_PENDING)) { + req = iter; break; + } } - if (&req->tl_requests == &connection->transfer_log) - req = NULL; connection->req_ack_pending = req; } @@ -384,17 +389,21 @@ static void set_if_null_req_not_net_done(struct drbd_peer_device *peer_device, s static void advance_conn_req_not_net_done(struct drbd_peer_device *peer_device, struct drbd_request *req) { struct drbd_connection *connection = peer_device ? peer_device->connection : NULL; + struct drbd_request *iter = req; if (!connection) return; if (connection->req_not_net_done != req) return; - list_for_each_entry_continue(req, &connection->transfer_log, tl_requests) { - const unsigned s = req->rq_state; - if ((s & RQ_NET_SENT) && !(s & RQ_NET_DONE)) + + req = NULL; + list_for_each_entry_continue(iter, &connection->transfer_log, tl_requests) { + const unsigned int s = iter->rq_state; + + if ((s & RQ_NET_SENT) && !(s & RQ_NET_DONE)) { + req = iter; break; + } } - if (&req->tl_requests == &connection->transfer_log) - req = NULL; connection->req_not_net_done = req; } @@ -510,16 +519,14 @@ static void mod_rq_state(struct drbd_request *req, struct bio_and_error *m, static void drbd_report_io_error(struct drbd_device *device, struct drbd_request *req) { - char b[BDEVNAME_SIZE]; - - if (!__ratelimit(&drbd_ratelimit_state)) + if (!drbd_ratelimit()) return; - drbd_warn(device, "local %s IO error sector %llu+%u on %s\n", + drbd_warn(device, "local %s IO error sector %llu+%u on %pg\n", (req->rq_state & RQ_WRITE) ? "WRITE" : "READ", (unsigned long long)req->i.sector, req->i.size >> 9, - bdevname(device->ldev->backing_bdev, b)); + device->ldev->backing_bdev); } /* Helper for HANDED_OVER_TO_NETWORK. @@ -546,12 +553,15 @@ static inline bool is_pending_write_protocol_A(struct drbd_request *req) * happen "atomically" within the req_lock, * and it enforces that we have to think in a very structured manner * about the "events" that may happen to a request during its life time ... + * + * + * peer_device == NULL means local disk */ int __req_mod(struct drbd_request *req, enum drbd_req_event what, + struct drbd_peer_device *peer_device, struct bio_and_error *m) { struct drbd_device *const device = req->device; - struct drbd_peer_device *const peer_device = first_peer_device(device); struct drbd_connection *const connection = peer_device ? peer_device->connection : NULL; struct net_conf *nc; int p, rv = 0; @@ -611,7 +621,7 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what, break; case READ_COMPLETED_WITH_ERROR: - drbd_set_out_of_sync(device, req->i.sector, req->i.size); + drbd_set_out_of_sync(peer_device, req->i.sector, req->i.size); drbd_report_io_error(device, req); __drbd_chk_io_error(device, DRBD_READ_ERROR); fallthrough; @@ -905,13 +915,11 @@ static bool drbd_may_do_local_read(struct drbd_device *device, sector_t sector, static bool remote_due_to_read_balancing(struct drbd_device *device, sector_t sector, enum drbd_read_balancing rbm) { - struct backing_dev_info *bdi; int stripe_shift; switch (rbm) { case RB_CONGESTED_REMOTE: - bdi = device->ldev->backing_bdev->bd_disk->queue->backing_dev_info; - return bdi_read_congested(bdi); + return false; case RB_LEAST_PENDING: return atomic_read(&device->local_cnt) > atomic_read(&device->ap_pending_cnt) + atomic_read(&device->rs_pending_cnt); @@ -1096,6 +1104,7 @@ static bool drbd_should_send_out_of_sync(union drbd_dev_state s) static int drbd_process_write_request(struct drbd_request *req) { struct drbd_device *device = req->device; + struct drbd_peer_device *peer_device = first_peer_device(device); int remote, send_oos; remote = drbd_should_do_remote(device->state); @@ -1111,7 +1120,7 @@ static int drbd_process_write_request(struct drbd_request *req) /* The only size==0 bios we expect are empty flushes. */ D_ASSERT(device, req->master_bio->bi_opf & REQ_PREFLUSH); if (remote) - _req_mod(req, QUEUE_AS_DRBD_BARRIER); + _req_mod(req, QUEUE_AS_DRBD_BARRIER, peer_device); return remote; } @@ -1121,10 +1130,10 @@ static int drbd_process_write_request(struct drbd_request *req) D_ASSERT(device, !(remote && send_oos)); if (remote) { - _req_mod(req, TO_BE_SENT); - _req_mod(req, QUEUE_FOR_NET_WRITE); - } else if (drbd_set_out_of_sync(device, req->i.sector, req->i.size)) - _req_mod(req, QUEUE_FOR_SEND_OOS); + _req_mod(req, TO_BE_SENT, peer_device); + _req_mod(req, QUEUE_FOR_NET_WRITE, peer_device); + } else if (drbd_set_out_of_sync(peer_device, req->i.sector, req->i.size)) + _req_mod(req, QUEUE_FOR_SEND_OOS, peer_device); return remote; } @@ -1152,8 +1161,6 @@ drbd_submit_req_private_bio(struct drbd_request *req) else type = DRBD_FAULT_DT_RD; - bio_set_dev(bio, device->ldev->backing_bdev); - /* State may have changed since we grabbed our reference on the * ->ldev member. Double check, and short-circuit to endio. * In case the last activity log transaction failed to get on @@ -1212,9 +1219,12 @@ drbd_request_prepare(struct drbd_device *device, struct bio *bio) /* Update disk stats */ req->start_jif = bio_start_io_acct(req->master_bio); - if (!get_ldev(device)) { - bio_put(req->private_bio); - req->private_bio = NULL; + if (get_ldev(device)) { + req->private_bio = bio_alloc_clone(device->ldev->backing_bdev, + bio, GFP_NOIO, + &drbd_io_bio_set); + req->private_bio->bi_private = req; + req->private_bio->bi_end_io = drbd_request_endio; } /* process discards always from our submitter thread */ @@ -1307,6 +1317,7 @@ static void drbd_update_plug(struct drbd_plug_cb *plug, struct drbd_request *req static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request *req) { struct drbd_resource *resource = device->resource; + struct drbd_peer_device *peer_device = first_peer_device(device); const int rw = bio_data_dir(req->master_bio); struct bio_and_error m = { NULL, }; bool no_remote = false; @@ -1370,8 +1381,8 @@ static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request /* We either have a private_bio, or we can read from remote. * Otherwise we had done the goto nodata above. */ if (req->private_bio == NULL) { - _req_mod(req, TO_BE_SENT); - _req_mod(req, QUEUE_FOR_NET_READ); + _req_mod(req, TO_BE_SENT, peer_device); + _req_mod(req, QUEUE_FOR_NET_READ, peer_device); } else no_remote = true; } @@ -1392,12 +1403,12 @@ static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request req->pre_submit_jif = jiffies; list_add_tail(&req->req_pending_local, &device->pending_completion[rw == WRITE]); - _req_mod(req, TO_BE_SUBMITTED); + _req_mod(req, TO_BE_SUBMITTED, NULL); /* but we need to give up the spinlock to submit */ submit_private_bio = true; } else if (no_remote) { nodata: - if (__ratelimit(&drbd_ratelimit_state)) + if (drbd_ratelimit()) drbd_err(device, "IO ERROR: neither local nor remote data, sector %llu+%u\n", (unsigned long long)req->i.sector, req->i.size >> 9); /* A write may have been queued for send_oos, however. @@ -1597,11 +1608,13 @@ void do_submit(struct work_struct *ws) } } -blk_qc_t drbd_submit_bio(struct bio *bio) +void drbd_submit_bio(struct bio *bio) { struct drbd_device *device = bio->bi_bdev->bd_disk->private_data; - blk_queue_split(&bio); + bio = bio_split_to_limits(bio); + if (!bio) + return; /* * what we "blindly" assume: @@ -1610,7 +1623,6 @@ blk_qc_t drbd_submit_bio(struct bio *bio) inc_ap_bio(device); __drbd_make_request(device, bio); - return BLK_QC_T_NONE; } static bool net_timeout_reached(struct drbd_request *net_req, @@ -1687,7 +1699,8 @@ static bool net_timeout_reached(struct drbd_request *net_req, void request_timer_fn(struct timer_list *t) { - struct drbd_device *device = from_timer(device, t, request_timer); + struct drbd_device *device = timer_container_of(device, t, + request_timer); struct drbd_connection *connection = first_peer_device(device)->connection; struct drbd_request *req_read, *req_write, *req_peer; /* oldest request */ struct net_conf *nc; |