summaryrefslogtreecommitdiff
path: root/block/blk.h
diff options
context:
space:
mode:
authorJens Axboe <axboe@kernel.dk>2021-10-14 14:39:59 -0600
committerJens Axboe <axboe@kernel.dk>2021-12-03 14:51:29 -0700
commit0a467d0fdd9594fbb449ebc93852533332c528fd (patch)
treefd368b2ce4f835da5d78e1bbd381b38aa98fd078 /block/blk.h
parentceaa762527f41a431b552bc000de4b626d2d8cb7 (diff)
block: switch to atomic_t for request references
refcount_t is not as expensive as it used to be, but it's still more expensive than the io_uring method of using atomic_t and just checking for potential over/underflow. This borrows that same implementation, which in turn is based on the mm implementation from Linus. Reviewed-by: Keith Busch <kbusch@kernel.org> Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'block/blk.h')
-rw-r--r--block/blk.h31
1 files changed, 31 insertions, 0 deletions
diff --git a/block/blk.h b/block/blk.h
index a55d82c3d1c2..24d8b333bb03 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -461,4 +461,35 @@ static inline bool should_fail_request(struct block_device *part,
}
#endif /* CONFIG_FAIL_MAKE_REQUEST */
+/*
+ * Optimized request reference counting. Ideally we'd make timeouts be more
+ * clever, as that's the only reason we need references at all... But until
+ * this happens, this is faster than using refcount_t. Also see:
+ *
+ * abc54d634334 ("io_uring: switch to atomic_t for io_kiocb reference count")
+ */
+#define req_ref_zero_or_close_to_overflow(req) \
+ ((unsigned int) atomic_read(&(req->ref)) + 127u <= 127u)
+
+static inline bool req_ref_inc_not_zero(struct request *req)
+{
+ return atomic_inc_not_zero(&req->ref);
+}
+
+static inline bool req_ref_put_and_test(struct request *req)
+{
+ WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
+ return atomic_dec_and_test(&req->ref);
+}
+
+static inline void req_ref_set(struct request *req, int value)
+{
+ atomic_set(&req->ref, value);
+}
+
+static inline int req_ref_read(struct request *req)
+{
+ return atomic_read(&req->ref);
+}
+
#endif /* BLK_INTERNAL_H */