summaryrefslogtreecommitdiff
path: root/tools/testing/selftests/ublk/kublk.h
diff options
context:
space:
mode:
Diffstat (limited to 'tools/testing/selftests/ublk/kublk.h')
-rw-r--r--tools/testing/selftests/ublk/kublk.h180
1 files changed, 81 insertions, 99 deletions
diff --git a/tools/testing/selftests/ublk/kublk.h b/tools/testing/selftests/ublk/kublk.h
index e34508bf5798..219233f8a053 100644
--- a/tools/testing/selftests/ublk/kublk.h
+++ b/tools/testing/selftests/ublk/kublk.h
@@ -29,13 +29,9 @@
#include "ublk_dep.h"
#include <linux/ublk_cmd.h>
-#define __maybe_unused __attribute__((unused))
-#define MAX_BACK_FILES 4
-#ifndef min
-#define min(a, b) ((a) < (b) ? (a) : (b))
-#endif
+#include "utils.h"
-#define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0]))
+#define MAX_BACK_FILES 4
/****************** part 1: libublk ********************/
@@ -45,22 +41,16 @@
#define UBLK_CTRL_RING_DEPTH 32
#define ERROR_EVTFD_DEVID -2
-/* queue idle timeout */
-#define UBLKSRV_IO_IDLE_SECS 20
-
#define UBLK_IO_MAX_BYTES (1 << 20)
-#define UBLK_MAX_QUEUES 32
+#define UBLK_MAX_QUEUES_SHIFT 5
+#define UBLK_MAX_QUEUES (1 << UBLK_MAX_QUEUES_SHIFT)
+#define UBLK_MAX_THREADS_SHIFT 5
+#define UBLK_MAX_THREADS (1 << UBLK_MAX_THREADS_SHIFT)
#define UBLK_QUEUE_DEPTH 1024
-#define UBLK_DBG_DEV (1U << 0)
-#define UBLK_DBG_QUEUE (1U << 1)
-#define UBLK_DBG_IO_CMD (1U << 2)
-#define UBLK_DBG_IO (1U << 3)
-#define UBLK_DBG_CTRL_CMD (1U << 4)
-#define UBLK_LOG (1U << 5)
-
struct ublk_dev;
struct ublk_queue;
+struct ublk_thread;
struct stripe_ctx {
/* stripe */
@@ -76,6 +66,7 @@ struct dev_ctx {
char tgt_type[16];
unsigned long flags;
unsigned nr_hw_queues;
+ unsigned short nthreads;
unsigned queue_depth;
int dev_id;
int nr_files;
@@ -85,6 +76,7 @@ struct dev_ctx {
unsigned int fg:1;
unsigned int recovery:1;
unsigned int auto_zc_fallback:1;
+ unsigned int per_io_tasks:1;
int _evtfd;
int _shmid;
@@ -115,16 +107,19 @@ struct ublk_ctrl_cmd_data {
struct ublk_io {
char *buf_addr;
-#define UBLKSRV_NEED_FETCH_RQ (1UL << 0)
-#define UBLKSRV_NEED_COMMIT_RQ_COMP (1UL << 1)
-#define UBLKSRV_IO_FREE (1UL << 2)
-#define UBLKSRV_NEED_GET_DATA (1UL << 3)
-#define UBLKSRV_NEED_REG_BUF (1UL << 4)
+#define UBLKS_IO_NEED_FETCH_RQ (1UL << 0)
+#define UBLKS_IO_NEED_COMMIT_RQ_COMP (1UL << 1)
+#define UBLKS_IO_FREE (1UL << 2)
+#define UBLKS_IO_NEED_GET_DATA (1UL << 3)
+#define UBLKS_IO_NEED_REG_BUF (1UL << 4)
unsigned short flags;
unsigned short refs; /* used by target code only */
+ int tag;
+
int result;
+ unsigned short buf_index;
unsigned short tgt_ios;
void *private_data;
};
@@ -134,9 +129,9 @@ struct ublk_tgt_ops {
int (*init_tgt)(const struct dev_ctx *ctx, struct ublk_dev *);
void (*deinit_tgt)(struct ublk_dev *);
- int (*queue_io)(struct ublk_queue *, int tag);
- void (*tgt_io_done)(struct ublk_queue *,
- int tag, const struct io_uring_cqe *);
+ int (*queue_io)(struct ublk_thread *, struct ublk_queue *, int tag);
+ void (*tgt_io_done)(struct ublk_thread *, struct ublk_queue *,
+ const struct io_uring_cqe *);
/*
* Target specific command line handling
@@ -165,28 +160,37 @@ struct ublk_tgt {
struct ublk_queue {
int q_id;
int q_depth;
- unsigned int cmd_inflight;
- unsigned int io_inflight;
struct ublk_dev *dev;
const struct ublk_tgt_ops *tgt_ops;
struct ublksrv_io_desc *io_cmd_buf;
- struct io_uring ring;
+
+/* borrow one bit of ublk uapi flags, which may never be used */
+#define UBLKS_Q_AUTO_BUF_REG_FALLBACK (1ULL << 63)
+ __u64 flags;
struct ublk_io ios[UBLK_QUEUE_DEPTH];
-#define UBLKSRV_QUEUE_STOPPING (1U << 0)
-#define UBLKSRV_QUEUE_IDLE (1U << 1)
-#define UBLKSRV_NO_BUF (1U << 2)
-#define UBLKSRV_ZC (1U << 3)
-#define UBLKSRV_AUTO_BUF_REG (1U << 4)
-#define UBLKSRV_AUTO_BUF_REG_FALLBACK (1U << 5)
- unsigned state;
- pid_t tid;
+};
+
+struct ublk_thread {
+ struct ublk_dev *dev;
+ struct io_uring ring;
+ unsigned int cmd_inflight;
+ unsigned int io_inflight;
+
pthread_t thread;
+ unsigned idx;
+
+#define UBLKS_T_STOPPING (1U << 0)
+#define UBLKS_T_IDLE (1U << 1)
+ unsigned state;
};
struct ublk_dev {
struct ublk_tgt tgt;
struct ublksrv_ctrl_dev_info dev_info;
struct ublk_queue q[UBLK_MAX_QUEUES];
+ struct ublk_thread threads[UBLK_MAX_THREADS];
+ unsigned nthreads;
+ unsigned per_io_tasks;
int fds[MAX_BACK_FILES + 1]; /* fds[0] points to /dev/ublkcN */
int nr_fds;
@@ -196,22 +200,7 @@ struct ublk_dev {
void *private_data;
};
-#ifndef offsetof
-#define offsetof(TYPE, MEMBER) ((size_t)&((TYPE *)0)->MEMBER)
-#endif
-
-#ifndef container_of
-#define container_of(ptr, type, member) ({ \
- unsigned long __mptr = (unsigned long)(ptr); \
- ((type *)(__mptr - offsetof(type, member))); })
-#endif
-
-#define round_up(val, rnd) \
- (((val) + ((rnd) - 1)) & ~((rnd) - 1))
-
-
-extern unsigned int ublk_dbg_mask;
-extern int ublk_queue_io_cmd(struct ublk_queue *q, struct ublk_io *io, unsigned tag);
+extern int ublk_queue_io_cmd(struct ublk_thread *t, struct ublk_io *io);
static inline int ublk_io_auto_zc_fallback(const struct ublksrv_io_desc *iod)
@@ -225,11 +214,14 @@ static inline int is_target_io(__u64 user_data)
}
static inline __u64 build_user_data(unsigned tag, unsigned op,
- unsigned tgt_data, unsigned is_target_io)
+ unsigned tgt_data, unsigned q_id, unsigned is_target_io)
{
- assert(!(tag >> 16) && !(op >> 8) && !(tgt_data >> 16));
+ /* we only have 7 bits to encode q_id */
+ _Static_assert(UBLK_MAX_QUEUES_SHIFT <= 7);
+ assert(!(tag >> 16) && !(op >> 8) && !(tgt_data >> 16) && !(q_id >> 7));
- return tag | (op << 16) | (tgt_data << 24) | (__u64)is_target_io << 63;
+ return tag | (op << 16) | (tgt_data << 24) |
+ (__u64)q_id << 56 | (__u64)is_target_io << 63;
}
static inline unsigned int user_data_to_tag(__u64 user_data)
@@ -247,50 +239,33 @@ static inline unsigned int user_data_to_tgt_data(__u64 user_data)
return (user_data >> 24) & 0xffff;
}
-static inline unsigned short ublk_cmd_op_nr(unsigned int op)
-{
- return _IOC_NR(op);
-}
-
-static inline void ublk_err(const char *fmt, ...)
+static inline unsigned int user_data_to_q_id(__u64 user_data)
{
- va_list ap;
-
- va_start(ap, fmt);
- vfprintf(stderr, fmt, ap);
+ return (user_data >> 56) & 0x7f;
}
-static inline void ublk_log(const char *fmt, ...)
+static inline unsigned short ublk_cmd_op_nr(unsigned int op)
{
- if (ublk_dbg_mask & UBLK_LOG) {
- va_list ap;
-
- va_start(ap, fmt);
- vfprintf(stdout, fmt, ap);
- }
+ return _IOC_NR(op);
}
-static inline void ublk_dbg(int level, const char *fmt, ...)
+static inline struct ublk_queue *ublk_io_to_queue(const struct ublk_io *io)
{
- if (level & ublk_dbg_mask) {
- va_list ap;
-
- va_start(ap, fmt);
- vfprintf(stdout, fmt, ap);
- }
+ return container_of(io, struct ublk_queue, ios[io->tag]);
}
-static inline int ublk_queue_alloc_sqes(struct ublk_queue *q,
+static inline int ublk_io_alloc_sqes(struct ublk_thread *t,
struct io_uring_sqe *sqes[], int nr_sqes)
{
- unsigned left = io_uring_sq_space_left(&q->ring);
+ struct io_uring *ring = &t->ring;
+ unsigned left = io_uring_sq_space_left(ring);
int i;
if (left < nr_sqes)
- io_uring_submit(&q->ring);
+ io_uring_submit(ring);
for (i = 0; i < nr_sqes; i++) {
- sqes[i] = io_uring_get_sqe(&q->ring);
+ sqes[i] = io_uring_get_sqe(ring);
if (!sqes[i])
return i;
}
@@ -345,7 +320,7 @@ static inline int ublk_get_io_res(const struct ublk_queue *q, unsigned tag)
static inline void ublk_mark_io_done(struct ublk_io *io, int res)
{
- io->flags |= (UBLKSRV_NEED_COMMIT_RQ_COMP | UBLKSRV_IO_FREE);
+ io->flags |= (UBLKS_IO_NEED_COMMIT_RQ_COMP | UBLKS_IO_FREE);
io->result = res;
}
@@ -367,45 +342,58 @@ static inline struct ublk_io *ublk_get_io(struct ublk_queue *q, unsigned tag)
return &q->ios[tag];
}
-static inline int ublk_complete_io(struct ublk_queue *q, unsigned tag, int res)
+static inline int ublk_complete_io(struct ublk_thread *t, struct ublk_queue *q,
+ unsigned tag, int res)
{
struct ublk_io *io = &q->ios[tag];
ublk_mark_io_done(io, res);
- return ublk_queue_io_cmd(q, io, tag);
+ return ublk_queue_io_cmd(t, io);
}
-static inline void ublk_queued_tgt_io(struct ublk_queue *q, unsigned tag, int queued)
+static inline void ublk_queued_tgt_io(struct ublk_thread *t, struct ublk_queue *q,
+ unsigned tag, int queued)
{
if (queued < 0)
- ublk_complete_io(q, tag, queued);
+ ublk_complete_io(t, q, tag, queued);
else {
struct ublk_io *io = ublk_get_io(q, tag);
- q->io_inflight += queued;
+ t->io_inflight += queued;
io->tgt_ios = queued;
io->result = 0;
}
}
-static inline int ublk_completed_tgt_io(struct ublk_queue *q, unsigned tag)
+static inline int ublk_completed_tgt_io(struct ublk_thread *t,
+ struct ublk_queue *q, unsigned tag)
{
struct ublk_io *io = ublk_get_io(q, tag);
- q->io_inflight--;
+ t->io_inflight--;
return --io->tgt_ios == 0;
}
static inline int ublk_queue_use_zc(const struct ublk_queue *q)
{
- return q->state & UBLKSRV_ZC;
+ return q->flags & UBLK_F_SUPPORT_ZERO_COPY;
}
static inline int ublk_queue_use_auto_zc(const struct ublk_queue *q)
{
- return q->state & UBLKSRV_AUTO_BUF_REG;
+ return q->flags & UBLK_F_AUTO_BUF_REG;
+}
+
+static inline int ublk_queue_auto_zc_fallback(const struct ublk_queue *q)
+{
+ return q->flags & UBLKS_Q_AUTO_BUF_REG_FALLBACK;
+}
+
+static inline int ublk_queue_no_buf(const struct ublk_queue *q)
+{
+ return ublk_queue_use_zc(q) || ublk_queue_use_auto_zc(q);
}
extern const struct ublk_tgt_ops null_tgt_ops;
@@ -416,10 +404,4 @@ extern const struct ublk_tgt_ops fault_inject_tgt_ops;
void backing_file_tgt_deinit(struct ublk_dev *dev);
int backing_file_tgt_init(struct ublk_dev *dev);
-static inline unsigned int ilog2(unsigned int x)
-{
- if (x == 0)
- return 0;
- return (sizeof(x) * 8 - 1) - __builtin_clz(x);
-}
#endif
generated by cgit (git 2.34.1) at 2025-07-30 15:35:44 +0000