summaryrefslogtreecommitdiff
path: root/drivers/infiniband/sw/rxe/rxe_verbs.h
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/infiniband/sw/rxe/rxe_verbs.h')
-rw-r--r--drivers/infiniband/sw/rxe/rxe_verbs.h220
1 files changed, 119 insertions, 101 deletions
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index 959a3260fcab..fd48075810dd 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -9,7 +9,6 @@
#include <linux/interrupt.h>
#include <linux/workqueue.h>
-#include <rdma/rdma_user_rxe.h>
#include "rxe_pool.h"
#include "rxe_task.h"
#include "rxe_hw_counters.h"
@@ -35,19 +34,20 @@ static inline int psn_compare(u32 psn_a, u32 psn_b)
struct rxe_ucontext {
struct ib_ucontext ibuc;
- struct rxe_pool_entry pelem;
+ struct rxe_pool_elem elem;
};
struct rxe_pd {
struct ib_pd ibpd;
- struct rxe_pool_entry pelem;
+ struct rxe_pool_elem elem;
};
struct rxe_ah {
struct ib_ah ibah;
- struct rxe_pool_entry pelem;
- struct rxe_pd *pd;
+ struct rxe_pool_elem elem;
struct rxe_av av;
+ bool is_user;
+ int ah_num;
};
struct rxe_cqe {
@@ -59,13 +59,12 @@ struct rxe_cqe {
struct rxe_cq {
struct ib_cq ibcq;
- struct rxe_pool_entry pelem;
+ struct rxe_pool_elem elem;
struct rxe_queue *queue;
spinlock_t cq_lock;
u8 notify;
- bool is_dying;
- int is_user;
- struct tasklet_struct comp_task;
+ bool is_user;
+ atomic_t num_wq;
};
enum wqe_state {
@@ -77,7 +76,6 @@ enum wqe_state {
};
struct rxe_sq {
- bool is_user;
int max_wr;
int max_sge;
int max_inline;
@@ -86,7 +84,6 @@ struct rxe_sq {
};
struct rxe_rq {
- bool is_user;
int max_wr;
int max_sge;
spinlock_t producer_lock; /* guard queue producer */
@@ -96,27 +93,16 @@ struct rxe_rq {
struct rxe_srq {
struct ib_srq ibsrq;
- struct rxe_pool_entry pelem;
+ struct rxe_pool_elem elem;
struct rxe_pd *pd;
struct rxe_rq rq;
u32 srq_num;
- bool is_user;
int limit;
int error;
};
-enum rxe_qp_state {
- QP_STATE_RESET,
- QP_STATE_INIT,
- QP_STATE_READY,
- QP_STATE_DRAIN, /* req only */
- QP_STATE_DRAINED, /* req only */
- QP_STATE_ERROR
-};
-
struct rxe_req_info {
- enum rxe_qp_state state;
int wqe_index;
u32 psn;
int opcode;
@@ -125,8 +111,9 @@ struct rxe_req_info {
int need_rd_atomic;
int wait_psn;
int need_retry;
+ int wait_for_rnr_timer;
int noack_pkts;
- struct rxe_task task;
+ int again;
};
struct rxe_comp_info {
@@ -137,7 +124,43 @@ struct rxe_comp_info {
int started_retry;
u32 retry_cnt;
u32 rnr_retry;
- struct rxe_task task;
+};
+
+/* responder states */
+enum resp_states {
+ RESPST_NONE,
+ RESPST_GET_REQ,
+ RESPST_CHK_PSN,
+ RESPST_CHK_OP_SEQ,
+ RESPST_CHK_OP_VALID,
+ RESPST_CHK_RESOURCE,
+ RESPST_CHK_LENGTH,
+ RESPST_CHK_RKEY,
+ RESPST_EXECUTE,
+ RESPST_READ_REPLY,
+ RESPST_ATOMIC_REPLY,
+ RESPST_ATOMIC_WRITE_REPLY,
+ RESPST_PROCESS_FLUSH,
+ RESPST_COMPLETE,
+ RESPST_ACKNOWLEDGE,
+ RESPST_CLEANUP,
+ RESPST_DUPLICATE_REQUEST,
+ RESPST_ERR_MALFORMED_WQE,
+ RESPST_ERR_UNSUPPORTED_OPCODE,
+ RESPST_ERR_MISALIGNED_ATOMIC,
+ RESPST_ERR_PSN_OUT_OF_SEQ,
+ RESPST_ERR_MISSING_OPCODE_FIRST,
+ RESPST_ERR_MISSING_OPCODE_LAST_C,
+ RESPST_ERR_MISSING_OPCODE_LAST_D1E,
+ RESPST_ERR_TOO_MANY_RDMA_ATM_REQ,
+ RESPST_ERR_RNR,
+ RESPST_ERR_RKEY_VIOLATION,
+ RESPST_ERR_INVALIDATE_RKEY,
+ RESPST_ERR_LENGTH,
+ RESPST_ERR_CQ_OVERFLOW,
+ RESPST_ERROR,
+ RESPST_DONE,
+ RESPST_EXIT,
};
enum rdatm_res_state {
@@ -156,21 +179,25 @@ struct resp_res {
union {
struct {
- struct sk_buff *skb;
+ u64 orig_val;
} atomic;
struct {
- struct rxe_mr *mr;
u64 va_org;
u32 rkey;
u32 length;
u64 va;
u32 resid;
} read;
+ struct {
+ u32 length;
+ u64 va;
+ u8 type;
+ u8 level;
+ } flush;
};
};
struct rxe_resp_info {
- enum rxe_qp_state state;
u32 msn;
u32 psn;
u32 ack_psn;
@@ -191,7 +218,6 @@ struct rxe_resp_info {
u32 resid;
u32 rkey;
u32 length;
- u64 atomic_orig;
/* SRQ only */
struct {
@@ -206,12 +232,11 @@ struct rxe_resp_info {
unsigned int res_head;
unsigned int res_tail;
struct resp_res *res;
- struct rxe_task task;
};
struct rxe_qp {
- struct rxe_pool_entry pelem;
struct ib_qp ibqp;
+ struct rxe_pool_elem elem;
struct ib_qp_attr attr;
unsigned int valid;
unsigned int mtu;
@@ -234,13 +259,13 @@ struct rxe_qp {
struct rxe_av pri_av;
struct rxe_av alt_av;
- /* list of mcast groups qp has joined (for cleanup) */
- struct list_head grp_list;
- spinlock_t grp_lock; /* guard grp_list */
+ atomic_t mcg_num;
struct sk_buff_head req_pkts;
struct sk_buff_head resp_pkts;
- struct sk_buff_head send_pkts;
+
+ struct rxe_task send_task;
+ struct rxe_task recv_task;
struct rxe_req_info req;
struct rxe_comp_info comp;
@@ -266,19 +291,28 @@ struct rxe_qp {
struct execute_work cleanup_work;
};
+enum {
+ RXE_ACCESS_REMOTE = IB_ACCESS_REMOTE_READ
+ | IB_ACCESS_REMOTE_WRITE
+ | IB_ACCESS_REMOTE_ATOMIC,
+ RXE_ACCESS_SUPPORTED_MR = RXE_ACCESS_REMOTE
+ | IB_ACCESS_LOCAL_WRITE
+ | IB_ACCESS_MW_BIND
+ | IB_ACCESS_ON_DEMAND
+ | IB_ACCESS_FLUSH_GLOBAL
+ | IB_ACCESS_FLUSH_PERSISTENT
+ | IB_ACCESS_OPTIONAL,
+ RXE_ACCESS_SUPPORTED_QP = RXE_ACCESS_SUPPORTED_MR,
+ RXE_ACCESS_SUPPORTED_MW = RXE_ACCESS_SUPPORTED_MR
+ | IB_ZERO_BASED,
+};
+
enum rxe_mr_state {
- RXE_MR_STATE_ZOMBIE,
RXE_MR_STATE_INVALID,
RXE_MR_STATE_FREE,
RXE_MR_STATE_VALID,
};
-enum rxe_mr_type {
- RXE_MR_TYPE_NONE,
- RXE_MR_TYPE_DMA,
- RXE_MR_TYPE_MR,
-};
-
enum rxe_mr_copy_dir {
RXE_TO_MR_OBJ,
RXE_FROM_MR_OBJ,
@@ -289,15 +323,9 @@ enum rxe_mr_lookup_type {
RXE_LOOKUP_REMOTE,
};
-#define RXE_BUF_PER_MAP (PAGE_SIZE / sizeof(struct rxe_phys_buf))
-
-struct rxe_phys_buf {
- u64 addr;
- u64 size;
-};
-
-struct rxe_map {
- struct rxe_phys_buf buf[RXE_BUF_PER_MAP];
+enum rxe_rereg {
+ RXE_MR_REREG_SUPPORTED = IB_MR_REREG_PD
+ | IB_MR_REREG_ACCESS,
};
static inline int rkey_is_mw(u32 rkey)
@@ -308,35 +336,32 @@ static inline int rkey_is_mw(u32 rkey)
}
struct rxe_mr {
- struct rxe_pool_entry pelem;
+ struct rxe_pool_elem elem;
struct ib_mr ibmr;
struct ib_umem *umem;
+ u32 lkey;
+ u32 rkey;
enum rxe_mr_state state;
- enum rxe_mr_type type;
- u64 va;
- u64 iova;
- size_t length;
- u32 offset;
int access;
+ atomic_t num_mw;
- int page_shift;
- int page_mask;
- int map_shift;
- int map_mask;
+ unsigned int page_offset;
+ unsigned int page_shift;
+ u64 page_mask;
u32 num_buf;
u32 nbuf;
- u32 max_buf;
- u32 num_map;
-
- atomic_t num_mw;
-
- struct rxe_map **map;
+ struct xarray page_list;
};
+static inline unsigned int mr_page_size(struct rxe_mr *mr)
+{
+ return mr ? mr->ibmr.page_size : PAGE_SIZE;
+}
+
enum rxe_mw_state {
RXE_MW_STATE_INVALID = RXE_MR_STATE_INVALID,
RXE_MW_STATE_FREE = RXE_MR_STATE_FREE,
@@ -345,33 +370,31 @@ enum rxe_mw_state {
struct rxe_mw {
struct ib_mw ibmw;
- struct rxe_pool_entry pelem;
+ struct rxe_pool_elem elem;
spinlock_t lock;
enum rxe_mw_state state;
struct rxe_qp *qp; /* Type 2 only */
struct rxe_mr *mr;
+ u32 rkey;
int access;
u64 addr;
u64 length;
};
-struct rxe_mc_grp {
- struct rxe_pool_entry pelem;
- spinlock_t mcg_lock; /* guard group */
+struct rxe_mcg {
+ struct rb_node node;
+ struct kref ref_cnt;
struct rxe_dev *rxe;
struct list_head qp_list;
union ib_gid mgid;
- int num_qp;
+ atomic_t qp_num;
u32 qkey;
u16 pkey;
};
-struct rxe_mc_elem {
- struct rxe_pool_entry pelem;
+struct rxe_mca {
struct list_head qp_list;
- struct list_head grp_list;
struct rxe_qp *qp;
- struct rxe_mc_grp *grp;
};
struct rxe_port {
@@ -381,20 +404,18 @@ struct rxe_port {
spinlock_t port_lock; /* guard port */
unsigned int mtu_cap;
/* special QPs */
- u32 qp_smi_index;
u32 qp_gsi_index;
};
+#define RXE_PORT 1
struct rxe_dev {
struct ib_device ib_dev;
struct ib_device_attr attr;
int max_ucontext;
int max_inline_data;
- struct mutex usdev_lock;
-
- struct net_device *ndev;
+ struct mutex usdev_lock;
- int xmit_errors;
+ char raw_gid[ETH_ALEN];
struct rxe_pool uc_pool;
struct rxe_pool pd_pool;
@@ -404,8 +425,12 @@ struct rxe_dev {
struct rxe_pool cq_pool;
struct rxe_pool mr_pool;
struct rxe_pool mw_pool;
- struct rxe_pool mc_grp_pool;
- struct rxe_pool mc_elem_pool;
+
+ /* multicast support */
+ spinlock_t mcg_lock;
+ struct rb_root mcg_tree;
+ atomic_t mcg_num;
+ atomic_t mcg_attach;
spinlock_t pending_lock; /* guard pending_mmaps */
struct list_head pending_mmaps;
@@ -416,9 +441,13 @@ struct rxe_dev {
atomic64_t stats_counters[RXE_NUM_OF_COUNTERS];
struct rxe_port port;
- struct crypto_shash *tfm;
};
+static inline struct net_device *rxe_ib_device_get_netdev(struct ib_device *dev)
+{
+ return ib_device_get_netdev(dev, RXE_PORT);
+}
+
static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)
{
atomic64_inc(&rxe->stats_counters[index]);
@@ -469,19 +498,14 @@ static inline struct rxe_mw *to_rmw(struct ib_mw *mw)
return mw ? container_of(mw, struct rxe_mw, ibmw) : NULL;
}
-static inline struct rxe_pd *mr_pd(struct rxe_mr *mr)
-{
- return to_rpd(mr->ibmr.pd);
-}
-
-static inline u32 mr_lkey(struct rxe_mr *mr)
+static inline struct rxe_pd *rxe_ah_pd(struct rxe_ah *ah)
{
- return mr->ibmr.lkey;
+ return to_rpd(ah->ibah.pd);
}
-static inline u32 mr_rkey(struct rxe_mr *mr)
+static inline struct rxe_pd *mr_pd(struct rxe_mr *mr)
{
- return mr->ibmr.rkey;
+ return to_rpd(mr->ibmr.pd);
}
static inline struct rxe_pd *rxe_mw_pd(struct rxe_mw *mw)
@@ -489,13 +513,7 @@ static inline struct rxe_pd *rxe_mw_pd(struct rxe_mw *mw)
return to_rpd(mw->ibmw.pd);
}
-static inline u32 rxe_mw_rkey(struct rxe_mw *mw)
-{
- return mw->ibmw.rkey;
-}
-
-int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name);
-
-void rxe_mc_cleanup(struct rxe_pool_entry *arg);
+int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name,
+ struct net_device *ndev);
#endif /* RXE_VERBS_H */
generated by cgit (git 2.34.1) at 2025-12-25 23:20:26 +0000