summaryrefslogtreecommitdiff
path: root/drivers/infiniband/sw/rxe/rxe_verbs.h
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/infiniband/sw/rxe/rxe_verbs.h')
-rw-r--r--drivers/infiniband/sw/rxe/rxe_verbs.h303
1 files changed, 174 insertions, 129 deletions
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index 74e04801d34d..fd48075810dd 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -1,34 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
* Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
*/
#ifndef RXE_VERBS_H
@@ -36,7 +9,6 @@
#include <linux/interrupt.h>
#include <linux/workqueue.h>
-#include <rdma/rdma_user_rxe.h>
#include "rxe_pool.h"
#include "rxe_task.h"
#include "rxe_hw_counters.h"
@@ -61,20 +33,21 @@ static inline int psn_compare(u32 psn_a, u32 psn_b)
}
struct rxe_ucontext {
- struct rxe_pool_entry pelem;
- struct ib_ucontext ibuc;
+ struct ib_ucontext ibuc;
+ struct rxe_pool_elem elem;
};
struct rxe_pd {
- struct rxe_pool_entry pelem;
- struct ib_pd ibpd;
+ struct ib_pd ibpd;
+ struct rxe_pool_elem elem;
};
struct rxe_ah {
- struct rxe_pool_entry pelem;
struct ib_ah ibah;
- struct rxe_pd *pd;
+ struct rxe_pool_elem elem;
struct rxe_av av;
+ bool is_user;
+ int ah_num;
};
struct rxe_cqe {
@@ -85,14 +58,13 @@ struct rxe_cqe {
};
struct rxe_cq {
- struct rxe_pool_entry pelem;
struct ib_cq ibcq;
+ struct rxe_pool_elem elem;
struct rxe_queue *queue;
spinlock_t cq_lock;
u8 notify;
- bool is_dying;
- int is_user;
- struct tasklet_struct comp_task;
+ bool is_user;
+ atomic_t num_wq;
};
enum wqe_state {
@@ -120,8 +92,8 @@ struct rxe_rq {
};
struct rxe_srq {
- struct rxe_pool_entry pelem;
struct ib_srq ibsrq;
+ struct rxe_pool_elem elem;
struct rxe_pd *pd;
struct rxe_rq rq;
u32 srq_num;
@@ -130,17 +102,7 @@ struct rxe_srq {
int error;
};
-enum rxe_qp_state {
- QP_STATE_RESET,
- QP_STATE_INIT,
- QP_STATE_READY,
- QP_STATE_DRAIN, /* req only */
- QP_STATE_DRAINED, /* req only */
- QP_STATE_ERROR
-};
-
struct rxe_req_info {
- enum rxe_qp_state state;
int wqe_index;
u32 psn;
int opcode;
@@ -149,8 +111,9 @@ struct rxe_req_info {
int need_rd_atomic;
int wait_psn;
int need_retry;
+ int wait_for_rnr_timer;
int noack_pkts;
- struct rxe_task task;
+ int again;
};
struct rxe_comp_info {
@@ -161,7 +124,43 @@ struct rxe_comp_info {
int started_retry;
u32 retry_cnt;
u32 rnr_retry;
- struct rxe_task task;
+};
+
+/* responder states */
+enum resp_states {
+ RESPST_NONE,
+ RESPST_GET_REQ,
+ RESPST_CHK_PSN,
+ RESPST_CHK_OP_SEQ,
+ RESPST_CHK_OP_VALID,
+ RESPST_CHK_RESOURCE,
+ RESPST_CHK_LENGTH,
+ RESPST_CHK_RKEY,
+ RESPST_EXECUTE,
+ RESPST_READ_REPLY,
+ RESPST_ATOMIC_REPLY,
+ RESPST_ATOMIC_WRITE_REPLY,
+ RESPST_PROCESS_FLUSH,
+ RESPST_COMPLETE,
+ RESPST_ACKNOWLEDGE,
+ RESPST_CLEANUP,
+ RESPST_DUPLICATE_REQUEST,
+ RESPST_ERR_MALFORMED_WQE,
+ RESPST_ERR_UNSUPPORTED_OPCODE,
+ RESPST_ERR_MISALIGNED_ATOMIC,
+ RESPST_ERR_PSN_OUT_OF_SEQ,
+ RESPST_ERR_MISSING_OPCODE_FIRST,
+ RESPST_ERR_MISSING_OPCODE_LAST_C,
+ RESPST_ERR_MISSING_OPCODE_LAST_D1E,
+ RESPST_ERR_TOO_MANY_RDMA_ATM_REQ,
+ RESPST_ERR_RNR,
+ RESPST_ERR_RKEY_VIOLATION,
+ RESPST_ERR_INVALIDATE_RKEY,
+ RESPST_ERR_LENGTH,
+ RESPST_ERR_CQ_OVERFLOW,
+ RESPST_ERROR,
+ RESPST_DONE,
+ RESPST_EXIT,
};
enum rdatm_res_state {
@@ -180,21 +179,25 @@ struct resp_res {
union {
struct {
- struct sk_buff *skb;
+ u64 orig_val;
} atomic;
struct {
- struct rxe_mem *mr;
u64 va_org;
u32 rkey;
u32 length;
u64 va;
u32 resid;
} read;
+ struct {
+ u32 length;
+ u64 va;
+ u8 type;
+ u8 level;
+ } flush;
};
};
struct rxe_resp_info {
- enum rxe_qp_state state;
u32 msn;
u32 psn;
u32 ack_psn;
@@ -210,10 +213,11 @@ struct rxe_resp_info {
/* RDMA read / atomic only */
u64 va;
- struct rxe_mem *mr;
+ u64 offset;
+ struct rxe_mr *mr;
u32 resid;
u32 rkey;
- u64 atomic_orig;
+ u32 length;
/* SRQ only */
struct {
@@ -228,16 +232,15 @@ struct rxe_resp_info {
unsigned int res_head;
unsigned int res_tail;
struct resp_res *res;
- struct rxe_task task;
};
struct rxe_qp {
- struct rxe_pool_entry pelem;
struct ib_qp ibqp;
+ struct rxe_pool_elem elem;
struct ib_qp_attr attr;
unsigned int valid;
unsigned int mtu;
- int is_user;
+ bool is_user;
struct rxe_pd *pd;
struct rxe_srq *srq;
@@ -256,13 +259,13 @@ struct rxe_qp {
struct rxe_av pri_av;
struct rxe_av alt_av;
- /* list of mcast groups qp has joined (for cleanup) */
- struct list_head grp_list;
- spinlock_t grp_lock; /* guard grp_list */
+ atomic_t mcg_num;
struct sk_buff_head req_pkts;
struct sk_buff_head resp_pkts;
- struct sk_buff_head send_pkts;
+
+ struct rxe_task send_task;
+ struct rxe_task recv_task;
struct rxe_req_info req;
struct rxe_comp_info comp;
@@ -288,109 +291,131 @@ struct rxe_qp {
struct execute_work cleanup_work;
};
-enum rxe_mem_state {
- RXE_MEM_STATE_ZOMBIE,
- RXE_MEM_STATE_INVALID,
- RXE_MEM_STATE_FREE,
- RXE_MEM_STATE_VALID,
+enum {
+ RXE_ACCESS_REMOTE = IB_ACCESS_REMOTE_READ
+ | IB_ACCESS_REMOTE_WRITE
+ | IB_ACCESS_REMOTE_ATOMIC,
+ RXE_ACCESS_SUPPORTED_MR = RXE_ACCESS_REMOTE
+ | IB_ACCESS_LOCAL_WRITE
+ | IB_ACCESS_MW_BIND
+ | IB_ACCESS_ON_DEMAND
+ | IB_ACCESS_FLUSH_GLOBAL
+ | IB_ACCESS_FLUSH_PERSISTENT
+ | IB_ACCESS_OPTIONAL,
+ RXE_ACCESS_SUPPORTED_QP = RXE_ACCESS_SUPPORTED_MR,
+ RXE_ACCESS_SUPPORTED_MW = RXE_ACCESS_SUPPORTED_MR
+ | IB_ZERO_BASED,
};
-enum rxe_mem_type {
- RXE_MEM_TYPE_NONE,
- RXE_MEM_TYPE_DMA,
- RXE_MEM_TYPE_MR,
- RXE_MEM_TYPE_FMR,
- RXE_MEM_TYPE_MW,
+enum rxe_mr_state {
+ RXE_MR_STATE_INVALID,
+ RXE_MR_STATE_FREE,
+ RXE_MR_STATE_VALID,
};
-#define RXE_BUF_PER_MAP (PAGE_SIZE / sizeof(struct rxe_phys_buf))
+enum rxe_mr_copy_dir {
+ RXE_TO_MR_OBJ,
+ RXE_FROM_MR_OBJ,
+};
-struct rxe_phys_buf {
- u64 addr;
- u64 size;
+enum rxe_mr_lookup_type {
+ RXE_LOOKUP_LOCAL,
+ RXE_LOOKUP_REMOTE,
};
-struct rxe_map {
- struct rxe_phys_buf buf[RXE_BUF_PER_MAP];
+enum rxe_rereg {
+ RXE_MR_REREG_SUPPORTED = IB_MR_REREG_PD
+ | IB_MR_REREG_ACCESS,
};
-struct rxe_mem {
- struct rxe_pool_entry pelem;
- union {
- struct ib_mr ibmr;
- struct ib_mw ibmw;
- };
+static inline int rkey_is_mw(u32 rkey)
+{
+ u32 index = rkey >> 8;
+
+ return (index >= RXE_MIN_MW_INDEX) && (index <= RXE_MAX_MW_INDEX);
+}
+
+struct rxe_mr {
+ struct rxe_pool_elem elem;
+ struct ib_mr ibmr;
- struct rxe_pd *pd;
struct ib_umem *umem;
u32 lkey;
u32 rkey;
-
- enum rxe_mem_state state;
- enum rxe_mem_type type;
- u64 va;
- u64 iova;
- size_t length;
- u32 offset;
+ enum rxe_mr_state state;
int access;
+ atomic_t num_mw;
- int page_shift;
- int page_mask;
- int map_shift;
- int map_mask;
+ unsigned int page_offset;
+ unsigned int page_shift;
+ u64 page_mask;
u32 num_buf;
u32 nbuf;
- u32 max_buf;
- u32 num_map;
+ struct xarray page_list;
+};
+
+static inline unsigned int mr_page_size(struct rxe_mr *mr)
+{
+ return mr ? mr->ibmr.page_size : PAGE_SIZE;
+}
+
+enum rxe_mw_state {
+ RXE_MW_STATE_INVALID = RXE_MR_STATE_INVALID,
+ RXE_MW_STATE_FREE = RXE_MR_STATE_FREE,
+ RXE_MW_STATE_VALID = RXE_MR_STATE_VALID,
+};
- struct rxe_map **map;
+struct rxe_mw {
+ struct ib_mw ibmw;
+ struct rxe_pool_elem elem;
+ spinlock_t lock;
+ enum rxe_mw_state state;
+ struct rxe_qp *qp; /* Type 2 only */
+ struct rxe_mr *mr;
+ u32 rkey;
+ int access;
+ u64 addr;
+ u64 length;
};
-struct rxe_mc_grp {
- struct rxe_pool_entry pelem;
- spinlock_t mcg_lock; /* guard group */
+struct rxe_mcg {
+ struct rb_node node;
+ struct kref ref_cnt;
struct rxe_dev *rxe;
struct list_head qp_list;
union ib_gid mgid;
- int num_qp;
+ atomic_t qp_num;
u32 qkey;
u16 pkey;
};
-struct rxe_mc_elem {
- struct rxe_pool_entry pelem;
+struct rxe_mca {
struct list_head qp_list;
- struct list_head grp_list;
struct rxe_qp *qp;
- struct rxe_mc_grp *grp;
};
struct rxe_port {
struct ib_port_attr attr;
- u16 *pkey_tbl;
__be64 port_guid;
__be64 subnet_prefix;
spinlock_t port_lock; /* guard port */
unsigned int mtu_cap;
/* special QPs */
- u32 qp_smi_index;
u32 qp_gsi_index;
};
+#define RXE_PORT 1
struct rxe_dev {
struct ib_device ib_dev;
struct ib_device_attr attr;
int max_ucontext;
int max_inline_data;
- struct kref ref_cnt;
- struct mutex usdev_lock;
-
- struct net_device *ndev;
+ struct mutex usdev_lock;
- int xmit_errors;
+ char raw_gid[ETH_ALEN];
struct rxe_pool uc_pool;
struct rxe_pool pd_pool;
@@ -400,22 +425,29 @@ struct rxe_dev {
struct rxe_pool cq_pool;
struct rxe_pool mr_pool;
struct rxe_pool mw_pool;
- struct rxe_pool mc_grp_pool;
- struct rxe_pool mc_elem_pool;
+
+ /* multicast support */
+ spinlock_t mcg_lock;
+ struct rb_root mcg_tree;
+ atomic_t mcg_num;
+ atomic_t mcg_attach;
spinlock_t pending_lock; /* guard pending_mmaps */
struct list_head pending_mmaps;
spinlock_t mmap_offset_lock; /* guard mmap_offset */
- int mmap_offset;
+ u64 mmap_offset;
atomic64_t stats_counters[RXE_NUM_OF_COUNTERS];
struct rxe_port port;
- struct list_head list;
- struct crypto_shash *tfm;
};
+static inline struct net_device *rxe_ib_device_get_netdev(struct ib_device *dev)
+{
+ return ib_device_get_netdev(dev, RXE_PORT);
+}
+
static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)
{
atomic64_inc(&rxe->stats_counters[index]);
@@ -456,19 +488,32 @@ static inline struct rxe_cq *to_rcq(struct ib_cq *cq)
return cq ? container_of(cq, struct rxe_cq, ibcq) : NULL;
}
-static inline struct rxe_mem *to_rmr(struct ib_mr *mr)
+static inline struct rxe_mr *to_rmr(struct ib_mr *mr)
+{
+ return mr ? container_of(mr, struct rxe_mr, ibmr) : NULL;
+}
+
+static inline struct rxe_mw *to_rmw(struct ib_mw *mw)
{
- return mr ? container_of(mr, struct rxe_mem, ibmr) : NULL;
+ return mw ? container_of(mw, struct rxe_mw, ibmw) : NULL;
}
-static inline struct rxe_mem *to_rmw(struct ib_mw *mw)
+static inline struct rxe_pd *rxe_ah_pd(struct rxe_ah *ah)
{
- return mw ? container_of(mw, struct rxe_mem, ibmw) : NULL;
+ return to_rpd(ah->ibah.pd);
}
-int rxe_register_device(struct rxe_dev *rxe);
-void rxe_unregister_device(struct rxe_dev *rxe);
+static inline struct rxe_pd *mr_pd(struct rxe_mr *mr)
+{
+ return to_rpd(mr->ibmr.pd);
+}
+
+static inline struct rxe_pd *rxe_mw_pd(struct rxe_mw *mw)
+{
+ return to_rpd(mw->ibmw.pd);
+}
-void rxe_mc_cleanup(struct rxe_pool_entry *arg);
+int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name,
+ struct net_device *ndev);
#endif /* RXE_VERBS_H */
generated by cgit (git 2.34.1) at 2025-12-26 00:14:18 +0000