summaryrefslogtreecommitdiff
path: root/drivers
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2023-04-29 17:21:24 -0700
committerLinus Torvalds <torvalds@linux-foundation.org>2023-04-29 17:21:24 -0700
commitaf3877265dd88d7e333f94fb37bc09554544adca (patch)
tree3f84b1a5d1e052039f510cece55f1c06e580ea8b /drivers
parent1ae78a14516b9372e4c90a89ac21b259339a3a3a (diff)
parent531094dc7164718d28ebb581d729807d7e846363 (diff)
Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma
Pull rdma updates from Jason Gunthorpe: "Usual wide collection of unrelated items in drivers: - Driver bug fixes and treewide cleanups in hfi1, siw, qib, mlx5, rxe, usnic, usnic, bnxt_re, ocrdma, iser: - remove unnecessary NULL checks - kmap obsolescence - pci_enable_pcie_error_reporting() obsolescence - unused variables and macros - trace event related warnings - casting warnings - Code cleanups for irdm and erdma - EFA reporting of 128 byte PCIe TLP support - mlx5 more agressively uses the out of order HW feature - Big rework of how state machines and tasks work in rxe - Fix a syzkaller found crash netdev refcount leak in siw - bnxt_re revises their HW description header - Congestion control for bnxt_re - Use mmu_notifiers more safely in hfi1 - mlx5 gets better support for PCIe relaxed ordering inside VMs" * tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma: (81 commits) RDMA/efa: Add rdma write capability to device caps RDMA/mlx5: Use correct device num_ports when modify DC RDMA/irdma: Drop spurious WQ_UNBOUND from alloc_ordered_workqueue() call RDMA/rxe: Fix spinlock recursion deadlock on requester RDMA/mlx5: Fix flow counter query via DEVX RDMA/rxe: Protect QP state with qp->state_lock RDMA/rxe: Move code to check if drained to subroutine RDMA/rxe: Remove qp->req.state RDMA/rxe: Remove qp->comp.state RDMA/rxe: Remove qp->resp.state RDMA/mlx5: Allow relaxed ordering read in VFs and VMs net/mlx5: Update relaxed ordering read HCA capabilities RDMA/mlx5: Check pcie_relaxed_ordering_enabled() in UMR RDMA/mlx5: Remove pcie_relaxed_ordering_enabled() check for RO write RDMA: Add ib_virt_dma_to_page() RDMA/rxe: Fix the error "trying to register non-static key in rxe_cleanup_task" RDMA/irdma: Slightly optimize irdma_form_ah_cm_frame() RDMA/rxe: Fix incorrect TASKLET_STATE_SCHED check in rxe_task.c IB/hfi1: Place struct mmu_rb_handler on cache line start IB/hfi1: Fix bugs with non-PAGE_SIZE-end multi-iovec user SDMA requests ...
Diffstat (limited to 'drivers')
-rw-r--r--drivers/infiniband/core/cm.c3
-rw-r--r--drivers/infiniband/core/cma.c6
-rw-r--r--drivers/infiniband/core/user_mad.c23
-rw-r--r--drivers/infiniband/hw/bnxt_re/ib_verbs.c109
-rw-r--r--drivers/infiniband/hw/bnxt_re/ib_verbs.h3
-rw-r--r--drivers/infiniband/hw/bnxt_re/main.c103
-rw-r--r--drivers/infiniband/hw/bnxt_re/qplib_fp.c211
-rw-r--r--drivers/infiniband/hw/bnxt_re/qplib_fp.h5
-rw-r--r--drivers/infiniband/hw/bnxt_re/qplib_rcfw.c97
-rw-r--r--drivers/infiniband/hw/bnxt_re/qplib_rcfw.h66
-rw-r--r--drivers/infiniband/hw/bnxt_re/qplib_sp.c337
-rw-r--r--drivers/infiniband/hw/bnxt_re/qplib_sp.h68
-rw-r--r--drivers/infiniband/hw/bnxt_re/qplib_tlv.h162
-rw-r--r--drivers/infiniband/hw/bnxt_re/roce_hsi.h7183
-rw-r--r--drivers/infiniband/hw/efa/efa_admin_cmds_defs.h17
-rw-r--r--drivers/infiniband/hw/efa/efa_io_defs.h42
-rw-r--r--drivers/infiniband/hw/efa/efa_verbs.c11
-rw-r--r--drivers/infiniband/hw/erdma/erdma.h2
-rw-r--r--drivers/infiniband/hw/erdma/erdma_cm.h10
-rw-r--r--drivers/infiniband/hw/erdma/erdma_cmdq.c42
-rw-r--r--drivers/infiniband/hw/erdma/erdma_cq.c2
-rw-r--r--drivers/infiniband/hw/erdma/erdma_eq.c9
-rw-r--r--drivers/infiniband/hw/erdma/erdma_hw.h4
-rw-r--r--drivers/infiniband/hw/erdma/erdma_main.c39
-rw-r--r--drivers/infiniband/hw/erdma/erdma_verbs.c17
-rw-r--r--drivers/infiniband/hw/hfi1/chip.c18
-rw-r--r--drivers/infiniband/hw/hfi1/driver.c2
-rw-r--r--drivers/infiniband/hw/hfi1/file_ops.c2
-rw-r--r--drivers/infiniband/hw/hfi1/init.c12
-rw-r--r--drivers/infiniband/hw/hfi1/ipoib_tx.c6
-rw-r--r--drivers/infiniband/hw/hfi1/mmu_rb.c84
-rw-r--r--drivers/infiniband/hw/hfi1/mmu_rb.h22
-rw-r--r--drivers/infiniband/hw/hfi1/pcie.c2
-rw-r--r--drivers/infiniband/hw/hfi1/pio.c2
-rw-r--r--drivers/infiniband/hw/hfi1/sdma.c21
-rw-r--r--drivers/infiniband/hw/hfi1/sdma.h16
-rw-r--r--drivers/infiniband/hw/hfi1/sdma_txreq.h1
-rw-r--r--drivers/infiniband/hw/hfi1/trace_dbg.h7
-rw-r--r--drivers/infiniband/hw/hfi1/trace_mmu.h4
-rw-r--r--drivers/infiniband/hw/hfi1/user_sdma.c600
-rw-r--r--drivers/infiniband/hw/hfi1/user_sdma.h5
-rw-r--r--drivers/infiniband/hw/hfi1/verbs.c4
-rw-r--r--drivers/infiniband/hw/hfi1/vnic_sdma.c1
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_hw_v2.c189
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_hw_v2.h31
-rw-r--r--drivers/infiniband/hw/irdma/cm.c2
-rw-r--r--drivers/infiniband/hw/irdma/ctrl.c324
-rw-r--r--drivers/infiniband/hw/irdma/defs.h9
-rw-r--r--drivers/infiniband/hw/irdma/hw.c17
-rw-r--r--drivers/infiniband/hw/irdma/i40iw_hw.c60
-rw-r--r--drivers/infiniband/hw/irdma/icrdma_hw.c51
-rw-r--r--drivers/infiniband/hw/irdma/irdma.h1
-rw-r--r--drivers/infiniband/hw/irdma/main.h3
-rw-r--r--drivers/infiniband/hw/irdma/pble.c16
-rw-r--r--drivers/infiniband/hw/irdma/pble.h2
-rw-r--r--drivers/infiniband/hw/irdma/protos.h8
-rw-r--r--drivers/infiniband/hw/irdma/type.h166
-rw-r--r--drivers/infiniband/hw/irdma/utils.c172
-rw-r--r--drivers/infiniband/hw/irdma/verbs.c200
-rw-r--r--drivers/infiniband/hw/mlx4/qp.c8
-rw-r--r--drivers/infiniband/hw/mlx5/counters.c171
-rw-r--r--drivers/infiniband/hw/mlx5/devx.c33
-rw-r--r--drivers/infiniband/hw/mlx5/mr.c12
-rw-r--r--drivers/infiniband/hw/mlx5/qp.c19
-rw-r--r--drivers/infiniband/hw/mlx5/umr.c7
-rw-r--r--drivers/infiniband/hw/mlx5/umr.h3
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_verbs.c2
-rw-r--r--drivers/infiniband/hw/qib/qib_file_ops.c9
-rw-r--r--drivers/infiniband/hw/qib/qib_pcie.c8
-rw-r--r--drivers/infiniband/hw/qib/qib_user_sdma.c11
-rw-r--r--drivers/infiniband/hw/usnic/usnic_ib_main.c2
-rw-r--r--drivers/infiniband/sw/rdmavt/qp.c4
-rw-r--r--drivers/infiniband/sw/rxe/rxe.c16
-rw-r--r--drivers/infiniband/sw/rxe/rxe.h46
-rw-r--r--drivers/infiniband/sw/rxe/rxe_comp.c161
-rw-r--r--drivers/infiniband/sw/rxe/rxe_cq.c39
-rw-r--r--drivers/infiniband/sw/rxe/rxe_icrc.c4
-rw-r--r--drivers/infiniband/sw/rxe/rxe_loc.h7
-rw-r--r--drivers/infiniband/sw/rxe/rxe_mmap.c6
-rw-r--r--drivers/infiniband/sw/rxe/rxe_mr.c29
-rw-r--r--drivers/infiniband/sw/rxe/rxe_net.c11
-rw-r--r--drivers/infiniband/sw/rxe/rxe_qp.c263
-rw-r--r--drivers/infiniband/sw/rxe/rxe_queue.c5
-rw-r--r--drivers/infiniband/sw/rxe/rxe_recv.c15
-rw-r--r--drivers/infiniband/sw/rxe/rxe_req.c104
-rw-r--r--drivers/infiniband/sw/rxe/rxe_resp.c126
-rw-r--r--drivers/infiniband/sw/rxe/rxe_srq.c6
-rw-r--r--drivers/infiniband/sw/rxe/rxe_task.c268
-rw-r--r--drivers/infiniband/sw/rxe/rxe_task.h23
-rw-r--r--drivers/infiniband/sw/rxe/rxe_verbs.c981
-rw-r--r--drivers/infiniband/sw/rxe/rxe_verbs.h14
-rw-r--r--drivers/infiniband/sw/siw/siw_main.c3
-rw-r--r--drivers/infiniband/sw/siw/siw_qp_rx.c6
-rw-r--r--drivers/infiniband/sw/siw/siw_qp_tx.c21
-rw-r--r--drivers/infiniband/sw/siw/siw_verbs.c4
-rw-r--r--drivers/infiniband/ulp/iser/iser_initiator.c17
-rw-r--r--drivers/infiniband/ulp/iser/iser_verbs.c6
-rw-r--r--drivers/infiniband/ulp/srpt/ib_srpt.c23
-rw-r--r--drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h81
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/params.c3
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_common.c9
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/main.c3
102 files changed, 7832 insertions, 5388 deletions
diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c
index 603c0aecc361..ff58058aeadc 100644
--- a/drivers/infiniband/core/cm.c
+++ b/drivers/infiniband/core/cm.c
@@ -2912,6 +2912,8 @@ static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
(ari && ari_length > IB_CM_REJ_ARI_LENGTH))
return -EINVAL;
+ trace_icm_send_rej(&cm_id_priv->id, reason);
+
switch (state) {
case IB_CM_REQ_SENT:
case IB_CM_MRA_REQ_RCVD:
@@ -2942,7 +2944,6 @@ static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
return -EINVAL;
}
- trace_icm_send_rej(&cm_id_priv->id, reason);
ret = ib_post_send_mad(msg, NULL);
if (ret) {
cm_free_msg(msg);
diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 6b9563d4f23c..93a1c48d0c32 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -709,8 +709,7 @@ cma_validate_port(struct ib_device *device, u32 port,
}
sgid_attr = rdma_find_gid_by_port(device, gid, gid_type, port, ndev);
- if (ndev)
- dev_put(ndev);
+ dev_put(ndev);
return sgid_attr;
}
@@ -2429,8 +2428,7 @@ err_unlock:
mutex_unlock(&listen_id->handler_mutex);
net_dev_put:
- if (net_dev)
- dev_put(net_dev);
+ dev_put(net_dev);
return ret;
}
diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c
index 0e9e04f8c685..7e5c33aad161 100644
--- a/drivers/infiniband/core/user_mad.c
+++ b/drivers/infiniband/core/user_mad.c
@@ -131,6 +131,11 @@ struct ib_umad_packet {
struct ib_user_mad mad;
};
+struct ib_rmpp_mad_hdr {
+ struct ib_mad_hdr mad_hdr;
+ struct ib_rmpp_hdr rmpp_hdr;
+} __packed;
+
#define CREATE_TRACE_POINTS
#include <trace/events/ib_umad.h>
@@ -494,11 +499,11 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf,
size_t count, loff_t *pos)
{
struct ib_umad_file *file = filp->private_data;
+ struct ib_rmpp_mad_hdr *rmpp_mad_hdr;
struct ib_umad_packet *packet;
struct ib_mad_agent *agent;
struct rdma_ah_attr ah_attr;
struct ib_ah *ah;
- struct ib_rmpp_mad *rmpp_mad;
__be64 *tid;
int ret, data_len, hdr_len, copy_offset, rmpp_active;
u8 base_version;
@@ -506,7 +511,7 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf,
if (count < hdr_size(file) + IB_MGMT_RMPP_HDR)
return -EINVAL;
- packet = kzalloc(sizeof *packet + IB_MGMT_RMPP_HDR, GFP_KERNEL);
+ packet = kzalloc(sizeof(*packet) + IB_MGMT_RMPP_HDR, GFP_KERNEL);
if (!packet)
return -ENOMEM;
@@ -560,13 +565,13 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf,
goto err_up;
}
- rmpp_mad = (struct ib_rmpp_mad *) packet->mad.data;
- hdr_len = ib_get_mad_data_offset(rmpp_mad->mad_hdr.mgmt_class);
+ rmpp_mad_hdr = (struct ib_rmpp_mad_hdr *)packet->mad.data;
+ hdr_len = ib_get_mad_data_offset(rmpp_mad_hdr->mad_hdr.mgmt_class);
- if (ib_is_mad_class_rmpp(rmpp_mad->mad_hdr.mgmt_class)
+ if (ib_is_mad_class_rmpp(rmpp_mad_hdr->mad_hdr.mgmt_class)
&& ib_mad_kernel_rmpp_agent(agent)) {
copy_offset = IB_MGMT_RMPP_HDR;
- rmpp_active = ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) &
+ rmpp_active = ib_get_rmpp_flags(&rmpp_mad_hdr->rmpp_hdr) &
IB_MGMT_RMPP_FLAG_ACTIVE;
} else {
copy_offset = IB_MGMT_MAD_HDR;
@@ -615,12 +620,12 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf,
tid = &((struct ib_mad_hdr *) packet->msg->mad)->tid;
*tid = cpu_to_be64(((u64) agent->hi_tid) << 32 |
(be64_to_cpup(tid) & 0xffffffff));
- rmpp_mad->mad_hdr.tid = *tid;
+ rmpp_mad_hdr->mad_hdr.tid = *tid;
}
if (!ib_mad_kernel_rmpp_agent(agent)
- && ib_is_mad_class_rmpp(rmpp_mad->mad_hdr.mgmt_class)
- && (ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) & IB_MGMT_RMPP_FLAG_ACTIVE)) {
+ && ib_is_mad_class_rmpp(rmpp_mad_hdr->mad_hdr.mgmt_class)
+ && (ib_get_rmpp_flags(&rmpp_mad_hdr->rmpp_hdr) & IB_MGMT_RMPP_FLAG_ACTIVE)) {
spin_lock_irq(&file->send_lock);
list_add_tail(&packet->list, &file->send_list);
spin_unlock_irq(&file->send_lock);
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index 989edc789633..e86afecfbe46 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -2912,6 +2912,106 @@ fail:
return rc;
}
+static void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq)
+{
+ struct bnxt_re_dev *rdev = cq->rdev;
+
+ bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq);
+
+ cq->qplib_cq.max_wqe = cq->resize_cqe;
+ if (cq->resize_umem) {
+ ib_umem_release(cq->umem);
+ cq->umem = cq->resize_umem;
+ cq->resize_umem = NULL;
+ cq->resize_cqe = 0;
+ }
+}
+
+int bnxt_re_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
+{
+ struct bnxt_qplib_sg_info sg_info = {};
+ struct bnxt_qplib_dpi *orig_dpi = NULL;
+ struct bnxt_qplib_dev_attr *dev_attr;
+ struct bnxt_re_ucontext *uctx = NULL;
+ struct bnxt_re_resize_cq_req req;
+ struct bnxt_re_dev *rdev;
+ struct bnxt_re_cq *cq;
+ int rc, entries;
+
+ cq = container_of(ibcq, struct bnxt_re_cq, ib_cq);
+ rdev = cq->rdev;
+ dev_attr = &rdev->dev_attr;
+ if (!ibcq->uobject) {
+ ibdev_err(&rdev->ibdev, "Kernel CQ Resize not supported");
+ return -EOPNOTSUPP;
+ }
+
+ if (cq->resize_umem) {
+ ibdev_err(&rdev->ibdev, "Resize CQ %#x failed - Busy",
+ cq->qplib_cq.id);
+ return -EBUSY;
+ }
+
+ /* Check the requested cq depth out of supported depth */
+ if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
+ ibdev_err(&rdev->ibdev, "Resize CQ %#x failed - out of range cqe %d",
+ cq->qplib_cq.id, cqe);
+ return -EINVAL;
+ }
+
+ entries = roundup_pow_of_two(cqe + 1);
+ if (entries > dev_attr->max_cq_wqes + 1)
+ entries = dev_attr->max_cq_wqes + 1;
+
+ uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext,
+ ib_uctx);
+ /* uverbs consumer */
+ if (ib_copy_from_udata(&req, udata, sizeof(req))) {
+ rc = -EFAULT;
+ goto fail;
+ }
+
+ cq->resize_umem = ib_umem_get(&rdev->ibdev, req.cq_va,
+ entries * sizeof(struct cq_base),
+ IB_ACCESS_LOCAL_WRITE);
+ if (IS_ERR(cq->resize_umem)) {
+ rc = PTR_ERR(cq->resize_umem);
+ cq->resize_umem = NULL;
+ ibdev_err(&rdev->ibdev, "%s: ib_umem_get failed! rc = %d\n",
+ __func__, rc);
+ goto fail;
+ }
+ cq->resize_cqe = entries;
+ memcpy(&sg_info, &cq->qplib_cq.sg_info, sizeof(sg_info));
+ orig_dpi = cq->qplib_cq.dpi;
+
+ cq->qplib_cq.sg_info.umem = cq->resize_umem;
+ cq->qplib_cq.sg_info.pgsize = PAGE_SIZE;
+ cq->qplib_cq.sg_info.pgshft = PAGE_SHIFT;
+ cq->qplib_cq.dpi = &uctx->dpi;
+
+ rc = bnxt_qplib_resize_cq(&rdev->qplib_res, &cq->qplib_cq, entries);
+ if (rc) {
+ ibdev_err(&rdev->ibdev, "Resize HW CQ %#x failed!",
+ cq->qplib_cq.id);
+ goto fail;
+ }
+
+ cq->ib_cq.cqe = cq->resize_cqe;
+
+ return 0;
+
+fail:
+ if (cq->resize_umem) {
+ ib_umem_release(cq->resize_umem);
+ cq->resize_umem = NULL;
+ cq->resize_cqe = 0;
+ memcpy(&cq->qplib_cq.sg_info, &sg_info, sizeof(sg_info));
+ cq->qplib_cq.dpi = orig_dpi;
+ }
+ return rc;
+}
+
static u8 __req_to_ib_wc_status(u8 qstatus)
{
switch (qstatus) {
@@ -3425,6 +3525,15 @@ int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)
struct bnxt_re_sqp_entries *sqp_entry = NULL;
unsigned long flags;
+ /* User CQ; the only processing we do is to
+ * complete any pending CQ resize operation.
+ */
+ if (cq->umem) {
+ if (cq->resize_umem)
+ bnxt_re_resize_cq_complete(cq);
+ return 0;
+ }
+
spin_lock_irqsave(&cq->cq_lock, flags);
budget = min_t(u32, num_entries, cq->max_cql);
num_entries = budget;
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
index 94326267f9bb..31f7e34040f7 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
@@ -104,6 +104,8 @@ struct bnxt_re_cq {
#define MAX_CQL_PER_POLL 1024
u32 max_cql;
struct ib_umem *umem;
+ struct ib_umem *resize_umem;
+ int resize_cqe;
};
struct bnxt_re_mr {
@@ -191,6 +193,7 @@ int bnxt_re_post_recv(struct ib_qp *qp, const struct ib_recv_wr *recv_wr,
const struct ib_recv_wr **bad_recv_wr);
int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
struct ib_udata *udata);
+int bnxt_re_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata);
int bnxt_re_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
int bnxt_re_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc);
int bnxt_re_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);
diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c
index c5867e78f231..b9e2f89337e8 100644
--- a/drivers/infiniband/hw/bnxt_re/main.c
+++ b/drivers/infiniband/hw/bnxt_re/main.c
@@ -553,6 +553,7 @@ static const struct ib_device_ops bnxt_re_dev_ops = {
.query_srq = bnxt_re_query_srq,
.reg_user_mr = bnxt_re_reg_user_mr,
.req_notify_cq = bnxt_re_req_notify_cq,
+ .resize_cq = bnxt_re_resize_cq,
INIT_RDMA_OBJ_SIZE(ib_ah, bnxt_re_ah, ib_ah),
INIT_RDMA_OBJ_SIZE(ib_cq, bnxt_re_cq, ib_cq),
INIT_RDMA_OBJ_SIZE(ib_pd, bnxt_re_pd, ib_pd),
@@ -584,6 +585,7 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)
return ret;
dma_set_max_seg_size(&rdev->en_dev->pdev->dev, UINT_MAX);
+ ibdev->uverbs_cmd_mask |= BIT_ULL(IB_USER_VERBS_CMD_POLL_CQ);
return ib_register_device(ibdev, "bnxt_re%d", &rdev->en_dev->pdev->dev);
}
@@ -919,49 +921,6 @@ static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp,
}
}
-#define HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN 0x02
-static int bnxt_re_query_hwrm_pri2cos(struct bnxt_re_dev *rdev, u8 dir,
- u64 *cid_map)
-{
- struct hwrm_queue_pri2cos_qcfg_input req = {0};
- struct hwrm_queue_pri2cos_qcfg_output resp;
- struct bnxt_en_dev *en_dev = rdev->en_dev;
- struct bnxt_fw_msg fw_msg;
- u32 flags = 0;
- u8 *qcfgmap, *tmp_map;
- int rc = 0, i;
-
- if (!cid_map)
- return -EINVAL;
-
- memset(&fw_msg, 0, sizeof(fw_msg));
- bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
- HWRM_QUEUE_PRI2COS_QCFG, -1, -1);
- flags |= (dir & 0x01);
- flags |= HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN;
- req.flags = cpu_to_le32(flags);
- req.port_id = en_dev->pf_port_id;
-
- bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
- sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
- rc = bnxt_send_msg(en_dev, &fw_msg);
- if (rc)
- return rc;
-
- if (resp.queue_cfg_info) {
- ibdev_warn(&rdev->ibdev,
- "Asymmetric cos queue configuration detected");
- ibdev_warn(&rdev->ibdev,
- " on device, QoS may not be fully functional\n");
- }
- qcfgmap = &resp.pri0_cos_queue_id;
- tmp_map = (u8 *)cid_map;
- for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++)
- tmp_map[i] = qcfgmap[i];
-
- return rc;
-}
-
static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev,
struct bnxt_re_qp *qp)
{
@@ -1054,26 +1013,9 @@ static u32 bnxt_re_get_priority_mask(struct bnxt_re_dev *rdev)
return prio_map;
}
-static void bnxt_re_parse_cid_map(u8 prio_map, u8 *cid_map, u16 *cosq)
-{
- u16 prio;
- u8 id;
-
- for (prio = 0, id = 0; prio < 8; prio++) {
- if (prio_map & (1 << prio)) {
- cosq[id] = cid_map[prio];
- id++;
- if (id == 2) /* Max 2 tcs supported */
- break;
- }
- }
-}
-
static int bnxt_re_setup_qos(struct bnxt_re_dev *rdev)
{
u8 prio_map = 0;
- u64 cid_map;
- int rc;
/* Get priority for roce */
prio_map = bnxt_re_get_priority_mask(rdev);
@@ -1081,23 +1023,6 @@ static int bnxt_re_setup_qos(struct bnxt_re_dev *rdev)
if (prio_map == rdev->cur_prio_map)
return 0;
rdev->cur_prio_map = prio_map;
- /* Get cosq id for this priority */
- rc = bnxt_re_query_hwrm_pri2cos(rdev, 0, &cid_map);
- if (rc) {
- ibdev_warn(&rdev->ibdev, "no cos for p_mask %x\n", prio_map);
- return rc;
- }
- /* Parse CoS IDs for app priority */
- bnxt_re_parse_cid_map(prio_map, (u8 *)&cid_map, rdev->cosq);
-
- /* Config BONO. */
- rc = bnxt_qplib_map_tc2cos(&rdev->qplib_res, rdev->cosq);
- if (rc) {
- ibdev_warn(&rdev->ibdev, "no tc for cos{%x, %x}\n",
- rdev->cosq[0], rdev->cosq[1]);
- return rc;
- }
-
/* Actual priorities are not programmed as they are already
* done by L2 driver; just enable or disable priority vlan tagging
*/
@@ -1407,6 +1332,27 @@ exit:
return rc;
}
+static void bnxt_re_setup_cc(struct bnxt_re_dev *rdev, bool enable)
+{
+ struct bnxt_qplib_cc_param cc_param = {};
+
+ /* Currently enabling only for GenP5 adapters */
+ if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx))
+ return;
+
+ if (enable) {
+ cc_param.enable = 1;
+ cc_param.cc_mode = CMDQ_MODIFY_ROCE_CC_CC_MODE_PROBABILISTIC_CC_MODE;
+ }
+
+ cc_param.mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE |
+ CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC |
+ CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN);
+
+ if (bnxt_qplib_modify_cc(&rdev->qplib_res, &cc_param))
+ ibdev_err(&rdev->ibdev, "Failed to setup CC enable = %d\n", enable);
+}
+
/*
* "Notifier chain callback can be invoked for the same chain from
* different CPUs at the same time".
@@ -1475,7 +1421,7 @@ static void bnxt_re_remove(struct auxiliary_device *adev)
*/
goto skip_remove;
}
-
+ bnxt_re_setup_cc(rdev, false);
ib_unregister_device(&rdev->ibdev);
ib_dealloc_device(&rdev->ibdev);
bnxt_re_dev_uninit(rdev);
@@ -1507,6 +1453,7 @@ static int bnxt_re_probe(struct auxiliary_device *adev,
goto err;
}
+ bnxt_re_setup_cc(rdev, true);
mutex_unlock(&bnxt_re_mutex);
return 0;
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
index 96e581ced50e..f139d4cd1712 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
@@ -300,8 +300,6 @@ static void bnxt_qplib_service_nq(struct tasklet_struct *t)
{
struct bnxt_qplib_nq *nq = from_tasklet(nq, t, nq_tasklet);
struct bnxt_qplib_hwq *hwq = &nq->hwq;
- int num_srqne_processed = 0;
- int num_cqne_processed = 0;
struct bnxt_qplib_cq *cq;
int budget = nq->budget;
u32 sw_cons, raw_cons;
@@ -340,9 +338,7 @@ static void bnxt_qplib_service_nq(struct tasklet_struct *t)
DBC_DBC_TYPE_CQ_ARMENA);
spin_lock_bh(&cq->compl_lock);
atomic_set(&cq->arm_state, 0);
- if (!nq->cqn_handler(nq, (cq)))
- num_cqne_processed++;
- else
+ if (nq->cqn_handler(nq, (cq)))
dev_warn(&nq->pdev->dev,
"cqn - type 0x%x not handled\n", type);
cq->cnq_events++;
@@ -361,11 +357,9 @@ static void bnxt_qplib_service_nq(struct tasklet_struct *t)
srq = (struct bnxt_qplib_srq *)q_handle;
bnxt_qplib_armen_db(&srq->dbinfo,
DBC_DBC_TYPE_SRQ_ARMENA);
- if (!nq->srqn_handler(nq,
- (struct bnxt_qplib_srq *)q_handle,
- nqsrqe->event))
- num_srqne_processed++;
- else
+ if (nq->srqn_handler(nq,
+ (struct bnxt_qplib_srq *)q_handle,
+ nqsrqe->event))
dev_warn(&nq->pdev->dev,
"SRQ event 0x%x not handled\n",
nqsrqe->event);
@@ -581,18 +575,20 @@ void bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
struct bnxt_qplib_srq *srq)
{
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
- struct cmdq_destroy_srq req;
- struct creq_destroy_srq_resp resp;
- u16 cmd_flags = 0;
+ struct creq_destroy_srq_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_destroy_srq req = {};
int rc;
- RCFW_CMD_PREP(req, DESTROY_SRQ, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_DESTROY_SRQ,
+ sizeof(req));
/* Configure the request */
req.srq_cid = cpu_to_le32(srq->id);
- rc = bnxt_qplib_rcfw_send_message(rcfw, (struct cmdq_base *)&req,
- (struct creq_base *)&resp, NULL, 0);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
kfree(srq->swq);
if (rc)
return;
@@ -604,10 +600,10 @@ int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
{
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
struct bnxt_qplib_hwq_attr hwq_attr = {};
- struct creq_create_srq_resp resp;
- struct cmdq_create_srq req;
+ struct creq_create_srq_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_create_srq req = {};
struct bnxt_qplib_pbl *pbl;
- u16 cmd_flags = 0;
u16 pg_sz_lvl;
int rc, idx;
@@ -627,7 +623,9 @@ int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
goto fail;
}
- RCFW_CMD_PREP(req, CREATE_SRQ, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_CREATE_SRQ,
+ sizeof(req));
/* Configure the request */
req.dpi = cpu_to_le32(srq->dpi->dpi);
@@ -644,8 +642,8 @@ int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
req.pd_id = cpu_to_le32(srq->pd->id);
req.eventq_id = cpu_to_le16(srq->eventq_hw_ring_id);
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
- (void *)&resp, NULL, 0);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc)
goto fail;
@@ -700,14 +698,16 @@ int bnxt_qplib_query_srq(struct bnxt_qplib_res *res,
struct bnxt_qplib_srq *srq)
{
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
- struct cmdq_query_srq req;
- struct creq_query_srq_resp resp;
+ struct creq_query_srq_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
struct bnxt_qplib_rcfw_sbuf *sbuf;
struct creq_query_srq_resp_sb *sb;
- u16 cmd_flags = 0;
+ struct cmdq_query_srq req = {};
int rc = 0;
- RCFW_CMD_PREP(req, QUERY_SRQ, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_QUERY_SRQ,
+ sizeof(req));
/* Configure the request */
sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));
@@ -716,8 +716,9 @@ int bnxt_qplib_query_srq(struct bnxt_qplib_res *res,
req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;
req.srq_cid = cpu_to_le32(srq->id);
sb = sbuf->sb;
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
- (void *)sbuf, 0);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, sbuf, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
srq->threshold = le16_to_cpu(sb->srq_limit);
bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf);
@@ -811,19 +812,20 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
{
struct bnxt_qplib_hwq_attr hwq_attr = {};
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct creq_create_qp1_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
struct bnxt_qplib_q *sq = &qp->sq;
struct bnxt_qplib_q *rq = &qp->rq;
- struct creq_create_qp1_resp resp;
- struct cmdq_create_qp1 req;
+ struct cmdq_create_qp1 req = {};
struct bnxt_qplib_pbl *pbl;
- u16 cmd_flags = 0;
u32 qp_flags = 0;
u8 pg_sz_lvl;
u32 tbl_indx;
int rc;
- RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags);
-
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_CREATE_QP1,
+ sizeof(req));
/* General */
req.type = qp->type;
req.dpi = cpu_to_le32(qp->dpi->dpi);
@@ -891,8 +893,8 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
req.qp_flags = cpu_to_le32(qp_flags);
req.pd_id = cpu_to_le32(qp->pd->id);
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
- (void *)&resp, NULL, 0);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc)
goto fail;
@@ -952,20 +954,22 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
struct bnxt_qplib_hwq_attr hwq_attr = {};
struct bnxt_qplib_sg_info sginfo = {};
+ struct creq_create_qp_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
struct bnxt_qplib_q *sq = &qp->sq;
struct bnxt_qplib_q *rq = &qp->rq;
- struct creq_create_qp_resp resp;
+ struct cmdq_create_qp req = {};
int rc, req_size, psn_sz = 0;
struct bnxt_qplib_hwq *xrrq;
struct bnxt_qplib_pbl *pbl;
- struct cmdq_create_qp req;
- u16 cmd_flags = 0;
u32 qp_flags = 0;
u8 pg_sz_lvl;
u32 tbl_indx;
u16 nsge;
- RCFW_CMD_PREP(req, CREATE_QP, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_CREATE_QP,
+ sizeof(req));
/* General */
req.type = qp->type;
@@ -1098,8 +1102,9 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
}
req.pd_id = cpu_to_le32(qp->pd->id);
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
- (void *)&resp, NULL, 0);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc)
goto fail;
@@ -1231,14 +1236,16 @@ static void __filter_modify_flags(struct bnxt_qplib_qp *qp)
int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
{
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
- struct cmdq_modify_qp req;
- struct creq_modify_qp_resp resp;
- u16 cmd_flags = 0;
+ struct creq_modify_qp_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_modify_qp req = {};
u32 temp32[4];
u32 bmask;
int rc;
- RCFW_CMD_PREP(req, MODIFY_QP, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_MODIFY_QP,
+ sizeof(req));
/* Filter out the qp_attr_mask based on the state->new transition */
__filter_modify_flags(qp);
@@ -1286,7 +1293,7 @@ int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
memcpy(req.dest_mac, qp->ah.dmac, 6);
if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)
- req.path_mtu = qp->path_mtu;
+ req.path_mtu_pingpong_push_enable |= qp->path_mtu;
if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT)
req.timeout = qp->timeout;
@@ -1324,8 +1331,8 @@ int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id);
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
- (void *)&resp, NULL, 0);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc)
return rc;
qp->cur_qp_state = qp->state;
@@ -1335,15 +1342,17 @@ int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
{
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
- struct cmdq_query_qp req;
- struct creq_query_qp_resp resp;
+ struct creq_query_qp_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
struct bnxt_qplib_rcfw_sbuf *sbuf;
struct creq_query_qp_resp_sb *sb;
- u16 cmd_flags = 0;
+ struct cmdq_query_qp req = {};
u32 temp32[4];
int i, rc = 0;
- RCFW_CMD_PREP(req, QUERY_QP, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_QUERY_QP,
+ sizeof(req));
sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));
if (!sbuf)
@@ -1352,8 +1361,9 @@ int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
req.qp_cid = cpu_to_le32(qp->id);
req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
- (void *)sbuf, 0);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, sbuf, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc)
goto bail;
/* Extract the context from the side buffer */
@@ -1460,9 +1470,9 @@ int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
struct bnxt_qplib_qp *qp)
{
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
- struct cmdq_destroy_qp req;
- struct creq_destroy_qp_resp resp;
- u16 cmd_flags = 0;
+ struct creq_destroy_qp_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_destroy_qp req = {};
u32 tbl_indx;
int rc;
@@ -1470,11 +1480,14 @@ int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
rcfw->qp_tbl[tbl_indx].qp_id = BNXT_QPLIB_QP_ID_INVALID;
rcfw->qp_tbl[tbl_indx].qp_handle = NULL;
- RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_DESTROY_QP,
+ sizeof(req));
req.qp_cid = cpu_to_le32(qp->id);
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
- (void *)&resp, NULL, 0);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc) {
rcfw->qp_tbl[tbl_indx].qp_id = qp->id;
rcfw->qp_tbl[tbl_indx].qp_handle = qp;
@@ -2036,10 +2049,10 @@ int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
{
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
struct bnxt_qplib_hwq_attr hwq_attr = {};
- struct creq_create_cq_resp resp;
+ struct creq_create_cq_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_create_cq req = {};
struct bnxt_qplib_pbl *pbl;
- struct cmdq_create_cq req;
- u16 cmd_flags = 0;
u32 pg_sz_lvl;
int rc;
@@ -2052,7 +2065,9 @@ int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
if (rc)
goto exit;
- RCFW_CMD_PREP(req, CREATE_CQ, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_CREATE_CQ,
+ sizeof(req));
if (!cq->dpi) {
dev_err(&rcfw->pdev->dev,
@@ -2071,9 +2086,9 @@ int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
req.cq_fco_cnq_id = cpu_to_le32(
(cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<
CMDQ_CREATE_CQ_CNQ_ID_SFT);
-
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
- (void *)&resp, NULL, 0);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc)
goto fail;
@@ -2100,20 +2115,70 @@ exit:
return rc;
}
+void bnxt_qplib_resize_cq_complete(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_cq *cq)
+{
+ bnxt_qplib_free_hwq(res, &cq->hwq);
+ memcpy(&cq->hwq, &cq->resize_hwq, sizeof(cq->hwq));
+}
+
+int bnxt_qplib_resize_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq,
+ int new_cqes)
+{
+ struct bnxt_qplib_hwq_attr hwq_attr = {};
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct creq_resize_cq_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_resize_cq req = {};
+ struct bnxt_qplib_pbl *pbl;
+ u32 pg_sz, lvl, new_sz;
+ int rc;
+
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_RESIZE_CQ,
+ sizeof(req));
+ hwq_attr.sginfo = &cq->sg_info;
+ hwq_attr.res = res;
+ hwq_attr.depth = new_cqes;
+ hwq_attr.stride = sizeof(struct cq_base);
+ hwq_attr.type = HWQ_TYPE_QUEUE;
+ rc = bnxt_qplib_alloc_init_hwq(&cq->resize_hwq, &hwq_attr);
+ if (rc)
+ return rc;
+
+ req.cq_cid = cpu_to_le32(cq->id);
+ pbl = &cq->resize_hwq.pbl[PBL_LVL_0];
+ pg_sz = bnxt_qplib_base_pg_size(&cq->resize_hwq);
+ lvl = (cq->resize_hwq.level << CMDQ_RESIZE_CQ_LVL_SFT) &
+ CMDQ_RESIZE_CQ_LVL_MASK;
+ new_sz = (new_cqes << CMDQ_RESIZE_CQ_NEW_CQ_SIZE_SFT) &
+ CMDQ_RESIZE_CQ_NEW_CQ_SIZE_MASK;
+ req.new_cq_size_pg_size_lvl = cpu_to_le32(new_sz | pg_sz | lvl);
+ req.new_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
+
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ return rc;
+}
+
int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
{
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
- struct cmdq_destroy_cq req;
- struct creq_destroy_cq_resp resp;
+ struct creq_destroy_cq_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_destroy_cq req = {};
u16 total_cnq_events;
- u16 cmd_flags = 0;
int rc;
- RCFW_CMD_PREP(req, DESTROY_CQ, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_DESTROY_CQ,
+ sizeof(req));
req.cq_cid = cpu_to_le32(cq->id);
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
- (void *)&resp, NULL, 0);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc)
return rc;
total_cnq_events = le16_to_cpu(resp.total_cnq_events);
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.h b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
index 037501952543..d74d5ead2e32 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
@@ -400,6 +400,7 @@ struct bnxt_qplib_cq {
u16 count;
u16 period;
struct bnxt_qplib_hwq hwq;
+ struct bnxt_qplib_hwq resize_hwq;
u32 cnq_hw_ring_id;
struct bnxt_qplib_nq *nq;
bool resize_in_progress;
@@ -532,6 +533,10 @@ void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp);
int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
struct bnxt_qplib_swqe *wqe);
int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq);
+int bnxt_qplib_resize_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq,
+ int new_cqes);
+void bnxt_qplib_resize_cq_complete(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_cq *cq);
int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq);
int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
int num, struct bnxt_qplib_qp **qp);
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
index 061b2895dd9b..de9069103177 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
@@ -49,6 +49,7 @@
#include "qplib_rcfw.h"
#include "qplib_sp.h"
#include "qplib_fp.h"
+#include "qplib_tlv.h"
static void bnxt_qplib_service_creq(struct tasklet_struct *t);
@@ -85,8 +86,8 @@ done:
return count ? 0 : -ETIMEDOUT;
};
-static int __send_message(struct bnxt_qplib_rcfw *rcfw, struct cmdq_base *req,
- struct creq_base *resp, void *sb, u8 is_block)
+static int __send_message(struct bnxt_qplib_rcfw *rcfw,
+ struct bnxt_qplib_cmdqmsg *msg)
{
struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq;
struct bnxt_qplib_hwq *hwq = &cmdq->hwq;
@@ -95,13 +96,13 @@ static int __send_message(struct bnxt_qplib_rcfw *rcfw, struct cmdq_base *req,
u32 sw_prod, cmdq_prod;
struct pci_dev *pdev;
unsigned long flags;
- u32 size, opcode;
+ u32 bsize, opcode;
u16 cookie, cbit;
u8 *preq;
pdev = rcfw->pdev;
- opcode = req->opcode;
+ opcode = __get_cmdq_base_opcode(msg->req, msg->req_sz);
if (!test_bit(FIRMWARE_INITIALIZED_FLAG, &cmdq->flags) &&
(opcode != CMDQ_BASE_OPCODE_QUERY_FUNC &&
opcode != CMDQ_BASE_OPCODE_INITIALIZE_FW &&
@@ -124,7 +125,7 @@ static int __send_message(struct bnxt_qplib_rcfw *rcfw, struct cmdq_base *req,
* cmdqe
*/
spin_lock_irqsave(&hwq->lock, flags);
- if (req->cmd_size >= HWQ_FREE_SLOTS(hwq)) {
+ if (msg->req->cmd_size >= HWQ_FREE_SLOTS(hwq)) {
dev_err(&pdev->dev, "RCFW: CMDQ is full!\n");
spin_unlock_irqrestore(&hwq->lock, flags);
return -EAGAIN;
@@ -133,36 +134,34 @@ static int __send_message(struct bnxt_qplib_rcfw *rcfw, struct cmdq_base *req,
cookie = cmdq->seq_num & RCFW_MAX_COOKIE_VALUE;
cbit = cookie % rcfw->cmdq_depth;
- if (is_block)
+ if (msg->block)
cookie |= RCFW_CMD_IS_BLOCKING;
set_bit(cbit, cmdq->cmdq_bitmap);
- req->cookie = cpu_to_le16(cookie);
+ __set_cmdq_base_cookie(msg->req, msg->req_sz, cpu_to_le16(cookie));
crsqe = &rcfw->crsqe_tbl[cbit];
if (crsqe->resp) {
spin_unlock_irqrestore(&hwq->lock, flags);
return -EBUSY;
}
- size = req->cmd_size;
/* change the cmd_size to the number of 16byte cmdq unit.
* req->cmd_size is modified here
*/
- bnxt_qplib_set_cmd_slots(req);
-
- memset(resp, 0, sizeof(*resp));
- crsqe->resp = (struct creq_qp_event *)resp;
- crsqe->resp->cookie = req->cookie;
- crsqe->req_size = req->cmd_size;
- if (req->resp_size && sb) {
- struct bnxt_qplib_rcfw_sbuf *sbuf = sb;
-
- req->resp_addr = cpu_to_le64(sbuf->dma_addr);
- req->resp_size = (sbuf->size + BNXT_QPLIB_CMDQE_UNITS - 1) /
- BNXT_QPLIB_CMDQE_UNITS;
+ bsize = bnxt_qplib_set_cmd_slots(msg->req);
+
+ memset(msg->resp, 0, sizeof(*msg->resp));
+ crsqe->resp = (struct creq_qp_event *)msg->resp;
+ crsqe->resp->cookie = cpu_to_le16(cookie);
+ crsqe->req_size = __get_cmdq_base_cmd_size(msg->req, msg->req_sz);
+ if (__get_cmdq_base_resp_size(msg->req, msg->req_sz) && msg->sb) {
+ struct bnxt_qplib_rcfw_sbuf *sbuf = msg->sb;
+ __set_cmdq_base_resp_addr(msg->req, msg->req_sz, cpu_to_le64(sbuf->dma_addr));
+ __set_cmdq_base_resp_size(msg->req, msg->req_sz,
+ ALIGN(sbuf->size, BNXT_QPLIB_CMDQE_UNITS));
}
- preq = (u8 *)req;
+ preq = (u8 *)msg->req;
do {
/* Locate the next cmdq slot */
sw_prod = HWQ_CMP(hwq->prod, hwq);
@@ -174,11 +173,11 @@ static int __send_message(struct bnxt_qplib_rcfw *rcfw, struct cmdq_base *req,
}
/* Copy a segment of the req cmd to the cmdq */
memset(cmdqe, 0, sizeof(*cmdqe));
- memcpy(cmdqe, preq, min_t(u32, size, sizeof(*cmdqe)));
- preq += min_t(u32, size, sizeof(*cmdqe));
- size -= min_t(u32, size, sizeof(*cmdqe));
+ memcpy(cmdqe, preq, min_t(u32, bsize, sizeof(*cmdqe)));
+ preq += min_t(u32, bsize, sizeof(*cmdqe));
+ bsize -= min_t(u32, bsize, sizeof(*cmdqe));
hwq->prod++;
- } while (size > 0);
+ } while (bsize > 0);
cmdq->seq_num++;
cmdq_prod = hwq->prod;
@@ -191,7 +190,6 @@ static int __send_message(struct bnxt_qplib_rcfw *rcfw, struct cmdq_base *req,
cmdq_prod |= BIT(FIRMWARE_FIRST_FLAG);
clear_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags);
}
-
/* ring CMDQ DB */
wmb();
writel(cmdq_prod, cmdq->cmdq_mbox.prod);
@@ -203,11 +201,9 @@ done:
}
int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,
- struct cmdq_base *req,
- struct creq_base *resp,
- void *sb, u8 is_block)
+ struct bnxt_qplib_cmdqmsg *msg)
{
- struct creq_qp_event *evnt = (struct creq_qp_event *)resp;
+ struct creq_qp_event *evnt = (struct creq_qp_event *)msg->resp;
u16 cookie;
u8 opcode, retry_cnt = 0xFF;
int rc = 0;
@@ -217,23 +213,23 @@ int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,
return 0;
do {
- opcode = req->opcode;
- rc = __send_message(rcfw, req, resp, sb, is_block);
- cookie = le16_to_cpu(req->cookie) & RCFW_MAX_COOKIE_VALUE;
+ opcode = __get_cmdq_base_opcode(msg->req, msg->req_sz);
+ rc = __send_message(rcfw, msg);
+ cookie = le16_to_cpu(__get_cmdq_base_cookie(msg->req, msg->req_sz)) &
+ RCFW_MAX_COOKIE_VALUE;
if (!rc)
break;
-
if (!retry_cnt || (rc != -EAGAIN && rc != -EBUSY)) {
/* send failed */
dev_err(&rcfw->pdev->dev, "cmdq[%#x]=%#x send failed\n",
cookie, opcode);
return rc;
}
- is_block ? mdelay(1) : usleep_range(500, 1000);
+ msg->block ? mdelay(1) : usleep_range(500, 1000);
} while (retry_cnt--);
- if (is_block)
+ if (msg->block)
rc = __block_for_resp(rcfw, cookie);
else
rc = __wait_for_resp(rcfw, cookie);
@@ -452,14 +448,17 @@ static irqreturn_t bnxt_qplib_creq_irq(int irq, void *dev_instance)
/* RCFW */
int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw)
{
- struct cmdq_deinitialize_fw req;
- struct creq_deinitialize_fw_resp resp;
- u16 cmd_flags = 0;
+ struct creq_deinitialize_fw_resp resp = {};
+ struct cmdq_deinitialize_fw req = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
int rc;
- RCFW_CMD_PREP(req, DEINITIALIZE_FW, cmd_flags);
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
- NULL, 0);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_DEINITIALIZE_FW,
+ sizeof(req));
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL,
+ sizeof(req), sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc)
return rc;
@@ -470,13 +469,15 @@ int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw)
int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw,
struct bnxt_qplib_ctx *ctx, int is_virtfn)
{
- struct creq_initialize_fw_resp resp;
- struct cmdq_initialize_fw req;
- u16 cmd_flags = 0;
+ struct creq_initialize_fw_resp resp = {};
+ struct cmdq_initialize_fw req = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
u8 pgsz, lvl;
int rc;
- RCFW_CMD_PREP(req, INITIALIZE_FW, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_INITIALIZE_FW,
+ sizeof(req));
/* Supply (log-base-2-of-host-page-size - base-page-shift)
* to bono to adjust the doorbell page sizes.
*/
@@ -545,8 +546,8 @@ config_vf_res:
skip_ctx_setup:
req.stat_ctx_id = cpu_to_le32(ctx->stats.fw_id);
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
- NULL, 0);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc)
return rc;
set_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->cmdq.flags);
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h
index 0a3d8e7da3d4..dd5651478bbb 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h
@@ -39,6 +39,8 @@
#ifndef __BNXT_QPLIB_RCFW_H__
#define __BNXT_QPLIB_RCFW_H__
+#include "qplib_tlv.h"
+
#define RCFW_CMDQ_TRIG_VAL 1
#define RCFW_COMM_PCI_BAR_REGION 0
#define RCFW_COMM_CONS_PCI_BAR_REGION 2
@@ -51,25 +53,25 @@
#define RCFW_DBR_PCI_BAR_REGION 2
#define RCFW_DBR_BASE_PAGE_SHIFT 12
-#define RCFW_CMD_PREP(req, CMD, cmd_flags) \
- do { \
- memset(&(req), 0, sizeof((req))); \
- (req).opcode = CMDQ_BASE_OPCODE_##CMD; \
- (req).cmd_size = sizeof((req)); \
- (req).flags = cpu_to_le16(cmd_flags); \
- } while (0)
-
-#define RCFW_CMD_WAIT_TIME_MS 20000 /* 20 Seconds timeout */
-
/* Cmdq contains a fix number of a 16-Byte slots */
struct bnxt_qplib_cmdqe {
u8 data[16];
};
+#define BNXT_QPLIB_CMDQE_UNITS sizeof(struct bnxt_qplib_cmdqe)
+
+static inline void bnxt_qplib_rcfw_cmd_prep(struct cmdq_base *req,
+ u8 opcode, u8 cmd_size)
+{
+ req->opcode = opcode;
+ req->cmd_size = cmd_size;
+}
+
+#define RCFW_CMD_WAIT_TIME_MS 20000 /* 20 Seconds timeout */
+
/* CMDQ elements */
#define BNXT_QPLIB_CMDQE_MAX_CNT_256 256
#define BNXT_QPLIB_CMDQE_MAX_CNT_8192 8192
-#define BNXT_QPLIB_CMDQE_UNITS sizeof(struct bnxt_qplib_cmdqe)
#define BNXT_QPLIB_CMDQE_BYTES(depth) ((depth) * BNXT_QPLIB_CMDQE_UNITS)
static inline u32 bnxt_qplib_cmdqe_npages(u32 depth)
@@ -87,11 +89,21 @@ static inline u32 bnxt_qplib_cmdqe_page_size(u32 depth)
return (bnxt_qplib_cmdqe_npages(depth) * PAGE_SIZE);
}
-/* Set the cmd_size to a factor of CMDQE unit */
-static inline void bnxt_qplib_set_cmd_slots(struct cmdq_base *req)
+static inline u32 bnxt_qplib_set_cmd_slots(struct cmdq_base *req)
{
- req->cmd_size = (req->cmd_size + BNXT_QPLIB_CMDQE_UNITS - 1) /
- BNXT_QPLIB_CMDQE_UNITS;
+ u32 cmd_byte = 0;
+
+ if (HAS_TLV_HEADER(req)) {
+ struct roce_tlv *tlv_req = (struct roce_tlv *)req;
+
+ cmd_byte = tlv_req->total_size * BNXT_QPLIB_CMDQE_UNITS;
+ } else {
+ cmd_byte = req->cmd_size;
+ req->cmd_size = (req->cmd_size + BNXT_QPLIB_CMDQE_UNITS - 1) /
+ BNXT_QPLIB_CMDQE_UNITS;
+ }
+
+ return cmd_byte;
}
#define RCFW_MAX_COOKIE_VALUE 0x7FFF
@@ -190,6 +202,27 @@ struct bnxt_qplib_rcfw {
u32 cmdq_depth;
};
+struct bnxt_qplib_cmdqmsg {
+ struct cmdq_base *req;
+ struct creq_base *resp;
+ void *sb;
+ u32 req_sz;
+ u32 res_sz;
+ u8 block;
+};
+
+static inline void bnxt_qplib_fill_cmdqmsg(struct bnxt_qplib_cmdqmsg *msg,
+ void *req, void *resp, void *sb,
+ u32 req_sz, u32 res_sz, u8 block)
+{
+ msg->req = req;
+ msg->resp = resp;
+ msg->sb = sb;
+ msg->req_sz = req_sz;
+ msg->res_sz = res_sz;
+ msg->block = block;
+}
+
void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw);
int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res,
struct bnxt_qplib_rcfw *rcfw,
@@ -210,8 +243,7 @@ struct bnxt_qplib_rcfw_sbuf *bnxt_qplib_rcfw_alloc_sbuf(
void bnxt_qplib_rcfw_free_sbuf(struct bnxt_qplib_rcfw *rcfw,
struct bnxt_qplib_rcfw_sbuf *sbuf);
int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,
- struct cmdq_base *req, struct creq_base *resp,
- void *sbuf, u8 is_block);
+ struct bnxt_qplib_cmdqmsg *msg);
int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw);
int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw,
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
index b802981b7171..1714a1e23113 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
@@ -48,6 +48,7 @@
#include "qplib_res.h"
#include "qplib_rcfw.h"
#include "qplib_sp.h"
+#include "qplib_tlv.h"
const struct bnxt_qplib_gid bnxt_qplib_gid_zero = {{ 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0 } };
@@ -68,15 +69,17 @@ static bool bnxt_qplib_is_atomic_cap(struct bnxt_qplib_rcfw *rcfw)
static void bnxt_qplib_query_version(struct bnxt_qplib_rcfw *rcfw,
char *fw_ver)
{
- struct cmdq_query_version req;
- struct creq_query_version_resp resp;
- u16 cmd_flags = 0;
+ struct creq_query_version_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_query_version req = {};
int rc = 0;
- RCFW_CMD_PREP(req, QUERY_VERSION, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_QUERY_VERSION,
+ sizeof(req));
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
- (void *)&resp, NULL, 0);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc)
return;
fw_ver[0] = resp.fw_maj;
@@ -88,16 +91,18 @@ static void bnxt_qplib_query_version(struct bnxt_qplib_rcfw *rcfw,
int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,
struct bnxt_qplib_dev_attr *attr, bool vf)
{
- struct cmdq_query_func req;
- struct creq_query_func_resp resp;
- struct bnxt_qplib_rcfw_sbuf *sbuf;
+ struct creq_query_func_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
struct creq_query_func_resp_sb *sb;
- u16 cmd_flags = 0;
- u32 temp;
+ struct bnxt_qplib_rcfw_sbuf *sbuf;
+ struct cmdq_query_func req = {};
u8 *tqm_alloc;
int i, rc = 0;
+ u32 temp;
- RCFW_CMD_PREP(req, QUERY_FUNC, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_QUERY_FUNC,
+ sizeof(req));
sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));
if (!sbuf) {
@@ -108,8 +113,9 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,
sb = sbuf->sb;
req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
- (void *)sbuf, 0);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, sbuf, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc)
goto bail;
@@ -174,12 +180,14 @@ int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res,
struct bnxt_qplib_rcfw *rcfw,
struct bnxt_qplib_ctx *ctx)
{
- struct cmdq_set_func_resources req;
- struct creq_set_func_resources_resp resp;
- u16 cmd_flags = 0;
+ struct creq_set_func_resources_resp resp = {};
+ struct cmdq_set_func_resources req = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
int rc = 0;
- RCFW_CMD_PREP(req, SET_FUNC_RESOURCES, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_SET_FUNC_RESOURCES,
+ sizeof(req));
req.number_of_qp = cpu_to_le32(ctx->qpc_count);
req.number_of_mrw = cpu_to_le32(ctx->mrw_count);
@@ -192,9 +200,9 @@ int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res,
req.max_cq_per_vf = cpu_to_le32(ctx->vf_res.max_cq_per_vf);
req.max_gid_per_vf = cpu_to_le32(ctx->vf_res.max_gid_per_vf);
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
- (void *)&resp,
- NULL, 0);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc) {
dev_err(&res->pdev->dev, "Failed to set function resources\n");
}
@@ -245,20 +253,23 @@ int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
}
/* Remove GID from the SGID table */
if (update) {
- struct cmdq_delete_gid req;
- struct creq_delete_gid_resp resp;
- u16 cmd_flags = 0;
+ struct creq_delete_gid_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_delete_gid req = {};
int rc;
- RCFW_CMD_PREP(req, DELETE_GID, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_DELETE_GID,
+ sizeof(req));
if (sgid_tbl->hw_id[index] == 0xFFFF) {
dev_err(&res->pdev->dev,
"GID entry contains an invalid HW id\n");
return -EINVAL;
}
req.gid_index = cpu_to_le16(sgid_tbl->hw_id[index]);
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
- (void *)&resp, NULL, 0);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc)
return rc;
}
@@ -315,12 +326,14 @@ int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
return -ENOMEM;
}
if (update) {
- struct cmdq_add_gid req;
- struct creq_add_gid_resp resp;
- u16 cmd_flags = 0;
+ struct creq_add_gid_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_add_gid req = {};
int rc;
- RCFW_CMD_PREP(req, ADD_GID, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_ADD_GID,
+ sizeof(req));
req.gid[0] = cpu_to_be32(((u32 *)gid->data)[3]);
req.gid[1] = cpu_to_be32(((u32 *)gid->data)[2]);
@@ -345,8 +358,9 @@ int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
req.src_mac[1] = cpu_to_be16(((u16 *)smac)[1]);
req.src_mac[2] = cpu_to_be16(((u16 *)smac)[2]);
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
- (void *)&resp, NULL, 0);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc)
return rc;
sgid_tbl->hw_id[free_idx] = le32_to_cpu(resp.xid);
@@ -375,12 +389,14 @@ int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
struct bnxt_qplib_res,
sgid_tbl);
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
- struct creq_modify_gid_resp resp;
- struct cmdq_modify_gid req;
+ struct creq_modify_gid_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_modify_gid req = {};
int rc;
- u16 cmd_flags = 0;
- RCFW_CMD_PREP(req, MODIFY_GID, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_MODIFY_GID,
+ sizeof(req));
req.gid[0] = cpu_to_be32(((u32 *)gid->data)[3]);
req.gid[1] = cpu_to_be32(((u32 *)gid->data)[2]);
@@ -399,8 +415,9 @@ int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
req.gid_index = cpu_to_le16(gid_idx);
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
- (void *)&resp, NULL, 0);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
return rc;
}
@@ -409,14 +426,16 @@ int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,
bool block)
{
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
- struct cmdq_create_ah req;
- struct creq_create_ah_resp resp;
- u16 cmd_flags = 0;
+ struct creq_create_ah_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_create_ah req = {};
u32 temp32[4];
u16 temp16[3];
int rc;
- RCFW_CMD_PREP(req, CREATE_AH, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_CREATE_AH,
+ sizeof(req));
memcpy(temp32, ah->dgid.data, sizeof(struct bnxt_qplib_gid));
req.dgid[0] = cpu_to_le32(temp32[0]);
@@ -439,8 +458,9 @@ int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,
req.dest_mac[1] = cpu_to_le16(temp16[1]);
req.dest_mac[2] = cpu_to_le16(temp16[2]);
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
- NULL, block);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), block);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc)
return rc;
@@ -452,26 +472,29 @@ void bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,
bool block)
{
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
- struct cmdq_destroy_ah req;
- struct creq_destroy_ah_resp resp;
- u16 cmd_flags = 0;
+ struct creq_destroy_ah_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_destroy_ah req = {};
/* Clean up the AH table in the device */
- RCFW_CMD_PREP(req, DESTROY_AH, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_DESTROY_AH,
+ sizeof(req));
req.ah_cid = cpu_to_le32(ah->id);
- bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp, NULL,
- block);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), block);
+ bnxt_qplib_rcfw_send_message(rcfw, &msg);
}
/* MRW */
int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw)
{
+ struct creq_deallocate_key_resp resp = {};
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
- struct cmdq_deallocate_key req;
- struct creq_deallocate_key_resp resp;
- u16 cmd_flags = 0;
+ struct cmdq_deallocate_key req = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
int rc;
if (mrw->lkey == 0xFFFFFFFF) {
@@ -479,7 +502,9 @@ int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw)
return 0;
}
- RCFW_CMD_PREP(req, DEALLOCATE_KEY, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_DEALLOCATE_KEY,
+ sizeof(req));
req.mrw_flags = mrw->type;
@@ -490,8 +515,9 @@ int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw)
else
req.key = cpu_to_le32(mrw->lkey);
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
- NULL, 0);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc)
return rc;
@@ -505,13 +531,15 @@ int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw)
int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw)
{
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
- struct cmdq_allocate_mrw req;
- struct creq_allocate_mrw_resp resp;
- u16 cmd_flags = 0;
+ struct creq_allocate_mrw_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_allocate_mrw req = {};
unsigned long tmp;
int rc;
- RCFW_CMD_PREP(req, ALLOCATE_MRW, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_ALLOCATE_MRW,
+ sizeof(req));
req.pd_id = cpu_to_le32(mrw->pd->id);
req.mrw_flags = mrw->type;
@@ -523,8 +551,9 @@ int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw)
tmp = (unsigned long)mrw;
req.mrw_handle = cpu_to_le64(tmp);
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
- (void *)&resp, NULL, 0);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc)
return rc;
@@ -541,16 +570,19 @@ int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw,
bool block)
{
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
- struct cmdq_deregister_mr req;
- struct creq_deregister_mr_resp resp;
- u16 cmd_flags = 0;
+ struct creq_deregister_mr_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_deregister_mr req = {};
int rc;
- RCFW_CMD_PREP(req, DEREGISTER_MR, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_DEREGISTER_MR,
+ sizeof(req));
req.lkey = cpu_to_le32(mrw->lkey);
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
- (void *)&resp, NULL, block);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), block);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc)
return rc;
@@ -570,11 +602,12 @@ int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
struct bnxt_qplib_hwq_attr hwq_attr = {};
struct bnxt_qplib_sg_info sginfo = {};
- struct creq_register_mr_resp resp;
- struct cmdq_register_mr req;
- u16 cmd_flags = 0, level;
+ struct creq_register_mr_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_register_mr req = {};
int pages, rc;
u32 pg_size;
+ u16 level;
if (num_pbls) {
pages = roundup_pow_of_two(num_pbls);
@@ -602,7 +635,9 @@ int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
}
}
- RCFW_CMD_PREP(req, REGISTER_MR, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_REGISTER_MR,
+ sizeof(req));
/* Configure the request */
if (mr->hwq.level == PBL_LVL_MAX) {
@@ -627,8 +662,9 @@ int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
req.key = cpu_to_le32(mr->lkey);
req.mr_size = cpu_to_le64(mr->total_size);
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
- (void *)&resp, NULL, false);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc)
goto fail;
@@ -679,32 +715,19 @@ int bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res,
return 0;
}
-int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids)
-{
- struct bnxt_qplib_rcfw *rcfw = res->rcfw;
- struct cmdq_map_tc_to_cos req;
- struct creq_map_tc_to_cos_resp resp;
- u16 cmd_flags = 0;
-
- RCFW_CMD_PREP(req, MAP_TC_TO_COS, cmd_flags);
- req.cos0 = cpu_to_le16(cids[0]);
- req.cos1 = cpu_to_le16(cids[1]);
-
- return bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
- NULL, 0);
-}
-
int bnxt_qplib_get_roce_stats(struct bnxt_qplib_rcfw *rcfw,
struct bnxt_qplib_roce_stats *stats)
{
- struct cmdq_query_roce_stats req;
- struct creq_query_roce_stats_resp resp;
- struct bnxt_qplib_rcfw_sbuf *sbuf;
+ struct creq_query_roce_stats_resp resp = {};
struct creq_query_roce_stats_resp_sb *sb;
- u16 cmd_flags = 0;
+ struct cmdq_query_roce_stats req = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct bnxt_qplib_rcfw_sbuf *sbuf;
int rc = 0;
- RCFW_CMD_PREP(req, QUERY_ROCE_STATS, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_BASE_OPCODE_QUERY_ROCE_STATS,
+ sizeof(req));
sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));
if (!sbuf) {
@@ -715,8 +738,9 @@ int bnxt_qplib_get_roce_stats(struct bnxt_qplib_rcfw *rcfw,
sb = sbuf->sb;
req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
- (void *)sbuf, 0);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, sbuf, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc)
goto bail;
/* Extract the context from the side buffer */
@@ -780,8 +804,8 @@ int bnxt_qplib_qext_stat(struct bnxt_qplib_rcfw *rcfw, u32 fid,
struct creq_query_roce_stats_ext_resp resp = {};
struct creq_query_roce_stats_ext_resp_sb *sb;
struct cmdq_query_roce_stats_ext req = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
struct bnxt_qplib_rcfw_sbuf *sbuf;
- u16 cmd_flags = 0;
int rc;
sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));
@@ -791,15 +815,18 @@ int bnxt_qplib_qext_stat(struct bnxt_qplib_rcfw *rcfw, u32 fid,
return -ENOMEM;
}
- RCFW_CMD_PREP(req, QUERY_ROCE_STATS_EXT, cmd_flags);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
+ CMDQ_QUERY_ROCE_STATS_EXT_OPCODE_QUERY_ROCE_STATS,
+ sizeof(req));
req.resp_size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS);
req.resp_addr = cpu_to_le64(sbuf->dma_addr);
req.function_id = cpu_to_le32(fid);
req.flags = cpu_to_le16(CMDQ_QUERY_ROCE_STATS_EXT_FLAGS_FUNCTION_ID);
- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
- (void *)&resp, (void *)sbuf, 0);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, sbuf, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc)
goto bail;
@@ -823,3 +850,111 @@ bail:
bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf);
return rc;
}
+
+static void bnxt_qplib_fill_cc_gen1(struct cmdq_modify_roce_cc_gen1_tlv *ext_req,
+ struct bnxt_qplib_cc_param_ext *cc_ext)
+{
+ ext_req->modify_mask = cpu_to_le64(cc_ext->ext_mask);
+ cc_ext->ext_mask = 0;
+ ext_req->inactivity_th_hi = cpu_to_le16(cc_ext->inact_th_hi);
+ ext_req->min_time_between_cnps = cpu_to_le16(cc_ext->min_delta_cnp);
+ ext_req->init_cp = cpu_to_le16(cc_ext->init_cp);
+ ext_req->tr_update_mode = cc_ext->tr_update_mode;
+ ext_req->tr_update_cycles = cc_ext->tr_update_cyls;
+ ext_req->fr_num_rtts = cc_ext->fr_rtt;
+ ext_req->ai_rate_increase = cc_ext->ai_rate_incr;
+ ext_req->reduction_relax_rtts_th = cpu_to_le16(cc_ext->rr_rtt_th);
+ ext_req->additional_relax_cr_th = cpu_to_le16(cc_ext->ar_cr_th);
+ ext_req->cr_min_th = cpu_to_le16(cc_ext->cr_min_th);
+ ext_req->bw_avg_weight = cc_ext->bw_avg_weight;
+ ext_req->actual_cr_factor = cc_ext->cr_factor;
+ ext_req->max_cp_cr_th = cpu_to_le16(cc_ext->cr_th_max_cp);
+ ext_req->cp_bias_en = cc_ext->cp_bias_en;
+ ext_req->cp_bias = cc_ext->cp_bias;
+ ext_req->cnp_ecn = cc_ext->cnp_ecn;
+ ext_req->rtt_jitter_en = cc_ext->rtt_jitter_en;
+ ext_req->link_bytes_per_usec = cpu_to_le16(cc_ext->bytes_per_usec);
+ ext_req->reset_cc_cr_th = cpu_to_le16(cc_ext->cc_cr_reset_th);
+ ext_req->cr_width = cc_ext->cr_width;
+ ext_req->quota_period_min = cc_ext->min_quota;
+ ext_req->quota_period_max = cc_ext->max_quota;
+ ext_req->quota_period_abs_max = cc_ext->abs_max_quota;
+ ext_req->tr_lower_bound = cpu_to_le16(cc_ext->tr_lb);
+ ext_req->cr_prob_factor = cc_ext->cr_prob_fac;
+ ext_req->tr_prob_factor = cc_ext->tr_prob_fac;
+ ext_req->fairness_cr_th = cpu_to_le16(cc_ext->fair_cr_th);
+ ext_req->red_div = cc_ext->red_div;
+ ext_req->cnp_ratio_th = cc_ext->cnp_ratio_th;
+ ext_req->exp_ai_rtts = cpu_to_le16(cc_ext->ai_ext_rtt);
+ ext_req->exp_ai_cr_cp_ratio = cc_ext->exp_crcp_ratio;
+ ext_req->use_rate_table = cc_ext->low_rate_en;
+ ext_req->cp_exp_update_th = cpu_to_le16(cc_ext->cpcr_update_th);
+ ext_req->high_exp_ai_rtts_th1 = cpu_to_le16(cc_ext->ai_rtt_th1);
+ ext_req->high_exp_ai_rtts_th2 = cpu_to_le16(cc_ext->ai_rtt_th2);
+ ext_req->actual_cr_cong_free_rtts_th = cpu_to_le16(cc_ext->cf_rtt_th);
+ ext_req->severe_cong_cr_th1 = cpu_to_le16(cc_ext->sc_cr_th1);
+ ext_req->severe_cong_cr_th2 = cpu_to_le16(cc_ext->sc_cr_th2);
+ ext_req->link64B_per_rtt = cpu_to_le32(cc_ext->l64B_per_rtt);
+ ext_req->cc_ack_bytes = cc_ext->cc_ack_bytes;
+}
+
+int bnxt_qplib_modify_cc(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_cc_param *cc_param)
+{
+ struct bnxt_qplib_tlv_modify_cc_req tlv_req = {};
+ struct creq_modify_roce_cc_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_modify_roce_cc *req;
+ int req_size;
+ void *cmd;
+ int rc;
+
+ /* Prepare the older base command */
+ req = &tlv_req.base_req;
+ cmd = req;
+ req_size = sizeof(*req);
+ bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)req, CMDQ_BASE_OPCODE_MODIFY_ROCE_CC,
+ sizeof(*req));
+ req->modify_mask = cpu_to_le32(cc_param->mask);
+ req->enable_cc = cc_param->enable;
+ req->g = cc_param->g;
+ req->num_phases_per_state = cc_param->nph_per_state;
+ req->time_per_phase = cc_param->time_pph;
+ req->pkts_per_phase = cc_param->pkts_pph;
+ req->init_cr = cpu_to_le16(cc_param->init_cr);
+ req->init_tr = cpu_to_le16(cc_param->init_tr);
+ req->tos_dscp_tos_ecn = (cc_param->tos_dscp << CMDQ_MODIFY_ROCE_CC_TOS_DSCP_SFT) |
+ (cc_param->tos_ecn & CMDQ_MODIFY_ROCE_CC_TOS_ECN_MASK);
+ req->alt_vlan_pcp = cc_param->alt_vlan_pcp;
+ req->alt_tos_dscp = cpu_to_le16(cc_param->alt_tos_dscp);
+ req->rtt = cpu_to_le16(cc_param->rtt);
+ req->tcp_cp = cpu_to_le16(cc_param->tcp_cp);
+ req->cc_mode = cc_param->cc_mode;
+ req->inactivity_th = cpu_to_le16(cc_param->inact_th);
+
+ /* For chip gen P5 onwards fill extended cmd and header */
+ if (bnxt_qplib_is_chip_gen_p5(res->cctx)) {
+ struct roce_tlv *hdr;
+ u32 payload;
+ u32 chunks;
+
+ cmd = &tlv_req;
+ req_size = sizeof(tlv_req);
+ /* Prepare primary tlv header */
+ hdr = &tlv_req.tlv_hdr;
+ chunks = CHUNKS(sizeof(struct bnxt_qplib_tlv_modify_cc_req));
+ payload = sizeof(struct cmdq_modify_roce_cc);
+ __roce_1st_tlv_prep(hdr, chunks, payload, true);
+ /* Prepare secondary tlv header */
+ hdr = (struct roce_tlv *)&tlv_req.ext_req;
+ payload = sizeof(struct cmdq_modify_roce_cc_gen1_tlv) -
+ sizeof(struct roce_tlv);
+ __roce_ext_tlv_prep(hdr, TLV_TYPE_MODIFY_ROCE_CC_GEN1, payload, false, true);
+ bnxt_qplib_fill_cc_gen1(&tlv_req.ext_req, &cc_param->cc_ext);
+ }
+
+ bnxt_qplib_fill_cmdqmsg(&msg, cmd, &resp, NULL, req_size,
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(res->rcfw, &msg);
+ return rc;
+}
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h
index 5939e8fc8353..5de874659cdf 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_sp.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.h
@@ -244,6 +244,71 @@ struct bnxt_qplib_ext_stat {
u64 rx_ecn_marked;
};
+struct bnxt_qplib_cc_param_ext {
+ u64 ext_mask;
+ u16 inact_th_hi;
+ u16 min_delta_cnp;
+ u16 init_cp;
+ u8 tr_update_mode;
+ u8 tr_update_cyls;
+ u8 fr_rtt;
+ u8 ai_rate_incr;
+ u16 rr_rtt_th;
+ u16 ar_cr_th;
+ u16 cr_min_th;
+ u8 bw_avg_weight;
+ u8 cr_factor;
+ u16 cr_th_max_cp;
+ u8 cp_bias_en;
+ u8 cp_bias;
+ u8 cnp_ecn;
+ u8 rtt_jitter_en;
+ u16 bytes_per_usec;
+ u16 cc_cr_reset_th;
+ u8 cr_width;
+ u8 min_quota;
+ u8 max_quota;
+ u8 abs_max_quota;
+ u16 tr_lb;
+ u8 cr_prob_fac;
+ u8 tr_prob_fac;
+ u16 fair_cr_th;
+ u8 red_div;
+ u8 cnp_ratio_th;
+ u16 ai_ext_rtt;
+ u8 exp_crcp_ratio;
+ u8 low_rate_en;
+ u16 cpcr_update_th;
+ u16 ai_rtt_th1;
+ u16 ai_rtt_th2;
+ u16 cf_rtt_th;
+ u16 sc_cr_th1; /* severe congestion cr threshold 1 */
+ u16 sc_cr_th2; /* severe congestion cr threshold 2 */
+ u32 l64B_per_rtt;
+ u8 cc_ack_bytes;
+ u16 reduce_cf_rtt_th;
+};
+
+struct bnxt_qplib_cc_param {
+ u8 alt_vlan_pcp;
+ u16 alt_tos_dscp;
+ u8 cc_mode;
+ u8 enable;
+ u16 inact_th;
+ u16 init_cr;
+ u16 init_tr;
+ u16 rtt;
+ u8 g;
+ u8 nph_per_state;
+ u8 time_pph;
+ u8 pkts_pph;
+ u8 tos_ecn;
+ u8 tos_dscp;
+ u16 tcp_cp;
+ struct bnxt_qplib_cc_param_ext cc_ext;
+ u32 mask;
+};
+
int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res,
struct bnxt_qplib_sgid_tbl *sgid_tbl, int index,
struct bnxt_qplib_gid *gid);
@@ -277,10 +342,11 @@ int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res,
struct bnxt_qplib_frpl *frpl, int max);
int bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res,
struct bnxt_qplib_frpl *frpl);
-int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids);
int bnxt_qplib_get_roce_stats(struct bnxt_qplib_rcfw *rcfw,
struct bnxt_qplib_roce_stats *stats);
int bnxt_qplib_qext_stat(struct bnxt_qplib_rcfw *rcfw, u32 fid,
struct bnxt_qplib_ext_stat *estat);
+int bnxt_qplib_modify_cc(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_cc_param *cc_param);
#endif /* __BNXT_QPLIB_SP_H__*/
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_tlv.h b/drivers/infiniband/hw/bnxt_re/qplib_tlv.h
new file mode 100644
index 000000000000..402c220734f6
--- /dev/null
+++ b/drivers/infiniband/hw/bnxt_re/qplib_tlv.h
@@ -0,0 +1,162 @@
+/* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */
+
+#ifndef __QPLIB_TLV_H__
+#define __QPLIB_TLV_H__
+
+struct roce_tlv {
+ struct tlv tlv;
+ u8 total_size; // in units of 16 byte chunks
+ u8 unused[7]; // for 16 byte alignment
+};
+
+#define CHUNK_SIZE 16
+#define CHUNKS(x) (((x) + CHUNK_SIZE - 1) / CHUNK_SIZE)
+
+static inline void __roce_1st_tlv_prep(struct roce_tlv *rtlv, u8 tot_chunks,
+ u16 content_bytes, u8 flags)
+{
+ rtlv->tlv.cmd_discr = cpu_to_le16(CMD_DISCR_TLV_ENCAP);
+ rtlv->tlv.tlv_type = cpu_to_le16(TLV_TYPE_ROCE_SP_COMMAND);
+ rtlv->tlv.length = cpu_to_le16(content_bytes);
+ rtlv->tlv.flags = TLV_FLAGS_REQUIRED;
+ rtlv->tlv.flags |= flags ? TLV_FLAGS_MORE : 0;
+ rtlv->total_size = (tot_chunks);
+}
+
+static inline void __roce_ext_tlv_prep(struct roce_tlv *rtlv, u16 tlv_type,
+ u16 content_bytes, u8 more, u8 flags)
+{
+ rtlv->tlv.cmd_discr = cpu_to_le16(CMD_DISCR_TLV_ENCAP);
+ rtlv->tlv.tlv_type = cpu_to_le16(tlv_type);
+ rtlv->tlv.length = cpu_to_le16(content_bytes);
+ rtlv->tlv.flags |= more ? TLV_FLAGS_MORE : 0;
+ rtlv->tlv.flags |= flags ? TLV_FLAGS_REQUIRED : 0;
+}
+
+/*
+ * TLV size in units of 16 byte chunks
+ */
+#define TLV_SIZE ((sizeof(struct roce_tlv) + 15) / 16)
+/*
+ * TLV length in bytes
+ */
+#define TLV_BYTES (TLV_SIZE * 16)
+
+#define HAS_TLV_HEADER(msg) (le16_to_cpu(((struct tlv *)(msg))->cmd_discr) == CMD_DISCR_TLV_ENCAP)
+#define GET_TLV_DATA(tlv) ((void *)&((uint8_t *)(tlv))[TLV_BYTES])
+
+static inline u8 __get_cmdq_base_opcode(struct cmdq_base *req, u32 size)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ return ((struct cmdq_base *)GET_TLV_DATA(req))->opcode;
+ else
+ return req->opcode;
+}
+
+static inline void __set_cmdq_base_opcode(struct cmdq_base *req,
+ u32 size, u8 val)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ ((struct cmdq_base *)GET_TLV_DATA(req))->opcode = val;
+ else
+ req->opcode = val;
+}
+
+static inline __le16 __get_cmdq_base_cookie(struct cmdq_base *req, u32 size)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ return ((struct cmdq_base *)GET_TLV_DATA(req))->cookie;
+ else
+ return req->cookie;
+}
+
+static inline void __set_cmdq_base_cookie(struct cmdq_base *req,
+ u32 size, __le16 val)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ ((struct cmdq_base *)GET_TLV_DATA(req))->cookie = val;
+ else
+ req->cookie = val;
+}
+
+static inline __le64 __get_cmdq_base_resp_addr(struct cmdq_base *req, u32 size)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ return ((struct cmdq_base *)GET_TLV_DATA(req))->resp_addr;
+ else
+ return req->resp_addr;
+}
+
+static inline void __set_cmdq_base_resp_addr(struct cmdq_base *req,
+ u32 size, __le64 val)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ ((struct cmdq_base *)GET_TLV_DATA(req))->resp_addr = val;
+ else
+ req->resp_addr = val;
+}
+
+static inline u8 __get_cmdq_base_resp_size(struct cmdq_base *req, u32 size)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ return ((struct cmdq_base *)GET_TLV_DATA(req))->resp_size;
+ else
+ return req->resp_size;
+}
+
+static inline void __set_cmdq_base_resp_size(struct cmdq_base *req,
+ u32 size, u8 val)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ ((struct cmdq_base *)GET_TLV_DATA(req))->resp_size = val;
+ else
+ req->resp_size = val;
+}
+
+static inline u8 __get_cmdq_base_cmd_size(struct cmdq_base *req, u32 size)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ return ((struct roce_tlv *)(req))->total_size;
+ else
+ return req->cmd_size;
+}
+
+static inline void __set_cmdq_base_cmd_size(struct cmdq_base *req,
+ u32 size, u8 val)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ ((struct cmdq_base *)GET_TLV_DATA(req))->cmd_size = val;
+ else
+ req->cmd_size = val;
+}
+
+static inline __le16 __get_cmdq_base_flags(struct cmdq_base *req, u32 size)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ return ((struct cmdq_base *)GET_TLV_DATA(req))->flags;
+ else
+ return req->flags;
+}
+
+static inline void __set_cmdq_base_flags(struct cmdq_base *req,
+ u32 size, __le16 val)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ ((struct cmdq_base *)GET_TLV_DATA(req))->flags = val;
+ else
+ req->flags = val;
+}
+
+struct bnxt_qplib_tlv_modify_cc_req {
+ struct roce_tlv tlv_hdr;
+ struct cmdq_modify_roce_cc base_req;
+ __le64 tlvpad;
+ struct cmdq_modify_roce_cc_gen1_tlv ext_req;
+};
+
+struct bnxt_qplib_tlv_query_rcc_sb {
+ struct roce_tlv tlv_hdr;
+ struct creq_query_roce_cc_resp_sb base_sb;
+ struct creq_query_roce_cc_gen1_resp_sb_tlv gen1_sb;
+};
+#endif /* __QPLIB_TLV_H__ */
diff --git a/drivers/infiniband/hw/bnxt_re/roce_hsi.h b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
index ecb719098b75..4a10303e0392 100644
--- a/drivers/infiniband/hw/bnxt_re/roce_hsi.h
+++ b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
@@ -42,1505 +42,1334 @@
/* include bnxt_hsi.h from bnxt_en driver */
#include "bnxt_hsi.h"
-/* CMP Door Bell Format (4 bytes) */
+/* tx_doorbell (size:32b/4B) */
+struct tx_doorbell {
+ __le32 key_idx;
+ #define TX_DOORBELL_IDX_MASK 0xffffffUL
+ #define TX_DOORBELL_IDX_SFT 0
+ #define TX_DOORBELL_KEY_MASK 0xf0000000UL
+ #define TX_DOORBELL_KEY_SFT 28
+ #define TX_DOORBELL_KEY_TX (0x0UL << 28)
+ #define TX_DOORBELL_KEY_LAST TX_DOORBELL_KEY_TX
+};
+
+/* rx_doorbell (size:32b/4B) */
+struct rx_doorbell {
+ __le32 key_idx;
+ #define RX_DOORBELL_IDX_MASK 0xffffffUL
+ #define RX_DOORBELL_IDX_SFT 0
+ #define RX_DOORBELL_KEY_MASK 0xf0000000UL
+ #define RX_DOORBELL_KEY_SFT 28
+ #define RX_DOORBELL_KEY_RX (0x1UL << 28)
+ #define RX_DOORBELL_KEY_LAST RX_DOORBELL_KEY_RX
+};
+
+/* cmpl_doorbell (size:32b/4B) */
struct cmpl_doorbell {
- __le32 key_mask_valid_idx;
- #define CMPL_DOORBELL_IDX_MASK 0xffffffUL
- #define CMPL_DOORBELL_IDX_SFT 0
- #define CMPL_DOORBELL_RESERVED_MASK 0x3000000UL
- #define CMPL_DOORBELL_RESERVED_SFT 24
- #define CMPL_DOORBELL_IDX_VALID 0x4000000UL
- #define CMPL_DOORBELL_MASK 0x8000000UL
- #define CMPL_DOORBELL_KEY_MASK 0xf0000000UL
- #define CMPL_DOORBELL_KEY_SFT 28
- #define CMPL_DOORBELL_KEY_CMPL (0x2UL << 28)
-};
-
-/* Status Door Bell Format (4 bytes) */
-struct status_doorbell {
- __le32 key_idx;
- #define STATUS_DOORBELL_IDX_MASK 0xffffffUL
- #define STATUS_DOORBELL_IDX_SFT 0
- #define STATUS_DOORBELL_RESERVED_MASK 0xf000000UL
- #define STATUS_DOORBELL_RESERVED_SFT 24
- #define STATUS_DOORBELL_KEY_MASK 0xf0000000UL
- #define STATUS_DOORBELL_KEY_SFT 28
- #define STATUS_DOORBELL_KEY_STAT (0x3UL << 28)
-};
-
-/* RoCE Host Structures */
-
-/* Doorbell Structures */
-/* dbc_dbc (size:64b/8B) */
-struct dbc_dbc {
- __le32 index;
- #define DBC_DBC_INDEX_MASK 0xffffffUL
- #define DBC_DBC_INDEX_SFT 0
- __le32 type_path_xid;
- #define DBC_DBC_XID_MASK 0xfffffUL
- #define DBC_DBC_XID_SFT 0
- #define DBC_DBC_PATH_MASK 0x3000000UL
- #define DBC_DBC_PATH_SFT 24
- #define DBC_DBC_PATH_ROCE (0x0UL << 24)
- #define DBC_DBC_PATH_L2 (0x1UL << 24)
- #define DBC_DBC_PATH_ENGINE (0x2UL << 24)
- #define DBC_DBC_PATH_LAST DBC_DBC_PATH_ENGINE
- #define DBC_DBC_DEBUG_TRACE 0x8000000UL
- #define DBC_DBC_TYPE_MASK 0xf0000000UL
- #define DBC_DBC_TYPE_SFT 28
- #define DBC_DBC_TYPE_SQ (0x0UL << 28)
- #define DBC_DBC_TYPE_RQ (0x1UL << 28)
- #define DBC_DBC_TYPE_SRQ (0x2UL << 28)
- #define DBC_DBC_TYPE_SRQ_ARM (0x3UL << 28)
- #define DBC_DBC_TYPE_CQ (0x4UL << 28)
- #define DBC_DBC_TYPE_CQ_ARMSE (0x5UL << 28)
- #define DBC_DBC_TYPE_CQ_ARMALL (0x6UL << 28)
- #define DBC_DBC_TYPE_CQ_ARMENA (0x7UL << 28)
- #define DBC_DBC_TYPE_SRQ_ARMENA (0x8UL << 28)
- #define DBC_DBC_TYPE_CQ_CUTOFF_ACK (0x9UL << 28)
- #define DBC_DBC_TYPE_NQ (0xaUL << 28)
- #define DBC_DBC_TYPE_NQ_ARM (0xbUL << 28)
- #define DBC_DBC_TYPE_NULL (0xfUL << 28)
- #define DBC_DBC_TYPE_LAST DBC_DBC_TYPE_NULL
-};
-
-/* dbc_dbc32 (size:32b/4B) */
-struct dbc_dbc32 {
- __le32 type_abs_incr_xid;
- #define DBC_DBC32_XID_MASK 0xfffffUL
- #define DBC_DBC32_XID_SFT 0
- #define DBC_DBC32_PATH_MASK 0xc00000UL
- #define DBC_DBC32_PATH_SFT 22
- #define DBC_DBC32_PATH_ROCE (0x0UL << 22)
- #define DBC_DBC32_PATH_L2 (0x1UL << 22)
- #define DBC_DBC32_PATH_LAST DBC_DBC32_PATH_L2
- #define DBC_DBC32_INCR_MASK 0xf000000UL
- #define DBC_DBC32_INCR_SFT 24
- #define DBC_DBC32_ABS 0x10000000UL
- #define DBC_DBC32_TYPE_MASK 0xe0000000UL
- #define DBC_DBC32_TYPE_SFT 29
- #define DBC_DBC32_TYPE_SQ (0x0UL << 29)
- #define DBC_DBC32_TYPE_LAST DBC_DBC32_TYPE_SQ
-};
-
-/* SQ WQE Structures */
-/* Base SQ WQE (8 bytes) */
-struct sq_base {
- u8 wqe_type;
- #define SQ_BASE_WQE_TYPE_SEND 0x0UL
- #define SQ_BASE_WQE_TYPE_SEND_W_IMMEAD 0x1UL
- #define SQ_BASE_WQE_TYPE_SEND_W_INVALID 0x2UL
- #define SQ_BASE_WQE_TYPE_WRITE_WQE 0x4UL
- #define SQ_BASE_WQE_TYPE_WRITE_W_IMMEAD 0x5UL
- #define SQ_BASE_WQE_TYPE_READ_WQE 0x6UL
- #define SQ_BASE_WQE_TYPE_ATOMIC_CS 0x8UL
- #define SQ_BASE_WQE_TYPE_ATOMIC_FA 0xbUL
- #define SQ_BASE_WQE_TYPE_LOCAL_INVALID 0xcUL
- #define SQ_BASE_WQE_TYPE_FR_PMR 0xdUL
- #define SQ_BASE_WQE_TYPE_BIND 0xeUL
- u8 unused_0[7];
-};
-
-/* WQE SGE (16 bytes) */
-struct sq_sge {
- __le64 va_or_pa;
- __le32 l_key;
- __le32 size;
+ __le32 key_mask_valid_idx;
+ #define CMPL_DOORBELL_IDX_MASK 0xffffffUL
+ #define CMPL_DOORBELL_IDX_SFT 0
+ #define CMPL_DOORBELL_IDX_VALID 0x4000000UL
+ #define CMPL_DOORBELL_MASK 0x8000000UL
+ #define CMPL_DOORBELL_KEY_MASK 0xf0000000UL
+ #define CMPL_DOORBELL_KEY_SFT 28
+ #define CMPL_DOORBELL_KEY_CMPL (0x2UL << 28)
+ #define CMPL_DOORBELL_KEY_LAST CMPL_DOORBELL_KEY_CMPL
};
-/* PSN Search Structure (8 bytes) */
-struct sq_psn_search {
- __le32 opcode_start_psn;
- #define SQ_PSN_SEARCH_START_PSN_MASK 0xffffffUL
- #define SQ_PSN_SEARCH_START_PSN_SFT 0
- #define SQ_PSN_SEARCH_OPCODE_MASK 0xff000000UL
- #define SQ_PSN_SEARCH_OPCODE_SFT 24
- __le32 flags_next_psn;
- #define SQ_PSN_SEARCH_NEXT_PSN_MASK 0xffffffUL
- #define SQ_PSN_SEARCH_NEXT_PSN_SFT 0
- #define SQ_PSN_SEARCH_FLAGS_MASK 0xff000000UL
- #define SQ_PSN_SEARCH_FLAGS_SFT 24
+/* status_doorbell (size:32b/4B) */
+struct status_doorbell {
+ __le32 key_idx;
+ #define STATUS_DOORBELL_IDX_MASK 0xffffffUL
+ #define STATUS_DOORBELL_IDX_SFT 0
+ #define STATUS_DOORBELL_KEY_MASK 0xf0000000UL
+ #define STATUS_DOORBELL_KEY_SFT 28
+ #define STATUS_DOORBELL_KEY_STAT (0x3UL << 28)
+ #define STATUS_DOORBELL_KEY_LAST STATUS_DOORBELL_KEY_STAT
};
-/* sq_psn_search_ext (size:128b/16B) */
-struct sq_psn_search_ext {
- __le32 opcode_start_psn;
- #define SQ_PSN_SEARCH_EXT_START_PSN_MASK 0xffffffUL
- #define SQ_PSN_SEARCH_EXT_START_PSN_SFT 0
- #define SQ_PSN_SEARCH_EXT_OPCODE_MASK 0xff000000UL
- #define SQ_PSN_SEARCH_EXT_OPCODE_SFT 24
- __le32 flags_next_psn;
- #define SQ_PSN_SEARCH_EXT_NEXT_PSN_MASK 0xffffffUL
- #define SQ_PSN_SEARCH_EXT_NEXT_PSN_SFT 0
- #define SQ_PSN_SEARCH_EXT_FLAGS_MASK 0xff000000UL
- #define SQ_PSN_SEARCH_EXT_FLAGS_SFT 24
- __le16 start_slot_idx;
- __le16 reserved16;
- __le32 reserved32;
-};
-
-/* Send SQ WQE (40 bytes) */
-struct sq_send {
- u8 wqe_type;
- #define SQ_SEND_WQE_TYPE_SEND 0x0UL
- #define SQ_SEND_WQE_TYPE_SEND_W_IMMEAD 0x1UL
- #define SQ_SEND_WQE_TYPE_SEND_W_INVALID 0x2UL
- u8 flags;
- #define SQ_SEND_FLAGS_SIGNAL_COMP 0x1UL
- #define SQ_SEND_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
- #define SQ_SEND_FLAGS_UC_FENCE 0x4UL
- #define SQ_SEND_FLAGS_SE 0x8UL
- #define SQ_SEND_FLAGS_INLINE 0x10UL
- u8 wqe_size;
- u8 reserved8_1;
- __le32 inv_key_or_imm_data;
- __le32 length;
- __le32 q_key;
- __le32 dst_qp;
- #define SQ_SEND_DST_QP_MASK 0xffffffUL
- #define SQ_SEND_DST_QP_SFT 0
- #define SQ_SEND_RESERVED8_2_MASK 0xff000000UL
- #define SQ_SEND_RESERVED8_2_SFT 24
- __le32 avid;
- #define SQ_SEND_AVID_MASK 0xfffffUL
- #define SQ_SEND_AVID_SFT 0
- #define SQ_SEND_RESERVED_AVID_MASK 0xfff00000UL
- #define SQ_SEND_RESERVED_AVID_SFT 20
- __le64 reserved64;
- __le32 data[24];
+/* cmdq_init (size:128b/16B) */
+struct cmdq_init {
+ __le64 cmdq_pbl;
+ __le16 cmdq_size_cmdq_lvl;
+ #define CMDQ_INIT_CMDQ_LVL_MASK 0x3UL
+ #define CMDQ_INIT_CMDQ_LVL_SFT 0
+ #define CMDQ_INIT_CMDQ_SIZE_MASK 0xfffcUL
+ #define CMDQ_INIT_CMDQ_SIZE_SFT 2
+ __le16 creq_ring_id;
+ __le32 prod_idx;
};
-/* sq_send_hdr (size:256b/32B) */
-struct sq_send_hdr {
- u8 wqe_type;
- u8 flags;
- u8 wqe_size;
- u8 reserved8_1;
- __le32 inv_key_or_imm_data;
- __le32 length;
- __le32 q_key;
- __le32 dst_qp;
- __le32 avid;
- __le64 reserved64;
+/* cmdq_base (size:128b/16B) */
+struct cmdq_base {
+ u8 opcode;
+ #define CMDQ_BASE_OPCODE_CREATE_QP 0x1UL
+ #define CMDQ_BASE_OPCODE_DESTROY_QP 0x2UL
+ #define CMDQ_BASE_OPCODE_MODIFY_QP 0x3UL
+ #define CMDQ_BASE_OPCODE_QUERY_QP 0x4UL
+ #define CMDQ_BASE_OPCODE_CREATE_SRQ 0x5UL
+ #define CMDQ_BASE_OPCODE_DESTROY_SRQ 0x6UL
+ #define CMDQ_BASE_OPCODE_QUERY_SRQ 0x8UL
+ #define CMDQ_BASE_OPCODE_CREATE_CQ 0x9UL
+ #define CMDQ_BASE_OPCODE_DESTROY_CQ 0xaUL
+ #define CMDQ_BASE_OPCODE_RESIZE_CQ 0xcUL
+ #define CMDQ_BASE_OPCODE_ALLOCATE_MRW 0xdUL
+ #define CMDQ_BASE_OPCODE_DEALLOCATE_KEY 0xeUL
+ #define CMDQ_BASE_OPCODE_REGISTER_MR 0xfUL
+ #define CMDQ_BASE_OPCODE_DEREGISTER_MR 0x10UL
+ #define CMDQ_BASE_OPCODE_ADD_GID 0x11UL
+ #define CMDQ_BASE_OPCODE_DELETE_GID 0x12UL
+ #define CMDQ_BASE_OPCODE_MODIFY_GID 0x17UL
+ #define CMDQ_BASE_OPCODE_QUERY_GID 0x18UL
+ #define CMDQ_BASE_OPCODE_CREATE_QP1 0x13UL
+ #define CMDQ_BASE_OPCODE_DESTROY_QP1 0x14UL
+ #define CMDQ_BASE_OPCODE_CREATE_AH 0x15UL
+ #define CMDQ_BASE_OPCODE_DESTROY_AH 0x16UL
+ #define CMDQ_BASE_OPCODE_INITIALIZE_FW 0x80UL
+ #define CMDQ_BASE_OPCODE_DEINITIALIZE_FW 0x81UL
+ #define CMDQ_BASE_OPCODE_STOP_FUNC 0x82UL
+ #define CMDQ_BASE_OPCODE_QUERY_FUNC 0x83UL
+ #define CMDQ_BASE_OPCODE_SET_FUNC_RESOURCES 0x84UL
+ #define CMDQ_BASE_OPCODE_READ_CONTEXT 0x85UL
+ #define CMDQ_BASE_OPCODE_VF_BACKCHANNEL_REQUEST 0x86UL
+ #define CMDQ_BASE_OPCODE_READ_VF_MEMORY 0x87UL
+ #define CMDQ_BASE_OPCODE_COMPLETE_VF_REQUEST 0x88UL
+ #define CMDQ_BASE_OPCODE_EXTEND_CONTEXT_ARRRAY 0x89UL
+ #define CMDQ_BASE_OPCODE_MAP_TC_TO_COS 0x8aUL
+ #define CMDQ_BASE_OPCODE_QUERY_VERSION 0x8bUL
+ #define CMDQ_BASE_OPCODE_MODIFY_ROCE_CC 0x8cUL
+ #define CMDQ_BASE_OPCODE_QUERY_ROCE_CC 0x8dUL
+ #define CMDQ_BASE_OPCODE_QUERY_ROCE_STATS 0x8eUL
+ #define CMDQ_BASE_OPCODE_SET_LINK_AGGR_MODE 0x8fUL
+ #define CMDQ_BASE_OPCODE_MODIFY_CQ 0x90UL
+ #define CMDQ_BASE_OPCODE_QUERY_QP_EXTEND 0x91UL
+ #define CMDQ_BASE_OPCODE_QUERY_ROCE_STATS_EXT 0x92UL
+ #define CMDQ_BASE_OPCODE_LAST CMDQ_BASE_OPCODE_QUERY_ROCE_STATS_EXT
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
};
-/* Send Raw Ethernet and QP1 SQ WQE (40 bytes) */
-struct sq_send_raweth_qp1 {
- u8 wqe_type;
- #define SQ_SEND_RAWETH_QP1_WQE_TYPE_SEND 0x0UL
- u8 flags;
- #define SQ_SEND_RAWETH_QP1_FLAGS_SIGNAL_COMP 0x1UL
- #define SQ_SEND_RAWETH_QP1_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
- #define SQ_SEND_RAWETH_QP1_FLAGS_UC_FENCE 0x4UL
- #define SQ_SEND_RAWETH_QP1_FLAGS_SE 0x8UL
- #define SQ_SEND_RAWETH_QP1_FLAGS_INLINE 0x10UL
- u8 wqe_size;
- u8 reserved8;
- __le16 lflags;
- #define SQ_SEND_RAWETH_QP1_LFLAGS_TCP_UDP_CHKSUM 0x1UL
- #define SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM 0x2UL
- #define SQ_SEND_RAWETH_QP1_LFLAGS_NOCRC 0x4UL
- #define SQ_SEND_RAWETH_QP1_LFLAGS_STAMP 0x8UL
- #define SQ_SEND_RAWETH_QP1_LFLAGS_T_IP_CHKSUM 0x10UL
- #define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_1 0x20UL
- #define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_2 0x40UL
- #define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_3 0x80UL
- #define SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC 0x100UL
- #define SQ_SEND_RAWETH_QP1_LFLAGS_FCOE_CRC 0x200UL
- __le16 cfa_action;
- __le32 length;
- __le32 reserved32_1;
- __le32 cfa_meta;
- #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK 0xfffUL
- #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT 0
- #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_DE 0x1000UL
- #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_MASK 0xe000UL
- #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_SFT 13
- #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_MASK 0x70000UL
- #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_SFT 16
- #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID88A8 (0x0UL << 16)
- #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID8100 (0x1UL << 16)
- #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9100 (0x2UL << 16)
- #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9200 (0x3UL << 16)
- #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9300 (0x4UL << 16)
- #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG (0x5UL << 16)
- #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_LAST \
- SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG
- #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_MASK 0xff80000UL
- #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_SFT 19
- #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_MASK 0xf0000000UL
- #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_SFT 28
- #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_NONE (0x0UL << 28)
- #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG (0x1UL << 28)
- #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_LAST \
- SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG
- __le32 reserved32_2;
- __le64 reserved64;
- __le32 data[24];
+/* creq_base (size:128b/16B) */
+struct creq_base {
+ u8 type;
+ #define CREQ_BASE_TYPE_MASK 0x3fUL
+ #define CREQ_BASE_TYPE_SFT 0
+ #define CREQ_BASE_TYPE_QP_EVENT 0x38UL
+ #define CREQ_BASE_TYPE_FUNC_EVENT 0x3aUL
+ #define CREQ_BASE_TYPE_LAST CREQ_BASE_TYPE_FUNC_EVENT
+ u8 reserved56[7];
+ u8 v;
+ #define CREQ_BASE_V 0x1UL
+ u8 event;
+ u8 reserved48[6];
};
-/* sq_send_raweth_qp1_hdr (size:256b/32B) */
-struct sq_send_raweth_qp1_hdr {
- u8 wqe_type;
- u8 flags;
- u8 wqe_size;
+/* cmdq_query_version (size:128b/16B) */
+struct cmdq_query_version {
+ u8 opcode;
+ #define CMDQ_QUERY_VERSION_OPCODE_QUERY_VERSION 0x8bUL
+ #define CMDQ_QUERY_VERSION_OPCODE_LAST CMDQ_QUERY_VERSION_OPCODE_QUERY_VERSION
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
u8 reserved8;
- __le16 lflags;
- __le16 cfa_action;
- __le32 length;
- __le32 reserved32_1;
- __le32 cfa_meta;
- __le32 reserved32_2;
- __le64 reserved64;
+ __le64 resp_addr;
};
-/* RDMA SQ WQE (40 bytes) */
-struct sq_rdma {
- u8 wqe_type;
- #define SQ_RDMA_WQE_TYPE_WRITE_WQE 0x4UL
- #define SQ_RDMA_WQE_TYPE_WRITE_W_IMMEAD 0x5UL
- #define SQ_RDMA_WQE_TYPE_READ_WQE 0x6UL
- u8 flags;
- #define SQ_RDMA_FLAGS_SIGNAL_COMP 0x1UL
- #define SQ_RDMA_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
- #define SQ_RDMA_FLAGS_UC_FENCE 0x4UL
- #define SQ_RDMA_FLAGS_SE 0x8UL
- #define SQ_RDMA_FLAGS_INLINE 0x10UL
- u8 wqe_size;
- u8 reserved8;
- __le32 imm_data;
- __le32 length;
- __le32 reserved32_1;
- __le64 remote_va;
- __le32 remote_key;
- __le32 reserved32_2;
- __le32 data[24];
+/* creq_query_version_resp (size:128b/16B) */
+struct creq_query_version_resp {
+ u8 type;
+ #define CREQ_QUERY_VERSION_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_QUERY_VERSION_RESP_TYPE_SFT 0
+ #define CREQ_QUERY_VERSION_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_QUERY_VERSION_RESP_TYPE_LAST CREQ_QUERY_VERSION_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ u8 fw_maj;
+ u8 fw_minor;
+ u8 fw_bld;
+ u8 fw_rsvd;
+ u8 v;
+ #define CREQ_QUERY_VERSION_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_QUERY_VERSION_RESP_EVENT_QUERY_VERSION 0x8bUL
+ #define CREQ_QUERY_VERSION_RESP_EVENT_LAST \
+ CREQ_QUERY_VERSION_RESP_EVENT_QUERY_VERSION
+ __le16 reserved16;
+ u8 intf_maj;
+ u8 intf_minor;
+ u8 intf_bld;
+ u8 intf_rsvd;
};
-/* sq_rdma_hdr (size:256b/32B) */
-struct sq_rdma_hdr {
- u8 wqe_type;
- u8 flags;
- u8 wqe_size;
+/* cmdq_initialize_fw (size:896b/112B) */
+struct cmdq_initialize_fw {
+ u8 opcode;
+ #define CMDQ_INITIALIZE_FW_OPCODE_INITIALIZE_FW 0x80UL
+ #define CMDQ_INITIALIZE_FW_OPCODE_LAST CMDQ_INITIALIZE_FW_OPCODE_INITIALIZE_FW
+ u8 cmd_size;
+ __le16 flags;
+ #define CMDQ_INITIALIZE_FW_FLAGS_MRAV_RESERVATION_SPLIT 0x1UL
+ #define CMDQ_INITIALIZE_FW_FLAGS_HW_REQUESTER_RETX_SUPPORTED 0x2UL
+ __le16 cookie;
+ u8 resp_size;
u8 reserved8;
- __le32 imm_data;
- __le32 length;
- __le32 reserved32_1;
- __le64 remote_va;
- __le32 remote_key;
- __le32 reserved32_2;
+ __le64 resp_addr;
+ u8 qpc_pg_size_qpc_lvl;
+ #define CMDQ_INITIALIZE_FW_QPC_LVL_MASK 0xfUL
+ #define CMDQ_INITIALIZE_FW_QPC_LVL_SFT 0
+ #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_0 0x0UL
+ #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_1 0x1UL
+ #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_2 0x2UL
+ #define CMDQ_INITIALIZE_FW_QPC_LVL_LAST CMDQ_INITIALIZE_FW_QPC_LVL_LVL_2
+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_MASK 0xf0UL
+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT 4
+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K (0x0UL << 4)
+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8K (0x1UL << 4)
+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_64K (0x2UL << 4)
+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_2M (0x3UL << 4)
+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8M (0x4UL << 4)
+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G (0x5UL << 4)
+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_LAST CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G
+ u8 mrw_pg_size_mrw_lvl;
+ #define CMDQ_INITIALIZE_FW_MRW_LVL_MASK 0xfUL
+ #define CMDQ_INITIALIZE_FW_MRW_LVL_SFT 0
+ #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_0 0x0UL
+ #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_1 0x1UL
+ #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_2 0x2UL
+ #define CMDQ_INITIALIZE_FW_MRW_LVL_LAST CMDQ_INITIALIZE_FW_MRW_LVL_LVL_2
+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_MASK 0xf0UL
+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_SFT 4
+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_4K (0x0UL << 4)
+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8K (0x1UL << 4)
+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_64K (0x2UL << 4)
+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_2M (0x3UL << 4)
+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8M (0x4UL << 4)
+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_1G (0x5UL << 4)
+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_LAST CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_1G
+ u8 srq_pg_size_srq_lvl;
+ #define CMDQ_INITIALIZE_FW_SRQ_LVL_MASK 0xfUL
+ #define CMDQ_INITIALIZE_FW_SRQ_LVL_SFT 0
+ #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_0 0x0UL
+ #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_1 0x1UL
+ #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_2 0x2UL
+ #define CMDQ_INITIALIZE_FW_SRQ_LVL_LAST CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_2
+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_MASK 0xf0UL
+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_SFT 4
+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_4K (0x0UL << 4)
+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8K (0x1UL << 4)
+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_64K (0x2UL << 4)
+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_2M (0x3UL << 4)
+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8M (0x4UL << 4)
+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_1G (0x5UL << 4)
+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_LAST CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_1G
+ u8 cq_pg_size_cq_lvl;
+ #define CMDQ_INITIALIZE_FW_CQ_LVL_MASK 0xfUL
+ #define CMDQ_INITIALIZE_FW_CQ_LVL_SFT 0
+ #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_0 0x0UL
+ #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_1 0x1UL
+ #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_2 0x2UL
+ #define CMDQ_INITIALIZE_FW_CQ_LVL_LAST CMDQ_INITIALIZE_FW_CQ_LVL_LVL_2
+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_MASK 0xf0UL
+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_SFT 4
+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_4K (0x0UL << 4)
+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8K (0x1UL << 4)
+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_64K (0x2UL << 4)
+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_2M (0x3UL << 4)
+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8M (0x4UL << 4)
+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_1G (0x5UL << 4)
+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_LAST CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_1G
+ u8 tqm_pg_size_tqm_lvl;
+ #define CMDQ_INITIALIZE_FW_TQM_LVL_MASK 0xfUL
+ #define CMDQ_INITIALIZE_FW_TQM_LVL_SFT 0
+ #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_0 0x0UL
+ #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_1 0x1UL
+ #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_2 0x2UL
+ #define CMDQ_INITIALIZE_FW_TQM_LVL_LAST CMDQ_INITIALIZE_FW_TQM_LVL_LVL_2
+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_MASK 0xf0UL
+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_SFT 4
+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_4K (0x0UL << 4)
+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8K (0x1UL << 4)
+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_64K (0x2UL << 4)
+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_2M (0x3UL << 4)
+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8M (0x4UL << 4)
+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_1G (0x5UL << 4)
+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_LAST CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_1G
+ u8 tim_pg_size_tim_lvl;
+ #define CMDQ_INITIALIZE_FW_TIM_LVL_MASK 0xfUL
+ #define CMDQ_INITIALIZE_FW_TIM_LVL_SFT 0
+ #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_0 0x0UL
+ #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_1 0x1UL
+ #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_2 0x2UL
+ #define CMDQ_INITIALIZE_FW_TIM_LVL_LAST CMDQ_INITIALIZE_FW_TIM_LVL_LVL_2
+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_MASK 0xf0UL
+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_SFT 4
+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_4K (0x0UL << 4)
+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8K (0x1UL << 4)
+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_64K (0x2UL << 4)
+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_2M (0x3UL << 4)
+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8M (0x4UL << 4)
+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_1G (0x5UL << 4)
+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_LAST CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_1G
+ __le16 log2_dbr_pg_size;
+ #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_MASK 0xfUL
+ #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_SFT 0
+ #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_4K 0x0UL
+ #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_8K 0x1UL
+ #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_16K 0x2UL
+ #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_32K 0x3UL
+ #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_64K 0x4UL
+ #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_128K 0x5UL
+ #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_256K 0x6UL
+ #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_512K 0x7UL
+ #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_1M 0x8UL
+ #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_2M 0x9UL
+ #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_4M 0xaUL
+ #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_8M 0xbUL
+ #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_16M 0xcUL
+ #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_32M 0xdUL
+ #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_64M 0xeUL
+ #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_128M 0xfUL
+ #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_LAST \
+ CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_128M
+ #define CMDQ_INITIALIZE_FW_RSVD_MASK 0xfff0UL
+ #define CMDQ_INITIALIZE_FW_RSVD_SFT 4
+ __le64 qpc_page_dir;
+ __le64 mrw_page_dir;
+ __le64 srq_page_dir;
+ __le64 cq_page_dir;
+ __le64 tqm_page_dir;
+ __le64 tim_page_dir;
+ __le32 number_of_qp;
+ __le32 number_of_mrw;
+ __le32 number_of_srq;
+ __le32 number_of_cq;
+ __le32 max_qp_per_vf;
+ __le32 max_mrw_per_vf;
+ __le32 max_srq_per_vf;
+ __le32 max_cq_per_vf;
+ __le32 max_gid_per_vf;
+ __le32 stat_ctx_id;
};
-/* Atomic SQ WQE (40 bytes) */
-struct sq_atomic {
- u8 wqe_type;
- #define SQ_ATOMIC_WQE_TYPE_ATOMIC_CS 0x8UL
- #define SQ_ATOMIC_WQE_TYPE_ATOMIC_FA 0xbUL
- u8 flags;
- #define SQ_ATOMIC_FLAGS_SIGNAL_COMP 0x1UL
- #define SQ_ATOMIC_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
- #define SQ_ATOMIC_FLAGS_UC_FENCE 0x4UL
- #define SQ_ATOMIC_FLAGS_SE 0x8UL
- #define SQ_ATOMIC_FLAGS_INLINE 0x10UL
- __le16 reserved16;
- __le32 remote_key;
- __le64 remote_va;
- __le64 swap_data;
- __le64 cmp_data;
- __le32 data[24];
+/* creq_initialize_fw_resp (size:128b/16B) */
+struct creq_initialize_fw_resp {
+ u8 type;
+ #define CREQ_INITIALIZE_FW_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_INITIALIZE_FW_RESP_TYPE_SFT 0
+ #define CREQ_INITIALIZE_FW_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_INITIALIZE_FW_RESP_TYPE_LAST CREQ_INITIALIZE_FW_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 reserved32;
+ u8 v;
+ #define CREQ_INITIALIZE_FW_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_INITIALIZE_FW_RESP_EVENT_INITIALIZE_FW 0x80UL
+ #define CREQ_INITIALIZE_FW_RESP_EVENT_LAST \
+ CREQ_INITIALIZE_FW_RESP_EVENT_INITIALIZE_FW
+ u8 reserved48[6];
};
-/* sq_atomic_hdr (size:256b/32B) */
-struct sq_atomic_hdr {
- u8 wqe_type;
- u8 flags;
- __le16 reserved16;
- __le32 remote_key;
- __le64 remote_va;
- __le64 swap_data;
- __le64 cmp_data;
+/* cmdq_deinitialize_fw (size:128b/16B) */
+struct cmdq_deinitialize_fw {
+ u8 opcode;
+ #define CMDQ_DEINITIALIZE_FW_OPCODE_DEINITIALIZE_FW 0x81UL
+ #define CMDQ_DEINITIALIZE_FW_OPCODE_LAST \
+ CMDQ_DEINITIALIZE_FW_OPCODE_DEINITIALIZE_FW
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
};
-/* Local Invalidate SQ WQE (40 bytes) */
-struct sq_localinvalidate {
- u8 wqe_type;
- #define SQ_LOCALINVALIDATE_WQE_TYPE_LOCAL_INVALID 0xcUL
- u8 flags;
- #define SQ_LOCALINVALIDATE_FLAGS_SIGNAL_COMP 0x1UL
- #define SQ_LOCALINVALIDATE_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
- #define SQ_LOCALINVALIDATE_FLAGS_UC_FENCE 0x4UL
- #define SQ_LOCALINVALIDATE_FLAGS_SE 0x8UL
- #define SQ_LOCALINVALIDATE_FLAGS_INLINE 0x10UL
- __le16 reserved16;
- __le32 inv_l_key;
- __le64 reserved64;
- __le32 reserved128[4];
- __le32 data[24];
+/* creq_deinitialize_fw_resp (size:128b/16B) */
+struct creq_deinitialize_fw_resp {
+ u8 type;
+ #define CREQ_DEINITIALIZE_FW_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_DEINITIALIZE_FW_RESP_TYPE_SFT 0
+ #define CREQ_DEINITIALIZE_FW_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_DEINITIALIZE_FW_RESP_TYPE_LAST CREQ_DEINITIALIZE_FW_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 reserved32;
+ u8 v;
+ #define CREQ_DEINITIALIZE_FW_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_DEINITIALIZE_FW_RESP_EVENT_DEINITIALIZE_FW 0x81UL
+ #define CREQ_DEINITIALIZE_FW_RESP_EVENT_LAST \
+ CREQ_DEINITIALIZE_FW_RESP_EVENT_DEINITIALIZE_FW
+ u8 reserved48[6];
};
-/* sq_localinvalidate_hdr (size:256b/32B) */
-struct sq_localinvalidate_hdr {
- u8 wqe_type;
- u8 flags;
- __le16 reserved16;
- __le32 inv_l_key;
- __le64 reserved64;
- u8 reserved128[16];
+/* cmdq_create_qp (size:768b/96B) */
+struct cmdq_create_qp {
+ u8 opcode;
+ #define CMDQ_CREATE_QP_OPCODE_CREATE_QP 0x1UL
+ #define CMDQ_CREATE_QP_OPCODE_LAST CMDQ_CREATE_QP_OPCODE_CREATE_QP
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __le64 qp_handle;
+ __le32 qp_flags;
+ #define CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED 0x1UL
+ #define CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION 0x2UL
+ #define CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE 0x4UL
+ #define CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED 0x8UL
+ #define CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED 0x10UL
+ #define CMDQ_CREATE_QP_QP_FLAGS_OPTIMIZED_TRANSMIT_ENABLED 0x20UL
+ #define CMDQ_CREATE_QP_QP_FLAGS_RESPONDER_UD_CQE_WITH_CFA 0x40UL
+ #define CMDQ_CREATE_QP_QP_FLAGS_EXT_STATS_ENABLED 0x80UL
+ #define CMDQ_CREATE_QP_QP_FLAGS_LAST \
+ CMDQ_CREATE_QP_QP_FLAGS_EXT_STATS_ENABLED
+ u8 type;
+ #define CMDQ_CREATE_QP_TYPE_RC 0x2UL
+ #define CMDQ_CREATE_QP_TYPE_UD 0x4UL
+ #define CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE 0x6UL
+ #define CMDQ_CREATE_QP_TYPE_GSI 0x7UL
+ #define CMDQ_CREATE_QP_TYPE_LAST CMDQ_CREATE_QP_TYPE_GSI
+ u8 sq_pg_size_sq_lvl;
+ #define CMDQ_CREATE_QP_SQ_LVL_MASK 0xfUL
+ #define CMDQ_CREATE_QP_SQ_LVL_SFT 0
+ #define CMDQ_CREATE_QP_SQ_LVL_LVL_0 0x0UL
+ #define CMDQ_CREATE_QP_SQ_LVL_LVL_1 0x1UL
+ #define CMDQ_CREATE_QP_SQ_LVL_LVL_2 0x2UL
+ #define CMDQ_CREATE_QP_SQ_LVL_LAST CMDQ_CREATE_QP_SQ_LVL_LVL_2
+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_MASK 0xf0UL
+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_SFT 4
+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K (0x0UL << 4)
+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K (0x1UL << 4)
+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K (0x2UL << 4)
+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M (0x3UL << 4)
+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M (0x4UL << 4)
+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G (0x5UL << 4)
+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_LAST CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G
+ u8 rq_pg_size_rq_lvl;
+ #define CMDQ_CREATE_QP_RQ_LVL_MASK 0xfUL
+ #define CMDQ_CREATE_QP_RQ_LVL_SFT 0
+ #define CMDQ_CREATE_QP_RQ_LVL_LVL_0 0x0UL
+ #define CMDQ_CREATE_QP_RQ_LVL_LVL_1 0x1UL
+ #define CMDQ_CREATE_QP_RQ_LVL_LVL_2 0x2UL
+ #define CMDQ_CREATE_QP_RQ_LVL_LAST CMDQ_CREATE_QP_RQ_LVL_LVL_2
+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_MASK 0xf0UL
+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_SFT 4
+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K (0x0UL << 4)
+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K (0x1UL << 4)
+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K (0x2UL << 4)
+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M (0x3UL << 4)
+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M (0x4UL << 4)
+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G (0x5UL << 4)
+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_LAST CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G
+ u8 unused_0;
+ __le32 dpi;
+ __le32 sq_size;
+ __le32 rq_size;
+ __le16 sq_fwo_sq_sge;
+ #define CMDQ_CREATE_QP_SQ_SGE_MASK 0xfUL
+ #define CMDQ_CREATE_QP_SQ_SGE_SFT 0
+ #define CMDQ_CREATE_QP_SQ_FWO_MASK 0xfff0UL
+ #define CMDQ_CREATE_QP_SQ_FWO_SFT 4
+ __le16 rq_fwo_rq_sge;
+ #define CMDQ_CREATE_QP_RQ_SGE_MASK 0xfUL
+ #define CMDQ_CREATE_QP_RQ_SGE_SFT 0
+ #define CMDQ_CREATE_QP_RQ_FWO_MASK 0xfff0UL
+ #define CMDQ_CREATE_QP_RQ_FWO_SFT 4
+ __le32 scq_cid;
+ __le32 rcq_cid;
+ __le32 srq_cid;
+ __le32 pd_id;
+ __le64 sq_pbl;
+ __le64 rq_pbl;
+ __le64 irrq_addr;
+ __le64 orrq_addr;
};
-/* FR-PMR SQ WQE (40 bytes) */
-struct sq_fr_pmr {
- u8 wqe_type;
- #define SQ_FR_PMR_WQE_TYPE_FR_PMR 0xdUL
- u8 flags;
- #define SQ_FR_PMR_FLAGS_SIGNAL_COMP 0x1UL
- #define SQ_FR_PMR_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
- #define SQ_FR_PMR_FLAGS_UC_FENCE 0x4UL
- #define SQ_FR_PMR_FLAGS_SE 0x8UL
- #define SQ_FR_PMR_FLAGS_INLINE 0x10UL
- u8 access_cntl;
- #define SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE 0x1UL
- #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ 0x2UL
- #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE 0x4UL
- #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC 0x8UL
- #define SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND 0x10UL
- u8 zero_based_page_size_log;
- #define SQ_FR_PMR_PAGE_SIZE_LOG_MASK 0x1fUL
- #define SQ_FR_PMR_PAGE_SIZE_LOG_SFT 0
- #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4K 0x0UL
- #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8K 0x1UL
- #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_64K 0x4UL
- #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_256K 0x6UL
- #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1M 0x8UL
- #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_2M 0x9UL
- #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4M 0xaUL
- #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1G 0x12UL
- #define SQ_FR_PMR_ZERO_BASED 0x20UL
- #define SQ_FR_PMR_RESERVED2_MASK 0xc0UL
- #define SQ_FR_PMR_RESERVED2_SFT 6
- __le32 l_key;
- u8 length[5];
- u8 reserved8_1;
- u8 reserved8_2;
- u8 numlevels_pbl_page_size_log;
- #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK 0x1fUL
- #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT 0
- #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4K 0x0UL
- #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8K 0x1UL
- #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_64K 0x4UL
- #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_256K 0x6UL
- #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1M 0x8UL
- #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_2M 0x9UL
- #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4M 0xaUL
- #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1G 0x12UL
- #define SQ_FR_PMR_RESERVED1 0x20UL
- #define SQ_FR_PMR_NUMLEVELS_MASK 0xc0UL
- #define SQ_FR_PMR_NUMLEVELS_SFT 6
- #define SQ_FR_PMR_NUMLEVELS_PHYSICAL (0x0UL << 6)
- #define SQ_FR_PMR_NUMLEVELS_LAYER1 (0x1UL << 6)
- #define SQ_FR_PMR_NUMLEVELS_LAYER2 (0x2UL << 6)
- __le64 pblptr;
- __le64 va;
- __le32 data[24];
+/* creq_create_qp_resp (size:128b/16B) */
+struct creq_create_qp_resp {
+ u8 type;
+ #define CREQ_CREATE_QP_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_CREATE_QP_RESP_TYPE_SFT 0
+ #define CREQ_CREATE_QP_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_CREATE_QP_RESP_TYPE_LAST CREQ_CREATE_QP_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 xid;
+ u8 v;
+ #define CREQ_CREATE_QP_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_CREATE_QP_RESP_EVENT_CREATE_QP 0x1UL
+ #define CREQ_CREATE_QP_RESP_EVENT_LAST CREQ_CREATE_QP_RESP_EVENT_CREATE_QP
+ u8 optimized_transmit_enabled;
+ u8 reserved48[5];
};
-/* sq_fr_pmr_hdr (size:256b/32B) */
-struct sq_fr_pmr_hdr {
- u8 wqe_type;
- u8 flags;
- u8 access_cntl;
- u8 zero_based_page_size_log;
- __le32 l_key;
- u8 length[5];
- u8 reserved8_1;
- u8 reserved8_2;
- u8 numlevels_pbl_page_size_log;
- __le64 pblptr;
- __le64 va;
+/* cmdq_destroy_qp (size:192b/24B) */
+struct cmdq_destroy_qp {
+ u8 opcode;
+ #define CMDQ_DESTROY_QP_OPCODE_DESTROY_QP 0x2UL
+ #define CMDQ_DESTROY_QP_OPCODE_LAST CMDQ_DESTROY_QP_OPCODE_DESTROY_QP
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __le32 qp_cid;
+ __le32 unused_0;
};
-/* Bind SQ WQE (40 bytes) */
-struct sq_bind {
- u8 wqe_type;
- #define SQ_BIND_WQE_TYPE_BIND 0xeUL
- u8 flags;
- #define SQ_BIND_FLAGS_SIGNAL_COMP 0x1UL
- #define SQ_BIND_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
- #define SQ_BIND_FLAGS_UC_FENCE 0x4UL
- #define SQ_BIND_FLAGS_SE 0x8UL
- #define SQ_BIND_FLAGS_INLINE 0x10UL
- u8 access_cntl;
- #define SQ_BIND_ACCESS_CNTL_LOCAL_WRITE 0x1UL
- #define SQ_BIND_ACCESS_CNTL_REMOTE_READ 0x2UL
- #define SQ_BIND_ACCESS_CNTL_REMOTE_WRITE 0x4UL
- #define SQ_BIND_ACCESS_CNTL_REMOTE_ATOMIC 0x8UL
- #define SQ_BIND_ACCESS_CNTL_WINDOW_BIND 0x10UL
- u8 reserved8_1;
- u8 mw_type_zero_based;
- #define SQ_BIND_ZERO_BASED 0x1UL
- #define SQ_BIND_MW_TYPE 0x2UL
- #define SQ_BIND_MW_TYPE_TYPE1 (0x0UL << 1)
- #define SQ_BIND_MW_TYPE_TYPE2 (0x1UL << 1)
- #define SQ_BIND_RESERVED6_MASK 0xfcUL
- #define SQ_BIND_RESERVED6_SFT 2
- u8 reserved8_2;
- __le16 reserved16;
- __le32 parent_l_key;
- __le32 l_key;
- __le64 va;
- u8 length[5];
- u8 data_reserved24[99];
- #define SQ_BIND_RESERVED24_MASK 0xffffff00UL
- #define SQ_BIND_RESERVED24_SFT 8
- #define SQ_BIND_DATA_MASK 0xffffffffUL
- #define SQ_BIND_DATA_SFT 0
+/* creq_destroy_qp_resp (size:128b/16B) */
+struct creq_destroy_qp_resp {
+ u8 type;
+ #define CREQ_DESTROY_QP_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_DESTROY_QP_RESP_TYPE_SFT 0
+ #define CREQ_DESTROY_QP_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_DESTROY_QP_RESP_TYPE_LAST CREQ_DESTROY_QP_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 xid;
+ u8 v;
+ #define CREQ_DESTROY_QP_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_DESTROY_QP_RESP_EVENT_DESTROY_QP 0x2UL
+ #define CREQ_DESTROY_QP_RESP_EVENT_LAST CREQ_DESTROY_QP_RESP_EVENT_DESTROY_QP
+ u8 reserved48[6];
};
-/* sq_bind_hdr (size:256b/32B) */
-struct sq_bind_hdr {
- u8 wqe_type;
- u8 flags;
- u8 access_cntl;
- u8 reserved8_1;
- u8 mw_type_zero_based;
- u8 reserved8_2;
- __le16 reserved16;
- __le32 parent_l_key;
- __le32 l_key;
- __le64 va;
- u8 length[5];
- u8 reserved24[3];
+/* cmdq_modify_qp (size:1024b/128B) */
+struct cmdq_modify_qp {
+ u8 opcode;
+ #define CMDQ_MODIFY_QP_OPCODE_MODIFY_QP 0x3UL
+ #define CMDQ_MODIFY_QP_OPCODE_LAST CMDQ_MODIFY_QP_OPCODE_MODIFY_QP
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __le32 modify_mask;
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_STATE 0x1UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY 0x2UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS 0x4UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_PKEY 0x8UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_QKEY 0x10UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_DGID 0x20UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL 0x40UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX 0x80UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT 0x100UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS 0x200UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC 0x400UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_PINGPONG_PUSH_MODE 0x800UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU 0x1000UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT 0x2000UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT 0x4000UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY 0x8000UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN 0x10000UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC 0x20000UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER 0x40000UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN 0x80000UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC 0x100000UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE 0x200000UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE 0x400000UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE 0x800000UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE 0x1000000UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA 0x2000000UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID 0x4000000UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC 0x8000000UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID 0x10000000UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_ENABLE_CC 0x20000000UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_TOS_ECN 0x40000000UL
+ #define CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP 0x80000000UL
+ __le32 qp_cid;
+ u8 network_type_en_sqd_async_notify_new_state;
+ #define CMDQ_MODIFY_QP_NEW_STATE_MASK 0xfUL
+ #define CMDQ_MODIFY_QP_NEW_STATE_SFT 0
+ #define CMDQ_MODIFY_QP_NEW_STATE_RESET 0x0UL
+ #define CMDQ_MODIFY_QP_NEW_STATE_INIT 0x1UL
+ #define CMDQ_MODIFY_QP_NEW_STATE_RTR 0x2UL
+ #define CMDQ_MODIFY_QP_NEW_STATE_RTS 0x3UL
+ #define CMDQ_MODIFY_QP_NEW_STATE_SQD 0x4UL
+ #define CMDQ_MODIFY_QP_NEW_STATE_SQE 0x5UL
+ #define CMDQ_MODIFY_QP_NEW_STATE_ERR 0x6UL
+ #define CMDQ_MODIFY_QP_NEW_STATE_LAST CMDQ_MODIFY_QP_NEW_STATE_ERR
+ #define CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY 0x10UL
+ #define CMDQ_MODIFY_QP_UNUSED1 0x20UL
+ #define CMDQ_MODIFY_QP_NETWORK_TYPE_MASK 0xc0UL
+ #define CMDQ_MODIFY_QP_NETWORK_TYPE_SFT 6
+ #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1 (0x0UL << 6)
+ #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4 (0x2UL << 6)
+ #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6 (0x3UL << 6)
+ #define CMDQ_MODIFY_QP_NETWORK_TYPE_LAST CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6
+ u8 access;
+ #define CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC_REMOTE_READ_REMOTE_WRITE_LOCAL_WRITE_MASK \
+ 0xffUL
+ #define CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC_REMOTE_READ_REMOTE_WRITE_LOCAL_WRITE_SFT \
+ 0
+ #define CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE 0x1UL
+ #define CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE 0x2UL
+ #define CMDQ_MODIFY_QP_ACCESS_REMOTE_READ 0x4UL
+ #define CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC 0x8UL
+ __le16 pkey;
+ __le32 qkey;
+ __le32 dgid[4];
+ __le32 flow_label;
+ __le16 sgid_index;
+ u8 hop_limit;
+ u8 traffic_class;
+ __le16 dest_mac[3];
+ u8 tos_dscp_tos_ecn;
+ #define CMDQ_MODIFY_QP_TOS_ECN_MASK 0x3UL
+ #define CMDQ_MODIFY_QP_TOS_ECN_SFT 0
+ #define CMDQ_MODIFY_QP_TOS_DSCP_MASK 0xfcUL
+ #define CMDQ_MODIFY_QP_TOS_DSCP_SFT 2
+ u8 path_mtu_pingpong_push_enable;
+ #define CMDQ_MODIFY_QP_PINGPONG_PUSH_ENABLE 0x1UL
+ #define CMDQ_MODIFY_QP_UNUSED3_MASK 0xeUL
+ #define CMDQ_MODIFY_QP_UNUSED3_SFT 1
+ #define CMDQ_MODIFY_QP_PATH_MTU_MASK 0xf0UL
+ #define CMDQ_MODIFY_QP_PATH_MTU_SFT 4
+ #define CMDQ_MODIFY_QP_PATH_MTU_MTU_256 (0x0UL << 4)
+ #define CMDQ_MODIFY_QP_PATH_MTU_MTU_512 (0x1UL << 4)
+ #define CMDQ_MODIFY_QP_PATH_MTU_MTU_1024 (0x2UL << 4)
+ #define CMDQ_MODIFY_QP_PATH_MTU_MTU_2048 (0x3UL << 4)
+ #define CMDQ_MODIFY_QP_PATH_MTU_MTU_4096 (0x4UL << 4)
+ #define CMDQ_MODIFY_QP_PATH_MTU_MTU_8192 (0x5UL << 4)
+ #define CMDQ_MODIFY_QP_PATH_MTU_LAST CMDQ_MODIFY_QP_PATH_MTU_MTU_8192
+ u8 timeout;
+ u8 retry_cnt;
+ u8 rnr_retry;
+ u8 min_rnr_timer;
+ __le32 rq_psn;
+ __le32 sq_psn;
+ u8 max_rd_atomic;
+ u8 max_dest_rd_atomic;
+ __le16 enable_cc;
+ #define CMDQ_MODIFY_QP_ENABLE_CC 0x1UL
+ #define CMDQ_MODIFY_QP_UNUSED15_MASK 0xfffeUL
+ #define CMDQ_MODIFY_QP_UNUSED15_SFT 1
+ __le32 sq_size;
+ __le32 rq_size;
+ __le16 sq_sge;
+ __le16 rq_sge;
+ __le32 max_inline_data;
+ __le32 dest_qp_id;
+ __le32 pingpong_push_dpi;
+ __le16 src_mac[3];
+ __le16 vlan_pcp_vlan_dei_vlan_id;
+ #define CMDQ_MODIFY_QP_VLAN_ID_MASK 0xfffUL
+ #define CMDQ_MODIFY_QP_VLAN_ID_SFT 0
+ #define CMDQ_MODIFY_QP_VLAN_DEI 0x1000UL
+ #define CMDQ_MODIFY_QP_VLAN_PCP_MASK 0xe000UL
+ #define CMDQ_MODIFY_QP_VLAN_PCP_SFT 13
+ __le64 irrq_addr;
+ __le64 orrq_addr;
};
-/* RQ/SRQ WQE Structures */
-/* RQ/SRQ WQE (40 bytes) */
-struct rq_wqe {
- u8 wqe_type;
- #define RQ_WQE_WQE_TYPE_RCV 0x80UL
- u8 flags;
- u8 wqe_size;
- u8 reserved8;
- __le32 reserved32;
- __le32 wr_id[2];
- #define RQ_WQE_WR_ID_MASK 0xfffffUL
- #define RQ_WQE_WR_ID_SFT 0
- #define RQ_WQE_RESERVED44_MASK 0xfff00000UL
- #define RQ_WQE_RESERVED44_SFT 20
- __le32 reserved128[4];
- __le32 data[24];
+/* creq_modify_qp_resp (size:128b/16B) */
+struct creq_modify_qp_resp {
+ u8 type;
+ #define CREQ_MODIFY_QP_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_MODIFY_QP_RESP_TYPE_SFT 0
+ #define CREQ_MODIFY_QP_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_MODIFY_QP_RESP_TYPE_LAST CREQ_MODIFY_QP_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 xid;
+ u8 v;
+ #define CREQ_MODIFY_QP_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_MODIFY_QP_RESP_EVENT_MODIFY_QP 0x3UL
+ #define CREQ_MODIFY_QP_RESP_EVENT_LAST CREQ_MODIFY_QP_RESP_EVENT_MODIFY_QP
+ u8 pingpong_push_state_index_enabled;
+ #define CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_ENABLED 0x1UL
+ #define CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_INDEX_MASK 0xeUL
+ #define CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_INDEX_SFT 1
+ #define CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_STATE 0x10UL
+ u8 reserved8;
+ __le32 lag_src_mac;
};
-/* rq_wqe_hdr (size:256b/32B) */
-struct rq_wqe_hdr {
- u8 wqe_type;
- u8 flags;
- u8 wqe_size;
+/* cmdq_query_qp (size:192b/24B) */
+struct cmdq_query_qp {
+ u8 opcode;
+ #define CMDQ_QUERY_QP_OPCODE_QUERY_QP 0x4UL
+ #define CMDQ_QUERY_QP_OPCODE_LAST CMDQ_QUERY_QP_OPCODE_QUERY_QP
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
u8 reserved8;
- __le32 reserved32;
- __le32 wr_id[2];
- u8 reserved128[16];
+ __le64 resp_addr;
+ __le32 qp_cid;
+ __le32 unused_0;
};
-/* CQ CQE Structures */
-/* Base CQE (32 bytes) */
-struct cq_base {
- __le64 reserved64_1;
- __le64 reserved64_2;
- __le64 reserved64_3;
- u8 cqe_type_toggle;
- #define CQ_BASE_TOGGLE 0x1UL
- #define CQ_BASE_CQE_TYPE_MASK 0x1eUL
- #define CQ_BASE_CQE_TYPE_SFT 1
- #define CQ_BASE_CQE_TYPE_REQ (0x0UL << 1)
- #define CQ_BASE_CQE_TYPE_RES_RC (0x1UL << 1)
- #define CQ_BASE_CQE_TYPE_RES_UD (0x2UL << 1)
- #define CQ_BASE_CQE_TYPE_RES_RAWETH_QP1 (0x3UL << 1)
- #define CQ_BASE_CQE_TYPE_TERMINAL (0xeUL << 1)
- #define CQ_BASE_CQE_TYPE_CUT_OFF (0xfUL << 1)
- #define CQ_BASE_RESERVED3_MASK 0xe0UL
- #define CQ_BASE_RESERVED3_SFT 5
- u8 status;
- __le16 reserved16;
- __le32 reserved32;
-};
-
-/* Requester CQ CQE (32 bytes) */
-struct cq_req {
- __le64 qp_handle;
- __le16 sq_cons_idx;
- __le16 reserved16_1;
- __le32 reserved32_2;
- __le64 reserved64;
- u8 cqe_type_toggle;
- #define CQ_REQ_TOGGLE 0x1UL
- #define CQ_REQ_CQE_TYPE_MASK 0x1eUL
- #define CQ_REQ_CQE_TYPE_SFT 1
- #define CQ_REQ_CQE_TYPE_REQ (0x0UL << 1)
- #define CQ_REQ_RESERVED3_MASK 0xe0UL
- #define CQ_REQ_RESERVED3_SFT 5
- u8 status;
- #define CQ_REQ_STATUS_OK 0x0UL
- #define CQ_REQ_STATUS_BAD_RESPONSE_ERR 0x1UL
- #define CQ_REQ_STATUS_LOCAL_LENGTH_ERR 0x2UL
- #define CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR 0x3UL
- #define CQ_REQ_STATUS_LOCAL_PROTECTION_ERR 0x4UL
- #define CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL
- #define CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR 0x6UL
- #define CQ_REQ_STATUS_REMOTE_ACCESS_ERR 0x7UL
- #define CQ_REQ_STATUS_REMOTE_OPERATION_ERR 0x8UL
- #define CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR 0x9UL
- #define CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR 0xaUL
- #define CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR 0xbUL
- __le16 reserved16_2;
- __le32 reserved32_1;
-};
-
-/* Responder RC CQE (32 bytes) */
-struct cq_res_rc {
- __le32 length;
- __le32 imm_data_or_inv_r_key;
- __le64 qp_handle;
- __le64 mr_handle;
- u8 cqe_type_toggle;
- #define CQ_RES_RC_TOGGLE 0x1UL
- #define CQ_RES_RC_CQE_TYPE_MASK 0x1eUL
- #define CQ_RES_RC_CQE_TYPE_SFT 1
- #define CQ_RES_RC_CQE_TYPE_RES_RC (0x1UL << 1)
- #define CQ_RES_RC_RESERVED3_MASK 0xe0UL
- #define CQ_RES_RC_RESERVED3_SFT 5
- u8 status;
- #define CQ_RES_RC_STATUS_OK 0x0UL
- #define CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR 0x1UL
- #define CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR 0x2UL
- #define CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR 0x3UL
- #define CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL
- #define CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL
- #define CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR 0x6UL
- #define CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL
- #define CQ_RES_RC_STATUS_HW_FLUSH_ERR 0x8UL
- __le16 flags;
- #define CQ_RES_RC_FLAGS_SRQ 0x1UL
- #define CQ_RES_RC_FLAGS_SRQ_RQ (0x0UL << 0)
- #define CQ_RES_RC_FLAGS_SRQ_SRQ (0x1UL << 0)
- #define CQ_RES_RC_FLAGS_SRQ_LAST CQ_RES_RC_FLAGS_SRQ_SRQ
- #define CQ_RES_RC_FLAGS_IMM 0x2UL
- #define CQ_RES_RC_FLAGS_INV 0x4UL
- #define CQ_RES_RC_FLAGS_RDMA 0x8UL
- #define CQ_RES_RC_FLAGS_RDMA_SEND (0x0UL << 3)
- #define CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE (0x1UL << 3)
- #define CQ_RES_RC_FLAGS_RDMA_LAST CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE
- __le32 srq_or_rq_wr_id;
- #define CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL
- #define CQ_RES_RC_SRQ_OR_RQ_WR_ID_SFT 0
- #define CQ_RES_RC_RESERVED12_MASK 0xfff00000UL
- #define CQ_RES_RC_RESERVED12_SFT 20
-};
-
-/* Responder UD CQE (32 bytes) */
-struct cq_res_ud {
- __le16 length;
- #define CQ_RES_UD_LENGTH_MASK 0x3fffUL
- #define CQ_RES_UD_LENGTH_SFT 0
- __le16 cfa_metadata;
- #define CQ_RES_UD_CFA_METADATA_VID_MASK 0xfffUL
- #define CQ_RES_UD_CFA_METADATA_VID_SFT 0
- #define CQ_RES_UD_CFA_METADATA_DE 0x1000UL
- #define CQ_RES_UD_CFA_METADATA_PRI_MASK 0xe000UL
- #define CQ_RES_UD_CFA_METADATA_PRI_SFT 13
- __le32 imm_data;
- __le64 qp_handle;
- __le16 src_mac[3];
- __le16 src_qp_low;
- u8 cqe_type_toggle;
- #define CQ_RES_UD_TOGGLE 0x1UL
- #define CQ_RES_UD_CQE_TYPE_MASK 0x1eUL
- #define CQ_RES_UD_CQE_TYPE_SFT 1
- #define CQ_RES_UD_CQE_TYPE_RES_UD (0x2UL << 1)
- u8 status;
- #define CQ_RES_UD_STATUS_OK 0x0UL
- #define CQ_RES_UD_STATUS_LOCAL_ACCESS_ERROR 0x1UL
- #define CQ_RES_UD_STATUS_HW_LOCAL_LENGTH_ERR 0x2UL
- #define CQ_RES_UD_STATUS_LOCAL_PROTECTION_ERR 0x3UL
- #define CQ_RES_UD_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL
- #define CQ_RES_UD_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL
- #define CQ_RES_UD_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL
- #define CQ_RES_UD_STATUS_HW_FLUSH_ERR 0x8UL
- __le16 flags;
- #define CQ_RES_UD_FLAGS_SRQ 0x1UL
- #define CQ_RES_UD_FLAGS_SRQ_RQ (0x0UL << 0)
- #define CQ_RES_UD_FLAGS_SRQ_SRQ (0x1UL << 0)
- #define CQ_RES_UD_FLAGS_SRQ_LAST CQ_RES_UD_FLAGS_SRQ_SRQ
- #define CQ_RES_UD_FLAGS_IMM 0x2UL
- #define CQ_RES_UD_FLAGS_UNUSED_MASK 0xcUL
- #define CQ_RES_UD_FLAGS_UNUSED_SFT 2
- #define CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK 0x30UL
- #define CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT 4
- #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V1 (0x0UL << 4)
- #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV4 (0x2UL << 4)
- #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6 (0x3UL << 4)
- #define CQ_RES_UD_FLAGS_ROCE_IP_VER_LAST \
- CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6
- #define CQ_RES_UD_FLAGS_META_FORMAT_MASK 0x3c0UL
- #define CQ_RES_UD_FLAGS_META_FORMAT_SFT 6
- #define CQ_RES_UD_FLAGS_META_FORMAT_NONE (0x0UL << 6)
- #define CQ_RES_UD_FLAGS_META_FORMAT_VLAN (0x1UL << 6)
- #define CQ_RES_UD_FLAGS_META_FORMAT_TUNNEL_ID (0x2UL << 6)
- #define CQ_RES_UD_FLAGS_META_FORMAT_CHDR_DATA (0x3UL << 6)
- #define CQ_RES_UD_FLAGS_META_FORMAT_HDR_OFFSET (0x4UL << 6)
- #define CQ_RES_UD_FLAGS_META_FORMAT_LAST \
- CQ_RES_UD_FLAGS_META_FORMAT_HDR_OFFSET
- #define CQ_RES_UD_FLAGS_EXT_META_FORMAT_MASK 0xc00UL
- #define CQ_RES_UD_FLAGS_EXT_META_FORMAT_SFT 10
-
- __le32 src_qp_high_srq_or_rq_wr_id;
- #define CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL
- #define CQ_RES_UD_SRQ_OR_RQ_WR_ID_SFT 0
- #define CQ_RES_UD_SRC_QP_HIGH_MASK 0xff000000UL
- #define CQ_RES_UD_SRC_QP_HIGH_SFT 24
-};
-
-/* Responder RawEth and QP1 CQE (32 bytes) */
-struct cq_res_raweth_qp1 {
- __le16 length;
- #define CQ_RES_RAWETH_QP1_LENGTH_MASK 0x3fffUL
- #define CQ_RES_RAWETH_QP1_LENGTH_SFT 0
- #define CQ_RES_RAWETH_QP1_RESERVED2_MASK 0xc000UL
- #define CQ_RES_RAWETH_QP1_RESERVED2_SFT 14
- __le16 raweth_qp1_flags;
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ERROR 0x1UL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_RESERVED5_1_MASK 0x3eUL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_RESERVED5_1_SFT 1
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_MASK 0x3c0UL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_SFT 6
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_NOT_KNOWN (0x0UL << 6)
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_IP (0x1UL << 6)
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_TCP (0x2UL << 6)
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_UDP (0x3UL << 6)
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_FCOE (0x4UL << 6)
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE (0x5UL << 6)
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ICMP (0x7UL << 6)
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_WO_TIMESTAMP \
- (0x8UL << 6)
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP \
- (0x9UL << 6)
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_LAST \
- CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_MASK 0x3ffUL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_SFT 0
- #define CQ_RES_RAWETH_QP1_RESERVED6_MASK 0xfc00UL
- #define CQ_RES_RAWETH_QP1_RESERVED6_SFT 10
- __le16 raweth_qp1_errors;
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_RESERVED4_MASK 0xfUL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_RESERVED4_SFT 0
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_IP_CS_ERROR 0x10UL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_L4_CS_ERROR 0x20UL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_IP_CS_ERROR 0x40UL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_L4_CS_ERROR 0x80UL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_CRC_ERROR 0x100UL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_MASK 0xe00UL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_SFT 9
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_NO_ERROR \
- (0x0UL << 9)
- #define \
- CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_VERSION \
- (0x1UL << 9)
- #define \
- CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_HDR_LEN \
- (0x2UL << 9)
- #define \
- CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_TUNNEL_TOTAL_ERROR \
- (0x3UL << 9)
- #define \
- CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_IP_TOTAL_ERROR \
- (0x4UL << 9)
- #define \
- CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_UDP_TOTAL_ERROR \
- (0x5UL << 9)
- #define \
- CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL \
- (0x6UL << 9)
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_LAST \
- CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_MASK 0xf000UL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_SFT 12
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_NO_ERROR \
- (0x0UL << 12)
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_VERSION \
- (0x1UL << 12)
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_HDR_LEN \
- (0x2UL << 12)
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_TTL \
- (0x3UL << 12)
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_IP_TOTAL_ERROR \
- (0x4UL << 12)
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_UDP_TOTAL_ERROR \
- (0x5UL << 12)
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN \
- (0x6UL << 12)
+/* creq_query_qp_resp (size:128b/16B) */
+struct creq_query_qp_resp {
+ u8 type;
+ #define CREQ_QUERY_QP_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_QUERY_QP_RESP_TYPE_SFT 0
+ #define CREQ_QUERY_QP_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_QUERY_QP_RESP_TYPE_LAST CREQ_QUERY_QP_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 size;
+ u8 v;
+ #define CREQ_QUERY_QP_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_QUERY_QP_RESP_EVENT_QUERY_QP 0x4UL
+ #define CREQ_QUERY_QP_RESP_EVENT_LAST CREQ_QUERY_QP_RESP_EVENT_QUERY_QP
+ u8 reserved48[6];
+};
+
+/* creq_query_qp_resp_sb (size:832b/104B) */
+struct creq_query_qp_resp_sb {
+ u8 opcode;
+ #define CREQ_QUERY_QP_RESP_SB_OPCODE_QUERY_QP 0x4UL
+ #define CREQ_QUERY_QP_RESP_SB_OPCODE_LAST CREQ_QUERY_QP_RESP_SB_OPCODE_QUERY_QP
+ u8 status;
+ __le16 cookie;
+ __le16 flags;
+ u8 resp_size;
+ u8 reserved8;
+ __le32 xid;
+ u8 en_sqd_async_notify_state;
+ #define CREQ_QUERY_QP_RESP_SB_STATE_MASK 0xfUL
+ #define CREQ_QUERY_QP_RESP_SB_STATE_SFT 0
+ #define CREQ_QUERY_QP_RESP_SB_STATE_RESET 0x0UL
+ #define CREQ_QUERY_QP_RESP_SB_STATE_INIT 0x1UL
+ #define CREQ_QUERY_QP_RESP_SB_STATE_RTR 0x2UL
+ #define CREQ_QUERY_QP_RESP_SB_STATE_RTS 0x3UL
+ #define CREQ_QUERY_QP_RESP_SB_STATE_SQD 0x4UL
+ #define CREQ_QUERY_QP_RESP_SB_STATE_SQE 0x5UL
+ #define CREQ_QUERY_QP_RESP_SB_STATE_ERR 0x6UL
+ #define CREQ_QUERY_QP_RESP_SB_STATE_LAST CREQ_QUERY_QP_RESP_SB_STATE_ERR
+ #define CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY 0x10UL
+ #define CREQ_QUERY_QP_RESP_SB_UNUSED3_MASK 0xe0UL
+ #define CREQ_QUERY_QP_RESP_SB_UNUSED3_SFT 5
+ u8 access;
#define \
- CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN_TOO_SMALL\
- (0x7UL << 12)
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN \
- (0x8UL << 12)
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_LAST \
- CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN
- __le16 raweth_qp1_cfa_code;
- __le64 qp_handle;
- __le32 raweth_qp1_flags2;
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC 0x1UL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC 0x2UL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_IP_CS_CALC 0x4UL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_L4_CS_CALC 0x8UL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_MASK 0xf0UL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_SFT 4
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_NONE \
- (0x0UL << 4)
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN \
- (0x1UL << 4)
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_LAST\
- CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE 0x100UL
- __le32 raweth_qp1_metadata;
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK 0xfffUL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_SFT 0
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_DE 0x1000UL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK 0xe000UL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT 13
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK 0xffff0000UL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT 16
- u8 cqe_type_toggle;
- #define CQ_RES_RAWETH_QP1_TOGGLE 0x1UL
- #define CQ_RES_RAWETH_QP1_CQE_TYPE_MASK 0x1eUL
- #define CQ_RES_RAWETH_QP1_CQE_TYPE_SFT 1
- #define CQ_RES_RAWETH_QP1_CQE_TYPE_RES_RAWETH_QP1 (0x3UL << 1)
- #define CQ_RES_RAWETH_QP1_RESERVED3_MASK 0xe0UL
- #define CQ_RES_RAWETH_QP1_RESERVED3_SFT 5
- u8 status;
- #define CQ_RES_RAWETH_QP1_STATUS_OK 0x0UL
- #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR 0x1UL
- #define CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR 0x2UL
- #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR 0x3UL
- #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL
- #define CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL
- #define CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL
- #define CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR 0x8UL
- __le16 flags;
- #define CQ_RES_RAWETH_QP1_FLAGS_SRQ 0x1UL
- #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_RQ 0x0UL
- #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ 0x1UL
- #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_LAST \
- CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ
- __le32 raweth_qp1_payload_offset_srq_or_rq_wr_id;
- #define CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL
- #define CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_SFT 0
- #define CQ_RES_RAWETH_QP1_RESERVED4_MASK 0xf00000UL
- #define CQ_RES_RAWETH_QP1_RESERVED4_SFT 20
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_MASK 0xff000000UL
- #define CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_SFT 24
-};
-
-/* Terminal CQE (32 bytes) */
-struct cq_terminal {
- __le64 qp_handle;
- __le16 sq_cons_idx;
- __le16 rq_cons_idx;
- __le32 reserved32_1;
- __le64 reserved64_3;
- u8 cqe_type_toggle;
- #define CQ_TERMINAL_TOGGLE 0x1UL
- #define CQ_TERMINAL_CQE_TYPE_MASK 0x1eUL
- #define CQ_TERMINAL_CQE_TYPE_SFT 1
- #define CQ_TERMINAL_CQE_TYPE_TERMINAL (0xeUL << 1)
- #define CQ_TERMINAL_RESERVED3_MASK 0xe0UL
- #define CQ_TERMINAL_RESERVED3_SFT 5
- u8 status;
- #define CQ_TERMINAL_STATUS_OK 0x0UL
- __le16 reserved16;
- __le32 reserved32_2;
-};
-
-/* Cutoff CQE (32 bytes) */
-struct cq_cutoff {
- __le64 reserved64_1;
- __le64 reserved64_2;
- __le64 reserved64_3;
- u8 cqe_type_toggle;
- #define CQ_CUTOFF_TOGGLE 0x1UL
- #define CQ_CUTOFF_CQE_TYPE_MASK 0x1eUL
- #define CQ_CUTOFF_CQE_TYPE_SFT 1
- #define CQ_CUTOFF_CQE_TYPE_CUT_OFF (0xfUL << 1)
- #define CQ_CUTOFF_RESERVED3_MASK 0xe0UL
- #define CQ_CUTOFF_RESERVED3_SFT 5
- u8 status;
- #define CQ_CUTOFF_STATUS_OK 0x0UL
- __le16 reserved16;
- __le32 reserved32;
-};
-
-/* Notification Queue (NQ) Structures */
-/* Base NQ Record (16 bytes) */
-struct nq_base {
- __le16 info10_type;
- #define NQ_BASE_TYPE_MASK 0x3fUL
- #define NQ_BASE_TYPE_SFT 0
- #define NQ_BASE_TYPE_CQ_NOTIFICATION 0x30UL
- #define NQ_BASE_TYPE_SRQ_EVENT 0x32UL
- #define NQ_BASE_TYPE_DBQ_EVENT 0x34UL
- #define NQ_BASE_TYPE_QP_EVENT 0x38UL
- #define NQ_BASE_TYPE_FUNC_EVENT 0x3aUL
- #define NQ_BASE_INFO10_MASK 0xffc0UL
- #define NQ_BASE_INFO10_SFT 6
- __le16 info16;
- __le32 info32;
- __le32 info63_v[2];
- #define NQ_BASE_V 0x1UL
- #define NQ_BASE_INFO63_MASK 0xfffffffeUL
- #define NQ_BASE_INFO63_SFT 1
-};
-
-/* Completion Queue Notification (16 bytes) */
-struct nq_cn {
- __le16 type;
- #define NQ_CN_TYPE_MASK 0x3fUL
- #define NQ_CN_TYPE_SFT 0
- #define NQ_CN_TYPE_CQ_NOTIFICATION 0x30UL
- #define NQ_CN_RESERVED9_MASK 0xffc0UL
- #define NQ_CN_RESERVED9_SFT 6
- __le16 reserved16;
- __le32 cq_handle_low;
- __le32 v;
- #define NQ_CN_V 0x1UL
- #define NQ_CN_RESERVED31_MASK 0xfffffffeUL
- #define NQ_CN_RESERVED31_SFT 1
- __le32 cq_handle_high;
-};
-
-/* SRQ Event Notification (16 bytes) */
-struct nq_srq_event {
- u8 type;
- #define NQ_SRQ_EVENT_TYPE_MASK 0x3fUL
- #define NQ_SRQ_EVENT_TYPE_SFT 0
- #define NQ_SRQ_EVENT_TYPE_SRQ_EVENT 0x32UL
- #define NQ_SRQ_EVENT_RESERVED1_MASK 0xc0UL
- #define NQ_SRQ_EVENT_RESERVED1_SFT 6
- u8 event;
- #define NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT 0x1UL
- __le16 reserved16;
- __le32 srq_handle_low;
- __le32 v;
- #define NQ_SRQ_EVENT_V 0x1UL
- #define NQ_SRQ_EVENT_RESERVED31_MASK 0xfffffffeUL
- #define NQ_SRQ_EVENT_RESERVED31_SFT 1
- __le32 srq_handle_high;
-};
-
-/* DBQ Async Event Notification (16 bytes) */
-struct nq_dbq_event {
- u8 type;
- #define NQ_DBQ_EVENT_TYPE_MASK 0x3fUL
- #define NQ_DBQ_EVENT_TYPE_SFT 0
- #define NQ_DBQ_EVENT_TYPE_DBQ_EVENT 0x34UL
- #define NQ_DBQ_EVENT_RESERVED1_MASK 0xc0UL
- #define NQ_DBQ_EVENT_RESERVED1_SFT 6
- u8 event;
- #define NQ_DBQ_EVENT_EVENT_DBQ_THRESHOLD_EVENT 0x1UL
- __le16 db_pfid;
- #define NQ_DBQ_EVENT_DB_PFID_MASK 0xfUL
- #define NQ_DBQ_EVENT_DB_PFID_SFT 0
- #define NQ_DBQ_EVENT_RESERVED12_MASK 0xfff0UL
- #define NQ_DBQ_EVENT_RESERVED12_SFT 4
- __le32 db_dpi;
- #define NQ_DBQ_EVENT_DB_DPI_MASK 0xfffffUL
- #define NQ_DBQ_EVENT_DB_DPI_SFT 0
- #define NQ_DBQ_EVENT_RESERVED12_2_MASK 0xfff00000UL
- #define NQ_DBQ_EVENT_RESERVED12_2_SFT 20
- __le32 v;
- #define NQ_DBQ_EVENT_V 0x1UL
- #define NQ_DBQ_EVENT_RESERVED32_MASK 0xfffffffeUL
- #define NQ_DBQ_EVENT_RESERVED32_SFT 1
- __le32 db_type_db_xid;
- #define NQ_DBQ_EVENT_DB_XID_MASK 0xfffffUL
- #define NQ_DBQ_EVENT_DB_XID_SFT 0
- #define NQ_DBQ_EVENT_RESERVED8_MASK 0xff00000UL
- #define NQ_DBQ_EVENT_RESERVED8_SFT 20
- #define NQ_DBQ_EVENT_DB_TYPE_MASK 0xf0000000UL
- #define NQ_DBQ_EVENT_DB_TYPE_SFT 28
-};
-
-/* Read Request/Response Queue Structures */
-/* Input Read Request Queue (IRRQ) Message (32 bytes) */
-struct xrrq_irrq {
- __le16 credits_type;
- #define XRRQ_IRRQ_TYPE 0x1UL
- #define XRRQ_IRRQ_TYPE_READ_REQ 0x0UL
- #define XRRQ_IRRQ_TYPE_ATOMIC_REQ 0x1UL
- #define XRRQ_IRRQ_RESERVED10_MASK 0x7feUL
- #define XRRQ_IRRQ_RESERVED10_SFT 1
- #define XRRQ_IRRQ_CREDITS_MASK 0xf800UL
- #define XRRQ_IRRQ_CREDITS_SFT 11
- __le16 reserved16;
- __le32 reserved32;
- __le32 psn;
- #define XRRQ_IRRQ_PSN_MASK 0xffffffUL
- #define XRRQ_IRRQ_PSN_SFT 0
- #define XRRQ_IRRQ_RESERVED8_1_MASK 0xff000000UL
- #define XRRQ_IRRQ_RESERVED8_1_SFT 24
- __le32 msn;
- #define XRRQ_IRRQ_MSN_MASK 0xffffffUL
- #define XRRQ_IRRQ_MSN_SFT 0
- #define XRRQ_IRRQ_RESERVED8_2_MASK 0xff000000UL
- #define XRRQ_IRRQ_RESERVED8_2_SFT 24
- __le64 va_or_atomic_result;
- __le32 rdma_r_key;
- __le32 length;
-};
-
-/* Output Read Request Queue (ORRQ) Message (32 bytes) */
-struct xrrq_orrq {
- __le16 num_sges_type;
- #define XRRQ_ORRQ_TYPE 0x1UL
- #define XRRQ_ORRQ_TYPE_READ_REQ 0x0UL
- #define XRRQ_ORRQ_TYPE_ATOMIC_REQ 0x1UL
- #define XRRQ_ORRQ_RESERVED10_MASK 0x7feUL
- #define XRRQ_ORRQ_RESERVED10_SFT 1
- #define XRRQ_ORRQ_NUM_SGES_MASK 0xf800UL
- #define XRRQ_ORRQ_NUM_SGES_SFT 11
- __le16 reserved16;
- __le32 length;
- __le32 psn;
- #define XRRQ_ORRQ_PSN_MASK 0xffffffUL
- #define XRRQ_ORRQ_PSN_SFT 0
- #define XRRQ_ORRQ_RESERVED8_1_MASK 0xff000000UL
- #define XRRQ_ORRQ_RESERVED8_1_SFT 24
- __le32 end_psn;
- #define XRRQ_ORRQ_END_PSN_MASK 0xffffffUL
- #define XRRQ_ORRQ_END_PSN_SFT 0
- #define XRRQ_ORRQ_RESERVED8_2_MASK 0xff000000UL
- #define XRRQ_ORRQ_RESERVED8_2_SFT 24
- __le64 first_sge_phy_or_sing_sge_va;
- __le32 single_sge_l_key;
- __le32 single_sge_size;
-};
-
-/* Page Buffer List Memory Structures (PBL) */
-/* Page Table Entry (PTE) (8 bytes) */
-struct ptu_pte {
- __le32 page_next_to_last_last_valid[2];
- #define PTU_PTE_VALID 0x1UL
- #define PTU_PTE_LAST 0x2UL
- #define PTU_PTE_NEXT_TO_LAST 0x4UL
- #define PTU_PTE_PAGE_MASK 0xfffff000UL
- #define PTU_PTE_PAGE_SFT 12
+ CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_ATOMIC_REMOTE_READ_REMOTE_WRITE_LOCAL_WRITE_MASK\
+ 0xffUL
+ #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_ATOMIC_REMOTE_READ_REMOTE_WRITE_LOCAL_WRITE_SFT\
+ 0
+ #define CREQ_QUERY_QP_RESP_SB_ACCESS_LOCAL_WRITE 0x1UL
+ #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_WRITE 0x2UL
+ #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_READ 0x4UL
+ #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_ATOMIC 0x8UL
+ __le16 pkey;
+ __le32 qkey;
+ __le32 reserved32;
+ __le32 dgid[4];
+ __le32 flow_label;
+ __le16 sgid_index;
+ u8 hop_limit;
+ u8 traffic_class;
+ __le16 dest_mac[3];
+ __le16 path_mtu_dest_vlan_id;
+ #define CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_MASK 0xfffUL
+ #define CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_SFT 0
+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK 0xf000UL
+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT 12
+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_256 (0x0UL << 12)
+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_512 (0x1UL << 12)
+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_1024 (0x2UL << 12)
+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_2048 (0x3UL << 12)
+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_4096 (0x4UL << 12)
+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_8192 (0x5UL << 12)
+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_LAST CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_8192
+ u8 timeout;
+ u8 retry_cnt;
+ u8 rnr_retry;
+ u8 min_rnr_timer;
+ __le32 rq_psn;
+ __le32 sq_psn;
+ u8 max_rd_atomic;
+ u8 max_dest_rd_atomic;
+ u8 tos_dscp_tos_ecn;
+ #define CREQ_QUERY_QP_RESP_SB_TOS_ECN_MASK 0x3UL
+ #define CREQ_QUERY_QP_RESP_SB_TOS_ECN_SFT 0
+ #define CREQ_QUERY_QP_RESP_SB_TOS_DSCP_MASK 0xfcUL
+ #define CREQ_QUERY_QP_RESP_SB_TOS_DSCP_SFT 2
+ u8 enable_cc;
+ #define CREQ_QUERY_QP_RESP_SB_ENABLE_CC 0x1UL
+ __le32 sq_size;
+ __le32 rq_size;
+ __le16 sq_sge;
+ __le16 rq_sge;
+ __le32 max_inline_data;
+ __le32 dest_qp_id;
+ __le16 port_id;
+ u8 unused_0;
+ u8 stat_collection_id;
+ __le16 src_mac[3];
+ __le16 vlan_pcp_vlan_dei_vlan_id;
+ #define CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK 0xfffUL
+ #define CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT 0
+ #define CREQ_QUERY_QP_RESP_SB_VLAN_DEI 0x1000UL
+ #define CREQ_QUERY_QP_RESP_SB_VLAN_PCP_MASK 0xe000UL
+ #define CREQ_QUERY_QP_RESP_SB_VLAN_PCP_SFT 13
};
-/* Page Directory Entry (PDE) (8 bytes) */
-struct ptu_pde {
- __le32 page_valid[2];
- #define PTU_PDE_VALID 0x1UL
- #define PTU_PDE_PAGE_MASK 0xfffff000UL
- #define PTU_PDE_PAGE_SFT 12
+/* cmdq_query_qp_extend (size:192b/24B) */
+struct cmdq_query_qp_extend {
+ u8 opcode;
+ #define CMDQ_QUERY_QP_EXTEND_OPCODE_QUERY_QP_EXTEND 0x91UL
+ #define CMDQ_QUERY_QP_EXTEND_OPCODE_LAST CMDQ_QUERY_QP_EXTEND_OPCODE_QUERY_QP_EXTEND
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 num_qps;
+ __le64 resp_addr;
+ __le32 function_id;
+ #define CMDQ_QUERY_QP_EXTEND_PF_NUM_MASK 0xffUL
+ #define CMDQ_QUERY_QP_EXTEND_PF_NUM_SFT 0
+ #define CMDQ_QUERY_QP_EXTEND_VF_NUM_MASK 0xffff00UL
+ #define CMDQ_QUERY_QP_EXTEND_VF_NUM_SFT 8
+ #define CMDQ_QUERY_QP_EXTEND_VF_VALID 0x1000000UL
+ __le32 current_index;
};
-/* RoCE Fastpath Host Structures */
-/* Command Queue (CMDQ) Interface */
-/* Init CMDQ (16 bytes) */
-struct cmdq_init {
- __le64 cmdq_pbl;
- __le16 cmdq_size_cmdq_lvl;
- #define CMDQ_INIT_CMDQ_LVL_MASK 0x3UL
- #define CMDQ_INIT_CMDQ_LVL_SFT 0
- #define CMDQ_INIT_CMDQ_SIZE_MASK 0xfffcUL
- #define CMDQ_INIT_CMDQ_SIZE_SFT 2
- __le16 creq_ring_id;
- __le32 prod_idx;
+/* creq_query_qp_extend_resp (size:128b/16B) */
+struct creq_query_qp_extend_resp {
+ u8 type;
+ #define CREQ_QUERY_QP_EXTEND_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_QUERY_QP_EXTEND_RESP_TYPE_SFT 0
+ #define CREQ_QUERY_QP_EXTEND_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_TYPE_LAST CREQ_QUERY_QP_EXTEND_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 size;
+ u8 v;
+ #define CREQ_QUERY_QP_EXTEND_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_QUERY_QP_EXTEND_RESP_EVENT_QUERY_QP_EXTEND 0x91UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_EVENT_LAST CREQ_QUERY_QP_EXTEND_RESP_EVENT_QUERY_QP_EXTEND
+ __le16 reserved16;
+ __le32 current_index;
};
-/* Update CMDQ producer index (16 bytes) */
-struct cmdq_update {
- __le64 reserved64;
- __le32 reserved32;
- __le32 prod_idx;
+/* creq_query_qp_extend_resp_sb (size:384b/48B) */
+struct creq_query_qp_extend_resp_sb {
+ u8 opcode;
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_OPCODE_QUERY_QP_EXTEND 0x91UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_OPCODE_LAST \
+ CREQ_QUERY_QP_EXTEND_RESP_SB_OPCODE_QUERY_QP_EXTEND
+ u8 status;
+ __le16 cookie;
+ __le16 flags;
+ u8 resp_size;
+ u8 reserved8;
+ __le32 xid;
+ u8 state;
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_MASK 0xfUL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_SFT 0
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_RESET 0x0UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_INIT 0x1UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_RTR 0x2UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_RTS 0x3UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_SQD 0x4UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_SQE 0x5UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_ERR 0x6UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_LAST CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_ERR
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_UNUSED4_MASK 0xf0UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_UNUSED4_SFT 4
+ u8 reserved_8;
+ __le16 port_id;
+ __le32 qkey;
+ __le16 sgid_index;
+ u8 network_type;
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_NETWORK_TYPE_ROCEV1 0x0UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_NETWORK_TYPE_ROCEV2_IPV4 0x2UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_NETWORK_TYPE_ROCEV2_IPV6 0x3UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_NETWORK_TYPE_LAST \
+ CREQ_QUERY_QP_EXTEND_RESP_SB_NETWORK_TYPE_ROCEV2_IPV6
+ u8 unused_0;
+ __le32 dgid[4];
+ __le32 dest_qp_id;
+ u8 stat_collection_id;
+ u8 reservred_8;
+ __le16 reserved_16;
};
-/* CMDQ common header structure (16 bytes) */
-struct cmdq_base {
- u8 opcode;
- #define CMDQ_BASE_OPCODE_CREATE_QP 0x1UL
- #define CMDQ_BASE_OPCODE_DESTROY_QP 0x2UL
- #define CMDQ_BASE_OPCODE_MODIFY_QP 0x3UL
- #define CMDQ_BASE_OPCODE_QUERY_QP 0x4UL
- #define CMDQ_BASE_OPCODE_CREATE_SRQ 0x5UL
- #define CMDQ_BASE_OPCODE_DESTROY_SRQ 0x6UL
- #define CMDQ_BASE_OPCODE_QUERY_SRQ 0x8UL
- #define CMDQ_BASE_OPCODE_CREATE_CQ 0x9UL
- #define CMDQ_BASE_OPCODE_DESTROY_CQ 0xaUL
- #define CMDQ_BASE_OPCODE_RESIZE_CQ 0xcUL
- #define CMDQ_BASE_OPCODE_ALLOCATE_MRW 0xdUL
- #define CMDQ_BASE_OPCODE_DEALLOCATE_KEY 0xeUL
- #define CMDQ_BASE_OPCODE_REGISTER_MR 0xfUL
- #define CMDQ_BASE_OPCODE_DEREGISTER_MR 0x10UL
- #define CMDQ_BASE_OPCODE_ADD_GID 0x11UL
- #define CMDQ_BASE_OPCODE_DELETE_GID 0x12UL
- #define CMDQ_BASE_OPCODE_MODIFY_GID 0x17UL
- #define CMDQ_BASE_OPCODE_QUERY_GID 0x18UL
- #define CMDQ_BASE_OPCODE_CREATE_QP1 0x13UL
- #define CMDQ_BASE_OPCODE_DESTROY_QP1 0x14UL
- #define CMDQ_BASE_OPCODE_CREATE_AH 0x15UL
- #define CMDQ_BASE_OPCODE_DESTROY_AH 0x16UL
- #define CMDQ_BASE_OPCODE_INITIALIZE_FW 0x80UL
- #define CMDQ_BASE_OPCODE_DEINITIALIZE_FW 0x81UL
- #define CMDQ_BASE_OPCODE_STOP_FUNC 0x82UL
- #define CMDQ_BASE_OPCODE_QUERY_FUNC 0x83UL
- #define CMDQ_BASE_OPCODE_SET_FUNC_RESOURCES 0x84UL
- #define CMDQ_BASE_OPCODE_READ_CONTEXT 0x85UL
- #define CMDQ_BASE_OPCODE_VF_BACKCHANNEL_REQUEST 0x86UL
- #define CMDQ_BASE_OPCODE_READ_VF_MEMORY 0x87UL
- #define CMDQ_BASE_OPCODE_COMPLETE_VF_REQUEST 0x88UL
- #define CMDQ_BASE_OPCODE_EXTEND_CONTEXT_ARRRAY 0x89UL
- #define CMDQ_BASE_OPCODE_MAP_TC_TO_COS 0x8aUL
- #define CMDQ_BASE_OPCODE_QUERY_VERSION 0x8bUL
- #define CMDQ_BASE_OPCODE_MODIFY_CC 0x8cUL
- #define CMDQ_BASE_OPCODE_QUERY_CC 0x8dUL
- #define CMDQ_BASE_OPCODE_QUERY_ROCE_STATS 0x8eUL
- #define CMDQ_BASE_OPCODE_QUERY_ROCE_STATS_EXT 0x92UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
-};
-
-/* Create QP command (96 bytes) */
-struct cmdq_create_qp {
- u8 opcode;
- #define CMDQ_CREATE_QP_OPCODE_CREATE_QP 0x1UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __le64 qp_handle;
- __le32 qp_flags;
- #define CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED 0x1UL
- #define CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION 0x2UL
- #define CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE 0x4UL
- #define CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED 0x8UL
- #define CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED 0x10UL
- #define CMDQ_CREATE_QP_QP_FLAGS_EXT_STATS_ENABLED 0x80UL
- #define CMDQ_CREATE_QP_QP_FLAGS_LAST \
- CMDQ_CREATE_QP_QP_FLAGS_EXT_STATS_ENABLED
+/* creq_query_qp_extend_resp_sb_tlv (size:512b/64B) */
+struct creq_query_qp_extend_resp_sb_tlv {
+ __le16 cmd_discr;
+ u8 reserved_8b;
+ u8 tlv_flags;
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_MORE 0x1UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_MORE_LAST 0x0UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_MORE_NOT_LAST 0x1UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_REQUIRED 0x2UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_REQUIRED_NO (0x0UL << 1)
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_REQUIRED_YES (0x1UL << 1)
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_REQUIRED_LAST \
+ CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_REQUIRED_YES
+ __le16 tlv_type;
+ __le16 length;
+ u8 total_size;
+ u8 reserved56[7];
+ u8 opcode;
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_OPCODE_QUERY_QP_EXTEND 0x91UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_OPCODE_LAST \
+ CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_OPCODE_QUERY_QP_EXTEND
+ u8 status;
+ __le16 cookie;
+ __le16 flags;
+ u8 resp_size;
+ u8 reserved8;
+ __le32 xid;
+ u8 state;
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_MASK 0xfUL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_SFT 0
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_RESET 0x0UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_INIT 0x1UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_RTR 0x2UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_RTS 0x3UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_SQD 0x4UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_SQE 0x5UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_ERR 0x6UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_LAST \
+ CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_ERR
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_UNUSED4_MASK 0xf0UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_UNUSED4_SFT 4
+ u8 reserved_8;
+ __le16 port_id;
+ __le32 qkey;
+ __le16 sgid_index;
+ u8 network_type;
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_NETWORK_TYPE_ROCEV1 0x0UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_NETWORK_TYPE_ROCEV2_IPV4 0x2UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_NETWORK_TYPE_ROCEV2_IPV6 0x3UL
+ #define CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_NETWORK_TYPE_LAST \
+ CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_NETWORK_TYPE_ROCEV2_IPV6
+ u8 unused_0;
+ __le32 dgid[4];
+ __le32 dest_qp_id;
+ u8 stat_collection_id;
+ u8 reservred_8;
+ __le16 reserved_16;
+};
- u8 type;
- #define CMDQ_CREATE_QP_TYPE_RC 0x2UL
- #define CMDQ_CREATE_QP_TYPE_UD 0x4UL
- #define CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE 0x6UL
- #define CMDQ_CREATE_QP_TYPE_GSI 0x7UL
- u8 sq_pg_size_sq_lvl;
- #define CMDQ_CREATE_QP_SQ_LVL_MASK 0xfUL
- #define CMDQ_CREATE_QP_SQ_LVL_SFT 0
- #define CMDQ_CREATE_QP_SQ_LVL_LVL_0 0x0UL
- #define CMDQ_CREATE_QP_SQ_LVL_LVL_1 0x1UL
- #define CMDQ_CREATE_QP_SQ_LVL_LVL_2 0x2UL
- #define CMDQ_CREATE_QP_SQ_PG_SIZE_MASK 0xf0UL
- #define CMDQ_CREATE_QP_SQ_PG_SIZE_SFT 4
- #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K (0x0UL << 4)
- #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K (0x1UL << 4)
- #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K (0x2UL << 4)
- #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M (0x3UL << 4)
- #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M (0x4UL << 4)
- #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G (0x5UL << 4)
- u8 rq_pg_size_rq_lvl;
- #define CMDQ_CREATE_QP_RQ_LVL_MASK 0xfUL
- #define CMDQ_CREATE_QP_RQ_LVL_SFT 0
- #define CMDQ_CREATE_QP_RQ_LVL_LVL_0 0x0UL
- #define CMDQ_CREATE_QP_RQ_LVL_LVL_1 0x1UL
- #define CMDQ_CREATE_QP_RQ_LVL_LVL_2 0x2UL
- #define CMDQ_CREATE_QP_RQ_PG_SIZE_MASK 0xf0UL
- #define CMDQ_CREATE_QP_RQ_PG_SIZE_SFT 4
- #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K (0x0UL << 4)
- #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K (0x1UL << 4)
- #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K (0x2UL << 4)
- #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M (0x3UL << 4)
- #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M (0x4UL << 4)
- #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G (0x5UL << 4)
- u8 unused_0;
- __le32 dpi;
- __le32 sq_size;
- __le32 rq_size;
- __le16 sq_fwo_sq_sge;
- #define CMDQ_CREATE_QP_SQ_SGE_MASK 0xfUL
- #define CMDQ_CREATE_QP_SQ_SGE_SFT 0
- #define CMDQ_CREATE_QP_SQ_FWO_MASK 0xfff0UL
- #define CMDQ_CREATE_QP_SQ_FWO_SFT 4
- __le16 rq_fwo_rq_sge;
- #define CMDQ_CREATE_QP_RQ_SGE_MASK 0xfUL
- #define CMDQ_CREATE_QP_RQ_SGE_SFT 0
- #define CMDQ_CREATE_QP_RQ_FWO_MASK 0xfff0UL
- #define CMDQ_CREATE_QP_RQ_FWO_SFT 4
- __le32 scq_cid;
- __le32 rcq_cid;
- __le32 srq_cid;
- __le32 pd_id;
- __le64 sq_pbl;
- __le64 rq_pbl;
- __le64 irrq_addr;
- __le64 orrq_addr;
-};
-
-/* Destroy QP command (24 bytes) */
-struct cmdq_destroy_qp {
- u8 opcode;
- #define CMDQ_DESTROY_QP_OPCODE_DESTROY_QP 0x2UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __le32 qp_cid;
- __le32 unused_0;
-};
-
-/* Modify QP command (112 bytes) */
-struct cmdq_modify_qp {
- u8 opcode;
- #define CMDQ_MODIFY_QP_OPCODE_MODIFY_QP 0x3UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __le32 modify_mask;
- #define CMDQ_MODIFY_QP_MODIFY_MASK_STATE 0x1UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY 0x2UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS 0x4UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_PKEY 0x8UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_QKEY 0x10UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_DGID 0x20UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL 0x40UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX 0x80UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT 0x100UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS 0x200UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC 0x400UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU 0x1000UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT 0x2000UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT 0x4000UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY 0x8000UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN 0x10000UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC 0x20000UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER 0x40000UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN 0x80000UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC 0x100000UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE 0x200000UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE 0x400000UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE 0x800000UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE 0x1000000UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA 0x2000000UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID 0x4000000UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC 0x8000000UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID 0x10000000UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_ENABLE_CC 0x20000000UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_TOS_ECN 0x40000000UL
- #define CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP 0x80000000UL
- __le32 qp_cid;
- u8 network_type_en_sqd_async_notify_new_state;
- #define CMDQ_MODIFY_QP_NEW_STATE_MASK 0xfUL
- #define CMDQ_MODIFY_QP_NEW_STATE_SFT 0
- #define CMDQ_MODIFY_QP_NEW_STATE_RESET 0x0UL
- #define CMDQ_MODIFY_QP_NEW_STATE_INIT 0x1UL
- #define CMDQ_MODIFY_QP_NEW_STATE_RTR 0x2UL
- #define CMDQ_MODIFY_QP_NEW_STATE_RTS 0x3UL
- #define CMDQ_MODIFY_QP_NEW_STATE_SQD 0x4UL
- #define CMDQ_MODIFY_QP_NEW_STATE_SQE 0x5UL
- #define CMDQ_MODIFY_QP_NEW_STATE_ERR 0x6UL
- #define CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY 0x10UL
- #define CMDQ_MODIFY_QP_NETWORK_TYPE_MASK 0xc0UL
- #define CMDQ_MODIFY_QP_NETWORK_TYPE_SFT 6
- #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1 (0x0UL << 6)
- #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4 (0x2UL << 6)
- #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6 (0x3UL << 6)
- u8 access;
- #define CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE 0x1UL
- #define CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE 0x2UL
- #define CMDQ_MODIFY_QP_ACCESS_REMOTE_READ 0x4UL
- #define CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC 0x8UL
- __le16 pkey;
- __le32 qkey;
- __le32 dgid[4];
- __le32 flow_label;
- __le16 sgid_index;
- u8 hop_limit;
- u8 traffic_class;
- __le16 dest_mac[3];
- u8 tos_dscp_tos_ecn;
- #define CMDQ_MODIFY_QP_TOS_ECN_MASK 0x3UL
- #define CMDQ_MODIFY_QP_TOS_ECN_SFT 0
- #define CMDQ_MODIFY_QP_TOS_DSCP_MASK 0xfcUL
- #define CMDQ_MODIFY_QP_TOS_DSCP_SFT 2
- u8 path_mtu;
- #define CMDQ_MODIFY_QP_PATH_MTU_MASK 0xf0UL
- #define CMDQ_MODIFY_QP_PATH_MTU_SFT 4
- #define CMDQ_MODIFY_QP_PATH_MTU_MTU_256 (0x0UL << 4)
- #define CMDQ_MODIFY_QP_PATH_MTU_MTU_512 (0x1UL << 4)
- #define CMDQ_MODIFY_QP_PATH_MTU_MTU_1024 (0x2UL << 4)
- #define CMDQ_MODIFY_QP_PATH_MTU_MTU_2048 (0x3UL << 4)
- #define CMDQ_MODIFY_QP_PATH_MTU_MTU_4096 (0x4UL << 4)
- #define CMDQ_MODIFY_QP_PATH_MTU_MTU_8192 (0x5UL << 4)
- u8 timeout;
- u8 retry_cnt;
- u8 rnr_retry;
- u8 min_rnr_timer;
- __le32 rq_psn;
- __le32 sq_psn;
- u8 max_rd_atomic;
- u8 max_dest_rd_atomic;
- __le16 enable_cc;
- #define CMDQ_MODIFY_QP_ENABLE_CC 0x1UL
- __le32 sq_size;
- __le32 rq_size;
- __le16 sq_sge;
- __le16 rq_sge;
- __le32 max_inline_data;
- __le32 dest_qp_id;
- __le32 unused_3;
- __le16 src_mac[3];
- __le16 vlan_pcp_vlan_dei_vlan_id;
- #define CMDQ_MODIFY_QP_VLAN_ID_MASK 0xfffUL
- #define CMDQ_MODIFY_QP_VLAN_ID_SFT 0
- #define CMDQ_MODIFY_QP_VLAN_DEI 0x1000UL
- #define CMDQ_MODIFY_QP_VLAN_PCP_MASK 0xe000UL
- #define CMDQ_MODIFY_QP_VLAN_PCP_SFT 13
-};
-
-/* Query QP command (24 bytes) */
-struct cmdq_query_qp {
- u8 opcode;
- #define CMDQ_QUERY_QP_OPCODE_QUERY_QP 0x4UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __le32 qp_cid;
- __le32 unused_0;
-};
-
-/* Create SRQ command (48 bytes) */
+/* cmdq_create_srq (size:384b/48B) */
struct cmdq_create_srq {
- u8 opcode;
- #define CMDQ_CREATE_SRQ_OPCODE_CREATE_SRQ 0x5UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __le64 srq_handle;
- __le16 pg_size_lvl;
- #define CMDQ_CREATE_SRQ_LVL_MASK 0x3UL
- #define CMDQ_CREATE_SRQ_LVL_SFT 0
- #define CMDQ_CREATE_SRQ_LVL_LVL_0 0x0UL
- #define CMDQ_CREATE_SRQ_LVL_LVL_1 0x1UL
- #define CMDQ_CREATE_SRQ_LVL_LVL_2 0x2UL
- #define CMDQ_CREATE_SRQ_PG_SIZE_MASK 0x1cUL
- #define CMDQ_CREATE_SRQ_PG_SIZE_SFT 2
- #define CMDQ_CREATE_SRQ_PG_SIZE_PG_4K (0x0UL << 2)
- #define CMDQ_CREATE_SRQ_PG_SIZE_PG_8K (0x1UL << 2)
- #define CMDQ_CREATE_SRQ_PG_SIZE_PG_64K (0x2UL << 2)
- #define CMDQ_CREATE_SRQ_PG_SIZE_PG_2M (0x3UL << 2)
- #define CMDQ_CREATE_SRQ_PG_SIZE_PG_8M (0x4UL << 2)
- #define CMDQ_CREATE_SRQ_PG_SIZE_PG_1G (0x5UL << 2)
- __le16 eventq_id;
- #define CMDQ_CREATE_SRQ_EVENTQ_ID_MASK 0xfffUL
- #define CMDQ_CREATE_SRQ_EVENTQ_ID_SFT 0
- __le16 srq_size;
- __le16 srq_fwo;
- __le32 dpi;
- __le32 pd_id;
- __le64 pbl;
-};
-
-/* Destroy SRQ command (24 bytes) */
+ u8 opcode;
+ #define CMDQ_CREATE_SRQ_OPCODE_CREATE_SRQ 0x5UL
+ #define CMDQ_CREATE_SRQ_OPCODE_LAST CMDQ_CREATE_SRQ_OPCODE_CREATE_SRQ
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __le64 srq_handle;
+ __le16 pg_size_lvl;
+ #define CMDQ_CREATE_SRQ_LVL_MASK 0x3UL
+ #define CMDQ_CREATE_SRQ_LVL_SFT 0
+ #define CMDQ_CREATE_SRQ_LVL_LVL_0 0x0UL
+ #define CMDQ_CREATE_SRQ_LVL_LVL_1 0x1UL
+ #define CMDQ_CREATE_SRQ_LVL_LVL_2 0x2UL
+ #define CMDQ_CREATE_SRQ_LVL_LAST CMDQ_CREATE_SRQ_LVL_LVL_2
+ #define CMDQ_CREATE_SRQ_PG_SIZE_MASK 0x1cUL
+ #define CMDQ_CREATE_SRQ_PG_SIZE_SFT 2
+ #define CMDQ_CREATE_SRQ_PG_SIZE_PG_4K (0x0UL << 2)
+ #define CMDQ_CREATE_SRQ_PG_SIZE_PG_8K (0x1UL << 2)
+ #define CMDQ_CREATE_SRQ_PG_SIZE_PG_64K (0x2UL << 2)
+ #define CMDQ_CREATE_SRQ_PG_SIZE_PG_2M (0x3UL << 2)
+ #define CMDQ_CREATE_SRQ_PG_SIZE_PG_8M (0x4UL << 2)
+ #define CMDQ_CREATE_SRQ_PG_SIZE_PG_1G (0x5UL << 2)
+ #define CMDQ_CREATE_SRQ_PG_SIZE_LAST CMDQ_CREATE_SRQ_PG_SIZE_PG_1G
+ #define CMDQ_CREATE_SRQ_UNUSED11_MASK 0xffe0UL
+ #define CMDQ_CREATE_SRQ_UNUSED11_SFT 5
+ __le16 eventq_id;
+ #define CMDQ_CREATE_SRQ_EVENTQ_ID_MASK 0xfffUL
+ #define CMDQ_CREATE_SRQ_EVENTQ_ID_SFT 0
+ #define CMDQ_CREATE_SRQ_UNUSED4_MASK 0xf000UL
+ #define CMDQ_CREATE_SRQ_UNUSED4_SFT 12
+ __le16 srq_size;
+ __le16 srq_fwo;
+ __le32 dpi;
+ __le32 pd_id;
+ __le64 pbl;
+};
+
+/* creq_create_srq_resp (size:128b/16B) */
+struct creq_create_srq_resp {
+ u8 type;
+ #define CREQ_CREATE_SRQ_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_CREATE_SRQ_RESP_TYPE_SFT 0
+ #define CREQ_CREATE_SRQ_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_CREATE_SRQ_RESP_TYPE_LAST CREQ_CREATE_SRQ_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 xid;
+ u8 v;
+ #define CREQ_CREATE_SRQ_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_CREATE_SRQ_RESP_EVENT_CREATE_SRQ 0x5UL
+ #define CREQ_CREATE_SRQ_RESP_EVENT_LAST CREQ_CREATE_SRQ_RESP_EVENT_CREATE_SRQ
+ u8 reserved48[6];
+};
+
+/* cmdq_destroy_srq (size:192b/24B) */
struct cmdq_destroy_srq {
- u8 opcode;
- #define CMDQ_DESTROY_SRQ_OPCODE_DESTROY_SRQ 0x6UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __le32 srq_cid;
- __le32 unused_0;
-};
-
-/* Query SRQ command (24 bytes) */
+ u8 opcode;
+ #define CMDQ_DESTROY_SRQ_OPCODE_DESTROY_SRQ 0x6UL
+ #define CMDQ_DESTROY_SRQ_OPCODE_LAST CMDQ_DESTROY_SRQ_OPCODE_DESTROY_SRQ
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __le32 srq_cid;
+ __le32 unused_0;
+};
+
+/* creq_destroy_srq_resp (size:128b/16B) */
+struct creq_destroy_srq_resp {
+ u8 type;
+ #define CREQ_DESTROY_SRQ_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_DESTROY_SRQ_RESP_TYPE_SFT 0
+ #define CREQ_DESTROY_SRQ_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_DESTROY_SRQ_RESP_TYPE_LAST CREQ_DESTROY_SRQ_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 xid;
+ u8 v;
+ #define CREQ_DESTROY_SRQ_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_DESTROY_SRQ_RESP_EVENT_DESTROY_SRQ 0x6UL
+ #define CREQ_DESTROY_SRQ_RESP_EVENT_LAST CREQ_DESTROY_SRQ_RESP_EVENT_DESTROY_SRQ
+ __le16 enable_for_arm[3];
+ #define CREQ_DESTROY_SRQ_RESP_UNUSED0_MASK 0xffffUL
+ #define CREQ_DESTROY_SRQ_RESP_UNUSED0_SFT 0
+ #define CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_MASK 0x30000UL
+ #define CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_SFT 16
+};
+
+/* cmdq_query_srq (size:192b/24B) */
struct cmdq_query_srq {
- u8 opcode;
- #define CMDQ_QUERY_SRQ_OPCODE_QUERY_SRQ 0x8UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __le32 srq_cid;
- __le32 unused_0;
-};
-
-/* Create CQ command (48 bytes) */
+ u8 opcode;
+ #define CMDQ_QUERY_SRQ_OPCODE_QUERY_SRQ 0x8UL
+ #define CMDQ_QUERY_SRQ_OPCODE_LAST CMDQ_QUERY_SRQ_OPCODE_QUERY_SRQ
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __le32 srq_cid;
+ __le32 unused_0;
+};
+
+/* creq_query_srq_resp (size:128b/16B) */
+struct creq_query_srq_resp {
+ u8 type;
+ #define CREQ_QUERY_SRQ_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_QUERY_SRQ_RESP_TYPE_SFT 0
+ #define CREQ_QUERY_SRQ_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_QUERY_SRQ_RESP_TYPE_LAST CREQ_QUERY_SRQ_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 size;
+ u8 v;
+ #define CREQ_QUERY_SRQ_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_QUERY_SRQ_RESP_EVENT_QUERY_SRQ 0x8UL
+ #define CREQ_QUERY_SRQ_RESP_EVENT_LAST CREQ_QUERY_SRQ_RESP_EVENT_QUERY_SRQ
+ u8 reserved48[6];
+};
+
+/* creq_query_srq_resp_sb (size:256b/32B) */
+struct creq_query_srq_resp_sb {
+ u8 opcode;
+ #define CREQ_QUERY_SRQ_RESP_SB_OPCODE_QUERY_SRQ 0x8UL
+ #define CREQ_QUERY_SRQ_RESP_SB_OPCODE_LAST CREQ_QUERY_SRQ_RESP_SB_OPCODE_QUERY_SRQ
+ u8 status;
+ __le16 cookie;
+ __le16 flags;
+ u8 resp_size;
+ u8 reserved8;
+ __le32 xid;
+ __le16 srq_limit;
+ __le16 reserved16;
+ __le32 data[4];
+};
+
+/* cmdq_create_cq (size:384b/48B) */
struct cmdq_create_cq {
- u8 opcode;
- #define CMDQ_CREATE_CQ_OPCODE_CREATE_CQ 0x9UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __le64 cq_handle;
- __le32 pg_size_lvl;
- #define CMDQ_CREATE_CQ_LVL_MASK 0x3UL
- #define CMDQ_CREATE_CQ_LVL_SFT 0
- #define CMDQ_CREATE_CQ_LVL_LVL_0 0x0UL
- #define CMDQ_CREATE_CQ_LVL_LVL_1 0x1UL
- #define CMDQ_CREATE_CQ_LVL_LVL_2 0x2UL
- #define CMDQ_CREATE_CQ_PG_SIZE_MASK 0x1cUL
- #define CMDQ_CREATE_CQ_PG_SIZE_SFT 2
- #define CMDQ_CREATE_CQ_PG_SIZE_PG_4K (0x0UL << 2)
- #define CMDQ_CREATE_CQ_PG_SIZE_PG_8K (0x1UL << 2)
- #define CMDQ_CREATE_CQ_PG_SIZE_PG_64K (0x2UL << 2)
- #define CMDQ_CREATE_CQ_PG_SIZE_PG_2M (0x3UL << 2)
- #define CMDQ_CREATE_CQ_PG_SIZE_PG_8M (0x4UL << 2)
- #define CMDQ_CREATE_CQ_PG_SIZE_PG_1G (0x5UL << 2)
- __le32 cq_fco_cnq_id;
- #define CMDQ_CREATE_CQ_CNQ_ID_MASK 0xfffUL
- #define CMDQ_CREATE_CQ_CNQ_ID_SFT 0
- #define CMDQ_CREATE_CQ_CQ_FCO_MASK 0xfffff000UL
- #define CMDQ_CREATE_CQ_CQ_FCO_SFT 12
- __le32 dpi;
- __le32 cq_size;
- __le64 pbl;
-};
-
-/* Destroy CQ command (24 bytes) */
+ u8 opcode;
+ #define CMDQ_CREATE_CQ_OPCODE_CREATE_CQ 0x9UL
+ #define CMDQ_CREATE_CQ_OPCODE_LAST CMDQ_CREATE_CQ_OPCODE_CREATE_CQ
+ u8 cmd_size;
+ __le16 flags;
+ #define CMDQ_CREATE_CQ_FLAGS_DISABLE_CQ_OVERFLOW_DETECTION 0x1UL
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __le64 cq_handle;
+ __le32 pg_size_lvl;
+ #define CMDQ_CREATE_CQ_LVL_MASK 0x3UL
+ #define CMDQ_CREATE_CQ_LVL_SFT 0
+ #define CMDQ_CREATE_CQ_LVL_LVL_0 0x0UL
+ #define CMDQ_CREATE_CQ_LVL_LVL_1 0x1UL
+ #define CMDQ_CREATE_CQ_LVL_LVL_2 0x2UL
+ #define CMDQ_CREATE_CQ_LVL_LAST CMDQ_CREATE_CQ_LVL_LVL_2
+ #define CMDQ_CREATE_CQ_PG_SIZE_MASK 0x1cUL
+ #define CMDQ_CREATE_CQ_PG_SIZE_SFT 2
+ #define CMDQ_CREATE_CQ_PG_SIZE_PG_4K (0x0UL << 2)
+ #define CMDQ_CREATE_CQ_PG_SIZE_PG_8K (0x1UL << 2)
+ #define CMDQ_CREATE_CQ_PG_SIZE_PG_64K (0x2UL << 2)
+ #define CMDQ_CREATE_CQ_PG_SIZE_PG_2M (0x3UL << 2)
+ #define CMDQ_CREATE_CQ_PG_SIZE_PG_8M (0x4UL << 2)
+ #define CMDQ_CREATE_CQ_PG_SIZE_PG_1G (0x5UL << 2)
+ #define CMDQ_CREATE_CQ_PG_SIZE_LAST CMDQ_CREATE_CQ_PG_SIZE_PG_1G
+ #define CMDQ_CREATE_CQ_UNUSED27_MASK 0xffffffe0UL
+ #define CMDQ_CREATE_CQ_UNUSED27_SFT 5
+ __le32 cq_fco_cnq_id;
+ #define CMDQ_CREATE_CQ_CNQ_ID_MASK 0xfffUL
+ #define CMDQ_CREATE_CQ_CNQ_ID_SFT 0
+ #define CMDQ_CREATE_CQ_CQ_FCO_MASK 0xfffff000UL
+ #define CMDQ_CREATE_CQ_CQ_FCO_SFT 12
+ __le32 dpi;
+ __le32 cq_size;
+ __le64 pbl;
+};
+
+/* creq_create_cq_resp (size:128b/16B) */
+struct creq_create_cq_resp {
+ u8 type;
+ #define CREQ_CREATE_CQ_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_CREATE_CQ_RESP_TYPE_SFT 0
+ #define CREQ_CREATE_CQ_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_CREATE_CQ_RESP_TYPE_LAST CREQ_CREATE_CQ_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 xid;
+ u8 v;
+ #define CREQ_CREATE_CQ_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_CREATE_CQ_RESP_EVENT_CREATE_CQ 0x9UL
+ #define CREQ_CREATE_CQ_RESP_EVENT_LAST CREQ_CREATE_CQ_RESP_EVENT_CREATE_CQ
+ u8 reserved48[6];
+};
+
+/* cmdq_destroy_cq (size:192b/24B) */
struct cmdq_destroy_cq {
- u8 opcode;
- #define CMDQ_DESTROY_CQ_OPCODE_DESTROY_CQ 0xaUL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __le32 cq_cid;
- __le32 unused_0;
-};
-
-/* Resize CQ command (40 bytes) */
+ u8 opcode;
+ #define CMDQ_DESTROY_CQ_OPCODE_DESTROY_CQ 0xaUL
+ #define CMDQ_DESTROY_CQ_OPCODE_LAST CMDQ_DESTROY_CQ_OPCODE_DESTROY_CQ
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __le32 cq_cid;
+ __le32 unused_0;
+};
+
+/* creq_destroy_cq_resp (size:128b/16B) */
+struct creq_destroy_cq_resp {
+ u8 type;
+ #define CREQ_DESTROY_CQ_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_DESTROY_CQ_RESP_TYPE_SFT 0
+ #define CREQ_DESTROY_CQ_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_DESTROY_CQ_RESP_TYPE_LAST CREQ_DESTROY_CQ_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 xid;
+ u8 v;
+ #define CREQ_DESTROY_CQ_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_DESTROY_CQ_RESP_EVENT_DESTROY_CQ 0xaUL
+ #define CREQ_DESTROY_CQ_RESP_EVENT_LAST CREQ_DESTROY_CQ_RESP_EVENT_DESTROY_CQ
+ __le16 cq_arm_lvl;
+ #define CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_MASK 0x3UL
+ #define CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_SFT 0
+ __le16 total_cnq_events;
+ __le16 reserved16;
+};
+
+/* cmdq_resize_cq (size:320b/40B) */
struct cmdq_resize_cq {
- u8 opcode;
- #define CMDQ_RESIZE_CQ_OPCODE_RESIZE_CQ 0xcUL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __le32 cq_cid;
- __le32 new_cq_size_pg_size_lvl;
- #define CMDQ_RESIZE_CQ_LVL_MASK 0x3UL
- #define CMDQ_RESIZE_CQ_LVL_SFT 0
- #define CMDQ_RESIZE_CQ_LVL_LVL_0 0x0UL
- #define CMDQ_RESIZE_CQ_LVL_LVL_1 0x1UL
- #define CMDQ_RESIZE_CQ_LVL_LVL_2 0x2UL
- #define CMDQ_RESIZE_CQ_PG_SIZE_MASK 0x1cUL
- #define CMDQ_RESIZE_CQ_PG_SIZE_SFT 2
- #define CMDQ_RESIZE_CQ_PG_SIZE_PG_4K (0x0UL << 2)
- #define CMDQ_RESIZE_CQ_PG_SIZE_PG_8K (0x1UL << 2)
- #define CMDQ_RESIZE_CQ_PG_SIZE_PG_64K (0x2UL << 2)
- #define CMDQ_RESIZE_CQ_PG_SIZE_PG_2M (0x3UL << 2)
- #define CMDQ_RESIZE_CQ_PG_SIZE_PG_8M (0x4UL << 2)
- #define CMDQ_RESIZE_CQ_PG_SIZE_PG_1G (0x5UL << 2)
- #define CMDQ_RESIZE_CQ_NEW_CQ_SIZE_MASK 0x1fffe0UL
- #define CMDQ_RESIZE_CQ_NEW_CQ_SIZE_SFT 5
- __le64 new_pbl;
- __le32 new_cq_fco;
- __le32 unused_2;
-};
-
-/* Allocate MRW command (32 bytes) */
+ u8 opcode;
+ #define CMDQ_RESIZE_CQ_OPCODE_RESIZE_CQ 0xcUL
+ #define CMDQ_RESIZE_CQ_OPCODE_LAST CMDQ_RESIZE_CQ_OPCODE_RESIZE_CQ
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __le32 cq_cid;
+ __le32 new_cq_size_pg_size_lvl;
+ #define CMDQ_RESIZE_CQ_LVL_MASK 0x3UL
+ #define CMDQ_RESIZE_CQ_LVL_SFT 0
+ #define CMDQ_RESIZE_CQ_LVL_LVL_0 0x0UL
+ #define CMDQ_RESIZE_CQ_LVL_LVL_1 0x1UL
+ #define CMDQ_RESIZE_CQ_LVL_LVL_2 0x2UL
+ #define CMDQ_RESIZE_CQ_LVL_LAST CMDQ_RESIZE_CQ_LVL_LVL_2
+ #define CMDQ_RESIZE_CQ_PG_SIZE_MASK 0x1cUL
+ #define CMDQ_RESIZE_CQ_PG_SIZE_SFT 2
+ #define CMDQ_RESIZE_CQ_PG_SIZE_PG_4K (0x0UL << 2)
+ #define CMDQ_RESIZE_CQ_PG_SIZE_PG_8K (0x1UL << 2)
+ #define CMDQ_RESIZE_CQ_PG_SIZE_PG_64K (0x2UL << 2)
+ #define CMDQ_RESIZE_CQ_PG_SIZE_PG_2M (0x3UL << 2)
+ #define CMDQ_RESIZE_CQ_PG_SIZE_PG_8M (0x4UL << 2)
+ #define CMDQ_RESIZE_CQ_PG_SIZE_PG_1G (0x5UL << 2)
+ #define CMDQ_RESIZE_CQ_PG_SIZE_LAST CMDQ_RESIZE_CQ_PG_SIZE_PG_1G
+ #define CMDQ_RESIZE_CQ_NEW_CQ_SIZE_MASK 0x1fffffe0UL
+ #define CMDQ_RESIZE_CQ_NEW_CQ_SIZE_SFT 5
+ __le64 new_pbl;
+ __le32 new_cq_fco;
+ __le32 unused_0;
+};
+
+/* creq_resize_cq_resp (size:128b/16B) */
+struct creq_resize_cq_resp {
+ u8 type;
+ #define CREQ_RESIZE_CQ_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_RESIZE_CQ_RESP_TYPE_SFT 0
+ #define CREQ_RESIZE_CQ_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_RESIZE_CQ_RESP_TYPE_LAST CREQ_RESIZE_CQ_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 xid;
+ u8 v;
+ #define CREQ_RESIZE_CQ_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_RESIZE_CQ_RESP_EVENT_RESIZE_CQ 0xcUL
+ #define CREQ_RESIZE_CQ_RESP_EVENT_LAST CREQ_RESIZE_CQ_RESP_EVENT_RESIZE_CQ
+ u8 reserved48[6];
+};
+
+/* cmdq_allocate_mrw (size:256b/32B) */
struct cmdq_allocate_mrw {
- u8 opcode;
- #define CMDQ_ALLOCATE_MRW_OPCODE_ALLOCATE_MRW 0xdUL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __le64 mrw_handle;
- u8 mrw_flags;
- #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MASK 0xfUL
- #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_SFT 0
- #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR 0x0UL
- #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR 0x1UL
- #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 0x2UL
- #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A 0x3UL
- #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B 0x4UL
- u8 access;
- #define CMDQ_ALLOCATE_MRW_ACCESS_RESERVED_MASK 0x1fUL
- #define CMDQ_ALLOCATE_MRW_ACCESS_RESERVED_SFT 0
- #define CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY 0x20UL
- __le16 unused_1;
- __le32 pd_id;
-};
-
-/* De-allocate key command (24 bytes) */
+ u8 opcode;
+ #define CMDQ_ALLOCATE_MRW_OPCODE_ALLOCATE_MRW 0xdUL
+ #define CMDQ_ALLOCATE_MRW_OPCODE_LAST CMDQ_ALLOCATE_MRW_OPCODE_ALLOCATE_MRW
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __le64 mrw_handle;
+ u8 mrw_flags;
+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MASK 0xfUL
+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_SFT 0
+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR 0x0UL
+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR 0x1UL
+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 0x2UL
+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A 0x3UL
+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B 0x4UL
+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_LAST CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B
+ #define CMDQ_ALLOCATE_MRW_UNUSED4_MASK 0xf0UL
+ #define CMDQ_ALLOCATE_MRW_UNUSED4_SFT 4
+ u8 access;
+ #define CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY 0x20UL
+ __le16 unused16;
+ __le32 pd_id;
+};
+
+/* creq_allocate_mrw_resp (size:128b/16B) */
+struct creq_allocate_mrw_resp {
+ u8 type;
+ #define CREQ_ALLOCATE_MRW_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_ALLOCATE_MRW_RESP_TYPE_SFT 0
+ #define CREQ_ALLOCATE_MRW_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_ALLOCATE_MRW_RESP_TYPE_LAST CREQ_ALLOCATE_MRW_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 xid;
+ u8 v;
+ #define CREQ_ALLOCATE_MRW_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_ALLOCATE_MRW_RESP_EVENT_ALLOCATE_MRW 0xdUL
+ #define CREQ_ALLOCATE_MRW_RESP_EVENT_LAST CREQ_ALLOCATE_MRW_RESP_EVENT_ALLOCATE_MRW
+ u8 reserved48[6];
+};
+
+/* cmdq_deallocate_key (size:192b/24B) */
struct cmdq_deallocate_key {
- u8 opcode;
- #define CMDQ_DEALLOCATE_KEY_OPCODE_DEALLOCATE_KEY 0xeUL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- u8 mrw_flags;
- #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MASK 0xfUL
- #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_SFT 0
- #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MR 0x0UL
- #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_PMR 0x1UL
- #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE1 0x2UL
- #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2A 0x3UL
- #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2B 0x4UL
- u8 unused_1[3];
- __le32 key;
-};
-
-/* Register MR command (48 bytes) */
+ u8 opcode;
+ #define CMDQ_DEALLOCATE_KEY_OPCODE_DEALLOCATE_KEY 0xeUL
+ #define CMDQ_DEALLOCATE_KEY_OPCODE_LAST CMDQ_DEALLOCATE_KEY_OPCODE_DEALLOCATE_KEY
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ u8 mrw_flags;
+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MASK 0xfUL
+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_SFT 0
+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MR 0x0UL
+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_PMR 0x1UL
+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE1 0x2UL
+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2A 0x3UL
+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2B 0x4UL
+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_LAST CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2B
+ #define CMDQ_DEALLOCATE_KEY_UNUSED4_MASK 0xf0UL
+ #define CMDQ_DEALLOCATE_KEY_UNUSED4_SFT 4
+ u8 unused24[3];
+ __le32 key;
+};
+
+/* creq_deallocate_key_resp (size:128b/16B) */
+struct creq_deallocate_key_resp {
+ u8 type;
+ #define CREQ_DEALLOCATE_KEY_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_DEALLOCATE_KEY_RESP_TYPE_SFT 0
+ #define CREQ_DEALLOCATE_KEY_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_DEALLOCATE_KEY_RESP_TYPE_LAST CREQ_DEALLOCATE_KEY_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 xid;
+ u8 v;
+ #define CREQ_DEALLOCATE_KEY_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_DEALLOCATE_KEY_RESP_EVENT_DEALLOCATE_KEY 0xeUL
+ #define CREQ_DEALLOCATE_KEY_RESP_EVENT_LAST CREQ_DEALLOCATE_KEY_RESP_EVENT_DEALLOCATE_KEY
+ __le16 reserved16;
+ __le32 bound_window_info;
+};
+
+/* cmdq_register_mr (size:384b/48B) */
struct cmdq_register_mr {
- u8 opcode;
- #define CMDQ_REGISTER_MR_OPCODE_REGISTER_MR 0xfUL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- u8 log2_pg_size_lvl;
- #define CMDQ_REGISTER_MR_LVL_MASK 0x3UL
- #define CMDQ_REGISTER_MR_LVL_SFT 0
- #define CMDQ_REGISTER_MR_LVL_LVL_0 0x0UL
- #define CMDQ_REGISTER_MR_LVL_LVL_1 0x1UL
- #define CMDQ_REGISTER_MR_LVL_LVL_2 0x2UL
+ u8 opcode;
+ #define CMDQ_REGISTER_MR_OPCODE_REGISTER_MR 0xfUL
+ #define CMDQ_REGISTER_MR_OPCODE_LAST CMDQ_REGISTER_MR_OPCODE_REGISTER_MR
+ u8 cmd_size;
+ __le16 flags;
+ #define CMDQ_REGISTER_MR_FLAGS_ALLOC_MR 0x1UL
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ u8 log2_pg_size_lvl;
+ #define CMDQ_REGISTER_MR_LVL_MASK 0x3UL
+ #define CMDQ_REGISTER_MR_LVL_SFT 0
+ #define CMDQ_REGISTER_MR_LVL_LVL_0 0x0UL
+ #define CMDQ_REGISTER_MR_LVL_LVL_1 0x1UL
+ #define CMDQ_REGISTER_MR_LVL_LVL_2 0x2UL
#define CMDQ_REGISTER_MR_LVL_LAST CMDQ_REGISTER_MR_LVL_LVL_2
- #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK 0x7cUL
- #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT 2
+ #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK 0x7cUL
+ #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT 2
#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_4K (0xcUL << 2)
#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_8K (0xdUL << 2)
#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_64K (0x10UL << 2)
@@ -1549,16 +1378,15 @@ struct cmdq_register_mr {
#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_2M (0x15UL << 2)
#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_4M (0x16UL << 2)
#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_1G (0x1eUL << 2)
- #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_LAST \
- CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_1G
+ #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_LAST CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_1G
#define CMDQ_REGISTER_MR_UNUSED1 0x80UL
- u8 access;
- #define CMDQ_REGISTER_MR_ACCESS_LOCAL_WRITE 0x1UL
- #define CMDQ_REGISTER_MR_ACCESS_REMOTE_READ 0x2UL
- #define CMDQ_REGISTER_MR_ACCESS_REMOTE_WRITE 0x4UL
- #define CMDQ_REGISTER_MR_ACCESS_REMOTE_ATOMIC 0x8UL
- #define CMDQ_REGISTER_MR_ACCESS_MW_BIND 0x10UL
- #define CMDQ_REGISTER_MR_ACCESS_ZERO_BASED 0x20UL
+ u8 access;
+ #define CMDQ_REGISTER_MR_ACCESS_LOCAL_WRITE 0x1UL
+ #define CMDQ_REGISTER_MR_ACCESS_REMOTE_READ 0x2UL
+ #define CMDQ_REGISTER_MR_ACCESS_REMOTE_WRITE 0x4UL
+ #define CMDQ_REGISTER_MR_ACCESS_REMOTE_ATOMIC 0x8UL
+ #define CMDQ_REGISTER_MR_ACCESS_MW_BIND 0x10UL
+ #define CMDQ_REGISTER_MR_ACCESS_ZERO_BASED 0x20UL
__le16 log2_pbl_pg_size;
#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_MASK 0x1fUL
#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_SFT 0
@@ -1570,1448 +1398,509 @@ struct cmdq_register_mr {
#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_2M 0x15UL
#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4M 0x16UL
#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G 0x1eUL
- #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_LAST \
- CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G
+ #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_LAST CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G
#define CMDQ_REGISTER_MR_UNUSED11_MASK 0xffe0UL
#define CMDQ_REGISTER_MR_UNUSED11_SFT 5
- __le32 key;
- __le64 pbl;
- __le64 va;
- __le64 mr_size;
+ __le32 key;
+ __le64 pbl;
+ __le64 va;
+ __le64 mr_size;
};
-/* Deregister MR command (24 bytes) */
-struct cmdq_deregister_mr {
- u8 opcode;
- #define CMDQ_DEREGISTER_MR_OPCODE_DEREGISTER_MR 0x10UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __le32 lkey;
- __le32 unused_0;
-};
-
-/* Add GID command (48 bytes) */
-struct cmdq_add_gid {
- u8 opcode;
- #define CMDQ_ADD_GID_OPCODE_ADD_GID 0x11UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __be32 gid[4];
- __be16 src_mac[3];
- __le16 vlan;
- #define CMDQ_ADD_GID_VLAN_VLAN_ID_MASK 0xfffUL
- #define CMDQ_ADD_GID_VLAN_VLAN_ID_SFT 0
- #define CMDQ_ADD_GID_VLAN_TPID_MASK 0x7000UL
- #define CMDQ_ADD_GID_VLAN_TPID_SFT 12
- #define CMDQ_ADD_GID_VLAN_TPID_TPID_88A8 (0x0UL << 12)
- #define CMDQ_ADD_GID_VLAN_TPID_TPID_8100 (0x1UL << 12)
- #define CMDQ_ADD_GID_VLAN_TPID_TPID_9100 (0x2UL << 12)
- #define CMDQ_ADD_GID_VLAN_TPID_TPID_9200 (0x3UL << 12)
- #define CMDQ_ADD_GID_VLAN_TPID_TPID_9300 (0x4UL << 12)
- #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG1 (0x5UL << 12)
- #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG2 (0x6UL << 12)
- #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3 (0x7UL << 12)
- #define CMDQ_ADD_GID_VLAN_TPID_LAST CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3
- #define CMDQ_ADD_GID_VLAN_VLAN_EN 0x8000UL
- __le16 ipid;
- __le16 stats_ctx;
- #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_MASK 0x7fffUL
- #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_SFT 0
- #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_VALID 0x8000UL
- __le32 unused_0;
-};
-
-/* Delete GID command (24 bytes) */
-struct cmdq_delete_gid {
- u8 opcode;
- #define CMDQ_DELETE_GID_OPCODE_DELETE_GID 0x12UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __le16 gid_index;
- __le16 unused_0;
- __le32 unused_1;
-};
-
-/* Modify GID command (48 bytes) */
-struct cmdq_modify_gid {
- u8 opcode;
- #define CMDQ_MODIFY_GID_OPCODE_MODIFY_GID 0x17UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __be32 gid[4];
- __be16 src_mac[3];
- __le16 vlan;
- #define CMDQ_MODIFY_GID_VLAN_VLAN_ID_MASK 0xfffUL
- #define CMDQ_MODIFY_GID_VLAN_VLAN_ID_SFT 0
- #define CMDQ_MODIFY_GID_VLAN_TPID_MASK 0x7000UL
- #define CMDQ_MODIFY_GID_VLAN_TPID_SFT 12
- #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_88A8 (0x0UL << 12)
- #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_8100 (0x1UL << 12)
- #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9100 (0x2UL << 12)
- #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9200 (0x3UL << 12)
- #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9300 (0x4UL << 12)
- #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG1 (0x5UL << 12)
- #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG2 (0x6UL << 12)
- #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3 (0x7UL << 12)
- #define CMDQ_MODIFY_GID_VLAN_TPID_LAST \
- CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3
- #define CMDQ_MODIFY_GID_VLAN_VLAN_EN 0x8000UL
- __le16 ipid;
- __le16 gid_index;
- __le16 stats_ctx;
- #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_MASK 0x7fffUL
- #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_SFT 0
- #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_VALID 0x8000UL
- __le16 unused_0;
-};
-
-/* Query GID command (24 bytes) */
-struct cmdq_query_gid {
- u8 opcode;
- #define CMDQ_QUERY_GID_OPCODE_QUERY_GID 0x18UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __le16 gid_index;
- __le16 unused_0;
- __le32 unused_1;
-};
-
-/* Create QP1 command (80 bytes) */
-struct cmdq_create_qp1 {
- u8 opcode;
- #define CMDQ_CREATE_QP1_OPCODE_CREATE_QP1 0x13UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __le64 qp_handle;
- __le32 qp_flags;
- #define CMDQ_CREATE_QP1_QP_FLAGS_SRQ_USED 0x1UL
- #define CMDQ_CREATE_QP1_QP_FLAGS_FORCE_COMPLETION 0x2UL
- #define CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE 0x4UL
- u8 type;
- #define CMDQ_CREATE_QP1_TYPE_GSI 0x1UL
- u8 sq_pg_size_sq_lvl;
- #define CMDQ_CREATE_QP1_SQ_LVL_MASK 0xfUL
- #define CMDQ_CREATE_QP1_SQ_LVL_SFT 0
- #define CMDQ_CREATE_QP1_SQ_LVL_LVL_0 0x0UL
- #define CMDQ_CREATE_QP1_SQ_LVL_LVL_1 0x1UL
- #define CMDQ_CREATE_QP1_SQ_LVL_LVL_2 0x2UL
- #define CMDQ_CREATE_QP1_SQ_PG_SIZE_MASK 0xf0UL
- #define CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT 4
- #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K (0x0UL << 4)
- #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K (0x1UL << 4)
- #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K (0x2UL << 4)
- #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M (0x3UL << 4)
- #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M (0x4UL << 4)
- #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G (0x5UL << 4)
- u8 rq_pg_size_rq_lvl;
- #define CMDQ_CREATE_QP1_RQ_LVL_MASK 0xfUL
- #define CMDQ_CREATE_QP1_RQ_LVL_SFT 0
- #define CMDQ_CREATE_QP1_RQ_LVL_LVL_0 0x0UL
- #define CMDQ_CREATE_QP1_RQ_LVL_LVL_1 0x1UL
- #define CMDQ_CREATE_QP1_RQ_LVL_LVL_2 0x2UL
- #define CMDQ_CREATE_QP1_RQ_PG_SIZE_MASK 0xf0UL
- #define CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT 4
- #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K (0x0UL << 4)
- #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K (0x1UL << 4)
- #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K (0x2UL << 4)
- #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M (0x3UL << 4)
- #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M (0x4UL << 4)
- #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G (0x5UL << 4)
- u8 unused_0;
- __le32 dpi;
- __le32 sq_size;
- __le32 rq_size;
- __le16 sq_fwo_sq_sge;
- #define CMDQ_CREATE_QP1_SQ_SGE_MASK 0xfUL
- #define CMDQ_CREATE_QP1_SQ_SGE_SFT 0
- #define CMDQ_CREATE_QP1_SQ_FWO_MASK 0xfff0UL
- #define CMDQ_CREATE_QP1_SQ_FWO_SFT 4
- __le16 rq_fwo_rq_sge;
- #define CMDQ_CREATE_QP1_RQ_SGE_MASK 0xfUL
- #define CMDQ_CREATE_QP1_RQ_SGE_SFT 0
- #define CMDQ_CREATE_QP1_RQ_FWO_MASK 0xfff0UL
- #define CMDQ_CREATE_QP1_RQ_FWO_SFT 4
- __le32 scq_cid;
- __le32 rcq_cid;
- __le32 srq_cid;
- __le32 pd_id;
- __le64 sq_pbl;
- __le64 rq_pbl;
-};
-
-/* Destroy QP1 command (24 bytes) */
-struct cmdq_destroy_qp1 {
- u8 opcode;
- #define CMDQ_DESTROY_QP1_OPCODE_DESTROY_QP1 0x14UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __le32 qp1_cid;
- __le32 unused_0;
-};
-
-/* Create AH command (64 bytes) */
-struct cmdq_create_ah {
- u8 opcode;
- #define CMDQ_CREATE_AH_OPCODE_CREATE_AH 0x15UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __le64 ah_handle;
- __le32 dgid[4];
- u8 type;
- #define CMDQ_CREATE_AH_TYPE_V1 0x0UL
- #define CMDQ_CREATE_AH_TYPE_V2IPV4 0x2UL
- #define CMDQ_CREATE_AH_TYPE_V2IPV6 0x3UL
- u8 hop_limit;
- __le16 sgid_index;
- __le32 dest_vlan_id_flow_label;
- #define CMDQ_CREATE_AH_FLOW_LABEL_MASK 0xfffffUL
- #define CMDQ_CREATE_AH_FLOW_LABEL_SFT 0
- #define CMDQ_CREATE_AH_DEST_VLAN_ID_MASK 0xfff00000UL
- #define CMDQ_CREATE_AH_DEST_VLAN_ID_SFT 20
- __le32 pd_id;
- __le32 unused_0;
- __le16 dest_mac[3];
- u8 traffic_class;
- u8 unused_1;
-};
-
-/* Destroy AH command (24 bytes) */
-struct cmdq_destroy_ah {
- u8 opcode;
- #define CMDQ_DESTROY_AH_OPCODE_DESTROY_AH 0x16UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __le32 ah_cid;
- __le32 unused_0;
-};
-
-/* Initialize Firmware command (112 bytes) */
-struct cmdq_initialize_fw {
- u8 opcode;
- #define CMDQ_INITIALIZE_FW_OPCODE_INITIALIZE_FW 0x80UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- u8 qpc_pg_size_qpc_lvl;
- #define CMDQ_INITIALIZE_FW_QPC_LVL_MASK 0xfUL
- #define CMDQ_INITIALIZE_FW_QPC_LVL_SFT 0
- #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_0 0x0UL
- #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_1 0x1UL
- #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_2 0x2UL
- #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_MASK 0xf0UL
- #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT 4
- #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K (0x0UL << 4)
- #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8K (0x1UL << 4)
- #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_64K (0x2UL << 4)
- #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_2M (0x3UL << 4)
- #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8M (0x4UL << 4)
- #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G (0x5UL << 4)
- u8 mrw_pg_size_mrw_lvl;
- #define CMDQ_INITIALIZE_FW_MRW_LVL_MASK 0xfUL
- #define CMDQ_INITIALIZE_FW_MRW_LVL_SFT 0
- #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_0 0x0UL
- #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_1 0x1UL
- #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_2 0x2UL
- #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_MASK 0xf0UL
- #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_SFT 4
- #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_4K (0x0UL << 4)
- #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8K (0x1UL << 4)
- #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_64K (0x2UL << 4)
- #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_2M (0x3UL << 4)
- #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8M (0x4UL << 4)
- #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_1G (0x5UL << 4)
- u8 srq_pg_size_srq_lvl;
- #define CMDQ_INITIALIZE_FW_SRQ_LVL_MASK 0xfUL
- #define CMDQ_INITIALIZE_FW_SRQ_LVL_SFT 0
- #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_0 0x0UL
- #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_1 0x1UL
- #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_2 0x2UL
- #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_MASK 0xf0UL
- #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_SFT 4
- #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_4K (0x0UL << 4)
- #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8K (0x1UL << 4)
- #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_64K (0x2UL << 4)
- #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_2M (0x3UL << 4)
- #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8M (0x4UL << 4)
- #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_1G (0x5UL << 4)
- u8 cq_pg_size_cq_lvl;
- #define CMDQ_INITIALIZE_FW_CQ_LVL_MASK 0xfUL
- #define CMDQ_INITIALIZE_FW_CQ_LVL_SFT 0
- #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_0 0x0UL
- #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_1 0x1UL
- #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_2 0x2UL
- #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_MASK 0xf0UL
- #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_SFT 4
- #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_4K (0x0UL << 4)
- #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8K (0x1UL << 4)
- #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_64K (0x2UL << 4)
- #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_2M (0x3UL << 4)
- #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8M (0x4UL << 4)
- #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_1G (0x5UL << 4)
- u8 tqm_pg_size_tqm_lvl;
- #define CMDQ_INITIALIZE_FW_TQM_LVL_MASK 0xfUL
- #define CMDQ_INITIALIZE_FW_TQM_LVL_SFT 0
- #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_0 0x0UL
- #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_1 0x1UL
- #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_2 0x2UL
- #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_MASK 0xf0UL
- #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_SFT 4
- #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_4K (0x0UL << 4)
- #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8K (0x1UL << 4)
- #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_64K (0x2UL << 4)
- #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_2M (0x3UL << 4)
- #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8M (0x4UL << 4)
- #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_1G (0x5UL << 4)
- u8 tim_pg_size_tim_lvl;
- #define CMDQ_INITIALIZE_FW_TIM_LVL_MASK 0xfUL
- #define CMDQ_INITIALIZE_FW_TIM_LVL_SFT 0
- #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_0 0x0UL
- #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_1 0x1UL
- #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_2 0x2UL
- #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_MASK 0xf0UL
- #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_SFT 4
- #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_4K (0x0UL << 4)
- #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8K (0x1UL << 4)
- #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_64K (0x2UL << 4)
- #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_2M (0x3UL << 4)
- #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8M (0x4UL << 4)
- #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_1G (0x5UL << 4)
- /* This value is (log-base-2-of-DBR-page-size - 12).
- * 0 for 4KB. HW supported values are enumerated below.
- */
- __le16 log2_dbr_pg_size;
- #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_MASK 0xfUL
- #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_SFT 0
- #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_4K 0x0UL
- #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_8K 0x1UL
- #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_16K 0x2UL
- #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_32K 0x3UL
- #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_64K 0x4UL
- #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_128K 0x5UL
- #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_256K 0x6UL
- #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_512K 0x7UL
- #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_1M 0x8UL
- #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_2M 0x9UL
- #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_4M 0xaUL
- #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_8M 0xbUL
- #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_16M 0xcUL
- #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_32M 0xdUL
- #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_64M 0xeUL
- #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_128M 0xfUL
- #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_LAST \
- CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_128M
- __le64 qpc_page_dir;
- __le64 mrw_page_dir;
- __le64 srq_page_dir;
- __le64 cq_page_dir;
- __le64 tqm_page_dir;
- __le64 tim_page_dir;
- __le32 number_of_qp;
- __le32 number_of_mrw;
- __le32 number_of_srq;
- __le32 number_of_cq;
- __le32 max_qp_per_vf;
- __le32 max_mrw_per_vf;
- __le32 max_srq_per_vf;
- __le32 max_cq_per_vf;
- __le32 max_gid_per_vf;
- __le32 stat_ctx_id;
-};
-
-/* De-initialize Firmware command (16 bytes) */
-struct cmdq_deinitialize_fw {
- u8 opcode;
- #define CMDQ_DEINITIALIZE_FW_OPCODE_DEINITIALIZE_FW 0x81UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
-};
-
-/* Stop function command (16 bytes) */
-struct cmdq_stop_func {
- u8 opcode;
- #define CMDQ_STOP_FUNC_OPCODE_STOP_FUNC 0x82UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
-};
-
-/* Query function command (16 bytes) */
-struct cmdq_query_func {
- u8 opcode;
- #define CMDQ_QUERY_FUNC_OPCODE_QUERY_FUNC 0x83UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
+/* creq_register_mr_resp (size:128b/16B) */
+struct creq_register_mr_resp {
+ u8 type;
+ #define CREQ_REGISTER_MR_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_REGISTER_MR_RESP_TYPE_SFT 0
+ #define CREQ_REGISTER_MR_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_REGISTER_MR_RESP_TYPE_LAST CREQ_REGISTER_MR_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 xid;
+ u8 v;
+ #define CREQ_REGISTER_MR_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_REGISTER_MR_RESP_EVENT_REGISTER_MR 0xfUL
+ #define CREQ_REGISTER_MR_RESP_EVENT_LAST CREQ_REGISTER_MR_RESP_EVENT_REGISTER_MR
+ u8 reserved48[6];
};
-/* Set function resources command (16 bytes) */
-struct cmdq_set_func_resources {
- u8 opcode;
- #define CMDQ_SET_FUNC_RESOURCES_OPCODE_SET_FUNC_RESOURCES 0x84UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __le32 number_of_qp;
- __le32 number_of_mrw;
- __le32 number_of_srq;
- __le32 number_of_cq;
- __le32 max_qp_per_vf;
- __le32 max_mrw_per_vf;
- __le32 max_srq_per_vf;
- __le32 max_cq_per_vf;
- __le32 max_gid_per_vf;
- __le32 stat_ctx_id;
-};
-
-/* Read hardware resource context command (24 bytes) */
-struct cmdq_read_context {
- u8 opcode;
- #define CMDQ_READ_CONTEXT_OPCODE_READ_CONTEXT 0x85UL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __le32 type_xid;
- #define CMDQ_READ_CONTEXT_XID_MASK 0xffffffUL
- #define CMDQ_READ_CONTEXT_XID_SFT 0
- #define CMDQ_READ_CONTEXT_TYPE_MASK 0xff000000UL
- #define CMDQ_READ_CONTEXT_TYPE_SFT 24
- #define CMDQ_READ_CONTEXT_TYPE_QPC (0x0UL << 24)
- #define CMDQ_READ_CONTEXT_TYPE_CQ (0x1UL << 24)
- #define CMDQ_READ_CONTEXT_TYPE_MRW (0x2UL << 24)
- #define CMDQ_READ_CONTEXT_TYPE_SRQ (0x3UL << 24)
- __le32 unused_0;
-};
-
-/* Map TC to COS. Can only be issued from a PF (24 bytes) */
-struct cmdq_map_tc_to_cos {
- u8 opcode;
- #define CMDQ_MAP_TC_TO_COS_OPCODE_MAP_TC_TO_COS 0x8aUL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
- __le16 cos0;
- #define CMDQ_MAP_TC_TO_COS_COS0_NO_CHANGE 0xffffUL
- __le16 cos1;
- #define CMDQ_MAP_TC_TO_COS_COS1_DISABLE 0x8000UL
- #define CMDQ_MAP_TC_TO_COS_COS1_NO_CHANGE 0xffffUL
- __le32 unused_0;
-};
-
-/* Query version command (16 bytes) */
-struct cmdq_query_version {
- u8 opcode;
- #define CMDQ_QUERY_VERSION_OPCODE_QUERY_VERSION 0x8bUL
- u8 cmd_size;
- __le16 flags;
- __le16 cookie;
- u8 resp_size;
- u8 reserved8;
- __le64 resp_addr;
-};
-
-/* Command-Response Event Queue (CREQ) Structures */
-/* Base CREQ Record (16 bytes) */
-struct creq_base {
- u8 type;
- #define CREQ_BASE_TYPE_MASK 0x3fUL
- #define CREQ_BASE_TYPE_SFT 0
- #define CREQ_BASE_TYPE_QP_EVENT 0x38UL
- #define CREQ_BASE_TYPE_FUNC_EVENT 0x3aUL
- #define CREQ_BASE_RESERVED2_MASK 0xc0UL
- #define CREQ_BASE_RESERVED2_SFT 6
- u8 reserved56[7];
- u8 v;
- #define CREQ_BASE_V 0x1UL
- #define CREQ_BASE_RESERVED7_MASK 0xfeUL
- #define CREQ_BASE_RESERVED7_SFT 1
- u8 event;
- __le16 reserved48[3];
-};
-
-/* RoCE Function Async Event Notification (16 bytes) */
-struct creq_func_event {
- u8 type;
- #define CREQ_FUNC_EVENT_TYPE_MASK 0x3fUL
- #define CREQ_FUNC_EVENT_TYPE_SFT 0
- #define CREQ_FUNC_EVENT_TYPE_FUNC_EVENT 0x3aUL
- #define CREQ_FUNC_EVENT_RESERVED2_MASK 0xc0UL
- #define CREQ_FUNC_EVENT_RESERVED2_SFT 6
- u8 reserved56[7];
- u8 v;
- #define CREQ_FUNC_EVENT_V 0x1UL
- #define CREQ_FUNC_EVENT_RESERVED7_MASK 0xfeUL
- #define CREQ_FUNC_EVENT_RESERVED7_SFT 1
- u8 event;
- #define CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR 0x1UL
- #define CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR 0x2UL
- #define CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR 0x3UL
- #define CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR 0x4UL
- #define CREQ_FUNC_EVENT_EVENT_CQ_ERROR 0x5UL
- #define CREQ_FUNC_EVENT_EVENT_TQM_ERROR 0x6UL
- #define CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR 0x7UL
- #define CREQ_FUNC_EVENT_EVENT_CFCS_ERROR 0x8UL
- #define CREQ_FUNC_EVENT_EVENT_CFCC_ERROR 0x9UL
- #define CREQ_FUNC_EVENT_EVENT_CFCM_ERROR 0xaUL
- #define CREQ_FUNC_EVENT_EVENT_TIM_ERROR 0xbUL
- #define CREQ_FUNC_EVENT_EVENT_VF_COMM_REQUEST 0x80UL
- #define CREQ_FUNC_EVENT_EVENT_RESOURCE_EXHAUSTED 0x81UL
- __le16 reserved48[3];
-};
-
-/* RoCE Slowpath Command Completion (16 bytes) */
-struct creq_qp_event {
- u8 type;
- #define CREQ_QP_EVENT_TYPE_MASK 0x3fUL
- #define CREQ_QP_EVENT_TYPE_SFT 0
- #define CREQ_QP_EVENT_TYPE_QP_EVENT 0x38UL
- #define CREQ_QP_EVENT_RESERVED2_MASK 0xc0UL
- #define CREQ_QP_EVENT_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 reserved32;
- u8 v;
- #define CREQ_QP_EVENT_V 0x1UL
- #define CREQ_QP_EVENT_RESERVED7_MASK 0xfeUL
- #define CREQ_QP_EVENT_RESERVED7_SFT 1
- u8 event;
- #define CREQ_QP_EVENT_EVENT_CREATE_QP 0x1UL
- #define CREQ_QP_EVENT_EVENT_DESTROY_QP 0x2UL
- #define CREQ_QP_EVENT_EVENT_MODIFY_QP 0x3UL
- #define CREQ_QP_EVENT_EVENT_QUERY_QP 0x4UL
- #define CREQ_QP_EVENT_EVENT_CREATE_SRQ 0x5UL
- #define CREQ_QP_EVENT_EVENT_DESTROY_SRQ 0x6UL
- #define CREQ_QP_EVENT_EVENT_QUERY_SRQ 0x8UL
- #define CREQ_QP_EVENT_EVENT_CREATE_CQ 0x9UL
- #define CREQ_QP_EVENT_EVENT_DESTROY_CQ 0xaUL
- #define CREQ_QP_EVENT_EVENT_RESIZE_CQ 0xcUL
- #define CREQ_QP_EVENT_EVENT_ALLOCATE_MRW 0xdUL
- #define CREQ_QP_EVENT_EVENT_DEALLOCATE_KEY 0xeUL
- #define CREQ_QP_EVENT_EVENT_REGISTER_MR 0xfUL
- #define CREQ_QP_EVENT_EVENT_DEREGISTER_MR 0x10UL
- #define CREQ_QP_EVENT_EVENT_ADD_GID 0x11UL
- #define CREQ_QP_EVENT_EVENT_DELETE_GID 0x12UL
- #define CREQ_QP_EVENT_EVENT_MODIFY_GID 0x17UL
- #define CREQ_QP_EVENT_EVENT_QUERY_GID 0x18UL
- #define CREQ_QP_EVENT_EVENT_CREATE_QP1 0x13UL
- #define CREQ_QP_EVENT_EVENT_DESTROY_QP1 0x14UL
- #define CREQ_QP_EVENT_EVENT_CREATE_AH 0x15UL
- #define CREQ_QP_EVENT_EVENT_DESTROY_AH 0x16UL
- #define CREQ_QP_EVENT_EVENT_INITIALIZE_FW 0x80UL
- #define CREQ_QP_EVENT_EVENT_DEINITIALIZE_FW 0x81UL
- #define CREQ_QP_EVENT_EVENT_STOP_FUNC 0x82UL
- #define CREQ_QP_EVENT_EVENT_QUERY_FUNC 0x83UL
- #define CREQ_QP_EVENT_EVENT_SET_FUNC_RESOURCES 0x84UL
- #define CREQ_QP_EVENT_EVENT_MAP_TC_TO_COS 0x8aUL
- #define CREQ_QP_EVENT_EVENT_QUERY_VERSION 0x8bUL
- #define CREQ_QP_EVENT_EVENT_MODIFY_CC 0x8cUL
- #define CREQ_QP_EVENT_EVENT_QUERY_CC 0x8dUL
- #define CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION 0xc0UL
- __le16 reserved48[3];
-};
-
-/* Create QP command response (16 bytes) */
-struct creq_create_qp_resp {
- u8 type;
- #define CREQ_CREATE_QP_RESP_TYPE_MASK 0x3fUL
- #define CREQ_CREATE_QP_RESP_TYPE_SFT 0
- #define CREQ_CREATE_QP_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_CREATE_QP_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_CREATE_QP_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 xid;
- u8 v;
- #define CREQ_CREATE_QP_RESP_V 0x1UL
- #define CREQ_CREATE_QP_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_CREATE_QP_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_CREATE_QP_RESP_EVENT_CREATE_QP 0x1UL
- __le16 reserved48[3];
-};
-
-/* Destroy QP command response (16 bytes) */
-struct creq_destroy_qp_resp {
- u8 type;
- #define CREQ_DESTROY_QP_RESP_TYPE_MASK 0x3fUL
- #define CREQ_DESTROY_QP_RESP_TYPE_SFT 0
- #define CREQ_DESTROY_QP_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_DESTROY_QP_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_DESTROY_QP_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 xid;
- u8 v;
- #define CREQ_DESTROY_QP_RESP_V 0x1UL
- #define CREQ_DESTROY_QP_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_DESTROY_QP_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_DESTROY_QP_RESP_EVENT_DESTROY_QP 0x2UL
- __le16 reserved48[3];
-};
-
-/* Modify QP command response (16 bytes) */
-struct creq_modify_qp_resp {
- u8 type;
- #define CREQ_MODIFY_QP_RESP_TYPE_MASK 0x3fUL
- #define CREQ_MODIFY_QP_RESP_TYPE_SFT 0
- #define CREQ_MODIFY_QP_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_MODIFY_QP_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_MODIFY_QP_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 xid;
- u8 v;
- #define CREQ_MODIFY_QP_RESP_V 0x1UL
- #define CREQ_MODIFY_QP_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_MODIFY_QP_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_MODIFY_QP_RESP_EVENT_MODIFY_QP 0x3UL
- __le16 reserved48[3];
-};
-
-/* cmdq_query_roce_stats (size:128b/16B) */
-struct cmdq_query_roce_stats {
+/* cmdq_deregister_mr (size:192b/24B) */
+struct cmdq_deregister_mr {
u8 opcode;
- #define CMDQ_QUERY_ROCE_STATS_OPCODE_QUERY_ROCE_STATS 0x8eUL
- #define CMDQ_QUERY_ROCE_STATS_OPCODE_LAST \
- CMDQ_QUERY_ROCE_STATS_OPCODE_QUERY_ROCE_STATS
+ #define CMDQ_DEREGISTER_MR_OPCODE_DEREGISTER_MR 0x10UL
+ #define CMDQ_DEREGISTER_MR_OPCODE_LAST CMDQ_DEREGISTER_MR_OPCODE_DEREGISTER_MR
u8 cmd_size;
__le16 flags;
__le16 cookie;
u8 resp_size;
u8 reserved8;
__le64 resp_addr;
+ __le32 lkey;
+ __le32 unused_0;
};
-/* Query QP command response (16 bytes) */
-struct creq_query_qp_resp {
- u8 type;
- #define CREQ_QUERY_QP_RESP_TYPE_MASK 0x3fUL
- #define CREQ_QUERY_QP_RESP_TYPE_SFT 0
- #define CREQ_QUERY_QP_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_QUERY_QP_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_QUERY_QP_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 size;
- u8 v;
- #define CREQ_QUERY_QP_RESP_V 0x1UL
- #define CREQ_QUERY_QP_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_QUERY_QP_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_QUERY_QP_RESP_EVENT_QUERY_QP 0x4UL
- __le16 reserved48[3];
-};
-
-/* Query QP command response side buffer structure (104 bytes) */
-struct creq_query_qp_resp_sb {
- u8 opcode;
- #define CREQ_QUERY_QP_RESP_SB_OPCODE_QUERY_QP 0x4UL
- u8 status;
- __le16 cookie;
- __le16 flags;
- u8 resp_size;
- u8 reserved8;
- __le32 xid;
- u8 en_sqd_async_notify_state;
- #define CREQ_QUERY_QP_RESP_SB_STATE_MASK 0xfUL
- #define CREQ_QUERY_QP_RESP_SB_STATE_SFT 0
- #define CREQ_QUERY_QP_RESP_SB_STATE_RESET 0x0UL
- #define CREQ_QUERY_QP_RESP_SB_STATE_INIT 0x1UL
- #define CREQ_QUERY_QP_RESP_SB_STATE_RTR 0x2UL
- #define CREQ_QUERY_QP_RESP_SB_STATE_RTS 0x3UL
- #define CREQ_QUERY_QP_RESP_SB_STATE_SQD 0x4UL
- #define CREQ_QUERY_QP_RESP_SB_STATE_SQE 0x5UL
- #define CREQ_QUERY_QP_RESP_SB_STATE_ERR 0x6UL
- #define CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY 0x10UL
- u8 access;
- #define CREQ_QUERY_QP_RESP_SB_ACCESS_LOCAL_WRITE 0x1UL
- #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_WRITE 0x2UL
- #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_READ 0x4UL
- #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_ATOMIC 0x8UL
- __le16 pkey;
- __le32 qkey;
- __le32 reserved32;
- __le32 dgid[4];
- __le32 flow_label;
- __le16 sgid_index;
- u8 hop_limit;
- u8 traffic_class;
- __le16 dest_mac[3];
- __le16 path_mtu_dest_vlan_id;
- #define CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_MASK 0xfffUL
- #define CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_SFT 0
- #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK 0xf000UL
- #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT 12
- #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_256 (0x0UL << 12)
- #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_512 (0x1UL << 12)
- #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_1024 (0x2UL << 12)
- #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_2048 (0x3UL << 12)
- #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_4096 (0x4UL << 12)
- #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_8192 (0x5UL << 12)
- u8 timeout;
- u8 retry_cnt;
- u8 rnr_retry;
- u8 min_rnr_timer;
- __le32 rq_psn;
- __le32 sq_psn;
- u8 max_rd_atomic;
- u8 max_dest_rd_atomic;
- u8 tos_dscp_tos_ecn;
- #define CREQ_QUERY_QP_RESP_SB_TOS_ECN_MASK 0x3UL
- #define CREQ_QUERY_QP_RESP_SB_TOS_ECN_SFT 0
- #define CREQ_QUERY_QP_RESP_SB_TOS_DSCP_MASK 0xfcUL
- #define CREQ_QUERY_QP_RESP_SB_TOS_DSCP_SFT 2
- u8 enable_cc;
- #define CREQ_QUERY_QP_RESP_SB_ENABLE_CC 0x1UL
- #define CREQ_QUERY_QP_RESP_SB_RESERVED7_MASK 0xfeUL
- #define CREQ_QUERY_QP_RESP_SB_RESERVED7_SFT 1
- __le32 sq_size;
- __le32 rq_size;
- __le16 sq_sge;
- __le16 rq_sge;
- __le32 max_inline_data;
- __le32 dest_qp_id;
- __le32 unused_1;
- __le16 src_mac[3];
- __le16 vlan_pcp_vlan_dei_vlan_id;
- #define CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK 0xfffUL
- #define CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT 0
- #define CREQ_QUERY_QP_RESP_SB_VLAN_DEI 0x1000UL
- #define CREQ_QUERY_QP_RESP_SB_VLAN_PCP_MASK 0xe000UL
- #define CREQ_QUERY_QP_RESP_SB_VLAN_PCP_SFT 13
-};
-
-/* Create SRQ command response (16 bytes) */
-struct creq_create_srq_resp {
- u8 type;
- #define CREQ_CREATE_SRQ_RESP_TYPE_MASK 0x3fUL
- #define CREQ_CREATE_SRQ_RESP_TYPE_SFT 0
- #define CREQ_CREATE_SRQ_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_CREATE_SRQ_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_CREATE_SRQ_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 xid;
- u8 v;
- #define CREQ_CREATE_SRQ_RESP_V 0x1UL
- #define CREQ_CREATE_SRQ_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_CREATE_SRQ_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_CREATE_SRQ_RESP_EVENT_CREATE_SRQ 0x5UL
- __le16 reserved48[3];
-};
-
-/* Destroy SRQ command response (16 bytes) */
-struct creq_destroy_srq_resp {
- u8 type;
- #define CREQ_DESTROY_SRQ_RESP_TYPE_MASK 0x3fUL
- #define CREQ_DESTROY_SRQ_RESP_TYPE_SFT 0
- #define CREQ_DESTROY_SRQ_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_DESTROY_SRQ_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_DESTROY_SRQ_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 xid;
- u8 v;
- #define CREQ_DESTROY_SRQ_RESP_V 0x1UL
- #define CREQ_DESTROY_SRQ_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_DESTROY_SRQ_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_DESTROY_SRQ_RESP_EVENT_DESTROY_SRQ 0x6UL
- __le16 enable_for_arm[3];
- #define CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_MASK 0x30000UL
- #define CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_SFT 16
- #define CREQ_DESTROY_SRQ_RESP_RESERVED46_MASK 0xfffc0000UL
- #define CREQ_DESTROY_SRQ_RESP_RESERVED46_SFT 18
-};
-
-/* Query SRQ command response (16 bytes) */
-struct creq_query_srq_resp {
- u8 type;
- #define CREQ_QUERY_SRQ_RESP_TYPE_MASK 0x3fUL
- #define CREQ_QUERY_SRQ_RESP_TYPE_SFT 0
- #define CREQ_QUERY_SRQ_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_QUERY_SRQ_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_QUERY_SRQ_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 size;
- u8 v;
- #define CREQ_QUERY_SRQ_RESP_V 0x1UL
- #define CREQ_QUERY_SRQ_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_QUERY_SRQ_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_QUERY_SRQ_RESP_EVENT_QUERY_SRQ 0x8UL
- __le16 reserved48[3];
-};
-
-/* Query SRQ command response side buffer structure (24 bytes) */
-struct creq_query_srq_resp_sb {
- u8 opcode;
- #define CREQ_QUERY_SRQ_RESP_SB_OPCODE_QUERY_SRQ 0x8UL
- u8 status;
- __le16 cookie;
- __le16 flags;
- u8 resp_size;
- u8 reserved8;
- __le32 xid;
- __le16 srq_limit;
- __le16 reserved16;
- __le32 data[4];
-};
-
-/* Create CQ command Response (16 bytes) */
-struct creq_create_cq_resp {
- u8 type;
- #define CREQ_CREATE_CQ_RESP_TYPE_MASK 0x3fUL
- #define CREQ_CREATE_CQ_RESP_TYPE_SFT 0
- #define CREQ_CREATE_CQ_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_CREATE_CQ_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_CREATE_CQ_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 xid;
- u8 v;
- #define CREQ_CREATE_CQ_RESP_V 0x1UL
- #define CREQ_CREATE_CQ_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_CREATE_CQ_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_CREATE_CQ_RESP_EVENT_CREATE_CQ 0x9UL
- __le16 reserved48[3];
-};
-
-/* Destroy CQ command response (16 bytes) */
-struct creq_destroy_cq_resp {
- u8 type;
- #define CREQ_DESTROY_CQ_RESP_TYPE_MASK 0x3fUL
- #define CREQ_DESTROY_CQ_RESP_TYPE_SFT 0
- #define CREQ_DESTROY_CQ_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_DESTROY_CQ_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_DESTROY_CQ_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 xid;
- u8 v;
- #define CREQ_DESTROY_CQ_RESP_V 0x1UL
- #define CREQ_DESTROY_CQ_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_DESTROY_CQ_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_DESTROY_CQ_RESP_EVENT_DESTROY_CQ 0xaUL
- __le16 cq_arm_lvl;
- #define CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_MASK 0x3UL
- #define CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_SFT 0
- #define CREQ_DESTROY_CQ_RESP_RESERVED14_MASK 0xfffcUL
- #define CREQ_DESTROY_CQ_RESP_RESERVED14_SFT 2
- __le16 total_cnq_events;
- __le16 reserved16;
-};
-
-/* Resize CQ command response (16 bytes) */
-struct creq_resize_cq_resp {
- u8 type;
- #define CREQ_RESIZE_CQ_RESP_TYPE_MASK 0x3fUL
- #define CREQ_RESIZE_CQ_RESP_TYPE_SFT 0
- #define CREQ_RESIZE_CQ_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_RESIZE_CQ_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_RESIZE_CQ_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 xid;
- u8 v;
- #define CREQ_RESIZE_CQ_RESP_V 0x1UL
- #define CREQ_RESIZE_CQ_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_RESIZE_CQ_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_RESIZE_CQ_RESP_EVENT_RESIZE_CQ 0xcUL
- __le16 reserved48[3];
-};
-
-/* Allocate MRW command response (16 bytes) */
-struct creq_allocate_mrw_resp {
- u8 type;
- #define CREQ_ALLOCATE_MRW_RESP_TYPE_MASK 0x3fUL
- #define CREQ_ALLOCATE_MRW_RESP_TYPE_SFT 0
- #define CREQ_ALLOCATE_MRW_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_ALLOCATE_MRW_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_ALLOCATE_MRW_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 xid;
- u8 v;
- #define CREQ_ALLOCATE_MRW_RESP_V 0x1UL
- #define CREQ_ALLOCATE_MRW_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_ALLOCATE_MRW_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_ALLOCATE_MRW_RESP_EVENT_ALLOCATE_MRW 0xdUL
- __le16 reserved48[3];
-};
-
-/* De-allocate key command response (16 bytes) */
-struct creq_deallocate_key_resp {
- u8 type;
- #define CREQ_DEALLOCATE_KEY_RESP_TYPE_MASK 0x3fUL
- #define CREQ_DEALLOCATE_KEY_RESP_TYPE_SFT 0
- #define CREQ_DEALLOCATE_KEY_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_DEALLOCATE_KEY_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_DEALLOCATE_KEY_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 xid;
- u8 v;
- #define CREQ_DEALLOCATE_KEY_RESP_V 0x1UL
- #define CREQ_DEALLOCATE_KEY_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_DEALLOCATE_KEY_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_DEALLOCATE_KEY_RESP_EVENT_DEALLOCATE_KEY 0xeUL
- __le16 reserved16;
- __le32 bound_window_info;
-};
-
-/* Register MR command response (16 bytes) */
-struct creq_register_mr_resp {
- u8 type;
- #define CREQ_REGISTER_MR_RESP_TYPE_MASK 0x3fUL
- #define CREQ_REGISTER_MR_RESP_TYPE_SFT 0
- #define CREQ_REGISTER_MR_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_REGISTER_MR_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_REGISTER_MR_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 xid;
- u8 v;
- #define CREQ_REGISTER_MR_RESP_V 0x1UL
- #define CREQ_REGISTER_MR_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_REGISTER_MR_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_REGISTER_MR_RESP_EVENT_REGISTER_MR 0xfUL
- __le16 reserved48[3];
-};
-
-/* Deregister MR command response (16 bytes) */
+/* creq_deregister_mr_resp (size:128b/16B) */
struct creq_deregister_mr_resp {
- u8 type;
- #define CREQ_DEREGISTER_MR_RESP_TYPE_MASK 0x3fUL
- #define CREQ_DEREGISTER_MR_RESP_TYPE_SFT 0
- #define CREQ_DEREGISTER_MR_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_DEREGISTER_MR_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_DEREGISTER_MR_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 xid;
- u8 v;
- #define CREQ_DEREGISTER_MR_RESP_V 0x1UL
- #define CREQ_DEREGISTER_MR_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_DEREGISTER_MR_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_DEREGISTER_MR_RESP_EVENT_DEREGISTER_MR 0x10UL
- __le16 reserved16;
- __le32 bound_windows;
-};
-
-/* Add GID command response (16 bytes) */
+ u8 type;
+ #define CREQ_DEREGISTER_MR_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_DEREGISTER_MR_RESP_TYPE_SFT 0
+ #define CREQ_DEREGISTER_MR_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_DEREGISTER_MR_RESP_TYPE_LAST CREQ_DEREGISTER_MR_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 xid;
+ u8 v;
+ #define CREQ_DEREGISTER_MR_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_DEREGISTER_MR_RESP_EVENT_DEREGISTER_MR 0x10UL
+ #define CREQ_DEREGISTER_MR_RESP_EVENT_LAST CREQ_DEREGISTER_MR_RESP_EVENT_DEREGISTER_MR
+ __le16 reserved16;
+ __le32 bound_windows;
+};
+
+/* cmdq_add_gid (size:384b/48B) */
+struct cmdq_add_gid {
+ u8 opcode;
+ #define CMDQ_ADD_GID_OPCODE_ADD_GID 0x11UL
+ #define CMDQ_ADD_GID_OPCODE_LAST CMDQ_ADD_GID_OPCODE_ADD_GID
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __be32 gid[4];
+ __be16 src_mac[3];
+ __le16 vlan;
+ #define CMDQ_ADD_GID_VLAN_VLAN_EN_TPID_VLAN_ID_MASK 0xffffUL
+ #define CMDQ_ADD_GID_VLAN_VLAN_EN_TPID_VLAN_ID_SFT 0
+ #define CMDQ_ADD_GID_VLAN_VLAN_ID_MASK 0xfffUL
+ #define CMDQ_ADD_GID_VLAN_VLAN_ID_SFT 0
+ #define CMDQ_ADD_GID_VLAN_TPID_MASK 0x7000UL
+ #define CMDQ_ADD_GID_VLAN_TPID_SFT 12
+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_88A8 (0x0UL << 12)
+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_8100 (0x1UL << 12)
+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_9100 (0x2UL << 12)
+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_9200 (0x3UL << 12)
+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_9300 (0x4UL << 12)
+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG1 (0x5UL << 12)
+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG2 (0x6UL << 12)
+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3 (0x7UL << 12)
+ #define CMDQ_ADD_GID_VLAN_TPID_LAST CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3
+ #define CMDQ_ADD_GID_VLAN_VLAN_EN 0x8000UL
+ __le16 ipid;
+ __le16 stats_ctx;
+ #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_VALID_STATS_CTX_ID_MASK 0xffffUL
+ #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_VALID_STATS_CTX_ID_SFT 0
+ #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_MASK 0x7fffUL
+ #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_SFT 0
+ #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_VALID 0x8000UL
+ __le32 unused_0;
+};
+
+/* creq_add_gid_resp (size:128b/16B) */
struct creq_add_gid_resp {
- u8 type;
- #define CREQ_ADD_GID_RESP_TYPE_MASK 0x3fUL
- #define CREQ_ADD_GID_RESP_TYPE_SFT 0
- #define CREQ_ADD_GID_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_ADD_GID_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_ADD_GID_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 xid;
- u8 v;
- #define CREQ_ADD_GID_RESP_V 0x1UL
- #define CREQ_ADD_GID_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_ADD_GID_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_ADD_GID_RESP_EVENT_ADD_GID 0x11UL
- __le16 reserved48[3];
-};
-
-/* Delete GID command response (16 bytes) */
+ u8 type;
+ #define CREQ_ADD_GID_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_ADD_GID_RESP_TYPE_SFT 0
+ #define CREQ_ADD_GID_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_ADD_GID_RESP_TYPE_LAST CREQ_ADD_GID_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 xid;
+ u8 v;
+ #define CREQ_ADD_GID_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_ADD_GID_RESP_EVENT_ADD_GID 0x11UL
+ #define CREQ_ADD_GID_RESP_EVENT_LAST CREQ_ADD_GID_RESP_EVENT_ADD_GID
+ u8 reserved48[6];
+};
+
+/* cmdq_delete_gid (size:192b/24B) */
+struct cmdq_delete_gid {
+ u8 opcode;
+ #define CMDQ_DELETE_GID_OPCODE_DELETE_GID 0x12UL
+ #define CMDQ_DELETE_GID_OPCODE_LAST CMDQ_DELETE_GID_OPCODE_DELETE_GID
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __le16 gid_index;
+ u8 unused_0[6];
+};
+
+/* creq_delete_gid_resp (size:128b/16B) */
struct creq_delete_gid_resp {
- u8 type;
- #define CREQ_DELETE_GID_RESP_TYPE_MASK 0x3fUL
- #define CREQ_DELETE_GID_RESP_TYPE_SFT 0
- #define CREQ_DELETE_GID_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_DELETE_GID_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_DELETE_GID_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 xid;
- u8 v;
- #define CREQ_DELETE_GID_RESP_V 0x1UL
- #define CREQ_DELETE_GID_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_DELETE_GID_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_DELETE_GID_RESP_EVENT_DELETE_GID 0x12UL
- __le16 reserved48[3];
-};
-
-/* Modify GID command response (16 bytes) */
+ u8 type;
+ #define CREQ_DELETE_GID_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_DELETE_GID_RESP_TYPE_SFT 0
+ #define CREQ_DELETE_GID_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_DELETE_GID_RESP_TYPE_LAST CREQ_DELETE_GID_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 xid;
+ u8 v;
+ #define CREQ_DELETE_GID_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_DELETE_GID_RESP_EVENT_DELETE_GID 0x12UL
+ #define CREQ_DELETE_GID_RESP_EVENT_LAST CREQ_DELETE_GID_RESP_EVENT_DELETE_GID
+ u8 reserved48[6];
+};
+
+/* cmdq_modify_gid (size:384b/48B) */
+struct cmdq_modify_gid {
+ u8 opcode;
+ #define CMDQ_MODIFY_GID_OPCODE_MODIFY_GID 0x17UL
+ #define CMDQ_MODIFY_GID_OPCODE_LAST CMDQ_MODIFY_GID_OPCODE_MODIFY_GID
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __be32 gid[4];
+ __be16 src_mac[3];
+ __le16 vlan;
+ #define CMDQ_MODIFY_GID_VLAN_VLAN_ID_MASK 0xfffUL
+ #define CMDQ_MODIFY_GID_VLAN_VLAN_ID_SFT 0
+ #define CMDQ_MODIFY_GID_VLAN_TPID_MASK 0x7000UL
+ #define CMDQ_MODIFY_GID_VLAN_TPID_SFT 12
+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_88A8 (0x0UL << 12)
+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_8100 (0x1UL << 12)
+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9100 (0x2UL << 12)
+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9200 (0x3UL << 12)
+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9300 (0x4UL << 12)
+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG1 (0x5UL << 12)
+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG2 (0x6UL << 12)
+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3 (0x7UL << 12)
+ #define CMDQ_MODIFY_GID_VLAN_TPID_LAST CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3
+ #define CMDQ_MODIFY_GID_VLAN_VLAN_EN 0x8000UL
+ __le16 ipid;
+ __le16 gid_index;
+ __le16 stats_ctx;
+ #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_MASK 0x7fffUL
+ #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_SFT 0
+ #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_VALID 0x8000UL
+ __le16 unused_0;
+};
+
+/* creq_modify_gid_resp (size:128b/16B) */
struct creq_modify_gid_resp {
- u8 type;
- #define CREQ_MODIFY_GID_RESP_TYPE_MASK 0x3fUL
- #define CREQ_MODIFY_GID_RESP_TYPE_SFT 0
- #define CREQ_MODIFY_GID_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_MODIFY_GID_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_MODIFY_GID_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 xid;
- u8 v;
- #define CREQ_MODIFY_GID_RESP_V 0x1UL
- #define CREQ_MODIFY_GID_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_MODIFY_GID_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_MODIFY_GID_RESP_EVENT_ADD_GID 0x11UL
- __le16 reserved48[3];
-};
-
-/* Query GID command response (16 bytes) */
+ u8 type;
+ #define CREQ_MODIFY_GID_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_MODIFY_GID_RESP_TYPE_SFT 0
+ #define CREQ_MODIFY_GID_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_MODIFY_GID_RESP_TYPE_LAST CREQ_MODIFY_GID_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 xid;
+ u8 v;
+ #define CREQ_MODIFY_GID_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_MODIFY_GID_RESP_EVENT_ADD_GID 0x11UL
+ #define CREQ_MODIFY_GID_RESP_EVENT_LAST CREQ_MODIFY_GID_RESP_EVENT_ADD_GID
+ u8 reserved48[6];
+};
+
+/* cmdq_query_gid (size:192b/24B) */
+struct cmdq_query_gid {
+ u8 opcode;
+ #define CMDQ_QUERY_GID_OPCODE_QUERY_GID 0x18UL
+ #define CMDQ_QUERY_GID_OPCODE_LAST CMDQ_QUERY_GID_OPCODE_QUERY_GID
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __le16 gid_index;
+ u8 unused16[6];
+};
+
+/* creq_query_gid_resp (size:128b/16B) */
struct creq_query_gid_resp {
- u8 type;
- #define CREQ_QUERY_GID_RESP_TYPE_MASK 0x3fUL
- #define CREQ_QUERY_GID_RESP_TYPE_SFT 0
- #define CREQ_QUERY_GID_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_QUERY_GID_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_QUERY_GID_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 size;
- u8 v;
- #define CREQ_QUERY_GID_RESP_V 0x1UL
- #define CREQ_QUERY_GID_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_QUERY_GID_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_QUERY_GID_RESP_EVENT_QUERY_GID 0x18UL
- __le16 reserved48[3];
-};
-
-/* Query GID command response side buffer structure (40 bytes) */
+ u8 type;
+ #define CREQ_QUERY_GID_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_QUERY_GID_RESP_TYPE_SFT 0
+ #define CREQ_QUERY_GID_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_QUERY_GID_RESP_TYPE_LAST CREQ_QUERY_GID_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 size;
+ u8 v;
+ #define CREQ_QUERY_GID_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_QUERY_GID_RESP_EVENT_QUERY_GID 0x18UL
+ #define CREQ_QUERY_GID_RESP_EVENT_LAST CREQ_QUERY_GID_RESP_EVENT_QUERY_GID
+ u8 reserved48[6];
+};
+
+/* creq_query_gid_resp_sb (size:320b/40B) */
struct creq_query_gid_resp_sb {
- u8 opcode;
- #define CREQ_QUERY_GID_RESP_SB_OPCODE_QUERY_GID 0x18UL
- u8 status;
- __le16 cookie;
- __le16 flags;
- u8 resp_size;
- u8 reserved8;
- __le32 gid[4];
- __le16 src_mac[3];
- __le16 vlan;
- #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_MASK 0xfffUL
- #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_SFT 0
- #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_MASK 0x7000UL
- #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_SFT 12
- #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_88A8 (0x0UL << 12)
- #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_8100 (0x1UL << 12)
- #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9100 (0x2UL << 12)
- #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9200 (0x3UL << 12)
- #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9300 (0x4UL << 12)
- #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG1 (0x5UL << 12)
- #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG2 (0x6UL << 12)
- #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3 (0x7UL << 12)
- #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_LAST \
- CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3
- #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_EN 0x8000UL
- __le16 ipid;
- __le16 gid_index;
- __le32 unused_0;
-};
-
-/* Create QP1 command response (16 bytes) */
+ u8 opcode;
+ #define CREQ_QUERY_GID_RESP_SB_OPCODE_QUERY_GID 0x18UL
+ #define CREQ_QUERY_GID_RESP_SB_OPCODE_LAST CREQ_QUERY_GID_RESP_SB_OPCODE_QUERY_GID
+ u8 status;
+ __le16 cookie;
+ __le16 flags;
+ u8 resp_size;
+ u8 reserved8;
+ __le32 gid[4];
+ __le16 src_mac[3];
+ __le16 vlan;
+ #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_EN_TPID_VLAN_ID_MASK 0xffffUL
+ #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_EN_TPID_VLAN_ID_SFT 0
+ #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_MASK 0xfffUL
+ #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_SFT 0
+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_MASK 0x7000UL
+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_SFT 12
+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_88A8 (0x0UL << 12)
+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_8100 (0x1UL << 12)
+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9100 (0x2UL << 12)
+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9200 (0x3UL << 12)
+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9300 (0x4UL << 12)
+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG1 (0x5UL << 12)
+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG2 (0x6UL << 12)
+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3 (0x7UL << 12)
+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_LAST CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3
+ #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_EN 0x8000UL
+ __le16 ipid;
+ __le16 gid_index;
+ __le32 unused_0;
+};
+
+/* cmdq_create_qp1 (size:640b/80B) */
+struct cmdq_create_qp1 {
+ u8 opcode;
+ #define CMDQ_CREATE_QP1_OPCODE_CREATE_QP1 0x13UL
+ #define CMDQ_CREATE_QP1_OPCODE_LAST CMDQ_CREATE_QP1_OPCODE_CREATE_QP1
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __le64 qp_handle;
+ __le32 qp_flags;
+ #define CMDQ_CREATE_QP1_QP_FLAGS_SRQ_USED 0x1UL
+ #define CMDQ_CREATE_QP1_QP_FLAGS_FORCE_COMPLETION 0x2UL
+ #define CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE 0x4UL
+ #define CMDQ_CREATE_QP1_QP_FLAGS_LAST CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE
+ u8 type;
+ #define CMDQ_CREATE_QP1_TYPE_GSI 0x1UL
+ #define CMDQ_CREATE_QP1_TYPE_LAST CMDQ_CREATE_QP1_TYPE_GSI
+ u8 sq_pg_size_sq_lvl;
+ #define CMDQ_CREATE_QP1_SQ_LVL_MASK 0xfUL
+ #define CMDQ_CREATE_QP1_SQ_LVL_SFT 0
+ #define CMDQ_CREATE_QP1_SQ_LVL_LVL_0 0x0UL
+ #define CMDQ_CREATE_QP1_SQ_LVL_LVL_1 0x1UL
+ #define CMDQ_CREATE_QP1_SQ_LVL_LVL_2 0x2UL
+ #define CMDQ_CREATE_QP1_SQ_LVL_LAST CMDQ_CREATE_QP1_SQ_LVL_LVL_2
+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_MASK 0xf0UL
+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT 4
+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K (0x0UL << 4)
+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K (0x1UL << 4)
+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K (0x2UL << 4)
+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M (0x3UL << 4)
+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M (0x4UL << 4)
+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G (0x5UL << 4)
+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_LAST CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G
+ u8 rq_pg_size_rq_lvl;
+ #define CMDQ_CREATE_QP1_RQ_LVL_MASK 0xfUL
+ #define CMDQ_CREATE_QP1_RQ_LVL_SFT 0
+ #define CMDQ_CREATE_QP1_RQ_LVL_LVL_0 0x0UL
+ #define CMDQ_CREATE_QP1_RQ_LVL_LVL_1 0x1UL
+ #define CMDQ_CREATE_QP1_RQ_LVL_LVL_2 0x2UL
+ #define CMDQ_CREATE_QP1_RQ_LVL_LAST CMDQ_CREATE_QP1_RQ_LVL_LVL_2
+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_MASK 0xf0UL
+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT 4
+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K (0x0UL << 4)
+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K (0x1UL << 4)
+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K (0x2UL << 4)
+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M (0x3UL << 4)
+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M (0x4UL << 4)
+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G (0x5UL << 4)
+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_LAST CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G
+ u8 unused_0;
+ __le32 dpi;
+ __le32 sq_size;
+ __le32 rq_size;
+ __le16 sq_fwo_sq_sge;
+ #define CMDQ_CREATE_QP1_SQ_SGE_MASK 0xfUL
+ #define CMDQ_CREATE_QP1_SQ_SGE_SFT 0
+ #define CMDQ_CREATE_QP1_SQ_FWO_MASK 0xfff0UL
+ #define CMDQ_CREATE_QP1_SQ_FWO_SFT 4
+ __le16 rq_fwo_rq_sge;
+ #define CMDQ_CREATE_QP1_RQ_SGE_MASK 0xfUL
+ #define CMDQ_CREATE_QP1_RQ_SGE_SFT 0
+ #define CMDQ_CREATE_QP1_RQ_FWO_MASK 0xfff0UL
+ #define CMDQ_CREATE_QP1_RQ_FWO_SFT 4
+ __le32 scq_cid;
+ __le32 rcq_cid;
+ __le32 srq_cid;
+ __le32 pd_id;
+ __le64 sq_pbl;
+ __le64 rq_pbl;
+};
+
+/* creq_create_qp1_resp (size:128b/16B) */
struct creq_create_qp1_resp {
- u8 type;
- #define CREQ_CREATE_QP1_RESP_TYPE_MASK 0x3fUL
- #define CREQ_CREATE_QP1_RESP_TYPE_SFT 0
- #define CREQ_CREATE_QP1_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_CREATE_QP1_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_CREATE_QP1_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 xid;
- u8 v;
- #define CREQ_CREATE_QP1_RESP_V 0x1UL
- #define CREQ_CREATE_QP1_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_CREATE_QP1_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_CREATE_QP1_RESP_EVENT_CREATE_QP1 0x13UL
- __le16 reserved48[3];
-};
-
-/* Destroy QP1 command response (16 bytes) */
+ u8 type;
+ #define CREQ_CREATE_QP1_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_CREATE_QP1_RESP_TYPE_SFT 0
+ #define CREQ_CREATE_QP1_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_CREATE_QP1_RESP_TYPE_LAST CREQ_CREATE_QP1_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 xid;
+ u8 v;
+ #define CREQ_CREATE_QP1_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_CREATE_QP1_RESP_EVENT_CREATE_QP1 0x13UL
+ #define CREQ_CREATE_QP1_RESP_EVENT_LAST CREQ_CREATE_QP1_RESP_EVENT_CREATE_QP1
+ u8 reserved48[6];
+};
+
+/* cmdq_destroy_qp1 (size:192b/24B) */
+struct cmdq_destroy_qp1 {
+ u8 opcode;
+ #define CMDQ_DESTROY_QP1_OPCODE_DESTROY_QP1 0x14UL
+ #define CMDQ_DESTROY_QP1_OPCODE_LAST CMDQ_DESTROY_QP1_OPCODE_DESTROY_QP1
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __le32 qp1_cid;
+ __le32 unused_0;
+};
+
+/* creq_destroy_qp1_resp (size:128b/16B) */
struct creq_destroy_qp1_resp {
- u8 type;
- #define CREQ_DESTROY_QP1_RESP_TYPE_MASK 0x3fUL
- #define CREQ_DESTROY_QP1_RESP_TYPE_SFT 0
- #define CREQ_DESTROY_QP1_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_DESTROY_QP1_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_DESTROY_QP1_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 xid;
- u8 v;
- #define CREQ_DESTROY_QP1_RESP_V 0x1UL
- #define CREQ_DESTROY_QP1_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_DESTROY_QP1_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_DESTROY_QP1_RESP_EVENT_DESTROY_QP1 0x14UL
- __le16 reserved48[3];
-};
-
-/* Create AH command response (16 bytes) */
+ u8 type;
+ #define CREQ_DESTROY_QP1_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_DESTROY_QP1_RESP_TYPE_SFT 0
+ #define CREQ_DESTROY_QP1_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_DESTROY_QP1_RESP_TYPE_LAST CREQ_DESTROY_QP1_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 xid;
+ u8 v;
+ #define CREQ_DESTROY_QP1_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_DESTROY_QP1_RESP_EVENT_DESTROY_QP1 0x14UL
+ #define CREQ_DESTROY_QP1_RESP_EVENT_LAST CREQ_DESTROY_QP1_RESP_EVENT_DESTROY_QP1
+ u8 reserved48[6];
+};
+
+/* cmdq_create_ah (size:512b/64B) */
+struct cmdq_create_ah {
+ u8 opcode;
+ #define CMDQ_CREATE_AH_OPCODE_CREATE_AH 0x15UL
+ #define CMDQ_CREATE_AH_OPCODE_LAST CMDQ_CREATE_AH_OPCODE_CREATE_AH
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __le64 ah_handle;
+ __le32 dgid[4];
+ u8 type;
+ #define CMDQ_CREATE_AH_TYPE_V1 0x0UL
+ #define CMDQ_CREATE_AH_TYPE_V2IPV4 0x2UL
+ #define CMDQ_CREATE_AH_TYPE_V2IPV6 0x3UL
+ #define CMDQ_CREATE_AH_TYPE_LAST CMDQ_CREATE_AH_TYPE_V2IPV6
+ u8 hop_limit;
+ __le16 sgid_index;
+ __le32 dest_vlan_id_flow_label;
+ #define CMDQ_CREATE_AH_FLOW_LABEL_MASK 0xfffffUL
+ #define CMDQ_CREATE_AH_FLOW_LABEL_SFT 0
+ #define CMDQ_CREATE_AH_DEST_VLAN_ID_MASK 0xfff00000UL
+ #define CMDQ_CREATE_AH_DEST_VLAN_ID_SFT 20
+ __le32 pd_id;
+ __le32 unused_0;
+ __le16 dest_mac[3];
+ u8 traffic_class;
+ u8 enable_cc;
+ #define CMDQ_CREATE_AH_ENABLE_CC 0x1UL
+};
+
+/* creq_create_ah_resp (size:128b/16B) */
struct creq_create_ah_resp {
- u8 type;
- #define CREQ_CREATE_AH_RESP_TYPE_MASK 0x3fUL
- #define CREQ_CREATE_AH_RESP_TYPE_SFT 0
- #define CREQ_CREATE_AH_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_CREATE_AH_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_CREATE_AH_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 xid;
- u8 v;
- #define CREQ_CREATE_AH_RESP_V 0x1UL
- #define CREQ_CREATE_AH_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_CREATE_AH_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_CREATE_AH_RESP_EVENT_CREATE_AH 0x15UL
- __le16 reserved48[3];
-};
-
-/* Destroy AH command response (16 bytes) */
+ u8 type;
+ #define CREQ_CREATE_AH_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_CREATE_AH_RESP_TYPE_SFT 0
+ #define CREQ_CREATE_AH_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_CREATE_AH_RESP_TYPE_LAST CREQ_CREATE_AH_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 xid;
+ u8 v;
+ #define CREQ_CREATE_AH_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_CREATE_AH_RESP_EVENT_CREATE_AH 0x15UL
+ #define CREQ_CREATE_AH_RESP_EVENT_LAST CREQ_CREATE_AH_RESP_EVENT_CREATE_AH
+ u8 reserved48[6];
+};
+
+/* cmdq_destroy_ah (size:192b/24B) */
+struct cmdq_destroy_ah {
+ u8 opcode;
+ #define CMDQ_DESTROY_AH_OPCODE_DESTROY_AH 0x16UL
+ #define CMDQ_DESTROY_AH_OPCODE_LAST CMDQ_DESTROY_AH_OPCODE_DESTROY_AH
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __le32 ah_cid;
+ __le32 unused_0;
+};
+
+/* creq_destroy_ah_resp (size:128b/16B) */
struct creq_destroy_ah_resp {
- u8 type;
- #define CREQ_DESTROY_AH_RESP_TYPE_MASK 0x3fUL
- #define CREQ_DESTROY_AH_RESP_TYPE_SFT 0
- #define CREQ_DESTROY_AH_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_DESTROY_AH_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_DESTROY_AH_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 xid;
- u8 v;
- #define CREQ_DESTROY_AH_RESP_V 0x1UL
- #define CREQ_DESTROY_AH_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_DESTROY_AH_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_DESTROY_AH_RESP_EVENT_DESTROY_AH 0x16UL
- __le16 reserved48[3];
-};
-
-/* Initialize Firmware command response (16 bytes) */
-struct creq_initialize_fw_resp {
- u8 type;
- #define CREQ_INITIALIZE_FW_RESP_TYPE_MASK 0x3fUL
- #define CREQ_INITIALIZE_FW_RESP_TYPE_SFT 0
- #define CREQ_INITIALIZE_FW_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_INITIALIZE_FW_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_INITIALIZE_FW_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 reserved32;
- u8 v;
- #define CREQ_INITIALIZE_FW_RESP_V 0x1UL
- #define CREQ_INITIALIZE_FW_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_INITIALIZE_FW_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_INITIALIZE_FW_RESP_EVENT_INITIALIZE_FW 0x80UL
- __le16 reserved48[3];
-};
-
-/* De-initialize Firmware command response (16 bytes) */
-struct creq_deinitialize_fw_resp {
- u8 type;
- #define CREQ_DEINITIALIZE_FW_RESP_TYPE_MASK 0x3fUL
- #define CREQ_DEINITIALIZE_FW_RESP_TYPE_SFT 0
- #define CREQ_DEINITIALIZE_FW_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_DEINITIALIZE_FW_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_DEINITIALIZE_FW_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 reserved32;
- u8 v;
- #define CREQ_DEINITIALIZE_FW_RESP_V 0x1UL
- #define CREQ_DEINITIALIZE_FW_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_DEINITIALIZE_FW_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_DEINITIALIZE_FW_RESP_EVENT_DEINITIALIZE_FW 0x81UL
- __le16 reserved48[3];
-};
-
-/* Stop function command response (16 bytes) */
-struct creq_stop_func_resp {
- u8 type;
- #define CREQ_STOP_FUNC_RESP_TYPE_MASK 0x3fUL
- #define CREQ_STOP_FUNC_RESP_TYPE_SFT 0
- #define CREQ_STOP_FUNC_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_STOP_FUNC_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_STOP_FUNC_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 reserved32;
- u8 v;
- #define CREQ_STOP_FUNC_RESP_V 0x1UL
- #define CREQ_STOP_FUNC_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_STOP_FUNC_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_STOP_FUNC_RESP_EVENT_STOP_FUNC 0x82UL
- __le16 reserved48[3];
-};
-
-/* Query function command response (16 bytes) */
-struct creq_query_func_resp {
- u8 type;
- #define CREQ_QUERY_FUNC_RESP_TYPE_MASK 0x3fUL
- #define CREQ_QUERY_FUNC_RESP_TYPE_SFT 0
- #define CREQ_QUERY_FUNC_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_QUERY_FUNC_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_QUERY_FUNC_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 size;
- u8 v;
- #define CREQ_QUERY_FUNC_RESP_V 0x1UL
- #define CREQ_QUERY_FUNC_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_QUERY_FUNC_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_QUERY_FUNC_RESP_EVENT_QUERY_FUNC 0x83UL
- __le16 reserved48[3];
-};
-
-/* Query function command response side buffer structure (88 bytes) */
-struct creq_query_func_resp_sb {
- u8 opcode;
- #define CREQ_QUERY_FUNC_RESP_SB_OPCODE_QUERY_FUNC 0x83UL
- u8 status;
- __le16 cookie;
- __le16 flags;
- u8 resp_size;
- u8 reserved8;
- __le64 max_mr_size;
- __le32 max_qp;
- __le16 max_qp_wr;
- __le16 dev_cap_flags;
- #define CREQ_QUERY_FUNC_RESP_SB_DEV_CAP_FLAGS_RESIZE_QP 0x1UL
- #define CREQ_QUERY_FUNC_RESP_SB_EXT_STATS 0x10UL
- __le32 max_cq;
- __le32 max_cqe;
- __le32 max_pd;
- u8 max_sge;
- u8 max_srq_sge;
- u8 max_qp_rd_atom;
- u8 max_qp_init_rd_atom;
- __le32 max_mr;
- __le32 max_mw;
- __le32 max_raw_eth_qp;
- __le32 max_ah;
- __le32 max_fmr;
- __le32 max_srq_wr;
- __le32 max_pkeys;
- __le32 max_inline_data;
- u8 max_map_per_fmr;
- u8 l2_db_space_size;
- __le16 max_srq;
- __le32 max_gid;
- __le32 tqm_alloc_reqs[12];
- __le32 max_dpi;
- __le32 reserved_32;
-};
-
-/* Set resources command response (16 bytes) */
-struct creq_set_func_resources_resp {
- u8 type;
- #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_MASK 0x3fUL
- #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_SFT 0
- #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 reserved32;
- u8 v;
- #define CREQ_SET_FUNC_RESOURCES_RESP_V 0x1UL
- #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_SET_FUNC_RESOURCES_RESP_EVENT_SET_FUNC_RESOURCES 0x84UL
- __le16 reserved48[3];
+ u8 type;
+ #define CREQ_DESTROY_AH_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_DESTROY_AH_RESP_TYPE_SFT 0
+ #define CREQ_DESTROY_AH_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_DESTROY_AH_RESP_TYPE_LAST CREQ_DESTROY_AH_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 xid;
+ u8 v;
+ #define CREQ_DESTROY_AH_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_DESTROY_AH_RESP_EVENT_DESTROY_AH 0x16UL
+ #define CREQ_DESTROY_AH_RESP_EVENT_LAST CREQ_DESTROY_AH_RESP_EVENT_DESTROY_AH
+ u8 reserved48[6];
};
-/* Map TC to COS response (16 bytes) */
-struct creq_map_tc_to_cos_resp {
- u8 type;
- #define CREQ_MAP_TC_TO_COS_RESP_TYPE_MASK 0x3fUL
- #define CREQ_MAP_TC_TO_COS_RESP_TYPE_SFT 0
- #define CREQ_MAP_TC_TO_COS_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_MAP_TC_TO_COS_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_MAP_TC_TO_COS_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 reserved32;
- u8 v;
- #define CREQ_MAP_TC_TO_COS_RESP_V 0x1UL
- #define CREQ_MAP_TC_TO_COS_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_MAP_TC_TO_COS_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_MAP_TC_TO_COS_RESP_EVENT_MAP_TC_TO_COS 0x8aUL
- __le16 reserved48[3];
-};
-
-/* Query version response (16 bytes) */
-struct creq_query_version_resp {
- u8 type;
- #define CREQ_QUERY_VERSION_RESP_TYPE_MASK 0x3fUL
- #define CREQ_QUERY_VERSION_RESP_TYPE_SFT 0
- #define CREQ_QUERY_VERSION_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_QUERY_VERSION_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_QUERY_VERSION_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- u8 fw_maj;
- u8 fw_minor;
- u8 fw_bld;
- u8 fw_rsvd;
- u8 v;
- #define CREQ_QUERY_VERSION_RESP_V 0x1UL
- #define CREQ_QUERY_VERSION_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_QUERY_VERSION_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_QUERY_VERSION_RESP_EVENT_QUERY_VERSION 0x8bUL
- __le16 reserved16;
- u8 intf_maj;
- u8 intf_minor;
- u8 intf_bld;
- u8 intf_rsvd;
-};
-
-/* Modify congestion control command response (16 bytes) */
-struct creq_modify_cc_resp {
- u8 type;
- #define CREQ_MODIFY_CC_RESP_TYPE_MASK 0x3fUL
- #define CREQ_MODIFY_CC_RESP_TYPE_SFT 0
- #define CREQ_MODIFY_CC_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_MODIFY_CC_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_MODIFY_CC_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 reserved32;
- u8 v;
- #define CREQ_MODIFY_CC_RESP_V 0x1UL
- #define CREQ_MODIFY_CC_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_MODIFY_CC_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_MODIFY_CC_RESP_EVENT_MODIFY_CC 0x8cUL
- __le16 reserved48[3];
-};
-
-/* Query congestion control command response (16 bytes) */
-struct creq_query_cc_resp {
- u8 type;
- #define CREQ_QUERY_CC_RESP_TYPE_MASK 0x3fUL
- #define CREQ_QUERY_CC_RESP_TYPE_SFT 0
- #define CREQ_QUERY_CC_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_QUERY_CC_RESP_RESERVED2_MASK 0xc0UL
- #define CREQ_QUERY_CC_RESP_RESERVED2_SFT 6
- u8 status;
- __le16 cookie;
- __le32 size;
- u8 v;
- #define CREQ_QUERY_CC_RESP_V 0x1UL
- #define CREQ_QUERY_CC_RESP_RESERVED7_MASK 0xfeUL
- #define CREQ_QUERY_CC_RESP_RESERVED7_SFT 1
- u8 event;
- #define CREQ_QUERY_CC_RESP_EVENT_QUERY_CC 0x8dUL
- __le16 reserved48[3];
-};
-
-/* Query congestion control command response side buffer structure (32 bytes) */
-struct creq_query_cc_resp_sb {
- u8 opcode;
- #define CREQ_QUERY_CC_RESP_SB_OPCODE_QUERY_CC 0x8dUL
- u8 status;
- __le16 cookie;
- __le16 flags;
- u8 resp_size;
- u8 reserved8;
- u8 enable_cc;
- #define CREQ_QUERY_CC_RESP_SB_ENABLE_CC 0x1UL
- u8 g;
- #define CREQ_QUERY_CC_RESP_SB_G_MASK 0x7UL
- #define CREQ_QUERY_CC_RESP_SB_G_SFT 0
- u8 num_phases_per_state;
- __le16 init_cr;
- u8 unused_2;
- __le16 unused_3;
- u8 unused_4;
- __le16 init_tr;
- u8 tos_dscp_tos_ecn;
- #define CREQ_QUERY_CC_RESP_SB_TOS_ECN_MASK 0x3UL
- #define CREQ_QUERY_CC_RESP_SB_TOS_ECN_SFT 0
- #define CREQ_QUERY_CC_RESP_SB_TOS_DSCP_MASK 0xfcUL
- #define CREQ_QUERY_CC_RESP_SB_TOS_DSCP_SFT 2
- __le64 reserved64;
- __le64 reserved64_1;
+/* cmdq_query_roce_stats (size:192b/24B) */
+struct cmdq_query_roce_stats {
+ u8 opcode;
+ #define CMDQ_QUERY_ROCE_STATS_OPCODE_QUERY_ROCE_STATS 0x8eUL
+ #define CMDQ_QUERY_ROCE_STATS_OPCODE_LAST CMDQ_QUERY_ROCE_STATS_OPCODE_QUERY_ROCE_STATS
+ u8 cmd_size;
+ __le16 flags;
+ #define CMDQ_QUERY_ROCE_STATS_FLAGS_COLLECTION_ID 0x1UL
+ #define CMDQ_QUERY_ROCE_STATS_FLAGS_FUNCTION_ID 0x2UL
+ __le16 cookie;
+ u8 resp_size;
+ u8 collection_id;
+ __le64 resp_addr;
+ __le32 function_id;
+ #define CMDQ_QUERY_ROCE_STATS_PF_NUM_MASK 0xffUL
+ #define CMDQ_QUERY_ROCE_STATS_PF_NUM_SFT 0
+ #define CMDQ_QUERY_ROCE_STATS_VF_NUM_MASK 0xffff00UL
+ #define CMDQ_QUERY_ROCE_STATS_VF_NUM_SFT 8
+ #define CMDQ_QUERY_ROCE_STATS_VF_VALID 0x1000000UL
+ __le32 reserved32;
};
/* creq_query_roce_stats_resp (size:128b/16B) */
@@ -3020,8 +1909,7 @@ struct creq_query_roce_stats_resp {
#define CREQ_QUERY_ROCE_STATS_RESP_TYPE_MASK 0x3fUL
#define CREQ_QUERY_ROCE_STATS_RESP_TYPE_SFT 0
#define CREQ_QUERY_ROCE_STATS_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_QUERY_ROCE_STATS_RESP_TYPE_LAST \
- CREQ_QUERY_ROCE_STATS_RESP_TYPE_QP_EVENT
+ #define CREQ_QUERY_ROCE_STATS_RESP_TYPE_LAST CREQ_QUERY_ROCE_STATS_RESP_TYPE_QP_EVENT
u8 status;
__le16 cookie;
__le32 size;
@@ -3029,17 +1917,17 @@ struct creq_query_roce_stats_resp {
#define CREQ_QUERY_ROCE_STATS_RESP_V 0x1UL
u8 event;
#define CREQ_QUERY_ROCE_STATS_RESP_EVENT_QUERY_ROCE_STATS 0x8eUL
- #define CREQ_QUERY_ROCE_STATS_RESP_EVENT_LAST \
- CREQ_QUERY_ROCE_STATS_RESP_EVENT_QUERY_ROCE_STATS
+ #define CREQ_QUERY_ROCE_STATS_RESP_EVENT_LAST \
+ CREQ_QUERY_ROCE_STATS_RESP_EVENT_QUERY_ROCE_STATS
u8 reserved48[6];
};
-/* creq_query_roce_stats_resp_sb (size:2624b/328B) */
+/* creq_query_roce_stats_resp_sb (size:2944b/368B) */
struct creq_query_roce_stats_resp_sb {
u8 opcode;
#define CREQ_QUERY_ROCE_STATS_RESP_SB_OPCODE_QUERY_ROCE_STATS 0x8eUL
#define CREQ_QUERY_ROCE_STATS_RESP_SB_OPCODE_LAST \
- CREQ_QUERY_ROCE_STATS_RESP_SB_OPCODE_QUERY_ROCE_STATS
+ CREQ_QUERY_ROCE_STATS_RESP_SB_OPCODE_QUERY_ROCE_STATS
u8 status;
__le16 cookie;
__le16 flags;
@@ -3086,125 +1974,2508 @@ struct creq_query_roce_stats_resp_sb {
__le64 res_srq_load_err;
__le64 res_tx_pci_err;
__le64 res_rx_pci_err;
- __le64 res_oos_drop_count;
- __le64 active_qp_count_p0;
- __le64 active_qp_count_p1;
- __le64 active_qp_count_p2;
- __le64 active_qp_count_p3;
+ __le64 res_oos_drop_count;
+ __le64 active_qp_count_p0;
+ __le64 active_qp_count_p1;
+ __le64 active_qp_count_p2;
+ __le64 active_qp_count_p3;
};
/* cmdq_query_roce_stats_ext (size:192b/24B) */
struct cmdq_query_roce_stats_ext {
- u8 opcode;
+ u8 opcode;
#define CMDQ_QUERY_ROCE_STATS_EXT_OPCODE_QUERY_ROCE_STATS 0x92UL
- #define CMDQ_QUERY_ROCE_STATS_EXT_OPCODE_LAST \
- CMDQ_QUERY_ROCE_STATS_EXT_OPCODE_QUERY_ROCE_STATS
- u8 cmd_size;
- __le16 flags;
+ #define CMDQ_QUERY_ROCE_STATS_EXT_OPCODE_LAST \
+ CMDQ_QUERY_ROCE_STATS_EXT_OPCODE_QUERY_ROCE_STATS
+ u8 cmd_size;
+ __le16 flags;
#define CMDQ_QUERY_ROCE_STATS_EXT_FLAGS_COLLECTION_ID 0x1UL
#define CMDQ_QUERY_ROCE_STATS_EXT_FLAGS_FUNCTION_ID 0x2UL
- __le16 cookie;
- u8 resp_size;
- u8 collection_id;
- __le64 resp_addr;
- __le32 function_id;
+ __le16 cookie;
+ u8 resp_size;
+ u8 collection_id;
+ __le64 resp_addr;
+ __le32 function_id;
#define CMDQ_QUERY_ROCE_STATS_EXT_PF_NUM_MASK 0xffUL
#define CMDQ_QUERY_ROCE_STATS_EXT_PF_NUM_SFT 0
#define CMDQ_QUERY_ROCE_STATS_EXT_VF_NUM_MASK 0xffff00UL
#define CMDQ_QUERY_ROCE_STATS_EXT_VF_NUM_SFT 8
#define CMDQ_QUERY_ROCE_STATS_EXT_VF_VALID 0x1000000UL
- __le32 reserved32;
+ __le32 reserved32;
};
/* creq_query_roce_stats_ext_resp (size:128b/16B) */
struct creq_query_roce_stats_ext_resp {
- u8 type;
+ u8 type;
#define CREQ_QUERY_ROCE_STATS_EXT_RESP_TYPE_MASK 0x3fUL
#define CREQ_QUERY_ROCE_STATS_EXT_RESP_TYPE_SFT 0
#define CREQ_QUERY_ROCE_STATS_EXT_RESP_TYPE_QP_EVENT 0x38UL
- #define CREQ_QUERY_ROCE_STATS_EXT_RESP_TYPE_LAST \
+ #define CREQ_QUERY_ROCE_STATS_EXT_RESP_TYPE_LAST \
CREQ_QUERY_ROCE_STATS_EXT_RESP_TYPE_QP_EVENT
- u8 status;
- __le16 cookie;
- __le32 size;
- u8 v;
+ u8 status;
+ __le16 cookie;
+ __le32 size;
+ u8 v;
#define CREQ_QUERY_ROCE_STATS_EXT_RESP_V 0x1UL
- u8 event;
+ u8 event;
#define CREQ_QUERY_ROCE_STATS_EXT_RESP_EVENT_QUERY_ROCE_STATS_EXT 0x92UL
#define CREQ_QUERY_ROCE_STATS_EXT_RESP_EVENT_LAST \
CREQ_QUERY_ROCE_STATS_EXT_RESP_EVENT_QUERY_ROCE_STATS_EXT
- u8 reserved48[6];
+ u8 reserved48[6];
};
-/* creq_query_roce_stats_ext_resp_sb (size:1536b/192B) */
+/* creq_query_roce_stats_ext_resp_sb (size:1856b/232B) */
struct creq_query_roce_stats_ext_resp_sb {
- u8 opcode;
+ u8 opcode;
#define CREQ_QUERY_ROCE_STATS_EXT_RESP_SB_OPCODE_QUERY_ROCE_STATS_EXT 0x92UL
#define CREQ_QUERY_ROCE_STATS_EXT_RESP_SB_OPCODE_LAST \
CREQ_QUERY_ROCE_STATS_EXT_RESP_SB_OPCODE_QUERY_ROCE_STATS_EXT
- u8 status;
- __le16 cookie;
- __le16 flags;
- u8 resp_size;
- u8 rsvd;
- __le64 tx_atomic_req_pkts;
- __le64 tx_read_req_pkts;
- __le64 tx_read_res_pkts;
- __le64 tx_write_req_pkts;
- __le64 tx_send_req_pkts;
- __le64 tx_roce_pkts;
- __le64 tx_roce_bytes;
- __le64 rx_atomic_req_pkts;
- __le64 rx_read_req_pkts;
- __le64 rx_read_res_pkts;
- __le64 rx_write_req_pkts;
- __le64 rx_send_req_pkts;
- __le64 rx_roce_pkts;
- __le64 rx_roce_bytes;
- __le64 rx_roce_good_pkts;
- __le64 rx_roce_good_bytes;
- __le64 rx_out_of_buffer_pkts;
- __le64 rx_out_of_sequence_pkts;
- __le64 tx_cnp_pkts;
- __le64 rx_cnp_pkts;
- __le64 rx_ecn_marked_pkts;
- __le64 tx_cnp_bytes;
- __le64 rx_cnp_bytes;
-};
-
-/* QP error notification event (16 bytes) */
+ u8 status;
+ __le16 cookie;
+ __le16 flags;
+ u8 resp_size;
+ u8 rsvd;
+ __le64 tx_atomic_req_pkts;
+ __le64 tx_read_req_pkts;
+ __le64 tx_read_res_pkts;
+ __le64 tx_write_req_pkts;
+ __le64 tx_send_req_pkts;
+ __le64 tx_roce_pkts;
+ __le64 tx_roce_bytes;
+ __le64 rx_atomic_req_pkts;
+ __le64 rx_read_req_pkts;
+ __le64 rx_read_res_pkts;
+ __le64 rx_write_req_pkts;
+ __le64 rx_send_req_pkts;
+ __le64 rx_roce_pkts;
+ __le64 rx_roce_bytes;
+ __le64 rx_roce_good_pkts;
+ __le64 rx_roce_good_bytes;
+ __le64 rx_out_of_buffer_pkts;
+ __le64 rx_out_of_sequence_pkts;
+ __le64 tx_cnp_pkts;
+ __le64 rx_cnp_pkts;
+ __le64 rx_ecn_marked_pkts;
+ __le64 tx_cnp_bytes;
+ __le64 rx_cnp_bytes;
+ __le64 seq_err_naks_rcvd;
+ __le64 rnr_naks_rcvd;
+ __le64 missing_resp;
+ __le64 to_retransmit;
+ __le64 dup_req;
+};
+
+/* cmdq_query_func (size:128b/16B) */
+struct cmdq_query_func {
+ u8 opcode;
+ #define CMDQ_QUERY_FUNC_OPCODE_QUERY_FUNC 0x83UL
+ #define CMDQ_QUERY_FUNC_OPCODE_LAST CMDQ_QUERY_FUNC_OPCODE_QUERY_FUNC
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+};
+
+/* creq_query_func_resp (size:128b/16B) */
+struct creq_query_func_resp {
+ u8 type;
+ #define CREQ_QUERY_FUNC_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_QUERY_FUNC_RESP_TYPE_SFT 0
+ #define CREQ_QUERY_FUNC_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_QUERY_FUNC_RESP_TYPE_LAST CREQ_QUERY_FUNC_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 size;
+ u8 v;
+ #define CREQ_QUERY_FUNC_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_QUERY_FUNC_RESP_EVENT_QUERY_FUNC 0x83UL
+ #define CREQ_QUERY_FUNC_RESP_EVENT_LAST CREQ_QUERY_FUNC_RESP_EVENT_QUERY_FUNC
+ u8 reserved48[6];
+};
+
+/* creq_query_func_resp_sb (size:1088b/136B) */
+struct creq_query_func_resp_sb {
+ u8 opcode;
+ #define CREQ_QUERY_FUNC_RESP_SB_OPCODE_QUERY_FUNC 0x83UL
+ #define CREQ_QUERY_FUNC_RESP_SB_OPCODE_LAST CREQ_QUERY_FUNC_RESP_SB_OPCODE_QUERY_FUNC
+ u8 status;
+ __le16 cookie;
+ __le16 flags;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 max_mr_size;
+ __le32 max_qp;
+ __le16 max_qp_wr;
+ __le16 dev_cap_flags;
+ #define CREQ_QUERY_FUNC_RESP_SB_RESIZE_QP 0x1UL
+ #define CREQ_QUERY_FUNC_RESP_SB_CC_GENERATION_MASK 0xeUL
+ #define CREQ_QUERY_FUNC_RESP_SB_CC_GENERATION_SFT 1
+ #define CREQ_QUERY_FUNC_RESP_SB_CC_GENERATION_CC_GEN0 (0x0UL << 1)
+ #define CREQ_QUERY_FUNC_RESP_SB_CC_GENERATION_CC_GEN1 (0x1UL << 1)
+ #define CREQ_QUERY_FUNC_RESP_SB_CC_GENERATION_CC_GEN1_EXT (0x2UL << 1)
+ #define CREQ_QUERY_FUNC_RESP_SB_CC_GENERATION_LAST \
+ CREQ_QUERY_FUNC_RESP_SB_CC_GENERATION_CC_GEN1_EXT
+ #define CREQ_QUERY_FUNC_RESP_SB_EXT_STATS 0x10UL
+ #define CREQ_QUERY_FUNC_RESP_SB_MR_REGISTER_ALLOC 0x20UL
+ #define CREQ_QUERY_FUNC_RESP_SB_OPTIMIZED_TRANSMIT_ENABLED 0x40UL
+ #define CREQ_QUERY_FUNC_RESP_SB_CQE_V2 0x80UL
+ #define CREQ_QUERY_FUNC_RESP_SB_PINGPONG_PUSH_MODE 0x100UL
+ #define CREQ_QUERY_FUNC_RESP_SB_HW_REQUESTER_RETX_ENABLED 0x200UL
+ #define CREQ_QUERY_FUNC_RESP_SB_HW_RESPONDER_RETX_ENABLED 0x400UL
+ __le32 max_cq;
+ __le32 max_cqe;
+ __le32 max_pd;
+ u8 max_sge;
+ u8 max_srq_sge;
+ u8 max_qp_rd_atom;
+ u8 max_qp_init_rd_atom;
+ __le32 max_mr;
+ __le32 max_mw;
+ __le32 max_raw_eth_qp;
+ __le32 max_ah;
+ __le32 max_fmr;
+ __le32 max_srq_wr;
+ __le32 max_pkeys;
+ __le32 max_inline_data;
+ u8 max_map_per_fmr;
+ u8 l2_db_space_size;
+ __le16 max_srq;
+ __le32 max_gid;
+ __le32 tqm_alloc_reqs[12];
+ __le32 max_dpi;
+ u8 max_sge_var_wqe;
+ u8 reserved_8;
+ __le16 max_inline_data_var_wqe;
+};
+
+/* cmdq_set_func_resources (size:448b/56B) */
+struct cmdq_set_func_resources {
+ u8 opcode;
+ #define CMDQ_SET_FUNC_RESOURCES_OPCODE_SET_FUNC_RESOURCES 0x84UL
+ #define CMDQ_SET_FUNC_RESOURCES_OPCODE_LAST\
+ CMDQ_SET_FUNC_RESOURCES_OPCODE_SET_FUNC_RESOURCES
+ u8 cmd_size;
+ __le16 flags;
+ #define CMDQ_SET_FUNC_RESOURCES_FLAGS_MRAV_RESERVATION_SPLIT 0x1UL
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __le32 number_of_qp;
+ __le32 number_of_mrw;
+ __le32 number_of_srq;
+ __le32 number_of_cq;
+ __le32 max_qp_per_vf;
+ __le32 max_mrw_per_vf;
+ __le32 max_srq_per_vf;
+ __le32 max_cq_per_vf;
+ __le32 max_gid_per_vf;
+ __le32 stat_ctx_id;
+};
+
+/* creq_set_func_resources_resp (size:128b/16B) */
+struct creq_set_func_resources_resp {
+ u8 type;
+ #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_SFT 0
+ #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_LAST CREQ_SET_FUNC_RESOURCES_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 reserved32;
+ u8 v;
+ #define CREQ_SET_FUNC_RESOURCES_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_SET_FUNC_RESOURCES_RESP_EVENT_SET_FUNC_RESOURCES 0x84UL
+ #define CREQ_SET_FUNC_RESOURCES_RESP_EVENT_LAST \
+ CREQ_SET_FUNC_RESOURCES_RESP_EVENT_SET_FUNC_RESOURCES
+ u8 reserved48[6];
+};
+
+/* cmdq_map_tc_to_cos (size:192b/24B) */
+struct cmdq_map_tc_to_cos {
+ u8 opcode;
+ #define CMDQ_MAP_TC_TO_COS_OPCODE_MAP_TC_TO_COS 0x8aUL
+ #define CMDQ_MAP_TC_TO_COS_OPCODE_LAST CMDQ_MAP_TC_TO_COS_OPCODE_MAP_TC_TO_COS
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __le16 cos0;
+ #define CMDQ_MAP_TC_TO_COS_COS0_NO_CHANGE 0xffffUL
+ #define CMDQ_MAP_TC_TO_COS_COS0_LAST CMDQ_MAP_TC_TO_COS_COS0_NO_CHANGE
+ __le16 cos1;
+ #define CMDQ_MAP_TC_TO_COS_COS1_DISABLE 0x8000UL
+ #define CMDQ_MAP_TC_TO_COS_COS1_NO_CHANGE 0xffffUL
+ #define CMDQ_MAP_TC_TO_COS_COS1_LAST CMDQ_MAP_TC_TO_COS_COS1_NO_CHANGE
+ __le32 unused_0;
+};
+
+/* creq_map_tc_to_cos_resp (size:128b/16B) */
+struct creq_map_tc_to_cos_resp {
+ u8 type;
+ #define CREQ_MAP_TC_TO_COS_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_MAP_TC_TO_COS_RESP_TYPE_SFT 0
+ #define CREQ_MAP_TC_TO_COS_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_MAP_TC_TO_COS_RESP_TYPE_LAST CREQ_MAP_TC_TO_COS_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 reserved32;
+ u8 v;
+ #define CREQ_MAP_TC_TO_COS_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_MAP_TC_TO_COS_RESP_EVENT_MAP_TC_TO_COS 0x8aUL
+ #define CREQ_MAP_TC_TO_COS_RESP_EVENT_LAST CREQ_MAP_TC_TO_COS_RESP_EVENT_MAP_TC_TO_COS
+ u8 reserved48[6];
+};
+
+/* cmdq_query_roce_cc (size:128b/16B) */
+struct cmdq_query_roce_cc {
+ u8 opcode;
+ #define CMDQ_QUERY_ROCE_CC_OPCODE_QUERY_ROCE_CC 0x8dUL
+ #define CMDQ_QUERY_ROCE_CC_OPCODE_LAST CMDQ_QUERY_ROCE_CC_OPCODE_QUERY_ROCE_CC
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+};
+
+/* creq_query_roce_cc_resp (size:128b/16B) */
+struct creq_query_roce_cc_resp {
+ u8 type;
+ #define CREQ_QUERY_ROCE_CC_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_QUERY_ROCE_CC_RESP_TYPE_SFT 0
+ #define CREQ_QUERY_ROCE_CC_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_QUERY_ROCE_CC_RESP_TYPE_LAST CREQ_QUERY_ROCE_CC_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 size;
+ u8 v;
+ #define CREQ_QUERY_ROCE_CC_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_QUERY_ROCE_CC_RESP_EVENT_QUERY_ROCE_CC 0x8dUL
+ #define CREQ_QUERY_ROCE_CC_RESP_EVENT_LAST CREQ_QUERY_ROCE_CC_RESP_EVENT_QUERY_ROCE_CC
+ u8 reserved48[6];
+};
+
+/* creq_query_roce_cc_resp_sb (size:256b/32B) */
+struct creq_query_roce_cc_resp_sb {
+ u8 opcode;
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_OPCODE_QUERY_ROCE_CC 0x8dUL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_OPCODE_LAST \
+ CREQ_QUERY_ROCE_CC_RESP_SB_OPCODE_QUERY_ROCE_CC
+ u8 status;
+ __le16 cookie;
+ __le16 flags;
+ u8 resp_size;
+ u8 reserved8;
+ u8 enable_cc;
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_ENABLE_CC 0x1UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_UNUSED7_MASK 0xfeUL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_UNUSED7_SFT 1
+ u8 tos_dscp_tos_ecn;
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TOS_ECN_MASK 0x3UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TOS_ECN_SFT 0
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TOS_DSCP_MASK 0xfcUL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TOS_DSCP_SFT 2
+ u8 g;
+ u8 num_phases_per_state;
+ __le16 init_cr;
+ __le16 init_tr;
+ u8 alt_vlan_pcp;
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_ALT_VLAN_PCP_MASK 0x7UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_ALT_VLAN_PCP_SFT 0
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD1_MASK 0xf8UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD1_SFT 3
+ u8 alt_tos_dscp;
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_ALT_TOS_DSCP_MASK 0x3fUL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_ALT_TOS_DSCP_SFT 0
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD4_MASK 0xc0UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD4_SFT 6
+ u8 cc_mode;
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_CC_MODE_DCTCP 0x0UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_CC_MODE_PROBABILISTIC 0x1UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_CC_MODE_LAST \
+ CREQ_QUERY_ROCE_CC_RESP_SB_CC_MODE_PROBABILISTIC
+ u8 tx_queue;
+ __le16 rtt;
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RTT_MASK 0x3fffUL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RTT_SFT 0
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD5_MASK 0xc000UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD5_SFT 14
+ __le16 tcp_cp;
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TCP_CP_MASK 0x3ffUL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TCP_CP_SFT 0
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD6_MASK 0xfc00UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD6_SFT 10
+ __le16 inactivity_th;
+ u8 pkts_per_phase;
+ u8 time_per_phase;
+ __le32 reserved32;
+};
+
+/* creq_query_roce_cc_resp_sb_tlv (size:384b/48B) */
+struct creq_query_roce_cc_resp_sb_tlv {
+ __le16 cmd_discr;
+ u8 reserved_8b;
+ u8 tlv_flags;
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_MORE 0x1UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_MORE_LAST 0x0UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_MORE_NOT_LAST 0x1UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_REQUIRED 0x2UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_REQUIRED_NO (0x0UL << 1)
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_REQUIRED_YES (0x1UL << 1)
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_REQUIRED_LAST \
+ CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_REQUIRED_YES
+ __le16 tlv_type;
+ __le16 length;
+ u8 total_size;
+ u8 reserved56[7];
+ u8 opcode;
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_OPCODE_QUERY_ROCE_CC 0x8dUL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_OPCODE_LAST \
+ CREQ_QUERY_ROCE_CC_RESP_SB_TLV_OPCODE_QUERY_ROCE_CC
+ u8 status;
+ __le16 cookie;
+ __le16 flags;
+ u8 resp_size;
+ u8 reserved8;
+ u8 enable_cc;
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_ENABLE_CC 0x1UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_UNUSED7_MASK 0xfeUL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_UNUSED7_SFT 1
+ u8 tos_dscp_tos_ecn;
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TOS_ECN_MASK 0x3UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TOS_ECN_SFT 0
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TOS_DSCP_MASK 0xfcUL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TOS_DSCP_SFT 2
+ u8 g;
+ u8 num_phases_per_state;
+ __le16 init_cr;
+ __le16 init_tr;
+ u8 alt_vlan_pcp;
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_ALT_VLAN_PCP_MASK 0x7UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_ALT_VLAN_PCP_SFT 0
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD1_MASK 0xf8UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD1_SFT 3
+ u8 alt_tos_dscp;
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_ALT_TOS_DSCP_MASK 0x3fUL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_ALT_TOS_DSCP_SFT 0
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD4_MASK 0xc0UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD4_SFT 6
+ u8 cc_mode;
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_CC_MODE_DCTCP 0x0UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_CC_MODE_PROBABILISTIC 0x1UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_CC_MODE_LAST\
+ CREQ_QUERY_ROCE_CC_RESP_SB_TLV_CC_MODE_PROBABILISTIC
+ u8 tx_queue;
+ __le16 rtt;
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RTT_MASK 0x3fffUL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RTT_SFT 0
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD5_MASK 0xc000UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD5_SFT 14
+ __le16 tcp_cp;
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TCP_CP_MASK 0x3ffUL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TCP_CP_SFT 0
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD6_MASK 0xfc00UL
+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD6_SFT 10
+ __le16 inactivity_th;
+ u8 pkts_per_phase;
+ u8 time_per_phase;
+ __le32 reserved32;
+};
+
+/* creq_query_roce_cc_gen1_resp_sb_tlv (size:704b/88B) */
+struct creq_query_roce_cc_gen1_resp_sb_tlv {
+ __le16 cmd_discr;
+ u8 reserved_8b;
+ u8 tlv_flags;
+ #define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_MORE 0x1UL
+ #define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_MORE_LAST 0x0UL
+ #define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_MORE_NOT_LAST 0x1UL
+ #define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_REQUIRED 0x2UL
+ #define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_REQUIRED_NO (0x0UL << 1)
+ #define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_REQUIRED_YES (0x1UL << 1)
+ #define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_REQUIRED_LAST \
+ CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_REQUIRED_YES
+ __le16 tlv_type;
+ __le16 length;
+ __le64 reserved64;
+ __le16 inactivity_th_hi;
+ __le16 min_time_between_cnps;
+ __le16 init_cp;
+ u8 tr_update_mode;
+ u8 tr_update_cycles;
+ u8 fr_num_rtts;
+ u8 ai_rate_increase;
+ __le16 reduction_relax_rtts_th;
+ __le16 additional_relax_cr_th;
+ __le16 cr_min_th;
+ u8 bw_avg_weight;
+ u8 actual_cr_factor;
+ __le16 max_cp_cr_th;
+ u8 cp_bias_en;
+ u8 cp_bias;
+ u8 cnp_ecn;
+ #define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_CNP_ECN_NOT_ECT 0x0UL
+ #define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_CNP_ECN_ECT_1 0x1UL
+ #define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_CNP_ECN_ECT_0 0x2UL
+ #define CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_CNP_ECN_LAST \
+ CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_CNP_ECN_ECT_0
+ u8 rtt_jitter_en;
+ __le16 link_bytes_per_usec;
+ __le16 reset_cc_cr_th;
+ u8 cr_width;
+ u8 quota_period_min;
+ u8 quota_period_max;
+ u8 quota_period_abs_max;
+ __le16 tr_lower_bound;
+ u8 cr_prob_factor;
+ u8 tr_prob_factor;
+ __le16 fairness_cr_th;
+ u8 red_div;
+ u8 cnp_ratio_th;
+ __le16 exp_ai_rtts;
+ u8 exp_ai_cr_cp_ratio;
+ u8 use_rate_table;
+ __le16 cp_exp_update_th;
+ __le16 high_exp_ai_rtts_th1;
+ __le16 high_exp_ai_rtts_th2;
+ __le16 actual_cr_cong_free_rtts_th;
+ __le16 severe_cong_cr_th1;
+ __le16 severe_cong_cr_th2;
+ __le32 link64B_per_rtt;
+ u8 cc_ack_bytes;
+ u8 reduce_init_en;
+ __le16 reduce_init_cong_free_rtts_th;
+ u8 random_no_red_en;
+ u8 actual_cr_shift_correction_en;
+ u8 quota_period_adjust_en;
+ u8 reserved[5];
+};
+
+/* cmdq_modify_roce_cc (size:448b/56B) */
+struct cmdq_modify_roce_cc {
+ u8 opcode;
+ #define CMDQ_MODIFY_ROCE_CC_OPCODE_MODIFY_ROCE_CC 0x8cUL
+ #define CMDQ_MODIFY_ROCE_CC_OPCODE_LAST CMDQ_MODIFY_ROCE_CC_OPCODE_MODIFY_ROCE_CC
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __le32 modify_mask;
+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC 0x1UL
+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_G 0x2UL
+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_NUMPHASEPERSTATE 0x4UL
+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_INIT_CR 0x8UL
+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_INIT_TR 0x10UL
+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN 0x20UL
+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP 0x40UL
+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP 0x80UL
+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP 0x100UL
+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_RTT 0x200UL
+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE 0x400UL
+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TCP_CP 0x800UL
+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TX_QUEUE 0x1000UL
+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_INACTIVITY_CP 0x2000UL
+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TIME_PER_PHASE 0x4000UL
+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_PKTS_PER_PHASE 0x8000UL
+ u8 enable_cc;
+ #define CMDQ_MODIFY_ROCE_CC_ENABLE_CC 0x1UL
+ #define CMDQ_MODIFY_ROCE_CC_RSVD1_MASK 0xfeUL
+ #define CMDQ_MODIFY_ROCE_CC_RSVD1_SFT 1
+ u8 g;
+ u8 num_phases_per_state;
+ u8 pkts_per_phase;
+ __le16 init_cr;
+ __le16 init_tr;
+ u8 tos_dscp_tos_ecn;
+ #define CMDQ_MODIFY_ROCE_CC_TOS_ECN_MASK 0x3UL
+ #define CMDQ_MODIFY_ROCE_CC_TOS_ECN_SFT 0
+ #define CMDQ_MODIFY_ROCE_CC_TOS_DSCP_MASK 0xfcUL
+ #define CMDQ_MODIFY_ROCE_CC_TOS_DSCP_SFT 2
+ u8 alt_vlan_pcp;
+ #define CMDQ_MODIFY_ROCE_CC_ALT_VLAN_PCP_MASK 0x7UL
+ #define CMDQ_MODIFY_ROCE_CC_ALT_VLAN_PCP_SFT 0
+ #define CMDQ_MODIFY_ROCE_CC_RSVD3_MASK 0xf8UL
+ #define CMDQ_MODIFY_ROCE_CC_RSVD3_SFT 3
+ __le16 alt_tos_dscp;
+ #define CMDQ_MODIFY_ROCE_CC_ALT_TOS_DSCP_MASK 0x3fUL
+ #define CMDQ_MODIFY_ROCE_CC_ALT_TOS_DSCP_SFT 0
+ #define CMDQ_MODIFY_ROCE_CC_RSVD4_MASK 0xffc0UL
+ #define CMDQ_MODIFY_ROCE_CC_RSVD4_SFT 6
+ __le16 rtt;
+ #define CMDQ_MODIFY_ROCE_CC_RTT_MASK 0x3fffUL
+ #define CMDQ_MODIFY_ROCE_CC_RTT_SFT 0
+ #define CMDQ_MODIFY_ROCE_CC_RSVD5_MASK 0xc000UL
+ #define CMDQ_MODIFY_ROCE_CC_RSVD5_SFT 14
+ __le16 tcp_cp;
+ #define CMDQ_MODIFY_ROCE_CC_TCP_CP_MASK 0x3ffUL
+ #define CMDQ_MODIFY_ROCE_CC_TCP_CP_SFT 0
+ #define CMDQ_MODIFY_ROCE_CC_RSVD6_MASK 0xfc00UL
+ #define CMDQ_MODIFY_ROCE_CC_RSVD6_SFT 10
+ u8 cc_mode;
+ #define CMDQ_MODIFY_ROCE_CC_CC_MODE_DCTCP_CC_MODE 0x0UL
+ #define CMDQ_MODIFY_ROCE_CC_CC_MODE_PROBABILISTIC_CC_MODE 0x1UL
+ #define CMDQ_MODIFY_ROCE_CC_CC_MODE_LAST CMDQ_MODIFY_ROCE_CC_CC_MODE_PROBABILISTIC_CC_MODE
+ u8 tx_queue;
+ __le16 inactivity_th;
+ u8 time_per_phase;
+ u8 reserved8_1;
+ __le16 reserved16;
+ __le32 reserved32;
+ __le64 reserved64;
+};
+
+/* cmdq_modify_roce_cc_tlv (size:640b/80B) */
+struct cmdq_modify_roce_cc_tlv {
+ __le16 cmd_discr;
+ u8 reserved_8b;
+ u8 tlv_flags;
+ #define CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_MORE 0x1UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_MORE_LAST 0x0UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_MORE_NOT_LAST 0x1UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_REQUIRED 0x2UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_REQUIRED_NO (0x0UL << 1)
+ #define CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_REQUIRED_YES (0x1UL << 1)
+ #define CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_REQUIRED_LAST \
+ CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_REQUIRED_YES
+ __le16 tlv_type;
+ __le16 length;
+ u8 total_size;
+ u8 reserved56[7];
+ u8 opcode;
+ #define CMDQ_MODIFY_ROCE_CC_TLV_OPCODE_MODIFY_ROCE_CC 0x8cUL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_OPCODE_LAST CMDQ_MODIFY_ROCE_CC_TLV_OPCODE_MODIFY_ROCE_CC
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __le32 modify_mask;
+ #define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_ENABLE_CC 0x1UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_G 0x2UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_NUMPHASEPERSTATE 0x4UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_INIT_CR 0x8UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_INIT_TR 0x10UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_TOS_ECN 0x20UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_TOS_DSCP 0x40UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_ALT_VLAN_PCP 0x80UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_ALT_TOS_DSCP 0x100UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_RTT 0x200UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_CC_MODE 0x400UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_TCP_CP 0x800UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_TX_QUEUE 0x1000UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_INACTIVITY_CP 0x2000UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_TIME_PER_PHASE 0x4000UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_PKTS_PER_PHASE 0x8000UL
+ u8 enable_cc;
+ #define CMDQ_MODIFY_ROCE_CC_TLV_ENABLE_CC 0x1UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_RSVD1_MASK 0xfeUL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_RSVD1_SFT 1
+ u8 g;
+ u8 num_phases_per_state;
+ u8 pkts_per_phase;
+ __le16 init_cr;
+ __le16 init_tr;
+ u8 tos_dscp_tos_ecn;
+ #define CMDQ_MODIFY_ROCE_CC_TLV_TOS_ECN_MASK 0x3UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_TOS_ECN_SFT 0
+ #define CMDQ_MODIFY_ROCE_CC_TLV_TOS_DSCP_MASK 0xfcUL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_TOS_DSCP_SFT 2
+ u8 alt_vlan_pcp;
+ #define CMDQ_MODIFY_ROCE_CC_TLV_ALT_VLAN_PCP_MASK 0x7UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_ALT_VLAN_PCP_SFT 0
+ #define CMDQ_MODIFY_ROCE_CC_TLV_RSVD3_MASK 0xf8UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_RSVD3_SFT 3
+ __le16 alt_tos_dscp;
+ #define CMDQ_MODIFY_ROCE_CC_TLV_ALT_TOS_DSCP_MASK 0x3fUL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_ALT_TOS_DSCP_SFT 0
+ #define CMDQ_MODIFY_ROCE_CC_TLV_RSVD4_MASK 0xffc0UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_RSVD4_SFT 6
+ __le16 rtt;
+ #define CMDQ_MODIFY_ROCE_CC_TLV_RTT_MASK 0x3fffUL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_RTT_SFT 0
+ #define CMDQ_MODIFY_ROCE_CC_TLV_RSVD5_MASK 0xc000UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_RSVD5_SFT 14
+ __le16 tcp_cp;
+ #define CMDQ_MODIFY_ROCE_CC_TLV_TCP_CP_MASK 0x3ffUL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_TCP_CP_SFT 0
+ #define CMDQ_MODIFY_ROCE_CC_TLV_RSVD6_MASK 0xfc00UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_RSVD6_SFT 10
+ u8 cc_mode;
+ #define CMDQ_MODIFY_ROCE_CC_TLV_CC_MODE_DCTCP_CC_MODE 0x0UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_CC_MODE_PROBABILISTIC_CC_MODE 0x1UL
+ #define CMDQ_MODIFY_ROCE_CC_TLV_CC_MODE_LAST\
+ CMDQ_MODIFY_ROCE_CC_TLV_CC_MODE_PROBABILISTIC_CC_MODE
+ u8 tx_queue;
+ __le16 inactivity_th;
+ u8 time_per_phase;
+ u8 reserved8_1;
+ __le16 reserved16;
+ __le32 reserved32;
+ __le64 reserved64;
+ __le64 reservedtlvpad;
+};
+
+/* cmdq_modify_roce_cc_gen1_tlv (size:768b/96B) */
+struct cmdq_modify_roce_cc_gen1_tlv {
+ __le16 cmd_discr;
+ u8 reserved_8b;
+ u8 tlv_flags;
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_MORE 0x1UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_MORE_LAST 0x0UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_MORE_NOT_LAST 0x1UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_REQUIRED 0x2UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_REQUIRED_NO (0x0UL << 1)
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_REQUIRED_YES (0x1UL << 1)
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_REQUIRED_LAST\
+ CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_REQUIRED_YES
+ __le16 tlv_type;
+ __le16 length;
+ __le64 reserved64;
+ __le64 modify_mask;
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_MIN_TIME_BETWEEN_CNPS 0x1UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_INIT_CP 0x2UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_TR_UPDATE_MODE 0x4UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_TR_UPDATE_CYCLES 0x8UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_FR_NUM_RTTS 0x10UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_AI_RATE_INCREASE 0x20UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_REDUCTION_RELAX_RTTS_TH 0x40UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_ADDITIONAL_RELAX_CR_TH 0x80UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CR_MIN_TH 0x100UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_BW_AVG_WEIGHT 0x200UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_ACTUAL_CR_FACTOR 0x400UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_MAX_CP_CR_TH 0x800UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CP_BIAS_EN 0x1000UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CP_BIAS 0x2000UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CNP_ECN 0x4000UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_RTT_JITTER_EN 0x8000UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_LINK_BYTES_PER_USEC 0x10000UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_RESET_CC_CR_TH 0x20000UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CR_WIDTH 0x40000UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_QUOTA_PERIOD_MIN 0x80000UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_QUOTA_PERIOD_MAX 0x100000UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_QUOTA_PERIOD_ABS_MAX 0x200000UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_TR_LOWER_BOUND 0x400000UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CR_PROB_FACTOR 0x800000UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_TR_PROB_FACTOR 0x1000000UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_FAIRNESS_CR_TH 0x2000000UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_RED_DIV 0x4000000UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CNP_RATIO_TH 0x8000000UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_EXP_AI_RTTS 0x10000000UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_EXP_AI_CR_CP_RATIO 0x20000000UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CP_EXP_UPDATE_TH 0x40000000UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_HIGH_EXP_AI_RTTS_TH1 0x80000000UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_HIGH_EXP_AI_RTTS_TH2 0x100000000ULL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_USE_RATE_TABLE 0x200000000ULL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_LINK64B_PER_RTT 0x400000000ULL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_ACTUAL_CR_CONG_FREE_RTTS_TH 0x800000000ULL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_SEVERE_CONG_CR_TH1 0x1000000000ULL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_SEVERE_CONG_CR_TH2 0x2000000000ULL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CC_ACK_BYTES 0x4000000000ULL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_REDUCE_INIT_EN 0x8000000000ULL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_REDUCE_INIT_CONG_FREE_RTTS_TH \
+ 0x10000000000ULL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_RANDOM_NO_RED_EN 0x20000000000ULL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_ACTUAL_CR_SHIFT_CORRECTION_EN \
+ 0x40000000000ULL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_QUOTA_PERIOD_ADJUST_EN 0x80000000000ULL
+ __le16 inactivity_th_hi;
+ __le16 min_time_between_cnps;
+ __le16 init_cp;
+ u8 tr_update_mode;
+ u8 tr_update_cycles;
+ u8 fr_num_rtts;
+ u8 ai_rate_increase;
+ __le16 reduction_relax_rtts_th;
+ __le16 additional_relax_cr_th;
+ __le16 cr_min_th;
+ u8 bw_avg_weight;
+ u8 actual_cr_factor;
+ __le16 max_cp_cr_th;
+ u8 cp_bias_en;
+ u8 cp_bias;
+ u8 cnp_ecn;
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_CNP_ECN_NOT_ECT 0x0UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_CNP_ECN_ECT_1 0x1UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_CNP_ECN_ECT_0 0x2UL
+ #define CMDQ_MODIFY_ROCE_CC_GEN1_TLV_CNP_ECN_LAST CMDQ_MODIFY_ROCE_CC_GEN1_TLV_CNP_ECN_ECT_0
+ u8 rtt_jitter_en;
+ __le16 link_bytes_per_usec;
+ __le16 reset_cc_cr_th;
+ u8 cr_width;
+ u8 quota_period_min;
+ u8 quota_period_max;
+ u8 quota_period_abs_max;
+ __le16 tr_lower_bound;
+ u8 cr_prob_factor;
+ u8 tr_prob_factor;
+ __le16 fairness_cr_th;
+ u8 red_div;
+ u8 cnp_ratio_th;
+ __le16 exp_ai_rtts;
+ u8 exp_ai_cr_cp_ratio;
+ u8 use_rate_table;
+ __le16 cp_exp_update_th;
+ __le16 high_exp_ai_rtts_th1;
+ __le16 high_exp_ai_rtts_th2;
+ __le16 actual_cr_cong_free_rtts_th;
+ __le16 severe_cong_cr_th1;
+ __le16 severe_cong_cr_th2;
+ __le32 link64B_per_rtt;
+ u8 cc_ack_bytes;
+ u8 reduce_init_en;
+ __le16 reduce_init_cong_free_rtts_th;
+ u8 random_no_red_en;
+ u8 actual_cr_shift_correction_en;
+ u8 quota_period_adjust_en;
+ u8 reserved[5];
+};
+
+/* creq_modify_roce_cc_resp (size:128b/16B) */
+struct creq_modify_roce_cc_resp {
+ u8 type;
+ #define CREQ_MODIFY_ROCE_CC_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_MODIFY_ROCE_CC_RESP_TYPE_SFT 0
+ #define CREQ_MODIFY_ROCE_CC_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_MODIFY_ROCE_CC_RESP_TYPE_LAST CREQ_MODIFY_ROCE_CC_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 reserved32;
+ u8 v;
+ #define CREQ_MODIFY_ROCE_CC_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_MODIFY_ROCE_CC_RESP_EVENT_MODIFY_ROCE_CC 0x8cUL
+ #define CREQ_MODIFY_ROCE_CC_RESP_EVENT_LAST CREQ_MODIFY_ROCE_CC_RESP_EVENT_MODIFY_ROCE_CC
+ u8 reserved48[6];
+};
+
+/* cmdq_set_link_aggr_mode_cc (size:320b/40B) */
+struct cmdq_set_link_aggr_mode_cc {
+ u8 opcode;
+ #define CMDQ_SET_LINK_AGGR_MODE_OPCODE_SET_LINK_AGGR_MODE 0x8fUL
+ #define CMDQ_SET_LINK_AGGR_MODE_OPCODE_LAST \
+ CMDQ_SET_LINK_AGGR_MODE_OPCODE_SET_LINK_AGGR_MODE
+ u8 cmd_size;
+ __le16 flags;
+ __le16 cookie;
+ u8 resp_size;
+ u8 reserved8;
+ __le64 resp_addr;
+ __le32 modify_mask;
+ #define CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_AGGR_EN 0x1UL
+ #define CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_ACTIVE_PORT_MAP 0x2UL
+ #define CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_MEMBER_PORT_MAP 0x4UL
+ #define CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_AGGR_MODE 0x8UL
+ #define CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_STAT_CTX_ID 0x10UL
+ u8 aggr_enable;
+ #define CMDQ_SET_LINK_AGGR_MODE_AGGR_ENABLE 0x1UL
+ #define CMDQ_SET_LINK_AGGR_MODE_RSVD1_MASK 0xfeUL
+ #define CMDQ_SET_LINK_AGGR_MODE_RSVD1_SFT 1
+ u8 active_port_map;
+ #define CMDQ_SET_LINK_AGGR_MODE_ACTIVE_PORT_MAP_MASK 0xfUL
+ #define CMDQ_SET_LINK_AGGR_MODE_ACTIVE_PORT_MAP_SFT 0
+ #define CMDQ_SET_LINK_AGGR_MODE_RSVD2_MASK 0xf0UL
+ #define CMDQ_SET_LINK_AGGR_MODE_RSVD2_SFT 4
+ u8 member_port_map;
+ u8 link_aggr_mode;
+ #define CMDQ_SET_LINK_AGGR_MODE_AGGR_MODE_ACTIVE_ACTIVE 0x1UL
+ #define CMDQ_SET_LINK_AGGR_MODE_AGGR_MODE_ACTIVE_BACKUP 0x2UL
+ #define CMDQ_SET_LINK_AGGR_MODE_AGGR_MODE_BALANCE_XOR 0x3UL
+ #define CMDQ_SET_LINK_AGGR_MODE_AGGR_MODE_802_3_AD 0x4UL
+ #define CMDQ_SET_LINK_AGGR_MODE_AGGR_MODE_LAST CMDQ_SET_LINK_AGGR_MODE_AGGR_MODE_802_3_AD
+ __le16 stat_ctx_id[4];
+ __le64 rsvd1;
+};
+
+/* creq_set_link_aggr_mode_resources_resp (size:128b/16B) */
+struct creq_set_link_aggr_mode_resources_resp {
+ u8 type;
+ #define CREQ_SET_LINK_AGGR_MODE_RESP_TYPE_MASK 0x3fUL
+ #define CREQ_SET_LINK_AGGR_MODE_RESP_TYPE_SFT 0
+ #define CREQ_SET_LINK_AGGR_MODE_RESP_TYPE_QP_EVENT 0x38UL
+ #define CREQ_SET_LINK_AGGR_MODE_RESP_TYPE_LAST CREQ_SET_LINK_AGGR_MODE_RESP_TYPE_QP_EVENT
+ u8 status;
+ __le16 cookie;
+ __le32 reserved32;
+ u8 v;
+ #define CREQ_SET_LINK_AGGR_MODE_RESP_V 0x1UL
+ u8 event;
+ #define CREQ_SET_LINK_AGGR_MODE_RESP_EVENT_SET_LINK_AGGR_MODE 0x8fUL
+ #define CREQ_SET_LINK_AGGR_MODE_RESP_EVENT_LAST\
+ CREQ_SET_LINK_AGGR_MODE_RESP_EVENT_SET_LINK_AGGR_MODE
+ u8 reserved48[6];
+};
+
+/* creq_func_event (size:128b/16B) */
+struct creq_func_event {
+ u8 type;
+ #define CREQ_FUNC_EVENT_TYPE_MASK 0x3fUL
+ #define CREQ_FUNC_EVENT_TYPE_SFT 0
+ #define CREQ_FUNC_EVENT_TYPE_FUNC_EVENT 0x3aUL
+ #define CREQ_FUNC_EVENT_TYPE_LAST CREQ_FUNC_EVENT_TYPE_FUNC_EVENT
+ u8 reserved56[7];
+ u8 v;
+ #define CREQ_FUNC_EVENT_V 0x1UL
+ u8 event;
+ #define CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR 0x1UL
+ #define CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR 0x2UL
+ #define CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR 0x3UL
+ #define CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR 0x4UL
+ #define CREQ_FUNC_EVENT_EVENT_CQ_ERROR 0x5UL
+ #define CREQ_FUNC_EVENT_EVENT_TQM_ERROR 0x6UL
+ #define CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR 0x7UL
+ #define CREQ_FUNC_EVENT_EVENT_CFCS_ERROR 0x8UL
+ #define CREQ_FUNC_EVENT_EVENT_CFCC_ERROR 0x9UL
+ #define CREQ_FUNC_EVENT_EVENT_CFCM_ERROR 0xaUL
+ #define CREQ_FUNC_EVENT_EVENT_TIM_ERROR 0xbUL
+ #define CREQ_FUNC_EVENT_EVENT_VF_COMM_REQUEST 0x80UL
+ #define CREQ_FUNC_EVENT_EVENT_RESOURCE_EXHAUSTED 0x81UL
+ #define CREQ_FUNC_EVENT_EVENT_LAST CREQ_FUNC_EVENT_EVENT_RESOURCE_EXHAUSTED
+ u8 reserved48[6];
+};
+
+/* creq_qp_event (size:128b/16B) */
+struct creq_qp_event {
+ u8 type;
+ #define CREQ_QP_EVENT_TYPE_MASK 0x3fUL
+ #define CREQ_QP_EVENT_TYPE_SFT 0
+ #define CREQ_QP_EVENT_TYPE_QP_EVENT 0x38UL
+ #define CREQ_QP_EVENT_TYPE_LAST CREQ_QP_EVENT_TYPE_QP_EVENT
+ u8 status;
+ #define CREQ_QP_EVENT_STATUS_SUCCESS 0x0UL
+ #define CREQ_QP_EVENT_STATUS_FAIL 0x1UL
+ #define CREQ_QP_EVENT_STATUS_RESOURCES 0x2UL
+ #define CREQ_QP_EVENT_STATUS_INVALID_CMD 0x3UL
+ #define CREQ_QP_EVENT_STATUS_NOT_IMPLEMENTED 0x4UL
+ #define CREQ_QP_EVENT_STATUS_INVALID_PARAMETER 0x5UL
+ #define CREQ_QP_EVENT_STATUS_HARDWARE_ERROR 0x6UL
+ #define CREQ_QP_EVENT_STATUS_INTERNAL_ERROR 0x7UL
+ #define CREQ_QP_EVENT_STATUS_LAST CREQ_QP_EVENT_STATUS_INTERNAL_ERROR
+ __le16 cookie;
+ __le32 reserved32;
+ u8 v;
+ #define CREQ_QP_EVENT_V 0x1UL
+ u8 event;
+ #define CREQ_QP_EVENT_EVENT_CREATE_QP 0x1UL
+ #define CREQ_QP_EVENT_EVENT_DESTROY_QP 0x2UL
+ #define CREQ_QP_EVENT_EVENT_MODIFY_QP 0x3UL
+ #define CREQ_QP_EVENT_EVENT_QUERY_QP 0x4UL
+ #define CREQ_QP_EVENT_EVENT_CREATE_SRQ 0x5UL
+ #define CREQ_QP_EVENT_EVENT_DESTROY_SRQ 0x6UL
+ #define CREQ_QP_EVENT_EVENT_QUERY_SRQ 0x8UL
+ #define CREQ_QP_EVENT_EVENT_CREATE_CQ 0x9UL
+ #define CREQ_QP_EVENT_EVENT_DESTROY_CQ 0xaUL
+ #define CREQ_QP_EVENT_EVENT_RESIZE_CQ 0xcUL
+ #define CREQ_QP_EVENT_EVENT_ALLOCATE_MRW 0xdUL
+ #define CREQ_QP_EVENT_EVENT_DEALLOCATE_KEY 0xeUL
+ #define CREQ_QP_EVENT_EVENT_REGISTER_MR 0xfUL
+ #define CREQ_QP_EVENT_EVENT_DEREGISTER_MR 0x10UL
+ #define CREQ_QP_EVENT_EVENT_ADD_GID 0x11UL
+ #define CREQ_QP_EVENT_EVENT_DELETE_GID 0x12UL
+ #define CREQ_QP_EVENT_EVENT_MODIFY_GID 0x17UL
+ #define CREQ_QP_EVENT_EVENT_QUERY_GID 0x18UL
+ #define CREQ_QP_EVENT_EVENT_CREATE_QP1 0x13UL
+ #define CREQ_QP_EVENT_EVENT_DESTROY_QP1 0x14UL
+ #define CREQ_QP_EVENT_EVENT_CREATE_AH 0x15UL
+ #define CREQ_QP_EVENT_EVENT_DESTROY_AH 0x16UL
+ #define CREQ_QP_EVENT_EVENT_INITIALIZE_FW 0x80UL
+ #define CREQ_QP_EVENT_EVENT_DEINITIALIZE_FW 0x81UL
+ #define CREQ_QP_EVENT_EVENT_STOP_FUNC 0x82UL
+ #define CREQ_QP_EVENT_EVENT_QUERY_FUNC 0x83UL
+ #define CREQ_QP_EVENT_EVENT_SET_FUNC_RESOURCES 0x84UL
+ #define CREQ_QP_EVENT_EVENT_READ_CONTEXT 0x85UL
+ #define CREQ_QP_EVENT_EVENT_MAP_TC_TO_COS 0x8aUL
+ #define CREQ_QP_EVENT_EVENT_QUERY_VERSION 0x8bUL
+ #define CREQ_QP_EVENT_EVENT_MODIFY_CC 0x8cUL
+ #define CREQ_QP_EVENT_EVENT_QUERY_CC 0x8dUL
+ #define CREQ_QP_EVENT_EVENT_QUERY_ROCE_STATS 0x8eUL
+ #define CREQ_QP_EVENT_EVENT_SET_LINK_AGGR_MODE 0x8fUL
+ #define CREQ_QP_EVENT_EVENT_QUERY_QP_EXTEND 0x91UL
+ #define CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION 0xc0UL
+ #define CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION 0xc1UL
+ #define CREQ_QP_EVENT_EVENT_LAST CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION
+ u8 reserved48[6];
+};
+
+/* creq_qp_error_notification (size:128b/16B) */
struct creq_qp_error_notification {
- u8 type;
- #define CREQ_QP_ERROR_NOTIFICATION_TYPE_MASK 0x3fUL
- #define CREQ_QP_ERROR_NOTIFICATION_TYPE_SFT 0
- #define CREQ_QP_ERROR_NOTIFICATION_TYPE_QP_EVENT 0x38UL
- #define CREQ_QP_ERROR_NOTIFICATION_RESERVED2_MASK 0xc0UL
- #define CREQ_QP_ERROR_NOTIFICATION_RESERVED2_SFT 6
- u8 status;
- u8 req_slow_path_state;
- u8 req_err_state_reason;
- __le32 xid;
- u8 v;
- #define CREQ_QP_ERROR_NOTIFICATION_V 0x1UL
- #define CREQ_QP_ERROR_NOTIFICATION_RESERVED7_MASK 0xfeUL
- #define CREQ_QP_ERROR_NOTIFICATION_RESERVED7_SFT 1
- u8 event;
+ u8 type;
+ #define CREQ_QP_ERROR_NOTIFICATION_TYPE_MASK 0x3fUL
+ #define CREQ_QP_ERROR_NOTIFICATION_TYPE_SFT 0
+ #define CREQ_QP_ERROR_NOTIFICATION_TYPE_QP_EVENT 0x38UL
+ #define CREQ_QP_ERROR_NOTIFICATION_TYPE_LAST CREQ_QP_ERROR_NOTIFICATION_TYPE_QP_EVENT
+ u8 status;
+ u8 req_slow_path_state;
+ u8 req_err_state_reason;
+ __le32 xid;
+ u8 v;
+ #define CREQ_QP_ERROR_NOTIFICATION_V 0x1UL
+ u8 event;
#define CREQ_QP_ERROR_NOTIFICATION_EVENT_QP_ERROR_NOTIFICATION 0xc0UL
- u8 res_slow_path_state;
- u8 res_err_state_reason;
- __le16 sq_cons_idx;
- __le16 rq_cons_idx;
+ #define CREQ_QP_ERROR_NOTIFICATION_EVENT_LAST \
+ CREQ_QP_ERROR_NOTIFICATION_EVENT_QP_ERROR_NOTIFICATION
+ u8 res_slow_path_state;
+ u8 res_err_state_reason;
+ __le16 sq_cons_idx;
+ __le16 rq_cons_idx;
};
-/* RoCE Slowpath HSI Specification 1.6.0 */
-#define ROCE_SP_HSI_VERSION_MAJOR 1
-#define ROCE_SP_HSI_VERSION_MINOR 6
-#define ROCE_SP_HSI_VERSION_UPDATE 0
+/* creq_cq_error_notification (size:128b/16B) */
+struct creq_cq_error_notification {
+ u8 type;
+ #define CREQ_CQ_ERROR_NOTIFICATION_TYPE_MASK 0x3fUL
+ #define CREQ_CQ_ERROR_NOTIFICATION_TYPE_SFT 0
+ #define CREQ_CQ_ERROR_NOTIFICATION_TYPE_CQ_EVENT 0x38UL
+ #define CREQ_CQ_ERROR_NOTIFICATION_TYPE_LAST CREQ_CQ_ERROR_NOTIFICATION_TYPE_CQ_EVENT
+ u8 status;
+ u8 cq_err_reason;
+ #define CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_INVALID_ERROR 0x1UL
+ #define CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_OVERFLOW_ERROR 0x2UL
+ #define CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_LOAD_ERROR 0x3UL
+ #define CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_INVALID_ERROR 0x4UL
+ #define CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_OVERFLOW_ERROR 0x5UL
+ #define CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_LOAD_ERROR 0x6UL
+ #define CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_LAST \
+ CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_LOAD_ERROR
+ u8 reserved8;
+ __le32 xid;
+ u8 v;
+ #define CREQ_CQ_ERROR_NOTIFICATION_V 0x1UL
+ u8 event;
+ #define CREQ_CQ_ERROR_NOTIFICATION_EVENT_CQ_ERROR_NOTIFICATION 0xc1UL
+ #define CREQ_CQ_ERROR_NOTIFICATION_EVENT_LAST \
+ CREQ_CQ_ERROR_NOTIFICATION_EVENT_CQ_ERROR_NOTIFICATION
+ u8 reserved48[6];
+};
-#define ROCE_SP_HSI_VERSION_STR "1.6.0"
-/*
- * Following is the signature for ROCE_SP_HSI message field that indicates not
- * applicable (All F's). Need to cast it the size of the field if needed.
- */
-#define ROCE_SP_HSI_NA_SIGNATURE ((__le32)(-1))
-#endif /* __BNXT_RE_HSI_H__ */
+/* sq_base (size:64b/8B) */
+struct sq_base {
+ u8 wqe_type;
+ #define SQ_BASE_WQE_TYPE_SEND 0x0UL
+ #define SQ_BASE_WQE_TYPE_SEND_W_IMMEAD 0x1UL
+ #define SQ_BASE_WQE_TYPE_SEND_W_INVALID 0x2UL
+ #define SQ_BASE_WQE_TYPE_WRITE_WQE 0x4UL
+ #define SQ_BASE_WQE_TYPE_WRITE_W_IMMEAD 0x5UL
+ #define SQ_BASE_WQE_TYPE_READ_WQE 0x6UL
+ #define SQ_BASE_WQE_TYPE_ATOMIC_CS 0x8UL
+ #define SQ_BASE_WQE_TYPE_ATOMIC_FA 0xbUL
+ #define SQ_BASE_WQE_TYPE_LOCAL_INVALID 0xcUL
+ #define SQ_BASE_WQE_TYPE_FR_PMR 0xdUL
+ #define SQ_BASE_WQE_TYPE_BIND 0xeUL
+ #define SQ_BASE_WQE_TYPE_FR_PPMR 0xfUL
+ #define SQ_BASE_WQE_TYPE_LAST SQ_BASE_WQE_TYPE_FR_PPMR
+ u8 unused_0[7];
+};
+
+/* sq_sge (size:128b/16B) */
+struct sq_sge {
+ __le64 va_or_pa;
+ __le32 l_key;
+ __le32 size;
+};
+
+/* sq_psn_search (size:64b/8B) */
+struct sq_psn_search {
+ __le32 opcode_start_psn;
+ #define SQ_PSN_SEARCH_START_PSN_MASK 0xffffffUL
+ #define SQ_PSN_SEARCH_START_PSN_SFT 0
+ #define SQ_PSN_SEARCH_OPCODE_MASK 0xff000000UL
+ #define SQ_PSN_SEARCH_OPCODE_SFT 24
+ __le32 flags_next_psn;
+ #define SQ_PSN_SEARCH_NEXT_PSN_MASK 0xffffffUL
+ #define SQ_PSN_SEARCH_NEXT_PSN_SFT 0
+ #define SQ_PSN_SEARCH_FLAGS_MASK 0xff000000UL
+ #define SQ_PSN_SEARCH_FLAGS_SFT 24
+};
+
+/* sq_psn_search_ext (size:128b/16B) */
+struct sq_psn_search_ext {
+ __le32 opcode_start_psn;
+ #define SQ_PSN_SEARCH_EXT_START_PSN_MASK 0xffffffUL
+ #define SQ_PSN_SEARCH_EXT_START_PSN_SFT 0
+ #define SQ_PSN_SEARCH_EXT_OPCODE_MASK 0xff000000UL
+ #define SQ_PSN_SEARCH_EXT_OPCODE_SFT 24
+ __le32 flags_next_psn;
+ #define SQ_PSN_SEARCH_EXT_NEXT_PSN_MASK 0xffffffUL
+ #define SQ_PSN_SEARCH_EXT_NEXT_PSN_SFT 0
+ #define SQ_PSN_SEARCH_EXT_FLAGS_MASK 0xff000000UL
+ #define SQ_PSN_SEARCH_EXT_FLAGS_SFT 24
+ __le16 start_slot_idx;
+ __le16 reserved16;
+ __le32 reserved32;
+};
+
+/* sq_send (size:1024b/128B) */
+struct sq_send {
+ u8 wqe_type;
+ #define SQ_SEND_WQE_TYPE_SEND 0x0UL
+ #define SQ_SEND_WQE_TYPE_SEND_W_IMMEAD 0x1UL
+ #define SQ_SEND_WQE_TYPE_SEND_W_INVALID 0x2UL
+ #define SQ_SEND_WQE_TYPE_LAST SQ_SEND_WQE_TYPE_SEND_W_INVALID
+ u8 flags;
+ #define SQ_SEND_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK 0xffUL
+ #define SQ_SEND_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT 0
+ #define SQ_SEND_FLAGS_SIGNAL_COMP 0x1UL
+ #define SQ_SEND_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
+ #define SQ_SEND_FLAGS_UC_FENCE 0x4UL
+ #define SQ_SEND_FLAGS_SE 0x8UL
+ #define SQ_SEND_FLAGS_INLINE 0x10UL
+ #define SQ_SEND_FLAGS_WQE_TS_EN 0x20UL
+ #define SQ_SEND_FLAGS_DEBUG_TRACE 0x40UL
+ u8 wqe_size;
+ u8 reserved8_1;
+ __le32 inv_key_or_imm_data;
+ __le32 length;
+ __le32 q_key;
+ __le32 dst_qp;
+ #define SQ_SEND_DST_QP_MASK 0xffffffUL
+ #define SQ_SEND_DST_QP_SFT 0
+ __le32 avid;
+ #define SQ_SEND_AVID_MASK 0xfffffUL
+ #define SQ_SEND_AVID_SFT 0
+ __le32 reserved32;
+ __le32 timestamp;
+ #define SQ_SEND_TIMESTAMP_MASK 0xffffffUL
+ #define SQ_SEND_TIMESTAMP_SFT 0
+ __le32 data[24];
+};
+
+/* sq_send_hdr (size:256b/32B) */
+struct sq_send_hdr {
+ u8 wqe_type;
+ #define SQ_SEND_HDR_WQE_TYPE_SEND 0x0UL
+ #define SQ_SEND_HDR_WQE_TYPE_SEND_W_IMMEAD 0x1UL
+ #define SQ_SEND_HDR_WQE_TYPE_SEND_W_INVALID 0x2UL
+ #define SQ_SEND_HDR_WQE_TYPE_LAST SQ_SEND_HDR_WQE_TYPE_SEND_W_INVALID
+ u8 flags;
+ #define SQ_SEND_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK 0xffUL
+ #define SQ_SEND_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT 0
+ #define SQ_SEND_HDR_FLAGS_SIGNAL_COMP 0x1UL
+ #define SQ_SEND_HDR_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
+ #define SQ_SEND_HDR_FLAGS_UC_FENCE 0x4UL
+ #define SQ_SEND_HDR_FLAGS_SE 0x8UL
+ #define SQ_SEND_HDR_FLAGS_INLINE 0x10UL
+ #define SQ_SEND_HDR_FLAGS_WQE_TS_EN 0x20UL
+ #define SQ_SEND_HDR_FLAGS_DEBUG_TRACE 0x40UL
+ u8 wqe_size;
+ u8 reserved8_1;
+ __le32 inv_key_or_imm_data;
+ __le32 length;
+ __le32 q_key;
+ __le32 dst_qp;
+ #define SQ_SEND_HDR_DST_QP_MASK 0xffffffUL
+ #define SQ_SEND_HDR_DST_QP_SFT 0
+ __le32 avid;
+ #define SQ_SEND_HDR_AVID_MASK 0xfffffUL
+ #define SQ_SEND_HDR_AVID_SFT 0
+ __le32 reserved32;
+ __le32 timestamp;
+ #define SQ_SEND_HDR_TIMESTAMP_MASK 0xffffffUL
+ #define SQ_SEND_HDR_TIMESTAMP_SFT 0
+};
+
+/* sq_send_raweth_qp1 (size:1024b/128B) */
+struct sq_send_raweth_qp1 {
+ u8 wqe_type;
+ #define SQ_SEND_RAWETH_QP1_WQE_TYPE_SEND 0x0UL
+ #define SQ_SEND_RAWETH_QP1_WQE_TYPE_LAST SQ_SEND_RAWETH_QP1_WQE_TYPE_SEND
+ u8 flags;
+ #define SQ_SEND_RAWETH_QP1_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK \
+ 0xffUL
+ #define SQ_SEND_RAWETH_QP1_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT \
+ 0
+ #define SQ_SEND_RAWETH_QP1_FLAGS_SIGNAL_COMP 0x1UL
+ #define SQ_SEND_RAWETH_QP1_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
+ #define SQ_SEND_RAWETH_QP1_FLAGS_UC_FENCE 0x4UL
+ #define SQ_SEND_RAWETH_QP1_FLAGS_SE 0x8UL
+ #define SQ_SEND_RAWETH_QP1_FLAGS_INLINE 0x10UL
+ #define SQ_SEND_RAWETH_QP1_FLAGS_WQE_TS_EN 0x20UL
+ #define SQ_SEND_RAWETH_QP1_FLAGS_DEBUG_TRACE 0x40UL
+ u8 wqe_size;
+ u8 reserved8;
+ __le16 lflags;
+ #define SQ_SEND_RAWETH_QP1_LFLAGS_TCP_UDP_CHKSUM 0x1UL
+ #define SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM 0x2UL
+ #define SQ_SEND_RAWETH_QP1_LFLAGS_NOCRC 0x4UL
+ #define SQ_SEND_RAWETH_QP1_LFLAGS_STAMP 0x8UL
+ #define SQ_SEND_RAWETH_QP1_LFLAGS_T_IP_CHKSUM 0x10UL
+ #define SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC 0x100UL
+ #define SQ_SEND_RAWETH_QP1_LFLAGS_FCOE_CRC 0x200UL
+ __le16 cfa_action;
+ __le32 length;
+ __le32 reserved32_1;
+ __le32 cfa_meta;
+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK 0xfffUL
+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT 0
+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_DE 0x1000UL
+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_MASK 0xe000UL
+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_SFT 13
+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_MASK 0x70000UL
+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_SFT 16
+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID88A8 (0x0UL << 16)
+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID8100 (0x1UL << 16)
+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9100 (0x2UL << 16)
+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9200 (0x3UL << 16)
+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9300 (0x4UL << 16)
+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG (0x5UL << 16)
+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_LAST\
+ SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG
+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_MASK 0xff80000UL
+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_SFT 19
+ #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_MASK 0xf0000000UL
+ #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_SFT 28
+ #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_NONE (0x0UL << 28)
+ #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG (0x1UL << 28)
+ #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_LAST SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG
+ __le32 reserved32_2;
+ __le32 reserved32_3;
+ __le32 timestamp;
+ #define SQ_SEND_RAWETH_QP1_TIMESTAMP_MASK 0xffffffUL
+ #define SQ_SEND_RAWETH_QP1_TIMESTAMP_SFT 0
+ __le32 data[24];
+};
+
+/* sq_send_raweth_qp1_hdr (size:256b/32B) */
+struct sq_send_raweth_qp1_hdr {
+ u8 wqe_type;
+ #define SQ_SEND_RAWETH_QP1_HDR_WQE_TYPE_SEND 0x0UL
+ #define SQ_SEND_RAWETH_QP1_HDR_WQE_TYPE_LAST SQ_SEND_RAWETH_QP1_HDR_WQE_TYPE_SEND
+ u8 flags;
+ #define \
+ SQ_SEND_RAWETH_QP1_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK 0xffUL
+ #define SQ_SEND_RAWETH_QP1_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT\
+ 0
+ #define SQ_SEND_RAWETH_QP1_HDR_FLAGS_SIGNAL_COMP 0x1UL
+ #define SQ_SEND_RAWETH_QP1_HDR_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
+ #define SQ_SEND_RAWETH_QP1_HDR_FLAGS_UC_FENCE 0x4UL
+ #define SQ_SEND_RAWETH_QP1_HDR_FLAGS_SE 0x8UL
+ #define SQ_SEND_RAWETH_QP1_HDR_FLAGS_INLINE 0x10UL
+ #define SQ_SEND_RAWETH_QP1_HDR_FLAGS_WQE_TS_EN 0x20UL
+ #define SQ_SEND_RAWETH_QP1_HDR_FLAGS_DEBUG_TRACE 0x40UL
+ u8 wqe_size;
+ u8 reserved8;
+ __le16 lflags;
+ #define SQ_SEND_RAWETH_QP1_HDR_LFLAGS_TCP_UDP_CHKSUM 0x1UL
+ #define SQ_SEND_RAWETH_QP1_HDR_LFLAGS_IP_CHKSUM 0x2UL
+ #define SQ_SEND_RAWETH_QP1_HDR_LFLAGS_NOCRC 0x4UL
+ #define SQ_SEND_RAWETH_QP1_HDR_LFLAGS_STAMP 0x8UL
+ #define SQ_SEND_RAWETH_QP1_HDR_LFLAGS_T_IP_CHKSUM 0x10UL
+ #define SQ_SEND_RAWETH_QP1_HDR_LFLAGS_ROCE_CRC 0x100UL
+ #define SQ_SEND_RAWETH_QP1_HDR_LFLAGS_FCOE_CRC 0x200UL
+ __le16 cfa_action;
+ __le32 length;
+ __le32 reserved32_1;
+ __le32 cfa_meta;
+ #define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_VID_MASK 0xfffUL
+ #define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_VID_SFT 0
+ #define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_DE 0x1000UL
+ #define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_PRI_MASK 0xe000UL
+ #define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_PRI_SFT 13
+ #define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_MASK 0x70000UL
+ #define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_SFT 16
+ #define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_TPID88A8 (0x0UL << 16)
+ #define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_TPID8100 (0x1UL << 16)
+ #define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_TPID9100 (0x2UL << 16)
+ #define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_TPID9200 (0x3UL << 16)
+ #define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_TPID9300 (0x4UL << 16)
+ #define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_TPIDCFG (0x5UL << 16)
+ #define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_LAST\
+ SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_TPIDCFG
+ #define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_RESERVED_MASK 0xff80000UL
+ #define SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_RESERVED_SFT 19
+ #define SQ_SEND_RAWETH_QP1_HDR_CFA_META_KEY_MASK 0xf0000000UL
+ #define SQ_SEND_RAWETH_QP1_HDR_CFA_META_KEY_SFT 28
+ #define SQ_SEND_RAWETH_QP1_HDR_CFA_META_KEY_NONE (0x0UL << 28)
+ #define SQ_SEND_RAWETH_QP1_HDR_CFA_META_KEY_VLAN_TAG (0x1UL << 28)
+ #define SQ_SEND_RAWETH_QP1_HDR_CFA_META_KEY_LAST\
+ SQ_SEND_RAWETH_QP1_HDR_CFA_META_KEY_VLAN_TAG
+ __le32 reserved32_2;
+ __le32 reserved32_3;
+ __le32 timestamp;
+ #define SQ_SEND_RAWETH_QP1_HDR_TIMESTAMP_MASK 0xffffffUL
+ #define SQ_SEND_RAWETH_QP1_HDR_TIMESTAMP_SFT 0
+};
+
+/* sq_rdma (size:1024b/128B) */
+struct sq_rdma {
+ u8 wqe_type;
+ #define SQ_RDMA_WQE_TYPE_WRITE_WQE 0x4UL
+ #define SQ_RDMA_WQE_TYPE_WRITE_W_IMMEAD 0x5UL
+ #define SQ_RDMA_WQE_TYPE_READ_WQE 0x6UL
+ #define SQ_RDMA_WQE_TYPE_LAST SQ_RDMA_WQE_TYPE_READ_WQE
+ u8 flags;
+ #define SQ_RDMA_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK 0xffUL
+ #define SQ_RDMA_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT 0
+ #define SQ_RDMA_FLAGS_SIGNAL_COMP 0x1UL
+ #define SQ_RDMA_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
+ #define SQ_RDMA_FLAGS_UC_FENCE 0x4UL
+ #define SQ_RDMA_FLAGS_SE 0x8UL
+ #define SQ_RDMA_FLAGS_INLINE 0x10UL
+ #define SQ_RDMA_FLAGS_WQE_TS_EN 0x20UL
+ #define SQ_RDMA_FLAGS_DEBUG_TRACE 0x40UL
+ u8 wqe_size;
+ u8 reserved8;
+ __le32 imm_data;
+ __le32 length;
+ __le32 reserved32_1;
+ __le64 remote_va;
+ __le32 remote_key;
+ __le32 timestamp;
+ #define SQ_RDMA_TIMESTAMP_MASK 0xffffffUL
+ #define SQ_RDMA_TIMESTAMP_SFT 0
+ __le32 data[24];
+};
+
+/* sq_rdma_hdr (size:256b/32B) */
+struct sq_rdma_hdr {
+ u8 wqe_type;
+ #define SQ_RDMA_HDR_WQE_TYPE_WRITE_WQE 0x4UL
+ #define SQ_RDMA_HDR_WQE_TYPE_WRITE_W_IMMEAD 0x5UL
+ #define SQ_RDMA_HDR_WQE_TYPE_READ_WQE 0x6UL
+ #define SQ_RDMA_HDR_WQE_TYPE_LAST SQ_RDMA_HDR_WQE_TYPE_READ_WQE
+ u8 flags;
+ #define SQ_RDMA_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK 0xffUL
+ #define SQ_RDMA_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT 0
+ #define SQ_RDMA_HDR_FLAGS_SIGNAL_COMP 0x1UL
+ #define SQ_RDMA_HDR_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
+ #define SQ_RDMA_HDR_FLAGS_UC_FENCE 0x4UL
+ #define SQ_RDMA_HDR_FLAGS_SE 0x8UL
+ #define SQ_RDMA_HDR_FLAGS_INLINE 0x10UL
+ #define SQ_RDMA_HDR_FLAGS_WQE_TS_EN 0x20UL
+ #define SQ_RDMA_HDR_FLAGS_DEBUG_TRACE 0x40UL
+ u8 wqe_size;
+ u8 reserved8;
+ __le32 imm_data;
+ __le32 length;
+ __le32 reserved32_1;
+ __le64 remote_va;
+ __le32 remote_key;
+ __le32 timestamp;
+ #define SQ_RDMA_HDR_TIMESTAMP_MASK 0xffffffUL
+ #define SQ_RDMA_HDR_TIMESTAMP_SFT 0
+};
+
+/* sq_atomic (size:1024b/128B) */
+struct sq_atomic {
+ u8 wqe_type;
+ #define SQ_ATOMIC_WQE_TYPE_ATOMIC_CS 0x8UL
+ #define SQ_ATOMIC_WQE_TYPE_ATOMIC_FA 0xbUL
+ #define SQ_ATOMIC_WQE_TYPE_LAST SQ_ATOMIC_WQE_TYPE_ATOMIC_FA
+ u8 flags;
+ #define SQ_ATOMIC_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK 0xffUL
+ #define SQ_ATOMIC_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT 0
+ #define SQ_ATOMIC_FLAGS_SIGNAL_COMP 0x1UL
+ #define SQ_ATOMIC_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
+ #define SQ_ATOMIC_FLAGS_UC_FENCE 0x4UL
+ #define SQ_ATOMIC_FLAGS_SE 0x8UL
+ #define SQ_ATOMIC_FLAGS_INLINE 0x10UL
+ #define SQ_ATOMIC_FLAGS_WQE_TS_EN 0x20UL
+ #define SQ_ATOMIC_FLAGS_DEBUG_TRACE 0x40UL
+ __le16 reserved16;
+ __le32 remote_key;
+ __le64 remote_va;
+ __le64 swap_data;
+ __le64 cmp_data;
+ __le32 data[24];
+};
+
+/* sq_atomic_hdr (size:256b/32B) */
+struct sq_atomic_hdr {
+ u8 wqe_type;
+ #define SQ_ATOMIC_HDR_WQE_TYPE_ATOMIC_CS 0x8UL
+ #define SQ_ATOMIC_HDR_WQE_TYPE_ATOMIC_FA 0xbUL
+ #define SQ_ATOMIC_HDR_WQE_TYPE_LAST SQ_ATOMIC_HDR_WQE_TYPE_ATOMIC_FA
+ u8 flags;
+ #define SQ_ATOMIC_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK 0xffUL
+ #define SQ_ATOMIC_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT 0
+ #define SQ_ATOMIC_HDR_FLAGS_SIGNAL_COMP 0x1UL
+ #define SQ_ATOMIC_HDR_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
+ #define SQ_ATOMIC_HDR_FLAGS_UC_FENCE 0x4UL
+ #define SQ_ATOMIC_HDR_FLAGS_SE 0x8UL
+ #define SQ_ATOMIC_HDR_FLAGS_INLINE 0x10UL
+ #define SQ_ATOMIC_HDR_FLAGS_WQE_TS_EN 0x20UL
+ #define SQ_ATOMIC_HDR_FLAGS_DEBUG_TRACE 0x40UL
+ __le16 reserved16;
+ __le32 remote_key;
+ __le64 remote_va;
+ __le64 swap_data;
+ __le64 cmp_data;
+};
+
+/* sq_localinvalidate (size:1024b/128B) */
+struct sq_localinvalidate {
+ u8 wqe_type;
+ #define SQ_LOCALINVALIDATE_WQE_TYPE_LOCAL_INVALID 0xcUL
+ #define SQ_LOCALINVALIDATE_WQE_TYPE_LAST SQ_LOCALINVALIDATE_WQE_TYPE_LOCAL_INVALID
+ u8 flags;
+ #define SQ_LOCALINVALIDATE_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK\
+ 0xffUL
+ #define SQ_LOCALINVALIDATE_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT\
+ 0
+ #define SQ_LOCALINVALIDATE_FLAGS_SIGNAL_COMP 0x1UL
+ #define SQ_LOCALINVALIDATE_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
+ #define SQ_LOCALINVALIDATE_FLAGS_UC_FENCE 0x4UL
+ #define SQ_LOCALINVALIDATE_FLAGS_SE 0x8UL
+ #define SQ_LOCALINVALIDATE_FLAGS_INLINE 0x10UL
+ #define SQ_LOCALINVALIDATE_FLAGS_WQE_TS_EN 0x20UL
+ #define SQ_LOCALINVALIDATE_FLAGS_DEBUG_TRACE 0x40UL
+ __le16 reserved16;
+ __le32 inv_l_key;
+ __le64 reserved64;
+ u8 reserved128[16];
+ __le32 data[24];
+};
+
+/* sq_localinvalidate_hdr (size:256b/32B) */
+struct sq_localinvalidate_hdr {
+ u8 wqe_type;
+ #define SQ_LOCALINVALIDATE_HDR_WQE_TYPE_LOCAL_INVALID 0xcUL
+ #define SQ_LOCALINVALIDATE_HDR_WQE_TYPE_LAST SQ_LOCALINVALIDATE_HDR_WQE_TYPE_LOCAL_INVALID
+ u8 flags;
+ #define \
+ SQ_LOCALINVALIDATE_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK 0xffUL
+ #define SQ_LOCALINVALIDATE_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT\
+ 0
+ #define SQ_LOCALINVALIDATE_HDR_FLAGS_SIGNAL_COMP 0x1UL
+ #define SQ_LOCALINVALIDATE_HDR_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
+ #define SQ_LOCALINVALIDATE_HDR_FLAGS_UC_FENCE 0x4UL
+ #define SQ_LOCALINVALIDATE_HDR_FLAGS_SE 0x8UL
+ #define SQ_LOCALINVALIDATE_HDR_FLAGS_INLINE 0x10UL
+ #define SQ_LOCALINVALIDATE_HDR_FLAGS_WQE_TS_EN 0x20UL
+ #define SQ_LOCALINVALIDATE_HDR_FLAGS_DEBUG_TRACE 0x40UL
+ __le16 reserved16;
+ __le32 inv_l_key;
+ __le64 reserved64;
+ u8 reserved128[16];
+};
+
+/* sq_fr_pmr (size:1024b/128B) */
+struct sq_fr_pmr {
+ u8 wqe_type;
+ #define SQ_FR_PMR_WQE_TYPE_FR_PMR 0xdUL
+ #define SQ_FR_PMR_WQE_TYPE_LAST SQ_FR_PMR_WQE_TYPE_FR_PMR
+ u8 flags;
+ #define SQ_FR_PMR_FLAGS_SIGNAL_COMP 0x1UL
+ #define SQ_FR_PMR_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
+ #define SQ_FR_PMR_FLAGS_UC_FENCE 0x4UL
+ #define SQ_FR_PMR_FLAGS_SE 0x8UL
+ #define SQ_FR_PMR_FLAGS_INLINE 0x10UL
+ #define SQ_FR_PMR_FLAGS_WQE_TS_EN 0x20UL
+ #define SQ_FR_PMR_FLAGS_DEBUG_TRACE 0x40UL
+ u8 access_cntl;
+ #define SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE 0x1UL
+ #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ 0x2UL
+ #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE 0x4UL
+ #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC 0x8UL
+ #define SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND 0x10UL
+ u8 zero_based_page_size_log;
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_MASK 0x1fUL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_SFT 0
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4K 0x0UL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8K 0x1UL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_16K 0x2UL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_32K 0x3UL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_64K 0x4UL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_128K 0x5UL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_256K 0x6UL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_512K 0x7UL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1M 0x8UL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_2M 0x9UL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4M 0xaUL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8M 0xbUL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_16M 0xcUL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_32M 0xdUL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_64M 0xeUL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_128M 0xfUL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_256M 0x10UL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_512M 0x11UL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1G 0x12UL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_2G 0x13UL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4G 0x14UL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8G 0x15UL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_16G 0x16UL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_32G 0x17UL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_64G 0x18UL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_128G 0x19UL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_256G 0x1aUL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_512G 0x1bUL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1T 0x1cUL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_2T 0x1dUL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4T 0x1eUL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8T 0x1fUL
+ #define SQ_FR_PMR_PAGE_SIZE_LOG_LAST SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8T
+ #define SQ_FR_PMR_ZERO_BASED 0x20UL
+ __le32 l_key;
+ u8 length[5];
+ u8 reserved8_1;
+ u8 reserved8_2;
+ u8 numlevels_pbl_page_size_log;
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK 0x1fUL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT 0
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4K 0x0UL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8K 0x1UL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_16K 0x2UL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_32K 0x3UL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_64K 0x4UL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_128K 0x5UL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_256K 0x6UL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_512K 0x7UL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1M 0x8UL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_2M 0x9UL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4M 0xaUL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8M 0xbUL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_16M 0xcUL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_32M 0xdUL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_64M 0xeUL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_128M 0xfUL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_256M 0x10UL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_512M 0x11UL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1G 0x12UL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_2G 0x13UL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4G 0x14UL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8G 0x15UL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_16G 0x16UL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_32G 0x17UL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_64G 0x18UL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_128G 0x19UL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_256G 0x1aUL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_512G 0x1bUL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1T 0x1cUL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_2T 0x1dUL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4T 0x1eUL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8T 0x1fUL
+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_LAST SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8T
+ #define SQ_FR_PMR_NUMLEVELS_MASK 0xc0UL
+ #define SQ_FR_PMR_NUMLEVELS_SFT 6
+ #define SQ_FR_PMR_NUMLEVELS_PHYSICAL (0x0UL << 6)
+ #define SQ_FR_PMR_NUMLEVELS_LAYER1 (0x1UL << 6)
+ #define SQ_FR_PMR_NUMLEVELS_LAYER2 (0x2UL << 6)
+ #define SQ_FR_PMR_NUMLEVELS_LAST SQ_FR_PMR_NUMLEVELS_LAYER2
+ __le64 pblptr;
+ __le64 va;
+ __le32 data[24];
+};
+
+/* sq_fr_pmr_hdr (size:256b/32B) */
+struct sq_fr_pmr_hdr {
+ u8 wqe_type;
+ #define SQ_FR_PMR_HDR_WQE_TYPE_FR_PMR 0xdUL
+ #define SQ_FR_PMR_HDR_WQE_TYPE_LAST SQ_FR_PMR_HDR_WQE_TYPE_FR_PMR
+ u8 flags;
+ #define SQ_FR_PMR_HDR_FLAGS_SIGNAL_COMP 0x1UL
+ #define SQ_FR_PMR_HDR_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
+ #define SQ_FR_PMR_HDR_FLAGS_UC_FENCE 0x4UL
+ #define SQ_FR_PMR_HDR_FLAGS_SE 0x8UL
+ #define SQ_FR_PMR_HDR_FLAGS_INLINE 0x10UL
+ #define SQ_FR_PMR_HDR_FLAGS_WQE_TS_EN 0x20UL
+ #define SQ_FR_PMR_HDR_FLAGS_DEBUG_TRACE 0x40UL
+ u8 access_cntl;
+ #define SQ_FR_PMR_HDR_ACCESS_CNTL_LOCAL_WRITE 0x1UL
+ #define SQ_FR_PMR_HDR_ACCESS_CNTL_REMOTE_READ 0x2UL
+ #define SQ_FR_PMR_HDR_ACCESS_CNTL_REMOTE_WRITE 0x4UL
+ #define SQ_FR_PMR_HDR_ACCESS_CNTL_REMOTE_ATOMIC 0x8UL
+ #define SQ_FR_PMR_HDR_ACCESS_CNTL_WINDOW_BIND 0x10UL
+ u8 zero_based_page_size_log;
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_MASK 0x1fUL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_SFT 0
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_4K 0x0UL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_8K 0x1UL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_16K 0x2UL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_32K 0x3UL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_64K 0x4UL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_128K 0x5UL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_256K 0x6UL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_512K 0x7UL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_1M 0x8UL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_2M 0x9UL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_4M 0xaUL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_8M 0xbUL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_16M 0xcUL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_32M 0xdUL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_64M 0xeUL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_128M 0xfUL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_256M 0x10UL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_512M 0x11UL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_1G 0x12UL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_2G 0x13UL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_4G 0x14UL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_8G 0x15UL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_16G 0x16UL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_32G 0x17UL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_64G 0x18UL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_128G 0x19UL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_256G 0x1aUL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_512G 0x1bUL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_1T 0x1cUL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_2T 0x1dUL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_4T 0x1eUL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_8T 0x1fUL
+ #define SQ_FR_PMR_HDR_PAGE_SIZE_LOG_LAST SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_8T
+ #define SQ_FR_PMR_HDR_ZERO_BASED 0x20UL
+ __le32 l_key;
+ u8 length[5];
+ u8 reserved8_1;
+ u8 reserved8_2;
+ u8 numlevels_pbl_page_size_log;
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_MASK 0x1fUL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_SFT 0
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_4K 0x0UL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_8K 0x1UL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_16K 0x2UL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_32K 0x3UL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_64K 0x4UL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_128K 0x5UL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_256K 0x6UL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_512K 0x7UL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_1M 0x8UL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_2M 0x9UL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_4M 0xaUL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_8M 0xbUL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_16M 0xcUL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_32M 0xdUL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_64M 0xeUL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_128M 0xfUL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_256M 0x10UL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_512M 0x11UL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_1G 0x12UL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_2G 0x13UL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_4G 0x14UL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_8G 0x15UL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_16G 0x16UL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_32G 0x17UL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_64G 0x18UL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_128G 0x19UL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_256G 0x1aUL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_512G 0x1bUL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_1T 0x1cUL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_2T 0x1dUL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_4T 0x1eUL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_8T 0x1fUL
+ #define SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_LAST SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_8T
+ #define SQ_FR_PMR_HDR_NUMLEVELS_MASK 0xc0UL
+ #define SQ_FR_PMR_HDR_NUMLEVELS_SFT 6
+ #define SQ_FR_PMR_HDR_NUMLEVELS_PHYSICAL (0x0UL << 6)
+ #define SQ_FR_PMR_HDR_NUMLEVELS_LAYER1 (0x1UL << 6)
+ #define SQ_FR_PMR_HDR_NUMLEVELS_LAYER2 (0x2UL << 6)
+ #define SQ_FR_PMR_HDR_NUMLEVELS_LAST SQ_FR_PMR_HDR_NUMLEVELS_LAYER2
+ __le64 pblptr;
+ __le64 va;
+};
+
+/* sq_bind (size:1024b/128B) */
+struct sq_bind {
+ u8 wqe_type;
+ #define SQ_BIND_WQE_TYPE_BIND 0xeUL
+ #define SQ_BIND_WQE_TYPE_LAST SQ_BIND_WQE_TYPE_BIND
+ u8 flags;
+ #define SQ_BIND_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK 0xffUL
+ #define SQ_BIND_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT 0
+ #define SQ_BIND_FLAGS_SIGNAL_COMP 0x1UL
+ #define SQ_BIND_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
+ #define SQ_BIND_FLAGS_UC_FENCE 0x4UL
+ #define SQ_BIND_FLAGS_SE 0x8UL
+ #define SQ_BIND_FLAGS_INLINE 0x10UL
+ #define SQ_BIND_FLAGS_WQE_TS_EN 0x20UL
+ #define SQ_BIND_FLAGS_DEBUG_TRACE 0x40UL
+ u8 access_cntl;
+ #define \
+ SQ_BIND_ACCESS_CNTL_WINDOW_BIND_REMOTE_ATOMIC_REMOTE_WRITE_REMOTE_READ_LOCAL_WRITE_MASK\
+ 0xffUL
+ #define \
+ SQ_BIND_ACCESS_CNTL_WINDOW_BIND_REMOTE_ATOMIC_REMOTE_WRITE_REMOTE_READ_LOCAL_WRITE_SFT 0
+ #define SQ_BIND_ACCESS_CNTL_LOCAL_WRITE 0x1UL
+ #define SQ_BIND_ACCESS_CNTL_REMOTE_READ 0x2UL
+ #define SQ_BIND_ACCESS_CNTL_REMOTE_WRITE 0x4UL
+ #define SQ_BIND_ACCESS_CNTL_REMOTE_ATOMIC 0x8UL
+ #define SQ_BIND_ACCESS_CNTL_WINDOW_BIND 0x10UL
+ u8 reserved8_1;
+ u8 mw_type_zero_based;
+ #define SQ_BIND_ZERO_BASED 0x1UL
+ #define SQ_BIND_MW_TYPE 0x2UL
+ #define SQ_BIND_MW_TYPE_TYPE1 (0x0UL << 1)
+ #define SQ_BIND_MW_TYPE_TYPE2 (0x1UL << 1)
+ #define SQ_BIND_MW_TYPE_LAST SQ_BIND_MW_TYPE_TYPE2
+ u8 reserved8_2;
+ __le16 reserved16;
+ __le32 parent_l_key;
+ __le32 l_key;
+ __le64 va;
+ u8 length[5];
+ u8 reserved24[3];
+ __le32 data[24];
+};
+
+/* sq_bind_hdr (size:256b/32B) */
+struct sq_bind_hdr {
+ u8 wqe_type;
+ #define SQ_BIND_HDR_WQE_TYPE_BIND 0xeUL
+ #define SQ_BIND_HDR_WQE_TYPE_LAST SQ_BIND_HDR_WQE_TYPE_BIND
+ u8 flags;
+ #define SQ_BIND_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK 0xffUL
+ #define SQ_BIND_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT 0
+ #define SQ_BIND_HDR_FLAGS_SIGNAL_COMP 0x1UL
+ #define SQ_BIND_HDR_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL
+ #define SQ_BIND_HDR_FLAGS_UC_FENCE 0x4UL
+ #define SQ_BIND_HDR_FLAGS_SE 0x8UL
+ #define SQ_BIND_HDR_FLAGS_INLINE 0x10UL
+ #define SQ_BIND_HDR_FLAGS_WQE_TS_EN 0x20UL
+ #define SQ_BIND_HDR_FLAGS_DEBUG_TRACE 0x40UL
+ u8 access_cntl;
+ #define \
+ SQ_BIND_HDR_ACCESS_CNTL_WINDOW_BIND_REMOTE_ATOMIC_REMOTE_WRITE_REMOTE_READ_LOCAL_WRITE_MASK\
+ 0xffUL
+ #define \
+ SQ_BIND_HDR_ACCESS_CNTL_WINDOW_BIND_REMOTE_ATOMIC_REMOTE_WRITE_REMOTE_READ_LOCAL_WRITE_SFT \
+ 0
+ #define SQ_BIND_HDR_ACCESS_CNTL_LOCAL_WRITE 0x1UL
+ #define SQ_BIND_HDR_ACCESS_CNTL_REMOTE_READ 0x2UL
+ #define SQ_BIND_HDR_ACCESS_CNTL_REMOTE_WRITE 0x4UL
+ #define SQ_BIND_HDR_ACCESS_CNTL_REMOTE_ATOMIC 0x8UL
+ #define SQ_BIND_HDR_ACCESS_CNTL_WINDOW_BIND 0x10UL
+ u8 reserved8_1;
+ u8 mw_type_zero_based;
+ #define SQ_BIND_HDR_ZERO_BASED 0x1UL
+ #define SQ_BIND_HDR_MW_TYPE 0x2UL
+ #define SQ_BIND_HDR_MW_TYPE_TYPE1 (0x0UL << 1)
+ #define SQ_BIND_HDR_MW_TYPE_TYPE2 (0x1UL << 1)
+ #define SQ_BIND_HDR_MW_TYPE_LAST SQ_BIND_HDR_MW_TYPE_TYPE2
+ u8 reserved8_2;
+ __le16 reserved16;
+ __le32 parent_l_key;
+ __le32 l_key;
+ __le64 va;
+ u8 length[5];
+ u8 reserved24[3];
+};
+
+/* rq_wqe (size:1024b/128B) */
+struct rq_wqe {
+ u8 wqe_type;
+ #define RQ_WQE_WQE_TYPE_RCV 0x80UL
+ #define RQ_WQE_WQE_TYPE_LAST RQ_WQE_WQE_TYPE_RCV
+ u8 flags;
+ u8 wqe_size;
+ u8 reserved8;
+ __le32 reserved32;
+ __le32 wr_id[2];
+ #define RQ_WQE_WR_ID_MASK 0xfffffUL
+ #define RQ_WQE_WR_ID_SFT 0
+ u8 reserved128[16];
+ __le32 data[24];
+};
+
+/* rq_wqe_hdr (size:256b/32B) */
+struct rq_wqe_hdr {
+ u8 wqe_type;
+ #define RQ_WQE_HDR_WQE_TYPE_RCV 0x80UL
+ #define RQ_WQE_HDR_WQE_TYPE_LAST RQ_WQE_HDR_WQE_TYPE_RCV
+ u8 flags;
+ u8 wqe_size;
+ u8 reserved8;
+ __le32 reserved32;
+ __le32 wr_id[2];
+ #define RQ_WQE_HDR_WR_ID_MASK 0xfffffUL
+ #define RQ_WQE_HDR_WR_ID_SFT 0
+ u8 reserved128[16];
+};
+
+/* cq_base (size:256b/32B) */
+struct cq_base {
+ __le64 reserved64_1;
+ __le64 reserved64_2;
+ __le64 reserved64_3;
+ u8 cqe_type_toggle;
+ #define CQ_BASE_TOGGLE 0x1UL
+ #define CQ_BASE_CQE_TYPE_MASK 0x1eUL
+ #define CQ_BASE_CQE_TYPE_SFT 1
+ #define CQ_BASE_CQE_TYPE_REQ (0x0UL << 1)
+ #define CQ_BASE_CQE_TYPE_RES_RC (0x1UL << 1)
+ #define CQ_BASE_CQE_TYPE_RES_UD (0x2UL << 1)
+ #define CQ_BASE_CQE_TYPE_RES_RAWETH_QP1 (0x3UL << 1)
+ #define CQ_BASE_CQE_TYPE_RES_UD_CFA (0x4UL << 1)
+ #define CQ_BASE_CQE_TYPE_NO_OP (0xdUL << 1)
+ #define CQ_BASE_CQE_TYPE_TERMINAL (0xeUL << 1)
+ #define CQ_BASE_CQE_TYPE_CUT_OFF (0xfUL << 1)
+ #define CQ_BASE_CQE_TYPE_LAST CQ_BASE_CQE_TYPE_CUT_OFF
+ u8 status;
+ __le16 reserved16;
+ __le32 reserved32;
+};
+
+/* cq_req (size:256b/32B) */
+struct cq_req {
+ __le64 qp_handle;
+ __le16 sq_cons_idx;
+ __le16 reserved16_1;
+ __le32 reserved32_2;
+ __le64 reserved64;
+ u8 cqe_type_toggle;
+ #define CQ_REQ_TOGGLE 0x1UL
+ #define CQ_REQ_CQE_TYPE_MASK 0x1eUL
+ #define CQ_REQ_CQE_TYPE_SFT 1
+ #define CQ_REQ_CQE_TYPE_REQ (0x0UL << 1)
+ #define CQ_REQ_CQE_TYPE_LAST CQ_REQ_CQE_TYPE_REQ
+ #define CQ_REQ_PUSH 0x20UL
+ u8 status;
+ #define CQ_REQ_STATUS_OK 0x0UL
+ #define CQ_REQ_STATUS_BAD_RESPONSE_ERR 0x1UL
+ #define CQ_REQ_STATUS_LOCAL_LENGTH_ERR 0x2UL
+ #define CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR 0x3UL
+ #define CQ_REQ_STATUS_LOCAL_PROTECTION_ERR 0x4UL
+ #define CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL
+ #define CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR 0x6UL
+ #define CQ_REQ_STATUS_REMOTE_ACCESS_ERR 0x7UL
+ #define CQ_REQ_STATUS_REMOTE_OPERATION_ERR 0x8UL
+ #define CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR 0x9UL
+ #define CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR 0xaUL
+ #define CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR 0xbUL
+ #define CQ_REQ_STATUS_LAST CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR
+ __le16 reserved16_2;
+ __le32 reserved32_1;
+};
+
+/* cq_res_rc (size:256b/32B) */
+struct cq_res_rc {
+ __le32 length;
+ __le32 imm_data_or_inv_r_key;
+ __le64 qp_handle;
+ __le64 mr_handle;
+ u8 cqe_type_toggle;
+ #define CQ_RES_RC_TOGGLE 0x1UL
+ #define CQ_RES_RC_CQE_TYPE_MASK 0x1eUL
+ #define CQ_RES_RC_CQE_TYPE_SFT 1
+ #define CQ_RES_RC_CQE_TYPE_RES_RC (0x1UL << 1)
+ #define CQ_RES_RC_CQE_TYPE_LAST CQ_RES_RC_CQE_TYPE_RES_RC
+ u8 status;
+ #define CQ_RES_RC_STATUS_OK 0x0UL
+ #define CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR 0x1UL
+ #define CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR 0x2UL
+ #define CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR 0x3UL
+ #define CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL
+ #define CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL
+ #define CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR 0x6UL
+ #define CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL
+ #define CQ_RES_RC_STATUS_HW_FLUSH_ERR 0x8UL
+ #define CQ_RES_RC_STATUS_LAST CQ_RES_RC_STATUS_HW_FLUSH_ERR
+ __le16 flags;
+ #define CQ_RES_RC_FLAGS_SRQ 0x1UL
+ #define CQ_RES_RC_FLAGS_SRQ_RQ 0x0UL
+ #define CQ_RES_RC_FLAGS_SRQ_SRQ 0x1UL
+ #define CQ_RES_RC_FLAGS_SRQ_LAST CQ_RES_RC_FLAGS_SRQ_SRQ
+ #define CQ_RES_RC_FLAGS_IMM 0x2UL
+ #define CQ_RES_RC_FLAGS_INV 0x4UL
+ #define CQ_RES_RC_FLAGS_RDMA 0x8UL
+ #define CQ_RES_RC_FLAGS_RDMA_SEND (0x0UL << 3)
+ #define CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE (0x1UL << 3)
+ #define CQ_RES_RC_FLAGS_RDMA_LAST CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE
+ __le32 srq_or_rq_wr_id;
+ #define CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL
+ #define CQ_RES_RC_SRQ_OR_RQ_WR_ID_SFT 0
+};
+
+/* cq_res_ud (size:256b/32B) */
+struct cq_res_ud {
+ __le16 length;
+ #define CQ_RES_UD_LENGTH_MASK 0x3fffUL
+ #define CQ_RES_UD_LENGTH_SFT 0
+ __le16 cfa_metadata;
+ #define CQ_RES_UD_CFA_METADATA_VID_MASK 0xfffUL
+ #define CQ_RES_UD_CFA_METADATA_VID_SFT 0
+ #define CQ_RES_UD_CFA_METADATA_DE 0x1000UL
+ #define CQ_RES_UD_CFA_METADATA_PRI_MASK 0xe000UL
+ #define CQ_RES_UD_CFA_METADATA_PRI_SFT 13
+ __le32 imm_data;
+ __le64 qp_handle;
+ __le16 src_mac[3];
+ __le16 src_qp_low;
+ u8 cqe_type_toggle;
+ #define CQ_RES_UD_TOGGLE 0x1UL
+ #define CQ_RES_UD_CQE_TYPE_MASK 0x1eUL
+ #define CQ_RES_UD_CQE_TYPE_SFT 1
+ #define CQ_RES_UD_CQE_TYPE_RES_UD (0x2UL << 1)
+ #define CQ_RES_UD_CQE_TYPE_LAST CQ_RES_UD_CQE_TYPE_RES_UD
+ u8 status;
+ #define CQ_RES_UD_STATUS_OK 0x0UL
+ #define CQ_RES_UD_STATUS_LOCAL_ACCESS_ERROR 0x1UL
+ #define CQ_RES_UD_STATUS_HW_LOCAL_LENGTH_ERR 0x2UL
+ #define CQ_RES_UD_STATUS_LOCAL_PROTECTION_ERR 0x3UL
+ #define CQ_RES_UD_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL
+ #define CQ_RES_UD_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL
+ #define CQ_RES_UD_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL
+ #define CQ_RES_UD_STATUS_HW_FLUSH_ERR 0x8UL
+ #define CQ_RES_UD_STATUS_LAST CQ_RES_UD_STATUS_HW_FLUSH_ERR
+ __le16 flags;
+ #define CQ_RES_UD_FLAGS_SRQ 0x1UL
+ #define CQ_RES_UD_FLAGS_SRQ_RQ 0x0UL
+ #define CQ_RES_UD_FLAGS_SRQ_SRQ 0x1UL
+ #define CQ_RES_UD_FLAGS_SRQ_LAST CQ_RES_UD_FLAGS_SRQ_SRQ
+ #define CQ_RES_UD_FLAGS_IMM 0x2UL
+ #define CQ_RES_UD_FLAGS_UNUSED_MASK 0xcUL
+ #define CQ_RES_UD_FLAGS_UNUSED_SFT 2
+ #define CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK 0x30UL
+ #define CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT 4
+ #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V1 (0x0UL << 4)
+ #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV4 (0x2UL << 4)
+ #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6 (0x3UL << 4)
+ #define CQ_RES_UD_FLAGS_ROCE_IP_VER_LAST CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6
+ #define CQ_RES_UD_FLAGS_META_FORMAT_MASK 0x3c0UL
+ #define CQ_RES_UD_FLAGS_META_FORMAT_SFT 6
+ #define CQ_RES_UD_FLAGS_META_FORMAT_NONE (0x0UL << 6)
+ #define CQ_RES_UD_FLAGS_META_FORMAT_VLAN (0x1UL << 6)
+ #define CQ_RES_UD_FLAGS_META_FORMAT_TUNNEL_ID (0x2UL << 6)
+ #define CQ_RES_UD_FLAGS_META_FORMAT_CHDR_DATA (0x3UL << 6)
+ #define CQ_RES_UD_FLAGS_META_FORMAT_HDR_OFFSET (0x4UL << 6)
+ #define CQ_RES_UD_FLAGS_META_FORMAT_LAST CQ_RES_UD_FLAGS_META_FORMAT_HDR_OFFSET
+ #define CQ_RES_UD_FLAGS_EXT_META_FORMAT_MASK 0xc00UL
+ #define CQ_RES_UD_FLAGS_EXT_META_FORMAT_SFT 10
+ __le32 src_qp_high_srq_or_rq_wr_id;
+ #define CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL
+ #define CQ_RES_UD_SRQ_OR_RQ_WR_ID_SFT 0
+ #define CQ_RES_UD_SRC_QP_HIGH_MASK 0xff000000UL
+ #define CQ_RES_UD_SRC_QP_HIGH_SFT 24
+};
+
+/* cq_res_ud_v2 (size:256b/32B) */
+struct cq_res_ud_v2 {
+ __le16 length;
+ #define CQ_RES_UD_V2_LENGTH_MASK 0x3fffUL
+ #define CQ_RES_UD_V2_LENGTH_SFT 0
+ __le16 cfa_metadata0;
+ #define CQ_RES_UD_V2_CFA_METADATA0_VID_MASK 0xfffUL
+ #define CQ_RES_UD_V2_CFA_METADATA0_VID_SFT 0
+ #define CQ_RES_UD_V2_CFA_METADATA0_DE 0x1000UL
+ #define CQ_RES_UD_V2_CFA_METADATA0_PRI_MASK 0xe000UL
+ #define CQ_RES_UD_V2_CFA_METADATA0_PRI_SFT 13
+ __le32 imm_data;
+ __le64 qp_handle;
+ __le16 src_mac[3];
+ __le16 src_qp_low;
+ u8 cqe_type_toggle;
+ #define CQ_RES_UD_V2_TOGGLE 0x1UL
+ #define CQ_RES_UD_V2_CQE_TYPE_MASK 0x1eUL
+ #define CQ_RES_UD_V2_CQE_TYPE_SFT 1
+ #define CQ_RES_UD_V2_CQE_TYPE_RES_UD (0x2UL << 1)
+ #define CQ_RES_UD_V2_CQE_TYPE_LAST CQ_RES_UD_V2_CQE_TYPE_RES_UD
+ u8 status;
+ #define CQ_RES_UD_V2_STATUS_OK 0x0UL
+ #define CQ_RES_UD_V2_STATUS_LOCAL_ACCESS_ERROR 0x1UL
+ #define CQ_RES_UD_V2_STATUS_HW_LOCAL_LENGTH_ERR 0x2UL
+ #define CQ_RES_UD_V2_STATUS_LOCAL_PROTECTION_ERR 0x3UL
+ #define CQ_RES_UD_V2_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL
+ #define CQ_RES_UD_V2_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL
+ #define CQ_RES_UD_V2_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL
+ #define CQ_RES_UD_V2_STATUS_HW_FLUSH_ERR 0x8UL
+ #define CQ_RES_UD_V2_STATUS_LAST CQ_RES_UD_V2_STATUS_HW_FLUSH_ERR
+ __le16 flags;
+ #define CQ_RES_UD_V2_FLAGS_SRQ 0x1UL
+ #define CQ_RES_UD_V2_FLAGS_SRQ_RQ 0x0UL
+ #define CQ_RES_UD_V2_FLAGS_SRQ_SRQ 0x1UL
+ #define CQ_RES_UD_V2_FLAGS_SRQ_LAST CQ_RES_UD_V2_FLAGS_SRQ_SRQ
+ #define CQ_RES_UD_V2_FLAGS_IMM 0x2UL
+ #define CQ_RES_UD_V2_FLAGS_UNUSED_MASK 0xcUL
+ #define CQ_RES_UD_V2_FLAGS_UNUSED_SFT 2
+ #define CQ_RES_UD_V2_FLAGS_ROCE_IP_VER_MASK 0x30UL
+ #define CQ_RES_UD_V2_FLAGS_ROCE_IP_VER_SFT 4
+ #define CQ_RES_UD_V2_FLAGS_ROCE_IP_VER_V1 (0x0UL << 4)
+ #define CQ_RES_UD_V2_FLAGS_ROCE_IP_VER_V2IPV4 (0x2UL << 4)
+ #define CQ_RES_UD_V2_FLAGS_ROCE_IP_VER_V2IPV6 (0x3UL << 4)
+ #define CQ_RES_UD_V2_FLAGS_ROCE_IP_VER_LAST CQ_RES_UD_V2_FLAGS_ROCE_IP_VER_V2IPV6
+ #define CQ_RES_UD_V2_FLAGS_META_FORMAT_MASK 0x3c0UL
+ #define CQ_RES_UD_V2_FLAGS_META_FORMAT_SFT 6
+ #define CQ_RES_UD_V2_FLAGS_META_FORMAT_NONE (0x0UL << 6)
+ #define CQ_RES_UD_V2_FLAGS_META_FORMAT_ACT_REC_PTR (0x1UL << 6)
+ #define CQ_RES_UD_V2_FLAGS_META_FORMAT_TUNNEL_ID (0x2UL << 6)
+ #define CQ_RES_UD_V2_FLAGS_META_FORMAT_CHDR_DATA (0x3UL << 6)
+ #define CQ_RES_UD_V2_FLAGS_META_FORMAT_HDR_OFFSET (0x4UL << 6)
+ #define CQ_RES_UD_V2_FLAGS_META_FORMAT_LAST CQ_RES_UD_V2_FLAGS_META_FORMAT_HDR_OFFSET
+ __le32 src_qp_high_srq_or_rq_wr_id;
+ #define CQ_RES_UD_V2_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL
+ #define CQ_RES_UD_V2_SRQ_OR_RQ_WR_ID_SFT 0
+ #define CQ_RES_UD_V2_CFA_METADATA1_MASK 0xf00000UL
+ #define CQ_RES_UD_V2_CFA_METADATA1_SFT 20
+ #define CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_MASK 0x700000UL
+ #define CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_SFT 20
+ #define CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_TPID88A8 (0x0UL << 20)
+ #define CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_TPID8100 (0x1UL << 20)
+ #define CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_TPID9100 (0x2UL << 20)
+ #define CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_TPID9200 (0x3UL << 20)
+ #define CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_TPID9300 (0x4UL << 20)
+ #define CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_TPIDCFG (0x5UL << 20)
+ #define CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_LAST CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_TPIDCFG
+ #define CQ_RES_UD_V2_CFA_METADATA1_VALID 0x800000UL
+ #define CQ_RES_UD_V2_SRC_QP_HIGH_MASK 0xff000000UL
+ #define CQ_RES_UD_V2_SRC_QP_HIGH_SFT 24
+};
+
+/* cq_res_ud_cfa (size:256b/32B) */
+struct cq_res_ud_cfa {
+ __le16 length;
+ #define CQ_RES_UD_CFA_LENGTH_MASK 0x3fffUL
+ #define CQ_RES_UD_CFA_LENGTH_SFT 0
+ __le16 cfa_code;
+ __le32 imm_data;
+ __le32 qid;
+ #define CQ_RES_UD_CFA_QID_MASK 0xfffffUL
+ #define CQ_RES_UD_CFA_QID_SFT 0
+ __le32 cfa_metadata;
+ #define CQ_RES_UD_CFA_CFA_METADATA_VID_MASK 0xfffUL
+ #define CQ_RES_UD_CFA_CFA_METADATA_VID_SFT 0
+ #define CQ_RES_UD_CFA_CFA_METADATA_DE 0x1000UL
+ #define CQ_RES_UD_CFA_CFA_METADATA_PRI_MASK 0xe000UL
+ #define CQ_RES_UD_CFA_CFA_METADATA_PRI_SFT 13
+ #define CQ_RES_UD_CFA_CFA_METADATA_TPID_MASK 0xffff0000UL
+ #define CQ_RES_UD_CFA_CFA_METADATA_TPID_SFT 16
+ __le16 src_mac[3];
+ __le16 src_qp_low;
+ u8 cqe_type_toggle;
+ #define CQ_RES_UD_CFA_TOGGLE 0x1UL
+ #define CQ_RES_UD_CFA_CQE_TYPE_MASK 0x1eUL
+ #define CQ_RES_UD_CFA_CQE_TYPE_SFT 1
+ #define CQ_RES_UD_CFA_CQE_TYPE_RES_UD_CFA (0x4UL << 1)
+ #define CQ_RES_UD_CFA_CQE_TYPE_LAST CQ_RES_UD_CFA_CQE_TYPE_RES_UD_CFA
+ u8 status;
+ #define CQ_RES_UD_CFA_STATUS_OK 0x0UL
+ #define CQ_RES_UD_CFA_STATUS_LOCAL_ACCESS_ERROR 0x1UL
+ #define CQ_RES_UD_CFA_STATUS_HW_LOCAL_LENGTH_ERR 0x2UL
+ #define CQ_RES_UD_CFA_STATUS_LOCAL_PROTECTION_ERR 0x3UL
+ #define CQ_RES_UD_CFA_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL
+ #define CQ_RES_UD_CFA_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL
+ #define CQ_RES_UD_CFA_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL
+ #define CQ_RES_UD_CFA_STATUS_HW_FLUSH_ERR 0x8UL
+ #define CQ_RES_UD_CFA_STATUS_LAST CQ_RES_UD_CFA_STATUS_HW_FLUSH_ERR
+ __le16 flags;
+ #define CQ_RES_UD_CFA_FLAGS_SRQ 0x1UL
+ #define CQ_RES_UD_CFA_FLAGS_SRQ_RQ 0x0UL
+ #define CQ_RES_UD_CFA_FLAGS_SRQ_SRQ 0x1UL
+ #define CQ_RES_UD_CFA_FLAGS_SRQ_LAST CQ_RES_UD_CFA_FLAGS_SRQ_SRQ
+ #define CQ_RES_UD_CFA_FLAGS_IMM 0x2UL
+ #define CQ_RES_UD_CFA_FLAGS_UNUSED_MASK 0xcUL
+ #define CQ_RES_UD_CFA_FLAGS_UNUSED_SFT 2
+ #define CQ_RES_UD_CFA_FLAGS_ROCE_IP_VER_MASK 0x30UL
+ #define CQ_RES_UD_CFA_FLAGS_ROCE_IP_VER_SFT 4
+ #define CQ_RES_UD_CFA_FLAGS_ROCE_IP_VER_V1 (0x0UL << 4)
+ #define CQ_RES_UD_CFA_FLAGS_ROCE_IP_VER_V2IPV4 (0x2UL << 4)
+ #define CQ_RES_UD_CFA_FLAGS_ROCE_IP_VER_V2IPV6 (0x3UL << 4)
+ #define CQ_RES_UD_CFA_FLAGS_ROCE_IP_VER_LAST CQ_RES_UD_CFA_FLAGS_ROCE_IP_VER_V2IPV6
+ #define CQ_RES_UD_CFA_FLAGS_META_FORMAT_MASK 0x3c0UL
+ #define CQ_RES_UD_CFA_FLAGS_META_FORMAT_SFT 6
+ #define CQ_RES_UD_CFA_FLAGS_META_FORMAT_NONE (0x0UL << 6)
+ #define CQ_RES_UD_CFA_FLAGS_META_FORMAT_VLAN (0x1UL << 6)
+ #define CQ_RES_UD_CFA_FLAGS_META_FORMAT_TUNNEL_ID (0x2UL << 6)
+ #define CQ_RES_UD_CFA_FLAGS_META_FORMAT_CHDR_DATA (0x3UL << 6)
+ #define CQ_RES_UD_CFA_FLAGS_META_FORMAT_HDR_OFFSET (0x4UL << 6)
+ #define CQ_RES_UD_CFA_FLAGS_META_FORMAT_LAST CQ_RES_UD_CFA_FLAGS_META_FORMAT_HDR_OFFSET
+ #define CQ_RES_UD_CFA_FLAGS_EXT_META_FORMAT_MASK 0xc00UL
+ #define CQ_RES_UD_CFA_FLAGS_EXT_META_FORMAT_SFT 10
+ __le32 src_qp_high_srq_or_rq_wr_id;
+ #define CQ_RES_UD_CFA_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL
+ #define CQ_RES_UD_CFA_SRQ_OR_RQ_WR_ID_SFT 0
+ #define CQ_RES_UD_CFA_SRC_QP_HIGH_MASK 0xff000000UL
+ #define CQ_RES_UD_CFA_SRC_QP_HIGH_SFT 24
+};
+
+/* cq_res_ud_cfa_v2 (size:256b/32B) */
+struct cq_res_ud_cfa_v2 {
+ __le16 length;
+ #define CQ_RES_UD_CFA_V2_LENGTH_MASK 0x3fffUL
+ #define CQ_RES_UD_CFA_V2_LENGTH_SFT 0
+ __le16 cfa_metadata0;
+ #define CQ_RES_UD_CFA_V2_CFA_METADATA0_VID_MASK 0xfffUL
+ #define CQ_RES_UD_CFA_V2_CFA_METADATA0_VID_SFT 0
+ #define CQ_RES_UD_CFA_V2_CFA_METADATA0_DE 0x1000UL
+ #define CQ_RES_UD_CFA_V2_CFA_METADATA0_PRI_MASK 0xe000UL
+ #define CQ_RES_UD_CFA_V2_CFA_METADATA0_PRI_SFT 13
+ __le32 imm_data;
+ __le32 qid;
+ #define CQ_RES_UD_CFA_V2_QID_MASK 0xfffffUL
+ #define CQ_RES_UD_CFA_V2_QID_SFT 0
+ __le32 cfa_metadata2;
+ __le16 src_mac[3];
+ __le16 src_qp_low;
+ u8 cqe_type_toggle;
+ #define CQ_RES_UD_CFA_V2_TOGGLE 0x1UL
+ #define CQ_RES_UD_CFA_V2_CQE_TYPE_MASK 0x1eUL
+ #define CQ_RES_UD_CFA_V2_CQE_TYPE_SFT 1
+ #define CQ_RES_UD_CFA_V2_CQE_TYPE_RES_UD_CFA (0x4UL << 1)
+ #define CQ_RES_UD_CFA_V2_CQE_TYPE_LAST CQ_RES_UD_CFA_V2_CQE_TYPE_RES_UD_CFA
+ u8 status;
+ #define CQ_RES_UD_CFA_V2_STATUS_OK 0x0UL
+ #define CQ_RES_UD_CFA_V2_STATUS_LOCAL_ACCESS_ERROR 0x1UL
+ #define CQ_RES_UD_CFA_V2_STATUS_HW_LOCAL_LENGTH_ERR 0x2UL
+ #define CQ_RES_UD_CFA_V2_STATUS_LOCAL_PROTECTION_ERR 0x3UL
+ #define CQ_RES_UD_CFA_V2_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL
+ #define CQ_RES_UD_CFA_V2_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL
+ #define CQ_RES_UD_CFA_V2_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL
+ #define CQ_RES_UD_CFA_V2_STATUS_HW_FLUSH_ERR 0x8UL
+ #define CQ_RES_UD_CFA_V2_STATUS_LAST CQ_RES_UD_CFA_V2_STATUS_HW_FLUSH_ERR
+ __le16 flags;
+ #define CQ_RES_UD_CFA_V2_FLAGS_SRQ 0x1UL
+ #define CQ_RES_UD_CFA_V2_FLAGS_SRQ_RQ 0x0UL
+ #define CQ_RES_UD_CFA_V2_FLAGS_SRQ_SRQ 0x1UL
+ #define CQ_RES_UD_CFA_V2_FLAGS_SRQ_LAST CQ_RES_UD_CFA_V2_FLAGS_SRQ_SRQ
+ #define CQ_RES_UD_CFA_V2_FLAGS_IMM 0x2UL
+ #define CQ_RES_UD_CFA_V2_FLAGS_UNUSED_MASK 0xcUL
+ #define CQ_RES_UD_CFA_V2_FLAGS_UNUSED_SFT 2
+ #define CQ_RES_UD_CFA_V2_FLAGS_ROCE_IP_VER_MASK 0x30UL
+ #define CQ_RES_UD_CFA_V2_FLAGS_ROCE_IP_VER_SFT 4
+ #define CQ_RES_UD_CFA_V2_FLAGS_ROCE_IP_VER_V1 (0x0UL << 4)
+ #define CQ_RES_UD_CFA_V2_FLAGS_ROCE_IP_VER_V2IPV4 (0x2UL << 4)
+ #define CQ_RES_UD_CFA_V2_FLAGS_ROCE_IP_VER_V2IPV6 (0x3UL << 4)
+ #define CQ_RES_UD_CFA_V2_FLAGS_ROCE_IP_VER_LAST CQ_RES_UD_CFA_V2_FLAGS_ROCE_IP_VER_V2IPV6
+ #define CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_MASK 0x3c0UL
+ #define CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_SFT 6
+ #define CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_NONE (0x0UL << 6)
+ #define CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_ACT_REC_PTR (0x1UL << 6)
+ #define CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_TUNNEL_ID (0x2UL << 6)
+ #define CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_CHDR_DATA (0x3UL << 6)
+ #define CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_HDR_OFFSET (0x4UL << 6)
+ #define CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_LAST \
+ CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_HDR_OFFSET
+ __le32 src_qp_high_srq_or_rq_wr_id;
+ #define CQ_RES_UD_CFA_V2_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL
+ #define CQ_RES_UD_CFA_V2_SRQ_OR_RQ_WR_ID_SFT 0
+ #define CQ_RES_UD_CFA_V2_CFA_METADATA1_MASK 0xf00000UL
+ #define CQ_RES_UD_CFA_V2_CFA_METADATA1_SFT 20
+ #define CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_MASK 0x700000UL
+ #define CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_SFT 20
+ #define CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_TPID88A8 (0x0UL << 20)
+ #define CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_TPID8100 (0x1UL << 20)
+ #define CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_TPID9100 (0x2UL << 20)
+ #define CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_TPID9200 (0x3UL << 20)
+ #define CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_TPID9300 (0x4UL << 20)
+ #define CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_TPIDCFG (0x5UL << 20)
+ #define CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_LAST \
+ CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_TPIDCFG
+ #define CQ_RES_UD_CFA_V2_CFA_METADATA1_VALID 0x800000UL
+ #define CQ_RES_UD_CFA_V2_SRC_QP_HIGH_MASK 0xff000000UL
+ #define CQ_RES_UD_CFA_V2_SRC_QP_HIGH_SFT 24
+};
+
+/* cq_res_raweth_qp1 (size:256b/32B) */
+struct cq_res_raweth_qp1 {
+ __le16 length;
+ #define CQ_RES_RAWETH_QP1_LENGTH_MASK 0x3fffUL
+ #define CQ_RES_RAWETH_QP1_LENGTH_SFT 0
+ __le16 raweth_qp1_flags;
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_MASK 0x3ffUL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_SFT 0
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ERROR 0x1UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_MASK 0x3c0UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_SFT 6
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_NOT_KNOWN (0x0UL << 6)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_IP (0x1UL << 6)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_TCP (0x2UL << 6)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_UDP (0x3UL << 6)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_FCOE (0x4UL << 6)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE (0x5UL << 6)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ICMP (0x7UL << 6)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_WO_TIMESTAMP (0x8UL << 6)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP (0x9UL << 6)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_LAST \
+ CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP
+ __le16 raweth_qp1_errors;
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_IP_CS_ERROR 0x10UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_L4_CS_ERROR 0x20UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_IP_CS_ERROR 0x40UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_L4_CS_ERROR 0x80UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_CRC_ERROR 0x100UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_MASK 0xe00UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_SFT 9
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_NO_ERROR (0x0UL << 9)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_VERSION (0x1UL << 9)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_HDR_LEN (0x2UL << 9)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_TUNNEL_TOTAL_ERROR (0x3UL << 9)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_IP_TOTAL_ERROR (0x4UL << 9)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_UDP_TOTAL_ERROR (0x5UL << 9)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL (0x6UL << 9)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_LAST \
+ CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_MASK 0xf000UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_SFT 12
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_NO_ERROR (0x0UL << 12)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_VERSION (0x1UL << 12)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_HDR_LEN (0x2UL << 12)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_TTL (0x3UL << 12)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_IP_TOTAL_ERROR (0x4UL << 12)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_UDP_TOTAL_ERROR (0x5UL << 12)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN (0x6UL << 12)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN_TOO_SMALL (0x7UL << 12)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN (0x8UL << 12)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_LAST \
+ CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN
+ __le16 raweth_qp1_cfa_code;
+ __le64 qp_handle;
+ __le32 raweth_qp1_flags2;
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC 0x1UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC 0x2UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_IP_CS_CALC 0x4UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_L4_CS_CALC 0x8UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_MASK 0xf0UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_SFT 4
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_NONE (0x0UL << 4)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN (0x1UL << 4)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_TUNNEL_ID (0x2UL << 4)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_CHDR_DATA (0x3UL << 4)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_HDR_OFFSET (0x4UL << 4)
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_LAST \
+ CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_HDR_OFFSET
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE 0x100UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_COMPLETE_CHECKSUM_CALC 0x200UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_EXT_META_FORMAT_MASK 0xc00UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_EXT_META_FORMAT_SFT 10
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_COMPLETE_CHECKSUM_MASK 0xffff0000UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_COMPLETE_CHECKSUM_SFT 16
+ __le32 raweth_qp1_metadata;
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_DE_VID_MASK 0xffffUL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_DE_VID_SFT 0
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK 0xfffUL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_SFT 0
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_DE 0x1000UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK 0xe000UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT 13
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK 0xffff0000UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT 16
+ u8 cqe_type_toggle;
+ #define CQ_RES_RAWETH_QP1_TOGGLE 0x1UL
+ #define CQ_RES_RAWETH_QP1_CQE_TYPE_MASK 0x1eUL
+ #define CQ_RES_RAWETH_QP1_CQE_TYPE_SFT 1
+ #define CQ_RES_RAWETH_QP1_CQE_TYPE_RES_RAWETH_QP1 (0x3UL << 1)
+ #define CQ_RES_RAWETH_QP1_CQE_TYPE_LAST CQ_RES_RAWETH_QP1_CQE_TYPE_RES_RAWETH_QP1
+ u8 status;
+ #define CQ_RES_RAWETH_QP1_STATUS_OK 0x0UL
+ #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR 0x1UL
+ #define CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR 0x2UL
+ #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR 0x3UL
+ #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL
+ #define CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL
+ #define CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL
+ #define CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR 0x8UL
+ #define CQ_RES_RAWETH_QP1_STATUS_LAST CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR
+ __le16 flags;
+ #define CQ_RES_RAWETH_QP1_FLAGS_SRQ 0x1UL
+ #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_RQ 0x0UL
+ #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ 0x1UL
+ #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_LAST CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ
+ __le32 raweth_qp1_payload_offset_srq_or_rq_wr_id;
+ #define CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL
+ #define CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_SFT 0
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_MASK 0xff000000UL
+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_SFT 24
+};
+
+/* cq_res_raweth_qp1_v2 (size:256b/32B) */
+struct cq_res_raweth_qp1_v2 {
+ __le16 length;
+ #define CQ_RES_RAWETH_QP1_V2_LENGTH_MASK 0x3fffUL
+ #define CQ_RES_RAWETH_QP1_V2_LENGTH_SFT 0
+ __le16 raweth_qp1_flags;
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_MASK 0x3ffUL
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_SFT 0
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ERROR 0x1UL
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_MASK 0x3c0UL
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_SFT 6
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_NOT_KNOWN (0x0UL << 6)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_IP (0x1UL << 6)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_TCP (0x2UL << 6)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_UDP (0x3UL << 6)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_FCOE (0x4UL << 6)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_ROCE (0x5UL << 6)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_ICMP (0x7UL << 6)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_PTP_WO_TIMESTAMP (0x8UL << 6)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP (0x9UL << 6)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_LAST \
+ CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP
+ __le16 raweth_qp1_errors;
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_IP_CS_ERROR 0x10UL
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_L4_CS_ERROR 0x20UL
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_IP_CS_ERROR 0x40UL
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_L4_CS_ERROR 0x80UL
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_CRC_ERROR 0x100UL
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_MASK 0xe00UL
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_SFT 9
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_NO_ERROR (0x0UL << 9)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_VERSION (0x1UL << 9)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_HDR_LEN (0x2UL << 9)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_TUNNEL_TOTAL_ERROR (0x3UL << 9)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_IP_TOTAL_ERROR (0x4UL << 9)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_UDP_TOTAL_ERROR (0x5UL << 9)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL (0x6UL << 9)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_LAST \
+ CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_MASK 0xf000UL
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_SFT 12
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_NO_ERROR (0x0UL << 12)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_VERSION (0x1UL << 12)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_HDR_LEN (0x2UL << 12)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_TTL (0x3UL << 12)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_IP_TOTAL_ERROR (0x4UL << 12)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_UDP_TOTAL_ERROR (0x5UL << 12)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN (0x6UL << 12)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN_TOO_SMALL \
+ (0x7UL << 12)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN \
+ (0x8UL << 12)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_LAST \
+ CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN
+ __le16 cfa_metadata0;
+ #define CQ_RES_RAWETH_QP1_V2_CFA_METADATA0_VID_MASK 0xfffUL
+ #define CQ_RES_RAWETH_QP1_V2_CFA_METADATA0_VID_SFT 0
+ #define CQ_RES_RAWETH_QP1_V2_CFA_METADATA0_DE 0x1000UL
+ #define CQ_RES_RAWETH_QP1_V2_CFA_METADATA0_PRI_MASK 0xe000UL
+ #define CQ_RES_RAWETH_QP1_V2_CFA_METADATA0_PRI_SFT 13
+ __le64 qp_handle;
+ __le32 raweth_qp1_flags2;
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_CS_ALL_OK_MODE 0x8UL
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_MASK 0xf0UL
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_SFT 4
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_NONE (0x0UL << 4)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_ACT_REC_PTR (0x1UL << 4)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_TUNNEL_ID (0x2UL << 4)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_CHDR_DATA (0x3UL << 4)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_HDR_OFFSET (0x4UL << 4)
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_LAST \
+ CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_HDR_OFFSET
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_IP_TYPE 0x100UL
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_COMPLETE_CHECKSUM_CALC 0x200UL
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_CS_OK_MASK 0xfc00UL
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_CS_OK_SFT 10
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_COMPLETE_CHECKSUM_MASK 0xffff0000UL
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_COMPLETE_CHECKSUM_SFT 16
+ __le32 cfa_metadata2;
+ u8 cqe_type_toggle;
+ #define CQ_RES_RAWETH_QP1_V2_TOGGLE 0x1UL
+ #define CQ_RES_RAWETH_QP1_V2_CQE_TYPE_MASK 0x1eUL
+ #define CQ_RES_RAWETH_QP1_V2_CQE_TYPE_SFT 1
+ #define CQ_RES_RAWETH_QP1_V2_CQE_TYPE_RES_RAWETH_QP1 (0x3UL << 1)
+ #define CQ_RES_RAWETH_QP1_V2_CQE_TYPE_LAST CQ_RES_RAWETH_QP1_V2_CQE_TYPE_RES_RAWETH_QP1
+ u8 status;
+ #define CQ_RES_RAWETH_QP1_V2_STATUS_OK 0x0UL
+ #define CQ_RES_RAWETH_QP1_V2_STATUS_LOCAL_ACCESS_ERROR 0x1UL
+ #define CQ_RES_RAWETH_QP1_V2_STATUS_HW_LOCAL_LENGTH_ERR 0x2UL
+ #define CQ_RES_RAWETH_QP1_V2_STATUS_LOCAL_PROTECTION_ERR 0x3UL
+ #define CQ_RES_RAWETH_QP1_V2_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL
+ #define CQ_RES_RAWETH_QP1_V2_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL
+ #define CQ_RES_RAWETH_QP1_V2_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL
+ #define CQ_RES_RAWETH_QP1_V2_STATUS_HW_FLUSH_ERR 0x8UL
+ #define CQ_RES_RAWETH_QP1_V2_STATUS_LAST CQ_RES_RAWETH_QP1_V2_STATUS_HW_FLUSH_ERR
+ __le16 flags;
+ #define CQ_RES_RAWETH_QP1_V2_FLAGS_SRQ 0x1UL
+ #define CQ_RES_RAWETH_QP1_V2_FLAGS_SRQ_RQ 0x0UL
+ #define CQ_RES_RAWETH_QP1_V2_FLAGS_SRQ_SRQ 0x1UL
+ #define CQ_RES_RAWETH_QP1_V2_FLAGS_SRQ_LAST CQ_RES_RAWETH_QP1_V2_FLAGS_SRQ_SRQ
+ __le32 raweth_qp1_payload_offset_srq_or_rq_wr_id;
+ #define CQ_RES_RAWETH_QP1_V2_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL
+ #define CQ_RES_RAWETH_QP1_V2_SRQ_OR_RQ_WR_ID_SFT 0
+ #define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_MASK 0xf00000UL
+ #define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_SFT 20
+ #define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_MASK 0x700000UL
+ #define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_SFT 20
+ #define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_TPID88A8 (0x0UL << 20)
+ #define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_TPID8100 (0x1UL << 20)
+ #define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_TPID9100 (0x2UL << 20)
+ #define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_TPID9200 (0x3UL << 20)
+ #define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_TPID9300 (0x4UL << 20)
+ #define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_TPIDCFG (0x5UL << 20)
+ #define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_LAST \
+ CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_TPIDCFG
+ #define CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_VALID 0x800000UL
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_PAYLOAD_OFFSET_MASK 0xff000000UL
+ #define CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_PAYLOAD_OFFSET_SFT 24
+};
+
+/* cq_terminal (size:256b/32B) */
+struct cq_terminal {
+ __le64 qp_handle;
+ __le16 sq_cons_idx;
+ __le16 rq_cons_idx;
+ __le32 reserved32_1;
+ __le64 reserved64_3;
+ u8 cqe_type_toggle;
+ #define CQ_TERMINAL_TOGGLE 0x1UL
+ #define CQ_TERMINAL_CQE_TYPE_MASK 0x1eUL
+ #define CQ_TERMINAL_CQE_TYPE_SFT 1
+ #define CQ_TERMINAL_CQE_TYPE_TERMINAL (0xeUL << 1)
+ #define CQ_TERMINAL_CQE_TYPE_LAST CQ_TERMINAL_CQE_TYPE_TERMINAL
+ u8 status;
+ #define CQ_TERMINAL_STATUS_OK 0x0UL
+ #define CQ_TERMINAL_STATUS_LAST CQ_TERMINAL_STATUS_OK
+ __le16 reserved16;
+ __le32 reserved32_2;
+};
+
+/* cq_cutoff (size:256b/32B) */
+struct cq_cutoff {
+ __le64 reserved64_1;
+ __le64 reserved64_2;
+ __le64 reserved64_3;
+ u8 cqe_type_toggle;
+ #define CQ_CUTOFF_TOGGLE 0x1UL
+ #define CQ_CUTOFF_CQE_TYPE_MASK 0x1eUL
+ #define CQ_CUTOFF_CQE_TYPE_SFT 1
+ #define CQ_CUTOFF_CQE_TYPE_CUT_OFF (0xfUL << 1)
+ #define CQ_CUTOFF_CQE_TYPE_LAST CQ_CUTOFF_CQE_TYPE_CUT_OFF
+ u8 status;
+ #define CQ_CUTOFF_STATUS_OK 0x0UL
+ #define CQ_CUTOFF_STATUS_LAST CQ_CUTOFF_STATUS_OK
+ __le16 reserved16;
+ __le32 reserved32;
+};
+
+/* nq_base (size:128b/16B) */
+struct nq_base {
+ __le16 info10_type;
+ #define NQ_BASE_TYPE_MASK 0x3fUL
+ #define NQ_BASE_TYPE_SFT 0
+ #define NQ_BASE_TYPE_CQ_NOTIFICATION 0x30UL
+ #define NQ_BASE_TYPE_SRQ_EVENT 0x32UL
+ #define NQ_BASE_TYPE_DBQ_EVENT 0x34UL
+ #define NQ_BASE_TYPE_QP_EVENT 0x38UL
+ #define NQ_BASE_TYPE_FUNC_EVENT 0x3aUL
+ #define NQ_BASE_TYPE_LAST NQ_BASE_TYPE_FUNC_EVENT
+ #define NQ_BASE_INFO10_MASK 0xffc0UL
+ #define NQ_BASE_INFO10_SFT 6
+ __le16 info16;
+ __le32 info32;
+ __le32 info63_v[2];
+ #define NQ_BASE_V 0x1UL
+ #define NQ_BASE_INFO63_MASK 0xfffffffeUL
+ #define NQ_BASE_INFO63_SFT 1
+};
+
+/* nq_cn (size:128b/16B) */
+struct nq_cn {
+ __le16 type;
+ #define NQ_CN_TYPE_MASK 0x3fUL
+ #define NQ_CN_TYPE_SFT 0
+ #define NQ_CN_TYPE_CQ_NOTIFICATION 0x30UL
+ #define NQ_CN_TYPE_LAST NQ_CN_TYPE_CQ_NOTIFICATION
+ #define NQ_CN_TOGGLE_MASK 0xc0UL
+ #define NQ_CN_TOGGLE_SFT 6
+ __le16 reserved16;
+ __le32 cq_handle_low;
+ __le32 v;
+ #define NQ_CN_V 0x1UL
+ __le32 cq_handle_high;
+};
+
+/* nq_srq_event (size:128b/16B) */
+struct nq_srq_event {
+ u8 type;
+ #define NQ_SRQ_EVENT_TYPE_MASK 0x3fUL
+ #define NQ_SRQ_EVENT_TYPE_SFT 0
+ #define NQ_SRQ_EVENT_TYPE_SRQ_EVENT 0x32UL
+ #define NQ_SRQ_EVENT_TYPE_LAST NQ_SRQ_EVENT_TYPE_SRQ_EVENT
+ u8 event;
+ #define NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT 0x1UL
+ #define NQ_SRQ_EVENT_EVENT_LAST NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT
+ __le16 reserved16;
+ __le32 srq_handle_low;
+ __le32 v;
+ #define NQ_SRQ_EVENT_V 0x1UL
+ __le32 srq_handle_high;
+};
+
+/* nq_dbq_event (size:128b/16B) */
+struct nq_dbq_event {
+ u8 type;
+ #define NQ_DBQ_EVENT_TYPE_MASK 0x3fUL
+ #define NQ_DBQ_EVENT_TYPE_SFT 0
+ #define NQ_DBQ_EVENT_TYPE_DBQ_EVENT 0x34UL
+ #define NQ_DBQ_EVENT_TYPE_LAST NQ_DBQ_EVENT_TYPE_DBQ_EVENT
+ u8 event;
+ #define NQ_DBQ_EVENT_EVENT_DBQ_THRESHOLD_EVENT 0x1UL
+ #define NQ_DBQ_EVENT_EVENT_LAST NQ_DBQ_EVENT_EVENT_DBQ_THRESHOLD_EVENT
+ __le16 db_pfid;
+ #define NQ_DBQ_EVENT_DB_PFID_MASK 0xfUL
+ #define NQ_DBQ_EVENT_DB_PFID_SFT 0
+ __le32 db_dpi;
+ #define NQ_DBQ_EVENT_DB_DPI_MASK 0xfffffUL
+ #define NQ_DBQ_EVENT_DB_DPI_SFT 0
+ __le32 v;
+ #define NQ_DBQ_EVENT_V 0x1UL
+ __le32 db_type_db_xid;
+ #define NQ_DBQ_EVENT_DB_XID_MASK 0xfffffUL
+ #define NQ_DBQ_EVENT_DB_XID_SFT 0
+ #define NQ_DBQ_EVENT_DB_TYPE_MASK 0xf0000000UL
+ #define NQ_DBQ_EVENT_DB_TYPE_SFT 28
+};
+
+/* xrrq_irrq (size:256b/32B) */
+struct xrrq_irrq {
+ __le16 credits_type;
+ #define XRRQ_IRRQ_TYPE 0x1UL
+ #define XRRQ_IRRQ_TYPE_READ_REQ 0x0UL
+ #define XRRQ_IRRQ_TYPE_ATOMIC_REQ 0x1UL
+ #define XRRQ_IRRQ_TYPE_LAST XRRQ_IRRQ_TYPE_ATOMIC_REQ
+ #define XRRQ_IRRQ_CREDITS_MASK 0xf800UL
+ #define XRRQ_IRRQ_CREDITS_SFT 11
+ __le16 reserved16;
+ __le32 reserved32;
+ __le32 psn;
+ #define XRRQ_IRRQ_PSN_MASK 0xffffffUL
+ #define XRRQ_IRRQ_PSN_SFT 0
+ __le32 msn;
+ #define XRRQ_IRRQ_MSN_MASK 0xffffffUL
+ #define XRRQ_IRRQ_MSN_SFT 0
+ __le64 va_or_atomic_result;
+ __le32 rdma_r_key;
+ __le32 length;
+};
+
+/* xrrq_orrq (size:256b/32B) */
+struct xrrq_orrq {
+ __le16 num_sges_type;
+ #define XRRQ_ORRQ_TYPE 0x1UL
+ #define XRRQ_ORRQ_TYPE_READ_REQ 0x0UL
+ #define XRRQ_ORRQ_TYPE_ATOMIC_REQ 0x1UL
+ #define XRRQ_ORRQ_TYPE_LAST XRRQ_ORRQ_TYPE_ATOMIC_REQ
+ #define XRRQ_ORRQ_NUM_SGES_MASK 0xf800UL
+ #define XRRQ_ORRQ_NUM_SGES_SFT 11
+ __le16 reserved16;
+ __le32 length;
+ __le32 psn;
+ #define XRRQ_ORRQ_PSN_MASK 0xffffffUL
+ #define XRRQ_ORRQ_PSN_SFT 0
+ __le32 end_psn;
+ #define XRRQ_ORRQ_END_PSN_MASK 0xffffffUL
+ #define XRRQ_ORRQ_END_PSN_SFT 0
+ __le64 first_sge_phy_or_sing_sge_va;
+ __le32 single_sge_l_key;
+ __le32 single_sge_size;
+};
+
+/* ptu_pte (size:64b/8B) */
+struct ptu_pte {
+ __le32 page_next_to_last_last_valid[2];
+ #define PTU_PTE_VALID 0x1UL
+ #define PTU_PTE_LAST 0x2UL
+ #define PTU_PTE_NEXT_TO_LAST 0x4UL
+ #define PTU_PTE_UNUSED_MASK 0xff8UL
+ #define PTU_PTE_UNUSED_SFT 3
+ #define PTU_PTE_PAGE_MASK 0xfffff000UL
+ #define PTU_PTE_PAGE_SFT 12
+};
+
+/* ptu_pde (size:64b/8B) */
+struct ptu_pde {
+ __le32 page_valid[2];
+ #define PTU_PDE_VALID 0x1UL
+ #define PTU_PDE_UNUSED_MASK 0xffeUL
+ #define PTU_PDE_UNUSED_SFT 1
+ #define PTU_PDE_PAGE_MASK 0xfffff000UL
+ #define PTU_PDE_PAGE_SFT 12
+};
+
+#endif /* ___BNXT_RE_HSI_H__ */
diff --git a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
index d4b9226088bd..4e93ef7f84ee 100644
--- a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
+++ b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
@@ -1,6 +1,6 @@
/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
/*
- * Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2023 Amazon.com, Inc. or its affiliates. All rights reserved.
*/
#ifndef _EFA_ADMIN_CMDS_H_
@@ -376,7 +376,9 @@ struct efa_admin_reg_mr_cmd {
* 0 : local_write_enable - Local write permissions:
* must be set for RQ buffers and buffers posted for
* RDMA Read requests
- * 1 : reserved1 - MBZ
+ * 1 : remote_write_enable - Remote write
+ * permissions: must be set to enable RDMA write to
+ * the region
* 2 : remote_read_enable - Remote read permissions:
* must be set to enable RDMA read from the region
* 7:3 : reserved2 - MBZ
@@ -618,7 +620,11 @@ struct efa_admin_feature_device_attr_desc {
* TX queues
* 1 : rnr_retry - If set, RNR retry is supported on
* modify QP command
- * 31:2 : reserved - MBZ
+ * 2 : data_polling_128 - If set, 128 bytes data
+ * polling is supported
+ * 3 : rdma_write - If set, RDMA Write is supported
+ * on TX queues
+ * 31:4 : reserved - MBZ
*/
u32 device_caps;
@@ -672,7 +678,7 @@ struct efa_admin_feature_queue_attr_desc {
/* The maximum size of LLQ in bytes */
u32 max_llq_size;
- /* Maximum number of SGEs for a single RDMA read WQE */
+ /* Maximum number of SGEs for a single RDMA read/write WQE */
u16 max_wr_rdma_sges;
/*
@@ -977,6 +983,7 @@ struct efa_admin_host_info {
#define EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT_MASK GENMASK(4, 0)
#define EFA_ADMIN_REG_MR_CMD_MEM_ADDR_PHY_MODE_EN_MASK BIT(7)
#define EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE_MASK BIT(0)
+#define EFA_ADMIN_REG_MR_CMD_REMOTE_WRITE_ENABLE_MASK BIT(1)
#define EFA_ADMIN_REG_MR_CMD_REMOTE_READ_ENABLE_MASK BIT(2)
/* create_cq_cmd */
@@ -991,6 +998,8 @@ struct efa_admin_host_info {
/* feature_device_attr_desc */
#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RDMA_READ_MASK BIT(0)
#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RNR_RETRY_MASK BIT(1)
+#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_DATA_POLLING_128_MASK BIT(2)
+#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RDMA_WRITE_MASK BIT(3)
/* create_eq_cmd */
#define EFA_ADMIN_CREATE_EQ_CMD_ENTRY_SIZE_WORDS_MASK GENMASK(4, 0)
diff --git a/drivers/infiniband/hw/efa/efa_io_defs.h b/drivers/infiniband/hw/efa/efa_io_defs.h
index 17ba8984b11e..2d8eb96eaa81 100644
--- a/drivers/infiniband/hw/efa/efa_io_defs.h
+++ b/drivers/infiniband/hw/efa/efa_io_defs.h
@@ -1,6 +1,6 @@
/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
/*
- * Copyright 2018-2022 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2023 Amazon.com, Inc. or its affiliates. All rights reserved.
*/
#ifndef _EFA_IO_H_
@@ -23,6 +23,8 @@ enum efa_io_send_op_type {
EFA_IO_SEND = 0,
/* RDMA read */
EFA_IO_RDMA_READ = 1,
+ /* RDMA write */
+ EFA_IO_RDMA_WRITE = 2,
};
enum efa_io_comp_status {
@@ -62,8 +64,7 @@ struct efa_io_tx_meta_desc {
/*
* control flags
- * 3:0 : op_type - operation type: send/rdma/fast mem
- * ops/etc
+ * 3:0 : op_type - enum efa_io_send_op_type
* 4 : has_imm - immediate_data field carries valid
* data.
* 5 : inline_msg - inline mode - inline message data
@@ -219,21 +220,22 @@ struct efa_io_cdesc_common {
* 2:1 : q_type - enum efa_io_queue_type: send/recv
* 3 : has_imm - indicates that immediate data is
* present - for RX completions only
- * 7:4 : reserved28 - MBZ
+ * 6:4 : op_type - enum efa_io_send_op_type
+ * 7 : reserved31 - MBZ
*/
u8 flags;
/* local QP number */
u16 qp_num;
-
- /* Transferred length */
- u16 length;
};
/* Tx completion descriptor */
struct efa_io_tx_cdesc {
/* Common completion info */
struct efa_io_cdesc_common common;
+
+ /* MBZ */
+ u16 reserved16;
};
/* Rx Completion Descriptor */
@@ -241,6 +243,9 @@ struct efa_io_rx_cdesc {
/* Common completion info */
struct efa_io_cdesc_common common;
+ /* Transferred length bits[15:0] */
+ u16 length;
+
/* Remote Address Handle FW index, 0xFFFF indicates invalid ah */
u16 ah;
@@ -250,16 +255,26 @@ struct efa_io_rx_cdesc {
u32 imm;
};
+/* Rx Completion Descriptor RDMA write info */
+struct efa_io_rx_cdesc_rdma_write {
+ /* Transferred length bits[31:16] */
+ u16 length_hi;
+};
+
/* Extended Rx Completion Descriptor */
struct efa_io_rx_cdesc_ex {
/* Base RX completion info */
- struct efa_io_rx_cdesc rx_cdesc_base;
+ struct efa_io_rx_cdesc base;
- /*
- * Valid only in case of unknown AH (0xFFFF) and CQ set_src_addr is
- * enabled.
- */
- u8 src_addr[16];
+ union {
+ struct efa_io_rx_cdesc_rdma_write rdma_write;
+
+ /*
+ * Valid only in case of unknown AH (0xFFFF) and CQ
+ * set_src_addr is enabled.
+ */
+ u8 src_addr[16];
+ } u;
};
/* tx_meta_desc */
@@ -285,5 +300,6 @@ struct efa_io_rx_cdesc_ex {
#define EFA_IO_CDESC_COMMON_PHASE_MASK BIT(0)
#define EFA_IO_CDESC_COMMON_Q_TYPE_MASK GENMASK(2, 1)
#define EFA_IO_CDESC_COMMON_HAS_IMM_MASK BIT(3)
+#define EFA_IO_CDESC_COMMON_OP_TYPE_MASK GENMASK(6, 4)
#endif /* _EFA_IO_H_ */
diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c
index 31454643f8c5..8eca6c14d0cf 100644
--- a/drivers/infiniband/hw/efa/efa_verbs.c
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
- * Copyright 2018-2022 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2023 Amazon.com, Inc. or its affiliates. All rights reserved.
*/
#include <linux/dma-buf.h>
@@ -250,6 +250,12 @@ int efa_query_device(struct ib_device *ibdev,
if (EFA_DEV_CAP(dev, RNR_RETRY))
resp.device_caps |= EFA_QUERY_DEVICE_CAPS_RNR_RETRY;
+ if (EFA_DEV_CAP(dev, DATA_POLLING_128))
+ resp.device_caps |= EFA_QUERY_DEVICE_CAPS_DATA_POLLING_128;
+
+ if (EFA_DEV_CAP(dev, RDMA_WRITE))
+ resp.device_caps |= EFA_QUERY_DEVICE_CAPS_RDMA_WRITE;
+
if (dev->neqs)
resp.device_caps |= EFA_QUERY_DEVICE_CAPS_CQ_NOTIFICATIONS;
@@ -1569,7 +1575,8 @@ static struct efa_mr *efa_alloc_mr(struct ib_pd *ibpd, int access_flags,
supp_access_flags =
IB_ACCESS_LOCAL_WRITE |
- (EFA_DEV_CAP(dev, RDMA_READ) ? IB_ACCESS_REMOTE_READ : 0);
+ (EFA_DEV_CAP(dev, RDMA_READ) ? IB_ACCESS_REMOTE_READ : 0) |
+ (EFA_DEV_CAP(dev, RDMA_WRITE) ? IB_ACCESS_REMOTE_WRITE : 0);
access_flags &= ~IB_ACCESS_OPTIONAL;
if (access_flags & ~supp_access_flags) {
diff --git a/drivers/infiniband/hw/erdma/erdma.h b/drivers/infiniband/hw/erdma/erdma.h
index 3d8c11aa23a2..e819e4032490 100644
--- a/drivers/infiniband/hw/erdma/erdma.h
+++ b/drivers/infiniband/hw/erdma/erdma.h
@@ -32,7 +32,7 @@ struct erdma_eq {
atomic64_t event_num;
atomic64_t notify_num;
- u64 __iomem *db_addr;
+ void __iomem *db;
u64 *db_record;
};
diff --git a/drivers/infiniband/hw/erdma/erdma_cm.h b/drivers/infiniband/hw/erdma/erdma_cm.h
index 8a3f998fec9b..a26d80770188 100644
--- a/drivers/infiniband/hw/erdma/erdma_cm.h
+++ b/drivers/infiniband/hw/erdma/erdma_cm.h
@@ -33,11 +33,11 @@ struct mpa_rr_params {
* MPA request/response Hdr bits & fields
*/
enum {
- MPA_RR_FLAG_MARKERS = __cpu_to_be16(0x8000),
- MPA_RR_FLAG_CRC = __cpu_to_be16(0x4000),
- MPA_RR_FLAG_REJECT = __cpu_to_be16(0x2000),
- MPA_RR_RESERVED = __cpu_to_be16(0x1f00),
- MPA_RR_MASK_REVISION = __cpu_to_be16(0x00ff)
+ MPA_RR_FLAG_MARKERS = cpu_to_be16(0x8000),
+ MPA_RR_FLAG_CRC = cpu_to_be16(0x4000),
+ MPA_RR_FLAG_REJECT = cpu_to_be16(0x2000),
+ MPA_RR_RESERVED = cpu_to_be16(0x1f00),
+ MPA_RR_MASK_REVISION = cpu_to_be16(0x00ff)
};
/*
diff --git a/drivers/infiniband/hw/erdma/erdma_cmdq.c b/drivers/infiniband/hw/erdma/erdma_cmdq.c
index 6ebfa6989b11..a151a7bdd504 100644
--- a/drivers/infiniband/hw/erdma/erdma_cmdq.c
+++ b/drivers/infiniband/hw/erdma/erdma_cmdq.c
@@ -166,8 +166,7 @@ static int erdma_cmdq_eq_init(struct erdma_dev *dev)
spin_lock_init(&eq->lock);
atomic64_set(&eq->event_num, 0);
- eq->db_addr =
- (u64 __iomem *)(dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG);
+ eq->db = dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG;
eq->db_record = (u64 *)(eq->qbuf + buf_size);
erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_ADDR_H_REG,
@@ -183,9 +182,8 @@ static int erdma_cmdq_eq_init(struct erdma_dev *dev)
int erdma_cmdq_init(struct erdma_dev *dev)
{
- int err, i;
struct erdma_cmdq *cmdq = &dev->cmdq;
- u32 sts, ctrl;
+ int err;
cmdq->max_outstandings = ERDMA_CMDQ_MAX_OUTSTANDING;
cmdq->use_event = false;
@@ -208,34 +206,10 @@ int erdma_cmdq_init(struct erdma_dev *dev)
if (err)
goto err_destroy_cq;
- ctrl = FIELD_PREP(ERDMA_REG_DEV_CTRL_INIT_MASK, 1);
- erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG, ctrl);
-
- for (i = 0; i < ERDMA_WAIT_DEV_DONE_CNT; i++) {
- sts = erdma_reg_read32_filed(dev, ERDMA_REGS_DEV_ST_REG,
- ERDMA_REG_DEV_ST_INIT_DONE_MASK);
- if (sts)
- break;
-
- msleep(ERDMA_REG_ACCESS_WAIT_MS);
- }
-
- if (i == ERDMA_WAIT_DEV_DONE_CNT) {
- dev_err(&dev->pdev->dev, "wait init done failed.\n");
- err = -ETIMEDOUT;
- goto err_destroy_eq;
- }
-
set_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state);
return 0;
-err_destroy_eq:
- dma_free_coherent(&dev->pdev->dev,
- (cmdq->eq.depth << EQE_SHIFT) +
- ERDMA_EXTRA_BUFFER_SIZE,
- cmdq->eq.qbuf, cmdq->eq.qbuf_dma_addr);
-
err_destroy_cq:
dma_free_coherent(&dev->pdev->dev,
(cmdq->cq.depth << CQE_SHIFT) +
@@ -283,7 +257,7 @@ static void *get_next_valid_cmdq_cqe(struct erdma_cmdq *cmdq)
__be32 *cqe = get_queue_entry(cmdq->cq.qbuf, cmdq->cq.ci,
cmdq->cq.depth, CQE_SHIFT);
u32 owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,
- __be32_to_cpu(READ_ONCE(*cqe)));
+ be32_to_cpu(READ_ONCE(*cqe)));
return owner ^ !!(cmdq->cq.ci & cmdq->cq.depth) ? cqe : NULL;
}
@@ -319,7 +293,6 @@ static int erdma_poll_single_cmd_completion(struct erdma_cmdq *cmdq)
__be32 *cqe;
u16 ctx_id;
u64 *sqe;
- int i;
cqe = get_next_valid_cmdq_cqe(cmdq);
if (!cqe)
@@ -328,8 +301,8 @@ static int erdma_poll_single_cmd_completion(struct erdma_cmdq *cmdq)
cmdq->cq.ci++;
dma_rmb();
- hdr0 = __be32_to_cpu(*cqe);
- sqe_idx = __be32_to_cpu(*(cqe + 1));
+ hdr0 = be32_to_cpu(*cqe);
+ sqe_idx = be32_to_cpu(*(cqe + 1));
sqe = get_queue_entry(cmdq->sq.qbuf, sqe_idx, cmdq->sq.depth,
SQEBB_SHIFT);
@@ -341,9 +314,8 @@ static int erdma_poll_single_cmd_completion(struct erdma_cmdq *cmdq)
comp_wait->cmd_status = ERDMA_CMD_STATUS_FINISHED;
comp_wait->comp_status = FIELD_GET(ERDMA_CQE_HDR_SYNDROME_MASK, hdr0);
cmdq->sq.ci += cmdq->sq.wqebb_cnt;
-
- for (i = 0; i < 4; i++)
- comp_wait->comp_data[i] = __be32_to_cpu(*(cqe + 2 + i));
+ /* Copy 16B comp data after cqe hdr to outer */
+ be32_to_cpu_array(comp_wait->comp_data, cqe + 2, 4);
if (cmdq->use_event)
complete(&comp_wait->wait_event);
diff --git a/drivers/infiniband/hw/erdma/erdma_cq.c b/drivers/infiniband/hw/erdma/erdma_cq.c
index 7bc354273d4e..c1cb5568eab2 100644
--- a/drivers/infiniband/hw/erdma/erdma_cq.c
+++ b/drivers/infiniband/hw/erdma/erdma_cq.c
@@ -11,7 +11,7 @@ static void *get_next_valid_cqe(struct erdma_cq *cq)
__be32 *cqe = get_queue_entry(cq->kern_cq.qbuf, cq->kern_cq.ci,
cq->depth, CQE_SHIFT);
u32 owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,
- __be32_to_cpu(READ_ONCE(*cqe)));
+ be32_to_cpu(READ_ONCE(*cqe)));
return owner ^ !!(cq->kern_cq.ci & cq->depth) ? cqe : NULL;
}
diff --git a/drivers/infiniband/hw/erdma/erdma_eq.c b/drivers/infiniband/hw/erdma/erdma_eq.c
index ed54130d924b..ea47cb21fdb8 100644
--- a/drivers/infiniband/hw/erdma/erdma_eq.c
+++ b/drivers/infiniband/hw/erdma/erdma_eq.c
@@ -14,7 +14,7 @@ void notify_eq(struct erdma_eq *eq)
FIELD_PREP(ERDMA_EQDB_ARM_MASK, 1);
*eq->db_record = db_data;
- writeq(db_data, eq->db_addr);
+ writeq(db_data, eq->db);
atomic64_inc(&eq->notify_num);
}
@@ -98,7 +98,7 @@ int erdma_aeq_init(struct erdma_dev *dev)
atomic64_set(&eq->event_num, 0);
atomic64_set(&eq->notify_num, 0);
- eq->db_addr = (u64 __iomem *)(dev->func_bar + ERDMA_REGS_AEQ_DB_REG);
+ eq->db = dev->func_bar + ERDMA_REGS_AEQ_DB_REG;
eq->db_record = (u64 *)(eq->qbuf + buf_size);
erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_H_REG,
@@ -243,9 +243,8 @@ static int erdma_ceq_init_one(struct erdma_dev *dev, u16 ceqn)
atomic64_set(&eq->notify_num, 0);
eq->depth = ERDMA_DEFAULT_EQ_DEPTH;
- eq->db_addr =
- (u64 __iomem *)(dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG +
- (ceqn + 1) * ERDMA_DB_SIZE);
+ eq->db = dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG +
+ (ceqn + 1) * ERDMA_DB_SIZE;
eq->db_record = (u64 *)(eq->qbuf + buf_size);
eq->ci = 0;
dev->ceqs[ceqn].dev = dev;
diff --git a/drivers/infiniband/hw/erdma/erdma_hw.h b/drivers/infiniband/hw/erdma/erdma_hw.h
index 37ad1bb1917c..76ce2856be28 100644
--- a/drivers/infiniband/hw/erdma/erdma_hw.h
+++ b/drivers/infiniband/hw/erdma/erdma_hw.h
@@ -112,6 +112,10 @@
#define ERDMA_PAGE_SIZE_SUPPORT 0x7FFFF000
+/* Hardware page size definition */
+#define ERDMA_HW_PAGE_SHIFT 12
+#define ERDMA_HW_PAGE_SIZE 4096
+
/* WQE related. */
#define EQE_SIZE 16
#define EQE_SHIFT 4
diff --git a/drivers/infiniband/hw/erdma/erdma_main.c b/drivers/infiniband/hw/erdma/erdma_main.c
index 4a29a53a6652..7c74abeee864 100644
--- a/drivers/infiniband/hw/erdma/erdma_main.c
+++ b/drivers/infiniband/hw/erdma/erdma_main.c
@@ -211,13 +211,36 @@ static int erdma_device_init(struct erdma_dev *dev, struct pci_dev *pdev)
return 0;
}
-static void erdma_device_uninit(struct erdma_dev *dev)
+static void erdma_hw_reset(struct erdma_dev *dev)
{
u32 ctrl = FIELD_PREP(ERDMA_REG_DEV_CTRL_RESET_MASK, 1);
erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG, ctrl);
}
+static int erdma_wait_hw_init_done(struct erdma_dev *dev)
+{
+ int i;
+
+ erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG,
+ FIELD_PREP(ERDMA_REG_DEV_CTRL_INIT_MASK, 1));
+
+ for (i = 0; i < ERDMA_WAIT_DEV_DONE_CNT; i++) {
+ if (erdma_reg_read32_filed(dev, ERDMA_REGS_DEV_ST_REG,
+ ERDMA_REG_DEV_ST_INIT_DONE_MASK))
+ break;
+
+ msleep(ERDMA_REG_ACCESS_WAIT_MS);
+ }
+
+ if (i == ERDMA_WAIT_DEV_DONE_CNT) {
+ dev_err(&dev->pdev->dev, "wait init done failed.\n");
+ return -ETIMEDOUT;
+ }
+
+ return 0;
+}
+
static const struct pci_device_id erdma_pci_tbl[] = {
{ PCI_DEVICE(PCI_VENDOR_ID_ALIBABA, 0x107f) },
{}
@@ -293,16 +316,22 @@ static int erdma_probe_dev(struct pci_dev *pdev)
if (err)
goto err_uninit_aeq;
- err = erdma_ceqs_init(dev);
+ err = erdma_wait_hw_init_done(dev);
if (err)
goto err_uninit_cmdq;
+ err = erdma_ceqs_init(dev);
+ if (err)
+ goto err_reset_hw;
+
erdma_finish_cmdq_init(dev);
return 0;
+err_reset_hw:
+ erdma_hw_reset(dev);
+
err_uninit_cmdq:
- erdma_device_uninit(dev);
erdma_cmdq_destroy(dev);
err_uninit_aeq:
@@ -334,9 +363,7 @@ static void erdma_remove_dev(struct pci_dev *pdev)
struct erdma_dev *dev = pci_get_drvdata(pdev);
erdma_ceqs_uninit(dev);
-
- erdma_device_uninit(dev);
-
+ erdma_hw_reset(dev);
erdma_cmdq_destroy(dev);
erdma_aeq_destroy(dev);
erdma_comm_irq_uninit(dev);
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c
index 9c30d78730aa..83e1b0d55977 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.c
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.c
@@ -38,7 +38,7 @@ static int create_qp_cmd(struct erdma_dev *dev, struct erdma_qp *qp)
FIELD_PREP(ERDMA_CMD_CREATE_QP_PD_MASK, pd->pdn);
if (rdma_is_kernel_res(&qp->ibqp.res)) {
- u32 pgsz_range = ilog2(SZ_1M) - PAGE_SHIFT;
+ u32 pgsz_range = ilog2(SZ_1M) - ERDMA_HW_PAGE_SHIFT;
req.sq_cqn_mtt_cfg =
FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
@@ -66,13 +66,13 @@ static int create_qp_cmd(struct erdma_dev *dev, struct erdma_qp *qp)
user_qp = &qp->user_qp;
req.sq_cqn_mtt_cfg = FIELD_PREP(
ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
- ilog2(user_qp->sq_mtt.page_size) - PAGE_SHIFT);
+ ilog2(user_qp->sq_mtt.page_size) - ERDMA_HW_PAGE_SHIFT);
req.sq_cqn_mtt_cfg |=
FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
req.rq_cqn_mtt_cfg = FIELD_PREP(
ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
- ilog2(user_qp->rq_mtt.page_size) - PAGE_SHIFT);
+ ilog2(user_qp->rq_mtt.page_size) - ERDMA_HW_PAGE_SHIFT);
req.rq_cqn_mtt_cfg |=
FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
@@ -162,7 +162,7 @@ static int create_cq_cmd(struct erdma_dev *dev, struct erdma_cq *cq)
if (rdma_is_kernel_res(&cq->ibcq.res)) {
page_size = SZ_32M;
req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
- ilog2(page_size) - PAGE_SHIFT);
+ ilog2(page_size) - ERDMA_HW_PAGE_SHIFT);
req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr);
req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr);
@@ -175,8 +175,9 @@ static int create_cq_cmd(struct erdma_dev *dev, struct erdma_cq *cq)
cq->kern_cq.qbuf_dma_addr + (cq->depth << CQE_SHIFT);
} else {
mtt = &cq->user_cq.qbuf_mtt;
- req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
- ilog2(mtt->page_size) - PAGE_SHIFT);
+ req.cfg0 |=
+ FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
+ ilog2(mtt->page_size) - ERDMA_HW_PAGE_SHIFT);
if (mtt->mtt_nents == 1) {
req.qbuf_addr_l = lower_32_bits(*(u64 *)mtt->mtt_buf);
req.qbuf_addr_h = upper_32_bits(*(u64 *)mtt->mtt_buf);
@@ -636,7 +637,7 @@ static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
u32 rq_offset;
int ret;
- if (len < (PAGE_ALIGN(qp->attrs.sq_size * SQEBB_SIZE) +
+ if (len < (ALIGN(qp->attrs.sq_size * SQEBB_SIZE, ERDMA_HW_PAGE_SIZE) +
qp->attrs.rq_size * RQE_SIZE))
return -EINVAL;
@@ -646,7 +647,7 @@ static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
if (ret)
return ret;
- rq_offset = PAGE_ALIGN(qp->attrs.sq_size << SQEBB_SHIFT);
+ rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE);
qp->user_qp.rq_offset = rq_offset;
ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mtt, va + rq_offset,
diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c
index 90b672feed83..9dbb89e9f4af 100644
--- a/drivers/infiniband/hw/hfi1/chip.c
+++ b/drivers/infiniband/hw/hfi1/chip.c
@@ -12135,7 +12135,7 @@ void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op,
set_intr_bits(dd, IS_RCVURGENT_START + rcd->ctxt,
IS_RCVURGENT_START + rcd->ctxt, false);
- hfi1_cdbg(RCVCTRL, "ctxt %d rcvctrl 0x%llx\n", ctxt, rcvctrl);
+ hfi1_cdbg(RCVCTRL, "ctxt %d rcvctrl 0x%llx", ctxt, rcvctrl);
write_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL, rcvctrl);
/* work around sticky RcvCtxtStatus.BlockedRHQFull */
@@ -12205,10 +12205,10 @@ u32 hfi1_read_cntrs(struct hfi1_devdata *dd, char **namep, u64 **cntrp)
hfi1_cdbg(CNTR, "reading %s", entry->name);
if (entry->flags & CNTR_DISABLED) {
/* Nothing */
- hfi1_cdbg(CNTR, "\tDisabled\n");
+ hfi1_cdbg(CNTR, "\tDisabled");
} else {
if (entry->flags & CNTR_VL) {
- hfi1_cdbg(CNTR, "\tPer VL\n");
+ hfi1_cdbg(CNTR, "\tPer VL");
for (j = 0; j < C_VL_COUNT; j++) {
val = entry->rw_cntr(entry,
dd, j,
@@ -12216,21 +12216,21 @@ u32 hfi1_read_cntrs(struct hfi1_devdata *dd, char **namep, u64 **cntrp)
0);
hfi1_cdbg(
CNTR,
- "\t\tRead 0x%llx for %d\n",
+ "\t\tRead 0x%llx for %d",
val, j);
dd->cntrs[entry->offset + j] =
val;
}
} else if (entry->flags & CNTR_SDMA) {
hfi1_cdbg(CNTR,
- "\t Per SDMA Engine\n");
+ "\t Per SDMA Engine");
for (j = 0; j < chip_sdma_engines(dd);
j++) {
val =
entry->rw_cntr(entry, dd, j,
CNTR_MODE_R, 0);
hfi1_cdbg(CNTR,
- "\t\tRead 0x%llx for %d\n",
+ "\t\tRead 0x%llx for %d",
val, j);
dd->cntrs[entry->offset + j] =
val;
@@ -12271,7 +12271,7 @@ u32 hfi1_read_portcntrs(struct hfi1_pportdata *ppd, char **namep, u64 **cntrp)
hfi1_cdbg(CNTR, "reading %s", entry->name);
if (entry->flags & CNTR_DISABLED) {
/* Nothing */
- hfi1_cdbg(CNTR, "\tDisabled\n");
+ hfi1_cdbg(CNTR, "\tDisabled");
continue;
}
@@ -12513,7 +12513,7 @@ static void do_update_synth_timer(struct work_struct *work)
hfi1_cdbg(
CNTR,
- "[%d] curr tx=0x%llx rx=0x%llx :: last tx=0x%llx rx=0x%llx\n",
+ "[%d] curr tx=0x%llx rx=0x%llx :: last tx=0x%llx rx=0x%llx",
dd->unit, cur_tx, cur_rx, dd->last_tx, dd->last_rx);
if ((cur_tx < dd->last_tx) || (cur_rx < dd->last_rx)) {
@@ -12527,7 +12527,7 @@ static void do_update_synth_timer(struct work_struct *work)
} else {
total_flits = (cur_tx - dd->last_tx) + (cur_rx - dd->last_rx);
hfi1_cdbg(CNTR,
- "[%d] total flits 0x%llx limit 0x%llx\n", dd->unit,
+ "[%d] total flits 0x%llx limit 0x%llx", dd->unit,
total_flits, (u64)CNTR_32BIT_MAX);
if (total_flits >= CNTR_32BIT_MAX) {
hfi1_cdbg(CNTR, "[%d] 32bit limit hit, updating",
diff --git a/drivers/infiniband/hw/hfi1/driver.c b/drivers/infiniband/hw/hfi1/driver.c
index bcc6bc0540f0..f4492fa407e0 100644
--- a/drivers/infiniband/hw/hfi1/driver.c
+++ b/drivers/infiniband/hw/hfi1/driver.c
@@ -1597,7 +1597,7 @@ static int hfi1_setup_bypass_packet(struct hfi1_packet *packet)
return 0;
drop:
- hfi1_cdbg(PKT, "%s: packet dropped\n", __func__);
+ hfi1_cdbg(PKT, "%s: packet dropped", __func__);
ibp->rvp.n_pkt_drops++;
return -EINVAL;
}
diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c
index f3d6ce45c397..a5ab22cedd41 100644
--- a/drivers/infiniband/hw/hfi1/file_ops.c
+++ b/drivers/infiniband/hw/hfi1/file_ops.c
@@ -975,7 +975,7 @@ static int allocate_ctxt(struct hfi1_filedata *fd, struct hfi1_devdata *dd,
ret = -ENOMEM;
goto ctxdata_free;
}
- hfi1_cdbg(PROC, "allocated send context %u(%u)\n", uctxt->sc->sw_index,
+ hfi1_cdbg(PROC, "allocated send context %u(%u)", uctxt->sc->sw_index,
uctxt->sc->hw_context);
ret = sc_enable(uctxt->sc);
if (ret)
diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c
index 62b6c5020039..6de37c5d7d27 100644
--- a/drivers/infiniband/hw/hfi1/init.c
+++ b/drivers/infiniband/hw/hfi1/init.c
@@ -342,7 +342,7 @@ int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa,
INIT_LIST_HEAD(&rcd->flow_queue.queue_head);
INIT_LIST_HEAD(&rcd->rarr_queue.queue_head);
- hfi1_cdbg(PROC, "setting up context %u\n", rcd->ctxt);
+ hfi1_cdbg(PROC, "setting up context %u", rcd->ctxt);
/*
* Calculate the context's RcvArray entry starting point.
@@ -400,7 +400,7 @@ int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa,
rcd->egrbufs.count = MAX_EAGER_ENTRIES;
}
hfi1_cdbg(PROC,
- "ctxt%u: max Eager buffer RcvArray entries: %u\n",
+ "ctxt%u: max Eager buffer RcvArray entries: %u",
rcd->ctxt, rcd->egrbufs.count);
/*
@@ -432,7 +432,7 @@ int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa,
if (rcd->egrbufs.size < hfi1_max_mtu) {
rcd->egrbufs.size = __roundup_pow_of_two(hfi1_max_mtu);
hfi1_cdbg(PROC,
- "ctxt%u: eager bufs size too small. Adjusting to %u\n",
+ "ctxt%u: eager bufs size too small. Adjusting to %u",
rcd->ctxt, rcd->egrbufs.size);
}
rcd->egrbufs.rcvtid_size = HFI1_MAX_EAGER_BUFFER_SIZE;
@@ -1920,7 +1920,7 @@ int hfi1_setup_eagerbufs(struct hfi1_ctxtdata *rcd)
rcd->egrbufs.size = alloced_bytes;
hfi1_cdbg(PROC,
- "ctxt%u: Alloced %u rcv tid entries @ %uKB, total %uKB\n",
+ "ctxt%u: Alloced %u rcv tid entries @ %uKB, total %uKB",
rcd->ctxt, rcd->egrbufs.alloced,
rcd->egrbufs.rcvtid_size / 1024, rcd->egrbufs.size / 1024);
@@ -1943,13 +1943,13 @@ int hfi1_setup_eagerbufs(struct hfi1_ctxtdata *rcd)
rcd->expected_count = MAX_TID_PAIR_ENTRIES * 2;
rcd->expected_base = rcd->eager_base + egrtop;
- hfi1_cdbg(PROC, "ctxt%u: eager:%u, exp:%u, egrbase:%u, expbase:%u\n",
+ hfi1_cdbg(PROC, "ctxt%u: eager:%u, exp:%u, egrbase:%u, expbase:%u",
rcd->ctxt, rcd->egrbufs.alloced, rcd->expected_count,
rcd->eager_base, rcd->expected_base);
if (!hfi1_rcvbuf_validate(rcd->egrbufs.rcvtid_size, PT_EAGER, &order)) {
hfi1_cdbg(PROC,
- "ctxt%u: current Eager buffer size is invalid %u\n",
+ "ctxt%u: current Eager buffer size is invalid %u",
rcd->ctxt, rcd->egrbufs.rcvtid_size);
ret = -EINVAL;
goto bail_rcvegrbuf_phys;
diff --git a/drivers/infiniband/hw/hfi1/ipoib_tx.c b/drivers/infiniband/hw/hfi1/ipoib_tx.c
index 5d9a7b09ca37..8973a081d641 100644
--- a/drivers/infiniband/hw/hfi1/ipoib_tx.c
+++ b/drivers/infiniband/hw/hfi1/ipoib_tx.c
@@ -215,6 +215,7 @@ static int hfi1_ipoib_build_ulp_payload(struct ipoib_txreq *tx,
const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
ret = sdma_txadd_page(dd,
+ NULL,
txreq,
skb_frag_page(frag),
frag->bv_offset,
@@ -737,10 +738,13 @@ int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv)
txq->tx_ring.shift = ilog2(tx_item_size);
txq->tx_ring.avail = hfi1_ipoib_ring_hwat(txq);
tx_ring = &txq->tx_ring;
- for (j = 0; j < tx_ring_size; j++)
+ for (j = 0; j < tx_ring_size; j++) {
hfi1_txreq_from_idx(tx_ring, j)->sdma_hdr =
kzalloc_node(sizeof(*tx->sdma_hdr),
GFP_KERNEL, priv->dd->node);
+ if (!hfi1_txreq_from_idx(tx_ring, j)->sdma_hdr)
+ goto free_txqs;
+ }
netif_napi_add_tx(dev, &txq->napi, hfi1_ipoib_poll_tx_ring);
}
diff --git a/drivers/infiniband/hw/hfi1/mmu_rb.c b/drivers/infiniband/hw/hfi1/mmu_rb.c
index 7333646021bb..1cea8b0c78e0 100644
--- a/drivers/infiniband/hw/hfi1/mmu_rb.c
+++ b/drivers/infiniband/hw/hfi1/mmu_rb.c
@@ -46,12 +46,14 @@ int hfi1_mmu_rb_register(void *ops_arg,
struct mmu_rb_handler **handler)
{
struct mmu_rb_handler *h;
+ void *free_ptr;
int ret;
- h = kzalloc(sizeof(*h), GFP_KERNEL);
- if (!h)
+ free_ptr = kzalloc(sizeof(*h) + cache_line_size() - 1, GFP_KERNEL);
+ if (!free_ptr)
return -ENOMEM;
+ h = PTR_ALIGN(free_ptr, cache_line_size());
h->root = RB_ROOT_CACHED;
h->ops = ops;
h->ops_arg = ops_arg;
@@ -62,10 +64,11 @@ int hfi1_mmu_rb_register(void *ops_arg,
INIT_LIST_HEAD(&h->del_list);
INIT_LIST_HEAD(&h->lru_list);
h->wq = wq;
+ h->free_ptr = free_ptr;
ret = mmu_notifier_register(&h->mn, current->mm);
if (ret) {
- kfree(h);
+ kfree(free_ptr);
return ret;
}
@@ -108,7 +111,7 @@ void hfi1_mmu_rb_unregister(struct mmu_rb_handler *handler)
/* Now the mm may be freed. */
mmdrop(handler->mn.mm);
- kfree(handler);
+ kfree(handler->free_ptr);
}
int hfi1_mmu_rb_insert(struct mmu_rb_handler *handler,
@@ -126,11 +129,11 @@ int hfi1_mmu_rb_insert(struct mmu_rb_handler *handler,
spin_lock_irqsave(&handler->lock, flags);
node = __mmu_rb_search(handler, mnode->addr, mnode->len);
if (node) {
- ret = -EINVAL;
+ ret = -EEXIST;
goto unlock;
}
__mmu_int_rb_insert(mnode, &handler->root);
- list_add(&mnode->list, &handler->lru_list);
+ list_add_tail(&mnode->list, &handler->lru_list);
ret = handler->ops->insert(handler->ops_arg, mnode);
if (ret) {
@@ -144,6 +147,19 @@ unlock:
}
/* Caller must hold handler lock */
+struct mmu_rb_node *hfi1_mmu_rb_get_first(struct mmu_rb_handler *handler,
+ unsigned long addr, unsigned long len)
+{
+ struct mmu_rb_node *node;
+
+ trace_hfi1_mmu_rb_search(addr, len);
+ node = __mmu_int_rb_iter_first(&handler->root, addr, (addr + len) - 1);
+ if (node)
+ list_move_tail(&node->list, &handler->lru_list);
+ return node;
+}
+
+/* Caller must hold handler lock */
static struct mmu_rb_node *__mmu_rb_search(struct mmu_rb_handler *handler,
unsigned long addr,
unsigned long len)
@@ -167,32 +183,6 @@ static struct mmu_rb_node *__mmu_rb_search(struct mmu_rb_handler *handler,
return node;
}
-bool hfi1_mmu_rb_remove_unless_exact(struct mmu_rb_handler *handler,
- unsigned long addr, unsigned long len,
- struct mmu_rb_node **rb_node)
-{
- struct mmu_rb_node *node;
- unsigned long flags;
- bool ret = false;
-
- if (current->mm != handler->mn.mm)
- return ret;
-
- spin_lock_irqsave(&handler->lock, flags);
- node = __mmu_rb_search(handler, addr, len);
- if (node) {
- if (node->addr == addr && node->len == len)
- goto unlock;
- __mmu_int_rb_remove(node, &handler->root);
- list_del(&node->list); /* remove from LRU list */
- ret = true;
- }
-unlock:
- spin_unlock_irqrestore(&handler->lock, flags);
- *rb_node = node;
- return ret;
-}
-
void hfi1_mmu_rb_evict(struct mmu_rb_handler *handler, void *evict_arg)
{
struct mmu_rb_node *rbnode, *ptr;
@@ -206,8 +196,7 @@ void hfi1_mmu_rb_evict(struct mmu_rb_handler *handler, void *evict_arg)
INIT_LIST_HEAD(&del_list);
spin_lock_irqsave(&handler->lock, flags);
- list_for_each_entry_safe_reverse(rbnode, ptr, &handler->lru_list,
- list) {
+ list_for_each_entry_safe(rbnode, ptr, &handler->lru_list, list) {
if (handler->ops->evict(handler->ops_arg, rbnode, evict_arg,
&stop)) {
__mmu_int_rb_remove(rbnode, &handler->root);
@@ -219,36 +208,11 @@ void hfi1_mmu_rb_evict(struct mmu_rb_handler *handler, void *evict_arg)
}
spin_unlock_irqrestore(&handler->lock, flags);
- while (!list_empty(&del_list)) {
- rbnode = list_first_entry(&del_list, struct mmu_rb_node, list);
- list_del(&rbnode->list);
+ list_for_each_entry_safe(rbnode, ptr, &del_list, list) {
handler->ops->remove(handler->ops_arg, rbnode);
}
}
-/*
- * It is up to the caller to ensure that this function does not race with the
- * mmu invalidate notifier which may be calling the users remove callback on
- * 'node'.
- */
-void hfi1_mmu_rb_remove(struct mmu_rb_handler *handler,
- struct mmu_rb_node *node)
-{
- unsigned long flags;
-
- if (current->mm != handler->mn.mm)
- return;
-
- /* Validity of handler and node pointers has been checked by caller. */
- trace_hfi1_mmu_rb_remove(node->addr, node->len);
- spin_lock_irqsave(&handler->lock, flags);
- __mmu_int_rb_remove(node, &handler->root);
- list_del(&node->list); /* remove from LRU list */
- spin_unlock_irqrestore(&handler->lock, flags);
-
- handler->ops->remove(handler->ops_arg, node);
-}
-
static int mmu_notifier_range_start(struct mmu_notifier *mn,
const struct mmu_notifier_range *range)
{
diff --git a/drivers/infiniband/hw/hfi1/mmu_rb.h b/drivers/infiniband/hw/hfi1/mmu_rb.h
index 7417be2b9dc8..c4da064188c9 100644
--- a/drivers/infiniband/hw/hfi1/mmu_rb.h
+++ b/drivers/infiniband/hw/hfi1/mmu_rb.h
@@ -33,15 +33,25 @@ struct mmu_rb_ops {
};
struct mmu_rb_handler {
+ /*
+ * struct mmu_notifier is 56 bytes, and spinlock_t is 4 bytes, so
+ * they fit together in one cache line. mn is relatively rarely
+ * accessed, so co-locating the spinlock with it achieves much of
+ * the cacheline contention reduction of giving the spinlock its own
+ * cacheline without the overhead of doing so.
+ */
struct mmu_notifier mn;
- struct rb_root_cached root;
- void *ops_arg;
spinlock_t lock; /* protect the RB tree */
+
+ /* Begin on a new cachline boundary here */
+ struct rb_root_cached root ____cacheline_aligned_in_smp;
+ void *ops_arg;
struct mmu_rb_ops *ops;
struct list_head lru_list;
struct work_struct del_work;
struct list_head del_list;
struct workqueue_struct *wq;
+ void *free_ptr;
};
int hfi1_mmu_rb_register(void *ops_arg,
@@ -52,10 +62,8 @@ void hfi1_mmu_rb_unregister(struct mmu_rb_handler *handler);
int hfi1_mmu_rb_insert(struct mmu_rb_handler *handler,
struct mmu_rb_node *mnode);
void hfi1_mmu_rb_evict(struct mmu_rb_handler *handler, void *evict_arg);
-void hfi1_mmu_rb_remove(struct mmu_rb_handler *handler,
- struct mmu_rb_node *mnode);
-bool hfi1_mmu_rb_remove_unless_exact(struct mmu_rb_handler *handler,
- unsigned long addr, unsigned long len,
- struct mmu_rb_node **rb_node);
+struct mmu_rb_node *hfi1_mmu_rb_get_first(struct mmu_rb_handler *handler,
+ unsigned long addr,
+ unsigned long len);
#endif /* _HFI1_MMU_RB_H */
diff --git a/drivers/infiniband/hw/hfi1/pcie.c b/drivers/infiniband/hw/hfi1/pcie.c
index a0802332c8cb..08732e1ac966 100644
--- a/drivers/infiniband/hw/hfi1/pcie.c
+++ b/drivers/infiniband/hw/hfi1/pcie.c
@@ -7,7 +7,6 @@
#include <linux/io.h>
#include <linux/delay.h>
#include <linux/vmalloc.h>
-#include <linux/aer.h>
#include <linux/module.h>
#include "hfi.h"
@@ -65,7 +64,6 @@ int hfi1_pcie_init(struct hfi1_devdata *dd)
}
pci_set_master(pdev);
- (void)pci_enable_pcie_error_reporting(pdev);
return 0;
bail:
diff --git a/drivers/infiniband/hw/hfi1/pio.c b/drivers/infiniband/hw/hfi1/pio.c
index 51ae58c02b15..62e7dc9bea7b 100644
--- a/drivers/infiniband/hw/hfi1/pio.c
+++ b/drivers/infiniband/hw/hfi1/pio.c
@@ -820,7 +820,7 @@ struct send_context *sc_alloc(struct hfi1_devdata *dd, int type,
}
hfi1_cdbg(PIO,
- "Send context %u(%u) %s group %u credits %u credit_ctrl 0x%llx threshold %u\n",
+ "Send context %u(%u) %s group %u credits %u credit_ctrl 0x%llx threshold %u",
sw_index,
hw_context,
sc_type_name(type),
diff --git a/drivers/infiniband/hw/hfi1/sdma.c b/drivers/infiniband/hw/hfi1/sdma.c
index 8ed20392e9f0..bb2552dd29c1 100644
--- a/drivers/infiniband/hw/hfi1/sdma.c
+++ b/drivers/infiniband/hw/hfi1/sdma.c
@@ -1593,22 +1593,7 @@ static inline void sdma_unmap_desc(
struct hfi1_devdata *dd,
struct sdma_desc *descp)
{
- switch (sdma_mapping_type(descp)) {
- case SDMA_MAP_SINGLE:
- dma_unmap_single(
- &dd->pcidev->dev,
- sdma_mapping_addr(descp),
- sdma_mapping_len(descp),
- DMA_TO_DEVICE);
- break;
- case SDMA_MAP_PAGE:
- dma_unmap_page(
- &dd->pcidev->dev,
- sdma_mapping_addr(descp),
- sdma_mapping_len(descp),
- DMA_TO_DEVICE);
- break;
- }
+ system_descriptor_complete(dd, descp);
}
/*
@@ -3128,7 +3113,7 @@ int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx,
/* Add descriptor for coalesce buffer */
tx->desc_limit = MAX_DESC;
- return _sdma_txadd_daddr(dd, SDMA_MAP_SINGLE, tx,
+ return _sdma_txadd_daddr(dd, SDMA_MAP_SINGLE, NULL, tx,
addr, tx->tlen);
}
@@ -3167,10 +3152,12 @@ int _pad_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx)
return rval;
}
}
+
/* finish the one just added */
make_tx_sdma_desc(
tx,
SDMA_MAP_NONE,
+ NULL,
dd->sdma_pad_phys,
sizeof(u32) - (tx->packet_len & (sizeof(u32) - 1)));
tx->num_desc++;
diff --git a/drivers/infiniband/hw/hfi1/sdma.h b/drivers/infiniband/hw/hfi1/sdma.h
index b023fc461bd5..95aaec14c6c2 100644
--- a/drivers/infiniband/hw/hfi1/sdma.h
+++ b/drivers/infiniband/hw/hfi1/sdma.h
@@ -594,6 +594,7 @@ static inline dma_addr_t sdma_mapping_addr(struct sdma_desc *d)
static inline void make_tx_sdma_desc(
struct sdma_txreq *tx,
int type,
+ void *pinning_ctx,
dma_addr_t addr,
size_t len)
{
@@ -612,6 +613,7 @@ static inline void make_tx_sdma_desc(
<< SDMA_DESC0_PHY_ADDR_SHIFT) |
(((u64)len & SDMA_DESC0_BYTE_COUNT_MASK)
<< SDMA_DESC0_BYTE_COUNT_SHIFT);
+ desc->pinning_ctx = pinning_ctx;
}
/* helper to extend txreq */
@@ -643,6 +645,7 @@ static inline void _sdma_close_tx(struct hfi1_devdata *dd,
static inline int _sdma_txadd_daddr(
struct hfi1_devdata *dd,
int type,
+ void *pinning_ctx,
struct sdma_txreq *tx,
dma_addr_t addr,
u16 len)
@@ -652,6 +655,7 @@ static inline int _sdma_txadd_daddr(
make_tx_sdma_desc(
tx,
type,
+ pinning_ctx,
addr, len);
WARN_ON(len > tx->tlen);
tx->num_desc++;
@@ -672,6 +676,7 @@ static inline int _sdma_txadd_daddr(
/**
* sdma_txadd_page() - add a page to the sdma_txreq
* @dd: the device to use for mapping
+ * @pinning_ctx: context to be released at descriptor retirement
* @tx: tx request to which the page is added
* @page: page to map
* @offset: offset within the page
@@ -687,6 +692,7 @@ static inline int _sdma_txadd_daddr(
*/
static inline int sdma_txadd_page(
struct hfi1_devdata *dd,
+ void *pinning_ctx,
struct sdma_txreq *tx,
struct page *page,
unsigned long offset,
@@ -714,8 +720,7 @@ static inline int sdma_txadd_page(
return -ENOSPC;
}
- return _sdma_txadd_daddr(
- dd, SDMA_MAP_PAGE, tx, addr, len);
+ return _sdma_txadd_daddr(dd, SDMA_MAP_PAGE, pinning_ctx, tx, addr, len);
}
/**
@@ -749,7 +754,8 @@ static inline int sdma_txadd_daddr(
return rval;
}
- return _sdma_txadd_daddr(dd, SDMA_MAP_NONE, tx, addr, len);
+ return _sdma_txadd_daddr(dd, SDMA_MAP_NONE, NULL, tx,
+ addr, len);
}
/**
@@ -795,8 +801,7 @@ static inline int sdma_txadd_kvaddr(
return -ENOSPC;
}
- return _sdma_txadd_daddr(
- dd, SDMA_MAP_SINGLE, tx, addr, len);
+ return _sdma_txadd_daddr(dd, SDMA_MAP_SINGLE, NULL, tx, addr, len);
}
struct iowait_work;
@@ -1030,4 +1035,5 @@ extern uint mod_num_sdma;
void sdma_update_lmc(struct hfi1_devdata *dd, u64 mask, u32 lid);
+void system_descriptor_complete(struct hfi1_devdata *dd, struct sdma_desc *descp);
#endif
diff --git a/drivers/infiniband/hw/hfi1/sdma_txreq.h b/drivers/infiniband/hw/hfi1/sdma_txreq.h
index e262fb5c5ec6..fad946cb5e0d 100644
--- a/drivers/infiniband/hw/hfi1/sdma_txreq.h
+++ b/drivers/infiniband/hw/hfi1/sdma_txreq.h
@@ -19,6 +19,7 @@
struct sdma_desc {
/* private: don't use directly */
u64 qw[2];
+ void *pinning_ctx;
};
/**
diff --git a/drivers/infiniband/hw/hfi1/trace_dbg.h b/drivers/infiniband/hw/hfi1/trace_dbg.h
index 582b6f68df3d..489395bfb5b3 100644
--- a/drivers/infiniband/hw/hfi1/trace_dbg.h
+++ b/drivers/infiniband/hw/hfi1/trace_dbg.h
@@ -22,6 +22,11 @@
#define MAX_MSG_LEN 512
+#pragma GCC diagnostic push
+#ifndef __clang__
+#pragma GCC diagnostic ignored "-Wsuggest-attribute=format"
+#endif
+
DECLARE_EVENT_CLASS(hfi1_trace_template,
TP_PROTO(const char *function, struct va_format *vaf),
TP_ARGS(function, vaf),
@@ -36,6 +41,8 @@ DECLARE_EVENT_CLASS(hfi1_trace_template,
__get_str(msg))
);
+#pragma GCC diagnostic pop
+
/*
* It may be nice to macroize the __hfi1_trace but the va_* stuff requires an
* actual function to work and can not be in a macro.
diff --git a/drivers/infiniband/hw/hfi1/trace_mmu.h b/drivers/infiniband/hw/hfi1/trace_mmu.h
index 187e9244fe5e..57900ebb7702 100644
--- a/drivers/infiniband/hw/hfi1/trace_mmu.h
+++ b/drivers/infiniband/hw/hfi1/trace_mmu.h
@@ -37,10 +37,6 @@ DEFINE_EVENT(hfi1_mmu_rb_template, hfi1_mmu_rb_search,
TP_PROTO(unsigned long addr, unsigned long len),
TP_ARGS(addr, len));
-DEFINE_EVENT(hfi1_mmu_rb_template, hfi1_mmu_rb_remove,
- TP_PROTO(unsigned long addr, unsigned long len),
- TP_ARGS(addr, len));
-
DEFINE_EVENT(hfi1_mmu_rb_template, hfi1_mmu_mem_invalidate,
TP_PROTO(unsigned long addr, unsigned long len),
TP_ARGS(addr, len));
diff --git a/drivers/infiniband/hw/hfi1/user_sdma.c b/drivers/infiniband/hw/hfi1/user_sdma.c
index a71c5a36ceba..ae58b48afe07 100644
--- a/drivers/infiniband/hw/hfi1/user_sdma.c
+++ b/drivers/infiniband/hw/hfi1/user_sdma.c
@@ -24,7 +24,6 @@
#include "hfi.h"
#include "sdma.h"
-#include "mmu_rb.h"
#include "user_sdma.h"
#include "verbs.h" /* for the headers */
#include "common.h" /* for struct hfi1_tid_info */
@@ -39,11 +38,7 @@ static unsigned initial_pkt_count = 8;
static int user_sdma_send_pkts(struct user_sdma_request *req, u16 maxpkts);
static void user_sdma_txreq_cb(struct sdma_txreq *txreq, int status);
static inline void pq_update(struct hfi1_user_sdma_pkt_q *pq);
-static void user_sdma_free_request(struct user_sdma_request *req, bool unpin);
-static int pin_vector_pages(struct user_sdma_request *req,
- struct user_sdma_iovec *iovec);
-static void unpin_vector_pages(struct mm_struct *mm, struct page **pages,
- unsigned start, unsigned npages);
+static void user_sdma_free_request(struct user_sdma_request *req);
static int check_header_template(struct user_sdma_request *req,
struct hfi1_pkt_header *hdr, u32 lrhlen,
u32 datalen);
@@ -81,6 +76,11 @@ static struct mmu_rb_ops sdma_rb_ops = {
.invalidate = sdma_rb_invalidate
};
+static int add_system_pages_to_sdma_packet(struct user_sdma_request *req,
+ struct user_sdma_txreq *tx,
+ struct user_sdma_iovec *iovec,
+ u32 *pkt_remaining);
+
static int defer_packet_queue(
struct sdma_engine *sde,
struct iowait_work *wait,
@@ -410,6 +410,7 @@ int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,
ret = -EINVAL;
goto free_req;
}
+
/* Copy the header from the user buffer */
ret = copy_from_user(&req->hdr, iovec[idx].iov_base + sizeof(info),
sizeof(req->hdr));
@@ -484,9 +485,8 @@ int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,
memcpy(&req->iovs[i].iov,
iovec + idx++,
sizeof(req->iovs[i].iov));
- ret = pin_vector_pages(req, &req->iovs[i]);
- if (ret) {
- req->data_iovs = i;
+ if (req->iovs[i].iov.iov_len == 0) {
+ ret = -EINVAL;
goto free_req;
}
req->data_len += req->iovs[i].iov.iov_len;
@@ -584,7 +584,7 @@ free_req:
if (req->seqsubmitted)
wait_event(pq->busy.wait_dma,
(req->seqcomp == req->seqsubmitted - 1));
- user_sdma_free_request(req, true);
+ user_sdma_free_request(req);
pq_update(pq);
set_comp_state(pq, cq, info.comp_idx, ERROR, ret);
}
@@ -696,48 +696,6 @@ static int user_sdma_txadd_ahg(struct user_sdma_request *req,
return ret;
}
-static int user_sdma_txadd(struct user_sdma_request *req,
- struct user_sdma_txreq *tx,
- struct user_sdma_iovec *iovec, u32 datalen,
- u32 *queued_ptr, u32 *data_sent_ptr,
- u64 *iov_offset_ptr)
-{
- int ret;
- unsigned int pageidx, len;
- unsigned long base, offset;
- u64 iov_offset = *iov_offset_ptr;
- u32 queued = *queued_ptr, data_sent = *data_sent_ptr;
- struct hfi1_user_sdma_pkt_q *pq = req->pq;
-
- base = (unsigned long)iovec->iov.iov_base;
- offset = offset_in_page(base + iovec->offset + iov_offset);
- pageidx = (((iovec->offset + iov_offset + base) - (base & PAGE_MASK)) >>
- PAGE_SHIFT);
- len = offset + req->info.fragsize > PAGE_SIZE ?
- PAGE_SIZE - offset : req->info.fragsize;
- len = min((datalen - queued), len);
- ret = sdma_txadd_page(pq->dd, &tx->txreq, iovec->pages[pageidx],
- offset, len);
- if (ret) {
- SDMA_DBG(req, "SDMA txreq add page failed %d\n", ret);
- return ret;
- }
- iov_offset += len;
- queued += len;
- data_sent += len;
- if (unlikely(queued < datalen && pageidx == iovec->npages &&
- req->iov_idx < req->data_iovs - 1)) {
- iovec->offset += iov_offset;
- iovec = &req->iovs[++req->iov_idx];
- iov_offset = 0;
- }
-
- *queued_ptr = queued;
- *data_sent_ptr = data_sent;
- *iov_offset_ptr = iov_offset;
- return ret;
-}
-
static int user_sdma_send_pkts(struct user_sdma_request *req, u16 maxpkts)
{
int ret = 0;
@@ -769,8 +727,7 @@ static int user_sdma_send_pkts(struct user_sdma_request *req, u16 maxpkts)
maxpkts = req->info.npkts - req->seqnum;
while (npkts < maxpkts) {
- u32 datalen = 0, queued = 0, data_sent = 0;
- u64 iov_offset = 0;
+ u32 datalen = 0;
/*
* Check whether any of the completions have come back
@@ -863,27 +820,17 @@ static int user_sdma_send_pkts(struct user_sdma_request *req, u16 maxpkts)
goto free_txreq;
}
- /*
- * If the request contains any data vectors, add up to
- * fragsize bytes to the descriptor.
- */
- while (queued < datalen &&
- (req->sent + data_sent) < req->data_len) {
- ret = user_sdma_txadd(req, tx, iovec, datalen,
- &queued, &data_sent, &iov_offset);
- if (ret)
- goto free_txreq;
- }
- /*
- * The txreq was submitted successfully so we can update
- * the counters.
- */
req->koffset += datalen;
if (req_opcode(req->info.ctrl) == EXPECTED)
req->tidoffset += datalen;
- req->sent += data_sent;
- if (req->data_len)
- iovec->offset += iov_offset;
+ req->sent += datalen;
+ while (datalen) {
+ ret = add_system_pages_to_sdma_packet(req, tx, iovec,
+ &datalen);
+ if (ret)
+ goto free_txreq;
+ iovec = &req->iovs[req->iov_idx];
+ }
list_add_tail(&tx->txreq.list, &req->txps);
/*
* It is important to increment this here as it is used to
@@ -920,133 +867,14 @@ free_tx:
static u32 sdma_cache_evict(struct hfi1_user_sdma_pkt_q *pq, u32 npages)
{
struct evict_data evict_data;
+ struct mmu_rb_handler *handler = pq->handler;
evict_data.cleared = 0;
evict_data.target = npages;
- hfi1_mmu_rb_evict(pq->handler, &evict_data);
+ hfi1_mmu_rb_evict(handler, &evict_data);
return evict_data.cleared;
}
-static int pin_sdma_pages(struct user_sdma_request *req,
- struct user_sdma_iovec *iovec,
- struct sdma_mmu_node *node,
- int npages)
-{
- int pinned, cleared;
- struct page **pages;
- struct hfi1_user_sdma_pkt_q *pq = req->pq;
-
- pages = kcalloc(npages, sizeof(*pages), GFP_KERNEL);
- if (!pages)
- return -ENOMEM;
- memcpy(pages, node->pages, node->npages * sizeof(*pages));
-
- npages -= node->npages;
-retry:
- if (!hfi1_can_pin_pages(pq->dd, current->mm,
- atomic_read(&pq->n_locked), npages)) {
- cleared = sdma_cache_evict(pq, npages);
- if (cleared >= npages)
- goto retry;
- }
- pinned = hfi1_acquire_user_pages(current->mm,
- ((unsigned long)iovec->iov.iov_base +
- (node->npages * PAGE_SIZE)), npages, 0,
- pages + node->npages);
- if (pinned < 0) {
- kfree(pages);
- return pinned;
- }
- if (pinned != npages) {
- unpin_vector_pages(current->mm, pages, node->npages, pinned);
- return -EFAULT;
- }
- kfree(node->pages);
- node->rb.len = iovec->iov.iov_len;
- node->pages = pages;
- atomic_add(pinned, &pq->n_locked);
- return pinned;
-}
-
-static void unpin_sdma_pages(struct sdma_mmu_node *node)
-{
- if (node->npages) {
- unpin_vector_pages(mm_from_sdma_node(node), node->pages, 0,
- node->npages);
- atomic_sub(node->npages, &node->pq->n_locked);
- }
-}
-
-static int pin_vector_pages(struct user_sdma_request *req,
- struct user_sdma_iovec *iovec)
-{
- int ret = 0, pinned, npages;
- struct hfi1_user_sdma_pkt_q *pq = req->pq;
- struct sdma_mmu_node *node = NULL;
- struct mmu_rb_node *rb_node;
- struct iovec *iov;
- bool extracted;
-
- extracted =
- hfi1_mmu_rb_remove_unless_exact(pq->handler,
- (unsigned long)
- iovec->iov.iov_base,
- iovec->iov.iov_len, &rb_node);
- if (rb_node) {
- node = container_of(rb_node, struct sdma_mmu_node, rb);
- if (!extracted) {
- atomic_inc(&node->refcount);
- iovec->pages = node->pages;
- iovec->npages = node->npages;
- iovec->node = node;
- return 0;
- }
- }
-
- if (!node) {
- node = kzalloc(sizeof(*node), GFP_KERNEL);
- if (!node)
- return -ENOMEM;
-
- node->rb.addr = (unsigned long)iovec->iov.iov_base;
- node->pq = pq;
- atomic_set(&node->refcount, 0);
- }
-
- iov = &iovec->iov;
- npages = num_user_pages((unsigned long)iov->iov_base, iov->iov_len);
- if (node->npages < npages) {
- pinned = pin_sdma_pages(req, iovec, node, npages);
- if (pinned < 0) {
- ret = pinned;
- goto bail;
- }
- node->npages += pinned;
- npages = node->npages;
- }
- iovec->pages = node->pages;
- iovec->npages = npages;
- iovec->node = node;
-
- ret = hfi1_mmu_rb_insert(req->pq->handler, &node->rb);
- if (ret) {
- iovec->node = NULL;
- goto bail;
- }
- return 0;
-bail:
- unpin_sdma_pages(node);
- kfree(node);
- return ret;
-}
-
-static void unpin_vector_pages(struct mm_struct *mm, struct page **pages,
- unsigned start, unsigned npages)
-{
- hfi1_release_user_pages(mm, pages + start, npages, false);
- kfree(pages);
-}
-
static int check_header_template(struct user_sdma_request *req,
struct hfi1_pkt_header *hdr, u32 lrhlen,
u32 datalen)
@@ -1388,7 +1216,7 @@ static void user_sdma_txreq_cb(struct sdma_txreq *txreq, int status)
if (req->seqcomp != req->info.npkts - 1)
return;
- user_sdma_free_request(req, false);
+ user_sdma_free_request(req);
set_comp_state(pq, cq, req->info.comp_idx, state, status);
pq_update(pq);
}
@@ -1399,10 +1227,8 @@ static inline void pq_update(struct hfi1_user_sdma_pkt_q *pq)
wake_up(&pq->wait);
}
-static void user_sdma_free_request(struct user_sdma_request *req, bool unpin)
+static void user_sdma_free_request(struct user_sdma_request *req)
{
- int i;
-
if (!list_empty(&req->txps)) {
struct sdma_txreq *t, *p;
@@ -1415,21 +1241,6 @@ static void user_sdma_free_request(struct user_sdma_request *req, bool unpin)
}
}
- for (i = 0; i < req->data_iovs; i++) {
- struct sdma_mmu_node *node = req->iovs[i].node;
-
- if (!node)
- continue;
-
- req->iovs[i].node = NULL;
-
- if (unpin)
- hfi1_mmu_rb_remove(req->pq->handler,
- &node->rb);
- else
- atomic_dec(&node->refcount);
- }
-
kfree(req->tids);
clear_bit(req->info.comp_idx, req->pq->req_in_use);
}
@@ -1447,6 +1258,368 @@ static inline void set_comp_state(struct hfi1_user_sdma_pkt_q *pq,
idx, state, ret);
}
+static void unpin_vector_pages(struct mm_struct *mm, struct page **pages,
+ unsigned int start, unsigned int npages)
+{
+ hfi1_release_user_pages(mm, pages + start, npages, false);
+ kfree(pages);
+}
+
+static void free_system_node(struct sdma_mmu_node *node)
+{
+ if (node->npages) {
+ unpin_vector_pages(mm_from_sdma_node(node), node->pages, 0,
+ node->npages);
+ atomic_sub(node->npages, &node->pq->n_locked);
+ }
+ kfree(node);
+}
+
+static inline void acquire_node(struct sdma_mmu_node *node)
+{
+ atomic_inc(&node->refcount);
+ WARN_ON(atomic_read(&node->refcount) < 0);
+}
+
+static inline void release_node(struct mmu_rb_handler *handler,
+ struct sdma_mmu_node *node)
+{
+ atomic_dec(&node->refcount);
+ WARN_ON(atomic_read(&node->refcount) < 0);
+}
+
+static struct sdma_mmu_node *find_system_node(struct mmu_rb_handler *handler,
+ unsigned long start,
+ unsigned long end)
+{
+ struct mmu_rb_node *rb_node;
+ struct sdma_mmu_node *node;
+ unsigned long flags;
+
+ spin_lock_irqsave(&handler->lock, flags);
+ rb_node = hfi1_mmu_rb_get_first(handler, start, (end - start));
+ if (!rb_node) {
+ spin_unlock_irqrestore(&handler->lock, flags);
+ return NULL;
+ }
+ node = container_of(rb_node, struct sdma_mmu_node, rb);
+ acquire_node(node);
+ spin_unlock_irqrestore(&handler->lock, flags);
+
+ return node;
+}
+
+static int pin_system_pages(struct user_sdma_request *req,
+ uintptr_t start_address, size_t length,
+ struct sdma_mmu_node *node, int npages)
+{
+ struct hfi1_user_sdma_pkt_q *pq = req->pq;
+ int pinned, cleared;
+ struct page **pages;
+
+ pages = kcalloc(npages, sizeof(*pages), GFP_KERNEL);
+ if (!pages)
+ return -ENOMEM;
+
+retry:
+ if (!hfi1_can_pin_pages(pq->dd, current->mm, atomic_read(&pq->n_locked),
+ npages)) {
+ SDMA_DBG(req, "Evicting: nlocked %u npages %u",
+ atomic_read(&pq->n_locked), npages);
+ cleared = sdma_cache_evict(pq, npages);
+ if (cleared >= npages)
+ goto retry;
+ }
+
+ SDMA_DBG(req, "Acquire user pages start_address %lx node->npages %u npages %u",
+ start_address, node->npages, npages);
+ pinned = hfi1_acquire_user_pages(current->mm, start_address, npages, 0,
+ pages);
+
+ if (pinned < 0) {
+ kfree(pages);
+ SDMA_DBG(req, "pinned %d", pinned);
+ return pinned;
+ }
+ if (pinned != npages) {
+ unpin_vector_pages(current->mm, pages, node->npages, pinned);
+ SDMA_DBG(req, "npages %u pinned %d", npages, pinned);
+ return -EFAULT;
+ }
+ node->rb.addr = start_address;
+ node->rb.len = length;
+ node->pages = pages;
+ node->npages = npages;
+ atomic_add(pinned, &pq->n_locked);
+ SDMA_DBG(req, "done. pinned %d", pinned);
+ return 0;
+}
+
+static int add_system_pinning(struct user_sdma_request *req,
+ struct sdma_mmu_node **node_p,
+ unsigned long start, unsigned long len)
+
+{
+ struct hfi1_user_sdma_pkt_q *pq = req->pq;
+ struct sdma_mmu_node *node;
+ int ret;
+
+ node = kzalloc(sizeof(*node), GFP_KERNEL);
+ if (!node)
+ return -ENOMEM;
+
+ node->pq = pq;
+ ret = pin_system_pages(req, start, len, node, PFN_DOWN(len));
+ if (ret == 0) {
+ ret = hfi1_mmu_rb_insert(pq->handler, &node->rb);
+ if (ret)
+ free_system_node(node);
+ else
+ *node_p = node;
+
+ return ret;
+ }
+
+ kfree(node);
+ return ret;
+}
+
+static int get_system_cache_entry(struct user_sdma_request *req,
+ struct sdma_mmu_node **node_p,
+ size_t req_start, size_t req_len)
+{
+ struct hfi1_user_sdma_pkt_q *pq = req->pq;
+ u64 start = ALIGN_DOWN(req_start, PAGE_SIZE);
+ u64 end = PFN_ALIGN(req_start + req_len);
+ struct mmu_rb_handler *handler = pq->handler;
+ int ret;
+
+ if ((end - start) == 0) {
+ SDMA_DBG(req,
+ "Request for empty cache entry req_start %lx req_len %lx start %llx end %llx",
+ req_start, req_len, start, end);
+ return -EINVAL;
+ }
+
+ SDMA_DBG(req, "req_start %lx req_len %lu", req_start, req_len);
+
+ while (1) {
+ struct sdma_mmu_node *node =
+ find_system_node(handler, start, end);
+ u64 prepend_len = 0;
+
+ SDMA_DBG(req, "node %p start %llx end %llu", node, start, end);
+ if (!node) {
+ ret = add_system_pinning(req, node_p, start,
+ end - start);
+ if (ret == -EEXIST) {
+ /*
+ * Another execution context has inserted a
+ * conficting entry first.
+ */
+ continue;
+ }
+ return ret;
+ }
+
+ if (node->rb.addr <= start) {
+ /*
+ * This entry covers at least part of the region. If it doesn't extend
+ * to the end, then this will be called again for the next segment.
+ */
+ *node_p = node;
+ return 0;
+ }
+
+ SDMA_DBG(req, "prepend: node->rb.addr %lx, node->refcount %d",
+ node->rb.addr, atomic_read(&node->refcount));
+ prepend_len = node->rb.addr - start;
+
+ /*
+ * This node will not be returned, instead a new node
+ * will be. So release the reference.
+ */
+ release_node(handler, node);
+
+ /* Prepend a node to cover the beginning of the allocation */
+ ret = add_system_pinning(req, node_p, start, prepend_len);
+ if (ret == -EEXIST) {
+ /* Another execution context has inserted a conficting entry first. */
+ continue;
+ }
+ return ret;
+ }
+}
+
+static int add_mapping_to_sdma_packet(struct user_sdma_request *req,
+ struct user_sdma_txreq *tx,
+ struct sdma_mmu_node *cache_entry,
+ size_t start,
+ size_t from_this_cache_entry)
+{
+ struct hfi1_user_sdma_pkt_q *pq = req->pq;
+ unsigned int page_offset;
+ unsigned int from_this_page;
+ size_t page_index;
+ void *ctx;
+ int ret;
+
+ /*
+ * Because the cache may be more fragmented than the memory that is being accessed,
+ * it's not strictly necessary to have a descriptor per cache entry.
+ */
+
+ while (from_this_cache_entry) {
+ page_index = PFN_DOWN(start - cache_entry->rb.addr);
+
+ if (page_index >= cache_entry->npages) {
+ SDMA_DBG(req,
+ "Request for page_index %zu >= cache_entry->npages %u",
+ page_index, cache_entry->npages);
+ return -EINVAL;
+ }
+
+ page_offset = start - ALIGN_DOWN(start, PAGE_SIZE);
+ from_this_page = PAGE_SIZE - page_offset;
+
+ if (from_this_page < from_this_cache_entry) {
+ ctx = NULL;
+ } else {
+ /*
+ * In the case they are equal the next line has no practical effect,
+ * but it's better to do a register to register copy than a conditional
+ * branch.
+ */
+ from_this_page = from_this_cache_entry;
+ ctx = cache_entry;
+ }
+
+ ret = sdma_txadd_page(pq->dd, ctx, &tx->txreq,
+ cache_entry->pages[page_index],
+ page_offset, from_this_page);
+ if (ret) {
+ /*
+ * When there's a failure, the entire request is freed by
+ * user_sdma_send_pkts().
+ */
+ SDMA_DBG(req,
+ "sdma_txadd_page failed %d page_index %lu page_offset %u from_this_page %u",
+ ret, page_index, page_offset, from_this_page);
+ return ret;
+ }
+ start += from_this_page;
+ from_this_cache_entry -= from_this_page;
+ }
+ return 0;
+}
+
+static int add_system_iovec_to_sdma_packet(struct user_sdma_request *req,
+ struct user_sdma_txreq *tx,
+ struct user_sdma_iovec *iovec,
+ size_t from_this_iovec)
+{
+ struct mmu_rb_handler *handler = req->pq->handler;
+
+ while (from_this_iovec > 0) {
+ struct sdma_mmu_node *cache_entry;
+ size_t from_this_cache_entry;
+ size_t start;
+ int ret;
+
+ start = (uintptr_t)iovec->iov.iov_base + iovec->offset;
+ ret = get_system_cache_entry(req, &cache_entry, start,
+ from_this_iovec);
+ if (ret) {
+ SDMA_DBG(req, "pin system segment failed %d", ret);
+ return ret;
+ }
+
+ from_this_cache_entry = cache_entry->rb.len - (start - cache_entry->rb.addr);
+ if (from_this_cache_entry > from_this_iovec)
+ from_this_cache_entry = from_this_iovec;
+
+ ret = add_mapping_to_sdma_packet(req, tx, cache_entry, start,
+ from_this_cache_entry);
+ if (ret) {
+ /*
+ * We're guaranteed that there will be no descriptor
+ * completion callback that releases this node
+ * because only the last descriptor referencing it
+ * has a context attached, and a failure means the
+ * last descriptor was never added.
+ */
+ release_node(handler, cache_entry);
+ SDMA_DBG(req, "add system segment failed %d", ret);
+ return ret;
+ }
+
+ iovec->offset += from_this_cache_entry;
+ from_this_iovec -= from_this_cache_entry;
+ }
+
+ return 0;
+}
+
+static int add_system_pages_to_sdma_packet(struct user_sdma_request *req,
+ struct user_sdma_txreq *tx,
+ struct user_sdma_iovec *iovec,
+ u32 *pkt_data_remaining)
+{
+ size_t remaining_to_add = *pkt_data_remaining;
+ /*
+ * Walk through iovec entries, ensure the associated pages
+ * are pinned and mapped, add data to the packet until no more
+ * data remains to be added.
+ */
+ while (remaining_to_add > 0) {
+ struct user_sdma_iovec *cur_iovec;
+ size_t from_this_iovec;
+ int ret;
+
+ cur_iovec = iovec;
+ from_this_iovec = iovec->iov.iov_len - iovec->offset;
+
+ if (from_this_iovec > remaining_to_add) {
+ from_this_iovec = remaining_to_add;
+ } else {
+ /* The current iovec entry will be consumed by this pass. */
+ req->iov_idx++;
+ iovec++;
+ }
+
+ ret = add_system_iovec_to_sdma_packet(req, tx, cur_iovec,
+ from_this_iovec);
+ if (ret)
+ return ret;
+
+ remaining_to_add -= from_this_iovec;
+ }
+ *pkt_data_remaining = remaining_to_add;
+
+ return 0;
+}
+
+void system_descriptor_complete(struct hfi1_devdata *dd,
+ struct sdma_desc *descp)
+{
+ switch (sdma_mapping_type(descp)) {
+ case SDMA_MAP_SINGLE:
+ dma_unmap_single(&dd->pcidev->dev, sdma_mapping_addr(descp),
+ sdma_mapping_len(descp), DMA_TO_DEVICE);
+ break;
+ case SDMA_MAP_PAGE:
+ dma_unmap_page(&dd->pcidev->dev, sdma_mapping_addr(descp),
+ sdma_mapping_len(descp), DMA_TO_DEVICE);
+ break;
+ }
+
+ if (descp->pinning_ctx) {
+ struct sdma_mmu_node *node = descp->pinning_ctx;
+
+ release_node(node->rb.handler, node);
+ }
+}
+
static bool sdma_rb_filter(struct mmu_rb_node *node, unsigned long addr,
unsigned long len)
{
@@ -1493,8 +1666,7 @@ static void sdma_rb_remove(void *arg, struct mmu_rb_node *mnode)
struct sdma_mmu_node *node =
container_of(mnode, struct sdma_mmu_node, rb);
- unpin_sdma_pages(node);
- kfree(node);
+ free_system_node(node);
}
static int sdma_rb_invalidate(void *arg, struct mmu_rb_node *mnode)
diff --git a/drivers/infiniband/hw/hfi1/user_sdma.h b/drivers/infiniband/hw/hfi1/user_sdma.h
index ea56eb57e656..a241836371dc 100644
--- a/drivers/infiniband/hw/hfi1/user_sdma.h
+++ b/drivers/infiniband/hw/hfi1/user_sdma.h
@@ -112,16 +112,11 @@ struct sdma_mmu_node {
struct user_sdma_iovec {
struct list_head list;
struct iovec iov;
- /* number of pages in this vector */
- unsigned int npages;
- /* array of pinned pages for this vector */
- struct page **pages;
/*
* offset into the virtual address space of the vector at
* which we last left off.
*/
u64 offset;
- struct sdma_mmu_node *node;
};
/* evict operation argument */
diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c
index 7f6d7fc7951d..fbdcfecb1768 100644
--- a/drivers/infiniband/hw/hfi1/verbs.c
+++ b/drivers/infiniband/hw/hfi1/verbs.c
@@ -778,8 +778,8 @@ static int build_verbs_tx_desc(
/* add icrc, lt byte, and padding to flit */
if (extra_bytes)
- ret = sdma_txadd_daddr(sde->dd, &tx->txreq,
- sde->dd->sdma_pad_phys, extra_bytes);
+ ret = sdma_txadd_daddr(sde->dd, &tx->txreq, sde->dd->sdma_pad_phys,
+ extra_bytes);
bail_txadd:
return ret;
diff --git a/drivers/infiniband/hw/hfi1/vnic_sdma.c b/drivers/infiniband/hw/hfi1/vnic_sdma.c
index c3f0f8d877c3..727eedfba332 100644
--- a/drivers/infiniband/hw/hfi1/vnic_sdma.c
+++ b/drivers/infiniband/hw/hfi1/vnic_sdma.c
@@ -64,6 +64,7 @@ static noinline int build_vnic_ulp_payload(struct sdma_engine *sde,
/* combine physically continuous fragments later? */
ret = sdma_txadd_page(sde->dd,
+ NULL,
&tx->txreq,
skb_frag_page(frag),
skb_frag_off(frag),
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
index dbf97fe5948f..84f1167de1d9 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
@@ -1960,100 +1960,6 @@ static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev)
return hns_roce_cmq_send(hr_dev, &desc, 1);
}
-/* Use default caps when hns_roce_query_pf_caps() failed or init VF profile */
-static void set_default_caps(struct hns_roce_dev *hr_dev)
-{
- struct hns_roce_caps *caps = &hr_dev->caps;
-
- caps->num_qps = HNS_ROCE_V2_MAX_QP_NUM;
- caps->max_wqes = HNS_ROCE_V2_MAX_WQE_NUM;
- caps->num_cqs = HNS_ROCE_V2_MAX_CQ_NUM;
- caps->num_srqs = HNS_ROCE_V2_MAX_SRQ_NUM;
- caps->min_cqes = HNS_ROCE_MIN_CQE_NUM;
- caps->max_cqes = HNS_ROCE_V2_MAX_CQE_NUM;
- caps->max_sq_sg = HNS_ROCE_V2_MAX_SQ_SGE_NUM;
- caps->max_rq_sg = HNS_ROCE_V2_MAX_RQ_SGE_NUM;
-
- caps->num_uars = HNS_ROCE_V2_UAR_NUM;
- caps->phy_num_uars = HNS_ROCE_V2_PHY_UAR_NUM;
- caps->num_aeq_vectors = HNS_ROCE_V2_AEQE_VEC_NUM;
- caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM;
- caps->num_comp_vectors = 0;
-
- caps->num_mtpts = HNS_ROCE_V2_MAX_MTPT_NUM;
- caps->num_pds = HNS_ROCE_V2_MAX_PD_NUM;
- caps->qpc_timer_bt_num = HNS_ROCE_V2_MAX_QPC_TIMER_BT_NUM;
- caps->cqc_timer_bt_num = HNS_ROCE_V2_MAX_CQC_TIMER_BT_NUM;
-
- caps->max_qp_init_rdma = HNS_ROCE_V2_MAX_QP_INIT_RDMA;
- caps->max_qp_dest_rdma = HNS_ROCE_V2_MAX_QP_DEST_RDMA;
- caps->max_sq_desc_sz = HNS_ROCE_V2_MAX_SQ_DESC_SZ;
- caps->max_rq_desc_sz = HNS_ROCE_V2_MAX_RQ_DESC_SZ;
- caps->irrl_entry_sz = HNS_ROCE_V2_IRRL_ENTRY_SZ;
- caps->trrl_entry_sz = HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ;
- caps->cqc_entry_sz = HNS_ROCE_V2_CQC_ENTRY_SZ;
- caps->srqc_entry_sz = HNS_ROCE_V2_SRQC_ENTRY_SZ;
- caps->mtpt_entry_sz = HNS_ROCE_V2_MTPT_ENTRY_SZ;
- caps->idx_entry_sz = HNS_ROCE_V2_IDX_ENTRY_SZ;
- caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
- caps->reserved_lkey = 0;
- caps->reserved_pds = 0;
- caps->reserved_mrws = 1;
- caps->reserved_uars = 0;
- caps->reserved_cqs = 0;
- caps->reserved_srqs = 0;
- caps->reserved_qps = HNS_ROCE_V2_RSV_QPS;
-
- caps->qpc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
- caps->srqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
- caps->cqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
- caps->mpt_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
- caps->sccc_hop_num = HNS_ROCE_SCCC_HOP_NUM;
-
- caps->mtt_hop_num = HNS_ROCE_MTT_HOP_NUM;
- caps->wqe_sq_hop_num = HNS_ROCE_SQWQE_HOP_NUM;
- caps->wqe_sge_hop_num = HNS_ROCE_EXT_SGE_HOP_NUM;
- caps->wqe_rq_hop_num = HNS_ROCE_RQWQE_HOP_NUM;
- caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM;
- caps->srqwqe_hop_num = HNS_ROCE_SRQWQE_HOP_NUM;
- caps->idx_hop_num = HNS_ROCE_IDX_HOP_NUM;
- caps->chunk_sz = HNS_ROCE_V2_TABLE_CHUNK_SIZE;
-
- caps->flags = HNS_ROCE_CAP_FLAG_REREG_MR |
- HNS_ROCE_CAP_FLAG_ROCE_V1_V2 |
- HNS_ROCE_CAP_FLAG_CQ_RECORD_DB |
- HNS_ROCE_CAP_FLAG_QP_RECORD_DB;
-
- caps->pkey_table_len[0] = 1;
- caps->ceqe_depth = HNS_ROCE_V2_COMP_EQE_NUM;
- caps->aeqe_depth = HNS_ROCE_V2_ASYNC_EQE_NUM;
- caps->local_ca_ack_delay = 0;
- caps->max_mtu = IB_MTU_4096;
-
- caps->max_srq_wrs = HNS_ROCE_V2_MAX_SRQ_WR;
- caps->max_srq_sges = HNS_ROCE_V2_MAX_SRQ_SGE;
-
- caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC | HNS_ROCE_CAP_FLAG_MW |
- HNS_ROCE_CAP_FLAG_SRQ | HNS_ROCE_CAP_FLAG_FRMR |
- HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL;
-
- caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;
-
- if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
- caps->flags |= HNS_ROCE_CAP_FLAG_STASH |
- HNS_ROCE_CAP_FLAG_DIRECT_WQE |
- HNS_ROCE_CAP_FLAG_XRC;
- caps->max_sq_inline = HNS_ROCE_V3_MAX_SQ_INLINE;
- } else {
- caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE;
-
- /* The following configuration are only valid for HIP08 */
- caps->qpc_sz = HNS_ROCE_V2_QPC_SZ;
- caps->sccc_sz = HNS_ROCE_V2_SCCC_SZ;
- caps->cqe_sz = HNS_ROCE_V2_CQE_SIZE;
- }
-}
-
static void calc_pg_sz(u32 obj_num, u32 obj_size, u32 hop_num, u32 ctx_bt_num,
u32 *buf_page_size, u32 *bt_page_size, u32 hem_type)
{
@@ -2239,7 +2145,7 @@ static void apply_func_caps(struct hns_roce_dev *hr_dev)
set_hem_page_size(hr_dev);
}
-static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev)
+static int hns_roce_query_caps(struct hns_roce_dev *hr_dev)
{
struct hns_roce_cmq_desc desc[HNS_ROCE_QUERY_PF_CAPS_CMD_NUM];
struct hns_roce_caps *caps = &hr_dev->caps;
@@ -2248,15 +2154,17 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev)
struct hns_roce_query_pf_caps_c *resp_c;
struct hns_roce_query_pf_caps_d *resp_d;
struct hns_roce_query_pf_caps_e *resp_e;
+ enum hns_roce_opcode_type cmd;
int ctx_hop_num;
int pbl_hop_num;
int ret;
int i;
+ cmd = hr_dev->is_vf ? HNS_ROCE_OPC_QUERY_VF_CAPS_NUM :
+ HNS_ROCE_OPC_QUERY_PF_CAPS_NUM;
+
for (i = 0; i < HNS_ROCE_QUERY_PF_CAPS_CMD_NUM; i++) {
- hns_roce_cmq_setup_basic_desc(&desc[i],
- HNS_ROCE_OPC_QUERY_PF_CAPS_NUM,
- true);
+ hns_roce_cmq_setup_basic_desc(&desc[i], cmd, true);
if (i < (HNS_ROCE_QUERY_PF_CAPS_CMD_NUM - 1))
desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
else
@@ -2273,35 +2181,33 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev)
resp_d = (struct hns_roce_query_pf_caps_d *)desc[3].data;
resp_e = (struct hns_roce_query_pf_caps_e *)desc[4].data;
- caps->local_ca_ack_delay = resp_a->local_ca_ack_delay;
- caps->max_sq_sg = le16_to_cpu(resp_a->max_sq_sg);
- caps->max_sq_inline = le16_to_cpu(resp_a->max_sq_inline);
- caps->max_rq_sg = le16_to_cpu(resp_a->max_rq_sg);
+ caps->local_ca_ack_delay = resp_a->local_ca_ack_delay;
+ caps->max_sq_sg = le16_to_cpu(resp_a->max_sq_sg);
+ caps->max_sq_inline = le16_to_cpu(resp_a->max_sq_inline);
+ caps->max_rq_sg = le16_to_cpu(resp_a->max_rq_sg);
caps->max_rq_sg = roundup_pow_of_two(caps->max_rq_sg);
- caps->max_srq_sges = le16_to_cpu(resp_a->max_srq_sges);
+ caps->max_srq_sges = le16_to_cpu(resp_a->max_srq_sges);
caps->max_srq_sges = roundup_pow_of_two(caps->max_srq_sges);
- caps->num_aeq_vectors = resp_a->num_aeq_vectors;
- caps->num_other_vectors = resp_a->num_other_vectors;
- caps->max_sq_desc_sz = resp_a->max_sq_desc_sz;
- caps->max_rq_desc_sz = resp_a->max_rq_desc_sz;
- caps->cqe_sz = resp_a->cqe_sz;
-
- caps->mtpt_entry_sz = resp_b->mtpt_entry_sz;
- caps->irrl_entry_sz = resp_b->irrl_entry_sz;
- caps->trrl_entry_sz = resp_b->trrl_entry_sz;
- caps->cqc_entry_sz = resp_b->cqc_entry_sz;
- caps->srqc_entry_sz = resp_b->srqc_entry_sz;
- caps->idx_entry_sz = resp_b->idx_entry_sz;
- caps->sccc_sz = resp_b->sccc_sz;
- caps->max_mtu = resp_b->max_mtu;
- caps->qpc_sz = le16_to_cpu(resp_b->qpc_sz);
- caps->min_cqes = resp_b->min_cqes;
- caps->min_wqes = resp_b->min_wqes;
- caps->page_size_cap = le32_to_cpu(resp_b->page_size_cap);
- caps->pkey_table_len[0] = resp_b->pkey_table_len;
- caps->phy_num_uars = resp_b->phy_num_uars;
- ctx_hop_num = resp_b->ctx_hop_num;
- pbl_hop_num = resp_b->pbl_hop_num;
+ caps->num_aeq_vectors = resp_a->num_aeq_vectors;
+ caps->num_other_vectors = resp_a->num_other_vectors;
+ caps->max_sq_desc_sz = resp_a->max_sq_desc_sz;
+ caps->max_rq_desc_sz = resp_a->max_rq_desc_sz;
+
+ caps->mtpt_entry_sz = resp_b->mtpt_entry_sz;
+ caps->irrl_entry_sz = resp_b->irrl_entry_sz;
+ caps->trrl_entry_sz = resp_b->trrl_entry_sz;
+ caps->cqc_entry_sz = resp_b->cqc_entry_sz;
+ caps->srqc_entry_sz = resp_b->srqc_entry_sz;
+ caps->idx_entry_sz = resp_b->idx_entry_sz;
+ caps->sccc_sz = resp_b->sccc_sz;
+ caps->max_mtu = resp_b->max_mtu;
+ caps->min_cqes = resp_b->min_cqes;
+ caps->min_wqes = resp_b->min_wqes;
+ caps->page_size_cap = le32_to_cpu(resp_b->page_size_cap);
+ caps->pkey_table_len[0] = resp_b->pkey_table_len;
+ caps->phy_num_uars = resp_b->phy_num_uars;
+ ctx_hop_num = resp_b->ctx_hop_num;
+ pbl_hop_num = resp_b->pbl_hop_num;
caps->num_pds = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_PDS);
@@ -2324,8 +2230,6 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev)
caps->ceqe_depth = 1 << hr_reg_read(resp_d, PF_CAPS_D_CEQ_DEPTH);
caps->num_comp_vectors = hr_reg_read(resp_d, PF_CAPS_D_NUM_CEQS);
caps->aeqe_depth = 1 << hr_reg_read(resp_d, PF_CAPS_D_AEQ_DEPTH);
- caps->default_aeq_arm_st = hr_reg_read(resp_d, PF_CAPS_D_AEQ_ARM_ST);
- caps->default_ceq_arm_st = hr_reg_read(resp_d, PF_CAPS_D_CEQ_ARM_ST);
caps->reserved_pds = hr_reg_read(resp_d, PF_CAPS_D_RSV_PDS);
caps->num_uars = 1 << hr_reg_read(resp_d, PF_CAPS_D_NUM_UARS);
caps->reserved_qps = hr_reg_read(resp_d, PF_CAPS_D_RSV_QPS);
@@ -2336,10 +2240,6 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev)
caps->reserved_cqs = hr_reg_read(resp_e, PF_CAPS_E_RSV_CQS);
caps->reserved_srqs = hr_reg_read(resp_e, PF_CAPS_E_RSV_SRQS);
caps->reserved_lkey = hr_reg_read(resp_e, PF_CAPS_E_RSV_LKEYS);
- caps->default_ceq_max_cnt = le16_to_cpu(resp_e->ceq_max_cnt);
- caps->default_ceq_period = le16_to_cpu(resp_e->ceq_period);
- caps->default_aeq_max_cnt = le16_to_cpu(resp_e->aeq_max_cnt);
- caps->default_aeq_period = le16_to_cpu(resp_e->aeq_period);
caps->qpc_hop_num = ctx_hop_num;
caps->sccc_hop_num = ctx_hop_num;
@@ -2357,6 +2257,19 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev)
if (!(caps->page_size_cap & PAGE_SIZE))
caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
+ if (!hr_dev->is_vf) {
+ caps->cqe_sz = resp_a->cqe_sz;
+ caps->qpc_sz = le16_to_cpu(resp_b->qpc_sz);
+ caps->default_aeq_arm_st =
+ hr_reg_read(resp_d, PF_CAPS_D_AEQ_ARM_ST);
+ caps->default_ceq_arm_st =
+ hr_reg_read(resp_d, PF_CAPS_D_CEQ_ARM_ST);
+ caps->default_ceq_max_cnt = le16_to_cpu(resp_e->ceq_max_cnt);
+ caps->default_ceq_period = le16_to_cpu(resp_e->ceq_period);
+ caps->default_aeq_max_cnt = le16_to_cpu(resp_e->aeq_max_cnt);
+ caps->default_aeq_period = le16_to_cpu(resp_e->aeq_period);
+ }
+
return 0;
}
@@ -2404,7 +2317,11 @@ static int hns_roce_v2_vf_profile(struct hns_roce_dev *hr_dev)
hr_dev->func_num = 1;
- set_default_caps(hr_dev);
+ ret = hns_roce_query_caps(hr_dev);
+ if (ret) {
+ dev_err(dev, "failed to query VF caps, ret = %d.\n", ret);
+ return ret;
+ }
ret = hns_roce_query_vf_resource(hr_dev);
if (ret) {
@@ -2444,9 +2361,11 @@ static int hns_roce_v2_pf_profile(struct hns_roce_dev *hr_dev)
return ret;
}
- ret = hns_roce_query_pf_caps(hr_dev);
- if (ret)
- set_default_caps(hr_dev);
+ ret = hns_roce_query_caps(hr_dev);
+ if (ret) {
+ dev_err(dev, "failed to query PF caps, ret = %d.\n", ret);
+ return ret;
+ }
ret = hns_roce_query_pf_resource(hr_dev);
if (ret) {
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
index af9d00225cdf..1b44d2434ab4 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
@@ -35,43 +35,15 @@
#include <linux/bitops.h>
-#define HNS_ROCE_V2_MAX_QP_NUM 0x1000
-#define HNS_ROCE_V2_MAX_WQE_NUM 0x8000
-#define HNS_ROCE_V2_MAX_SRQ_WR 0x8000
-#define HNS_ROCE_V2_MAX_SRQ_SGE 64
-#define HNS_ROCE_V2_MAX_CQ_NUM 0x100000
-#define HNS_ROCE_V2_MAX_QPC_TIMER_BT_NUM 0x100
-#define HNS_ROCE_V2_MAX_CQC_TIMER_BT_NUM 0x100
-#define HNS_ROCE_V2_MAX_SRQ_NUM 0x100000
-#define HNS_ROCE_V2_MAX_CQE_NUM 0x400000
-#define HNS_ROCE_V2_MAX_RQ_SGE_NUM 64
-#define HNS_ROCE_V2_MAX_SQ_SGE_NUM 64
-#define HNS_ROCE_V2_MAX_SQ_INLINE 0x20
-#define HNS_ROCE_V3_MAX_SQ_INLINE 0x400
#define HNS_ROCE_V2_MAX_RC_INL_INN_SZ 32
-#define HNS_ROCE_V2_UAR_NUM 256
-#define HNS_ROCE_V2_PHY_UAR_NUM 1
+#define HNS_ROCE_V2_MTT_ENTRY_SZ 64
#define HNS_ROCE_V2_AEQE_VEC_NUM 1
#define HNS_ROCE_V2_ABNORMAL_VEC_NUM 1
-#define HNS_ROCE_V2_MAX_MTPT_NUM 0x100000
#define HNS_ROCE_V2_MAX_SRQWQE_SEGS 0x1000000
#define HNS_ROCE_V2_MAX_IDX_SEGS 0x1000000
-#define HNS_ROCE_V2_MAX_PD_NUM 0x1000000
#define HNS_ROCE_V2_MAX_XRCD_NUM 0x1000000
#define HNS_ROCE_V2_RSV_XRCD_NUM 0
-#define HNS_ROCE_V2_MAX_QP_INIT_RDMA 128
-#define HNS_ROCE_V2_MAX_QP_DEST_RDMA 128
-#define HNS_ROCE_V2_MAX_SQ_DESC_SZ 64
-#define HNS_ROCE_V2_MAX_RQ_DESC_SZ 16
-#define HNS_ROCE_V2_IRRL_ENTRY_SZ 64
-#define HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ 100
-#define HNS_ROCE_V2_CQC_ENTRY_SZ 64
-#define HNS_ROCE_V2_SRQC_ENTRY_SZ 64
-#define HNS_ROCE_V2_MTPT_ENTRY_SZ 64
-#define HNS_ROCE_V2_MTT_ENTRY_SZ 64
-#define HNS_ROCE_V2_IDX_ENTRY_SZ 4
-#define HNS_ROCE_V2_SCCC_SZ 32
#define HNS_ROCE_V3_SCCC_SZ 64
#define HNS_ROCE_V3_GMV_ENTRY_SZ 32
@@ -232,6 +204,7 @@ enum hns_roce_opcode_type {
HNS_ROCE_OPC_QUERY_FUNC_INFO = 0x8407,
HNS_ROCE_OPC_QUERY_PF_CAPS_NUM = 0x8408,
HNS_ROCE_OPC_CFG_ENTRY_SIZE = 0x8409,
+ HNS_ROCE_OPC_QUERY_VF_CAPS_NUM = 0x8410,
HNS_ROCE_OPC_CFG_SGID_TB = 0x8500,
HNS_ROCE_OPC_CFG_SMAC_TB = 0x8501,
HNS_ROCE_OPC_POST_MB = 0x8504,
diff --git a/drivers/infiniband/hw/irdma/cm.c b/drivers/infiniband/hw/irdma/cm.c
index 8817864154af..70009b970e08 100644
--- a/drivers/infiniband/hw/irdma/cm.c
+++ b/drivers/infiniband/hw/irdma/cm.c
@@ -337,7 +337,7 @@ static struct irdma_puda_buf *irdma_form_ah_cm_frame(struct irdma_cm_node *cm_no
pktsize = sizeof(*tcph) + opts_len + hdr_len + pd_len;
- memset(buf, 0, pktsize);
+ memset(buf, 0, sizeof(*tcph));
sqbuf->totallen = pktsize;
sqbuf->tcphlen = sizeof(*tcph) + opts_len;
diff --git a/drivers/infiniband/hw/irdma/ctrl.c b/drivers/infiniband/hw/irdma/ctrl.c
index a41e0d21143a..d88c9184007e 100644
--- a/drivers/infiniband/hw/irdma/ctrl.c
+++ b/drivers/infiniband/hw/irdma/ctrl.c
@@ -1867,8 +1867,6 @@ void irdma_sc_vsi_init(struct irdma_sc_vsi *vsi,
vsi->mtu = info->params->mtu;
vsi->exception_lan_q = info->exception_lan_q;
vsi->vsi_idx = info->pf_data_vsi_num;
- if (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
- vsi->fcn_id = info->dev->hmc_fn_id;
irdma_set_qos_info(vsi, info->params);
for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
@@ -1887,32 +1885,56 @@ void irdma_sc_vsi_init(struct irdma_sc_vsi *vsi,
}
/**
- * irdma_get_fcn_id - Return the function id
+ * irdma_get_stats_idx - Return stats index
* @vsi: pointer to the vsi
*/
-static u8 irdma_get_fcn_id(struct irdma_sc_vsi *vsi)
+static u8 irdma_get_stats_idx(struct irdma_sc_vsi *vsi)
{
struct irdma_stats_inst_info stats_info = {};
struct irdma_sc_dev *dev = vsi->dev;
- u8 fcn_id = IRDMA_INVALID_FCN_ID;
- u8 start_idx, max_stats, i;
+ u8 i;
- if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {
+ if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
if (!irdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_ALLOCATE,
&stats_info))
return stats_info.stats_idx;
}
- start_idx = 1;
- max_stats = 16;
- for (i = start_idx; i < max_stats; i++)
- if (!dev->fcn_id_array[i]) {
- fcn_id = i;
- dev->fcn_id_array[i] = true;
- break;
+ for (i = 0; i < IRDMA_MAX_STATS_COUNT_GEN_1; i++) {
+ if (!dev->stats_idx_array[i]) {
+ dev->stats_idx_array[i] = true;
+ return i;
}
+ }
- return fcn_id;
+ return IRDMA_INVALID_STATS_IDX;
+}
+
+/**
+ * irdma_hw_stats_init_gen1 - Initialize stat reg table used for gen1
+ * @vsi: vsi structure where hw_regs are set
+ *
+ * Populate the HW stats table
+ */
+static void irdma_hw_stats_init_gen1(struct irdma_sc_vsi *vsi)
+{
+ struct irdma_sc_dev *dev = vsi->dev;
+ const struct irdma_hw_stat_map *map;
+ u64 *stat_reg = vsi->hw_stats_regs;
+ u64 *regs = dev->hw_stats_regs;
+ u16 i, stats_reg_set = vsi->stats_idx;
+
+ map = dev->hw_stats_map;
+
+ /* First 4 stat instances are reserved for port level statistics. */
+ stats_reg_set += vsi->stats_inst_alloc ? IRDMA_FIRST_NON_PF_STAT : 0;
+
+ for (i = 0; i < dev->hw_attrs.max_stat_idx; i++) {
+ if (map[i].bitmask <= IRDMA_MAX_STATS_32)
+ stat_reg[i] = regs[i] + stats_reg_set * sizeof(u32);
+ else
+ stat_reg[i] = regs[i] + stats_reg_set * sizeof(u64);
+ }
}
/**
@@ -1923,7 +1945,6 @@ static u8 irdma_get_fcn_id(struct irdma_sc_vsi *vsi)
int irdma_vsi_stats_init(struct irdma_sc_vsi *vsi,
struct irdma_vsi_stats_info *info)
{
- u8 fcn_id = info->fcn_id;
struct irdma_dma_mem *stats_buff_mem;
vsi->pestat = info->pestat;
@@ -1944,26 +1965,24 @@ int irdma_vsi_stats_init(struct irdma_sc_vsi *vsi,
IRDMA_GATHER_STATS_BUF_SIZE);
irdma_hw_stats_start_timer(vsi);
- if (info->alloc_fcn_id)
- fcn_id = irdma_get_fcn_id(vsi);
- if (fcn_id == IRDMA_INVALID_FCN_ID)
- goto stats_error;
-
- vsi->stats_fcn_id_alloc = info->alloc_fcn_id;
- vsi->fcn_id = fcn_id;
- if (info->alloc_fcn_id) {
- vsi->pestat->gather_info.use_stats_inst = true;
- vsi->pestat->gather_info.stats_inst_index = fcn_id;
- }
- return 0;
+ /* when stat allocation is not required default to fcn_id. */
+ vsi->stats_idx = info->fcn_id;
+ if (info->alloc_stats_inst) {
+ u8 stats_idx = irdma_get_stats_idx(vsi);
-stats_error:
- dma_free_coherent(vsi->pestat->hw->device, stats_buff_mem->size,
- stats_buff_mem->va, stats_buff_mem->pa);
- stats_buff_mem->va = NULL;
+ if (stats_idx != IRDMA_INVALID_STATS_IDX) {
+ vsi->stats_inst_alloc = true;
+ vsi->stats_idx = stats_idx;
+ vsi->pestat->gather_info.use_stats_inst = true;
+ vsi->pestat->gather_info.stats_inst_index = stats_idx;
+ }
+ }
+
+ if (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
+ irdma_hw_stats_init_gen1(vsi);
- return -EIO;
+ return 0;
}
/**
@@ -1973,19 +1992,19 @@ stats_error:
void irdma_vsi_stats_free(struct irdma_sc_vsi *vsi)
{
struct irdma_stats_inst_info stats_info = {};
- u8 fcn_id = vsi->fcn_id;
struct irdma_sc_dev *dev = vsi->dev;
+ u8 stats_idx = vsi->stats_idx;
- if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {
- if (vsi->stats_fcn_id_alloc) {
- stats_info.stats_idx = vsi->fcn_id;
+ if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
+ if (vsi->stats_inst_alloc) {
+ stats_info.stats_idx = vsi->stats_idx;
irdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_FREE,
&stats_info);
}
} else {
- if (vsi->stats_fcn_id_alloc &&
- fcn_id < vsi->dev->hw_attrs.max_stat_inst)
- vsi->dev->fcn_id_array[fcn_id] = false;
+ if (vsi->stats_inst_alloc &&
+ stats_idx < vsi->dev->hw_attrs.max_stat_inst)
+ vsi->dev->stats_idx_array[stats_idx] = false;
}
if (!vsi->pestat)
@@ -5297,7 +5316,8 @@ void sc_vsi_update_stats(struct irdma_sc_vsi *vsi)
gather_stats = vsi->pestat->gather_info.gather_stats_va;
last_gather_stats = vsi->pestat->gather_info.last_gather_stats_va;
irdma_update_stats(&vsi->pestat->hw_stats, gather_stats,
- last_gather_stats);
+ last_gather_stats, vsi->dev->hw_stats_map,
+ vsi->dev->hw_attrs.max_stat_idx);
}
/**
@@ -5405,185 +5425,61 @@ int irdma_sc_dev_init(enum irdma_vers ver, struct irdma_sc_dev *dev,
}
/**
+ * irdma_stat_val - Extract HW counter value from statistics buffer
+ * @stats_val: pointer to statistics buffer
+ * @byteoff: byte offset of counter value in the buffer (8B-aligned)
+ * @bitoff: bit offset of counter value within 8B entry
+ * @bitmask: maximum counter value (e.g. 0xffffff for 24-bit counter)
+ */
+static inline u64 irdma_stat_val(const u64 *stats_val, u16 byteoff, u8 bitoff,
+ u64 bitmask)
+{
+ u16 idx = byteoff / sizeof(*stats_val);
+
+ return (stats_val[idx] >> bitoff) & bitmask;
+}
+
+/**
+ * irdma_stat_delta - Calculate counter delta
+ * @new_val: updated counter value
+ * @old_val: last counter value
+ * @max_val: maximum counter value (e.g. 0xffffff for 24-bit counter)
+ */
+static inline u64 irdma_stat_delta(u64 new_val, u64 old_val, u64 max_val)
+{
+ if (new_val >= old_val)
+ return new_val - old_val;
+
+ /* roll-over case */
+ return max_val - old_val + new_val + 1;
+}
+
+/**
* irdma_update_stats - Update statistics
* @hw_stats: hw_stats instance to update
* @gather_stats: updated stat counters
* @last_gather_stats: last stat counters
+ * @map: HW stat map (hw_stats => gather_stats)
+ * @max_stat_idx: number of HW stats
*/
void irdma_update_stats(struct irdma_dev_hw_stats *hw_stats,
struct irdma_gather_stats *gather_stats,
- struct irdma_gather_stats *last_gather_stats)
-{
- u64 *stats_val = hw_stats->stats_val_32;
-
- stats_val[IRDMA_HW_STAT_INDEX_RXVLANERR] +=
- IRDMA_STATS_DELTA(gather_stats->rxvlanerr,
- last_gather_stats->rxvlanerr,
- IRDMA_MAX_STATS_32);
- stats_val[IRDMA_HW_STAT_INDEX_IP4RXDISCARD] +=
- IRDMA_STATS_DELTA(gather_stats->ip4rxdiscard,
- last_gather_stats->ip4rxdiscard,
- IRDMA_MAX_STATS_32);
- stats_val[IRDMA_HW_STAT_INDEX_IP4RXTRUNC] +=
- IRDMA_STATS_DELTA(gather_stats->ip4rxtrunc,
- last_gather_stats->ip4rxtrunc,
- IRDMA_MAX_STATS_32);
- stats_val[IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] +=
- IRDMA_STATS_DELTA(gather_stats->ip4txnoroute,
- last_gather_stats->ip4txnoroute,
- IRDMA_MAX_STATS_32);
- stats_val[IRDMA_HW_STAT_INDEX_IP6RXDISCARD] +=
- IRDMA_STATS_DELTA(gather_stats->ip6rxdiscard,
- last_gather_stats->ip6rxdiscard,
- IRDMA_MAX_STATS_32);
- stats_val[IRDMA_HW_STAT_INDEX_IP6RXTRUNC] +=
- IRDMA_STATS_DELTA(gather_stats->ip6rxtrunc,
- last_gather_stats->ip6rxtrunc,
- IRDMA_MAX_STATS_32);
- stats_val[IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] +=
- IRDMA_STATS_DELTA(gather_stats->ip6txnoroute,
- last_gather_stats->ip6txnoroute,
- IRDMA_MAX_STATS_32);
- stats_val[IRDMA_HW_STAT_INDEX_TCPRTXSEG] +=
- IRDMA_STATS_DELTA(gather_stats->tcprtxseg,
- last_gather_stats->tcprtxseg,
- IRDMA_MAX_STATS_32);
- stats_val[IRDMA_HW_STAT_INDEX_TCPRXOPTERR] +=
- IRDMA_STATS_DELTA(gather_stats->tcprxopterr,
- last_gather_stats->tcprxopterr,
- IRDMA_MAX_STATS_32);
- stats_val[IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] +=
- IRDMA_STATS_DELTA(gather_stats->tcprxprotoerr,
- last_gather_stats->tcprxprotoerr,
- IRDMA_MAX_STATS_32);
- stats_val[IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED] +=
- IRDMA_STATS_DELTA(gather_stats->rxrpcnphandled,
- last_gather_stats->rxrpcnphandled,
- IRDMA_MAX_STATS_32);
- stats_val[IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED] +=
- IRDMA_STATS_DELTA(gather_stats->rxrpcnpignored,
- last_gather_stats->rxrpcnpignored,
- IRDMA_MAX_STATS_32);
- stats_val[IRDMA_HW_STAT_INDEX_TXNPCNPSENT] +=
- IRDMA_STATS_DELTA(gather_stats->txnpcnpsent,
- last_gather_stats->txnpcnpsent,
- IRDMA_MAX_STATS_32);
- stats_val = hw_stats->stats_val_64;
- stats_val[IRDMA_HW_STAT_INDEX_IP4RXOCTS] +=
- IRDMA_STATS_DELTA(gather_stats->ip4rxocts,
- last_gather_stats->ip4rxocts,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_IP4RXPKTS] +=
- IRDMA_STATS_DELTA(gather_stats->ip4rxpkts,
- last_gather_stats->ip4rxpkts,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_IP4RXFRAGS] +=
- IRDMA_STATS_DELTA(gather_stats->ip4txfrag,
- last_gather_stats->ip4txfrag,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] +=
- IRDMA_STATS_DELTA(gather_stats->ip4rxmcpkts,
- last_gather_stats->ip4rxmcpkts,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_IP4TXOCTS] +=
- IRDMA_STATS_DELTA(gather_stats->ip4txocts,
- last_gather_stats->ip4txocts,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_IP4TXPKTS] +=
- IRDMA_STATS_DELTA(gather_stats->ip4txpkts,
- last_gather_stats->ip4txpkts,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_IP4TXFRAGS] +=
- IRDMA_STATS_DELTA(gather_stats->ip4txfrag,
- last_gather_stats->ip4txfrag,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] +=
- IRDMA_STATS_DELTA(gather_stats->ip4txmcpkts,
- last_gather_stats->ip4txmcpkts,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_IP6RXOCTS] +=
- IRDMA_STATS_DELTA(gather_stats->ip6rxocts,
- last_gather_stats->ip6rxocts,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_IP6RXPKTS] +=
- IRDMA_STATS_DELTA(gather_stats->ip6rxpkts,
- last_gather_stats->ip6rxpkts,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_IP6RXFRAGS] +=
- IRDMA_STATS_DELTA(gather_stats->ip6txfrags,
- last_gather_stats->ip6txfrags,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] +=
- IRDMA_STATS_DELTA(gather_stats->ip6rxmcpkts,
- last_gather_stats->ip6rxmcpkts,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_IP6TXOCTS] +=
- IRDMA_STATS_DELTA(gather_stats->ip6txocts,
- last_gather_stats->ip6txocts,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_IP6TXPKTS] +=
- IRDMA_STATS_DELTA(gather_stats->ip6txpkts,
- last_gather_stats->ip6txpkts,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_IP6TXFRAGS] +=
- IRDMA_STATS_DELTA(gather_stats->ip6txfrags,
- last_gather_stats->ip6txfrags,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] +=
- IRDMA_STATS_DELTA(gather_stats->ip6txmcpkts,
- last_gather_stats->ip6txmcpkts,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_TCPRXSEGS] +=
- IRDMA_STATS_DELTA(gather_stats->tcprxsegs,
- last_gather_stats->tcprxsegs,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_TCPTXSEG] +=
- IRDMA_STATS_DELTA(gather_stats->tcptxsegs,
- last_gather_stats->tcptxsegs,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_RDMARXRDS] +=
- IRDMA_STATS_DELTA(gather_stats->rdmarxrds,
- last_gather_stats->rdmarxrds,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_RDMARXSNDS] +=
- IRDMA_STATS_DELTA(gather_stats->rdmarxsnds,
- last_gather_stats->rdmarxsnds,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_RDMARXWRS] +=
- IRDMA_STATS_DELTA(gather_stats->rdmarxwrs,
- last_gather_stats->rdmarxwrs,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_RDMATXRDS] +=
- IRDMA_STATS_DELTA(gather_stats->rdmatxrds,
- last_gather_stats->rdmatxrds,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_RDMATXSNDS] +=
- IRDMA_STATS_DELTA(gather_stats->rdmatxsnds,
- last_gather_stats->rdmatxsnds,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_RDMATXWRS] +=
- IRDMA_STATS_DELTA(gather_stats->rdmatxwrs,
- last_gather_stats->rdmatxwrs,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_RDMAVBND] +=
- IRDMA_STATS_DELTA(gather_stats->rdmavbn,
- last_gather_stats->rdmavbn,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_RDMAVINV] +=
- IRDMA_STATS_DELTA(gather_stats->rdmavinv,
- last_gather_stats->rdmavinv,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_UDPRXPKTS] +=
- IRDMA_STATS_DELTA(gather_stats->udprxpkts,
- last_gather_stats->udprxpkts,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_UDPTXPKTS] +=
- IRDMA_STATS_DELTA(gather_stats->udptxpkts,
- last_gather_stats->udptxpkts,
- IRDMA_MAX_STATS_48);
- stats_val[IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS] +=
- IRDMA_STATS_DELTA(gather_stats->rxnpecnmrkpkts,
- last_gather_stats->rxnpecnmrkpkts,
- IRDMA_MAX_STATS_48);
+ struct irdma_gather_stats *last_gather_stats,
+ const struct irdma_hw_stat_map *map, u16 max_stat_idx)
+{
+ u64 *stats_val = hw_stats->stats_val;
+ u16 i;
+
+ for (i = 0; i < max_stat_idx; i++) {
+ u64 new_val = irdma_stat_val(gather_stats->val, map[i].byteoff,
+ map[i].bitoff, map[i].bitmask);
+ u64 last_val = irdma_stat_val(last_gather_stats->val,
+ map[i].byteoff, map[i].bitoff,
+ map[i].bitmask);
+
+ stats_val[i] +=
+ irdma_stat_delta(new_val, last_val, map[i].bitmask);
+ }
+
memcpy(last_gather_stats, gather_stats, sizeof(*last_gather_stats));
}
diff --git a/drivers/infiniband/hw/irdma/defs.h b/drivers/infiniband/hw/irdma/defs.h
index c1906cab5c8a..6014b9d06a9b 100644
--- a/drivers/infiniband/hw/irdma/defs.h
+++ b/drivers/infiniband/hw/irdma/defs.h
@@ -36,6 +36,7 @@ enum irdma_protocol_used {
#define IRDMA_QP_STATE_ERROR 6
#define IRDMA_MAX_TRAFFIC_CLASS 8
+#define IRDMA_MAX_STATS_COUNT_GEN_1 12
#define IRDMA_MAX_USER_PRIORITY 8
#define IRDMA_MAX_APPS 8
#define IRDMA_MAX_STATS_COUNT 128
@@ -365,9 +366,11 @@ enum irdma_cqp_op_type {
#define FLD_RS_32(dev, val, field) \
((u64)((val) & (dev)->hw_masks[field ## _M]) >> (dev)->hw_shifts[field ## _S])
-#define IRDMA_STATS_DELTA(a, b, c) ((a) >= (b) ? (a) - (b) : (a) + (c) - (b))
-#define IRDMA_MAX_STATS_32 0xFFFFFFFFULL
-#define IRDMA_MAX_STATS_48 0xFFFFFFFFFFFFULL
+#define IRDMA_MAX_STATS_24 0xffffffULL
+#define IRDMA_MAX_STATS_32 0xffffffffULL
+#define IRDMA_MAX_STATS_48 0xffffffffffffULL
+#define IRDMA_MAX_STATS_56 0xffffffffffffffULL
+#define IRDMA_MAX_STATS_64 0xffffffffffffffffULL
#define IRDMA_MAX_CQ_READ_THRESH 0x3FFFF
#define IRDMA_CQPSQ_QHASH_VLANID GENMASK_ULL(43, 32)
diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c
index 43dfa4761f06..795f7fd4f257 100644
--- a/drivers/infiniband/hw/irdma/hw.c
+++ b/drivers/infiniband/hw/irdma/hw.c
@@ -1092,14 +1092,19 @@ static int irdma_cfg_ceq_vector(struct irdma_pci_f *rf, struct irdma_ceq *iwceq,
int status;
if (rf->msix_shared && !ceq_id) {
+ snprintf(msix_vec->name, sizeof(msix_vec->name) - 1,
+ "irdma-%s-AEQCEQ-0", dev_name(&rf->pcidev->dev));
tasklet_setup(&rf->dpc_tasklet, irdma_dpc);
status = request_irq(msix_vec->irq, irdma_irq_handler, 0,
- "AEQCEQ", rf);
+ msix_vec->name, rf);
} else {
+ snprintf(msix_vec->name, sizeof(msix_vec->name) - 1,
+ "irdma-%s-CEQ-%d",
+ dev_name(&rf->pcidev->dev), ceq_id);
tasklet_setup(&iwceq->dpc_tasklet, irdma_ceq_dpc);
status = request_irq(msix_vec->irq, irdma_ceq_handler, 0,
- "CEQ", iwceq);
+ msix_vec->name, iwceq);
}
cpumask_clear(&msix_vec->mask);
cpumask_set_cpu(msix_vec->cpu_affinity, &msix_vec->mask);
@@ -1128,9 +1133,11 @@ static int irdma_cfg_aeq_vector(struct irdma_pci_f *rf)
u32 ret = 0;
if (!rf->msix_shared) {
+ snprintf(msix_vec->name, sizeof(msix_vec->name) - 1,
+ "irdma-%s-AEQ", dev_name(&rf->pcidev->dev));
tasklet_setup(&rf->dpc_tasklet, irdma_dpc);
ret = request_irq(msix_vec->irq, irdma_irq_handler, 0,
- "irdma", rf);
+ msix_vec->name, rf);
}
if (ret) {
ibdev_dbg(&rf->iwdev->ibdev, "ERR: aeq irq config fail\n");
@@ -1904,8 +1911,8 @@ int irdma_ctrl_init_hw(struct irdma_pci_f *rf)
break;
rf->init_state = CEQ0_CREATED;
/* Handles processing of CQP completions */
- rf->cqp_cmpl_wq = alloc_ordered_workqueue("cqp_cmpl_wq",
- WQ_HIGHPRI | WQ_UNBOUND);
+ rf->cqp_cmpl_wq =
+ alloc_ordered_workqueue("cqp_cmpl_wq", WQ_HIGHPRI);
if (!rf->cqp_cmpl_wq) {
status = -ENOMEM;
break;
diff --git a/drivers/infiniband/hw/irdma/i40iw_hw.c b/drivers/infiniband/hw/irdma/i40iw_hw.c
index 50299f58b6b3..37a40fb4d0d7 100644
--- a/drivers/infiniband/hw/irdma/i40iw_hw.c
+++ b/drivers/infiniband/hw/irdma/i40iw_hw.c
@@ -32,7 +32,7 @@ static u32 i40iw_regs[IRDMA_MAX_REGS] = {
0xffffffff /* PFINT_RATEN not used in FPK */
};
-static u32 i40iw_stat_offsets_32[IRDMA_HW_STAT_INDEX_MAX_32] = {
+static u32 i40iw_stat_offsets[] = {
I40E_GLPES_PFIP4RXDISCARD(0),
I40E_GLPES_PFIP4RXTRUNC(0),
I40E_GLPES_PFIP4TXNOROUTE(0),
@@ -42,10 +42,8 @@ static u32 i40iw_stat_offsets_32[IRDMA_HW_STAT_INDEX_MAX_32] = {
I40E_GLPES_PFTCPRTXSEG(0),
I40E_GLPES_PFTCPRXOPTERR(0),
I40E_GLPES_PFTCPRXPROTOERR(0),
- I40E_GLPES_PFRXVLANERR(0)
-};
+ I40E_GLPES_PFRXVLANERR(0),
-static u32 i40iw_stat_offsets_64[IRDMA_HW_STAT_INDEX_MAX_64] = {
I40E_GLPES_PFIP4RXOCTSLO(0),
I40E_GLPES_PFIP4RXPKTSLO(0),
I40E_GLPES_PFIP4RXFRAGSLO(0),
@@ -158,6 +156,51 @@ static const struct irdma_irq_ops i40iw_irq_ops = {
.irdma_en_irq = i40iw_ena_irq,
};
+static const struct irdma_hw_stat_map i40iw_hw_stat_map[] = {
+ [IRDMA_HW_STAT_INDEX_RXVLANERR] = { 0, 0, IRDMA_MAX_STATS_24 },
+ [IRDMA_HW_STAT_INDEX_IP4RXOCTS] = { 8, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP4RXPKTS] = { 16, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP4RXDISCARD] = { 24, 0, IRDMA_MAX_STATS_32 },
+ [IRDMA_HW_STAT_INDEX_IP4RXTRUNC] = { 32, 0, IRDMA_MAX_STATS_32 },
+ [IRDMA_HW_STAT_INDEX_IP4RXFRAGS] = { 40, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] = { 48, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP6RXOCTS] = { 56, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP6RXPKTS] = { 64, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP6RXDISCARD] = { 72, 0, IRDMA_MAX_STATS_32 },
+ [IRDMA_HW_STAT_INDEX_IP6RXTRUNC] = { 80, 0, IRDMA_MAX_STATS_32 },
+ [IRDMA_HW_STAT_INDEX_IP6RXFRAGS] = { 88, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] = { 96, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP4TXOCTS] = { 104, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP4TXPKTS] = { 112, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP4TXFRAGS] = { 120, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] = { 128, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP6TXOCTS] = { 136, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP6TXPKTS] = { 144, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP6TXFRAGS] = { 152, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] = { 160, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] = { 168, 0, IRDMA_MAX_STATS_24 },
+ [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] = { 176, 0, IRDMA_MAX_STATS_24 },
+ [IRDMA_HW_STAT_INDEX_TCPRXSEGS] = { 184, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_TCPRXOPTERR] = { 192, 0, IRDMA_MAX_STATS_24 },
+ [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] = { 200, 0, IRDMA_MAX_STATS_24 },
+ [IRDMA_HW_STAT_INDEX_TCPTXSEG] = { 208, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_TCPRTXSEG] = { 216, 0, IRDMA_MAX_STATS_32 },
+ [IRDMA_HW_STAT_INDEX_RDMARXWRS] = { 224, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_RDMARXRDS] = { 232, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_RDMARXSNDS] = { 240, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_RDMATXWRS] = { 248, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_RDMATXRDS] = { 256, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_RDMATXSNDS] = { 264, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_RDMAVBND] = { 272, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_RDMAVINV] = { 280, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS] = { 288, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS] = { 296, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS] = { 304, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS] = { 312, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_UDPRXPKTS] = { 320, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_UDPTXPKTS] = { 328, 0, IRDMA_MAX_STATS_48 },
+};
+
void i40iw_init_hw(struct irdma_sc_dev *dev)
{
int i;
@@ -172,11 +215,8 @@ void i40iw_init_hw(struct irdma_sc_dev *dev)
dev->hw_regs[i] = (u32 __iomem *)(i40iw_regs[i] + hw_addr);
}
- for (i = 0; i < IRDMA_HW_STAT_INDEX_MAX_32; ++i)
- dev->hw_stats_regs_32[i] = i40iw_stat_offsets_32[i];
-
- for (i = 0; i < IRDMA_HW_STAT_INDEX_MAX_64; ++i)
- dev->hw_stats_regs_64[i] = i40iw_stat_offsets_64[i];
+ for (i = 0; i < IRDMA_HW_STAT_INDEX_MAX_GEN_1; ++i)
+ dev->hw_stats_regs[i] = i40iw_stat_offsets[i];
dev->hw_attrs.first_hw_vf_fpm_id = I40IW_FIRST_VF_FPM_ID;
dev->hw_attrs.max_hw_vf_fpm_id = IRDMA_MAX_VF_FPM_ID;
@@ -195,6 +235,7 @@ void i40iw_init_hw(struct irdma_sc_dev *dev)
dev->ceq_itr_mask_db = NULL;
dev->aeq_itr_mask_db = NULL;
dev->irq_ops = &i40iw_irq_ops;
+ dev->hw_stats_map = i40iw_hw_stat_map;
/* Setup the hardware limits, hmc may limit further */
dev->hw_attrs.uk_attrs.max_hw_wq_frags = I40IW_MAX_WQ_FRAGMENT_COUNT;
@@ -210,6 +251,7 @@ void i40iw_init_hw(struct irdma_sc_dev *dev)
dev->hw_attrs.uk_attrs.max_hw_sq_chunk = I40IW_MAX_QUANTA_PER_WR;
dev->hw_attrs.max_hw_pds = I40IW_MAX_PDS;
dev->hw_attrs.max_stat_inst = I40IW_MAX_STATS_COUNT;
+ dev->hw_attrs.max_stat_idx = IRDMA_HW_STAT_INDEX_MAX_GEN_1;
dev->hw_attrs.max_hw_outbound_msg_size = I40IW_MAX_OUTBOUND_MSG_SIZE;
dev->hw_attrs.max_hw_inbound_msg_size = I40IW_MAX_INBOUND_MSG_SIZE;
dev->hw_attrs.max_qp_wr = I40IW_MAX_QP_WRS;
diff --git a/drivers/infiniband/hw/irdma/icrdma_hw.c b/drivers/infiniband/hw/irdma/icrdma_hw.c
index 5986fd906308..298d14905993 100644
--- a/drivers/infiniband/hw/irdma/icrdma_hw.c
+++ b/drivers/infiniband/hw/irdma/icrdma_hw.c
@@ -111,6 +111,55 @@ static const struct irdma_irq_ops icrdma_irq_ops = {
.irdma_en_irq = icrdma_ena_irq,
};
+static const struct irdma_hw_stat_map icrdma_hw_stat_map[] = {
+ [IRDMA_HW_STAT_INDEX_RXVLANERR] = { 0, 32, IRDMA_MAX_STATS_24 },
+ [IRDMA_HW_STAT_INDEX_IP4RXOCTS] = { 8, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP4RXPKTS] = { 16, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP4RXDISCARD] = { 24, 32, IRDMA_MAX_STATS_32 },
+ [IRDMA_HW_STAT_INDEX_IP4RXTRUNC] = { 24, 0, IRDMA_MAX_STATS_32 },
+ [IRDMA_HW_STAT_INDEX_IP4RXFRAGS] = { 32, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS] = { 40, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] = { 48, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP6RXOCTS] = { 56, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP6RXPKTS] = { 64, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP6RXDISCARD] = { 72, 32, IRDMA_MAX_STATS_32 },
+ [IRDMA_HW_STAT_INDEX_IP6RXTRUNC] = { 72, 0, IRDMA_MAX_STATS_32 },
+ [IRDMA_HW_STAT_INDEX_IP6RXFRAGS] = { 80, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS] = { 88, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] = { 96, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP4TXOCTS] = { 104, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP4TXPKTS] = { 112, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP4TXFRAGS] = { 120, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS] = { 128, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] = { 136, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP6TXOCTS] = { 144, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP6TXPKTS] = { 152, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP6TXFRAGS] = { 160, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS] = { 168, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] = { 176, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] = { 184, 32, IRDMA_MAX_STATS_24 },
+ [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] = { 184, 0, IRDMA_MAX_STATS_24 },
+ [IRDMA_HW_STAT_INDEX_TCPRXSEGS] = { 192, 32, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_TCPRXOPTERR] = { 200, 32, IRDMA_MAX_STATS_24 },
+ [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] = { 200, 0, IRDMA_MAX_STATS_24 },
+ [IRDMA_HW_STAT_INDEX_TCPTXSEG] = { 208, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_TCPRTXSEG] = { 216, 32, IRDMA_MAX_STATS_32 },
+ [IRDMA_HW_STAT_INDEX_UDPRXPKTS] = { 224, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_UDPTXPKTS] = { 232, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_RDMARXWRS] = { 240, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_RDMARXRDS] = { 248, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_RDMARXSNDS] = { 256, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_RDMATXWRS] = { 264, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_RDMATXRDS] = { 272, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_RDMATXSNDS] = { 280, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_RDMAVBND] = { 288, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_RDMAVINV] = { 296, 0, IRDMA_MAX_STATS_48 },
+ [IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS] = { 304, 0, IRDMA_MAX_STATS_56 },
+ [IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED] = { 312, 32, IRDMA_MAX_STATS_24 },
+ [IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED] = { 312, 0, IRDMA_MAX_STATS_32 },
+ [IRDMA_HW_STAT_INDEX_TXNPCNPSENT] = { 320, 0, IRDMA_MAX_STATS_32 },
+};
+
void icrdma_init_hw(struct irdma_sc_dev *dev)
{
int i;
@@ -140,9 +189,11 @@ void icrdma_init_hw(struct irdma_sc_dev *dev)
dev->cq_ack_db = dev->hw_regs[IRDMA_CQACK];
dev->irq_ops = &icrdma_irq_ops;
dev->hw_attrs.page_size_cap = SZ_4K | SZ_2M | SZ_1G;
+ dev->hw_stats_map = icrdma_hw_stat_map;
dev->hw_attrs.max_hw_ird = ICRDMA_MAX_IRD_SIZE;
dev->hw_attrs.max_hw_ord = ICRDMA_MAX_ORD_SIZE;
dev->hw_attrs.max_stat_inst = ICRDMA_MAX_STATS_COUNT;
+ dev->hw_attrs.max_stat_idx = IRDMA_HW_STAT_INDEX_MAX_GEN_2;
dev->hw_attrs.uk_attrs.max_hw_sq_chunk = IRDMA_MAX_QUANTA_PER_WR;
dev->hw_attrs.uk_attrs.feature_flags |= IRDMA_FEATURE_RTS_AE |
diff --git a/drivers/infiniband/hw/irdma/irdma.h b/drivers/infiniband/hw/irdma/irdma.h
index 4789e85d717b..173e2dc2fc35 100644
--- a/drivers/infiniband/hw/irdma/irdma.h
+++ b/drivers/infiniband/hw/irdma/irdma.h
@@ -147,6 +147,7 @@ struct irdma_hw_attrs {
u32 max_sleep_count;
u32 max_cqp_compl_wait_time_ms;
u16 max_stat_inst;
+ u16 max_stat_idx;
};
void i40iw_init_hw(struct irdma_sc_dev *dev);
diff --git a/drivers/infiniband/hw/irdma/main.h b/drivers/infiniband/hw/irdma/main.h
index 65e966ad3453..def6dd58dcd4 100644
--- a/drivers/infiniband/hw/irdma/main.h
+++ b/drivers/infiniband/hw/irdma/main.h
@@ -115,6 +115,8 @@ extern struct auxiliary_driver i40iw_auxiliary_drv;
#define IRDMA_REFLUSH BIT(2)
#define IRDMA_FLUSH_WAIT BIT(3)
+#define IRDMA_IRQ_NAME_STR_LEN (64)
+
enum init_completion_state {
INVALID_STATE = 0,
INITIAL_STATE,
@@ -212,6 +214,7 @@ struct irdma_msix_vector {
u32 cpu_affinity;
u32 ceq_id;
cpumask_t mask;
+ char name[IRDMA_IRQ_NAME_STR_LEN];
};
struct irdma_mc_table_info {
diff --git a/drivers/infiniband/hw/irdma/pble.c b/drivers/infiniband/hw/irdma/pble.c
index cdc0b8a6ed48..c0bef11436b9 100644
--- a/drivers/infiniband/hw/irdma/pble.c
+++ b/drivers/infiniband/hw/irdma/pble.c
@@ -423,15 +423,15 @@ static int get_lvl1_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
* get_lvl1_lvl2_pble - calls get_lvl1 and get_lvl2 pble routine
* @pble_rsrc: pble resources
* @palloc: contains all inforamtion regarding pble (idx + pble addr)
- * @level1_only: flag for a level 1 PBLE
+ * @lvl: Bitmask for requested pble level
*/
static int get_lvl1_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
- struct irdma_pble_alloc *palloc, bool level1_only)
+ struct irdma_pble_alloc *palloc, u8 lvl)
{
int status = 0;
status = get_lvl1_pble(pble_rsrc, palloc);
- if (!status || level1_only || palloc->total_cnt <= PBLE_PER_PAGE)
+ if (!status || lvl == PBLE_LEVEL_1 || palloc->total_cnt <= PBLE_PER_PAGE)
return status;
status = get_lvl2_pble(pble_rsrc, palloc);
@@ -444,11 +444,11 @@ static int get_lvl1_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
* @pble_rsrc: pble resources
* @palloc: contains all inforamtion regarding pble (idx + pble addr)
* @pble_cnt: #of pbles requested
- * @level1_only: true if only pble level 1 to acquire
+ * @lvl: requested pble level mask
*/
int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc, u32 pble_cnt,
- bool level1_only)
+ u8 lvl)
{
int status = 0;
int max_sds = 0;
@@ -462,7 +462,7 @@ int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
/*check first to see if we can get pble's without acquiring
* additional sd's
*/
- status = get_lvl1_lvl2_pble(pble_rsrc, palloc, level1_only);
+ status = get_lvl1_lvl2_pble(pble_rsrc, palloc, lvl);
if (!status)
goto exit;
@@ -472,9 +472,9 @@ int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
if (status)
break;
- status = get_lvl1_lvl2_pble(pble_rsrc, palloc, level1_only);
+ status = get_lvl1_lvl2_pble(pble_rsrc, palloc, lvl);
/* if level1_only, only go through it once */
- if (!status || level1_only)
+ if (!status || lvl)
break;
}
diff --git a/drivers/infiniband/hw/irdma/pble.h b/drivers/infiniband/hw/irdma/pble.h
index 29d295463559..b31b7c5d66fe 100644
--- a/drivers/infiniband/hw/irdma/pble.h
+++ b/drivers/infiniband/hw/irdma/pble.h
@@ -114,7 +114,7 @@ void irdma_free_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc);
int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc, u32 pble_cnt,
- bool level1_only);
+ u8 lvl);
int irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,
struct irdma_chunk *pchunk);
int irdma_prm_get_pbles(struct irdma_pble_prm *pprm,
diff --git a/drivers/infiniband/hw/irdma/protos.h b/drivers/infiniband/hw/irdma/protos.h
index 9b6e919ae2a9..113096b60323 100644
--- a/drivers/infiniband/hw/irdma/protos.h
+++ b/drivers/infiniband/hw/irdma/protos.h
@@ -28,9 +28,7 @@ int irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev,
struct irdma_vsi_pestat *pestat);
void irdma_hw_stats_read_all(struct irdma_vsi_pestat *stats,
- struct irdma_dev_hw_stats *stats_values,
- u64 *hw_stats_regs_32, u64 *hw_stats_regs_64,
- u8 hw_rev);
+ const u64 *hw_stats_regs);
int irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd,
struct irdma_ws_node_info *node_info);
int irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq,
@@ -43,7 +41,9 @@ u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev);
void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id);
void irdma_update_stats(struct irdma_dev_hw_stats *hw_stats,
struct irdma_gather_stats *gather_stats,
- struct irdma_gather_stats *last_gather_stats);
+ struct irdma_gather_stats *last_gather_stats,
+ const struct irdma_hw_stat_map *map, u16 max_stat_idx);
+
/* vsi functions */
int irdma_vsi_stats_init(struct irdma_sc_vsi *vsi,
struct irdma_vsi_stats_info *info);
diff --git a/drivers/infiniband/hw/irdma/type.h b/drivers/infiniband/hw/irdma/type.h
index 517d41a1c289..5ee68604e59f 100644
--- a/drivers/infiniband/hw/irdma/type.h
+++ b/drivers/infiniband/hw/irdma/type.h
@@ -101,7 +101,8 @@ enum irdma_qp_event_type {
IRDMA_QP_EVENT_REQ_ERR,
};
-enum irdma_hw_stats_index_32b {
+enum irdma_hw_stats_index {
+ /* gen1 - 32-bit */
IRDMA_HW_STAT_INDEX_IP4RXDISCARD = 0,
IRDMA_HW_STAT_INDEX_IP4RXTRUNC = 1,
IRDMA_HW_STAT_INDEX_IP4TXNOROUTE = 2,
@@ -111,50 +112,48 @@ enum irdma_hw_stats_index_32b {
IRDMA_HW_STAT_INDEX_TCPRTXSEG = 6,
IRDMA_HW_STAT_INDEX_TCPRXOPTERR = 7,
IRDMA_HW_STAT_INDEX_TCPRXPROTOERR = 8,
- IRDMA_HW_STAT_INDEX_MAX_32_GEN_1 = 9, /* Must be same value as next entry */
IRDMA_HW_STAT_INDEX_RXVLANERR = 9,
- IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED = 10,
- IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED = 11,
- IRDMA_HW_STAT_INDEX_TXNPCNPSENT = 12,
- IRDMA_HW_STAT_INDEX_MAX_32, /* Must be last entry */
-};
-
-enum irdma_hw_stats_index_64b {
- IRDMA_HW_STAT_INDEX_IP4RXOCTS = 0,
- IRDMA_HW_STAT_INDEX_IP4RXPKTS = 1,
- IRDMA_HW_STAT_INDEX_IP4RXFRAGS = 2,
- IRDMA_HW_STAT_INDEX_IP4RXMCPKTS = 3,
- IRDMA_HW_STAT_INDEX_IP4TXOCTS = 4,
- IRDMA_HW_STAT_INDEX_IP4TXPKTS = 5,
- IRDMA_HW_STAT_INDEX_IP4TXFRAGS = 6,
- IRDMA_HW_STAT_INDEX_IP4TXMCPKTS = 7,
- IRDMA_HW_STAT_INDEX_IP6RXOCTS = 8,
- IRDMA_HW_STAT_INDEX_IP6RXPKTS = 9,
- IRDMA_HW_STAT_INDEX_IP6RXFRAGS = 10,
- IRDMA_HW_STAT_INDEX_IP6RXMCPKTS = 11,
- IRDMA_HW_STAT_INDEX_IP6TXOCTS = 12,
- IRDMA_HW_STAT_INDEX_IP6TXPKTS = 13,
- IRDMA_HW_STAT_INDEX_IP6TXFRAGS = 14,
- IRDMA_HW_STAT_INDEX_IP6TXMCPKTS = 15,
- IRDMA_HW_STAT_INDEX_TCPRXSEGS = 16,
- IRDMA_HW_STAT_INDEX_TCPTXSEG = 17,
- IRDMA_HW_STAT_INDEX_RDMARXRDS = 18,
- IRDMA_HW_STAT_INDEX_RDMARXSNDS = 19,
- IRDMA_HW_STAT_INDEX_RDMARXWRS = 20,
- IRDMA_HW_STAT_INDEX_RDMATXRDS = 21,
- IRDMA_HW_STAT_INDEX_RDMATXSNDS = 22,
- IRDMA_HW_STAT_INDEX_RDMATXWRS = 23,
- IRDMA_HW_STAT_INDEX_RDMAVBND = 24,
- IRDMA_HW_STAT_INDEX_RDMAVINV = 25,
- IRDMA_HW_STAT_INDEX_MAX_64_GEN_1 = 26, /* Must be same value as next entry */
- IRDMA_HW_STAT_INDEX_IP4RXMCOCTS = 26,
- IRDMA_HW_STAT_INDEX_IP4TXMCOCTS = 27,
- IRDMA_HW_STAT_INDEX_IP6RXMCOCTS = 28,
- IRDMA_HW_STAT_INDEX_IP6TXMCOCTS = 29,
- IRDMA_HW_STAT_INDEX_UDPRXPKTS = 30,
- IRDMA_HW_STAT_INDEX_UDPTXPKTS = 31,
- IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS = 32,
- IRDMA_HW_STAT_INDEX_MAX_64, /* Must be last entry */
+ /* gen1 - 64-bit */
+ IRDMA_HW_STAT_INDEX_IP4RXOCTS = 10,
+ IRDMA_HW_STAT_INDEX_IP4RXPKTS = 11,
+ IRDMA_HW_STAT_INDEX_IP4RXFRAGS = 12,
+ IRDMA_HW_STAT_INDEX_IP4RXMCPKTS = 13,
+ IRDMA_HW_STAT_INDEX_IP4TXOCTS = 14,
+ IRDMA_HW_STAT_INDEX_IP4TXPKTS = 15,
+ IRDMA_HW_STAT_INDEX_IP4TXFRAGS = 16,
+ IRDMA_HW_STAT_INDEX_IP4TXMCPKTS = 17,
+ IRDMA_HW_STAT_INDEX_IP6RXOCTS = 18,
+ IRDMA_HW_STAT_INDEX_IP6RXPKTS = 19,
+ IRDMA_HW_STAT_INDEX_IP6RXFRAGS = 20,
+ IRDMA_HW_STAT_INDEX_IP6RXMCPKTS = 21,
+ IRDMA_HW_STAT_INDEX_IP6TXOCTS = 22,
+ IRDMA_HW_STAT_INDEX_IP6TXPKTS = 23,
+ IRDMA_HW_STAT_INDEX_IP6TXFRAGS = 24,
+ IRDMA_HW_STAT_INDEX_IP6TXMCPKTS = 25,
+ IRDMA_HW_STAT_INDEX_TCPRXSEGS = 26,
+ IRDMA_HW_STAT_INDEX_TCPTXSEG = 27,
+ IRDMA_HW_STAT_INDEX_RDMARXRDS = 28,
+ IRDMA_HW_STAT_INDEX_RDMARXSNDS = 29,
+ IRDMA_HW_STAT_INDEX_RDMARXWRS = 30,
+ IRDMA_HW_STAT_INDEX_RDMATXRDS = 31,
+ IRDMA_HW_STAT_INDEX_RDMATXSNDS = 32,
+ IRDMA_HW_STAT_INDEX_RDMATXWRS = 33,
+ IRDMA_HW_STAT_INDEX_RDMAVBND = 34,
+ IRDMA_HW_STAT_INDEX_RDMAVINV = 35,
+ IRDMA_HW_STAT_INDEX_IP4RXMCOCTS = 36,
+ IRDMA_HW_STAT_INDEX_IP4TXMCOCTS = 37,
+ IRDMA_HW_STAT_INDEX_IP6RXMCOCTS = 38,
+ IRDMA_HW_STAT_INDEX_IP6TXMCOCTS = 39,
+ IRDMA_HW_STAT_INDEX_UDPRXPKTS = 40,
+ IRDMA_HW_STAT_INDEX_UDPTXPKTS = 41,
+ IRDMA_HW_STAT_INDEX_MAX_GEN_1 = 42, /* Must be same value as next entry */
+ /* gen2 - 64-bit */
+ IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS = 42,
+ /* gen2 - 32-bit */
+ IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED = 43,
+ IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED = 44,
+ IRDMA_HW_STAT_INDEX_TXNPCNPSENT = 45,
+ IRDMA_HW_STAT_INDEX_MAX_GEN_2 = 46,
};
enum irdma_feature_type {
@@ -274,65 +273,21 @@ struct irdma_cq_shadow_area {
};
struct irdma_dev_hw_stats_offsets {
- u32 stats_offset_32[IRDMA_HW_STAT_INDEX_MAX_32];
- u32 stats_offset_64[IRDMA_HW_STAT_INDEX_MAX_64];
+ u32 stats_offset[IRDMA_HW_STAT_INDEX_MAX_GEN_1];
};
struct irdma_dev_hw_stats {
- u64 stats_val_32[IRDMA_HW_STAT_INDEX_MAX_32];
- u64 stats_val_64[IRDMA_HW_STAT_INDEX_MAX_64];
+ u64 stats_val[IRDMA_GATHER_STATS_BUF_SIZE / sizeof(u64)];
};
struct irdma_gather_stats {
- u32 rsvd1;
- u32 rxvlanerr;
- u64 ip4rxocts;
- u64 ip4rxpkts;
- u32 ip4rxtrunc;
- u32 ip4rxdiscard;
- u64 ip4rxfrags;
- u64 ip4rxmcocts;
- u64 ip4rxmcpkts;
- u64 ip6rxocts;
- u64 ip6rxpkts;
- u32 ip6rxtrunc;
- u32 ip6rxdiscard;
- u64 ip6rxfrags;
- u64 ip6rxmcocts;
- u64 ip6rxmcpkts;
- u64 ip4txocts;
- u64 ip4txpkts;
- u64 ip4txfrag;
- u64 ip4txmcocts;
- u64 ip4txmcpkts;
- u64 ip6txocts;
- u64 ip6txpkts;
- u64 ip6txfrags;
- u64 ip6txmcocts;
- u64 ip6txmcpkts;
- u32 ip6txnoroute;
- u32 ip4txnoroute;
- u64 tcprxsegs;
- u32 tcprxprotoerr;
- u32 tcprxopterr;
- u64 tcptxsegs;
- u32 rsvd2;
- u32 tcprtxseg;
- u64 udprxpkts;
- u64 udptxpkts;
- u64 rdmarxwrs;
- u64 rdmarxrds;
- u64 rdmarxsnds;
- u64 rdmatxwrs;
- u64 rdmatxrds;
- u64 rdmatxsnds;
- u64 rdmavbn;
- u64 rdmavinv;
- u64 rxnpecnmrkpkts;
- u32 rxrpcnphandled;
- u32 rxrpcnpignored;
- u32 txnpcnpsent;
- u32 rsvd3[88];
+ u64 val[IRDMA_GATHER_STATS_BUF_SIZE / sizeof(u64)];
+};
+
+struct irdma_hw_stat_map {
+ u16 byteoff;
+ u8 bitoff;
+ u64 bitmask;
};
struct irdma_stats_gather_info {
@@ -584,7 +539,7 @@ struct irdma_qos {
bool valid;
};
-#define IRDMA_INVALID_FCN_ID 0xff
+#define IRDMA_INVALID_STATS_IDX 0xff
struct irdma_sc_vsi {
u16 vsi_idx;
struct irdma_sc_dev *dev;
@@ -598,11 +553,9 @@ struct irdma_sc_vsi {
u32 exception_lan_q;
u16 mtu;
u16 vm_id;
- u8 fcn_id;
enum irdma_vm_vf_type vm_vf_type;
- bool stats_fcn_id_alloc:1;
+ bool stats_inst_alloc:1;
bool tc_change_pending:1;
- struct irdma_qos qos[IRDMA_MAX_USER_PRIORITY];
struct irdma_vsi_pestat *pestat;
atomic_t qp_suspend_reqs;
int (*register_qset)(struct irdma_sc_vsi *vsi,
@@ -611,14 +564,17 @@ struct irdma_sc_vsi {
struct irdma_ws_node *tc_node);
u8 qos_rel_bw;
u8 qos_prio_type;
+ u8 stats_idx;
u8 dscp_map[IIDC_MAX_DSCP_MAPPING];
+ struct irdma_qos qos[IRDMA_MAX_USER_PRIORITY];
+ u64 hw_stats_regs[IRDMA_HW_STAT_INDEX_MAX_GEN_1];
bool dscp_mode:1;
};
struct irdma_sc_dev {
struct list_head cqp_cmd_head; /* head of the CQP command list */
spinlock_t cqp_lock; /* protect CQP list access */
- bool fcn_id_array[IRDMA_MAX_STATS_COUNT];
+ bool stats_idx_array[IRDMA_MAX_STATS_COUNT_GEN_1];
struct irdma_dma_mem vf_fpm_query_buf[IRDMA_MAX_PE_ENA_VF_COUNT];
u64 fpm_query_buf_pa;
u64 fpm_commit_buf_pa;
@@ -637,8 +593,8 @@ struct irdma_sc_dev {
u32 ceq_itr; /* Interrupt throttle, usecs between interrupts: 0 disabled. 2 - 8160 */
u64 hw_masks[IRDMA_MAX_MASKS];
u64 hw_shifts[IRDMA_MAX_SHIFTS];
- u64 hw_stats_regs_32[IRDMA_HW_STAT_INDEX_MAX_32];
- u64 hw_stats_regs_64[IRDMA_HW_STAT_INDEX_MAX_64];
+ const struct irdma_hw_stat_map *hw_stats_map;
+ u64 hw_stats_regs[IRDMA_HW_STAT_INDEX_MAX_GEN_1];
u64 feature_info[IRDMA_MAX_FEATURES];
u64 cqp_cmd_stats[IRDMA_MAX_CQP_OPS];
struct irdma_hw_attrs hw_attrs;
@@ -763,7 +719,7 @@ struct irdma_vsi_init_info {
struct irdma_vsi_stats_info {
struct irdma_vsi_pestat *pestat;
u8 fcn_id;
- bool alloc_fcn_id;
+ bool alloc_stats_inst;
};
struct irdma_device_init_info {
diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c
index 7887230c867b..71e1c5d34709 100644
--- a/drivers/infiniband/hw/irdma/utils.c
+++ b/drivers/infiniband/hw/irdma/utils.c
@@ -1634,10 +1634,10 @@ static void irdma_hw_stats_timeout(struct timer_list *t)
from_timer(pf_devstat, t, stats_timer);
struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi;
- if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
- irdma_cqp_gather_stats_gen1(sc_vsi->dev, sc_vsi->pestat);
- else
+ if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false);
+ else
+ irdma_cqp_gather_stats_gen1(sc_vsi->dev, sc_vsi->pestat);
mod_timer(&pf_devstat->stats_timer,
jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
@@ -1686,164 +1686,28 @@ void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev,
{
struct irdma_gather_stats *gather_stats =
pestat->gather_info.gather_stats_va;
+ const struct irdma_hw_stat_map *map = dev->hw_stats_map;
+ u16 max_stats_idx = dev->hw_attrs.max_stat_idx;
u32 stats_inst_offset_32;
u32 stats_inst_offset_64;
+ u64 new_val;
+ u16 i;
stats_inst_offset_32 = (pestat->gather_info.use_stats_inst) ?
- pestat->gather_info.stats_inst_index :
- pestat->hw->hmc.hmc_fn_id;
+ pestat->gather_info.stats_inst_index :
+ pestat->hw->hmc.hmc_fn_id;
stats_inst_offset_32 *= 4;
stats_inst_offset_64 = stats_inst_offset_32 * 2;
- gather_stats->rxvlanerr =
- rd32(dev->hw,
- dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_RXVLANERR]
- + stats_inst_offset_32);
- gather_stats->ip4rxdiscard =
- rd32(dev->hw,
- dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4RXDISCARD]
- + stats_inst_offset_32);
- gather_stats->ip4rxtrunc =
- rd32(dev->hw,
- dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4RXTRUNC]
- + stats_inst_offset_32);
- gather_stats->ip4txnoroute =
- rd32(dev->hw,
- dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4TXNOROUTE]
- + stats_inst_offset_32);
- gather_stats->ip6rxdiscard =
- rd32(dev->hw,
- dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6RXDISCARD]
- + stats_inst_offset_32);
- gather_stats->ip6rxtrunc =
- rd32(dev->hw,
- dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6RXTRUNC]
- + stats_inst_offset_32);
- gather_stats->ip6txnoroute =
- rd32(dev->hw,
- dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6TXNOROUTE]
- + stats_inst_offset_32);
- gather_stats->tcprtxseg =
- rd32(dev->hw,
- dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_TCPRTXSEG]
- + stats_inst_offset_32);
- gather_stats->tcprxopterr =
- rd32(dev->hw,
- dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_TCPRXOPTERR]
- + stats_inst_offset_32);
-
- gather_stats->ip4rxocts =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXOCTS]
- + stats_inst_offset_64);
- gather_stats->ip4rxpkts =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXPKTS]
- + stats_inst_offset_64);
- gather_stats->ip4txfrag =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXFRAGS]
- + stats_inst_offset_64);
- gather_stats->ip4rxmcpkts =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXMCPKTS]
- + stats_inst_offset_64);
- gather_stats->ip4txocts =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXOCTS]
- + stats_inst_offset_64);
- gather_stats->ip4txpkts =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXPKTS]
- + stats_inst_offset_64);
- gather_stats->ip4txfrag =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXFRAGS]
- + stats_inst_offset_64);
- gather_stats->ip4txmcpkts =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXMCPKTS]
- + stats_inst_offset_64);
- gather_stats->ip6rxocts =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXOCTS]
- + stats_inst_offset_64);
- gather_stats->ip6rxpkts =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXPKTS]
- + stats_inst_offset_64);
- gather_stats->ip6txfrags =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXFRAGS]
- + stats_inst_offset_64);
- gather_stats->ip6rxmcpkts =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXMCPKTS]
- + stats_inst_offset_64);
- gather_stats->ip6txocts =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXOCTS]
- + stats_inst_offset_64);
- gather_stats->ip6txpkts =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXPKTS]
- + stats_inst_offset_64);
- gather_stats->ip6txfrags =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXFRAGS]
- + stats_inst_offset_64);
- gather_stats->ip6txmcpkts =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXMCPKTS]
- + stats_inst_offset_64);
- gather_stats->tcprxsegs =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_TCPRXSEGS]
- + stats_inst_offset_64);
- gather_stats->tcptxsegs =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_TCPTXSEG]
- + stats_inst_offset_64);
- gather_stats->rdmarxrds =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXRDS]
- + stats_inst_offset_64);
- gather_stats->rdmarxsnds =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXSNDS]
- + stats_inst_offset_64);
- gather_stats->rdmarxwrs =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXWRS]
- + stats_inst_offset_64);
- gather_stats->rdmatxrds =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXRDS]
- + stats_inst_offset_64);
- gather_stats->rdmatxsnds =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXSNDS]
- + stats_inst_offset_64);
- gather_stats->rdmatxwrs =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXWRS]
- + stats_inst_offset_64);
- gather_stats->rdmavbn =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMAVBND]
- + stats_inst_offset_64);
- gather_stats->rdmavinv =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMAVINV]
- + stats_inst_offset_64);
- gather_stats->udprxpkts =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_UDPRXPKTS]
- + stats_inst_offset_64);
- gather_stats->udptxpkts =
- rd64(dev->hw,
- dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_UDPTXPKTS]
- + stats_inst_offset_64);
+ for (i = 0; i < max_stats_idx; i++) {
+ if (map[i].bitmask <= IRDMA_MAX_STATS_32)
+ new_val = rd32(dev->hw,
+ dev->hw_stats_regs[i] + stats_inst_offset_32);
+ else
+ new_val = rd64(dev->hw,
+ dev->hw_stats_regs[i] + stats_inst_offset_64);
+ gather_stats->val[map[i].byteoff / sizeof(u64)] = new_val;
+ }
irdma_process_stats(pestat);
}
diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c
index 1b2e3e800c9a..ab5cdf782785 100644
--- a/drivers/infiniband/hw/irdma/verbs.c
+++ b/drivers/infiniband/hw/irdma/verbs.c
@@ -1226,10 +1226,6 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,
udp_info->ipv4 = false;
irdma_copy_ip_ntohl(local_ip, daddr);
- udp_info->arp_idx = irdma_arp_table(iwdev->rf,
- &local_ip[0],
- false, NULL,
- IRDMA_ARP_RESOLVE);
} else if (av->net_type == RDMA_NETWORK_IPV4) {
__be32 saddr = av->sgid_addr.saddr_in.sin_addr.s_addr;
__be32 daddr = av->dgid_addr.saddr_in.sin_addr.s_addr;
@@ -2329,11 +2325,10 @@ static bool irdma_check_mr_contiguous(struct irdma_pble_alloc *palloc,
* irdma_setup_pbles - copy user pg address to pble's
* @rf: RDMA PCI function
* @iwmr: mr pointer for this memory registration
- * @use_pbles: flag if to use pble's
- * @lvl_1_only: request only level 1 pble if true
+ * @lvl: requested pble levels
*/
static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr,
- bool use_pbles, bool lvl_1_only)
+ u8 lvl)
{
struct irdma_pbl *iwpbl = &iwmr->iwpbl;
struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
@@ -2342,9 +2337,9 @@ static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr,
int status;
enum irdma_pble_level level = PBLE_LEVEL_1;
- if (use_pbles) {
+ if (lvl) {
status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt,
- lvl_1_only);
+ lvl);
if (status)
return status;
@@ -2359,7 +2354,7 @@ static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr,
irdma_copy_user_pgaddrs(iwmr, pbl, level);
- if (use_pbles)
+ if (lvl)
iwmr->pgaddrmem[0] = *pbl;
return 0;
@@ -2370,11 +2365,11 @@ static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr,
* @iwdev: irdma device
* @req: information for q memory management
* @iwpbl: pble struct
- * @use_pbles: flag to use pble
+ * @lvl: pble level mask
*/
static int irdma_handle_q_mem(struct irdma_device *iwdev,
struct irdma_mem_reg_req *req,
- struct irdma_pbl *iwpbl, bool use_pbles)
+ struct irdma_pbl *iwpbl, u8 lvl)
{
struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
struct irdma_mr *iwmr = iwpbl->iwmr;
@@ -2387,11 +2382,11 @@ static int irdma_handle_q_mem(struct irdma_device *iwdev,
bool ret = true;
pg_size = iwmr->page_size;
- err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, true);
+ err = irdma_setup_pbles(iwdev->rf, iwmr, lvl);
if (err)
return err;
- if (use_pbles)
+ if (lvl)
arr = palloc->level1.addr;
switch (iwmr->type) {
@@ -2400,7 +2395,7 @@ static int irdma_handle_q_mem(struct irdma_device *iwdev,
hmc_p = &qpmr->sq_pbl;
qpmr->shadow = (dma_addr_t)arr[total];
- if (use_pbles) {
+ if (lvl) {
ret = irdma_check_mem_contiguous(arr, req->sq_pages,
pg_size);
if (ret)
@@ -2425,7 +2420,7 @@ static int irdma_handle_q_mem(struct irdma_device *iwdev,
if (!cqmr->split)
cqmr->shadow = (dma_addr_t)arr[req->cq_pages];
- if (use_pbles)
+ if (lvl)
ret = irdma_check_mem_contiguous(arr, req->cq_pages,
pg_size);
@@ -2439,7 +2434,7 @@ static int irdma_handle_q_mem(struct irdma_device *iwdev,
err = -EINVAL;
}
- if (use_pbles && ret) {
+ if (lvl && ret) {
irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
iwpbl->pbl_allocated = false;
}
@@ -2749,17 +2744,17 @@ static int irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access)
{
struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);
struct irdma_pbl *iwpbl = &iwmr->iwpbl;
- bool use_pbles;
u32 stag;
+ u8 lvl;
int err;
- use_pbles = iwmr->page_cnt != 1;
+ lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0;
- err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, false);
+ err = irdma_setup_pbles(iwdev->rf, iwmr, lvl);
if (err)
return err;
- if (use_pbles) {
+ if (lvl) {
err = irdma_check_mr_contiguous(&iwpbl->pble_alloc,
iwmr->page_size);
if (err) {
@@ -2843,17 +2838,17 @@ static int irdma_reg_user_mr_type_qp(struct irdma_mem_reg_req req,
struct irdma_pbl *iwpbl = &iwmr->iwpbl;
struct irdma_ucontext *ucontext = NULL;
unsigned long flags;
- bool use_pbles;
u32 total;
int err;
+ u8 lvl;
total = req.sq_pages + req.rq_pages + 1;
if (total > iwmr->page_cnt)
return -EINVAL;
total = req.sq_pages + req.rq_pages;
- use_pbles = total > 2;
- err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
+ lvl = total > 2 ? PBLE_LEVEL_1 : PBLE_LEVEL_0;
+ err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl);
if (err)
return err;
@@ -2876,9 +2871,9 @@ static int irdma_reg_user_mr_type_cq(struct irdma_mem_reg_req req,
struct irdma_ucontext *ucontext = NULL;
u8 shadow_pgcnt = 1;
unsigned long flags;
- bool use_pbles;
u32 total;
int err;
+ u8 lvl;
if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE)
shadow_pgcnt = 0;
@@ -2886,8 +2881,8 @@ static int irdma_reg_user_mr_type_cq(struct irdma_mem_reg_req req,
if (total > iwmr->page_cnt)
return -EINVAL;
- use_pbles = req.cq_pages > 1;
- err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
+ lvl = req.cq_pages > 1 ? PBLE_LEVEL_1 : PBLE_LEVEL_0;
+ err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl);
if (err)
return err;
@@ -3708,89 +3703,59 @@ static int irdma_iw_port_immutable(struct ib_device *ibdev, u32 port_num,
return 0;
}
-static const struct rdma_stat_desc irdma_hw_stat_descs[] = {
- /* 32bit names */
- [IRDMA_HW_STAT_INDEX_RXVLANERR].name = "rxVlanErrors",
- [IRDMA_HW_STAT_INDEX_IP4RXDISCARD].name = "ip4InDiscards",
- [IRDMA_HW_STAT_INDEX_IP4RXTRUNC].name = "ip4InTruncatedPkts",
- [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE].name = "ip4OutNoRoutes",
- [IRDMA_HW_STAT_INDEX_IP6RXDISCARD].name = "ip6InDiscards",
- [IRDMA_HW_STAT_INDEX_IP6RXTRUNC].name = "ip6InTruncatedPkts",
- [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE].name = "ip6OutNoRoutes",
- [IRDMA_HW_STAT_INDEX_TCPRTXSEG].name = "tcpRetransSegs",
- [IRDMA_HW_STAT_INDEX_TCPRXOPTERR].name = "tcpInOptErrors",
- [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR].name = "tcpInProtoErrors",
- [IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED].name = "cnpHandled",
- [IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED].name = "cnpIgnored",
- [IRDMA_HW_STAT_INDEX_TXNPCNPSENT].name = "cnpSent",
-
- /* 64bit names */
- [IRDMA_HW_STAT_INDEX_IP4RXOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "ip4InOctets",
- [IRDMA_HW_STAT_INDEX_IP4RXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "ip4InPkts",
- [IRDMA_HW_STAT_INDEX_IP4RXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "ip4InReasmRqd",
- [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "ip4InMcastOctets",
- [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "ip4InMcastPkts",
- [IRDMA_HW_STAT_INDEX_IP4TXOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "ip4OutOctets",
- [IRDMA_HW_STAT_INDEX_IP4TXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "ip4OutPkts",
- [IRDMA_HW_STAT_INDEX_IP4TXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "ip4OutSegRqd",
- [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "ip4OutMcastOctets",
- [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "ip4OutMcastPkts",
- [IRDMA_HW_STAT_INDEX_IP6RXOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "ip6InOctets",
- [IRDMA_HW_STAT_INDEX_IP6RXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "ip6InPkts",
- [IRDMA_HW_STAT_INDEX_IP6RXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "ip6InReasmRqd",
- [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "ip6InMcastOctets",
- [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "ip6InMcastPkts",
- [IRDMA_HW_STAT_INDEX_IP6TXOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "ip6OutOctets",
- [IRDMA_HW_STAT_INDEX_IP6TXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "ip6OutPkts",
- [IRDMA_HW_STAT_INDEX_IP6TXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "ip6OutSegRqd",
- [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "ip6OutMcastOctets",
- [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "ip6OutMcastPkts",
- [IRDMA_HW_STAT_INDEX_TCPRXSEGS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "tcpInSegs",
- [IRDMA_HW_STAT_INDEX_TCPTXSEG + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "tcpOutSegs",
- [IRDMA_HW_STAT_INDEX_RDMARXRDS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "iwInRdmaReads",
- [IRDMA_HW_STAT_INDEX_RDMARXSNDS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "iwInRdmaSends",
- [IRDMA_HW_STAT_INDEX_RDMARXWRS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "iwInRdmaWrites",
- [IRDMA_HW_STAT_INDEX_RDMATXRDS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "iwOutRdmaReads",
- [IRDMA_HW_STAT_INDEX_RDMATXSNDS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "iwOutRdmaSends",
- [IRDMA_HW_STAT_INDEX_RDMATXWRS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "iwOutRdmaWrites",
- [IRDMA_HW_STAT_INDEX_RDMAVBND + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "iwRdmaBnd",
- [IRDMA_HW_STAT_INDEX_RDMAVINV + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "iwRdmaInv",
- [IRDMA_HW_STAT_INDEX_UDPRXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "RxUDP",
- [IRDMA_HW_STAT_INDEX_UDPTXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name =
- "TxUDP",
- [IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS + IRDMA_HW_STAT_INDEX_MAX_32]
- .name = "RxECNMrkd",
+static const struct rdma_stat_desc irdma_hw_stat_names[] = {
+ /* gen1 - 32-bit */
+ [IRDMA_HW_STAT_INDEX_IP4RXDISCARD].name = "ip4InDiscards",
+ [IRDMA_HW_STAT_INDEX_IP4RXTRUNC].name = "ip4InTruncatedPkts",
+ [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE].name = "ip4OutNoRoutes",
+ [IRDMA_HW_STAT_INDEX_IP6RXDISCARD].name = "ip6InDiscards",
+ [IRDMA_HW_STAT_INDEX_IP6RXTRUNC].name = "ip6InTruncatedPkts",
+ [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE].name = "ip6OutNoRoutes",
+ [IRDMA_HW_STAT_INDEX_TCPRTXSEG].name = "tcpRetransSegs",
+ [IRDMA_HW_STAT_INDEX_TCPRXOPTERR].name = "tcpInOptErrors",
+ [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR].name = "tcpInProtoErrors",
+ [IRDMA_HW_STAT_INDEX_RXVLANERR].name = "rxVlanErrors",
+ /* gen1 - 64-bit */
+ [IRDMA_HW_STAT_INDEX_IP4RXOCTS].name = "ip4InOctets",
+ [IRDMA_HW_STAT_INDEX_IP4RXPKTS].name = "ip4InPkts",
+ [IRDMA_HW_STAT_INDEX_IP4RXFRAGS].name = "ip4InReasmRqd",
+ [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS].name = "ip4InMcastPkts",
+ [IRDMA_HW_STAT_INDEX_IP4TXOCTS].name = "ip4OutOctets",
+ [IRDMA_HW_STAT_INDEX_IP4TXPKTS].name = "ip4OutPkts",
+ [IRDMA_HW_STAT_INDEX_IP4TXFRAGS].name = "ip4OutSegRqd",
+ [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS].name = "ip4OutMcastPkts",
+ [IRDMA_HW_STAT_INDEX_IP6RXOCTS].name = "ip6InOctets",
+ [IRDMA_HW_STAT_INDEX_IP6RXPKTS].name = "ip6InPkts",
+ [IRDMA_HW_STAT_INDEX_IP6RXFRAGS].name = "ip6InReasmRqd",
+ [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS].name = "ip6InMcastPkts",
+ [IRDMA_HW_STAT_INDEX_IP6TXOCTS].name = "ip6OutOctets",
+ [IRDMA_HW_STAT_INDEX_IP6TXPKTS].name = "ip6OutPkts",
+ [IRDMA_HW_STAT_INDEX_IP6TXFRAGS].name = "ip6OutSegRqd",
+ [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS].name = "ip6OutMcastPkts",
+ [IRDMA_HW_STAT_INDEX_TCPRXSEGS].name = "tcpInSegs",
+ [IRDMA_HW_STAT_INDEX_TCPTXSEG].name = "tcpOutSegs",
+ [IRDMA_HW_STAT_INDEX_RDMARXRDS].name = "iwInRdmaReads",
+ [IRDMA_HW_STAT_INDEX_RDMARXSNDS].name = "iwInRdmaSends",
+ [IRDMA_HW_STAT_INDEX_RDMARXWRS].name = "iwInRdmaWrites",
+ [IRDMA_HW_STAT_INDEX_RDMATXRDS].name = "iwOutRdmaReads",
+ [IRDMA_HW_STAT_INDEX_RDMATXSNDS].name = "iwOutRdmaSends",
+ [IRDMA_HW_STAT_INDEX_RDMATXWRS].name = "iwOutRdmaWrites",
+ [IRDMA_HW_STAT_INDEX_RDMAVBND].name = "iwRdmaBnd",
+ [IRDMA_HW_STAT_INDEX_RDMAVINV].name = "iwRdmaInv",
+
+ /* gen2 - 32-bit */
+ [IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED].name = "cnpHandled",
+ [IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED].name = "cnpIgnored",
+ [IRDMA_HW_STAT_INDEX_TXNPCNPSENT].name = "cnpSent",
+ /* gen2 - 64-bit */
+ [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS].name = "ip4InMcastOctets",
+ [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS].name = "ip4OutMcastOctets",
+ [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS].name = "ip6InMcastOctets",
+ [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS].name = "ip6OutMcastOctets",
+ [IRDMA_HW_STAT_INDEX_UDPRXPKTS].name = "RxUDP",
+ [IRDMA_HW_STAT_INDEX_UDPTXPKTS].name = "TxUDP",
+ [IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS].name = "RxECNMrkd",
+
};
static void irdma_get_dev_fw_str(struct ib_device *dev, char *str)
@@ -3810,14 +3775,13 @@ static void irdma_get_dev_fw_str(struct ib_device *dev, char *str)
static struct rdma_hw_stats *irdma_alloc_hw_port_stats(struct ib_device *ibdev,
u32 port_num)
{
- int num_counters = IRDMA_HW_STAT_INDEX_MAX_32 +
- IRDMA_HW_STAT_INDEX_MAX_64;
- unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN;
+ struct irdma_device *iwdev = to_iwdev(ibdev);
+ struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
- BUILD_BUG_ON(ARRAY_SIZE(irdma_hw_stat_descs) !=
- (IRDMA_HW_STAT_INDEX_MAX_32 + IRDMA_HW_STAT_INDEX_MAX_64));
+ int num_counters = dev->hw_attrs.max_stat_idx;
+ unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN;
- return rdma_alloc_hw_stats_struct(irdma_hw_stat_descs, num_counters,
+ return rdma_alloc_hw_stats_struct(irdma_hw_stat_names, num_counters,
lifespan);
}
@@ -3840,7 +3804,7 @@ static int irdma_get_hw_stats(struct ib_device *ibdev,
else
irdma_cqp_gather_stats_gen1(&iwdev->rf->sc_dev, iwdev->vsi.pestat);
- memcpy(&stats->value[0], hw_stats, sizeof(*hw_stats));
+ memcpy(&stats->value[0], hw_stats, sizeof(u64) * stats->num_counters);
return stats->num_counters;
}
@@ -4054,7 +4018,7 @@ static int irdma_attach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid)
mc_qht_elem->mc_grp_ctx.vlan_id = vlan_id;
if (vlan_id < VLAN_N_VID)
mc_qht_elem->mc_grp_ctx.vlan_valid = true;
- mc_qht_elem->mc_grp_ctx.hmc_fcn_id = iwdev->vsi.fcn_id;
+ mc_qht_elem->mc_grp_ctx.hmc_fcn_id = iwdev->rf->sc_dev.hmc_fn_id;
mc_qht_elem->mc_grp_ctx.qs_handle =
iwqp->sc_qp.vsi->qos[iwqp->sc_qp.user_pri].qs_handle;
ether_addr_copy(mc_qht_elem->mc_grp_ctx.dest_mac_addr, dmac);
diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
index 884825b2e5f7..456656617c33 100644
--- a/drivers/infiniband/hw/mlx4/qp.c
+++ b/drivers/infiniband/hw/mlx4/qp.c
@@ -447,9 +447,13 @@ static int set_user_sq_size(struct mlx4_ib_dev *dev,
struct mlx4_ib_qp *qp,
struct mlx4_ib_create_qp *ucmd)
{
+ u32 cnt;
+
/* Sanity check SQ size before proceeding */
- if ((1 << ucmd->log_sq_bb_count) > dev->dev->caps.max_wqes ||
- ucmd->log_sq_stride >
+ if (check_shl_overflow(1, ucmd->log_sq_bb_count, &cnt) ||
+ cnt > dev->dev->caps.max_wqes)
+ return -EINVAL;
+ if (ucmd->log_sq_stride >
ilog2(roundup_pow_of_two(dev->dev->caps.max_sq_desc_sz)) ||
ucmd->log_sq_stride < MLX4_IB_MIN_SQ_STRIDE)
return -EINVAL;
diff --git a/drivers/infiniband/hw/mlx5/counters.c b/drivers/infiniband/hw/mlx5/counters.c
index 3e1272695d99..1c06920505d2 100644
--- a/drivers/infiniband/hw/mlx5/counters.c
+++ b/drivers/infiniband/hw/mlx5/counters.c
@@ -5,6 +5,7 @@
#include "mlx5_ib.h"
#include <linux/mlx5/eswitch.h>
+#include <linux/mlx5/vport.h>
#include "counters.h"
#include "ib_rep.h"
#include "qp.h"
@@ -18,6 +19,10 @@ struct mlx5_ib_counter {
#define INIT_Q_COUNTER(_name) \
{ .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)}
+#define INIT_VPORT_Q_COUNTER(_name) \
+ { .name = "vport_" #_name, .offset = \
+ MLX5_BYTE_OFF(query_q_counter_out, _name)}
+
static const struct mlx5_ib_counter basic_q_cnts[] = {
INIT_Q_COUNTER(rx_write_requests),
INIT_Q_COUNTER(rx_read_requests),
@@ -37,6 +42,25 @@ static const struct mlx5_ib_counter retrans_q_cnts[] = {
INIT_Q_COUNTER(local_ack_timeout_err),
};
+static const struct mlx5_ib_counter vport_basic_q_cnts[] = {
+ INIT_VPORT_Q_COUNTER(rx_write_requests),
+ INIT_VPORT_Q_COUNTER(rx_read_requests),
+ INIT_VPORT_Q_COUNTER(rx_atomic_requests),
+ INIT_VPORT_Q_COUNTER(out_of_buffer),
+};
+
+static const struct mlx5_ib_counter vport_out_of_seq_q_cnts[] = {
+ INIT_VPORT_Q_COUNTER(out_of_sequence),
+};
+
+static const struct mlx5_ib_counter vport_retrans_q_cnts[] = {
+ INIT_VPORT_Q_COUNTER(duplicate_request),
+ INIT_VPORT_Q_COUNTER(rnr_nak_retry_err),
+ INIT_VPORT_Q_COUNTER(packet_seq_err),
+ INIT_VPORT_Q_COUNTER(implied_nak_seq_err),
+ INIT_VPORT_Q_COUNTER(local_ack_timeout_err),
+};
+
#define INIT_CONG_COUNTER(_name) \
{ .name = #_name, .offset = \
MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)}
@@ -67,6 +91,25 @@ static const struct mlx5_ib_counter roce_accl_cnts[] = {
INIT_Q_COUNTER(roce_slow_restart_trans),
};
+static const struct mlx5_ib_counter vport_extended_err_cnts[] = {
+ INIT_VPORT_Q_COUNTER(resp_local_length_error),
+ INIT_VPORT_Q_COUNTER(resp_cqe_error),
+ INIT_VPORT_Q_COUNTER(req_cqe_error),
+ INIT_VPORT_Q_COUNTER(req_remote_invalid_request),
+ INIT_VPORT_Q_COUNTER(req_remote_access_errors),
+ INIT_VPORT_Q_COUNTER(resp_remote_access_errors),
+ INIT_VPORT_Q_COUNTER(resp_cqe_flush_error),
+ INIT_VPORT_Q_COUNTER(req_cqe_flush_error),
+};
+
+static const struct mlx5_ib_counter vport_roce_accl_cnts[] = {
+ INIT_VPORT_Q_COUNTER(roce_adp_retrans),
+ INIT_VPORT_Q_COUNTER(roce_adp_retrans_to),
+ INIT_VPORT_Q_COUNTER(roce_slow_restart),
+ INIT_VPORT_Q_COUNTER(roce_slow_restart_cnps),
+ INIT_VPORT_Q_COUNTER(roce_slow_restart_trans),
+};
+
#define INIT_EXT_PPCNT_COUNTER(_name) \
{ .name = #_name, .offset = \
MLX5_BYTE_OFF(ppcnt_reg, \
@@ -153,12 +196,20 @@ static int mlx5_ib_create_counters(struct ib_counters *counters,
return 0;
}
+static bool vport_qcounters_supported(struct mlx5_ib_dev *dev)
+{
+ return MLX5_CAP_GEN(dev->mdev, q_counter_other_vport) &&
+ MLX5_CAP_GEN(dev->mdev, q_counter_aggregation);
+}
static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev,
u32 port_num)
{
- return is_mdev_switchdev_mode(dev->mdev) ? &dev->port[0].cnts :
- &dev->port[port_num].cnts;
+ if ((is_mdev_switchdev_mode(dev->mdev) &&
+ !vport_qcounters_supported(dev)) || !port_num)
+ return &dev->port[0].cnts;
+
+ return &dev->port[port_num - 1].cnts;
}
/**
@@ -172,7 +223,7 @@ static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev,
*/
u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u32 port_num)
{
- const struct mlx5_ib_counters *cnts = get_counters(dev, port_num);
+ const struct mlx5_ib_counters *cnts = get_counters(dev, port_num + 1);
return cnts->set_id;
}
@@ -270,12 +321,44 @@ free:
return ret;
}
+static int mlx5_ib_query_q_counters_vport(struct mlx5_ib_dev *dev,
+ u32 port_num,
+ const struct mlx5_ib_counters *cnts,
+ struct rdma_hw_stats *stats)
+
+{
+ u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {};
+ u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {};
+ __be32 val;
+ int ret, i;
+
+ if (!dev->port[port_num].rep ||
+ dev->port[port_num].rep->vport == MLX5_VPORT_UPLINK)
+ return 0;
+
+ MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);
+ MLX5_SET(query_q_counter_in, in, other_vport, 1);
+ MLX5_SET(query_q_counter_in, in, vport_number,
+ dev->port[port_num].rep->vport);
+ MLX5_SET(query_q_counter_in, in, aggregate, 1);
+ ret = mlx5_cmd_exec_inout(dev->mdev, query_q_counter, in, out);
+ if (ret)
+ return ret;
+
+ for (i = 0; i < cnts->num_q_counters; i++) {
+ val = *(__be32 *)((void *)out + cnts->offsets[i]);
+ stats->value[i] = (u64)be32_to_cpu(val);
+ }
+
+ return 0;
+}
+
static int do_get_hw_stats(struct ib_device *ibdev,
struct rdma_hw_stats *stats,
u32 port_num, int index)
{
struct mlx5_ib_dev *dev = to_mdev(ibdev);
- const struct mlx5_ib_counters *cnts = get_counters(dev, port_num - 1);
+ const struct mlx5_ib_counters *cnts = get_counters(dev, port_num);
struct mlx5_core_dev *mdev;
int ret, num_counters;
@@ -286,11 +369,19 @@ static int do_get_hw_stats(struct ib_device *ibdev,
cnts->num_cong_counters +
cnts->num_ext_ppcnt_counters;
- /* q_counters are per IB device, query the master mdev */
- ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats, cnts->set_id);
+ if (is_mdev_switchdev_mode(dev->mdev) && dev->is_rep && port_num != 0)
+ ret = mlx5_ib_query_q_counters_vport(dev, port_num - 1, cnts,
+ stats);
+ else
+ ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats,
+ cnts->set_id);
if (ret)
return ret;
+ /* We don't expose device counters over Vports */
+ if (is_mdev_switchdev_mode(dev->mdev) && port_num != 0)
+ goto done;
+
if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
ret = mlx5_ib_query_ext_ppcnt_counters(dev, cnts, stats);
if (ret)
@@ -335,7 +426,8 @@ static int do_get_op_stat(struct ib_device *ibdev,
u32 type;
int ret;
- cnts = get_counters(dev, port_num - 1);
+ cnts = get_counters(dev, port_num);
+
opfcs = cnts->opfcs;
type = *(u32 *)cnts->descs[index].priv;
if (type >= MLX5_IB_OPCOUNTER_MAX)
@@ -362,7 +454,7 @@ static int do_get_op_stats(struct ib_device *ibdev,
const struct mlx5_ib_counters *cnts;
int index, ret, num_hw_counters;
- cnts = get_counters(dev, port_num - 1);
+ cnts = get_counters(dev, port_num);
num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
cnts->num_ext_ppcnt_counters;
for (index = num_hw_counters;
@@ -383,7 +475,7 @@ static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,
struct mlx5_ib_dev *dev = to_mdev(ibdev);
const struct mlx5_ib_counters *cnts;
- cnts = get_counters(dev, port_num - 1);
+ cnts = get_counters(dev, port_num);
num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
cnts->num_ext_ppcnt_counters;
num_counters = num_hw_counters + cnts->num_op_counters;
@@ -410,8 +502,7 @@ static struct rdma_hw_stats *
mlx5_ib_counter_alloc_stats(struct rdma_counter *counter)
{
struct mlx5_ib_dev *dev = to_mdev(counter->device);
- const struct mlx5_ib_counters *cnts =
- get_counters(dev, counter->port - 1);
+ const struct mlx5_ib_counters *cnts = get_counters(dev, counter->port);
return do_alloc_stats(cnts);
}
@@ -419,8 +510,7 @@ mlx5_ib_counter_alloc_stats(struct rdma_counter *counter)
static int mlx5_ib_counter_update_stats(struct rdma_counter *counter)
{
struct mlx5_ib_dev *dev = to_mdev(counter->device);
- const struct mlx5_ib_counters *cnts =
- get_counters(dev, counter->port - 1);
+ const struct mlx5_ib_counters *cnts = get_counters(dev, counter->port);
return mlx5_ib_query_q_counters(dev->mdev, cnts,
counter->stats, counter->id);
@@ -479,44 +569,55 @@ static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp)
}
static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,
- struct rdma_stat_desc *descs, size_t *offsets)
+ struct rdma_stat_desc *descs, size_t *offsets,
+ u32 port_num)
{
- int i;
- int j = 0;
+ bool is_vport = is_mdev_switchdev_mode(dev->mdev) &&
+ port_num != MLX5_VPORT_PF;
+ const struct mlx5_ib_counter *names;
+ int j = 0, i;
+ names = is_vport ? vport_basic_q_cnts : basic_q_cnts;
for (i = 0; i < ARRAY_SIZE(basic_q_cnts); i++, j++) {
- descs[j].name = basic_q_cnts[i].name;
+ descs[j].name = names[i].name;
offsets[j] = basic_q_cnts[i].offset;
}
+ names = is_vport ? vport_out_of_seq_q_cnts : out_of_seq_q_cnts;
if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) {
for (i = 0; i < ARRAY_SIZE(out_of_seq_q_cnts); i++, j++) {
- descs[j].name = out_of_seq_q_cnts[i].name;
+ descs[j].name = names[i].name;
offsets[j] = out_of_seq_q_cnts[i].offset;
}
}
+ names = is_vport ? vport_retrans_q_cnts : retrans_q_cnts;
if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) {
for (i = 0; i < ARRAY_SIZE(retrans_q_cnts); i++, j++) {
- descs[j].name = retrans_q_cnts[i].name;
+ descs[j].name = names[i].name;
offsets[j] = retrans_q_cnts[i].offset;
}
}
+ names = is_vport ? vport_extended_err_cnts : extended_err_cnts;
if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) {
for (i = 0; i < ARRAY_SIZE(extended_err_cnts); i++, j++) {
- descs[j].name = extended_err_cnts[i].name;
+ descs[j].name = names[i].name;
offsets[j] = extended_err_cnts[i].offset;
}
}
+ names = is_vport ? vport_roce_accl_cnts : roce_accl_cnts;
if (MLX5_CAP_GEN(dev->mdev, roce_accl)) {
for (i = 0; i < ARRAY_SIZE(roce_accl_cnts); i++, j++) {
- descs[j].name = roce_accl_cnts[i].name;
+ descs[j].name = names[i].name;
offsets[j] = roce_accl_cnts[i].offset;
}
}
+ if (is_vport)
+ return;
+
if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) {
descs[j].name = cong_cnts[i].name;
@@ -558,9 +659,9 @@ static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,
static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
- struct mlx5_ib_counters *cnts)
+ struct mlx5_ib_counters *cnts, u32 port_num)
{
- u32 num_counters, num_op_counters;
+ u32 num_counters, num_op_counters = 0;
num_counters = ARRAY_SIZE(basic_q_cnts);
@@ -578,6 +679,9 @@ static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
cnts->num_q_counters = num_counters;
+ if (is_mdev_switchdev_mode(dev->mdev) && port_num != MLX5_VPORT_PF)
+ goto skip_non_qcounters;
+
if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
cnts->num_cong_counters = ARRAY_SIZE(cong_cnts);
num_counters += ARRAY_SIZE(cong_cnts);
@@ -597,6 +701,7 @@ static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
ft_field_support_2_nic_transmit_rdma.bth_opcode))
num_op_counters += ARRAY_SIZE(rdmatx_cnp_op_cnts);
+skip_non_qcounters:
cnts->num_op_counters = num_op_counters;
num_counters += num_op_counters;
cnts->descs = kcalloc(num_counters,
@@ -623,7 +728,8 @@ static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev)
int num_cnt_ports;
int i, j;
- num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports;
+ num_cnt_ports = (!is_mdev_switchdev_mode(dev->mdev) ||
+ vport_qcounters_supported(dev)) ? dev->num_ports : 1;
MLX5_SET(dealloc_q_counter_in, in, opcode,
MLX5_CMD_OP_DEALLOC_Q_COUNTER);
@@ -662,15 +768,16 @@ static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev)
MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);
is_shared = MLX5_CAP_GEN(dev->mdev, log_max_uctx) != 0;
- num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports;
+ num_cnt_ports = (!is_mdev_switchdev_mode(dev->mdev) ||
+ vport_qcounters_supported(dev)) ? dev->num_ports : 1;
for (i = 0; i < num_cnt_ports; i++) {
- err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts);
+ err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts, i);
if (err)
goto err_alloc;
mlx5_ib_fill_counters(dev, dev->port[i].cnts.descs,
- dev->port[i].cnts.offsets);
+ dev->port[i].cnts.offsets, i);
MLX5_SET(alloc_q_counter_in, in, uid,
is_shared ? MLX5_SHARED_RESOURCE_UID : 0);
@@ -889,6 +996,10 @@ static const struct ib_device_ops hw_stats_ops = {
mlx5_ib_modify_stat : NULL,
};
+static const struct ib_device_ops hw_switchdev_vport_op = {
+ .alloc_hw_port_stats = mlx5_ib_alloc_hw_port_stats,
+};
+
static const struct ib_device_ops hw_switchdev_stats_ops = {
.alloc_hw_device_stats = mlx5_ib_alloc_hw_device_stats,
.get_hw_stats = mlx5_ib_get_hw_stats,
@@ -914,9 +1025,11 @@ int mlx5_ib_counters_init(struct mlx5_ib_dev *dev)
if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
return 0;
- if (is_mdev_switchdev_mode(dev->mdev))
+ if (is_mdev_switchdev_mode(dev->mdev)) {
ib_set_device_ops(&dev->ib_dev, &hw_switchdev_stats_ops);
- else
+ if (vport_qcounters_supported(dev))
+ ib_set_device_ops(&dev->ib_dev, &hw_switchdev_vport_op);
+ } else
ib_set_device_ops(&dev->ib_dev, &hw_stats_ops);
return mlx5_ib_alloc_counters(dev);
}
diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c
index 2211a0be16f3..db5fb196c728 100644
--- a/drivers/infiniband/hw/mlx5/devx.c
+++ b/drivers/infiniband/hw/mlx5/devx.c
@@ -666,7 +666,21 @@ static bool devx_is_valid_obj_id(struct uverbs_attr_bundle *attrs,
obj_id;
case MLX5_IB_OBJECT_DEVX_OBJ:
- return ((struct devx_obj *)uobj->object)->obj_id == obj_id;
+ {
+ u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
+ struct devx_obj *devx_uobj = uobj->object;
+
+ if (opcode == MLX5_CMD_OP_QUERY_FLOW_COUNTER &&
+ devx_uobj->flow_counter_bulk_size) {
+ u64 end;
+
+ end = devx_uobj->obj_id +
+ devx_uobj->flow_counter_bulk_size;
+ return devx_uobj->obj_id <= obj_id && end > obj_id;
+ }
+
+ return devx_uobj->obj_id == obj_id;
+ }
default:
return false;
@@ -1517,10 +1531,17 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
goto obj_free;
if (opcode == MLX5_CMD_OP_ALLOC_FLOW_COUNTER) {
- u8 bulk = MLX5_GET(alloc_flow_counter_in,
- cmd_in,
- flow_counter_bulk);
- obj->flow_counter_bulk_size = 128UL * bulk;
+ u32 bulk = MLX5_GET(alloc_flow_counter_in,
+ cmd_in,
+ flow_counter_bulk_log_size);
+
+ if (bulk)
+ bulk = 1 << bulk;
+ else
+ bulk = 128UL * MLX5_GET(alloc_flow_counter_in,
+ cmd_in,
+ flow_counter_bulk);
+ obj->flow_counter_bulk_size = bulk;
}
uobj->object = obj;
@@ -1993,7 +2014,6 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(
int redirect_fd;
bool use_eventfd = false;
int num_events;
- int num_alloc_xa_entries = 0;
u16 obj_type = 0;
u64 cookie = 0;
u32 obj_id = 0;
@@ -2075,7 +2095,6 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(
if (err)
goto err;
- num_alloc_xa_entries++;
event_sub = kzalloc(sizeof(*event_sub), GFP_KERNEL);
if (!event_sub) {
err = -ENOMEM;
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index 67356f515261..2017ede100a6 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -67,11 +67,14 @@ static void set_mkc_access_pd_addr_fields(void *mkc, int acc, u64 start_addr,
MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE));
MLX5_SET(mkc, mkc, lr, 1);
- if ((acc & IB_ACCESS_RELAXED_ORDERING) &&
- pcie_relaxed_ordering_enabled(dev->mdev->pdev)) {
+ if (acc & IB_ACCESS_RELAXED_ORDERING) {
if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write))
MLX5_SET(mkc, mkc, relaxed_ordering_write, 1);
- if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read))
+
+ if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) ||
+ (MLX5_CAP_GEN(dev->mdev,
+ relaxed_ordering_read_pci_enabled) &&
+ pcie_relaxed_ordering_enabled(dev->mdev->pdev)))
MLX5_SET(mkc, mkc, relaxed_ordering_read, 1);
}
@@ -791,7 +794,8 @@ static int get_unchangeable_access_flags(struct mlx5_ib_dev *dev,
ret |= IB_ACCESS_RELAXED_ORDERING;
if ((access_flags & IB_ACCESS_RELAXED_ORDERING) &&
- MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) &&
+ (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) ||
+ MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_pci_enabled)) &&
!MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr))
ret |= IB_ACCESS_RELAXED_ORDERING;
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index 7cc3b973dec7..70ca8ffa9256 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -60,6 +60,10 @@ enum raw_qp_set_mask_map {
MLX5_RAW_QP_RATE_LIMIT = 1UL << 1,
};
+enum {
+ MLX5_QP_RM_GO_BACK_N = 0x1,
+};
+
struct mlx5_modify_raw_qp_param {
u16 operation;
@@ -2519,6 +2523,10 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO)
MLX5_SET(qpc, qpc, ulp_stateless_offload_mode, 1);
+ if (qp->flags & IB_QP_CREATE_INTEGRITY_EN &&
+ MLX5_CAP_GEN(mdev, go_back_n))
+ MLX5_SET(qpc, qpc, retry_mode, MLX5_QP_RM_GO_BACK_N);
+
err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
kvfree(in);
if (err)
@@ -2846,9 +2854,9 @@ static void process_vendor_flag(struct mlx5_ib_dev *dev, int *flags, int flag,
case MLX5_QP_FLAG_SCATTER_CQE:
case MLX5_QP_FLAG_ALLOW_SCATTER_CQE:
/*
- * We don't return error if these flags were provided,
- * and mlx5 doesn't have right capability.
- */
+ * We don't return error if these flags were provided,
+ * and mlx5 doesn't have right capability.
+ */
*flags &= ~(MLX5_QP_FLAG_SCATTER_CQE |
MLX5_QP_FLAG_ALLOW_SCATTER_CQE);
return;
@@ -4485,7 +4493,7 @@ static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr,
return -EINVAL;
if (attr->port_num == 0 ||
- attr->port_num > MLX5_CAP_GEN(dev->mdev, num_ports)) {
+ attr->port_num > dev->num_ports) {
mlx5_ib_dbg(dev, "invalid port number %d. number of ports is %d\n",
attr->port_num, dev->num_ports);
return -EINVAL;
@@ -5592,8 +5600,7 @@ int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
if (wq_attr->flags_mask & IB_WQ_FLAGS_CVLAN_STRIPPING) {
if (!(MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
MLX5_CAP_ETH(dev->mdev, vlan_cap))) {
- mlx5_ib_dbg(dev, "VLAN offloads are not "
- "supported\n");
+ mlx5_ib_dbg(dev, "VLAN offloads are not supported\n");
err = -EOPNOTSUPP;
goto out;
}
diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c
index 55f4e048d947..234bf30db731 100644
--- a/drivers/infiniband/hw/mlx5/umr.c
+++ b/drivers/infiniband/hw/mlx5/umr.c
@@ -380,6 +380,10 @@ static void mlx5r_umr_set_access_flags(struct mlx5_ib_dev *dev,
struct mlx5_mkey_seg *seg,
unsigned int access_flags)
{
+ bool ro_read = (access_flags & IB_ACCESS_RELAXED_ORDERING) &&
+ (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) ||
+ pcie_relaxed_ordering_enabled(dev->mdev->pdev));
+
MLX5_SET(mkc, seg, a, !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
MLX5_SET(mkc, seg, rw, !!(access_flags & IB_ACCESS_REMOTE_WRITE));
MLX5_SET(mkc, seg, rr, !!(access_flags & IB_ACCESS_REMOTE_READ));
@@ -387,8 +391,7 @@ static void mlx5r_umr_set_access_flags(struct mlx5_ib_dev *dev,
MLX5_SET(mkc, seg, lr, 1);
MLX5_SET(mkc, seg, relaxed_ordering_write,
!!(access_flags & IB_ACCESS_RELAXED_ORDERING));
- MLX5_SET(mkc, seg, relaxed_ordering_read,
- !!(access_flags & IB_ACCESS_RELAXED_ORDERING));
+ MLX5_SET(mkc, seg, relaxed_ordering_read, ro_read);
}
int mlx5r_umr_rereg_pd_access(struct mlx5_ib_mr *mr, struct ib_pd *pd,
diff --git a/drivers/infiniband/hw/mlx5/umr.h b/drivers/infiniband/hw/mlx5/umr.h
index c9d0021381a2..3799bb758e49 100644
--- a/drivers/infiniband/hw/mlx5/umr.h
+++ b/drivers/infiniband/hw/mlx5/umr.h
@@ -62,7 +62,8 @@ static inline bool mlx5r_umr_can_reconfig(struct mlx5_ib_dev *dev,
return false;
if ((diffs & IB_ACCESS_RELAXED_ORDERING) &&
- MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) &&
+ (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) ||
+ MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_pci_enabled)) &&
!MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr))
return false;
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
index dd4021b11963..58f994341e9a 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
@@ -1589,7 +1589,6 @@ static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq)
{
unsigned long cq_flags;
unsigned long flags;
- int discard_cnt = 0;
u32 cur_getp, stop_getp;
struct ocrdma_cqe *cqe;
u32 qpn = 0, wqe_idx = 0;
@@ -1641,7 +1640,6 @@ static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq)
/* mark cqe discarded so that it is not picked up later
* in the poll_cq().
*/
- discard_cnt += 1;
cqe->cmn.qpn = 0;
skip_cqe:
cur_getp = (cur_getp + 1) % cq->max_hw_cqe;
diff --git a/drivers/infiniband/hw/qib/qib_file_ops.c b/drivers/infiniband/hw/qib/qib_file_ops.c
index f22352f2b369..ef85bc8d9384 100644
--- a/drivers/infiniband/hw/qib/qib_file_ops.c
+++ b/drivers/infiniband/hw/qib/qib_file_ops.c
@@ -484,7 +484,7 @@ static int qib_tid_free(struct qib_ctxtdata *rcd, unsigned subctxt,
const struct qib_tid_info *ti)
{
int ret = 0;
- u32 tid, ctxttid, cnt, limit, tidcnt;
+ u32 tid, ctxttid, limit, tidcnt;
struct qib_devdata *dd = rcd->dd;
u64 __iomem *tidbase;
unsigned long tidmap[8];
@@ -520,7 +520,7 @@ static int qib_tid_free(struct qib_ctxtdata *rcd, unsigned subctxt,
/* just in case size changes in future */
limit = tidcnt;
tid = find_first_bit(tidmap, limit);
- for (cnt = 0; tid < limit; tid++) {
+ for (; tid < limit; tid++) {
/*
* small optimization; if we detect a run of 3 or so without
* any set, use find_first_bit again. That's mainly to
@@ -530,7 +530,7 @@ static int qib_tid_free(struct qib_ctxtdata *rcd, unsigned subctxt,
*/
if (!test_bit(tid, tidmap))
continue;
- cnt++;
+
if (dd->pageshadow[ctxttid + tid]) {
struct page *p;
dma_addr_t phys;
@@ -1768,7 +1768,7 @@ static void unlock_expected_tids(struct qib_ctxtdata *rcd)
{
struct qib_devdata *dd = rcd->dd;
int ctxt_tidbase = rcd->ctxt * dd->rcvtidcnt;
- int i, cnt = 0, maxtid = ctxt_tidbase + dd->rcvtidcnt;
+ int i, maxtid = ctxt_tidbase + dd->rcvtidcnt;
for (i = ctxt_tidbase; i < maxtid; i++) {
struct page *p = dd->pageshadow[i];
@@ -1783,7 +1783,6 @@ static void unlock_expected_tids(struct qib_ctxtdata *rcd)
dma_unmap_page(&dd->pcidev->dev, phys, PAGE_SIZE,
DMA_FROM_DEVICE);
qib_release_user_pages(&p, 1);
- cnt++;
}
}
diff --git a/drivers/infiniband/hw/qib/qib_pcie.c b/drivers/infiniband/hw/qib/qib_pcie.c
index 692b64efad97..47bf64ace05c 100644
--- a/drivers/infiniband/hw/qib/qib_pcie.c
+++ b/drivers/infiniband/hw/qib/qib_pcie.c
@@ -35,7 +35,6 @@
#include <linux/io.h>
#include <linux/delay.h>
#include <linux/vmalloc.h>
-#include <linux/aer.h>
#include <linux/module.h>
#include "qib.h"
@@ -105,13 +104,6 @@ int qib_pcie_init(struct pci_dev *pdev, const struct pci_device_id *ent)
}
pci_set_master(pdev);
- ret = pci_enable_pcie_error_reporting(pdev);
- if (ret) {
- qib_early_err(&pdev->dev,
- "Unable to enable pcie error reporting: %d\n",
- ret);
- ret = 0;
- }
goto done;
bail:
diff --git a/drivers/infiniband/hw/qib/qib_user_sdma.c b/drivers/infiniband/hw/qib/qib_user_sdma.c
index 9fe03d6ffac1..336eb15a721f 100644
--- a/drivers/infiniband/hw/qib/qib_user_sdma.c
+++ b/drivers/infiniband/hw/qib/qib_user_sdma.c
@@ -320,7 +320,6 @@ static int qib_user_sdma_page_to_frags(const struct qib_devdata *dd,
unpin_user_page(page);
} else {
/* coalesce case */
- kunmap(page);
__free_page(page);
}
ret = -ENOMEM;
@@ -572,7 +571,7 @@ static int qib_user_sdma_coalesce(const struct qib_devdata *dd,
goto done;
}
- mpage = kmap(page);
+ mpage = page_address(page);
mpage_save = mpage;
for (i = 0; i < niov; i++) {
int cfur;
@@ -581,7 +580,7 @@ static int qib_user_sdma_coalesce(const struct qib_devdata *dd,
iov[i].iov_base, iov[i].iov_len);
if (cfur) {
ret = -EFAULT;
- goto free_unmap;
+ goto page_free;
}
mpage += iov[i].iov_len;
@@ -592,8 +591,7 @@ static int qib_user_sdma_coalesce(const struct qib_devdata *dd,
page, 0, 0, len, mpage_save);
goto done;
-free_unmap:
- kunmap(page);
+page_free:
__free_page(page);
done:
return ret;
@@ -627,9 +625,6 @@ static void qib_user_sdma_free_pkt_frag(struct device *dev,
pkt->addr[i].dma_length,
DMA_TO_DEVICE);
- if (pkt->addr[i].kvaddr)
- kunmap(pkt->addr[i].page);
-
if (pkt->addr[i].put_page)
unpin_user_page(pkt->addr[i].page);
else
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_main.c b/drivers/infiniband/hw/usnic/usnic_ib_main.c
index 46653ad56f5a..13b654ddd3cc 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_main.c
+++ b/drivers/infiniband/hw/usnic/usnic_ib_main.c
@@ -602,7 +602,6 @@ out_clean_vnic:
usnic_vnic_free(vf->vnic);
out_release_regions:
pci_set_drvdata(pdev, NULL);
- pci_clear_master(pdev);
pci_release_regions(pdev);
out_disable_device:
pci_disable_device(pdev);
@@ -623,7 +622,6 @@ static void usnic_ib_pci_remove(struct pci_dev *pdev)
kref_put(&pf->vf_cnt, usnic_ib_undiscover_pf);
usnic_vnic_free(vf->vnic);
pci_set_drvdata(pdev, NULL);
- pci_clear_master(pdev);
pci_release_regions(pdev);
pci_disable_device(pdev);
kfree(vf);
diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index 9b4c0389d2c0..dc83d0ac6a38 100644
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -464,8 +464,6 @@ void rvt_qp_exit(struct rvt_dev_info *rdi)
if (qps_inuse)
rvt_pr_err(rdi, "QP memory leak! %u still in use\n",
qps_inuse);
- if (!rdi->qp_dev)
- return;
kfree(rdi->qp_dev->qp_table);
free_qpn_table(&rdi->qp_dev->qpn_table);
@@ -2040,7 +2038,7 @@ static int rvt_post_one_wr(struct rvt_qp *qp,
wqe = rvt_get_swqe_ptr(qp, qp->s_head);
/* cplen has length from above */
- memcpy(&wqe->wr, wr, cplen);
+ memcpy(&wqe->ud_wr, wr, cplen);
wqe->length = 0;
j = 0;
diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c
index 136c2efe3466..7a7e713de52d 100644
--- a/drivers/infiniband/sw/rxe/rxe.c
+++ b/drivers/infiniband/sw/rxe/rxe.c
@@ -160,6 +160,8 @@ void rxe_set_mtu(struct rxe_dev *rxe, unsigned int ndev_mtu)
port->attr.active_mtu = mtu;
port->mtu_cap = ib_mtu_enum_to_int(mtu);
+
+ rxe_info_dev(rxe, "Set mtu to %d", port->mtu_cap);
}
/* called by ifc layer to create new rxe device.
@@ -175,26 +177,26 @@ int rxe_add(struct rxe_dev *rxe, unsigned int mtu, const char *ibdev_name)
static int rxe_newlink(const char *ibdev_name, struct net_device *ndev)
{
- struct rxe_dev *exists;
+ struct rxe_dev *rxe;
int err = 0;
if (is_vlan_dev(ndev)) {
- pr_err("rxe creation allowed on top of a real device only\n");
+ rxe_err("rxe creation allowed on top of a real device only");
err = -EPERM;
goto err;
}
- exists = rxe_get_dev_from_net(ndev);
- if (exists) {
- ib_device_put(&exists->ib_dev);
- rxe_dbg(exists, "already configured on %s\n", ndev->name);
+ rxe = rxe_get_dev_from_net(ndev);
+ if (rxe) {
+ ib_device_put(&rxe->ib_dev);
+ rxe_err_dev(rxe, "already configured on %s", ndev->name);
err = -EEXIST;
goto err;
}
err = rxe_net_add(ibdev_name, ndev);
if (err) {
- rxe_dbg(exists, "failed to add %s\n", ndev->name);
+ rxe_err("failed to add %s\n", ndev->name);
goto err;
}
err:
diff --git a/drivers/infiniband/sw/rxe/rxe.h b/drivers/infiniband/sw/rxe/rxe.h
index 2415f3704f57..d33dd6cf83d3 100644
--- a/drivers/infiniband/sw/rxe/rxe.h
+++ b/drivers/infiniband/sw/rxe/rxe.h
@@ -38,7 +38,8 @@
#define RXE_ROCE_V2_SPORT (0xc000)
-#define rxe_dbg(rxe, fmt, ...) ibdev_dbg(&(rxe)->ib_dev, \
+#define rxe_dbg(fmt, ...) pr_debug("%s: " fmt "\n", __func__, ##__VA_ARGS__)
+#define rxe_dbg_dev(rxe, fmt, ...) ibdev_dbg(&(rxe)->ib_dev, \
"%s: " fmt, __func__, ##__VA_ARGS__)
#define rxe_dbg_uc(uc, fmt, ...) ibdev_dbg((uc)->ibuc.device, \
"uc#%d %s: " fmt, (uc)->elem.index, __func__, ##__VA_ARGS__)
@@ -57,6 +58,48 @@
#define rxe_dbg_mw(mw, fmt, ...) ibdev_dbg((mw)->ibmw.device, \
"mw#%d %s: " fmt, (mw)->elem.index, __func__, ##__VA_ARGS__)
+#define rxe_err(fmt, ...) pr_err_ratelimited("%s: " fmt "\n", __func__, \
+ ##__VA_ARGS__)
+#define rxe_err_dev(rxe, fmt, ...) ibdev_err_ratelimited(&(rxe)->ib_dev, \
+ "%s: " fmt, __func__, ##__VA_ARGS__)
+#define rxe_err_uc(uc, fmt, ...) ibdev_err_ratelimited((uc)->ibuc.device, \
+ "uc#%d %s: " fmt, (uc)->elem.index, __func__, ##__VA_ARGS__)
+#define rxe_err_pd(pd, fmt, ...) ibdev_err_ratelimited((pd)->ibpd.device, \
+ "pd#%d %s: " fmt, (pd)->elem.index, __func__, ##__VA_ARGS__)
+#define rxe_err_ah(ah, fmt, ...) ibdev_err_ratelimited((ah)->ibah.device, \
+ "ah#%d %s: " fmt, (ah)->elem.index, __func__, ##__VA_ARGS__)
+#define rxe_err_srq(srq, fmt, ...) ibdev_err_ratelimited((srq)->ibsrq.device, \
+ "srq#%d %s: " fmt, (srq)->elem.index, __func__, ##__VA_ARGS__)
+#define rxe_err_qp(qp, fmt, ...) ibdev_err_ratelimited((qp)->ibqp.device, \
+ "qp#%d %s: " fmt, (qp)->elem.index, __func__, ##__VA_ARGS__)
+#define rxe_err_cq(cq, fmt, ...) ibdev_err_ratelimited((cq)->ibcq.device, \
+ "cq#%d %s: " fmt, (cq)->elem.index, __func__, ##__VA_ARGS__)
+#define rxe_err_mr(mr, fmt, ...) ibdev_err_ratelimited((mr)->ibmr.device, \
+ "mr#%d %s: " fmt, (mr)->elem.index, __func__, ##__VA_ARGS__)
+#define rxe_err_mw(mw, fmt, ...) ibdev_err_ratelimited((mw)->ibmw.device, \
+ "mw#%d %s: " fmt, (mw)->elem.index, __func__, ##__VA_ARGS__)
+
+#define rxe_info(fmt, ...) pr_info_ratelimited("%s: " fmt "\n", __func__, \
+ ##__VA_ARGS__)
+#define rxe_info_dev(rxe, fmt, ...) ibdev_info_ratelimited(&(rxe)->ib_dev, \
+ "%s: " fmt, __func__, ##__VA_ARGS__)
+#define rxe_info_uc(uc, fmt, ...) ibdev_info_ratelimited((uc)->ibuc.device, \
+ "uc#%d %s: " fmt, (uc)->elem.index, __func__, ##__VA_ARGS__)
+#define rxe_info_pd(pd, fmt, ...) ibdev_info_ratelimited((pd)->ibpd.device, \
+ "pd#%d %s: " fmt, (pd)->elem.index, __func__, ##__VA_ARGS__)
+#define rxe_info_ah(ah, fmt, ...) ibdev_info_ratelimited((ah)->ibah.device, \
+ "ah#%d %s: " fmt, (ah)->elem.index, __func__, ##__VA_ARGS__)
+#define rxe_info_srq(srq, fmt, ...) ibdev_info_ratelimited((srq)->ibsrq.device, \
+ "srq#%d %s: " fmt, (srq)->elem.index, __func__, ##__VA_ARGS__)
+#define rxe_info_qp(qp, fmt, ...) ibdev_info_ratelimited((qp)->ibqp.device, \
+ "qp#%d %s: " fmt, (qp)->elem.index, __func__, ##__VA_ARGS__)
+#define rxe_info_cq(cq, fmt, ...) ibdev_info_ratelimited((cq)->ibcq.device, \
+ "cq#%d %s: " fmt, (cq)->elem.index, __func__, ##__VA_ARGS__)
+#define rxe_info_mr(mr, fmt, ...) ibdev_info_ratelimited((mr)->ibmr.device, \
+ "mr#%d %s: " fmt, (mr)->elem.index, __func__, ##__VA_ARGS__)
+#define rxe_info_mw(mw, fmt, ...) ibdev_info_ratelimited((mw)->ibmw.device, \
+ "mw#%d %s: " fmt, (mw)->elem.index, __func__, ##__VA_ARGS__)
+
/* responder states */
enum resp_states {
RESPST_NONE,
@@ -90,7 +133,6 @@ enum resp_states {
RESPST_ERR_LENGTH,
RESPST_ERR_CQ_OVERFLOW,
RESPST_ERROR,
- RESPST_RESET,
RESPST_DONE,
RESPST_EXIT,
};
diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c
index 20737fec392b..db18ace74d2b 100644
--- a/drivers/infiniband/sw/rxe/rxe_comp.c
+++ b/drivers/infiniband/sw/rxe/rxe_comp.c
@@ -118,10 +118,12 @@ void retransmit_timer(struct timer_list *t)
rxe_dbg_qp(qp, "retransmit timer fired\n");
+ spin_lock_bh(&qp->state_lock);
if (qp->valid) {
qp->comp.timeout = 1;
rxe_sched_task(&qp->comp.task);
}
+ spin_unlock_bh(&qp->state_lock);
}
void rxe_comp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb)
@@ -322,7 +324,7 @@ static inline enum comp_state check_ack(struct rxe_qp *qp,
qp->comp.psn = pkt->psn;
if (qp->req.wait_psn) {
qp->req.wait_psn = 0;
- rxe_run_task(&qp->req.task);
+ rxe_sched_task(&qp->req.task);
}
}
return COMPST_ERROR_RETRY;
@@ -428,6 +430,10 @@ static void make_send_cqe(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
uwc->wc_flags = IB_WC_WITH_IMM;
uwc->byte_len = wqe->dma.length;
}
+ } else {
+ if (wqe->status != IB_WC_WR_FLUSH_ERR)
+ rxe_err_qp(qp, "non-flush error status = %d",
+ wqe->status);
}
}
@@ -469,30 +475,16 @@ static void do_complete(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
*/
if (qp->req.wait_fence) {
qp->req.wait_fence = 0;
- rxe_run_task(&qp->req.task);
+ rxe_sched_task(&qp->req.task);
}
}
-static inline enum comp_state complete_ack(struct rxe_qp *qp,
- struct rxe_pkt_info *pkt,
- struct rxe_send_wqe *wqe)
+static void comp_check_sq_drain_done(struct rxe_qp *qp)
{
- if (wqe->has_rd_atomic) {
- wqe->has_rd_atomic = 0;
- atomic_inc(&qp->req.rd_atomic);
- if (qp->req.need_rd_atomic) {
- qp->comp.timeout_retry = 0;
- qp->req.need_rd_atomic = 0;
- rxe_run_task(&qp->req.task);
- }
- }
-
- if (unlikely(qp->req.state == QP_STATE_DRAIN)) {
- /* state_lock used by requester & completer */
- spin_lock_bh(&qp->state_lock);
- if ((qp->req.state == QP_STATE_DRAIN) &&
- (qp->comp.psn == qp->req.psn)) {
- qp->req.state = QP_STATE_DRAINED;
+ spin_lock_bh(&qp->state_lock);
+ if (unlikely(qp_state(qp) == IB_QPS_SQD)) {
+ if (qp->attr.sq_draining && qp->comp.psn == qp->req.psn) {
+ qp->attr.sq_draining = 0;
spin_unlock_bh(&qp->state_lock);
if (qp->ibqp.event_handler) {
@@ -504,10 +496,27 @@ static inline enum comp_state complete_ack(struct rxe_qp *qp,
qp->ibqp.event_handler(&ev,
qp->ibqp.qp_context);
}
- } else {
- spin_unlock_bh(&qp->state_lock);
+ return;
}
}
+ spin_unlock_bh(&qp->state_lock);
+}
+
+static inline enum comp_state complete_ack(struct rxe_qp *qp,
+ struct rxe_pkt_info *pkt,
+ struct rxe_send_wqe *wqe)
+{
+ if (wqe->has_rd_atomic) {
+ wqe->has_rd_atomic = 0;
+ atomic_inc(&qp->req.rd_atomic);
+ if (qp->req.need_rd_atomic) {
+ qp->comp.timeout_retry = 0;
+ qp->req.need_rd_atomic = 0;
+ rxe_sched_task(&qp->req.task);
+ }
+ }
+
+ comp_check_sq_drain_done(qp);
do_complete(qp, wqe);
@@ -538,25 +547,60 @@ static inline enum comp_state complete_wqe(struct rxe_qp *qp,
return COMPST_GET_WQE;
}
-static void rxe_drain_resp_pkts(struct rxe_qp *qp, bool notify)
+/* drain incoming response packet queue */
+static void drain_resp_pkts(struct rxe_qp *qp)
{
struct sk_buff *skb;
- struct rxe_send_wqe *wqe;
- struct rxe_queue *q = qp->sq.queue;
while ((skb = skb_dequeue(&qp->resp_pkts))) {
rxe_put(qp);
kfree_skb(skb);
ib_device_put(qp->ibqp.device);
}
+}
+
+/* complete send wqe with flush error */
+static int flush_send_wqe(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
+{
+ struct rxe_cqe cqe = {};
+ struct ib_wc *wc = &cqe.ibwc;
+ struct ib_uverbs_wc *uwc = &cqe.uibwc;
+ int err;
+
+ if (qp->is_user) {
+ uwc->wr_id = wqe->wr.wr_id;
+ uwc->status = IB_WC_WR_FLUSH_ERR;
+ uwc->qp_num = qp->ibqp.qp_num;
+ } else {
+ wc->wr_id = wqe->wr.wr_id;
+ wc->status = IB_WC_WR_FLUSH_ERR;
+ wc->qp = &qp->ibqp;
+ }
+
+ err = rxe_cq_post(qp->scq, &cqe, 0);
+ if (err)
+ rxe_dbg_cq(qp->scq, "post cq failed, err = %d", err);
+
+ return err;
+}
+
+/* drain and optionally complete the send queue
+ * if unable to complete a wqe, i.e. cq is full, stop
+ * completing and flush the remaining wqes
+ */
+static void flush_send_queue(struct rxe_qp *qp, bool notify)
+{
+ struct rxe_send_wqe *wqe;
+ struct rxe_queue *q = qp->sq.queue;
+ int err;
while ((wqe = queue_head(q, q->type))) {
if (notify) {
- wqe->status = IB_WC_WR_FLUSH_ERR;
- do_complete(qp, wqe);
- } else {
- queue_advance_consumer(q, q->type);
+ err = flush_send_wqe(qp, wqe);
+ if (err)
+ notify = 0;
}
+ queue_advance_consumer(q, q->type);
}
}
@@ -571,9 +615,28 @@ static void free_pkt(struct rxe_pkt_info *pkt)
ib_device_put(dev);
}
-int rxe_completer(void *arg)
+/* reset the retry timer if
+ * - QP is type RC
+ * - there is a packet sent by the requester that
+ * might be acked (we still might get spurious
+ * timeouts but try to keep them as few as possible)
+ * - the timeout parameter is set
+ * - the QP is alive
+ */
+static void reset_retry_timer(struct rxe_qp *qp)
+{
+ if (qp_type(qp) == IB_QPT_RC && qp->qp_timeout_jiffies) {
+ spin_lock_bh(&qp->state_lock);
+ if (qp_state(qp) >= IB_QPS_RTS &&
+ psn_compare(qp->req.psn, qp->comp.psn) > 0)
+ mod_timer(&qp->retrans_timer,
+ jiffies + qp->qp_timeout_jiffies);
+ spin_unlock_bh(&qp->state_lock);
+ }
+}
+
+int rxe_completer(struct rxe_qp *qp)
{
- struct rxe_qp *qp = (struct rxe_qp *)arg;
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
struct rxe_send_wqe *wqe = NULL;
struct sk_buff *skb = NULL;
@@ -581,15 +644,17 @@ int rxe_completer(void *arg)
enum comp_state state;
int ret;
- if (!rxe_get(qp))
- return -EAGAIN;
+ spin_lock_bh(&qp->state_lock);
+ if (!qp->valid || qp_state(qp) == IB_QPS_ERR ||
+ qp_state(qp) == IB_QPS_RESET) {
+ bool notify = qp->valid && (qp_state(qp) == IB_QPS_ERR);
- if (!qp->valid || qp->comp.state == QP_STATE_ERROR ||
- qp->comp.state == QP_STATE_RESET) {
- rxe_drain_resp_pkts(qp, qp->valid &&
- qp->comp.state == QP_STATE_ERROR);
+ drain_resp_pkts(qp);
+ flush_send_queue(qp, notify);
+ spin_unlock_bh(&qp->state_lock);
goto exit;
}
+ spin_unlock_bh(&qp->state_lock);
if (qp->comp.timeout) {
qp->comp.timeout_retry = 1;
@@ -677,20 +742,7 @@ int rxe_completer(void *arg)
break;
}
- /* re reset the timeout counter if
- * (1) QP is type RC
- * (2) the QP is alive
- * (3) there is a packet sent by the requester that
- * might be acked (we still might get spurious
- * timeouts but try to keep them as few as possible)
- * (4) the timeout parameter is set
- */
- if ((qp_type(qp) == IB_QPT_RC) &&
- (qp->req.state == QP_STATE_READY) &&
- (psn_compare(qp->req.psn, qp->comp.psn) > 0) &&
- qp->qp_timeout_jiffies)
- mod_timer(&qp->retrans_timer,
- jiffies + qp->qp_timeout_jiffies);
+ reset_retry_timer(qp);
goto exit;
case COMPST_ERROR_RETRY:
@@ -730,7 +782,7 @@ int rxe_completer(void *arg)
RXE_CNT_COMP_RETRY);
qp->req.need_retry = 1;
qp->comp.started_retry = 1;
- rxe_run_task(&qp->req.task);
+ rxe_sched_task(&qp->req.task);
}
goto done;
@@ -752,6 +804,7 @@ int rxe_completer(void *arg)
*/
qp->req.wait_for_rnr_timer = 1;
rxe_dbg_qp(qp, "set rnr nak timer\n");
+ // TODO who protects from destroy_qp??
mod_timer(&qp->rnr_nak_timer,
jiffies + rnrnak_jiffies(aeth_syn(pkt)
& ~AETH_TYPE_MASK));
@@ -784,7 +837,5 @@ exit:
out:
if (pkt)
free_pkt(pkt);
- rxe_put(qp);
-
return ret;
}
diff --git a/drivers/infiniband/sw/rxe/rxe_cq.c b/drivers/infiniband/sw/rxe/rxe_cq.c
index 1df186534639..20ff0c0c4605 100644
--- a/drivers/infiniband/sw/rxe/rxe_cq.c
+++ b/drivers/infiniband/sw/rxe/rxe_cq.c
@@ -14,12 +14,12 @@ int rxe_cq_chk_attr(struct rxe_dev *rxe, struct rxe_cq *cq,
int count;
if (cqe <= 0) {
- rxe_dbg(rxe, "cqe(%d) <= 0\n", cqe);
+ rxe_dbg_dev(rxe, "cqe(%d) <= 0\n", cqe);
goto err1;
}
if (cqe > rxe->attr.max_cqe) {
- rxe_dbg(rxe, "cqe(%d) > max_cqe(%d)\n",
+ rxe_dbg_dev(rxe, "cqe(%d) > max_cqe(%d)\n",
cqe, rxe->attr.max_cqe);
goto err1;
}
@@ -39,21 +39,6 @@ err1:
return -EINVAL;
}
-static void rxe_send_complete(struct tasklet_struct *t)
-{
- struct rxe_cq *cq = from_tasklet(cq, t, comp_task);
- unsigned long flags;
-
- spin_lock_irqsave(&cq->cq_lock, flags);
- if (cq->is_dying) {
- spin_unlock_irqrestore(&cq->cq_lock, flags);
- return;
- }
- spin_unlock_irqrestore(&cq->cq_lock, flags);
-
- cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
-}
-
int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe,
int comp_vector, struct ib_udata *udata,
struct rxe_create_cq_resp __user *uresp)
@@ -65,7 +50,7 @@ int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe,
cq->queue = rxe_queue_init(rxe, &cqe,
sizeof(struct rxe_cqe), type);
if (!cq->queue) {
- rxe_dbg(rxe, "unable to create cq\n");
+ rxe_dbg_dev(rxe, "unable to create cq\n");
return -ENOMEM;
}
@@ -79,10 +64,6 @@ int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe,
cq->is_user = uresp;
- cq->is_dying = false;
-
- tasklet_setup(&cq->comp_task, rxe_send_complete);
-
spin_lock_init(&cq->cq_lock);
cq->ibcq.cqe = cqe;
return 0;
@@ -103,6 +84,7 @@ int rxe_cq_resize_queue(struct rxe_cq *cq, int cqe,
return err;
}
+/* caller holds reference to cq */
int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited)
{
struct ib_event ev;
@@ -114,6 +96,7 @@ int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited)
full = queue_full(cq->queue, QUEUE_TYPE_TO_CLIENT);
if (unlikely(full)) {
+ rxe_err_cq(cq, "queue full");
spin_unlock_irqrestore(&cq->cq_lock, flags);
if (cq->ibcq.event_handler) {
ev.device = cq->ibcq.device;
@@ -135,21 +118,13 @@ int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited)
if ((cq->notify == IB_CQ_NEXT_COMP) ||
(cq->notify == IB_CQ_SOLICITED && solicited)) {
cq->notify = 0;
- tasklet_schedule(&cq->comp_task);
+
+ cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
}
return 0;
}
-void rxe_cq_disable(struct rxe_cq *cq)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&cq->cq_lock, flags);
- cq->is_dying = true;
- spin_unlock_irqrestore(&cq->cq_lock, flags);
-}
-
void rxe_cq_cleanup(struct rxe_pool_elem *elem)
{
struct rxe_cq *cq = container_of(elem, typeof(*cq), elem);
diff --git a/drivers/infiniband/sw/rxe/rxe_icrc.c b/drivers/infiniband/sw/rxe/rxe_icrc.c
index 71bc2c189588..fdf5f08cd8f1 100644
--- a/drivers/infiniband/sw/rxe/rxe_icrc.c
+++ b/drivers/infiniband/sw/rxe/rxe_icrc.c
@@ -21,7 +21,7 @@ int rxe_icrc_init(struct rxe_dev *rxe)
tfm = crypto_alloc_shash("crc32", 0, 0);
if (IS_ERR(tfm)) {
- rxe_dbg(rxe, "failed to init crc32 algorithm err: %ld\n",
+ rxe_dbg_dev(rxe, "failed to init crc32 algorithm err: %ld\n",
PTR_ERR(tfm));
return PTR_ERR(tfm);
}
@@ -51,7 +51,7 @@ static __be32 rxe_crc32(struct rxe_dev *rxe, __be32 crc, void *next, size_t len)
*(__be32 *)shash_desc_ctx(shash) = crc;
err = crypto_shash_update(shash, next, len);
if (unlikely(err)) {
- rxe_dbg(rxe, "failed crc calculation, err: %d\n", err);
+ rxe_dbg_dev(rxe, "failed crc calculation, err: %d\n", err);
return (__force __be32)crc32_le((__force u32)crc, next, len);
}
diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h
index 1bb0cb479eb1..804b15e929dd 100644
--- a/drivers/infiniband/sw/rxe/rxe_loc.h
+++ b/drivers/infiniband/sw/rxe/rxe_loc.h
@@ -80,7 +80,6 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
int advance_dma_data(struct rxe_dma_info *dma, unsigned int length);
int rxe_invalidate_mr(struct rxe_qp *qp, u32 key);
int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe);
-int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
void rxe_mr_cleanup(struct rxe_pool_elem *elem);
/* rxe_mw.c */
@@ -171,9 +170,9 @@ void rxe_srq_cleanup(struct rxe_pool_elem *elem);
void rxe_dealloc(struct ib_device *ib_dev);
-int rxe_completer(void *arg);
-int rxe_requester(void *arg);
-int rxe_responder(void *arg);
+int rxe_completer(struct rxe_qp *qp);
+int rxe_requester(struct rxe_qp *qp);
+int rxe_responder(struct rxe_qp *qp);
/* rxe_icrc.c */
int rxe_icrc_init(struct rxe_dev *rxe);
diff --git a/drivers/infiniband/sw/rxe/rxe_mmap.c b/drivers/infiniband/sw/rxe/rxe_mmap.c
index a47d72dbc537..6b7f2bd69879 100644
--- a/drivers/infiniband/sw/rxe/rxe_mmap.c
+++ b/drivers/infiniband/sw/rxe/rxe_mmap.c
@@ -79,7 +79,7 @@ int rxe_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
/* Don't allow a mmap larger than the object. */
if (size > ip->info.size) {
- rxe_dbg(rxe, "mmap region is larger than the object!\n");
+ rxe_dbg_dev(rxe, "mmap region is larger than the object!\n");
spin_unlock_bh(&rxe->pending_lock);
ret = -EINVAL;
goto done;
@@ -87,7 +87,7 @@ int rxe_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
goto found_it;
}
- rxe_dbg(rxe, "unable to find pending mmap info\n");
+ rxe_dbg_dev(rxe, "unable to find pending mmap info\n");
spin_unlock_bh(&rxe->pending_lock);
ret = -EINVAL;
goto done;
@@ -98,7 +98,7 @@ found_it:
ret = remap_vmalloc_range(vma, ip->obj, 0);
if (ret) {
- rxe_dbg(rxe, "err %d from remap_vmalloc_range\n", ret);
+ rxe_dbg_dev(rxe, "err %d from remap_vmalloc_range\n", ret);
goto done;
}
diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c
index b10aa1580a64..0e538fafcc20 100644
--- a/drivers/infiniband/sw/rxe/rxe_mr.c
+++ b/drivers/infiniband/sw/rxe/rxe_mr.c
@@ -210,10 +210,10 @@ err1:
return err;
}
-static int rxe_set_page(struct ib_mr *ibmr, u64 iova)
+static int rxe_set_page(struct ib_mr *ibmr, u64 dma_addr)
{
struct rxe_mr *mr = to_rmr(ibmr);
- struct page *page = virt_to_page(iova & mr->page_mask);
+ struct page *page = ib_virt_dma_to_page(dma_addr);
bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT);
int err;
@@ -279,16 +279,16 @@ static int rxe_mr_copy_xarray(struct rxe_mr *mr, u64 iova, void *addr,
return 0;
}
-static void rxe_mr_copy_dma(struct rxe_mr *mr, u64 iova, void *addr,
+static void rxe_mr_copy_dma(struct rxe_mr *mr, u64 dma_addr, void *addr,
unsigned int length, enum rxe_mr_copy_dir dir)
{
- unsigned int page_offset = iova & (PAGE_SIZE - 1);
+ unsigned int page_offset = dma_addr & (PAGE_SIZE - 1);
unsigned int bytes;
struct page *page;
u8 *va;
while (length) {
- page = virt_to_page(iova & mr->page_mask);
+ page = ib_virt_dma_to_page(dma_addr);
bytes = min_t(unsigned int, length,
PAGE_SIZE - page_offset);
va = kmap_local_page(page);
@@ -300,7 +300,7 @@ static void rxe_mr_copy_dma(struct rxe_mr *mr, u64 iova, void *addr,
kunmap_local(va);
page_offset = 0;
- iova += bytes;
+ dma_addr += bytes;
addr += bytes;
length -= bytes;
}
@@ -488,7 +488,7 @@ int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
if (mr->ibmr.type == IB_MR_TYPE_DMA) {
page_offset = iova & (PAGE_SIZE - 1);
- page = virt_to_page(iova & PAGE_MASK);
+ page = ib_virt_dma_to_page(iova);
} else {
unsigned long index;
int err;
@@ -545,7 +545,7 @@ int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)
if (mr->ibmr.type == IB_MR_TYPE_DMA) {
page_offset = iova & (PAGE_SIZE - 1);
- page = virt_to_page(iova & PAGE_MASK);
+ page = ib_virt_dma_to_page(iova);
} else {
unsigned long index;
int err;
@@ -722,19 +722,6 @@ int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
return 0;
}
-int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
-{
- struct rxe_mr *mr = to_rmr(ibmr);
-
- /* See IBA 10.6.7.2.6 */
- if (atomic_read(&mr->num_mw) > 0)
- return -EINVAL;
-
- rxe_cleanup(mr);
- kfree_rcu(mr);
- return 0;
-}
-
void rxe_mr_cleanup(struct rxe_pool_elem *elem)
{
struct rxe_mr *mr = container_of(elem, typeof(*mr), elem);
diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
index e02e1624bcf4..2bc7361152ea 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.c
+++ b/drivers/infiniband/sw/rxe/rxe_net.c
@@ -413,11 +413,14 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
int is_request = pkt->mask & RXE_REQ_MASK;
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
- if ((is_request && (qp->req.state != QP_STATE_READY)) ||
- (!is_request && (qp->resp.state != QP_STATE_READY))) {
+ spin_lock_bh(&qp->state_lock);
+ if ((is_request && (qp_state(qp) < IB_QPS_RTS)) ||
+ (!is_request && (qp_state(qp) < IB_QPS_RTR))) {
+ spin_unlock_bh(&qp->state_lock);
rxe_dbg_qp(qp, "Packet dropped. QP is not in ready state\n");
goto drop;
}
+ spin_unlock_bh(&qp->state_lock);
rxe_icrc_generate(skb, pkt);
@@ -596,7 +599,7 @@ static int rxe_notify(struct notifier_block *not_blk,
rxe_port_down(rxe);
break;
case NETDEV_CHANGEMTU:
- rxe_dbg(rxe, "%s changed mtu to %d\n", ndev->name, ndev->mtu);
+ rxe_dbg_dev(rxe, "%s changed mtu to %d\n", ndev->name, ndev->mtu);
rxe_set_mtu(rxe, ndev->mtu);
break;
case NETDEV_CHANGE:
@@ -608,7 +611,7 @@ static int rxe_notify(struct notifier_block *not_blk,
case NETDEV_CHANGENAME:
case NETDEV_FEAT_CHANGE:
default:
- rxe_dbg(rxe, "ignoring netdev event = %ld for %s\n",
+ rxe_dbg_dev(rxe, "ignoring netdev event = %ld for %s\n",
event, ndev->name);
break;
}
diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c
index ab72db68b58f..c5451a4488ca 100644
--- a/drivers/infiniband/sw/rxe/rxe_qp.c
+++ b/drivers/infiniband/sw/rxe/rxe_qp.c
@@ -19,33 +19,33 @@ static int rxe_qp_chk_cap(struct rxe_dev *rxe, struct ib_qp_cap *cap,
int has_srq)
{
if (cap->max_send_wr > rxe->attr.max_qp_wr) {
- rxe_dbg(rxe, "invalid send wr = %u > %d\n",
+ rxe_dbg_dev(rxe, "invalid send wr = %u > %d\n",
cap->max_send_wr, rxe->attr.max_qp_wr);
goto err1;
}
if (cap->max_send_sge > rxe->attr.max_send_sge) {
- rxe_dbg(rxe, "invalid send sge = %u > %d\n",
+ rxe_dbg_dev(rxe, "invalid send sge = %u > %d\n",
cap->max_send_sge, rxe->attr.max_send_sge);
goto err1;
}
if (!has_srq) {
if (cap->max_recv_wr > rxe->attr.max_qp_wr) {
- rxe_dbg(rxe, "invalid recv wr = %u > %d\n",
+ rxe_dbg_dev(rxe, "invalid recv wr = %u > %d\n",
cap->max_recv_wr, rxe->attr.max_qp_wr);
goto err1;
}
if (cap->max_recv_sge > rxe->attr.max_recv_sge) {
- rxe_dbg(rxe, "invalid recv sge = %u > %d\n",
+ rxe_dbg_dev(rxe, "invalid recv sge = %u > %d\n",
cap->max_recv_sge, rxe->attr.max_recv_sge);
goto err1;
}
}
if (cap->max_inline_data > rxe->max_inline_data) {
- rxe_dbg(rxe, "invalid max inline data = %u > %d\n",
+ rxe_dbg_dev(rxe, "invalid max inline data = %u > %d\n",
cap->max_inline_data, rxe->max_inline_data);
goto err1;
}
@@ -73,7 +73,7 @@ int rxe_qp_chk_init(struct rxe_dev *rxe, struct ib_qp_init_attr *init)
}
if (!init->recv_cq || !init->send_cq) {
- rxe_dbg(rxe, "missing cq\n");
+ rxe_dbg_dev(rxe, "missing cq\n");
goto err1;
}
@@ -82,14 +82,14 @@ int rxe_qp_chk_init(struct rxe_dev *rxe, struct ib_qp_init_attr *init)
if (init->qp_type == IB_QPT_GSI) {
if (!rdma_is_port_valid(&rxe->ib_dev, port_num)) {
- rxe_dbg(rxe, "invalid port = %d\n", port_num);
+ rxe_dbg_dev(rxe, "invalid port = %d\n", port_num);
goto err1;
}
port = &rxe->port;
if (init->qp_type == IB_QPT_GSI && port->qp_gsi_index) {
- rxe_dbg(rxe, "GSI QP exists for port %d\n", port_num);
+ rxe_dbg_dev(rxe, "GSI QP exists for port %d\n", port_num);
goto err1;
}
}
@@ -231,8 +231,6 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp,
qp->req.wqe_index = queue_get_producer(qp->sq.queue,
QUEUE_TYPE_FROM_CLIENT);
- qp->req.state = QP_STATE_RESET;
- qp->comp.state = QP_STATE_RESET;
qp->req.opcode = -1;
qp->comp.opcode = -1;
@@ -287,7 +285,6 @@ static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp,
qp->resp.opcode = OPCODE_NONE;
qp->resp.msn = 0;
- qp->resp.state = QP_STATE_RESET;
return 0;
}
@@ -328,8 +325,10 @@ int rxe_qp_from_init(struct rxe_dev *rxe, struct rxe_qp *qp, struct rxe_pd *pd,
if (err)
goto err2;
+ spin_lock_bh(&qp->state_lock);
qp->attr.qp_state = IB_QPS_RESET;
qp->valid = 1;
+ spin_unlock_bh(&qp->state_lock);
return 0;
@@ -380,30 +379,9 @@ int rxe_qp_to_init(struct rxe_qp *qp, struct ib_qp_init_attr *init)
return 0;
}
-/* called by the modify qp verb, this routine checks all the parameters before
- * making any changes
- */
int rxe_qp_chk_attr(struct rxe_dev *rxe, struct rxe_qp *qp,
struct ib_qp_attr *attr, int mask)
{
- enum ib_qp_state cur_state = (mask & IB_QP_CUR_STATE) ?
- attr->cur_qp_state : qp->attr.qp_state;
- enum ib_qp_state new_state = (mask & IB_QP_STATE) ?
- attr->qp_state : cur_state;
-
- if (!ib_modify_qp_is_ok(cur_state, new_state, qp_type(qp), mask)) {
- rxe_dbg_qp(qp, "invalid mask or state\n");
- goto err1;
- }
-
- if (mask & IB_QP_STATE) {
- if (cur_state == IB_QPS_SQD) {
- if (qp->req.state == QP_STATE_DRAIN &&
- new_state != IB_QPS_ERR)
- goto err1;
- }
- }
-
if (mask & IB_QP_PORT) {
if (!rdma_is_port_valid(&rxe->ib_dev, attr->port_num)) {
rxe_dbg_qp(qp, "invalid port %d\n", attr->port_num);
@@ -473,29 +451,18 @@ static void rxe_qp_reset(struct rxe_qp *qp)
{
/* stop tasks from running */
rxe_disable_task(&qp->resp.task);
+ rxe_disable_task(&qp->comp.task);
+ rxe_disable_task(&qp->req.task);
- /* stop request/comp */
- if (qp->sq.queue) {
- if (qp_type(qp) == IB_QPT_RC)
- rxe_disable_task(&qp->comp.task);
- rxe_disable_task(&qp->req.task);
- }
-
- /* move qp to the reset state */
- qp->req.state = QP_STATE_RESET;
- qp->comp.state = QP_STATE_RESET;
- qp->resp.state = QP_STATE_RESET;
+ /* drain work and packet queuesc */
+ rxe_requester(qp);
+ rxe_completer(qp);
+ rxe_responder(qp);
- /* let state machines reset themselves drain work and packet queues
- * etc.
- */
- __rxe_do_task(&qp->resp.task);
-
- if (qp->sq.queue) {
- __rxe_do_task(&qp->comp.task);
- __rxe_do_task(&qp->req.task);
+ if (qp->rq.queue)
+ rxe_queue_reset(qp->rq.queue);
+ if (qp->sq.queue)
rxe_queue_reset(qp->sq.queue);
- }
/* cleanup attributes */
atomic_set(&qp->ssn, 0);
@@ -518,54 +485,103 @@ static void rxe_qp_reset(struct rxe_qp *qp)
/* reenable tasks */
rxe_enable_task(&qp->resp.task);
-
- if (qp->sq.queue) {
- if (qp_type(qp) == IB_QPT_RC)
- rxe_enable_task(&qp->comp.task);
-
- rxe_enable_task(&qp->req.task);
- }
-}
-
-/* drain the send queue */
-static void rxe_qp_drain(struct rxe_qp *qp)
-{
- if (qp->sq.queue) {
- if (qp->req.state != QP_STATE_DRAINED) {
- qp->req.state = QP_STATE_DRAIN;
- if (qp_type(qp) == IB_QPT_RC)
- rxe_sched_task(&qp->comp.task);
- else
- __rxe_do_task(&qp->comp.task);
- rxe_sched_task(&qp->req.task);
- }
- }
+ rxe_enable_task(&qp->comp.task);
+ rxe_enable_task(&qp->req.task);
}
/* move the qp to the error state */
void rxe_qp_error(struct rxe_qp *qp)
{
- qp->req.state = QP_STATE_ERROR;
- qp->resp.state = QP_STATE_ERROR;
- qp->comp.state = QP_STATE_ERROR;
+ spin_lock_bh(&qp->state_lock);
qp->attr.qp_state = IB_QPS_ERR;
/* drain work and packet queues */
rxe_sched_task(&qp->resp.task);
+ rxe_sched_task(&qp->comp.task);
+ rxe_sched_task(&qp->req.task);
+ spin_unlock_bh(&qp->state_lock);
+}
- if (qp_type(qp) == IB_QPT_RC)
- rxe_sched_task(&qp->comp.task);
- else
- __rxe_do_task(&qp->comp.task);
+static void rxe_qp_sqd(struct rxe_qp *qp, struct ib_qp_attr *attr,
+ int mask)
+{
+ spin_lock_bh(&qp->state_lock);
+ qp->attr.sq_draining = 1;
+ rxe_sched_task(&qp->comp.task);
rxe_sched_task(&qp->req.task);
+ spin_unlock_bh(&qp->state_lock);
}
+/* caller should hold qp->state_lock */
+static int __qp_chk_state(struct rxe_qp *qp, struct ib_qp_attr *attr,
+ int mask)
+{
+ enum ib_qp_state cur_state;
+ enum ib_qp_state new_state;
+
+ cur_state = (mask & IB_QP_CUR_STATE) ?
+ attr->cur_qp_state : qp->attr.qp_state;
+ new_state = (mask & IB_QP_STATE) ?
+ attr->qp_state : cur_state;
+
+ if (!ib_modify_qp_is_ok(cur_state, new_state, qp_type(qp), mask))
+ return -EINVAL;
+
+ if (mask & IB_QP_STATE && cur_state == IB_QPS_SQD) {
+ if (qp->attr.sq_draining && new_state != IB_QPS_ERR)
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static const char *const qps2str[] = {
+ [IB_QPS_RESET] = "RESET",
+ [IB_QPS_INIT] = "INIT",
+ [IB_QPS_RTR] = "RTR",
+ [IB_QPS_RTS] = "RTS",
+ [IB_QPS_SQD] = "SQD",
+ [IB_QPS_SQE] = "SQE",
+ [IB_QPS_ERR] = "ERR",
+};
+
/* called by the modify qp verb */
int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask,
struct ib_udata *udata)
{
int err;
+ if (mask & IB_QP_CUR_STATE)
+ qp->attr.cur_qp_state = attr->qp_state;
+
+ if (mask & IB_QP_STATE) {
+ spin_lock_bh(&qp->state_lock);
+ err = __qp_chk_state(qp, attr, mask);
+ if (!err) {
+ qp->attr.qp_state = attr->qp_state;
+ rxe_dbg_qp(qp, "state -> %s\n",
+ qps2str[attr->qp_state]);
+ }
+ spin_unlock_bh(&qp->state_lock);
+
+ if (err)
+ return err;
+
+ switch (attr->qp_state) {
+ case IB_QPS_RESET:
+ rxe_qp_reset(qp);
+ break;
+ case IB_QPS_SQD:
+ rxe_qp_sqd(qp, attr, mask);
+ break;
+ case IB_QPS_ERR:
+ rxe_qp_error(qp);
+ break;
+ default:
+ break;
+ }
+ }
+
if (mask & IB_QP_MAX_QP_RD_ATOMIC) {
int max_rd_atomic = attr->max_rd_atomic ?
roundup_pow_of_two(attr->max_rd_atomic) : 0;
@@ -587,9 +603,6 @@ int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask,
return err;
}
- if (mask & IB_QP_CUR_STATE)
- qp->attr.cur_qp_state = attr->qp_state;
-
if (mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
qp->attr.en_sqd_async_notify = attr->en_sqd_async_notify;
@@ -669,50 +682,6 @@ int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask,
if (mask & IB_QP_DEST_QPN)
qp->attr.dest_qp_num = attr->dest_qp_num;
- if (mask & IB_QP_STATE) {
- qp->attr.qp_state = attr->qp_state;
-
- switch (attr->qp_state) {
- case IB_QPS_RESET:
- rxe_dbg_qp(qp, "state -> RESET\n");
- rxe_qp_reset(qp);
- break;
-
- case IB_QPS_INIT:
- rxe_dbg_qp(qp, "state -> INIT\n");
- qp->req.state = QP_STATE_INIT;
- qp->resp.state = QP_STATE_INIT;
- qp->comp.state = QP_STATE_INIT;
- break;
-
- case IB_QPS_RTR:
- rxe_dbg_qp(qp, "state -> RTR\n");
- qp->resp.state = QP_STATE_READY;
- break;
-
- case IB_QPS_RTS:
- rxe_dbg_qp(qp, "state -> RTS\n");
- qp->req.state = QP_STATE_READY;
- qp->comp.state = QP_STATE_READY;
- break;
-
- case IB_QPS_SQD:
- rxe_dbg_qp(qp, "state -> SQD\n");
- rxe_qp_drain(qp);
- break;
-
- case IB_QPS_SQE:
- rxe_dbg_qp(qp, "state -> SQE !!?\n");
- /* Not possible from modify_qp. */
- break;
-
- case IB_QPS_ERR:
- rxe_dbg_qp(qp, "state -> ERR\n");
- rxe_qp_error(qp);
- break;
- }
- }
-
return 0;
}
@@ -736,18 +705,15 @@ int rxe_qp_to_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask)
rxe_av_to_attr(&qp->pri_av, &attr->ah_attr);
rxe_av_to_attr(&qp->alt_av, &attr->alt_ah_attr);
- if (qp->req.state == QP_STATE_DRAIN) {
- attr->sq_draining = 1;
- /* applications that get this state
- * typically spin on it. yield the
- * processor
- */
+ /* Applications that get this state typically spin on it.
+ * Yield the processor
+ */
+ spin_lock_bh(&qp->state_lock);
+ if (qp->attr.sq_draining) {
+ spin_unlock_bh(&qp->state_lock);
cond_resched();
- } else {
- attr->sq_draining = 0;
}
-
- rxe_dbg_qp(qp, "attr->sq_draining = %d\n", attr->sq_draining);
+ spin_unlock_bh(&qp->state_lock);
return 0;
}
@@ -771,26 +737,29 @@ static void rxe_qp_do_cleanup(struct work_struct *work)
{
struct rxe_qp *qp = container_of(work, typeof(*qp), cleanup_work.work);
+ spin_lock_bh(&qp->state_lock);
qp->valid = 0;
+ spin_unlock_bh(&qp->state_lock);
qp->qp_timeout_jiffies = 0;
- rxe_cleanup_task(&qp->resp.task);
if (qp_type(qp) == IB_QPT_RC) {
del_timer_sync(&qp->retrans_timer);
del_timer_sync(&qp->rnr_nak_timer);
}
- rxe_cleanup_task(&qp->req.task);
- rxe_cleanup_task(&qp->comp.task);
+ if (qp->resp.task.func)
+ rxe_cleanup_task(&qp->resp.task);
- /* flush out any receive wr's or pending requests */
if (qp->req.task.func)
- __rxe_do_task(&qp->req.task);
+ rxe_cleanup_task(&qp->req.task);
- if (qp->sq.queue) {
- __rxe_do_task(&qp->comp.task);
- __rxe_do_task(&qp->req.task);
- }
+ if (qp->comp.task.func)
+ rxe_cleanup_task(&qp->comp.task);
+
+ /* flush out any receive wr's or pending requests */
+ rxe_requester(qp);
+ rxe_completer(qp);
+ rxe_responder(qp);
if (qp->sq.queue)
rxe_queue_cleanup(qp->sq.queue);
diff --git a/drivers/infiniband/sw/rxe/rxe_queue.c b/drivers/infiniband/sw/rxe/rxe_queue.c
index d6dbf5a0058d..9611ee191a46 100644
--- a/drivers/infiniband/sw/rxe/rxe_queue.c
+++ b/drivers/infiniband/sw/rxe/rxe_queue.c
@@ -61,11 +61,11 @@ struct rxe_queue *rxe_queue_init(struct rxe_dev *rxe, int *num_elem,
/* num_elem == 0 is allowed, but uninteresting */
if (*num_elem < 0)
- goto err1;
+ return NULL;
q = kzalloc(sizeof(*q), GFP_KERNEL);
if (!q)
- goto err1;
+ return NULL;
q->rxe = rxe;
q->type = type;
@@ -100,7 +100,6 @@ struct rxe_queue *rxe_queue_init(struct rxe_dev *rxe, int *num_elem,
err2:
kfree(q);
-err1:
return NULL;
}
diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c
index 434a693cd4a5..2f953cc74256 100644
--- a/drivers/infiniband/sw/rxe/rxe_recv.c
+++ b/drivers/infiniband/sw/rxe/rxe_recv.c
@@ -38,12 +38,19 @@ static int check_type_state(struct rxe_dev *rxe, struct rxe_pkt_info *pkt,
return -EINVAL;
}
+ spin_lock_bh(&qp->state_lock);
if (pkt->mask & RXE_REQ_MASK) {
- if (unlikely(qp->resp.state != QP_STATE_READY))
+ if (unlikely(qp_state(qp) < IB_QPS_RTR)) {
+ spin_unlock_bh(&qp->state_lock);
return -EINVAL;
- } else if (unlikely(qp->req.state < QP_STATE_READY ||
- qp->req.state > QP_STATE_DRAINED))
- return -EINVAL;
+ }
+ } else {
+ if (unlikely(qp_state(qp) < IB_QPS_RTS)) {
+ spin_unlock_bh(&qp->state_lock);
+ return -EINVAL;
+ }
+ }
+ spin_unlock_bh(&qp->state_lock);
return 0;
}
diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c
index 899c8779f800..65134a9aefe7 100644
--- a/drivers/infiniband/sw/rxe/rxe_req.c
+++ b/drivers/infiniband/sw/rxe/rxe_req.c
@@ -102,44 +102,44 @@ void rnr_nak_timer(struct timer_list *t)
rxe_dbg_qp(qp, "nak timer fired\n");
- /* request a send queue retry */
- qp->req.need_retry = 1;
- qp->req.wait_for_rnr_timer = 0;
- rxe_sched_task(&qp->req.task);
+ spin_lock_bh(&qp->state_lock);
+ if (qp->valid) {
+ /* request a send queue retry */
+ qp->req.need_retry = 1;
+ qp->req.wait_for_rnr_timer = 0;
+ rxe_sched_task(&qp->req.task);
+ }
+ spin_unlock_bh(&qp->state_lock);
}
-static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp)
+static void req_check_sq_drain_done(struct rxe_qp *qp)
{
- struct rxe_send_wqe *wqe;
- struct rxe_queue *q = qp->sq.queue;
- unsigned int index = qp->req.wqe_index;
+ struct rxe_queue *q;
+ unsigned int index;
unsigned int cons;
- unsigned int prod;
+ struct rxe_send_wqe *wqe;
- wqe = queue_head(q, QUEUE_TYPE_FROM_CLIENT);
- cons = queue_get_consumer(q, QUEUE_TYPE_FROM_CLIENT);
- prod = queue_get_producer(q, QUEUE_TYPE_FROM_CLIENT);
+ spin_lock_bh(&qp->state_lock);
+ if (qp_state(qp) == IB_QPS_SQD) {
+ q = qp->sq.queue;
+ index = qp->req.wqe_index;
+ cons = queue_get_consumer(q, QUEUE_TYPE_FROM_CLIENT);
+ wqe = queue_addr_from_index(q, cons);
- if (unlikely(qp->req.state == QP_STATE_DRAIN)) {
/* check to see if we are drained;
* state_lock used by requester and completer
*/
- spin_lock_bh(&qp->state_lock);
do {
- if (qp->req.state != QP_STATE_DRAIN) {
+ if (!qp->attr.sq_draining)
/* comp just finished */
- spin_unlock_bh(&qp->state_lock);
break;
- }
if (wqe && ((index != cons) ||
- (wqe->state != wqe_state_posted))) {
+ (wqe->state != wqe_state_posted)))
/* comp not done yet */
- spin_unlock_bh(&qp->state_lock);
break;
- }
- qp->req.state = QP_STATE_DRAINED;
+ qp->attr.sq_draining = 0;
spin_unlock_bh(&qp->state_lock);
if (qp->ibqp.event_handler) {
@@ -151,18 +151,42 @@ static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp)
qp->ibqp.event_handler(&ev,
qp->ibqp.qp_context);
}
+ return;
} while (0);
}
+ spin_unlock_bh(&qp->state_lock);
+}
+
+static struct rxe_send_wqe *__req_next_wqe(struct rxe_qp *qp)
+{
+ struct rxe_queue *q = qp->sq.queue;
+ unsigned int index = qp->req.wqe_index;
+ unsigned int prod;
+ prod = queue_get_producer(q, QUEUE_TYPE_FROM_CLIENT);
if (index == prod)
return NULL;
+ else
+ return queue_addr_from_index(q, index);
+}
+
+static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp)
+{
+ struct rxe_send_wqe *wqe;
+
+ req_check_sq_drain_done(qp);
- wqe = queue_addr_from_index(q, index);
+ wqe = __req_next_wqe(qp);
+ if (wqe == NULL)
+ return NULL;
- if (unlikely((qp->req.state == QP_STATE_DRAIN ||
- qp->req.state == QP_STATE_DRAINED) &&
- (wqe->state != wqe_state_processing)))
+ spin_lock_bh(&qp->state_lock);
+ if (unlikely((qp_state(qp) == IB_QPS_SQD) &&
+ (wqe->state != wqe_state_processing))) {
+ spin_unlock_bh(&qp->state_lock);
return NULL;
+ }
+ spin_unlock_bh(&qp->state_lock);
wqe->mask = wr_opcode_mask(wqe->wr.opcode, qp);
return wqe;
@@ -635,9 +659,8 @@ static int rxe_do_local_ops(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
return 0;
}
-int rxe_requester(void *arg)
+int rxe_requester(struct rxe_qp *qp)
{
- struct rxe_qp *qp = (struct rxe_qp *)arg;
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
struct rxe_pkt_info pkt;
struct sk_buff *skb;
@@ -654,24 +677,22 @@ int rxe_requester(void *arg)
struct rxe_ah *ah;
struct rxe_av *av;
- if (!rxe_get(qp))
- return -EAGAIN;
-
- if (unlikely(!qp->valid))
+ spin_lock_bh(&qp->state_lock);
+ if (unlikely(!qp->valid)) {
+ spin_unlock_bh(&qp->state_lock);
goto exit;
+ }
- if (unlikely(qp->req.state == QP_STATE_ERROR)) {
- wqe = req_next_wqe(qp);
+ if (unlikely(qp_state(qp) == IB_QPS_ERR)) {
+ wqe = __req_next_wqe(qp);
+ spin_unlock_bh(&qp->state_lock);
if (wqe)
- /*
- * Generate an error completion for error qp state
- */
goto err;
else
goto exit;
}
- if (unlikely(qp->req.state == QP_STATE_RESET)) {
+ if (unlikely(qp_state(qp) == IB_QPS_RESET)) {
qp->req.wqe_index = queue_get_consumer(q,
QUEUE_TYPE_FROM_CLIENT);
qp->req.opcode = -1;
@@ -679,8 +700,10 @@ int rxe_requester(void *arg)
qp->req.wait_psn = 0;
qp->req.need_retry = 0;
qp->req.wait_for_rnr_timer = 0;
+ spin_unlock_bh(&qp->state_lock);
goto exit;
}
+ spin_unlock_bh(&qp->state_lock);
/* we come here if the retransmit timer has fired
* or if the rnr timer has fired. If the retransmit
@@ -757,7 +780,7 @@ int rxe_requester(void *arg)
qp->req.wqe_index);
wqe->state = wqe_state_done;
wqe->status = IB_WC_SUCCESS;
- rxe_run_task(&qp->comp.task);
+ rxe_sched_task(&qp->comp.task);
goto done;
}
payload = mtu;
@@ -840,12 +863,9 @@ err:
/* update wqe_index for each wqe completion */
qp->req.wqe_index = queue_next_index(qp->sq.queue, qp->req.wqe_index);
wqe->state = wqe_state_error;
- qp->req.state = QP_STATE_ERROR;
- rxe_run_task(&qp->comp.task);
+ rxe_qp_error(qp);
exit:
ret = -EAGAIN;
out:
- rxe_put(qp);
-
return ret;
}
diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c
index 0cc1ba91d48c..68f6cd188d8e 100644
--- a/drivers/infiniband/sw/rxe/rxe_resp.c
+++ b/drivers/infiniband/sw/rxe/rxe_resp.c
@@ -42,7 +42,6 @@ static char *resp_state_name[] = {
[RESPST_ERR_LENGTH] = "ERR_LENGTH",
[RESPST_ERR_CQ_OVERFLOW] = "ERR_CQ_OVERFLOW",
[RESPST_ERROR] = "ERROR",
- [RESPST_RESET] = "RESET",
[RESPST_DONE] = "DONE",
[RESPST_EXIT] = "EXIT",
};
@@ -69,17 +68,6 @@ static inline enum resp_states get_req(struct rxe_qp *qp,
{
struct sk_buff *skb;
- if (qp->resp.state == QP_STATE_ERROR) {
- while ((skb = skb_dequeue(&qp->req_pkts))) {
- rxe_put(qp);
- kfree_skb(skb);
- ib_device_put(qp->ibqp.device);
- }
-
- /* go drain recv wr queue */
- return RESPST_CHK_RESOURCE;
- }
-
skb = skb_peek(&qp->req_pkts);
if (!skb)
return RESPST_EXIT;
@@ -334,24 +322,6 @@ static enum resp_states check_resource(struct rxe_qp *qp,
{
struct rxe_srq *srq = qp->srq;
- if (qp->resp.state == QP_STATE_ERROR) {
- if (qp->resp.wqe) {
- qp->resp.status = IB_WC_WR_FLUSH_ERR;
- return RESPST_COMPLETE;
- } else if (!srq) {
- qp->resp.wqe = queue_head(qp->rq.queue,
- QUEUE_TYPE_FROM_CLIENT);
- if (qp->resp.wqe) {
- qp->resp.status = IB_WC_WR_FLUSH_ERR;
- return RESPST_COMPLETE;
- } else {
- return RESPST_EXIT;
- }
- } else {
- return RESPST_EXIT;
- }
- }
-
if (pkt->mask & (RXE_READ_OR_ATOMIC_MASK | RXE_ATOMIC_WRITE_MASK)) {
/* it is the requesters job to not send
* too many read/atomic ops, we just
@@ -1151,6 +1121,10 @@ static enum resp_states do_complete(struct rxe_qp *qp,
wc->port_num = qp->attr.port_num;
}
+ } else {
+ if (wc->status != IB_WC_WR_FLUSH_ERR)
+ rxe_err_qp(qp, "non-flush error status = %d",
+ wc->status);
}
/* have copy for srq and reference for !srq */
@@ -1163,8 +1137,13 @@ static enum resp_states do_complete(struct rxe_qp *qp,
return RESPST_ERR_CQ_OVERFLOW;
finish:
- if (unlikely(qp->resp.state == QP_STATE_ERROR))
+ spin_lock_bh(&qp->state_lock);
+ if (unlikely(qp_state(qp) == IB_QPS_ERR)) {
+ spin_unlock_bh(&qp->state_lock);
return RESPST_CHK_RESOURCE;
+ }
+ spin_unlock_bh(&qp->state_lock);
+
if (unlikely(!pkt))
return RESPST_DONE;
if (qp_type(qp) == IB_QPT_RC)
@@ -1421,49 +1400,90 @@ static enum resp_states do_class_d1e_error(struct rxe_qp *qp)
}
}
-static void rxe_drain_req_pkts(struct rxe_qp *qp, bool notify)
+/* drain incoming request packet queue */
+static void drain_req_pkts(struct rxe_qp *qp)
{
struct sk_buff *skb;
- struct rxe_queue *q = qp->rq.queue;
while ((skb = skb_dequeue(&qp->req_pkts))) {
rxe_put(qp);
kfree_skb(skb);
ib_device_put(qp->ibqp.device);
}
+}
+
+/* complete receive wqe with flush error */
+static int flush_recv_wqe(struct rxe_qp *qp, struct rxe_recv_wqe *wqe)
+{
+ struct rxe_cqe cqe = {};
+ struct ib_wc *wc = &cqe.ibwc;
+ struct ib_uverbs_wc *uwc = &cqe.uibwc;
+ int err;
+
+ if (qp->rcq->is_user) {
+ uwc->wr_id = wqe->wr_id;
+ uwc->status = IB_WC_WR_FLUSH_ERR;
+ uwc->qp_num = qp_num(qp);
+ } else {
+ wc->wr_id = wqe->wr_id;
+ wc->status = IB_WC_WR_FLUSH_ERR;
+ wc->qp = &qp->ibqp;
+ }
- if (notify)
+ err = rxe_cq_post(qp->rcq, &cqe, 0);
+ if (err)
+ rxe_dbg_cq(qp->rcq, "post cq failed err = %d", err);
+
+ return err;
+}
+
+/* drain and optionally complete the recive queue
+ * if unable to complete a wqe stop completing and
+ * just flush the remaining wqes
+ */
+static void flush_recv_queue(struct rxe_qp *qp, bool notify)
+{
+ struct rxe_queue *q = qp->rq.queue;
+ struct rxe_recv_wqe *wqe;
+ int err;
+
+ if (qp->srq)
return;
- while (!qp->srq && q && queue_head(q, q->type))
+ while ((wqe = queue_head(q, q->type))) {
+ if (notify) {
+ err = flush_recv_wqe(qp, wqe);
+ if (err)
+ notify = 0;
+ }
queue_advance_consumer(q, q->type);
+ }
+
+ qp->resp.wqe = NULL;
}
-int rxe_responder(void *arg)
+int rxe_responder(struct rxe_qp *qp)
{
- struct rxe_qp *qp = (struct rxe_qp *)arg;
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
enum resp_states state;
struct rxe_pkt_info *pkt = NULL;
int ret;
- if (!rxe_get(qp))
- return -EAGAIN;
+ spin_lock_bh(&qp->state_lock);
+ if (!qp->valid || qp_state(qp) == IB_QPS_ERR ||
+ qp_state(qp) == IB_QPS_RESET) {
+ bool notify = qp->valid && (qp_state(qp) == IB_QPS_ERR);
- qp->resp.aeth_syndrome = AETH_ACK_UNLIMITED;
-
- if (!qp->valid)
+ drain_req_pkts(qp);
+ flush_recv_queue(qp, notify);
+ spin_unlock_bh(&qp->state_lock);
goto exit;
+ }
+ spin_unlock_bh(&qp->state_lock);
- switch (qp->resp.state) {
- case QP_STATE_RESET:
- state = RESPST_RESET;
- break;
+ qp->resp.aeth_syndrome = AETH_ACK_UNLIMITED;
- default:
- state = RESPST_GET_REQ;
- break;
- }
+ state = RESPST_GET_REQ;
while (1) {
rxe_dbg_qp(qp, "state = %s\n", resp_state_name[state]);
@@ -1622,11 +1642,6 @@ int rxe_responder(void *arg)
goto exit;
- case RESPST_RESET:
- rxe_drain_req_pkts(qp, false);
- qp->resp.wqe = NULL;
- goto exit;
-
case RESPST_ERROR:
qp->resp.goto_error = 0;
rxe_dbg_qp(qp, "moved to error state\n");
@@ -1648,6 +1663,5 @@ done:
exit:
ret = -EAGAIN;
out:
- rxe_put(qp);
return ret;
}
diff --git a/drivers/infiniband/sw/rxe/rxe_srq.c b/drivers/infiniband/sw/rxe/rxe_srq.c
index 82e37a41ced4..27ca82ec0826 100644
--- a/drivers/infiniband/sw/rxe/rxe_srq.c
+++ b/drivers/infiniband/sw/rxe/rxe_srq.c
@@ -13,13 +13,13 @@ int rxe_srq_chk_init(struct rxe_dev *rxe, struct ib_srq_init_attr *init)
struct ib_srq_attr *attr = &init->attr;
if (attr->max_wr > rxe->attr.max_srq_wr) {
- rxe_dbg(rxe, "max_wr(%d) > max_srq_wr(%d)\n",
+ rxe_dbg_dev(rxe, "max_wr(%d) > max_srq_wr(%d)\n",
attr->max_wr, rxe->attr.max_srq_wr);
goto err1;
}
if (attr->max_wr <= 0) {
- rxe_dbg(rxe, "max_wr(%d) <= 0\n", attr->max_wr);
+ rxe_dbg_dev(rxe, "max_wr(%d) <= 0\n", attr->max_wr);
goto err1;
}
@@ -27,7 +27,7 @@ int rxe_srq_chk_init(struct rxe_dev *rxe, struct ib_srq_init_attr *init)
attr->max_wr = RXE_MIN_SRQ_WR;
if (attr->max_sge > rxe->attr.max_srq_sge) {
- rxe_dbg(rxe, "max_sge(%d) > max_srq_sge(%d)\n",
+ rxe_dbg_dev(rxe, "max_sge(%d) > max_srq_sge(%d)\n",
attr->max_sge, rxe->attr.max_srq_sge);
goto err1;
}
diff --git a/drivers/infiniband/sw/rxe/rxe_task.c b/drivers/infiniband/sw/rxe/rxe_task.c
index 60b90e33a884..fb9a6bc8e620 100644
--- a/drivers/infiniband/sw/rxe/rxe_task.c
+++ b/drivers/infiniband/sw/rxe/rxe_task.c
@@ -6,69 +6,128 @@
#include "rxe.h"
-int __rxe_do_task(struct rxe_task *task)
+/* Check if task is idle i.e. not running, not scheduled in
+ * tasklet queue and not draining. If so move to busy to
+ * reserve a slot in do_task() by setting to busy and taking
+ * a qp reference to cover the gap from now until the task finishes.
+ * state will move out of busy if task returns a non zero value
+ * in do_task(). If state is already busy it is raised to armed
+ * to indicate to do_task that additional pass should be made
+ * over the task.
+ * Context: caller should hold task->lock.
+ * Returns: true if state transitioned from idle to busy else false.
+ */
+static bool __reserve_if_idle(struct rxe_task *task)
+{
+ WARN_ON(rxe_read(task->qp) <= 0);
+
+ if (task->tasklet.state & BIT(TASKLET_STATE_SCHED))
+ return false;
+
+ if (task->state == TASK_STATE_IDLE) {
+ rxe_get(task->qp);
+ task->state = TASK_STATE_BUSY;
+ task->num_sched++;
+ return true;
+ }
+
+ if (task->state == TASK_STATE_BUSY)
+ task->state = TASK_STATE_ARMED;
+ return false;
+}
+
+/* check if task is idle or drained and not currently
+ * scheduled in the tasklet queue. This routine is
+ * called by rxe_cleanup_task or rxe_disable_task to
+ * see if the queue is empty.
+ * Context: caller should hold task->lock.
+ * Returns true if done else false.
+ */
+static bool __is_done(struct rxe_task *task)
{
- int ret;
+ if (task->tasklet.state & BIT(TASKLET_STATE_SCHED))
+ return false;
- while ((ret = task->func(task->arg)) == 0)
- ;
+ if (task->state == TASK_STATE_IDLE ||
+ task->state == TASK_STATE_DRAINED) {
+ return true;
+ }
- task->ret = ret;
+ return false;
+}
- return ret;
+/* a locked version of __is_done */
+static bool is_done(struct rxe_task *task)
+{
+ unsigned long flags;
+ int done;
+
+ spin_lock_irqsave(&task->lock, flags);
+ done = __is_done(task);
+ spin_unlock_irqrestore(&task->lock, flags);
+
+ return done;
}
-/*
- * this locking is due to a potential race where
- * a second caller finds the task already running
- * but looks just after the last call to func
+/* do_task is a wrapper for the three tasks (requester,
+ * completer, responder) and calls them in a loop until
+ * they return a non-zero value. It is called either
+ * directly by rxe_run_task or indirectly if rxe_sched_task
+ * schedules the task. They must call __reserve_if_idle to
+ * move the task to busy before calling or scheduling.
+ * The task can also be moved to drained or invalid
+ * by calls to rxe-cleanup_task or rxe_disable_task.
+ * In that case tasks which get here are not executed but
+ * just flushed. The tasks are designed to look to see if
+ * there is work to do and do part of it before returning
+ * here with a return value of zero until all the work
+ * has been consumed then it retuens a non-zero value.
+ * The number of times the task can be run is limited by
+ * max iterations so one task cannot hold the cpu forever.
*/
static void do_task(struct tasklet_struct *t)
{
int cont;
int ret;
struct rxe_task *task = from_tasklet(task, t, tasklet);
- struct rxe_qp *qp = (struct rxe_qp *)task->arg;
- unsigned int iterations = RXE_MAX_ITERATIONS;
+ unsigned int iterations;
+ unsigned long flags;
+ int resched = 0;
- spin_lock_bh(&task->lock);
- switch (task->state) {
- case TASK_STATE_START:
- task->state = TASK_STATE_BUSY;
- spin_unlock_bh(&task->lock);
- break;
-
- case TASK_STATE_BUSY:
- task->state = TASK_STATE_ARMED;
- fallthrough;
- case TASK_STATE_ARMED:
- spin_unlock_bh(&task->lock);
- return;
+ WARN_ON(rxe_read(task->qp) <= 0);
- default:
- spin_unlock_bh(&task->lock);
- rxe_dbg_qp(qp, "failed with bad state %d\n", task->state);
+ spin_lock_irqsave(&task->lock, flags);
+ if (task->state >= TASK_STATE_DRAINED) {
+ rxe_put(task->qp);
+ task->num_done++;
+ spin_unlock_irqrestore(&task->lock, flags);
return;
}
+ spin_unlock_irqrestore(&task->lock, flags);
do {
+ iterations = RXE_MAX_ITERATIONS;
cont = 0;
- ret = task->func(task->arg);
- spin_lock_bh(&task->lock);
+ do {
+ ret = task->func(task->qp);
+ } while (ret == 0 && iterations-- > 0);
+
+ spin_lock_irqsave(&task->lock, flags);
switch (task->state) {
case TASK_STATE_BUSY:
if (ret) {
- task->state = TASK_STATE_START;
- } else if (iterations--) {
- cont = 1;
+ task->state = TASK_STATE_IDLE;
} else {
- /* reschedule the tasklet and exit
+ /* This can happen if the client
+ * can add work faster than the
+ * tasklet can finish it.
+ * Reschedule the tasklet and exit
* the loop to give up the cpu
*/
- tasklet_schedule(&task->tasklet);
- task->state = TASK_STATE_START;
+ task->state = TASK_STATE_IDLE;
+ resched = 1;
}
break;
@@ -81,71 +140,158 @@ static void do_task(struct tasklet_struct *t)
cont = 1;
break;
+ case TASK_STATE_DRAINING:
+ if (ret)
+ task->state = TASK_STATE_DRAINED;
+ else
+ cont = 1;
+ break;
+
default:
- rxe_dbg_qp(qp, "failed with bad state %d\n",
- task->state);
+ WARN_ON(1);
+ rxe_info_qp(task->qp, "unexpected task state = %d", task->state);
+ }
+
+ if (!cont) {
+ task->num_done++;
+ if (WARN_ON(task->num_done != task->num_sched))
+ rxe_err_qp(task->qp, "%ld tasks scheduled, %ld tasks done",
+ task->num_sched, task->num_done);
}
- spin_unlock_bh(&task->lock);
+ spin_unlock_irqrestore(&task->lock, flags);
} while (cont);
task->ret = ret;
+
+ if (resched)
+ rxe_sched_task(task);
+
+ rxe_put(task->qp);
}
-int rxe_init_task(struct rxe_task *task, void *arg, int (*func)(void *))
+int rxe_init_task(struct rxe_task *task, struct rxe_qp *qp,
+ int (*func)(struct rxe_qp *))
{
- task->arg = arg;
- task->func = func;
- task->destroyed = false;
+ WARN_ON(rxe_read(qp) <= 0);
+
+ task->qp = qp;
+ task->func = func;
tasklet_setup(&task->tasklet, do_task);
- task->state = TASK_STATE_START;
+ task->state = TASK_STATE_IDLE;
spin_lock_init(&task->lock);
return 0;
}
+/* rxe_cleanup_task is only called from rxe_do_qp_cleanup in
+ * process context. The qp is already completed with no
+ * remaining references. Once the queue is drained the
+ * task is moved to invalid and returns. The qp cleanup
+ * code then calls the task functions directly without
+ * using the task struct to drain any late arriving packets
+ * or work requests.
+ */
void rxe_cleanup_task(struct rxe_task *task)
{
- bool idle;
+ unsigned long flags;
- /*
- * Mark the task, then wait for it to finish. It might be
- * running in a non-tasklet (direct call) context.
- */
- task->destroyed = true;
+ spin_lock_irqsave(&task->lock, flags);
+ if (!__is_done(task) && task->state < TASK_STATE_DRAINED) {
+ task->state = TASK_STATE_DRAINING;
+ } else {
+ task->state = TASK_STATE_INVALID;
+ spin_unlock_irqrestore(&task->lock, flags);
+ return;
+ }
+ spin_unlock_irqrestore(&task->lock, flags);
- do {
- spin_lock_bh(&task->lock);
- idle = (task->state == TASK_STATE_START);
- spin_unlock_bh(&task->lock);
- } while (!idle);
+ /* now the task cannot be scheduled or run just wait
+ * for the previously scheduled tasks to finish.
+ */
+ while (!is_done(task))
+ cond_resched();
tasklet_kill(&task->tasklet);
+
+ spin_lock_irqsave(&task->lock, flags);
+ task->state = TASK_STATE_INVALID;
+ spin_unlock_irqrestore(&task->lock, flags);
}
+/* run the task inline if it is currently idle
+ * cannot call do_task holding the lock
+ */
void rxe_run_task(struct rxe_task *task)
{
- if (task->destroyed)
- return;
+ unsigned long flags;
+ int run;
+
+ WARN_ON(rxe_read(task->qp) <= 0);
+
+ spin_lock_irqsave(&task->lock, flags);
+ run = __reserve_if_idle(task);
+ spin_unlock_irqrestore(&task->lock, flags);
- do_task(&task->tasklet);
+ if (run)
+ do_task(&task->tasklet);
}
+/* schedule the task to run later as a tasklet.
+ * the tasklet)schedule call can be called holding
+ * the lock.
+ */
void rxe_sched_task(struct rxe_task *task)
{
- if (task->destroyed)
- return;
+ unsigned long flags;
- tasklet_schedule(&task->tasklet);
+ WARN_ON(rxe_read(task->qp) <= 0);
+
+ spin_lock_irqsave(&task->lock, flags);
+ if (__reserve_if_idle(task))
+ tasklet_schedule(&task->tasklet);
+ spin_unlock_irqrestore(&task->lock, flags);
}
+/* rxe_disable/enable_task are only called from
+ * rxe_modify_qp in process context. Task is moved
+ * to the drained state by do_task.
+ */
void rxe_disable_task(struct rxe_task *task)
{
+ unsigned long flags;
+
+ WARN_ON(rxe_read(task->qp) <= 0);
+
+ spin_lock_irqsave(&task->lock, flags);
+ if (!__is_done(task) && task->state < TASK_STATE_DRAINED) {
+ task->state = TASK_STATE_DRAINING;
+ } else {
+ task->state = TASK_STATE_DRAINED;
+ spin_unlock_irqrestore(&task->lock, flags);
+ return;
+ }
+ spin_unlock_irqrestore(&task->lock, flags);
+
+ while (!is_done(task))
+ cond_resched();
+
tasklet_disable(&task->tasklet);
}
void rxe_enable_task(struct rxe_task *task)
{
+ unsigned long flags;
+
+ WARN_ON(rxe_read(task->qp) <= 0);
+
+ spin_lock_irqsave(&task->lock, flags);
+ if (task->state == TASK_STATE_INVALID) {
+ spin_unlock_irqrestore(&task->lock, flags);
+ return;
+ }
+ task->state = TASK_STATE_IDLE;
tasklet_enable(&task->tasklet);
+ spin_unlock_irqrestore(&task->lock, flags);
}
diff --git a/drivers/infiniband/sw/rxe/rxe_task.h b/drivers/infiniband/sw/rxe/rxe_task.h
index 7b88129702ac..facb7c8e3729 100644
--- a/drivers/infiniband/sw/rxe/rxe_task.h
+++ b/drivers/infiniband/sw/rxe/rxe_task.h
@@ -8,9 +8,12 @@
#define RXE_TASK_H
enum {
- TASK_STATE_START = 0,
+ TASK_STATE_IDLE = 0,
TASK_STATE_BUSY = 1,
TASK_STATE_ARMED = 2,
+ TASK_STATE_DRAINING = 3,
+ TASK_STATE_DRAINED = 4,
+ TASK_STATE_INVALID = 5,
};
/*
@@ -22,28 +25,24 @@ struct rxe_task {
struct tasklet_struct tasklet;
int state;
spinlock_t lock;
- void *arg;
- int (*func)(void *arg);
+ struct rxe_qp *qp;
+ int (*func)(struct rxe_qp *qp);
int ret;
- bool destroyed;
+ long num_sched;
+ long num_done;
};
/*
* init rxe_task structure
- * arg => parameter to pass to fcn
+ * qp => parameter to pass to func
* func => function to call until it returns != 0
*/
-int rxe_init_task(struct rxe_task *task, void *arg, int (*func)(void *));
+int rxe_init_task(struct rxe_task *task, struct rxe_qp *qp,
+ int (*func)(struct rxe_qp *));
/* cleanup task */
void rxe_cleanup_task(struct rxe_task *task);
-/*
- * raw call to func in loop without any checking
- * can call when tasklets are disabled
- */
-int __rxe_do_task(struct rxe_task *task);
-
void rxe_run_task(struct rxe_task *task);
void rxe_sched_task(struct rxe_task *task);
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index e14050a69276..dea605b7f683 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -12,31 +12,48 @@
#include "rxe_queue.h"
#include "rxe_hw_counters.h"
-static int rxe_query_device(struct ib_device *dev,
+static int post_one_recv(struct rxe_rq *rq, const struct ib_recv_wr *ibwr);
+
+/* dev */
+static int rxe_query_device(struct ib_device *ibdev,
struct ib_device_attr *attr,
- struct ib_udata *uhw)
+ struct ib_udata *udata)
{
- struct rxe_dev *rxe = to_rdev(dev);
+ struct rxe_dev *rxe = to_rdev(ibdev);
+ int err;
- if (uhw->inlen || uhw->outlen)
- return -EINVAL;
+ if (udata->inlen || udata->outlen) {
+ rxe_dbg_dev(rxe, "malformed udata");
+ err = -EINVAL;
+ goto err_out;
+ }
+
+ memcpy(attr, &rxe->attr, sizeof(*attr));
- *attr = rxe->attr;
return 0;
+
+err_out:
+ rxe_err_dev(rxe, "returned err = %d", err);
+ return err;
}
-static int rxe_query_port(struct ib_device *dev,
+static int rxe_query_port(struct ib_device *ibdev,
u32 port_num, struct ib_port_attr *attr)
{
- struct rxe_dev *rxe = to_rdev(dev);
- int rc;
+ struct rxe_dev *rxe = to_rdev(ibdev);
+ int err, ret;
+
+ if (port_num != 1) {
+ err = -EINVAL;
+ rxe_dbg_dev(rxe, "bad port_num = %d", port_num);
+ goto err_out;
+ }
- /* *attr being zeroed by the caller, avoid zeroing it here */
- *attr = rxe->port.attr;
+ memcpy(attr, &rxe->port.attr, sizeof(*attr));
mutex_lock(&rxe->usdev_lock);
- rc = ib_get_eth_speed(dev, port_num, &attr->active_speed,
- &attr->active_width);
+ ret = ib_get_eth_speed(ibdev, port_num, &attr->active_speed,
+ &attr->active_width);
if (attr->state == IB_PORT_ACTIVE)
attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
@@ -47,27 +64,45 @@ static int rxe_query_port(struct ib_device *dev,
mutex_unlock(&rxe->usdev_lock);
- return rc;
+ return ret;
+
+err_out:
+ rxe_err_dev(rxe, "returned err = %d", err);
+ return err;
}
-static int rxe_query_pkey(struct ib_device *device,
+static int rxe_query_pkey(struct ib_device *ibdev,
u32 port_num, u16 index, u16 *pkey)
{
- if (index > 0)
- return -EINVAL;
+ struct rxe_dev *rxe = to_rdev(ibdev);
+ int err;
+
+ if (index != 0) {
+ err = -EINVAL;
+ rxe_dbg_dev(rxe, "bad pkey index = %d", index);
+ goto err_out;
+ }
*pkey = IB_DEFAULT_PKEY_FULL;
return 0;
+
+err_out:
+ rxe_err_dev(rxe, "returned err = %d", err);
+ return err;
}
-static int rxe_modify_device(struct ib_device *dev,
+static int rxe_modify_device(struct ib_device *ibdev,
int mask, struct ib_device_modify *attr)
{
- struct rxe_dev *rxe = to_rdev(dev);
+ struct rxe_dev *rxe = to_rdev(ibdev);
+ int err;
if (mask & ~(IB_DEVICE_MODIFY_SYS_IMAGE_GUID |
- IB_DEVICE_MODIFY_NODE_DESC))
- return -EOPNOTSUPP;
+ IB_DEVICE_MODIFY_NODE_DESC)) {
+ err = -EOPNOTSUPP;
+ rxe_dbg_dev(rxe, "unsupported mask = 0x%x", mask);
+ goto err_out;
+ }
if (mask & IB_DEVICE_MODIFY_SYS_IMAGE_GUID)
rxe->attr.sys_image_guid = cpu_to_be64(attr->sys_image_guid);
@@ -78,16 +113,33 @@ static int rxe_modify_device(struct ib_device *dev,
}
return 0;
+
+err_out:
+ rxe_err_dev(rxe, "returned err = %d", err);
+ return err;
}
-static int rxe_modify_port(struct ib_device *dev,
- u32 port_num, int mask, struct ib_port_modify *attr)
+static int rxe_modify_port(struct ib_device *ibdev, u32 port_num,
+ int mask, struct ib_port_modify *attr)
{
- struct rxe_dev *rxe = to_rdev(dev);
+ struct rxe_dev *rxe = to_rdev(ibdev);
struct rxe_port *port;
+ int err;
- port = &rxe->port;
+ if (port_num != 1) {
+ err = -EINVAL;
+ rxe_dbg_dev(rxe, "bad port_num = %d", port_num);
+ goto err_out;
+ }
+ //TODO is shutdown useful
+ if (mask & ~(IB_PORT_RESET_QKEY_CNTR)) {
+ err = -EOPNOTSUPP;
+ rxe_dbg_dev(rxe, "unsupported mask = 0x%x", mask);
+ goto err_out;
+ }
+
+ port = &rxe->port;
port->attr.port_cap_flags |= attr->set_port_cap_mask;
port->attr.port_cap_flags &= ~attr->clr_port_cap_mask;
@@ -95,73 +147,125 @@ static int rxe_modify_port(struct ib_device *dev,
port->attr.qkey_viol_cntr = 0;
return 0;
-}
-static enum rdma_link_layer rxe_get_link_layer(struct ib_device *dev,
- u32 port_num)
-{
- return IB_LINK_LAYER_ETHERNET;
+err_out:
+ rxe_err_dev(rxe, "returned err = %d", err);
+ return err;
}
-static int rxe_alloc_ucontext(struct ib_ucontext *ibuc, struct ib_udata *udata)
+static enum rdma_link_layer rxe_get_link_layer(struct ib_device *ibdev,
+ u32 port_num)
{
- struct rxe_dev *rxe = to_rdev(ibuc->device);
- struct rxe_ucontext *uc = to_ruc(ibuc);
+ struct rxe_dev *rxe = to_rdev(ibdev);
+ int err;
- return rxe_add_to_pool(&rxe->uc_pool, uc);
-}
+ if (port_num != 1) {
+ err = -EINVAL;
+ rxe_dbg_dev(rxe, "bad port_num = %d", port_num);
+ goto err_out;
+ }
-static void rxe_dealloc_ucontext(struct ib_ucontext *ibuc)
-{
- struct rxe_ucontext *uc = to_ruc(ibuc);
+ return IB_LINK_LAYER_ETHERNET;
- rxe_cleanup(uc);
+err_out:
+ rxe_err_dev(rxe, "returned err = %d", err);
+ return err;
}
-static int rxe_port_immutable(struct ib_device *dev, u32 port_num,
+static int rxe_port_immutable(struct ib_device *ibdev, u32 port_num,
struct ib_port_immutable *immutable)
{
+ struct rxe_dev *rxe = to_rdev(ibdev);
+ struct ib_port_attr attr = {};
int err;
- struct ib_port_attr attr;
- immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
+ if (port_num != 1) {
+ err = -EINVAL;
+ rxe_dbg_dev(rxe, "bad port_num = %d", port_num);
+ goto err_out;
+ }
- err = ib_query_port(dev, port_num, &attr);
+ err = ib_query_port(ibdev, port_num, &attr);
if (err)
- return err;
+ goto err_out;
+ immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
immutable->pkey_tbl_len = attr.pkey_tbl_len;
immutable->gid_tbl_len = attr.gid_tbl_len;
immutable->max_mad_size = IB_MGMT_MAD_SIZE;
return 0;
+
+err_out:
+ rxe_err_dev(rxe, "returned err = %d", err);
+ return err;
}
+/* uc */
+static int rxe_alloc_ucontext(struct ib_ucontext *ibuc, struct ib_udata *udata)
+{
+ struct rxe_dev *rxe = to_rdev(ibuc->device);
+ struct rxe_ucontext *uc = to_ruc(ibuc);
+ int err;
+
+ err = rxe_add_to_pool(&rxe->uc_pool, uc);
+ if (err)
+ rxe_err_dev(rxe, "unable to create uc");
+
+ return err;
+}
+
+static void rxe_dealloc_ucontext(struct ib_ucontext *ibuc)
+{
+ struct rxe_ucontext *uc = to_ruc(ibuc);
+ int err;
+
+ err = rxe_cleanup(uc);
+ if (err)
+ rxe_err_uc(uc, "cleanup failed, err = %d", err);
+}
+
+/* pd */
static int rxe_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
{
struct rxe_dev *rxe = to_rdev(ibpd->device);
struct rxe_pd *pd = to_rpd(ibpd);
+ int err;
- return rxe_add_to_pool(&rxe->pd_pool, pd);
+ err = rxe_add_to_pool(&rxe->pd_pool, pd);
+ if (err) {
+ rxe_dbg_dev(rxe, "unable to alloc pd");
+ goto err_out;
+ }
+
+ return 0;
+
+err_out:
+ rxe_err_dev(rxe, "returned err = %d", err);
+ return err;
}
static int rxe_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
{
struct rxe_pd *pd = to_rpd(ibpd);
+ int err;
+
+ err = rxe_cleanup(pd);
+ if (err)
+ rxe_err_pd(pd, "cleanup failed, err = %d", err);
- rxe_cleanup(pd);
return 0;
}
+/* ah */
static int rxe_create_ah(struct ib_ah *ibah,
struct rdma_ah_init_attr *init_attr,
struct ib_udata *udata)
-
{
struct rxe_dev *rxe = to_rdev(ibah->device);
struct rxe_ah *ah = to_rah(ibah);
struct rxe_create_ah_resp __user *uresp = NULL;
- int err;
+ int err, cleanup_err;
if (udata) {
/* test if new user provider */
@@ -174,16 +278,18 @@ static int rxe_create_ah(struct ib_ah *ibah,
err = rxe_add_to_pool_ah(&rxe->ah_pool, ah,
init_attr->flags & RDMA_CREATE_AH_SLEEPABLE);
- if (err)
- return err;
+ if (err) {
+ rxe_dbg_dev(rxe, "unable to create ah");
+ goto err_out;
+ }
/* create index > 0 */
ah->ah_num = ah->elem.index;
err = rxe_ah_chk_attr(ah, init_attr->ah_attr);
if (err) {
- rxe_cleanup(ah);
- return err;
+ rxe_dbg_ah(ah, "bad attr");
+ goto err_cleanup;
}
if (uresp) {
@@ -191,8 +297,9 @@ static int rxe_create_ah(struct ib_ah *ibah,
err = copy_to_user(&uresp->ah_num, &ah->ah_num,
sizeof(uresp->ah_num));
if (err) {
- rxe_cleanup(ah);
- return -EFAULT;
+ err = -EFAULT;
+ rxe_dbg_ah(ah, "unable to copy to user");
+ goto err_cleanup;
}
} else if (ah->is_user) {
/* only if old user provider */
@@ -203,19 +310,34 @@ static int rxe_create_ah(struct ib_ah *ibah,
rxe_finalize(ah);
return 0;
+
+err_cleanup:
+ cleanup_err = rxe_cleanup(ah);
+ if (cleanup_err)
+ rxe_err_ah(ah, "cleanup failed, err = %d", cleanup_err);
+err_out:
+ rxe_err_ah(ah, "returned err = %d", err);
+ return err;
}
static int rxe_modify_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr)
{
- int err;
struct rxe_ah *ah = to_rah(ibah);
+ int err;
err = rxe_ah_chk_attr(ah, attr);
- if (err)
- return err;
+ if (err) {
+ rxe_dbg_ah(ah, "bad attr");
+ goto err_out;
+ }
rxe_init_av(attr, &ah->av);
+
return 0;
+
+err_out:
+ rxe_err_ah(ah, "returned err = %d", err);
+ return err;
}
static int rxe_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr)
@@ -225,92 +347,77 @@ static int rxe_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr)
memset(attr, 0, sizeof(*attr));
attr->type = ibah->type;
rxe_av_to_attr(&ah->av, attr);
+
return 0;
}
static int rxe_destroy_ah(struct ib_ah *ibah, u32 flags)
{
struct rxe_ah *ah = to_rah(ibah);
+ int err;
- rxe_cleanup_ah(ah, flags & RDMA_DESTROY_AH_SLEEPABLE);
-
- return 0;
-}
-
-static int post_one_recv(struct rxe_rq *rq, const struct ib_recv_wr *ibwr)
-{
- int i;
- u32 length;
- struct rxe_recv_wqe *recv_wqe;
- int num_sge = ibwr->num_sge;
- int full;
-
- full = queue_full(rq->queue, QUEUE_TYPE_FROM_ULP);
- if (unlikely(full))
- return -ENOMEM;
-
- if (unlikely(num_sge > rq->max_sge))
- return -EINVAL;
-
- length = 0;
- for (i = 0; i < num_sge; i++)
- length += ibwr->sg_list[i].length;
-
- recv_wqe = queue_producer_addr(rq->queue, QUEUE_TYPE_FROM_ULP);
- recv_wqe->wr_id = ibwr->wr_id;
-
- memcpy(recv_wqe->dma.sge, ibwr->sg_list,
- num_sge * sizeof(struct ib_sge));
-
- recv_wqe->dma.length = length;
- recv_wqe->dma.resid = length;
- recv_wqe->dma.num_sge = num_sge;
- recv_wqe->dma.cur_sge = 0;
- recv_wqe->dma.sge_offset = 0;
-
- queue_advance_producer(rq->queue, QUEUE_TYPE_FROM_ULP);
+ err = rxe_cleanup_ah(ah, flags & RDMA_DESTROY_AH_SLEEPABLE);
+ if (err)
+ rxe_err_ah(ah, "cleanup failed, err = %d", err);
return 0;
}
+/* srq */
static int rxe_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init,
struct ib_udata *udata)
{
- int err;
struct rxe_dev *rxe = to_rdev(ibsrq->device);
struct rxe_pd *pd = to_rpd(ibsrq->pd);
struct rxe_srq *srq = to_rsrq(ibsrq);
struct rxe_create_srq_resp __user *uresp = NULL;
+ int err, cleanup_err;
if (udata) {
- if (udata->outlen < sizeof(*uresp))
- return -EINVAL;
+ if (udata->outlen < sizeof(*uresp)) {
+ err = -EINVAL;
+ rxe_err_dev(rxe, "malformed udata");
+ goto err_out;
+ }
uresp = udata->outbuf;
}
- if (init->srq_type != IB_SRQT_BASIC)
- return -EOPNOTSUPP;
+ if (init->srq_type != IB_SRQT_BASIC) {
+ err = -EOPNOTSUPP;
+ rxe_dbg_dev(rxe, "srq type = %d, not supported",
+ init->srq_type);
+ goto err_out;
+ }
err = rxe_srq_chk_init(rxe, init);
- if (err)
- return err;
+ if (err) {
+ rxe_dbg_dev(rxe, "invalid init attributes");
+ goto err_out;
+ }
err = rxe_add_to_pool(&rxe->srq_pool, srq);
- if (err)
- return err;
+ if (err) {
+ rxe_dbg_dev(rxe, "unable to create srq, err = %d", err);
+ goto err_out;
+ }
rxe_get(pd);
srq->pd = pd;
err = rxe_srq_from_init(rxe, srq, init, udata, uresp);
- if (err)
+ if (err) {
+ rxe_dbg_srq(srq, "create srq failed, err = %d", err);
goto err_cleanup;
+ }
return 0;
err_cleanup:
- rxe_cleanup(srq);
-
+ cleanup_err = rxe_cleanup(srq);
+ if (cleanup_err)
+ rxe_err_srq(srq, "cleanup failed, err = %d", cleanup_err);
+err_out:
+ rxe_err_dev(rxe, "returned err = %d", err);
return err;
}
@@ -318,46 +425,64 @@ static int rxe_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
enum ib_srq_attr_mask mask,
struct ib_udata *udata)
{
- int err;
struct rxe_srq *srq = to_rsrq(ibsrq);
struct rxe_dev *rxe = to_rdev(ibsrq->device);
- struct rxe_modify_srq_cmd ucmd = {};
+ struct rxe_modify_srq_cmd cmd = {};
+ int err;
if (udata) {
- if (udata->inlen < sizeof(ucmd))
- return -EINVAL;
+ if (udata->inlen < sizeof(cmd)) {
+ err = -EINVAL;
+ rxe_dbg_srq(srq, "malformed udata");
+ goto err_out;
+ }
- err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));
- if (err)
- return err;
+ err = ib_copy_from_udata(&cmd, udata, sizeof(cmd));
+ if (err) {
+ err = -EFAULT;
+ rxe_dbg_srq(srq, "unable to read udata");
+ goto err_out;
+ }
}
err = rxe_srq_chk_attr(rxe, srq, attr, mask);
- if (err)
- return err;
+ if (err) {
+ rxe_dbg_srq(srq, "bad init attributes");
+ goto err_out;
+ }
+
+ err = rxe_srq_from_attr(rxe, srq, attr, mask, &cmd, udata);
+ if (err) {
+ rxe_dbg_srq(srq, "bad attr");
+ goto err_out;
+ }
+
+ return 0;
- return rxe_srq_from_attr(rxe, srq, attr, mask, &ucmd, udata);
+err_out:
+ rxe_err_srq(srq, "returned err = %d", err);
+ return err;
}
static int rxe_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
{
struct rxe_srq *srq = to_rsrq(ibsrq);
+ int err;
- if (srq->error)
- return -EINVAL;
+ if (srq->error) {
+ err = -EINVAL;
+ rxe_dbg_srq(srq, "srq in error state");
+ goto err_out;
+ }
attr->max_wr = srq->rq.queue->buf->index_mask;
attr->max_sge = srq->rq.max_sge;
attr->srq_limit = srq->limit;
return 0;
-}
-static int rxe_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
-{
- struct rxe_srq *srq = to_rsrq(ibsrq);
-
- rxe_cleanup(srq);
- return 0;
+err_out:
+ rxe_err_srq(srq, "returned err = %d", err);
+ return err;
}
static int rxe_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
@@ -378,76 +503,116 @@ static int rxe_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
spin_unlock_irqrestore(&srq->rq.producer_lock, flags);
- if (err)
+ if (err) {
*bad_wr = wr;
+ rxe_err_srq(srq, "returned err = %d", err);
+ }
return err;
}
+static int rxe_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
+{
+ struct rxe_srq *srq = to_rsrq(ibsrq);
+ int err;
+
+ err = rxe_cleanup(srq);
+ if (err)
+ rxe_err_srq(srq, "cleanup failed, err = %d", err);
+
+ return 0;
+}
+
+/* qp */
static int rxe_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init,
struct ib_udata *udata)
{
- int err;
struct rxe_dev *rxe = to_rdev(ibqp->device);
struct rxe_pd *pd = to_rpd(ibqp->pd);
struct rxe_qp *qp = to_rqp(ibqp);
struct rxe_create_qp_resp __user *uresp = NULL;
+ int err, cleanup_err;
if (udata) {
- if (udata->outlen < sizeof(*uresp))
- return -EINVAL;
- uresp = udata->outbuf;
- }
-
- if (init->create_flags)
- return -EOPNOTSUPP;
-
- err = rxe_qp_chk_init(rxe, init);
- if (err)
- return err;
+ if (udata->inlen) {
+ err = -EINVAL;
+ rxe_dbg_dev(rxe, "malformed udata, err = %d", err);
+ goto err_out;
+ }
- if (udata) {
- if (udata->inlen)
- return -EINVAL;
+ if (udata->outlen < sizeof(*uresp)) {
+ err = -EINVAL;
+ rxe_dbg_dev(rxe, "malformed udata, err = %d", err);
+ goto err_out;
+ }
qp->is_user = true;
+ uresp = udata->outbuf;
} else {
qp->is_user = false;
}
+ if (init->create_flags) {
+ err = -EOPNOTSUPP;
+ rxe_dbg_dev(rxe, "unsupported create_flags, err = %d", err);
+ goto err_out;
+ }
+
+ err = rxe_qp_chk_init(rxe, init);
+ if (err) {
+ rxe_dbg_dev(rxe, "bad init attr, err = %d", err);
+ goto err_out;
+ }
+
err = rxe_add_to_pool(&rxe->qp_pool, qp);
- if (err)
- return err;
+ if (err) {
+ rxe_dbg_dev(rxe, "unable to create qp, err = %d", err);
+ goto err_out;
+ }
err = rxe_qp_from_init(rxe, qp, pd, init, uresp, ibqp->pd, udata);
- if (err)
- goto qp_init;
+ if (err) {
+ rxe_dbg_qp(qp, "create qp failed, err = %d", err);
+ goto err_cleanup;
+ }
rxe_finalize(qp);
return 0;
-qp_init:
- rxe_cleanup(qp);
+err_cleanup:
+ cleanup_err = rxe_cleanup(qp);
+ if (cleanup_err)
+ rxe_err_qp(qp, "cleanup failed, err = %d", cleanup_err);
+err_out:
+ rxe_err_dev(rxe, "returned err = %d", err);
return err;
}
static int rxe_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
int mask, struct ib_udata *udata)
{
- int err;
struct rxe_dev *rxe = to_rdev(ibqp->device);
struct rxe_qp *qp = to_rqp(ibqp);
+ int err;
- if (mask & ~IB_QP_ATTR_STANDARD_BITS)
- return -EOPNOTSUPP;
+ if (mask & ~IB_QP_ATTR_STANDARD_BITS) {
+ err = -EOPNOTSUPP;
+ rxe_dbg_qp(qp, "unsupported mask = 0x%x, err = %d",
+ mask, err);
+ goto err_out;
+ }
err = rxe_qp_chk_attr(rxe, qp, attr, mask);
- if (err)
- return err;
+ if (err) {
+ rxe_dbg_qp(qp, "bad mask/attr, err = %d", err);
+ goto err_out;
+ }
err = rxe_qp_from_attr(qp, attr, mask, udata);
- if (err)
- return err;
+ if (err) {
+ rxe_dbg_qp(qp, "modify qp failed, err = %d", err);
+ goto err_out;
+ }
if ((mask & IB_QP_AV) && (attr->ah_attr.ah_flags & IB_AH_GRH))
qp->src_port = rdma_get_udp_sport(attr->ah_attr.grh.flow_label,
@@ -455,6 +620,10 @@ static int rxe_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
qp->attr.dest_qp_num);
return 0;
+
+err_out:
+ rxe_err_qp(qp, "returned err = %d", err);
+ return err;
}
static int rxe_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
@@ -471,41 +640,90 @@ static int rxe_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
static int rxe_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
{
struct rxe_qp *qp = to_rqp(ibqp);
- int ret;
+ int err;
- ret = rxe_qp_chk_destroy(qp);
- if (ret)
- return ret;
+ err = rxe_qp_chk_destroy(qp);
+ if (err) {
+ rxe_dbg_qp(qp, "unable to destroy qp, err = %d", err);
+ goto err_out;
+ }
+
+ err = rxe_cleanup(qp);
+ if (err)
+ rxe_err_qp(qp, "cleanup failed, err = %d", err);
- rxe_cleanup(qp);
return 0;
+
+err_out:
+ rxe_err_qp(qp, "returned err = %d", err);
+ return err;
}
+/* send wr */
+
+/* sanity check incoming send work request */
static int validate_send_wr(struct rxe_qp *qp, const struct ib_send_wr *ibwr,
- unsigned int mask, unsigned int length)
+ unsigned int *maskp, unsigned int *lengthp)
{
int num_sge = ibwr->num_sge;
struct rxe_sq *sq = &qp->sq;
+ unsigned int mask = 0;
+ unsigned long length = 0;
+ int err = -EINVAL;
+ int i;
- if (unlikely(num_sge > sq->max_sge))
- return -EINVAL;
+ do {
+ mask = wr_opcode_mask(ibwr->opcode, qp);
+ if (!mask) {
+ rxe_err_qp(qp, "bad wr opcode for qp type");
+ break;
+ }
- if (unlikely(mask & WR_ATOMIC_MASK)) {
- if (length < 8)
- return -EINVAL;
+ if (num_sge > sq->max_sge) {
+ rxe_err_qp(qp, "num_sge > max_sge");
+ break;
+ }
- if (atomic_wr(ibwr)->remote_addr & 0x7)
- return -EINVAL;
- }
+ length = 0;
+ for (i = 0; i < ibwr->num_sge; i++)
+ length += ibwr->sg_list[i].length;
- if (unlikely((ibwr->send_flags & IB_SEND_INLINE) &&
- (length > sq->max_inline)))
- return -EINVAL;
+ if (length > (1UL << 31)) {
+ rxe_err_qp(qp, "message length too long");
+ break;
+ }
- return 0;
+ if (mask & WR_ATOMIC_MASK) {
+ if (length != 8) {
+ rxe_err_qp(qp, "atomic length != 8");
+ break;
+ }
+ if (atomic_wr(ibwr)->remote_addr & 0x7) {
+ rxe_err_qp(qp, "misaligned atomic address");
+ break;
+ }
+ }
+ if (ibwr->send_flags & IB_SEND_INLINE) {
+ if (!(mask & WR_INLINE_MASK)) {
+ rxe_err_qp(qp, "opcode doesn't support inline data");
+ break;
+ }
+ if (length > sq->max_inline) {
+ rxe_err_qp(qp, "inline length too big");
+ break;
+ }
+ }
+
+ err = 0;
+ } while (0);
+
+ *maskp = mask;
+ *lengthp = (int)length;
+
+ return err;
}
-static void init_send_wr(struct rxe_qp *qp, struct rxe_send_wr *wr,
+static int init_send_wr(struct rxe_qp *qp, struct rxe_send_wr *wr,
const struct ib_send_wr *ibwr)
{
wr->wr_id = ibwr->wr_id;
@@ -521,8 +739,18 @@ static void init_send_wr(struct rxe_qp *qp, struct rxe_send_wr *wr,
wr->wr.ud.ah_num = to_rah(ibah)->ah_num;
if (qp_type(qp) == IB_QPT_GSI)
wr->wr.ud.pkey_index = ud_wr(ibwr)->pkey_index;
- if (wr->opcode == IB_WR_SEND_WITH_IMM)
+
+ switch (wr->opcode) {
+ case IB_WR_SEND_WITH_IMM:
wr->ex.imm_data = ibwr->ex.imm_data;
+ break;
+ case IB_WR_SEND:
+ break;
+ default:
+ rxe_err_qp(qp, "bad wr opcode %d for UD/GSI QP",
+ wr->opcode);
+ return -EINVAL;
+ }
} else {
switch (wr->opcode) {
case IB_WR_RDMA_WRITE_WITH_IMM:
@@ -539,6 +767,11 @@ static void init_send_wr(struct rxe_qp *qp, struct rxe_send_wr *wr,
case IB_WR_SEND_WITH_INV:
wr->ex.invalidate_rkey = ibwr->ex.invalidate_rkey;
break;
+ case IB_WR_RDMA_READ_WITH_INV:
+ wr->ex.invalidate_rkey = ibwr->ex.invalidate_rkey;
+ wr->wr.rdma.remote_addr = rdma_wr(ibwr)->remote_addr;
+ wr->wr.rdma.rkey = rdma_wr(ibwr)->rkey;
+ break;
case IB_WR_ATOMIC_CMP_AND_SWP:
case IB_WR_ATOMIC_FETCH_AND_ADD:
wr->wr.atomic.remote_addr =
@@ -550,16 +783,26 @@ static void init_send_wr(struct rxe_qp *qp, struct rxe_send_wr *wr,
break;
case IB_WR_LOCAL_INV:
wr->ex.invalidate_rkey = ibwr->ex.invalidate_rkey;
- break;
+ break;
case IB_WR_REG_MR:
wr->wr.reg.mr = reg_wr(ibwr)->mr;
wr->wr.reg.key = reg_wr(ibwr)->key;
wr->wr.reg.access = reg_wr(ibwr)->access;
- break;
+ break;
+ case IB_WR_SEND:
+ case IB_WR_BIND_MW:
+ case IB_WR_FLUSH:
+ case IB_WR_ATOMIC_WRITE:
+ break;
default:
+ rxe_err_qp(qp, "unsupported wr opcode %d",
+ wr->opcode);
+ return -EINVAL;
break;
}
}
+
+ return 0;
}
static void copy_inline_data_to_wqe(struct rxe_send_wqe *wqe,
@@ -570,24 +813,27 @@ static void copy_inline_data_to_wqe(struct rxe_send_wqe *wqe,
int i;
for (i = 0; i < ibwr->num_sge; i++, sge++) {
- memcpy(p, (void *)(uintptr_t)sge->addr, sge->length);
+ memcpy(p, ib_virt_dma_to_page(sge->addr), sge->length);
p += sge->length;
}
}
-static void init_send_wqe(struct rxe_qp *qp, const struct ib_send_wr *ibwr,
+static int init_send_wqe(struct rxe_qp *qp, const struct ib_send_wr *ibwr,
unsigned int mask, unsigned int length,
struct rxe_send_wqe *wqe)
{
int num_sge = ibwr->num_sge;
+ int err;
- init_send_wr(qp, &wqe->wr, ibwr);
+ err = init_send_wr(qp, &wqe->wr, ibwr);
+ if (err)
+ return err;
/* local operation */
if (unlikely(mask & WR_LOCAL_OP_MASK)) {
wqe->mask = mask;
wqe->state = wqe_state_posted;
- return;
+ return 0;
}
if (unlikely(ibwr->send_flags & IB_SEND_INLINE))
@@ -606,82 +852,62 @@ static void init_send_wqe(struct rxe_qp *qp, const struct ib_send_wr *ibwr,
wqe->dma.sge_offset = 0;
wqe->state = wqe_state_posted;
wqe->ssn = atomic_add_return(1, &qp->ssn);
+
+ return 0;
}
-static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr,
- unsigned int mask, u32 length)
+static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr)
{
int err;
struct rxe_sq *sq = &qp->sq;
struct rxe_send_wqe *send_wqe;
- unsigned long flags;
+ unsigned int mask;
+ unsigned int length;
int full;
- err = validate_send_wr(qp, ibwr, mask, length);
+ err = validate_send_wr(qp, ibwr, &mask, &length);
if (err)
return err;
- spin_lock_irqsave(&qp->sq.sq_lock, flags);
-
full = queue_full(sq->queue, QUEUE_TYPE_FROM_ULP);
-
if (unlikely(full)) {
- spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
+ rxe_err_qp(qp, "send queue full");
return -ENOMEM;
}
send_wqe = queue_producer_addr(sq->queue, QUEUE_TYPE_FROM_ULP);
- init_send_wqe(qp, ibwr, mask, length, send_wqe);
+ err = init_send_wqe(qp, ibwr, mask, length, send_wqe);
+ if (!err)
+ queue_advance_producer(sq->queue, QUEUE_TYPE_FROM_ULP);
- queue_advance_producer(sq->queue, QUEUE_TYPE_FROM_ULP);
-
- spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
-
- return 0;
+ return err;
}
-static int rxe_post_send_kernel(struct rxe_qp *qp, const struct ib_send_wr *wr,
+static int rxe_post_send_kernel(struct rxe_qp *qp,
+ const struct ib_send_wr *ibwr,
const struct ib_send_wr **bad_wr)
{
int err = 0;
- unsigned int mask;
- unsigned int length = 0;
- int i;
- struct ib_send_wr *next;
-
- while (wr) {
- mask = wr_opcode_mask(wr->opcode, qp);
- if (unlikely(!mask)) {
- err = -EINVAL;
- *bad_wr = wr;
- break;
- }
-
- if (unlikely((wr->send_flags & IB_SEND_INLINE) &&
- !(mask & WR_INLINE_MASK))) {
- err = -EINVAL;
- *bad_wr = wr;
- break;
- }
-
- next = wr->next;
-
- length = 0;
- for (i = 0; i < wr->num_sge; i++)
- length += wr->sg_list[i].length;
-
- err = post_one_send(qp, wr, mask, length);
+ unsigned long flags;
+ spin_lock_irqsave(&qp->sq.sq_lock, flags);
+ while (ibwr) {
+ err = post_one_send(qp, ibwr);
if (err) {
- *bad_wr = wr;
+ *bad_wr = ibwr;
break;
}
- wr = next;
+ ibwr = ibwr->next;
}
+ spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
- rxe_sched_task(&qp->req.task);
- if (unlikely(qp->req.state == QP_STATE_ERROR))
+ if (!err)
+ rxe_sched_task(&qp->req.task);
+
+ spin_lock_bh(&qp->state_lock);
+ if (qp_state(qp) == IB_QPS_ERR)
rxe_sched_task(&qp->comp.task);
+ spin_unlock_bh(&qp->state_lock);
return err;
}
@@ -690,23 +916,88 @@ static int rxe_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
const struct ib_send_wr **bad_wr)
{
struct rxe_qp *qp = to_rqp(ibqp);
+ int err;
- if (unlikely(!qp->valid)) {
- *bad_wr = wr;
+ spin_lock_bh(&qp->state_lock);
+ /* caller has already called destroy_qp */
+ if (WARN_ON_ONCE(!qp->valid)) {
+ spin_unlock_bh(&qp->state_lock);
+ rxe_err_qp(qp, "qp has been destroyed");
return -EINVAL;
}
- if (unlikely(qp->req.state < QP_STATE_READY)) {
+ if (unlikely(qp_state(qp) < IB_QPS_RTS)) {
+ spin_unlock_bh(&qp->state_lock);
*bad_wr = wr;
+ rxe_err_qp(qp, "qp not ready to send");
return -EINVAL;
}
+ spin_unlock_bh(&qp->state_lock);
if (qp->is_user) {
/* Utilize process context to do protocol processing */
rxe_run_task(&qp->req.task);
- return 0;
- } else
- return rxe_post_send_kernel(qp, wr, bad_wr);
+ } else {
+ err = rxe_post_send_kernel(qp, wr, bad_wr);
+ if (err)
+ return err;
+ }
+
+ return 0;
+}
+
+/* recv wr */
+static int post_one_recv(struct rxe_rq *rq, const struct ib_recv_wr *ibwr)
+{
+ int i;
+ unsigned long length;
+ struct rxe_recv_wqe *recv_wqe;
+ int num_sge = ibwr->num_sge;
+ int full;
+ int err;
+
+ full = queue_full(rq->queue, QUEUE_TYPE_FROM_ULP);
+ if (unlikely(full)) {
+ err = -ENOMEM;
+ rxe_dbg("queue full");
+ goto err_out;
+ }
+
+ if (unlikely(num_sge > rq->max_sge)) {
+ err = -EINVAL;
+ rxe_dbg("bad num_sge > max_sge");
+ goto err_out;
+ }
+
+ length = 0;
+ for (i = 0; i < num_sge; i++)
+ length += ibwr->sg_list[i].length;
+
+ /* IBA max message size is 2^31 */
+ if (length >= (1UL<<31)) {
+ err = -EINVAL;
+ rxe_dbg("message length too long");
+ goto err_out;
+ }
+
+ recv_wqe = queue_producer_addr(rq->queue, QUEUE_TYPE_FROM_ULP);
+
+ recv_wqe->wr_id = ibwr->wr_id;
+ recv_wqe->dma.length = length;
+ recv_wqe->dma.resid = length;
+ recv_wqe->dma.num_sge = num_sge;
+ recv_wqe->dma.cur_sge = 0;
+ recv_wqe->dma.sge_offset = 0;
+ memcpy(recv_wqe->dma.sge, ibwr->sg_list,
+ num_sge * sizeof(struct ib_sge));
+
+ queue_advance_producer(rq->queue, QUEUE_TYPE_FROM_ULP);
+
+ return 0;
+
+err_out:
+ rxe_dbg("returned err = %d", err);
+ return err;
}
static int rxe_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
@@ -717,13 +1008,26 @@ static int rxe_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
struct rxe_rq *rq = &qp->rq;
unsigned long flags;
- if (unlikely((qp_state(qp) < IB_QPS_INIT) || !qp->valid)) {
+ spin_lock_bh(&qp->state_lock);
+ /* caller has already called destroy_qp */
+ if (WARN_ON_ONCE(!qp->valid)) {
+ spin_unlock_bh(&qp->state_lock);
+ rxe_err_qp(qp, "qp has been destroyed");
+ return -EINVAL;
+ }
+
+ /* see C10-97.2.1 */
+ if (unlikely((qp_state(qp) < IB_QPS_INIT))) {
+ spin_unlock_bh(&qp->state_lock);
*bad_wr = wr;
+ rxe_dbg_qp(qp, "qp not ready to post recv");
return -EINVAL;
}
+ spin_unlock_bh(&qp->state_lock);
if (unlikely(qp->srq)) {
*bad_wr = wr;
+ rxe_dbg_qp(qp, "qp has srq, use post_srq_recv instead");
return -EINVAL;
}
@@ -740,76 +1044,102 @@ static int rxe_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
spin_unlock_irqrestore(&rq->producer_lock, flags);
- if (qp->resp.state == QP_STATE_ERROR)
+ spin_lock_bh(&qp->state_lock);
+ if (qp_state(qp) == IB_QPS_ERR)
rxe_sched_task(&qp->resp.task);
+ spin_unlock_bh(&qp->state_lock);
return err;
}
+/* cq */
static int rxe_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
struct ib_udata *udata)
{
- int err;
struct ib_device *dev = ibcq->device;
struct rxe_dev *rxe = to_rdev(dev);
struct rxe_cq *cq = to_rcq(ibcq);
struct rxe_create_cq_resp __user *uresp = NULL;
+ int err, cleanup_err;
if (udata) {
- if (udata->outlen < sizeof(*uresp))
- return -EINVAL;
+ if (udata->outlen < sizeof(*uresp)) {
+ err = -EINVAL;
+ rxe_dbg_dev(rxe, "malformed udata, err = %d", err);
+ goto err_out;
+ }
uresp = udata->outbuf;
}
- if (attr->flags)
- return -EOPNOTSUPP;
+ if (attr->flags) {
+ err = -EOPNOTSUPP;
+ rxe_dbg_dev(rxe, "bad attr->flags, err = %d", err);
+ goto err_out;
+ }
err = rxe_cq_chk_attr(rxe, NULL, attr->cqe, attr->comp_vector);
- if (err)
- return err;
+ if (err) {
+ rxe_dbg_dev(rxe, "bad init attributes, err = %d", err);
+ goto err_out;
+ }
+
+ err = rxe_add_to_pool(&rxe->cq_pool, cq);
+ if (err) {
+ rxe_dbg_dev(rxe, "unable to create cq, err = %d", err);
+ goto err_out;
+ }
err = rxe_cq_from_init(rxe, cq, attr->cqe, attr->comp_vector, udata,
uresp);
- if (err)
- return err;
-
- return rxe_add_to_pool(&rxe->cq_pool, cq);
-}
-
-static int rxe_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
-{
- struct rxe_cq *cq = to_rcq(ibcq);
-
- /* See IBA C11-17: The CI shall return an error if this Verb is
- * invoked while a Work Queue is still associated with the CQ.
- */
- if (atomic_read(&cq->num_wq))
- return -EINVAL;
-
- rxe_cq_disable(cq);
+ if (err) {
+ rxe_dbg_cq(cq, "create cq failed, err = %d", err);
+ goto err_cleanup;
+ }
- rxe_cleanup(cq);
return 0;
+
+err_cleanup:
+ cleanup_err = rxe_cleanup(cq);
+ if (cleanup_err)
+ rxe_err_cq(cq, "cleanup failed, err = %d", cleanup_err);
+err_out:
+ rxe_err_dev(rxe, "returned err = %d", err);
+ return err;
}
static int rxe_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
{
- int err;
struct rxe_cq *cq = to_rcq(ibcq);
struct rxe_dev *rxe = to_rdev(ibcq->device);
struct rxe_resize_cq_resp __user *uresp = NULL;
+ int err;
if (udata) {
- if (udata->outlen < sizeof(*uresp))
- return -EINVAL;
+ if (udata->outlen < sizeof(*uresp)) {
+ err = -EINVAL;
+ rxe_dbg_cq(cq, "malformed udata");
+ goto err_out;
+ }
uresp = udata->outbuf;
}
err = rxe_cq_chk_attr(rxe, cq, cqe, 0);
- if (err)
- return err;
+ if (err) {
+ rxe_dbg_cq(cq, "bad attr, err = %d", err);
+ goto err_out;
+ }
- return rxe_cq_resize_queue(cq, cqe, uresp, udata);
+ err = rxe_cq_resize_queue(cq, cqe, uresp, udata);
+ if (err) {
+ rxe_dbg_cq(cq, "resize cq failed, err = %d", err);
+ goto err_out;
+ }
+
+ return 0;
+
+err_out:
+ rxe_err_cq(cq, "returned err = %d", err);
+ return err;
}
static int rxe_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
@@ -823,7 +1153,7 @@ static int rxe_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
for (i = 0; i < num_entries; i++) {
cqe = queue_head(cq->queue, QUEUE_TYPE_TO_ULP);
if (!cqe)
- break;
+ break; /* queue empty */
memcpy(wc++, &cqe->ibwc, sizeof(*wc));
queue_advance_consumer(cq->queue, QUEUE_TYPE_TO_ULP);
@@ -864,6 +1194,32 @@ static int rxe_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
return ret;
}
+static int rxe_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
+{
+ struct rxe_cq *cq = to_rcq(ibcq);
+ int err;
+
+ /* See IBA C11-17: The CI shall return an error if this Verb is
+ * invoked while a Work Queue is still associated with the CQ.
+ */
+ if (atomic_read(&cq->num_wq)) {
+ err = -EINVAL;
+ rxe_dbg_cq(cq, "still in use");
+ goto err_out;
+ }
+
+ err = rxe_cleanup(cq);
+ if (err)
+ rxe_err_cq(cq, "cleanup failed, err = %d", err);
+
+ return 0;
+
+err_out:
+ rxe_err_cq(cq, "returned err = %d", err);
+ return err;
+}
+
+/* mr */
static struct ib_mr *rxe_get_dma_mr(struct ib_pd *ibpd, int access)
{
struct rxe_dev *rxe = to_rdev(ibpd->device);
@@ -872,14 +1228,14 @@ static struct ib_mr *rxe_get_dma_mr(struct ib_pd *ibpd, int access)
int err;
mr = kzalloc(sizeof(*mr), GFP_KERNEL);
- if (!mr) {
- err = -ENOMEM;
- goto err_out;
- }
+ if (!mr)
+ return ERR_PTR(-ENOMEM);
err = rxe_add_to_pool(&rxe->mr_pool, mr);
- if (err)
+ if (err) {
+ rxe_dbg_dev(rxe, "unable to create mr");
goto err_free;
+ }
rxe_get(pd);
mr->ibmr.pd = ibpd;
@@ -891,47 +1247,49 @@ static struct ib_mr *rxe_get_dma_mr(struct ib_pd *ibpd, int access)
err_free:
kfree(mr);
-err_out:
+ rxe_err_pd(pd, "returned err = %d", err);
return ERR_PTR(err);
}
-static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd,
- u64 start,
- u64 length,
- u64 iova,
- int access, struct ib_udata *udata)
+static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd, u64 start,
+ u64 length, u64 iova, int access,
+ struct ib_udata *udata)
{
- int err;
struct rxe_dev *rxe = to_rdev(ibpd->device);
struct rxe_pd *pd = to_rpd(ibpd);
struct rxe_mr *mr;
+ int err, cleanup_err;
mr = kzalloc(sizeof(*mr), GFP_KERNEL);
- if (!mr) {
- err = -ENOMEM;
- goto err_out;
- }
+ if (!mr)
+ return ERR_PTR(-ENOMEM);
err = rxe_add_to_pool(&rxe->mr_pool, mr);
- if (err)
+ if (err) {
+ rxe_dbg_pd(pd, "unable to create mr");
goto err_free;
+ }
rxe_get(pd);
mr->ibmr.pd = ibpd;
mr->ibmr.device = ibpd->device;
err = rxe_mr_init_user(rxe, start, length, iova, access, mr);
- if (err)
+ if (err) {
+ rxe_dbg_mr(mr, "reg_user_mr failed, err = %d", err);
goto err_cleanup;
+ }
rxe_finalize(mr);
return &mr->ibmr;
err_cleanup:
- rxe_cleanup(mr);
+ cleanup_err = rxe_cleanup(mr);
+ if (cleanup_err)
+ rxe_err_mr(mr, "cleanup failed, err = %d", cleanup_err);
err_free:
kfree(mr);
-err_out:
+ rxe_err_pd(pd, "returned err = %d", err);
return ERR_PTR(err);
}
@@ -941,17 +1299,19 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
struct rxe_dev *rxe = to_rdev(ibpd->device);
struct rxe_pd *pd = to_rpd(ibpd);
struct rxe_mr *mr;
- int err;
+ int err, cleanup_err;
- if (mr_type != IB_MR_TYPE_MEM_REG)
- return ERR_PTR(-EINVAL);
-
- mr = kzalloc(sizeof(*mr), GFP_KERNEL);
- if (!mr) {
- err = -ENOMEM;
+ if (mr_type != IB_MR_TYPE_MEM_REG) {
+ err = -EINVAL;
+ rxe_dbg_pd(pd, "mr type %d not supported, err = %d",
+ mr_type, err);
goto err_out;
}
+ mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+ if (!mr)
+ return ERR_PTR(-ENOMEM);
+
err = rxe_add_to_pool(&rxe->mr_pool, mr);
if (err)
goto err_free;
@@ -961,20 +1321,49 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
mr->ibmr.device = ibpd->device;
err = rxe_mr_init_fast(max_num_sg, mr);
- if (err)
+ if (err) {
+ rxe_dbg_mr(mr, "alloc_mr failed, err = %d", err);
goto err_cleanup;
+ }
rxe_finalize(mr);
return &mr->ibmr;
err_cleanup:
- rxe_cleanup(mr);
+ cleanup_err = rxe_cleanup(mr);
+ if (cleanup_err)
+ rxe_err_mr(mr, "cleanup failed, err = %d", err);
err_free:
kfree(mr);
err_out:
+ rxe_err_pd(pd, "returned err = %d", err);
return ERR_PTR(err);
}
+static int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
+{
+ struct rxe_mr *mr = to_rmr(ibmr);
+ int err, cleanup_err;
+
+ /* See IBA 10.6.7.2.6 */
+ if (atomic_read(&mr->num_mw) > 0) {
+ err = -EINVAL;
+ rxe_dbg_mr(mr, "mr has mw's bound");
+ goto err_out;
+ }
+
+ cleanup_err = rxe_cleanup(mr);
+ if (cleanup_err)
+ rxe_err_mr(mr, "cleanup failed, err = %d", cleanup_err);
+
+ kfree_rcu(mr);
+ return 0;
+
+err_out:
+ rxe_err_mr(mr, "returned err = %d", err);
+ return err;
+}
+
static ssize_t parent_show(struct device *device,
struct device_attribute *attr, char *buf)
{
@@ -1095,7 +1484,7 @@ int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name)
err = ib_register_device(dev, ibdev_name, NULL);
if (err)
- rxe_dbg(rxe, "failed with error %d\n", err);
+ rxe_dbg_dev(rxe, "failed with error %d\n", err);
/*
* Note that rxe may be invalid at this point if another thread
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index c269ae2a3224..26a20f088692 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -63,9 +63,7 @@ struct rxe_cq {
struct rxe_queue *queue;
spinlock_t cq_lock;
u8 notify;
- bool is_dying;
bool is_user;
- struct tasklet_struct comp_task;
atomic_t num_wq;
};
@@ -104,17 +102,7 @@ struct rxe_srq {
int error;
};
-enum rxe_qp_state {
- QP_STATE_RESET,
- QP_STATE_INIT,
- QP_STATE_READY,
- QP_STATE_DRAIN, /* req only */
- QP_STATE_DRAINED, /* req only */
- QP_STATE_ERROR
-};
-
struct rxe_req_info {
- enum rxe_qp_state state;
int wqe_index;
u32 psn;
int opcode;
@@ -129,7 +117,6 @@ struct rxe_req_info {
};
struct rxe_comp_info {
- enum rxe_qp_state state;
u32 psn;
int opcode;
int timeout;
@@ -175,7 +162,6 @@ struct resp_res {
};
struct rxe_resp_info {
- enum rxe_qp_state state;
u32 msn;
u32 psn;
u32 ack_psn;
diff --git a/drivers/infiniband/sw/siw/siw_main.c b/drivers/infiniband/sw/siw/siw_main.c
index dacc174604bf..65b5cda5457b 100644
--- a/drivers/infiniband/sw/siw/siw_main.c
+++ b/drivers/infiniband/sw/siw/siw_main.c
@@ -437,9 +437,6 @@ static int siw_netdev_event(struct notifier_block *nb, unsigned long event,
dev_dbg(&netdev->dev, "siw: event %lu\n", event);
- if (dev_net(netdev) != &init_net)
- return NOTIFY_OK;
-
base_dev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_SIW);
if (!base_dev)
return NOTIFY_OK;
diff --git a/drivers/infiniband/sw/siw/siw_qp_rx.c b/drivers/infiniband/sw/siw/siw_qp_rx.c
index fd721cc19682..58bbf738e4e5 100644
--- a/drivers/infiniband/sw/siw/siw_qp_rx.c
+++ b/drivers/infiniband/sw/siw/siw_qp_rx.c
@@ -139,7 +139,7 @@ static int siw_rx_pbl(struct siw_rx_stream *srx, int *pbl_idx,
break;
bytes = min(bytes, len);
- if (siw_rx_kva(srx, (void *)(uintptr_t)buf_addr, bytes) ==
+ if (siw_rx_kva(srx, ib_virt_dma_to_ptr(buf_addr), bytes) ==
bytes) {
copied += bytes;
offset += bytes;
@@ -487,7 +487,7 @@ int siw_proc_send(struct siw_qp *qp)
mem_p = *mem;
if (mem_p->mem_obj == NULL)
rv = siw_rx_kva(srx,
- (void *)(uintptr_t)(sge->laddr + frx->sge_off),
+ ib_virt_dma_to_ptr(sge->laddr + frx->sge_off),
sge_bytes);
else if (!mem_p->is_pbl)
rv = siw_rx_umem(srx, mem_p->umem,
@@ -852,7 +852,7 @@ int siw_proc_rresp(struct siw_qp *qp)
if (mem_p->mem_obj == NULL)
rv = siw_rx_kva(srx,
- (void *)(uintptr_t)(sge->laddr + wqe->processed),
+ ib_virt_dma_to_ptr(sge->laddr + wqe->processed),
bytes);
else if (!mem_p->is_pbl)
rv = siw_rx_umem(srx, mem_p->umem, sge->laddr + wqe->processed,
diff --git a/drivers/infiniband/sw/siw/siw_qp_tx.c b/drivers/infiniband/sw/siw/siw_qp_tx.c
index 05052b49107f..4b292e0504f1 100644
--- a/drivers/infiniband/sw/siw/siw_qp_tx.c
+++ b/drivers/infiniband/sw/siw/siw_qp_tx.c
@@ -29,7 +29,7 @@ static struct page *siw_get_pblpage(struct siw_mem *mem, u64 addr, int *idx)
dma_addr_t paddr = siw_pbl_get_buffer(pbl, offset, NULL, idx);
if (paddr)
- return virt_to_page((void *)(uintptr_t)paddr);
+ return ib_virt_dma_to_page(paddr);
return NULL;
}
@@ -56,8 +56,7 @@ static int siw_try_1seg(struct siw_iwarp_tx *c_tx, void *paddr)
if (!mem->mem_obj) {
/* Kernel client using kva */
- memcpy(paddr,
- (const void *)(uintptr_t)sge->laddr, bytes);
+ memcpy(paddr, ib_virt_dma_to_ptr(sge->laddr), bytes);
} else if (c_tx->in_syscall) {
if (copy_from_user(paddr, u64_to_user_ptr(sge->laddr),
bytes))
@@ -477,7 +476,7 @@ static int siw_tx_hdt(struct siw_iwarp_tx *c_tx, struct socket *s)
* or memory region with assigned kernel buffer
*/
iov[seg].iov_base =
- (void *)(uintptr_t)(sge->laddr + sge_off);
+ ib_virt_dma_to_ptr(sge->laddr + sge_off);
iov[seg].iov_len = sge_len;
if (do_crc)
@@ -537,19 +536,13 @@ static int siw_tx_hdt(struct siw_iwarp_tx *c_tx, struct socket *s)
* Cast to an uintptr_t to preserve all 64 bits
* in sge->laddr.
*/
- uintptr_t va = (uintptr_t)(sge->laddr + sge_off);
+ u64 va = sge->laddr + sge_off;
- /*
- * virt_to_page() takes a (void *) pointer
- * so cast to a (void *) meaning it will be 64
- * bits on a 64 bit platform and 32 bits on a
- * 32 bit platform.
- */
- page_array[seg] = virt_to_page((void *)(va & PAGE_MASK));
+ page_array[seg] = ib_virt_dma_to_page(va);
if (do_crc)
crypto_shash_update(
c_tx->mpa_crc_hd,
- (void *)va,
+ ib_virt_dma_to_ptr(va),
plen);
}
@@ -558,7 +551,7 @@ static int siw_tx_hdt(struct siw_iwarp_tx *c_tx, struct socket *s)
data_len -= plen;
fp_off = 0;
- if (++seg > (int)MAX_ARRAY) {
+ if (++seg >= (int)MAX_ARRAY) {
siw_dbg_qp(tx_qp(c_tx), "to many fragments\n");
siw_unmap_pages(iov, kmap_mask, seg-1);
wqe->processed -= c_tx->bytes_unsent;
diff --git a/drivers/infiniband/sw/siw/siw_verbs.c b/drivers/infiniband/sw/siw/siw_verbs.c
index 906fde1a2a0d..398ec13db624 100644
--- a/drivers/infiniband/sw/siw/siw_verbs.c
+++ b/drivers/infiniband/sw/siw/siw_verbs.c
@@ -660,7 +660,7 @@ static int siw_copy_inline_sgl(const struct ib_send_wr *core_wr,
bytes = -EINVAL;
break;
}
- memcpy(kbuf, (void *)(uintptr_t)core_sge->addr,
+ memcpy(kbuf, ib_virt_dma_to_ptr(core_sge->addr),
core_sge->length);
kbuf += core_sge->length;
@@ -1523,7 +1523,7 @@ int siw_map_mr_sg(struct ib_mr *base_mr, struct scatterlist *sl, int num_sle,
}
siw_dbg_mem(mem,
"sge[%d], size %u, addr 0x%p, total %lu\n",
- i, pble->size, (void *)(uintptr_t)pble->addr,
+ i, pble->size, ib_virt_dma_to_ptr(pble->addr),
pbl_size);
}
rv = ib_sg_to_pages(base_mr, sl, num_sle, sg_off, siw_set_pbl_page);
diff --git a/drivers/infiniband/ulp/iser/iser_initiator.c b/drivers/infiniband/ulp/iser/iser_initiator.c
index 7b83f48f60c5..39ea73f69016 100644
--- a/drivers/infiniband/ulp/iser/iser_initiator.c
+++ b/drivers/infiniband/ulp/iser/iser_initiator.c
@@ -141,10 +141,14 @@ out_err:
/* creates a new tx descriptor and adds header regd buffer */
static void iser_create_send_desc(struct iser_conn *iser_conn,
- struct iser_tx_desc *tx_desc)
+ struct iser_tx_desc *tx_desc, enum iser_desc_type type,
+ void (*done)(struct ib_cq *cq, struct ib_wc *wc))
{
struct iser_device *device = iser_conn->ib_conn.device;
+ tx_desc->type = type;
+ tx_desc->cqe.done = done;
+
ib_dma_sync_single_for_cpu(device->ib_device,
tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE);
@@ -349,9 +353,8 @@ int iser_send_command(struct iscsi_conn *conn, struct iscsi_task *task)
edtl = ntohl(hdr->data_length);
/* build the tx desc regd header and add it to the tx desc dto */
- tx_desc->type = ISCSI_TX_SCSI_COMMAND;
- tx_desc->cqe.done = iser_cmd_comp;
- iser_create_send_desc(iser_conn, tx_desc);
+ iser_create_send_desc(iser_conn, tx_desc, ISCSI_TX_SCSI_COMMAND,
+ iser_cmd_comp);
if (hdr->flags & ISCSI_FLAG_CMD_READ) {
data_buf = &iser_task->data[ISER_DIR_IN];
@@ -457,7 +460,6 @@ int iser_send_data_out(struct iscsi_conn *conn, struct iscsi_task *task,
iser_dbg("data-out itt: %d, offset: %ld, sz: %ld\n",
itt, buf_offset, data_seg_len);
-
err = iser_post_send(&iser_conn->ib_conn, tx_desc);
if (!err)
return 0;
@@ -478,9 +480,8 @@ int iser_send_control(struct iscsi_conn *conn, struct iscsi_task *task)
struct iser_device *device;
/* build the tx desc regd header and add it to the tx desc dto */
- mdesc->type = ISCSI_TX_CONTROL;
- mdesc->cqe.done = iser_ctrl_comp;
- iser_create_send_desc(iser_conn, mdesc);
+ iser_create_send_desc(iser_conn, mdesc, ISCSI_TX_CONTROL,
+ iser_ctrl_comp);
device = iser_conn->ib_conn.device;
diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c
index 1b8eda0dae4e..95b8eebf7e04 100644
--- a/drivers/infiniband/ulp/iser/iser_verbs.c
+++ b/drivers/infiniband/ulp/iser/iser_verbs.c
@@ -37,12 +37,6 @@
#include "iscsi_iser.h"
-#define ISCSI_ISER_MAX_CONN 8
-#define ISER_MAX_RX_LEN (ISER_QP_MAX_RECV_DTOS * ISCSI_ISER_MAX_CONN)
-#define ISER_MAX_TX_LEN (ISER_QP_MAX_REQ_DTOS * ISCSI_ISER_MAX_CONN)
-#define ISER_MAX_CQ_LEN (ISER_MAX_RX_LEN + ISER_MAX_TX_LEN + \
- ISCSI_ISER_MAX_CONN)
-
static void iser_qp_event_callback(struct ib_event *cause, void *context)
{
iser_err("qp event %s (%d)\n",
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c
index 3e9343fd2d8e..c12005eab14c 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.c
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.c
@@ -549,6 +549,7 @@ static int srpt_format_guid(char *buf, unsigned int size, const __be64 *guid)
*/
static int srpt_refresh_port(struct srpt_port *sport)
{
+ struct ib_mad_agent *mad_agent;
struct ib_mad_reg_req reg_req;
struct ib_port_modify port_modify;
struct ib_port_attr port_attr;
@@ -593,24 +594,26 @@ static int srpt_refresh_port(struct srpt_port *sport)
set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
- sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
- sport->port,
- IB_QPT_GSI,
- &reg_req, 0,
- srpt_mad_send_handler,
- srpt_mad_recv_handler,
- sport, 0);
- if (IS_ERR(sport->mad_agent)) {
+ mad_agent = ib_register_mad_agent(sport->sdev->device,
+ sport->port,
+ IB_QPT_GSI,
+ &reg_req, 0,
+ srpt_mad_send_handler,
+ srpt_mad_recv_handler,
+ sport, 0);
+ if (IS_ERR(mad_agent)) {
pr_err("%s-%d: MAD agent registration failed (%ld). Note: this is expected if SR-IOV is enabled.\n",
dev_name(&sport->sdev->device->dev), sport->port,
- PTR_ERR(sport->mad_agent));
+ PTR_ERR(mad_agent));
sport->mad_agent = NULL;
memset(&port_modify, 0, sizeof(port_modify));
port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
ib_modify_port(sport->sdev->device, sport->port, 0,
&port_modify);
-
+ return 0;
}
+
+ sport->mad_agent = mad_agent;
}
return 0;
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h
index a5408879e077..b31de4cf6534 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h
@@ -10321,6 +10321,87 @@ struct hwrm_selftest_irq_output {
u8 valid;
};
+/* dbc_dbc (size:64b/8B) */
+struct dbc_dbc {
+ u32 index;
+ #define DBC_DBC_INDEX_MASK 0xffffffUL
+ #define DBC_DBC_INDEX_SFT 0
+ #define DBC_DBC_EPOCH 0x1000000UL
+ #define DBC_DBC_TOGGLE_MASK 0x6000000UL
+ #define DBC_DBC_TOGGLE_SFT 25
+ u32 type_path_xid;
+ #define DBC_DBC_XID_MASK 0xfffffUL
+ #define DBC_DBC_XID_SFT 0
+ #define DBC_DBC_PATH_MASK 0x3000000UL
+ #define DBC_DBC_PATH_SFT 24
+ #define DBC_DBC_PATH_ROCE (0x0UL << 24)
+ #define DBC_DBC_PATH_L2 (0x1UL << 24)
+ #define DBC_DBC_PATH_ENGINE (0x2UL << 24)
+ #define DBC_DBC_PATH_LAST DBC_DBC_PATH_ENGINE
+ #define DBC_DBC_VALID 0x4000000UL
+ #define DBC_DBC_DEBUG_TRACE 0x8000000UL
+ #define DBC_DBC_TYPE_MASK 0xf0000000UL
+ #define DBC_DBC_TYPE_SFT 28
+ #define DBC_DBC_TYPE_SQ (0x0UL << 28)
+ #define DBC_DBC_TYPE_RQ (0x1UL << 28)
+ #define DBC_DBC_TYPE_SRQ (0x2UL << 28)
+ #define DBC_DBC_TYPE_SRQ_ARM (0x3UL << 28)
+ #define DBC_DBC_TYPE_CQ (0x4UL << 28)
+ #define DBC_DBC_TYPE_CQ_ARMSE (0x5UL << 28)
+ #define DBC_DBC_TYPE_CQ_ARMALL (0x6UL << 28)
+ #define DBC_DBC_TYPE_CQ_ARMENA (0x7UL << 28)
+ #define DBC_DBC_TYPE_SRQ_ARMENA (0x8UL << 28)
+ #define DBC_DBC_TYPE_CQ_CUTOFF_ACK (0x9UL << 28)
+ #define DBC_DBC_TYPE_NQ (0xaUL << 28)
+ #define DBC_DBC_TYPE_NQ_ARM (0xbUL << 28)
+ #define DBC_DBC_TYPE_NQ_MASK (0xeUL << 28)
+ #define DBC_DBC_TYPE_NULL (0xfUL << 28)
+ #define DBC_DBC_TYPE_LAST DBC_DBC_TYPE_NULL
+};
+
+/* db_push_start (size:64b/8B) */
+struct db_push_start {
+ u64 db;
+ #define DB_PUSH_START_DB_INDEX_MASK 0xffffffUL
+ #define DB_PUSH_START_DB_INDEX_SFT 0
+ #define DB_PUSH_START_DB_PI_LO_MASK 0xff000000UL
+ #define DB_PUSH_START_DB_PI_LO_SFT 24
+ #define DB_PUSH_START_DB_XID_MASK 0xfffff00000000ULL
+ #define DB_PUSH_START_DB_XID_SFT 32
+ #define DB_PUSH_START_DB_PI_HI_MASK 0xf0000000000000ULL
+ #define DB_PUSH_START_DB_PI_HI_SFT 52
+ #define DB_PUSH_START_DB_TYPE_MASK 0xf000000000000000ULL
+ #define DB_PUSH_START_DB_TYPE_SFT 60
+ #define DB_PUSH_START_DB_TYPE_PUSH_START (0xcULL << 60)
+ #define DB_PUSH_START_DB_TYPE_PUSH_END (0xdULL << 60)
+ #define DB_PUSH_START_DB_TYPE_LAST DB_PUSH_START_DB_TYPE_PUSH_END
+};
+
+/* db_push_end (size:64b/8B) */
+struct db_push_end {
+ u64 db;
+ #define DB_PUSH_END_DB_INDEX_MASK 0xffffffUL
+ #define DB_PUSH_END_DB_INDEX_SFT 0
+ #define DB_PUSH_END_DB_PI_LO_MASK 0xff000000UL
+ #define DB_PUSH_END_DB_PI_LO_SFT 24
+ #define DB_PUSH_END_DB_XID_MASK 0xfffff00000000ULL
+ #define DB_PUSH_END_DB_XID_SFT 32
+ #define DB_PUSH_END_DB_PI_HI_MASK 0xf0000000000000ULL
+ #define DB_PUSH_END_DB_PI_HI_SFT 52
+ #define DB_PUSH_END_DB_PATH_MASK 0x300000000000000ULL
+ #define DB_PUSH_END_DB_PATH_SFT 56
+ #define DB_PUSH_END_DB_PATH_ROCE (0x0ULL << 56)
+ #define DB_PUSH_END_DB_PATH_L2 (0x1ULL << 56)
+ #define DB_PUSH_END_DB_PATH_ENGINE (0x2ULL << 56)
+ #define DB_PUSH_END_DB_PATH_LAST DB_PUSH_END_DB_PATH_ENGINE
+ #define DB_PUSH_END_DB_DEBUG_TRACE 0x800000000000000ULL
+ #define DB_PUSH_END_DB_TYPE_MASK 0xf000000000000000ULL
+ #define DB_PUSH_END_DB_TYPE_SFT 60
+ #define DB_PUSH_END_DB_TYPE_PUSH_START (0xcULL << 60)
+ #define DB_PUSH_END_DB_TYPE_PUSH_END (0xdULL << 60)
+ #define DB_PUSH_END_DB_TYPE_LAST DB_PUSH_END_DB_TYPE_PUSH_END
+};
+
/* db_push_info (size:64b/8B) */
struct db_push_info {
u32 push_size_push_index;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
index ef546ed8b4d9..9c94807097cb 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
@@ -930,8 +930,7 @@ static void mlx5e_build_rx_cq_param(struct mlx5_core_dev *mdev,
static u8 rq_end_pad_mode(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
{
bool lro_en = params->packet_merge.type == MLX5E_PACKET_MERGE_LRO;
- bool ro = pcie_relaxed_ordering_enabled(mdev->pdev) &&
- MLX5_CAP_GEN(mdev, relaxed_ordering_write);
+ bool ro = MLX5_CAP_GEN(mdev, relaxed_ordering_write);
return ro && lro_en ?
MLX5_WQ_END_PAD_MODE_NONE : MLX5_WQ_END_PAD_MODE_ALIGN;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c
index 4c9a3210600c..1f90594499c6 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c
@@ -39,12 +39,13 @@
void mlx5e_mkey_set_relaxed_ordering(struct mlx5_core_dev *mdev, void *mkc)
{
- bool ro_pci_enable = pcie_relaxed_ordering_enabled(mdev->pdev);
bool ro_write = MLX5_CAP_GEN(mdev, relaxed_ordering_write);
- bool ro_read = MLX5_CAP_GEN(mdev, relaxed_ordering_read);
+ bool ro_read = MLX5_CAP_GEN(mdev, relaxed_ordering_read) ||
+ (pcie_relaxed_ordering_enabled(mdev->pdev) &&
+ MLX5_CAP_GEN(mdev, relaxed_ordering_read_pci_enabled));
- MLX5_SET(mkc, mkc, relaxed_ordering_read, ro_pci_enable && ro_read);
- MLX5_SET(mkc, mkc, relaxed_ordering_write, ro_pci_enable && ro_write);
+ MLX5_SET(mkc, mkc, relaxed_ordering_read, ro_read);
+ MLX5_SET(mkc, mkc, relaxed_ordering_write, ro_write);
}
int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, u32 *mkey)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index edc738e86cac..995eb2d5ace0 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -691,6 +691,9 @@ static int handle_hca_cap_roce(struct mlx5_core_dev *dev, void *set_ctx)
MLX5_ST_SZ_BYTES(roce_cap));
MLX5_SET(roce_cap, set_hca_cap, sw_r_roce_src_udp_port, 1);
+ if (MLX5_CAP_ROCE_MAX(dev, qp_ooo_transmit_default))
+ MLX5_SET(roce_cap, set_hca_cap, qp_ooo_transmit_default, 1);
+
err = set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_ROCE);
return err;
}