diff options
Diffstat (limited to 'drivers/infiniband/hw/hns')
20 files changed, 6969 insertions, 697 deletions
diff --git a/drivers/infiniband/hw/hns/Kconfig b/drivers/infiniband/hw/hns/Kconfig index 61c93bbd230d..fddb5fdf92de 100644 --- a/drivers/infiniband/hw/hns/Kconfig +++ b/drivers/infiniband/hw/hns/Kconfig @@ -1,10 +1,31 @@ config INFINIBAND_HNS tristate "HNS RoCE Driver" depends on NET_VENDOR_HISILICON - depends on (ARM64 || (COMPILE_TEST && 64BIT)) && HNS && HNS_DSAF && HNS_ENET + depends on ARM64 || (COMPILE_TEST && 64BIT) ---help--- This is a RoCE/RDMA driver for the Hisilicon RoCE engine. The engine - is used in Hisilicon Hi1610 and more further ICT SoC. + is used in Hisilicon Hip06 and more further ICT SoC based on + platform device. To compile this driver as a module, choose M here: the module will be called hns-roce. + +config INFINIBAND_HNS_HIP06 + tristate "Hisilicon Hip06 Family RoCE support" + depends on INFINIBAND_HNS && HNS && HNS_DSAF && HNS_ENET + ---help--- + RoCE driver support for Hisilicon RoCE engine in Hisilicon Hip06 and + Hip07 SoC. These RoCE engines are platform devices. + + To compile this driver as a module, choose M here: the module + will be called hns-roce-hw-v1. + +config INFINIBAND_HNS_HIP08 + tristate "Hisilicon Hip08 Family RoCE support" + depends on INFINIBAND_HNS && PCI && HNS3 + ---help--- + RoCE driver support for Hisilicon RoCE engine in Hisilicon Hip08 SoC. + The RoCE engine is a PCI device. + + To compile this driver as a module, choose M here: the module + will be called hns-roce-hw-v2. diff --git a/drivers/infiniband/hw/hns/Makefile b/drivers/infiniband/hw/hns/Makefile index 7e8ebd24dcae..ff426a625e13 100644 --- a/drivers/infiniband/hw/hns/Makefile +++ b/drivers/infiniband/hw/hns/Makefile @@ -2,7 +2,13 @@ # Makefile for the Hisilicon RoCE drivers. # +ccflags-y := -Idrivers/net/ethernet/hisilicon/hns3 + obj-$(CONFIG_INFINIBAND_HNS) += hns-roce.o hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_eq.o hns_roce_pd.o \ hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \ - hns_roce_cq.o hns_roce_alloc.o hns_roce_hw_v1.o + hns_roce_cq.o hns_roce_alloc.o +obj-$(CONFIG_INFINIBAND_HNS_HIP06) += hns-roce-hw-v1.o +hns-roce-hw-v1-objs := hns_roce_hw_v1.o +obj-$(CONFIG_INFINIBAND_HNS_HIP08) += hns-roce-hw-v2.o +hns-roce-hw-v2-objs := hns_roce_hw_v2.o diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c index d545302b8ef8..7dd6a66ea244 100644 --- a/drivers/infiniband/hw/hns/hns_roce_ah.c +++ b/drivers/infiniband/hw/hns/hns_roce_ah.c @@ -44,11 +44,10 @@ struct ib_ah *hns_roce_create_ah(struct ib_pd *ibpd, struct ib_udata *udata) { struct hns_roce_dev *hr_dev = to_hr_dev(ibpd->device); - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; struct ib_gid_attr gid_attr; struct hns_roce_ah *ah; u16 vlan_tag = 0xffff; - struct in6_addr in6; const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr); union ib_gid sgid; int ret; @@ -58,18 +57,7 @@ struct ib_ah *hns_roce_create_ah(struct ib_pd *ibpd, return ERR_PTR(-ENOMEM); /* Get mac address */ - memcpy(&in6, grh->dgid.raw, sizeof(grh->dgid.raw)); - if (rdma_is_multicast_addr(&in6)) { - rdma_get_mcast_mac(&in6, ah->av.mac); - } else { - u8 *dmac = rdma_ah_retrieve_dmac(ah_attr); - - if (!dmac) { - kfree(ah); - return ERR_PTR(-EINVAL); - } - memcpy(ah->av.mac, dmac, ETH_ALEN); - } + memcpy(ah->av.mac, ah_attr->roce.dmac, ETH_ALEN); /* Get source gid */ ret = ib_get_cached_gid(ibpd->device, rdma_ah_get_port_num(ah_attr), diff --git a/drivers/infiniband/hw/hns/hns_roce_alloc.c b/drivers/infiniband/hw/hns/hns_roce_alloc.c index e1b433cdd5e2..3e4c5253ab5c 100644 --- a/drivers/infiniband/hw/hns/hns_roce_alloc.c +++ b/drivers/infiniband/hw/hns/hns_roce_alloc.c @@ -67,6 +67,7 @@ void hns_roce_bitmap_free(struct hns_roce_bitmap *bitmap, unsigned long obj, { hns_roce_bitmap_free_range(bitmap, obj, 1, rr); } +EXPORT_SYMBOL_GPL(hns_roce_bitmap_free); int hns_roce_bitmap_alloc_range(struct hns_roce_bitmap *bitmap, int cnt, int align, unsigned long *obj) @@ -160,39 +161,47 @@ void hns_roce_buf_free(struct hns_roce_dev *hr_dev, u32 size, struct hns_roce_buf *buf) { int i; - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; u32 bits_per_long = BITS_PER_LONG; if (buf->nbufs == 1) { dma_free_coherent(dev, size, buf->direct.buf, buf->direct.map); } else { - if (bits_per_long == 64) + if (bits_per_long == 64 && buf->page_shift == PAGE_SHIFT) vunmap(buf->direct.buf); for (i = 0; i < buf->nbufs; ++i) if (buf->page_list[i].buf) - dma_free_coherent(&hr_dev->pdev->dev, PAGE_SIZE, + dma_free_coherent(dev, 1 << buf->page_shift, buf->page_list[i].buf, buf->page_list[i].map); kfree(buf->page_list); } } +EXPORT_SYMBOL_GPL(hns_roce_buf_free); int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct, - struct hns_roce_buf *buf) + struct hns_roce_buf *buf, u32 page_shift) { int i = 0; dma_addr_t t; struct page **pages; - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; u32 bits_per_long = BITS_PER_LONG; + u32 page_size = 1 << page_shift; + u32 order; /* SQ/RQ buf lease than one page, SQ + RQ = 8K */ if (size <= max_direct) { buf->nbufs = 1; /* Npages calculated by page_size */ - buf->npages = 1 << get_order(size); - buf->page_shift = PAGE_SHIFT; + order = get_order(size); + if (order <= page_shift - PAGE_SHIFT) + order = 0; + else + order -= page_shift - PAGE_SHIFT; + buf->npages = 1 << order; + buf->page_shift = page_shift; /* MTT PA must be recorded in 4k alignment, t is 4k aligned */ buf->direct.buf = dma_alloc_coherent(dev, size, &t, GFP_KERNEL); if (!buf->direct.buf) @@ -207,9 +216,9 @@ int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct, memset(buf->direct.buf, 0, size); } else { - buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE; + buf->nbufs = (size + page_size - 1) / page_size; buf->npages = buf->nbufs; - buf->page_shift = PAGE_SHIFT; + buf->page_shift = page_shift; buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list), GFP_KERNEL); @@ -218,16 +227,16 @@ int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct, for (i = 0; i < buf->nbufs; ++i) { buf->page_list[i].buf = dma_alloc_coherent(dev, - PAGE_SIZE, &t, + page_size, &t, GFP_KERNEL); if (!buf->page_list[i].buf) goto err_free; buf->page_list[i].map = t; - memset(buf->page_list[i].buf, 0, PAGE_SIZE); + memset(buf->page_list[i].buf, 0, page_size); } - if (bits_per_long == 64) { + if (bits_per_long == 64 && page_shift == PAGE_SHIFT) { pages = kmalloc_array(buf->nbufs, sizeof(*pages), GFP_KERNEL); if (!pages) @@ -241,6 +250,8 @@ int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct, kfree(pages); if (!buf->direct.buf) goto err_free; + } else { + buf->direct.buf = NULL; } } diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.c b/drivers/infiniband/hw/hns/hns_roce_cmd.c index b94dcd823ad1..1085cb249bc1 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cmd.c +++ b/drivers/infiniband/hw/hns/hns_roce_cmd.c @@ -38,69 +38,7 @@ #define CMD_POLL_TOKEN 0xffff #define CMD_MAX_NUM 32 -#define STATUS_MASK 0xff #define CMD_TOKEN_MASK 0x1f -#define GO_BIT_TIMEOUT_MSECS 10000 - -enum { - HCR_TOKEN_OFFSET = 0x14, - HCR_STATUS_OFFSET = 0x18, - HCR_GO_BIT = 15, -}; - -static int cmd_pending(struct hns_roce_dev *hr_dev) -{ - u32 status = readl(hr_dev->cmd.hcr + HCR_TOKEN_OFFSET); - - return (!!(status & (1 << HCR_GO_BIT))); -} - -/* this function should be serialized with "hcr_mutex" */ -static int __hns_roce_cmd_mbox_post_hw(struct hns_roce_dev *hr_dev, - u64 in_param, u64 out_param, - u32 in_modifier, u8 op_modifier, u16 op, - u16 token, int event) -{ - struct hns_roce_cmdq *cmd = &hr_dev->cmd; - struct device *dev = &hr_dev->pdev->dev; - u32 __iomem *hcr = (u32 *)cmd->hcr; - int ret = -EAGAIN; - unsigned long end; - u32 val = 0; - - end = msecs_to_jiffies(GO_BIT_TIMEOUT_MSECS) + jiffies; - while (cmd_pending(hr_dev)) { - if (time_after(jiffies, end)) { - dev_dbg(dev, "jiffies=%d end=%d\n", (int)jiffies, - (int)end); - goto out; - } - cond_resched(); - } - - roce_set_field(val, ROCEE_MB6_ROCEE_MB_CMD_M, ROCEE_MB6_ROCEE_MB_CMD_S, - op); - roce_set_field(val, ROCEE_MB6_ROCEE_MB_CMD_MDF_M, - ROCEE_MB6_ROCEE_MB_CMD_MDF_S, op_modifier); - roce_set_bit(val, ROCEE_MB6_ROCEE_MB_EVENT_S, event); - roce_set_bit(val, ROCEE_MB6_ROCEE_MB_HW_RUN_S, 1); - roce_set_field(val, ROCEE_MB6_ROCEE_MB_TOKEN_M, - ROCEE_MB6_ROCEE_MB_TOKEN_S, token); - - __raw_writeq(cpu_to_le64(in_param), hcr + 0); - __raw_writeq(cpu_to_le64(out_param), hcr + 2); - __raw_writel(cpu_to_le32(in_modifier), hcr + 4); - /* Memory barrier */ - wmb(); - - __raw_writel(cpu_to_le32(val), hcr + 5); - - mmiowb(); - ret = 0; - -out: - return ret; -} static int hns_roce_cmd_mbox_post_hw(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param, u32 in_modifier, @@ -108,12 +46,11 @@ static int hns_roce_cmd_mbox_post_hw(struct hns_roce_dev *hr_dev, u64 in_param, int event) { struct hns_roce_cmdq *cmd = &hr_dev->cmd; - int ret = -EAGAIN; + int ret; mutex_lock(&cmd->hcr_mutex); - ret = __hns_roce_cmd_mbox_post_hw(hr_dev, in_param, out_param, - in_modifier, op_modifier, op, token, - event); + ret = hr_dev->hw->post_mbox(hr_dev, in_param, out_param, in_modifier, + op_modifier, op, token, event); mutex_unlock(&cmd->hcr_mutex); return ret; @@ -125,10 +62,7 @@ static int __hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param, u8 op_modifier, u16 op, unsigned long timeout) { - struct device *dev = &hr_dev->pdev->dev; - u8 __iomem *hcr = hr_dev->cmd.hcr; - unsigned long end = 0; - u32 status = 0; + struct device *dev = hr_dev->dev; int ret; ret = hns_roce_cmd_mbox_post_hw(hr_dev, in_param, out_param, @@ -136,29 +70,10 @@ static int __hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param, CMD_POLL_TOKEN, 0); if (ret) { dev_err(dev, "[cmd_poll]hns_roce_cmd_mbox_post_hw failed\n"); - goto out; - } - - end = msecs_to_jiffies(timeout) + jiffies; - while (cmd_pending(hr_dev) && time_before(jiffies, end)) - cond_resched(); - - if (cmd_pending(hr_dev)) { - dev_err(dev, "[cmd_poll]hw run cmd TIMEDOUT!\n"); - ret = -ETIMEDOUT; - goto out; + return ret; } - status = le32_to_cpu((__force __be32) - __raw_readl(hcr + HCR_STATUS_OFFSET)); - if ((status & STATUS_MASK) != 0x1) { - dev_err(dev, "mailbox status 0x%x!\n", status); - ret = -EBUSY; - goto out; - } - -out: - return ret; + return hr_dev->hw->chk_mbox(hr_dev, timeout); } static int hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param, @@ -196,9 +111,9 @@ static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param, unsigned long timeout) { struct hns_roce_cmdq *cmd = &hr_dev->cmd; - struct device *dev = &hr_dev->pdev->dev; struct hns_roce_cmd_context *context; - int ret = 0; + struct device *dev = hr_dev->dev; + int ret; spin_lock(&cmd->context_lock); WARN_ON(cmd->free_head < 0); @@ -269,17 +184,17 @@ int hns_roce_cmd_mbox(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param, in_modifier, op_modifier, op, timeout); } +EXPORT_SYMBOL_GPL(hns_roce_cmd_mbox); int hns_roce_cmd_init(struct hns_roce_dev *hr_dev) { - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; mutex_init(&hr_dev->cmd.hcr_mutex); sema_init(&hr_dev->cmd.poll_sem, 1); hr_dev->cmd.use_events = 0; hr_dev->cmd.toggle = 1; hr_dev->cmd.max_cmds = CMD_MAX_NUM; - hr_dev->cmd.hcr = hr_dev->reg_base + ROCEE_MB1_REG; hr_dev->cmd.pool = dma_pool_create("hns_roce_cmd", dev, HNS_ROCE_MAILBOX_SIZE, HNS_ROCE_MAILBOX_SIZE, 0); @@ -356,6 +271,7 @@ struct hns_roce_cmd_mailbox return mailbox; } +EXPORT_SYMBOL_GPL(hns_roce_alloc_cmd_mailbox); void hns_roce_free_cmd_mailbox(struct hns_roce_dev *hr_dev, struct hns_roce_cmd_mailbox *mailbox) @@ -366,3 +282,4 @@ void hns_roce_free_cmd_mailbox(struct hns_roce_dev *hr_dev, dma_pool_free(hr_dev->cmd.pool, mailbox->buf, mailbox->dma); kfree(mailbox); } +EXPORT_SYMBOL_GPL(hns_roce_free_cmd_mailbox); diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.h b/drivers/infiniband/hw/hns/hns_roce_cmd.h index f5a9ee2fc53d..b1c94223c28b 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cmd.h +++ b/drivers/infiniband/hw/hns/hns_roce_cmd.h @@ -37,6 +37,60 @@ #define HNS_ROCE_CMD_TIMEOUT_MSECS 10000 enum { + /* QPC BT commands */ + HNS_ROCE_CMD_WRITE_QPC_BT0 = 0x0, + HNS_ROCE_CMD_WRITE_QPC_BT1 = 0x1, + HNS_ROCE_CMD_WRITE_QPC_BT2 = 0x2, + HNS_ROCE_CMD_READ_QPC_BT0 = 0x4, + HNS_ROCE_CMD_READ_QPC_BT1 = 0x5, + HNS_ROCE_CMD_READ_QPC_BT2 = 0x6, + HNS_ROCE_CMD_DESTROY_QPC_BT0 = 0x8, + HNS_ROCE_CMD_DESTROY_QPC_BT1 = 0x9, + HNS_ROCE_CMD_DESTROY_QPC_BT2 = 0xa, + + /* QPC operation */ + HNS_ROCE_CMD_MODIFY_QPC = 0x41, + HNS_ROCE_CMD_QUERY_QPC = 0x42, + + HNS_ROCE_CMD_MODIFY_CQC = 0x52, + /* CQC BT commands */ + HNS_ROCE_CMD_WRITE_CQC_BT0 = 0x10, + HNS_ROCE_CMD_WRITE_CQC_BT1 = 0x11, + HNS_ROCE_CMD_WRITE_CQC_BT2 = 0x12, + HNS_ROCE_CMD_READ_CQC_BT0 = 0x14, + HNS_ROCE_CMD_READ_CQC_BT1 = 0x15, + HNS_ROCE_CMD_READ_CQC_BT2 = 0x1b, + HNS_ROCE_CMD_DESTROY_CQC_BT0 = 0x18, + HNS_ROCE_CMD_DESTROY_CQC_BT1 = 0x19, + HNS_ROCE_CMD_DESTROY_CQC_BT2 = 0x1a, + + /* MPT BT commands */ + HNS_ROCE_CMD_WRITE_MPT_BT0 = 0x20, + HNS_ROCE_CMD_WRITE_MPT_BT1 = 0x21, + HNS_ROCE_CMD_WRITE_MPT_BT2 = 0x22, + HNS_ROCE_CMD_READ_MPT_BT0 = 0x24, + HNS_ROCE_CMD_READ_MPT_BT1 = 0x25, + HNS_ROCE_CMD_READ_MPT_BT2 = 0x26, + HNS_ROCE_CMD_DESTROY_MPT_BT0 = 0x28, + HNS_ROCE_CMD_DESTROY_MPT_BT1 = 0x29, + HNS_ROCE_CMD_DESTROY_MPT_BT2 = 0x2a, + + /* MPT commands */ + HNS_ROCE_CMD_QUERY_MPT = 0x62, + + /* SRQC BT commands */ + HNS_ROCE_CMD_WRITE_SRQC_BT0 = 0x30, + HNS_ROCE_CMD_WRITE_SRQC_BT1 = 0x31, + HNS_ROCE_CMD_WRITE_SRQC_BT2 = 0x32, + HNS_ROCE_CMD_READ_SRQC_BT0 = 0x34, + HNS_ROCE_CMD_READ_SRQC_BT1 = 0x35, + HNS_ROCE_CMD_READ_SRQC_BT2 = 0x36, + HNS_ROCE_CMD_DESTROY_SRQC_BT0 = 0x38, + HNS_ROCE_CMD_DESTROY_SRQC_BT1 = 0x39, + HNS_ROCE_CMD_DESTROY_SRQC_BT2 = 0x3a, +}; + +enum { /* TPT commands */ HNS_ROCE_CMD_SW2HW_MPT = 0xd, HNS_ROCE_CMD_HW2SW_MPT = 0xf, diff --git a/drivers/infiniband/hw/hns/hns_roce_common.h b/drivers/infiniband/hw/hns/hns_roce_common.h index 4af403e1348c..7ecb7a4147a8 100644 --- a/drivers/infiniband/hw/hns/hns_roce_common.h +++ b/drivers/infiniband/hw/hns/hns_roce_common.h @@ -341,6 +341,7 @@ #define ROCEE_BT_CMD_L_REG 0x200 #define ROCEE_MB1_REG 0x210 +#define ROCEE_MB6_REG 0x224 #define ROCEE_DB_SQ_L_0_REG 0x230 #define ROCEE_DB_OTHERS_L_0_REG 0x238 #define ROCEE_QP1C_CFG0_0_REG 0x270 @@ -362,4 +363,26 @@ #define ROCEE_ECC_UCERR_ALM0_REG 0xB34 #define ROCEE_ECC_CERR_ALM0_REG 0xB40 +/* V2 ROCEE REG */ +#define ROCEE_TX_CMQ_BASEADDR_L_REG 0x07000 +#define ROCEE_TX_CMQ_BASEADDR_H_REG 0x07004 +#define ROCEE_TX_CMQ_DEPTH_REG 0x07008 +#define ROCEE_TX_CMQ_TAIL_REG 0x07010 +#define ROCEE_TX_CMQ_HEAD_REG 0x07014 + +#define ROCEE_RX_CMQ_BASEADDR_L_REG 0x07018 +#define ROCEE_RX_CMQ_BASEADDR_H_REG 0x0701c +#define ROCEE_RX_CMQ_DEPTH_REG 0x07020 +#define ROCEE_RX_CMQ_TAIL_REG 0x07024 +#define ROCEE_RX_CMQ_HEAD_REG 0x07028 + +#define ROCEE_VF_SMAC_CFG0_REG 0x12000 +#define ROCEE_VF_SMAC_CFG1_REG 0x12004 + +#define ROCEE_VF_SGID_CFG0_REG 0x10000 +#define ROCEE_VF_SGID_CFG1_REG 0x10004 +#define ROCEE_VF_SGID_CFG2_REG 0x10008 +#define ROCEE_VF_SGID_CFG3_REG 0x1000c +#define ROCEE_VF_SGID_CFG4_REG 0x10010 + #endif /* _HNS_ROCE_COMMON_H */ diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index b89fd711019e..2111b57a3489 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -58,7 +58,7 @@ static void hns_roce_ib_cq_event(struct hns_roce_cq *hr_cq, if (event_type != HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID && event_type != HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR && event_type != HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW) { - dev_err(&hr_dev->pdev->dev, + dev_err(hr_dev->dev, "hns_roce_ib: Unexpected event type 0x%x on CQ %06lx\n", event_type, hr_cq->cqn); return; @@ -85,17 +85,23 @@ static int hns_roce_cq_alloc(struct hns_roce_dev *hr_dev, int nent, struct hns_roce_uar *hr_uar, struct hns_roce_cq *hr_cq, int vector) { - struct hns_roce_cmd_mailbox *mailbox = NULL; - struct hns_roce_cq_table *cq_table = NULL; - struct device *dev = &hr_dev->pdev->dev; + struct hns_roce_cmd_mailbox *mailbox; + struct hns_roce_hem_table *mtt_table; + struct hns_roce_cq_table *cq_table; + struct device *dev = hr_dev->dev; dma_addr_t dma_handle; - u64 *mtts = NULL; - int ret = 0; + u64 *mtts; + int ret; cq_table = &hr_dev->cq_table; /* Get the physical address of cq buf */ - mtts = hns_roce_table_find(&hr_dev->mr_table.mtt_table, + if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) + mtt_table = &hr_dev->mr_table.mtt_cqe_table; + else + mtt_table = &hr_dev->mr_table.mtt_table; + + mtts = hns_roce_table_find(hr_dev, mtt_table, hr_mtt->first_seg, &dma_handle); if (!mtts) { dev_err(dev, "CQ alloc.Failed to find cq buf addr.\n"); @@ -150,6 +156,7 @@ static int hns_roce_cq_alloc(struct hns_roce_dev *hr_dev, int nent, } hr_cq->cons_index = 0; + hr_cq->arm_sn = 1; hr_cq->uar = hr_uar; atomic_set(&hr_cq->refcount, 1); @@ -182,21 +189,22 @@ static int hns_roce_hw2sw_cq(struct hns_roce_dev *dev, void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) { struct hns_roce_cq_table *cq_table = &hr_dev->cq_table; - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; int ret; ret = hns_roce_hw2sw_cq(hr_dev, NULL, hr_cq->cqn); if (ret) dev_err(dev, "HW2SW_CQ failed (%d) for CQN %06lx\n", ret, hr_cq->cqn); - - /* Waiting interrupt process procedure carried out */ - synchronize_irq(hr_dev->eq_table.eq[hr_cq->vector].irq); - - /* wait for all interrupt processed */ - if (atomic_dec_and_test(&hr_cq->refcount)) - complete(&hr_cq->free); - wait_for_completion(&hr_cq->free); + if (hr_dev->eq_table.eq) { + /* Waiting interrupt process procedure carried out */ + synchronize_irq(hr_dev->eq_table.eq[hr_cq->vector].irq); + + /* wait for all interrupt processed */ + if (atomic_dec_and_test(&hr_cq->refcount)) + complete(&hr_cq->free); + wait_for_completion(&hr_cq->free); + } spin_lock_irq(&cq_table->lock); radix_tree_delete(&cq_table->tree, hr_cq->cqn); @@ -205,6 +213,7 @@ void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn); hns_roce_bitmap_free(&cq_table->bitmap, hr_cq->cqn, BITMAP_NO_RR); } +EXPORT_SYMBOL_GPL(hns_roce_free_cq); static int hns_roce_ib_get_cq_umem(struct hns_roce_dev *hr_dev, struct ib_ucontext *context, @@ -212,14 +221,31 @@ static int hns_roce_ib_get_cq_umem(struct hns_roce_dev *hr_dev, struct ib_umem **umem, u64 buf_addr, int cqe) { int ret; + u32 page_shift; + u32 npages; *umem = ib_umem_get(context, buf_addr, cqe * hr_dev->caps.cq_entry_sz, IB_ACCESS_LOCAL_WRITE, 1); if (IS_ERR(*umem)) return PTR_ERR(*umem); - ret = hns_roce_mtt_init(hr_dev, ib_umem_page_count(*umem), - (*umem)->page_shift, &buf->hr_mtt); + if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) + buf->hr_mtt.mtt_type = MTT_TYPE_CQE; + else + buf->hr_mtt.mtt_type = MTT_TYPE_WQE; + + if (hr_dev->caps.cqe_buf_pg_sz) { + npages = (ib_umem_page_count(*umem) + + (1 << hr_dev->caps.cqe_buf_pg_sz) - 1) / + (1 << hr_dev->caps.cqe_buf_pg_sz); + page_shift = PAGE_SHIFT + hr_dev->caps.cqe_buf_pg_sz; + ret = hns_roce_mtt_init(hr_dev, npages, page_shift, + &buf->hr_mtt); + } else { + ret = hns_roce_mtt_init(hr_dev, ib_umem_page_count(*umem), + (*umem)->page_shift, + &buf->hr_mtt); + } if (ret) goto err_buf; @@ -241,12 +267,19 @@ static int hns_roce_ib_alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq_buf *buf, u32 nent) { int ret; + u32 page_shift = PAGE_SHIFT + hr_dev->caps.cqe_buf_pg_sz; ret = hns_roce_buf_alloc(hr_dev, nent * hr_dev->caps.cq_entry_sz, - PAGE_SIZE * 2, &buf->hr_buf); + (1 << page_shift) * 2, &buf->hr_buf, + page_shift); if (ret) goto out; + if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) + buf->hr_mtt.mtt_type = MTT_TYPE_CQE; + else + buf->hr_mtt.mtt_type = MTT_TYPE_WQE; + ret = hns_roce_mtt_init(hr_dev, buf->hr_buf.npages, buf->hr_buf.page_shift, &buf->hr_mtt); if (ret) @@ -281,13 +314,13 @@ struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev, struct ib_udata *udata) { struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; struct hns_roce_ib_create_cq ucmd; struct hns_roce_cq *hr_cq = NULL; struct hns_roce_uar *uar = NULL; int vector = attr->comp_vector; int cq_entries = attr->cqe; - int ret = 0; + int ret; if (cq_entries < 1 || cq_entries > hr_dev->caps.max_cqes) { dev_err(dev, "Creat CQ failed. entries=%d, max=%d\n", @@ -295,13 +328,12 @@ struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev, return ERR_PTR(-EINVAL); } - hr_cq = kmalloc(sizeof(*hr_cq), GFP_KERNEL); + hr_cq = kzalloc(sizeof(*hr_cq), GFP_KERNEL); if (!hr_cq) return ERR_PTR(-ENOMEM); - /* In v1 engine, parameter verification */ - if (cq_entries < HNS_ROCE_MIN_CQE_NUM) - cq_entries = HNS_ROCE_MIN_CQE_NUM; + if (hr_dev->caps.min_cqes) + cq_entries = max(cq_entries, hr_dev->caps.min_cqes); cq_entries = roundup_pow_of_two((unsigned int)cq_entries); hr_cq->ib_cq.cqe = cq_entries - 1; @@ -335,8 +367,8 @@ struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev, } uar = &hr_dev->priv_uar; - hr_cq->cq_db_l = hr_dev->reg_base + ROCEE_DB_OTHERS_L_0_REG + - 0x1000 * uar->index; + hr_cq->cq_db_l = hr_dev->reg_base + hr_dev->odb_offset + + DB_REG_OFFSET * uar->index; } /* Allocate cq index, fill cq_context */ @@ -353,7 +385,7 @@ struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev, * problems if tptr is set to zero here, so we initialze it in user * space. */ - if (!context) + if (!context && hr_cq->tptr_addr) *hr_cq->tptr_addr = 0; /* Get created cq handler and carry out event */ @@ -385,6 +417,7 @@ err_cq: kfree(hr_cq); return ERR_PTR(ret); } +EXPORT_SYMBOL_GPL(hns_roce_ib_create_cq); int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq) { @@ -410,10 +443,11 @@ int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq) return ret; } +EXPORT_SYMBOL_GPL(hns_roce_ib_destroy_cq); void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn) { - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; struct hns_roce_cq *cq; cq = radix_tree_lookup(&hr_dev->cq_table.tree, @@ -423,13 +457,14 @@ void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn) return; } + ++cq->arm_sn; cq->comp(cq); } void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type) { struct hns_roce_cq_table *cq_table = &hr_dev->cq_table; - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; struct hns_roce_cq *cq; cq = radix_tree_lookup(&cq_table->tree, diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index e493a61e14e1..01d3d695cbba 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -78,6 +78,8 @@ #define HNS_ROCE_MAX_GID_NUM 16 #define HNS_ROCE_GID_SIZE 16 +#define HNS_ROCE_HOP_NUM_0 0xff + #define BITMAP_NO_RR 0 #define BITMAP_RR 1 @@ -168,6 +170,16 @@ enum { HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE = 0x07, }; +enum { + HNS_ROCE_CAP_FLAG_REREG_MR = BIT(0), + HNS_ROCE_CAP_FLAG_ROCE_V1_V2 = BIT(1), +}; + +enum hns_roce_mtt_type { + MTT_TYPE_WQE, + MTT_TYPE_CQE, +}; + #define HNS_ROCE_CMD_SUCCESS 1 #define HNS_ROCE_PORT_DOWN 0 @@ -229,15 +241,21 @@ struct hns_roce_hem_table { unsigned long num_obj; /*Single obj size */ unsigned long obj_size; + unsigned long table_chunk_size; int lowmem; struct mutex mutex; struct hns_roce_hem **hem; + u64 **bt_l1; + dma_addr_t *bt_l1_dma_addr; + u64 **bt_l0; + dma_addr_t *bt_l0_dma_addr; }; struct hns_roce_mtt { - unsigned long first_seg; - int order; - int page_shift; + unsigned long first_seg; + int order; + int page_shift; + enum hns_roce_mtt_type mtt_type; }; /* Only support 4K page size for mr register */ @@ -255,6 +273,19 @@ struct hns_roce_mr { int type; /* MR's register type */ u64 *pbl_buf;/* MR's PBL space */ dma_addr_t pbl_dma_addr; /* MR's PBL space PA */ + u32 pbl_size;/* PA number in the PBL */ + u64 pbl_ba;/* page table address */ + u32 l0_chunk_last_num;/* L0 last number */ + u32 l1_chunk_last_num;/* L1 last number */ + u64 **pbl_bt_l2;/* PBL BT L2 */ + u64 **pbl_bt_l1;/* PBL BT L1 */ + u64 *pbl_bt_l0;/* PBL BT L0 */ + dma_addr_t *pbl_l2_dma_addr;/* PBL BT L2 dma addr */ + dma_addr_t *pbl_l1_dma_addr;/* PBL BT L1 dma addr */ + dma_addr_t pbl_l0_dma_addr;/* PBL BT L0 dma addr */ + u32 pbl_ba_pg_sz;/* BT chunk page size */ + u32 pbl_buf_pg_sz;/* buf chunk page size */ + u32 pbl_hop_num;/* multi-hop number */ }; struct hns_roce_mr_table { @@ -262,6 +293,8 @@ struct hns_roce_mr_table { struct hns_roce_buddy mtt_buddy; struct hns_roce_hem_table mtt_table; struct hns_roce_hem_table mtpt_table; + struct hns_roce_buddy mtt_cqe_buddy; + struct hns_roce_hem_table mtt_cqe_table; }; struct hns_roce_wq { @@ -277,6 +310,12 @@ struct hns_roce_wq { void __iomem *db_reg_l; }; +struct hns_roce_sge { + int sge_cnt; /* SGE num */ + int offset; + int sge_shift;/* SGE size */ +}; + struct hns_roce_buf_list { void *buf; dma_addr_t map; @@ -308,6 +347,7 @@ struct hns_roce_cq { u32 cons_index; void __iomem *cq_db_l; u16 *tptr_addr; + int arm_sn; unsigned long cqn; u32 vector; atomic_t refcount; @@ -328,6 +368,7 @@ struct hns_roce_qp_table { spinlock_t lock; struct hns_roce_hem_table qp_table; struct hns_roce_hem_table irrl_table; + struct hns_roce_hem_table trrl_table; }; struct hns_roce_cq_table { @@ -367,7 +408,6 @@ struct hns_roce_cmd_context { struct hns_roce_cmdq { struct dma_pool *pool; - u8 __iomem *hcr; struct mutex hcr_mutex; struct semaphore poll_sem; /* @@ -429,6 +469,9 @@ struct hns_roce_qp { atomic_t refcount; struct completion free; + + struct hns_roce_sge sge; + u32 next_sge; }; struct hns_roce_sqp { @@ -439,7 +482,6 @@ struct hns_roce_ib_iboe { spinlock_t lock; struct net_device *netdevs[HNS_ROCE_MAX_PORTS]; struct notifier_block nb; - struct notifier_block nb_inet; u8 phy_port[HNS_ROCE_MAX_PORTS]; }; @@ -477,16 +519,20 @@ struct hns_roce_caps { u32 max_wqes; /* 16k */ u32 max_sq_desc_sz; /* 64 */ u32 max_rq_desc_sz; /* 64 */ + u32 max_srq_desc_sz; int max_qp_init_rdma; int max_qp_dest_rdma; int num_cqs; int max_cqes; + int min_cqes; + u32 min_wqes; int reserved_cqs; int num_aeq_vectors; /* 1 */ int num_comp_vectors; /* 32 ceq */ int num_other_vectors; int num_mtpts; u32 num_mtt_segs; + u32 num_cqe_segs; int reserved_mrws; int reserved_uars; int num_pds; @@ -498,29 +544,70 @@ struct hns_roce_caps { int mtpt_entry_sz; int qpc_entry_sz; int irrl_entry_sz; + int trrl_entry_sz; int cqc_entry_sz; + u32 pbl_ba_pg_sz; + u32 pbl_buf_pg_sz; + u32 pbl_hop_num; int aeqe_depth; int ceqe_depth[HNS_ROCE_COMP_VEC_NUM]; enum ib_mtu max_mtu; + u32 qpc_bt_num; + u32 srqc_bt_num; + u32 cqc_bt_num; + u32 mpt_bt_num; + u32 qpc_ba_pg_sz; + u32 qpc_buf_pg_sz; + u32 qpc_hop_num; + u32 srqc_ba_pg_sz; + u32 srqc_buf_pg_sz; + u32 srqc_hop_num; + u32 cqc_ba_pg_sz; + u32 cqc_buf_pg_sz; + u32 cqc_hop_num; + u32 mpt_ba_pg_sz; + u32 mpt_buf_pg_sz; + u32 mpt_hop_num; + u32 mtt_ba_pg_sz; + u32 mtt_buf_pg_sz; + u32 mtt_hop_num; + u32 cqe_ba_pg_sz; + u32 cqe_buf_pg_sz; + u32 cqe_hop_num; + u32 chunk_sz; /* chunk size in non multihop mode*/ + u64 flags; }; struct hns_roce_hw { int (*reset)(struct hns_roce_dev *hr_dev, bool enable); - void (*hw_profile)(struct hns_roce_dev *hr_dev); + int (*cmq_init)(struct hns_roce_dev *hr_dev); + void (*cmq_exit)(struct hns_roce_dev *hr_dev); + int (*hw_profile)(struct hns_roce_dev *hr_dev); int (*hw_init)(struct hns_roce_dev *hr_dev); void (*hw_exit)(struct hns_roce_dev *hr_dev); - void (*set_gid)(struct hns_roce_dev *hr_dev, u8 port, int gid_index, - union ib_gid *gid); - void (*set_mac)(struct hns_roce_dev *hr_dev, u8 phy_port, u8 *addr); + int (*post_mbox)(struct hns_roce_dev *hr_dev, u64 in_param, + u64 out_param, u32 in_modifier, u8 op_modifier, u16 op, + u16 token, int event); + int (*chk_mbox)(struct hns_roce_dev *hr_dev, unsigned long timeout); + int (*set_gid)(struct hns_roce_dev *hr_dev, u8 port, int gid_index, + union ib_gid *gid, const struct ib_gid_attr *attr); + int (*set_mac)(struct hns_roce_dev *hr_dev, u8 phy_port, u8 *addr); void (*set_mtu)(struct hns_roce_dev *hr_dev, u8 phy_port, enum ib_mtu mtu); int (*write_mtpt)(void *mb_buf, struct hns_roce_mr *mr, unsigned long mtpt_idx); + int (*rereg_write_mtpt)(struct hns_roce_dev *hr_dev, + struct hns_roce_mr *mr, int flags, u32 pdn, + int mr_access_flags, u64 iova, u64 size, + void *mb_buf); void (*write_cqc)(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, void *mb_buf, u64 *mtts, dma_addr_t dma_handle, int nent, u32 vector); + int (*set_hem)(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, int obj, int step_idx); int (*clear_hem)(struct hns_roce_dev *hr_dev, - struct hns_roce_hem_table *table, int obj); + struct hns_roce_hem_table *table, int obj, + int step_idx); int (*query_qp)(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); int (*modify_qp)(struct ib_qp *ibqp, const struct ib_qp_attr *attr, @@ -535,12 +622,14 @@ struct hns_roce_hw { int (*poll_cq)(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); int (*dereg_mr)(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr); int (*destroy_cq)(struct ib_cq *ibcq); - void *priv; + int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period); }; struct hns_roce_dev { struct ib_device ib_dev; struct platform_device *pdev; + struct pci_dev *pci_dev; + struct device *dev; struct hns_roce_uar priv_uar; const char *irq_names[HNS_ROCE_MAX_IRQ_NUM]; spinlock_t sm_lock; @@ -569,9 +658,12 @@ struct hns_roce_dev { int cmd_mod; int loop_idc; + u32 sdb_offset; + u32 odb_offset; dma_addr_t tptr_dma_addr; /*only for hw v1*/ u32 tptr_size; /*only for hw v1*/ - struct hns_roce_hw *hw; + const struct hns_roce_hw *hw; + void *priv; }; static inline struct hns_roce_dev *to_hr_dev(struct ib_device *ib_dev) @@ -635,12 +727,14 @@ static inline struct hns_roce_qp static inline void *hns_roce_buf_offset(struct hns_roce_buf *buf, int offset) { u32 bits_per_long_val = BITS_PER_LONG; + u32 page_size = 1 << buf->page_shift; - if (bits_per_long_val == 64 || buf->nbufs == 1) + if ((bits_per_long_val == 64 && buf->page_shift == PAGE_SHIFT) || + buf->nbufs == 1) return (char *)(buf->direct.buf) + offset; else - return (char *)(buf->page_list[offset >> PAGE_SHIFT].buf) + - (offset & (PAGE_SIZE - 1)); + return (char *)(buf->page_list[offset >> buf->page_shift].buf) + + (offset & (page_size - 1)); } int hns_roce_init_uar_table(struct hns_roce_dev *dev); @@ -702,6 +796,9 @@ struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc); struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, u64 virt_addr, int access_flags, struct ib_udata *udata); +int hns_roce_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, u64 length, + u64 virt_addr, int mr_access_flags, struct ib_pd *pd, + struct ib_udata *udata); int hns_roce_dereg_mr(struct ib_mr *ibmr); int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev, struct hns_roce_cmd_mailbox *mailbox, @@ -711,7 +808,7 @@ unsigned long key_to_hw_index(u32 key); void hns_roce_buf_free(struct hns_roce_dev *hr_dev, u32 size, struct hns_roce_buf *buf); int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct, - struct hns_roce_buf *buf); + struct hns_roce_buf *buf, u32 page_shift); int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt, struct ib_umem *umem); @@ -723,6 +820,7 @@ int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata); void *get_recv_wqe(struct hns_roce_qp *hr_qp, int n); void *get_send_wqe(struct hns_roce_qp *hr_qp, int n); +void *get_send_extend_sge(struct hns_roce_qp *hr_qp, int n); bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq, struct ib_cq *ib_cq); enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state); @@ -749,7 +847,7 @@ void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn); void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type); void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type); int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index); - -extern struct hns_roce_hw hns_roce_hw_v1; +int hns_roce_init(struct hns_roce_dev *hr_dev); +void hns_roce_exit(struct hns_roce_dev *hr_dev); #endif /* _HNS_ROCE_DEVICE_H */ diff --git a/drivers/infiniband/hw/hns/hns_roce_eq.c b/drivers/infiniband/hw/hns/hns_roce_eq.c index b0f43735de1a..d184431e2bf5 100644 --- a/drivers/infiniband/hw/hns/hns_roce_eq.c +++ b/drivers/infiniband/hw/hns/hns_roce_eq.c @@ -558,7 +558,7 @@ static int hns_roce_create_eq(struct hns_roce_dev *hr_dev, writel(eqshift_val, eqc); /* Configure eq extended address 12~44bit */ - writel((u32)(eq->buf_list[0].map >> 12), (u8 *)eqc + 4); + writel((u32)(eq->buf_list[0].map >> 12), eqc + 4); /* * Configure eq extended address 45~49 bit. @@ -572,13 +572,13 @@ static int hns_roce_create_eq(struct hns_roce_dev *hr_dev, roce_set_field(eqcuridx_val, ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_M, ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S, 0); - writel(eqcuridx_val, (u8 *)eqc + 8); + writel(eqcuridx_val, eqc + 8); /* Configure eq consumer index */ roce_set_field(eqconsindx_val, ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_M, ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S, 0); - writel(eqconsindx_val, (u8 *)eqc + 0xc); + writel(eqconsindx_val, eqc + 0xc); return 0; diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index c5104e0b2916..8b733a66fae5 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -36,14 +36,165 @@ #include "hns_roce_hem.h" #include "hns_roce_common.h" -#define HNS_ROCE_HEM_ALLOC_SIZE (1 << 17) -#define HNS_ROCE_TABLE_CHUNK_SIZE (1 << 17) - #define DMA_ADDR_T_SHIFT 12 #define BT_BA_SHIFT 32 -struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, int npages, - gfp_t gfp_mask) +bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type) +{ + if ((hr_dev->caps.qpc_hop_num && type == HEM_TYPE_QPC) || + (hr_dev->caps.mpt_hop_num && type == HEM_TYPE_MTPT) || + (hr_dev->caps.cqc_hop_num && type == HEM_TYPE_CQC) || + (hr_dev->caps.srqc_hop_num && type == HEM_TYPE_SRQC) || + (hr_dev->caps.cqe_hop_num && type == HEM_TYPE_CQE) || + (hr_dev->caps.mtt_hop_num && type == HEM_TYPE_MTT)) + return true; + + return false; +} +EXPORT_SYMBOL_GPL(hns_roce_check_whether_mhop); + +static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 start_idx, + u32 bt_chunk_num) +{ + int i; + + for (i = 0; i < bt_chunk_num; i++) + if (hem[start_idx + i]) + return false; + + return true; +} + +static bool hns_roce_check_bt_null(u64 **bt, u64 start_idx, u32 bt_chunk_num) +{ + int i; + + for (i = 0; i < bt_chunk_num; i++) + if (bt[start_idx + i]) + return false; + + return true; +} + +static int hns_roce_get_bt_num(u32 table_type, u32 hop_num) +{ + if (check_whether_bt_num_3(table_type, hop_num)) + return 3; + else if (check_whether_bt_num_2(table_type, hop_num)) + return 2; + else if (check_whether_bt_num_1(table_type, hop_num)) + return 1; + else + return 0; +} + +int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, unsigned long *obj, + struct hns_roce_hem_mhop *mhop) +{ + struct device *dev = hr_dev->dev; + u32 chunk_ba_num; + u32 table_idx; + u32 bt_num; + u32 chunk_size; + + switch (table->type) { + case HEM_TYPE_QPC: + mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = hr_dev->caps.qpc_bt_num; + mhop->hop_num = hr_dev->caps.qpc_hop_num; + break; + case HEM_TYPE_MTPT: + mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = hr_dev->caps.mpt_bt_num; + mhop->hop_num = hr_dev->caps.mpt_hop_num; + break; + case HEM_TYPE_CQC: + mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = hr_dev->caps.cqc_bt_num; + mhop->hop_num = hr_dev->caps.cqc_hop_num; + break; + case HEM_TYPE_SRQC: + mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = hr_dev->caps.srqc_bt_num; + mhop->hop_num = hr_dev->caps.srqc_hop_num; + break; + case HEM_TYPE_MTT: + mhop->buf_chunk_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = mhop->bt_chunk_size / 8; + mhop->hop_num = hr_dev->caps.mtt_hop_num; + break; + case HEM_TYPE_CQE: + mhop->buf_chunk_size = 1 << (hr_dev->caps.cqe_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = mhop->bt_chunk_size / 8; + mhop->hop_num = hr_dev->caps.cqe_hop_num; + break; + default: + dev_err(dev, "Table %d not support multi-hop addressing!\n", + table->type); + return -EINVAL; + } + + if (!obj) + return 0; + + /* + * QPC/MTPT/CQC/SRQC alloc hem for buffer pages. + * MTT/CQE alloc hem for bt pages. + */ + bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); + chunk_ba_num = mhop->bt_chunk_size / 8; + chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : + mhop->bt_chunk_size; + table_idx = (*obj & (table->num_obj - 1)) / + (chunk_size / table->obj_size); + switch (bt_num) { + case 3: + mhop->l2_idx = table_idx & (chunk_ba_num - 1); + mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1); + mhop->l0_idx = table_idx / chunk_ba_num / chunk_ba_num; + break; + case 2: + mhop->l1_idx = table_idx & (chunk_ba_num - 1); + mhop->l0_idx = table_idx / chunk_ba_num; + break; + case 1: + mhop->l0_idx = table_idx; + break; + default: + dev_err(dev, "Table %d not support hop_num = %d!\n", + table->type, mhop->hop_num); + return -EINVAL; + } + if (mhop->l0_idx >= mhop->ba_l0_num) + mhop->l0_idx %= mhop->ba_l0_num; + + return 0; +} +EXPORT_SYMBOL_GPL(hns_roce_calc_hem_mhop); + +static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, + int npages, + unsigned long hem_alloc_size, + gfp_t gfp_mask) { struct hns_roce_hem_chunk *chunk = NULL; struct hns_roce_hem *hem; @@ -61,7 +212,7 @@ struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, int npages, hem->refcount = 0; INIT_LIST_HEAD(&hem->chunk_list); - order = get_order(HNS_ROCE_HEM_ALLOC_SIZE); + order = get_order(hem_alloc_size); while (npages > 0) { if (!chunk) { @@ -84,7 +235,7 @@ struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, int npages, * memory, directly return fail. */ mem = &chunk->mem[chunk->npages]; - buf = dma_alloc_coherent(&hr_dev->pdev->dev, PAGE_SIZE << order, + buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order, &sg_dma_address(mem), gfp_mask); if (!buf) goto fail; @@ -115,7 +266,7 @@ void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem) list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) { for (i = 0; i < chunk->npages; ++i) - dma_free_coherent(&hr_dev->pdev->dev, + dma_free_coherent(hr_dev->dev, chunk->mem[i].length, lowmem_page_address(sg_page(&chunk->mem[i])), sg_dma_address(&chunk->mem[i])); @@ -128,8 +279,8 @@ void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem) static int hns_roce_set_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table, unsigned long obj) { - struct device *dev = &hr_dev->pdev->dev; spinlock_t *lock = &hr_dev->bt_cmd_lock; + struct device *dev = hr_dev->dev; unsigned long end = 0; unsigned long flags; struct hns_roce_hem_iter iter; @@ -142,7 +293,7 @@ static int hns_roce_set_hem(struct hns_roce_dev *hr_dev, /* Find the HEM(Hardware Entry Memory) entry */ unsigned long i = (obj & (table->num_obj - 1)) / - (HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size); + (table->table_chunk_size / table->obj_size); switch (table->type) { case HEM_TYPE_QPC: @@ -209,14 +360,185 @@ static int hns_roce_set_hem(struct hns_roce_dev *hr_dev, return ret; } +static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, + unsigned long obj) +{ + struct device *dev = hr_dev->dev; + struct hns_roce_hem_mhop mhop; + struct hns_roce_hem_iter iter; + u32 buf_chunk_size; + u32 bt_chunk_size; + u32 chunk_ba_num; + u32 hop_num; + u32 size; + u32 bt_num; + u64 hem_idx; + u64 bt_l1_idx = 0; + u64 bt_l0_idx = 0; + u64 bt_ba; + unsigned long mhop_obj = obj; + int bt_l1_allocated = 0; + int bt_l0_allocated = 0; + int step_idx; + int ret; + + ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); + if (ret) + return ret; + + buf_chunk_size = mhop.buf_chunk_size; + bt_chunk_size = mhop.bt_chunk_size; + hop_num = mhop.hop_num; + chunk_ba_num = bt_chunk_size / 8; + + bt_num = hns_roce_get_bt_num(table->type, hop_num); + switch (bt_num) { + case 3: + hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num + + mhop.l1_idx * chunk_ba_num + mhop.l2_idx; + bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; + bt_l0_idx = mhop.l0_idx; + break; + case 2: + hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; + bt_l0_idx = mhop.l0_idx; + break; + case 1: + hem_idx = mhop.l0_idx; + break; + default: + dev_err(dev, "Table %d not support hop_num = %d!\n", + table->type, hop_num); + return -EINVAL; + } + + mutex_lock(&table->mutex); + + if (table->hem[hem_idx]) { + ++table->hem[hem_idx]->refcount; + goto out; + } + + /* alloc L1 BA's chunk */ + if ((check_whether_bt_num_3(table->type, hop_num) || + check_whether_bt_num_2(table->type, hop_num)) && + !table->bt_l0[bt_l0_idx]) { + table->bt_l0[bt_l0_idx] = dma_alloc_coherent(dev, bt_chunk_size, + &(table->bt_l0_dma_addr[bt_l0_idx]), + GFP_KERNEL); + if (!table->bt_l0[bt_l0_idx]) { + ret = -ENOMEM; + goto out; + } + bt_l0_allocated = 1; + + /* set base address to hardware */ + if (table->type < HEM_TYPE_MTT) { + step_idx = 0; + if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) { + ret = -ENODEV; + dev_err(dev, "set HEM base address to HW failed!\n"); + goto err_dma_alloc_l1; + } + } + } + + /* alloc L2 BA's chunk */ + if (check_whether_bt_num_3(table->type, hop_num) && + !table->bt_l1[bt_l1_idx]) { + table->bt_l1[bt_l1_idx] = dma_alloc_coherent(dev, bt_chunk_size, + &(table->bt_l1_dma_addr[bt_l1_idx]), + GFP_KERNEL); + if (!table->bt_l1[bt_l1_idx]) { + ret = -ENOMEM; + goto err_dma_alloc_l1; + } + bt_l1_allocated = 1; + *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = + table->bt_l1_dma_addr[bt_l1_idx]; + + /* set base address to hardware */ + step_idx = 1; + if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) { + ret = -ENODEV; + dev_err(dev, "set HEM base address to HW failed!\n"); + goto err_alloc_hem_buf; + } + } + + /* + * alloc buffer space chunk for QPC/MTPT/CQC/SRQC. + * alloc bt space chunk for MTT/CQE. + */ + size = table->type < HEM_TYPE_MTT ? buf_chunk_size : bt_chunk_size; + table->hem[hem_idx] = hns_roce_alloc_hem(hr_dev, + size >> PAGE_SHIFT, + size, + (table->lowmem ? GFP_KERNEL : + GFP_HIGHUSER) | __GFP_NOWARN); + if (!table->hem[hem_idx]) { + ret = -ENOMEM; + goto err_alloc_hem_buf; + } + + hns_roce_hem_first(table->hem[hem_idx], &iter); + bt_ba = hns_roce_hem_addr(&iter); + + if (table->type < HEM_TYPE_MTT) { + if (hop_num == 2) { + *(table->bt_l1[bt_l1_idx] + mhop.l2_idx) = bt_ba; + step_idx = 2; + } else if (hop_num == 1) { + *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba; + step_idx = 1; + } else if (hop_num == HNS_ROCE_HOP_NUM_0) { + step_idx = 0; + } + + /* set HEM base address to hardware */ + if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) { + ret = -ENODEV; + dev_err(dev, "set HEM base address to HW failed!\n"); + goto err_alloc_hem_buf; + } + } else if (hop_num == 2) { + *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba; + } + + ++table->hem[hem_idx]->refcount; + goto out; + +err_alloc_hem_buf: + if (bt_l1_allocated) { + dma_free_coherent(dev, bt_chunk_size, table->bt_l1[bt_l1_idx], + table->bt_l1_dma_addr[bt_l1_idx]); + table->bt_l1[bt_l1_idx] = NULL; + } + +err_dma_alloc_l1: + if (bt_l0_allocated) { + dma_free_coherent(dev, bt_chunk_size, table->bt_l0[bt_l0_idx], + table->bt_l0_dma_addr[bt_l0_idx]); + table->bt_l0[bt_l0_idx] = NULL; + } + +out: + mutex_unlock(&table->mutex); + return ret; +} + int hns_roce_table_get(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table, unsigned long obj) { - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; int ret = 0; unsigned long i; - i = (obj & (table->num_obj - 1)) / (HNS_ROCE_TABLE_CHUNK_SIZE / + if (hns_roce_check_whether_mhop(hr_dev, table->type)) + return hns_roce_table_mhop_get(hr_dev, table, obj); + + i = (obj & (table->num_obj - 1)) / (table->table_chunk_size / table->obj_size); mutex_lock(&table->mutex); @@ -227,7 +549,8 @@ int hns_roce_table_get(struct hns_roce_dev *hr_dev, } table->hem[i] = hns_roce_alloc_hem(hr_dev, - HNS_ROCE_TABLE_CHUNK_SIZE >> PAGE_SHIFT, + table->table_chunk_size >> PAGE_SHIFT, + table->table_chunk_size, (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | __GFP_NOWARN); if (!table->hem[i]) { @@ -237,6 +560,8 @@ int hns_roce_table_get(struct hns_roce_dev *hr_dev, /* Set HEM base address(128K/page, pa) to Hardware */ if (hns_roce_set_hem(hr_dev, table, obj)) { + hns_roce_free_hem(hr_dev, table->hem[i]); + table->hem[i] = NULL; ret = -ENODEV; dev_err(dev, "set HEM base address to HW failed.\n"); goto out; @@ -248,20 +573,139 @@ out: return ret; } +static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, + unsigned long obj, + int check_refcount) +{ + struct device *dev = hr_dev->dev; + struct hns_roce_hem_mhop mhop; + unsigned long mhop_obj = obj; + u32 bt_chunk_size; + u32 chunk_ba_num; + u32 hop_num; + u32 start_idx; + u32 bt_num; + u64 hem_idx; + u64 bt_l1_idx = 0; + int ret; + + ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); + if (ret) + return; + + bt_chunk_size = mhop.bt_chunk_size; + hop_num = mhop.hop_num; + chunk_ba_num = bt_chunk_size / 8; + + bt_num = hns_roce_get_bt_num(table->type, hop_num); + switch (bt_num) { + case 3: + hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num + + mhop.l1_idx * chunk_ba_num + mhop.l2_idx; + bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; + break; + case 2: + hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; + break; + case 1: + hem_idx = mhop.l0_idx; + break; + default: + dev_err(dev, "Table %d not support hop_num = %d!\n", + table->type, hop_num); + return; + } + + mutex_lock(&table->mutex); + + if (check_refcount && (--table->hem[hem_idx]->refcount > 0)) { + mutex_unlock(&table->mutex); + return; + } + + if (table->type < HEM_TYPE_MTT && hop_num == 1) { + if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) + dev_warn(dev, "Clear HEM base address failed.\n"); + } else if (table->type < HEM_TYPE_MTT && hop_num == 2) { + if (hr_dev->hw->clear_hem(hr_dev, table, obj, 2)) + dev_warn(dev, "Clear HEM base address failed.\n"); + } else if (table->type < HEM_TYPE_MTT && + hop_num == HNS_ROCE_HOP_NUM_0) { + if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) + dev_warn(dev, "Clear HEM base address failed.\n"); + } + + /* + * free buffer space chunk for QPC/MTPT/CQC/SRQC. + * free bt space chunk for MTT/CQE. + */ + hns_roce_free_hem(hr_dev, table->hem[hem_idx]); + table->hem[hem_idx] = NULL; + + if (check_whether_bt_num_2(table->type, hop_num)) { + start_idx = mhop.l0_idx * chunk_ba_num; + if (hns_roce_check_hem_null(table->hem, start_idx, + chunk_ba_num)) { + if (table->type < HEM_TYPE_MTT && + hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) + dev_warn(dev, "Clear HEM base address failed.\n"); + + dma_free_coherent(dev, bt_chunk_size, + table->bt_l0[mhop.l0_idx], + table->bt_l0_dma_addr[mhop.l0_idx]); + table->bt_l0[mhop.l0_idx] = NULL; + } + } else if (check_whether_bt_num_3(table->type, hop_num)) { + start_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num + + mhop.l1_idx * chunk_ba_num; + if (hns_roce_check_hem_null(table->hem, start_idx, + chunk_ba_num)) { + if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) + dev_warn(dev, "Clear HEM base address failed.\n"); + + dma_free_coherent(dev, bt_chunk_size, + table->bt_l1[bt_l1_idx], + table->bt_l1_dma_addr[bt_l1_idx]); + table->bt_l1[bt_l1_idx] = NULL; + + start_idx = mhop.l0_idx * chunk_ba_num; + if (hns_roce_check_bt_null(table->bt_l1, start_idx, + chunk_ba_num)) { + if (hr_dev->hw->clear_hem(hr_dev, table, obj, + 0)) + dev_warn(dev, "Clear HEM base address failed.\n"); + + dma_free_coherent(dev, bt_chunk_size, + table->bt_l0[mhop.l0_idx], + table->bt_l0_dma_addr[mhop.l0_idx]); + table->bt_l0[mhop.l0_idx] = NULL; + } + } + } + + mutex_unlock(&table->mutex); +} + void hns_roce_table_put(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table, unsigned long obj) { - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; unsigned long i; + if (hns_roce_check_whether_mhop(hr_dev, table->type)) { + hns_roce_table_mhop_put(hr_dev, table, obj, 1); + return; + } + i = (obj & (table->num_obj - 1)) / - (HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size); + (table->table_chunk_size / table->obj_size); mutex_lock(&table->mutex); if (--table->hem[i]->refcount == 0) { /* Clear HEM base address */ - if (hr_dev->hw->clear_hem(hr_dev, table, obj)) + if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) dev_warn(dev, "Clear HEM base address failed.\n"); hns_roce_free_hem(hr_dev, table->hem[i]); @@ -271,23 +715,48 @@ void hns_roce_table_put(struct hns_roce_dev *hr_dev, mutex_unlock(&table->mutex); } -void *hns_roce_table_find(struct hns_roce_hem_table *table, unsigned long obj, - dma_addr_t *dma_handle) +void *hns_roce_table_find(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, + unsigned long obj, dma_addr_t *dma_handle) { struct hns_roce_hem_chunk *chunk; - unsigned long idx; - int i; - int offset, dma_offset; + struct hns_roce_hem_mhop mhop; struct hns_roce_hem *hem; struct page *page = NULL; + unsigned long mhop_obj = obj; + unsigned long obj_per_chunk; + unsigned long idx_offset; + int offset, dma_offset; + int i, j; + u32 hem_idx = 0; if (!table->lowmem) return NULL; mutex_lock(&table->mutex); - idx = (obj & (table->num_obj - 1)) * table->obj_size; - hem = table->hem[idx / HNS_ROCE_TABLE_CHUNK_SIZE]; - dma_offset = offset = idx % HNS_ROCE_TABLE_CHUNK_SIZE; + + if (!hns_roce_check_whether_mhop(hr_dev, table->type)) { + obj_per_chunk = table->table_chunk_size / table->obj_size; + hem = table->hem[(obj & (table->num_obj - 1)) / obj_per_chunk]; + idx_offset = (obj & (table->num_obj - 1)) % obj_per_chunk; + dma_offset = offset = idx_offset * table->obj_size; + } else { + hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); + /* mtt mhop */ + i = mhop.l0_idx; + j = mhop.l1_idx; + if (mhop.hop_num == 2) + hem_idx = i * (mhop.bt_chunk_size / 8) + j; + else if (mhop.hop_num == 1 || + mhop.hop_num == HNS_ROCE_HOP_NUM_0) + hem_idx = i; + + hem = table->hem[hem_idx]; + dma_offset = offset = (obj & (table->num_obj - 1)) * + table->obj_size % mhop.bt_chunk_size; + if (mhop.hop_num == 2) + dma_offset = offset = 0; + } if (!hem) goto out; @@ -314,14 +783,21 @@ out: mutex_unlock(&table->mutex); return page ? lowmem_page_address(page) + offset : NULL; } +EXPORT_SYMBOL_GPL(hns_roce_table_find); int hns_roce_table_get_range(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table, unsigned long start, unsigned long end) { - unsigned long inc = HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size; - unsigned long i = 0; - int ret = 0; + struct hns_roce_hem_mhop mhop; + unsigned long inc = table->table_chunk_size / table->obj_size; + unsigned long i; + int ret; + + if (hns_roce_check_whether_mhop(hr_dev, table->type)) { + hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop); + inc = mhop.bt_chunk_size / table->obj_size; + } /* Allocate MTT entry memory according to chunk(128K) */ for (i = start; i <= end; i += inc) { @@ -344,10 +820,16 @@ void hns_roce_table_put_range(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table, unsigned long start, unsigned long end) { + struct hns_roce_hem_mhop mhop; + unsigned long inc = table->table_chunk_size / table->obj_size; unsigned long i; - for (i = start; i <= end; - i += HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size) + if (hns_roce_check_whether_mhop(hr_dev, table->type)) { + hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop); + inc = mhop.bt_chunk_size / table->obj_size; + } + + for (i = start; i <= end; i += inc) hns_roce_table_put(hr_dev, table, i); } @@ -356,15 +838,120 @@ int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, unsigned long obj_size, unsigned long nobj, int use_lowmem) { + struct device *dev = hr_dev->dev; unsigned long obj_per_chunk; unsigned long num_hem; - obj_per_chunk = HNS_ROCE_TABLE_CHUNK_SIZE / obj_size; - num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk; + if (!hns_roce_check_whether_mhop(hr_dev, type)) { + table->table_chunk_size = hr_dev->caps.chunk_sz; + obj_per_chunk = table->table_chunk_size / obj_size; + num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk; + + table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL); + if (!table->hem) + return -ENOMEM; + } else { + unsigned long buf_chunk_size; + unsigned long bt_chunk_size; + unsigned long bt_chunk_num; + unsigned long num_bt_l0 = 0; + u32 hop_num; + + switch (type) { + case HEM_TYPE_QPC: + buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz + + PAGE_SHIFT); + bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz + + PAGE_SHIFT); + num_bt_l0 = hr_dev->caps.qpc_bt_num; + hop_num = hr_dev->caps.qpc_hop_num; + break; + case HEM_TYPE_MTPT: + buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz + + PAGE_SHIFT); + bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz + + PAGE_SHIFT); + num_bt_l0 = hr_dev->caps.mpt_bt_num; + hop_num = hr_dev->caps.mpt_hop_num; + break; + case HEM_TYPE_CQC: + buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz + + PAGE_SHIFT); + bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz + + PAGE_SHIFT); + num_bt_l0 = hr_dev->caps.cqc_bt_num; + hop_num = hr_dev->caps.cqc_hop_num; + break; + case HEM_TYPE_SRQC: + buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz + + PAGE_SHIFT); + bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz + + PAGE_SHIFT); + num_bt_l0 = hr_dev->caps.srqc_bt_num; + hop_num = hr_dev->caps.srqc_hop_num; + break; + case HEM_TYPE_MTT: + buf_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + + PAGE_SHIFT); + bt_chunk_size = buf_chunk_size; + hop_num = hr_dev->caps.mtt_hop_num; + break; + case HEM_TYPE_CQE: + buf_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + + PAGE_SHIFT); + bt_chunk_size = buf_chunk_size; + hop_num = hr_dev->caps.cqe_hop_num; + break; + default: + dev_err(dev, + "Table %d not support to init hem table here!\n", + type); + return -EINVAL; + } + obj_per_chunk = buf_chunk_size / obj_size; + num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk; + bt_chunk_num = bt_chunk_size / 8; + if (table->type >= HEM_TYPE_MTT) + num_bt_l0 = bt_chunk_num; + + table->hem = kcalloc(num_hem, sizeof(*table->hem), + GFP_KERNEL); + if (!table->hem) + goto err_kcalloc_hem_buf; + + if (check_whether_bt_num_3(table->type, hop_num)) { + unsigned long num_bt_l1; + + num_bt_l1 = (num_hem + bt_chunk_num - 1) / + bt_chunk_num; + table->bt_l1 = kcalloc(num_bt_l1, + sizeof(*table->bt_l1), + GFP_KERNEL); + if (!table->bt_l1) + goto err_kcalloc_bt_l1; + + table->bt_l1_dma_addr = kcalloc(num_bt_l1, + sizeof(*table->bt_l1_dma_addr), + GFP_KERNEL); + + if (!table->bt_l1_dma_addr) + goto err_kcalloc_l1_dma; + } - table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL); - if (!table->hem) - return -ENOMEM; + if (check_whether_bt_num_2(table->type, hop_num) || + check_whether_bt_num_3(table->type, hop_num)) { + table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0), + GFP_KERNEL); + if (!table->bt_l0) + goto err_kcalloc_bt_l0; + + table->bt_l0_dma_addr = kcalloc(num_bt_l0, + sizeof(*table->bt_l0_dma_addr), + GFP_KERNEL); + if (!table->bt_l0_dma_addr) + goto err_kcalloc_l0_dma; + } + } table->type = type; table->num_hem = num_hem; @@ -374,18 +961,72 @@ int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, mutex_init(&table->mutex); return 0; + +err_kcalloc_l0_dma: + kfree(table->bt_l0); + table->bt_l0 = NULL; + +err_kcalloc_bt_l0: + kfree(table->bt_l1_dma_addr); + table->bt_l1_dma_addr = NULL; + +err_kcalloc_l1_dma: + kfree(table->bt_l1); + table->bt_l1 = NULL; + +err_kcalloc_bt_l1: + kfree(table->hem); + table->hem = NULL; + +err_kcalloc_hem_buf: + return -ENOMEM; +} + +static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table) +{ + struct hns_roce_hem_mhop mhop; + u32 buf_chunk_size; + int i; + u64 obj; + + hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop); + buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size : + mhop.bt_chunk_size; + + for (i = 0; i < table->num_hem; ++i) { + obj = i * buf_chunk_size / table->obj_size; + if (table->hem[i]) + hns_roce_table_mhop_put(hr_dev, table, obj, 0); + } + + kfree(table->hem); + table->hem = NULL; + kfree(table->bt_l1); + table->bt_l1 = NULL; + kfree(table->bt_l1_dma_addr); + table->bt_l1_dma_addr = NULL; + kfree(table->bt_l0); + table->bt_l0 = NULL; + kfree(table->bt_l0_dma_addr); + table->bt_l0_dma_addr = NULL; } void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table) { - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; unsigned long i; + if (hns_roce_check_whether_mhop(hr_dev, table->type)) { + hns_roce_cleanup_mhop_hem_table(hr_dev, table); + return; + } + for (i = 0; i < table->num_hem; ++i) if (table->hem[i]) { if (hr_dev->hw->clear_hem(hr_dev, table, - i * HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size)) + i * table->table_chunk_size / table->obj_size, 0)) dev_err(dev, "Clear HEM base address failed.\n"); hns_roce_free_hem(hr_dev, table->hem[i]); @@ -398,7 +1039,13 @@ void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev) { hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table); hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table); + if (hr_dev->caps.trrl_entry_sz) + hns_roce_cleanup_hem_table(hr_dev, + &hr_dev->qp_table.trrl_table); hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table); hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table); hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table); + if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) + hns_roce_cleanup_hem_table(hr_dev, + &hr_dev->mr_table.mtt_cqe_table); } diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.h b/drivers/infiniband/hw/hns/hns_roce_hem.h index 435748858252..db66db12075e 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.h +++ b/drivers/infiniband/hw/hns/hns_roce_hem.h @@ -47,13 +47,27 @@ enum { /* UNMAP HEM */ HEM_TYPE_MTT, + HEM_TYPE_CQE, HEM_TYPE_IRRL, + HEM_TYPE_TRRL, }; #define HNS_ROCE_HEM_CHUNK_LEN \ ((256 - sizeof(struct list_head) - 2 * sizeof(int)) / \ (sizeof(struct scatterlist))) +#define check_whether_bt_num_3(type, hop_num) \ + (type < HEM_TYPE_MTT && hop_num == 2) + +#define check_whether_bt_num_2(type, hop_num) \ + ((type < HEM_TYPE_MTT && hop_num == 1) || \ + (type >= HEM_TYPE_MTT && hop_num == 2)) + +#define check_whether_bt_num_1(type, hop_num) \ + ((type < HEM_TYPE_MTT && hop_num == HNS_ROCE_HOP_NUM_0) || \ + (type >= HEM_TYPE_MTT && hop_num == 1) || \ + (type >= HEM_TYPE_MTT && hop_num == HNS_ROCE_HOP_NUM_0)) + enum { HNS_ROCE_HEM_PAGE_SHIFT = 12, HNS_ROCE_HEM_PAGE_SIZE = 1 << HNS_ROCE_HEM_PAGE_SHIFT, @@ -77,12 +91,23 @@ struct hns_roce_hem_iter { int page_idx; }; +struct hns_roce_hem_mhop { + u32 hop_num; + u32 buf_chunk_size; + u32 bt_chunk_size; + u32 ba_l0_num; + u32 l0_idx;/* level 0 base address table index */ + u32 l1_idx;/* level 1 base address table index */ + u32 l2_idx;/* level 2 base address table index */ +}; + void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem); int hns_roce_table_get(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table, unsigned long obj); void hns_roce_table_put(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table, unsigned long obj); -void *hns_roce_table_find(struct hns_roce_hem_table *table, unsigned long obj, +void *hns_roce_table_find(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, unsigned long obj, dma_addr_t *dma_handle); int hns_roce_table_get_range(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table, @@ -97,6 +122,10 @@ int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table); void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev); +int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, unsigned long *obj, + struct hns_roce_hem_mhop *mhop); +bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type); static inline void hns_roce_hem_first(struct hns_roce_hem *hem, struct hns_roce_hem_iter *iter) @@ -105,7 +134,7 @@ static inline void hns_roce_hem_first(struct hns_roce_hem *hem, iter->chunk = list_empty(&hem->chunk_list) ? NULL : list_entry(hem->chunk_list.next, struct hns_roce_hem_chunk, list); - iter->page_idx = 0; + iter->page_idx = 0; } static inline int hns_roce_hem_last(struct hns_roce_hem_iter *iter) diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c index 747efd1ae5a6..af27168faf0f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c @@ -34,6 +34,7 @@ #include <linux/acpi.h> #include <linux/etherdevice.h> #include <linux/of.h> +#include <linux/of_platform.h> #include <rdma/ib_umem.h> #include "hns_roce_common.h" #include "hns_roce_device.h" @@ -56,8 +57,8 @@ static void set_raddr_seg(struct hns_roce_wqe_raddr_seg *rseg, u64 remote_addr, rseg->len = 0; } -int hns_roce_v1_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, - struct ib_send_wr **bad_wr) +static int hns_roce_v1_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, + struct ib_send_wr **bad_wr) { struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah); @@ -316,8 +317,8 @@ out: return ret; } -int hns_roce_v1_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr, - struct ib_recv_wr **bad_wr) +static int hns_roce_v1_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr, + struct ib_recv_wr **bad_wr) { int ret = 0; int nreq = 0; @@ -472,7 +473,7 @@ static void hns_roce_set_sdb_ext(struct hns_roce_dev *hr_dev, u32 ext_sdb_alept, dma_addr_t sdb_dma_addr; u32 val; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; db = &priv->db_table; /* Configure extend SDB threshold */ @@ -511,7 +512,7 @@ static void hns_roce_set_odb_ext(struct hns_roce_dev *hr_dev, u32 ext_odb_alept, dma_addr_t odb_dma_addr; u32 val; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; db = &priv->db_table; /* Configure extend ODB threshold */ @@ -547,7 +548,7 @@ static int hns_roce_db_ext_init(struct hns_roce_dev *hr_dev, u32 sdb_ext_mod, dma_addr_t odb_dma_addr; int ret = 0; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; db = &priv->db_table; db->ext_db = kmalloc(sizeof(*db->ext_db), GFP_KERNEL); @@ -668,7 +669,7 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev) u8 port = 0; u8 sl; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; free_mr = &priv->free_mr; /* Reserved cq for loop qp */ @@ -816,7 +817,7 @@ static void hns_roce_v1_release_lp_qp(struct hns_roce_dev *hr_dev) int ret; int i; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; free_mr = &priv->free_mr; for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) { @@ -850,7 +851,7 @@ static int hns_roce_db_init(struct hns_roce_dev *hr_dev) u32 odb_evt_mod; int ret = 0; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; db = &priv->db_table; memset(db, 0, sizeof(*db)); @@ -876,7 +877,7 @@ static int hns_roce_db_init(struct hns_roce_dev *hr_dev) return 0; } -void hns_roce_v1_recreate_lp_qp_work_fn(struct work_struct *work) +static void hns_roce_v1_recreate_lp_qp_work_fn(struct work_struct *work) { struct hns_roce_recreate_lp_qp_work *lp_qp_work; struct hns_roce_dev *hr_dev; @@ -906,11 +907,13 @@ static int hns_roce_v1_recreate_lp_qp(struct hns_roce_dev *hr_dev) unsigned long end = msecs_to_jiffies(HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS) + jiffies; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; free_mr = &priv->free_mr; lp_qp_work = kzalloc(sizeof(struct hns_roce_recreate_lp_qp_work), GFP_KERNEL); + if (!lp_qp_work) + return -ENOMEM; INIT_WORK(&(lp_qp_work->work), hns_roce_v1_recreate_lp_qp_work_fn); @@ -982,7 +985,7 @@ static void hns_roce_v1_mr_free_work_fn(struct work_struct *work) hr_dev = to_hr_dev(mr_work->ib_dev); dev = &hr_dev->pdev->dev; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; free_mr = &priv->free_mr; mr_free_cq = free_mr->mr_free_cq; @@ -1001,6 +1004,11 @@ static void hns_roce_v1_mr_free_work_fn(struct work_struct *work) } } + if (!ne) { + dev_err(dev, "Reserved loop qp is absent!\n"); + goto free_work; + } + do { ret = hns_roce_v1_poll_cq(&mr_free_cq->ib_cq, ne, wc); if (ret < 0) { @@ -1025,7 +1033,8 @@ free_work: kfree(mr_work); } -int hns_roce_v1_dereg_mr(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr) +static int hns_roce_v1_dereg_mr(struct hns_roce_dev *hr_dev, + struct hns_roce_mr *mr) { struct device *dev = &hr_dev->pdev->dev; struct hns_roce_mr_free_work *mr_work; @@ -1038,7 +1047,7 @@ int hns_roce_v1_dereg_mr(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr) int npages; int ret = 0; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; free_mr = &priv->free_mr; if (mr->enabled) { @@ -1103,7 +1112,7 @@ static void hns_roce_db_free(struct hns_roce_dev *hr_dev) struct hns_roce_v1_priv *priv; struct hns_roce_db_table *db; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; db = &priv->db_table; if (db->sdb_ext_mod) { @@ -1133,7 +1142,7 @@ static int hns_roce_raq_init(struct hns_roce_dev *hr_dev) struct hns_roce_raq_table *raq; struct device *dev = &hr_dev->pdev->dev; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; raq = &priv->raq_table; raq->e_raq_buf = kzalloc(sizeof(*(raq->e_raq_buf)), GFP_KERNEL); @@ -1210,7 +1219,7 @@ static void hns_roce_raq_free(struct hns_roce_dev *hr_dev) struct hns_roce_v1_priv *priv; struct hns_roce_raq_table *raq; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; raq = &priv->raq_table; dma_free_coherent(dev, HNS_ROCE_V1_RAQ_SIZE, raq->e_raq_buf->buf, @@ -1244,7 +1253,7 @@ static int hns_roce_bt_init(struct hns_roce_dev *hr_dev) struct hns_roce_v1_priv *priv; int ret; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; priv->bt_table.qpc_buf.buf = dma_alloc_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.qpc_buf.map, @@ -1286,7 +1295,7 @@ static void hns_roce_bt_free(struct hns_roce_dev *hr_dev) struct device *dev = &hr_dev->pdev->dev; struct hns_roce_v1_priv *priv; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE, priv->bt_table.cqc_buf.buf, priv->bt_table.cqc_buf.map); @@ -1304,7 +1313,7 @@ static int hns_roce_tptr_init(struct hns_roce_dev *hr_dev) struct hns_roce_buf_list *tptr_buf; struct hns_roce_v1_priv *priv; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; tptr_buf = &priv->tptr_table.tptr_buf; /* @@ -1330,7 +1339,7 @@ static void hns_roce_tptr_free(struct hns_roce_dev *hr_dev) struct hns_roce_buf_list *tptr_buf; struct hns_roce_v1_priv *priv; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; tptr_buf = &priv->tptr_table.tptr_buf; dma_free_coherent(dev, HNS_ROCE_V1_TPTR_BUF_SIZE, @@ -1344,7 +1353,7 @@ static int hns_roce_free_mr_init(struct hns_roce_dev *hr_dev) struct hns_roce_v1_priv *priv; int ret = 0; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; free_mr = &priv->free_mr; free_mr->free_mr_wq = create_singlethread_workqueue("hns_roce_free_mr"); @@ -1368,7 +1377,7 @@ static void hns_roce_free_mr_free(struct hns_roce_dev *hr_dev) struct hns_roce_free_mr *free_mr; struct hns_roce_v1_priv *priv; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; free_mr = &priv->free_mr; flush_workqueue(free_mr->free_mr_wq); @@ -1383,7 +1392,7 @@ static void hns_roce_free_mr_free(struct hns_roce_dev *hr_dev) * @enable: true -- drop reset, false -- reset * return 0 - success , negative --fail */ -int hns_roce_v1_reset(struct hns_roce_dev *hr_dev, bool dereset) +static int hns_roce_v1_reset(struct hns_roce_dev *hr_dev, bool dereset) { struct device_node *dsaf_node; struct device *dev = &hr_dev->pdev->dev; @@ -1432,7 +1441,7 @@ static int hns_roce_des_qp_init(struct hns_roce_dev *hr_dev) struct hns_roce_v1_priv *priv; struct hns_roce_des_qp *des_qp; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; des_qp = &priv->des_qp; des_qp->requeue_flag = 1; @@ -1450,7 +1459,7 @@ static void hns_roce_des_qp_free(struct hns_roce_dev *hr_dev) struct hns_roce_v1_priv *priv; struct hns_roce_des_qp *des_qp; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; des_qp = &priv->des_qp; des_qp->requeue_flag = 0; @@ -1458,7 +1467,7 @@ static void hns_roce_des_qp_free(struct hns_roce_dev *hr_dev) destroy_workqueue(des_qp->qp_wq); } -void hns_roce_v1_profile(struct hns_roce_dev *hr_dev) +static int hns_roce_v1_profile(struct hns_roce_dev *hr_dev) { int i = 0; struct hns_roce_caps *caps = &hr_dev->caps; @@ -1474,7 +1483,9 @@ void hns_roce_v1_profile(struct hns_roce_dev *hr_dev) caps->num_qps = HNS_ROCE_V1_MAX_QP_NUM; caps->max_wqes = HNS_ROCE_V1_MAX_WQE_NUM; + caps->min_wqes = HNS_ROCE_MIN_WQE_NUM; caps->num_cqs = HNS_ROCE_V1_MAX_CQ_NUM; + caps->min_cqes = HNS_ROCE_MIN_CQE_NUM; caps->max_cqes = HNS_ROCE_V1_MAX_CQE_NUM; caps->max_sq_sg = HNS_ROCE_V1_SG_NUM; caps->max_rq_sg = HNS_ROCE_V1_SG_NUM; @@ -1503,6 +1514,7 @@ void hns_roce_v1_profile(struct hns_roce_dev *hr_dev) caps->reserved_mrws = 1; caps->reserved_uars = 0; caps->reserved_cqs = 0; + caps->chunk_sz = HNS_ROCE_V1_TABLE_CHUNK_SIZE; for (i = 0; i < caps->num_ports; i++) caps->pkey_table_len[i] = 1; @@ -1524,9 +1536,11 @@ void hns_roce_v1_profile(struct hns_roce_dev *hr_dev) caps->local_ca_ack_delay = le32_to_cpu(roce_read(hr_dev, ROCEE_ACK_DELAY_REG)); caps->max_mtu = IB_MTU_2048; + + return 0; } -int hns_roce_v1_init(struct hns_roce_dev *hr_dev) +static int hns_roce_v1_init(struct hns_roce_dev *hr_dev) { int ret; u32 val; @@ -1605,7 +1619,7 @@ error_failed_raq_init: return ret; } -void hns_roce_v1_exit(struct hns_roce_dev *hr_dev) +static void hns_roce_v1_exit(struct hns_roce_dev *hr_dev) { hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_DOWN); hns_roce_free_mr_free(hr_dev); @@ -1616,8 +1630,82 @@ void hns_roce_v1_exit(struct hns_roce_dev *hr_dev) hns_roce_db_free(hr_dev); } -void hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u8 port, int gid_index, - union ib_gid *gid) +static int hns_roce_v1_cmd_pending(struct hns_roce_dev *hr_dev) +{ + u32 status = readl(hr_dev->reg_base + ROCEE_MB6_REG); + + return (!!(status & (1 << HCR_GO_BIT))); +} + +static int hns_roce_v1_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param, + u64 out_param, u32 in_modifier, u8 op_modifier, + u16 op, u16 token, int event) +{ + u32 __iomem *hcr = (u32 __iomem *)(hr_dev->reg_base + ROCEE_MB1_REG); + unsigned long end; + u32 val = 0; + + end = msecs_to_jiffies(GO_BIT_TIMEOUT_MSECS) + jiffies; + while (hns_roce_v1_cmd_pending(hr_dev)) { + if (time_after(jiffies, end)) { + dev_err(hr_dev->dev, "jiffies=%d end=%d\n", + (int)jiffies, (int)end); + return -EAGAIN; + } + cond_resched(); + } + + roce_set_field(val, ROCEE_MB6_ROCEE_MB_CMD_M, ROCEE_MB6_ROCEE_MB_CMD_S, + op); + roce_set_field(val, ROCEE_MB6_ROCEE_MB_CMD_MDF_M, + ROCEE_MB6_ROCEE_MB_CMD_MDF_S, op_modifier); + roce_set_bit(val, ROCEE_MB6_ROCEE_MB_EVENT_S, event); + roce_set_bit(val, ROCEE_MB6_ROCEE_MB_HW_RUN_S, 1); + roce_set_field(val, ROCEE_MB6_ROCEE_MB_TOKEN_M, + ROCEE_MB6_ROCEE_MB_TOKEN_S, token); + + __raw_writeq(cpu_to_le64(in_param), hcr + 0); + __raw_writeq(cpu_to_le64(out_param), hcr + 2); + __raw_writel(cpu_to_le32(in_modifier), hcr + 4); + /* Memory barrier */ + wmb(); + + __raw_writel(cpu_to_le32(val), hcr + 5); + + mmiowb(); + + return 0; +} + +static int hns_roce_v1_chk_mbox(struct hns_roce_dev *hr_dev, + unsigned long timeout) +{ + u8 __iomem *hcr = hr_dev->reg_base + ROCEE_MB1_REG; + unsigned long end = 0; + u32 status = 0; + + end = msecs_to_jiffies(timeout) + jiffies; + while (hns_roce_v1_cmd_pending(hr_dev) && time_before(jiffies, end)) + cond_resched(); + + if (hns_roce_v1_cmd_pending(hr_dev)) { + dev_err(hr_dev->dev, "[cmd_poll]hw run cmd TIMEDOUT!\n"); + return -ETIMEDOUT; + } + + status = le32_to_cpu((__force __be32) + __raw_readl(hcr + HCR_STATUS_OFFSET)); + if ((status & STATUS_MASK) != 0x1) { + dev_err(hr_dev->dev, "mailbox status 0x%x!\n", status); + return -EBUSY; + } + + return 0; +} + +static int hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u8 port, + int gid_index, union ib_gid *gid, + const struct ib_gid_attr *attr) { u32 *p = NULL; u8 gid_idx = 0; @@ -1639,9 +1727,12 @@ void hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u8 port, int gid_index, p = (u32 *)&gid->raw[0xc]; roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_H_0_REG + (HNS_ROCE_V1_GID_NUM * gid_idx)); + + return 0; } -void hns_roce_v1_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port, u8 *addr) +static int hns_roce_v1_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port, + u8 *addr) { u32 reg_smac_l; u16 reg_smac_h; @@ -1654,8 +1745,13 @@ void hns_roce_v1_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port, u8 *addr) * because of smac not equal to dmac. * We Need to release and create reserved qp again. */ - if (hr_dev->hw->dereg_mr && hns_roce_v1_recreate_lp_qp(hr_dev)) - dev_warn(&hr_dev->pdev->dev, "recreate lp qp timeout!\n"); + if (hr_dev->hw->dereg_mr) { + int ret; + + ret = hns_roce_v1_recreate_lp_qp(hr_dev); + if (ret && ret != -ETIMEDOUT) + return ret; + } p = (u32 *)(&addr[0]); reg_smac_l = *p; @@ -1670,10 +1766,12 @@ void hns_roce_v1_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port, u8 *addr) ROCEE_SMAC_H_ROCEE_SMAC_H_S, reg_smac_h); roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET, val); + + return 0; } -void hns_roce_v1_set_mtu(struct hns_roce_dev *hr_dev, u8 phy_port, - enum ib_mtu mtu) +static void hns_roce_v1_set_mtu(struct hns_roce_dev *hr_dev, u8 phy_port, + enum ib_mtu mtu) { u32 val; @@ -1685,8 +1783,8 @@ void hns_roce_v1_set_mtu(struct hns_roce_dev *hr_dev, u8 phy_port, val); } -int hns_roce_v1_write_mtpt(void *mb_buf, struct hns_roce_mr *mr, - unsigned long mtpt_idx) +static int hns_roce_v1_write_mtpt(void *mb_buf, struct hns_roce_mr *mr, + unsigned long mtpt_idx) { struct hns_roce_v1_mpt_entry *mpt_entry; struct scatterlist *sg; @@ -1858,7 +1956,7 @@ static struct hns_roce_cqe *next_cqe_sw(struct hns_roce_cq *hr_cq) return get_sw_cqe(hr_cq, hr_cq->cons_index); } -void hns_roce_v1_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index) +static void hns_roce_v1_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index) { u32 doorbell[2]; @@ -1931,9 +2029,10 @@ static void hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, spin_unlock_irq(&hr_cq->lock); } -void hns_roce_v1_write_cqc(struct hns_roce_dev *hr_dev, - struct hns_roce_cq *hr_cq, void *mb_buf, u64 *mtts, - dma_addr_t dma_handle, int nent, u32 vector) +static void hns_roce_v1_write_cqc(struct hns_roce_dev *hr_dev, + struct hns_roce_cq *hr_cq, void *mb_buf, + u64 *mtts, dma_addr_t dma_handle, int nent, + u32 vector) { struct hns_roce_cq_context *cq_context = NULL; struct hns_roce_buf_list *tptr_buf; @@ -1941,7 +2040,7 @@ void hns_roce_v1_write_cqc(struct hns_roce_dev *hr_dev, dma_addr_t tptr_dma_addr; int offset; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; tptr_buf = &priv->tptr_table.tptr_buf; cq_context = mb_buf; @@ -2018,7 +2117,13 @@ void hns_roce_v1_write_cqc(struct hns_roce_dev *hr_dev, cq_context->cqc_byte_32 = cpu_to_le32(cq_context->cqc_byte_32); } -int hns_roce_v1_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) +static int hns_roce_v1_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) +{ + return -EOPNOTSUPP; +} + +static int hns_roce_v1_req_notify_cq(struct ib_cq *ibcq, + enum ib_cq_notify_flags flags) { struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); u32 notification_flag; @@ -2279,8 +2384,9 @@ int hns_roce_v1_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) return ret; } -int hns_roce_v1_clear_hem(struct hns_roce_dev *hr_dev, - struct hns_roce_hem_table *table, int obj) +static int hns_roce_v1_clear_hem(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, int obj, + int step_idx) { struct device *dev = &hr_dev->pdev->dev; struct hns_roce_v1_priv *priv; @@ -2289,7 +2395,7 @@ int hns_roce_v1_clear_hem(struct hns_roce_dev *hr_dev, void __iomem *bt_cmd; u64 bt_ba = 0; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; switch (table->type) { case HEM_TYPE_QPC: @@ -2441,14 +2547,14 @@ static int hns_roce_v1_m_sqp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, int rq_pa_start; u32 reg_val; u64 *mtts; - u32 *addr; + u32 __iomem *addr; context = kzalloc(sizeof(*context), GFP_KERNEL); if (!context) return -ENOMEM; /* Search QP buf's MTTs */ - mtts = hns_roce_table_find(&hr_dev->mr_table.mtt_table, + mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table, hr_qp->mtt.first_seg, &dma_handle); if (!mtts) { dev_err(dev, "qp buf pa find failed\n"); @@ -2523,8 +2629,9 @@ static int hns_roce_v1_m_sqp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, QP1C_BYTES_40_SQ_CUR_IDX_S, 0); /* Copy context to QP1C register */ - addr = (u32 *)(hr_dev->reg_base + ROCEE_QP1C_CFG0_0_REG + - hr_qp->phy_port * sizeof(*context)); + addr = (u32 __iomem *)(hr_dev->reg_base + + ROCEE_QP1C_CFG0_0_REG + + hr_qp->phy_port * sizeof(*context)); writel(context->qp1c_bytes_4, addr); writel(context->sq_rq_bt_l, addr + 1); @@ -2595,7 +2702,7 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, return -ENOMEM; /* Search qp buf's mtts */ - mtts = hns_roce_table_find(&hr_dev->mr_table.mtt_table, + mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table, hr_qp->mtt.first_seg, &dma_handle); if (mtts == NULL) { dev_err(dev, "qp buf pa find failed\n"); @@ -2603,8 +2710,8 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, } /* Search IRRL's mtts */ - mtts_2 = hns_roce_table_find(&hr_dev->qp_table.irrl_table, hr_qp->qpn, - &dma_handle_2); + mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table, + hr_qp->qpn, &dma_handle_2); if (mtts_2 == NULL) { dev_err(dev, "qp irrl_table find failed\n"); goto out; @@ -2800,10 +2907,11 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S, ilog2((unsigned int)attr->max_dest_rd_atomic)); - roce_set_field(context->qpc_bytes_36, - QP_CONTEXT_QPC_BYTES_36_DEST_QP_M, - QP_CONTEXT_QPC_BYTES_36_DEST_QP_S, - attr->dest_qp_num); + if (attr_mask & IB_QP_DEST_QPN) + roce_set_field(context->qpc_bytes_36, + QP_CONTEXT_QPC_BYTES_36_DEST_QP_M, + QP_CONTEXT_QPC_BYTES_36_DEST_QP_S, + attr->dest_qp_num); /* Configure GID index */ port_num = rdma_ah_get_port_num(&attr->ah_attr); @@ -3143,7 +3251,7 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, if (ibqp->uobject) { hr_qp->rq.db_reg_l = hr_dev->reg_base + - ROCEE_DB_OTHERS_L_0_REG + + hr_dev->odb_offset + DB_REG_OFFSET * hr_dev->priv_uar.index; } @@ -3177,9 +3285,10 @@ out: return ret; } -int hns_roce_v1_modify_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, - int attr_mask, enum ib_qp_state cur_state, - enum ib_qp_state new_state) +static int hns_roce_v1_modify_qp(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, int attr_mask, + enum ib_qp_state cur_state, + enum ib_qp_state new_state) { if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI) @@ -3270,6 +3379,7 @@ static int hns_roce_v1_q_sqp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, qp_attr->path_mtu = IB_MTU_256; qp_attr->path_mig_state = IB_MIG_ARMED; qp_attr->qkey = QKEY_VAL; + qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; qp_attr->rq_psn = 0; qp_attr->sq_psn = 0; qp_attr->dest_qp_num = 1; @@ -3351,6 +3461,7 @@ static int hns_roce_v1_q_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, QP_CONTEXT_QPC_BYTES_48_MTU_M, QP_CONTEXT_QPC_BYTES_48_MTU_S); qp_attr->path_mig_state = IB_MIG_ARMED; + qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; if (hr_qp->ibqp.qp_type == IB_QPT_UD) qp_attr->qkey = QKEY_VAL; @@ -3406,10 +3517,10 @@ static int hns_roce_v1_q_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S); qp_attr->port_num = hr_qp->port + 1; qp_attr->sq_draining = 0; - qp_attr->max_rd_atomic = roce_get_field(context->qpc_bytes_156, + qp_attr->max_rd_atomic = 1 << roce_get_field(context->qpc_bytes_156, QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M, QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S); - qp_attr->max_dest_rd_atomic = roce_get_field(context->qpc_bytes_32, + qp_attr->max_dest_rd_atomic = 1 << roce_get_field(context->qpc_bytes_32, QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M, QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S); qp_attr->min_rnr_timer = (u8)(roce_get_field(context->qpc_bytes_24, @@ -3444,8 +3555,9 @@ out: return ret; } -int hns_roce_v1_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, - int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) +static int hns_roce_v1_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, + struct ib_qp_init_attr *qp_init_attr) { struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); @@ -3454,6 +3566,53 @@ int hns_roce_v1_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, hns_roce_v1_q_qp(ibqp, qp_attr, qp_attr_mask, qp_init_attr); } +static void hns_roce_check_sdb_status(struct hns_roce_dev *hr_dev, + u32 *old_send, u32 *old_retry, + u32 *tsp_st, u32 *success_flags) +{ + u32 sdb_retry_cnt; + u32 sdb_send_ptr; + u32 cur_cnt, old_cnt; + u32 send_ptr; + + sdb_send_ptr = roce_read(hr_dev, ROCEE_SDB_SEND_PTR_REG); + sdb_retry_cnt = roce_read(hr_dev, ROCEE_SDB_RETRY_CNT_REG); + cur_cnt = roce_get_field(sdb_send_ptr, + ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M, + ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) + + roce_get_field(sdb_retry_cnt, + ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M, + ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S); + if (!roce_get_bit(*tsp_st, ROCEE_CNT_CLR_CE_CNT_CLR_CE_S)) { + old_cnt = roce_get_field(*old_send, + ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M, + ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) + + roce_get_field(*old_retry, + ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M, + ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S); + if (cur_cnt - old_cnt > SDB_ST_CMP_VAL) + *success_flags = 1; + } else { + old_cnt = roce_get_field(*old_send, + ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M, + ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S); + if (cur_cnt - old_cnt > SDB_ST_CMP_VAL) { + *success_flags = 1; + } else { + send_ptr = roce_get_field(*old_send, + ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M, + ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) + + roce_get_field(sdb_retry_cnt, + ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M, + ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S); + roce_set_field(*old_send, + ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M, + ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S, + send_ptr); + } + } +} + static int check_qp_db_process_status(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, u32 sdb_issue_ptr, @@ -3461,12 +3620,10 @@ static int check_qp_db_process_status(struct hns_roce_dev *hr_dev, u32 *wait_stage) { struct device *dev = &hr_dev->pdev->dev; - u32 sdb_retry_cnt, old_retry; u32 sdb_send_ptr, old_send; u32 success_flags = 0; - u32 cur_cnt, old_cnt; unsigned long end; - u32 send_ptr; + u32 old_retry; u32 inv_cnt; u32 tsp_st; @@ -3524,47 +3681,9 @@ static int check_qp_db_process_status(struct hns_roce_dev *hr_dev, msleep(HNS_ROCE_V1_CHECK_DB_SLEEP_MSECS); - sdb_send_ptr = roce_read(hr_dev, - ROCEE_SDB_SEND_PTR_REG); - sdb_retry_cnt = roce_read(hr_dev, - ROCEE_SDB_RETRY_CNT_REG); - cur_cnt = roce_get_field(sdb_send_ptr, - ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M, - ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) + - roce_get_field(sdb_retry_cnt, - ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M, - ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S); - if (!roce_get_bit(tsp_st, - ROCEE_CNT_CLR_CE_CNT_CLR_CE_S)) { - old_cnt = roce_get_field(old_send, - ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M, - ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) + - roce_get_field(old_retry, - ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M, - ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S); - if (cur_cnt - old_cnt > SDB_ST_CMP_VAL) - success_flags = 1; - } else { - old_cnt = roce_get_field(old_send, - ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M, - ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S); - if (cur_cnt - old_cnt > - SDB_ST_CMP_VAL) { - success_flags = 1; - } else { - send_ptr = - roce_get_field(old_send, - ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M, - ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) + - roce_get_field(sdb_retry_cnt, - ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M, - ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S); - roce_set_field(old_send, - ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M, - ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S, - send_ptr); - } - } + hns_roce_check_sdb_status(hr_dev, &old_send, + &old_retry, &tsp_st, + &success_flags); } while (!success_flags); } @@ -3664,7 +3783,7 @@ static void hns_roce_v1_destroy_qp_work_fn(struct work_struct *work) qp_work_entry = container_of(work, struct hns_roce_qp_work, work); hr_dev = to_hr_dev(qp_work_entry->ib_dev); dev = &hr_dev->pdev->dev; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; hr_qp = qp_work_entry->qp; qpn = hr_qp->qpn; @@ -3781,7 +3900,7 @@ int hns_roce_v1_destroy_qp(struct ib_qp *ibqp) qp_work->sdb_inv_cnt = qp_work_entry.sdb_inv_cnt; qp_work->sche_cnt = qp_work_entry.sche_cnt; - priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; + priv = (struct hns_roce_v1_priv *)hr_dev->priv; queue_work(priv->des_qp.qp_wq, &qp_work->work); dev_dbg(dev, "Begin destroy QP(0x%lx) work.\n", hr_qp->qpn); } @@ -3789,7 +3908,7 @@ int hns_roce_v1_destroy_qp(struct ib_qp *ibqp) return 0; } -int hns_roce_v1_destroy_cq(struct ib_cq *ibcq) +static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq) { struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device); struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); @@ -3841,18 +3960,19 @@ int hns_roce_v1_destroy_cq(struct ib_cq *ibcq) return ret; } -struct hns_roce_v1_priv hr_v1_priv; - -struct hns_roce_hw hns_roce_hw_v1 = { +static const struct hns_roce_hw hns_roce_hw_v1 = { .reset = hns_roce_v1_reset, .hw_profile = hns_roce_v1_profile, .hw_init = hns_roce_v1_init, .hw_exit = hns_roce_v1_exit, + .post_mbox = hns_roce_v1_post_mbox, + .chk_mbox = hns_roce_v1_chk_mbox, .set_gid = hns_roce_v1_set_gid, .set_mac = hns_roce_v1_set_mac, .set_mtu = hns_roce_v1_set_mtu, .write_mtpt = hns_roce_v1_write_mtpt, .write_cqc = hns_roce_v1_write_cqc, + .modify_cq = hns_roce_v1_modify_cq, .clear_hem = hns_roce_v1_clear_hem, .modify_qp = hns_roce_v1_modify_qp, .query_qp = hns_roce_v1_query_qp, @@ -3863,5 +3983,258 @@ struct hns_roce_hw hns_roce_hw_v1 = { .poll_cq = hns_roce_v1_poll_cq, .dereg_mr = hns_roce_v1_dereg_mr, .destroy_cq = hns_roce_v1_destroy_cq, - .priv = &hr_v1_priv, }; + +static const struct of_device_id hns_roce_of_match[] = { + { .compatible = "hisilicon,hns-roce-v1", .data = &hns_roce_hw_v1, }, + {}, +}; +MODULE_DEVICE_TABLE(of, hns_roce_of_match); + +static const struct acpi_device_id hns_roce_acpi_match[] = { + { "HISI00D1", (kernel_ulong_t)&hns_roce_hw_v1 }, + {}, +}; +MODULE_DEVICE_TABLE(acpi, hns_roce_acpi_match); + +static int hns_roce_node_match(struct device *dev, void *fwnode) +{ + return dev->fwnode == fwnode; +} + +static struct +platform_device *hns_roce_find_pdev(struct fwnode_handle *fwnode) +{ + struct device *dev; + + /* get the 'device' corresponding to the matching 'fwnode' */ + dev = bus_find_device(&platform_bus_type, NULL, + fwnode, hns_roce_node_match); + /* get the platform device */ + return dev ? to_platform_device(dev) : NULL; +} + +static int hns_roce_get_cfg(struct hns_roce_dev *hr_dev) +{ + struct device *dev = &hr_dev->pdev->dev; + struct platform_device *pdev = NULL; + struct net_device *netdev = NULL; + struct device_node *net_node; + struct resource *res; + int port_cnt = 0; + u8 phy_port; + int ret; + int i; + + /* check if we are compatible with the underlying SoC */ + if (dev_of_node(dev)) { + const struct of_device_id *of_id; + + of_id = of_match_node(hns_roce_of_match, dev->of_node); + if (!of_id) { + dev_err(dev, "device is not compatible!\n"); + return -ENXIO; + } + hr_dev->hw = (const struct hns_roce_hw *)of_id->data; + if (!hr_dev->hw) { + dev_err(dev, "couldn't get H/W specific DT data!\n"); + return -ENXIO; + } + } else if (is_acpi_device_node(dev->fwnode)) { + const struct acpi_device_id *acpi_id; + + acpi_id = acpi_match_device(hns_roce_acpi_match, dev); + if (!acpi_id) { + dev_err(dev, "device is not compatible!\n"); + return -ENXIO; + } + hr_dev->hw = (const struct hns_roce_hw *) acpi_id->driver_data; + if (!hr_dev->hw) { + dev_err(dev, "couldn't get H/W specific ACPI data!\n"); + return -ENXIO; + } + } else { + dev_err(dev, "can't read compatibility data from DT or ACPI\n"); + return -ENXIO; + } + + /* get the mapped register base address */ + res = platform_get_resource(hr_dev->pdev, IORESOURCE_MEM, 0); + if (!res) { + dev_err(dev, "memory resource not found!\n"); + return -EINVAL; + } + hr_dev->reg_base = devm_ioremap_resource(dev, res); + if (IS_ERR(hr_dev->reg_base)) + return PTR_ERR(hr_dev->reg_base); + + /* read the node_guid of IB device from the DT or ACPI */ + ret = device_property_read_u8_array(dev, "node-guid", + (u8 *)&hr_dev->ib_dev.node_guid, + GUID_LEN); + if (ret) { + dev_err(dev, "couldn't get node_guid from DT or ACPI!\n"); + return ret; + } + + /* get the RoCE associated ethernet ports or netdevices */ + for (i = 0; i < HNS_ROCE_MAX_PORTS; i++) { + if (dev_of_node(dev)) { + net_node = of_parse_phandle(dev->of_node, "eth-handle", + i); + if (!net_node) + continue; + pdev = of_find_device_by_node(net_node); + } else if (is_acpi_device_node(dev->fwnode)) { + struct acpi_reference_args args; + struct fwnode_handle *fwnode; + + ret = acpi_node_get_property_reference(dev->fwnode, + "eth-handle", + i, &args); + if (ret) + continue; + fwnode = acpi_fwnode_handle(args.adev); + pdev = hns_roce_find_pdev(fwnode); + } else { + dev_err(dev, "cannot read data from DT or ACPI\n"); + return -ENXIO; + } + + if (pdev) { + netdev = platform_get_drvdata(pdev); + phy_port = (u8)i; + if (netdev) { + hr_dev->iboe.netdevs[port_cnt] = netdev; + hr_dev->iboe.phy_port[port_cnt] = phy_port; + } else { + dev_err(dev, "no netdev found with pdev %s\n", + pdev->name); + return -ENODEV; + } + port_cnt++; + } + } + + if (port_cnt == 0) { + dev_err(dev, "unable to get eth-handle for available ports!\n"); + return -EINVAL; + } + + hr_dev->caps.num_ports = port_cnt; + + /* cmd issue mode: 0 is poll, 1 is event */ + hr_dev->cmd_mod = 1; + hr_dev->loop_idc = 0; + hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG; + hr_dev->odb_offset = ROCEE_DB_OTHERS_L_0_REG; + + /* read the interrupt names from the DT or ACPI */ + ret = device_property_read_string_array(dev, "interrupt-names", + hr_dev->irq_names, + HNS_ROCE_MAX_IRQ_NUM); + if (ret < 0) { + dev_err(dev, "couldn't get interrupt names from DT or ACPI!\n"); + return ret; + } + + /* fetch the interrupt numbers */ + for (i = 0; i < HNS_ROCE_MAX_IRQ_NUM; i++) { + hr_dev->irq[i] = platform_get_irq(hr_dev->pdev, i); + if (hr_dev->irq[i] <= 0) { + dev_err(dev, "platform get of irq[=%d] failed!\n", i); + return -EINVAL; + } + } + + return 0; +} + +/** + * hns_roce_probe - RoCE driver entrance + * @pdev: pointer to platform device + * Return : int + * + */ +static int hns_roce_probe(struct platform_device *pdev) +{ + int ret; + struct hns_roce_dev *hr_dev; + struct device *dev = &pdev->dev; + + hr_dev = (struct hns_roce_dev *)ib_alloc_device(sizeof(*hr_dev)); + if (!hr_dev) + return -ENOMEM; + + hr_dev->priv = kzalloc(sizeof(struct hns_roce_v1_priv), GFP_KERNEL); + if (!hr_dev->priv) { + ret = -ENOMEM; + goto error_failed_kzalloc; + } + + hr_dev->pdev = pdev; + hr_dev->dev = dev; + platform_set_drvdata(pdev, hr_dev); + + if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64ULL)) && + dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32ULL))) { + dev_err(dev, "Not usable DMA addressing mode\n"); + ret = -EIO; + goto error_failed_get_cfg; + } + + ret = hns_roce_get_cfg(hr_dev); + if (ret) { + dev_err(dev, "Get Configuration failed!\n"); + goto error_failed_get_cfg; + } + + ret = hns_roce_init(hr_dev); + if (ret) { + dev_err(dev, "RoCE engine init failed!\n"); + goto error_failed_get_cfg; + } + + return 0; + +error_failed_get_cfg: + kfree(hr_dev->priv); + +error_failed_kzalloc: + ib_dealloc_device(&hr_dev->ib_dev); + + return ret; +} + +/** + * hns_roce_remove - remove RoCE device + * @pdev: pointer to platform device + */ +static int hns_roce_remove(struct platform_device *pdev) +{ + struct hns_roce_dev *hr_dev = platform_get_drvdata(pdev); + + hns_roce_exit(hr_dev); + kfree(hr_dev->priv); + ib_dealloc_device(&hr_dev->ib_dev); + + return 0; +} + +static struct platform_driver hns_roce_driver = { + .probe = hns_roce_probe, + .remove = hns_roce_remove, + .driver = { + .name = DRV_NAME, + .of_match_table = hns_roce_of_match, + .acpi_match_table = ACPI_PTR(hns_roce_acpi_match), + }, +}; + +module_platform_driver(hns_roce_driver); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>"); +MODULE_AUTHOR("Nenglong Zhao <zhaonenglong@hisilicon.com>"); +MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>"); +MODULE_DESCRIPTION("Hisilicon Hip06 Family RoCE Driver"); diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.h b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h index b213b5e6fef1..21a07ef0afc9 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h @@ -72,6 +72,8 @@ #define HNS_ROCE_V1_CQE_ENTRY_SIZE 32 #define HNS_ROCE_V1_PAGE_SIZE_SUPPORT 0xFFFFF000 +#define HNS_ROCE_V1_TABLE_CHUNK_SIZE (1 << 17) + #define HNS_ROCE_V1_EXT_RAQ_WF 8 #define HNS_ROCE_V1_RAQ_ENTRY 64 #define HNS_ROCE_V1_RAQ_DEPTH 32768 @@ -948,6 +950,11 @@ struct hns_roce_qp_context { #define QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_M \ (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S) +#define STATUS_MASK 0xff +#define GO_BIT_TIMEOUT_MSECS 10000 +#define HCR_STATUS_OFFSET 0x18 +#define HCR_GO_BIT 15 + struct hns_roce_rq_db { u32 u32_4; u32 u32_8; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c new file mode 100644 index 000000000000..8f719c00467b --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -0,0 +1,3296 @@ +/* + * Copyright (c) 2016-2017 Hisilicon Limited. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/acpi.h> +#include <linux/etherdevice.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <rdma/ib_umem.h> + +#include "hnae3.h" +#include "hns_roce_common.h" +#include "hns_roce_device.h" +#include "hns_roce_cmd.h" +#include "hns_roce_hem.h" +#include "hns_roce_hw_v2.h" + +static void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg, + struct ib_sge *sg) +{ + dseg->lkey = cpu_to_le32(sg->lkey); + dseg->addr = cpu_to_le64(sg->addr); + dseg->len = cpu_to_le32(sg->length); +} + +static int hns_roce_v2_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, + struct ib_send_wr **bad_wr) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_v2_rc_send_wqe *rc_sq_wqe; + struct hns_roce_qp *qp = to_hr_qp(ibqp); + struct hns_roce_v2_wqe_data_seg *dseg; + struct device *dev = hr_dev->dev; + struct hns_roce_v2_db sq_db; + unsigned int sge_ind = 0; + unsigned int wqe_sz = 0; + unsigned int owner_bit; + unsigned long flags; + unsigned int ind; + void *wqe = NULL; + int ret = 0; + int nreq; + int i; + + if (unlikely(ibqp->qp_type != IB_QPT_RC)) { + dev_err(dev, "Not supported QP(0x%x)type!\n", ibqp->qp_type); + *bad_wr = NULL; + return -EOPNOTSUPP; + } + + if (unlikely(qp->state != IB_QPS_RTS && qp->state != IB_QPS_SQD)) { + dev_err(dev, "Post WQE fail, QP state %d err!\n", qp->state); + *bad_wr = wr; + return -EINVAL; + } + + spin_lock_irqsave(&qp->sq.lock, flags); + ind = qp->sq_next_wqe; + sge_ind = qp->next_sge; + + for (nreq = 0; wr; ++nreq, wr = wr->next) { + if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { + ret = -ENOMEM; + *bad_wr = wr; + goto out; + } + + if (unlikely(wr->num_sge > qp->sq.max_gs)) { + dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n", + wr->num_sge, qp->sq.max_gs); + ret = -EINVAL; + *bad_wr = wr; + goto out; + } + + wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1)); + qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] = + wr->wr_id; + + owner_bit = ~(qp->sq.head >> ilog2(qp->sq.wqe_cnt)) & 0x1; + rc_sq_wqe = wqe; + memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe)); + for (i = 0; i < wr->num_sge; i++) + rc_sq_wqe->msg_len += wr->sg_list[i].length; + + rc_sq_wqe->inv_key_immtdata = send_ieth(wr); + + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_FENCE_S, + (wr->send_flags & IB_SEND_FENCE) ? 1 : 0); + + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_SE_S, + (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0); + + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_CQE_S, + (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0); + + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OWNER_S, + owner_bit); + + switch (wr->opcode) { + case IB_WR_RDMA_READ: + roce_set_field(rc_sq_wqe->byte_4, + V2_RC_SEND_WQE_BYTE_4_OPCODE_M, + V2_RC_SEND_WQE_BYTE_4_OPCODE_S, + HNS_ROCE_V2_WQE_OP_RDMA_READ); + rc_sq_wqe->rkey = cpu_to_le32(rdma_wr(wr)->rkey); + rc_sq_wqe->va = cpu_to_le64(rdma_wr(wr)->remote_addr); + break; + case IB_WR_RDMA_WRITE: + roce_set_field(rc_sq_wqe->byte_4, + V2_RC_SEND_WQE_BYTE_4_OPCODE_M, + V2_RC_SEND_WQE_BYTE_4_OPCODE_S, + HNS_ROCE_V2_WQE_OP_RDMA_WRITE); + rc_sq_wqe->rkey = cpu_to_le32(rdma_wr(wr)->rkey); + rc_sq_wqe->va = cpu_to_le64(rdma_wr(wr)->remote_addr); + break; + case IB_WR_RDMA_WRITE_WITH_IMM: + roce_set_field(rc_sq_wqe->byte_4, + V2_RC_SEND_WQE_BYTE_4_OPCODE_M, + V2_RC_SEND_WQE_BYTE_4_OPCODE_S, + HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM); + rc_sq_wqe->rkey = cpu_to_le32(rdma_wr(wr)->rkey); + rc_sq_wqe->va = cpu_to_le64(rdma_wr(wr)->remote_addr); + break; + case IB_WR_SEND: + roce_set_field(rc_sq_wqe->byte_4, + V2_RC_SEND_WQE_BYTE_4_OPCODE_M, + V2_RC_SEND_WQE_BYTE_4_OPCODE_S, + HNS_ROCE_V2_WQE_OP_SEND); + break; + case IB_WR_SEND_WITH_INV: + roce_set_field(rc_sq_wqe->byte_4, + V2_RC_SEND_WQE_BYTE_4_OPCODE_M, + V2_RC_SEND_WQE_BYTE_4_OPCODE_S, + HNS_ROCE_V2_WQE_OP_SEND_WITH_INV); + break; + case IB_WR_SEND_WITH_IMM: + roce_set_field(rc_sq_wqe->byte_4, + V2_RC_SEND_WQE_BYTE_4_OPCODE_M, + V2_RC_SEND_WQE_BYTE_4_OPCODE_S, + HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM); + break; + case IB_WR_LOCAL_INV: + roce_set_field(rc_sq_wqe->byte_4, + V2_RC_SEND_WQE_BYTE_4_OPCODE_M, + V2_RC_SEND_WQE_BYTE_4_OPCODE_S, + HNS_ROCE_V2_WQE_OP_LOCAL_INV); + break; + case IB_WR_ATOMIC_CMP_AND_SWP: + roce_set_field(rc_sq_wqe->byte_4, + V2_RC_SEND_WQE_BYTE_4_OPCODE_M, + V2_RC_SEND_WQE_BYTE_4_OPCODE_S, + HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP); + break; + case IB_WR_ATOMIC_FETCH_AND_ADD: + roce_set_field(rc_sq_wqe->byte_4, + V2_RC_SEND_WQE_BYTE_4_OPCODE_M, + V2_RC_SEND_WQE_BYTE_4_OPCODE_S, + HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD); + break; + case IB_WR_MASKED_ATOMIC_CMP_AND_SWP: + roce_set_field(rc_sq_wqe->byte_4, + V2_RC_SEND_WQE_BYTE_4_OPCODE_M, + V2_RC_SEND_WQE_BYTE_4_OPCODE_S, + HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP); + break; + case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD: + roce_set_field(rc_sq_wqe->byte_4, + V2_RC_SEND_WQE_BYTE_4_OPCODE_M, + V2_RC_SEND_WQE_BYTE_4_OPCODE_S, + HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD); + break; + default: + roce_set_field(rc_sq_wqe->byte_4, + V2_RC_SEND_WQE_BYTE_4_OPCODE_M, + V2_RC_SEND_WQE_BYTE_4_OPCODE_S, + HNS_ROCE_V2_WQE_OP_MASK); + break; + } + + wqe += sizeof(struct hns_roce_v2_rc_send_wqe); + dseg = wqe; + if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) { + if (rc_sq_wqe->msg_len > + hr_dev->caps.max_sq_inline) { + ret = -EINVAL; + *bad_wr = wr; + dev_err(dev, "inline len(1-%d)=%d, illegal", + rc_sq_wqe->msg_len, + hr_dev->caps.max_sq_inline); + goto out; + } + + for (i = 0; i < wr->num_sge; i++) { + memcpy(wqe, ((void *)wr->sg_list[i].addr), + wr->sg_list[i].length); + wqe += wr->sg_list[i].length; + wqe_sz += wr->sg_list[i].length; + } + + roce_set_bit(rc_sq_wqe->byte_4, + V2_RC_SEND_WQE_BYTE_4_INLINE_S, 1); + } else { + if (wr->num_sge <= 2) { + for (i = 0; i < wr->num_sge; i++) + set_data_seg_v2(dseg + i, + wr->sg_list + i); + } else { + roce_set_field(rc_sq_wqe->byte_20, + V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M, + V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S, + sge_ind & (qp->sge.sge_cnt - 1)); + + for (i = 0; i < 2; i++) + set_data_seg_v2(dseg + i, + wr->sg_list + i); + + dseg = get_send_extend_sge(qp, + sge_ind & (qp->sge.sge_cnt - 1)); + + for (i = 0; i < wr->num_sge - 2; i++) { + set_data_seg_v2(dseg + i, + wr->sg_list + 2 + i); + sge_ind++; + } + } + + roce_set_field(rc_sq_wqe->byte_16, + V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M, + V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, + wr->num_sge); + wqe_sz += wr->num_sge * + sizeof(struct hns_roce_v2_wqe_data_seg); + } + ind++; + } + +out: + if (likely(nreq)) { + qp->sq.head += nreq; + /* Memory barrier */ + wmb(); + + sq_db.byte_4 = 0; + sq_db.parameter = 0; + + roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_TAG_M, + V2_DB_BYTE_4_TAG_S, qp->doorbell_qpn); + roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M, + V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB); + roce_set_field(sq_db.parameter, V2_DB_PARAMETER_CONS_IDX_M, + V2_DB_PARAMETER_CONS_IDX_S, + qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1)); + roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M, + V2_DB_PARAMETER_SL_S, qp->sl); + + hns_roce_write64_k((__be32 *)&sq_db, qp->sq.db_reg_l); + + qp->sq_next_wqe = ind; + qp->next_sge = sge_ind; + } + + spin_unlock_irqrestore(&qp->sq.lock, flags); + + return ret; +} + +static int hns_roce_v2_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr, + struct ib_recv_wr **bad_wr) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + struct hns_roce_v2_wqe_data_seg *dseg; + struct device *dev = hr_dev->dev; + struct hns_roce_v2_db rq_db; + unsigned long flags; + void *wqe = NULL; + int ret = 0; + int nreq; + int ind; + int i; + + spin_lock_irqsave(&hr_qp->rq.lock, flags); + ind = hr_qp->rq.head & (hr_qp->rq.wqe_cnt - 1); + + if (hr_qp->state == IB_QPS_RESET || hr_qp->state == IB_QPS_ERR) { + spin_unlock_irqrestore(&hr_qp->rq.lock, flags); + *bad_wr = wr; + return -EINVAL; + } + + for (nreq = 0; wr; ++nreq, wr = wr->next) { + if (hns_roce_wq_overflow(&hr_qp->rq, nreq, + hr_qp->ibqp.recv_cq)) { + ret = -ENOMEM; + *bad_wr = wr; + goto out; + } + + if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) { + dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n", + wr->num_sge, hr_qp->rq.max_gs); + ret = -EINVAL; + *bad_wr = wr; + goto out; + } + + wqe = get_recv_wqe(hr_qp, ind); + dseg = (struct hns_roce_v2_wqe_data_seg *)wqe; + for (i = 0; i < wr->num_sge; i++) { + if (!wr->sg_list[i].length) + continue; + set_data_seg_v2(dseg, wr->sg_list + i); + dseg++; + } + + if (i < hr_qp->rq.max_gs) { + dseg[i].lkey = cpu_to_be32(HNS_ROCE_INVALID_LKEY); + dseg[i].addr = 0; + } + + hr_qp->rq.wrid[ind] = wr->wr_id; + + ind = (ind + 1) & (hr_qp->rq.wqe_cnt - 1); + } + +out: + if (likely(nreq)) { + hr_qp->rq.head += nreq; + /* Memory barrier */ + wmb(); + + rq_db.byte_4 = 0; + rq_db.parameter = 0; + + roce_set_field(rq_db.byte_4, V2_DB_BYTE_4_TAG_M, + V2_DB_BYTE_4_TAG_S, hr_qp->qpn); + roce_set_field(rq_db.byte_4, V2_DB_BYTE_4_CMD_M, + V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_RQ_DB); + roce_set_field(rq_db.parameter, V2_DB_PARAMETER_CONS_IDX_M, + V2_DB_PARAMETER_CONS_IDX_S, hr_qp->rq.head); + + hns_roce_write64_k((__be32 *)&rq_db, hr_qp->rq.db_reg_l); + } + spin_unlock_irqrestore(&hr_qp->rq.lock, flags); + + return ret; +} + +static int hns_roce_cmq_space(struct hns_roce_v2_cmq_ring *ring) +{ + int ntu = ring->next_to_use; + int ntc = ring->next_to_clean; + int used = (ntu - ntc + ring->desc_num) % ring->desc_num; + + return ring->desc_num - used - 1; +} + +static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev, + struct hns_roce_v2_cmq_ring *ring) +{ + int size = ring->desc_num * sizeof(struct hns_roce_cmq_desc); + + ring->desc = kzalloc(size, GFP_KERNEL); + if (!ring->desc) + return -ENOMEM; + + ring->desc_dma_addr = dma_map_single(hr_dev->dev, ring->desc, size, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(hr_dev->dev, ring->desc_dma_addr)) { + ring->desc_dma_addr = 0; + kfree(ring->desc); + ring->desc = NULL; + return -ENOMEM; + } + + return 0; +} + +static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev, + struct hns_roce_v2_cmq_ring *ring) +{ + dma_unmap_single(hr_dev->dev, ring->desc_dma_addr, + ring->desc_num * sizeof(struct hns_roce_cmq_desc), + DMA_BIDIRECTIONAL); + kfree(ring->desc); +} + +static int hns_roce_init_cmq_ring(struct hns_roce_dev *hr_dev, bool ring_type) +{ + struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; + struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ? + &priv->cmq.csq : &priv->cmq.crq; + + ring->flag = ring_type; + ring->next_to_clean = 0; + ring->next_to_use = 0; + + return hns_roce_alloc_cmq_desc(hr_dev, ring); +} + +static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type) +{ + struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; + struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ? + &priv->cmq.csq : &priv->cmq.crq; + dma_addr_t dma = ring->desc_dma_addr; + + if (ring_type == TYPE_CSQ) { + roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, (u32)dma); + roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG, + upper_32_bits(dma)); + roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG, + (ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S) | + HNS_ROCE_CMQ_ENABLE); + roce_write(hr_dev, ROCEE_TX_CMQ_HEAD_REG, 0); + roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, 0); + } else { + roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_L_REG, (u32)dma); + roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_H_REG, + upper_32_bits(dma)); + roce_write(hr_dev, ROCEE_RX_CMQ_DEPTH_REG, + (ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S) | + HNS_ROCE_CMQ_ENABLE); + roce_write(hr_dev, ROCEE_RX_CMQ_HEAD_REG, 0); + roce_write(hr_dev, ROCEE_RX_CMQ_TAIL_REG, 0); + } +} + +static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; + int ret; + + /* Setup the queue entries for command queue */ + priv->cmq.csq.desc_num = 1024; + priv->cmq.crq.desc_num = 1024; + + /* Setup the lock for command queue */ + spin_lock_init(&priv->cmq.csq.lock); + spin_lock_init(&priv->cmq.crq.lock); + + /* Setup Tx write back timeout */ + priv->cmq.tx_timeout = HNS_ROCE_CMQ_TX_TIMEOUT; + + /* Init CSQ */ + ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CSQ); + if (ret) { + dev_err(hr_dev->dev, "Init CSQ error, ret = %d.\n", ret); + return ret; + } + + /* Init CRQ */ + ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CRQ); + if (ret) { + dev_err(hr_dev->dev, "Init CRQ error, ret = %d.\n", ret); + goto err_crq; + } + + /* Init CSQ REG */ + hns_roce_cmq_init_regs(hr_dev, TYPE_CSQ); + + /* Init CRQ REG */ + hns_roce_cmq_init_regs(hr_dev, TYPE_CRQ); + + return 0; + +err_crq: + hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq); + + return ret; +} + +static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; + + hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq); + hns_roce_free_cmq_desc(hr_dev, &priv->cmq.crq); +} + +static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc, + enum hns_roce_opcode_type opcode, + bool is_read) +{ + memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc)); + desc->opcode = cpu_to_le16(opcode); + desc->flag = + cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN); + if (is_read) + desc->flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR); + else + desc->flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR); +} + +static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; + u32 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG); + + return head == priv->cmq.csq.next_to_use; +} + +static int hns_roce_cmq_csq_clean(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; + struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq; + struct hns_roce_cmq_desc *desc; + u16 ntc = csq->next_to_clean; + u32 head; + int clean = 0; + + desc = &csq->desc[ntc]; + head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG); + while (head != ntc) { + memset(desc, 0, sizeof(*desc)); + ntc++; + if (ntc == csq->desc_num) + ntc = 0; + desc = &csq->desc[ntc]; + clean++; + } + csq->next_to_clean = ntc; + + return clean; +} + +static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev, + struct hns_roce_cmq_desc *desc, int num) +{ + struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; + struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq; + struct hns_roce_cmq_desc *desc_to_use; + bool complete = false; + u32 timeout = 0; + int handle = 0; + u16 desc_ret; + int ret = 0; + int ntc; + + spin_lock_bh(&csq->lock); + + if (num > hns_roce_cmq_space(csq)) { + spin_unlock_bh(&csq->lock); + return -EBUSY; + } + + /* + * Record the location of desc in the cmq for this time + * which will be use for hardware to write back + */ + ntc = csq->next_to_use; + + while (handle < num) { + desc_to_use = &csq->desc[csq->next_to_use]; + *desc_to_use = desc[handle]; + dev_dbg(hr_dev->dev, "set cmq desc:\n"); + csq->next_to_use++; + if (csq->next_to_use == csq->desc_num) + csq->next_to_use = 0; + handle++; + } + + /* Write to hardware */ + roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, csq->next_to_use); + + /* + * If the command is sync, wait for the firmware to write back, + * if multi descriptors to be sent, use the first one to check + */ + if ((desc->flag) & HNS_ROCE_CMD_FLAG_NO_INTR) { + do { + if (hns_roce_cmq_csq_done(hr_dev)) + break; + udelay(1); + timeout++; + } while (timeout < priv->cmq.tx_timeout); + } + + if (hns_roce_cmq_csq_done(hr_dev)) { + complete = true; + handle = 0; + while (handle < num) { + /* get the result of hardware write back */ + desc_to_use = &csq->desc[ntc]; + desc[handle] = *desc_to_use; + dev_dbg(hr_dev->dev, "Get cmq desc:\n"); + desc_ret = desc[handle].retval; + if (desc_ret == CMD_EXEC_SUCCESS) + ret = 0; + else + ret = -EIO; + priv->cmq.last_status = desc_ret; + ntc++; + handle++; + if (ntc == csq->desc_num) + ntc = 0; + } + } + + if (!complete) + ret = -EAGAIN; + + /* clean the command send queue */ + handle = hns_roce_cmq_csq_clean(hr_dev); + if (handle != num) + dev_warn(hr_dev->dev, "Cleaned %d, need to clean %d\n", + handle, num); + + spin_unlock_bh(&csq->lock); + + return ret; +} + +static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_query_version *resp; + struct hns_roce_cmq_desc desc; + int ret; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_HW_VER, true); + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) + return ret; + + resp = (struct hns_roce_query_version *)desc.data; + hr_dev->hw_rev = le32_to_cpu(resp->rocee_hw_version); + hr_dev->vendor_id = le32_to_cpu(resp->rocee_vendor_id); + + return 0; +} + +static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_cfg_global_param *req; + struct hns_roce_cmq_desc desc; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM, + false); + + req = (struct hns_roce_cfg_global_param *)desc.data; + memset(req, 0, sizeof(*req)); + roce_set_field(req->time_cfg_udp_port, + CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_M, + CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_S, 0x3e8); + roce_set_field(req->time_cfg_udp_port, + CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_M, + CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_S, 0x12b7); + + return hns_roce_cmq_send(hr_dev, &desc, 1); +} + +static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_cmq_desc desc[2]; + struct hns_roce_pf_res *res; + int ret; + int i; + + for (i = 0; i < 2; i++) { + hns_roce_cmq_setup_basic_desc(&desc[i], + HNS_ROCE_OPC_QUERY_PF_RES, true); + + if (i == 0) + desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); + else + desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); + } + + ret = hns_roce_cmq_send(hr_dev, desc, 2); + if (ret) + return ret; + + res = (struct hns_roce_pf_res *)desc[0].data; + + hr_dev->caps.qpc_bt_num = roce_get_field(res->qpc_bt_idx_num, + PF_RES_DATA_1_PF_QPC_BT_NUM_M, + PF_RES_DATA_1_PF_QPC_BT_NUM_S); + hr_dev->caps.srqc_bt_num = roce_get_field(res->srqc_bt_idx_num, + PF_RES_DATA_2_PF_SRQC_BT_NUM_M, + PF_RES_DATA_2_PF_SRQC_BT_NUM_S); + hr_dev->caps.cqc_bt_num = roce_get_field(res->cqc_bt_idx_num, + PF_RES_DATA_3_PF_CQC_BT_NUM_M, + PF_RES_DATA_3_PF_CQC_BT_NUM_S); + hr_dev->caps.mpt_bt_num = roce_get_field(res->mpt_bt_idx_num, + PF_RES_DATA_4_PF_MPT_BT_NUM_M, + PF_RES_DATA_4_PF_MPT_BT_NUM_S); + + return 0; +} + +static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_cmq_desc desc[2]; + struct hns_roce_vf_res_a *req_a; + struct hns_roce_vf_res_b *req_b; + int i; + + req_a = (struct hns_roce_vf_res_a *)desc[0].data; + req_b = (struct hns_roce_vf_res_b *)desc[1].data; + memset(req_a, 0, sizeof(*req_a)); + memset(req_b, 0, sizeof(*req_b)); + for (i = 0; i < 2; i++) { + hns_roce_cmq_setup_basic_desc(&desc[i], + HNS_ROCE_OPC_ALLOC_VF_RES, false); + + if (i == 0) + desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); + else + desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); + + if (i == 0) { + roce_set_field(req_a->vf_qpc_bt_idx_num, + VF_RES_A_DATA_1_VF_QPC_BT_IDX_M, + VF_RES_A_DATA_1_VF_QPC_BT_IDX_S, 0); + roce_set_field(req_a->vf_qpc_bt_idx_num, + VF_RES_A_DATA_1_VF_QPC_BT_NUM_M, + VF_RES_A_DATA_1_VF_QPC_BT_NUM_S, + HNS_ROCE_VF_QPC_BT_NUM); + + roce_set_field(req_a->vf_srqc_bt_idx_num, + VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M, + VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S, 0); + roce_set_field(req_a->vf_srqc_bt_idx_num, + VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M, + VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S, + HNS_ROCE_VF_SRQC_BT_NUM); + + roce_set_field(req_a->vf_cqc_bt_idx_num, + VF_RES_A_DATA_3_VF_CQC_BT_IDX_M, + VF_RES_A_DATA_3_VF_CQC_BT_IDX_S, 0); + roce_set_field(req_a->vf_cqc_bt_idx_num, + VF_RES_A_DATA_3_VF_CQC_BT_NUM_M, + VF_RES_A_DATA_3_VF_CQC_BT_NUM_S, + HNS_ROCE_VF_CQC_BT_NUM); + + roce_set_field(req_a->vf_mpt_bt_idx_num, + VF_RES_A_DATA_4_VF_MPT_BT_IDX_M, + VF_RES_A_DATA_4_VF_MPT_BT_IDX_S, 0); + roce_set_field(req_a->vf_mpt_bt_idx_num, + VF_RES_A_DATA_4_VF_MPT_BT_NUM_M, + VF_RES_A_DATA_4_VF_MPT_BT_NUM_S, + HNS_ROCE_VF_MPT_BT_NUM); + + roce_set_field(req_a->vf_eqc_bt_idx_num, + VF_RES_A_DATA_5_VF_EQC_IDX_M, + VF_RES_A_DATA_5_VF_EQC_IDX_S, 0); + roce_set_field(req_a->vf_eqc_bt_idx_num, + VF_RES_A_DATA_5_VF_EQC_NUM_M, + VF_RES_A_DATA_5_VF_EQC_NUM_S, + HNS_ROCE_VF_EQC_NUM); + } else { + roce_set_field(req_b->vf_smac_idx_num, + VF_RES_B_DATA_1_VF_SMAC_IDX_M, + VF_RES_B_DATA_1_VF_SMAC_IDX_S, 0); + roce_set_field(req_b->vf_smac_idx_num, + VF_RES_B_DATA_1_VF_SMAC_NUM_M, + VF_RES_B_DATA_1_VF_SMAC_NUM_S, + HNS_ROCE_VF_SMAC_NUM); + + roce_set_field(req_b->vf_sgid_idx_num, + VF_RES_B_DATA_2_VF_SGID_IDX_M, + VF_RES_B_DATA_2_VF_SGID_IDX_S, 0); + roce_set_field(req_b->vf_sgid_idx_num, + VF_RES_B_DATA_2_VF_SGID_NUM_M, + VF_RES_B_DATA_2_VF_SGID_NUM_S, + HNS_ROCE_VF_SGID_NUM); + + roce_set_field(req_b->vf_qid_idx_sl_num, + VF_RES_B_DATA_3_VF_QID_IDX_M, + VF_RES_B_DATA_3_VF_QID_IDX_S, 0); + roce_set_field(req_b->vf_qid_idx_sl_num, + VF_RES_B_DATA_3_VF_SL_NUM_M, + VF_RES_B_DATA_3_VF_SL_NUM_S, + HNS_ROCE_VF_SL_NUM); + } + } + + return hns_roce_cmq_send(hr_dev, desc, 2); +} + +static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev) +{ + u8 srqc_hop_num = hr_dev->caps.srqc_hop_num; + u8 qpc_hop_num = hr_dev->caps.qpc_hop_num; + u8 cqc_hop_num = hr_dev->caps.cqc_hop_num; + u8 mpt_hop_num = hr_dev->caps.mpt_hop_num; + struct hns_roce_cfg_bt_attr *req; + struct hns_roce_cmq_desc desc; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_BT_ATTR, false); + req = (struct hns_roce_cfg_bt_attr *)desc.data; + memset(req, 0, sizeof(*req)); + + roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M, + CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S, + hr_dev->caps.qpc_ba_pg_sz); + roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M, + CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S, + hr_dev->caps.qpc_buf_pg_sz); + roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M, + CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S, + qpc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : qpc_hop_num); + + roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M, + CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S, + hr_dev->caps.srqc_ba_pg_sz); + roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M, + CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S, + hr_dev->caps.srqc_buf_pg_sz); + roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M, + CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S, + srqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : srqc_hop_num); + + roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M, + CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S, + hr_dev->caps.cqc_ba_pg_sz); + roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M, + CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S, + hr_dev->caps.cqc_buf_pg_sz); + roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M, + CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S, + cqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : cqc_hop_num); + + roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M, + CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S, + hr_dev->caps.mpt_ba_pg_sz); + roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M, + CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S, + hr_dev->caps.mpt_buf_pg_sz); + roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M, + CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S, + mpt_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : mpt_hop_num); + + return hns_roce_cmq_send(hr_dev, &desc, 1); +} + +static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_caps *caps = &hr_dev->caps; + int ret; + + ret = hns_roce_cmq_query_hw_info(hr_dev); + if (ret) { + dev_err(hr_dev->dev, "Query firmware version fail, ret = %d.\n", + ret); + return ret; + } + + ret = hns_roce_config_global_param(hr_dev); + if (ret) { + dev_err(hr_dev->dev, "Configure global param fail, ret = %d.\n", + ret); + } + + /* Get pf resource owned by every pf */ + ret = hns_roce_query_pf_resource(hr_dev); + if (ret) { + dev_err(hr_dev->dev, "Query pf resource fail, ret = %d.\n", + ret); + return ret; + } + + ret = hns_roce_alloc_vf_resource(hr_dev); + if (ret) { + dev_err(hr_dev->dev, "Allocate vf resource fail, ret = %d.\n", + ret); + return ret; + } + + hr_dev->vendor_part_id = 0; + hr_dev->sys_image_guid = 0; + + caps->num_qps = HNS_ROCE_V2_MAX_QP_NUM; + caps->max_wqes = HNS_ROCE_V2_MAX_WQE_NUM; + caps->num_cqs = HNS_ROCE_V2_MAX_CQ_NUM; + caps->max_cqes = HNS_ROCE_V2_MAX_CQE_NUM; + caps->max_sq_sg = HNS_ROCE_V2_MAX_SQ_SGE_NUM; + caps->max_rq_sg = HNS_ROCE_V2_MAX_RQ_SGE_NUM; + caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE; + caps->num_uars = HNS_ROCE_V2_UAR_NUM; + caps->phy_num_uars = HNS_ROCE_V2_PHY_UAR_NUM; + caps->num_aeq_vectors = 1; + caps->num_comp_vectors = 63; + caps->num_other_vectors = 0; + caps->num_mtpts = HNS_ROCE_V2_MAX_MTPT_NUM; + caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS; + caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS; + caps->num_pds = HNS_ROCE_V2_MAX_PD_NUM; + caps->max_qp_init_rdma = HNS_ROCE_V2_MAX_QP_INIT_RDMA; + caps->max_qp_dest_rdma = HNS_ROCE_V2_MAX_QP_DEST_RDMA; + caps->max_sq_desc_sz = HNS_ROCE_V2_MAX_SQ_DESC_SZ; + caps->max_rq_desc_sz = HNS_ROCE_V2_MAX_RQ_DESC_SZ; + caps->max_srq_desc_sz = HNS_ROCE_V2_MAX_SRQ_DESC_SZ; + caps->qpc_entry_sz = HNS_ROCE_V2_QPC_ENTRY_SZ; + caps->irrl_entry_sz = HNS_ROCE_V2_IRRL_ENTRY_SZ; + caps->trrl_entry_sz = HNS_ROCE_V2_TRRL_ENTRY_SZ; + caps->cqc_entry_sz = HNS_ROCE_V2_CQC_ENTRY_SZ; + caps->mtpt_entry_sz = HNS_ROCE_V2_MTPT_ENTRY_SZ; + caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ; + caps->cq_entry_sz = HNS_ROCE_V2_CQE_ENTRY_SIZE; + caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED; + caps->reserved_lkey = 0; + caps->reserved_pds = 0; + caps->reserved_mrws = 1; + caps->reserved_uars = 0; + caps->reserved_cqs = 0; + + caps->qpc_ba_pg_sz = 0; + caps->qpc_buf_pg_sz = 0; + caps->qpc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; + caps->srqc_ba_pg_sz = 0; + caps->srqc_buf_pg_sz = 0; + caps->srqc_hop_num = HNS_ROCE_HOP_NUM_0; + caps->cqc_ba_pg_sz = 0; + caps->cqc_buf_pg_sz = 0; + caps->cqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; + caps->mpt_ba_pg_sz = 0; + caps->mpt_buf_pg_sz = 0; + caps->mpt_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; + caps->pbl_ba_pg_sz = 0; + caps->pbl_buf_pg_sz = 0; + caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM; + caps->mtt_ba_pg_sz = 0; + caps->mtt_buf_pg_sz = 0; + caps->mtt_hop_num = HNS_ROCE_MTT_HOP_NUM; + caps->cqe_ba_pg_sz = 0; + caps->cqe_buf_pg_sz = 0; + caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM; + caps->chunk_sz = HNS_ROCE_V2_TABLE_CHUNK_SIZE; + + caps->flags = HNS_ROCE_CAP_FLAG_REREG_MR | + HNS_ROCE_CAP_FLAG_ROCE_V1_V2; + caps->pkey_table_len[0] = 1; + caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM; + caps->local_ca_ack_delay = 0; + caps->max_mtu = IB_MTU_4096; + + ret = hns_roce_v2_set_bt(hr_dev); + if (ret) + dev_err(hr_dev->dev, "Configure bt attribute fail, ret = %d.\n", + ret); + + return ret; +} + +static int hns_roce_v2_cmd_pending(struct hns_roce_dev *hr_dev) +{ + u32 status = readl(hr_dev->reg_base + ROCEE_VF_MB_STATUS_REG); + + return status >> HNS_ROCE_HW_RUN_BIT_SHIFT; +} + +static int hns_roce_v2_cmd_complete(struct hns_roce_dev *hr_dev) +{ + u32 status = readl(hr_dev->reg_base + ROCEE_VF_MB_STATUS_REG); + + return status & HNS_ROCE_HW_MB_STATUS_MASK; +} + +static int hns_roce_v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param, + u64 out_param, u32 in_modifier, u8 op_modifier, + u16 op, u16 token, int event) +{ + struct device *dev = hr_dev->dev; + u32 __iomem *hcr = (u32 __iomem *)(hr_dev->reg_base + + ROCEE_VF_MB_CFG0_REG); + unsigned long end; + u32 val0 = 0; + u32 val1 = 0; + + end = msecs_to_jiffies(HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS) + jiffies; + while (hns_roce_v2_cmd_pending(hr_dev)) { + if (time_after(jiffies, end)) { + dev_dbg(dev, "jiffies=%d end=%d\n", (int)jiffies, + (int)end); + return -EAGAIN; + } + cond_resched(); + } + + roce_set_field(val0, HNS_ROCE_VF_MB4_TAG_MASK, + HNS_ROCE_VF_MB4_TAG_SHIFT, in_modifier); + roce_set_field(val0, HNS_ROCE_VF_MB4_CMD_MASK, + HNS_ROCE_VF_MB4_CMD_SHIFT, op); + roce_set_field(val1, HNS_ROCE_VF_MB5_EVENT_MASK, + HNS_ROCE_VF_MB5_EVENT_SHIFT, event); + roce_set_field(val1, HNS_ROCE_VF_MB5_TOKEN_MASK, + HNS_ROCE_VF_MB5_TOKEN_SHIFT, token); + + __raw_writeq(cpu_to_le64(in_param), hcr + 0); + __raw_writeq(cpu_to_le64(out_param), hcr + 2); + + /* Memory barrier */ + wmb(); + + __raw_writel(cpu_to_le32(val0), hcr + 4); + __raw_writel(cpu_to_le32(val1), hcr + 5); + + mmiowb(); + + return 0; +} + +static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev, + unsigned long timeout) +{ + struct device *dev = hr_dev->dev; + unsigned long end = 0; + u32 status; + + end = msecs_to_jiffies(timeout) + jiffies; + while (hns_roce_v2_cmd_pending(hr_dev) && time_before(jiffies, end)) + cond_resched(); + + if (hns_roce_v2_cmd_pending(hr_dev)) { + dev_err(dev, "[cmd_poll]hw run cmd TIMEDOUT!\n"); + return -ETIMEDOUT; + } + + status = hns_roce_v2_cmd_complete(hr_dev); + if (status != 0x1) { + dev_err(dev, "mailbox status 0x%x!\n", status); + return -EBUSY; + } + + return 0; +} + +static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u8 port, + int gid_index, union ib_gid *gid, + const struct ib_gid_attr *attr) +{ + enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1; + u32 *p; + u32 val; + + if (!gid || !attr) + return -EINVAL; + + if (attr->gid_type == IB_GID_TYPE_ROCE) + sgid_type = GID_TYPE_FLAG_ROCE_V1; + + if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { + if (ipv6_addr_v4mapped((void *)gid)) + sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4; + else + sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6; + } + + p = (u32 *)&gid->raw[0]; + roce_raw_write(*p, hr_dev->reg_base + ROCEE_VF_SGID_CFG0_REG + + 0x20 * gid_index); + + p = (u32 *)&gid->raw[4]; + roce_raw_write(*p, hr_dev->reg_base + ROCEE_VF_SGID_CFG1_REG + + 0x20 * gid_index); + + p = (u32 *)&gid->raw[8]; + roce_raw_write(*p, hr_dev->reg_base + ROCEE_VF_SGID_CFG2_REG + + 0x20 * gid_index); + + p = (u32 *)&gid->raw[0xc]; + roce_raw_write(*p, hr_dev->reg_base + ROCEE_VF_SGID_CFG3_REG + + 0x20 * gid_index); + + val = roce_read(hr_dev, ROCEE_VF_SGID_CFG4_REG + 0x20 * gid_index); + roce_set_field(val, ROCEE_VF_SGID_CFG4_SGID_TYPE_M, + ROCEE_VF_SGID_CFG4_SGID_TYPE_S, sgid_type); + + roce_write(hr_dev, ROCEE_VF_SGID_CFG4_REG + 0x20 * gid_index, val); + + return 0; +} + +static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port, + u8 *addr) +{ + u16 reg_smac_h; + u32 reg_smac_l; + u32 val; + + reg_smac_l = *(u32 *)(&addr[0]); + roce_raw_write(reg_smac_l, hr_dev->reg_base + ROCEE_VF_SMAC_CFG0_REG + + 0x08 * phy_port); + val = roce_read(hr_dev, ROCEE_VF_SMAC_CFG1_REG + 0x08 * phy_port); + + reg_smac_h = *(u16 *)(&addr[4]); + roce_set_field(val, ROCEE_VF_SMAC_CFG1_VF_SMAC_H_M, + ROCEE_VF_SMAC_CFG1_VF_SMAC_H_S, reg_smac_h); + roce_write(hr_dev, ROCEE_VF_SMAC_CFG1_REG + 0x08 * phy_port, val); + + return 0; +} + +static int hns_roce_v2_write_mtpt(void *mb_buf, struct hns_roce_mr *mr, + unsigned long mtpt_idx) +{ + struct hns_roce_v2_mpt_entry *mpt_entry; + struct scatterlist *sg; + u64 *pages; + int entry; + int i; + + mpt_entry = mb_buf; + memset(mpt_entry, 0, sizeof(*mpt_entry)); + + roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M, + V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID); + roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M, + V2_MPT_BYTE_4_PBL_HOP_NUM_S, mr->pbl_hop_num == + HNS_ROCE_HOP_NUM_0 ? 0 : mr->pbl_hop_num); + roce_set_field(mpt_entry->byte_4_pd_hop_st, + V2_MPT_BYTE_4_PBL_BA_PG_SZ_M, + V2_MPT_BYTE_4_PBL_BA_PG_SZ_S, mr->pbl_ba_pg_sz); + roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M, + V2_MPT_BYTE_4_PD_S, mr->pd); + mpt_entry->byte_4_pd_hop_st = cpu_to_le32(mpt_entry->byte_4_pd_hop_st); + + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 0); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 0); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_BIND_EN_S, + (mr->access & IB_ACCESS_MW_BIND ? 1 : 0)); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_ATOMIC_EN_S, 0); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S, + (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0)); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S, + (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0)); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S, + (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0)); + mpt_entry->byte_8_mw_cnt_en = cpu_to_le32(mpt_entry->byte_8_mw_cnt_en); + + roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, + mr->type == MR_TYPE_MR ? 0 : 1); + mpt_entry->byte_12_mw_pa = cpu_to_le32(mpt_entry->byte_12_mw_pa); + + mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size)); + mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size)); + mpt_entry->lkey = cpu_to_le32(mr->key); + mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova)); + mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova)); + + if (mr->type == MR_TYPE_DMA) + return 0; + + mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size); + + mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3)); + roce_set_field(mpt_entry->byte_48_mode_ba, V2_MPT_BYTE_48_PBL_BA_H_M, + V2_MPT_BYTE_48_PBL_BA_H_S, + upper_32_bits(mr->pbl_ba >> 3)); + mpt_entry->byte_48_mode_ba = cpu_to_le32(mpt_entry->byte_48_mode_ba); + + pages = (u64 *)__get_free_page(GFP_KERNEL); + if (!pages) + return -ENOMEM; + + i = 0; + for_each_sg(mr->umem->sg_head.sgl, sg, mr->umem->nmap, entry) { + pages[i] = ((u64)sg_dma_address(sg)) >> 6; + + /* Record the first 2 entry directly to MTPT table */ + if (i >= HNS_ROCE_V2_MAX_INNER_MTPT_NUM - 1) + break; + i++; + } + + mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0])); + roce_set_field(mpt_entry->byte_56_pa0_h, V2_MPT_BYTE_56_PA0_H_M, + V2_MPT_BYTE_56_PA0_H_S, + upper_32_bits(pages[0])); + mpt_entry->byte_56_pa0_h = cpu_to_le32(mpt_entry->byte_56_pa0_h); + + mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1])); + roce_set_field(mpt_entry->byte_64_buf_pa1, V2_MPT_BYTE_64_PA1_H_M, + V2_MPT_BYTE_64_PA1_H_S, upper_32_bits(pages[1])); + + free_page((unsigned long)pages); + + roce_set_field(mpt_entry->byte_64_buf_pa1, + V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M, + V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S, mr->pbl_buf_pg_sz); + mpt_entry->byte_64_buf_pa1 = cpu_to_le32(mpt_entry->byte_64_buf_pa1); + + return 0; +} + +static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev, + struct hns_roce_mr *mr, int flags, + u32 pdn, int mr_access_flags, u64 iova, + u64 size, void *mb_buf) +{ + struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf; + + if (flags & IB_MR_REREG_PD) { + roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M, + V2_MPT_BYTE_4_PD_S, pdn); + mr->pd = pdn; + } + + if (flags & IB_MR_REREG_ACCESS) { + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, + V2_MPT_BYTE_8_BIND_EN_S, + (mr_access_flags & IB_ACCESS_MW_BIND ? 1 : 0)); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, + V2_MPT_BYTE_8_ATOMIC_EN_S, + (mr_access_flags & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0)); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S, + (mr_access_flags & IB_ACCESS_REMOTE_READ ? 1 : 0)); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S, + (mr_access_flags & IB_ACCESS_REMOTE_WRITE ? 1 : 0)); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S, + (mr_access_flags & IB_ACCESS_LOCAL_WRITE ? 1 : 0)); + } + + if (flags & IB_MR_REREG_TRANS) { + mpt_entry->va_l = cpu_to_le32(lower_32_bits(iova)); + mpt_entry->va_h = cpu_to_le32(upper_32_bits(iova)); + mpt_entry->len_l = cpu_to_le32(lower_32_bits(size)); + mpt_entry->len_h = cpu_to_le32(upper_32_bits(size)); + + mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size); + mpt_entry->pbl_ba_l = + cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3)); + roce_set_field(mpt_entry->byte_48_mode_ba, + V2_MPT_BYTE_48_PBL_BA_H_M, + V2_MPT_BYTE_48_PBL_BA_H_S, + upper_32_bits(mr->pbl_ba >> 3)); + mpt_entry->byte_48_mode_ba = + cpu_to_le32(mpt_entry->byte_48_mode_ba); + + mr->iova = iova; + mr->size = size; + } + + return 0; +} + +static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n) +{ + return hns_roce_buf_offset(&hr_cq->hr_buf.hr_buf, + n * HNS_ROCE_V2_CQE_ENTRY_SIZE); +} + +static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, int n) +{ + struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n & hr_cq->ib_cq.cqe); + + /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */ + return (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_OWNER_S) ^ + !!(n & (hr_cq->ib_cq.cqe + 1))) ? cqe : NULL; +} + +static struct hns_roce_v2_cqe *next_cqe_sw_v2(struct hns_roce_cq *hr_cq) +{ + return get_sw_cqe_v2(hr_cq, hr_cq->cons_index); +} + +static void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index) +{ + struct hns_roce_v2_cq_db cq_db; + + cq_db.byte_4 = 0; + cq_db.parameter = 0; + + roce_set_field(cq_db.byte_4, V2_CQ_DB_BYTE_4_TAG_M, + V2_CQ_DB_BYTE_4_TAG_S, hr_cq->cqn); + roce_set_field(cq_db.byte_4, V2_CQ_DB_BYTE_4_CMD_M, + V2_CQ_DB_BYTE_4_CMD_S, HNS_ROCE_V2_CQ_DB_PTR); + + roce_set_field(cq_db.parameter, V2_CQ_DB_PARAMETER_CONS_IDX_M, + V2_CQ_DB_PARAMETER_CONS_IDX_S, + cons_index & ((hr_cq->cq_depth << 1) - 1)); + roce_set_field(cq_db.parameter, V2_CQ_DB_PARAMETER_CMD_SN_M, + V2_CQ_DB_PARAMETER_CMD_SN_S, 1); + + hns_roce_write64_k((__be32 *)&cq_db, hr_cq->cq_db_l); + +} + +static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, + struct hns_roce_srq *srq) +{ + struct hns_roce_v2_cqe *cqe, *dest; + u32 prod_index; + int nfreed = 0; + u8 owner_bit; + + for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index); + ++prod_index) { + if (prod_index == hr_cq->cons_index + hr_cq->ib_cq.cqe) + break; + } + + /* + * Now backwards through the CQ, removing CQ entries + * that match our QP by overwriting them with next entries. + */ + while ((int) --prod_index - (int) hr_cq->cons_index >= 0) { + cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe); + if ((roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M, + V2_CQE_BYTE_16_LCL_QPN_S) & + HNS_ROCE_V2_CQE_QPN_MASK) == qpn) { + /* In v1 engine, not support SRQ */ + ++nfreed; + } else if (nfreed) { + dest = get_cqe_v2(hr_cq, (prod_index + nfreed) & + hr_cq->ib_cq.cqe); + owner_bit = roce_get_bit(dest->byte_4, + V2_CQE_BYTE_4_OWNER_S); + memcpy(dest, cqe, sizeof(*cqe)); + roce_set_bit(dest->byte_4, V2_CQE_BYTE_4_OWNER_S, + owner_bit); + } + } + + if (nfreed) { + hr_cq->cons_index += nfreed; + /* + * Make sure update of buffer contents is done before + * updating consumer index. + */ + wmb(); + hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index); + } +} + +static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, + struct hns_roce_srq *srq) +{ + spin_lock_irq(&hr_cq->lock); + __hns_roce_v2_cq_clean(hr_cq, qpn, srq); + spin_unlock_irq(&hr_cq->lock); +} + +static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev, + struct hns_roce_cq *hr_cq, void *mb_buf, + u64 *mtts, dma_addr_t dma_handle, int nent, + u32 vector) +{ + struct hns_roce_v2_cq_context *cq_context; + + cq_context = mb_buf; + memset(cq_context, 0, sizeof(*cq_context)); + + roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CQ_ST_M, + V2_CQC_BYTE_4_CQ_ST_S, V2_CQ_STATE_VALID); + roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_SHIFT_M, + V2_CQC_BYTE_4_SHIFT_S, ilog2((unsigned int)nent)); + roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CEQN_M, + V2_CQC_BYTE_4_CEQN_S, vector); + cq_context->byte_4_pg_ceqn = cpu_to_le32(cq_context->byte_4_pg_ceqn); + + roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQN_M, + V2_CQC_BYTE_8_CQN_S, hr_cq->cqn); + + cq_context->cqe_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT); + cq_context->cqe_cur_blk_addr = + cpu_to_le32(cq_context->cqe_cur_blk_addr); + + roce_set_field(cq_context->byte_16_hop_addr, + V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M, + V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S, + cpu_to_le32((mtts[0]) >> (32 + PAGE_ADDR_SHIFT))); + roce_set_field(cq_context->byte_16_hop_addr, + V2_CQC_BYTE_16_CQE_HOP_NUM_M, + V2_CQC_BYTE_16_CQE_HOP_NUM_S, hr_dev->caps.cqe_hop_num == + HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num); + + cq_context->cqe_nxt_blk_addr = (u32)(mtts[1] >> PAGE_ADDR_SHIFT); + roce_set_field(cq_context->byte_24_pgsz_addr, + V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M, + V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S, + cpu_to_le32((mtts[1]) >> (32 + PAGE_ADDR_SHIFT))); + roce_set_field(cq_context->byte_24_pgsz_addr, + V2_CQC_BYTE_24_CQE_BA_PG_SZ_M, + V2_CQC_BYTE_24_CQE_BA_PG_SZ_S, + hr_dev->caps.cqe_ba_pg_sz); + roce_set_field(cq_context->byte_24_pgsz_addr, + V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M, + V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S, + hr_dev->caps.cqe_buf_pg_sz); + + cq_context->cqe_ba = (u32)(dma_handle >> 3); + + roce_set_field(cq_context->byte_40_cqe_ba, V2_CQC_BYTE_40_CQE_BA_M, + V2_CQC_BYTE_40_CQE_BA_S, (dma_handle >> (32 + 3))); +} + +static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq, + enum ib_cq_notify_flags flags) +{ + struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); + u32 notification_flag; + u32 doorbell[2]; + + doorbell[0] = 0; + doorbell[1] = 0; + + notification_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? + V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL; + /* + * flags = 0; Notification Flag = 1, next + * flags = 1; Notification Flag = 0, solocited + */ + roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_TAG_M, V2_DB_BYTE_4_TAG_S, + hr_cq->cqn); + roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_CMD_M, V2_DB_BYTE_4_CMD_S, + HNS_ROCE_V2_CQ_DB_NTR); + roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CONS_IDX_M, + V2_CQ_DB_PARAMETER_CONS_IDX_S, + hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1)); + roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CMD_SN_M, + V2_CQ_DB_PARAMETER_CMD_SN_S, hr_cq->arm_sn & 0x3); + roce_set_bit(doorbell[1], V2_CQ_DB_PARAMETER_NOTIFY_S, + notification_flag); + + hns_roce_write64_k(doorbell, hr_cq->cq_db_l); + + return 0; +} + +static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq, + struct hns_roce_qp **cur_qp, struct ib_wc *wc) +{ + struct hns_roce_dev *hr_dev; + struct hns_roce_v2_cqe *cqe; + struct hns_roce_qp *hr_qp; + struct hns_roce_wq *wq; + int is_send; + u16 wqe_ctr; + u32 opcode; + u32 status; + int qpn; + + /* Find cqe according to consumer index */ + cqe = next_cqe_sw_v2(hr_cq); + if (!cqe) + return -EAGAIN; + + ++hr_cq->cons_index; + /* Memory barrier */ + rmb(); + + /* 0->SQ, 1->RQ */ + is_send = !roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S); + + qpn = roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M, + V2_CQE_BYTE_16_LCL_QPN_S); + + if (!*cur_qp || (qpn & HNS_ROCE_V2_CQE_QPN_MASK) != (*cur_qp)->qpn) { + hr_dev = to_hr_dev(hr_cq->ib_cq.device); + hr_qp = __hns_roce_qp_lookup(hr_dev, qpn); + if (unlikely(!hr_qp)) { + dev_err(hr_dev->dev, "CQ %06lx with entry for unknown QPN %06x\n", + hr_cq->cqn, (qpn & HNS_ROCE_V2_CQE_QPN_MASK)); + return -EINVAL; + } + *cur_qp = hr_qp; + } + + wc->qp = &(*cur_qp)->ibqp; + wc->vendor_err = 0; + + status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M, + V2_CQE_BYTE_4_STATUS_S); + switch (status & HNS_ROCE_V2_CQE_STATUS_MASK) { + case HNS_ROCE_CQE_V2_SUCCESS: + wc->status = IB_WC_SUCCESS; + break; + case HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR: + wc->status = IB_WC_LOC_LEN_ERR; + break; + case HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR: + wc->status = IB_WC_LOC_QP_OP_ERR; + break; + case HNS_ROCE_CQE_V2_LOCAL_PROT_ERR: + wc->status = IB_WC_LOC_PROT_ERR; + break; + case HNS_ROCE_CQE_V2_WR_FLUSH_ERR: + wc->status = IB_WC_WR_FLUSH_ERR; + break; + case HNS_ROCE_CQE_V2_MW_BIND_ERR: + wc->status = IB_WC_MW_BIND_ERR; + break; + case HNS_ROCE_CQE_V2_BAD_RESP_ERR: + wc->status = IB_WC_BAD_RESP_ERR; + break; + case HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR: + wc->status = IB_WC_LOC_ACCESS_ERR; + break; + case HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR: + wc->status = IB_WC_REM_INV_REQ_ERR; + break; + case HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR: + wc->status = IB_WC_REM_ACCESS_ERR; + break; + case HNS_ROCE_CQE_V2_REMOTE_OP_ERR: + wc->status = IB_WC_REM_OP_ERR; + break; + case HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR: + wc->status = IB_WC_RETRY_EXC_ERR; + break; + case HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR: + wc->status = IB_WC_RNR_RETRY_EXC_ERR; + break; + case HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR: + wc->status = IB_WC_REM_ABORT_ERR; + break; + default: + wc->status = IB_WC_GENERAL_ERR; + break; + } + + /* CQE status error, directly return */ + if (wc->status != IB_WC_SUCCESS) + return 0; + + if (is_send) { + wc->wc_flags = 0; + /* SQ corresponding to CQE */ + switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M, + V2_CQE_BYTE_4_OPCODE_S) & 0x1f) { + case HNS_ROCE_SQ_OPCODE_SEND: + wc->opcode = IB_WC_SEND; + break; + case HNS_ROCE_SQ_OPCODE_SEND_WITH_INV: + wc->opcode = IB_WC_SEND; + break; + case HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM: + wc->opcode = IB_WC_SEND; + wc->wc_flags |= IB_WC_WITH_IMM; + break; + case HNS_ROCE_SQ_OPCODE_RDMA_READ: + wc->opcode = IB_WC_RDMA_READ; + wc->byte_len = le32_to_cpu(cqe->byte_cnt); + break; + case HNS_ROCE_SQ_OPCODE_RDMA_WRITE: + wc->opcode = IB_WC_RDMA_WRITE; + break; + case HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM: + wc->opcode = IB_WC_RDMA_WRITE; + wc->wc_flags |= IB_WC_WITH_IMM; + break; + case HNS_ROCE_SQ_OPCODE_LOCAL_INV: + wc->opcode = IB_WC_LOCAL_INV; + wc->wc_flags |= IB_WC_WITH_INVALIDATE; + break; + case HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP: + wc->opcode = IB_WC_COMP_SWAP; + wc->byte_len = 8; + break; + case HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD: + wc->opcode = IB_WC_FETCH_ADD; + wc->byte_len = 8; + break; + case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP: + wc->opcode = IB_WC_MASKED_COMP_SWAP; + wc->byte_len = 8; + break; + case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD: + wc->opcode = IB_WC_MASKED_FETCH_ADD; + wc->byte_len = 8; + break; + case HNS_ROCE_SQ_OPCODE_FAST_REG_WR: + wc->opcode = IB_WC_REG_MR; + break; + case HNS_ROCE_SQ_OPCODE_BIND_MW: + wc->opcode = IB_WC_REG_MR; + break; + default: + wc->status = IB_WC_GENERAL_ERR; + break; + } + + wq = &(*cur_qp)->sq; + if ((*cur_qp)->sq_signal_bits) { + /* + * If sg_signal_bit is 1, + * firstly tail pointer updated to wqe + * which current cqe correspond to + */ + wqe_ctr = (u16)roce_get_field(cqe->byte_4, + V2_CQE_BYTE_4_WQE_INDX_M, + V2_CQE_BYTE_4_WQE_INDX_S); + wq->tail += (wqe_ctr - (u16)wq->tail) & + (wq->wqe_cnt - 1); + } + + wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; + ++wq->tail; + } else { + /* RQ correspond to CQE */ + wc->byte_len = le32_to_cpu(cqe->byte_cnt); + + opcode = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M, + V2_CQE_BYTE_4_OPCODE_S); + switch (opcode & 0x1f) { + case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM: + wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; + wc->wc_flags = IB_WC_WITH_IMM; + wc->ex.imm_data = le32_to_cpu(cqe->rkey_immtdata); + break; + case HNS_ROCE_V2_OPCODE_SEND: + wc->opcode = IB_WC_RECV; + wc->wc_flags = 0; + break; + case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM: + wc->opcode = IB_WC_RECV; + wc->wc_flags = IB_WC_WITH_IMM; + wc->ex.imm_data = le32_to_cpu(cqe->rkey_immtdata); + break; + case HNS_ROCE_V2_OPCODE_SEND_WITH_INV: + wc->opcode = IB_WC_RECV; + wc->wc_flags = IB_WC_WITH_INVALIDATE; + wc->ex.invalidate_rkey = cqe->rkey_immtdata; + break; + default: + wc->status = IB_WC_GENERAL_ERR; + break; + } + + /* Update tail pointer, record wr_id */ + wq = &(*cur_qp)->rq; + wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; + ++wq->tail; + + wc->sl = (u8)roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_SL_M, + V2_CQE_BYTE_32_SL_S); + wc->src_qp = (u8)roce_get_field(cqe->byte_32, + V2_CQE_BYTE_32_RMT_QPN_M, + V2_CQE_BYTE_32_RMT_QPN_S); + wc->wc_flags |= (roce_get_bit(cqe->byte_32, + V2_CQE_BYTE_32_GRH_S) ? + IB_WC_GRH : 0); + } + + return 0; +} + +static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries, + struct ib_wc *wc) +{ + struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); + struct hns_roce_qp *cur_qp = NULL; + unsigned long flags; + int npolled; + + spin_lock_irqsave(&hr_cq->lock, flags); + + for (npolled = 0; npolled < num_entries; ++npolled) { + if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled)) + break; + } + + if (npolled) { + /* Memory barrier */ + wmb(); + hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index); + } + + spin_unlock_irqrestore(&hr_cq->lock, flags); + + return npolled; +} + +static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, int obj, + int step_idx) +{ + struct device *dev = hr_dev->dev; + struct hns_roce_cmd_mailbox *mailbox; + struct hns_roce_hem_iter iter; + struct hns_roce_hem_mhop mhop; + struct hns_roce_hem *hem; + unsigned long mhop_obj = obj; + int i, j, k; + int ret = 0; + u64 hem_idx = 0; + u64 l1_idx = 0; + u64 bt_ba = 0; + u32 chunk_ba_num; + u32 hop_num; + u16 op = 0xff; + + if (!hns_roce_check_whether_mhop(hr_dev, table->type)) + return 0; + + hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); + i = mhop.l0_idx; + j = mhop.l1_idx; + k = mhop.l2_idx; + hop_num = mhop.hop_num; + chunk_ba_num = mhop.bt_chunk_size / 8; + + if (hop_num == 2) { + hem_idx = i * chunk_ba_num * chunk_ba_num + j * chunk_ba_num + + k; + l1_idx = i * chunk_ba_num + j; + } else if (hop_num == 1) { + hem_idx = i * chunk_ba_num + j; + } else if (hop_num == HNS_ROCE_HOP_NUM_0) { + hem_idx = i; + } + + switch (table->type) { + case HEM_TYPE_QPC: + op = HNS_ROCE_CMD_WRITE_QPC_BT0; + break; + case HEM_TYPE_MTPT: + op = HNS_ROCE_CMD_WRITE_MPT_BT0; + break; + case HEM_TYPE_CQC: + op = HNS_ROCE_CMD_WRITE_CQC_BT0; + break; + case HEM_TYPE_SRQC: + op = HNS_ROCE_CMD_WRITE_SRQC_BT0; + break; + default: + dev_warn(dev, "Table %d not to be written by mailbox!\n", + table->type); + return 0; + } + op += step_idx; + + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + if (check_whether_last_step(hop_num, step_idx)) { + hem = table->hem[hem_idx]; + for (hns_roce_hem_first(hem, &iter); + !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) { + bt_ba = hns_roce_hem_addr(&iter); + + /* configure the ba, tag, and op */ + ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, + obj, 0, op, + HNS_ROCE_CMD_TIMEOUT_MSECS); + } + } else { + if (step_idx == 0) + bt_ba = table->bt_l0_dma_addr[i]; + else if (step_idx == 1 && hop_num == 2) + bt_ba = table->bt_l1_dma_addr[l1_idx]; + + /* configure the ba, tag, and op */ + ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj, + 0, op, HNS_ROCE_CMD_TIMEOUT_MSECS); + } + + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + return ret; +} + +static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, int obj, + int step_idx) +{ + struct device *dev = hr_dev->dev; + struct hns_roce_cmd_mailbox *mailbox; + int ret = 0; + u16 op = 0xff; + + if (!hns_roce_check_whether_mhop(hr_dev, table->type)) + return 0; + + switch (table->type) { + case HEM_TYPE_QPC: + op = HNS_ROCE_CMD_DESTROY_QPC_BT0; + break; + case HEM_TYPE_MTPT: + op = HNS_ROCE_CMD_DESTROY_MPT_BT0; + break; + case HEM_TYPE_CQC: + op = HNS_ROCE_CMD_DESTROY_CQC_BT0; + break; + case HEM_TYPE_SRQC: + op = HNS_ROCE_CMD_DESTROY_SRQC_BT0; + break; + default: + dev_warn(dev, "Table %d not to be destroyed by mailbox!\n", + table->type); + return 0; + } + op += step_idx; + + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + /* configure the tag and op */ + ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, obj, 0, op, + HNS_ROCE_CMD_TIMEOUT_MSECS); + + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + return ret; +} + +static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev, + struct hns_roce_mtt *mtt, + enum ib_qp_state cur_state, + enum ib_qp_state new_state, + struct hns_roce_v2_qp_context *context, + struct hns_roce_qp *hr_qp) +{ + struct hns_roce_cmd_mailbox *mailbox; + int ret; + + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + memcpy(mailbox->buf, context, sizeof(*context) * 2); + + ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0, + HNS_ROCE_CMD_MODIFY_QPC, + HNS_ROCE_CMD_TIMEOUT_MSECS); + + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + + return ret; +} + +static void modify_qp_reset_to_init(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, + struct hns_roce_v2_qp_context *context, + struct hns_roce_v2_qp_context *qpc_mask) +{ + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + + /* + * In v2 engine, software pass context and context mask to hardware + * when modifying qp. If software need modify some fields in context, + * we should set all bits of the relevant fields in context mask to + * 0 at the same time, else set them to 0x1. + */ + roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M, + V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type)); + roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M, + V2_QPC_BYTE_4_TST_S, 0); + + roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M, + V2_QPC_BYTE_4_SGE_SHIFT_S, hr_qp->sq.max_gs > 2 ? + ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0); + roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M, + V2_QPC_BYTE_4_SGE_SHIFT_S, 0); + + roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M, + V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn); + roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M, + V2_QPC_BYTE_4_SQPN_S, 0); + + roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M, + V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn); + roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M, + V2_QPC_BYTE_16_PD_S, 0); + + roce_set_field(context->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M, + V2_QPC_BYTE_20_RQWS_S, ilog2(hr_qp->rq.max_gs)); + roce_set_field(qpc_mask->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M, + V2_QPC_BYTE_20_RQWS_S, 0); + + roce_set_field(context->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, + ilog2((unsigned int)hr_qp->sq.wqe_cnt)); + roce_set_field(qpc_mask->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0); + + roce_set_field(context->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, + ilog2((unsigned int)hr_qp->rq.wqe_cnt)); + roce_set_field(qpc_mask->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0); + + /* No VLAN need to set 0xFFF */ + roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_IDX_M, + V2_QPC_BYTE_24_VLAN_IDX_S, 0xfff); + roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_IDX_M, + V2_QPC_BYTE_24_VLAN_IDX_S, 0); + + /* + * Set some fields in context to zero, Because the default values + * of all fields in context are zero, we need not set them to 0 again. + * but we should set the relevant fields of context mask to 0. + */ + roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_TX_ERR_S, 0); + roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_RX_ERR_S, 0); + roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_TX_ERR_S, 0); + roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_RX_ERR_S, 0); + + roce_set_field(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_MAPID_M, + V2_QPC_BYTE_60_MAPID_S, 0); + + roce_set_bit(qpc_mask->byte_60_qpst_mapid, + V2_QPC_BYTE_60_INNER_MAP_IND_S, 0); + roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_SQ_MAP_IND_S, + 0); + roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_RQ_MAP_IND_S, + 0); + roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_EXT_MAP_IND_S, + 0); + roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_SQ_RLS_IND_S, + 0); + roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_SQ_EXT_IND_S, + 0); + roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CNP_TX_FLAG_S, 0); + roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CE_FLAG_S, 0); + + roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, + !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 0); + + roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, + !!(attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 0); + + roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, + !!(attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 0); + + roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 0); + + roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M, + V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn); + roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M, + V2_QPC_BYTE_80_RX_CQN_S, 0); + if (ibqp->srq) { + roce_set_field(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, + to_hr_srq(ibqp->srq)->srqn); + roce_set_field(qpc_mask->byte_76_srqn_op_en, + V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0); + roce_set_bit(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_SRQ_EN_S, 1); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, + V2_QPC_BYTE_76_SRQ_EN_S, 0); + } + + roce_set_field(qpc_mask->byte_84_rq_ci_pi, + V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M, + V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0); + roce_set_field(qpc_mask->byte_84_rq_ci_pi, + V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M, + V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0); + + roce_set_field(qpc_mask->byte_92_srq_info, V2_QPC_BYTE_92_SRQ_INFO_M, + V2_QPC_BYTE_92_SRQ_INFO_S, 0); + + roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M, + V2_QPC_BYTE_96_RX_REQ_MSN_S, 0); + + roce_set_field(qpc_mask->byte_104_rq_sge, + V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_M, + V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_S, 0); + + roce_set_bit(qpc_mask->byte_108_rx_reqepsn, + V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0); + roce_set_field(qpc_mask->byte_108_rx_reqepsn, + V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M, + V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0); + roce_set_bit(qpc_mask->byte_108_rx_reqepsn, + V2_QPC_BYTE_108_RX_REQ_RNR_S, 0); + + qpc_mask->rq_rnr_timer = 0; + qpc_mask->rx_msg_len = 0; + qpc_mask->rx_rkey_pkt_info = 0; + qpc_mask->rx_va = 0; + + roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M, + V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0); + roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M, + V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0); + + roce_set_bit(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RSVD_RAQ_MAP_S, 0); + roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_HEAD_M, + V2_QPC_BYTE_140_RAQ_TRRL_HEAD_S, 0); + roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_TAIL_M, + V2_QPC_BYTE_140_RAQ_TRRL_TAIL_S, 0); + + roce_set_field(qpc_mask->byte_144_raq, + V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_M, + V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_S, 0); + roce_set_bit(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RAQ_RTY_INI_IND_S, + 0); + roce_set_field(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RAQ_CREDIT_M, + V2_QPC_BYTE_144_RAQ_CREDIT_S, 0); + roce_set_bit(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RESP_RTY_FLG_S, 0); + + roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RQ_MSN_M, + V2_QPC_BYTE_148_RQ_MSN_S, 0); + roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RAQ_SYNDROME_M, + V2_QPC_BYTE_148_RAQ_SYNDROME_S, 0); + + roce_set_field(qpc_mask->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M, + V2_QPC_BYTE_152_RAQ_PSN_S, 0); + roce_set_field(qpc_mask->byte_152_raq, + V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_M, + V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_S, 0); + + roce_set_field(qpc_mask->byte_156_raq, V2_QPC_BYTE_156_RAQ_USE_PKTN_M, + V2_QPC_BYTE_156_RAQ_USE_PKTN_S, 0); + + roce_set_field(qpc_mask->byte_160_sq_ci_pi, + V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M, + V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0); + roce_set_field(qpc_mask->byte_160_sq_ci_pi, + V2_QPC_BYTE_160_SQ_CONSUMER_IDX_M, + V2_QPC_BYTE_160_SQ_CONSUMER_IDX_S, 0); + + roce_set_field(context->byte_168_irrl_idx, + V2_QPC_BYTE_168_SQ_SHIFT_BAK_M, + V2_QPC_BYTE_168_SQ_SHIFT_BAK_S, + ilog2((unsigned int)hr_qp->sq.wqe_cnt)); + roce_set_field(qpc_mask->byte_168_irrl_idx, + V2_QPC_BYTE_168_SQ_SHIFT_BAK_M, + V2_QPC_BYTE_168_SQ_SHIFT_BAK_S, 0); + + roce_set_bit(qpc_mask->byte_168_irrl_idx, + V2_QPC_BYTE_168_MSG_RTY_LP_FLG_S, 0); + roce_set_bit(qpc_mask->byte_168_irrl_idx, + V2_QPC_BYTE_168_SQ_INVLD_FLG_S, 0); + roce_set_field(qpc_mask->byte_168_irrl_idx, + V2_QPC_BYTE_168_IRRL_IDX_LSB_M, + V2_QPC_BYTE_168_IRRL_IDX_LSB_S, 0); + + roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_ACK_REQ_FREQ_M, + V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 4); + roce_set_field(qpc_mask->byte_172_sq_psn, + V2_QPC_BYTE_172_ACK_REQ_FREQ_M, + V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 0); + + roce_set_bit(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_MSG_RNR_FLG_S, + 0); + + roce_set_field(qpc_mask->byte_176_msg_pktn, + V2_QPC_BYTE_176_MSG_USE_PKTN_M, + V2_QPC_BYTE_176_MSG_USE_PKTN_S, 0); + roce_set_field(qpc_mask->byte_176_msg_pktn, + V2_QPC_BYTE_176_IRRL_HEAD_PRE_M, + V2_QPC_BYTE_176_IRRL_HEAD_PRE_S, 0); + + roce_set_field(qpc_mask->byte_184_irrl_idx, + V2_QPC_BYTE_184_IRRL_IDX_MSB_M, + V2_QPC_BYTE_184_IRRL_IDX_MSB_S, 0); + + qpc_mask->cur_sge_offset = 0; + + roce_set_field(qpc_mask->byte_192_ext_sge, + V2_QPC_BYTE_192_CUR_SGE_IDX_M, + V2_QPC_BYTE_192_CUR_SGE_IDX_S, 0); + roce_set_field(qpc_mask->byte_192_ext_sge, + V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_M, + V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_S, 0); + + roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M, + V2_QPC_BYTE_196_IRRL_HEAD_S, 0); + + roce_set_field(qpc_mask->byte_200_sq_max, V2_QPC_BYTE_200_SQ_MAX_IDX_M, + V2_QPC_BYTE_200_SQ_MAX_IDX_S, 0); + roce_set_field(qpc_mask->byte_200_sq_max, + V2_QPC_BYTE_200_LCL_OPERATED_CNT_M, + V2_QPC_BYTE_200_LCL_OPERATED_CNT_S, 0); + + roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RNR_FLG_S, 0); + roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RTY_FLG_S, 0); + + roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M, + V2_QPC_BYTE_212_CHECK_FLG_S, 0); + + qpc_mask->sq_timer = 0; + + roce_set_field(qpc_mask->byte_220_retry_psn_msn, + V2_QPC_BYTE_220_RETRY_MSG_MSN_M, + V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0); + roce_set_field(qpc_mask->byte_232_irrl_sge, + V2_QPC_BYTE_232_IRRL_SGE_IDX_M, + V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0); + + qpc_mask->irrl_cur_sge_offset = 0; + + roce_set_field(qpc_mask->byte_240_irrl_tail, + V2_QPC_BYTE_240_IRRL_TAIL_REAL_M, + V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0); + roce_set_field(qpc_mask->byte_240_irrl_tail, + V2_QPC_BYTE_240_IRRL_TAIL_RD_M, + V2_QPC_BYTE_240_IRRL_TAIL_RD_S, 0); + roce_set_field(qpc_mask->byte_240_irrl_tail, + V2_QPC_BYTE_240_RX_ACK_MSN_M, + V2_QPC_BYTE_240_RX_ACK_MSN_S, 0); + + roce_set_field(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_M, + V2_QPC_BYTE_248_IRRL_PSN_S, 0); + roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_ACK_PSN_ERR_S, + 0); + roce_set_field(qpc_mask->byte_248_ack_psn, + V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M, + V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0); + roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_VLD_S, + 0); + roce_set_bit(qpc_mask->byte_248_ack_psn, + V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0); + roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_CQ_ERR_IND_S, + 0); + + hr_qp->access_flags = attr->qp_access_flags; + hr_qp->pkey_index = attr->pkey_index; + roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M, + V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn); + roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M, + V2_QPC_BYTE_252_TX_CQN_S, 0); + + roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_ERR_TYPE_M, + V2_QPC_BYTE_252_ERR_TYPE_S, 0); + + roce_set_field(qpc_mask->byte_256_sqflush_rqcqe, + V2_QPC_BYTE_256_RQ_CQE_IDX_M, + V2_QPC_BYTE_256_RQ_CQE_IDX_S, 0); + roce_set_field(qpc_mask->byte_256_sqflush_rqcqe, + V2_QPC_BYTE_256_SQ_FLUSH_IDX_M, + V2_QPC_BYTE_256_SQ_FLUSH_IDX_S, 0); +} + +static void modify_qp_init_to_init(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, int attr_mask, + struct hns_roce_v2_qp_context *context, + struct hns_roce_v2_qp_context *qpc_mask) +{ + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + + /* + * In v2 engine, software pass context and context mask to hardware + * when modifying qp. If software need modify some fields in context, + * we should set all bits of the relevant fields in context mask to + * 0 at the same time, else set them to 0x1. + */ + roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M, + V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type)); + roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M, + V2_QPC_BYTE_4_TST_S, 0); + + roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M, + V2_QPC_BYTE_4_SGE_SHIFT_S, hr_qp->sq.max_gs > 2 ? + ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0); + roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M, + V2_QPC_BYTE_4_SGE_SHIFT_S, 0); + + if (attr_mask & IB_QP_ACCESS_FLAGS) { + roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, + !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, + 0); + + roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, + !!(attr->qp_access_flags & + IB_ACCESS_REMOTE_WRITE)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, + 0); + + roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, + !!(attr->qp_access_flags & + IB_ACCESS_REMOTE_ATOMIC)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, + 0); + } else { + roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, + !!(hr_qp->access_flags & IB_ACCESS_REMOTE_READ)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, + 0); + + roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, + !!(hr_qp->access_flags & IB_ACCESS_REMOTE_WRITE)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, + 0); + + roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, + !!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, + 0); + } + + roce_set_field(context->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, + ilog2((unsigned int)hr_qp->sq.wqe_cnt)); + roce_set_field(qpc_mask->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0); + + roce_set_field(context->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, + ilog2((unsigned int)hr_qp->rq.wqe_cnt)); + roce_set_field(qpc_mask->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0); + + roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M, + V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn); + roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M, + V2_QPC_BYTE_16_PD_S, 0); + + roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M, + V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn); + roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M, + V2_QPC_BYTE_80_RX_CQN_S, 0); + + roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M, + V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn); + roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M, + V2_QPC_BYTE_252_TX_CQN_S, 0); + + if (ibqp->srq) { + roce_set_bit(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_SRQ_EN_S, 1); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, + V2_QPC_BYTE_76_SRQ_EN_S, 0); + roce_set_field(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, + to_hr_srq(ibqp->srq)->srqn); + roce_set_field(qpc_mask->byte_76_srqn_op_en, + V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0); + } + + if (attr_mask & IB_QP_PKEY_INDEX) + context->qkey_xrcd = attr->pkey_index; + else + context->qkey_xrcd = hr_qp->pkey_index; + + roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M, + V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn); + roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M, + V2_QPC_BYTE_4_SQPN_S, 0); + + roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M, + V2_QPC_BYTE_56_DQPN_S, hr_qp->qpn); + roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M, + V2_QPC_BYTE_56_DQPN_S, 0); + roce_set_field(context->byte_168_irrl_idx, + V2_QPC_BYTE_168_SQ_SHIFT_BAK_M, + V2_QPC_BYTE_168_SQ_SHIFT_BAK_S, + ilog2((unsigned int)hr_qp->sq.wqe_cnt)); + roce_set_field(qpc_mask->byte_168_irrl_idx, + V2_QPC_BYTE_168_SQ_SHIFT_BAK_M, + V2_QPC_BYTE_168_SQ_SHIFT_BAK_S, 0); +} + +static int modify_qp_init_to_rtr(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, int attr_mask, + struct hns_roce_v2_qp_context *context, + struct hns_roce_v2_qp_context *qpc_mask) +{ + const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + struct device *dev = hr_dev->dev; + dma_addr_t dma_handle_3; + dma_addr_t dma_handle_2; + dma_addr_t dma_handle; + u32 page_size; + u8 port_num; + u64 *mtts_3; + u64 *mtts_2; + u64 *mtts; + u8 *dmac; + u8 *smac; + int port; + + /* Search qp buf's mtts */ + mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table, + hr_qp->mtt.first_seg, &dma_handle); + if (!mtts) { + dev_err(dev, "qp buf pa find failed\n"); + return -EINVAL; + } + + /* Search IRRL's mtts */ + mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table, + hr_qp->qpn, &dma_handle_2); + if (!mtts_2) { + dev_err(dev, "qp irrl_table find failed\n"); + return -EINVAL; + } + + /* Search TRRL's mtts */ + mtts_3 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table, + hr_qp->qpn, &dma_handle_3); + if (!mtts_3) { + dev_err(dev, "qp trrl_table find failed\n"); + return -EINVAL; + } + + if ((attr_mask & IB_QP_ALT_PATH) || (attr_mask & IB_QP_ACCESS_FLAGS) || + (attr_mask & IB_QP_PKEY_INDEX) || (attr_mask & IB_QP_QKEY)) { + dev_err(dev, "INIT2RTR attr_mask (0x%x) error\n", attr_mask); + return -EINVAL; + } + + dmac = (u8 *)attr->ah_attr.roce.dmac; + context->wqe_sge_ba = (u32)(dma_handle >> 3); + qpc_mask->wqe_sge_ba = 0; + + /* + * In v2 engine, software pass context and context mask to hardware + * when modifying qp. If software need modify some fields in context, + * we should set all bits of the relevant fields in context mask to + * 0 at the same time, else set them to 0x1. + */ + roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M, + V2_QPC_BYTE_12_WQE_SGE_BA_S, dma_handle >> (32 + 3)); + roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M, + V2_QPC_BYTE_12_WQE_SGE_BA_S, 0); + + roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M, + V2_QPC_BYTE_12_SQ_HOP_NUM_S, + hr_dev->caps.mtt_hop_num == HNS_ROCE_HOP_NUM_0 ? + 0 : hr_dev->caps.mtt_hop_num); + roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M, + V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0); + + roce_set_field(context->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_SGE_HOP_NUM_M, + V2_QPC_BYTE_20_SGE_HOP_NUM_S, + hr_qp->sq.max_gs > 2 ? hr_dev->caps.mtt_hop_num : 0); + roce_set_field(qpc_mask->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_SGE_HOP_NUM_M, + V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0); + + roce_set_field(context->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_RQ_HOP_NUM_M, + V2_QPC_BYTE_20_RQ_HOP_NUM_S, + hr_dev->caps.mtt_hop_num == HNS_ROCE_HOP_NUM_0 ? + 0 : hr_dev->caps.mtt_hop_num); + roce_set_field(qpc_mask->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_RQ_HOP_NUM_M, + V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0); + + roce_set_field(context->byte_16_buf_ba_pg_sz, + V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M, + V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, + hr_dev->caps.mtt_ba_pg_sz); + roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, + V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M, + V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0); + + roce_set_field(context->byte_16_buf_ba_pg_sz, + V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M, + V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, + hr_dev->caps.mtt_buf_pg_sz); + roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, + V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M, + V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0); + + roce_set_field(context->byte_80_rnr_rx_cqn, + V2_QPC_BYTE_80_MIN_RNR_TIME_M, + V2_QPC_BYTE_80_MIN_RNR_TIME_S, attr->min_rnr_timer); + roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, + V2_QPC_BYTE_80_MIN_RNR_TIME_M, + V2_QPC_BYTE_80_MIN_RNR_TIME_S, 0); + + page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); + context->rq_cur_blk_addr = (u32)(mtts[hr_qp->rq.offset / page_size] + >> PAGE_ADDR_SHIFT); + qpc_mask->rq_cur_blk_addr = 0; + + roce_set_field(context->byte_92_srq_info, + V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M, + V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, + mtts[hr_qp->rq.offset / page_size] + >> (32 + PAGE_ADDR_SHIFT)); + roce_set_field(qpc_mask->byte_92_srq_info, + V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M, + V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0); + + context->rq_nxt_blk_addr = (u32)(mtts[hr_qp->rq.offset / page_size + 1] + >> PAGE_ADDR_SHIFT); + qpc_mask->rq_nxt_blk_addr = 0; + + roce_set_field(context->byte_104_rq_sge, + V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M, + V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, + mtts[hr_qp->rq.offset / page_size + 1] + >> (32 + PAGE_ADDR_SHIFT)); + roce_set_field(qpc_mask->byte_104_rq_sge, + V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M, + V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0); + + roce_set_field(context->byte_108_rx_reqepsn, + V2_QPC_BYTE_108_RX_REQ_EPSN_M, + V2_QPC_BYTE_108_RX_REQ_EPSN_S, attr->rq_psn); + roce_set_field(qpc_mask->byte_108_rx_reqepsn, + V2_QPC_BYTE_108_RX_REQ_EPSN_M, + V2_QPC_BYTE_108_RX_REQ_EPSN_S, 0); + + roce_set_field(context->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M, + V2_QPC_BYTE_132_TRRL_BA_S, dma_handle_3 >> 4); + roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M, + V2_QPC_BYTE_132_TRRL_BA_S, 0); + context->trrl_ba = (u32)(dma_handle_3 >> (16 + 4)); + qpc_mask->trrl_ba = 0; + roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M, + V2_QPC_BYTE_140_TRRL_BA_S, + (u32)(dma_handle_3 >> (32 + 16 + 4))); + roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M, + V2_QPC_BYTE_140_TRRL_BA_S, 0); + + context->irrl_ba = (u32)(dma_handle_2 >> 6); + qpc_mask->irrl_ba = 0; + roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M, + V2_QPC_BYTE_208_IRRL_BA_S, + dma_handle_2 >> (32 + 6)); + roce_set_field(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M, + V2_QPC_BYTE_208_IRRL_BA_S, 0); + + roce_set_bit(context->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 1); + roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 0); + + roce_set_bit(context->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S, + hr_qp->sq_signal_bits); + roce_set_bit(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S, + 0); + + port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port; + + smac = (u8 *)hr_dev->dev_addr[port]; + /* when dmac equals smac or loop_idc is 1, it should loopback */ + if (ether_addr_equal_unaligned(dmac, smac) || + hr_dev->loop_idc == 0x1) { + roce_set_bit(context->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 1); + roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 0); + } + + roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M, + V2_QPC_BYTE_140_RR_MAX_S, + ilog2((unsigned int)attr->max_dest_rd_atomic)); + roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M, + V2_QPC_BYTE_140_RR_MAX_S, 0); + + roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M, + V2_QPC_BYTE_56_DQPN_S, attr->dest_qp_num); + roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M, + V2_QPC_BYTE_56_DQPN_S, 0); + + /* Configure GID index */ + port_num = rdma_ah_get_port_num(&attr->ah_attr); + roce_set_field(context->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_SGID_IDX_M, + V2_QPC_BYTE_20_SGID_IDX_S, + hns_get_gid_index(hr_dev, port_num - 1, + grh->sgid_index)); + roce_set_field(qpc_mask->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_SGID_IDX_M, + V2_QPC_BYTE_20_SGID_IDX_S, 0); + memcpy(&(context->dmac), dmac, 4); + roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M, + V2_QPC_BYTE_52_DMAC_S, *((u16 *)(&dmac[4]))); + qpc_mask->dmac = 0; + roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M, + V2_QPC_BYTE_52_DMAC_S, 0); + + roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M, + V2_QPC_BYTE_56_LP_PKTN_INI_S, 4); + roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M, + V2_QPC_BYTE_56_LP_PKTN_INI_S, 0); + + roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M, + V2_QPC_BYTE_24_HOP_LIMIT_S, grh->hop_limit); + roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M, + V2_QPC_BYTE_24_HOP_LIMIT_S, 0); + + roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M, + V2_QPC_BYTE_28_FL_S, grh->flow_label); + roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_FL_M, + V2_QPC_BYTE_28_FL_S, 0); + + roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M, + V2_QPC_BYTE_24_TC_S, grh->traffic_class); + roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M, + V2_QPC_BYTE_24_TC_S, 0); + + roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M, + V2_QPC_BYTE_24_MTU_S, attr->path_mtu); + roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M, + V2_QPC_BYTE_24_MTU_S, 0); + + memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw)); + memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw)); + + roce_set_field(context->byte_84_rq_ci_pi, + V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M, + V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, hr_qp->rq.head); + roce_set_field(qpc_mask->byte_84_rq_ci_pi, + V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M, + V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0); + + roce_set_field(qpc_mask->byte_84_rq_ci_pi, + V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M, + V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0); + roce_set_bit(qpc_mask->byte_108_rx_reqepsn, + V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0); + roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M, + V2_QPC_BYTE_96_RX_REQ_MSN_S, 0); + roce_set_field(qpc_mask->byte_108_rx_reqepsn, + V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M, + V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0); + + context->rq_rnr_timer = 0; + qpc_mask->rq_rnr_timer = 0; + + roce_set_field(context->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M, + V2_QPC_BYTE_152_RAQ_PSN_S, attr->rq_psn - 1); + roce_set_field(qpc_mask->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M, + V2_QPC_BYTE_152_RAQ_PSN_S, 0); + + roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M, + V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0); + roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M, + V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0); + + roce_set_field(context->byte_168_irrl_idx, + V2_QPC_BYTE_168_LP_SGEN_INI_M, + V2_QPC_BYTE_168_LP_SGEN_INI_S, 3); + roce_set_field(qpc_mask->byte_168_irrl_idx, + V2_QPC_BYTE_168_LP_SGEN_INI_M, + V2_QPC_BYTE_168_LP_SGEN_INI_S, 0); + + roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_SR_MAX_M, + V2_QPC_BYTE_208_SR_MAX_S, + ilog2((unsigned int)attr->max_rd_atomic)); + roce_set_field(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_SR_MAX_M, + V2_QPC_BYTE_208_SR_MAX_S, 0); + + roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_SL_M, + V2_QPC_BYTE_28_SL_S, rdma_ah_get_sl(&attr->ah_attr)); + roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_SL_M, + V2_QPC_BYTE_28_SL_S, 0); + hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); + + return 0; +} + +static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, int attr_mask, + struct hns_roce_v2_qp_context *context, + struct hns_roce_v2_qp_context *qpc_mask) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + struct device *dev = hr_dev->dev; + dma_addr_t dma_handle; + u32 page_size; + u64 *mtts; + + /* Search qp buf's mtts */ + mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table, + hr_qp->mtt.first_seg, &dma_handle); + if (!mtts) { + dev_err(dev, "qp buf pa find failed\n"); + return -EINVAL; + } + + /* If exist optional param, return error */ + if ((attr_mask & IB_QP_ALT_PATH) || (attr_mask & IB_QP_ACCESS_FLAGS) || + (attr_mask & IB_QP_QKEY) || (attr_mask & IB_QP_PATH_MIG_STATE) || + (attr_mask & IB_QP_CUR_STATE) || + (attr_mask & IB_QP_MIN_RNR_TIMER)) { + dev_err(dev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask); + return -EINVAL; + } + + /* + * In v2 engine, software pass context and context mask to hardware + * when modifying qp. If software need modify some fields in context, + * we should set all bits of the relevant fields in context mask to + * 0 at the same time, else set them to 0x1. + */ + roce_set_field(context->byte_60_qpst_mapid, + V2_QPC_BYTE_60_RTY_NUM_INI_BAK_M, + V2_QPC_BYTE_60_RTY_NUM_INI_BAK_S, attr->retry_cnt); + roce_set_field(qpc_mask->byte_60_qpst_mapid, + V2_QPC_BYTE_60_RTY_NUM_INI_BAK_M, + V2_QPC_BYTE_60_RTY_NUM_INI_BAK_S, 0); + + context->sq_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT); + roce_set_field(context->byte_168_irrl_idx, + V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M, + V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, + mtts[0] >> (32 + PAGE_ADDR_SHIFT)); + qpc_mask->sq_cur_blk_addr = 0; + roce_set_field(qpc_mask->byte_168_irrl_idx, + V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M, + V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0); + + page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); + context->sq_cur_sge_blk_addr = hr_qp->sq.max_gs > 2 ? + ((u32)(mtts[hr_qp->sge.offset / page_size] + >> PAGE_ADDR_SHIFT)) : 0; + roce_set_field(context->byte_184_irrl_idx, + V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M, + V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, + hr_qp->sq.max_gs > 2 ? + (mtts[hr_qp->sge.offset / page_size] >> + (32 + PAGE_ADDR_SHIFT)) : 0); + qpc_mask->sq_cur_sge_blk_addr = 0; + roce_set_field(qpc_mask->byte_184_irrl_idx, + V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M, + V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 0); + + context->rx_sq_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT); + roce_set_field(context->byte_232_irrl_sge, + V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M, + V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, + mtts[0] >> (32 + PAGE_ADDR_SHIFT)); + qpc_mask->rx_sq_cur_blk_addr = 0; + roce_set_field(qpc_mask->byte_232_irrl_sge, + V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M, + V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, 0); + + /* + * Set some fields in context to zero, Because the default values + * of all fields in context are zero, we need not set them to 0 again. + * but we should set the relevant fields of context mask to 0. + */ + roce_set_field(qpc_mask->byte_232_irrl_sge, + V2_QPC_BYTE_232_IRRL_SGE_IDX_M, + V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0); + + roce_set_field(qpc_mask->byte_240_irrl_tail, + V2_QPC_BYTE_240_RX_ACK_MSN_M, + V2_QPC_BYTE_240_RX_ACK_MSN_S, 0); + + roce_set_field(context->byte_244_rnr_rxack, + V2_QPC_BYTE_244_RX_ACK_EPSN_M, + V2_QPC_BYTE_244_RX_ACK_EPSN_S, attr->sq_psn); + roce_set_field(qpc_mask->byte_244_rnr_rxack, + V2_QPC_BYTE_244_RX_ACK_EPSN_M, + V2_QPC_BYTE_244_RX_ACK_EPSN_S, 0); + + roce_set_field(qpc_mask->byte_248_ack_psn, + V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M, + V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0); + roce_set_bit(qpc_mask->byte_248_ack_psn, + V2_QPC_BYTE_248_IRRL_PSN_VLD_S, 0); + roce_set_field(qpc_mask->byte_248_ack_psn, + V2_QPC_BYTE_248_IRRL_PSN_M, + V2_QPC_BYTE_248_IRRL_PSN_S, 0); + + roce_set_field(qpc_mask->byte_240_irrl_tail, + V2_QPC_BYTE_240_IRRL_TAIL_REAL_M, + V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0); + + roce_set_field(context->byte_220_retry_psn_msn, + V2_QPC_BYTE_220_RETRY_MSG_PSN_M, + V2_QPC_BYTE_220_RETRY_MSG_PSN_S, attr->sq_psn); + roce_set_field(qpc_mask->byte_220_retry_psn_msn, + V2_QPC_BYTE_220_RETRY_MSG_PSN_M, + V2_QPC_BYTE_220_RETRY_MSG_PSN_S, 0); + + roce_set_field(context->byte_224_retry_msg, + V2_QPC_BYTE_224_RETRY_MSG_PSN_M, + V2_QPC_BYTE_224_RETRY_MSG_PSN_S, attr->sq_psn >> 16); + roce_set_field(qpc_mask->byte_224_retry_msg, + V2_QPC_BYTE_224_RETRY_MSG_PSN_M, + V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 0); + + roce_set_field(context->byte_224_retry_msg, + V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M, + V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, attr->sq_psn); + roce_set_field(qpc_mask->byte_224_retry_msg, + V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M, + V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, 0); + + roce_set_field(qpc_mask->byte_220_retry_psn_msn, + V2_QPC_BYTE_220_RETRY_MSG_MSN_M, + V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0); + + roce_set_bit(qpc_mask->byte_248_ack_psn, + V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0); + + roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M, + V2_QPC_BYTE_212_CHECK_FLG_S, 0); + + roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_RETRY_CNT_M, + V2_QPC_BYTE_212_RETRY_CNT_S, attr->retry_cnt); + roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_RETRY_CNT_M, + V2_QPC_BYTE_212_RETRY_CNT_S, 0); + + roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_RETRY_NUM_INIT_M, + V2_QPC_BYTE_212_RETRY_NUM_INIT_S, attr->retry_cnt); + roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_RETRY_NUM_INIT_M, + V2_QPC_BYTE_212_RETRY_NUM_INIT_S, 0); + + roce_set_field(context->byte_244_rnr_rxack, + V2_QPC_BYTE_244_RNR_NUM_INIT_M, + V2_QPC_BYTE_244_RNR_NUM_INIT_S, attr->rnr_retry); + roce_set_field(qpc_mask->byte_244_rnr_rxack, + V2_QPC_BYTE_244_RNR_NUM_INIT_M, + V2_QPC_BYTE_244_RNR_NUM_INIT_S, 0); + + roce_set_field(context->byte_244_rnr_rxack, V2_QPC_BYTE_244_RNR_CNT_M, + V2_QPC_BYTE_244_RNR_CNT_S, attr->rnr_retry); + roce_set_field(qpc_mask->byte_244_rnr_rxack, V2_QPC_BYTE_244_RNR_CNT_M, + V2_QPC_BYTE_244_RNR_CNT_S, 0); + + roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_LSN_M, + V2_QPC_BYTE_212_LSN_S, 0x100); + roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_LSN_M, + V2_QPC_BYTE_212_LSN_S, 0); + + if (attr_mask & IB_QP_TIMEOUT) { + roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_AT_M, + V2_QPC_BYTE_28_AT_S, attr->timeout); + roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_AT_M, + V2_QPC_BYTE_28_AT_S, 0); + } + + roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_SL_M, + V2_QPC_BYTE_28_SL_S, + rdma_ah_get_sl(&attr->ah_attr)); + roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_SL_M, + V2_QPC_BYTE_28_SL_S, 0); + hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); + + roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_SQ_CUR_PSN_M, + V2_QPC_BYTE_172_SQ_CUR_PSN_S, attr->sq_psn); + roce_set_field(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_SQ_CUR_PSN_M, + V2_QPC_BYTE_172_SQ_CUR_PSN_S, 0); + + roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M, + V2_QPC_BYTE_196_IRRL_HEAD_S, 0); + roce_set_field(context->byte_196_sq_psn, V2_QPC_BYTE_196_SQ_MAX_PSN_M, + V2_QPC_BYTE_196_SQ_MAX_PSN_S, attr->sq_psn); + roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_SQ_MAX_PSN_M, + V2_QPC_BYTE_196_SQ_MAX_PSN_S, 0); + + return 0; +} + +static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, + int attr_mask, enum ib_qp_state cur_state, + enum ib_qp_state new_state) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + struct hns_roce_v2_qp_context *context; + struct hns_roce_v2_qp_context *qpc_mask; + struct device *dev = hr_dev->dev; + int ret = -EINVAL; + + context = kzalloc(2 * sizeof(*context), GFP_KERNEL); + if (!context) + return -ENOMEM; + + qpc_mask = context + 1; + /* + * In v2 engine, software pass context and context mask to hardware + * when modifying qp. If software need modify some fields in context, + * we should set all bits of the relevant fields in context mask to + * 0 at the same time, else set them to 0x1. + */ + memset(qpc_mask, 0xff, sizeof(*qpc_mask)); + if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { + modify_qp_reset_to_init(ibqp, attr, context, qpc_mask); + } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) { + modify_qp_init_to_init(ibqp, attr, attr_mask, context, + qpc_mask); + } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) { + ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context, + qpc_mask); + if (ret) + goto out; + } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) { + ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context, + qpc_mask); + if (ret) + goto out; + } else if ((cur_state == IB_QPS_RTS && new_state == IB_QPS_RTS) || + (cur_state == IB_QPS_SQE && new_state == IB_QPS_RTS) || + (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD) || + (cur_state == IB_QPS_SQD && new_state == IB_QPS_SQD) || + (cur_state == IB_QPS_SQD && new_state == IB_QPS_RTS) || + (cur_state == IB_QPS_INIT && new_state == IB_QPS_RESET) || + (cur_state == IB_QPS_RTR && new_state == IB_QPS_RESET) || + (cur_state == IB_QPS_RTS && new_state == IB_QPS_RESET) || + (cur_state == IB_QPS_ERR && new_state == IB_QPS_RESET) || + (cur_state == IB_QPS_INIT && new_state == IB_QPS_ERR) || + (cur_state == IB_QPS_RTR && new_state == IB_QPS_ERR) || + (cur_state == IB_QPS_RTS && new_state == IB_QPS_ERR) || + (cur_state == IB_QPS_SQD && new_state == IB_QPS_ERR) || + (cur_state == IB_QPS_SQE && new_state == IB_QPS_ERR)) { + /* Nothing */ + ; + } else { + dev_err(dev, "Illegal state for QP!\n"); + goto out; + } + + /* Every status migrate must change state */ + roce_set_field(context->byte_60_qpst_mapid, V2_QPC_BYTE_60_QP_ST_M, + V2_QPC_BYTE_60_QP_ST_S, new_state); + roce_set_field(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_QP_ST_M, + V2_QPC_BYTE_60_QP_ST_S, 0); + + /* SW pass context to HW */ + ret = hns_roce_v2_qp_modify(hr_dev, &hr_qp->mtt, cur_state, new_state, + context, hr_qp); + if (ret) { + dev_err(dev, "hns_roce_qp_modify failed(%d)\n", ret); + goto out; + } + + hr_qp->state = new_state; + + if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) + hr_qp->resp_depth = attr->max_dest_rd_atomic; + if (attr_mask & IB_QP_PORT) { + hr_qp->port = attr->port_num - 1; + hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; + } + + if (new_state == IB_QPS_RESET && !ibqp->uobject) { + hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn, + ibqp->srq ? to_hr_srq(ibqp->srq) : NULL); + if (ibqp->send_cq != ibqp->recv_cq) + hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq), + hr_qp->qpn, NULL); + + hr_qp->rq.head = 0; + hr_qp->rq.tail = 0; + hr_qp->sq.head = 0; + hr_qp->sq.tail = 0; + hr_qp->sq_next_wqe = 0; + hr_qp->next_sge = 0; + } + +out: + kfree(context); + return ret; +} + +static inline enum ib_qp_state to_ib_qp_st(enum hns_roce_v2_qp_state state) +{ + switch (state) { + case HNS_ROCE_QP_ST_RST: return IB_QPS_RESET; + case HNS_ROCE_QP_ST_INIT: return IB_QPS_INIT; + case HNS_ROCE_QP_ST_RTR: return IB_QPS_RTR; + case HNS_ROCE_QP_ST_RTS: return IB_QPS_RTS; + case HNS_ROCE_QP_ST_SQ_DRAINING: + case HNS_ROCE_QP_ST_SQD: return IB_QPS_SQD; + case HNS_ROCE_QP_ST_SQER: return IB_QPS_SQE; + case HNS_ROCE_QP_ST_ERR: return IB_QPS_ERR; + default: return -1; + } +} + +static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct hns_roce_v2_qp_context *hr_context) +{ + struct hns_roce_cmd_mailbox *mailbox; + int ret; + + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0, + HNS_ROCE_CMD_QUERY_QPC, + HNS_ROCE_CMD_TIMEOUT_MSECS); + if (ret) { + dev_err(hr_dev->dev, "QUERY QP cmd process error\n"); + goto out; + } + + memcpy(hr_context, mailbox->buf, sizeof(*hr_context)); + +out: + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + return ret; +} + +static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, + struct ib_qp_init_attr *qp_init_attr) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + struct hns_roce_v2_qp_context *context; + struct device *dev = hr_dev->dev; + int tmp_qp_state; + int state; + int ret; + + context = kzalloc(sizeof(*context), GFP_KERNEL); + if (!context) + return -ENOMEM; + + memset(qp_attr, 0, sizeof(*qp_attr)); + memset(qp_init_attr, 0, sizeof(*qp_init_attr)); + + mutex_lock(&hr_qp->mutex); + + if (hr_qp->state == IB_QPS_RESET) { + qp_attr->qp_state = IB_QPS_RESET; + ret = 0; + goto done; + } + + ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, context); + if (ret) { + dev_err(dev, "query qpc error\n"); + ret = -EINVAL; + goto out; + } + + state = roce_get_field(context->byte_60_qpst_mapid, + V2_QPC_BYTE_60_QP_ST_M, V2_QPC_BYTE_60_QP_ST_S); + tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state); + if (tmp_qp_state == -1) { + dev_err(dev, "Illegal ib_qp_state\n"); + ret = -EINVAL; + goto out; + } + hr_qp->state = (u8)tmp_qp_state; + qp_attr->qp_state = (enum ib_qp_state)hr_qp->state; + qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context->byte_24_mtu_tc, + V2_QPC_BYTE_24_MTU_M, + V2_QPC_BYTE_24_MTU_S); + qp_attr->path_mig_state = IB_MIG_ARMED; + qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; + if (hr_qp->ibqp.qp_type == IB_QPT_UD) + qp_attr->qkey = V2_QKEY_VAL; + + qp_attr->rq_psn = roce_get_field(context->byte_108_rx_reqepsn, + V2_QPC_BYTE_108_RX_REQ_EPSN_M, + V2_QPC_BYTE_108_RX_REQ_EPSN_S); + qp_attr->sq_psn = (u32)roce_get_field(context->byte_172_sq_psn, + V2_QPC_BYTE_172_SQ_CUR_PSN_M, + V2_QPC_BYTE_172_SQ_CUR_PSN_S); + qp_attr->dest_qp_num = (u8)roce_get_field(context->byte_56_dqpn_err, + V2_QPC_BYTE_56_DQPN_M, + V2_QPC_BYTE_56_DQPN_S); + qp_attr->qp_access_flags = ((roce_get_bit(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_RRE_S)) << 2) | + ((roce_get_bit(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_RWE_S)) << 1) | + ((roce_get_bit(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_ATE_S)) << 3); + if (hr_qp->ibqp.qp_type == IB_QPT_RC || + hr_qp->ibqp.qp_type == IB_QPT_UC) { + struct ib_global_route *grh = + rdma_ah_retrieve_grh(&qp_attr->ah_attr); + + rdma_ah_set_sl(&qp_attr->ah_attr, + roce_get_field(context->byte_28_at_fl, + V2_QPC_BYTE_28_SL_M, + V2_QPC_BYTE_28_SL_S)); + grh->flow_label = roce_get_field(context->byte_28_at_fl, + V2_QPC_BYTE_28_FL_M, + V2_QPC_BYTE_28_FL_S); + grh->sgid_index = roce_get_field(context->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_SGID_IDX_M, + V2_QPC_BYTE_20_SGID_IDX_S); + grh->hop_limit = roce_get_field(context->byte_24_mtu_tc, + V2_QPC_BYTE_24_HOP_LIMIT_M, + V2_QPC_BYTE_24_HOP_LIMIT_S); + grh->traffic_class = roce_get_field(context->byte_24_mtu_tc, + V2_QPC_BYTE_24_TC_M, + V2_QPC_BYTE_24_TC_S); + + memcpy(grh->dgid.raw, context->dgid, sizeof(grh->dgid.raw)); + } + + qp_attr->port_num = hr_qp->port + 1; + qp_attr->sq_draining = 0; + qp_attr->max_rd_atomic = 1 << roce_get_field(context->byte_208_irrl, + V2_QPC_BYTE_208_SR_MAX_M, + V2_QPC_BYTE_208_SR_MAX_S); + qp_attr->max_dest_rd_atomic = 1 << roce_get_field(context->byte_140_raq, + V2_QPC_BYTE_140_RR_MAX_M, + V2_QPC_BYTE_140_RR_MAX_S); + qp_attr->min_rnr_timer = (u8)roce_get_field(context->byte_80_rnr_rx_cqn, + V2_QPC_BYTE_80_MIN_RNR_TIME_M, + V2_QPC_BYTE_80_MIN_RNR_TIME_S); + qp_attr->timeout = (u8)roce_get_field(context->byte_28_at_fl, + V2_QPC_BYTE_28_AT_M, + V2_QPC_BYTE_28_AT_S); + qp_attr->retry_cnt = roce_get_field(context->byte_212_lsn, + V2_QPC_BYTE_212_RETRY_CNT_M, + V2_QPC_BYTE_212_RETRY_CNT_S); + qp_attr->rnr_retry = context->rq_rnr_timer; + +done: + qp_attr->cur_qp_state = qp_attr->qp_state; + qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt; + qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs; + + if (!ibqp->uobject) { + qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt; + qp_attr->cap.max_send_sge = hr_qp->sq.max_gs; + } else { + qp_attr->cap.max_send_wr = 0; + qp_attr->cap.max_send_sge = 0; + } + + qp_init_attr->cap = qp_attr->cap; + +out: + mutex_unlock(&hr_qp->mutex); + kfree(context); + return ret; +} + +static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + int is_user) +{ + struct hns_roce_cq *send_cq, *recv_cq; + struct device *dev = hr_dev->dev; + int ret; + + if (hr_qp->ibqp.qp_type == IB_QPT_RC && hr_qp->state != IB_QPS_RESET) { + /* Modify qp to reset before destroying qp */ + ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0, + hr_qp->state, IB_QPS_RESET); + if (ret) { + dev_err(dev, "modify QP %06lx to ERR failed.\n", + hr_qp->qpn); + return ret; + } + } + + send_cq = to_hr_cq(hr_qp->ibqp.send_cq); + recv_cq = to_hr_cq(hr_qp->ibqp.recv_cq); + + hns_roce_lock_cqs(send_cq, recv_cq); + + if (!is_user) { + __hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn, hr_qp->ibqp.srq ? + to_hr_srq(hr_qp->ibqp.srq) : NULL); + if (send_cq != recv_cq) + __hns_roce_v2_cq_clean(send_cq, hr_qp->qpn, NULL); + } + + hns_roce_qp_remove(hr_dev, hr_qp); + + hns_roce_unlock_cqs(send_cq, recv_cq); + + hns_roce_qp_free(hr_dev, hr_qp); + + /* Not special_QP, free their QPN */ + if ((hr_qp->ibqp.qp_type == IB_QPT_RC) || + (hr_qp->ibqp.qp_type == IB_QPT_UC) || + (hr_qp->ibqp.qp_type == IB_QPT_UD)) + hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1); + + hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt); + + if (is_user) { + ib_umem_release(hr_qp->umem); + } else { + kfree(hr_qp->sq.wrid); + kfree(hr_qp->rq.wrid); + hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf); + } + + return 0; +} + +static int hns_roce_v2_destroy_qp(struct ib_qp *ibqp) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + int ret; + + ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, !!ibqp->pd->uobject); + if (ret) { + dev_err(hr_dev->dev, "Destroy qp failed(%d)\n", ret); + return ret; + } + + if (hr_qp->ibqp.qp_type == IB_QPT_GSI) + kfree(hr_to_hr_sqp(hr_qp)); + else + kfree(hr_qp); + + return 0; +} + +static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(cq->device); + struct hns_roce_v2_cq_context *cq_context; + struct hns_roce_cq *hr_cq = to_hr_cq(cq); + struct hns_roce_v2_cq_context *cqc_mask; + struct hns_roce_cmd_mailbox *mailbox; + int ret; + + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + cq_context = mailbox->buf; + cqc_mask = (struct hns_roce_v2_cq_context *)mailbox->buf + 1; + + memset(cqc_mask, 0xff, sizeof(*cqc_mask)); + + roce_set_field(cq_context->byte_56_cqe_period_maxcnt, + V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S, + cq_count); + roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt, + V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S, + 0); + roce_set_field(cq_context->byte_56_cqe_period_maxcnt, + V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S, + cq_period); + roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt, + V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S, + 0); + + ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_cq->cqn, 1, + HNS_ROCE_CMD_MODIFY_CQC, + HNS_ROCE_CMD_TIMEOUT_MSECS); + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + if (ret) + dev_err(hr_dev->dev, "MODIFY CQ Failed to cmd mailbox.\n"); + + return ret; +} + +static const struct hns_roce_hw hns_roce_hw_v2 = { + .cmq_init = hns_roce_v2_cmq_init, + .cmq_exit = hns_roce_v2_cmq_exit, + .hw_profile = hns_roce_v2_profile, + .post_mbox = hns_roce_v2_post_mbox, + .chk_mbox = hns_roce_v2_chk_mbox, + .set_gid = hns_roce_v2_set_gid, + .set_mac = hns_roce_v2_set_mac, + .write_mtpt = hns_roce_v2_write_mtpt, + .rereg_write_mtpt = hns_roce_v2_rereg_write_mtpt, + .write_cqc = hns_roce_v2_write_cqc, + .set_hem = hns_roce_v2_set_hem, + .clear_hem = hns_roce_v2_clear_hem, + .modify_qp = hns_roce_v2_modify_qp, + .query_qp = hns_roce_v2_query_qp, + .destroy_qp = hns_roce_v2_destroy_qp, + .modify_cq = hns_roce_v2_modify_cq, + .post_send = hns_roce_v2_post_send, + .post_recv = hns_roce_v2_post_recv, + .req_notify_cq = hns_roce_v2_req_notify_cq, + .poll_cq = hns_roce_v2_poll_cq, +}; + +static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = { + {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0}, + {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0}, + {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0}, + /* required last entry */ + {0, } +}; + +static int hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev, + struct hnae3_handle *handle) +{ + const struct pci_device_id *id; + + id = pci_match_id(hns_roce_hw_v2_pci_tbl, hr_dev->pci_dev); + if (!id) { + dev_err(hr_dev->dev, "device is not compatible!\n"); + return -ENXIO; + } + + hr_dev->hw = &hns_roce_hw_v2; + hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG; + hr_dev->odb_offset = hr_dev->sdb_offset; + + /* Get info from NIC driver. */ + hr_dev->reg_base = handle->rinfo.roce_io_base; + hr_dev->caps.num_ports = 1; + hr_dev->iboe.netdevs[0] = handle->rinfo.netdev; + hr_dev->iboe.phy_port[0] = 0; + + /* cmd issue mode: 0 is poll, 1 is event */ + hr_dev->cmd_mod = 0; + hr_dev->loop_idc = 0; + + return 0; +} + +static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle) +{ + struct hns_roce_dev *hr_dev; + int ret; + + hr_dev = (struct hns_roce_dev *)ib_alloc_device(sizeof(*hr_dev)); + if (!hr_dev) + return -ENOMEM; + + hr_dev->priv = kzalloc(sizeof(struct hns_roce_v2_priv), GFP_KERNEL); + if (!hr_dev->priv) { + ret = -ENOMEM; + goto error_failed_kzalloc; + } + + hr_dev->pci_dev = handle->pdev; + hr_dev->dev = &handle->pdev->dev; + handle->priv = hr_dev; + + ret = hns_roce_hw_v2_get_cfg(hr_dev, handle); + if (ret) { + dev_err(hr_dev->dev, "Get Configuration failed!\n"); + goto error_failed_get_cfg; + } + + ret = hns_roce_init(hr_dev); + if (ret) { + dev_err(hr_dev->dev, "RoCE Engine init failed!\n"); + goto error_failed_get_cfg; + } + + return 0; + +error_failed_get_cfg: + kfree(hr_dev->priv); + +error_failed_kzalloc: + ib_dealloc_device(&hr_dev->ib_dev); + + return ret; +} + +static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, + bool reset) +{ + struct hns_roce_dev *hr_dev = (struct hns_roce_dev *)handle->priv; + + hns_roce_exit(hr_dev); + kfree(hr_dev->priv); + ib_dealloc_device(&hr_dev->ib_dev); +} + +static const struct hnae3_client_ops hns_roce_hw_v2_ops = { + .init_instance = hns_roce_hw_v2_init_instance, + .uninit_instance = hns_roce_hw_v2_uninit_instance, +}; + +static struct hnae3_client hns_roce_hw_v2_client = { + .name = "hns_roce_hw_v2", + .type = HNAE3_CLIENT_ROCE, + .ops = &hns_roce_hw_v2_ops, +}; + +static int __init hns_roce_hw_v2_init(void) +{ + return hnae3_register_client(&hns_roce_hw_v2_client); +} + +static void __exit hns_roce_hw_v2_exit(void) +{ + hnae3_unregister_client(&hns_roce_hw_v2_client); +} + +module_init(hns_roce_hw_v2_init); +module_exit(hns_roce_hw_v2_exit); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>"); +MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>"); +MODULE_AUTHOR("Shaobo Xu <xushaobo2@huawei.com>"); +MODULE_DESCRIPTION("Hisilicon Hip08 Family RoCE Driver"); diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h new file mode 100644 index 000000000000..04b7a51b8efb --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -0,0 +1,1177 @@ +/* + * Copyright (c) 2016-2017 Hisilicon Limited. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _HNS_ROCE_HW_V2_H +#define _HNS_ROCE_HW_V2_H + +#include <linux/bitops.h> + +#define HNS_ROCE_VF_QPC_BT_NUM 256 +#define HNS_ROCE_VF_SRQC_BT_NUM 64 +#define HNS_ROCE_VF_CQC_BT_NUM 64 +#define HNS_ROCE_VF_MPT_BT_NUM 64 +#define HNS_ROCE_VF_EQC_NUM 64 +#define HNS_ROCE_VF_SMAC_NUM 32 +#define HNS_ROCE_VF_SGID_NUM 32 +#define HNS_ROCE_VF_SL_NUM 8 + +#define HNS_ROCE_V2_MAX_QP_NUM 0x2000 +#define HNS_ROCE_V2_MAX_WQE_NUM 0x8000 +#define HNS_ROCE_V2_MAX_CQ_NUM 0x8000 +#define HNS_ROCE_V2_MAX_CQE_NUM 0x10000 +#define HNS_ROCE_V2_MAX_RQ_SGE_NUM 0x100 +#define HNS_ROCE_V2_MAX_SQ_SGE_NUM 0xff +#define HNS_ROCE_V2_MAX_SQ_INLINE 0x20 +#define HNS_ROCE_V2_UAR_NUM 256 +#define HNS_ROCE_V2_PHY_UAR_NUM 1 +#define HNS_ROCE_V2_MAX_MTPT_NUM 0x8000 +#define HNS_ROCE_V2_MAX_MTT_SEGS 0x1000000 +#define HNS_ROCE_V2_MAX_CQE_SEGS 0x1000000 +#define HNS_ROCE_V2_MAX_PD_NUM 0x1000000 +#define HNS_ROCE_V2_MAX_QP_INIT_RDMA 128 +#define HNS_ROCE_V2_MAX_QP_DEST_RDMA 128 +#define HNS_ROCE_V2_MAX_SQ_DESC_SZ 64 +#define HNS_ROCE_V2_MAX_RQ_DESC_SZ 16 +#define HNS_ROCE_V2_MAX_SRQ_DESC_SZ 64 +#define HNS_ROCE_V2_QPC_ENTRY_SZ 256 +#define HNS_ROCE_V2_IRRL_ENTRY_SZ 64 +#define HNS_ROCE_V2_TRRL_ENTRY_SZ 48 +#define HNS_ROCE_V2_CQC_ENTRY_SZ 64 +#define HNS_ROCE_V2_MTPT_ENTRY_SZ 64 +#define HNS_ROCE_V2_MTT_ENTRY_SZ 64 +#define HNS_ROCE_V2_CQE_ENTRY_SIZE 32 +#define HNS_ROCE_V2_PAGE_SIZE_SUPPORTED 0xFFFFF000 +#define HNS_ROCE_V2_MAX_INNER_MTPT_NUM 2 +#define HNS_ROCE_INVALID_LKEY 0x100 +#define HNS_ROCE_CMQ_TX_TIMEOUT 200 + +#define HNS_ROCE_CONTEXT_HOP_NUM 1 +#define HNS_ROCE_MTT_HOP_NUM 1 +#define HNS_ROCE_CQE_HOP_NUM 1 +#define HNS_ROCE_PBL_HOP_NUM 2 +#define HNS_ROCE_V2_GID_INDEX_NUM 256 + +#define HNS_ROCE_V2_TABLE_CHUNK_SIZE (1 << 18) + +#define HNS_ROCE_CMD_FLAG_IN_VALID_SHIFT 0 +#define HNS_ROCE_CMD_FLAG_OUT_VALID_SHIFT 1 +#define HNS_ROCE_CMD_FLAG_NEXT_SHIFT 2 +#define HNS_ROCE_CMD_FLAG_WR_OR_RD_SHIFT 3 +#define HNS_ROCE_CMD_FLAG_NO_INTR_SHIFT 4 +#define HNS_ROCE_CMD_FLAG_ERR_INTR_SHIFT 5 + +#define HNS_ROCE_CMD_FLAG_IN BIT(HNS_ROCE_CMD_FLAG_IN_VALID_SHIFT) +#define HNS_ROCE_CMD_FLAG_OUT BIT(HNS_ROCE_CMD_FLAG_OUT_VALID_SHIFT) +#define HNS_ROCE_CMD_FLAG_NEXT BIT(HNS_ROCE_CMD_FLAG_NEXT_SHIFT) +#define HNS_ROCE_CMD_FLAG_WR BIT(HNS_ROCE_CMD_FLAG_WR_OR_RD_SHIFT) +#define HNS_ROCE_CMD_FLAG_NO_INTR BIT(HNS_ROCE_CMD_FLAG_NO_INTR_SHIFT) +#define HNS_ROCE_CMD_FLAG_ERR_INTR BIT(HNS_ROCE_CMD_FLAG_ERR_INTR_SHIFT) + +#define HNS_ROCE_CMQ_DESC_NUM_S 3 +#define HNS_ROCE_CMQ_EN_B 16 +#define HNS_ROCE_CMQ_ENABLE BIT(HNS_ROCE_CMQ_EN_B) + +#define check_whether_last_step(hop_num, step_idx) \ + ((step_idx == 0 && hop_num == HNS_ROCE_HOP_NUM_0) || \ + (step_idx == 1 && hop_num == 1) || \ + (step_idx == 2 && hop_num == 2)) + +#define V2_CQ_DB_REQ_NOT_SOL 0 +#define V2_CQ_DB_REQ_NOT 1 + +#define V2_CQ_STATE_VALID 1 +#define V2_QKEY_VAL 0x80010000 + +#define GID_LEN_V2 16 + +#define HNS_ROCE_V2_CQE_QPN_MASK 0x3ffff + +enum { + HNS_ROCE_V2_WQE_OP_SEND = 0x0, + HNS_ROCE_V2_WQE_OP_SEND_WITH_INV = 0x1, + HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM = 0x2, + HNS_ROCE_V2_WQE_OP_RDMA_WRITE = 0x3, + HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM = 0x4, + HNS_ROCE_V2_WQE_OP_RDMA_READ = 0x5, + HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP = 0x6, + HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD = 0x7, + HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP = 0x8, + HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD = 0x9, + HNS_ROCE_V2_WQE_OP_FAST_REG_PMR = 0xa, + HNS_ROCE_V2_WQE_OP_LOCAL_INV = 0xb, + HNS_ROCE_V2_WQE_OP_BIND_MW_TYPE = 0xc, + HNS_ROCE_V2_WQE_OP_MASK = 0x1f, +}; + +enum { + HNS_ROCE_SQ_OPCODE_SEND = 0x0, + HNS_ROCE_SQ_OPCODE_SEND_WITH_INV = 0x1, + HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM = 0x2, + HNS_ROCE_SQ_OPCODE_RDMA_WRITE = 0x3, + HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM = 0x4, + HNS_ROCE_SQ_OPCODE_RDMA_READ = 0x5, + HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP = 0x6, + HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD = 0x7, + HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP = 0x8, + HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD = 0x9, + HNS_ROCE_SQ_OPCODE_FAST_REG_WR = 0xa, + HNS_ROCE_SQ_OPCODE_LOCAL_INV = 0xb, + HNS_ROCE_SQ_OPCODE_BIND_MW = 0xc, +}; + +enum { + /* rq operations */ + HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM = 0x0, + HNS_ROCE_V2_OPCODE_SEND = 0x1, + HNS_ROCE_V2_OPCODE_SEND_WITH_IMM = 0x2, + HNS_ROCE_V2_OPCODE_SEND_WITH_INV = 0x3, +}; + +enum { + HNS_ROCE_V2_SQ_DB = 0x0, + HNS_ROCE_V2_RQ_DB = 0x1, + HNS_ROCE_V2_SRQ_DB = 0x2, + HNS_ROCE_V2_CQ_DB_PTR = 0x3, + HNS_ROCE_V2_CQ_DB_NTR = 0x4, +}; + +enum { + HNS_ROCE_CQE_V2_SUCCESS = 0x00, + HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR = 0x01, + HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR = 0x02, + HNS_ROCE_CQE_V2_LOCAL_PROT_ERR = 0x04, + HNS_ROCE_CQE_V2_WR_FLUSH_ERR = 0x05, + HNS_ROCE_CQE_V2_MW_BIND_ERR = 0x06, + HNS_ROCE_CQE_V2_BAD_RESP_ERR = 0x10, + HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR = 0x11, + HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR = 0x12, + HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR = 0x13, + HNS_ROCE_CQE_V2_REMOTE_OP_ERR = 0x14, + HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR = 0x15, + HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR = 0x16, + HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR = 0x22, + + HNS_ROCE_V2_CQE_STATUS_MASK = 0xff, +}; + +/* CMQ command */ +enum hns_roce_opcode_type { + HNS_ROCE_OPC_QUERY_HW_VER = 0x8000, + HNS_ROCE_OPC_CFG_GLOBAL_PARAM = 0x8001, + HNS_ROCE_OPC_ALLOC_PF_RES = 0x8004, + HNS_ROCE_OPC_QUERY_PF_RES = 0x8400, + HNS_ROCE_OPC_ALLOC_VF_RES = 0x8401, + HNS_ROCE_OPC_CFG_BT_ATTR = 0x8506, +}; + +enum { + TYPE_CRQ, + TYPE_CSQ, +}; + +enum hns_roce_cmd_return_status { + CMD_EXEC_SUCCESS = 0, + CMD_NO_AUTH = 1, + CMD_NOT_EXEC = 2, + CMD_QUEUE_FULL = 3, +}; + +enum hns_roce_sgid_type { + GID_TYPE_FLAG_ROCE_V1 = 0, + GID_TYPE_FLAG_ROCE_V2_IPV4, + GID_TYPE_FLAG_ROCE_V2_IPV6, +}; + +struct hns_roce_v2_cq_context { + u32 byte_4_pg_ceqn; + u32 byte_8_cqn; + u32 cqe_cur_blk_addr; + u32 byte_16_hop_addr; + u32 cqe_nxt_blk_addr; + u32 byte_24_pgsz_addr; + u32 byte_28_cq_pi; + u32 byte_32_cq_ci; + u32 cqe_ba; + u32 byte_40_cqe_ba; + u32 byte_44_db_record; + u32 db_record_addr; + u32 byte_52_cqe_cnt; + u32 byte_56_cqe_period_maxcnt; + u32 cqe_report_timer; + u32 byte_64_se_cqe_idx; +}; +#define V2_CQC_BYTE_4_CQ_ST_S 0 +#define V2_CQC_BYTE_4_CQ_ST_M GENMASK(1, 0) + +#define V2_CQC_BYTE_4_POLL_S 2 + +#define V2_CQC_BYTE_4_SE_S 3 + +#define V2_CQC_BYTE_4_OVER_IGNORE_S 4 + +#define V2_CQC_BYTE_4_COALESCE_S 5 + +#define V2_CQC_BYTE_4_ARM_ST_S 6 +#define V2_CQC_BYTE_4_ARM_ST_M GENMASK(7, 6) + +#define V2_CQC_BYTE_4_SHIFT_S 8 +#define V2_CQC_BYTE_4_SHIFT_M GENMASK(12, 8) + +#define V2_CQC_BYTE_4_CMD_SN_S 13 +#define V2_CQC_BYTE_4_CMD_SN_M GENMASK(14, 13) + +#define V2_CQC_BYTE_4_CEQN_S 15 +#define V2_CQC_BYTE_4_CEQN_M GENMASK(23, 15) + +#define V2_CQC_BYTE_4_PAGE_OFFSET_S 24 +#define V2_CQC_BYTE_4_PAGE_OFFSET_M GENMASK(31, 24) + +#define V2_CQC_BYTE_8_CQN_S 0 +#define V2_CQC_BYTE_8_CQN_M GENMASK(23, 0) + +#define V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S 0 +#define V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M GENMASK(19, 0) + +#define V2_CQC_BYTE_16_CQE_HOP_NUM_S 30 +#define V2_CQC_BYTE_16_CQE_HOP_NUM_M GENMASK(31, 30) + +#define V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S 0 +#define V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M GENMASK(19, 0) + +#define V2_CQC_BYTE_24_CQE_BA_PG_SZ_S 24 +#define V2_CQC_BYTE_24_CQE_BA_PG_SZ_M GENMASK(27, 24) + +#define V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S 28 +#define V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M GENMASK(31, 28) + +#define V2_CQC_BYTE_28_CQ_PRODUCER_IDX_S 0 +#define V2_CQC_BYTE_28_CQ_PRODUCER_IDX_M GENMASK(23, 0) + +#define V2_CQC_BYTE_32_CQ_CONSUMER_IDX_S 0 +#define V2_CQC_BYTE_32_CQ_CONSUMER_IDX_M GENMASK(23, 0) + +#define V2_CQC_BYTE_40_CQE_BA_S 0 +#define V2_CQC_BYTE_40_CQE_BA_M GENMASK(28, 0) + +#define V2_CQC_BYTE_44_DB_RECORD_EN_S 0 + +#define V2_CQC_BYTE_52_CQE_CNT_S 0 +#define V2_CQC_BYTE_52_CQE_CNT_M GENMASK(23, 0) + +#define V2_CQC_BYTE_56_CQ_MAX_CNT_S 0 +#define V2_CQC_BYTE_56_CQ_MAX_CNT_M GENMASK(15, 0) + +#define V2_CQC_BYTE_56_CQ_PERIOD_S 16 +#define V2_CQC_BYTE_56_CQ_PERIOD_M GENMASK(31, 16) + +#define V2_CQC_BYTE_64_SE_CQE_IDX_S 0 +#define V2_CQC_BYTE_64_SE_CQE_IDX_M GENMASK(23, 0) + +enum{ + V2_MPT_ST_VALID = 0x1, +}; + +enum hns_roce_v2_qp_state { + HNS_ROCE_QP_ST_RST, + HNS_ROCE_QP_ST_INIT, + HNS_ROCE_QP_ST_RTR, + HNS_ROCE_QP_ST_RTS, + HNS_ROCE_QP_ST_SQER, + HNS_ROCE_QP_ST_SQD, + HNS_ROCE_QP_ST_ERR, + HNS_ROCE_QP_ST_SQ_DRAINING, + HNS_ROCE_QP_NUM_ST +}; + +struct hns_roce_v2_qp_context { + u32 byte_4_sqpn_tst; + u32 wqe_sge_ba; + u32 byte_12_sq_hop; + u32 byte_16_buf_ba_pg_sz; + u32 byte_20_smac_sgid_idx; + u32 byte_24_mtu_tc; + u32 byte_28_at_fl; + u8 dgid[GID_LEN_V2]; + u32 dmac; + u32 byte_52_udpspn_dmac; + u32 byte_56_dqpn_err; + u32 byte_60_qpst_mapid; + u32 qkey_xrcd; + u32 byte_68_rq_db; + u32 rq_db_record_addr; + u32 byte_76_srqn_op_en; + u32 byte_80_rnr_rx_cqn; + u32 byte_84_rq_ci_pi; + u32 rq_cur_blk_addr; + u32 byte_92_srq_info; + u32 byte_96_rx_reqmsn; + u32 rq_nxt_blk_addr; + u32 byte_104_rq_sge; + u32 byte_108_rx_reqepsn; + u32 rq_rnr_timer; + u32 rx_msg_len; + u32 rx_rkey_pkt_info; + u64 rx_va; + u32 byte_132_trrl; + u32 trrl_ba; + u32 byte_140_raq; + u32 byte_144_raq; + u32 byte_148_raq; + u32 byte_152_raq; + u32 byte_156_raq; + u32 byte_160_sq_ci_pi; + u32 sq_cur_blk_addr; + u32 byte_168_irrl_idx; + u32 byte_172_sq_psn; + u32 byte_176_msg_pktn; + u32 sq_cur_sge_blk_addr; + u32 byte_184_irrl_idx; + u32 cur_sge_offset; + u32 byte_192_ext_sge; + u32 byte_196_sq_psn; + u32 byte_200_sq_max; + u32 irrl_ba; + u32 byte_208_irrl; + u32 byte_212_lsn; + u32 sq_timer; + u32 byte_220_retry_psn_msn; + u32 byte_224_retry_msg; + u32 rx_sq_cur_blk_addr; + u32 byte_232_irrl_sge; + u32 irrl_cur_sge_offset; + u32 byte_240_irrl_tail; + u32 byte_244_rnr_rxack; + u32 byte_248_ack_psn; + u32 byte_252_err_txcqn; + u32 byte_256_sqflush_rqcqe; +}; + +#define V2_QPC_BYTE_4_TST_S 0 +#define V2_QPC_BYTE_4_TST_M GENMASK(2, 0) + +#define V2_QPC_BYTE_4_SGE_SHIFT_S 3 +#define V2_QPC_BYTE_4_SGE_SHIFT_M GENMASK(7, 3) + +#define V2_QPC_BYTE_4_SQPN_S 8 +#define V2_QPC_BYTE_4_SQPN_M GENMASK(31, 8) + +#define V2_QPC_BYTE_12_WQE_SGE_BA_S 0 +#define V2_QPC_BYTE_12_WQE_SGE_BA_M GENMASK(28, 0) + +#define V2_QPC_BYTE_12_SQ_HOP_NUM_S 29 +#define V2_QPC_BYTE_12_SQ_HOP_NUM_M GENMASK(30, 29) + +#define V2_QPC_BYTE_12_RSVD_LKEY_EN_S 31 + +#define V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S 0 +#define V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M GENMASK(3, 0) + +#define V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S 4 +#define V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M GENMASK(7, 4) + +#define V2_QPC_BYTE_16_PD_S 8 +#define V2_QPC_BYTE_16_PD_M GENMASK(31, 8) + +#define V2_QPC_BYTE_20_RQ_HOP_NUM_S 0 +#define V2_QPC_BYTE_20_RQ_HOP_NUM_M GENMASK(1, 0) + +#define V2_QPC_BYTE_20_SGE_HOP_NUM_S 2 +#define V2_QPC_BYTE_20_SGE_HOP_NUM_M GENMASK(3, 2) + +#define V2_QPC_BYTE_20_RQWS_S 4 +#define V2_QPC_BYTE_20_RQWS_M GENMASK(7, 4) + +#define V2_QPC_BYTE_20_SQ_SHIFT_S 8 +#define V2_QPC_BYTE_20_SQ_SHIFT_M GENMASK(11, 8) + +#define V2_QPC_BYTE_20_RQ_SHIFT_S 12 +#define V2_QPC_BYTE_20_RQ_SHIFT_M GENMASK(15, 12) + +#define V2_QPC_BYTE_20_SGID_IDX_S 16 +#define V2_QPC_BYTE_20_SGID_IDX_M GENMASK(23, 16) + +#define V2_QPC_BYTE_20_SMAC_IDX_S 24 +#define V2_QPC_BYTE_20_SMAC_IDX_M GENMASK(31, 24) + +#define V2_QPC_BYTE_24_HOP_LIMIT_S 0 +#define V2_QPC_BYTE_24_HOP_LIMIT_M GENMASK(7, 0) + +#define V2_QPC_BYTE_24_TC_S 8 +#define V2_QPC_BYTE_24_TC_M GENMASK(15, 8) + +#define V2_QPC_BYTE_24_VLAN_IDX_S 16 +#define V2_QPC_BYTE_24_VLAN_IDX_M GENMASK(27, 16) + +#define V2_QPC_BYTE_24_MTU_S 28 +#define V2_QPC_BYTE_24_MTU_M GENMASK(31, 28) + +#define V2_QPC_BYTE_28_FL_S 0 +#define V2_QPC_BYTE_28_FL_M GENMASK(19, 0) + +#define V2_QPC_BYTE_28_SL_S 20 +#define V2_QPC_BYTE_28_SL_M GENMASK(23, 20) + +#define V2_QPC_BYTE_28_CNP_TX_FLAG_S 24 + +#define V2_QPC_BYTE_28_CE_FLAG_S 25 + +#define V2_QPC_BYTE_28_LBI_S 26 + +#define V2_QPC_BYTE_28_AT_S 27 +#define V2_QPC_BYTE_28_AT_M GENMASK(31, 27) + +#define V2_QPC_BYTE_52_DMAC_S 0 +#define V2_QPC_BYTE_52_DMAC_M GENMASK(15, 0) + +#define V2_QPC_BYTE_52_UDPSPN_S 16 +#define V2_QPC_BYTE_52_UDPSPN_M GENMASK(31, 16) + +#define V2_QPC_BYTE_56_DQPN_S 0 +#define V2_QPC_BYTE_56_DQPN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_56_SQ_TX_ERR_S 24 +#define V2_QPC_BYTE_56_SQ_RX_ERR_S 25 +#define V2_QPC_BYTE_56_RQ_TX_ERR_S 26 +#define V2_QPC_BYTE_56_RQ_RX_ERR_S 27 + +#define V2_QPC_BYTE_56_LP_PKTN_INI_S 28 +#define V2_QPC_BYTE_56_LP_PKTN_INI_M GENMASK(31, 28) + +#define V2_QPC_BYTE_60_MAPID_S 0 +#define V2_QPC_BYTE_60_MAPID_M GENMASK(12, 0) + +#define V2_QPC_BYTE_60_INNER_MAP_IND_S 13 + +#define V2_QPC_BYTE_60_SQ_MAP_IND_S 14 + +#define V2_QPC_BYTE_60_RQ_MAP_IND_S 15 + +#define V2_QPC_BYTE_60_TEMPID_S 16 +#define V2_QPC_BYTE_60_TEMPID_M GENMASK(22, 16) + +#define V2_QPC_BYTE_60_EXT_MAP_IND_S 23 + +#define V2_QPC_BYTE_60_RTY_NUM_INI_BAK_S 24 +#define V2_QPC_BYTE_60_RTY_NUM_INI_BAK_M GENMASK(26, 24) + +#define V2_QPC_BYTE_60_SQ_RLS_IND_S 27 + +#define V2_QPC_BYTE_60_SQ_EXT_IND_S 28 + +#define V2_QPC_BYTE_60_QP_ST_S 29 +#define V2_QPC_BYTE_60_QP_ST_M GENMASK(31, 29) + +#define V2_QPC_BYTE_68_RQ_RECORD_EN_S 0 + +#define V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S 1 +#define V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M GENMASK(31, 1) + +#define V2_QPC_BYTE_76_SRQN_S 0 +#define V2_QPC_BYTE_76_SRQN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_76_SRQ_EN_S 24 + +#define V2_QPC_BYTE_76_RRE_S 25 + +#define V2_QPC_BYTE_76_RWE_S 26 + +#define V2_QPC_BYTE_76_ATE_S 27 + +#define V2_QPC_BYTE_76_RQIE_S 28 + +#define V2_QPC_BYTE_80_RX_CQN_S 0 +#define V2_QPC_BYTE_80_RX_CQN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_80_MIN_RNR_TIME_S 27 +#define V2_QPC_BYTE_80_MIN_RNR_TIME_M GENMASK(31, 27) + +#define V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S 0 +#define V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M GENMASK(15, 0) + +#define V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S 16 +#define V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M GENMASK(31, 16) + +#define V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S 0 +#define V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M GENMASK(19, 0) + +#define V2_QPC_BYTE_92_SRQ_INFO_S 20 +#define V2_QPC_BYTE_92_SRQ_INFO_M GENMASK(31, 20) + +#define V2_QPC_BYTE_96_RX_REQ_MSN_S 0 +#define V2_QPC_BYTE_96_RX_REQ_MSN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S 0 +#define V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M GENMASK(19, 0) + +#define V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_S 24 +#define V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_M GENMASK(31, 24) + +#define V2_QPC_BYTE_108_INV_CREDIT_S 0 + +#define V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S 3 + +#define V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S 4 +#define V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M GENMASK(6, 4) + +#define V2_QPC_BYTE_108_RX_REQ_RNR_S 7 + +#define V2_QPC_BYTE_108_RX_REQ_EPSN_S 8 +#define V2_QPC_BYTE_108_RX_REQ_EPSN_M GENMASK(31, 8) + +#define V2_QPC_BYTE_132_TRRL_HEAD_MAX_S 0 +#define V2_QPC_BYTE_132_TRRL_HEAD_MAX_M GENMASK(7, 0) + +#define V2_QPC_BYTE_132_TRRL_TAIL_MAX_S 8 +#define V2_QPC_BYTE_132_TRRL_TAIL_MAX_M GENMASK(15, 8) + +#define V2_QPC_BYTE_132_TRRL_BA_S 16 +#define V2_QPC_BYTE_132_TRRL_BA_M GENMASK(31, 16) + +#define V2_QPC_BYTE_140_TRRL_BA_S 0 +#define V2_QPC_BYTE_140_TRRL_BA_M GENMASK(11, 0) + +#define V2_QPC_BYTE_140_RR_MAX_S 12 +#define V2_QPC_BYTE_140_RR_MAX_M GENMASK(14, 12) + +#define V2_QPC_BYTE_140_RSVD_RAQ_MAP_S 15 + +#define V2_QPC_BYTE_140_RAQ_TRRL_HEAD_S 16 +#define V2_QPC_BYTE_140_RAQ_TRRL_HEAD_M GENMASK(23, 16) + +#define V2_QPC_BYTE_140_RAQ_TRRL_TAIL_S 24 +#define V2_QPC_BYTE_140_RAQ_TRRL_TAIL_M GENMASK(31, 24) + +#define V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_S 0 +#define V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_144_RAQ_RTY_INI_IND_S 24 + +#define V2_QPC_BYTE_144_RAQ_CREDIT_S 25 +#define V2_QPC_BYTE_144_RAQ_CREDIT_M GENMASK(29, 25) + +#define V2_QPC_BYTE_144_RESP_RTY_FLG_S 31 + +#define V2_QPC_BYTE_148_RQ_MSN_S 0 +#define V2_QPC_BYTE_148_RQ_MSN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_148_RAQ_SYNDROME_S 24 +#define V2_QPC_BYTE_148_RAQ_SYNDROME_M GENMASK(31, 24) + +#define V2_QPC_BYTE_152_RAQ_PSN_S 8 +#define V2_QPC_BYTE_152_RAQ_PSN_M GENMASK(31, 8) + +#define V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_S 24 +#define V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_M GENMASK(31, 24) + +#define V2_QPC_BYTE_156_RAQ_USE_PKTN_S 0 +#define V2_QPC_BYTE_156_RAQ_USE_PKTN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S 0 +#define V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M GENMASK(15, 0) + +#define V2_QPC_BYTE_160_SQ_CONSUMER_IDX_S 16 +#define V2_QPC_BYTE_160_SQ_CONSUMER_IDX_M GENMASK(31, 16) + +#define V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S 0 +#define V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M GENMASK(19, 0) + +#define V2_QPC_BYTE_168_MSG_RTY_LP_FLG_S 20 + +#define V2_QPC_BYTE_168_SQ_INVLD_FLG_S 21 + +#define V2_QPC_BYTE_168_LP_SGEN_INI_S 22 +#define V2_QPC_BYTE_168_LP_SGEN_INI_M GENMASK(23, 22) + +#define V2_QPC_BYTE_168_SQ_SHIFT_BAK_S 24 +#define V2_QPC_BYTE_168_SQ_SHIFT_BAK_M GENMASK(27, 24) + +#define V2_QPC_BYTE_168_IRRL_IDX_LSB_S 28 +#define V2_QPC_BYTE_168_IRRL_IDX_LSB_M GENMASK(31, 28) + +#define V2_QPC_BYTE_172_ACK_REQ_FREQ_S 0 +#define V2_QPC_BYTE_172_ACK_REQ_FREQ_M GENMASK(5, 0) + +#define V2_QPC_BYTE_172_MSG_RNR_FLG_S 6 + +#define V2_QPC_BYTE_172_FRE_S 7 + +#define V2_QPC_BYTE_172_SQ_CUR_PSN_S 8 +#define V2_QPC_BYTE_172_SQ_CUR_PSN_M GENMASK(31, 8) + +#define V2_QPC_BYTE_176_MSG_USE_PKTN_S 0 +#define V2_QPC_BYTE_176_MSG_USE_PKTN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_176_IRRL_HEAD_PRE_S 24 +#define V2_QPC_BYTE_176_IRRL_HEAD_PRE_M GENMASK(31, 24) + +#define V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S 0 +#define V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M GENMASK(19, 0) + +#define V2_QPC_BYTE_184_IRRL_IDX_MSB_S 20 +#define V2_QPC_BYTE_184_IRRL_IDX_MSB_M GENMASK(31, 20) + +#define V2_QPC_BYTE_192_CUR_SGE_IDX_S 0 +#define V2_QPC_BYTE_192_CUR_SGE_IDX_M GENMASK(23, 0) + +#define V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_S 24 +#define V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_M GENMASK(31, 24) + +#define V2_QPC_BYTE_196_IRRL_HEAD_S 0 +#define V2_QPC_BYTE_196_IRRL_HEAD_M GENMASK(7, 0) + +#define V2_QPC_BYTE_196_SQ_MAX_PSN_S 8 +#define V2_QPC_BYTE_196_SQ_MAX_PSN_M GENMASK(31, 8) + +#define V2_QPC_BYTE_200_SQ_MAX_IDX_S 0 +#define V2_QPC_BYTE_200_SQ_MAX_IDX_M GENMASK(15, 0) + +#define V2_QPC_BYTE_200_LCL_OPERATED_CNT_S 16 +#define V2_QPC_BYTE_200_LCL_OPERATED_CNT_M GENMASK(31, 16) + +#define V2_QPC_BYTE_208_IRRL_BA_S 0 +#define V2_QPC_BYTE_208_IRRL_BA_M GENMASK(25, 0) + +#define V2_QPC_BYTE_208_PKT_RNR_FLG_S 26 + +#define V2_QPC_BYTE_208_PKT_RTY_FLG_S 27 + +#define V2_QPC_BYTE_208_RMT_E2E_S 28 + +#define V2_QPC_BYTE_208_SR_MAX_S 29 +#define V2_QPC_BYTE_208_SR_MAX_M GENMASK(31, 29) + +#define V2_QPC_BYTE_212_LSN_S 0 +#define V2_QPC_BYTE_212_LSN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_212_RETRY_NUM_INIT_S 24 +#define V2_QPC_BYTE_212_RETRY_NUM_INIT_M GENMASK(26, 24) + +#define V2_QPC_BYTE_212_CHECK_FLG_S 27 +#define V2_QPC_BYTE_212_CHECK_FLG_M GENMASK(28, 27) + +#define V2_QPC_BYTE_212_RETRY_CNT_S 29 +#define V2_QPC_BYTE_212_RETRY_CNT_M GENMASK(31, 29) + +#define V2_QPC_BYTE_220_RETRY_MSG_MSN_S 0 +#define V2_QPC_BYTE_220_RETRY_MSG_MSN_M GENMASK(15, 0) + +#define V2_QPC_BYTE_220_RETRY_MSG_PSN_S 16 +#define V2_QPC_BYTE_220_RETRY_MSG_PSN_M GENMASK(31, 16) + +#define V2_QPC_BYTE_224_RETRY_MSG_PSN_S 0 +#define V2_QPC_BYTE_224_RETRY_MSG_PSN_M GENMASK(7, 0) + +#define V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S 8 +#define V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M GENMASK(31, 8) + +#define V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S 0 +#define V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M GENMASK(19, 0) + +#define V2_QPC_BYTE_232_IRRL_SGE_IDX_S 20 +#define V2_QPC_BYTE_232_IRRL_SGE_IDX_M GENMASK(28, 20) + +#define V2_QPC_BYTE_240_IRRL_TAIL_REAL_S 0 +#define V2_QPC_BYTE_240_IRRL_TAIL_REAL_M GENMASK(7, 0) + +#define V2_QPC_BYTE_240_IRRL_TAIL_RD_S 8 +#define V2_QPC_BYTE_240_IRRL_TAIL_RD_M GENMASK(15, 8) + +#define V2_QPC_BYTE_240_RX_ACK_MSN_S 16 +#define V2_QPC_BYTE_240_RX_ACK_MSN_M GENMASK(31, 16) + +#define V2_QPC_BYTE_244_RX_ACK_EPSN_S 0 +#define V2_QPC_BYTE_244_RX_ACK_EPSN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_244_RNR_NUM_INIT_S 24 +#define V2_QPC_BYTE_244_RNR_NUM_INIT_M GENMASK(26, 24) + +#define V2_QPC_BYTE_244_RNR_CNT_S 27 +#define V2_QPC_BYTE_244_RNR_CNT_M GENMASK(29, 27) + +#define V2_QPC_BYTE_248_IRRL_PSN_S 0 +#define V2_QPC_BYTE_248_IRRL_PSN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_248_ACK_PSN_ERR_S 24 + +#define V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S 25 +#define V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M GENMASK(26, 25) + +#define V2_QPC_BYTE_248_IRRL_PSN_VLD_S 27 + +#define V2_QPC_BYTE_248_RNR_RETRY_FLAG_S 28 + +#define V2_QPC_BYTE_248_CQ_ERR_IND_S 31 + +#define V2_QPC_BYTE_252_TX_CQN_S 0 +#define V2_QPC_BYTE_252_TX_CQN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_252_SIG_TYPE_S 24 + +#define V2_QPC_BYTE_252_ERR_TYPE_S 25 +#define V2_QPC_BYTE_252_ERR_TYPE_M GENMASK(31, 25) + +#define V2_QPC_BYTE_256_RQ_CQE_IDX_S 0 +#define V2_QPC_BYTE_256_RQ_CQE_IDX_M GENMASK(15, 0) + +#define V2_QPC_BYTE_256_SQ_FLUSH_IDX_S 16 +#define V2_QPC_BYTE_256_SQ_FLUSH_IDX_M GENMASK(31, 16) + +struct hns_roce_v2_cqe { + u32 byte_4; + u32 rkey_immtdata; + u32 byte_12; + u32 byte_16; + u32 byte_cnt; + u32 smac; + u32 byte_28; + u32 byte_32; +}; + +#define V2_CQE_BYTE_4_OPCODE_S 0 +#define V2_CQE_BYTE_4_OPCODE_M GENMASK(4, 0) + +#define V2_CQE_BYTE_4_RQ_INLINE_S 5 + +#define V2_CQE_BYTE_4_S_R_S 6 + +#define V2_CQE_BYTE_4_OWNER_S 7 + +#define V2_CQE_BYTE_4_STATUS_S 8 +#define V2_CQE_BYTE_4_STATUS_M GENMASK(15, 8) + +#define V2_CQE_BYTE_4_WQE_INDX_S 16 +#define V2_CQE_BYTE_4_WQE_INDX_M GENMASK(31, 16) + +#define V2_CQE_BYTE_12_XRC_SRQN_S 0 +#define V2_CQE_BYTE_12_XRC_SRQN_M GENMASK(23, 0) + +#define V2_CQE_BYTE_16_LCL_QPN_S 0 +#define V2_CQE_BYTE_16_LCL_QPN_M GENMASK(23, 0) + +#define V2_CQE_BYTE_16_SUB_STATUS_S 24 +#define V2_CQE_BYTE_16_SUB_STATUS_M GENMASK(31, 24) + +#define V2_CQE_BYTE_28_SMAC_4_S 0 +#define V2_CQE_BYTE_28_SMAC_4_M GENMASK(7, 0) + +#define V2_CQE_BYTE_28_SMAC_5_S 8 +#define V2_CQE_BYTE_28_SMAC_5_M GENMASK(15, 8) + +#define V2_CQE_BYTE_28_PORT_TYPE_S 16 +#define V2_CQE_BYTE_28_PORT_TYPE_M GENMASK(17, 16) + +#define V2_CQE_BYTE_32_RMT_QPN_S 0 +#define V2_CQE_BYTE_32_RMT_QPN_M GENMASK(23, 0) + +#define V2_CQE_BYTE_32_SL_S 24 +#define V2_CQE_BYTE_32_SL_M GENMASK(26, 24) + +#define V2_CQE_BYTE_32_PORTN_S 27 +#define V2_CQE_BYTE_32_PORTN_M GENMASK(29, 27) + +#define V2_CQE_BYTE_32_GRH_S 30 + +#define V2_CQE_BYTE_32_LPK_S 31 + +struct hns_roce_v2_mpt_entry { + __le32 byte_4_pd_hop_st; + __le32 byte_8_mw_cnt_en; + __le32 byte_12_mw_pa; + __le32 bound_lkey; + __le32 len_l; + __le32 len_h; + __le32 lkey; + __le32 va_l; + __le32 va_h; + __le32 pbl_size; + __le32 pbl_ba_l; + __le32 byte_48_mode_ba; + __le32 pa0_l; + __le32 byte_56_pa0_h; + __le32 pa1_l; + __le32 byte_64_buf_pa1; +}; + +#define V2_MPT_BYTE_4_MPT_ST_S 0 +#define V2_MPT_BYTE_4_MPT_ST_M GENMASK(1, 0) + +#define V2_MPT_BYTE_4_PBL_HOP_NUM_S 2 +#define V2_MPT_BYTE_4_PBL_HOP_NUM_M GENMASK(3, 2) + +#define V2_MPT_BYTE_4_PBL_BA_PG_SZ_S 4 +#define V2_MPT_BYTE_4_PBL_BA_PG_SZ_M GENMASK(7, 4) + +#define V2_MPT_BYTE_4_PD_S 8 +#define V2_MPT_BYTE_4_PD_M GENMASK(31, 8) + +#define V2_MPT_BYTE_8_RA_EN_S 0 + +#define V2_MPT_BYTE_8_R_INV_EN_S 1 + +#define V2_MPT_BYTE_8_L_INV_EN_S 2 + +#define V2_MPT_BYTE_8_BIND_EN_S 3 + +#define V2_MPT_BYTE_8_ATOMIC_EN_S 4 + +#define V2_MPT_BYTE_8_RR_EN_S 5 + +#define V2_MPT_BYTE_8_RW_EN_S 6 + +#define V2_MPT_BYTE_8_LW_EN_S 7 + +#define V2_MPT_BYTE_12_PA_S 1 + +#define V2_MPT_BYTE_12_INNER_PA_VLD_S 7 + +#define V2_MPT_BYTE_12_MW_BIND_QPN_S 8 +#define V2_MPT_BYTE_12_MW_BIND_QPN_M GENMASK(31, 8) + +#define V2_MPT_BYTE_48_PBL_BA_H_S 0 +#define V2_MPT_BYTE_48_PBL_BA_H_M GENMASK(28, 0) + +#define V2_MPT_BYTE_48_BLK_MODE_S 29 + +#define V2_MPT_BYTE_56_PA0_H_S 0 +#define V2_MPT_BYTE_56_PA0_H_M GENMASK(25, 0) + +#define V2_MPT_BYTE_64_PA1_H_S 0 +#define V2_MPT_BYTE_64_PA1_H_M GENMASK(25, 0) + +#define V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S 28 +#define V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M GENMASK(31, 28) + +#define V2_DB_BYTE_4_TAG_S 0 +#define V2_DB_BYTE_4_TAG_M GENMASK(23, 0) + +#define V2_DB_BYTE_4_CMD_S 24 +#define V2_DB_BYTE_4_CMD_M GENMASK(27, 24) + +#define V2_DB_PARAMETER_CONS_IDX_S 0 +#define V2_DB_PARAMETER_CONS_IDX_M GENMASK(15, 0) + +#define V2_DB_PARAMETER_SL_S 16 +#define V2_DB_PARAMETER_SL_M GENMASK(18, 16) + +struct hns_roce_v2_cq_db { + u32 byte_4; + u32 parameter; +}; + +#define V2_CQ_DB_BYTE_4_TAG_S 0 +#define V2_CQ_DB_BYTE_4_TAG_M GENMASK(23, 0) + +#define V2_CQ_DB_BYTE_4_CMD_S 24 +#define V2_CQ_DB_BYTE_4_CMD_M GENMASK(27, 24) + +#define V2_CQ_DB_PARAMETER_CONS_IDX_S 0 +#define V2_CQ_DB_PARAMETER_CONS_IDX_M GENMASK(23, 0) + +#define V2_CQ_DB_PARAMETER_CMD_SN_S 25 +#define V2_CQ_DB_PARAMETER_CMD_SN_M GENMASK(26, 25) + +#define V2_CQ_DB_PARAMETER_NOTIFY_S 24 + +struct hns_roce_v2_rc_send_wqe { + u32 byte_4; + u32 msg_len; + u32 inv_key_immtdata; + u32 byte_16; + u32 byte_20; + u32 rkey; + u64 va; +}; + +#define V2_RC_SEND_WQE_BYTE_4_OPCODE_S 0 +#define V2_RC_SEND_WQE_BYTE_4_OPCODE_M GENMASK(4, 0) + +#define V2_RC_SEND_WQE_BYTE_4_OWNER_S 7 + +#define V2_RC_SEND_WQE_BYTE_4_CQE_S 8 + +#define V2_RC_SEND_WQE_BYTE_4_FENCE_S 9 + +#define V2_RC_SEND_WQE_BYTE_4_SO_S 10 + +#define V2_RC_SEND_WQE_BYTE_4_SE_S 11 + +#define V2_RC_SEND_WQE_BYTE_4_INLINE_S 12 + +#define V2_RC_SEND_WQE_BYTE_16_XRC_SRQN_S 0 +#define V2_RC_SEND_WQE_BYTE_16_XRC_SRQN_M GENMASK(23, 0) + +#define V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S 24 +#define V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M GENMASK(31, 24) + +#define V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S 0 +#define V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M GENMASK(23, 0) + +struct hns_roce_v2_wqe_data_seg { + __be32 len; + __be32 lkey; + __be64 addr; +}; + +struct hns_roce_v2_db { + u32 byte_4; + u32 parameter; +}; + +struct hns_roce_query_version { + __le16 rocee_vendor_id; + __le16 rocee_hw_version; + __le32 rsv[5]; +}; + +struct hns_roce_cfg_global_param { + __le32 time_cfg_udp_port; + __le32 rsv[5]; +}; + +#define CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_S 0 +#define CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_M GENMASK(9, 0) + +#define CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_S 16 +#define CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_M GENMASK(31, 16) + +struct hns_roce_pf_res { + __le32 rsv; + __le32 qpc_bt_idx_num; + __le32 srqc_bt_idx_num; + __le32 cqc_bt_idx_num; + __le32 mpt_bt_idx_num; + __le32 eqc_bt_idx_num; +}; + +#define PF_RES_DATA_1_PF_QPC_BT_IDX_S 0 +#define PF_RES_DATA_1_PF_QPC_BT_IDX_M GENMASK(10, 0) + +#define PF_RES_DATA_1_PF_QPC_BT_NUM_S 16 +#define PF_RES_DATA_1_PF_QPC_BT_NUM_M GENMASK(27, 16) + +#define PF_RES_DATA_2_PF_SRQC_BT_IDX_S 0 +#define PF_RES_DATA_2_PF_SRQC_BT_IDX_M GENMASK(8, 0) + +#define PF_RES_DATA_2_PF_SRQC_BT_NUM_S 16 +#define PF_RES_DATA_2_PF_SRQC_BT_NUM_M GENMASK(25, 16) + +#define PF_RES_DATA_3_PF_CQC_BT_IDX_S 0 +#define PF_RES_DATA_3_PF_CQC_BT_IDX_M GENMASK(8, 0) + +#define PF_RES_DATA_3_PF_CQC_BT_NUM_S 16 +#define PF_RES_DATA_3_PF_CQC_BT_NUM_M GENMASK(25, 16) + +#define PF_RES_DATA_4_PF_MPT_BT_IDX_S 0 +#define PF_RES_DATA_4_PF_MPT_BT_IDX_M GENMASK(8, 0) + +#define PF_RES_DATA_4_PF_MPT_BT_NUM_S 16 +#define PF_RES_DATA_4_PF_MPT_BT_NUM_M GENMASK(25, 16) + +#define PF_RES_DATA_5_PF_EQC_BT_IDX_S 0 +#define PF_RES_DATA_5_PF_EQC_BT_IDX_M GENMASK(8, 0) + +#define PF_RES_DATA_5_PF_EQC_BT_NUM_S 16 +#define PF_RES_DATA_5_PF_EQC_BT_NUM_M GENMASK(25, 16) + +struct hns_roce_vf_res_a { + u32 vf_id; + u32 vf_qpc_bt_idx_num; + u32 vf_srqc_bt_idx_num; + u32 vf_cqc_bt_idx_num; + u32 vf_mpt_bt_idx_num; + u32 vf_eqc_bt_idx_num; +}; + +#define VF_RES_A_DATA_1_VF_QPC_BT_IDX_S 0 +#define VF_RES_A_DATA_1_VF_QPC_BT_IDX_M GENMASK(10, 0) + +#define VF_RES_A_DATA_1_VF_QPC_BT_NUM_S 16 +#define VF_RES_A_DATA_1_VF_QPC_BT_NUM_M GENMASK(27, 16) + +#define VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S 0 +#define VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M GENMASK(8, 0) + +#define VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S 16 +#define VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M GENMASK(25, 16) + +#define VF_RES_A_DATA_3_VF_CQC_BT_IDX_S 0 +#define VF_RES_A_DATA_3_VF_CQC_BT_IDX_M GENMASK(8, 0) + +#define VF_RES_A_DATA_3_VF_CQC_BT_NUM_S 16 +#define VF_RES_A_DATA_3_VF_CQC_BT_NUM_M GENMASK(25, 16) + +#define VF_RES_A_DATA_4_VF_MPT_BT_IDX_S 0 +#define VF_RES_A_DATA_4_VF_MPT_BT_IDX_M GENMASK(8, 0) + +#define VF_RES_A_DATA_4_VF_MPT_BT_NUM_S 16 +#define VF_RES_A_DATA_4_VF_MPT_BT_NUM_M GENMASK(25, 16) + +#define VF_RES_A_DATA_5_VF_EQC_IDX_S 0 +#define VF_RES_A_DATA_5_VF_EQC_IDX_M GENMASK(8, 0) + +#define VF_RES_A_DATA_5_VF_EQC_NUM_S 16 +#define VF_RES_A_DATA_5_VF_EQC_NUM_M GENMASK(25, 16) + +struct hns_roce_vf_res_b { + u32 rsv0; + u32 vf_smac_idx_num; + u32 vf_sgid_idx_num; + u32 vf_qid_idx_sl_num; + u32 rsv[2]; +}; + +#define VF_RES_B_DATA_0_VF_ID_S 0 +#define VF_RES_B_DATA_0_VF_ID_M GENMASK(7, 0) + +#define VF_RES_B_DATA_1_VF_SMAC_IDX_S 0 +#define VF_RES_B_DATA_1_VF_SMAC_IDX_M GENMASK(7, 0) + +#define VF_RES_B_DATA_1_VF_SMAC_NUM_S 8 +#define VF_RES_B_DATA_1_VF_SMAC_NUM_M GENMASK(16, 8) + +#define VF_RES_B_DATA_2_VF_SGID_IDX_S 0 +#define VF_RES_B_DATA_2_VF_SGID_IDX_M GENMASK(7, 0) + +#define VF_RES_B_DATA_2_VF_SGID_NUM_S 8 +#define VF_RES_B_DATA_2_VF_SGID_NUM_M GENMASK(16, 8) + +#define VF_RES_B_DATA_3_VF_QID_IDX_S 0 +#define VF_RES_B_DATA_3_VF_QID_IDX_M GENMASK(9, 0) + +#define VF_RES_B_DATA_3_VF_SL_NUM_S 16 +#define VF_RES_B_DATA_3_VF_SL_NUM_M GENMASK(19, 16) + +/* Reg field definition */ +#define ROCEE_VF_SMAC_CFG1_VF_SMAC_H_S 0 +#define ROCEE_VF_SMAC_CFG1_VF_SMAC_H_M GENMASK(15, 0) + +#define ROCEE_VF_SGID_CFG4_SGID_TYPE_S 0 +#define ROCEE_VF_SGID_CFG4_SGID_TYPE_M GENMASK(1, 0) + +struct hns_roce_cfg_bt_attr { + u32 vf_qpc_cfg; + u32 vf_srqc_cfg; + u32 vf_cqc_cfg; + u32 vf_mpt_cfg; + u32 rsv[2]; +}; + +#define CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S 0 +#define CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M GENMASK(3, 0) + +#define CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S 4 +#define CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M GENMASK(7, 4) + +#define CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S 8 +#define CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M GENMASK(9, 8) + +#define CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S 0 +#define CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M GENMASK(3, 0) + +#define CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S 4 +#define CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M GENMASK(7, 4) + +#define CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S 8 +#define CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M GENMASK(9, 8) + +#define CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S 0 +#define CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M GENMASK(3, 0) + +#define CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S 4 +#define CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M GENMASK(7, 4) + +#define CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S 8 +#define CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M GENMASK(9, 8) + +#define CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S 0 +#define CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M GENMASK(3, 0) + +#define CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S 4 +#define CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M GENMASK(7, 4) + +#define CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S 8 +#define CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M GENMASK(9, 8) + +struct hns_roce_cmq_desc { + u16 opcode; + u16 flag; + u16 retval; + u16 rsv; + u32 data[6]; +}; + +#define ROCEE_VF_MB_CFG0_REG 0x40 +#define ROCEE_VF_MB_STATUS_REG 0x58 + +#define HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS 10000 + +#define HNS_ROCE_HW_RUN_BIT_SHIFT 31 +#define HNS_ROCE_HW_MB_STATUS_MASK 0xFF + +#define HNS_ROCE_VF_MB4_TAG_MASK 0xFFFFFF00 +#define HNS_ROCE_VF_MB4_TAG_SHIFT 8 + +#define HNS_ROCE_VF_MB4_CMD_MASK 0xFF +#define HNS_ROCE_VF_MB4_CMD_SHIFT 0 + +#define HNS_ROCE_VF_MB5_EVENT_MASK 0x10000 +#define HNS_ROCE_VF_MB5_EVENT_SHIFT 16 + +#define HNS_ROCE_VF_MB5_TOKEN_MASK 0xFFFF +#define HNS_ROCE_VF_MB5_TOKEN_SHIFT 0 + +struct hns_roce_v2_cmq_ring { + dma_addr_t desc_dma_addr; + struct hns_roce_cmq_desc *desc; + u32 head; + u32 tail; + + u16 buf_size; + u16 desc_num; + int next_to_use; + int next_to_clean; + u8 flag; + spinlock_t lock; /* command queue lock */ +}; + +struct hns_roce_v2_cmq { + struct hns_roce_v2_cmq_ring csq; + struct hns_roce_v2_cmq_ring crq; + u16 tx_timeout; + u16 last_status; +}; + +struct hns_roce_v2_priv { + struct hns_roce_v2_cmq cmq; +}; + +#endif diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index d9777b662eba..cf02ac2d3596 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -57,20 +57,21 @@ int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index) { return gid_index * hr_dev->caps.num_ports + port; } +EXPORT_SYMBOL_GPL(hns_get_gid_index); -static void hns_roce_set_mac(struct hns_roce_dev *hr_dev, u8 port, u8 *addr) +static int hns_roce_set_mac(struct hns_roce_dev *hr_dev, u8 port, u8 *addr) { u8 phy_port; u32 i = 0; if (!memcmp(hr_dev->dev_addr[port], addr, MAC_ADDR_OCTET_NUM)) - return; + return 0; for (i = 0; i < MAC_ADDR_OCTET_NUM; i++) hr_dev->dev_addr[port][i] = addr[i]; phy_port = hr_dev->iboe.phy_port[port]; - hr_dev->hw->set_mac(hr_dev, phy_port, addr); + return hr_dev->hw->set_mac(hr_dev, phy_port, addr); } static int hns_roce_add_gid(struct ib_device *device, u8 port_num, @@ -80,17 +81,19 @@ static int hns_roce_add_gid(struct ib_device *device, u8 port_num, struct hns_roce_dev *hr_dev = to_hr_dev(device); u8 port = port_num - 1; unsigned long flags; + int ret; if (port >= hr_dev->caps.num_ports) return -EINVAL; spin_lock_irqsave(&hr_dev->iboe.lock, flags); - hr_dev->hw->set_gid(hr_dev, port, index, (union ib_gid *)gid); + ret = hr_dev->hw->set_gid(hr_dev, port, index, (union ib_gid *)gid, + attr); spin_unlock_irqrestore(&hr_dev->iboe.lock, flags); - return 0; + return ret; } static int hns_roce_del_gid(struct ib_device *device, u8 port_num, @@ -100,24 +103,26 @@ static int hns_roce_del_gid(struct ib_device *device, u8 port_num, union ib_gid zgid = { {0} }; u8 port = port_num - 1; unsigned long flags; + int ret; if (port >= hr_dev->caps.num_ports) return -EINVAL; spin_lock_irqsave(&hr_dev->iboe.lock, flags); - hr_dev->hw->set_gid(hr_dev, port, index, &zgid); + ret = hr_dev->hw->set_gid(hr_dev, port, index, &zgid, NULL); spin_unlock_irqrestore(&hr_dev->iboe.lock, flags); - return 0; + return ret; } static int handle_en_event(struct hns_roce_dev *hr_dev, u8 port, unsigned long event) { - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; struct net_device *netdev; + int ret = 0; netdev = hr_dev->iboe.netdevs[port]; if (!netdev) { @@ -130,7 +135,7 @@ static int handle_en_event(struct hns_roce_dev *hr_dev, u8 port, case NETDEV_CHANGE: case NETDEV_REGISTER: case NETDEV_CHANGEADDR: - hns_roce_set_mac(hr_dev, port, netdev->dev_addr); + ret = hns_roce_set_mac(hr_dev, port, netdev->dev_addr); break; case NETDEV_DOWN: /* @@ -142,7 +147,7 @@ static int handle_en_event(struct hns_roce_dev *hr_dev, u8 port, break; } - return 0; + return ret; } static int hns_roce_netdev_event(struct notifier_block *self, @@ -171,12 +176,17 @@ static int hns_roce_netdev_event(struct notifier_block *self, static int hns_roce_setup_mtu_mac(struct hns_roce_dev *hr_dev) { + int ret; u8 i; for (i = 0; i < hr_dev->caps.num_ports; i++) { - hr_dev->hw->set_mtu(hr_dev, hr_dev->iboe.phy_port[i], - hr_dev->caps.max_mtu); - hns_roce_set_mac(hr_dev, i, hr_dev->iboe.netdevs[i]->dev_addr); + if (hr_dev->hw->set_mtu) + hr_dev->hw->set_mtu(hr_dev, hr_dev->iboe.phy_port[i], + hr_dev->caps.max_mtu); + ret = hns_roce_set_mac(hr_dev, i, + hr_dev->iboe.netdevs[i]->dev_addr); + if (ret) + return ret; } return 0; @@ -200,7 +210,7 @@ static int hns_roce_query_device(struct ib_device *ib_dev, props->max_qp_wr = hr_dev->caps.max_wqes; props->device_cap_flags = IB_DEVICE_PORT_ACTIVE_EVENT | IB_DEVICE_RC_RNR_NAK_GEN; - props->max_sge = hr_dev->caps.max_sq_sg; + props->max_sge = max(hr_dev->caps.max_sq_sg, hr_dev->caps.max_rq_sg); props->max_sge_rd = 1; props->max_cq = hr_dev->caps.num_cqs; props->max_cqe = hr_dev->caps.max_cqes; @@ -238,7 +248,7 @@ static int hns_roce_query_port(struct ib_device *ib_dev, u8 port_num, struct ib_port_attr *props) { struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; struct net_device *net_dev; unsigned long flags; enum ib_mtu mtu; @@ -379,7 +389,8 @@ static int hns_roce_mmap(struct ib_ucontext *context, to_hr_ucontext(context)->uar.pfn, PAGE_SIZE, vma->vm_page_prot)) return -EAGAIN; - } else if (vma->vm_pgoff == 1 && hr_dev->hw_rev == HNS_ROCE_HW_VER1) { + } else if (vma->vm_pgoff == 1 && hr_dev->tptr_dma_addr && + hr_dev->tptr_size) { /* vm_pgoff: 1 -- TPTR */ if (io_remap_pfn_range(vma, vma->vm_start, hr_dev->tptr_dma_addr >> PAGE_SHIFT, @@ -398,8 +409,6 @@ static int hns_roce_port_immutable(struct ib_device *ib_dev, u8 port_num, struct ib_port_attr attr; int ret; - immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; - ret = ib_query_port(ib_dev, port_num, &attr); if (ret) return ret; @@ -408,6 +417,9 @@ static int hns_roce_port_immutable(struct ib_device *ib_dev, u8 port_num, immutable->gid_tbl_len = attr.gid_tbl_len; immutable->max_mad_size = IB_MGMT_MAD_SIZE; + immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; + if (to_hr_dev(ib_dev)->caps.flags & HNS_ROCE_CAP_FLAG_ROCE_V1_V2) + immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; return 0; } @@ -416,7 +428,6 @@ static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev) { struct hns_roce_ib_iboe *iboe = &hr_dev->iboe; - unregister_inetaddr_notifier(&iboe->nb_inet); unregister_netdevice_notifier(&iboe->nb); ib_unregister_device(&hr_dev->ib_dev); } @@ -426,7 +437,7 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev) int ret; struct hns_roce_ib_iboe *iboe = NULL; struct ib_device *ib_dev = NULL; - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; iboe = &hr_dev->iboe; spin_lock_init(&iboe->lock); @@ -492,6 +503,7 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev) /* CQ */ ib_dev->create_cq = hns_roce_ib_create_cq; + ib_dev->modify_cq = hr_dev->hw->modify_cq; ib_dev->destroy_cq = hns_roce_ib_destroy_cq; ib_dev->req_notify_cq = hr_dev->hw->req_notify_cq; ib_dev->poll_cq = hr_dev->hw->poll_cq; @@ -500,6 +512,10 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev) ib_dev->get_dma_mr = hns_roce_get_dma_mr; ib_dev->reg_user_mr = hns_roce_reg_user_mr; ib_dev->dereg_mr = hns_roce_dereg_mr; + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_REREG_MR) { + ib_dev->rereg_user_mr = hns_roce_rereg_user_mr; + ib_dev->uverbs_cmd_mask |= (1ULL << IB_USER_VERBS_CMD_REREG_MR); + } /* OTHERS */ ib_dev->get_port_immutable = hns_roce_port_immutable; @@ -531,173 +547,10 @@ error_failed_setup_mtu_mac: return ret; } -static const struct of_device_id hns_roce_of_match[] = { - { .compatible = "hisilicon,hns-roce-v1", .data = &hns_roce_hw_v1, }, - {}, -}; -MODULE_DEVICE_TABLE(of, hns_roce_of_match); - -static const struct acpi_device_id hns_roce_acpi_match[] = { - { "HISI00D1", (kernel_ulong_t)&hns_roce_hw_v1 }, - {}, -}; -MODULE_DEVICE_TABLE(acpi, hns_roce_acpi_match); - -static int hns_roce_node_match(struct device *dev, void *fwnode) -{ - return dev->fwnode == fwnode; -} - -static struct -platform_device *hns_roce_find_pdev(struct fwnode_handle *fwnode) -{ - struct device *dev; - - /* get the 'device'corresponding to matching 'fwnode' */ - dev = bus_find_device(&platform_bus_type, NULL, - fwnode, hns_roce_node_match); - /* get the platform device */ - return dev ? to_platform_device(dev) : NULL; -} - -static int hns_roce_get_cfg(struct hns_roce_dev *hr_dev) -{ - int i; - int ret; - u8 phy_port; - int port_cnt = 0; - struct device *dev = &hr_dev->pdev->dev; - struct device_node *net_node; - struct net_device *netdev = NULL; - struct platform_device *pdev = NULL; - struct resource *res; - - /* check if we are compatible with the underlying SoC */ - if (dev_of_node(dev)) { - const struct of_device_id *of_id; - - of_id = of_match_node(hns_roce_of_match, dev->of_node); - if (!of_id) { - dev_err(dev, "device is not compatible!\n"); - return -ENXIO; - } - hr_dev->hw = (struct hns_roce_hw *)of_id->data; - if (!hr_dev->hw) { - dev_err(dev, "couldn't get H/W specific DT data!\n"); - return -ENXIO; - } - } else if (is_acpi_device_node(dev->fwnode)) { - const struct acpi_device_id *acpi_id; - - acpi_id = acpi_match_device(hns_roce_acpi_match, dev); - if (!acpi_id) { - dev_err(dev, "device is not compatible!\n"); - return -ENXIO; - } - hr_dev->hw = (struct hns_roce_hw *) acpi_id->driver_data; - if (!hr_dev->hw) { - dev_err(dev, "couldn't get H/W specific ACPI data!\n"); - return -ENXIO; - } - } else { - dev_err(dev, "can't read compatibility data from DT or ACPI\n"); - return -ENXIO; - } - - /* get the mapped register base address */ - res = platform_get_resource(hr_dev->pdev, IORESOURCE_MEM, 0); - if (!res) { - dev_err(dev, "memory resource not found!\n"); - return -EINVAL; - } - hr_dev->reg_base = devm_ioremap_resource(dev, res); - if (IS_ERR(hr_dev->reg_base)) - return PTR_ERR(hr_dev->reg_base); - - /* read the node_guid of IB device from the DT or ACPI */ - ret = device_property_read_u8_array(dev, "node-guid", - (u8 *)&hr_dev->ib_dev.node_guid, - GUID_LEN); - if (ret) { - dev_err(dev, "couldn't get node_guid from DT or ACPI!\n"); - return ret; - } - - /* get the RoCE associated ethernet ports or netdevices */ - for (i = 0; i < HNS_ROCE_MAX_PORTS; i++) { - if (dev_of_node(dev)) { - net_node = of_parse_phandle(dev->of_node, "eth-handle", - i); - if (!net_node) - continue; - pdev = of_find_device_by_node(net_node); - } else if (is_acpi_device_node(dev->fwnode)) { - struct acpi_reference_args args; - struct fwnode_handle *fwnode; - - ret = acpi_node_get_property_reference(dev->fwnode, - "eth-handle", - i, &args); - if (ret) - continue; - fwnode = acpi_fwnode_handle(args.adev); - pdev = hns_roce_find_pdev(fwnode); - } else { - dev_err(dev, "cannot read data from DT or ACPI\n"); - return -ENXIO; - } - - if (pdev) { - netdev = platform_get_drvdata(pdev); - phy_port = (u8)i; - if (netdev) { - hr_dev->iboe.netdevs[port_cnt] = netdev; - hr_dev->iboe.phy_port[port_cnt] = phy_port; - } else { - dev_err(dev, "no netdev found with pdev %s\n", - pdev->name); - return -ENODEV; - } - port_cnt++; - } - } - - if (port_cnt == 0) { - dev_err(dev, "unable to get eth-handle for available ports!\n"); - return -EINVAL; - } - - hr_dev->caps.num_ports = port_cnt; - - /* cmd issue mode: 0 is poll, 1 is event */ - hr_dev->cmd_mod = 1; - hr_dev->loop_idc = 0; - - /* read the interrupt names from the DT or ACPI */ - ret = device_property_read_string_array(dev, "interrupt-names", - hr_dev->irq_names, - HNS_ROCE_MAX_IRQ_NUM); - if (ret < 0) { - dev_err(dev, "couldn't get interrupt names from DT or ACPI!\n"); - return ret; - } - - /* fetch the interrupt numbers */ - for (i = 0; i < HNS_ROCE_MAX_IRQ_NUM; i++) { - hr_dev->irq[i] = platform_get_irq(hr_dev->pdev, i); - if (hr_dev->irq[i] <= 0) { - dev_err(dev, "platform get of irq[=%d] failed!\n", i); - return -EINVAL; - } - } - - return 0; -} - static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) { int ret; - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; ret = hns_roce_init_hem_table(hr_dev, &hr_dev->mr_table.mtt_table, HEM_TYPE_MTT, hr_dev->caps.mtt_entry_sz, @@ -707,6 +560,17 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) return ret; } + if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) { + ret = hns_roce_init_hem_table(hr_dev, + &hr_dev->mr_table.mtt_cqe_table, + HEM_TYPE_CQE, hr_dev->caps.mtt_entry_sz, + hr_dev->caps.num_cqe_segs, 1); + if (ret) { + dev_err(dev, "Failed to init MTT CQE context memory, aborting.\n"); + goto err_unmap_cqe; + } + } + ret = hns_roce_init_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table, HEM_TYPE_MTPT, hr_dev->caps.mtpt_entry_sz, hr_dev->caps.num_mtpts, 1); @@ -733,16 +597,35 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) goto err_unmap_qp; } + if (hr_dev->caps.trrl_entry_sz) { + ret = hns_roce_init_hem_table(hr_dev, + &hr_dev->qp_table.trrl_table, + HEM_TYPE_TRRL, + hr_dev->caps.trrl_entry_sz * + hr_dev->caps.max_qp_dest_rdma, + hr_dev->caps.num_qps, 1); + if (ret) { + dev_err(dev, + "Failed to init trrl_table memory, aborting.\n"); + goto err_unmap_irrl; + } + } + ret = hns_roce_init_hem_table(hr_dev, &hr_dev->cq_table.table, HEM_TYPE_CQC, hr_dev->caps.cqc_entry_sz, hr_dev->caps.num_cqs, 1); if (ret) { dev_err(dev, "Failed to init CQ context memory, aborting.\n"); - goto err_unmap_irrl; + goto err_unmap_trrl; } return 0; +err_unmap_trrl: + if (hr_dev->caps.trrl_entry_sz) + hns_roce_cleanup_hem_table(hr_dev, + &hr_dev->qp_table.trrl_table); + err_unmap_irrl: hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table); @@ -754,6 +637,12 @@ err_unmap_dmpt: err_unmap_mtt: hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table); + if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) + hns_roce_cleanup_hem_table(hr_dev, + &hr_dev->mr_table.mtt_cqe_table); + +err_unmap_cqe: + hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table); return ret; } @@ -766,7 +655,7 @@ err_unmap_mtt: static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev) { int ret; - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; spin_lock_init(&hr_dev->sm_lock); spin_lock_init(&hr_dev->bt_cmd_lock); @@ -826,56 +715,45 @@ err_uar_table_free: return ret; } -/** - * hns_roce_probe - RoCE driver entrance - * @pdev: pointer to platform device - * Return : int - * - */ -static int hns_roce_probe(struct platform_device *pdev) +int hns_roce_init(struct hns_roce_dev *hr_dev) { int ret; - struct hns_roce_dev *hr_dev; - struct device *dev = &pdev->dev; - - hr_dev = (struct hns_roce_dev *)ib_alloc_device(sizeof(*hr_dev)); - if (!hr_dev) - return -ENOMEM; - - hr_dev->pdev = pdev; - platform_set_drvdata(pdev, hr_dev); + struct device *dev = hr_dev->dev; - if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64ULL)) && - dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32ULL))) { - dev_err(dev, "Not usable DMA addressing mode\n"); - ret = -EIO; - goto error_failed_get_cfg; + if (hr_dev->hw->reset) { + ret = hr_dev->hw->reset(hr_dev, true); + if (ret) { + dev_err(dev, "Reset RoCE engine failed!\n"); + return ret; + } } - ret = hns_roce_get_cfg(hr_dev); - if (ret) { - dev_err(dev, "Get Configuration failed!\n"); - goto error_failed_get_cfg; + if (hr_dev->hw->cmq_init) { + ret = hr_dev->hw->cmq_init(hr_dev); + if (ret) { + dev_err(dev, "Init RoCE Command Queue failed!\n"); + goto error_failed_cmq_init; + } } - ret = hr_dev->hw->reset(hr_dev, true); + ret = hr_dev->hw->hw_profile(hr_dev); if (ret) { - dev_err(dev, "Reset RoCE engine failed!\n"); - goto error_failed_get_cfg; + dev_err(dev, "Get RoCE engine profile failed!\n"); + goto error_failed_cmd_init; } - hr_dev->hw->hw_profile(hr_dev); - ret = hns_roce_cmd_init(hr_dev); if (ret) { dev_err(dev, "cmd init failed!\n"); goto error_failed_cmd_init; } - ret = hns_roce_init_eq_table(hr_dev); - if (ret) { - dev_err(dev, "eq init failed!\n"); - goto error_failed_eq_table; + if (hr_dev->cmd_mod) { + ret = hns_roce_init_eq_table(hr_dev); + if (ret) { + dev_err(dev, "eq init failed!\n"); + goto error_failed_eq_table; + } } if (hr_dev->cmd_mod) { @@ -898,10 +776,12 @@ static int hns_roce_probe(struct platform_device *pdev) goto error_failed_setup_hca; } - ret = hr_dev->hw->hw_init(hr_dev); - if (ret) { - dev_err(dev, "hw_init failed!\n"); - goto error_failed_engine_init; + if (hr_dev->hw->hw_init) { + ret = hr_dev->hw->hw_init(hr_dev); + if (ret) { + dev_err(dev, "hw_init failed!\n"); + goto error_failed_engine_init; + } } ret = hns_roce_register_device(hr_dev); @@ -911,7 +791,8 @@ static int hns_roce_probe(struct platform_device *pdev) return 0; error_failed_register_device: - hr_dev->hw->hw_exit(hr_dev); + if (hr_dev->hw->hw_exit) + hr_dev->hw->hw_exit(hr_dev); error_failed_engine_init: hns_roce_cleanup_bitmap(hr_dev); @@ -924,58 +805,47 @@ error_failed_init_hem: hns_roce_cmd_use_polling(hr_dev); error_failed_use_event: - hns_roce_cleanup_eq_table(hr_dev); + if (hr_dev->cmd_mod) + hns_roce_cleanup_eq_table(hr_dev); error_failed_eq_table: hns_roce_cmd_cleanup(hr_dev); error_failed_cmd_init: - ret = hr_dev->hw->reset(hr_dev, false); - if (ret) - dev_err(&hr_dev->pdev->dev, "roce_engine reset fail\n"); + if (hr_dev->hw->cmq_exit) + hr_dev->hw->cmq_exit(hr_dev); -error_failed_get_cfg: - ib_dealloc_device(&hr_dev->ib_dev); +error_failed_cmq_init: + if (hr_dev->hw->reset) { + ret = hr_dev->hw->reset(hr_dev, false); + if (ret) + dev_err(dev, "Dereset RoCE engine failed!\n"); + } return ret; } +EXPORT_SYMBOL_GPL(hns_roce_init); -/** - * hns_roce_remove - remove RoCE device - * @pdev: pointer to platform device - */ -static int hns_roce_remove(struct platform_device *pdev) +void hns_roce_exit(struct hns_roce_dev *hr_dev) { - struct hns_roce_dev *hr_dev = platform_get_drvdata(pdev); - hns_roce_unregister_device(hr_dev); - hr_dev->hw->hw_exit(hr_dev); + if (hr_dev->hw->hw_exit) + hr_dev->hw->hw_exit(hr_dev); hns_roce_cleanup_bitmap(hr_dev); hns_roce_cleanup_hem(hr_dev); if (hr_dev->cmd_mod) hns_roce_cmd_use_polling(hr_dev); - hns_roce_cleanup_eq_table(hr_dev); + if (hr_dev->cmd_mod) + hns_roce_cleanup_eq_table(hr_dev); hns_roce_cmd_cleanup(hr_dev); - hr_dev->hw->reset(hr_dev, false); - - ib_dealloc_device(&hr_dev->ib_dev); - - return 0; + if (hr_dev->hw->cmq_exit) + hr_dev->hw->cmq_exit(hr_dev); + if (hr_dev->hw->reset) + hr_dev->hw->reset(hr_dev, false); } - -static struct platform_driver hns_roce_driver = { - .probe = hns_roce_probe, - .remove = hns_roce_remove, - .driver = { - .name = DRV_NAME, - .of_match_table = hns_roce_of_match, - .acpi_match_table = ACPI_PTR(hns_roce_acpi_match), - }, -}; - -module_platform_driver(hns_roce_driver); +EXPORT_SYMBOL_GPL(hns_roce_exit); MODULE_LICENSE("Dual BSD/GPL"); MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>"); diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index e387360e3780..da86a8117bd5 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -47,6 +47,7 @@ unsigned long key_to_hw_index(u32 key) { return (key << 24) | (key >> 8); } +EXPORT_SYMBOL_GPL(key_to_hw_index); static int hns_roce_sw2hw_mpt(struct hns_roce_dev *hr_dev, struct hns_roce_cmd_mailbox *mailbox, @@ -65,6 +66,7 @@ int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev, mpt_index, !mailbox, HNS_ROCE_CMD_HW2SW_MPT, HNS_ROCE_CMD_TIMEOUT_MSECS); } +EXPORT_SYMBOL_GPL(hns_roce_hw2sw_mpt); static int hns_roce_buddy_alloc(struct hns_roce_buddy *buddy, int order, unsigned long *seg) @@ -175,18 +177,28 @@ static void hns_roce_buddy_cleanup(struct hns_roce_buddy *buddy) } static int hns_roce_alloc_mtt_range(struct hns_roce_dev *hr_dev, int order, - unsigned long *seg) + unsigned long *seg, u32 mtt_type) { struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; - int ret = 0; + struct hns_roce_hem_table *table; + struct hns_roce_buddy *buddy; + int ret; + + if (mtt_type == MTT_TYPE_WQE) { + buddy = &mr_table->mtt_buddy; + table = &mr_table->mtt_table; + } else { + buddy = &mr_table->mtt_cqe_buddy; + table = &mr_table->mtt_cqe_table; + } - ret = hns_roce_buddy_alloc(&mr_table->mtt_buddy, order, seg); + ret = hns_roce_buddy_alloc(buddy, order, seg); if (ret == -1) return -1; - if (hns_roce_table_get_range(hr_dev, &mr_table->mtt_table, *seg, + if (hns_roce_table_get_range(hr_dev, table, *seg, *seg + (1 << order) - 1)) { - hns_roce_buddy_free(&mr_table->mtt_buddy, *seg, order); + hns_roce_buddy_free(buddy, *seg, order); return -1; } @@ -196,7 +208,7 @@ static int hns_roce_alloc_mtt_range(struct hns_roce_dev *hr_dev, int order, int hns_roce_mtt_init(struct hns_roce_dev *hr_dev, int npages, int page_shift, struct hns_roce_mtt *mtt) { - int ret = 0; + int ret; int i; /* Page num is zero, correspond to DMA memory register */ @@ -215,7 +227,8 @@ int hns_roce_mtt_init(struct hns_roce_dev *hr_dev, int npages, int page_shift, ++mtt->order; /* Allocate MTT entry */ - ret = hns_roce_alloc_mtt_range(hr_dev, mtt->order, &mtt->first_seg); + ret = hns_roce_alloc_mtt_range(hr_dev, mtt->order, &mtt->first_seg, + mtt->mtt_type); if (ret == -1) return -ENOMEM; @@ -229,18 +242,261 @@ void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt) if (mtt->order < 0) return; - hns_roce_buddy_free(&mr_table->mtt_buddy, mtt->first_seg, mtt->order); - hns_roce_table_put_range(hr_dev, &mr_table->mtt_table, mtt->first_seg, - mtt->first_seg + (1 << mtt->order) - 1); + if (mtt->mtt_type == MTT_TYPE_WQE) { + hns_roce_buddy_free(&mr_table->mtt_buddy, mtt->first_seg, + mtt->order); + hns_roce_table_put_range(hr_dev, &mr_table->mtt_table, + mtt->first_seg, + mtt->first_seg + (1 << mtt->order) - 1); + } else { + hns_roce_buddy_free(&mr_table->mtt_cqe_buddy, mtt->first_seg, + mtt->order); + hns_roce_table_put_range(hr_dev, &mr_table->mtt_cqe_table, + mtt->first_seg, + mtt->first_seg + (1 << mtt->order) - 1); + } +} +EXPORT_SYMBOL_GPL(hns_roce_mtt_cleanup); + +static void hns_roce_loop_free(struct hns_roce_dev *hr_dev, + struct hns_roce_mr *mr, int err_loop_index, + int loop_i, int loop_j) +{ + struct device *dev = hr_dev->dev; + u32 mhop_num; + u32 pbl_bt_sz; + u64 bt_idx; + int i, j; + + pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT); + mhop_num = hr_dev->caps.pbl_hop_num; + + i = loop_i; + if (mhop_num == 3 && err_loop_index == 2) { + for (; i >= 0; i--) { + dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i], + mr->pbl_l1_dma_addr[i]); + + for (j = 0; j < pbl_bt_sz / 8; j++) { + if (i == loop_i && j >= loop_j) + break; + + bt_idx = i * pbl_bt_sz / 8 + j; + dma_free_coherent(dev, pbl_bt_sz, + mr->pbl_bt_l2[bt_idx], + mr->pbl_l2_dma_addr[bt_idx]); + } + } + } else if (mhop_num == 3 && err_loop_index == 1) { + for (i -= 1; i >= 0; i--) { + dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i], + mr->pbl_l1_dma_addr[i]); + + for (j = 0; j < pbl_bt_sz / 8; j++) { + bt_idx = i * pbl_bt_sz / 8 + j; + dma_free_coherent(dev, pbl_bt_sz, + mr->pbl_bt_l2[bt_idx], + mr->pbl_l2_dma_addr[bt_idx]); + } + } + } else if (mhop_num == 2 && err_loop_index == 1) { + for (i -= 1; i >= 0; i--) + dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i], + mr->pbl_l1_dma_addr[i]); + } else { + dev_warn(dev, "not support: mhop_num=%d, err_loop_index=%d.", + mhop_num, err_loop_index); + return; + } + + dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l0, mr->pbl_l0_dma_addr); + mr->pbl_bt_l0 = NULL; + mr->pbl_l0_dma_addr = 0; +} + +/* PBL multi hop addressing */ +static int hns_roce_mhop_alloc(struct hns_roce_dev *hr_dev, int npages, + struct hns_roce_mr *mr) +{ + struct device *dev = hr_dev->dev; + int mr_alloc_done = 0; + int npages_allocated; + int i = 0, j = 0; + u32 pbl_bt_sz; + u32 mhop_num; + u64 pbl_last_bt_num; + u64 pbl_bt_cnt = 0; + u64 bt_idx; + u64 size; + + mhop_num = hr_dev->caps.pbl_hop_num; + pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT); + pbl_last_bt_num = (npages + pbl_bt_sz / 8 - 1) / (pbl_bt_sz / 8); + + if (mhop_num == HNS_ROCE_HOP_NUM_0) + return 0; + + /* hop_num = 1 */ + if (mhop_num == 1) { + if (npages > pbl_bt_sz / 8) { + dev_err(dev, "npages %d is larger than buf_pg_sz!", + npages); + return -EINVAL; + } + mr->pbl_buf = dma_alloc_coherent(dev, npages * 8, + &(mr->pbl_dma_addr), + GFP_KERNEL); + if (!mr->pbl_buf) + return -ENOMEM; + + mr->pbl_size = npages; + mr->pbl_ba = mr->pbl_dma_addr; + mr->pbl_hop_num = hr_dev->caps.pbl_hop_num; + mr->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz; + mr->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz; + return 0; + } + + mr->pbl_l1_dma_addr = kcalloc(pbl_bt_sz / 8, + sizeof(*mr->pbl_l1_dma_addr), + GFP_KERNEL); + if (!mr->pbl_l1_dma_addr) + return -ENOMEM; + + mr->pbl_bt_l1 = kcalloc(pbl_bt_sz / 8, sizeof(*mr->pbl_bt_l1), + GFP_KERNEL); + if (!mr->pbl_bt_l1) + goto err_kcalloc_bt_l1; + + if (mhop_num == 3) { + mr->pbl_l2_dma_addr = kcalloc(pbl_last_bt_num, + sizeof(*mr->pbl_l2_dma_addr), + GFP_KERNEL); + if (!mr->pbl_l2_dma_addr) + goto err_kcalloc_l2_dma; + + mr->pbl_bt_l2 = kcalloc(pbl_last_bt_num, + sizeof(*mr->pbl_bt_l2), + GFP_KERNEL); + if (!mr->pbl_bt_l2) + goto err_kcalloc_bt_l2; + } + + /* alloc L0 BT */ + mr->pbl_bt_l0 = dma_alloc_coherent(dev, pbl_bt_sz, + &(mr->pbl_l0_dma_addr), + GFP_KERNEL); + if (!mr->pbl_bt_l0) + goto err_dma_alloc_l0; + + if (mhop_num == 2) { + /* alloc L1 BT */ + for (i = 0; i < pbl_bt_sz / 8; i++) { + if (pbl_bt_cnt + 1 < pbl_last_bt_num) { + size = pbl_bt_sz; + } else { + npages_allocated = i * (pbl_bt_sz / 8); + size = (npages - npages_allocated) * 8; + } + mr->pbl_bt_l1[i] = dma_alloc_coherent(dev, size, + &(mr->pbl_l1_dma_addr[i]), + GFP_KERNEL); + if (!mr->pbl_bt_l1[i]) { + hns_roce_loop_free(hr_dev, mr, 1, i, 0); + goto err_dma_alloc_l0; + } + + *(mr->pbl_bt_l0 + i) = mr->pbl_l1_dma_addr[i]; + + pbl_bt_cnt++; + if (pbl_bt_cnt >= pbl_last_bt_num) + break; + } + } else if (mhop_num == 3) { + /* alloc L1, L2 BT */ + for (i = 0; i < pbl_bt_sz / 8; i++) { + mr->pbl_bt_l1[i] = dma_alloc_coherent(dev, pbl_bt_sz, + &(mr->pbl_l1_dma_addr[i]), + GFP_KERNEL); + if (!mr->pbl_bt_l1[i]) { + hns_roce_loop_free(hr_dev, mr, 1, i, 0); + goto err_dma_alloc_l0; + } + + *(mr->pbl_bt_l0 + i) = mr->pbl_l1_dma_addr[i]; + + for (j = 0; j < pbl_bt_sz / 8; j++) { + bt_idx = i * pbl_bt_sz / 8 + j; + + if (pbl_bt_cnt + 1 < pbl_last_bt_num) { + size = pbl_bt_sz; + } else { + npages_allocated = bt_idx * + (pbl_bt_sz / 8); + size = (npages - npages_allocated) * 8; + } + mr->pbl_bt_l2[bt_idx] = dma_alloc_coherent( + dev, size, + &(mr->pbl_l2_dma_addr[bt_idx]), + GFP_KERNEL); + if (!mr->pbl_bt_l2[bt_idx]) { + hns_roce_loop_free(hr_dev, mr, 2, i, j); + goto err_dma_alloc_l0; + } + + *(mr->pbl_bt_l1[i] + j) = + mr->pbl_l2_dma_addr[bt_idx]; + + pbl_bt_cnt++; + if (pbl_bt_cnt >= pbl_last_bt_num) { + mr_alloc_done = 1; + break; + } + } + + if (mr_alloc_done) + break; + } + } + + mr->l0_chunk_last_num = i + 1; + if (mhop_num == 3) + mr->l1_chunk_last_num = j + 1; + + mr->pbl_size = npages; + mr->pbl_ba = mr->pbl_l0_dma_addr; + mr->pbl_hop_num = hr_dev->caps.pbl_hop_num; + mr->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz; + mr->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz; + + return 0; + +err_dma_alloc_l0: + kfree(mr->pbl_bt_l2); + mr->pbl_bt_l2 = NULL; + +err_kcalloc_bt_l2: + kfree(mr->pbl_l2_dma_addr); + mr->pbl_l2_dma_addr = NULL; + +err_kcalloc_l2_dma: + kfree(mr->pbl_bt_l1); + mr->pbl_bt_l1 = NULL; + +err_kcalloc_bt_l1: + kfree(mr->pbl_l1_dma_addr); + mr->pbl_l1_dma_addr = NULL; + + return -ENOMEM; } static int hns_roce_mr_alloc(struct hns_roce_dev *hr_dev, u32 pd, u64 iova, u64 size, u32 access, int npages, struct hns_roce_mr *mr) { + struct device *dev = hr_dev->dev; unsigned long index = 0; int ret = 0; - struct device *dev = &hr_dev->pdev->dev; /* Allocate a key for mr from mr_table */ ret = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &index); @@ -258,22 +514,117 @@ static int hns_roce_mr_alloc(struct hns_roce_dev *hr_dev, u32 pd, u64 iova, mr->type = MR_TYPE_DMA; mr->pbl_buf = NULL; mr->pbl_dma_addr = 0; + /* PBL multi-hop addressing parameters */ + mr->pbl_bt_l2 = NULL; + mr->pbl_bt_l1 = NULL; + mr->pbl_bt_l0 = NULL; + mr->pbl_l2_dma_addr = NULL; + mr->pbl_l1_dma_addr = NULL; + mr->pbl_l0_dma_addr = 0; } else { mr->type = MR_TYPE_MR; - mr->pbl_buf = dma_alloc_coherent(dev, npages * 8, - &(mr->pbl_dma_addr), - GFP_KERNEL); - if (!mr->pbl_buf) - return -ENOMEM; + if (!hr_dev->caps.pbl_hop_num) { + mr->pbl_buf = dma_alloc_coherent(dev, npages * 8, + &(mr->pbl_dma_addr), + GFP_KERNEL); + if (!mr->pbl_buf) + return -ENOMEM; + } else { + ret = hns_roce_mhop_alloc(hr_dev, npages, mr); + } } - return 0; + return ret; +} + +static void hns_roce_mhop_free(struct hns_roce_dev *hr_dev, + struct hns_roce_mr *mr) +{ + struct device *dev = hr_dev->dev; + int npages_allocated; + int npages; + int i, j; + u32 pbl_bt_sz; + u32 mhop_num; + u64 bt_idx; + + npages = ib_umem_page_count(mr->umem); + pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT); + mhop_num = hr_dev->caps.pbl_hop_num; + + if (mhop_num == HNS_ROCE_HOP_NUM_0) + return; + + /* hop_num = 1 */ + if (mhop_num == 1) { + dma_free_coherent(dev, (unsigned int)(npages * 8), + mr->pbl_buf, mr->pbl_dma_addr); + return; + } + + dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l0, + mr->pbl_l0_dma_addr); + + if (mhop_num == 2) { + for (i = 0; i < mr->l0_chunk_last_num; i++) { + if (i == mr->l0_chunk_last_num - 1) { + npages_allocated = i * (pbl_bt_sz / 8); + + dma_free_coherent(dev, + (npages - npages_allocated) * 8, + mr->pbl_bt_l1[i], + mr->pbl_l1_dma_addr[i]); + + break; + } + + dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i], + mr->pbl_l1_dma_addr[i]); + } + } else if (mhop_num == 3) { + for (i = 0; i < mr->l0_chunk_last_num; i++) { + dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i], + mr->pbl_l1_dma_addr[i]); + + for (j = 0; j < pbl_bt_sz / 8; j++) { + bt_idx = i * (pbl_bt_sz / 8) + j; + + if ((i == mr->l0_chunk_last_num - 1) + && j == mr->l1_chunk_last_num - 1) { + npages_allocated = bt_idx * + (pbl_bt_sz / 8); + + dma_free_coherent(dev, + (npages - npages_allocated) * 8, + mr->pbl_bt_l2[bt_idx], + mr->pbl_l2_dma_addr[bt_idx]); + + break; + } + + dma_free_coherent(dev, pbl_bt_sz, + mr->pbl_bt_l2[bt_idx], + mr->pbl_l2_dma_addr[bt_idx]); + } + } + } + + kfree(mr->pbl_bt_l1); + kfree(mr->pbl_l1_dma_addr); + mr->pbl_bt_l1 = NULL; + mr->pbl_l1_dma_addr = NULL; + if (mhop_num == 3) { + kfree(mr->pbl_bt_l2); + kfree(mr->pbl_l2_dma_addr); + mr->pbl_bt_l2 = NULL; + mr->pbl_l2_dma_addr = NULL; + } } static void hns_roce_mr_free(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr) { - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; int npages = 0; int ret; @@ -286,10 +637,18 @@ static void hns_roce_mr_free(struct hns_roce_dev *hr_dev, if (mr->size != ~0ULL) { npages = ib_umem_page_count(mr->umem); - dma_free_coherent(dev, (unsigned int)(npages * 8), mr->pbl_buf, - mr->pbl_dma_addr); + + if (!hr_dev->caps.pbl_hop_num) + dma_free_coherent(dev, (unsigned int)(npages * 8), + mr->pbl_buf, mr->pbl_dma_addr); + else + hns_roce_mhop_free(hr_dev, mr); } + if (mr->enabled) + hns_roce_table_put(hr_dev, &hr_dev->mr_table.mtpt_table, + key_to_hw_index(mr->key)); + hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap, key_to_hw_index(mr->key), BITMAP_NO_RR); } @@ -299,7 +658,7 @@ static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev, { int ret; unsigned long mtpt_idx = key_to_hw_index(mr->key); - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; struct hns_roce_cmd_mailbox *mailbox; struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; @@ -345,28 +704,44 @@ static int hns_roce_write_mtt_chunk(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt, u32 start_index, u32 npages, u64 *page_list) { - u32 i = 0; - __le64 *mtts = NULL; + struct hns_roce_hem_table *table; dma_addr_t dma_handle; + __le64 *mtts; u32 s = start_index * sizeof(u64); + u32 bt_page_size; + u32 i; + + if (mtt->mtt_type == MTT_TYPE_WQE) + bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT); + else + bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT); /* All MTTs must fit in the same page */ - if (start_index / (PAGE_SIZE / sizeof(u64)) != - (start_index + npages - 1) / (PAGE_SIZE / sizeof(u64))) + if (start_index / (bt_page_size / sizeof(u64)) != + (start_index + npages - 1) / (bt_page_size / sizeof(u64))) return -EINVAL; if (start_index & (HNS_ROCE_MTT_ENTRY_PER_SEG - 1)) return -EINVAL; - mtts = hns_roce_table_find(&hr_dev->mr_table.mtt_table, + if (mtt->mtt_type == MTT_TYPE_WQE) + table = &hr_dev->mr_table.mtt_table; + else + table = &hr_dev->mr_table.mtt_cqe_table; + + mtts = hns_roce_table_find(hr_dev, table, mtt->first_seg + s / hr_dev->caps.mtt_entry_sz, &dma_handle); if (!mtts) return -ENOMEM; /* Save page addr, low 12 bits : 0 */ - for (i = 0; i < npages; ++i) - mtts[i] = (cpu_to_le64(page_list[i])) >> PAGE_ADDR_SHIFT; + for (i = 0; i < npages; ++i) { + if (!hr_dev->caps.mtt_hop_num) + mtts[i] = cpu_to_le64(page_list[i] >> PAGE_ADDR_SHIFT); + else + mtts[i] = cpu_to_le64(page_list[i]); + } return 0; } @@ -377,12 +752,18 @@ static int hns_roce_write_mtt(struct hns_roce_dev *hr_dev, { int chunk; int ret; + u32 bt_page_size; if (mtt->order < 0) return -EINVAL; + if (mtt->mtt_type == MTT_TYPE_WQE) + bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT); + else + bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT); + while (npages > 0) { - chunk = min_t(int, PAGE_SIZE / sizeof(u64), npages); + chunk = min_t(int, bt_page_size / sizeof(u64), npages); ret = hns_roce_write_mtt_chunk(hr_dev, mtt, start_index, chunk, page_list); @@ -400,9 +781,9 @@ static int hns_roce_write_mtt(struct hns_roce_dev *hr_dev, int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt, struct hns_roce_buf *buf) { - u32 i = 0; - int ret = 0; - u64 *page_list = NULL; + u64 *page_list; + int ret; + u32 i; page_list = kmalloc_array(buf->npages, sizeof(*page_list), GFP_KERNEL); if (!page_list) @@ -425,7 +806,7 @@ int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev, int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev) { struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; - int ret = 0; + int ret; ret = hns_roce_bitmap_init(&mr_table->mtpt_bitmap, hr_dev->caps.num_mtpts, @@ -439,8 +820,17 @@ int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev) if (ret) goto err_buddy; + if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) { + ret = hns_roce_buddy_init(&mr_table->mtt_cqe_buddy, + ilog2(hr_dev->caps.num_cqe_segs)); + if (ret) + goto err_buddy_cqe; + } return 0; +err_buddy_cqe: + hns_roce_buddy_cleanup(&mr_table->mtt_buddy); + err_buddy: hns_roce_bitmap_cleanup(&mr_table->mtpt_bitmap); return ret; @@ -451,13 +841,15 @@ void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev) struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; hns_roce_buddy_cleanup(&mr_table->mtt_buddy); + if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) + hns_roce_buddy_cleanup(&mr_table->mtt_cqe_buddy); hns_roce_bitmap_cleanup(&mr_table->mtpt_bitmap); } struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc) { - int ret = 0; - struct hns_roce_mr *mr = NULL; + struct hns_roce_mr *mr; + int ret; mr = kmalloc(sizeof(*mr), GFP_KERNEL); if (mr == NULL) @@ -489,25 +881,44 @@ err_free: int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt, struct ib_umem *umem) { + struct device *dev = hr_dev->dev; struct scatterlist *sg; + unsigned int order; int i, k, entry; + int npage = 0; int ret = 0; + int len; + u64 page_addr; u64 *pages; + u32 bt_page_size; u32 n; - int len; - pages = (u64 *) __get_free_page(GFP_KERNEL); + order = mtt->mtt_type == MTT_TYPE_WQE ? hr_dev->caps.mtt_ba_pg_sz : + hr_dev->caps.cqe_ba_pg_sz; + bt_page_size = 1 << (order + PAGE_SHIFT); + + pages = (u64 *) __get_free_pages(GFP_KERNEL, order); if (!pages) return -ENOMEM; i = n = 0; for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { - len = sg_dma_len(sg) >> mtt->page_shift; + len = sg_dma_len(sg) >> PAGE_SHIFT; for (k = 0; k < len; ++k) { - pages[i++] = sg_dma_address(sg) + - (k << umem->page_shift); - if (i == PAGE_SIZE / sizeof(u64)) { + page_addr = + sg_dma_address(sg) + (k << umem->page_shift); + if (!(npage % (1 << (mtt->page_shift - PAGE_SHIFT)))) { + if (page_addr & ((1 << mtt->page_shift) - 1)) { + dev_err(dev, "page_addr 0x%llx is not page_shift %d alignment!\n", + page_addr, mtt->page_shift); + ret = -EINVAL; + goto out; + } + pages[i++] = page_addr; + } + npage++; + if (i == bt_page_size / sizeof(u64)) { ret = hns_roce_write_mtt(hr_dev, mtt, n, i, pages); if (ret) @@ -526,16 +937,44 @@ out: return ret; } -static int hns_roce_ib_umem_write_mr(struct hns_roce_mr *mr, +static int hns_roce_ib_umem_write_mr(struct hns_roce_dev *hr_dev, + struct hns_roce_mr *mr, struct ib_umem *umem) { - int i = 0; - int entry; struct scatterlist *sg; + int i = 0, j = 0, k; + int entry; + int len; + u64 page_addr; + u32 pbl_bt_sz; + + if (hr_dev->caps.pbl_hop_num == HNS_ROCE_HOP_NUM_0) + return 0; + pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT); for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { - mr->pbl_buf[i] = ((u64)sg_dma_address(sg)) >> 12; - i++; + len = sg_dma_len(sg) >> PAGE_SHIFT; + for (k = 0; k < len; ++k) { + page_addr = sg_dma_address(sg) + + (k << umem->page_shift); + + if (!hr_dev->caps.pbl_hop_num) { + mr->pbl_buf[i++] = page_addr >> 12; + } else if (hr_dev->caps.pbl_hop_num == 1) { + mr->pbl_buf[i++] = page_addr; + } else { + if (hr_dev->caps.pbl_hop_num == 2) + mr->pbl_bt_l1[i][j] = page_addr; + else if (hr_dev->caps.pbl_hop_num == 3) + mr->pbl_bt_l2[i][j] = page_addr; + + j++; + if (j >= (pbl_bt_sz / 8)) { + i++; + j = 0; + } + } + } } /* Memory barrier */ @@ -549,10 +988,12 @@ struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, struct ib_udata *udata) { struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); - struct device *dev = &hr_dev->pdev->dev; - struct hns_roce_mr *mr = NULL; - int ret = 0; - int n = 0; + struct device *dev = hr_dev->dev; + struct hns_roce_mr *mr; + int bt_size; + int ret; + int n; + int i; mr = kmalloc(sizeof(*mr), GFP_KERNEL); if (!mr) @@ -573,11 +1014,27 @@ struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, goto err_umem; } - if (n > HNS_ROCE_MAX_MTPT_PBL_NUM) { - dev_err(dev, " MR len %lld err. MR is limited to 4G at most!\n", - length); - ret = -EINVAL; - goto err_umem; + if (!hr_dev->caps.pbl_hop_num) { + if (n > HNS_ROCE_MAX_MTPT_PBL_NUM) { + dev_err(dev, + " MR len %lld err. MR is limited to 4G at most!\n", + length); + ret = -EINVAL; + goto err_umem; + } + } else { + int pbl_size = 1; + + bt_size = (1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT)) / 8; + for (i = 0; i < hr_dev->caps.pbl_hop_num; i++) + pbl_size *= bt_size; + if (n > pbl_size) { + dev_err(dev, + " MR len %lld err. MR page num is limited to %d!\n", + length, pbl_size); + ret = -EINVAL; + goto err_umem; + } } ret = hns_roce_mr_alloc(hr_dev, to_hr_pd(pd)->pdn, virt_addr, length, @@ -585,7 +1042,7 @@ struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, if (ret) goto err_umem; - ret = hns_roce_ib_umem_write_mr(mr, mr->umem); + ret = hns_roce_ib_umem_write_mr(hr_dev, mr, mr->umem); if (ret) goto err_mr; @@ -608,6 +1065,129 @@ err_free: return ERR_PTR(ret); } +int hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, u64 length, + u64 virt_addr, int mr_access_flags, struct ib_pd *pd, + struct ib_udata *udata) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device); + struct hns_roce_mr *mr = to_hr_mr(ibmr); + struct hns_roce_cmd_mailbox *mailbox; + struct device *dev = hr_dev->dev; + unsigned long mtpt_idx; + u32 pdn = 0; + int npages; + int ret; + + if (!mr->enabled) + return -EINVAL; + + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + mtpt_idx = key_to_hw_index(mr->key) & (hr_dev->caps.num_mtpts - 1); + ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, mtpt_idx, 0, + HNS_ROCE_CMD_QUERY_MPT, + HNS_ROCE_CMD_TIMEOUT_MSECS); + if (ret) + goto free_cmd_mbox; + + ret = hns_roce_hw2sw_mpt(hr_dev, NULL, mtpt_idx); + if (ret) + dev_warn(dev, "HW2SW_MPT failed (%d)\n", ret); + + mr->enabled = 0; + + if (flags & IB_MR_REREG_PD) + pdn = to_hr_pd(pd)->pdn; + + if (flags & IB_MR_REREG_TRANS) { + if (mr->size != ~0ULL) { + npages = ib_umem_page_count(mr->umem); + + if (hr_dev->caps.pbl_hop_num) + hns_roce_mhop_free(hr_dev, mr); + else + dma_free_coherent(dev, npages * 8, mr->pbl_buf, + mr->pbl_dma_addr); + } + ib_umem_release(mr->umem); + + mr->umem = ib_umem_get(ibmr->uobject->context, start, length, + mr_access_flags, 0); + if (IS_ERR(mr->umem)) { + ret = PTR_ERR(mr->umem); + mr->umem = NULL; + goto free_cmd_mbox; + } + npages = ib_umem_page_count(mr->umem); + + if (hr_dev->caps.pbl_hop_num) { + ret = hns_roce_mhop_alloc(hr_dev, npages, mr); + if (ret) + goto release_umem; + } else { + mr->pbl_buf = dma_alloc_coherent(dev, npages * 8, + &(mr->pbl_dma_addr), + GFP_KERNEL); + if (!mr->pbl_buf) { + ret = -ENOMEM; + goto release_umem; + } + } + } + + ret = hr_dev->hw->rereg_write_mtpt(hr_dev, mr, flags, pdn, + mr_access_flags, virt_addr, + length, mailbox->buf); + if (ret) { + if (flags & IB_MR_REREG_TRANS) + goto release_umem; + else + goto free_cmd_mbox; + } + + if (flags & IB_MR_REREG_TRANS) { + ret = hns_roce_ib_umem_write_mr(hr_dev, mr, mr->umem); + if (ret) { + if (mr->size != ~0ULL) { + npages = ib_umem_page_count(mr->umem); + + if (hr_dev->caps.pbl_hop_num) + hns_roce_mhop_free(hr_dev, mr); + else + dma_free_coherent(dev, npages * 8, + mr->pbl_buf, + mr->pbl_dma_addr); + } + + goto release_umem; + } + } + + ret = hns_roce_sw2hw_mpt(hr_dev, mailbox, mtpt_idx); + if (ret) { + dev_err(dev, "SW2HW_MPT failed (%d)\n", ret); + goto release_umem; + } + + mr->enabled = 1; + if (flags & IB_MR_REREG_ACCESS) + mr->access = mr_access_flags; + + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + + return 0; + +release_umem: + ib_umem_release(mr->umem); + +free_cmd_mbox: + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + + return ret; +} + int hns_roce_dereg_mr(struct ib_mr *ibmr) { struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device); diff --git a/drivers/infiniband/hw/hns/hns_roce_pd.c b/drivers/infiniband/hw/hns/hns_roce_pd.c index a64500fa1145..bdab2188c04a 100644 --- a/drivers/infiniband/hw/hns/hns_roce_pd.c +++ b/drivers/infiniband/hw/hns/hns_roce_pd.c @@ -31,6 +31,7 @@ */ #include <linux/platform_device.h> +#include <linux/pci.h> #include "hns_roce_device.h" static int hns_roce_pd_alloc(struct hns_roce_dev *hr_dev, unsigned long *pdn) @@ -60,7 +61,7 @@ struct ib_pd *hns_roce_alloc_pd(struct ib_device *ib_dev, struct ib_udata *udata) { struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; struct hns_roce_pd *pd; int ret; @@ -86,6 +87,7 @@ struct ib_pd *hns_roce_alloc_pd(struct ib_device *ib_dev, return &pd->ibpd; } +EXPORT_SYMBOL_GPL(hns_roce_alloc_pd); int hns_roce_dealloc_pd(struct ib_pd *pd) { @@ -94,6 +96,7 @@ int hns_roce_dealloc_pd(struct ib_pd *pd) return 0; } +EXPORT_SYMBOL_GPL(hns_roce_dealloc_pd); int hns_roce_uar_alloc(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar) { @@ -109,12 +112,17 @@ int hns_roce_uar_alloc(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar) uar->index = (uar->index - 1) % (hr_dev->caps.phy_num_uars - 1) + 1; - res = platform_get_resource(hr_dev->pdev, IORESOURCE_MEM, 0); - if (!res) { - dev_err(&hr_dev->pdev->dev, "memory resource not found!\n"); - return -EINVAL; + if (!dev_is_pci(hr_dev->dev)) { + res = platform_get_resource(hr_dev->pdev, IORESOURCE_MEM, 0); + if (!res) { + dev_err(&hr_dev->pdev->dev, "memory resource not found!\n"); + return -EINVAL; + } + uar->pfn = ((res->start) >> PAGE_SHIFT) + uar->index; + } else { + uar->pfn = ((pci_resource_start(hr_dev->pci_dev, 2)) + >> PAGE_SHIFT); } - uar->pfn = ((res->start) >> PAGE_SHIFT) + uar->index; return 0; } diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index f5dd21c2d275..49586ec8126a 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -44,7 +44,7 @@ void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type) { struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; struct hns_roce_qp *qp; spin_lock(&qp_table->lock); @@ -136,6 +136,7 @@ enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state) return HNS_ROCE_QP_NUM_STATE; } } +EXPORT_SYMBOL_GPL(to_hns_roce_state); static int hns_roce_gsi_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn, struct hns_roce_qp *hr_qp) @@ -153,7 +154,7 @@ static int hns_roce_gsi_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn, hr_qp->qpn & (hr_dev->caps.num_qps - 1), hr_qp); spin_unlock_irq(&qp_table->lock); if (ret) { - dev_err(&hr_dev->pdev->dev, "QPC radix_tree_insert failed\n"); + dev_err(hr_dev->dev, "QPC radix_tree_insert failed\n"); goto err_put_irrl; } @@ -171,7 +172,7 @@ static int hns_roce_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn, struct hns_roce_qp *hr_qp) { struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; int ret; if (!qpn) @@ -193,13 +194,23 @@ static int hns_roce_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn, goto err_put_qp; } + if (hr_dev->caps.trrl_entry_sz) { + /* Alloc memory for TRRL */ + ret = hns_roce_table_get(hr_dev, &qp_table->trrl_table, + hr_qp->qpn); + if (ret) { + dev_err(dev, "TRRL table get failed\n"); + goto err_put_irrl; + } + } + spin_lock_irq(&qp_table->lock); ret = radix_tree_insert(&hr_dev->qp_table_tree, hr_qp->qpn & (hr_dev->caps.num_qps - 1), hr_qp); spin_unlock_irq(&qp_table->lock); if (ret) { dev_err(dev, "QPC radix_tree_insert failed\n"); - goto err_put_irrl; + goto err_put_trrl; } atomic_set(&hr_qp->refcount, 1); @@ -207,6 +218,10 @@ static int hns_roce_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn, return 0; +err_put_trrl: + if (hr_dev->caps.trrl_entry_sz) + hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); + err_put_irrl: hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); @@ -227,6 +242,7 @@ void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) hr_qp->qpn & (hr_dev->caps.num_qps - 1)); spin_unlock_irqrestore(&qp_table->lock, flags); } +EXPORT_SYMBOL_GPL(hns_roce_qp_remove); void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) { @@ -237,10 +253,14 @@ void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) wait_for_completion(&hr_qp->free); if ((hr_qp->ibqp.qp_type) != IB_QPT_GSI) { + if (hr_dev->caps.trrl_entry_sz) + hns_roce_table_put(hr_dev, &qp_table->trrl_table, + hr_qp->qpn); hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn); } } +EXPORT_SYMBOL_GPL(hns_roce_qp_free); void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn, int cnt) @@ -252,13 +272,14 @@ void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn, hns_roce_bitmap_free_range(&qp_table->bitmap, base_qpn, cnt, BITMAP_RR); } +EXPORT_SYMBOL_GPL(hns_roce_release_range_qp); static int hns_roce_set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap, int is_user, int has_srq, struct hns_roce_qp *hr_qp) { + struct device *dev = hr_dev->dev; u32 max_cnt; - struct device *dev = &hr_dev->pdev->dev; /* Check the validity of QP support capacity */ if (cap->max_recv_wr > hr_dev->caps.max_wqes || @@ -282,20 +303,27 @@ static int hns_roce_set_rq_size(struct hns_roce_dev *hr_dev, return -EINVAL; } - /* In v1 engine, parameter verification procession */ - max_cnt = cap->max_recv_wr > HNS_ROCE_MIN_WQE_NUM ? - cap->max_recv_wr : HNS_ROCE_MIN_WQE_NUM; + if (hr_dev->caps.min_wqes) + max_cnt = max(cap->max_recv_wr, hr_dev->caps.min_wqes); + else + max_cnt = cap->max_recv_wr; + hr_qp->rq.wqe_cnt = roundup_pow_of_two(max_cnt); if ((u32)hr_qp->rq.wqe_cnt > hr_dev->caps.max_wqes) { - dev_err(dev, "hns_roce_set_rq_size rq.wqe_cnt too large\n"); + dev_err(dev, "while setting rq size, rq.wqe_cnt too large\n"); return -EINVAL; } max_cnt = max(1U, cap->max_recv_sge); hr_qp->rq.max_gs = roundup_pow_of_two(max_cnt); - /* WQE is fixed for 64B */ - hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz); + if (hr_dev->caps.max_rq_sg <= 2) + hr_qp->rq.wqe_shift = + ilog2(hr_dev->caps.max_rq_desc_sz); + else + hr_qp->rq.wqe_shift = + ilog2(hr_dev->caps.max_rq_desc_sz + * hr_qp->rq.max_gs); } cap->max_recv_wr = hr_qp->rq.max_post = hr_qp->rq.wqe_cnt; @@ -305,32 +333,79 @@ static int hns_roce_set_rq_size(struct hns_roce_dev *hr_dev, } static int hns_roce_set_user_sq_size(struct hns_roce_dev *hr_dev, + struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp, struct hns_roce_ib_create_qp *ucmd) { u32 roundup_sq_stride = roundup_pow_of_two(hr_dev->caps.max_sq_desc_sz); u8 max_sq_stride = ilog2(roundup_sq_stride); + u32 page_size; + u32 max_cnt; /* Sanity check SQ size before proceeding */ if ((u32)(1 << ucmd->log_sq_bb_count) > hr_dev->caps.max_wqes || ucmd->log_sq_stride > max_sq_stride || ucmd->log_sq_stride < HNS_ROCE_IB_MIN_SQ_STRIDE) { - dev_err(&hr_dev->pdev->dev, "check SQ size error!\n"); + dev_err(hr_dev->dev, "check SQ size error!\n"); + return -EINVAL; + } + + if (cap->max_send_sge > hr_dev->caps.max_sq_sg) { + dev_err(hr_dev->dev, "SQ sge error! max_send_sge=%d\n", + cap->max_send_sge); return -EINVAL; } hr_qp->sq.wqe_cnt = 1 << ucmd->log_sq_bb_count; hr_qp->sq.wqe_shift = ucmd->log_sq_stride; + max_cnt = max(1U, cap->max_send_sge); + if (hr_dev->caps.max_sq_sg <= 2) + hr_qp->sq.max_gs = roundup_pow_of_two(max_cnt); + else + hr_qp->sq.max_gs = max_cnt; + + if (hr_qp->sq.max_gs > 2) + hr_qp->sge.sge_cnt = roundup_pow_of_two(hr_qp->sq.wqe_cnt * + (hr_qp->sq.max_gs - 2)); + hr_qp->sge.sge_shift = 4; + /* Get buf size, SQ and RQ are aligned to page_szie */ - hr_qp->buff_size = HNS_ROCE_ALOGN_UP((hr_qp->rq.wqe_cnt << + if (hr_dev->caps.max_sq_sg <= 2) { + hr_qp->buff_size = HNS_ROCE_ALOGN_UP((hr_qp->rq.wqe_cnt << hr_qp->rq.wqe_shift), PAGE_SIZE) + - HNS_ROCE_ALOGN_UP((hr_qp->sq.wqe_cnt << + HNS_ROCE_ALOGN_UP((hr_qp->sq.wqe_cnt << hr_qp->sq.wqe_shift), PAGE_SIZE); - hr_qp->sq.offset = 0; - hr_qp->rq.offset = HNS_ROCE_ALOGN_UP((hr_qp->sq.wqe_cnt << + hr_qp->sq.offset = 0; + hr_qp->rq.offset = HNS_ROCE_ALOGN_UP((hr_qp->sq.wqe_cnt << hr_qp->sq.wqe_shift), PAGE_SIZE); + } else { + page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); + hr_qp->buff_size = HNS_ROCE_ALOGN_UP((hr_qp->rq.wqe_cnt << + hr_qp->rq.wqe_shift), page_size) + + HNS_ROCE_ALOGN_UP((hr_qp->sge.sge_cnt << + hr_qp->sge.sge_shift), page_size) + + HNS_ROCE_ALOGN_UP((hr_qp->sq.wqe_cnt << + hr_qp->sq.wqe_shift), page_size); + + hr_qp->sq.offset = 0; + if (hr_qp->sge.sge_cnt) { + hr_qp->sge.offset = HNS_ROCE_ALOGN_UP( + (hr_qp->sq.wqe_cnt << + hr_qp->sq.wqe_shift), + page_size); + hr_qp->rq.offset = hr_qp->sge.offset + + HNS_ROCE_ALOGN_UP((hr_qp->sge.sge_cnt << + hr_qp->sge.sge_shift), + page_size); + } else { + hr_qp->rq.offset = HNS_ROCE_ALOGN_UP( + (hr_qp->sq.wqe_cnt << + hr_qp->sq.wqe_shift), + page_size); + } + } return 0; } @@ -339,13 +414,15 @@ static int hns_roce_set_kernel_sq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp) { - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; + u32 page_size; u32 max_cnt; + int size; if (cap->max_send_wr > hr_dev->caps.max_wqes || cap->max_send_sge > hr_dev->caps.max_sq_sg || cap->max_inline_data > hr_dev->caps.max_sq_inline) { - dev_err(dev, "hns_roce_set_kernel_sq_size error1\n"); + dev_err(dev, "SQ WR or sge or inline data error!\n"); return -EINVAL; } @@ -353,27 +430,46 @@ static int hns_roce_set_kernel_sq_size(struct hns_roce_dev *hr_dev, hr_qp->sq_max_wqes_per_wr = 1; hr_qp->sq_spare_wqes = 0; - /* In v1 engine, parameter verification procession */ - max_cnt = cap->max_send_wr > HNS_ROCE_MIN_WQE_NUM ? - cap->max_send_wr : HNS_ROCE_MIN_WQE_NUM; + if (hr_dev->caps.min_wqes) + max_cnt = max(cap->max_send_wr, hr_dev->caps.min_wqes); + else + max_cnt = cap->max_send_wr; + hr_qp->sq.wqe_cnt = roundup_pow_of_two(max_cnt); if ((u32)hr_qp->sq.wqe_cnt > hr_dev->caps.max_wqes) { - dev_err(dev, "hns_roce_set_kernel_sq_size sq.wqe_cnt too large\n"); + dev_err(dev, "while setting kernel sq size, sq.wqe_cnt too large\n"); return -EINVAL; } /* Get data_seg numbers */ max_cnt = max(1U, cap->max_send_sge); - hr_qp->sq.max_gs = roundup_pow_of_two(max_cnt); + if (hr_dev->caps.max_sq_sg <= 2) + hr_qp->sq.max_gs = roundup_pow_of_two(max_cnt); + else + hr_qp->sq.max_gs = max_cnt; - /* Get buf size, SQ and RQ are aligned to page_szie */ - hr_qp->buff_size = HNS_ROCE_ALOGN_UP((hr_qp->rq.wqe_cnt << - hr_qp->rq.wqe_shift), PAGE_SIZE) + - HNS_ROCE_ALOGN_UP((hr_qp->sq.wqe_cnt << - hr_qp->sq.wqe_shift), PAGE_SIZE); + if (hr_qp->sq.max_gs > 2) { + hr_qp->sge.sge_cnt = roundup_pow_of_two(hr_qp->sq.wqe_cnt * + (hr_qp->sq.max_gs - 2)); + hr_qp->sge.sge_shift = 4; + } + + /* Get buf size, SQ and RQ are aligned to PAGE_SIZE */ + page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); hr_qp->sq.offset = 0; - hr_qp->rq.offset = HNS_ROCE_ALOGN_UP((hr_qp->sq.wqe_cnt << - hr_qp->sq.wqe_shift), PAGE_SIZE); + size = HNS_ROCE_ALOGN_UP(hr_qp->sq.wqe_cnt << hr_qp->sq.wqe_shift, + page_size); + + if (hr_dev->caps.max_sq_sg > 2 && hr_qp->sge.sge_cnt) { + hr_qp->sge.offset = size; + size += HNS_ROCE_ALOGN_UP(hr_qp->sge.sge_cnt << + hr_qp->sge.sge_shift, page_size); + } + + hr_qp->rq.offset = size; + size += HNS_ROCE_ALOGN_UP((hr_qp->rq.wqe_cnt << hr_qp->rq.wqe_shift), + page_size); + hr_qp->buff_size = size; /* Get wr and sge number which send */ cap->max_send_wr = hr_qp->sq.max_post = hr_qp->sq.wqe_cnt; @@ -391,10 +487,12 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, struct ib_udata *udata, unsigned long sqpn, struct hns_roce_qp *hr_qp) { - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; struct hns_roce_ib_create_qp ucmd; unsigned long qpn = 0; int ret = 0; + u32 page_shift; + u32 npages; mutex_init(&hr_qp->mutex); spin_lock_init(&hr_qp->sq.lock); @@ -421,7 +519,8 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, goto err_out; } - ret = hns_roce_set_user_sq_size(hr_dev, hr_qp, &ucmd); + ret = hns_roce_set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, + &ucmd); if (ret) { dev_err(dev, "hns_roce_set_user_sq_size error for create qp\n"); goto err_out; @@ -436,8 +535,21 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, goto err_out; } - ret = hns_roce_mtt_init(hr_dev, ib_umem_page_count(hr_qp->umem), - hr_qp->umem->page_shift, &hr_qp->mtt); + hr_qp->mtt.mtt_type = MTT_TYPE_WQE; + if (hr_dev->caps.mtt_buf_pg_sz) { + npages = (ib_umem_page_count(hr_qp->umem) + + (1 << hr_dev->caps.mtt_buf_pg_sz) - 1) / + (1 << hr_dev->caps.mtt_buf_pg_sz); + page_shift = PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz; + ret = hns_roce_mtt_init(hr_dev, npages, + page_shift, + &hr_qp->mtt); + } else { + ret = hns_roce_mtt_init(hr_dev, + ib_umem_page_count(hr_qp->umem), + hr_qp->umem->page_shift, + &hr_qp->mtt); + } if (ret) { dev_err(dev, "hns_roce_mtt_init error for create qp\n"); goto err_buf; @@ -472,20 +584,22 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, } /* QP doorbell register address */ - hr_qp->sq.db_reg_l = hr_dev->reg_base + ROCEE_DB_SQ_L_0_REG + + hr_qp->sq.db_reg_l = hr_dev->reg_base + hr_dev->sdb_offset + DB_REG_OFFSET * hr_dev->priv_uar.index; - hr_qp->rq.db_reg_l = hr_dev->reg_base + - ROCEE_DB_OTHERS_L_0_REG + + hr_qp->rq.db_reg_l = hr_dev->reg_base + hr_dev->odb_offset + DB_REG_OFFSET * hr_dev->priv_uar.index; /* Allocate QP buf */ - if (hns_roce_buf_alloc(hr_dev, hr_qp->buff_size, PAGE_SIZE * 2, - &hr_qp->hr_buf)) { + page_shift = PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz; + if (hns_roce_buf_alloc(hr_dev, hr_qp->buff_size, + (1 << page_shift) * 2, + &hr_qp->hr_buf, page_shift)) { dev_err(dev, "hns_roce_buf_alloc error!\n"); ret = -ENOMEM; goto err_out; } + hr_qp->mtt.mtt_type = MTT_TYPE_WQE; /* Write MTT */ ret = hns_roce_mtt_init(hr_dev, hr_qp->hr_buf.npages, hr_qp->hr_buf.page_shift, &hr_qp->mtt); @@ -522,7 +636,9 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, } } - if ((init_attr->qp_type) == IB_QPT_GSI) { + if (init_attr->qp_type == IB_QPT_GSI && + hr_dev->hw_rev == HNS_ROCE_HW_VER1) { + /* In v1 engine, GSI QP context in RoCE engine's register */ ret = hns_roce_gsi_qp_alloc(hr_dev, qpn, hr_qp); if (ret) { dev_err(dev, "hns_roce_qp_alloc failed!\n"); @@ -571,7 +687,7 @@ struct ib_qp *hns_roce_create_qp(struct ib_pd *pd, struct ib_udata *udata) { struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; struct hns_roce_sqp *hr_sqp; struct hns_roce_qp *hr_qp; int ret; @@ -629,6 +745,7 @@ struct ib_qp *hns_roce_create_qp(struct ib_pd *pd, return &hr_qp->ibqp; } +EXPORT_SYMBOL_GPL(hns_roce_create_qp); int to_hr_qp_type(int qp_type) { @@ -647,6 +764,7 @@ int to_hr_qp_type(int qp_type) return transport_type; } +EXPORT_SYMBOL_GPL(to_hr_qp_type); int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) @@ -654,7 +772,7 @@ int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); enum ib_qp_state cur_state, new_state; - struct device *dev = &hr_dev->pdev->dev; + struct device *dev = hr_dev->dev; int ret = -EINVAL; int p; enum ib_mtu active_mtu; @@ -692,7 +810,10 @@ int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; active_mtu = iboe_get_mtu(hr_dev->iboe.netdevs[p]->mtu); - if (attr->path_mtu > IB_MTU_2048 || + if ((hr_dev->caps.max_mtu == IB_MTU_4096 && + attr->path_mtu > IB_MTU_4096) || + (hr_dev->caps.max_mtu == IB_MTU_2048 && + attr->path_mtu > IB_MTU_2048) || attr->path_mtu < IB_MTU_256 || attr->path_mtu > active_mtu) { dev_err(dev, "attr path_mtu(%d)invalid while modify qp", @@ -716,9 +837,7 @@ int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, } if (cur_state == new_state && cur_state == IB_QPS_RESET) { - ret = -EPERM; - dev_err(dev, "cur_state=%d new_state=%d\n", cur_state, - new_state); + ret = 0; goto out; } @@ -745,6 +864,7 @@ void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq) spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); } } +EXPORT_SYMBOL_GPL(hns_roce_lock_cqs); void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq) __releases(&send_cq->lock) @@ -761,6 +881,7 @@ void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, spin_unlock_irq(&recv_cq->lock); } } +EXPORT_SYMBOL_GPL(hns_roce_unlock_cqs); __be32 send_ieth(struct ib_send_wr *wr) { @@ -774,6 +895,7 @@ __be32 send_ieth(struct ib_send_wr *wr) return 0; } } +EXPORT_SYMBOL_GPL(send_ieth); static void *get_wqe(struct hns_roce_qp *hr_qp, int offset) { @@ -785,11 +907,20 @@ void *get_recv_wqe(struct hns_roce_qp *hr_qp, int n) { return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift)); } +EXPORT_SYMBOL_GPL(get_recv_wqe); void *get_send_wqe(struct hns_roce_qp *hr_qp, int n) { return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift)); } +EXPORT_SYMBOL_GPL(get_send_wqe); + +void *get_send_extend_sge(struct hns_roce_qp *hr_qp, int n) +{ + return hns_roce_buf_offset(&hr_qp->hr_buf, hr_qp->sge.offset + + (n << hr_qp->sge.sge_shift)); +} +EXPORT_SYMBOL_GPL(get_send_extend_sge); bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq, struct ib_cq *ib_cq) @@ -808,6 +939,7 @@ bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq, return cur + nreq >= hr_wq->max_post; } +EXPORT_SYMBOL_GPL(hns_roce_wq_overflow); int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev) { @@ -823,7 +955,7 @@ int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev) hr_dev->caps.num_qps - 1, SQP_NUM, reserved_from_top); if (ret) { - dev_err(&hr_dev->pdev->dev, "qp bitmap init failed!error=%d\n", + dev_err(hr_dev->dev, "qp bitmap init failed!error=%d\n", ret); return ret; } |