diff options
Diffstat (limited to 'drivers/net/ethernet/intel/ice/ice_base.c')
| -rw-r--r-- | drivers/net/ethernet/intel/ice/ice_base.c | 850 |
1 files changed, 687 insertions, 163 deletions
diff --git a/drivers/net/ethernet/intel/ice/ice_base.c b/drivers/net/ethernet/intel/ice/ice_base.c index 554095b25f44..eadb1e3d12b3 100644 --- a/drivers/net/ethernet/intel/ice/ice_base.c +++ b/drivers/net/ethernet/intel/ice/ice_base.c @@ -2,6 +2,7 @@ /* Copyright (c) 2019, Intel Corporation. */ #include <net/xdp_sock_drv.h> +#include <linux/net/intel/libie/rx.h> #include "ice_base.h" #include "ice_lib.h" #include "ice_dcb_lib.h" @@ -103,10 +104,10 @@ static int ice_vsi_alloc_q_vector(struct ice_vsi *vsi, u16 v_idx) { struct ice_pf *pf = vsi->back; struct ice_q_vector *q_vector; + int err; /* allocate q_vector */ - q_vector = devm_kzalloc(ice_pf_to_dev(pf), sizeof(*q_vector), - GFP_KERNEL); + q_vector = kzalloc(sizeof(*q_vector), GFP_KERNEL); if (!q_vector) return -ENOMEM; @@ -118,25 +119,53 @@ static int ice_vsi_alloc_q_vector(struct ice_vsi *vsi, u16 v_idx) q_vector->rx.itr_mode = ITR_DYNAMIC; q_vector->tx.type = ICE_TX_CONTAINER; q_vector->rx.type = ICE_RX_CONTAINER; + q_vector->irq.index = -ENOENT; - if (vsi->type == ICE_VSI_VF) + if (vsi->type == ICE_VSI_VF) { + ice_calc_vf_reg_idx(vsi->vf, q_vector); goto out; - /* only set affinity_mask if the CPU is online */ - if (cpu_online(v_idx)) - cpumask_set_cpu(v_idx, &q_vector->affinity_mask); + } else if (vsi->type == ICE_VSI_CTRL && vsi->vf) { + struct ice_vsi *ctrl_vsi = ice_get_vf_ctrl_vsi(pf, vsi); + + if (ctrl_vsi) { + if (unlikely(!ctrl_vsi->q_vectors)) { + err = -ENOENT; + goto err_free_q_vector; + } + + q_vector->irq = ctrl_vsi->q_vectors[0]->irq; + goto skip_alloc; + } + } + + q_vector->irq = ice_alloc_irq(pf, vsi->irq_dyn_alloc); + if (q_vector->irq.index < 0) { + err = -ENOMEM; + goto err_free_q_vector; + } + +skip_alloc: + q_vector->reg_idx = q_vector->irq.index; + q_vector->vf_reg_idx = q_vector->irq.index; /* This will not be called in the driver load path because the netdev * will not be created yet. All other cases with register the NAPI * handler here (i.e. resume, reset/rebuild, etc.) */ if (vsi->netdev) - netif_napi_add(vsi->netdev, &q_vector->napi, ice_napi_poll); + netif_napi_add_config(vsi->netdev, &q_vector->napi, + ice_napi_poll, v_idx); out: /* tie q_vector and VSI together */ vsi->q_vectors[v_idx] = q_vector; return 0; + +err_free_q_vector: + kfree(q_vector); + + return err; } /** @@ -159,16 +188,29 @@ static void ice_free_q_vector(struct ice_vsi *vsi, int v_idx) } q_vector = vsi->q_vectors[v_idx]; - ice_for_each_tx_ring(tx_ring, q_vector->tx) + ice_for_each_tx_ring(tx_ring, vsi->q_vectors[v_idx]->tx) tx_ring->q_vector = NULL; - ice_for_each_rx_ring(rx_ring, q_vector->rx) + + ice_for_each_rx_ring(rx_ring, vsi->q_vectors[v_idx]->rx) rx_ring->q_vector = NULL; /* only VSI with an associated netdev is set up with NAPI */ if (vsi->netdev) netif_napi_del(&q_vector->napi); - devm_kfree(dev, q_vector); + /* release MSIX interrupt if q_vector had interrupt allocated */ + if (q_vector->irq.index < 0) + goto free_q_vector; + + /* only free last VF ctrl vsi interrupt */ + if (vsi->type == ICE_VSI_CTRL && vsi->vf && + ice_get_vf_ctrl_vsi(pf, vsi)) + goto free_q_vector; + + ice_free_irq(pf, q_vector->irq); + +free_q_vector: + kfree(q_vector); vsi->q_vectors[v_idx] = NULL; } @@ -182,24 +224,16 @@ static void ice_cfg_itr_gran(struct ice_hw *hw) /* no need to update global register if ITR gran is already set */ if (!(regval & GLINT_CTL_DIS_AUTOMASK_M) && - (((regval & GLINT_CTL_ITR_GRAN_200_M) >> - GLINT_CTL_ITR_GRAN_200_S) == ICE_ITR_GRAN_US) && - (((regval & GLINT_CTL_ITR_GRAN_100_M) >> - GLINT_CTL_ITR_GRAN_100_S) == ICE_ITR_GRAN_US) && - (((regval & GLINT_CTL_ITR_GRAN_50_M) >> - GLINT_CTL_ITR_GRAN_50_S) == ICE_ITR_GRAN_US) && - (((regval & GLINT_CTL_ITR_GRAN_25_M) >> - GLINT_CTL_ITR_GRAN_25_S) == ICE_ITR_GRAN_US)) + (FIELD_GET(GLINT_CTL_ITR_GRAN_200_M, regval) == ICE_ITR_GRAN_US) && + (FIELD_GET(GLINT_CTL_ITR_GRAN_100_M, regval) == ICE_ITR_GRAN_US) && + (FIELD_GET(GLINT_CTL_ITR_GRAN_50_M, regval) == ICE_ITR_GRAN_US) && + (FIELD_GET(GLINT_CTL_ITR_GRAN_25_M, regval) == ICE_ITR_GRAN_US)) return; - regval = ((ICE_ITR_GRAN_US << GLINT_CTL_ITR_GRAN_200_S) & - GLINT_CTL_ITR_GRAN_200_M) | - ((ICE_ITR_GRAN_US << GLINT_CTL_ITR_GRAN_100_S) & - GLINT_CTL_ITR_GRAN_100_M) | - ((ICE_ITR_GRAN_US << GLINT_CTL_ITR_GRAN_50_S) & - GLINT_CTL_ITR_GRAN_50_M) | - ((ICE_ITR_GRAN_US << GLINT_CTL_ITR_GRAN_25_S) & - GLINT_CTL_ITR_GRAN_25_M); + regval = FIELD_PREP(GLINT_CTL_ITR_GRAN_200_M, ICE_ITR_GRAN_US) | + FIELD_PREP(GLINT_CTL_ITR_GRAN_100_M, ICE_ITR_GRAN_US) | + FIELD_PREP(GLINT_CTL_ITR_GRAN_50_M, ICE_ITR_GRAN_US) | + FIELD_PREP(GLINT_CTL_ITR_GRAN_25_M, ICE_ITR_GRAN_US); wr32(hw, GLINT_CTL, regval); } @@ -209,7 +243,8 @@ static void ice_cfg_itr_gran(struct ice_hw *hw) * @ring: ring to get the absolute queue index * @tc: traffic class number */ -static u16 ice_calc_txq_handle(struct ice_vsi *vsi, struct ice_tx_ring *ring, u8 tc) +static u16 +ice_calc_txq_handle(const struct ice_vsi *vsi, struct ice_tx_ring *ring, u8 tc) { WARN_ONCE(ice_ring_is_xdp(ring) && tc, "XDP ring can't belong to TC other than 0\n"); @@ -217,37 +252,13 @@ static u16 ice_calc_txq_handle(struct ice_vsi *vsi, struct ice_tx_ring *ring, u8 return ring->q_index - ring->ch->base_q; /* Idea here for calculation is that we subtract the number of queue - * count from TC that ring belongs to from it's absolute queue index + * count from TC that ring belongs to from its absolute queue index * and as a result we get the queue's index within TC. */ return ring->q_index - vsi->tc_cfg.tc_info[tc].qoffset; } /** - * ice_eswitch_calc_txq_handle - * @ring: pointer to ring which unique index is needed - * - * To correctly work with many netdevs ring->q_index of Tx rings on switchdev - * VSI can repeat. Hardware ring setup requires unique q_index. Calculate it - * here by finding index in vsi->tx_rings of this ring. - * - * Return ICE_INVAL_Q_INDEX when index wasn't found. Should never happen, - * because VSI is get from ring->vsi, so it has to be present in this VSI. - */ -static u16 ice_eswitch_calc_txq_handle(struct ice_tx_ring *ring) -{ - struct ice_vsi *vsi = ring->vsi; - int i; - - ice_for_each_txq(vsi, i) { - if (vsi->tx_rings[i] == ring) - return i; - } - - return ICE_INVAL_Q_INDEX; -} - -/** * ice_cfg_xps_tx_ring - Configure XPS for a Tx ring * @ring: The Tx ring to configure * @@ -263,35 +274,26 @@ static void ice_cfg_xps_tx_ring(struct ice_tx_ring *ring) if (test_and_set_bit(ICE_TX_XPS_INIT_DONE, ring->xps_state)) return; - netif_set_xps_queue(ring->netdev, &ring->q_vector->affinity_mask, + netif_set_xps_queue(ring->netdev, + &ring->q_vector->napi.config->affinity_mask, ring->q_index); } /** - * ice_setup_tx_ctx - setup a struct ice_tlan_ctx instance - * @ring: The Tx ring to configure - * @tlan_ctx: Pointer to the Tx LAN queue context structure to be initialized - * @pf_q: queue index in the PF space + * ice_set_txq_ctx_vmvf - set queue context VM/VF type and number by VSI type + * @ring: the Tx ring to configure + * @vmvf_type: VM/VF type + * @vmvf_num: VM/VF number * - * Configure the Tx descriptor ring in TLAN context. + * Return: 0 on success and a negative value on error. */ -static void -ice_setup_tx_ctx(struct ice_tx_ring *ring, struct ice_tlan_ctx *tlan_ctx, u16 pf_q) +static int +ice_set_txq_ctx_vmvf(struct ice_tx_ring *ring, u8 *vmvf_type, u16 *vmvf_num) { struct ice_vsi *vsi = ring->vsi; - struct ice_hw *hw = &vsi->back->hw; - - tlan_ctx->base = ring->dma >> ICE_TLAN_CTX_BASE_S; - - tlan_ctx->port_num = vsi->port_info->lport; - - /* Transmit Queue Length */ - tlan_ctx->qlen = ring->count; - - ice_set_cgd_num(tlan_ctx, ring->dcb_tc); + struct ice_hw *hw; - /* PF number */ - tlan_ctx->pf_num = hw->pf_id; + hw = &vsi->back->hw; /* queue belongs to a specific VSI type * VF / VM index should be programmed per vmvf_type setting: @@ -304,21 +306,60 @@ ice_setup_tx_ctx(struct ice_tx_ring *ring, struct ice_tlan_ctx *tlan_ctx, u16 pf case ICE_VSI_CTRL: case ICE_VSI_PF: if (ring->ch) - tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VMQ; + *vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VMQ; else - tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_PF; + *vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_PF; break; case ICE_VSI_VF: /* Firmware expects vmvf_num to be absolute VF ID */ - tlan_ctx->vmvf_num = hw->func_caps.vf_base_id + vsi->vf->vf_id; - tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VF; + *vmvf_num = hw->func_caps.vf_base_id + vsi->vf->vf_id; + *vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VF; break; - case ICE_VSI_SWITCHDEV_CTRL: - tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VMQ; + case ICE_VSI_SF: + *vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VMQ; break; default: - return; + dev_info(ice_pf_to_dev(vsi->back), + "Unable to set VMVF type for VSI type %d\n", + vsi->type); + return -EINVAL; } + return 0; +} + +/** + * ice_setup_tx_ctx - setup a struct ice_tlan_ctx instance + * @ring: the Tx ring to configure + * @tlan_ctx: pointer to the Tx LAN queue context structure to be initialized + * @pf_q: queue index in the PF space + * + * Configure the Tx descriptor ring in TLAN context. + * + * Return: 0 on success and a negative value on error. + */ +static int +ice_setup_tx_ctx(struct ice_tx_ring *ring, struct ice_tlan_ctx *tlan_ctx, u16 pf_q) +{ + struct ice_vsi *vsi = ring->vsi; + struct ice_hw *hw; + int err; + + hw = &vsi->back->hw; + tlan_ctx->base = ring->dma >> ICE_TLAN_CTX_BASE_S; + tlan_ctx->port_num = vsi->port_info->lport; + + /* Transmit Queue Length */ + tlan_ctx->qlen = ring->count; + + ice_set_cgd_num(tlan_ctx, ring->dcb_tc); + + /* PF number */ + tlan_ctx->pf_num = hw->pf_id; + + err = ice_set_txq_ctx_vmvf(ring, &tlan_ctx->vmvf_type, + &tlan_ctx->vmvf_num); + if (err) + return err; /* make sure the context is associated with the right VSI */ if (ring->ch) @@ -335,6 +376,8 @@ ice_setup_tx_ctx(struct ice_tx_ring *ring, struct ice_tlan_ctx *tlan_ctx, u16 pf break; } + tlan_ctx->quanta_prof_idx = ring->quanta_prof_id; + tlan_ctx->tso_ena = ICE_TX_LEGACY; tlan_ctx->tso_qnum = pf_q; @@ -343,25 +386,83 @@ ice_setup_tx_ctx(struct ice_tx_ring *ring, struct ice_tlan_ctx *tlan_ctx, u16 pf * 1: Legacy Host Interface */ tlan_ctx->legacy_int = ICE_TX_LEGACY; + + return 0; } /** - * ice_rx_offset - Return expected offset into page to access data - * @rx_ring: Ring we are requesting offset of + * ice_setup_txtime_ctx - setup a struct ice_txtime_ctx instance + * @ring: the tstamp ring to configure + * @txtime_ctx: pointer to the Tx time queue context structure to be initialized * - * Returns the offset value for ring into the data buffer. + * Return: 0 on success and a negative value on error. */ -static unsigned int ice_rx_offset(struct ice_rx_ring *rx_ring) +static int +ice_setup_txtime_ctx(const struct ice_tstamp_ring *ring, + struct ice_txtime_ctx *txtime_ctx) { - if (ice_ring_uses_build_skb(rx_ring)) - return ICE_SKB_PAD; - else if (ice_is_xdp_ena_vsi(rx_ring->vsi)) - return XDP_PACKET_HEADROOM; + struct ice_tx_ring *tx_ring = ring->tx_ring; + struct ice_vsi *vsi = tx_ring->vsi; + struct ice_hw *hw = &vsi->back->hw; + int err; + + txtime_ctx->base = ring->dma >> ICE_TXTIME_CTX_BASE_S; + + /* Tx time Queue Length */ + txtime_ctx->qlen = ring->count; + txtime_ctx->txtime_ena_q = 1; + + /* PF number */ + txtime_ctx->pf_num = hw->pf_id; + + err = ice_set_txq_ctx_vmvf(tx_ring, &txtime_ctx->vmvf_type, + &txtime_ctx->vmvf_num); + if (err) + return err; + + /* make sure the context is associated with the right VSI */ + if (tx_ring->ch) + txtime_ctx->src_vsi = tx_ring->ch->vsi_num; + else + txtime_ctx->src_vsi = ice_get_hw_vsi_num(hw, vsi->idx); + + txtime_ctx->ts_res = ICE_TXTIME_CTX_RESOLUTION_128NS; + txtime_ctx->drbell_mode_32 = ICE_TXTIME_CTX_DRBELL_MODE_32; + txtime_ctx->ts_fetch_prof_id = ICE_TXTIME_CTX_FETCH_PROF_ID_0; return 0; } /** + * ice_calc_ts_ring_count - calculate the number of Tx time stamp descriptors + * @tx_ring: Tx ring to calculate the count for + * + * Return: the number of Tx time stamp descriptors. + */ +u16 ice_calc_ts_ring_count(struct ice_tx_ring *tx_ring) +{ + u16 prof = ICE_TXTIME_CTX_FETCH_PROF_ID_0; + struct ice_vsi *vsi = tx_ring->vsi; + struct ice_hw *hw = &vsi->back->hw; + u16 max_fetch_desc = 0, fetch, i; + u32 reg; + + for (i = 0; i < ICE_TXTIME_FETCH_PROFILE_CNT; i++) { + reg = rd32(hw, E830_GLTXTIME_FETCH_PROFILE(prof, 0)); + fetch = FIELD_GET(E830_GLTXTIME_FETCH_PROFILE_FETCH_TS_DESC_M, + reg); + max_fetch_desc = max(fetch, max_fetch_desc); + } + + if (!max_fetch_desc) + max_fetch_desc = ICE_TXTIME_FETCH_TS_DESC_DFLT; + + max_fetch_desc = ALIGN(max_fetch_desc, ICE_REQ_DESC_MULTIPLE); + + return tx_ring->count + max_fetch_desc; +} + +/** * ice_setup_rx_ctx - Configure a receive ring context * @ring: The Rx ring to configure * @@ -369,7 +470,6 @@ static unsigned int ice_rx_offset(struct ice_rx_ring *rx_ring) */ static int ice_setup_rx_ctx(struct ice_rx_ring *ring) { - int chain_len = ICE_MAX_CHAINED_RX_BUFS; struct ice_vsi *vsi = ring->vsi; u32 rxdid = ICE_RXDID_FLEX_NIC; struct ice_rlan_ctx rlan_ctx; @@ -396,7 +496,8 @@ static int ice_setup_rx_ctx(struct ice_rx_ring *ring) /* Receive Packet Data Buffer Size. * The Packet Data Buffer Size is defined in 128 byte units. */ - rlan_ctx.dbuf = ring->rx_buf_len >> ICE_RLAN_CTX_DBUF_S; + rlan_ctx.dbuf = DIV_ROUND_UP(ring->rx_buf_len, + BIT_ULL(ICE_RLAN_CTX_DBUF_S)); /* use 32 byte descriptors */ rlan_ctx.dsize = 1; @@ -423,8 +524,29 @@ static int ice_setup_rx_ctx(struct ice_rx_ring *ring) else rlan_ctx.l2tsel = 1; - rlan_ctx.dtype = ICE_RX_DTYPE_NO_SPLIT; - rlan_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_NO_SPLIT; + if (ring->hdr_pp) { + rlan_ctx.hbuf = ring->rx_hdr_len >> ICE_RLAN_CTX_HBUF_S; + rlan_ctx.dtype = ICE_RX_DTYPE_HEADER_SPLIT; + + /* + * If the frame is TCP/UDP/SCTP, it will be split by the + * payload. + * If not, but it's an IPv4/IPv6 frame, it will be split by + * the IP header. + * If not IP, it will be split by the Ethernet header. + * + * In any case, the header buffer will never be left empty. + */ + rlan_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_SPLIT_L2 | + ICE_RLAN_RX_HSPLIT_0_SPLIT_IP | + ICE_RLAN_RX_HSPLIT_0_SPLIT_TCP_UDP | + ICE_RLAN_RX_HSPLIT_0_SPLIT_SCTP; + } else { + rlan_ctx.hbuf = 0; + rlan_ctx.dtype = ICE_RX_DTYPE_NO_SPLIT; + rlan_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_NO_SPLIT; + } + rlan_ctx.hsplit_1 = ICE_RLAN_RX_HSPLIT_1_NO_SPLIT; /* This controls whether VLAN is stripped from inner headers @@ -433,21 +555,26 @@ static int ice_setup_rx_ctx(struct ice_rx_ring *ring) */ rlan_ctx.showiv = 0; - /* For AF_XDP ZC, we disallow packets to span on - * multiple buffers, thus letting us skip that - * handling in the fast-path. - */ - if (ring->xsk_pool) - chain_len = 1; /* Max packet size for this queue - must not be set to a larger value * than 5 x DBUF */ rlan_ctx.rxmax = min_t(u32, vsi->max_frame, - chain_len * ring->rx_buf_len); + ICE_MAX_CHAINED_RX_BUFS * ring->rx_buf_len); /* Rx queue threshold in units of 64 */ rlan_ctx.lrxqthresh = 1; + /* Enable descriptor prefetch */ + rlan_ctx.prefena = 1; + + /* PF acts as uplink for switchdev; set flex descriptor with src_vsi + * metadata and flags to allow redirecting to PR netdev + */ + if (ice_is_eswitch_mode_switchdev(vsi->back)) { + ring->flags |= ICE_RX_FLAGS_MULTIDEV; + rxdid = ICE_RXDID_FLEX_NIC_2; + } + /* Enable Flexible Descriptors in the queue context which * allows this driver to select a specific receive descriptor format * increasing context priority to pick up profile ID; default is 0x01; @@ -456,9 +583,6 @@ static int ice_setup_rx_ctx(struct ice_rx_ring *ring) */ if (vsi->type != ICE_VSI_VF) ice_write_qrxflxp_cntxt(hw, pf_q, rxdid, 0x3, true); - else - ice_write_qrxflxp_cntxt(hw, pf_q, ICE_RXDID_LEGACY_1, 0x3, - false); /* Absolute queue number out of 2K needs to be passed */ err = ice_write_rxq_ctx(hw, &rlan_ctx, pf_q); @@ -471,14 +595,6 @@ static int ice_setup_rx_ctx(struct ice_rx_ring *ring) if (vsi->type == ICE_VSI_VF) return 0; - /* configure Rx buffer alignment */ - if (!vsi->netdev || test_bit(ICE_FLAG_LEGACY_RX, vsi->back->flags)) - ice_clear_ring_build_skb_ena(ring); - else - ice_set_ring_build_skb_ena(ring); - - ring->rx_offset = ice_rx_offset(ring); - /* init queue specific tail register */ ring->tail = hw->hw_addr + QRX_TAIL(pf_q); writel(0, ring->tail); @@ -486,32 +602,92 @@ static int ice_setup_rx_ctx(struct ice_rx_ring *ring) return 0; } +static int ice_rxq_pp_create(struct ice_rx_ring *rq) +{ + struct libeth_fq fq = { + .count = rq->count, + .nid = NUMA_NO_NODE, + .hsplit = rq->vsi->hsplit, + .xdp = ice_is_xdp_ena_vsi(rq->vsi), + .buf_len = LIBIE_MAX_RX_BUF_LEN, + }; + int err; + + err = libeth_rx_fq_create(&fq, &rq->q_vector->napi); + if (err) + return err; + + rq->pp = fq.pp; + rq->rx_fqes = fq.fqes; + rq->truesize = fq.truesize; + rq->rx_buf_len = fq.buf_len; + + if (!fq.hsplit) + return 0; + + fq = (struct libeth_fq){ + .count = rq->count, + .type = LIBETH_FQE_HDR, + .nid = NUMA_NO_NODE, + .xdp = ice_is_xdp_ena_vsi(rq->vsi), + }; + + err = libeth_rx_fq_create(&fq, &rq->q_vector->napi); + if (err) + goto destroy; + + rq->hdr_pp = fq.pp; + rq->hdr_fqes = fq.fqes; + rq->hdr_truesize = fq.truesize; + rq->rx_hdr_len = fq.buf_len; + + return 0; + +destroy: + ice_rxq_pp_destroy(rq); + + return err; +} + /** * ice_vsi_cfg_rxq - Configure an Rx queue * @ring: the ring being configured * * Return 0 on success and a negative value on error. */ -int ice_vsi_cfg_rxq(struct ice_rx_ring *ring) +static int ice_vsi_cfg_rxq(struct ice_rx_ring *ring) { struct device *dev = ice_pf_to_dev(ring->vsi->back); - u16 num_bufs = ICE_DESC_UNUSED(ring); + u32 num_bufs = ICE_DESC_UNUSED(ring); + u32 rx_buf_len; int err; - ring->rx_buf_len = ring->vsi->rx_buf_len; + if (ring->vsi->type == ICE_VSI_PF || ring->vsi->type == ICE_VSI_SF) { + if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) { + err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, + ring->q_index, + ring->q_vector->napi.napi_id, + ring->rx_buf_len); + if (err) + return err; + } - if (ring->vsi->type == ICE_VSI_PF) { - if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) - /* coverity[check_return] */ - xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, - ring->q_index, ring->q_vector->napi.napi_id); + ice_rx_xsk_pool(ring); + err = ice_realloc_rx_xdp_bufs(ring, ring->xsk_pool); + if (err) + return err; - ring->xsk_pool = ice_xsk_pool(ring); if (ring->xsk_pool) { - xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq); + xdp_rxq_info_unreg(&ring->xdp_rxq); - ring->rx_buf_len = + rx_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool); + err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, + ring->q_index, + ring->q_vector->napi.napi_id, + rx_buf_len); + if (err) + return err; err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, MEM_TYPE_XSK_BUFF_POOL, NULL); @@ -522,25 +698,29 @@ int ice_vsi_cfg_rxq(struct ice_rx_ring *ring) dev_info(dev, "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n", ring->q_index); } else { - if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) - /* coverity[check_return] */ - xdp_rxq_info_reg(&ring->xdp_rxq, - ring->netdev, - ring->q_index, ring->q_vector->napi.napi_id); - - err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, - MEM_TYPE_PAGE_SHARED, - NULL); + err = ice_rxq_pp_create(ring); if (err) return err; + + if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) { + err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, + ring->q_index, + ring->q_vector->napi.napi_id, + ring->rx_buf_len); + if (err) + goto err_destroy_fq; + } + xdp_rxq_info_attach_page_pool(&ring->xdp_rxq, + ring->pp); } } + ring->xdp.data = NULL; err = ice_setup_rx_ctx(ring); if (err) { dev_err(dev, "ice_setup_rx_ctx failed for RxQ %d, err %d\n", ring->q_index, err); - return err; + goto err_destroy_fq; } if (ring->xsk_pool) { @@ -554,7 +734,7 @@ int ice_vsi_cfg_rxq(struct ice_rx_ring *ring) return 0; } - ok = ice_alloc_rx_bufs_zc(ring, num_bufs); + ok = ice_alloc_rx_bufs_zc(ring, ring->xsk_pool, num_bufs); if (!ok) { u16 pf_q = ring->vsi->rxq_map[ring->q_index]; @@ -565,7 +745,70 @@ int ice_vsi_cfg_rxq(struct ice_rx_ring *ring) return 0; } - ice_alloc_rx_bufs(ring, num_bufs); + if (ring->vsi->type == ICE_VSI_CTRL) + ice_init_ctrl_rx_descs(ring, num_bufs); + else + err = ice_alloc_rx_bufs(ring, num_bufs); + + if (err) + goto err_destroy_fq; + + return 0; + +err_destroy_fq: + ice_rxq_pp_destroy(ring); + + return err; +} + +int ice_vsi_cfg_single_rxq(struct ice_vsi *vsi, u16 q_idx) +{ + if (q_idx >= vsi->num_rxq) + return -EINVAL; + + return ice_vsi_cfg_rxq(vsi->rx_rings[q_idx]); +} + +/** + * ice_vsi_cfg_frame_size - setup max frame size and Rx buffer length + * @vsi: VSI + * @ring: Rx ring to configure + * + * Determine the maximum frame size and Rx buffer length to use for a PF VSI. + * Set these in the associated Rx ring structure. + */ +static void ice_vsi_cfg_frame_size(struct ice_vsi *vsi, struct ice_rx_ring *ring) +{ + if (!vsi->netdev) { + vsi->max_frame = ICE_MAX_FRAME_LEGACY_RX; + } else { + vsi->max_frame = ICE_AQ_SET_MAC_FRAME_SIZE_MAX; + } +} + +/** + * ice_vsi_cfg_rxqs - Configure the VSI for Rx + * @vsi: the VSI being configured + * + * Return 0 on success and a negative value on error + * Configure the Rx VSI for operation. + */ +int ice_vsi_cfg_rxqs(struct ice_vsi *vsi) +{ + u16 i; + + /* set up individual rings */ + ice_for_each_rxq(vsi, i) { + struct ice_rx_ring *ring = vsi->rx_rings[i]; + int err; + + if (vsi->type != ICE_VSI_VF) + ice_vsi_cfg_frame_size(vsi, ring); + + err = ice_vsi_cfg_rxq(ring); + if (err) + return err; + } return 0; } @@ -677,13 +920,11 @@ int ice_vsi_alloc_q_vectors(struct ice_vsi *vsi) return 0; err_out: - while (v_idx--) - ice_free_q_vector(vsi, v_idx); - dev_err(dev, "Failed to allocate %d q_vector for VSI %d, ret=%d\n", - vsi->num_q_vectors, vsi->vsi_num, err); - vsi->num_q_vectors = 0; - return err; + dev_info(dev, "Failed to allocate %d q_vectors for VSI %d, new value %d", + vsi->num_q_vectors, vsi->vsi_num, v_idx); + vsi->num_q_vectors = v_idx; + return v_idx ? 0 : err; } /** @@ -743,6 +984,9 @@ void ice_vsi_map_rings_to_vectors(struct ice_vsi *vsi) } rx_rings_rem -= rx_rings_per_v; } + + if (ice_is_xdp_ena_vsi(vsi)) + ice_map_xdp_rings(vsi); } /** @@ -755,6 +999,42 @@ void ice_vsi_free_q_vectors(struct ice_vsi *vsi) ice_for_each_q_vector(vsi, v_idx) ice_free_q_vector(vsi, v_idx); + + vsi->num_q_vectors = 0; +} + +/** + * ice_cfg_tstamp - Configure Tx time stamp queue + * @tx_ring: Tx ring to be configured with timestamping + * + * Return: 0 on success and a negative value on error. + */ +static int +ice_cfg_tstamp(struct ice_tx_ring *tx_ring) +{ + DEFINE_RAW_FLEX(struct ice_aqc_set_txtime_qgrp, txtime_qg_buf, + txtimeqs, 1); + u8 txtime_buf_len = struct_size(txtime_qg_buf, txtimeqs, 1); + struct ice_tstamp_ring *tstamp_ring = tx_ring->tstamp_ring; + struct ice_txtime_ctx txtime_ctx = {}; + struct ice_vsi *vsi = tx_ring->vsi; + struct ice_pf *pf = vsi->back; + struct ice_hw *hw = &pf->hw; + u16 pf_q = tx_ring->reg_idx; + int err; + + err = ice_setup_txtime_ctx(tstamp_ring, &txtime_ctx); + if (err) { + dev_err(ice_pf_to_dev(pf), "Failed to setup Tx time queue context for queue %d, error: %d\n", + pf_q, err); + return err; + } + ice_pack_txtime_ctx(&txtime_ctx, + &txtime_qg_buf->txtimeqs[0].txtime_ctx); + + tstamp_ring->tail = hw->hw_addr + E830_GLQTX_TXTIME_DBELL_LSB(pf_q); + return ice_aq_set_txtimeq(hw, pf_q, 1, txtime_qg_buf, + txtime_buf_len, NULL); } /** @@ -762,9 +1042,11 @@ void ice_vsi_free_q_vectors(struct ice_vsi *vsi) * @vsi: the VSI that queue belongs to * @ring: Tx ring to be configured * @qg_buf: queue group buffer + * + * Return: 0 on success and a negative value on error. */ -int -ice_vsi_cfg_txq(struct ice_vsi *vsi, struct ice_tx_ring *ring, +static int +ice_vsi_cfg_txq(const struct ice_vsi *vsi, struct ice_tx_ring *ring, struct ice_aqc_add_tx_qgrp *qg_buf) { u8 buf_len = struct_size(qg_buf, txqs, 1); @@ -773,19 +1055,23 @@ ice_vsi_cfg_txq(struct ice_vsi *vsi, struct ice_tx_ring *ring, struct ice_channel *ch = ring->ch; struct ice_pf *pf = vsi->back; struct ice_hw *hw = &pf->hw; + u32 pf_q, vsi_idx; int status; - u16 pf_q; u8 tc; /* Configure XPS */ ice_cfg_xps_tx_ring(ring); pf_q = ring->reg_idx; - ice_setup_tx_ctx(ring, &tlan_ctx, pf_q); + status = ice_setup_tx_ctx(ring, &tlan_ctx, pf_q); + if (status) { + dev_err(ice_pf_to_dev(pf), "Failed to setup Tx context for queue %d, error: %d\n", + pf_q, status); + return status; + } /* copy context contents into the qg_buf */ qg_buf->txqs[0].txq_id = cpu_to_le16(pf_q); - ice_set_ctx(hw, (u8 *)&tlan_ctx, qg_buf->txqs[0].txq_ctx, - ice_tlan_ctx_info); + ice_pack_txq_ctx(&tlan_ctx, &qg_buf->txqs[0].txq_ctx); /* init queue specific tail reg. It is referred as * transmit comm scheduler queue doorbell. @@ -800,23 +1086,17 @@ ice_vsi_cfg_txq(struct ice_vsi *vsi, struct ice_tx_ring *ring, /* Add unique software queue handle of the Tx queue per * TC into the VSI Tx ring */ - if (vsi->type == ICE_VSI_SWITCHDEV_CTRL) { - ring->q_handle = ice_eswitch_calc_txq_handle(ring); + ring->q_handle = ice_calc_txq_handle(vsi, ring, tc); - if (ring->q_handle == ICE_INVAL_Q_INDEX) - return -ENODEV; + if (ch) { + tc = 0; + vsi_idx = ch->ch_vsi->idx; } else { - ring->q_handle = ice_calc_txq_handle(vsi, ring, tc); + vsi_idx = vsi->idx; } - if (ch) - status = ice_ena_vsi_txq(vsi->port_info, ch->ch_vsi->idx, 0, - ring->q_handle, 1, qg_buf, buf_len, - NULL); - else - status = ice_ena_vsi_txq(vsi->port_info, vsi->idx, tc, - ring->q_handle, 1, qg_buf, buf_len, - NULL); + status = ice_ena_vsi_txq(vsi->port_info, vsi_idx, tc, ring->q_handle, + 1, qg_buf, buf_len, NULL); if (status) { dev_err(ice_pf_to_dev(pf), "Failed to set LAN Tx queue context, error: %d\n", status); @@ -831,6 +1111,105 @@ ice_vsi_cfg_txq(struct ice_vsi *vsi, struct ice_tx_ring *ring, if (pf_q == le16_to_cpu(txq->txq_id)) ring->txq_teid = le32_to_cpu(txq->q_teid); + if (ice_is_txtime_ena(ring)) { + status = ice_alloc_setup_tstamp_ring(ring); + if (status) { + dev_err(ice_pf_to_dev(pf), + "Failed to allocate Tx timestamp ring, error: %d\n", + status); + goto err_setup_tstamp; + } + + status = ice_cfg_tstamp(ring); + if (status) { + dev_err(ice_pf_to_dev(pf), "Failed to set Tx Time queue context, error: %d\n", + status); + goto err_cfg_tstamp; + } + } + return 0; + +err_cfg_tstamp: + ice_free_tx_tstamp_ring(ring); +err_setup_tstamp: + ice_dis_vsi_txq(vsi->port_info, vsi_idx, tc, 1, &ring->q_handle, + &ring->reg_idx, &ring->txq_teid, ICE_NO_RESET, + tlan_ctx.vmvf_num, NULL); + + return status; +} + +int ice_vsi_cfg_single_txq(struct ice_vsi *vsi, struct ice_tx_ring **tx_rings, + u16 q_idx) +{ + DEFINE_RAW_FLEX(struct ice_aqc_add_tx_qgrp, qg_buf, txqs, 1); + + if (q_idx >= vsi->alloc_txq || !tx_rings || !tx_rings[q_idx]) + return -EINVAL; + + qg_buf->num_txqs = 1; + + return ice_vsi_cfg_txq(vsi, tx_rings[q_idx], qg_buf); +} + +/** + * ice_vsi_cfg_txqs - Configure the VSI for Tx + * @vsi: the VSI being configured + * @rings: Tx ring array to be configured + * @count: number of Tx ring array elements + * + * Return 0 on success and a negative value on error + * Configure the Tx VSI for operation. + */ +static int +ice_vsi_cfg_txqs(struct ice_vsi *vsi, struct ice_tx_ring **rings, u16 count) +{ + DEFINE_RAW_FLEX(struct ice_aqc_add_tx_qgrp, qg_buf, txqs, 1); + int err = 0; + u16 q_idx; + + qg_buf->num_txqs = 1; + + for (q_idx = 0; q_idx < count; q_idx++) { + err = ice_vsi_cfg_txq(vsi, rings[q_idx], qg_buf); + if (err) + break; + } + + return err; +} + +/** + * ice_vsi_cfg_lan_txqs - Configure the VSI for Tx + * @vsi: the VSI being configured + * + * Return 0 on success and a negative value on error + * Configure the Tx VSI for operation. + */ +int ice_vsi_cfg_lan_txqs(struct ice_vsi *vsi) +{ + return ice_vsi_cfg_txqs(vsi, vsi->tx_rings, vsi->num_txq); +} + +/** + * ice_vsi_cfg_xdp_txqs - Configure Tx queues dedicated for XDP in given VSI + * @vsi: the VSI being configured + * + * Return 0 on success and a negative value on error + * Configure the Tx queues dedicated for XDP in given VSI for operation. + */ +int ice_vsi_cfg_xdp_txqs(struct ice_vsi *vsi) +{ + int ret; + int i; + + ret = ice_vsi_cfg_txqs(vsi, vsi->xdp_rings, vsi->num_xdp_txq); + if (ret) + return ret; + + ice_for_each_rxq(vsi, i) + ice_tx_xsk_pool(vsi, i); + return 0; } @@ -872,10 +1251,10 @@ ice_cfg_txq_interrupt(struct ice_vsi *vsi, u16 txq, u16 msix_idx, u16 itr_idx) struct ice_hw *hw = &pf->hw; u32 val; - itr_idx = (itr_idx << QINT_TQCTL_ITR_INDX_S) & QINT_TQCTL_ITR_INDX_M; + itr_idx = FIELD_PREP(QINT_TQCTL_ITR_INDX_M, itr_idx); val = QINT_TQCTL_CAUSE_ENA_M | itr_idx | - ((msix_idx << QINT_TQCTL_MSIX_INDX_S) & QINT_TQCTL_MSIX_INDX_M); + FIELD_PREP(QINT_TQCTL_MSIX_INDX_M, msix_idx); wr32(hw, QINT_TQCTL(vsi->txq_map[txq]), val); if (ice_is_xdp_ena_vsi(vsi)) { @@ -904,10 +1283,10 @@ ice_cfg_rxq_interrupt(struct ice_vsi *vsi, u16 rxq, u16 msix_idx, u16 itr_idx) struct ice_hw *hw = &pf->hw; u32 val; - itr_idx = (itr_idx << QINT_RQCTL_ITR_INDX_S) & QINT_RQCTL_ITR_INDX_M; + itr_idx = FIELD_PREP(QINT_RQCTL_ITR_INDX_M, itr_idx); val = QINT_RQCTL_CAUSE_ENA_M | itr_idx | - ((msix_idx << QINT_RQCTL_MSIX_INDX_S) & QINT_RQCTL_MSIX_INDX_M); + FIELD_PREP(QINT_RQCTL_MSIX_INDX_M, msix_idx); wr32(hw, QINT_RQCTL(vsi->rxq_map[rxq]), val); @@ -919,7 +1298,7 @@ ice_cfg_rxq_interrupt(struct ice_vsi *vsi, u16 rxq, u16 msix_idx, u16 itr_idx) * @hw: pointer to the HW structure * @q_vector: interrupt vector to trigger the software interrupt for */ -void ice_trigger_sw_intr(struct ice_hw *hw, struct ice_q_vector *q_vector) +void ice_trigger_sw_intr(struct ice_hw *hw, const struct ice_q_vector *q_vector) { wr32(hw, GLINT_DYN_CTL(q_vector->reg_idx), (ICE_ITR_NONE << GLINT_DYN_CTL_ITR_INDX_S) | @@ -994,7 +1373,7 @@ ice_vsi_stop_tx_ring(struct ice_vsi *vsi, enum ice_disq_rst_src rst_src, * are needed for stopping Tx queue */ void -ice_fill_txq_meta(struct ice_vsi *vsi, struct ice_tx_ring *ring, +ice_fill_txq_meta(const struct ice_vsi *vsi, struct ice_tx_ring *ring, struct ice_txq_meta *txq_meta) { struct ice_channel *ch = ring->ch; @@ -1016,3 +1395,148 @@ ice_fill_txq_meta(struct ice_vsi *vsi, struct ice_tx_ring *ring, txq_meta->tc = tc; } } + +/** + * ice_qp_reset_stats - Resets all stats for rings of given index + * @vsi: VSI that contains rings of interest + * @q_idx: ring index in array + */ +static void ice_qp_reset_stats(struct ice_vsi *vsi, u16 q_idx) +{ + struct ice_vsi_stats *vsi_stat; + struct ice_pf *pf; + + pf = vsi->back; + if (!pf->vsi_stats) + return; + + vsi_stat = pf->vsi_stats[vsi->idx]; + if (!vsi_stat) + return; + + memset(&vsi_stat->rx_ring_stats[q_idx]->rx_stats, 0, + sizeof(vsi_stat->rx_ring_stats[q_idx]->rx_stats)); + memset(&vsi_stat->tx_ring_stats[q_idx]->stats, 0, + sizeof(vsi_stat->tx_ring_stats[q_idx]->stats)); + if (vsi->xdp_rings) + memset(&vsi->xdp_rings[q_idx]->ring_stats->stats, 0, + sizeof(vsi->xdp_rings[q_idx]->ring_stats->stats)); +} + +/** + * ice_qp_clean_rings - Cleans all the rings of a given index + * @vsi: VSI that contains rings of interest + * @q_idx: ring index in array + */ +static void ice_qp_clean_rings(struct ice_vsi *vsi, u16 q_idx) +{ + ice_clean_tx_ring(vsi->tx_rings[q_idx]); + if (vsi->xdp_rings) + ice_clean_tx_ring(vsi->xdp_rings[q_idx]); + ice_clean_rx_ring(vsi->rx_rings[q_idx]); +} + +/** + * ice_qp_dis - Disables a queue pair + * @vsi: VSI of interest + * @q_idx: ring index in array + * + * Returns 0 on success, negative on failure. + */ +int ice_qp_dis(struct ice_vsi *vsi, u16 q_idx) +{ + struct ice_txq_meta txq_meta = { }; + struct ice_q_vector *q_vector; + struct ice_tx_ring *tx_ring; + struct ice_rx_ring *rx_ring; + int fail = 0; + int err; + + if (q_idx >= vsi->num_rxq || q_idx >= vsi->num_txq) + return -EINVAL; + + tx_ring = vsi->tx_rings[q_idx]; + rx_ring = vsi->rx_rings[q_idx]; + q_vector = rx_ring->q_vector; + + synchronize_net(); + netif_carrier_off(vsi->netdev); + netif_tx_stop_queue(netdev_get_tx_queue(vsi->netdev, q_idx)); + + ice_qvec_dis_irq(vsi, rx_ring, q_vector); + ice_qvec_toggle_napi(vsi, q_vector, false); + + ice_fill_txq_meta(vsi, tx_ring, &txq_meta); + err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, tx_ring, &txq_meta); + if (!fail) + fail = err; + if (vsi->xdp_rings) { + struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_idx]; + + memset(&txq_meta, 0, sizeof(txq_meta)); + ice_fill_txq_meta(vsi, xdp_ring, &txq_meta); + err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, xdp_ring, + &txq_meta); + if (!fail) + fail = err; + } + + ice_vsi_ctrl_one_rx_ring(vsi, false, q_idx, false); + ice_qp_clean_rings(vsi, q_idx); + ice_qp_reset_stats(vsi, q_idx); + + return fail; +} + +/** + * ice_qp_ena - Enables a queue pair + * @vsi: VSI of interest + * @q_idx: ring index in array + * + * Returns 0 on success, negative on failure. + */ +int ice_qp_ena(struct ice_vsi *vsi, u16 q_idx) +{ + struct ice_q_vector *q_vector; + int fail = 0; + bool link_up; + int err; + + err = ice_vsi_cfg_single_txq(vsi, vsi->tx_rings, q_idx); + if (!fail) + fail = err; + + if (ice_is_xdp_ena_vsi(vsi)) { + struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_idx]; + + err = ice_vsi_cfg_single_txq(vsi, vsi->xdp_rings, q_idx); + if (!fail) + fail = err; + ice_set_ring_xdp(xdp_ring); + ice_tx_xsk_pool(vsi, q_idx); + } + + err = ice_vsi_cfg_single_rxq(vsi, q_idx); + if (!fail) + fail = err; + + q_vector = vsi->rx_rings[q_idx]->q_vector; + ice_qvec_cfg_msix(vsi, q_vector, q_idx); + + err = ice_vsi_ctrl_one_rx_ring(vsi, true, q_idx, true); + if (!fail) + fail = err; + + ice_qvec_toggle_napi(vsi, q_vector, true); + ice_qvec_ena_irq(vsi, q_vector); + + /* make sure NAPI sees updated ice_{t,x}_ring::xsk_pool */ + synchronize_net(); + ice_get_link_status(vsi->port_info, &link_up); + if (link_up) { + netif_tx_start_queue(netdev_get_tx_queue(vsi->netdev, q_idx)); + netif_carrier_on(vsi->netdev); + } + + return fail; +} |