diff options
Diffstat (limited to 'drivers/net/ethernet/intel/ice/ice_lib.c')
| -rw-r--r-- | drivers/net/ethernet/intel/ice/ice_lib.c | 3280 |
1 files changed, 1881 insertions, 1399 deletions
diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c index dde9802c6c72..15621707fbf8 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_lib.c @@ -7,7 +7,8 @@ #include "ice_lib.h" #include "ice_fltr.h" #include "ice_dcb_lib.h" -#include "ice_devlink.h" +#include "ice_type.h" +#include "ice_vsi_vlan_ops.h" /** * ice_vsi_type_str - maps VSI type enum to string equivalents @@ -20,8 +21,12 @@ const char *ice_vsi_type_str(enum ice_vsi_type vsi_type) return "ICE_VSI_PF"; case ICE_VSI_VF: return "ICE_VSI_VF"; + case ICE_VSI_SF: + return "ICE_VSI_SF"; case ICE_VSI_CTRL: return "ICE_VSI_CTRL"; + case ICE_VSI_CHNL: + return "ICE_VSI_CHNL"; case ICE_VSI_LB: return "ICE_VSI_LB"; default: @@ -44,12 +49,12 @@ static int ice_vsi_ctrl_all_rx_rings(struct ice_vsi *vsi, bool ena) int ret = 0; u16 i; - for (i = 0; i < vsi->num_rxq; i++) + ice_for_each_rxq(vsi, i) ice_vsi_ctrl_one_rx_ring(vsi, ena, i, false); ice_flush(&vsi->back->hw); - for (i = 0; i < vsi->num_rxq; i++) { + ice_for_each_rxq(vsi, i) { ret = ice_vsi_wait_one_rx_ring(vsi, ena, i); if (ret) break; @@ -71,6 +76,8 @@ static int ice_vsi_alloc_arrays(struct ice_vsi *vsi) struct device *dev; dev = ice_pf_to_dev(pf); + if (vsi->type == ICE_VSI_CHNL) + return 0; /* allocate memory for both Tx and Rx ring pointers */ vsi->tx_rings = devm_kcalloc(dev, vsi->alloc_txq, @@ -83,8 +90,13 @@ static int ice_vsi_alloc_arrays(struct ice_vsi *vsi) if (!vsi->rx_rings) goto err_rings; - /* XDP will have vsi->alloc_txq Tx queues as well, so double the size */ - vsi->txq_map = devm_kcalloc(dev, (2 * vsi->alloc_txq), + /* txq_map needs to have enough space to track both Tx (stack) rings + * and XDP rings; at this point vsi->num_xdp_txq might not be set, + * so use num_possible_cpus() as we want to always provide XDP ring + * per CPU, regardless of queue count settings from user that might + * have come from ethtool's set_channels() callback; + */ + vsi->txq_map = devm_kcalloc(dev, (vsi->alloc_txq + num_possible_cpus()), sizeof(*vsi->txq_map), GFP_KERNEL); if (!vsi->txq_map) @@ -105,14 +117,8 @@ static int ice_vsi_alloc_arrays(struct ice_vsi *vsi) if (!vsi->q_vectors) goto err_vectors; - vsi->af_xdp_zc_qps = bitmap_zalloc(max_t(int, vsi->alloc_txq, vsi->alloc_rxq), GFP_KERNEL); - if (!vsi->af_xdp_zc_qps) - goto err_zc_qps; - return 0; -err_zc_qps: - devm_kfree(dev, vsi->q_vectors); err_vectors: devm_kfree(dev, vsi->rxq_map); err_rxq_map: @@ -132,6 +138,7 @@ static void ice_vsi_set_num_desc(struct ice_vsi *vsi) { switch (vsi->type) { case ICE_VSI_PF: + case ICE_VSI_SF: case ICE_VSI_CTRL: case ICE_VSI_LB: /* a user could change the values of num_[tr]x_desc using @@ -150,32 +157,38 @@ static void ice_vsi_set_num_desc(struct ice_vsi *vsi) } } +static u16 ice_get_rxq_count(struct ice_pf *pf) +{ + return min(ice_get_avail_rxq_count(pf), num_online_cpus()); +} + +static u16 ice_get_txq_count(struct ice_pf *pf) +{ + return min(ice_get_avail_txq_count(pf), num_online_cpus()); +} + /** * ice_vsi_set_num_qs - Set number of queues, descriptors and vectors for a VSI * @vsi: the VSI being configured - * @vf_id: ID of the VF being configured * * Return 0 on success and a negative value on error */ -static void ice_vsi_set_num_qs(struct ice_vsi *vsi, u16 vf_id) +static void ice_vsi_set_num_qs(struct ice_vsi *vsi) { + enum ice_vsi_type vsi_type = vsi->type; struct ice_pf *pf = vsi->back; - struct ice_vf *vf = NULL; + struct ice_vf *vf = vsi->vf; - if (vsi->type == ICE_VSI_VF) - vsi->vf_id = vf_id; - else - vsi->vf_id = ICE_INVAL_VFID; + if (WARN_ON(vsi_type == ICE_VSI_VF && !vf)) + return; - switch (vsi->type) { + switch (vsi_type) { case ICE_VSI_PF: if (vsi->req_txq) { vsi->alloc_txq = vsi->req_txq; vsi->num_txq = vsi->req_txq; } else { - vsi->alloc_txq = min3(pf->num_lan_msix, - ice_get_avail_txq_count(pf), - (u16)num_online_cpus()); + vsi->alloc_txq = ice_get_txq_count(pf); } pf->num_lan_tx = vsi->alloc_txq; @@ -188,42 +201,47 @@ static void ice_vsi_set_num_qs(struct ice_vsi *vsi, u16 vf_id) vsi->alloc_rxq = vsi->req_rxq; vsi->num_rxq = vsi->req_rxq; } else { - vsi->alloc_rxq = min3(pf->num_lan_msix, - ice_get_avail_rxq_count(pf), - (u16)num_online_cpus()); + vsi->alloc_rxq = ice_get_rxq_count(pf); } } pf->num_lan_rx = vsi->alloc_rxq; - vsi->num_q_vectors = min_t(int, pf->num_lan_msix, - max_t(int, vsi->alloc_rxq, - vsi->alloc_txq)); + vsi->num_q_vectors = max(vsi->alloc_rxq, vsi->alloc_txq); + break; + case ICE_VSI_SF: + vsi->alloc_txq = 1; + vsi->alloc_rxq = 1; + vsi->num_q_vectors = 1; + vsi->irq_dyn_alloc = true; break; case ICE_VSI_VF: - vf = &pf->vf[vsi->vf_id]; if (vf->num_req_qs) vf->num_vf_qs = vf->num_req_qs; vsi->alloc_txq = vf->num_vf_qs; vsi->alloc_rxq = vf->num_vf_qs; - /* pf->num_msix_per_vf includes (VF miscellaneous vector + + /* pf->vfs.num_msix_per includes (VF miscellaneous vector + * data queue interrupts). Since vsi->num_q_vectors is number * of queues vectors, subtract 1 (ICE_NONQ_VECS_VF) from the * original vector count */ - vsi->num_q_vectors = pf->num_msix_per_vf - ICE_NONQ_VECS_VF; + vsi->num_q_vectors = vf->num_msix - ICE_NONQ_VECS_VF; break; case ICE_VSI_CTRL: vsi->alloc_txq = 1; vsi->alloc_rxq = 1; vsi->num_q_vectors = 1; break; + case ICE_VSI_CHNL: + vsi->alloc_txq = 0; + vsi->alloc_rxq = 0; + break; case ICE_VSI_LB: vsi->alloc_txq = 1; vsi->alloc_rxq = 1; break; default: - dev_warn(ice_pf_to_dev(pf), "Unknown VSI type %d\n", vsi->type); + dev_warn(ice_pf_to_dev(pf), "Unknown VSI type %d\n", vsi_type); break; } @@ -260,29 +278,30 @@ static int ice_get_free_slot(void *array, int size, int curr) } /** - * ice_vsi_delete - delete a VSI from the switch + * ice_vsi_delete_from_hw - delete a VSI from the switch * @vsi: pointer to VSI being removed */ -static void ice_vsi_delete(struct ice_vsi *vsi) +static void ice_vsi_delete_from_hw(struct ice_vsi *vsi) { struct ice_pf *pf = vsi->back; struct ice_vsi_ctx *ctxt; - enum ice_status status; + int status; + ice_fltr_remove_all(vsi); ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL); if (!ctxt) return; if (vsi->type == ICE_VSI_VF) - ctxt->vf_num = vsi->vf_id; + ctxt->vf_num = vsi->vf->vf_id; ctxt->vsi_num = vsi->vsi_num; memcpy(&ctxt->info, &vsi->info, sizeof(ctxt->info)); status = ice_free_vsi(&pf->hw, vsi->idx, ctxt, false, NULL); if (status) - dev_err(ice_pf_to_dev(pf), "Failed to delete VSI %i in FW - error: %s\n", - vsi->vsi_num, ice_stat_str(status)); + dev_err(ice_pf_to_dev(pf), "Failed to delete VSI %i in FW - error: %d\n", + vsi->vsi_num, status); kfree(ctxt); } @@ -298,76 +317,159 @@ static void ice_vsi_free_arrays(struct ice_vsi *vsi) dev = ice_pf_to_dev(pf); - if (vsi->af_xdp_zc_qps) { - bitmap_free(vsi->af_xdp_zc_qps); - vsi->af_xdp_zc_qps = NULL; - } /* free the ring and vector containers */ - if (vsi->q_vectors) { - devm_kfree(dev, vsi->q_vectors); - vsi->q_vectors = NULL; - } - if (vsi->tx_rings) { - devm_kfree(dev, vsi->tx_rings); - vsi->tx_rings = NULL; + devm_kfree(dev, vsi->q_vectors); + vsi->q_vectors = NULL; + devm_kfree(dev, vsi->tx_rings); + vsi->tx_rings = NULL; + devm_kfree(dev, vsi->rx_rings); + vsi->rx_rings = NULL; + devm_kfree(dev, vsi->txq_map); + vsi->txq_map = NULL; + devm_kfree(dev, vsi->rxq_map); + vsi->rxq_map = NULL; +} + +/** + * ice_vsi_free_stats - Free the ring statistics structures + * @vsi: VSI pointer + */ +static void ice_vsi_free_stats(struct ice_vsi *vsi) +{ + struct ice_vsi_stats *vsi_stat; + struct ice_pf *pf = vsi->back; + int i; + + if (vsi->type == ICE_VSI_CHNL) + return; + if (!pf->vsi_stats) + return; + + vsi_stat = pf->vsi_stats[vsi->idx]; + if (!vsi_stat) + return; + + ice_for_each_alloc_txq(vsi, i) { + if (vsi_stat->tx_ring_stats[i]) { + kfree_rcu(vsi_stat->tx_ring_stats[i], rcu); + WRITE_ONCE(vsi_stat->tx_ring_stats[i], NULL); + } } - if (vsi->rx_rings) { - devm_kfree(dev, vsi->rx_rings); - vsi->rx_rings = NULL; + + ice_for_each_alloc_rxq(vsi, i) { + if (vsi_stat->rx_ring_stats[i]) { + kfree_rcu(vsi_stat->rx_ring_stats[i], rcu); + WRITE_ONCE(vsi_stat->rx_ring_stats[i], NULL); + } } - if (vsi->txq_map) { - devm_kfree(dev, vsi->txq_map); - vsi->txq_map = NULL; + + kfree(vsi_stat->tx_ring_stats); + kfree(vsi_stat->rx_ring_stats); + kfree(vsi_stat); + pf->vsi_stats[vsi->idx] = NULL; +} + +/** + * ice_vsi_alloc_ring_stats - Allocates Tx and Rx ring stats for the VSI + * @vsi: VSI which is having stats allocated + */ +static int ice_vsi_alloc_ring_stats(struct ice_vsi *vsi) +{ + struct ice_ring_stats **tx_ring_stats; + struct ice_ring_stats **rx_ring_stats; + struct ice_vsi_stats *vsi_stats; + struct ice_pf *pf = vsi->back; + u16 i; + + vsi_stats = pf->vsi_stats[vsi->idx]; + tx_ring_stats = vsi_stats->tx_ring_stats; + rx_ring_stats = vsi_stats->rx_ring_stats; + + /* Allocate Tx ring stats */ + ice_for_each_alloc_txq(vsi, i) { + struct ice_ring_stats *ring_stats; + struct ice_tx_ring *ring; + + ring = vsi->tx_rings[i]; + ring_stats = tx_ring_stats[i]; + + if (!ring_stats) { + ring_stats = kzalloc(sizeof(*ring_stats), GFP_KERNEL); + if (!ring_stats) + goto err_out; + + WRITE_ONCE(tx_ring_stats[i], ring_stats); + } + + ring->ring_stats = ring_stats; } - if (vsi->rxq_map) { - devm_kfree(dev, vsi->rxq_map); - vsi->rxq_map = NULL; + + /* Allocate Rx ring stats */ + ice_for_each_alloc_rxq(vsi, i) { + struct ice_ring_stats *ring_stats; + struct ice_rx_ring *ring; + + ring = vsi->rx_rings[i]; + ring_stats = rx_ring_stats[i]; + + if (!ring_stats) { + ring_stats = kzalloc(sizeof(*ring_stats), GFP_KERNEL); + if (!ring_stats) + goto err_out; + + WRITE_ONCE(rx_ring_stats[i], ring_stats); + } + + ring->ring_stats = ring_stats; } + + return 0; + +err_out: + ice_vsi_free_stats(vsi); + return -ENOMEM; } /** - * ice_vsi_clear - clean up and deallocate the provided VSI + * ice_vsi_free - clean up and deallocate the provided VSI * @vsi: pointer to VSI being cleared * * This deallocates the VSI's queue resources, removes it from the PF's * VSI array if necessary, and deallocates the VSI - * - * Returns 0 on success, negative on failure */ -static int ice_vsi_clear(struct ice_vsi *vsi) +void ice_vsi_free(struct ice_vsi *vsi) { struct ice_pf *pf = NULL; struct device *dev; - if (!vsi) - return 0; - - if (!vsi->back) - return -EINVAL; + if (!vsi || !vsi->back) + return; pf = vsi->back; dev = ice_pf_to_dev(pf); if (!pf->vsi[vsi->idx] || pf->vsi[vsi->idx] != vsi) { dev_dbg(dev, "vsi does not exist at pf->vsi[%d]\n", vsi->idx); - return -EINVAL; + return; } mutex_lock(&pf->sw_mutex); /* updates the PF for this cleared VSI */ pf->vsi[vsi->idx] = NULL; - if (vsi->idx < pf->next_vsi && vsi->type != ICE_VSI_CTRL) - pf->next_vsi = vsi->idx; - if (vsi->idx < pf->next_vsi && vsi->type == ICE_VSI_CTRL && - vsi->vf_id != ICE_INVAL_VFID) - pf->next_vsi = vsi->idx; + pf->next_vsi = vsi->idx; + ice_vsi_free_stats(vsi); ice_vsi_free_arrays(vsi); + mutex_destroy(&vsi->xdp_state_lock); mutex_unlock(&pf->sw_mutex); devm_kfree(dev, vsi); +} - return 0; +void ice_vsi_delete(struct ice_vsi *vsi) +{ + ice_vsi_delete_from_hw(vsi); + ice_vsi_free(vsi); } /** @@ -379,12 +481,11 @@ static irqreturn_t ice_msix_clean_ctrl_vsi(int __always_unused irq, void *data) { struct ice_q_vector *q_vector = (struct ice_q_vector *)data; - if (!q_vector->tx.ring) + if (!q_vector->tx.tx_ring) return IRQ_HANDLED; -#define FDIR_RX_DESC_CLEAN_BUDGET 64 - ice_clean_rx_irq(q_vector->rx.ring, FDIR_RX_DESC_CLEAN_BUDGET); - ice_clean_ctrl_tx_irq(q_vector->tx.ring); + ice_clean_ctrl_rx_irq(q_vector->rx.rx_ring); + ice_clean_ctrl_tx_irq(q_vector->tx.tx_ring); return IRQ_HANDLED; } @@ -398,7 +499,7 @@ static irqreturn_t ice_msix_clean_rings(int __always_unused irq, void *data) { struct ice_q_vector *q_vector = (struct ice_q_vector *)data; - if (!q_vector->tx.ring && !q_vector->rx.ring) + if (!q_vector->tx.tx_ring && !q_vector->rx.rx_ring) return IRQ_HANDLED; q_vector->total_events++; @@ -409,15 +510,108 @@ static irqreturn_t ice_msix_clean_rings(int __always_unused irq, void *data) } /** + * ice_vsi_alloc_stat_arrays - Allocate statistics arrays + * @vsi: VSI pointer + */ +static int ice_vsi_alloc_stat_arrays(struct ice_vsi *vsi) +{ + struct ice_vsi_stats *vsi_stat; + struct ice_pf *pf = vsi->back; + + if (vsi->type == ICE_VSI_CHNL) + return 0; + if (!pf->vsi_stats) + return -ENOENT; + + if (pf->vsi_stats[vsi->idx]) + /* realloc will happen in rebuild path */ + return 0; + + vsi_stat = kzalloc(sizeof(*vsi_stat), GFP_KERNEL); + if (!vsi_stat) + return -ENOMEM; + + vsi_stat->tx_ring_stats = + kcalloc(vsi->alloc_txq, sizeof(*vsi_stat->tx_ring_stats), + GFP_KERNEL); + if (!vsi_stat->tx_ring_stats) + goto err_alloc_tx; + + vsi_stat->rx_ring_stats = + kcalloc(vsi->alloc_rxq, sizeof(*vsi_stat->rx_ring_stats), + GFP_KERNEL); + if (!vsi_stat->rx_ring_stats) + goto err_alloc_rx; + + pf->vsi_stats[vsi->idx] = vsi_stat; + + return 0; + +err_alloc_rx: + kfree(vsi_stat->rx_ring_stats); +err_alloc_tx: + kfree(vsi_stat->tx_ring_stats); + kfree(vsi_stat); + pf->vsi_stats[vsi->idx] = NULL; + return -ENOMEM; +} + +/** + * ice_vsi_alloc_def - set default values for already allocated VSI + * @vsi: ptr to VSI + * @ch: ptr to channel + */ +static int +ice_vsi_alloc_def(struct ice_vsi *vsi, struct ice_channel *ch) +{ + if (vsi->type != ICE_VSI_CHNL) { + ice_vsi_set_num_qs(vsi); + if (ice_vsi_alloc_arrays(vsi)) + return -ENOMEM; + } + + vsi->irq_dyn_alloc = pci_msix_can_alloc_dyn(vsi->back->pdev); + + switch (vsi->type) { + case ICE_VSI_PF: + case ICE_VSI_SF: + /* Setup default MSIX irq handler for VSI */ + vsi->irq_handler = ice_msix_clean_rings; + break; + case ICE_VSI_CTRL: + /* Setup ctrl VSI MSIX irq handler */ + vsi->irq_handler = ice_msix_clean_ctrl_vsi; + break; + case ICE_VSI_CHNL: + if (!ch) + return -EINVAL; + + vsi->num_rxq = ch->num_rxq; + vsi->num_txq = ch->num_txq; + vsi->next_base_q = ch->base_q; + break; + case ICE_VSI_VF: + case ICE_VSI_LB: + break; + default: + ice_vsi_free_arrays(vsi); + return -EINVAL; + } + + return 0; +} + +/** * ice_vsi_alloc - Allocates the next available struct VSI in the PF * @pf: board private structure - * @vsi_type: type of VSI - * @vf_id: ID of the VF being configured + * + * Reserves a VSI index from the PF and allocates an empty VSI structure + * without a type. The VSI structure must later be initialized by calling + * ice_vsi_cfg(). * * returns a pointer to a VSI on success, NULL on failure. */ -static struct ice_vsi * -ice_vsi_alloc(struct ice_pf *pf, enum ice_vsi_type vsi_type, u16 vf_id) +struct ice_vsi *ice_vsi_alloc(struct ice_pf *pf) { struct device *dev = ice_pf_to_dev(pf); struct ice_vsi *vsi = NULL; @@ -438,65 +632,19 @@ ice_vsi_alloc(struct ice_pf *pf, enum ice_vsi_type vsi_type, u16 vf_id) if (!vsi) goto unlock_pf; - vsi->type = vsi_type; vsi->back = pf; set_bit(ICE_VSI_DOWN, vsi->state); - if (vsi_type == ICE_VSI_VF) - ice_vsi_set_num_qs(vsi, vf_id); - else - ice_vsi_set_num_qs(vsi, ICE_INVAL_VFID); - - switch (vsi->type) { - case ICE_VSI_PF: - if (ice_vsi_alloc_arrays(vsi)) - goto err_rings; - - /* Setup default MSIX irq handler for VSI */ - vsi->irq_handler = ice_msix_clean_rings; - break; - case ICE_VSI_CTRL: - if (ice_vsi_alloc_arrays(vsi)) - goto err_rings; + /* fill slot and make note of the index */ + vsi->idx = pf->next_vsi; + pf->vsi[pf->next_vsi] = vsi; - /* Setup ctrl VSI MSIX irq handler */ - vsi->irq_handler = ice_msix_clean_ctrl_vsi; - break; - case ICE_VSI_VF: - if (ice_vsi_alloc_arrays(vsi)) - goto err_rings; - break; - case ICE_VSI_LB: - if (ice_vsi_alloc_arrays(vsi)) - goto err_rings; - break; - default: - dev_warn(dev, "Unknown VSI type %d\n", vsi->type); - goto unlock_pf; - } - - if (vsi->type == ICE_VSI_CTRL && vf_id == ICE_INVAL_VFID) { - /* Use the last VSI slot as the index for PF control VSI */ - vsi->idx = pf->num_alloc_vsi - 1; - pf->ctrl_vsi_idx = vsi->idx; - pf->vsi[vsi->idx] = vsi; - } else { - /* fill slot and make note of the index */ - vsi->idx = pf->next_vsi; - pf->vsi[pf->next_vsi] = vsi; - - /* prepare pf->next_vsi for next use */ - pf->next_vsi = ice_get_free_slot(pf->vsi, pf->num_alloc_vsi, - pf->next_vsi); - } + /* prepare pf->next_vsi for next use */ + pf->next_vsi = ice_get_free_slot(pf->vsi, pf->num_alloc_vsi, + pf->next_vsi); - if (vsi->type == ICE_VSI_CTRL && vf_id != ICE_INVAL_VFID) - pf->vf[vf_id].ctrl_vsi_idx = vsi->idx; - goto unlock_pf; + mutex_init(&vsi->xdp_state_lock); -err_rings: - devm_kfree(dev, vsi); - vsi = NULL; unlock_pf: mutex_unlock(&pf->sw_mutex); return vsi; @@ -515,10 +663,16 @@ static int ice_alloc_fd_res(struct ice_vsi *vsi) struct ice_pf *pf = vsi->back; u32 g_val, b_val; - /* Flow Director filters are only allocated/assigned to the PF VSI which - * passes the traffic. The CTRL VSI is only used to add/delete filters - * so we don't allocate resources to it + /* Flow Director filters are only allocated/assigned to the PF VSI or + * CHNL VSI which passes the traffic. The CTRL VSI is only used to + * add/delete filters so resources are not allocated to it */ + if (!test_bit(ICE_FLAG_FD_ENA, pf->flags)) + return -EPERM; + + if (!(vsi->type == ICE_VSI_PF || vsi->type == ICE_VSI_VF || + vsi->type == ICE_VSI_CHNL)) + return -EPERM; /* FD filters from guaranteed pool per VSI */ g_val = pf->hw.func_caps.fd_fltr_guar; @@ -530,19 +684,56 @@ static int ice_alloc_fd_res(struct ice_vsi *vsi) if (!b_val) return -EPERM; - if (!(vsi->type == ICE_VSI_PF || vsi->type == ICE_VSI_VF)) - return -EPERM; + /* PF main VSI gets only 64 FD resources from guaranteed pool + * when ADQ is configured. + */ +#define ICE_PF_VSI_GFLTR 64 - if (!test_bit(ICE_FLAG_FD_ENA, pf->flags)) - return -EPERM; + /* determine FD filter resources per VSI from shared(best effort) and + * dedicated pool + */ + if (vsi->type == ICE_VSI_PF) { + vsi->num_gfltr = g_val; + /* if MQPRIO is configured, main VSI doesn't get all FD + * resources from guaranteed pool. PF VSI gets 64 FD resources + */ + if (test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) { + if (g_val < ICE_PF_VSI_GFLTR) + return -EPERM; + /* allow bare minimum entries for PF VSI */ + vsi->num_gfltr = ICE_PF_VSI_GFLTR; + } - vsi->num_gfltr = g_val / pf->num_alloc_vsi; + /* each VSI gets same "best_effort" quota */ + vsi->num_bfltr = b_val; + } else if (vsi->type == ICE_VSI_VF) { + vsi->num_gfltr = 0; - /* each VSI gets same "best_effort" quota */ - vsi->num_bfltr = b_val; + /* each VSI gets same "best_effort" quota */ + vsi->num_bfltr = b_val; + } else { + struct ice_vsi *main_vsi; + int numtc; - if (vsi->type == ICE_VSI_VF) { - vsi->num_gfltr = 0; + main_vsi = ice_get_main_vsi(pf); + if (!main_vsi) + return -EPERM; + + if (!main_vsi->all_numtc) + return -EINVAL; + + /* figure out ADQ numtc */ + numtc = main_vsi->all_numtc - ICE_CHNL_START_TC; + + /* only one TC but still asking resources for channels, + * invalid config + */ + if (numtc < ICE_CHNL_START_TC) + return -EPERM; + + g_val -= ICE_PF_VSI_GFLTR; + /* channel VSIs gets equal share from guaranteed pool */ + vsi->num_gfltr = g_val / numtc; /* each VSI gets same "best_effort" quota */ vsi->num_bfltr = b_val; @@ -582,6 +773,9 @@ static int ice_vsi_get_qs(struct ice_vsi *vsi) }; int ret; + if (vsi->type == ICE_VSI_CHNL) + return 0; + ret = __ice_vsi_get_qs(&tx_qs_cfg); if (ret) return ret; @@ -606,12 +800,12 @@ static void ice_vsi_put_qs(struct ice_vsi *vsi) mutex_lock(&pf->avail_q_mutex); - for (i = 0; i < vsi->alloc_txq; i++) { + ice_for_each_alloc_txq(vsi, i) { clear_bit(vsi->txq_map[i], pf->avail_txqs); vsi->txq_map[i] = ICE_INVAL_Q_INDEX; } - for (i = 0; i < vsi->alloc_rxq; i++) { + ice_for_each_alloc_rxq(vsi, i) { clear_bit(vsi->rxq_map[i], pf->avail_rxqs); vsi->rxq_map[i] = ICE_INVAL_Q_INDEX; } @@ -631,14 +825,20 @@ bool ice_is_safe_mode(struct ice_pf *pf) } /** - * ice_is_aux_ena + * ice_is_rdma_ena * @pf: pointer to the PF struct * - * returns true if AUX devices/drivers are supported, false otherwise + * returns true if RDMA is currently supported, false otherwise */ -bool ice_is_aux_ena(struct ice_pf *pf) +bool ice_is_rdma_ena(struct ice_pf *pf) { - return test_bit(ICE_FLAG_AUX_ENA, pf->flags); + union devlink_param_value value; + int err; + + err = devl_param_driverinit_value_get(priv_to_devlink(pf), + DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA, + &value); + return err ? test_bit(ICE_FLAG_RDMA_ENA, pf->flags) : value.vbool; } /** @@ -651,15 +851,15 @@ bool ice_is_aux_ena(struct ice_pf *pf) static void ice_vsi_clean_rss_flow_fld(struct ice_vsi *vsi) { struct ice_pf *pf = vsi->back; - enum ice_status status; + int status; if (ice_is_safe_mode(pf)) return; status = ice_rem_vsi_rss_cfg(&pf->hw, vsi->idx); if (status) - dev_dbg(ice_pf_to_dev(pf), "ice_rem_vsi_rss_cfg failed for vsi = %d, error = %s\n", - vsi->vsi_num, ice_stat_str(status)); + dev_dbg(ice_pf_to_dev(pf), "ice_rem_vsi_rss_cfg failed for vsi = %d, error = %d\n", + vsi->vsi_num, status); } /** @@ -673,10 +873,8 @@ static void ice_rss_clean(struct ice_vsi *vsi) dev = ice_pf_to_dev(pf); - if (vsi->rss_hkey_user) - devm_kfree(dev, vsi->rss_hkey_user); - if (vsi->rss_lut_user) - devm_kfree(dev, vsi->rss_lut_user); + devm_kfree(dev, vsi->rss_hkey_user); + devm_kfree(dev, vsi->rss_lut_user); ice_vsi_clean_rss_flow_fld(vsi); /* remove RSS replay list */ @@ -692,6 +890,7 @@ static void ice_vsi_set_rss_params(struct ice_vsi *vsi) { struct ice_hw_common_caps *cap; struct ice_pf *pf = vsi->back; + u16 max_rss_size; if (!test_bit(ICE_FLAG_RSS_ENA, pf->flags)) { vsi->rss_size = 1; @@ -699,21 +898,31 @@ static void ice_vsi_set_rss_params(struct ice_vsi *vsi) } cap = &pf->hw.func_caps.common_cap; + max_rss_size = BIT(cap->rss_table_entry_width); switch (vsi->type) { + case ICE_VSI_CHNL: case ICE_VSI_PF: /* PF VSI will inherit RSS instance of PF */ vsi->rss_table_size = (u16)cap->rss_table_size; - vsi->rss_size = min_t(u16, num_online_cpus(), - BIT(cap->rss_table_entry_width)); - vsi->rss_lut_type = ICE_AQC_GSET_RSS_LUT_TABLE_TYPE_PF; + if (vsi->type == ICE_VSI_CHNL) + vsi->rss_size = min_t(u16, vsi->num_rxq, max_rss_size); + else + vsi->rss_size = min_t(u16, num_online_cpus(), + max_rss_size); + vsi->rss_lut_type = ICE_LUT_PF; + break; + case ICE_VSI_SF: + vsi->rss_table_size = ICE_LUT_VSI_SIZE; + vsi->rss_size = min_t(u16, num_online_cpus(), max_rss_size); + vsi->rss_lut_type = ICE_LUT_VSI; break; case ICE_VSI_VF: /* VF VSI will get a small RSS table. * For VSI_LUT, LUT size should be set to 64 bytes. */ - vsi->rss_table_size = ICE_VSIQF_HLUT_ARRAY_SIZE; + vsi->rss_table_size = ICE_LUT_VSI_SIZE; vsi->rss_size = ICE_MAX_RSS_QS_PER_VF; - vsi->rss_lut_type = ICE_AQC_GSET_RSS_LUT_TABLE_TYPE_VSI; + vsi->rss_lut_type = ICE_LUT_VSI; break; case ICE_VSI_LB: break; @@ -726,11 +935,12 @@ static void ice_vsi_set_rss_params(struct ice_vsi *vsi) /** * ice_set_dflt_vsi_ctx - Set default VSI context before adding a VSI + * @hw: HW structure used to determine the VLAN mode of the device * @ctxt: the VSI context being set * * This initializes a default VSI context for all sections except the Queues. */ -static void ice_set_dflt_vsi_ctx(struct ice_vsi_ctx *ctxt) +static void ice_set_dflt_vsi_ctx(struct ice_hw *hw, struct ice_vsi_ctx *ctxt) { u32 table = 0; @@ -741,13 +951,28 @@ static void ice_set_dflt_vsi_ctx(struct ice_vsi_ctx *ctxt) ctxt->info.sw_flags = ICE_AQ_VSI_SW_FLAG_SRC_PRUNE; /* Traffic from VSI can be sent to LAN */ ctxt->info.sw_flags2 = ICE_AQ_VSI_SW_FLAG_LAN_ENA; - /* By default bits 3 and 4 in vlan_flags are 0's which results in legacy - * behavior (show VLAN, DEI, and UP) in descriptor. Also, allow all - * packets untagged/tagged. + /* allow all untagged/tagged packets by default on Tx */ + ctxt->info.inner_vlan_flags = FIELD_PREP(ICE_AQ_VSI_INNER_VLAN_TX_MODE_M, + ICE_AQ_VSI_INNER_VLAN_TX_MODE_ALL); + /* SVM - by default bits 3 and 4 in inner_vlan_flags are 0's which + * results in legacy behavior (show VLAN, DEI, and UP) in descriptor. + * + * DVM - leave inner VLAN in packet by default */ - ctxt->info.vlan_flags = ((ICE_AQ_VSI_VLAN_MODE_ALL & - ICE_AQ_VSI_VLAN_MODE_M) >> - ICE_AQ_VSI_VLAN_MODE_S); + if (ice_is_dvm_ena(hw)) { + ctxt->info.inner_vlan_flags |= + FIELD_PREP(ICE_AQ_VSI_INNER_VLAN_EMODE_M, + ICE_AQ_VSI_INNER_VLAN_EMODE_NOTHING); + ctxt->info.outer_vlan_flags = + FIELD_PREP(ICE_AQ_VSI_OUTER_VLAN_TX_MODE_M, + ICE_AQ_VSI_OUTER_VLAN_TX_MODE_ALL); + ctxt->info.outer_vlan_flags |= + FIELD_PREP(ICE_AQ_VSI_OUTER_TAG_TYPE_M, + ICE_AQ_VSI_OUTER_TAG_VLAN_8100); + ctxt->info.outer_vlan_flags |= + FIELD_PREP(ICE_AQ_VSI_OUTER_VLAN_EMODE_M, + ICE_AQ_VSI_OUTER_VLAN_EMODE_NOTHING); + } /* Have 1:1 UP mapping for both ingress/egress tables */ table |= ICE_UP_TABLE_TRANSLATE(0, 0); table |= ICE_UP_TABLE_TRANSLATE(1, 1); @@ -769,27 +994,19 @@ static void ice_set_dflt_vsi_ctx(struct ice_vsi_ctx *ctxt) * @vsi: the VSI being configured * @ctxt: VSI context structure */ -static void ice_vsi_setup_q_map(struct ice_vsi *vsi, struct ice_vsi_ctx *ctxt) +static int ice_vsi_setup_q_map(struct ice_vsi *vsi, struct ice_vsi_ctx *ctxt) { - u16 offset = 0, qmap = 0, tx_count = 0, pow = 0; + u16 offset = 0, qmap = 0, tx_count = 0, rx_count = 0, pow = 0; u16 num_txq_per_tc, num_rxq_per_tc; u16 qcount_tx = vsi->alloc_txq; u16 qcount_rx = vsi->alloc_rxq; - bool ena_tc0 = false; u8 netdev_tc = 0; int i; - /* at least TC0 should be enabled by default */ - if (vsi->tc_cfg.numtc) { - if (!(vsi->tc_cfg.ena_tc & BIT(0))) - ena_tc0 = true; - } else { - ena_tc0 = true; - } - - if (ena_tc0) { - vsi->tc_cfg.numtc++; - vsi->tc_cfg.ena_tc |= 1; + if (!vsi->tc_cfg.numtc) { + /* at least TC0 should be enabled by default */ + vsi->tc_cfg.numtc = 1; + vsi->tc_cfg.ena_tc = 1; } num_rxq_per_tc = min_t(u16, qcount_rx / vsi->tc_cfg.numtc, ICE_MAX_RXQS_PER_TC); @@ -830,10 +1047,8 @@ static void ice_vsi_setup_q_map(struct ice_vsi *vsi, struct ice_vsi_ctx *ctxt) vsi->tc_cfg.tc_info[i].qcount_tx = num_txq_per_tc; vsi->tc_cfg.tc_info[i].netdev_tc = netdev_tc++; - qmap = ((offset << ICE_AQ_VSI_TC_Q_OFFSET_S) & - ICE_AQ_VSI_TC_Q_OFFSET_M) | - ((pow << ICE_AQ_VSI_TC_Q_NUM_S) & - ICE_AQ_VSI_TC_Q_NUM_M); + qmap = FIELD_PREP(ICE_AQ_VSI_TC_Q_OFFSET_M, offset); + qmap |= FIELD_PREP(ICE_AQ_VSI_TC_Q_NUM_M, pow); offset += num_rxq_per_tc; tx_count += num_txq_per_tc; ctxt->info.tc_mapping[i] = cpu_to_le16(qmap); @@ -846,11 +1061,24 @@ static void ice_vsi_setup_q_map(struct ice_vsi *vsi, struct ice_vsi_ctx *ctxt) * at least 1) */ if (offset) - vsi->num_rxq = offset; + rx_count = offset; else - vsi->num_rxq = num_rxq_per_tc; + rx_count = num_rxq_per_tc; + + if (rx_count > vsi->alloc_rxq) { + dev_err(ice_pf_to_dev(vsi->back), "Trying to use more Rx queues (%u), than were allocated (%u)!\n", + rx_count, vsi->alloc_rxq); + return -EINVAL; + } + + if (tx_count > vsi->alloc_txq) { + dev_err(ice_pf_to_dev(vsi->back), "Trying to use more Tx queues (%u), than were allocated (%u)!\n", + tx_count, vsi->alloc_txq); + return -EINVAL; + } vsi->num_txq = tx_count; + vsi->num_rxq = rx_count; if (vsi->type == ICE_VSI_VF && vsi->num_txq != vsi->num_rxq) { dev_dbg(ice_pf_to_dev(vsi->back), "VF VSI should have same number of Tx and Rx queues. Hence making them equal\n"); @@ -868,6 +1096,8 @@ static void ice_vsi_setup_q_map(struct ice_vsi *vsi, struct ice_vsi_ctx *ctxt) */ ctxt->info.q_mapping[0] = cpu_to_le16(vsi->rxq_map[0]); ctxt->info.q_mapping[1] = cpu_to_le16(vsi->num_rxq); + + return 0; } /** @@ -881,7 +1111,7 @@ static void ice_set_fd_vsi_ctx(struct ice_vsi_ctx *ctxt, struct ice_vsi *vsi) u16 dflt_q, report_q, val; if (vsi->type != ICE_VSI_PF && vsi->type != ICE_VSI_CTRL && - vsi->type != ICE_VSI_VF) + vsi->type != ICE_VSI_VF && vsi->type != ICE_VSI_CHNL) return; val = ICE_AQ_VSI_PROP_FLOW_DIR_VALID; @@ -901,18 +1131,14 @@ static void ice_set_fd_vsi_ctx(struct ice_vsi_ctx *ctxt, struct ice_vsi *vsi) ctxt->info.max_fd_fltr_shared = cpu_to_le16(vsi->num_bfltr); /* default queue index within the VSI of the default FD */ - val = ((dflt_q << ICE_AQ_VSI_FD_DEF_Q_S) & - ICE_AQ_VSI_FD_DEF_Q_M); + val = FIELD_PREP(ICE_AQ_VSI_FD_DEF_Q_M, dflt_q); /* target queue or queue group to the FD filter */ - val |= ((dflt_q_group << ICE_AQ_VSI_FD_DEF_GRP_S) & - ICE_AQ_VSI_FD_DEF_GRP_M); + val |= FIELD_PREP(ICE_AQ_VSI_FD_DEF_GRP_M, dflt_q_group); ctxt->info.fd_def_q = cpu_to_le16(val); /* queue index on which FD filter completion is reported */ - val = ((report_q << ICE_AQ_VSI_FD_REPORT_Q_S) & - ICE_AQ_VSI_FD_REPORT_Q_M); + val = FIELD_PREP(ICE_AQ_VSI_FD_REPORT_Q_M, report_q); /* priority of the default qindex action */ - val |= ((dflt_q_prio << ICE_AQ_VSI_FD_DEF_PRIORITY_S) & - ICE_AQ_VSI_FD_DEF_PRIORITY_M); + val |= FIELD_PREP(ICE_AQ_VSI_FD_DEF_PRIORITY_M, dflt_q_prio); ctxt->info.fd_report_opt = cpu_to_le16(val); } @@ -931,15 +1157,15 @@ static void ice_set_rss_vsi_ctx(struct ice_vsi_ctx *ctxt, struct ice_vsi *vsi) dev = ice_pf_to_dev(pf); switch (vsi->type) { + case ICE_VSI_CHNL: case ICE_VSI_PF: /* PF VSI will inherit RSS instance of PF */ lut_type = ICE_AQ_VSI_Q_OPT_RSS_LUT_PF; - hash_type = ICE_AQ_VSI_Q_OPT_RSS_TPLZ; break; case ICE_VSI_VF: + case ICE_VSI_SF: /* VF VSI will gets a small RSS table which is a VSI LUT type */ lut_type = ICE_AQ_VSI_Q_OPT_RSS_LUT_VSI; - hash_type = ICE_AQ_VSI_Q_OPT_RSS_TPLZ; break; default: dev_dbg(dev, "Unsupported VSI type %s\n", @@ -947,21 +1173,56 @@ static void ice_set_rss_vsi_ctx(struct ice_vsi_ctx *ctxt, struct ice_vsi *vsi) return; } - ctxt->info.q_opt_rss = ((lut_type << ICE_AQ_VSI_Q_OPT_RSS_LUT_S) & - ICE_AQ_VSI_Q_OPT_RSS_LUT_M) | - ((hash_type << ICE_AQ_VSI_Q_OPT_RSS_HASH_S) & - ICE_AQ_VSI_Q_OPT_RSS_HASH_M); + hash_type = ICE_AQ_VSI_Q_OPT_RSS_HASH_TPLZ; + vsi->rss_hfunc = hash_type; + + ctxt->info.q_opt_rss = + FIELD_PREP(ICE_AQ_VSI_Q_OPT_RSS_LUT_M, lut_type) | + FIELD_PREP(ICE_AQ_VSI_Q_OPT_RSS_HASH_M, hash_type); +} + +static void +ice_chnl_vsi_setup_q_map(struct ice_vsi *vsi, struct ice_vsi_ctx *ctxt) +{ + u16 qcount, qmap; + u8 offset = 0; + int pow; + + qcount = vsi->num_rxq; + + pow = order_base_2(qcount); + qmap = FIELD_PREP(ICE_AQ_VSI_TC_Q_OFFSET_M, offset); + qmap |= FIELD_PREP(ICE_AQ_VSI_TC_Q_NUM_M, pow); + + ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); + ctxt->info.mapping_flags |= cpu_to_le16(ICE_AQ_VSI_Q_MAP_CONTIG); + ctxt->info.q_mapping[0] = cpu_to_le16(vsi->next_base_q); + ctxt->info.q_mapping[1] = cpu_to_le16(qcount); +} + +/** + * ice_vsi_is_vlan_pruning_ena - check if VLAN pruning is enabled or not + * @vsi: VSI to check whether or not VLAN pruning is enabled. + * + * returns true if Rx VLAN pruning is enabled and false otherwise. + */ +static bool ice_vsi_is_vlan_pruning_ena(struct ice_vsi *vsi) +{ + return vsi->info.sw_flags2 & ICE_AQ_VSI_SW_FLAG_RX_VLAN_PRUNE_ENA; } /** * ice_vsi_init - Create and initialize a VSI * @vsi: the VSI being configured - * @init_vsi: is this call creating a VSI + * @vsi_flags: VSI configuration flags + * + * Set ICE_FLAG_VSI_INIT to initialize a new VSI context, clear it to + * reconfigure an existing context. * * This initializes a VSI context depending on the VSI type to be added and * passes it down to the add_vsi aq command to create a new VSI. */ -static int ice_vsi_init(struct ice_vsi *vsi, bool init_vsi) +static int ice_vsi_init(struct ice_vsi *vsi, u32 vsi_flags) { struct ice_pf *pf = vsi->back; struct ice_hw *hw = &pf->hw; @@ -980,17 +1241,36 @@ static int ice_vsi_init(struct ice_vsi *vsi, bool init_vsi) case ICE_VSI_PF: ctxt->flags = ICE_AQ_VSI_TYPE_PF; break; + case ICE_VSI_SF: + case ICE_VSI_CHNL: + ctxt->flags = ICE_AQ_VSI_TYPE_VMDQ2; + break; case ICE_VSI_VF: ctxt->flags = ICE_AQ_VSI_TYPE_VF; /* VF number here is the absolute VF number (0-255) */ - ctxt->vf_num = vsi->vf_id + hw->func_caps.vf_base_id; + ctxt->vf_num = vsi->vf->vf_id + hw->func_caps.vf_base_id; break; default: ret = -ENODEV; goto out; } - ice_set_dflt_vsi_ctx(ctxt); + /* Handle VLAN pruning for channel VSI if main VSI has VLAN + * prune enabled + */ + if (vsi->type == ICE_VSI_CHNL) { + struct ice_vsi *main_vsi; + + main_vsi = ice_get_main_vsi(pf); + if (main_vsi && ice_vsi_is_vlan_pruning_ena(main_vsi)) + ctxt->info.sw_flags2 |= + ICE_AQ_VSI_SW_FLAG_RX_VLAN_PRUNE_ENA; + else + ctxt->info.sw_flags2 &= + ~ICE_AQ_VSI_SW_FLAG_RX_VLAN_PRUNE_ENA; + } + + ice_set_dflt_vsi_ctx(hw, ctxt); if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) ice_set_fd_vsi_ctx(ctxt, vsi); /* if the switch is in VEB mode, allow VSI loopback */ @@ -1004,37 +1284,26 @@ static int ice_vsi_init(struct ice_vsi *vsi, bool init_vsi) /* if updating VSI context, make sure to set valid_section: * to indicate which section of VSI context being updated */ - if (!init_vsi) + if (!(vsi_flags & ICE_VSI_FLAG_INIT)) ctxt->info.valid_sections |= cpu_to_le16(ICE_AQ_VSI_PROP_Q_OPT_VALID); } ctxt->info.sw_id = vsi->port_info->sw_id; - ice_vsi_setup_q_map(vsi, ctxt); - if (!init_vsi) /* means VSI being updated */ - /* must to indicate which section of VSI context are - * being modified - */ - ctxt->info.valid_sections |= - cpu_to_le16(ICE_AQ_VSI_PROP_RXQ_MAP_VALID); + if (vsi->type == ICE_VSI_CHNL) { + ice_chnl_vsi_setup_q_map(vsi, ctxt); + } else { + ret = ice_vsi_setup_q_map(vsi, ctxt); + if (ret) + goto out; - /* enable/disable MAC and VLAN anti-spoof when spoofchk is on/off - * respectively - */ - if (vsi->type == ICE_VSI_VF) { - ctxt->info.valid_sections |= - cpu_to_le16(ICE_AQ_VSI_PROP_SECURITY_VALID); - if (pf->vf[vsi->vf_id].spoofchk) { - ctxt->info.sec_flags |= - ICE_AQ_VSI_SEC_FLAG_ENA_MAC_ANTI_SPOOF | - (ICE_AQ_VSI_SEC_TX_VLAN_PRUNE_ENA << - ICE_AQ_VSI_SEC_TX_PRUNE_ENA_S); - } else { - ctxt->info.sec_flags &= - ~(ICE_AQ_VSI_SEC_FLAG_ENA_MAC_ANTI_SPOOF | - (ICE_AQ_VSI_SEC_TX_VLAN_PRUNE_ENA << - ICE_AQ_VSI_SEC_TX_PRUNE_ENA_S)); - } + if (!(vsi_flags & ICE_VSI_FLAG_INIT)) + /* means VSI being updated */ + /* must to indicate which section of VSI context are + * being modified + */ + ctxt->info.valid_sections |= + cpu_to_le16(ICE_AQ_VSI_PROP_RXQ_MAP_VALID); } /* Allow control frames out of main VSI */ @@ -1044,7 +1313,7 @@ static int ice_vsi_init(struct ice_vsi *vsi, bool init_vsi) cpu_to_le16(ICE_AQ_VSI_PROP_SECURITY_VALID); } - if (init_vsi) { + if (vsi_flags & ICE_VSI_FLAG_INIT) { ret = ice_add_vsi(hw, vsi->idx, ctxt, NULL); if (ret) { dev_err(dev, "Add VSI failed, err %d\n", ret); @@ -1072,170 +1341,6 @@ out: } /** - * ice_free_res - free a block of resources - * @res: pointer to the resource - * @index: starting index previously returned by ice_get_res - * @id: identifier to track owner - * - * Returns number of resources freed - */ -int ice_free_res(struct ice_res_tracker *res, u16 index, u16 id) -{ - int count = 0; - int i; - - if (!res || index >= res->end) - return -EINVAL; - - id |= ICE_RES_VALID_BIT; - for (i = index; i < res->end && res->list[i] == id; i++) { - res->list[i] = 0; - count++; - } - - return count; -} - -/** - * ice_search_res - Search the tracker for a block of resources - * @res: pointer to the resource - * @needed: size of the block needed - * @id: identifier to track owner - * - * Returns the base item index of the block, or -ENOMEM for error - */ -static int ice_search_res(struct ice_res_tracker *res, u16 needed, u16 id) -{ - u16 start = 0, end = 0; - - if (needed > res->end) - return -ENOMEM; - - id |= ICE_RES_VALID_BIT; - - do { - /* skip already allocated entries */ - if (res->list[end++] & ICE_RES_VALID_BIT) { - start = end; - if ((start + needed) > res->end) - break; - } - - if (end == (start + needed)) { - int i = start; - - /* there was enough, so assign it to the requestor */ - while (i != end) - res->list[i++] = id; - - return start; - } - } while (end < res->end); - - return -ENOMEM; -} - -/** - * ice_get_free_res_count - Get free count from a resource tracker - * @res: Resource tracker instance - */ -static u16 ice_get_free_res_count(struct ice_res_tracker *res) -{ - u16 i, count = 0; - - for (i = 0; i < res->end; i++) - if (!(res->list[i] & ICE_RES_VALID_BIT)) - count++; - - return count; -} - -/** - * ice_get_res - get a block of resources - * @pf: board private structure - * @res: pointer to the resource - * @needed: size of the block needed - * @id: identifier to track owner - * - * Returns the base item index of the block, or negative for error - */ -int -ice_get_res(struct ice_pf *pf, struct ice_res_tracker *res, u16 needed, u16 id) -{ - if (!res || !pf) - return -EINVAL; - - if (!needed || needed > res->num_entries || id >= ICE_RES_VALID_BIT) { - dev_err(ice_pf_to_dev(pf), "param err: needed=%d, num_entries = %d id=0x%04x\n", - needed, res->num_entries, id); - return -EINVAL; - } - - return ice_search_res(res, needed, id); -} - -/** - * ice_vsi_setup_vector_base - Set up the base vector for the given VSI - * @vsi: ptr to the VSI - * - * This should only be called after ice_vsi_alloc() which allocates the - * corresponding SW VSI structure and initializes num_queue_pairs for the - * newly allocated VSI. - * - * Returns 0 on success or negative on failure - */ -static int ice_vsi_setup_vector_base(struct ice_vsi *vsi) -{ - struct ice_pf *pf = vsi->back; - struct device *dev; - u16 num_q_vectors; - int base; - - dev = ice_pf_to_dev(pf); - /* SRIOV doesn't grab irq_tracker entries for each VSI */ - if (vsi->type == ICE_VSI_VF) - return 0; - - if (vsi->base_vector) { - dev_dbg(dev, "VSI %d has non-zero base vector %d\n", - vsi->vsi_num, vsi->base_vector); - return -EEXIST; - } - - num_q_vectors = vsi->num_q_vectors; - /* reserve slots from OS requested IRQs */ - if (vsi->type == ICE_VSI_CTRL && vsi->vf_id != ICE_INVAL_VFID) { - int i; - - ice_for_each_vf(pf, i) { - struct ice_vf *vf = &pf->vf[i]; - - if (i != vsi->vf_id && vf->ctrl_vsi_idx != ICE_NO_VSI) { - base = pf->vsi[vf->ctrl_vsi_idx]->base_vector; - break; - } - } - if (i == pf->num_alloc_vfs) - base = ice_get_res(pf, pf->irq_tracker, num_q_vectors, - ICE_RES_VF_CTRL_VEC_ID); - } else { - base = ice_get_res(pf, pf->irq_tracker, num_q_vectors, - vsi->idx); - } - - if (base < 0) { - dev_err(dev, "%d MSI-X interrupts available. %s %d failed to get %d MSI-X vectors\n", - ice_get_free_res_count(pf->irq_tracker), - ice_vsi_type_str(vsi->type), vsi->idx, num_q_vectors); - return -ENOENT; - } - vsi->base_vector = (u16)base; - pf->num_avail_sw_msix -= num_q_vectors; - - return 0; -} - -/** * ice_vsi_clear_rings - Deallocates the Tx and Rx rings for VSI * @vsi: the VSI having rings deallocated */ @@ -1249,14 +1354,14 @@ static void ice_vsi_clear_rings(struct ice_vsi *vsi) struct ice_q_vector *q_vector = vsi->q_vectors[i]; if (q_vector) { - q_vector->tx.ring = NULL; - q_vector->rx.ring = NULL; + q_vector->tx.tx_ring = NULL; + q_vector->rx.rx_ring = NULL; } } } if (vsi->tx_rings) { - for (i = 0; i < vsi->alloc_txq; i++) { + ice_for_each_alloc_txq(vsi, i) { if (vsi->tx_rings[i]) { kfree_rcu(vsi->tx_rings[i], rcu); WRITE_ONCE(vsi->tx_rings[i], NULL); @@ -1264,7 +1369,7 @@ static void ice_vsi_clear_rings(struct ice_vsi *vsi) } } if (vsi->rx_rings) { - for (i = 0; i < vsi->alloc_rxq; i++) { + ice_for_each_alloc_rxq(vsi, i) { if (vsi->rx_rings[i]) { kfree_rcu(vsi->rx_rings[i], rcu); WRITE_ONCE(vsi->rx_rings[i], NULL); @@ -1279,14 +1384,15 @@ static void ice_vsi_clear_rings(struct ice_vsi *vsi) */ static int ice_vsi_alloc_rings(struct ice_vsi *vsi) { + bool dvm_ena = ice_is_dvm_ena(&vsi->back->hw); struct ice_pf *pf = vsi->back; struct device *dev; u16 i; dev = ice_pf_to_dev(pf); /* Allocate Tx rings */ - for (i = 0; i < vsi->alloc_txq; i++) { - struct ice_ring *ring; + ice_for_each_alloc_txq(vsi, i) { + struct ice_tx_ring *ring; /* allocate with kzalloc(), free with kfree_rcu() */ ring = kzalloc(sizeof(*ring), GFP_KERNEL); @@ -1296,17 +1402,21 @@ static int ice_vsi_alloc_rings(struct ice_vsi *vsi) ring->q_index = i; ring->reg_idx = vsi->txq_map[i]; - ring->ring_active = false; ring->vsi = vsi; ring->tx_tstamps = &pf->ptp.port.tx; ring->dev = dev; ring->count = vsi->num_tx_desc; + ring->txq_teid = ICE_INVAL_TEID; + if (dvm_ena) + ring->flags |= ICE_TX_FLAGS_RING_VLAN_L2TAG2; + else + ring->flags |= ICE_TX_FLAGS_RING_VLAN_L2TAG1; WRITE_ONCE(vsi->tx_rings[i], ring); } /* Allocate Rx rings */ - for (i = 0; i < vsi->alloc_rxq; i++) { - struct ice_ring *ring; + ice_for_each_alloc_rxq(vsi, i) { + struct ice_rx_ring *ring; /* allocate with kzalloc(), free with kfree_rcu() */ ring = kzalloc(sizeof(*ring), GFP_KERNEL); @@ -1315,11 +1425,14 @@ static int ice_vsi_alloc_rings(struct ice_vsi *vsi) ring->q_index = i; ring->reg_idx = vsi->rxq_map[i]; - ring->ring_active = false; ring->vsi = vsi; ring->netdev = vsi->netdev; - ring->dev = dev; ring->count = vsi->num_rx_desc; + ring->cached_phctime = pf->ptp.cached_phc_time; + + if (ice_is_feature_supported(pf, ICE_F_GCS)) + ring->flags |= ICE_RX_FLAGS_RING_GCS; + WRITE_ONCE(vsi->rx_rings[i], ring); } @@ -1360,10 +1473,26 @@ void ice_vsi_manage_rss_lut(struct ice_vsi *vsi, bool ena) } /** + * ice_vsi_cfg_crc_strip - Configure CRC stripping for a VSI + * @vsi: VSI to be configured + * @disable: set to true to have FCS / CRC in the frame data + */ +void ice_vsi_cfg_crc_strip(struct ice_vsi *vsi, bool disable) +{ + int i; + + ice_for_each_rxq(vsi, i) + if (disable) + vsi->rx_rings[i]->flags |= ICE_RX_FLAGS_CRC_STRIP_DIS; + else + vsi->rx_rings[i]->flags &= ~ICE_RX_FLAGS_CRC_STRIP_DIS; +} + +/** * ice_vsi_cfg_rss_lut_key - Configure RSS params for a VSI * @vsi: VSI to be configured */ -static int ice_vsi_cfg_rss_lut_key(struct ice_vsi *vsi) +int ice_vsi_cfg_rss_lut_key(struct ice_vsi *vsi) { struct ice_pf *pf = vsi->back; struct device *dev; @@ -1371,7 +1500,25 @@ static int ice_vsi_cfg_rss_lut_key(struct ice_vsi *vsi) int err; dev = ice_pf_to_dev(pf); - vsi->rss_size = min_t(u16, vsi->rss_size, vsi->num_rxq); + if (vsi->type == ICE_VSI_PF && vsi->ch_rss_size && + (test_bit(ICE_FLAG_TC_MQPRIO, pf->flags))) { + vsi->rss_size = min_t(u16, vsi->rss_size, vsi->ch_rss_size); + } else { + vsi->rss_size = min_t(u16, vsi->rss_size, vsi->num_rxq); + + /* If orig_rss_size is valid and it is less than determined + * main VSI's rss_size, update main VSI's rss_size to be + * orig_rss_size so that when tc-qdisc is deleted, main VSI + * RSS table gets programmed to be correct (whatever it was + * to begin with (prior to setup-tc for ADQ config) + */ + if (vsi->orig_rss_size && vsi->rss_size < vsi->orig_rss_size && + vsi->orig_rss_size <= vsi->num_rxq) { + vsi->rss_size = vsi->orig_rss_size; + /* now orig_rss_size is used, reset it to zero */ + vsi->orig_rss_size = 0; + } + } lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); if (!lut) @@ -1420,8 +1567,8 @@ ice_vsi_cfg_rss_exit: static void ice_vsi_set_vf_rss_flow_fld(struct ice_vsi *vsi) { struct ice_pf *pf = vsi->back; - enum ice_status status; struct device *dev; + int status; dev = ice_pf_to_dev(pf); if (ice_is_safe_mode(pf)) { @@ -1430,12 +1577,81 @@ static void ice_vsi_set_vf_rss_flow_fld(struct ice_vsi *vsi) return; } - status = ice_add_avf_rss_cfg(&pf->hw, vsi->idx, ICE_DEFAULT_RSS_HENA); + status = ice_add_avf_rss_cfg(&pf->hw, vsi, ICE_DEFAULT_RSS_HASHCFG); if (status) - dev_dbg(dev, "ice_add_avf_rss_cfg failed for vsi = %d, error = %s\n", - vsi->vsi_num, ice_stat_str(status)); + dev_dbg(dev, "ice_add_avf_rss_cfg failed for vsi = %d, error = %d\n", + vsi->vsi_num, status); } +static const struct ice_rss_hash_cfg default_rss_cfgs[] = { + /* configure RSS for IPv4 with input set IP src/dst */ + {ICE_FLOW_SEG_HDR_IPV4, ICE_FLOW_HASH_IPV4, ICE_RSS_ANY_HEADERS, false}, + /* configure RSS for IPv6 with input set IPv6 src/dst */ + {ICE_FLOW_SEG_HDR_IPV6, ICE_FLOW_HASH_IPV6, ICE_RSS_ANY_HEADERS, false}, + /* configure RSS for tcp4 with input set IP src/dst, TCP src/dst */ + {ICE_FLOW_SEG_HDR_TCP | ICE_FLOW_SEG_HDR_IPV4, + ICE_HASH_TCP_IPV4, ICE_RSS_ANY_HEADERS, false}, + /* configure RSS for udp4 with input set IP src/dst, UDP src/dst */ + {ICE_FLOW_SEG_HDR_UDP | ICE_FLOW_SEG_HDR_IPV4, + ICE_HASH_UDP_IPV4, ICE_RSS_ANY_HEADERS, false}, + /* configure RSS for sctp4 with input set IP src/dst - only support + * RSS on SCTPv4 on outer headers (non-tunneled) + */ + {ICE_FLOW_SEG_HDR_SCTP | ICE_FLOW_SEG_HDR_IPV4, + ICE_HASH_SCTP_IPV4, ICE_RSS_OUTER_HEADERS, false}, + /* configure RSS for gtpc4 with input set IPv4 src/dst */ + {ICE_FLOW_SEG_HDR_GTPC | ICE_FLOW_SEG_HDR_IPV4, + ICE_FLOW_HASH_IPV4, ICE_RSS_OUTER_HEADERS, false}, + /* configure RSS for gtpc4t with input set IPv4 src/dst */ + {ICE_FLOW_SEG_HDR_GTPC_TEID | ICE_FLOW_SEG_HDR_IPV4, + ICE_FLOW_HASH_GTP_C_IPV4_TEID, ICE_RSS_OUTER_HEADERS, false}, + /* configure RSS for gtpu4 with input set IPv4 src/dst */ + {ICE_FLOW_SEG_HDR_GTPU_IP | ICE_FLOW_SEG_HDR_IPV4, + ICE_FLOW_HASH_GTP_U_IPV4_TEID, ICE_RSS_OUTER_HEADERS, false}, + /* configure RSS for gtpu4e with input set IPv4 src/dst */ + {ICE_FLOW_SEG_HDR_GTPU_EH | ICE_FLOW_SEG_HDR_IPV4, + ICE_FLOW_HASH_GTP_U_IPV4_EH, ICE_RSS_OUTER_HEADERS, false}, + /* configure RSS for gtpu4u with input set IPv4 src/dst */ + { ICE_FLOW_SEG_HDR_GTPU_UP | ICE_FLOW_SEG_HDR_IPV4, + ICE_FLOW_HASH_GTP_U_IPV4_UP, ICE_RSS_OUTER_HEADERS, false}, + /* configure RSS for gtpu4d with input set IPv4 src/dst */ + {ICE_FLOW_SEG_HDR_GTPU_DWN | ICE_FLOW_SEG_HDR_IPV4, + ICE_FLOW_HASH_GTP_U_IPV4_DWN, ICE_RSS_OUTER_HEADERS, false}, + + /* configure RSS for tcp6 with input set IPv6 src/dst, TCP src/dst */ + {ICE_FLOW_SEG_HDR_TCP | ICE_FLOW_SEG_HDR_IPV6, + ICE_HASH_TCP_IPV6, ICE_RSS_ANY_HEADERS, false}, + /* configure RSS for udp6 with input set IPv6 src/dst, UDP src/dst */ + {ICE_FLOW_SEG_HDR_UDP | ICE_FLOW_SEG_HDR_IPV6, + ICE_HASH_UDP_IPV6, ICE_RSS_ANY_HEADERS, false}, + /* configure RSS for sctp6 with input set IPv6 src/dst - only support + * RSS on SCTPv6 on outer headers (non-tunneled) + */ + {ICE_FLOW_SEG_HDR_SCTP | ICE_FLOW_SEG_HDR_IPV6, + ICE_HASH_SCTP_IPV6, ICE_RSS_OUTER_HEADERS, false}, + /* configure RSS for IPSEC ESP SPI with input set MAC_IPV4_SPI */ + {ICE_FLOW_SEG_HDR_ESP, + ICE_FLOW_HASH_ESP_SPI, ICE_RSS_OUTER_HEADERS, false}, + /* configure RSS for gtpc6 with input set IPv6 src/dst */ + {ICE_FLOW_SEG_HDR_GTPC | ICE_FLOW_SEG_HDR_IPV6, + ICE_FLOW_HASH_IPV6, ICE_RSS_OUTER_HEADERS, false}, + /* configure RSS for gtpc6t with input set IPv6 src/dst */ + {ICE_FLOW_SEG_HDR_GTPC_TEID | ICE_FLOW_SEG_HDR_IPV6, + ICE_FLOW_HASH_GTP_C_IPV6_TEID, ICE_RSS_OUTER_HEADERS, false}, + /* configure RSS for gtpu6 with input set IPv6 src/dst */ + {ICE_FLOW_SEG_HDR_GTPU_IP | ICE_FLOW_SEG_HDR_IPV6, + ICE_FLOW_HASH_GTP_U_IPV6_TEID, ICE_RSS_OUTER_HEADERS, false}, + /* configure RSS for gtpu6e with input set IPv6 src/dst */ + {ICE_FLOW_SEG_HDR_GTPU_EH | ICE_FLOW_SEG_HDR_IPV6, + ICE_FLOW_HASH_GTP_U_IPV6_EH, ICE_RSS_OUTER_HEADERS, false}, + /* configure RSS for gtpu6u with input set IPv6 src/dst */ + { ICE_FLOW_SEG_HDR_GTPU_UP | ICE_FLOW_SEG_HDR_IPV6, + ICE_FLOW_HASH_GTP_U_IPV6_UP, ICE_RSS_OUTER_HEADERS, false}, + /* configure RSS for gtpu6d with input set IPv6 src/dst */ + {ICE_FLOW_SEG_HDR_GTPU_DWN | ICE_FLOW_SEG_HDR_IPV6, + ICE_FLOW_HASH_GTP_U_IPV6_DWN, ICE_RSS_OUTER_HEADERS, false}, +}; + /** * ice_vsi_set_rss_flow_fld - Sets RSS input set for different flows * @vsi: VSI to be configured @@ -1449,11 +1665,12 @@ static void ice_vsi_set_vf_rss_flow_fld(struct ice_vsi *vsi) */ static void ice_vsi_set_rss_flow_fld(struct ice_vsi *vsi) { - u16 vsi_handle = vsi->idx, vsi_num = vsi->vsi_num; + u16 vsi_num = vsi->vsi_num; struct ice_pf *pf = vsi->back; struct ice_hw *hw = &pf->hw; - enum ice_status status; struct device *dev; + int status; + u32 i; dev = ice_pf_to_dev(pf); if (ice_is_safe_mode(pf)) { @@ -1461,61 +1678,15 @@ static void ice_vsi_set_rss_flow_fld(struct ice_vsi *vsi) vsi_num); return; } - /* configure RSS for IPv4 with input set IP src/dst */ - status = ice_add_rss_cfg(hw, vsi_handle, ICE_FLOW_HASH_IPV4, - ICE_FLOW_SEG_HDR_IPV4); - if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for ipv4 flow, vsi = %d, error = %s\n", - vsi_num, ice_stat_str(status)); - - /* configure RSS for IPv6 with input set IPv6 src/dst */ - status = ice_add_rss_cfg(hw, vsi_handle, ICE_FLOW_HASH_IPV6, - ICE_FLOW_SEG_HDR_IPV6); - if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for ipv6 flow, vsi = %d, error = %s\n", - vsi_num, ice_stat_str(status)); - - /* configure RSS for tcp4 with input set IP src/dst, TCP src/dst */ - status = ice_add_rss_cfg(hw, vsi_handle, ICE_HASH_TCP_IPV4, - ICE_FLOW_SEG_HDR_TCP | ICE_FLOW_SEG_HDR_IPV4); - if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for tcp4 flow, vsi = %d, error = %s\n", - vsi_num, ice_stat_str(status)); - - /* configure RSS for udp4 with input set IP src/dst, UDP src/dst */ - status = ice_add_rss_cfg(hw, vsi_handle, ICE_HASH_UDP_IPV4, - ICE_FLOW_SEG_HDR_UDP | ICE_FLOW_SEG_HDR_IPV4); - if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for udp4 flow, vsi = %d, error = %s\n", - vsi_num, ice_stat_str(status)); - - /* configure RSS for sctp4 with input set IP src/dst */ - status = ice_add_rss_cfg(hw, vsi_handle, ICE_FLOW_HASH_IPV4, - ICE_FLOW_SEG_HDR_SCTP | ICE_FLOW_SEG_HDR_IPV4); - if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for sctp4 flow, vsi = %d, error = %s\n", - vsi_num, ice_stat_str(status)); - - /* configure RSS for tcp6 with input set IPv6 src/dst, TCP src/dst */ - status = ice_add_rss_cfg(hw, vsi_handle, ICE_HASH_TCP_IPV6, - ICE_FLOW_SEG_HDR_TCP | ICE_FLOW_SEG_HDR_IPV6); - if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for tcp6 flow, vsi = %d, error = %s\n", - vsi_num, ice_stat_str(status)); + for (i = 0; i < ARRAY_SIZE(default_rss_cfgs); i++) { + const struct ice_rss_hash_cfg *cfg = &default_rss_cfgs[i]; - /* configure RSS for udp6 with input set IPv6 src/dst, UDP src/dst */ - status = ice_add_rss_cfg(hw, vsi_handle, ICE_HASH_UDP_IPV6, - ICE_FLOW_SEG_HDR_UDP | ICE_FLOW_SEG_HDR_IPV6); - if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for udp6 flow, vsi = %d, error = %s\n", - vsi_num, ice_stat_str(status)); - - /* configure RSS for sctp6 with input set IPv6 src/dst */ - status = ice_add_rss_cfg(hw, vsi_handle, ICE_FLOW_HASH_IPV6, - ICE_FLOW_SEG_HDR_SCTP | ICE_FLOW_SEG_HDR_IPV6); - if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for sctp6 flow, vsi = %d, error = %s\n", - vsi_num, ice_stat_str(status)); + status = ice_add_rss_cfg(hw, vsi, cfg); + if (status) + dev_dbg(dev, "ice_add_rss_cfg failed, addl_hdrs = %x, hash_flds = %llx, hdr_type = %d, symm = %d\n", + cfg->addl_hdrs, cfg->hash_flds, + cfg->hdr_type, cfg->symm); + } } /** @@ -1542,6 +1713,12 @@ bool ice_pf_state_is_nominal(struct ice_pf *pf) return true; } +#define ICE_FW_MODE_REC_M BIT(1) +bool ice_is_recovery_mode(struct ice_hw *hw) +{ + return rd32(hw, GL_MNG_FWSM) & ICE_FW_MODE_REC_M; +} + /** * ice_update_eth_stats - Update VSI-specific ethernet statistics counters * @vsi: the VSI to be updated @@ -1550,11 +1727,15 @@ void ice_update_eth_stats(struct ice_vsi *vsi) { struct ice_eth_stats *prev_es, *cur_es; struct ice_hw *hw = &vsi->back->hw; + struct ice_pf *pf = vsi->back; u16 vsi_num = vsi->vsi_num; /* HW absolute index of a VSI */ prev_es = &vsi->eth_stats_prev; cur_es = &vsi->eth_stats; + if (ice_is_reset_in_progress(pf->state)) + vsi->stat_offsets_loaded = false; + ice_stat_update40(hw, GLV_GORCL(vsi_num), vsi->stat_offsets_loaded, &prev_es->rx_bytes, &cur_es->rx_bytes); @@ -1589,88 +1770,6 @@ void ice_update_eth_stats(struct ice_vsi *vsi) } /** - * ice_vsi_add_vlan - Add VSI membership for given VLAN - * @vsi: the VSI being configured - * @vid: VLAN ID to be added - * @action: filter action to be performed on match - */ -int -ice_vsi_add_vlan(struct ice_vsi *vsi, u16 vid, enum ice_sw_fwd_act_type action) -{ - struct ice_pf *pf = vsi->back; - struct device *dev; - int err = 0; - - dev = ice_pf_to_dev(pf); - - if (!ice_fltr_add_vlan(vsi, vid, action)) { - vsi->num_vlan++; - } else { - err = -ENODEV; - dev_err(dev, "Failure Adding VLAN %d on VSI %i\n", vid, - vsi->vsi_num); - } - - return err; -} - -/** - * ice_vsi_kill_vlan - Remove VSI membership for a given VLAN - * @vsi: the VSI being configured - * @vid: VLAN ID to be removed - * - * Returns 0 on success and negative on failure - */ -int ice_vsi_kill_vlan(struct ice_vsi *vsi, u16 vid) -{ - struct ice_pf *pf = vsi->back; - enum ice_status status; - struct device *dev; - int err = 0; - - dev = ice_pf_to_dev(pf); - - status = ice_fltr_remove_vlan(vsi, vid, ICE_FWD_TO_VSI); - if (!status) { - vsi->num_vlan--; - } else if (status == ICE_ERR_DOES_NOT_EXIST) { - dev_dbg(dev, "Failed to remove VLAN %d on VSI %i, it does not exist, status: %s\n", - vid, vsi->vsi_num, ice_stat_str(status)); - } else { - dev_err(dev, "Error removing VLAN %d on vsi %i error: %s\n", - vid, vsi->vsi_num, ice_stat_str(status)); - err = -EIO; - } - - return err; -} - -/** - * ice_vsi_cfg_frame_size - setup max frame size and Rx buffer length - * @vsi: VSI - */ -void ice_vsi_cfg_frame_size(struct ice_vsi *vsi) -{ - if (!vsi->netdev || test_bit(ICE_FLAG_LEGACY_RX, vsi->back->flags)) { - vsi->max_frame = ICE_AQ_SET_MAC_FRAME_SIZE_MAX; - vsi->rx_buf_len = ICE_RXBUF_2048; -#if (PAGE_SIZE < 8192) - } else if (!ICE_2K_TOO_SMALL_WITH_PADDING && - (vsi->netdev->mtu <= ETH_DATA_LEN)) { - vsi->max_frame = ICE_RXBUF_1536 - NET_IP_ALIGN; - vsi->rx_buf_len = ICE_RXBUF_1536 - NET_IP_ALIGN; -#endif - } else { - vsi->max_frame = ICE_AQ_SET_MAC_FRAME_SIZE_MAX; -#if (PAGE_SIZE < 8192) - vsi->rx_buf_len = ICE_RXBUF_3072; -#else - vsi->rx_buf_len = ICE_RXBUF_2048; -#endif - } -} - -/** * ice_write_qrxflxp_cntxt - write/configure QRXFLXP_CNTXT register * @hw: HW pointer * @pf_q: index of the Rx queue in the PF's queue space @@ -1678,9 +1777,8 @@ void ice_vsi_cfg_frame_size(struct ice_vsi *vsi) * @prio: priority for the RXDID for this queue * @ena_ts: true to enable timestamp and false to disable timestamp */ -void -ice_write_qrxflxp_cntxt(struct ice_hw *hw, u16 pf_q, u32 rxdid, u32 prio, - bool ena_ts) +void ice_write_qrxflxp_cntxt(struct ice_hw *hw, u16 pf_q, u32 rxdid, u32 prio, + bool ena_ts) { int regval = rd32(hw, QRXFLXP_CNTXT(pf_q)); @@ -1689,11 +1787,8 @@ ice_write_qrxflxp_cntxt(struct ice_hw *hw, u16 pf_q, u32 rxdid, u32 prio, QRXFLXP_CNTXT_RXDID_PRIO_M | QRXFLXP_CNTXT_TS_M); - regval |= (rxdid << QRXFLXP_CNTXT_RXDID_IDX_S) & - QRXFLXP_CNTXT_RXDID_IDX_M; - - regval |= (prio << QRXFLXP_CNTXT_RXDID_PRIO_S) & - QRXFLXP_CNTXT_RXDID_PRIO_M; + regval |= FIELD_PREP(QRXFLXP_CNTXT_RXDID_IDX_M, rxdid); + regval |= FIELD_PREP(QRXFLXP_CNTXT_RXDID_PRIO_M, prio); if (ena_ts) /* Enable TimeSync on this queue */ @@ -1702,127 +1797,6 @@ ice_write_qrxflxp_cntxt(struct ice_hw *hw, u16 pf_q, u32 rxdid, u32 prio, wr32(hw, QRXFLXP_CNTXT(pf_q), regval); } -int ice_vsi_cfg_single_rxq(struct ice_vsi *vsi, u16 q_idx) -{ - if (q_idx >= vsi->num_rxq) - return -EINVAL; - - return ice_vsi_cfg_rxq(vsi->rx_rings[q_idx]); -} - -int ice_vsi_cfg_single_txq(struct ice_vsi *vsi, struct ice_ring **tx_rings, u16 q_idx) -{ - struct ice_aqc_add_tx_qgrp *qg_buf; - int err; - - if (q_idx >= vsi->alloc_txq || !tx_rings || !tx_rings[q_idx]) - return -EINVAL; - - qg_buf = kzalloc(struct_size(qg_buf, txqs, 1), GFP_KERNEL); - if (!qg_buf) - return -ENOMEM; - - qg_buf->num_txqs = 1; - - err = ice_vsi_cfg_txq(vsi, tx_rings[q_idx], qg_buf); - kfree(qg_buf); - return err; -} - -/** - * ice_vsi_cfg_rxqs - Configure the VSI for Rx - * @vsi: the VSI being configured - * - * Return 0 on success and a negative value on error - * Configure the Rx VSI for operation. - */ -int ice_vsi_cfg_rxqs(struct ice_vsi *vsi) -{ - u16 i; - - if (vsi->type == ICE_VSI_VF) - goto setup_rings; - - ice_vsi_cfg_frame_size(vsi); -setup_rings: - /* set up individual rings */ - ice_for_each_rxq(vsi, i) { - int err = ice_vsi_cfg_rxq(vsi->rx_rings[i]); - - if (err) - return err; - } - - return 0; -} - -/** - * ice_vsi_cfg_txqs - Configure the VSI for Tx - * @vsi: the VSI being configured - * @rings: Tx ring array to be configured - * @count: number of Tx ring array elements - * - * Return 0 on success and a negative value on error - * Configure the Tx VSI for operation. - */ -static int -ice_vsi_cfg_txqs(struct ice_vsi *vsi, struct ice_ring **rings, u16 count) -{ - struct ice_aqc_add_tx_qgrp *qg_buf; - u16 q_idx = 0; - int err = 0; - - qg_buf = kzalloc(struct_size(qg_buf, txqs, 1), GFP_KERNEL); - if (!qg_buf) - return -ENOMEM; - - qg_buf->num_txqs = 1; - - for (q_idx = 0; q_idx < count; q_idx++) { - err = ice_vsi_cfg_txq(vsi, rings[q_idx], qg_buf); - if (err) - goto err_cfg_txqs; - } - -err_cfg_txqs: - kfree(qg_buf); - return err; -} - -/** - * ice_vsi_cfg_lan_txqs - Configure the VSI for Tx - * @vsi: the VSI being configured - * - * Return 0 on success and a negative value on error - * Configure the Tx VSI for operation. - */ -int ice_vsi_cfg_lan_txqs(struct ice_vsi *vsi) -{ - return ice_vsi_cfg_txqs(vsi, vsi->tx_rings, vsi->num_txq); -} - -/** - * ice_vsi_cfg_xdp_txqs - Configure Tx queues dedicated for XDP in given VSI - * @vsi: the VSI being configured - * - * Return 0 on success and a negative value on error - * Configure the Tx queues dedicated for XDP in given VSI for operation. - */ -int ice_vsi_cfg_xdp_txqs(struct ice_vsi *vsi) -{ - int ret; - int i; - - ret = ice_vsi_cfg_txqs(vsi, vsi->xdp_rings, vsi->num_xdp_txq); - if (ret) - return ret; - - for (i = 0; i < vsi->num_xdp_txq; i++) - vsi->xdp_rings[i]->xsk_pool = ice_xsk_pool(vsi->xdp_rings[i]); - - return ret; -} - /** * ice_intrl_usec_to_reg - convert interrupt rate limit to register value * @intrl: interrupt rate limit in usecs @@ -1853,6 +1827,24 @@ void ice_write_intrl(struct ice_q_vector *q_vector, u8 intrl) ice_intrl_usec_to_reg(intrl, ICE_INTRL_GRAN_ABOVE_25)); } +static struct ice_q_vector *ice_pull_qvec_from_rc(struct ice_ring_container *rc) +{ + switch (rc->type) { + case ICE_RX_CONTAINER: + if (rc->rx_ring) + return rc->rx_ring->q_vector; + break; + case ICE_TX_CONTAINER: + if (rc->tx_ring) + return rc->tx_ring->q_vector; + break; + default: + break; + } + + return NULL; +} + /** * __ice_write_itr - write throttle rate to register * @q_vector: pointer to interrupt data structure @@ -1877,15 +1869,39 @@ void ice_write_itr(struct ice_ring_container *rc, u16 itr) { struct ice_q_vector *q_vector; - if (!rc->ring) + q_vector = ice_pull_qvec_from_rc(rc); + if (!q_vector) return; - q_vector = rc->ring->q_vector; - __ice_write_itr(q_vector, rc, itr); } /** + * ice_set_q_vector_intrl - set up interrupt rate limiting + * @q_vector: the vector to be configured + * + * Interrupt rate limiting is local to the vector, not per-queue so we must + * detect if either ring container has dynamic moderation enabled to decide + * what to set the interrupt rate limit to via INTRL settings. In the case that + * dynamic moderation is disabled on both, write the value with the cached + * setting to make sure INTRL register matches the user visible value. + */ +void ice_set_q_vector_intrl(struct ice_q_vector *q_vector) +{ + if (ITR_IS_DYNAMIC(&q_vector->tx) || ITR_IS_DYNAMIC(&q_vector->rx)) { + /* in the case of dynamic enabled, cap each vector to no more + * than (4 us) 250,000 ints/sec, which allows low latency + * but still less than 500,000 interrupts per second, which + * reduces CPU a bit in the case of the lowest latency + * setting. The 4 here is a value in microseconds. + */ + ice_write_intrl(q_vector, 4); + } else { + ice_write_intrl(q_vector, q_vector->intrl); + } +} + +/** * ice_vsi_cfg_msix - MSIX mode Interrupt Config in the HW * @vsi: the VSI being configured * @@ -1899,7 +1915,7 @@ void ice_vsi_cfg_msix(struct ice_vsi *vsi) u16 txq = 0, rxq = 0; int i, q; - for (i = 0; i < vsi->num_q_vectors; i++) { + ice_for_each_q_vector(vsi, i) { struct ice_q_vector *q_vector = vsi->q_vectors[i]; u16 reg_idx = q_vector->reg_idx; @@ -1931,101 +1947,6 @@ void ice_vsi_cfg_msix(struct ice_vsi *vsi) } /** - * ice_vsi_manage_vlan_insertion - Manage VLAN insertion for the VSI for Tx - * @vsi: the VSI being changed - */ -int ice_vsi_manage_vlan_insertion(struct ice_vsi *vsi) -{ - struct ice_hw *hw = &vsi->back->hw; - struct ice_vsi_ctx *ctxt; - enum ice_status status; - int ret = 0; - - ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL); - if (!ctxt) - return -ENOMEM; - - /* Here we are configuring the VSI to let the driver add VLAN tags by - * setting vlan_flags to ICE_AQ_VSI_VLAN_MODE_ALL. The actual VLAN tag - * insertion happens in the Tx hot path, in ice_tx_map. - */ - ctxt->info.vlan_flags = ICE_AQ_VSI_VLAN_MODE_ALL; - - /* Preserve existing VLAN strip setting */ - ctxt->info.vlan_flags |= (vsi->info.vlan_flags & - ICE_AQ_VSI_VLAN_EMOD_M); - - ctxt->info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_VLAN_VALID); - - status = ice_update_vsi(hw, vsi->idx, ctxt, NULL); - if (status) { - dev_err(ice_pf_to_dev(vsi->back), "update VSI for VLAN insert failed, err %s aq_err %s\n", - ice_stat_str(status), - ice_aq_str(hw->adminq.sq_last_status)); - ret = -EIO; - goto out; - } - - vsi->info.vlan_flags = ctxt->info.vlan_flags; -out: - kfree(ctxt); - return ret; -} - -/** - * ice_vsi_manage_vlan_stripping - Manage VLAN stripping for the VSI for Rx - * @vsi: the VSI being changed - * @ena: boolean value indicating if this is a enable or disable request - */ -int ice_vsi_manage_vlan_stripping(struct ice_vsi *vsi, bool ena) -{ - struct ice_hw *hw = &vsi->back->hw; - struct ice_vsi_ctx *ctxt; - enum ice_status status; - int ret = 0; - - /* do not allow modifying VLAN stripping when a port VLAN is configured - * on this VSI - */ - if (vsi->info.pvid) - return 0; - - ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL); - if (!ctxt) - return -ENOMEM; - - /* Here we are configuring what the VSI should do with the VLAN tag in - * the Rx packet. We can either leave the tag in the packet or put it in - * the Rx descriptor. - */ - if (ena) - /* Strip VLAN tag from Rx packet and put it in the desc */ - ctxt->info.vlan_flags = ICE_AQ_VSI_VLAN_EMOD_STR_BOTH; - else - /* Disable stripping. Leave tag in packet */ - ctxt->info.vlan_flags = ICE_AQ_VSI_VLAN_EMOD_NOTHING; - - /* Allow all packets untagged/tagged */ - ctxt->info.vlan_flags |= ICE_AQ_VSI_VLAN_MODE_ALL; - - ctxt->info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_VLAN_VALID); - - status = ice_update_vsi(hw, vsi->idx, ctxt, NULL); - if (status) { - dev_err(ice_pf_to_dev(vsi->back), "update VSI for VLAN strip failed, ena = %d err %s aq_err %s\n", - ena, ice_stat_str(status), - ice_aq_str(hw->adminq.sq_last_status)); - ret = -EIO; - goto out; - } - - vsi->info.vlan_flags = ctxt->info.vlan_flags; -out: - kfree(ctxt); - return ret; -} - -/** * ice_vsi_start_all_rx_rings - start/enable all of a VSI's Rx rings * @vsi: the VSI whose rings are to be enabled * @@ -2057,7 +1978,7 @@ int ice_vsi_stop_all_rx_rings(struct ice_vsi *vsi) */ static int ice_vsi_stop_tx_rings(struct ice_vsi *vsi, enum ice_disq_rst_src rst_src, - u16 rel_vmvf_num, struct ice_ring **rings, u16 count) + u16 rel_vmvf_num, struct ice_tx_ring **rings, u16 count) { u16 q_idx; @@ -2105,143 +2026,58 @@ int ice_vsi_stop_xdp_tx_rings(struct ice_vsi *vsi) } /** - * ice_vsi_is_vlan_pruning_ena - check if VLAN pruning is enabled or not - * @vsi: VSI to check whether or not VLAN pruning is enabled. - * - * returns true if Rx VLAN pruning is enabled and false otherwise. - */ -bool ice_vsi_is_vlan_pruning_ena(struct ice_vsi *vsi) -{ - if (!vsi) - return false; - - return (vsi->info.sw_flags2 & ICE_AQ_VSI_SW_FLAG_RX_VLAN_PRUNE_ENA); -} - -/** - * ice_cfg_vlan_pruning - enable or disable VLAN pruning on the VSI - * @vsi: VSI to enable or disable VLAN pruning on - * @ena: set to true to enable VLAN pruning and false to disable it - * @vlan_promisc: enable valid security flags if not in VLAN promiscuous mode + * ice_vsi_is_rx_queue_active + * @vsi: the VSI being configured * - * returns 0 if VSI is updated, negative otherwise + * Return true if at least one queue is active. */ -int ice_cfg_vlan_pruning(struct ice_vsi *vsi, bool ena, bool vlan_promisc) +bool ice_vsi_is_rx_queue_active(struct ice_vsi *vsi) { - struct ice_vsi_ctx *ctxt; - struct ice_pf *pf; - int status; - - if (!vsi) - return -EINVAL; - - /* Don't enable VLAN pruning if the netdev is currently in promiscuous - * mode. VLAN pruning will be enabled when the interface exits - * promiscuous mode if any VLAN filters are active. - */ - if (vsi->netdev && vsi->netdev->flags & IFF_PROMISC && ena) - return 0; - - pf = vsi->back; - ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL); - if (!ctxt) - return -ENOMEM; - - ctxt->info = vsi->info; - - if (ena) - ctxt->info.sw_flags2 |= ICE_AQ_VSI_SW_FLAG_RX_VLAN_PRUNE_ENA; - else - ctxt->info.sw_flags2 &= ~ICE_AQ_VSI_SW_FLAG_RX_VLAN_PRUNE_ENA; + struct ice_pf *pf = vsi->back; + struct ice_hw *hw = &pf->hw; + int i; - if (!vlan_promisc) - ctxt->info.valid_sections = - cpu_to_le16(ICE_AQ_VSI_PROP_SW_VALID); + ice_for_each_rxq(vsi, i) { + u32 rx_reg; + int pf_q; - status = ice_update_vsi(&pf->hw, vsi->idx, ctxt, NULL); - if (status) { - netdev_err(vsi->netdev, "%sabling VLAN pruning on VSI handle: %d, VSI HW ID: %d failed, err = %s, aq_err = %s\n", - ena ? "En" : "Dis", vsi->idx, vsi->vsi_num, - ice_stat_str(status), - ice_aq_str(pf->hw.adminq.sq_last_status)); - goto err_out; + pf_q = vsi->rxq_map[i]; + rx_reg = rd32(hw, QRX_CTRL(pf_q)); + if (rx_reg & QRX_CTRL_QENA_STAT_M) + return true; } - vsi->info.sw_flags2 = ctxt->info.sw_flags2; - - kfree(ctxt); - return 0; - -err_out: - kfree(ctxt); - return -EIO; + return false; } static void ice_vsi_set_tc_cfg(struct ice_vsi *vsi) { - struct ice_dcbx_cfg *cfg = &vsi->port_info->qos_cfg.local_dcbx_cfg; - - vsi->tc_cfg.ena_tc = ice_dcb_get_ena_tc(cfg); - vsi->tc_cfg.numtc = ice_dcb_get_num_tc(cfg); -} - -/** - * ice_vsi_set_q_vectors_reg_idx - set the HW register index for all q_vectors - * @vsi: VSI to set the q_vectors register index on - */ -static int -ice_vsi_set_q_vectors_reg_idx(struct ice_vsi *vsi) -{ - u16 i; - - if (!vsi || !vsi->q_vectors) - return -EINVAL; - - ice_for_each_q_vector(vsi, i) { - struct ice_q_vector *q_vector = vsi->q_vectors[i]; - - if (!q_vector) { - dev_err(ice_pf_to_dev(vsi->back), "Failed to set reg_idx on q_vector %d VSI %d\n", - i, vsi->vsi_num); - goto clear_reg_idx; - } - - if (vsi->type == ICE_VSI_VF) { - struct ice_vf *vf = &vsi->back->vf[vsi->vf_id]; - - q_vector->reg_idx = ice_calc_vf_reg_idx(vf, q_vector); - } else { - q_vector->reg_idx = - q_vector->v_idx + vsi->base_vector; - } - } - - return 0; - -clear_reg_idx: - ice_for_each_q_vector(vsi, i) { - struct ice_q_vector *q_vector = vsi->q_vectors[i]; - - if (q_vector) - q_vector->reg_idx = 0; + if (!test_bit(ICE_FLAG_DCB_ENA, vsi->back->flags)) { + vsi->tc_cfg.ena_tc = ICE_DFLT_TRAFFIC_CLASS; + vsi->tc_cfg.numtc = 1; + return; } - return -EINVAL; + /* set VSI TC information based on DCB config */ + ice_vsi_set_dcb_tc_cfg(vsi); } /** - * ice_cfg_sw_lldp - Config switch rules for LLDP packet handling + * ice_vsi_cfg_sw_lldp - Config switch rules for LLDP packet handling * @vsi: the VSI being configured * @tx: bool to determine Tx or Rx rule * @create: bool to determine create or remove Rule + * + * Adding an ethtype Tx rule to the uplink VSI results in it being applied + * to the whole port, so LLDP transmission for VFs will be blocked too. */ -void ice_cfg_sw_lldp(struct ice_vsi *vsi, bool tx, bool create) +void ice_vsi_cfg_sw_lldp(struct ice_vsi *vsi, bool tx, bool create) { - enum ice_status (*eth_fltr)(struct ice_vsi *v, u16 type, u16 flag, - enum ice_sw_fwd_act_type act); + int (*eth_fltr)(struct ice_vsi *v, u16 type, u16 flag, + enum ice_sw_fwd_act_type act); struct ice_pf *pf = vsi->back; - enum ice_status status; struct device *dev; + int status; dev = ice_pf_to_dev(pf); eth_fltr = create ? ice_fltr_add_eth : ice_fltr_remove_eth; @@ -2250,19 +2086,59 @@ void ice_cfg_sw_lldp(struct ice_vsi *vsi, bool tx, bool create) status = eth_fltr(vsi, ETH_P_LLDP, ICE_FLTR_TX, ICE_DROP_PACKET); } else { - if (ice_fw_supports_lldp_fltr_ctrl(&pf->hw)) { - status = ice_lldp_fltr_add_remove(&pf->hw, vsi->vsi_num, - create); - } else { + if (!test_bit(ICE_FLAG_LLDP_AQ_FLTR, pf->flags)) { status = eth_fltr(vsi, ETH_P_LLDP, ICE_FLTR_RX, ICE_FWD_TO_VSI); + if (!status || !create) + goto report; + + dev_info(dev, + "Failed to add generic LLDP Rx filter on VSI %i error: %d, falling back to specialized AQ control\n", + vsi->vsi_num, status); } + + status = ice_lldp_fltr_add_remove(&pf->hw, vsi, create); + if (!status) + set_bit(ICE_FLAG_LLDP_AQ_FLTR, pf->flags); + } +report: if (status) - dev_dbg(dev, "Fail %s %s LLDP rule on VSI %i error: %s\n", - create ? "adding" : "removing", tx ? "TX" : "RX", - vsi->vsi_num, ice_stat_str(status)); + dev_warn(dev, "Failed to %s %s LLDP rule on VSI %i error: %d\n", + create ? "add" : "remove", tx ? "Tx" : "Rx", + vsi->vsi_num, status); +} + +/** + * ice_cfg_sw_rx_lldp - Enable/disable software handling of LLDP + * @pf: the PF being configured + * @enable: enable or disable + * + * Configure switch rules to enable/disable LLDP handling by software + * across PF. + */ +void ice_cfg_sw_rx_lldp(struct ice_pf *pf, bool enable) +{ + struct ice_vsi *vsi; + struct ice_vf *vf; + unsigned int bkt; + + vsi = ice_get_main_vsi(pf); + ice_vsi_cfg_sw_lldp(vsi, false, enable); + + if (!test_bit(ICE_FLAG_SRIOV_ENA, pf->flags)) + return; + + ice_for_each_vf(pf, bkt, vf) { + vsi = ice_get_vf_vsi(vf); + + if (WARN_ON(!vsi)) + continue; + + if (ice_vf_is_lldp_ena(vf)) + ice_vsi_cfg_sw_lldp(vsi, false, enable); + } } /** @@ -2282,7 +2158,7 @@ static void ice_set_agg_vsi(struct ice_vsi *vsi) struct ice_port_info *port_info; struct ice_pf *pf = vsi->back; u32 agg_node_id_start = 0; - enum ice_status status; + int status; /* create (as needed) scheduler aggregator node and move VSI into * corresponding aggregator node @@ -2295,8 +2171,10 @@ static void ice_set_agg_vsi(struct ice_vsi *vsi) switch (vsi->type) { case ICE_VSI_CTRL: + case ICE_VSI_CHNL: case ICE_VSI_LB: case ICE_VSI_PF: + case ICE_VSI_SF: max_agg_nodes = ICE_MAX_PF_AGG_NODES; agg_node_id_start = ICE_PF_AGG_NODE_ID_START; agg_node_iter = &pf->pf_agg_node[0]; @@ -2359,7 +2237,7 @@ static void ice_set_agg_vsi(struct ice_vsi *vsi) agg_id); return; } - /* aggregator node is created, store the neeeded info */ + /* aggregator node is created, store the needed info */ agg_node->valid = true; agg_node->agg_id = agg_id; } @@ -2385,54 +2263,70 @@ static void ice_set_agg_vsi(struct ice_vsi *vsi) vsi->agg_node->num_vsis); } -/** - * ice_vsi_setup - Set up a VSI by a given type - * @pf: board private structure - * @pi: pointer to the port_info instance - * @vsi_type: VSI type - * @vf_id: defines VF ID to which this VSI connects. This field is meant to be - * used only for ICE_VSI_VF VSI type. For other VSI types, should - * fill-in ICE_INVAL_VFID as input. - * - * This allocates the sw VSI structure and its queue resources. - * - * Returns pointer to the successfully allocated and configured VSI sw struct on - * success, NULL on failure. - */ -struct ice_vsi * -ice_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi, - enum ice_vsi_type vsi_type, u16 vf_id) +static int ice_vsi_cfg_tc_lan(struct ice_pf *pf, struct ice_vsi *vsi) { u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 }; struct device *dev = ice_pf_to_dev(pf); - enum ice_status status; - struct ice_vsi *vsi; int ret, i; - if (vsi_type == ICE_VSI_VF || vsi_type == ICE_VSI_CTRL) - vsi = ice_vsi_alloc(pf, vsi_type, vf_id); - else - vsi = ice_vsi_alloc(pf, vsi_type, ICE_INVAL_VFID); + /* configure VSI nodes based on number of queues and TC's */ + ice_for_each_traffic_class(i) { + if (!(vsi->tc_cfg.ena_tc & BIT(i))) + continue; - if (!vsi) { - dev_err(dev, "could not allocate VSI\n"); - return NULL; + if (vsi->type == ICE_VSI_CHNL) { + if (!vsi->alloc_txq && vsi->num_txq) + max_txqs[i] = vsi->num_txq; + else + max_txqs[i] = pf->num_lan_tx; + } else { + max_txqs[i] = vsi->alloc_txq; + } + + if (vsi->type == ICE_VSI_PF) + max_txqs[i] += vsi->num_xdp_txq; } - vsi->port_info = pi; + dev_dbg(dev, "vsi->tc_cfg.ena_tc = %d\n", vsi->tc_cfg.ena_tc); + ret = ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc, + max_txqs); + if (ret) { + dev_err(dev, "VSI %d failed lan queue config, error %d\n", + vsi->vsi_num, ret); + return ret; + } + + return 0; +} + +/** + * ice_vsi_cfg_def - configure default VSI based on the type + * @vsi: pointer to VSI + */ +static int ice_vsi_cfg_def(struct ice_vsi *vsi) +{ + struct device *dev = ice_pf_to_dev(vsi->back); + struct ice_pf *pf = vsi->back; + int ret; + vsi->vsw = pf->first_sw; - if (vsi->type == ICE_VSI_PF) - vsi->ethtype = ETH_P_PAUSE; - if (vsi->type == ICE_VSI_VF || vsi->type == ICE_VSI_CTRL) - vsi->vf_id = vf_id; + ret = ice_vsi_alloc_def(vsi, vsi->ch); + if (ret) + return ret; + + /* allocate memory for Tx/Rx ring stat pointers */ + ret = ice_vsi_alloc_stat_arrays(vsi); + if (ret) + goto unroll_vsi_alloc; ice_alloc_fd_res(vsi); - if (ice_vsi_get_qs(vsi)) { + ret = ice_vsi_get_qs(vsi); + if (ret) { dev_err(dev, "Failed to allocate queues. vsi->idx = %d\n", vsi->idx); - goto unroll_vsi_alloc; + goto unroll_vsi_alloc_stat; } /* set RSS capabilities */ @@ -2442,42 +2336,42 @@ ice_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi, ice_vsi_set_tc_cfg(vsi); /* create the VSI */ - ret = ice_vsi_init(vsi, true); + ret = ice_vsi_init(vsi, vsi->flags); if (ret) goto unroll_get_qs; + ice_vsi_init_vlan_ops(vsi); + switch (vsi->type) { case ICE_VSI_CTRL: + case ICE_VSI_SF: case ICE_VSI_PF: ret = ice_vsi_alloc_q_vectors(vsi); if (ret) goto unroll_vsi_init; - ret = ice_vsi_setup_vector_base(vsi); - if (ret) - goto unroll_alloc_q_vector; - - ret = ice_vsi_set_q_vectors_reg_idx(vsi); + ret = ice_vsi_alloc_rings(vsi); if (ret) goto unroll_vector_base; - ret = ice_vsi_alloc_rings(vsi); + ret = ice_vsi_alloc_ring_stats(vsi); if (ret) goto unroll_vector_base; - /* Always add VLAN ID 0 switch rule by default. This is needed - * in order to allow all untagged and 0 tagged priority traffic - * if Rx VLAN pruning is enabled. Also there are cases where we - * don't get the call to add VLAN 0 via ice_vlan_rx_add_vid() - * so this handles those cases (i.e. adding the PF to a bridge - * without the 8021q module loaded). - */ - ret = ice_vsi_add_vlan(vsi, 0, ICE_FWD_TO_VSI); - if (ret) - goto unroll_clear_rings; + if (ice_is_xdp_ena_vsi(vsi)) { + ret = ice_vsi_determine_xdp_res(vsi); + if (ret) + goto unroll_vector_base; + ret = ice_prepare_xdp_rings(vsi, vsi->xdp_prog, + ICE_XDP_CFG_PART); + if (ret) + goto unroll_vector_base; + } ice_vsi_map_rings_to_vectors(vsi); + vsi->stat_offsets_loaded = false; + /* ICE_VSI_CTRL does not need RSS so skip RSS processing */ if (vsi->type != ICE_VSI_CTRL) /* Do not exit if configuring RSS had an issue, at @@ -2490,6 +2384,12 @@ ice_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi, } ice_init_arfs(vsi); break; + case ICE_VSI_CHNL: + if (test_bit(ICE_FLAG_RSS_ENA, pf->flags)) { + ice_vsi_cfg_rss_lut_key(vsi); + ice_vsi_set_rss_flow_fld(vsi); + } + break; case ICE_VSI_VF: /* VF driver will take care of creating netdev for this type and * map queues to vectors through Virtchnl, PF driver only @@ -2504,10 +2404,12 @@ ice_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi, if (ret) goto unroll_alloc_q_vector; - ret = ice_vsi_set_q_vectors_reg_idx(vsi); + ret = ice_vsi_alloc_ring_stats(vsi); if (ret) goto unroll_vector_base; + vsi->stat_offsets_loaded = false; + /* Do not exit if configuring RSS had an issue, at least * receive traffic on first queue. Hence no need to capture * return value @@ -2521,24 +2423,140 @@ ice_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi, ret = ice_vsi_alloc_rings(vsi); if (ret) goto unroll_vsi_init; + + ret = ice_vsi_alloc_ring_stats(vsi); + if (ret) + goto unroll_vector_base; + break; default: /* clean up the resources and exit */ + ret = -EINVAL; goto unroll_vsi_init; } - /* configure VSI nodes based on number of queues and TC's */ - for (i = 0; i < vsi->tc_cfg.numtc; i++) - max_txqs[i] = vsi->alloc_txq; + return 0; - status = ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc, - max_txqs); - if (status) { - dev_err(dev, "VSI %d failed lan queue config, error %s\n", - vsi->vsi_num, ice_stat_str(status)); - goto unroll_clear_rings; +unroll_vector_base: + /* reclaim SW interrupts back to the common pool */ +unroll_alloc_q_vector: + ice_vsi_free_q_vectors(vsi); +unroll_vsi_init: + ice_vsi_delete_from_hw(vsi); +unroll_get_qs: + ice_vsi_put_qs(vsi); +unroll_vsi_alloc_stat: + ice_vsi_free_stats(vsi); +unroll_vsi_alloc: + ice_vsi_free_arrays(vsi); + return ret; +} + +/** + * ice_vsi_cfg - configure a previously allocated VSI + * @vsi: pointer to VSI + */ +int ice_vsi_cfg(struct ice_vsi *vsi) +{ + struct ice_pf *pf = vsi->back; + int ret; + + if (WARN_ON(vsi->type == ICE_VSI_VF && !vsi->vf)) + return -EINVAL; + + ret = ice_vsi_cfg_def(vsi); + if (ret) + return ret; + + ret = ice_vsi_cfg_tc_lan(vsi->back, vsi); + if (ret) + ice_vsi_decfg(vsi); + + if (vsi->type == ICE_VSI_CTRL) { + if (vsi->vf) { + WARN_ON(vsi->vf->ctrl_vsi_idx != ICE_NO_VSI); + vsi->vf->ctrl_vsi_idx = vsi->idx; + } else { + WARN_ON(pf->ctrl_vsi_idx != ICE_NO_VSI); + pf->ctrl_vsi_idx = vsi->idx; + } + } + + return ret; +} + +/** + * ice_vsi_decfg - remove all VSI configuration + * @vsi: pointer to VSI + */ +void ice_vsi_decfg(struct ice_vsi *vsi) +{ + struct ice_pf *pf = vsi->back; + int err; + + ice_rm_vsi_lan_cfg(vsi->port_info, vsi->idx); + err = ice_rm_vsi_rdma_cfg(vsi->port_info, vsi->idx); + if (err) + dev_err(ice_pf_to_dev(pf), "Failed to remove RDMA scheduler config for VSI %u, err %d\n", + vsi->vsi_num, err); + + if (vsi->xdp_rings) + /* return value check can be skipped here, it always returns + * 0 if reset is in progress + */ + ice_destroy_xdp_rings(vsi, ICE_XDP_CFG_PART); + + ice_vsi_clear_rings(vsi); + ice_vsi_free_q_vectors(vsi); + ice_vsi_put_qs(vsi); + ice_vsi_free_arrays(vsi); + + /* SR-IOV determines needed MSIX resources all at once instead of per + * VSI since when VFs are spawned we know how many VFs there are and how + * many interrupts each VF needs. SR-IOV MSIX resources are also + * cleared in the same manner. + */ + + if (vsi->type == ICE_VSI_VF && + vsi->agg_node && vsi->agg_node->valid) + vsi->agg_node->num_vsis--; +} + +/** + * ice_vsi_setup - Set up a VSI by a given type + * @pf: board private structure + * @params: parameters to use when creating the VSI + * + * This allocates the sw VSI structure and its queue resources. + * + * Returns pointer to the successfully allocated and configured VSI sw struct on + * success, NULL on failure. + */ +struct ice_vsi * +ice_vsi_setup(struct ice_pf *pf, struct ice_vsi_cfg_params *params) +{ + struct device *dev = ice_pf_to_dev(pf); + struct ice_vsi *vsi; + int ret; + + /* ice_vsi_setup can only initialize a new VSI, and we must have + * a port_info structure for it. + */ + if (WARN_ON(!(params->flags & ICE_VSI_FLAG_INIT)) || + WARN_ON(!params->port_info)) + return NULL; + + vsi = ice_vsi_alloc(pf); + if (!vsi) { + dev_err(dev, "could not allocate VSI\n"); + return NULL; } + vsi->params = *params; + ret = ice_vsi_cfg(vsi); + if (ret) + goto err_vsi_cfg; + /* Add switch rule to drop all Tx Flow Control Frames, of look up * type ETHERTYPE from VSIs, and restrict malicious VF from sending * out PAUSE or PFC frames. If enabled, FW can still send FC frames. @@ -2548,33 +2566,19 @@ ice_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi, * be dropped so that VFs cannot send LLDP packets to reconfig DCB * settings in the HW. */ - if (!ice_is_safe_mode(pf)) - if (vsi->type == ICE_VSI_PF) { - ice_fltr_add_eth(vsi, ETH_P_PAUSE, ICE_FLTR_TX, - ICE_DROP_PACKET); - ice_cfg_sw_lldp(vsi, true, true); - } + if (!ice_is_safe_mode(pf) && vsi->type == ICE_VSI_PF) { + ice_fltr_add_eth(vsi, ETH_P_PAUSE, ICE_FLTR_TX, + ICE_DROP_PACKET); + ice_vsi_cfg_sw_lldp(vsi, true, true); + } if (!vsi->agg_node) ice_set_agg_vsi(vsi); + return vsi; -unroll_clear_rings: - ice_vsi_clear_rings(vsi); -unroll_vector_base: - /* reclaim SW interrupts back to the common pool */ - ice_free_res(pf->irq_tracker, vsi->base_vector, vsi->idx); - pf->num_avail_sw_msix += vsi->num_q_vectors; -unroll_alloc_q_vector: - ice_vsi_free_q_vectors(vsi); -unroll_vsi_init: - ice_vsi_delete(vsi); -unroll_get_qs: - ice_vsi_put_qs(vsi); -unroll_vsi_alloc: - if (vsi_type == ICE_VSI_VF) - ice_enable_lag(pf->lag); - ice_vsi_clear(vsi); +err_vsi_cfg: + ice_vsi_free(vsi); return NULL; } @@ -2591,14 +2595,14 @@ static void ice_vsi_release_msix(struct ice_vsi *vsi) u32 rxq = 0; int i, q; - for (i = 0; i < vsi->num_q_vectors; i++) { + ice_for_each_q_vector(vsi, i) { struct ice_q_vector *q_vector = vsi->q_vectors[i]; ice_write_intrl(q_vector, 0); for (q = 0; q < q_vector->num_ring_tx; q++) { ice_write_itr(&q_vector->tx, 0); wr32(hw, QINT_TQCTL(vsi->txq_map[txq]), 0); - if (ice_is_xdp_ena_vsi(vsi)) { + if (vsi->xdp_rings) { u32 xdp_txq = txq + vsi->num_xdp_txq; wr32(hw, QINT_TQCTL(vsi->txq_map[xdp_txq]), 0); @@ -2623,7 +2627,6 @@ static void ice_vsi_release_msix(struct ice_vsi *vsi) void ice_vsi_free_irq(struct ice_vsi *vsi) { struct ice_pf *pf = vsi->back; - int base = vsi->base_vector; int i; if (!vsi->q_vectors || !vsi->irqs_ready) @@ -2634,11 +2637,11 @@ void ice_vsi_free_irq(struct ice_vsi *vsi) return; vsi->irqs_ready = false; + ice_for_each_q_vector(vsi, i) { - u16 vector = i + base; int irq_num; - irq_num = pf->msix_entries[vector].vector; + irq_num = vsi->q_vectors[i]->irq.virq; /* free only the irqs that were actually requested */ if (!vsi->q_vectors[i] || @@ -2646,11 +2649,6 @@ void ice_vsi_free_irq(struct ice_vsi *vsi) vsi->q_vectors[i]->num_ring_rx)) continue; - /* clear the affinity notifier in the IRQ descriptor */ - irq_set_affinity_notifier(irq_num, NULL); - - /* clear the affinity_mask in the IRQ descriptor */ - irq_set_affinity_hint(irq_num, NULL); synchronize_irq(irq_num); devm_free_irq(ice_pf_to_dev(pf), irq_num, vsi->q_vectors[i]); } @@ -2697,6 +2695,7 @@ void ice_vsi_close(struct ice_vsi *vsi) if (!test_and_set_bit(ICE_VSI_DOWN, vsi->state)) ice_down(vsi); + ice_vsi_clear_napi_queues(vsi); ice_vsi_free_irq(vsi); ice_vsi_free_tx_rings(vsi); ice_vsi_free_rx_rings(vsi); @@ -2716,7 +2715,8 @@ int ice_ena_vsi(struct ice_vsi *vsi, bool locked) clear_bit(ICE_VSI_NEEDS_RESTART, vsi->state); - if (vsi->netdev && vsi->type == ICE_VSI_PF) { + if (vsi->netdev && (vsi->type == ICE_VSI_PF || + vsi->type == ICE_VSI_SF)) { if (netif_running(vsi->netdev)) { if (!locked) rtnl_lock(); @@ -2740,89 +2740,97 @@ int ice_ena_vsi(struct ice_vsi *vsi, bool locked) */ void ice_dis_vsi(struct ice_vsi *vsi, bool locked) { - if (test_bit(ICE_VSI_DOWN, vsi->state)) - return; + bool already_down = test_bit(ICE_VSI_DOWN, vsi->state); set_bit(ICE_VSI_NEEDS_RESTART, vsi->state); - if (vsi->type == ICE_VSI_PF && vsi->netdev) { + if (vsi->netdev && (vsi->type == ICE_VSI_PF || + vsi->type == ICE_VSI_SF)) { if (netif_running(vsi->netdev)) { if (!locked) rtnl_lock(); - - ice_vsi_close(vsi); + already_down = test_bit(ICE_VSI_DOWN, vsi->state); + if (!already_down) + ice_vsi_close(vsi); if (!locked) rtnl_unlock(); - } else { + } else if (!already_down) { ice_vsi_close(vsi); } - } else if (vsi->type == ICE_VSI_CTRL) { + } else if (vsi->type == ICE_VSI_CTRL && !already_down) { ice_vsi_close(vsi); } } /** - * ice_vsi_dis_irq - Mask off queue interrupt generation on the VSI - * @vsi: the VSI being un-configured + * ice_vsi_set_napi_queues - associate netdev queues with napi + * @vsi: VSI pointer + * + * Associate queue[s] with napi for all vectors. */ -void ice_vsi_dis_irq(struct ice_vsi *vsi) +void ice_vsi_set_napi_queues(struct ice_vsi *vsi) { - int base = vsi->base_vector; - struct ice_pf *pf = vsi->back; - struct ice_hw *hw = &pf->hw; - u32 val; - int i; - - /* disable interrupt causation from each queue */ - if (vsi->tx_rings) { - ice_for_each_txq(vsi, i) { - if (vsi->tx_rings[i]) { - u16 reg; + struct net_device *netdev = vsi->netdev; + int q_idx, v_idx; - reg = vsi->tx_rings[i]->reg_idx; - val = rd32(hw, QINT_TQCTL(reg)); - val &= ~QINT_TQCTL_CAUSE_ENA_M; - wr32(hw, QINT_TQCTL(reg), val); - } - } - } + if (!netdev) + return; - if (vsi->rx_rings) { - ice_for_each_rxq(vsi, i) { - if (vsi->rx_rings[i]) { - u16 reg; + ASSERT_RTNL(); + ice_for_each_rxq(vsi, q_idx) + netif_queue_set_napi(netdev, q_idx, NETDEV_QUEUE_TYPE_RX, + &vsi->rx_rings[q_idx]->q_vector->napi); - reg = vsi->rx_rings[i]->reg_idx; - val = rd32(hw, QINT_RQCTL(reg)); - val &= ~QINT_RQCTL_CAUSE_ENA_M; - wr32(hw, QINT_RQCTL(reg), val); - } - } - } + ice_for_each_txq(vsi, q_idx) + netif_queue_set_napi(netdev, q_idx, NETDEV_QUEUE_TYPE_TX, + &vsi->tx_rings[q_idx]->q_vector->napi); + /* Also set the interrupt number for the NAPI */ + ice_for_each_q_vector(vsi, v_idx) { + struct ice_q_vector *q_vector = vsi->q_vectors[v_idx]; - /* disable each interrupt */ - ice_for_each_q_vector(vsi, i) { - if (!vsi->q_vectors[i]) - continue; - wr32(hw, GLINT_DYN_CTL(vsi->q_vectors[i]->reg_idx), 0); + netif_napi_set_irq(&q_vector->napi, q_vector->irq.virq); } +} - ice_flush(hw); +/** + * ice_vsi_clear_napi_queues - dissociate netdev queues from napi + * @vsi: VSI pointer + * + * Clear the association between all VSI queues queue[s] and napi. + */ +void ice_vsi_clear_napi_queues(struct ice_vsi *vsi) +{ + struct net_device *netdev = vsi->netdev; + int q_idx, v_idx; - /* don't call synchronize_irq() for VF's from the host */ - if (vsi->type == ICE_VSI_VF) + if (!netdev) return; - ice_for_each_q_vector(vsi, i) - synchronize_irq(pf->msix_entries[i + base].vector); + ASSERT_RTNL(); + /* Clear the NAPI's interrupt number */ + ice_for_each_q_vector(vsi, v_idx) { + struct ice_q_vector *q_vector = vsi->q_vectors[v_idx]; + + netif_napi_set_irq(&q_vector->napi, -1); + } + + ice_for_each_txq(vsi, q_idx) + netif_queue_set_napi(netdev, q_idx, NETDEV_QUEUE_TYPE_TX, NULL); + + ice_for_each_rxq(vsi, q_idx) + netif_queue_set_napi(netdev, q_idx, NETDEV_QUEUE_TYPE_RX, NULL); } /** - * ice_napi_del - Remove NAPI handler for the VSI - * @vsi: VSI for which NAPI handler is to be removed + * ice_napi_add - register NAPI handler for the VSI + * @vsi: VSI for which NAPI handler is to be registered + * + * This function is only called in the driver's load path. Registering the NAPI + * handler is done in ice_vsi_alloc_q_vector() for all other cases (i.e. resume, + * reset/rebuild, etc.) */ -void ice_napi_del(struct ice_vsi *vsi) +void ice_napi_add(struct ice_vsi *vsi) { int v_idx; @@ -2830,7 +2838,10 @@ void ice_napi_del(struct ice_vsi *vsi) return; ice_for_each_q_vector(vsi, v_idx) - netif_napi_del(&vsi->q_vectors[v_idx]->napi); + netif_napi_add_config(vsi->netdev, + &vsi->q_vectors[v_idx]->napi, + ice_napi_poll, + v_idx); } /** @@ -2847,99 +2858,28 @@ int ice_vsi_release(struct ice_vsi *vsi) return -ENODEV; pf = vsi->back; - /* do not unregister while driver is in the reset recovery pending - * state. Since reset/rebuild happens through PF service task workqueue, - * it's not a good idea to unregister netdev that is associated to the - * PF that is running the work queue items currently. This is done to - * avoid check_flush_dependency() warning on this wq - */ - if (vsi->netdev && !ice_is_reset_in_progress(pf->state) && - (test_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state))) { - unregister_netdev(vsi->netdev); - clear_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state); - } - - ice_devlink_destroy_port(vsi); - if (test_bit(ICE_FLAG_RSS_ENA, pf->flags)) ice_rss_clean(vsi); - /* Disable VSI and free resources */ - if (vsi->type != ICE_VSI_LB) - ice_vsi_dis_irq(vsi); ice_vsi_close(vsi); - /* SR-IOV determines needed MSIX resources all at once instead of per - * VSI since when VFs are spawned we know how many VFs there are and how - * many interrupts each VF needs. SR-IOV MSIX resources are also - * cleared in the same manner. + /* The Rx rule will only exist to remove if the LLDP FW + * engine is currently stopped */ - if (vsi->type == ICE_VSI_CTRL && vsi->vf_id != ICE_INVAL_VFID) { - int i; - - ice_for_each_vf(pf, i) { - struct ice_vf *vf = &pf->vf[i]; - - if (i != vsi->vf_id && vf->ctrl_vsi_idx != ICE_NO_VSI) - break; - } - if (i == pf->num_alloc_vfs) { - /* No other VFs left that have control VSI, reclaim SW - * interrupts back to the common pool - */ - ice_free_res(pf->irq_tracker, vsi->base_vector, - ICE_RES_VF_CTRL_VEC_ID); - pf->num_avail_sw_msix += vsi->num_q_vectors; - } - } else if (vsi->type != ICE_VSI_VF) { - /* reclaim SW interrupts back to the common pool */ - ice_free_res(pf->irq_tracker, vsi->base_vector, vsi->idx); - pf->num_avail_sw_msix += vsi->num_q_vectors; - } - - if (!ice_is_safe_mode(pf)) { - if (vsi->type == ICE_VSI_PF) { - ice_fltr_remove_eth(vsi, ETH_P_PAUSE, ICE_FLTR_TX, - ICE_DROP_PACKET); - ice_cfg_sw_lldp(vsi, true, false); - /* The Rx rule will only exist to remove if the LLDP FW - * engine is currently stopped - */ - if (!test_bit(ICE_FLAG_FW_LLDP_AGENT, pf->flags)) - ice_cfg_sw_lldp(vsi, false, false); - } - } - - ice_fltr_remove_all(vsi); - ice_rm_vsi_lan_cfg(vsi->port_info, vsi->idx); - ice_vsi_delete(vsi); - ice_vsi_free_q_vectors(vsi); - - if (vsi->netdev) { - if (test_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state)) { - unregister_netdev(vsi->netdev); - clear_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state); - } - if (test_bit(ICE_VSI_NETDEV_ALLOCD, vsi->state)) { - free_netdev(vsi->netdev); - vsi->netdev = NULL; - clear_bit(ICE_VSI_NETDEV_ALLOCD, vsi->state); - } - } + if (!ice_is_safe_mode(pf) && + !test_bit(ICE_FLAG_FW_LLDP_AGENT, pf->flags) && + (vsi->type == ICE_VSI_PF || (vsi->type == ICE_VSI_VF && + ice_vf_is_lldp_ena(vsi->vf)))) + ice_vsi_cfg_sw_lldp(vsi, false, false); - if (vsi->type == ICE_VSI_VF && - vsi->agg_node && vsi->agg_node->valid) - vsi->agg_node->num_vsis--; - ice_vsi_clear_rings(vsi); - - ice_vsi_put_qs(vsi); + ice_vsi_decfg(vsi); /* retain SW VSI data structure since it is needed to unregister and * free VSI netdev when PF is not in reset recovery pending state,\ * for ex: during rmmod. */ if (!ice_is_reset_in_progress(pf->state)) - ice_vsi_clear(vsi); + ice_vsi_delete(vsi); return 0; } @@ -2960,8 +2900,8 @@ ice_vsi_rebuild_get_coalesce(struct ice_vsi *vsi, ice_for_each_q_vector(vsi, i) { struct ice_q_vector *q_vector = vsi->q_vectors[i]; - coalesce[i].itr_tx = q_vector->tx.itr_setting; - coalesce[i].itr_rx = q_vector->rx.itr_setting; + coalesce[i].itr_tx = q_vector->tx.itr_settings; + coalesce[i].itr_rx = q_vector->rx.itr_settings; coalesce[i].intrl = q_vector->intrl; if (i < vsi->num_txq) @@ -3017,26 +2957,26 @@ ice_vsi_rebuild_set_coalesce(struct ice_vsi *vsi, */ if (i < vsi->alloc_rxq && coalesce[i].rx_valid) { rc = &vsi->q_vectors[i]->rx; - rc->itr_setting = coalesce[i].itr_rx; + rc->itr_settings = coalesce[i].itr_rx; ice_write_itr(rc, rc->itr_setting); } else if (i < vsi->alloc_rxq) { rc = &vsi->q_vectors[i]->rx; - rc->itr_setting = coalesce[0].itr_rx; + rc->itr_settings = coalesce[0].itr_rx; ice_write_itr(rc, rc->itr_setting); } if (i < vsi->alloc_txq && coalesce[i].tx_valid) { rc = &vsi->q_vectors[i]->tx; - rc->itr_setting = coalesce[i].itr_tx; + rc->itr_settings = coalesce[i].itr_tx; ice_write_itr(rc, rc->itr_setting); } else if (i < vsi->alloc_txq) { rc = &vsi->q_vectors[i]->tx; - rc->itr_setting = coalesce[0].itr_tx; + rc->itr_settings = coalesce[0].itr_tx; ice_write_itr(rc, rc->itr_setting); } vsi->q_vectors[i]->intrl = coalesce[i].intrl; - ice_write_intrl(vsi->q_vectors[i], coalesce[i].intrl); + ice_set_q_vector_intrl(vsi->q_vectors[i]); } /* the number of queue vectors increased so write whatever is in @@ -3045,185 +2985,145 @@ ice_vsi_rebuild_set_coalesce(struct ice_vsi *vsi, for (; i < vsi->num_q_vectors; i++) { /* transmit */ rc = &vsi->q_vectors[i]->tx; - rc->itr_setting = coalesce[0].itr_tx; + rc->itr_settings = coalesce[0].itr_tx; ice_write_itr(rc, rc->itr_setting); /* receive */ rc = &vsi->q_vectors[i]->rx; - rc->itr_setting = coalesce[0].itr_rx; + rc->itr_settings = coalesce[0].itr_rx; ice_write_itr(rc, rc->itr_setting); vsi->q_vectors[i]->intrl = coalesce[0].intrl; - ice_write_intrl(vsi->q_vectors[i], coalesce[0].intrl); + ice_set_q_vector_intrl(vsi->q_vectors[i]); + } +} + +/** + * ice_vsi_realloc_stat_arrays - Frees unused stat structures or alloc new ones + * @vsi: VSI pointer + */ +static int +ice_vsi_realloc_stat_arrays(struct ice_vsi *vsi) +{ + u16 req_txq = vsi->req_txq ? vsi->req_txq : vsi->alloc_txq; + u16 req_rxq = vsi->req_rxq ? vsi->req_rxq : vsi->alloc_rxq; + struct ice_ring_stats **tx_ring_stats; + struct ice_ring_stats **rx_ring_stats; + struct ice_vsi_stats *vsi_stat; + struct ice_pf *pf = vsi->back; + u16 prev_txq = vsi->alloc_txq; + u16 prev_rxq = vsi->alloc_rxq; + int i; + + vsi_stat = pf->vsi_stats[vsi->idx]; + + if (req_txq < prev_txq) { + for (i = req_txq; i < prev_txq; i++) { + if (vsi_stat->tx_ring_stats[i]) { + kfree_rcu(vsi_stat->tx_ring_stats[i], rcu); + WRITE_ONCE(vsi_stat->tx_ring_stats[i], NULL); + } + } } + + tx_ring_stats = vsi_stat->tx_ring_stats; + vsi_stat->tx_ring_stats = + krealloc_array(vsi_stat->tx_ring_stats, req_txq, + sizeof(*vsi_stat->tx_ring_stats), + GFP_KERNEL | __GFP_ZERO); + if (!vsi_stat->tx_ring_stats) { + vsi_stat->tx_ring_stats = tx_ring_stats; + return -ENOMEM; + } + + if (req_rxq < prev_rxq) { + for (i = req_rxq; i < prev_rxq; i++) { + if (vsi_stat->rx_ring_stats[i]) { + kfree_rcu(vsi_stat->rx_ring_stats[i], rcu); + WRITE_ONCE(vsi_stat->rx_ring_stats[i], NULL); + } + } + } + + rx_ring_stats = vsi_stat->rx_ring_stats; + vsi_stat->rx_ring_stats = + krealloc_array(vsi_stat->rx_ring_stats, req_rxq, + sizeof(*vsi_stat->rx_ring_stats), + GFP_KERNEL | __GFP_ZERO); + if (!vsi_stat->rx_ring_stats) { + vsi_stat->rx_ring_stats = rx_ring_stats; + return -ENOMEM; + } + + return 0; } /** * ice_vsi_rebuild - Rebuild VSI after reset * @vsi: VSI to be rebuild - * @init_vsi: is this an initialization or a reconfigure of the VSI + * @vsi_flags: flags used for VSI rebuild flow + * + * Set vsi_flags to ICE_VSI_FLAG_INIT to initialize a new VSI, or + * ICE_VSI_FLAG_NO_INIT to rebuild an existing VSI in hardware. * * Returns 0 on success and negative value on failure */ -int ice_vsi_rebuild(struct ice_vsi *vsi, bool init_vsi) +int ice_vsi_rebuild(struct ice_vsi *vsi, u32 vsi_flags) { - u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 }; struct ice_coalesce_stored *coalesce; - int prev_num_q_vectors = 0; - struct ice_vf *vf = NULL; - enum ice_vsi_type vtype; - enum ice_status status; + int prev_num_q_vectors; struct ice_pf *pf; - int ret, i; + int ret; if (!vsi) return -EINVAL; + vsi->flags = vsi_flags; pf = vsi->back; - vtype = vsi->type; - if (vtype == ICE_VSI_VF) - vf = &pf->vf[vsi->vf_id]; + if (WARN_ON(vsi->type == ICE_VSI_VF && !vsi->vf)) + return -EINVAL; - coalesce = kcalloc(vsi->num_q_vectors, - sizeof(struct ice_coalesce_stored), GFP_KERNEL); - if (!coalesce) - return -ENOMEM; + mutex_lock(&vsi->xdp_state_lock); - prev_num_q_vectors = ice_vsi_rebuild_get_coalesce(vsi, coalesce); + ret = ice_vsi_realloc_stat_arrays(vsi); + if (ret) + goto unlock; - ice_rm_vsi_lan_cfg(vsi->port_info, vsi->idx); - ice_vsi_free_q_vectors(vsi); + ice_vsi_decfg(vsi); + ret = ice_vsi_cfg_def(vsi); + if (ret) + goto unlock; - /* SR-IOV determines needed MSIX resources all at once instead of per - * VSI since when VFs are spawned we know how many VFs there are and how - * many interrupts each VF needs. SR-IOV MSIX resources are also - * cleared in the same manner. - */ - if (vtype != ICE_VSI_VF) { - /* reclaim SW interrupts back to the common pool */ - ice_free_res(pf->irq_tracker, vsi->base_vector, vsi->idx); - pf->num_avail_sw_msix += vsi->num_q_vectors; - vsi->base_vector = 0; + coalesce = kcalloc(vsi->num_q_vectors, + sizeof(struct ice_coalesce_stored), GFP_KERNEL); + if (!coalesce) { + ret = -ENOMEM; + goto decfg; } - if (ice_is_xdp_ena_vsi(vsi)) - /* return value check can be skipped here, it always returns - * 0 if reset is in progress - */ - ice_destroy_xdp_rings(vsi); - ice_vsi_put_qs(vsi); - ice_vsi_clear_rings(vsi); - ice_vsi_free_arrays(vsi); - if (vtype == ICE_VSI_VF) - ice_vsi_set_num_qs(vsi, vf->vf_id); - else - ice_vsi_set_num_qs(vsi, ICE_INVAL_VFID); - - ret = ice_vsi_alloc_arrays(vsi); - if (ret < 0) - goto err_vsi; - - ice_vsi_get_qs(vsi); - - ice_alloc_fd_res(vsi); - ice_vsi_set_tc_cfg(vsi); - - /* Initialize VSI struct elements and create VSI in FW */ - ret = ice_vsi_init(vsi, init_vsi); - if (ret < 0) - goto err_vsi; - - switch (vtype) { - case ICE_VSI_CTRL: - case ICE_VSI_PF: - ret = ice_vsi_alloc_q_vectors(vsi); - if (ret) - goto err_rings; - - ret = ice_vsi_setup_vector_base(vsi); - if (ret) - goto err_vectors; - - ret = ice_vsi_set_q_vectors_reg_idx(vsi); - if (ret) - goto err_vectors; - - ret = ice_vsi_alloc_rings(vsi); - if (ret) - goto err_vectors; + prev_num_q_vectors = ice_vsi_rebuild_get_coalesce(vsi, coalesce); - ice_vsi_map_rings_to_vectors(vsi); - if (ice_is_xdp_ena_vsi(vsi)) { - vsi->num_xdp_txq = vsi->alloc_rxq; - ret = ice_prepare_xdp_rings(vsi, vsi->xdp_prog); - if (ret) - goto err_vectors; + ret = ice_vsi_cfg_tc_lan(pf, vsi); + if (ret) { + if (vsi_flags & ICE_VSI_FLAG_INIT) { + ret = -EIO; + goto free_coalesce; } - /* ICE_VSI_CTRL does not need RSS so skip RSS processing */ - if (vtype != ICE_VSI_CTRL) - /* Do not exit if configuring RSS had an issue, at - * least receive traffic on first queue. Hence no - * need to capture return value - */ - if (test_bit(ICE_FLAG_RSS_ENA, pf->flags)) - ice_vsi_cfg_rss_lut_key(vsi); - break; - case ICE_VSI_VF: - ret = ice_vsi_alloc_q_vectors(vsi); - if (ret) - goto err_rings; - ret = ice_vsi_set_q_vectors_reg_idx(vsi); - if (ret) - goto err_vectors; - - ret = ice_vsi_alloc_rings(vsi); - if (ret) - goto err_vectors; - - break; - default: - break; + ret = ice_schedule_reset(pf, ICE_RESET_PFR); + goto free_coalesce; } - /* configure VSI nodes based on number of queues and TC's */ - for (i = 0; i < vsi->tc_cfg.numtc; i++) { - max_txqs[i] = vsi->alloc_txq; - - if (ice_is_xdp_ena_vsi(vsi)) - max_txqs[i] += vsi->num_xdp_txq; - } - - status = ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc, - max_txqs); - if (status) { - dev_err(ice_pf_to_dev(pf), "VSI %d failed lan queue config, error %s\n", - vsi->vsi_num, ice_stat_str(status)); - if (init_vsi) { - ret = -EIO; - goto err_vectors; - } else { - return ice_schedule_reset(pf, ICE_RESET_PFR); - } - } ice_vsi_rebuild_set_coalesce(vsi, coalesce, prev_num_q_vectors); - kfree(coalesce); + clear_bit(ICE_VSI_REBUILD_PENDING, vsi->state); - return 0; - -err_vectors: - ice_vsi_free_q_vectors(vsi); -err_rings: - if (vsi->netdev) { - vsi->current_netdev_flags = 0; - unregister_netdev(vsi->netdev); - free_netdev(vsi->netdev); - vsi->netdev = NULL; - } -err_vsi: - ice_vsi_clear(vsi); - set_bit(ICE_RESET_FAILED, pf->state); +free_coalesce: kfree(coalesce); +decfg: + if (ret) + ice_vsi_decfg(vsi); +unlock: + mutex_unlock(&vsi->xdp_state_lock); return ret; } @@ -3267,7 +3167,6 @@ int ice_wait_for_reset(struct ice_pf *pf, unsigned long timeout) return 0; } -#ifdef CONFIG_DCB /** * ice_vsi_update_q_map - update our copy of the VSI info with new queue map * @vsi: VSI being configured @@ -3283,6 +3182,162 @@ static void ice_vsi_update_q_map(struct ice_vsi *vsi, struct ice_vsi_ctx *ctx) } /** + * ice_vsi_cfg_netdev_tc - Setup the netdev TC configuration + * @vsi: the VSI being configured + * @ena_tc: TC map to be enabled + */ +void ice_vsi_cfg_netdev_tc(struct ice_vsi *vsi, u8 ena_tc) +{ + struct net_device *netdev = vsi->netdev; + struct ice_pf *pf = vsi->back; + int numtc = vsi->tc_cfg.numtc; + struct ice_dcbx_cfg *dcbcfg; + u8 netdev_tc; + int i; + + if (!netdev) + return; + + /* CHNL VSI doesn't have its own netdev, hence, no netdev_tc */ + if (vsi->type == ICE_VSI_CHNL) + return; + + if (!ena_tc) { + netdev_reset_tc(netdev); + return; + } + + if (vsi->type == ICE_VSI_PF && ice_is_adq_active(pf)) + numtc = vsi->all_numtc; + + if (netdev_set_num_tc(netdev, numtc)) + return; + + dcbcfg = &pf->hw.port_info->qos_cfg.local_dcbx_cfg; + + ice_for_each_traffic_class(i) + if (vsi->tc_cfg.ena_tc & BIT(i)) + netdev_set_tc_queue(netdev, + vsi->tc_cfg.tc_info[i].netdev_tc, + vsi->tc_cfg.tc_info[i].qcount_tx, + vsi->tc_cfg.tc_info[i].qoffset); + /* setup TC queue map for CHNL TCs */ + ice_for_each_chnl_tc(i) { + if (!(vsi->all_enatc & BIT(i))) + break; + if (!vsi->mqprio_qopt.qopt.count[i]) + break; + netdev_set_tc_queue(netdev, i, + vsi->mqprio_qopt.qopt.count[i], + vsi->mqprio_qopt.qopt.offset[i]); + } + + if (test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) + return; + + for (i = 0; i < ICE_MAX_USER_PRIORITY; i++) { + u8 ets_tc = dcbcfg->etscfg.prio_table[i]; + + /* Get the mapped netdev TC# for the UP */ + netdev_tc = vsi->tc_cfg.tc_info[ets_tc].netdev_tc; + netdev_set_prio_tc_map(netdev, i, netdev_tc); + } +} + +/** + * ice_vsi_setup_q_map_mqprio - Prepares mqprio based tc_config + * @vsi: the VSI being configured, + * @ctxt: VSI context structure + * @ena_tc: number of traffic classes to enable + * + * Prepares VSI tc_config to have queue configurations based on MQPRIO options. + */ +static int +ice_vsi_setup_q_map_mqprio(struct ice_vsi *vsi, struct ice_vsi_ctx *ctxt, + u8 ena_tc) +{ + u16 pow, offset = 0, qcount_tx = 0, qcount_rx = 0, qmap; + u16 tc0_offset = vsi->mqprio_qopt.qopt.offset[0]; + int tc0_qcount = vsi->mqprio_qopt.qopt.count[0]; + u16 new_txq, new_rxq; + u8 netdev_tc = 0; + int i; + + vsi->tc_cfg.ena_tc = ena_tc ? ena_tc : 1; + + pow = order_base_2(tc0_qcount); + qmap = FIELD_PREP(ICE_AQ_VSI_TC_Q_OFFSET_M, tc0_offset); + qmap |= FIELD_PREP(ICE_AQ_VSI_TC_Q_NUM_M, pow); + + ice_for_each_traffic_class(i) { + if (!(vsi->tc_cfg.ena_tc & BIT(i))) { + /* TC is not enabled */ + vsi->tc_cfg.tc_info[i].qoffset = 0; + vsi->tc_cfg.tc_info[i].qcount_rx = 1; + vsi->tc_cfg.tc_info[i].qcount_tx = 1; + vsi->tc_cfg.tc_info[i].netdev_tc = 0; + ctxt->info.tc_mapping[i] = 0; + continue; + } + + offset = vsi->mqprio_qopt.qopt.offset[i]; + qcount_rx = vsi->mqprio_qopt.qopt.count[i]; + qcount_tx = vsi->mqprio_qopt.qopt.count[i]; + vsi->tc_cfg.tc_info[i].qoffset = offset; + vsi->tc_cfg.tc_info[i].qcount_rx = qcount_rx; + vsi->tc_cfg.tc_info[i].qcount_tx = qcount_tx; + vsi->tc_cfg.tc_info[i].netdev_tc = netdev_tc++; + } + + if (vsi->all_numtc && vsi->all_numtc != vsi->tc_cfg.numtc) { + ice_for_each_chnl_tc(i) { + if (!(vsi->all_enatc & BIT(i))) + continue; + offset = vsi->mqprio_qopt.qopt.offset[i]; + qcount_rx = vsi->mqprio_qopt.qopt.count[i]; + qcount_tx = vsi->mqprio_qopt.qopt.count[i]; + } + } + + new_txq = offset + qcount_tx; + if (new_txq > vsi->alloc_txq) { + dev_err(ice_pf_to_dev(vsi->back), "Trying to use more Tx queues (%u), than were allocated (%u)!\n", + new_txq, vsi->alloc_txq); + return -EINVAL; + } + + new_rxq = offset + qcount_rx; + if (new_rxq > vsi->alloc_rxq) { + dev_err(ice_pf_to_dev(vsi->back), "Trying to use more Rx queues (%u), than were allocated (%u)!\n", + new_rxq, vsi->alloc_rxq); + return -EINVAL; + } + + /* Set actual Tx/Rx queue pairs */ + vsi->num_txq = new_txq; + vsi->num_rxq = new_rxq; + + /* Setup queue TC[0].qmap for given VSI context */ + ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); + ctxt->info.q_mapping[0] = cpu_to_le16(vsi->rxq_map[0]); + ctxt->info.q_mapping[1] = cpu_to_le16(tc0_qcount); + + /* Find queue count available for channel VSIs and starting offset + * for channel VSIs + */ + if (tc0_qcount && tc0_qcount < vsi->num_rxq) { + vsi->cnt_q_avail = vsi->num_rxq - tc0_qcount; + vsi->next_base_q = tc0_qcount; + } + dev_dbg(ice_pf_to_dev(vsi->back), "vsi->num_txq = %d\n", vsi->num_txq); + dev_dbg(ice_pf_to_dev(vsi->back), "vsi->num_rxq = %d\n", vsi->num_rxq); + dev_dbg(ice_pf_to_dev(vsi->back), "all_numtc %u, all_enatc: 0x%04x, tc_cfg.numtc %u\n", + vsi->all_numtc, vsi->all_enatc, vsi->tc_cfg.numtc); + + return 0; +} + +/** * ice_vsi_cfg_tc - Configure VSI Tx Sched for given TC map * @vsi: VSI to be configured * @ena_tc: TC bitmap @@ -3293,13 +3348,16 @@ int ice_vsi_cfg_tc(struct ice_vsi *vsi, u8 ena_tc) { u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 }; struct ice_pf *pf = vsi->back; + struct ice_tc_cfg old_tc_cfg; struct ice_vsi_ctx *ctx; - enum ice_status status; struct device *dev; int i, ret = 0; u8 num_tc = 0; dev = ice_pf_to_dev(pf); + if (vsi->tc_cfg.ena_tc == ena_tc && + vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL) + return 0; ice_for_each_traffic_class(i) { /* build bitmap of enabled TCs */ @@ -3307,8 +3365,15 @@ int ice_vsi_cfg_tc(struct ice_vsi *vsi, u8 ena_tc) num_tc++; /* populate max_txqs per TC */ max_txqs[i] = vsi->alloc_txq; + /* Update max_txqs if it is CHNL VSI, because alloc_t[r]xq are + * zero for CHNL VSI, hence use num_txq instead as max_txqs + */ + if (vsi->type == ICE_VSI_CHNL && + test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) + max_txqs[i] = vsi->num_txq; } + memcpy(&old_tc_cfg, &vsi->tc_cfg, sizeof(old_tc_cfg)); vsi->tc_cfg.ena_tc = ena_tc; vsi->tc_cfg.numtc = num_tc; @@ -3319,24 +3384,35 @@ int ice_vsi_cfg_tc(struct ice_vsi *vsi, u8 ena_tc) ctx->vf_num = 0; ctx->info = vsi->info; - ice_vsi_setup_q_map(vsi, ctx); + if (vsi->type == ICE_VSI_PF && + test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) + ret = ice_vsi_setup_q_map_mqprio(vsi, ctx, ena_tc); + else + ret = ice_vsi_setup_q_map(vsi, ctx); + + if (ret) { + memcpy(&vsi->tc_cfg, &old_tc_cfg, sizeof(vsi->tc_cfg)); + goto out; + } /* must to indicate which section of VSI context are being modified */ ctx->info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_RXQ_MAP_VALID); - status = ice_update_vsi(&pf->hw, vsi->idx, ctx, NULL); - if (status) { + ret = ice_update_vsi(&pf->hw, vsi->idx, ctx, NULL); + if (ret) { dev_info(dev, "Failed VSI Update\n"); - ret = -EIO; goto out; } - status = ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc, - max_txqs); + if (vsi->type == ICE_VSI_PF && + test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) + ret = ice_cfg_vsi_lan(vsi->port_info, vsi->idx, 1, max_txqs); + else + ret = ice_cfg_vsi_lan(vsi->port_info, vsi->idx, + vsi->tc_cfg.ena_tc, max_txqs); - if (status) { - dev_err(dev, "VSI %d failed TC config, error %s\n", - vsi->vsi_num, ice_stat_str(status)); - ret = -EIO; + if (ret) { + dev_err(dev, "VSI %d failed TC config, error %d\n", + vsi->vsi_num, ret); goto out; } ice_vsi_update_q_map(vsi, ctx); @@ -3347,20 +3423,19 @@ out: kfree(ctx); return ret; } -#endif /* CONFIG_DCB */ /** * ice_update_ring_stats - Update ring statistics - * @ring: ring to update + * @stats: stats to be updated * @pkts: number of processed packets * @bytes: number of processed bytes * * This function assumes that caller has acquired a u64_stats_sync lock. */ -static void ice_update_ring_stats(struct ice_ring *ring, u64 pkts, u64 bytes) +static void ice_update_ring_stats(struct ice_q_stats *stats, u64 pkts, u64 bytes) { - ring->stats.bytes += bytes; - ring->stats.pkts += pkts; + stats->bytes += bytes; + stats->pkts += pkts; } /** @@ -3369,11 +3444,11 @@ static void ice_update_ring_stats(struct ice_ring *ring, u64 pkts, u64 bytes) * @pkts: number of processed packets * @bytes: number of processed bytes */ -void ice_update_tx_ring_stats(struct ice_ring *tx_ring, u64 pkts, u64 bytes) +void ice_update_tx_ring_stats(struct ice_tx_ring *tx_ring, u64 pkts, u64 bytes) { - u64_stats_update_begin(&tx_ring->syncp); - ice_update_ring_stats(tx_ring, pkts, bytes); - u64_stats_update_end(&tx_ring->syncp); + u64_stats_update_begin(&tx_ring->ring_stats->syncp); + ice_update_ring_stats(&tx_ring->ring_stats->stats, pkts, bytes); + u64_stats_update_end(&tx_ring->ring_stats->syncp); } /** @@ -3382,157 +3457,263 @@ void ice_update_tx_ring_stats(struct ice_ring *tx_ring, u64 pkts, u64 bytes) * @pkts: number of processed packets * @bytes: number of processed bytes */ -void ice_update_rx_ring_stats(struct ice_ring *rx_ring, u64 pkts, u64 bytes) -{ - u64_stats_update_begin(&rx_ring->syncp); - ice_update_ring_stats(rx_ring, pkts, bytes); - u64_stats_update_end(&rx_ring->syncp); -} - -/** - * ice_status_to_errno - convert from enum ice_status to Linux errno - * @err: ice_status value to convert - */ -int ice_status_to_errno(enum ice_status err) +void ice_update_rx_ring_stats(struct ice_rx_ring *rx_ring, u64 pkts, u64 bytes) { - switch (err) { - case ICE_SUCCESS: - return 0; - case ICE_ERR_DOES_NOT_EXIST: - return -ENOENT; - case ICE_ERR_OUT_OF_RANGE: - case ICE_ERR_AQ_ERROR: - case ICE_ERR_AQ_TIMEOUT: - case ICE_ERR_AQ_EMPTY: - case ICE_ERR_AQ_FW_CRITICAL: - return -EIO; - case ICE_ERR_PARAM: - case ICE_ERR_INVAL_SIZE: - return -EINVAL; - case ICE_ERR_NO_MEMORY: - return -ENOMEM; - case ICE_ERR_MAX_LIMIT: - return -EAGAIN; - case ICE_ERR_RESET_ONGOING: - return -EBUSY; - case ICE_ERR_AQ_FULL: - return -ENOSPC; - default: - return -EINVAL; - } + u64_stats_update_begin(&rx_ring->ring_stats->syncp); + ice_update_ring_stats(&rx_ring->ring_stats->stats, pkts, bytes); + u64_stats_update_end(&rx_ring->ring_stats->syncp); } /** * ice_is_dflt_vsi_in_use - check if the default forwarding VSI is being used - * @sw: switch to check if its default forwarding VSI is free + * @pi: port info of the switch with default VSI * - * Return true if the default forwarding VSI is already being used, else returns - * false signalling that it's available to use. + * Return true if the there is a single VSI in default forwarding VSI list */ -bool ice_is_dflt_vsi_in_use(struct ice_sw *sw) +bool ice_is_dflt_vsi_in_use(struct ice_port_info *pi) { - return (sw->dflt_vsi && sw->dflt_vsi_ena); + bool exists = false; + + ice_check_if_dflt_vsi(pi, 0, &exists); + return exists; } /** * ice_is_vsi_dflt_vsi - check if the VSI passed in is the default VSI - * @sw: switch for the default forwarding VSI to compare against * @vsi: VSI to compare against default forwarding VSI * * If this VSI passed in is the default forwarding VSI then return true, else * return false */ -bool ice_is_vsi_dflt_vsi(struct ice_sw *sw, struct ice_vsi *vsi) +bool ice_is_vsi_dflt_vsi(struct ice_vsi *vsi) { - return (sw->dflt_vsi == vsi && sw->dflt_vsi_ena); + return ice_check_if_dflt_vsi(vsi->port_info, vsi->idx, NULL); } /** * ice_set_dflt_vsi - set the default forwarding VSI - * @sw: switch used to assign the default forwarding VSI * @vsi: VSI getting set as the default forwarding VSI on the switch * * If the VSI passed in is already the default VSI and it's enabled just return * success. * - * If there is already a default VSI on the switch and it's enabled then return - * -EEXIST since there can only be one default VSI per switch. - * - * Otherwise try to set the VSI passed in as the switch's default VSI and - * return the result. + * Otherwise try to set the VSI passed in as the switch's default VSI and + * return the result. */ -int ice_set_dflt_vsi(struct ice_sw *sw, struct ice_vsi *vsi) +int ice_set_dflt_vsi(struct ice_vsi *vsi) { - enum ice_status status; struct device *dev; + int status; - if (!sw || !vsi) + if (!vsi) return -EINVAL; dev = ice_pf_to_dev(vsi->back); - /* the VSI passed in is already the default VSI */ - if (ice_is_vsi_dflt_vsi(sw, vsi)) { - dev_dbg(dev, "VSI %d passed in is already the default forwarding VSI, nothing to do\n", + if (ice_lag_is_switchdev_running(vsi->back)) { + dev_dbg(dev, "VSI %d passed is a part of LAG containing interfaces in switchdev mode, nothing to do\n", vsi->vsi_num); return 0; } - /* another VSI is already the default VSI for this switch */ - if (ice_is_dflt_vsi_in_use(sw)) { - dev_err(dev, "Default forwarding VSI %d already in use, disable it and try again\n", - sw->dflt_vsi->vsi_num); - return -EEXIST; + /* the VSI passed in is already the default VSI */ + if (ice_is_vsi_dflt_vsi(vsi)) { + dev_dbg(dev, "VSI %d passed in is already the default forwarding VSI, nothing to do\n", + vsi->vsi_num); + return 0; } - status = ice_cfg_dflt_vsi(&vsi->back->hw, vsi->idx, true, ICE_FLTR_RX); + status = ice_cfg_dflt_vsi(vsi->port_info, vsi->idx, true, ICE_FLTR_RX); if (status) { - dev_err(dev, "Failed to set VSI %d as the default forwarding VSI, error %s\n", - vsi->vsi_num, ice_stat_str(status)); - return -EIO; + dev_err(dev, "Failed to set VSI %d as the default forwarding VSI, error %d\n", + vsi->vsi_num, status); + return status; } - sw->dflt_vsi = vsi; - sw->dflt_vsi_ena = true; - return 0; } /** * ice_clear_dflt_vsi - clear the default forwarding VSI - * @sw: switch used to clear the default VSI + * @vsi: VSI to remove from filter list * * If the switch has no default VSI or it's not enabled then return error. * * Otherwise try to clear the default VSI and return the result. */ -int ice_clear_dflt_vsi(struct ice_sw *sw) +int ice_clear_dflt_vsi(struct ice_vsi *vsi) { - struct ice_vsi *dflt_vsi; - enum ice_status status; struct device *dev; + int status; - if (!sw) + if (!vsi) return -EINVAL; - dev = ice_pf_to_dev(sw->pf); - - dflt_vsi = sw->dflt_vsi; + dev = ice_pf_to_dev(vsi->back); /* there is no default VSI configured */ - if (!ice_is_dflt_vsi_in_use(sw)) + if (!ice_is_dflt_vsi_in_use(vsi->port_info)) return -ENODEV; - status = ice_cfg_dflt_vsi(&dflt_vsi->back->hw, dflt_vsi->idx, false, + status = ice_cfg_dflt_vsi(vsi->port_info, vsi->idx, false, ICE_FLTR_RX); if (status) { - dev_err(dev, "Failed to clear the default forwarding VSI %d, error %s\n", - dflt_vsi->vsi_num, ice_stat_str(status)); + dev_err(dev, "Failed to clear the default forwarding VSI %d, error %d\n", + vsi->vsi_num, status); return -EIO; } - sw->dflt_vsi = NULL; - sw->dflt_vsi_ena = false; + return 0; +} + +/** + * ice_get_link_speed_mbps - get link speed in Mbps + * @vsi: the VSI whose link speed is being queried + * + * Return current VSI link speed and 0 if the speed is unknown. + */ +int ice_get_link_speed_mbps(struct ice_vsi *vsi) +{ + unsigned int link_speed; + + link_speed = vsi->port_info->phy.link_info.link_speed; + + return (int)ice_get_link_speed(fls(link_speed) - 1); +} + +/** + * ice_get_link_speed_kbps - get link speed in Kbps + * @vsi: the VSI whose link speed is being queried + * + * Return current VSI link speed and 0 if the speed is unknown. + */ +int ice_get_link_speed_kbps(struct ice_vsi *vsi) +{ + int speed_mbps; + + speed_mbps = ice_get_link_speed_mbps(vsi); + + return speed_mbps * 1000; +} + +/** + * ice_set_min_bw_limit - setup minimum BW limit for Tx based on min_tx_rate + * @vsi: VSI to be configured + * @min_tx_rate: min Tx rate in Kbps to be configured as BW limit + * + * If the min_tx_rate is specified as 0 that means to clear the minimum BW limit + * profile, otherwise a non-zero value will force a minimum BW limit for the VSI + * on TC 0. + */ +int ice_set_min_bw_limit(struct ice_vsi *vsi, u64 min_tx_rate) +{ + struct ice_pf *pf = vsi->back; + struct device *dev; + int status; + int speed; + + dev = ice_pf_to_dev(pf); + if (!vsi->port_info) { + dev_dbg(dev, "VSI %d, type %u specified doesn't have valid port_info\n", + vsi->idx, vsi->type); + return -EINVAL; + } + + speed = ice_get_link_speed_kbps(vsi); + if (min_tx_rate > (u64)speed) { + dev_err(dev, "invalid min Tx rate %llu Kbps specified for %s %d is greater than current link speed %u Kbps\n", + min_tx_rate, ice_vsi_type_str(vsi->type), vsi->idx, + speed); + return -EINVAL; + } + + /* Configure min BW for VSI limit */ + if (min_tx_rate) { + status = ice_cfg_vsi_bw_lmt_per_tc(vsi->port_info, vsi->idx, 0, + ICE_MIN_BW, min_tx_rate); + if (status) { + dev_err(dev, "failed to set min Tx rate(%llu Kbps) for %s %d\n", + min_tx_rate, ice_vsi_type_str(vsi->type), + vsi->idx); + return status; + } + + dev_dbg(dev, "set min Tx rate(%llu Kbps) for %s\n", + min_tx_rate, ice_vsi_type_str(vsi->type)); + } else { + status = ice_cfg_vsi_bw_dflt_lmt_per_tc(vsi->port_info, + vsi->idx, 0, + ICE_MIN_BW); + if (status) { + dev_err(dev, "failed to clear min Tx rate configuration for %s %d\n", + ice_vsi_type_str(vsi->type), vsi->idx); + return status; + } + + dev_dbg(dev, "cleared min Tx rate configuration for %s %d\n", + ice_vsi_type_str(vsi->type), vsi->idx); + } + + return 0; +} + +/** + * ice_set_max_bw_limit - setup maximum BW limit for Tx based on max_tx_rate + * @vsi: VSI to be configured + * @max_tx_rate: max Tx rate in Kbps to be configured as BW limit + * + * If the max_tx_rate is specified as 0 that means to clear the maximum BW limit + * profile, otherwise a non-zero value will force a maximum BW limit for the VSI + * on TC 0. + */ +int ice_set_max_bw_limit(struct ice_vsi *vsi, u64 max_tx_rate) +{ + struct ice_pf *pf = vsi->back; + struct device *dev; + int status; + int speed; + + dev = ice_pf_to_dev(pf); + if (!vsi->port_info) { + dev_dbg(dev, "VSI %d, type %u specified doesn't have valid port_info\n", + vsi->idx, vsi->type); + return -EINVAL; + } + + speed = ice_get_link_speed_kbps(vsi); + if (max_tx_rate > (u64)speed) { + dev_err(dev, "invalid max Tx rate %llu Kbps specified for %s %d is greater than current link speed %u Kbps\n", + max_tx_rate, ice_vsi_type_str(vsi->type), vsi->idx, + speed); + return -EINVAL; + } + + /* Configure max BW for VSI limit */ + if (max_tx_rate) { + status = ice_cfg_vsi_bw_lmt_per_tc(vsi->port_info, vsi->idx, 0, + ICE_MAX_BW, max_tx_rate); + if (status) { + dev_err(dev, "failed setting max Tx rate(%llu Kbps) for %s %d\n", + max_tx_rate, ice_vsi_type_str(vsi->type), + vsi->idx); + return status; + } + + dev_dbg(dev, "set max Tx rate(%llu Kbps) for %s %d\n", + max_tx_rate, ice_vsi_type_str(vsi->type), vsi->idx); + } else { + status = ice_cfg_vsi_bw_dflt_lmt_per_tc(vsi->port_info, + vsi->idx, 0, + ICE_MAX_BW); + if (status) { + dev_err(dev, "failed clearing max Tx rate configuration for %s %d\n", + ice_vsi_type_str(vsi->type), vsi->idx); + return status; + } + + dev_dbg(dev, "cleared max Tx rate configuration for %s %d\n", + ice_vsi_type_str(vsi->type), vsi->idx); + } return 0; } @@ -3547,29 +3728,330 @@ int ice_set_link(struct ice_vsi *vsi, bool ena) struct device *dev = ice_pf_to_dev(vsi->back); struct ice_port_info *pi = vsi->port_info; struct ice_hw *hw = pi->hw; - enum ice_status status; + int status; if (vsi->type != ICE_VSI_PF) return -EINVAL; status = ice_aq_set_link_restart_an(pi, ena, NULL); - /* if link is owned by manageability, FW will return ICE_AQ_RC_EMODE. + /* if link is owned by manageability, FW will return LIBIE_AQ_RC_EMODE. * this is not a fatal error, so print a warning message and return * a success code. Return an error if FW returns an error code other - * than ICE_AQ_RC_EMODE + * than LIBIE_AQ_RC_EMODE */ - if (status == ICE_ERR_AQ_ERROR) { - if (hw->adminq.sq_last_status == ICE_AQ_RC_EMODE) - dev_warn(dev, "can't set link to %s, err %s aq_err %s. not fatal, continuing\n", - (ena ? "ON" : "OFF"), ice_stat_str(status), - ice_aq_str(hw->adminq.sq_last_status)); + if (status == -EIO) { + if (hw->adminq.sq_last_status == LIBIE_AQ_RC_EMODE) + dev_dbg(dev, "can't set link to %s, err %d aq_err %s. not fatal, continuing\n", + (ena ? "ON" : "OFF"), status, + libie_aq_str(hw->adminq.sq_last_status)); } else if (status) { - dev_err(dev, "can't set link to %s, err %s aq_err %s\n", - (ena ? "ON" : "OFF"), ice_stat_str(status), - ice_aq_str(hw->adminq.sq_last_status)); - return -EIO; + dev_err(dev, "can't set link to %s, err %d aq_err %s\n", + (ena ? "ON" : "OFF"), status, + libie_aq_str(hw->adminq.sq_last_status)); + return status; } return 0; } + +/** + * ice_vsi_add_vlan_zero - add VLAN 0 filter(s) for this VSI + * @vsi: VSI used to add VLAN filters + * + * In Single VLAN Mode (SVM), single VLAN filters via ICE_SW_LKUP_VLAN are based + * on the inner VLAN ID, so the VLAN TPID (i.e. 0x8100 or 0x888a8) doesn't + * matter. In Double VLAN Mode (DVM), outer/single VLAN filters via + * ICE_SW_LKUP_VLAN are based on the outer/single VLAN ID + VLAN TPID. + * + * For both modes add a VLAN 0 + no VLAN TPID filter to handle untagged traffic + * when VLAN pruning is enabled. Also, this handles VLAN 0 priority tagged + * traffic in SVM, since the VLAN TPID isn't part of filtering. + * + * If DVM is enabled then an explicit VLAN 0 + VLAN TPID filter needs to be + * added to allow VLAN 0 priority tagged traffic in DVM, since the VLAN TPID is + * part of filtering. + */ +int ice_vsi_add_vlan_zero(struct ice_vsi *vsi) +{ + struct ice_vsi_vlan_ops *vlan_ops = ice_get_compat_vsi_vlan_ops(vsi); + struct ice_vlan vlan; + int err; + + vlan = ICE_VLAN(0, 0, 0); + err = vlan_ops->add_vlan(vsi, &vlan); + if (err && err != -EEXIST) + return err; + + /* in SVM both VLAN 0 filters are identical */ + if (!ice_is_dvm_ena(&vsi->back->hw)) + return 0; + + vlan = ICE_VLAN(ETH_P_8021Q, 0, 0); + err = vlan_ops->add_vlan(vsi, &vlan); + if (err && err != -EEXIST) + return err; + + return 0; +} + +/** + * ice_vsi_del_vlan_zero - delete VLAN 0 filter(s) for this VSI + * @vsi: VSI used to add VLAN filters + * + * Delete the VLAN 0 filters in the same manner that they were added in + * ice_vsi_add_vlan_zero. + */ +int ice_vsi_del_vlan_zero(struct ice_vsi *vsi) +{ + struct ice_vsi_vlan_ops *vlan_ops = ice_get_compat_vsi_vlan_ops(vsi); + struct ice_vlan vlan; + int err; + + vlan = ICE_VLAN(0, 0, 0); + err = vlan_ops->del_vlan(vsi, &vlan); + if (err && err != -EEXIST) + return err; + + /* in SVM both VLAN 0 filters are identical */ + if (!ice_is_dvm_ena(&vsi->back->hw)) + return 0; + + vlan = ICE_VLAN(ETH_P_8021Q, 0, 0); + err = vlan_ops->del_vlan(vsi, &vlan); + if (err && err != -EEXIST) + return err; + + /* when deleting the last VLAN filter, make sure to disable the VLAN + * promisc mode so the filter isn't left by accident + */ + return ice_clear_vsi_promisc(&vsi->back->hw, vsi->idx, + ICE_MCAST_VLAN_PROMISC_BITS, 0); +} + +/** + * ice_vsi_num_zero_vlans - get number of VLAN 0 filters based on VLAN mode + * @vsi: VSI used to get the VLAN mode + * + * If DVM is enabled then 2 VLAN 0 filters are added, else if SVM is enabled + * then 1 VLAN 0 filter is added. See ice_vsi_add_vlan_zero for more details. + */ +static u16 ice_vsi_num_zero_vlans(struct ice_vsi *vsi) +{ +#define ICE_DVM_NUM_ZERO_VLAN_FLTRS 2 +#define ICE_SVM_NUM_ZERO_VLAN_FLTRS 1 + /* no VLAN 0 filter is created when a port VLAN is active */ + if (vsi->type == ICE_VSI_VF) { + if (WARN_ON(!vsi->vf)) + return 0; + + if (ice_vf_is_port_vlan_ena(vsi->vf)) + return 0; + } + + if (ice_is_dvm_ena(&vsi->back->hw)) + return ICE_DVM_NUM_ZERO_VLAN_FLTRS; + else + return ICE_SVM_NUM_ZERO_VLAN_FLTRS; +} + +/** + * ice_vsi_has_non_zero_vlans - check if VSI has any non-zero VLANs + * @vsi: VSI used to determine if any non-zero VLANs have been added + */ +bool ice_vsi_has_non_zero_vlans(struct ice_vsi *vsi) +{ + return (vsi->num_vlan > ice_vsi_num_zero_vlans(vsi)); +} + +/** + * ice_vsi_num_non_zero_vlans - get the number of non-zero VLANs for this VSI + * @vsi: VSI used to get the number of non-zero VLANs added + */ +u16 ice_vsi_num_non_zero_vlans(struct ice_vsi *vsi) +{ + return (vsi->num_vlan - ice_vsi_num_zero_vlans(vsi)); +} + +/** + * ice_is_feature_supported + * @pf: pointer to the struct ice_pf instance + * @f: feature enum to be checked + * + * returns true if feature is supported, false otherwise + */ +bool ice_is_feature_supported(struct ice_pf *pf, enum ice_feature f) +{ + if (f < 0 || f >= ICE_F_MAX) + return false; + + return test_bit(f, pf->features); +} + +/** + * ice_set_feature_support + * @pf: pointer to the struct ice_pf instance + * @f: feature enum to set + */ +void ice_set_feature_support(struct ice_pf *pf, enum ice_feature f) +{ + if (f < 0 || f >= ICE_F_MAX) + return; + + set_bit(f, pf->features); +} + +/** + * ice_clear_feature_support + * @pf: pointer to the struct ice_pf instance + * @f: feature enum to clear + */ +void ice_clear_feature_support(struct ice_pf *pf, enum ice_feature f) +{ + if (f < 0 || f >= ICE_F_MAX) + return; + + clear_bit(f, pf->features); +} + +/** + * ice_init_feature_support + * @pf: pointer to the struct ice_pf instance + * + * called during init to setup supported feature + */ +void ice_init_feature_support(struct ice_pf *pf) +{ + switch (pf->hw.device_id) { + case ICE_DEV_ID_E810C_BACKPLANE: + case ICE_DEV_ID_E810C_QSFP: + case ICE_DEV_ID_E810C_SFP: + case ICE_DEV_ID_E810_XXV_BACKPLANE: + case ICE_DEV_ID_E810_XXV_QSFP: + case ICE_DEV_ID_E810_XXV_SFP: + ice_set_feature_support(pf, ICE_F_DSCP); + if (ice_is_phy_rclk_in_netlist(&pf->hw)) + ice_set_feature_support(pf, ICE_F_PHY_RCLK); + /* If we don't own the timer - don't enable other caps */ + if (!ice_pf_src_tmr_owned(pf)) + break; + if (ice_is_cgu_in_netlist(&pf->hw)) + ice_set_feature_support(pf, ICE_F_CGU); + if (ice_is_clock_mux_in_netlist(&pf->hw)) + ice_set_feature_support(pf, ICE_F_SMA_CTRL); + if (ice_gnss_is_module_present(&pf->hw)) + ice_set_feature_support(pf, ICE_F_GNSS); + break; + default: + break; + } + + if (pf->hw.mac_type == ICE_MAC_E830) { + ice_set_feature_support(pf, ICE_F_MBX_LIMIT); + ice_set_feature_support(pf, ICE_F_GCS); + ice_set_feature_support(pf, ICE_F_TXTIME); + } +} + +/** + * ice_vsi_update_security - update security block in VSI + * @vsi: pointer to VSI structure + * @fill: function pointer to fill ctx + */ +int +ice_vsi_update_security(struct ice_vsi *vsi, void (*fill)(struct ice_vsi_ctx *)) +{ + struct ice_vsi_ctx ctx = { 0 }; + + ctx.info = vsi->info; + ctx.info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_SECURITY_VALID); + fill(&ctx); + + if (ice_update_vsi(&vsi->back->hw, vsi->idx, &ctx, NULL)) + return -ENODEV; + + vsi->info = ctx.info; + return 0; +} + +/** + * ice_vsi_ctx_set_antispoof - set antispoof function in VSI ctx + * @ctx: pointer to VSI ctx structure + */ +void ice_vsi_ctx_set_antispoof(struct ice_vsi_ctx *ctx) +{ + ctx->info.sec_flags |= ICE_AQ_VSI_SEC_FLAG_ENA_MAC_ANTI_SPOOF | + (ICE_AQ_VSI_SEC_TX_VLAN_PRUNE_ENA << + ICE_AQ_VSI_SEC_TX_PRUNE_ENA_S); +} + +/** + * ice_vsi_ctx_clear_antispoof - clear antispoof function in VSI ctx + * @ctx: pointer to VSI ctx structure + */ +void ice_vsi_ctx_clear_antispoof(struct ice_vsi_ctx *ctx) +{ + ctx->info.sec_flags &= ~ICE_AQ_VSI_SEC_FLAG_ENA_MAC_ANTI_SPOOF & + ~(ICE_AQ_VSI_SEC_TX_VLAN_PRUNE_ENA << + ICE_AQ_VSI_SEC_TX_PRUNE_ENA_S); +} + +/** + * ice_vsi_update_local_lb - update sw block in VSI with local loopback bit + * @vsi: pointer to VSI structure + * @set: set or unset the bit + */ +int +ice_vsi_update_local_lb(struct ice_vsi *vsi, bool set) +{ + struct ice_vsi_ctx ctx = { + .info = vsi->info, + }; + + ctx.info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_SW_VALID); + if (set) + ctx.info.sw_flags |= ICE_AQ_VSI_SW_FLAG_LOCAL_LB; + else + ctx.info.sw_flags &= ~ICE_AQ_VSI_SW_FLAG_LOCAL_LB; + + if (ice_update_vsi(&vsi->back->hw, vsi->idx, &ctx, NULL)) + return -ENODEV; + + vsi->info = ctx.info; + return 0; +} + +/** + * ice_vsi_update_l2tsel - update l2tsel field for all Rx rings on this VSI + * @vsi: VSI used to update l2tsel on + * @l2tsel: l2tsel setting requested + * + * Use the l2tsel setting to update all of the Rx queue context bits for l2tsel. + * This will modify which descriptor field the first offloaded VLAN will be + * stripped into. + */ +void ice_vsi_update_l2tsel(struct ice_vsi *vsi, enum ice_l2tsel l2tsel) +{ + struct ice_hw *hw = &vsi->back->hw; + u32 l2tsel_bit; + int i; + + if (l2tsel == ICE_L2TSEL_EXTRACT_FIRST_TAG_L2TAG2_2ND) + l2tsel_bit = 0; + else + l2tsel_bit = BIT(ICE_L2TSEL_BIT_OFFSET); + + for (i = 0; i < vsi->alloc_rxq; i++) { + u16 pfq = vsi->rxq_map[i]; + u32 qrx_context_offset; + u32 regval; + + qrx_context_offset = + QRX_CONTEXT(ICE_L2TSEL_QRX_CONTEXT_REG_IDX, pfq); + + regval = rd32(hw, qrx_context_offset); + regval &= ~BIT(ICE_L2TSEL_BIT_OFFSET); + regval |= l2tsel_bit; + wr32(hw, qrx_context_offset, regval); + } +} |