diff options
Diffstat (limited to 'drivers/net/ethernet')
64 files changed, 2888 insertions, 410 deletions
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c index 6b3d4f4c2a75..14df8cfc2946 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c @@ -20,6 +20,8 @@ #include "bnxt_ulp.h" #include "bnxt_ptp.h" #include "bnxt_coredump.h" +#include "bnxt_nvm_defs.h" +#include "bnxt_ethtool.h" static void __bnxt_fw_recover(struct bnxt *bp) { @@ -610,6 +612,63 @@ static int bnxt_dl_reload_up(struct devlink *dl, enum devlink_reload_action acti return rc; } +static bool bnxt_nvm_test(struct bnxt *bp, struct netlink_ext_ack *extack) +{ + u32 datalen; + u16 index; + u8 *buf; + + if (bnxt_find_nvram_item(bp->dev, BNX_DIR_TYPE_VPD, + BNX_DIR_ORDINAL_FIRST, BNX_DIR_EXT_NONE, + &index, NULL, &datalen) || !datalen) { + NL_SET_ERR_MSG_MOD(extack, "nvm test vpd entry error"); + return false; + } + + buf = kzalloc(datalen, GFP_KERNEL); + if (!buf) { + NL_SET_ERR_MSG_MOD(extack, "insufficient memory for nvm test"); + return false; + } + + if (bnxt_get_nvram_item(bp->dev, index, 0, datalen, buf)) { + NL_SET_ERR_MSG_MOD(extack, "nvm test vpd read error"); + goto err; + } + + if (bnxt_flash_nvram(bp->dev, BNX_DIR_TYPE_VPD, BNX_DIR_ORDINAL_FIRST, + BNX_DIR_EXT_NONE, 0, 0, buf, datalen)) { + NL_SET_ERR_MSG_MOD(extack, "nvm test vpd write error"); + goto err; + } + + return true; + +err: + kfree(buf); + return false; +} + +static bool bnxt_dl_selftest_check(struct devlink *dl, unsigned int id, + struct netlink_ext_ack *extack) +{ + return id == DEVLINK_ATTR_SELFTEST_ID_FLASH; +} + +static enum devlink_selftest_status bnxt_dl_selftest_run(struct devlink *dl, + unsigned int id, + struct netlink_ext_ack *extack) +{ + struct bnxt *bp = bnxt_get_bp_from_dl(dl); + + if (id == DEVLINK_ATTR_SELFTEST_ID_FLASH) + return bnxt_nvm_test(bp, extack) ? + DEVLINK_SELFTEST_STATUS_PASS : + DEVLINK_SELFTEST_STATUS_FAIL; + + return DEVLINK_SELFTEST_STATUS_SKIP; +} + static const struct devlink_ops bnxt_dl_ops = { #ifdef CONFIG_BNXT_SRIOV .eswitch_mode_set = bnxt_dl_eswitch_mode_set, @@ -622,6 +681,8 @@ static const struct devlink_ops bnxt_dl_ops = { .reload_limits = BIT(DEVLINK_RELOAD_LIMIT_NO_RESET), .reload_down = bnxt_dl_reload_down, .reload_up = bnxt_dl_reload_up, + .selftest_check = bnxt_dl_selftest_check, + .selftest_run = bnxt_dl_selftest_run, }; static const struct devlink_ops bnxt_vf_dl_ops; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index 7191e5d74208..87eb5362ad70 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -2176,14 +2176,14 @@ static void bnxt_print_admin_err(struct bnxt *bp) netdev_info(bp->dev, "PF does not have admin privileges to flash or reset the device\n"); } -static int bnxt_find_nvram_item(struct net_device *dev, u16 type, u16 ordinal, - u16 ext, u16 *index, u32 *item_length, - u32 *data_length); +int bnxt_find_nvram_item(struct net_device *dev, u16 type, u16 ordinal, + u16 ext, u16 *index, u32 *item_length, + u32 *data_length); -static int bnxt_flash_nvram(struct net_device *dev, u16 dir_type, - u16 dir_ordinal, u16 dir_ext, u16 dir_attr, - u32 dir_item_len, const u8 *data, - size_t data_len) +int bnxt_flash_nvram(struct net_device *dev, u16 dir_type, + u16 dir_ordinal, u16 dir_ext, u16 dir_attr, + u32 dir_item_len, const u8 *data, + size_t data_len) { struct bnxt *bp = netdev_priv(dev); struct hwrm_nvm_write_input *req; @@ -2836,8 +2836,8 @@ static int bnxt_get_nvram_directory(struct net_device *dev, u32 len, u8 *data) return rc; } -static int bnxt_get_nvram_item(struct net_device *dev, u32 index, u32 offset, - u32 length, u8 *data) +int bnxt_get_nvram_item(struct net_device *dev, u32 index, u32 offset, + u32 length, u8 *data) { struct bnxt *bp = netdev_priv(dev); int rc; @@ -2871,9 +2871,9 @@ static int bnxt_get_nvram_item(struct net_device *dev, u32 index, u32 offset, return rc; } -static int bnxt_find_nvram_item(struct net_device *dev, u16 type, u16 ordinal, - u16 ext, u16 *index, u32 *item_length, - u32 *data_length) +int bnxt_find_nvram_item(struct net_device *dev, u16 type, u16 ordinal, + u16 ext, u16 *index, u32 *item_length, + u32 *data_length) { struct hwrm_nvm_find_dir_entry_output *output; struct hwrm_nvm_find_dir_entry_input *req; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.h index a59284215e78..a8ecef8ab82c 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.h @@ -58,5 +58,17 @@ int bnxt_flash_package_from_fw_obj(struct net_device *dev, const struct firmware int bnxt_get_pkginfo(struct net_device *dev, char *ver, int size); void bnxt_ethtool_init(struct bnxt *bp); void bnxt_ethtool_free(struct bnxt *bp); +int bnxt_find_nvram_item(struct net_device *dev, u16 type, u16 ordinal, + u16 ext, u16 *index, u32 *item_length, + u32 *data_length); +int bnxt_find_nvram_item(struct net_device *dev, u16 type, u16 ordinal, + u16 ext, u16 *index, u32 *item_length, + u32 *data_length); +int bnxt_flash_nvram(struct net_device *dev, u16 dir_type, + u16 dir_ordinal, u16 dir_ext, u16 dir_attr, + u32 dir_item_len, const u8 *data, + size_t data_len); +int bnxt_get_nvram_item(struct net_device *dev, u32 index, u32 offset, + u32 length, u8 *data); #endif diff --git a/drivers/net/ethernet/fungible/funeth/funeth_rx.c b/drivers/net/ethernet/fungible/funeth/funeth_rx.c index 0f6a549b9f67..29a6c2ede43a 100644 --- a/drivers/net/ethernet/fungible/funeth/funeth_rx.c +++ b/drivers/net/ethernet/fungible/funeth/funeth_rx.c @@ -142,6 +142,7 @@ static void *fun_run_xdp(struct funeth_rxq *q, skb_frag_t *frags, void *buf_va, int ref_ok, struct funeth_txq *xdp_q) { struct bpf_prog *xdp_prog; + struct xdp_frame *xdpf; struct xdp_buff xdp; u32 act; @@ -163,7 +164,9 @@ static void *fun_run_xdp(struct funeth_rxq *q, skb_frag_t *frags, void *buf_va, case XDP_TX: if (unlikely(!ref_ok)) goto pass; - if (!fun_xdp_tx(xdp_q, xdp.data, xdp.data_end - xdp.data)) + + xdpf = xdp_convert_buff_to_frame(&xdp); + if (!xdpf || !fun_xdp_tx(xdp_q, xdpf)) goto xdp_error; FUN_QSTAT_INC(q, xdp_tx); q->xdp_flush |= FUN_XDP_FLUSH_TX; diff --git a/drivers/net/ethernet/fungible/funeth/funeth_tx.c b/drivers/net/ethernet/fungible/funeth/funeth_tx.c index a97e3af00cb9..54bdeb65a2bd 100644 --- a/drivers/net/ethernet/fungible/funeth/funeth_tx.c +++ b/drivers/net/ethernet/fungible/funeth/funeth_tx.c @@ -487,7 +487,7 @@ static unsigned int fun_xdpq_clean(struct funeth_txq *q, unsigned int budget) do { fun_xdp_unmap(q, reclaim_idx); - page_frag_free(q->info[reclaim_idx].vaddr); + xdp_return_frame(q->info[reclaim_idx].xdpf); trace_funeth_tx_free(q, reclaim_idx, 1, head); @@ -500,11 +500,11 @@ static unsigned int fun_xdpq_clean(struct funeth_txq *q, unsigned int budget) return npkts; } -bool fun_xdp_tx(struct funeth_txq *q, void *data, unsigned int len) +bool fun_xdp_tx(struct funeth_txq *q, struct xdp_frame *xdpf) { struct fun_eth_tx_req *req; struct fun_dataop_gl *gle; - unsigned int idx; + unsigned int idx, len; dma_addr_t dma; if (fun_txq_avail(q) < FUN_XDP_CLEAN_THRES) @@ -515,7 +515,8 @@ bool fun_xdp_tx(struct funeth_txq *q, void *data, unsigned int len) return false; } - dma = dma_map_single(q->dma_dev, data, len, DMA_TO_DEVICE); + len = xdpf->len; + dma = dma_map_single(q->dma_dev, xdpf->data, len, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(q->dma_dev, dma))) { FUN_QSTAT_INC(q, tx_map_err); return false; @@ -535,7 +536,7 @@ bool fun_xdp_tx(struct funeth_txq *q, void *data, unsigned int len) gle = (struct fun_dataop_gl *)req->dataop.imm; fun_dataop_gl_init(gle, 0, 0, len, dma); - q->info[idx].vaddr = data; + q->info[idx].xdpf = xdpf; u64_stats_update_begin(&q->syncp); q->stats.tx_bytes += len; @@ -566,12 +567,9 @@ int fun_xdp_xmit_frames(struct net_device *dev, int n, if (unlikely(q_idx >= fp->num_xdpqs)) return -ENXIO; - for (q = xdpqs[q_idx], i = 0; i < n; i++) { - const struct xdp_frame *xdpf = frames[i]; - - if (!fun_xdp_tx(q, xdpf->data, xdpf->len)) + for (q = xdpqs[q_idx], i = 0; i < n; i++) + if (!fun_xdp_tx(q, frames[i])) break; - } if (unlikely(flags & XDP_XMIT_FLUSH)) fun_txq_wr_db(q); @@ -598,7 +596,7 @@ static void fun_xdpq_purge(struct funeth_txq *q) unsigned int idx = q->cons_cnt & q->mask; fun_xdp_unmap(q, idx); - page_frag_free(q->info[idx].vaddr); + xdp_return_frame(q->info[idx].xdpf); q->cons_cnt++; } } diff --git a/drivers/net/ethernet/fungible/funeth/funeth_txrx.h b/drivers/net/ethernet/fungible/funeth/funeth_txrx.h index 1711f82cad71..53b7e95213a8 100644 --- a/drivers/net/ethernet/fungible/funeth/funeth_txrx.h +++ b/drivers/net/ethernet/fungible/funeth/funeth_txrx.h @@ -96,8 +96,8 @@ struct funeth_txq_stats { /* per Tx queue SW counters */ struct funeth_tx_info { /* per Tx descriptor state */ union { - struct sk_buff *skb; /* associated packet */ - void *vaddr; /* start address for XDP */ + struct sk_buff *skb; /* associated packet (sk_buff path) */ + struct xdp_frame *xdpf; /* associated XDP frame (XDP path) */ }; }; @@ -246,7 +246,7 @@ static inline int fun_irq_node(const struct fun_irq *p) int fun_rxq_napi_poll(struct napi_struct *napi, int budget); int fun_txq_napi_poll(struct napi_struct *napi, int budget); netdev_tx_t fun_start_xmit(struct sk_buff *skb, struct net_device *netdev); -bool fun_xdp_tx(struct funeth_txq *q, void *data, unsigned int len); +bool fun_xdp_tx(struct funeth_txq *q, struct xdp_frame *xdpf); int fun_xdp_xmit_frames(struct net_device *dev, int n, struct xdp_frame **frames, u32 flags); diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index fb9f476fb33c..b36bf9c3e1e4 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -2033,11 +2033,15 @@ static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, * non-zero req_queue_pairs says that user requested a new * queue count via ethtool's set_channels, so use this * value for queues distribution across traffic classes + * We need at least one queue pair for the interface + * to be usable as we see in else statement. */ if (vsi->req_queue_pairs > 0) vsi->num_queue_pairs = vsi->req_queue_pairs; else if (pf->flags & I40E_FLAG_MSIX_ENABLED) vsi->num_queue_pairs = pf->num_lan_msix; + else + vsi->num_queue_pairs = 1; } /* Number of queues per enabled TC */ diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c index 340dc5a0832c..a6fff8ebaf9d 100644 --- a/drivers/net/ethernet/intel/ice/ice_ethtool.c +++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c @@ -658,7 +658,8 @@ static int ice_lbtest_receive_frames(struct ice_rx_ring *rx_ring) rx_desc = ICE_RX_DESC(rx_ring, i); if (!(rx_desc->wb.status_error0 & - cpu_to_le16(ICE_TX_DESC_CMD_EOP | ICE_TX_DESC_CMD_RS))) + (cpu_to_le16(BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S)) | + cpu_to_le16(BIT(ICE_RX_FLEX_DESC_STATUS0_EOF_S))))) continue; rx_buf = &rx_ring->rx_buf[i]; diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index cbbbb6788baa..9a3b14d42835 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -4657,6 +4657,8 @@ ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) ice_set_safe_mode_caps(hw); } + hw->ucast_shared = true; + err = ice_init_pf(pf); if (err) { dev_err(dev, "ice_init_pf failed: %d\n", err); @@ -6033,10 +6035,12 @@ int ice_vsi_cfg(struct ice_vsi *vsi) if (vsi->netdev) { ice_set_rx_mode(vsi->netdev); - err = ice_vsi_vlan_setup(vsi); + if (vsi->type != ICE_VSI_LB) { + err = ice_vsi_vlan_setup(vsi); - if (err) - return err; + if (err) + return err; + } } ice_vsi_cfg_dcb_rings(vsi); diff --git a/drivers/net/ethernet/intel/ice/ice_sriov.c b/drivers/net/ethernet/intel/ice/ice_sriov.c index 86093b2511d8..3ba1408c56a9 100644 --- a/drivers/net/ethernet/intel/ice/ice_sriov.c +++ b/drivers/net/ethernet/intel/ice/ice_sriov.c @@ -1310,39 +1310,6 @@ out_put_vf: } /** - * ice_unicast_mac_exists - check if the unicast MAC exists on the PF's switch - * @pf: PF used to reference the switch's rules - * @umac: unicast MAC to compare against existing switch rules - * - * Return true on the first/any match, else return false - */ -static bool ice_unicast_mac_exists(struct ice_pf *pf, u8 *umac) -{ - struct ice_sw_recipe *mac_recipe_list = - &pf->hw.switch_info->recp_list[ICE_SW_LKUP_MAC]; - struct ice_fltr_mgmt_list_entry *list_itr; - struct list_head *rule_head; - struct mutex *rule_lock; /* protect MAC filter list access */ - - rule_head = &mac_recipe_list->filt_rules; - rule_lock = &mac_recipe_list->filt_rule_lock; - - mutex_lock(rule_lock); - list_for_each_entry(list_itr, rule_head, list_entry) { - u8 *existing_mac = &list_itr->fltr_info.l_data.mac.mac_addr[0]; - - if (ether_addr_equal(existing_mac, umac)) { - mutex_unlock(rule_lock); - return true; - } - } - - mutex_unlock(rule_lock); - - return false; -} - -/** * ice_set_vf_mac * @netdev: network interface device structure * @vf_id: VF identifier @@ -1376,13 +1343,6 @@ int ice_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac) if (ret) goto out_put_vf; - if (ice_unicast_mac_exists(pf, mac)) { - netdev_err(netdev, "Unicast MAC %pM already exists on this PF. Preventing setting VF %u unicast MAC address to %pM\n", - mac, vf_id, mac); - ret = -EINVAL; - goto out_put_vf; - } - mutex_lock(&vf->cfg_lock); /* VF is notified of its new MAC via the PF's response to the diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c index 3f8b7274ed2f..836dce840712 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx.c @@ -1751,11 +1751,13 @@ int ice_tx_csum(struct ice_tx_buf *first, struct ice_tx_offload_params *off) protocol = vlan_get_protocol(skb); - if (eth_p_mpls(protocol)) + if (eth_p_mpls(protocol)) { ip.hdr = skb_inner_network_header(skb); - else + l4.hdr = skb_checksum_start(skb); + } else { ip.hdr = skb_network_header(skb); - l4.hdr = skb_checksum_start(skb); + l4.hdr = skb_transport_header(skb); + } /* compute outer L2 header size */ l2_len = ip.hdr - skb->data; diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl.c b/drivers/net/ethernet/intel/ice/ice_virtchnl.c index d46786cdc162..094e3c97a1ea 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl.c +++ b/drivers/net/ethernet/intel/ice/ice_virtchnl.c @@ -2972,7 +2972,8 @@ ice_vc_validate_add_vlan_filter_list(struct ice_vsi *vsi, struct virtchnl_vlan_filtering_caps *vfc, struct virtchnl_vlan_filter_list_v2 *vfl) { - u16 num_requested_filters = vsi->num_vlan + vfl->num_elements; + u16 num_requested_filters = ice_vsi_num_non_zero_vlans(vsi) + + vfl->num_elements; if (num_requested_filters > vfc->max_filters) return false; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c index 28b19945d716..e64318c110fd 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c @@ -28,6 +28,9 @@ #define MAX_RATE_EXPONENT 0x0FULL #define MAX_RATE_MANTISSA 0xFFULL +#define CN10K_MAX_BURST_MANTISSA 0x7FFFULL +#define CN10K_MAX_BURST_SIZE 8453888ULL + /* Bitfields in NIX_TLX_PIR register */ #define TLX_RATE_MANTISSA GENMASK_ULL(8, 1) #define TLX_RATE_EXPONENT GENMASK_ULL(12, 9) @@ -35,6 +38,9 @@ #define TLX_BURST_MANTISSA GENMASK_ULL(36, 29) #define TLX_BURST_EXPONENT GENMASK_ULL(40, 37) +#define CN10K_TLX_BURST_MANTISSA GENMASK_ULL(43, 29) +#define CN10K_TLX_BURST_EXPONENT GENMASK_ULL(47, 44) + struct otx2_tc_flow_stats { u64 bytes; u64 pkts; @@ -77,33 +83,42 @@ int otx2_tc_alloc_ent_bitmap(struct otx2_nic *nic) } EXPORT_SYMBOL(otx2_tc_alloc_ent_bitmap); -static void otx2_get_egress_burst_cfg(u32 burst, u32 *burst_exp, - u32 *burst_mantissa) +static void otx2_get_egress_burst_cfg(struct otx2_nic *nic, u32 burst, + u32 *burst_exp, u32 *burst_mantissa) { + int max_burst, max_mantissa; unsigned int tmp; + if (is_dev_otx2(nic->pdev)) { + max_burst = MAX_BURST_SIZE; + max_mantissa = MAX_BURST_MANTISSA; + } else { + max_burst = CN10K_MAX_BURST_SIZE; + max_mantissa = CN10K_MAX_BURST_MANTISSA; + } + /* Burst is calculated as * ((256 + BURST_MANTISSA) << (1 + BURST_EXPONENT)) / 256 * Max supported burst size is 130,816 bytes. */ - burst = min_t(u32, burst, MAX_BURST_SIZE); + burst = min_t(u32, burst, max_burst); if (burst) { *burst_exp = ilog2(burst) ? ilog2(burst) - 1 : 0; tmp = burst - rounddown_pow_of_two(burst); - if (burst < MAX_BURST_MANTISSA) + if (burst < max_mantissa) *burst_mantissa = tmp * 2; else *burst_mantissa = tmp / (1ULL << (*burst_exp - 7)); } else { *burst_exp = MAX_BURST_EXPONENT; - *burst_mantissa = MAX_BURST_MANTISSA; + *burst_mantissa = max_mantissa; } } -static void otx2_get_egress_rate_cfg(u32 maxrate, u32 *exp, +static void otx2_get_egress_rate_cfg(u64 maxrate, u32 *exp, u32 *mantissa, u32 *div_exp) { - unsigned int tmp; + u64 tmp; /* Rate calculation by hardware * @@ -132,21 +147,44 @@ static void otx2_get_egress_rate_cfg(u32 maxrate, u32 *exp, } } -static int otx2_set_matchall_egress_rate(struct otx2_nic *nic, u32 burst, u32 maxrate) +static u64 otx2_get_txschq_rate_regval(struct otx2_nic *nic, + u64 maxrate, u32 burst) { - struct otx2_hw *hw = &nic->hw; - struct nix_txschq_config *req; u32 burst_exp, burst_mantissa; u32 exp, mantissa, div_exp; + u64 regval = 0; + + /* Get exponent and mantissa values from the desired rate */ + otx2_get_egress_burst_cfg(nic, burst, &burst_exp, &burst_mantissa); + otx2_get_egress_rate_cfg(maxrate, &exp, &mantissa, &div_exp); + + if (is_dev_otx2(nic->pdev)) { + regval = FIELD_PREP(TLX_BURST_EXPONENT, (u64)burst_exp) | + FIELD_PREP(TLX_BURST_MANTISSA, (u64)burst_mantissa) | + FIELD_PREP(TLX_RATE_DIVIDER_EXPONENT, div_exp) | + FIELD_PREP(TLX_RATE_EXPONENT, exp) | + FIELD_PREP(TLX_RATE_MANTISSA, mantissa) | BIT_ULL(0); + } else { + regval = FIELD_PREP(CN10K_TLX_BURST_EXPONENT, (u64)burst_exp) | + FIELD_PREP(CN10K_TLX_BURST_MANTISSA, (u64)burst_mantissa) | + FIELD_PREP(TLX_RATE_DIVIDER_EXPONENT, div_exp) | + FIELD_PREP(TLX_RATE_EXPONENT, exp) | + FIELD_PREP(TLX_RATE_MANTISSA, mantissa) | BIT_ULL(0); + } + + return regval; +} + +static int otx2_set_matchall_egress_rate(struct otx2_nic *nic, + u32 burst, u64 maxrate) +{ + struct otx2_hw *hw = &nic->hw; + struct nix_txschq_config *req; int txschq, err; /* All SQs share the same TL4, so pick the first scheduler */ txschq = hw->txschq_list[NIX_TXSCH_LVL_TL4][0]; - /* Get exponent and mantissa values from the desired rate */ - otx2_get_egress_burst_cfg(burst, &burst_exp, &burst_mantissa); - otx2_get_egress_rate_cfg(maxrate, &exp, &mantissa, &div_exp); - mutex_lock(&nic->mbox.lock); req = otx2_mbox_alloc_msg_nix_txschq_cfg(&nic->mbox); if (!req) { @@ -157,11 +195,7 @@ static int otx2_set_matchall_egress_rate(struct otx2_nic *nic, u32 burst, u32 ma req->lvl = NIX_TXSCH_LVL_TL4; req->num_regs = 1; req->reg[0] = NIX_AF_TL4X_PIR(txschq); - req->regval[0] = FIELD_PREP(TLX_BURST_EXPONENT, burst_exp) | - FIELD_PREP(TLX_BURST_MANTISSA, burst_mantissa) | - FIELD_PREP(TLX_RATE_DIVIDER_EXPONENT, div_exp) | - FIELD_PREP(TLX_RATE_EXPONENT, exp) | - FIELD_PREP(TLX_RATE_MANTISSA, mantissa) | BIT_ULL(0); + req->regval[0] = otx2_get_txschq_rate_regval(nic, maxrate, burst); err = otx2_sync_mbox_msg(&nic->mbox); mutex_unlock(&nic->mbox.lock); @@ -230,7 +264,7 @@ static int otx2_tc_egress_matchall_install(struct otx2_nic *nic, struct netlink_ext_ack *extack = cls->common.extack; struct flow_action *actions = &cls->rule->action; struct flow_action_entry *entry; - u32 rate; + u64 rate; int err; err = otx2_tc_validate_flow(nic, actions, extack); @@ -256,7 +290,7 @@ static int otx2_tc_egress_matchall_install(struct otx2_nic *nic, } /* Convert bytes per second to Mbps */ rate = entry->police.rate_bytes_ps * 8; - rate = max_t(u32, rate / 1000000, 1); + rate = max_t(u64, rate / 1000000, 1); err = otx2_set_matchall_egress_rate(nic, entry->police.burst, rate); if (err) return err; @@ -614,21 +648,27 @@ static int otx2_tc_prepare_flow(struct otx2_nic *nic, struct otx2_tc_flow *node, flow_spec->dport = match.key->dst; flow_mask->dport = match.mask->dst; - if (ip_proto == IPPROTO_UDP) - req->features |= BIT_ULL(NPC_DPORT_UDP); - else if (ip_proto == IPPROTO_TCP) - req->features |= BIT_ULL(NPC_DPORT_TCP); - else if (ip_proto == IPPROTO_SCTP) - req->features |= BIT_ULL(NPC_DPORT_SCTP); + + if (flow_mask->dport) { + if (ip_proto == IPPROTO_UDP) + req->features |= BIT_ULL(NPC_DPORT_UDP); + else if (ip_proto == IPPROTO_TCP) + req->features |= BIT_ULL(NPC_DPORT_TCP); + else if (ip_proto == IPPROTO_SCTP) + req->features |= BIT_ULL(NPC_DPORT_SCTP); + } flow_spec->sport = match.key->src; flow_mask->sport = match.mask->src; - if (ip_proto == IPPROTO_UDP) - req->features |= BIT_ULL(NPC_SPORT_UDP); - else if (ip_proto == IPPROTO_TCP) - req->features |= BIT_ULL(NPC_SPORT_TCP); - else if (ip_proto == IPPROTO_SCTP) - req->features |= BIT_ULL(NPC_SPORT_SCTP); + + if (flow_mask->sport) { + if (ip_proto == IPPROTO_UDP) + req->features |= BIT_ULL(NPC_SPORT_UDP); + else if (ip_proto == IPPROTO_TCP) + req->features |= BIT_ULL(NPC_SPORT_TCP); + else if (ip_proto == IPPROTO_SCTP) + req->features |= BIT_ULL(NPC_SPORT_SCTP); + } } return otx2_tc_parse_actions(nic, &rule->action, req, f, node); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_main.c b/drivers/net/ethernet/marvell/prestera/prestera_main.c index c267ca1ccdba..4b64bda3f9c2 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_main.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_main.c @@ -847,7 +847,7 @@ static void prestera_event_handlers_unregister(struct prestera_switch *sw) static int prestera_switch_set_base_mac_addr(struct prestera_switch *sw) { struct device_node *base_mac_np; - int ret; + int ret = 0; if (sw->np) { base_mac_np = of_parse_phandle(sw->np, "base-mac-provider", 0); diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index c370d6589596..d9426b01f462 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -1001,7 +1001,7 @@ static int txd_to_idx(struct mtk_tx_ring *ring, void *dma, u32 txd_size) } static void mtk_tx_unmap(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf, - bool napi) + struct xdp_frame_bulk *bq, bool napi) { if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) { if (tx_buf->flags & MTK_TX_FLAGS_SINGLE0) { @@ -1031,23 +1031,24 @@ static void mtk_tx_unmap(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf, } } - if (tx_buf->type == MTK_TYPE_SKB) { - if (tx_buf->data && - tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) { + if (tx_buf->data && tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) { + if (tx_buf->type == MTK_TYPE_SKB) { struct sk_buff *skb = tx_buf->data; if (napi) napi_consume_skb(skb, napi); else dev_kfree_skb_any(skb); - } - } else if (tx_buf->data) { - struct xdp_frame *xdpf = tx_buf->data; + } else { + struct xdp_frame *xdpf = tx_buf->data; - if (napi && tx_buf->type == MTK_TYPE_XDP_TX) - xdp_return_frame_rx_napi(xdpf); - else - xdp_return_frame(xdpf); + if (napi && tx_buf->type == MTK_TYPE_XDP_TX) + xdp_return_frame_rx_napi(xdpf); + else if (bq) + xdp_return_frame_bulk(xdpf, bq); + else + xdp_return_frame(xdpf); + } } tx_buf->flags = 0; tx_buf->data = NULL; @@ -1297,7 +1298,7 @@ err_dma: tx_buf = mtk_desc_to_tx_buf(ring, itxd, soc->txrx.txd_size); /* unmap dma */ - mtk_tx_unmap(eth, tx_buf, false); + mtk_tx_unmap(eth, tx_buf, NULL, false); itxd->txd3 = TX_DMA_LS0 | TX_DMA_OWNER_CPU; if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA)) @@ -1523,68 +1524,112 @@ static void mtk_rx_put_buff(struct mtk_rx_ring *ring, void *data, bool napi) skb_free_frag(data); } +static int mtk_xdp_frame_map(struct mtk_eth *eth, struct net_device *dev, + struct mtk_tx_dma_desc_info *txd_info, + struct mtk_tx_dma *txd, struct mtk_tx_buf *tx_buf, + void *data, u16 headroom, int index, bool dma_map) +{ + struct mtk_tx_ring *ring = ð->tx_ring; + struct mtk_mac *mac = netdev_priv(dev); + struct mtk_tx_dma *txd_pdma; + + if (dma_map) { /* ndo_xdp_xmit */ + txd_info->addr = dma_map_single(eth->dma_dev, data, + txd_info->size, DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(eth->dma_dev, txd_info->addr))) + return -ENOMEM; + + tx_buf->flags |= MTK_TX_FLAGS_SINGLE0; + } else { + struct page *page = virt_to_head_page(data); + + txd_info->addr = page_pool_get_dma_addr(page) + + sizeof(struct xdp_frame) + headroom; + dma_sync_single_for_device(eth->dma_dev, txd_info->addr, + txd_info->size, DMA_BIDIRECTIONAL); + } + mtk_tx_set_dma_desc(dev, txd, txd_info); + + tx_buf->flags |= !mac->id ? MTK_TX_FLAGS_FPORT0 : MTK_TX_FLAGS_FPORT1; + tx_buf->type = dma_map ? MTK_TYPE_XDP_NDO : MTK_TYPE_XDP_TX; + tx_buf->data = (void *)MTK_DMA_DUMMY_DESC; + + txd_pdma = qdma_to_pdma(ring, txd); + setup_tx_buf(eth, tx_buf, txd_pdma, txd_info->addr, txd_info->size, + index); + + return 0; +} + static int mtk_xdp_submit_frame(struct mtk_eth *eth, struct xdp_frame *xdpf, struct net_device *dev, bool dma_map) { + struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf); const struct mtk_soc_data *soc = eth->soc; struct mtk_tx_ring *ring = ð->tx_ring; struct mtk_tx_dma_desc_info txd_info = { .size = xdpf->len, .first = true, - .last = true, + .last = !xdp_frame_has_frags(xdpf), }; - struct mtk_mac *mac = netdev_priv(dev); - struct mtk_tx_dma *txd, *txd_pdma; - int err = 0, index = 0, n_desc = 1; - struct mtk_tx_buf *tx_buf; + int err, index = 0, n_desc = 1, nr_frags; + struct mtk_tx_dma *htxd, *txd, *txd_pdma; + struct mtk_tx_buf *htx_buf, *tx_buf; + void *data = xdpf->data; if (unlikely(test_bit(MTK_RESETTING, ð->state))) return -EBUSY; - if (unlikely(atomic_read(&ring->free_count) <= 1)) + nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0; + if (unlikely(atomic_read(&ring->free_count) <= 1 + nr_frags)) return -EBUSY; spin_lock(ð->page_lock); txd = ring->next_free; if (txd == ring->last_free) { - err = -ENOMEM; - goto out; + spin_unlock(ð->page_lock); + return -ENOMEM; } + htxd = txd; tx_buf = mtk_desc_to_tx_buf(ring, txd, soc->txrx.txd_size); memset(tx_buf, 0, sizeof(*tx_buf)); + htx_buf = tx_buf; - if (dma_map) { /* ndo_xdp_xmit */ - txd_info.addr = dma_map_single(eth->dma_dev, xdpf->data, - txd_info.size, DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(eth->dma_dev, txd_info.addr))) { - err = -ENOMEM; - goto out; - } - tx_buf->flags |= MTK_TX_FLAGS_SINGLE0; - } else { - struct page *page = virt_to_head_page(xdpf->data); + for (;;) { + err = mtk_xdp_frame_map(eth, dev, &txd_info, txd, tx_buf, + data, xdpf->headroom, index, dma_map); + if (err < 0) + goto unmap; - txd_info.addr = page_pool_get_dma_addr(page) + - sizeof(*xdpf) + xdpf->headroom; - dma_sync_single_for_device(eth->dma_dev, txd_info.addr, - txd_info.size, - DMA_BIDIRECTIONAL); - } - mtk_tx_set_dma_desc(dev, txd, &txd_info); + if (txd_info.last) + break; - tx_buf->flags |= !mac->id ? MTK_TX_FLAGS_FPORT0 : MTK_TX_FLAGS_FPORT1; + if (MTK_HAS_CAPS(soc->caps, MTK_QDMA) || (index & 0x1)) { + txd = mtk_qdma_phys_to_virt(ring, txd->txd2); + txd_pdma = qdma_to_pdma(ring, txd); + if (txd == ring->last_free) + goto unmap; - txd_pdma = qdma_to_pdma(ring, txd); - setup_tx_buf(eth, tx_buf, txd_pdma, txd_info.addr, txd_info.size, - index++); + tx_buf = mtk_desc_to_tx_buf(ring, txd, + soc->txrx.txd_size); + memset(tx_buf, 0, sizeof(*tx_buf)); + n_desc++; + } + + memset(&txd_info, 0, sizeof(struct mtk_tx_dma_desc_info)); + txd_info.size = skb_frag_size(&sinfo->frags[index]); + txd_info.last = index + 1 == nr_frags; + data = skb_frag_address(&sinfo->frags[index]); + index++; + } /* store xdpf for cleanup */ - tx_buf->type = dma_map ? MTK_TYPE_XDP_NDO : MTK_TYPE_XDP_TX; - tx_buf->data = xdpf; + htx_buf->data = xdpf; if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA)) { + txd_pdma = qdma_to_pdma(ring, txd); if (index & 1) txd_pdma->txd2 |= TX_DMA_LS0; else @@ -1608,7 +1653,24 @@ static int mtk_xdp_submit_frame(struct mtk_eth *eth, struct xdp_frame *xdpf, mtk_w32(eth, NEXT_DESP_IDX(idx, ring->dma_size), MT7628_TX_CTX_IDX0); } -out: + + spin_unlock(ð->page_lock); + + return 0; + +unmap: + while (htxd != txd) { + txd_pdma = qdma_to_pdma(ring, htxd); + tx_buf = mtk_desc_to_tx_buf(ring, htxd, soc->txrx.txd_size); + mtk_tx_unmap(eth, tx_buf, NULL, false); + + htxd->txd3 = TX_DMA_LS0 | TX_DMA_OWNER_CPU; + if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA)) + txd_pdma->txd2 = TX_DMA_DESP2_DEF; + + htxd = mtk_qdma_phys_to_virt(ring, htxd->txd2); + } + spin_unlock(ð->page_lock); return err; @@ -1913,6 +1975,7 @@ static int mtk_poll_tx_qdma(struct mtk_eth *eth, int budget, const struct mtk_reg_map *reg_map = eth->soc->reg_map; struct mtk_tx_ring *ring = ð->tx_ring; struct mtk_tx_buf *tx_buf; + struct xdp_frame_bulk bq; struct mtk_tx_dma *desc; u32 cpu, dma; @@ -1920,6 +1983,7 @@ static int mtk_poll_tx_qdma(struct mtk_eth *eth, int budget, dma = mtk_r32(eth, reg_map->qdma.drx_ptr); desc = mtk_qdma_phys_to_virt(ring, cpu); + xdp_frame_bulk_init(&bq); while ((cpu != dma) && budget) { u32 next_cpu = desc->txd2; @@ -1937,25 +2001,23 @@ static int mtk_poll_tx_qdma(struct mtk_eth *eth, int budget, if (!tx_buf->data) break; - if (tx_buf->type == MTK_TYPE_SKB && - tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) { - struct sk_buff *skb = tx_buf->data; + if (tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) { + if (tx_buf->type == MTK_TYPE_SKB) { + struct sk_buff *skb = tx_buf->data; - bytes[mac] += skb->len; - done[mac]++; - budget--; - } else if (tx_buf->type == MTK_TYPE_XDP_TX || - tx_buf->type == MTK_TYPE_XDP_NDO) { + bytes[mac] += skb->len; + done[mac]++; + } budget--; } - - mtk_tx_unmap(eth, tx_buf, true); + mtk_tx_unmap(eth, tx_buf, &bq, true); ring->last_free = desc; atomic_inc(&ring->free_count); cpu = next_cpu; } + xdp_flush_frame_bulk(&bq); ring->last_free_ptr = cpu; mtk_w32(eth, cpu, reg_map->qdma.crx_ptr); @@ -1968,29 +2030,29 @@ static int mtk_poll_tx_pdma(struct mtk_eth *eth, int budget, { struct mtk_tx_ring *ring = ð->tx_ring; struct mtk_tx_buf *tx_buf; + struct xdp_frame_bulk bq; struct mtk_tx_dma *desc; u32 cpu, dma; cpu = ring->cpu_idx; dma = mtk_r32(eth, MT7628_TX_DTX_IDX0); + xdp_frame_bulk_init(&bq); while ((cpu != dma) && budget) { tx_buf = &ring->buf[cpu]; if (!tx_buf->data) break; - if (tx_buf->type == MTK_TYPE_SKB && - tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) { - struct sk_buff *skb = tx_buf->data; - bytes[0] += skb->len; - done[0]++; - budget--; - } else if (tx_buf->type == MTK_TYPE_XDP_TX || - tx_buf->type == MTK_TYPE_XDP_NDO) { + if (tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) { + if (tx_buf->type == MTK_TYPE_SKB) { + struct sk_buff *skb = tx_buf->data; + + bytes[0] += skb->len; + done[0]++; + } budget--; } - - mtk_tx_unmap(eth, tx_buf, true); + mtk_tx_unmap(eth, tx_buf, &bq, true); desc = ring->dma + cpu * eth->soc->txrx.txd_size; ring->last_free = desc; @@ -1998,6 +2060,7 @@ static int mtk_poll_tx_pdma(struct mtk_eth *eth, int budget, cpu = NEXT_DESP_IDX(cpu, ring->dma_size); } + xdp_flush_frame_bulk(&bq); ring->cpu_idx = cpu; @@ -2207,7 +2270,7 @@ static void mtk_tx_clean(struct mtk_eth *eth) if (ring->buf) { for (i = 0; i < MTK_DMA_SIZE; i++) - mtk_tx_unmap(eth, &ring->buf[i], false); + mtk_tx_unmap(eth, &ring->buf[i], NULL, false); kfree(ring->buf); ring->buf = NULL; } diff --git a/drivers/net/ethernet/mellanox/mlx4/catas.c b/drivers/net/ethernet/mellanox/mlx4/catas.c index 5b11557f1ae4..0eb7b83637d8 100644 --- a/drivers/net/ethernet/mellanox/mlx4/catas.c +++ b/drivers/net/ethernet/mellanox/mlx4/catas.c @@ -204,9 +204,13 @@ out: static void mlx4_handle_error_state(struct mlx4_dev_persistent *persist) { + struct mlx4_dev *dev = persist->dev; + struct devlink *devlink; int err = 0; mlx4_enter_error_state(persist); + devlink = priv_to_devlink(mlx4_priv(dev)); + devl_lock(devlink); mutex_lock(&persist->interface_state_mutex); if (persist->interface_state & MLX4_INTERFACE_STATE_UP && !(persist->interface_state & MLX4_INTERFACE_STATE_DELETION)) { @@ -215,6 +219,7 @@ static void mlx4_handle_error_state(struct mlx4_dev_persistent *persist) err); } mutex_unlock(&persist->interface_state_mutex); + devl_unlock(devlink); } static void dump_err_buf(struct mlx4_dev *dev) diff --git a/drivers/net/ethernet/mellanox/mlx4/crdump.c b/drivers/net/ethernet/mellanox/mlx4/crdump.c index ac5468b77488..82a07a31cde7 100644 --- a/drivers/net/ethernet/mellanox/mlx4/crdump.c +++ b/drivers/net/ethernet/mellanox/mlx4/crdump.c @@ -226,10 +226,10 @@ int mlx4_crdump_init(struct mlx4_dev *dev) /* Create cr-space region */ crdump->region_crspace = - devlink_region_create(devlink, - ®ion_cr_space_ops, - MAX_NUM_OF_DUMPS_TO_STORE, - pci_resource_len(pdev, 0)); + devl_region_create(devlink, + ®ion_cr_space_ops, + MAX_NUM_OF_DUMPS_TO_STORE, + pci_resource_len(pdev, 0)); if (IS_ERR(crdump->region_crspace)) mlx4_warn(dev, "crdump: create devlink region %s err %ld\n", region_cr_space_str, @@ -237,10 +237,10 @@ int mlx4_crdump_init(struct mlx4_dev *dev) /* Create fw-health region */ crdump->region_fw_health = - devlink_region_create(devlink, - ®ion_fw_health_ops, - MAX_NUM_OF_DUMPS_TO_STORE, - HEALTH_BUFFER_SIZE); + devl_region_create(devlink, + ®ion_fw_health_ops, + MAX_NUM_OF_DUMPS_TO_STORE, + HEALTH_BUFFER_SIZE); if (IS_ERR(crdump->region_fw_health)) mlx4_warn(dev, "crdump: create devlink region %s err %ld\n", region_fw_health_str, @@ -253,6 +253,6 @@ void mlx4_crdump_end(struct mlx4_dev *dev) { struct mlx4_fw_crdump *crdump = &dev->persist->crdump; - devlink_region_destroy(crdump->region_fw_health); - devlink_region_destroy(crdump->region_crspace); + devl_region_destroy(crdump->region_fw_health); + devl_region_destroy(crdump->region_crspace); } diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c index b187c210d4d6..2c764d1d897d 100644 --- a/drivers/net/ethernet/mellanox/mlx4/main.c +++ b/drivers/net/ethernet/mellanox/mlx4/main.c @@ -3033,7 +3033,7 @@ static int mlx4_init_port_info(struct mlx4_dev *dev, int port) struct mlx4_port_info *info = &mlx4_priv(dev)->port[port]; int err; - err = devlink_port_register(devlink, &info->devlink_port, port); + err = devl_port_register(devlink, &info->devlink_port, port); if (err) return err; @@ -3071,7 +3071,7 @@ static int mlx4_init_port_info(struct mlx4_dev *dev, int port) err = device_create_file(&dev->persist->pdev->dev, &info->port_attr); if (err) { mlx4_err(dev, "Failed to create file for port %d\n", port); - devlink_port_unregister(&info->devlink_port); + devl_port_unregister(&info->devlink_port); info->port = -1; return err; } @@ -3093,7 +3093,7 @@ static int mlx4_init_port_info(struct mlx4_dev *dev, int port) mlx4_err(dev, "Failed to create mtu file for port %d\n", port); device_remove_file(&info->dev->persist->pdev->dev, &info->port_attr); - devlink_port_unregister(&info->devlink_port); + devl_port_unregister(&info->devlink_port); info->port = -1; return err; } @@ -3109,7 +3109,7 @@ static void mlx4_cleanup_port_info(struct mlx4_port_info *info) device_remove_file(&info->dev->persist->pdev->dev, &info->port_attr); device_remove_file(&info->dev->persist->pdev->dev, &info->port_mtu_attr); - devlink_port_unregister(&info->devlink_port); + devl_port_unregister(&info->devlink_port); #ifdef CONFIG_RFS_ACCEL free_irq_cpu_rmap(info->rmap); @@ -3333,6 +3333,7 @@ static int mlx4_load_one(struct pci_dev *pdev, int pci_dev_data, int total_vfs, int *nvfs, struct mlx4_priv *priv, int reset_flow) { + struct devlink *devlink = priv_to_devlink(priv); struct mlx4_dev *dev; unsigned sum = 0; int err; @@ -3341,6 +3342,7 @@ static int mlx4_load_one(struct pci_dev *pdev, int pci_dev_data, struct mlx4_dev_cap *dev_cap = NULL; int existing_vfs = 0; + devl_assert_locked(devlink); dev = &priv->dev; INIT_LIST_HEAD(&priv->ctx_list); @@ -3956,9 +3958,11 @@ static int mlx4_devlink_reload_down(struct devlink *devlink, bool netns_change, NL_SET_ERR_MSG_MOD(extack, "Namespace change is not supported"); return -EOPNOTSUPP; } + devl_lock(devlink); if (persist->num_vfs) mlx4_warn(persist->dev, "Reload performed on PF, will cause reset on operating Virtual Functions\n"); mlx4_restart_one_down(persist->pdev); + devl_unlock(devlink); return 0; } @@ -3971,8 +3975,10 @@ static int mlx4_devlink_reload_up(struct devlink *devlink, enum devlink_reload_a struct mlx4_dev_persistent *persist = dev->persist; int err; + devl_lock(devlink); *actions_performed = BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT); err = mlx4_restart_one_up(persist->pdev, true, devlink); + devl_unlock(devlink); if (err) mlx4_err(persist->dev, "mlx4_restart_one_up failed, ret=%d\n", err); @@ -3999,6 +4005,7 @@ static int mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id) devlink = devlink_alloc(&mlx4_devlink_ops, sizeof(*priv), &pdev->dev); if (!devlink) return -ENOMEM; + devl_lock(devlink); priv = devlink_priv(devlink); dev = &priv->dev; @@ -4026,6 +4033,7 @@ static int mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id) pci_save_state(pdev); devlink_set_features(devlink, DEVLINK_F_RELOAD); + devl_unlock(devlink); devlink_register(devlink); return 0; @@ -4035,6 +4043,7 @@ err_params_unregister: err_devlink_unregister: kfree(dev->persist); err_devlink_free: + devl_unlock(devlink); devlink_free(devlink); return ret; } @@ -4056,8 +4065,11 @@ static void mlx4_unload_one(struct pci_dev *pdev) struct mlx4_dev *dev = persist->dev; struct mlx4_priv *priv = mlx4_priv(dev); int pci_dev_data; + struct devlink *devlink; int p, i; + devlink = priv_to_devlink(priv); + devl_assert_locked(devlink); if (priv->removed) return; @@ -4137,6 +4149,7 @@ static void mlx4_remove_one(struct pci_dev *pdev) devlink_unregister(devlink); + devl_lock(devlink); if (mlx4_is_slave(dev)) persist->interface_state |= MLX4_INTERFACE_STATE_NOWAIT; @@ -4172,6 +4185,7 @@ static void mlx4_remove_one(struct pci_dev *pdev) devlink_params_unregister(devlink, mlx4_devlink_params, ARRAY_SIZE(mlx4_devlink_params)); kfree(dev->persist); + devl_unlock(devlink); devlink_free(devlink); } @@ -4292,15 +4306,20 @@ static pci_ers_result_t mlx4_pci_err_detected(struct pci_dev *pdev, pci_channel_state_t state) { struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev); + struct mlx4_dev *dev = persist->dev; + struct devlink *devlink; mlx4_err(persist->dev, "mlx4_pci_err_detected was called\n"); mlx4_enter_error_state(persist); + devlink = priv_to_devlink(mlx4_priv(dev)); + devl_lock(devlink); mutex_lock(&persist->interface_state_mutex); if (persist->interface_state & MLX4_INTERFACE_STATE_UP) mlx4_unload_one(pdev); mutex_unlock(&persist->interface_state_mutex); + devl_unlock(devlink); if (state == pci_channel_io_perm_failure) return PCI_ERS_RESULT_DISCONNECT; @@ -4333,6 +4352,7 @@ static void mlx4_pci_resume(struct pci_dev *pdev) struct mlx4_dev *dev = persist->dev; struct mlx4_priv *priv = mlx4_priv(dev); int nvfs[MLX4_MAX_PORTS + 1] = {0, 0, 0}; + struct devlink *devlink; int total_vfs; int err; @@ -4340,6 +4360,8 @@ static void mlx4_pci_resume(struct pci_dev *pdev) total_vfs = dev->persist->num_vfs; memcpy(nvfs, dev->persist->nvfs, sizeof(dev->persist->nvfs)); + devlink = priv_to_devlink(priv); + devl_lock(devlink); mutex_lock(&persist->interface_state_mutex); if (!(persist->interface_state & MLX4_INTERFACE_STATE_UP)) { err = mlx4_load_one(pdev, priv->pci_dev_data, total_vfs, nvfs, @@ -4358,19 +4380,23 @@ static void mlx4_pci_resume(struct pci_dev *pdev) } end: mutex_unlock(&persist->interface_state_mutex); - + devl_unlock(devlink); } static void mlx4_shutdown(struct pci_dev *pdev) { struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev); struct mlx4_dev *dev = persist->dev; + struct devlink *devlink; mlx4_info(persist->dev, "mlx4_shutdown was called\n"); + devlink = priv_to_devlink(mlx4_priv(dev)); + devl_lock(devlink); mutex_lock(&persist->interface_state_mutex); if (persist->interface_state & MLX4_INTERFACE_STATE_UP) mlx4_unload_one(pdev); mutex_unlock(&persist->interface_state_mutex); + devl_unlock(devlink); mlx4_pci_disable_device(dev); } @@ -4385,12 +4411,16 @@ static int __maybe_unused mlx4_suspend(struct device *dev_d) struct pci_dev *pdev = to_pci_dev(dev_d); struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev); struct mlx4_dev *dev = persist->dev; + struct devlink *devlink; mlx4_err(dev, "suspend was called\n"); + devlink = priv_to_devlink(mlx4_priv(dev)); + devl_lock(devlink); mutex_lock(&persist->interface_state_mutex); if (persist->interface_state & MLX4_INTERFACE_STATE_UP) mlx4_unload_one(pdev); mutex_unlock(&persist->interface_state_mutex); + devl_unlock(devlink); return 0; } @@ -4402,6 +4432,7 @@ static int __maybe_unused mlx4_resume(struct device *dev_d) struct mlx4_dev *dev = persist->dev; struct mlx4_priv *priv = mlx4_priv(dev); int nvfs[MLX4_MAX_PORTS + 1] = {0, 0, 0}; + struct devlink *devlink; int total_vfs; int ret = 0; @@ -4409,6 +4440,8 @@ static int __maybe_unused mlx4_resume(struct device *dev_d) total_vfs = dev->persist->num_vfs; memcpy(nvfs, dev->persist->nvfs, sizeof(dev->persist->nvfs)); + devlink = priv_to_devlink(priv); + devl_lock(devlink); mutex_lock(&persist->interface_state_mutex); if (!(persist->interface_state & MLX4_INTERFACE_STATE_UP)) { ret = mlx4_load_one(pdev, priv->pci_dev_data, total_vfs, @@ -4422,6 +4455,7 @@ static int __maybe_unused mlx4_resume(struct device *dev_d) } } mutex_unlock(&persist->interface_state_mutex); + devl_unlock(devlink); return ret; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/dev.c b/drivers/net/ethernet/mellanox/mlx5/core/dev.c index ccf2068d2e79..0571e40c6ee5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/dev.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/dev.c @@ -335,13 +335,12 @@ static void del_adev(struct auxiliary_device *adev) int mlx5_attach_device(struct mlx5_core_dev *dev) { - struct devlink *devlink = priv_to_devlink(dev); struct mlx5_priv *priv = &dev->priv; struct auxiliary_device *adev; struct auxiliary_driver *adrv; int ret = 0, i; - devl_lock(devlink); + devl_assert_locked(priv_to_devlink(dev)); mutex_lock(&mlx5_intf_mutex); priv->flags &= ~MLX5_PRIV_FLAGS_DETACH; priv->flags |= MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; @@ -394,20 +393,18 @@ int mlx5_attach_device(struct mlx5_core_dev *dev) } priv->flags &= ~MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; mutex_unlock(&mlx5_intf_mutex); - devl_unlock(devlink); return ret; } void mlx5_detach_device(struct mlx5_core_dev *dev) { - struct devlink *devlink = priv_to_devlink(dev); struct mlx5_priv *priv = &dev->priv; struct auxiliary_device *adev; struct auxiliary_driver *adrv; pm_message_t pm = {}; int i; - devl_lock(devlink); + devl_assert_locked(priv_to_devlink(dev)); mutex_lock(&mlx5_intf_mutex); priv->flags |= MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; for (i = ARRAY_SIZE(mlx5_adev_devices) - 1; i >= 0; i--) { @@ -441,21 +438,17 @@ skip_suspend: priv->flags &= ~MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; priv->flags |= MLX5_PRIV_FLAGS_DETACH; mutex_unlock(&mlx5_intf_mutex); - devl_unlock(devlink); } int mlx5_register_device(struct mlx5_core_dev *dev) { - struct devlink *devlink; int ret; - devlink = priv_to_devlink(dev); - devl_lock(devlink); + devl_assert_locked(priv_to_devlink(dev)); mutex_lock(&mlx5_intf_mutex); dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV; ret = mlx5_rescan_drivers_locked(dev); mutex_unlock(&mlx5_intf_mutex); - devl_unlock(devlink); if (ret) mlx5_unregister_device(dev); @@ -464,15 +457,11 @@ int mlx5_register_device(struct mlx5_core_dev *dev) void mlx5_unregister_device(struct mlx5_core_dev *dev) { - struct devlink *devlink; - - devlink = priv_to_devlink(dev); - devl_lock(devlink); + devl_assert_locked(priv_to_devlink(dev)); mutex_lock(&mlx5_intf_mutex); dev->priv.flags = MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV; mlx5_rescan_drivers_locked(dev); mutex_unlock(&mlx5_intf_mutex); - devl_unlock(devlink); } static int add_drivers(struct mlx5_core_dev *dev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c index f85166e587f2..1c05a7091698 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c @@ -104,7 +104,16 @@ static int mlx5_devlink_reload_fw_activate(struct devlink *devlink, struct netli if (err) return err; - return mlx5_fw_reset_wait_reset_done(dev); + err = mlx5_fw_reset_wait_reset_done(dev); + if (err) + return err; + + mlx5_unload_one_devl_locked(dev); + err = mlx5_health_wait_pci_up(dev); + if (err) + NL_SET_ERR_MSG_MOD(extack, "FW activate aborted, PCI reads fail after reset"); + + return err; } static int mlx5_devlink_trigger_fw_live_patch(struct devlink *devlink, @@ -134,6 +143,7 @@ static int mlx5_devlink_reload_down(struct devlink *devlink, bool netns_change, struct mlx5_core_dev *dev = devlink_priv(devlink); struct pci_dev *pdev = dev->pdev; bool sf_dev_allocated; + int ret = 0; sf_dev_allocated = mlx5_sf_dev_allocated(dev); if (sf_dev_allocated) { @@ -154,19 +164,25 @@ static int mlx5_devlink_reload_down(struct devlink *devlink, bool netns_change, NL_SET_ERR_MSG_MOD(extack, "reload while VFs are present is unfavorable"); } + devl_lock(devlink); switch (action) { case DEVLINK_RELOAD_ACTION_DRIVER_REINIT: - mlx5_unload_one(dev); - return 0; + mlx5_unload_one_devl_locked(dev); + break; case DEVLINK_RELOAD_ACTION_FW_ACTIVATE: if (limit == DEVLINK_RELOAD_LIMIT_NO_RESET) - return mlx5_devlink_trigger_fw_live_patch(devlink, extack); - return mlx5_devlink_reload_fw_activate(devlink, extack); + ret = mlx5_devlink_trigger_fw_live_patch(devlink, extack); + else + ret = mlx5_devlink_reload_fw_activate(devlink, extack); + break; default: /* Unsupported action should not get to this function */ WARN_ON(1); - return -EOPNOTSUPP; + ret = -EOPNOTSUPP; } + + devl_unlock(devlink); + return ret; } static int mlx5_devlink_reload_up(struct devlink *devlink, enum devlink_reload_action action, @@ -174,24 +190,29 @@ static int mlx5_devlink_reload_up(struct devlink *devlink, enum devlink_reload_a struct netlink_ext_ack *extack) { struct mlx5_core_dev *dev = devlink_priv(devlink); + int ret = 0; + devl_lock(devlink); *actions_performed = BIT(action); switch (action) { case DEVLINK_RELOAD_ACTION_DRIVER_REINIT: - return mlx5_load_one(dev, false); + ret = mlx5_load_one_devl_locked(dev, false); + break; case DEVLINK_RELOAD_ACTION_FW_ACTIVATE: if (limit == DEVLINK_RELOAD_LIMIT_NO_RESET) break; /* On fw_activate action, also driver is reloaded and reinit performed */ *actions_performed |= BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT); - return mlx5_load_one(dev, false); + ret = mlx5_load_one_devl_locked(dev, false); + break; default: /* Unsupported action should not get to this function */ WARN_ON(1); - return -EOPNOTSUPP; + ret = -EOPNOTSUPP; } - return 0; + devl_unlock(devlink); + return ret; } static struct mlx5_devlink_trap *mlx5_find_trap_by_id(struct mlx5_core_dev *dev, int trap_id) @@ -828,28 +849,28 @@ static int mlx5_devlink_traps_register(struct devlink *devlink) struct mlx5_core_dev *core_dev = devlink_priv(devlink); int err; - err = devlink_trap_groups_register(devlink, mlx5_trap_groups_arr, - ARRAY_SIZE(mlx5_trap_groups_arr)); + err = devl_trap_groups_register(devlink, mlx5_trap_groups_arr, + ARRAY_SIZE(mlx5_trap_groups_arr)); if (err) return err; - err = devlink_traps_register(devlink, mlx5_traps_arr, ARRAY_SIZE(mlx5_traps_arr), - &core_dev->priv); + err = devl_traps_register(devlink, mlx5_traps_arr, ARRAY_SIZE(mlx5_traps_arr), + &core_dev->priv); if (err) goto err_trap_group; return 0; err_trap_group: - devlink_trap_groups_unregister(devlink, mlx5_trap_groups_arr, - ARRAY_SIZE(mlx5_trap_groups_arr)); + devl_trap_groups_unregister(devlink, mlx5_trap_groups_arr, + ARRAY_SIZE(mlx5_trap_groups_arr)); return err; } static void mlx5_devlink_traps_unregister(struct devlink *devlink) { - devlink_traps_unregister(devlink, mlx5_traps_arr, ARRAY_SIZE(mlx5_traps_arr)); - devlink_trap_groups_unregister(devlink, mlx5_trap_groups_arr, - ARRAY_SIZE(mlx5_trap_groups_arr)); + devl_traps_unregister(devlink, mlx5_traps_arr, ARRAY_SIZE(mlx5_traps_arr)); + devl_trap_groups_unregister(devlink, mlx5_trap_groups_arr, + ARRAY_SIZE(mlx5_trap_groups_arr)); } int mlx5_devlink_register(struct devlink *devlink) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h index 04c0a5e1c89a..1839f1ab1ddd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h @@ -194,4 +194,14 @@ static inline void mlx5e_accel_cleanup_rx(struct mlx5e_priv *priv) { mlx5e_ktls_cleanup_rx(priv); } + +static inline int mlx5e_accel_init_tx(struct mlx5e_priv *priv) +{ + return mlx5e_ktls_init_tx(priv); +} + +static inline void mlx5e_accel_cleanup_tx(struct mlx5e_priv *priv) +{ + mlx5e_ktls_cleanup_tx(priv); +} #endif /* __MLX5E_EN_ACCEL_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h index d016624fbc9d..948400dee525 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h @@ -42,6 +42,8 @@ static inline bool mlx5e_ktls_type_check(struct mlx5_core_dev *mdev, } void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv); +int mlx5e_ktls_init_tx(struct mlx5e_priv *priv); +void mlx5e_ktls_cleanup_tx(struct mlx5e_priv *priv); int mlx5e_ktls_init_rx(struct mlx5e_priv *priv); void mlx5e_ktls_cleanup_rx(struct mlx5e_priv *priv); int mlx5e_ktls_set_feature_rx(struct net_device *netdev, bool enable); @@ -62,6 +64,8 @@ static inline bool mlx5e_is_ktls_rx(struct mlx5_core_dev *mdev) struct mlx5e_tls_sw_stats { atomic64_t tx_tls_ctx; atomic64_t tx_tls_del; + atomic64_t tx_tls_pool_alloc; + atomic64_t tx_tls_pool_free; atomic64_t rx_tls_ctx; atomic64_t rx_tls_del; }; @@ -69,6 +73,7 @@ struct mlx5e_tls_sw_stats { struct mlx5e_tls { struct mlx5e_tls_sw_stats sw_stats; struct workqueue_struct *rx_wq; + struct mlx5e_tls_tx_pool *tx_pool; }; int mlx5e_ktls_init(struct mlx5e_priv *priv); @@ -83,6 +88,15 @@ static inline void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv) { } +static inline int mlx5e_ktls_init_tx(struct mlx5e_priv *priv) +{ + return 0; +} + +static inline void mlx5e_ktls_cleanup_tx(struct mlx5e_priv *priv) +{ +} + static inline int mlx5e_ktls_init_rx(struct mlx5e_priv *priv) { return 0; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_stats.c index 2ab46c4247ff..7c1c0eb16787 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_stats.c @@ -41,6 +41,8 @@ static const struct counter_desc mlx5e_ktls_sw_stats_desc[] = { { MLX5E_DECLARE_STAT(struct mlx5e_tls_sw_stats, tx_tls_ctx) }, { MLX5E_DECLARE_STAT(struct mlx5e_tls_sw_stats, tx_tls_del) }, + { MLX5E_DECLARE_STAT(struct mlx5e_tls_sw_stats, tx_tls_pool_alloc) }, + { MLX5E_DECLARE_STAT(struct mlx5e_tls_sw_stats, tx_tls_pool_free) }, { MLX5E_DECLARE_STAT(struct mlx5e_tls_sw_stats, rx_tls_ctx) }, { MLX5E_DECLARE_STAT(struct mlx5e_tls_sw_stats, rx_tls_del) }, }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c index fba21edf88d8..6b6c7044b64a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c @@ -35,30 +35,70 @@ u16 mlx5e_ktls_get_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *pa stop_room += mlx5e_stop_room_for_wqe(mdev, MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS); stop_room += mlx5e_stop_room_for_wqe(mdev, MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS); stop_room += num_dumps * mlx5e_stop_room_for_wqe(mdev, MLX5E_KTLS_DUMP_WQEBBS); + stop_room += 1; /* fence nop */ return stop_room; } +static void mlx5e_ktls_set_tisc(struct mlx5_core_dev *mdev, void *tisc) +{ + MLX5_SET(tisc, tisc, tls_en, 1); + MLX5_SET(tisc, tisc, pd, mdev->mlx5e_res.hw_objs.pdn); + MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.hw_objs.td.tdn); +} + static int mlx5e_ktls_create_tis(struct mlx5_core_dev *mdev, u32 *tisn) { u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; - void *tisc; - tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); + mlx5e_ktls_set_tisc(mdev, MLX5_ADDR_OF(create_tis_in, in, ctx)); - MLX5_SET(tisc, tisc, tls_en, 1); + return mlx5_core_create_tis(mdev, in, tisn); +} + +static int mlx5e_ktls_create_tis_cb(struct mlx5_core_dev *mdev, + struct mlx5_async_ctx *async_ctx, + u32 *out, int outlen, + mlx5_async_cbk_t callback, + struct mlx5_async_work *context) +{ + u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; + + mlx5e_ktls_set_tisc(mdev, MLX5_ADDR_OF(create_tis_in, in, ctx)); + MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS); + + return mlx5_cmd_exec_cb(async_ctx, in, sizeof(in), + out, outlen, callback, context); +} + +static int mlx5e_ktls_destroy_tis_cb(struct mlx5_core_dev *mdev, u32 tisn, + struct mlx5_async_ctx *async_ctx, + u32 *out, int outlen, + mlx5_async_cbk_t callback, + struct mlx5_async_work *context) +{ + u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {}; + + MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS); + MLX5_SET(destroy_tis_in, in, tisn, tisn); - return mlx5e_create_tis(mdev, in, tisn); + return mlx5_cmd_exec_cb(async_ctx, in, sizeof(in), + out, outlen, callback, context); } struct mlx5e_ktls_offload_context_tx { - struct tls_offload_context_tx *tx_ctx; - struct tls12_crypto_info_aes_gcm_128 crypto_info; - struct mlx5e_tls_sw_stats *sw_stats; + /* fast path */ u32 expected_seq; u32 tisn; - u32 key_id; bool ctx_post_pending; + /* control / resync */ + struct list_head list_node; /* member of the pool */ + struct tls12_crypto_info_aes_gcm_128 crypto_info; + struct tls_offload_context_tx *tx_ctx; + struct mlx5_core_dev *mdev; + struct mlx5e_tls_sw_stats *sw_stats; + u32 key_id; + u8 create_err : 1; }; static void @@ -82,28 +122,368 @@ mlx5e_get_ktls_tx_priv_ctx(struct tls_context *tls_ctx) return *ctx; } +/* struct for callback API management */ +struct mlx5e_async_ctx { + struct mlx5_async_work context; + struct mlx5_async_ctx async_ctx; + struct work_struct work; + struct mlx5e_ktls_offload_context_tx *priv_tx; + struct completion complete; + int err; + union { + u32 out_create[MLX5_ST_SZ_DW(create_tis_out)]; + u32 out_destroy[MLX5_ST_SZ_DW(destroy_tis_out)]; + }; +}; + +static struct mlx5e_async_ctx *mlx5e_bulk_async_init(struct mlx5_core_dev *mdev, int n) +{ + struct mlx5e_async_ctx *bulk_async; + int i; + + bulk_async = kvcalloc(n, sizeof(struct mlx5e_async_ctx), GFP_KERNEL); + if (!bulk_async) + return NULL; + + for (i = 0; i < n; i++) { + struct mlx5e_async_ctx *async = &bulk_async[i]; + + mlx5_cmd_init_async_ctx(mdev, &async->async_ctx); + init_completion(&async->complete); + } + + return bulk_async; +} + +static void mlx5e_bulk_async_cleanup(struct mlx5e_async_ctx *bulk_async, int n) +{ + int i; + + for (i = 0; i < n; i++) { + struct mlx5e_async_ctx *async = &bulk_async[i]; + + mlx5_cmd_cleanup_async_ctx(&async->async_ctx); + } + kvfree(bulk_async); +} + +static void create_tis_callback(int status, struct mlx5_async_work *context) +{ + struct mlx5e_async_ctx *async = + container_of(context, struct mlx5e_async_ctx, context); + struct mlx5e_ktls_offload_context_tx *priv_tx = async->priv_tx; + + if (status) { + async->err = status; + priv_tx->create_err = 1; + goto out; + } + + priv_tx->tisn = MLX5_GET(create_tis_out, async->out_create, tisn); +out: + complete(&async->complete); +} + +static void destroy_tis_callback(int status, struct mlx5_async_work *context) +{ + struct mlx5e_async_ctx *async = + container_of(context, struct mlx5e_async_ctx, context); + struct mlx5e_ktls_offload_context_tx *priv_tx = async->priv_tx; + + complete(&async->complete); + kfree(priv_tx); +} + +static struct mlx5e_ktls_offload_context_tx * +mlx5e_tls_priv_tx_init(struct mlx5_core_dev *mdev, struct mlx5e_tls_sw_stats *sw_stats, + struct mlx5e_async_ctx *async) +{ + struct mlx5e_ktls_offload_context_tx *priv_tx; + int err; + + priv_tx = kzalloc(sizeof(*priv_tx), GFP_KERNEL); + if (!priv_tx) + return ERR_PTR(-ENOMEM); + + priv_tx->mdev = mdev; + priv_tx->sw_stats = sw_stats; + + if (!async) { + err = mlx5e_ktls_create_tis(mdev, &priv_tx->tisn); + if (err) + goto err_out; + } else { + async->priv_tx = priv_tx; + err = mlx5e_ktls_create_tis_cb(mdev, &async->async_ctx, + async->out_create, sizeof(async->out_create), + create_tis_callback, &async->context); + if (err) + goto err_out; + } + + return priv_tx; + +err_out: + kfree(priv_tx); + return ERR_PTR(err); +} + +static void mlx5e_tls_priv_tx_cleanup(struct mlx5e_ktls_offload_context_tx *priv_tx, + struct mlx5e_async_ctx *async) +{ + if (priv_tx->create_err) { + complete(&async->complete); + kfree(priv_tx); + return; + } + async->priv_tx = priv_tx; + mlx5e_ktls_destroy_tis_cb(priv_tx->mdev, priv_tx->tisn, + &async->async_ctx, + async->out_destroy, sizeof(async->out_destroy), + destroy_tis_callback, &async->context); +} + +static void mlx5e_tls_priv_tx_list_cleanup(struct mlx5_core_dev *mdev, + struct list_head *list, int size) +{ + struct mlx5e_ktls_offload_context_tx *obj; + struct mlx5e_async_ctx *bulk_async; + int i; + + bulk_async = mlx5e_bulk_async_init(mdev, size); + if (!bulk_async) + return; + + i = 0; + list_for_each_entry(obj, list, list_node) { + mlx5e_tls_priv_tx_cleanup(obj, &bulk_async[i]); + i++; + } + + for (i = 0; i < size; i++) { + struct mlx5e_async_ctx *async = &bulk_async[i]; + + wait_for_completion(&async->complete); + } + mlx5e_bulk_async_cleanup(bulk_async, size); +} + +/* Recycling pool API */ + +#define MLX5E_TLS_TX_POOL_BULK (16) +#define MLX5E_TLS_TX_POOL_HIGH (4 * 1024) +#define MLX5E_TLS_TX_POOL_LOW (MLX5E_TLS_TX_POOL_HIGH / 4) + +struct mlx5e_tls_tx_pool { + struct mlx5_core_dev *mdev; + struct mlx5e_tls_sw_stats *sw_stats; + struct mutex lock; /* Protects access to the pool */ + struct list_head list; + size_t size; + + struct workqueue_struct *wq; + struct work_struct create_work; + struct work_struct destroy_work; +}; + +static void create_work(struct work_struct *work) +{ + struct mlx5e_tls_tx_pool *pool = + container_of(work, struct mlx5e_tls_tx_pool, create_work); + struct mlx5e_ktls_offload_context_tx *obj; + struct mlx5e_async_ctx *bulk_async; + LIST_HEAD(local_list); + int i, j, err = 0; + + bulk_async = mlx5e_bulk_async_init(pool->mdev, MLX5E_TLS_TX_POOL_BULK); + if (!bulk_async) + return; + + for (i = 0; i < MLX5E_TLS_TX_POOL_BULK; i++) { + obj = mlx5e_tls_priv_tx_init(pool->mdev, pool->sw_stats, &bulk_async[i]); + if (IS_ERR(obj)) { + err = PTR_ERR(obj); + break; + } + list_add(&obj->list_node, &local_list); + } + + for (j = 0; j < i; j++) { + struct mlx5e_async_ctx *async = &bulk_async[j]; + + wait_for_completion(&async->complete); + if (!err && async->err) + err = async->err; + } + atomic64_add(i, &pool->sw_stats->tx_tls_pool_alloc); + mlx5e_bulk_async_cleanup(bulk_async, MLX5E_TLS_TX_POOL_BULK); + if (err) + goto err_out; + + mutex_lock(&pool->lock); + if (pool->size + MLX5E_TLS_TX_POOL_BULK >= MLX5E_TLS_TX_POOL_HIGH) { + mutex_unlock(&pool->lock); + goto err_out; + } + list_splice(&local_list, &pool->list); + pool->size += MLX5E_TLS_TX_POOL_BULK; + if (pool->size <= MLX5E_TLS_TX_POOL_LOW) + queue_work(pool->wq, work); + mutex_unlock(&pool->lock); + return; + +err_out: + mlx5e_tls_priv_tx_list_cleanup(pool->mdev, &local_list, i); + atomic64_add(i, &pool->sw_stats->tx_tls_pool_free); +} + +static void destroy_work(struct work_struct *work) +{ + struct mlx5e_tls_tx_pool *pool = + container_of(work, struct mlx5e_tls_tx_pool, destroy_work); + struct mlx5e_ktls_offload_context_tx *obj; + LIST_HEAD(local_list); + int i = 0; + + mutex_lock(&pool->lock); + if (pool->size < MLX5E_TLS_TX_POOL_HIGH) { + mutex_unlock(&pool->lock); + return; + } + + list_for_each_entry(obj, &pool->list, list_node) + if (++i == MLX5E_TLS_TX_POOL_BULK) + break; + + list_cut_position(&local_list, &pool->list, &obj->list_node); + pool->size -= MLX5E_TLS_TX_POOL_BULK; + if (pool->size >= MLX5E_TLS_TX_POOL_HIGH) + queue_work(pool->wq, work); + mutex_unlock(&pool->lock); + + mlx5e_tls_priv_tx_list_cleanup(pool->mdev, &local_list, MLX5E_TLS_TX_POOL_BULK); + atomic64_add(MLX5E_TLS_TX_POOL_BULK, &pool->sw_stats->tx_tls_pool_free); +} + +static struct mlx5e_tls_tx_pool *mlx5e_tls_tx_pool_init(struct mlx5_core_dev *mdev, + struct mlx5e_tls_sw_stats *sw_stats) +{ + struct mlx5e_tls_tx_pool *pool; + + BUILD_BUG_ON(MLX5E_TLS_TX_POOL_LOW + MLX5E_TLS_TX_POOL_BULK >= MLX5E_TLS_TX_POOL_HIGH); + + pool = kvzalloc(sizeof(*pool), GFP_KERNEL); + if (!pool) + return NULL; + + pool->wq = create_singlethread_workqueue("mlx5e_tls_tx_pool"); + if (!pool->wq) + goto err_free; + + INIT_LIST_HEAD(&pool->list); + mutex_init(&pool->lock); + + INIT_WORK(&pool->create_work, create_work); + INIT_WORK(&pool->destroy_work, destroy_work); + + pool->mdev = mdev; + pool->sw_stats = sw_stats; + + return pool; + +err_free: + kvfree(pool); + return NULL; +} + +static void mlx5e_tls_tx_pool_list_cleanup(struct mlx5e_tls_tx_pool *pool) +{ + while (pool->size > MLX5E_TLS_TX_POOL_BULK) { + struct mlx5e_ktls_offload_context_tx *obj; + LIST_HEAD(local_list); + int i = 0; + + list_for_each_entry(obj, &pool->list, list_node) + if (++i == MLX5E_TLS_TX_POOL_BULK) + break; + + list_cut_position(&local_list, &pool->list, &obj->list_node); + mlx5e_tls_priv_tx_list_cleanup(pool->mdev, &local_list, MLX5E_TLS_TX_POOL_BULK); + atomic64_add(MLX5E_TLS_TX_POOL_BULK, &pool->sw_stats->tx_tls_pool_free); + pool->size -= MLX5E_TLS_TX_POOL_BULK; + } + if (pool->size) { + mlx5e_tls_priv_tx_list_cleanup(pool->mdev, &pool->list, pool->size); + atomic64_add(pool->size, &pool->sw_stats->tx_tls_pool_free); + } +} + +static void mlx5e_tls_tx_pool_cleanup(struct mlx5e_tls_tx_pool *pool) +{ + mlx5e_tls_tx_pool_list_cleanup(pool); + destroy_workqueue(pool->wq); + kvfree(pool); +} + +static void pool_push(struct mlx5e_tls_tx_pool *pool, struct mlx5e_ktls_offload_context_tx *obj) +{ + mutex_lock(&pool->lock); + list_add(&obj->list_node, &pool->list); + if (++pool->size == MLX5E_TLS_TX_POOL_HIGH) + queue_work(pool->wq, &pool->destroy_work); + mutex_unlock(&pool->lock); +} + +static struct mlx5e_ktls_offload_context_tx *pool_pop(struct mlx5e_tls_tx_pool *pool) +{ + struct mlx5e_ktls_offload_context_tx *obj; + + mutex_lock(&pool->lock); + if (unlikely(pool->size == 0)) { + /* pool is empty: + * - trigger the populating work, and + * - serve the current context via the regular blocking api. + */ + queue_work(pool->wq, &pool->create_work); + mutex_unlock(&pool->lock); + obj = mlx5e_tls_priv_tx_init(pool->mdev, pool->sw_stats, NULL); + if (!IS_ERR(obj)) + atomic64_inc(&pool->sw_stats->tx_tls_pool_alloc); + return obj; + } + + obj = list_first_entry(&pool->list, struct mlx5e_ktls_offload_context_tx, + list_node); + list_del(&obj->list_node); + if (--pool->size == MLX5E_TLS_TX_POOL_LOW) + queue_work(pool->wq, &pool->create_work); + mutex_unlock(&pool->lock); + return obj; +} + +/* End of pool API */ + int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk, struct tls_crypto_info *crypto_info, u32 start_offload_tcp_sn) { struct mlx5e_ktls_offload_context_tx *priv_tx; + struct mlx5e_tls_tx_pool *pool; struct tls_context *tls_ctx; - struct mlx5_core_dev *mdev; struct mlx5e_priv *priv; int err; tls_ctx = tls_get_ctx(sk); priv = netdev_priv(netdev); - mdev = priv->mdev; + pool = priv->tls->tx_pool; - priv_tx = kzalloc(sizeof(*priv_tx), GFP_KERNEL); - if (!priv_tx) - return -ENOMEM; + priv_tx = pool_pop(pool); + if (IS_ERR(priv_tx)) + return PTR_ERR(priv_tx); - err = mlx5_ktls_create_key(mdev, crypto_info, &priv_tx->key_id); + err = mlx5_ktls_create_key(pool->mdev, crypto_info, &priv_tx->key_id); if (err) goto err_create_key; - priv_tx->sw_stats = &priv->tls->sw_stats; priv_tx->expected_seq = start_offload_tcp_sn; priv_tx->crypto_info = *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info; @@ -111,36 +491,29 @@ int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk, mlx5e_set_ktls_tx_priv_ctx(tls_ctx, priv_tx); - err = mlx5e_ktls_create_tis(mdev, &priv_tx->tisn); - if (err) - goto err_create_tis; - priv_tx->ctx_post_pending = true; atomic64_inc(&priv_tx->sw_stats->tx_tls_ctx); return 0; -err_create_tis: - mlx5_ktls_destroy_key(mdev, priv_tx->key_id); err_create_key: - kfree(priv_tx); + pool_push(pool, priv_tx); return err; } void mlx5e_ktls_del_tx(struct net_device *netdev, struct tls_context *tls_ctx) { struct mlx5e_ktls_offload_context_tx *priv_tx; - struct mlx5_core_dev *mdev; + struct mlx5e_tls_tx_pool *pool; struct mlx5e_priv *priv; priv_tx = mlx5e_get_ktls_tx_priv_ctx(tls_ctx); priv = netdev_priv(netdev); - mdev = priv->mdev; + pool = priv->tls->tx_pool; atomic64_inc(&priv_tx->sw_stats->tx_tls_del); - mlx5e_destroy_tis(mdev, priv_tx->tisn); - mlx5_ktls_destroy_key(mdev, priv_tx->key_id); - kfree(priv_tx); + mlx5_ktls_destroy_key(priv_tx->mdev, priv_tx->key_id); + pool_push(pool, priv_tx); } static void tx_fill_wi(struct mlx5e_txqsq *sq, @@ -201,6 +574,16 @@ post_progress_params(struct mlx5e_txqsq *sq, sq->pc += num_wqebbs; } +static void tx_post_fence_nop(struct mlx5e_txqsq *sq) +{ + struct mlx5_wq_cyc *wq = &sq->wq; + u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); + + tx_fill_wi(sq, pi, 1, 0, NULL); + + mlx5e_post_nop_fence(wq, sq->sqn, &sq->pc); +} + static void mlx5e_ktls_tx_post_param_wqes(struct mlx5e_txqsq *sq, struct mlx5e_ktls_offload_context_tx *priv_tx, @@ -212,6 +595,7 @@ mlx5e_ktls_tx_post_param_wqes(struct mlx5e_txqsq *sq, post_static_params(sq, priv_tx, fence_first_post); post_progress_params(sq, priv_tx, progress_fence); + tx_post_fence_nop(sq); } struct tx_sync_info { @@ -304,7 +688,7 @@ tx_post_resync_params(struct mlx5e_txqsq *sq, } static int -tx_post_resync_dump(struct mlx5e_txqsq *sq, skb_frag_t *frag, u32 tisn, bool first) +tx_post_resync_dump(struct mlx5e_txqsq *sq, skb_frag_t *frag, u32 tisn) { struct mlx5_wqe_ctrl_seg *cseg; struct mlx5_wqe_data_seg *dseg; @@ -326,7 +710,6 @@ tx_post_resync_dump(struct mlx5e_txqsq *sq, skb_frag_t *frag, u32 tisn, bool fir cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_DUMP); cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); cseg->tis_tir_num = cpu_to_be32(tisn << 8); - cseg->fm_ce_se = first ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0; fsz = skb_frag_size(frag); dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz, @@ -361,67 +744,39 @@ void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq, stats->tls_dump_bytes += wi->num_bytes; } -static void tx_post_fence_nop(struct mlx5e_txqsq *sq) -{ - struct mlx5_wq_cyc *wq = &sq->wq; - u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); - - tx_fill_wi(sq, pi, 1, 0, NULL); - - mlx5e_post_nop_fence(wq, sq->sqn, &sq->pc); -} - static enum mlx5e_ktls_sync_retval mlx5e_ktls_tx_handle_ooo(struct mlx5e_ktls_offload_context_tx *priv_tx, struct mlx5e_txqsq *sq, int datalen, u32 seq) { - struct mlx5e_sq_stats *stats = sq->stats; enum mlx5e_ktls_sync_retval ret; struct tx_sync_info info = {}; - int i = 0; + int i; ret = tx_sync_info_get(priv_tx, seq, datalen, &info); - if (unlikely(ret != MLX5E_KTLS_SYNC_DONE)) { - if (ret == MLX5E_KTLS_SYNC_SKIP_NO_DATA) { - stats->tls_skip_no_sync_data++; - return MLX5E_KTLS_SYNC_SKIP_NO_DATA; - } - /* We might get here if a retransmission reaches the driver - * after the relevant record is acked. + if (unlikely(ret != MLX5E_KTLS_SYNC_DONE)) + /* We might get here with ret == FAIL if a retransmission + * reaches the driver after the relevant record is acked. * It should be safe to drop the packet in this case */ - stats->tls_drop_no_sync_data++; - goto err_out; - } - - stats->tls_ooo++; + return ret; tx_post_resync_params(sq, priv_tx, info.rcd_sn); - /* If no dump WQE was sent, we need to have a fence NOP WQE before the - * actual data xmit. - */ - if (!info.nr_frags) { - tx_post_fence_nop(sq); - return MLX5E_KTLS_SYNC_DONE; - } - - for (; i < info.nr_frags; i++) { + for (i = 0; i < info.nr_frags; i++) { unsigned int orig_fsz, frag_offset = 0, n = 0; skb_frag_t *f = &info.frags[i]; orig_fsz = skb_frag_size(f); do { - bool fence = !(i || frag_offset); unsigned int fsz; n++; fsz = min_t(unsigned int, sq->hw_mtu, orig_fsz - frag_offset); skb_frag_size_set(f, fsz); - if (tx_post_resync_dump(sq, f, priv_tx->tisn, fence)) { + if (tx_post_resync_dump(sq, f, priv_tx->tisn)) { page_ref_add(skb_frag_page(f), n - 1); goto err_out; } @@ -469,24 +824,27 @@ bool mlx5e_ktls_handle_tx_skb(struct net_device *netdev, struct mlx5e_txqsq *sq, priv_tx = mlx5e_get_ktls_tx_priv_ctx(tls_ctx); - if (unlikely(mlx5e_ktls_tx_offload_test_and_clear_pending(priv_tx))) { + if (unlikely(mlx5e_ktls_tx_offload_test_and_clear_pending(priv_tx))) mlx5e_ktls_tx_post_param_wqes(sq, priv_tx, false, false); - } seq = ntohl(tcp_hdr(skb)->seq); if (unlikely(priv_tx->expected_seq != seq)) { enum mlx5e_ktls_sync_retval ret = mlx5e_ktls_tx_handle_ooo(priv_tx, sq, datalen, seq); + stats->tls_ooo++; + switch (ret) { case MLX5E_KTLS_SYNC_DONE: break; case MLX5E_KTLS_SYNC_SKIP_NO_DATA: + stats->tls_skip_no_sync_data++; if (likely(!skb->decrypted)) goto out; WARN_ON_ONCE(1); - fallthrough; + goto err_out; case MLX5E_KTLS_SYNC_FAIL: + stats->tls_drop_no_sync_data++; goto err_out; } } @@ -505,3 +863,24 @@ err_out: dev_kfree_skb_any(skb); return false; } + +int mlx5e_ktls_init_tx(struct mlx5e_priv *priv) +{ + if (!mlx5e_is_ktls_tx(priv->mdev)) + return 0; + + priv->tls->tx_pool = mlx5e_tls_tx_pool_init(priv->mdev, &priv->tls->sw_stats); + if (!priv->tls->tx_pool) + return -ENOMEM; + + return 0; +} + +void mlx5e_ktls_cleanup_tx(struct mlx5e_priv *priv) +{ + if (!mlx5e_is_ktls_tx(priv->mdev)) + return; + + mlx5e_tls_tx_pool_cleanup(priv->tls->tx_pool); + priv->tls->tx_pool = NULL; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 180b2f418339..24ddd438c066 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -3144,6 +3144,7 @@ static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv) mlx5e_mqprio_rl_free(priv->mqprio_rl); priv->mqprio_rl = NULL; } + mlx5e_accel_cleanup_tx(priv); mlx5e_destroy_tises(priv); } @@ -5147,9 +5148,17 @@ static int mlx5e_init_nic_tx(struct mlx5e_priv *priv) return err; } + err = mlx5e_accel_init_tx(priv); + if (err) + goto err_destroy_tises; + mlx5e_set_mqprio_rl(priv); mlx5e_dcbnl_initialize(priv); return 0; + +err_destroy_tises: + mlx5e_destroy_tises(priv); + return err; } static void mlx5e_nic_enable(struct mlx5e_priv *priv) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index 30a6c9fbf1b6..6aa58044b949 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -1300,20 +1300,19 @@ abort: */ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs) { - struct devlink *devlink; bool toggle_lag; int ret; if (!mlx5_esw_allowed(esw)) return 0; + devl_assert_locked(priv_to_devlink(esw->dev)); + toggle_lag = !mlx5_esw_is_fdb_created(esw); if (toggle_lag) mlx5_lag_disable_change(esw->dev); - devlink = priv_to_devlink(esw->dev); - devl_lock(devlink); down_write(&esw->mode_lock); if (!mlx5_esw_is_fdb_created(esw)) { ret = mlx5_eswitch_enable_locked(esw, num_vfs); @@ -1327,7 +1326,6 @@ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs) esw->esw_funcs.num_vfs = num_vfs; } up_write(&esw->mode_lock); - devl_unlock(devlink); if (toggle_lag) mlx5_lag_enable_change(esw->dev); @@ -1338,13 +1336,10 @@ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs) /* When disabling sriov, free driver level resources. */ void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf) { - struct devlink *devlink; - if (!mlx5_esw_allowed(esw)) return; - devlink = priv_to_devlink(esw->dev); - devl_lock(devlink); + devl_assert_locked(priv_to_devlink(esw->dev)); down_write(&esw->mode_lock); /* If driver is unloaded, this function is called twice by remove_one() * and mlx5_unload(). Prevent the second call. @@ -1373,7 +1368,6 @@ void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf) unlock: up_write(&esw->mode_lock); - devl_unlock(devlink); } /* Free resources for corresponding eswitch mode. It is called by devlink @@ -1407,18 +1401,14 @@ void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw) void mlx5_eswitch_disable(struct mlx5_eswitch *esw) { - struct devlink *devlink; - if (!mlx5_esw_allowed(esw)) return; + devl_assert_locked(priv_to_devlink(esw->dev)); mlx5_lag_disable_change(esw->dev); - devlink = priv_to_devlink(esw->dev); - devl_lock(devlink); down_write(&esw->mode_lock); mlx5_eswitch_disable_locked(esw); up_write(&esw->mode_lock); - devl_unlock(devlink); mlx5_lag_enable_change(esw->dev); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c index 052af4901c0b..e8896f368362 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c @@ -149,6 +149,9 @@ static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev) if (test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags)) { complete(&fw_reset->done); } else { + mlx5_unload_one(dev); + if (mlx5_health_wait_pci_up(dev)) + mlx5_core_err(dev, "reset reload flow aborted, PCI reads still not working\n"); mlx5_load_one(dev, false); devlink_remote_reload_actions_performed(priv_to_devlink(dev), 0, BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT) | @@ -183,15 +186,9 @@ static void mlx5_sync_reset_reload_work(struct work_struct *work) struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset, reset_reload_work); struct mlx5_core_dev *dev = fw_reset->dev; - int err; mlx5_sync_reset_clear_reset_requested(dev, false); mlx5_enter_error_state(dev, true); - mlx5_unload_one(dev); - err = mlx5_health_wait_pci_up(dev); - if (err) - mlx5_core_err(dev, "reset reload flow aborted, PCI reads still not working\n"); - fw_reset->ret = err; mlx5_fw_reset_complete_reload(dev); } @@ -395,7 +392,6 @@ static void mlx5_sync_reset_now_event(struct work_struct *work) } mlx5_enter_error_state(dev, true); - mlx5_unload_one(dev); done: fw_reset->ret = err; mlx5_fw_reset_complete_reload(dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c index 659021c31cbd..2cf2c9948446 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/health.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c @@ -666,16 +666,20 @@ static void mlx5_fw_fatal_reporter_err_work(struct work_struct *work) struct mlx5_fw_reporter_ctx fw_reporter_ctx; struct mlx5_core_health *health; struct mlx5_core_dev *dev; + struct devlink *devlink; struct mlx5_priv *priv; health = container_of(work, struct mlx5_core_health, fatal_report_work); priv = container_of(health, struct mlx5_priv, health); dev = container_of(priv, struct mlx5_core_dev, priv); + devlink = priv_to_devlink(dev); enter_error_state(dev, false); if (IS_ERR_OR_NULL(health->fw_fatal_reporter)) { + devl_lock(devlink); if (mlx5_health_try_recover(dev)) mlx5_core_err(dev, "health recovery failed\n"); + devl_unlock(devlink); return; } fw_reporter_ctx.err_synd = health->synd; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 8b621c1ddd14..1de9b39a6359 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -1304,8 +1304,10 @@ static void mlx5_unload(struct mlx5_core_dev *dev) int mlx5_init_one(struct mlx5_core_dev *dev) { + struct devlink *devlink = priv_to_devlink(dev); int err = 0; + devl_lock(devlink); mutex_lock(&dev->intf_state_mutex); dev->state = MLX5_DEVICE_STATE_UP; @@ -1334,6 +1336,7 @@ int mlx5_init_one(struct mlx5_core_dev *dev) goto err_register; mutex_unlock(&dev->intf_state_mutex); + devl_unlock(devlink); return 0; err_register: @@ -1348,11 +1351,15 @@ function_teardown: err_function: dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; mutex_unlock(&dev->intf_state_mutex); + devl_unlock(devlink); return err; } void mlx5_uninit_one(struct mlx5_core_dev *dev) { + struct devlink *devlink = priv_to_devlink(dev); + + devl_lock(devlink); mutex_lock(&dev->intf_state_mutex); mlx5_unregister_device(dev); @@ -1371,13 +1378,15 @@ void mlx5_uninit_one(struct mlx5_core_dev *dev) mlx5_function_teardown(dev, true); out: mutex_unlock(&dev->intf_state_mutex); + devl_unlock(devlink); } -int mlx5_load_one(struct mlx5_core_dev *dev, bool recovery) +int mlx5_load_one_devl_locked(struct mlx5_core_dev *dev, bool recovery) { int err = 0; u64 timeout; + devl_assert_locked(priv_to_devlink(dev)); mutex_lock(&dev->intf_state_mutex); if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { mlx5_core_warn(dev, "interface is up, NOP\n"); @@ -1419,8 +1428,20 @@ out: return err; } -void mlx5_unload_one(struct mlx5_core_dev *dev) +int mlx5_load_one(struct mlx5_core_dev *dev, bool recovery) +{ + struct devlink *devlink = priv_to_devlink(dev); + int ret; + + devl_lock(devlink); + ret = mlx5_load_one_devl_locked(dev, recovery); + devl_unlock(devlink); + return ret; +} + +void mlx5_unload_one_devl_locked(struct mlx5_core_dev *dev) { + devl_assert_locked(priv_to_devlink(dev)); mutex_lock(&dev->intf_state_mutex); mlx5_detach_device(dev); @@ -1438,6 +1459,15 @@ out: mutex_unlock(&dev->intf_state_mutex); } +void mlx5_unload_one(struct mlx5_core_dev *dev) +{ + struct devlink *devlink = priv_to_devlink(dev); + + devl_lock(devlink); + mlx5_unload_one_devl_locked(dev); + devl_unlock(devlink); +} + static const int types[] = { MLX5_CAP_GENERAL, MLX5_CAP_GENERAL_2, @@ -1902,7 +1932,7 @@ MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table); void mlx5_disable_device(struct mlx5_core_dev *dev) { mlx5_error_sw_reset(dev); - mlx5_unload_one(dev); + mlx5_unload_one_devl_locked(dev); } int mlx5_recover_device(struct mlx5_core_dev *dev) @@ -1913,7 +1943,7 @@ int mlx5_recover_device(struct mlx5_core_dev *dev) return -EIO; } - return mlx5_load_one(dev, true); + return mlx5_load_one_devl_locked(dev, true); } static struct pci_driver mlx5_core_driver = { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index 9cc7afea2758..ad61b86d5769 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -290,7 +290,9 @@ void mlx5_mdev_uninit(struct mlx5_core_dev *dev); int mlx5_init_one(struct mlx5_core_dev *dev); void mlx5_uninit_one(struct mlx5_core_dev *dev); void mlx5_unload_one(struct mlx5_core_dev *dev); +void mlx5_unload_one_devl_locked(struct mlx5_core_dev *dev); int mlx5_load_one(struct mlx5_core_dev *dev, bool recovery); +int mlx5_load_one_devl_locked(struct mlx5_core_dev *dev, bool recovery); int mlx5_vport_get_other_func_cap(struct mlx5_core_dev *dev, u16 function_id, void *out); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c index 5757cd6e1819..ee2e1b7c1310 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c @@ -154,13 +154,16 @@ mlx5_device_disable_sriov(struct mlx5_core_dev *dev, int num_vfs, bool clear_vf) static int mlx5_sriov_enable(struct pci_dev *pdev, int num_vfs) { struct mlx5_core_dev *dev = pci_get_drvdata(pdev); + struct devlink *devlink = priv_to_devlink(dev); int err; + devl_lock(devlink); err = mlx5_device_enable_sriov(dev, num_vfs); if (err) { mlx5_core_warn(dev, "mlx5_device_enable_sriov failed : %d\n", err); return err; } + devl_unlock(devlink); err = pci_enable_sriov(pdev, num_vfs); if (err) { @@ -173,10 +176,13 @@ static int mlx5_sriov_enable(struct pci_dev *pdev, int num_vfs) void mlx5_sriov_disable(struct pci_dev *pdev) { struct mlx5_core_dev *dev = pci_get_drvdata(pdev); + struct devlink *devlink = priv_to_devlink(dev); int num_vfs = pci_num_vf(dev->pdev); pci_disable_sriov(pdev); + devl_lock(devlink); mlx5_device_disable_sriov(dev, num_vfs, true); + devl_unlock(devlink); } int mlx5_core_sriov_configure(struct pci_dev *pdev, int num_vfs) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c index a48f893cf7b0..7c93bd04a3a1 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core.c @@ -3335,6 +3335,24 @@ u32 mlxsw_core_read_frc_l(struct mlxsw_core *mlxsw_core) } EXPORT_SYMBOL(mlxsw_core_read_frc_l); +u32 mlxsw_core_read_utc_sec(struct mlxsw_core *mlxsw_core) +{ + return mlxsw_core->bus->read_utc_sec(mlxsw_core->bus_priv); +} +EXPORT_SYMBOL(mlxsw_core_read_utc_sec); + +u32 mlxsw_core_read_utc_nsec(struct mlxsw_core *mlxsw_core) +{ + return mlxsw_core->bus->read_utc_nsec(mlxsw_core->bus_priv); +} +EXPORT_SYMBOL(mlxsw_core_read_utc_nsec); + +bool mlxsw_core_sdq_supports_cqe_v2(struct mlxsw_core *mlxsw_core) +{ + return mlxsw_core->driver->sdq_supports_cqe_v2; +} +EXPORT_SYMBOL(mlxsw_core_sdq_supports_cqe_v2); + void mlxsw_core_emad_string_tlv_enable(struct mlxsw_core *mlxsw_core) { mlxsw_core->emad.enable_string_tlv = true; diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.h b/drivers/net/ethernet/mellanox/mlxsw/core.h index 7213e4528298..02d9cc2ef0c8 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.h +++ b/drivers/net/ethernet/mellanox/mlxsw/core.h @@ -427,6 +427,7 @@ struct mlxsw_driver { u8 txhdr_len; const struct mlxsw_config_profile *profile; + bool sdq_supports_cqe_v2; }; int mlxsw_core_kvd_sizes_get(struct mlxsw_core *mlxsw_core, @@ -437,6 +438,11 @@ int mlxsw_core_kvd_sizes_get(struct mlxsw_core *mlxsw_core, u32 mlxsw_core_read_frc_h(struct mlxsw_core *mlxsw_core); u32 mlxsw_core_read_frc_l(struct mlxsw_core *mlxsw_core); +u32 mlxsw_core_read_utc_sec(struct mlxsw_core *mlxsw_core); +u32 mlxsw_core_read_utc_nsec(struct mlxsw_core *mlxsw_core); + +bool mlxsw_core_sdq_supports_cqe_v2(struct mlxsw_core *mlxsw_core); + void mlxsw_core_emad_string_tlv_enable(struct mlxsw_core *mlxsw_core); bool mlxsw_core_res_valid(struct mlxsw_core *mlxsw_core, @@ -476,6 +482,8 @@ struct mlxsw_bus { u8 *p_status); u32 (*read_frc_h)(void *bus_priv); u32 (*read_frc_l)(void *bus_priv); + u32 (*read_utc_sec)(void *bus_priv); + u32 (*read_utc_nsec)(void *bus_priv); u8 features; }; @@ -550,11 +558,17 @@ enum mlxsw_devlink_param_id { MLXSW_DEVLINK_PARAM_ID_ACL_REGION_REHASH_INTERVAL, }; +struct mlxsw_cqe_ts { + u8 sec; + u32 nsec; +}; + struct mlxsw_skb_cb { union { struct mlxsw_tx_info tx_info; struct mlxsw_rx_md_info rx_md_info; }; + struct mlxsw_cqe_ts cqe_ts; }; static inline struct mlxsw_skb_cb *mlxsw_skb_cb(struct sk_buff *skb) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_linecard_dev.c b/drivers/net/ethernet/mellanox/mlxsw/core_linecard_dev.c index 49fee038a99c..af37e650a8ad 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_linecard_dev.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_linecard_dev.c @@ -5,7 +5,6 @@ #include <linux/module.h> #include <linux/err.h> #include <linux/types.h> -#include <linux/err.h> #include <linux/auxiliary_bus.h> #include <linux/idr.h> #include <linux/gfp.h> diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index 83659fb0559a..50527adc5b5a 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -103,6 +103,8 @@ struct mlxsw_pci { struct pci_dev *pdev; u8 __iomem *hw_addr; u64 free_running_clock_offset; + u64 utc_sec_offset; + u64 utc_nsec_offset; struct mlxsw_pci_queue_type_group queues[MLXSW_PCI_QUEUE_TYPE_COUNT]; u32 doorbell_offset; struct mlxsw_core *core; @@ -456,9 +458,9 @@ static void mlxsw_pci_cq_pre_init(struct mlxsw_pci *mlxsw_pci, { q->u.cq.v = mlxsw_pci->max_cqe_ver; - /* For SDQ it is pointless to use CQEv2, so use CQEv1 instead */ if (q->u.cq.v == MLXSW_PCI_CQE_V2 && - q->num < mlxsw_pci->num_sdq_cqs) + q->num < mlxsw_pci->num_sdq_cqs && + !mlxsw_core_sdq_supports_cqe_v2(mlxsw_pci->core)) q->u.cq.v = MLXSW_PCI_CQE_V1; } @@ -511,9 +513,26 @@ static unsigned int mlxsw_pci_read32_off(struct mlxsw_pci *mlxsw_pci, return ioread32be(mlxsw_pci->hw_addr + off); } +static void mlxsw_pci_skb_cb_ts_set(struct mlxsw_pci *mlxsw_pci, + struct sk_buff *skb, + enum mlxsw_pci_cqe_v cqe_v, char *cqe) +{ + if (cqe_v != MLXSW_PCI_CQE_V2) + return; + + if (mlxsw_pci_cqe2_time_stamp_type_get(cqe) != + MLXSW_PCI_CQE_TIME_STAMP_TYPE_UTC) + return; + + mlxsw_skb_cb(skb)->cqe_ts.sec = mlxsw_pci_cqe2_time_stamp_sec_get(cqe); + mlxsw_skb_cb(skb)->cqe_ts.nsec = + mlxsw_pci_cqe2_time_stamp_nsec_get(cqe); +} + static void mlxsw_pci_cqe_sdq_handle(struct mlxsw_pci *mlxsw_pci, struct mlxsw_pci_queue *q, u16 consumer_counter_limit, + enum mlxsw_pci_cqe_v cqe_v, char *cqe) { struct pci_dev *pdev = mlxsw_pci->pdev; @@ -533,6 +552,7 @@ static void mlxsw_pci_cqe_sdq_handle(struct mlxsw_pci *mlxsw_pci, if (unlikely(!tx_info.is_emad && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { + mlxsw_pci_skb_cb_ts_set(mlxsw_pci, skb, cqe_v, cqe); mlxsw_core_ptp_transmitted(mlxsw_pci->core, skb, tx_info.local_port); skb = NULL; @@ -653,6 +673,8 @@ static void mlxsw_pci_cqe_rdq_handle(struct mlxsw_pci *mlxsw_pci, mlxsw_pci_cqe_rdq_md_tx_port_init(skb, cqe); } + mlxsw_pci_skb_cb_ts_set(mlxsw_pci, skb, cqe_v, cqe); + byte_count = mlxsw_pci_cqe_byte_count_get(cqe); if (mlxsw_pci_cqe_crc_get(cqe_v, cqe)) byte_count -= ETH_FCS_LEN; @@ -704,7 +726,7 @@ static void mlxsw_pci_cq_tasklet(struct tasklet_struct *t) sdq = mlxsw_pci_sdq_get(mlxsw_pci, dqn); mlxsw_pci_cqe_sdq_handle(mlxsw_pci, sdq, - wqe_counter, ncqe); + wqe_counter, q->u.cq.v, ncqe); q->u.cq.comp_sdq_count++; } else { struct mlxsw_pci_queue *rdq; @@ -1537,6 +1559,24 @@ static int mlxsw_pci_init(void *bus_priv, struct mlxsw_core *mlxsw_core, mlxsw_pci->free_running_clock_offset = mlxsw_cmd_mbox_query_fw_free_running_clock_offset_get(mbox); + if (mlxsw_cmd_mbox_query_fw_utc_sec_bar_get(mbox) != 0) { + dev_err(&pdev->dev, "Unsupported UTC sec BAR queried from hw\n"); + err = -EINVAL; + goto err_utc_sec_bar; + } + + mlxsw_pci->utc_sec_offset = + mlxsw_cmd_mbox_query_fw_utc_sec_offset_get(mbox); + + if (mlxsw_cmd_mbox_query_fw_utc_nsec_bar_get(mbox) != 0) { + dev_err(&pdev->dev, "Unsupported UTC nsec BAR queried from hw\n"); + err = -EINVAL; + goto err_utc_nsec_bar; + } + + mlxsw_pci->utc_nsec_offset = + mlxsw_cmd_mbox_query_fw_utc_nsec_offset_get(mbox); + num_pages = mlxsw_cmd_mbox_query_fw_fw_pages_get(mbox); err = mlxsw_pci_fw_area_init(mlxsw_pci, mbox, num_pages); if (err) @@ -1601,6 +1641,8 @@ err_query_resources: err_boardinfo: mlxsw_pci_fw_area_fini(mlxsw_pci); err_fw_area_init: +err_utc_nsec_bar: +err_utc_sec_bar: err_fr_rn_clk_bar: err_doorbell_page_bar: err_iface_rev: @@ -1830,6 +1872,20 @@ static u32 mlxsw_pci_read_frc_l(void *bus_priv) return mlxsw_pci_read32_off(mlxsw_pci, frc_offset_l); } +static u32 mlxsw_pci_read_utc_sec(void *bus_priv) +{ + struct mlxsw_pci *mlxsw_pci = bus_priv; + + return mlxsw_pci_read32_off(mlxsw_pci, mlxsw_pci->utc_sec_offset); +} + +static u32 mlxsw_pci_read_utc_nsec(void *bus_priv) +{ + struct mlxsw_pci *mlxsw_pci = bus_priv; + + return mlxsw_pci_read32_off(mlxsw_pci, mlxsw_pci->utc_nsec_offset); +} + static const struct mlxsw_bus mlxsw_pci_bus = { .kind = "pci", .init = mlxsw_pci_init, @@ -1839,6 +1895,8 @@ static const struct mlxsw_bus mlxsw_pci_bus = { .cmd_exec = mlxsw_pci_cmd_exec, .read_frc_h = mlxsw_pci_read_frc_h, .read_frc_l = mlxsw_pci_read_frc_l, + .read_utc_sec = mlxsw_pci_read_utc_sec, + .read_utc_nsec = mlxsw_pci_read_utc_nsec, .features = MLXSW_BUS_F_TXRX | MLXSW_BUS_F_RESET, }; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c index 641078060b02..1e240cdd9cbd 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c @@ -29,6 +29,7 @@ #include <net/pkt_cls.h> #include <net/netevent.h> #include <net/addrconf.h> +#include <linux/ptp_classify.h> #include "spectrum.h" #include "pci.h" @@ -230,8 +231,8 @@ void mlxsw_sp_flow_counter_free(struct mlxsw_sp *mlxsw_sp, counter_index); } -static void mlxsw_sp_txhdr_construct(struct sk_buff *skb, - const struct mlxsw_tx_info *tx_info) +void mlxsw_sp_txhdr_construct(struct sk_buff *skb, + const struct mlxsw_tx_info *tx_info) { char *txhdr = skb_push(skb, MLXSW_TXHDR_LEN); @@ -246,6 +247,82 @@ static void mlxsw_sp_txhdr_construct(struct sk_buff *skb, mlxsw_tx_hdr_type_set(txhdr, MLXSW_TXHDR_TYPE_CONTROL); } +int +mlxsw_sp_txhdr_ptp_data_construct(struct mlxsw_core *mlxsw_core, + struct mlxsw_sp_port *mlxsw_sp_port, + struct sk_buff *skb, + const struct mlxsw_tx_info *tx_info) +{ + char *txhdr; + u16 max_fid; + int err; + + if (skb_cow_head(skb, MLXSW_TXHDR_LEN)) { + err = -ENOMEM; + goto err_skb_cow_head; + } + + if (!MLXSW_CORE_RES_VALID(mlxsw_core, FID)) { + err = -EIO; + goto err_res_valid; + } + max_fid = MLXSW_CORE_RES_GET(mlxsw_core, FID); + + txhdr = skb_push(skb, MLXSW_TXHDR_LEN); + memset(txhdr, 0, MLXSW_TXHDR_LEN); + + mlxsw_tx_hdr_version_set(txhdr, MLXSW_TXHDR_VERSION_1); + mlxsw_tx_hdr_proto_set(txhdr, MLXSW_TXHDR_PROTO_ETH); + mlxsw_tx_hdr_rx_is_router_set(txhdr, true); + mlxsw_tx_hdr_fid_valid_set(txhdr, true); + mlxsw_tx_hdr_fid_set(txhdr, max_fid + tx_info->local_port - 1); + mlxsw_tx_hdr_type_set(txhdr, MLXSW_TXHDR_TYPE_DATA); + return 0; + +err_res_valid: +err_skb_cow_head: + this_cpu_inc(mlxsw_sp_port->pcpu_stats->tx_dropped); + dev_kfree_skb_any(skb); + return err; +} + +static bool mlxsw_sp_skb_requires_ts(struct sk_buff *skb) +{ + unsigned int type; + + if (!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) + return false; + + type = ptp_classify_raw(skb); + return !!ptp_parse_header(skb, type); +} + +static int mlxsw_sp_txhdr_handle(struct mlxsw_core *mlxsw_core, + struct mlxsw_sp_port *mlxsw_sp_port, + struct sk_buff *skb, + const struct mlxsw_tx_info *tx_info) +{ + struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core); + + /* In Spectrum-2 and Spectrum-3, PTP events that require a time stamp + * need special handling and cannot be transmitted as regular control + * packets. + */ + if (unlikely(mlxsw_sp_skb_requires_ts(skb))) + return mlxsw_sp->ptp_ops->txhdr_construct(mlxsw_core, + mlxsw_sp_port, skb, + tx_info); + + if (skb_cow_head(skb, MLXSW_TXHDR_LEN)) { + this_cpu_inc(mlxsw_sp_port->pcpu_stats->tx_dropped); + dev_kfree_skb_any(skb); + return -ENOMEM; + } + + mlxsw_sp_txhdr_construct(skb, tx_info); + return 0; +} + enum mlxsw_reg_spms_state mlxsw_sp_stp_spms_state(u8 state) { switch (state) { @@ -648,12 +725,6 @@ static netdev_tx_t mlxsw_sp_port_xmit(struct sk_buff *skb, u64 len; int err; - if (skb_cow_head(skb, MLXSW_TXHDR_LEN)) { - this_cpu_inc(mlxsw_sp_port->pcpu_stats->tx_dropped); - dev_kfree_skb_any(skb); - return NETDEV_TX_OK; - } - memset(skb->cb, 0, sizeof(struct mlxsw_skb_cb)); if (mlxsw_core_skb_transmit_busy(mlxsw_sp->core, &tx_info)) @@ -664,7 +735,11 @@ static netdev_tx_t mlxsw_sp_port_xmit(struct sk_buff *skb, return NETDEV_TX_OK; } - mlxsw_sp_txhdr_construct(skb, &tx_info); + err = mlxsw_sp_txhdr_handle(mlxsw_sp->core, mlxsw_sp_port, skb, + &tx_info); + if (err) + return NETDEV_TX_OK; + /* TX header is consumed by HW on the way so we shouldn't count its * bytes as being sent. */ @@ -2666,6 +2741,7 @@ static const struct mlxsw_sp_ptp_ops mlxsw_sp1_ptp_ops = { .get_stats_count = mlxsw_sp1_get_stats_count, .get_stats_strings = mlxsw_sp1_get_stats_strings, .get_stats = mlxsw_sp1_get_stats, + .txhdr_construct = mlxsw_sp_ptp_txhdr_construct, }; static const struct mlxsw_sp_ptp_ops mlxsw_sp2_ptp_ops = { @@ -2682,6 +2758,24 @@ static const struct mlxsw_sp_ptp_ops mlxsw_sp2_ptp_ops = { .get_stats_count = mlxsw_sp2_get_stats_count, .get_stats_strings = mlxsw_sp2_get_stats_strings, .get_stats = mlxsw_sp2_get_stats, + .txhdr_construct = mlxsw_sp2_ptp_txhdr_construct, +}; + +static const struct mlxsw_sp_ptp_ops mlxsw_sp4_ptp_ops = { + .clock_init = mlxsw_sp2_ptp_clock_init, + .clock_fini = mlxsw_sp2_ptp_clock_fini, + .init = mlxsw_sp2_ptp_init, + .fini = mlxsw_sp2_ptp_fini, + .receive = mlxsw_sp2_ptp_receive, + .transmitted = mlxsw_sp2_ptp_transmitted, + .hwtstamp_get = mlxsw_sp2_ptp_hwtstamp_get, + .hwtstamp_set = mlxsw_sp2_ptp_hwtstamp_set, + .shaper_work = mlxsw_sp2_ptp_shaper_work, + .get_ts_info = mlxsw_sp2_ptp_get_ts_info, + .get_stats_count = mlxsw_sp2_get_stats_count, + .get_stats_strings = mlxsw_sp2_get_stats_strings, + .get_stats = mlxsw_sp2_get_stats, + .txhdr_construct = mlxsw_sp_ptp_txhdr_construct, }; struct mlxsw_sp_sample_trigger_node { @@ -3327,7 +3421,7 @@ static int mlxsw_sp4_init(struct mlxsw_core *mlxsw_core, mlxsw_sp->sb_vals = &mlxsw_sp2_sb_vals; mlxsw_sp->sb_ops = &mlxsw_sp3_sb_ops; mlxsw_sp->port_type_speed_ops = &mlxsw_sp2_port_type_speed_ops; - mlxsw_sp->ptp_ops = &mlxsw_sp2_ptp_ops; + mlxsw_sp->ptp_ops = &mlxsw_sp4_ptp_ops; mlxsw_sp->span_ops = &mlxsw_sp3_span_ops; mlxsw_sp->policer_core_ops = &mlxsw_sp2_policer_core_ops; mlxsw_sp->trap_ops = &mlxsw_sp2_trap_ops; @@ -3831,6 +3925,7 @@ static struct mlxsw_driver mlxsw_sp1_driver = { .ptp_transmitted = mlxsw_sp_ptp_transmitted, .txhdr_len = MLXSW_TXHDR_LEN, .profile = &mlxsw_sp1_config_profile, + .sdq_supports_cqe_v2 = false, }; static struct mlxsw_driver mlxsw_sp2_driver = { @@ -3869,6 +3964,7 @@ static struct mlxsw_driver mlxsw_sp2_driver = { .ptp_transmitted = mlxsw_sp_ptp_transmitted, .txhdr_len = MLXSW_TXHDR_LEN, .profile = &mlxsw_sp2_config_profile, + .sdq_supports_cqe_v2 = true, }; static struct mlxsw_driver mlxsw_sp3_driver = { @@ -3907,6 +4003,7 @@ static struct mlxsw_driver mlxsw_sp3_driver = { .ptp_transmitted = mlxsw_sp_ptp_transmitted, .txhdr_len = MLXSW_TXHDR_LEN, .profile = &mlxsw_sp2_config_profile, + .sdq_supports_cqe_v2 = true, }; static struct mlxsw_driver mlxsw_sp4_driver = { @@ -3943,6 +4040,7 @@ static struct mlxsw_driver mlxsw_sp4_driver = { .ptp_transmitted = mlxsw_sp_ptp_transmitted, .txhdr_len = MLXSW_TXHDR_LEN, .profile = &mlxsw_sp2_config_profile, + .sdq_supports_cqe_v2 = true, }; bool mlxsw_sp_port_dev_check(const struct net_device *dev) diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h index 50a9380b76e9..c8ff2a6d7e90 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h @@ -243,6 +243,10 @@ struct mlxsw_sp_ptp_ops { void (*get_stats_strings)(u8 **p); void (*get_stats)(struct mlxsw_sp_port *mlxsw_sp_port, u64 *data, int data_index); + int (*txhdr_construct)(struct mlxsw_core *mlxsw_core, + struct mlxsw_sp_port *mlxsw_sp_port, + struct sk_buff *skb, + const struct mlxsw_tx_info *tx_info); }; static inline struct mlxsw_sp_upper * @@ -700,6 +704,12 @@ int mlxsw_sp_flow_counter_alloc(struct mlxsw_sp *mlxsw_sp, unsigned int *p_counter_index); void mlxsw_sp_flow_counter_free(struct mlxsw_sp *mlxsw_sp, unsigned int counter_index); +void mlxsw_sp_txhdr_construct(struct sk_buff *skb, + const struct mlxsw_tx_info *tx_info); +int mlxsw_sp_txhdr_ptp_data_construct(struct mlxsw_core *mlxsw_core, + struct mlxsw_sp_port *mlxsw_sp_port, + struct sk_buff *skb, + const struct mlxsw_tx_info *tx_info); bool mlxsw_sp_port_dev_check(const struct net_device *dev); struct mlxsw_sp *mlxsw_sp_lower_get(struct net_device *dev); struct mlxsw_sp_port *mlxsw_sp_port_dev_lower_find(struct net_device *dev); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c index 5116d7ebe258..2e0b704b8a31 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c @@ -11,6 +11,7 @@ #include <linux/if_ether.h> #include <linux/if_vlan.h> #include <linux/net_tstamp.h> +#include <linux/refcount.h> #include "spectrum.h" #include "spectrum_ptp.h" @@ -39,6 +40,14 @@ struct mlxsw_sp1_ptp_state { u32 gc_cycle; }; +struct mlxsw_sp2_ptp_state { + struct mlxsw_sp_ptp_state common; + refcount_t ptp_port_enabled_ref; /* Number of ports with time stamping + * enabled. + */ + struct hwtstamp_config config; +}; + struct mlxsw_sp1_ptp_key { u16 local_port; u8 message_type; @@ -85,6 +94,13 @@ mlxsw_sp1_ptp_state(struct mlxsw_sp *mlxsw_sp) common); } +static struct mlxsw_sp2_ptp_state * +mlxsw_sp2_ptp_state(struct mlxsw_sp *mlxsw_sp) +{ + return container_of(mlxsw_sp->ptp_state, struct mlxsw_sp2_ptp_state, + common); +} + static struct mlxsw_sp1_ptp_clock * mlxsw_sp1_ptp_clock(struct ptp_clock_info *ptp) { @@ -328,6 +344,153 @@ void mlxsw_sp1_ptp_clock_fini(struct mlxsw_sp_ptp_clock *clock_common) kfree(clock); } +static u64 mlxsw_sp2_ptp_read_utc(struct mlxsw_sp_ptp_clock *clock, + struct ptp_system_timestamp *sts) +{ + struct mlxsw_core *mlxsw_core = clock->core; + u32 utc_sec1, utc_sec2, utc_nsec; + + utc_sec1 = mlxsw_core_read_utc_sec(mlxsw_core); + ptp_read_system_prets(sts); + utc_nsec = mlxsw_core_read_utc_nsec(mlxsw_core); + ptp_read_system_postts(sts); + utc_sec2 = mlxsw_core_read_utc_sec(mlxsw_core); + + if (utc_sec1 != utc_sec2) { + /* Wrap around. */ + ptp_read_system_prets(sts); + utc_nsec = mlxsw_core_read_utc_nsec(mlxsw_core); + ptp_read_system_postts(sts); + } + + return (u64)utc_sec2 * NSEC_PER_SEC + utc_nsec; +} + +static int +mlxsw_sp2_ptp_phc_settime(struct mlxsw_sp_ptp_clock *clock, u64 nsec) +{ + struct mlxsw_core *mlxsw_core = clock->core; + char mtutc_pl[MLXSW_REG_MTUTC_LEN]; + u32 sec, nsec_rem; + + sec = div_u64_rem(nsec, NSEC_PER_SEC, &nsec_rem); + mlxsw_reg_mtutc_pack(mtutc_pl, + MLXSW_REG_MTUTC_OPERATION_SET_TIME_IMMEDIATE, + 0, sec, nsec_rem, 0); + return mlxsw_reg_write(mlxsw_core, MLXSW_REG(mtutc), mtutc_pl); +} + +static int mlxsw_sp2_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) +{ + struct mlxsw_sp_ptp_clock *clock = + container_of(ptp, struct mlxsw_sp_ptp_clock, ptp_info); + s32 ppb = scaled_ppm_to_ppb(scaled_ppm); + + /* In Spectrum-2 and newer ASICs, the frequency adjustment in MTUTC is + * reversed, positive values mean to decrease the frequency. Adjust the + * sign of PPB to this behavior. + */ + return mlxsw_sp_ptp_phc_adjfreq(clock, -ppb); +} + +static int mlxsw_sp2_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) +{ + struct mlxsw_sp_ptp_clock *clock = + container_of(ptp, struct mlxsw_sp_ptp_clock, ptp_info); + struct mlxsw_core *mlxsw_core = clock->core; + char mtutc_pl[MLXSW_REG_MTUTC_LEN]; + + /* HW time adjustment range is s16. If out of range, set time instead. */ + if (delta < S16_MIN || delta > S16_MAX) { + u64 nsec; + + nsec = mlxsw_sp2_ptp_read_utc(clock, NULL); + nsec += delta; + + return mlxsw_sp2_ptp_phc_settime(clock, nsec); + } + + mlxsw_reg_mtutc_pack(mtutc_pl, + MLXSW_REG_MTUTC_OPERATION_ADJUST_TIME, + 0, 0, 0, delta); + return mlxsw_reg_write(mlxsw_core, MLXSW_REG(mtutc), mtutc_pl); +} + +static int mlxsw_sp2_ptp_gettimex(struct ptp_clock_info *ptp, + struct timespec64 *ts, + struct ptp_system_timestamp *sts) +{ + struct mlxsw_sp_ptp_clock *clock = + container_of(ptp, struct mlxsw_sp_ptp_clock, ptp_info); + u64 nsec; + + nsec = mlxsw_sp2_ptp_read_utc(clock, sts); + *ts = ns_to_timespec64(nsec); + + return 0; +} + +static int mlxsw_sp2_ptp_settime(struct ptp_clock_info *ptp, + const struct timespec64 *ts) +{ + struct mlxsw_sp_ptp_clock *clock = + container_of(ptp, struct mlxsw_sp_ptp_clock, ptp_info); + u64 nsec = timespec64_to_ns(ts); + + return mlxsw_sp2_ptp_phc_settime(clock, nsec); +} + +static const struct ptp_clock_info mlxsw_sp2_ptp_clock_info = { + .owner = THIS_MODULE, + .name = "mlxsw_sp_clock", + .max_adj = MLXSW_REG_MTUTC_MAX_FREQ_ADJ, + .adjfine = mlxsw_sp2_ptp_adjfine, + .adjtime = mlxsw_sp2_ptp_adjtime, + .gettimex64 = mlxsw_sp2_ptp_gettimex, + .settime64 = mlxsw_sp2_ptp_settime, +}; + +struct mlxsw_sp_ptp_clock * +mlxsw_sp2_ptp_clock_init(struct mlxsw_sp *mlxsw_sp, struct device *dev) +{ + struct mlxsw_sp_ptp_clock *clock; + int err; + + clock = kzalloc(sizeof(*clock), GFP_KERNEL); + if (!clock) + return ERR_PTR(-ENOMEM); + + clock->core = mlxsw_sp->core; + + clock->ptp_info = mlxsw_sp2_ptp_clock_info; + + err = mlxsw_sp2_ptp_phc_settime(clock, 0); + if (err) { + dev_err(dev, "setting UTC time failed %d\n", err); + goto err_ptp_phc_settime; + } + + clock->ptp = ptp_clock_register(&clock->ptp_info, dev); + if (IS_ERR(clock->ptp)) { + err = PTR_ERR(clock->ptp); + dev_err(dev, "ptp_clock_register failed %d\n", err); + goto err_ptp_clock_register; + } + + return clock; + +err_ptp_clock_register: +err_ptp_phc_settime: + kfree(clock); + return ERR_PTR(err); +} + +void mlxsw_sp2_ptp_clock_fini(struct mlxsw_sp_ptp_clock *clock) +{ + ptp_clock_unregister(clock->ptp); + kfree(clock); +} + static int mlxsw_sp_ptp_parse(struct sk_buff *skb, u8 *p_domain_number, u8 *p_message_type, @@ -835,10 +998,44 @@ static int mlxsw_sp1_ptp_shaper_params_set(struct mlxsw_sp *mlxsw_sp) return 0; } +static int mlxsw_sp_ptp_traps_set(struct mlxsw_sp *mlxsw_sp) +{ + u16 event_message_type; + int err; + + /* Deliver these message types as PTP0. */ + event_message_type = BIT(PTP_MSGTYPE_SYNC) | + BIT(PTP_MSGTYPE_DELAY_REQ) | + BIT(PTP_MSGTYPE_PDELAY_REQ) | + BIT(PTP_MSGTYPE_PDELAY_RESP); + + err = mlxsw_sp_ptp_mtptpt_set(mlxsw_sp, MLXSW_REG_MTPTPT_TRAP_ID_PTP0, + event_message_type); + if (err) + return err; + + /* Everything else is PTP1. */ + err = mlxsw_sp_ptp_mtptpt_set(mlxsw_sp, MLXSW_REG_MTPTPT_TRAP_ID_PTP1, + ~event_message_type); + if (err) + goto err_mtptpt1_set; + + return 0; + +err_mtptpt1_set: + mlxsw_sp_ptp_mtptpt_set(mlxsw_sp, MLXSW_REG_MTPTPT_TRAP_ID_PTP0, 0); + return err; +} + +static void mlxsw_sp_ptp_traps_unset(struct mlxsw_sp *mlxsw_sp) +{ + mlxsw_sp_ptp_mtptpt_set(mlxsw_sp, MLXSW_REG_MTPTPT_TRAP_ID_PTP1, 0); + mlxsw_sp_ptp_mtptpt_set(mlxsw_sp, MLXSW_REG_MTPTPT_TRAP_ID_PTP0, 0); +} + struct mlxsw_sp_ptp_state *mlxsw_sp1_ptp_init(struct mlxsw_sp *mlxsw_sp) { struct mlxsw_sp1_ptp_state *ptp_state; - u16 message_type; int err; err = mlxsw_sp1_ptp_shaper_params_set(mlxsw_sp); @@ -857,22 +1054,9 @@ struct mlxsw_sp_ptp_state *mlxsw_sp1_ptp_init(struct mlxsw_sp *mlxsw_sp) if (err) goto err_hashtable_init; - /* Delive these message types as PTP0. */ - message_type = BIT(PTP_MSGTYPE_SYNC) | - BIT(PTP_MSGTYPE_DELAY_REQ) | - BIT(PTP_MSGTYPE_PDELAY_REQ) | - BIT(PTP_MSGTYPE_PDELAY_RESP); - err = mlxsw_sp_ptp_mtptpt_set(mlxsw_sp, MLXSW_REG_MTPTPT_TRAP_ID_PTP0, - message_type); + err = mlxsw_sp_ptp_traps_set(mlxsw_sp); if (err) - goto err_mtptpt_set; - - /* Everything else is PTP1. */ - message_type = ~message_type; - err = mlxsw_sp_ptp_mtptpt_set(mlxsw_sp, MLXSW_REG_MTPTPT_TRAP_ID_PTP1, - message_type); - if (err) - goto err_mtptpt1_set; + goto err_ptp_traps_set; err = mlxsw_sp1_ptp_set_fifo_clr_on_trap(mlxsw_sp, true); if (err) @@ -884,10 +1068,8 @@ struct mlxsw_sp_ptp_state *mlxsw_sp1_ptp_init(struct mlxsw_sp *mlxsw_sp) return &ptp_state->common; err_fifo_clr: - mlxsw_sp_ptp_mtptpt_set(mlxsw_sp, MLXSW_REG_MTPTPT_TRAP_ID_PTP1, 0); -err_mtptpt1_set: - mlxsw_sp_ptp_mtptpt_set(mlxsw_sp, MLXSW_REG_MTPTPT_TRAP_ID_PTP0, 0); -err_mtptpt_set: + mlxsw_sp_ptp_traps_unset(mlxsw_sp); +err_ptp_traps_set: rhltable_destroy(&ptp_state->unmatched_ht); err_hashtable_init: kfree(ptp_state); @@ -904,8 +1086,7 @@ void mlxsw_sp1_ptp_fini(struct mlxsw_sp_ptp_state *ptp_state_common) cancel_delayed_work_sync(&ptp_state->ht_gc_dw); mlxsw_sp1_ptp_mtpppc_set(mlxsw_sp, 0, 0); mlxsw_sp1_ptp_set_fifo_clr_on_trap(mlxsw_sp, false); - mlxsw_sp_ptp_mtptpt_set(mlxsw_sp, MLXSW_REG_MTPTPT_TRAP_ID_PTP1, 0); - mlxsw_sp_ptp_mtptpt_set(mlxsw_sp, MLXSW_REG_MTPTPT_TRAP_ID_PTP0, 0); + mlxsw_sp_ptp_traps_unset(mlxsw_sp); rhltable_free_and_destroy(&ptp_state->unmatched_ht, &mlxsw_sp1_ptp_unmatched_free_fn, NULL); kfree(ptp_state); @@ -1176,3 +1357,354 @@ void mlxsw_sp1_get_stats(struct mlxsw_sp_port *mlxsw_sp_port, *data++ = *(u64 *)(stats + offset); } } + +struct mlxsw_sp_ptp_state *mlxsw_sp2_ptp_init(struct mlxsw_sp *mlxsw_sp) +{ + struct mlxsw_sp2_ptp_state *ptp_state; + int err; + + ptp_state = kzalloc(sizeof(*ptp_state), GFP_KERNEL); + if (!ptp_state) + return ERR_PTR(-ENOMEM); + + ptp_state->common.mlxsw_sp = mlxsw_sp; + + err = mlxsw_sp_ptp_traps_set(mlxsw_sp); + if (err) + goto err_ptp_traps_set; + + refcount_set(&ptp_state->ptp_port_enabled_ref, 0); + return &ptp_state->common; + +err_ptp_traps_set: + kfree(ptp_state); + return ERR_PTR(err); +} + +void mlxsw_sp2_ptp_fini(struct mlxsw_sp_ptp_state *ptp_state_common) +{ + struct mlxsw_sp *mlxsw_sp = ptp_state_common->mlxsw_sp; + struct mlxsw_sp2_ptp_state *ptp_state; + + ptp_state = mlxsw_sp2_ptp_state(mlxsw_sp); + + mlxsw_sp_ptp_traps_unset(mlxsw_sp); + kfree(ptp_state); +} + +static u32 mlxsw_ptp_utc_time_stamp_sec_get(struct mlxsw_core *mlxsw_core, + u8 cqe_ts_sec) +{ + u32 utc_sec = mlxsw_core_read_utc_sec(mlxsw_core); + + if (cqe_ts_sec > (utc_sec & 0xff)) + /* Time stamp above the last bits of UTC (UTC & 0xff) means the + * latter has wrapped after the time stamp was collected. + */ + utc_sec -= 256; + + utc_sec &= ~0xff; + utc_sec |= cqe_ts_sec; + + return utc_sec; +} + +static void mlxsw_sp2_ptp_hwtstamp_fill(struct mlxsw_core *mlxsw_core, + const struct mlxsw_skb_cb *cb, + struct skb_shared_hwtstamps *hwtstamps) +{ + u64 ts_sec, ts_nsec, nsec; + + WARN_ON_ONCE(!cb->cqe_ts.sec && !cb->cqe_ts.nsec); + + /* The time stamp in the CQE is represented by 38 bits, which is a short + * representation of UTC time. Software should create the full time + * stamp using the global UTC clock. The seconds have only 8 bits in the + * CQE, to create the full time stamp, use the current UTC time and fix + * the seconds according to the relation between UTC seconds and CQE + * seconds. + */ + ts_sec = mlxsw_ptp_utc_time_stamp_sec_get(mlxsw_core, cb->cqe_ts.sec); + ts_nsec = cb->cqe_ts.nsec; + + nsec = ts_sec * NSEC_PER_SEC + ts_nsec; + + hwtstamps->hwtstamp = ns_to_ktime(nsec); +} + +void mlxsw_sp2_ptp_receive(struct mlxsw_sp *mlxsw_sp, struct sk_buff *skb, + u16 local_port) +{ + struct skb_shared_hwtstamps hwtstamps; + + mlxsw_sp2_ptp_hwtstamp_fill(mlxsw_sp->core, mlxsw_skb_cb(skb), + &hwtstamps); + *skb_hwtstamps(skb) = hwtstamps; + mlxsw_sp_rx_listener_no_mark_func(skb, local_port, mlxsw_sp); +} + +void mlxsw_sp2_ptp_transmitted(struct mlxsw_sp *mlxsw_sp, + struct sk_buff *skb, u16 local_port) +{ + struct skb_shared_hwtstamps hwtstamps; + + mlxsw_sp2_ptp_hwtstamp_fill(mlxsw_sp->core, mlxsw_skb_cb(skb), + &hwtstamps); + skb_tstamp_tx(skb, &hwtstamps); + dev_kfree_skb_any(skb); +} + +int mlxsw_sp2_ptp_hwtstamp_get(struct mlxsw_sp_port *mlxsw_sp_port, + struct hwtstamp_config *config) +{ + struct mlxsw_sp2_ptp_state *ptp_state; + + ptp_state = mlxsw_sp2_ptp_state(mlxsw_sp_port->mlxsw_sp); + + *config = ptp_state->config; + return 0; +} + +static int +mlxsw_sp2_ptp_get_message_types(const struct hwtstamp_config *config, + u16 *p_ing_types, u16 *p_egr_types, + enum hwtstamp_rx_filters *p_rx_filter) +{ + enum hwtstamp_rx_filters rx_filter = config->rx_filter; + enum hwtstamp_tx_types tx_type = config->tx_type; + u16 ing_types = 0x00; + u16 egr_types = 0x00; + + *p_rx_filter = rx_filter; + + switch (rx_filter) { + case HWTSTAMP_FILTER_NONE: + ing_types = 0x00; + break; + case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_SYNC: + case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: + case HWTSTAMP_FILTER_PTP_V2_EVENT: + /* In Spectrum-2 and above, all packets get time stamp by + * default and the driver fill the time stamp only for event + * packets. Return all event types even if only specific types + * were required. + */ + ing_types = 0x0f; + *p_rx_filter = HWTSTAMP_FILTER_SOME; + break; + case HWTSTAMP_FILTER_ALL: + case HWTSTAMP_FILTER_SOME: + case HWTSTAMP_FILTER_NTP_ALL: + return -ERANGE; + default: + return -EINVAL; + } + + switch (tx_type) { + case HWTSTAMP_TX_OFF: + egr_types = 0x00; + break; + case HWTSTAMP_TX_ON: + egr_types = 0x0f; + break; + case HWTSTAMP_TX_ONESTEP_SYNC: + case HWTSTAMP_TX_ONESTEP_P2P: + return -ERANGE; + default: + return -EINVAL; + } + + *p_ing_types = ing_types; + *p_egr_types = egr_types; + return 0; +} + +static int mlxsw_sp2_ptp_mtpcpc_set(struct mlxsw_sp *mlxsw_sp, bool ptp_trap_en, + u16 ing_types, u16 egr_types) +{ + char mtpcpc_pl[MLXSW_REG_MTPCPC_LEN]; + + mlxsw_reg_mtpcpc_pack(mtpcpc_pl, false, 0, ptp_trap_en, ing_types, + egr_types); + return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(mtpcpc), mtpcpc_pl); +} + +static int mlxsw_sp2_ptp_enable(struct mlxsw_sp *mlxsw_sp, u16 ing_types, + u16 egr_types, + struct hwtstamp_config new_config) +{ + struct mlxsw_sp2_ptp_state *ptp_state = mlxsw_sp2_ptp_state(mlxsw_sp); + int err; + + err = mlxsw_sp2_ptp_mtpcpc_set(mlxsw_sp, true, ing_types, egr_types); + if (err) + return err; + + ptp_state->config = new_config; + return 0; +} + +static int mlxsw_sp2_ptp_disable(struct mlxsw_sp *mlxsw_sp, + struct hwtstamp_config new_config) +{ + struct mlxsw_sp2_ptp_state *ptp_state = mlxsw_sp2_ptp_state(mlxsw_sp); + int err; + + err = mlxsw_sp2_ptp_mtpcpc_set(mlxsw_sp, false, 0, 0); + if (err) + return err; + + ptp_state->config = new_config; + return 0; +} + +static int mlxsw_sp2_ptp_configure_port(struct mlxsw_sp_port *mlxsw_sp_port, + u16 ing_types, u16 egr_types, + struct hwtstamp_config new_config) +{ + struct mlxsw_sp2_ptp_state *ptp_state; + int err; + + ASSERT_RTNL(); + + ptp_state = mlxsw_sp2_ptp_state(mlxsw_sp_port->mlxsw_sp); + + if (refcount_inc_not_zero(&ptp_state->ptp_port_enabled_ref)) + return 0; + + err = mlxsw_sp2_ptp_enable(mlxsw_sp_port->mlxsw_sp, ing_types, + egr_types, new_config); + if (err) + return err; + + refcount_set(&ptp_state->ptp_port_enabled_ref, 1); + + return 0; +} + +static int mlxsw_sp2_ptp_deconfigure_port(struct mlxsw_sp_port *mlxsw_sp_port, + struct hwtstamp_config new_config) +{ + struct mlxsw_sp2_ptp_state *ptp_state; + int err; + + ASSERT_RTNL(); + + ptp_state = mlxsw_sp2_ptp_state(mlxsw_sp_port->mlxsw_sp); + + if (!refcount_dec_and_test(&ptp_state->ptp_port_enabled_ref)) + return 0; + + err = mlxsw_sp2_ptp_disable(mlxsw_sp_port->mlxsw_sp, new_config); + if (err) + goto err_ptp_disable; + + return 0; + +err_ptp_disable: + refcount_set(&ptp_state->ptp_port_enabled_ref, 1); + return err; +} + +int mlxsw_sp2_ptp_hwtstamp_set(struct mlxsw_sp_port *mlxsw_sp_port, + struct hwtstamp_config *config) +{ + enum hwtstamp_rx_filters rx_filter; + struct hwtstamp_config new_config; + u16 new_ing_types, new_egr_types; + bool ptp_enabled; + int err; + + err = mlxsw_sp2_ptp_get_message_types(config, &new_ing_types, + &new_egr_types, &rx_filter); + if (err) + return err; + + new_config.flags = config->flags; + new_config.tx_type = config->tx_type; + new_config.rx_filter = rx_filter; + + ptp_enabled = mlxsw_sp_port->ptp.ing_types || + mlxsw_sp_port->ptp.egr_types; + + if ((new_ing_types || new_egr_types) && !ptp_enabled) { + err = mlxsw_sp2_ptp_configure_port(mlxsw_sp_port, new_ing_types, + new_egr_types, new_config); + if (err) + return err; + } else if (!new_ing_types && !new_egr_types && ptp_enabled) { + err = mlxsw_sp2_ptp_deconfigure_port(mlxsw_sp_port, new_config); + if (err) + return err; + } + + mlxsw_sp_port->ptp.ing_types = new_ing_types; + mlxsw_sp_port->ptp.egr_types = new_egr_types; + + /* Notify the ioctl caller what we are actually timestamping. */ + config->rx_filter = rx_filter; + + return 0; +} + +int mlxsw_sp2_ptp_get_ts_info(struct mlxsw_sp *mlxsw_sp, + struct ethtool_ts_info *info) +{ + info->phc_index = ptp_clock_index(mlxsw_sp->clock->ptp); + + info->so_timestamping = SOF_TIMESTAMPING_TX_HARDWARE | + SOF_TIMESTAMPING_RX_HARDWARE | + SOF_TIMESTAMPING_RAW_HARDWARE; + + info->tx_types = BIT(HWTSTAMP_TX_OFF) | + BIT(HWTSTAMP_TX_ON); + + info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) | + BIT(HWTSTAMP_FILTER_PTP_V1_L4_EVENT) | + BIT(HWTSTAMP_FILTER_PTP_V2_EVENT); + + return 0; +} + +int mlxsw_sp_ptp_txhdr_construct(struct mlxsw_core *mlxsw_core, + struct mlxsw_sp_port *mlxsw_sp_port, + struct sk_buff *skb, + const struct mlxsw_tx_info *tx_info) +{ + mlxsw_sp_txhdr_construct(skb, tx_info); + return 0; +} + +int mlxsw_sp2_ptp_txhdr_construct(struct mlxsw_core *mlxsw_core, + struct mlxsw_sp_port *mlxsw_sp_port, + struct sk_buff *skb, + const struct mlxsw_tx_info *tx_info) +{ + /* In Spectrum-2 and Spectrum-3, in order for PTP event packets to have + * their correction field correctly set on the egress port they must be + * transmitted as data packets. Such packets ingress the ASIC via the + * CPU port and must have a VLAN tag, as the CPU port is not configured + * with a PVID. Push the default VLAN (4095), which is configured as + * egress untagged on all the ports. + */ + if (!skb_vlan_tagged(skb)) { + skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q), + MLXSW_SP_DEFAULT_VID); + if (!skb) { + this_cpu_inc(mlxsw_sp_port->pcpu_stats->tx_dropped); + return -ENOMEM; + } + } + + return mlxsw_sp_txhdr_ptp_data_construct(mlxsw_core, mlxsw_sp_port, skb, + tx_info); +} diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.h index c06cd1384bca..2d1628fdefc1 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.h +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.h @@ -57,6 +57,40 @@ void mlxsw_sp1_get_stats_strings(u8 **p); void mlxsw_sp1_get_stats(struct mlxsw_sp_port *mlxsw_sp_port, u64 *data, int data_index); +int mlxsw_sp_ptp_txhdr_construct(struct mlxsw_core *mlxsw_core, + struct mlxsw_sp_port *mlxsw_sp_port, + struct sk_buff *skb, + const struct mlxsw_tx_info *tx_info); + +struct mlxsw_sp_ptp_clock * +mlxsw_sp2_ptp_clock_init(struct mlxsw_sp *mlxsw_sp, struct device *dev); + +void mlxsw_sp2_ptp_clock_fini(struct mlxsw_sp_ptp_clock *clock); + +struct mlxsw_sp_ptp_state *mlxsw_sp2_ptp_init(struct mlxsw_sp *mlxsw_sp); + +void mlxsw_sp2_ptp_fini(struct mlxsw_sp_ptp_state *ptp_state); + +void mlxsw_sp2_ptp_receive(struct mlxsw_sp *mlxsw_sp, struct sk_buff *skb, + u16 local_port); + +void mlxsw_sp2_ptp_transmitted(struct mlxsw_sp *mlxsw_sp, + struct sk_buff *skb, u16 local_port); + +int mlxsw_sp2_ptp_hwtstamp_get(struct mlxsw_sp_port *mlxsw_sp_port, + struct hwtstamp_config *config); + +int mlxsw_sp2_ptp_hwtstamp_set(struct mlxsw_sp_port *mlxsw_sp_port, + struct hwtstamp_config *config); + +int mlxsw_sp2_ptp_get_ts_info(struct mlxsw_sp *mlxsw_sp, + struct ethtool_ts_info *info); + +int mlxsw_sp2_ptp_txhdr_construct(struct mlxsw_core *mlxsw_core, + struct mlxsw_sp_port *mlxsw_sp_port, + struct sk_buff *skb, + const struct mlxsw_tx_info *tx_info); + #else static inline struct mlxsw_sp_ptp_clock * @@ -136,7 +170,14 @@ static inline void mlxsw_sp1_get_stats(struct mlxsw_sp_port *mlxsw_sp_port, u64 *data, int data_index) { } -#endif + +int mlxsw_sp_ptp_txhdr_construct(struct mlxsw_core *mlxsw_core, + struct mlxsw_sp_port *mlxsw_sp_port, + struct sk_buff *skb, + const struct mlxsw_tx_info *tx_info) +{ + return -EOPNOTSUPP; +} static inline struct mlxsw_sp_ptp_clock * mlxsw_sp2_ptp_clock_init(struct mlxsw_sp *mlxsw_sp, struct device *dev) @@ -184,16 +225,25 @@ mlxsw_sp2_ptp_hwtstamp_set(struct mlxsw_sp_port *mlxsw_sp_port, return -EOPNOTSUPP; } -static inline void mlxsw_sp2_ptp_shaper_work(struct work_struct *work) -{ -} - static inline int mlxsw_sp2_ptp_get_ts_info(struct mlxsw_sp *mlxsw_sp, struct ethtool_ts_info *info) { return mlxsw_sp_ptp_get_ts_info_noptp(info); } +int mlxsw_sp2_ptp_txhdr_construct(struct mlxsw_core *mlxsw_core, + struct mlxsw_sp_port *mlxsw_sp_port, + struct sk_buff *skb, + const struct mlxsw_tx_info *tx_info) +{ + return -EOPNOTSUPP; +} +#endif + +static inline void mlxsw_sp2_ptp_shaper_work(struct work_struct *work) +{ +} + static inline int mlxsw_sp2_get_stats_count(void) { return 0; diff --git a/drivers/net/ethernet/netronome/nfp/bpf/jit.c b/drivers/net/ethernet/netronome/nfp/bpf/jit.c index e31f8fbbc696..df2ab5cbd49b 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/jit.c +++ b/drivers/net/ethernet/netronome/nfp/bpf/jit.c @@ -4233,7 +4233,7 @@ static void nfp_bpf_opt_ldst_gather(struct nfp_prog *nfp_prog) } /* If the chain is ended by an load/store pair then this - * could serve as the new head of the the next chain. + * could serve as the new head of the next chain. */ if (curr_pair_is_memcpy(meta1, meta2)) { head_ld_meta = meta1; diff --git a/drivers/net/ethernet/sfc/Makefile b/drivers/net/ethernet/sfc/Makefile index 4c759488fc77..bb06fa228367 100644 --- a/drivers/net/ethernet/sfc/Makefile +++ b/drivers/net/ethernet/sfc/Makefile @@ -8,7 +8,8 @@ sfc-y += efx.o efx_common.o efx_channels.o nic.o \ ef100.o ef100_nic.o ef100_netdev.o \ ef100_ethtool.o ef100_rx.o ef100_tx.o sfc-$(CONFIG_SFC_MTD) += mtd.o -sfc-$(CONFIG_SFC_SRIOV) += sriov.o ef10_sriov.o ef100_sriov.o ef100_rep.o mae.o +sfc-$(CONFIG_SFC_SRIOV) += sriov.o ef10_sriov.o ef100_sriov.o ef100_rep.o \ + mae.o tc.o obj-$(CONFIG_SFC) += sfc.o diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c index ab979fd11133..ee734b69150f 100644 --- a/drivers/net/ethernet/sfc/ef10.c +++ b/drivers/net/ethernet/sfc/ef10.c @@ -2538,23 +2538,33 @@ static int efx_ef10_filter_table_probe(struct efx_nic *efx) if (rc) return rc; + down_write(&efx->filter_sem); rc = efx_mcdi_filter_table_probe(efx, nic_data->workaround_26807); if (rc) - return rc; + goto out_unlock; list_for_each_entry(vlan, &nic_data->vlan_list, list) { rc = efx_mcdi_filter_add_vlan(efx, vlan->vid); if (rc) goto fail_add_vlan; } - return 0; + goto out_unlock; fail_add_vlan: efx_mcdi_filter_table_remove(efx); +out_unlock: + up_write(&efx->filter_sem); return rc; } +static void efx_ef10_filter_table_remove(struct efx_nic *efx) +{ + down_write(&efx->filter_sem); + efx_mcdi_filter_table_remove(efx); + up_write(&efx->filter_sem); +} + /* This creates an entry in the RX descriptor queue */ static inline void efx_ef10_build_rx_desc(struct efx_rx_queue *rx_queue, unsigned int index) @@ -3211,9 +3221,7 @@ static int efx_ef10_vport_set_mac_address(struct efx_nic *efx) efx_device_detach_sync(efx); efx_net_stop(efx->net_dev); - down_write(&efx->filter_sem); - efx_mcdi_filter_table_remove(efx); - up_write(&efx->filter_sem); + efx_ef10_filter_table_remove(efx); rc = efx_ef10_vadaptor_free(efx, efx->vport_id); if (rc) @@ -3243,9 +3251,7 @@ restore_vadaptor: if (rc2) goto reset_nic; restore_filters: - down_write(&efx->filter_sem); rc2 = efx_ef10_filter_table_probe(efx); - up_write(&efx->filter_sem); if (rc2) goto reset_nic; @@ -3275,8 +3281,7 @@ static int efx_ef10_set_mac_address(struct efx_nic *efx) efx_net_stop(efx->net_dev); mutex_lock(&efx->mac_lock); - down_write(&efx->filter_sem); - efx_mcdi_filter_table_remove(efx); + efx_ef10_filter_table_remove(efx); ether_addr_copy(MCDI_PTR(inbuf, VADAPTOR_SET_MAC_IN_MACADDR), efx->net_dev->dev_addr); @@ -3286,7 +3291,6 @@ static int efx_ef10_set_mac_address(struct efx_nic *efx) sizeof(inbuf), NULL, 0, NULL); efx_ef10_filter_table_probe(efx); - up_write(&efx->filter_sem); mutex_unlock(&efx->mac_lock); if (was_enabled) @@ -4092,7 +4096,7 @@ const struct efx_nic_type efx_hunt_a0_vf_nic_type = { .ev_test_generate = efx_ef10_ev_test_generate, .filter_table_probe = efx_ef10_filter_table_probe, .filter_table_restore = efx_mcdi_filter_table_restore, - .filter_table_remove = efx_mcdi_filter_table_remove, + .filter_table_remove = efx_ef10_filter_table_remove, .filter_update_rx_scatter = efx_mcdi_update_rx_scatter, .filter_insert = efx_mcdi_filter_insert, .filter_remove_safe = efx_mcdi_filter_remove_safe, diff --git a/drivers/net/ethernet/sfc/ef100.c b/drivers/net/ethernet/sfc/ef100.c index 425017fbcb25..71aab3d0480f 100644 --- a/drivers/net/ethernet/sfc/ef100.c +++ b/drivers/net/ethernet/sfc/ef100.c @@ -431,6 +431,9 @@ static void ef100_pci_remove(struct pci_dev *pci_dev) probe_data = container_of(efx, struct efx_probe_data, efx); ef100_remove_netdev(probe_data); +#ifdef CONFIG_SFC_SRIOV + efx_fini_struct_tc(efx); +#endif ef100_remove(efx); efx_fini_io(efx); diff --git a/drivers/net/ethernet/sfc/ef100_netdev.c b/drivers/net/ethernet/sfc/ef100_netdev.c index 9e65de1ab889..17b9d37218cb 100644 --- a/drivers/net/ethernet/sfc/ef100_netdev.c +++ b/drivers/net/ethernet/sfc/ef100_netdev.c @@ -329,6 +329,10 @@ void ef100_remove_netdev(struct efx_probe_data *probe_data) ef100_unregister_netdev(efx); +#ifdef CONFIG_SFC_SRIOV + efx_fini_tc(efx); +#endif + down_write(&efx->filter_sem); efx_mcdi_filter_table_remove(efx); up_write(&efx->filter_sem); diff --git a/drivers/net/ethernet/sfc/ef100_nic.c b/drivers/net/ethernet/sfc/ef100_nic.c index 4625d35269e6..8061efdaf82c 100644 --- a/drivers/net/ethernet/sfc/ef100_nic.c +++ b/drivers/net/ethernet/sfc/ef100_nic.c @@ -24,6 +24,8 @@ #include "ef100_tx.h" #include "ef100_sriov.h" #include "ef100_netdev.h" +#include "tc.h" +#include "mae.h" #include "rx_common.h" #define EF100_MAX_VIS 4096 @@ -374,26 +376,46 @@ static int ef100_filter_table_up(struct efx_nic *efx) { int rc; + down_write(&efx->filter_sem); rc = efx_mcdi_filter_add_vlan(efx, EFX_FILTER_VID_UNSPEC); - if (rc) { - efx_mcdi_filter_table_down(efx); - return rc; - } + if (rc) + goto fail_unspec; rc = efx_mcdi_filter_add_vlan(efx, 0); - if (rc) { - efx_mcdi_filter_del_vlan(efx, EFX_FILTER_VID_UNSPEC); - efx_mcdi_filter_table_down(efx); - } + if (rc) + goto fail_vlan0; + /* Drop the lock: we've finished altering table existence, and + * filter insertion will need to take the lock for read. + */ + up_write(&efx->filter_sem); +#ifdef CONFIG_SFC_SRIOV + rc = efx_tc_insert_rep_filters(efx); + /* Rep filter failure is nonfatal */ + if (rc) + netif_warn(efx, drv, efx->net_dev, + "Failed to insert representor filters, rc %d\n", + rc); +#endif + return 0; +fail_vlan0: + efx_mcdi_filter_del_vlan(efx, EFX_FILTER_VID_UNSPEC); +fail_unspec: + efx_mcdi_filter_table_down(efx); + up_write(&efx->filter_sem); return rc; } static void ef100_filter_table_down(struct efx_nic *efx) { +#ifdef CONFIG_SFC_SRIOV + efx_tc_remove_rep_filters(efx); +#endif + down_write(&efx->filter_sem); efx_mcdi_filter_del_vlan(efx, 0); efx_mcdi_filter_del_vlan(efx, EFX_FILTER_VID_UNSPEC); efx_mcdi_filter_table_down(efx); + up_write(&efx->filter_sem); } /* Other @@ -704,6 +726,31 @@ static unsigned int efx_ef100_recycle_ring_size(const struct efx_nic *efx) return 10 * EFX_RECYCLE_RING_SIZE_10G; } +#ifdef CONFIG_SFC_SRIOV +static int efx_ef100_get_base_mport(struct efx_nic *efx) +{ + struct ef100_nic_data *nic_data = efx->nic_data; + u32 selector, id; + int rc; + + /* Construct mport selector for "physical network port" */ + efx_mae_mport_wire(efx, &selector); + /* Look up actual mport ID */ + rc = efx_mae_lookup_mport(efx, selector, &id); + if (rc) + return rc; + /* The ID should always fit in 16 bits, because that's how wide the + * corresponding fields in the RX prefix & TX override descriptor are + */ + if (id >> 16) + netif_warn(efx, probe, efx->net_dev, "Bad base m-port id %#x\n", + id); + nic_data->base_mport = id; + nic_data->have_mport = true; + return 0; +} +#endif + static int compare_versions(const char *a, const char *b) { int a_major, a_minor, a_point, a_patch; @@ -1064,6 +1111,34 @@ int ef100_probe_netdev_pf(struct efx_nic *efx) eth_hw_addr_set(net_dev, net_dev->perm_addr); memcpy(nic_data->port_id, net_dev->perm_addr, ETH_ALEN); + if (!nic_data->grp_mae) + return 0; + +#ifdef CONFIG_SFC_SRIOV + rc = efx_init_struct_tc(efx); + if (rc) + return rc; + + rc = efx_ef100_get_base_mport(efx); + if (rc) { + netif_warn(efx, probe, net_dev, + "Failed to probe base mport rc %d; representors will not function\n", + rc); + } + + rc = efx_init_tc(efx); + if (rc) { + /* Either we don't have an MAE at all (i.e. legacy v-switching), + * or we do but we failed to probe it. In the latter case, we + * may not have set up default rules, in which case we won't be + * able to pass any traffic. However, we don't fail the probe, + * because the user might need to use the netdevice to apply + * configuration changes to fix whatever's wrong with the MAE. + */ + netif_warn(efx, probe, net_dev, "Failed to probe MAE rc %d\n", + rc); + } +#endif return 0; fail: diff --git a/drivers/net/ethernet/sfc/ef100_nic.h b/drivers/net/ethernet/sfc/ef100_nic.h index 40f84a275057..0295933145fa 100644 --- a/drivers/net/ethernet/sfc/ef100_nic.h +++ b/drivers/net/ethernet/sfc/ef100_nic.h @@ -72,6 +72,8 @@ struct ef100_nic_data { u8 port_id[ETH_ALEN]; DECLARE_BITMAP(evq_phases, EFX_MAX_CHANNELS); u64 stats[EF100_STAT_COUNT]; + u32 base_mport; + bool have_mport; /* base_mport was populated successfully */ bool grp_mae; /* MAE Privilege */ u16 tso_max_hdr_len; u16 tso_max_payload_num_segs; diff --git a/drivers/net/ethernet/sfc/ef100_rep.c b/drivers/net/ethernet/sfc/ef100_rep.c index d07539f091b8..73ae4656a6e7 100644 --- a/drivers/net/ethernet/sfc/ef100_rep.c +++ b/drivers/net/ethernet/sfc/ef100_rep.c @@ -13,15 +13,24 @@ #include "ef100_netdev.h" #include "ef100_nic.h" #include "mae.h" +#include "rx_common.h" #define EFX_EF100_REP_DRIVER "efx_ef100_rep" +#define EFX_REP_DEFAULT_PSEUDO_RING_SIZE 64 + +static int efx_ef100_rep_poll(struct napi_struct *napi, int weight); + static int efx_ef100_rep_init_struct(struct efx_nic *efx, struct efx_rep *efv, unsigned int i) { efv->parent = efx; efv->idx = i; INIT_LIST_HEAD(&efv->list); + efv->dflt.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; + INIT_LIST_HEAD(&efv->dflt.acts.list); + INIT_LIST_HEAD(&efv->rx_list); + spin_lock_init(&efv->rx_lock); efv->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP | NETIF_MSG_RX_ERR | @@ -29,6 +38,25 @@ static int efx_ef100_rep_init_struct(struct efx_nic *efx, struct efx_rep *efv, return 0; } +static int efx_ef100_rep_open(struct net_device *net_dev) +{ + struct efx_rep *efv = netdev_priv(net_dev); + + netif_napi_add(net_dev, &efv->napi, efx_ef100_rep_poll, + NAPI_POLL_WEIGHT); + napi_enable(&efv->napi); + return 0; +} + +static int efx_ef100_rep_close(struct net_device *net_dev) +{ + struct efx_rep *efv = netdev_priv(net_dev); + + napi_disable(&efv->napi); + netif_napi_del(&efv->napi); + return 0; +} + static netdev_tx_t efx_ef100_rep_xmit(struct sk_buff *skb, struct net_device *dev) { @@ -79,10 +107,26 @@ static int efx_ef100_rep_get_phys_port_name(struct net_device *dev, return 0; } +static void efx_ef100_rep_get_stats64(struct net_device *dev, + struct rtnl_link_stats64 *stats) +{ + struct efx_rep *efv = netdev_priv(dev); + + stats->rx_packets = atomic64_read(&efv->stats.rx_packets); + stats->tx_packets = atomic64_read(&efv->stats.tx_packets); + stats->rx_bytes = atomic64_read(&efv->stats.rx_bytes); + stats->tx_bytes = atomic64_read(&efv->stats.tx_bytes); + stats->rx_dropped = atomic64_read(&efv->stats.rx_dropped); + stats->tx_errors = atomic64_read(&efv->stats.tx_errors); +} + static const struct net_device_ops efx_ef100_rep_netdev_ops = { + .ndo_open = efx_ef100_rep_open, + .ndo_stop = efx_ef100_rep_close, .ndo_start_xmit = efx_ef100_rep_xmit, .ndo_get_port_parent_id = efx_ef100_rep_get_port_parent_id, .ndo_get_phys_port_name = efx_ef100_rep_get_phys_port_name, + .ndo_get_stats64 = efx_ef100_rep_get_stats64, }; static void efx_ef100_rep_get_drvinfo(struct net_device *dev, @@ -106,10 +150,37 @@ static void efx_ef100_rep_ethtool_set_msglevel(struct net_device *net_dev, efv->msg_enable = msg_enable; } +static void efx_ef100_rep_ethtool_get_ringparam(struct net_device *net_dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kring, + struct netlink_ext_ack *ext_ack) +{ + struct efx_rep *efv = netdev_priv(net_dev); + + ring->rx_max_pending = U32_MAX; + ring->rx_pending = efv->rx_pring_size; +} + +static int efx_ef100_rep_ethtool_set_ringparam(struct net_device *net_dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kring, + struct netlink_ext_ack *ext_ack) +{ + struct efx_rep *efv = netdev_priv(net_dev); + + if (ring->rx_mini_pending || ring->rx_jumbo_pending || ring->tx_pending) + return -EINVAL; + + efv->rx_pring_size = ring->rx_pending; + return 0; +} + static const struct ethtool_ops efx_ef100_rep_ethtool_ops = { .get_drvinfo = efx_ef100_rep_get_drvinfo, .get_msglevel = efx_ef100_rep_ethtool_get_msglevel, .set_msglevel = efx_ef100_rep_ethtool_set_msglevel, + .get_ringparam = efx_ef100_rep_ethtool_get_ringparam, + .set_ringparam = efx_ef100_rep_ethtool_set_ringparam, }; static struct efx_rep *efx_ef100_rep_create_netdev(struct efx_nic *efx, @@ -159,6 +230,7 @@ static int efx_ef100_configure_rep(struct efx_rep *efv) u32 selector; int rc; + efv->rx_pring_size = EFX_REP_DEFAULT_PSEUDO_RING_SIZE; /* Construct mport selector for corresponding VF */ efx_mae_mport_vf(efx, efv->idx, &selector); /* Look up actual mport ID */ @@ -169,7 +241,14 @@ static int efx_ef100_configure_rep(struct efx_rep *efv) /* mport label should fit in 16 bits */ WARN_ON(efv->mport >> 16); - return 0; + return efx_tc_configure_default_rule_rep(efv); +} + +static void efx_ef100_deconfigure_rep(struct efx_rep *efv) +{ + struct efx_nic *efx = efv->parent; + + efx_tc_deconfigure_default_rule(efx, &efv->dflt); } static void efx_ef100_rep_destroy_netdev(struct efx_rep *efv) @@ -181,6 +260,7 @@ static void efx_ef100_rep_destroy_netdev(struct efx_rep *efv) list_del(&efv->list); spin_unlock_bh(&efx->vf_reps_lock); rtnl_unlock(); + synchronize_rcu(); free_netdev(efv->net_dev); } @@ -202,19 +282,21 @@ int efx_ef100_vfrep_create(struct efx_nic *efx, unsigned int i) pci_err(efx->pci_dev, "Failed to configure representor for VF %d, rc %d\n", i, rc); - goto fail; + goto fail1; } rc = register_netdev(efv->net_dev); if (rc) { pci_err(efx->pci_dev, "Failed to register representor for VF %d, rc %d\n", i, rc); - goto fail; + goto fail2; } pci_dbg(efx->pci_dev, "Representor for VF %d is %s\n", i, efv->net_dev->name); return 0; -fail: +fail2: + efx_ef100_deconfigure_rep(efv); +fail1: efx_ef100_rep_destroy_netdev(efv); return rc; } @@ -228,6 +310,7 @@ void efx_ef100_vfrep_destroy(struct efx_nic *efx, struct efx_rep *efv) return; netif_dbg(efx, drv, rep_dev, "Removing VF representor\n"); unregister_netdev(rep_dev); + efx_ef100_deconfigure_rep(efv); efx_ef100_rep_destroy_netdev(efv); } @@ -242,3 +325,111 @@ void efx_ef100_fini_vfreps(struct efx_nic *efx) list_for_each_entry_safe(efv, next, &efx->vf_reps, list) efx_ef100_vfrep_destroy(efx, efv); } + +static int efx_ef100_rep_poll(struct napi_struct *napi, int weight) +{ + struct efx_rep *efv = container_of(napi, struct efx_rep, napi); + unsigned int read_index; + struct list_head head; + struct sk_buff *skb; + bool need_resched; + int spent = 0; + + INIT_LIST_HEAD(&head); + /* Grab up to 'weight' pending SKBs */ + spin_lock_bh(&efv->rx_lock); + read_index = efv->write_index; + while (spent < weight && !list_empty(&efv->rx_list)) { + skb = list_first_entry(&efv->rx_list, struct sk_buff, list); + list_del(&skb->list); + list_add_tail(&skb->list, &head); + spent++; + } + spin_unlock_bh(&efv->rx_lock); + /* Receive them */ + netif_receive_skb_list(&head); + if (spent < weight) + if (napi_complete_done(napi, spent)) { + spin_lock_bh(&efv->rx_lock); + efv->read_index = read_index; + /* If write_index advanced while we were doing the + * RX, then storing our read_index won't re-prime the + * fake-interrupt. In that case, we need to schedule + * NAPI again to consume the additional packet(s). + */ + need_resched = efv->write_index != read_index; + spin_unlock_bh(&efv->rx_lock); + if (need_resched) + napi_schedule(&efv->napi); + } + return spent; +} + +void efx_ef100_rep_rx_packet(struct efx_rep *efv, struct efx_rx_buffer *rx_buf) +{ + u8 *eh = efx_rx_buf_va(rx_buf); + struct sk_buff *skb; + bool primed; + + /* Don't allow too many queued SKBs to build up, as they consume + * GFP_ATOMIC memory. If we overrun, just start dropping. + */ + if (efv->write_index - READ_ONCE(efv->read_index) > efv->rx_pring_size) { + atomic64_inc(&efv->stats.rx_dropped); + if (net_ratelimit()) + netif_dbg(efv->parent, rx_err, efv->net_dev, + "nodesc-dropped packet of length %u\n", + rx_buf->len); + return; + } + + skb = netdev_alloc_skb(efv->net_dev, rx_buf->len); + if (!skb) { + atomic64_inc(&efv->stats.rx_dropped); + if (net_ratelimit()) + netif_dbg(efv->parent, rx_err, efv->net_dev, + "noskb-dropped packet of length %u\n", + rx_buf->len); + return; + } + memcpy(skb->data, eh, rx_buf->len); + __skb_put(skb, rx_buf->len); + + skb_record_rx_queue(skb, 0); /* rep is single-queue */ + + /* Move past the ethernet header */ + skb->protocol = eth_type_trans(skb, efv->net_dev); + + skb_checksum_none_assert(skb); + + atomic64_inc(&efv->stats.rx_packets); + atomic64_add(rx_buf->len, &efv->stats.rx_bytes); + + /* Add it to the rx list */ + spin_lock_bh(&efv->rx_lock); + primed = efv->read_index == efv->write_index; + list_add_tail(&skb->list, &efv->rx_list); + efv->write_index++; + spin_unlock_bh(&efv->rx_lock); + /* Trigger rx work */ + if (primed) + napi_schedule(&efv->napi); +} + +struct efx_rep *efx_ef100_find_rep_by_mport(struct efx_nic *efx, u16 mport) +{ + struct efx_rep *efv, *out = NULL; + + /* spinlock guards against list mutation while we're walking it; + * but caller must also hold rcu_read_lock() to ensure the netdev + * isn't freed after we drop the spinlock. + */ + spin_lock_bh(&efx->vf_reps_lock); + list_for_each_entry(efv, &efx->vf_reps, list) + if (efv->mport == mport) { + out = efv; + break; + } + spin_unlock_bh(&efx->vf_reps_lock); + return out; +} diff --git a/drivers/net/ethernet/sfc/ef100_rep.h b/drivers/net/ethernet/sfc/ef100_rep.h index d47fd8ff6220..070f700893c1 100644 --- a/drivers/net/ethernet/sfc/ef100_rep.h +++ b/drivers/net/ethernet/sfc/ef100_rep.h @@ -14,6 +14,7 @@ #define EF100_REP_H #include "net_driver.h" +#include "tc.h" struct efx_rep_sw_stats { atomic64_t rx_packets, tx_packets; @@ -29,7 +30,14 @@ struct efx_rep_sw_stats { * @msg_enable: log message enable flags * @mport: m-port ID of corresponding VF * @idx: VF index + * @write_index: number of packets enqueued to @rx_list + * @read_index: number of packets consumed from @rx_list + * @rx_pring_size: max length of RX list + * @dflt: default-rule for MAE switching * @list: entry on efx->vf_reps + * @rx_list: list of SKBs queued for receive in NAPI poll + * @rx_lock: protects @rx_list + * @napi: NAPI control structure * @stats: software traffic counters for netdev stats */ struct efx_rep { @@ -38,7 +46,13 @@ struct efx_rep { u32 msg_enable; u32 mport; unsigned int idx; + unsigned int write_index, read_index; + unsigned int rx_pring_size; + struct efx_tc_flow_rule dflt; struct list_head list; + struct list_head rx_list; + spinlock_t rx_lock; + struct napi_struct napi; struct efx_rep_sw_stats stats; }; @@ -46,4 +60,10 @@ int efx_ef100_vfrep_create(struct efx_nic *efx, unsigned int i); void efx_ef100_vfrep_destroy(struct efx_nic *efx, struct efx_rep *efv); void efx_ef100_fini_vfreps(struct efx_nic *efx); +void efx_ef100_rep_rx_packet(struct efx_rep *efv, struct efx_rx_buffer *rx_buf); +/* Returns the representor corresponding to a VF m-port, or NULL + * @mport is an m-port label, *not* an m-port ID! + * Caller must hold rcu_read_lock(). + */ +struct efx_rep *efx_ef100_find_rep_by_mport(struct efx_nic *efx, u16 mport); #endif /* EF100_REP_H */ diff --git a/drivers/net/ethernet/sfc/ef100_rx.c b/drivers/net/ethernet/sfc/ef100_rx.c index 85207acf7dee..65bbe37753e6 100644 --- a/drivers/net/ethernet/sfc/ef100_rx.c +++ b/drivers/net/ethernet/sfc/ef100_rx.c @@ -55,10 +55,14 @@ static bool ef100_has_fcs_error(struct efx_channel *channel, u32 *prefix) void __ef100_rx_packet(struct efx_channel *channel) { - struct efx_rx_buffer *rx_buf = efx_rx_buffer(&channel->rx_queue, channel->rx_pkt_index); + struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel); + struct efx_rx_buffer *rx_buf = efx_rx_buffer(rx_queue, + channel->rx_pkt_index); struct efx_nic *efx = channel->efx; + struct ef100_nic_data *nic_data; u8 *eh = efx_rx_buf_va(rx_buf); __wsum csum = 0; + u16 ing_port; u32 *prefix; prefix = (u32 *)(eh - ESE_GZ_RX_PKT_PREFIX_LEN); @@ -76,6 +80,37 @@ void __ef100_rx_packet(struct efx_channel *channel) goto out; } + ing_port = le16_to_cpu((__force __le16) PREFIX_FIELD(prefix, INGRESS_MPORT)); + + nic_data = efx->nic_data; + + if (nic_data->have_mport && ing_port != nic_data->base_mport) { +#ifdef CONFIG_SFC_SRIOV + struct efx_rep *efv; + + rcu_read_lock(); + efv = efx_ef100_find_rep_by_mport(efx, ing_port); + if (efv) { + if (efv->net_dev->flags & IFF_UP) + efx_ef100_rep_rx_packet(efv, rx_buf); + rcu_read_unlock(); + /* Representor Rx doesn't care about PF Rx buffer + * ownership, it just makes a copy. So, we are done + * with the Rx buffer from PF point of view and should + * free it. + */ + goto free_rx_buffer; + } + rcu_read_unlock(); +#endif + if (net_ratelimit()) + netif_warn(efx, drv, efx->net_dev, + "Unrecognised ing_port %04x (base %04x), dropping\n", + ing_port, nic_data->base_mport); + channel->n_rx_mport_bad++; + goto free_rx_buffer; + } + if (likely(efx->net_dev->features & NETIF_F_RXCSUM)) { if (PREFIX_FIELD(prefix, NT_OR_INNER_L3_CLASS) == 1) { ++channel->n_rx_ip_hdr_chksum_err; @@ -87,17 +122,16 @@ void __ef100_rx_packet(struct efx_channel *channel) } if (channel->type->receive_skb) { - struct efx_rx_queue *rx_queue = - efx_channel_get_rx_queue(channel); - /* no support for special channels yet, so just discard */ WARN_ON_ONCE(1); - efx_free_rx_buffers(rx_queue, rx_buf, 1); - goto out; + goto free_rx_buffer; } efx_rx_packet_gro(channel, rx_buf, channel->rx_pkt_n_frags, eh, csum); + goto out; +free_rx_buffer: + efx_free_rx_buffers(rx_queue, rx_buf, 1); out: channel->rx_pkt_n_frags = 0; } diff --git a/drivers/net/ethernet/sfc/ef10_sriov.c b/drivers/net/ethernet/sfc/ef10_sriov.c index 92550c7e85ce..9aae0d8b713f 100644 --- a/drivers/net/ethernet/sfc/ef10_sriov.c +++ b/drivers/net/ethernet/sfc/ef10_sriov.c @@ -501,14 +501,11 @@ int efx_ef10_sriov_set_vf_mac(struct efx_nic *efx, int vf_i, const u8 *mac) efx_device_detach_sync(vf->efx); efx_net_stop(vf->efx->net_dev); - down_write(&vf->efx->filter_sem); vf->efx->type->filter_table_remove(vf->efx); rc = efx_ef10_vadaptor_free(vf->efx, EVB_PORT_ID_ASSIGNED); - if (rc) { - up_write(&vf->efx->filter_sem); + if (rc) return rc; - } } rc = efx_ef10_evb_port_assign(efx, EVB_PORT_ID_NULL, vf_i); @@ -539,12 +536,9 @@ int efx_ef10_sriov_set_vf_mac(struct efx_nic *efx, int vf_i, const u8 *mac) if (vf->efx) { /* VF cannot use the vport_id that the PF created */ rc = efx_ef10_vadaptor_alloc(vf->efx, EVB_PORT_ID_ASSIGNED); - if (rc) { - up_write(&vf->efx->filter_sem); + if (rc) return rc; - } vf->efx->type->filter_table_probe(vf->efx); - up_write(&vf->efx->filter_sem); efx_net_open(vf->efx->net_dev); efx_device_attach_if_not_resetting(vf->efx); } @@ -580,7 +574,6 @@ int efx_ef10_sriov_set_vf_vlan(struct efx_nic *efx, int vf_i, u16 vlan, efx_net_stop(vf->efx->net_dev); mutex_lock(&vf->efx->mac_lock); - down_write(&vf->efx->filter_sem); vf->efx->type->filter_table_remove(vf->efx); rc = efx_ef10_vadaptor_free(vf->efx, EVB_PORT_ID_ASSIGNED); @@ -654,7 +647,6 @@ restore_filters: if (rc2) goto reset_nic_up_write; - up_write(&vf->efx->filter_sem); mutex_unlock(&vf->efx->mac_lock); rc2 = efx_net_open(vf->efx->net_dev); @@ -666,10 +658,8 @@ restore_filters: return rc; reset_nic_up_write: - if (vf->efx) { - up_write(&vf->efx->filter_sem); + if (vf->efx) mutex_unlock(&vf->efx->mac_lock); - } reset_nic: if (vf->efx) { netif_err(efx, drv, efx->net_dev, diff --git a/drivers/net/ethernet/sfc/ethtool_common.c b/drivers/net/ethernet/sfc/ethtool_common.c index 58ad9d665805..bc840ede3053 100644 --- a/drivers/net/ethernet/sfc/ethtool_common.c +++ b/drivers/net/ethernet/sfc/ethtool_common.c @@ -91,6 +91,7 @@ static const struct efx_sw_stat_desc efx_sw_stat_desc[] = { EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_xdp_bad_drops), EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_xdp_tx), EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_xdp_redirect), + EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_mport_bad), #ifdef CONFIG_RFS_ACCEL EFX_ETHTOOL_UINT_CHANNEL_STAT_NO_N(rfs_filter_count), EFX_ETHTOOL_UINT_CHANNEL_STAT(rfs_succeeded), diff --git a/drivers/net/ethernet/sfc/filter.h b/drivers/net/ethernet/sfc/filter.h index 40b2af8bfb81..4d928839d292 100644 --- a/drivers/net/ethernet/sfc/filter.h +++ b/drivers/net/ethernet/sfc/filter.h @@ -88,6 +88,7 @@ enum efx_filter_priority { * the automatic filter in its place. * @EFX_FILTER_FLAG_RX: Filter is for RX * @EFX_FILTER_FLAG_TX: Filter is for TX + * @EFX_FILTER_FLAG_VPORT_ID: Virtual port ID for adapter switching. */ enum efx_filter_flags { EFX_FILTER_FLAG_RX_RSS = 0x01, @@ -95,6 +96,7 @@ enum efx_filter_flags { EFX_FILTER_FLAG_RX_OVER_AUTO = 0x04, EFX_FILTER_FLAG_RX = 0x08, EFX_FILTER_FLAG_TX = 0x10, + EFX_FILTER_FLAG_VPORT_ID = 0x20, }; /** enum efx_encap_type - types of encapsulation @@ -127,6 +129,9 @@ enum efx_encap_type { * MCFW context_id. * @dmaq_id: Source/target queue index, or %EFX_FILTER_RX_DMAQ_ID_DROP for * an RX drop filter + * @vport_id: Virtual port ID associated with RX queue, for adapter switching, + * if %EFX_FILTER_FLAG_VPORT_ID is set. This is an MCFW vport_id, or on + * EF100 an mport selector. * @outer_vid: Outer VLAN ID to match, if %EFX_FILTER_MATCH_OUTER_VID is set * @inner_vid: Inner VLAN ID to match, if %EFX_FILTER_MATCH_INNER_VID is set * @loc_mac: Local MAC address to match, if %EFX_FILTER_MATCH_LOC_MAC or @@ -156,6 +161,7 @@ struct efx_filter_spec { u32 priority:2; u32 flags:6; u32 dmaq_id:12; + u32 vport_id; u32 rss_context; __be16 outer_vid __aligned(4); /* allow jhash2() of match values */ __be16 inner_vid; @@ -292,6 +298,18 @@ static inline int efx_filter_set_mc_def(struct efx_filter_spec *spec) return 0; } +/** + * efx_filter_set_vport_id - override virtual port id relating to filter + * @spec: Specification to initialise + * @vport_id: firmware ID of the virtual port + */ +static inline void efx_filter_set_vport_id(struct efx_filter_spec *spec, + u32 vport_id) +{ + spec->flags |= EFX_FILTER_FLAG_VPORT_ID; + spec->vport_id = vport_id; +} + static inline void efx_filter_set_encap_type(struct efx_filter_spec *spec, enum efx_encap_type encap_type) { diff --git a/drivers/net/ethernet/sfc/mae.c b/drivers/net/ethernet/sfc/mae.c index 011ebd46ada5..97627f5e3674 100644 --- a/drivers/net/ethernet/sfc/mae.c +++ b/drivers/net/ethernet/sfc/mae.c @@ -11,7 +11,65 @@ #include "mae.h" #include "mcdi.h" -#include "mcdi_pcol.h" +#include "mcdi_pcol_mae.h" + +int efx_mae_allocate_mport(struct efx_nic *efx, u32 *id, u32 *label) +{ + MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_MPORT_ALLOC_ALIAS_OUT_LEN); + MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_MPORT_ALLOC_ALIAS_IN_LEN); + size_t outlen; + int rc; + + if (WARN_ON_ONCE(!id)) + return -EINVAL; + if (WARN_ON_ONCE(!label)) + return -EINVAL; + + MCDI_SET_DWORD(inbuf, MAE_MPORT_ALLOC_ALIAS_IN_TYPE, + MC_CMD_MAE_MPORT_ALLOC_ALIAS_IN_MPORT_TYPE_ALIAS); + MCDI_SET_DWORD(inbuf, MAE_MPORT_ALLOC_ALIAS_IN_DELIVER_MPORT, + MAE_MPORT_SELECTOR_ASSIGNED); + rc = efx_mcdi_rpc(efx, MC_CMD_MAE_MPORT_ALLOC, inbuf, sizeof(inbuf), + outbuf, sizeof(outbuf), &outlen); + if (rc) + return rc; + if (outlen < sizeof(outbuf)) + return -EIO; + *id = MCDI_DWORD(outbuf, MAE_MPORT_ALLOC_ALIAS_OUT_MPORT_ID); + *label = MCDI_DWORD(outbuf, MAE_MPORT_ALLOC_ALIAS_OUT_LABEL); + return 0; +} + +int efx_mae_free_mport(struct efx_nic *efx, u32 id) +{ + MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_MPORT_FREE_IN_LEN); + + BUILD_BUG_ON(MC_CMD_MAE_MPORT_FREE_OUT_LEN); + MCDI_SET_DWORD(inbuf, MAE_MPORT_FREE_IN_MPORT_ID, id); + return efx_mcdi_rpc(efx, MC_CMD_MAE_MPORT_FREE, inbuf, sizeof(inbuf), + NULL, 0, NULL); +} + +void efx_mae_mport_wire(struct efx_nic *efx, u32 *out) +{ + efx_dword_t mport; + + EFX_POPULATE_DWORD_2(mport, + MAE_MPORT_SELECTOR_TYPE, MAE_MPORT_SELECTOR_TYPE_PPORT, + MAE_MPORT_SELECTOR_PPORT_ID, efx->port_num); + *out = EFX_DWORD_VAL(mport); +} + +void efx_mae_mport_uplink(struct efx_nic *efx __always_unused, u32 *out) +{ + efx_dword_t mport; + + EFX_POPULATE_DWORD_3(mport, + MAE_MPORT_SELECTOR_TYPE, MAE_MPORT_SELECTOR_TYPE_FUNC, + MAE_MPORT_SELECTOR_FUNC_PF_ID, MAE_MPORT_SELECTOR_FUNC_PF_ID_CALLER, + MAE_MPORT_SELECTOR_FUNC_VF_ID, MAE_MPORT_SELECTOR_FUNC_VF_ID_NULL); + *out = EFX_DWORD_VAL(mport); +} void efx_mae_mport_vf(struct efx_nic *efx __always_unused, u32 vf_id, u32 *out) { @@ -24,6 +82,17 @@ void efx_mae_mport_vf(struct efx_nic *efx __always_unused, u32 vf_id, u32 *out) *out = EFX_DWORD_VAL(mport); } +/* Constructs an mport selector from an mport ID, because they're not the same */ +void efx_mae_mport_mport(struct efx_nic *efx __always_unused, u32 mport_id, u32 *out) +{ + efx_dword_t mport; + + EFX_POPULATE_DWORD_2(mport, + MAE_MPORT_SELECTOR_TYPE, MAE_MPORT_SELECTOR_TYPE_MPORT_ID, + MAE_MPORT_SELECTOR_MPORT_ID, mport_id); + *out = EFX_DWORD_VAL(mport); +} + /* id is really only 24 bits wide */ int efx_mae_lookup_mport(struct efx_nic *efx, u32 selector, u32 *id) { @@ -42,3 +111,236 @@ int efx_mae_lookup_mport(struct efx_nic *efx, u32 selector, u32 *id) *id = MCDI_DWORD(outbuf, MAE_MPORT_LOOKUP_OUT_MPORT_ID); return 0; } + +static bool efx_mae_asl_id(u32 id) +{ + return !!(id & BIT(31)); +} + +int efx_mae_alloc_action_set(struct efx_nic *efx, struct efx_tc_action_set *act) +{ + MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_ACTION_SET_ALLOC_OUT_LEN); + MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_ACTION_SET_ALLOC_IN_LEN); + size_t outlen; + int rc; + + MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_SRC_MAC_ID, + MC_CMD_MAE_MAC_ADDR_ALLOC_OUT_MAC_ID_NULL); + MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_DST_MAC_ID, + MC_CMD_MAE_MAC_ADDR_ALLOC_OUT_MAC_ID_NULL); + MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_COUNTER_ID, + MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_NULL); + MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_COUNTER_LIST_ID, + MC_CMD_MAE_COUNTER_LIST_ALLOC_OUT_COUNTER_LIST_ID_NULL); + MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_ENCAP_HEADER_ID, + MC_CMD_MAE_ENCAP_HEADER_ALLOC_OUT_ENCAP_HEADER_ID_NULL); + if (act->deliver) + MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_DELIVER, + act->dest_mport); + BUILD_BUG_ON(MAE_MPORT_SELECTOR_NULL); + rc = efx_mcdi_rpc(efx, MC_CMD_MAE_ACTION_SET_ALLOC, inbuf, sizeof(inbuf), + outbuf, sizeof(outbuf), &outlen); + if (rc) + return rc; + if (outlen < sizeof(outbuf)) + return -EIO; + act->fw_id = MCDI_DWORD(outbuf, MAE_ACTION_SET_ALLOC_OUT_AS_ID); + /* We rely on the high bit of AS IDs always being clear. + * The firmware API guarantees this, but let's check it ourselves. + */ + if (WARN_ON_ONCE(efx_mae_asl_id(act->fw_id))) { + efx_mae_free_action_set(efx, act->fw_id); + return -EIO; + } + return 0; +} + +int efx_mae_free_action_set(struct efx_nic *efx, u32 fw_id) +{ + MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_ACTION_SET_FREE_OUT_LEN(1)); + MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_ACTION_SET_FREE_IN_LEN(1)); + size_t outlen; + int rc; + + MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_FREE_IN_AS_ID, fw_id); + rc = efx_mcdi_rpc(efx, MC_CMD_MAE_ACTION_SET_FREE, inbuf, sizeof(inbuf), + outbuf, sizeof(outbuf), &outlen); + if (rc) + return rc; + if (outlen < sizeof(outbuf)) + return -EIO; + /* FW freed a different ID than we asked for, should never happen. + * Warn because it means we've now got a different idea to the FW of + * what action-sets exist, which could cause mayhem later. + */ + if (WARN_ON(MCDI_DWORD(outbuf, MAE_ACTION_SET_FREE_OUT_FREED_AS_ID) != fw_id)) + return -EIO; + return 0; +} + +int efx_mae_alloc_action_set_list(struct efx_nic *efx, + struct efx_tc_action_set_list *acts) +{ + MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_LEN); + struct efx_tc_action_set *act; + size_t inlen, outlen, i = 0; + efx_dword_t *inbuf; + int rc; + + list_for_each_entry(act, &acts->list, list) + i++; + if (i == 0) + return -EINVAL; + if (i == 1) { + /* Don't wrap an ASL around a single AS, just use the AS_ID + * directly. ASLs are a more limited resource. + */ + act = list_first_entry(&acts->list, struct efx_tc_action_set, list); + acts->fw_id = act->fw_id; + return 0; + } + if (i > MC_CMD_MAE_ACTION_SET_LIST_ALLOC_IN_AS_IDS_MAXNUM_MCDI2) + return -EOPNOTSUPP; /* Too many actions */ + inlen = MC_CMD_MAE_ACTION_SET_LIST_ALLOC_IN_LEN(i); + inbuf = kzalloc(inlen, GFP_KERNEL); + if (!inbuf) + return -ENOMEM; + i = 0; + list_for_each_entry(act, &acts->list, list) { + MCDI_SET_ARRAY_DWORD(inbuf, MAE_ACTION_SET_LIST_ALLOC_IN_AS_IDS, + i, act->fw_id); + i++; + } + MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_LIST_ALLOC_IN_COUNT, i); + rc = efx_mcdi_rpc(efx, MC_CMD_MAE_ACTION_SET_LIST_ALLOC, inbuf, inlen, + outbuf, sizeof(outbuf), &outlen); + if (rc) + goto out_free; + if (outlen < sizeof(outbuf)) { + rc = -EIO; + goto out_free; + } + acts->fw_id = MCDI_DWORD(outbuf, MAE_ACTION_SET_LIST_ALLOC_OUT_ASL_ID); + /* We rely on the high bit of ASL IDs always being set. + * The firmware API guarantees this, but let's check it ourselves. + */ + if (WARN_ON_ONCE(!efx_mae_asl_id(acts->fw_id))) { + efx_mae_free_action_set_list(efx, acts); + rc = -EIO; + } +out_free: + kfree(inbuf); + return rc; +} + +int efx_mae_free_action_set_list(struct efx_nic *efx, + struct efx_tc_action_set_list *acts) +{ + MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_ACTION_SET_LIST_FREE_OUT_LEN(1)); + MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_ACTION_SET_LIST_FREE_IN_LEN(1)); + size_t outlen; + int rc; + + /* If this is just an AS_ID with no ASL wrapper, then there is + * nothing for us to free. (The AS will be freed later.) + */ + if (efx_mae_asl_id(acts->fw_id)) { + MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_LIST_FREE_IN_ASL_ID, + acts->fw_id); + rc = efx_mcdi_rpc(efx, MC_CMD_MAE_ACTION_SET_LIST_FREE, inbuf, + sizeof(inbuf), outbuf, sizeof(outbuf), &outlen); + if (rc) + return rc; + if (outlen < sizeof(outbuf)) + return -EIO; + /* FW freed a different ID than we asked for, should never happen. + * Warn because it means we've now got a different idea to the FW of + * what action-set-lists exist, which could cause mayhem later. + */ + if (WARN_ON(MCDI_DWORD(outbuf, MAE_ACTION_SET_LIST_FREE_OUT_FREED_ASL_ID) != acts->fw_id)) + return -EIO; + } + /* We're probably about to free @acts, but let's just make sure its + * fw_id is blatted so that it won't look valid if it leaks out. + */ + acts->fw_id = MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_ACTION_SET_LIST_ID_NULL; + return 0; +} + +static int efx_mae_populate_match_criteria(MCDI_DECLARE_STRUCT_PTR(match_crit), + const struct efx_tc_match *match) +{ + if (match->mask.ingress_port) { + if (~match->mask.ingress_port) + return -EOPNOTSUPP; + MCDI_STRUCT_SET_DWORD(match_crit, + MAE_FIELD_MASK_VALUE_PAIRS_V2_INGRESS_MPORT_SELECTOR, + match->value.ingress_port); + } + MCDI_STRUCT_SET_DWORD(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_INGRESS_MPORT_SELECTOR_MASK, + match->mask.ingress_port); + return 0; +} + +int efx_mae_insert_rule(struct efx_nic *efx, const struct efx_tc_match *match, + u32 prio, u32 acts_id, u32 *id) +{ + MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_ACTION_RULE_INSERT_IN_LEN(MAE_FIELD_MASK_VALUE_PAIRS_V2_LEN)); + MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_ACTION_RULE_INSERT_OUT_LEN); + MCDI_DECLARE_STRUCT_PTR(match_crit); + MCDI_DECLARE_STRUCT_PTR(response); + size_t outlen; + int rc; + + if (!id) + return -EINVAL; + + match_crit = _MCDI_DWORD(inbuf, MAE_ACTION_RULE_INSERT_IN_MATCH_CRITERIA); + response = _MCDI_DWORD(inbuf, MAE_ACTION_RULE_INSERT_IN_RESPONSE); + if (efx_mae_asl_id(acts_id)) { + MCDI_STRUCT_SET_DWORD(response, MAE_ACTION_RULE_RESPONSE_ASL_ID, acts_id); + MCDI_STRUCT_SET_DWORD(response, MAE_ACTION_RULE_RESPONSE_AS_ID, + MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL); + } else { + /* We only had one AS, so we didn't wrap it in an ASL */ + MCDI_STRUCT_SET_DWORD(response, MAE_ACTION_RULE_RESPONSE_ASL_ID, + MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_ACTION_SET_LIST_ID_NULL); + MCDI_STRUCT_SET_DWORD(response, MAE_ACTION_RULE_RESPONSE_AS_ID, acts_id); + } + MCDI_SET_DWORD(inbuf, MAE_ACTION_RULE_INSERT_IN_PRIO, prio); + rc = efx_mae_populate_match_criteria(match_crit, match); + if (rc) + return rc; + + rc = efx_mcdi_rpc(efx, MC_CMD_MAE_ACTION_RULE_INSERT, inbuf, sizeof(inbuf), + outbuf, sizeof(outbuf), &outlen); + if (rc) + return rc; + if (outlen < sizeof(outbuf)) + return -EIO; + *id = MCDI_DWORD(outbuf, MAE_ACTION_RULE_INSERT_OUT_AR_ID); + return 0; +} + +int efx_mae_delete_rule(struct efx_nic *efx, u32 id) +{ + MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_ACTION_RULE_DELETE_OUT_LEN(1)); + MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_ACTION_RULE_DELETE_IN_LEN(1)); + size_t outlen; + int rc; + + MCDI_SET_DWORD(inbuf, MAE_ACTION_RULE_DELETE_IN_AR_ID, id); + rc = efx_mcdi_rpc(efx, MC_CMD_MAE_ACTION_RULE_DELETE, inbuf, sizeof(inbuf), + outbuf, sizeof(outbuf), &outlen); + if (rc) + return rc; + if (outlen < sizeof(outbuf)) + return -EIO; + /* FW freed a different ID than we asked for, should also never happen. + * Warn because it means we've now got a different idea to the FW of + * what rules exist, which could cause mayhem later. + */ + if (WARN_ON(MCDI_DWORD(outbuf, MAE_ACTION_RULE_DELETE_OUT_DELETED_AR_ID) != id)) + return -EIO; + return 0; +} diff --git a/drivers/net/ethernet/sfc/mae.h b/drivers/net/ethernet/sfc/mae.h index 27e69e8a54b6..0369be4d8983 100644 --- a/drivers/net/ethernet/sfc/mae.h +++ b/drivers/net/ethernet/sfc/mae.h @@ -14,9 +14,29 @@ /* MCDI interface for the ef100 Match-Action Engine */ #include "net_driver.h" +#include "tc.h" +#include "mcdi_pcol.h" /* needed for various MC_CMD_MAE_*_NULL defines */ +int efx_mae_allocate_mport(struct efx_nic *efx, u32 *id, u32 *label); +int efx_mae_free_mport(struct efx_nic *efx, u32 id); + +void efx_mae_mport_wire(struct efx_nic *efx, u32 *out); +void efx_mae_mport_uplink(struct efx_nic *efx, u32 *out); void efx_mae_mport_vf(struct efx_nic *efx, u32 vf_id, u32 *out); +void efx_mae_mport_mport(struct efx_nic *efx, u32 mport_id, u32 *out); int efx_mae_lookup_mport(struct efx_nic *efx, u32 selector, u32 *id); +int efx_mae_alloc_action_set(struct efx_nic *efx, struct efx_tc_action_set *act); +int efx_mae_free_action_set(struct efx_nic *efx, u32 fw_id); + +int efx_mae_alloc_action_set_list(struct efx_nic *efx, + struct efx_tc_action_set_list *acts); +int efx_mae_free_action_set_list(struct efx_nic *efx, + struct efx_tc_action_set_list *acts); + +int efx_mae_insert_rule(struct efx_nic *efx, const struct efx_tc_match *match, + u32 prio, u32 acts_id, u32 *id); +int efx_mae_delete_rule(struct efx_nic *efx, u32 id); + #endif /* EF100_MAE_H */ diff --git a/drivers/net/ethernet/sfc/mcdi.h b/drivers/net/ethernet/sfc/mcdi.h index f74f6ce8b27d..26bc69f76801 100644 --- a/drivers/net/ethernet/sfc/mcdi.h +++ b/drivers/net/ethernet/sfc/mcdi.h @@ -205,6 +205,8 @@ void efx_mcdi_sensor_event(struct efx_nic *efx, efx_qword_t *ev); ((_ofst) + BUILD_BUG_ON_ZERO((_ofst) & (_align - 1))) #define _MCDI_DWORD(_buf, _field) \ ((_buf) + (_MCDI_CHECK_ALIGN(MC_CMD_ ## _field ## _OFST, 4) >> 2)) +#define _MCDI_STRUCT_DWORD(_buf, _field) \ + ((_buf) + (_MCDI_CHECK_ALIGN(_field ## _OFST, 4) >> 2)) #define MCDI_BYTE(_buf, _field) \ ((void)BUILD_BUG_ON_ZERO(MC_CMD_ ## _field ## _LEN != 1), \ @@ -214,6 +216,8 @@ void efx_mcdi_sensor_event(struct efx_nic *efx, efx_qword_t *ev); le16_to_cpu(*(__force const __le16 *)MCDI_PTR(_buf, _field))) #define MCDI_SET_DWORD(_buf, _field, _value) \ EFX_POPULATE_DWORD_1(*_MCDI_DWORD(_buf, _field), EFX_DWORD_0, _value) +#define MCDI_STRUCT_SET_DWORD(_buf, _field, _value) \ + EFX_POPULATE_DWORD_1(*_MCDI_STRUCT_DWORD(_buf, _field), EFX_DWORD_0, _value) #define MCDI_DWORD(_buf, _field) \ EFX_DWORD_FIELD(*_MCDI_DWORD(_buf, _field), EFX_DWORD_0) #define MCDI_POPULATE_DWORD_1(_buf, _field, _name1, _value1) \ diff --git a/drivers/net/ethernet/sfc/mcdi_filters.c b/drivers/net/ethernet/sfc/mcdi_filters.c index 1523be77b9db..4ff6586116ee 100644 --- a/drivers/net/ethernet/sfc/mcdi_filters.c +++ b/drivers/net/ethernet/sfc/mcdi_filters.c @@ -221,7 +221,10 @@ static void efx_mcdi_filter_push_prep(struct efx_nic *efx, efx_mcdi_filter_push_prep_set_match_fields(efx, spec, inbuf); } - MCDI_SET_DWORD(inbuf, FILTER_OP_IN_PORT_ID, efx->vport_id); + if (flags & EFX_FILTER_FLAG_VPORT_ID) + MCDI_SET_DWORD(inbuf, FILTER_OP_IN_PORT_ID, spec->vport_id); + else + MCDI_SET_DWORD(inbuf, FILTER_OP_IN_PORT_ID, efx->vport_id); MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_DEST, spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP ? MC_CMD_FILTER_OP_IN_RX_DEST_DROP : @@ -488,6 +491,7 @@ static s32 efx_mcdi_filter_insert_locked(struct efx_nic *efx, saved_spec->flags |= spec->flags; saved_spec->rss_context = spec->rss_context; saved_spec->dmaq_id = spec->dmaq_id; + saved_spec->vport_id = spec->vport_id; } } else if (!replacing) { kfree(saved_spec); diff --git a/drivers/net/ethernet/sfc/mcdi_filters.h b/drivers/net/ethernet/sfc/mcdi_filters.h index 06426aa9f2f3..c0d6558b9fd2 100644 --- a/drivers/net/ethernet/sfc/mcdi_filters.h +++ b/drivers/net/ethernet/sfc/mcdi_filters.h @@ -89,6 +89,7 @@ struct efx_mcdi_filter_table { */ bool mc_chaining; bool vlan_filter; + /* Entries on the vlan_list are added/removed under filter_sem */ struct list_head vlan_list; }; diff --git a/drivers/net/ethernet/sfc/mcdi_pcol_mae.h b/drivers/net/ethernet/sfc/mcdi_pcol_mae.h new file mode 100644 index 000000000000..ff6d80c8e486 --- /dev/null +++ b/drivers/net/ethernet/sfc/mcdi_pcol_mae.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/**************************************************************************** + * Driver for Solarflare network controllers and boards + * Copyright 2019 Solarflare Communications Inc. + * Copyright 2019-2022 Xilinx, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, incorporated herein by reference. + */ + +#ifndef MCDI_PCOL_MAE_H +#define MCDI_PCOL_MAE_H +/* MCDI definitions for Match-Action Engine functionality, that are + * missing from the main mcdi_pcol.h + */ + +/* MC_CMD_MAE_COUNTER_LIST_ALLOC is not (yet) a released API, but the + * following value is needed as an argument to MC_CMD_MAE_ACTION_SET_ALLOC. + */ +/* enum: A counter ID that is guaranteed never to represent a real counter */ +#define MC_CMD_MAE_COUNTER_LIST_ALLOC_OUT_COUNTER_LIST_ID_NULL 0xffffffff + +#endif /* MCDI_PCOL_MAE_H */ diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h index 4cde54cf77b9..7ef823d7a89a 100644 --- a/drivers/net/ethernet/sfc/net_driver.h +++ b/drivers/net/ethernet/sfc/net_driver.h @@ -478,6 +478,8 @@ enum efx_sync_events_state { * @n_rx_xdp_bad_drops: Count of RX packets dropped due to XDP errors * @n_rx_xdp_tx: Count of RX packets retransmitted due to XDP * @n_rx_xdp_redirect: Count of RX packets redirected to a different NIC by XDP + * @n_rx_mport_bad: Count of RX packets dropped because their ingress mport was + * not recognised * @rx_pkt_n_frags: Number of fragments in next packet to be delivered by * __efx_rx_packet(), or zero if there is none * @rx_pkt_index: Ring index of first buffer for next packet to be delivered @@ -540,6 +542,7 @@ struct efx_channel { unsigned int n_rx_xdp_bad_drops; unsigned int n_rx_xdp_tx; unsigned int n_rx_xdp_redirect; + unsigned int n_rx_mport_bad; unsigned int rx_pkt_n_frags; unsigned int rx_pkt_index; @@ -975,6 +978,7 @@ enum efx_xdp_tx_queues_mode { * @xdp_rxq_info_failed: Have any of the rx queues failed to initialise their * xdp_rxq_info structures? * @netdev_notifier: Netdevice notifier. + * @tc: state for TC offload (EF100). * @mem_bar: The BAR that is mapped into membase. * @reg_base: Offset from the start of the bar to the function control window. * @monitor_work: Hardware monitor workitem @@ -1158,6 +1162,7 @@ struct efx_nic { bool xdp_rxq_info_failed; struct notifier_block netdev_notifier; + struct efx_tc_state *tc; unsigned int mem_bar; u32 reg_base; diff --git a/drivers/net/ethernet/sfc/ptp.c b/drivers/net/ethernet/sfc/ptp.c index 4625f85acab2..10ad0b93d283 100644 --- a/drivers/net/ethernet/sfc/ptp.c +++ b/drivers/net/ethernet/sfc/ptp.c @@ -1100,7 +1100,29 @@ static void efx_ptp_xmit_skb_queue(struct efx_nic *efx, struct sk_buff *skb) tx_queue = efx_channel_get_tx_queue(ptp_data->channel, type); if (tx_queue && tx_queue->timestamping) { + /* This code invokes normal driver TX code which is always + * protected from softirqs when called from generic TX code, + * which in turn disables preemption. Look at __dev_queue_xmit + * which uses rcu_read_lock_bh disabling preemption for RCU + * plus disabling softirqs. We do not need RCU reader + * protection here. + * + * Although it is theoretically safe for current PTP TX/RX code + * running without disabling softirqs, there are three good + * reasond for doing so: + * + * 1) The code invoked is mainly implemented for non-PTP + * packets and it is always executed with softirqs + * disabled. + * 2) This being a single PTP packet, better to not + * interrupt its processing by softirqs which can lead + * to high latencies. + * 3) netdev_xmit_more checks preemption is disabled and + * triggers a BUG_ON if not. + */ + local_bh_disable(); efx_enqueue_skb(tx_queue, skb); + local_bh_enable(); } else { WARN_ONCE(1, "PTP channel has no timestamped tx queue\n"); dev_kfree_skb_any(skb); diff --git a/drivers/net/ethernet/sfc/rx_common.c b/drivers/net/ethernet/sfc/rx_common.c index bd21d6ac778a..4826e6a7e4ce 100644 --- a/drivers/net/ethernet/sfc/rx_common.c +++ b/drivers/net/ethernet/sfc/rx_common.c @@ -793,7 +793,6 @@ int efx_probe_filters(struct efx_nic *efx) int rc; mutex_lock(&efx->mac_lock); - down_write(&efx->filter_sem); rc = efx->type->filter_table_probe(efx); if (rc) goto out_unlock; @@ -830,7 +829,6 @@ int efx_probe_filters(struct efx_nic *efx) } #endif out_unlock: - up_write(&efx->filter_sem); mutex_unlock(&efx->mac_lock); return rc; } @@ -846,9 +844,7 @@ void efx_remove_filters(struct efx_nic *efx) channel->rps_flow_id = NULL; } #endif - down_write(&efx->filter_sem); efx->type->filter_table_remove(efx); - up_write(&efx->filter_sem); } #ifdef CONFIG_RFS_ACCEL diff --git a/drivers/net/ethernet/sfc/tc.c b/drivers/net/ethernet/sfc/tc.c new file mode 100644 index 000000000000..0c0aeb91f500 --- /dev/null +++ b/drivers/net/ethernet/sfc/tc.c @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: GPL-2.0-only +/**************************************************************************** + * Driver for Solarflare network controllers and boards + * Copyright 2019 Solarflare Communications Inc. + * Copyright 2020-2022 Xilinx Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, incorporated herein by reference. + */ + +#include "tc.h" +#include "mae.h" +#include "ef100_rep.h" +#include "efx.h" + +static void efx_tc_free_action_set(struct efx_nic *efx, + struct efx_tc_action_set *act, bool in_hw) +{ + /* Failure paths calling this on the 'running action' set in_hw=false, + * because if the alloc had succeeded we'd've put it in acts.list and + * not still have it in act. + */ + if (in_hw) { + efx_mae_free_action_set(efx, act->fw_id); + /* in_hw is true iff we are on an acts.list; make sure to + * remove ourselves from that list before we are freed. + */ + list_del(&act->list); + } + kfree(act); +} + +static void efx_tc_free_action_set_list(struct efx_nic *efx, + struct efx_tc_action_set_list *acts, + bool in_hw) +{ + struct efx_tc_action_set *act, *next; + + /* Failure paths set in_hw=false, because usually the acts didn't get + * to efx_mae_alloc_action_set_list(); if they did, the failure tree + * has a separate efx_mae_free_action_set_list() before calling us. + */ + if (in_hw) + efx_mae_free_action_set_list(efx, acts); + /* Any act that's on the list will be in_hw even if the list isn't */ + list_for_each_entry_safe(act, next, &acts->list, list) + efx_tc_free_action_set(efx, act, true); + /* Don't kfree, as acts is embedded inside a struct efx_tc_flow_rule */ +} + +static void efx_tc_delete_rule(struct efx_nic *efx, struct efx_tc_flow_rule *rule) +{ + efx_mae_delete_rule(efx, rule->fw_id); + + /* Release entries in subsidiary tables */ + efx_tc_free_action_set_list(efx, &rule->acts, true); + rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; +} + +static int efx_tc_configure_default_rule(struct efx_nic *efx, u32 ing_port, + u32 eg_port, struct efx_tc_flow_rule *rule) +{ + struct efx_tc_action_set_list *acts = &rule->acts; + struct efx_tc_match *match = &rule->match; + struct efx_tc_action_set *act; + int rc; + + match->value.ingress_port = ing_port; + match->mask.ingress_port = ~0; + act = kzalloc(sizeof(*act), GFP_KERNEL); + if (!act) + return -ENOMEM; + act->deliver = 1; + act->dest_mport = eg_port; + rc = efx_mae_alloc_action_set(efx, act); + if (rc) + goto fail1; + EFX_WARN_ON_PARANOID(!list_empty(&acts->list)); + list_add_tail(&act->list, &acts->list); + rc = efx_mae_alloc_action_set_list(efx, acts); + if (rc) + goto fail2; + rc = efx_mae_insert_rule(efx, match, EFX_TC_PRIO_DFLT, + acts->fw_id, &rule->fw_id); + if (rc) + goto fail3; + return 0; +fail3: + efx_mae_free_action_set_list(efx, acts); +fail2: + list_del(&act->list); + efx_mae_free_action_set(efx, act->fw_id); +fail1: + kfree(act); + return rc; +} + +static int efx_tc_configure_default_rule_pf(struct efx_nic *efx) +{ + struct efx_tc_flow_rule *rule = &efx->tc->dflt.pf; + u32 ing_port, eg_port; + + efx_mae_mport_uplink(efx, &ing_port); + efx_mae_mport_wire(efx, &eg_port); + return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule); +} + +static int efx_tc_configure_default_rule_wire(struct efx_nic *efx) +{ + struct efx_tc_flow_rule *rule = &efx->tc->dflt.wire; + u32 ing_port, eg_port; + + efx_mae_mport_wire(efx, &ing_port); + efx_mae_mport_uplink(efx, &eg_port); + return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule); +} + +int efx_tc_configure_default_rule_rep(struct efx_rep *efv) +{ + struct efx_tc_flow_rule *rule = &efv->dflt; + struct efx_nic *efx = efv->parent; + u32 ing_port, eg_port; + + efx_mae_mport_mport(efx, efv->mport, &ing_port); + efx_mae_mport_mport(efx, efx->tc->reps_mport_id, &eg_port); + return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule); +} + +void efx_tc_deconfigure_default_rule(struct efx_nic *efx, + struct efx_tc_flow_rule *rule) +{ + if (rule->fw_id != MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL) + efx_tc_delete_rule(efx, rule); + rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; +} + +static int efx_tc_configure_rep_mport(struct efx_nic *efx) +{ + u32 rep_mport_label; + int rc; + + rc = efx_mae_allocate_mport(efx, &efx->tc->reps_mport_id, &rep_mport_label); + if (rc) + return rc; + pci_dbg(efx->pci_dev, "created rep mport 0x%08x (0x%04x)\n", + efx->tc->reps_mport_id, rep_mport_label); + /* Use mport *selector* as vport ID */ + efx_mae_mport_mport(efx, efx->tc->reps_mport_id, + &efx->tc->reps_mport_vport_id); + return 0; +} + +static void efx_tc_deconfigure_rep_mport(struct efx_nic *efx) +{ + efx_mae_free_mport(efx, efx->tc->reps_mport_id); + efx->tc->reps_mport_id = MAE_MPORT_SELECTOR_NULL; +} + +int efx_tc_insert_rep_filters(struct efx_nic *efx) +{ + struct efx_filter_spec promisc, allmulti; + int rc; + + if (efx->type->is_vf) + return 0; + if (!efx->tc) + return 0; + efx_filter_init_rx(&promisc, EFX_FILTER_PRI_REQUIRED, 0, 0); + efx_filter_set_uc_def(&promisc); + efx_filter_set_vport_id(&promisc, efx->tc->reps_mport_vport_id); + rc = efx_filter_insert_filter(efx, &promisc, false); + if (rc < 0) + return rc; + efx->tc->reps_filter_uc = rc; + efx_filter_init_rx(&allmulti, EFX_FILTER_PRI_REQUIRED, 0, 0); + efx_filter_set_mc_def(&allmulti); + efx_filter_set_vport_id(&allmulti, efx->tc->reps_mport_vport_id); + rc = efx_filter_insert_filter(efx, &allmulti, false); + if (rc < 0) + return rc; + efx->tc->reps_filter_mc = rc; + return 0; +} + +void efx_tc_remove_rep_filters(struct efx_nic *efx) +{ + if (efx->type->is_vf) + return; + if (!efx->tc) + return; + if (efx->tc->reps_filter_mc >= 0) + efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_mc); + efx->tc->reps_filter_mc = -1; + if (efx->tc->reps_filter_uc >= 0) + efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_uc); + efx->tc->reps_filter_uc = -1; +} + +int efx_init_tc(struct efx_nic *efx) +{ + int rc; + + rc = efx_tc_configure_default_rule_pf(efx); + if (rc) + return rc; + rc = efx_tc_configure_default_rule_wire(efx); + if (rc) + return rc; + return efx_tc_configure_rep_mport(efx); +} + +void efx_fini_tc(struct efx_nic *efx) +{ + /* We can get called even if efx_init_struct_tc() failed */ + if (!efx->tc) + return; + efx_tc_deconfigure_rep_mport(efx); + efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.pf); + efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.wire); +} + +int efx_init_struct_tc(struct efx_nic *efx) +{ + if (efx->type->is_vf) + return 0; + + efx->tc = kzalloc(sizeof(*efx->tc), GFP_KERNEL); + if (!efx->tc) + return -ENOMEM; + + efx->tc->reps_filter_uc = -1; + efx->tc->reps_filter_mc = -1; + INIT_LIST_HEAD(&efx->tc->dflt.pf.acts.list); + efx->tc->dflt.pf.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; + INIT_LIST_HEAD(&efx->tc->dflt.wire.acts.list); + efx->tc->dflt.wire.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; + return 0; +} + +void efx_fini_struct_tc(struct efx_nic *efx) +{ + if (!efx->tc) + return; + + EFX_WARN_ON_PARANOID(efx->tc->dflt.pf.fw_id != + MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL); + EFX_WARN_ON_PARANOID(efx->tc->dflt.wire.fw_id != + MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL); + kfree(efx->tc); + efx->tc = NULL; +} diff --git a/drivers/net/ethernet/sfc/tc.h b/drivers/net/ethernet/sfc/tc.h new file mode 100644 index 000000000000..309123c6b386 --- /dev/null +++ b/drivers/net/ethernet/sfc/tc.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/**************************************************************************** + * Driver for Solarflare network controllers and boards + * Copyright 2019 Solarflare Communications Inc. + * Copyright 2020-2022 Xilinx Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, incorporated herein by reference. + */ + +#ifndef EFX_TC_H +#define EFX_TC_H +#include "net_driver.h" + +struct efx_tc_action_set { + u16 deliver:1; + u32 dest_mport; + u32 fw_id; /* index of this entry in firmware actions table */ + struct list_head list; +}; + +struct efx_tc_match_fields { + /* L1 */ + u32 ingress_port; +}; + +struct efx_tc_match { + struct efx_tc_match_fields value; + struct efx_tc_match_fields mask; +}; + +struct efx_tc_action_set_list { + struct list_head list; + u32 fw_id; +}; + +struct efx_tc_flow_rule { + struct efx_tc_match match; + struct efx_tc_action_set_list acts; + u32 fw_id; +}; + +enum efx_tc_rule_prios { + EFX_TC_PRIO_DFLT, /* Default switch rule; one of efx_tc_default_rules */ + EFX_TC_PRIO__NUM +}; + +/** + * struct efx_tc_state - control plane data for TC offload + * + * @reps_mport_id: MAE port allocated for representor RX + * @reps_filter_uc: VNIC filter for representor unicast RX (promisc) + * @reps_filter_mc: VNIC filter for representor multicast RX (allmulti) + * @reps_mport_vport_id: vport_id for representor RX filters + * @dflt: Match-action rules for default switching; at priority + * %EFX_TC_PRIO_DFLT. Named by *ingress* port + * @dflt.pf: rule for traffic ingressing from PF (egresses to wire) + * @dflt.wire: rule for traffic ingressing from wire (egresses to PF) + */ +struct efx_tc_state { + u32 reps_mport_id, reps_mport_vport_id; + s32 reps_filter_uc, reps_filter_mc; + struct { + struct efx_tc_flow_rule pf; + struct efx_tc_flow_rule wire; + } dflt; +}; + +struct efx_rep; + +int efx_tc_configure_default_rule_rep(struct efx_rep *efv); +void efx_tc_deconfigure_default_rule(struct efx_nic *efx, + struct efx_tc_flow_rule *rule); + +int efx_tc_insert_rep_filters(struct efx_nic *efx); +void efx_tc_remove_rep_filters(struct efx_nic *efx); + +int efx_init_tc(struct efx_nic *efx); +void efx_fini_tc(struct efx_nic *efx); + +int efx_init_struct_tc(struct efx_nic *efx); +void efx_fini_struct_tc(struct efx_nic *efx); + +#endif /* EFX_TC_H */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c index ca8ab290013c..d42e1afb6521 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c @@ -688,18 +688,19 @@ static int mediatek_dwmac_probe(struct platform_device *pdev) ret = mediatek_dwmac_clks_config(priv_plat, true); if (ret) - return ret; + goto err_remove_config_dt; ret = stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res); - if (ret) { - stmmac_remove_config_dt(pdev, plat_dat); + if (ret) goto err_drv_probe; - } return 0; err_drv_probe: mediatek_dwmac_clks_config(priv_plat, false); +err_remove_config_dt: + stmmac_remove_config_dt(pdev, plat_dat); + return ret; } |