diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2020-06-03 16:27:18 -0700 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2020-06-03 16:27:18 -0700 |
| commit | cb8e59cc87201af93dfbb6c3dccc8fcad72a09c2 (patch) | |
| tree | a334db9022f89654b777bbce8c4c6632e65b9031 /drivers/net/ethernet/intel | |
| parent | 2e63f6ce7ed2c4ff83ba30ad9ccad422289a6c63 (diff) | |
| parent | 065fcfd49763ec71ae345bb5c5a74f961031e70e (diff) | |
Merge git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next
Pull networking updates from David Miller:
1) Allow setting bluetooth L2CAP modes via socket option, from Luiz
Augusto von Dentz.
2) Add GSO partial support to igc, from Sasha Neftin.
3) Several cleanups and improvements to r8169 from Heiner Kallweit.
4) Add IF_OPER_TESTING link state and use it when ethtool triggers a
device self-test. From Andrew Lunn.
5) Start moving away from custom driver versions, use the globally
defined kernel version instead, from Leon Romanovsky.
6) Support GRO vis gro_cells in DSA layer, from Alexander Lobakin.
7) Allow hard IRQ deferral during NAPI, from Eric Dumazet.
8) Add sriov and vf support to hinic, from Luo bin.
9) Support Media Redundancy Protocol (MRP) in the bridging code, from
Horatiu Vultur.
10) Support netmap in the nft_nat code, from Pablo Neira Ayuso.
11) Allow UDPv6 encapsulation of ESP in the ipsec code, from Sabrina
Dubroca. Also add ipv6 support for espintcp.
12) Lots of ReST conversions of the networking documentation, from Mauro
Carvalho Chehab.
13) Support configuration of ethtool rxnfc flows in bcmgenet driver,
from Doug Berger.
14) Allow to dump cgroup id and filter by it in inet_diag code, from
Dmitry Yakunin.
15) Add infrastructure to export netlink attribute policies to
userspace, from Johannes Berg.
16) Several optimizations to sch_fq scheduler, from Eric Dumazet.
17) Fallback to the default qdisc if qdisc init fails because otherwise
a packet scheduler init failure will make a device inoperative. From
Jesper Dangaard Brouer.
18) Several RISCV bpf jit optimizations, from Luke Nelson.
19) Correct the return type of the ->ndo_start_xmit() method in several
drivers, it's netdev_tx_t but many drivers were using
'int'. From Yunjian Wang.
20) Add an ethtool interface for PHY master/slave config, from Oleksij
Rempel.
21) Add BPF iterators, from Yonghang Song.
22) Add cable test infrastructure, including ethool interfaces, from
Andrew Lunn. Marvell PHY driver is the first to support this
facility.
23) Remove zero-length arrays all over, from Gustavo A. R. Silva.
24) Calculate and maintain an explicit frame size in XDP, from Jesper
Dangaard Brouer.
25) Add CAP_BPF, from Alexei Starovoitov.
26) Support terse dumps in the packet scheduler, from Vlad Buslov.
27) Support XDP_TX bulking in dpaa2 driver, from Ioana Ciornei.
28) Add devm_register_netdev(), from Bartosz Golaszewski.
29) Minimize qdisc resets, from Cong Wang.
30) Get rid of kernel_getsockopt and kernel_setsockopt in order to
eliminate set_fs/get_fs calls. From Christoph Hellwig.
* git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next: (2517 commits)
selftests: net: ip_defrag: ignore EPERM
net_failover: fixed rollback in net_failover_open()
Revert "tipc: Fix potential tipc_aead refcnt leak in tipc_crypto_rcv"
Revert "tipc: Fix potential tipc_node refcnt leak in tipc_rcv"
vmxnet3: allow rx flow hash ops only when rss is enabled
hinic: add set_channels ethtool_ops support
selftests/bpf: Add a default $(CXX) value
tools/bpf: Don't use $(COMPILE.c)
bpf, selftests: Use bpf_probe_read_kernel
s390/bpf: Use bcr 0,%0 as tail call nop filler
s390/bpf: Maintain 8-byte stack alignment
selftests/bpf: Fix verifier test
selftests/bpf: Fix sample_cnt shared between two threads
bpf, selftests: Adapt cls_redirect to call csum_level helper
bpf: Add csum_level helper for fixing up csum levels
bpf: Fix up bpf_skb_adjust_room helper's skb csum setting
sfc: add missing annotation for efx_ef10_try_update_nic_stats_vf()
crypto/chtls: IPv6 support for inline TLS
Crypto/chcr: Fixes a coccinile check error
Crypto/chcr: Fixes compilations warnings
...
Diffstat (limited to 'drivers/net/ethernet/intel')
86 files changed, 10756 insertions, 3507 deletions
diff --git a/drivers/net/ethernet/intel/e1000/e1000_hw.c b/drivers/net/ethernet/intel/e1000/e1000_hw.c index 48428d6a00be..623e516a9630 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_hw.c +++ b/drivers/net/ethernet/intel/e1000/e1000_hw.c @@ -3960,7 +3960,7 @@ static s32 e1000_do_read_eeprom(struct e1000_hw *hw, u16 offset, u16 words, * @hw: Struct containing variables accessed by shared code * * Reads the first 64 16 bit words of the EEPROM and sums the values read. - * If the the sum of the 64 16 bit words is 0xBABA, the EEPROM's checksum is + * If the sum of the 64 16 bit words is 0xBABA, the EEPROM's checksum is * valid. */ s32 e1000_validate_eeprom_checksum(struct e1000_hw *hw) diff --git a/drivers/net/ethernet/intel/e1000/e1000_main.c b/drivers/net/ethernet/intel/e1000/e1000_main.c index 0d51cbc88028..d9fa4600f745 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_main.c +++ b/drivers/net/ethernet/intel/e1000/e1000_main.c @@ -542,8 +542,13 @@ void e1000_reinit_locked(struct e1000_adapter *adapter) WARN_ON(in_interrupt()); while (test_and_set_bit(__E1000_RESETTING, &adapter->flags)) msleep(1); - e1000_down(adapter); - e1000_up(adapter); + + /* only run the task if not already down */ + if (!test_bit(__E1000_DOWN, &adapter->flags)) { + e1000_down(adapter); + e1000_up(adapter); + } + clear_bit(__E1000_RESETTING, &adapter->flags); } @@ -1433,10 +1438,15 @@ int e1000_close(struct net_device *netdev) struct e1000_hw *hw = &adapter->hw; int count = E1000_CHECK_RESET_COUNT; - while (test_bit(__E1000_RESETTING, &adapter->flags) && count--) + while (test_and_set_bit(__E1000_RESETTING, &adapter->flags) && count--) usleep_range(10000, 20000); - WARN_ON(test_bit(__E1000_RESETTING, &adapter->flags)); + WARN_ON(count < 0); + + /* signal that we're down so that the reset task will no longer run */ + set_bit(__E1000_DOWN, &adapter->flags); + clear_bit(__E1000_RESETTING, &adapter->flags); + e1000_down(adapter); e1000_power_down_phy(adapter); e1000_free_irq(adapter); @@ -3136,8 +3146,9 @@ static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb, hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); if (skb->data_len && hdr_len == len) { switch (hw->mac_type) { + case e1000_82544: { unsigned int pull_size; - case e1000_82544: + /* Make sure we have room to chop off 4 bytes, * and that the end alignment will work out to * this hardware's requirements @@ -3158,6 +3169,7 @@ static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb, } len = skb_headlen(skb); break; + } default: /* do nothing */ break; diff --git a/drivers/net/ethernet/intel/e1000e/e1000.h b/drivers/net/ethernet/intel/e1000e/e1000.h index 37a2314d3e6b..944abd5eae11 100644 --- a/drivers/net/ethernet/intel/e1000e/e1000.h +++ b/drivers/net/ethernet/intel/e1000e/e1000.h @@ -576,7 +576,6 @@ static inline u32 __er32(struct e1000_hw *hw, unsigned long reg) #define er32(reg) __er32(hw, E1000_##reg) -s32 __ew32_prepare(struct e1000_hw *hw); void __ew32(struct e1000_hw *hw, unsigned long reg, u32 val); #define ew32(reg, val) __ew32(hw, E1000_##reg, (val)) diff --git a/drivers/net/ethernet/intel/e1000e/ich8lan.c b/drivers/net/ethernet/intel/e1000e/ich8lan.c index 735bf25952fc..f999cca37a8a 100644 --- a/drivers/net/ethernet/intel/e1000e/ich8lan.c +++ b/drivers/net/ethernet/intel/e1000e/ich8lan.c @@ -300,7 +300,11 @@ static s32 e1000_init_phy_workarounds_pchlan(struct e1000_hw *hw) * so forcibly disable it. */ hw->dev_spec.ich8lan.ulp_state = e1000_ulp_state_unknown; - e1000_disable_ulp_lpt_lp(hw, true); + ret_val = e1000_disable_ulp_lpt_lp(hw, true); + if (ret_val) { + e_warn("Failed to disable ULP\n"); + goto out; + } ret_val = hw->phy.ops.acquire(hw); if (ret_val) { diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c index 2730b1c7dddb..a279f4fa9962 100644 --- a/drivers/net/ethernet/intel/e1000e/netdev.c +++ b/drivers/net/ethernet/intel/e1000e/netdev.c @@ -107,6 +107,45 @@ static const struct e1000_reg_info e1000_reg_info_tbl[] = { {0, NULL} }; +struct e1000e_me_supported { + u16 device_id; /* supported device ID */ +}; + +static const struct e1000e_me_supported me_supported[] = { + {E1000_DEV_ID_PCH_LPT_I217_LM}, + {E1000_DEV_ID_PCH_LPTLP_I218_LM}, + {E1000_DEV_ID_PCH_I218_LM2}, + {E1000_DEV_ID_PCH_I218_LM3}, + {E1000_DEV_ID_PCH_SPT_I219_LM}, + {E1000_DEV_ID_PCH_SPT_I219_LM2}, + {E1000_DEV_ID_PCH_LBG_I219_LM3}, + {E1000_DEV_ID_PCH_SPT_I219_LM4}, + {E1000_DEV_ID_PCH_SPT_I219_LM5}, + {E1000_DEV_ID_PCH_CNP_I219_LM6}, + {E1000_DEV_ID_PCH_CNP_I219_LM7}, + {E1000_DEV_ID_PCH_ICP_I219_LM8}, + {E1000_DEV_ID_PCH_ICP_I219_LM9}, + {E1000_DEV_ID_PCH_CMP_I219_LM10}, + {E1000_DEV_ID_PCH_CMP_I219_LM11}, + {E1000_DEV_ID_PCH_CMP_I219_LM12}, + {E1000_DEV_ID_PCH_TGP_I219_LM13}, + {E1000_DEV_ID_PCH_TGP_I219_LM14}, + {E1000_DEV_ID_PCH_TGP_I219_LM15}, + {0} +}; + +static bool e1000e_check_me(u16 device_id) +{ + struct e1000e_me_supported *id; + + for (id = (struct e1000e_me_supported *)me_supported; + id->device_id; id++) + if (device_id == id->device_id) + return true; + + return false; +} + /** * __ew32_prepare - prepare to write to MAC CSR register on certain parts * @hw: pointer to the HW structure @@ -119,14 +158,12 @@ static const struct e1000_reg_info e1000_reg_info_tbl[] = { * has bit 24 set while ME is accessing MAC CSR registers, wait if it is set * and try again a number of times. **/ -s32 __ew32_prepare(struct e1000_hw *hw) +static void __ew32_prepare(struct e1000_hw *hw) { s32 i = E1000_ICH_FWSM_PCIM2PCI_COUNT; while ((er32(FWSM) & E1000_ICH_FWSM_PCIM2PCI) && --i) udelay(50); - - return i; } void __ew32(struct e1000_hw *hw, unsigned long reg, u32 val) @@ -607,11 +644,11 @@ static void e1000e_update_rdt_wa(struct e1000_ring *rx_ring, unsigned int i) { struct e1000_adapter *adapter = rx_ring->adapter; struct e1000_hw *hw = &adapter->hw; - s32 ret_val = __ew32_prepare(hw); + __ew32_prepare(hw); writel(i, rx_ring->tail); - if (unlikely(!ret_val && (i != readl(rx_ring->tail)))) { + if (unlikely(i != readl(rx_ring->tail))) { u32 rctl = er32(RCTL); ew32(RCTL, rctl & ~E1000_RCTL_EN); @@ -624,11 +661,11 @@ static void e1000e_update_tdt_wa(struct e1000_ring *tx_ring, unsigned int i) { struct e1000_adapter *adapter = tx_ring->adapter; struct e1000_hw *hw = &adapter->hw; - s32 ret_val = __ew32_prepare(hw); + __ew32_prepare(hw); writel(i, tx_ring->tail); - if (unlikely(!ret_val && (i != readl(tx_ring->tail)))) { + if (unlikely(i != readl(tx_ring->tail))) { u32 tctl = er32(TCTL); ew32(TCTL, tctl & ~E1000_TCTL_EN); @@ -5294,6 +5331,10 @@ static void e1000_watchdog_task(struct work_struct *work) /* oops */ break; } + if (hw->mac.type == e1000_pch_spt) { + netdev->features &= ~NETIF_F_TSO; + netdev->features &= ~NETIF_F_TSO6; + } } /* enable transmits in the hardware, need to do this @@ -6404,6 +6445,31 @@ static void e1000e_s0ix_entry_flow(struct e1000_adapter *adapter) mac_data |= BIT(3); ew32(CTRL_EXT, mac_data); + /* Disable disconnected cable conditioning for Power Gating */ + mac_data = er32(DPGFR); + mac_data |= BIT(2); + ew32(DPGFR, mac_data); + + /* Don't wake from dynamic Power Gating with clock request */ + mac_data = er32(FEXTNVM12); + mac_data |= BIT(12); + ew32(FEXTNVM12, mac_data); + + /* Ungate PGCB clock */ + mac_data = er32(FEXTNVM9); + mac_data |= BIT(28); + ew32(FEXTNVM9, mac_data); + + /* Enable K1 off to enable mPHY Power Gating */ + mac_data = er32(FEXTNVM6); + mac_data |= BIT(31); + ew32(FEXTNVM12, mac_data); + + /* Enable mPHY power gating for any link and speed */ + mac_data = er32(FEXTNVM8); + mac_data |= BIT(9); + ew32(FEXTNVM8, mac_data); + /* Enable the Dynamic Clock Gating in the DMA and MAC */ mac_data = er32(CTRL_EXT); mac_data |= E1000_CTRL_EXT_DMA_DYN_CLK_EN; @@ -6433,6 +6499,35 @@ static void e1000e_s0ix_exit_flow(struct e1000_adapter *adapter) mac_data |= BIT(0); ew32(FEXTNVM7, mac_data); + /* Disable mPHY power gating for any link and speed */ + mac_data = er32(FEXTNVM8); + mac_data &= ~BIT(9); + ew32(FEXTNVM8, mac_data); + + /* Disable K1 off */ + mac_data = er32(FEXTNVM6); + mac_data &= ~BIT(31); + ew32(FEXTNVM12, mac_data); + + /* Disable Ungate PGCB clock */ + mac_data = er32(FEXTNVM9); + mac_data &= ~BIT(28); + ew32(FEXTNVM9, mac_data); + + /* Cancel not waking from dynamic + * Power Gating with clock request + */ + mac_data = er32(FEXTNVM12); + mac_data &= ~BIT(12); + ew32(FEXTNVM12, mac_data); + + /* Cancel disable disconnected cable conditioning + * for Power Gating + */ + mac_data = er32(DPGFR); + mac_data &= ~BIT(2); + ew32(DPGFR, mac_data); + /* Disable Dynamic Power Gating */ mac_data = er32(CTRL_EXT); mac_data &= 0xFFFFFFF7; @@ -6858,7 +6953,8 @@ static int e1000e_pm_suspend(struct device *dev) e1000e_pm_thaw(dev); /* Introduce S0ix implementation */ - if (hw->mac.type >= e1000_pch_cnp) + if (hw->mac.type >= e1000_pch_cnp && + !e1000e_check_me(hw->adapter->pdev->device)) e1000e_s0ix_entry_flow(adapter); return rc; @@ -6873,7 +6969,8 @@ static int e1000e_pm_resume(struct device *dev) int rc; /* Introduce S0ix implementation */ - if (hw->mac.type >= e1000_pch_cnp) + if (hw->mac.type >= e1000_pch_cnp && + !e1000e_check_me(hw->adapter->pdev->device)) e1000e_s0ix_exit_flow(adapter); rc = __e1000_resume(pdev); diff --git a/drivers/net/ethernet/intel/e1000e/regs.h b/drivers/net/ethernet/intel/e1000e/regs.h index df59fd1d660c..8165ba2619a4 100644 --- a/drivers/net/ethernet/intel/e1000e/regs.h +++ b/drivers/net/ethernet/intel/e1000e/regs.h @@ -21,9 +21,12 @@ #define E1000_FEXTNVM5 0x00014 /* Future Extended NVM 5 - RW */ #define E1000_FEXTNVM6 0x00010 /* Future Extended NVM 6 - RW */ #define E1000_FEXTNVM7 0x000E4 /* Future Extended NVM 7 - RW */ +#define E1000_FEXTNVM8 0x5BB0 /* Future Extended NVM 8 - RW */ #define E1000_FEXTNVM9 0x5BB4 /* Future Extended NVM 9 - RW */ #define E1000_FEXTNVM11 0x5BBC /* Future Extended NVM 11 - RW */ +#define E1000_FEXTNVM12 0x5BC0 /* Future Extended NVM 12 - RW */ #define E1000_PCIEANACFG 0x00F18 /* PCIE Analog Config */ +#define E1000_DPGFR 0x00FAC /* Dynamic Power Gate Force Control Register */ #define E1000_FCT 0x00030 /* Flow Control Type - RW */ #define E1000_VET 0x00038 /* VLAN Ether Type - RW */ #define E1000_ICR 0x000C0 /* Interrupt Cause Read - R/clr */ diff --git a/drivers/net/ethernet/intel/i40e/i40e_adminq.c b/drivers/net/ethernet/intel/i40e/i40e_adminq.c index 37514a75f928..6a089848c857 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_adminq.c +++ b/drivers/net/ethernet/intel/i40e/i40e_adminq.c @@ -694,10 +694,8 @@ init_adminq_exit: * i40e_shutdown_adminq - shutdown routine for the Admin Queue * @hw: pointer to the hardware structure **/ -i40e_status i40e_shutdown_adminq(struct i40e_hw *hw) +void i40e_shutdown_adminq(struct i40e_hw *hw) { - i40e_status ret_code = 0; - if (i40e_check_asq_alive(hw)) i40e_aq_queue_shutdown(hw, true); @@ -706,8 +704,6 @@ i40e_status i40e_shutdown_adminq(struct i40e_hw *hw) if (hw->nvm_buff.va) i40e_free_virt_mem(hw, &hw->nvm_buff); - - return ret_code; } /** diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index 2a037ec244b9..5d807c8004f8 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -11,7 +11,7 @@ #include "i40e_diag.h" #include "i40e_xsk.h" #include <net/udp_tunnel.h> -#include <net/xdp_sock.h> +#include <net/xdp_sock_drv.h> /* All i40e tracepoints are defined by the include below, which * must be included exactly once across the whole kernel with * CREATE_TRACE_POINTS defined @@ -3260,26 +3260,31 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring) if (ring->vsi->type == I40E_VSI_MAIN) xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq); + kfree(ring->rx_bi); ring->xsk_umem = i40e_xsk_umem(ring); if (ring->xsk_umem) { - ring->rx_buf_len = ring->xsk_umem->chunk_size_nohr - - XDP_PACKET_HEADROOM; + ret = i40e_alloc_rx_bi_zc(ring); + if (ret) + return ret; + ring->rx_buf_len = xsk_umem_get_rx_frame_size(ring->xsk_umem); /* For AF_XDP ZC, we disallow packets to span on * multiple buffers, thus letting us skip that * handling in the fast-path. */ chain_len = 1; - ring->zca.free = i40e_zca_free; ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, - MEM_TYPE_ZERO_COPY, - &ring->zca); + MEM_TYPE_XSK_BUFF_POOL, + NULL); if (ret) return ret; dev_info(&vsi->back->pdev->dev, - "Registered XDP mem model MEM_TYPE_ZERO_COPY on Rx ring %d\n", + "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n", ring->queue_index); } else { + ret = i40e_alloc_rx_bi(ring); + if (ret) + return ret; ring->rx_buf_len = vsi->rx_buf_len; if (ring->vsi->type == I40E_VSI_MAIN) { ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, @@ -3344,9 +3349,12 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring) ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q); writel(0, ring->tail); - ok = ring->xsk_umem ? - i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring)) : - !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring)); + if (ring->xsk_umem) { + xsk_buff_set_rxq_info(ring->xsk_umem, &ring->xdp_rxq); + ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring)); + } else { + ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring)); + } if (!ok) { /* Log this in case the user has forgotten to give the kernel * any buffers, even later in the application. @@ -14478,29 +14486,29 @@ static void i40e_print_features(struct i40e_pf *pf) i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id); #ifdef CONFIG_PCI_IOV - i += snprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs); + i += scnprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs); #endif - i += snprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d", + i += scnprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d", pf->hw.func_caps.num_vsis, pf->vsi[pf->lan_vsi]->num_queue_pairs); if (pf->flags & I40E_FLAG_RSS_ENABLED) - i += snprintf(&buf[i], REMAIN(i), " RSS"); + i += scnprintf(&buf[i], REMAIN(i), " RSS"); if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) - i += snprintf(&buf[i], REMAIN(i), " FD_ATR"); + i += scnprintf(&buf[i], REMAIN(i), " FD_ATR"); if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { - i += snprintf(&buf[i], REMAIN(i), " FD_SB"); - i += snprintf(&buf[i], REMAIN(i), " NTUPLE"); + i += scnprintf(&buf[i], REMAIN(i), " FD_SB"); + i += scnprintf(&buf[i], REMAIN(i), " NTUPLE"); } if (pf->flags & I40E_FLAG_DCB_CAPABLE) - i += snprintf(&buf[i], REMAIN(i), " DCB"); - i += snprintf(&buf[i], REMAIN(i), " VxLAN"); - i += snprintf(&buf[i], REMAIN(i), " Geneve"); + i += scnprintf(&buf[i], REMAIN(i), " DCB"); + i += scnprintf(&buf[i], REMAIN(i), " VxLAN"); + i += scnprintf(&buf[i], REMAIN(i), " Geneve"); if (pf->flags & I40E_FLAG_PTP) - i += snprintf(&buf[i], REMAIN(i), " PTP"); + i += scnprintf(&buf[i], REMAIN(i), " PTP"); if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) - i += snprintf(&buf[i], REMAIN(i), " VEB"); + i += scnprintf(&buf[i], REMAIN(i), " VEB"); else - i += snprintf(&buf[i], REMAIN(i), " VEPA"); + i += scnprintf(&buf[i], REMAIN(i), " VEPA"); dev_info(&pf->pdev->dev, "%s\n", buf); kfree(buf); diff --git a/drivers/net/ethernet/intel/i40e/i40e_prototype.h b/drivers/net/ethernet/intel/i40e/i40e_prototype.h index bbb478f09093..5c1378641b3b 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_prototype.h +++ b/drivers/net/ethernet/intel/i40e/i40e_prototype.h @@ -17,7 +17,7 @@ /* adminq functions */ i40e_status i40e_init_adminq(struct i40e_hw *hw); -i40e_status i40e_shutdown_adminq(struct i40e_hw *hw); +void i40e_shutdown_adminq(struct i40e_hw *hw); void i40e_adminq_init_ring_data(struct i40e_hw *hw); i40e_status i40e_clean_arq_element(struct i40e_hw *hw, struct i40e_arq_event_info *e, diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c index b8496037ef7f..f9555c847f73 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c @@ -521,28 +521,29 @@ int i40e_add_del_fdir(struct i40e_vsi *vsi, /** * i40e_fd_handle_status - check the Programming Status for FD * @rx_ring: the Rx ring for this descriptor - * @rx_desc: the Rx descriptor for programming Status, not a packet descriptor. + * @qword0_raw: qword0 + * @qword1: qword1 after le_to_cpu * @prog_id: the id originally used for programming * * This is used to verify if the FD programming or invalidation * requested by SW to the HW is successful or not and take actions accordingly. **/ -void i40e_fd_handle_status(struct i40e_ring *rx_ring, - union i40e_rx_desc *rx_desc, u8 prog_id) +static void i40e_fd_handle_status(struct i40e_ring *rx_ring, u64 qword0_raw, + u64 qword1, u8 prog_id) { struct i40e_pf *pf = rx_ring->vsi->back; struct pci_dev *pdev = pf->pdev; + struct i40e_32b_rx_wb_qw0 *qw0; u32 fcnt_prog, fcnt_avail; u32 error; - u64 qw; - qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len); - error = (qw & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >> + qw0 = (struct i40e_32b_rx_wb_qw0 *)&qword0_raw; + error = (qword1 & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >> I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT; if (error == BIT(I40E_RX_PROG_STATUS_DESC_FD_TBL_FULL_SHIFT)) { - pf->fd_inv = le32_to_cpu(rx_desc->wb.qword0.hi_dword.fd_id); - if ((rx_desc->wb.qword0.hi_dword.fd_id != 0) || + pf->fd_inv = le32_to_cpu(qw0->hi_dword.fd_id); + if (qw0->hi_dword.fd_id != 0 || (I40E_DEBUG_FD & pf->hw.debug_mask)) dev_warn(&pdev->dev, "ntuple filter loc = %d, could not be added\n", pf->fd_inv); @@ -560,7 +561,7 @@ void i40e_fd_handle_status(struct i40e_ring *rx_ring, /* store the current atr filter count */ pf->fd_atr_cnt = i40e_get_current_atr_cnt(pf); - if ((rx_desc->wb.qword0.hi_dword.fd_id == 0) && + if (qw0->hi_dword.fd_id == 0 && test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) { /* These set_bit() calls aren't atomic with the * test_bit() here, but worse case we potentially @@ -589,7 +590,7 @@ void i40e_fd_handle_status(struct i40e_ring *rx_ring, } else if (error == BIT(I40E_RX_PROG_STATUS_DESC_NO_FD_ENTRY_SHIFT)) { if (I40E_DEBUG_FD & pf->hw.debug_mask) dev_info(&pdev->dev, "ntuple filter fd_id = %d, could not be removed\n", - rx_desc->wb.qword0.hi_dword.fd_id); + qw0->hi_dword.fd_id); } } @@ -1195,6 +1196,11 @@ clear_counts: rc->total_packets = 0; } +static struct i40e_rx_buffer *i40e_rx_bi(struct i40e_ring *rx_ring, u32 idx) +{ + return &rx_ring->rx_bi[idx]; +} + /** * i40e_reuse_rx_page - page flip buffer and store it back on the ring * @rx_ring: rx descriptor ring to store buffers on @@ -1208,7 +1214,7 @@ static void i40e_reuse_rx_page(struct i40e_ring *rx_ring, struct i40e_rx_buffer *new_buff; u16 nta = rx_ring->next_to_alloc; - new_buff = &rx_ring->rx_bi[nta]; + new_buff = i40e_rx_bi(rx_ring, nta); /* update, and store next to alloc */ nta++; @@ -1227,29 +1233,10 @@ static void i40e_reuse_rx_page(struct i40e_ring *rx_ring, } /** - * i40e_rx_is_programming_status - check for programming status descriptor - * @qw: qword representing status_error_len in CPU ordering - * - * The value of in the descriptor length field indicate if this - * is a programming status descriptor for flow director or FCoE - * by the value of I40E_RX_PROG_STATUS_DESC_LENGTH, otherwise - * it is a packet descriptor. - **/ -static inline bool i40e_rx_is_programming_status(u64 qw) -{ - /* The Rx filter programming status and SPH bit occupy the same - * spot in the descriptor. Since we don't support packet split we - * can just reuse the bit as an indication that this is a - * programming status descriptor. - */ - return qw & I40E_RXD_QW1_LENGTH_SPH_MASK; -} - -/** - * i40e_clean_programming_status - try clean the programming status descriptor + * i40e_clean_programming_status - clean the programming status descriptor * @rx_ring: the rx ring that has this descriptor - * @rx_desc: the rx descriptor written back by HW - * @qw: qword representing status_error_len in CPU ordering + * @qword0_raw: qword0 + * @qword1: qword1 representing status_error_len in CPU ordering * * Flow director should handle FD_FILTER_STATUS to check its filter programming * status being successful or not and take actions accordingly. FCoE should @@ -1257,34 +1244,16 @@ static inline bool i40e_rx_is_programming_status(u64 qw) * * Returns an i40e_rx_buffer to reuse if the cleanup occurred, otherwise NULL. **/ -struct i40e_rx_buffer *i40e_clean_programming_status( - struct i40e_ring *rx_ring, - union i40e_rx_desc *rx_desc, - u64 qw) +void i40e_clean_programming_status(struct i40e_ring *rx_ring, u64 qword0_raw, + u64 qword1) { - struct i40e_rx_buffer *rx_buffer; - u32 ntc; u8 id; - if (!i40e_rx_is_programming_status(qw)) - return NULL; - - ntc = rx_ring->next_to_clean; - - /* fetch, update, and store next to clean */ - rx_buffer = &rx_ring->rx_bi[ntc++]; - ntc = (ntc < rx_ring->count) ? ntc : 0; - rx_ring->next_to_clean = ntc; - - prefetch(I40E_RX_DESC(rx_ring, ntc)); - - id = (qw & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >> + id = (qword1 & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >> I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT; if (id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS) - i40e_fd_handle_status(rx_ring, rx_desc, id); - - return rx_buffer; + i40e_fd_handle_status(rx_ring, qword0_raw, qword1, id); } /** @@ -1336,13 +1305,25 @@ err: return -ENOMEM; } +int i40e_alloc_rx_bi(struct i40e_ring *rx_ring) +{ + unsigned long sz = sizeof(*rx_ring->rx_bi) * rx_ring->count; + + rx_ring->rx_bi = kzalloc(sz, GFP_KERNEL); + return rx_ring->rx_bi ? 0 : -ENOMEM; +} + +static void i40e_clear_rx_bi(struct i40e_ring *rx_ring) +{ + memset(rx_ring->rx_bi, 0, sizeof(*rx_ring->rx_bi) * rx_ring->count); +} + /** * i40e_clean_rx_ring - Free Rx buffers * @rx_ring: ring to be cleaned **/ void i40e_clean_rx_ring(struct i40e_ring *rx_ring) { - unsigned long bi_size; u16 i; /* ring already cleared, nothing to do */ @@ -1361,7 +1342,7 @@ void i40e_clean_rx_ring(struct i40e_ring *rx_ring) /* Free all the Rx ring sk_buffs */ for (i = 0; i < rx_ring->count; i++) { - struct i40e_rx_buffer *rx_bi = &rx_ring->rx_bi[i]; + struct i40e_rx_buffer *rx_bi = i40e_rx_bi(rx_ring, i); if (!rx_bi->page) continue; @@ -1388,8 +1369,10 @@ void i40e_clean_rx_ring(struct i40e_ring *rx_ring) } skip_free: - bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count; - memset(rx_ring->rx_bi, 0, bi_size); + if (rx_ring->xsk_umem) + i40e_clear_rx_bi_zc(rx_ring); + else + i40e_clear_rx_bi(rx_ring); /* Zero out the descriptor ring */ memset(rx_ring->desc, 0, rx_ring->size); @@ -1430,15 +1413,7 @@ void i40e_free_rx_resources(struct i40e_ring *rx_ring) int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring) { struct device *dev = rx_ring->dev; - int err = -ENOMEM; - int bi_size; - - /* warn if we are about to overwrite the pointer */ - WARN_ON(rx_ring->rx_bi); - bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count; - rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL); - if (!rx_ring->rx_bi) - goto err; + int err; u64_stats_init(&rx_ring->syncp); @@ -1451,7 +1426,7 @@ int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring) if (!rx_ring->desc) { dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n", rx_ring->size); - goto err; + return -ENOMEM; } rx_ring->next_to_alloc = 0; @@ -1463,16 +1438,12 @@ int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring) err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->queue_index); if (err < 0) - goto err; + return err; } rx_ring->xdp_prog = rx_ring->vsi->xdp_prog; return 0; -err: - kfree(rx_ring->rx_bi); - rx_ring->rx_bi = NULL; - return err; } /** @@ -1507,6 +1478,22 @@ static inline unsigned int i40e_rx_offset(struct i40e_ring *rx_ring) return ring_uses_build_skb(rx_ring) ? I40E_SKB_PAD : 0; } +static unsigned int i40e_rx_frame_truesize(struct i40e_ring *rx_ring, + unsigned int size) +{ + unsigned int truesize; + +#if (PAGE_SIZE < 8192) + truesize = i40e_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */ +#else + truesize = i40e_rx_offset(rx_ring) ? + SKB_DATA_ALIGN(size + i40e_rx_offset(rx_ring)) + + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) : + SKB_DATA_ALIGN(size); +#endif + return truesize; +} + /** * i40e_alloc_mapped_page - recycle or make a new page * @rx_ring: ring to use @@ -1576,7 +1563,7 @@ bool i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count) return false; rx_desc = I40E_RX_DESC(rx_ring, ntu); - bi = &rx_ring->rx_bi[ntu]; + bi = i40e_rx_bi(rx_ring, ntu); do { if (!i40e_alloc_mapped_page(rx_ring, bi)) @@ -1598,7 +1585,7 @@ bool i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count) ntu++; if (unlikely(ntu == rx_ring->count)) { rx_desc = I40E_RX_DESC(rx_ring, 0); - bi = rx_ring->rx_bi; + bi = i40e_rx_bi(rx_ring, 0); ntu = 0; } @@ -1965,7 +1952,7 @@ static struct i40e_rx_buffer *i40e_get_rx_buffer(struct i40e_ring *rx_ring, { struct i40e_rx_buffer *rx_buffer; - rx_buffer = &rx_ring->rx_bi[rx_ring->next_to_clean]; + rx_buffer = i40e_rx_bi(rx_ring, rx_ring->next_to_clean); prefetchw(rx_buffer->page); /* we are reusing so sync this buffer for CPU use */ @@ -2180,7 +2167,7 @@ static int i40e_xmit_xdp_ring(struct xdp_frame *xdpf, int i40e_xmit_xdp_tx_ring(struct xdp_buff *xdp, struct i40e_ring *xdp_ring) { - struct xdp_frame *xdpf = convert_to_xdp_frame(xdp); + struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); if (unlikely(!xdpf)) return I40E_XDP_CONSUMED; @@ -2246,13 +2233,11 @@ static void i40e_rx_buffer_flip(struct i40e_ring *rx_ring, struct i40e_rx_buffer *rx_buffer, unsigned int size) { -#if (PAGE_SIZE < 8192) - unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2; + unsigned int truesize = i40e_rx_frame_truesize(rx_ring, size); +#if (PAGE_SIZE < 8192) rx_buffer->page_offset ^= truesize; #else - unsigned int truesize = SKB_DATA_ALIGN(i40e_rx_offset(rx_ring) + size); - rx_buffer->page_offset += truesize; #endif } @@ -2335,6 +2320,9 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget) bool failure = false; struct xdp_buff xdp; +#if (PAGE_SIZE < 8192) + xdp.frame_sz = i40e_rx_frame_truesize(rx_ring, 0); +#endif xdp.rxq = &rx_ring->xdp_rxq; while (likely(total_rx_packets < (unsigned int)budget)) { @@ -2365,9 +2353,12 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget) */ dma_rmb(); - rx_buffer = i40e_clean_programming_status(rx_ring, rx_desc, - qword); - if (unlikely(rx_buffer)) { + if (i40e_rx_is_programming_status(qword)) { + i40e_clean_programming_status(rx_ring, + rx_desc->raw.qword[0], + qword); + rx_buffer = i40e_rx_bi(rx_ring, rx_ring->next_to_clean); + i40e_inc_ntc(rx_ring); i40e_reuse_rx_page(rx_ring, rx_buffer); cleaned_count++; continue; @@ -2389,7 +2380,10 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget) xdp.data_hard_start = xdp.data - i40e_rx_offset(rx_ring); xdp.data_end = xdp.data + size; - +#if (PAGE_SIZE > 4096) + /* At larger PAGE_SIZE, frame_sz depend on len size */ + xdp.frame_sz = i40e_rx_frame_truesize(rx_ring, size); +#endif skb = i40e_run_xdp(rx_ring, &xdp); } diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.h b/drivers/net/ethernet/intel/i40e/i40e_txrx.h index 36d37f31a287..5c255977fd58 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.h +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.h @@ -296,17 +296,9 @@ struct i40e_tx_buffer { struct i40e_rx_buffer { dma_addr_t dma; - union { - struct { - struct page *page; - __u32 page_offset; - __u16 pagecnt_bias; - }; - struct { - void *addr; - u64 handle; - }; - }; + struct page *page; + __u32 page_offset; + __u16 pagecnt_bias; }; struct i40e_queue_stats { @@ -358,6 +350,7 @@ struct i40e_ring { union { struct i40e_tx_buffer *tx_bi; struct i40e_rx_buffer *rx_bi; + struct xdp_buff **rx_bi_zc; }; DECLARE_BITMAP(state, __I40E_RING_STATE_NBITS); u16 queue_index; /* Queue number of ring */ @@ -419,7 +412,6 @@ struct i40e_ring { struct i40e_channel *ch; struct xdp_rxq_info xdp_rxq; struct xdp_umem *xsk_umem; - struct zero_copy_allocator zca; /* ZC allocator anchor */ } ____cacheline_internodealigned_in_smp; static inline bool ring_uses_build_skb(struct i40e_ring *ring) @@ -495,6 +487,7 @@ int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size); bool __i40e_chk_linearize(struct sk_buff *skb); int i40e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, u32 flags); +int i40e_alloc_rx_bi(struct i40e_ring *rx_ring); /** * i40e_get_head - Retrieve head from head writeback diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx_common.h b/drivers/net/ethernet/intel/i40e/i40e_txrx_common.h index 8af0e99c6c0d..667c4dc4b39f 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx_common.h +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx_common.h @@ -4,13 +4,9 @@ #ifndef I40E_TXRX_COMMON_ #define I40E_TXRX_COMMON_ -void i40e_fd_handle_status(struct i40e_ring *rx_ring, - union i40e_rx_desc *rx_desc, u8 prog_id); int i40e_xmit_xdp_tx_ring(struct xdp_buff *xdp, struct i40e_ring *xdp_ring); -struct i40e_rx_buffer *i40e_clean_programming_status( - struct i40e_ring *rx_ring, - union i40e_rx_desc *rx_desc, - u64 qw); +void i40e_clean_programming_status(struct i40e_ring *rx_ring, u64 qword0_raw, + u64 qword1); void i40e_process_skb_fields(struct i40e_ring *rx_ring, union i40e_rx_desc *rx_desc, struct sk_buff *skb); void i40e_xdp_ring_update_tail(struct i40e_ring *xdp_ring); @@ -84,6 +80,38 @@ static inline void i40e_arm_wb(struct i40e_ring *tx_ring, } } +/** + * i40e_rx_is_programming_status - check for programming status descriptor + * @qword1: qword1 representing status_error_len in CPU ordering + * + * The value of in the descriptor length field indicate if this + * is a programming status descriptor for flow director or FCoE + * by the value of I40E_RX_PROG_STATUS_DESC_LENGTH, otherwise + * it is a packet descriptor. + **/ +static inline bool i40e_rx_is_programming_status(u64 qword1) +{ + /* The Rx filter programming status and SPH bit occupy the same + * spot in the descriptor. Since we don't support packet split we + * can just reuse the bit as an indication that this is a + * programming status descriptor. + */ + return qword1 & I40E_RXD_QW1_LENGTH_SPH_MASK; +} + +/** + * i40e_inc_ntc: Advance the next_to_clean index + * @rx_ring: Rx ring + **/ +static inline void i40e_inc_ntc(struct i40e_ring *rx_ring) +{ + u32 ntc = rx_ring->next_to_clean + 1; + + ntc = (ntc < rx_ring->count) ? ntc : 0; + rx_ring->next_to_clean = ntc; + prefetch(I40E_RX_DESC(rx_ring, ntc)); +} + void i40e_xsk_clean_rx_ring(struct i40e_ring *rx_ring); void i40e_xsk_clean_tx_ring(struct i40e_ring *tx_ring); bool i40e_xsk_any_rx_ring_enabled(struct i40e_vsi *vsi); diff --git a/drivers/net/ethernet/intel/i40e/i40e_type.h b/drivers/net/ethernet/intel/i40e/i40e_type.h index 6ea2867ff60f..63e098f7cb63 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_type.h +++ b/drivers/net/ethernet/intel/i40e/i40e_type.h @@ -689,7 +689,7 @@ union i40e_32byte_rx_desc { __le64 rsvd2; } read; struct { - struct { + struct i40e_32b_rx_wb_qw0 { struct { union { __le16 mirroring_status; @@ -727,6 +727,9 @@ union i40e_32byte_rx_desc { } hi_dword; } qword3; } wb; /* writeback */ + struct { + u64 qword[4]; + } raw; }; enum i40e_rx_desc_status_bits { diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c index 0b7d29192b2c..7276580cbe64 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c +++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c @@ -2,68 +2,30 @@ /* Copyright(c) 2018 Intel Corporation. */ #include <linux/bpf_trace.h> -#include <net/xdp_sock.h> +#include <net/xdp_sock_drv.h> #include <net/xdp.h> #include "i40e.h" #include "i40e_txrx_common.h" #include "i40e_xsk.h" -/** - * i40e_xsk_umem_dma_map - DMA maps all UMEM memory for the netdev - * @vsi: Current VSI - * @umem: UMEM to DMA map - * - * Returns 0 on success, <0 on failure - **/ -static int i40e_xsk_umem_dma_map(struct i40e_vsi *vsi, struct xdp_umem *umem) +int i40e_alloc_rx_bi_zc(struct i40e_ring *rx_ring) { - struct i40e_pf *pf = vsi->back; - struct device *dev; - unsigned int i, j; - dma_addr_t dma; - - dev = &pf->pdev->dev; - for (i = 0; i < umem->npgs; i++) { - dma = dma_map_page_attrs(dev, umem->pgs[i], 0, PAGE_SIZE, - DMA_BIDIRECTIONAL, I40E_RX_DMA_ATTR); - if (dma_mapping_error(dev, dma)) - goto out_unmap; + unsigned long sz = sizeof(*rx_ring->rx_bi_zc) * rx_ring->count; - umem->pages[i].dma = dma; - } - - return 0; - -out_unmap: - for (j = 0; j < i; j++) { - dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE, - DMA_BIDIRECTIONAL, I40E_RX_DMA_ATTR); - umem->pages[i].dma = 0; - } - - return -1; + rx_ring->rx_bi_zc = kzalloc(sz, GFP_KERNEL); + return rx_ring->rx_bi_zc ? 0 : -ENOMEM; } -/** - * i40e_xsk_umem_dma_unmap - DMA unmaps all UMEM memory for the netdev - * @vsi: Current VSI - * @umem: UMEM to DMA map - **/ -static void i40e_xsk_umem_dma_unmap(struct i40e_vsi *vsi, struct xdp_umem *umem) +void i40e_clear_rx_bi_zc(struct i40e_ring *rx_ring) { - struct i40e_pf *pf = vsi->back; - struct device *dev; - unsigned int i; - - dev = &pf->pdev->dev; - - for (i = 0; i < umem->npgs; i++) { - dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE, - DMA_BIDIRECTIONAL, I40E_RX_DMA_ATTR); + memset(rx_ring->rx_bi_zc, 0, + sizeof(*rx_ring->rx_bi_zc) * rx_ring->count); +} - umem->pages[i].dma = 0; - } +static struct xdp_buff **i40e_rx_bi(struct i40e_ring *rx_ring, u32 idx) +{ + return &rx_ring->rx_bi_zc[idx]; } /** @@ -78,7 +40,6 @@ static int i40e_xsk_umem_enable(struct i40e_vsi *vsi, struct xdp_umem *umem, u16 qid) { struct net_device *netdev = vsi->netdev; - struct xdp_umem_fq_reuse *reuseq; bool if_running; int err; @@ -92,13 +53,7 @@ static int i40e_xsk_umem_enable(struct i40e_vsi *vsi, struct xdp_umem *umem, qid >= netdev->real_num_tx_queues) return -EINVAL; - reuseq = xsk_reuseq_prepare(vsi->rx_rings[0]->count); - if (!reuseq) - return -ENOMEM; - - xsk_reuseq_free(xsk_reuseq_swap(umem, reuseq)); - - err = i40e_xsk_umem_dma_map(vsi, umem); + err = xsk_buff_dma_map(umem, &vsi->back->pdev->dev, I40E_RX_DMA_ATTR); if (err) return err; @@ -151,7 +106,7 @@ static int i40e_xsk_umem_disable(struct i40e_vsi *vsi, u16 qid) } clear_bit(qid, vsi->af_xdp_zc_qps); - i40e_xsk_umem_dma_unmap(vsi, umem); + xsk_buff_dma_unmap(umem, I40E_RX_DMA_ATTR); if (if_running) { err = i40e_queue_pair_enable(vsi, qid); @@ -184,17 +139,13 @@ int i40e_xsk_umem_setup(struct i40e_vsi *vsi, struct xdp_umem *umem, * @rx_ring: Rx ring * @xdp: xdp_buff used as input to the XDP program * - * This function enables or disables a UMEM to a certain ring. - * * Returns any of I40E_XDP_{PASS, CONSUMED, TX, REDIR} **/ static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp) { - struct xdp_umem *umem = rx_ring->xsk_umem; int err, result = I40E_XDP_PASS; struct i40e_ring *xdp_ring; struct bpf_prog *xdp_prog; - u64 offset; u32 act; rcu_read_lock(); @@ -203,9 +154,6 @@ static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp) */ xdp_prog = READ_ONCE(rx_ring->xdp_prog); act = bpf_prog_run_xdp(xdp_prog, xdp); - offset = xdp->data - xdp->data_hard_start; - - xdp->handle = xsk_umem_adjust_offset(umem, xdp->handle, offset); switch (act) { case XDP_PASS: @@ -232,107 +180,26 @@ static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp) return result; } -/** - * i40e_alloc_buffer_zc - Allocates an i40e_rx_buffer - * @rx_ring: Rx ring - * @bi: Rx buffer to populate - * - * This function allocates an Rx buffer. The buffer can come from fill - * queue, or via the recycle queue (next_to_alloc). - * - * Returns true for a successful allocation, false otherwise - **/ -static bool i40e_alloc_buffer_zc(struct i40e_ring *rx_ring, - struct i40e_rx_buffer *bi) -{ - struct xdp_umem *umem = rx_ring->xsk_umem; - void *addr = bi->addr; - u64 handle, hr; - - if (addr) { - rx_ring->rx_stats.page_reuse_count++; - return true; - } - - if (!xsk_umem_peek_addr(umem, &handle)) { - rx_ring->rx_stats.alloc_page_failed++; - return false; - } - - hr = umem->headroom + XDP_PACKET_HEADROOM; - - bi->dma = xdp_umem_get_dma(umem, handle); - bi->dma += hr; - - bi->addr = xdp_umem_get_data(umem, handle); - bi->addr += hr; - - bi->handle = xsk_umem_adjust_offset(umem, handle, umem->headroom); - - xsk_umem_release_addr(umem); - return true; -} - -/** - * i40e_alloc_buffer_slow_zc - Allocates an i40e_rx_buffer - * @rx_ring: Rx ring - * @bi: Rx buffer to populate - * - * This function allocates an Rx buffer. The buffer can come from fill - * queue, or via the reuse queue. - * - * Returns true for a successful allocation, false otherwise - **/ -static bool i40e_alloc_buffer_slow_zc(struct i40e_ring *rx_ring, - struct i40e_rx_buffer *bi) -{ - struct xdp_umem *umem = rx_ring->xsk_umem; - u64 handle, hr; - - if (!xsk_umem_peek_addr_rq(umem, &handle)) { - rx_ring->rx_stats.alloc_page_failed++; - return false; - } - - handle &= rx_ring->xsk_umem->chunk_mask; - - hr = umem->headroom + XDP_PACKET_HEADROOM; - - bi->dma = xdp_umem_get_dma(umem, handle); - bi->dma += hr; - - bi->addr = xdp_umem_get_data(umem, handle); - bi->addr += hr; - - bi->handle = xsk_umem_adjust_offset(umem, handle, umem->headroom); - - xsk_umem_release_addr_rq(umem); - return true; -} - -static __always_inline bool -__i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count, - bool alloc(struct i40e_ring *rx_ring, - struct i40e_rx_buffer *bi)) +bool i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count) { u16 ntu = rx_ring->next_to_use; union i40e_rx_desc *rx_desc; - struct i40e_rx_buffer *bi; + struct xdp_buff **bi, *xdp; + dma_addr_t dma; bool ok = true; rx_desc = I40E_RX_DESC(rx_ring, ntu); - bi = &rx_ring->rx_bi[ntu]; + bi = i40e_rx_bi(rx_ring, ntu); do { - if (!alloc(rx_ring, bi)) { + xdp = xsk_buff_alloc(rx_ring->xsk_umem); + if (!xdp) { ok = false; goto no_buffers; } - - dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 0, - rx_ring->rx_buf_len, - DMA_BIDIRECTIONAL); - - rx_desc->read.pkt_addr = cpu_to_le64(bi->dma); + *bi = xdp; + dma = xsk_buff_xdp_get_dma(xdp); + rx_desc->read.pkt_addr = cpu_to_le64(dma); + rx_desc->read.hdr_addr = 0; rx_desc++; bi++; @@ -340,11 +207,10 @@ __i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count, if (unlikely(ntu == rx_ring->count)) { rx_desc = I40E_RX_DESC(rx_ring, 0); - bi = rx_ring->rx_bi; + bi = i40e_rx_bi(rx_ring, 0); ntu = 0; } - rx_desc->wb.qword1.status_error_len = 0; count--; } while (count); @@ -356,127 +222,8 @@ no_buffers: } /** - * i40e_alloc_rx_buffers_zc - Allocates a number of Rx buffers - * @rx_ring: Rx ring - * @count: The number of buffers to allocate - * - * This function allocates a number of Rx buffers from the reuse queue - * or fill ring and places them on the Rx ring. - * - * Returns true for a successful allocation, false otherwise - **/ -bool i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count) -{ - return __i40e_alloc_rx_buffers_zc(rx_ring, count, - i40e_alloc_buffer_slow_zc); -} - -/** - * i40e_alloc_rx_buffers_fast_zc - Allocates a number of Rx buffers - * @rx_ring: Rx ring - * @count: The number of buffers to allocate - * - * This function allocates a number of Rx buffers from the fill ring - * or the internal recycle mechanism and places them on the Rx ring. - * - * Returns true for a successful allocation, false otherwise - **/ -static bool i40e_alloc_rx_buffers_fast_zc(struct i40e_ring *rx_ring, u16 count) -{ - return __i40e_alloc_rx_buffers_zc(rx_ring, count, - i40e_alloc_buffer_zc); -} - -/** - * i40e_get_rx_buffer_zc - Return the current Rx buffer - * @rx_ring: Rx ring - * @size: The size of the rx buffer (read from descriptor) - * - * This function returns the current, received Rx buffer, and also - * does DMA synchronization. the Rx ring. - * - * Returns the received Rx buffer - **/ -static struct i40e_rx_buffer *i40e_get_rx_buffer_zc(struct i40e_ring *rx_ring, - const unsigned int size) -{ - struct i40e_rx_buffer *bi; - - bi = &rx_ring->rx_bi[rx_ring->next_to_clean]; - - /* we are reusing so sync this buffer for CPU use */ - dma_sync_single_range_for_cpu(rx_ring->dev, - bi->dma, 0, - size, - DMA_BIDIRECTIONAL); - - return bi; -} - -/** - * i40e_reuse_rx_buffer_zc - Recycle an Rx buffer - * @rx_ring: Rx ring - * @old_bi: The Rx buffer to recycle - * - * This function recycles a finished Rx buffer, and places it on the - * recycle queue (next_to_alloc). - **/ -static void i40e_reuse_rx_buffer_zc(struct i40e_ring *rx_ring, - struct i40e_rx_buffer *old_bi) -{ - struct i40e_rx_buffer *new_bi = &rx_ring->rx_bi[rx_ring->next_to_alloc]; - u16 nta = rx_ring->next_to_alloc; - - /* update, and store next to alloc */ - nta++; - rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; - - /* transfer page from old buffer to new buffer */ - new_bi->dma = old_bi->dma; - new_bi->addr = old_bi->addr; - new_bi->handle = old_bi->handle; - - old_bi->addr = NULL; -} - -/** - * i40e_zca_free - Free callback for MEM_TYPE_ZERO_COPY allocations - * @alloc: Zero-copy allocator - * @handle: Buffer handle - **/ -void i40e_zca_free(struct zero_copy_allocator *alloc, unsigned long handle) -{ - struct i40e_rx_buffer *bi; - struct i40e_ring *rx_ring; - u64 hr, mask; - u16 nta; - - rx_ring = container_of(alloc, struct i40e_ring, zca); - hr = rx_ring->xsk_umem->headroom + XDP_PACKET_HEADROOM; - mask = rx_ring->xsk_umem->chunk_mask; - - nta = rx_ring->next_to_alloc; - bi = &rx_ring->rx_bi[nta]; - - nta++; - rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; - - handle &= mask; - - bi->dma = xdp_umem_get_dma(rx_ring->xsk_umem, handle); - bi->dma += hr; - - bi->addr = xdp_umem_get_data(rx_ring->xsk_umem, handle); - bi->addr += hr; - - bi->handle = xsk_umem_adjust_offset(rx_ring->xsk_umem, (u64)handle, - rx_ring->xsk_umem->headroom); -} - -/** - * i40e_construct_skb_zc - Create skbufff from zero-copy Rx buffer + * i40e_construct_skb_zc - Create skbuff from zero-copy Rx buffer * @rx_ring: Rx ring - * @bi: Rx buffer * @xdp: xdp_buff * * This functions allocates a new skb from a zero-copy Rx buffer. @@ -484,7 +231,6 @@ void i40e_zca_free(struct zero_copy_allocator *alloc, unsigned long handle) * Returns the skb, or NULL on failure. **/ static struct sk_buff *i40e_construct_skb_zc(struct i40e_ring *rx_ring, - struct i40e_rx_buffer *bi, struct xdp_buff *xdp) { unsigned int metasize = xdp->data - xdp->data_meta; @@ -503,24 +249,11 @@ static struct sk_buff *i40e_construct_skb_zc(struct i40e_ring *rx_ring, if (metasize) skb_metadata_set(skb, metasize); - i40e_reuse_rx_buffer_zc(rx_ring, bi); + xsk_buff_free(xdp); return skb; } /** - * i40e_inc_ntc: Advance the next_to_clean index - * @rx_ring: Rx ring - **/ -static void i40e_inc_ntc(struct i40e_ring *rx_ring) -{ - u32 ntc = rx_ring->next_to_clean + 1; - - ntc = (ntc < rx_ring->count) ? ntc : 0; - rx_ring->next_to_clean = ntc; - prefetch(I40E_RX_DESC(rx_ring, ntc)); -} - -/** * i40e_clean_rx_irq_zc - Consumes Rx packets from the hardware ring * @rx_ring: Rx ring * @budget: NAPI budget @@ -534,20 +267,17 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget) unsigned int xdp_res, xdp_xmit = 0; bool failure = false; struct sk_buff *skb; - struct xdp_buff xdp; - - xdp.rxq = &rx_ring->xdp_rxq; while (likely(total_rx_packets < (unsigned int)budget)) { - struct i40e_rx_buffer *bi; union i40e_rx_desc *rx_desc; + struct xdp_buff **bi; unsigned int size; u64 qword; if (cleaned_count >= I40E_RX_BUFFER_WRITE) { failure = failure || - !i40e_alloc_rx_buffers_fast_zc(rx_ring, - cleaned_count); + !i40e_alloc_rx_buffers_zc(rx_ring, + cleaned_count); cleaned_count = 0; } @@ -560,35 +290,36 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget) */ dma_rmb(); - bi = i40e_clean_programming_status(rx_ring, rx_desc, - qword); - if (unlikely(bi)) { - i40e_reuse_rx_buffer_zc(rx_ring, bi); + if (i40e_rx_is_programming_status(qword)) { + i40e_clean_programming_status(rx_ring, + rx_desc->raw.qword[0], + qword); + bi = i40e_rx_bi(rx_ring, rx_ring->next_to_clean); + xsk_buff_free(*bi); + *bi = NULL; cleaned_count++; + i40e_inc_ntc(rx_ring); continue; } + bi = i40e_rx_bi(rx_ring, rx_ring->next_to_clean); size = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >> I40E_RXD_QW1_LENGTH_PBUF_SHIFT; if (!size) break; - bi = i40e_get_rx_buffer_zc(rx_ring, size); - xdp.data = bi->addr; - xdp.data_meta = xdp.data; - xdp.data_hard_start = xdp.data - XDP_PACKET_HEADROOM; - xdp.data_end = xdp.data + size; - xdp.handle = bi->handle; + bi = i40e_rx_bi(rx_ring, rx_ring->next_to_clean); + (*bi)->data_end = (*bi)->data + size; + xsk_buff_dma_sync_for_cpu(*bi); - xdp_res = i40e_run_xdp_zc(rx_ring, &xdp); + xdp_res = i40e_run_xdp_zc(rx_ring, *bi); if (xdp_res) { - if (xdp_res & (I40E_XDP_TX | I40E_XDP_REDIR)) { + if (xdp_res & (I40E_XDP_TX | I40E_XDP_REDIR)) xdp_xmit |= xdp_res; - bi->addr = NULL; - } else { - i40e_reuse_rx_buffer_zc(rx_ring, bi); - } + else + xsk_buff_free(*bi); + *bi = NULL; total_rx_bytes += size; total_rx_packets++; @@ -604,7 +335,8 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget) * BIT(I40E_RXD_QW1_ERROR_SHIFT). This is due to that * SBP is *not* set in PRT_SBPVSI (default not set). */ - skb = i40e_construct_skb_zc(rx_ring, bi, &xdp); + skb = i40e_construct_skb_zc(rx_ring, *bi); + *bi = NULL; if (!skb) { rx_ring->rx_stats.alloc_buff_failed++; break; @@ -662,10 +394,9 @@ static bool i40e_xmit_zc(struct i40e_ring *xdp_ring, unsigned int budget) if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &desc)) break; - dma = xdp_umem_get_dma(xdp_ring->xsk_umem, desc.addr); - - dma_sync_single_for_device(xdp_ring->dev, dma, desc.len, - DMA_BIDIRECTIONAL); + dma = xsk_buff_raw_get_dma(xdp_ring->xsk_umem, desc.addr); + xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_umem, dma, + desc.len); tx_bi = &xdp_ring->tx_bi[xdp_ring->next_to_use]; tx_bi->bytecount = desc.len; @@ -824,13 +555,13 @@ void i40e_xsk_clean_rx_ring(struct i40e_ring *rx_ring) u16 i; for (i = 0; i < rx_ring->count; i++) { - struct i40e_rx_buffer *rx_bi = &rx_ring->rx_bi[i]; + struct xdp_buff *rx_bi = *i40e_rx_bi(rx_ring, i); - if (!rx_bi->addr) + if (!rx_bi) continue; - xsk_umem_fq_reuse(rx_ring->xsk_umem, rx_bi->handle); - rx_bi->addr = NULL; + xsk_buff_free(rx_bi); + rx_bi = NULL; } } diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.h b/drivers/net/ethernet/intel/i40e/i40e_xsk.h index 9ed59c14eb55..ea919a7d60ec 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_xsk.h +++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.h @@ -12,12 +12,13 @@ int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair); int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair); int i40e_xsk_umem_setup(struct i40e_vsi *vsi, struct xdp_umem *umem, u16 qid); -void i40e_zca_free(struct zero_copy_allocator *alloc, unsigned long handle); bool i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 cleaned_count); int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget); bool i40e_clean_xdp_tx_irq(struct i40e_vsi *vsi, struct i40e_ring *tx_ring, int napi_budget); int i40e_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags); +int i40e_alloc_rx_bi_zc(struct i40e_ring *rx_ring); +void i40e_clear_rx_bi_zc(struct i40e_ring *rx_ring); #endif /* _I40E_XSK_H_ */ diff --git a/drivers/net/ethernet/intel/ice/Makefile b/drivers/net/ethernet/intel/ice/Makefile index 29c6c6743450..980bbcc64b4b 100644 --- a/drivers/net/ethernet/intel/ice/Makefile +++ b/drivers/net/ethernet/intel/ice/Makefile @@ -17,10 +17,14 @@ ice-y := ice_main.o \ ice_lib.o \ ice_txrx_lib.o \ ice_txrx.o \ + ice_fltr.o \ + ice_fdir.o \ + ice_ethtool_fdir.o \ ice_flex_pipe.o \ ice_flow.o \ ice_devlink.o \ ice_ethtool.o ice-$(CONFIG_PCI_IOV) += ice_virtchnl_pf.o ice_sriov.o ice-$(CONFIG_DCB) += ice_dcb.o ice_dcb_nl.o ice_dcb_lib.o +ice-$(CONFIG_RFS_ACCEL) += ice_arfs.o ice-$(CONFIG_XDP_SOCKETS) += ice_xsk.o diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h index 5c11448bfbb3..5792ee616b5c 100644 --- a/drivers/net/ethernet/intel/ice/ice.h +++ b/drivers/net/ethernet/intel/ice/ice.h @@ -34,9 +34,14 @@ #include <linux/ctype.h> #include <linux/bpf.h> #include <linux/avf/virtchnl.h> +#include <linux/cpu_rmap.h> #include <net/devlink.h> #include <net/ipv6.h> #include <net/xdp_sock.h> +#include <net/geneve.h> +#include <net/gre.h> +#include <net/udp_tunnel.h> +#include <net/vxlan.h> #include "ice_devids.h" #include "ice_type.h" #include "ice_txrx.h" @@ -46,7 +51,9 @@ #include "ice_sched.h" #include "ice_virtchnl_pf.h" #include "ice_sriov.h" +#include "ice_fdir.h" #include "ice_xsk.h" +#include "ice_arfs.h" extern const char ice_drv_ver[]; #define ICE_BAR0 0 @@ -62,6 +69,7 @@ extern const char ice_drv_ver[]; #define ICE_AQ_LEN 64 #define ICE_MBXSQ_LEN 64 #define ICE_MIN_MSIX 2 +#define ICE_FDIR_MSIX 1 #define ICE_NO_VSI 0xffff #define ICE_VSI_MAP_CONTIG 0 #define ICE_VSI_MAP_SCATTER 1 @@ -90,6 +98,7 @@ extern const char ice_drv_ver[]; #define ICE_TX_DESC(R, i) (&(((struct ice_tx_desc *)((R)->desc))[i])) #define ICE_RX_DESC(R, i) (&(((union ice_32b_rx_flex_desc *)((R)->desc))[i])) #define ICE_TX_CTX_DESC(R, i) (&(((struct ice_tx_ctx_desc *)((R)->desc))[i])) +#define ICE_TX_FDIRDESC(R, i) (&(((struct ice_fltr_desc *)((R)->desc))[i])) /* Macro for each VSI in a PF */ #define ice_for_each_vsi(pf, i) \ @@ -210,6 +219,7 @@ enum ice_state { __ICE_CFG_BUSY, __ICE_SERVICE_SCHED, __ICE_SERVICE_DIS, + __ICE_FD_FLUSH_REQ, __ICE_OICR_INTR_DIS, /* Global OICR interrupt disabled */ __ICE_MDD_VF_PRINT_PENDING, /* set when MDD event handle */ __ICE_VF_RESETS_DISABLED, /* disable resets during ice_remove */ @@ -244,8 +254,8 @@ struct ice_vsi { u32 tx_busy; u32 rx_buf_failed; u32 rx_page_failed; - int num_q_vectors; - int base_vector; /* IRQ base for OS reserved vectors */ + u16 num_q_vectors; + u16 base_vector; /* IRQ base for OS reserved vectors */ enum ice_vsi_type type; u16 vsi_num; /* HW (absolute) index of this VSI */ u16 idx; /* software index in pf->vsi[] */ @@ -253,6 +263,8 @@ struct ice_vsi { s16 vf_id; /* VF ID for SR-IOV VSIs */ u16 ethtype; /* Ethernet protocol for pause frame */ + u16 num_gfltr; + u16 num_bfltr; /* RSS config */ u16 rss_table_size; /* HW RSS table size */ @@ -261,6 +273,14 @@ struct ice_vsi { u8 *rss_lut_user; /* User configured lookup table entries */ u8 rss_lut_type; /* used to configure Get/Set RSS LUT AQ call */ + /* aRFS members only allocated for the PF VSI */ +#define ICE_MAX_ARFS_LIST 1024 +#define ICE_ARFS_LST_MASK (ICE_MAX_ARFS_LIST - 1) + struct hlist_head *arfs_fltr_list; + struct ice_arfs_active_fltr_cntrs *arfs_fltr_cntrs; + spinlock_t arfs_lock; /* protects aRFS hash table and filter state */ + atomic_t *arfs_last_fltr_id; + u16 max_frame; u16 rx_buf_len; @@ -335,12 +355,14 @@ enum ice_pf_flags { ICE_FLAG_SRIOV_CAPABLE, ICE_FLAG_DCB_CAPABLE, ICE_FLAG_DCB_ENA, + ICE_FLAG_FD_ENA, ICE_FLAG_ADV_FEATURES, ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, ICE_FLAG_NO_MEDIA, ICE_FLAG_FW_LLDP_AGENT, ICE_FLAG_ETHTOOL_CTXT, /* set when ethtool holds RTNL lock */ ICE_FLAG_LEGACY_RX, + ICE_FLAG_VF_TRUE_PROMISC_ENA, ICE_FLAG_MDD_AUTO_RESET_VF, ICE_PF_FLAGS_NBITS /* must be last */ }; @@ -362,11 +384,13 @@ struct ice_pf { */ u16 sriov_base_vector; + u16 ctrl_vsi_idx; /* control VSI index in pf->vsi array */ + struct ice_vsi **vsi; /* VSIs created by the driver */ struct ice_sw *first_sw; /* first switch created by firmware */ /* Virtchnl/SR-IOV config info */ struct ice_vf *vf; - int num_alloc_vfs; /* actual number of VFs allocated */ + u16 num_alloc_vfs; /* actual number of VFs allocated */ u16 num_vfs_supported; /* num VFs supported for this PF */ u16 num_qps_per_vf; u16 num_msix_per_vf; @@ -385,11 +409,11 @@ struct ice_pf { struct mutex tc_mutex; /* lock to protect TC changes */ u32 msg_enable; u32 hw_csum_rx_error; - u32 oicr_idx; /* Other interrupt cause MSIX vector index */ - u32 num_avail_sw_msix; /* remaining MSIX SW vectors left unclaimed */ + u16 oicr_idx; /* Other interrupt cause MSIX vector index */ + u16 num_avail_sw_msix; /* remaining MSIX SW vectors left unclaimed */ u16 max_pf_txqs; /* Total Tx queues PF wide */ u16 max_pf_rxqs; /* Total Rx queues PF wide */ - u32 num_lan_msix; /* Total MSIX vectors for base driver */ + u16 num_lan_msix; /* Total MSIX vectors for base driver */ u16 num_lan_tx; /* num LAN Tx queues setup */ u16 num_lan_rx; /* num LAN Rx queues setup */ u16 next_vsi; /* Next free slot in pf->vsi[] - 0-based! */ @@ -500,8 +524,27 @@ static inline struct ice_vsi *ice_get_main_vsi(struct ice_pf *pf) return NULL; } +/** + * ice_get_ctrl_vsi - Get the control VSI + * @pf: PF instance + */ +static inline struct ice_vsi *ice_get_ctrl_vsi(struct ice_pf *pf) +{ + /* if pf->ctrl_vsi_idx is ICE_NO_VSI, control VSI was not set up */ + if (!pf->vsi || pf->ctrl_vsi_idx == ICE_NO_VSI) + return NULL; + + return pf->vsi[pf->ctrl_vsi_idx]; +} + +#define ICE_FD_STAT_CTR_BLOCK_COUNT 256 +#define ICE_FD_STAT_PF_IDX(base_idx) \ + ((base_idx) * ICE_FD_STAT_CTR_BLOCK_COUNT) +#define ICE_FD_SB_STAT_IDX(base_idx) ICE_FD_STAT_PF_IDX(base_idx) + int ice_vsi_setup_tx_rings(struct ice_vsi *vsi); int ice_vsi_setup_rx_rings(struct ice_vsi *vsi); +int ice_vsi_open_ctrl(struct ice_vsi *vsi); void ice_set_ethtool_ops(struct net_device *netdev); void ice_set_ethtool_safe_mode_ops(struct net_device *netdev); u16 ice_get_avail_txq_count(struct ice_pf *pf); @@ -523,7 +566,24 @@ int ice_get_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size); void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size); int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset); void ice_print_link_msg(struct ice_vsi *vsi, bool isup); +const char *ice_stat_str(enum ice_status stat_err); +const char *ice_aq_str(enum ice_aq_err aq_err); +int +ice_fdir_write_fltr(struct ice_pf *pf, struct ice_fdir_fltr *input, bool add, + bool is_tun); +void ice_vsi_manage_fdir(struct ice_vsi *vsi, bool ena); +int ice_add_fdir_ethtool(struct ice_vsi *vsi, struct ethtool_rxnfc *cmd); +int ice_del_fdir_ethtool(struct ice_vsi *vsi, struct ethtool_rxnfc *cmd); +int ice_get_ethtool_fdir_entry(struct ice_hw *hw, struct ethtool_rxnfc *cmd); +int +ice_get_fdir_fltr_ids(struct ice_hw *hw, struct ethtool_rxnfc *cmd, + u32 *rule_locs); +void ice_fdir_release_flows(struct ice_hw *hw); +void ice_fdir_replay_flows(struct ice_hw *hw); +void ice_fdir_replay_fltrs(struct ice_pf *pf); +int ice_fdir_create_dflt_rules(struct ice_pf *pf); int ice_open(struct net_device *netdev); int ice_stop(struct net_device *netdev); +void ice_service_task_schedule(struct ice_pf *pf); #endif /* _ICE_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h index 2381b4014ed6..92f82f2a8af4 100644 --- a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h +++ b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h @@ -107,6 +107,7 @@ struct ice_aqc_list_caps_elem { #define ICE_AQC_CAPS_RXQS 0x0041 #define ICE_AQC_CAPS_TXQS 0x0042 #define ICE_AQC_CAPS_MSIX 0x0043 +#define ICE_AQC_CAPS_FD 0x0045 #define ICE_AQC_CAPS_MAX_MTU 0x0047 u8 major_ver; @@ -155,13 +156,11 @@ struct ice_aqc_manage_mac_write { #define ICE_AQC_MAN_MAC_WR_MC_MAG_EN BIT(0) #define ICE_AQC_MAN_MAC_WR_WOL_LAA_PFR_KEEP BIT(1) #define ICE_AQC_MAN_MAC_WR_S 6 -#define ICE_AQC_MAN_MAC_WR_M (3 << ICE_AQC_MAN_MAC_WR_S) +#define ICE_AQC_MAN_MAC_WR_M ICE_M(3, ICE_AQC_MAN_MAC_WR_S) #define ICE_AQC_MAN_MAC_UPDATE_LAA 0 -#define ICE_AQC_MAN_MAC_UPDATE_LAA_WOL (BIT(0) << ICE_AQC_MAN_MAC_WR_S) - /* High 16 bits of MAC address in big endian order */ - __be16 sah; - /* Low 32 bits of MAC address in big endian order */ - __be32 sal; +#define ICE_AQC_MAN_MAC_UPDATE_LAA_WOL BIT(ICE_AQC_MAN_MAC_WR_S) + /* byte stream in network order */ + u8 mac_addr[ETH_ALEN]; __le32 addr_high; __le32 addr_low; }; @@ -232,6 +231,11 @@ struct ice_aqc_get_sw_cfg_resp { */ #define ICE_AQC_RES_TYPE_VSI_LIST_REP 0x03 #define ICE_AQC_RES_TYPE_VSI_LIST_PRUNE 0x04 +#define ICE_AQC_RES_TYPE_FDIR_COUNTER_BLOCK 0x21 +#define ICE_AQC_RES_TYPE_FDIR_GUARANTEED_ENTRIES 0x22 +#define ICE_AQC_RES_TYPE_FDIR_SHARED_ENTRIES 0x23 +#define ICE_AQC_RES_TYPE_FD_PROF_BLDR_PROFID 0x58 +#define ICE_AQC_RES_TYPE_FD_PROF_BLDR_TCAM 0x59 #define ICE_AQC_RES_TYPE_HASH_PROF_BLDR_PROFID 0x60 #define ICE_AQC_RES_TYPE_HASH_PROF_BLDR_TCAM 0x61 @@ -240,6 +244,9 @@ struct ice_aqc_get_sw_cfg_resp { #define ICE_AQC_RES_TYPE_FLAG_DEDICATED 0x00 +#define ICE_AQC_RES_TYPE_S 0 +#define ICE_AQC_RES_TYPE_M (0x07F << ICE_AQC_RES_TYPE_S) + /* Allocate Resources command (indirect 0x0208) * Free Resources command (indirect 0x0209) */ @@ -541,7 +548,7 @@ struct ice_sw_rule_lkup_rx_tx { #define ICE_SINGLE_ACT_OTHER_ACTS 0x3 #define ICE_SINGLE_OTHER_ACT_IDENTIFIER_S 17 #define ICE_SINGLE_OTHER_ACT_IDENTIFIER_M \ - (0x3 << \ ICE_SINGLE_OTHER_ACT_IDENTIFIER_S) + (0x3 << ICE_SINGLE_OTHER_ACT_IDENTIFIER_S) /* Bit 17:18 - Defines other actions */ /* Other action = 0 - Mirror VSI */ @@ -967,7 +974,7 @@ struct ice_aqc_get_phy_caps { #define ICE_PHY_TYPE_HIGH_100G_CAUI2 BIT_ULL(2) #define ICE_PHY_TYPE_HIGH_100G_AUI2_AOC_ACC BIT_ULL(3) #define ICE_PHY_TYPE_HIGH_100G_AUI2 BIT_ULL(4) -#define ICE_PHY_TYPE_HIGH_MAX_INDEX 19 +#define ICE_PHY_TYPE_HIGH_MAX_INDEX 5 struct ice_aqc_get_phy_caps_data { __le64 phy_type_low; /* Use values from ICE_PHY_TYPE_LOW_* */ @@ -1059,6 +1066,25 @@ struct ice_aqc_set_phy_cfg_data { u8 rsvd1; }; +/* Set MAC Config command data structure (direct 0x0603) */ +struct ice_aqc_set_mac_cfg { + __le16 max_frame_size; + u8 params; +#define ICE_AQ_SET_MAC_PACE_S 3 +#define ICE_AQ_SET_MAC_PACE_M (0xF << ICE_AQ_SET_MAC_PACE_S) +#define ICE_AQ_SET_MAC_PACE_TYPE_M BIT(7) +#define ICE_AQ_SET_MAC_PACE_TYPE_RATE 0 +#define ICE_AQ_SET_MAC_PACE_TYPE_FIXED ICE_AQ_SET_MAC_PACE_TYPE_M + u8 tx_tmr_priority; + __le16 tx_tmr_value; + __le16 fc_refresh_threshold; + u8 drop_opts; +#define ICE_AQ_SET_MAC_AUTO_DROP_MASK BIT(0) +#define ICE_AQ_SET_MAC_AUTO_DROP_NONE 0 +#define ICE_AQ_SET_MAC_AUTO_DROP_BLOCKING_PKTS BIT(0) + u8 reserved[7]; +}; + /* Restart AN command data structure (direct 0x0605) * Also used for response, with only the lport_num field present. */ @@ -1264,6 +1290,33 @@ struct ice_aqc_nvm_checksum { u8 rsvd2[12]; }; +/* The result of netlist NVM read comes in a TLV format. The actual data + * (netlist header) starts from word offset 1 (byte 2). The FW strips + * out the type field from the TLV header so all the netlist fields + * should adjust their offset value by 1 word (2 bytes) in order to map + * their correct location. + */ +#define ICE_AQC_NVM_LINK_TOPO_NETLIST_MOD_ID 0x11B +#define ICE_AQC_NVM_LINK_TOPO_NETLIST_LEN_OFFSET 1 +#define ICE_AQC_NVM_LINK_TOPO_NETLIST_LEN 2 /* In bytes */ +#define ICE_AQC_NVM_NETLIST_NODE_COUNT_OFFSET 2 +#define ICE_AQC_NVM_NETLIST_NODE_COUNT_LEN 2 /* In bytes */ +#define ICE_AQC_NVM_NETLIST_NODE_COUNT_M ICE_M(0x3FF, 0) +#define ICE_AQC_NVM_NETLIST_ID_BLK_START_OFFSET 5 +#define ICE_AQC_NVM_NETLIST_ID_BLK_LEN 0x30 /* In words */ + +/* netlist ID block field offsets (word offsets) */ +#define ICE_AQC_NVM_NETLIST_ID_BLK_MAJOR_VER_LOW 2 +#define ICE_AQC_NVM_NETLIST_ID_BLK_MAJOR_VER_HIGH 3 +#define ICE_AQC_NVM_NETLIST_ID_BLK_MINOR_VER_LOW 4 +#define ICE_AQC_NVM_NETLIST_ID_BLK_MINOR_VER_HIGH 5 +#define ICE_AQC_NVM_NETLIST_ID_BLK_TYPE_LOW 6 +#define ICE_AQC_NVM_NETLIST_ID_BLK_TYPE_HIGH 7 +#define ICE_AQC_NVM_NETLIST_ID_BLK_REV_LOW 8 +#define ICE_AQC_NVM_NETLIST_ID_BLK_REV_HIGH 9 +#define ICE_AQC_NVM_NETLIST_ID_BLK_SHA_HASH 0xA +#define ICE_AQC_NVM_NETLIST_ID_BLK_CUST_VER 0x2F + /** * Send to PF command (indirect 0x0801) ID is only used by PF * @@ -1648,10 +1701,12 @@ struct ice_pkg_ver { }; #define ICE_PKG_NAME_SIZE 32 +#define ICE_SEG_NAME_SIZE 28 struct ice_aqc_get_pkg_info { struct ice_pkg_ver ver; - char name[ICE_PKG_NAME_SIZE]; + char name[ICE_SEG_NAME_SIZE]; + __le32 track_id; u8 is_in_nvm; u8 is_active; u8 is_active_at_boot; @@ -1738,6 +1793,7 @@ struct ice_aq_desc { struct ice_aqc_download_pkg download_pkg; struct ice_aqc_set_mac_lb set_mac_lb; struct ice_aqc_alloc_free_res_cmd sw_res_ctrl; + struct ice_aqc_set_mac_cfg set_mac_cfg; struct ice_aqc_set_event_mask set_event_mask; struct ice_aqc_get_link_status get_link_status; struct ice_aqc_event_lan_overflow lan_overflow; @@ -1770,6 +1826,7 @@ enum ice_aq_err { ICE_AQ_RC_EINVAL = 14, /* Invalid argument */ ICE_AQ_RC_ENOSPC = 16, /* No space left or allocation failure */ ICE_AQ_RC_ENOSYS = 17, /* Function not implemented */ + ICE_AQ_RC_EMODE = 21, /* Op not allowed in current dev mode */ ICE_AQ_RC_ENOSEC = 24, /* Missing security manifest */ ICE_AQ_RC_EBADSIG = 25, /* Bad RSA signature */ ICE_AQ_RC_ESVN = 26, /* SVN number prohibits this package */ @@ -1834,6 +1891,7 @@ enum ice_adminq_opc { /* PHY commands */ ice_aqc_opc_get_phy_caps = 0x0600, ice_aqc_opc_set_phy_cfg = 0x0601, + ice_aqc_opc_set_mac_cfg = 0x0603, ice_aqc_opc_restart_an = 0x0605, ice_aqc_opc_get_link_status = 0x0607, ice_aqc_opc_set_event_mask = 0x0613, diff --git a/drivers/net/ethernet/intel/ice/ice_arfs.c b/drivers/net/ethernet/intel/ice/ice_arfs.c new file mode 100644 index 000000000000..6560acd76c94 --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_arfs.c @@ -0,0 +1,663 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2018-2020, Intel Corporation. */ + +#include "ice.h" + +/** + * ice_is_arfs_active - helper to check is aRFS is active + * @vsi: VSI to check + */ +static bool ice_is_arfs_active(struct ice_vsi *vsi) +{ + return !!vsi->arfs_fltr_list; +} + +/** + * ice_is_arfs_using_perfect_flow - check if aRFS has active perfect filters + * @hw: pointer to the HW structure + * @flow_type: flow type as Flow Director understands it + * + * Flow Director will query this function to see if aRFS is currently using + * the specified flow_type for perfect (4-tuple) filters. + */ +bool +ice_is_arfs_using_perfect_flow(struct ice_hw *hw, enum ice_fltr_ptype flow_type) +{ + struct ice_arfs_active_fltr_cntrs *arfs_fltr_cntrs; + struct ice_pf *pf = hw->back; + struct ice_vsi *vsi; + + vsi = ice_get_main_vsi(pf); + if (!vsi) + return false; + + arfs_fltr_cntrs = vsi->arfs_fltr_cntrs; + + /* active counters can be updated by multiple CPUs */ + smp_mb__before_atomic(); + switch (flow_type) { + case ICE_FLTR_PTYPE_NONF_IPV4_UDP: + return atomic_read(&arfs_fltr_cntrs->active_udpv4_cnt) > 0; + case ICE_FLTR_PTYPE_NONF_IPV6_UDP: + return atomic_read(&arfs_fltr_cntrs->active_udpv6_cnt) > 0; + case ICE_FLTR_PTYPE_NONF_IPV4_TCP: + return atomic_read(&arfs_fltr_cntrs->active_tcpv4_cnt) > 0; + case ICE_FLTR_PTYPE_NONF_IPV6_TCP: + return atomic_read(&arfs_fltr_cntrs->active_tcpv6_cnt) > 0; + default: + return false; + } +} + +/** + * ice_arfs_update_active_fltr_cntrs - update active filter counters for aRFS + * @vsi: VSI that aRFS is active on + * @entry: aRFS entry used to change counters + * @add: true to increment counter, false to decrement + */ +static void +ice_arfs_update_active_fltr_cntrs(struct ice_vsi *vsi, + struct ice_arfs_entry *entry, bool add) +{ + struct ice_arfs_active_fltr_cntrs *fltr_cntrs = vsi->arfs_fltr_cntrs; + + switch (entry->fltr_info.flow_type) { + case ICE_FLTR_PTYPE_NONF_IPV4_TCP: + if (add) + atomic_inc(&fltr_cntrs->active_tcpv4_cnt); + else + atomic_dec(&fltr_cntrs->active_tcpv4_cnt); + break; + case ICE_FLTR_PTYPE_NONF_IPV6_TCP: + if (add) + atomic_inc(&fltr_cntrs->active_tcpv6_cnt); + else + atomic_dec(&fltr_cntrs->active_tcpv6_cnt); + break; + case ICE_FLTR_PTYPE_NONF_IPV4_UDP: + if (add) + atomic_inc(&fltr_cntrs->active_udpv4_cnt); + else + atomic_dec(&fltr_cntrs->active_udpv4_cnt); + break; + case ICE_FLTR_PTYPE_NONF_IPV6_UDP: + if (add) + atomic_inc(&fltr_cntrs->active_udpv6_cnt); + else + atomic_dec(&fltr_cntrs->active_udpv6_cnt); + break; + default: + dev_err(ice_pf_to_dev(vsi->back), "aRFS: Failed to update filter counters, invalid filter type %d\n", + entry->fltr_info.flow_type); + } +} + +/** + * ice_arfs_del_flow_rules - delete the rules passed in from HW + * @vsi: VSI for the flow rules that need to be deleted + * @del_list_head: head of the list of ice_arfs_entry(s) for rule deletion + * + * Loop through the delete list passed in and remove the rules from HW. After + * each rule is deleted, disconnect and free the ice_arfs_entry because it is no + * longer being referenced by the aRFS hash table. + */ +static void +ice_arfs_del_flow_rules(struct ice_vsi *vsi, struct hlist_head *del_list_head) +{ + struct ice_arfs_entry *e; + struct hlist_node *n; + struct device *dev; + + dev = ice_pf_to_dev(vsi->back); + + hlist_for_each_entry_safe(e, n, del_list_head, list_entry) { + int result; + + result = ice_fdir_write_fltr(vsi->back, &e->fltr_info, false, + false); + if (!result) + ice_arfs_update_active_fltr_cntrs(vsi, e, false); + else + dev_dbg(dev, "Unable to delete aRFS entry, err %d fltr_state %d fltr_id %d flow_id %d Q %d\n", + result, e->fltr_state, e->fltr_info.fltr_id, + e->flow_id, e->fltr_info.q_index); + + /* The aRFS hash table is no longer referencing this entry */ + hlist_del(&e->list_entry); + devm_kfree(dev, e); + } +} + +/** + * ice_arfs_add_flow_rules - add the rules passed in from HW + * @vsi: VSI for the flow rules that need to be added + * @add_list_head: head of the list of ice_arfs_entry_ptr(s) for rule addition + * + * Loop through the add list passed in and remove the rules from HW. After each + * rule is added, disconnect and free the ice_arfs_entry_ptr node. Don't free + * the ice_arfs_entry(s) because they are still being referenced in the aRFS + * hash table. + */ +static void +ice_arfs_add_flow_rules(struct ice_vsi *vsi, struct hlist_head *add_list_head) +{ + struct ice_arfs_entry_ptr *ep; + struct hlist_node *n; + struct device *dev; + + dev = ice_pf_to_dev(vsi->back); + + hlist_for_each_entry_safe(ep, n, add_list_head, list_entry) { + int result; + + result = ice_fdir_write_fltr(vsi->back, + &ep->arfs_entry->fltr_info, true, + false); + if (!result) + ice_arfs_update_active_fltr_cntrs(vsi, ep->arfs_entry, + true); + else + dev_dbg(dev, "Unable to add aRFS entry, err %d fltr_state %d fltr_id %d flow_id %d Q %d\n", + result, ep->arfs_entry->fltr_state, + ep->arfs_entry->fltr_info.fltr_id, + ep->arfs_entry->flow_id, + ep->arfs_entry->fltr_info.q_index); + + hlist_del(&ep->list_entry); + devm_kfree(dev, ep); + } +} + +/** + * ice_arfs_is_flow_expired - check if the aRFS entry has expired + * @vsi: VSI containing the aRFS entry + * @arfs_entry: aRFS entry that's being checked for expiration + * + * Return true if the flow has expired, else false. This function should be used + * to determine whether or not an aRFS entry should be removed from the hardware + * and software structures. + */ +static bool +ice_arfs_is_flow_expired(struct ice_vsi *vsi, struct ice_arfs_entry *arfs_entry) +{ +#define ICE_ARFS_TIME_DELTA_EXPIRATION msecs_to_jiffies(5000) + if (rps_may_expire_flow(vsi->netdev, arfs_entry->fltr_info.q_index, + arfs_entry->flow_id, + arfs_entry->fltr_info.fltr_id)) + return true; + + /* expiration timer only used for UDP filters */ + if (arfs_entry->fltr_info.flow_type != ICE_FLTR_PTYPE_NONF_IPV4_UDP && + arfs_entry->fltr_info.flow_type != ICE_FLTR_PTYPE_NONF_IPV6_UDP) + return false; + + return time_in_range64(arfs_entry->time_activated + + ICE_ARFS_TIME_DELTA_EXPIRATION, + arfs_entry->time_activated, get_jiffies_64()); +} + +/** + * ice_arfs_update_flow_rules - add/delete aRFS rules in HW + * @vsi: the VSI to be forwarded to + * @idx: index into the table of aRFS filter lists. Obtained from skb->hash + * @add_list: list to populate with filters to be added to Flow Director + * @del_list: list to populate with filters to be deleted from Flow Director + * + * Iterate over the hlist at the index given in the aRFS hash table and + * determine if there are any aRFS entries that need to be either added or + * deleted in the HW. If the aRFS entry is marked as ICE_ARFS_INACTIVE the + * filter needs to be added to HW, else if it's marked as ICE_ARFS_ACTIVE and + * the flow has expired delete the filter from HW. The caller of this function + * is expected to add/delete rules on the add_list/del_list respectively. + */ +static void +ice_arfs_update_flow_rules(struct ice_vsi *vsi, u16 idx, + struct hlist_head *add_list, + struct hlist_head *del_list) +{ + struct ice_arfs_entry *e; + struct hlist_node *n; + struct device *dev; + + dev = ice_pf_to_dev(vsi->back); + + /* go through the aRFS hlist at this idx and check for needed updates */ + hlist_for_each_entry_safe(e, n, &vsi->arfs_fltr_list[idx], list_entry) + /* check if filter needs to be added to HW */ + if (e->fltr_state == ICE_ARFS_INACTIVE) { + enum ice_fltr_ptype flow_type = e->fltr_info.flow_type; + struct ice_arfs_entry_ptr *ep = + devm_kzalloc(dev, sizeof(*ep), GFP_ATOMIC); + + if (!ep) + continue; + INIT_HLIST_NODE(&ep->list_entry); + /* reference aRFS entry to add HW filter */ + ep->arfs_entry = e; + hlist_add_head(&ep->list_entry, add_list); + e->fltr_state = ICE_ARFS_ACTIVE; + /* expiration timer only used for UDP flows */ + if (flow_type == ICE_FLTR_PTYPE_NONF_IPV4_UDP || + flow_type == ICE_FLTR_PTYPE_NONF_IPV6_UDP) + e->time_activated = get_jiffies_64(); + } else if (e->fltr_state == ICE_ARFS_ACTIVE) { + /* check if filter needs to be removed from HW */ + if (ice_arfs_is_flow_expired(vsi, e)) { + /* remove aRFS entry from hash table for delete + * and to prevent referencing it the next time + * through this hlist index + */ + hlist_del(&e->list_entry); + e->fltr_state = ICE_ARFS_TODEL; + /* save reference to aRFS entry for delete */ + hlist_add_head(&e->list_entry, del_list); + } + } +} + +/** + * ice_sync_arfs_fltrs - update all aRFS filters + * @pf: board private structure + */ +void ice_sync_arfs_fltrs(struct ice_pf *pf) +{ + HLIST_HEAD(tmp_del_list); + HLIST_HEAD(tmp_add_list); + struct ice_vsi *pf_vsi; + unsigned int i; + + pf_vsi = ice_get_main_vsi(pf); + if (!pf_vsi) + return; + + if (!ice_is_arfs_active(pf_vsi)) + return; + + spin_lock_bh(&pf_vsi->arfs_lock); + /* Once we process aRFS for the PF VSI get out */ + for (i = 0; i < ICE_MAX_ARFS_LIST; i++) + ice_arfs_update_flow_rules(pf_vsi, i, &tmp_add_list, + &tmp_del_list); + spin_unlock_bh(&pf_vsi->arfs_lock); + + /* use list of ice_arfs_entry(s) for delete */ + ice_arfs_del_flow_rules(pf_vsi, &tmp_del_list); + + /* use list of ice_arfs_entry_ptr(s) for add */ + ice_arfs_add_flow_rules(pf_vsi, &tmp_add_list); +} + +/** + * ice_arfs_build_entry - builds an aRFS entry based on input + * @vsi: destination VSI for this flow + * @fk: flow dissector keys for creating the tuple + * @rxq_idx: Rx queue to steer this flow to + * @flow_id: passed down from the stack and saved for flow expiration + * + * returns an aRFS entry on success and NULL on failure + */ +static struct ice_arfs_entry * +ice_arfs_build_entry(struct ice_vsi *vsi, const struct flow_keys *fk, + u16 rxq_idx, u32 flow_id) +{ + struct ice_arfs_entry *arfs_entry; + struct ice_fdir_fltr *fltr_info; + u8 ip_proto; + + arfs_entry = devm_kzalloc(ice_pf_to_dev(vsi->back), + sizeof(*arfs_entry), + GFP_ATOMIC | __GFP_NOWARN); + if (!arfs_entry) + return NULL; + + fltr_info = &arfs_entry->fltr_info; + fltr_info->q_index = rxq_idx; + fltr_info->dest_ctl = ICE_FLTR_PRGM_DESC_DEST_DIRECT_PKT_QINDEX; + fltr_info->dest_vsi = vsi->idx; + ip_proto = fk->basic.ip_proto; + + if (fk->basic.n_proto == htons(ETH_P_IP)) { + fltr_info->ip.v4.proto = ip_proto; + fltr_info->flow_type = (ip_proto == IPPROTO_TCP) ? + ICE_FLTR_PTYPE_NONF_IPV4_TCP : + ICE_FLTR_PTYPE_NONF_IPV4_UDP; + fltr_info->ip.v4.src_ip = fk->addrs.v4addrs.src; + fltr_info->ip.v4.dst_ip = fk->addrs.v4addrs.dst; + fltr_info->ip.v4.src_port = fk->ports.src; + fltr_info->ip.v4.dst_port = fk->ports.dst; + } else { /* ETH_P_IPV6 */ + fltr_info->ip.v6.proto = ip_proto; + fltr_info->flow_type = (ip_proto == IPPROTO_TCP) ? + ICE_FLTR_PTYPE_NONF_IPV6_TCP : + ICE_FLTR_PTYPE_NONF_IPV6_UDP; + memcpy(&fltr_info->ip.v6.src_ip, &fk->addrs.v6addrs.src, + sizeof(struct in6_addr)); + memcpy(&fltr_info->ip.v6.dst_ip, &fk->addrs.v6addrs.dst, + sizeof(struct in6_addr)); + fltr_info->ip.v6.src_port = fk->ports.src; + fltr_info->ip.v6.dst_port = fk->ports.dst; + } + + arfs_entry->flow_id = flow_id; + fltr_info->fltr_id = + atomic_inc_return(vsi->arfs_last_fltr_id) % RPS_NO_FILTER; + + return arfs_entry; +} + +/** + * ice_arfs_is_perfect_flow_set - Check to see if perfect flow is set + * @hw: pointer to HW structure + * @l3_proto: ETH_P_IP or ETH_P_IPV6 in network order + * @l4_proto: IPPROTO_UDP or IPPROTO_TCP + * + * We only support perfect (4-tuple) filters for aRFS. This function allows aRFS + * to check if perfect (4-tuple) flow rules are currently in place by Flow + * Director. + */ +static bool +ice_arfs_is_perfect_flow_set(struct ice_hw *hw, __be16 l3_proto, u8 l4_proto) +{ + unsigned long *perfect_fltr = hw->fdir_perfect_fltr; + + /* advanced Flow Director disabled, perfect filters always supported */ + if (!perfect_fltr) + return true; + + if (l3_proto == htons(ETH_P_IP) && l4_proto == IPPROTO_UDP) + return test_bit(ICE_FLTR_PTYPE_NONF_IPV4_UDP, perfect_fltr); + else if (l3_proto == htons(ETH_P_IP) && l4_proto == IPPROTO_TCP) + return test_bit(ICE_FLTR_PTYPE_NONF_IPV4_TCP, perfect_fltr); + else if (l3_proto == htons(ETH_P_IPV6) && l4_proto == IPPROTO_UDP) + return test_bit(ICE_FLTR_PTYPE_NONF_IPV6_UDP, perfect_fltr); + else if (l3_proto == htons(ETH_P_IPV6) && l4_proto == IPPROTO_TCP) + return test_bit(ICE_FLTR_PTYPE_NONF_IPV6_TCP, perfect_fltr); + + return false; +} + +/** + * ice_rx_flow_steer - steer the Rx flow to where application is being run + * @netdev: ptr to the netdev being adjusted + * @skb: buffer with required header information + * @rxq_idx: queue to which the flow needs to move + * @flow_id: flow identifier provided by the netdev + * + * Based on the skb, rxq_idx, and flow_id passed in add/update an entry in the + * aRFS hash table. Iterate over one of the hlists in the aRFS hash table and + * if the flow_id already exists in the hash table but the rxq_idx has changed + * mark the entry as ICE_ARFS_INACTIVE so it can get updated in HW, else + * if the entry is marked as ICE_ARFS_TODEL delete it from the aRFS hash table. + * If neither of the previous conditions are true then add a new entry in the + * aRFS hash table, which gets set to ICE_ARFS_INACTIVE by default so it can be + * added to HW. + */ +int +ice_rx_flow_steer(struct net_device *netdev, const struct sk_buff *skb, + u16 rxq_idx, u32 flow_id) +{ + struct ice_netdev_priv *np = netdev_priv(netdev); + struct ice_arfs_entry *arfs_entry; + struct ice_vsi *vsi = np->vsi; + struct flow_keys fk; + struct ice_pf *pf; + __be16 n_proto; + u8 ip_proto; + u16 idx; + int ret; + + /* failed to allocate memory for aRFS so don't crash */ + if (unlikely(!vsi->arfs_fltr_list)) + return -ENODEV; + + pf = vsi->back; + + if (skb->encapsulation) + return -EPROTONOSUPPORT; + + if (!skb_flow_dissect_flow_keys(skb, &fk, 0)) + return -EPROTONOSUPPORT; + + n_proto = fk.basic.n_proto; + /* Support only IPV4 and IPV6 */ + if ((n_proto == htons(ETH_P_IP) && !ip_is_fragment(ip_hdr(skb))) || + n_proto == htons(ETH_P_IPV6)) + ip_proto = fk.basic.ip_proto; + else + return -EPROTONOSUPPORT; + + /* Support only TCP and UDP */ + if (ip_proto != IPPROTO_TCP && ip_proto != IPPROTO_UDP) + return -EPROTONOSUPPORT; + + /* only support 4-tuple filters for aRFS */ + if (!ice_arfs_is_perfect_flow_set(&pf->hw, n_proto, ip_proto)) + return -EOPNOTSUPP; + + /* choose the aRFS list bucket based on skb hash */ + idx = skb_get_hash_raw(skb) & ICE_ARFS_LST_MASK; + /* search for entry in the bucket */ + spin_lock_bh(&vsi->arfs_lock); + hlist_for_each_entry(arfs_entry, &vsi->arfs_fltr_list[idx], + list_entry) { + struct ice_fdir_fltr *fltr_info; + + /* keep searching for the already existing arfs_entry flow */ + if (arfs_entry->flow_id != flow_id) + continue; + + fltr_info = &arfs_entry->fltr_info; + ret = fltr_info->fltr_id; + + if (fltr_info->q_index == rxq_idx || + arfs_entry->fltr_state != ICE_ARFS_ACTIVE) + goto out; + + /* update the queue to forward to on an already existing flow */ + fltr_info->q_index = rxq_idx; + arfs_entry->fltr_state = ICE_ARFS_INACTIVE; + ice_arfs_update_active_fltr_cntrs(vsi, arfs_entry, false); + goto out_schedule_service_task; + } + + arfs_entry = ice_arfs_build_entry(vsi, &fk, rxq_idx, flow_id); + if (!arfs_entry) { + ret = -ENOMEM; + goto out; + } + + ret = arfs_entry->fltr_info.fltr_id; + INIT_HLIST_NODE(&arfs_entry->list_entry); + hlist_add_head(&arfs_entry->list_entry, &vsi->arfs_fltr_list[idx]); +out_schedule_service_task: + ice_service_task_schedule(pf); +out: + spin_unlock_bh(&vsi->arfs_lock); + return ret; +} + +/** + * ice_init_arfs_cntrs - initialize aRFS counter values + * @vsi: VSI that aRFS counters need to be initialized on + */ +static int ice_init_arfs_cntrs(struct ice_vsi *vsi) +{ + if (!vsi || vsi->type != ICE_VSI_PF) + return -EINVAL; + + vsi->arfs_fltr_cntrs = kzalloc(sizeof(*vsi->arfs_fltr_cntrs), + GFP_KERNEL); + if (!vsi->arfs_fltr_cntrs) + return -ENOMEM; + + vsi->arfs_last_fltr_id = kzalloc(sizeof(*vsi->arfs_last_fltr_id), + GFP_KERNEL); + if (!vsi->arfs_last_fltr_id) { + kfree(vsi->arfs_fltr_cntrs); + vsi->arfs_fltr_cntrs = NULL; + return -ENOMEM; + } + + return 0; +} + +/** + * ice_init_arfs - initialize aRFS resources + * @vsi: the VSI to be forwarded to + */ +void ice_init_arfs(struct ice_vsi *vsi) +{ + struct hlist_head *arfs_fltr_list; + unsigned int i; + + if (!vsi || vsi->type != ICE_VSI_PF) + return; + + arfs_fltr_list = kzalloc(sizeof(*arfs_fltr_list) * ICE_MAX_ARFS_LIST, + GFP_KERNEL); + if (!arfs_fltr_list) + return; + + if (ice_init_arfs_cntrs(vsi)) + goto free_arfs_fltr_list; + + for (i = 0; i < ICE_MAX_ARFS_LIST; i++) + INIT_HLIST_HEAD(&arfs_fltr_list[i]); + + spin_lock_init(&vsi->arfs_lock); + + vsi->arfs_fltr_list = arfs_fltr_list; + + return; + +free_arfs_fltr_list: + kfree(arfs_fltr_list); +} + +/** + * ice_clear_arfs - clear the aRFS hash table and any memory used for aRFS + * @vsi: the VSI to be forwarded to + */ +void ice_clear_arfs(struct ice_vsi *vsi) +{ + struct device *dev; + unsigned int i; + + if (!vsi || vsi->type != ICE_VSI_PF || !vsi->back || + !vsi->arfs_fltr_list) + return; + + dev = ice_pf_to_dev(vsi->back); + for (i = 0; i < ICE_MAX_ARFS_LIST; i++) { + struct ice_arfs_entry *r; + struct hlist_node *n; + + spin_lock_bh(&vsi->arfs_lock); + hlist_for_each_entry_safe(r, n, &vsi->arfs_fltr_list[i], + list_entry) { + hlist_del(&r->list_entry); + devm_kfree(dev, r); + } + spin_unlock_bh(&vsi->arfs_lock); + } + + kfree(vsi->arfs_fltr_list); + vsi->arfs_fltr_list = NULL; + kfree(vsi->arfs_last_fltr_id); + vsi->arfs_last_fltr_id = NULL; + kfree(vsi->arfs_fltr_cntrs); + vsi->arfs_fltr_cntrs = NULL; +} + +/** + * ice_free_cpu_rx_rmap - free setup CPU reverse map + * @vsi: the VSI to be forwarded to + */ +void ice_free_cpu_rx_rmap(struct ice_vsi *vsi) +{ + struct net_device *netdev; + + if (!vsi || vsi->type != ICE_VSI_PF || !vsi->arfs_fltr_list) + return; + + netdev = vsi->netdev; + if (!netdev || !netdev->rx_cpu_rmap || + netdev->reg_state != NETREG_REGISTERED) + return; + + free_irq_cpu_rmap(netdev->rx_cpu_rmap); + netdev->rx_cpu_rmap = NULL; +} + +/** + * ice_set_cpu_rx_rmap - setup CPU reverse map for each queue + * @vsi: the VSI to be forwarded to + */ +int ice_set_cpu_rx_rmap(struct ice_vsi *vsi) +{ + struct net_device *netdev; + struct ice_pf *pf; + int base_idx, i; + + if (!vsi || vsi->type != ICE_VSI_PF) + return -EINVAL; + + pf = vsi->back; + netdev = vsi->netdev; + if (!pf || !netdev || !vsi->num_q_vectors || + vsi->netdev->reg_state != NETREG_REGISTERED) + return -EINVAL; + + netdev_dbg(netdev, "Setup CPU RMAP: vsi type 0x%x, ifname %s, q_vectors %d\n", + vsi->type, netdev->name, vsi->num_q_vectors); + + netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(vsi->num_q_vectors); + if (unlikely(!netdev->rx_cpu_rmap)) + return -EINVAL; + + base_idx = vsi->base_vector; + for (i = 0; i < vsi->num_q_vectors; i++) + if (irq_cpu_rmap_add(netdev->rx_cpu_rmap, + pf->msix_entries[base_idx + i].vector)) { + ice_free_cpu_rx_rmap(vsi); + return -EINVAL; + } + + return 0; +} + +/** + * ice_remove_arfs - remove/clear all aRFS resources + * @pf: device private structure + */ +void ice_remove_arfs(struct ice_pf *pf) +{ + struct ice_vsi *pf_vsi; + + pf_vsi = ice_get_main_vsi(pf); + if (!pf_vsi) + return; + + ice_free_cpu_rx_rmap(pf_vsi); + ice_clear_arfs(pf_vsi); +} + +/** + * ice_rebuild_arfs - remove/clear all aRFS resources and rebuild after reset + * @pf: device private structure + */ +void ice_rebuild_arfs(struct ice_pf *pf) +{ + struct ice_vsi *pf_vsi; + + pf_vsi = ice_get_main_vsi(pf); + if (!pf_vsi) + return; + + ice_remove_arfs(pf); + if (ice_set_cpu_rx_rmap(pf_vsi)) { + dev_err(ice_pf_to_dev(pf), "Failed to rebuild aRFS\n"); + return; + } + ice_init_arfs(pf_vsi); +} diff --git a/drivers/net/ethernet/intel/ice/ice_arfs.h b/drivers/net/ethernet/intel/ice/ice_arfs.h new file mode 100644 index 000000000000..f39cd16403ed --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_arfs.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2018-2020, Intel Corporation. */ + +#ifndef _ICE_ARFS_H_ +#define _ICE_ARFS_H_ +enum ice_arfs_fltr_state { + ICE_ARFS_INACTIVE, + ICE_ARFS_ACTIVE, + ICE_ARFS_TODEL, +}; + +struct ice_arfs_entry { + struct ice_fdir_fltr fltr_info; + struct hlist_node list_entry; + u64 time_activated; /* only valid for UDP flows */ + u32 flow_id; + /* fltr_state = 0 - ICE_ARFS_INACTIVE: + * filter needs to be updated or programmed in HW. + * fltr_state = 1 - ICE_ARFS_ACTIVE: + * filter is active and programmed in HW. + * fltr_state = 2 - ICE_ARFS_TODEL: + * filter has been deleted from HW and needs to be removed from + * the aRFS hash table. + */ + u8 fltr_state; +}; + +struct ice_arfs_entry_ptr { + struct ice_arfs_entry *arfs_entry; + struct hlist_node list_entry; +}; + +struct ice_arfs_active_fltr_cntrs { + atomic_t active_tcpv4_cnt; + atomic_t active_tcpv6_cnt; + atomic_t active_udpv4_cnt; + atomic_t active_udpv6_cnt; +}; + +#ifdef CONFIG_RFS_ACCEL +int +ice_rx_flow_steer(struct net_device *netdev, const struct sk_buff *skb, + u16 rxq_idx, u32 flow_id); +void ice_clear_arfs(struct ice_vsi *vsi); +void ice_free_cpu_rx_rmap(struct ice_vsi *vsi); +void ice_init_arfs(struct ice_vsi *vsi); +void ice_sync_arfs_fltrs(struct ice_pf *pf); +int ice_set_cpu_rx_rmap(struct ice_vsi *vsi); +void ice_remove_arfs(struct ice_pf *pf); +void ice_rebuild_arfs(struct ice_pf *pf); +bool +ice_is_arfs_using_perfect_flow(struct ice_hw *hw, + enum ice_fltr_ptype flow_type); +#else +#define ice_sync_arfs_fltrs(pf) do {} while (0) +#define ice_init_arfs(vsi) do {} while (0) +#define ice_clear_arfs(vsi) do {} while (0) +#define ice_remove_arfs(pf) do {} while (0) +#define ice_free_cpu_rx_rmap(vsi) do {} while (0) +#define ice_rebuild_arfs(pf) do {} while (0) + +static inline int ice_set_cpu_rx_rmap(struct ice_vsi __always_unused *vsi) +{ + return 0; +} + +static inline int +ice_rx_flow_steer(struct net_device __always_unused *netdev, + const struct sk_buff __always_unused *skb, + u16 __always_unused rxq_idx, u32 __always_unused flow_id) +{ + return -EOPNOTSUPP; +} + +static inline bool +ice_is_arfs_using_perfect_flow(struct ice_hw __always_unused *hw, + enum ice_fltr_ptype __always_unused flow_type) +{ + return false; +} +#endif /* CONFIG_RFS_ACCEL */ +#endif /* _ICE_ARFS_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_base.c b/drivers/net/ethernet/intel/ice/ice_base.c index a19cd6f5436b..d620d26d42ed 100644 --- a/drivers/net/ethernet/intel/ice/ice_base.c +++ b/drivers/net/ethernet/intel/ice/ice_base.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2019, Intel Corporation. */ +#include <net/xdp_sock_drv.h> #include "ice_base.h" +#include "ice_lib.h" #include "ice_dcb_lib.h" /** @@ -12,7 +14,7 @@ */ static int __ice_vsi_get_qs_contig(struct ice_qs_cfg *qs_cfg) { - int offset, i; + unsigned int offset, i; mutex_lock(qs_cfg->qs_mutex); offset = bitmap_find_next_zero_area(qs_cfg->pf_map, qs_cfg->pf_map_size, @@ -24,7 +26,7 @@ static int __ice_vsi_get_qs_contig(struct ice_qs_cfg *qs_cfg) bitmap_set(qs_cfg->pf_map, offset, qs_cfg->q_count); for (i = 0; i < qs_cfg->q_count; i++) - qs_cfg->vsi_map[i + qs_cfg->vsi_map_offset] = i + offset; + qs_cfg->vsi_map[i + qs_cfg->vsi_map_offset] = (u16)(i + offset); mutex_unlock(qs_cfg->qs_mutex); return 0; @@ -38,7 +40,7 @@ static int __ice_vsi_get_qs_contig(struct ice_qs_cfg *qs_cfg) */ static int __ice_vsi_get_qs_sc(struct ice_qs_cfg *qs_cfg) { - int i, index = 0; + unsigned int i, index = 0; mutex_lock(qs_cfg->qs_mutex); for (i = 0; i < qs_cfg->q_count; i++) { @@ -47,7 +49,7 @@ static int __ice_vsi_get_qs_sc(struct ice_qs_cfg *qs_cfg) if (index >= qs_cfg->pf_map_size) goto err_scatter; set_bit(index, qs_cfg->pf_map); - qs_cfg->vsi_map[i + qs_cfg->vsi_map_offset] = index; + qs_cfg->vsi_map[i + qs_cfg->vsi_map_offset] = (u16)index; } mutex_unlock(qs_cfg->qs_mutex); @@ -96,7 +98,7 @@ static int ice_pf_rxq_wait(struct ice_pf *pf, int pf_q, bool ena) * We allocate one q_vector and set default value for ITR setting associated * with this q_vector. If allocation fails we return -ENOMEM. */ -static int ice_vsi_alloc_q_vector(struct ice_vsi *vsi, int v_idx) +static int ice_vsi_alloc_q_vector(struct ice_vsi *vsi, u16 v_idx) { struct ice_pf *pf = vsi->back; struct ice_q_vector *q_vector; @@ -246,6 +248,7 @@ ice_setup_tx_ctx(struct ice_ring *ring, struct ice_tlan_ctx *tlan_ctx, u16 pf_q) */ switch (vsi->type) { case ICE_VSI_LB: + case ICE_VSI_CTRL: case ICE_VSI_PF: tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_PF; break; @@ -279,12 +282,13 @@ ice_setup_tx_ctx(struct ice_ring *ring, struct ice_tlan_ctx *tlan_ctx, u16 pf_q) */ int ice_setup_rx_ctx(struct ice_ring *ring) { + struct device *dev = ice_pf_to_dev(ring->vsi->back); int chain_len = ICE_MAX_CHAINED_RX_BUFS; + u16 num_bufs = ICE_DESC_UNUSED(ring); struct ice_vsi *vsi = ring->vsi; u32 rxdid = ICE_RXDID_FLEX_NIC; struct ice_rlan_ctx rlan_ctx; struct ice_hw *hw; - u32 regval; u16 pf_q; int err; @@ -308,24 +312,23 @@ int ice_setup_rx_ctx(struct ice_ring *ring) if (ring->xsk_umem) { xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq); - ring->rx_buf_len = ring->xsk_umem->chunk_size_nohr - - XDP_PACKET_HEADROOM; + ring->rx_buf_len = + xsk_umem_get_rx_frame_size(ring->xsk_umem); /* For AF_XDP ZC, we disallow packets to span on * multiple buffers, thus letting us skip that * handling in the fast-path. */ chain_len = 1; - ring->zca.free = ice_zca_free; err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, - MEM_TYPE_ZERO_COPY, - &ring->zca); + MEM_TYPE_XSK_BUFF_POOL, + NULL); if (err) return err; + xsk_buff_set_rxq_info(ring->xsk_umem, &ring->xdp_rxq); - dev_info(ice_pf_to_dev(vsi->back), "Registered XDP mem model MEM_TYPE_ZERO_COPY on Rx ring %d\n", + dev_info(dev, "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n", ring->q_index); } else { - ring->zca.free = NULL; if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) /* coverity[check_return] */ xdp_rxq_info_reg(&ring->xdp_rxq, @@ -376,38 +379,27 @@ int ice_setup_rx_ctx(struct ice_ring *ring) /* Max packet size for this queue - must not be set to a larger value * than 5 x DBUF */ - rlan_ctx.rxmax = min_t(u16, vsi->max_frame, + rlan_ctx.rxmax = min_t(u32, vsi->max_frame, chain_len * ring->rx_buf_len); /* Rx queue threshold in units of 64 */ rlan_ctx.lrxqthresh = 1; - /* Enable Flexible Descriptors in the queue context which - * allows this driver to select a specific receive descriptor format - */ - regval = rd32(hw, QRXFLXP_CNTXT(pf_q)); - if (vsi->type != ICE_VSI_VF) { - regval |= (rxdid << QRXFLXP_CNTXT_RXDID_IDX_S) & - QRXFLXP_CNTXT_RXDID_IDX_M; - - /* increasing context priority to pick up profile ID; - * default is 0x01; setting to 0x03 to ensure profile - * is programming if prev context is of same priority - */ - regval |= (0x03 << QRXFLXP_CNTXT_RXDID_PRIO_S) & - QRXFLXP_CNTXT_RXDID_PRIO_M; - - } else { - regval &= ~(QRXFLXP_CNTXT_RXDID_IDX_M | - QRXFLXP_CNTXT_RXDID_PRIO_M | - QRXFLXP_CNTXT_TS_M); - } - wr32(hw, QRXFLXP_CNTXT(pf_q), regval); + /* Enable Flexible Descriptors in the queue context which + * allows this driver to select a specific receive descriptor format + * increasing context priority to pick up profile ID; default is 0x01; + * setting to 0x03 to ensure profile is programming if prev context is + * of same priority + */ + if (vsi->type != ICE_VSI_VF) + ice_write_qrxflxp_cntxt(hw, pf_q, rxdid, 0x3); + else + ice_write_qrxflxp_cntxt(hw, pf_q, ICE_RXDID_LEGACY_1, 0x3); /* Absolute queue number out of 2K needs to be passed */ err = ice_write_rxq_ctx(hw, &rlan_ctx, pf_q); if (err) { - dev_err(ice_pf_to_dev(vsi->back), "Failed to set LAN Rx queue context for absolute Rx queue %d error: %d\n", + dev_err(dev, "Failed to set LAN Rx queue context for absolute Rx queue %d error: %d\n", pf_q, err); return -EIO; } @@ -425,13 +417,23 @@ int ice_setup_rx_ctx(struct ice_ring *ring) ring->tail = hw->hw_addr + QRX_TAIL(pf_q); writel(0, ring->tail); - err = ring->xsk_umem ? - ice_alloc_rx_bufs_slow_zc(ring, ICE_DESC_UNUSED(ring)) : - ice_alloc_rx_bufs(ring, ICE_DESC_UNUSED(ring)); - if (err) - dev_info(ice_pf_to_dev(vsi->back), "Failed allocate some buffers on %sRx ring %d (pf_q %d)\n", - ring->xsk_umem ? "UMEM enabled " : "", - ring->q_index, pf_q); + if (ring->xsk_umem) { + if (!xsk_buff_can_alloc(ring->xsk_umem, num_bufs)) { + dev_warn(dev, "UMEM does not provide enough addresses to fill %d buffers on Rx ring %d\n", + num_bufs, ring->q_index); + dev_warn(dev, "Change Rx ring/fill queue size to avoid performance issues\n"); + + return 0; + } + + err = ice_alloc_rx_bufs_zc(ring, num_bufs); + if (err) + dev_info(dev, "Failed to allocate some buffers on UMEM enabled Rx ring %d (pf_q %d)\n", + ring->q_index, pf_q); + return 0; + } + + ice_alloc_rx_bufs(ring, num_bufs); return 0; } @@ -453,7 +455,7 @@ int __ice_vsi_get_qs(struct ice_qs_cfg *qs_cfg) if (ret) { /* contig failed, so try with scatter approach */ qs_cfg->mapping_mode = ICE_VSI_MAP_SCATTER; - qs_cfg->q_count = min_t(u16, qs_cfg->q_count, + qs_cfg->q_count = min_t(unsigned int, qs_cfg->q_count, qs_cfg->scatter_count); ret = __ice_vsi_get_qs_sc(qs_cfg); } @@ -526,7 +528,8 @@ int ice_vsi_wait_one_rx_ring(struct ice_vsi *vsi, bool ena, u16 rxq_idx) int ice_vsi_alloc_q_vectors(struct ice_vsi *vsi) { struct device *dev = ice_pf_to_dev(vsi->back); - int v_idx, err; + u16 v_idx; + int err; if (vsi->q_vectors[0]) { dev_dbg(dev, "VSI %d has existing q_vectors\n", vsi->vsi_num); @@ -562,7 +565,7 @@ err_out: void ice_vsi_map_rings_to_vectors(struct ice_vsi *vsi) { int q_vectors = vsi->num_q_vectors; - int tx_rings_rem, rx_rings_rem; + u16 tx_rings_rem, rx_rings_rem; int v_id; /* initially assigning remaining rings count to VSIs num queue value */ @@ -571,10 +574,12 @@ void ice_vsi_map_rings_to_vectors(struct ice_vsi *vsi) for (v_id = 0; v_id < q_vectors; v_id++) { struct ice_q_vector *q_vector = vsi->q_vectors[v_id]; - int tx_rings_per_v, rx_rings_per_v, q_id, q_base; + u8 tx_rings_per_v, rx_rings_per_v; + u16 q_id, q_base; /* Tx rings mapping to vector */ - tx_rings_per_v = DIV_ROUND_UP(tx_rings_rem, q_vectors - v_id); + tx_rings_per_v = (u8)DIV_ROUND_UP(tx_rings_rem, + q_vectors - v_id); q_vector->num_ring_tx = tx_rings_per_v; q_vector->tx.ring = NULL; q_vector->tx.itr_idx = ICE_TX_ITR; @@ -590,7 +595,8 @@ void ice_vsi_map_rings_to_vectors(struct ice_vsi *vsi) tx_rings_rem -= tx_rings_per_v; /* Rx rings mapping to vector */ - rx_rings_per_v = DIV_ROUND_UP(rx_rings_rem, q_vectors - v_id); + rx_rings_per_v = (u8)DIV_ROUND_UP(rx_rings_rem, + q_vectors - v_id); q_vector->num_ring_rx = rx_rings_per_v; q_vector->rx.ring = NULL; q_vector->rx.itr_idx = ICE_RX_ITR; @@ -633,6 +639,7 @@ ice_vsi_cfg_txq(struct ice_vsi *vsi, struct ice_ring *ring, struct ice_aqc_add_txqs_perq *txq; struct ice_pf *pf = vsi->back; u8 buf_len = sizeof(*qg_buf); + struct ice_hw *hw = &pf->hw; enum ice_status status; u16 pf_q; u8 tc; @@ -641,13 +648,13 @@ ice_vsi_cfg_txq(struct ice_vsi *vsi, struct ice_ring *ring, ice_setup_tx_ctx(ring, &tlan_ctx, pf_q); /* copy context contents into the qg_buf */ qg_buf->txqs[0].txq_id = cpu_to_le16(pf_q); - ice_set_ctx((u8 *)&tlan_ctx, qg_buf->txqs[0].txq_ctx, + ice_set_ctx(hw, (u8 *)&tlan_ctx, qg_buf->txqs[0].txq_ctx, ice_tlan_ctx_info); /* init queue specific tail reg. It is referred as * transmit comm scheduler queue doorbell. */ - ring->tail = pf->hw.hw_addr + QTX_COMM_DBELL(pf_q); + ring->tail = hw->hw_addr + QTX_COMM_DBELL(pf_q); if (IS_ENABLED(CONFIG_DCB)) tc = ring->dcb_tc; @@ -662,8 +669,8 @@ ice_vsi_cfg_txq(struct ice_vsi *vsi, struct ice_ring *ring, status = ice_ena_vsi_txq(vsi->port_info, vsi->idx, tc, ring->q_handle, 1, qg_buf, buf_len, NULL); if (status) { - dev_err(ice_pf_to_dev(pf), "Failed to set LAN Tx queue context, error: %d\n", - status); + dev_err(ice_pf_to_dev(pf), "Failed to set LAN Tx queue context, error: %s\n", + ice_stat_str(status)); return -ENODEV; } @@ -832,8 +839,8 @@ ice_vsi_stop_tx_ring(struct ice_vsi *vsi, enum ice_disq_rst_src rst_src, } else if (status == ICE_ERR_DOES_NOT_EXIST) { dev_dbg(ice_pf_to_dev(vsi->back), "LAN Tx queues do not exist, nothing to disable\n"); } else if (status) { - dev_err(ice_pf_to_dev(vsi->back), "Failed to disable LAN Tx queues, error: %d\n", - status); + dev_err(ice_pf_to_dev(vsi->back), "Failed to disable LAN Tx queues, error: %s\n", + ice_stat_str(status)); return -ENODEV; } diff --git a/drivers/net/ethernet/intel/ice/ice_common.c b/drivers/net/ethernet/intel/ice/ice_common.c index 2c0d8fd3d5cd..bce0e1281168 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.c +++ b/drivers/net/ethernet/intel/ice/ice_common.c @@ -316,12 +316,78 @@ ice_aq_get_link_info(struct ice_port_info *pi, bool ena_lse, } /** + * ice_fill_tx_timer_and_fc_thresh + * @hw: pointer to the HW struct + * @cmd: pointer to MAC cfg structure + * + * Add Tx timer and FC refresh threshold info to Set MAC Config AQ command + * descriptor + */ +static void +ice_fill_tx_timer_and_fc_thresh(struct ice_hw *hw, + struct ice_aqc_set_mac_cfg *cmd) +{ + u16 fc_thres_val, tx_timer_val; + u32 val; + + /* We read back the transmit timer and FC threshold value of + * LFC. Thus, we will use index = + * PRTMAC_HSEC_CTL_TX_PAUSE_QUANTA_MAX_INDEX. + * + * Also, because we are operating on transmit timer and FC + * threshold of LFC, we don't turn on any bit in tx_tmr_priority + */ +#define IDX_OF_LFC PRTMAC_HSEC_CTL_TX_PAUSE_QUANTA_MAX_INDEX + + /* Retrieve the transmit timer */ + val = rd32(hw, PRTMAC_HSEC_CTL_TX_PAUSE_QUANTA(IDX_OF_LFC)); + tx_timer_val = val & + PRTMAC_HSEC_CTL_TX_PAUSE_QUANTA_HSEC_CTL_TX_PAUSE_QUANTA_M; + cmd->tx_tmr_value = cpu_to_le16(tx_timer_val); + + /* Retrieve the FC threshold */ + val = rd32(hw, PRTMAC_HSEC_CTL_TX_PAUSE_REFRESH_TIMER(IDX_OF_LFC)); + fc_thres_val = val & PRTMAC_HSEC_CTL_TX_PAUSE_REFRESH_TIMER_M; + + cmd->fc_refresh_threshold = cpu_to_le16(fc_thres_val); +} + +/** + * ice_aq_set_mac_cfg + * @hw: pointer to the HW struct + * @max_frame_size: Maximum Frame Size to be supported + * @cd: pointer to command details structure or NULL + * + * Set MAC configuration (0x0603) + */ +enum ice_status +ice_aq_set_mac_cfg(struct ice_hw *hw, u16 max_frame_size, struct ice_sq_cd *cd) +{ + struct ice_aqc_set_mac_cfg *cmd; + struct ice_aq_desc desc; + + cmd = &desc.params.set_mac_cfg; + + if (max_frame_size == 0) + return ICE_ERR_PARAM; + + ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_set_mac_cfg); + + cmd->max_frame_size = cpu_to_le16(max_frame_size); + + ice_fill_tx_timer_and_fc_thresh(hw, cmd); + + return ice_aq_send_cmd(hw, &desc, NULL, 0, cd); +} + +/** * ice_init_fltr_mgmt_struct - initializes filter management list and locks * @hw: pointer to the HW struct */ static enum ice_status ice_init_fltr_mgmt_struct(struct ice_hw *hw) { struct ice_switch_info *sw; + enum ice_status status; hw->switch_info = devm_kzalloc(ice_hw_to_dev(hw), sizeof(*hw->switch_info), GFP_KERNEL); @@ -332,7 +398,12 @@ static enum ice_status ice_init_fltr_mgmt_struct(struct ice_hw *hw) INIT_LIST_HEAD(&sw->vsi_list_map_head); - return ice_init_def_sw_recp(hw); + status = ice_init_def_sw_recp(hw); + if (status) { + devm_kfree(ice_hw_to_dev(hw), hw->switch_info); + return status; + } + return 0; } /** @@ -653,6 +724,10 @@ enum ice_status ice_init_hw(struct ice_hw *hw) if (status) goto err_unroll_cqinit; + /* Set bit to enable Flow Director filters */ + wr32(hw, PFQF_FD_ENA, PFQF_FD_ENA_FD_ENA_M); + INIT_LIST_HEAD(&hw->fdir_list_head); + ice_clear_pxe_mode(hw); status = ice_init_nvm(hw); @@ -743,9 +818,18 @@ enum ice_status ice_init_hw(struct ice_hw *hw) if (status) goto err_unroll_fltr_mgmt_struct; + /* enable jumbo frame support at MAC level */ + status = ice_aq_set_mac_cfg(hw, ICE_AQ_SET_MAC_FRAME_SIZE_MAX, NULL); + if (status) + goto err_unroll_fltr_mgmt_struct; + /* Obtain counter base index which would be used by flow director */ + status = ice_alloc_fd_res_cntr(hw, &hw->fd_ctr_base); + if (status) + goto err_unroll_fltr_mgmt_struct; status = ice_init_hw_tbls(hw); if (status) goto err_unroll_fltr_mgmt_struct; + mutex_init(&hw->tnl_lock); return 0; err_unroll_fltr_mgmt_struct: @@ -769,12 +853,14 @@ err_unroll_cqinit: */ void ice_deinit_hw(struct ice_hw *hw) { + ice_free_fd_res_cntr(hw, hw->fd_ctr_base); ice_cleanup_fltr_mgmt_struct(hw); ice_sched_cleanup_all(hw); ice_sched_clear_agg(hw); ice_free_seg(hw); ice_free_hw_tbls(hw); + mutex_destroy(&hw->tnl_lock); if (hw->port_info) { devm_kfree(ice_hw_to_dev(hw), hw->port_info); @@ -878,7 +964,12 @@ static enum ice_status ice_pf_reset(struct ice_hw *hw) wr32(hw, PFGEN_CTRL, (reg | PFGEN_CTRL_PFSWR_M)); - for (cnt = 0; cnt < ICE_PF_RESET_WAIT_COUNT; cnt++) { + /* Wait for the PFR to complete. The wait time is the global config lock + * timeout plus the PFR timeout which will account for a possible reset + * that is occurring during a download package operation. + */ + for (cnt = 0; cnt < ICE_GLOBAL_CFG_LOCK_TIMEOUT + + ICE_PF_RESET_WAIT_COUNT; cnt++) { reg = rd32(hw, PFGEN_CTRL); if (!(reg & PFGEN_CTRL_PFSWR_M)) break; @@ -1012,7 +1103,7 @@ ice_write_rxq_ctx(struct ice_hw *hw, struct ice_rlan_ctx *rlan_ctx, rlan_ctx->prefena = 1; - ice_set_ctx((u8 *)rlan_ctx, ctx_buf, ice_rlan_ctx_info); + ice_set_ctx(hw, (u8 *)rlan_ctx, ctx_buf, ice_rlan_ctx_info); return ice_copy_rxq_ctx_to_hw(hw, ctx_buf, rxq_index); } @@ -1678,6 +1769,33 @@ ice_parse_caps(struct ice_hw *hw, void *buf, u32 cap_count, "%s: msix_vector_first_id = %d\n", prefix, caps->msix_vector_first_id); break; + case ICE_AQC_CAPS_FD: + if (dev_p) { + dev_p->num_flow_director_fltr = number; + ice_debug(hw, ICE_DBG_INIT, + "%s: num_flow_director_fltr = %d\n", + prefix, + dev_p->num_flow_director_fltr); + } + if (func_p) { + u32 reg_val, val; + + reg_val = rd32(hw, GLQF_FD_SIZE); + val = (reg_val & GLQF_FD_SIZE_FD_GSIZE_M) >> + GLQF_FD_SIZE_FD_GSIZE_S; + func_p->fd_fltr_guar = + ice_get_num_per_func(hw, val); + val = (reg_val & GLQF_FD_SIZE_FD_BSIZE_M) >> + GLQF_FD_SIZE_FD_BSIZE_S; + func_p->fd_fltr_best_effort = val; + ice_debug(hw, ICE_DBG_INIT, + "%s: fd_fltr_guar = %d\n", + prefix, func_p->fd_fltr_guar); + ice_debug(hw, ICE_DBG_INIT, + "%s: fd_fltr_best_effort = %d\n", + prefix, func_p->fd_fltr_best_effort); + } + break; case ICE_AQC_CAPS_MAX_MTU: caps->max_mtu = number; ice_debug(hw, ICE_DBG_INIT, "%s: max_mtu = %d\n", @@ -1887,10 +2005,7 @@ ice_aq_manage_mac_write(struct ice_hw *hw, const u8 *mac_addr, u8 flags, ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_manage_mac_write); cmd->flags = flags; - - /* Prep values for flags, sah, sal */ - cmd->sah = htons(*((const u16 *)mac_addr)); - cmd->sal = htonl(*((const u32 *)(mac_addr + 2))); + ether_addr_copy(cmd->mac_addr, mac_addr); return ice_aq_send_cmd(hw, &desc, NULL, 0, cd); } @@ -2117,6 +2232,7 @@ ice_aq_set_phy_cfg(struct ice_hw *hw, u8 lport, struct ice_aqc_set_phy_cfg_data *cfg, struct ice_sq_cd *cd) { struct ice_aq_desc desc; + enum ice_status status; if (!cfg) return ICE_ERR_PARAM; @@ -2145,7 +2261,11 @@ ice_aq_set_phy_cfg(struct ice_hw *hw, u8 lport, ice_debug(hw, ICE_DBG_LINK, "eeer_value = 0x%x\n", cfg->eeer_value); ice_debug(hw, ICE_DBG_LINK, "link_fec_opt = 0x%x\n", cfg->link_fec_opt); - return ice_aq_send_cmd(hw, &desc, cfg, sizeof(*cfg), cd); + status = ice_aq_send_cmd(hw, &desc, cfg, sizeof(*cfg), cd); + if (hw->adminq.sq_last_status == ICE_AQ_RC_EMODE) + status = 0; + + return status; } /** @@ -3089,12 +3209,14 @@ ice_write_qword(u8 *src_ctx, u8 *dest_ctx, const struct ice_ctx_ele *ce_info) /** * ice_set_ctx - set context bits in packed structure + * @hw: pointer to the hardware structure * @src_ctx: pointer to a generic non-packed context structure * @dest_ctx: pointer to memory for the packed structure * @ce_info: a description of the structure to be transformed */ enum ice_status -ice_set_ctx(u8 *src_ctx, u8 *dest_ctx, const struct ice_ctx_ele *ce_info) +ice_set_ctx(struct ice_hw *hw, u8 *src_ctx, u8 *dest_ctx, + const struct ice_ctx_ele *ce_info) { int f; @@ -3103,6 +3225,12 @@ ice_set_ctx(u8 *src_ctx, u8 *dest_ctx, const struct ice_ctx_ele *ce_info) * using the correct size so that we are correct regardless * of the endianness of the machine. */ + if (ce_info[f].width > (ce_info[f].size_of * BITS_PER_BYTE)) { + ice_debug(hw, ICE_DBG_QCTX, + "Field %d width of %d bits larger than size of %d byte(s) ... skipping write\n", + f, ce_info[f].width, ce_info[f].size_of); + continue; + } switch (ce_info[f].size_of) { case sizeof(u8): ice_write_byte(src_ctx, dest_ctx, &ce_info[f]); diff --git a/drivers/net/ethernet/intel/ice/ice_common.h b/drivers/net/ethernet/intel/ice/ice_common.h index 8104f3d64d96..9b9e50d2398b 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.h +++ b/drivers/net/ethernet/intel/ice/ice_common.h @@ -70,7 +70,8 @@ enum ice_status ice_aq_q_shutdown(struct ice_hw *hw, bool unloading); void ice_fill_dflt_direct_cmd_desc(struct ice_aq_desc *desc, u16 opcode); extern const struct ice_ctx_ele ice_tlan_ctx_info[]; enum ice_status -ice_set_ctx(u8 *src_ctx, u8 *dest_ctx, const struct ice_ctx_ele *ce_info); +ice_set_ctx(struct ice_hw *hw, u8 *src_ctx, u8 *dest_ctx, + const struct ice_ctx_ele *ce_info); extern struct mutex ice_global_cfg_lock_sw; @@ -108,6 +109,8 @@ enum ice_status ice_aq_set_link_restart_an(struct ice_port_info *pi, bool ena_link, struct ice_sq_cd *cd); enum ice_status +ice_aq_set_mac_cfg(struct ice_hw *hw, u16 max_frame_size, struct ice_sq_cd *cd); +enum ice_status ice_aq_get_link_info(struct ice_port_info *pi, bool ena_lse, struct ice_link_status *link, struct ice_sq_cd *cd); enum ice_status diff --git a/drivers/net/ethernet/intel/ice/ice_controlq.c b/drivers/net/ethernet/intel/ice/ice_controlq.c index dd946866d7b8..1e18021aa073 100644 --- a/drivers/net/ethernet/intel/ice/ice_controlq.c +++ b/drivers/net/ethernet/intel/ice/ice_controlq.c @@ -12,6 +12,7 @@ do { \ (qinfo)->sq.bal = prefix##_ATQBAL; \ (qinfo)->sq.len_mask = prefix##_ATQLEN_ATQLEN_M; \ (qinfo)->sq.len_ena_mask = prefix##_ATQLEN_ATQENABLE_M; \ + (qinfo)->sq.len_crit_mask = prefix##_ATQLEN_ATQCRIT_M; \ (qinfo)->sq.head_mask = prefix##_ATQH_ATQH_M; \ (qinfo)->rq.head = prefix##_ARQH; \ (qinfo)->rq.tail = prefix##_ARQT; \ @@ -20,6 +21,7 @@ do { \ (qinfo)->rq.bal = prefix##_ARQBAL; \ (qinfo)->rq.len_mask = prefix##_ARQLEN_ARQLEN_M; \ (qinfo)->rq.len_ena_mask = prefix##_ARQLEN_ARQENABLE_M; \ + (qinfo)->rq.len_crit_mask = prefix##_ARQLEN_ARQCRIT_M; \ (qinfo)->rq.head_mask = prefix##_ARQH_ARQH_M; \ } while (0) @@ -199,7 +201,9 @@ unwind_alloc_rq_bufs: cq->rq.r.rq_bi[i].pa = 0; cq->rq.r.rq_bi[i].size = 0; } + cq->rq.r.rq_bi = NULL; devm_kfree(ice_hw_to_dev(hw), cq->rq.dma_head); + cq->rq.dma_head = NULL; return ICE_ERR_NO_MEMORY; } @@ -245,7 +249,9 @@ unwind_alloc_sq_bufs: cq->sq.r.sq_bi[i].pa = 0; cq->sq.r.sq_bi[i].size = 0; } + cq->sq.r.sq_bi = NULL; devm_kfree(ice_hw_to_dev(hw), cq->sq.dma_head); + cq->sq.dma_head = NULL; return ICE_ERR_NO_MEMORY; } @@ -304,6 +310,28 @@ ice_cfg_rq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq) return 0; } +#define ICE_FREE_CQ_BUFS(hw, qi, ring) \ +do { \ + int i; \ + /* free descriptors */ \ + if ((qi)->ring.r.ring##_bi) \ + for (i = 0; i < (qi)->num_##ring##_entries; i++) \ + if ((qi)->ring.r.ring##_bi[i].pa) { \ + dmam_free_coherent(ice_hw_to_dev(hw), \ + (qi)->ring.r.ring##_bi[i].size, \ + (qi)->ring.r.ring##_bi[i].va, \ + (qi)->ring.r.ring##_bi[i].pa); \ + (qi)->ring.r.ring##_bi[i].va = NULL;\ + (qi)->ring.r.ring##_bi[i].pa = 0;\ + (qi)->ring.r.ring##_bi[i].size = 0;\ + } \ + /* free the buffer info list */ \ + if ((qi)->ring.cmd_buf) \ + devm_kfree(ice_hw_to_dev(hw), (qi)->ring.cmd_buf); \ + /* free DMA head */ \ + devm_kfree(ice_hw_to_dev(hw), (qi)->ring.dma_head); \ +} while (0) + /** * ice_init_sq - main initialization routine for Control ATQ * @hw: pointer to the hardware structure @@ -357,6 +385,7 @@ static enum ice_status ice_init_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq) goto init_ctrlq_exit; init_ctrlq_free_rings: + ICE_FREE_CQ_BUFS(hw, cq, sq); ice_free_cq_ring(hw, &cq->sq); init_ctrlq_exit: @@ -416,33 +445,13 @@ static enum ice_status ice_init_rq(struct ice_hw *hw, struct ice_ctl_q_info *cq) goto init_ctrlq_exit; init_ctrlq_free_rings: + ICE_FREE_CQ_BUFS(hw, cq, rq); ice_free_cq_ring(hw, &cq->rq); init_ctrlq_exit: return ret_code; } -#define ICE_FREE_CQ_BUFS(hw, qi, ring) \ -do { \ - int i; \ - /* free descriptors */ \ - for (i = 0; i < (qi)->num_##ring##_entries; i++) \ - if ((qi)->ring.r.ring##_bi[i].pa) { \ - dmam_free_coherent(ice_hw_to_dev(hw), \ - (qi)->ring.r.ring##_bi[i].size,\ - (qi)->ring.r.ring##_bi[i].va,\ - (qi)->ring.r.ring##_bi[i].pa);\ - (qi)->ring.r.ring##_bi[i].va = NULL; \ - (qi)->ring.r.ring##_bi[i].pa = 0; \ - (qi)->ring.r.ring##_bi[i].size = 0; \ - } \ - /* free the buffer info list */ \ - if ((qi)->ring.cmd_buf) \ - devm_kfree(ice_hw_to_dev(hw), (qi)->ring.cmd_buf); \ - /* free DMA head */ \ - devm_kfree(ice_hw_to_dev(hw), (qi)->ring.dma_head); \ -} while (0) - /** * ice_shutdown_sq - shutdown the Control ATQ * @hw: pointer to the hardware structure @@ -635,6 +644,50 @@ init_ctrlq_free_sq: } /** + * ice_shutdown_ctrlq - shutdown routine for any control queue + * @hw: pointer to the hardware structure + * @q_type: specific Control queue type + * + * NOTE: this function does not destroy the control queue locks. + */ +static void ice_shutdown_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type) +{ + struct ice_ctl_q_info *cq; + + switch (q_type) { + case ICE_CTL_Q_ADMIN: + cq = &hw->adminq; + if (ice_check_sq_alive(hw, cq)) + ice_aq_q_shutdown(hw, true); + break; + case ICE_CTL_Q_MAILBOX: + cq = &hw->mailboxq; + break; + default: + return; + } + + ice_shutdown_sq(hw, cq); + ice_shutdown_rq(hw, cq); +} + +/** + * ice_shutdown_all_ctrlq - shutdown routine for all control queues + * @hw: pointer to the hardware structure + * + * NOTE: this function does not destroy the control queue locks. The driver + * may call this at runtime to shutdown and later restart control queues, such + * as in response to a reset event. + */ +void ice_shutdown_all_ctrlq(struct ice_hw *hw) +{ + /* Shutdown FW admin queue */ + ice_shutdown_ctrlq(hw, ICE_CTL_Q_ADMIN); + /* Shutdown PF-VF Mailbox */ + ice_shutdown_ctrlq(hw, ICE_CTL_Q_MAILBOX); +} + +/** * ice_init_all_ctrlq - main initialization routine for all control queues * @hw: pointer to the hardware structure * @@ -649,17 +702,27 @@ init_ctrlq_free_sq: */ enum ice_status ice_init_all_ctrlq(struct ice_hw *hw) { - enum ice_status ret_code; + enum ice_status status; + u32 retry = 0; /* Init FW admin queue */ - ret_code = ice_init_ctrlq(hw, ICE_CTL_Q_ADMIN); - if (ret_code) - return ret_code; + do { + status = ice_init_ctrlq(hw, ICE_CTL_Q_ADMIN); + if (status) + return status; - ret_code = ice_init_check_adminq(hw); - if (ret_code) - return ret_code; + status = ice_init_check_adminq(hw); + if (status != ICE_ERR_AQ_FW_CRITICAL) + break; + + ice_debug(hw, ICE_DBG_AQ_MSG, + "Retry Admin Queue init due to FW critical error\n"); + ice_shutdown_ctrlq(hw, ICE_CTL_Q_ADMIN); + msleep(ICE_CTL_Q_ADMIN_INIT_MSEC); + } while (retry++ < ICE_CTL_Q_ADMIN_INIT_TIMEOUT); + if (status) + return status; /* Init Mailbox queue */ return ice_init_ctrlq(hw, ICE_CTL_Q_MAILBOX); } @@ -701,57 +764,12 @@ enum ice_status ice_create_all_ctrlq(struct ice_hw *hw) } /** - * ice_shutdown_ctrlq - shutdown routine for any control queue - * @hw: pointer to the hardware structure - * @q_type: specific Control queue type - * - * NOTE: this function does not destroy the control queue locks. - */ -static void ice_shutdown_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type) -{ - struct ice_ctl_q_info *cq; - - switch (q_type) { - case ICE_CTL_Q_ADMIN: - cq = &hw->adminq; - if (ice_check_sq_alive(hw, cq)) - ice_aq_q_shutdown(hw, true); - break; - case ICE_CTL_Q_MAILBOX: - cq = &hw->mailboxq; - break; - default: - return; - } - - ice_shutdown_sq(hw, cq); - ice_shutdown_rq(hw, cq); -} - -/** - * ice_shutdown_all_ctrlq - shutdown routine for all control queues - * @hw: pointer to the hardware structure - * - * NOTE: this function does not destroy the control queue locks. The driver - * may call this at runtime to shutdown and later restart control queues, such - * as in response to a reset event. - */ -void ice_shutdown_all_ctrlq(struct ice_hw *hw) -{ - /* Shutdown FW admin queue */ - ice_shutdown_ctrlq(hw, ICE_CTL_Q_ADMIN); - /* Shutdown PF-VF Mailbox */ - ice_shutdown_ctrlq(hw, ICE_CTL_Q_MAILBOX); -} - -/** * ice_destroy_ctrlq_locks - Destroy locks for a control queue * @cq: pointer to the control queue * * Destroys the send and receive queue locks for a given control queue. */ -static void -ice_destroy_ctrlq_locks(struct ice_ctl_q_info *cq) +static void ice_destroy_ctrlq_locks(struct ice_ctl_q_info *cq) { mutex_destroy(&cq->sq_lock); mutex_destroy(&cq->rq_lock); @@ -1042,9 +1060,15 @@ ice_sq_send_cmd(struct ice_hw *hw, struct ice_ctl_q_info *cq, /* update the error if time out occurred */ if (!cmd_completed) { - ice_debug(hw, ICE_DBG_AQ_MSG, - "Control Send Queue Writeback timeout.\n"); - status = ICE_ERR_AQ_TIMEOUT; + if (rd32(hw, cq->rq.len) & cq->rq.len_crit_mask || + rd32(hw, cq->sq.len) & cq->sq.len_crit_mask) { + ice_debug(hw, ICE_DBG_AQ_MSG, "Critical FW error.\n"); + status = ICE_ERR_AQ_FW_CRITICAL; + } else { + ice_debug(hw, ICE_DBG_AQ_MSG, + "Control Send Queue Writeback timeout.\n"); + status = ICE_ERR_AQ_TIMEOUT; + } } sq_send_command_error: @@ -1128,7 +1152,7 @@ ice_clean_rq_elem(struct ice_hw *hw, struct ice_ctl_q_info *cq, } memcpy(&e->desc, desc, sizeof(e->desc)); datalen = le16_to_cpu(desc->datalen); - e->msg_len = min(datalen, e->buf_len); + e->msg_len = min_t(u16, datalen, e->buf_len); if (e->msg_buf && e->msg_len) memcpy(e->msg_buf, cq->rq.r.rq_bi[desc_idx].va, e->msg_len); diff --git a/drivers/net/ethernet/intel/ice/ice_controlq.h b/drivers/net/ethernet/intel/ice/ice_controlq.h index bf0ebe6149e8..faaa08e8171b 100644 --- a/drivers/net/ethernet/intel/ice/ice_controlq.h +++ b/drivers/net/ethernet/intel/ice/ice_controlq.h @@ -34,6 +34,8 @@ enum ice_ctl_q { /* Control Queue timeout settings - max delay 250ms */ #define ICE_CTL_Q_SQ_CMD_TIMEOUT 2500 /* Count 2500 times */ #define ICE_CTL_Q_SQ_CMD_USEC 100 /* Check every 100usec */ +#define ICE_CTL_Q_ADMIN_INIT_TIMEOUT 10 /* Count 10 times */ +#define ICE_CTL_Q_ADMIN_INIT_MSEC 100 /* Check every 100msec */ struct ice_ctl_q_ring { void *dma_head; /* Virtual address to DMA head */ @@ -59,6 +61,7 @@ struct ice_ctl_q_ring { u32 bal; u32 len_mask; u32 len_ena_mask; + u32 len_crit_mask; u32 head_mask; }; diff --git a/drivers/net/ethernet/intel/ice/ice_dcb_lib.c b/drivers/net/ethernet/intel/ice/ice_dcb_lib.c index 7bea09363b42..979af197f8a3 100644 --- a/drivers/net/ethernet/intel/ice/ice_dcb_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_dcb_lib.c @@ -63,6 +63,64 @@ u8 ice_dcb_get_ena_tc(struct ice_dcbx_cfg *dcbcfg) } /** + * ice_is_pfc_causing_hung_q + * @pf: pointer to PF structure + * @txqueue: Tx queue which is supposedly hung queue + * + * find if PFC is causing the hung queue, if yes return true else false + */ +bool ice_is_pfc_causing_hung_q(struct ice_pf *pf, unsigned int txqueue) +{ + u8 num_tcs = 0, i, tc, up_mapped_tc, up_in_tc = 0; + u64 ref_prio_xoff[ICE_MAX_UP]; + struct ice_vsi *vsi; + u32 up2tc; + + vsi = ice_get_main_vsi(pf); + if (!vsi) + return false; + + ice_for_each_traffic_class(i) + if (vsi->tc_cfg.ena_tc & BIT(i)) + num_tcs++; + + /* first find out the TC to which the hung queue belongs to */ + for (tc = 0; tc < num_tcs - 1; tc++) + if (ice_find_q_in_range(vsi->tc_cfg.tc_info[tc].qoffset, + vsi->tc_cfg.tc_info[tc + 1].qoffset, + txqueue)) + break; + + /* Build a bit map of all UPs associated to the suspect hung queue TC, + * so that we check for its counter increment. + */ + up2tc = rd32(&pf->hw, PRTDCB_TUP2TC); + for (i = 0; i < ICE_MAX_UP; i++) { + up_mapped_tc = (up2tc >> (i * 3)) & 0x7; + if (up_mapped_tc == tc) + up_in_tc |= BIT(i); + } + + /* Now that we figured out that hung queue is PFC enabled, still the + * Tx timeout can be legitimate. So to make sure Tx timeout is + * absolutely caused by PFC storm, check if the counters are + * incrementing. + */ + for (i = 0; i < ICE_MAX_UP; i++) + if (up_in_tc & BIT(i)) + ref_prio_xoff[i] = pf->stats.priority_xoff_rx[i]; + + ice_update_dcb_stats(pf); + + for (i = 0; i < ICE_MAX_UP; i++) + if (up_in_tc & BIT(i)) + if (pf->stats.priority_xoff_rx[i] > ref_prio_xoff[i]) + return true; + + return false; +} + +/** * ice_dcb_get_mode - gets the DCB mode * @port_info: pointer to port info structure * @host: if set it's HOST if not it's MANAGED @@ -526,16 +584,21 @@ static int ice_dcb_sw_dflt_cfg(struct ice_pf *pf, bool ets_willing, bool locked) */ static bool ice_dcb_tc_contig(u8 *prio_table) { - u8 max_tc = 0; + bool found_empty = false; + u8 used_tc = 0; int i; - for (i = 0; i < CEE_DCBX_MAX_PRIO; i++) { - u8 cur_tc = prio_table[i]; + /* Create a bitmap of used TCs */ + for (i = 0; i < CEE_DCBX_MAX_PRIO; i++) + used_tc |= BIT(prio_table[i]); - if (cur_tc > max_tc) - return false; - else if (cur_tc == max_tc) - max_tc++; + for (i = 0; i < CEE_DCBX_MAX_PRIO; i++) { + if (used_tc & BIT(i)) { + if (found_empty) + return false; + } else { + found_empty = true; + } } return true; @@ -728,39 +791,31 @@ void ice_update_dcb_stats(struct ice_pf *pf) * ice_tx_prepare_vlan_flags_dcb - prepare VLAN tagging for DCB * @tx_ring: ring to send buffer on * @first: pointer to struct ice_tx_buf + * + * This should not be called if the outer VLAN is software offloaded as the VLAN + * tag will already be configured with the correct ID and priority bits */ -int +void ice_tx_prepare_vlan_flags_dcb(struct ice_ring *tx_ring, struct ice_tx_buf *first) { struct sk_buff *skb = first->skb; if (!test_bit(ICE_FLAG_DCB_ENA, tx_ring->vsi->back->flags)) - return 0; + return; /* Insert 802.1p priority into VLAN header */ - if ((first->tx_flags & (ICE_TX_FLAGS_HW_VLAN | ICE_TX_FLAGS_SW_VLAN)) || + if ((first->tx_flags & ICE_TX_FLAGS_HW_VLAN) || skb->priority != TC_PRIO_CONTROL) { first->tx_flags &= ~ICE_TX_FLAGS_VLAN_PR_M; /* Mask the lower 3 bits to set the 802.1p priority */ first->tx_flags |= (skb->priority & 0x7) << ICE_TX_FLAGS_VLAN_PR_S; - if (first->tx_flags & ICE_TX_FLAGS_SW_VLAN) { - struct vlan_ethhdr *vhdr; - int rc; - - rc = skb_cow_head(skb, 0); - if (rc < 0) - return rc; - vhdr = (struct vlan_ethhdr *)skb->data; - vhdr->h_vlan_TCI = htons(first->tx_flags >> - ICE_TX_FLAGS_VLAN_S); - } else { - first->tx_flags |= ICE_TX_FLAGS_HW_VLAN; - } + /* if this is not already set it means a VLAN 0 + priority needs + * to be offloaded + */ + first->tx_flags |= ICE_TX_FLAGS_HW_VLAN; } - - return 0; } /** diff --git a/drivers/net/ethernet/intel/ice/ice_dcb_lib.h b/drivers/net/ethernet/intel/ice/ice_dcb_lib.h index 37680e815b02..323238669572 100644 --- a/drivers/net/ethernet/intel/ice/ice_dcb_lib.h +++ b/drivers/net/ethernet/intel/ice/ice_dcb_lib.h @@ -17,6 +17,8 @@ void ice_dcb_rebuild(struct ice_pf *pf); u8 ice_dcb_get_ena_tc(struct ice_dcbx_cfg *dcbcfg); u8 ice_dcb_get_num_tc(struct ice_dcbx_cfg *dcbcfg); +void ice_vsi_set_dcb_tc_cfg(struct ice_vsi *vsi); +bool ice_is_pfc_causing_hung_q(struct ice_pf *pf, unsigned int txqueue); u8 ice_dcb_get_tc(struct ice_vsi *vsi, int queue_index); int ice_pf_dcb_cfg(struct ice_pf *pf, struct ice_dcbx_cfg *new_cfg, bool locked); @@ -25,13 +27,27 @@ void ice_pf_dcb_recfg(struct ice_pf *pf); void ice_vsi_cfg_dcb_rings(struct ice_vsi *vsi); int ice_init_pf_dcb(struct ice_pf *pf, bool locked); void ice_update_dcb_stats(struct ice_pf *pf); -int +void ice_tx_prepare_vlan_flags_dcb(struct ice_ring *tx_ring, struct ice_tx_buf *first); void ice_dcb_process_lldp_set_mib_change(struct ice_pf *pf, struct ice_rq_event_info *event); void ice_vsi_cfg_netdev_tc(struct ice_vsi *vsi, u8 ena_tc); + +/** + * ice_find_q_in_range + * @low: start of queue range for a TC i.e. offset of TC + * @high: start of queue for next TC + * @tx_q: hung_queue/tx_queue + * + * finds if queue 'tx_q' falls between the two offsets of any given TC + */ +static inline bool ice_find_q_in_range(u16 low, u16 high, unsigned int tx_q) +{ + return (tx_q >= low) && (tx_q < high); +} + static inline void ice_set_cgd_num(struct ice_tlan_ctx *tlan_ctx, struct ice_ring *ring) { @@ -79,6 +95,13 @@ ice_tx_prepare_vlan_flags_dcb(struct ice_ring __always_unused *tx_ring, return 0; } +static inline bool +ice_is_pfc_causing_hung_q(struct ice_pf __always_unused *pf, + unsigned int __always_unused txqueue) +{ + return false; +} + #define ice_update_dcb_stats(pf) do {} while (0) #define ice_pf_dcb_recfg(pf) do {} while (0) #define ice_vsi_cfg_dcb_rings(vsi) do {} while (0) diff --git a/drivers/net/ethernet/intel/ice/ice_dcb_nl.c b/drivers/net/ethernet/intel/ice/ice_dcb_nl.c index c4c12414083a..87f91b750d59 100644 --- a/drivers/net/ethernet/intel/ice/ice_dcb_nl.c +++ b/drivers/net/ethernet/intel/ice/ice_dcb_nl.c @@ -7,8 +7,6 @@ #include "ice_dcb_nl.h" #include <net/dcbnl.h> -#define ICE_APP_PROT_ID_ROCE 0x8915 - /** * ice_dcbnl_devreset - perform enough of a ifdown/ifup to sync DCBNL info * @netdev: device associated with interface that needs reset @@ -671,7 +669,7 @@ static bool ice_dcbnl_find_app(struct ice_dcbx_cfg *cfg, struct ice_dcb_app_priority_table *app) { - int i; + unsigned int i; for (i = 0; i < cfg->numapps; i++) { if (app->selector == cfg->app[i].selector && @@ -746,7 +744,8 @@ static int ice_dcbnl_delapp(struct net_device *netdev, struct dcb_app *app) { struct ice_pf *pf = ice_netdev_to_pf(netdev); struct ice_dcbx_cfg *old_cfg, *new_cfg; - int i, j, ret = 0; + unsigned int i, j; + int ret = 0; if (pf->dcbx_cap & DCB_CAP_DCBX_LLD_MANAGED) return -EINVAL; @@ -869,7 +868,7 @@ void ice_dcbnl_set_all(struct ice_vsi *vsi) struct ice_port_info *pi; struct dcb_app sapp; struct ice_pf *pf; - int i; + unsigned int i; if (!netdev) return; @@ -941,7 +940,7 @@ ice_dcbnl_flush_apps(struct ice_pf *pf, struct ice_dcbx_cfg *old_cfg, struct ice_dcbx_cfg *new_cfg) { struct ice_vsi *main_vsi = ice_get_main_vsi(pf); - int i; + unsigned int i; if (!main_vsi) return; diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c index c6833944b90a..a73d06e06b5d 100644 --- a/drivers/net/ethernet/intel/ice/ice_devlink.c +++ b/drivers/net/ethernet/intel/ice/ice_devlink.c @@ -105,6 +105,27 @@ static int ice_info_ddp_pkg_version(struct ice_pf *pf, char *buf, size_t len) return 0; } +static int ice_info_netlist_ver(struct ice_pf *pf, char *buf, size_t len) +{ + struct ice_netlist_ver_info *netlist = &pf->hw.netlist_ver; + + /* The netlist version fields are BCD formatted */ + snprintf(buf, len, "%x.%x.%x-%x.%x.%x", netlist->major, netlist->minor, + netlist->type >> 16, netlist->type & 0xFFFF, netlist->rev, + netlist->cust_ver); + + return 0; +} + +static int ice_info_netlist_build(struct ice_pf *pf, char *buf, size_t len) +{ + struct ice_netlist_ver_info *netlist = &pf->hw.netlist_ver; + + snprintf(buf, len, "0x%08x", netlist->hash); + + return 0; +} + #define fixed(key, getter) { ICE_VERSION_FIXED, key, getter } #define running(key, getter) { ICE_VERSION_RUNNING, key, getter } @@ -128,6 +149,8 @@ static const struct ice_devlink_version { running(DEVLINK_INFO_VERSION_GENERIC_FW_BUNDLE_ID, ice_info_eetrack), running("fw.app.name", ice_info_ddp_pkg_name), running(DEVLINK_INFO_VERSION_GENERIC_FW_APP, ice_info_ddp_pkg_version), + running("fw.netlist", ice_info_netlist_ver), + running("fw.netlist.build", ice_info_netlist_build), }; /** diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c index 593fb37bd59e..68c38004a088 100644 --- a/drivers/net/ethernet/intel/ice/ice_ethtool.c +++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c @@ -5,6 +5,7 @@ #include "ice.h" #include "ice_flow.h" +#include "ice_fltr.h" #include "ice_lib.h" #include "ice_dcb_lib.h" @@ -129,6 +130,8 @@ static const struct ice_stats ice_gstrings_pf_stats[] = { ICE_PF_STAT("illegal_bytes.nic", stats.illegal_bytes), ICE_PF_STAT("mac_local_faults.nic", stats.mac_local_faults), ICE_PF_STAT("mac_remote_faults.nic", stats.mac_remote_faults), + ICE_PF_STAT("fdir_sb_match.nic", stats.fd_sb_match), + ICE_PF_STAT("fdir_sb_status.nic", stats.fd_sb_status), }; static const u32 ice_regs_dump_list[] = { @@ -139,9 +142,6 @@ static const u32 ice_regs_dump_list[] = { QINT_RQCTL(0), PFINT_OICR_ENA, QRX_ITR(0), - PF0INT_ITR_0(0), - PF0INT_ITR_1(0), - PF0INT_ITR_2(0), }; struct ice_priv_flag { @@ -157,6 +157,8 @@ struct ice_priv_flag { static const struct ice_priv_flag ice_gstrings_priv_flags[] = { ICE_PRIV_FLAG("link-down-on-close", ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA), ICE_PRIV_FLAG("fw-lldp-agent", ICE_FLAG_FW_LLDP_AGENT), + ICE_PRIV_FLAG("vf-true-promisc-support", + ICE_FLAG_VF_TRUE_PROMISC_ENA), ICE_PRIV_FLAG("mdd-auto-reset-vf", ICE_FLAG_MDD_AUTO_RESET_VF), ICE_PRIV_FLAG("legacy-rx", ICE_FLAG_LEGACY_RX), }; @@ -203,7 +205,7 @@ ice_get_regs(struct net_device *netdev, struct ethtool_regs *regs, void *p) struct ice_pf *pf = np->vsi->back; struct ice_hw *hw = &pf->hw; u32 *regs_buf = (u32 *)p; - int i; + unsigned int i; regs->version = 1; @@ -273,8 +275,9 @@ ice_get_eeprom(struct net_device *netdev, struct ethtool_eeprom *eeprom, status = ice_acquire_nvm(hw, ICE_RES_READ); if (status) { - dev_err(dev, "ice_acquire_nvm failed, err %d aq_err %d\n", - status, hw->adminq.sq_last_status); + dev_err(dev, "ice_acquire_nvm failed, err %s aq_err %s\n", + ice_stat_str(status), + ice_aq_str(hw->adminq.sq_last_status)); ret = -EIO; goto out; } @@ -282,8 +285,9 @@ ice_get_eeprom(struct net_device *netdev, struct ethtool_eeprom *eeprom, status = ice_read_flat_nvm(hw, eeprom->offset, &eeprom->len, buf, false); if (status) { - dev_err(dev, "ice_read_flat_nvm failed, err %d aq_err %d\n", - status, hw->adminq.sq_last_status); + dev_err(dev, "ice_read_flat_nvm failed, err %s aq_err %s\n", + ice_stat_str(status), + ice_aq_str(hw->adminq.sq_last_status)); ret = -EIO; goto release; } @@ -304,7 +308,7 @@ out: */ static bool ice_active_vfs(struct ice_pf *pf) { - int i; + unsigned int i; ice_for_each_vf(pf, i) { struct ice_vf *vf = &pf->vf[i]; @@ -332,7 +336,8 @@ static u64 ice_link_test(struct net_device *netdev) netdev_info(netdev, "link test\n"); status = ice_get_link_status(np->vsi->port_info, &link_up); if (status) { - netdev_err(netdev, "link query error, status = %d\n", status); + netdev_err(netdev, "link query error, status = %s\n", + ice_stat_str(status)); return 1; } @@ -373,7 +378,7 @@ static int ice_reg_pattern_test(struct ice_hw *hw, u32 reg, u32 mask) 0x00000000, 0xFFFFFFFF }; u32 val, orig_val; - int i; + unsigned int i; orig_val = rd32(hw, reg); for (i = 0; i < ARRAY_SIZE(patterns); ++i) { @@ -426,7 +431,7 @@ static u64 ice_reg_test(struct net_device *netdev) GLINT_ITR(2, 1) - GLINT_ITR(2, 0)}, {GLINT_CTL, 0xffff0001, 1, 0} }; - int i; + unsigned int i; netdev_dbg(netdev, "Register test\n"); for (i = 0; i < ARRAY_SIZE(ice_reg_list); ++i) { @@ -671,7 +676,6 @@ static u64 ice_loopback_test(struct net_device *netdev) struct ice_ring *tx_ring, *rx_ring; u8 broadcast[ETH_ALEN], ret = 0; int num_frames, valid_frames; - LIST_HEAD(tmp_list); struct device *dev; u8 *tx_frame; int i; @@ -707,16 +711,11 @@ static u64 ice_loopback_test(struct net_device *netdev) /* Test VSI needs to receive broadcast packets */ eth_broadcast_addr(broadcast); - if (ice_add_mac_to_list(test_vsi, &tmp_list, broadcast)) { + if (ice_fltr_add_mac(test_vsi, broadcast, ICE_FWD_TO_VSI)) { ret = 5; goto lbtest_mac_dis; } - if (ice_add_mac(&pf->hw, &tmp_list)) { - ret = 6; - goto free_mac_list; - } - if (ice_lbtest_create_frame(pf, &tx_frame, ICE_LB_FRAME_SIZE)) { ret = 7; goto remove_mac_filters; @@ -739,10 +738,8 @@ static u64 ice_loopback_test(struct net_device *netdev) lbtest_free_frame: devm_kfree(dev, tx_frame); remove_mac_filters: - if (ice_remove_mac(&pf->hw, &tmp_list)) + if (ice_fltr_remove_mac(test_vsi, broadcast, ICE_FWD_TO_VSI)) netdev_err(netdev, "Could not remove MAC filter for the test VSI\n"); -free_mac_list: - ice_free_fltr_list(dev, &tmp_list); lbtest_mac_dis: /* Disable MAC loopback after the test is completed. */ if (ice_aq_set_mac_loopback(&pf->hw, false, NULL)) @@ -1158,8 +1155,9 @@ static int ice_nway_reset(struct net_device *netdev) status = ice_aq_set_link_restart_an(pi, false, NULL); if (status) { - netdev_info(netdev, "link restart failed, err %d aq_err %d\n", - status, pi->hw->adminq.sq_last_status); + netdev_info(netdev, "link restart failed, err %s aq_err %s\n", + ice_stat_str(status), + ice_aq_str(pi->hw->adminq.sq_last_status)); return -EIO; } @@ -1308,6 +1306,16 @@ static int ice_set_priv_flags(struct net_device *netdev, u32 flags) ice_down(vsi); ice_up(vsi); } + /* don't allow modification of this flag when a single VF is in + * promiscuous mode because it's not supported + */ + if (test_bit(ICE_FLAG_VF_TRUE_PROMISC_ENA, change_flags) && + ice_is_any_vf_in_promisc(pf)) { + dev_err(dev, "Changing vf-true-promisc-support flag while VF(s) are in promiscuous mode not supported\n"); + /* toggle bit back to previous state */ + change_bit(ICE_FLAG_VF_TRUE_PROMISC_ENA, pf->flags); + ret = -EAGAIN; + } clear_bit(ICE_FLAG_ETHTOOL_CTXT, pf->flags); return ret; } @@ -2450,8 +2458,8 @@ ice_set_rss_hash_opt(struct ice_vsi *vsi, struct ethtool_rxnfc *nfc) status = ice_add_rss_cfg(&pf->hw, vsi->idx, hashed_flds, hdrs); if (status) { - dev_dbg(dev, "ice_add_rss_cfg failed, vsi num = %d, error = %d\n", - vsi->vsi_num, status); + dev_dbg(dev, "ice_add_rss_cfg failed, vsi num = %d, error = %s\n", + vsi->vsi_num, ice_stat_str(status)); return -EINVAL; } @@ -2526,6 +2534,10 @@ static int ice_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) struct ice_vsi *vsi = np->vsi; switch (cmd->cmd) { + case ETHTOOL_SRXCLSRLINS: + return ice_add_fdir_ethtool(vsi, cmd); + case ETHTOOL_SRXCLSRLDEL: + return ice_del_fdir_ethtool(vsi, cmd); case ETHTOOL_SRXFH: return ice_set_rss_hash_opt(vsi, cmd); default: @@ -2549,12 +2561,27 @@ ice_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, struct ice_netdev_priv *np = netdev_priv(netdev); struct ice_vsi *vsi = np->vsi; int ret = -EOPNOTSUPP; + struct ice_hw *hw; + + hw = &vsi->back->hw; switch (cmd->cmd) { case ETHTOOL_GRXRINGS: cmd->data = vsi->rss_size; ret = 0; break; + case ETHTOOL_GRXCLSRLCNT: + cmd->rule_cnt = hw->fdir_active_fltr; + /* report total rule count */ + cmd->data = ice_get_fdir_cnt_all(hw); + ret = 0; + break; + case ETHTOOL_GRXCLSRULE: + ret = ice_get_ethtool_fdir_entry(hw, cmd); + break; + case ETHTOOL_GRXCLSRLALL: + ret = ice_get_fdir_fltr_ids(hw, cmd, (u32 *)rule_locs); + break; case ETHTOOL_GRXFH: ice_get_rss_hash_opt(vsi, cmd); ret = 0; @@ -2593,7 +2620,7 @@ ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring) struct ice_vsi *vsi = np->vsi; struct ice_pf *pf = vsi->back; int i, timeout = 50, err = 0; - u32 new_rx_cnt, new_tx_cnt; + u16 new_rx_cnt, new_tx_cnt; if (ring->tx_pending > ICE_MAX_NUM_DESC || ring->tx_pending < ICE_MIN_NUM_DESC || @@ -2645,8 +2672,8 @@ ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring) if (ice_is_xdp_ena_vsi(vsi)) for (i = 0; i < vsi->num_xdp_txq; i++) vsi->xdp_rings[i]->count = new_tx_cnt; - vsi->num_tx_desc = new_tx_cnt; - vsi->num_rx_desc = new_rx_cnt; + vsi->num_tx_desc = (u16)new_tx_cnt; + vsi->num_rx_desc = (u16)new_rx_cnt; netdev_dbg(netdev, "Link is down, descriptor count change happens when link is brought up\n"); goto done; } @@ -2952,31 +2979,22 @@ ice_set_pauseparam(struct net_device *netdev, struct ethtool_pauseparam *pause) status = ice_set_fc(pi, &aq_failures, link_up); if (aq_failures & ICE_SET_FC_AQ_FAIL_GET) { - netdev_info(netdev, "Set fc failed on the get_phy_capabilities call with err %d aq_err %d\n", - status, hw->adminq.sq_last_status); + netdev_info(netdev, "Set fc failed on the get_phy_capabilities call with err %s aq_err %s\n", + ice_stat_str(status), + ice_aq_str(hw->adminq.sq_last_status)); err = -EAGAIN; } else if (aq_failures & ICE_SET_FC_AQ_FAIL_SET) { - netdev_info(netdev, "Set fc failed on the set_phy_config call with err %d aq_err %d\n", - status, hw->adminq.sq_last_status); + netdev_info(netdev, "Set fc failed on the set_phy_config call with err %s aq_err %s\n", + ice_stat_str(status), + ice_aq_str(hw->adminq.sq_last_status)); err = -EAGAIN; } else if (aq_failures & ICE_SET_FC_AQ_FAIL_UPDATE) { - netdev_info(netdev, "Set fc failed on the get_link_info call with err %d aq_err %d\n", - status, hw->adminq.sq_last_status); + netdev_info(netdev, "Set fc failed on the get_link_info call with err %s aq_err %s\n", + ice_stat_str(status), + ice_aq_str(hw->adminq.sq_last_status)); err = -EAGAIN; } - if (!test_bit(__ICE_DOWN, pf->state)) { - /* Give it a little more time to try to come back. If still - * down, restart autoneg link or reinitialize the interface. - */ - msleep(75); - if (!test_bit(__ICE_DOWN, pf->state)) - return ice_nway_reset(netdev); - - ice_down(vsi); - ice_up(vsi); - } - return err; } @@ -3171,10 +3189,6 @@ ice_get_channels(struct net_device *dev, struct ethtool_channels *ch) struct ice_vsi *vsi = np->vsi; struct ice_pf *pf = vsi->back; - /* check to see if VSI is active */ - if (test_bit(__ICE_DOWN, vsi->state)) - return; - /* report maximum channels */ ch->max_rx = ice_get_max_rxq(pf); ch->max_tx = ice_get_max_txq(pf); @@ -3184,6 +3198,10 @@ ice_get_channels(struct net_device *dev, struct ethtool_channels *ch) ch->combined_count = ice_get_combined_cnt(vsi); ch->rx_count = vsi->num_rxq - ch->combined_count; ch->tx_count = vsi->num_txq - ch->combined_count; + + /* report other queues */ + ch->other_count = test_bit(ICE_FLAG_FD_ENA, pf->flags) ? 1 : 0; + ch->max_other = ch->other_count; } /** @@ -3227,8 +3245,9 @@ static int ice_vsi_set_dflt_rss_lut(struct ice_vsi *vsi, int req_rss_size) status = ice_aq_set_rss_lut(hw, vsi->idx, vsi->rss_lut_type, lut, vsi->rss_table_size); if (status) { - dev_err(dev, "Cannot set RSS lut, err %d aq_err %d\n", - status, hw->adminq.rq_last_status); + dev_err(dev, "Cannot set RSS lut, err %s aq_err %s\n", + ice_stat_str(status), + ice_aq_str(hw->adminq.sq_last_status)); err = -EIO; } @@ -3255,9 +3274,14 @@ static int ice_set_channels(struct net_device *dev, struct ethtool_channels *ch) return -EOPNOTSUPP; } /* do not support changing other_count */ - if (ch->other_count) + if (ch->other_count != (test_bit(ICE_FLAG_FD_ENA, pf->flags) ? 1U : 0U)) return -EINVAL; + if (test_bit(ICE_FLAG_FD_ENA, pf->flags) && pf->hw.fdir_active_fltr) { + netdev_err(dev, "Cannot set channels when Flow Director filters are active\n"); + return -EOPNOTSUPP; + } + curr_combined = ice_get_combined_cnt(vsi); /* these checks are for cases where user didn't specify a particular @@ -3731,10 +3755,10 @@ ice_get_module_eeprom(struct net_device *netdev, struct ice_hw *hw = &pf->hw; enum ice_status status; bool is_sfp = false; + unsigned int i; u16 offset = 0; u8 value = 0; u8 page = 0; - int i; status = ice_aq_sff_eeprom(hw, 0, addr, offset, page, 0, &value, 1, 0, NULL); diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool_fdir.c b/drivers/net/ethernet/intel/ice/ice_ethtool_fdir.c new file mode 100644 index 000000000000..d7430ce6af26 --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_ethtool_fdir.c @@ -0,0 +1,1697 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2018-2020, Intel Corporation. */ + +/* flow director ethtool support for ice */ + +#include "ice.h" +#include "ice_lib.h" +#include "ice_flow.h" + +static struct in6_addr full_ipv6_addr_mask = { + .in6_u = { + .u6_addr8 = { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + } + } +}; + +static struct in6_addr zero_ipv6_addr_mask = { + .in6_u = { + .u6_addr8 = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + } + } +}; + +/* calls to ice_flow_add_prof require the number of segments in the array + * for segs_cnt. In this code that is one more than the index. + */ +#define TNL_SEG_CNT(_TNL_) ((_TNL_) + 1) + +/** + * ice_fltr_to_ethtool_flow - convert filter type values to ethtool + * flow type values + * @flow: filter type to be converted + * + * Returns the corresponding ethtool flow type. + */ +static int ice_fltr_to_ethtool_flow(enum ice_fltr_ptype flow) +{ + switch (flow) { + case ICE_FLTR_PTYPE_NONF_IPV4_TCP: + return TCP_V4_FLOW; + case ICE_FLTR_PTYPE_NONF_IPV4_UDP: + return UDP_V4_FLOW; + case ICE_FLTR_PTYPE_NONF_IPV4_SCTP: + return SCTP_V4_FLOW; + case ICE_FLTR_PTYPE_NONF_IPV4_OTHER: + return IPV4_USER_FLOW; + case ICE_FLTR_PTYPE_NONF_IPV6_TCP: + return TCP_V6_FLOW; + case ICE_FLTR_PTYPE_NONF_IPV6_UDP: + return UDP_V6_FLOW; + case ICE_FLTR_PTYPE_NONF_IPV6_SCTP: + return SCTP_V6_FLOW; + case ICE_FLTR_PTYPE_NONF_IPV6_OTHER: + return IPV6_USER_FLOW; + default: + /* 0 is undefined ethtool flow */ + return 0; + } +} + +/** + * ice_ethtool_flow_to_fltr - convert ethtool flow type to filter enum + * @eth: Ethtool flow type to be converted + * + * Returns flow enum + */ +static enum ice_fltr_ptype ice_ethtool_flow_to_fltr(int eth) +{ + switch (eth) { + case TCP_V4_FLOW: + return ICE_FLTR_PTYPE_NONF_IPV4_TCP; + case UDP_V4_FLOW: + return ICE_FLTR_PTYPE_NONF_IPV4_UDP; + case SCTP_V4_FLOW: + return ICE_FLTR_PTYPE_NONF_IPV4_SCTP; + case IPV4_USER_FLOW: + return ICE_FLTR_PTYPE_NONF_IPV4_OTHER; + case TCP_V6_FLOW: + return ICE_FLTR_PTYPE_NONF_IPV6_TCP; + case UDP_V6_FLOW: + return ICE_FLTR_PTYPE_NONF_IPV6_UDP; + case SCTP_V6_FLOW: + return ICE_FLTR_PTYPE_NONF_IPV6_SCTP; + case IPV6_USER_FLOW: + return ICE_FLTR_PTYPE_NONF_IPV6_OTHER; + default: + return ICE_FLTR_PTYPE_NONF_NONE; + } +} + +/** + * ice_is_mask_valid - check mask field set + * @mask: full mask to check + * @field: field for which mask should be valid + * + * If the mask is fully set return true. If it is not valid for field return + * false. + */ +static bool ice_is_mask_valid(u64 mask, u64 field) +{ + return (mask & field) == field; +} + +/** + * ice_get_ethtool_fdir_entry - fill ethtool structure with fdir filter data + * @hw: hardware structure that contains filter list + * @cmd: ethtool command data structure to receive the filter data + * + * Returns 0 on success and -EINVAL on failure + */ +int ice_get_ethtool_fdir_entry(struct ice_hw *hw, struct ethtool_rxnfc *cmd) +{ + struct ethtool_rx_flow_spec *fsp; + struct ice_fdir_fltr *rule; + int ret = 0; + u16 idx; + + fsp = (struct ethtool_rx_flow_spec *)&cmd->fs; + + mutex_lock(&hw->fdir_fltr_lock); + + rule = ice_fdir_find_fltr_by_idx(hw, fsp->location); + + if (!rule || fsp->location != rule->fltr_id) { + ret = -EINVAL; + goto release_lock; + } + + fsp->flow_type = ice_fltr_to_ethtool_flow(rule->flow_type); + + memset(&fsp->m_u, 0, sizeof(fsp->m_u)); + memset(&fsp->m_ext, 0, sizeof(fsp->m_ext)); + + switch (fsp->flow_type) { + case IPV4_USER_FLOW: + fsp->h_u.usr_ip4_spec.ip_ver = ETH_RX_NFC_IP4; + fsp->h_u.usr_ip4_spec.proto = 0; + fsp->h_u.usr_ip4_spec.l4_4_bytes = rule->ip.v4.l4_header; + fsp->h_u.usr_ip4_spec.tos = rule->ip.v4.tos; + fsp->h_u.usr_ip4_spec.ip4src = rule->ip.v4.src_ip; + fsp->h_u.usr_ip4_spec.ip4dst = rule->ip.v4.dst_ip; + fsp->m_u.usr_ip4_spec.ip4src = rule->mask.v4.src_ip; + fsp->m_u.usr_ip4_spec.ip4dst = rule->mask.v4.dst_ip; + fsp->m_u.usr_ip4_spec.ip_ver = 0xFF; + fsp->m_u.usr_ip4_spec.proto = 0; + fsp->m_u.usr_ip4_spec.l4_4_bytes = rule->mask.v4.l4_header; + fsp->m_u.usr_ip4_spec.tos = rule->mask.v4.tos; + break; + case TCP_V4_FLOW: + case UDP_V4_FLOW: + case SCTP_V4_FLOW: + fsp->h_u.tcp_ip4_spec.psrc = rule->ip.v4.src_port; + fsp->h_u.tcp_ip4_spec.pdst = rule->ip.v4.dst_port; + fsp->h_u.tcp_ip4_spec.ip4src = rule->ip.v4.src_ip; + fsp->h_u.tcp_ip4_spec.ip4dst = rule->ip.v4.dst_ip; + fsp->m_u.tcp_ip4_spec.psrc = rule->mask.v4.src_port; + fsp->m_u.tcp_ip4_spec.pdst = rule->mask.v4.dst_port; + fsp->m_u.tcp_ip4_spec.ip4src = rule->mask.v4.src_ip; + fsp->m_u.tcp_ip4_spec.ip4dst = rule->mask.v4.dst_ip; + break; + case IPV6_USER_FLOW: + fsp->h_u.usr_ip6_spec.l4_4_bytes = rule->ip.v6.l4_header; + fsp->h_u.usr_ip6_spec.tclass = rule->ip.v6.tc; + fsp->h_u.usr_ip6_spec.l4_proto = rule->ip.v6.proto; + memcpy(fsp->h_u.tcp_ip6_spec.ip6src, rule->ip.v6.src_ip, + sizeof(struct in6_addr)); + memcpy(fsp->h_u.tcp_ip6_spec.ip6dst, rule->ip.v6.dst_ip, + sizeof(struct in6_addr)); + memcpy(fsp->m_u.tcp_ip6_spec.ip6src, rule->mask.v6.src_ip, + sizeof(struct in6_addr)); + memcpy(fsp->m_u.tcp_ip6_spec.ip6dst, rule->mask.v6.dst_ip, + sizeof(struct in6_addr)); + fsp->m_u.usr_ip6_spec.l4_4_bytes = rule->mask.v6.l4_header; + fsp->m_u.usr_ip6_spec.tclass = rule->mask.v6.tc; + fsp->m_u.usr_ip6_spec.l4_proto = rule->mask.v6.proto; + break; + case TCP_V6_FLOW: + case UDP_V6_FLOW: + case SCTP_V6_FLOW: + memcpy(fsp->h_u.tcp_ip6_spec.ip6src, rule->ip.v6.src_ip, + sizeof(struct in6_addr)); + memcpy(fsp->h_u.tcp_ip6_spec.ip6dst, rule->ip.v6.dst_ip, + sizeof(struct in6_addr)); + fsp->h_u.tcp_ip6_spec.psrc = rule->ip.v6.src_port; + fsp->h_u.tcp_ip6_spec.pdst = rule->ip.v6.dst_port; + memcpy(fsp->m_u.tcp_ip6_spec.ip6src, + rule->mask.v6.src_ip, + sizeof(struct in6_addr)); + memcpy(fsp->m_u.tcp_ip6_spec.ip6dst, + rule->mask.v6.dst_ip, + sizeof(struct in6_addr)); + fsp->m_u.tcp_ip6_spec.psrc = rule->mask.v6.src_port; + fsp->m_u.tcp_ip6_spec.pdst = rule->mask.v6.dst_port; + fsp->h_u.tcp_ip6_spec.tclass = rule->ip.v6.tc; + fsp->m_u.tcp_ip6_spec.tclass = rule->mask.v6.tc; + break; + default: + break; + } + + if (rule->dest_ctl == ICE_FLTR_PRGM_DESC_DEST_DROP_PKT) + fsp->ring_cookie = RX_CLS_FLOW_DISC; + else + fsp->ring_cookie = rule->q_index; + + idx = ice_ethtool_flow_to_fltr(fsp->flow_type); + if (idx == ICE_FLTR_PTYPE_NONF_NONE) { + dev_err(ice_hw_to_dev(hw), "Missing input index for flow_type %d\n", + rule->flow_type); + ret = -EINVAL; + } + +release_lock: + mutex_unlock(&hw->fdir_fltr_lock); + return ret; +} + +/** + * ice_get_fdir_fltr_ids - fill buffer with filter IDs of active filters + * @hw: hardware structure containing the filter list + * @cmd: ethtool command data structure + * @rule_locs: ethtool array passed in from OS to receive filter IDs + * + * Returns 0 as expected for success by ethtool + */ +int +ice_get_fdir_fltr_ids(struct ice_hw *hw, struct ethtool_rxnfc *cmd, + u32 *rule_locs) +{ + struct ice_fdir_fltr *f_rule; + unsigned int cnt = 0; + int val = 0; + + /* report total rule count */ + cmd->data = ice_get_fdir_cnt_all(hw); + + mutex_lock(&hw->fdir_fltr_lock); + + list_for_each_entry(f_rule, &hw->fdir_list_head, fltr_node) { + if (cnt == cmd->rule_cnt) { + val = -EMSGSIZE; + goto release_lock; + } + rule_locs[cnt] = f_rule->fltr_id; + cnt++; + } + +release_lock: + mutex_unlock(&hw->fdir_fltr_lock); + if (!val) + cmd->rule_cnt = cnt; + return val; +} + +/** + * ice_fdir_get_hw_prof - return the ice_fd_hw_proc associated with a flow + * @hw: hardware structure containing the filter list + * @blk: hardware block + * @flow: FDir flow type to release + */ +static struct ice_fd_hw_prof * +ice_fdir_get_hw_prof(struct ice_hw *hw, enum ice_block blk, int flow) +{ + if (blk == ICE_BLK_FD && hw->fdir_prof) + return hw->fdir_prof[flow]; + + return NULL; +} + +/** + * ice_fdir_erase_flow_from_hw - remove a flow from the HW profile tables + * @hw: hardware structure containing the filter list + * @blk: hardware block + * @flow: FDir flow type to release + */ +static void +ice_fdir_erase_flow_from_hw(struct ice_hw *hw, enum ice_block blk, int flow) +{ + struct ice_fd_hw_prof *prof = ice_fdir_get_hw_prof(hw, blk, flow); + int tun; + + if (!prof) + return; + + for (tun = 0; tun < ICE_FD_HW_SEG_MAX; tun++) { + u64 prof_id; + int j; + + prof_id = flow + tun * ICE_FLTR_PTYPE_MAX; + for (j = 0; j < prof->cnt; j++) { + u16 vsi_num; + + if (!prof->entry_h[j][tun] || !prof->vsi_h[j]) + continue; + vsi_num = ice_get_hw_vsi_num(hw, prof->vsi_h[j]); + ice_rem_prof_id_flow(hw, blk, vsi_num, prof_id); + ice_flow_rem_entry(hw, blk, prof->entry_h[j][tun]); + prof->entry_h[j][tun] = 0; + } + ice_flow_rem_prof(hw, blk, prof_id); + } +} + +/** + * ice_fdir_rem_flow - release the ice_flow structures for a filter type + * @hw: hardware structure containing the filter list + * @blk: hardware block + * @flow_type: FDir flow type to release + */ +static void +ice_fdir_rem_flow(struct ice_hw *hw, enum ice_block blk, + enum ice_fltr_ptype flow_type) +{ + int flow = (int)flow_type & ~FLOW_EXT; + struct ice_fd_hw_prof *prof; + int tun, i; + + prof = ice_fdir_get_hw_prof(hw, blk, flow); + if (!prof) + return; + + ice_fdir_erase_flow_from_hw(hw, blk, flow); + for (i = 0; i < prof->cnt; i++) + prof->vsi_h[i] = 0; + for (tun = 0; tun < ICE_FD_HW_SEG_MAX; tun++) { + if (!prof->fdir_seg[tun]) + continue; + devm_kfree(ice_hw_to_dev(hw), prof->fdir_seg[tun]); + prof->fdir_seg[tun] = NULL; + } + prof->cnt = 0; +} + +/** + * ice_fdir_release_flows - release all flows in use for later replay + * @hw: pointer to HW instance + */ +void ice_fdir_release_flows(struct ice_hw *hw) +{ + int flow; + + /* release Flow Director HW table entries */ + for (flow = 0; flow < ICE_FLTR_PTYPE_MAX; flow++) + ice_fdir_erase_flow_from_hw(hw, ICE_BLK_FD, flow); +} + +/** + * ice_fdir_replay_flows - replay HW Flow Director filter info + * @hw: pointer to HW instance + */ +void ice_fdir_replay_flows(struct ice_hw *hw) +{ + int flow; + + for (flow = 0; flow < ICE_FLTR_PTYPE_MAX; flow++) { + int tun; + + if (!hw->fdir_prof[flow] || !hw->fdir_prof[flow]->cnt) + continue; + for (tun = 0; tun < ICE_FD_HW_SEG_MAX; tun++) { + struct ice_flow_prof *hw_prof; + struct ice_fd_hw_prof *prof; + u64 prof_id; + int j; + + prof = hw->fdir_prof[flow]; + prof_id = flow + tun * ICE_FLTR_PTYPE_MAX; + ice_flow_add_prof(hw, ICE_BLK_FD, ICE_FLOW_RX, prof_id, + prof->fdir_seg[tun], TNL_SEG_CNT(tun), + &hw_prof); + for (j = 0; j < prof->cnt; j++) { + enum ice_flow_priority prio; + u64 entry_h = 0; + int err; + + prio = ICE_FLOW_PRIO_NORMAL; + err = ice_flow_add_entry(hw, ICE_BLK_FD, + prof_id, + prof->vsi_h[0], + prof->vsi_h[j], + prio, prof->fdir_seg, + &entry_h); + if (err) { + dev_err(ice_hw_to_dev(hw), "Could not replay Flow Director, flow type %d\n", + flow); + continue; + } + prof->entry_h[j][tun] = entry_h; + } + } + } +} + +/** + * ice_parse_rx_flow_user_data - deconstruct user-defined data + * @fsp: pointer to ethtool Rx flow specification + * @data: pointer to userdef data structure for storage + * + * Returns 0 on success, negative error value on failure + */ +static int +ice_parse_rx_flow_user_data(struct ethtool_rx_flow_spec *fsp, + struct ice_rx_flow_userdef *data) +{ + u64 value, mask; + + memset(data, 0, sizeof(*data)); + if (!(fsp->flow_type & FLOW_EXT)) + return 0; + + value = be64_to_cpu(*((__force __be64 *)fsp->h_ext.data)); + mask = be64_to_cpu(*((__force __be64 *)fsp->m_ext.data)); + if (!mask) + return 0; + +#define ICE_USERDEF_FLEX_WORD_M GENMASK_ULL(15, 0) +#define ICE_USERDEF_FLEX_OFFS_S 16 +#define ICE_USERDEF_FLEX_OFFS_M GENMASK_ULL(31, ICE_USERDEF_FLEX_OFFS_S) +#define ICE_USERDEF_FLEX_FLTR_M GENMASK_ULL(31, 0) + + /* 0x1fe is the maximum value for offsets stored in the internal + * filtering tables. + */ +#define ICE_USERDEF_FLEX_MAX_OFFS_VAL 0x1fe + + if (!ice_is_mask_valid(mask, ICE_USERDEF_FLEX_FLTR_M) || + value > ICE_USERDEF_FLEX_FLTR_M) + return -EINVAL; + + data->flex_word = value & ICE_USERDEF_FLEX_WORD_M; + data->flex_offset = (value & ICE_USERDEF_FLEX_OFFS_M) >> + ICE_USERDEF_FLEX_OFFS_S; + if (data->flex_offset > ICE_USERDEF_FLEX_MAX_OFFS_VAL) + return -EINVAL; + + data->flex_fltr = true; + + return 0; +} + +/** + * ice_fdir_num_avail_fltr - return the number of unused flow director filters + * @hw: pointer to hardware structure + * @vsi: software VSI structure + * + * There are 2 filter pools: guaranteed and best effort(shared). Each VSI can + * use filters from either pool. The guaranteed pool is divided between VSIs. + * The best effort filter pool is common to all VSIs and is a device shared + * resource pool. The number of filters available to this VSI is the sum of + * the VSIs guaranteed filter pool and the global available best effort + * filter pool. + * + * Returns the number of available flow director filters to this VSI + */ +static int ice_fdir_num_avail_fltr(struct ice_hw *hw, struct ice_vsi *vsi) +{ + u16 vsi_num = ice_get_hw_vsi_num(hw, vsi->idx); + u16 num_guar; + u16 num_be; + + /* total guaranteed filters assigned to this VSI */ + num_guar = vsi->num_gfltr; + + /* minus the guaranteed filters programed by this VSI */ + num_guar -= (rd32(hw, VSIQF_FD_CNT(vsi_num)) & + VSIQF_FD_CNT_FD_GCNT_M) >> VSIQF_FD_CNT_FD_GCNT_S; + + /* total global best effort filters */ + num_be = hw->func_caps.fd_fltr_best_effort; + + /* minus the global best effort filters programmed */ + num_be -= (rd32(hw, GLQF_FD_CNT) & GLQF_FD_CNT_FD_BCNT_M) >> + GLQF_FD_CNT_FD_BCNT_S; + + return num_guar + num_be; +} + +/** + * ice_fdir_alloc_flow_prof - allocate FDir flow profile structure(s) + * @hw: HW structure containing the FDir flow profile structure(s) + * @flow: flow type to allocate the flow profile for + * + * Allocate the fdir_prof and fdir_prof[flow] if not already created. Return 0 + * on success and negative on error. + */ +static int +ice_fdir_alloc_flow_prof(struct ice_hw *hw, enum ice_fltr_ptype flow) +{ + if (!hw) + return -EINVAL; + + if (!hw->fdir_prof) { + hw->fdir_prof = devm_kcalloc(ice_hw_to_dev(hw), + ICE_FLTR_PTYPE_MAX, + sizeof(*hw->fdir_prof), + GFP_KERNEL); + if (!hw->fdir_prof) + return -ENOMEM; + } + + if (!hw->fdir_prof[flow]) { + hw->fdir_prof[flow] = devm_kzalloc(ice_hw_to_dev(hw), + sizeof(**hw->fdir_prof), + GFP_KERNEL); + if (!hw->fdir_prof[flow]) + return -ENOMEM; + } + + return 0; +} + +/** + * ice_fdir_set_hw_fltr_rule - Configure HW tables to generate a FDir rule + * @pf: pointer to the PF structure + * @seg: protocol header description pointer + * @flow: filter enum + * @tun: FDir segment to program + */ +static int +ice_fdir_set_hw_fltr_rule(struct ice_pf *pf, struct ice_flow_seg_info *seg, + enum ice_fltr_ptype flow, enum ice_fd_hw_seg tun) +{ + struct device *dev = ice_pf_to_dev(pf); + struct ice_vsi *main_vsi, *ctrl_vsi; + struct ice_flow_seg_info *old_seg; + struct ice_flow_prof *prof = NULL; + struct ice_fd_hw_prof *hw_prof; + struct ice_hw *hw = &pf->hw; + enum ice_status status; + u64 entry1_h = 0; + u64 entry2_h = 0; + u64 prof_id; + int err; + + main_vsi = ice_get_main_vsi(pf); + if (!main_vsi) + return -EINVAL; + + ctrl_vsi = ice_get_ctrl_vsi(pf); + if (!ctrl_vsi) + return -EINVAL; + + err = ice_fdir_alloc_flow_prof(hw, flow); + if (err) + return err; + + hw_prof = hw->fdir_prof[flow]; + old_seg = hw_prof->fdir_seg[tun]; + if (old_seg) { + /* This flow_type already has a changed input set. + * If it matches the requested input set then we are + * done. Or, if it's different then it's an error. + */ + if (!memcmp(old_seg, seg, sizeof(*seg))) + return -EEXIST; + + /* if there are FDir filters using this flow, + * then return error. + */ + if (hw->fdir_fltr_cnt[flow]) { + dev_err(dev, "Failed to add filter. Flow director filters on each port must have the same input set.\n"); + return -EINVAL; + } + + if (ice_is_arfs_using_perfect_flow(hw, flow)) { + dev_err(dev, "aRFS using perfect flow type %d, cannot change input set\n", + flow); + return -EINVAL; + } + + /* remove HW filter definition */ + ice_fdir_rem_flow(hw, ICE_BLK_FD, flow); + } + + /* Adding a profile, but there is only one header supported. + * That is the final parameters are 1 header (segment), no + * actions (NULL) and zero actions 0. + */ + prof_id = flow + tun * ICE_FLTR_PTYPE_MAX; + status = ice_flow_add_prof(hw, ICE_BLK_FD, ICE_FLOW_RX, prof_id, seg, + TNL_SEG_CNT(tun), &prof); + if (status) + return ice_status_to_errno(status); + status = ice_flow_add_entry(hw, ICE_BLK_FD, prof_id, main_vsi->idx, + main_vsi->idx, ICE_FLOW_PRIO_NORMAL, + seg, &entry1_h); + if (status) { + err = ice_status_to_errno(status); + goto err_prof; + } + status = ice_flow_add_entry(hw, ICE_BLK_FD, prof_id, main_vsi->idx, + ctrl_vsi->idx, ICE_FLOW_PRIO_NORMAL, + seg, &entry2_h); + if (status) { + err = ice_status_to_errno(status); + goto err_entry; + } + + hw_prof->fdir_seg[tun] = seg; + hw_prof->entry_h[0][tun] = entry1_h; + hw_prof->entry_h[1][tun] = entry2_h; + hw_prof->vsi_h[0] = main_vsi->idx; + hw_prof->vsi_h[1] = ctrl_vsi->idx; + if (!hw_prof->cnt) + hw_prof->cnt = 2; + + return 0; + +err_entry: + ice_rem_prof_id_flow(hw, ICE_BLK_FD, + ice_get_hw_vsi_num(hw, main_vsi->idx), prof_id); + ice_flow_rem_entry(hw, ICE_BLK_FD, entry1_h); +err_prof: + ice_flow_rem_prof(hw, ICE_BLK_FD, prof_id); + dev_err(dev, "Failed to add filter. Flow director filters on each port must have the same input set.\n"); + + return err; +} + +/** + * ice_set_init_fdir_seg + * @seg: flow segment for programming + * @l3_proto: ICE_FLOW_SEG_HDR_IPV4 or ICE_FLOW_SEG_HDR_IPV6 + * @l4_proto: ICE_FLOW_SEG_HDR_TCP or ICE_FLOW_SEG_HDR_UDP + * + * Set the configuration for perfect filters to the provided flow segment for + * programming the HW filter. This is to be called only when initializing + * filters as this function it assumes no filters exist. + */ +static int +ice_set_init_fdir_seg(struct ice_flow_seg_info *seg, + enum ice_flow_seg_hdr l3_proto, + enum ice_flow_seg_hdr l4_proto) +{ + enum ice_flow_field src_addr, dst_addr, src_port, dst_port; + + if (!seg) + return -EINVAL; + + if (l3_proto == ICE_FLOW_SEG_HDR_IPV4) { + src_addr = ICE_FLOW_FIELD_IDX_IPV4_SA; + dst_addr = ICE_FLOW_FIELD_IDX_IPV4_DA; + } else if (l3_proto == ICE_FLOW_SEG_HDR_IPV6) { + src_addr = ICE_FLOW_FIELD_IDX_IPV6_SA; + dst_addr = ICE_FLOW_FIELD_IDX_IPV6_DA; + } else { + return -EINVAL; + } + + if (l4_proto == ICE_FLOW_SEG_HDR_TCP) { + src_port = ICE_FLOW_FIELD_IDX_TCP_SRC_PORT; + dst_port = ICE_FLOW_FIELD_IDX_TCP_DST_PORT; + } else if (l4_proto == ICE_FLOW_SEG_HDR_UDP) { + src_port = ICE_FLOW_FIELD_IDX_UDP_SRC_PORT; + dst_port = ICE_FLOW_FIELD_IDX_UDP_DST_PORT; + } else { + return -EINVAL; + } + + ICE_FLOW_SET_HDRS(seg, l3_proto | l4_proto); + + /* IP source address */ + ice_flow_set_fld(seg, src_addr, ICE_FLOW_FLD_OFF_INVAL, + ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, false); + + /* IP destination address */ + ice_flow_set_fld(seg, dst_addr, ICE_FLOW_FLD_OFF_INVAL, + ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, false); + + /* Layer 4 source port */ + ice_flow_set_fld(seg, src_port, ICE_FLOW_FLD_OFF_INVAL, + ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, false); + + /* Layer 4 destination port */ + ice_flow_set_fld(seg, dst_port, ICE_FLOW_FLD_OFF_INVAL, + ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, false); + + return 0; +} + +/** + * ice_create_init_fdir_rule + * @pf: PF structure + * @flow: filter enum + * + * Return error value or 0 on success. + */ +static int +ice_create_init_fdir_rule(struct ice_pf *pf, enum ice_fltr_ptype flow) +{ + struct ice_flow_seg_info *seg, *tun_seg; + struct device *dev = ice_pf_to_dev(pf); + struct ice_hw *hw = &pf->hw; + int ret; + + /* if there is already a filter rule for kind return -EINVAL */ + if (hw->fdir_prof && hw->fdir_prof[flow] && + hw->fdir_prof[flow]->fdir_seg[0]) + return -EINVAL; + + seg = devm_kzalloc(dev, sizeof(*seg), GFP_KERNEL); + if (!seg) + return -ENOMEM; + + tun_seg = devm_kzalloc(dev, sizeof(*seg) * ICE_FD_HW_SEG_MAX, + GFP_KERNEL); + if (!tun_seg) { + devm_kfree(dev, seg); + return -ENOMEM; + } + + if (flow == ICE_FLTR_PTYPE_NONF_IPV4_TCP) + ret = ice_set_init_fdir_seg(seg, ICE_FLOW_SEG_HDR_IPV4, + ICE_FLOW_SEG_HDR_TCP); + else if (flow == ICE_FLTR_PTYPE_NONF_IPV4_UDP) + ret = ice_set_init_fdir_seg(seg, ICE_FLOW_SEG_HDR_IPV4, + ICE_FLOW_SEG_HDR_UDP); + else if (flow == ICE_FLTR_PTYPE_NONF_IPV6_TCP) + ret = ice_set_init_fdir_seg(seg, ICE_FLOW_SEG_HDR_IPV6, + ICE_FLOW_SEG_HDR_TCP); + else if (flow == ICE_FLTR_PTYPE_NONF_IPV6_UDP) + ret = ice_set_init_fdir_seg(seg, ICE_FLOW_SEG_HDR_IPV6, + ICE_FLOW_SEG_HDR_UDP); + else + ret = -EINVAL; + if (ret) + goto err_exit; + + /* add filter for outer headers */ + ret = ice_fdir_set_hw_fltr_rule(pf, seg, flow, ICE_FD_HW_SEG_NON_TUN); + if (ret) + /* could not write filter, free memory */ + goto err_exit; + + /* make tunneled filter HW entries if possible */ + memcpy(&tun_seg[1], seg, sizeof(*seg)); + ret = ice_fdir_set_hw_fltr_rule(pf, tun_seg, flow, ICE_FD_HW_SEG_TUN); + if (ret) + /* could not write tunnel filter, but outer header filter + * exists + */ + devm_kfree(dev, tun_seg); + + set_bit(flow, hw->fdir_perfect_fltr); + return ret; +err_exit: + devm_kfree(dev, tun_seg); + devm_kfree(dev, seg); + + return -EOPNOTSUPP; +} + +/** + * ice_set_fdir_ip4_seg + * @seg: flow segment for programming + * @tcp_ip4_spec: mask data from ethtool + * @l4_proto: Layer 4 protocol to program + * @perfect_fltr: only valid on success; returns true if perfect filter, + * false if not + * + * Set the mask data into the flow segment to be used to program HW + * table based on provided L4 protocol for IPv4 + */ +static int +ice_set_fdir_ip4_seg(struct ice_flow_seg_info *seg, + struct ethtool_tcpip4_spec *tcp_ip4_spec, + enum ice_flow_seg_hdr l4_proto, bool *perfect_fltr) +{ + enum ice_flow_field src_port, dst_port; + + /* make sure we don't have any empty rule */ + if (!tcp_ip4_spec->psrc && !tcp_ip4_spec->ip4src && + !tcp_ip4_spec->pdst && !tcp_ip4_spec->ip4dst) + return -EINVAL; + + /* filtering on TOS not supported */ + if (tcp_ip4_spec->tos) + return -EOPNOTSUPP; + + if (l4_proto == ICE_FLOW_SEG_HDR_TCP) { + src_port = ICE_FLOW_FIELD_IDX_TCP_SRC_PORT; + dst_port = ICE_FLOW_FIELD_IDX_TCP_DST_PORT; + } else if (l4_proto == ICE_FLOW_SEG_HDR_UDP) { + src_port = ICE_FLOW_FIELD_IDX_UDP_SRC_PORT; + dst_port = ICE_FLOW_FIELD_IDX_UDP_DST_PORT; + } else if (l4_proto == ICE_FLOW_SEG_HDR_SCTP) { + src_port = ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT; + dst_port = ICE_FLOW_FIELD_IDX_SCTP_DST_PORT; + } else { + return -EOPNOTSUPP; + } + + *perfect_fltr = true; + ICE_FLOW_SET_HDRS(seg, ICE_FLOW_SEG_HDR_IPV4 | l4_proto); + + /* IP source address */ + if (tcp_ip4_spec->ip4src == htonl(0xFFFFFFFF)) + ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV4_SA, + ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, + ICE_FLOW_FLD_OFF_INVAL, false); + else if (!tcp_ip4_spec->ip4src) + *perfect_fltr = false; + else + return -EOPNOTSUPP; + + /* IP destination address */ + if (tcp_ip4_spec->ip4dst == htonl(0xFFFFFFFF)) + ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV4_DA, + ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, + ICE_FLOW_FLD_OFF_INVAL, false); + else if (!tcp_ip4_spec->ip4dst) + *perfect_fltr = false; + else + return -EOPNOTSUPP; + + /* Layer 4 source port */ + if (tcp_ip4_spec->psrc == htons(0xFFFF)) + ice_flow_set_fld(seg, src_port, ICE_FLOW_FLD_OFF_INVAL, + ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, + false); + else if (!tcp_ip4_spec->psrc) + *perfect_fltr = false; + else + return -EOPNOTSUPP; + + /* Layer 4 destination port */ + if (tcp_ip4_spec->pdst == htons(0xFFFF)) + ice_flow_set_fld(seg, dst_port, ICE_FLOW_FLD_OFF_INVAL, + ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, + false); + else if (!tcp_ip4_spec->pdst) + *perfect_fltr = false; + else + return -EOPNOTSUPP; + + return 0; +} + +/** + * ice_set_fdir_ip4_usr_seg + * @seg: flow segment for programming + * @usr_ip4_spec: ethtool userdef packet offset + * @perfect_fltr: only valid on success; returns true if perfect filter, + * false if not + * + * Set the offset data into the flow segment to be used to program HW + * table for IPv4 + */ +static int +ice_set_fdir_ip4_usr_seg(struct ice_flow_seg_info *seg, + struct ethtool_usrip4_spec *usr_ip4_spec, + bool *perfect_fltr) +{ + /* first 4 bytes of Layer 4 header */ + if (usr_ip4_spec->l4_4_bytes) + return -EINVAL; + if (usr_ip4_spec->tos) + return -EINVAL; + if (usr_ip4_spec->ip_ver) + return -EINVAL; + /* Filtering on Layer 4 protocol not supported */ + if (usr_ip4_spec->proto) + return -EOPNOTSUPP; + /* empty rules are not valid */ + if (!usr_ip4_spec->ip4src && !usr_ip4_spec->ip4dst) + return -EINVAL; + + *perfect_fltr = true; + ICE_FLOW_SET_HDRS(seg, ICE_FLOW_SEG_HDR_IPV4); + + /* IP source address */ + if (usr_ip4_spec->ip4src == htonl(0xFFFFFFFF)) + ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV4_SA, + ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, + ICE_FLOW_FLD_OFF_INVAL, false); + else if (!usr_ip4_spec->ip4src) + *perfect_fltr = false; + else + return -EOPNOTSUPP; + + /* IP destination address */ + if (usr_ip4_spec->ip4dst == htonl(0xFFFFFFFF)) + ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV4_DA, + ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, + ICE_FLOW_FLD_OFF_INVAL, false); + else if (!usr_ip4_spec->ip4dst) + *perfect_fltr = false; + else + return -EOPNOTSUPP; + + return 0; +} + +/** + * ice_set_fdir_ip6_seg + * @seg: flow segment for programming + * @tcp_ip6_spec: mask data from ethtool + * @l4_proto: Layer 4 protocol to program + * @perfect_fltr: only valid on success; returns true if perfect filter, + * false if not + * + * Set the mask data into the flow segment to be used to program HW + * table based on provided L4 protocol for IPv6 + */ +static int +ice_set_fdir_ip6_seg(struct ice_flow_seg_info *seg, + struct ethtool_tcpip6_spec *tcp_ip6_spec, + enum ice_flow_seg_hdr l4_proto, bool *perfect_fltr) +{ + enum ice_flow_field src_port, dst_port; + + /* make sure we don't have any empty rule */ + if (!memcmp(tcp_ip6_spec->ip6src, &zero_ipv6_addr_mask, + sizeof(struct in6_addr)) && + !memcmp(tcp_ip6_spec->ip6dst, &zero_ipv6_addr_mask, + sizeof(struct in6_addr)) && + !tcp_ip6_spec->psrc && !tcp_ip6_spec->pdst) + return -EINVAL; + + /* filtering on TC not supported */ + if (tcp_ip6_spec->tclass) + return -EOPNOTSUPP; + + if (l4_proto == ICE_FLOW_SEG_HDR_TCP) { + src_port = ICE_FLOW_FIELD_IDX_TCP_SRC_PORT; + dst_port = ICE_FLOW_FIELD_IDX_TCP_DST_PORT; + } else if (l4_proto == ICE_FLOW_SEG_HDR_UDP) { + src_port = ICE_FLOW_FIELD_IDX_UDP_SRC_PORT; + dst_port = ICE_FLOW_FIELD_IDX_UDP_DST_PORT; + } else if (l4_proto == ICE_FLOW_SEG_HDR_SCTP) { + src_port = ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT; + dst_port = ICE_FLOW_FIELD_IDX_SCTP_DST_PORT; + } else { + return -EINVAL; + } + + *perfect_fltr = true; + ICE_FLOW_SET_HDRS(seg, ICE_FLOW_SEG_HDR_IPV6 | l4_proto); + + if (!memcmp(tcp_ip6_spec->ip6src, &full_ipv6_addr_mask, + sizeof(struct in6_addr))) + ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV6_SA, + ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, + ICE_FLOW_FLD_OFF_INVAL, false); + else if (!memcmp(tcp_ip6_spec->ip6src, &zero_ipv6_addr_mask, + sizeof(struct in6_addr))) + *perfect_fltr = false; + else + return -EOPNOTSUPP; + + if (!memcmp(tcp_ip6_spec->ip6dst, &full_ipv6_addr_mask, + sizeof(struct in6_addr))) + ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV6_DA, + ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, + ICE_FLOW_FLD_OFF_INVAL, false); + else if (!memcmp(tcp_ip6_spec->ip6dst, &zero_ipv6_addr_mask, + sizeof(struct in6_addr))) + *perfect_fltr = false; + else + return -EOPNOTSUPP; + + /* Layer 4 source port */ + if (tcp_ip6_spec->psrc == htons(0xFFFF)) + ice_flow_set_fld(seg, src_port, ICE_FLOW_FLD_OFF_INVAL, + ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, + false); + else if (!tcp_ip6_spec->psrc) + *perfect_fltr = false; + else + return -EOPNOTSUPP; + + /* Layer 4 destination port */ + if (tcp_ip6_spec->pdst == htons(0xFFFF)) + ice_flow_set_fld(seg, dst_port, ICE_FLOW_FLD_OFF_INVAL, + ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, + false); + else if (!tcp_ip6_spec->pdst) + *perfect_fltr = false; + else + return -EOPNOTSUPP; + + return 0; +} + +/** + * ice_set_fdir_ip6_usr_seg + * @seg: flow segment for programming + * @usr_ip6_spec: ethtool userdef packet offset + * @perfect_fltr: only valid on success; returns true if perfect filter, + * false if not + * + * Set the offset data into the flow segment to be used to program HW + * table for IPv6 + */ +static int +ice_set_fdir_ip6_usr_seg(struct ice_flow_seg_info *seg, + struct ethtool_usrip6_spec *usr_ip6_spec, + bool *perfect_fltr) +{ + /* filtering on Layer 4 bytes not supported */ + if (usr_ip6_spec->l4_4_bytes) + return -EOPNOTSUPP; + /* filtering on TC not supported */ + if (usr_ip6_spec->tclass) + return -EOPNOTSUPP; + /* filtering on Layer 4 protocol not supported */ + if (usr_ip6_spec->l4_proto) + return -EOPNOTSUPP; + /* empty rules are not valid */ + if (!memcmp(usr_ip6_spec->ip6src, &zero_ipv6_addr_mask, + sizeof(struct in6_addr)) && + !memcmp(usr_ip6_spec->ip6dst, &zero_ipv6_addr_mask, + sizeof(struct in6_addr))) + return -EINVAL; + + *perfect_fltr = true; + ICE_FLOW_SET_HDRS(seg, ICE_FLOW_SEG_HDR_IPV6); + + if (!memcmp(usr_ip6_spec->ip6src, &full_ipv6_addr_mask, + sizeof(struct in6_addr))) + ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV6_SA, + ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, + ICE_FLOW_FLD_OFF_INVAL, false); + else if (!memcmp(usr_ip6_spec->ip6src, &zero_ipv6_addr_mask, + sizeof(struct in6_addr))) + *perfect_fltr = false; + else + return -EOPNOTSUPP; + + if (!memcmp(usr_ip6_spec->ip6dst, &full_ipv6_addr_mask, + sizeof(struct in6_addr))) + ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV6_DA, + ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, + ICE_FLOW_FLD_OFF_INVAL, false); + else if (!memcmp(usr_ip6_spec->ip6dst, &zero_ipv6_addr_mask, + sizeof(struct in6_addr))) + *perfect_fltr = false; + else + return -EOPNOTSUPP; + + return 0; +} + +/** + * ice_cfg_fdir_xtrct_seq - Configure extraction sequence for the given filter + * @pf: PF structure + * @fsp: pointer to ethtool Rx flow specification + * @user: user defined data from flow specification + * + * Returns 0 on success. + */ +static int +ice_cfg_fdir_xtrct_seq(struct ice_pf *pf, struct ethtool_rx_flow_spec *fsp, + struct ice_rx_flow_userdef *user) +{ + struct ice_flow_seg_info *seg, *tun_seg; + struct device *dev = ice_pf_to_dev(pf); + enum ice_fltr_ptype fltr_idx; + struct ice_hw *hw = &pf->hw; + bool perfect_filter; + int ret; + + seg = devm_kzalloc(dev, sizeof(*seg), GFP_KERNEL); + if (!seg) + return -ENOMEM; + + tun_seg = devm_kzalloc(dev, sizeof(*seg) * ICE_FD_HW_SEG_MAX, + GFP_KERNEL); + if (!tun_seg) { + devm_kfree(dev, seg); + return -ENOMEM; + } + + switch (fsp->flow_type & ~FLOW_EXT) { + case TCP_V4_FLOW: + ret = ice_set_fdir_ip4_seg(seg, &fsp->m_u.tcp_ip4_spec, + ICE_FLOW_SEG_HDR_TCP, + &perfect_filter); + break; + case UDP_V4_FLOW: + ret = ice_set_fdir_ip4_seg(seg, &fsp->m_u.tcp_ip4_spec, + ICE_FLOW_SEG_HDR_UDP, + &perfect_filter); + break; + case SCTP_V4_FLOW: + ret = ice_set_fdir_ip4_seg(seg, &fsp->m_u.tcp_ip4_spec, + ICE_FLOW_SEG_HDR_SCTP, + &perfect_filter); + break; + case IPV4_USER_FLOW: + ret = ice_set_fdir_ip4_usr_seg(seg, &fsp->m_u.usr_ip4_spec, + &perfect_filter); + break; + case TCP_V6_FLOW: + ret = ice_set_fdir_ip6_seg(seg, &fsp->m_u.tcp_ip6_spec, + ICE_FLOW_SEG_HDR_TCP, + &perfect_filter); + break; + case UDP_V6_FLOW: + ret = ice_set_fdir_ip6_seg(seg, &fsp->m_u.tcp_ip6_spec, + ICE_FLOW_SEG_HDR_UDP, + &perfect_filter); + break; + case SCTP_V6_FLOW: + ret = ice_set_fdir_ip6_seg(seg, &fsp->m_u.tcp_ip6_spec, + ICE_FLOW_SEG_HDR_SCTP, + &perfect_filter); + break; + case IPV6_USER_FLOW: + ret = ice_set_fdir_ip6_usr_seg(seg, &fsp->m_u.usr_ip6_spec, + &perfect_filter); + break; + default: + ret = -EINVAL; + } + if (ret) + goto err_exit; + + /* tunnel segments are shifted up one. */ + memcpy(&tun_seg[1], seg, sizeof(*seg)); + + if (user && user->flex_fltr) { + perfect_filter = false; + ice_flow_add_fld_raw(seg, user->flex_offset, + ICE_FLTR_PRGM_FLEX_WORD_SIZE, + ICE_FLOW_FLD_OFF_INVAL, + ICE_FLOW_FLD_OFF_INVAL); + ice_flow_add_fld_raw(&tun_seg[1], user->flex_offset, + ICE_FLTR_PRGM_FLEX_WORD_SIZE, + ICE_FLOW_FLD_OFF_INVAL, + ICE_FLOW_FLD_OFF_INVAL); + } + + /* add filter for outer headers */ + fltr_idx = ice_ethtool_flow_to_fltr(fsp->flow_type & ~FLOW_EXT); + ret = ice_fdir_set_hw_fltr_rule(pf, seg, fltr_idx, + ICE_FD_HW_SEG_NON_TUN); + if (ret == -EEXIST) + /* Rule already exists, free memory and continue */ + devm_kfree(dev, seg); + else if (ret) + /* could not write filter, free memory */ + goto err_exit; + + /* make tunneled filter HW entries if possible */ + memcpy(&tun_seg[1], seg, sizeof(*seg)); + ret = ice_fdir_set_hw_fltr_rule(pf, tun_seg, fltr_idx, + ICE_FD_HW_SEG_TUN); + if (ret == -EEXIST) { + /* Rule already exists, free memory and count as success */ + devm_kfree(dev, tun_seg); + ret = 0; + } else if (ret) { + /* could not write tunnel filter, but outer filter exists */ + devm_kfree(dev, tun_seg); + } + + if (perfect_filter) + set_bit(fltr_idx, hw->fdir_perfect_fltr); + else + clear_bit(fltr_idx, hw->fdir_perfect_fltr); + + return ret; + +err_exit: + devm_kfree(dev, tun_seg); + devm_kfree(dev, seg); + + return -EOPNOTSUPP; +} + +/** + * ice_fdir_write_fltr - send a flow director filter to the hardware + * @pf: PF data structure + * @input: filter structure + * @add: true adds filter and false removed filter + * @is_tun: true adds inner filter on tunnel and false outer headers + * + * returns 0 on success and negative value on error + */ +int +ice_fdir_write_fltr(struct ice_pf *pf, struct ice_fdir_fltr *input, bool add, + bool is_tun) +{ + struct device *dev = ice_pf_to_dev(pf); + struct ice_hw *hw = &pf->hw; + struct ice_fltr_desc desc; + struct ice_vsi *ctrl_vsi; + enum ice_status status; + u8 *pkt, *frag_pkt; + bool has_frag; + int err; + + ctrl_vsi = ice_get_ctrl_vsi(pf); + if (!ctrl_vsi) + return -EINVAL; + + pkt = devm_kzalloc(dev, ICE_FDIR_MAX_RAW_PKT_SIZE, GFP_KERNEL); + if (!pkt) + return -ENOMEM; + frag_pkt = devm_kzalloc(dev, ICE_FDIR_MAX_RAW_PKT_SIZE, GFP_KERNEL); + if (!frag_pkt) { + err = -ENOMEM; + goto err_free; + } + + ice_fdir_get_prgm_desc(hw, input, &desc, add); + status = ice_fdir_get_gen_prgm_pkt(hw, input, pkt, false, is_tun); + if (status) { + err = ice_status_to_errno(status); + goto err_free_all; + } + err = ice_prgm_fdir_fltr(ctrl_vsi, &desc, pkt); + if (err) + goto err_free_all; + + /* repeat for fragment packet */ + has_frag = ice_fdir_has_frag(input->flow_type); + if (has_frag) { + /* does not return error */ + ice_fdir_get_prgm_desc(hw, input, &desc, add); + status = ice_fdir_get_gen_prgm_pkt(hw, input, frag_pkt, true, + is_tun); + if (status) { + err = ice_status_to_errno(status); + goto err_frag; + } + err = ice_prgm_fdir_fltr(ctrl_vsi, &desc, frag_pkt); + if (err) + goto err_frag; + } else { + devm_kfree(dev, frag_pkt); + } + + return 0; + +err_free_all: + devm_kfree(dev, frag_pkt); +err_free: + devm_kfree(dev, pkt); + return err; + +err_frag: + devm_kfree(dev, frag_pkt); + return err; +} + +/** + * ice_fdir_write_all_fltr - send a flow director filter to the hardware + * @pf: PF data structure + * @input: filter structure + * @add: true adds filter and false removed filter + * + * returns 0 on success and negative value on error + */ +static int +ice_fdir_write_all_fltr(struct ice_pf *pf, struct ice_fdir_fltr *input, + bool add) +{ + u16 port_num; + int tun; + + for (tun = 0; tun < ICE_FD_HW_SEG_MAX; tun++) { + bool is_tun = tun == ICE_FD_HW_SEG_TUN; + int err; + + if (is_tun && !ice_get_open_tunnel_port(&pf->hw, TNL_ALL, + &port_num)) + continue; + err = ice_fdir_write_fltr(pf, input, add, is_tun); + if (err) + return err; + } + return 0; +} + +/** + * ice_fdir_replay_fltrs - replay filters from the HW filter list + * @pf: board private structure + */ +void ice_fdir_replay_fltrs(struct ice_pf *pf) +{ + struct ice_fdir_fltr *f_rule; + struct ice_hw *hw = &pf->hw; + + list_for_each_entry(f_rule, &hw->fdir_list_head, fltr_node) { + int err = ice_fdir_write_all_fltr(pf, f_rule, true); + + if (err) + dev_dbg(ice_pf_to_dev(pf), "Flow Director error %d, could not reprogram filter %d\n", + err, f_rule->fltr_id); + } +} + +/** + * ice_fdir_create_dflt_rules - create default perfect filters + * @pf: PF data structure + * + * Returns 0 for success or error. + */ +int ice_fdir_create_dflt_rules(struct ice_pf *pf) +{ + int err; + + /* Create perfect TCP and UDP rules in hardware. */ + err = ice_create_init_fdir_rule(pf, ICE_FLTR_PTYPE_NONF_IPV4_TCP); + if (err) + return err; + + err = ice_create_init_fdir_rule(pf, ICE_FLTR_PTYPE_NONF_IPV4_UDP); + if (err) + return err; + + err = ice_create_init_fdir_rule(pf, ICE_FLTR_PTYPE_NONF_IPV6_TCP); + if (err) + return err; + + err = ice_create_init_fdir_rule(pf, ICE_FLTR_PTYPE_NONF_IPV6_UDP); + + return err; +} + +/** + * ice_vsi_manage_fdir - turn on/off flow director + * @vsi: the VSI being changed + * @ena: boolean value indicating if this is an enable or disable request + */ +void ice_vsi_manage_fdir(struct ice_vsi *vsi, bool ena) +{ + struct ice_fdir_fltr *f_rule, *tmp; + struct ice_pf *pf = vsi->back; + struct ice_hw *hw = &pf->hw; + enum ice_fltr_ptype flow; + + if (ena) { + set_bit(ICE_FLAG_FD_ENA, pf->flags); + ice_fdir_create_dflt_rules(pf); + return; + } + + mutex_lock(&hw->fdir_fltr_lock); + if (!test_and_clear_bit(ICE_FLAG_FD_ENA, pf->flags)) + goto release_lock; + list_for_each_entry_safe(f_rule, tmp, &hw->fdir_list_head, fltr_node) { + /* ignore return value */ + ice_fdir_write_all_fltr(pf, f_rule, false); + ice_fdir_update_cntrs(hw, f_rule->flow_type, false); + list_del(&f_rule->fltr_node); + devm_kfree(ice_hw_to_dev(hw), f_rule); + } + + if (hw->fdir_prof) + for (flow = ICE_FLTR_PTYPE_NONF_NONE; flow < ICE_FLTR_PTYPE_MAX; + flow++) + if (hw->fdir_prof[flow]) + ice_fdir_rem_flow(hw, ICE_BLK_FD, flow); + +release_lock: + mutex_unlock(&hw->fdir_fltr_lock); +} + +/** + * ice_fdir_do_rem_flow - delete flow and possibly add perfect flow + * @pf: PF structure + * @flow_type: FDir flow type to release + */ +static void +ice_fdir_do_rem_flow(struct ice_pf *pf, enum ice_fltr_ptype flow_type) +{ + struct ice_hw *hw = &pf->hw; + bool need_perfect = false; + + if (flow_type == ICE_FLTR_PTYPE_NONF_IPV4_TCP || + flow_type == ICE_FLTR_PTYPE_NONF_IPV4_UDP || + flow_type == ICE_FLTR_PTYPE_NONF_IPV6_TCP || + flow_type == ICE_FLTR_PTYPE_NONF_IPV6_UDP) + need_perfect = true; + + if (need_perfect && test_bit(flow_type, hw->fdir_perfect_fltr)) + return; + + ice_fdir_rem_flow(hw, ICE_BLK_FD, flow_type); + if (need_perfect) + ice_create_init_fdir_rule(pf, flow_type); +} + +/** + * ice_fdir_update_list_entry - add or delete a filter from the filter list + * @pf: PF structure + * @input: filter structure + * @fltr_idx: ethtool index of filter to modify + * + * returns 0 on success and negative on errors + */ +static int +ice_fdir_update_list_entry(struct ice_pf *pf, struct ice_fdir_fltr *input, + int fltr_idx) +{ + struct ice_fdir_fltr *old_fltr; + struct ice_hw *hw = &pf->hw; + int err = -ENOENT; + + /* Do not update filters during reset */ + if (ice_is_reset_in_progress(pf->state)) + return -EBUSY; + + old_fltr = ice_fdir_find_fltr_by_idx(hw, fltr_idx); + if (old_fltr) { + err = ice_fdir_write_all_fltr(pf, old_fltr, false); + if (err) + return err; + ice_fdir_update_cntrs(hw, old_fltr->flow_type, false); + if (!input && !hw->fdir_fltr_cnt[old_fltr->flow_type]) + /* we just deleted the last filter of flow_type so we + * should also delete the HW filter info. + */ + ice_fdir_do_rem_flow(pf, old_fltr->flow_type); + list_del(&old_fltr->fltr_node); + devm_kfree(ice_hw_to_dev(hw), old_fltr); + } + if (!input) + return err; + ice_fdir_list_add_fltr(hw, input); + ice_fdir_update_cntrs(hw, input->flow_type, true); + return 0; +} + +/** + * ice_del_fdir_ethtool - delete Flow Director filter + * @vsi: pointer to target VSI + * @cmd: command to add or delete Flow Director filter + * + * Returns 0 on success and negative values for failure + */ +int ice_del_fdir_ethtool(struct ice_vsi *vsi, struct ethtool_rxnfc *cmd) +{ + struct ethtool_rx_flow_spec *fsp = + (struct ethtool_rx_flow_spec *)&cmd->fs; + struct ice_pf *pf = vsi->back; + struct ice_hw *hw = &pf->hw; + int val; + + if (!test_bit(ICE_FLAG_FD_ENA, pf->flags)) + return -EOPNOTSUPP; + + /* Do not delete filters during reset */ + if (ice_is_reset_in_progress(pf->state)) { + dev_err(ice_pf_to_dev(pf), "Device is resetting - deleting Flow Director filters not supported during reset\n"); + return -EBUSY; + } + + if (test_bit(__ICE_FD_FLUSH_REQ, pf->state)) + return -EBUSY; + + mutex_lock(&hw->fdir_fltr_lock); + val = ice_fdir_update_list_entry(pf, NULL, fsp->location); + mutex_unlock(&hw->fdir_fltr_lock); + + return val; +} + +/** + * ice_set_fdir_input_set - Set the input set for Flow Director + * @vsi: pointer to target VSI + * @fsp: pointer to ethtool Rx flow specification + * @input: filter structure + */ +static int +ice_set_fdir_input_set(struct ice_vsi *vsi, struct ethtool_rx_flow_spec *fsp, + struct ice_fdir_fltr *input) +{ + u16 dest_vsi, q_index = 0; + struct ice_pf *pf; + struct ice_hw *hw; + int flow_type; + u8 dest_ctl; + + if (!vsi || !fsp || !input) + return -EINVAL; + + pf = vsi->back; + hw = &pf->hw; + + dest_vsi = vsi->idx; + if (fsp->ring_cookie == RX_CLS_FLOW_DISC) { + dest_ctl = ICE_FLTR_PRGM_DESC_DEST_DROP_PKT; + } else { + u32 ring = ethtool_get_flow_spec_ring(fsp->ring_cookie); + u8 vf = ethtool_get_flow_spec_ring_vf(fsp->ring_cookie); + + if (vf) { + dev_err(ice_pf_to_dev(pf), "Failed to add filter. Flow director filters are not supported on VF queues.\n"); + return -EINVAL; + } + + if (ring >= vsi->num_rxq) + return -EINVAL; + + dest_ctl = ICE_FLTR_PRGM_DESC_DEST_DIRECT_PKT_QINDEX; + q_index = ring; + } + + input->fltr_id = fsp->location; + input->q_index = q_index; + flow_type = fsp->flow_type & ~FLOW_EXT; + + input->dest_vsi = dest_vsi; + input->dest_ctl = dest_ctl; + input->fltr_status = ICE_FLTR_PRGM_DESC_FD_STATUS_FD_ID; + input->cnt_index = ICE_FD_SB_STAT_IDX(hw->fd_ctr_base); + input->flow_type = ice_ethtool_flow_to_fltr(flow_type); + + if (fsp->flow_type & FLOW_EXT) { + memcpy(input->ext_data.usr_def, fsp->h_ext.data, + sizeof(input->ext_data.usr_def)); + input->ext_data.vlan_type = fsp->h_ext.vlan_etype; + input->ext_data.vlan_tag = fsp->h_ext.vlan_tci; + memcpy(input->ext_mask.usr_def, fsp->m_ext.data, + sizeof(input->ext_mask.usr_def)); + input->ext_mask.vlan_type = fsp->m_ext.vlan_etype; + input->ext_mask.vlan_tag = fsp->m_ext.vlan_tci; + } + + switch (flow_type) { + case TCP_V4_FLOW: + case UDP_V4_FLOW: + case SCTP_V4_FLOW: + input->ip.v4.dst_port = fsp->h_u.tcp_ip4_spec.pdst; + input->ip.v4.src_port = fsp->h_u.tcp_ip4_spec.psrc; + input->ip.v4.dst_ip = fsp->h_u.tcp_ip4_spec.ip4dst; + input->ip.v4.src_ip = fsp->h_u.tcp_ip4_spec.ip4src; + input->mask.v4.dst_port = fsp->m_u.tcp_ip4_spec.pdst; + input->mask.v4.src_port = fsp->m_u.tcp_ip4_spec.psrc; + input->mask.v4.dst_ip = fsp->m_u.tcp_ip4_spec.ip4dst; + input->mask.v4.src_ip = fsp->m_u.tcp_ip4_spec.ip4src; + break; + case IPV4_USER_FLOW: + input->ip.v4.dst_ip = fsp->h_u.usr_ip4_spec.ip4dst; + input->ip.v4.src_ip = fsp->h_u.usr_ip4_spec.ip4src; + input->ip.v4.l4_header = fsp->h_u.usr_ip4_spec.l4_4_bytes; + input->ip.v4.proto = fsp->h_u.usr_ip4_spec.proto; + input->ip.v4.ip_ver = fsp->h_u.usr_ip4_spec.ip_ver; + input->ip.v4.tos = fsp->h_u.usr_ip4_spec.tos; + input->mask.v4.dst_ip = fsp->m_u.usr_ip4_spec.ip4dst; + input->mask.v4.src_ip = fsp->m_u.usr_ip4_spec.ip4src; + input->mask.v4.l4_header = fsp->m_u.usr_ip4_spec.l4_4_bytes; + input->mask.v4.proto = fsp->m_u.usr_ip4_spec.proto; + input->mask.v4.ip_ver = fsp->m_u.usr_ip4_spec.ip_ver; + input->mask.v4.tos = fsp->m_u.usr_ip4_spec.tos; + break; + case TCP_V6_FLOW: + case UDP_V6_FLOW: + case SCTP_V6_FLOW: + memcpy(input->ip.v6.dst_ip, fsp->h_u.usr_ip6_spec.ip6dst, + sizeof(struct in6_addr)); + memcpy(input->ip.v6.src_ip, fsp->h_u.usr_ip6_spec.ip6src, + sizeof(struct in6_addr)); + input->ip.v6.dst_port = fsp->h_u.tcp_ip6_spec.pdst; + input->ip.v6.src_port = fsp->h_u.tcp_ip6_spec.psrc; + input->ip.v6.tc = fsp->h_u.tcp_ip6_spec.tclass; + memcpy(input->mask.v6.dst_ip, fsp->m_u.tcp_ip6_spec.ip6dst, + sizeof(struct in6_addr)); + memcpy(input->mask.v6.src_ip, fsp->m_u.tcp_ip6_spec.ip6src, + sizeof(struct in6_addr)); + input->mask.v6.dst_port = fsp->m_u.tcp_ip6_spec.pdst; + input->mask.v6.src_port = fsp->m_u.tcp_ip6_spec.psrc; + input->mask.v6.tc = fsp->m_u.tcp_ip6_spec.tclass; + break; + case IPV6_USER_FLOW: + memcpy(input->ip.v6.dst_ip, fsp->h_u.usr_ip6_spec.ip6dst, + sizeof(struct in6_addr)); + memcpy(input->ip.v6.src_ip, fsp->h_u.usr_ip6_spec.ip6src, + sizeof(struct in6_addr)); + input->ip.v6.l4_header = fsp->h_u.usr_ip6_spec.l4_4_bytes; + input->ip.v6.tc = fsp->h_u.usr_ip6_spec.tclass; + input->ip.v6.proto = fsp->h_u.usr_ip6_spec.l4_proto; + memcpy(input->mask.v6.dst_ip, fsp->m_u.usr_ip6_spec.ip6dst, + sizeof(struct in6_addr)); + memcpy(input->mask.v6.src_ip, fsp->m_u.usr_ip6_spec.ip6src, + sizeof(struct in6_addr)); + input->mask.v6.l4_header = fsp->m_u.usr_ip6_spec.l4_4_bytes; + input->mask.v6.tc = fsp->m_u.usr_ip6_spec.tclass; + input->mask.v6.proto = fsp->m_u.usr_ip6_spec.l4_proto; + break; + default: + /* not doing un-parsed flow types */ + return -EINVAL; + } + + return 0; +} + +/** + * ice_add_fdir_ethtool - Add/Remove Flow Director filter + * @vsi: pointer to target VSI + * @cmd: command to add or delete Flow Director filter + * + * Returns 0 on success and negative values for failure + */ +int ice_add_fdir_ethtool(struct ice_vsi *vsi, struct ethtool_rxnfc *cmd) +{ + struct ice_rx_flow_userdef userdata; + struct ethtool_rx_flow_spec *fsp; + struct ice_fdir_fltr *input; + struct device *dev; + struct ice_pf *pf; + struct ice_hw *hw; + int fltrs_needed; + u16 tunnel_port; + int ret; + + if (!vsi) + return -EINVAL; + + pf = vsi->back; + hw = &pf->hw; + dev = ice_pf_to_dev(pf); + + if (!test_bit(ICE_FLAG_FD_ENA, pf->flags)) + return -EOPNOTSUPP; + + /* Do not program filters during reset */ + if (ice_is_reset_in_progress(pf->state)) { + dev_err(dev, "Device is resetting - adding Flow Director filters not supported during reset\n"); + return -EBUSY; + } + + fsp = (struct ethtool_rx_flow_spec *)&cmd->fs; + + if (ice_parse_rx_flow_user_data(fsp, &userdata)) + return -EINVAL; + + if (fsp->flow_type & FLOW_MAC_EXT) + return -EINVAL; + + ret = ice_cfg_fdir_xtrct_seq(pf, fsp, &userdata); + if (ret) + return ret; + + if (fsp->location >= ice_get_fdir_cnt_all(hw)) { + dev_err(dev, "Failed to add filter. The maximum number of flow director filters has been reached.\n"); + return -ENOSPC; + } + + /* return error if not an update and no available filters */ + fltrs_needed = ice_get_open_tunnel_port(hw, TNL_ALL, &tunnel_port) ? + 2 : 1; + if (!ice_fdir_find_fltr_by_idx(hw, fsp->location) && + ice_fdir_num_avail_fltr(hw, pf->vsi[vsi->idx]) < fltrs_needed) { + dev_err(dev, "Failed to add filter. The maximum number of flow director filters has been reached.\n"); + return -ENOSPC; + } + + input = devm_kzalloc(dev, sizeof(*input), GFP_KERNEL); + if (!input) + return -ENOMEM; + + ret = ice_set_fdir_input_set(vsi, fsp, input); + if (ret) + goto free_input; + + mutex_lock(&hw->fdir_fltr_lock); + if (ice_fdir_is_dup_fltr(hw, input)) { + ret = -EINVAL; + goto release_lock; + } + + if (userdata.flex_fltr) { + input->flex_fltr = true; + input->flex_word = cpu_to_be16(userdata.flex_word); + input->flex_offset = userdata.flex_offset; + } + + /* input struct is added to the HW filter list */ + ice_fdir_update_list_entry(pf, input, fsp->location); + + ret = ice_fdir_write_all_fltr(pf, input, true); + if (ret) + goto remove_sw_rule; + + goto release_lock; + +remove_sw_rule: + ice_fdir_update_cntrs(hw, input->flow_type, false); + list_del(&input->fltr_node); +release_lock: + mutex_unlock(&hw->fdir_fltr_lock); +free_input: + if (ret) + devm_kfree(dev, input); + + return ret; +} diff --git a/drivers/net/ethernet/intel/ice/ice_fdir.c b/drivers/net/ethernet/intel/ice/ice_fdir.c new file mode 100644 index 000000000000..6834df14332f --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_fdir.c @@ -0,0 +1,840 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2018-2020, Intel Corporation. */ + +#include "ice_common.h" + +/* These are training packet headers used to program flow director filters. */ +static const u8 ice_fdir_tcpv4_pkt[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x45, 0x00, + 0x00, 0x28, 0x00, 0x01, 0x00, 0x00, 0x40, 0x06, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +static const u8 ice_fdir_udpv4_pkt[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x45, 0x00, + 0x00, 0x1C, 0x00, 0x00, 0x40, 0x00, 0x40, 0x11, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, +}; + +static const u8 ice_fdir_sctpv4_pkt[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x45, 0x00, + 0x00, 0x20, 0x00, 0x00, 0x40, 0x00, 0x40, 0x84, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +static const u8 ice_fdir_ipv4_pkt[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x45, 0x00, + 0x00, 0x14, 0x00, 0x00, 0x40, 0x00, 0x40, 0x10, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00 +}; + +static const u8 ice_fdir_tcpv6_pkt[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x86, 0xDD, 0x60, 0x00, + 0x00, 0x00, 0x00, 0x14, 0x06, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x50, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x00, 0x00, +}; + +static const u8 ice_fdir_udpv6_pkt[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x86, 0xDD, 0x60, 0x00, + 0x00, 0x00, 0x00, 0x08, 0x11, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, +}; + +static const u8 ice_fdir_sctpv6_pkt[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x86, 0xDD, 0x60, 0x00, + 0x00, 0x00, 0x00, 0x0C, 0x84, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, +}; + +static const u8 ice_fdir_ipv6_pkt[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x86, 0xDD, 0x60, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3B, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +static const u8 ice_fdir_tcp4_tun_pkt[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x45, 0x00, + 0x00, 0x5a, 0x00, 0x00, 0x40, 0x00, 0x40, 0x11, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, + 0x45, 0x00, 0x00, 0x28, 0x00, 0x00, 0x40, 0x00, + 0x40, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x50, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +static const u8 ice_fdir_udp4_tun_pkt[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x45, 0x00, + 0x00, 0x4e, 0x00, 0x00, 0x40, 0x00, 0x40, 0x11, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, + 0x45, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x40, 0x00, + 0x40, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, +}; + +static const u8 ice_fdir_sctp4_tun_pkt[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x45, 0x00, + 0x00, 0x52, 0x00, 0x00, 0x40, 0x00, 0x40, 0x11, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, + 0x45, 0x00, 0x00, 0x20, 0x00, 0x01, 0x00, 0x00, + 0x40, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +static const u8 ice_fdir_ip4_tun_pkt[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x45, 0x00, + 0x00, 0x46, 0x00, 0x00, 0x40, 0x00, 0x40, 0x11, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, + 0x45, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, +}; + +static const u8 ice_fdir_tcp6_tun_pkt[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x45, 0x00, + 0x00, 0x6e, 0x00, 0x00, 0x40, 0x00, 0x40, 0x11, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x86, 0xdd, + 0x60, 0x00, 0x00, 0x00, 0x00, 0x14, 0x06, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x50, 0x00, 0x20, 0x00, + 0x00, 0x00, 0x00, 0x00, +}; + +static const u8 ice_fdir_udp6_tun_pkt[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x45, 0x00, + 0x00, 0x62, 0x00, 0x00, 0x40, 0x00, 0x40, 0x11, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x86, 0xdd, + 0x60, 0x00, 0x00, 0x00, 0x00, 0x08, 0x11, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +static const u8 ice_fdir_sctp6_tun_pkt[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x45, 0x00, + 0x00, 0x66, 0x00, 0x00, 0x40, 0x00, 0x40, 0x11, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x86, 0xdd, + 0x60, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x84, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, +}; + +static const u8 ice_fdir_ip6_tun_pkt[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x45, 0x00, + 0x00, 0x5a, 0x00, 0x00, 0x40, 0x00, 0x40, 0x11, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x86, 0xdd, + 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3b, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +/* Flow Director no-op training packet table */ +static const struct ice_fdir_base_pkt ice_fdir_pkt[] = { + { + ICE_FLTR_PTYPE_NONF_IPV4_TCP, + sizeof(ice_fdir_tcpv4_pkt), ice_fdir_tcpv4_pkt, + sizeof(ice_fdir_tcp4_tun_pkt), ice_fdir_tcp4_tun_pkt, + }, + { + ICE_FLTR_PTYPE_NONF_IPV4_UDP, + sizeof(ice_fdir_udpv4_pkt), ice_fdir_udpv4_pkt, + sizeof(ice_fdir_udp4_tun_pkt), ice_fdir_udp4_tun_pkt, + }, + { + ICE_FLTR_PTYPE_NONF_IPV4_SCTP, + sizeof(ice_fdir_sctpv4_pkt), ice_fdir_sctpv4_pkt, + sizeof(ice_fdir_sctp4_tun_pkt), ice_fdir_sctp4_tun_pkt, + }, + { + ICE_FLTR_PTYPE_NONF_IPV4_OTHER, + sizeof(ice_fdir_ipv4_pkt), ice_fdir_ipv4_pkt, + sizeof(ice_fdir_ip4_tun_pkt), ice_fdir_ip4_tun_pkt, + }, + { + ICE_FLTR_PTYPE_NONF_IPV6_TCP, + sizeof(ice_fdir_tcpv6_pkt), ice_fdir_tcpv6_pkt, + sizeof(ice_fdir_tcp6_tun_pkt), ice_fdir_tcp6_tun_pkt, + }, + { + ICE_FLTR_PTYPE_NONF_IPV6_UDP, + sizeof(ice_fdir_udpv6_pkt), ice_fdir_udpv6_pkt, + sizeof(ice_fdir_udp6_tun_pkt), ice_fdir_udp6_tun_pkt, + }, + { + ICE_FLTR_PTYPE_NONF_IPV6_SCTP, + sizeof(ice_fdir_sctpv6_pkt), ice_fdir_sctpv6_pkt, + sizeof(ice_fdir_sctp6_tun_pkt), ice_fdir_sctp6_tun_pkt, + }, + { + ICE_FLTR_PTYPE_NONF_IPV6_OTHER, + sizeof(ice_fdir_ipv6_pkt), ice_fdir_ipv6_pkt, + sizeof(ice_fdir_ip6_tun_pkt), ice_fdir_ip6_tun_pkt, + }, +}; + +#define ICE_FDIR_NUM_PKT ARRAY_SIZE(ice_fdir_pkt) + +/** + * ice_set_dflt_val_fd_desc + * @fd_fltr_ctx: pointer to fd filter descriptor + */ +static void ice_set_dflt_val_fd_desc(struct ice_fd_fltr_desc_ctx *fd_fltr_ctx) +{ + fd_fltr_ctx->comp_q = ICE_FXD_FLTR_QW0_COMP_Q_ZERO; + fd_fltr_ctx->comp_report = ICE_FXD_FLTR_QW0_COMP_REPORT_SW_FAIL; + fd_fltr_ctx->fd_space = ICE_FXD_FLTR_QW0_FD_SPACE_GUAR_BEST; + fd_fltr_ctx->cnt_ena = ICE_FXD_FLTR_QW0_STAT_ENA_PKTS; + fd_fltr_ctx->evict_ena = ICE_FXD_FLTR_QW0_EVICT_ENA_TRUE; + fd_fltr_ctx->toq = ICE_FXD_FLTR_QW0_TO_Q_EQUALS_QINDEX; + fd_fltr_ctx->toq_prio = ICE_FXD_FLTR_QW0_TO_Q_PRIO1; + fd_fltr_ctx->dpu_recipe = ICE_FXD_FLTR_QW0_DPU_RECIPE_DFLT; + fd_fltr_ctx->drop = ICE_FXD_FLTR_QW0_DROP_NO; + fd_fltr_ctx->flex_prio = ICE_FXD_FLTR_QW0_FLEX_PRI_NONE; + fd_fltr_ctx->flex_mdid = ICE_FXD_FLTR_QW0_FLEX_MDID0; + fd_fltr_ctx->flex_val = ICE_FXD_FLTR_QW0_FLEX_VAL0; + fd_fltr_ctx->dtype = ICE_TX_DESC_DTYPE_FLTR_PROG; + fd_fltr_ctx->desc_prof_prio = ICE_FXD_FLTR_QW1_PROF_PRIO_ZERO; + fd_fltr_ctx->desc_prof = ICE_FXD_FLTR_QW1_PROF_ZERO; + fd_fltr_ctx->swap = ICE_FXD_FLTR_QW1_SWAP_SET; + fd_fltr_ctx->fdid_prio = ICE_FXD_FLTR_QW1_FDID_PRI_ONE; + fd_fltr_ctx->fdid_mdid = ICE_FXD_FLTR_QW1_FDID_MDID_FD; + fd_fltr_ctx->fdid = ICE_FXD_FLTR_QW1_FDID_ZERO; +} + +/** + * ice_set_fd_desc_val + * @ctx: pointer to fd filter descriptor context + * @fdir_desc: populated with fd filter descriptor values + */ +static void +ice_set_fd_desc_val(struct ice_fd_fltr_desc_ctx *ctx, + struct ice_fltr_desc *fdir_desc) +{ + u64 qword; + + /* prep QW0 of FD filter programming desc */ + qword = ((u64)ctx->qindex << ICE_FXD_FLTR_QW0_QINDEX_S) & + ICE_FXD_FLTR_QW0_QINDEX_M; + qword |= ((u64)ctx->comp_q << ICE_FXD_FLTR_QW0_COMP_Q_S) & + ICE_FXD_FLTR_QW0_COMP_Q_M; + qword |= ((u64)ctx->comp_report << ICE_FXD_FLTR_QW0_COMP_REPORT_S) & + ICE_FXD_FLTR_QW0_COMP_REPORT_M; + qword |= ((u64)ctx->fd_space << ICE_FXD_FLTR_QW0_FD_SPACE_S) & + ICE_FXD_FLTR_QW0_FD_SPACE_M; + qword |= ((u64)ctx->cnt_index << ICE_FXD_FLTR_QW0_STAT_CNT_S) & + ICE_FXD_FLTR_QW0_STAT_CNT_M; + qword |= ((u64)ctx->cnt_ena << ICE_FXD_FLTR_QW0_STAT_ENA_S) & + ICE_FXD_FLTR_QW0_STAT_ENA_M; + qword |= ((u64)ctx->evict_ena << ICE_FXD_FLTR_QW0_EVICT_ENA_S) & + ICE_FXD_FLTR_QW0_EVICT_ENA_M; + qword |= ((u64)ctx->toq << ICE_FXD_FLTR_QW0_TO_Q_S) & + ICE_FXD_FLTR_QW0_TO_Q_M; + qword |= ((u64)ctx->toq_prio << ICE_FXD_FLTR_QW0_TO_Q_PRI_S) & + ICE_FXD_FLTR_QW0_TO_Q_PRI_M; + qword |= ((u64)ctx->dpu_recipe << ICE_FXD_FLTR_QW0_DPU_RECIPE_S) & + ICE_FXD_FLTR_QW0_DPU_RECIPE_M; + qword |= ((u64)ctx->drop << ICE_FXD_FLTR_QW0_DROP_S) & + ICE_FXD_FLTR_QW0_DROP_M; + qword |= ((u64)ctx->flex_prio << ICE_FXD_FLTR_QW0_FLEX_PRI_S) & + ICE_FXD_FLTR_QW0_FLEX_PRI_M; + qword |= ((u64)ctx->flex_mdid << ICE_FXD_FLTR_QW0_FLEX_MDID_S) & + ICE_FXD_FLTR_QW0_FLEX_MDID_M; + qword |= ((u64)ctx->flex_val << ICE_FXD_FLTR_QW0_FLEX_VAL_S) & + ICE_FXD_FLTR_QW0_FLEX_VAL_M; + fdir_desc->qidx_compq_space_stat = cpu_to_le64(qword); + + /* prep QW1 of FD filter programming desc */ + qword = ((u64)ctx->dtype << ICE_FXD_FLTR_QW1_DTYPE_S) & + ICE_FXD_FLTR_QW1_DTYPE_M; + qword |= ((u64)ctx->pcmd << ICE_FXD_FLTR_QW1_PCMD_S) & + ICE_FXD_FLTR_QW1_PCMD_M; + qword |= ((u64)ctx->desc_prof_prio << ICE_FXD_FLTR_QW1_PROF_PRI_S) & + ICE_FXD_FLTR_QW1_PROF_PRI_M; + qword |= ((u64)ctx->desc_prof << ICE_FXD_FLTR_QW1_PROF_S) & + ICE_FXD_FLTR_QW1_PROF_M; + qword |= ((u64)ctx->fd_vsi << ICE_FXD_FLTR_QW1_FD_VSI_S) & + ICE_FXD_FLTR_QW1_FD_VSI_M; + qword |= ((u64)ctx->swap << ICE_FXD_FLTR_QW1_SWAP_S) & + ICE_FXD_FLTR_QW1_SWAP_M; + qword |= ((u64)ctx->fdid_prio << ICE_FXD_FLTR_QW1_FDID_PRI_S) & + ICE_FXD_FLTR_QW1_FDID_PRI_M; + qword |= ((u64)ctx->fdid_mdid << ICE_FXD_FLTR_QW1_FDID_MDID_S) & + ICE_FXD_FLTR_QW1_FDID_MDID_M; + qword |= ((u64)ctx->fdid << ICE_FXD_FLTR_QW1_FDID_S) & + ICE_FXD_FLTR_QW1_FDID_M; + fdir_desc->dtype_cmd_vsi_fdid = cpu_to_le64(qword); +} + +/** + * ice_fdir_get_prgm_desc - set a fdir descriptor from a fdir filter struct + * @hw: pointer to the hardware structure + * @input: filter + * @fdesc: filter descriptor + * @add: if add is true, this is an add operation, false implies delete + */ +void +ice_fdir_get_prgm_desc(struct ice_hw *hw, struct ice_fdir_fltr *input, + struct ice_fltr_desc *fdesc, bool add) +{ + struct ice_fd_fltr_desc_ctx fdir_fltr_ctx = { 0 }; + + /* set default context info */ + ice_set_dflt_val_fd_desc(&fdir_fltr_ctx); + + /* change sideband filtering values */ + fdir_fltr_ctx.fdid = input->fltr_id; + if (input->dest_ctl == ICE_FLTR_PRGM_DESC_DEST_DROP_PKT) { + fdir_fltr_ctx.drop = ICE_FXD_FLTR_QW0_DROP_YES; + fdir_fltr_ctx.qindex = 0; + } else { + fdir_fltr_ctx.drop = ICE_FXD_FLTR_QW0_DROP_NO; + fdir_fltr_ctx.qindex = input->q_index; + } + fdir_fltr_ctx.cnt_ena = ICE_FXD_FLTR_QW0_STAT_ENA_PKTS; + fdir_fltr_ctx.cnt_index = input->cnt_index; + fdir_fltr_ctx.fd_vsi = ice_get_hw_vsi_num(hw, input->dest_vsi); + fdir_fltr_ctx.evict_ena = ICE_FXD_FLTR_QW0_EVICT_ENA_FALSE; + fdir_fltr_ctx.toq_prio = 3; + fdir_fltr_ctx.pcmd = add ? ICE_FXD_FLTR_QW1_PCMD_ADD : + ICE_FXD_FLTR_QW1_PCMD_REMOVE; + fdir_fltr_ctx.swap = ICE_FXD_FLTR_QW1_SWAP_NOT_SET; + fdir_fltr_ctx.comp_q = ICE_FXD_FLTR_QW0_COMP_Q_ZERO; + fdir_fltr_ctx.comp_report = ICE_FXD_FLTR_QW0_COMP_REPORT_SW_FAIL; + fdir_fltr_ctx.fdid_prio = 3; + fdir_fltr_ctx.desc_prof = 1; + fdir_fltr_ctx.desc_prof_prio = 3; + ice_set_fd_desc_val(&fdir_fltr_ctx, fdesc); +} + +/** + * ice_alloc_fd_res_cntr - obtain counter resource for FD type + * @hw: pointer to the hardware structure + * @cntr_id: returns counter index + */ +enum ice_status ice_alloc_fd_res_cntr(struct ice_hw *hw, u16 *cntr_id) +{ + return ice_alloc_res_cntr(hw, ICE_AQC_RES_TYPE_FDIR_COUNTER_BLOCK, + ICE_AQC_RES_TYPE_FLAG_DEDICATED, 1, cntr_id); +} + +/** + * ice_free_fd_res_cntr - Free counter resource for FD type + * @hw: pointer to the hardware structure + * @cntr_id: counter index to be freed + */ +enum ice_status ice_free_fd_res_cntr(struct ice_hw *hw, u16 cntr_id) +{ + return ice_free_res_cntr(hw, ICE_AQC_RES_TYPE_FDIR_COUNTER_BLOCK, + ICE_AQC_RES_TYPE_FLAG_DEDICATED, 1, cntr_id); +} + +/** + * ice_alloc_fd_guar_item - allocate resource for FD guaranteed entries + * @hw: pointer to the hardware structure + * @cntr_id: returns counter index + * @num_fltr: number of filter entries to be allocated + */ +enum ice_status +ice_alloc_fd_guar_item(struct ice_hw *hw, u16 *cntr_id, u16 num_fltr) +{ + return ice_alloc_res_cntr(hw, ICE_AQC_RES_TYPE_FDIR_GUARANTEED_ENTRIES, + ICE_AQC_RES_TYPE_FLAG_DEDICATED, num_fltr, + cntr_id); +} + +/** + * ice_alloc_fd_shrd_item - allocate resource for flow director shared entries + * @hw: pointer to the hardware structure + * @cntr_id: returns counter index + * @num_fltr: number of filter entries to be allocated + */ +enum ice_status +ice_alloc_fd_shrd_item(struct ice_hw *hw, u16 *cntr_id, u16 num_fltr) +{ + return ice_alloc_res_cntr(hw, ICE_AQC_RES_TYPE_FDIR_SHARED_ENTRIES, + ICE_AQC_RES_TYPE_FLAG_DEDICATED, num_fltr, + cntr_id); +} + +/** + * ice_get_fdir_cnt_all - get the number of Flow Director filters + * @hw: hardware data structure + * + * Returns the number of filters available on device + */ +int ice_get_fdir_cnt_all(struct ice_hw *hw) +{ + return hw->func_caps.fd_fltr_guar + hw->func_caps.fd_fltr_best_effort; +} + +/** + * ice_pkt_insert_ipv6_addr - insert a be32 IPv6 address into a memory buffer + * @pkt: packet buffer + * @offset: offset into buffer + * @addr: IPv6 address to convert and insert into pkt at offset + */ +static void ice_pkt_insert_ipv6_addr(u8 *pkt, int offset, __be32 *addr) +{ + int idx; + + for (idx = 0; idx < ICE_IPV6_ADDR_LEN_AS_U32; idx++) + memcpy(pkt + offset + idx * sizeof(*addr), &addr[idx], + sizeof(*addr)); +} + +/** + * ice_pkt_insert_u16 - insert a be16 value into a memory buffer + * @pkt: packet buffer + * @offset: offset into buffer + * @data: 16 bit value to convert and insert into pkt at offset + */ +static void ice_pkt_insert_u16(u8 *pkt, int offset, __be16 data) +{ + memcpy(pkt + offset, &data, sizeof(data)); +} + +/** + * ice_pkt_insert_u32 - insert a be32 value into a memory buffer + * @pkt: packet buffer + * @offset: offset into buffer + * @data: 32 bit value to convert and insert into pkt at offset + */ +static void ice_pkt_insert_u32(u8 *pkt, int offset, __be32 data) +{ + memcpy(pkt + offset, &data, sizeof(data)); +} + +/** + * ice_fdir_get_gen_prgm_pkt - generate a training packet + * @hw: pointer to the hardware structure + * @input: flow director filter data structure + * @pkt: pointer to return filter packet + * @frag: generate a fragment packet + * @tun: true implies generate a tunnel packet + */ +enum ice_status +ice_fdir_get_gen_prgm_pkt(struct ice_hw *hw, struct ice_fdir_fltr *input, + u8 *pkt, bool frag, bool tun) +{ + enum ice_fltr_ptype flow; + u16 tnl_port; + u8 *loc; + u16 idx; + + if (input->flow_type == ICE_FLTR_PTYPE_NONF_IPV4_OTHER) { + switch (input->ip.v4.proto) { + case IPPROTO_TCP: + flow = ICE_FLTR_PTYPE_NONF_IPV4_TCP; + break; + case IPPROTO_UDP: + flow = ICE_FLTR_PTYPE_NONF_IPV4_UDP; + break; + case IPPROTO_SCTP: + flow = ICE_FLTR_PTYPE_NONF_IPV4_SCTP; + break; + case IPPROTO_IP: + flow = ICE_FLTR_PTYPE_NONF_IPV4_OTHER; + break; + default: + return ICE_ERR_PARAM; + } + } else if (input->flow_type == ICE_FLTR_PTYPE_NONF_IPV6_OTHER) { + switch (input->ip.v6.proto) { + case IPPROTO_TCP: + flow = ICE_FLTR_PTYPE_NONF_IPV6_TCP; + break; + case IPPROTO_UDP: + flow = ICE_FLTR_PTYPE_NONF_IPV6_UDP; + break; + case IPPROTO_SCTP: + flow = ICE_FLTR_PTYPE_NONF_IPV6_SCTP; + break; + case IPPROTO_IP: + flow = ICE_FLTR_PTYPE_NONF_IPV6_OTHER; + break; + default: + return ICE_ERR_PARAM; + } + } else { + flow = input->flow_type; + } + + for (idx = 0; idx < ICE_FDIR_NUM_PKT; idx++) + if (ice_fdir_pkt[idx].flow == flow) + break; + if (idx == ICE_FDIR_NUM_PKT) + return ICE_ERR_PARAM; + if (!tun) { + memcpy(pkt, ice_fdir_pkt[idx].pkt, ice_fdir_pkt[idx].pkt_len); + loc = pkt; + } else { + if (!ice_get_open_tunnel_port(hw, TNL_ALL, &tnl_port)) + return ICE_ERR_DOES_NOT_EXIST; + if (!ice_fdir_pkt[idx].tun_pkt) + return ICE_ERR_PARAM; + memcpy(pkt, ice_fdir_pkt[idx].tun_pkt, + ice_fdir_pkt[idx].tun_pkt_len); + ice_pkt_insert_u16(pkt, ICE_IPV4_UDP_DST_PORT_OFFSET, + htons(tnl_port)); + loc = &pkt[ICE_FDIR_TUN_PKT_OFF]; + } + + /* Reverse the src and dst, since the HW expects them to be from Tx + * perspective. The input from user is from Rx filter perspective. + */ + switch (flow) { + case ICE_FLTR_PTYPE_NONF_IPV4_TCP: + ice_pkt_insert_u32(loc, ICE_IPV4_DST_ADDR_OFFSET, + input->ip.v4.src_ip); + ice_pkt_insert_u16(loc, ICE_IPV4_TCP_DST_PORT_OFFSET, + input->ip.v4.src_port); + ice_pkt_insert_u32(loc, ICE_IPV4_SRC_ADDR_OFFSET, + input->ip.v4.dst_ip); + ice_pkt_insert_u16(loc, ICE_IPV4_TCP_SRC_PORT_OFFSET, + input->ip.v4.dst_port); + if (frag) + loc[20] = ICE_FDIR_IPV4_PKT_FLAG_DF; + break; + case ICE_FLTR_PTYPE_NONF_IPV4_UDP: + ice_pkt_insert_u32(loc, ICE_IPV4_DST_ADDR_OFFSET, + input->ip.v4.src_ip); + ice_pkt_insert_u16(loc, ICE_IPV4_UDP_DST_PORT_OFFSET, + input->ip.v4.src_port); + ice_pkt_insert_u32(loc, ICE_IPV4_SRC_ADDR_OFFSET, + input->ip.v4.dst_ip); + ice_pkt_insert_u16(loc, ICE_IPV4_UDP_SRC_PORT_OFFSET, + input->ip.v4.dst_port); + break; + case ICE_FLTR_PTYPE_NONF_IPV4_SCTP: + ice_pkt_insert_u32(loc, ICE_IPV4_DST_ADDR_OFFSET, + input->ip.v4.src_ip); + ice_pkt_insert_u16(loc, ICE_IPV4_SCTP_DST_PORT_OFFSET, + input->ip.v4.src_port); + ice_pkt_insert_u32(loc, ICE_IPV4_SRC_ADDR_OFFSET, + input->ip.v4.dst_ip); + ice_pkt_insert_u16(loc, ICE_IPV4_SCTP_SRC_PORT_OFFSET, + input->ip.v4.dst_port); + break; + case ICE_FLTR_PTYPE_NONF_IPV4_OTHER: + ice_pkt_insert_u32(loc, ICE_IPV4_DST_ADDR_OFFSET, + input->ip.v4.src_ip); + ice_pkt_insert_u32(loc, ICE_IPV4_SRC_ADDR_OFFSET, + input->ip.v4.dst_ip); + ice_pkt_insert_u16(loc, ICE_IPV4_PROTO_OFFSET, 0); + break; + case ICE_FLTR_PTYPE_NONF_IPV6_TCP: + ice_pkt_insert_ipv6_addr(loc, ICE_IPV6_DST_ADDR_OFFSET, + input->ip.v6.src_ip); + ice_pkt_insert_ipv6_addr(loc, ICE_IPV6_SRC_ADDR_OFFSET, + input->ip.v6.dst_ip); + ice_pkt_insert_u16(loc, ICE_IPV6_TCP_DST_PORT_OFFSET, + input->ip.v6.src_port); + ice_pkt_insert_u16(loc, ICE_IPV6_TCP_SRC_PORT_OFFSET, + input->ip.v6.dst_port); + break; + case ICE_FLTR_PTYPE_NONF_IPV6_UDP: + ice_pkt_insert_ipv6_addr(loc, ICE_IPV6_DST_ADDR_OFFSET, + input->ip.v6.src_ip); + ice_pkt_insert_ipv6_addr(loc, ICE_IPV6_SRC_ADDR_OFFSET, + input->ip.v6.dst_ip); + ice_pkt_insert_u16(loc, ICE_IPV6_UDP_DST_PORT_OFFSET, + input->ip.v6.src_port); + ice_pkt_insert_u16(loc, ICE_IPV6_UDP_SRC_PORT_OFFSET, + input->ip.v6.dst_port); + break; + case ICE_FLTR_PTYPE_NONF_IPV6_SCTP: + ice_pkt_insert_ipv6_addr(loc, ICE_IPV6_DST_ADDR_OFFSET, + input->ip.v6.src_ip); + ice_pkt_insert_ipv6_addr(loc, ICE_IPV6_SRC_ADDR_OFFSET, + input->ip.v6.dst_ip); + ice_pkt_insert_u16(loc, ICE_IPV6_SCTP_DST_PORT_OFFSET, + input->ip.v6.src_port); + ice_pkt_insert_u16(loc, ICE_IPV6_SCTP_SRC_PORT_OFFSET, + input->ip.v6.dst_port); + break; + case ICE_FLTR_PTYPE_NONF_IPV6_OTHER: + ice_pkt_insert_ipv6_addr(loc, ICE_IPV6_DST_ADDR_OFFSET, + input->ip.v6.src_ip); + ice_pkt_insert_ipv6_addr(loc, ICE_IPV6_SRC_ADDR_OFFSET, + input->ip.v6.dst_ip); + break; + default: + return ICE_ERR_PARAM; + } + + if (input->flex_fltr) + ice_pkt_insert_u16(loc, input->flex_offset, input->flex_word); + + return 0; +} + +/** + * ice_fdir_has_frag - does flow type have 2 ptypes + * @flow: flow ptype + * + * returns true is there is a fragment packet for this ptype + */ +bool ice_fdir_has_frag(enum ice_fltr_ptype flow) +{ + if (flow == ICE_FLTR_PTYPE_NONF_IPV4_OTHER) + return true; + else + return false; +} + +/** + * ice_fdir_find_by_idx - find filter with idx + * @hw: pointer to hardware structure + * @fltr_idx: index to find. + * + * Returns pointer to filter if found or null + */ +struct ice_fdir_fltr * +ice_fdir_find_fltr_by_idx(struct ice_hw *hw, u32 fltr_idx) +{ + struct ice_fdir_fltr *rule; + + list_for_each_entry(rule, &hw->fdir_list_head, fltr_node) { + /* rule ID found in the list */ + if (fltr_idx == rule->fltr_id) + return rule; + if (fltr_idx < rule->fltr_id) + break; + } + return NULL; +} + +/** + * ice_fdir_list_add_fltr - add a new node to the flow director filter list + * @hw: hardware structure + * @fltr: filter node to add to structure + */ +void ice_fdir_list_add_fltr(struct ice_hw *hw, struct ice_fdir_fltr *fltr) +{ + struct ice_fdir_fltr *rule, *parent = NULL; + + list_for_each_entry(rule, &hw->fdir_list_head, fltr_node) { + /* rule ID found or pass its spot in the list */ + if (rule->fltr_id >= fltr->fltr_id) + break; + parent = rule; + } + + if (parent) + list_add(&fltr->fltr_node, &parent->fltr_node); + else + list_add(&fltr->fltr_node, &hw->fdir_list_head); +} + +/** + * ice_fdir_update_cntrs - increment / decrement filter counter + * @hw: pointer to hardware structure + * @flow: filter flow type + * @add: true implies filters added + */ +void +ice_fdir_update_cntrs(struct ice_hw *hw, enum ice_fltr_ptype flow, bool add) +{ + int incr; + + incr = add ? 1 : -1; + hw->fdir_active_fltr += incr; + + if (flow == ICE_FLTR_PTYPE_NONF_NONE || flow >= ICE_FLTR_PTYPE_MAX) + ice_debug(hw, ICE_DBG_SW, "Unknown filter type %d\n", flow); + else + hw->fdir_fltr_cnt[flow] += incr; +} + +/** + * ice_cmp_ipv6_addr - compare 2 IP v6 addresses + * @a: IP v6 address + * @b: IP v6 address + * + * Returns 0 on equal, returns non-0 if different + */ +static int ice_cmp_ipv6_addr(__be32 *a, __be32 *b) +{ + return memcmp(a, b, 4 * sizeof(__be32)); +} + +/** + * ice_fdir_comp_rules - compare 2 filters + * @a: a Flow Director filter data structure + * @b: a Flow Director filter data structure + * @v6: bool true if v6 filter + * + * Returns true if the filters match + */ +static bool +ice_fdir_comp_rules(struct ice_fdir_fltr *a, struct ice_fdir_fltr *b, bool v6) +{ + enum ice_fltr_ptype flow_type = a->flow_type; + + /* The calling function already checks that the two filters have the + * same flow_type. + */ + if (!v6) { + if (flow_type == ICE_FLTR_PTYPE_NONF_IPV4_TCP || + flow_type == ICE_FLTR_PTYPE_NONF_IPV4_UDP || + flow_type == ICE_FLTR_PTYPE_NONF_IPV4_SCTP) { + if (a->ip.v4.dst_ip == b->ip.v4.dst_ip && + a->ip.v4.src_ip == b->ip.v4.src_ip && + a->ip.v4.dst_port == b->ip.v4.dst_port && + a->ip.v4.src_port == b->ip.v4.src_port) + return true; + } else if (flow_type == ICE_FLTR_PTYPE_NONF_IPV4_OTHER) { + if (a->ip.v4.dst_ip == b->ip.v4.dst_ip && + a->ip.v4.src_ip == b->ip.v4.src_ip && + a->ip.v4.l4_header == b->ip.v4.l4_header && + a->ip.v4.proto == b->ip.v4.proto && + a->ip.v4.ip_ver == b->ip.v4.ip_ver && + a->ip.v4.tos == b->ip.v4.tos) + return true; + } + } else { + if (flow_type == ICE_FLTR_PTYPE_NONF_IPV6_UDP || + flow_type == ICE_FLTR_PTYPE_NONF_IPV6_TCP || + flow_type == ICE_FLTR_PTYPE_NONF_IPV6_SCTP) { + if (a->ip.v6.dst_port == b->ip.v6.dst_port && + a->ip.v6.src_port == b->ip.v6.src_port && + !ice_cmp_ipv6_addr(a->ip.v6.dst_ip, + b->ip.v6.dst_ip) && + !ice_cmp_ipv6_addr(a->ip.v6.src_ip, + b->ip.v6.src_ip)) + return true; + } else if (flow_type == ICE_FLTR_PTYPE_NONF_IPV6_OTHER) { + if (a->ip.v6.dst_port == b->ip.v6.dst_port && + a->ip.v6.src_port == b->ip.v6.src_port) + return true; + } + } + + return false; +} + +/** + * ice_fdir_is_dup_fltr - test if filter is already in list for PF + * @hw: hardware data structure + * @input: Flow Director filter data structure + * + * Returns true if the filter is found in the list + */ +bool ice_fdir_is_dup_fltr(struct ice_hw *hw, struct ice_fdir_fltr *input) +{ + struct ice_fdir_fltr *rule; + bool ret = false; + + list_for_each_entry(rule, &hw->fdir_list_head, fltr_node) { + enum ice_fltr_ptype flow_type; + + if (rule->flow_type != input->flow_type) + continue; + + flow_type = input->flow_type; + if (flow_type == ICE_FLTR_PTYPE_NONF_IPV4_TCP || + flow_type == ICE_FLTR_PTYPE_NONF_IPV4_UDP || + flow_type == ICE_FLTR_PTYPE_NONF_IPV4_SCTP || + flow_type == ICE_FLTR_PTYPE_NONF_IPV4_OTHER) + ret = ice_fdir_comp_rules(rule, input, false); + else + ret = ice_fdir_comp_rules(rule, input, true); + if (ret) { + if (rule->fltr_id == input->fltr_id && + rule->q_index != input->q_index) + ret = false; + else + break; + } + } + + return ret; +} diff --git a/drivers/net/ethernet/intel/ice/ice_fdir.h b/drivers/net/ethernet/intel/ice/ice_fdir.h new file mode 100644 index 000000000000..1c587766daab --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_fdir.h @@ -0,0 +1,166 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2018-2020, Intel Corporation. */ + +#ifndef _ICE_FDIR_H_ +#define _ICE_FDIR_H_ + +#define ICE_FDIR_TUN_PKT_OFF 50 +#define ICE_FDIR_MAX_RAW_PKT_SIZE (512 + ICE_FDIR_TUN_PKT_OFF) + +/* macros for offsets into packets for flow director programming */ +#define ICE_IPV4_SRC_ADDR_OFFSET 26 +#define ICE_IPV4_DST_ADDR_OFFSET 30 +#define ICE_IPV4_TCP_SRC_PORT_OFFSET 34 +#define ICE_IPV4_TCP_DST_PORT_OFFSET 36 +#define ICE_IPV4_UDP_SRC_PORT_OFFSET 34 +#define ICE_IPV4_UDP_DST_PORT_OFFSET 36 +#define ICE_IPV4_SCTP_SRC_PORT_OFFSET 34 +#define ICE_IPV4_SCTP_DST_PORT_OFFSET 36 +#define ICE_IPV4_PROTO_OFFSET 23 +#define ICE_IPV6_SRC_ADDR_OFFSET 22 +#define ICE_IPV6_DST_ADDR_OFFSET 38 +#define ICE_IPV6_TCP_SRC_PORT_OFFSET 54 +#define ICE_IPV6_TCP_DST_PORT_OFFSET 56 +#define ICE_IPV6_UDP_SRC_PORT_OFFSET 54 +#define ICE_IPV6_UDP_DST_PORT_OFFSET 56 +#define ICE_IPV6_SCTP_SRC_PORT_OFFSET 54 +#define ICE_IPV6_SCTP_DST_PORT_OFFSET 56 +/* IP v4 has 2 flag bits that enable fragment processing: DF and MF. DF + * requests that the packet not be fragmented. MF indicates that a packet has + * been fragmented. + */ +#define ICE_FDIR_IPV4_PKT_FLAG_DF 0x20 + +enum ice_fltr_prgm_desc_dest { + ICE_FLTR_PRGM_DESC_DEST_DROP_PKT, + ICE_FLTR_PRGM_DESC_DEST_DIRECT_PKT_QINDEX, +}; + +enum ice_fltr_prgm_desc_fd_status { + ICE_FLTR_PRGM_DESC_FD_STATUS_NONE, + ICE_FLTR_PRGM_DESC_FD_STATUS_FD_ID, +}; + +/* Flow Director (FD) Filter Programming descriptor */ +struct ice_fd_fltr_desc_ctx { + u32 fdid; + u16 qindex; + u16 cnt_index; + u16 fd_vsi; + u16 flex_val; + u8 comp_q; + u8 comp_report; + u8 fd_space; + u8 cnt_ena; + u8 evict_ena; + u8 toq; + u8 toq_prio; + u8 dpu_recipe; + u8 drop; + u8 flex_prio; + u8 flex_mdid; + u8 dtype; + u8 pcmd; + u8 desc_prof_prio; + u8 desc_prof; + u8 swap; + u8 fdid_prio; + u8 fdid_mdid; +}; + +#define ICE_FLTR_PRGM_FLEX_WORD_SIZE sizeof(__be16) + +struct ice_rx_flow_userdef { + u16 flex_word; + u16 flex_offset; + u16 flex_fltr; +}; + +struct ice_fdir_v4 { + __be32 dst_ip; + __be32 src_ip; + __be16 dst_port; + __be16 src_port; + __be32 l4_header; + __be32 sec_parm_idx; /* security parameter index */ + u8 tos; + u8 ip_ver; + u8 proto; +}; + +#define ICE_IPV6_ADDR_LEN_AS_U32 4 + +struct ice_fdir_v6 { + __be32 dst_ip[ICE_IPV6_ADDR_LEN_AS_U32]; + __be32 src_ip[ICE_IPV6_ADDR_LEN_AS_U32]; + __be16 dst_port; + __be16 src_port; + __be32 l4_header; /* next header */ + __be32 sec_parm_idx; /* security parameter index */ + u8 tc; + u8 proto; +}; + +struct ice_fdir_extra { + u8 dst_mac[ETH_ALEN]; /* dest MAC address */ + u32 usr_def[2]; /* user data */ + __be16 vlan_type; /* VLAN ethertype */ + __be16 vlan_tag; /* VLAN tag info */ +}; + +struct ice_fdir_fltr { + struct list_head fltr_node; + enum ice_fltr_ptype flow_type; + + union { + struct ice_fdir_v4 v4; + struct ice_fdir_v6 v6; + } ip, mask; + + struct ice_fdir_extra ext_data; + struct ice_fdir_extra ext_mask; + + /* flex byte filter data */ + __be16 flex_word; + u16 flex_offset; + u16 flex_fltr; + + /* filter control */ + u16 q_index; + u16 dest_vsi; + u8 dest_ctl; + u8 fltr_status; + u16 cnt_index; + u32 fltr_id; +}; + +/* Dummy packet filter definition structure */ +struct ice_fdir_base_pkt { + enum ice_fltr_ptype flow; + u16 pkt_len; + const u8 *pkt; + u16 tun_pkt_len; + const u8 *tun_pkt; +}; + +enum ice_status ice_alloc_fd_res_cntr(struct ice_hw *hw, u16 *cntr_id); +enum ice_status ice_free_fd_res_cntr(struct ice_hw *hw, u16 cntr_id); +enum ice_status +ice_alloc_fd_guar_item(struct ice_hw *hw, u16 *cntr_id, u16 num_fltr); +enum ice_status +ice_alloc_fd_shrd_item(struct ice_hw *hw, u16 *cntr_id, u16 num_fltr); +void +ice_fdir_get_prgm_desc(struct ice_hw *hw, struct ice_fdir_fltr *input, + struct ice_fltr_desc *fdesc, bool add); +enum ice_status +ice_fdir_get_gen_prgm_pkt(struct ice_hw *hw, struct ice_fdir_fltr *input, + u8 *pkt, bool frag, bool tun); +int ice_get_fdir_cnt_all(struct ice_hw *hw); +bool ice_fdir_is_dup_fltr(struct ice_hw *hw, struct ice_fdir_fltr *input); +bool ice_fdir_has_frag(enum ice_fltr_ptype flow); +struct ice_fdir_fltr * +ice_fdir_find_fltr_by_idx(struct ice_hw *hw, u32 fltr_idx); +void +ice_fdir_update_cntrs(struct ice_hw *hw, enum ice_fltr_ptype flow, bool add); +void ice_fdir_list_add_fltr(struct ice_hw *hw, struct ice_fdir_fltr *input); +#endif /* _ICE_FDIR_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_flex_pipe.c b/drivers/net/ethernet/intel/ice/ice_flex_pipe.c index 42bac3ec5526..4420fc02f7e7 100644 --- a/drivers/net/ethernet/intel/ice/ice_flex_pipe.c +++ b/drivers/net/ethernet/intel/ice/ice_flex_pipe.c @@ -5,6 +5,15 @@ #include "ice_flex_pipe.h" #include "ice_flow.h" +/* To support tunneling entries by PF, the package will append the PF number to + * the label; for example TNL_VXLAN_PF0, TNL_VXLAN_PF1, TNL_VXLAN_PF2, etc. + */ +static const struct ice_tunnel_type_scan tnls[] = { + { TNL_VXLAN, "TNL_VXLAN_PF" }, + { TNL_GENEVE, "TNL_GENEVE_PF" }, + { TNL_LAST, "" } +}; + static const u32 ice_sect_lkup[ICE_BLK_COUNT][ICE_SECT_COUNT] = { /* SWITCH */ { @@ -239,6 +248,268 @@ ice_pkg_enum_section(struct ice_seg *ice_seg, struct ice_pkg_enum *state, return state->sect; } +/** + * ice_pkg_enum_entry + * @ice_seg: pointer to the ice segment (or NULL on subsequent calls) + * @state: pointer to the enum state + * @sect_type: section type to enumerate + * @offset: pointer to variable that receives the offset in the table (optional) + * @handler: function that handles access to the entries into the section type + * + * This function will enumerate all the entries in particular section type in + * the ice segment. The first call is made with the ice_seg parameter non-NULL; + * on subsequent calls, ice_seg is set to NULL which continues the enumeration. + * When the function returns a NULL pointer, then the end of the entries has + * been reached. + * + * Since each section may have a different header and entry size, the handler + * function is needed to determine the number and location entries in each + * section. + * + * The offset parameter is optional, but should be used for sections that + * contain an offset for each section table. For such cases, the section handler + * function must return the appropriate offset + index to give the absolution + * offset for each entry. For example, if the base for a section's header + * indicates a base offset of 10, and the index for the entry is 2, then + * section handler function should set the offset to 10 + 2 = 12. + */ +static void * +ice_pkg_enum_entry(struct ice_seg *ice_seg, struct ice_pkg_enum *state, + u32 sect_type, u32 *offset, + void *(*handler)(u32 sect_type, void *section, + u32 index, u32 *offset)) +{ + void *entry; + + if (ice_seg) { + if (!handler) + return NULL; + + if (!ice_pkg_enum_section(ice_seg, state, sect_type)) + return NULL; + + state->entry_idx = 0; + state->handler = handler; + } else { + state->entry_idx++; + } + + if (!state->handler) + return NULL; + + /* get entry */ + entry = state->handler(state->sect_type, state->sect, state->entry_idx, + offset); + if (!entry) { + /* end of a section, look for another section of this type */ + if (!ice_pkg_enum_section(NULL, state, 0)) + return NULL; + + state->entry_idx = 0; + entry = state->handler(state->sect_type, state->sect, + state->entry_idx, offset); + } + + return entry; +} + +/** + * ice_boost_tcam_handler + * @sect_type: section type + * @section: pointer to section + * @index: index of the boost TCAM entry to be returned + * @offset: pointer to receive absolute offset, always 0 for boost TCAM sections + * + * This is a callback function that can be passed to ice_pkg_enum_entry. + * Handles enumeration of individual boost TCAM entries. + */ +static void * +ice_boost_tcam_handler(u32 sect_type, void *section, u32 index, u32 *offset) +{ + struct ice_boost_tcam_section *boost; + + if (!section) + return NULL; + + if (sect_type != ICE_SID_RXPARSER_BOOST_TCAM) + return NULL; + + if (index > ICE_MAX_BST_TCAMS_IN_BUF) + return NULL; + + if (offset) + *offset = 0; + + boost = section; + if (index >= le16_to_cpu(boost->count)) + return NULL; + + return boost->tcam + index; +} + +/** + * ice_find_boost_entry + * @ice_seg: pointer to the ice segment (non-NULL) + * @addr: Boost TCAM address of entry to search for + * @entry: returns pointer to the entry + * + * Finds a particular Boost TCAM entry and returns a pointer to that entry + * if it is found. The ice_seg parameter must not be NULL since the first call + * to ice_pkg_enum_entry requires a pointer to an actual ice_segment structure. + */ +static enum ice_status +ice_find_boost_entry(struct ice_seg *ice_seg, u16 addr, + struct ice_boost_tcam_entry **entry) +{ + struct ice_boost_tcam_entry *tcam; + struct ice_pkg_enum state; + + memset(&state, 0, sizeof(state)); + + if (!ice_seg) + return ICE_ERR_PARAM; + + do { + tcam = ice_pkg_enum_entry(ice_seg, &state, + ICE_SID_RXPARSER_BOOST_TCAM, NULL, + ice_boost_tcam_handler); + if (tcam && le16_to_cpu(tcam->addr) == addr) { + *entry = tcam; + return 0; + } + + ice_seg = NULL; + } while (tcam); + + *entry = NULL; + return ICE_ERR_CFG; +} + +/** + * ice_label_enum_handler + * @sect_type: section type + * @section: pointer to section + * @index: index of the label entry to be returned + * @offset: pointer to receive absolute offset, always zero for label sections + * + * This is a callback function that can be passed to ice_pkg_enum_entry. + * Handles enumeration of individual label entries. + */ +static void * +ice_label_enum_handler(u32 __always_unused sect_type, void *section, u32 index, + u32 *offset) +{ + struct ice_label_section *labels; + + if (!section) + return NULL; + + if (index > ICE_MAX_LABELS_IN_BUF) + return NULL; + + if (offset) + *offset = 0; + + labels = section; + if (index >= le16_to_cpu(labels->count)) + return NULL; + + return labels->label + index; +} + +/** + * ice_enum_labels + * @ice_seg: pointer to the ice segment (NULL on subsequent calls) + * @type: the section type that will contain the label (0 on subsequent calls) + * @state: ice_pkg_enum structure that will hold the state of the enumeration + * @value: pointer to a value that will return the label's value if found + * + * Enumerates a list of labels in the package. The caller will call + * ice_enum_labels(ice_seg, type, ...) to start the enumeration, then call + * ice_enum_labels(NULL, 0, ...) to continue. When the function returns a NULL + * the end of the list has been reached. + */ +static char * +ice_enum_labels(struct ice_seg *ice_seg, u32 type, struct ice_pkg_enum *state, + u16 *value) +{ + struct ice_label *label; + + /* Check for valid label section on first call */ + if (type && !(type >= ICE_SID_LBL_FIRST && type <= ICE_SID_LBL_LAST)) + return NULL; + + label = ice_pkg_enum_entry(ice_seg, state, type, NULL, + ice_label_enum_handler); + if (!label) + return NULL; + + *value = le16_to_cpu(label->value); + return label->name; +} + +/** + * ice_init_pkg_hints + * @hw: pointer to the HW structure + * @ice_seg: pointer to the segment of the package scan (non-NULL) + * + * This function will scan the package and save off relevant information + * (hints or metadata) for driver use. The ice_seg parameter must not be NULL + * since the first call to ice_enum_labels requires a pointer to an actual + * ice_seg structure. + */ +static void ice_init_pkg_hints(struct ice_hw *hw, struct ice_seg *ice_seg) +{ + struct ice_pkg_enum state; + char *label_name; + u16 val; + int i; + + memset(&hw->tnl, 0, sizeof(hw->tnl)); + memset(&state, 0, sizeof(state)); + + if (!ice_seg) + return; + + label_name = ice_enum_labels(ice_seg, ICE_SID_LBL_RXPARSER_TMEM, &state, + &val); + + while (label_name && hw->tnl.count < ICE_TUNNEL_MAX_ENTRIES) { + for (i = 0; tnls[i].type != TNL_LAST; i++) { + size_t len = strlen(tnls[i].label_prefix); + + /* Look for matching label start, before continuing */ + if (strncmp(label_name, tnls[i].label_prefix, len)) + continue; + + /* Make sure this label matches our PF. Note that the PF + * character ('0' - '7') will be located where our + * prefix string's null terminator is located. + */ + if ((label_name[len] - '0') == hw->pf_id) { + hw->tnl.tbl[hw->tnl.count].type = tnls[i].type; + hw->tnl.tbl[hw->tnl.count].valid = false; + hw->tnl.tbl[hw->tnl.count].in_use = false; + hw->tnl.tbl[hw->tnl.count].marked = false; + hw->tnl.tbl[hw->tnl.count].boost_addr = val; + hw->tnl.tbl[hw->tnl.count].port = 0; + hw->tnl.count++; + break; + } + } + + label_name = ice_enum_labels(NULL, 0, &state, &val); + } + + /* Cache the appropriate boost TCAM entry pointers */ + for (i = 0; i < hw->tnl.count; i++) { + ice_find_boost_entry(ice_seg, hw->tnl.tbl[i].boost_addr, + &hw->tnl.tbl[i].boost_entry); + if (hw->tnl.tbl[i].boost_entry) + hw->tnl.tbl[i].valid = true; + } +} + /* Key creation */ #define ICE_DC_KEY 0x1 /* don't care */ @@ -593,8 +864,9 @@ ice_find_seg_in_pkg(struct ice_hw *hw, u32 seg_type, u32 i; ice_debug(hw, ICE_DBG_PKG, "Package format version: %d.%d.%d.%d\n", - pkg_hdr->format_ver.major, pkg_hdr->format_ver.minor, - pkg_hdr->format_ver.update, pkg_hdr->format_ver.draft); + pkg_hdr->pkg_format_ver.major, pkg_hdr->pkg_format_ver.minor, + pkg_hdr->pkg_format_ver.update, + pkg_hdr->pkg_format_ver.draft); /* Search all package segments for the requested segment type */ for (i = 0; i < le32_to_cpu(pkg_hdr->seg_count); i++) { @@ -764,13 +1036,15 @@ ice_download_pkg(struct ice_hw *hw, struct ice_seg *ice_seg) { struct ice_buf_table *ice_buf_tbl; - ice_debug(hw, ICE_DBG_PKG, "Segment version: %d.%d.%d.%d\n", - ice_seg->hdr.seg_ver.major, ice_seg->hdr.seg_ver.minor, - ice_seg->hdr.seg_ver.update, ice_seg->hdr.seg_ver.draft); + ice_debug(hw, ICE_DBG_PKG, "Segment format version: %d.%d.%d.%d\n", + ice_seg->hdr.seg_format_ver.major, + ice_seg->hdr.seg_format_ver.minor, + ice_seg->hdr.seg_format_ver.update, + ice_seg->hdr.seg_format_ver.draft); ice_debug(hw, ICE_DBG_PKG, "Seg: type 0x%X, size %d, name %s\n", le32_to_cpu(ice_seg->hdr.seg_type), - le32_to_cpu(ice_seg->hdr.seg_size), ice_seg->hdr.seg_name); + le32_to_cpu(ice_seg->hdr.seg_size), ice_seg->hdr.seg_id); ice_buf_tbl = ice_find_buf_table(ice_seg); @@ -815,14 +1089,16 @@ ice_init_pkg_info(struct ice_hw *hw, struct ice_pkg_hdr *pkg_hdr) seg_hdr = ice_find_seg_in_pkg(hw, SEGMENT_TYPE_ICE, pkg_hdr); if (seg_hdr) { - hw->ice_pkg_ver = seg_hdr->seg_ver; - memcpy(hw->ice_pkg_name, seg_hdr->seg_name, + hw->ice_pkg_ver = seg_hdr->seg_format_ver; + memcpy(hw->ice_pkg_name, seg_hdr->seg_id, sizeof(hw->ice_pkg_name)); - ice_debug(hw, ICE_DBG_PKG, "Ice Pkg: %d.%d.%d.%d, %s\n", - seg_hdr->seg_ver.major, seg_hdr->seg_ver.minor, - seg_hdr->seg_ver.update, seg_hdr->seg_ver.draft, - seg_hdr->seg_name); + ice_debug(hw, ICE_DBG_PKG, "Ice Seg: %d.%d.%d.%d, %s\n", + seg_hdr->seg_format_ver.major, + seg_hdr->seg_format_ver.minor, + seg_hdr->seg_format_ver.update, + seg_hdr->seg_format_ver.draft, + seg_hdr->seg_id); } else { ice_debug(hw, ICE_DBG_INIT, "Did not find ice segment in driver package\n"); @@ -863,9 +1139,11 @@ static enum ice_status ice_get_pkg_info(struct ice_hw *hw) if (pkg_info->pkg_info[i].is_active) { flags[place++] = 'A'; hw->active_pkg_ver = pkg_info->pkg_info[i].ver; + hw->active_track_id = + le32_to_cpu(pkg_info->pkg_info[i].track_id); memcpy(hw->active_pkg_name, pkg_info->pkg_info[i].name, - sizeof(hw->active_pkg_name)); + sizeof(pkg_info->pkg_info[i].name)); hw->active_pkg_in_nvm = pkg_info->pkg_info[i].is_in_nvm; } if (pkg_info->pkg_info[i].is_active_at_boot) @@ -905,10 +1183,10 @@ static enum ice_status ice_verify_pkg(struct ice_pkg_hdr *pkg, u32 len) if (len < sizeof(*pkg)) return ICE_ERR_BUF_TOO_SHORT; - if (pkg->format_ver.major != ICE_PKG_FMT_VER_MAJ || - pkg->format_ver.minor != ICE_PKG_FMT_VER_MNR || - pkg->format_ver.update != ICE_PKG_FMT_VER_UPD || - pkg->format_ver.draft != ICE_PKG_FMT_VER_DFT) + if (pkg->pkg_format_ver.major != ICE_PKG_FMT_VER_MAJ || + pkg->pkg_format_ver.minor != ICE_PKG_FMT_VER_MNR || + pkg->pkg_format_ver.update != ICE_PKG_FMT_VER_UPD || + pkg->pkg_format_ver.draft != ICE_PKG_FMT_VER_DFT) return ICE_ERR_CFG; /* pkg must have at least one segment */ @@ -990,6 +1268,68 @@ static enum ice_status ice_chk_pkg_version(struct ice_pkg_ver *pkg_ver) } /** + * ice_chk_pkg_compat + * @hw: pointer to the hardware structure + * @ospkg: pointer to the package hdr + * @seg: pointer to the package segment hdr + * + * This function checks the package version compatibility with driver and NVM + */ +static enum ice_status +ice_chk_pkg_compat(struct ice_hw *hw, struct ice_pkg_hdr *ospkg, + struct ice_seg **seg) +{ + struct ice_aqc_get_pkg_info_resp *pkg; + enum ice_status status; + u16 size; + u32 i; + + /* Check package version compatibility */ + status = ice_chk_pkg_version(&hw->pkg_ver); + if (status) { + ice_debug(hw, ICE_DBG_INIT, "Package version check failed.\n"); + return status; + } + + /* find ICE segment in given package */ + *seg = (struct ice_seg *)ice_find_seg_in_pkg(hw, SEGMENT_TYPE_ICE, + ospkg); + if (!*seg) { + ice_debug(hw, ICE_DBG_INIT, "no ice segment in package.\n"); + return ICE_ERR_CFG; + } + + /* Check if FW is compatible with the OS package */ + size = struct_size(pkg, pkg_info, ICE_PKG_CNT - 1); + pkg = kzalloc(size, GFP_KERNEL); + if (!pkg) + return ICE_ERR_NO_MEMORY; + + status = ice_aq_get_pkg_info_list(hw, pkg, size, NULL); + if (status) + goto fw_ddp_compat_free_alloc; + + for (i = 0; i < le32_to_cpu(pkg->count); i++) { + /* loop till we find the NVM package */ + if (!pkg->pkg_info[i].is_in_nvm) + continue; + if ((*seg)->hdr.seg_format_ver.major != + pkg->pkg_info[i].ver.major || + (*seg)->hdr.seg_format_ver.minor > + pkg->pkg_info[i].ver.minor) { + status = ICE_ERR_FW_DDP_MISMATCH; + ice_debug(hw, ICE_DBG_INIT, + "OS package is not compatible with NVM.\n"); + } + /* done processing NVM package so break */ + break; + } +fw_ddp_compat_free_alloc: + kfree(pkg); + return status; +} + +/** * ice_init_pkg - initialize/download package * @hw: pointer to the hardware structure * @buf: pointer to the package buffer @@ -1039,18 +1379,12 @@ enum ice_status ice_init_pkg(struct ice_hw *hw, u8 *buf, u32 len) /* before downloading the package, check package version for * compatibility with driver */ - status = ice_chk_pkg_version(&hw->pkg_ver); + status = ice_chk_pkg_compat(hw, pkg, &seg); if (status) return status; - /* find segment in given package */ - seg = (struct ice_seg *)ice_find_seg_in_pkg(hw, SEGMENT_TYPE_ICE, pkg); - if (!seg) { - ice_debug(hw, ICE_DBG_INIT, "no ice segment in package.\n"); - return ICE_ERR_CFG; - } - - /* download package */ + /* initialize package hints and then download package */ + ice_init_pkg_hints(hw, seg); status = ice_download_pkg(hw, seg); if (status == ICE_ERR_AQ_NO_WORK) { ice_debug(hw, ICE_DBG_INIT, @@ -1292,6 +1626,284 @@ static struct ice_buf *ice_pkg_buf(struct ice_buf_build *bld) return &bld->buf; } +/** + * ice_tunnel_port_in_use_hlpr - helper function to determine tunnel usage + * @hw: pointer to the HW structure + * @port: port to search for + * @index: optionally returns index + * + * Returns whether a port is already in use as a tunnel, and optionally its + * index + */ +static bool ice_tunnel_port_in_use_hlpr(struct ice_hw *hw, u16 port, u16 *index) +{ + u16 i; + + for (i = 0; i < hw->tnl.count && i < ICE_TUNNEL_MAX_ENTRIES; i++) + if (hw->tnl.tbl[i].in_use && hw->tnl.tbl[i].port == port) { + if (index) + *index = i; + return true; + } + + return false; +} + +/** + * ice_tunnel_port_in_use + * @hw: pointer to the HW structure + * @port: port to search for + * @index: optionally returns index + * + * Returns whether a port is already in use as a tunnel, and optionally its + * index + */ +bool ice_tunnel_port_in_use(struct ice_hw *hw, u16 port, u16 *index) +{ + bool res; + + mutex_lock(&hw->tnl_lock); + res = ice_tunnel_port_in_use_hlpr(hw, port, index); + mutex_unlock(&hw->tnl_lock); + + return res; +} + +/** + * ice_find_free_tunnel_entry + * @hw: pointer to the HW structure + * @type: tunnel type + * @index: optionally returns index + * + * Returns whether there is a free tunnel entry, and optionally its index + */ +static bool +ice_find_free_tunnel_entry(struct ice_hw *hw, enum ice_tunnel_type type, + u16 *index) +{ + u16 i; + + for (i = 0; i < hw->tnl.count && i < ICE_TUNNEL_MAX_ENTRIES; i++) + if (hw->tnl.tbl[i].valid && !hw->tnl.tbl[i].in_use && + hw->tnl.tbl[i].type == type) { + if (index) + *index = i; + return true; + } + + return false; +} + +/** + * ice_get_open_tunnel_port - retrieve an open tunnel port + * @hw: pointer to the HW structure + * @type: tunnel type (TNL_ALL will return any open port) + * @port: returns open port + */ +bool +ice_get_open_tunnel_port(struct ice_hw *hw, enum ice_tunnel_type type, + u16 *port) +{ + bool res = false; + u16 i; + + mutex_lock(&hw->tnl_lock); + + for (i = 0; i < hw->tnl.count && i < ICE_TUNNEL_MAX_ENTRIES; i++) + if (hw->tnl.tbl[i].valid && hw->tnl.tbl[i].in_use && + (type == TNL_ALL || hw->tnl.tbl[i].type == type)) { + *port = hw->tnl.tbl[i].port; + res = true; + break; + } + + mutex_unlock(&hw->tnl_lock); + + return res; +} + +/** + * ice_create_tunnel + * @hw: pointer to the HW structure + * @type: type of tunnel + * @port: port of tunnel to create + * + * Create a tunnel by updating the parse graph in the parser. We do that by + * creating a package buffer with the tunnel info and issuing an update package + * command. + */ +enum ice_status +ice_create_tunnel(struct ice_hw *hw, enum ice_tunnel_type type, u16 port) +{ + struct ice_boost_tcam_section *sect_rx, *sect_tx; + enum ice_status status = ICE_ERR_MAX_LIMIT; + struct ice_buf_build *bld; + u16 index; + + mutex_lock(&hw->tnl_lock); + + if (ice_tunnel_port_in_use_hlpr(hw, port, &index)) { + hw->tnl.tbl[index].ref++; + status = 0; + goto ice_create_tunnel_end; + } + + if (!ice_find_free_tunnel_entry(hw, type, &index)) { + status = ICE_ERR_OUT_OF_RANGE; + goto ice_create_tunnel_end; + } + + bld = ice_pkg_buf_alloc(hw); + if (!bld) { + status = ICE_ERR_NO_MEMORY; + goto ice_create_tunnel_end; + } + + /* allocate 2 sections, one for Rx parser, one for Tx parser */ + if (ice_pkg_buf_reserve_section(bld, 2)) + goto ice_create_tunnel_err; + + sect_rx = ice_pkg_buf_alloc_section(bld, ICE_SID_RXPARSER_BOOST_TCAM, + sizeof(*sect_rx)); + if (!sect_rx) + goto ice_create_tunnel_err; + sect_rx->count = cpu_to_le16(1); + + sect_tx = ice_pkg_buf_alloc_section(bld, ICE_SID_TXPARSER_BOOST_TCAM, + sizeof(*sect_tx)); + if (!sect_tx) + goto ice_create_tunnel_err; + sect_tx->count = cpu_to_le16(1); + + /* copy original boost entry to update package buffer */ + memcpy(sect_rx->tcam, hw->tnl.tbl[index].boost_entry, + sizeof(*sect_rx->tcam)); + + /* over-write the never-match dest port key bits with the encoded port + * bits + */ + ice_set_key((u8 *)§_rx->tcam[0].key, sizeof(sect_rx->tcam[0].key), + (u8 *)&port, NULL, NULL, NULL, + (u16)offsetof(struct ice_boost_key_value, hv_dst_port_key), + sizeof(sect_rx->tcam[0].key.key.hv_dst_port_key)); + + /* exact copy of entry to Tx section entry */ + memcpy(sect_tx->tcam, sect_rx->tcam, sizeof(*sect_tx->tcam)); + + status = ice_update_pkg(hw, ice_pkg_buf(bld), 1); + if (!status) { + hw->tnl.tbl[index].port = port; + hw->tnl.tbl[index].in_use = true; + hw->tnl.tbl[index].ref = 1; + } + +ice_create_tunnel_err: + ice_pkg_buf_free(hw, bld); + +ice_create_tunnel_end: + mutex_unlock(&hw->tnl_lock); + + return status; +} + +/** + * ice_destroy_tunnel + * @hw: pointer to the HW structure + * @port: port of tunnel to destroy (ignored if the all parameter is true) + * @all: flag that states to destroy all tunnels + * + * Destroys a tunnel or all tunnels by creating an update package buffer + * targeting the specific updates requested and then performing an update + * package. + */ +enum ice_status ice_destroy_tunnel(struct ice_hw *hw, u16 port, bool all) +{ + struct ice_boost_tcam_section *sect_rx, *sect_tx; + enum ice_status status = ICE_ERR_MAX_LIMIT; + struct ice_buf_build *bld; + u16 count = 0; + u16 index; + u16 size; + u16 i; + + mutex_lock(&hw->tnl_lock); + + if (!all && ice_tunnel_port_in_use_hlpr(hw, port, &index)) + if (hw->tnl.tbl[index].ref > 1) { + hw->tnl.tbl[index].ref--; + status = 0; + goto ice_destroy_tunnel_end; + } + + /* determine count */ + for (i = 0; i < hw->tnl.count && i < ICE_TUNNEL_MAX_ENTRIES; i++) + if (hw->tnl.tbl[i].valid && hw->tnl.tbl[i].in_use && + (all || hw->tnl.tbl[i].port == port)) + count++; + + if (!count) { + status = ICE_ERR_PARAM; + goto ice_destroy_tunnel_end; + } + + /* size of section - there is at least one entry */ + size = struct_size(sect_rx, tcam, count - 1); + + bld = ice_pkg_buf_alloc(hw); + if (!bld) { + status = ICE_ERR_NO_MEMORY; + goto ice_destroy_tunnel_end; + } + + /* allocate 2 sections, one for Rx parser, one for Tx parser */ + if (ice_pkg_buf_reserve_section(bld, 2)) + goto ice_destroy_tunnel_err; + + sect_rx = ice_pkg_buf_alloc_section(bld, ICE_SID_RXPARSER_BOOST_TCAM, + size); + if (!sect_rx) + goto ice_destroy_tunnel_err; + sect_rx->count = cpu_to_le16(1); + + sect_tx = ice_pkg_buf_alloc_section(bld, ICE_SID_TXPARSER_BOOST_TCAM, + size); + if (!sect_tx) + goto ice_destroy_tunnel_err; + sect_tx->count = cpu_to_le16(1); + + /* copy original boost entry to update package buffer, one copy to Rx + * section, another copy to the Tx section + */ + for (i = 0; i < hw->tnl.count && i < ICE_TUNNEL_MAX_ENTRIES; i++) + if (hw->tnl.tbl[i].valid && hw->tnl.tbl[i].in_use && + (all || hw->tnl.tbl[i].port == port)) { + memcpy(sect_rx->tcam + i, hw->tnl.tbl[i].boost_entry, + sizeof(*sect_rx->tcam)); + memcpy(sect_tx->tcam + i, hw->tnl.tbl[i].boost_entry, + sizeof(*sect_tx->tcam)); + hw->tnl.tbl[i].marked = true; + } + + status = ice_update_pkg(hw, ice_pkg_buf(bld), 1); + if (!status) + for (i = 0; i < hw->tnl.count && + i < ICE_TUNNEL_MAX_ENTRIES; i++) + if (hw->tnl.tbl[i].marked) { + hw->tnl.tbl[i].ref = 0; + hw->tnl.tbl[i].port = 0; + hw->tnl.tbl[i].in_use = false; + hw->tnl.tbl[i].marked = false; + } + +ice_destroy_tunnel_err: + ice_pkg_buf_free(hw, bld); + +ice_destroy_tunnel_end: + mutex_unlock(&hw->tnl_lock); + + return status; +} + /* PTG Management */ /** @@ -1807,9 +2419,16 @@ ice_find_prof_id(struct ice_hw *hw, enum ice_block blk, struct ice_fv_word *fv, u8 *prof_id) { struct ice_es *es = &hw->blk[blk].es; - u16 off, i; + u16 off; + u8 i; - for (i = 0; i < es->count; i++) { + /* For FD, we don't want to re-use a existed profile with the same + * field vector and mask. This will cause rule interference. + */ + if (blk == ICE_BLK_FD) + return ICE_ERR_DOES_NOT_EXIST; + + for (i = 0; i < (u8)es->count; i++) { off = i * es->fvw; if (memcmp(&es->t[off], fv, es->fvw * sizeof(*fv))) @@ -1830,6 +2449,9 @@ ice_find_prof_id(struct ice_hw *hw, enum ice_block blk, static bool ice_prof_id_rsrc_type(enum ice_block blk, u16 *rsrc_type) { switch (blk) { + case ICE_BLK_FD: + *rsrc_type = ICE_AQC_RES_TYPE_FD_PROF_BLDR_PROFID; + break; case ICE_BLK_RSS: *rsrc_type = ICE_AQC_RES_TYPE_HASH_PROF_BLDR_PROFID; break; @@ -1847,6 +2469,9 @@ static bool ice_prof_id_rsrc_type(enum ice_block blk, u16 *rsrc_type) static bool ice_tcam_ent_rsrc_type(enum ice_block blk, u16 *rsrc_type) { switch (blk) { + case ICE_BLK_FD: + *rsrc_type = ICE_AQC_RES_TYPE_FD_PROF_BLDR_TCAM; + break; case ICE_BLK_RSS: *rsrc_type = ICE_AQC_RES_TYPE_HASH_PROF_BLDR_TCAM; break; @@ -2290,6 +2915,12 @@ static void ice_free_flow_profs(struct ice_hw *hw, u8 blk_idx) mutex_lock(&hw->fl_profs_locks[blk_idx]); list_for_each_entry_safe(p, tmp, &hw->fl_profs[blk_idx], l_entry) { + struct ice_flow_entry *e, *t; + + list_for_each_entry_safe(e, t, &p->entries, l_entry) + ice_flow_rem_entry(hw, (enum ice_block)blk_idx, + ICE_FLOW_ENTRY_HNDL(e)); + list_del(&p->l_entry); devm_kfree(ice_hw_to_dev(hw), p); } @@ -2919,6 +3550,212 @@ error_tmp: } /** + * ice_update_fd_mask - set Flow Director Field Vector mask for a profile + * @hw: pointer to the HW struct + * @prof_id: profile ID + * @mask_sel: mask select + * + * This function enable any of the masks selected by the mask select parameter + * for the profile specified. + */ +static void ice_update_fd_mask(struct ice_hw *hw, u16 prof_id, u32 mask_sel) +{ + wr32(hw, GLQF_FDMASK_SEL(prof_id), mask_sel); + + ice_debug(hw, ICE_DBG_INIT, "fd mask(%d): %x = %x\n", prof_id, + GLQF_FDMASK_SEL(prof_id), mask_sel); +} + +struct ice_fd_src_dst_pair { + u8 prot_id; + u8 count; + u16 off; +}; + +static const struct ice_fd_src_dst_pair ice_fd_pairs[] = { + /* These are defined in pairs */ + { ICE_PROT_IPV4_OF_OR_S, 2, 12 }, + { ICE_PROT_IPV4_OF_OR_S, 2, 16 }, + + { ICE_PROT_IPV4_IL, 2, 12 }, + { ICE_PROT_IPV4_IL, 2, 16 }, + + { ICE_PROT_IPV6_OF_OR_S, 8, 8 }, + { ICE_PROT_IPV6_OF_OR_S, 8, 24 }, + + { ICE_PROT_IPV6_IL, 8, 8 }, + { ICE_PROT_IPV6_IL, 8, 24 }, + + { ICE_PROT_TCP_IL, 1, 0 }, + { ICE_PROT_TCP_IL, 1, 2 }, + + { ICE_PROT_UDP_OF, 1, 0 }, + { ICE_PROT_UDP_OF, 1, 2 }, + + { ICE_PROT_UDP_IL_OR_S, 1, 0 }, + { ICE_PROT_UDP_IL_OR_S, 1, 2 }, + + { ICE_PROT_SCTP_IL, 1, 0 }, + { ICE_PROT_SCTP_IL, 1, 2 } +}; + +#define ICE_FD_SRC_DST_PAIR_COUNT ARRAY_SIZE(ice_fd_pairs) + +/** + * ice_update_fd_swap - set register appropriately for a FD FV extraction + * @hw: pointer to the HW struct + * @prof_id: profile ID + * @es: extraction sequence (length of array is determined by the block) + */ +static enum ice_status +ice_update_fd_swap(struct ice_hw *hw, u16 prof_id, struct ice_fv_word *es) +{ + DECLARE_BITMAP(pair_list, ICE_FD_SRC_DST_PAIR_COUNT); + u8 pair_start[ICE_FD_SRC_DST_PAIR_COUNT] = { 0 }; +#define ICE_FD_FV_NOT_FOUND (-2) + s8 first_free = ICE_FD_FV_NOT_FOUND; + u8 used[ICE_MAX_FV_WORDS] = { 0 }; + s8 orig_free, si; + u32 mask_sel = 0; + u8 i, j, k; + + bitmap_zero(pair_list, ICE_FD_SRC_DST_PAIR_COUNT); + + /* This code assumes that the Flow Director field vectors are assigned + * from the end of the FV indexes working towards the zero index, that + * only complete fields will be included and will be consecutive, and + * that there are no gaps between valid indexes. + */ + + /* Determine swap fields present */ + for (i = 0; i < hw->blk[ICE_BLK_FD].es.fvw; i++) { + /* Find the first free entry, assuming right to left population. + * This is where we can start adding additional pairs if needed. + */ + if (first_free == ICE_FD_FV_NOT_FOUND && es[i].prot_id != + ICE_PROT_INVALID) + first_free = i - 1; + + for (j = 0; j < ICE_FD_SRC_DST_PAIR_COUNT; j++) + if (es[i].prot_id == ice_fd_pairs[j].prot_id && + es[i].off == ice_fd_pairs[j].off) { + set_bit(j, pair_list); + pair_start[j] = i; + } + } + + orig_free = first_free; + + /* determine missing swap fields that need to be added */ + for (i = 0; i < ICE_FD_SRC_DST_PAIR_COUNT; i += 2) { + u8 bit1 = test_bit(i + 1, pair_list); + u8 bit0 = test_bit(i, pair_list); + + if (bit0 ^ bit1) { + u8 index; + + /* add the appropriate 'paired' entry */ + if (!bit0) + index = i; + else + index = i + 1; + + /* check for room */ + if (first_free + 1 < (s8)ice_fd_pairs[index].count) + return ICE_ERR_MAX_LIMIT; + + /* place in extraction sequence */ + for (k = 0; k < ice_fd_pairs[index].count; k++) { + es[first_free - k].prot_id = + ice_fd_pairs[index].prot_id; + es[first_free - k].off = + ice_fd_pairs[index].off + (k * 2); + + if (k > first_free) + return ICE_ERR_OUT_OF_RANGE; + + /* keep track of non-relevant fields */ + mask_sel |= BIT(first_free - k); + } + + pair_start[index] = first_free; + first_free -= ice_fd_pairs[index].count; + } + } + + /* fill in the swap array */ + si = hw->blk[ICE_BLK_FD].es.fvw - 1; + while (si >= 0) { + u8 indexes_used = 1; + + /* assume flat at this index */ +#define ICE_SWAP_VALID 0x80 + used[si] = si | ICE_SWAP_VALID; + + if (orig_free == ICE_FD_FV_NOT_FOUND || si <= orig_free) { + si -= indexes_used; + continue; + } + + /* check for a swap location */ + for (j = 0; j < ICE_FD_SRC_DST_PAIR_COUNT; j++) + if (es[si].prot_id == ice_fd_pairs[j].prot_id && + es[si].off == ice_fd_pairs[j].off) { + u8 idx; + + /* determine the appropriate matching field */ + idx = j + ((j % 2) ? -1 : 1); + + indexes_used = ice_fd_pairs[idx].count; + for (k = 0; k < indexes_used; k++) { + used[si - k] = (pair_start[idx] - k) | + ICE_SWAP_VALID; + } + + break; + } + + si -= indexes_used; + } + + /* for each set of 4 swap and 4 inset indexes, write the appropriate + * register + */ + for (j = 0; j < hw->blk[ICE_BLK_FD].es.fvw / 4; j++) { + u32 raw_swap = 0; + u32 raw_in = 0; + + for (k = 0; k < 4; k++) { + u8 idx; + + idx = (j * 4) + k; + if (used[idx] && !(mask_sel & BIT(idx))) { + raw_swap |= used[idx] << (k * BITS_PER_BYTE); +#define ICE_INSET_DFLT 0x9f + raw_in |= ICE_INSET_DFLT << (k * BITS_PER_BYTE); + } + } + + /* write the appropriate swap register set */ + wr32(hw, GLQF_FDSWAP(prof_id, j), raw_swap); + + ice_debug(hw, ICE_DBG_INIT, "swap wr(%d, %d): %x = %08x\n", + prof_id, j, GLQF_FDSWAP(prof_id, j), raw_swap); + + /* write the appropriate inset register set */ + wr32(hw, GLQF_FDINSET(prof_id, j), raw_in); + + ice_debug(hw, ICE_DBG_INIT, "inset wr(%d, %d): %x = %08x\n", + prof_id, j, GLQF_FDINSET(prof_id, j), raw_in); + } + + /* initially clear the mask select for this profile */ + ice_update_fd_mask(hw, prof_id, 0); + + return 0; +} + +/** * ice_add_prof - add profile * @hw: pointer to the HW struct * @blk: hardware block @@ -2939,7 +3776,7 @@ ice_add_prof(struct ice_hw *hw, enum ice_block blk, u64 id, u8 ptypes[], DECLARE_BITMAP(ptgs_used, ICE_XLT1_CNT); struct ice_prof_map *prof; enum ice_status status; - u32 byte = 0; + u8 byte = 0; u8 prof_id; bitmap_zero(ptgs_used, ICE_XLT1_CNT); @@ -2953,6 +3790,18 @@ ice_add_prof(struct ice_hw *hw, enum ice_block blk, u64 id, u8 ptypes[], status = ice_alloc_prof_id(hw, blk, &prof_id); if (status) goto err_ice_add_prof; + if (blk == ICE_BLK_FD) { + /* For Flow Director block, the extraction sequence may + * need to be altered in the case where there are paired + * fields that have no match. This is necessary because + * for Flow Director, src and dest fields need to paired + * for filter programming and these values are swapped + * during Tx. + */ + status = ice_update_fd_swap(hw, prof_id, es); + if (status) + goto err_ice_add_prof; + } /* and write new es */ ice_write_es(hw, blk, prof_id, es); @@ -2962,8 +3811,10 @@ ice_add_prof(struct ice_hw *hw, enum ice_block blk, u64 id, u8 ptypes[], /* add profile info */ prof = devm_kzalloc(ice_hw_to_dev(hw), sizeof(*prof), GFP_KERNEL); - if (!prof) + if (!prof) { + status = ICE_ERR_NO_MEMORY; goto err_ice_add_prof; + } prof->profile_cookie = id; prof->prof_id = prof_id; @@ -2972,7 +3823,7 @@ ice_add_prof(struct ice_hw *hw, enum ice_block blk, u64 id, u8 ptypes[], /* build list of ptgs */ while (bytes && prof->ptg_cnt < ICE_MAX_PTG_PER_PROFILE) { - u32 bit; + u8 bit; if (!ptypes[byte]) { bytes--; @@ -3006,7 +3857,7 @@ ice_add_prof(struct ice_hw *hw, enum ice_block blk, u64 id, u8 ptypes[], break; /* nothing left in byte, then exit */ - m = ~((1 << (bit + 1)) - 1); + m = ~(u8)((1 << (bit + 1)) - 1); if (!(ptypes[byte] & m)) break; } @@ -3703,8 +4554,10 @@ ice_add_prof_id_vsig(struct ice_hw *hw, enum ice_block blk, u16 vsig, u64 hdl, t->tcam[i].prof_id, t->tcam[i].ptg, vsig, 0, 0, vl_msk, dc_msk, nm_msk); - if (status) + if (status) { + devm_kfree(ice_hw_to_dev(hw), p); goto err_ice_add_prof_id_vsig; + } /* log change */ list_add(&p->list_entry, chg); diff --git a/drivers/net/ethernet/intel/ice/ice_flex_pipe.h b/drivers/net/ethernet/intel/ice/ice_flex_pipe.h index c7b5e1a6ea2b..568ea519af51 100644 --- a/drivers/net/ethernet/intel/ice/ice_flex_pipe.h +++ b/drivers/net/ethernet/intel/ice/ice_flex_pipe.h @@ -18,6 +18,14 @@ #define ICE_PKG_CNT 4 +bool +ice_get_open_tunnel_port(struct ice_hw *hw, enum ice_tunnel_type type, + u16 *port); +enum ice_status +ice_create_tunnel(struct ice_hw *hw, enum ice_tunnel_type type, u16 port); +enum ice_status ice_destroy_tunnel(struct ice_hw *hw, u16 port, bool all); +bool ice_tunnel_port_in_use(struct ice_hw *hw, u16 port, u16 *index); + enum ice_status ice_add_prof(struct ice_hw *hw, enum ice_block blk, u64 id, u8 ptypes[], struct ice_fv_word *es); diff --git a/drivers/net/ethernet/intel/ice/ice_flex_type.h b/drivers/net/ethernet/intel/ice/ice_flex_type.h index 0fb3fe3ff3ea..a6f391eac8ff 100644 --- a/drivers/net/ethernet/intel/ice/ice_flex_type.h +++ b/drivers/net/ethernet/intel/ice/ice_flex_type.h @@ -20,7 +20,7 @@ struct ice_fv { /* Package and segment headers and tables */ struct ice_pkg_hdr { - struct ice_pkg_ver format_ver; + struct ice_pkg_ver pkg_format_ver; __le32 seg_count; __le32 seg_offset[1]; }; @@ -30,9 +30,9 @@ struct ice_generic_seg_hdr { #define SEGMENT_TYPE_METADATA 0x00000001 #define SEGMENT_TYPE_ICE 0x00000010 __le32 seg_type; - struct ice_pkg_ver seg_ver; + struct ice_pkg_ver seg_format_ver; __le32 seg_size; - char seg_name[ICE_PKG_NAME_SIZE]; + char seg_id[ICE_PKG_NAME_SIZE]; }; /* ice specific segment */ @@ -75,7 +75,7 @@ struct ice_buf_table { struct ice_global_metadata_seg { struct ice_generic_seg_hdr hdr; struct ice_pkg_ver pkg_ver; - __le32 track_id; + __le32 rsvd; char pkg_name[ICE_PKG_NAME_SIZE]; }; @@ -149,6 +149,7 @@ struct ice_buf_hdr { #define ICE_SID_CDID_REDIR_RSS 48 #define ICE_SID_RXPARSER_BOOST_TCAM 56 +#define ICE_SID_TXPARSER_BOOST_TCAM 66 #define ICE_SID_XLT0_PE 80 #define ICE_SID_XLT_KEY_BUILDER_PE 81 @@ -291,6 +292,38 @@ struct ice_pkg_enum { void *(*handler)(u32 sect_type, void *section, u32 index, u32 *offset); }; +/* Tunnel enabling */ + +enum ice_tunnel_type { + TNL_VXLAN = 0, + TNL_GENEVE, + TNL_LAST = 0xFF, + TNL_ALL = 0xFF, +}; + +struct ice_tunnel_type_scan { + enum ice_tunnel_type type; + const char *label_prefix; +}; + +struct ice_tunnel_entry { + enum ice_tunnel_type type; + u16 boost_addr; + u16 port; + u16 ref; + struct ice_boost_tcam_entry *boost_entry; + u8 valid; + u8 in_use; + u8 marked; +}; + +#define ICE_TUNNEL_MAX_ENTRIES 16 + +struct ice_tunnel_table { + struct ice_tunnel_entry tbl[ICE_TUNNEL_MAX_ENTRIES]; + u16 count; +}; + struct ice_pkg_es { __le16 count; __le16 offset; diff --git a/drivers/net/ethernet/intel/ice/ice_flow.c b/drivers/net/ethernet/intel/ice/ice_flow.c index 3de862a3c789..d74e5290677f 100644 --- a/drivers/net/ethernet/intel/ice/ice_flow.c +++ b/drivers/net/ethernet/intel/ice/ice_flow.c @@ -42,7 +42,10 @@ struct ice_flow_field_info ice_flds_info[ICE_FLOW_FIELD_IDX_MAX] = { ICE_FLOW_FLD_INFO(ICE_FLOW_SEG_HDR_SCTP, 0, sizeof(__be16)), /* ICE_FLOW_FIELD_IDX_SCTP_DST_PORT */ ICE_FLOW_FLD_INFO(ICE_FLOW_SEG_HDR_SCTP, 2, sizeof(__be16)), - + /* GRE */ + /* ICE_FLOW_FIELD_IDX_GRE_KEYID */ + ICE_FLOW_FLD_INFO(ICE_FLOW_SEG_HDR_GRE, 12, + sizeof_field(struct gre_full_hdr, key)), }; /* Bitmaps indicating relevant packet types for a particular protocol header @@ -134,6 +137,18 @@ static const u32 ice_ptypes_sctp_il[] = { 0x00000000, 0x00000000, 0x00000000, 0x00000000, }; +/* Packet types for packets with an Outermost/First GRE header */ +static const u32 ice_ptypes_gre_of[] = { + 0x00000000, 0xBFBF7800, 0x000001DF, 0xFEFDE000, + 0x0000017E, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, +}; + /* Manage parameters and info. used during the creation of a flow profile */ struct ice_flow_prof_params { enum ice_block blk; @@ -178,6 +193,40 @@ ice_flow_val_hdrs(struct ice_flow_seg_info *segs, u8 segs_cnt) return 0; } +/* Sizes of fixed known protocol headers without header options */ +#define ICE_FLOW_PROT_HDR_SZ_MAC 14 +#define ICE_FLOW_PROT_HDR_SZ_IPV4 20 +#define ICE_FLOW_PROT_HDR_SZ_IPV6 40 +#define ICE_FLOW_PROT_HDR_SZ_TCP 20 +#define ICE_FLOW_PROT_HDR_SZ_UDP 8 +#define ICE_FLOW_PROT_HDR_SZ_SCTP 12 + +/** + * ice_flow_calc_seg_sz - calculates size of a packet segment based on headers + * @params: information about the flow to be processed + * @seg: index of packet segment whose header size is to be determined + */ +static u16 ice_flow_calc_seg_sz(struct ice_flow_prof_params *params, u8 seg) +{ + u16 sz = ICE_FLOW_PROT_HDR_SZ_MAC; + + /* L3 headers */ + if (params->prof->segs[seg].hdrs & ICE_FLOW_SEG_HDR_IPV4) + sz += ICE_FLOW_PROT_HDR_SZ_IPV4; + else if (params->prof->segs[seg].hdrs & ICE_FLOW_SEG_HDR_IPV6) + sz += ICE_FLOW_PROT_HDR_SZ_IPV6; + + /* L4 headers */ + if (params->prof->segs[seg].hdrs & ICE_FLOW_SEG_HDR_TCP) + sz += ICE_FLOW_PROT_HDR_SZ_TCP; + else if (params->prof->segs[seg].hdrs & ICE_FLOW_SEG_HDR_UDP) + sz += ICE_FLOW_PROT_HDR_SZ_UDP; + else if (params->prof->segs[seg].hdrs & ICE_FLOW_SEG_HDR_SCTP) + sz += ICE_FLOW_PROT_HDR_SZ_SCTP; + + return sz; +} + /** * ice_flow_proc_seg_hdrs - process protocol headers present in pkt segments * @params: information about the flow to be processed @@ -225,6 +274,12 @@ ice_flow_proc_seg_hdrs(struct ice_flow_prof_params *params) src = (const unsigned long *)ice_ptypes_sctp_il; bitmap_and(params->ptypes, params->ptypes, src, ICE_FLOW_PTYPE_MAX); + } else if (hdrs & ICE_FLOW_SEG_HDR_GRE) { + if (!i) { + src = (const unsigned long *)ice_ptypes_gre_of; + bitmap_and(params->ptypes, params->ptypes, + src, ICE_FLOW_PTYPE_MAX); + } } } @@ -275,6 +330,9 @@ ice_flow_xtract_fld(struct ice_hw *hw, struct ice_flow_prof_params *params, case ICE_FLOW_FIELD_IDX_SCTP_DST_PORT: prot_id = ICE_PROT_SCTP_IL; break; + case ICE_FLOW_FIELD_IDX_GRE_KEYID: + prot_id = ICE_PROT_GRE_OF; + break; default: return ICE_ERR_NOT_IMPL; } @@ -324,6 +382,81 @@ ice_flow_xtract_fld(struct ice_hw *hw, struct ice_flow_prof_params *params, } /** + * ice_flow_xtract_raws - Create extract sequence entries for raw bytes + * @hw: pointer to the HW struct + * @params: information about the flow to be processed + * @seg: index of packet segment whose raw fields are to be be extracted + */ +static enum ice_status +ice_flow_xtract_raws(struct ice_hw *hw, struct ice_flow_prof_params *params, + u8 seg) +{ + u16 fv_words; + u16 hdrs_sz; + u8 i; + + if (!params->prof->segs[seg].raws_cnt) + return 0; + + if (params->prof->segs[seg].raws_cnt > + ARRAY_SIZE(params->prof->segs[seg].raws)) + return ICE_ERR_MAX_LIMIT; + + /* Offsets within the segment headers are not supported */ + hdrs_sz = ice_flow_calc_seg_sz(params, seg); + if (!hdrs_sz) + return ICE_ERR_PARAM; + + fv_words = hw->blk[params->blk].es.fvw; + + for (i = 0; i < params->prof->segs[seg].raws_cnt; i++) { + struct ice_flow_seg_fld_raw *raw; + u16 off, cnt, j; + + raw = ¶ms->prof->segs[seg].raws[i]; + + /* Storing extraction information */ + raw->info.xtrct.prot_id = ICE_PROT_MAC_OF_OR_S; + raw->info.xtrct.off = (raw->off / ICE_FLOW_FV_EXTRACT_SZ) * + ICE_FLOW_FV_EXTRACT_SZ; + raw->info.xtrct.disp = (raw->off % ICE_FLOW_FV_EXTRACT_SZ) * + BITS_PER_BYTE; + raw->info.xtrct.idx = params->es_cnt; + + /* Determine the number of field vector entries this raw field + * consumes. + */ + cnt = DIV_ROUND_UP(raw->info.xtrct.disp + + (raw->info.src.last * BITS_PER_BYTE), + (ICE_FLOW_FV_EXTRACT_SZ * BITS_PER_BYTE)); + off = raw->info.xtrct.off; + for (j = 0; j < cnt; j++) { + u16 idx; + + /* Make sure the number of extraction sequence required + * does not exceed the block's capability + */ + if (params->es_cnt >= hw->blk[params->blk].es.count || + params->es_cnt >= ICE_MAX_FV_WORDS) + return ICE_ERR_MAX_LIMIT; + + /* some blocks require a reversed field vector layout */ + if (hw->blk[params->blk].es.reverse) + idx = fv_words - params->es_cnt - 1; + else + idx = params->es_cnt; + + params->es[idx].prot_id = raw->info.xtrct.prot_id; + params->es[idx].off = off; + params->es_cnt++; + off += ICE_FLOW_FV_EXTRACT_SZ; + } + } + + return 0; +} + +/** * ice_flow_create_xtrct_seq - Create an extraction sequence for given segments * @hw: pointer to the HW struct * @params: information about the flow to be processed @@ -349,6 +482,11 @@ ice_flow_create_xtrct_seq(struct ice_hw *hw, if (status) return status; } + + /* Process raw matching bytes */ + status = ice_flow_xtract_raws(hw, params, i); + if (status) + return status; } return status; @@ -373,10 +511,8 @@ ice_flow_proc_segs(struct ice_hw *hw, struct ice_flow_prof_params *params) return status; switch (params->blk) { + case ICE_BLK_FD: case ICE_BLK_RSS: - /* Only header information is provided for RSS configuration. - * No further processing is needed. - */ status = 0; break; default: @@ -458,6 +594,43 @@ ice_flow_find_prof_id(struct ice_hw *hw, enum ice_block blk, u64 prof_id) } /** + * ice_dealloc_flow_entry - Deallocate flow entry memory + * @hw: pointer to the HW struct + * @entry: flow entry to be removed + */ +static void +ice_dealloc_flow_entry(struct ice_hw *hw, struct ice_flow_entry *entry) +{ + if (!entry) + return; + + if (entry->entry) + devm_kfree(ice_hw_to_dev(hw), entry->entry); + + devm_kfree(ice_hw_to_dev(hw), entry); +} + +/** + * ice_flow_rem_entry_sync - Remove a flow entry + * @hw: pointer to the HW struct + * @blk: classification stage + * @entry: flow entry to be removed + */ +static enum ice_status +ice_flow_rem_entry_sync(struct ice_hw *hw, enum ice_block __always_unused blk, + struct ice_flow_entry *entry) +{ + if (!entry) + return ICE_ERR_BAD_PTR; + + list_del(&entry->l_entry); + + ice_dealloc_flow_entry(hw, entry); + + return 0; +} + +/** * ice_flow_add_prof_sync - Add a flow profile for packet segments and fields * @hw: pointer to the HW struct * @blk: classification stage @@ -544,6 +717,21 @@ ice_flow_rem_prof_sync(struct ice_hw *hw, enum ice_block blk, { enum ice_status status; + /* Remove all remaining flow entries before removing the flow profile */ + if (!list_empty(&prof->entries)) { + struct ice_flow_entry *e, *t; + + mutex_lock(&prof->entries_lock); + + list_for_each_entry_safe(e, t, &prof->entries, l_entry) { + status = ice_flow_rem_entry_sync(hw, blk, e); + if (status) + break; + } + + mutex_unlock(&prof->entries_lock); + } + /* Remove all hardware profiles associated with this flow profile */ status = ice_rem_prof(hw, blk, prof->id); if (!status) { @@ -629,7 +817,7 @@ ice_flow_disassoc_prof(struct ice_hw *hw, enum ice_block blk, * @segs_cnt: number of packet segments provided * @prof: stores the returned flow profile added */ -static enum ice_status +enum ice_status ice_flow_add_prof(struct ice_hw *hw, enum ice_block blk, enum ice_flow_dir dir, u64 prof_id, struct ice_flow_seg_info *segs, u8 segs_cnt, struct ice_flow_prof **prof) @@ -667,7 +855,7 @@ ice_flow_add_prof(struct ice_hw *hw, enum ice_block blk, enum ice_flow_dir dir, * @blk: the block for which the flow profile is to be removed * @prof_id: unique ID of the flow profile to be removed */ -static enum ice_status +enum ice_status ice_flow_rem_prof(struct ice_hw *hw, enum ice_block blk, u64 prof_id) { struct ice_flow_prof *prof; @@ -691,6 +879,113 @@ out: } /** + * ice_flow_add_entry - Add a flow entry + * @hw: pointer to the HW struct + * @blk: classification stage + * @prof_id: ID of the profile to add a new flow entry to + * @entry_id: unique ID to identify this flow entry + * @vsi_handle: software VSI handle for the flow entry + * @prio: priority of the flow entry + * @data: pointer to a data buffer containing flow entry's match values/masks + * @entry_h: pointer to buffer that receives the new flow entry's handle + */ +enum ice_status +ice_flow_add_entry(struct ice_hw *hw, enum ice_block blk, u64 prof_id, + u64 entry_id, u16 vsi_handle, enum ice_flow_priority prio, + void *data, u64 *entry_h) +{ + struct ice_flow_entry *e = NULL; + struct ice_flow_prof *prof; + enum ice_status status; + + /* No flow entry data is expected for RSS */ + if (!entry_h || (!data && blk != ICE_BLK_RSS)) + return ICE_ERR_BAD_PTR; + + if (!ice_is_vsi_valid(hw, vsi_handle)) + return ICE_ERR_PARAM; + + mutex_lock(&hw->fl_profs_locks[blk]); + + prof = ice_flow_find_prof_id(hw, blk, prof_id); + if (!prof) { + status = ICE_ERR_DOES_NOT_EXIST; + } else { + /* Allocate memory for the entry being added and associate + * the VSI to the found flow profile + */ + e = devm_kzalloc(ice_hw_to_dev(hw), sizeof(*e), GFP_KERNEL); + if (!e) + status = ICE_ERR_NO_MEMORY; + else + status = ice_flow_assoc_prof(hw, blk, prof, vsi_handle); + } + + mutex_unlock(&hw->fl_profs_locks[blk]); + if (status) + goto out; + + e->id = entry_id; + e->vsi_handle = vsi_handle; + e->prof = prof; + e->priority = prio; + + switch (blk) { + case ICE_BLK_FD: + case ICE_BLK_RSS: + break; + default: + status = ICE_ERR_NOT_IMPL; + goto out; + } + + mutex_lock(&prof->entries_lock); + list_add(&e->l_entry, &prof->entries); + mutex_unlock(&prof->entries_lock); + + *entry_h = ICE_FLOW_ENTRY_HNDL(e); + +out: + if (status && e) { + if (e->entry) + devm_kfree(ice_hw_to_dev(hw), e->entry); + devm_kfree(ice_hw_to_dev(hw), e); + } + + return status; +} + +/** + * ice_flow_rem_entry - Remove a flow entry + * @hw: pointer to the HW struct + * @blk: classification stage + * @entry_h: handle to the flow entry to be removed + */ +enum ice_status ice_flow_rem_entry(struct ice_hw *hw, enum ice_block blk, + u64 entry_h) +{ + struct ice_flow_entry *entry; + struct ice_flow_prof *prof; + enum ice_status status = 0; + + if (entry_h == ICE_FLOW_ENTRY_HANDLE_INVAL) + return ICE_ERR_PARAM; + + entry = ICE_FLOW_ENTRY_PTR(entry_h); + + /* Retain the pointer to the flow profile as the entry will be freed */ + prof = entry->prof; + + if (prof) { + mutex_lock(&prof->entries_lock); + status = ice_flow_rem_entry_sync(hw, blk, entry); + mutex_unlock(&prof->entries_lock); + } + + return status; +} + +/** * ice_flow_set_fld_ext - specifies locations of field from entry's input buffer * @seg: packet segment the field being set belongs to * @fld: field to be set @@ -752,7 +1047,7 @@ ice_flow_set_fld_ext(struct ice_flow_seg_info *seg, enum ice_flow_field fld, * create the content of a match entry. This function should only be used for * fixed-size data structures. */ -static void +void ice_flow_set_fld(struct ice_flow_seg_info *seg, enum ice_flow_field fld, u16 val_loc, u16 mask_loc, u16 last_loc, bool range) { @@ -762,6 +1057,42 @@ ice_flow_set_fld(struct ice_flow_seg_info *seg, enum ice_flow_field fld, ice_flow_set_fld_ext(seg, fld, t, val_loc, mask_loc, last_loc); } +/** + * ice_flow_add_fld_raw - sets locations of a raw field from entry's input buf + * @seg: packet segment the field being set belongs to + * @off: offset of the raw field from the beginning of the segment in bytes + * @len: length of the raw pattern to be matched + * @val_loc: location of the value to match from entry's input buffer + * @mask_loc: location of mask value from entry's input buffer + * + * This function specifies the offset of the raw field to be match from the + * beginning of the specified packet segment, and the locations, in the form of + * byte offsets from the start of the input buffer for a flow entry, from where + * the value to match and the mask value to be extracted. These locations are + * then stored in the flow profile. When adding flow entries to the associated + * flow profile, these locations can be used to quickly extract the values to + * create the content of a match entry. This function should only be used for + * fixed-size data structures. + */ +void +ice_flow_add_fld_raw(struct ice_flow_seg_info *seg, u16 off, u8 len, + u16 val_loc, u16 mask_loc) +{ + if (seg->raws_cnt < ICE_FLOW_SEG_RAW_FLD_MAX) { + seg->raws[seg->raws_cnt].off = off; + seg->raws[seg->raws_cnt].info.type = ICE_FLOW_FLD_TYPE_SIZE; + seg->raws[seg->raws_cnt].info.src.val = val_loc; + seg->raws[seg->raws_cnt].info.src.mask = mask_loc; + /* The "last" field is used to store the length of the field */ + seg->raws[seg->raws_cnt].info.src.last = len; + } + + /* Overflows of "raws" will be handled as an error condition later in + * the flow when this information is processed. + */ + seg->raws_cnt++; +} + #define ICE_FLOW_RSS_SEG_HDR_L3_MASKS \ (ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV6) @@ -945,6 +1276,7 @@ ice_add_rss_list(struct ice_hw *hw, u16 vsi_handle, struct ice_flow_prof *prof) #define ICE_FLOW_PROF_ENCAP_M (BIT_ULL(ICE_FLOW_PROF_ENCAP_S)) #define ICE_RSS_OUTER_HEADERS 1 +#define ICE_RSS_INNER_HEADERS 2 /* Flow profile ID format: * [0:31] - Packet match fields @@ -1085,6 +1417,9 @@ ice_add_rss_cfg(struct ice_hw *hw, u16 vsi_handle, u64 hashed_flds, mutex_lock(&hw->rss_locks); status = ice_add_rss_cfg_sync(hw, vsi_handle, hashed_flds, addl_hdrs, ICE_RSS_OUTER_HEADERS); + if (!status) + status = ice_add_rss_cfg_sync(hw, vsi_handle, hashed_flds, + addl_hdrs, ICE_RSS_INNER_HEADERS); mutex_unlock(&hw->rss_locks); return status; @@ -1238,6 +1573,12 @@ enum ice_status ice_replay_rss_cfg(struct ice_hw *hw, u16 vsi_handle) ICE_RSS_OUTER_HEADERS); if (status) break; + status = ice_add_rss_cfg_sync(hw, vsi_handle, + r->hashed_flds, + r->packet_hdr, + ICE_RSS_INNER_HEADERS); + if (status) + break; } } mutex_unlock(&hw->rss_locks); diff --git a/drivers/net/ethernet/intel/ice/ice_flow.h b/drivers/net/ethernet/intel/ice/ice_flow.h index 5558627bd5eb..3913da2116d2 100644 --- a/drivers/net/ethernet/intel/ice/ice_flow.h +++ b/drivers/net/ethernet/intel/ice/ice_flow.h @@ -43,6 +43,7 @@ enum ice_flow_seg_hdr { ICE_FLOW_SEG_HDR_TCP = 0x00000040, ICE_FLOW_SEG_HDR_UDP = 0x00000080, ICE_FLOW_SEG_HDR_SCTP = 0x00000100, + ICE_FLOW_SEG_HDR_GRE = 0x00000200, }; enum ice_flow_field { @@ -58,6 +59,8 @@ enum ice_flow_field { ICE_FLOW_FIELD_IDX_UDP_DST_PORT, ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT, ICE_FLOW_FIELD_IDX_SCTP_DST_PORT, + /* GRE */ + ICE_FLOW_FIELD_IDX_GRE_KEYID, /* The total number of enums must not exceed 64 */ ICE_FLOW_FIELD_IDX_MAX }; @@ -125,6 +128,7 @@ enum ice_flow_priority { }; #define ICE_FLOW_SEG_MAX 2 +#define ICE_FLOW_SEG_RAW_FLD_MAX 2 #define ICE_FLOW_FV_EXTRACT_SZ 2 #define ICE_FLOW_SET_HDRS(seg, val) ((seg)->hdrs |= (u32)(val)) @@ -161,14 +165,38 @@ struct ice_flow_fld_info { struct ice_flow_seg_xtrct xtrct; }; +struct ice_flow_seg_fld_raw { + struct ice_flow_fld_info info; + u16 off; /* Offset from the start of the segment */ +}; + struct ice_flow_seg_info { u32 hdrs; /* Bitmask indicating protocol headers present */ u64 match; /* Bitmask indicating header fields to be matched */ u64 range; /* Bitmask indicating header fields matched as ranges */ struct ice_flow_fld_info fields[ICE_FLOW_FIELD_IDX_MAX]; + + u8 raws_cnt; /* Number of raw fields to be matched */ + struct ice_flow_seg_fld_raw raws[ICE_FLOW_SEG_RAW_FLD_MAX]; }; +/* This structure describes a flow entry, and is tracked only in this file */ +struct ice_flow_entry { + struct list_head l_entry; + + u64 id; + struct ice_flow_prof *prof; + /* Flow entry's content */ + void *entry; + enum ice_flow_priority priority; + u16 vsi_handle; + u16 entry_sz; +}; + +#define ICE_FLOW_ENTRY_HNDL(e) ((u64)e) +#define ICE_FLOW_ENTRY_PTR(h) ((struct ice_flow_entry *)(h)) + struct ice_flow_prof { struct list_head l_entry; @@ -194,7 +222,24 @@ struct ice_rss_cfg { u32 packet_hdr; }; -enum ice_status ice_flow_rem_entry(struct ice_hw *hw, u64 entry_h); +enum ice_status +ice_flow_add_prof(struct ice_hw *hw, enum ice_block blk, enum ice_flow_dir dir, + u64 prof_id, struct ice_flow_seg_info *segs, u8 segs_cnt, + struct ice_flow_prof **prof); +enum ice_status +ice_flow_rem_prof(struct ice_hw *hw, enum ice_block blk, u64 prof_id); +enum ice_status +ice_flow_add_entry(struct ice_hw *hw, enum ice_block blk, u64 prof_id, + u64 entry_id, u16 vsi, enum ice_flow_priority prio, + void *data, u64 *entry_h); +enum ice_status +ice_flow_rem_entry(struct ice_hw *hw, enum ice_block blk, u64 entry_h); +void +ice_flow_set_fld(struct ice_flow_seg_info *seg, enum ice_flow_field fld, + u16 val_loc, u16 mask_loc, u16 last_loc, bool range); +void +ice_flow_add_fld_raw(struct ice_flow_seg_info *seg, u16 off, u8 len, + u16 val_loc, u16 mask_loc); void ice_rem_vsi_rss_list(struct ice_hw *hw, u16 vsi_handle); enum ice_status ice_replay_rss_cfg(struct ice_hw *hw, u16 vsi_handle); enum ice_status diff --git a/drivers/net/ethernet/intel/ice/ice_fltr.c b/drivers/net/ethernet/intel/ice/ice_fltr.c new file mode 100644 index 000000000000..2418d4fff037 --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_fltr.c @@ -0,0 +1,397 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2018-2020, Intel Corporation. */ + +#include "ice.h" +#include "ice_fltr.h" + +/** + * ice_fltr_free_list - free filter lists helper + * @dev: pointer to the device struct + * @h: pointer to the list head to be freed + * + * Helper function to free filter lists previously created using + * ice_fltr_add_mac_to_list + */ +void ice_fltr_free_list(struct device *dev, struct list_head *h) +{ + struct ice_fltr_list_entry *e, *tmp; + + list_for_each_entry_safe(e, tmp, h, list_entry) { + list_del(&e->list_entry); + devm_kfree(dev, e); + } +} + +/** + * ice_fltr_add_entry_to_list - allocate and add filter entry to list + * @dev: pointer to device needed by alloc function + * @info: filter info struct that gets added to the passed in list + * @list: pointer to the list which contains MAC filters entry + */ +static int +ice_fltr_add_entry_to_list(struct device *dev, struct ice_fltr_info *info, + struct list_head *list) +{ + struct ice_fltr_list_entry *entry; + + entry = devm_kzalloc(dev, sizeof(*entry), GFP_ATOMIC); + if (!entry) + return -ENOMEM; + + entry->fltr_info = *info; + + INIT_LIST_HEAD(&entry->list_entry); + list_add(&entry->list_entry, list); + + return 0; +} + +/** + * ice_fltr_add_mac_list - add list of MAC filters + * @vsi: pointer to VSI struct + * @list: list of filters + */ +enum ice_status +ice_fltr_add_mac_list(struct ice_vsi *vsi, struct list_head *list) +{ + return ice_add_mac(&vsi->back->hw, list); +} + +/** + * ice_fltr_remove_mac_list - remove list of MAC filters + * @vsi: pointer to VSI struct + * @list: list of filters + */ +enum ice_status +ice_fltr_remove_mac_list(struct ice_vsi *vsi, struct list_head *list) +{ + return ice_remove_mac(&vsi->back->hw, list); +} + +/** + * ice_fltr_add_vlan_list - add list of VLAN filters + * @vsi: pointer to VSI struct + * @list: list of filters + */ +static enum ice_status +ice_fltr_add_vlan_list(struct ice_vsi *vsi, struct list_head *list) +{ + return ice_add_vlan(&vsi->back->hw, list); +} + +/** + * ice_fltr_remove_vlan_list - remove list of VLAN filters + * @vsi: pointer to VSI struct + * @list: list of filters + */ +static enum ice_status +ice_fltr_remove_vlan_list(struct ice_vsi *vsi, struct list_head *list) +{ + return ice_remove_vlan(&vsi->back->hw, list); +} + +/** + * ice_fltr_add_eth_list - add list of ethertype filters + * @vsi: pointer to VSI struct + * @list: list of filters + */ +static enum ice_status +ice_fltr_add_eth_list(struct ice_vsi *vsi, struct list_head *list) +{ + return ice_add_eth_mac(&vsi->back->hw, list); +} + +/** + * ice_fltr_remove_eth_list - remove list of ethertype filters + * @vsi: pointer to VSI struct + * @list: list of filters + */ +static enum ice_status +ice_fltr_remove_eth_list(struct ice_vsi *vsi, struct list_head *list) +{ + return ice_remove_eth_mac(&vsi->back->hw, list); +} + +/** + * ice_fltr_remove_all - remove all filters associated with VSI + * @vsi: pointer to VSI struct + */ +void ice_fltr_remove_all(struct ice_vsi *vsi) +{ + ice_remove_vsi_fltr(&vsi->back->hw, vsi->idx); +} + +/** + * ice_fltr_add_mac_to_list - add MAC filter info to exsisting list + * @vsi: pointer to VSI struct + * @list: list to add filter info to + * @mac: MAC address to add + * @action: filter action + */ +int +ice_fltr_add_mac_to_list(struct ice_vsi *vsi, struct list_head *list, + const u8 *mac, enum ice_sw_fwd_act_type action) +{ + struct ice_fltr_info info = { 0 }; + + info.flag = ICE_FLTR_TX; + info.src_id = ICE_SRC_ID_VSI; + info.lkup_type = ICE_SW_LKUP_MAC; + info.fltr_act = action; + info.vsi_handle = vsi->idx; + + ether_addr_copy(info.l_data.mac.mac_addr, mac); + + return ice_fltr_add_entry_to_list(ice_pf_to_dev(vsi->back), &info, + list); +} + +/** + * ice_fltr_add_vlan_to_list - add VLAN filter info to exsisting list + * @vsi: pointer to VSI struct + * @list: list to add filter info to + * @vlan_id: VLAN ID to add + * @action: filter action + */ +static int +ice_fltr_add_vlan_to_list(struct ice_vsi *vsi, struct list_head *list, + u16 vlan_id, enum ice_sw_fwd_act_type action) +{ + struct ice_fltr_info info = { 0 }; + + info.flag = ICE_FLTR_TX; + info.src_id = ICE_SRC_ID_VSI; + info.lkup_type = ICE_SW_LKUP_VLAN; + info.fltr_act = action; + info.vsi_handle = vsi->idx; + info.l_data.vlan.vlan_id = vlan_id; + + return ice_fltr_add_entry_to_list(ice_pf_to_dev(vsi->back), &info, + list); +} + +/** + * ice_fltr_add_eth_to_list - add ethertype filter info to exsisting list + * @vsi: pointer to VSI struct + * @list: list to add filter info to + * @ethertype: ethertype of packet that matches filter + * @flag: filter direction, Tx or Rx + * @action: filter action + */ +static int +ice_fltr_add_eth_to_list(struct ice_vsi *vsi, struct list_head *list, + u16 ethertype, u16 flag, + enum ice_sw_fwd_act_type action) +{ + struct ice_fltr_info info = { 0 }; + + info.flag = flag; + info.lkup_type = ICE_SW_LKUP_ETHERTYPE; + info.fltr_act = action; + info.vsi_handle = vsi->idx; + info.l_data.ethertype_mac.ethertype = ethertype; + + if (flag == ICE_FLTR_TX) + info.src_id = ICE_SRC_ID_VSI; + else + info.src_id = ICE_SRC_ID_LPORT; + + return ice_fltr_add_entry_to_list(ice_pf_to_dev(vsi->back), &info, + list); +} + +/** + * ice_fltr_prepare_mac - add or remove MAC rule + * @vsi: pointer to VSI struct + * @mac: MAC address to add + * @action: action to be performed on filter match + * @mac_action: pointer to add or remove MAC function + */ +static enum ice_status +ice_fltr_prepare_mac(struct ice_vsi *vsi, const u8 *mac, + enum ice_sw_fwd_act_type action, + enum ice_status (*mac_action)(struct ice_vsi *, + struct list_head *)) +{ + enum ice_status result; + LIST_HEAD(tmp_list); + + if (ice_fltr_add_mac_to_list(vsi, &tmp_list, mac, action)) { + ice_fltr_free_list(ice_pf_to_dev(vsi->back), &tmp_list); + return ICE_ERR_NO_MEMORY; + } + + result = mac_action(vsi, &tmp_list); + ice_fltr_free_list(ice_pf_to_dev(vsi->back), &tmp_list); + return result; +} + +/** + * ice_fltr_prepare_mac_and_broadcast - add or remove MAC and broadcast filter + * @vsi: pointer to VSI struct + * @mac: MAC address to add + * @action: action to be performed on filter match + * @mac_action: pointer to add or remove MAC function + */ +static enum ice_status +ice_fltr_prepare_mac_and_broadcast(struct ice_vsi *vsi, const u8 *mac, + enum ice_sw_fwd_act_type action, + enum ice_status(*mac_action) + (struct ice_vsi *, struct list_head *)) +{ + u8 broadcast[ETH_ALEN]; + enum ice_status result; + LIST_HEAD(tmp_list); + + eth_broadcast_addr(broadcast); + if (ice_fltr_add_mac_to_list(vsi, &tmp_list, mac, action) || + ice_fltr_add_mac_to_list(vsi, &tmp_list, broadcast, action)) { + ice_fltr_free_list(ice_pf_to_dev(vsi->back), &tmp_list); + return ICE_ERR_NO_MEMORY; + } + + result = mac_action(vsi, &tmp_list); + ice_fltr_free_list(ice_pf_to_dev(vsi->back), &tmp_list); + return result; +} + +/** + * ice_fltr_prepare_vlan - add or remove VLAN filter + * @vsi: pointer to VSI struct + * @vlan_id: VLAN ID to add + * @action: action to be performed on filter match + * @vlan_action: pointer to add or remove VLAN function + */ +static enum ice_status +ice_fltr_prepare_vlan(struct ice_vsi *vsi, u16 vlan_id, + enum ice_sw_fwd_act_type action, + enum ice_status (*vlan_action)(struct ice_vsi *, + struct list_head *)) +{ + enum ice_status result; + LIST_HEAD(tmp_list); + + if (ice_fltr_add_vlan_to_list(vsi, &tmp_list, vlan_id, action)) + return ICE_ERR_NO_MEMORY; + + result = vlan_action(vsi, &tmp_list); + ice_fltr_free_list(ice_pf_to_dev(vsi->back), &tmp_list); + return result; +} + +/** + * ice_fltr_prepare_eth - add or remove ethertype filter + * @vsi: pointer to VSI struct + * @ethertype: ethertype of packet to be filtered + * @flag: direction of packet, Tx or Rx + * @action: action to be performed on filter match + * @eth_action: pointer to add or remove ethertype function + */ +static enum ice_status +ice_fltr_prepare_eth(struct ice_vsi *vsi, u16 ethertype, u16 flag, + enum ice_sw_fwd_act_type action, + enum ice_status (*eth_action)(struct ice_vsi *, + struct list_head *)) +{ + enum ice_status result; + LIST_HEAD(tmp_list); + + if (ice_fltr_add_eth_to_list(vsi, &tmp_list, ethertype, flag, action)) + return ICE_ERR_NO_MEMORY; + + result = eth_action(vsi, &tmp_list); + ice_fltr_free_list(ice_pf_to_dev(vsi->back), &tmp_list); + return result; +} + +/** + * ice_fltr_add_mac - add single MAC filter + * @vsi: pointer to VSI struct + * @mac: MAC to add + * @action: action to be performed on filter match + */ +enum ice_status ice_fltr_add_mac(struct ice_vsi *vsi, const u8 *mac, + enum ice_sw_fwd_act_type action) +{ + return ice_fltr_prepare_mac(vsi, mac, action, ice_fltr_add_mac_list); +} + +/** + * ice_fltr_add_mac_and_broadcast - add single MAC and broadcast + * @vsi: pointer to VSI struct + * @mac: MAC to add + * @action: action to be performed on filter match + */ +enum ice_status +ice_fltr_add_mac_and_broadcast(struct ice_vsi *vsi, const u8 *mac, + enum ice_sw_fwd_act_type action) +{ + return ice_fltr_prepare_mac_and_broadcast(vsi, mac, action, + ice_fltr_add_mac_list); +} + +/** + * ice_fltr_remove_mac - remove MAC filter + * @vsi: pointer to VSI struct + * @mac: filter MAC to remove + * @action: action to remove + */ +enum ice_status ice_fltr_remove_mac(struct ice_vsi *vsi, const u8 *mac, + enum ice_sw_fwd_act_type action) +{ + return ice_fltr_prepare_mac(vsi, mac, action, ice_fltr_remove_mac_list); +} + +/** + * ice_fltr_add_vlan - add single VLAN filter + * @vsi: pointer to VSI struct + * @vlan_id: VLAN ID to add + * @action: action to be performed on filter match + */ +enum ice_status ice_fltr_add_vlan(struct ice_vsi *vsi, u16 vlan_id, + enum ice_sw_fwd_act_type action) +{ + return ice_fltr_prepare_vlan(vsi, vlan_id, action, + ice_fltr_add_vlan_list); +} + +/** + * ice_fltr_remove_vlan - remove VLAN filter + * @vsi: pointer to VSI struct + * @vlan_id: filter VLAN to remove + * @action: action to remove + */ +enum ice_status ice_fltr_remove_vlan(struct ice_vsi *vsi, u16 vlan_id, + enum ice_sw_fwd_act_type action) +{ + return ice_fltr_prepare_vlan(vsi, vlan_id, action, + ice_fltr_remove_vlan_list); +} + +/** + * ice_fltr_add_eth - add specyfic ethertype filter + * @vsi: pointer to VSI struct + * @ethertype: ethertype of filter + * @flag: direction of packet to be filtered, Tx or Rx + * @action: action to be performed on filter match + */ +enum ice_status ice_fltr_add_eth(struct ice_vsi *vsi, u16 ethertype, u16 flag, + enum ice_sw_fwd_act_type action) +{ + return ice_fltr_prepare_eth(vsi, ethertype, flag, action, + ice_fltr_add_eth_list); +} + +/** + * ice_fltr_remove_eth - remove ethertype filter + * @vsi: pointer to VSI struct + * @ethertype: ethertype of filter + * @flag: direction of filter + * @action: action to remove + */ +enum ice_status ice_fltr_remove_eth(struct ice_vsi *vsi, u16 ethertype, + u16 flag, enum ice_sw_fwd_act_type action) +{ + return ice_fltr_prepare_eth(vsi, ethertype, flag, action, + ice_fltr_remove_eth_list); +} diff --git a/drivers/net/ethernet/intel/ice/ice_fltr.h b/drivers/net/ethernet/intel/ice/ice_fltr.h new file mode 100644 index 000000000000..361cb4da9b43 --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_fltr.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2018-2020, Intel Corporation. */ + +#ifndef _ICE_FLTR_H_ +#define _ICE_FLTR_H_ + +void ice_fltr_free_list(struct device *dev, struct list_head *h); +enum ice_status +ice_fltr_add_mac_to_list(struct ice_vsi *vsi, struct list_head *list, + const u8 *mac, enum ice_sw_fwd_act_type action); +enum ice_status +ice_fltr_add_mac(struct ice_vsi *vsi, const u8 *mac, + enum ice_sw_fwd_act_type action); +enum ice_status +ice_fltr_add_mac_and_broadcast(struct ice_vsi *vsi, const u8 *mac, + enum ice_sw_fwd_act_type action); +enum ice_status +ice_fltr_add_mac_list(struct ice_vsi *vsi, struct list_head *list); +enum ice_status +ice_fltr_remove_mac(struct ice_vsi *vsi, const u8 *mac, + enum ice_sw_fwd_act_type action); +enum ice_status +ice_fltr_remove_mac_list(struct ice_vsi *vsi, struct list_head *list); + +enum ice_status +ice_fltr_add_vlan(struct ice_vsi *vsi, u16 vid, + enum ice_sw_fwd_act_type action); +enum ice_status +ice_fltr_remove_vlan(struct ice_vsi *vsi, u16 vid, + enum ice_sw_fwd_act_type action); + +enum ice_status +ice_fltr_add_eth(struct ice_vsi *vsi, u16 ethertype, u16 flag, + enum ice_sw_fwd_act_type action); +enum ice_status +ice_fltr_remove_eth(struct ice_vsi *vsi, u16 ethertype, u16 flag, + enum ice_sw_fwd_act_type action); +void ice_fltr_remove_all(struct ice_vsi *vsi); +#endif diff --git a/drivers/net/ethernet/intel/ice/ice_hw_autogen.h b/drivers/net/ethernet/intel/ice/ice_hw_autogen.h index 1d37a9f02c1c..1086c9f778b4 100644 --- a/drivers/net/ethernet/intel/ice/ice_hw_autogen.h +++ b/drivers/net/ethernet/intel/ice/ice_hw_autogen.h @@ -6,9 +6,6 @@ #ifndef _ICE_HW_AUTOGEN_H_ #define _ICE_HW_AUTOGEN_H_ -#define PF0INT_ITR_0(_i) (0x03000004 + ((_i) * 4096)) -#define PF0INT_ITR_1(_i) (0x03000008 + ((_i) * 4096)) -#define PF0INT_ITR_2(_i) (0x0300000C + ((_i) * 4096)) #define QTX_COMM_DBELL(_DBQM) (0x002C0000 + ((_DBQM) * 4)) #define QTX_COMM_HEAD(_DBQM) (0x000E0000 + ((_DBQM) * 4)) #define QTX_COMM_HEAD_HEAD_S 0 @@ -42,6 +39,7 @@ #define PF_MBX_ARQH_ARQH_M ICE_M(0x3FF, 0) #define PF_MBX_ARQLEN 0x0022E480 #define PF_MBX_ARQLEN_ARQLEN_M ICE_M(0x3FF, 0) +#define PF_MBX_ARQLEN_ARQCRIT_M BIT(30) #define PF_MBX_ARQLEN_ARQENABLE_M BIT(31) #define PF_MBX_ARQT 0x0022E580 #define PF_MBX_ATQBAH 0x0022E180 @@ -50,6 +48,7 @@ #define PF_MBX_ATQH_ATQH_M ICE_M(0x3FF, 0) #define PF_MBX_ATQLEN 0x0022E200 #define PF_MBX_ATQLEN_ATQLEN_M ICE_M(0x3FF, 0) +#define PF_MBX_ATQLEN_ATQCRIT_M BIT(30) #define PF_MBX_ATQLEN_ATQENABLE_M BIT(31) #define PF_MBX_ATQT 0x0022E300 #define PRTDCB_GENC 0x00083000 @@ -58,6 +57,7 @@ #define PRTDCB_GENS 0x00083020 #define PRTDCB_GENS_DCBX_STATUS_S 0 #define PRTDCB_GENS_DCBX_STATUS_M ICE_M(0x7, 0) +#define PRTDCB_TUP2TC 0x001D26C0 /* Reset Source: CORER */ #define GL_PREEXT_L2_PMASK0(_i) (0x0020F0FC + ((_i) * 4)) #define GL_PREEXT_L2_PMASK1(_i) (0x0020F108 + ((_i) * 4)) #define GLFLXP_RXDID_FLX_WRD_0(_i) (0x0045c800 + ((_i) * 4)) @@ -218,6 +218,11 @@ #define VPLAN_TX_QBASE_VFNUMQ_M ICE_M(0xFF, 16) #define VPLAN_TXQ_MAPENA(_VF) (0x00073800 + ((_VF) * 4)) #define VPLAN_TXQ_MAPENA_TX_ENA_M BIT(0) +#define PRTMAC_HSEC_CTL_TX_PAUSE_QUANTA(_i) (0x001E36E0 + ((_i) * 32)) +#define PRTMAC_HSEC_CTL_TX_PAUSE_QUANTA_MAX_INDEX 8 +#define PRTMAC_HSEC_CTL_TX_PAUSE_QUANTA_HSEC_CTL_TX_PAUSE_QUANTA_M ICE_M(0xFFFF, 0) +#define PRTMAC_HSEC_CTL_TX_PAUSE_REFRESH_TIMER(_i) (0x001E3800 + ((_i) * 32)) +#define PRTMAC_HSEC_CTL_TX_PAUSE_REFRESH_TIMER_M ICE_M(0xFFFF, 0) #define GL_MDCK_TX_TDPU 0x00049348 #define GL_MDCK_TX_TDPU_RCU_ANTISPOOF_ITR_DIS_M BIT(1) #define GL_MDET_RX 0x00294C00 @@ -289,6 +294,20 @@ #define GL_PWR_MODE_CTL 0x000B820C #define GL_PWR_MODE_CTL_CAR_MAX_BW_S 30 #define GL_PWR_MODE_CTL_CAR_MAX_BW_M ICE_M(0x3, 30) +#define GLQF_FD_CNT 0x00460018 +#define GLQF_FD_CNT_FD_BCNT_S 16 +#define GLQF_FD_CNT_FD_BCNT_M ICE_M(0x7FFF, 16) +#define GLQF_FD_SIZE 0x00460010 +#define GLQF_FD_SIZE_FD_GSIZE_S 0 +#define GLQF_FD_SIZE_FD_GSIZE_M ICE_M(0x7FFF, 0) +#define GLQF_FD_SIZE_FD_BSIZE_S 16 +#define GLQF_FD_SIZE_FD_BSIZE_M ICE_M(0x7FFF, 16) +#define GLQF_FDINSET(_i, _j) (0x00412000 + ((_i) * 4 + (_j) * 512)) +#define GLQF_FDMASK_SEL(_i) (0x00410400 + ((_i) * 4)) +#define GLQF_FDSWAP(_i, _j) (0x00413000 + ((_i) * 4 + (_j) * 512)) +#define PFQF_FD_ENA 0x0043A000 +#define PFQF_FD_ENA_FD_ENA_M BIT(0) +#define PFQF_FD_SIZE 0x00460100 #define GLDCB_RTCTQ_RXQNUM_S 0 #define GLDCB_RTCTQ_RXQNUM_M ICE_M(0x7FF, 0) #define GLPRT_BPRCL(_i) (0x00381380 + ((_i) * 8)) @@ -332,6 +351,7 @@ #define GLPRT_TDOLD(_i) (0x00381280 + ((_i) * 8)) #define GLPRT_UPRCL(_i) (0x00381300 + ((_i) * 8)) #define GLPRT_UPTCL(_i) (0x003811C0 + ((_i) * 8)) +#define GLSTAT_FD_CNT0L(_i) (0x003A0000 + ((_i) * 8)) #define GLV_BPRCL(_i) (0x003B6000 + ((_i) * 8)) #define GLV_BPTCL(_i) (0x0030E000 + ((_i) * 8)) #define GLV_GORCL(_i) (0x003B0000 + ((_i) * 8)) @@ -342,6 +362,9 @@ #define GLV_TEPC(_VSI) (0x00312000 + ((_VSI) * 4)) #define GLV_UPRCL(_i) (0x003B2000 + ((_i) * 8)) #define GLV_UPTCL(_i) (0x0030A000 + ((_i) * 8)) +#define VSIQF_FD_CNT(_VSI) (0x00464000 + ((_VSI) * 4)) +#define VSIQF_FD_CNT_FD_GCNT_S 0 +#define VSIQF_FD_CNT_FD_GCNT_M ICE_M(0x3FFF, 0) #define VSIQF_HKEY_MAX_INDEX 12 #define VSIQF_HLUT_MAX_INDEX 15 #define VFINT_DYN_CTLN(_i) (0x00003800 + ((_i) * 4)) diff --git a/drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h b/drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h index 878e125d8b42..14dfbbc1b2cf 100644 --- a/drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h +++ b/drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h @@ -40,6 +40,104 @@ union ice_32byte_rx_desc { } wb; /* writeback */ }; +struct ice_fltr_desc { + __le64 qidx_compq_space_stat; + __le64 dtype_cmd_vsi_fdid; +}; + +#define ICE_FXD_FLTR_QW0_QINDEX_S 0 +#define ICE_FXD_FLTR_QW0_QINDEX_M (0x7FFULL << ICE_FXD_FLTR_QW0_QINDEX_S) +#define ICE_FXD_FLTR_QW0_COMP_Q_S 11 +#define ICE_FXD_FLTR_QW0_COMP_Q_M BIT_ULL(ICE_FXD_FLTR_QW0_COMP_Q_S) +#define ICE_FXD_FLTR_QW0_COMP_Q_ZERO 0x0ULL + +#define ICE_FXD_FLTR_QW0_COMP_REPORT_S 12 +#define ICE_FXD_FLTR_QW0_COMP_REPORT_M \ + (0x3ULL << ICE_FXD_FLTR_QW0_COMP_REPORT_S) +#define ICE_FXD_FLTR_QW0_COMP_REPORT_SW_FAIL 0x1ULL + +#define ICE_FXD_FLTR_QW0_FD_SPACE_S 14 +#define ICE_FXD_FLTR_QW0_FD_SPACE_M (0x3ULL << ICE_FXD_FLTR_QW0_FD_SPACE_S) +#define ICE_FXD_FLTR_QW0_FD_SPACE_GUAR_BEST 0x2ULL + +#define ICE_FXD_FLTR_QW0_STAT_CNT_S 16 +#define ICE_FXD_FLTR_QW0_STAT_CNT_M \ + (0x1FFFULL << ICE_FXD_FLTR_QW0_STAT_CNT_S) +#define ICE_FXD_FLTR_QW0_STAT_ENA_S 29 +#define ICE_FXD_FLTR_QW0_STAT_ENA_M (0x3ULL << ICE_FXD_FLTR_QW0_STAT_ENA_S) +#define ICE_FXD_FLTR_QW0_STAT_ENA_PKTS 0x1ULL + +#define ICE_FXD_FLTR_QW0_EVICT_ENA_S 31 +#define ICE_FXD_FLTR_QW0_EVICT_ENA_M BIT_ULL(ICE_FXD_FLTR_QW0_EVICT_ENA_S) +#define ICE_FXD_FLTR_QW0_EVICT_ENA_FALSE 0x0ULL +#define ICE_FXD_FLTR_QW0_EVICT_ENA_TRUE 0x1ULL + +#define ICE_FXD_FLTR_QW0_TO_Q_S 32 +#define ICE_FXD_FLTR_QW0_TO_Q_M (0x7ULL << ICE_FXD_FLTR_QW0_TO_Q_S) +#define ICE_FXD_FLTR_QW0_TO_Q_EQUALS_QINDEX 0x0ULL + +#define ICE_FXD_FLTR_QW0_TO_Q_PRI_S 35 +#define ICE_FXD_FLTR_QW0_TO_Q_PRI_M (0x7ULL << ICE_FXD_FLTR_QW0_TO_Q_PRI_S) +#define ICE_FXD_FLTR_QW0_TO_Q_PRIO1 0x1ULL + +#define ICE_FXD_FLTR_QW0_DPU_RECIPE_S 38 +#define ICE_FXD_FLTR_QW0_DPU_RECIPE_M \ + (0x3ULL << ICE_FXD_FLTR_QW0_DPU_RECIPE_S) +#define ICE_FXD_FLTR_QW0_DPU_RECIPE_DFLT 0x0ULL + +#define ICE_FXD_FLTR_QW0_DROP_S 40 +#define ICE_FXD_FLTR_QW0_DROP_M BIT_ULL(ICE_FXD_FLTR_QW0_DROP_S) +#define ICE_FXD_FLTR_QW0_DROP_NO 0x0ULL +#define ICE_FXD_FLTR_QW0_DROP_YES 0x1ULL + +#define ICE_FXD_FLTR_QW0_FLEX_PRI_S 41 +#define ICE_FXD_FLTR_QW0_FLEX_PRI_M (0x7ULL << ICE_FXD_FLTR_QW0_FLEX_PRI_S) +#define ICE_FXD_FLTR_QW0_FLEX_PRI_NONE 0x0ULL + +#define ICE_FXD_FLTR_QW0_FLEX_MDID_S 44 +#define ICE_FXD_FLTR_QW0_FLEX_MDID_M (0xFULL << ICE_FXD_FLTR_QW0_FLEX_MDID_S) +#define ICE_FXD_FLTR_QW0_FLEX_MDID0 0x0ULL + +#define ICE_FXD_FLTR_QW0_FLEX_VAL_S 48 +#define ICE_FXD_FLTR_QW0_FLEX_VAL_M \ + (0xFFFFULL << ICE_FXD_FLTR_QW0_FLEX_VAL_S) +#define ICE_FXD_FLTR_QW0_FLEX_VAL0 0x0ULL + +#define ICE_FXD_FLTR_QW1_DTYPE_S 0 +#define ICE_FXD_FLTR_QW1_DTYPE_M (0xFULL << ICE_FXD_FLTR_QW1_DTYPE_S) +#define ICE_FXD_FLTR_QW1_PCMD_S 4 +#define ICE_FXD_FLTR_QW1_PCMD_M BIT_ULL(ICE_FXD_FLTR_QW1_PCMD_S) +#define ICE_FXD_FLTR_QW1_PCMD_ADD 0x0ULL +#define ICE_FXD_FLTR_QW1_PCMD_REMOVE 0x1ULL + +#define ICE_FXD_FLTR_QW1_PROF_PRI_S 5 +#define ICE_FXD_FLTR_QW1_PROF_PRI_M (0x7ULL << ICE_FXD_FLTR_QW1_PROF_PRI_S) +#define ICE_FXD_FLTR_QW1_PROF_PRIO_ZERO 0x0ULL + +#define ICE_FXD_FLTR_QW1_PROF_S 8 +#define ICE_FXD_FLTR_QW1_PROF_M (0x3FULL << ICE_FXD_FLTR_QW1_PROF_S) +#define ICE_FXD_FLTR_QW1_PROF_ZERO 0x0ULL + +#define ICE_FXD_FLTR_QW1_FD_VSI_S 14 +#define ICE_FXD_FLTR_QW1_FD_VSI_M (0x3FFULL << ICE_FXD_FLTR_QW1_FD_VSI_S) +#define ICE_FXD_FLTR_QW1_SWAP_S 24 +#define ICE_FXD_FLTR_QW1_SWAP_M BIT_ULL(ICE_FXD_FLTR_QW1_SWAP_S) +#define ICE_FXD_FLTR_QW1_SWAP_NOT_SET 0x0ULL +#define ICE_FXD_FLTR_QW1_SWAP_SET 0x1ULL + +#define ICE_FXD_FLTR_QW1_FDID_PRI_S 25 +#define ICE_FXD_FLTR_QW1_FDID_PRI_M (0x7ULL << ICE_FXD_FLTR_QW1_FDID_PRI_S) +#define ICE_FXD_FLTR_QW1_FDID_PRI_ONE 0x1ULL + +#define ICE_FXD_FLTR_QW1_FDID_MDID_S 28 +#define ICE_FXD_FLTR_QW1_FDID_MDID_M (0xFULL << ICE_FXD_FLTR_QW1_FDID_MDID_S) +#define ICE_FXD_FLTR_QW1_FDID_MDID_FD 0x05ULL + +#define ICE_FXD_FLTR_QW1_FDID_S 32 +#define ICE_FXD_FLTR_QW1_FDID_M \ + (0xFFFFFFFFULL << ICE_FXD_FLTR_QW1_FDID_S) +#define ICE_FXD_FLTR_QW1_FDID_ZERO 0x0ULL + struct ice_rx_ptype_decoded { u32 ptype:10; u32 known:1; @@ -262,6 +360,12 @@ enum ice_rx_flex_desc_status_error_0_bits { ICE_RX_FLEX_DESC_STATUS0_LAST /* this entry must be last!!! */ }; +enum ice_rx_flex_desc_status_error_1_bits { + /* Note: These are predefined bit offsets */ + ICE_RX_FLEX_DESC_STATUS1_NAT_S = 4, + ICE_RX_FLEX_DESC_STATUS1_LAST /* this entry must be last!!! */ +}; + #define ICE_RXQ_CTX_SIZE_DWORDS 8 #define ICE_RXQ_CTX_SZ (ICE_RXQ_CTX_SIZE_DWORDS * sizeof(u32)) #define ICE_TX_CMPLTNQ_CTX_SIZE_DWORDS 22 @@ -340,6 +444,7 @@ struct ice_tx_desc { enum ice_tx_desc_dtype_value { ICE_TX_DESC_DTYPE_DATA = 0x0, ICE_TX_DESC_DTYPE_CTX = 0x1, + ICE_TX_DESC_DTYPE_FLTR_PROG = 0x8, /* DESC_DONE - HW has completed write-back of descriptor */ ICE_TX_DESC_DTYPE_DESC_DONE = 0xF, }; @@ -351,12 +456,14 @@ enum ice_tx_desc_cmd_bits { ICE_TX_DESC_CMD_EOP = 0x0001, ICE_TX_DESC_CMD_RS = 0x0002, ICE_TX_DESC_CMD_IL2TAG1 = 0x0008, + ICE_TX_DESC_CMD_DUMMY = 0x0010, ICE_TX_DESC_CMD_IIPT_IPV6 = 0x0020, ICE_TX_DESC_CMD_IIPT_IPV4 = 0x0040, ICE_TX_DESC_CMD_IIPT_IPV4_CSUM = 0x0060, ICE_TX_DESC_CMD_L4T_EOFT_TCP = 0x0100, ICE_TX_DESC_CMD_L4T_EOFT_SCTP = 0x0200, ICE_TX_DESC_CMD_L4T_EOFT_UDP = 0x0300, + ICE_TX_DESC_CMD_RE = 0x0400, }; #define ICE_TXD_QW1_OFFSET_S 16 @@ -413,6 +520,25 @@ enum ice_tx_ctx_desc_cmd_bits { ICE_TX_CTX_DESC_RESERVED = 0x40 }; +enum ice_tx_ctx_desc_eipt_offload { + ICE_TX_CTX_EIPT_NONE = 0x0, + ICE_TX_CTX_EIPT_IPV6 = 0x1, + ICE_TX_CTX_EIPT_IPV4_NO_CSUM = 0x2, + ICE_TX_CTX_EIPT_IPV4 = 0x3 +}; + +#define ICE_TXD_CTX_QW0_EIPLEN_S 2 + +#define ICE_TXD_CTX_QW0_L4TUNT_S 9 + +#define ICE_TXD_CTX_UDP_TUNNELING BIT_ULL(ICE_TXD_CTX_QW0_L4TUNT_S) +#define ICE_TXD_CTX_GRE_TUNNELING (0x2ULL << ICE_TXD_CTX_QW0_L4TUNT_S) + +#define ICE_TXD_CTX_QW0_NATLEN_S 12 + +#define ICE_TXD_CTX_QW0_L4T_CS_S 23 +#define ICE_TXD_CTX_QW0_L4T_CS_M BIT_ULL(ICE_TXD_CTX_QW0_L4T_CS_S) + #define ICE_LAN_TXQ_MAX_QGRPS 127 #define ICE_LAN_TXQ_MAX_QDIS 1023 @@ -455,7 +581,7 @@ struct ice_tlan_ctx { u8 drop_ena; u8 cache_prof_idx; u8 pkt_shaper_prof_idx; - u8 int_q_state; /* width not needed - internal do not write */ + u8 int_q_state; /* width not needed - internal - DO NOT WRITE!!! */ }; /* macro to make the table lines short */ diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c index 2f256bf45efc..28b46cc9f5cb 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_lib.c @@ -5,6 +5,7 @@ #include "ice_base.h" #include "ice_flow.h" #include "ice_lib.h" +#include "ice_fltr.h" #include "ice_dcb_lib.h" /** @@ -18,6 +19,8 @@ const char *ice_vsi_type_str(enum ice_vsi_type vsi_type) return "ICE_VSI_PF"; case ICE_VSI_VF: return "ICE_VSI_VF"; + case ICE_VSI_CTRL: + return "ICE_VSI_CTRL"; case ICE_VSI_LB: return "ICE_VSI_LB"; default: @@ -37,7 +40,8 @@ const char *ice_vsi_type_str(enum ice_vsi_type vsi_type) */ static int ice_vsi_ctrl_all_rx_rings(struct ice_vsi *vsi, bool ena) { - int i, ret = 0; + int ret = 0; + u16 i; for (i = 0; i < vsi->num_rxq; i++) ice_vsi_ctrl_one_rx_ring(vsi, ena, i, false); @@ -121,6 +125,7 @@ static void ice_vsi_set_num_desc(struct ice_vsi *vsi) { switch (vsi->type) { case ICE_VSI_PF: + case ICE_VSI_CTRL: case ICE_VSI_LB: vsi->num_rx_desc = ICE_DFLT_NUM_RX_DESC; vsi->num_tx_desc = ICE_DFLT_NUM_TX_DESC; @@ -185,6 +190,11 @@ static void ice_vsi_set_num_qs(struct ice_vsi *vsi, u16 vf_id) */ vsi->num_q_vectors = pf->num_msix_per_vf - ICE_NONQ_VECS_VF; break; + case ICE_VSI_CTRL: + vsi->alloc_txq = 1; + vsi->alloc_rxq = 1; + vsi->num_q_vectors = 1; + break; case ICE_VSI_LB: vsi->alloc_txq = 1; vsi->alloc_rxq = 1; @@ -248,8 +258,8 @@ void ice_vsi_delete(struct ice_vsi *vsi) status = ice_free_vsi(&pf->hw, vsi->idx, ctxt, false, NULL); if (status) - dev_err(ice_pf_to_dev(pf), "Failed to delete VSI %i in FW - error: %d\n", - vsi->vsi_num, status); + dev_err(ice_pf_to_dev(pf), "Failed to delete VSI %i in FW - error: %s\n", + vsi->vsi_num, ice_stat_str(status)); kfree(ctxt); } @@ -320,7 +330,7 @@ int ice_vsi_clear(struct ice_vsi *vsi) /* updates the PF for this cleared VSI */ pf->vsi[vsi->idx] = NULL; - if (vsi->idx < pf->next_vsi) + if (vsi->idx < pf->next_vsi && vsi->type != ICE_VSI_CTRL) pf->next_vsi = vsi->idx; ice_vsi_free_arrays(vsi); @@ -331,6 +341,25 @@ int ice_vsi_clear(struct ice_vsi *vsi) } /** + * ice_msix_clean_ctrl_vsi - MSIX mode interrupt handler for ctrl VSI + * @irq: interrupt number + * @data: pointer to a q_vector + */ +static irqreturn_t ice_msix_clean_ctrl_vsi(int __always_unused irq, void *data) +{ + struct ice_q_vector *q_vector = (struct ice_q_vector *)data; + + if (!q_vector->tx.ring) + return IRQ_HANDLED; + +#define FDIR_RX_DESC_CLEAN_BUDGET 64 + ice_clean_rx_irq(q_vector->rx.ring, FDIR_RX_DESC_CLEAN_BUDGET); + ice_clean_ctrl_tx_irq(q_vector->tx.ring); + + return IRQ_HANDLED; +} + +/** * ice_msix_clean_rings - MSIX mode Interrupt Handler * @irq: interrupt number * @data: pointer to a q_vector @@ -381,8 +410,6 @@ ice_vsi_alloc(struct ice_pf *pf, enum ice_vsi_type vsi_type, u16 vf_id) vsi->back = pf; set_bit(__ICE_DOWN, vsi->state); - vsi->idx = pf->next_vsi; - if (vsi_type == ICE_VSI_VF) ice_vsi_set_num_qs(vsi, vf_id); else @@ -396,6 +423,13 @@ ice_vsi_alloc(struct ice_pf *pf, enum ice_vsi_type vsi_type, u16 vf_id) /* Setup default MSIX irq handler for VSI */ vsi->irq_handler = ice_msix_clean_rings; break; + case ICE_VSI_CTRL: + if (ice_vsi_alloc_arrays(vsi)) + goto err_rings; + + /* Setup ctrl VSI MSIX irq handler */ + vsi->irq_handler = ice_msix_clean_ctrl_vsi; + break; case ICE_VSI_VF: if (ice_vsi_alloc_arrays(vsi)) goto err_rings; @@ -409,12 +443,20 @@ ice_vsi_alloc(struct ice_pf *pf, enum ice_vsi_type vsi_type, u16 vf_id) goto unlock_pf; } - /* fill VSI slot in the PF struct */ - pf->vsi[pf->next_vsi] = vsi; + if (vsi->type == ICE_VSI_CTRL) { + /* Use the last VSI slot as the index for the control VSI */ + vsi->idx = pf->num_alloc_vsi - 1; + pf->ctrl_vsi_idx = vsi->idx; + pf->vsi[vsi->idx] = vsi; + } else { + /* fill slot and make note of the index */ + vsi->idx = pf->next_vsi; + pf->vsi[pf->next_vsi] = vsi; - /* prepare pf->next_vsi for next use */ - pf->next_vsi = ice_get_free_slot(pf->vsi, pf->num_alloc_vsi, - pf->next_vsi); + /* prepare pf->next_vsi for next use */ + pf->next_vsi = ice_get_free_slot(pf->vsi, pf->num_alloc_vsi, + pf->next_vsi); + } goto unlock_pf; err_rings: @@ -426,6 +468,48 @@ unlock_pf: } /** + * ice_alloc_fd_res - Allocate FD resource for a VSI + * @vsi: pointer to the ice_vsi + * + * This allocates the FD resources + * + * Returns 0 on success, -EPERM on no-op or -EIO on failure + */ +static int ice_alloc_fd_res(struct ice_vsi *vsi) +{ + struct ice_pf *pf = vsi->back; + u32 g_val, b_val; + + /* Flow Director filters are only allocated/assigned to the PF VSI which + * passes the traffic. The CTRL VSI is only used to add/delete filters + * so we don't allocate resources to it + */ + + /* FD filters from guaranteed pool per VSI */ + g_val = pf->hw.func_caps.fd_fltr_guar; + if (!g_val) + return -EPERM; + + /* FD filters from best effort pool */ + b_val = pf->hw.func_caps.fd_fltr_best_effort; + if (!b_val) + return -EPERM; + + if (vsi->type != ICE_VSI_PF) + return -EPERM; + + if (!test_bit(ICE_FLAG_FD_ENA, pf->flags)) + return -EPERM; + + vsi->num_gfltr = g_val / pf->num_alloc_vsi; + + /* each VSI gets same "best_effort" quota */ + vsi->num_bfltr = b_val; + + return 0; +} + +/** * ice_vsi_get_qs - Assign queues from PF to VSI * @vsi: the VSI to assign queues to * @@ -521,8 +605,8 @@ static void ice_vsi_clean_rss_flow_fld(struct ice_vsi *vsi) status = ice_rem_vsi_rss_cfg(&pf->hw, vsi->idx); if (status) - dev_dbg(ice_pf_to_dev(pf), "ice_rem_vsi_rss_cfg failed for vsi = %d, error = %d\n", - vsi->vsi_num, status); + dev_dbg(ice_pf_to_dev(pf), "ice_rem_vsi_rss_cfg failed for vsi = %d, error = %s\n", + vsi->vsi_num, ice_stat_str(status)); } /** @@ -565,8 +649,8 @@ static void ice_vsi_set_rss_params(struct ice_vsi *vsi) switch (vsi->type) { case ICE_VSI_PF: /* PF VSI will inherit RSS instance of PF */ - vsi->rss_table_size = cap->rss_table_size; - vsi->rss_size = min_t(int, num_online_cpus(), + vsi->rss_table_size = (u16)cap->rss_table_size; + vsi->rss_size = min_t(u16, num_online_cpus(), BIT(cap->rss_table_entry_width)); vsi->rss_lut_type = ICE_AQC_GSET_RSS_LUT_TABLE_TYPE_PF; break; @@ -581,8 +665,8 @@ static void ice_vsi_set_rss_params(struct ice_vsi *vsi) case ICE_VSI_LB: break; default: - dev_warn(ice_pf_to_dev(pf), "Unknown VSI type %d\n", - vsi->type); + dev_dbg(ice_pf_to_dev(pf), "Unsupported VSI type %s\n", + ice_vsi_type_str(vsi->type)); break; } } @@ -684,15 +768,15 @@ static void ice_vsi_setup_q_map(struct ice_vsi *vsi, struct ice_vsi_ctx *ctxt) max_rss = ICE_MAX_LG_RSS_QS; else max_rss = ICE_MAX_RSS_QS_PER_VF; - qcount_rx = min_t(int, rx_numq_tc, max_rss); + qcount_rx = min_t(u16, rx_numq_tc, max_rss); if (!vsi->req_rxq) - qcount_rx = min_t(int, qcount_rx, + qcount_rx = min_t(u16, qcount_rx, vsi->rss_size); } } /* find the (rounded up) power-of-2 of qcount */ - pow = order_base_2(qcount_rx); + pow = (u16)order_base_2(qcount_rx); ice_for_each_traffic_class(i) { if (!(vsi->tc_cfg.ena_tc & BIT(i))) { @@ -752,6 +836,51 @@ static void ice_vsi_setup_q_map(struct ice_vsi *vsi, struct ice_vsi_ctx *ctxt) } /** + * ice_set_fd_vsi_ctx - Set FD VSI context before adding a VSI + * @ctxt: the VSI context being set + * @vsi: the VSI being configured + */ +static void ice_set_fd_vsi_ctx(struct ice_vsi_ctx *ctxt, struct ice_vsi *vsi) +{ + u8 dflt_q_group, dflt_q_prio; + u16 dflt_q, report_q, val; + + if (vsi->type != ICE_VSI_PF && vsi->type != ICE_VSI_CTRL) + return; + + val = ICE_AQ_VSI_PROP_FLOW_DIR_VALID; + ctxt->info.valid_sections |= cpu_to_le16(val); + dflt_q = 0; + dflt_q_group = 0; + report_q = 0; + dflt_q_prio = 0; + + /* enable flow director filtering/programming */ + val = ICE_AQ_VSI_FD_ENABLE | ICE_AQ_VSI_FD_PROG_ENABLE; + ctxt->info.fd_options = cpu_to_le16(val); + /* max of allocated flow director filters */ + ctxt->info.max_fd_fltr_dedicated = + cpu_to_le16(vsi->num_gfltr); + /* max of shared flow director filters any VSI may program */ + ctxt->info.max_fd_fltr_shared = + cpu_to_le16(vsi->num_bfltr); + /* default queue index within the VSI of the default FD */ + val = ((dflt_q << ICE_AQ_VSI_FD_DEF_Q_S) & + ICE_AQ_VSI_FD_DEF_Q_M); + /* target queue or queue group to the FD filter */ + val |= ((dflt_q_group << ICE_AQ_VSI_FD_DEF_GRP_S) & + ICE_AQ_VSI_FD_DEF_GRP_M); + ctxt->info.fd_def_q = cpu_to_le16(val); + /* queue index on which FD filter completion is reported */ + val = ((report_q << ICE_AQ_VSI_FD_REPORT_Q_S) & + ICE_AQ_VSI_FD_REPORT_Q_M); + /* priority of the default qindex action */ + val |= ((dflt_q_prio << ICE_AQ_VSI_FD_DEF_PRIORITY_S) & + ICE_AQ_VSI_FD_DEF_PRIORITY_M); + ctxt->info.fd_report_opt = cpu_to_le16(val); +} + +/** * ice_set_rss_vsi_ctx - Set RSS VSI context before adding a VSI * @ctxt: the VSI context being set * @vsi: the VSI being configured @@ -776,13 +905,10 @@ static void ice_set_rss_vsi_ctx(struct ice_vsi_ctx *ctxt, struct ice_vsi *vsi) lut_type = ICE_AQ_VSI_Q_OPT_RSS_LUT_VSI; hash_type = ICE_AQ_VSI_Q_OPT_RSS_TPLZ; break; - case ICE_VSI_LB: + default: dev_dbg(dev, "Unsupported VSI type %s\n", ice_vsi_type_str(vsi->type)); return; - default: - dev_warn(dev, "Unknown VSI type %d\n", vsi->type); - return; } ctxt->info.q_opt_rss = ((lut_type << ICE_AQ_VSI_Q_OPT_RSS_LUT_S) & @@ -812,8 +938,8 @@ static int ice_vsi_init(struct ice_vsi *vsi, bool init_vsi) if (!ctxt) return -ENOMEM; - ctxt->info = vsi->info; switch (vsi->type) { + case ICE_VSI_CTRL: case ICE_VSI_LB: case ICE_VSI_PF: ctxt->flags = ICE_AQ_VSI_TYPE_PF; @@ -829,12 +955,15 @@ static int ice_vsi_init(struct ice_vsi *vsi, bool init_vsi) } ice_set_dflt_vsi_ctx(ctxt); + if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) + ice_set_fd_vsi_ctx(ctxt, vsi); /* if the switch is in VEB mode, allow VSI loopback */ if (vsi->vsw->bridge_mode == BRIDGE_MODE_VEB) ctxt->info.sw_flags |= ICE_AQ_VSI_SW_FLAG_ALLOW_LB; /* Set LUT type and HASH type if RSS is enabled */ - if (test_bit(ICE_FLAG_RSS_ENA, pf->flags)) { + if (test_bit(ICE_FLAG_RSS_ENA, pf->flags) && + vsi->type != ICE_VSI_CTRL) { ice_set_rss_vsi_ctx(ctxt, vsi); /* if updating VSI context, make sure to set valid_section: * to indicate which section of VSI context being updated @@ -941,7 +1070,7 @@ int ice_free_res(struct ice_res_tracker *res, u16 index, u16 id) */ static int ice_search_res(struct ice_res_tracker *res, u16 needed, u16 id) { - int start = 0, end = 0; + u16 start = 0, end = 0; if (needed > res->end) return -ENOMEM; @@ -1024,6 +1153,7 @@ static int ice_vsi_setup_vector_base(struct ice_vsi *vsi) struct ice_pf *pf = vsi->back; struct device *dev; u16 num_q_vectors; + int base; dev = ice_pf_to_dev(pf); /* SRIOV doesn't grab irq_tracker entries for each VSI */ @@ -1038,14 +1168,15 @@ static int ice_vsi_setup_vector_base(struct ice_vsi *vsi) num_q_vectors = vsi->num_q_vectors; /* reserve slots from OS requested IRQs */ - vsi->base_vector = ice_get_res(pf, pf->irq_tracker, num_q_vectors, - vsi->idx); - if (vsi->base_vector < 0) { + base = ice_get_res(pf, pf->irq_tracker, num_q_vectors, vsi->idx); + + if (base < 0) { dev_err(dev, "%d MSI-X interrupts available. %s %d failed to get %d MSI-X vectors\n", ice_get_free_res_count(pf->irq_tracker), ice_vsi_type_str(vsi->type), vsi->idx, num_q_vectors); return -ENOENT; } + vsi->base_vector = (u16)base; pf->num_avail_sw_msix -= num_q_vectors; return 0; @@ -1085,7 +1216,7 @@ static int ice_vsi_alloc_rings(struct ice_vsi *vsi) { struct ice_pf *pf = vsi->back; struct device *dev; - int i; + u16 i; dev = ice_pf_to_dev(pf); /* Allocate Tx rings */ @@ -1178,7 +1309,7 @@ static int ice_vsi_cfg_rss_lut_key(struct ice_vsi *vsi) u8 *lut; dev = ice_pf_to_dev(pf); - vsi->rss_size = min_t(int, vsi->rss_size, vsi->num_rxq); + vsi->rss_size = min_t(u16, vsi->rss_size, vsi->num_rxq); lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); if (!lut) @@ -1193,7 +1324,8 @@ static int ice_vsi_cfg_rss_lut_key(struct ice_vsi *vsi) vsi->rss_table_size); if (status) { - dev_err(dev, "set_rss_lut failed, error %d\n", status); + dev_err(dev, "set_rss_lut failed, error %s\n", + ice_stat_str(status)); err = -EIO; goto ice_vsi_cfg_rss_exit; } @@ -1215,7 +1347,8 @@ static int ice_vsi_cfg_rss_lut_key(struct ice_vsi *vsi) status = ice_aq_set_rss_key(&pf->hw, vsi->idx, key); if (status) { - dev_err(dev, "set_rss_key failed, error %d\n", status); + dev_err(dev, "set_rss_key failed, error %s\n", + ice_stat_str(status)); err = -EIO; } @@ -1248,8 +1381,8 @@ static void ice_vsi_set_vf_rss_flow_fld(struct ice_vsi *vsi) status = ice_add_avf_rss_cfg(&pf->hw, vsi->idx, ICE_DEFAULT_RSS_HENA); if (status) - dev_dbg(dev, "ice_add_avf_rss_cfg failed for vsi = %d, error = %d\n", - vsi->vsi_num, status); + dev_dbg(dev, "ice_add_avf_rss_cfg failed for vsi = %d, error = %s\n", + vsi->vsi_num, ice_stat_str(status)); } /** @@ -1281,91 +1414,57 @@ static void ice_vsi_set_rss_flow_fld(struct ice_vsi *vsi) status = ice_add_rss_cfg(hw, vsi_handle, ICE_FLOW_HASH_IPV4, ICE_FLOW_SEG_HDR_IPV4); if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for ipv4 flow, vsi = %d, error = %d\n", - vsi_num, status); + dev_dbg(dev, "ice_add_rss_cfg failed for ipv4 flow, vsi = %d, error = %s\n", + vsi_num, ice_stat_str(status)); /* configure RSS for IPv6 with input set IPv6 src/dst */ status = ice_add_rss_cfg(hw, vsi_handle, ICE_FLOW_HASH_IPV6, ICE_FLOW_SEG_HDR_IPV6); if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for ipv6 flow, vsi = %d, error = %d\n", - vsi_num, status); + dev_dbg(dev, "ice_add_rss_cfg failed for ipv6 flow, vsi = %d, error = %s\n", + vsi_num, ice_stat_str(status)); /* configure RSS for tcp4 with input set IP src/dst, TCP src/dst */ status = ice_add_rss_cfg(hw, vsi_handle, ICE_HASH_TCP_IPV4, ICE_FLOW_SEG_HDR_TCP | ICE_FLOW_SEG_HDR_IPV4); if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for tcp4 flow, vsi = %d, error = %d\n", - vsi_num, status); + dev_dbg(dev, "ice_add_rss_cfg failed for tcp4 flow, vsi = %d, error = %s\n", + vsi_num, ice_stat_str(status)); /* configure RSS for udp4 with input set IP src/dst, UDP src/dst */ status = ice_add_rss_cfg(hw, vsi_handle, ICE_HASH_UDP_IPV4, ICE_FLOW_SEG_HDR_UDP | ICE_FLOW_SEG_HDR_IPV4); if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for udp4 flow, vsi = %d, error = %d\n", - vsi_num, status); + dev_dbg(dev, "ice_add_rss_cfg failed for udp4 flow, vsi = %d, error = %s\n", + vsi_num, ice_stat_str(status)); /* configure RSS for sctp4 with input set IP src/dst */ status = ice_add_rss_cfg(hw, vsi_handle, ICE_FLOW_HASH_IPV4, ICE_FLOW_SEG_HDR_SCTP | ICE_FLOW_SEG_HDR_IPV4); if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for sctp4 flow, vsi = %d, error = %d\n", - vsi_num, status); + dev_dbg(dev, "ice_add_rss_cfg failed for sctp4 flow, vsi = %d, error = %s\n", + vsi_num, ice_stat_str(status)); /* configure RSS for tcp6 with input set IPv6 src/dst, TCP src/dst */ status = ice_add_rss_cfg(hw, vsi_handle, ICE_HASH_TCP_IPV6, ICE_FLOW_SEG_HDR_TCP | ICE_FLOW_SEG_HDR_IPV6); if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for tcp6 flow, vsi = %d, error = %d\n", - vsi_num, status); + dev_dbg(dev, "ice_add_rss_cfg failed for tcp6 flow, vsi = %d, error = %s\n", + vsi_num, ice_stat_str(status)); /* configure RSS for udp6 with input set IPv6 src/dst, UDP src/dst */ status = ice_add_rss_cfg(hw, vsi_handle, ICE_HASH_UDP_IPV6, ICE_FLOW_SEG_HDR_UDP | ICE_FLOW_SEG_HDR_IPV6); if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for udp6 flow, vsi = %d, error = %d\n", - vsi_num, status); + dev_dbg(dev, "ice_add_rss_cfg failed for udp6 flow, vsi = %d, error = %s\n", + vsi_num, ice_stat_str(status)); /* configure RSS for sctp6 with input set IPv6 src/dst */ status = ice_add_rss_cfg(hw, vsi_handle, ICE_FLOW_HASH_IPV6, ICE_FLOW_SEG_HDR_SCTP | ICE_FLOW_SEG_HDR_IPV6); if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for sctp6 flow, vsi = %d, error = %d\n", - vsi_num, status); -} - -/** - * ice_add_mac_to_list - Add a MAC address filter entry to the list - * @vsi: the VSI to be forwarded to - * @add_list: pointer to the list which contains MAC filter entries - * @macaddr: the MAC address to be added. - * - * Adds MAC address filter entry to the temp list - * - * Returns 0 on success or ENOMEM on failure. - */ -int -ice_add_mac_to_list(struct ice_vsi *vsi, struct list_head *add_list, - const u8 *macaddr) -{ - struct ice_fltr_list_entry *tmp; - struct ice_pf *pf = vsi->back; - - tmp = devm_kzalloc(ice_pf_to_dev(pf), sizeof(*tmp), GFP_ATOMIC); - if (!tmp) - return -ENOMEM; - - tmp->fltr_info.flag = ICE_FLTR_TX; - tmp->fltr_info.src_id = ICE_SRC_ID_VSI; - tmp->fltr_info.lkup_type = ICE_SW_LKUP_MAC; - tmp->fltr_info.fltr_act = ICE_FWD_TO_VSI; - tmp->fltr_info.vsi_handle = vsi->idx; - ether_addr_copy(tmp->fltr_info.l_data.mac.mac_addr, macaddr); - - INIT_LIST_HEAD(&tmp->list_entry); - list_add(&tmp->list_entry, add_list); - - return 0; + dev_dbg(dev, "ice_add_rss_cfg failed for sctp6 flow, vsi = %d, error = %s\n", + vsi_num, ice_stat_str(status)); } /** @@ -1415,54 +1514,21 @@ void ice_update_eth_stats(struct ice_vsi *vsi) } /** - * ice_free_fltr_list - free filter lists helper - * @dev: pointer to the device struct - * @h: pointer to the list head to be freed - * - * Helper function to free filter lists previously created using - * ice_add_mac_to_list - */ -void ice_free_fltr_list(struct device *dev, struct list_head *h) -{ - struct ice_fltr_list_entry *e, *tmp; - - list_for_each_entry_safe(e, tmp, h, list_entry) { - list_del(&e->list_entry); - devm_kfree(dev, e); - } -} - -/** * ice_vsi_add_vlan - Add VSI membership for given VLAN * @vsi: the VSI being configured * @vid: VLAN ID to be added + * @action: filter action to be performed on match */ -int ice_vsi_add_vlan(struct ice_vsi *vsi, u16 vid) +int +ice_vsi_add_vlan(struct ice_vsi *vsi, u16 vid, enum ice_sw_fwd_act_type action) { - struct ice_fltr_list_entry *tmp; struct ice_pf *pf = vsi->back; - LIST_HEAD(tmp_add_list); - enum ice_status status; struct device *dev; int err = 0; dev = ice_pf_to_dev(pf); - tmp = devm_kzalloc(dev, sizeof(*tmp), GFP_KERNEL); - if (!tmp) - return -ENOMEM; - tmp->fltr_info.lkup_type = ICE_SW_LKUP_VLAN; - tmp->fltr_info.fltr_act = ICE_FWD_TO_VSI; - tmp->fltr_info.flag = ICE_FLTR_TX; - tmp->fltr_info.src_id = ICE_SRC_ID_VSI; - tmp->fltr_info.vsi_handle = vsi->idx; - tmp->fltr_info.l_data.vlan.vlan_id = vid; - - INIT_LIST_HEAD(&tmp->list_entry); - list_add(&tmp->list_entry, &tmp_add_list); - - status = ice_add_vlan(&pf->hw, &tmp_add_list); - if (!status) { + if (!ice_fltr_add_vlan(vsi, vid, action)) { vsi->num_vlan++; } else { err = -ENODEV; @@ -1470,7 +1536,6 @@ int ice_vsi_add_vlan(struct ice_vsi *vsi, u16 vid) vsi->vsi_num); } - ice_free_fltr_list(dev, &tmp_add_list); return err; } @@ -1483,41 +1548,25 @@ int ice_vsi_add_vlan(struct ice_vsi *vsi, u16 vid) */ int ice_vsi_kill_vlan(struct ice_vsi *vsi, u16 vid) { - struct ice_fltr_list_entry *list; struct ice_pf *pf = vsi->back; - LIST_HEAD(tmp_add_list); enum ice_status status; struct device *dev; int err = 0; dev = ice_pf_to_dev(pf); - list = devm_kzalloc(dev, sizeof(*list), GFP_KERNEL); - if (!list) - return -ENOMEM; - - list->fltr_info.lkup_type = ICE_SW_LKUP_VLAN; - list->fltr_info.vsi_handle = vsi->idx; - list->fltr_info.fltr_act = ICE_FWD_TO_VSI; - list->fltr_info.l_data.vlan.vlan_id = vid; - list->fltr_info.flag = ICE_FLTR_TX; - list->fltr_info.src_id = ICE_SRC_ID_VSI; - INIT_LIST_HEAD(&list->list_entry); - list_add(&list->list_entry, &tmp_add_list); - - status = ice_remove_vlan(&pf->hw, &tmp_add_list); + status = ice_fltr_remove_vlan(vsi, vid, ICE_FWD_TO_VSI); if (!status) { vsi->num_vlan--; } else if (status == ICE_ERR_DOES_NOT_EXIST) { - dev_dbg(dev, "Failed to remove VLAN %d on VSI %i, it does not exist, status: %d\n", - vid, vsi->vsi_num, status); + dev_dbg(dev, "Failed to remove VLAN %d on VSI %i, it does not exist, status: %s\n", + vid, vsi->vsi_num, ice_stat_str(status)); } else { - dev_err(dev, "Error removing VLAN %d on vsi %i error: %d\n", - vid, vsi->vsi_num, status); + dev_err(dev, "Error removing VLAN %d on vsi %i error: %s\n", + vid, vsi->vsi_num, ice_stat_str(status)); err = -EIO; } - ice_free_fltr_list(dev, &tmp_add_list); return err; } @@ -1547,6 +1596,32 @@ void ice_vsi_cfg_frame_size(struct ice_vsi *vsi) } /** + * ice_write_qrxflxp_cntxt - write/configure QRXFLXP_CNTXT register + * @hw: HW pointer + * @pf_q: index of the Rx queue in the PF's queue space + * @rxdid: flexible descriptor RXDID + * @prio: priority for the RXDID for this queue + */ +void +ice_write_qrxflxp_cntxt(struct ice_hw *hw, u16 pf_q, u32 rxdid, u32 prio) +{ + int regval = rd32(hw, QRXFLXP_CNTXT(pf_q)); + + /* clear any previous values */ + regval &= ~(QRXFLXP_CNTXT_RXDID_IDX_M | + QRXFLXP_CNTXT_RXDID_PRIO_M | + QRXFLXP_CNTXT_TS_M); + + regval |= (rxdid << QRXFLXP_CNTXT_RXDID_IDX_S) & + QRXFLXP_CNTXT_RXDID_IDX_M; + + regval |= (prio << QRXFLXP_CNTXT_RXDID_PRIO_S) & + QRXFLXP_CNTXT_RXDID_PRIO_M; + + wr32(hw, QRXFLXP_CNTXT(pf_q), regval); +} + +/** * ice_vsi_cfg_rxqs - Configure the VSI for Rx * @vsi: the VSI being configured * @@ -1671,7 +1746,7 @@ void ice_vsi_cfg_msix(struct ice_vsi *vsi) { struct ice_pf *pf = vsi->back; struct ice_hw *hw = &pf->hw; - u32 txq = 0, rxq = 0; + u16 txq = 0, rxq = 0; int i, q; for (i = 0; i < vsi->num_q_vectors; i++) { @@ -1737,8 +1812,9 @@ int ice_vsi_manage_vlan_insertion(struct ice_vsi *vsi) status = ice_update_vsi(hw, vsi->idx, ctxt, NULL); if (status) { - dev_err(ice_pf_to_dev(vsi->back), "update VSI for VLAN insert failed, err %d aq_err %d\n", - status, hw->adminq.sq_last_status); + dev_err(ice_pf_to_dev(vsi->back), "update VSI for VLAN insert failed, err %s aq_err %s\n", + ice_stat_str(status), + ice_aq_str(hw->adminq.sq_last_status)); ret = -EIO; goto out; } @@ -1761,6 +1837,12 @@ int ice_vsi_manage_vlan_stripping(struct ice_vsi *vsi, bool ena) enum ice_status status; int ret = 0; + /* do not allow modifying VLAN stripping when a port VLAN is configured + * on this VSI + */ + if (vsi->info.pvid) + return 0; + ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL); if (!ctxt) return -ENOMEM; @@ -1783,8 +1865,9 @@ int ice_vsi_manage_vlan_stripping(struct ice_vsi *vsi, bool ena) status = ice_update_vsi(hw, vsi->idx, ctxt, NULL); if (status) { - dev_err(ice_pf_to_dev(vsi->back), "update VSI for VLAN strip failed, ena = %d err %d aq_err %d\n", - ena, status, hw->adminq.sq_last_status); + dev_err(ice_pf_to_dev(vsi->back), "update VSI for VLAN strip failed, ena = %d err %s aq_err %s\n", + ena, ice_stat_str(status), + ice_aq_str(hw->adminq.sq_last_status)); ret = -EIO; goto out; } @@ -1922,9 +2005,10 @@ int ice_cfg_vlan_pruning(struct ice_vsi *vsi, bool ena, bool vlan_promisc) status = ice_update_vsi(&pf->hw, vsi->idx, ctxt, NULL); if (status) { - netdev_err(vsi->netdev, "%sabling VLAN pruning on VSI handle: %d, VSI HW ID: %d failed, err = %d, aq_err = %d\n", - ena ? "En" : "Dis", vsi->idx, vsi->vsi_num, status, - pf->hw.adminq.sq_last_status); + netdev_err(vsi->netdev, "%sabling VLAN pruning on VSI handle: %d, VSI HW ID: %d failed, err = %s, aq_err = %s\n", + ena ? "En" : "Dis", vsi->idx, vsi->vsi_num, + ice_stat_str(status), + ice_aq_str(pf->hw.adminq.sq_last_status)); goto err_out; } @@ -1991,47 +2075,6 @@ clear_reg_idx: } /** - * ice_vsi_add_rem_eth_mac - Program VSI ethertype based filter with rule - * @vsi: the VSI being configured - * @add_rule: boolean value to add or remove ethertype filter rule - */ -static void -ice_vsi_add_rem_eth_mac(struct ice_vsi *vsi, bool add_rule) -{ - struct ice_fltr_list_entry *list; - struct ice_pf *pf = vsi->back; - LIST_HEAD(tmp_add_list); - enum ice_status status; - struct device *dev; - - dev = ice_pf_to_dev(pf); - list = devm_kzalloc(dev, sizeof(*list), GFP_KERNEL); - if (!list) - return; - - list->fltr_info.lkup_type = ICE_SW_LKUP_ETHERTYPE; - list->fltr_info.fltr_act = ICE_DROP_PACKET; - list->fltr_info.flag = ICE_FLTR_TX; - list->fltr_info.src_id = ICE_SRC_ID_VSI; - list->fltr_info.vsi_handle = vsi->idx; - list->fltr_info.l_data.ethertype_mac.ethertype = vsi->ethtype; - - INIT_LIST_HEAD(&list->list_entry); - list_add(&list->list_entry, &tmp_add_list); - - if (add_rule) - status = ice_add_eth_mac(&pf->hw, &tmp_add_list); - else - status = ice_remove_eth_mac(&pf->hw, &tmp_add_list); - - if (status) - dev_err(dev, "Failure Adding or Removing Ethertype on VSI %i error: %d\n", - vsi->vsi_num, status); - - ice_free_fltr_list(dev, &tmp_add_list); -} - -/** * ice_cfg_sw_lldp - Config switch rules for LLDP packet handling * @vsi: the VSI being configured * @tx: bool to determine Tx or Rx rule @@ -2039,45 +2082,25 @@ ice_vsi_add_rem_eth_mac(struct ice_vsi *vsi, bool add_rule) */ void ice_cfg_sw_lldp(struct ice_vsi *vsi, bool tx, bool create) { - struct ice_fltr_list_entry *list; + enum ice_status (*eth_fltr)(struct ice_vsi *v, u16 type, u16 flag, + enum ice_sw_fwd_act_type act); struct ice_pf *pf = vsi->back; - LIST_HEAD(tmp_add_list); enum ice_status status; struct device *dev; dev = ice_pf_to_dev(pf); - list = devm_kzalloc(dev, sizeof(*list), GFP_KERNEL); - if (!list) - return; + eth_fltr = create ? ice_fltr_add_eth : ice_fltr_remove_eth; - list->fltr_info.lkup_type = ICE_SW_LKUP_ETHERTYPE; - list->fltr_info.vsi_handle = vsi->idx; - list->fltr_info.l_data.ethertype_mac.ethertype = ETH_P_LLDP; - - if (tx) { - list->fltr_info.fltr_act = ICE_DROP_PACKET; - list->fltr_info.flag = ICE_FLTR_TX; - list->fltr_info.src_id = ICE_SRC_ID_VSI; - } else { - list->fltr_info.fltr_act = ICE_FWD_TO_VSI; - list->fltr_info.flag = ICE_FLTR_RX; - list->fltr_info.src_id = ICE_SRC_ID_LPORT; - } - - INIT_LIST_HEAD(&list->list_entry); - list_add(&list->list_entry, &tmp_add_list); - - if (create) - status = ice_add_eth_mac(&pf->hw, &tmp_add_list); + if (tx) + status = eth_fltr(vsi, ETH_P_LLDP, ICE_FLTR_TX, + ICE_DROP_PACKET); else - status = ice_remove_eth_mac(&pf->hw, &tmp_add_list); + status = eth_fltr(vsi, ETH_P_LLDP, ICE_FLTR_RX, ICE_FWD_TO_VSI); if (status) - dev_err(dev, "Fail %s %s LLDP rule on VSI %i error: %d\n", + dev_err(dev, "Fail %s %s LLDP rule on VSI %i error: %s\n", create ? "adding" : "removing", tx ? "TX" : "RX", - vsi->vsi_num, status); - - ice_free_fltr_list(dev, &tmp_add_list); + vsi->vsi_num, ice_stat_str(status)); } /** @@ -2122,10 +2145,12 @@ ice_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi, if (vsi->type == ICE_VSI_VF) vsi->vf_id = vf_id; + ice_alloc_fd_res(vsi); + if (ice_vsi_get_qs(vsi)) { dev_err(dev, "Failed to allocate queues. vsi->idx = %d\n", vsi->idx); - goto unroll_get_qs; + goto unroll_vsi_alloc; } /* set RSS capabilities */ @@ -2140,6 +2165,7 @@ ice_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi, goto unroll_get_qs; switch (vsi->type) { + case ICE_VSI_CTRL: case ICE_VSI_PF: ret = ice_vsi_alloc_q_vectors(vsi); if (ret) @@ -2164,20 +2190,23 @@ ice_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi, * so this handles those cases (i.e. adding the PF to a bridge * without the 8021q module loaded). */ - ret = ice_vsi_add_vlan(vsi, 0); + ret = ice_vsi_add_vlan(vsi, 0, ICE_FWD_TO_VSI); if (ret) goto unroll_clear_rings; ice_vsi_map_rings_to_vectors(vsi); - /* Do not exit if configuring RSS had an issue, at least - * receive traffic on first queue. Hence no need to capture - * return value - */ - if (test_bit(ICE_FLAG_RSS_ENA, pf->flags)) { - ice_vsi_cfg_rss_lut_key(vsi); - ice_vsi_set_rss_flow_fld(vsi); - } + /* ICE_VSI_CTRL does not need RSS so skip RSS processing */ + if (vsi->type != ICE_VSI_CTRL) + /* Do not exit if configuring RSS had an issue, at + * least receive traffic on first queue. Hence no + * need to capture return value + */ + if (test_bit(ICE_FLAG_RSS_ENA, pf->flags)) { + ice_vsi_cfg_rss_lut_key(vsi); + ice_vsi_set_rss_flow_fld(vsi); + } + ice_init_arfs(vsi); break; case ICE_VSI_VF: /* VF driver will take care of creating netdev for this type and @@ -2223,8 +2252,8 @@ ice_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi, status = ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc, max_txqs); if (status) { - dev_err(dev, "VSI %d failed lan queue config, error %d\n", - vsi->vsi_num, status); + dev_err(dev, "VSI %d failed lan queue config, error %s\n", + vsi->vsi_num, ice_stat_str(status)); goto unroll_vector_base; } @@ -2239,9 +2268,8 @@ ice_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi, */ if (!ice_is_safe_mode(pf)) if (vsi->type == ICE_VSI_PF) { - ice_vsi_add_rem_eth_mac(vsi, true); - - /* Tx LLDP packets */ + ice_fltr_add_eth(vsi, ETH_P_PAUSE, ICE_FLTR_TX, + ICE_DROP_PACKET); ice_cfg_sw_lldp(vsi, true, true); } @@ -2259,6 +2287,7 @@ unroll_vsi_init: ice_vsi_delete(vsi); unroll_get_qs: ice_vsi_put_qs(vsi); +unroll_vsi_alloc: ice_vsi_clear(vsi); return NULL; @@ -2411,6 +2440,8 @@ int ice_ena_vsi(struct ice_vsi *vsi, bool locked) if (!locked) rtnl_unlock(); } + } else if (vsi->type == ICE_VSI_CTRL) { + err = ice_vsi_open_ctrl(vsi); } return err; @@ -2440,6 +2471,8 @@ void ice_dis_vsi(struct ice_vsi *vsi, bool locked) } else { ice_vsi_close(vsi); } + } else if (vsi->type == ICE_VSI_CTRL) { + ice_vsi_close(vsi); } } @@ -2558,7 +2591,8 @@ int ice_vsi_release(struct ice_vsi *vsi) if (!ice_is_safe_mode(pf)) { if (vsi->type == ICE_VSI_PF) { - ice_vsi_add_rem_eth_mac(vsi, false); + ice_fltr_remove_eth(vsi, ETH_P_PAUSE, ICE_FLTR_TX, + ICE_DROP_PACKET); ice_cfg_sw_lldp(vsi, true, false); /* The Rx rule will only exist to remove if the LLDP FW * engine is currently stopped @@ -2568,7 +2602,7 @@ int ice_vsi_release(struct ice_vsi *vsi) } } - ice_remove_vsi_fltr(&pf->hw, vsi->idx); + ice_fltr_remove_all(vsi); ice_rm_vsi_lan_cfg(vsi->port_info, vsi->idx); ice_vsi_delete(vsi); ice_vsi_free_q_vectors(vsi); @@ -2673,15 +2707,13 @@ ice_vsi_rebuild_set_coalesce(struct ice_vsi *vsi, ice_vsi_rebuild_update_coalesce(vsi->q_vectors[i], &coalesce[i]); - for (; i < vsi->num_q_vectors; i++) { - struct ice_coalesce_stored coalesce_dflt = { - .itr_tx = ICE_DFLT_TX_ITR, - .itr_rx = ICE_DFLT_RX_ITR, - .intrl = 0 - }; + /* number of q_vectors increased, so assume coalesce settings were + * changed globally (i.e. ethtool -C eth0 instead of per-queue) and use + * the previous settings from q_vector 0 for all of the new q_vectors + */ + for (; i < vsi->num_q_vectors; i++) ice_vsi_rebuild_update_coalesce(vsi->q_vectors[i], - &coalesce_dflt); - } + &coalesce[0]); } /** @@ -2746,6 +2778,8 @@ int ice_vsi_rebuild(struct ice_vsi *vsi, bool init_vsi) goto err_vsi; ice_vsi_get_qs(vsi); + + ice_alloc_fd_res(vsi); ice_vsi_set_tc_cfg(vsi); /* Initialize VSI struct elements and create VSI in FW */ @@ -2754,6 +2788,7 @@ int ice_vsi_rebuild(struct ice_vsi *vsi, bool init_vsi) goto err_vsi; switch (vsi->type) { + case ICE_VSI_CTRL: case ICE_VSI_PF: ret = ice_vsi_alloc_q_vectors(vsi); if (ret) @@ -2773,17 +2808,19 @@ int ice_vsi_rebuild(struct ice_vsi *vsi, bool init_vsi) ice_vsi_map_rings_to_vectors(vsi); if (ice_is_xdp_ena_vsi(vsi)) { - vsi->num_xdp_txq = vsi->alloc_txq; + vsi->num_xdp_txq = vsi->alloc_rxq; ret = ice_prepare_xdp_rings(vsi, vsi->xdp_prog); if (ret) goto err_vectors; } - /* Do not exit if configuring RSS had an issue, at least - * receive traffic on first queue. Hence no need to capture - * return value - */ - if (test_bit(ICE_FLAG_RSS_ENA, pf->flags)) - ice_vsi_cfg_rss_lut_key(vsi); + /* ICE_VSI_CTRL does not need RSS so skip RSS processing */ + if (vsi->type != ICE_VSI_CTRL) + /* Do not exit if configuring RSS had an issue, at + * least receive traffic on first queue. Hence no + * need to capture return value + */ + if (test_bit(ICE_FLAG_RSS_ENA, pf->flags)) + ice_vsi_cfg_rss_lut_key(vsi); break; case ICE_VSI_VF: ret = ice_vsi_alloc_q_vectors(vsi); @@ -2814,8 +2851,8 @@ int ice_vsi_rebuild(struct ice_vsi *vsi, bool init_vsi) status = ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc, max_txqs); if (status) { - dev_err(ice_pf_to_dev(pf), "VSI %d failed lan queue config, error %d\n", - vsi->vsi_num, status); + dev_err(ice_pf_to_dev(pf), "VSI %d failed lan queue config, error %s\n", + vsi->vsi_num, ice_stat_str(status)); if (init_vsi) { ret = -EIO; goto err_vectors; @@ -2924,8 +2961,8 @@ int ice_vsi_cfg_tc(struct ice_vsi *vsi, u8 ena_tc) max_txqs); if (status) { - dev_err(dev, "VSI %d failed TC config, error %d\n", - vsi->vsi_num, status); + dev_err(dev, "VSI %d failed TC config, error %s\n", + vsi->vsi_num, ice_stat_str(status)); ret = -EIO; goto out; } @@ -2985,33 +3022,27 @@ void ice_update_rx_ring_stats(struct ice_ring *rx_ring, u64 pkts, u64 bytes) } /** - * ice_vsi_cfg_mac_fltr - Add or remove a MAC address filter for a VSI - * @vsi: the VSI being configured MAC filter - * @macaddr: the MAC address to be added. - * @set: Add or delete a MAC filter - * - * Adds or removes MAC address filter entry for VF VSI + * ice_status_to_errno - convert from enum ice_status to Linux errno + * @err: ice_status value to convert */ -enum ice_status -ice_vsi_cfg_mac_fltr(struct ice_vsi *vsi, const u8 *macaddr, bool set) +int ice_status_to_errno(enum ice_status err) { - LIST_HEAD(tmp_add_list); - enum ice_status status; - - /* Update MAC filter list to be added or removed for a VSI */ - if (ice_add_mac_to_list(vsi, &tmp_add_list, macaddr)) { - status = ICE_ERR_NO_MEMORY; - goto cfg_mac_fltr_exit; + switch (err) { + case ICE_SUCCESS: + return 0; + case ICE_ERR_DOES_NOT_EXIST: + return -ENOENT; + case ICE_ERR_OUT_OF_RANGE: + return -ENOTTY; + case ICE_ERR_PARAM: + return -EINVAL; + case ICE_ERR_NO_MEMORY: + return -ENOMEM; + case ICE_ERR_MAX_LIMIT: + return -EAGAIN; + default: + return -EINVAL; } - - if (set) - status = ice_add_mac(&vsi->back->hw, &tmp_add_list); - else - status = ice_remove_mac(&vsi->back->hw, &tmp_add_list); - -cfg_mac_fltr_exit: - ice_free_fltr_list(ice_pf_to_dev(vsi->back), &tmp_add_list); - return status; } /** @@ -3079,8 +3110,8 @@ int ice_set_dflt_vsi(struct ice_sw *sw, struct ice_vsi *vsi) status = ice_cfg_dflt_vsi(&vsi->back->hw, vsi->idx, true, ICE_FLTR_RX); if (status) { - dev_err(dev, "Failed to set VSI %d as the default forwarding VSI, error %d\n", - vsi->vsi_num, status); + dev_err(dev, "Failed to set VSI %d as the default forwarding VSI, error %s\n", + vsi->vsi_num, ice_stat_str(status)); return -EIO; } @@ -3118,8 +3149,8 @@ int ice_clear_dflt_vsi(struct ice_sw *sw) status = ice_cfg_dflt_vsi(&dflt_vsi->back->hw, dflt_vsi->idx, false, ICE_FLTR_RX); if (status) { - dev_err(dev, "Failed to clear the default forwarding VSI %d, error %d\n", - dflt_vsi->vsi_num, status); + dev_err(dev, "Failed to clear the default forwarding VSI %d, error %s\n", + dflt_vsi->vsi_num, ice_stat_str(status)); return -EIO; } diff --git a/drivers/net/ethernet/intel/ice/ice_lib.h b/drivers/net/ethernet/intel/ice/ice_lib.h index 04ca00799364..d80e6afa4511 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.h +++ b/drivers/net/ethernet/intel/ice/ice_lib.h @@ -8,12 +8,6 @@ const char *ice_vsi_type_str(enum ice_vsi_type vsi_type); -int -ice_add_mac_to_list(struct ice_vsi *vsi, struct list_head *add_list, - const u8 *macaddr); - -void ice_free_fltr_list(struct device *dev, struct list_head *h); - void ice_update_eth_stats(struct ice_vsi *vsi); int ice_vsi_cfg_rxqs(struct ice_vsi *vsi); @@ -22,7 +16,8 @@ int ice_vsi_cfg_lan_txqs(struct ice_vsi *vsi); void ice_vsi_cfg_msix(struct ice_vsi *vsi); -int ice_vsi_add_vlan(struct ice_vsi *vsi, u16 vid); +int +ice_vsi_add_vlan(struct ice_vsi *vsi, u16 vid, enum ice_sw_fwd_act_type action); int ice_vsi_kill_vlan(struct ice_vsi *vsi, u16 vid); @@ -79,6 +74,9 @@ int ice_vsi_rebuild(struct ice_vsi *vsi, bool init_vsi); bool ice_is_reset_in_progress(unsigned long *state); +void +ice_write_qrxflxp_cntxt(struct ice_hw *hw, u16 pf_q, u32 rxdid, u32 prio); + void ice_vsi_put_qs(struct ice_vsi *vsi); void ice_vsi_dis_irq(struct ice_vsi *vsi); @@ -97,6 +95,8 @@ void ice_update_rx_ring_stats(struct ice_ring *ring, u64 pkts, u64 bytes); void ice_vsi_cfg_frame_size(struct ice_vsi *vsi); +int ice_status_to_errno(enum ice_status err); + u32 ice_intrl_usec_to_reg(u8 intrl, u8 gran); enum ice_status diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index 5b190c257124..082825e3cb39 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -8,6 +8,7 @@ #include "ice.h" #include "ice_base.h" #include "ice_lib.h" +#include "ice_fltr.h" #include "ice_dcb_lib.h" #include "ice_dcb_nl.h" #include "ice_devlink.h" @@ -133,38 +134,24 @@ static void ice_check_for_hang_subtask(struct ice_pf *pf) static int ice_init_mac_fltr(struct ice_pf *pf) { enum ice_status status; - u8 broadcast[ETH_ALEN]; struct ice_vsi *vsi; + u8 *perm_addr; vsi = ice_get_main_vsi(pf); if (!vsi) return -EINVAL; - /* To add a MAC filter, first add the MAC to a list and then - * pass the list to ice_add_mac. - */ - - /* Add a unicast MAC filter so the VSI can get its packets */ - status = ice_vsi_cfg_mac_fltr(vsi, vsi->port_info->mac.perm_addr, true); - if (status) - goto unregister; - - /* VSI needs to receive broadcast traffic, so add the broadcast - * MAC address to the list as well. - */ - eth_broadcast_addr(broadcast); - status = ice_vsi_cfg_mac_fltr(vsi, broadcast, true); - if (status) - goto unregister; + perm_addr = vsi->port_info->mac.perm_addr; + status = ice_fltr_add_mac_and_broadcast(vsi, perm_addr, ICE_FWD_TO_VSI); + if (!status) + return 0; - return 0; -unregister: /* We aren't useful with no MAC filters, so unregister if we * had an error */ - if (status && vsi->netdev->reg_state == NETREG_REGISTERED) { - dev_err(ice_pf_to_dev(pf), "Could not add MAC filters error %d. Unregistering device\n", - status); + if (vsi->netdev->reg_state == NETREG_REGISTERED) { + dev_err(ice_pf_to_dev(pf), "Could not add MAC filters error %s. Unregistering device\n", + ice_stat_str(status)); unregister_netdev(vsi->netdev); free_netdev(vsi->netdev); vsi->netdev = NULL; @@ -188,7 +175,8 @@ static int ice_add_mac_to_sync_list(struct net_device *netdev, const u8 *addr) struct ice_netdev_priv *np = netdev_priv(netdev); struct ice_vsi *vsi = np->vsi; - if (ice_add_mac_to_list(vsi, &vsi->tmp_sync_list, addr)) + if (ice_fltr_add_mac_to_list(vsi, &vsi->tmp_sync_list, addr, + ICE_FWD_TO_VSI)) return -EINVAL; return 0; @@ -209,7 +197,8 @@ static int ice_add_mac_to_unsync_list(struct net_device *netdev, const u8 *addr) struct ice_netdev_priv *np = netdev_priv(netdev); struct ice_vsi *vsi = np->vsi; - if (ice_add_mac_to_list(vsi, &vsi->tmp_unsync_list, addr)) + if (ice_fltr_add_mac_to_list(vsi, &vsi->tmp_unsync_list, addr, + ICE_FWD_TO_VSI)) return -EINVAL; return 0; @@ -307,8 +296,8 @@ static int ice_vsi_sync_fltr(struct ice_vsi *vsi) } /* Remove MAC addresses in the unsync list */ - status = ice_remove_mac(hw, &vsi->tmp_unsync_list); - ice_free_fltr_list(dev, &vsi->tmp_unsync_list); + status = ice_fltr_remove_mac_list(vsi, &vsi->tmp_unsync_list); + ice_fltr_free_list(dev, &vsi->tmp_unsync_list); if (status) { netdev_err(netdev, "Failed to delete MAC filters\n"); /* if we failed because of alloc failures, just bail */ @@ -319,8 +308,8 @@ static int ice_vsi_sync_fltr(struct ice_vsi *vsi) } /* Add MAC addresses in the sync list */ - status = ice_add_mac(hw, &vsi->tmp_sync_list); - ice_free_fltr_list(dev, &vsi->tmp_sync_list); + status = ice_fltr_add_mac_list(vsi, &vsi->tmp_sync_list); + ice_fltr_free_list(dev, &vsi->tmp_sync_list); /* If filter is added successfully or already exists, do not go into * 'if' condition and report it as error. Instead continue processing * rest of the function. @@ -357,7 +346,8 @@ static int ice_vsi_sync_fltr(struct ice_vsi *vsi) vsi->current_netdev_flags &= ~IFF_ALLMULTI; goto out_promisc; } - } else if (!(vsi->current_netdev_flags & IFF_ALLMULTI)) { + } else { + /* !(vsi->current_netdev_flags & IFF_ALLMULTI) */ if (vsi->vlan_ena) promisc_m = ICE_MCAST_VLAN_PROMISC_BITS; else @@ -462,7 +452,7 @@ static void ice_prepare_for_reset(struct ice_pf *pf) { struct ice_hw *hw = &pf->hw; - int i; + unsigned int i; /* already prepared for reset */ if (test_bit(__ICE_PREPARED_FOR_RESET, pf->state)) @@ -1017,8 +1007,8 @@ static int __ice_clean_ctrlq(struct ice_pf *pf, enum ice_ctl_q q_type) if (ret == ICE_ERR_AQ_NO_WORK) break; if (ret) { - dev_err(dev, "%s Receive Queue event error %d\n", qtype, - ret); + dev_err(dev, "%s Receive Queue event error %s\n", qtype, + ice_stat_str(ret)); break; } @@ -1123,7 +1113,7 @@ static void ice_clean_mailboxq_subtask(struct ice_pf *pf) * * If not already scheduled, this puts the task into the work queue. */ -static void ice_service_task_schedule(struct ice_pf *pf) +void ice_service_task_schedule(struct ice_pf *pf) { if (!test_bit(__ICE_SERVICE_DIS, pf->state) && !test_and_set_bit(__ICE_SERVICE_SCHED, pf->state) && @@ -1198,8 +1188,8 @@ static void ice_handle_mdd_event(struct ice_pf *pf) { struct device *dev = ice_pf_to_dev(pf); struct ice_hw *hw = &pf->hw; + unsigned int i; u32 reg; - int i; if (!test_and_clear_bit(__ICE_MDD_EVENT_PENDING, pf->state)) { /* Since the VF MDD event logging is rate limited, check if @@ -1332,8 +1322,13 @@ static void ice_handle_mdd_event(struct ice_pf *pf) * PF can be configured to reset the VF through ethtool * private flag mdd-auto-reset-vf. */ - if (test_bit(ICE_FLAG_MDD_AUTO_RESET_VF, pf->flags)) + if (test_bit(ICE_FLAG_MDD_AUTO_RESET_VF, pf->flags)) { + /* VF MDD event counters will be cleared by + * reset, so print the event prior to reset. + */ + ice_print_vf_rx_mdd_event(vf); ice_reset_vf(&pf->vf[i], false); + } } } @@ -1493,7 +1488,7 @@ static void ice_service_task(struct work_struct *work) ice_process_vflr_event(pf); ice_clean_mailboxq_subtask(pf); - + ice_sync_arfs_fltrs(pf); /* Clear __ICE_SERVICE_SCHED flag to allow scheduling next event */ ice_service_task_complete(pf); @@ -1652,9 +1647,14 @@ static int ice_vsi_req_irq_msix(struct ice_vsi *vsi, char *basename) } /* register for affinity change notifications */ - q_vector->affinity_notify.notify = ice_irq_affinity_notify; - q_vector->affinity_notify.release = ice_irq_affinity_release; - irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); + if (!IS_ENABLED(CONFIG_RFS_ACCEL)) { + struct irq_affinity_notify *affinity_notify; + + affinity_notify = &q_vector->affinity_notify; + affinity_notify->notify = ice_irq_affinity_notify; + affinity_notify->release = ice_irq_affinity_release; + irq_set_affinity_notifier(irq_num, affinity_notify); + } /* assign the mask for this irq */ irq_set_affinity_hint(irq_num, &q_vector->affinity_mask); @@ -1666,8 +1666,9 @@ static int ice_vsi_req_irq_msix(struct ice_vsi *vsi, char *basename) free_q_irqs: while (vector) { vector--; - irq_num = pf->msix_entries[base + vector].vector, - irq_set_affinity_notifier(irq_num, NULL); + irq_num = pf->msix_entries[base + vector].vector; + if (!IS_ENABLED(CONFIG_RFS_ACCEL)) + irq_set_affinity_notifier(irq_num, NULL); irq_set_affinity_hint(irq_num, NULL); devm_free_irq(dev, irq_num, &vsi->q_vectors[vector]); } @@ -1809,8 +1810,8 @@ int ice_prepare_xdp_rings(struct ice_vsi *vsi, struct bpf_prog *prog) status = ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc, max_txqs); if (status) { - dev_err(dev, "Failed VSI LAN queue config for XDP, error:%d\n", - status); + dev_err(dev, "Failed VSI LAN queue config for XDP, error: %s\n", + ice_stat_str(status)); goto clear_xdp_rings; } ice_vsi_assign_bpf_prog(vsi, prog); @@ -1898,6 +1899,9 @@ free_qmap: for (i = 0; i < vsi->tc_cfg.numtc; i++) max_txqs[i] = vsi->num_txq; + /* change number of XDP Tx queues to 0 */ + vsi->num_xdp_txq = 0; + return ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc, max_txqs); } @@ -1931,7 +1935,7 @@ ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog, } if (!ice_is_xdp_ena_vsi(vsi) && prog) { - vsi->num_xdp_txq = vsi->alloc_txq; + vsi->num_xdp_txq = vsi->alloc_rxq; xdp_ring_err = ice_prepare_xdp_rings(vsi, prog); if (xdp_ring_err) NL_SET_ERR_MSG_MOD(extack, "Setting up XDP Tx resources failed"); @@ -2137,10 +2141,8 @@ static irqreturn_t ice_misc_intr(int __always_unused irq, void *data) } ret = IRQ_HANDLED; - if (!test_bit(__ICE_DOWN, pf->state)) { - ice_service_task_schedule(pf); - ice_irq_dynamic_ena(hw, NULL, NULL); - } + ice_service_task_schedule(pf); + ice_irq_dynamic_ena(hw, NULL, NULL); return ret; } @@ -2247,7 +2249,7 @@ static int ice_req_irq_msix_misc(struct ice_pf *pf) return oicr_idx; pf->num_avail_sw_msix -= 1; - pf->oicr_idx = oicr_idx; + pf->oicr_idx = (u16)oicr_idx; err = devm_request_irq(dev, pf->msix_entries[pf->oicr_idx].vector, ice_misc_intr, 0, pf->int_name, pf); @@ -2331,6 +2333,7 @@ static void ice_set_netdev_features(struct net_device *netdev) dflt_features = NETIF_F_SG | NETIF_F_HIGHDMA | + NETIF_F_NTUPLE | NETIF_F_RXHASH; csumo_features = NETIF_F_RXCSUM | @@ -2342,13 +2345,27 @@ static void ice_set_netdev_features(struct net_device *netdev) NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX; - tso_features = NETIF_F_TSO | + tso_features = NETIF_F_TSO | + NETIF_F_TSO_ECN | + NETIF_F_TSO6 | + NETIF_F_GSO_GRE | + NETIF_F_GSO_UDP_TUNNEL | + NETIF_F_GSO_GRE_CSUM | + NETIF_F_GSO_UDP_TUNNEL_CSUM | + NETIF_F_GSO_PARTIAL | + NETIF_F_GSO_IPXIP4 | + NETIF_F_GSO_IPXIP6 | NETIF_F_GSO_UDP_L4; + netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM | + NETIF_F_GSO_GRE_CSUM; /* set features that user can change */ netdev->hw_features = dflt_features | csumo_features | vlano_features | tso_features; + /* add support for HW_CSUM on packets with MPLS header */ + netdev->mpls_features = NETIF_F_HW_CSUM; + /* enable features */ netdev->features |= netdev->hw_features; /* encap and VLAN devices inherit default, csumo and tso features */ @@ -2411,7 +2428,7 @@ static int ice_cfg_netdev(struct ice_vsi *vsi) err = register_netdev(vsi->netdev); if (err) - goto err_destroy_devlink_port; + goto err_free_netdev; devlink_port_type_eth_set(&pf->devlink_port, vsi->netdev); @@ -2422,9 +2439,11 @@ static int ice_cfg_netdev(struct ice_vsi *vsi) return 0; +err_free_netdev: + free_netdev(vsi->netdev); + vsi->netdev = NULL; err_destroy_devlink_port: ice_devlink_destroy_port(pf); - return err; } @@ -2457,6 +2476,20 @@ ice_pf_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) } /** + * ice_ctrl_vsi_setup - Set up a control VSI + * @pf: board private structure + * @pi: pointer to the port_info instance + * + * Returns pointer to the successfully allocated VSI software struct + * on success, otherwise returns NULL on failure. + */ +static struct ice_vsi * +ice_ctrl_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) +{ + return ice_vsi_setup(pf, pi, ICE_VSI_CTRL, ICE_INVAL_VFID); +} + +/** * ice_lb_vsi_setup - Set up a loopback VSI * @pf: board private structure * @pi: pointer to the port_info instance @@ -2509,7 +2542,7 @@ ice_vlan_rx_add_vid(struct net_device *netdev, __always_unused __be16 proto, /* Add a switch rule for this VLAN ID so its corresponding VLAN tagged * packets aren't pruned by the device's internal switch on Rx */ - ret = ice_vsi_add_vlan(vsi, vid); + ret = ice_vsi_add_vlan(vsi, vid, ICE_FWD_TO_VSI); if (!ret) { vsi->vlan_ena = true; set_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags); @@ -2594,12 +2627,22 @@ static int ice_setup_pf_sw(struct ice_pf *pf) */ ice_napi_add(vsi); + status = ice_set_cpu_rx_rmap(vsi); + if (status) { + dev_err(ice_pf_to_dev(pf), "Failed to set CPU Rx map VSI %d error %d\n", + vsi->vsi_num, status); + status = -EINVAL; + goto unroll_napi_add; + } status = ice_init_mac_fltr(pf); if (status) - goto unroll_napi_add; + goto free_cpu_rx_map; return status; +free_cpu_rx_map: + ice_free_cpu_rx_rmap(vsi); + unroll_napi_add: if (vsi) { ice_napi_del(vsi); @@ -2630,7 +2673,8 @@ unroll_vsi_setup: static u16 ice_get_avail_q_count(unsigned long *pf_qmap, struct mutex *lock, u16 size) { - u16 count = 0, bit; + unsigned long bit; + u16 count = 0; mutex_lock(lock); for_each_clear_bit(bit, pf_qmap, size) @@ -2703,6 +2747,23 @@ static void ice_set_pf_caps(struct ice_pf *pf) if (func_caps->common_cap.rss_table_size) set_bit(ICE_FLAG_RSS_ENA, pf->flags); + clear_bit(ICE_FLAG_FD_ENA, pf->flags); + if (func_caps->fd_fltr_guar > 0 || func_caps->fd_fltr_best_effort > 0) { + u16 unused; + + /* ctrl_vsi_idx will be set to a valid value when flow director + * is setup by ice_init_fdir + */ + pf->ctrl_vsi_idx = ICE_NO_VSI; + set_bit(ICE_FLAG_FD_ENA, pf->flags); + /* force guaranteed filter pool for PF */ + ice_alloc_fd_guar_item(&pf->hw, &unused, + func_caps->fd_fltr_guar); + /* force shared filter pool for PF */ + ice_alloc_fd_shrd_item(&pf->hw, &unused, + func_caps->fd_fltr_best_effort); + } + pf->max_pf_txqs = func_caps->common_cap.num_txq; pf->max_pf_rxqs = func_caps->common_cap.num_rxq; } @@ -2769,6 +2830,15 @@ static int ice_ena_msix_range(struct ice_pf *pf) v_budget += needed; v_left -= needed; + /* reserve one vector for flow director */ + if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) { + needed = ICE_FDIR_MSIX; + if (v_left < needed) + goto no_hw_vecs_left_err; + v_budget += needed; + v_left -= needed; + } + pf->msix_entries = devm_kcalloc(dev, v_budget, sizeof(*pf->msix_entries), GFP_KERNEL); @@ -2793,8 +2863,10 @@ static int ice_ena_msix_range(struct ice_pf *pf) if (v_actual < v_budget) { dev_warn(dev, "not enough OS MSI-X vectors. requested = %d, obtained = %d\n", v_budget, v_actual); -/* 2 vectors for LAN (traffic + OICR) */ +/* 2 vectors each for LAN and RDMA (traffic + OICR), one for flow director */ #define ICE_MIN_LAN_VECS 2 +#define ICE_MIN_RDMA_VECS 2 +#define ICE_MIN_VECS (ICE_MIN_LAN_VECS + ICE_MIN_RDMA_VECS + 1) if (v_actual < ICE_MIN_LAN_VECS) { /* error if we can't get minimum vectors */ @@ -2869,8 +2941,8 @@ static int ice_init_interrupt_scheme(struct ice_pf *pf) } /* populate SW interrupts pool with number of OS granted IRQs. */ - pf->num_avail_sw_msix = vectors; - pf->irq_tracker->num_entries = vectors; + pf->num_avail_sw_msix = (u16)vectors; + pf->irq_tracker->num_entries = (u16)vectors; pf->irq_tracker->end = pf->irq_tracker->num_entries; return 0; @@ -2902,9 +2974,9 @@ int ice_vsi_recfg_qs(struct ice_vsi *vsi, int new_rx, int new_tx) } if (new_tx) - vsi->req_txq = new_tx; + vsi->req_txq = (u16)new_tx; if (new_rx) - vsi->req_rxq = new_rx; + vsi->req_rxq = (u16)new_rx; /* set for the next time the netdev is started */ if (!netif_running(vsi->netdev)) { @@ -2985,6 +3057,9 @@ ice_log_pkg_init(struct ice_hw *hw, enum ice_status *status) *status = ICE_ERR_NOT_SUPPORTED; } break; + case ICE_ERR_FW_DDP_MISMATCH: + dev_err(dev, "The firmware loaded on the device is not compatible with the DDP package. Please update the device's NVM. Entering safe mode.\n"); + break; case ICE_ERR_BUF_TOO_SHORT: case ICE_ERR_CFG: dev_err(dev, "The DDP package file is invalid. Entering Safe Mode.\n"); @@ -3013,6 +3088,9 @@ ice_log_pkg_init(struct ice_hw *hw, enum ice_status *status) case ICE_AQ_RC_EBADMAN: case ICE_AQ_RC_EBADBUF: dev_err(dev, "An error occurred on the device while loading the DDP package. The device will be reset.\n"); + /* poll for reset to complete */ + if (ice_check_reset(hw)) + dev_err(dev, "Error resetting device. Please reload the driver\n"); return; default: break; @@ -3100,6 +3178,53 @@ static enum ice_status ice_send_version(struct ice_pf *pf) } /** + * ice_init_fdir - Initialize flow director VSI and configuration + * @pf: pointer to the PF instance + * + * returns 0 on success, negative on error + */ +static int ice_init_fdir(struct ice_pf *pf) +{ + struct device *dev = ice_pf_to_dev(pf); + struct ice_vsi *ctrl_vsi; + int err; + + /* Side Band Flow Director needs to have a control VSI. + * Allocate it and store it in the PF. + */ + ctrl_vsi = ice_ctrl_vsi_setup(pf, pf->hw.port_info); + if (!ctrl_vsi) { + dev_dbg(dev, "could not create control VSI\n"); + return -ENOMEM; + } + + err = ice_vsi_open_ctrl(ctrl_vsi); + if (err) { + dev_dbg(dev, "could not open control VSI\n"); + goto err_vsi_open; + } + + mutex_init(&pf->hw.fdir_fltr_lock); + + err = ice_fdir_create_dflt_rules(pf); + if (err) + goto err_fdir_rule; + + return 0; + +err_fdir_rule: + ice_fdir_release_flows(&pf->hw); + ice_vsi_close(ctrl_vsi); +err_vsi_open: + ice_vsi_release(ctrl_vsi); + if (pf->ctrl_vsi_idx != ICE_NO_VSI) { + pf->vsi[pf->ctrl_vsi_idx] = NULL; + pf->ctrl_vsi_idx = ICE_NO_VSI; + } + return err; +} + +/** * ice_get_opt_fw_name - return optional firmware file name or NULL * @pf: pointer to the PF instance */ @@ -3123,7 +3248,7 @@ static char *ice_get_opt_fw_name(struct ice_pf *pf) if (!opt_fw_filename) return NULL; - snprintf(opt_fw_filename, NAME_MAX, "%sice-%016llX.pkg", + snprintf(opt_fw_filename, NAME_MAX, "%sice-%016llx.pkg", ICE_DDP_PKG_PATH, dsn); return opt_fw_filename; @@ -3295,12 +3420,9 @@ ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) if (err) { dev_err(dev, "ice_init_interrupt_scheme failed: %d\n", err); err = -EIO; - goto err_init_interrupt_unroll; + goto err_init_vsi_unroll; } - /* Driver is mostly up */ - clear_bit(__ICE_DOWN, pf->state); - /* In case of MSIX we are going to setup the misc vector right here * to handle admin queue events etc. In case of legacy and MSI * the misc functionality and queue processing is combined in @@ -3356,12 +3478,16 @@ ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) ice_verify_cacheline_size(pf); - /* If no DDP driven features have to be setup, return here */ + /* If no DDP driven features have to be setup, we are done with probe */ if (ice_is_safe_mode(pf)) - return 0; + goto probe_done; /* initialize DDP driven features */ + /* Note: Flow director init failure is non-fatal to load */ + if (ice_init_fdir(pf)) + dev_err(dev, "could not initialize flow director\n"); + /* Note: DCB init failure is non-fatal to load */ if (ice_init_pf_dcb(pf, false)) { clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags); @@ -3373,6 +3499,9 @@ ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) /* print PCI link speed and width */ pcie_print_link_status(pf->pdev); +probe_done: + /* ready to go, so clear down state bit */ + clear_bit(__ICE_DOWN, pf->state); return 0; err_alloc_sw_unroll: @@ -3384,6 +3513,7 @@ err_msix_misc_unroll: ice_free_irq_msix_misc(pf); err_init_interrupt_unroll: ice_clear_interrupt_scheme(pf); +err_init_vsi_unroll: devm_kfree(dev, pf->vsi); err_init_pf_unroll: ice_deinit_pf(pf); @@ -3421,6 +3551,9 @@ static void ice_remove(struct pci_dev *pdev) set_bit(__ICE_DOWN, pf->state); ice_service_task_stop(pf); + mutex_destroy(&(&pf->hw)->fdir_fltr_lock); + if (!ice_is_safe_mode(pf)) + ice_remove_arfs(pf); ice_devlink_destroy_port(pf); ice_vsi_release_all(pf); ice_free_irq_msix_misc(pf); @@ -3705,25 +3838,24 @@ static int ice_set_mac_address(struct net_device *netdev, void *pi) return -EBUSY; } - /* When we change the MAC address we also have to change the MAC address - * based filter rules that were created previously for the old MAC - * address. So first, we remove the old filter rule using ice_remove_mac - * and then create a new filter rule using ice_add_mac via - * ice_vsi_cfg_mac_fltr function call for both add and/or remove - * filters. - */ - status = ice_vsi_cfg_mac_fltr(vsi, netdev->dev_addr, false); - if (status) { + /* Clean up old MAC filter. Not an error if old filter doesn't exist */ + status = ice_fltr_remove_mac(vsi, netdev->dev_addr, ICE_FWD_TO_VSI); + if (status && status != ICE_ERR_DOES_NOT_EXIST) { err = -EADDRNOTAVAIL; goto err_update_filters; } - status = ice_vsi_cfg_mac_fltr(vsi, mac, true); - if (status) { - err = -EADDRNOTAVAIL; - goto err_update_filters; + /* Add filter for new MAC. If filter exists, just return success */ + status = ice_fltr_add_mac(vsi, mac, ICE_FWD_TO_VSI); + if (status == ICE_ERR_ALREADY_EXISTS) { + netdev_dbg(netdev, "filter for MAC %pM already exists\n", mac); + return 0; } + /* error if the new filter addition failed */ + if (status) + err = -EADDRNOTAVAIL; + err_update_filters: if (err) { netdev_err(netdev, "can't set MAC %pM. filter update failed\n", @@ -3740,8 +3872,8 @@ err_update_filters: flags = ICE_AQC_MAN_MAC_UPDATE_LAA_WOL; status = ice_aq_manage_mac_write(hw, mac, flags, NULL); if (status) { - netdev_err(netdev, "can't set MAC %pM. write to firmware failed error %d\n", - mac, status); + netdev_err(netdev, "can't set MAC %pM. write to firmware failed error %s\n", + mac, ice_stat_str(status)); } return 0; } @@ -3805,8 +3937,8 @@ ice_set_tx_maxrate(struct net_device *netdev, int queue_index, u32 maxrate) status = ice_cfg_q_bw_lmt(vsi->port_info, vsi->idx, tc, q_handle, ICE_MAX_BW, maxrate * 1000); if (status) { - netdev_err(netdev, "Unable to set Tx max rate, error %d\n", - status); + netdev_err(netdev, "Unable to set Tx max rate, error %s\n", + ice_stat_str(status)); return -EIO; } @@ -3938,6 +4070,16 @@ ice_set_features(struct net_device *netdev, netdev_features_t features) (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER)) ret = ice_cfg_vlan_pruning(vsi, false, false); + if ((features & NETIF_F_NTUPLE) && + !(netdev->features & NETIF_F_NTUPLE)) { + ice_vsi_manage_fdir(vsi, true); + ice_init_arfs(vsi); + } else if (!(features & NETIF_F_NTUPLE) && + (netdev->features & NETIF_F_NTUPLE)) { + ice_vsi_manage_fdir(vsi, false); + ice_clear_arfs(vsi); + } + return ret; } @@ -4084,6 +4226,33 @@ ice_fetch_u64_stats_per_ring(struct ice_ring *ring, u64 *pkts, u64 *bytes) } /** + * ice_update_vsi_tx_ring_stats - Update VSI Tx ring stats counters + * @vsi: the VSI to be updated + * @rings: rings to work on + * @count: number of rings + */ +static void +ice_update_vsi_tx_ring_stats(struct ice_vsi *vsi, struct ice_ring **rings, + u16 count) +{ + struct rtnl_link_stats64 *vsi_stats = &vsi->net_stats; + u16 i; + + for (i = 0; i < count; i++) { + struct ice_ring *ring; + u64 pkts, bytes; + + ring = READ_ONCE(rings[i]); + ice_fetch_u64_stats_per_ring(ring, &pkts, &bytes); + vsi_stats->tx_packets += pkts; + vsi_stats->tx_bytes += bytes; + vsi->tx_restart += ring->tx_stats.restart_q; + vsi->tx_busy += ring->tx_stats.tx_busy; + vsi->tx_linearize += ring->tx_stats.tx_linearize; + } +} + +/** * ice_update_vsi_ring_stats - Update VSI stats counters * @vsi: the VSI to be updated */ @@ -4110,15 +4279,7 @@ static void ice_update_vsi_ring_stats(struct ice_vsi *vsi) rcu_read_lock(); /* update Tx rings counters */ - ice_for_each_txq(vsi, i) { - ring = READ_ONCE(vsi->tx_rings[i]); - ice_fetch_u64_stats_per_ring(ring, &pkts, &bytes); - vsi_stats->tx_packets += pkts; - vsi_stats->tx_bytes += bytes; - vsi->tx_restart += ring->tx_stats.restart_q; - vsi->tx_busy += ring->tx_stats.tx_busy; - vsi->tx_linearize += ring->tx_stats.tx_linearize; - } + ice_update_vsi_tx_ring_stats(vsi, vsi->tx_rings, vsi->num_txq); /* update Rx rings counters */ ice_for_each_rxq(vsi, i) { @@ -4130,6 +4291,11 @@ static void ice_update_vsi_ring_stats(struct ice_vsi *vsi) vsi->rx_page_failed += ring->rx_stats.alloc_page_failed; } + /* update XDP Tx rings counters */ + if (ice_is_xdp_ena_vsi(vsi)) + ice_update_vsi_tx_ring_stats(vsi, vsi->xdp_rings, + vsi->num_xdp_txq); + rcu_read_unlock(); } @@ -4162,7 +4328,13 @@ void ice_update_vsi_stats(struct ice_vsi *vsi) if (vsi->type == ICE_VSI_PF) { cur_ns->rx_crc_errors = pf->stats.crc_errors; cur_ns->rx_errors = pf->stats.crc_errors + - pf->stats.illegal_bytes; + pf->stats.illegal_bytes + + pf->stats.rx_len_errors + + pf->stats.rx_undersize + + pf->hw_csum_rx_error + + pf->stats.rx_jabber + + pf->stats.rx_fragments + + pf->stats.rx_oversize; cur_ns->rx_length_errors = pf->stats.rx_len_errors; /* record drops from the port level */ cur_ns->rx_missed_errors = pf->stats.eth.rx_discards; @@ -4177,6 +4349,7 @@ void ice_update_pf_stats(struct ice_pf *pf) { struct ice_hw_port_stats *prev_ps, *cur_ps; struct ice_hw *hw = &pf->hw; + u16 fd_ctr_base; u8 port; port = hw->port_info->lport; @@ -4265,6 +4438,12 @@ void ice_update_pf_stats(struct ice_pf *pf) ice_stat_update40(hw, GLPRT_PTC9522L(port), pf->stat_prev_loaded, &prev_ps->tx_size_big, &cur_ps->tx_size_big); + fd_ctr_base = hw->fd_ctr_base; + + ice_stat_update40(hw, + GLSTAT_FD_CNT0L(ICE_FD_SB_STAT_IDX(fd_ctr_base)), + pf->stat_prev_loaded, &prev_ps->fd_sb_match, + &cur_ps->fd_sb_match); ice_stat_update32(hw, GLPRT_LXONRXC(port), pf->stat_prev_loaded, &prev_ps->link_xon_rx, &cur_ps->link_xon_rx); @@ -4308,6 +4487,8 @@ void ice_update_pf_stats(struct ice_pf *pf) ice_stat_update32(hw, GLPRT_RJC(port), pf->stat_prev_loaded, &prev_ps->rx_jabber, &cur_ps->rx_jabber); + cur_ps->fd_sb_status = test_bit(ICE_FLAG_FD_ENA, pf->flags) ? 1 : 0; + pf->stat_prev_loaded = true; } @@ -4493,6 +4674,62 @@ int ice_vsi_setup_rx_rings(struct ice_vsi *vsi) } /** + * ice_vsi_open_ctrl - open control VSI for use + * @vsi: the VSI to open + * + * Initialization of the Control VSI + * + * Returns 0 on success, negative value on error + */ +int ice_vsi_open_ctrl(struct ice_vsi *vsi) +{ + char int_name[ICE_INT_NAME_STR_LEN]; + struct ice_pf *pf = vsi->back; + struct device *dev; + int err; + + dev = ice_pf_to_dev(pf); + /* allocate descriptors */ + err = ice_vsi_setup_tx_rings(vsi); + if (err) + goto err_setup_tx; + + err = ice_vsi_setup_rx_rings(vsi); + if (err) + goto err_setup_rx; + + err = ice_vsi_cfg(vsi); + if (err) + goto err_setup_rx; + + snprintf(int_name, sizeof(int_name) - 1, "%s-%s:ctrl", + dev_driver_string(dev), dev_name(dev)); + err = ice_vsi_req_irq_msix(vsi, int_name); + if (err) + goto err_setup_rx; + + ice_vsi_cfg_msix(vsi); + + err = ice_vsi_start_all_rx_rings(vsi); + if (err) + goto err_up_complete; + + clear_bit(__ICE_DOWN, vsi->state); + ice_vsi_ena_irq(vsi); + + return 0; + +err_up_complete: + ice_down(vsi); +err_setup_rx: + ice_vsi_free_rx_rings(vsi); +err_setup_tx: + ice_vsi_free_tx_rings(vsi); + + return err; +} + +/** * ice_vsi_open - Called when a network interface is made active * @vsi: the VSI to open * @@ -4604,8 +4841,9 @@ static int ice_vsi_rebuild_by_type(struct ice_pf *pf, enum ice_vsi_type type) /* replay filters for the VSI */ status = ice_replay_vsi(&pf->hw, vsi->idx); if (status) { - dev_err(dev, "replay VSI failed, status %d, VSI index %d, type %s\n", - status, vsi->idx, ice_vsi_type_str(type)); + dev_err(dev, "replay VSI failed, status %s, VSI index %d, type %s\n", + ice_stat_str(status), vsi->idx, + ice_vsi_type_str(type)); return -EIO; } @@ -4659,6 +4897,11 @@ static void ice_update_pf_netdev_link(struct ice_pf *pf) * ice_rebuild - rebuild after reset * @pf: PF to rebuild * @reset_type: type of reset + * + * Do not rebuild VF VSI in this flow because that is already handled via + * ice_reset_all_vfs(). This is because requirements for resetting a VF after a + * PFR/CORER/GLOBER/etc. are different than the normal flow. Also, we don't want + * to reset/rebuild all the VF VSI twice. */ static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) { @@ -4674,7 +4917,8 @@ static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) ret = ice_init_all_ctrlq(hw); if (ret) { - dev_err(dev, "control queues init failed %d\n", ret); + dev_err(dev, "control queues init failed %s\n", + ice_stat_str(ret)); goto err_init_ctrlq; } @@ -4690,7 +4934,8 @@ static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) ret = ice_clear_pf_cfg(hw); if (ret) { - dev_err(dev, "clear PF configuration failed %d\n", ret); + dev_err(dev, "clear PF configuration failed %s\n", + ice_stat_str(ret)); goto err_init_ctrlq; } @@ -4704,7 +4949,13 @@ static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) ret = ice_get_caps(hw); if (ret) { - dev_err(dev, "ice_get_caps failed %d\n", ret); + dev_err(dev, "ice_get_caps failed %s\n", ice_stat_str(ret)); + goto err_init_ctrlq; + } + + ret = ice_aq_set_mac_cfg(hw, ICE_AQ_SET_MAC_FRAME_SIZE_MAX, NULL); + if (ret) { + dev_err(dev, "set_mac_cfg failed %s\n", ice_stat_str(ret)); goto err_init_ctrlq; } @@ -4723,6 +4974,21 @@ static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) goto err_sched_init_port; } + if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) { + wr32(hw, PFQF_FD_ENA, PFQF_FD_ENA_FD_ENA_M); + if (!rd32(hw, PFQF_FD_SIZE)) { + u16 unused, guar, b_effort; + + guar = hw->func_caps.fd_fltr_guar; + b_effort = hw->func_caps.fd_fltr_best_effort; + + /* force guaranteed filter pool for PF */ + ice_alloc_fd_guar_item(hw, &unused, guar); + /* force shared filter pool for PF */ + ice_alloc_fd_shrd_item(hw, &unused, b_effort); + } + } + if (test_bit(ICE_FLAG_DCB_ENA, pf->flags)) ice_dcb_rebuild(pf); @@ -4733,12 +4999,22 @@ static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) goto err_vsi_rebuild; } - if (test_bit(ICE_FLAG_SRIOV_ENA, pf->flags)) { - err = ice_vsi_rebuild_by_type(pf, ICE_VSI_VF); + /* If Flow Director is active */ + if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) { + err = ice_vsi_rebuild_by_type(pf, ICE_VSI_CTRL); if (err) { - dev_err(dev, "VF VSI rebuild failed: %d\n", err); + dev_err(dev, "control VSI rebuild failed: %d\n", err); goto err_vsi_rebuild; } + + /* replay HW Flow Director recipes */ + if (hw->fdir_prof) + ice_fdir_replay_flows(hw); + + /* replay Flow Director filters */ + ice_fdir_replay_fltrs(pf); + + ice_rebuild_arfs(pf); } ice_update_pf_netdev_link(pf); @@ -4746,8 +5022,8 @@ static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) /* tell the firmware we are up */ ret = ice_send_version(pf); if (ret) { - dev_err(dev, "Rebuild failed due to error sending driver version: %d\n", - ret); + dev_err(dev, "Rebuild failed due to error sending driver version: %s\n", + ice_stat_str(ret)); goto err_vsi_rebuild; } @@ -4795,7 +5071,7 @@ static int ice_change_mtu(struct net_device *netdev, int new_mtu) struct ice_pf *pf = vsi->back; u8 count = 0; - if (new_mtu == netdev->mtu) { + if (new_mtu == (int)netdev->mtu) { netdev_warn(netdev, "MTU is already %u\n", netdev->mtu); return 0; } @@ -4810,11 +5086,11 @@ static int ice_change_mtu(struct net_device *netdev, int new_mtu) } } - if (new_mtu < netdev->min_mtu) { + if (new_mtu < (int)netdev->min_mtu) { netdev_err(netdev, "new MTU invalid. min_mtu is %d\n", netdev->min_mtu); return -EINVAL; - } else if (new_mtu > netdev->max_mtu) { + } else if (new_mtu > (int)netdev->max_mtu) { netdev_err(netdev, "new MTU invalid. max_mtu is %d\n", netdev->min_mtu); return -EINVAL; @@ -4835,7 +5111,7 @@ static int ice_change_mtu(struct net_device *netdev, int new_mtu) return -EBUSY; } - netdev->mtu = new_mtu; + netdev->mtu = (unsigned int)new_mtu; /* if VSI is up, bring it down and then back up */ if (!test_and_set_bit(__ICE_DOWN, vsi->state)) { @@ -4859,6 +5135,118 @@ static int ice_change_mtu(struct net_device *netdev, int new_mtu) } /** + * ice_aq_str - convert AQ err code to a string + * @aq_err: the AQ error code to convert + */ +const char *ice_aq_str(enum ice_aq_err aq_err) +{ + switch (aq_err) { + case ICE_AQ_RC_OK: + return "OK"; + case ICE_AQ_RC_EPERM: + return "ICE_AQ_RC_EPERM"; + case ICE_AQ_RC_ENOENT: + return "ICE_AQ_RC_ENOENT"; + case ICE_AQ_RC_ENOMEM: + return "ICE_AQ_RC_ENOMEM"; + case ICE_AQ_RC_EBUSY: + return "ICE_AQ_RC_EBUSY"; + case ICE_AQ_RC_EEXIST: + return "ICE_AQ_RC_EEXIST"; + case ICE_AQ_RC_EINVAL: + return "ICE_AQ_RC_EINVAL"; + case ICE_AQ_RC_ENOSPC: + return "ICE_AQ_RC_ENOSPC"; + case ICE_AQ_RC_ENOSYS: + return "ICE_AQ_RC_ENOSYS"; + case ICE_AQ_RC_EMODE: + return "ICE_AQ_RC_EMODE"; + case ICE_AQ_RC_ENOSEC: + return "ICE_AQ_RC_ENOSEC"; + case ICE_AQ_RC_EBADSIG: + return "ICE_AQ_RC_EBADSIG"; + case ICE_AQ_RC_ESVN: + return "ICE_AQ_RC_ESVN"; + case ICE_AQ_RC_EBADMAN: + return "ICE_AQ_RC_EBADMAN"; + case ICE_AQ_RC_EBADBUF: + return "ICE_AQ_RC_EBADBUF"; + } + + return "ICE_AQ_RC_UNKNOWN"; +} + +/** + * ice_stat_str - convert status err code to a string + * @stat_err: the status error code to convert + */ +const char *ice_stat_str(enum ice_status stat_err) +{ + switch (stat_err) { + case ICE_SUCCESS: + return "OK"; + case ICE_ERR_PARAM: + return "ICE_ERR_PARAM"; + case ICE_ERR_NOT_IMPL: + return "ICE_ERR_NOT_IMPL"; + case ICE_ERR_NOT_READY: + return "ICE_ERR_NOT_READY"; + case ICE_ERR_NOT_SUPPORTED: + return "ICE_ERR_NOT_SUPPORTED"; + case ICE_ERR_BAD_PTR: + return "ICE_ERR_BAD_PTR"; + case ICE_ERR_INVAL_SIZE: + return "ICE_ERR_INVAL_SIZE"; + case ICE_ERR_DEVICE_NOT_SUPPORTED: + return "ICE_ERR_DEVICE_NOT_SUPPORTED"; + case ICE_ERR_RESET_FAILED: + return "ICE_ERR_RESET_FAILED"; + case ICE_ERR_FW_API_VER: + return "ICE_ERR_FW_API_VER"; + case ICE_ERR_NO_MEMORY: + return "ICE_ERR_NO_MEMORY"; + case ICE_ERR_CFG: + return "ICE_ERR_CFG"; + case ICE_ERR_OUT_OF_RANGE: + return "ICE_ERR_OUT_OF_RANGE"; + case ICE_ERR_ALREADY_EXISTS: + return "ICE_ERR_ALREADY_EXISTS"; + case ICE_ERR_NVM_CHECKSUM: + return "ICE_ERR_NVM_CHECKSUM"; + case ICE_ERR_BUF_TOO_SHORT: + return "ICE_ERR_BUF_TOO_SHORT"; + case ICE_ERR_NVM_BLANK_MODE: + return "ICE_ERR_NVM_BLANK_MODE"; + case ICE_ERR_IN_USE: + return "ICE_ERR_IN_USE"; + case ICE_ERR_MAX_LIMIT: + return "ICE_ERR_MAX_LIMIT"; + case ICE_ERR_RESET_ONGOING: + return "ICE_ERR_RESET_ONGOING"; + case ICE_ERR_HW_TABLE: + return "ICE_ERR_HW_TABLE"; + case ICE_ERR_DOES_NOT_EXIST: + return "ICE_ERR_DOES_NOT_EXIST"; + case ICE_ERR_FW_DDP_MISMATCH: + return "ICE_ERR_FW_DDP_MISMATCH"; + case ICE_ERR_AQ_ERROR: + return "ICE_ERR_AQ_ERROR"; + case ICE_ERR_AQ_TIMEOUT: + return "ICE_ERR_AQ_TIMEOUT"; + case ICE_ERR_AQ_FULL: + return "ICE_ERR_AQ_FULL"; + case ICE_ERR_AQ_NO_WORK: + return "ICE_ERR_AQ_NO_WORK"; + case ICE_ERR_AQ_EMPTY: + return "ICE_ERR_AQ_EMPTY"; + case ICE_ERR_AQ_FW_CRITICAL: + return "ICE_ERR_AQ_FW_CRITICAL"; + } + + return "ICE_ERR_UNKNOWN"; +} + +/** * ice_set_rss - Set RSS keys and lut * @vsi: Pointer to VSI structure * @seed: RSS hash seed @@ -4882,8 +5270,9 @@ int ice_set_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) status = ice_aq_set_rss_key(hw, vsi->idx, buf); if (status) { - dev_err(dev, "Cannot set RSS key, err %d aq_err %d\n", - status, hw->adminq.rq_last_status); + dev_err(dev, "Cannot set RSS key, err %s aq_err %s\n", + ice_stat_str(status), + ice_aq_str(hw->adminq.sq_last_status)); return -EIO; } } @@ -4892,8 +5281,9 @@ int ice_set_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) status = ice_aq_set_rss_lut(hw, vsi->idx, vsi->rss_lut_type, lut, lut_size); if (status) { - dev_err(dev, "Cannot set RSS lut, err %d aq_err %d\n", - status, hw->adminq.rq_last_status); + dev_err(dev, "Cannot set RSS lut, err %s aq_err %s\n", + ice_stat_str(status), + ice_aq_str(hw->adminq.sq_last_status)); return -EIO; } } @@ -4924,8 +5314,9 @@ int ice_get_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) status = ice_aq_get_rss_key(hw, vsi->idx, buf); if (status) { - dev_err(dev, "Cannot get RSS key, err %d aq_err %d\n", - status, hw->adminq.rq_last_status); + dev_err(dev, "Cannot get RSS key, err %s aq_err %s\n", + ice_stat_str(status), + ice_aq_str(hw->adminq.sq_last_status)); return -EIO; } } @@ -4934,8 +5325,9 @@ int ice_get_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) status = ice_aq_get_rss_lut(hw, vsi->idx, vsi->rss_lut_type, lut, lut_size); if (status) { - dev_err(dev, "Cannot get RSS lut, err %d aq_err %d\n", - status, hw->adminq.rq_last_status); + dev_err(dev, "Cannot get RSS lut, err %s aq_err %s\n", + ice_stat_str(status), + ice_aq_str(hw->adminq.sq_last_status)); return -EIO; } } @@ -5002,8 +5394,9 @@ static int ice_vsi_update_bridge_mode(struct ice_vsi *vsi, u16 bmode) status = ice_update_vsi(hw, vsi->idx, ctxt, NULL); if (status) { - dev_err(ice_pf_to_dev(vsi->back), "update VSI for bridge mode failed, bmode = %d err %d aq_err %d\n", - bmode, status, hw->adminq.sq_last_status); + dev_err(ice_pf_to_dev(vsi->back), "update VSI for bridge mode failed, bmode = %d err %s aq_err %s\n", + bmode, ice_stat_str(status), + ice_aq_str(hw->adminq.sq_last_status)); ret = -EIO; goto out; } @@ -5072,8 +5465,9 @@ ice_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, */ status = ice_update_sw_rule_bridge_mode(hw); if (status) { - netdev_err(dev, "switch rule update failed, mode = %d err %d aq_err %d\n", - mode, status, hw->adminq.sq_last_status); + netdev_err(dev, "switch rule update failed, mode = %d err %s aq_err %s\n", + mode, ice_stat_str(status), + ice_aq_str(hw->adminq.sq_last_status)); /* revert hw->evb_veb */ hw->evb_veb = (pf_sw->bridge_mode == BRIDGE_MODE_VEB); return -EIO; @@ -5100,6 +5494,16 @@ static void ice_tx_timeout(struct net_device *netdev, unsigned int txqueue) pf->tx_timeout_count++; + /* Check if PFC is enabled for the TC to which the queue belongs + * to. If yes then Tx timeout is not caused by a hung queue, no + * need to reset and rebuild + */ + if (ice_is_pfc_causing_hung_q(pf, txqueue)) { + dev_info(ice_pf_to_dev(pf), "Fake Tx hang detected on queue %u, timeout caused by PFC storm\n", + txqueue); + return; + } + /* now that we have an index, find the tx_ring struct */ for (i = 0; i < vsi->num_txq; i++) if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) @@ -5158,6 +5562,70 @@ static void ice_tx_timeout(struct net_device *netdev, unsigned int txqueue) } /** + * ice_udp_tunnel_add - Get notifications about UDP tunnel ports that come up + * @netdev: This physical port's netdev + * @ti: Tunnel endpoint information + */ +static void +ice_udp_tunnel_add(struct net_device *netdev, struct udp_tunnel_info *ti) +{ + struct ice_netdev_priv *np = netdev_priv(netdev); + struct ice_vsi *vsi = np->vsi; + struct ice_pf *pf = vsi->back; + enum ice_tunnel_type tnl_type; + u16 port = ntohs(ti->port); + enum ice_status status; + + switch (ti->type) { + case UDP_TUNNEL_TYPE_VXLAN: + tnl_type = TNL_VXLAN; + break; + case UDP_TUNNEL_TYPE_GENEVE: + tnl_type = TNL_GENEVE; + break; + default: + netdev_err(netdev, "Unknown tunnel type\n"); + return; + } + + status = ice_create_tunnel(&pf->hw, tnl_type, port); + if (status == ICE_ERR_OUT_OF_RANGE) + netdev_info(netdev, "Max tunneled UDP ports reached, port %d not added\n", + port); + else if (status) + netdev_err(netdev, "Error adding UDP tunnel - %s\n", + ice_stat_str(status)); +} + +/** + * ice_udp_tunnel_del - Get notifications about UDP tunnel ports that go away + * @netdev: This physical port's netdev + * @ti: Tunnel endpoint information + */ +static void +ice_udp_tunnel_del(struct net_device *netdev, struct udp_tunnel_info *ti) +{ + struct ice_netdev_priv *np = netdev_priv(netdev); + struct ice_vsi *vsi = np->vsi; + struct ice_pf *pf = vsi->back; + u16 port = ntohs(ti->port); + enum ice_status status; + bool retval; + + retval = ice_tunnel_port_in_use(&pf->hw, port, NULL); + if (!retval) { + netdev_info(netdev, "port %d not found in UDP tunnels list\n", + port); + return; + } + + status = ice_destroy_tunnel(&pf->hw, port, false); + if (status) + netdev_err(netdev, "error deleting port %d from UDP tunnels list\n", + port); +} + +/** * ice_open - Called when a network interface becomes active * @netdev: network interface device structure * @@ -5173,14 +5641,20 @@ int ice_open(struct net_device *netdev) { struct ice_netdev_priv *np = netdev_priv(netdev); struct ice_vsi *vsi = np->vsi; + struct ice_pf *pf = vsi->back; struct ice_port_info *pi; int err; - if (test_bit(__ICE_NEEDS_RESTART, vsi->back->state)) { + if (test_bit(__ICE_NEEDS_RESTART, pf->state)) { netdev_err(netdev, "driver needs to be unloaded and reloaded\n"); return -EIO; } + if (test_bit(__ICE_DOWN, pf->state)) { + netdev_err(netdev, "device is not ready yet\n"); + return -EBUSY; + } + netif_carrier_off(netdev); pi = vsi->port_info; @@ -5213,6 +5687,10 @@ int ice_open(struct net_device *netdev) if (err) netdev_err(netdev, "Failed to open VSI 0x%04X on switch 0x%04X\n", vsi->vsi_num, vsi->vsw->sw_id); + + /* Update existing tunnels information */ + udp_tunnel_get_rx_info(netdev); + return err; } @@ -5263,21 +5741,21 @@ ice_features_check(struct sk_buff *skb, features &= ~NETIF_F_GSO_MASK; len = skb_network_header(skb) - skb->data; - if (len & ~(ICE_TXD_MACLEN_MAX)) + if (len > ICE_TXD_MACLEN_MAX || len & 0x1) goto out_rm_features; len = skb_transport_header(skb) - skb_network_header(skb); - if (len & ~(ICE_TXD_IPLEN_MAX)) + if (len > ICE_TXD_IPLEN_MAX || len & 0x1) goto out_rm_features; if (skb->encapsulation) { len = skb_inner_network_header(skb) - skb_transport_header(skb); - if (len & ~(ICE_TXD_L4LEN_MAX)) + if (len > ICE_TXD_L4LEN_MAX || len & 0x1) goto out_rm_features; len = skb_inner_transport_header(skb) - skb_inner_network_header(skb); - if (len & ~(ICE_TXD_IPLEN_MAX)) + if (len > ICE_TXD_IPLEN_MAX || len & 0x1) goto out_rm_features; } @@ -5322,8 +5800,13 @@ static const struct net_device_ops ice_netdev_ops = { .ndo_bridge_setlink = ice_bridge_setlink, .ndo_fdb_add = ice_fdb_add, .ndo_fdb_del = ice_fdb_del, +#ifdef CONFIG_RFS_ACCEL + .ndo_rx_flow_steer = ice_rx_flow_steer, +#endif .ndo_tx_timeout = ice_tx_timeout, .ndo_bpf = ice_xdp, .ndo_xdp_xmit = ice_xdp_xmit, .ndo_xsk_wakeup = ice_xsk_wakeup, + .ndo_udp_tunnel_add = ice_udp_tunnel_add, + .ndo_udp_tunnel_del = ice_udp_tunnel_del, }; diff --git a/drivers/net/ethernet/intel/ice/ice_nvm.c b/drivers/net/ethernet/intel/ice/ice_nvm.c index 8beb675d676b..b049c1c30c88 100644 --- a/drivers/net/ethernet/intel/ice/ice_nvm.c +++ b/drivers/net/ethernet/intel/ice/ice_nvm.c @@ -172,7 +172,8 @@ void ice_release_nvm(struct ice_hw *hw) * * Reads one 16 bit word from the Shadow RAM using the ice_read_sr_word_aq. */ -enum ice_status ice_read_sr_word(struct ice_hw *hw, u16 offset, u16 *data) +static enum ice_status +ice_read_sr_word(struct ice_hw *hw, u16 offset, u16 *data) { enum ice_status status; @@ -196,7 +197,7 @@ enum ice_status ice_read_sr_word(struct ice_hw *hw, u16 offset, u16 *data) * Area (PFA) and returns the TLV pointer and length. The caller can * use these to read the variable length TLV value. */ -enum ice_status +static enum ice_status ice_get_pfa_module_tlv(struct ice_hw *hw, u16 *module_tlv, u16 *module_tlv_len, u16 module_type) { @@ -367,6 +368,87 @@ static enum ice_status ice_get_orom_ver_info(struct ice_hw *hw) } /** + * ice_get_netlist_ver_info + * @hw: pointer to the HW struct + * + * Get the netlist version information + */ +static enum ice_status ice_get_netlist_ver_info(struct ice_hw *hw) +{ + struct ice_netlist_ver_info *ver = &hw->netlist_ver; + enum ice_status ret; + u32 id_blk_start; + __le16 raw_data; + u16 data, i; + u16 *buff; + + ret = ice_acquire_nvm(hw, ICE_RES_READ); + if (ret) + return ret; + buff = kcalloc(ICE_AQC_NVM_NETLIST_ID_BLK_LEN, sizeof(*buff), + GFP_KERNEL); + if (!buff) { + ret = ICE_ERR_NO_MEMORY; + goto exit_no_mem; + } + + /* read module length */ + ret = ice_aq_read_nvm(hw, ICE_AQC_NVM_LINK_TOPO_NETLIST_MOD_ID, + ICE_AQC_NVM_LINK_TOPO_NETLIST_LEN_OFFSET * 2, + ICE_AQC_NVM_LINK_TOPO_NETLIST_LEN, &raw_data, + false, false, NULL); + if (ret) + goto exit_error; + + data = le16_to_cpu(raw_data); + /* exit if length is = 0 */ + if (!data) + goto exit_error; + + /* read node count */ + ret = ice_aq_read_nvm(hw, ICE_AQC_NVM_LINK_TOPO_NETLIST_MOD_ID, + ICE_AQC_NVM_NETLIST_NODE_COUNT_OFFSET * 2, + ICE_AQC_NVM_NETLIST_NODE_COUNT_LEN, &raw_data, + false, false, NULL); + if (ret) + goto exit_error; + data = le16_to_cpu(raw_data) & ICE_AQC_NVM_NETLIST_NODE_COUNT_M; + + /* netlist ID block starts from offset 4 + node count * 2 */ + id_blk_start = ICE_AQC_NVM_NETLIST_ID_BLK_START_OFFSET + data * 2; + + /* read the entire netlist ID block */ + ret = ice_aq_read_nvm(hw, ICE_AQC_NVM_LINK_TOPO_NETLIST_MOD_ID, + id_blk_start * 2, + ICE_AQC_NVM_NETLIST_ID_BLK_LEN * 2, buff, false, + false, NULL); + if (ret) + goto exit_error; + + for (i = 0; i < ICE_AQC_NVM_NETLIST_ID_BLK_LEN; i++) + buff[i] = le16_to_cpu(((__force __le16 *)buff)[i]); + + ver->major = (buff[ICE_AQC_NVM_NETLIST_ID_BLK_MAJOR_VER_HIGH] << 16) | + buff[ICE_AQC_NVM_NETLIST_ID_BLK_MAJOR_VER_LOW]; + ver->minor = (buff[ICE_AQC_NVM_NETLIST_ID_BLK_MINOR_VER_HIGH] << 16) | + buff[ICE_AQC_NVM_NETLIST_ID_BLK_MINOR_VER_LOW]; + ver->type = (buff[ICE_AQC_NVM_NETLIST_ID_BLK_TYPE_HIGH] << 16) | + buff[ICE_AQC_NVM_NETLIST_ID_BLK_TYPE_LOW]; + ver->rev = (buff[ICE_AQC_NVM_NETLIST_ID_BLK_REV_HIGH] << 16) | + buff[ICE_AQC_NVM_NETLIST_ID_BLK_REV_LOW]; + ver->cust_ver = buff[ICE_AQC_NVM_NETLIST_ID_BLK_CUST_VER]; + /* Read the left most 4 bytes of SHA */ + ver->hash = buff[ICE_AQC_NVM_NETLIST_ID_BLK_SHA_HASH + 15] << 16 | + buff[ICE_AQC_NVM_NETLIST_ID_BLK_SHA_HASH + 14]; + +exit_error: + kfree(buff); +exit_no_mem: + ice_release_nvm(hw); + return ret; +} + +/** * ice_discover_flash_size - Discover the available flash size. * @hw: pointer to the HW struct * @@ -515,6 +597,11 @@ enum ice_status ice_init_nvm(struct ice_hw *hw) return status; } + /* read the netlist version information */ + status = ice_get_netlist_ver_info(hw); + if (status) + ice_debug(hw, ICE_DBG_INIT, "Failed to read netlist info.\n"); + return 0; } diff --git a/drivers/net/ethernet/intel/ice/ice_nvm.h b/drivers/net/ethernet/intel/ice/ice_nvm.h index 999f273ba6ad..165eda07b93d 100644 --- a/drivers/net/ethernet/intel/ice/ice_nvm.h +++ b/drivers/net/ethernet/intel/ice/ice_nvm.h @@ -11,10 +11,6 @@ enum ice_status ice_read_flat_nvm(struct ice_hw *hw, u32 offset, u32 *length, u8 *data, bool read_shadow_ram); enum ice_status -ice_get_pfa_module_tlv(struct ice_hw *hw, u16 *module_tlv, u16 *module_tlv_len, - u16 module_type); -enum ice_status ice_read_pba_string(struct ice_hw *hw, u8 *pba_num, u32 pba_num_size); enum ice_status ice_init_nvm(struct ice_hw *hw); -enum ice_status ice_read_sr_word(struct ice_hw *hw, u16 offset, u16 *data); #endif /* _ICE_NVM_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_protocol_type.h b/drivers/net/ethernet/intel/ice/ice_protocol_type.h index 71647566964e..7f4c1ec1eff2 100644 --- a/drivers/net/ethernet/intel/ice/ice_protocol_type.h +++ b/drivers/net/ethernet/intel/ice/ice_protocol_type.h @@ -12,12 +12,15 @@ */ enum ice_prot_id { ICE_PROT_ID_INVAL = 0, + ICE_PROT_MAC_OF_OR_S = 1, ICE_PROT_IPV4_OF_OR_S = 32, ICE_PROT_IPV4_IL = 33, ICE_PROT_IPV6_OF_OR_S = 40, ICE_PROT_IPV6_IL = 41, ICE_PROT_TCP_IL = 49, + ICE_PROT_UDP_OF = 52, ICE_PROT_UDP_IL_OR_S = 53, + ICE_PROT_GRE_OF = 64, ICE_PROT_SCTP_IL = 96, ICE_PROT_META_ID = 255, /* when offset == metadata */ ICE_PROT_INVALID = 255 /* when offset == ICE_FV_OFFSET_INVAL */ diff --git a/drivers/net/ethernet/intel/ice/ice_sched.c b/drivers/net/ethernet/intel/ice/ice_sched.c index eae707ddf8e8..0475134295e4 100644 --- a/drivers/net/ethernet/intel/ice/ice_sched.c +++ b/drivers/net/ethernet/intel/ice/ice_sched.c @@ -1714,8 +1714,7 @@ ice_sched_cfg_vsi(struct ice_port_info *pi, u16 vsi_handle, u8 tc, u16 maxqs, * This function removes single aggregator VSI info entry from * aggregator list. */ -static void -ice_sched_rm_agg_vsi_info(struct ice_port_info *pi, u16 vsi_handle) +static void ice_sched_rm_agg_vsi_info(struct ice_port_info *pi, u16 vsi_handle) { struct ice_sched_agg_info *agg_info; struct ice_sched_agg_info *atmp; @@ -1917,7 +1916,7 @@ ice_sched_update_elem(struct ice_hw *hw, struct ice_sched_node *node, */ static enum ice_status ice_sched_cfg_node_bw_alloc(struct ice_hw *hw, struct ice_sched_node *node, - enum ice_rl_type rl_type, u8 bw_alloc) + enum ice_rl_type rl_type, u16 bw_alloc) { struct ice_aqc_txsched_elem_data buf; struct ice_aqc_txsched_elem *data; @@ -1947,8 +1946,7 @@ ice_sched_cfg_node_bw_alloc(struct ice_hw *hw, struct ice_sched_node *node, * * Save or clear CIR bandwidth (BW) in the passed param bw_t_info. */ -static void -ice_set_clear_cir_bw(struct ice_bw_type_info *bw_t_info, u32 bw) +static void ice_set_clear_cir_bw(struct ice_bw_type_info *bw_t_info, u32 bw) { if (bw == ICE_SCHED_DFLT_BW) { clear_bit(ICE_BW_TYPE_CIR, bw_t_info->bw_t_bitmap); @@ -1967,8 +1965,7 @@ ice_set_clear_cir_bw(struct ice_bw_type_info *bw_t_info, u32 bw) * * Save or clear EIR bandwidth (BW) in the passed param bw_t_info. */ -static void -ice_set_clear_eir_bw(struct ice_bw_type_info *bw_t_info, u32 bw) +static void ice_set_clear_eir_bw(struct ice_bw_type_info *bw_t_info, u32 bw) { if (bw == ICE_SCHED_DFLT_BW) { clear_bit(ICE_BW_TYPE_EIR, bw_t_info->bw_t_bitmap); @@ -1993,8 +1990,7 @@ ice_set_clear_eir_bw(struct ice_bw_type_info *bw_t_info, u32 bw) * * Save or clear shared bandwidth (BW) in the passed param bw_t_info. */ -static void -ice_set_clear_shared_bw(struct ice_bw_type_info *bw_t_info, u32 bw) +static void ice_set_clear_shared_bw(struct ice_bw_type_info *bw_t_info, u32 bw) { if (bw == ICE_SCHED_DFLT_BW) { clear_bit(ICE_BW_TYPE_SHARED, bw_t_info->bw_t_bitmap); diff --git a/drivers/net/ethernet/intel/ice/ice_status.h b/drivers/net/ethernet/intel/ice/ice_status.h index a9a8bc3aca42..4028c6365172 100644 --- a/drivers/net/ethernet/intel/ice/ice_status.h +++ b/drivers/net/ethernet/intel/ice/ice_status.h @@ -27,6 +27,8 @@ enum ice_status { ICE_ERR_MAX_LIMIT = -17, ICE_ERR_RESET_ONGOING = -18, ICE_ERR_HW_TABLE = -19, + ICE_ERR_FW_DDP_MISMATCH = -20, + ICE_ERR_NVM_CHECKSUM = -51, ICE_ERR_BUF_TOO_SHORT = -52, ICE_ERR_NVM_BLANK_MODE = -53, @@ -35,6 +37,7 @@ enum ice_status { ICE_ERR_AQ_FULL = -102, ICE_ERR_AQ_NO_WORK = -103, ICE_ERR_AQ_EMPTY = -104, + ICE_ERR_AQ_FW_CRITICAL = -105, }; #endif /* _ICE_STATUS_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_switch.c b/drivers/net/ethernet/intel/ice/ice_switch.c index 51825a203e35..ff7d16ac693e 100644 --- a/drivers/net/ethernet/intel/ice/ice_switch.c +++ b/drivers/net/ethernet/intel/ice/ice_switch.c @@ -593,8 +593,8 @@ enum ice_status ice_get_initial_sw_cfg(struct ice_hw *hw) ICE_AQC_GET_SW_CONF_RESP_IS_VF) is_vf = true; - res_type = le16_to_cpu(ele->vsi_port_num) >> - ICE_AQC_GET_SW_CONF_RESP_TYPE_S; + res_type = (u8)(le16_to_cpu(ele->vsi_port_num) >> + ICE_AQC_GET_SW_CONF_RESP_TYPE_S); if (res_type == ICE_AQC_GET_SW_CONF_RESP_VSI) { /* FW VSI is not needed. Just continue. */ @@ -1612,18 +1612,17 @@ exit: * check for duplicates in this case, removing duplicates from a given * list should be taken care of in the caller of this function. */ -enum ice_status -ice_add_mac(struct ice_hw *hw, struct list_head *m_list) +enum ice_status ice_add_mac(struct ice_hw *hw, struct list_head *m_list) { struct ice_aqc_sw_rules_elem *s_rule, *r_iter; struct ice_fltr_list_entry *m_list_itr; struct list_head *rule_head; - u16 elem_sent, total_elem_left; + u16 total_elem_left, s_rule_size; struct ice_switch_info *sw; struct mutex *rule_lock; /* Lock to protect filter rule list */ enum ice_status status = 0; u16 num_unicast = 0; - u16 s_rule_size; + u8 elem_sent; if (!m_list || !hw) return ICE_ERR_PARAM; @@ -1707,8 +1706,8 @@ ice_add_mac(struct ice_hw *hw, struct list_head *m_list) total_elem_left -= elem_sent) { struct ice_aqc_sw_rules_elem *entry = r_iter; - elem_sent = min(total_elem_left, - (u16)(ICE_AQ_MAX_BUF_LEN / s_rule_size)); + elem_sent = min_t(u8, total_elem_left, + (ICE_AQ_MAX_BUF_LEN / s_rule_size)); status = ice_aq_sw_rules(hw, entry, elem_sent * s_rule_size, elem_sent, ice_aqc_opc_add_sw_rules, NULL); @@ -1914,8 +1913,7 @@ exit: * @hw: pointer to the hardware structure * @v_list: list of VLAN entries and forwarding information */ -enum ice_status -ice_add_vlan(struct ice_hw *hw, struct list_head *v_list) +enum ice_status ice_add_vlan(struct ice_hw *hw, struct list_head *v_list) { struct ice_fltr_list_entry *v_list_itr; @@ -2145,8 +2143,7 @@ ice_find_ucast_rule_entry(struct ice_hw *hw, u8 recp_id, * the entries passed into m_list were added previously. It will not attempt to * do a partial remove of entries that were found. */ -enum ice_status -ice_remove_mac(struct ice_hw *hw, struct list_head *m_list) +enum ice_status ice_remove_mac(struct ice_hw *hw, struct list_head *m_list) { struct ice_fltr_list_entry *list_itr, *tmp; struct mutex *rule_lock; /* Lock to protect filter rule list */ @@ -2678,6 +2675,81 @@ void ice_remove_vsi_fltr(struct ice_hw *hw, u16 vsi_handle) } /** + * ice_alloc_res_cntr - allocating resource counter + * @hw: pointer to the hardware structure + * @type: type of resource + * @alloc_shared: if set it is shared else dedicated + * @num_items: number of entries requested for FD resource type + * @counter_id: counter index returned by AQ call + */ +enum ice_status +ice_alloc_res_cntr(struct ice_hw *hw, u8 type, u8 alloc_shared, u16 num_items, + u16 *counter_id) +{ + struct ice_aqc_alloc_free_res_elem *buf; + enum ice_status status; + u16 buf_len; + + /* Allocate resource */ + buf_len = sizeof(*buf); + buf = kzalloc(buf_len, GFP_KERNEL); + if (!buf) + return ICE_ERR_NO_MEMORY; + + buf->num_elems = cpu_to_le16(num_items); + buf->res_type = cpu_to_le16(((type << ICE_AQC_RES_TYPE_S) & + ICE_AQC_RES_TYPE_M) | alloc_shared); + + status = ice_aq_alloc_free_res(hw, 1, buf, buf_len, + ice_aqc_opc_alloc_res, NULL); + if (status) + goto exit; + + *counter_id = le16_to_cpu(buf->elem[0].e.sw_resp); + +exit: + kfree(buf); + return status; +} + +/** + * ice_free_res_cntr - free resource counter + * @hw: pointer to the hardware structure + * @type: type of resource + * @alloc_shared: if set it is shared else dedicated + * @num_items: number of entries to be freed for FD resource type + * @counter_id: counter ID resource which needs to be freed + */ +enum ice_status +ice_free_res_cntr(struct ice_hw *hw, u8 type, u8 alloc_shared, u16 num_items, + u16 counter_id) +{ + struct ice_aqc_alloc_free_res_elem *buf; + enum ice_status status; + u16 buf_len; + + /* Free resource */ + buf_len = sizeof(*buf); + buf = kzalloc(buf_len, GFP_KERNEL); + if (!buf) + return ICE_ERR_NO_MEMORY; + + buf->num_elems = cpu_to_le16(num_items); + buf->res_type = cpu_to_le16(((type << ICE_AQC_RES_TYPE_S) & + ICE_AQC_RES_TYPE_M) | alloc_shared); + buf->elem[0].e.sw_resp = cpu_to_le16(counter_id); + + status = ice_aq_alloc_free_res(hw, 1, buf, buf_len, + ice_aqc_opc_free_res, NULL); + if (status) + ice_debug(hw, ICE_DBG_SW, + "counter resource could not be freed\n"); + + kfree(buf); + return status; +} + +/** * ice_replay_vsi_fltr - Replay filters for requested VSI * @hw: pointer to the hardware structure * @vsi_handle: driver VSI handle diff --git a/drivers/net/ethernet/intel/ice/ice_switch.h b/drivers/net/ethernet/intel/ice/ice_switch.h index fa14b9545dab..8b4f9d35c860 100644 --- a/drivers/net/ethernet/intel/ice/ice_switch.h +++ b/drivers/net/ethernet/intel/ice/ice_switch.h @@ -208,6 +208,13 @@ void ice_clear_all_vsi_ctx(struct ice_hw *hw); /* Switch config */ enum ice_status ice_get_initial_sw_cfg(struct ice_hw *hw); +enum ice_status +ice_alloc_res_cntr(struct ice_hw *hw, u8 type, u8 alloc_shared, u16 num_items, + u16 *counter_id); +enum ice_status +ice_free_res_cntr(struct ice_hw *hw, u8 type, u8 alloc_shared, u16 num_items, + u16 counter_id); + /* Switch/bridge related commands */ enum ice_status ice_update_sw_rule_bridge_mode(struct ice_hw *hw); enum ice_status ice_add_mac(struct ice_hw *hw, struct list_head *m_lst); diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c index f67e8362958c..abdb137c8bb7 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx.c @@ -15,6 +15,90 @@ #define ICE_RX_HDR_SIZE 256 +#define FDIR_DESC_RXDID 0x40 +#define ICE_FDIR_CLEAN_DELAY 10 + +/** + * ice_prgm_fdir_fltr - Program a Flow Director filter + * @vsi: VSI to send dummy packet + * @fdir_desc: flow director descriptor + * @raw_packet: allocated buffer for flow director + */ +int +ice_prgm_fdir_fltr(struct ice_vsi *vsi, struct ice_fltr_desc *fdir_desc, + u8 *raw_packet) +{ + struct ice_tx_buf *tx_buf, *first; + struct ice_fltr_desc *f_desc; + struct ice_tx_desc *tx_desc; + struct ice_ring *tx_ring; + struct device *dev; + dma_addr_t dma; + u32 td_cmd; + u16 i; + + /* VSI and Tx ring */ + if (!vsi) + return -ENOENT; + tx_ring = vsi->tx_rings[0]; + if (!tx_ring || !tx_ring->desc) + return -ENOENT; + dev = tx_ring->dev; + + /* we are using two descriptors to add/del a filter and we can wait */ + for (i = ICE_FDIR_CLEAN_DELAY; ICE_DESC_UNUSED(tx_ring) < 2; i--) { + if (!i) + return -EAGAIN; + msleep_interruptible(1); + } + + dma = dma_map_single(dev, raw_packet, ICE_FDIR_MAX_RAW_PKT_SIZE, + DMA_TO_DEVICE); + + if (dma_mapping_error(dev, dma)) + return -EINVAL; + + /* grab the next descriptor */ + i = tx_ring->next_to_use; + first = &tx_ring->tx_buf[i]; + f_desc = ICE_TX_FDIRDESC(tx_ring, i); + memcpy(f_desc, fdir_desc, sizeof(*f_desc)); + + i++; + i = (i < tx_ring->count) ? i : 0; + tx_desc = ICE_TX_DESC(tx_ring, i); + tx_buf = &tx_ring->tx_buf[i]; + + i++; + tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; + + memset(tx_buf, 0, sizeof(*tx_buf)); + dma_unmap_len_set(tx_buf, len, ICE_FDIR_MAX_RAW_PKT_SIZE); + dma_unmap_addr_set(tx_buf, dma, dma); + + tx_desc->buf_addr = cpu_to_le64(dma); + td_cmd = ICE_TXD_LAST_DESC_CMD | ICE_TX_DESC_CMD_DUMMY | + ICE_TX_DESC_CMD_RE; + + tx_buf->tx_flags = ICE_TX_FLAGS_DUMMY_PKT; + tx_buf->raw_buf = raw_packet; + + tx_desc->cmd_type_offset_bsz = + ice_build_ctob(td_cmd, 0, ICE_FDIR_MAX_RAW_PKT_SIZE, 0); + + /* Force memory write to complete before letting h/w know + * there are new descriptors to fetch. + */ + wmb(); + + /* mark the data descriptor to be watched */ + first->next_to_watch = tx_desc; + + writel(tx_ring->next_to_use, tx_ring->tail); + + return 0; +} + /** * ice_unmap_and_free_tx_buf - Release a Tx buffer * @ring: the ring that owns the buffer @@ -24,7 +108,9 @@ static void ice_unmap_and_free_tx_buf(struct ice_ring *ring, struct ice_tx_buf *tx_buf) { if (tx_buf->skb) { - if (ice_ring_is_xdp(ring)) + if (tx_buf->tx_flags & ICE_TX_FLAGS_DUMMY_PKT) + devm_kfree(ring->dev, tx_buf->raw_buf); + else if (ice_ring_is_xdp(ring)) page_frag_free(tx_buf->raw_buf); else dev_kfree_skb_any(tx_buf->skb); @@ -423,6 +509,22 @@ static unsigned int ice_rx_offset(struct ice_ring *rx_ring) return 0; } +static unsigned int ice_rx_frame_truesize(struct ice_ring *rx_ring, + unsigned int size) +{ + unsigned int truesize; + +#if (PAGE_SIZE < 8192) + truesize = ice_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */ +#else + truesize = ice_rx_offset(rx_ring) ? + SKB_DATA_ALIGN(ice_rx_offset(rx_ring) + size) + + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) : + SKB_DATA_ALIGN(size); +#endif + return truesize; +} + /** * ice_run_xdp - Executes an XDP program on initialized xdp_buff * @rx_ring: Rx ring @@ -583,7 +685,8 @@ bool ice_alloc_rx_bufs(struct ice_ring *rx_ring, u16 cleaned_count) struct ice_rx_buf *bi; /* do nothing if no valid netdev defined */ - if (!rx_ring->netdev || !cleaned_count) + if ((!rx_ring->netdev && rx_ring->vsi->type != ICE_VSI_CTRL) || + !cleaned_count) return false; /* get the Rx descriptor and buffer based on next_to_use */ @@ -803,7 +906,7 @@ static struct sk_buff * ice_build_skb(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf, struct xdp_buff *xdp) { - unsigned int metasize = xdp->data - xdp->data_meta; + u8 metasize = xdp->data - xdp->data_meta; #if (PAGE_SIZE < 8192) unsigned int truesize = ice_rx_pg_size(rx_ring) / 2; #else @@ -918,7 +1021,7 @@ ice_construct_skb(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf, */ static void ice_put_rx_buf(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf) { - u32 ntc = rx_ring->next_to_clean + 1; + u16 ntc = rx_ring->next_to_clean + 1; /* fetch, update, and store next to clean */ ntc = (ntc < rx_ring->count) ? ntc : 0; @@ -981,7 +1084,7 @@ ice_is_non_eop(struct ice_ring *rx_ring, union ice_32b_rx_flex_desc *rx_desc, * * Returns amount of work completed */ -static int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget) +int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget) { unsigned int total_rx_bytes = 0, total_rx_pkts = 0; u16 cleaned_count = ICE_DESC_UNUSED(rx_ring); @@ -991,6 +1094,10 @@ static int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget) bool failure; xdp.rxq = &rx_ring->xdp_rxq; + /* Frame size depend on rx_ring setup when PAGE_SIZE=4K */ +#if (PAGE_SIZE < 8192) + xdp.frame_sz = ice_rx_frame_truesize(rx_ring, 0); +#endif /* start the loop to process Rx packets bounded by 'budget' */ while (likely(total_rx_pkts < (unsigned int)budget)) { @@ -1020,6 +1127,12 @@ static int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget) */ dma_rmb(); + if (rx_desc->wb.rxdid == FDIR_DESC_RXDID || !rx_ring->netdev) { + ice_put_rx_buf(rx_ring, NULL); + cleaned_count++; + continue; + } + size = le16_to_cpu(rx_desc->wb.pkt_len) & ICE_RX_FLX_DESC_PKT_LEN_M; @@ -1038,6 +1151,10 @@ static int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget) xdp.data_hard_start = xdp.data - ice_rx_offset(rx_ring); xdp.data_meta = xdp.data; xdp.data_end = xdp.data + size; +#if (PAGE_SIZE > 4096) + /* At larger PAGE_SIZE, frame_sz depend on len size */ + xdp.frame_sz = ice_rx_frame_truesize(rx_ring, size); +#endif rcu_read_lock(); xdp_prog = READ_ONCE(rx_ring->xdp_prog); @@ -1051,16 +1168,8 @@ static int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget) if (!xdp_res) goto construct_skb; if (xdp_res & (ICE_XDP_TX | ICE_XDP_REDIR)) { - unsigned int truesize; - -#if (PAGE_SIZE < 8192) - truesize = ice_rx_pg_size(rx_ring) / 2; -#else - truesize = SKB_DATA_ALIGN(ice_rx_offset(rx_ring) + - size); -#endif xdp_xmit |= xdp_res; - ice_rx_buf_adjust_pg_offset(rx_buf, truesize); + ice_rx_buf_adjust_pg_offset(rx_buf, xdp.frame_sz); } else { rx_buf->pagecnt_bias++; } @@ -1528,7 +1637,7 @@ int ice_napi_poll(struct napi_struct *napi, int budget) * don't allow the budget to go below 1 because that would exit * polling early. */ - budget_per_ring = max(budget / q_vector->num_ring_rx, 1); + budget_per_ring = max_t(int, budget / q_vector->num_ring_rx, 1); else /* Max of 1 Rx ring in this q_vector so give it the budget */ budget_per_ring = budget; @@ -1664,7 +1773,8 @@ ice_tx_map(struct ice_ring *tx_ring, struct ice_tx_buf *first, */ while (unlikely(size > ICE_MAX_DATA_PER_TXD)) { tx_desc->cmd_type_offset_bsz = - build_ctob(td_cmd, td_offset, max_data, td_tag); + ice_build_ctob(td_cmd, td_offset, max_data, + td_tag); tx_desc++; i++; @@ -1684,8 +1794,8 @@ ice_tx_map(struct ice_ring *tx_ring, struct ice_tx_buf *first, if (likely(!data_len)) break; - tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset, - size, td_tag); + tx_desc->cmd_type_offset_bsz = ice_build_ctob(td_cmd, td_offset, + size, td_tag); tx_desc++; i++; @@ -1716,8 +1826,8 @@ ice_tx_map(struct ice_ring *tx_ring, struct ice_tx_buf *first, /* write last descriptor with RS and EOP bits */ td_cmd |= (u64)ICE_TXD_LAST_DESC_CMD; - tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset, size, - td_tag); + tx_desc->cmd_type_offset_bsz = + ice_build_ctob(td_cmd, td_offset, size, td_tag); /* Force memory writes to complete before letting h/w know there * are new descriptors to fetch. @@ -1791,12 +1901,94 @@ int ice_tx_csum(struct ice_tx_buf *first, struct ice_tx_offload_params *off) l2_len = ip.hdr - skb->data; offset = (l2_len / 2) << ICE_TX_DESC_LEN_MACLEN_S; - if (skb->encapsulation) - return -1; + protocol = vlan_get_protocol(skb); + + if (protocol == htons(ETH_P_IP)) + first->tx_flags |= ICE_TX_FLAGS_IPV4; + else if (protocol == htons(ETH_P_IPV6)) + first->tx_flags |= ICE_TX_FLAGS_IPV6; + + if (skb->encapsulation) { + bool gso_ena = false; + u32 tunnel = 0; + + /* define outer network header type */ + if (first->tx_flags & ICE_TX_FLAGS_IPV4) { + tunnel |= (first->tx_flags & ICE_TX_FLAGS_TSO) ? + ICE_TX_CTX_EIPT_IPV4 : + ICE_TX_CTX_EIPT_IPV4_NO_CSUM; + l4_proto = ip.v4->protocol; + } else if (first->tx_flags & ICE_TX_FLAGS_IPV6) { + tunnel |= ICE_TX_CTX_EIPT_IPV6; + exthdr = ip.hdr + sizeof(*ip.v6); + l4_proto = ip.v6->nexthdr; + if (l4.hdr != exthdr) + ipv6_skip_exthdr(skb, exthdr - skb->data, + &l4_proto, &frag_off); + } + + /* define outer transport */ + switch (l4_proto) { + case IPPROTO_UDP: + tunnel |= ICE_TXD_CTX_UDP_TUNNELING; + first->tx_flags |= ICE_TX_FLAGS_TUNNEL; + break; + case IPPROTO_GRE: + tunnel |= ICE_TXD_CTX_GRE_TUNNELING; + first->tx_flags |= ICE_TX_FLAGS_TUNNEL; + break; + case IPPROTO_IPIP: + case IPPROTO_IPV6: + first->tx_flags |= ICE_TX_FLAGS_TUNNEL; + l4.hdr = skb_inner_network_header(skb); + break; + default: + if (first->tx_flags & ICE_TX_FLAGS_TSO) + return -1; + + skb_checksum_help(skb); + return 0; + } + + /* compute outer L3 header size */ + tunnel |= ((l4.hdr - ip.hdr) / 4) << + ICE_TXD_CTX_QW0_EIPLEN_S; + + /* switch IP header pointer from outer to inner header */ + ip.hdr = skb_inner_network_header(skb); + + /* compute tunnel header size */ + tunnel |= ((ip.hdr - l4.hdr) / 2) << + ICE_TXD_CTX_QW0_NATLEN_S; + + gso_ena = skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL; + /* indicate if we need to offload outer UDP header */ + if ((first->tx_flags & ICE_TX_FLAGS_TSO) && !gso_ena && + (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)) + tunnel |= ICE_TXD_CTX_QW0_L4T_CS_M; + + /* record tunnel offload values */ + off->cd_tunnel_params |= tunnel; + + /* set DTYP=1 to indicate that it's an Tx context descriptor + * in IPsec tunnel mode with Tx offloads in Quad word 1 + */ + off->cd_qw1 |= (u64)ICE_TX_DESC_DTYPE_CTX; + + /* switch L4 header pointer from outer to inner */ + l4.hdr = skb_inner_transport_header(skb); + l4_proto = 0; + + /* reset type as we transition from outer to inner headers */ + first->tx_flags &= ~(ICE_TX_FLAGS_IPV4 | ICE_TX_FLAGS_IPV6); + if (ip.v4->version == 4) + first->tx_flags |= ICE_TX_FLAGS_IPV4; + if (ip.v6->version == 6) + first->tx_flags |= ICE_TX_FLAGS_IPV6; + } /* Enable IP checksum offloads */ - protocol = vlan_get_protocol(skb); - if (protocol == htons(ETH_P_IP)) { + if (first->tx_flags & ICE_TX_FLAGS_IPV4) { l4_proto = ip.v4->protocol; /* the stack computes the IP header already, the only time we * need the hardware to recompute it is in the case of TSO. @@ -1806,7 +1998,7 @@ int ice_tx_csum(struct ice_tx_buf *first, struct ice_tx_offload_params *off) else cmd |= ICE_TX_DESC_CMD_IIPT_IPV4; - } else if (protocol == htons(ETH_P_IPV6)) { + } else if (first->tx_flags & ICE_TX_FLAGS_IPV6) { cmd |= ICE_TX_DESC_CMD_IIPT_IPV6; exthdr = ip.hdr + sizeof(*ip.v6); l4_proto = ip.v6->nexthdr; @@ -1861,49 +2053,25 @@ int ice_tx_csum(struct ice_tx_buf *first, struct ice_tx_offload_params *off) * * Checks the skb and set up correspondingly several generic transmit flags * related to VLAN tagging for the HW, such as VLAN, DCB, etc. - * - * Returns error code indicate the frame should be dropped upon error and the - * otherwise returns 0 to indicate the flags has been set properly. */ -static int +static void ice_tx_prepare_vlan_flags(struct ice_ring *tx_ring, struct ice_tx_buf *first) { struct sk_buff *skb = first->skb; - __be16 protocol = skb->protocol; - - if (protocol == htons(ETH_P_8021Q) && - !(tx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) { - /* when HW VLAN acceleration is turned off by the user the - * stack sets the protocol to 8021q so that the driver - * can take any steps required to support the SW only - * VLAN handling. In our case the driver doesn't need - * to take any further steps so just set the protocol - * to the encapsulated ethertype. - */ - skb->protocol = vlan_get_protocol(skb); - return 0; - } - /* if we have a HW VLAN tag being added, default to the HW one */ + /* nothing left to do, software offloaded VLAN */ + if (!skb_vlan_tag_present(skb) && eth_type_vlan(skb->protocol)) + return; + + /* currently, we always assume 802.1Q for VLAN insertion as VLAN + * insertion for 802.1AD is not supported + */ if (skb_vlan_tag_present(skb)) { first->tx_flags |= skb_vlan_tag_get(skb) << ICE_TX_FLAGS_VLAN_S; first->tx_flags |= ICE_TX_FLAGS_HW_VLAN; - } else if (protocol == htons(ETH_P_8021Q)) { - struct vlan_hdr *vhdr, _vhdr; - - /* for SW VLAN, check the next protocol and store the tag */ - vhdr = (struct vlan_hdr *)skb_header_pointer(skb, ETH_HLEN, - sizeof(_vhdr), - &_vhdr); - if (!vhdr) - return -EINVAL; - - first->tx_flags |= ntohs(vhdr->h_vlan_TCI) << - ICE_TX_FLAGS_VLAN_S; - first->tx_flags |= ICE_TX_FLAGS_SW_VLAN; } - return ice_tx_prepare_vlan_flags_dcb(tx_ring, first); + ice_tx_prepare_vlan_flags_dcb(tx_ring, first); } /** @@ -1928,7 +2096,8 @@ int ice_tso(struct ice_tx_buf *first, struct ice_tx_offload_params *off) unsigned char *hdr; } l4; u64 cd_mss, cd_tso_len; - u32 paylen, l4_start; + u32 paylen; + u8 l4_start; int err; if (skb->ip_summed != CHECKSUM_PARTIAL) @@ -1953,8 +2122,42 @@ int ice_tso(struct ice_tx_buf *first, struct ice_tx_offload_params *off) ip.v6->payload_len = 0; } + if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE | + SKB_GSO_GRE_CSUM | + SKB_GSO_IPXIP4 | + SKB_GSO_IPXIP6 | + SKB_GSO_UDP_TUNNEL | + SKB_GSO_UDP_TUNNEL_CSUM)) { + if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) && + (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)) { + l4.udp->len = 0; + + /* determine offset of outer transport header */ + l4_start = (u8)(l4.hdr - skb->data); + + /* remove payload length from outer checksum */ + paylen = skb->len - l4_start; + csum_replace_by_diff(&l4.udp->check, + (__force __wsum)htonl(paylen)); + } + + /* reset pointers to inner headers */ + + /* cppcheck-suppress unreadVariable */ + ip.hdr = skb_inner_network_header(skb); + l4.hdr = skb_inner_transport_header(skb); + + /* initialize inner IP header fields */ + if (ip.v4->version == 4) { + ip.v4->tot_len = 0; + ip.v4->check = 0; + } else { + ip.v6->payload_len = 0; + } + } + /* determine offset of transport header */ - l4_start = l4.hdr - skb->data; + l4_start = (u8)(l4.hdr - skb->data); /* remove payload length from checksum */ paylen = skb->len - l4_start; @@ -1963,12 +2166,12 @@ int ice_tso(struct ice_tx_buf *first, struct ice_tx_offload_params *off) csum_replace_by_diff(&l4.udp->check, (__force __wsum)htonl(paylen)); /* compute length of UDP segmentation header */ - off->header_len = sizeof(l4.udp) + l4_start; + off->header_len = (u8)sizeof(l4.udp) + l4_start; } else { csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen)); /* compute length of TCP segmentation header */ - off->header_len = (l4.tcp->doff * 4) + l4_start; + off->header_len = (u8)((l4.tcp->doff * 4) + l4_start); } /* update gso_segs and bytecount */ @@ -2176,8 +2379,7 @@ ice_xmit_frame_ring(struct sk_buff *skb, struct ice_ring *tx_ring) first->tx_flags = 0; /* prepare the VLAN tagging flags for Tx */ - if (ice_tx_prepare_vlan_flags(tx_ring, first)) - goto out_drop; + ice_tx_prepare_vlan_flags(tx_ring, first); /* set up TSO offload */ tso = ice_tso(first, &offload); @@ -2199,7 +2401,7 @@ ice_xmit_frame_ring(struct sk_buff *skb, struct ice_ring *tx_ring) if (offload.cd_qw1 & ICE_TX_DESC_DTYPE_CTX) { struct ice_tx_ctx_desc *cdesc; - int i = tx_ring->next_to_use; + u16 i = tx_ring->next_to_use; /* grab the next descriptor */ cdesc = ICE_TX_CTX_DESC(tx_ring, i); @@ -2244,3 +2446,86 @@ netdev_tx_t ice_start_xmit(struct sk_buff *skb, struct net_device *netdev) return ice_xmit_frame_ring(skb, tx_ring); } + +/** + * ice_clean_ctrl_tx_irq - interrupt handler for flow director Tx queue + * @tx_ring: tx_ring to clean + */ +void ice_clean_ctrl_tx_irq(struct ice_ring *tx_ring) +{ + struct ice_vsi *vsi = tx_ring->vsi; + s16 i = tx_ring->next_to_clean; + int budget = ICE_DFLT_IRQ_WORK; + struct ice_tx_desc *tx_desc; + struct ice_tx_buf *tx_buf; + + tx_buf = &tx_ring->tx_buf[i]; + tx_desc = ICE_TX_DESC(tx_ring, i); + i -= tx_ring->count; + + do { + struct ice_tx_desc *eop_desc = tx_buf->next_to_watch; + + /* if next_to_watch is not set then there is no pending work */ + if (!eop_desc) + break; + + /* prevent any other reads prior to eop_desc */ + smp_rmb(); + + /* if the descriptor isn't done, no work to do */ + if (!(eop_desc->cmd_type_offset_bsz & + cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE))) + break; + + /* clear next_to_watch to prevent false hangs */ + tx_buf->next_to_watch = NULL; + tx_desc->buf_addr = 0; + tx_desc->cmd_type_offset_bsz = 0; + + /* move past filter desc */ + tx_buf++; + tx_desc++; + i++; + if (unlikely(!i)) { + i -= tx_ring->count; + tx_buf = tx_ring->tx_buf; + tx_desc = ICE_TX_DESC(tx_ring, 0); + } + + /* unmap the data header */ + if (dma_unmap_len(tx_buf, len)) + dma_unmap_single(tx_ring->dev, + dma_unmap_addr(tx_buf, dma), + dma_unmap_len(tx_buf, len), + DMA_TO_DEVICE); + if (tx_buf->tx_flags & ICE_TX_FLAGS_DUMMY_PKT) + devm_kfree(tx_ring->dev, tx_buf->raw_buf); + + /* clear next_to_watch to prevent false hangs */ + tx_buf->raw_buf = NULL; + tx_buf->tx_flags = 0; + tx_buf->next_to_watch = NULL; + dma_unmap_len_set(tx_buf, len, 0); + tx_desc->buf_addr = 0; + tx_desc->cmd_type_offset_bsz = 0; + + /* move past eop_desc for start of next FD desc */ + tx_buf++; + tx_desc++; + i++; + if (unlikely(!i)) { + i -= tx_ring->count; + tx_buf = tx_ring->tx_buf; + tx_desc = ICE_TX_DESC(tx_ring, 0); + } + + budget--; + } while (likely(budget)); + + i += tx_ring->count; + tx_ring->next_to_clean = i; + + /* re-enable interrupt if needed */ + ice_irq_dynamic_ena(&vsi->back->hw, vsi, vsi->q_vectors[0]); +} diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.h b/drivers/net/ethernet/intel/ice/ice_txrx.h index 7ee00a128663..e70c4619edc3 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.h +++ b/drivers/net/ethernet/intel/ice/ice_txrx.h @@ -38,7 +38,8 @@ */ #if (PAGE_SIZE < 8192) #define ICE_2K_TOO_SMALL_WITH_PADDING \ -((NET_SKB_PAD + ICE_RXBUF_1536) > SKB_WITH_OVERHEAD(ICE_RXBUF_2048)) + ((unsigned int)(NET_SKB_PAD + ICE_RXBUF_1536) > \ + SKB_WITH_OVERHEAD(ICE_RXBUF_2048)) /** * ice_compute_pad - compute the padding @@ -107,12 +108,19 @@ static inline int ice_skb_pad(void) #define DESC_NEEDED (MAX_SKB_FRAGS + ICE_DESCS_FOR_CTX_DESC + \ ICE_DESCS_PER_CACHE_LINE + ICE_DESCS_FOR_SKB_DATA_PTR) #define ICE_DESC_UNUSED(R) \ - ((((R)->next_to_clean > (R)->next_to_use) ? 0 : (R)->count) + \ - (R)->next_to_clean - (R)->next_to_use - 1) + (u16)((((R)->next_to_clean > (R)->next_to_use) ? 0 : (R)->count) + \ + (R)->next_to_clean - (R)->next_to_use - 1) #define ICE_TX_FLAGS_TSO BIT(0) #define ICE_TX_FLAGS_HW_VLAN BIT(1) #define ICE_TX_FLAGS_SW_VLAN BIT(2) +/* ICE_TX_FLAGS_DUMMY_PKT is used to mark dummy packets that should be + * freed instead of returned like skb packets. + */ +#define ICE_TX_FLAGS_DUMMY_PKT BIT(3) +#define ICE_TX_FLAGS_IPV4 BIT(5) +#define ICE_TX_FLAGS_IPV6 BIT(6) +#define ICE_TX_FLAGS_TUNNEL BIT(7) #define ICE_TX_FLAGS_VLAN_M 0xffff0000 #define ICE_TX_FLAGS_VLAN_PR_M 0xe0000000 #define ICE_TX_FLAGS_VLAN_PR_S 29 @@ -155,17 +163,16 @@ struct ice_tx_offload_params { }; struct ice_rx_buf { - struct sk_buff *skb; - dma_addr_t dma; union { struct { + struct sk_buff *skb; + dma_addr_t dma; struct page *page; unsigned int page_offset; u16 pagecnt_bias; }; struct { - void *addr; - u64 handle; + struct xdp_buff *xdp; }; }; }; @@ -289,7 +296,6 @@ struct ice_ring { struct rcu_head rcu; /* to avoid race on free */ struct bpf_prog *xdp_prog; struct xdp_umem *xsk_umem; - struct zero_copy_allocator zca; /* CL3 - 3rd cacheline starts here */ struct xdp_rxq_info xdp_rxq; /* CLX - the below items are only accessed infrequently and should be @@ -373,5 +379,9 @@ int ice_setup_rx_ring(struct ice_ring *rx_ring); void ice_free_tx_ring(struct ice_ring *tx_ring); void ice_free_rx_ring(struct ice_ring *rx_ring); int ice_napi_poll(struct napi_struct *napi, int budget); - +int +ice_prgm_fdir_fltr(struct ice_vsi *vsi, struct ice_fltr_desc *fdir_desc, + u8 *raw_packet); +int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget); +void ice_clean_ctrl_tx_irq(struct ice_ring *tx_ring); #endif /* _ICE_TXRX_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_txrx_lib.c b/drivers/net/ethernet/intel/ice/ice_txrx_lib.c index 6da048a6ca7c..02b12736ea80 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx_lib.c @@ -8,7 +8,7 @@ * @rx_ring: ring to bump * @val: new head index */ -void ice_release_rx_desc(struct ice_ring *rx_ring, u32 val) +void ice_release_rx_desc(struct ice_ring *rx_ring, u16 val) { u16 prev_ntu = rx_ring->next_to_use & ~0x7; @@ -84,11 +84,11 @@ ice_rx_csum(struct ice_ring *ring, struct sk_buff *skb, union ice_32b_rx_flex_desc *rx_desc, u8 ptype) { struct ice_rx_ptype_decoded decoded; - u32 rx_error, rx_status; + u16 rx_status0, rx_status1; bool ipv4, ipv6; - rx_status = le16_to_cpu(rx_desc->wb.status_error0); - rx_error = rx_status; + rx_status0 = le16_to_cpu(rx_desc->wb.status_error0); + rx_status1 = le16_to_cpu(rx_desc->wb.status_error1); decoded = ice_decode_rx_desc_ptype(ptype); @@ -101,7 +101,7 @@ ice_rx_csum(struct ice_ring *ring, struct sk_buff *skb, return; /* check if HW has decoded the packet and checksum */ - if (!(rx_status & BIT(ICE_RX_FLEX_DESC_STATUS0_L3L4P_S))) + if (!(rx_status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_L3L4P_S))) return; if (!(decoded.known && decoded.outer_ip)) @@ -112,19 +112,31 @@ ice_rx_csum(struct ice_ring *ring, struct sk_buff *skb, ipv6 = (decoded.outer_ip == ICE_RX_PTYPE_OUTER_IP) && (decoded.outer_ip_ver == ICE_RX_PTYPE_OUTER_IPV6); - if (ipv4 && (rx_error & (BIT(ICE_RX_FLEX_DESC_STATUS0_XSUM_IPE_S) | - BIT(ICE_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S)))) + if (ipv4 && (rx_status0 & (BIT(ICE_RX_FLEX_DESC_STATUS0_XSUM_IPE_S) | + BIT(ICE_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S)))) goto checksum_fail; - else if (ipv6 && (rx_status & - (BIT(ICE_RX_FLEX_DESC_STATUS0_IPV6EXADD_S)))) + + if (ipv6 && (rx_status0 & (BIT(ICE_RX_FLEX_DESC_STATUS0_IPV6EXADD_S)))) goto checksum_fail; /* check for L4 errors and handle packets that were not able to be * checksummed due to arrival speed */ - if (rx_error & BIT(ICE_RX_FLEX_DESC_STATUS0_XSUM_L4E_S)) + if (rx_status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_XSUM_L4E_S)) + goto checksum_fail; + + /* check for outer UDP checksum error in tunneled packets */ + if ((rx_status1 & BIT(ICE_RX_FLEX_DESC_STATUS1_NAT_S)) && + (rx_status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S))) goto checksum_fail; + /* If there is an outer header present that might contain a checksum + * we need to bump the checksum level by 1 to reflect the fact that + * we are indicating we validated the inner checksum. + */ + if (decoded.tunnel_type >= ICE_RX_PTYPE_TUNNEL_IP_GRENAT) + skb->csum_level = 1; + /* Only report checksum unnecessary for TCP, UDP, or SCTP */ switch (decoded.inner_prot) { case ICE_RX_PTYPE_INNER_PROT_TCP: @@ -215,8 +227,8 @@ int ice_xmit_xdp_ring(void *data, u16 size, struct ice_ring *xdp_ring) tx_desc = ICE_TX_DESC(xdp_ring, i); tx_desc->buf_addr = cpu_to_le64(dma); - tx_desc->cmd_type_offset_bsz = build_ctob(ICE_TXD_LAST_DESC_CMD, 0, - size, 0); + tx_desc->cmd_type_offset_bsz = ice_build_ctob(ICE_TXD_LAST_DESC_CMD, 0, + size, 0); /* Make certain all of the status bits have been updated * before next_to_watch is written. @@ -242,7 +254,7 @@ int ice_xmit_xdp_ring(void *data, u16 size, struct ice_ring *xdp_ring) */ int ice_xmit_xdp_buff(struct xdp_buff *xdp, struct ice_ring *xdp_ring) { - struct xdp_frame *xdpf = convert_to_xdp_frame(xdp); + struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); if (unlikely(!xdpf)) return ICE_XDP_CONSUMED; diff --git a/drivers/net/ethernet/intel/ice/ice_txrx_lib.h b/drivers/net/ethernet/intel/ice/ice_txrx_lib.h index ba9164dad9ae..58ff58f0f972 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx_lib.h +++ b/drivers/net/ethernet/intel/ice/ice_txrx_lib.h @@ -22,7 +22,7 @@ ice_test_staterr(union ice_32b_rx_flex_desc *rx_desc, const u16 stat_err_bits) } static inline __le64 -build_ctob(u64 td_cmd, u64 td_offset, unsigned int size, u64 td_tag) +ice_build_ctob(u64 td_cmd, u64 td_offset, unsigned int size, u64 td_tag) { return cpu_to_le64(ICE_TX_DESC_DTYPE_DATA | (td_cmd << ICE_TXD_QW1_CMD_S) | @@ -49,7 +49,7 @@ static inline void ice_xdp_ring_update_tail(struct ice_ring *xdp_ring) void ice_finalize_xdp_rx(struct ice_ring *rx_ring, unsigned int xdp_res); int ice_xmit_xdp_buff(struct xdp_buff *xdp, struct ice_ring *xdp_ring); int ice_xmit_xdp_ring(void *data, u16 size, struct ice_ring *xdp_ring); -void ice_release_rx_desc(struct ice_ring *rx_ring, u32 val); +void ice_release_rx_desc(struct ice_ring *rx_ring, u16 val); void ice_process_skb_fields(struct ice_ring *rx_ring, union ice_32b_rx_flex_desc *rx_desc, diff --git a/drivers/net/ethernet/intel/ice/ice_type.h b/drivers/net/ethernet/intel/ice/ice_type.h index 4ce5f92fca4a..c1ad8622e65c 100644 --- a/drivers/net/ethernet/intel/ice/ice_type.h +++ b/drivers/net/ethernet/intel/ice/ice_type.h @@ -118,7 +118,8 @@ enum ice_media_type { enum ice_vsi_type { ICE_VSI_PF = 0, - ICE_VSI_VF, + ICE_VSI_VF = 1, + ICE_VSI_CTRL = 3, /* equates to ICE_VSI_PF with 1 queue pair */ ICE_VSI_LB = 6, }; @@ -161,6 +162,38 @@ struct ice_phy_info { u8 get_link_info; }; +/* protocol enumeration for filters */ +enum ice_fltr_ptype { + /* NONE - used for undef/error */ + ICE_FLTR_PTYPE_NONF_NONE = 0, + ICE_FLTR_PTYPE_NONF_IPV4_UDP, + ICE_FLTR_PTYPE_NONF_IPV4_TCP, + ICE_FLTR_PTYPE_NONF_IPV4_SCTP, + ICE_FLTR_PTYPE_NONF_IPV4_OTHER, + ICE_FLTR_PTYPE_FRAG_IPV4, + ICE_FLTR_PTYPE_NONF_IPV6_UDP, + ICE_FLTR_PTYPE_NONF_IPV6_TCP, + ICE_FLTR_PTYPE_NONF_IPV6_SCTP, + ICE_FLTR_PTYPE_NONF_IPV6_OTHER, + ICE_FLTR_PTYPE_MAX, +}; + +enum ice_fd_hw_seg { + ICE_FD_HW_SEG_NON_TUN = 0, + ICE_FD_HW_SEG_TUN, + ICE_FD_HW_SEG_MAX, +}; + +/* 2 VSI = 1 ICE_VSI_PF + 1 ICE_VSI_CTRL */ +#define ICE_MAX_FDIR_VSI_PER_FILTER 2 + +struct ice_fd_hw_prof { + struct ice_flow_seg_info *fdir_seg[ICE_FD_HW_SEG_MAX]; + int cnt; + u64 entry_h[ICE_MAX_FDIR_VSI_PER_FILTER][ICE_FD_HW_SEG_MAX]; + u16 vsi_h[ICE_MAX_FDIR_VSI_PER_FILTER]; +}; + /* Common HW capabilities for SW use */ struct ice_hw_common_caps { u32 valid_functions; @@ -197,6 +230,8 @@ struct ice_hw_func_caps { u32 num_allocd_vfs; /* Number of allocated VFs */ u32 vf_base_id; /* Logical ID of the first VF */ u32 guar_num_vsi; + u32 fd_fltr_guar; /* Number of filters guaranteed */ + u32 fd_fltr_best_effort; /* Number of best effort filters */ }; /* Device wide capabilities */ @@ -204,6 +239,7 @@ struct ice_hw_dev_caps { struct ice_hw_common_caps common_cap; u32 num_vfs_exposed; /* Total number of VFs exposed */ u32 num_vsi_allocd_to_host; /* Excluding EMP VSI */ + u32 num_flow_director_fltr; /* Number of FD filters available */ u32 num_funcs; }; @@ -259,6 +295,16 @@ struct ice_nvm_info { #define ICE_NVM_VER_LEN 32 +/* netlist version information */ +struct ice_netlist_ver_info { + u32 major; /* major high/low */ + u32 minor; /* minor high/low */ + u32 type; /* type high/low */ + u32 rev; /* revision high/low */ + u32 hash; /* SHA-1 hash word */ + u16 cust_ver; /* customer version */ +}; + /* Max number of port to queue branches w.r.t topology */ #define ICE_MAX_TRAFFIC_CLASS 8 #define ICE_TXSCHED_MAX_BRANCHES ICE_MAX_TRAFFIC_CLASS @@ -479,6 +525,8 @@ struct ice_hw { u64 debug_mask; /* bitmap for debug mask */ enum ice_mac_type mac_type; + u16 fd_ctr_base; /* FD counter base index */ + /* pci info */ u16 device_id; u16 vendor_id; @@ -491,8 +539,8 @@ struct ice_hw { u16 max_burst_size; /* driver sets this value */ /* Tx Scheduler values */ - u16 num_tx_sched_layers; - u16 num_tx_sched_phys_layers; + u8 num_tx_sched_layers; + u8 num_tx_sched_phys_layers; u8 flattened_layers; u8 max_cgds; u8 sw_entry_point_layer; @@ -506,6 +554,7 @@ struct ice_hw { struct ice_nvm_info nvm; struct ice_hw_dev_caps dev_caps; /* device capabilities */ struct ice_hw_func_caps func_caps; /* function capabilities */ + struct ice_netlist_ver_info netlist_ver; /* netlist version info */ struct ice_switch_info *switch_info; /* switch filter lists */ @@ -548,6 +597,7 @@ struct ice_hw { /* Active package version (currently active) */ struct ice_pkg_ver active_pkg_ver; + u32 active_track_id; u8 active_pkg_name[ICE_PKG_NAME_SIZE]; u8 active_pkg_in_nvm; @@ -568,10 +618,29 @@ struct ice_hw { u8 *pkg_copy; u32 pkg_size; + /* tunneling info */ + struct mutex tnl_lock; + struct ice_tunnel_table tnl; + /* HW block tables */ struct ice_blk_info blk[ICE_BLK_COUNT]; struct mutex fl_profs_locks[ICE_BLK_COUNT]; /* lock fltr profiles */ struct list_head fl_profs[ICE_BLK_COUNT]; + + /* Flow Director filter info */ + int fdir_active_fltr; + + struct mutex fdir_fltr_lock; /* protect Flow Director */ + struct list_head fdir_list_head; + + /* Book-keeping of side-band filter count per flow-type. + * This is used to detect and handle input set changes for + * respective flow-type. + */ + u16 fdir_fltr_cnt[ICE_FLTR_PTYPE_MAX]; + + struct ice_fd_hw_prof **fdir_prof; + DECLARE_BITMAP(fdir_perfect_fltr, ICE_FLTR_PTYPE_MAX); struct mutex rss_locks; /* protect RSS configuration */ struct list_head rss_list_head; }; @@ -592,6 +661,8 @@ struct ice_eth_stats { u64 tx_errors; /* tepc */ }; +#define ICE_MAX_UP 8 + /* Statistics collected by the MAC */ struct ice_hw_port_stats { /* eth stats collected by the port */ @@ -631,6 +702,9 @@ struct ice_hw_port_stats { u64 tx_size_1023; /* ptc1023 */ u64 tx_size_1522; /* ptc1522 */ u64 tx_size_big; /* ptc9522 */ + /* flow director stats */ + u32 fd_sb_status; + u64 fd_sb_match; }; /* Checksum and Shadow RAM pointers */ diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c index 15191a325918..16a2f2526ccc 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c +++ b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c @@ -4,16 +4,18 @@ #include "ice.h" #include "ice_base.h" #include "ice_lib.h" +#include "ice_fltr.h" /** * ice_validate_vf_id - helper to check if VF ID is valid * @pf: pointer to the PF structure * @vf_id: the ID of the VF to check */ -static int ice_validate_vf_id(struct ice_pf *pf, int vf_id) +static int ice_validate_vf_id(struct ice_pf *pf, u16 vf_id) { + /* vf_id range is only valid for 0-255, and should always be unsigned */ if (vf_id >= pf->num_alloc_vfs) { - dev_err(ice_pf_to_dev(pf), "Invalid VF ID: %d\n", vf_id); + dev_err(ice_pf_to_dev(pf), "Invalid VF ID: %u\n", vf_id); return -EINVAL; } return 0; @@ -27,7 +29,7 @@ static int ice_validate_vf_id(struct ice_pf *pf, int vf_id) static int ice_check_vf_init(struct ice_pf *pf, struct ice_vf *vf) { if (!test_bit(ICE_VF_STATE_INIT, vf->vf_states)) { - dev_err(ice_pf_to_dev(pf), "VF ID: %d in reset. Try again.\n", + dev_err(ice_pf_to_dev(pf), "VF ID: %u in reset. Try again.\n", vf->vf_id); return -EBUSY; } @@ -35,6 +37,37 @@ static int ice_check_vf_init(struct ice_pf *pf, struct ice_vf *vf) } /** + * ice_err_to_virt_err - translate errors for VF return code + * @ice_err: error return code + */ +static enum virtchnl_status_code ice_err_to_virt_err(enum ice_status ice_err) +{ + switch (ice_err) { + case ICE_SUCCESS: + return VIRTCHNL_STATUS_SUCCESS; + case ICE_ERR_BAD_PTR: + case ICE_ERR_INVAL_SIZE: + case ICE_ERR_DEVICE_NOT_SUPPORTED: + case ICE_ERR_PARAM: + case ICE_ERR_CFG: + return VIRTCHNL_STATUS_ERR_PARAM; + case ICE_ERR_NO_MEMORY: + return VIRTCHNL_STATUS_ERR_NO_MEMORY; + case ICE_ERR_NOT_READY: + case ICE_ERR_RESET_FAILED: + case ICE_ERR_FW_API_VER: + case ICE_ERR_AQ_ERROR: + case ICE_ERR_AQ_TIMEOUT: + case ICE_ERR_AQ_FULL: + case ICE_ERR_AQ_NO_WORK: + case ICE_ERR_AQ_EMPTY: + return VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR; + default: + return VIRTCHNL_STATUS_ERR_NOT_SUPPORTED; + } +} + +/** * ice_vc_vf_broadcast - Broadcast a message to all VFs on PF * @pf: pointer to the PF structure * @v_opcode: operation code @@ -47,7 +80,7 @@ ice_vc_vf_broadcast(struct ice_pf *pf, enum virtchnl_ops v_opcode, enum virtchnl_status_code v_retval, u8 *msg, u16 msglen) { struct ice_hw *hw = &pf->hw; - int i; + unsigned int i; ice_for_each_vf(pf, i) { struct ice_vf *vf = &pf->vf[i]; @@ -149,6 +182,26 @@ static void ice_vc_notify_vf_link_state(struct ice_vf *vf) } /** + * ice_vf_invalidate_vsi - invalidate vsi_idx/vsi_num to remove VSI access + * @vf: VF to remove access to VSI for + */ +static void ice_vf_invalidate_vsi(struct ice_vf *vf) +{ + vf->lan_vsi_idx = ICE_NO_VSI; + vf->lan_vsi_num = ICE_NO_VSI; +} + +/** + * ice_vf_vsi_release - invalidate the VF's VSI after freeing it + * @vf: invalidate this VF's VSI after freeing it + */ +static void ice_vf_vsi_release(struct ice_vf *vf) +{ + ice_vsi_release(vf->pf->vsi[vf->lan_vsi_idx]); + ice_vf_invalidate_vsi(vf); +} + +/** * ice_free_vf_res - Free a VF's resources * @vf: pointer to the VF info */ @@ -163,10 +216,8 @@ static void ice_free_vf_res(struct ice_vf *vf) clear_bit(ICE_VF_STATE_INIT, vf->vf_states); /* free VSI and disconnect it from the parent uplink */ - if (vf->lan_vsi_idx) { - ice_vsi_release(pf->vsi[vf->lan_vsi_idx]); - vf->lan_vsi_idx = 0; - vf->lan_vsi_num = 0; + if (vf->lan_vsi_idx != ICE_NO_VSI) { + ice_vf_vsi_release(vf); vf->num_mac = 0; } @@ -292,7 +343,7 @@ void ice_free_vfs(struct ice_pf *pf) { struct device *dev = ice_pf_to_dev(pf); struct ice_hw *hw = &pf->hw; - int tmp, i; + unsigned int tmp, i; if (!pf->vf) return; @@ -337,7 +388,7 @@ void ice_free_vfs(struct ice_pf *pf) * before this function ever gets called. */ if (!pci_vfs_assigned(pf->pdev)) { - int vf_id; + unsigned int vf_id; /* Acknowledge VFLR for all VFs. Without this, VFs will fail to * work correctly when SR-IOV gets re-enabled. @@ -368,9 +419,9 @@ static void ice_trigger_vf_reset(struct ice_vf *vf, bool is_vflr, bool is_pfr) { struct ice_pf *pf = vf->pf; u32 reg, reg_idx, bit_idx; + unsigned int vf_abs_id, i; struct device *dev; struct ice_hw *hw; - int vf_abs_id, i; dev = ice_pf_to_dev(pf); hw = &pf->hw; @@ -380,10 +431,7 @@ static void ice_trigger_vf_reset(struct ice_vf *vf, bool is_vflr, bool is_pfr) clear_bit(ICE_VF_STATE_ACTIVE, vf->vf_states); /* Disable VF's configuration API during reset. The flag is re-enabled - * in ice_alloc_vf_res(), when it's safe again to access VF's VSI. - * It's normally disabled in ice_free_vf_res(), but it's safer - * to do it earlier to give some time to finish to any VF config - * functions that may still be running at this point. + * when it's safe again to access VF's VSI. */ clear_bit(ICE_VF_STATE_INIT, vf->vf_states); @@ -418,7 +466,7 @@ static void ice_trigger_vf_reset(struct ice_vf *vf, bool is_vflr, bool is_pfr) if ((reg & VF_TRANS_PENDING_M) == 0) break; - dev_err(dev, "VF %d PCI transactions stuck\n", vf->vf_id); + dev_err(dev, "VF %u PCI transactions stuck\n", vf->vf_id); udelay(ICE_PCI_CIAD_WAIT_DELAY_US); } } @@ -460,8 +508,9 @@ static int ice_vsi_manage_pvid(struct ice_vsi *vsi, u16 pvid_info, bool enable) status = ice_update_vsi(hw, vsi->idx, ctxt, NULL); if (status) { - dev_info(ice_hw_to_dev(hw), "update VSI for port VLAN failed, err %d aq_err %d\n", - status, hw->adminq.sq_last_status); + dev_info(ice_hw_to_dev(hw), "update VSI for port VLAN failed, err %s aq_err %s\n", + ice_stat_str(status), + ice_aq_str(hw->adminq.sq_last_status)); ret = -EIO; goto out; } @@ -475,18 +524,39 @@ out: } /** + * ice_vf_get_port_info - Get the VF's port info structure + * @vf: VF used to get the port info structure for + */ +static struct ice_port_info *ice_vf_get_port_info(struct ice_vf *vf) +{ + return vf->pf->hw.port_info; +} + +/** * ice_vf_vsi_setup - Set up a VF VSI - * @pf: board private structure - * @pi: pointer to the port_info instance - * @vf_id: defines VF ID to which this VSI connects. + * @vf: VF to setup VSI for * * Returns pointer to the successfully allocated VSI struct on success, * otherwise returns NULL on failure. */ -static struct ice_vsi * -ice_vf_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi, u16 vf_id) +static struct ice_vsi *ice_vf_vsi_setup(struct ice_vf *vf) { - return ice_vsi_setup(pf, pi, ICE_VSI_VF, vf_id); + struct ice_port_info *pi = ice_vf_get_port_info(vf); + struct ice_pf *pf = vf->pf; + struct ice_vsi *vsi; + + vsi = ice_vsi_setup(pf, pi, ICE_VSI_VF, vf->vf_id); + + if (!vsi) { + dev_err(ice_pf_to_dev(pf), "Failed to create VF VSI\n"); + ice_vf_invalidate_vsi(vf); + return NULL; + } + + vf->lan_vsi_idx = vsi->idx; + vf->lan_vsi_num = vsi->vsi_num; + + return vsi; } /** @@ -507,170 +577,158 @@ static int ice_calc_vf_first_vector_idx(struct ice_pf *pf, struct ice_vf *vf) } /** - * ice_alloc_vsi_res - Setup VF VSI and its resources - * @vf: pointer to the VF structure + * ice_vf_rebuild_host_vlan_cfg - add VLAN 0 filter or rebuild the Port VLAN + * @vf: VF to add MAC filters for * - * Returns 0 on success, negative value on failure + * Called after a VF VSI has been re-added/rebuilt during reset. The PF driver + * always re-adds either a VLAN 0 or port VLAN based filter after reset. */ -static int ice_alloc_vsi_res(struct ice_vf *vf) +static int ice_vf_rebuild_host_vlan_cfg(struct ice_vf *vf) { - struct ice_pf *pf = vf->pf; - LIST_HEAD(tmp_add_list); - u8 broadcast[ETH_ALEN]; - struct ice_vsi *vsi; - struct device *dev; - int status = 0; + struct ice_vsi *vsi = vf->pf->vsi[vf->lan_vsi_idx]; + struct device *dev = ice_pf_to_dev(vf->pf); + u16 vlan_id = 0; + int err; - dev = ice_pf_to_dev(pf); - /* first vector index is the VFs OICR index */ - vf->first_vector_idx = ice_calc_vf_first_vector_idx(pf, vf); + if (vf->port_vlan_info) { + err = ice_vsi_manage_pvid(vsi, vf->port_vlan_info, true); + if (err) { + dev_err(dev, "failed to configure port VLAN via VSI parameters for VF %u, error %d\n", + vf->vf_id, err); + return err; + } - vsi = ice_vf_vsi_setup(pf, pf->hw.port_info, vf->vf_id); - if (!vsi) { - dev_err(dev, "Failed to create VF VSI\n"); - return -ENOMEM; + vlan_id = vf->port_vlan_info & VLAN_VID_MASK; } - vf->lan_vsi_idx = vsi->idx; - vf->lan_vsi_num = vsi->vsi_num; - - /* Check if port VLAN exist before, and restore it accordingly */ - if (vf->port_vlan_info) { - ice_vsi_manage_pvid(vsi, vf->port_vlan_info, true); - if (ice_vsi_add_vlan(vsi, vf->port_vlan_info & VLAN_VID_MASK)) - dev_warn(ice_pf_to_dev(pf), "Failed to add Port VLAN %d filter for VF %d\n", - vf->port_vlan_info & VLAN_VID_MASK, vf->vf_id); - } else { - /* set VLAN 0 filter by default when no port VLAN is - * enabled. If a port VLAN is enabled we don't want - * untagged broadcast/multicast traffic seen on the VF - * interface. - */ - if (ice_vsi_add_vlan(vsi, 0)) - dev_warn(ice_pf_to_dev(pf), "Failed to add VLAN 0 filter for VF %d, MDD events will trigger. Reset the VF, disable spoofchk, or enable 8021q module on the guest\n", - vf->vf_id); + /* vlan_id will either be 0 or the port VLAN number */ + err = ice_vsi_add_vlan(vsi, vlan_id, ICE_FWD_TO_VSI); + if (err) { + dev_err(dev, "failed to add %s VLAN %u filter for VF %u, error %d\n", + vf->port_vlan_info ? "port" : "", vlan_id, vf->vf_id, + err); + return err; } + return 0; +} + +/** + * ice_vf_rebuild_host_mac_cfg - add broadcast and the VF's perm_addr/LAA + * @vf: VF to add MAC filters for + * + * Called after a VF VSI has been re-added/rebuilt during reset. The PF driver + * always re-adds a broadcast filter and the VF's perm_addr/LAA after reset. + */ +static int ice_vf_rebuild_host_mac_cfg(struct ice_vf *vf) +{ + struct ice_vsi *vsi = vf->pf->vsi[vf->lan_vsi_idx]; + struct device *dev = ice_pf_to_dev(vf->pf); + enum ice_status status; + u8 broadcast[ETH_ALEN]; + eth_broadcast_addr(broadcast); + status = ice_fltr_add_mac(vsi, broadcast, ICE_FWD_TO_VSI); + if (status) { + dev_err(dev, "failed to add broadcast MAC filter for VF %u, error %s\n", + vf->vf_id, ice_stat_str(status)); + return ice_status_to_errno(status); + } - status = ice_add_mac_to_list(vsi, &tmp_add_list, broadcast); - if (status) - goto ice_alloc_vsi_res_exit; + vf->num_mac++; if (is_valid_ether_addr(vf->dflt_lan_addr.addr)) { - status = ice_add_mac_to_list(vsi, &tmp_add_list, - vf->dflt_lan_addr.addr); - if (status) - goto ice_alloc_vsi_res_exit; + status = ice_fltr_add_mac(vsi, vf->dflt_lan_addr.addr, + ICE_FWD_TO_VSI); + if (status) { + dev_err(dev, "failed to add default unicast MAC filter %pM for VF %u, error %s\n", + &vf->dflt_lan_addr.addr[0], vf->vf_id, + ice_stat_str(status)); + return ice_status_to_errno(status); + } + vf->num_mac++; } - status = ice_add_mac(&pf->hw, &tmp_add_list); - if (status) - dev_err(dev, "could not add mac filters error %d\n", status); - else - vf->num_mac = 1; - - /* Clear this bit after VF initialization since we shouldn't reclaim - * and reassign interrupts for synchronous or asynchronous VFR events. - * We don't want to reconfigure interrupts since AVF driver doesn't - * expect vector assignment to be changed unless there is a request for - * more vectors. - */ -ice_alloc_vsi_res_exit: - ice_free_fltr_list(dev, &tmp_add_list); - return status; + return 0; } /** - * ice_alloc_vf_res - Allocate VF resources - * @vf: pointer to the VF structure + * ice_vf_set_host_trust_cfg - set trust setting based on pre-reset value + * @vf: VF to configure trust setting for */ -static int ice_alloc_vf_res(struct ice_vf *vf) +static void ice_vf_set_host_trust_cfg(struct ice_vf *vf) { - struct ice_pf *pf = vf->pf; - int tx_rx_queue_left; - int status; - - /* Update number of VF queues, in case VF had requested for queue - * changes - */ - tx_rx_queue_left = min_t(int, ice_get_avail_txq_count(pf), - ice_get_avail_rxq_count(pf)); - tx_rx_queue_left += pf->num_qps_per_vf; - if (vf->num_req_qs && vf->num_req_qs <= tx_rx_queue_left && - vf->num_req_qs != vf->num_vf_qs) - vf->num_vf_qs = vf->num_req_qs; - - /* setup VF VSI and necessary resources */ - status = ice_alloc_vsi_res(vf); - if (status) - goto ice_alloc_vf_res_exit; - if (vf->trusted) set_bit(ICE_VIRTCHNL_VF_CAP_PRIVILEGE, &vf->vf_caps); else clear_bit(ICE_VIRTCHNL_VF_CAP_PRIVILEGE, &vf->vf_caps); - - /* VF is now completely initialized */ - set_bit(ICE_VF_STATE_INIT, vf->vf_states); - - return status; - -ice_alloc_vf_res_exit: - ice_free_vf_res(vf); - return status; } /** - * ice_ena_vf_mappings - * @vf: pointer to the VF structure + * ice_ena_vf_msix_mappings - enable VF MSIX mappings in hardware + * @vf: VF to enable MSIX mappings for * - * Enable VF vectors and queues allocation by writing the details into - * respective registers. + * Some of the registers need to be indexed/configured using hardware global + * device values and other registers need 0-based values, which represent PF + * based values. */ -static void ice_ena_vf_mappings(struct ice_vf *vf) +static void ice_ena_vf_msix_mappings(struct ice_vf *vf) { - int abs_vf_id, abs_first, abs_last; + int device_based_first_msix, device_based_last_msix; + int pf_based_first_msix, pf_based_last_msix, v; struct ice_pf *pf = vf->pf; - struct ice_vsi *vsi; - struct device *dev; - int first, last, v; + int device_based_vf_id; struct ice_hw *hw; u32 reg; - dev = ice_pf_to_dev(pf); hw = &pf->hw; - vsi = pf->vsi[vf->lan_vsi_idx]; - first = vf->first_vector_idx; - last = (first + pf->num_msix_per_vf) - 1; - abs_first = first + pf->hw.func_caps.common_cap.msix_vector_first_id; - abs_last = (abs_first + pf->num_msix_per_vf) - 1; - abs_vf_id = vf->vf_id + hw->func_caps.vf_base_id; - - /* VF Vector allocation */ - reg = (((abs_first << VPINT_ALLOC_FIRST_S) & VPINT_ALLOC_FIRST_M) | - ((abs_last << VPINT_ALLOC_LAST_S) & VPINT_ALLOC_LAST_M) | - VPINT_ALLOC_VALID_M); + pf_based_first_msix = vf->first_vector_idx; + pf_based_last_msix = (pf_based_first_msix + pf->num_msix_per_vf) - 1; + + device_based_first_msix = pf_based_first_msix + + pf->hw.func_caps.common_cap.msix_vector_first_id; + device_based_last_msix = + (device_based_first_msix + pf->num_msix_per_vf) - 1; + device_based_vf_id = vf->vf_id + hw->func_caps.vf_base_id; + + reg = (((device_based_first_msix << VPINT_ALLOC_FIRST_S) & + VPINT_ALLOC_FIRST_M) | + ((device_based_last_msix << VPINT_ALLOC_LAST_S) & + VPINT_ALLOC_LAST_M) | VPINT_ALLOC_VALID_M); wr32(hw, VPINT_ALLOC(vf->vf_id), reg); - reg = (((abs_first << VPINT_ALLOC_PCI_FIRST_S) + reg = (((device_based_first_msix << VPINT_ALLOC_PCI_FIRST_S) & VPINT_ALLOC_PCI_FIRST_M) | - ((abs_last << VPINT_ALLOC_PCI_LAST_S) & VPINT_ALLOC_PCI_LAST_M) | - VPINT_ALLOC_PCI_VALID_M); + ((device_based_last_msix << VPINT_ALLOC_PCI_LAST_S) & + VPINT_ALLOC_PCI_LAST_M) | VPINT_ALLOC_PCI_VALID_M); wr32(hw, VPINT_ALLOC_PCI(vf->vf_id), reg); + /* map the interrupts to its functions */ - for (v = first; v <= last; v++) { - reg = (((abs_vf_id << GLINT_VECT2FUNC_VF_NUM_S) & + for (v = pf_based_first_msix; v <= pf_based_last_msix; v++) { + reg = (((device_based_vf_id << GLINT_VECT2FUNC_VF_NUM_S) & GLINT_VECT2FUNC_VF_NUM_M) | ((hw->pf_id << GLINT_VECT2FUNC_PF_NUM_S) & GLINT_VECT2FUNC_PF_NUM_M)); wr32(hw, GLINT_VECT2FUNC(v), reg); } - /* Map mailbox interrupt. We put an explicit 0 here to remind us that - * VF admin queue interrupts will go to VF MSI-X vector 0. - */ - wr32(hw, VPINT_MBX_CTL(abs_vf_id), VPINT_MBX_CTL_CAUSE_ENA_M | 0); + /* Map mailbox interrupt to VF MSI-X vector 0 */ + wr32(hw, VPINT_MBX_CTL(device_based_vf_id), VPINT_MBX_CTL_CAUSE_ENA_M); +} + +/** + * ice_ena_vf_q_mappings - enable Rx/Tx queue mappings for a VF + * @vf: VF to enable the mappings for + * @max_txq: max Tx queues allowed on the VF's VSI + * @max_rxq: max Rx queues allowed on the VF's VSI + */ +static void ice_ena_vf_q_mappings(struct ice_vf *vf, u16 max_txq, u16 max_rxq) +{ + struct ice_vsi *vsi = vf->pf->vsi[vf->lan_vsi_idx]; + struct device *dev = ice_pf_to_dev(vf->pf); + struct ice_hw *hw = &vf->pf->hw; + u32 reg; + /* set regardless of mapping mode */ wr32(hw, VPLAN_TXQ_MAPENA(vf->vf_id), VPLAN_TXQ_MAPENA_TX_ENA_M); @@ -682,7 +740,7 @@ static void ice_ena_vf_mappings(struct ice_vf *vf) */ reg = (((vsi->txq_map[0] << VPLAN_TX_QBASE_VFFIRSTQ_S) & VPLAN_TX_QBASE_VFFIRSTQ_M) | - (((vsi->alloc_txq - 1) << VPLAN_TX_QBASE_VFNUMQ_S) & + (((max_txq - 1) << VPLAN_TX_QBASE_VFNUMQ_S) & VPLAN_TX_QBASE_VFNUMQ_M)); wr32(hw, VPLAN_TX_QBASE(vf->vf_id), reg); } else { @@ -700,7 +758,7 @@ static void ice_ena_vf_mappings(struct ice_vf *vf) */ reg = (((vsi->rxq_map[0] << VPLAN_RX_QBASE_VFFIRSTQ_S) & VPLAN_RX_QBASE_VFFIRSTQ_M) | - (((vsi->alloc_txq - 1) << VPLAN_RX_QBASE_VFNUMQ_S) & + (((max_rxq - 1) << VPLAN_RX_QBASE_VFNUMQ_S) & VPLAN_RX_QBASE_VFNUMQ_M)); wr32(hw, VPLAN_RX_QBASE(vf->vf_id), reg); } else { @@ -709,6 +767,18 @@ static void ice_ena_vf_mappings(struct ice_vf *vf) } /** + * ice_ena_vf_mappings - enable VF MSIX and queue mapping + * @vf: pointer to the VF structure + */ +static void ice_ena_vf_mappings(struct ice_vf *vf) +{ + struct ice_vsi *vsi = vf->pf->vsi[vf->lan_vsi_idx]; + + ice_ena_vf_msix_mappings(vf); + ice_ena_vf_q_mappings(vf, vsi->alloc_txq, vsi->alloc_rxq); +} + +/** * ice_determine_res * @pf: pointer to the PF structure * @avail_res: available resources in the PF structure @@ -906,51 +976,18 @@ static int ice_set_per_vf_res(struct ice_pf *pf) } /** - * ice_cleanup_and_realloc_vf - Clean up VF and reallocate resources after reset - * @vf: pointer to the VF structure - * - * Cleanup a VF after the hardware reset is finished. Expects the caller to - * have verified whether the reset is finished properly, and ensure the - * minimum amount of wait time has passed. Reallocate VF resources back to make - * VF state active + * ice_clear_vf_reset_trigger - enable VF to access hardware + * @vf: VF to enabled hardware access for */ -static void ice_cleanup_and_realloc_vf(struct ice_vf *vf) +static void ice_clear_vf_reset_trigger(struct ice_vf *vf) { - struct ice_pf *pf = vf->pf; - struct ice_hw *hw; + struct ice_hw *hw = &vf->pf->hw; u32 reg; - hw = &pf->hw; - - /* PF software completes the flow by notifying VF that reset flow is - * completed. This is done by enabling hardware by clearing the reset - * bit in the VPGEN_VFRTRIG reg and setting VFR_STATE in the VFGEN_RSTAT - * register to VFR completed (done at the end of this function) - * By doing this we allow HW to access VF memory at any point. If we - * did it any sooner, HW could access memory while it was being freed - * in ice_free_vf_res(), causing an IOMMU fault. - * - * On the other hand, this needs to be done ASAP, because the VF driver - * is waiting for this to happen and may report a timeout. It's - * harmless, but it gets logged into Guest OS kernel log, so best avoid - * it. - */ reg = rd32(hw, VPGEN_VFRTRIG(vf->vf_id)); reg &= ~VPGEN_VFRTRIG_VFSWR_M; wr32(hw, VPGEN_VFRTRIG(vf->vf_id), reg); - - /* reallocate VF resources to finish resetting the VSI state */ - if (!ice_alloc_vf_res(vf)) { - ice_ena_vf_mappings(vf); - set_bit(ICE_VF_STATE_ACTIVE, vf->vf_states); - clear_bit(ICE_VF_STATE_DIS, vf->vf_states); - } - - /* Tell the VF driver the reset is done. This needs to be done only - * after VF has been fully initialized, because the VF driver may - * request resources immediately after setting this flag. - */ - wr32(hw, VFGEN_RSTAT(vf->vf_id), VIRTCHNL_VFR_VFACTIVE); + ice_flush(hw); } /** @@ -994,44 +1031,124 @@ ice_vf_set_vsi_promisc(struct ice_vf *vf, struct ice_vsi *vsi, u8 promisc_m, return status; } +static void ice_vf_clear_counters(struct ice_vf *vf) +{ + struct ice_vsi *vsi = vf->pf->vsi[vf->lan_vsi_idx]; + + vf->num_mac = 0; + vsi->num_vlan = 0; + memset(&vf->mdd_tx_events, 0, sizeof(vf->mdd_tx_events)); + memset(&vf->mdd_rx_events, 0, sizeof(vf->mdd_rx_events)); +} + /** - * ice_config_res_vfs - Finalize allocation of VFs resources in one go - * @pf: pointer to the PF structure + * ice_vf_pre_vsi_rebuild - tasks to be done prior to VSI rebuild + * @vf: VF to perform pre VSI rebuild tasks * - * This function is being called as last part of resetting all VFs, or when - * configuring VFs for the first time, where there is no resource to be freed - * Returns true if resources were properly allocated for all VFs, and false - * otherwise. + * These tasks are items that don't need to be amortized since they are most + * likely called in a for loop with all VF(s) in the reset_all_vfs() case. */ -static bool ice_config_res_vfs(struct ice_pf *pf) +static void ice_vf_pre_vsi_rebuild(struct ice_vf *vf) { - struct device *dev = ice_pf_to_dev(pf); - struct ice_hw *hw = &pf->hw; - int v; + ice_vf_clear_counters(vf); + ice_clear_vf_reset_trigger(vf); +} - if (ice_set_per_vf_res(pf)) { - dev_err(dev, "Cannot allocate VF resources, try with fewer number of VFs\n"); - return false; - } +/** + * ice_vf_rebuild_host_cfg - host admin configuration is persistent across reset + * @vf: VF to rebuild host configuration on + */ +static void ice_vf_rebuild_host_cfg(struct ice_vf *vf) +{ + struct device *dev = ice_pf_to_dev(vf->pf); - /* rearm global interrupts */ - if (test_and_clear_bit(__ICE_OICR_INTR_DIS, pf->state)) - ice_irq_dynamic_ena(hw, NULL, NULL); + ice_vf_set_host_trust_cfg(vf); - /* Finish resetting each VF and allocate resources */ - ice_for_each_vf(pf, v) { - struct ice_vf *vf = &pf->vf[v]; + if (ice_vf_rebuild_host_mac_cfg(vf)) + dev_err(dev, "failed to rebuild default MAC configuration for VF %d\n", + vf->vf_id); - vf->num_vf_qs = pf->num_qps_per_vf; - dev_dbg(dev, "VF-id %d has %d queues configured\n", vf->vf_id, - vf->num_vf_qs); - ice_cleanup_and_realloc_vf(vf); + if (ice_vf_rebuild_host_vlan_cfg(vf)) + dev_err(dev, "failed to rebuild VLAN configuration for VF %u\n", + vf->vf_id); +} + +/** + * ice_vf_rebuild_vsi_with_release - release and setup the VF's VSI + * @vf: VF to release and setup the VSI for + * + * This is only called when a single VF is being reset (i.e. VFR, VFLR, host VF + * configuration change, etc.). + */ +static int ice_vf_rebuild_vsi_with_release(struct ice_vf *vf) +{ + ice_vf_vsi_release(vf); + if (!ice_vf_vsi_setup(vf)) + return -ENOMEM; + + return 0; +} + +/** + * ice_vf_rebuild_vsi - rebuild the VF's VSI + * @vf: VF to rebuild the VSI for + * + * This is only called when all VF(s) are being reset (i.e. PCIe Reset on the + * host, PFR, CORER, etc.). + */ +static int ice_vf_rebuild_vsi(struct ice_vf *vf) +{ + struct ice_pf *pf = vf->pf; + struct ice_vsi *vsi; + + vsi = pf->vsi[vf->lan_vsi_idx]; + + if (ice_vsi_rebuild(vsi, true)) { + dev_err(ice_pf_to_dev(pf), "failed to rebuild VF %d VSI\n", + vf->vf_id); + return -EIO; } + /* vsi->idx will remain the same in this case so don't update + * vf->lan_vsi_idx + */ + vsi->vsi_num = ice_get_hw_vsi_num(&pf->hw, vsi->idx); + vf->lan_vsi_num = vsi->vsi_num; - ice_flush(hw); - clear_bit(__ICE_VF_DIS, pf->state); + return 0; +} - return true; +/** + * ice_vf_set_initialized - VF is ready for VIRTCHNL communication + * @vf: VF to set in initialized state + * + * After this function the VF will be ready to receive/handle the + * VIRTCHNL_OP_GET_VF_RESOURCES message + */ +static void ice_vf_set_initialized(struct ice_vf *vf) +{ + ice_set_vf_state_qs_dis(vf); + clear_bit(ICE_VF_STATE_MC_PROMISC, vf->vf_states); + clear_bit(ICE_VF_STATE_UC_PROMISC, vf->vf_states); + clear_bit(ICE_VF_STATE_DIS, vf->vf_states); + set_bit(ICE_VF_STATE_INIT, vf->vf_states); +} + +/** + * ice_vf_post_vsi_rebuild - tasks to do after the VF's VSI have been rebuilt + * @vf: VF to perform tasks on + */ +static void ice_vf_post_vsi_rebuild(struct ice_vf *vf) +{ + struct ice_pf *pf = vf->pf; + struct ice_hw *hw; + + hw = &pf->hw; + + ice_vf_rebuild_host_cfg(vf); + + ice_vf_set_initialized(vf); + ice_ena_vf_mappings(vf); + wr32(hw, VFGEN_RSTAT(vf->vf_id), VIRTCHNL_VFR_VFACTIVE); } /** @@ -1065,17 +1182,6 @@ bool ice_reset_all_vfs(struct ice_pf *pf, bool is_vflr) ice_for_each_vf(pf, v) ice_trigger_vf_reset(&pf->vf[v], is_vflr, true); - ice_for_each_vf(pf, v) { - struct ice_vsi *vsi; - - vf = &pf->vf[v]; - vsi = pf->vsi[vf->lan_vsi_idx]; - if (test_bit(ICE_VF_STATE_QS_ENA, vf->vf_states)) - ice_dis_vf_qs(vf); - ice_dis_vsi_txq(vsi->port_info, vsi->idx, 0, 0, NULL, NULL, - NULL, ICE_VF_RESET, vf->vf_id, NULL); - } - /* HW requires some time to make sure it can flush the FIFO for a VF * when it resets it. Poll the VPGEN_VFRSTAT register for each VF in * sequence to make sure that it has completed. We'll keep track of @@ -1112,21 +1218,13 @@ bool ice_reset_all_vfs(struct ice_pf *pf, bool is_vflr) ice_for_each_vf(pf, v) { vf = &pf->vf[v]; - ice_free_vf_res(vf); - - /* Free VF queues as well, and reallocate later. - * If a given VF has different number of queues - * configured, the request for update will come - * via mailbox communication. - */ - vf->num_vf_qs = 0; + ice_vf_pre_vsi_rebuild(vf); + ice_vf_rebuild_vsi(vf); + ice_vf_post_vsi_rebuild(vf); } - if (ice_sriov_free_msix_res(pf)) - dev_err(dev, "Failed to free MSIX resources used by SR-IOV\n"); - - if (!ice_config_res_vfs(pf)) - return false; + ice_flush(hw); + clear_bit(__ICE_VF_DIS, pf->state); return true; } @@ -1238,12 +1336,9 @@ bool ice_reset_vf(struct ice_vf *vf, bool is_vflr) dev_err(dev, "disabling promiscuous mode failed\n"); } - /* free VF resources to begin resetting the VSI state */ - ice_free_vf_res(vf); - - ice_cleanup_and_realloc_vf(vf); - - ice_flush(hw); + ice_vf_pre_vsi_rebuild(vf); + ice_vf_rebuild_vsi_with_release(vf); + ice_vf_post_vsi_rebuild(vf); return true; } @@ -1311,16 +1406,144 @@ static void ice_vc_notify_vf_reset(struct ice_vf *vf) } /** - * ice_alloc_vfs - Allocate and set up VFs resources + * ice_init_vf_vsi_res - initialize/setup VF VSI resources + * @vf: VF to initialize/setup the VSI for + * + * This function creates a VSI for the VF, adds a VLAN 0 filter, and sets up the + * VF VSI's broadcast filter and is only used during initial VF creation. + */ +static int ice_init_vf_vsi_res(struct ice_vf *vf) +{ + struct ice_pf *pf = vf->pf; + u8 broadcast[ETH_ALEN]; + enum ice_status status; + struct ice_vsi *vsi; + struct device *dev; + int err; + + vf->first_vector_idx = ice_calc_vf_first_vector_idx(pf, vf); + + dev = ice_pf_to_dev(pf); + vsi = ice_vf_vsi_setup(vf); + if (!vsi) + return -ENOMEM; + + err = ice_vsi_add_vlan(vsi, 0, ICE_FWD_TO_VSI); + if (err) { + dev_warn(dev, "Failed to add VLAN 0 filter for VF %d\n", + vf->vf_id); + goto release_vsi; + } + + eth_broadcast_addr(broadcast); + status = ice_fltr_add_mac(vsi, broadcast, ICE_FWD_TO_VSI); + if (status) { + dev_err(dev, "Failed to add broadcast MAC filter for VF %d, status %s\n", + vf->vf_id, ice_stat_str(status)); + err = ice_status_to_errno(status); + goto release_vsi; + } + + vf->num_mac = 1; + + return 0; + +release_vsi: + ice_vf_vsi_release(vf); + return err; +} + +/** + * ice_start_vfs - start VFs so they are ready to be used by SR-IOV + * @pf: PF the VFs are associated with + */ +static int ice_start_vfs(struct ice_pf *pf) +{ + struct ice_hw *hw = &pf->hw; + int retval, i; + + ice_for_each_vf(pf, i) { + struct ice_vf *vf = &pf->vf[i]; + + ice_clear_vf_reset_trigger(vf); + + retval = ice_init_vf_vsi_res(vf); + if (retval) { + dev_err(ice_pf_to_dev(pf), "Failed to initialize VSI resources for VF %d, error %d\n", + vf->vf_id, retval); + goto teardown; + } + + set_bit(ICE_VF_STATE_INIT, vf->vf_states); + ice_ena_vf_mappings(vf); + wr32(hw, VFGEN_RSTAT(vf->vf_id), VIRTCHNL_VFR_VFACTIVE); + } + + ice_flush(hw); + return 0; + +teardown: + for (i = i - 1; i >= 0; i--) { + struct ice_vf *vf = &pf->vf[i]; + + ice_dis_vf_mappings(vf); + ice_vf_vsi_release(vf); + } + + return retval; +} + +/** + * ice_set_dflt_settings - set VF defaults during initialization/creation + * @pf: PF holding reference to all VFs for default configuration + */ +static void ice_set_dflt_settings_vfs(struct ice_pf *pf) +{ + int i; + + ice_for_each_vf(pf, i) { + struct ice_vf *vf = &pf->vf[i]; + + vf->pf = pf; + vf->vf_id = i; + vf->vf_sw_id = pf->first_sw; + /* assign default capabilities */ + set_bit(ICE_VIRTCHNL_VF_CAP_L2, &vf->vf_caps); + vf->spoofchk = true; + vf->num_vf_qs = pf->num_qps_per_vf; + } +} + +/** + * ice_alloc_vfs - allocate num_vfs in the PF structure + * @pf: PF to store the allocated VFs in + * @num_vfs: number of VFs to allocate + */ +static int ice_alloc_vfs(struct ice_pf *pf, int num_vfs) +{ + struct ice_vf *vfs; + + vfs = devm_kcalloc(ice_pf_to_dev(pf), num_vfs, sizeof(*vfs), + GFP_KERNEL); + if (!vfs) + return -ENOMEM; + + pf->vf = vfs; + pf->num_alloc_vfs = num_vfs; + + return 0; +} + +/** + * ice_ena_vfs - enable VFs so they are ready to be used * @pf: pointer to the PF structure - * @num_alloc_vfs: number of VFs to allocate + * @num_vfs: number of VFs to enable */ -static int ice_alloc_vfs(struct ice_pf *pf, u16 num_alloc_vfs) +static int ice_ena_vfs(struct ice_pf *pf, u16 num_vfs) { struct device *dev = ice_pf_to_dev(pf); struct ice_hw *hw = &pf->hw; - struct ice_vf *vfs; - int i, ret; + int ret; /* Disable global interrupt 0 so we don't try to handle the VFLR. */ wr32(hw, GLINT_DYN_CTL(pf->oicr_idx), @@ -1328,43 +1551,37 @@ static int ice_alloc_vfs(struct ice_pf *pf, u16 num_alloc_vfs) set_bit(__ICE_OICR_INTR_DIS, pf->state); ice_flush(hw); - ret = pci_enable_sriov(pf->pdev, num_alloc_vfs); + ret = pci_enable_sriov(pf->pdev, num_vfs); if (ret) { pf->num_alloc_vfs = 0; goto err_unroll_intr; } - /* allocate memory */ - vfs = devm_kcalloc(dev, num_alloc_vfs, sizeof(*vfs), GFP_KERNEL); - if (!vfs) { - ret = -ENOMEM; - goto err_pci_disable_sriov; - } - pf->vf = vfs; - pf->num_alloc_vfs = num_alloc_vfs; - /* apply default profile */ - ice_for_each_vf(pf, i) { - vfs[i].pf = pf; - vfs[i].vf_sw_id = pf->first_sw; - vfs[i].vf_id = i; + ret = ice_alloc_vfs(pf, num_vfs); + if (ret) + goto err_pci_disable_sriov; - /* assign default capabilities */ - set_bit(ICE_VIRTCHNL_VF_CAP_L2, &vfs[i].vf_caps); - vfs[i].spoofchk = true; + if (ice_set_per_vf_res(pf)) { + dev_err(dev, "Not enough resources for %d VFs, try with fewer number of VFs\n", + num_vfs); + ret = -ENOSPC; + goto err_unroll_sriov; } - /* VF resources get allocated with initialization */ - if (!ice_config_res_vfs(pf)) { - ret = -EIO; + ice_set_dflt_settings_vfs(pf); + + if (ice_start_vfs(pf)) { + dev_err(dev, "Failed to start VF(s)\n"); + ret = -EAGAIN; goto err_unroll_sriov; } - return ret; + clear_bit(__ICE_VF_DIS, pf->state); + return 0; err_unroll_sriov: + devm_kfree(dev, pf->vf); pf->vf = NULL; - devm_kfree(dev, vfs); - vfs = NULL; pf->num_alloc_vfs = 0; err_pci_disable_sriov: pci_disable_sriov(pf->pdev); @@ -1404,6 +1621,8 @@ static bool ice_pf_state_is_nominal(struct ice_pf *pf) * ice_pci_sriov_ena - Enable or change number of VFs * @pf: pointer to the PF structure * @num_vfs: number of VFs to allocate + * + * Returns 0 on success and negative on failure */ static int ice_pci_sriov_ena(struct ice_pf *pf, int num_vfs) { @@ -1411,20 +1630,10 @@ static int ice_pci_sriov_ena(struct ice_pf *pf, int num_vfs) struct device *dev = ice_pf_to_dev(pf); int err; - if (!ice_pf_state_is_nominal(pf)) { - dev_err(dev, "Cannot enable SR-IOV, device not ready\n"); - return -EBUSY; - } - - if (!test_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags)) { - dev_err(dev, "This device is not capable of SR-IOV\n"); - return -EOPNOTSUPP; - } - if (pre_existing_vfs && pre_existing_vfs != num_vfs) ice_free_vfs(pf); else if (pre_existing_vfs && pre_existing_vfs == num_vfs) - return num_vfs; + return 0; if (num_vfs > pf->num_vfs_supported) { dev_err(dev, "Can't enable %d VFs, max VFs supported is %d\n", @@ -1432,45 +1641,77 @@ static int ice_pci_sriov_ena(struct ice_pf *pf, int num_vfs) return -EOPNOTSUPP; } - dev_info(dev, "Allocating %d VFs\n", num_vfs); - err = ice_alloc_vfs(pf, num_vfs); + dev_info(dev, "Enabling %d VFs\n", num_vfs); + err = ice_ena_vfs(pf, num_vfs); if (err) { dev_err(dev, "Failed to enable SR-IOV: %d\n", err); return err; } set_bit(ICE_FLAG_SRIOV_ENA, pf->flags); - return num_vfs; + return 0; +} + +/** + * ice_check_sriov_allowed - check if SR-IOV is allowed based on various checks + * @pf: PF to enabled SR-IOV on + */ +static int ice_check_sriov_allowed(struct ice_pf *pf) +{ + struct device *dev = ice_pf_to_dev(pf); + + if (!test_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags)) { + dev_err(dev, "This device is not capable of SR-IOV\n"); + return -EOPNOTSUPP; + } + + if (ice_is_safe_mode(pf)) { + dev_err(dev, "SR-IOV cannot be configured - Device is in Safe Mode\n"); + return -EOPNOTSUPP; + } + + if (!ice_pf_state_is_nominal(pf)) { + dev_err(dev, "Cannot enable SR-IOV, device not ready\n"); + return -EBUSY; + } + + return 0; } /** * ice_sriov_configure - Enable or change number of VFs via sysfs * @pdev: pointer to a pci_dev structure - * @num_vfs: number of VFs to allocate + * @num_vfs: number of VFs to allocate or 0 to free VFs * - * This function is called when the user updates the number of VFs in sysfs. + * This function is called when the user updates the number of VFs in sysfs. On + * success return whatever num_vfs was set to by the caller. Return negative on + * failure. */ int ice_sriov_configure(struct pci_dev *pdev, int num_vfs) { struct ice_pf *pf = pci_get_drvdata(pdev); struct device *dev = ice_pf_to_dev(pf); + int err; - if (ice_is_safe_mode(pf)) { - dev_err(dev, "SR-IOV cannot be configured - Device is in Safe Mode\n"); - return -EOPNOTSUPP; - } + err = ice_check_sriov_allowed(pf); + if (err) + return err; - if (num_vfs) - return ice_pci_sriov_ena(pf, num_vfs); + if (!num_vfs) { + if (!pci_vfs_assigned(pdev)) { + ice_free_vfs(pf); + return 0; + } - if (!pci_vfs_assigned(pdev)) { - ice_free_vfs(pf); - } else { dev_err(dev, "can't free VFs because some are assigned to VMs.\n"); return -EBUSY; } - return 0; + err = ice_pci_sriov_ena(pf, num_vfs); + if (err) + return err; + + return num_vfs; } /** @@ -1483,7 +1724,7 @@ int ice_sriov_configure(struct pci_dev *pdev, int num_vfs) void ice_process_vflr_event(struct ice_pf *pf) { struct ice_hw *hw = &pf->hw; - int vf_id; + unsigned int vf_id; u32 reg; if (!test_and_clear_bit(__ICE_VFLR_EVENT_PENDING, pf->state) || @@ -1524,7 +1765,7 @@ static void ice_vc_reset_vf(struct ice_vf *vf) */ static struct ice_vf *ice_get_vf_from_pfq(struct ice_pf *pf, u16 pfq) { - int vf_id; + unsigned int vf_id; ice_for_each_vf(pf, vf_id) { struct ice_vf *vf = &pf->vf[vf_id]; @@ -1628,8 +1869,9 @@ ice_vc_send_msg_to_vf(struct ice_vf *vf, u32 v_opcode, aq_ret = ice_aq_send_msg_to_vf(&pf->hw, vf->vf_id, v_opcode, v_retval, msg, msglen, NULL); if (aq_ret && pf->hw.mailboxq.sq_last_status != ICE_AQ_RC_ENOSYS) { - dev_info(dev, "Unable to send the message to VF %d ret %d aq_err %d\n", - vf->vf_id, aq_ret, pf->hw.mailboxq.sq_last_status); + dev_info(dev, "Unable to send the message to VF %d ret %s aq_err %s\n", + vf->vf_id, ice_stat_str(aq_ret), + ice_aq_str(pf->hw.mailboxq.sq_last_status)); return -EIO; } @@ -1772,7 +2014,7 @@ err: */ static void ice_vc_reset_vf_msg(struct ice_vf *vf) { - if (test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) + if (test_bit(ICE_VF_STATE_INIT, vf->vf_states)) ice_reset_vf(vf, false); } @@ -2044,8 +2286,9 @@ int ice_set_vf_spoofchk(struct net_device *netdev, int vf_id, bool ena) status = ice_update_vsi(&pf->hw, vf_vsi->idx, ctx, NULL); if (status) { - dev_err(dev, "Failed to %sable spoofchk on VF %d VSI %d\n error %d\n", - ena ? "en" : "dis", vf->vf_id, vf_vsi->vsi_num, status); + dev_err(dev, "Failed to %sable spoofchk on VF %d VSI %d\n error %s\n", + ena ? "en" : "dis", vf->vf_id, vf_vsi->vsi_num, + ice_stat_str(status)); ret = -EIO; goto out; } @@ -2060,6 +2303,174 @@ out: } /** + * ice_is_any_vf_in_promisc - check if any VF(s) are in promiscuous mode + * @pf: PF structure for accessing VF(s) + * + * Return false if no VF(s) are in unicast and/or multicast promiscuous mode, + * else return true + */ +bool ice_is_any_vf_in_promisc(struct ice_pf *pf) +{ + int vf_idx; + + ice_for_each_vf(pf, vf_idx) { + struct ice_vf *vf = &pf->vf[vf_idx]; + + /* found a VF that has promiscuous mode configured */ + if (test_bit(ICE_VF_STATE_UC_PROMISC, vf->vf_states) || + test_bit(ICE_VF_STATE_MC_PROMISC, vf->vf_states)) + return true; + } + + return false; +} + +/** + * ice_vc_cfg_promiscuous_mode_msg + * @vf: pointer to the VF info + * @msg: pointer to the msg buffer + * + * called from the VF to configure VF VSIs promiscuous mode + */ +static int ice_vc_cfg_promiscuous_mode_msg(struct ice_vf *vf, u8 *msg) +{ + enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; + struct virtchnl_promisc_info *info = + (struct virtchnl_promisc_info *)msg; + struct ice_pf *pf = vf->pf; + struct ice_vsi *vsi; + struct device *dev; + bool rm_promisc; + int ret = 0; + + if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (!ice_vc_isvalid_vsi_id(vf, info->vsi_id)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + vsi = pf->vsi[vf->lan_vsi_idx]; + if (!vsi) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + dev = ice_pf_to_dev(pf); + if (!test_bit(ICE_VIRTCHNL_VF_CAP_PRIVILEGE, &vf->vf_caps)) { + dev_err(dev, "Unprivileged VF %d is attempting to configure promiscuous mode\n", + vf->vf_id); + /* Leave v_ret alone, lie to the VF on purpose. */ + goto error_param; + } + + rm_promisc = !(info->flags & FLAG_VF_UNICAST_PROMISC) && + !(info->flags & FLAG_VF_MULTICAST_PROMISC); + + if (vsi->num_vlan || vf->port_vlan_info) { + struct ice_vsi *pf_vsi = ice_get_main_vsi(pf); + struct net_device *pf_netdev; + + if (!pf_vsi) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + pf_netdev = pf_vsi->netdev; + + ret = ice_set_vf_spoofchk(pf_netdev, vf->vf_id, rm_promisc); + if (ret) { + dev_err(dev, "Failed to update spoofchk to %s for VF %d VSI %d when setting promiscuous mode\n", + rm_promisc ? "ON" : "OFF", vf->vf_id, + vsi->vsi_num); + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + } + + ret = ice_cfg_vlan_pruning(vsi, true, !rm_promisc); + if (ret) { + dev_err(dev, "Failed to configure VLAN pruning in promiscuous mode\n"); + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + } + + if (!test_bit(ICE_FLAG_VF_TRUE_PROMISC_ENA, pf->flags)) { + bool set_dflt_vsi = !!(info->flags & FLAG_VF_UNICAST_PROMISC); + + if (set_dflt_vsi && !ice_is_dflt_vsi_in_use(pf->first_sw)) + /* only attempt to set the default forwarding VSI if + * it's not currently set + */ + ret = ice_set_dflt_vsi(pf->first_sw, vsi); + else if (!set_dflt_vsi && + ice_is_vsi_dflt_vsi(pf->first_sw, vsi)) + /* only attempt to free the default forwarding VSI if we + * are the owner + */ + ret = ice_clear_dflt_vsi(pf->first_sw); + + if (ret) { + dev_err(dev, "%sable VF %d as the default VSI failed, error %d\n", + set_dflt_vsi ? "en" : "dis", vf->vf_id, ret); + v_ret = VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR; + goto error_param; + } + } else { + enum ice_status status; + u8 promisc_m; + + if (info->flags & FLAG_VF_UNICAST_PROMISC) { + if (vf->port_vlan_info || vsi->num_vlan) + promisc_m = ICE_UCAST_VLAN_PROMISC_BITS; + else + promisc_m = ICE_UCAST_PROMISC_BITS; + } else if (info->flags & FLAG_VF_MULTICAST_PROMISC) { + if (vf->port_vlan_info || vsi->num_vlan) + promisc_m = ICE_MCAST_VLAN_PROMISC_BITS; + else + promisc_m = ICE_MCAST_PROMISC_BITS; + } else { + if (vf->port_vlan_info || vsi->num_vlan) + promisc_m = ICE_UCAST_VLAN_PROMISC_BITS; + else + promisc_m = ICE_UCAST_PROMISC_BITS; + } + + /* Configure multicast/unicast with or without VLAN promiscuous + * mode + */ + status = ice_vf_set_vsi_promisc(vf, vsi, promisc_m, rm_promisc); + if (status) { + dev_err(dev, "%sable Tx/Rx filter promiscuous mode on VF-%d failed, error: %s\n", + rm_promisc ? "dis" : "en", vf->vf_id, + ice_stat_str(status)); + v_ret = ice_err_to_virt_err(status); + goto error_param; + } else { + dev_dbg(dev, "%sable Tx/Rx filter promiscuous mode on VF-%d succeeded\n", + rm_promisc ? "dis" : "en", vf->vf_id); + } + } + + if (info->flags & FLAG_VF_MULTICAST_PROMISC) + set_bit(ICE_VF_STATE_MC_PROMISC, vf->vf_states); + else + clear_bit(ICE_VF_STATE_MC_PROMISC, vf->vf_states); + + if (info->flags & FLAG_VF_UNICAST_PROMISC) + set_bit(ICE_VF_STATE_UC_PROMISC, vf->vf_states); + else + clear_bit(ICE_VF_STATE_UC_PROMISC, vf->vf_states); + +error_param: + return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, + v_ret, NULL, 0); +} + +/** * ice_vc_get_stats_msg * @vf: pointer to the VF info * @msg: pointer to the msg buffer @@ -2118,6 +2529,52 @@ static bool ice_vc_validate_vqs_bitmaps(struct virtchnl_queue_select *vqs) } /** + * ice_vf_ena_txq_interrupt - enable Tx queue interrupt via QINT_TQCTL + * @vsi: VSI of the VF to configure + * @q_idx: VF queue index used to determine the queue in the PF's space + */ +static void ice_vf_ena_txq_interrupt(struct ice_vsi *vsi, u32 q_idx) +{ + struct ice_hw *hw = &vsi->back->hw; + u32 pfq = vsi->txq_map[q_idx]; + u32 reg; + + reg = rd32(hw, QINT_TQCTL(pfq)); + + /* MSI-X index 0 in the VF's space is always for the OICR, which means + * this is most likely a poll mode VF driver, so don't enable an + * interrupt that was never configured via VIRTCHNL_OP_CONFIG_IRQ_MAP + */ + if (!(reg & QINT_TQCTL_MSIX_INDX_M)) + return; + + wr32(hw, QINT_TQCTL(pfq), reg | QINT_TQCTL_CAUSE_ENA_M); +} + +/** + * ice_vf_ena_rxq_interrupt - enable Tx queue interrupt via QINT_RQCTL + * @vsi: VSI of the VF to configure + * @q_idx: VF queue index used to determine the queue in the PF's space + */ +static void ice_vf_ena_rxq_interrupt(struct ice_vsi *vsi, u32 q_idx) +{ + struct ice_hw *hw = &vsi->back->hw; + u32 pfq = vsi->rxq_map[q_idx]; + u32 reg; + + reg = rd32(hw, QINT_RQCTL(pfq)); + + /* MSI-X index 0 in the VF's space is always for the OICR, which means + * this is most likely a poll mode VF driver, so don't enable an + * interrupt that was never configured via VIRTCHNL_OP_CONFIG_IRQ_MAP + */ + if (!(reg & QINT_RQCTL_MSIX_INDX_M)) + return; + + wr32(hw, QINT_RQCTL(pfq), reg | QINT_RQCTL_CAUSE_ENA_M); +} + +/** * ice_vc_ena_qs_msg * @vf: pointer to the VF info * @msg: pointer to the msg buffer @@ -2177,6 +2634,7 @@ static int ice_vc_ena_qs_msg(struct ice_vf *vf, u8 *msg) goto error_param; } + ice_vf_ena_rxq_interrupt(vsi, vf_q_id); set_bit(vf_q_id, vf->rxq_ena); } @@ -2192,6 +2650,7 @@ static int ice_vc_ena_qs_msg(struct ice_vf *vf, u8 *msg) if (test_bit(vf_q_id, vf->txq_ena)) continue; + ice_vf_ena_txq_interrupt(vsi, vf_q_id); set_bit(vf_q_id, vf->txq_ena); } @@ -2604,20 +3063,22 @@ ice_vc_add_mac_addr(struct ice_vf *vf, struct ice_vsi *vsi, u8 *mac_addr) return -EPERM; } - status = ice_vsi_cfg_mac_fltr(vsi, mac_addr, true); + status = ice_fltr_add_mac(vsi, mac_addr, ICE_FWD_TO_VSI); if (status == ICE_ERR_ALREADY_EXISTS) { dev_err(dev, "MAC %pM already exists for VF %d\n", mac_addr, vf->vf_id); return -EEXIST; } else if (status) { - dev_err(dev, "Failed to add MAC %pM for VF %d\n, error %d\n", - mac_addr, vf->vf_id, status); + dev_err(dev, "Failed to add MAC %pM for VF %d\n, error %s\n", + mac_addr, vf->vf_id, ice_stat_str(status)); return -EIO; } - /* only set dflt_lan_addr once */ - if (is_zero_ether_addr(vf->dflt_lan_addr.addr) && - is_unicast_ether_addr(mac_addr)) + /* Set the default LAN address to the latest unicast MAC address added + * by the VF. The default LAN address is reported by the PF via + * ndo_get_vf_config. + */ + if (is_unicast_ether_addr(mac_addr)) ether_addr_copy(vf->dflt_lan_addr.addr, mac_addr); vf->num_mac++; @@ -2641,14 +3102,14 @@ ice_vc_del_mac_addr(struct ice_vf *vf, struct ice_vsi *vsi, u8 *mac_addr) ether_addr_equal(mac_addr, vf->dflt_lan_addr.addr)) return 0; - status = ice_vsi_cfg_mac_fltr(vsi, mac_addr, false); + status = ice_fltr_remove_mac(vsi, mac_addr, ICE_FWD_TO_VSI); if (status == ICE_ERR_DOES_NOT_EXIST) { dev_err(dev, "MAC %pM does not exist for VF %d\n", mac_addr, vf->vf_id); return -ENOENT; } else if (status) { - dev_err(dev, "Failed to delete MAC %pM for VF %d, error %d\n", - mac_addr, vf->vf_id, status); + dev_err(dev, "Failed to delete MAC %pM for VF %d, error %s\n", + mac_addr, vf->vf_id, ice_stat_str(status)); return -EIO; } @@ -2834,7 +3295,6 @@ ice_set_vf_port_vlan(struct net_device *netdev, int vf_id, u16 vlan_id, u8 qos, __be16 vlan_proto) { struct ice_pf *pf = ice_netdev_to_pf(netdev); - struct ice_vsi *vsi; struct device *dev; struct ice_vf *vf; u16 vlanprio; @@ -2856,8 +3316,6 @@ ice_set_vf_port_vlan(struct net_device *netdev, int vf_id, u16 vlan_id, u8 qos, } vf = &pf->vf[vf_id]; - vsi = pf->vsi[vf->lan_vsi_idx]; - ret = ice_check_vf_ready_for_cfg(vf); if (ret) return ret; @@ -2870,44 +3328,15 @@ ice_set_vf_port_vlan(struct net_device *netdev, int vf_id, u16 vlan_id, u8 qos, return 0; } - if (vlan_id || qos) { - /* remove VLAN 0 filter set by default when transitioning from - * no port VLAN to a port VLAN. No change to old port VLAN on - * failure. - */ - ret = ice_vsi_kill_vlan(vsi, 0); - if (ret) - return ret; - ret = ice_vsi_manage_pvid(vsi, vlanprio, true); - if (ret) - return ret; - } else { - /* add VLAN 0 filter back when transitioning from port VLAN to - * no port VLAN. No change to old port VLAN on failure. - */ - ret = ice_vsi_add_vlan(vsi, 0); - if (ret) - return ret; - ret = ice_vsi_manage_pvid(vsi, 0, false); - if (ret) - return ret; - } + vf->port_vlan_info = vlanprio; - if (vlan_id) { + if (vf->port_vlan_info) dev_info(dev, "Setting VLAN %d, QoS 0x%x on VF %d\n", vlan_id, qos, vf_id); + else + dev_info(dev, "Clearing port VLAN on VF %d\n", vf_id); - /* add VLAN filter for the port VLAN */ - ret = ice_vsi_add_vlan(vsi, vlan_id); - if (ret) - return ret; - } - /* remove old port VLAN filter with valid VLAN ID or QoS fields */ - if (vf->port_vlan_info) - ice_vsi_kill_vlan(vsi, vf->port_vlan_info & VLAN_VID_MASK); - - /* keep port VLAN information persistent on resets */ - vf->port_vlan_info = le16_to_cpu(vsi->info.pvid); + ice_vc_reset_vf(vf); return 0; } @@ -2992,8 +3421,9 @@ static int ice_vc_process_vlan_msg(struct ice_vf *vf, u8 *msg, bool add_v) goto error_param; } - if (test_bit(ICE_VF_STATE_UC_PROMISC, vf->vf_states) || - test_bit(ICE_VF_STATE_MC_PROMISC, vf->vf_states)) + if ((test_bit(ICE_VF_STATE_UC_PROMISC, vf->vf_states) || + test_bit(ICE_VF_STATE_MC_PROMISC, vf->vf_states)) && + test_bit(ICE_FLAG_VF_TRUE_PROMISC_ENA, pf->flags)) vlan_promisc = true; if (add_v) { @@ -3018,7 +3448,7 @@ static int ice_vc_process_vlan_msg(struct ice_vf *vf, u8 *msg, bool add_v) if (!vid) continue; - status = ice_vsi_add_vlan(vsi, vid); + status = ice_vsi_add_vlan(vsi, vid, ICE_FWD_TO_VSI); if (status) { v_ret = VIRTCHNL_STATUS_ERR_PARAM; goto error_param; @@ -3317,6 +3747,9 @@ error_handler: case VIRTCHNL_OP_GET_STATS: err = ice_vc_get_stats_msg(vf, msg); break; + case VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: + err = ice_vc_cfg_promiscuous_mode_msg(vf, msg); + break; case VIRTCHNL_OP_ADD_VLAN: err = ice_vc_add_vlan_msg(vf, msg); break; @@ -3390,6 +3823,39 @@ ice_get_vf_cfg(struct net_device *netdev, int vf_id, struct ifla_vf_info *ivi) } /** + * ice_unicast_mac_exists - check if the unicast MAC exists on the PF's switch + * @pf: PF used to reference the switch's rules + * @umac: unicast MAC to compare against existing switch rules + * + * Return true on the first/any match, else return false + */ +static bool ice_unicast_mac_exists(struct ice_pf *pf, u8 *umac) +{ + struct ice_sw_recipe *mac_recipe_list = + &pf->hw.switch_info->recp_list[ICE_SW_LKUP_MAC]; + struct ice_fltr_mgmt_list_entry *list_itr; + struct list_head *rule_head; + struct mutex *rule_lock; /* protect MAC filter list access */ + + rule_head = &mac_recipe_list->filt_rules; + rule_lock = &mac_recipe_list->filt_rule_lock; + + mutex_lock(rule_lock); + list_for_each_entry(list_itr, rule_head, list_entry) { + u8 *existing_mac = &list_itr->fltr_info.l_data.mac.mac_addr[0]; + + if (ether_addr_equal(existing_mac, umac)) { + mutex_unlock(rule_lock); + return true; + } + } + + mutex_unlock(rule_lock); + + return false; +} + +/** * ice_set_vf_mac * @netdev: network interface device structure * @vf_id: VF identifier @@ -3406,25 +3872,41 @@ int ice_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac) if (ice_validate_vf_id(pf, vf_id)) return -EINVAL; - if (is_zero_ether_addr(mac) || is_multicast_ether_addr(mac)) { + if (is_multicast_ether_addr(mac)) { netdev_err(netdev, "%pM not a valid unicast address\n", mac); return -EINVAL; } vf = &pf->vf[vf_id]; + /* nothing left to do, unicast MAC already set */ + if (ether_addr_equal(vf->dflt_lan_addr.addr, mac)) + return 0; + ret = ice_check_vf_ready_for_cfg(vf); if (ret) return ret; - /* copy MAC into dflt_lan_addr and trigger a VF reset. The reset - * flow will use the updated dflt_lan_addr and add a MAC filter - * using ice_add_mac. Also set pf_set_mac to indicate that the PF has - * set the MAC address for this VF. + if (ice_unicast_mac_exists(pf, mac)) { + netdev_err(netdev, "Unicast MAC %pM already exists on this PF. Preventing setting VF %u unicast MAC address to %pM\n", + mac, vf_id, mac); + return -EINVAL; + } + + /* VF is notified of its new MAC via the PF's response to the + * VIRTCHNL_OP_GET_VF_RESOURCES message after the VF has been reset */ ether_addr_copy(vf->dflt_lan_addr.addr, mac); - vf->pf_set_mac = true; - netdev_info(netdev, "MAC on VF %d set to %pM. VF driver will be reinitialized\n", - vf_id, mac); + if (is_zero_ether_addr(mac)) { + /* VF will send VIRTCHNL_OP_ADD_ETH_ADDR message with its MAC */ + vf->pf_set_mac = false; + netdev_info(netdev, "Removing MAC on VF %d. VF driver will be reinitialized\n", + vf->vf_id); + } else { + /* PF will add MAC rule for the VF */ + vf->pf_set_mac = true; + netdev_info(netdev, "Setting MAC %pM on VF %d. VF driver will be reinitialized\n", + mac, vf_id); + } ice_vc_reset_vf(vf); return 0; @@ -3554,6 +4036,24 @@ int ice_get_vf_stats(struct net_device *netdev, int vf_id, } /** + * ice_print_vf_rx_mdd_event - print VF Rx malicious driver detect event + * @vf: pointer to the VF structure + */ +void ice_print_vf_rx_mdd_event(struct ice_vf *vf) +{ + struct ice_pf *pf = vf->pf; + struct device *dev; + + dev = ice_pf_to_dev(pf); + + dev_info(dev, "%d Rx Malicious Driver Detection events detected on PF %d VF %d MAC %pM. mdd-auto-reset-vfs=%s\n", + vf->mdd_rx_events.count, pf->hw.pf_id, vf->vf_id, + vf->dflt_lan_addr.addr, + test_bit(ICE_FLAG_MDD_AUTO_RESET_VF, pf->flags) + ? "on" : "off"); +} + +/** * ice_print_vfs_mdd_event - print VFs malicious driver detect event * @pf: pointer to the PF structure * @@ -3582,12 +4082,7 @@ void ice_print_vfs_mdd_events(struct ice_pf *pf) if (vf->mdd_rx_events.count != vf->mdd_rx_events.last_printed) { vf->mdd_rx_events.last_printed = vf->mdd_rx_events.count; - - dev_info(dev, "%d Rx Malicious Driver Detection events detected on PF %d VF %d MAC %pM. mdd-auto-reset-vfs=%s\n", - vf->mdd_rx_events.count, hw->pf_id, i, - vf->dflt_lan_addr.addr, - test_bit(ICE_FLAG_MDD_AUTO_RESET_VF, pf->flags) - ? "on" : "off"); + ice_print_vf_rx_mdd_event(vf); } /* only print Tx MDD event message if there are new events */ diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h index 3f9464269bd2..67aa9110fdd1 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h +++ b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h @@ -7,7 +7,10 @@ /* Restrict number of MAC Addr and VLAN that non-trusted VF can programmed */ #define ICE_MAX_VLAN_PER_VF 8 -#define ICE_MAX_MACADDR_PER_VF 12 +/* MAC filters: 1 is reserved for the VF's default/perm_addr/LAA MAC, 1 for + * broadcast, and 16 for additional unicast/multicast filters + */ +#define ICE_MAX_MACADDR_PER_VF 18 /* Malicious Driver Detection */ #define ICE_DFLT_NUM_INVAL_MSGS_ALLOWED 10 @@ -64,7 +67,7 @@ struct ice_mdd_vf_events { struct ice_vf { struct ice_pf *pf; - s16 vf_id; /* VF ID in the PF space */ + u16 vf_id; /* VF ID in the PF space */ u16 lan_vsi_idx; /* index into PF struct */ /* first vector index of this VF in the PF space */ int first_vector_idx; @@ -128,9 +131,11 @@ void ice_set_vf_state_qs_dis(struct ice_vf *vf); int ice_get_vf_stats(struct net_device *netdev, int vf_id, struct ifla_vf_stats *vf_stats); +bool ice_is_any_vf_in_promisc(struct ice_pf *pf); void ice_vf_lan_overflow_event(struct ice_pf *pf, struct ice_rq_event_info *event); void ice_print_vfs_mdd_events(struct ice_pf *pf); +void ice_print_vf_rx_mdd_event(struct ice_vf *vf); #else /* CONFIG_PCI_IOV */ #define ice_process_vflr_event(pf) do {} while (0) #define ice_free_vfs(pf) do {} while (0) @@ -140,6 +145,7 @@ void ice_print_vfs_mdd_events(struct ice_pf *pf); #define ice_set_vf_state_qs_dis(vf) do {} while (0) #define ice_vf_lan_overflow_event(pf, event) do {} while (0) #define ice_print_vfs_mdd_events(pf) do {} while (0) +#define ice_print_vf_rx_mdd_event(vf) do {} while (0) static inline bool ice_reset_all_vfs(struct ice_pf __always_unused *pf, @@ -219,5 +225,10 @@ ice_get_vf_stats(struct net_device __always_unused *netdev, { return -EOPNOTSUPP; } + +static inline bool ice_is_any_vf_in_promisc(struct ice_pf __always_unused *pf) +{ + return false; +} #endif /* CONFIG_PCI_IOV */ #endif /* _ICE_VIRTCHNL_PF_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.c b/drivers/net/ethernet/intel/ice/ice_xsk.c index 8279db15e870..b6f928c9e9c9 100644 --- a/drivers/net/ethernet/intel/ice/ice_xsk.c +++ b/drivers/net/ethernet/intel/ice/ice_xsk.c @@ -2,7 +2,7 @@ /* Copyright (c) 2019, Intel Corporation. */ #include <linux/bpf_trace.h> -#include <net/xdp_sock.h> +#include <net/xdp_sock_drv.h> #include <net/xdp.h> #include "ice.h" #include "ice_base.h" @@ -280,28 +280,6 @@ static int ice_xsk_alloc_umems(struct ice_vsi *vsi) } /** - * ice_xsk_add_umem - add a UMEM region for XDP sockets - * @vsi: VSI to which the UMEM will be added - * @umem: pointer to a requested UMEM region - * @qid: queue ID - * - * Returns 0 on success, negative on error - */ -static int ice_xsk_add_umem(struct ice_vsi *vsi, struct xdp_umem *umem, u16 qid) -{ - int err; - - err = ice_xsk_alloc_umems(vsi); - if (err) - return err; - - vsi->xsk_umems[qid] = umem; - vsi->num_xsk_umems_used++; - - return 0; -} - -/** * ice_xsk_remove_umem - Remove an UMEM for a certain ring/qid * @vsi: VSI from which the VSI will be removed * @qid: Ring/qid associated with the UMEM @@ -318,65 +296,6 @@ static void ice_xsk_remove_umem(struct ice_vsi *vsi, u16 qid) } } -/** - * ice_xsk_umem_dma_map - DMA map UMEM region for XDP sockets - * @vsi: VSI to map the UMEM region - * @umem: UMEM to map - * - * Returns 0 on success, negative on error - */ -static int ice_xsk_umem_dma_map(struct ice_vsi *vsi, struct xdp_umem *umem) -{ - struct ice_pf *pf = vsi->back; - struct device *dev; - unsigned int i; - - dev = ice_pf_to_dev(pf); - for (i = 0; i < umem->npgs; i++) { - dma_addr_t dma = dma_map_page_attrs(dev, umem->pgs[i], 0, - PAGE_SIZE, - DMA_BIDIRECTIONAL, - ICE_RX_DMA_ATTR); - if (dma_mapping_error(dev, dma)) { - dev_dbg(dev, "XSK UMEM DMA mapping error on page num %d\n", - i); - goto out_unmap; - } - - umem->pages[i].dma = dma; - } - - return 0; - -out_unmap: - for (; i > 0; i--) { - dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE, - DMA_BIDIRECTIONAL, ICE_RX_DMA_ATTR); - umem->pages[i].dma = 0; - } - - return -EFAULT; -} - -/** - * ice_xsk_umem_dma_unmap - DMA unmap UMEM region for XDP sockets - * @vsi: VSI from which the UMEM will be unmapped - * @umem: UMEM to unmap - */ -static void ice_xsk_umem_dma_unmap(struct ice_vsi *vsi, struct xdp_umem *umem) -{ - struct ice_pf *pf = vsi->back; - struct device *dev; - unsigned int i; - - dev = ice_pf_to_dev(pf); - for (i = 0; i < umem->npgs; i++) { - dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE, - DMA_BIDIRECTIONAL, ICE_RX_DMA_ATTR); - - umem->pages[i].dma = 0; - } -} /** * ice_xsk_umem_disable - disable a UMEM region @@ -391,7 +310,7 @@ static int ice_xsk_umem_disable(struct ice_vsi *vsi, u16 qid) !vsi->xsk_umems[qid]) return -EINVAL; - ice_xsk_umem_dma_unmap(vsi, vsi->xsk_umems[qid]); + xsk_buff_dma_unmap(vsi->xsk_umems[qid], ICE_RX_DMA_ATTR); ice_xsk_remove_umem(vsi, qid); return 0; @@ -408,7 +327,6 @@ static int ice_xsk_umem_disable(struct ice_vsi *vsi, u16 qid) static int ice_xsk_umem_enable(struct ice_vsi *vsi, struct xdp_umem *umem, u16 qid) { - struct xdp_umem_fq_reuse *reuseq; int err; if (vsi->type != ICE_VSI_PF) @@ -419,20 +337,18 @@ ice_xsk_umem_enable(struct ice_vsi *vsi, struct xdp_umem *umem, u16 qid) if (qid >= vsi->num_xsk_umems) return -EINVAL; + err = ice_xsk_alloc_umems(vsi); + if (err) + return err; + if (vsi->xsk_umems && vsi->xsk_umems[qid]) return -EBUSY; - reuseq = xsk_reuseq_prepare(vsi->rx_rings[0]->count); - if (!reuseq) - return -ENOMEM; - - xsk_reuseq_free(xsk_reuseq_swap(umem, reuseq)); - - err = ice_xsk_umem_dma_map(vsi, umem); - if (err) - return err; + vsi->xsk_umems[qid] = umem; + vsi->num_xsk_umems_used++; - err = ice_xsk_add_umem(vsi, umem, qid); + err = xsk_buff_dma_map(vsi->xsk_umems[qid], ice_pf_to_dev(vsi->back), + ICE_RX_DMA_ATTR); if (err) return err; @@ -484,137 +400,22 @@ xsk_umem_if_up: } /** - * ice_zca_free - Callback for MEM_TYPE_ZERO_COPY allocations - * @zca: zero-cpoy allocator - * @handle: Buffer handle - */ -void ice_zca_free(struct zero_copy_allocator *zca, unsigned long handle) -{ - struct ice_rx_buf *rx_buf; - struct ice_ring *rx_ring; - struct xdp_umem *umem; - u64 hr, mask; - u16 nta; - - rx_ring = container_of(zca, struct ice_ring, zca); - umem = rx_ring->xsk_umem; - hr = umem->headroom + XDP_PACKET_HEADROOM; - - mask = umem->chunk_mask; - - nta = rx_ring->next_to_alloc; - rx_buf = &rx_ring->rx_buf[nta]; - - nta++; - rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; - - handle &= mask; - - rx_buf->dma = xdp_umem_get_dma(umem, handle); - rx_buf->dma += hr; - - rx_buf->addr = xdp_umem_get_data(umem, handle); - rx_buf->addr += hr; - - rx_buf->handle = (u64)handle + umem->headroom; -} - -/** - * ice_alloc_buf_fast_zc - Retrieve buffer address from XDP umem - * @rx_ring: ring with an xdp_umem bound to it - * @rx_buf: buffer to which xsk page address will be assigned - * - * This function allocates an Rx buffer in the hot path. - * The buffer can come from fill queue or recycle queue. - * - * Returns true if an assignment was successful, false if not. - */ -static __always_inline bool -ice_alloc_buf_fast_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf) -{ - struct xdp_umem *umem = rx_ring->xsk_umem; - void *addr = rx_buf->addr; - u64 handle, hr; - - if (addr) { - rx_ring->rx_stats.page_reuse_count++; - return true; - } - - if (!xsk_umem_peek_addr(umem, &handle)) { - rx_ring->rx_stats.alloc_page_failed++; - return false; - } - - hr = umem->headroom + XDP_PACKET_HEADROOM; - - rx_buf->dma = xdp_umem_get_dma(umem, handle); - rx_buf->dma += hr; - - rx_buf->addr = xdp_umem_get_data(umem, handle); - rx_buf->addr += hr; - - rx_buf->handle = handle + umem->headroom; - - xsk_umem_release_addr(umem); - return true; -} - -/** - * ice_alloc_buf_slow_zc - Retrieve buffer address from XDP umem - * @rx_ring: ring with an xdp_umem bound to it - * @rx_buf: buffer to which xsk page address will be assigned - * - * This function allocates an Rx buffer in the slow path. - * The buffer can come from fill queue or recycle queue. - * - * Returns true if an assignment was successful, false if not. - */ -static __always_inline bool -ice_alloc_buf_slow_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf) -{ - struct xdp_umem *umem = rx_ring->xsk_umem; - u64 handle, headroom; - - if (!xsk_umem_peek_addr_rq(umem, &handle)) { - rx_ring->rx_stats.alloc_page_failed++; - return false; - } - - handle &= umem->chunk_mask; - headroom = umem->headroom + XDP_PACKET_HEADROOM; - - rx_buf->dma = xdp_umem_get_dma(umem, handle); - rx_buf->dma += headroom; - - rx_buf->addr = xdp_umem_get_data(umem, handle); - rx_buf->addr += headroom; - - rx_buf->handle = handle + umem->headroom; - - xsk_umem_release_addr_rq(umem); - return true; -} - -/** * ice_alloc_rx_bufs_zc - allocate a number of Rx buffers * @rx_ring: Rx ring * @count: The number of buffers to allocate - * @alloc: the function pointer to call for allocation * * This function allocates a number of Rx buffers from the fill ring * or the internal recycle mechanism and places them on the Rx ring. * * Returns false if all allocations were successful, true if any fail. */ -static bool -ice_alloc_rx_bufs_zc(struct ice_ring *rx_ring, int count, - bool (*alloc)(struct ice_ring *, struct ice_rx_buf *)) +bool ice_alloc_rx_bufs_zc(struct ice_ring *rx_ring, u16 count) { union ice_32b_rx_flex_desc *rx_desc; u16 ntu = rx_ring->next_to_use; struct ice_rx_buf *rx_buf; bool ret = false; + dma_addr_t dma; if (!count) return false; @@ -623,16 +424,14 @@ ice_alloc_rx_bufs_zc(struct ice_ring *rx_ring, int count, rx_buf = &rx_ring->rx_buf[ntu]; do { - if (!alloc(rx_ring, rx_buf)) { + rx_buf->xdp = xsk_buff_alloc(rx_ring->xsk_umem); + if (!rx_buf->xdp) { ret = true; break; } - dma_sync_single_range_for_device(rx_ring->dev, rx_buf->dma, 0, - rx_ring->rx_buf_len, - DMA_BIDIRECTIONAL); - - rx_desc->read.pkt_addr = cpu_to_le64(rx_buf->dma); + dma = xsk_buff_xdp_get_dma(rx_buf->xdp); + rx_desc->read.pkt_addr = cpu_to_le64(dma); rx_desc->wb.status_error0 = 0; rx_desc++; @@ -653,32 +452,6 @@ ice_alloc_rx_bufs_zc(struct ice_ring *rx_ring, int count, } /** - * ice_alloc_rx_bufs_fast_zc - allocate zero copy bufs in the hot path - * @rx_ring: Rx ring - * @count: number of bufs to allocate - * - * Returns false on success, true on failure. - */ -static bool ice_alloc_rx_bufs_fast_zc(struct ice_ring *rx_ring, u16 count) -{ - return ice_alloc_rx_bufs_zc(rx_ring, count, - ice_alloc_buf_fast_zc); -} - -/** - * ice_alloc_rx_bufs_slow_zc - allocate zero copy bufs in the slow path - * @rx_ring: Rx ring - * @count: number of bufs to allocate - * - * Returns false on success, true on failure. - */ -bool ice_alloc_rx_bufs_slow_zc(struct ice_ring *rx_ring, u16 count) -{ - return ice_alloc_rx_bufs_zc(rx_ring, count, - ice_alloc_buf_slow_zc); -} - -/** * ice_bump_ntc - Bump the next_to_clean counter of an Rx ring * @rx_ring: Rx ring */ @@ -692,76 +465,21 @@ static void ice_bump_ntc(struct ice_ring *rx_ring) } /** - * ice_get_rx_buf_zc - Fetch the current Rx buffer - * @rx_ring: Rx ring - * @size: size of a buffer - * - * This function returns the current, received Rx buffer and does - * DMA synchronization. - * - * Returns a pointer to the received Rx buffer. - */ -static struct ice_rx_buf *ice_get_rx_buf_zc(struct ice_ring *rx_ring, int size) -{ - struct ice_rx_buf *rx_buf; - - rx_buf = &rx_ring->rx_buf[rx_ring->next_to_clean]; - - dma_sync_single_range_for_cpu(rx_ring->dev, rx_buf->dma, 0, - size, DMA_BIDIRECTIONAL); - - return rx_buf; -} - -/** - * ice_reuse_rx_buf_zc - reuse an Rx buffer - * @rx_ring: Rx ring - * @old_buf: The buffer to recycle - * - * This function recycles a finished Rx buffer, and places it on the recycle - * queue (next_to_alloc). - */ -static void -ice_reuse_rx_buf_zc(struct ice_ring *rx_ring, struct ice_rx_buf *old_buf) -{ - unsigned long mask = (unsigned long)rx_ring->xsk_umem->chunk_mask; - u64 hr = rx_ring->xsk_umem->headroom + XDP_PACKET_HEADROOM; - u16 nta = rx_ring->next_to_alloc; - struct ice_rx_buf *new_buf; - - new_buf = &rx_ring->rx_buf[nta++]; - rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; - - new_buf->dma = old_buf->dma & mask; - new_buf->dma += hr; - - new_buf->addr = (void *)((unsigned long)old_buf->addr & mask); - new_buf->addr += hr; - - new_buf->handle = old_buf->handle & mask; - new_buf->handle += rx_ring->xsk_umem->headroom; - - old_buf->addr = NULL; -} - -/** * ice_construct_skb_zc - Create an sk_buff from zero-copy buffer * @rx_ring: Rx ring * @rx_buf: zero-copy Rx buffer - * @xdp: XDP buffer * * This function allocates a new skb from a zero-copy Rx buffer. * * Returns the skb on success, NULL on failure. */ static struct sk_buff * -ice_construct_skb_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf, - struct xdp_buff *xdp) +ice_construct_skb_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf) { - unsigned int metasize = xdp->data - xdp->data_meta; - unsigned int datasize = xdp->data_end - xdp->data; - unsigned int datasize_hard = xdp->data_end - - xdp->data_hard_start; + unsigned int metasize = rx_buf->xdp->data - rx_buf->xdp->data_meta; + unsigned int datasize = rx_buf->xdp->data_end - rx_buf->xdp->data; + unsigned int datasize_hard = rx_buf->xdp->data_end - + rx_buf->xdp->data_hard_start; struct sk_buff *skb; skb = __napi_alloc_skb(&rx_ring->q_vector->napi, datasize_hard, @@ -769,13 +487,13 @@ ice_construct_skb_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf, if (unlikely(!skb)) return NULL; - skb_reserve(skb, xdp->data - xdp->data_hard_start); - memcpy(__skb_put(skb, datasize), xdp->data, datasize); + skb_reserve(skb, rx_buf->xdp->data - rx_buf->xdp->data_hard_start); + memcpy(__skb_put(skb, datasize), rx_buf->xdp->data, datasize); if (metasize) skb_metadata_set(skb, metasize); - ice_reuse_rx_buf_zc(rx_ring, rx_buf); - + xsk_buff_free(rx_buf->xdp); + rx_buf->xdp = NULL; return skb; } @@ -802,7 +520,6 @@ ice_run_xdp_zc(struct ice_ring *rx_ring, struct xdp_buff *xdp) } act = bpf_prog_run_xdp(xdp_prog, xdp); - xdp->handle += xdp->data - xdp->data_hard_start; switch (act) { case XDP_PASS: break; @@ -842,9 +559,6 @@ int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget) u16 cleaned_count = ICE_DESC_UNUSED(rx_ring); unsigned int xdp_xmit = 0; bool failure = false; - struct xdp_buff xdp; - - xdp.rxq = &rx_ring->xdp_rxq; while (likely(total_rx_packets < (unsigned int)budget)) { union ice_32b_rx_flex_desc *rx_desc; @@ -856,8 +570,8 @@ int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget) u8 rx_ptype; if (cleaned_count >= ICE_RX_BUF_WRITE) { - failure |= ice_alloc_rx_bufs_fast_zc(rx_ring, - cleaned_count); + failure |= ice_alloc_rx_bufs_zc(rx_ring, + cleaned_count); cleaned_count = 0; } @@ -878,25 +592,19 @@ int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget) if (!size) break; - rx_buf = ice_get_rx_buf_zc(rx_ring, size); - if (!rx_buf->addr) - break; - xdp.data = rx_buf->addr; - xdp.data_meta = xdp.data; - xdp.data_hard_start = xdp.data - XDP_PACKET_HEADROOM; - xdp.data_end = xdp.data + size; - xdp.handle = rx_buf->handle; + rx_buf = &rx_ring->rx_buf[rx_ring->next_to_clean]; + rx_buf->xdp->data_end = rx_buf->xdp->data + size; + xsk_buff_dma_sync_for_cpu(rx_buf->xdp); - xdp_res = ice_run_xdp_zc(rx_ring, &xdp); + xdp_res = ice_run_xdp_zc(rx_ring, rx_buf->xdp); if (xdp_res) { - if (xdp_res & (ICE_XDP_TX | ICE_XDP_REDIR)) { + if (xdp_res & (ICE_XDP_TX | ICE_XDP_REDIR)) xdp_xmit |= xdp_res; - rx_buf->addr = NULL; - } else { - ice_reuse_rx_buf_zc(rx_ring, rx_buf); - } + else + xsk_buff_free(rx_buf->xdp); + rx_buf->xdp = NULL; total_rx_bytes += size; total_rx_packets++; cleaned_count++; @@ -906,7 +614,7 @@ int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget) } /* XDP_PASS path */ - skb = ice_construct_skb_zc(rx_ring, rx_buf, &xdp); + skb = ice_construct_skb_zc(rx_ring, rx_buf); if (!skb) { rx_ring->rx_stats.alloc_buf_failed++; break; @@ -977,17 +685,16 @@ static bool ice_xmit_zc(struct ice_ring *xdp_ring, int budget) if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &desc)) break; - dma = xdp_umem_get_dma(xdp_ring->xsk_umem, desc.addr); - - dma_sync_single_for_device(xdp_ring->dev, dma, desc.len, - DMA_BIDIRECTIONAL); + dma = xsk_buff_raw_get_dma(xdp_ring->xsk_umem, desc.addr); + xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_umem, dma, + desc.len); tx_buf->bytecount = desc.len; tx_desc = ICE_TX_DESC(xdp_ring, xdp_ring->next_to_use); tx_desc->buf_addr = cpu_to_le64(dma); - tx_desc->cmd_type_offset_bsz = build_ctob(ICE_TXD_LAST_DESC_CMD, - 0, desc.len, 0); + tx_desc->cmd_type_offset_bsz = + ice_build_ctob(ICE_TXD_LAST_DESC_CMD, 0, desc.len, 0); xdp_ring->next_to_use++; if (xdp_ring->next_to_use == xdp_ring->count) @@ -1163,11 +870,10 @@ void ice_xsk_clean_rx_ring(struct ice_ring *rx_ring) for (i = 0; i < rx_ring->count; i++) { struct ice_rx_buf *rx_buf = &rx_ring->rx_buf[i]; - if (!rx_buf->addr) + if (!rx_buf->xdp) continue; - xsk_umem_fq_reuse(rx_ring->xsk_umem, rx_buf->handle); - rx_buf->addr = NULL; + rx_buf->xdp = NULL; } } diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.h b/drivers/net/ethernet/intel/ice/ice_xsk.h index 8a4ba7c6d549..fc1a06b4df36 100644 --- a/drivers/net/ethernet/intel/ice/ice_xsk.h +++ b/drivers/net/ethernet/intel/ice/ice_xsk.h @@ -10,11 +10,10 @@ struct ice_vsi; #ifdef CONFIG_XDP_SOCKETS int ice_xsk_umem_setup(struct ice_vsi *vsi, struct xdp_umem *umem, u16 qid); -void ice_zca_free(struct zero_copy_allocator *zca, unsigned long handle); int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget); bool ice_clean_tx_irq_zc(struct ice_ring *xdp_ring, int budget); int ice_xsk_wakeup(struct net_device *netdev, u32 queue_id, u32 flags); -bool ice_alloc_rx_bufs_slow_zc(struct ice_ring *rx_ring, u16 count); +bool ice_alloc_rx_bufs_zc(struct ice_ring *rx_ring, u16 count); bool ice_xsk_any_rx_ring_ena(struct ice_vsi *vsi); void ice_xsk_clean_rx_ring(struct ice_ring *rx_ring); void ice_xsk_clean_xdp_ring(struct ice_ring *xdp_ring); @@ -27,12 +26,6 @@ ice_xsk_umem_setup(struct ice_vsi __always_unused *vsi, return -EOPNOTSUPP; } -static inline void -ice_zca_free(struct zero_copy_allocator __always_unused *zca, - unsigned long __always_unused handle) -{ -} - static inline int ice_clean_rx_irq_zc(struct ice_ring __always_unused *rx_ring, int __always_unused budget) @@ -48,8 +41,8 @@ ice_clean_tx_irq_zc(struct ice_ring __always_unused *xdp_ring, } static inline bool -ice_alloc_rx_bufs_slow_zc(struct ice_ring __always_unused *rx_ring, - u16 __always_unused count) +ice_alloc_rx_bufs_zc(struct ice_ring __always_unused *rx_ring, + u16 __always_unused count) { return false; } diff --git a/drivers/net/ethernet/intel/igb/e1000_mac.c b/drivers/net/ethernet/intel/igb/e1000_mac.c index 79ee0a747260..3254737c07a3 100644 --- a/drivers/net/ethernet/intel/igb/e1000_mac.c +++ b/drivers/net/ethernet/intel/igb/e1000_mac.c @@ -12,7 +12,7 @@ #include "igb.h" static s32 igb_set_default_fc(struct e1000_hw *hw); -static s32 igb_set_fc_watermarks(struct e1000_hw *hw); +static void igb_set_fc_watermarks(struct e1000_hw *hw); /** * igb_get_bus_info_pcie - Get PCIe bus information @@ -687,7 +687,7 @@ s32 igb_setup_link(struct e1000_hw *hw) wr32(E1000_FCTTV, hw->fc.pause_time); - ret_val = igb_set_fc_watermarks(hw); + igb_set_fc_watermarks(hw); out: @@ -723,9 +723,8 @@ void igb_config_collision_dist(struct e1000_hw *hw) * flow control XON frame transmission is enabled, then set XON frame * tansmission as well. **/ -static s32 igb_set_fc_watermarks(struct e1000_hw *hw) +static void igb_set_fc_watermarks(struct e1000_hw *hw) { - s32 ret_val = 0; u32 fcrtl = 0, fcrth = 0; /* Set the flow control receive threshold registers. Normally, @@ -747,8 +746,6 @@ static s32 igb_set_fc_watermarks(struct e1000_hw *hw) } wr32(E1000_FCRTL, fcrtl); wr32(E1000_FCRTH, fcrth); - - return ret_val; } /** diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c index 39d3b76a6f5d..2cd003c5ad43 100644 --- a/drivers/net/ethernet/intel/igb/igb_ethtool.c +++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c @@ -143,7 +143,8 @@ static int igb_get_link_ksettings(struct net_device *netdev, u32 speed; u32 supported, advertising; - status = rd32(E1000_STATUS); + status = pm_runtime_suspended(&adapter->pdev->dev) ? + 0 : rd32(E1000_STATUS); if (hw->phy.media_type == e1000_media_type_copper) { supported = (SUPPORTED_10baseT_Half | diff --git a/drivers/net/ethernet/intel/igc/Makefile b/drivers/net/ethernet/intel/igc/Makefile index e3c164c12e10..1c3051db9085 100644 --- a/drivers/net/ethernet/intel/igc/Makefile +++ b/drivers/net/ethernet/intel/igc/Makefile @@ -8,4 +8,4 @@ obj-$(CONFIG_IGC) += igc.o igc-objs := igc_main.o igc_mac.o igc_i225.o igc_base.o igc_nvm.o igc_phy.o \ -igc_ethtool.o igc_ptp.o igc_dump.o +igc_diag.o igc_ethtool.o igc_ptp.o igc_dump.o igc_tsn.o diff --git a/drivers/net/ethernet/intel/igc/igc.h b/drivers/net/ethernet/intel/igc/igc.h index a1f845a2aa80..5dbc5a156626 100644 --- a/drivers/net/ethernet/intel/igc/igc.h +++ b/drivers/net/ethernet/intel/igc/igc.h @@ -16,14 +16,210 @@ #include "igc_hw.h" -/* forward declaration */ -void igc_set_ethtool_ops(struct net_device *); +void igc_ethtool_set_ops(struct net_device *); -struct igc_adapter; -struct igc_ring; +/* Transmit and receive queues */ +#define IGC_MAX_RX_QUEUES 4 +#define IGC_MAX_TX_QUEUES 4 + +#define MAX_Q_VECTORS 8 +#define MAX_STD_JUMBO_FRAME_SIZE 9216 + +#define MAX_ETYPE_FILTER 8 +#define IGC_RETA_SIZE 128 + +enum igc_mac_filter_type { + IGC_MAC_FILTER_TYPE_DST = 0, + IGC_MAC_FILTER_TYPE_SRC +}; + +struct igc_tx_queue_stats { + u64 packets; + u64 bytes; + u64 restart_queue; + u64 restart_queue2; +}; + +struct igc_rx_queue_stats { + u64 packets; + u64 bytes; + u64 drops; + u64 csum_err; + u64 alloc_failed; +}; + +struct igc_rx_packet_stats { + u64 ipv4_packets; /* IPv4 headers processed */ + u64 ipv4e_packets; /* IPv4E headers with extensions processed */ + u64 ipv6_packets; /* IPv6 headers processed */ + u64 ipv6e_packets; /* IPv6E headers with extensions processed */ + u64 tcp_packets; /* TCP headers processed */ + u64 udp_packets; /* UDP headers processed */ + u64 sctp_packets; /* SCTP headers processed */ + u64 nfs_packets; /* NFS headers processe */ + u64 other_packets; +}; + +struct igc_ring_container { + struct igc_ring *ring; /* pointer to linked list of rings */ + unsigned int total_bytes; /* total bytes processed this int */ + unsigned int total_packets; /* total packets processed this int */ + u16 work_limit; /* total work allowed per interrupt */ + u8 count; /* total number of rings in vector */ + u8 itr; /* current ITR setting for ring */ +}; + +struct igc_ring { + struct igc_q_vector *q_vector; /* backlink to q_vector */ + struct net_device *netdev; /* back pointer to net_device */ + struct device *dev; /* device for dma mapping */ + union { /* array of buffer info structs */ + struct igc_tx_buffer *tx_buffer_info; + struct igc_rx_buffer *rx_buffer_info; + }; + void *desc; /* descriptor ring memory */ + unsigned long flags; /* ring specific flags */ + void __iomem *tail; /* pointer to ring tail register */ + dma_addr_t dma; /* phys address of the ring */ + unsigned int size; /* length of desc. ring in bytes */ + + u16 count; /* number of desc. in the ring */ + u8 queue_index; /* logical index of the ring*/ + u8 reg_idx; /* physical index of the ring */ + bool launchtime_enable; /* true if LaunchTime is enabled */ + + u32 start_time; + u32 end_time; + + /* everything past this point are written often */ + u16 next_to_clean; + u16 next_to_use; + u16 next_to_alloc; + + union { + /* TX */ + struct { + struct igc_tx_queue_stats tx_stats; + struct u64_stats_sync tx_syncp; + struct u64_stats_sync tx_syncp2; + }; + /* RX */ + struct { + struct igc_rx_queue_stats rx_stats; + struct igc_rx_packet_stats pkt_stats; + struct u64_stats_sync rx_syncp; + struct sk_buff *skb; + }; + }; +} ____cacheline_internodealigned_in_smp; + +/* Board specific private data structure */ +struct igc_adapter { + struct net_device *netdev; + + unsigned long state; + unsigned int flags; + unsigned int num_q_vectors; + + struct msix_entry *msix_entries; + + /* TX */ + u16 tx_work_limit; + u32 tx_timeout_count; + int num_tx_queues; + struct igc_ring *tx_ring[IGC_MAX_TX_QUEUES]; + + /* RX */ + int num_rx_queues; + struct igc_ring *rx_ring[IGC_MAX_RX_QUEUES]; + + struct timer_list watchdog_timer; + struct timer_list dma_err_timer; + struct timer_list phy_info_timer; + + u32 wol; + u32 en_mng_pt; + u16 link_speed; + u16 link_duplex; + + u8 port_num; + + u8 __iomem *io_addr; + /* Interrupt Throttle Rate */ + u32 rx_itr_setting; + u32 tx_itr_setting; + + struct work_struct reset_task; + struct work_struct watchdog_task; + struct work_struct dma_err_task; + bool fc_autoneg; + + u8 tx_timeout_factor; + + int msg_enable; + u32 max_frame_size; + u32 min_frame_size; + + ktime_t base_time; + ktime_t cycle_time; + + /* OS defined structs */ + struct pci_dev *pdev; + /* lock for statistics */ + spinlock_t stats64_lock; + struct rtnl_link_stats64 stats64; + + /* structs defined in igc_hw.h */ + struct igc_hw hw; + struct igc_hw_stats stats; + + struct igc_q_vector *q_vector[MAX_Q_VECTORS]; + u32 eims_enable_mask; + u32 eims_other; + + u16 tx_ring_count; + u16 rx_ring_count; + + u32 tx_hwtstamp_timeouts; + u32 tx_hwtstamp_skipped; + u32 rx_hwtstamp_cleared; + + u32 rss_queues; + u32 rss_indir_tbl_init; + + /* Any access to elements in nfc_rule_list is protected by the + * nfc_rule_lock. + */ + struct mutex nfc_rule_lock; + struct list_head nfc_rule_list; + unsigned int nfc_rule_count; + + u8 rss_indir_tbl[IGC_RETA_SIZE]; + + unsigned long link_check_timeout; + struct igc_info ei; + + u32 test_icr; + + struct ptp_clock *ptp_clock; + struct ptp_clock_info ptp_caps; + struct work_struct ptp_tx_work; + struct sk_buff *ptp_tx_skb; + struct hwtstamp_config tstamp_config; + unsigned long ptp_tx_start; + unsigned long last_rx_ptp_check; + unsigned long last_rx_timestamp; + unsigned int ptp_flags; + /* System time value lock */ + spinlock_t tmreg_lock; + struct cyclecounter cc; + struct timecounter tc; +}; void igc_up(struct igc_adapter *adapter); void igc_down(struct igc_adapter *adapter); +int igc_open(struct net_device *netdev); +int igc_close(struct net_device *netdev); int igc_setup_tx_resources(struct igc_ring *ring); int igc_setup_rx_resources(struct igc_ring *ring); void igc_free_tx_resources(struct igc_ring *ring); @@ -36,10 +232,6 @@ void igc_write_rss_indir_tbl(struct igc_adapter *adapter); bool igc_has_link(struct igc_adapter *adapter); void igc_reset(struct igc_adapter *adapter); int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx); -int igc_add_mac_steering_filter(struct igc_adapter *adapter, - const u8 *addr, u8 queue, u8 flags); -int igc_del_mac_steering_filter(struct igc_adapter *adapter, - const u8 *addr, u8 queue, u8 flags); void igc_update_stats(struct igc_adapter *adapter); /* igc_dump declarations */ @@ -50,14 +242,10 @@ extern char igc_driver_name[]; extern char igc_driver_version[]; #define IGC_REGS_LEN 740 -#define IGC_RETA_SIZE 128 /* flags controlling PTP/1588 function */ #define IGC_PTP_ENABLED BIT(0) -/* Interrupt defines */ -#define IGC_START_ITR 648 /* ~6000 ints/sec */ - /* Flags definitions */ #define IGC_FLAG_HAS_MSI BIT(0) #define IGC_FLAG_QUEUE_PAIRS BIT(3) @@ -70,6 +258,7 @@ extern char igc_driver_version[]; #define IGC_FLAG_HAS_MSIX BIT(13) #define IGC_FLAG_VLAN_PROMISC BIT(15) #define IGC_FLAG_RX_LEGACY BIT(16) +#define IGC_FLAG_TSN_QBV_ENABLED BIT(17) #define IGC_FLAG_RSS_FIELD_IPV4_UDP BIT(6) #define IGC_FLAG_RSS_FIELD_IPV6_UDP BIT(7) @@ -78,6 +267,7 @@ extern char igc_driver_version[]; #define IGC_MRQC_RSS_FIELD_IPV4_UDP 0x00400000 #define IGC_MRQC_RSS_FIELD_IPV6_UDP 0x00800000 +/* Interrupt defines */ #define IGC_START_ITR 648 /* ~6000 ints/sec */ #define IGC_4K_ITR 980 #define IGC_20K_ITR 196 @@ -99,13 +289,6 @@ extern char igc_driver_version[]; #define IGC_MIN_RXD 80 #define IGC_MAX_RXD 4096 -/* Transmit and receive queues */ -#define IGC_MAX_RX_QUEUES 4 -#define IGC_MAX_TX_QUEUES 4 - -#define MAX_Q_VECTORS 8 -#define MAX_STD_JUMBO_FRAME_SIZE 9216 - /* Supported Rx Buffer Sizes */ #define IGC_RXBUFFER_256 256 #define IGC_RXBUFFER_2048 2048 @@ -232,83 +415,6 @@ struct igc_rx_buffer { __u16 pagecnt_bias; }; -struct igc_tx_queue_stats { - u64 packets; - u64 bytes; - u64 restart_queue; - u64 restart_queue2; -}; - -struct igc_rx_queue_stats { - u64 packets; - u64 bytes; - u64 drops; - u64 csum_err; - u64 alloc_failed; -}; - -struct igc_rx_packet_stats { - u64 ipv4_packets; /* IPv4 headers processed */ - u64 ipv4e_packets; /* IPv4E headers with extensions processed */ - u64 ipv6_packets; /* IPv6 headers processed */ - u64 ipv6e_packets; /* IPv6E headers with extensions processed */ - u64 tcp_packets; /* TCP headers processed */ - u64 udp_packets; /* UDP headers processed */ - u64 sctp_packets; /* SCTP headers processed */ - u64 nfs_packets; /* NFS headers processe */ - u64 other_packets; -}; - -struct igc_ring_container { - struct igc_ring *ring; /* pointer to linked list of rings */ - unsigned int total_bytes; /* total bytes processed this int */ - unsigned int total_packets; /* total packets processed this int */ - u16 work_limit; /* total work allowed per interrupt */ - u8 count; /* total number of rings in vector */ - u8 itr; /* current ITR setting for ring */ -}; - -struct igc_ring { - struct igc_q_vector *q_vector; /* backlink to q_vector */ - struct net_device *netdev; /* back pointer to net_device */ - struct device *dev; /* device for dma mapping */ - union { /* array of buffer info structs */ - struct igc_tx_buffer *tx_buffer_info; - struct igc_rx_buffer *rx_buffer_info; - }; - void *desc; /* descriptor ring memory */ - unsigned long flags; /* ring specific flags */ - void __iomem *tail; /* pointer to ring tail register */ - dma_addr_t dma; /* phys address of the ring */ - unsigned int size; /* length of desc. ring in bytes */ - - u16 count; /* number of desc. in the ring */ - u8 queue_index; /* logical index of the ring*/ - u8 reg_idx; /* physical index of the ring */ - bool launchtime_enable; /* true if LaunchTime is enabled */ - - /* everything past this point are written often */ - u16 next_to_clean; - u16 next_to_use; - u16 next_to_alloc; - - union { - /* TX */ - struct { - struct igc_tx_queue_stats tx_stats; - struct u64_stats_sync tx_syncp; - struct u64_stats_sync tx_syncp2; - }; - /* RX */ - struct { - struct igc_rx_queue_stats rx_stats; - struct igc_rx_packet_stats pkt_stats; - struct u64_stats_sync rx_syncp; - struct sk_buff *skb; - }; - }; -} ____cacheline_internodealigned_in_smp; - struct igc_q_vector { struct igc_adapter *adapter; /* backlink */ void __iomem *itr_register; @@ -329,8 +435,6 @@ struct igc_q_vector { struct igc_ring ring[] ____cacheline_internodealigned_in_smp; }; -#define MAX_ETYPE_FILTER (4 - 1) - enum igc_filter_match_flags { IGC_FILTER_FLAG_ETHER_TYPE = 0x1, IGC_FILTER_FLAG_VLAN_TCI = 0x2, @@ -338,143 +442,25 @@ enum igc_filter_match_flags { IGC_FILTER_FLAG_DST_MAC_ADDR = 0x8, }; -/* RX network flow classification data structure */ -struct igc_nfc_input { - /* Byte layout in order, all values with MSB first: - * match_flags - 1 byte - * etype - 2 bytes - * vlan_tci - 2 bytes - */ +struct igc_nfc_filter { u8 match_flags; - __be16 etype; - __be16 vlan_tci; + u16 etype; + u16 vlan_tci; u8 src_addr[ETH_ALEN]; u8 dst_addr[ETH_ALEN]; }; -struct igc_nfc_filter { - struct hlist_node nfc_node; - struct igc_nfc_input filter; - unsigned long cookie; - u16 etype_reg_index; - u16 sw_idx; +struct igc_nfc_rule { + struct list_head list; + struct igc_nfc_filter filter; + u32 location; u16 action; }; -struct igc_mac_addr { - u8 addr[ETH_ALEN]; - u8 queue; - u8 state; /* bitmask */ -}; - -#define IGC_MAC_STATE_DEFAULT 0x1 -#define IGC_MAC_STATE_IN_USE 0x2 -#define IGC_MAC_STATE_SRC_ADDR 0x4 -#define IGC_MAC_STATE_QUEUE_STEERING 0x8 - -#define IGC_MAX_RXNFC_FILTERS 16 - -/* Board specific private data structure */ -struct igc_adapter { - struct net_device *netdev; - - unsigned long state; - unsigned int flags; - unsigned int num_q_vectors; - - struct msix_entry *msix_entries; - - /* TX */ - u16 tx_work_limit; - u32 tx_timeout_count; - int num_tx_queues; - struct igc_ring *tx_ring[IGC_MAX_TX_QUEUES]; - - /* RX */ - int num_rx_queues; - struct igc_ring *rx_ring[IGC_MAX_RX_QUEUES]; - - struct timer_list watchdog_timer; - struct timer_list dma_err_timer; - struct timer_list phy_info_timer; - - u32 wol; - u32 en_mng_pt; - u16 link_speed; - u16 link_duplex; - - u8 port_num; - - u8 __iomem *io_addr; - /* Interrupt Throttle Rate */ - u32 rx_itr_setting; - u32 tx_itr_setting; - - struct work_struct reset_task; - struct work_struct watchdog_task; - struct work_struct dma_err_task; - bool fc_autoneg; - - u8 tx_timeout_factor; - - int msg_enable; - u32 max_frame_size; - u32 min_frame_size; - - /* OS defined structs */ - struct pci_dev *pdev; - /* lock for statistics */ - spinlock_t stats64_lock; - struct rtnl_link_stats64 stats64; - - /* structs defined in igc_hw.h */ - struct igc_hw hw; - struct igc_hw_stats stats; - - struct igc_q_vector *q_vector[MAX_Q_VECTORS]; - u32 eims_enable_mask; - u32 eims_other; - - u16 tx_ring_count; - u16 rx_ring_count; - - u32 tx_hwtstamp_timeouts; - u32 tx_hwtstamp_skipped; - u32 rx_hwtstamp_cleared; - - u32 rss_queues; - u32 rss_indir_tbl_init; - - /* RX network flow classification support */ - struct hlist_head nfc_filter_list; - struct hlist_head cls_flower_list; - unsigned int nfc_filter_count; - - /* lock for RX network flow classification filter */ - spinlock_t nfc_lock; - bool etype_bitmap[MAX_ETYPE_FILTER]; - - struct igc_mac_addr *mac_table; - - u8 rss_indir_tbl[IGC_RETA_SIZE]; - - unsigned long link_check_timeout; - struct igc_info ei; - - struct ptp_clock *ptp_clock; - struct ptp_clock_info ptp_caps; - struct work_struct ptp_tx_work; - struct sk_buff *ptp_tx_skb; - struct hwtstamp_config tstamp_config; - unsigned long ptp_tx_start; - unsigned long last_rx_ptp_check; - unsigned long last_rx_timestamp; - unsigned int ptp_flags; - /* System time value lock */ - spinlock_t tmreg_lock; - struct cyclecounter cc; - struct timecounter tc; -}; +/* IGC supports a total of 32 NFC rules: 16 MAC address based,, 8 VLAN priority + * based, and 8 ethertype based. + */ +#define IGC_MAX_RXNFC_RULES 32 /* igc_desc_unused - calculate if we have unused descriptors */ static inline u16 igc_desc_unused(const struct igc_ring *ring) @@ -550,12 +536,11 @@ static inline s32 igc_read_phy_reg(struct igc_hw *hw, u32 offset, u16 *data) return 0; } -/* forward declaration */ void igc_reinit_locked(struct igc_adapter *); -int igc_add_filter(struct igc_adapter *adapter, - struct igc_nfc_filter *input); -int igc_erase_filter(struct igc_adapter *adapter, - struct igc_nfc_filter *input); +struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter, + u32 location); +int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule); +void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule); void igc_ptp_init(struct igc_adapter *adapter); void igc_ptp_reset(struct igc_adapter *adapter); diff --git a/drivers/net/ethernet/intel/igc/igc_base.c b/drivers/net/ethernet/intel/igc/igc_base.c index 5a506440560a..cc5a6cf531c7 100644 --- a/drivers/net/ethernet/intel/igc/igc_base.c +++ b/drivers/net/ethernet/intel/igc/igc_base.c @@ -26,7 +26,7 @@ static s32 igc_reset_hw_base(struct igc_hw *hw) */ ret_val = igc_disable_pcie_master(hw); if (ret_val) - hw_dbg("PCI-E Master disable polling has failed.\n"); + hw_dbg("PCI-E Master disable polling has failed\n"); hw_dbg("Masking off all interrupts\n"); wr32(IGC_IMC, 0xffffffff); @@ -177,7 +177,7 @@ static s32 igc_init_phy_params_base(struct igc_hw *hw) */ ret_val = hw->phy.ops.reset(hw); if (ret_val) { - hw_dbg("Error resetting the PHY.\n"); + hw_dbg("Error resetting the PHY\n"); goto out; } @@ -212,6 +212,9 @@ static s32 igc_get_invariants_base(struct igc_hw *hw) case IGC_DEV_ID_I225_I: case IGC_DEV_ID_I220_V: case IGC_DEV_ID_I225_K: + case IGC_DEV_ID_I225_K2: + case IGC_DEV_ID_I225_LMVP: + case IGC_DEV_ID_I225_IT: case IGC_DEV_ID_I225_BLANK_NVM: mac->type = igc_i225; break; @@ -364,7 +367,7 @@ void igc_rx_fifo_flush_base(struct igc_hw *hw) } if (ms_wait == 10) - pr_debug("Queue disable timed out after 10ms\n"); + hw_dbg("Queue disable timed out after 10ms\n"); /* Clear RLPML, RCTL.SBP, RFCTL.LEF, and set RCTL.LPE so that all * incoming packets are rejected. Set enable and wait 2ms so that diff --git a/drivers/net/ethernet/intel/igc/igc_defines.h b/drivers/net/ethernet/intel/igc/igc_defines.h index 4ddccccf42cc..186deb1d9375 100644 --- a/drivers/net/ethernet/intel/igc/igc_defines.h +++ b/drivers/net/ethernet/intel/igc/igc_defines.h @@ -44,13 +44,9 @@ /* Wake Up Packet Memory stores the first 128 bytes of the wake up packet */ #define IGC_WUPM_BYTES 128 -/* Physical Func Reset Done Indication */ -#define IGC_CTRL_EXT_LINK_MODE_MASK 0x00C00000 - /* Loop limit on how long we wait for auto-negotiation to complete */ #define COPPER_LINK_UP_LIMIT 10 #define PHY_AUTO_NEG_LIMIT 45 -#define PHY_FORCE_LIMIT 20 /* Number of 100 microseconds we wait for PCI Express master disable */ #define MASTER_DISABLE_TIMEOUT 800 @@ -66,8 +62,14 @@ * (RAR[15]) for our directed address used by controllers with * manageability enabled, allowing us room for 15 multicast addresses. */ +#define IGC_RAH_RAH_MASK 0x0000FFFF +#define IGC_RAH_ASEL_MASK 0x00030000 +#define IGC_RAH_ASEL_SRC_ADDR BIT(16) +#define IGC_RAH_QSEL_MASK 0x000C0000 +#define IGC_RAH_QSEL_SHIFT 18 +#define IGC_RAH_QSEL_ENABLE BIT(28) #define IGC_RAH_AV 0x80000000 /* Receive descriptor valid */ -#define IGC_RAH_POOL_1 0x00040000 + #define IGC_RAL_MAC_ADDR_LEN 4 #define IGC_RAH_MAC_ADDR_LEN 2 @@ -94,8 +96,6 @@ #define IGC_CTRL_RFCE 0x08000000 /* Receive Flow Control enable */ #define IGC_CTRL_TFCE 0x10000000 /* Transmit flow control enable */ -#define IGC_CONNSW_AUTOSENSE_EN 0x1 - /* As per the EAS the maximum supported size is 9.5KB (9728 bytes) */ #define MAX_JUMBO_FRAME_SIZE 0x2600 @@ -166,11 +166,6 @@ /* For checksumming, the sum of all words in the NVM should equal 0xBABA. */ #define NVM_SUM 0xBABA - -#define NVM_PBA_OFFSET_0 8 -#define NVM_PBA_OFFSET_1 9 -#define NVM_RESERVED_WORD 0xFFFF -#define NVM_PBA_PTR_GUARD 0xFAFA #define NVM_WORD_SIZE_BASE_SHIFT 6 /* Collision related configuration parameters */ @@ -252,7 +247,6 @@ /* Interrupt Cause Set */ #define IGC_ICS_LSC IGC_ICR_LSC /* Link Status Change */ #define IGC_ICS_RXDMT0 IGC_ICR_RXDMT0 /* rx desc min. threshold */ -#define IGC_ICS_DRSTA IGC_ICR_DRSTA /* Device Reset Aserted */ #define IGC_ICR_DOUTSYNC 0x10000000 /* NIC DMA out of sync */ #define IGC_EITR_CNT_IGNR 0x80000000 /* Don't reset counters on write */ @@ -271,21 +265,13 @@ #define IGC_TXD_POPTS_IXSM 0x01 /* Insert IP checksum */ #define IGC_TXD_POPTS_TXSM 0x02 /* Insert TCP/UDP checksum */ #define IGC_TXD_CMD_EOP 0x01000000 /* End of Packet */ -#define IGC_TXD_CMD_IFCS 0x02000000 /* Insert FCS (Ethernet CRC) */ #define IGC_TXD_CMD_IC 0x04000000 /* Insert Checksum */ -#define IGC_TXD_CMD_RS 0x08000000 /* Report Status */ -#define IGC_TXD_CMD_RPS 0x10000000 /* Report Packet Sent */ #define IGC_TXD_CMD_DEXT 0x20000000 /* Desc extension (0 = legacy) */ #define IGC_TXD_CMD_VLE 0x40000000 /* Add VLAN tag */ -#define IGC_TXD_CMD_IDE 0x80000000 /* Enable Tidv register */ #define IGC_TXD_STAT_DD 0x00000001 /* Descriptor Done */ -#define IGC_TXD_STAT_EC 0x00000002 /* Excess Collisions */ -#define IGC_TXD_STAT_LC 0x00000004 /* Late Collisions */ -#define IGC_TXD_STAT_TU 0x00000008 /* Transmit underrun */ #define IGC_TXD_CMD_TCP 0x01000000 /* TCP packet */ #define IGC_TXD_CMD_IP 0x02000000 /* IP packet */ #define IGC_TXD_CMD_TSE 0x04000000 /* TCP Seg enable */ -#define IGC_TXD_STAT_TC 0x00000004 /* Tx Underrun */ #define IGC_TXD_EXTCMD_TSTAMP 0x00000010 /* IEEE1588 Timestamp packet */ /* IPSec Encrypt Enable */ @@ -377,6 +363,11 @@ #define I225_TXPBSIZE_DEFAULT 0x04000014 /* TXPBSIZE default */ #define IGC_RXPBS_CFG_TS_EN 0x80000000 /* Timestamp in Rx buffer */ +#define IGC_TXPBSIZE_TSN 0x04145145 /* 5k bytes buffer for each queue */ + +#define IGC_DTXMXPKTSZ_TSN 0x19 /* 1600 bytes of max TX DMA packet size */ +#define IGC_DTXMXPKTSZ_DEFAULT 0x98 /* 9728-byte Jumbo frames */ + /* Time Sync Interrupt Causes */ #define IGC_TSICR_SYS_WRAP BIT(0) /* SYSTIM Wrap around. */ #define IGC_TSICR_TXTS BIT(1) /* Transmit Timestamp. */ @@ -387,9 +378,6 @@ #define IGC_TSICR_INTERRUPTS IGC_TSICR_TXTS -/* PTP Queue Filter */ -#define IGC_ETQF_1588 BIT(30) - #define IGC_FTQF_VF_BP 0x00008000 #define IGC_FTQF_1588_TIME_STAMP 0x08000000 #define IGC_FTQF_MASK 0xF0000000 @@ -431,6 +419,14 @@ #define IGC_TSYNCTXCTL_START_SYNC 0x80000000 /* initiate sync */ #define IGC_TSYNCTXCTL_TXSYNSIG 0x00000020 /* Sample TX tstamp in PHY sop */ +/* Transmit Scheduling */ +#define IGC_TQAVCTRL_TRANSMIT_MODE_TSN 0x00000001 +#define IGC_TQAVCTRL_ENHANCED_QAV 0x00000008 + +#define IGC_TXQCTL_QUEUE_MODE_LAUNCHT 0x00000001 +#define IGC_TXQCTL_STRICT_CYCLE 0x00000002 +#define IGC_TXQCTL_STRICT_END 0x00000004 + /* Receive Checksum Control */ #define IGC_RXCSUM_CRCOFL 0x00000800 /* CRC32 offload enable */ #define IGC_RXCSUM_PCSD 0x00002000 /* packet checksum disabled */ @@ -497,16 +493,15 @@ #define IGC_MDIC_READY 0x10000000 #define IGC_MDIC_INT_EN 0x20000000 #define IGC_MDIC_ERROR 0x40000000 -#define IGC_MDIC_DEST 0x80000000 #define IGC_N0_QUEUE -1 #define IGC_MAX_MAC_HDR_LEN 127 #define IGC_MAX_NETWORK_HDR_LEN 511 -#define IGC_VLAPQF_QUEUE_SEL(_n, q_idx) ((q_idx) << ((_n) * 4)) -#define IGC_VLAPQF_P_VALID(_n) (0x1 << (3 + (_n) * 4)) -#define IGC_VLAPQF_QUEUE_MASK 0x03 +#define IGC_VLANPQF_QSEL(_n, q_idx) ((q_idx) << ((_n) * 4)) +#define IGC_VLANPQF_VALID(_n) (0x1 << (3 + (_n) * 4)) +#define IGC_VLANPQF_QUEUE_MASK 0x03 #define IGC_ADVTXD_MACLEN_SHIFT 9 /* Adv ctxt desc mac len shift */ #define IGC_ADVTXD_TUCMD_IPV4 0x00000400 /* IP Packet Type:1=IPv4 */ diff --git a/drivers/net/ethernet/intel/igc/igc_diag.c b/drivers/net/ethernet/intel/igc/igc_diag.c new file mode 100644 index 000000000000..cc621970c0cd --- /dev/null +++ b/drivers/net/ethernet/intel/igc/igc_diag.c @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2020 Intel Corporation */ + +#include "igc.h" +#include "igc_diag.h" + +static struct igc_reg_test reg_test[] = { + { IGC_FCAL, 1, PATTERN_TEST, 0xFFFFFFFF, 0xFFFFFFFF }, + { IGC_FCAH, 1, PATTERN_TEST, 0x0000FFFF, 0xFFFFFFFF }, + { IGC_FCT, 1, PATTERN_TEST, 0x0000FFFF, 0xFFFFFFFF }, + { IGC_RDBAH(0), 4, PATTERN_TEST, 0xFFFFFFFF, 0xFFFFFFFF }, + { IGC_RDBAL(0), 4, PATTERN_TEST, 0xFFFFFF80, 0xFFFFFF80 }, + { IGC_RDLEN(0), 4, PATTERN_TEST, 0x000FFF80, 0x000FFFFF }, + { IGC_RDT(0), 4, PATTERN_TEST, 0x0000FFFF, 0x0000FFFF }, + { IGC_FCRTH, 1, PATTERN_TEST, 0x0003FFF0, 0x0003FFF0 }, + { IGC_FCTTV, 1, PATTERN_TEST, 0x0000FFFF, 0x0000FFFF }, + { IGC_TIPG, 1, PATTERN_TEST, 0x3FFFFFFF, 0x3FFFFFFF }, + { IGC_TDBAH(0), 4, PATTERN_TEST, 0xFFFFFFFF, 0xFFFFFFFF }, + { IGC_TDBAL(0), 4, PATTERN_TEST, 0xFFFFFF80, 0xFFFFFF80 }, + { IGC_TDLEN(0), 4, PATTERN_TEST, 0x000FFF80, 0x000FFFFF }, + { IGC_TDT(0), 4, PATTERN_TEST, 0x0000FFFF, 0x0000FFFF }, + { IGC_RCTL, 1, SET_READ_TEST, 0xFFFFFFFF, 0x00000000 }, + { IGC_RCTL, 1, SET_READ_TEST, 0x04CFB2FE, 0x003FFFFB }, + { IGC_RCTL, 1, SET_READ_TEST, 0x04CFB2FE, 0xFFFFFFFF }, + { IGC_TCTL, 1, SET_READ_TEST, 0xFFFFFFFF, 0x00000000 }, + { IGC_RA, 16, TABLE64_TEST_LO, + 0xFFFFFFFF, 0xFFFFFFFF }, + { IGC_RA, 16, TABLE64_TEST_HI, + 0x900FFFFF, 0xFFFFFFFF }, + { IGC_MTA, 128, TABLE32_TEST, + 0xFFFFFFFF, 0xFFFFFFFF }, + { 0, 0, 0, 0} +}; + +static bool reg_pattern_test(struct igc_adapter *adapter, u64 *data, int reg, + u32 mask, u32 write) +{ + struct igc_hw *hw = &adapter->hw; + u32 pat, val, before; + static const u32 test_pattern[] = { + 0x5A5A5A5A, 0xA5A5A5A5, 0x00000000, 0xFFFFFFFF + }; + + for (pat = 0; pat < ARRAY_SIZE(test_pattern); pat++) { + before = rd32(reg); + wr32(reg, test_pattern[pat] & write); + val = rd32(reg); + if (val != (test_pattern[pat] & write & mask)) { + netdev_err(adapter->netdev, + "pattern test reg %04X failed: got 0x%08X expected 0x%08X", + reg, val, test_pattern[pat] & write & mask); + *data = reg; + wr32(reg, before); + return false; + } + wr32(reg, before); + } + return true; +} + +static bool reg_set_and_check(struct igc_adapter *adapter, u64 *data, int reg, + u32 mask, u32 write) +{ + struct igc_hw *hw = &adapter->hw; + u32 val, before; + + before = rd32(reg); + wr32(reg, write & mask); + val = rd32(reg); + if ((write & mask) != (val & mask)) { + netdev_err(adapter->netdev, + "set/check reg %04X test failed: got 0x%08X expected 0x%08X", + reg, (val & mask), (write & mask)); + *data = reg; + wr32(reg, before); + return false; + } + wr32(reg, before); + return true; +} + +bool igc_reg_test(struct igc_adapter *adapter, u64 *data) +{ + struct igc_reg_test *test = reg_test; + struct igc_hw *hw = &adapter->hw; + u32 value, before, after; + u32 i, toggle, b = false; + + /* Because the status register is such a special case, + * we handle it separately from the rest of the register + * tests. Some bits are read-only, some toggle, and some + * are writeable. + */ + toggle = 0x6800D3; + before = rd32(IGC_STATUS); + value = before & toggle; + wr32(IGC_STATUS, toggle); + after = rd32(IGC_STATUS) & toggle; + if (value != after) { + netdev_err(adapter->netdev, + "failed STATUS register test got: 0x%08X expected: 0x%08X", + after, value); + *data = 1; + return false; + } + /* restore previous status */ + wr32(IGC_STATUS, before); + + /* Perform the remainder of the register test, looping through + * the test table until we either fail or reach the null entry. + */ + while (test->reg) { + for (i = 0; i < test->array_len; i++) { + switch (test->test_type) { + case PATTERN_TEST: + b = reg_pattern_test(adapter, data, + test->reg + (i * 0x40), + test->mask, + test->write); + break; + case SET_READ_TEST: + b = reg_set_and_check(adapter, data, + test->reg + (i * 0x40), + test->mask, + test->write); + break; + case TABLE64_TEST_LO: + b = reg_pattern_test(adapter, data, + test->reg + (i * 8), + test->mask, + test->write); + break; + case TABLE64_TEST_HI: + b = reg_pattern_test(adapter, data, + test->reg + 4 + (i * 8), + test->mask, + test->write); + break; + case TABLE32_TEST: + b = reg_pattern_test(adapter, data, + test->reg + (i * 4), + test->mask, + test->write); + break; + } + if (!b) + return false; + } + test++; + } + *data = 0; + return true; +} + +bool igc_eeprom_test(struct igc_adapter *adapter, u64 *data) +{ + struct igc_hw *hw = &adapter->hw; + + *data = 0; + + if (hw->nvm.ops.validate(hw) != IGC_SUCCESS) { + *data = 1; + return false; + } + + return true; +} + +bool igc_link_test(struct igc_adapter *adapter, u64 *data) +{ + bool link_up; + + *data = 0; + + /* add delay to give enough time for autonegotioation to finish */ + if (adapter->hw.mac.autoneg) + ssleep(5); + + link_up = igc_has_link(adapter); + if (!link_up) { + *data = 1; + return false; + } + + return true; +} diff --git a/drivers/net/ethernet/intel/igc/igc_diag.h b/drivers/net/ethernet/intel/igc/igc_diag.h new file mode 100644 index 000000000000..600658e33bec --- /dev/null +++ b/drivers/net/ethernet/intel/igc/igc_diag.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2020 Intel Corporation */ + +bool igc_reg_test(struct igc_adapter *adapter, u64 *data); +bool igc_eeprom_test(struct igc_adapter *adapter, u64 *data); +bool igc_link_test(struct igc_adapter *adapter, u64 *data); + +struct igc_reg_test { + u16 reg; + u8 array_len; + u8 test_type; + u32 mask; + u32 write; +}; + +/* In the hardware, registers are laid out either singly, in arrays + * spaced 0x40 bytes apart, or in contiguous tables. We assume + * most tests take place on arrays or single registers (handled + * as a single-element array) and special-case the tables. + * Table tests are always pattern tests. + * + * We also make provision for some required setup steps by specifying + * registers to be written without any read-back testing. + */ + +#define PATTERN_TEST 1 +#define SET_READ_TEST 2 +#define TABLE32_TEST 3 +#define TABLE64_TEST_LO 4 +#define TABLE64_TEST_HI 5 diff --git a/drivers/net/ethernet/intel/igc/igc_dump.c b/drivers/net/ethernet/intel/igc/igc_dump.c index 657ab50ae296..4b9ec7d0b727 100644 --- a/drivers/net/ethernet/intel/igc/igc_dump.c +++ b/drivers/net/ethernet/intel/igc/igc_dump.c @@ -35,10 +35,6 @@ static const struct igc_reg_info igc_reg_info_tbl[] = { {IGC_TDH(0), "TDH"}, {IGC_TDT(0), "TDT"}, {IGC_TXDCTL(0), "TXDCTL"}, - {IGC_TDFH, "TDFH"}, - {IGC_TDFT, "TDFT"}, - {IGC_TDFHS, "TDFHS"}, - {IGC_TDFPC, "TDFPC"}, /* List Terminator */ {} @@ -47,6 +43,7 @@ static const struct igc_reg_info igc_reg_info_tbl[] = { /* igc_regdump - register printout routine */ static void igc_regdump(struct igc_hw *hw, struct igc_reg_info *reginfo) { + struct net_device *dev = igc_get_hw_dev(hw); int n = 0; char rname[16]; u32 regs[8]; @@ -101,13 +98,14 @@ static void igc_regdump(struct igc_hw *hw, struct igc_reg_info *reginfo) regs[n] = rd32(IGC_TXDCTL(n)); break; default: - pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs)); + netdev_info(dev, "%-15s %08x\n", reginfo->name, + rd32(reginfo->ofs)); return; } snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]"); - pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1], - regs[2], regs[3]); + netdev_info(dev, "%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1], + regs[2], regs[3]); } /* igc_rings_dump - Tx-rings and Rx-rings */ @@ -125,39 +123,34 @@ void igc_rings_dump(struct igc_adapter *adapter) if (!netif_msg_hw(adapter)) return; - /* Print netdevice Info */ - if (netdev) { - dev_info(&adapter->pdev->dev, "Net device Info\n"); - pr_info("Device Name state trans_start\n"); - pr_info("%-15s %016lX %016lX\n", netdev->name, - netdev->state, dev_trans_start(netdev)); - } + netdev_info(netdev, "Device info: state %016lX trans_start %016lX\n", + netdev->state, dev_trans_start(netdev)); /* Print TX Ring Summary */ - if (!netdev || !netif_running(netdev)) + if (!netif_running(netdev)) goto exit; - dev_info(&adapter->pdev->dev, "TX Rings Summary\n"); - pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n"); + netdev_info(netdev, "TX Rings Summary\n"); + netdev_info(netdev, "Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n"); for (n = 0; n < adapter->num_tx_queues; n++) { struct igc_tx_buffer *buffer_info; tx_ring = adapter->tx_ring[n]; buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean]; - pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n", - n, tx_ring->next_to_use, tx_ring->next_to_clean, - (u64)dma_unmap_addr(buffer_info, dma), - dma_unmap_len(buffer_info, len), - buffer_info->next_to_watch, - (u64)buffer_info->time_stamp); + netdev_info(netdev, "%5d %5X %5X %016llX %04X %p %016llX\n", + n, tx_ring->next_to_use, tx_ring->next_to_clean, + (u64)dma_unmap_addr(buffer_info, dma), + dma_unmap_len(buffer_info, len), + buffer_info->next_to_watch, + (u64)buffer_info->time_stamp); } /* Print TX Rings */ if (!netif_msg_tx_done(adapter)) goto rx_ring_summary; - dev_info(&adapter->pdev->dev, "TX Rings Dump\n"); + netdev_info(netdev, "TX Rings Dump\n"); /* Transmit Descriptor Formats * @@ -172,10 +165,11 @@ void igc_rings_dump(struct igc_adapter *adapter) for (n = 0; n < adapter->num_tx_queues; n++) { tx_ring = adapter->tx_ring[n]; - pr_info("------------------------------------\n"); - pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index); - pr_info("------------------------------------\n"); - pr_info("T [desc] [address 63:0 ] [PlPOCIStDDM Ln] [bi->dma ] leng ntw timestamp bi->skb\n"); + netdev_info(netdev, "------------------------------------\n"); + netdev_info(netdev, "TX QUEUE INDEX = %d\n", + tx_ring->queue_index); + netdev_info(netdev, "------------------------------------\n"); + netdev_info(netdev, "T [desc] [address 63:0 ] [PlPOCIStDDM Ln] [bi->dma ] leng ntw timestamp bi->skb\n"); for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) { const char *next_desc; @@ -194,14 +188,14 @@ void igc_rings_dump(struct igc_adapter *adapter) else next_desc = ""; - pr_info("T [0x%03X] %016llX %016llX %016llX %04X %p %016llX %p%s\n", - i, le64_to_cpu(u0->a), - le64_to_cpu(u0->b), - (u64)dma_unmap_addr(buffer_info, dma), - dma_unmap_len(buffer_info, len), - buffer_info->next_to_watch, - (u64)buffer_info->time_stamp, - buffer_info->skb, next_desc); + netdev_info(netdev, "T [0x%03X] %016llX %016llX %016llX %04X %p %016llX %p%s\n", + i, le64_to_cpu(u0->a), + le64_to_cpu(u0->b), + (u64)dma_unmap_addr(buffer_info, dma), + dma_unmap_len(buffer_info, len), + buffer_info->next_to_watch, + (u64)buffer_info->time_stamp, + buffer_info->skb, next_desc); if (netif_msg_pktdata(adapter) && buffer_info->skb) print_hex_dump(KERN_INFO, "", @@ -214,19 +208,19 @@ void igc_rings_dump(struct igc_adapter *adapter) /* Print RX Rings Summary */ rx_ring_summary: - dev_info(&adapter->pdev->dev, "RX Rings Summary\n"); - pr_info("Queue [NTU] [NTC]\n"); + netdev_info(netdev, "RX Rings Summary\n"); + netdev_info(netdev, "Queue [NTU] [NTC]\n"); for (n = 0; n < adapter->num_rx_queues; n++) { rx_ring = adapter->rx_ring[n]; - pr_info(" %5d %5X %5X\n", - n, rx_ring->next_to_use, rx_ring->next_to_clean); + netdev_info(netdev, "%5d %5X %5X\n", n, rx_ring->next_to_use, + rx_ring->next_to_clean); } /* Print RX Rings */ if (!netif_msg_rx_status(adapter)) goto exit; - dev_info(&adapter->pdev->dev, "RX Rings Dump\n"); + netdev_info(netdev, "RX Rings Dump\n"); /* Advanced Receive Descriptor (Read) Format * 63 1 0 @@ -251,11 +245,12 @@ rx_ring_summary: for (n = 0; n < adapter->num_rx_queues; n++) { rx_ring = adapter->rx_ring[n]; - pr_info("------------------------------------\n"); - pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index); - pr_info("------------------------------------\n"); - pr_info("R [desc] [ PktBuf A0] [ HeadBuf DD] [bi->dma ] [bi->skb] <-- Adv Rx Read format\n"); - pr_info("RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] ---------------- [bi->skb] <-- Adv Rx Write-Back format\n"); + netdev_info(netdev, "------------------------------------\n"); + netdev_info(netdev, "RX QUEUE INDEX = %d\n", + rx_ring->queue_index); + netdev_info(netdev, "------------------------------------\n"); + netdev_info(netdev, "R [desc] [ PktBuf A0] [ HeadBuf DD] [bi->dma ] [bi->skb] <-- Adv Rx Read format\n"); + netdev_info(netdev, "RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] ---------------- [bi->skb] <-- Adv Rx Write-Back format\n"); for (i = 0; i < rx_ring->count; i++) { const char *next_desc; @@ -275,18 +270,18 @@ rx_ring_summary: if (staterr & IGC_RXD_STAT_DD) { /* Descriptor Done */ - pr_info("%s[0x%03X] %016llX %016llX ---------------- %s\n", - "RWB", i, - le64_to_cpu(u0->a), - le64_to_cpu(u0->b), - next_desc); + netdev_info(netdev, "%s[0x%03X] %016llX %016llX ---------------- %s\n", + "RWB", i, + le64_to_cpu(u0->a), + le64_to_cpu(u0->b), + next_desc); } else { - pr_info("%s[0x%03X] %016llX %016llX %016llX %s\n", - "R ", i, - le64_to_cpu(u0->a), - le64_to_cpu(u0->b), - (u64)buffer_info->dma, - next_desc); + netdev_info(netdev, "%s[0x%03X] %016llX %016llX %016llX %s\n", + "R ", i, + le64_to_cpu(u0->a), + le64_to_cpu(u0->b), + (u64)buffer_info->dma, + next_desc); if (netif_msg_pktdata(adapter) && buffer_info->dma && buffer_info->page) { @@ -314,8 +309,8 @@ void igc_regs_dump(struct igc_adapter *adapter) struct igc_reg_info *reginfo; /* Print Registers */ - dev_info(&adapter->pdev->dev, "Register Dump\n"); - pr_info(" Register Name Value\n"); + netdev_info(adapter->netdev, "Register Dump\n"); + netdev_info(adapter->netdev, "Register Name Value\n"); for (reginfo = (struct igc_reg_info *)igc_reg_info_tbl; reginfo->name; reginfo++) { igc_regdump(hw, reginfo); diff --git a/drivers/net/ethernet/intel/igc/igc_ethtool.c b/drivers/net/ethernet/intel/igc/igc_ethtool.c index f530fc29b074..a938ec8db681 100644 --- a/drivers/net/ethernet/intel/igc/igc_ethtool.c +++ b/drivers/net/ethernet/intel/igc/igc_ethtool.c @@ -6,6 +6,7 @@ #include <linux/pm_runtime.h> #include "igc.h" +#include "igc_diag.h" /* forward declaration */ struct igc_stats { @@ -123,8 +124,8 @@ static const char igc_priv_flags_strings[][ETH_GSTRING_LEN] = { #define IGC_PRIV_FLAGS_STR_LEN ARRAY_SIZE(igc_priv_flags_strings) -static void igc_get_drvinfo(struct net_device *netdev, - struct ethtool_drvinfo *drvinfo) +static void igc_ethtool_get_drvinfo(struct net_device *netdev, + struct ethtool_drvinfo *drvinfo) { struct igc_adapter *adapter = netdev_priv(netdev); @@ -138,13 +139,13 @@ static void igc_get_drvinfo(struct net_device *netdev, drvinfo->n_priv_flags = IGC_PRIV_FLAGS_STR_LEN; } -static int igc_get_regs_len(struct net_device *netdev) +static int igc_ethtool_get_regs_len(struct net_device *netdev) { return IGC_REGS_LEN * sizeof(u32); } -static void igc_get_regs(struct net_device *netdev, - struct ethtool_regs *regs, void *p) +static void igc_ethtool_get_regs(struct net_device *netdev, + struct ethtool_regs *regs, void *p) { struct igc_adapter *adapter = netdev_priv(netdev); struct igc_hw *hw = &adapter->hw; @@ -153,7 +154,7 @@ static void igc_get_regs(struct net_device *netdev, memset(p, 0, IGC_REGS_LEN * sizeof(u32)); - regs->version = (1u << 24) | (hw->revision_id << 16) | hw->device_id; + regs->version = (2u << 24) | (hw->revision_id << 16) | hw->device_id; /* General Registers */ regs_buff[0] = rd32(IGC_CTRL); @@ -306,9 +307,24 @@ static void igc_get_regs(struct net_device *netdev, regs_buff[164 + i] = rd32(IGC_TDT(i)); for (i = 0; i < 4; i++) regs_buff[168 + i] = rd32(IGC_TXDCTL(i)); + + /* XXX: Due to a bug few lines above, RAL and RAH registers are + * overwritten. To preserve the ABI, we write these registers again in + * regs_buff. + */ + for (i = 0; i < 16; i++) + regs_buff[172 + i] = rd32(IGC_RAL(i)); + for (i = 0; i < 16; i++) + regs_buff[188 + i] = rd32(IGC_RAH(i)); + + regs_buff[204] = rd32(IGC_VLANPQF); + + for (i = 0; i < 8; i++) + regs_buff[205 + i] = rd32(IGC_ETQF(i)); } -static void igc_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) +static void igc_ethtool_get_wol(struct net_device *netdev, + struct ethtool_wolinfo *wol) { struct igc_adapter *adapter = netdev_priv(netdev); @@ -339,7 +355,8 @@ static void igc_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) wol->wolopts |= WAKE_PHY; } -static int igc_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) +static int igc_ethtool_set_wol(struct net_device *netdev, + struct ethtool_wolinfo *wol) { struct igc_adapter *adapter = netdev_priv(netdev); @@ -367,21 +384,21 @@ static int igc_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) return 0; } -static u32 igc_get_msglevel(struct net_device *netdev) +static u32 igc_ethtool_get_msglevel(struct net_device *netdev) { struct igc_adapter *adapter = netdev_priv(netdev); return adapter->msg_enable; } -static void igc_set_msglevel(struct net_device *netdev, u32 data) +static void igc_ethtool_set_msglevel(struct net_device *netdev, u32 data) { struct igc_adapter *adapter = netdev_priv(netdev); adapter->msg_enable = data; } -static int igc_nway_reset(struct net_device *netdev) +static int igc_ethtool_nway_reset(struct net_device *netdev) { struct igc_adapter *adapter = netdev_priv(netdev); @@ -390,7 +407,7 @@ static int igc_nway_reset(struct net_device *netdev) return 0; } -static u32 igc_get_link(struct net_device *netdev) +static u32 igc_ethtool_get_link(struct net_device *netdev) { struct igc_adapter *adapter = netdev_priv(netdev); struct igc_mac_info *mac = &adapter->hw.mac; @@ -407,15 +424,15 @@ static u32 igc_get_link(struct net_device *netdev) return igc_has_link(adapter); } -static int igc_get_eeprom_len(struct net_device *netdev) +static int igc_ethtool_get_eeprom_len(struct net_device *netdev) { struct igc_adapter *adapter = netdev_priv(netdev); return adapter->hw.nvm.word_size * 2; } -static int igc_get_eeprom(struct net_device *netdev, - struct ethtool_eeprom *eeprom, u8 *bytes) +static int igc_ethtool_get_eeprom(struct net_device *netdev, + struct ethtool_eeprom *eeprom, u8 *bytes) { struct igc_adapter *adapter = netdev_priv(netdev); struct igc_hw *hw = &adapter->hw; @@ -461,8 +478,8 @@ static int igc_get_eeprom(struct net_device *netdev, return ret_val; } -static int igc_set_eeprom(struct net_device *netdev, - struct ethtool_eeprom *eeprom, u8 *bytes) +static int igc_ethtool_set_eeprom(struct net_device *netdev, + struct ethtool_eeprom *eeprom, u8 *bytes) { struct igc_adapter *adapter = netdev_priv(netdev); struct igc_hw *hw = &adapter->hw; @@ -529,8 +546,8 @@ static int igc_set_eeprom(struct net_device *netdev, return ret_val; } -static void igc_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) +static void igc_ethtool_get_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ring) { struct igc_adapter *adapter = netdev_priv(netdev); @@ -540,8 +557,8 @@ static void igc_get_ringparam(struct net_device *netdev, ring->tx_pending = adapter->tx_ring_count; } -static int igc_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) +static int igc_ethtool_set_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ring) { struct igc_adapter *adapter = netdev_priv(netdev); struct igc_ring *temp_ring; @@ -655,8 +672,8 @@ clear_reset: return err; } -static void igc_get_pauseparam(struct net_device *netdev, - struct ethtool_pauseparam *pause) +static void igc_ethtool_get_pauseparam(struct net_device *netdev, + struct ethtool_pauseparam *pause) { struct igc_adapter *adapter = netdev_priv(netdev); struct igc_hw *hw = &adapter->hw; @@ -674,8 +691,8 @@ static void igc_get_pauseparam(struct net_device *netdev, } } -static int igc_set_pauseparam(struct net_device *netdev, - struct ethtool_pauseparam *pause) +static int igc_ethtool_set_pauseparam(struct net_device *netdev, + struct ethtool_pauseparam *pause) { struct igc_adapter *adapter = netdev_priv(netdev); struct igc_hw *hw = &adapter->hw; @@ -714,7 +731,8 @@ static int igc_set_pauseparam(struct net_device *netdev, return retval; } -static void igc_get_strings(struct net_device *netdev, u32 stringset, u8 *data) +static void igc_ethtool_get_strings(struct net_device *netdev, u32 stringset, + u8 *data) { struct igc_adapter *adapter = netdev_priv(netdev); u8 *p = data; @@ -765,7 +783,7 @@ static void igc_get_strings(struct net_device *netdev, u32 stringset, u8 *data) } } -static int igc_get_sset_count(struct net_device *netdev, int sset) +static int igc_ethtool_get_sset_count(struct net_device *netdev, int sset) { switch (sset) { case ETH_SS_STATS: @@ -779,7 +797,7 @@ static int igc_get_sset_count(struct net_device *netdev, int sset) } } -static void igc_get_ethtool_stats(struct net_device *netdev, +static void igc_ethtool_get_stats(struct net_device *netdev, struct ethtool_stats *stats, u64 *data) { struct igc_adapter *adapter = netdev_priv(netdev); @@ -835,8 +853,8 @@ static void igc_get_ethtool_stats(struct net_device *netdev, spin_unlock(&adapter->stats64_lock); } -static int igc_get_coalesce(struct net_device *netdev, - struct ethtool_coalesce *ec) +static int igc_ethtool_get_coalesce(struct net_device *netdev, + struct ethtool_coalesce *ec) { struct igc_adapter *adapter = netdev_priv(netdev); @@ -855,8 +873,8 @@ static int igc_get_coalesce(struct net_device *netdev, return 0; } -static int igc_set_coalesce(struct net_device *netdev, - struct ethtool_coalesce *ec) +static int igc_ethtool_set_coalesce(struct net_device *netdev, + struct ethtool_coalesce *ec) { struct igc_adapter *adapter = netdev_priv(netdev); int i; @@ -913,81 +931,83 @@ static int igc_set_coalesce(struct net_device *netdev, } #define ETHER_TYPE_FULL_MASK ((__force __be16)~0) -static int igc_get_ethtool_nfc_entry(struct igc_adapter *adapter, - struct ethtool_rxnfc *cmd) +static int igc_ethtool_get_nfc_rule(struct igc_adapter *adapter, + struct ethtool_rxnfc *cmd) { struct ethtool_rx_flow_spec *fsp = &cmd->fs; - struct igc_nfc_filter *rule = NULL; + struct igc_nfc_rule *rule = NULL; - /* report total rule count */ - cmd->data = IGC_MAX_RXNFC_FILTERS; + cmd->data = IGC_MAX_RXNFC_RULES; - hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) { - if (fsp->location <= rule->sw_idx) - break; + mutex_lock(&adapter->nfc_rule_lock); + + rule = igc_get_nfc_rule(adapter, fsp->location); + if (!rule) + goto out; + + fsp->flow_type = ETHER_FLOW; + fsp->ring_cookie = rule->action; + + if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { + fsp->h_u.ether_spec.h_proto = htons(rule->filter.etype); + fsp->m_u.ether_spec.h_proto = ETHER_TYPE_FULL_MASK; } - if (!rule || fsp->location != rule->sw_idx) - return -EINVAL; + if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { + fsp->flow_type |= FLOW_EXT; + fsp->h_ext.vlan_tci = htons(rule->filter.vlan_tci); + fsp->m_ext.vlan_tci = htons(VLAN_PRIO_MASK); + } - if (rule->filter.match_flags) { - fsp->flow_type = ETHER_FLOW; - fsp->ring_cookie = rule->action; - if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { - fsp->h_u.ether_spec.h_proto = rule->filter.etype; - fsp->m_u.ether_spec.h_proto = ETHER_TYPE_FULL_MASK; - } - if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { - fsp->flow_type |= FLOW_EXT; - fsp->h_ext.vlan_tci = rule->filter.vlan_tci; - fsp->m_ext.vlan_tci = htons(VLAN_PRIO_MASK); - } - if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) { - ether_addr_copy(fsp->h_u.ether_spec.h_dest, - rule->filter.dst_addr); - /* As we only support matching by the full - * mask, return the mask to userspace - */ - eth_broadcast_addr(fsp->m_u.ether_spec.h_dest); - } - if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) { - ether_addr_copy(fsp->h_u.ether_spec.h_source, - rule->filter.src_addr); - /* As we only support matching by the full - * mask, return the mask to userspace - */ - eth_broadcast_addr(fsp->m_u.ether_spec.h_source); - } + if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) { + ether_addr_copy(fsp->h_u.ether_spec.h_dest, + rule->filter.dst_addr); + eth_broadcast_addr(fsp->m_u.ether_spec.h_dest); + } - return 0; + if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) { + ether_addr_copy(fsp->h_u.ether_spec.h_source, + rule->filter.src_addr); + eth_broadcast_addr(fsp->m_u.ether_spec.h_source); } + + mutex_unlock(&adapter->nfc_rule_lock); + return 0; + +out: + mutex_unlock(&adapter->nfc_rule_lock); return -EINVAL; } -static int igc_get_ethtool_nfc_all(struct igc_adapter *adapter, - struct ethtool_rxnfc *cmd, - u32 *rule_locs) +static int igc_ethtool_get_nfc_rules(struct igc_adapter *adapter, + struct ethtool_rxnfc *cmd, + u32 *rule_locs) { - struct igc_nfc_filter *rule; + struct igc_nfc_rule *rule; int cnt = 0; - /* report total rule count */ - cmd->data = IGC_MAX_RXNFC_FILTERS; + cmd->data = IGC_MAX_RXNFC_RULES; + + mutex_lock(&adapter->nfc_rule_lock); - hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) { - if (cnt == cmd->rule_cnt) + list_for_each_entry(rule, &adapter->nfc_rule_list, list) { + if (cnt == cmd->rule_cnt) { + mutex_unlock(&adapter->nfc_rule_lock); return -EMSGSIZE; - rule_locs[cnt] = rule->sw_idx; + } + rule_locs[cnt] = rule->location; cnt++; } + mutex_unlock(&adapter->nfc_rule_lock); + cmd->rule_cnt = cnt; return 0; } -static int igc_get_rss_hash_opts(struct igc_adapter *adapter, - struct ethtool_rxnfc *cmd) +static int igc_ethtool_get_rss_hash_opts(struct igc_adapter *adapter, + struct ethtool_rxnfc *cmd) { cmd->data = 0; @@ -1036,41 +1056,33 @@ static int igc_get_rss_hash_opts(struct igc_adapter *adapter, return 0; } -static int igc_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd, - u32 *rule_locs) +static int igc_ethtool_get_rxnfc(struct net_device *dev, + struct ethtool_rxnfc *cmd, u32 *rule_locs) { struct igc_adapter *adapter = netdev_priv(dev); - int ret = -EOPNOTSUPP; switch (cmd->cmd) { case ETHTOOL_GRXRINGS: cmd->data = adapter->num_rx_queues; - ret = 0; - break; + return 0; case ETHTOOL_GRXCLSRLCNT: - cmd->rule_cnt = adapter->nfc_filter_count; - ret = 0; - break; + cmd->rule_cnt = adapter->nfc_rule_count; + return 0; case ETHTOOL_GRXCLSRULE: - ret = igc_get_ethtool_nfc_entry(adapter, cmd); - break; + return igc_ethtool_get_nfc_rule(adapter, cmd); case ETHTOOL_GRXCLSRLALL: - ret = igc_get_ethtool_nfc_all(adapter, cmd, rule_locs); - break; + return igc_ethtool_get_nfc_rules(adapter, cmd, rule_locs); case ETHTOOL_GRXFH: - ret = igc_get_rss_hash_opts(adapter, cmd); - break; + return igc_ethtool_get_rss_hash_opts(adapter, cmd); default: - break; + return -EOPNOTSUPP; } - - return ret; } #define UDP_RSS_FLAGS (IGC_FLAG_RSS_FIELD_IPV4_UDP | \ IGC_FLAG_RSS_FIELD_IPV6_UDP) -static int igc_set_rss_hash_opt(struct igc_adapter *adapter, - struct ethtool_rxnfc *nfc) +static int igc_ethtool_set_rss_hash_opt(struct igc_adapter *adapter, + struct ethtool_rxnfc *nfc) { u32 flags = adapter->flags; @@ -1145,8 +1157,8 @@ static int igc_set_rss_hash_opt(struct igc_adapter *adapter, if ((flags & UDP_RSS_FLAGS) && !(adapter->flags & UDP_RSS_FLAGS)) - dev_err(&adapter->pdev->dev, - "enabling UDP RSS: fragmented packets may arrive out of order to the stack above\n"); + netdev_err(adapter->netdev, + "Enabling UDP RSS: fragmented packets may arrive out of order to the stack above\n"); adapter->flags = flags; @@ -1171,344 +1183,184 @@ static int igc_set_rss_hash_opt(struct igc_adapter *adapter, return 0; } -static int igc_rxnfc_write_etype_filter(struct igc_adapter *adapter, - struct igc_nfc_filter *input) +static void igc_ethtool_init_nfc_rule(struct igc_nfc_rule *rule, + const struct ethtool_rx_flow_spec *fsp) { - struct igc_hw *hw = &adapter->hw; - u8 i; - u32 etqf; - u16 etype; - - /* find an empty etype filter register */ - for (i = 0; i < MAX_ETYPE_FILTER; ++i) { - if (!adapter->etype_bitmap[i]) - break; - } - if (i == MAX_ETYPE_FILTER) { - dev_err(&adapter->pdev->dev, "ethtool -N: etype filters are all used.\n"); - return -EINVAL; - } + INIT_LIST_HEAD(&rule->list); - adapter->etype_bitmap[i] = true; + rule->action = fsp->ring_cookie; + rule->location = fsp->location; - etqf = rd32(IGC_ETQF(i)); - etype = ntohs(input->filter.etype & ETHER_TYPE_FULL_MASK); - - etqf |= IGC_ETQF_FILTER_ENABLE; - etqf &= ~IGC_ETQF_ETYPE_MASK; - etqf |= (etype & IGC_ETQF_ETYPE_MASK); - - etqf &= ~IGC_ETQF_QUEUE_MASK; - etqf |= ((input->action << IGC_ETQF_QUEUE_SHIFT) - & IGC_ETQF_QUEUE_MASK); - etqf |= IGC_ETQF_QUEUE_ENABLE; - - wr32(IGC_ETQF(i), etqf); - - input->etype_reg_index = i; - - return 0; -} - -static int igc_rxnfc_write_vlan_prio_filter(struct igc_adapter *adapter, - struct igc_nfc_filter *input) -{ - struct igc_hw *hw = &adapter->hw; - u8 vlan_priority; - u16 queue_index; - u32 vlapqf; - - vlapqf = rd32(IGC_VLAPQF); - vlan_priority = (ntohs(input->filter.vlan_tci) & VLAN_PRIO_MASK) - >> VLAN_PRIO_SHIFT; - queue_index = (vlapqf >> (vlan_priority * 4)) & IGC_VLAPQF_QUEUE_MASK; - - /* check whether this vlan prio is already set */ - if (vlapqf & IGC_VLAPQF_P_VALID(vlan_priority) && - queue_index != input->action) { - dev_err(&adapter->pdev->dev, "ethtool rxnfc set vlan prio filter failed.\n"); - return -EEXIST; - } - - vlapqf |= IGC_VLAPQF_P_VALID(vlan_priority); - vlapqf |= IGC_VLAPQF_QUEUE_SEL(vlan_priority, input->action); - - wr32(IGC_VLAPQF, vlapqf); - - return 0; -} - -int igc_add_filter(struct igc_adapter *adapter, struct igc_nfc_filter *input) -{ - struct igc_hw *hw = &adapter->hw; - int err = -EINVAL; - - if (hw->mac.type == igc_i225 && - !(input->filter.match_flags & ~IGC_FILTER_FLAG_SRC_MAC_ADDR)) { - dev_err(&adapter->pdev->dev, - "i225 doesn't support flow classification rules specifying only source addresses.\n"); - return -EOPNOTSUPP; + if ((fsp->flow_type & FLOW_EXT) && fsp->m_ext.vlan_tci) { + rule->filter.vlan_tci = ntohs(fsp->h_ext.vlan_tci); + rule->filter.match_flags |= IGC_FILTER_FLAG_VLAN_TCI; } - if (input->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { - err = igc_rxnfc_write_etype_filter(adapter, input); - if (err) - return err; + if (fsp->m_u.ether_spec.h_proto == ETHER_TYPE_FULL_MASK) { + rule->filter.etype = ntohs(fsp->h_u.ether_spec.h_proto); + rule->filter.match_flags = IGC_FILTER_FLAG_ETHER_TYPE; } - if (input->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) { - err = igc_add_mac_steering_filter(adapter, - input->filter.dst_addr, - input->action, 0); - err = min_t(int, err, 0); - if (err) - return err; + /* Both source and destination address filters only support the full + * mask. + */ + if (is_broadcast_ether_addr(fsp->m_u.ether_spec.h_source)) { + rule->filter.match_flags |= IGC_FILTER_FLAG_SRC_MAC_ADDR; + ether_addr_copy(rule->filter.src_addr, + fsp->h_u.ether_spec.h_source); } - if (input->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) { - err = igc_add_mac_steering_filter(adapter, - input->filter.src_addr, - input->action, - IGC_MAC_STATE_SRC_ADDR); - err = min_t(int, err, 0); - if (err) - return err; + if (is_broadcast_ether_addr(fsp->m_u.ether_spec.h_dest)) { + rule->filter.match_flags |= IGC_FILTER_FLAG_DST_MAC_ADDR; + ether_addr_copy(rule->filter.dst_addr, + fsp->h_u.ether_spec.h_dest); } - - if (input->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) - err = igc_rxnfc_write_vlan_prio_filter(adapter, input); - - return err; -} - -static void igc_clear_etype_filter_regs(struct igc_adapter *adapter, - u16 reg_index) -{ - struct igc_hw *hw = &adapter->hw; - u32 etqf = rd32(IGC_ETQF(reg_index)); - - etqf &= ~IGC_ETQF_QUEUE_ENABLE; - etqf &= ~IGC_ETQF_QUEUE_MASK; - etqf &= ~IGC_ETQF_FILTER_ENABLE; - - wr32(IGC_ETQF(reg_index), etqf); - - adapter->etype_bitmap[reg_index] = false; -} - -static void igc_clear_vlan_prio_filter(struct igc_adapter *adapter, - u16 vlan_tci) -{ - struct igc_hw *hw = &adapter->hw; - u8 vlan_priority; - u32 vlapqf; - - vlan_priority = (vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; - - vlapqf = rd32(IGC_VLAPQF); - vlapqf &= ~IGC_VLAPQF_P_VALID(vlan_priority); - vlapqf &= ~IGC_VLAPQF_QUEUE_SEL(vlan_priority, - IGC_VLAPQF_QUEUE_MASK); - - wr32(IGC_VLAPQF, vlapqf); } -int igc_erase_filter(struct igc_adapter *adapter, struct igc_nfc_filter *input) +/** + * igc_ethtool_check_nfc_rule() - Check if NFC rule is valid + * @adapter: Pointer to adapter + * @rule: Rule under evaluation + * + * The driver doesn't support rules with multiple matches so if more than + * one bit in filter flags is set, @rule is considered invalid. + * + * Also, if there is already another rule with the same filter in a different + * location, @rule is considered invalid. + * + * Context: Expects adapter->nfc_rule_lock to be held by caller. + * + * Return: 0 in case of success, negative errno code otherwise. + */ +static int igc_ethtool_check_nfc_rule(struct igc_adapter *adapter, + struct igc_nfc_rule *rule) { - if (input->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) - igc_clear_etype_filter_regs(adapter, - input->etype_reg_index); - - if (input->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) - igc_clear_vlan_prio_filter(adapter, - ntohs(input->filter.vlan_tci)); - - if (input->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) - igc_del_mac_steering_filter(adapter, input->filter.src_addr, - input->action, - IGC_MAC_STATE_SRC_ADDR); - - if (input->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) - igc_del_mac_steering_filter(adapter, input->filter.dst_addr, - input->action, 0); - - return 0; -} - -static int igc_update_ethtool_nfc_entry(struct igc_adapter *adapter, - struct igc_nfc_filter *input, - u16 sw_idx) -{ - struct igc_nfc_filter *rule, *parent; - int err = -EINVAL; - - parent = NULL; - rule = NULL; + struct net_device *dev = adapter->netdev; + u8 flags = rule->filter.match_flags; + struct igc_nfc_rule *tmp; - hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) { - /* hash found, or no matching entry */ - if (rule->sw_idx >= sw_idx) - break; - parent = rule; + if (!flags) { + netdev_dbg(dev, "Rule with no match\n"); + return -EINVAL; } - /* if there is an old rule occupying our place remove it */ - if (rule && rule->sw_idx == sw_idx) { - if (!input) - err = igc_erase_filter(adapter, rule); - - hlist_del(&rule->nfc_node); - kfree(rule); - adapter->nfc_filter_count--; + if (flags & (flags - 1)) { + netdev_dbg(dev, "Rule with multiple matches not supported\n"); + return -EOPNOTSUPP; } - /* If no input this was a delete, err should be 0 if a rule was - * successfully found and removed from the list else -EINVAL - */ - if (!input) - return err; - - /* initialize node */ - INIT_HLIST_NODE(&input->nfc_node); - - /* add filter to the list */ - if (parent) - hlist_add_behind(&input->nfc_node, &parent->nfc_node); - else - hlist_add_head(&input->nfc_node, &adapter->nfc_filter_list); - - /* update counts */ - adapter->nfc_filter_count++; + list_for_each_entry(tmp, &adapter->nfc_rule_list, list) { + if (!memcmp(&rule->filter, &tmp->filter, + sizeof(rule->filter)) && + tmp->location != rule->location) { + netdev_dbg(dev, "Rule already exists\n"); + return -EEXIST; + } + } return 0; } -static int igc_add_ethtool_nfc_entry(struct igc_adapter *adapter, - struct ethtool_rxnfc *cmd) +static int igc_ethtool_add_nfc_rule(struct igc_adapter *adapter, + struct ethtool_rxnfc *cmd) { struct net_device *netdev = adapter->netdev; struct ethtool_rx_flow_spec *fsp = (struct ethtool_rx_flow_spec *)&cmd->fs; - struct igc_nfc_filter *input, *rule; - int err = 0; + struct igc_nfc_rule *rule, *old_rule; + int err; - if (!(netdev->hw_features & NETIF_F_NTUPLE)) + if (!(netdev->hw_features & NETIF_F_NTUPLE)) { + netdev_dbg(netdev, "N-tuple filters disabled\n"); return -EOPNOTSUPP; + } - /* Don't allow programming if the action is a queue greater than - * the number of online Rx queues. - */ - if (fsp->ring_cookie == RX_CLS_FLOW_DISC || - fsp->ring_cookie >= adapter->num_rx_queues) { - dev_err(&adapter->pdev->dev, "ethtool -N: The specified action is invalid\n"); - return -EINVAL; + if ((fsp->flow_type & ~FLOW_EXT) != ETHER_FLOW) { + netdev_dbg(netdev, "Only ethernet flow type is supported\n"); + return -EOPNOTSUPP; } - /* Don't allow indexes to exist outside of available space */ - if (fsp->location >= IGC_MAX_RXNFC_FILTERS) { - dev_err(&adapter->pdev->dev, "Location out of range\n"); - return -EINVAL; + if ((fsp->flow_type & FLOW_EXT) && + fsp->m_ext.vlan_tci != htons(VLAN_PRIO_MASK)) { + netdev_dbg(netdev, "VLAN mask not supported\n"); + return -EOPNOTSUPP; } - if ((fsp->flow_type & ~FLOW_EXT) != ETHER_FLOW) + if (fsp->ring_cookie >= adapter->num_rx_queues) { + netdev_dbg(netdev, "Invalid action\n"); return -EINVAL; - - input = kzalloc(sizeof(*input), GFP_KERNEL); - if (!input) - return -ENOMEM; - - if (fsp->m_u.ether_spec.h_proto == ETHER_TYPE_FULL_MASK) { - input->filter.etype = fsp->h_u.ether_spec.h_proto; - input->filter.match_flags = IGC_FILTER_FLAG_ETHER_TYPE; } - /* Only support matching addresses by the full mask */ - if (is_broadcast_ether_addr(fsp->m_u.ether_spec.h_source)) { - input->filter.match_flags |= IGC_FILTER_FLAG_SRC_MAC_ADDR; - ether_addr_copy(input->filter.src_addr, - fsp->h_u.ether_spec.h_source); + if (fsp->location >= IGC_MAX_RXNFC_RULES) { + netdev_dbg(netdev, "Invalid location\n"); + return -EINVAL; } - /* Only support matching addresses by the full mask */ - if (is_broadcast_ether_addr(fsp->m_u.ether_spec.h_dest)) { - input->filter.match_flags |= IGC_FILTER_FLAG_DST_MAC_ADDR; - ether_addr_copy(input->filter.dst_addr, - fsp->h_u.ether_spec.h_dest); - } + rule = kzalloc(sizeof(*rule), GFP_KERNEL); + if (!rule) + return -ENOMEM; - if ((fsp->flow_type & FLOW_EXT) && fsp->m_ext.vlan_tci) { - if (fsp->m_ext.vlan_tci != htons(VLAN_PRIO_MASK)) { - err = -EINVAL; - goto err_out; - } - input->filter.vlan_tci = fsp->h_ext.vlan_tci; - input->filter.match_flags |= IGC_FILTER_FLAG_VLAN_TCI; - } + igc_ethtool_init_nfc_rule(rule, fsp); - input->action = fsp->ring_cookie; - input->sw_idx = fsp->location; + mutex_lock(&adapter->nfc_rule_lock); - spin_lock(&adapter->nfc_lock); + err = igc_ethtool_check_nfc_rule(adapter, rule); + if (err) + goto err; - hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) { - if (!memcmp(&input->filter, &rule->filter, - sizeof(input->filter))) { - err = -EEXIST; - dev_err(&adapter->pdev->dev, - "ethtool: this filter is already set\n"); - goto err_out_w_lock; - } - } + old_rule = igc_get_nfc_rule(adapter, fsp->location); + if (old_rule) + igc_del_nfc_rule(adapter, old_rule); - err = igc_add_filter(adapter, input); + err = igc_add_nfc_rule(adapter, rule); if (err) - goto err_out_w_lock; - - igc_update_ethtool_nfc_entry(adapter, input, input->sw_idx); + goto err; - spin_unlock(&adapter->nfc_lock); + mutex_unlock(&adapter->nfc_rule_lock); return 0; -err_out_w_lock: - spin_unlock(&adapter->nfc_lock); -err_out: - kfree(input); +err: + mutex_unlock(&adapter->nfc_rule_lock); + kfree(rule); return err; } -static int igc_del_ethtool_nfc_entry(struct igc_adapter *adapter, - struct ethtool_rxnfc *cmd) +static int igc_ethtool_del_nfc_rule(struct igc_adapter *adapter, + struct ethtool_rxnfc *cmd) { struct ethtool_rx_flow_spec *fsp = (struct ethtool_rx_flow_spec *)&cmd->fs; - int err; + struct igc_nfc_rule *rule; - spin_lock(&adapter->nfc_lock); - err = igc_update_ethtool_nfc_entry(adapter, NULL, fsp->location); - spin_unlock(&adapter->nfc_lock); + mutex_lock(&adapter->nfc_rule_lock); - return err; + rule = igc_get_nfc_rule(adapter, fsp->location); + if (!rule) { + mutex_unlock(&adapter->nfc_rule_lock); + return -EINVAL; + } + + igc_del_nfc_rule(adapter, rule); + + mutex_unlock(&adapter->nfc_rule_lock); + return 0; } -static int igc_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd) +static int igc_ethtool_set_rxnfc(struct net_device *dev, + struct ethtool_rxnfc *cmd) { struct igc_adapter *adapter = netdev_priv(dev); - int ret = -EOPNOTSUPP; switch (cmd->cmd) { case ETHTOOL_SRXFH: - ret = igc_set_rss_hash_opt(adapter, cmd); - break; + return igc_ethtool_set_rss_hash_opt(adapter, cmd); case ETHTOOL_SRXCLSRLINS: - ret = igc_add_ethtool_nfc_entry(adapter, cmd); - break; + return igc_ethtool_add_nfc_rule(adapter, cmd); case ETHTOOL_SRXCLSRLDEL: - ret = igc_del_ethtool_nfc_entry(adapter, cmd); + return igc_ethtool_del_nfc_rule(adapter, cmd); default: - break; + return -EOPNOTSUPP; } - - return ret; } void igc_write_rss_indir_tbl(struct igc_adapter *adapter) @@ -1533,13 +1385,13 @@ void igc_write_rss_indir_tbl(struct igc_adapter *adapter) } } -static u32 igc_get_rxfh_indir_size(struct net_device *netdev) +static u32 igc_ethtool_get_rxfh_indir_size(struct net_device *netdev) { return IGC_RETA_SIZE; } -static int igc_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, - u8 *hfunc) +static int igc_ethtool_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, + u8 *hfunc) { struct igc_adapter *adapter = netdev_priv(netdev); int i; @@ -1554,8 +1406,8 @@ static int igc_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, return 0; } -static int igc_set_rxfh(struct net_device *netdev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int igc_ethtool_set_rxfh(struct net_device *netdev, const u32 *indir, + const u8 *key, const u8 hfunc) { struct igc_adapter *adapter = netdev_priv(netdev); u32 num_queues; @@ -1583,18 +1435,13 @@ static int igc_set_rxfh(struct net_device *netdev, const u32 *indir, return 0; } -static unsigned int igc_max_channels(struct igc_adapter *adapter) -{ - return igc_get_max_rss_queues(adapter); -} - -static void igc_get_channels(struct net_device *netdev, - struct ethtool_channels *ch) +static void igc_ethtool_get_channels(struct net_device *netdev, + struct ethtool_channels *ch) { struct igc_adapter *adapter = netdev_priv(netdev); /* Report maximum channels */ - ch->max_combined = igc_max_channels(adapter); + ch->max_combined = igc_get_max_rss_queues(adapter); /* Report info for other vector */ if (adapter->flags & IGC_FLAG_HAS_MSIX) { @@ -1605,8 +1452,8 @@ static void igc_get_channels(struct net_device *netdev, ch->combined_count = adapter->rss_queues; } -static int igc_set_channels(struct net_device *netdev, - struct ethtool_channels *ch) +static int igc_ethtool_set_channels(struct net_device *netdev, + struct ethtool_channels *ch) { struct igc_adapter *adapter = netdev_priv(netdev); unsigned int count = ch->combined_count; @@ -1621,7 +1468,7 @@ static int igc_set_channels(struct net_device *netdev, return -EINVAL; /* Verify the number of channels doesn't exceed hw limits */ - max_combined = igc_max_channels(adapter); + max_combined = igc_get_max_rss_queues(adapter); if (count > max_combined) return -EINVAL; @@ -1638,8 +1485,8 @@ static int igc_set_channels(struct net_device *netdev, return 0; } -static int igc_get_ts_info(struct net_device *dev, - struct ethtool_ts_info *info) +static int igc_ethtool_get_ts_info(struct net_device *dev, + struct ethtool_ts_info *info) { struct igc_adapter *adapter = netdev_priv(dev); @@ -1671,7 +1518,7 @@ static int igc_get_ts_info(struct net_device *dev, } } -static u32 igc_get_priv_flags(struct net_device *netdev) +static u32 igc_ethtool_get_priv_flags(struct net_device *netdev) { struct igc_adapter *adapter = netdev_priv(netdev); u32 priv_flags = 0; @@ -1682,7 +1529,7 @@ static u32 igc_get_priv_flags(struct net_device *netdev) return priv_flags; } -static int igc_set_priv_flags(struct net_device *netdev, u32 priv_flags) +static int igc_ethtool_set_priv_flags(struct net_device *netdev, u32 priv_flags) { struct igc_adapter *adapter = netdev_priv(netdev); unsigned int flags = adapter->flags; @@ -1717,8 +1564,8 @@ static void igc_ethtool_complete(struct net_device *netdev) pm_runtime_put(&adapter->pdev->dev); } -static int igc_get_link_ksettings(struct net_device *netdev, - struct ethtool_link_ksettings *cmd) +static int igc_ethtool_get_link_ksettings(struct net_device *netdev, + struct ethtool_link_ksettings *cmd) { struct igc_adapter *adapter = netdev_priv(netdev); struct igc_hw *hw = &adapter->hw; @@ -1824,10 +1671,12 @@ static int igc_get_link_ksettings(struct net_device *netdev, return 0; } -static int igc_set_link_ksettings(struct net_device *netdev, - const struct ethtool_link_ksettings *cmd) +static int +igc_ethtool_set_link_ksettings(struct net_device *netdev, + const struct ethtool_link_ksettings *cmd) { struct igc_adapter *adapter = netdev_priv(netdev); + struct net_device *dev = adapter->netdev; struct igc_hw *hw = &adapter->hw; u32 advertising; @@ -1835,8 +1684,7 @@ static int igc_set_link_ksettings(struct net_device *netdev, * cannot be changed */ if (igc_check_reset_block(hw)) { - dev_err(&adapter->pdev->dev, - "Cannot change link characteristics when reset is active.\n"); + netdev_err(dev, "Cannot change link characteristics when reset is active\n"); return -EINVAL; } @@ -1847,7 +1695,7 @@ static int igc_set_link_ksettings(struct net_device *netdev, if (cmd->base.eth_tp_mdix_ctrl) { if (cmd->base.eth_tp_mdix_ctrl != ETH_TP_MDI_AUTO && cmd->base.autoneg != AUTONEG_ENABLE) { - dev_err(&adapter->pdev->dev, "forcing MDI/MDI-X state is not supported when link speed and/or duplex are forced\n"); + netdev_err(dev, "Forcing MDI/MDI-X state is not supported when link speed and/or duplex are forced\n"); return -EINVAL; } } @@ -1864,9 +1712,7 @@ static int igc_set_link_ksettings(struct net_device *netdev, if (adapter->fc_autoneg) hw->fc.requested_mode = igc_fc_default; } else { - /* calling this overrides forced MDI setting */ - dev_info(&adapter->pdev->dev, - "Force mode currently not supported\n"); + netdev_info(dev, "Force mode currently not supported\n"); } /* MDI-X => 2; MDI => 1; Auto => 3 */ @@ -1893,46 +1739,105 @@ static int igc_set_link_ksettings(struct net_device *netdev, return 0; } +static void igc_ethtool_diag_test(struct net_device *netdev, + struct ethtool_test *eth_test, u64 *data) +{ + struct igc_adapter *adapter = netdev_priv(netdev); + bool if_running = netif_running(netdev); + + if (eth_test->flags == ETH_TEST_FL_OFFLINE) { + netdev_info(adapter->netdev, "Offline testing starting"); + set_bit(__IGC_TESTING, &adapter->state); + + /* Link test performed before hardware reset so autoneg doesn't + * interfere with test result + */ + if (!igc_link_test(adapter, &data[TEST_LINK])) + eth_test->flags |= ETH_TEST_FL_FAILED; + + if (if_running) + igc_close(netdev); + else + igc_reset(adapter); + + netdev_info(adapter->netdev, "Register testing starting"); + if (!igc_reg_test(adapter, &data[TEST_REG])) + eth_test->flags |= ETH_TEST_FL_FAILED; + + igc_reset(adapter); + + netdev_info(adapter->netdev, "EEPROM testing starting"); + if (!igc_eeprom_test(adapter, &data[TEST_EEP])) + eth_test->flags |= ETH_TEST_FL_FAILED; + + igc_reset(adapter); + + /* loopback and interrupt tests + * will be implemented in the future + */ + data[TEST_LOOP] = 0; + data[TEST_IRQ] = 0; + + clear_bit(__IGC_TESTING, &adapter->state); + if (if_running) + igc_open(netdev); + } else { + netdev_info(adapter->netdev, "Online testing starting"); + + /* register, eeprom, intr and loopback tests not run online */ + data[TEST_REG] = 0; + data[TEST_EEP] = 0; + data[TEST_IRQ] = 0; + data[TEST_LOOP] = 0; + + if (!igc_link_test(adapter, &data[TEST_LINK])) + eth_test->flags |= ETH_TEST_FL_FAILED; + } + + msleep_interruptible(4 * 1000); +} + static const struct ethtool_ops igc_ethtool_ops = { .supported_coalesce_params = ETHTOOL_COALESCE_USECS, - .get_drvinfo = igc_get_drvinfo, - .get_regs_len = igc_get_regs_len, - .get_regs = igc_get_regs, - .get_wol = igc_get_wol, - .set_wol = igc_set_wol, - .get_msglevel = igc_get_msglevel, - .set_msglevel = igc_set_msglevel, - .nway_reset = igc_nway_reset, - .get_link = igc_get_link, - .get_eeprom_len = igc_get_eeprom_len, - .get_eeprom = igc_get_eeprom, - .set_eeprom = igc_set_eeprom, - .get_ringparam = igc_get_ringparam, - .set_ringparam = igc_set_ringparam, - .get_pauseparam = igc_get_pauseparam, - .set_pauseparam = igc_set_pauseparam, - .get_strings = igc_get_strings, - .get_sset_count = igc_get_sset_count, - .get_ethtool_stats = igc_get_ethtool_stats, - .get_coalesce = igc_get_coalesce, - .set_coalesce = igc_set_coalesce, - .get_rxnfc = igc_get_rxnfc, - .set_rxnfc = igc_set_rxnfc, - .get_rxfh_indir_size = igc_get_rxfh_indir_size, - .get_rxfh = igc_get_rxfh, - .set_rxfh = igc_set_rxfh, - .get_ts_info = igc_get_ts_info, - .get_channels = igc_get_channels, - .set_channels = igc_set_channels, - .get_priv_flags = igc_get_priv_flags, - .set_priv_flags = igc_set_priv_flags, + .get_drvinfo = igc_ethtool_get_drvinfo, + .get_regs_len = igc_ethtool_get_regs_len, + .get_regs = igc_ethtool_get_regs, + .get_wol = igc_ethtool_get_wol, + .set_wol = igc_ethtool_set_wol, + .get_msglevel = igc_ethtool_get_msglevel, + .set_msglevel = igc_ethtool_set_msglevel, + .nway_reset = igc_ethtool_nway_reset, + .get_link = igc_ethtool_get_link, + .get_eeprom_len = igc_ethtool_get_eeprom_len, + .get_eeprom = igc_ethtool_get_eeprom, + .set_eeprom = igc_ethtool_set_eeprom, + .get_ringparam = igc_ethtool_get_ringparam, + .set_ringparam = igc_ethtool_set_ringparam, + .get_pauseparam = igc_ethtool_get_pauseparam, + .set_pauseparam = igc_ethtool_set_pauseparam, + .get_strings = igc_ethtool_get_strings, + .get_sset_count = igc_ethtool_get_sset_count, + .get_ethtool_stats = igc_ethtool_get_stats, + .get_coalesce = igc_ethtool_get_coalesce, + .set_coalesce = igc_ethtool_set_coalesce, + .get_rxnfc = igc_ethtool_get_rxnfc, + .set_rxnfc = igc_ethtool_set_rxnfc, + .get_rxfh_indir_size = igc_ethtool_get_rxfh_indir_size, + .get_rxfh = igc_ethtool_get_rxfh, + .set_rxfh = igc_ethtool_set_rxfh, + .get_ts_info = igc_ethtool_get_ts_info, + .get_channels = igc_ethtool_get_channels, + .set_channels = igc_ethtool_set_channels, + .get_priv_flags = igc_ethtool_get_priv_flags, + .set_priv_flags = igc_ethtool_set_priv_flags, .begin = igc_ethtool_begin, .complete = igc_ethtool_complete, - .get_link_ksettings = igc_get_link_ksettings, - .set_link_ksettings = igc_set_link_ksettings, + .get_link_ksettings = igc_ethtool_get_link_ksettings, + .set_link_ksettings = igc_ethtool_set_link_ksettings, + .self_test = igc_ethtool_diag_test, }; -void igc_set_ethtool_ops(struct net_device *netdev) +void igc_ethtool_set_ops(struct net_device *netdev) { netdev->ethtool_ops = &igc_ethtool_ops; } diff --git a/drivers/net/ethernet/intel/igc/igc_hw.h b/drivers/net/ethernet/intel/igc/igc_hw.h index 90ac0e0144d8..af34ae310327 100644 --- a/drivers/net/ethernet/intel/igc/igc_hw.h +++ b/drivers/net/ethernet/intel/igc/igc_hw.h @@ -21,6 +21,9 @@ #define IGC_DEV_ID_I225_I 0x15F8 #define IGC_DEV_ID_I220_V 0x15F7 #define IGC_DEV_ID_I225_K 0x3100 +#define IGC_DEV_ID_I225_K2 0x3101 +#define IGC_DEV_ID_I225_LMVP 0x5502 +#define IGC_DEV_ID_I225_IT 0x0D9F #define IGC_DEV_ID_I225_BLANK_NVM 0x15FD /* Function pointers for the MAC. */ diff --git a/drivers/net/ethernet/intel/igc/igc_mac.c b/drivers/net/ethernet/intel/igc/igc_mac.c index 12aa6b5fcb5d..410aeb01de5c 100644 --- a/drivers/net/ethernet/intel/igc/igc_mac.c +++ b/drivers/net/ethernet/intel/igc/igc_mac.c @@ -235,15 +235,14 @@ out: void igc_clear_hw_cntrs_base(struct igc_hw *hw) { rd32(IGC_CRCERRS); - rd32(IGC_SYMERRS); rd32(IGC_MPC); rd32(IGC_SCC); rd32(IGC_ECOL); rd32(IGC_MCC); rd32(IGC_LATECOL); rd32(IGC_COLC); + rd32(IGC_RERC); rd32(IGC_DC); - rd32(IGC_SEC); rd32(IGC_RLEC); rd32(IGC_XONRXC); rd32(IGC_XONTXC); @@ -288,7 +287,7 @@ void igc_clear_hw_cntrs_base(struct igc_hw *hw) rd32(IGC_ALGNERRC); rd32(IGC_RXERRC); rd32(IGC_TNCRS); - rd32(IGC_CEXTERR); + rd32(IGC_HTDPMC); rd32(IGC_TSCTC); rd32(IGC_TSCTFC); @@ -307,12 +306,8 @@ void igc_clear_hw_cntrs_base(struct igc_hw *hw) rd32(IGC_ICTXQMTC); rd32(IGC_ICRXDMTC); - rd32(IGC_CBTMPC); - rd32(IGC_HTDPMC); - rd32(IGC_CBRMPC); rd32(IGC_RPTHC); rd32(IGC_HGPTC); - rd32(IGC_HTCBDPC); rd32(IGC_HGORCL); rd32(IGC_HGORCH); rd32(IGC_HGOTCL); diff --git a/drivers/net/ethernet/intel/igc/igc_mac.h b/drivers/net/ethernet/intel/igc/igc_mac.h index 832cccec87cd..b5963f86defb 100644 --- a/drivers/net/ethernet/intel/igc/igc_mac.h +++ b/drivers/net/ethernet/intel/igc/igc_mac.h @@ -8,10 +8,6 @@ #include "igc_phy.h" #include "igc_defines.h" -#ifndef IGC_REMOVED -#define IGC_REMOVED(a) (0) -#endif /* IGC_REMOVED */ - /* forward declaration */ s32 igc_disable_pcie_master(struct igc_hw *hw); s32 igc_check_for_copper_link(struct igc_hw *hw); diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 59fc0097438f..6919c50e449a 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -9,11 +9,13 @@ #include <linux/udp.h> #include <linux/ip.h> #include <linux/pm_runtime.h> +#include <net/pkt_sched.h> #include <net/ipv6.h> #include "igc.h" #include "igc_hw.h" +#include "igc_tsn.h" #define DRV_VERSION "0.0.1-k" #define DRV_SUMMARY "Intel(R) 2.5G Ethernet Linux Driver" @@ -45,6 +47,9 @@ static const struct pci_device_id igc_pci_tbl[] = { { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base }, { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base }, { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base }, + { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), board_base }, + { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), board_base }, + { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), board_base }, { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), board_base }, /* required last entry */ {0, } @@ -71,7 +76,7 @@ static void igc_power_down_link(struct igc_adapter *adapter) void igc_reset(struct igc_adapter *adapter) { - struct pci_dev *pdev = adapter->pdev; + struct net_device *dev = adapter->netdev; struct igc_hw *hw = &adapter->hw; struct igc_fc_info *fc = &hw->fc; u32 pba, hwm; @@ -98,7 +103,7 @@ void igc_reset(struct igc_adapter *adapter) hw->mac.ops.reset_hw(hw); if (hw->mac.ops.init_hw(hw)) - dev_err(&pdev->dev, "Hardware Error\n"); + netdev_err(dev, "Error on hardware initialization\n"); if (!netif_running(adapter->netdev)) igc_power_down_link(adapter); @@ -106,6 +111,9 @@ void igc_reset(struct igc_adapter *adapter) /* Re-enable PTP, where applicable. */ igc_ptp_reset(adapter); + /* Re-enable TSN offloading, where applicable. */ + igc_tsn_offload_apply(adapter); + igc_get_phy_info(hw); } @@ -280,6 +288,7 @@ static void igc_clean_all_tx_rings(struct igc_adapter *adapter) */ int igc_setup_tx_resources(struct igc_ring *tx_ring) { + struct net_device *ndev = tx_ring->netdev; struct device *dev = tx_ring->dev; int size = 0; @@ -305,8 +314,7 @@ int igc_setup_tx_resources(struct igc_ring *tx_ring) err: vfree(tx_ring->tx_buffer_info); - dev_err(dev, - "Unable to allocate memory for the transmit descriptor ring\n"); + netdev_err(ndev, "Unable to allocate memory for Tx descriptor ring\n"); return -ENOMEM; } @@ -318,14 +326,13 @@ err: */ static int igc_setup_all_tx_resources(struct igc_adapter *adapter) { - struct pci_dev *pdev = adapter->pdev; + struct net_device *dev = adapter->netdev; int i, err = 0; for (i = 0; i < adapter->num_tx_queues; i++) { err = igc_setup_tx_resources(adapter->tx_ring[i]); if (err) { - dev_err(&pdev->dev, - "Allocation for Tx Queue %u failed\n", i); + netdev_err(dev, "Error on Tx queue %u setup\n", i); for (i--; i >= 0; i--) igc_free_tx_resources(adapter->tx_ring[i]); break; @@ -436,6 +443,7 @@ static void igc_free_all_rx_resources(struct igc_adapter *adapter) */ int igc_setup_rx_resources(struct igc_ring *rx_ring) { + struct net_device *ndev = rx_ring->netdev; struct device *dev = rx_ring->dev; int size, desc_len; @@ -465,8 +473,7 @@ int igc_setup_rx_resources(struct igc_ring *rx_ring) err: vfree(rx_ring->rx_buffer_info); rx_ring->rx_buffer_info = NULL; - dev_err(dev, - "Unable to allocate memory for the receive descriptor ring\n"); + netdev_err(ndev, "Unable to allocate memory for Rx descriptor ring\n"); return -ENOMEM; } @@ -479,14 +486,13 @@ err: */ static int igc_setup_all_rx_resources(struct igc_adapter *adapter) { - struct pci_dev *pdev = adapter->pdev; + struct net_device *dev = adapter->netdev; int i, err = 0; for (i = 0; i < adapter->num_rx_queues; i++) { err = igc_setup_rx_resources(adapter->rx_ring[i]); if (err) { - dev_err(&pdev->dev, - "Allocation for Rx Queue %u failed\n", i); + netdev_err(dev, "Error on Rx queue %u setup\n", i); for (i--; i >= 0; i--) igc_free_rx_resources(adapter->rx_ring[i]); break; @@ -757,48 +763,76 @@ static void igc_setup_tctl(struct igc_adapter *adapter) } /** - * igc_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table - * @adapter: address of board private structure - * @index: Index of the RAR entry which need to be synced with MAC table + * igc_set_mac_filter_hw() - Set MAC address filter in hardware + * @adapter: Pointer to adapter where the filter should be set + * @index: Filter index + * @type: MAC address filter type (source or destination) + * @addr: MAC address + * @queue: If non-negative, queue assignment feature is enabled and frames + * matching the filter are enqueued onto 'queue'. Otherwise, queue + * assignment is disabled. */ -static void igc_rar_set_index(struct igc_adapter *adapter, u32 index) +static void igc_set_mac_filter_hw(struct igc_adapter *adapter, int index, + enum igc_mac_filter_type type, + const u8 *addr, int queue) { - u8 *addr = adapter->mac_table[index].addr; + struct net_device *dev = adapter->netdev; struct igc_hw *hw = &adapter->hw; - u32 rar_low, rar_high; + u32 ral, rah; - /* HW expects these to be in network order when they are plugged - * into the registers which are little endian. In order to guarantee - * that ordering we need to do an leXX_to_cpup here in order to be - * ready for the byteswap that occurs with writel - */ - rar_low = le32_to_cpup((__le32 *)(addr)); - rar_high = le16_to_cpup((__le16 *)(addr + 4)); + if (WARN_ON(index >= hw->mac.rar_entry_count)) + return; - /* Indicate to hardware the Address is Valid. */ - if (adapter->mac_table[index].state & IGC_MAC_STATE_IN_USE) { - if (is_valid_ether_addr(addr)) - rar_high |= IGC_RAH_AV; + ral = le32_to_cpup((__le32 *)(addr)); + rah = le16_to_cpup((__le16 *)(addr + 4)); - rar_high |= IGC_RAH_POOL_1 << - adapter->mac_table[index].queue; + if (type == IGC_MAC_FILTER_TYPE_SRC) { + rah &= ~IGC_RAH_ASEL_MASK; + rah |= IGC_RAH_ASEL_SRC_ADDR; } - wr32(IGC_RAL(index), rar_low); - wrfl(); - wr32(IGC_RAH(index), rar_high); - wrfl(); + if (queue >= 0) { + rah &= ~IGC_RAH_QSEL_MASK; + rah |= (queue << IGC_RAH_QSEL_SHIFT); + rah |= IGC_RAH_QSEL_ENABLE; + } + + rah |= IGC_RAH_AV; + + wr32(IGC_RAL(index), ral); + wr32(IGC_RAH(index), rah); + + netdev_dbg(dev, "MAC address filter set in HW: index %d", index); +} + +/** + * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware + * @adapter: Pointer to adapter where the filter should be cleared + * @index: Filter index + */ +static void igc_clear_mac_filter_hw(struct igc_adapter *adapter, int index) +{ + struct net_device *dev = adapter->netdev; + struct igc_hw *hw = &adapter->hw; + + if (WARN_ON(index >= hw->mac.rar_entry_count)) + return; + + wr32(IGC_RAL(index), 0); + wr32(IGC_RAH(index), 0); + + netdev_dbg(dev, "MAC address filter cleared in HW: index %d", index); } /* Set default MAC address for the PF in the first RAR entry */ static void igc_set_default_mac_filter(struct igc_adapter *adapter) { - struct igc_mac_addr *mac_table = &adapter->mac_table[0]; + struct net_device *dev = adapter->netdev; + u8 *addr = adapter->hw.mac.addr; - ether_addr_copy(mac_table->addr, adapter->hw.mac.addr); - mac_table->state = IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE; + netdev_dbg(dev, "Set default MAC address filter: address %pM", addr); - igc_rar_set_index(adapter, 0); + igc_set_mac_filter_hw(adapter, 0, IGC_MAC_FILTER_TYPE_DST, addr, -1); } /** @@ -864,6 +898,23 @@ static int igc_write_mc_addr_list(struct net_device *netdev) return netdev_mc_count(netdev); } +static __le32 igc_tx_launchtime(struct igc_adapter *adapter, ktime_t txtime) +{ + ktime_t cycle_time = adapter->cycle_time; + ktime_t base_time = adapter->base_time; + u32 launchtime; + + /* FIXME: when using ETF together with taprio, we may have a + * case where 'delta' is larger than the cycle_time, this may + * cause problems if we don't read the current value of + * IGC_BASET, as the value writen into the launchtime + * descriptor field may be misinterpreted. + */ + div_s64_rem(ktime_sub_ns(txtime, base_time), cycle_time, &launchtime); + + return cpu_to_le32(launchtime); +} + static void igc_tx_ctxtdesc(struct igc_ring *tx_ring, struct igc_tx_buffer *first, u32 vlan_macip_lens, u32 type_tucmd, @@ -871,7 +922,6 @@ static void igc_tx_ctxtdesc(struct igc_ring *tx_ring, { struct igc_adv_tx_context_desc *context_desc; u16 i = tx_ring->next_to_use; - struct timespec64 ts; context_desc = IGC_TX_CTXTDESC(tx_ring, i); @@ -893,9 +943,12 @@ static void igc_tx_ctxtdesc(struct igc_ring *tx_ring, * should have been handled by the upper layers. */ if (tx_ring->launchtime_enable) { - ts = ktime_to_timespec64(first->skb->tstamp); + struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); + ktime_t txtime = first->skb->tstamp; + first->skb->tstamp = ktime_set(0, 0); - context_desc->launch_time = cpu_to_le32(ts.tv_nsec / 32); + context_desc->launch_time = igc_tx_launchtime(adapter, + txtime); } else { context_desc->launch_time = 0; } @@ -1143,7 +1196,7 @@ static int igc_tx_map(struct igc_ring *tx_ring, return 0; dma_error: - dev_err(tx_ring->dev, "TX DMA map failed\n"); + netdev_err(tx_ring->netdev, "TX DMA map failed\n"); tx_buffer = &tx_ring->tx_buffer_info[i]; /* clear dma mappings for failed tx_buffer_info map */ @@ -1406,8 +1459,8 @@ static void igc_rx_checksum(struct igc_ring *ring, IGC_RXD_STAT_UDPCS)) skb->ip_summed = CHECKSUM_UNNECESSARY; - dev_dbg(ring->dev, "cksum success: bits %08X\n", - le32_to_cpu(rx_desc->wb.upper.status_error)); + netdev_dbg(ring->netdev, "cksum success: bits %08X\n", + le32_to_cpu(rx_desc->wb.upper.status_error)); } static inline void igc_rx_hash(struct igc_ring *ring, @@ -2069,27 +2122,27 @@ static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) (adapter->tx_timeout_factor * HZ)) && !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF)) { /* detected Tx unit hang */ - dev_err(tx_ring->dev, - "Detected Tx Unit Hang\n" - " Tx Queue <%d>\n" - " TDH <%x>\n" - " TDT <%x>\n" - " next_to_use <%x>\n" - " next_to_clean <%x>\n" - "buffer_info[next_to_clean]\n" - " time_stamp <%lx>\n" - " next_to_watch <%p>\n" - " jiffies <%lx>\n" - " desc.status <%x>\n", - tx_ring->queue_index, - rd32(IGC_TDH(tx_ring->reg_idx)), - readl(tx_ring->tail), - tx_ring->next_to_use, - tx_ring->next_to_clean, - tx_buffer->time_stamp, - tx_buffer->next_to_watch, - jiffies, - tx_buffer->next_to_watch->wb.status); + netdev_err(tx_ring->netdev, + "Detected Tx Unit Hang\n" + " Tx Queue <%d>\n" + " TDH <%x>\n" + " TDT <%x>\n" + " next_to_use <%x>\n" + " next_to_clean <%x>\n" + "buffer_info[next_to_clean]\n" + " time_stamp <%lx>\n" + " next_to_watch <%p>\n" + " jiffies <%lx>\n" + " desc.status <%x>\n", + tx_ring->queue_index, + rd32(IGC_TDH(tx_ring->reg_idx)), + readl(tx_ring->tail), + tx_ring->next_to_use, + tx_ring->next_to_clean, + tx_buffer->time_stamp, + tx_buffer->next_to_watch, + jiffies, + tx_buffer->next_to_watch->wb.status); netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); @@ -2121,140 +2174,435 @@ static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) return !!budget; } -static void igc_nfc_filter_restore(struct igc_adapter *adapter) +static int igc_find_mac_filter(struct igc_adapter *adapter, + enum igc_mac_filter_type type, const u8 *addr) { - struct igc_nfc_filter *rule; + struct igc_hw *hw = &adapter->hw; + int max_entries = hw->mac.rar_entry_count; + u32 ral, rah; + int i; - spin_lock(&adapter->nfc_lock); + for (i = 0; i < max_entries; i++) { + ral = rd32(IGC_RAL(i)); + rah = rd32(IGC_RAH(i)); - hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) - igc_add_filter(adapter, rule); + if (!(rah & IGC_RAH_AV)) + continue; + if (!!(rah & IGC_RAH_ASEL_SRC_ADDR) != type) + continue; + if ((rah & IGC_RAH_RAH_MASK) != + le16_to_cpup((__le16 *)(addr + 4))) + continue; + if (ral != le32_to_cpup((__le32 *)(addr))) + continue; + + return i; + } - spin_unlock(&adapter->nfc_lock); + return -1; } -/* If the filter to be added and an already existing filter express - * the same address and address type, it should be possible to only - * override the other configurations, for example the queue to steer - * traffic. - */ -static bool igc_mac_entry_can_be_used(const struct igc_mac_addr *entry, - const u8 *addr, const u8 flags) +static int igc_get_avail_mac_filter_slot(struct igc_adapter *adapter) { - if (!(entry->state & IGC_MAC_STATE_IN_USE)) - return true; + struct igc_hw *hw = &adapter->hw; + int max_entries = hw->mac.rar_entry_count; + u32 rah; + int i; - if ((entry->state & IGC_MAC_STATE_SRC_ADDR) != - (flags & IGC_MAC_STATE_SRC_ADDR)) - return false; + for (i = 0; i < max_entries; i++) { + rah = rd32(IGC_RAH(i)); - if (!ether_addr_equal(addr, entry->addr)) - return false; + if (!(rah & IGC_RAH_AV)) + return i; + } - return true; + return -1; } -/* Add a MAC filter for 'addr' directing matching traffic to 'queue', - * 'flags' is used to indicate what kind of match is made, match is by - * default for the destination address, if matching by source address - * is desired the flag IGC_MAC_STATE_SRC_ADDR can be used. +/** + * igc_add_mac_filter() - Add MAC address filter + * @adapter: Pointer to adapter where the filter should be added + * @type: MAC address filter type (source or destination) + * @addr: MAC address + * @queue: If non-negative, queue assignment feature is enabled and frames + * matching the filter are enqueued onto 'queue'. Otherwise, queue + * assignment is disabled. + * + * Return: 0 in case of success, negative errno code otherwise. */ static int igc_add_mac_filter(struct igc_adapter *adapter, - const u8 *addr, const u8 queue) + enum igc_mac_filter_type type, const u8 *addr, + int queue) +{ + struct net_device *dev = adapter->netdev; + int index; + + index = igc_find_mac_filter(adapter, type, addr); + if (index >= 0) + goto update_filter; + + index = igc_get_avail_mac_filter_slot(adapter); + if (index < 0) + return -ENOSPC; + + netdev_dbg(dev, "Add MAC address filter: index %d type %s address %pM queue %d\n", + index, type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", + addr, queue); + +update_filter: + igc_set_mac_filter_hw(adapter, index, type, addr, queue); + return 0; +} + +/** + * igc_del_mac_filter() - Delete MAC address filter + * @adapter: Pointer to adapter where the filter should be deleted from + * @type: MAC address filter type (source or destination) + * @addr: MAC address + */ +static void igc_del_mac_filter(struct igc_adapter *adapter, + enum igc_mac_filter_type type, const u8 *addr) { + struct net_device *dev = adapter->netdev; + int index; + + index = igc_find_mac_filter(adapter, type, addr); + if (index < 0) + return; + + if (index == 0) { + /* If this is the default filter, we don't actually delete it. + * We just reset to its default value i.e. disable queue + * assignment. + */ + netdev_dbg(dev, "Disable default MAC filter queue assignment"); + + igc_set_mac_filter_hw(adapter, 0, type, addr, -1); + } else { + netdev_dbg(dev, "Delete MAC address filter: index %d type %s address %pM\n", + index, + type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", + addr); + + igc_clear_mac_filter_hw(adapter, index); + } +} + +/** + * igc_add_vlan_prio_filter() - Add VLAN priority filter + * @adapter: Pointer to adapter where the filter should be added + * @prio: VLAN priority value + * @queue: Queue number which matching frames are assigned to + * + * Return: 0 in case of success, negative errno code otherwise. + */ +static int igc_add_vlan_prio_filter(struct igc_adapter *adapter, int prio, + int queue) +{ + struct net_device *dev = adapter->netdev; struct igc_hw *hw = &adapter->hw; - int rar_entries = hw->mac.rar_entry_count; - int i; + u32 vlanpqf; - if (is_zero_ether_addr(addr)) - return -EINVAL; + vlanpqf = rd32(IGC_VLANPQF); - /* Search for the first empty entry in the MAC table. - * Do not touch entries at the end of the table reserved for the VF MAC - * addresses. - */ - for (i = 0; i < rar_entries; i++) { - if (!igc_mac_entry_can_be_used(&adapter->mac_table[i], - addr, 0)) - continue; + if (vlanpqf & IGC_VLANPQF_VALID(prio)) { + netdev_dbg(dev, "VLAN priority filter already in use\n"); + return -EEXIST; + } - ether_addr_copy(adapter->mac_table[i].addr, addr); - adapter->mac_table[i].queue = queue; - adapter->mac_table[i].state |= IGC_MAC_STATE_IN_USE; + vlanpqf |= IGC_VLANPQF_QSEL(prio, queue); + vlanpqf |= IGC_VLANPQF_VALID(prio); - igc_rar_set_index(adapter, i); - return i; + wr32(IGC_VLANPQF, vlanpqf); + + netdev_dbg(dev, "Add VLAN priority filter: prio %d queue %d\n", + prio, queue); + return 0; +} + +/** + * igc_del_vlan_prio_filter() - Delete VLAN priority filter + * @adapter: Pointer to adapter where the filter should be deleted from + * @prio: VLAN priority value + */ +static void igc_del_vlan_prio_filter(struct igc_adapter *adapter, int prio) +{ + struct igc_hw *hw = &adapter->hw; + u32 vlanpqf; + + vlanpqf = rd32(IGC_VLANPQF); + + vlanpqf &= ~IGC_VLANPQF_VALID(prio); + vlanpqf &= ~IGC_VLANPQF_QSEL(prio, IGC_VLANPQF_QUEUE_MASK); + + wr32(IGC_VLANPQF, vlanpqf); + + netdev_dbg(adapter->netdev, "Delete VLAN priority filter: prio %d\n", + prio); +} + +static int igc_get_avail_etype_filter_slot(struct igc_adapter *adapter) +{ + struct igc_hw *hw = &adapter->hw; + int i; + + for (i = 0; i < MAX_ETYPE_FILTER; i++) { + u32 etqf = rd32(IGC_ETQF(i)); + + if (!(etqf & IGC_ETQF_FILTER_ENABLE)) + return i; } - return -ENOSPC; + return -1; } -/* Remove a MAC filter for 'addr' directing matching traffic to - * 'queue', 'flags' is used to indicate what kind of match need to be - * removed, match is by default for the destination address, if - * matching by source address is to be removed the flag - * IGC_MAC_STATE_SRC_ADDR can be used. +/** + * igc_add_etype_filter() - Add ethertype filter + * @adapter: Pointer to adapter where the filter should be added + * @etype: Ethertype value + * @queue: If non-negative, queue assignment feature is enabled and frames + * matching the filter are enqueued onto 'queue'. Otherwise, queue + * assignment is disabled. + * + * Return: 0 in case of success, negative errno code otherwise. */ -static int igc_del_mac_filter(struct igc_adapter *adapter, - const u8 *addr, const u8 queue) +static int igc_add_etype_filter(struct igc_adapter *adapter, u16 etype, + int queue) +{ + struct igc_hw *hw = &adapter->hw; + int index; + u32 etqf; + + index = igc_get_avail_etype_filter_slot(adapter); + if (index < 0) + return -ENOSPC; + + etqf = rd32(IGC_ETQF(index)); + + etqf &= ~IGC_ETQF_ETYPE_MASK; + etqf |= etype; + + if (queue >= 0) { + etqf &= ~IGC_ETQF_QUEUE_MASK; + etqf |= (queue << IGC_ETQF_QUEUE_SHIFT); + etqf |= IGC_ETQF_QUEUE_ENABLE; + } + + etqf |= IGC_ETQF_FILTER_ENABLE; + + wr32(IGC_ETQF(index), etqf); + + netdev_dbg(adapter->netdev, "Add ethertype filter: etype %04x queue %d\n", + etype, queue); + return 0; +} + +static int igc_find_etype_filter(struct igc_adapter *adapter, u16 etype) { struct igc_hw *hw = &adapter->hw; - int rar_entries = hw->mac.rar_entry_count; int i; - if (is_zero_ether_addr(addr)) - return -EINVAL; + for (i = 0; i < MAX_ETYPE_FILTER; i++) { + u32 etqf = rd32(IGC_ETQF(i)); - /* Search for matching entry in the MAC table based on given address - * and queue. Do not touch entries at the end of the table reserved - * for the VF MAC addresses. - */ - for (i = 0; i < rar_entries; i++) { - if (!(adapter->mac_table[i].state & IGC_MAC_STATE_IN_USE)) - continue; - if (adapter->mac_table[i].state != 0) - continue; - if (adapter->mac_table[i].queue != queue) - continue; - if (!ether_addr_equal(adapter->mac_table[i].addr, addr)) - continue; + if ((etqf & IGC_ETQF_ETYPE_MASK) == etype) + return i; + } - /* When a filter for the default address is "deleted", - * we return it to its initial configuration - */ - if (adapter->mac_table[i].state & IGC_MAC_STATE_DEFAULT) { - adapter->mac_table[i].state = - IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE; - adapter->mac_table[i].queue = 0; - } else { - adapter->mac_table[i].state = 0; - adapter->mac_table[i].queue = 0; - memset(adapter->mac_table[i].addr, 0, ETH_ALEN); - } + return -1; +} - igc_rar_set_index(adapter, i); - return 0; +/** + * igc_del_etype_filter() - Delete ethertype filter + * @adapter: Pointer to adapter where the filter should be deleted from + * @etype: Ethertype value + */ +static void igc_del_etype_filter(struct igc_adapter *adapter, u16 etype) +{ + struct igc_hw *hw = &adapter->hw; + int index; + + index = igc_find_etype_filter(adapter, etype); + if (index < 0) + return; + + wr32(IGC_ETQF(index), 0); + + netdev_dbg(adapter->netdev, "Delete ethertype filter: etype %04x\n", + etype); +} + +static int igc_enable_nfc_rule(struct igc_adapter *adapter, + const struct igc_nfc_rule *rule) +{ + int err; + + if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { + err = igc_add_etype_filter(adapter, rule->filter.etype, + rule->action); + if (err) + return err; + } + + if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) { + err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, + rule->filter.src_addr, rule->action); + if (err) + return err; + } + + if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) { + err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, + rule->filter.dst_addr, rule->action); + if (err) + return err; + } + + if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { + int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >> + VLAN_PRIO_SHIFT; + + err = igc_add_vlan_prio_filter(adapter, prio, rule->action); + if (err) + return err; + } + + return 0; +} + +static void igc_disable_nfc_rule(struct igc_adapter *adapter, + const struct igc_nfc_rule *rule) +{ + if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) + igc_del_etype_filter(adapter, rule->filter.etype); + + if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { + int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >> + VLAN_PRIO_SHIFT; + + igc_del_vlan_prio_filter(adapter, prio); } - return -ENOENT; + if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) + igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, + rule->filter.src_addr); + + if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) + igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, + rule->filter.dst_addr); +} + +/** + * igc_get_nfc_rule() - Get NFC rule + * @adapter: Pointer to adapter + * @location: Rule location + * + * Context: Expects adapter->nfc_rule_lock to be held by caller. + * + * Return: Pointer to NFC rule at @location. If not found, NULL. + */ +struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter, + u32 location) +{ + struct igc_nfc_rule *rule; + + list_for_each_entry(rule, &adapter->nfc_rule_list, list) { + if (rule->location == location) + return rule; + if (rule->location > location) + break; + } + + return NULL; +} + +/** + * igc_del_nfc_rule() - Delete NFC rule + * @adapter: Pointer to adapter + * @rule: Pointer to rule to be deleted + * + * Disable NFC rule in hardware and delete it from adapter. + * + * Context: Expects adapter->nfc_rule_lock to be held by caller. + */ +void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) +{ + igc_disable_nfc_rule(adapter, rule); + + list_del(&rule->list); + adapter->nfc_rule_count--; + + kfree(rule); +} + +static void igc_flush_nfc_rules(struct igc_adapter *adapter) +{ + struct igc_nfc_rule *rule, *tmp; + + mutex_lock(&adapter->nfc_rule_lock); + + list_for_each_entry_safe(rule, tmp, &adapter->nfc_rule_list, list) + igc_del_nfc_rule(adapter, rule); + + mutex_unlock(&adapter->nfc_rule_lock); +} + +/** + * igc_add_nfc_rule() - Add NFC rule + * @adapter: Pointer to adapter + * @rule: Pointer to rule to be added + * + * Enable NFC rule in hardware and add it to adapter. + * + * Context: Expects adapter->nfc_rule_lock to be held by caller. + * + * Return: 0 on success, negative errno on failure. + */ +int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) +{ + struct igc_nfc_rule *pred, *cur; + int err; + + err = igc_enable_nfc_rule(adapter, rule); + if (err) + return err; + + pred = NULL; + list_for_each_entry(cur, &adapter->nfc_rule_list, list) { + if (cur->location >= rule->location) + break; + pred = cur; + } + + list_add(&rule->list, pred ? &pred->list : &adapter->nfc_rule_list); + adapter->nfc_rule_count++; + return 0; +} + +static void igc_restore_nfc_rules(struct igc_adapter *adapter) +{ + struct igc_nfc_rule *rule; + + mutex_lock(&adapter->nfc_rule_lock); + + list_for_each_entry_reverse(rule, &adapter->nfc_rule_list, list) + igc_enable_nfc_rule(adapter, rule); + + mutex_unlock(&adapter->nfc_rule_lock); } static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr) { struct igc_adapter *adapter = netdev_priv(netdev); - int ret; - - ret = igc_add_mac_filter(adapter, addr, adapter->num_rx_queues); - return min_t(int, ret, 0); + return igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr, -1); } static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr) { struct igc_adapter *adapter = netdev_priv(netdev); - igc_del_mac_filter(adapter, addr, adapter->num_rx_queues); - + igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr); return 0; } @@ -2325,7 +2673,9 @@ static void igc_configure(struct igc_adapter *adapter) igc_setup_mrqc(adapter); igc_setup_rctl(adapter); - igc_nfc_filter_restore(adapter); + igc_set_default_mac_filter(adapter); + igc_restore_nfc_rules(adapter); + igc_configure_tx(adapter); igc_configure_rx(adapter); @@ -2518,12 +2868,7 @@ void igc_set_flag_queue_pairs(struct igc_adapter *adapter, unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter) { - unsigned int max_rss_queues; - - /* Determine the maximum number of RSS queues supported. */ - max_rss_queues = IGC_MAX_RX_QUEUES; - - return max_rss_queues; + return IGC_MAX_RX_QUEUES; } static void igc_init_queue_configuration(struct igc_adapter *adapter) @@ -3164,14 +3509,14 @@ err_out: */ static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix) { - struct pci_dev *pdev = adapter->pdev; + struct net_device *dev = adapter->netdev; int err = 0; igc_set_interrupt_capability(adapter, msix); err = igc_alloc_q_vectors(adapter); if (err) { - dev_err(&pdev->dev, "Unable to allocate memory for vectors\n"); + netdev_err(dev, "Unable to allocate memory for vectors\n"); goto err_alloc_q_vectors; } @@ -3198,8 +3543,6 @@ static int igc_sw_init(struct igc_adapter *adapter) struct pci_dev *pdev = adapter->pdev; struct igc_hw *hw = &adapter->hw; - int size = sizeof(struct igc_mac_addr) * hw->mac.rar_entry_count; - pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word); /* set default ring sizes */ @@ -3218,20 +3561,19 @@ static int igc_sw_init(struct igc_adapter *adapter) VLAN_HLEN; adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; - spin_lock_init(&adapter->nfc_lock); + mutex_init(&adapter->nfc_rule_lock); + INIT_LIST_HEAD(&adapter->nfc_rule_list); + adapter->nfc_rule_count = 0; + spin_lock_init(&adapter->stats64_lock); /* Assume MSI-X interrupts, will be checked during IRQ allocation */ adapter->flags |= IGC_FLAG_HAS_MSIX; - adapter->mac_table = kzalloc(size, GFP_ATOMIC); - if (!adapter->mac_table) - return -ENOMEM; - igc_init_queue_configuration(adapter); /* This call may decrease the number of queues */ if (igc_init_interrupt_scheme(adapter, true)) { - dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); + netdev_err(netdev, "Unable to allocate memory for queues\n"); return -ENOMEM; } @@ -3359,8 +3701,6 @@ void igc_update_stats(struct igc_adapter *adapter) adapter->stats.prc511 += rd32(IGC_PRC511); adapter->stats.prc1023 += rd32(IGC_PRC1023); adapter->stats.prc1522 += rd32(IGC_PRC1522); - adapter->stats.symerrs += rd32(IGC_SYMERRS); - adapter->stats.sec += rd32(IGC_SEC); mpc = rd32(IGC_MPC); adapter->stats.mpc += mpc; @@ -3399,6 +3739,7 @@ void igc_update_stats(struct igc_adapter *adapter) adapter->stats.tpt += rd32(IGC_TPT); adapter->stats.colc += rd32(IGC_COLC); + adapter->stats.colc += rd32(IGC_RERC); adapter->stats.algnerrc += rd32(IGC_ALGNERRC); @@ -3449,21 +3790,6 @@ void igc_update_stats(struct igc_adapter *adapter) adapter->stats.mgpdc += rd32(IGC_MGTPDC); } -static void igc_nfc_filter_exit(struct igc_adapter *adapter) -{ - struct igc_nfc_filter *rule; - - spin_lock(&adapter->nfc_lock); - - hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) - igc_erase_filter(adapter, rule); - - hlist_for_each_entry(rule, &adapter->cls_flower_list, nfc_node) - igc_erase_filter(adapter, rule); - - spin_unlock(&adapter->nfc_lock); -} - /** * igc_down - Close the interface * @adapter: board private structure @@ -3482,8 +3808,6 @@ void igc_down(struct igc_adapter *adapter) wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN); /* flush and sleep below */ - igc_nfc_filter_exit(adapter); - /* set trans_start so we don't get spurious watchdogs during reset */ netif_trans_update(netdev); @@ -3577,8 +3901,7 @@ static int igc_change_mtu(struct net_device *netdev, int new_mtu) if (netif_running(netdev)) igc_down(adapter); - netdev_dbg(netdev, "changing MTU from %d to %d\n", - netdev->mtu, new_mtu); + netdev_dbg(netdev, "changing MTU from %d to %d\n", netdev->mtu, new_mtu); netdev->mtu = new_mtu; if (netif_running(netdev)) @@ -3633,20 +3956,8 @@ static int igc_set_features(struct net_device *netdev, if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) return 0; - if (!(features & NETIF_F_NTUPLE)) { - struct hlist_node *node2; - struct igc_nfc_filter *rule; - - spin_lock(&adapter->nfc_lock); - hlist_for_each_entry_safe(rule, node2, - &adapter->nfc_filter_list, nfc_node) { - igc_erase_filter(adapter, rule); - hlist_del(&rule->nfc_node); - kfree(rule); - } - spin_unlock(&adapter->nfc_lock); - adapter->nfc_filter_count = 0; - } + if (!(features & NETIF_F_NTUPLE)) + igc_flush_nfc_rules(adapter); netdev->features = features; @@ -3689,106 +4000,6 @@ igc_features_check(struct sk_buff *skb, struct net_device *dev, return features; } -/* Add a MAC filter for 'addr' directing matching traffic to 'queue', - * 'flags' is used to indicate what kind of match is made, match is by - * default for the destination address, if matching by source address - * is desired the flag IGC_MAC_STATE_SRC_ADDR can be used. - */ -static int igc_add_mac_filter_flags(struct igc_adapter *adapter, - const u8 *addr, const u8 queue, - const u8 flags) -{ - struct igc_hw *hw = &adapter->hw; - int rar_entries = hw->mac.rar_entry_count; - int i; - - if (is_zero_ether_addr(addr)) - return -EINVAL; - - /* Search for the first empty entry in the MAC table. - * Do not touch entries at the end of the table reserved for the VF MAC - * addresses. - */ - for (i = 0; i < rar_entries; i++) { - if (!igc_mac_entry_can_be_used(&adapter->mac_table[i], - addr, flags)) - continue; - - ether_addr_copy(adapter->mac_table[i].addr, addr); - adapter->mac_table[i].queue = queue; - adapter->mac_table[i].state |= IGC_MAC_STATE_IN_USE | flags; - - igc_rar_set_index(adapter, i); - return i; - } - - return -ENOSPC; -} - -int igc_add_mac_steering_filter(struct igc_adapter *adapter, - const u8 *addr, u8 queue, u8 flags) -{ - return igc_add_mac_filter_flags(adapter, addr, queue, - IGC_MAC_STATE_QUEUE_STEERING | flags); -} - -/* Remove a MAC filter for 'addr' directing matching traffic to - * 'queue', 'flags' is used to indicate what kind of match need to be - * removed, match is by default for the destination address, if - * matching by source address is to be removed the flag - * IGC_MAC_STATE_SRC_ADDR can be used. - */ -static int igc_del_mac_filter_flags(struct igc_adapter *adapter, - const u8 *addr, const u8 queue, - const u8 flags) -{ - struct igc_hw *hw = &adapter->hw; - int rar_entries = hw->mac.rar_entry_count; - int i; - - if (is_zero_ether_addr(addr)) - return -EINVAL; - - /* Search for matching entry in the MAC table based on given address - * and queue. Do not touch entries at the end of the table reserved - * for the VF MAC addresses. - */ - for (i = 0; i < rar_entries; i++) { - if (!(adapter->mac_table[i].state & IGC_MAC_STATE_IN_USE)) - continue; - if ((adapter->mac_table[i].state & flags) != flags) - continue; - if (adapter->mac_table[i].queue != queue) - continue; - if (!ether_addr_equal(adapter->mac_table[i].addr, addr)) - continue; - - /* When a filter for the default address is "deleted", - * we return it to its initial configuration - */ - if (adapter->mac_table[i].state & IGC_MAC_STATE_DEFAULT) { - adapter->mac_table[i].state = - IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE; - } else { - adapter->mac_table[i].state = 0; - adapter->mac_table[i].queue = 0; - memset(adapter->mac_table[i].addr, 0, ETH_ALEN); - } - - igc_rar_set_index(adapter, i); - return 0; - } - - return -ENOENT; -} - -int igc_del_mac_steering_filter(struct igc_adapter *adapter, - const u8 *addr, u8 queue, u8 flags) -{ - return igc_del_mac_filter_flags(adapter, addr, queue, - IGC_MAC_STATE_QUEUE_STEERING | flags); -} - static void igc_tsync_interrupt(struct igc_adapter *adapter) { struct igc_hw *hw = &adapter->hw; @@ -4009,7 +4220,6 @@ static void igc_watchdog_task(struct work_struct *work) struct igc_hw *hw = &adapter->hw; struct igc_phy_info *phy = &hw->phy; u16 phy_data, retry_count = 20; - u32 connsw; u32 link; int i; @@ -4022,14 +4232,6 @@ static void igc_watchdog_task(struct work_struct *work) link = false; } - /* Force link down if we have fiber to swap to */ - if (adapter->flags & IGC_FLAG_MAS_ENABLE) { - if (hw->phy.media_type == igc_media_type_copper) { - connsw = rd32(IGC_CONNSW); - if (!(connsw & IGC_CONNSW_AUTOSENSE_EN)) - link = 0; - } - } if (link) { /* Cancel scheduled suspend requests. */ pm_runtime_resume(netdev->dev.parent); @@ -4044,8 +4246,7 @@ static void igc_watchdog_task(struct work_struct *work) ctrl = rd32(IGC_CTRL); /* Link status message must follow this format */ netdev_info(netdev, - "igc: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", - netdev->name, + "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", adapter->link_speed, adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half", @@ -4083,10 +4284,10 @@ retry_read_status: retry_count--; goto retry_read_status; } else if (!retry_count) { - dev_err(&adapter->pdev->dev, "exceed max 2 second\n"); + netdev_err(netdev, "exceed max 2 second\n"); } } else { - dev_err(&adapter->pdev->dev, "read 1000Base-T Status Reg\n"); + netdev_err(netdev, "read 1000Base-T Status Reg\n"); } no_wait: netif_carrier_on(netdev); @@ -4102,8 +4303,7 @@ no_wait: adapter->link_duplex = 0; /* Links status message must follow this format */ - netdev_info(netdev, "igc: %s NIC Link is Down\n", - netdev->name); + netdev_info(netdev, "NIC Link is Down\n"); netif_carrier_off(netdev); /* link state has changed, schedule phy info update */ @@ -4321,8 +4521,7 @@ static int igc_request_irq(struct igc_adapter *adapter) netdev->name, adapter); if (err) - dev_err(&pdev->dev, "Error %d getting interrupt\n", - err); + netdev_err(netdev, "Error %d getting interrupt\n", err); request_done: return err; @@ -4424,7 +4623,7 @@ err_setup_tx: return err; } -static int igc_open(struct net_device *netdev) +int igc_open(struct net_device *netdev) { return __igc_open(netdev, false); } @@ -4466,7 +4665,7 @@ static int __igc_close(struct net_device *netdev, bool suspending) return 0; } -static int igc_close(struct net_device *netdev) +int igc_close(struct net_device *netdev) { if (netif_device_present(netdev) || netdev->dismantle) return __igc_close(netdev, false); @@ -4491,6 +4690,158 @@ static int igc_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) } } +static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue, + bool enable) +{ + struct igc_ring *ring; + int i; + + if (queue < 0 || queue >= adapter->num_tx_queues) + return -EINVAL; + + ring = adapter->tx_ring[queue]; + ring->launchtime_enable = enable; + + if (adapter->base_time) + return 0; + + adapter->cycle_time = NSEC_PER_SEC; + + for (i = 0; i < adapter->num_tx_queues; i++) { + ring = adapter->tx_ring[i]; + ring->start_time = 0; + ring->end_time = NSEC_PER_SEC; + } + + return 0; +} + +static bool validate_schedule(const struct tc_taprio_qopt_offload *qopt) +{ + int queue_uses[IGC_MAX_TX_QUEUES] = { }; + size_t n; + + if (qopt->cycle_time_extension) + return false; + + for (n = 0; n < qopt->num_entries; n++) { + const struct tc_taprio_sched_entry *e; + int i; + + e = &qopt->entries[n]; + + /* i225 only supports "global" frame preemption + * settings. + */ + if (e->command != TC_TAPRIO_CMD_SET_GATES) + return false; + + for (i = 0; i < IGC_MAX_TX_QUEUES; i++) { + if (e->gate_mask & BIT(i)) + queue_uses[i]++; + + if (queue_uses[i] > 1) + return false; + } + } + + return true; +} + +static int igc_tsn_enable_launchtime(struct igc_adapter *adapter, + struct tc_etf_qopt_offload *qopt) +{ + struct igc_hw *hw = &adapter->hw; + int err; + + if (hw->mac.type != igc_i225) + return -EOPNOTSUPP; + + err = igc_save_launchtime_params(adapter, qopt->queue, qopt->enable); + if (err) + return err; + + return igc_tsn_offload_apply(adapter); +} + +static int igc_save_qbv_schedule(struct igc_adapter *adapter, + struct tc_taprio_qopt_offload *qopt) +{ + u32 start_time = 0, end_time = 0; + size_t n; + + if (!qopt->enable) { + adapter->base_time = 0; + return 0; + } + + if (adapter->base_time) + return -EALREADY; + + if (!validate_schedule(qopt)) + return -EINVAL; + + adapter->cycle_time = qopt->cycle_time; + adapter->base_time = qopt->base_time; + + /* FIXME: be a little smarter about cases when the gate for a + * queue stays open for more than one entry. + */ + for (n = 0; n < qopt->num_entries; n++) { + struct tc_taprio_sched_entry *e = &qopt->entries[n]; + int i; + + end_time += e->interval; + + for (i = 0; i < IGC_MAX_TX_QUEUES; i++) { + struct igc_ring *ring = adapter->tx_ring[i]; + + if (!(e->gate_mask & BIT(i))) + continue; + + ring->start_time = start_time; + ring->end_time = end_time; + } + + start_time += e->interval; + } + + return 0; +} + +static int igc_tsn_enable_qbv_scheduling(struct igc_adapter *adapter, + struct tc_taprio_qopt_offload *qopt) +{ + struct igc_hw *hw = &adapter->hw; + int err; + + if (hw->mac.type != igc_i225) + return -EOPNOTSUPP; + + err = igc_save_qbv_schedule(adapter, qopt); + if (err) + return err; + + return igc_tsn_offload_apply(adapter); +} + +static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type, + void *type_data) +{ + struct igc_adapter *adapter = netdev_priv(dev); + + switch (type) { + case TC_SETUP_QDISC_TAPRIO: + return igc_tsn_enable_qbv_scheduling(adapter, type_data); + + case TC_SETUP_QDISC_ETF: + return igc_tsn_enable_launchtime(adapter, type_data); + + default: + return -EOPNOTSUPP; + } +} + static const struct net_device_ops igc_netdev_ops = { .ndo_open = igc_open, .ndo_stop = igc_close, @@ -4503,6 +4854,7 @@ static const struct net_device_ops igc_netdev_ops = { .ndo_set_features = igc_set_features, .ndo_features_check = igc_features_check, .ndo_do_ioctl = igc_ioctl, + .ndo_setup_tc = igc_setup_tc, }; /* PCIe configuration access */ @@ -4550,9 +4902,6 @@ u32 igc_rd32(struct igc_hw *hw, u32 reg) u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr); u32 value = 0; - if (IGC_REMOVED(hw_addr)) - return ~value; - value = readl(&hw_addr[reg]); /* reads should not return all F's */ @@ -4571,7 +4920,6 @@ u32 igc_rd32(struct igc_hw *hw, u32 reg) int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx) { - struct pci_dev *pdev = adapter->pdev; struct igc_mac_info *mac = &adapter->hw.mac; mac->autoneg = 0; @@ -4616,7 +4964,7 @@ int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx) return 0; err_inval: - dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n"); + netdev_err(adapter->netdev, "Unsupported Speed/Duplex configuration\n"); return -EINVAL; } @@ -4697,7 +5045,7 @@ static int igc_probe(struct pci_dev *pdev, hw->hw_addr = adapter->io_addr; netdev->netdev_ops = &igc_netdev_ops; - igc_set_ethtool_ops(netdev); + igc_ethtool_set_ops(netdev); netdev->watchdog_timeo = 5 * HZ; netdev->mem_start = pci_resource_start(pdev, 0); @@ -4723,9 +5071,21 @@ static int igc_probe(struct pci_dev *pdev, netdev->features |= NETIF_F_SG; netdev->features |= NETIF_F_TSO; netdev->features |= NETIF_F_TSO6; + netdev->features |= NETIF_F_TSO_ECN; netdev->features |= NETIF_F_RXCSUM; netdev->features |= NETIF_F_HW_CSUM; netdev->features |= NETIF_F_SCTP_CRC; + netdev->features |= NETIF_F_HW_TC; + +#define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ + NETIF_F_GSO_GRE_CSUM | \ + NETIF_F_GSO_IPXIP4 | \ + NETIF_F_GSO_IPXIP6 | \ + NETIF_F_GSO_UDP_TUNNEL | \ + NETIF_F_GSO_UDP_TUNNEL_CSUM) + + netdev->gso_partial_features = IGC_GSO_PARTIAL_FEATURES; + netdev->features |= NETIF_F_GSO_PARTIAL | IGC_GSO_PARTIAL_FEATURES; /* setup the private structure */ err = igc_sw_init(adapter); @@ -4750,8 +5110,7 @@ static int igc_probe(struct pci_dev *pdev, if (igc_get_flash_presence_i225(hw)) { if (hw->nvm.ops.validate(hw) < 0) { - dev_err(&pdev->dev, - "The NVM Checksum Is Not Valid\n"); + dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n"); err = -EIO; goto err_eeprom; } @@ -4865,6 +5224,8 @@ static void igc_remove(struct pci_dev *pdev) pm_runtime_get_noresume(&pdev->dev); + igc_flush_nfc_rules(adapter); + igc_ptp_stop(adapter); set_bit(__IGC_DOWN, &adapter->state); @@ -4885,7 +5246,6 @@ static void igc_remove(struct pci_dev *pdev) pci_iounmap(pdev, adapter->io_addr); pci_release_mem_regions(pdev); - kfree(adapter->mac_table); free_netdev(netdev); pci_disable_pcie_error_reporting(pdev); @@ -5014,8 +5374,7 @@ static int __maybe_unused igc_resume(struct device *dev) return -ENODEV; err = pci_enable_device_mem(pdev); if (err) { - dev_err(&pdev->dev, - "igc: Cannot enable PCI device from suspend\n"); + netdev_err(netdev, "Cannot enable PCI device from suspend\n"); return err; } pci_set_master(pdev); @@ -5024,7 +5383,7 @@ static int __maybe_unused igc_resume(struct device *dev) pci_enable_wake(pdev, PCI_D3cold, 0); if (igc_init_interrupt_scheme(adapter, true)) { - dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); + netdev_err(netdev, "Unable to allocate memory for queues\n"); return -ENOMEM; } @@ -5128,8 +5487,7 @@ static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev) pci_ers_result_t result; if (pci_enable_device_mem(pdev)) { - dev_err(&pdev->dev, - "Could not re-enable PCI device after reset.\n"); + netdev_err(netdev, "Could not re-enable PCI device after reset\n"); result = PCI_ERS_RESULT_DISCONNECT; } else { pci_set_master(pdev); @@ -5168,7 +5526,7 @@ static void igc_io_resume(struct pci_dev *pdev) rtnl_lock(); if (netif_running(netdev)) { if (igc_open(netdev)) { - dev_err(&pdev->dev, "igc_open failed after reset\n"); + netdev_err(netdev, "igc_open failed after reset\n"); return; } } @@ -5215,7 +5573,6 @@ static struct pci_driver igc_driver = { int igc_reinit_queues(struct igc_adapter *adapter) { struct net_device *netdev = adapter->netdev; - struct pci_dev *pdev = adapter->pdev; int err = 0; if (netif_running(netdev)) @@ -5224,7 +5581,7 @@ int igc_reinit_queues(struct igc_adapter *adapter) igc_reset_interrupt_capability(adapter); if (igc_init_interrupt_scheme(adapter, true)) { - dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); + netdev_err(netdev, "Unable to allocate memory for queues\n"); return -ENOMEM; } diff --git a/drivers/net/ethernet/intel/igc/igc_ptp.c b/drivers/net/ethernet/intel/igc/igc_ptp.c index f99c514ad0f4..0d746f8588c8 100644 --- a/drivers/net/ethernet/intel/igc/igc_ptp.c +++ b/drivers/net/ethernet/intel/igc/igc_ptp.c @@ -305,7 +305,6 @@ static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter, struct igc_hw *hw = &adapter->hw; u32 tsync_rx_cfg = 0; bool is_l4 = false; - bool is_l2 = false; u32 regval; /* reserved for future extensions */ @@ -346,7 +345,6 @@ static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter, case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: tsync_rx_ctl |= IGC_TSYNCRXCTL_TYPE_EVENT_V2; config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; - is_l2 = true; is_l4 = true; break; case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: @@ -370,7 +368,6 @@ static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter, tsync_rx_ctl |= IGC_TSYNCRXCTL_TYPE_ALL; tsync_rx_ctl |= IGC_TSYNCRXCTL_RXSYNSIG; config->rx_filter = HWTSTAMP_FILTER_ALL; - is_l2 = true; is_l4 = true; if (hw->mac.type == igc_i225) { @@ -405,15 +402,6 @@ static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter, /* define which PTP packets are time stamped */ wr32(IGC_TSYNCRXCFG, tsync_rx_cfg); - /* define ethertype filter for timestamped packets */ - if (is_l2) - wr32(IGC_ETQF(3), - (IGC_ETQF_FILTER_ENABLE | /* enable filter */ - IGC_ETQF_1588 | /* enable timestamping */ - ETH_P_1588)); /* 1588 eth protocol type */ - else - wr32(IGC_ETQF(3), 0); - /* L4 Queue Filter[3]: filter by destination port and protocol */ if (is_l4) { u32 ftqf = (IPPROTO_UDP /* UDP */ @@ -466,7 +454,7 @@ void igc_ptp_tx_hang(struct igc_adapter *adapter) * interrupt */ rd32(IGC_TXSTMPH); - dev_warn(&adapter->pdev->dev, "clearing Tx timestamp hang\n"); + netdev_warn(adapter->netdev, "Clearing Tx timestamp hang\n"); } } @@ -529,7 +517,7 @@ static void igc_ptp_tx_work(struct work_struct *work) * interrupt */ rd32(IGC_TXSTMPH); - dev_warn(&adapter->pdev->dev, "clearing Tx timestamp hang\n"); + netdev_warn(adapter->netdev, "Clearing Tx timestamp hang\n"); return; } @@ -626,10 +614,9 @@ void igc_ptp_init(struct igc_adapter *adapter) &adapter->pdev->dev); if (IS_ERR(adapter->ptp_clock)) { adapter->ptp_clock = NULL; - dev_err(&adapter->pdev->dev, "ptp_clock_register failed\n"); + netdev_err(netdev, "ptp_clock_register failed\n"); } else if (adapter->ptp_clock) { - dev_info(&adapter->pdev->dev, "added PHC on %s\n", - adapter->netdev->name); + netdev_info(netdev, "PHC added\n"); adapter->ptp_flags |= IGC_PTP_ENABLED; } } @@ -666,8 +653,7 @@ void igc_ptp_stop(struct igc_adapter *adapter) if (adapter->ptp_clock) { ptp_clock_unregister(adapter->ptp_clock); - dev_info(&adapter->pdev->dev, "removed PHC on %s\n", - adapter->netdev->name); + netdev_info(adapter->netdev, "PHC removed\n"); adapter->ptp_flags &= ~IGC_PTP_ENABLED; } } diff --git a/drivers/net/ethernet/intel/igc/igc_regs.h b/drivers/net/ethernet/intel/igc/igc_regs.h index d4af53a80f11..232e82dec62e 100644 --- a/drivers/net/ethernet/intel/igc/igc_regs.h +++ b/drivers/net/ethernet/intel/igc/igc_regs.h @@ -17,11 +17,6 @@ /* Internal Packet Buffer Size Registers */ #define IGC_RXPBS 0x02404 /* Rx Packet Buffer Size - RW */ #define IGC_TXPBS 0x03404 /* Tx Packet Buffer Size - RW */ -#define IGC_TDFH 0x03410 /* Tx Data FIFO Head - RW */ -#define IGC_TDFT 0x03418 /* Tx Data FIFO Tail - RW */ -#define IGC_TDFHS 0x03420 /* Tx Data FIFO Head Saved - RW */ -#define IGC_TDFTS 0x03428 /* Tx Data FIFO Tail Saved - RW */ -#define IGC_TDFPC 0x03430 /* Tx Data FIFO Packet Count - RW */ /* NVM Register Descriptions */ #define IGC_EERD 0x12014 /* EEprom mode read - RW */ @@ -35,10 +30,6 @@ #define IGC_FCRTL 0x02160 /* FC Receive Threshold Low - RW */ #define IGC_FCRTH 0x02168 /* FC Receive Threshold High - RW */ #define IGC_FCRTV 0x02460 /* FC Refresh Timer Value - RW */ -#define IGC_FCSTS 0x02464 /* FC Status - RO */ - -/* PCIe Register Description */ -#define IGC_GCR 0x05B00 /* PCIe control- RW */ /* Semaphore registers */ #define IGC_SW_FW_SYNC 0x05B5C /* SW-FW Synchronization - RW */ @@ -49,6 +40,7 @@ #define IGC_FACTPS 0x05B30 /* Interrupt Register Description */ +#define IGC_EICR 0x01580 /* Ext. Interrupt Cause read - W0 */ #define IGC_EICS 0x01520 /* Ext. Interrupt Cause Set - W0 */ #define IGC_EIMS 0x01524 /* Ext. Interrupt Mask Set/Read - RW */ #define IGC_EIMC 0x01528 /* Ext. Interrupt Mask Clear - WO */ @@ -76,13 +68,6 @@ #define IGC_ICRXDMTC 0x04120 /* Rx Descriptor Min Threshold Count */ #define IGC_ICRXOC 0x04124 /* Receiver Overrun Count */ -#define IGC_CBTMPC 0x0402C /* Circuit Breaker TX Packet Count */ -#define IGC_HTDPMC 0x0403C /* Host Transmit Discarded Packets */ -#define IGC_CBRMPC 0x040FC /* Circuit Breaker RX Packet Count */ -#define IGC_RPTHC 0x04104 /* Rx Packets To Host */ -#define IGC_HGPTC 0x04118 /* Host Good Packets TX Count */ -#define IGC_HTCBDPC 0x04124 /* Host TX Circ.Breaker Drop Count */ - /* MSI-X Table Register Descriptions */ #define IGC_PBACL 0x05B68 /* MSIx PBA Clear - R/W 1 to clear */ @@ -119,10 +104,11 @@ #define IGC_RLPML 0x05004 /* Rx Long Packet Max Length */ #define IGC_RFCTL 0x05008 /* Receive Filter Control*/ #define IGC_MTA 0x05200 /* Multicast Table Array - RW Array */ +#define IGC_RA 0x05400 /* Receive Address - RW Array */ #define IGC_UTA 0x0A000 /* Unicast Table Array - RW */ #define IGC_RAL(_n) (0x05400 + ((_n) * 0x08)) #define IGC_RAH(_n) (0x05404 + ((_n) * 0x08)) -#define IGC_VLAPQF 0x055B0 /* VLAN Priority Queue Filter VLAPQF */ +#define IGC_VLANPQF 0x055B0 /* VLAN Priority Queue Filter - RW */ /* Transmit Register Descriptions */ #define IGC_TCTL 0x00400 /* Tx Control - RW */ @@ -138,13 +124,9 @@ #define IGC_MMDAC 13 /* MMD Access Control */ #define IGC_MMDAAD 14 /* MMD Access Address/Data */ -/* Good transmitted packets counter registers */ -#define IGC_PQGPTC(_n) (0x010014 + (0x100 * (_n))) - /* Statistics Register Descriptions */ #define IGC_CRCERRS 0x04000 /* CRC Error Count - R/clr */ #define IGC_ALGNERRC 0x04004 /* Alignment Error Count - R/clr */ -#define IGC_SYMERRS 0x04008 /* Symbol Error Count - R/clr */ #define IGC_RXERRC 0x0400C /* Receive Error Count - R/clr */ #define IGC_MPC 0x04010 /* Missed Packet Count - R/clr */ #define IGC_SCC 0x04014 /* Single Collision Count - R/clr */ @@ -152,10 +134,10 @@ #define IGC_MCC 0x0401C /* Multiple Collision Count - R/clr */ #define IGC_LATECOL 0x04020 /* Late Collision Count - R/clr */ #define IGC_COLC 0x04028 /* Collision Count - R/clr */ +#define IGC_RERC 0x0402C /* Receive Error Count - R/clr */ #define IGC_DC 0x04030 /* Defer Count - R/clr */ #define IGC_TNCRS 0x04034 /* Tx-No CRS - R/clr */ -#define IGC_SEC 0x04038 /* Sequence Error Count - R/clr */ -#define IGC_CEXTERR 0x0403C /* Carrier Extension Error Count - R/clr */ +#define IGC_HTDPMC 0x0403C /* Host Transmit Discarded by MAC - R/clr */ #define IGC_RLEC 0x04040 /* Receive Length Error Count - R/clr */ #define IGC_XONRXC 0x04048 /* XON Rx Count - R/clr */ #define IGC_XONTXC 0x0404C /* XON Tx Count - R/clr */ @@ -213,7 +195,6 @@ #define IGC_HGOTCL 0x04130 /* Host Good Octets Transmit Count Low */ #define IGC_HGOTCH 0x04134 /* Host Good Octets Transmit Count High */ #define IGC_LENERRS 0x04138 /* Length Errors Count */ -#define IGC_HRMPC 0x0A018 /* Header Redirection Missed Packet Count */ /* Time sync registers */ #define IGC_TSICR 0x0B66C /* Time Sync Interrupt Cause */ @@ -229,7 +210,17 @@ #define IGC_FTQF(_n) (0x059E0 + (4 * (_n))) /* 5-tuple Queue Fltr */ -#define IGC_RXPBS 0x02404 /* Rx Packet Buffer Size - RW */ +/* Transmit Scheduling Registers */ +#define IGC_TQAVCTRL 0x3570 +#define IGC_TXQCTL(_n) (0x3344 + 0x4 * (_n)) +#define IGC_BASET_L 0x3314 +#define IGC_BASET_H 0x3318 +#define IGC_QBVCYCLET 0x331C +#define IGC_QBVCYCLET_S 0x3320 + +#define IGC_STQT(_n) (0x3324 + 0x4 * (_n)) +#define IGC_ENDQT(_n) (0x3334 + 0x4 * (_n)) +#define IGC_DTXMXPKTSZ 0x355C /* System Time Registers */ #define IGC_SYSTIML 0x0B600 /* System time register Low - RO */ @@ -265,8 +256,7 @@ u32 igc_rd32(struct igc_hw *hw, u32 reg); #define wr32(reg, val) \ do { \ u8 __iomem *hw_addr = READ_ONCE((hw)->hw_addr); \ - if (!IGC_REMOVED(hw_addr)) \ - writel((val), &hw_addr[(reg)]); \ + writel((val), &hw_addr[(reg)]); \ } while (0) #define rd32(reg) (igc_rd32(hw, reg)) diff --git a/drivers/net/ethernet/intel/igc/igc_tsn.c b/drivers/net/ethernet/intel/igc/igc_tsn.c new file mode 100644 index 000000000000..174103c4bea6 --- /dev/null +++ b/drivers/net/ethernet/intel/igc/igc_tsn.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 Intel Corporation */ + +#include "igc.h" +#include "igc_tsn.h" + +static bool is_any_launchtime(struct igc_adapter *adapter) +{ + int i; + + for (i = 0; i < adapter->num_tx_queues; i++) { + struct igc_ring *ring = adapter->tx_ring[i]; + + if (ring->launchtime_enable) + return true; + } + + return false; +} + +/* Returns the TSN specific registers to their default values after + * TSN offloading is disabled. + */ +static int igc_tsn_disable_offload(struct igc_adapter *adapter) +{ + struct igc_hw *hw = &adapter->hw; + u32 tqavctrl; + int i; + + if (!(adapter->flags & IGC_FLAG_TSN_QBV_ENABLED)) + return 0; + + adapter->cycle_time = 0; + + wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT); + wr32(IGC_DTXMXPKTSZ, IGC_DTXMXPKTSZ_DEFAULT); + + tqavctrl = rd32(IGC_TQAVCTRL); + tqavctrl &= ~(IGC_TQAVCTRL_TRANSMIT_MODE_TSN | + IGC_TQAVCTRL_ENHANCED_QAV); + wr32(IGC_TQAVCTRL, tqavctrl); + + for (i = 0; i < adapter->num_tx_queues; i++) { + struct igc_ring *ring = adapter->tx_ring[i]; + + ring->start_time = 0; + ring->end_time = 0; + ring->launchtime_enable = false; + + wr32(IGC_TXQCTL(i), 0); + wr32(IGC_STQT(i), 0); + wr32(IGC_ENDQT(i), NSEC_PER_SEC); + } + + wr32(IGC_QBVCYCLET_S, NSEC_PER_SEC); + wr32(IGC_QBVCYCLET, NSEC_PER_SEC); + + adapter->flags &= ~IGC_FLAG_TSN_QBV_ENABLED; + + return 0; +} + +static int igc_tsn_enable_offload(struct igc_adapter *adapter) +{ + struct igc_hw *hw = &adapter->hw; + u32 tqavctrl, baset_l, baset_h; + u32 sec, nsec, cycle; + ktime_t base_time, systim; + int i; + + if (adapter->flags & IGC_FLAG_TSN_QBV_ENABLED) + return 0; + + cycle = adapter->cycle_time; + base_time = adapter->base_time; + + wr32(IGC_TSAUXC, 0); + wr32(IGC_DTXMXPKTSZ, IGC_DTXMXPKTSZ_TSN); + wr32(IGC_TXPBS, IGC_TXPBSIZE_TSN); + + tqavctrl = rd32(IGC_TQAVCTRL); + tqavctrl |= IGC_TQAVCTRL_TRANSMIT_MODE_TSN | IGC_TQAVCTRL_ENHANCED_QAV; + wr32(IGC_TQAVCTRL, tqavctrl); + + wr32(IGC_QBVCYCLET_S, cycle); + wr32(IGC_QBVCYCLET, cycle); + + for (i = 0; i < adapter->num_tx_queues; i++) { + struct igc_ring *ring = adapter->tx_ring[i]; + u32 txqctl = 0; + + wr32(IGC_STQT(i), ring->start_time); + wr32(IGC_ENDQT(i), ring->end_time); + + if (adapter->base_time) { + /* If we have a base_time we are in "taprio" + * mode and we need to be strict about the + * cycles: only transmit a packet if it can be + * completed during that cycle. + */ + txqctl |= IGC_TXQCTL_STRICT_CYCLE | + IGC_TXQCTL_STRICT_END; + } + + if (ring->launchtime_enable) + txqctl |= IGC_TXQCTL_QUEUE_MODE_LAUNCHT; + + wr32(IGC_TXQCTL(i), txqctl); + } + + nsec = rd32(IGC_SYSTIML); + sec = rd32(IGC_SYSTIMH); + + systim = ktime_set(sec, nsec); + + if (ktime_compare(systim, base_time) > 0) { + s64 n; + + n = div64_s64(ktime_sub_ns(systim, base_time), cycle); + base_time = ktime_add_ns(base_time, (n + 1) * cycle); + } + + baset_h = div_s64_rem(base_time, NSEC_PER_SEC, &baset_l); + + wr32(IGC_BASET_H, baset_h); + wr32(IGC_BASET_L, baset_l); + + adapter->flags |= IGC_FLAG_TSN_QBV_ENABLED; + + return 0; +} + +int igc_tsn_offload_apply(struct igc_adapter *adapter) +{ + bool is_any_enabled = adapter->base_time || is_any_launchtime(adapter); + + if (!(adapter->flags & IGC_FLAG_TSN_QBV_ENABLED) && !is_any_enabled) + return 0; + + if (!is_any_enabled) { + int err = igc_tsn_disable_offload(adapter); + + if (err < 0) + return err; + + /* The BASET registers aren't cleared when writing + * into them, force a reset if the interface is + * running. + */ + if (netif_running(adapter->netdev)) + schedule_work(&adapter->reset_task); + + return 0; + } + + return igc_tsn_enable_offload(adapter); +} diff --git a/drivers/net/ethernet/intel/igc/igc_tsn.h b/drivers/net/ethernet/intel/igc/igc_tsn.h new file mode 100644 index 000000000000..f76bc86ddccd --- /dev/null +++ b/drivers/net/ethernet/intel/igc/igc_tsn.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2020 Intel Corporation */ + +#ifndef _IGC_TSN_H_ +#define _IGC_TSN_H_ + +int igc_tsn_offload_apply(struct igc_adapter *adapter); + +#endif /* _IGC_BASE_H */ diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index 2833e4f041ce..5ddfc83a1e46 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -224,17 +224,17 @@ struct ixgbe_tx_buffer { }; struct ixgbe_rx_buffer { - struct sk_buff *skb; - dma_addr_t dma; union { struct { + struct sk_buff *skb; + dma_addr_t dma; struct page *page; __u32 page_offset; __u16 pagecnt_bias; }; struct { - void *addr; - u64 handle; + bool discard; + struct xdp_buff *xdp; }; }; }; @@ -351,7 +351,6 @@ struct ixgbe_ring { }; struct xdp_rxq_info xdp_rxq; struct xdp_umem *xsk_umem; - struct zero_copy_allocator zca; /* ZC allocator anchor */ u16 ring_idx; /* {rx,tx,xdp}_ring back reference idx */ u16 rx_buf_len; } ____cacheline_internodealigned_in_smp; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c index 0bd1294ba517..17357a12cbdc 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c @@ -64,8 +64,7 @@ bool ixgbe_device_supports_autoneg_fc(struct ixgbe_hw *hw) hw->mac.ops.check_link(hw, &speed, &link_up, false); /* if link is down, assume supported */ if (link_up) - supported = speed == IXGBE_LINK_SPEED_1GB_FULL ? - true : false; + supported = speed == IXGBE_LINK_SPEED_1GB_FULL; else supported = true; } @@ -2243,7 +2242,7 @@ s32 ixgbe_fc_enable_generic(struct ixgbe_hw *hw) } /* Configure pause time (2 TCs per register) */ - reg = hw->fc.pause_time * 0x00010001; + reg = hw->fc.pause_time * 0x00010001U; for (i = 0; i < (MAX_TRAFFIC_CLASS / 2); i++) IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 718931d951bc..f162b8b8f345 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -35,7 +35,7 @@ #include <net/tc_act/tc_mirred.h> #include <net/vxlan.h> #include <net/mpls.h> -#include <net/xdp_sock.h> +#include <net/xdp_sock_drv.h> #include <net/xfrm.h> #include "ixgbe.h" @@ -2215,7 +2215,7 @@ static struct sk_buff *ixgbe_run_xdp(struct ixgbe_adapter *adapter, case XDP_PASS: break; case XDP_TX: - xdpf = convert_to_xdp_frame(xdp); + xdpf = xdp_convert_buff_to_frame(xdp); if (unlikely(!xdpf)) { result = IXGBE_XDP_CONSUMED; break; @@ -2244,19 +2244,30 @@ xdp_out: return ERR_PTR(-result); } +static unsigned int ixgbe_rx_frame_truesize(struct ixgbe_ring *rx_ring, + unsigned int size) +{ + unsigned int truesize; + +#if (PAGE_SIZE < 8192) + truesize = ixgbe_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */ +#else + truesize = ring_uses_build_skb(rx_ring) ? + SKB_DATA_ALIGN(IXGBE_SKB_PAD + size) + + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) : + SKB_DATA_ALIGN(size); +#endif + return truesize; +} + static void ixgbe_rx_buffer_flip(struct ixgbe_ring *rx_ring, struct ixgbe_rx_buffer *rx_buffer, unsigned int size) { + unsigned int truesize = ixgbe_rx_frame_truesize(rx_ring, size); #if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2; - rx_buffer->page_offset ^= truesize; #else - unsigned int truesize = ring_uses_build_skb(rx_ring) ? - SKB_DATA_ALIGN(IXGBE_SKB_PAD + size) : - SKB_DATA_ALIGN(size); - rx_buffer->page_offset += truesize; #endif } @@ -2290,6 +2301,11 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, xdp.rxq = &rx_ring->xdp_rxq; + /* Frame size depend on rx_ring setup when PAGE_SIZE=4K */ +#if (PAGE_SIZE < 8192) + xdp.frame_sz = ixgbe_rx_frame_truesize(rx_ring, 0); +#endif + while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; struct ixgbe_rx_buffer *rx_buffer; @@ -2323,7 +2339,10 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, xdp.data_hard_start = xdp.data - ixgbe_rx_offset(rx_ring); xdp.data_end = xdp.data + size; - +#if (PAGE_SIZE > 4096) + /* At larger PAGE_SIZE, frame_sz depend on len size */ + xdp.frame_sz = ixgbe_rx_frame_truesize(rx_ring, size); +#endif skb = ixgbe_run_xdp(adapter, rx_ring, &xdp); } @@ -2954,35 +2973,6 @@ static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter, /* skip the flush */ } -static inline void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter, - u64 qmask) -{ - u32 mask; - struct ixgbe_hw *hw = &adapter->hw; - - switch (hw->mac.type) { - case ixgbe_mac_82598EB: - mask = (IXGBE_EIMS_RTX_QUEUE & qmask); - IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); - break; - case ixgbe_mac_82599EB: - case ixgbe_mac_X540: - case ixgbe_mac_X550: - case ixgbe_mac_X550EM_x: - case ixgbe_mac_x550em_a: - mask = (qmask & 0xFFFFFFFF); - if (mask) - IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); - mask = (qmask >> 32); - if (mask) - IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); - break; - default: - break; - } - /* skip the flush */ -} - /** * ixgbe_irq_enable - Enable default interrupt generation settings * @adapter: board private structure @@ -3726,8 +3716,7 @@ static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter, /* configure the packet buffer length */ if (rx_ring->xsk_umem) { - u32 xsk_buf_len = rx_ring->xsk_umem->chunk_size_nohr - - XDP_PACKET_HEADROOM; + u32 xsk_buf_len = xsk_umem_get_rx_frame_size(rx_ring->xsk_umem); /* If the MAC support setting RXDCTL.RLPML, the * SRRCTL[n].BSIZEPKT is set to PAGE_SIZE and @@ -4074,11 +4063,10 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq); ring->xsk_umem = ixgbe_xsk_umem(adapter, ring); if (ring->xsk_umem) { - ring->zca.free = ixgbe_zca_free; WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, - MEM_TYPE_ZERO_COPY, - &ring->zca)); - + MEM_TYPE_XSK_BUFF_POOL, + NULL)); + xsk_buff_set_rxq_info(ring->xsk_umem, &ring->xdp_rxq); } else { WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, MEM_TYPE_PAGE_SHARED, NULL)); @@ -4134,8 +4122,7 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, } if (ring->xsk_umem && hw->mac.type != ixgbe_mac_82599EB) { - u32 xsk_buf_len = ring->xsk_umem->chunk_size_nohr - - XDP_PACKET_HEADROOM; + u32 xsk_buf_len = xsk_umem_get_rx_frame_size(ring->xsk_umem); rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | IXGBE_RXDCTL_RLPML_EN); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c index 537dfff585e0..d05a5690e66b 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c @@ -102,7 +102,7 @@ static int __ixgbe_enable_sriov(struct ixgbe_adapter *adapter, * indirection table and RSS hash key with PF therefore * we want to disable the querying by default. */ - adapter->vfinfo[i].rss_query_enabled = 0; + adapter->vfinfo[i].rss_query_enabled = false; /* Untrust all VFs */ adapter->vfinfo[i].trusted = false; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h index 6d01700b46bc..7887ae4aaf4f 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h @@ -35,7 +35,7 @@ int ixgbe_xsk_umem_setup(struct ixgbe_adapter *adapter, struct xdp_umem *umem, void ixgbe_zca_free(struct zero_copy_allocator *alloc, unsigned long handle); -void ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 cleaned_count); +bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 cleaned_count); int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, struct ixgbe_ring *rx_ring, const int budget); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c index 74b540ebb3dc..be9d2a8da515 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c @@ -2,7 +2,7 @@ /* Copyright(c) 2018 Intel Corporation. */ #include <linux/bpf_trace.h> -#include <net/xdp_sock.h> +#include <net/xdp_sock_drv.h> #include <net/xdp.h> #include "ixgbe.h" @@ -20,54 +20,11 @@ struct xdp_umem *ixgbe_xsk_umem(struct ixgbe_adapter *adapter, return xdp_get_umem_from_qid(adapter->netdev, qid); } -static int ixgbe_xsk_umem_dma_map(struct ixgbe_adapter *adapter, - struct xdp_umem *umem) -{ - struct device *dev = &adapter->pdev->dev; - unsigned int i, j; - dma_addr_t dma; - - for (i = 0; i < umem->npgs; i++) { - dma = dma_map_page_attrs(dev, umem->pgs[i], 0, PAGE_SIZE, - DMA_BIDIRECTIONAL, IXGBE_RX_DMA_ATTR); - if (dma_mapping_error(dev, dma)) - goto out_unmap; - - umem->pages[i].dma = dma; - } - - return 0; - -out_unmap: - for (j = 0; j < i; j++) { - dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE, - DMA_BIDIRECTIONAL, IXGBE_RX_DMA_ATTR); - umem->pages[i].dma = 0; - } - - return -1; -} - -static void ixgbe_xsk_umem_dma_unmap(struct ixgbe_adapter *adapter, - struct xdp_umem *umem) -{ - struct device *dev = &adapter->pdev->dev; - unsigned int i; - - for (i = 0; i < umem->npgs; i++) { - dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE, - DMA_BIDIRECTIONAL, IXGBE_RX_DMA_ATTR); - - umem->pages[i].dma = 0; - } -} - static int ixgbe_xsk_umem_enable(struct ixgbe_adapter *adapter, struct xdp_umem *umem, u16 qid) { struct net_device *netdev = adapter->netdev; - struct xdp_umem_fq_reuse *reuseq; bool if_running; int err; @@ -78,13 +35,7 @@ static int ixgbe_xsk_umem_enable(struct ixgbe_adapter *adapter, qid >= netdev->real_num_tx_queues) return -EINVAL; - reuseq = xsk_reuseq_prepare(adapter->rx_ring[0]->count); - if (!reuseq) - return -ENOMEM; - - xsk_reuseq_free(xsk_reuseq_swap(umem, reuseq)); - - err = ixgbe_xsk_umem_dma_map(adapter, umem); + err = xsk_buff_dma_map(umem, &adapter->pdev->dev, IXGBE_RX_DMA_ATTR); if (err) return err; @@ -124,7 +75,7 @@ static int ixgbe_xsk_umem_disable(struct ixgbe_adapter *adapter, u16 qid) ixgbe_txrx_ring_disable(adapter, qid); clear_bit(qid, adapter->af_xdp_zc_qps); - ixgbe_xsk_umem_dma_unmap(adapter, umem); + xsk_buff_dma_unmap(umem, IXGBE_RX_DMA_ATTR); if (if_running) ixgbe_txrx_ring_enable(adapter, qid); @@ -143,25 +94,20 @@ static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter, struct ixgbe_ring *rx_ring, struct xdp_buff *xdp) { - struct xdp_umem *umem = rx_ring->xsk_umem; int err, result = IXGBE_XDP_PASS; struct bpf_prog *xdp_prog; struct xdp_frame *xdpf; - u64 offset; u32 act; rcu_read_lock(); xdp_prog = READ_ONCE(rx_ring->xdp_prog); act = bpf_prog_run_xdp(xdp_prog, xdp); - offset = xdp->data - xdp->data_hard_start; - - xdp->handle = xsk_umem_adjust_offset(umem, xdp->handle, offset); switch (act) { case XDP_PASS: break; case XDP_TX: - xdpf = convert_to_xdp_frame(xdp); + xdpf = xdp_convert_buff_to_frame(xdp); if (unlikely(!xdpf)) { result = IXGBE_XDP_CONSUMED; break; @@ -186,140 +132,16 @@ static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter, return result; } -static struct -ixgbe_rx_buffer *ixgbe_get_rx_buffer_zc(struct ixgbe_ring *rx_ring, - unsigned int size) -{ - struct ixgbe_rx_buffer *bi; - - bi = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; - - /* we are reusing so sync this buffer for CPU use */ - dma_sync_single_range_for_cpu(rx_ring->dev, - bi->dma, 0, - size, - DMA_BIDIRECTIONAL); - - return bi; -} - -static void ixgbe_reuse_rx_buffer_zc(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *obi) -{ - u16 nta = rx_ring->next_to_alloc; - struct ixgbe_rx_buffer *nbi; - - nbi = &rx_ring->rx_buffer_info[rx_ring->next_to_alloc]; - /* update, and store next to alloc */ - nta++; - rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; - - /* transfer page from old buffer to new buffer */ - nbi->dma = obi->dma; - nbi->addr = obi->addr; - nbi->handle = obi->handle; - - obi->addr = NULL; - obi->skb = NULL; -} - -void ixgbe_zca_free(struct zero_copy_allocator *alloc, unsigned long handle) -{ - struct ixgbe_rx_buffer *bi; - struct ixgbe_ring *rx_ring; - u64 hr, mask; - u16 nta; - - rx_ring = container_of(alloc, struct ixgbe_ring, zca); - hr = rx_ring->xsk_umem->headroom + XDP_PACKET_HEADROOM; - mask = rx_ring->xsk_umem->chunk_mask; - - nta = rx_ring->next_to_alloc; - bi = rx_ring->rx_buffer_info; - - nta++; - rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; - - handle &= mask; - - bi->dma = xdp_umem_get_dma(rx_ring->xsk_umem, handle); - bi->dma += hr; - - bi->addr = xdp_umem_get_data(rx_ring->xsk_umem, handle); - bi->addr += hr; - - bi->handle = xsk_umem_adjust_offset(rx_ring->xsk_umem, (u64)handle, - rx_ring->xsk_umem->headroom); -} - -static bool ixgbe_alloc_buffer_zc(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *bi) -{ - struct xdp_umem *umem = rx_ring->xsk_umem; - void *addr = bi->addr; - u64 handle, hr; - - if (addr) - return true; - - if (!xsk_umem_peek_addr(umem, &handle)) { - rx_ring->rx_stats.alloc_rx_page_failed++; - return false; - } - - hr = umem->headroom + XDP_PACKET_HEADROOM; - - bi->dma = xdp_umem_get_dma(umem, handle); - bi->dma += hr; - - bi->addr = xdp_umem_get_data(umem, handle); - bi->addr += hr; - - bi->handle = xsk_umem_adjust_offset(umem, handle, umem->headroom); - - xsk_umem_release_addr(umem); - return true; -} - -static bool ixgbe_alloc_buffer_slow_zc(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *bi) -{ - struct xdp_umem *umem = rx_ring->xsk_umem; - u64 handle, hr; - - if (!xsk_umem_peek_addr_rq(umem, &handle)) { - rx_ring->rx_stats.alloc_rx_page_failed++; - return false; - } - - handle &= rx_ring->xsk_umem->chunk_mask; - - hr = umem->headroom + XDP_PACKET_HEADROOM; - - bi->dma = xdp_umem_get_dma(umem, handle); - bi->dma += hr; - - bi->addr = xdp_umem_get_data(umem, handle); - bi->addr += hr; - - bi->handle = xsk_umem_adjust_offset(umem, handle, umem->headroom); - - xsk_umem_release_addr_rq(umem); - return true; -} - -static __always_inline bool -__ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 cleaned_count, - bool alloc(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *bi)) +bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count) { union ixgbe_adv_rx_desc *rx_desc; struct ixgbe_rx_buffer *bi; u16 i = rx_ring->next_to_use; + dma_addr_t dma; bool ok = true; /* nothing to do */ - if (!cleaned_count) + if (!count) return true; rx_desc = IXGBE_RX_DESC(rx_ring, i); @@ -327,21 +149,18 @@ __ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 cleaned_count, i -= rx_ring->count; do { - if (!alloc(rx_ring, bi)) { + bi->xdp = xsk_buff_alloc(rx_ring->xsk_umem); + if (!bi->xdp) { ok = false; break; } - /* sync the buffer for use by the device */ - dma_sync_single_range_for_device(rx_ring->dev, bi->dma, - bi->page_offset, - rx_ring->rx_buf_len, - DMA_BIDIRECTIONAL); + dma = xsk_buff_xdp_get_dma(bi->xdp); /* Refresh the desc even if buffer_addrs didn't change * because each write-back erases this info. */ - rx_desc->read.pkt_addr = cpu_to_le64(bi->dma); + rx_desc->read.pkt_addr = cpu_to_le64(dma); rx_desc++; bi++; @@ -355,17 +174,14 @@ __ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 cleaned_count, /* clear the length for the next_to_use descriptor */ rx_desc->wb.upper.length = 0; - cleaned_count--; - } while (cleaned_count); + count--; + } while (count); i += rx_ring->count; if (rx_ring->next_to_use != i) { rx_ring->next_to_use = i; - /* update next to alloc since we have filled the ring */ - rx_ring->next_to_alloc = i; - /* Force memory writes to complete before letting h/w * know there are new descriptors to fetch. (Only * applicable for weak-ordered memory model archs, @@ -378,40 +194,27 @@ __ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 cleaned_count, return ok; } -void ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count) -{ - __ixgbe_alloc_rx_buffers_zc(rx_ring, count, - ixgbe_alloc_buffer_slow_zc); -} - -static bool ixgbe_alloc_rx_buffers_fast_zc(struct ixgbe_ring *rx_ring, - u16 count) -{ - return __ixgbe_alloc_rx_buffers_zc(rx_ring, count, - ixgbe_alloc_buffer_zc); -} - static struct sk_buff *ixgbe_construct_skb_zc(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *bi, - struct xdp_buff *xdp) + struct ixgbe_rx_buffer *bi) { - unsigned int metasize = xdp->data - xdp->data_meta; - unsigned int datasize = xdp->data_end - xdp->data; + unsigned int metasize = bi->xdp->data - bi->xdp->data_meta; + unsigned int datasize = bi->xdp->data_end - bi->xdp->data; struct sk_buff *skb; /* allocate a skb to store the frags */ skb = __napi_alloc_skb(&rx_ring->q_vector->napi, - xdp->data_end - xdp->data_hard_start, + bi->xdp->data_end - bi->xdp->data_hard_start, GFP_ATOMIC | __GFP_NOWARN); if (unlikely(!skb)) return NULL; - skb_reserve(skb, xdp->data - xdp->data_hard_start); - memcpy(__skb_put(skb, datasize), xdp->data, datasize); + skb_reserve(skb, bi->xdp->data - bi->xdp->data_hard_start); + memcpy(__skb_put(skb, datasize), bi->xdp->data, datasize); if (metasize) skb_metadata_set(skb, metasize); - ixgbe_reuse_rx_buffer_zc(rx_ring, bi); + xsk_buff_free(bi->xdp); + bi->xdp = NULL; return skb; } @@ -434,9 +237,6 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, unsigned int xdp_res, xdp_xmit = 0; bool failure = false; struct sk_buff *skb; - struct xdp_buff xdp; - - xdp.rxq = &rx_ring->xdp_rxq; while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; @@ -446,8 +246,8 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, /* return some buffers to hardware, one at a time is too slow */ if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) { failure = failure || - !ixgbe_alloc_rx_buffers_fast_zc(rx_ring, - cleaned_count); + !ixgbe_alloc_rx_buffers_zc(rx_ring, + cleaned_count); cleaned_count = 0; } @@ -462,42 +262,40 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, */ dma_rmb(); - bi = ixgbe_get_rx_buffer_zc(rx_ring, size); + bi = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; if (unlikely(!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) { struct ixgbe_rx_buffer *next_bi; - ixgbe_reuse_rx_buffer_zc(rx_ring, bi); + xsk_buff_free(bi->xdp); + bi->xdp = NULL; ixgbe_inc_ntc(rx_ring); next_bi = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; - next_bi->skb = ERR_PTR(-EINVAL); + next_bi->discard = true; continue; } - if (unlikely(bi->skb)) { - ixgbe_reuse_rx_buffer_zc(rx_ring, bi); + if (unlikely(bi->discard)) { + xsk_buff_free(bi->xdp); + bi->xdp = NULL; + bi->discard = false; ixgbe_inc_ntc(rx_ring); continue; } - xdp.data = bi->addr; - xdp.data_meta = xdp.data; - xdp.data_hard_start = xdp.data - XDP_PACKET_HEADROOM; - xdp.data_end = xdp.data + size; - xdp.handle = bi->handle; - - xdp_res = ixgbe_run_xdp_zc(adapter, rx_ring, &xdp); + bi->xdp->data_end = bi->xdp->data + size; + xsk_buff_dma_sync_for_cpu(bi->xdp); + xdp_res = ixgbe_run_xdp_zc(adapter, rx_ring, bi->xdp); if (xdp_res) { - if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) { + if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) xdp_xmit |= xdp_res; - bi->addr = NULL; - bi->skb = NULL; - } else { - ixgbe_reuse_rx_buffer_zc(rx_ring, bi); - } + else + xsk_buff_free(bi->xdp); + + bi->xdp = NULL; total_rx_packets++; total_rx_bytes += size; @@ -507,7 +305,7 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, } /* XDP_PASS path */ - skb = ixgbe_construct_skb_zc(rx_ring, bi, &xdp); + skb = ixgbe_construct_skb_zc(rx_ring, bi); if (!skb) { rx_ring->rx_stats.alloc_rx_buff_failed++; break; @@ -559,17 +357,17 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, void ixgbe_xsk_clean_rx_ring(struct ixgbe_ring *rx_ring) { - u16 i = rx_ring->next_to_clean; - struct ixgbe_rx_buffer *bi = &rx_ring->rx_buffer_info[i]; + struct ixgbe_rx_buffer *bi; + u16 i; - while (i != rx_ring->next_to_alloc) { - xsk_umem_fq_reuse(rx_ring->xsk_umem, bi->handle); - i++; - bi++; - if (i == rx_ring->count) { - i = 0; - bi = rx_ring->rx_buffer_info; - } + for (i = 0; i < rx_ring->count; i++) { + bi = &rx_ring->rx_buffer_info[i]; + + if (!bi->xdp) + continue; + + xsk_buff_free(bi->xdp); + bi->xdp = NULL; } } @@ -592,10 +390,9 @@ static bool ixgbe_xmit_zc(struct ixgbe_ring *xdp_ring, unsigned int budget) if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &desc)) break; - dma = xdp_umem_get_dma(xdp_ring->xsk_umem, desc.addr); - - dma_sync_single_for_device(xdp_ring->dev, dma, desc.len, - DMA_BIDIRECTIONAL); + dma = xsk_buff_raw_get_dma(xdp_ring->xsk_umem, desc.addr); + xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_umem, dma, + desc.len); tx_bi = &xdp_ring->tx_buffer_info[xdp_ring->next_to_use]; tx_bi->bytecount = desc.len; diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index 4622c4ea2e46..a39e2cb384dd 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -1095,19 +1095,31 @@ xdp_out: return ERR_PTR(-result); } +static unsigned int ixgbevf_rx_frame_truesize(struct ixgbevf_ring *rx_ring, + unsigned int size) +{ + unsigned int truesize; + +#if (PAGE_SIZE < 8192) + truesize = ixgbevf_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */ +#else + truesize = ring_uses_build_skb(rx_ring) ? + SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) + + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) : + SKB_DATA_ALIGN(size); +#endif + return truesize; +} + static void ixgbevf_rx_buffer_flip(struct ixgbevf_ring *rx_ring, struct ixgbevf_rx_buffer *rx_buffer, unsigned int size) { -#if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; + unsigned int truesize = ixgbevf_rx_frame_truesize(rx_ring, size); +#if (PAGE_SIZE < 8192) rx_buffer->page_offset ^= truesize; #else - unsigned int truesize = ring_uses_build_skb(rx_ring) ? - SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) : - SKB_DATA_ALIGN(size); - rx_buffer->page_offset += truesize; #endif } @@ -1125,6 +1137,11 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, xdp.rxq = &rx_ring->xdp_rxq; + /* Frame size depend on rx_ring setup when PAGE_SIZE=4K */ +#if (PAGE_SIZE < 8192) + xdp.frame_sz = ixgbevf_rx_frame_truesize(rx_ring, 0); +#endif + while (likely(total_rx_packets < budget)) { struct ixgbevf_rx_buffer *rx_buffer; union ixgbe_adv_rx_desc *rx_desc; @@ -1157,7 +1174,10 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, xdp.data_hard_start = xdp.data - ixgbevf_rx_offset(rx_ring); xdp.data_end = xdp.data + size; - +#if (PAGE_SIZE > 4096) + /* At larger PAGE_SIZE, frame_sz depend on len size */ + xdp.frame_sz = ixgbevf_rx_frame_truesize(rx_ring, size); +#endif skb = ixgbevf_run_xdp(adapter, rx_ring, &xdp); } |