diff options
Diffstat (limited to 'drivers/net/ethernet/freescale/fec_main.c')
| -rw-r--r-- | drivers/net/ethernet/freescale/fec_main.c | 1615 |
1 files changed, 923 insertions, 692 deletions
diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index 2341597408d1..c685a5c0cc51 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -22,66 +22,69 @@ * Copyright (C) 2010-2011 Freescale Semiconductor, Inc. */ -#include <linux/module.h> -#include <linux/kernel.h> -#include <linux/string.h> -#include <linux/pm_runtime.h> -#include <linux/ptrace.h> -#include <linux/errno.h> -#include <linux/ioport.h> -#include <linux/slab.h> -#include <linux/interrupt.h> +#include <linux/bitops.h> +#include <linux/bpf.h> +#include <linux/bpf_trace.h> +#include <linux/cacheflush.h> +#include <linux/clk.h> +#include <linux/crc32.h> #include <linux/delay.h> -#include <linux/netdevice.h> +#include <linux/errno.h> #include <linux/etherdevice.h> -#include <linux/skbuff.h> -#include <linux/in.h> -#include <linux/ip.h> -#include <net/ip.h> -#include <net/selftests.h> -#include <net/tso.h> -#include <linux/tcp.h> -#include <linux/udp.h> +#include <linux/fec.h> +#include <linux/filter.h> +#include <linux/gpio/consumer.h> #include <linux/icmp.h> -#include <linux/spinlock.h> -#include <linux/workqueue.h> -#include <linux/bitops.h> +#include <linux/if_vlan.h> +#include <linux/in.h> +#include <linux/interrupt.h> #include <linux/io.h> +#include <linux/ioport.h> +#include <linux/ip.h> #include <linux/irq.h> -#include <linux/clk.h> -#include <linux/crc32.h> -#include <linux/platform_device.h> +#include <linux/kernel.h> #include <linux/mdio.h> -#include <linux/phy.h> -#include <linux/fec.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/netdevice.h> #include <linux/of.h> -#include <linux/of_device.h> -#include <linux/of_gpio.h> #include <linux/of_mdio.h> #include <linux/of_net.h> -#include <linux/regulator/consumer.h> -#include <linux/if_vlan.h> +#include <linux/phy.h> #include <linux/pinctrl/consumer.h> +#include <linux/phy_fixed.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> #include <linux/prefetch.h> -#include <linux/mfd/syscon.h> +#include <linux/property.h> +#include <linux/ptrace.h> #include <linux/regmap.h> +#include <linux/regulator/consumer.h> +#include <linux/skbuff.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/string.h> +#include <linux/tcp.h> +#include <linux/udp.h> +#include <linux/workqueue.h> +#include <net/ip.h> +#include <net/page_pool/helpers.h> +#include <net/selftests.h> +#include <net/tso.h> #include <soc/imx/cpuidle.h> -#include <linux/filter.h> -#include <linux/bpf.h> - -#include <asm/cacheflush.h> #include "fec.h" static void set_multicast_list(struct net_device *ndev); static void fec_enet_itr_coal_set(struct net_device *ndev); +static int fec_enet_xdp_tx_xmit(struct fec_enet_private *fep, + int cpu, struct xdp_buff *xdp, + u32 dma_sync_len); #define DRIVER_NAME "fec" static const u16 fec_enet_vlan_pri_to_queue[8] = {0, 0, 1, 1, 1, 2, 2, 2}; -/* Pause frame feild and FIFO threshold */ -#define FEC_ENET_FCE (1 << 5) #define FEC_ENET_RSEM_V 0x84 #define FEC_ENET_RSFL_V 16 #define FEC_ENET_RAEM_V 0x8 @@ -100,18 +103,19 @@ struct fec_devinfo { static const struct fec_devinfo fec_imx25_info = { .quirks = FEC_QUIRK_USE_GASKET | FEC_QUIRK_MIB_CLEAR | - FEC_QUIRK_HAS_FRREG, + FEC_QUIRK_HAS_FRREG | FEC_QUIRK_HAS_MDIO_C45, }; static const struct fec_devinfo fec_imx27_info = { - .quirks = FEC_QUIRK_MIB_CLEAR | FEC_QUIRK_HAS_FRREG, + .quirks = FEC_QUIRK_MIB_CLEAR | FEC_QUIRK_HAS_FRREG | + FEC_QUIRK_HAS_MDIO_C45, }; static const struct fec_devinfo fec_imx28_info = { .quirks = FEC_QUIRK_ENET_MAC | FEC_QUIRK_SWAP_FRAME | FEC_QUIRK_SINGLE_MDIO | FEC_QUIRK_HAS_RACC | FEC_QUIRK_HAS_FRREG | FEC_QUIRK_CLEAR_SETUP_MII | - FEC_QUIRK_NO_HARD_RESET, + FEC_QUIRK_NO_HARD_RESET | FEC_QUIRK_HAS_MDIO_C45, }; static const struct fec_devinfo fec_imx6q_info = { @@ -119,20 +123,22 @@ static const struct fec_devinfo fec_imx6q_info = { FEC_QUIRK_HAS_BUFDESC_EX | FEC_QUIRK_HAS_CSUM | FEC_QUIRK_HAS_VLAN | FEC_QUIRK_ERR006358 | FEC_QUIRK_HAS_RACC | FEC_QUIRK_CLEAR_SETUP_MII | - FEC_QUIRK_HAS_PMQOS, + FEC_QUIRK_HAS_PMQOS | FEC_QUIRK_HAS_MDIO_C45, }; static const struct fec_devinfo fec_mvf600_info = { - .quirks = FEC_QUIRK_ENET_MAC | FEC_QUIRK_HAS_RACC, + .quirks = FEC_QUIRK_ENET_MAC | FEC_QUIRK_HAS_RACC | + FEC_QUIRK_HAS_MDIO_C45, }; -static const struct fec_devinfo fec_imx6x_info = { +static const struct fec_devinfo fec_imx6sx_info = { .quirks = FEC_QUIRK_ENET_MAC | FEC_QUIRK_HAS_GBIT | FEC_QUIRK_HAS_BUFDESC_EX | FEC_QUIRK_HAS_CSUM | FEC_QUIRK_HAS_VLAN | FEC_QUIRK_HAS_AVB | FEC_QUIRK_ERR007885 | FEC_QUIRK_BUG_CAPTURE | FEC_QUIRK_HAS_RACC | FEC_QUIRK_HAS_COALESCE | - FEC_QUIRK_CLEAR_SETUP_MII | FEC_QUIRK_HAS_MULTI_QUEUES, + FEC_QUIRK_CLEAR_SETUP_MII | FEC_QUIRK_HAS_MULTI_QUEUES | + FEC_QUIRK_HAS_MDIO_C45, }; static const struct fec_devinfo fec_imx6ul_info = { @@ -140,7 +146,8 @@ static const struct fec_devinfo fec_imx6ul_info = { FEC_QUIRK_HAS_BUFDESC_EX | FEC_QUIRK_HAS_CSUM | FEC_QUIRK_HAS_VLAN | FEC_QUIRK_ERR007885 | FEC_QUIRK_BUG_CAPTURE | FEC_QUIRK_HAS_RACC | - FEC_QUIRK_HAS_COALESCE | FEC_QUIRK_CLEAR_SETUP_MII, + FEC_QUIRK_HAS_COALESCE | FEC_QUIRK_CLEAR_SETUP_MII | + FEC_QUIRK_HAS_MDIO_C45, }; static const struct fec_devinfo fec_imx8mq_info = { @@ -150,7 +157,8 @@ static const struct fec_devinfo fec_imx8mq_info = { FEC_QUIRK_ERR007885 | FEC_QUIRK_BUG_CAPTURE | FEC_QUIRK_HAS_RACC | FEC_QUIRK_HAS_COALESCE | FEC_QUIRK_CLEAR_SETUP_MII | FEC_QUIRK_HAS_MULTI_QUEUES | - FEC_QUIRK_HAS_EEE | FEC_QUIRK_WAKEUP_FROM_INT2, + FEC_QUIRK_HAS_EEE | FEC_QUIRK_WAKEUP_FROM_INT2 | + FEC_QUIRK_HAS_MDIO_C45, }; static const struct fec_devinfo fec_imx8qm_info = { @@ -160,14 +168,16 @@ static const struct fec_devinfo fec_imx8qm_info = { FEC_QUIRK_ERR007885 | FEC_QUIRK_BUG_CAPTURE | FEC_QUIRK_HAS_RACC | FEC_QUIRK_HAS_COALESCE | FEC_QUIRK_CLEAR_SETUP_MII | FEC_QUIRK_HAS_MULTI_QUEUES | - FEC_QUIRK_DELAYED_CLKS_SUPPORT, + FEC_QUIRK_DELAYED_CLKS_SUPPORT | FEC_QUIRK_HAS_MDIO_C45 | + FEC_QUIRK_JUMBO_FRAME, }; static const struct fec_devinfo fec_s32v234_info = { .quirks = FEC_QUIRK_ENET_MAC | FEC_QUIRK_HAS_GBIT | FEC_QUIRK_HAS_BUFDESC_EX | FEC_QUIRK_HAS_CSUM | FEC_QUIRK_HAS_VLAN | FEC_QUIRK_HAS_AVB | - FEC_QUIRK_ERR007885 | FEC_QUIRK_BUG_CAPTURE, + FEC_QUIRK_ERR007885 | FEC_QUIRK_BUG_CAPTURE | + FEC_QUIRK_HAS_MDIO_C45, }; static struct platform_device_id fec_devtype[] = { @@ -176,65 +186,22 @@ static struct platform_device_id fec_devtype[] = { .name = DRIVER_NAME, .driver_data = 0, }, { - .name = "imx25-fec", - .driver_data = (kernel_ulong_t)&fec_imx25_info, - }, { - .name = "imx27-fec", - .driver_data = (kernel_ulong_t)&fec_imx27_info, - }, { - .name = "imx28-fec", - .driver_data = (kernel_ulong_t)&fec_imx28_info, - }, { - .name = "imx6q-fec", - .driver_data = (kernel_ulong_t)&fec_imx6q_info, - }, { - .name = "mvf600-fec", - .driver_data = (kernel_ulong_t)&fec_mvf600_info, - }, { - .name = "imx6sx-fec", - .driver_data = (kernel_ulong_t)&fec_imx6x_info, - }, { - .name = "imx6ul-fec", - .driver_data = (kernel_ulong_t)&fec_imx6ul_info, - }, { - .name = "imx8mq-fec", - .driver_data = (kernel_ulong_t)&fec_imx8mq_info, - }, { - .name = "imx8qm-fec", - .driver_data = (kernel_ulong_t)&fec_imx8qm_info, - }, { - .name = "s32v234-fec", - .driver_data = (kernel_ulong_t)&fec_s32v234_info, - }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(platform, fec_devtype); -enum imx_fec_type { - IMX25_FEC = 1, /* runs on i.mx25/50/53 */ - IMX27_FEC, /* runs on i.mx27/35/51 */ - IMX28_FEC, - IMX6Q_FEC, - MVF600_FEC, - IMX6SX_FEC, - IMX6UL_FEC, - IMX8MQ_FEC, - IMX8QM_FEC, - S32V234_FEC, -}; - static const struct of_device_id fec_dt_ids[] = { - { .compatible = "fsl,imx25-fec", .data = &fec_devtype[IMX25_FEC], }, - { .compatible = "fsl,imx27-fec", .data = &fec_devtype[IMX27_FEC], }, - { .compatible = "fsl,imx28-fec", .data = &fec_devtype[IMX28_FEC], }, - { .compatible = "fsl,imx6q-fec", .data = &fec_devtype[IMX6Q_FEC], }, - { .compatible = "fsl,mvf600-fec", .data = &fec_devtype[MVF600_FEC], }, - { .compatible = "fsl,imx6sx-fec", .data = &fec_devtype[IMX6SX_FEC], }, - { .compatible = "fsl,imx6ul-fec", .data = &fec_devtype[IMX6UL_FEC], }, - { .compatible = "fsl,imx8mq-fec", .data = &fec_devtype[IMX8MQ_FEC], }, - { .compatible = "fsl,imx8qm-fec", .data = &fec_devtype[IMX8QM_FEC], }, - { .compatible = "fsl,s32v234-fec", .data = &fec_devtype[S32V234_FEC], }, + { .compatible = "fsl,imx25-fec", .data = &fec_imx25_info, }, + { .compatible = "fsl,imx27-fec", .data = &fec_imx27_info, }, + { .compatible = "fsl,imx28-fec", .data = &fec_imx28_info, }, + { .compatible = "fsl,imx6q-fec", .data = &fec_imx6q_info, }, + { .compatible = "fsl,mvf600-fec", .data = &fec_mvf600_info, }, + { .compatible = "fsl,imx6sx-fec", .data = &fec_imx6sx_info, }, + { .compatible = "fsl,imx6ul-fec", .data = &fec_imx6ul_info, }, + { .compatible = "fsl,imx8mq-fec", .data = &fec_imx8mq_info, }, + { .compatible = "fsl,imx8qm-fec", .data = &fec_imx8qm_info, }, + { .compatible = "fsl,s32v234-fec", .data = &fec_s32v234_info, }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, fec_dt_ids); @@ -268,12 +235,13 @@ MODULE_PARM_DESC(macaddr, "FEC Ethernet MAC address"); * 2048 byte skbufs are allocated. However, alignment requirements * varies between FEC variants. Worst case is 64, so round down by 64. */ +#define MAX_JUMBO_BUF_SIZE (round_down(16384 - FEC_DRV_RESERVE_SPACE - 64, 64)) #define PKT_MAXBUF_SIZE (round_down(2048 - 64, 64)) #define PKT_MINBUF_SIZE 64 /* FEC receive acceleration */ -#define FEC_RACC_IPDIS (1 << 1) -#define FEC_RACC_PRODIS (1 << 2) +#define FEC_RACC_IPDIS BIT(1) +#define FEC_RACC_PRODIS BIT(2) #define FEC_RACC_SHIFT16 BIT(7) #define FEC_RACC_OPTIONS (FEC_RACC_IPDIS | FEC_RACC_PRODIS) @@ -285,12 +253,10 @@ MODULE_PARM_DESC(macaddr, "FEC Ethernet MAC address"); * size bits. Other FEC hardware does not, so we need to take that into * account when setting it. */ -#if defined(CONFIG_M523x) || defined(CONFIG_M527x) || defined(CONFIG_M528x) || \ - defined(CONFIG_M520x) || defined(CONFIG_M532x) || defined(CONFIG_ARM) || \ - defined(CONFIG_ARM64) -#define OPT_FRAME_SIZE (PKT_MAXBUF_SIZE << 16) +#ifndef CONFIG_M5272 +#define OPT_ARCH_HAS_MAX_FL 1 #else -#define OPT_FRAME_SIZE 0 +#define OPT_ARCH_HAS_MAX_FL 0 #endif /* FEC MII MMFR bits definition */ @@ -305,8 +271,26 @@ MODULE_PARM_DESC(macaddr, "FEC Ethernet MAC address"); #define FEC_MMFR_TA (2 << 16) #define FEC_MMFR_DATA(v) (v & 0xffff) /* FEC ECR bits definition */ -#define FEC_ECR_MAGICEN (1 << 2) -#define FEC_ECR_SLEEP (1 << 3) +#define FEC_ECR_RESET BIT(0) +#define FEC_ECR_ETHEREN BIT(1) +#define FEC_ECR_MAGICEN BIT(2) +#define FEC_ECR_SLEEP BIT(3) +#define FEC_ECR_EN1588 BIT(4) +#define FEC_ECR_SPEED BIT(5) +#define FEC_ECR_BYTESWP BIT(8) +/* FEC RCR bits definition */ +#define FEC_RCR_LOOP BIT(0) +#define FEC_RCR_DRT BIT(1) +#define FEC_RCR_MII BIT(2) +#define FEC_RCR_PROMISC BIT(3) +#define FEC_RCR_BC_REJ BIT(4) +#define FEC_RCR_FLOWCTL BIT(5) +#define FEC_RCR_RGMII BIT(6) +#define FEC_RCR_RMII BIT(8) +#define FEC_RCR_10BASET BIT(9) +#define FEC_RCR_NLC BIT(30) +/* TX WMARK bits */ +#define FEC_TXWMRK_STRFWD BIT(8) #define FEC_MII_TIMEOUT 30000 /* us */ @@ -319,8 +303,6 @@ MODULE_PARM_DESC(macaddr, "FEC Ethernet MAC address"); #define FEC_WOL_FLAG_ENABLE (0x1 << 1) #define FEC_WOL_FLAG_SLEEP_ON (0x1 << 2) -#define COPYBREAK_DEFAULT 256 - /* Max number of allowed TCP segments for software TSO */ #define FEC_MAX_TSO_SEGS 100 #define FEC_MAX_SKB_DESCS (FEC_MAX_TSO_SEGS * 2 + MAX_SKB_FRAGS) @@ -391,12 +373,76 @@ static void fec_dump(struct net_device *ndev) fec16_to_cpu(bdp->cbd_sc), fec32_to_cpu(bdp->cbd_bufaddr), fec16_to_cpu(bdp->cbd_datlen), - txq->tx_skbuff[index]); + txq->tx_buf[index].buf_p); bdp = fec_enet_get_nextdesc(bdp, &txq->bd); index++; } while (bdp != txq->bd.base); } +/* + * Coldfire does not support DMA coherent allocations, and has historically used + * a band-aid with a manual flush in fec_enet_rx_queue. + */ +#if defined(CONFIG_COLDFIRE) && !defined(CONFIG_COLDFIRE_COHERENT_DMA) +static void *fec_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, + gfp_t gfp) +{ + return dma_alloc_noncoherent(dev, size, handle, DMA_BIDIRECTIONAL, gfp); +} + +static void fec_dma_free(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t handle) +{ + dma_free_noncoherent(dev, size, cpu_addr, handle, DMA_BIDIRECTIONAL); +} +#else /* !CONFIG_COLDFIRE || CONFIG_COLDFIRE_COHERENT_DMA */ +static void *fec_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, + gfp_t gfp) +{ + return dma_alloc_coherent(dev, size, handle, gfp); +} + +static void fec_dma_free(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t handle) +{ + dma_free_coherent(dev, size, cpu_addr, handle); +} +#endif /* !CONFIG_COLDFIRE || CONFIG_COLDFIRE_COHERENT_DMA */ + +struct fec_dma_devres { + size_t size; + void *vaddr; + dma_addr_t dma_handle; +}; + +static void fec_dmam_release(struct device *dev, void *res) +{ + struct fec_dma_devres *this = res; + + fec_dma_free(dev, this->size, this->vaddr, this->dma_handle); +} + +static void *fec_dmam_alloc(struct device *dev, size_t size, dma_addr_t *handle, + gfp_t gfp) +{ + struct fec_dma_devres *dr; + void *vaddr; + + dr = devres_alloc(fec_dmam_release, sizeof(*dr), gfp); + if (!dr) + return NULL; + vaddr = fec_dma_alloc(dev, size, handle, gfp); + if (!vaddr) { + devres_free(dr); + return NULL; + } + dr->vaddr = vaddr; + dr->dma_handle = *handle; + dr->size = size; + devres_add(dev, dr); + return vaddr; +} + static inline bool is_ipv4_pkt(struct sk_buff *skb) { return skb->protocol == htons(ETH_P_IP) && ip_hdr(skb)->version == 4; @@ -425,14 +471,14 @@ fec_enet_create_page_pool(struct fec_enet_private *fep, { struct bpf_prog *xdp_prog = READ_ONCE(fep->xdp_prog); struct page_pool_params pp_params = { - .order = 0, + .order = fep->pagepool_order, .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, .pool_size = size, .nid = dev_to_node(&fep->pdev->dev), .dev = &fep->pdev->dev, .dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE, .offset = FEC_ENET_XDP_HEADROOM, - .max_len = FEC_ENET_RX_FRSIZE, + .max_len = fep->rx_frame_size, }; int err; @@ -648,7 +694,7 @@ static int fec_enet_txq_submit_skb(struct fec_enet_priv_tx_q *txq, index = fec_enet_get_bd_index(last_bdp, &txq->bd); /* Save skb pointer */ - txq->tx_skbuff[index] = skb; + txq->tx_buf[index].buf_p = skb; /* Make sure the updates to rest of the descriptor are performed before * transferring ownership. @@ -666,14 +712,17 @@ static int fec_enet_txq_submit_skb(struct fec_enet_priv_tx_q *txq, skb_tx_timestamp(skb); - /* Make sure the update to bdp and tx_skbuff are performed before - * txq->bd.cur. - */ + /* Make sure the update to bdp is performed before txq->bd.cur. */ wmb(); txq->bd.cur = bdp; /* Trigger transmission start */ - writel(0, txq->bd.reg_desc_active); + if (!(fep->quirks & FEC_QUIRK_ERR007885) || + !readl(txq->bd.reg_desc_active) || + !readl(txq->bd.reg_desc_active) || + !readl(txq->bd.reg_desc_active) || + !readl(txq->bd.reg_desc_active)) + writel(0, txq->bd.reg_desc_active); return 0; } @@ -799,6 +848,8 @@ static int fec_enet_txq_submit_tso(struct fec_enet_priv_tx_q *txq, struct fec_enet_private *fep = netdev_priv(ndev); int hdr_len, total_len, data_left; struct bufdesc *bdp = txq->bd.cur; + struct bufdesc *tmp_bdp; + struct bufdesc_ex *ebdp; struct tso_t tso; unsigned int index = 0; int ret; @@ -856,7 +907,7 @@ static int fec_enet_txq_submit_tso(struct fec_enet_priv_tx_q *txq, } /* Save skb pointer */ - txq->tx_skbuff[index] = skb; + txq->tx_buf[index].buf_p = skb; skb_tx_timestamp(skb); txq->bd.cur = bdp; @@ -872,7 +923,34 @@ static int fec_enet_txq_submit_tso(struct fec_enet_priv_tx_q *txq, return 0; err_release: - /* TODO: Release all used data descriptors for TSO */ + /* Release all used data descriptors for TSO */ + tmp_bdp = txq->bd.cur; + + while (tmp_bdp != bdp) { + /* Unmap data buffers */ + if (tmp_bdp->cbd_bufaddr && + !IS_TSO_HEADER(txq, fec32_to_cpu(tmp_bdp->cbd_bufaddr))) + dma_unmap_single(&fep->pdev->dev, + fec32_to_cpu(tmp_bdp->cbd_bufaddr), + fec16_to_cpu(tmp_bdp->cbd_datlen), + DMA_TO_DEVICE); + + /* Clear standard buffer descriptor fields */ + tmp_bdp->cbd_sc = 0; + tmp_bdp->cbd_datlen = 0; + tmp_bdp->cbd_bufaddr = 0; + + /* Handle extended descriptor if enabled */ + if (fep->bufdesc_ex) { + ebdp = (struct bufdesc_ex *)tmp_bdp; + ebdp->cbd_esc = 0; + } + + tmp_bdp = fec_enet_get_nextdesc(tmp_bdp, &txq->bd); + } + + dev_kfree_skb_any(skb); + return ret; } @@ -932,7 +1010,7 @@ static void fec_enet_bd_init(struct net_device *dev) /* Set the last buffer to wrap */ bdp = fec_enet_get_prevdesc(bdp, &rxq->bd); - bdp->cbd_sc |= cpu_to_fec16(BD_SC_WRAP); + bdp->cbd_sc |= cpu_to_fec16(BD_ENET_RX_WRAP); rxq->bd.cur = rxq->bd.base; } @@ -946,23 +1024,43 @@ static void fec_enet_bd_init(struct net_device *dev) for (i = 0; i < txq->bd.ring_size; i++) { /* Initialize the BD for every fragment in the page. */ bdp->cbd_sc = cpu_to_fec16(0); - if (bdp->cbd_bufaddr && - !IS_TSO_HEADER(txq, fec32_to_cpu(bdp->cbd_bufaddr))) - dma_unmap_single(&fep->pdev->dev, - fec32_to_cpu(bdp->cbd_bufaddr), - fec16_to_cpu(bdp->cbd_datlen), - DMA_TO_DEVICE); - if (txq->tx_skbuff[i]) { - dev_kfree_skb_any(txq->tx_skbuff[i]); - txq->tx_skbuff[i] = NULL; + if (txq->tx_buf[i].type == FEC_TXBUF_T_SKB) { + if (bdp->cbd_bufaddr && + !IS_TSO_HEADER(txq, fec32_to_cpu(bdp->cbd_bufaddr))) + dma_unmap_single(&fep->pdev->dev, + fec32_to_cpu(bdp->cbd_bufaddr), + fec16_to_cpu(bdp->cbd_datlen), + DMA_TO_DEVICE); + if (txq->tx_buf[i].buf_p) + dev_kfree_skb_any(txq->tx_buf[i].buf_p); + } else if (txq->tx_buf[i].type == FEC_TXBUF_T_XDP_NDO) { + if (bdp->cbd_bufaddr) + dma_unmap_single(&fep->pdev->dev, + fec32_to_cpu(bdp->cbd_bufaddr), + fec16_to_cpu(bdp->cbd_datlen), + DMA_TO_DEVICE); + + if (txq->tx_buf[i].buf_p) + xdp_return_frame(txq->tx_buf[i].buf_p); + } else { + struct page *page = txq->tx_buf[i].buf_p; + + if (page) + page_pool_put_page(pp_page_to_nmdesc(page)->pp, + page, 0, + false); } + + txq->tx_buf[i].buf_p = NULL; + /* restore default tx buffer type: FEC_TXBUF_T_SKB */ + txq->tx_buf[i].type = FEC_TXBUF_T_SKB; bdp->cbd_bufaddr = cpu_to_fec32(0); bdp = fec_enet_get_nextdesc(bdp, &txq->bd); } /* Set the last buffer to wrap */ bdp = fec_enet_get_prevdesc(bdp, &txq->bd); - bdp->cbd_sc |= cpu_to_fec16(BD_SC_WRAP); + bdp->cbd_sc |= cpu_to_fec16(BD_ENET_TX_WRAP); txq->dirty_tx = bdp; } } @@ -986,7 +1084,7 @@ static void fec_enet_enable_ring(struct net_device *ndev) for (i = 0; i < fep->num_rx_queues; i++) { rxq = fep->rx_queue[i]; writel(rxq->bd.dma, fep->hwp + FEC_R_DES_START(i)); - writel(PKT_MAXBUF_SIZE, fep->hwp + FEC_R_BUFF_SIZE(i)); + writel(fep->max_buf_size, fep->hwp + FEC_R_BUFF_SIZE(i)); /* enable DMA1/2 */ if (i) @@ -1005,24 +1103,40 @@ static void fec_enet_enable_ring(struct net_device *ndev) } } -static void fec_enet_reset_skb(struct net_device *ndev) +/* Whack a reset. We should wait for this. + * For i.MX6SX SOC, enet use AXI bus, we use disable MAC + * instead of reset MAC itself. + */ +static void fec_ctrl_reset(struct fec_enet_private *fep, bool allow_wol) { - struct fec_enet_private *fep = netdev_priv(ndev); - struct fec_enet_priv_tx_q *txq; - int i, j; - - for (i = 0; i < fep->num_tx_queues; i++) { - txq = fep->tx_queue[i]; + u32 val; - for (j = 0; j < txq->bd.ring_size; j++) { - if (txq->tx_skbuff[j]) { - dev_kfree_skb_any(txq->tx_skbuff[j]); - txq->tx_skbuff[j] = NULL; - } + if (!allow_wol || !(fep->wol_flag & FEC_WOL_FLAG_SLEEP_ON)) { + if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES || + ((fep->quirks & FEC_QUIRK_NO_HARD_RESET) && fep->link)) { + writel(0, fep->hwp + FEC_ECNTRL); + } else { + writel(FEC_ECR_RESET, fep->hwp + FEC_ECNTRL); + udelay(10); } + } else { + val = readl(fep->hwp + FEC_ECNTRL); + val |= (FEC_ECR_MAGICEN | FEC_ECR_SLEEP); + writel(val, fep->hwp + FEC_ECNTRL); } } +static void fec_set_hw_mac_addr(struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + + writel(ndev->dev_addr[3] | (ndev->dev_addr[2] << 8) | + (ndev->dev_addr[1] << 16) | (ndev->dev_addr[0] << 24), + fep->hwp + FEC_ADDR_LOW); + writel((ndev->dev_addr[5] << 16) | (ndev->dev_addr[4] << 24), + fep->hwp + FEC_ADDR_HIGH); +} + /* * This function is called to start or restart the FEC during a link * change, transmit timeout, or to reconfigure the FEC. The network @@ -1032,31 +1146,22 @@ static void fec_restart(struct net_device *ndev) { struct fec_enet_private *fep = netdev_priv(ndev); - u32 temp_mac[2]; - u32 rcntl = OPT_FRAME_SIZE | 0x04; - u32 ecntl = 0x2; /* ETHEREN */ + u32 ecntl = FEC_ECR_ETHEREN; + u32 rcntl = FEC_RCR_MII; - /* Whack a reset. We should wait for this. - * For i.MX6SX SOC, enet use AXI bus, we use disable MAC - * instead of reset MAC itself. - */ - if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES || - ((fep->quirks & FEC_QUIRK_NO_HARD_RESET) && fep->link)) { - writel(0, fep->hwp + FEC_ECNTRL); - } else { - writel(1, fep->hwp + FEC_ECNTRL); - udelay(10); - } + if (OPT_ARCH_HAS_MAX_FL) + rcntl |= (fep->netdev->mtu + ETH_HLEN + ETH_FCS_LEN) << 16; + + if (fep->bufdesc_ex) + fec_ptp_save_state(fep); + + fec_ctrl_reset(fep, false); /* * enet-mac reset will reset mac address registers too, * so need to reconfigure it. */ - memcpy(&temp_mac, ndev->dev_addr, ETH_ALEN); - writel((__force u32)cpu_to_be32(temp_mac[0]), - fep->hwp + FEC_ADDR_LOW); - writel((__force u32)cpu_to_be32(temp_mac[1]), - fep->hwp + FEC_ADDR_HIGH); + fec_set_hw_mac_addr(ndev); /* Clear any outstanding interrupt, except MDIO. */ writel((0xffffffff & ~FEC_ENET_MII), fep->hwp + FEC_IEVENT); @@ -1065,16 +1170,13 @@ fec_restart(struct net_device *ndev) fec_enet_enable_ring(ndev); - /* Reset tx SKB buffers. */ - fec_enet_reset_skb(ndev); - /* Enable MII mode */ if (fep->full_duplex == DUPLEX_FULL) { /* FD enable */ writel(0x04, fep->hwp + FEC_X_CNTRL); } else { /* No Rcv on Xmit */ - rcntl |= 0x02; + rcntl |= FEC_RCR_DRT; writel(0x0, fep->hwp + FEC_X_CNTRL); } @@ -1093,7 +1195,7 @@ fec_restart(struct net_device *ndev) else val &= ~FEC_RACC_OPTIONS; writel(val, fep->hwp + FEC_RACC); - writel(PKT_MAXBUF_SIZE, fep->hwp + FEC_FTRL); + writel(min(fep->rx_frame_size, fep->max_buf_size), fep->hwp + FEC_FTRL); } #endif @@ -1103,27 +1205,24 @@ fec_restart(struct net_device *ndev) */ if (fep->quirks & FEC_QUIRK_ENET_MAC) { /* Enable flow control and length check */ - rcntl |= 0x40000000 | 0x00000020; + rcntl |= FEC_RCR_NLC | FEC_RCR_FLOWCTL; /* RGMII, RMII or MII */ - if (fep->phy_interface == PHY_INTERFACE_MODE_RGMII || - fep->phy_interface == PHY_INTERFACE_MODE_RGMII_ID || - fep->phy_interface == PHY_INTERFACE_MODE_RGMII_RXID || - fep->phy_interface == PHY_INTERFACE_MODE_RGMII_TXID) - rcntl |= (1 << 6); + if (phy_interface_mode_is_rgmii(fep->phy_interface)) + rcntl |= FEC_RCR_RGMII; else if (fep->phy_interface == PHY_INTERFACE_MODE_RMII) - rcntl |= (1 << 8); + rcntl |= FEC_RCR_RMII; else - rcntl &= ~(1 << 8); + rcntl &= ~FEC_RCR_RMII; /* 1G, 100M or 10M */ if (ndev->phydev) { if (ndev->phydev->speed == SPEED_1000) - ecntl |= (1 << 5); + ecntl |= FEC_ECR_SPEED; else if (ndev->phydev->speed == SPEED_100) - rcntl &= ~(1 << 9); + rcntl &= ~FEC_RCR_10BASET; else - rcntl |= (1 << 9); + rcntl |= FEC_RCR_10BASET; } } else { #ifdef FEC_MIIGSK_ENR @@ -1156,7 +1255,7 @@ fec_restart(struct net_device *ndev) if ((fep->pause_flag & FEC_PAUSE_FLAG_ENABLE) || ((fep->pause_flag & FEC_PAUSE_FLAG_AUTONEG) && ndev->phydev && ndev->phydev->pause)) { - rcntl |= FEC_ENET_FCE; + rcntl |= FEC_RCR_FLOWCTL; /* set FIFO threshold parameter to reduce overrun */ writel(FEC_ENET_RSEM_V, fep->hwp + FEC_R_FIFO_RSEM); @@ -1167,7 +1266,7 @@ fec_restart(struct net_device *ndev) /* OPD */ writel(FEC_ENET_OPD_V, fep->hwp + FEC_OPD); } else { - rcntl &= ~FEC_ENET_FCE; + rcntl &= ~FEC_RCR_FLOWCTL; } #endif /* !defined(CONFIG_M5272) */ @@ -1182,13 +1281,23 @@ fec_restart(struct net_device *ndev) if (fep->quirks & FEC_QUIRK_ENET_MAC) { /* enable ENET endian swap */ - ecntl |= (1 << 8); - /* enable ENET store and forward mode */ - writel(1 << 8, fep->hwp + FEC_X_WMRK); + ecntl |= FEC_ECR_BYTESWP; + + /* When Jumbo Frame is enabled, the FIFO may not be large enough + * to hold an entire frame. In such cases, if the MTU exceeds + * (PKT_MAXBUF_SIZE - ETH_HLEN - ETH_FCS_LEN), configure the interface + * to operate in cut-through mode, triggered by the FIFO threshold. + * Otherwise, enable the ENET store-and-forward mode. + */ + if ((fep->quirks & FEC_QUIRK_JUMBO_FRAME) && + (ndev->mtu > (PKT_MAXBUF_SIZE - ETH_HLEN - ETH_FCS_LEN))) + writel(0xF, fep->hwp + FEC_X_WMRK); + else + writel(FEC_TXWMRK_STRFWD, fep->hwp + FEC_X_WMRK); } if (fep->bufdesc_ex) - ecntl |= (1 << 4); + ecntl |= FEC_ECR_EN1588; if (fep->quirks & FEC_QUIRK_DELAYED_CLKS_SUPPORT && fep->rgmii_txc_dly) @@ -1206,8 +1315,10 @@ fec_restart(struct net_device *ndev) writel(ecntl, fep->hwp + FEC_ECNTRL); fec_enet_active_rxring(ndev); - if (fep->bufdesc_ex) + if (fep->bufdesc_ex) { fec_ptp_start_cyclecounter(ndev); + fec_ptp_restore_state(fep); + } /* Enable interrupts we wish to service */ if (fep->link) @@ -1287,7 +1398,7 @@ static void fec_stop(struct net_device *ndev) { struct fec_enet_private *fep = netdev_priv(ndev); - u32 rmii_mode = readl(fep->hwp + FEC_R_CNTRL) & (1 << 8); + u32 rmii_mode = readl(fep->hwp + FEC_R_CNTRL) & FEC_RCR_RMII; u32 val; /* We cannot expect a graceful transmit stop without link !!! */ @@ -1298,33 +1409,29 @@ fec_stop(struct net_device *ndev) netdev_err(ndev, "Graceful transmit stop did not complete!\n"); } - /* Whack a reset. We should wait for this. - * For i.MX6SX SOC, enet use AXI bus, we use disable MAC - * instead of reset MAC itself. - */ - if (!(fep->wol_flag & FEC_WOL_FLAG_SLEEP_ON)) { - if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES) { - writel(0, fep->hwp + FEC_ECNTRL); - } else { - writel(1, fep->hwp + FEC_ECNTRL); - udelay(10); - } - } else { - val = readl(fep->hwp + FEC_ECNTRL); - val |= (FEC_ECR_MAGICEN | FEC_ECR_SLEEP); - writel(val, fep->hwp + FEC_ECNTRL); - } + if (fep->bufdesc_ex) + fec_ptp_save_state(fep); + + fec_ctrl_reset(fep, true); writel(fep->phy_speed, fep->hwp + FEC_MII_SPEED); writel(FEC_DEFAULT_IMASK, fep->hwp + FEC_IMASK); /* We have to keep ENET enabled to have MII interrupt stay working */ if (fep->quirks & FEC_QUIRK_ENET_MAC && !(fep->wol_flag & FEC_WOL_FLAG_SLEEP_ON)) { - writel(2, fep->hwp + FEC_ECNTRL); + writel(FEC_ECR_ETHEREN, fep->hwp + FEC_ECNTRL); writel(rmii_mode, fep->hwp + FEC_R_CNTRL); } -} + if (fep->bufdesc_ex) { + val = readl(fep->hwp + FEC_ECNTRL); + val |= FEC_ECR_EN1588; + writel(val, fep->hwp + FEC_ECNTRL); + + fec_ptp_start_cyclecounter(ndev); + fec_ptp_restore_state(fep); + } +} static void fec_timeout(struct net_device *ndev, unsigned int txqueue) @@ -1372,9 +1479,10 @@ fec_enet_hwtstamp(struct fec_enet_private *fep, unsigned ts, } static void -fec_enet_tx_queue(struct net_device *ndev, u16 queue_id) +fec_enet_tx_queue(struct net_device *ndev, u16 queue_id, int budget) { struct fec_enet_private *fep; + struct xdp_frame *xdpf; struct bufdesc *bdp; unsigned short status; struct sk_buff *skb; @@ -1382,6 +1490,8 @@ fec_enet_tx_queue(struct net_device *ndev, u16 queue_id) struct netdev_queue *nq; int index = 0; int entries_free; + struct page *page; + int frame_len; fep = netdev_priv(ndev); @@ -1402,16 +1512,45 @@ fec_enet_tx_queue(struct net_device *ndev, u16 queue_id) index = fec_enet_get_bd_index(bdp, &txq->bd); - skb = txq->tx_skbuff[index]; - txq->tx_skbuff[index] = NULL; - if (!IS_TSO_HEADER(txq, fec32_to_cpu(bdp->cbd_bufaddr))) - dma_unmap_single(&fep->pdev->dev, - fec32_to_cpu(bdp->cbd_bufaddr), - fec16_to_cpu(bdp->cbd_datlen), - DMA_TO_DEVICE); - bdp->cbd_bufaddr = cpu_to_fec32(0); - if (!skb) - goto skb_done; + if (txq->tx_buf[index].type == FEC_TXBUF_T_SKB) { + skb = txq->tx_buf[index].buf_p; + if (bdp->cbd_bufaddr && + !IS_TSO_HEADER(txq, fec32_to_cpu(bdp->cbd_bufaddr))) + dma_unmap_single(&fep->pdev->dev, + fec32_to_cpu(bdp->cbd_bufaddr), + fec16_to_cpu(bdp->cbd_datlen), + DMA_TO_DEVICE); + bdp->cbd_bufaddr = cpu_to_fec32(0); + if (!skb) + goto tx_buf_done; + } else { + /* Tx processing cannot call any XDP (or page pool) APIs if + * the "budget" is 0. Because NAPI is called with budget of + * 0 (such as netpoll) indicates we may be in an IRQ context, + * however, we can't use the page pool from IRQ context. + */ + if (unlikely(!budget)) + break; + + if (txq->tx_buf[index].type == FEC_TXBUF_T_XDP_NDO) { + xdpf = txq->tx_buf[index].buf_p; + if (bdp->cbd_bufaddr) + dma_unmap_single(&fep->pdev->dev, + fec32_to_cpu(bdp->cbd_bufaddr), + fec16_to_cpu(bdp->cbd_datlen), + DMA_TO_DEVICE); + } else { + page = txq->tx_buf[index].buf_p; + } + + bdp->cbd_bufaddr = cpu_to_fec32(0); + if (unlikely(!txq->tx_buf[index].buf_p)) { + txq->tx_buf[index].type = FEC_TXBUF_T_SKB; + goto tx_buf_done; + } + + frame_len = fec16_to_cpu(bdp->cbd_datlen); + } /* Check for errors. */ if (status & (BD_ENET_TX_HB | BD_ENET_TX_LC | @@ -1430,21 +1569,11 @@ fec_enet_tx_queue(struct net_device *ndev, u16 queue_id) ndev->stats.tx_carrier_errors++; } else { ndev->stats.tx_packets++; - ndev->stats.tx_bytes += skb->len; - } - /* NOTE: SKBTX_IN_PROGRESS being set does not imply it's we who - * are to time stamp the packet, so we still need to check time - * stamping enabled flag. - */ - if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS && - fep->hwts_tx_en) && - fep->bufdesc_ex) { - struct skb_shared_hwtstamps shhwtstamps; - struct bufdesc_ex *ebdp = (struct bufdesc_ex *)bdp; - - fec_enet_hwtstamp(fep, fec32_to_cpu(ebdp->ts), &shhwtstamps); - skb_tstamp_tx(skb, &shhwtstamps); + if (txq->tx_buf[index].type == FEC_TXBUF_T_SKB) + ndev->stats.tx_bytes += skb->len; + else + ndev->stats.tx_bytes += frame_len; } /* Deferred means some collisions occurred during transmit, @@ -1453,10 +1582,36 @@ fec_enet_tx_queue(struct net_device *ndev, u16 queue_id) if (status & BD_ENET_TX_DEF) ndev->stats.collisions++; - /* Free the sk buffer associated with this last transmit */ - dev_kfree_skb_any(skb); -skb_done: - /* Make sure the update to bdp and tx_skbuff are performed + if (txq->tx_buf[index].type == FEC_TXBUF_T_SKB) { + /* NOTE: SKBTX_IN_PROGRESS being set does not imply it's we who + * are to time stamp the packet, so we still need to check time + * stamping enabled flag. + */ + if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS && + fep->hwts_tx_en) && fep->bufdesc_ex) { + struct skb_shared_hwtstamps shhwtstamps; + struct bufdesc_ex *ebdp = (struct bufdesc_ex *)bdp; + + fec_enet_hwtstamp(fep, fec32_to_cpu(ebdp->ts), &shhwtstamps); + skb_tstamp_tx(skb, &shhwtstamps); + } + + /* Free the sk buffer associated with this last transmit */ + napi_consume_skb(skb, budget); + } else if (txq->tx_buf[index].type == FEC_TXBUF_T_XDP_NDO) { + xdp_return_frame_rx_napi(xdpf); + } else { /* recycle pages of XDP_TX frames */ + /* The dma_sync_size = 0 as XDP_TX has already synced DMA for_device */ + page_pool_put_page(pp_page_to_nmdesc(page)->pp, page, + 0, true); + } + + txq->tx_buf[index].buf_p = NULL; + /* restore default tx buffer type: FEC_TXBUF_T_SKB */ + txq->tx_buf[index].type = FEC_TXBUF_T_SKB; + +tx_buf_done: + /* Make sure the update to bdp and tx_buf are performed * before dirty_tx */ wmb(); @@ -1480,34 +1635,36 @@ skb_done: writel(0, txq->bd.reg_desc_active); } -static void fec_enet_tx(struct net_device *ndev) +static void fec_enet_tx(struct net_device *ndev, int budget) { struct fec_enet_private *fep = netdev_priv(ndev); int i; /* Make sure that AVB queues are processed first. */ for (i = fep->num_tx_queues - 1; i >= 0; i--) - fec_enet_tx_queue(ndev, i); + fec_enet_tx_queue(ndev, i, budget); } -static void fec_enet_update_cbd(struct fec_enet_priv_rx_q *rxq, +static int fec_enet_update_cbd(struct fec_enet_priv_rx_q *rxq, struct bufdesc *bdp, int index) { struct page *new_page; dma_addr_t phys_addr; new_page = page_pool_dev_alloc_pages(rxq->page_pool); - WARN_ON(!new_page); - rxq->rx_skb_info[index].page = new_page; + if (unlikely(!new_page)) + return -ENOMEM; - rxq->rx_skb_info[index].offset = FEC_ENET_XDP_HEADROOM; + rxq->rx_buf[index] = new_page; phys_addr = page_pool_get_dma_addr(new_page) + FEC_ENET_XDP_HEADROOM; bdp->cbd_bufaddr = cpu_to_fec32(phys_addr); + + return 0; } static u32 fec_enet_run_xdp(struct fec_enet_private *fep, struct bpf_prog *prog, - struct xdp_buff *xdp, struct fec_enet_priv_rx_q *rxq, int index) + struct xdp_buff *xdp, struct fec_enet_priv_rx_q *rxq, int cpu) { unsigned int sync, len = xdp->data_end - xdp->data; u32 ret = FEC_ENET_XDP_PASS; @@ -1517,8 +1674,10 @@ fec_enet_run_xdp(struct fec_enet_private *fep, struct bpf_prog *prog, act = bpf_prog_run_xdp(prog, xdp); - /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU touch */ - sync = xdp->data_end - xdp->data_hard_start - FEC_ENET_XDP_HEADROOM; + /* Due xdp_adjust_tail and xdp_adjust_head: DMA sync for_device cover + * max len CPU touch + */ + sync = xdp->data_end - xdp->data; sync = max(sync, len); switch (act) { @@ -1530,44 +1689,67 @@ fec_enet_run_xdp(struct fec_enet_private *fep, struct bpf_prog *prog, case XDP_REDIRECT: rxq->stats[RX_XDP_REDIRECT]++; err = xdp_do_redirect(fep->netdev, xdp, prog); - if (!err) { - ret = FEC_ENET_XDP_REDIR; - } else { - ret = FEC_ENET_XDP_CONSUMED; - page = virt_to_head_page(xdp->data); - page_pool_put_page(rxq->page_pool, page, sync, true); + if (unlikely(err)) + goto xdp_err; + + ret = FEC_ENET_XDP_REDIR; + break; + + case XDP_TX: + rxq->stats[RX_XDP_TX]++; + err = fec_enet_xdp_tx_xmit(fep, cpu, xdp, sync); + if (unlikely(err)) { + rxq->stats[RX_XDP_TX_ERRORS]++; + goto xdp_err; } + + ret = FEC_ENET_XDP_TX; break; default: bpf_warn_invalid_xdp_action(fep->netdev, prog, act); fallthrough; - case XDP_TX: - bpf_warn_invalid_xdp_action(fep->netdev, prog, act); - fallthrough; - case XDP_ABORTED: fallthrough; /* handle aborts by dropping packet */ case XDP_DROP: rxq->stats[RX_XDP_DROP]++; +xdp_err: ret = FEC_ENET_XDP_CONSUMED; page = virt_to_head_page(xdp->data); page_pool_put_page(rxq->page_pool, page, sync, true); + if (act != XDP_DROP) + trace_xdp_exception(fep->netdev, prog, act); break; } return ret; } +static void fec_enet_rx_vlan(const struct net_device *ndev, struct sk_buff *skb) +{ + if (ndev->features & NETIF_F_HW_VLAN_CTAG_RX) { + const struct vlan_ethhdr *vlan_header = skb_vlan_eth_hdr(skb); + const u16 vlan_tag = ntohs(vlan_header->h_vlan_TCI); + + /* Push and remove the vlan tag */ + + memmove(skb->data + VLAN_HLEN, skb->data, ETH_ALEN * 2); + skb_pull(skb, VLAN_HLEN); + __vlan_hwaccel_put_tag(skb, + htons(ETH_P_8021Q), + vlan_tag); + } +} + /* During a receive, the bd_rx.cur points to the current incoming buffer. * When we update through the ring, if the next incoming buffer has * not been given to the system, we just set the empty indicator, * effectively tossing the packet. */ static int -fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) +fec_enet_rx_queue(struct net_device *ndev, u16 queue_id, int budget) { struct fec_enet_private *fep = netdev_priv(ndev); struct fec_enet_priv_rx_q *rxq; @@ -1575,21 +1757,19 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) unsigned short status; struct sk_buff *skb; ushort pkt_len; - __u8 *data; int pkt_received = 0; struct bufdesc_ex *ebdp = NULL; - bool vlan_packet_rcvd = false; - u16 vlan_tag; int index = 0; bool need_swap = fep->quirks & FEC_QUIRK_SWAP_FRAME; struct bpf_prog *xdp_prog = READ_ONCE(fep->xdp_prog); u32 ret, xdp_result = FEC_ENET_XDP_PASS; u32 data_start = FEC_ENET_XDP_HEADROOM; + int cpu = smp_processor_id(); struct xdp_buff xdp; struct page *page; + __fec32 cbd_bufaddr; u32 sub_len = 4; -#if !defined(CONFIG_M5272) /*If it has the FEC_QUIRK_HAS_RACC quirk property, the bit of * FEC_RACC_SHIFT16 is set by default in the probe function. */ @@ -1597,9 +1777,12 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) data_start += 2; sub_len += 2; } -#endif -#ifdef CONFIG_M532x +#if defined(CONFIG_COLDFIRE) && !defined(CONFIG_COLDFIRE_COHERENT_DMA) + /* + * Hacky flush of all caches instead of using the DMA API for the TSO + * headers. + */ flush_cache_all(); #endif rxq = fep->rx_queue[queue_id]; @@ -1608,7 +1791,7 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) * These get messed up if we get called due to a busy condition. */ bdp = rxq->bd.cur; - xdp_init_buff(&xdp, PAGE_SIZE, &rxq->xdp_rxq); + xdp_init_buff(&xdp, PAGE_SIZE << fep->pagepool_order, &rxq->xdp_rxq); while (!((status = fec16_to_cpu(bdp->cbd_sc)) & BD_ENET_RX_EMPTY)) { @@ -1648,22 +1831,29 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) ndev->stats.rx_packets++; pkt_len = fec16_to_cpu(bdp->cbd_datlen); ndev->stats.rx_bytes += pkt_len; + if (fep->quirks & FEC_QUIRK_HAS_RACC) + ndev->stats.rx_bytes -= 2; index = fec_enet_get_bd_index(bdp, &rxq->bd); - page = rxq->rx_skb_info[index].page; + page = rxq->rx_buf[index]; + cbd_bufaddr = bdp->cbd_bufaddr; + if (fec_enet_update_cbd(rxq, bdp, index)) { + ndev->stats.rx_dropped++; + goto rx_processing_done; + } + dma_sync_single_for_cpu(&fep->pdev->dev, - fec32_to_cpu(bdp->cbd_bufaddr), + fec32_to_cpu(cbd_bufaddr), pkt_len, DMA_FROM_DEVICE); prefetch(page_address(page)); - fec_enet_update_cbd(rxq, bdp, index); if (xdp_prog) { xdp_buff_clear_frags_flag(&xdp); /* subtract 16bit shift and FCS */ xdp_prepare_buff(&xdp, page_address(page), data_start, pkt_len - sub_len, false); - ret = fec_enet_run_xdp(fep, xdp_prog, &xdp, rxq, index); + ret = fec_enet_run_xdp(fep, xdp_prog, &xdp, rxq, cpu); xdp_result |= ret; if (ret != FEC_ENET_XDP_PASS) goto rx_processing_done; @@ -1673,7 +1863,8 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) * include that when passing upstream as it messes up * bridging applications. */ - skb = build_skb(page_address(page), PAGE_SIZE); + skb = build_skb(page_address(page), + PAGE_SIZE << fep->pagepool_order); if (unlikely(!skb)) { page_pool_recycle_direct(rxq->page_pool, page); ndev->stats.rx_dropped++; @@ -1687,10 +1878,11 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) skb_mark_for_recycle(skb); if (unlikely(need_swap)) { + u8 *data; + data = page_address(page) + FEC_ENET_XDP_HEADROOM; swap_buffer(data, pkt_len); } - data = skb->data; /* Extract the enhanced buffer descriptor */ ebdp = NULL; @@ -1698,20 +1890,9 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) ebdp = (struct bufdesc_ex *)bdp; /* If this is a VLAN packet remove the VLAN Tag */ - vlan_packet_rcvd = false; - if ((ndev->features & NETIF_F_HW_VLAN_CTAG_RX) && - fep->bufdesc_ex && - (ebdp->cbd_esc & cpu_to_fec32(BD_ENET_RX_VLAN))) { - /* Push and remove the vlan tag */ - struct vlan_hdr *vlan_header = - (struct vlan_hdr *) (data + ETH_HLEN); - vlan_tag = ntohs(vlan_header->h_vlan_TCI); - - vlan_packet_rcvd = true; - - memmove(skb->data + VLAN_HLEN, data, ETH_ALEN * 2); - skb_pull(skb, VLAN_HLEN); - } + if (fep->bufdesc_ex && + (ebdp->cbd_esc & cpu_to_fec32(BD_ENET_RX_VLAN))) + fec_enet_rx_vlan(ndev, skb); skb->protocol = eth_type_trans(skb, ndev); @@ -1730,12 +1911,6 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) } } - /* Handle received VLAN packets */ - if (vlan_packet_rcvd) - __vlan_hwaccel_put_tag(skb, - htons(ETH_P_8021Q), - vlan_tag); - skb_record_rx_queue(skb, queue_id); napi_gro_receive(&fep->napi, skb); @@ -1771,7 +1946,7 @@ rx_processing_done: rxq->bd.cur = bdp; if (xdp_result & FEC_ENET_XDP_REDIR) - xdp_do_flush_map(); + xdp_do_flush(); return pkt_received; } @@ -1783,7 +1958,7 @@ static int fec_enet_rx(struct net_device *ndev, int budget) /* Make sure that AVB queues are processed first. */ for (i = fep->num_rx_queues - 1; i >= 0; i--) - done += fec_enet_rx_queue(ndev, budget - done, i); + done += fec_enet_rx_queue(ndev, i, budget - done); return done; } @@ -1830,7 +2005,7 @@ static int fec_enet_rx_napi(struct napi_struct *napi, int budget) do { done += fec_enet_rx(ndev, budget - done); - fec_enet_tx(ndev); + fec_enet_tx(ndev, budget); } while ((done < budget) && fec_enet_collect_events(fep)); if (done < budget) { @@ -1919,6 +2094,37 @@ static int fec_get_mac(struct net_device *ndev) /* * Phy section */ + +/* LPI Sleep Ts count base on tx clk (clk_ref). + * The lpi sleep cnt value = X us / (cycle_ns). + */ +static int fec_enet_us_to_tx_cycle(struct net_device *ndev, int us) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + + return us * (fep->clk_ref_rate / 1000) / 1000; +} + +static int fec_enet_eee_mode_set(struct net_device *ndev, u32 lpi_timer, + bool enable) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + unsigned int sleep_cycle, wake_cycle; + + if (enable) { + sleep_cycle = fec_enet_us_to_tx_cycle(ndev, lpi_timer); + wake_cycle = sleep_cycle; + } else { + sleep_cycle = 0; + wake_cycle = 0; + } + + writel(sleep_cycle, fep->hwp + FEC_LPI_SLEEP); + writel(wake_cycle, fep->hwp + FEC_LPI_WAKE); + + return 0; +} + static void fec_enet_adjust_link(struct net_device *ndev) { struct fec_enet_private *fep = netdev_priv(ndev); @@ -1950,6 +2156,7 @@ static void fec_enet_adjust_link(struct net_device *ndev) /* if any of the above changed restart the FEC */ if (status_change) { + netif_stop_queue(ndev); napi_disable(&fep->napi); netif_tx_lock_bh(ndev); fec_restart(ndev); @@ -1957,8 +2164,13 @@ static void fec_enet_adjust_link(struct net_device *ndev) netif_tx_unlock_bh(ndev); napi_enable(&fep->napi); } + if (fep->quirks & FEC_QUIRK_HAS_EEE) + fec_enet_eee_mode_set(ndev, + phy_dev->eee_cfg.tx_lpi_timer, + phy_dev->enable_tx_lpi); } else { if (fep->link) { + netif_stop_queue(ndev); napi_disable(&fep->napi); netif_tx_lock_bh(ndev); fec_stop(ndev); @@ -1987,47 +2199,73 @@ static int fec_enet_mdio_wait(struct fec_enet_private *fep) return ret; } -static int fec_enet_mdio_read(struct mii_bus *bus, int mii_id, int regnum) +static int fec_enet_mdio_read_c22(struct mii_bus *bus, int mii_id, int regnum) { struct fec_enet_private *fep = bus->priv; struct device *dev = &fep->pdev->dev; int ret = 0, frame_start, frame_addr, frame_op; - bool is_c45 = !!(regnum & MII_ADDR_C45); ret = pm_runtime_resume_and_get(dev); if (ret < 0) return ret; - if (is_c45) { - frame_start = FEC_MMFR_ST_C45; + /* C22 read */ + frame_op = FEC_MMFR_OP_READ; + frame_start = FEC_MMFR_ST; + frame_addr = regnum; - /* write address */ - frame_addr = (regnum >> 16); - writel(frame_start | FEC_MMFR_OP_ADDR_WRITE | - FEC_MMFR_PA(mii_id) | FEC_MMFR_RA(frame_addr) | - FEC_MMFR_TA | (regnum & 0xFFFF), - fep->hwp + FEC_MII_DATA); + /* start a read op */ + writel(frame_start | frame_op | + FEC_MMFR_PA(mii_id) | FEC_MMFR_RA(frame_addr) | + FEC_MMFR_TA, fep->hwp + FEC_MII_DATA); - /* wait for end of transfer */ - ret = fec_enet_mdio_wait(fep); - if (ret) { - netdev_err(fep->netdev, "MDIO address write timeout\n"); - goto out; - } + /* wait for end of transfer */ + ret = fec_enet_mdio_wait(fep); + if (ret) { + netdev_err(fep->netdev, "MDIO read timeout\n"); + goto out; + } - frame_op = FEC_MMFR_OP_READ_C45; + ret = FEC_MMFR_DATA(readl(fep->hwp + FEC_MII_DATA)); - } else { - /* C22 read */ - frame_op = FEC_MMFR_OP_READ; - frame_start = FEC_MMFR_ST; - frame_addr = regnum; +out: + pm_runtime_put_autosuspend(dev); + + return ret; +} + +static int fec_enet_mdio_read_c45(struct mii_bus *bus, int mii_id, + int devad, int regnum) +{ + struct fec_enet_private *fep = bus->priv; + struct device *dev = &fep->pdev->dev; + int ret = 0, frame_start, frame_op; + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) + return ret; + + frame_start = FEC_MMFR_ST_C45; + + /* write address */ + writel(frame_start | FEC_MMFR_OP_ADDR_WRITE | + FEC_MMFR_PA(mii_id) | FEC_MMFR_RA(devad) | + FEC_MMFR_TA | (regnum & 0xFFFF), + fep->hwp + FEC_MII_DATA); + + /* wait for end of transfer */ + ret = fec_enet_mdio_wait(fep); + if (ret) { + netdev_err(fep->netdev, "MDIO address write timeout\n"); + goto out; } + frame_op = FEC_MMFR_OP_READ_C45; + /* start a read op */ writel(frame_start | frame_op | - FEC_MMFR_PA(mii_id) | FEC_MMFR_RA(frame_addr) | - FEC_MMFR_TA, fep->hwp + FEC_MII_DATA); + FEC_MMFR_PA(mii_id) | FEC_MMFR_RA(devad) | + FEC_MMFR_TA, fep->hwp + FEC_MII_DATA); /* wait for end of transfer */ ret = fec_enet_mdio_wait(fep); @@ -2039,51 +2277,73 @@ static int fec_enet_mdio_read(struct mii_bus *bus, int mii_id, int regnum) ret = FEC_MMFR_DATA(readl(fep->hwp + FEC_MII_DATA)); out: - pm_runtime_mark_last_busy(dev); pm_runtime_put_autosuspend(dev); return ret; } -static int fec_enet_mdio_write(struct mii_bus *bus, int mii_id, int regnum, - u16 value) +static int fec_enet_mdio_write_c22(struct mii_bus *bus, int mii_id, int regnum, + u16 value) { struct fec_enet_private *fep = bus->priv; struct device *dev = &fep->pdev->dev; int ret, frame_start, frame_addr; - bool is_c45 = !!(regnum & MII_ADDR_C45); ret = pm_runtime_resume_and_get(dev); if (ret < 0) return ret; - if (is_c45) { - frame_start = FEC_MMFR_ST_C45; + /* C22 write */ + frame_start = FEC_MMFR_ST; + frame_addr = regnum; + + /* start a write op */ + writel(frame_start | FEC_MMFR_OP_WRITE | + FEC_MMFR_PA(mii_id) | FEC_MMFR_RA(frame_addr) | + FEC_MMFR_TA | FEC_MMFR_DATA(value), + fep->hwp + FEC_MII_DATA); - /* write address */ - frame_addr = (regnum >> 16); - writel(frame_start | FEC_MMFR_OP_ADDR_WRITE | - FEC_MMFR_PA(mii_id) | FEC_MMFR_RA(frame_addr) | - FEC_MMFR_TA | (regnum & 0xFFFF), - fep->hwp + FEC_MII_DATA); + /* wait for end of transfer */ + ret = fec_enet_mdio_wait(fep); + if (ret) + netdev_err(fep->netdev, "MDIO write timeout\n"); - /* wait for end of transfer */ - ret = fec_enet_mdio_wait(fep); - if (ret) { - netdev_err(fep->netdev, "MDIO address write timeout\n"); - goto out; - } - } else { - /* C22 write */ - frame_start = FEC_MMFR_ST; - frame_addr = regnum; + pm_runtime_put_autosuspend(dev); + + return ret; +} + +static int fec_enet_mdio_write_c45(struct mii_bus *bus, int mii_id, + int devad, int regnum, u16 value) +{ + struct fec_enet_private *fep = bus->priv; + struct device *dev = &fep->pdev->dev; + int ret, frame_start; + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) + return ret; + + frame_start = FEC_MMFR_ST_C45; + + /* write address */ + writel(frame_start | FEC_MMFR_OP_ADDR_WRITE | + FEC_MMFR_PA(mii_id) | FEC_MMFR_RA(devad) | + FEC_MMFR_TA | (regnum & 0xFFFF), + fep->hwp + FEC_MII_DATA); + + /* wait for end of transfer */ + ret = fec_enet_mdio_wait(fep); + if (ret) { + netdev_err(fep->netdev, "MDIO address write timeout\n"); + goto out; } /* start a write op */ writel(frame_start | FEC_MMFR_OP_WRITE | - FEC_MMFR_PA(mii_id) | FEC_MMFR_RA(frame_addr) | - FEC_MMFR_TA | FEC_MMFR_DATA(value), - fep->hwp + FEC_MII_DATA); + FEC_MMFR_PA(mii_id) | FEC_MMFR_RA(devad) | + FEC_MMFR_TA | FEC_MMFR_DATA(value), + fep->hwp + FEC_MII_DATA); /* wait for end of transfer */ ret = fec_enet_mdio_wait(fep); @@ -2091,7 +2351,6 @@ static int fec_enet_mdio_write(struct mii_bus *bus, int mii_id, int regnum, netdev_err(fep->netdev, "MDIO write timeout\n"); out: - pm_runtime_mark_last_busy(dev); pm_runtime_put_autosuspend(dev); return ret; @@ -2114,7 +2373,8 @@ static void fec_enet_phy_reset_after_clk_enable(struct net_device *ndev) */ phy_dev = of_phy_find_device(fep->phy_node); phy_reset_after_clk_enable(phy_dev); - put_device(&phy_dev->mdio.dev); + if (phy_dev) + put_device(&phy_dev->mdio.dev); } } @@ -2210,11 +2470,8 @@ static int fec_enet_parse_rgmii_delay(struct fec_enet_private *fep, static int fec_enet_mii_probe(struct net_device *ndev) { struct fec_enet_private *fep = netdev_priv(ndev); - struct phy_device *phy_dev = NULL; - char mdio_bus_id[MII_BUS_ID_SIZE]; - char phy_name[MII_BUS_ID_SIZE + 3]; - int phy_id; - int dev_id = fep->dev_id; + struct phy_device *phy_dev; + int ret; if (fep->phy_node) { phy_dev = of_phy_connect(ndev, fep->phy_node, @@ -2226,30 +2483,28 @@ static int fec_enet_mii_probe(struct net_device *ndev) } } else { /* check for attached phy */ - for (phy_id = 0; (phy_id < PHY_MAX_ADDR); phy_id++) { - if (!mdiobus_is_registered_device(fep->mii_bus, phy_id)) - continue; - if (dev_id--) - continue; - strscpy(mdio_bus_id, fep->mii_bus->id, MII_BUS_ID_SIZE); - break; - } + phy_dev = phy_find_first(fep->mii_bus); + if (fep->dev_id && phy_dev) + phy_dev = phy_find_next(fep->mii_bus, phy_dev); - if (phy_id >= PHY_MAX_ADDR) { + if (!phy_dev) { netdev_info(ndev, "no PHY, assuming direct connection to switch\n"); - strscpy(mdio_bus_id, "fixed-0", MII_BUS_ID_SIZE); - phy_id = 0; + phy_dev = fixed_phy_register_100fd(); + if (IS_ERR(phy_dev)) { + netdev_err(ndev, "could not register fixed PHY\n"); + return PTR_ERR(phy_dev); + } } - snprintf(phy_name, sizeof(phy_name), - PHY_ID_FMT, mdio_bus_id, phy_id); - phy_dev = phy_connect(ndev, phy_name, &fec_enet_adjust_link, - fep->phy_interface); - } + ret = phy_connect_direct(ndev, phy_dev, &fec_enet_adjust_link, + fep->phy_interface); + if (ret) { + if (phy_is_pseudo_fixed_link(phy_dev)) + fixed_phy_unregister(phy_dev); + netdev_err(ndev, "could not attach to PHY\n"); + return ret; + } - if (IS_ERR(phy_dev)) { - netdev_err(ndev, "could not attach to PHY\n"); - return PTR_ERR(phy_dev); } /* mask with MAC supported features */ @@ -2257,18 +2512,17 @@ static int fec_enet_mii_probe(struct net_device *ndev) phy_set_max_speed(phy_dev, 1000); phy_remove_link_mode(phy_dev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT); -#if !defined(CONFIG_M5272) phy_support_sym_pause(phy_dev); -#endif } else phy_set_max_speed(phy_dev, 100); + if (fep->quirks & FEC_QUIRK_HAS_EEE) + phy_support_eee(phy_dev); + fep->link = 0; fep->full_duplex = 0; - phy_dev->mac_managed_pm = true; - phy_attached_info(phy_dev); return 0; @@ -2280,6 +2534,7 @@ static int fec_enet_mii_init(struct platform_device *pdev) struct net_device *ndev = platform_get_drvdata(pdev); struct fec_enet_private *fep = netdev_priv(ndev); bool suppress_preamble = false; + struct phy_device *phydev; struct device_node *node; int err = -ENXIO; u32 mii_speed, holdtime; @@ -2381,8 +2636,12 @@ static int fec_enet_mii_init(struct platform_device *pdev) } fep->mii_bus->name = "fec_enet_mii_bus"; - fep->mii_bus->read = fec_enet_mdio_read; - fep->mii_bus->write = fec_enet_mdio_write; + fep->mii_bus->read = fec_enet_mdio_read_c22; + fep->mii_bus->write = fec_enet_mdio_write_c22; + if (fep->quirks & FEC_QUIRK_HAS_MDIO_C45) { + fep->mii_bus->read_c45 = fec_enet_mdio_read_c45; + fep->mii_bus->write_c45 = fec_enet_mdio_write_c45; + } snprintf(fep->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", pdev->name, fep->dev_id + 1); fep->mii_bus->priv = fep; @@ -2393,6 +2652,10 @@ static int fec_enet_mii_init(struct platform_device *pdev) goto err_out_free_mdiobus; of_node_put(node); + /* find all the PHY devices on the bus and set mac_managed_pm to true */ + mdiobus_for_each_phy(fep->mii_bus, phydev) + phydev->mac_managed_pm = true; + mii_cnt++; /* save fec0 mii_bus */ @@ -2440,9 +2703,7 @@ static int fec_enet_get_regs_len(struct net_device *ndev) } /* List of registers that can be safety be read to dump them with ethtool */ -#if defined(CONFIG_M523x) || defined(CONFIG_M527x) || defined(CONFIG_M528x) || \ - defined(CONFIG_M520x) || defined(CONFIG_M532x) || defined(CONFIG_ARM) || \ - defined(CONFIG_ARM64) || defined(CONFIG_COMPILE_TEST) +#if !defined(CONFIG_M5272) || defined(CONFIG_COMPILE_TEST) static __u32 fec_enet_register_version = 2; static u32 fec_enet_register_offset[] = { FEC_IEVENT, FEC_IMASK, FEC_R_DES_ACTIVE_0, FEC_X_DES_ACTIVE_0, @@ -2516,30 +2777,22 @@ static u32 fec_enet_register_offset[] = { static void fec_enet_get_regs(struct net_device *ndev, struct ethtool_regs *regs, void *regbuf) { + u32 reg_cnt = ARRAY_SIZE(fec_enet_register_offset); struct fec_enet_private *fep = netdev_priv(ndev); u32 __iomem *theregs = (u32 __iomem *)fep->hwp; + u32 *reg_list = fec_enet_register_offset; struct device *dev = &fep->pdev->dev; u32 *buf = (u32 *)regbuf; u32 i, off; int ret; -#if defined(CONFIG_M523x) || defined(CONFIG_M527x) || defined(CONFIG_M528x) || \ - defined(CONFIG_M520x) || defined(CONFIG_M532x) || defined(CONFIG_ARM) || \ - defined(CONFIG_ARM64) || defined(CONFIG_COMPILE_TEST) - u32 *reg_list; - u32 reg_cnt; - - if (!of_machine_is_compatible("fsl,imx6ul")) { - reg_list = fec_enet_register_offset; - reg_cnt = ARRAY_SIZE(fec_enet_register_offset); - } else { + +#if !defined(CONFIG_M5272) || defined(CONFIG_COMPILE_TEST) + if (of_machine_is_compatible("fsl,imx6ul")) { reg_list = fec_enet_register_offset_6ul; reg_cnt = ARRAY_SIZE(fec_enet_register_offset_6ul); } -#else - /* coldfire */ - static u32 *reg_list = fec_enet_register_offset; - static const u32 reg_cnt = ARRAY_SIZE(fec_enet_register_offset); #endif + ret = pm_runtime_resume_and_get(dev); if (ret < 0) return; @@ -2559,27 +2812,22 @@ static void fec_enet_get_regs(struct net_device *ndev, buf[off] = readl(&theregs[off]); } - pm_runtime_mark_last_busy(dev); pm_runtime_put_autosuspend(dev); } static int fec_enet_get_ts_info(struct net_device *ndev, - struct ethtool_ts_info *info) + struct kernel_ethtool_ts_info *info) { struct fec_enet_private *fep = netdev_priv(ndev); if (fep->bufdesc_ex) { info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | - SOF_TIMESTAMPING_RX_SOFTWARE | - SOF_TIMESTAMPING_SOFTWARE | SOF_TIMESTAMPING_TX_HARDWARE | SOF_TIMESTAMPING_RX_HARDWARE | SOF_TIMESTAMPING_RAW_HARDWARE; if (fep->ptp_clock) info->phc_index = ptp_clock_index(fep->ptp_clock); - else - info->phc_index = -1; info->tx_types = (1 << HWTSTAMP_TX_OFF) | (1 << HWTSTAMP_TX_ON); @@ -2749,6 +2997,7 @@ static void fec_enet_get_xdp_stats(struct fec_enet_private *fep, u64 *data) static void fec_enet_page_pool_stats(struct fec_enet_private *fep, u64 *data) { +#ifdef CONFIG_PAGE_POOL_STATS struct page_pool_stats stats = {}; struct fec_enet_priv_rx_q *rxq; int i; @@ -2763,6 +3012,7 @@ static void fec_enet_page_pool_stats(struct fec_enet_private *fep, u64 *data) } page_pool_ethtool_stats_get(data, &stats); +#endif } static void fec_enet_get_ethtool_stats(struct net_device *dev, @@ -2789,12 +3039,10 @@ static void fec_enet_get_strings(struct net_device *netdev, switch (stringset) { case ETH_SS_STATS: for (i = 0; i < ARRAY_SIZE(fec_stats); i++) { - memcpy(data, fec_stats[i].name, ETH_GSTRING_LEN); - data += ETH_GSTRING_LEN; + ethtool_puts(&data, fec_stats[i].name); } for (i = 0; i < ARRAY_SIZE(fec_xdp_stat_strs); i++) { - strncpy(data, fec_xdp_stat_strs[i], ETH_GSTRING_LEN); - data += ETH_GSTRING_LEN; + ethtool_puts(&data, fec_xdp_stat_strs[i]); } page_pool_ethtool_stats_get_strings(data); @@ -2870,27 +3118,25 @@ static int fec_enet_us_to_itr_clock(struct net_device *ndev, int us) static void fec_enet_itr_coal_set(struct net_device *ndev) { struct fec_enet_private *fep = netdev_priv(ndev); - int rx_itr, tx_itr; + u32 rx_itr = 0, tx_itr = 0; + int rx_ictt, tx_ictt; - /* Must be greater than zero to avoid unpredictable behavior */ - if (!fep->rx_time_itr || !fep->rx_pkts_itr || - !fep->tx_time_itr || !fep->tx_pkts_itr) - return; - - /* Select enet system clock as Interrupt Coalescing - * timer Clock Source - */ - rx_itr = FEC_ITR_CLK_SEL; - tx_itr = FEC_ITR_CLK_SEL; + rx_ictt = fec_enet_us_to_itr_clock(ndev, fep->rx_time_itr); + tx_ictt = fec_enet_us_to_itr_clock(ndev, fep->tx_time_itr); - /* set ICFT and ICTT */ - rx_itr |= FEC_ITR_ICFT(fep->rx_pkts_itr); - rx_itr |= FEC_ITR_ICTT(fec_enet_us_to_itr_clock(ndev, fep->rx_time_itr)); - tx_itr |= FEC_ITR_ICFT(fep->tx_pkts_itr); - tx_itr |= FEC_ITR_ICTT(fec_enet_us_to_itr_clock(ndev, fep->tx_time_itr)); + if (rx_ictt > 0 && fep->rx_pkts_itr > 1) { + /* Enable with enet system clock as Interrupt Coalescing timer Clock Source */ + rx_itr = FEC_ITR_EN | FEC_ITR_CLK_SEL; + rx_itr |= FEC_ITR_ICFT(fep->rx_pkts_itr); + rx_itr |= FEC_ITR_ICTT(rx_ictt); + } - rx_itr |= FEC_ITR_EN; - tx_itr |= FEC_ITR_EN; + if (tx_ictt > 0 && fep->tx_pkts_itr > 1) { + /* Enable with enet system clock as Interrupt Coalescing timer Clock Source */ + tx_itr = FEC_ITR_EN | FEC_ITR_CLK_SEL; + tx_itr |= FEC_ITR_ICFT(fep->tx_pkts_itr); + tx_itr |= FEC_ITR_ICTT(tx_ictt); + } writel(tx_itr, fep->hwp + FEC_TXIC0); writel(rx_itr, fep->hwp + FEC_RXIC0); @@ -2966,88 +3212,10 @@ static int fec_enet_set_coalesce(struct net_device *ndev, return 0; } -static int fec_enet_get_tunable(struct net_device *netdev, - const struct ethtool_tunable *tuna, - void *data) -{ - struct fec_enet_private *fep = netdev_priv(netdev); - int ret = 0; - - switch (tuna->id) { - case ETHTOOL_RX_COPYBREAK: - *(u32 *)data = fep->rx_copybreak; - break; - default: - ret = -EINVAL; - break; - } - - return ret; -} - -static int fec_enet_set_tunable(struct net_device *netdev, - const struct ethtool_tunable *tuna, - const void *data) -{ - struct fec_enet_private *fep = netdev_priv(netdev); - int ret = 0; - - switch (tuna->id) { - case ETHTOOL_RX_COPYBREAK: - fep->rx_copybreak = *(u32 *)data; - break; - default: - ret = -EINVAL; - break; - } - - return ret; -} - -/* LPI Sleep Ts count base on tx clk (clk_ref). - * The lpi sleep cnt value = X us / (cycle_ns). - */ -static int fec_enet_us_to_tx_cycle(struct net_device *ndev, int us) -{ - struct fec_enet_private *fep = netdev_priv(ndev); - - return us * (fep->clk_ref_rate / 1000) / 1000; -} - -static int fec_enet_eee_mode_set(struct net_device *ndev, bool enable) -{ - struct fec_enet_private *fep = netdev_priv(ndev); - struct ethtool_eee *p = &fep->eee; - unsigned int sleep_cycle, wake_cycle; - int ret = 0; - - if (enable) { - ret = phy_init_eee(ndev->phydev, false); - if (ret) - return ret; - - sleep_cycle = fec_enet_us_to_tx_cycle(ndev, p->tx_lpi_timer); - wake_cycle = sleep_cycle; - } else { - sleep_cycle = 0; - wake_cycle = 0; - } - - p->tx_lpi_enabled = enable; - p->eee_enabled = enable; - p->eee_active = enable; - - writel(sleep_cycle, fep->hwp + FEC_LPI_SLEEP); - writel(wake_cycle, fep->hwp + FEC_LPI_WAKE); - - return 0; -} - static int -fec_enet_get_eee(struct net_device *ndev, struct ethtool_eee *edata) +fec_enet_get_eee(struct net_device *ndev, struct ethtool_keee *edata) { struct fec_enet_private *fep = netdev_priv(ndev); - struct ethtool_eee *p = &fep->eee; if (!(fep->quirks & FEC_QUIRK_HAS_EEE)) return -EOPNOTSUPP; @@ -3055,20 +3223,13 @@ fec_enet_get_eee(struct net_device *ndev, struct ethtool_eee *edata) if (!netif_running(ndev)) return -ENETDOWN; - edata->eee_enabled = p->eee_enabled; - edata->eee_active = p->eee_active; - edata->tx_lpi_timer = p->tx_lpi_timer; - edata->tx_lpi_enabled = p->tx_lpi_enabled; - return phy_ethtool_get_eee(ndev->phydev, edata); } static int -fec_enet_set_eee(struct net_device *ndev, struct ethtool_eee *edata) +fec_enet_set_eee(struct net_device *ndev, struct ethtool_keee *edata) { struct fec_enet_private *fep = netdev_priv(ndev); - struct ethtool_eee *p = &fep->eee; - int ret = 0; if (!(fep->quirks & FEC_QUIRK_HAS_EEE)) return -EOPNOTSUPP; @@ -3076,17 +3237,6 @@ fec_enet_set_eee(struct net_device *ndev, struct ethtool_eee *edata) if (!netif_running(ndev)) return -ENETDOWN; - p->tx_lpi_timer = edata->tx_lpi_timer; - - if (!edata->eee_enabled || !edata->tx_lpi_enabled || - !edata->tx_lpi_timer) - ret = fec_enet_eee_mode_set(ndev, false); - else - ret = fec_enet_eee_mode_set(ndev, true); - - if (ret) - return ret; - return phy_ethtool_set_eee(ndev->phydev, edata); } @@ -3141,8 +3291,6 @@ static const struct ethtool_ops fec_enet_ethtool_ops = { .get_sset_count = fec_enet_get_sset_count, #endif .get_ts_info = fec_enet_get_ts_info, - .get_tunable = fec_enet_get_tunable, - .set_tunable = fec_enet_set_tunable, .get_wol = fec_enet_get_wol, .set_wol = fec_enet_set_wol, .get_eee = fec_enet_get_eee, @@ -3152,38 +3300,10 @@ static const struct ethtool_ops fec_enet_ethtool_ops = { .self_test = net_selftest, }; -static int fec_enet_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) -{ - struct fec_enet_private *fep = netdev_priv(ndev); - struct phy_device *phydev = ndev->phydev; - - if (!netif_running(ndev)) - return -EINVAL; - - if (!phydev) - return -ENODEV; - - if (fep->bufdesc_ex) { - bool use_fec_hwts = !phy_has_hwtstamp(phydev); - - if (cmd == SIOCSHWTSTAMP) { - if (use_fec_hwts) - return fec_ptp_set(ndev, rq); - fec_ptp_disable_hwts(ndev); - } else if (cmd == SIOCGHWTSTAMP) { - if (use_fec_hwts) - return fec_ptp_get(ndev, rq); - } - } - - return phy_mii_ioctl(phydev, rq, cmd); -} - static void fec_enet_free_buffers(struct net_device *ndev) { struct fec_enet_private *fep = netdev_priv(ndev); unsigned int i; - struct sk_buff *skb; struct fec_enet_priv_tx_q *txq; struct fec_enet_priv_rx_q *rxq; unsigned int q; @@ -3191,7 +3311,8 @@ static void fec_enet_free_buffers(struct net_device *ndev) for (q = 0; q < fep->num_rx_queues; q++) { rxq = fep->rx_queue[q]; for (i = 0; i < rxq->bd.ring_size; i++) - page_pool_put_full_page(rxq->page_pool, rxq->rx_skb_info[i].page, false); + page_pool_put_full_page(rxq->page_pool, rxq->rx_buf[i], + false); for (i = 0; i < XDP_STATS_TOTAL; i++) rxq->stats[i] = 0; @@ -3207,9 +3328,25 @@ static void fec_enet_free_buffers(struct net_device *ndev) for (i = 0; i < txq->bd.ring_size; i++) { kfree(txq->tx_bounce[i]); txq->tx_bounce[i] = NULL; - skb = txq->tx_skbuff[i]; - txq->tx_skbuff[i] = NULL; - dev_kfree_skb(skb); + + if (!txq->tx_buf[i].buf_p) { + txq->tx_buf[i].type = FEC_TXBUF_T_SKB; + continue; + } + + if (txq->tx_buf[i].type == FEC_TXBUF_T_SKB) { + dev_kfree_skb(txq->tx_buf[i].buf_p); + } else if (txq->tx_buf[i].type == FEC_TXBUF_T_XDP_NDO) { + xdp_return_frame(txq->tx_buf[i].buf_p); + } else { + struct page *page = txq->tx_buf[i].buf_p; + + page_pool_put_page(pp_page_to_nmdesc(page)->pp, + page, 0, false); + } + + txq->tx_buf[i].buf_p = NULL; + txq->tx_buf[i].type = FEC_TXBUF_T_SKB; } } } @@ -3223,10 +3360,9 @@ static void fec_enet_free_queue(struct net_device *ndev) for (i = 0; i < fep->num_tx_queues; i++) if (fep->tx_queue[i] && fep->tx_queue[i]->tso_hdrs) { txq = fep->tx_queue[i]; - dma_free_coherent(&fep->pdev->dev, - txq->bd.ring_size * TSO_HEADER_SIZE, - txq->tso_hdrs, - txq->tso_hdrs_dma); + fec_dma_free(&fep->pdev->dev, + txq->bd.ring_size * TSO_HEADER_SIZE, + txq->tso_hdrs, txq->tso_hdrs_dma); } for (i = 0; i < fep->num_rx_queues; i++) @@ -3254,13 +3390,11 @@ static int fec_enet_alloc_queue(struct net_device *ndev) fep->total_tx_ring_size += fep->tx_queue[i]->bd.ring_size; txq->tx_stop_threshold = FEC_MAX_SKB_DESCS; - txq->tx_wake_threshold = - (txq->bd.ring_size - txq->tx_stop_threshold) / 2; + txq->tx_wake_threshold = FEC_MAX_SKB_DESCS + 2 * MAX_SKB_FRAGS; - txq->tso_hdrs = dma_alloc_coherent(&fep->pdev->dev, + txq->tso_hdrs = fec_dma_alloc(&fep->pdev->dev, txq->bd.ring_size * TSO_HEADER_SIZE, - &txq->tso_hdrs_dma, - GFP_KERNEL); + &txq->tso_hdrs_dma, GFP_KERNEL); if (!txq->tso_hdrs) { ret = -ENOMEM; goto alloc_failed; @@ -3304,6 +3438,19 @@ fec_enet_alloc_rxq_buffers(struct net_device *ndev, unsigned int queue) return err; } + /* Some platforms require the RX buffer must be 64 bytes alignment. + * Some platforms require 16 bytes alignment. And some platforms + * require 4 bytes alignment. But since the page pool have been + * introduced into the driver, the address of RX buffer is always + * the page address plus FEC_ENET_XDP_HEADROOM, and + * FEC_ENET_XDP_HEADROOM is 256 bytes. Therefore, this address can + * satisfy all platforms. To prevent future modifications to + * FEC_ENET_XDP_HEADROOM from ignoring this hardware limitation, a + * BUILD_BUG_ON() test has been added, which ensures that + * FEC_ENET_XDP_HEADROOM provides the required alignment. + */ + BUILD_BUG_ON(FEC_ENET_XDP_HEADROOM & 0x3f); + for (i = 0; i < rxq->bd.ring_size; i++) { page = page_pool_dev_alloc_pages(rxq->page_pool); if (!page) @@ -3312,8 +3459,7 @@ fec_enet_alloc_rxq_buffers(struct net_device *ndev, unsigned int queue) phys_addr = page_pool_get_dma_addr(page) + FEC_ENET_XDP_HEADROOM; bdp->cbd_bufaddr = cpu_to_fec32(phys_addr); - rxq->rx_skb_info[i].page = page; - rxq->rx_skb_info[i].offset = FEC_ENET_XDP_HEADROOM; + rxq->rx_buf[i] = page; bdp->cbd_sc = cpu_to_fec16(BD_ENET_RX_EMPTY); if (fep->bufdesc_ex) { @@ -3326,7 +3472,7 @@ fec_enet_alloc_rxq_buffers(struct net_device *ndev, unsigned int queue) /* Set the last buffer to wrap. */ bdp = fec_enet_get_prevdesc(bdp, &rxq->bd); - bdp->cbd_sc |= cpu_to_fec16(BD_SC_WRAP); + bdp->cbd_sc |= cpu_to_fec16(BD_ENET_RX_WRAP); return 0; err_alloc: @@ -3362,7 +3508,7 @@ fec_enet_alloc_txq_buffers(struct net_device *ndev, unsigned int queue) /* Set the last buffer to wrap. */ bdp = fec_enet_get_prevdesc(bdp, &txq->bd); - bdp->cbd_sc |= cpu_to_fec16(BD_SC_WRAP); + bdp->cbd_sc |= cpu_to_fec16(BD_ENET_TX_WRAP); return 0; @@ -3455,7 +3601,6 @@ err_enet_mii_probe: err_enet_alloc: fec_enet_clk_enable(ndev, false); clk_enable: - pm_runtime_mark_last_busy(&fep->pdev->dev); pm_runtime_put_autosuspend(&fep->pdev->dev); pinctrl_pm_select_sleep_state(&fep->pdev->dev); return ret; @@ -3465,8 +3610,9 @@ static int fec_enet_close(struct net_device *ndev) { struct fec_enet_private *fep = netdev_priv(ndev); + struct phy_device *phy_dev = ndev->phydev; - phy_stop(ndev->phydev); + phy_stop(phy_dev); if (netif_device_present(ndev)) { napi_disable(&fep->napi); @@ -3474,7 +3620,10 @@ fec_enet_close(struct net_device *ndev) fec_stop(ndev); } - phy_disconnect(ndev->phydev); + phy_disconnect(phy_dev); + + if (!fep->phy_node && phy_is_pseudo_fixed_link(phy_dev)) + fixed_phy_unregister(phy_dev); if (fep->quirks & FEC_QUIRK_ERR006687) imx6q_cpuidle_fec_irqs_unused(); @@ -3486,7 +3635,6 @@ fec_enet_close(struct net_device *ndev) cpu_latency_qos_remove_request(&fep->pm_qos_req); pinctrl_pm_select_sleep_state(&fep->pdev->dev); - pm_runtime_mark_last_busy(&fep->pdev->dev); pm_runtime_put_autosuspend(&fep->pdev->dev); fec_enet_free_buffers(ndev); @@ -3559,7 +3707,6 @@ static void set_multicast_list(struct net_device *ndev) static int fec_set_mac_address(struct net_device *ndev, void *p) { - struct fec_enet_private *fep = netdev_priv(ndev); struct sockaddr *addr = p; if (addr) { @@ -3576,36 +3723,10 @@ fec_set_mac_address(struct net_device *ndev, void *p) if (!netif_running(ndev)) return 0; - writel(ndev->dev_addr[3] | (ndev->dev_addr[2] << 8) | - (ndev->dev_addr[1] << 16) | (ndev->dev_addr[0] << 24), - fep->hwp + FEC_ADDR_LOW); - writel((ndev->dev_addr[5] << 16) | (ndev->dev_addr[4] << 24), - fep->hwp + FEC_ADDR_HIGH); - return 0; -} - -#ifdef CONFIG_NET_POLL_CONTROLLER -/** - * fec_poll_controller - FEC Poll controller function - * @dev: The FEC network adapter - * - * Polled functionality used by netconsole and others in non interrupt mode - * - */ -static void fec_poll_controller(struct net_device *dev) -{ - int i; - struct fec_enet_private *fep = netdev_priv(dev); + fec_set_hw_mac_addr(ndev); - for (i = 0; i < FEC_IRQ_NUM; i++) { - if (fep->irq[i] > 0) { - disable_irq(fep->irq[i]); - fec_enet_interrupt(fep->irq[i], dev); - enable_irq(fep->irq[i]); - } - } + return 0; } -#endif static inline void fec_enet_set_netdev_features(struct net_device *netdev, netdev_features_t features) @@ -3646,31 +3767,26 @@ static int fec_set_features(struct net_device *netdev, return 0; } -static u16 fec_enet_get_raw_vlan_tci(struct sk_buff *skb) -{ - struct vlan_ethhdr *vhdr; - unsigned short vlan_TCI = 0; - - if (skb->protocol == htons(ETH_P_ALL)) { - vhdr = (struct vlan_ethhdr *)(skb->data); - vlan_TCI = ntohs(vhdr->h_vlan_TCI); - } - - return vlan_TCI; -} - static u16 fec_enet_select_queue(struct net_device *ndev, struct sk_buff *skb, struct net_device *sb_dev) { struct fec_enet_private *fep = netdev_priv(ndev); - u16 vlan_tag; + u16 vlan_tag = 0; if (!(fep->quirks & FEC_QUIRK_HAS_AVB)) return netdev_pick_tx(ndev, skb, NULL); - vlan_tag = fec_enet_get_raw_vlan_tci(skb); - if (!vlan_tag) + /* VLAN is present in the payload.*/ + if (eth_type_vlan(skb->protocol)) { + struct vlan_ethhdr *vhdr = skb_vlan_eth_hdr(skb); + + vlan_tag = ntohs(vhdr->h_vlan_TCI); + /* VLAN is present in the skb but not yet pushed in the payload.*/ + } else if (skb_vlan_tag_present(skb)) { + vlan_tag = skb->vlan_tci; + } else { return vlan_tag; + } return fec_enet_vlan_pri_to_queue[vlan_tag >> 13]; } @@ -3690,12 +3806,18 @@ static int fec_enet_bpf(struct net_device *dev, struct netdev_bpf *bpf) if (fep->quirks & FEC_QUIRK_SWAP_FRAME) return -EOPNOTSUPP; + if (!bpf->prog) + xdp_features_clear_redirect_target(dev); + if (is_run) { napi_disable(&fep->napi); netif_tx_disable(dev); } old_prog = xchg(&fep->xdp_prog, bpf->prog); + if (old_prog) + bpf_prog_put(old_prog); + fec_restart(dev); if (is_run) { @@ -3703,8 +3825,8 @@ static int fec_enet_bpf(struct net_device *dev, struct netdev_bpf *bpf) netif_tx_start_all_queues(dev); } - if (old_prog) - bpf_prog_put(old_prog); + if (bpf->prog) + xdp_features_set_redirect_target(dev, false); return 0; @@ -3727,38 +3849,59 @@ fec_enet_xdp_get_tx_queue(struct fec_enet_private *fep, int index) static int fec_enet_txq_xmit_frame(struct fec_enet_private *fep, struct fec_enet_priv_tx_q *txq, - struct xdp_frame *frame) + void *frame, u32 dma_sync_len, + bool ndo_xmit) { unsigned int index, status, estatus; - struct bufdesc *bdp, *last_bdp; + struct bufdesc *bdp; dma_addr_t dma_addr; int entries_free; + u16 frame_len; entries_free = fec_enet_get_free_txdesc_num(txq); if (entries_free < MAX_SKB_FRAGS + 1) { - netdev_err(fep->netdev, "NOT enough BD for SG!\n"); - return NETDEV_TX_OK; + netdev_err_once(fep->netdev, "NOT enough BD for SG!\n"); + return -EBUSY; } /* Fill in a Tx ring entry */ bdp = txq->bd.cur; - last_bdp = bdp; status = fec16_to_cpu(bdp->cbd_sc); status &= ~BD_ENET_TX_STATS; index = fec_enet_get_bd_index(bdp, &txq->bd); - dma_addr = dma_map_single(&fep->pdev->dev, frame->data, - frame->len, DMA_TO_DEVICE); - if (dma_mapping_error(&fep->pdev->dev, dma_addr)) - return FEC_ENET_XDP_CONSUMED; + if (ndo_xmit) { + struct xdp_frame *xdpf = frame; + + dma_addr = dma_map_single(&fep->pdev->dev, xdpf->data, + xdpf->len, DMA_TO_DEVICE); + if (dma_mapping_error(&fep->pdev->dev, dma_addr)) + return -ENOMEM; + + frame_len = xdpf->len; + txq->tx_buf[index].buf_p = xdpf; + txq->tx_buf[index].type = FEC_TXBUF_T_XDP_NDO; + } else { + struct xdp_buff *xdpb = frame; + struct page *page; + + page = virt_to_page(xdpb->data); + dma_addr = page_pool_get_dma_addr(page) + + (xdpb->data - xdpb->data_hard_start); + dma_sync_single_for_device(&fep->pdev->dev, dma_addr, + dma_sync_len, DMA_BIDIRECTIONAL); + frame_len = xdpb->data_end - xdpb->data; + txq->tx_buf[index].buf_p = page; + txq->tx_buf[index].type = FEC_TXBUF_T_XDP_TX; + } status |= (BD_ENET_TX_INTR | BD_ENET_TX_LAST); if (fep->bufdesc_ex) estatus = BD_ENET_TX_INT; bdp->cbd_bufaddr = cpu_to_fec32(dma_addr); - bdp->cbd_datlen = cpu_to_fec16(frame->len); + bdp->cbd_datlen = cpu_to_fec16(frame_len); if (fep->bufdesc_ex) { struct bufdesc_ex *ebdp = (struct bufdesc_ex *)bdp; @@ -3770,8 +3913,10 @@ static int fec_enet_txq_xmit_frame(struct fec_enet_private *fep, ebdp->cbd_esc = cpu_to_fec32(estatus); } - index = fec_enet_get_bd_index(last_bdp, &txq->bd); - txq->tx_skbuff[index] = NULL; + /* Make sure the updates to rest of the descriptor are performed before + * transferring ownership. + */ + dma_wmb(); /* Send it on its way. Tell FEC it's ready, interrupt when done, * it's the last BD of the frame, and to put the CRC on the end. @@ -3780,13 +3925,42 @@ static int fec_enet_txq_xmit_frame(struct fec_enet_private *fep, bdp->cbd_sc = cpu_to_fec16(status); /* If this was the last BD in the ring, start at the beginning again. */ - bdp = fec_enet_get_nextdesc(last_bdp, &txq->bd); + bdp = fec_enet_get_nextdesc(bdp, &txq->bd); + + /* Make sure the update to bdp are performed before txq->bd.cur. */ + dma_wmb(); txq->bd.cur = bdp; + /* Trigger transmission start */ + writel(0, txq->bd.reg_desc_active); + return 0; } +static int fec_enet_xdp_tx_xmit(struct fec_enet_private *fep, + int cpu, struct xdp_buff *xdp, + u32 dma_sync_len) +{ + struct fec_enet_priv_tx_q *txq; + struct netdev_queue *nq; + int queue, ret; + + queue = fec_enet_xdp_get_tx_queue(fep, cpu); + txq = fep->tx_queue[queue]; + nq = netdev_get_tx_queue(fep->netdev, queue); + + __netif_tx_lock(nq, cpu); + + /* Avoid tx timeout as XDP shares the queue with kernel stack */ + txq_trans_cond_update(nq); + ret = fec_enet_txq_xmit_frame(fep, txq, xdp, dma_sync_len, false); + + __netif_tx_unlock(nq); + + return ret; +} + static int fec_enet_xdp_xmit(struct net_device *dev, int num_frames, struct xdp_frame **frames, @@ -3795,6 +3969,7 @@ static int fec_enet_xdp_xmit(struct net_device *dev, struct fec_enet_private *fep = netdev_priv(dev); struct fec_enet_priv_tx_q *txq; int cpu = smp_processor_id(); + unsigned int sent_frames = 0; struct netdev_queue *nq; unsigned int queue; int i; @@ -3805,18 +3980,65 @@ static int fec_enet_xdp_xmit(struct net_device *dev, __netif_tx_lock(nq, cpu); - for (i = 0; i < num_frames; i++) - fec_enet_txq_xmit_frame(fep, txq, frames[i]); + /* Avoid tx timeout as XDP shares the queue with kernel stack */ + txq_trans_cond_update(nq); + for (i = 0; i < num_frames; i++) { + if (fec_enet_txq_xmit_frame(fep, txq, frames[i], 0, true) < 0) + break; + sent_frames++; + } - /* Make sure the update to bdp and tx_skbuff are performed. */ - wmb(); + __netif_tx_unlock(nq); - /* Trigger transmission start */ - writel(0, txq->bd.reg_desc_active); + return sent_frames; +} - __netif_tx_unlock(nq); +static int fec_hwtstamp_get(struct net_device *ndev, + struct kernel_hwtstamp_config *config) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + + if (!netif_running(ndev)) + return -EINVAL; + + if (!fep->bufdesc_ex) + return -EOPNOTSUPP; + + fec_ptp_get(ndev, config); + + return 0; +} + +static int fec_hwtstamp_set(struct net_device *ndev, + struct kernel_hwtstamp_config *config, + struct netlink_ext_ack *extack) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + + if (!netif_running(ndev)) + return -EINVAL; + + if (!fep->bufdesc_ex) + return -EOPNOTSUPP; + + return fec_ptp_set(ndev, config, extack); +} - return num_frames; +static int fec_change_mtu(struct net_device *ndev, int new_mtu) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + int order; + + if (netif_running(ndev)) + return -EBUSY; + + order = get_order(new_mtu + ETH_HLEN + ETH_FCS_LEN + + FEC_DRV_RESERVE_SPACE); + fep->rx_frame_size = (PAGE_SIZE << order) - FEC_DRV_RESERVE_SPACE; + fep->pagepool_order = order; + WRITE_ONCE(ndev->mtu, new_mtu); + + return 0; } static const struct net_device_ops fec_netdev_ops = { @@ -3828,13 +4050,13 @@ static const struct net_device_ops fec_netdev_ops = { .ndo_validate_addr = eth_validate_addr, .ndo_tx_timeout = fec_timeout, .ndo_set_mac_address = fec_set_mac_address, - .ndo_eth_ioctl = fec_enet_ioctl, -#ifdef CONFIG_NET_POLL_CONTROLLER - .ndo_poll_controller = fec_poll_controller, -#endif + .ndo_change_mtu = fec_change_mtu, + .ndo_eth_ioctl = phy_do_ioctl_running, .ndo_set_features = fec_set_features, .ndo_bpf = fec_enet_bpf, .ndo_xdp_xmit = fec_enet_xdp_xmit, + .ndo_hwtstamp_get = fec_hwtstamp_get, + .ndo_hwtstamp_set = fec_hwtstamp_set, }; static const unsigned short offset_des_active_rxq[] = { @@ -3863,10 +4085,8 @@ static int fec_enet_init(struct net_device *ndev) WARN_ON(dsize != (1 << dsize_log2)); #if defined(CONFIG_ARM) || defined(CONFIG_ARM64) - fep->rx_align = 0xf; fep->tx_align = 0xf; #else - fep->rx_align = 0x3; fep->tx_align = 0x3; #endif fep->rx_pkts_itr = FEC_ITR_ICFT_DEFAULT; @@ -3888,8 +4108,8 @@ static int fec_enet_init(struct net_device *ndev) bd_size = (fep->total_tx_ring_size + fep->total_rx_ring_size) * dsize; /* Allocate memory for buffer descriptors. */ - cbd_base = dmam_alloc_coherent(&fep->pdev->dev, bd_size, &bd_dma, - GFP_KERNEL); + cbd_base = fec_dmam_alloc(&fep->pdev->dev, bd_size, &bd_dma, + GFP_KERNEL); if (!cbd_base) { ret = -ENOMEM; goto free_queue_mem; @@ -3900,9 +4120,6 @@ static int fec_enet_init(struct net_device *ndev) if (ret) goto free_queue_mem; - /* make sure MAC we just acquired is programmed into the hw */ - fec_set_mac_address(ndev, NULL); - /* Set receive and transmit descriptor base. */ for (i = 0; i < fep->num_rx_queues; i++) { struct fec_enet_priv_rx_q *rxq = fep->rx_queue[i]; @@ -3958,13 +4175,15 @@ static int fec_enet_init(struct net_device *ndev) fep->csum_flags |= FLAG_RX_CSUM_ENABLED; } - if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES) { + if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES) fep->tx_align = 0; - fep->rx_align = 0x3f; - } ndev->hw_features = ndev->features; + if (!(fep->quirks & FEC_QUIRK_SWAP_FRAME)) + ndev->xdp_features = NETDEV_XDP_ACT_BASIC | + NETDEV_XDP_ACT_REDIRECT; + fec_restart(ndev); if (fep->quirks & FEC_QUIRK_MIB_CLEAR) @@ -3979,13 +4198,21 @@ free_queue_mem: return ret; } +static void fec_enet_deinit(struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + + netif_napi_del(&fep->napi); + fec_enet_free_queue(ndev); +} + #ifdef CONFIG_OF static int fec_reset_phy(struct platform_device *pdev) { - int err, phy_reset; - bool active_high = false; + struct gpio_desc *phy_reset; int msec = 1, phy_post_delay = 0; struct device_node *np = pdev->dev.of_node; + int err; if (!np) return 0; @@ -3995,33 +4222,26 @@ static int fec_reset_phy(struct platform_device *pdev) if (!err && msec > 1000) msec = 1; - phy_reset = of_get_named_gpio(np, "phy-reset-gpios", 0); - if (phy_reset == -EPROBE_DEFER) - return phy_reset; - else if (!gpio_is_valid(phy_reset)) - return 0; - err = of_property_read_u32(np, "phy-reset-post-delay", &phy_post_delay); /* valid reset duration should be less than 1s */ if (!err && phy_post_delay > 1000) return -EINVAL; - active_high = of_property_read_bool(np, "phy-reset-active-high"); + phy_reset = devm_gpiod_get_optional(&pdev->dev, "phy-reset", + GPIOD_OUT_HIGH); + if (IS_ERR(phy_reset)) + return dev_err_probe(&pdev->dev, PTR_ERR(phy_reset), + "failed to get phy-reset-gpios\n"); - err = devm_gpio_request_one(&pdev->dev, phy_reset, - active_high ? GPIOF_OUT_INIT_HIGH : GPIOF_OUT_INIT_LOW, - "phy-reset"); - if (err) { - dev_err(&pdev->dev, "failed to get phy-reset-gpios: %d\n", err); - return err; - } + if (!phy_reset) + return 0; if (msec > 20) msleep(msec); else usleep_range(msec * 1000, msec * 1000 + 1000); - gpio_set_value_cansleep(phy_reset, !active_high); + gpiod_set_value_cansleep(phy_reset, 0); if (!phy_post_delay) return 0; @@ -4143,14 +4363,13 @@ fec_probe(struct platform_device *pdev) phy_interface_t interface; struct net_device *ndev; int i, irq, ret = 0; - const struct of_device_id *of_id; static int dev_id; struct device_node *np = pdev->dev.of_node, *phy_node; int num_tx_qs; int num_rx_qs; char irq_name[8]; int irq_cnt; - struct fec_devinfo *dev_info; + const struct fec_devinfo *dev_info; fec_enet_get_queue_num(pdev, &num_tx_qs, &num_rx_qs); @@ -4165,10 +4384,9 @@ fec_probe(struct platform_device *pdev) /* setup board info structure */ fep = netdev_priv(ndev); - of_id = of_match_device(fec_dt_ids, &pdev->dev); - if (of_id) - pdev->id_entry = of_id->data; - dev_info = (struct fec_devinfo *)pdev->id_entry->driver_data; + dev_info = device_get_match_data(&pdev->dev); + if (!dev_info) + dev_info = (const struct fec_devinfo *)pdev->id_entry->driver_data; if (dev_info) fep->quirks = dev_info->quirks; @@ -4176,11 +4394,9 @@ fec_probe(struct platform_device *pdev) fep->num_rx_queues = num_rx_qs; fep->num_tx_queues = num_tx_qs; -#if !defined(CONFIG_M5272) /* default enable pause frame auto negotiation */ if (fep->quirks & FEC_QUIRK_HAS_GBIT) fep->pause_flag |= FEC_PAUSE_FLAG_AUTONEG; -#endif /* Select default pin state */ pinctrl_pm_select_default_state(&pdev->dev); @@ -4205,7 +4421,7 @@ fec_probe(struct platform_device *pdev) if (ret) goto failed_ipc_init; - if (of_get_property(np, "fsl,magic-packet", NULL)) + if (of_property_read_bool(np, "fsl,magic-packet")) fep->wol_flag |= FEC_WOL_HAS_MAGIC_PACKET; ret = fec_enet_init_stop_mode(fep, np); @@ -4359,7 +4575,15 @@ fec_probe(struct platform_device *pdev) fec_enet_clk_enable(ndev, false); pinctrl_pm_select_sleep_state(&pdev->dev); - ndev->max_mtu = PKT_MAXBUF_SIZE - ETH_HLEN - ETH_FCS_LEN; + fep->pagepool_order = 0; + fep->rx_frame_size = FEC_ENET_RX_FRSIZE; + + if (fep->quirks & FEC_QUIRK_JUMBO_FRAME) + fep->max_buf_size = MAX_JUMBO_BUF_SIZE; + else + fep->max_buf_size = PKT_MAXBUF_SIZE; + + ndev->max_mtu = fep->max_buf_size - ETH_HLEN - ETH_FCS_LEN; ret = register_netdev(ndev); if (ret) @@ -4371,10 +4595,8 @@ fec_probe(struct platform_device *pdev) if (fep->bufdesc_ex && fep->ptp_clock) netdev_info(ndev, "registered PHC device %d\n", fep->dev_id); - fep->rx_copybreak = COPYBREAK_DEFAULT; INIT_WORK(&fep->tx_timeout_work, fec_enet_timeout_work); - pm_runtime_mark_last_busy(&pdev->dev); pm_runtime_put_autosuspend(&pdev->dev); return 0; @@ -4383,6 +4605,7 @@ failed_register: fec_enet_mii_remove(fep); failed_mii_init: failed_irq: + fec_enet_deinit(ndev); failed_init: fec_ptp_stop(pdev); failed_reset: @@ -4411,7 +4634,7 @@ failed_ioremap: return ret; } -static int +static void fec_drv_remove(struct platform_device *pdev) { struct net_device *ndev = platform_get_drvdata(pdev); @@ -4419,9 +4642,11 @@ fec_drv_remove(struct platform_device *pdev) struct device_node *np = pdev->dev.of_node; int ret; - ret = pm_runtime_resume_and_get(&pdev->dev); + ret = pm_runtime_get_sync(&pdev->dev); if (ret < 0) - return ret; + dev_err(&pdev->dev, + "Failed to resume device in remove callback (%pe)\n", + ERR_PTR(ret)); cancel_work_sync(&fep->tx_timeout_work); fec_ptp_stop(pdev); @@ -4434,16 +4659,21 @@ fec_drv_remove(struct platform_device *pdev) of_phy_deregister_fixed_link(np); of_node_put(fep->phy_node); - clk_disable_unprepare(fep->clk_ahb); - clk_disable_unprepare(fep->clk_ipg); + /* After pm_runtime_get_sync() failed, the clks are still off, so skip + * disabling them again. + */ + if (ret >= 0) { + clk_disable_unprepare(fep->clk_ahb); + clk_disable_unprepare(fep->clk_ipg); + } pm_runtime_put_noidle(&pdev->dev); pm_runtime_disable(&pdev->dev); + fec_enet_deinit(ndev); free_netdev(ndev); - return 0; } -static int __maybe_unused fec_suspend(struct device *dev) +static int fec_suspend(struct device *dev) { struct net_device *ndev = dev_get_drvdata(dev); struct fec_enet_private *fep = netdev_priv(ndev); @@ -4496,7 +4726,7 @@ static int __maybe_unused fec_suspend(struct device *dev) return 0; } -static int __maybe_unused fec_resume(struct device *dev) +static int fec_resume(struct device *dev) { struct net_device *ndev = dev_get_drvdata(dev); struct fec_enet_private *fep = netdev_priv(ndev); @@ -4551,7 +4781,7 @@ failed_clk: return ret; } -static int __maybe_unused fec_runtime_suspend(struct device *dev) +static int fec_runtime_suspend(struct device *dev) { struct net_device *ndev = dev_get_drvdata(dev); struct fec_enet_private *fep = netdev_priv(ndev); @@ -4562,7 +4792,7 @@ static int __maybe_unused fec_runtime_suspend(struct device *dev) return 0; } -static int __maybe_unused fec_runtime_resume(struct device *dev) +static int fec_runtime_resume(struct device *dev) { struct net_device *ndev = dev_get_drvdata(dev); struct fec_enet_private *fep = netdev_priv(ndev); @@ -4583,22 +4813,23 @@ failed_clk_ipg: } static const struct dev_pm_ops fec_pm_ops = { - SET_SYSTEM_SLEEP_PM_OPS(fec_suspend, fec_resume) - SET_RUNTIME_PM_OPS(fec_runtime_suspend, fec_runtime_resume, NULL) + SYSTEM_SLEEP_PM_OPS(fec_suspend, fec_resume) + RUNTIME_PM_OPS(fec_runtime_suspend, fec_runtime_resume, NULL) }; static struct platform_driver fec_driver = { .driver = { .name = DRIVER_NAME, - .pm = &fec_pm_ops, + .pm = pm_ptr(&fec_pm_ops), .of_match_table = fec_dt_ids, .suppress_bind_attrs = true, }, .id_table = fec_devtype, .probe = fec_probe, - .remove = fec_drv_remove, + .remove = fec_drv_remove, }; module_platform_driver(fec_driver); +MODULE_DESCRIPTION("NXP Fast Ethernet Controller (FEC) driver"); MODULE_LICENSE("GPL"); |