diff options
Diffstat (limited to 'drivers/net/ethernet/cadence/macb_main.c')
| -rw-r--r-- | drivers/net/ethernet/cadence/macb_main.c | 4155 |
1 files changed, 3219 insertions, 936 deletions
diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index 26d25749c3e4..e461f5072884 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -1,82 +1,96 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Cadence MACB/GEM Ethernet Controller driver * * Copyright (C) 2004-2006 Atmel Corporation - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include <linux/clk.h> -#include <linux/module.h> -#include <linux/moduleparam.h> -#include <linux/kernel.h> -#include <linux/types.h> #include <linux/circ_buf.h> -#include <linux/slab.h> +#include <linux/clk-provider.h> +#include <linux/clk.h> +#include <linux/crc32.h> +#include <linux/dma-mapping.h> +#include <linux/etherdevice.h> +#include <linux/firmware/xlnx-zynqmp.h> +#include <linux/inetdevice.h> #include <linux/init.h> -#include <linux/io.h> -#include <linux/gpio.h> -#include <linux/gpio/consumer.h> #include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/ip.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/moduleparam.h> #include <linux/netdevice.h> -#include <linux/etherdevice.h> -#include <linux/dma-mapping.h> -#include <linux/platform_data/macb.h> -#include <linux/platform_device.h> -#include <linux/phy.h> #include <linux/of.h> -#include <linux/of_device.h> -#include <linux/of_gpio.h> #include <linux/of_mdio.h> #include <linux/of_net.h> -#include <linux/ip.h> -#include <linux/udp.h> +#include <linux/phy/phy.h> +#include <linux/phylink.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/ptp_classify.h> +#include <linux/reset.h> +#include <linux/slab.h> #include <linux/tcp.h> +#include <linux/types.h> +#include <linux/udp.h> +#include <net/pkt_sched.h> #include "macb.h" +/* This structure is only used for MACB on SiFive FU540 devices */ +struct sifive_fu540_macb_mgmt { + void __iomem *reg; + unsigned long rate; + struct clk_hw hw; +}; + #define MACB_RX_BUFFER_SIZE 128 #define RX_BUFFER_MULTIPLE 64 /* bytes */ #define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */ #define MIN_RX_RING_SIZE 64 #define MAX_RX_RING_SIZE 8192 -#define RX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ - * (bp)->rx_ring_size) #define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */ #define MIN_TX_RING_SIZE 64 #define MAX_TX_RING_SIZE 4096 -#define TX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ - * (bp)->tx_ring_size) /* level of occupied TX descriptors under which we wake up TX process */ #define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4) -#define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(RXUBR) \ - | MACB_BIT(ISR_ROVR)) +#define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(ISR_ROVR)) #define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \ | MACB_BIT(ISR_RLE) \ | MACB_BIT(TXERR)) -#define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP)) +#define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP) \ + | MACB_BIT(TXUBR)) /* Max length of transmit frame must be a multiple of 8 bytes */ #define MACB_TX_LEN_ALIGN 8 #define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) -#define GEM_MAX_TX_LEN ((unsigned int)((1 << GEM_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) +/* Limit maximum TX length as per Cadence TSO errata. This is to avoid a + * false amba_error in TX path from the DMA assuming there is not enough + * space in the SRAM (16KB) even when there is. + */ +#define GEM_MAX_TX_LEN (unsigned int)(0x3FC0) #define GEM_MTU_MIN_SIZE ETH_MIN_MTU -#define MACB_NETIF_LSO (NETIF_F_TSO | NETIF_F_UFO) +#define MACB_NETIF_LSO NETIF_F_TSO -#define MACB_WOL_HAS_MAGIC_PACKET (0x1 << 0) -#define MACB_WOL_ENABLED (0x1 << 1) +#define MACB_WOL_ENABLED BIT(0) + +#define HS_SPEED_10000M 4 +#define MACB_SERDES_RATE_10G 1 /* Graceful stop timeouts in us. We should allow up to * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions) */ -#define MACB_HALT_TIMEOUT 1230 +#define MACB_HALT_TIMEOUT 14000 +#define MACB_PM_TIMEOUT 100 /* ms */ + +#define MACB_MDIO_TIMEOUT 1000000 /* in usecs */ /* DMA buffer descriptor might be different size * depends on hardware configuration: @@ -107,57 +121,26 @@ */ static unsigned int macb_dma_desc_get_size(struct macb *bp) { -#ifdef MACB_EXT_DESC - unsigned int desc_size; + unsigned int desc_size = sizeof(struct macb_dma_desc); + + if (macb_dma64(bp)) + desc_size += sizeof(struct macb_dma_desc_64); + if (macb_dma_ptp(bp)) + desc_size += sizeof(struct macb_dma_desc_ptp); - switch (bp->hw_dma_cap) { - case HW_DMA_CAP_64B: - desc_size = sizeof(struct macb_dma_desc) - + sizeof(struct macb_dma_desc_64); - break; - case HW_DMA_CAP_PTP: - desc_size = sizeof(struct macb_dma_desc) - + sizeof(struct macb_dma_desc_ptp); - break; - case HW_DMA_CAP_64B_PTP: - desc_size = sizeof(struct macb_dma_desc) - + sizeof(struct macb_dma_desc_64) - + sizeof(struct macb_dma_desc_ptp); - break; - default: - desc_size = sizeof(struct macb_dma_desc); - } return desc_size; -#endif - return sizeof(struct macb_dma_desc); } static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx) { -#ifdef MACB_EXT_DESC - switch (bp->hw_dma_cap) { - case HW_DMA_CAP_64B: - case HW_DMA_CAP_PTP: - desc_idx <<= 1; - break; - case HW_DMA_CAP_64B_PTP: - desc_idx *= 3; - break; - default: - break; - } -#endif - return desc_idx; + return desc_idx * (1 + macb_dma64(bp) + macb_dma_ptp(bp)); } -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc) { - if (bp->hw_dma_cap & HW_DMA_CAP_64B) - return (struct macb_dma_desc_64 *)((void *)desc + sizeof(struct macb_dma_desc)); - return NULL; + return (struct macb_dma_desc_64 *)((void *)desc + + sizeof(struct macb_dma_desc)); } -#endif /* Ring buffer accessors */ static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index) @@ -194,17 +177,17 @@ static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index) return index & (bp->rx_ring_size - 1); } -static struct macb_dma_desc *macb_rx_desc(struct macb *bp, unsigned int index) +static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index) { - index = macb_rx_ring_wrap(bp, index); - index = macb_adj_dma_desc_idx(bp, index); - return &bp->rx_ring[index]; + index = macb_rx_ring_wrap(queue->bp, index); + index = macb_adj_dma_desc_idx(queue->bp, index); + return &queue->rx_ring[index]; } -static void *macb_rx_buffer(struct macb *bp, unsigned int index) +static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index) { - return bp->rx_buffers + bp->rx_buffer_size * - macb_rx_ring_wrap(bp, index); + return queue->rx_buffers + queue->bp->rx_buffer_size * + macb_rx_ring_wrap(queue->bp, index); } /* I/O accessors */ @@ -262,11 +245,16 @@ static void macb_set_hwaddr(struct macb *bp) u32 bottom; u16 top; - bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr)); + bottom = get_unaligned_le32(bp->dev->dev_addr); macb_or_gem_writel(bp, SA1B, bottom); - top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4))); + top = get_unaligned_le16(bp->dev->dev_addr + 4); macb_or_gem_writel(bp, SA1T, top); + if (gem_has_ptp(bp)) { + gem_writel(bp, RXPTPUNI, bottom); + gem_writel(bp, TXPTPUNI, bottom); + } + /* Clear unused address register sets */ macb_or_gem_writel(bp, SA2B, 0); macb_or_gem_writel(bp, SA2T, 0); @@ -278,37 +266,25 @@ static void macb_set_hwaddr(struct macb *bp) static void macb_get_hwaddr(struct macb *bp) { - struct macb_platform_data *pdata; u32 bottom; u16 top; u8 addr[6]; int i; - pdata = dev_get_platdata(&bp->pdev->dev); - /* Check all 4 address register for valid address */ for (i = 0; i < 4; i++) { bottom = macb_or_gem_readl(bp, SA1B + i * 8); top = macb_or_gem_readl(bp, SA1T + i * 8); - if (pdata && pdata->rev_eth_addr) { - addr[5] = bottom & 0xff; - addr[4] = (bottom >> 8) & 0xff; - addr[3] = (bottom >> 16) & 0xff; - addr[2] = (bottom >> 24) & 0xff; - addr[1] = top & 0xff; - addr[0] = (top & 0xff00) >> 8; - } else { - addr[0] = bottom & 0xff; - addr[1] = (bottom >> 8) & 0xff; - addr[2] = (bottom >> 16) & 0xff; - addr[3] = (bottom >> 24) & 0xff; - addr[4] = top & 0xff; - addr[5] = (top >> 8) & 0xff; - } + addr[0] = bottom & 0xff; + addr[1] = (bottom >> 8) & 0xff; + addr[2] = (bottom >> 16) & 0xff; + addr[3] = (bottom >> 24) & 0xff; + addr[4] = top & 0xff; + addr[5] = (top >> 8) & 0xff; if (is_valid_ether_addr(addr)) { - memcpy(bp->dev->dev_addr, addr, sizeof(addr)); + eth_hw_addr_set(bp->dev, addr); return; } } @@ -317,73 +293,206 @@ static void macb_get_hwaddr(struct macb *bp) eth_hw_addr_random(bp->dev); } -static int macb_mdio_read(struct mii_bus *bus, int mii_id, int regnum) +static int macb_mdio_wait_for_idle(struct macb *bp) +{ + u32 val; + + return readx_poll_timeout(MACB_READ_NSR, bp, val, val & MACB_BIT(IDLE), + 1, MACB_MDIO_TIMEOUT); +} + +static int macb_mdio_read_c22(struct mii_bus *bus, int mii_id, int regnum) { struct macb *bp = bus->priv; - int value; + int status; + + status = pm_runtime_resume_and_get(&bp->pdev->dev); + if (status < 0) + goto mdio_pm_exit; - macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_SOF) - | MACB_BF(RW, MACB_MAN_READ) + status = macb_mdio_wait_for_idle(bp); + if (status < 0) + goto mdio_read_exit; + + macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF) + | MACB_BF(RW, MACB_MAN_C22_READ) | MACB_BF(PHYA, mii_id) | MACB_BF(REGA, regnum) - | MACB_BF(CODE, MACB_MAN_CODE))); + | MACB_BF(CODE, MACB_MAN_C22_CODE))); - /* wait for end of transfer */ - while (!MACB_BFEXT(IDLE, macb_readl(bp, NSR))) - cpu_relax(); + status = macb_mdio_wait_for_idle(bp); + if (status < 0) + goto mdio_read_exit; - value = MACB_BFEXT(DATA, macb_readl(bp, MAN)); + status = MACB_BFEXT(DATA, macb_readl(bp, MAN)); - return value; +mdio_read_exit: + pm_runtime_put_autosuspend(&bp->pdev->dev); +mdio_pm_exit: + return status; } -static int macb_mdio_write(struct mii_bus *bus, int mii_id, int regnum, - u16 value) +static int macb_mdio_read_c45(struct mii_bus *bus, int mii_id, int devad, + int regnum) { struct macb *bp = bus->priv; + int status; + + status = pm_runtime_get_sync(&bp->pdev->dev); + if (status < 0) { + pm_runtime_put_noidle(&bp->pdev->dev); + goto mdio_pm_exit; + } - macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_SOF) - | MACB_BF(RW, MACB_MAN_WRITE) + status = macb_mdio_wait_for_idle(bp); + if (status < 0) + goto mdio_read_exit; + + macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) + | MACB_BF(RW, MACB_MAN_C45_ADDR) + | MACB_BF(PHYA, mii_id) + | MACB_BF(REGA, devad & 0x1F) + | MACB_BF(DATA, regnum & 0xFFFF) + | MACB_BF(CODE, MACB_MAN_C45_CODE))); + + status = macb_mdio_wait_for_idle(bp); + if (status < 0) + goto mdio_read_exit; + + macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) + | MACB_BF(RW, MACB_MAN_C45_READ) + | MACB_BF(PHYA, mii_id) + | MACB_BF(REGA, devad & 0x1F) + | MACB_BF(CODE, MACB_MAN_C45_CODE))); + + status = macb_mdio_wait_for_idle(bp); + if (status < 0) + goto mdio_read_exit; + + status = MACB_BFEXT(DATA, macb_readl(bp, MAN)); + +mdio_read_exit: + pm_runtime_put_autosuspend(&bp->pdev->dev); +mdio_pm_exit: + return status; +} + +static int macb_mdio_write_c22(struct mii_bus *bus, int mii_id, int regnum, + u16 value) +{ + struct macb *bp = bus->priv; + int status; + + status = pm_runtime_resume_and_get(&bp->pdev->dev); + if (status < 0) + goto mdio_pm_exit; + + status = macb_mdio_wait_for_idle(bp); + if (status < 0) + goto mdio_write_exit; + + macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF) + | MACB_BF(RW, MACB_MAN_C22_WRITE) | MACB_BF(PHYA, mii_id) | MACB_BF(REGA, regnum) - | MACB_BF(CODE, MACB_MAN_CODE) + | MACB_BF(CODE, MACB_MAN_C22_CODE) | MACB_BF(DATA, value))); - /* wait for end of transfer */ - while (!MACB_BFEXT(IDLE, macb_readl(bp, NSR))) - cpu_relax(); + status = macb_mdio_wait_for_idle(bp); + if (status < 0) + goto mdio_write_exit; - return 0; +mdio_write_exit: + pm_runtime_put_autosuspend(&bp->pdev->dev); +mdio_pm_exit: + return status; +} + +static int macb_mdio_write_c45(struct mii_bus *bus, int mii_id, + int devad, int regnum, + u16 value) +{ + struct macb *bp = bus->priv; + int status; + + status = pm_runtime_get_sync(&bp->pdev->dev); + if (status < 0) { + pm_runtime_put_noidle(&bp->pdev->dev); + goto mdio_pm_exit; + } + + status = macb_mdio_wait_for_idle(bp); + if (status < 0) + goto mdio_write_exit; + + macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) + | MACB_BF(RW, MACB_MAN_C45_ADDR) + | MACB_BF(PHYA, mii_id) + | MACB_BF(REGA, devad & 0x1F) + | MACB_BF(DATA, regnum & 0xFFFF) + | MACB_BF(CODE, MACB_MAN_C45_CODE))); + + status = macb_mdio_wait_for_idle(bp); + if (status < 0) + goto mdio_write_exit; + + macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) + | MACB_BF(RW, MACB_MAN_C45_WRITE) + | MACB_BF(PHYA, mii_id) + | MACB_BF(REGA, devad & 0x1F) + | MACB_BF(CODE, MACB_MAN_C45_CODE) + | MACB_BF(DATA, value))); + + status = macb_mdio_wait_for_idle(bp); + if (status < 0) + goto mdio_write_exit; + +mdio_write_exit: + pm_runtime_put_autosuspend(&bp->pdev->dev); +mdio_pm_exit: + return status; +} + +static void macb_init_buffers(struct macb *bp) +{ + struct macb_queue *queue; + unsigned int q; + + /* Single register for all queues' high 32 bits. */ + if (macb_dma64(bp)) { + macb_writel(bp, RBQPH, + upper_32_bits(bp->queues[0].rx_ring_dma)); + macb_writel(bp, TBQPH, + upper_32_bits(bp->queues[0].tx_ring_dma)); + } + + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); + queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); + } } /** * macb_set_tx_clk() - Set a clock to a new frequency - * @clk Pointer to the clock to change - * @rate New frequency in Hz - * @dev Pointer to the struct net_device + * @bp: pointer to struct macb + * @speed: New frequency in Hz */ -static void macb_set_tx_clk(struct clk *clk, int speed, struct net_device *dev) +static void macb_set_tx_clk(struct macb *bp, int speed) { long ferr, rate, rate_rounded; - if (!clk) + if (!bp->tx_clk || (bp->caps & MACB_CAPS_CLK_HW_CHG)) return; - switch (speed) { - case SPEED_10: - rate = 2500000; - break; - case SPEED_100: - rate = 25000000; - break; - case SPEED_1000: - rate = 125000000; - break; - default: + /* In case of MII the PHY is the clock master */ + if (bp->phy_interface == PHY_INTERFACE_MODE_MII) return; - } - rate_rounded = clk_round_rate(clk, rate); + rate = rgmii_clock(speed); + if (rate < 0) + return; + + rate_rounded = clk_round_rate(bp->tx_clk, rate); if (rate_rounded < 0) return; @@ -393,147 +502,406 @@ static void macb_set_tx_clk(struct clk *clk, int speed, struct net_device *dev) ferr = abs(rate_rounded - rate); ferr = DIV_ROUND_UP(ferr, rate / 100000); if (ferr > 5) - netdev_warn(dev, "unable to generate target frequency: %ld Hz\n", + netdev_warn(bp->dev, + "unable to generate target frequency: %ld Hz\n", rate); - if (clk_set_rate(clk, rate_rounded)) - netdev_err(dev, "adjusting tx_clk failed.\n"); + if (clk_set_rate(bp->tx_clk, rate_rounded)) + netdev_err(bp->dev, "adjusting tx_clk failed.\n"); } -static void macb_handle_link_change(struct net_device *dev) +static void macb_usx_pcs_link_up(struct phylink_pcs *pcs, unsigned int neg_mode, + phy_interface_t interface, int speed, + int duplex) { - struct macb *bp = netdev_priv(dev); - struct phy_device *phydev = dev->phydev; - unsigned long flags; - int status_change = 0; + struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs); + u32 config; - spin_lock_irqsave(&bp->lock, flags); + config = gem_readl(bp, USX_CONTROL); + config = GEM_BFINS(SERDES_RATE, MACB_SERDES_RATE_10G, config); + config = GEM_BFINS(USX_CTRL_SPEED, HS_SPEED_10000M, config); + config &= ~(GEM_BIT(TX_SCR_BYPASS) | GEM_BIT(RX_SCR_BYPASS)); + config |= GEM_BIT(TX_EN); + gem_writel(bp, USX_CONTROL, config); +} - if (phydev->link) { - if ((bp->speed != phydev->speed) || - (bp->duplex != phydev->duplex)) { - u32 reg; +static void macb_usx_pcs_get_state(struct phylink_pcs *pcs, + unsigned int neg_mode, + struct phylink_link_state *state) +{ + struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs); + u32 val; - reg = macb_readl(bp, NCFGR); - reg &= ~(MACB_BIT(SPD) | MACB_BIT(FD)); - if (macb_is_gem(bp)) - reg &= ~GEM_BIT(GBE); + state->speed = SPEED_10000; + state->duplex = 1; + state->an_complete = 1; + + val = gem_readl(bp, USX_STATUS); + state->link = !!(val & GEM_BIT(USX_BLOCK_LOCK)); + val = gem_readl(bp, NCFGR); + if (val & GEM_BIT(PAE)) + state->pause = MLO_PAUSE_RX; +} + +static int macb_usx_pcs_config(struct phylink_pcs *pcs, + unsigned int neg_mode, + phy_interface_t interface, + const unsigned long *advertising, + bool permit_pause_to_mac) +{ + struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs); + + gem_writel(bp, USX_CONTROL, gem_readl(bp, USX_CONTROL) | + GEM_BIT(SIGNAL_OK)); + + return 0; +} + +static void macb_pcs_get_state(struct phylink_pcs *pcs, unsigned int neg_mode, + struct phylink_link_state *state) +{ + state->link = 0; +} + +static void macb_pcs_an_restart(struct phylink_pcs *pcs) +{ + /* Not supported */ +} + +static int macb_pcs_config(struct phylink_pcs *pcs, + unsigned int neg_mode, + phy_interface_t interface, + const unsigned long *advertising, + bool permit_pause_to_mac) +{ + return 0; +} - if (phydev->duplex) - reg |= MACB_BIT(FD); - if (phydev->speed == SPEED_100) - reg |= MACB_BIT(SPD); - if (phydev->speed == SPEED_1000 && - bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE) - reg |= GEM_BIT(GBE); +static const struct phylink_pcs_ops macb_phylink_usx_pcs_ops = { + .pcs_get_state = macb_usx_pcs_get_state, + .pcs_config = macb_usx_pcs_config, + .pcs_link_up = macb_usx_pcs_link_up, +}; + +static const struct phylink_pcs_ops macb_phylink_pcs_ops = { + .pcs_get_state = macb_pcs_get_state, + .pcs_an_restart = macb_pcs_an_restart, + .pcs_config = macb_pcs_config, +}; + +static void macb_mac_config(struct phylink_config *config, unsigned int mode, + const struct phylink_link_state *state) +{ + struct net_device *ndev = to_net_dev(config->dev); + struct macb *bp = netdev_priv(ndev); + unsigned long flags; + u32 old_ctrl, ctrl; + u32 old_ncr, ncr; - macb_or_gem_writel(bp, NCFGR, reg); + spin_lock_irqsave(&bp->lock, flags); - bp->speed = phydev->speed; - bp->duplex = phydev->duplex; - status_change = 1; + old_ctrl = ctrl = macb_or_gem_readl(bp, NCFGR); + old_ncr = ncr = macb_or_gem_readl(bp, NCR); + + if (bp->caps & MACB_CAPS_MACB_IS_EMAC) { + if (state->interface == PHY_INTERFACE_MODE_RMII) + ctrl |= MACB_BIT(RM9200_RMII); + } else if (macb_is_gem(bp)) { + ctrl &= ~(GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL)); + ncr &= ~GEM_BIT(ENABLE_HS_MAC); + + if (state->interface == PHY_INTERFACE_MODE_SGMII) { + ctrl |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); + } else if (state->interface == PHY_INTERFACE_MODE_10GBASER) { + ctrl |= GEM_BIT(PCSSEL); + ncr |= GEM_BIT(ENABLE_HS_MAC); + } else if (bp->caps & MACB_CAPS_MIIONRGMII && + bp->phy_interface == PHY_INTERFACE_MODE_MII) { + ncr |= MACB_BIT(MIIONRGMII); } } - if (phydev->link != bp->link) { - if (!phydev->link) { - bp->speed = 0; - bp->duplex = -1; - } - bp->link = phydev->link; + /* Apply the new configuration, if any */ + if (old_ctrl ^ ctrl) + macb_or_gem_writel(bp, NCFGR, ctrl); + + if (old_ncr ^ ncr) + macb_or_gem_writel(bp, NCR, ncr); + + /* Disable AN for SGMII fixed link configuration, enable otherwise. + * Must be written after PCSSEL is set in NCFGR, + * otherwise writes will not take effect. + */ + if (macb_is_gem(bp) && state->interface == PHY_INTERFACE_MODE_SGMII) { + u32 pcsctrl, old_pcsctrl; - status_change = 1; + old_pcsctrl = gem_readl(bp, PCSCNTRL); + if (mode == MLO_AN_FIXED) + pcsctrl = old_pcsctrl & ~GEM_BIT(PCSAUTONEG); + else + pcsctrl = old_pcsctrl | GEM_BIT(PCSAUTONEG); + if (old_pcsctrl != pcsctrl) + gem_writel(bp, PCSCNTRL, pcsctrl); } spin_unlock_irqrestore(&bp->lock, flags); +} - if (status_change) { - if (phydev->link) { - /* Update the TX clock rate if and only if the link is - * up and there has been a link change. - */ - macb_set_tx_clk(bp->tx_clk, phydev->speed, dev); +static void macb_mac_link_down(struct phylink_config *config, unsigned int mode, + phy_interface_t interface) +{ + struct net_device *ndev = to_net_dev(config->dev); + struct macb *bp = netdev_priv(ndev); + struct macb_queue *queue; + unsigned int q; + u32 ctrl; - netif_carrier_on(dev); - netdev_info(dev, "link up (%d/%s)\n", - phydev->speed, - phydev->duplex == DUPLEX_FULL ? - "Full" : "Half"); - } else { - netif_carrier_off(dev); - netdev_info(dev, "link down\n"); + if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) + queue_writel(queue, IDR, + bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); + + /* Disable Rx and Tx */ + ctrl = macb_readl(bp, NCR) & ~(MACB_BIT(RE) | MACB_BIT(TE)); + macb_writel(bp, NCR, ctrl); + + netif_tx_stop_all_queues(ndev); +} + +static void macb_mac_link_up(struct phylink_config *config, + struct phy_device *phy, + unsigned int mode, phy_interface_t interface, + int speed, int duplex, + bool tx_pause, bool rx_pause) +{ + struct net_device *ndev = to_net_dev(config->dev); + struct macb *bp = netdev_priv(ndev); + struct macb_queue *queue; + unsigned long flags; + unsigned int q; + u32 ctrl; + + spin_lock_irqsave(&bp->lock, flags); + + ctrl = macb_or_gem_readl(bp, NCFGR); + + ctrl &= ~(MACB_BIT(SPD) | MACB_BIT(FD)); + + if (speed == SPEED_100) + ctrl |= MACB_BIT(SPD); + + if (duplex) + ctrl |= MACB_BIT(FD); + + if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { + ctrl &= ~MACB_BIT(PAE); + if (macb_is_gem(bp)) { + ctrl &= ~GEM_BIT(GBE); + + if (speed == SPEED_1000) + ctrl |= GEM_BIT(GBE); } + + if (rx_pause) + ctrl |= MACB_BIT(PAE); + + /* Initialize rings & buffers as clearing MACB_BIT(TE) in link down + * cleared the pipeline and control registers. + */ + bp->macbgem_ops.mog_init_rings(bp); + macb_init_buffers(bp); + + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) + queue_writel(queue, IER, + bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); } + + macb_or_gem_writel(bp, NCFGR, ctrl); + + if (bp->phy_interface == PHY_INTERFACE_MODE_10GBASER) + gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_10000M, + gem_readl(bp, HS_MAC_CONFIG))); + + spin_unlock_irqrestore(&bp->lock, flags); + + if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) + macb_set_tx_clk(bp, speed); + + /* Enable Rx and Tx; Enable PTP unicast */ + ctrl = macb_readl(bp, NCR); + if (gem_has_ptp(bp)) + ctrl |= MACB_BIT(PTPUNI); + + macb_writel(bp, NCR, ctrl | MACB_BIT(RE) | MACB_BIT(TE)); + + netif_tx_wake_all_queues(ndev); } -/* based on au1000_eth. c*/ -static int macb_mii_probe(struct net_device *dev) +static struct phylink_pcs *macb_mac_select_pcs(struct phylink_config *config, + phy_interface_t interface) { - struct macb *bp = netdev_priv(dev); - struct macb_platform_data *pdata; + struct net_device *ndev = to_net_dev(config->dev); + struct macb *bp = netdev_priv(ndev); + + if (interface == PHY_INTERFACE_MODE_10GBASER) + return &bp->phylink_usx_pcs; + else if (interface == PHY_INTERFACE_MODE_SGMII) + return &bp->phylink_sgmii_pcs; + else + return NULL; +} + +static const struct phylink_mac_ops macb_phylink_ops = { + .mac_select_pcs = macb_mac_select_pcs, + .mac_config = macb_mac_config, + .mac_link_down = macb_mac_link_down, + .mac_link_up = macb_mac_link_up, +}; + +static bool macb_phy_handle_exists(struct device_node *dn) +{ + dn = of_parse_phandle(dn, "phy-handle", 0); + of_node_put(dn); + return dn != NULL; +} + +static int macb_phylink_connect(struct macb *bp) +{ + struct device_node *dn = bp->pdev->dev.of_node; + struct net_device *dev = bp->dev; struct phy_device *phydev; - int phy_irq; int ret; - if (bp->phy_node) { - phydev = of_phy_connect(dev, bp->phy_node, - &macb_handle_link_change, 0, - bp->phy_interface); - if (!phydev) - return -ENODEV; - } else { + if (dn) + ret = phylink_of_phy_connect(bp->phylink, dn, 0); + + if (!dn || (ret && !macb_phy_handle_exists(dn))) { phydev = phy_find_first(bp->mii_bus); if (!phydev) { netdev_err(dev, "no PHY found\n"); return -ENXIO; } - pdata = dev_get_platdata(&bp->pdev->dev); - if (pdata) { - if (gpio_is_valid(pdata->phy_irq_pin)) { - ret = devm_gpio_request(&bp->pdev->dev, - pdata->phy_irq_pin, "phy int"); - if (!ret) { - phy_irq = gpio_to_irq(pdata->phy_irq_pin); - phydev->irq = (phy_irq < 0) ? PHY_POLL : phy_irq; - } - } else { - phydev->irq = PHY_POLL; - } - } - /* attach the mac to the phy */ - ret = phy_connect_direct(dev, phydev, &macb_handle_link_change, - bp->phy_interface); - if (ret) { - netdev_err(dev, "Could not attach to PHY\n"); - return ret; - } + ret = phylink_connect_phy(bp->phylink, phydev); } - /* mask with MAC supported features */ - if (macb_is_gem(bp) && bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE) - phydev->supported &= PHY_GBIT_FEATURES; - else - phydev->supported &= PHY_BASIC_FEATURES; + if (ret) { + netdev_err(dev, "Could not attach PHY (%d)\n", ret); + return ret; + } + + phylink_start(bp->phylink); - if (bp->caps & MACB_CAPS_NO_GIGABIT_HALF) - phydev->supported &= ~SUPPORTED_1000baseT_Half; + return 0; +} - phydev->advertising = phydev->supported; +static void macb_get_pcs_fixed_state(struct phylink_config *config, + struct phylink_link_state *state) +{ + struct net_device *ndev = to_net_dev(config->dev); + struct macb *bp = netdev_priv(ndev); - bp->link = 0; - bp->speed = 0; - bp->duplex = -1; + state->link = (macb_readl(bp, NSR) & MACB_BIT(NSR_LINK)) != 0; +} + +/* based on au1000_eth. c*/ +static int macb_mii_probe(struct net_device *dev) +{ + struct macb *bp = netdev_priv(dev); + + bp->phylink_sgmii_pcs.ops = &macb_phylink_pcs_ops; + bp->phylink_usx_pcs.ops = &macb_phylink_usx_pcs_ops; + + bp->phylink_config.dev = &dev->dev; + bp->phylink_config.type = PHYLINK_NETDEV; + bp->phylink_config.mac_managed_pm = true; + + if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { + bp->phylink_config.poll_fixed_state = true; + bp->phylink_config.get_fixed_state = macb_get_pcs_fixed_state; + } + + bp->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | + MAC_10 | MAC_100; + + __set_bit(PHY_INTERFACE_MODE_MII, + bp->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_RMII, + bp->phylink_config.supported_interfaces); + + /* Determine what modes are supported */ + if (macb_is_gem(bp) && (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE)) { + bp->phylink_config.mac_capabilities |= MAC_1000FD; + if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF)) + bp->phylink_config.mac_capabilities |= MAC_1000HD; + + __set_bit(PHY_INTERFACE_MODE_GMII, + bp->phylink_config.supported_interfaces); + phy_interface_set_rgmii(bp->phylink_config.supported_interfaces); + + if (bp->caps & MACB_CAPS_PCS) + __set_bit(PHY_INTERFACE_MODE_SGMII, + bp->phylink_config.supported_interfaces); + + if (bp->caps & MACB_CAPS_HIGH_SPEED) { + __set_bit(PHY_INTERFACE_MODE_10GBASER, + bp->phylink_config.supported_interfaces); + bp->phylink_config.mac_capabilities |= MAC_10000FD; + } + } + + bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode, + bp->phy_interface, &macb_phylink_ops); + if (IS_ERR(bp->phylink)) { + netdev_err(dev, "Could not create a phylink instance (%ld)\n", + PTR_ERR(bp->phylink)); + return PTR_ERR(bp->phylink); + } return 0; } +static int macb_mdiobus_register(struct macb *bp, struct device_node *mdio_np) +{ + struct device_node *child, *np = bp->pdev->dev.of_node; + + /* If we have a child named mdio, probe it instead of looking for PHYs + * directly under the MAC node + */ + if (mdio_np) + return of_mdiobus_register(bp->mii_bus, mdio_np); + + /* Only create the PHY from the device tree if at least one PHY is + * described. Otherwise scan the entire MDIO bus. We do this to support + * old device tree that did not follow the best practices and did not + * describe their network PHYs. + */ + for_each_available_child_of_node(np, child) + if (of_mdiobus_child_is_phy(child)) { + /* The loop increments the child refcount, + * decrement it before returning. + */ + of_node_put(child); + + return of_mdiobus_register(bp->mii_bus, np); + } + + return mdiobus_register(bp->mii_bus); +} + static int macb_mii_init(struct macb *bp) { - struct macb_platform_data *pdata; - struct device_node *np; - int err = -ENXIO, i; + struct device_node *mdio_np, *np = bp->pdev->dev.of_node; + int err = -ENXIO; + + /* With fixed-link, we don't need to register the MDIO bus, + * except if we have a child named "mdio" in the device tree. + * In that case, some devices may be attached to the MACB's MDIO bus. + */ + mdio_np = of_get_child_by_name(np, "mdio"); + if (!mdio_np && of_phy_is_fixed_link(np)) + return macb_mii_probe(bp->dev); /* Enable management port */ macb_writel(bp, NCR, MACB_BIT(MPE)); @@ -545,60 +913,18 @@ static int macb_mii_init(struct macb *bp) } bp->mii_bus->name = "MACB_mii_bus"; - bp->mii_bus->read = &macb_mdio_read; - bp->mii_bus->write = &macb_mdio_write; + bp->mii_bus->read = &macb_mdio_read_c22; + bp->mii_bus->write = &macb_mdio_write_c22; + bp->mii_bus->read_c45 = &macb_mdio_read_c45; + bp->mii_bus->write_c45 = &macb_mdio_write_c45; snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", bp->pdev->name, bp->pdev->id); bp->mii_bus->priv = bp; bp->mii_bus->parent = &bp->pdev->dev; - pdata = dev_get_platdata(&bp->pdev->dev); dev_set_drvdata(&bp->dev->dev, bp->mii_bus); - np = bp->pdev->dev.of_node; - if (np) { - if (of_phy_is_fixed_link(np)) { - if (of_phy_register_fixed_link(np) < 0) { - dev_err(&bp->pdev->dev, - "broken fixed-link specification\n"); - goto err_out_unregister_bus; - } - bp->phy_node = of_node_get(np); - - err = mdiobus_register(bp->mii_bus); - } else { - /* try dt phy registration */ - err = of_mdiobus_register(bp->mii_bus, np); - - /* fallback to standard phy registration if no phy were - * found during dt phy registration - */ - if (!err && !phy_find_first(bp->mii_bus)) { - for (i = 0; i < PHY_MAX_ADDR; i++) { - struct phy_device *phydev; - - phydev = mdiobus_scan(bp->mii_bus, i); - if (IS_ERR(phydev) && - PTR_ERR(phydev) != -ENODEV) { - err = PTR_ERR(phydev); - break; - } - } - - if (err) - goto err_out_unregister_bus; - } - } - } else { - for (i = 0; i < PHY_MAX_ADDR; i++) - bp->mii_bus->irq[i] = PHY_POLL; - - if (pdata) - bp->mii_bus->phy_mask = pdata->phy_mask; - - err = mdiobus_register(bp->mii_bus); - } - + err = macb_mdiobus_register(bp, mdio_np); if (err) goto err_out_free_mdiobus; @@ -613,13 +939,15 @@ err_out_unregister_bus: err_out_free_mdiobus: mdiobus_free(bp->mii_bus); err_out: + of_node_put(mdio_np); + return err; } static void macb_update_stats(struct macb *bp) { - u32 *p = &bp->hw_stats.macb.rx_pause_frames; - u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1; + u64 *p = &bp->hw_stats.macb.rx_pause_frames; + u64 *end = &bp->hw_stats.macb.tx_pause_frames + 1; int offset = MACB_PFR; WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4); @@ -630,25 +958,18 @@ static void macb_update_stats(struct macb *bp) static int macb_halt_tx(struct macb *bp) { - unsigned long halt_time, timeout; - u32 status; + u32 status; macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT)); - timeout = jiffies + usecs_to_jiffies(MACB_HALT_TIMEOUT); - do { - halt_time = jiffies; - status = macb_readl(bp, TSR); - if (!(status & MACB_BIT(TGO))) - return 0; - - usleep_range(10, 250); - } while (time_before(halt_time, timeout)); - - return -ETIMEDOUT; + /* Poll TSR until TGO is cleared or timeout. */ + return read_poll_timeout_atomic(macb_readl, status, + !(status & MACB_BIT(TGO)), + 250, MACB_HALT_TIMEOUT, false, + bp, TSR); } -static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb) +static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb, int budget) { if (tx_skb->mapping) { if (tx_skb->mapped_as_page) @@ -661,36 +982,41 @@ static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb) } if (tx_skb->skb) { - dev_kfree_skb_any(tx_skb->skb); + napi_consume_skb(tx_skb->skb, budget); tx_skb->skb = NULL; } } static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr) { -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - struct macb_dma_desc_64 *desc_64; + if (macb_dma64(bp)) { + struct macb_dma_desc_64 *desc_64; - if (bp->hw_dma_cap & HW_DMA_CAP_64B) { desc_64 = macb_64b_desc(bp, desc); desc_64->addrh = upper_32_bits(addr); + /* The low bits of RX address contain the RX_USED bit, clearing + * of which allows packet RX. Make sure the high bits are also + * visible to HW at that point. + */ + dma_wmb(); } -#endif + desc->addr = lower_32_bits(addr); } static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc) { dma_addr_t addr = 0; -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - struct macb_dma_desc_64 *desc_64; - if (bp->hw_dma_cap & HW_DMA_CAP_64B) { + if (macb_dma64(bp)) { + struct macb_dma_desc_64 *desc_64; + desc_64 = macb_64b_desc(bp, desc); addr = ((u64)(desc_64->addrh) << 32); } -#endif addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr)); + if (macb_dma_ptp(bp)) + addr &= ~GEM_BIT(DMA_RXVALID); return addr; } @@ -698,23 +1024,28 @@ static void macb_tx_error_task(struct work_struct *work) { struct macb_queue *queue = container_of(work, struct macb_queue, tx_error_task); + bool halt_timeout = false; struct macb *bp = queue->bp; + u32 queue_index; + u32 packets = 0; + u32 bytes = 0; struct macb_tx_skb *tx_skb; struct macb_dma_desc *desc; struct sk_buff *skb; unsigned int tail; unsigned long flags; + queue_index = queue - bp->queues; netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n", - (unsigned int)(queue - bp->queues), - queue->tx_tail, queue->tx_head); + queue_index, queue->tx_tail, queue->tx_head); - /* Prevent the queue IRQ handlers from running: each of them may call - * macb_tx_interrupt(), which in turn may call netif_wake_subqueue(). + /* Prevent the queue NAPI TX poll from running, as it calls + * macb_tx_complete(), which in turn may call netif_wake_subqueue(). * As explained below, we have to halt the transmission before updating * TBQP registers so we call netif_tx_stop_all_queues() to notify the * network engine about the macb/gem being halted. */ + napi_disable(&queue->napi_tx); spin_lock_irqsave(&bp->lock, flags); /* Make sure nobody is trying to queue up new packets */ @@ -724,9 +1055,11 @@ static void macb_tx_error_task(struct work_struct *work) * (in case we have just queued new packets) * macb/gem must be halted to write TBQP register */ - if (macb_halt_tx(bp)) - /* Just complain for now, reinitializing TX path can be good */ + if (macb_halt_tx(bp)) { netdev_err(bp->dev, "BUG: halt tx timed out\n"); + macb_writel(bp, NCR, macb_readl(bp, NCR) & (~MACB_BIT(TE))); + halt_timeout = true; + } /* Treat frames in TX queue including the ones that caused the error. * Free transmit buffers in upper layer. @@ -742,7 +1075,7 @@ static void macb_tx_error_task(struct work_struct *work) if (ctrl & MACB_BIT(TX_USED)) { /* skb is set for the last buffer of the frame */ while (!skb) { - macb_tx_unmap(bp, tx_skb); + macb_tx_unmap(bp, tx_skb, 0); tail++; tx_skb = macb_tx_skb(queue, tail); skb = tx_skb->skb; @@ -756,7 +1089,11 @@ static void macb_tx_error_task(struct work_struct *work) macb_tx_ring_wrap(bp, tail), skb->data); bp->dev->stats.tx_packets++; + queue->stats.tx_packets++; + packets++; bp->dev->stats.tx_bytes += skb->len; + queue->stats.tx_bytes += skb->len; + bytes += skb->len; } } else { /* "Buffers exhausted mid-frame" errors may only happen @@ -770,9 +1107,12 @@ static void macb_tx_error_task(struct work_struct *work) desc->ctrl = ctrl | MACB_BIT(TX_USED); } - macb_tx_unmap(bp, tx_skb); + macb_tx_unmap(bp, tx_skb, 0); } + netdev_tx_completed_queue(netdev_get_tx_queue(bp->dev, queue_index), + packets, bytes); + /* Set end of TX queue */ desc = macb_tx_desc(queue, 0); macb_set_addr(bp, desc, 0); @@ -783,10 +1123,6 @@ static void macb_tx_error_task(struct work_struct *work) /* Reinitialize the TX desc queue */ queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - if (bp->hw_dma_cap & HW_DMA_CAP_64B) - queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); -#endif /* Make TX ring reflect state of hardware */ queue->tx_head = 0; queue->tx_tail = 0; @@ -795,32 +1131,60 @@ static void macb_tx_error_task(struct work_struct *work) macb_writel(bp, TSR, macb_readl(bp, TSR)); queue_writel(queue, IER, MACB_TX_INT_FLAGS); + if (halt_timeout) + macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TE)); + /* Now we are ready to start transmission again */ netif_tx_start_all_queues(bp->dev); macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); spin_unlock_irqrestore(&bp->lock, flags); + napi_enable(&queue->napi_tx); } -static void macb_tx_interrupt(struct macb_queue *queue) +static bool ptp_one_step_sync(struct sk_buff *skb) { - unsigned int tail; - unsigned int head; - u32 status; - struct macb *bp = queue->bp; - u16 queue_index = queue - bp->queues; + struct ptp_header *hdr; + unsigned int ptp_class; + u8 msgtype; - status = macb_readl(bp, TSR); - macb_writel(bp, TSR, status); + /* No need to parse packet if PTP TS is not involved */ + if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) + goto not_oss; - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, MACB_BIT(TCOMP)); + /* Identify and return whether PTP one step sync is being processed */ + ptp_class = ptp_classify_raw(skb); + if (ptp_class == PTP_CLASS_NONE) + goto not_oss; - netdev_vdbg(bp->dev, "macb_tx_interrupt status = 0x%03lx\n", - (unsigned long)status); + hdr = ptp_parse_header(skb, ptp_class); + if (!hdr) + goto not_oss; + if (hdr->flag_field[0] & PTP_FLAG_TWOSTEP) + goto not_oss; + + msgtype = ptp_get_msgtype(hdr, ptp_class); + if (msgtype == PTP_MSGTYPE_SYNC) + return true; + +not_oss: + return false; +} + +static int macb_tx_complete(struct macb_queue *queue, int budget) +{ + struct macb *bp = queue->bp; + u16 queue_index = queue - bp->queues; + unsigned long flags; + unsigned int tail; + unsigned int head; + int packets = 0; + u32 bytes = 0; + + spin_lock_irqsave(&queue->tx_ptr_lock, flags); head = queue->tx_head; - for (tail = queue->tx_tail; tail != head; tail++) { + for (tail = queue->tx_tail; tail != head && packets < budget; tail++) { struct macb_tx_skb *tx_skb; struct sk_buff *skb; struct macb_dma_desc *desc; @@ -846,21 +1210,23 @@ static void macb_tx_interrupt(struct macb_queue *queue) /* First, update TX stats if needed */ if (skb) { - if (gem_ptp_do_txstamp(queue, skb, desc) == 0) { - /* skb now belongs to timestamp buffer - * and will be removed later - */ - tx_skb->skb = NULL; - } + if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && + !ptp_one_step_sync(skb)) + gem_ptp_do_txstamp(bp, skb, desc); + netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n", macb_tx_ring_wrap(bp, tail), skb->data); bp->dev->stats.tx_packets++; + queue->stats.tx_packets++; bp->dev->stats.tx_bytes += skb->len; + queue->stats.tx_bytes += skb->len; + packets++; + bytes += skb->len; } /* Now we can safely release resources */ - macb_tx_unmap(bp, tx_skb); + macb_tx_unmap(bp, tx_skb, budget); /* skb is set only for the last buffer of the frame. * WARNING: at this point skb has been freed by @@ -871,31 +1237,37 @@ static void macb_tx_interrupt(struct macb_queue *queue) } } + netdev_tx_completed_queue(netdev_get_tx_queue(bp->dev, queue_index), + packets, bytes); + queue->tx_tail = tail; if (__netif_subqueue_stopped(bp->dev, queue_index) && CIRC_CNT(queue->tx_head, queue->tx_tail, bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) netif_wake_subqueue(bp->dev, queue_index); + spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); + + return packets; } -static void gem_rx_refill(struct macb *bp) +static void gem_rx_refill(struct macb_queue *queue) { unsigned int entry; struct sk_buff *skb; dma_addr_t paddr; + struct macb *bp = queue->bp; struct macb_dma_desc *desc; - while (CIRC_SPACE(bp->rx_prepared_head, bp->rx_tail, - bp->rx_ring_size) > 0) { - entry = macb_rx_ring_wrap(bp, bp->rx_prepared_head); + while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail, + bp->rx_ring_size) > 0) { + entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head); /* Make hw descriptor updates visible to CPU */ rmb(); - bp->rx_prepared_head++; - desc = macb_rx_desc(bp, entry); + desc = macb_rx_desc(queue, entry); - if (!bp->rx_skbuff[entry]) { + if (!queue->rx_skbuff[entry]) { /* allocate sk_buff for this free entry in ring */ skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size); if (unlikely(!skb)) { @@ -913,36 +1285,53 @@ static void gem_rx_refill(struct macb *bp) break; } - bp->rx_skbuff[entry] = skb; + queue->rx_skbuff[entry] = skb; if (entry == bp->rx_ring_size - 1) paddr |= MACB_BIT(RX_WRAP); - macb_set_addr(bp, desc, paddr); desc->ctrl = 0; + /* Setting addr clears RX_USED and allows reception, + * make sure ctrl is cleared first to avoid a race. + */ + dma_wmb(); + macb_set_addr(bp, desc, paddr); - /* properly align Ethernet header */ - skb_reserve(skb, NET_IP_ALIGN); + /* Properly align Ethernet header. + * + * Hardware can add dummy bytes if asked using the RBOF + * field inside the NCFGR register. That feature isn't + * available if hardware is RSC capable. + * + * We cannot fallback to doing the 2-byte shift before + * DMA mapping because the address field does not allow + * setting the low 2/3 bits. + * It is 3 bits if HW_DMA_CAP_PTP, else 2 bits. + */ + if (!(bp->caps & MACB_CAPS_RSC)) + skb_reserve(skb, NET_IP_ALIGN); } else { - desc->addr &= ~MACB_BIT(RX_USED); desc->ctrl = 0; + dma_wmb(); + desc->addr &= ~MACB_BIT(RX_USED); } + queue->rx_prepared_head++; } /* Make descriptor updates visible to hardware */ wmb(); - netdev_vdbg(bp->dev, "rx ring: prepared head %d, tail %d\n", - bp->rx_prepared_head, bp->rx_tail); + netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n", + queue, queue->rx_prepared_head, queue->rx_tail); } /* Mark DMA descriptors from begin up to and not including end as unused */ -static void discard_partial_frame(struct macb *bp, unsigned int begin, +static void discard_partial_frame(struct macb_queue *queue, unsigned int begin, unsigned int end) { unsigned int frag; for (frag = begin; frag != end; frag++) { - struct macb_dma_desc *desc = macb_rx_desc(bp, frag); + struct macb_dma_desc *desc = macb_rx_desc(queue, frag); desc->addr &= ~MACB_BIT(RX_USED); } @@ -956,8 +1345,10 @@ static void discard_partial_frame(struct macb *bp, unsigned int begin, */ } -static int gem_rx(struct macb *bp, int budget) +static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, + int budget) { + struct macb *bp = queue->bp; unsigned int len; unsigned int entry; struct sk_buff *skb; @@ -969,37 +1360,43 @@ static int gem_rx(struct macb *bp, int budget) dma_addr_t addr; bool rxused; - entry = macb_rx_ring_wrap(bp, bp->rx_tail); - desc = macb_rx_desc(bp, entry); + entry = macb_rx_ring_wrap(bp, queue->rx_tail); + desc = macb_rx_desc(queue, entry); /* Make hw descriptor updates visible to CPU */ rmb(); rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false; addr = macb_get_addr(bp, desc); - ctrl = desc->ctrl; if (!rxused) break; - bp->rx_tail++; + /* Ensure ctrl is at least as up-to-date as rxused */ + dma_rmb(); + + ctrl = desc->ctrl; + + queue->rx_tail++; count++; if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) { netdev_err(bp->dev, "not whole frame pointed by descriptor\n"); bp->dev->stats.rx_dropped++; + queue->stats.rx_dropped++; break; } - skb = bp->rx_skbuff[entry]; + skb = queue->rx_skbuff[entry]; if (unlikely(!skb)) { netdev_err(bp->dev, "inconsistent Rx descriptor chain\n"); bp->dev->stats.rx_dropped++; + queue->stats.rx_dropped++; break; } /* now everything is ready for receiving packet */ - bp->rx_skbuff[entry] = NULL; + queue->rx_skbuff[entry] = NULL; len = ctrl & bp->rx_frm_len_mask; netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len); @@ -1016,7 +1413,9 @@ static int gem_rx(struct macb *bp, int budget) skb->ip_summed = CHECKSUM_UNNECESSARY; bp->dev->stats.rx_packets++; + queue->stats.rx_packets++; bp->dev->stats.rx_bytes += skb->len; + queue->stats.rx_bytes += skb->len; gem_ptp_do_rxstamp(bp, skb, desc); @@ -1029,24 +1428,25 @@ static int gem_rx(struct macb *bp, int budget) skb->data, 32, true); #endif - netif_receive_skb(skb); + napi_gro_receive(napi, skb); } - gem_rx_refill(bp); + gem_rx_refill(queue); return count; } -static int macb_rx_frame(struct macb *bp, unsigned int first_frag, - unsigned int last_frag) +static int macb_rx_frame(struct macb_queue *queue, struct napi_struct *napi, + unsigned int first_frag, unsigned int last_frag) { unsigned int len; unsigned int frag; unsigned int offset; struct sk_buff *skb; struct macb_dma_desc *desc; + struct macb *bp = queue->bp; - desc = macb_rx_desc(bp, last_frag); + desc = macb_rx_desc(queue, last_frag); len = desc->ctrl & bp->rx_frm_len_mask; netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n", @@ -1065,7 +1465,7 @@ static int macb_rx_frame(struct macb *bp, unsigned int first_frag, if (!skb) { bp->dev->stats.rx_dropped++; for (frag = first_frag; ; frag++) { - desc = macb_rx_desc(bp, frag); + desc = macb_rx_desc(queue, frag); desc->addr &= ~MACB_BIT(RX_USED); if (frag == last_frag) break; @@ -1093,10 +1493,10 @@ static int macb_rx_frame(struct macb *bp, unsigned int first_frag, frag_len = len - offset; } skb_copy_to_linear_data_offset(skb, offset, - macb_rx_buffer(bp, frag), + macb_rx_buffer(queue, frag), frag_len); offset += bp->rx_buffer_size; - desc = macb_rx_desc(bp, frag); + desc = macb_rx_desc(queue, frag); desc->addr &= ~MACB_BIT(RX_USED); if (frag == last_frag) @@ -1113,50 +1513,56 @@ static int macb_rx_frame(struct macb *bp, unsigned int first_frag, bp->dev->stats.rx_bytes += skb->len; netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", skb->len, skb->csum); - netif_receive_skb(skb); + napi_gro_receive(napi, skb); return 0; } -static inline void macb_init_rx_ring(struct macb *bp) +static inline void macb_init_rx_ring(struct macb_queue *queue) { + struct macb *bp = queue->bp; dma_addr_t addr; struct macb_dma_desc *desc = NULL; int i; - addr = bp->rx_buffers_dma; + addr = queue->rx_buffers_dma; for (i = 0; i < bp->rx_ring_size; i++) { - desc = macb_rx_desc(bp, i); + desc = macb_rx_desc(queue, i); macb_set_addr(bp, desc, addr); desc->ctrl = 0; addr += bp->rx_buffer_size; } desc->addr |= MACB_BIT(RX_WRAP); - bp->rx_tail = 0; + queue->rx_tail = 0; } -static int macb_rx(struct macb *bp, int budget) +static int macb_rx(struct macb_queue *queue, struct napi_struct *napi, + int budget) { + struct macb *bp = queue->bp; bool reset_rx_queue = false; int received = 0; unsigned int tail; int first_frag = -1; - for (tail = bp->rx_tail; budget > 0; tail++) { - struct macb_dma_desc *desc = macb_rx_desc(bp, tail); + for (tail = queue->rx_tail; budget > 0; tail++) { + struct macb_dma_desc *desc = macb_rx_desc(queue, tail); u32 ctrl; /* Make hw descriptor updates visible to CPU */ rmb(); - ctrl = desc->ctrl; - if (!(desc->addr & MACB_BIT(RX_USED))) break; + /* Ensure ctrl is at least as up-to-date as addr */ + dma_rmb(); + + ctrl = desc->ctrl; + if (ctrl & MACB_BIT(RX_SOF)) { if (first_frag != -1) - discard_partial_frame(bp, first_frag, tail); + discard_partial_frame(queue, first_frag, tail); first_frag = tail; } @@ -1168,7 +1574,7 @@ static int macb_rx(struct macb *bp, int budget) continue; } - dropped = macb_rx_frame(bp, first_frag, tail); + dropped = macb_rx_frame(queue, napi, first_frag, tail); first_frag = -1; if (unlikely(dropped < 0)) { reset_rx_queue = true; @@ -1192,8 +1598,8 @@ static int macb_rx(struct macb *bp, int budget) ctrl = macb_readl(bp, NCR); macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); - macb_init_rx_ring(bp); - macb_writel(bp, RBQP, bp->rx_ring_dma); + macb_init_rx_ring(queue); + queue_writel(queue, RBQP, queue->rx_ring_dma); macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); @@ -1202,39 +1608,58 @@ static int macb_rx(struct macb *bp, int budget) } if (first_frag != -1) - bp->rx_tail = first_frag; + queue->rx_tail = first_frag; else - bp->rx_tail = tail; + queue->rx_tail = tail; return received; } -static int macb_poll(struct napi_struct *napi, int budget) +static bool macb_rx_pending(struct macb_queue *queue) { - struct macb *bp = container_of(napi, struct macb, napi); - int work_done; - u32 status; + struct macb *bp = queue->bp; + unsigned int entry; + struct macb_dma_desc *desc; - status = macb_readl(bp, RSR); - macb_writel(bp, RSR, status); + entry = macb_rx_ring_wrap(bp, queue->rx_tail); + desc = macb_rx_desc(queue, entry); - work_done = 0; + /* Make hw descriptor updates visible to CPU */ + rmb(); - netdev_vdbg(bp->dev, "poll: status = %08lx, budget = %d\n", - (unsigned long)status, budget); + return (desc->addr & MACB_BIT(RX_USED)) != 0; +} - work_done = bp->macbgem_ops.mog_rx(bp, budget); - if (work_done < budget) { - napi_complete_done(napi, work_done); +static int macb_rx_poll(struct napi_struct *napi, int budget) +{ + struct macb_queue *queue = container_of(napi, struct macb_queue, napi_rx); + struct macb *bp = queue->bp; + int work_done; - /* Packets received while interrupts were disabled */ - status = macb_readl(bp, RSR); - if (status) { + work_done = bp->macbgem_ops.mog_rx(queue, napi, budget); + + netdev_vdbg(bp->dev, "RX poll: queue = %u, work_done = %d, budget = %d\n", + (unsigned int)(queue - bp->queues), work_done, budget); + + if (work_done < budget && napi_complete_done(napi, work_done)) { + queue_writel(queue, IER, bp->rx_intr_mask); + + /* Packet completions only seem to propagate to raise + * interrupts when interrupts are enabled at the time, so if + * packets were received while interrupts were disabled, + * they will not cause another interrupt to be generated when + * interrupts are re-enabled. + * Check for this case here to avoid losing a wakeup. This can + * potentially race with the interrupt handler doing the same + * actions if an interrupt is raised just after enabling them, + * but this should be harmless. + */ + if (macb_rx_pending(queue)) { + queue_writel(queue, IDR, bp->rx_intr_mask); if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - macb_writel(bp, ISR, MACB_BIT(RCOMP)); - napi_reschedule(napi); - } else { - macb_writel(bp, IER, MACB_RX_INT_FLAGS); + queue_writel(queue, ISR, MACB_BIT(RCOMP)); + netdev_vdbg(bp->dev, "poll: packets pending, reschedule\n"); + napi_schedule(napi); } } @@ -1243,6 +1668,189 @@ static int macb_poll(struct napi_struct *napi, int budget) return work_done; } +static void macb_tx_restart(struct macb_queue *queue) +{ + struct macb *bp = queue->bp; + unsigned int head_idx, tbqp; + unsigned long flags; + + spin_lock_irqsave(&queue->tx_ptr_lock, flags); + + if (queue->tx_head == queue->tx_tail) + goto out_tx_ptr_unlock; + + tbqp = queue_readl(queue, TBQP) / macb_dma_desc_get_size(bp); + tbqp = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, tbqp)); + head_idx = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, queue->tx_head)); + + if (tbqp == head_idx) + goto out_tx_ptr_unlock; + + spin_lock(&bp->lock); + macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); + spin_unlock(&bp->lock); + +out_tx_ptr_unlock: + spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); +} + +static bool macb_tx_complete_pending(struct macb_queue *queue) +{ + bool retval = false; + unsigned long flags; + + spin_lock_irqsave(&queue->tx_ptr_lock, flags); + if (queue->tx_head != queue->tx_tail) { + /* Make hw descriptor updates visible to CPU */ + rmb(); + + if (macb_tx_desc(queue, queue->tx_tail)->ctrl & MACB_BIT(TX_USED)) + retval = true; + } + spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); + return retval; +} + +static int macb_tx_poll(struct napi_struct *napi, int budget) +{ + struct macb_queue *queue = container_of(napi, struct macb_queue, napi_tx); + struct macb *bp = queue->bp; + int work_done; + + work_done = macb_tx_complete(queue, budget); + + rmb(); // ensure txubr_pending is up to date + if (queue->txubr_pending) { + queue->txubr_pending = false; + netdev_vdbg(bp->dev, "poll: tx restart\n"); + macb_tx_restart(queue); + } + + netdev_vdbg(bp->dev, "TX poll: queue = %u, work_done = %d, budget = %d\n", + (unsigned int)(queue - bp->queues), work_done, budget); + + if (work_done < budget && napi_complete_done(napi, work_done)) { + queue_writel(queue, IER, MACB_BIT(TCOMP)); + + /* Packet completions only seem to propagate to raise + * interrupts when interrupts are enabled at the time, so if + * packets were sent while interrupts were disabled, + * they will not cause another interrupt to be generated when + * interrupts are re-enabled. + * Check for this case here to avoid losing a wakeup. This can + * potentially race with the interrupt handler doing the same + * actions if an interrupt is raised just after enabling them, + * but this should be harmless. + */ + if (macb_tx_complete_pending(queue)) { + queue_writel(queue, IDR, MACB_BIT(TCOMP)); + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, MACB_BIT(TCOMP)); + netdev_vdbg(bp->dev, "TX poll: packets pending, reschedule\n"); + napi_schedule(napi); + } + } + + return work_done; +} + +static void macb_hresp_error_task(struct work_struct *work) +{ + struct macb *bp = from_work(bp, work, hresp_err_bh_work); + struct net_device *dev = bp->dev; + struct macb_queue *queue; + unsigned int q; + u32 ctrl; + + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + queue_writel(queue, IDR, bp->rx_intr_mask | + MACB_TX_INT_FLAGS | + MACB_BIT(HRESP)); + } + ctrl = macb_readl(bp, NCR); + ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); + macb_writel(bp, NCR, ctrl); + + netif_tx_stop_all_queues(dev); + netif_carrier_off(dev); + + bp->macbgem_ops.mog_init_rings(bp); + + /* Initialize TX and RX buffers */ + macb_init_buffers(bp); + + /* Enable interrupts */ + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) + queue_writel(queue, IER, + bp->rx_intr_mask | + MACB_TX_INT_FLAGS | + MACB_BIT(HRESP)); + + ctrl |= MACB_BIT(RE) | MACB_BIT(TE); + macb_writel(bp, NCR, ctrl); + + netif_carrier_on(dev); + netif_tx_start_all_queues(dev); +} + +static irqreturn_t macb_wol_interrupt(int irq, void *dev_id) +{ + struct macb_queue *queue = dev_id; + struct macb *bp = queue->bp; + u32 status; + + status = queue_readl(queue, ISR); + + if (unlikely(!status)) + return IRQ_NONE; + + spin_lock(&bp->lock); + + if (status & MACB_BIT(WOL)) { + queue_writel(queue, IDR, MACB_BIT(WOL)); + macb_writel(bp, WOL, 0); + netdev_vdbg(bp->dev, "MACB WoL: queue = %u, isr = 0x%08lx\n", + (unsigned int)(queue - bp->queues), + (unsigned long)status); + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, MACB_BIT(WOL)); + pm_wakeup_event(&bp->pdev->dev, 0); + } + + spin_unlock(&bp->lock); + + return IRQ_HANDLED; +} + +static irqreturn_t gem_wol_interrupt(int irq, void *dev_id) +{ + struct macb_queue *queue = dev_id; + struct macb *bp = queue->bp; + u32 status; + + status = queue_readl(queue, ISR); + + if (unlikely(!status)) + return IRQ_NONE; + + spin_lock(&bp->lock); + + if (status & GEM_BIT(WOL)) { + queue_writel(queue, IDR, GEM_BIT(WOL)); + gem_writel(bp, WOL, 0); + netdev_vdbg(bp->dev, "GEM WoL: queue = %u, isr = 0x%08lx\n", + (unsigned int)(queue - bp->queues), + (unsigned long)status); + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, GEM_BIT(WOL)); + pm_wakeup_event(&bp->pdev->dev, 0); + } + + spin_unlock(&bp->lock); + + return IRQ_HANDLED; +} + static irqreturn_t macb_interrupt(int irq, void *dev_id) { struct macb_queue *queue = dev_id; @@ -1270,20 +1878,38 @@ static irqreturn_t macb_interrupt(int irq, void *dev_id) (unsigned int)(queue - bp->queues), (unsigned long)status); - if (status & MACB_RX_INT_FLAGS) { + if (status & bp->rx_intr_mask) { /* There's no point taking any more interrupts * until we have processed the buffers. The * scheduling call may fail if the poll routine * is already scheduled, so disable interrupts * now. */ - queue_writel(queue, IDR, MACB_RX_INT_FLAGS); + queue_writel(queue, IDR, bp->rx_intr_mask); if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) queue_writel(queue, ISR, MACB_BIT(RCOMP)); - if (napi_schedule_prep(&bp->napi)) { + if (napi_schedule_prep(&queue->napi_rx)) { netdev_vdbg(bp->dev, "scheduling RX softirq\n"); - __napi_schedule(&bp->napi); + __napi_schedule(&queue->napi_rx); + } + } + + if (status & (MACB_BIT(TCOMP) | + MACB_BIT(TXUBR))) { + queue_writel(queue, IDR, MACB_BIT(TCOMP)); + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, MACB_BIT(TCOMP) | + MACB_BIT(TXUBR)); + + if (status & MACB_BIT(TXUBR)) { + queue->txubr_pending = true; + wmb(); // ensure softirq can see update + } + + if (napi_schedule_prep(&queue->napi_tx)) { + netdev_vdbg(bp->dev, "scheduling TX softirq\n"); + __napi_schedule(&queue->napi_tx); } } @@ -1297,9 +1923,6 @@ static irqreturn_t macb_interrupt(int irq, void *dev_id) break; } - if (status & MACB_BIT(TCOMP)) - macb_tx_interrupt(queue); - /* Link change detection isn't possible with RMII, so we'll * add that if/when we get our hands on a full-blown MII PHY. */ @@ -1307,8 +1930,9 @@ static irqreturn_t macb_interrupt(int irq, void *dev_id) /* There is a hardware issue under heavy load where DMA can * stop, this causes endless "used buffer descriptor read" * interrupts but it can be cleared by re-enabling RX. See - * the at91 manual, section 41.3.1 or the Zynq manual - * section 16.7.4 for details. + * the at91rm9200 manual, section 41.3.1 or the Zynq manual + * section 16.7.4 for details. RXUBR is only enabled for + * these two versions. */ if (status & MACB_BIT(RXUBR)) { ctrl = macb_readl(bp, NCR); @@ -1322,20 +1946,19 @@ static irqreturn_t macb_interrupt(int irq, void *dev_id) if (status & MACB_BIT(ISR_ROVR)) { /* We missed at least one packet */ + spin_lock(&bp->stats_lock); if (macb_is_gem(bp)) bp->hw_stats.gem.rx_overruns++; else bp->hw_stats.macb.rx_overruns++; + spin_unlock(&bp->stats_lock); if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) queue_writel(queue, ISR, MACB_BIT(ISR_ROVR)); } if (status & MACB_BIT(HRESP)) { - /* TODO: Reset the hardware, and maybe move the - * netdev_err to a lower-priority context as well - * (work queue?) - */ + queue_work(system_bh_wq, &bp->hresp_err_bh_work); netdev_err(dev, "DMA bus error: HRESP not OK\n"); if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) @@ -1372,14 +1995,14 @@ static unsigned int macb_tx_map(struct macb *bp, struct sk_buff *skb, unsigned int hdrlen) { - dma_addr_t mapping; - unsigned int len, entry, i, tx_head = queue->tx_head; - struct macb_tx_skb *tx_skb = NULL; - struct macb_dma_desc *desc; - unsigned int offset, size, count = 0; unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; - unsigned int eof = 1, mss_mfs = 0; + unsigned int len, i, tx_head = queue->tx_head; u32 ctrl, lso_ctrl = 0, seq_ctrl = 0; + unsigned int eof = 1, mss_mfs = 0; + struct macb_tx_skb *tx_skb = NULL; + struct macb_dma_desc *desc; + unsigned int offset, size; + dma_addr_t mapping; /* LSO */ if (skb_shinfo(skb)->gso_size != 0) { @@ -1399,8 +2022,7 @@ static unsigned int macb_tx_map(struct macb *bp, offset = 0; while (len) { - entry = macb_tx_ring_wrap(bp, tx_head); - tx_skb = &queue->tx_skb[entry]; + tx_skb = macb_tx_skb(queue, tx_head); mapping = dma_map_single(&bp->pdev->dev, skb->data + offset, @@ -1416,10 +2038,9 @@ static unsigned int macb_tx_map(struct macb *bp, len -= size; offset += size; - count++; tx_head++; - size = min(len, bp->max_tx_length); + size = umin(len, bp->max_tx_length); } /* Then, map paged data from fragments */ @@ -1429,9 +2050,8 @@ static unsigned int macb_tx_map(struct macb *bp, len = skb_frag_size(frag); offset = 0; while (len) { - size = min(len, bp->max_tx_length); - entry = macb_tx_ring_wrap(bp, tx_head); - tx_skb = &queue->tx_skb[entry]; + size = umin(len, bp->max_tx_length); + tx_skb = macb_tx_skb(queue, tx_head); mapping = skb_frag_dma_map(&bp->pdev->dev, frag, offset, size, DMA_TO_DEVICE); @@ -1446,7 +2066,6 @@ static unsigned int macb_tx_map(struct macb *bp, len -= size; offset += size; - count++; tx_head++; } } @@ -1468,9 +2087,8 @@ static unsigned int macb_tx_map(struct macb *bp, * to set the end of TX queue */ i = tx_head; - entry = macb_tx_ring_wrap(bp, i); ctrl = MACB_BIT(TX_USED); - desc = macb_tx_desc(queue, entry); + desc = macb_tx_desc(queue, i); desc->ctrl = ctrl; if (lso_ctrl) { @@ -1490,22 +2108,25 @@ static unsigned int macb_tx_map(struct macb *bp, do { i--; - entry = macb_tx_ring_wrap(bp, i); - tx_skb = &queue->tx_skb[entry]; - desc = macb_tx_desc(queue, entry); + tx_skb = macb_tx_skb(queue, i); + desc = macb_tx_desc(queue, i); ctrl = (u32)tx_skb->size; if (eof) { ctrl |= MACB_BIT(TX_LAST); eof = 0; } - if (unlikely(entry == (bp->tx_ring_size - 1))) + if (unlikely(macb_tx_ring_wrap(bp, i) == bp->tx_ring_size - 1)) ctrl |= MACB_BIT(TX_WRAP); /* First descriptor is header descriptor */ if (i == queue->tx_head) { ctrl |= MACB_BF(TX_LSO, lso_ctrl); ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl); + if ((bp->dev->features & NETIF_F_HW_CSUM) && + skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl && + !ptp_one_step_sync(skb)) + ctrl |= MACB_BIT(TX_NOCRC); } else /* Only set MSS/MFS on payload descriptors * (second or later descriptor) @@ -1523,7 +2144,7 @@ static unsigned int macb_tx_map(struct macb *bp, queue->tx_head = tx_head; - return count; + return 0; dma_error: netdev_err(bp->dev, "TX DMA map failed\n"); @@ -1531,10 +2152,10 @@ dma_error: for (i = queue->tx_head; i != tx_head; i++) { tx_skb = macb_tx_skb(queue, i); - macb_tx_unmap(bp, tx_skb); + macb_tx_unmap(bp, tx_skb, 0); } - return 0; + return -ENOMEM; } static netdev_features_t macb_features_check(struct sk_buff *skb, @@ -1546,16 +2167,14 @@ static netdev_features_t macb_features_check(struct sk_buff *skb, /* Validate LSO compatibility */ - /* there is only one buffer */ - if (!skb_is_nonlinear(skb)) + /* there is only one buffer or protocol is not UDP */ + if (!skb_is_nonlinear(skb) || (ip_hdr(skb)->protocol != IPPROTO_UDP)) return features; /* length of header */ hdrlen = skb_transport_offset(skb); - if (ip_hdr(skb)->protocol == IPPROTO_TCP) - hdrlen += tcp_hdrlen(skb); - /* For LSO: + /* For UFO only: * When software supplies two or more payload buffers all payload buffers * apart from the last must be a multiple of 8 bytes in size. */ @@ -1592,34 +2211,98 @@ static inline int macb_clear_csum(struct sk_buff *skb) return 0; } -static int macb_start_xmit(struct sk_buff *skb, struct net_device *dev) +static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev) +{ + bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb) || + skb_is_nonlinear(*skb); + int padlen = ETH_ZLEN - (*skb)->len; + int tailroom = skb_tailroom(*skb); + struct sk_buff *nskb; + u32 fcs; + + if (!(ndev->features & NETIF_F_HW_CSUM) || + !((*skb)->ip_summed != CHECKSUM_PARTIAL) || + skb_shinfo(*skb)->gso_size || ptp_one_step_sync(*skb)) + return 0; + + if (padlen <= 0) { + /* FCS could be appeded to tailroom. */ + if (tailroom >= ETH_FCS_LEN) + goto add_fcs; + /* No room for FCS, need to reallocate skb. */ + else + padlen = ETH_FCS_LEN; + } else { + /* Add room for FCS. */ + padlen += ETH_FCS_LEN; + } + + if (cloned || tailroom < padlen) { + nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC); + if (!nskb) + return -ENOMEM; + + dev_consume_skb_any(*skb); + *skb = nskb; + } + + if (padlen > ETH_FCS_LEN) + skb_put_zero(*skb, padlen - ETH_FCS_LEN); + +add_fcs: + /* set FCS to packet */ + fcs = crc32_le(~0, (*skb)->data, (*skb)->len); + fcs = ~fcs; + + skb_put_u8(*skb, fcs & 0xff); + skb_put_u8(*skb, (fcs >> 8) & 0xff); + skb_put_u8(*skb, (fcs >> 16) & 0xff); + skb_put_u8(*skb, (fcs >> 24) & 0xff); + + return 0; +} + +static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) { u16 queue_index = skb_get_queue_mapping(skb); struct macb *bp = netdev_priv(dev); struct macb_queue *queue = &bp->queues[queue_index]; - unsigned long flags; unsigned int desc_cnt, nr_frags, frag_size, f; unsigned int hdrlen; - bool is_lso, is_udp = 0; + unsigned long flags; + bool is_lso; + netdev_tx_t ret = NETDEV_TX_OK; + + if (macb_clear_csum(skb)) { + dev_kfree_skb_any(skb); + return ret; + } + + if (macb_pad_and_fcs(&skb, dev)) { + dev_kfree_skb_any(skb); + return ret; + } + + if (macb_dma_ptp(bp) && + (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; is_lso = (skb_shinfo(skb)->gso_size != 0); if (is_lso) { - is_udp = !!(ip_hdr(skb)->protocol == IPPROTO_UDP); - /* length of headers */ - if (is_udp) + if (ip_hdr(skb)->protocol == IPPROTO_UDP) /* only queue eth + ip headers separately for UDP */ hdrlen = skb_transport_offset(skb); else - hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); + hdrlen = skb_tcp_all_headers(skb); if (skb_headlen(skb) < hdrlen) { netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n"); /* if this is required, would need to copy to single buffer */ return NETDEV_TX_BUSY; } } else - hdrlen = min(skb_headlen(skb), bp->max_tx_length); + hdrlen = umin(skb_headlen(skb), bp->max_tx_length); #if defined(DEBUG) && defined(VERBOSE_DEBUG) netdev_vdbg(bp->dev, @@ -1645,25 +2328,20 @@ static int macb_start_xmit(struct sk_buff *skb, struct net_device *dev) desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); } - spin_lock_irqsave(&bp->lock, flags); + spin_lock_irqsave(&queue->tx_ptr_lock, flags); /* This is a hard error, log it. */ if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < desc_cnt) { netif_stop_subqueue(dev, queue_index); - spin_unlock_irqrestore(&bp->lock, flags); netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n", queue->tx_head, queue->tx_tail); - return NETDEV_TX_BUSY; - } - - if (macb_clear_csum(skb)) { - dev_kfree_skb_any(skb); + ret = NETDEV_TX_BUSY; goto unlock; } /* Map socket buffer for DMA transfer */ - if (!macb_tx_map(bp, queue, skb, hdrlen)) { + if (macb_tx_map(bp, queue, skb, hdrlen)) { dev_kfree_skb_any(skb); goto unlock; } @@ -1671,16 +2349,20 @@ static int macb_start_xmit(struct sk_buff *skb, struct net_device *dev) /* Make newly initialized descriptor visible to hardware */ wmb(); skb_tx_timestamp(skb); + netdev_tx_sent_queue(netdev_get_tx_queue(bp->dev, queue_index), + skb->len); + spin_lock(&bp->lock); macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); + spin_unlock(&bp->lock); if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) netif_stop_subqueue(dev, queue_index); unlock: - spin_unlock_irqrestore(&bp->lock, flags); + spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); - return NETDEV_TX_OK; + return ret; } static void macb_init_rx_buffer_size(struct macb *bp, size_t size) @@ -1707,131 +2389,177 @@ static void gem_free_rx_buffers(struct macb *bp) { struct sk_buff *skb; struct macb_dma_desc *desc; + struct macb_queue *queue; dma_addr_t addr; + unsigned int q; int i; - if (!bp->rx_skbuff) - return; + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + if (!queue->rx_skbuff) + continue; - for (i = 0; i < bp->rx_ring_size; i++) { - skb = bp->rx_skbuff[i]; + for (i = 0; i < bp->rx_ring_size; i++) { + skb = queue->rx_skbuff[i]; - if (!skb) - continue; + if (!skb) + continue; - desc = macb_rx_desc(bp, i); - addr = macb_get_addr(bp, desc); + desc = macb_rx_desc(queue, i); + addr = macb_get_addr(bp, desc); - dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, - DMA_FROM_DEVICE); - dev_kfree_skb_any(skb); - skb = NULL; - } + dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, + DMA_FROM_DEVICE); + dev_kfree_skb_any(skb); + skb = NULL; + } - kfree(bp->rx_skbuff); - bp->rx_skbuff = NULL; + kfree(queue->rx_skbuff); + queue->rx_skbuff = NULL; + } } static void macb_free_rx_buffers(struct macb *bp) { - if (bp->rx_buffers) { + struct macb_queue *queue = &bp->queues[0]; + + if (queue->rx_buffers) { dma_free_coherent(&bp->pdev->dev, bp->rx_ring_size * bp->rx_buffer_size, - bp->rx_buffers, bp->rx_buffers_dma); - bp->rx_buffers = NULL; + queue->rx_buffers, queue->rx_buffers_dma); + queue->rx_buffers = NULL; } } +static unsigned int macb_tx_ring_size_per_queue(struct macb *bp) +{ + return macb_dma_desc_get_size(bp) * bp->tx_ring_size + bp->tx_bd_rd_prefetch; +} + +static unsigned int macb_rx_ring_size_per_queue(struct macb *bp) +{ + return macb_dma_desc_get_size(bp) * bp->rx_ring_size + bp->rx_bd_rd_prefetch; +} + static void macb_free_consistent(struct macb *bp) { + struct device *dev = &bp->pdev->dev; struct macb_queue *queue; unsigned int q; + size_t size; - bp->macbgem_ops.mog_free_rx_buffers(bp); - if (bp->rx_ring) { - dma_free_coherent(&bp->pdev->dev, RX_RING_BYTES(bp), - bp->rx_ring, bp->rx_ring_dma); - bp->rx_ring = NULL; + if (bp->rx_ring_tieoff) { + dma_free_coherent(dev, macb_dma_desc_get_size(bp), + bp->rx_ring_tieoff, bp->rx_ring_tieoff_dma); + bp->rx_ring_tieoff = NULL; } + bp->macbgem_ops.mog_free_rx_buffers(bp); + + size = bp->num_queues * macb_tx_ring_size_per_queue(bp); + dma_free_coherent(dev, size, bp->queues[0].tx_ring, bp->queues[0].tx_ring_dma); + + size = bp->num_queues * macb_rx_ring_size_per_queue(bp); + dma_free_coherent(dev, size, bp->queues[0].rx_ring, bp->queues[0].rx_ring_dma); + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { kfree(queue->tx_skb); queue->tx_skb = NULL; - if (queue->tx_ring) { - dma_free_coherent(&bp->pdev->dev, TX_RING_BYTES(bp), - queue->tx_ring, queue->tx_ring_dma); - queue->tx_ring = NULL; - } + queue->tx_ring = NULL; + queue->rx_ring = NULL; } } static int gem_alloc_rx_buffers(struct macb *bp) { + struct macb_queue *queue; + unsigned int q; int size; - size = bp->rx_ring_size * sizeof(struct sk_buff *); - bp->rx_skbuff = kzalloc(size, GFP_KERNEL); - if (!bp->rx_skbuff) - return -ENOMEM; - else - netdev_dbg(bp->dev, - "Allocated %d RX struct sk_buff entries at %p\n", - bp->rx_ring_size, bp->rx_skbuff); + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + size = bp->rx_ring_size * sizeof(struct sk_buff *); + queue->rx_skbuff = kzalloc(size, GFP_KERNEL); + if (!queue->rx_skbuff) + return -ENOMEM; + else + netdev_dbg(bp->dev, + "Allocated %d RX struct sk_buff entries at %p\n", + bp->rx_ring_size, queue->rx_skbuff); + } return 0; } static int macb_alloc_rx_buffers(struct macb *bp) { + struct macb_queue *queue = &bp->queues[0]; int size; size = bp->rx_ring_size * bp->rx_buffer_size; - bp->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size, - &bp->rx_buffers_dma, GFP_KERNEL); - if (!bp->rx_buffers) + queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size, + &queue->rx_buffers_dma, GFP_KERNEL); + if (!queue->rx_buffers) return -ENOMEM; netdev_dbg(bp->dev, "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n", - size, (unsigned long)bp->rx_buffers_dma, bp->rx_buffers); + size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers); return 0; } static int macb_alloc_consistent(struct macb *bp) { + struct device *dev = &bp->pdev->dev; + dma_addr_t tx_dma, rx_dma; struct macb_queue *queue; unsigned int q; - int size; + void *tx, *rx; + size_t size; + + /* + * Upper 32-bits of Tx/Rx DMA descriptor for each queues much match! + * We cannot enforce this guarantee, the best we can do is do a single + * allocation and hope it will land into alloc_pages() that guarantees + * natural alignment of physical addresses. + */ + + size = bp->num_queues * macb_tx_ring_size_per_queue(bp); + tx = dma_alloc_coherent(dev, size, &tx_dma, GFP_KERNEL); + if (!tx || upper_32_bits(tx_dma) != upper_32_bits(tx_dma + size - 1)) + goto out_err; + netdev_dbg(bp->dev, "Allocated %zu bytes for %u TX rings at %08lx (mapped %p)\n", + size, bp->num_queues, (unsigned long)tx_dma, tx); + + size = bp->num_queues * macb_rx_ring_size_per_queue(bp); + rx = dma_alloc_coherent(dev, size, &rx_dma, GFP_KERNEL); + if (!rx || upper_32_bits(rx_dma) != upper_32_bits(rx_dma + size - 1)) + goto out_err; + netdev_dbg(bp->dev, "Allocated %zu bytes for %u RX rings at %08lx (mapped %p)\n", + size, bp->num_queues, (unsigned long)rx_dma, rx); for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - size = TX_RING_BYTES(bp); - queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size, - &queue->tx_ring_dma, - GFP_KERNEL); - if (!queue->tx_ring) - goto out_err; - netdev_dbg(bp->dev, - "Allocated TX ring for queue %u of %d bytes at %08lx (mapped %p)\n", - q, size, (unsigned long)queue->tx_ring_dma, - queue->tx_ring); + queue->tx_ring = tx + macb_tx_ring_size_per_queue(bp) * q; + queue->tx_ring_dma = tx_dma + macb_tx_ring_size_per_queue(bp) * q; + + queue->rx_ring = rx + macb_rx_ring_size_per_queue(bp) * q; + queue->rx_ring_dma = rx_dma + macb_rx_ring_size_per_queue(bp) * q; size = bp->tx_ring_size * sizeof(struct macb_tx_skb); queue->tx_skb = kmalloc(size, GFP_KERNEL); if (!queue->tx_skb) goto out_err; } - - size = RX_RING_BYTES(bp); - bp->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size, - &bp->rx_ring_dma, GFP_KERNEL); - if (!bp->rx_ring) - goto out_err; - netdev_dbg(bp->dev, - "Allocated RX ring of %d bytes at %08lx (mapped %p)\n", - size, (unsigned long)bp->rx_ring_dma, bp->rx_ring); - if (bp->macbgem_ops.mog_alloc_rx_buffers(bp)) goto out_err; + /* Required for tie off descriptor for PM cases */ + if (!(bp->caps & MACB_CAPS_QUEUE_DISABLE)) { + bp->rx_ring_tieoff = dma_alloc_coherent(&bp->pdev->dev, + macb_dma_desc_get_size(bp), + &bp->rx_ring_tieoff_dma, + GFP_KERNEL); + if (!bp->rx_ring_tieoff) + goto out_err; + } + return 0; out_err: @@ -1839,6 +2567,19 @@ out_err: return -ENOMEM; } +static void macb_init_tieoff(struct macb *bp) +{ + struct macb_dma_desc *desc = bp->rx_ring_tieoff; + + if (bp->caps & MACB_CAPS_QUEUE_DISABLE) + return; + /* Setup a wrapping descriptor with no free slots + * (WRAP and USED) to tie off/disable unused RX queues. + */ + macb_set_addr(bp, desc, MACB_BIT(RX_WRAP) | MACB_BIT(RX_USED)); + desc->ctrl = 0; +} + static void gem_init_rings(struct macb *bp) { struct macb_queue *queue; @@ -1855,12 +2596,14 @@ static void gem_init_rings(struct macb *bp) desc->ctrl |= MACB_BIT(TX_WRAP); queue->tx_head = 0; queue->tx_tail = 0; - } - bp->rx_tail = 0; - bp->rx_prepared_head = 0; + queue->rx_tail = 0; + queue->rx_prepared_head = 0; - gem_rx_refill(bp); + gem_rx_refill(queue); + } + + macb_init_tieoff(bp); } static void macb_init_rings(struct macb *bp) @@ -1868,7 +2611,7 @@ static void macb_init_rings(struct macb *bp) int i; struct macb_dma_desc *desc = NULL; - macb_init_rx_ring(bp); + macb_init_rx_ring(&bp->queues[0]); for (i = 0; i < bp->tx_ring_size; i++) { desc = macb_tx_desc(&bp->queues[0], i); @@ -1878,25 +2621,33 @@ static void macb_init_rings(struct macb *bp) bp->queues[0].tx_head = 0; bp->queues[0].tx_tail = 0; desc->ctrl |= MACB_BIT(TX_WRAP); + + macb_init_tieoff(bp); } static void macb_reset_hw(struct macb *bp) { struct macb_queue *queue; unsigned int q; + u32 ctrl = macb_readl(bp, NCR); /* Disable RX and TX (XXX: Should we halt the transmission * more gracefully?) */ - macb_writel(bp, NCR, 0); + ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); /* Clear the stats registers (XXX: Update stats first?) */ - macb_writel(bp, NCR, MACB_BIT(CLRSTAT)); + ctrl |= MACB_BIT(CLRSTAT); + + macb_writel(bp, NCR, ctrl); /* Clear all status flags */ macb_writel(bp, TSR, -1); macb_writel(bp, RSR, -1); + /* Disable RX partial store and forward and reset watermark value */ + gem_writel(bp, PBUFRXCUT, 0); + /* Disable all interrupts */ for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { queue_writel(queue, IDR, -1); @@ -1921,8 +2672,12 @@ static u32 gem_mdc_clk_div(struct macb *bp) config = GEM_BF(CLK, GEM_CLK_DIV48); else if (pclk_hz <= 160000000) config = GEM_BF(CLK, GEM_CLK_DIV64); - else + else if (pclk_hz <= 240000000) config = GEM_BF(CLK, GEM_CLK_DIV96); + else if (pclk_hz <= 320000000) + config = GEM_BF(CLK, GEM_CLK_DIV128); + else + config = GEM_BF(CLK, GEM_CLK_DIV224); return config; } @@ -1977,11 +2732,20 @@ static u32 macb_dbw(struct macb *bp) */ static void macb_configure_dma(struct macb *bp) { + struct macb_queue *queue; + u32 buffer_size; + unsigned int q; u32 dmacfg; + buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE; if (macb_is_gem(bp)) { dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L); - dmacfg |= GEM_BF(RXBS, bp->rx_buffer_size / RX_BUFFER_MULTIPLE); + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + if (q) + queue_writel(queue, RBQS, buffer_size); + else + dmacfg |= GEM_BF(RXBS, buffer_size); + } if (bp->dma_burst_length) dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg); dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L); @@ -1997,14 +2761,11 @@ static void macb_configure_dma(struct macb *bp) else dmacfg &= ~GEM_BIT(TXCOEN); -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - if (bp->hw_dma_cap & HW_DMA_CAP_64B) + dmacfg &= ~GEM_BIT(ADDR64); + if (macb_dma64(bp)) dmacfg |= GEM_BIT(ADDR64); -#endif -#ifdef CONFIG_MACB_USE_HWSTAMP - if (bp->hw_dma_cap & HW_DMA_CAP_PTP) + if (macb_dma_ptp(bp)) dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT); -#endif netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n", dmacfg); gem_writel(bp, DMACFG, dmacfg); @@ -2013,19 +2774,17 @@ static void macb_configure_dma(struct macb *bp) static void macb_init_hw(struct macb *bp) { - struct macb_queue *queue; - unsigned int q; - u32 config; macb_reset_hw(bp); macb_set_hwaddr(bp); config = macb_mdc_clk_div(bp); - if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) - config |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); - config |= MACB_BF(RBOF, NET_IP_ALIGN); /* Make eth data aligned */ - config |= MACB_BIT(PAE); /* PAuse Enable */ + /* Make eth data aligned. + * If RSC capable, that offset is ignored by HW. + */ + if (!(bp->caps & MACB_CAPS_RSC)) + config |= MACB_BF(RBOF, NET_IP_ALIGN); config |= MACB_BIT(DRFCS); /* Discard Rx FCS */ if (bp->caps & MACB_CAPS_JUMBO) config |= MACB_BIT(JFRAME); /* Enable jumbo frames */ @@ -2041,36 +2800,15 @@ static void macb_init_hw(struct macb *bp) macb_writel(bp, NCFGR, config); if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) gem_writel(bp, JML, bp->jumbo_max_len); - bp->speed = SPEED_10; - bp->duplex = DUPLEX_HALF; bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK; if (bp->caps & MACB_CAPS_JUMBO) bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK; macb_configure_dma(bp); - /* Initialize TX and RX buffers */ - macb_writel(bp, RBQP, lower_32_bits(bp->rx_ring_dma)); -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - if (bp->hw_dma_cap & HW_DMA_CAP_64B) - macb_writel(bp, RBQPH, upper_32_bits(bp->rx_ring_dma)); -#endif - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - if (bp->hw_dma_cap & HW_DMA_CAP_64B) - queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); -#endif - - /* Enable interrupts */ - queue_writel(queue, IER, - MACB_RX_INT_FLAGS | - MACB_TX_INT_FLAGS | - MACB_BIT(HRESP)); - } - - /* Enable TX and RX */ - macb_writel(bp, NCR, MACB_BIT(RE) | MACB_BIT(TE) | MACB_BIT(MPE)); + /* Enable RX partial store and forward and set watermark */ + if (bp->rx_watermark) + gem_writel(bp, PBUFRXCUT, (bp->rx_watermark | GEM_BIT(ENCUTTHRU))); } /* The hash address register is 64 bits long and takes up two @@ -2194,18 +2932,17 @@ static void macb_set_rx_mode(struct net_device *dev) static int macb_open(struct net_device *dev) { - struct macb *bp = netdev_priv(dev); size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; + struct macb *bp = netdev_priv(dev); + struct macb_queue *queue; + unsigned int q; int err; netdev_dbg(bp->dev, "open\n"); - /* carrier starts down */ - netif_carrier_off(dev); - - /* if the phy is not yet register, retry later*/ - if (!dev->phydev) - return -EAGAIN; + err = pm_runtime_resume_and_get(&bp->pdev->dev); + if (err < 0) + return err; /* RX buffers initialization */ macb_init_rx_buffer_size(bp, bufsz); @@ -2214,16 +2951,27 @@ static int macb_open(struct net_device *dev) if (err) { netdev_err(dev, "Unable to allocate DMA memory (error %d)\n", err); - return err; + goto pm_exit; } - napi_enable(&bp->napi); + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + napi_enable(&queue->napi_rx); + napi_enable(&queue->napi_tx); + } - bp->macbgem_ops.mog_init_rings(bp); macb_init_hw(bp); - /* schedule a link state check */ - phy_start(dev->phydev); + err = phy_set_mode_ext(bp->phy, PHY_MODE_ETHERNET, bp->phy_interface); + if (err) + goto reset_hw; + + err = phy_power_on(bp->phy); + if (err) + goto reset_hw; + + err = macb_phylink_connect(bp); + if (err) + goto phy_off; netif_tx_start_all_queues(dev); @@ -2231,18 +2979,41 @@ static int macb_open(struct net_device *dev) bp->ptp_info->ptp_init(dev); return 0; + +phy_off: + phy_power_off(bp->phy); + +reset_hw: + macb_reset_hw(bp); + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + napi_disable(&queue->napi_rx); + napi_disable(&queue->napi_tx); + } + macb_free_consistent(bp); +pm_exit: + pm_runtime_put_sync(&bp->pdev->dev); + return err; } static int macb_close(struct net_device *dev) { struct macb *bp = netdev_priv(dev); + struct macb_queue *queue; unsigned long flags; + unsigned int q; netif_tx_stop_all_queues(dev); - napi_disable(&bp->napi); - if (dev->phydev) - phy_stop(dev->phydev); + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + napi_disable(&queue->napi_rx); + napi_disable(&queue->napi_tx); + netdev_tx_reset_queue(netdev_get_tx_queue(dev, q)); + } + + phylink_stop(bp->phylink); + phylink_disconnect_phy(bp->phylink); + + phy_power_off(bp->phy); spin_lock_irqsave(&bp->lock, flags); macb_reset_hw(bp); @@ -2254,6 +3025,8 @@ static int macb_close(struct net_device *dev) if (bp->ptp_info) bp->ptp_info->ptp_remove(dev); + pm_runtime_put(&bp->pdev->dev); + return 0; } @@ -2262,15 +3035,30 @@ static int macb_change_mtu(struct net_device *dev, int new_mtu) if (netif_running(dev)) return -EBUSY; - dev->mtu = new_mtu; + WRITE_ONCE(dev->mtu, new_mtu); return 0; } +static int macb_set_mac_addr(struct net_device *dev, void *addr) +{ + int err; + + err = eth_mac_addr(dev, addr); + if (err < 0) + return err; + + macb_set_hwaddr(netdev_priv(dev)); + return 0; +} + static void gem_update_stats(struct macb *bp) { - unsigned int i; - u32 *p = &bp->hw_stats.gem.tx_octets_31_0; + struct macb_queue *queue; + unsigned int i, q, idx; + unsigned long *stat; + + u64 *p = &bp->hw_stats.gem.tx_octets; for (i = 0; i < GEM_STATS_LEN; ++i, ++p) { u32 offset = gem_statistics[i].offset; @@ -2283,17 +3071,23 @@ static void gem_update_stats(struct macb *bp) /* Add GEM_OCTTXH, GEM_OCTRXH */ val = bp->macb_reg_readl(bp, offset + 4); bp->ethtool_stats[i] += ((u64)val) << 32; - *(++p) += val; + *p += ((u64)val) << 32; } } + + idx = GEM_STATS_LEN; + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) + for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat) + bp->ethtool_stats[idx++] = *stat; } -static struct net_device_stats *gem_get_stats(struct macb *bp) +static void gem_get_stats(struct macb *bp, struct rtnl_link_stats64 *nstat) { struct gem_stats *hwstat = &bp->hw_stats.gem; - struct net_device_stats *nstat = &bp->dev->stats; - gem_update_stats(bp); + spin_lock_irq(&bp->stats_lock); + if (netif_running(bp->dev)) + gem_update_stats(bp); nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + hwstat->rx_alignment_errors + @@ -2322,25 +3116,28 @@ static struct net_device_stats *gem_get_stats(struct macb *bp) nstat->tx_aborted_errors = hwstat->tx_excessive_collisions; nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors; nstat->tx_fifo_errors = hwstat->tx_underrun; - - return nstat; + spin_unlock_irq(&bp->stats_lock); } static void gem_get_ethtool_stats(struct net_device *dev, struct ethtool_stats *stats, u64 *data) { - struct macb *bp; + struct macb *bp = netdev_priv(dev); - bp = netdev_priv(dev); + spin_lock_irq(&bp->stats_lock); gem_update_stats(bp); - memcpy(data, &bp->ethtool_stats, sizeof(u64) * GEM_STATS_LEN); + memcpy(data, &bp->ethtool_stats, sizeof(u64) + * (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES)); + spin_unlock_irq(&bp->stats_lock); } static int gem_get_sset_count(struct net_device *dev, int sset) { + struct macb *bp = netdev_priv(dev); + switch (sset) { case ETH_SS_STATS: - return GEM_STATS_LEN; + return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN; default: return -EOPNOTSUPP; } @@ -2348,27 +3145,43 @@ static int gem_get_sset_count(struct net_device *dev, int sset) static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p) { + char stat_string[ETH_GSTRING_LEN]; + struct macb *bp = netdev_priv(dev); + struct macb_queue *queue; unsigned int i; + unsigned int q; switch (sset) { case ETH_SS_STATS: for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN) memcpy(p, gem_statistics[i].stat_string, ETH_GSTRING_LEN); + + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) { + snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s", + q, queue_statistics[i].stat_string); + memcpy(p, stat_string, ETH_GSTRING_LEN); + } + } break; } } -static struct net_device_stats *macb_get_stats(struct net_device *dev) +static void macb_get_stats(struct net_device *dev, + struct rtnl_link_stats64 *nstat) { struct macb *bp = netdev_priv(dev); - struct net_device_stats *nstat = &bp->dev->stats; struct macb_stats *hwstat = &bp->hw_stats.macb; - if (macb_is_gem(bp)) - return gem_get_stats(bp); + netdev_stats_to_stats64(nstat, &bp->dev->stats); + if (macb_is_gem(bp)) { + gem_get_stats(bp, nstat); + return; + } /* read stats from hardware */ + spin_lock_irq(&bp->stats_lock); macb_update_stats(bp); /* Convert HW stats into netdevice stats */ @@ -2402,8 +3215,171 @@ static struct net_device_stats *macb_get_stats(struct net_device *dev) nstat->tx_carrier_errors = hwstat->tx_carrier_errors; nstat->tx_fifo_errors = hwstat->tx_underruns; /* Don't know about heartbeat or window errors... */ + spin_unlock_irq(&bp->stats_lock); +} + +static void macb_get_pause_stats(struct net_device *dev, + struct ethtool_pause_stats *pause_stats) +{ + struct macb *bp = netdev_priv(dev); + struct macb_stats *hwstat = &bp->hw_stats.macb; + + spin_lock_irq(&bp->stats_lock); + macb_update_stats(bp); + pause_stats->tx_pause_frames = hwstat->tx_pause_frames; + pause_stats->rx_pause_frames = hwstat->rx_pause_frames; + spin_unlock_irq(&bp->stats_lock); +} + +static void gem_get_pause_stats(struct net_device *dev, + struct ethtool_pause_stats *pause_stats) +{ + struct macb *bp = netdev_priv(dev); + struct gem_stats *hwstat = &bp->hw_stats.gem; - return nstat; + spin_lock_irq(&bp->stats_lock); + gem_update_stats(bp); + pause_stats->tx_pause_frames = hwstat->tx_pause_frames; + pause_stats->rx_pause_frames = hwstat->rx_pause_frames; + spin_unlock_irq(&bp->stats_lock); +} + +static void macb_get_eth_mac_stats(struct net_device *dev, + struct ethtool_eth_mac_stats *mac_stats) +{ + struct macb *bp = netdev_priv(dev); + struct macb_stats *hwstat = &bp->hw_stats.macb; + + spin_lock_irq(&bp->stats_lock); + macb_update_stats(bp); + mac_stats->FramesTransmittedOK = hwstat->tx_ok; + mac_stats->SingleCollisionFrames = hwstat->tx_single_cols; + mac_stats->MultipleCollisionFrames = hwstat->tx_multiple_cols; + mac_stats->FramesReceivedOK = hwstat->rx_ok; + mac_stats->FrameCheckSequenceErrors = hwstat->rx_fcs_errors; + mac_stats->AlignmentErrors = hwstat->rx_align_errors; + mac_stats->FramesWithDeferredXmissions = hwstat->tx_deferred; + mac_stats->LateCollisions = hwstat->tx_late_cols; + mac_stats->FramesAbortedDueToXSColls = hwstat->tx_excessive_cols; + mac_stats->FramesLostDueToIntMACXmitError = hwstat->tx_underruns; + mac_stats->CarrierSenseErrors = hwstat->tx_carrier_errors; + mac_stats->FramesLostDueToIntMACRcvError = hwstat->rx_overruns; + mac_stats->InRangeLengthErrors = hwstat->rx_length_mismatch; + mac_stats->FrameTooLongErrors = hwstat->rx_oversize_pkts; + spin_unlock_irq(&bp->stats_lock); +} + +static void gem_get_eth_mac_stats(struct net_device *dev, + struct ethtool_eth_mac_stats *mac_stats) +{ + struct macb *bp = netdev_priv(dev); + struct gem_stats *hwstat = &bp->hw_stats.gem; + + spin_lock_irq(&bp->stats_lock); + gem_update_stats(bp); + mac_stats->FramesTransmittedOK = hwstat->tx_frames; + mac_stats->SingleCollisionFrames = hwstat->tx_single_collision_frames; + mac_stats->MultipleCollisionFrames = + hwstat->tx_multiple_collision_frames; + mac_stats->FramesReceivedOK = hwstat->rx_frames; + mac_stats->FrameCheckSequenceErrors = + hwstat->rx_frame_check_sequence_errors; + mac_stats->AlignmentErrors = hwstat->rx_alignment_errors; + mac_stats->OctetsTransmittedOK = hwstat->tx_octets; + mac_stats->FramesWithDeferredXmissions = hwstat->tx_deferred_frames; + mac_stats->LateCollisions = hwstat->tx_late_collisions; + mac_stats->FramesAbortedDueToXSColls = hwstat->tx_excessive_collisions; + mac_stats->FramesLostDueToIntMACXmitError = hwstat->tx_underrun; + mac_stats->CarrierSenseErrors = hwstat->tx_carrier_sense_errors; + mac_stats->OctetsReceivedOK = hwstat->rx_octets; + mac_stats->MulticastFramesXmittedOK = hwstat->tx_multicast_frames; + mac_stats->BroadcastFramesXmittedOK = hwstat->tx_broadcast_frames; + mac_stats->MulticastFramesReceivedOK = hwstat->rx_multicast_frames; + mac_stats->BroadcastFramesReceivedOK = hwstat->rx_broadcast_frames; + mac_stats->InRangeLengthErrors = hwstat->rx_length_field_frame_errors; + mac_stats->FrameTooLongErrors = hwstat->rx_oversize_frames; + spin_unlock_irq(&bp->stats_lock); +} + +/* TODO: Report SQE test errors when added to phy_stats */ +static void macb_get_eth_phy_stats(struct net_device *dev, + struct ethtool_eth_phy_stats *phy_stats) +{ + struct macb *bp = netdev_priv(dev); + struct macb_stats *hwstat = &bp->hw_stats.macb; + + spin_lock_irq(&bp->stats_lock); + macb_update_stats(bp); + phy_stats->SymbolErrorDuringCarrier = hwstat->rx_symbol_errors; + spin_unlock_irq(&bp->stats_lock); +} + +static void gem_get_eth_phy_stats(struct net_device *dev, + struct ethtool_eth_phy_stats *phy_stats) +{ + struct macb *bp = netdev_priv(dev); + struct gem_stats *hwstat = &bp->hw_stats.gem; + + spin_lock_irq(&bp->stats_lock); + gem_update_stats(bp); + phy_stats->SymbolErrorDuringCarrier = hwstat->rx_symbol_errors; + spin_unlock_irq(&bp->stats_lock); +} + +static void macb_get_rmon_stats(struct net_device *dev, + struct ethtool_rmon_stats *rmon_stats, + const struct ethtool_rmon_hist_range **ranges) +{ + struct macb *bp = netdev_priv(dev); + struct macb_stats *hwstat = &bp->hw_stats.macb; + + spin_lock_irq(&bp->stats_lock); + macb_update_stats(bp); + rmon_stats->undersize_pkts = hwstat->rx_undersize_pkts; + rmon_stats->oversize_pkts = hwstat->rx_oversize_pkts; + rmon_stats->jabbers = hwstat->rx_jabbers; + spin_unlock_irq(&bp->stats_lock); +} + +static const struct ethtool_rmon_hist_range gem_rmon_ranges[] = { + { 64, 64 }, + { 65, 127 }, + { 128, 255 }, + { 256, 511 }, + { 512, 1023 }, + { 1024, 1518 }, + { 1519, 16384 }, + { }, +}; + +static void gem_get_rmon_stats(struct net_device *dev, + struct ethtool_rmon_stats *rmon_stats, + const struct ethtool_rmon_hist_range **ranges) +{ + struct macb *bp = netdev_priv(dev); + struct gem_stats *hwstat = &bp->hw_stats.gem; + + spin_lock_irq(&bp->stats_lock); + gem_update_stats(bp); + rmon_stats->undersize_pkts = hwstat->rx_undersized_frames; + rmon_stats->oversize_pkts = hwstat->rx_oversize_frames; + rmon_stats->jabbers = hwstat->rx_jabbers; + rmon_stats->hist[0] = hwstat->rx_64_byte_frames; + rmon_stats->hist[1] = hwstat->rx_65_127_byte_frames; + rmon_stats->hist[2] = hwstat->rx_128_255_byte_frames; + rmon_stats->hist[3] = hwstat->rx_256_511_byte_frames; + rmon_stats->hist[4] = hwstat->rx_512_1023_byte_frames; + rmon_stats->hist[5] = hwstat->rx_1024_1518_byte_frames; + rmon_stats->hist[6] = hwstat->rx_greater_than_1518_byte_frames; + rmon_stats->hist_tx[0] = hwstat->tx_64_byte_frames; + rmon_stats->hist_tx[1] = hwstat->tx_65_127_byte_frames; + rmon_stats->hist_tx[2] = hwstat->tx_128_255_byte_frames; + rmon_stats->hist_tx[3] = hwstat->tx_256_511_byte_frames; + rmon_stats->hist_tx[4] = hwstat->tx_512_1023_byte_frames; + rmon_stats->hist_tx[5] = hwstat->tx_1024_1518_byte_frames; + rmon_stats->hist_tx[6] = hwstat->tx_greater_than_1518_byte_frames; + spin_unlock_irq(&bp->stats_lock); + *ranges = gem_rmon_ranges; } static int macb_get_regs_len(struct net_device *netdev) @@ -2448,37 +3424,53 @@ static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) { struct macb *bp = netdev_priv(netdev); - wol->supported = 0; - wol->wolopts = 0; - - if (bp->wol & MACB_WOL_HAS_MAGIC_PACKET) { - wol->supported = WAKE_MAGIC; + phylink_ethtool_get_wol(bp->phylink, wol); + wol->supported |= (WAKE_MAGIC | WAKE_ARP); - if (bp->wol & MACB_WOL_ENABLED) - wol->wolopts |= WAKE_MAGIC; - } + /* Add macb wolopts to phy wolopts */ + wol->wolopts |= bp->wolopts; } static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) { struct macb *bp = netdev_priv(netdev); + int ret; - if (!(bp->wol & MACB_WOL_HAS_MAGIC_PACKET) || - (wol->wolopts & ~WAKE_MAGIC)) - return -EOPNOTSUPP; + /* Pass the order to phylink layer */ + ret = phylink_ethtool_set_wol(bp->phylink, wol); + /* Don't manage WoL on MAC, if PHY set_wol() fails */ + if (ret && ret != -EOPNOTSUPP) + return ret; - if (wol->wolopts & WAKE_MAGIC) - bp->wol |= MACB_WOL_ENABLED; - else - bp->wol &= ~MACB_WOL_ENABLED; + bp->wolopts = (wol->wolopts & WAKE_MAGIC) ? WAKE_MAGIC : 0; + bp->wolopts |= (wol->wolopts & WAKE_ARP) ? WAKE_ARP : 0; + bp->wol = (wol->wolopts) ? MACB_WOL_ENABLED : 0; - device_set_wakeup_enable(&bp->pdev->dev, bp->wol & MACB_WOL_ENABLED); + device_set_wakeup_enable(&bp->pdev->dev, bp->wol); return 0; } +static int macb_get_link_ksettings(struct net_device *netdev, + struct ethtool_link_ksettings *kset) +{ + struct macb *bp = netdev_priv(netdev); + + return phylink_ethtool_ksettings_get(bp->phylink, kset); +} + +static int macb_set_link_ksettings(struct net_device *netdev, + const struct ethtool_link_ksettings *kset) +{ + struct macb *bp = netdev_priv(netdev); + + return phylink_ethtool_ksettings_set(bp->phylink, kset); +} + static void macb_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct macb *bp = netdev_priv(netdev); @@ -2490,7 +3482,9 @@ static void macb_get_ringparam(struct net_device *netdev, } static int macb_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct macb *bp = netdev_priv(netdev); u32 new_rx_size, new_tx_size; @@ -2551,19 +3545,17 @@ static s32 gem_get_ptp_max_adj(void) } static int gem_get_ts_info(struct net_device *dev, - struct ethtool_ts_info *info) + struct kernel_ethtool_ts_info *info) { struct macb *bp = netdev_priv(dev); - if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) { + if (!macb_dma_ptp(bp)) { ethtool_op_get_ts_info(dev, info); return 0; } info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | - SOF_TIMESTAMPING_RX_SOFTWARE | - SOF_TIMESTAMPING_SOFTWARE | SOF_TIMESTAMPING_TX_HARDWARE | SOF_TIMESTAMPING_RX_HARDWARE | SOF_TIMESTAMPING_RAW_HARDWARE; @@ -2575,7 +3567,8 @@ static int gem_get_ts_info(struct net_device *dev, (1 << HWTSTAMP_FILTER_NONE) | (1 << HWTSTAMP_FILTER_ALL); - info->phc_index = bp->ptp_clock ? ptp_clock_index(bp->ptp_clock) : -1; + if (bp->ptp_clock) + info->phc_index = ptp_clock_index(bp->ptp_clock); return 0; } @@ -2592,7 +3585,7 @@ static struct macb_ptp_info gem_ptp_info = { #endif static int macb_get_ts_info(struct net_device *netdev, - struct ethtool_ts_info *info) + struct kernel_ethtool_ts_info *info) { struct macb *bp = netdev_priv(netdev); @@ -2602,15 +3595,327 @@ static int macb_get_ts_info(struct net_device *netdev, return ethtool_op_get_ts_info(netdev, info); } +static void gem_enable_flow_filters(struct macb *bp, bool enable) +{ + struct net_device *netdev = bp->dev; + struct ethtool_rx_fs_item *item; + u32 t2_scr; + int num_t2_scr; + + if (!(netdev->features & NETIF_F_NTUPLE)) + return; + + num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8)); + + list_for_each_entry(item, &bp->rx_fs_list.list, list) { + struct ethtool_rx_flow_spec *fs = &item->fs; + struct ethtool_tcpip4_spec *tp4sp_m; + + if (fs->location >= num_t2_scr) + continue; + + t2_scr = gem_readl_n(bp, SCRT2, fs->location); + + /* enable/disable screener regs for the flow entry */ + t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr); + + /* only enable fields with no masking */ + tp4sp_m = &(fs->m_u.tcp_ip4_spec); + + if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF)) + t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr); + else + t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr); + + if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF)) + t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr); + else + t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr); + + if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF))) + t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr); + else + t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr); + + gem_writel_n(bp, SCRT2, fs->location, t2_scr); + } +} + +static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs) +{ + struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m; + uint16_t index = fs->location; + u32 w0, w1, t2_scr; + bool cmp_a = false; + bool cmp_b = false; + bool cmp_c = false; + + if (!macb_is_gem(bp)) + return; + + tp4sp_v = &(fs->h_u.tcp_ip4_spec); + tp4sp_m = &(fs->m_u.tcp_ip4_spec); + + /* ignore field if any masking set */ + if (tp4sp_m->ip4src == 0xFFFFFFFF) { + /* 1st compare reg - IP source address */ + w0 = 0; + w1 = 0; + w0 = tp4sp_v->ip4src; + w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ + w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); + w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1); + gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0); + gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1); + cmp_a = true; + } + + /* ignore field if any masking set */ + if (tp4sp_m->ip4dst == 0xFFFFFFFF) { + /* 2nd compare reg - IP destination address */ + w0 = 0; + w1 = 0; + w0 = tp4sp_v->ip4dst; + w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ + w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); + w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1); + gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0); + gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1); + cmp_b = true; + } + + /* ignore both port fields if masking set in both */ + if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) { + /* 3rd compare reg - source port, destination port */ + w0 = 0; + w1 = 0; + w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1); + if (tp4sp_m->psrc == tp4sp_m->pdst) { + w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0); + w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); + w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ + w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); + } else { + /* only one port definition */ + w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */ + w0 = GEM_BFINS(T2MASK, 0xFFFF, w0); + if (tp4sp_m->psrc == 0xFFFF) { /* src port */ + w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0); + w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); + } else { /* dst port */ + w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); + w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1); + } + } + gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0); + gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1); + cmp_c = true; + } + + t2_scr = 0; + t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr); + t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr); + if (cmp_a) + t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr); + if (cmp_b) + t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr); + if (cmp_c) + t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr); + gem_writel_n(bp, SCRT2, index, t2_scr); +} + +static int gem_add_flow_filter(struct net_device *netdev, + struct ethtool_rxnfc *cmd) +{ + struct macb *bp = netdev_priv(netdev); + struct ethtool_rx_flow_spec *fs = &cmd->fs; + struct ethtool_rx_fs_item *item, *newfs; + unsigned long flags; + int ret = -EINVAL; + bool added = false; + + newfs = kmalloc(sizeof(*newfs), GFP_KERNEL); + if (newfs == NULL) + return -ENOMEM; + memcpy(&newfs->fs, fs, sizeof(newfs->fs)); + + netdev_dbg(netdev, + "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", + fs->flow_type, (int)fs->ring_cookie, fs->location, + htonl(fs->h_u.tcp_ip4_spec.ip4src), + htonl(fs->h_u.tcp_ip4_spec.ip4dst), + be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc), + be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst)); + + spin_lock_irqsave(&bp->rx_fs_lock, flags); + + /* find correct place to add in list */ + list_for_each_entry(item, &bp->rx_fs_list.list, list) { + if (item->fs.location > newfs->fs.location) { + list_add_tail(&newfs->list, &item->list); + added = true; + break; + } else if (item->fs.location == fs->location) { + netdev_err(netdev, "Rule not added: location %d not free!\n", + fs->location); + ret = -EBUSY; + goto err; + } + } + if (!added) + list_add_tail(&newfs->list, &bp->rx_fs_list.list); + + gem_prog_cmp_regs(bp, fs); + bp->rx_fs_list.count++; + /* enable filtering if NTUPLE on */ + gem_enable_flow_filters(bp, 1); + + spin_unlock_irqrestore(&bp->rx_fs_lock, flags); + return 0; + +err: + spin_unlock_irqrestore(&bp->rx_fs_lock, flags); + kfree(newfs); + return ret; +} + +static int gem_del_flow_filter(struct net_device *netdev, + struct ethtool_rxnfc *cmd) +{ + struct macb *bp = netdev_priv(netdev); + struct ethtool_rx_fs_item *item; + struct ethtool_rx_flow_spec *fs; + unsigned long flags; + + spin_lock_irqsave(&bp->rx_fs_lock, flags); + + list_for_each_entry(item, &bp->rx_fs_list.list, list) { + if (item->fs.location == cmd->fs.location) { + /* disable screener regs for the flow entry */ + fs = &(item->fs); + netdev_dbg(netdev, + "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", + fs->flow_type, (int)fs->ring_cookie, fs->location, + htonl(fs->h_u.tcp_ip4_spec.ip4src), + htonl(fs->h_u.tcp_ip4_spec.ip4dst), + be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc), + be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst)); + + gem_writel_n(bp, SCRT2, fs->location, 0); + + list_del(&item->list); + bp->rx_fs_list.count--; + spin_unlock_irqrestore(&bp->rx_fs_lock, flags); + kfree(item); + return 0; + } + } + + spin_unlock_irqrestore(&bp->rx_fs_lock, flags); + return -EINVAL; +} + +static int gem_get_flow_entry(struct net_device *netdev, + struct ethtool_rxnfc *cmd) +{ + struct macb *bp = netdev_priv(netdev); + struct ethtool_rx_fs_item *item; + + list_for_each_entry(item, &bp->rx_fs_list.list, list) { + if (item->fs.location == cmd->fs.location) { + memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs)); + return 0; + } + } + return -EINVAL; +} + +static int gem_get_all_flow_entries(struct net_device *netdev, + struct ethtool_rxnfc *cmd, u32 *rule_locs) +{ + struct macb *bp = netdev_priv(netdev); + struct ethtool_rx_fs_item *item; + uint32_t cnt = 0; + + list_for_each_entry(item, &bp->rx_fs_list.list, list) { + if (cnt == cmd->rule_cnt) + return -EMSGSIZE; + rule_locs[cnt] = item->fs.location; + cnt++; + } + cmd->data = bp->max_tuples; + cmd->rule_cnt = cnt; + + return 0; +} + +static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, + u32 *rule_locs) +{ + struct macb *bp = netdev_priv(netdev); + int ret = 0; + + switch (cmd->cmd) { + case ETHTOOL_GRXRINGS: + cmd->data = bp->num_queues; + break; + case ETHTOOL_GRXCLSRLCNT: + cmd->rule_cnt = bp->rx_fs_list.count; + break; + case ETHTOOL_GRXCLSRULE: + ret = gem_get_flow_entry(netdev, cmd); + break; + case ETHTOOL_GRXCLSRLALL: + ret = gem_get_all_flow_entries(netdev, cmd, rule_locs); + break; + default: + netdev_err(netdev, + "Command parameter %d is not supported\n", cmd->cmd); + ret = -EOPNOTSUPP; + } + + return ret; +} + +static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) +{ + struct macb *bp = netdev_priv(netdev); + int ret; + + switch (cmd->cmd) { + case ETHTOOL_SRXCLSRLINS: + if ((cmd->fs.location >= bp->max_tuples) + || (cmd->fs.ring_cookie >= bp->num_queues)) { + ret = -EINVAL; + break; + } + ret = gem_add_flow_filter(netdev, cmd); + break; + case ETHTOOL_SRXCLSRLDEL: + ret = gem_del_flow_filter(netdev, cmd); + break; + default: + netdev_err(netdev, + "Command parameter %d is not supported\n", cmd->cmd); + ret = -EOPNOTSUPP; + } + + return ret; +} + static const struct ethtool_ops macb_ethtool_ops = { .get_regs_len = macb_get_regs_len, .get_regs = macb_get_regs, .get_link = ethtool_op_get_link, .get_ts_info = ethtool_op_get_ts_info, + .get_pause_stats = macb_get_pause_stats, + .get_eth_mac_stats = macb_get_eth_mac_stats, + .get_eth_phy_stats = macb_get_eth_phy_stats, + .get_rmon_stats = macb_get_rmon_stats, .get_wol = macb_get_wol, .set_wol = macb_set_wol, - .get_link_ksettings = phy_ethtool_get_link_ksettings, - .set_link_ksettings = phy_ethtool_set_link_ksettings, + .get_link_ksettings = macb_get_link_ksettings, + .set_link_ksettings = macb_set_link_ksettings, .get_ringparam = macb_get_ringparam, .set_ringparam = macb_set_ringparam, }; @@ -2618,39 +3923,106 @@ static const struct ethtool_ops macb_ethtool_ops = { static const struct ethtool_ops gem_ethtool_ops = { .get_regs_len = macb_get_regs_len, .get_regs = macb_get_regs, + .get_wol = macb_get_wol, + .set_wol = macb_set_wol, .get_link = ethtool_op_get_link, .get_ts_info = macb_get_ts_info, .get_ethtool_stats = gem_get_ethtool_stats, .get_strings = gem_get_ethtool_strings, .get_sset_count = gem_get_sset_count, - .get_link_ksettings = phy_ethtool_get_link_ksettings, - .set_link_ksettings = phy_ethtool_set_link_ksettings, + .get_pause_stats = gem_get_pause_stats, + .get_eth_mac_stats = gem_get_eth_mac_stats, + .get_eth_phy_stats = gem_get_eth_phy_stats, + .get_rmon_stats = gem_get_rmon_stats, + .get_link_ksettings = macb_get_link_ksettings, + .set_link_ksettings = macb_set_link_ksettings, .get_ringparam = macb_get_ringparam, .set_ringparam = macb_set_ringparam, + .get_rxnfc = gem_get_rxnfc, + .set_rxnfc = gem_set_rxnfc, }; static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) { - struct phy_device *phydev = dev->phydev; struct macb *bp = netdev_priv(dev); if (!netif_running(dev)) return -EINVAL; - if (!phydev) - return -ENODEV; + return phylink_mii_ioctl(bp->phylink, rq, cmd); +} + +static int macb_hwtstamp_get(struct net_device *dev, + struct kernel_hwtstamp_config *cfg) +{ + struct macb *bp = netdev_priv(dev); + + if (!netif_running(dev)) + return -EINVAL; if (!bp->ptp_info) - return phy_mii_ioctl(phydev, rq, cmd); + return -EOPNOTSUPP; - switch (cmd) { - case SIOCSHWTSTAMP: - return bp->ptp_info->set_hwtst(dev, rq, cmd); - case SIOCGHWTSTAMP: - return bp->ptp_info->get_hwtst(dev, rq); - default: - return phy_mii_ioctl(phydev, rq, cmd); - } + return bp->ptp_info->get_hwtst(dev, cfg); +} + +static int macb_hwtstamp_set(struct net_device *dev, + struct kernel_hwtstamp_config *cfg, + struct netlink_ext_ack *extack) +{ + struct macb *bp = netdev_priv(dev); + + if (!netif_running(dev)) + return -EINVAL; + + if (!bp->ptp_info) + return -EOPNOTSUPP; + + return bp->ptp_info->set_hwtst(dev, cfg, extack); +} + +static inline void macb_set_txcsum_feature(struct macb *bp, + netdev_features_t features) +{ + u32 val; + + if (!macb_is_gem(bp)) + return; + + val = gem_readl(bp, DMACFG); + if (features & NETIF_F_HW_CSUM) + val |= GEM_BIT(TXCOEN); + else + val &= ~GEM_BIT(TXCOEN); + + gem_writel(bp, DMACFG, val); +} + +static inline void macb_set_rxcsum_feature(struct macb *bp, + netdev_features_t features) +{ + struct net_device *netdev = bp->dev; + u32 val; + + if (!macb_is_gem(bp)) + return; + + val = gem_readl(bp, NCFGR); + if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC)) + val |= GEM_BIT(RXCOEN); + else + val &= ~GEM_BIT(RXCOEN); + + gem_writel(bp, NCFGR, val); +} + +static inline void macb_set_rxflow_feature(struct macb *bp, + netdev_features_t features) +{ + if (!macb_is_gem(bp)) + return; + + gem_enable_flow_filters(bp, !!(features & NETIF_F_NTUPLE)); } static int macb_set_features(struct net_device *netdev, @@ -2660,31 +4032,259 @@ static int macb_set_features(struct net_device *netdev, netdev_features_t changed = features ^ netdev->features; /* TX checksum offload */ - if ((changed & NETIF_F_HW_CSUM) && macb_is_gem(bp)) { - u32 dmacfg; + if (changed & NETIF_F_HW_CSUM) + macb_set_txcsum_feature(bp, features); - dmacfg = gem_readl(bp, DMACFG); - if (features & NETIF_F_HW_CSUM) - dmacfg |= GEM_BIT(TXCOEN); - else - dmacfg &= ~GEM_BIT(TXCOEN); - gem_writel(bp, DMACFG, dmacfg); - } + /* RX checksum offload */ + if (changed & NETIF_F_RXCSUM) + macb_set_rxcsum_feature(bp, features); + + /* RX Flow Filters */ + if (changed & NETIF_F_NTUPLE) + macb_set_rxflow_feature(bp, features); + + return 0; +} + +static void macb_restore_features(struct macb *bp) +{ + struct net_device *netdev = bp->dev; + netdev_features_t features = netdev->features; + struct ethtool_rx_fs_item *item; + + /* TX checksum offload */ + macb_set_txcsum_feature(bp, features); /* RX checksum offload */ - if ((changed & NETIF_F_RXCSUM) && macb_is_gem(bp)) { - u32 netcfg; + macb_set_rxcsum_feature(bp, features); - netcfg = gem_readl(bp, NCFGR); - if (features & NETIF_F_RXCSUM && - !(netdev->flags & IFF_PROMISC)) - netcfg |= GEM_BIT(RXCOEN); - else - netcfg &= ~GEM_BIT(RXCOEN); - gem_writel(bp, NCFGR, netcfg); + /* RX Flow Filters */ + list_for_each_entry(item, &bp->rx_fs_list.list, list) + gem_prog_cmp_regs(bp, &item->fs); + + macb_set_rxflow_feature(bp, features); +} + +static int macb_taprio_setup_replace(struct net_device *ndev, + struct tc_taprio_qopt_offload *conf) +{ + u64 total_on_time = 0, start_time_sec = 0, start_time = conf->base_time; + u32 configured_queues = 0, speed = 0, start_time_nsec; + struct macb_queue_enst_config *enst_queue; + struct tc_taprio_sched_entry *entry; + struct macb *bp = netdev_priv(ndev); + struct ethtool_link_ksettings kset; + struct macb_queue *queue; + u32 queue_mask; + u8 queue_id; + size_t i; + int err; + + if (conf->num_entries > bp->num_queues) { + netdev_err(ndev, "Too many TAPRIO entries: %zu > %d queues\n", + conf->num_entries, bp->num_queues); + return -EINVAL; } - return 0; + if (conf->base_time < 0) { + netdev_err(ndev, "Invalid base_time: must be 0 or positive, got %lld\n", + conf->base_time); + return -ERANGE; + } + + /* Get the current link speed */ + err = phylink_ethtool_ksettings_get(bp->phylink, &kset); + if (unlikely(err)) { + netdev_err(ndev, "Failed to get link settings: %d\n", err); + return err; + } + + speed = kset.base.speed; + if (unlikely(speed <= 0)) { + netdev_err(ndev, "Invalid speed: %d\n", speed); + return -EINVAL; + } + + enst_queue = kcalloc(conf->num_entries, sizeof(*enst_queue), GFP_KERNEL); + if (unlikely(!enst_queue)) + return -ENOMEM; + + /* Pre-validate all entries before making any hardware changes */ + for (i = 0; i < conf->num_entries; i++) { + entry = &conf->entries[i]; + + if (entry->command != TC_TAPRIO_CMD_SET_GATES) { + netdev_err(ndev, "Entry %zu: unsupported command %d\n", + i, entry->command); + err = -EOPNOTSUPP; + goto cleanup; + } + + /* Validate gate_mask: must be nonzero, single queue, and within range */ + if (!is_power_of_2(entry->gate_mask)) { + netdev_err(ndev, "Entry %zu: gate_mask 0x%x is not a power of 2 (only one queue per entry allowed)\n", + i, entry->gate_mask); + err = -EINVAL; + goto cleanup; + } + + /* gate_mask must not select queues outside the valid queues */ + queue_id = order_base_2(entry->gate_mask); + if (queue_id >= bp->num_queues) { + netdev_err(ndev, "Entry %zu: gate_mask 0x%x exceeds queue range (max_queues=%d)\n", + i, entry->gate_mask, bp->num_queues); + err = -EINVAL; + goto cleanup; + } + + /* Check for start time limits */ + start_time_sec = start_time; + start_time_nsec = do_div(start_time_sec, NSEC_PER_SEC); + if (start_time_sec > GENMASK(GEM_START_TIME_SEC_SIZE - 1, 0)) { + netdev_err(ndev, "Entry %zu: Start time %llu s exceeds hardware limit\n", + i, start_time_sec); + err = -ERANGE; + goto cleanup; + } + + /* Check for on time limit */ + if (entry->interval > enst_max_hw_interval(speed)) { + netdev_err(ndev, "Entry %zu: interval %u ns exceeds hardware limit %llu ns\n", + i, entry->interval, enst_max_hw_interval(speed)); + err = -ERANGE; + goto cleanup; + } + + /* Check for off time limit*/ + if ((conf->cycle_time - entry->interval) > enst_max_hw_interval(speed)) { + netdev_err(ndev, "Entry %zu: off_time %llu ns exceeds hardware limit %llu ns\n", + i, conf->cycle_time - entry->interval, + enst_max_hw_interval(speed)); + err = -ERANGE; + goto cleanup; + } + + enst_queue[i].queue_id = queue_id; + enst_queue[i].start_time_mask = + (start_time_sec << GEM_START_TIME_SEC_OFFSET) | + start_time_nsec; + enst_queue[i].on_time_bytes = + enst_ns_to_hw_units(entry->interval, speed); + enst_queue[i].off_time_bytes = + enst_ns_to_hw_units(conf->cycle_time - entry->interval, speed); + + configured_queues |= entry->gate_mask; + total_on_time += entry->interval; + start_time += entry->interval; + } + + /* Check total interval doesn't exceed cycle time */ + if (total_on_time > conf->cycle_time) { + netdev_err(ndev, "Total ON %llu ns exceeds cycle time %llu ns\n", + total_on_time, conf->cycle_time); + err = -EINVAL; + goto cleanup; + } + + netdev_dbg(ndev, "TAPRIO setup: %zu entries, base_time=%lld ns, cycle_time=%llu ns\n", + conf->num_entries, conf->base_time, conf->cycle_time); + + /* All validations passed - proceed with hardware configuration */ + scoped_guard(spinlock_irqsave, &bp->lock) { + /* Disable ENST queues if running before configuring */ + queue_mask = BIT_U32(bp->num_queues) - 1; + gem_writel(bp, ENST_CONTROL, + queue_mask << GEM_ENST_DISABLE_QUEUE_OFFSET); + + for (i = 0; i < conf->num_entries; i++) { + queue = &bp->queues[enst_queue[i].queue_id]; + /* Configure queue timing registers */ + queue_writel(queue, ENST_START_TIME, + enst_queue[i].start_time_mask); + queue_writel(queue, ENST_ON_TIME, + enst_queue[i].on_time_bytes); + queue_writel(queue, ENST_OFF_TIME, + enst_queue[i].off_time_bytes); + } + + /* Enable ENST for all configured queues in one write */ + gem_writel(bp, ENST_CONTROL, configured_queues); + } + + netdev_info(ndev, "TAPRIO configuration completed successfully: %zu entries, %d queues configured\n", + conf->num_entries, hweight32(configured_queues)); + +cleanup: + kfree(enst_queue); + return err; +} + +static void macb_taprio_destroy(struct net_device *ndev) +{ + struct macb *bp = netdev_priv(ndev); + struct macb_queue *queue; + u32 queue_mask; + unsigned int q; + + netdev_reset_tc(ndev); + queue_mask = BIT_U32(bp->num_queues) - 1; + + scoped_guard(spinlock_irqsave, &bp->lock) { + /* Single disable command for all queues */ + gem_writel(bp, ENST_CONTROL, + queue_mask << GEM_ENST_DISABLE_QUEUE_OFFSET); + + /* Clear all queue ENST registers in batch */ + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + queue_writel(queue, ENST_START_TIME, 0); + queue_writel(queue, ENST_ON_TIME, 0); + queue_writel(queue, ENST_OFF_TIME, 0); + } + } + netdev_info(ndev, "TAPRIO destroy: All gates disabled\n"); +} + +static int macb_setup_taprio(struct net_device *ndev, + struct tc_taprio_qopt_offload *taprio) +{ + struct macb *bp = netdev_priv(ndev); + int err = 0; + + if (unlikely(!(ndev->hw_features & NETIF_F_HW_TC))) + return -EOPNOTSUPP; + + /* Check if Device is in runtime suspend */ + if (unlikely(pm_runtime_suspended(&bp->pdev->dev))) { + netdev_err(ndev, "Device is in runtime suspend\n"); + return -EOPNOTSUPP; + } + + switch (taprio->cmd) { + case TAPRIO_CMD_REPLACE: + err = macb_taprio_setup_replace(ndev, taprio); + break; + case TAPRIO_CMD_DESTROY: + macb_taprio_destroy(ndev); + break; + default: + err = -EOPNOTSUPP; + } + + return err; +} + +static int macb_setup_tc(struct net_device *dev, enum tc_setup_type type, + void *type_data) +{ + if (!dev || !type_data) + return -EINVAL; + + switch (type) { + case TC_SETUP_QDISC_TAPRIO: + return macb_setup_taprio(dev, type_data); + default: + return -EOPNOTSUPP; + } } static const struct net_device_ops macb_netdev_ops = { @@ -2692,16 +4292,19 @@ static const struct net_device_ops macb_netdev_ops = { .ndo_stop = macb_close, .ndo_start_xmit = macb_start_xmit, .ndo_set_rx_mode = macb_set_rx_mode, - .ndo_get_stats = macb_get_stats, - .ndo_do_ioctl = macb_ioctl, + .ndo_get_stats64 = macb_get_stats, + .ndo_eth_ioctl = macb_ioctl, .ndo_validate_addr = eth_validate_addr, .ndo_change_mtu = macb_change_mtu, - .ndo_set_mac_address = eth_mac_addr, + .ndo_set_mac_address = macb_set_mac_addr, #ifdef CONFIG_NET_POLL_CONTROLLER .ndo_poll_controller = macb_poll_controller, #endif .ndo_set_features = macb_set_features, .ndo_features_check = macb_features_check, + .ndo_hwtstamp_set = macb_hwtstamp_set, + .ndo_hwtstamp_get = macb_hwtstamp_get, + .ndo_setup_tc = macb_setup_tc, }; /* Configure peripheral capabilities according to device tree @@ -2710,8 +4313,12 @@ static const struct net_device_ops macb_netdev_ops = { static void macb_configure_caps(struct macb *bp, const struct macb_config *dt_conf) { + struct device_node *np = bp->pdev->dev.of_node; + bool refclk_ext; u32 dcfg; + refclk_ext = of_property_read_bool(np, "cdns,refclk-ext"); + if (dt_conf) bp->caps = dt_conf->caps; @@ -2721,56 +4328,73 @@ static void macb_configure_caps(struct macb *bp, dcfg = gem_readl(bp, DCFG1); if (GEM_BFEXT(IRQCOR, dcfg) == 0) bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE; + if (GEM_BFEXT(NO_PCS, dcfg) == 0) + bp->caps |= MACB_CAPS_PCS; + dcfg = gem_readl(bp, DCFG12); + if (GEM_BFEXT(HIGH_SPEED, dcfg) == 1) + bp->caps |= MACB_CAPS_HIGH_SPEED; dcfg = gem_readl(bp, DCFG2); if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0) bp->caps |= MACB_CAPS_FIFO_MODE; -#ifdef CONFIG_MACB_USE_HWSTAMP + if (GEM_BFEXT(PBUF_RSC, gem_readl(bp, DCFG6))) + bp->caps |= MACB_CAPS_RSC; if (gem_has_ptp(bp)) { if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5))) - pr_err("GEM doesn't support hardware ptp.\n"); + dev_err(&bp->pdev->dev, + "GEM doesn't support hardware ptp.\n"); else { - bp->hw_dma_cap |= HW_DMA_CAP_PTP; +#ifdef CONFIG_MACB_USE_HWSTAMP + bp->caps |= MACB_CAPS_DMA_PTP; bp->ptp_info = &gem_ptp_info; +#endif } } -#endif } + if (refclk_ext) + bp->caps |= MACB_CAPS_USRIO_HAS_CLKEN; + dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps); } -static void macb_probe_queues(void __iomem *mem, - bool native_io, - unsigned int *queue_mask, - unsigned int *num_queues) +static int macb_probe_queues(struct device *dev, void __iomem *mem, bool native_io) { - unsigned int hw_q; - - *queue_mask = 0x1; - *num_queues = 1; + /* BIT(0) is never set but queue 0 always exists. */ + unsigned int queue_mask = 0x1; - /* is it macb or gem ? - * - * We need to read directly from the hardware here because - * we are early in the probe process and don't have the - * MACB_CAPS_MACB_IS_GEM flag positioned - */ - if (!hw_is_gem(mem, native_io)) - return; + /* Use hw_is_gem() as MACB_CAPS_MACB_IS_GEM is not yet positioned. */ + if (hw_is_gem(mem, native_io)) { + if (native_io) + queue_mask |= __raw_readl(mem + GEM_DCFG6) & 0xFF; + else + queue_mask |= readl_relaxed(mem + GEM_DCFG6) & 0xFF; - /* bit 0 is never set but queue 0 always exists */ - *queue_mask = readl_relaxed(mem + GEM_DCFG6) & 0xff; + if (fls(queue_mask) != ffz(queue_mask)) { + dev_err(dev, "queue mask %#x has a hole\n", queue_mask); + return -EINVAL; + } + } - *queue_mask |= 0x1; + return hweight32(queue_mask); +} - for (hw_q = 1; hw_q < MACB_MAX_QUEUES; ++hw_q) - if (*queue_mask & (1 << hw_q)) - (*num_queues)++; +static void macb_clks_disable(struct clk *pclk, struct clk *hclk, struct clk *tx_clk, + struct clk *rx_clk, struct clk *tsu_clk) +{ + struct clk_bulk_data clks[] = { + { .clk = tsu_clk, }, + { .clk = rx_clk, }, + { .clk = pclk, }, + { .clk = hclk, }, + { .clk = tx_clk }, + }; + + clk_bulk_disable_unprepare(ARRAY_SIZE(clks), clks); } static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, struct clk **hclk, struct clk **tx_clk, - struct clk **rx_clk) + struct clk **rx_clk, struct clk **tsu_clk) { struct macb_platform_data *pdata; int err; @@ -2784,52 +4408,63 @@ static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, *hclk = devm_clk_get(&pdev->dev, "hclk"); } - if (IS_ERR(*pclk)) { - err = PTR_ERR(*pclk); - dev_err(&pdev->dev, "failed to get macb_clk (%u)\n", err); - return err; - } + if (IS_ERR_OR_NULL(*pclk)) + return dev_err_probe(&pdev->dev, + IS_ERR(*pclk) ? PTR_ERR(*pclk) : -ENODEV, + "failed to get pclk\n"); - if (IS_ERR(*hclk)) { - err = PTR_ERR(*hclk); - dev_err(&pdev->dev, "failed to get hclk (%u)\n", err); - return err; - } + if (IS_ERR_OR_NULL(*hclk)) + return dev_err_probe(&pdev->dev, + IS_ERR(*hclk) ? PTR_ERR(*hclk) : -ENODEV, + "failed to get hclk\n"); - *tx_clk = devm_clk_get(&pdev->dev, "tx_clk"); + *tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk"); if (IS_ERR(*tx_clk)) - *tx_clk = NULL; + return PTR_ERR(*tx_clk); - *rx_clk = devm_clk_get(&pdev->dev, "rx_clk"); + *rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk"); if (IS_ERR(*rx_clk)) - *rx_clk = NULL; + return PTR_ERR(*rx_clk); + + *tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk"); + if (IS_ERR(*tsu_clk)) + return PTR_ERR(*tsu_clk); err = clk_prepare_enable(*pclk); if (err) { - dev_err(&pdev->dev, "failed to enable pclk (%u)\n", err); + dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); return err; } err = clk_prepare_enable(*hclk); if (err) { - dev_err(&pdev->dev, "failed to enable hclk (%u)\n", err); + dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err); goto err_disable_pclk; } err = clk_prepare_enable(*tx_clk); if (err) { - dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); + dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err); goto err_disable_hclk; } err = clk_prepare_enable(*rx_clk); if (err) { - dev_err(&pdev->dev, "failed to enable rx_clk (%u)\n", err); + dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err); goto err_disable_txclk; } + err = clk_prepare_enable(*tsu_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err); + goto err_disable_rxclk; + } + return 0; +err_disable_rxclk: + clk_disable_unprepare(*rx_clk); + err_disable_txclk: clk_disable_unprepare(*tx_clk); @@ -2849,7 +4484,7 @@ static int macb_init(struct platform_device *pdev) struct macb *bp = netdev_priv(dev); struct macb_queue *queue; int err; - u32 val; + u32 val, reg; bp->tx_ring_size = DEFAULT_TX_RING_SIZE; bp->rx_ring_size = DEFAULT_RX_RING_SIZE; @@ -2858,22 +4493,20 @@ static int macb_init(struct platform_device *pdev) * register mapping but we don't want to test the queue index then * compute the corresponding register offset at run time. */ - for (hw_q = 0, q = 0; hw_q < MACB_MAX_QUEUES; ++hw_q) { - if (!(bp->queue_mask & (1 << hw_q))) - continue; - + for (hw_q = 0, q = 0; hw_q < bp->num_queues; ++hw_q) { queue = &bp->queues[q]; queue->bp = bp; + spin_lock_init(&queue->tx_ptr_lock); + netif_napi_add(dev, &queue->napi_rx, macb_rx_poll); + netif_napi_add(dev, &queue->napi_tx, macb_tx_poll); if (hw_q) { queue->ISR = GEM_ISR(hw_q - 1); queue->IER = GEM_IER(hw_q - 1); queue->IDR = GEM_IDR(hw_q - 1); queue->IMR = GEM_IMR(hw_q - 1); queue->TBQP = GEM_TBQP(hw_q - 1); -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - if (bp->hw_dma_cap & HW_DMA_CAP_64B) - queue->TBQPH = GEM_TBQPH(hw_q - 1); -#endif + queue->RBQP = GEM_RBQP(hw_q - 1); + queue->RBQS = GEM_RBQS(hw_q - 1); } else { /* queue0 uses legacy registers */ queue->ISR = MACB_ISR; @@ -2881,12 +4514,13 @@ static int macb_init(struct platform_device *pdev) queue->IDR = MACB_IDR; queue->IMR = MACB_IMR; queue->TBQP = MACB_TBQP; -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - if (bp->hw_dma_cap & HW_DMA_CAP_64B) - queue->TBQPH = MACB_TBQPH; -#endif + queue->RBQP = MACB_RBQP; } + queue->ENST_START_TIME = GEM_ENST_START_TIME(hw_q); + queue->ENST_ON_TIME = GEM_ENST_ON_TIME(hw_q); + queue->ENST_OFF_TIME = GEM_ENST_OFF_TIME(hw_q); + /* get irq: here we use the linux queue index, not the hardware * queue index. the queue irq definitions in the device tree * must remove the optional gaps that could exist in the @@ -2907,18 +4541,15 @@ static int macb_init(struct platform_device *pdev) } dev->netdev_ops = &macb_netdev_ops; - netif_napi_add(dev, &bp->napi, macb_poll, 64); /* setup appropriated routines according to adapter type */ if (macb_is_gem(bp)) { - bp->max_tx_length = GEM_MAX_TX_LEN; bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers; bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers; bp->macbgem_ops.mog_init_rings = gem_init_rings; bp->macbgem_ops.mog_rx = gem_rx; dev->ethtool_ops = &gem_ethtool_ops; } else { - bp->max_tx_length = MACB_MAX_TX_LEN; bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers; bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers; bp->macbgem_ops.mog_init_rings = macb_init_rings; @@ -2926,11 +4557,18 @@ static int macb_init(struct platform_device *pdev) dev->ethtool_ops = &macb_ethtool_ops; } + netdev_sw_irq_coalesce_default_on(dev); + + dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; + /* Set features */ dev->hw_features = NETIF_F_SG; - /* Check LSO capability */ - if (GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6))) + /* Check LSO capability; runtime detection can be overridden by a cap + * flag if the hardware is known to be buggy + */ + if (!(bp->caps & MACB_CAPS_NO_LSO) && + GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6))) dev->hw_features |= MACB_NETIF_LSO; /* Checksum offload is only available on gem with packet buffer */ @@ -2938,20 +4576,48 @@ static int macb_init(struct platform_device *pdev) dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; if (bp->caps & MACB_CAPS_SG_DISABLED) dev->hw_features &= ~NETIF_F_SG; + /* Enable HW_TC if hardware supports QBV */ + if (bp->caps & MACB_CAPS_QBV) + dev->hw_features |= NETIF_F_HW_TC; + dev->features = dev->hw_features; + /* Check RX Flow Filters support. + * Max Rx flows set by availability of screeners & compare regs: + * each 4-tuple define requires 1 T2 screener reg + 3 compare regs + */ + reg = gem_readl(bp, DCFG8); + bp->max_tuples = umin((GEM_BFEXT(SCR2CMP, reg) / 3), + GEM_BFEXT(T2SCR, reg)); + INIT_LIST_HEAD(&bp->rx_fs_list.list); + if (bp->max_tuples > 0) { + /* also needs one ethtype match to check IPv4 */ + if (GEM_BFEXT(SCR2ETH, reg) > 0) { + /* program this reg now */ + reg = 0; + reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg); + gem_writel_n(bp, ETHT, SCRT2_ETHT, reg); + /* Filtering is supported in hw but don't enable it in kernel now */ + dev->hw_features |= NETIF_F_NTUPLE; + /* init Rx flow definitions */ + bp->rx_fs_list.count = 0; + spin_lock_init(&bp->rx_fs_lock); + } else + bp->max_tuples = 0; + } + if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) { val = 0; - if (bp->phy_interface == PHY_INTERFACE_MODE_RGMII) - val = GEM_BIT(RGMII); + if (phy_interface_mode_is_rgmii(bp->phy_interface)) + val = bp->usrio->rgmii; else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) - val = MACB_BIT(RMII); + val = bp->usrio->rmii; else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) - val = MACB_BIT(MII); + val = bp->usrio->mii; if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN) - val |= MACB_BIT(CLKEN); + val |= bp->usrio->refclk; macb_or_gem_writel(bp, USRIO, val); } @@ -2966,44 +4632,85 @@ static int macb_init(struct platform_device *pdev) return 0; } +static const struct macb_usrio_config macb_default_usrio = { + .mii = MACB_BIT(MII), + .rmii = MACB_BIT(RMII), + .rgmii = GEM_BIT(RGMII), + .refclk = MACB_BIT(CLKEN), +}; + #if defined(CONFIG_OF) /* 1518 rounded up */ #define AT91ETHER_MAX_RBUFF_SZ 0x600 /* max number of receive buffers */ #define AT91ETHER_MAX_RX_DESCR 9 -/* Initialize and start the Receiver and Transmit subsystems */ -static int at91ether_start(struct net_device *dev) +static struct sifive_fu540_macb_mgmt *mgmt; + +static int at91ether_alloc_coherent(struct macb *lp) { - struct macb *lp = netdev_priv(dev); - struct macb_dma_desc *desc; - dma_addr_t addr; - u32 ctl; - int i; + struct macb_queue *q = &lp->queues[0]; - lp->rx_ring = dma_alloc_coherent(&lp->pdev->dev, + q->rx_ring = dma_alloc_coherent(&lp->pdev->dev, (AT91ETHER_MAX_RX_DESCR * macb_dma_desc_get_size(lp)), - &lp->rx_ring_dma, GFP_KERNEL); - if (!lp->rx_ring) + &q->rx_ring_dma, GFP_KERNEL); + if (!q->rx_ring) return -ENOMEM; - lp->rx_buffers = dma_alloc_coherent(&lp->pdev->dev, + q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev, AT91ETHER_MAX_RX_DESCR * AT91ETHER_MAX_RBUFF_SZ, - &lp->rx_buffers_dma, GFP_KERNEL); - if (!lp->rx_buffers) { + &q->rx_buffers_dma, GFP_KERNEL); + if (!q->rx_buffers) { dma_free_coherent(&lp->pdev->dev, AT91ETHER_MAX_RX_DESCR * macb_dma_desc_get_size(lp), - lp->rx_ring, lp->rx_ring_dma); - lp->rx_ring = NULL; + q->rx_ring, q->rx_ring_dma); + q->rx_ring = NULL; return -ENOMEM; } - addr = lp->rx_buffers_dma; + return 0; +} + +static void at91ether_free_coherent(struct macb *lp) +{ + struct macb_queue *q = &lp->queues[0]; + + if (q->rx_ring) { + dma_free_coherent(&lp->pdev->dev, + AT91ETHER_MAX_RX_DESCR * + macb_dma_desc_get_size(lp), + q->rx_ring, q->rx_ring_dma); + q->rx_ring = NULL; + } + + if (q->rx_buffers) { + dma_free_coherent(&lp->pdev->dev, + AT91ETHER_MAX_RX_DESCR * + AT91ETHER_MAX_RBUFF_SZ, + q->rx_buffers, q->rx_buffers_dma); + q->rx_buffers = NULL; + } +} + +/* Initialize and start the Receiver and Transmit subsystems */ +static int at91ether_start(struct macb *lp) +{ + struct macb_queue *q = &lp->queues[0]; + struct macb_dma_desc *desc; + dma_addr_t addr; + u32 ctl; + int i, ret; + + ret = at91ether_alloc_coherent(lp); + if (ret) + return ret; + + addr = q->rx_buffers_dma; for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) { - desc = macb_rx_desc(lp, i); + desc = macb_rx_desc(q, i); macb_set_addr(lp, desc, addr); desc->ctrl = 0; addr += AT91ETHER_MAX_RBUFF_SZ; @@ -3013,18 +4720,48 @@ static int at91ether_start(struct net_device *dev) desc->addr |= MACB_BIT(RX_WRAP); /* Reset buffer index */ - lp->rx_tail = 0; + q->rx_tail = 0; /* Program address of descriptor list in Rx Buffer Queue register */ - macb_writel(lp, RBQP, lp->rx_ring_dma); + macb_writel(lp, RBQP, q->rx_ring_dma); /* Enable Receive and Transmit */ ctl = macb_readl(lp, NCR); macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE)); + /* Enable MAC interrupts */ + macb_writel(lp, IER, MACB_BIT(RCOMP) | + MACB_BIT(RXUBR) | + MACB_BIT(ISR_TUND) | + MACB_BIT(ISR_RLE) | + MACB_BIT(TCOMP) | + MACB_BIT(ISR_ROVR) | + MACB_BIT(HRESP)); + return 0; } +static void at91ether_stop(struct macb *lp) +{ + u32 ctl; + + /* Disable MAC interrupts */ + macb_writel(lp, IDR, MACB_BIT(RCOMP) | + MACB_BIT(RXUBR) | + MACB_BIT(ISR_TUND) | + MACB_BIT(ISR_RLE) | + MACB_BIT(TCOMP) | + MACB_BIT(ISR_ROVR) | + MACB_BIT(HRESP)); + + /* Disable Receiver and Transmitter */ + ctl = macb_readl(lp, NCR); + macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE))); + + /* Free resources. */ + at91ether_free_coherent(lp); +} + /* Open the ethernet interface */ static int at91ether_open(struct net_device *dev) { @@ -3032,82 +4769,67 @@ static int at91ether_open(struct net_device *dev) u32 ctl; int ret; + ret = pm_runtime_resume_and_get(&lp->pdev->dev); + if (ret < 0) + return ret; + /* Clear internal statistics */ ctl = macb_readl(lp, NCR); macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT)); macb_set_hwaddr(lp); - ret = at91ether_start(dev); + ret = at91ether_start(lp); if (ret) - return ret; + goto pm_exit; - /* Enable MAC interrupts */ - macb_writel(lp, IER, MACB_BIT(RCOMP) | - MACB_BIT(RXUBR) | - MACB_BIT(ISR_TUND) | - MACB_BIT(ISR_RLE) | - MACB_BIT(TCOMP) | - MACB_BIT(ISR_ROVR) | - MACB_BIT(HRESP)); - - /* schedule a link state check */ - phy_start(dev->phydev); + ret = macb_phylink_connect(lp); + if (ret) + goto stop; netif_start_queue(dev); return 0; + +stop: + at91ether_stop(lp); +pm_exit: + pm_runtime_put_sync(&lp->pdev->dev); + return ret; } /* Close the interface */ static int at91ether_close(struct net_device *dev) { struct macb *lp = netdev_priv(dev); - u32 ctl; - - /* Disable Receiver and Transmitter */ - ctl = macb_readl(lp, NCR); - macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE))); - - /* Disable MAC interrupts */ - macb_writel(lp, IDR, MACB_BIT(RCOMP) | - MACB_BIT(RXUBR) | - MACB_BIT(ISR_TUND) | - MACB_BIT(ISR_RLE) | - MACB_BIT(TCOMP) | - MACB_BIT(ISR_ROVR) | - MACB_BIT(HRESP)); netif_stop_queue(dev); - dma_free_coherent(&lp->pdev->dev, - AT91ETHER_MAX_RX_DESCR * - macb_dma_desc_get_size(lp), - lp->rx_ring, lp->rx_ring_dma); - lp->rx_ring = NULL; + phylink_stop(lp->phylink); + phylink_disconnect_phy(lp->phylink); - dma_free_coherent(&lp->pdev->dev, - AT91ETHER_MAX_RX_DESCR * AT91ETHER_MAX_RBUFF_SZ, - lp->rx_buffers, lp->rx_buffers_dma); - lp->rx_buffers = NULL; + at91ether_stop(lp); - return 0; + return pm_runtime_put(&lp->pdev->dev); } /* Transmit packet */ -static int at91ether_start_xmit(struct sk_buff *skb, struct net_device *dev) +static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, + struct net_device *dev) { struct macb *lp = netdev_priv(dev); if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) { + int desc = 0; + netif_stop_queue(dev); /* Store packet information (to free when Tx completed) */ - lp->skb = skb; - lp->skb_length = skb->len; - lp->skb_physaddr = dma_map_single(NULL, skb->data, skb->len, - DMA_TO_DEVICE); - if (dma_mapping_error(NULL, lp->skb_physaddr)) { + lp->rm9200_txq[desc].skb = skb; + lp->rm9200_txq[desc].size = skb->len; + lp->rm9200_txq[desc].mapping = dma_map_single(&lp->pdev->dev, skb->data, + skb->len, DMA_TO_DEVICE); + if (dma_mapping_error(&lp->pdev->dev, lp->rm9200_txq[desc].mapping)) { dev_kfree_skb_any(skb); dev->stats.tx_dropped++; netdev_err(dev, "%s: DMA mapping error\n", __func__); @@ -3115,7 +4837,7 @@ static int at91ether_start_xmit(struct sk_buff *skb, struct net_device *dev) } /* Set address of the data in the Transmit Address register */ - macb_writel(lp, TAR, lp->skb_physaddr); + macb_writel(lp, TAR, lp->rm9200_txq[desc].mapping); /* Set length of the packet in the Transmit Control register */ macb_writel(lp, TCR, skb->len); @@ -3133,14 +4855,15 @@ static int at91ether_start_xmit(struct sk_buff *skb, struct net_device *dev) static void at91ether_rx(struct net_device *dev) { struct macb *lp = netdev_priv(dev); + struct macb_queue *q = &lp->queues[0]; struct macb_dma_desc *desc; unsigned char *p_recv; struct sk_buff *skb; unsigned int pktlen; - desc = macb_rx_desc(lp, lp->rx_tail); + desc = macb_rx_desc(q, q->rx_tail); while (desc->addr & MACB_BIT(RX_USED)) { - p_recv = lp->rx_buffers + lp->rx_tail * AT91ETHER_MAX_RBUFF_SZ; + p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ; pktlen = MACB_BF(RX_FRMLEN, desc->ctrl); skb = netdev_alloc_skb(dev, pktlen + 2); if (skb) { @@ -3162,12 +4885,12 @@ static void at91ether_rx(struct net_device *dev) desc->addr &= ~MACB_BIT(RX_USED); /* wrap after last buffer */ - if (lp->rx_tail == AT91ETHER_MAX_RX_DESCR - 1) - lp->rx_tail = 0; + if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1) + q->rx_tail = 0; else - lp->rx_tail++; + q->rx_tail++; - desc = macb_rx_desc(lp, lp->rx_tail); + desc = macb_rx_desc(q, q->rx_tail); } } @@ -3177,6 +4900,7 @@ static irqreturn_t at91ether_interrupt(int irq, void *dev_id) struct net_device *dev = dev_id; struct macb *lp = netdev_priv(dev); u32 intstatus, ctl; + unsigned int desc; /* MAC Interrupt Status register indicates what interrupts are pending. * It is automatically cleared once read. @@ -3193,13 +4917,14 @@ static irqreturn_t at91ether_interrupt(int irq, void *dev_id) if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE))) dev->stats.tx_errors++; - if (lp->skb) { - dev_kfree_skb_irq(lp->skb); - lp->skb = NULL; - dma_unmap_single(NULL, lp->skb_physaddr, - lp->skb_length, DMA_TO_DEVICE); + desc = 0; + if (lp->rm9200_txq[desc].skb) { + dev_consume_skb_irq(lp->rm9200_txq[desc].skb); + lp->rm9200_txq[desc].skb = NULL; + dma_unmap_single(&lp->pdev->dev, lp->rm9200_txq[desc].mapping, + lp->rm9200_txq[desc].size, DMA_TO_DEVICE); dev->stats.tx_packets++; - dev->stats.tx_bytes += lp->skb_length; + dev->stats.tx_bytes += lp->rm9200_txq[desc].size; } netif_wake_queue(dev); } @@ -3233,25 +4958,28 @@ static const struct net_device_ops at91ether_netdev_ops = { .ndo_open = at91ether_open, .ndo_stop = at91ether_close, .ndo_start_xmit = at91ether_start_xmit, - .ndo_get_stats = macb_get_stats, + .ndo_get_stats64 = macb_get_stats, .ndo_set_rx_mode = macb_set_rx_mode, .ndo_set_mac_address = eth_mac_addr, - .ndo_do_ioctl = macb_ioctl, + .ndo_eth_ioctl = macb_ioctl, .ndo_validate_addr = eth_validate_addr, #ifdef CONFIG_NET_POLL_CONTROLLER .ndo_poll_controller = at91ether_poll_controller, #endif + .ndo_hwtstamp_set = macb_hwtstamp_set, + .ndo_hwtstamp_get = macb_hwtstamp_get, }; static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk, struct clk **hclk, struct clk **tx_clk, - struct clk **rx_clk) + struct clk **rx_clk, struct clk **tsu_clk) { int err; *hclk = NULL; *tx_clk = NULL; *rx_clk = NULL; + *tsu_clk = NULL; *pclk = devm_clk_get(&pdev->dev, "ether_clk"); if (IS_ERR(*pclk)) @@ -3259,7 +4987,7 @@ static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk, err = clk_prepare_enable(*pclk); if (err) { - dev_err(&pdev->dev, "failed to enable pclk (%u)\n", err); + dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); return err; } @@ -3271,7 +4999,8 @@ static int at91ether_init(struct platform_device *pdev) struct net_device *dev = platform_get_drvdata(pdev); struct macb *bp = netdev_priv(dev); int err; - u32 reg; + + bp->queues[0].bp = bp; dev->netdev_ops = &at91ether_netdev_ops; dev->ethtool_ops = &macb_ethtool_ops; @@ -3283,19 +5012,236 @@ static int at91ether_init(struct platform_device *pdev) macb_writel(bp, NCR, 0); - reg = MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG); - if (bp->phy_interface == PHY_INTERFACE_MODE_RMII) - reg |= MACB_BIT(RM9200_RMII); + macb_writel(bp, NCFGR, MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG)); - macb_writel(bp, NCFGR, reg); + return 0; +} + +static unsigned long fu540_macb_tx_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + return mgmt->rate; +} + +static int fu540_macb_tx_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + if (WARN_ON(req->rate < 2500000)) + req->rate = 2500000; + else if (req->rate == 2500000) + req->rate = 2500000; + else if (WARN_ON(req->rate < 13750000)) + req->rate = 2500000; + else if (WARN_ON(req->rate < 25000000)) + req->rate = 25000000; + else if (req->rate == 25000000) + req->rate = 25000000; + else if (WARN_ON(req->rate < 75000000)) + req->rate = 25000000; + else if (WARN_ON(req->rate < 125000000)) + req->rate = 125000000; + else if (req->rate == 125000000) + req->rate = 125000000; + else if (WARN_ON(req->rate > 125000000)) + req->rate = 125000000; + else + req->rate = 125000000; return 0; } +static int fu540_macb_tx_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_rate_request req; + int ret; + + clk_hw_init_rate_request(hw, &req, rate); + ret = fu540_macb_tx_determine_rate(hw, &req); + if (ret != 0) + return ret; + + if (req.rate != 125000000) + iowrite32(1, mgmt->reg); + else + iowrite32(0, mgmt->reg); + mgmt->rate = rate; + + return 0; +} + +static const struct clk_ops fu540_c000_ops = { + .recalc_rate = fu540_macb_tx_recalc_rate, + .determine_rate = fu540_macb_tx_determine_rate, + .set_rate = fu540_macb_tx_set_rate, +}; + +static int fu540_c000_clk_init(struct platform_device *pdev, struct clk **pclk, + struct clk **hclk, struct clk **tx_clk, + struct clk **rx_clk, struct clk **tsu_clk) +{ + struct clk_init_data init; + int err = 0; + + err = macb_clk_init(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk); + if (err) + return err; + + mgmt = devm_kzalloc(&pdev->dev, sizeof(*mgmt), GFP_KERNEL); + if (!mgmt) { + err = -ENOMEM; + goto err_disable_clks; + } + + init.name = "sifive-gemgxl-mgmt"; + init.ops = &fu540_c000_ops; + init.flags = 0; + init.num_parents = 0; + + mgmt->rate = 0; + mgmt->hw.init = &init; + + *tx_clk = devm_clk_register(&pdev->dev, &mgmt->hw); + if (IS_ERR(*tx_clk)) { + err = PTR_ERR(*tx_clk); + goto err_disable_clks; + } + + err = clk_prepare_enable(*tx_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); + *tx_clk = NULL; + goto err_disable_clks; + } else { + dev_info(&pdev->dev, "Registered clk switch '%s'\n", init.name); + } + + return 0; + +err_disable_clks: + macb_clks_disable(*pclk, *hclk, *tx_clk, *rx_clk, *tsu_clk); + + return err; +} + +static int fu540_c000_init(struct platform_device *pdev) +{ + mgmt->reg = devm_platform_ioremap_resource(pdev, 1); + if (IS_ERR(mgmt->reg)) + return PTR_ERR(mgmt->reg); + + return macb_init(pdev); +} + +static int init_reset_optional(struct platform_device *pdev) +{ + struct net_device *dev = platform_get_drvdata(pdev); + struct macb *bp = netdev_priv(dev); + int ret; + + if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { + /* Ensure PHY device used in SGMII mode is ready */ + bp->phy = devm_phy_optional_get(&pdev->dev, NULL); + + if (IS_ERR(bp->phy)) + return dev_err_probe(&pdev->dev, PTR_ERR(bp->phy), + "failed to get SGMII PHY\n"); + + ret = phy_init(bp->phy); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "failed to init SGMII PHY\n"); + + ret = zynqmp_pm_is_function_supported(PM_IOCTL, IOCTL_SET_GEM_CONFIG); + if (!ret) { + u32 pm_info[2]; + + ret = of_property_read_u32_array(pdev->dev.of_node, "power-domains", + pm_info, ARRAY_SIZE(pm_info)); + if (ret) { + dev_err(&pdev->dev, "Failed to read power management information\n"); + goto err_out_phy_exit; + } + ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_FIXED, 0); + if (ret) + goto err_out_phy_exit; + + ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_SGMII_MODE, 1); + if (ret) + goto err_out_phy_exit; + } + + } + + /* Fully reset controller at hardware level if mapped in device tree */ + ret = device_reset_optional(&pdev->dev); + if (ret) { + phy_exit(bp->phy); + return dev_err_probe(&pdev->dev, ret, "failed to reset controller"); + } + + ret = macb_init(pdev); + +err_out_phy_exit: + if (ret) + phy_exit(bp->phy); + + return ret; +} + +static int eyeq5_init(struct platform_device *pdev) +{ + struct net_device *netdev = platform_get_drvdata(pdev); + struct macb *bp = netdev_priv(netdev); + struct device *dev = &pdev->dev; + int ret; + + bp->phy = devm_phy_get(dev, NULL); + if (IS_ERR(bp->phy)) + return dev_err_probe(dev, PTR_ERR(bp->phy), + "failed to get PHY\n"); + + ret = phy_init(bp->phy); + if (ret) + return dev_err_probe(dev, ret, "failed to init PHY\n"); + + ret = macb_init(pdev); + if (ret) + phy_exit(bp->phy); + return ret; +} + +static const struct macb_usrio_config sama7g5_usrio = { + .mii = 0, + .rmii = 1, + .rgmii = 2, + .refclk = BIT(2), + .hdfctlen = BIT(6), +}; + +static const struct macb_config fu540_c000_config = { + .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | + MACB_CAPS_GEM_HAS_PTP, + .dma_burst_length = 16, + .clk_init = fu540_c000_clk_init, + .init = fu540_c000_init, + .jumbo_max_len = 10240, + .usrio = &macb_default_usrio, +}; + static const struct macb_config at91sam9260_config = { .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, .clk_init = macb_clk_init, .init = macb_init, + .usrio = &macb_default_usrio, +}; + +static const struct macb_config sama5d3macb_config = { + .caps = MACB_CAPS_SG_DISABLED | + MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, + .clk_init = macb_clk_init, + .init = macb_init, + .usrio = &macb_default_usrio, }; static const struct macb_config pc302gem_config = { @@ -3303,22 +5249,34 @@ static const struct macb_config pc302gem_config = { .dma_burst_length = 16, .clk_init = macb_clk_init, .init = macb_init, + .usrio = &macb_default_usrio, }; static const struct macb_config sama5d2_config = { - .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, + .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, .dma_burst_length = 16, .clk_init = macb_clk_init, .init = macb_init, + .jumbo_max_len = 10240, + .usrio = &macb_default_usrio, +}; + +static const struct macb_config sama5d29_config = { + .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_GEM_HAS_PTP, + .dma_burst_length = 16, + .clk_init = macb_clk_init, + .init = macb_init, + .usrio = &macb_default_usrio, }; static const struct macb_config sama5d3_config = { - .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE - | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, + .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE | + MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, .dma_burst_length = 16, .clk_init = macb_clk_init, .init = macb_init, .jumbo_max_len = 10240, + .usrio = &macb_default_usrio, }; static const struct macb_config sama5d4_config = { @@ -3326,50 +5284,134 @@ static const struct macb_config sama5d4_config = { .dma_burst_length = 4, .clk_init = macb_clk_init, .init = macb_init, + .usrio = &macb_default_usrio, }; static const struct macb_config emac_config = { + .caps = MACB_CAPS_NEEDS_RSTONUBR | MACB_CAPS_MACB_IS_EMAC, .clk_init = at91ether_clk_init, .init = at91ether_init, + .usrio = &macb_default_usrio, }; static const struct macb_config np4_config = { .caps = MACB_CAPS_USRIO_DISABLED, .clk_init = macb_clk_init, .init = macb_init, + .usrio = &macb_default_usrio, }; static const struct macb_config zynqmp_config = { .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | - MACB_CAPS_JUMBO | - MACB_CAPS_GEM_HAS_PTP, + MACB_CAPS_JUMBO | + MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH, .dma_burst_length = 16, .clk_init = macb_clk_init, - .init = macb_init, + .init = init_reset_optional, .jumbo_max_len = 10240, + .usrio = &macb_default_usrio, }; static const struct macb_config zynq_config = { - .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF, + .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF | + MACB_CAPS_NEEDS_RSTONUBR, .dma_burst_length = 16, .clk_init = macb_clk_init, .init = macb_init, + .usrio = &macb_default_usrio, +}; + +static const struct macb_config mpfs_config = { + .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | + MACB_CAPS_JUMBO | + MACB_CAPS_GEM_HAS_PTP, + .dma_burst_length = 16, + .clk_init = macb_clk_init, + .init = init_reset_optional, + .usrio = &macb_default_usrio, + .max_tx_length = 4040, /* Cadence Erratum 1686 */ + .jumbo_max_len = 4040, +}; + +static const struct macb_config sama7g5_gem_config = { + .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_CLK_HW_CHG | + MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | + MACB_CAPS_MIIONRGMII | MACB_CAPS_GEM_HAS_PTP, + .dma_burst_length = 16, + .clk_init = macb_clk_init, + .init = macb_init, + .usrio = &sama7g5_usrio, +}; + +static const struct macb_config sama7g5_emac_config = { + .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | + MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_MIIONRGMII | + MACB_CAPS_GEM_HAS_PTP, + .dma_burst_length = 16, + .clk_init = macb_clk_init, + .init = macb_init, + .usrio = &sama7g5_usrio, +}; + +static const struct macb_config versal_config = { + .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | + MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH | + MACB_CAPS_NEED_TSUCLK | MACB_CAPS_QUEUE_DISABLE | + MACB_CAPS_QBV, + .dma_burst_length = 16, + .clk_init = macb_clk_init, + .init = init_reset_optional, + .jumbo_max_len = 10240, + .usrio = &macb_default_usrio, +}; + +static const struct macb_config eyeq5_config = { + .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | + MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_QUEUE_DISABLE | + MACB_CAPS_NO_LSO, + .dma_burst_length = 16, + .clk_init = macb_clk_init, + .init = eyeq5_init, + .jumbo_max_len = 10240, + .usrio = &macb_default_usrio, +}; + +static const struct macb_config raspberrypi_rp1_config = { + .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_CLK_HW_CHG | + MACB_CAPS_JUMBO | + MACB_CAPS_GEM_HAS_PTP, + .dma_burst_length = 16, + .clk_init = macb_clk_init, + .init = macb_init, + .usrio = &macb_default_usrio, + .jumbo_max_len = 10240, }; static const struct of_device_id macb_dt_ids[] = { - { .compatible = "cdns,at32ap7000-macb" }, { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, { .compatible = "cdns,macb" }, { .compatible = "cdns,np4-macb", .data = &np4_config }, { .compatible = "cdns,pc302-gem", .data = &pc302gem_config }, { .compatible = "cdns,gem", .data = &pc302gem_config }, + { .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config }, { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config }, + { .compatible = "atmel,sama5d29-gem", .data = &sama5d29_config }, { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config }, + { .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config }, { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config }, { .compatible = "cdns,at91rm9200-emac", .data = &emac_config }, { .compatible = "cdns,emac", .data = &emac_config }, - { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, - { .compatible = "cdns,zynq-gem", .data = &zynq_config }, + { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, /* deprecated */ + { .compatible = "cdns,zynq-gem", .data = &zynq_config }, /* deprecated */ + { .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config }, + { .compatible = "microchip,mpfs-macb", .data = &mpfs_config }, + { .compatible = "microchip,sama7g5-gem", .data = &sama7g5_gem_config }, + { .compatible = "microchip,sama7g5-emac", .data = &sama7g5_emac_config }, + { .compatible = "mobileye,eyeq5-gem", .data = &eyeq5_config }, + { .compatible = "raspberrypi,rp1-gem", .data = &raspberrypi_rp1_config }, + { .compatible = "xlnx,zynqmp-gem", .data = &zynqmp_config}, + { .compatible = "xlnx,zynq-gem", .data = &zynq_config }, + { .compatible = "xlnx,versal-gem", .data = &versal_config}, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, macb_dt_ids); @@ -3377,37 +5419,32 @@ MODULE_DEVICE_TABLE(of, macb_dt_ids); static const struct macb_config default_gem_config = { .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | - MACB_CAPS_JUMBO | - MACB_CAPS_GEM_HAS_PTP, + MACB_CAPS_JUMBO | + MACB_CAPS_GEM_HAS_PTP, .dma_burst_length = 16, .clk_init = macb_clk_init, .init = macb_init, + .usrio = &macb_default_usrio, .jumbo_max_len = 10240, }; static int macb_probe(struct platform_device *pdev) { const struct macb_config *macb_config = &default_gem_config; - int (*clk_init)(struct platform_device *, struct clk **, - struct clk **, struct clk **, struct clk **) - = macb_config->clk_init; - int (*init)(struct platform_device *) = macb_config->init; struct device_node *np = pdev->dev.of_node; - struct device_node *phy_node; struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL; - unsigned int queue_mask, num_queues; - struct macb_platform_data *pdata; - bool native_io; - struct phy_device *phydev; + struct clk *tsu_clk = NULL; + phy_interface_t interface; struct net_device *dev; struct resource *regs; + u32 wtrmrk_rst_val; void __iomem *mem; - const char *mac; struct macb *bp; - int err; + int num_queues; + bool native_io; + int err, val; - regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); - mem = devm_ioremap_resource(&pdev->dev, regs); + mem = devm_platform_get_and_ioremap_resource(pdev, 0, ®s); if (IS_ERR(mem)) return PTR_ERR(mem); @@ -3415,20 +5452,27 @@ static int macb_probe(struct platform_device *pdev) const struct of_device_id *match; match = of_match_node(macb_dt_ids, np); - if (match && match->data) { + if (match && match->data) macb_config = match->data; - clk_init = macb_config->clk_init; - init = macb_config->init; - } } - err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk); + err = macb_config->clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk); if (err) return err; + pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT); + pm_runtime_use_autosuspend(&pdev->dev); + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); native_io = hw_is_native_io(mem); - macb_probe_queues(mem, native_io, &queue_mask, &num_queues); + num_queues = macb_probe_queues(&pdev->dev, mem, native_io); + if (num_queues < 0) { + err = num_queues; + goto err_disable_clocks; + } + dev = alloc_etherdev_mq(sizeof(*bp), num_queues); if (!dev) { err = -ENOMEM; @@ -3452,30 +5496,59 @@ static int macb_probe(struct platform_device *pdev) bp->macb_reg_writel = hw_writel; } bp->num_queues = num_queues; - bp->queue_mask = queue_mask; - if (macb_config) - bp->dma_burst_length = macb_config->dma_burst_length; + bp->dma_burst_length = macb_config->dma_burst_length; bp->pclk = pclk; bp->hclk = hclk; bp->tx_clk = tx_clk; bp->rx_clk = rx_clk; - if (macb_config) - bp->jumbo_max_len = macb_config->jumbo_max_len; + bp->tsu_clk = tsu_clk; + bp->jumbo_max_len = macb_config->jumbo_max_len; + + if (!hw_is_gem(bp->regs, bp->native_io)) + bp->max_tx_length = MACB_MAX_TX_LEN; + else if (macb_config->max_tx_length) + bp->max_tx_length = macb_config->max_tx_length; + else + bp->max_tx_length = GEM_MAX_TX_LEN; bp->wol = 0; - if (of_get_property(np, "magic-packet", NULL)) - bp->wol |= MACB_WOL_HAS_MAGIC_PACKET; - device_init_wakeup(&pdev->dev, bp->wol & MACB_WOL_HAS_MAGIC_PACKET); + device_set_wakeup_capable(&pdev->dev, 1); + + bp->usrio = macb_config->usrio; + /* By default we set to partial store and forward mode for zynqmp. + * Disable if not set in devicetree. + */ + if (GEM_BFEXT(PBUF_CUTTHRU, gem_readl(bp, DCFG6))) { + err = of_property_read_u32(bp->pdev->dev.of_node, + "cdns,rx-watermark", + &bp->rx_watermark); + + if (!err) { + /* Disable partial store and forward in case of error or + * invalid watermark value + */ + wtrmrk_rst_val = (1 << (GEM_BFEXT(RX_PBUF_ADDR, gem_readl(bp, DCFG2)))) - 1; + if (bp->rx_watermark > wtrmrk_rst_val || !bp->rx_watermark) { + dev_info(&bp->pdev->dev, "Invalid watermark value\n"); + bp->rx_watermark = 0; + } + } + } spin_lock_init(&bp->lock); + spin_lock_init(&bp->stats_lock); /* setup capabilities */ macb_configure_caps(bp, macb_config); #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) { - dma_set_mask(&pdev->dev, DMA_BIT_MASK(44)); - bp->hw_dma_cap |= HW_DMA_CAP_64B; + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44)); + if (err) { + dev_err(&pdev->dev, "failed to set DMA mask\n"); + goto err_out_free_netdev; + } + bp->caps |= MACB_CAPS_DMA_64B; } #endif platform_set_drvdata(pdev, dev); @@ -3486,52 +5559,50 @@ static int macb_probe(struct platform_device *pdev) goto err_out_free_netdev; } - /* MTU range: 68 - 1500 or 10240 */ + /* MTU range: 68 - 1518 or 10240 */ dev->min_mtu = GEM_MTU_MIN_SIZE; - if (bp->caps & MACB_CAPS_JUMBO) - dev->max_mtu = gem_readl(bp, JML) - ETH_HLEN - ETH_FCS_LEN; - else - dev->max_mtu = ETH_DATA_LEN; - - mac = of_get_mac_address(np); - if (mac) - ether_addr_copy(bp->dev->dev_addr, mac); + if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) + dev->max_mtu = bp->jumbo_max_len - ETH_HLEN - ETH_FCS_LEN; else - macb_get_hwaddr(bp); + dev->max_mtu = 1536 - ETH_HLEN - ETH_FCS_LEN; + + if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) { + val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10)); + if (val) + bp->rx_bd_rd_prefetch = (2 << (val - 1)) * + macb_dma_desc_get_size(bp); + + val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10)); + if (val) + bp->tx_bd_rd_prefetch = (2 << (val - 1)) * + macb_dma_desc_get_size(bp); + } - /* Power up the PHY if there is a GPIO reset */ - phy_node = of_get_next_available_child(np, NULL); - if (phy_node) { - int gpio = of_get_named_gpio(phy_node, "reset-gpios", 0); + bp->rx_intr_mask = MACB_RX_INT_FLAGS; + if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR) + bp->rx_intr_mask |= MACB_BIT(RXUBR); - if (gpio_is_valid(gpio)) { - bp->reset_gpio = gpio_to_desc(gpio); - gpiod_direction_output(bp->reset_gpio, 1); - } - } - of_node_put(phy_node); + err = of_get_ethdev_address(np, bp->dev); + if (err == -EPROBE_DEFER) + goto err_out_free_netdev; + else if (err) + macb_get_hwaddr(bp); - err = of_get_phy_mode(np); - if (err < 0) { - pdata = dev_get_platdata(&pdev->dev); - if (pdata && pdata->is_rmii) - bp->phy_interface = PHY_INTERFACE_MODE_RMII; - else - bp->phy_interface = PHY_INTERFACE_MODE_MII; - } else { - bp->phy_interface = err; - } + err = of_get_phy_mode(np, &interface); + if (err) + /* not found in DT, MII by default */ + bp->phy_interface = PHY_INTERFACE_MODE_MII; + else + bp->phy_interface = interface; /* IP specific init */ - err = init(pdev); + err = macb_config->init(pdev); if (err) goto err_out_free_netdev; err = macb_mii_init(bp); if (err) - goto err_out_free_netdev; - - phydev = dev->phydev; + goto err_out_phy_exit; netif_carrier_off(dev); @@ -3541,36 +5612,36 @@ static int macb_probe(struct platform_device *pdev) goto err_out_unregister_mdio; } - phy_attached_info(phydev); + INIT_WORK(&bp->hresp_err_bh_work, macb_hresp_error_task); netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID), dev->base_addr, dev->irq, dev->dev_addr); + pm_runtime_put_autosuspend(&bp->pdev->dev); + return 0; err_out_unregister_mdio: - phy_disconnect(dev->phydev); mdiobus_unregister(bp->mii_bus); mdiobus_free(bp->mii_bus); - /* Shutdown the PHY if there is a GPIO reset */ - if (bp->reset_gpio) - gpiod_set_value(bp->reset_gpio, 0); +err_out_phy_exit: + phy_exit(bp->phy); err_out_free_netdev: free_netdev(dev); err_disable_clocks: - clk_disable_unprepare(tx_clk); - clk_disable_unprepare(hclk); - clk_disable_unprepare(pclk); - clk_disable_unprepare(rx_clk); + macb_clks_disable(pclk, hclk, tx_clk, rx_clk, tsu_clk); + pm_runtime_disable(&pdev->dev); + pm_runtime_set_suspended(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); return err; } -static int macb_remove(struct platform_device *pdev) +static void macb_remove(struct platform_device *pdev) { struct net_device *dev; struct macb *bp; @@ -3579,74 +5650,285 @@ static int macb_remove(struct platform_device *pdev) if (dev) { bp = netdev_priv(dev); - if (dev->phydev) - phy_disconnect(dev->phydev); + unregister_netdev(dev); + phy_exit(bp->phy); mdiobus_unregister(bp->mii_bus); - dev->phydev = NULL; mdiobus_free(bp->mii_bus); - /* Shutdown the PHY if there is a GPIO reset */ - if (bp->reset_gpio) - gpiod_set_value(bp->reset_gpio, 0); - - unregister_netdev(dev); - clk_disable_unprepare(bp->tx_clk); - clk_disable_unprepare(bp->hclk); - clk_disable_unprepare(bp->pclk); - clk_disable_unprepare(bp->rx_clk); - of_node_put(bp->phy_node); + device_set_wakeup_enable(&bp->pdev->dev, 0); + cancel_work_sync(&bp->hresp_err_bh_work); + pm_runtime_disable(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); + pm_runtime_set_suspended(&pdev->dev); + phylink_destroy(bp->phylink); free_netdev(dev); } - - return 0; } static int __maybe_unused macb_suspend(struct device *dev) { - struct platform_device *pdev = to_platform_device(dev); - struct net_device *netdev = platform_get_drvdata(pdev); + struct net_device *netdev = dev_get_drvdata(dev); struct macb *bp = netdev_priv(netdev); + struct in_ifaddr *ifa = NULL; + struct macb_queue *queue; + struct in_device *idev; + unsigned long flags; + unsigned int q; + int err; + u32 tmp; - netif_carrier_off(netdev); - netif_device_detach(netdev); + if (!device_may_wakeup(&bp->dev->dev)) + phy_exit(bp->phy); + + if (!netif_running(netdev)) + return 0; if (bp->wol & MACB_WOL_ENABLED) { - macb_writel(bp, IER, MACB_BIT(WOL)); - macb_writel(bp, WOL, MACB_BIT(MAG)); + /* Check for IP address in WOL ARP mode */ + idev = __in_dev_get_rcu(bp->dev); + if (idev) + ifa = rcu_dereference(idev->ifa_list); + if ((bp->wolopts & WAKE_ARP) && !ifa) { + netdev_err(netdev, "IP address not assigned as required by WoL walk ARP\n"); + return -EOPNOTSUPP; + } + spin_lock_irqsave(&bp->lock, flags); + + /* Disable Tx and Rx engines before disabling the queues, + * this is mandatory as per the IP spec sheet + */ + tmp = macb_readl(bp, NCR); + macb_writel(bp, NCR, tmp & ~(MACB_BIT(TE) | MACB_BIT(RE))); +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + if (!(bp->caps & MACB_CAPS_QUEUE_DISABLE)) + macb_writel(bp, RBQPH, + upper_32_bits(bp->rx_ring_tieoff_dma)); +#endif + for (q = 0, queue = bp->queues; q < bp->num_queues; + ++q, ++queue) { + /* Disable RX queues */ + if (bp->caps & MACB_CAPS_QUEUE_DISABLE) { + queue_writel(queue, RBQP, MACB_BIT(QUEUE_DISABLE)); + } else { + /* Tie off RX queues */ + queue_writel(queue, RBQP, + lower_32_bits(bp->rx_ring_tieoff_dma)); + } + /* Disable all interrupts */ + queue_writel(queue, IDR, -1); + queue_readl(queue, ISR); + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, -1); + } + /* Enable Receive engine */ + macb_writel(bp, NCR, tmp | MACB_BIT(RE)); + /* Flush all status bits */ + macb_writel(bp, TSR, -1); + macb_writel(bp, RSR, -1); + + tmp = (bp->wolopts & WAKE_MAGIC) ? MACB_BIT(MAG) : 0; + if (bp->wolopts & WAKE_ARP) { + tmp |= MACB_BIT(ARP); + /* write IP address into register */ + tmp |= MACB_BFEXT(IP, be32_to_cpu(ifa->ifa_local)); + } + + /* Change interrupt handler and + * Enable WoL IRQ on queue 0 + */ + devm_free_irq(dev, bp->queues[0].irq, bp->queues); + if (macb_is_gem(bp)) { + err = devm_request_irq(dev, bp->queues[0].irq, gem_wol_interrupt, + IRQF_SHARED, netdev->name, bp->queues); + if (err) { + dev_err(dev, + "Unable to request IRQ %d (error %d)\n", + bp->queues[0].irq, err); + spin_unlock_irqrestore(&bp->lock, flags); + return err; + } + queue_writel(bp->queues, IER, GEM_BIT(WOL)); + gem_writel(bp, WOL, tmp); + } else { + err = devm_request_irq(dev, bp->queues[0].irq, macb_wol_interrupt, + IRQF_SHARED, netdev->name, bp->queues); + if (err) { + dev_err(dev, + "Unable to request IRQ %d (error %d)\n", + bp->queues[0].irq, err); + spin_unlock_irqrestore(&bp->lock, flags); + return err; + } + queue_writel(bp->queues, IER, MACB_BIT(WOL)); + macb_writel(bp, WOL, tmp); + } + spin_unlock_irqrestore(&bp->lock, flags); + enable_irq_wake(bp->queues[0].irq); - } else { - clk_disable_unprepare(bp->tx_clk); - clk_disable_unprepare(bp->hclk); - clk_disable_unprepare(bp->pclk); - clk_disable_unprepare(bp->rx_clk); } + netif_device_detach(netdev); + for (q = 0, queue = bp->queues; q < bp->num_queues; + ++q, ++queue) { + napi_disable(&queue->napi_rx); + napi_disable(&queue->napi_tx); + } + + if (!(bp->wol & MACB_WOL_ENABLED)) { + rtnl_lock(); + phylink_stop(bp->phylink); + rtnl_unlock(); + spin_lock_irqsave(&bp->lock, flags); + macb_reset_hw(bp); + spin_unlock_irqrestore(&bp->lock, flags); + } + + if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) + bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO); + + if (netdev->hw_features & NETIF_F_NTUPLE) + bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT); + + if (bp->ptp_info) + bp->ptp_info->ptp_remove(netdev); + if (!device_may_wakeup(dev)) + pm_runtime_force_suspend(dev); + return 0; } static int __maybe_unused macb_resume(struct device *dev) { - struct platform_device *pdev = to_platform_device(dev); - struct net_device *netdev = platform_get_drvdata(pdev); + struct net_device *netdev = dev_get_drvdata(dev); struct macb *bp = netdev_priv(netdev); + struct macb_queue *queue; + unsigned long flags; + unsigned int q; + int err; + + if (!device_may_wakeup(&bp->dev->dev)) + phy_init(bp->phy); + + if (!netif_running(netdev)) + return 0; + + if (!device_may_wakeup(dev)) + pm_runtime_force_resume(dev); if (bp->wol & MACB_WOL_ENABLED) { - macb_writel(bp, IDR, MACB_BIT(WOL)); - macb_writel(bp, WOL, 0); + spin_lock_irqsave(&bp->lock, flags); + /* Disable WoL */ + if (macb_is_gem(bp)) { + queue_writel(bp->queues, IDR, GEM_BIT(WOL)); + gem_writel(bp, WOL, 0); + } else { + queue_writel(bp->queues, IDR, MACB_BIT(WOL)); + macb_writel(bp, WOL, 0); + } + /* Clear ISR on queue 0 */ + queue_readl(bp->queues, ISR); + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(bp->queues, ISR, -1); + /* Replace interrupt handler on queue 0 */ + devm_free_irq(dev, bp->queues[0].irq, bp->queues); + err = devm_request_irq(dev, bp->queues[0].irq, macb_interrupt, + IRQF_SHARED, netdev->name, bp->queues); + if (err) { + dev_err(dev, + "Unable to request IRQ %d (error %d)\n", + bp->queues[0].irq, err); + spin_unlock_irqrestore(&bp->lock, flags); + return err; + } + spin_unlock_irqrestore(&bp->lock, flags); + disable_irq_wake(bp->queues[0].irq); - } else { + + /* Now make sure we disable phy before moving + * to common restore path + */ + rtnl_lock(); + phylink_stop(bp->phylink); + rtnl_unlock(); + } + + for (q = 0, queue = bp->queues; q < bp->num_queues; + ++q, ++queue) { + napi_enable(&queue->napi_rx); + napi_enable(&queue->napi_tx); + } + + if (netdev->hw_features & NETIF_F_NTUPLE) + gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2); + + if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) + macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio); + + macb_writel(bp, NCR, MACB_BIT(MPE)); + macb_init_hw(bp); + macb_set_rx_mode(netdev); + macb_restore_features(bp); + rtnl_lock(); + + phylink_start(bp->phylink); + rtnl_unlock(); + + netif_device_attach(netdev); + if (bp->ptp_info) + bp->ptp_info->ptp_init(netdev); + + return 0; +} + +static int __maybe_unused macb_runtime_suspend(struct device *dev) +{ + struct net_device *netdev = dev_get_drvdata(dev); + struct macb *bp = netdev_priv(netdev); + + if (!(device_may_wakeup(dev))) + macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk, bp->rx_clk, bp->tsu_clk); + else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) + macb_clks_disable(NULL, NULL, NULL, NULL, bp->tsu_clk); + + return 0; +} + +static int __maybe_unused macb_runtime_resume(struct device *dev) +{ + struct net_device *netdev = dev_get_drvdata(dev); + struct macb *bp = netdev_priv(netdev); + + if (!(device_may_wakeup(dev))) { clk_prepare_enable(bp->pclk); clk_prepare_enable(bp->hclk); clk_prepare_enable(bp->tx_clk); clk_prepare_enable(bp->rx_clk); + clk_prepare_enable(bp->tsu_clk); + } else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) { + clk_prepare_enable(bp->tsu_clk); } - netif_device_attach(netdev); - return 0; } -static SIMPLE_DEV_PM_OPS(macb_pm_ops, macb_suspend, macb_resume); +static void macb_shutdown(struct platform_device *pdev) +{ + struct net_device *netdev = platform_get_drvdata(pdev); + + rtnl_lock(); + + if (netif_running(netdev)) + dev_close(netdev); + + netif_device_detach(netdev); + + rtnl_unlock(); +} + +static const struct dev_pm_ops macb_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(macb_suspend, macb_resume) + SET_RUNTIME_PM_OPS(macb_runtime_suspend, macb_runtime_resume, NULL) +}; static struct platform_driver macb_driver = { .probe = macb_probe, @@ -3656,6 +5938,7 @@ static struct platform_driver macb_driver = { .of_match_table = of_match_ptr(macb_dt_ids), .pm = &macb_pm_ops, }, + .shutdown = macb_shutdown, }; module_platform_driver(macb_driver); |