diff options
author | David S. Miller <davem@davemloft.net> | 2023-12-23 01:01:20 +0000 |
---|---|---|
committer | David S. Miller <davem@davemloft.net> | 2023-12-23 01:01:20 +0000 |
commit | d11db8ad388281cbf7aa3e6834253d844210f1e0 (patch) | |
tree | 8349f515f59cdef643b9870681de548ad7327ac5 | |
parent | 2437c0f5147b82b92075af6085b99051753f4a3c (diff) | |
parent | e4918f9d48823ea184534848d32a61a4fc02753f (diff) |
Merge branch 'am65-cpsw-preemption-coalescing'
Roger Quadros says:
====================
net: ethernet: am65-cpsw: Add mqprio, frame preemption & coalescing
This series adds mqprio qdisc offload in channel mode,
Frame Preemption MAC merge support and RX/TX coalesing
for AM65 CPSW driver.
In v11 following changes were made
- Fix patch "net: ethernet: ti: am65-cpsw: add mqprio qdisc offload in channel mode"
by including units.h
Changelog information in each patch file.
====================
Signed-off-by: David S. Miller <davem@davemloft.net>
-rw-r--r-- | drivers/net/ethernet/ti/Kconfig | 14 | ||||
-rw-r--r-- | drivers/net/ethernet/ti/Makefile | 3 | ||||
-rw-r--r-- | drivers/net/ethernet/ti/am65-cpsw-ethtool.c | 246 | ||||
-rw-r--r-- | drivers/net/ethernet/ti/am65-cpsw-nuss.c | 64 | ||||
-rw-r--r-- | drivers/net/ethernet/ti/am65-cpsw-nuss.h | 9 | ||||
-rw-r--r-- | drivers/net/ethernet/ti/am65-cpsw-qos.c | 708 | ||||
-rw-r--r-- | drivers/net/ethernet/ti/am65-cpsw-qos.h | 186 | ||||
-rwxr-xr-x | tools/testing/selftests/net/forwarding/ethtool_mm.sh | 48 | ||||
-rwxr-xr-x | tools/testing/selftests/net/forwarding/lib.sh | 9 |
9 files changed, 1100 insertions, 187 deletions
diff --git a/drivers/net/ethernet/ti/Kconfig b/drivers/net/ethernet/ti/Kconfig index e60b557d59b9..be01450c20dc 100644 --- a/drivers/net/ethernet/ti/Kconfig +++ b/drivers/net/ethernet/ti/Kconfig @@ -134,14 +134,16 @@ config TI_K3_AM65_CPTS protocol, Ethernet Enhanced Scheduled Traffic Operations (CPTS_ESTFn) and PCIe Subsystem Precision Time Measurement (PTM). -config TI_AM65_CPSW_TAS - bool "Enable TAS offload in AM65 CPSW" +config TI_AM65_CPSW_QOS + bool "Enable QoS offload features in AM65 CPSW" depends on TI_K3_AM65_CPSW_NUSS && NET_SCH_TAPRIO && TI_K3_AM65_CPTS help - Say y here to support Time Aware Shaper(TAS) offload in AM65 CPSW. - AM65 CPSW hardware supports Enhanced Scheduled Traffic (EST) - defined in IEEE 802.1Q 2018. The EST scheduler runs on CPTS and the - TAS/EST schedule is updated in the Fetch RAM memory of the CPSW. + This option enables QoS offload features in AM65 CPSW like + Time Aware Shaper (TAS) / Enhanced Scheduled Traffic (EST), + MQPRIO qdisc offload and Frame-Preemption MAC Merge / Interspersing + Express Traffic (IET). + The EST scheduler runs on CPTS and the TAS/EST schedule is + updated in the Fetch RAM memory of the CPSW. config TI_KEYSTONE_NETCP tristate "TI Keystone NETCP Core Support" diff --git a/drivers/net/ethernet/ti/Makefile b/drivers/net/ethernet/ti/Makefile index 27de1d697134..d8590304f3df 100644 --- a/drivers/net/ethernet/ti/Makefile +++ b/drivers/net/ethernet/ti/Makefile @@ -26,7 +26,8 @@ keystone_netcp_ethss-y := netcp_ethss.o netcp_sgmii.o netcp_xgbepcsr.o cpsw_ale. obj-$(CONFIG_TI_K3_CPPI_DESC_POOL) += k3-cppi-desc-pool.o obj-$(CONFIG_TI_K3_AM65_CPSW_NUSS) += ti-am65-cpsw-nuss.o -ti-am65-cpsw-nuss-y := am65-cpsw-nuss.o cpsw_sl.o am65-cpsw-ethtool.o cpsw_ale.o am65-cpsw-qos.o +ti-am65-cpsw-nuss-y := am65-cpsw-nuss.o cpsw_sl.o am65-cpsw-ethtool.o cpsw_ale.o +ti-am65-cpsw-nuss-$(CONFIG_TI_AM65_CPSW_QOS) += am65-cpsw-qos.o ti-am65-cpsw-nuss-$(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV) += am65-cpsw-switchdev.o obj-$(CONFIG_TI_K3_AM65_CPTS) += am65-cpts.o diff --git a/drivers/net/ethernet/ti/am65-cpsw-ethtool.c b/drivers/net/ethernet/ti/am65-cpsw-ethtool.c index b9e1d568604b..35fceba01ea4 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-ethtool.c +++ b/drivers/net/ethernet/ti/am65-cpsw-ethtool.c @@ -11,6 +11,7 @@ #include <linux/pm_runtime.h> #include "am65-cpsw-nuss.h" +#include "am65-cpsw-qos.h" #include "cpsw_ale.h" #include "am65-cpts.h" @@ -670,6 +671,9 @@ static void am65_cpsw_get_eth_mac_stats(struct net_device *ndev, stats = port->stat_base; + if (s->src != ETHTOOL_MAC_STATS_SRC_AGGREGATE) + return; + s->FramesTransmittedOK = readl_relaxed(&stats->tx_good_frames); s->SingleCollisionFrames = readl_relaxed(&stats->tx_single_coll_frames); s->MultipleCollisionFrames = readl_relaxed(&stats->tx_mult_coll_frames); @@ -740,6 +744,240 @@ static int am65_cpsw_set_ethtool_priv_flags(struct net_device *ndev, u32 flags) return 0; } +static void am65_cpsw_port_iet_rx_enable(struct am65_cpsw_port *port, bool enable) +{ + u32 val; + + val = readl(port->port_base + AM65_CPSW_PN_REG_CTL); + if (enable) + val |= AM65_CPSW_PN_CTL_IET_PORT_EN; + else + val &= ~AM65_CPSW_PN_CTL_IET_PORT_EN; + + writel(val, port->port_base + AM65_CPSW_PN_REG_CTL); + am65_cpsw_iet_common_enable(port->common); +} + +static void am65_cpsw_port_iet_tx_enable(struct am65_cpsw_port *port, bool enable) +{ + u32 val; + + val = readl(port->port_base + AM65_CPSW_PN_REG_IET_CTRL); + if (enable) + val |= AM65_CPSW_PN_IET_MAC_PENABLE; + else + val &= ~AM65_CPSW_PN_IET_MAC_PENABLE; + + writel(val, port->port_base + AM65_CPSW_PN_REG_IET_CTRL); +} + +static int am65_cpsw_get_mm(struct net_device *ndev, struct ethtool_mm_state *state) +{ + struct am65_cpsw_port *port = am65_ndev_to_port(ndev); + struct am65_cpsw_ndev_priv *priv = netdev_priv(ndev); + u32 port_ctrl, iet_ctrl, iet_status; + u32 add_frag_size; + + if (!IS_ENABLED(CONFIG_TI_AM65_CPSW_QOS)) + return -EOPNOTSUPP; + + mutex_lock(&priv->mm_lock); + + iet_ctrl = readl(port->port_base + AM65_CPSW_PN_REG_IET_CTRL); + port_ctrl = readl(port->port_base + AM65_CPSW_PN_REG_CTL); + + state->tx_enabled = !!(iet_ctrl & AM65_CPSW_PN_IET_MAC_PENABLE); + state->pmac_enabled = !!(port_ctrl & AM65_CPSW_PN_CTL_IET_PORT_EN); + + iet_status = readl(port->port_base + AM65_CPSW_PN_REG_IET_STATUS); + + if (iet_ctrl & AM65_CPSW_PN_IET_MAC_DISABLEVERIFY) + state->verify_status = ETHTOOL_MM_VERIFY_STATUS_DISABLED; + else if (iet_status & AM65_CPSW_PN_MAC_VERIFIED) + state->verify_status = ETHTOOL_MM_VERIFY_STATUS_SUCCEEDED; + else if (iet_status & AM65_CPSW_PN_MAC_VERIFY_FAIL) + state->verify_status = ETHTOOL_MM_VERIFY_STATUS_FAILED; + else + state->verify_status = ETHTOOL_MM_VERIFY_STATUS_UNKNOWN; + + add_frag_size = AM65_CPSW_PN_IET_MAC_GET_ADDFRAGSIZE(iet_ctrl); + state->tx_min_frag_size = ethtool_mm_frag_size_add_to_min(add_frag_size); + + /* Errata i2208: RX min fragment size cannot be less than 124 */ + state->rx_min_frag_size = 124; + + /* FPE active if common tx_enabled and verification success or disabled (forced) */ + state->tx_active = state->tx_enabled && + (state->verify_status == ETHTOOL_MM_VERIFY_STATUS_SUCCEEDED || + state->verify_status == ETHTOOL_MM_VERIFY_STATUS_DISABLED); + state->verify_enabled = !(iet_ctrl & AM65_CPSW_PN_IET_MAC_DISABLEVERIFY); + + state->verify_time = port->qos.iet.verify_time_ms; + + /* 802.3-2018 clause 30.14.1.6, says that the aMACMergeVerifyTime + * variable has a range between 1 and 128 ms inclusive. Limit to that. + */ + state->max_verify_time = 128; + + mutex_unlock(&priv->mm_lock); + + return 0; +} + +static int am65_cpsw_set_mm(struct net_device *ndev, struct ethtool_mm_cfg *cfg, + struct netlink_ext_ack *extack) +{ + struct am65_cpsw_port *port = am65_ndev_to_port(ndev); + struct am65_cpsw_ndev_priv *priv = netdev_priv(ndev); + struct am65_cpsw_iet *iet = &port->qos.iet; + u32 val, add_frag_size; + int err; + + if (!IS_ENABLED(CONFIG_TI_AM65_CPSW_QOS)) + return -EOPNOTSUPP; + + err = ethtool_mm_frag_size_min_to_add(cfg->tx_min_frag_size, &add_frag_size, extack); + if (err) + return err; + + mutex_lock(&priv->mm_lock); + + if (cfg->pmac_enabled) { + /* change TX & RX FIFO MAX_BLKS as per TRM recommendation */ + if (!iet->original_max_blks) + iet->original_max_blks = readl(port->port_base + AM65_CPSW_PN_REG_MAX_BLKS); + + writel(AM65_CPSW_PN_TX_RX_MAX_BLKS_IET, + port->port_base + AM65_CPSW_PN_REG_MAX_BLKS); + } else if (iet->original_max_blks) { + /* restore RX & TX FIFO MAX_BLKS */ + writel(iet->original_max_blks, + port->port_base + AM65_CPSW_PN_REG_MAX_BLKS); + } + + am65_cpsw_port_iet_rx_enable(port, cfg->pmac_enabled); + am65_cpsw_port_iet_tx_enable(port, cfg->tx_enabled); + + val = readl(port->port_base + AM65_CPSW_PN_REG_IET_CTRL); + if (cfg->verify_enabled) { + val &= ~AM65_CPSW_PN_IET_MAC_DISABLEVERIFY; + /* Reset Verify state machine. Verification won't start here. + * Verification will be done once link-up. + */ + val |= AM65_CPSW_PN_IET_MAC_LINKFAIL; + } else { + val |= AM65_CPSW_PN_IET_MAC_DISABLEVERIFY; + /* Clear LINKFAIL to allow verify/response packets */ + val &= ~AM65_CPSW_PN_IET_MAC_LINKFAIL; + } + + val &= ~AM65_CPSW_PN_IET_MAC_MAC_ADDFRAGSIZE_MASK; + val |= AM65_CPSW_PN_IET_MAC_SET_ADDFRAGSIZE(add_frag_size); + writel(val, port->port_base + AM65_CPSW_PN_REG_IET_CTRL); + + /* verify_timeout_count can only be set at valid link */ + port->qos.iet.verify_time_ms = cfg->verify_time; + + /* enable/disable preemption based on link status */ + am65_cpsw_iet_commit_preemptible_tcs(port); + + mutex_unlock(&priv->mm_lock); + + return 0; +} + +static void am65_cpsw_get_mm_stats(struct net_device *ndev, + struct ethtool_mm_stats *s) +{ + struct am65_cpsw_port *port = am65_ndev_to_port(ndev); + void __iomem *base = port->stat_base; + + s->MACMergeFrameAssOkCount = readl(base + AM65_CPSW_STATN_IET_RX_ASSEMBLY_OK); + s->MACMergeFrameAssErrorCount = readl(base + AM65_CPSW_STATN_IET_RX_ASSEMBLY_ERROR); + s->MACMergeFrameSmdErrorCount = readl(base + AM65_CPSW_STATN_IET_RX_SMD_ERROR); + /* CPSW Functional Spec states: + * "The IET stat aMACMergeFragCountRx is derived by adding the + * Receive Assembly Error count to this value. i.e. AM65_CPSW_STATN_IET_RX_FRAG" + */ + s->MACMergeFragCountRx = readl(base + AM65_CPSW_STATN_IET_RX_FRAG) + s->MACMergeFrameAssErrorCount; + s->MACMergeFragCountTx = readl(base + AM65_CPSW_STATN_IET_TX_FRAG); + s->MACMergeHoldCount = readl(base + AM65_CPSW_STATN_IET_TX_HOLD); +} + +static int am65_cpsw_get_coalesce(struct net_device *ndev, struct ethtool_coalesce *coal, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + struct am65_cpsw_common *common = am65_ndev_to_common(ndev); + struct am65_cpsw_tx_chn *tx_chn; + + tx_chn = &common->tx_chns[0]; + + coal->rx_coalesce_usecs = common->rx_pace_timeout / 1000; + coal->tx_coalesce_usecs = tx_chn->tx_pace_timeout / 1000; + + return 0; +} + +static int am65_cpsw_get_per_queue_coalesce(struct net_device *ndev, u32 queue, + struct ethtool_coalesce *coal) +{ + struct am65_cpsw_common *common = am65_ndev_to_common(ndev); + struct am65_cpsw_tx_chn *tx_chn; + + if (queue >= AM65_CPSW_MAX_TX_QUEUES) + return -EINVAL; + + tx_chn = &common->tx_chns[queue]; + + coal->tx_coalesce_usecs = tx_chn->tx_pace_timeout / 1000; + + return 0; +} + +static int am65_cpsw_set_coalesce(struct net_device *ndev, struct ethtool_coalesce *coal, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + struct am65_cpsw_common *common = am65_ndev_to_common(ndev); + struct am65_cpsw_tx_chn *tx_chn; + + tx_chn = &common->tx_chns[0]; + + if (coal->rx_coalesce_usecs && coal->rx_coalesce_usecs < 20) + return -EINVAL; + + if (coal->tx_coalesce_usecs && coal->tx_coalesce_usecs < 20) + return -EINVAL; + + common->rx_pace_timeout = coal->rx_coalesce_usecs * 1000; + tx_chn->tx_pace_timeout = coal->tx_coalesce_usecs * 1000; + + return 0; +} + +static int am65_cpsw_set_per_queue_coalesce(struct net_device *ndev, u32 queue, + struct ethtool_coalesce *coal) +{ + struct am65_cpsw_common *common = am65_ndev_to_common(ndev); + struct am65_cpsw_tx_chn *tx_chn; + + if (queue >= AM65_CPSW_MAX_TX_QUEUES) + return -EINVAL; + + tx_chn = &common->tx_chns[queue]; + + if (coal->tx_coalesce_usecs && coal->tx_coalesce_usecs < 20) { + dev_info(common->dev, "defaulting to min value of 20us for tx-usecs for tx-%u\n", + queue); + coal->tx_coalesce_usecs = 20; + } + + tx_chn->tx_pace_timeout = coal->tx_coalesce_usecs * 1000; + + return 0; +} + const struct ethtool_ops am65_cpsw_ethtool_ops_slave = { .begin = am65_cpsw_ethtool_op_begin, .complete = am65_cpsw_ethtool_op_complete, @@ -758,6 +996,11 @@ const struct ethtool_ops am65_cpsw_ethtool_ops_slave = { .get_ts_info = am65_cpsw_get_ethtool_ts_info, .get_priv_flags = am65_cpsw_get_ethtool_priv_flags, .set_priv_flags = am65_cpsw_set_ethtool_priv_flags, + .supported_coalesce_params = ETHTOOL_COALESCE_USECS, + .get_coalesce = am65_cpsw_get_coalesce, + .set_coalesce = am65_cpsw_set_coalesce, + .get_per_queue_coalesce = am65_cpsw_get_per_queue_coalesce, + .set_per_queue_coalesce = am65_cpsw_set_per_queue_coalesce, .get_link = ethtool_op_get_link, .get_link_ksettings = am65_cpsw_get_link_ksettings, @@ -769,4 +1012,7 @@ const struct ethtool_ops am65_cpsw_ethtool_ops_slave = { .get_eee = am65_cpsw_get_eee, .set_eee = am65_cpsw_set_eee, .nway_reset = am65_cpsw_nway_reset, + .get_mm = am65_cpsw_get_mm, + .set_mm = am65_cpsw_set_mm, + .get_mm_stats = am65_cpsw_get_mm_stats, }; diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c index 7651f90f51f2..faa0561e988e 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c @@ -596,8 +596,10 @@ static int am65_cpsw_nuss_common_stop(struct am65_cpsw_common *common) msecs_to_jiffies(1000)); if (!i) dev_err(common->dev, "tx timeout\n"); - for (i = 0; i < common->tx_ch_num; i++) + for (i = 0; i < common->tx_ch_num; i++) { napi_disable(&common->tx_chns[i].napi_tx); + hrtimer_cancel(&common->tx_chns[i].tx_hrtimer); + } for (i = 0; i < common->tx_ch_num; i++) { k3_udma_glue_reset_tx_chn(common->tx_chns[i].tx_chn, @@ -616,6 +618,7 @@ static int am65_cpsw_nuss_common_stop(struct am65_cpsw_common *common) } napi_disable(&common->napi_rx); + hrtimer_cancel(&common->rx_hrtimer); for (i = 0; i < AM65_CPSW_MAX_RX_FLOWS; i++) k3_udma_glue_reset_rx_chn(common->rx_chns.rx_chn, i, @@ -885,6 +888,15 @@ static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_common *common, return ret; } +static enum hrtimer_restart am65_cpsw_nuss_rx_timer_callback(struct hrtimer *timer) +{ + struct am65_cpsw_common *common = + container_of(timer, struct am65_cpsw_common, rx_hrtimer); + + enable_irq(common->rx_chns.irq); + return HRTIMER_NORESTART; +} + static int am65_cpsw_nuss_rx_poll(struct napi_struct *napi_rx, int budget) { struct am65_cpsw_common *common = am65_cpsw_napi_to_common(napi_rx); @@ -912,7 +924,13 @@ static int am65_cpsw_nuss_rx_poll(struct napi_struct *napi_rx, int budget) if (num_rx < budget && napi_complete_done(napi_rx, num_rx)) { if (common->rx_irq_disabled) { common->rx_irq_disabled = false; - enable_irq(common->rx_chns.irq); + if (unlikely(common->rx_pace_timeout)) { + hrtimer_start(&common->rx_hrtimer, + ns_to_ktime(common->rx_pace_timeout), + HRTIMER_MODE_REL_PINNED); + } else { + enable_irq(common->rx_chns.irq); + } } } @@ -968,7 +986,7 @@ static void am65_cpsw_nuss_tx_wake(struct am65_cpsw_tx_chn *tx_chn, struct net_d } static int am65_cpsw_nuss_tx_compl_packets(struct am65_cpsw_common *common, - int chn, unsigned int budget) + int chn, unsigned int budget, bool *tdown) { struct device *dev = common->dev; struct am65_cpsw_tx_chn *tx_chn; @@ -991,6 +1009,7 @@ static int am65_cpsw_nuss_tx_compl_packets(struct am65_cpsw_common *common, if (cppi5_desc_is_tdcm(desc_dma)) { if (atomic_dec_and_test(&common->tdown_cnt)) complete(&common->tdown_complete); + *tdown = true; break; } @@ -1013,7 +1032,7 @@ static int am65_cpsw_nuss_tx_compl_packets(struct am65_cpsw_common *common, } static int am65_cpsw_nuss_tx_compl_packets_2g(struct am65_cpsw_common *common, - int chn, unsigned int budget) + int chn, unsigned int budget, bool *tdown) { struct device *dev = common->dev; struct am65_cpsw_tx_chn *tx_chn; @@ -1034,6 +1053,7 @@ static int am65_cpsw_nuss_tx_compl_packets_2g(struct am65_cpsw_common *common, if (cppi5_desc_is_tdcm(desc_dma)) { if (atomic_dec_and_test(&common->tdown_cnt)) complete(&common->tdown_complete); + *tdown = true; break; } @@ -1059,21 +1079,40 @@ static int am65_cpsw_nuss_tx_compl_packets_2g(struct am65_cpsw_common *common, return num_tx; } +static enum hrtimer_restart am65_cpsw_nuss_tx_timer_callback(struct hrtimer *timer) +{ + struct am65_cpsw_tx_chn *tx_chns = + container_of(timer, struct am65_cpsw_tx_chn, tx_hrtimer); + + enable_irq(tx_chns->irq); + return HRTIMER_NORESTART; +} + static int am65_cpsw_nuss_tx_poll(struct napi_struct *napi_tx, int budget) { struct am65_cpsw_tx_chn *tx_chn = am65_cpsw_napi_to_tx_chn(napi_tx); + bool tdown = false; int num_tx; if (AM65_CPSW_IS_CPSW2G(tx_chn->common)) - num_tx = am65_cpsw_nuss_tx_compl_packets_2g(tx_chn->common, tx_chn->id, budget); + num_tx = am65_cpsw_nuss_tx_compl_packets_2g(tx_chn->common, tx_chn->id, + budget, &tdown); else - num_tx = am65_cpsw_nuss_tx_compl_packets(tx_chn->common, tx_chn->id, budget); + num_tx = am65_cpsw_nuss_tx_compl_packets(tx_chn->common, + tx_chn->id, budget, &tdown); if (num_tx >= budget) return budget; - if (napi_complete_done(napi_tx, num_tx)) - enable_irq(tx_chn->irq); + if (napi_complete_done(napi_tx, num_tx)) { + if (unlikely(tx_chn->tx_pace_timeout && !tdown)) { + hrtimer_start(&tx_chn->tx_hrtimer, + ns_to_ktime(tx_chn->tx_pace_timeout), + HRTIMER_MODE_REL_PINNED); + } else { + enable_irq(tx_chn->irq); + } + } return 0; } @@ -1705,6 +1744,8 @@ static int am65_cpsw_nuss_ndev_add_tx_napi(struct am65_cpsw_common *common) netif_napi_add_tx(common->dma_ndev, &tx_chn->napi_tx, am65_cpsw_nuss_tx_poll); + hrtimer_init(&tx_chn->tx_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); + tx_chn->tx_hrtimer.function = &am65_cpsw_nuss_tx_timer_callback; ret = devm_request_irq(dev, tx_chn->irq, am65_cpsw_nuss_tx_irq, @@ -1930,6 +1971,8 @@ static int am65_cpsw_nuss_init_rx_chns(struct am65_cpsw_common *common) netif_napi_add(common->dma_ndev, &common->napi_rx, am65_cpsw_nuss_rx_poll); + hrtimer_init(&common->rx_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); + common->rx_hrtimer.function = &am65_cpsw_nuss_rx_timer_callback; ret = devm_request_irq(dev, rx_chn->irq, am65_cpsw_nuss_rx_irq, @@ -2127,6 +2170,9 @@ static int am65_cpsw_nuss_init_slave_ports(struct am65_cpsw_common *common) dev_err(dev, "Use random MAC address\n"); } } + + /* Reset all Queue priorities to 0 */ + writel(0, port->port_base + AM65_CPSW_PN_REG_TX_PRI_MAP); } of_node_put(node); @@ -2191,6 +2237,8 @@ am65_cpsw_nuss_init_port_ndev(struct am65_cpsw_common *common, u32 port_idx) ndev_priv = netdev_priv(port->ndev); ndev_priv->port = port; ndev_priv->msg_enable = AM65_CPSW_DEBUG; + mutex_init(&ndev_priv->mm_lock); + port->qos.link_speed = SPEED_UNKNOWN; SET_NETDEV_DEV(port->ndev, dev); eth_hw_addr_set(port->ndev, port->slave.mac_addr); diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.h b/drivers/net/ethernet/ti/am65-cpsw-nuss.h index f3dad2ab9828..7da0492dc091 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.h +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.h @@ -75,6 +75,8 @@ struct am65_cpsw_tx_chn { struct k3_cppi_desc_pool *desc_pool; struct k3_udma_glue_tx_channel *tx_chn; spinlock_t lock; /* protect TX rings in multi-port mode */ + struct hrtimer tx_hrtimer; + unsigned long tx_pace_timeout; int irq; u32 id; u32 descs_num; @@ -138,6 +140,8 @@ struct am65_cpsw_common { struct napi_struct napi_rx; bool rx_irq_disabled; + struct hrtimer rx_hrtimer; + unsigned long rx_pace_timeout; u32 nuss_ver; u32 cpsw_ver; @@ -145,6 +149,7 @@ struct am65_cpsw_common { bool pf_p0_rx_ptype_rrobin; struct am65_cpts *cpts; int est_enabled; + bool iet_enabled; bool is_emac_mode; u16 br_members; @@ -170,6 +175,10 @@ struct am65_cpsw_ndev_priv { struct am65_cpsw_port *port; struct am65_cpsw_ndev_stats __percpu *stats; bool offload_fwd_mark; + /* Serialize access to MAC Merge state between ethtool requests + * and link state updates + */ + struct mutex mm_lock; }; #define am65_ndev_to_priv(ndev) \ diff --git a/drivers/net/ethernet/ti/am65-cpsw-qos.c b/drivers/net/ethernet/ti/am65-cpsw-qos.c index 9ac2ff05d501..816e73a3d6e4 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-qos.c +++ b/drivers/net/ethernet/ti/am65-cpsw-qos.c @@ -4,10 +4,13 @@ * * quality of service module includes: * Enhanced Scheduler Traffic (EST - P802.1Qbv/D2.2) + * Interspersed Express Traffic (IET - P802.3br/D2.0) */ #include <linux/pm_runtime.h> +#include <linux/math.h> #include <linux/time.h> +#include <linux/units.h> #include <net/pkt_cls.h> #include "am65-cpsw-nuss.h" @@ -15,40 +18,7 @@ #include "am65-cpts.h" #include "cpsw_ale.h" -#define AM65_CPSW_REG_CTL 0x004 -#define AM65_CPSW_PN_REG_CTL 0x004 -#define AM65_CPSW_PN_REG_FIFO_STATUS 0x050 -#define AM65_CPSW_PN_REG_EST_CTL 0x060 -#define AM65_CPSW_PN_REG_PRI_CIR(pri) (0x140 + 4 * (pri)) - -/* AM65_CPSW_REG_CTL register fields */ -#define AM65_CPSW_CTL_EST_EN BIT(18) - -/* AM65_CPSW_PN_REG_CTL register fields */ -#define AM65_CPSW_PN_CTL_EST_PORT_EN BIT(17) - -/* AM65_CPSW_PN_REG_EST_CTL register fields */ -#define AM65_CPSW_PN_EST_ONEBUF BIT(0) -#define AM65_CPSW_PN_EST_BUFSEL BIT(1) -#define AM65_CPSW_PN_EST_TS_EN BIT(2) -#define AM65_CPSW_PN_EST_TS_FIRST BIT(3) -#define AM65_CPSW_PN_EST_ONEPRI BIT(4) -#define AM65_CPSW_PN_EST_TS_PRI_MSK GENMASK(7, 5) - -/* AM65_CPSW_PN_REG_FIFO_STATUS register fields */ -#define AM65_CPSW_PN_FST_TX_PRI_ACTIVE_MSK GENMASK(7, 0) -#define AM65_CPSW_PN_FST_TX_E_MAC_ALLOW_MSK GENMASK(15, 8) -#define AM65_CPSW_PN_FST_EST_CNT_ERR BIT(16) -#define AM65_CPSW_PN_FST_EST_ADD_ERR BIT(17) -#define AM65_CPSW_PN_FST_EST_BUFACT BIT(18) - -/* EST FETCH COMMAND RAM */ -#define AM65_CPSW_FETCH_RAM_CMD_NUM 0x80 -#define AM65_CPSW_FETCH_CNT_MSK GENMASK(21, 8) -#define AM65_CPSW_FETCH_CNT_MAX (AM65_CPSW_FETCH_CNT_MSK >> 8) -#define AM65_CPSW_FETCH_CNT_OFFSET 8 -#define AM65_CPSW_FETCH_ALLOW_MSK GENMASK(7, 0) -#define AM65_CPSW_FETCH_ALLOW_MAX AM65_CPSW_FETCH_ALLOW_MSK +#define TO_MBPS(x) DIV_ROUND_UP((x), BYTES_PER_MBIT) enum timer_act { TACT_PROG, /* need program timer */ @@ -56,6 +26,412 @@ enum timer_act { TACT_SKIP_PROG, /* just buffer can be updated */ }; +static void am65_cpsw_iet_change_preemptible_tcs(struct am65_cpsw_port *port, u8 preemptible_tcs); + +static u32 +am65_cpsw_qos_tx_rate_calc(u32 rate_mbps, unsigned long bus_freq) +{ + u32 ir; + + bus_freq /= 1000000; + ir = DIV_ROUND_UP(((u64)rate_mbps * 32768), bus_freq); + return ir; +} + +static void am65_cpsw_tx_pn_shaper_reset(struct am65_cpsw_port *port) +{ + int prio; + + for (prio = 0; prio < AM65_CPSW_PN_FIFO_PRIO_NUM; prio++) { + writel(0, port->port_base + AM65_CPSW_PN_REG_PRI_CIR(prio)); + writel(0, port->port_base + AM65_CPSW_PN_REG_PRI_EIR(prio)); + } +} + +static void am65_cpsw_tx_pn_shaper_apply(struct am65_cpsw_port *port) +{ + struct am65_cpsw_mqprio *p_mqprio = &port->qos.mqprio; + struct am65_cpsw_common *common = port->common; + struct tc_mqprio_qopt_offload *mqprio; + bool enable, shaper_susp = false; + u32 rate_mbps; + int tc, prio; + + mqprio = &p_mqprio->mqprio_hw; + /* takes care of no link case as well */ + if (p_mqprio->max_rate_total > port->qos.link_speed) + shaper_susp = true; + + am65_cpsw_tx_pn_shaper_reset(port); + + enable = p_mqprio->shaper_en && !shaper_susp; + if (!enable) + return; + + /* Rate limit is specified per Traffic Class but + * for CPSW, rate limit can be applied per priority + * at port FIFO. + * + * We have assigned the same priority (TCn) to all queues + * of a Traffic Class so they share the same shaper + * bandwidth. + */ + for (tc = 0; tc < mqprio->qopt.num_tc; tc++) { + prio = tc; + + rate_mbps = TO_MBPS(mqprio->min_rate[tc]); + rate_mbps = am65_cpsw_qos_tx_rate_calc(rate_mbps, + common->bus_freq); + writel(rate_mbps, + port->port_base + AM65_CPSW_PN_REG_PRI_CIR(prio)); + + rate_mbps = 0; + + if (mqprio->max_rate[tc]) { + rate_mbps = mqprio->max_rate[tc] - mqprio->min_rate[tc]; + rate_mbps = TO_MBPS(rate_mbps); + rate_mbps = am65_cpsw_qos_tx_rate_calc(rate_mbps, + common->bus_freq); + } + + writel(rate_mbps, + port->port_base + AM65_CPSW_PN_REG_PRI_EIR(prio)); + } +} + +static int am65_cpsw_mqprio_verify_shaper(struct am65_cpsw_port *port, + struct tc_mqprio_qopt_offload *mqprio) +{ + struct am65_cpsw_mqprio *p_mqprio = &port->qos.mqprio; + struct netlink_ext_ack *extack = mqprio->extack; + u64 min_rate_total = 0, max_rate_total = 0; + u32 min_rate_msk = 0, max_rate_msk = 0; + bool has_min_rate, has_max_rate; + int num_tc, i; + + if (!(mqprio->flags & TC_MQPRIO_F_SHAPER)) + return 0; + + if (mqprio->shaper != TC_MQPRIO_SHAPER_BW_RATE) + return 0; + + has_min_rate = !!(mqprio->flags & TC_MQPRIO_F_MIN_RATE); + has_max_rate = !!(mqprio->flags & TC_MQPRIO_F_MAX_RATE); + + if (!has_min_rate && has_max_rate) { + NL_SET_ERR_MSG_MOD(extack, "min_rate is required with max_rate"); + return -EOPNOTSUPP; + } + + if (!has_min_rate) + return 0; + + num_tc = mqprio->qopt.num_tc; + + for (i = num_tc - 1; i >= 0; i--) { + u32 ch_msk; + + if (mqprio->min_rate[i]) + min_rate_msk |= BIT(i); + min_rate_total += mqprio->min_rate[i]; + + if (has_max_rate) { + if (mqprio->max_rate[i]) + max_rate_msk |= BIT(i); + max_rate_total += mqprio->max_rate[i]; + + if (!mqprio->min_rate[i] && mqprio->max_rate[i]) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "TX tc%d rate max>0 but min=0", + i); + return -EINVAL; + } + + if (mqprio->max_rate[i] && + mqprio->max_rate[i] < mqprio->min_rate[i]) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "TX tc%d rate min(%llu)>max(%llu)", + i, mqprio->min_rate[i], + mqprio->max_rate[i]); + return -EINVAL; + } + } + + ch_msk = GENMASK(num_tc - 1, i); + if ((min_rate_msk & BIT(i)) && (min_rate_msk ^ ch_msk)) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "Min rate must be set sequentially hi->lo tx_rate_msk%x", + min_rate_msk); + return -EINVAL; + } + + if ((max_rate_msk & BIT(i)) && (max_rate_msk ^ ch_msk)) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "Max rate must be set sequentially hi->lo tx_rate_msk%x", + max_rate_msk); + return -EINVAL; + } + } + + min_rate_total = TO_MBPS(min_rate_total); + max_rate_total = TO_MBPS(max_rate_total); + + p_mqprio->shaper_en = true; + p_mqprio->max_rate_total = max_t(u64, min_rate_total, max_rate_total); + + return 0; +} + +static void am65_cpsw_reset_tc_mqprio(struct net_device *ndev) +{ + struct am65_cpsw_port *port = am65_ndev_to_port(ndev); + struct am65_cpsw_mqprio *p_mqprio = &port->qos.mqprio; + + p_mqprio->shaper_en = false; + p_mqprio->max_rate_total = 0; + + am65_cpsw_tx_pn_shaper_reset(port); + netdev_reset_tc(ndev); + + /* Reset all Queue priorities to 0 */ + writel(0, port->port_base + AM65_CPSW_PN_REG_TX_PRI_MAP); + + am65_cpsw_iet_change_preemptible_tcs(port, 0); +} + +static int am65_cpsw_setup_mqprio(struct net_device *ndev, void *type_data) +{ + struct am65_cpsw_port *port = am65_ndev_to_port(ndev); + struct am65_cpsw_mqprio *p_mqprio = &port->qos.mqprio; + struct tc_mqprio_qopt_offload *mqprio = type_data; + struct am65_cpsw_common *common = port->common; + struct tc_mqprio_qopt *qopt = &mqprio->qopt; + int i, tc, offset, count, prio, ret; + u8 num_tc = qopt->num_tc; + u32 tx_prio_map = 0; + + memcpy(&p_mqprio->mqprio_hw, mqprio, sizeof(*mqprio)); + + ret = pm_runtime_get_sync(common->dev); + if (ret < 0) { + pm_runtime_put_noidle(common->dev); + return ret; + } + + if (!num_tc) { + am65_cpsw_reset_tc_mqprio(ndev); + ret = 0; + goto exit_put; + } + + ret = am65_cpsw_mqprio_verify_shaper(port, mqprio); + if (ret) + goto exit_put; + + netdev_set_num_tc(ndev, num_tc); + + /* Multiple Linux priorities can map to a Traffic Class + * A Traffic Class can have multiple contiguous Queues, + * Queues get mapped to Channels (thread_id), + * if not VLAN tagged, thread_id is used as packet_priority + * if VLAN tagged. VLAN priority is used as packet_priority + * packet_priority gets mapped to header_priority in p0_rx_pri_map, + * header_priority gets mapped to switch_priority in pn_tx_pri_map. + * As p0_rx_pri_map is left at defaults (0x76543210), we can + * assume that Queue_n gets mapped to header_priority_n. We can then + * set the switch priority in pn_tx_pri_map. + */ + + for (tc = 0; tc < num_tc; tc++) { + prio = tc; + + /* For simplicity we assign the same priority (TCn) to + * all queues of a Traffic Class. + */ + for (i = qopt->offset[tc]; i < qopt->offset[tc] + qopt->count[tc]; i++) + tx_prio_map |= prio << (4 * i); + + count = qopt->count[tc]; + offset = qopt->offset[tc]; + netdev_set_tc_queue(ndev, tc, count, offset); + } + + writel(tx_prio_map, port->port_base + AM65_CPSW_PN_REG_TX_PRI_MAP); + + am65_cpsw_tx_pn_shaper_apply(port); + am65_cpsw_iet_change_preemptible_tcs(port, mqprio->preemptible_tcs); + +exit_put: + pm_runtime_put(common->dev); + + return ret; +} + +static int am65_cpsw_iet_set_verify_timeout_count(struct am65_cpsw_port *port) +{ + int verify_time_ms = port->qos.iet.verify_time_ms; + u32 val; + + /* The number of wireside clocks contained in the verify + * timeout counter. The default is 0x1312d0 + * (10ms at 125Mhz in 1G mode). + */ + val = 125 * HZ_PER_MHZ; /* assuming 125MHz wireside clock */ + + val /= MILLIHZ_PER_HZ; /* count per ms timeout */ + val *= verify_time_ms; /* count for timeout ms */ + + if (val > AM65_CPSW_PN_MAC_VERIFY_CNT_MASK) + return -EINVAL; + + writel(val, port->port_base + AM65_CPSW_PN_REG_IET_VERIFY); + + return 0; +} + +static int am65_cpsw_iet_verify_wait(struct am65_cpsw_port *port) +{ + u32 ctrl, status; + int try; + + try = 20; + do { + /* Reset the verify state machine by writing 1 + * to LINKFAIL + */ + ctrl = readl(port->port_base + AM65_CPSW_PN_REG_IET_CTRL); + ctrl |= AM65_CPSW_PN_IET_MAC_LINKFAIL; + writel(ctrl, port->port_base + AM65_CPSW_PN_REG_IET_CTRL); + + /* Clear MAC_LINKFAIL bit to start Verify. */ + ctrl = readl(port->port_base + AM65_CPSW_PN_REG_IET_CTRL); + ctrl &= ~AM65_CPSW_PN_IET_MAC_LINKFAIL; + writel(ctrl, port->port_base + AM65_CPSW_PN_REG_IET_CTRL); + + msleep(port->qos.iet.verify_time_ms); + + status = readl(port->port_base + AM65_CPSW_PN_REG_IET_STATUS); + if (status & AM65_CPSW_PN_MAC_VERIFIED) + return 0; + + if (status & AM65_CPSW_PN_MAC_VERIFY_FAIL) { + netdev_dbg(port->ndev, + "MAC Merge verify failed, trying again\n"); + continue; + } + + if (status & AM65_CPSW_PN_MAC_RESPOND_ERR) { + netdev_dbg(port->ndev, "MAC Merge respond error\n"); + return -ENODEV; + } + + if (status & AM65_CPSW_PN_MAC_VERIFY_ERR) { + netdev_dbg(port->ndev, "MAC Merge verify error\n"); + return -ENODEV; + } + } while (try-- > 0); + + netdev_dbg(port->ndev, "MAC Merge verify timeout\n"); + return -ETIMEDOUT; +} + +static void am65_cpsw_iet_set_preempt_mask(struct am65_cpsw_port *port, u8 preemptible_tcs) +{ + u32 val; + + val = readl(port->port_base + AM65_CPSW_PN_REG_IET_CTRL); + val &= ~AM65_CPSW_PN_IET_MAC_PREMPT_MASK; + val |= AM65_CPSW_PN_IET_MAC_SET_PREEMPT(preemptible_tcs); + writel(val, port->port_base + AM65_CPSW_PN_REG_IET_CTRL); +} + +/* enable common IET_ENABLE only if at least 1 port has rx IET enabled. + * UAPI doesn't allow tx enable without rx enable. + */ +void am65_cpsw_iet_common_enable(struct am65_cpsw_common *common) +{ + struct am65_cpsw_port *port; + bool rx_enable = false; + u32 val; + int i; + + for (i = 0; i < common->port_num; i++) { + port = &common->ports[i]; + val = readl(port->port_base + AM65_CPSW_PN_REG_CTL); + rx_enable = !!(val & AM65_CPSW_PN_CTL_IET_PORT_EN); + if (rx_enable) + break; + } + + val = readl(common->cpsw_base + AM65_CPSW_REG_CTL); + + if (rx_enable) + val |= AM65_CPSW_CTL_IET_EN; + else + val &= ~AM65_CPSW_CTL_IET_EN; + + writel(val, common->cpsw_base + AM65_CPSW_REG_CTL); + common->iet_enabled = rx_enable; +} + +/* CPSW does not have an IRQ to notify changes to the MAC Merge TX status + * (active/inactive), but the preemptible traffic classes should only be + * committed to hardware once TX is active. Resort to polling. + */ +void am65_cpsw_iet_commit_preemptible_tcs(struct am65_cpsw_port *port) +{ + u8 preemptible_tcs; + int err; + u32 val; + + if (port->qos.link_speed == SPEED_UNKNOWN) + return; + + val = readl(port->port_base + AM65_CPSW_PN_REG_CTL); + if (!(val & AM65_CPSW_PN_CTL_IET_PORT_EN)) + return; + + /* update common IET enable */ + am65_cpsw_iet_common_enable(port->common); + + /* update verify count */ + err = am65_cpsw_iet_set_verify_timeout_count(port); + if (err) { + netdev_err(port->ndev, "couldn't set verify count: %d\n", err); + return; + } + + val = readl(port->port_base + AM65_CPSW_PN_REG_IET_CTRL); + if (!(val & AM65_CPSW_PN_IET_MAC_DISABLEVERIFY)) { + err = am65_cpsw_iet_verify_wait(port); + if (err) + return; + } + + preemptible_tcs = port->qos.iet.preemptible_tcs; + am65_cpsw_iet_set_preempt_mask(port, preemptible_tcs); +} + +static void am65_cpsw_iet_change_preemptible_tcs(struct am65_cpsw_port *port, u8 preemptible_tcs) +{ + struct am65_cpsw_ndev_priv *priv = am65_ndev_to_priv(port->ndev); + + port->qos.iet.preemptible_tcs = preemptible_tcs; + mutex_lock(&priv->mm_lock); + am65_cpsw_iet_commit_preemptible_tcs(port); + mutex_unlock(&priv->mm_lock); +} + +static void am65_cpsw_iet_link_state_update(struct net_device *ndev) +{ + struct am65_cpsw_ndev_priv *priv = am65_ndev_to_priv(ndev); + struct am65_cpsw_port *port = am65_ndev_to_port(ndev); + + mutex_lock(&priv->mm_lock); + am65_cpsw_iet_commit_preemptible_tcs(port); + mutex_unlock(&priv->mm_lock); +} + static int am65_cpsw_port_est_enabled(struct am65_cpsw_port *port) { return port->qos.est_oper || port->qos.est_admin; @@ -428,7 +804,7 @@ static void am65_cpsw_stop_est(struct net_device *ndev) am65_cpsw_timer_stop(ndev); } -static void am65_cpsw_purge_est(struct net_device *ndev) +static void am65_cpsw_taprio_destroy(struct net_device *ndev) { struct am65_cpsw_port *port = am65_ndev_to_port(ndev); @@ -439,31 +815,74 @@ static void am65_cpsw_purge_est(struct net_device *ndev) port->qos.est_oper = NULL; port->qos.est_admin = NULL; + + am65_cpsw_reset_tc_mqprio(ndev); } -static int am65_cpsw_configure_taprio(struct net_device *ndev, - struct am65_cpsw_est *est_new) +static void am65_cpsw_cp_taprio(struct tc_taprio_qopt_offload *from, + struct tc_taprio_qopt_offload *to) +{ + int i; + + *to = *from; + for (i = 0; i < from->num_entries; i++) + to->entries[i] = from->entries[i]; +} + +static int am65_cpsw_taprio_replace(struct net_device *ndev, + struct tc_taprio_qopt_offload *taprio) { struct am65_cpsw_common *common = am65_ndev_to_common(ndev); + struct netlink_ext_ack *extack = taprio->mqprio.extack; + struct am65_cpsw_port *port = am65_ndev_to_port(ndev); struct am65_cpts *cpts = common->cpts; - int ret = 0, tact = TACT_PROG; + struct am65_cpsw_est *est_new; + int ret, tact; - am65_cpsw_est_update_state(ndev); + if (!netif_running(ndev)) { + NL_SET_ERR_MSG_MOD(extack, "interface is down, link speed unknown"); + return -ENETDOWN; + } - if (est_new->taprio.cmd == TAPRIO_CMD_DESTROY) { - am65_cpsw_stop_est(ndev); - return ret; + if (common->pf_p0_rx_ptype_rrobin) { + NL_SET_ERR_MSG_MOD(extack, + "p0-rx-ptype-rrobin flag conflicts with taprio qdisc"); + return -EINVAL; } + if (port->qos.link_speed == SPEED_UNKNOWN) + return -ENOLINK; + + if (taprio->cycle_time_extension) { + NL_SET_ERR_MSG_MOD(extack, + "cycle time extension not supported"); + return -EOPNOTSUPP; + } + + est_new = devm_kzalloc(&ndev->dev, + struct_size(est_new, taprio.entries, taprio->num_entries), + GFP_KERNEL); + if (!est_new) + return -ENOMEM; + + ret = am65_cpsw_setup_mqprio(ndev, &taprio->mqprio); + if (ret) + return ret; + + am65_cpsw_cp_taprio(taprio, &est_new->taprio); + + am65_cpsw_est_update_state(ndev); + ret = am65_cpsw_est_check_scheds(ndev, est_new); if (ret < 0) - return ret; + goto fail; tact = am65_cpsw_timer_act(ndev, est_new); if (tact == TACT_NEED_STOP) { - dev_err(&ndev->dev, - "Can't toggle estf timer, stop taprio first"); - return -EINVAL; + NL_SET_ERR_MSG_MOD(extack, + "Can't toggle estf timer, stop taprio first"); + ret = -EINVAL; + goto fail; } if (tact == TACT_PROG) @@ -476,62 +895,26 @@ static int am65_cpsw_configure_taprio(struct net_device *ndev, am65_cpsw_est_set_sched_list(ndev, est_new); am65_cpsw_port_est_assign_buf_num(ndev, est_new->buf); - am65_cpsw_est_set(ndev, est_new->taprio.cmd == TAPRIO_CMD_REPLACE); + am65_cpsw_est_set(ndev, 1); if (tact == TACT_PROG) { ret = am65_cpsw_timer_set(ndev, est_new); if (ret) { - dev_err(&ndev->dev, "Failed to set cycle time"); - return ret; + NL_SET_ERR_MSG_MOD(extack, + "Failed to set cycle time"); + goto fail; } } - return ret; -} - -static void am65_cpsw_cp_taprio(struct tc_taprio_qopt_offload *from, - struct tc_taprio_qopt_offload *to) -{ - int i; - - *to = *from; - for (i = 0; i < from->num_entries; i++) - to->entries[i] = from->entries[i]; -} - -static int am65_cpsw_set_taprio(struct net_device *ndev, void *type_data) -{ - struct am65_cpsw_port *port = am65_ndev_to_port(ndev); - struct tc_taprio_qopt_offload *taprio = type_data; - struct am65_cpsw_est *est_new; - int ret = 0; - - if (taprio->cycle_time_extension) { - dev_err(&ndev->dev, "Failed to set cycle time extension"); - return -EOPNOTSUPP; - } - - est_new = devm_kzalloc(&ndev->dev, - struct_size(est_new, taprio.entries, taprio->num_entries), - GFP_KERNEL); - if (!est_new) - return -ENOMEM; - - am65_cpsw_cp_taprio(taprio, &est_new->taprio); - ret = am65_cpsw_configure_taprio(ndev, est_new); - if (!ret) { - if (taprio->cmd == TAPRIO_CMD_REPLACE) { - devm_kfree(&ndev->dev, port->qos.est_admin); + devm_kfree(&ndev->dev, port->qos.est_admin); + port->qos.est_admin = est_new; + am65_cpsw_iet_change_preemptible_tcs(port, taprio->mqprio.preemptible_tcs); - port->qos.est_admin = est_new; - } else { - devm_kfree(&ndev->dev, est_new); - am65_cpsw_purge_est(ndev); - } - } else { - devm_kfree(&ndev->dev, est_new); - } + return 0; +fail: + am65_cpsw_reset_tc_mqprio(ndev); + devm_kfree(&ndev->dev, est_new); return ret; } @@ -541,7 +924,6 @@ static void am65_cpsw_est_link_up(struct net_device *ndev, int link_speed) ktime_t cur_time; s64 delta; - port->qos.link_speed = link_speed; if (!am65_cpsw_port_est_enabled(port)) return; @@ -558,37 +940,26 @@ static void am65_cpsw_est_link_up(struct net_device *ndev, int link_speed) return; purge_est: - am65_cpsw_purge_est(ndev); + am65_cpsw_taprio_destroy(ndev); } static int am65_cpsw_setup_taprio(struct net_device *ndev, void *type_data) { - struct am65_cpsw_port *port = am65_ndev_to_port(ndev); struct tc_taprio_qopt_offload *taprio = type_data; - struct am65_cpsw_common *common = port->common; - - if (taprio->cmd != TAPRIO_CMD_REPLACE && - taprio->cmd != TAPRIO_CMD_DESTROY) - return -EOPNOTSUPP; - - if (!IS_ENABLED(CONFIG_TI_AM65_CPSW_TAS)) - return -ENODEV; - - if (!netif_running(ndev)) { - dev_err(&ndev->dev, "interface is down, link speed unknown\n"); - return -ENETDOWN; - } - - if (common->pf_p0_rx_ptype_rrobin) { - dev_err(&ndev->dev, - "p0-rx-ptype-rrobin flag conflicts with taprio qdisc\n"); - return -EINVAL; + int err = 0; + + switch (taprio->cmd) { + case TAPRIO_CMD_REPLACE: + err = am65_cpsw_taprio_replace(ndev, taprio); + break; + case TAPRIO_CMD_DESTROY: + am65_cpsw_taprio_destroy(ndev); + break; + default: + err = -EOPNOTSUPP; } - if (port->qos.link_speed == SPEED_UNKNOWN) - return -ENOLINK; - - return am65_cpsw_set_taprio(ndev, type_data); + return err; } static int am65_cpsw_tc_query_caps(struct net_device *ndev, void *type_data) @@ -596,12 +967,17 @@ static int am65_cpsw_tc_query_caps(struct net_device *ndev, void *type_data) struct tc_query_caps_base *base = type_data; switch (base->type) { + case TC_SETUP_QDISC_MQPRIO: { + struct tc_mqprio_caps *caps = base->caps; + + caps->validate_queue_counts = true; + + return 0; + } + case TC_SETUP_QDISC_TAPRIO: { struct tc_taprio_caps *caps = base->caps; - if (!IS_ENABLED(CONFIG_TI_AM65_CPSW_TAS)) - return -EOPNOTSUPP; - caps->gate_mask_per_txq = true; return 0; @@ -787,55 +1163,6 @@ static int am65_cpsw_qos_setup_tc_block(struct net_device *ndev, struct flow_blo port, port, true); } -int am65_cpsw_qos_ndo_setup_tc(struct net_device *ndev, enum tc_setup_type type, - void *type_data) -{ - switch (type) { - case TC_QUERY_CAPS: - return am65_cpsw_tc_query_caps(ndev, type_data); - case TC_SETUP_QDISC_TAPRIO: - return am65_cpsw_setup_taprio(ndev, type_data); - case TC_SETUP_BLOCK: - return am65_cpsw_qos_setup_tc_block(ndev, type_data); - default: - return -EOPNOTSUPP; - } -} - -void am65_cpsw_qos_link_up(struct net_device *ndev, int link_speed) -{ - struct am65_cpsw_port *port = am65_ndev_to_port(ndev); - - if (!IS_ENABLED(CONFIG_TI_AM65_CPSW_TAS)) - return; - - am65_cpsw_est_link_up(ndev, link_speed); - port->qos.link_down_time = 0; -} - -void am65_cpsw_qos_link_down(struct net_device *ndev) -{ - struct am65_cpsw_port *port = am65_ndev_to_port(ndev); - - if (!IS_ENABLED(CONFIG_TI_AM65_CPSW_TAS)) - return; - - if (!port->qos.link_down_time) - port->qos.link_down_time = ktime_get(); - - port->qos.link_speed = SPEED_UNKNOWN; -} - -static u32 -am65_cpsw_qos_tx_rate_calc(u32 rate_mbps, unsigned long bus_freq) -{ - u32 ir; - - bus_freq /= 1000000; - ir = DIV_ROUND_UP(((u64)rate_mbps * 32768), bus_freq); - return ir; -} - static void am65_cpsw_qos_tx_p0_rate_apply(struct am65_cpsw_common *common, int tx_ch, u32 rate_mbps) @@ -937,3 +1264,44 @@ void am65_cpsw_qos_tx_p0_rate_init(struct am65_cpsw_common *common) host->port_base + AM65_CPSW_PN_REG_PRI_CIR(tx_ch)); } } + +int am65_cpsw_qos_ndo_setup_tc(struct net_device *ndev, enum tc_setup_type type, + void *type_data) +{ + switch (type) { + case TC_QUERY_CAPS: + return am65_cpsw_tc_query_caps(ndev, type_data); + case TC_SETUP_QDISC_TAPRIO: + return am65_cpsw_setup_taprio(ndev, type_data); + case TC_SETUP_QDISC_MQPRIO: + return am65_cpsw_setup_mqprio(ndev, type_data); + case TC_SETUP_BLOCK: + return am65_cpsw_qos_setup_tc_block(ndev, type_data); + default: + return -EOPNOTSUPP; + } +} + +void am65_cpsw_qos_link_up(struct net_device *ndev, int link_speed) +{ + struct am65_cpsw_port *port = am65_ndev_to_port(ndev); + + port->qos.link_speed = link_speed; + am65_cpsw_tx_pn_shaper_apply(port); + am65_cpsw_iet_link_state_update(ndev); + + am65_cpsw_est_link_up(ndev, link_speed); + port->qos.link_down_time = 0; +} + +void am65_cpsw_qos_link_down(struct net_device *ndev) +{ + struct am65_cpsw_port *port = am65_ndev_to_port(ndev); + + port->qos.link_speed = SPEED_UNKNOWN; + am65_cpsw_tx_pn_shaper_apply(port); + am65_cpsw_iet_link_state_update(ndev); + + if (!port->qos.link_down_time) + port->qos.link_down_time = ktime_get(); +} diff --git a/drivers/net/ethernet/ti/am65-cpsw-qos.h b/drivers/net/ethernet/ti/am65-cpsw-qos.h index 0cc2a3b3d7f9..b328e56c5b2b 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-qos.h +++ b/drivers/net/ethernet/ti/am65-cpsw-qos.h @@ -9,6 +9,7 @@ #include <net/pkt_sched.h> struct am65_cpsw_common; +struct am65_cpsw_port; struct am65_cpsw_est { int buf; @@ -16,6 +17,18 @@ struct am65_cpsw_est { struct tc_taprio_qopt_offload taprio; }; +struct am65_cpsw_mqprio { + struct tc_mqprio_qopt_offload mqprio_hw; + u64 max_rate_total; + bool shaper_en; +}; + +struct am65_cpsw_iet { + u8 preemptible_tcs; + u32 original_max_blks; + int verify_time_ms; +}; + struct am65_cpsw_ale_ratelimit { unsigned long cookie; u64 rate_packet_ps; @@ -26,16 +39,189 @@ struct am65_cpsw_qos { struct am65_cpsw_est *est_oper; ktime_t link_down_time; int link_speed; + struct am65_cpsw_mqprio mqprio; + struct am65_cpsw_iet iet; struct am65_cpsw_ale_ratelimit ale_bc_ratelimit; struct am65_cpsw_ale_ratelimit ale_mc_ratelimit; }; +#define AM65_CPSW_REG_CTL 0x004 +#define AM65_CPSW_PN_REG_CTL 0x004 +#define AM65_CPSW_PN_REG_FIFO_STATUS 0x050 +#define AM65_CPSW_PN_REG_EST_CTL 0x060 +#define AM65_CPSW_PN_REG_PRI_CIR(pri) (0x140 + 4 * (pri)) +#define AM65_CPSW_P0_REG_PRI_EIR(pri) (0x160 + 4 * (pri)) + +#define AM65_CPSW_PN_REG_CTL 0x004 +#define AM65_CPSW_PN_REG_TX_PRI_MAP 0x018 +#define AM65_CPSW_PN_REG_RX_PRI_MAP 0x020 +#define AM65_CPSW_PN_REG_FIFO_STATUS 0x050 +#define AM65_CPSW_PN_REG_EST_CTL 0x060 +#define AM65_CPSW_PN_REG_PRI_CIR(pri) (0x140 + 4 * (pri)) +#define AM65_CPSW_PN_REG_PRI_EIR(pri) (0x160 + 4 * (pri)) + +/* AM65_CPSW_REG_CTL register fields */ +#define AM65_CPSW_CTL_EST_EN BIT(18) + +/* AM65_CPSW_PN_REG_CTL register fields */ +#define AM65_CPSW_PN_CTL_EST_PORT_EN BIT(17) + +/* AM65_CPSW_PN_REG_EST_CTL register fields */ +#define AM65_CPSW_PN_EST_ONEBUF BIT(0) +#define AM65_CPSW_PN_EST_BUFSEL BIT(1) +#define AM65_CPSW_PN_EST_TS_EN BIT(2) +#define AM65_CPSW_PN_EST_TS_FIRST BIT(3) +#define AM65_CPSW_PN_EST_ONEPRI BIT(4) +#define AM65_CPSW_PN_EST_TS_PRI_MSK GENMASK(7, 5) + +/* AM65_CPSW_PN_REG_FIFO_STATUS register fields */ +#define AM65_CPSW_PN_FST_TX_PRI_ACTIVE_MSK GENMASK(7, 0) +#define AM65_CPSW_PN_FST_TX_E_MAC_ALLOW_MSK GENMASK(15, 8) +#define AM65_CPSW_PN_FST_EST_CNT_ERR BIT(16) +#define AM65_CPSW_PN_FST_EST_ADD_ERR BIT(17) +#define AM65_CPSW_PN_FST_EST_BUFACT BIT(18) + +/* EST FETCH COMMAND RAM */ +#define AM65_CPSW_FETCH_RAM_CMD_NUM 0x80 +#define AM65_CPSW_FETCH_CNT_MSK GENMASK(21, 8) +#define AM65_CPSW_FETCH_CNT_MAX (AM65_CPSW_FETCH_CNT_MSK >> 8) +#define AM65_CPSW_FETCH_CNT_OFFSET 8 +#define AM65_CPSW_FETCH_ALLOW_MSK GENMASK(7, 0) +#define AM65_CPSW_FETCH_ALLOW_MAX AM65_CPSW_FETCH_ALLOW_MSK + +/* number of priority queues per port FIFO */ +#define AM65_CPSW_PN_FIFO_PRIO_NUM 8 + +#if IS_ENABLED(CONFIG_TI_AM65_CPSW_QOS) int am65_cpsw_qos_ndo_setup_tc(struct net_device *ndev, enum tc_setup_type type, void *type_data); void am65_cpsw_qos_link_up(struct net_device *ndev, int link_speed); void am65_cpsw_qos_link_down(struct net_device *ndev); int am65_cpsw_qos_ndo_tx_p0_set_maxrate(struct net_device *ndev, int queue, u32 rate_mbps); void am65_cpsw_qos_tx_p0_rate_init(struct am65_cpsw_common *common); +void am65_cpsw_iet_commit_preemptible_tcs(struct am65_cpsw_port *port); +void am65_cpsw_iet_common_enable(struct am65_cpsw_common *common); +#else +static inline int am65_cpsw_qos_ndo_setup_tc(struct net_device *ndev, + enum tc_setup_type type, + void *type_data) +{ + return -EOPNOTSUPP; +} + +static inline void am65_cpsw_qos_link_up(struct net_device *ndev, + int link_speed) +{ } + +static inline void am65_cpsw_qos_link_down(struct net_device *ndev) +{ } + +static inline int am65_cpsw_qos_ndo_tx_p0_set_maxrate(struct net_device *ndev, + int queue, + u32 rate_mbps) +{ + return 0; +} + +static inline void am65_cpsw_qos_tx_p0_rate_init(struct am65_cpsw_common *common) +{ } +static inline void am65_cpsw_iet_commit_preemptible_tcs(struct am65_cpsw_port *port) +{ } +static inline void am65_cpsw_iet_common_enable(struct am65_cpsw_common *common) +{ } +#endif + +#define AM65_CPSW_REG_CTL 0x004 +#define AM65_CPSW_PN_REG_CTL 0x004 +#define AM65_CPSW_PN_REG_MAX_BLKS 0x008 +#define AM65_CPSW_PN_REG_TX_PRI_MAP 0x018 +#define AM65_CPSW_PN_REG_RX_PRI_MAP 0x020 +#define AM65_CPSW_PN_REG_IET_CTRL 0x040 +#define AM65_CPSW_PN_REG_IET_STATUS 0x044 +#define AM65_CPSW_PN_REG_IET_VERIFY 0x048 +#define AM65_CPSW_PN_REG_FIFO_STATUS 0x050 +#define AM65_CPSW_PN_REG_EST_CTL 0x060 +#define AM65_CPSW_PN_REG_PRI_CIR(pri) (0x140 + 4 * (pri)) +#define AM65_CPSW_PN_REG_PRI_EIR(pri) (0x160 + 4 * (pri)) + +/* AM65_CPSW_REG_CTL register fields */ +#define AM65_CPSW_CTL_IET_EN BIT(17) +#define AM65_CPSW_CTL_EST_EN BIT(18) + +/* AM65_CPSW_PN_REG_CTL register fields */ +#define AM65_CPSW_PN_CTL_IET_PORT_EN BIT(16) +#define AM65_CPSW_PN_CTL_EST_PORT_EN BIT(17) + +/* AM65_CPSW_PN_REG_EST_CTL register fields */ +#define AM65_CPSW_PN_EST_ONEBUF BIT(0) +#define AM65_CPSW_PN_EST_BUFSEL BIT(1) +#define AM65_CPSW_PN_EST_TS_EN BIT(2) +#define AM65_CPSW_PN_EST_TS_FIRST BIT(3) +#define AM65_CPSW_PN_EST_ONEPRI BIT(4) +#define AM65_CPSW_PN_EST_TS_PRI_MSK GENMASK(7, 5) + +/* AM65_CPSW_PN_REG_IET_CTRL register fields */ +#define AM65_CPSW_PN_IET_MAC_PENABLE BIT(0) +#define AM65_CPSW_PN_IET_MAC_DISABLEVERIFY BIT(2) +#define AM65_CPSW_PN_IET_MAC_LINKFAIL BIT(3) +#define AM65_CPSW_PN_IET_MAC_MAC_ADDFRAGSIZE_MASK GENMASK(10, 8) +#define AM65_CPSW_PN_IET_MAC_MAC_ADDFRAGSIZE_OFFSET 8 +#define AM65_CPSW_PN_IET_MAC_PREMPT_MASK GENMASK(23, 16) +#define AM65_CPSW_PN_IET_MAC_PREMPT_OFFSET 16 + +#define AM65_CPSW_PN_IET_MAC_SET_ADDFRAGSIZE(n) (((n) << AM65_CPSW_PN_IET_MAC_MAC_ADDFRAGSIZE_OFFSET) & \ + AM65_CPSW_PN_IET_MAC_MAC_ADDFRAGSIZE_MASK) +#define AM65_CPSW_PN_IET_MAC_GET_ADDFRAGSIZE(n) (((n) & AM65_CPSW_PN_IET_MAC_MAC_ADDFRAGSIZE_MASK) >> \ + AM65_CPSW_PN_IET_MAC_MAC_ADDFRAGSIZE_OFFSET) +#define AM65_CPSW_PN_IET_MAC_SET_PREEMPT(n) (((n) << AM65_CPSW_PN_IET_MAC_PREMPT_OFFSET) & \ + AM65_CPSW_PN_IET_MAC_PREMPT_MASK) +#define AM65_CPSW_PN_IET_MAC_GET_PREEMPT(n) (((n) & AM65_CPSW_PN_IET_MAC_PREMPT_MASK) >> \ + AM65_CPSW_PN_IET_MAC_PREMPT_OFFSET) + +/* AM65_CPSW_PN_REG_IET_STATUS register fields */ +#define AM65_CPSW_PN_MAC_STATUS GENMASK(3, 0) +#define AM65_CPSW_PN_MAC_VERIFIED BIT(0) +#define AM65_CPSW_PN_MAC_VERIFY_FAIL BIT(1) +#define AM65_CPSW_PN_MAC_RESPOND_ERR BIT(2) +#define AM65_CPSW_PN_MAC_VERIFY_ERR BIT(3) + +/* AM65_CPSW_PN_REG_IET_VERIFY register fields */ +#define AM65_CPSW_PN_MAC_VERIFY_CNT_MASK GENMASK(23, 0) +#define AM65_CPSW_PN_MAC_GET_VERIFY_CNT(n) ((n) & AM65_CPSW_PN_MAC_VERIFY_CNT_MASK) +/* 10 msec converted to NSEC */ +#define AM65_CPSW_IET_VERIFY_CNT_MS (10) +#define AM65_CPSW_IET_VERIFY_CNT_NS (AM65_CPSW_IET_VERIFY_CNT_MS * \ + NSEC_PER_MSEC) + +/* AM65_CPSW_PN_REG_FIFO_STATUS register fields */ +#define AM65_CPSW_PN_FST_TX_PRI_ACTIVE_MSK GENMASK(7, 0) +#define AM65_CPSW_PN_FST_TX_E_MAC_ALLOW_MSK GENMASK(15, 8) +#define AM65_CPSW_PN_FST_EST_CNT_ERR BIT(16) +#define AM65_CPSW_PN_FST_EST_ADD_ERR BIT(17) +#define AM65_CPSW_PN_FST_EST_BUFACT BIT(18) + +/* EST FETCH COMMAND RAM */ +#define AM65_CPSW_FETCH_RAM_CMD_NUM 0x80 +#define AM65_CPSW_FETCH_CNT_MSK GENMASK(21, 8) +#define AM65_CPSW_FETCH_CNT_MAX (AM65_CPSW_FETCH_CNT_MSK >> 8) +#define AM65_CPSW_FETCH_CNT_OFFSET 8 +#define AM65_CPSW_FETCH_ALLOW_MSK GENMASK(7, 0) +#define AM65_CPSW_FETCH_ALLOW_MAX AM65_CPSW_FETCH_ALLOW_MSK + +/* AM65_CPSW_PN_REG_MAX_BLKS fields for IET and No IET cases */ +/* 7 blocks for pn_rx_max_blks, 13 for pn_tx_max_blks*/ +#define AM65_CPSW_PN_TX_RX_MAX_BLKS_IET 0xD07 + +/* Slave IET Stats. register offsets */ +#define AM65_CPSW_STATN_IET_RX_ASSEMBLY_ERROR 0x140 +#define AM65_CPSW_STATN_IET_RX_ASSEMBLY_OK 0x144 +#define AM65_CPSW_STATN_IET_RX_SMD_ERROR 0x148 +#define AM65_CPSW_STATN_IET_RX_FRAG 0x14c +#define AM65_CPSW_STATN_IET_TX_HOLD 0x150 +#define AM65_CPSW_STATN_IET_TX_FRAG 0x154 + +/* number of priority queues per port FIFO */ +#define AM65_CPSW_PN_FIFO_PRIO_NUM 8 #endif /* AM65_CPSW_QOS_H_ */ diff --git a/tools/testing/selftests/net/forwarding/ethtool_mm.sh b/tools/testing/selftests/net/forwarding/ethtool_mm.sh index 39e736f30322..50d5bfb17ef1 100755 --- a/tools/testing/selftests/net/forwarding/ethtool_mm.sh +++ b/tools/testing/selftests/net/forwarding/ethtool_mm.sh @@ -25,6 +25,10 @@ traffic_test() local after= local delta= + if [ ${has_pmac_stats[$if]} = false ]; then + src="aggregate" + fi + before=$(ethtool_std_stats_get $if "eth-mac" "FramesTransmittedOK" $src) $MZ $if -q -c $num_pkts -p 64 -b bcast -t ip -R $PREEMPTIBLE_PRIO @@ -155,15 +159,48 @@ manual_failed_verification_h2_to_h1() manual_failed_verification $h2 $h1 } +smallest_supported_add_frag_size() +{ + local iface=$1 + local rx_min_frag_size= + + rx_min_frag_size=$(ethtool --json --show-mm $iface | \ + jq '.[]."rx-min-frag-size"') + + if [ $rx_min_frag_size -le 60 ]; then + echo 0 + elif [ $rx_min_frag_size -le 124 ]; then + echo 1 + elif [ $rx_min_frag_size -le 188 ]; then + echo 2 + elif [ $rx_min_frag_size -le 252 ]; then + echo 3 + else + echo "$iface: RX min frag size $rx_min_frag_size cannot be advertised over LLDP" + exit 1 + fi +} + +expected_add_frag_size() +{ + local iface=$1 + local requested=$2 + local min=$(smallest_supported_add_frag_size $iface) + + [ $requested -le $min ] && echo $min || echo $requested +} + lldp_change_add_frag_size() { local add_frag_size=$1 + local pattern= lldptool -T -i $h1 -V addEthCaps addFragSize=$add_frag_size >/dev/null # Wait for TLVs to be received sleep 2 - lldptool -i $h2 -t -n -V addEthCaps | \ - grep -q "Additional fragment size: $add_frag_size" + pattern=$(printf "Additional fragment size: %d" \ + $(expected_add_frag_size $h1 $add_frag_size)) + lldptool -i $h2 -t -n -V addEthCaps | grep -q "$pattern" } lldp() @@ -284,6 +321,13 @@ for netif in ${NETIFS[@]}; do echo "SKIP: $netif does not support MAC Merge" exit $ksft_skip fi + + if check_ethtool_pmac_std_stats_support $netif eth-mac; then + has_pmac_stats[$netif]=true + else + has_pmac_stats[$netif]=false + echo "$netif does not report pMAC statistics, falling back to aggregate" + fi done trap cleanup EXIT diff --git a/tools/testing/selftests/net/forwarding/lib.sh b/tools/testing/selftests/net/forwarding/lib.sh index e3740163c384..69ef2a40df21 100755 --- a/tools/testing/selftests/net/forwarding/lib.sh +++ b/tools/testing/selftests/net/forwarding/lib.sh @@ -155,6 +155,15 @@ check_ethtool_counter_group_support() fi } +check_ethtool_pmac_std_stats_support() +{ + local dev=$1; shift + local grp=$1; shift + + [ 0 -ne $(ethtool --json -S $dev --all-groups --src pmac 2>/dev/null \ + | jq ".[].\"$grp\" | length") ] +} + check_locked_port_support() { if ! bridge -d link show | grep -q " locked"; then |