diff options
Diffstat (limited to 'drivers/net/ethernet/mellanox/mlx5')
57 files changed, 3535 insertions, 1189 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index 9ea867a45764..a3773a8177ed 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -17,7 +17,7 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \ fs_counters.o fs_ft_pool.o rl.o lag/debugfs.o lag/lag.o dev.o events.o wq.o lib/gid.o \ lib/devcom.o lib/pci_vsc.o lib/dm.o lib/fs_ttc.o diag/fs_tracepoint.o \ diag/fw_tracer.o diag/crdump.o devlink.o diag/rsc_dump.o \ - fw_reset.o qos.o lib/tout.o + fw_reset.o qos.o lib/tout.o lib/aso.o # # Netdev basic @@ -28,7 +28,8 @@ mlx5_core-$(CONFIG_MLX5_CORE_EN) += en/rqt.o en/tir.o en/rss.o en/rx_res.o \ en_selftest.o en/port.o en/monitor_stats.o en/health.o \ en/reporter_tx.o en/reporter_rx.o en/params.o en/xsk/pool.o \ en/xsk/setup.o en/xsk/rx.o en/xsk/tx.o en/devlink.o en/ptp.o \ - en/qos.o en/trap.o en/fs_tt_redirect.o en/selq.o lib/crypto.o + en/qos.o en/htb.o en/trap.o en/fs_tt_redirect.o en/selq.o \ + lib/crypto.o # # Netdev extra @@ -45,7 +46,8 @@ mlx5_core-$(CONFIG_MLX5_CLS_ACT) += en_tc.o en/rep/tc.o en/rep/neigh.o \ esw/indir_table.o en/tc_tun_encap.o \ en/tc_tun_vxlan.o en/tc_tun_gre.o en/tc_tun_geneve.o \ en/tc_tun_mplsoudp.o diag/en_tc_tracepoint.o \ - en/tc/post_act.o en/tc/int_port.o + en/tc/post_act.o en/tc/int_port.o en/tc/meter.o \ + en/tc/post_meter.o mlx5_core-$(CONFIG_MLX5_CLS_ACT) += en/tc/act/act.o en/tc/act/drop.o en/tc/act/trap.o \ en/tc/act/accept.o en/tc/act/mark.o en/tc/act/goto.o \ @@ -53,7 +55,7 @@ mlx5_core-$(CONFIG_MLX5_CLS_ACT) += en/tc/act/act.o en/tc/act/drop.o en/tc/a en/tc/act/vlan.o en/tc/act/vlan_mangle.o en/tc/act/mpls.o \ en/tc/act/mirred.o en/tc/act/mirred_nic.o \ en/tc/act/ct.o en/tc/act/sample.o en/tc/act/ptype.o \ - en/tc/act/redirect_ingress.o + en/tc/act/redirect_ingress.o en/tc/act/police.o ifneq ($(CONFIG_MLX5_TC_CT),) mlx5_core-y += en/tc_ct.o en/tc/ct_fs_dmfs.o @@ -67,7 +69,7 @@ mlx5_core-$(CONFIG_MLX5_TC_SAMPLE) += en/tc/sample.o # mlx5_core-$(CONFIG_MLX5_ESWITCH) += eswitch.o eswitch_offloads.o eswitch_offloads_termtbl.o \ ecpf.o rdma.o esw/legacy.o \ - esw/devlink_port.o esw/vporttbl.o esw/qos.o + esw/debugfs.o esw/devlink_port.o esw/vporttbl.o esw/qos.o mlx5_core-$(CONFIG_MLX5_ESWITCH) += esw/acl/helper.o \ esw/acl/egress_lgcy.o esw/acl/egress_ofld.o \ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c b/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c index 9caa1b52321b..3e232a65a0c3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c @@ -166,6 +166,28 @@ static const struct file_operations stats_fops = { .write = average_write, }; +static ssize_t slots_read(struct file *filp, char __user *buf, size_t count, + loff_t *pos) +{ + struct mlx5_cmd *cmd; + char tbuf[6]; + int weight; + int field; + int ret; + + cmd = filp->private_data; + weight = bitmap_weight(&cmd->bitmask, cmd->max_reg_cmds); + field = cmd->max_reg_cmds - weight; + ret = snprintf(tbuf, sizeof(tbuf), "%d\n", field); + return simple_read_from_buffer(buf, count, pos, tbuf, ret); +} + +static const struct file_operations slots_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = slots_read, +}; + void mlx5_cmdif_debugfs_init(struct mlx5_core_dev *dev) { struct mlx5_cmd_stats *stats; @@ -176,6 +198,8 @@ void mlx5_cmdif_debugfs_init(struct mlx5_core_dev *dev) cmd = &dev->priv.dbg.cmdif_debugfs; *cmd = debugfs_create_dir("commands", dev->priv.dbg.dbg_root); + debugfs_create_file("slots_inuse", 0400, *cmd, &dev->cmd, &slots_fops); + for (i = 0; i < MLX5_CMD_OP_MAX; i++) { stats = &dev->cmd.stats[i]; namep = mlx5_command_str(i); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/dev.c b/drivers/net/ethernet/mellanox/mlx5/core/dev.c index 50422b56a64d..ccf2068d2e79 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/dev.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/dev.c @@ -335,13 +335,16 @@ static void del_adev(struct auxiliary_device *adev) int mlx5_attach_device(struct mlx5_core_dev *dev) { + struct devlink *devlink = priv_to_devlink(dev); struct mlx5_priv *priv = &dev->priv; struct auxiliary_device *adev; struct auxiliary_driver *adrv; int ret = 0, i; + devl_lock(devlink); mutex_lock(&mlx5_intf_mutex); priv->flags &= ~MLX5_PRIV_FLAGS_DETACH; + priv->flags |= MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; for (i = 0; i < ARRAY_SIZE(mlx5_adev_devices); i++) { if (!priv->adev[i]) { bool is_supported = false; @@ -389,19 +392,24 @@ int mlx5_attach_device(struct mlx5_core_dev *dev) break; } } + priv->flags &= ~MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; mutex_unlock(&mlx5_intf_mutex); + devl_unlock(devlink); return ret; } void mlx5_detach_device(struct mlx5_core_dev *dev) { + struct devlink *devlink = priv_to_devlink(dev); struct mlx5_priv *priv = &dev->priv; struct auxiliary_device *adev; struct auxiliary_driver *adrv; pm_message_t pm = {}; int i; + devl_lock(devlink); mutex_lock(&mlx5_intf_mutex); + priv->flags |= MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; for (i = ARRAY_SIZE(mlx5_adev_devices) - 1; i >= 0; i--) { if (!priv->adev[i]) continue; @@ -430,18 +438,24 @@ skip_suspend: del_adev(&priv->adev[i]->adev); priv->adev[i] = NULL; } + priv->flags &= ~MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; priv->flags |= MLX5_PRIV_FLAGS_DETACH; mutex_unlock(&mlx5_intf_mutex); + devl_unlock(devlink); } int mlx5_register_device(struct mlx5_core_dev *dev) { + struct devlink *devlink; int ret; + devlink = priv_to_devlink(dev); + devl_lock(devlink); mutex_lock(&mlx5_intf_mutex); dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV; ret = mlx5_rescan_drivers_locked(dev); mutex_unlock(&mlx5_intf_mutex); + devl_unlock(devlink); if (ret) mlx5_unregister_device(dev); @@ -450,10 +464,15 @@ int mlx5_register_device(struct mlx5_core_dev *dev) void mlx5_unregister_device(struct mlx5_core_dev *dev) { + struct devlink *devlink; + + devlink = priv_to_devlink(dev); + devl_lock(devlink); mutex_lock(&mlx5_intf_mutex); dev->priv.flags = MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV; mlx5_rescan_drivers_locked(dev); mutex_unlock(&mlx5_intf_mutex); + devl_unlock(devlink); } static int add_drivers(struct mlx5_core_dev *dev) @@ -526,16 +545,22 @@ del_adev: int mlx5_rescan_drivers_locked(struct mlx5_core_dev *dev) { struct mlx5_priv *priv = &dev->priv; + int err = 0; lockdep_assert_held(&mlx5_intf_mutex); if (priv->flags & MLX5_PRIV_FLAGS_DETACH) return 0; + priv->flags |= MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; delete_drivers(dev); if (priv->flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV) - return 0; + goto out; + + err = add_drivers(dev); - return add_drivers(dev); +out: + priv->flags &= ~MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; + return err; } bool mlx5_same_hw_devs(struct mlx5_core_dev *dev, struct mlx5_core_dev *peer_dev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index b6c15efe92ad..b07228f69b91 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -321,7 +321,8 @@ struct mlx5e_params { u8 num_tc; struct netdev_tc_txq tc_to_txq[TC_MAX_QUEUE]; struct { - struct mlx5e_mqprio_rl *rl; + u64 max_rate[TC_MAX_QUEUE]; + u32 hw_id[TC_MAX_QUEUE]; } channel; } mqprio; bool rx_cqe_compress_def; @@ -898,16 +899,8 @@ struct mlx5e_scratchpad { cpumask_var_t cpumask; }; -struct mlx5e_htb { - DECLARE_HASHTABLE(qos_tc2node, order_base_2(MLX5E_QOS_MAX_LEAF_NODES)); - DECLARE_BITMAP(qos_used_qids, MLX5E_QOS_MAX_LEAF_NODES); - struct mlx5e_sq_stats **qos_sq_stats; - u16 max_qos_sqs; - u16 maj_id; - u16 defcls; -}; - struct mlx5e_trap; +struct mlx5e_htb; struct mlx5e_priv { /* priv data path fields - start */ @@ -945,6 +938,8 @@ struct mlx5e_priv { struct mlx5e_channel_stats **channel_stats; struct mlx5e_channel_stats trap_stats; struct mlx5e_ptp_stats ptp_stats; + struct mlx5e_sq_stats **htb_qos_sq_stats; + u16 htb_max_qos_sqs; u16 stats_nch; u16 max_nch; u8 max_opened_tc; @@ -976,7 +971,7 @@ struct mlx5e_priv { struct mlx5e_hv_vhca_stats_agent stats_agent; #endif struct mlx5e_scratchpad scratchpad; - struct mlx5e_htb htb; + struct mlx5e_htb *htb; struct mlx5e_mqprio_rl *mqprio_rl; }; @@ -1181,7 +1176,8 @@ int mlx5e_ethtool_get_sset_count(struct mlx5e_priv *priv, int sset); void mlx5e_ethtool_get_ethtool_stats(struct mlx5e_priv *priv, struct ethtool_stats *stats, u64 *data); void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv, - struct ethtool_ringparam *param); + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param); int mlx5e_ethtool_set_ringparam(struct mlx5e_priv *priv, struct ethtool_ringparam *param); void mlx5e_ethtool_get_channels(struct mlx5e_priv *priv, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c index ae52e7f38306..b69f9d10ccbd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c @@ -21,6 +21,7 @@ int mlx5e_devlink_port_register(struct mlx5e_priv *priv) struct netdev_phys_item_id ppid = {}; struct devlink_port *dl_port; unsigned int dl_port_index; + int ret; if (mlx5_core_is_pf(priv->mdev)) { attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL; @@ -41,7 +42,13 @@ int mlx5e_devlink_port_register(struct mlx5e_priv *priv) memset(dl_port, 0, sizeof(*dl_port)); devlink_port_attrs_set(dl_port, &attrs); - return devlink_port_register(devlink, dl_port, dl_port_index); + if (!(priv->mdev->priv.flags & MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW)) + devl_lock(devlink); + ret = devl_port_register(devlink, dl_port, dl_port_index); + if (!(priv->mdev->priv.flags & MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW)) + devl_unlock(devlink); + + return ret; } void mlx5e_devlink_port_type_eth_set(struct mlx5e_priv *priv) @@ -54,8 +61,13 @@ void mlx5e_devlink_port_type_eth_set(struct mlx5e_priv *priv) void mlx5e_devlink_port_unregister(struct mlx5e_priv *priv) { struct devlink_port *dl_port = mlx5e_devlink_get_dl_port(priv); + struct devlink *devlink = priv_to_devlink(priv->mdev); - devlink_port_unregister(dl_port); + if (!(priv->mdev->priv.flags & MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW)) + devl_lock(devlink); + devl_port_unregister(dl_port); + if (!(priv->mdev->priv.flags & MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW)) + devl_unlock(devlink); } struct devlink_port *mlx5e_get_devlink_port(struct net_device *dev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/htb.c b/drivers/net/ethernet/mellanox/mlx5/core/en/htb.c new file mode 100644 index 000000000000..6dac76fa58a3 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/htb.c @@ -0,0 +1,722 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#include <net/pkt_cls.h> +#include "htb.h" +#include "en.h" +#include "../qos.h" + +struct mlx5e_qos_node { + struct hlist_node hnode; + struct mlx5e_qos_node *parent; + u64 rate; + u32 bw_share; + u32 max_average_bw; + u32 hw_id; + u32 classid; /* 16-bit, except root. */ + u16 qid; +}; + +struct mlx5e_htb { + DECLARE_HASHTABLE(qos_tc2node, order_base_2(MLX5E_QOS_MAX_LEAF_NODES)); + DECLARE_BITMAP(qos_used_qids, MLX5E_QOS_MAX_LEAF_NODES); + struct mlx5_core_dev *mdev; + struct net_device *netdev; + struct mlx5e_priv *priv; + struct mlx5e_selq *selq; +}; + +#define MLX5E_QOS_QID_INNER 0xffff +#define MLX5E_HTB_CLASSID_ROOT 0xffffffff + +/* Software representation of the QoS tree */ + +int mlx5e_htb_enumerate_leaves(struct mlx5e_htb *htb, mlx5e_fp_htb_enumerate callback, void *data) +{ + struct mlx5e_qos_node *node = NULL; + int bkt, err; + + hash_for_each(htb->qos_tc2node, bkt, node, hnode) { + if (node->qid == MLX5E_QOS_QID_INNER) + continue; + err = callback(data, node->qid, node->hw_id); + if (err) + return err; + } + return 0; +} + +int mlx5e_htb_cur_leaf_nodes(struct mlx5e_htb *htb) +{ + int last; + + last = find_last_bit(htb->qos_used_qids, mlx5e_qos_max_leaf_nodes(htb->mdev)); + return last == mlx5e_qos_max_leaf_nodes(htb->mdev) ? 0 : last + 1; +} + +static int mlx5e_htb_find_unused_qos_qid(struct mlx5e_htb *htb) +{ + int size = mlx5e_qos_max_leaf_nodes(htb->mdev); + struct mlx5e_priv *priv = htb->priv; + int res; + + WARN_ONCE(!mutex_is_locked(&priv->state_lock), "%s: state_lock is not held\n", __func__); + res = find_first_zero_bit(htb->qos_used_qids, size); + + return res == size ? -ENOSPC : res; +} + +static struct mlx5e_qos_node * +mlx5e_htb_node_create_leaf(struct mlx5e_htb *htb, u16 classid, u16 qid, + struct mlx5e_qos_node *parent) +{ + struct mlx5e_qos_node *node; + + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (!node) + return ERR_PTR(-ENOMEM); + + node->parent = parent; + + node->qid = qid; + __set_bit(qid, htb->qos_used_qids); + + node->classid = classid; + hash_add_rcu(htb->qos_tc2node, &node->hnode, classid); + + mlx5e_update_tx_netdev_queues(htb->priv); + + return node; +} + +static struct mlx5e_qos_node *mlx5e_htb_node_create_root(struct mlx5e_htb *htb) +{ + struct mlx5e_qos_node *node; + + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (!node) + return ERR_PTR(-ENOMEM); + + node->qid = MLX5E_QOS_QID_INNER; + node->classid = MLX5E_HTB_CLASSID_ROOT; + hash_add_rcu(htb->qos_tc2node, &node->hnode, node->classid); + + return node; +} + +static struct mlx5e_qos_node *mlx5e_htb_node_find(struct mlx5e_htb *htb, u32 classid) +{ + struct mlx5e_qos_node *node = NULL; + + hash_for_each_possible(htb->qos_tc2node, node, hnode, classid) { + if (node->classid == classid) + break; + } + + return node; +} + +static struct mlx5e_qos_node *mlx5e_htb_node_find_rcu(struct mlx5e_htb *htb, u32 classid) +{ + struct mlx5e_qos_node *node = NULL; + + hash_for_each_possible_rcu(htb->qos_tc2node, node, hnode, classid) { + if (node->classid == classid) + break; + } + + return node; +} + +static void mlx5e_htb_node_delete(struct mlx5e_htb *htb, struct mlx5e_qos_node *node) +{ + hash_del_rcu(&node->hnode); + if (node->qid != MLX5E_QOS_QID_INNER) { + __clear_bit(node->qid, htb->qos_used_qids); + mlx5e_update_tx_netdev_queues(htb->priv); + } + /* Make sure this qid is no longer selected by mlx5e_select_queue, so + * that mlx5e_reactivate_qos_sq can safely restart the netdev TX queue. + */ + synchronize_net(); + kfree(node); +} + +/* TX datapath API */ + +int mlx5e_htb_get_txq_by_classid(struct mlx5e_htb *htb, u16 classid) +{ + struct mlx5e_qos_node *node; + u16 qid; + int res; + + rcu_read_lock(); + + node = mlx5e_htb_node_find_rcu(htb, classid); + if (!node) { + res = -ENOENT; + goto out; + } + qid = READ_ONCE(node->qid); + if (qid == MLX5E_QOS_QID_INNER) { + res = -EINVAL; + goto out; + } + res = mlx5e_qid_from_qos(&htb->priv->channels, qid); + +out: + rcu_read_unlock(); + return res; +} + +/* HTB TC handlers */ + +static int +mlx5e_htb_root_add(struct mlx5e_htb *htb, u16 htb_maj_id, u16 htb_defcls, + struct netlink_ext_ack *extack) +{ + struct mlx5e_priv *priv = htb->priv; + struct mlx5e_qos_node *root; + bool opened; + int err; + + qos_dbg(htb->mdev, "TC_HTB_CREATE handle %04x:, default :%04x\n", htb_maj_id, htb_defcls); + + mlx5e_selq_prepare_htb(htb->selq, htb_maj_id, htb_defcls); + + opened = test_bit(MLX5E_STATE_OPENED, &priv->state); + if (opened) { + err = mlx5e_qos_alloc_queues(priv, &priv->channels); + if (err) + goto err_cancel_selq; + } + + root = mlx5e_htb_node_create_root(htb); + if (IS_ERR(root)) { + err = PTR_ERR(root); + goto err_free_queues; + } + + err = mlx5_qos_create_root_node(htb->mdev, &root->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error. Try upgrading firmware."); + goto err_sw_node_delete; + } + + mlx5e_selq_apply(htb->selq); + + return 0; + +err_sw_node_delete: + mlx5e_htb_node_delete(htb, root); + +err_free_queues: + if (opened) + mlx5e_qos_close_all_queues(&priv->channels); +err_cancel_selq: + mlx5e_selq_cancel(htb->selq); + return err; +} + +static int mlx5e_htb_root_del(struct mlx5e_htb *htb) +{ + struct mlx5e_priv *priv = htb->priv; + struct mlx5e_qos_node *root; + int err; + + qos_dbg(htb->mdev, "TC_HTB_DESTROY\n"); + + /* Wait until real_num_tx_queues is updated for mlx5e_select_queue, + * so that we can safely switch to its non-HTB non-PTP fastpath. + */ + synchronize_net(); + + mlx5e_selq_prepare_htb(htb->selq, 0, 0); + mlx5e_selq_apply(htb->selq); + + root = mlx5e_htb_node_find(htb, MLX5E_HTB_CLASSID_ROOT); + if (!root) { + qos_err(htb->mdev, "Failed to find the root node in the QoS tree\n"); + return -ENOENT; + } + err = mlx5_qos_destroy_node(htb->mdev, root->hw_id); + if (err) + qos_err(htb->mdev, "Failed to destroy root node %u, err = %d\n", + root->hw_id, err); + mlx5e_htb_node_delete(htb, root); + + mlx5e_qos_deactivate_all_queues(&priv->channels); + mlx5e_qos_close_all_queues(&priv->channels); + + return err; +} + +static int mlx5e_htb_convert_rate(struct mlx5e_htb *htb, u64 rate, + struct mlx5e_qos_node *parent, u32 *bw_share) +{ + u64 share = 0; + + while (parent->classid != MLX5E_HTB_CLASSID_ROOT && !parent->max_average_bw) + parent = parent->parent; + + if (parent->max_average_bw) + share = div64_u64(div_u64(rate * 100, BYTES_IN_MBIT), + parent->max_average_bw); + else + share = 101; + + *bw_share = share == 0 ? 1 : share > 100 ? 0 : share; + + qos_dbg(htb->mdev, "Convert: rate %llu, parent ceil %llu -> bw_share %u\n", + rate, (u64)parent->max_average_bw * BYTES_IN_MBIT, *bw_share); + + return 0; +} + +static void mlx5e_htb_convert_ceil(struct mlx5e_htb *htb, u64 ceil, u32 *max_average_bw) +{ + /* Hardware treats 0 as "unlimited", set at least 1. */ + *max_average_bw = max_t(u32, div_u64(ceil, BYTES_IN_MBIT), 1); + + qos_dbg(htb->mdev, "Convert: ceil %llu -> max_average_bw %u\n", + ceil, *max_average_bw); +} + +int +mlx5e_htb_leaf_alloc_queue(struct mlx5e_htb *htb, u16 classid, + u32 parent_classid, u64 rate, u64 ceil, + struct netlink_ext_ack *extack) +{ + struct mlx5e_qos_node *node, *parent; + struct mlx5e_priv *priv = htb->priv; + int qid; + int err; + + qos_dbg(htb->mdev, "TC_HTB_LEAF_ALLOC_QUEUE classid %04x, parent %04x, rate %llu, ceil %llu\n", + classid, parent_classid, rate, ceil); + + qid = mlx5e_htb_find_unused_qos_qid(htb); + if (qid < 0) { + NL_SET_ERR_MSG_MOD(extack, "Maximum amount of leaf classes is reached."); + return qid; + } + + parent = mlx5e_htb_node_find(htb, parent_classid); + if (!parent) + return -EINVAL; + + node = mlx5e_htb_node_create_leaf(htb, classid, qid, parent); + if (IS_ERR(node)) + return PTR_ERR(node); + + node->rate = rate; + mlx5e_htb_convert_rate(htb, rate, node->parent, &node->bw_share); + mlx5e_htb_convert_ceil(htb, ceil, &node->max_average_bw); + + err = mlx5_qos_create_leaf_node(htb->mdev, node->parent->hw_id, + node->bw_share, node->max_average_bw, + &node->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); + qos_err(htb->mdev, "Failed to create a leaf node (class %04x), err = %d\n", + classid, err); + mlx5e_htb_node_delete(htb, node); + return err; + } + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + err = mlx5e_open_qos_sq(priv, &priv->channels, node->qid, node->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); + qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", + classid, err); + } else { + mlx5e_activate_qos_sq(priv, node->qid, node->hw_id); + } + } + + return mlx5e_qid_from_qos(&priv->channels, node->qid); +} + +int +mlx5e_htb_leaf_to_inner(struct mlx5e_htb *htb, u16 classid, u16 child_classid, + u64 rate, u64 ceil, struct netlink_ext_ack *extack) +{ + struct mlx5e_qos_node *node, *child; + struct mlx5e_priv *priv = htb->priv; + int err, tmp_err; + u32 new_hw_id; + u16 qid; + + qos_dbg(htb->mdev, "TC_HTB_LEAF_TO_INNER classid %04x, upcoming child %04x, rate %llu, ceil %llu\n", + classid, child_classid, rate, ceil); + + node = mlx5e_htb_node_find(htb, classid); + if (!node) + return -ENOENT; + + err = mlx5_qos_create_inner_node(htb->mdev, node->parent->hw_id, + node->bw_share, node->max_average_bw, + &new_hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating an inner node."); + qos_err(htb->mdev, "Failed to create an inner node (class %04x), err = %d\n", + classid, err); + return err; + } + + /* Intentionally reuse the qid for the upcoming first child. */ + child = mlx5e_htb_node_create_leaf(htb, child_classid, node->qid, node); + if (IS_ERR(child)) { + err = PTR_ERR(child); + goto err_destroy_hw_node; + } + + child->rate = rate; + mlx5e_htb_convert_rate(htb, rate, node, &child->bw_share); + mlx5e_htb_convert_ceil(htb, ceil, &child->max_average_bw); + + err = mlx5_qos_create_leaf_node(htb->mdev, new_hw_id, child->bw_share, + child->max_average_bw, &child->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); + qos_err(htb->mdev, "Failed to create a leaf node (class %04x), err = %d\n", + classid, err); + goto err_delete_sw_node; + } + + /* No fail point. */ + + qid = node->qid; + /* Pairs with mlx5e_htb_get_txq_by_classid. */ + WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + mlx5e_deactivate_qos_sq(priv, qid); + mlx5e_close_qos_sq(priv, qid); + } + + err = mlx5_qos_destroy_node(htb->mdev, node->hw_id); + if (err) /* Not fatal. */ + qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", + node->hw_id, classid, err); + + node->hw_id = new_hw_id; + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + err = mlx5e_open_qos_sq(priv, &priv->channels, child->qid, child->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); + qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", + classid, err); + } else { + mlx5e_activate_qos_sq(priv, child->qid, child->hw_id); + } + } + + return 0; + +err_delete_sw_node: + child->qid = MLX5E_QOS_QID_INNER; + mlx5e_htb_node_delete(htb, child); + +err_destroy_hw_node: + tmp_err = mlx5_qos_destroy_node(htb->mdev, new_hw_id); + if (tmp_err) /* Not fatal. */ + qos_warn(htb->mdev, "Failed to roll back creation of an inner node %u (class %04x), err = %d\n", + new_hw_id, classid, tmp_err); + return err; +} + +static struct mlx5e_qos_node *mlx5e_htb_node_find_by_qid(struct mlx5e_htb *htb, u16 qid) +{ + struct mlx5e_qos_node *node = NULL; + int bkt; + + hash_for_each(htb->qos_tc2node, bkt, node, hnode) + if (node->qid == qid) + break; + + return node; +} + +int mlx5e_htb_leaf_del(struct mlx5e_htb *htb, u16 *classid, + struct netlink_ext_ack *extack) +{ + struct mlx5e_priv *priv = htb->priv; + struct mlx5e_qos_node *node; + struct netdev_queue *txq; + u16 qid, moved_qid; + bool opened; + int err; + + qos_dbg(htb->mdev, "TC_HTB_LEAF_DEL classid %04x\n", *classid); + + node = mlx5e_htb_node_find(htb, *classid); + if (!node) + return -ENOENT; + + /* Store qid for reuse. */ + qid = node->qid; + + opened = test_bit(MLX5E_STATE_OPENED, &priv->state); + if (opened) { + txq = netdev_get_tx_queue(htb->netdev, + mlx5e_qid_from_qos(&priv->channels, qid)); + mlx5e_deactivate_qos_sq(priv, qid); + mlx5e_close_qos_sq(priv, qid); + } + + err = mlx5_qos_destroy_node(htb->mdev, node->hw_id); + if (err) /* Not fatal. */ + qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", + node->hw_id, *classid, err); + + mlx5e_htb_node_delete(htb, node); + + moved_qid = mlx5e_htb_cur_leaf_nodes(htb); + + if (moved_qid == 0) { + /* The last QoS SQ was just destroyed. */ + if (opened) + mlx5e_reactivate_qos_sq(priv, qid, txq); + return 0; + } + moved_qid--; + + if (moved_qid < qid) { + /* The highest QoS SQ was just destroyed. */ + WARN(moved_qid != qid - 1, "Gaps in queue numeration: destroyed queue %u, the highest queue is %u", + qid, moved_qid); + if (opened) + mlx5e_reactivate_qos_sq(priv, qid, txq); + return 0; + } + + WARN(moved_qid == qid, "Can't move node with qid %u to itself", qid); + qos_dbg(htb->mdev, "Moving QoS SQ %u to %u\n", moved_qid, qid); + + node = mlx5e_htb_node_find_by_qid(htb, moved_qid); + WARN(!node, "Could not find a node with qid %u to move to queue %u", + moved_qid, qid); + + /* Stop traffic to the old queue. */ + WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); + __clear_bit(moved_qid, priv->htb->qos_used_qids); + + if (opened) { + txq = netdev_get_tx_queue(htb->netdev, + mlx5e_qid_from_qos(&priv->channels, moved_qid)); + mlx5e_deactivate_qos_sq(priv, moved_qid); + mlx5e_close_qos_sq(priv, moved_qid); + } + + /* Prevent packets from the old class from getting into the new one. */ + mlx5e_reset_qdisc(htb->netdev, moved_qid); + + __set_bit(qid, htb->qos_used_qids); + WRITE_ONCE(node->qid, qid); + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + err = mlx5e_open_qos_sq(priv, &priv->channels, node->qid, node->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); + qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x) while moving qid %u to %u, err = %d\n", + node->classid, moved_qid, qid, err); + } else { + mlx5e_activate_qos_sq(priv, node->qid, node->hw_id); + } + } + + mlx5e_update_tx_netdev_queues(priv); + if (opened) + mlx5e_reactivate_qos_sq(priv, moved_qid, txq); + + *classid = node->classid; + return 0; +} + +int +mlx5e_htb_leaf_del_last(struct mlx5e_htb *htb, u16 classid, bool force, + struct netlink_ext_ack *extack) +{ + struct mlx5e_qos_node *node, *parent; + struct mlx5e_priv *priv = htb->priv; + u32 old_hw_id, new_hw_id; + int err, saved_err = 0; + u16 qid; + + qos_dbg(htb->mdev, "TC_HTB_LEAF_DEL_LAST%s classid %04x\n", + force ? "_FORCE" : "", classid); + + node = mlx5e_htb_node_find(htb, classid); + if (!node) + return -ENOENT; + + err = mlx5_qos_create_leaf_node(htb->mdev, node->parent->parent->hw_id, + node->parent->bw_share, + node->parent->max_average_bw, + &new_hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); + qos_err(htb->mdev, "Failed to create a leaf node (class %04x), err = %d\n", + classid, err); + if (!force) + return err; + saved_err = err; + } + + /* Store qid for reuse and prevent clearing the bit. */ + qid = node->qid; + /* Pairs with mlx5e_htb_get_txq_by_classid. */ + WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + mlx5e_deactivate_qos_sq(priv, qid); + mlx5e_close_qos_sq(priv, qid); + } + + /* Prevent packets from the old class from getting into the new one. */ + mlx5e_reset_qdisc(htb->netdev, qid); + + err = mlx5_qos_destroy_node(htb->mdev, node->hw_id); + if (err) /* Not fatal. */ + qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", + node->hw_id, classid, err); + + parent = node->parent; + mlx5e_htb_node_delete(htb, node); + + node = parent; + WRITE_ONCE(node->qid, qid); + + /* Early return on error in force mode. Parent will still be an inner + * node to be deleted by a following delete operation. + */ + if (saved_err) + return saved_err; + + old_hw_id = node->hw_id; + node->hw_id = new_hw_id; + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + err = mlx5e_open_qos_sq(priv, &priv->channels, node->qid, node->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); + qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", + classid, err); + } else { + mlx5e_activate_qos_sq(priv, node->qid, node->hw_id); + } + } + + err = mlx5_qos_destroy_node(htb->mdev, old_hw_id); + if (err) /* Not fatal. */ + qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", + node->hw_id, classid, err); + + return 0; +} + +static int +mlx5e_htb_update_children(struct mlx5e_htb *htb, struct mlx5e_qos_node *node, + struct netlink_ext_ack *extack) +{ + struct mlx5e_qos_node *child; + int err = 0; + int bkt; + + hash_for_each(htb->qos_tc2node, bkt, child, hnode) { + u32 old_bw_share = child->bw_share; + int err_one; + + if (child->parent != node) + continue; + + mlx5e_htb_convert_rate(htb, child->rate, node, &child->bw_share); + if (child->bw_share == old_bw_share) + continue; + + err_one = mlx5_qos_update_node(htb->mdev, child->hw_id, child->bw_share, + child->max_average_bw, child->hw_id); + if (!err && err_one) { + err = err_one; + + NL_SET_ERR_MSG_MOD(extack, "Firmware error when modifying a child node."); + qos_err(htb->mdev, "Failed to modify a child node (class %04x), err = %d\n", + node->classid, err); + } + } + + return err; +} + +int +mlx5e_htb_node_modify(struct mlx5e_htb *htb, u16 classid, u64 rate, u64 ceil, + struct netlink_ext_ack *extack) +{ + u32 bw_share, max_average_bw; + struct mlx5e_qos_node *node; + bool ceil_changed = false; + int err; + + qos_dbg(htb->mdev, "TC_HTB_LEAF_MODIFY classid %04x, rate %llu, ceil %llu\n", + classid, rate, ceil); + + node = mlx5e_htb_node_find(htb, classid); + if (!node) + return -ENOENT; + + node->rate = rate; + mlx5e_htb_convert_rate(htb, rate, node->parent, &bw_share); + mlx5e_htb_convert_ceil(htb, ceil, &max_average_bw); + + err = mlx5_qos_update_node(htb->mdev, node->parent->hw_id, bw_share, + max_average_bw, node->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error when modifying a node."); + qos_err(htb->mdev, "Failed to modify a node (class %04x), err = %d\n", + classid, err); + return err; + } + + if (max_average_bw != node->max_average_bw) + ceil_changed = true; + + node->bw_share = bw_share; + node->max_average_bw = max_average_bw; + + if (ceil_changed) + err = mlx5e_htb_update_children(htb, node, extack); + + return err; +} + +struct mlx5e_htb *mlx5e_htb_alloc(void) +{ + return kvzalloc(sizeof(struct mlx5e_htb), GFP_KERNEL); +} + +void mlx5e_htb_free(struct mlx5e_htb *htb) +{ + kvfree(htb); +} + +int mlx5e_htb_init(struct mlx5e_htb *htb, struct tc_htb_qopt_offload *htb_qopt, + struct net_device *netdev, struct mlx5_core_dev *mdev, + struct mlx5e_selq *selq, struct mlx5e_priv *priv) +{ + htb->mdev = mdev; + htb->netdev = netdev; + htb->selq = selq; + htb->priv = priv; + hash_init(htb->qos_tc2node); + return mlx5e_htb_root_add(htb, htb_qopt->parent_classid, htb_qopt->classid, + htb_qopt->extack); +} + +void mlx5e_htb_cleanup(struct mlx5e_htb *htb) +{ + mlx5e_htb_root_del(htb); +} + diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/htb.h b/drivers/net/ethernet/mellanox/mlx5/core/en/htb.h new file mode 100644 index 000000000000..8386f1ea4559 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/htb.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5E_EN_HTB_H_ +#define __MLX5E_EN_HTB_H_ + +#include "qos.h" + +#define MLX5E_QOS_MAX_LEAF_NODES 256 + +struct mlx5e_selq; +struct mlx5e_htb; + +typedef int (*mlx5e_fp_htb_enumerate)(void *data, u16 qid, u32 hw_id); +int mlx5e_htb_enumerate_leaves(struct mlx5e_htb *htb, mlx5e_fp_htb_enumerate callback, void *data); + +int mlx5e_htb_cur_leaf_nodes(struct mlx5e_htb *htb); + +/* TX datapath API */ +int mlx5e_htb_get_txq_by_classid(struct mlx5e_htb *htb, u16 classid); + +/* HTB TC handlers */ + +int +mlx5e_htb_leaf_alloc_queue(struct mlx5e_htb *htb, u16 classid, + u32 parent_classid, u64 rate, u64 ceil, + struct netlink_ext_ack *extack); +int +mlx5e_htb_leaf_to_inner(struct mlx5e_htb *htb, u16 classid, u16 child_classid, + u64 rate, u64 ceil, struct netlink_ext_ack *extack); +int mlx5e_htb_leaf_del(struct mlx5e_htb *htb, u16 *classid, + struct netlink_ext_ack *extack); +int +mlx5e_htb_leaf_del_last(struct mlx5e_htb *htb, u16 classid, bool force, + struct netlink_ext_ack *extack); +int +mlx5e_htb_node_modify(struct mlx5e_htb *htb, u16 classid, u64 rate, u64 ceil, + struct netlink_ext_ack *extack); +struct mlx5e_htb *mlx5e_htb_alloc(void); +void mlx5e_htb_free(struct mlx5e_htb *htb); +int mlx5e_htb_init(struct mlx5e_htb *htb, struct tc_htb_qopt_offload *htb_qopt, + struct net_device *netdev, struct mlx5_core_dev *mdev, + struct mlx5e_selq *selq, struct mlx5e_priv *priv); +void mlx5e_htb_cleanup(struct mlx5e_htb *htb); +#endif + diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c index 047f88f09203..78ad96cf4222 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c @@ -79,19 +79,49 @@ void mlx5e_skb_cb_hwtstamp_handler(struct sk_buff *skb, int hwtstamp_type, memset(skb->cb, 0, sizeof(struct mlx5e_skb_cb_hwtstamp)); } +#define PTP_WQE_CTR2IDX(val) ((val) & ptpsq->ts_cqe_ctr_mask) + +static bool mlx5e_ptp_ts_cqe_drop(struct mlx5e_ptpsq *ptpsq, u16 skb_cc, u16 skb_id) +{ + return (ptpsq->ts_cqe_ctr_mask && (skb_cc != skb_id)); +} + +static void mlx5e_ptp_skb_fifo_ts_cqe_resync(struct mlx5e_ptpsq *ptpsq, u16 skb_cc, u16 skb_id) +{ + struct skb_shared_hwtstamps hwts = {}; + struct sk_buff *skb; + + ptpsq->cq_stats->resync_event++; + + while (skb_cc != skb_id) { + skb = mlx5e_skb_fifo_pop(&ptpsq->skb_fifo); + hwts.hwtstamp = mlx5e_skb_cb_get_hwts(skb)->cqe_hwtstamp; + skb_tstamp_tx(skb, &hwts); + ptpsq->cq_stats->resync_cqe++; + skb_cc = PTP_WQE_CTR2IDX(ptpsq->skb_fifo_cc); + } +} + static void mlx5e_ptp_handle_ts_cqe(struct mlx5e_ptpsq *ptpsq, struct mlx5_cqe64 *cqe, int budget) { - struct sk_buff *skb = mlx5e_skb_fifo_pop(&ptpsq->skb_fifo); + u16 skb_id = PTP_WQE_CTR2IDX(be16_to_cpu(cqe->wqe_counter)); + u16 skb_cc = PTP_WQE_CTR2IDX(ptpsq->skb_fifo_cc); struct mlx5e_txqsq *sq = &ptpsq->txqsq; + struct sk_buff *skb; ktime_t hwtstamp; if (unlikely(MLX5E_RX_ERR_CQE(cqe))) { + skb = mlx5e_skb_fifo_pop(&ptpsq->skb_fifo); ptpsq->cq_stats->err_cqe++; goto out; } + if (mlx5e_ptp_ts_cqe_drop(ptpsq, skb_cc, skb_id)) + mlx5e_ptp_skb_fifo_ts_cqe_resync(ptpsq, skb_cc, skb_id); + + skb = mlx5e_skb_fifo_pop(&ptpsq->skb_fifo); hwtstamp = mlx5e_cqe_ts_to_ns(sq->ptp_cyc2time, sq->clock, get_cqe_ts(cqe)); mlx5e_skb_cb_hwtstamp_handler(skb, MLX5E_SKB_CB_PORT_HWTSTAMP, hwtstamp, ptpsq->cq_stats); @@ -241,6 +271,7 @@ static void mlx5e_ptp_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn) static int mlx5e_ptp_alloc_traffic_db(struct mlx5e_ptpsq *ptpsq, int numa) { int wq_sz = mlx5_wq_cyc_get_size(&ptpsq->txqsq.wq); + struct mlx5_core_dev *mdev = ptpsq->txqsq.mdev; ptpsq->skb_fifo.fifo = kvzalloc_node(array_size(wq_sz, sizeof(*ptpsq->skb_fifo.fifo)), GFP_KERNEL, numa); @@ -250,7 +281,9 @@ static int mlx5e_ptp_alloc_traffic_db(struct mlx5e_ptpsq *ptpsq, int numa) ptpsq->skb_fifo.pc = &ptpsq->skb_fifo_pc; ptpsq->skb_fifo.cc = &ptpsq->skb_fifo_cc; ptpsq->skb_fifo.mask = wq_sz - 1; - + if (MLX5_CAP_GEN_2(mdev, ts_cqe_metadata_size2wqe_counter)) + ptpsq->ts_cqe_ctr_mask = + (1 << MLX5_CAP_GEN_2(mdev, ts_cqe_metadata_size2wqe_counter)) - 1; return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h index a71a32e00ebb..92dbbec472ec 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h @@ -17,6 +17,7 @@ struct mlx5e_ptpsq { u16 skb_fifo_pc; struct mlx5e_skb_fifo skb_fifo; struct mlx5e_ptp_cq_stats *cq_stats; + u16 ts_cqe_ctr_mask; }; enum { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c index 9db677e9ca9c..2842195ee548 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c @@ -2,11 +2,16 @@ /* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ #include <net/sch_generic.h> +#include <net/pkt_cls.h> #include "en.h" #include "params.h" #include "../qos.h" +#include "en/htb.h" -#define BYTES_IN_MBIT 125000 +struct qos_sq_callback_params { + struct mlx5e_priv *priv; + struct mlx5e_channels *chs; +}; int mlx5e_qos_bytes_rate_check(struct mlx5_core_dev *mdev, u64 nbytes) { @@ -28,124 +33,14 @@ int mlx5e_qos_max_leaf_nodes(struct mlx5_core_dev *mdev) return min(MLX5E_QOS_MAX_LEAF_NODES, mlx5_qos_max_leaf_nodes(mdev)); } -int mlx5e_qos_cur_leaf_nodes(struct mlx5e_priv *priv) -{ - int last = find_last_bit(priv->htb.qos_used_qids, mlx5e_qos_max_leaf_nodes(priv->mdev)); - - return last == mlx5e_qos_max_leaf_nodes(priv->mdev) ? 0 : last + 1; -} - -/* Software representation of the QoS tree (internal to this file) */ - -static int mlx5e_find_unused_qos_qid(struct mlx5e_priv *priv) -{ - int size = mlx5e_qos_max_leaf_nodes(priv->mdev); - int res; - - WARN_ONCE(!mutex_is_locked(&priv->state_lock), "%s: state_lock is not held\n", __func__); - res = find_first_zero_bit(priv->htb.qos_used_qids, size); - - return res == size ? -ENOSPC : res; -} - -struct mlx5e_qos_node { - struct hlist_node hnode; - struct mlx5e_qos_node *parent; - u64 rate; - u32 bw_share; - u32 max_average_bw; - u32 hw_id; - u32 classid; /* 16-bit, except root. */ - u16 qid; -}; - -#define MLX5E_QOS_QID_INNER 0xffff -#define MLX5E_HTB_CLASSID_ROOT 0xffffffff - -static struct mlx5e_qos_node * -mlx5e_sw_node_create_leaf(struct mlx5e_priv *priv, u16 classid, u16 qid, - struct mlx5e_qos_node *parent) -{ - struct mlx5e_qos_node *node; - - node = kzalloc(sizeof(*node), GFP_KERNEL); - if (!node) - return ERR_PTR(-ENOMEM); - - node->parent = parent; - - node->qid = qid; - __set_bit(qid, priv->htb.qos_used_qids); - - node->classid = classid; - hash_add_rcu(priv->htb.qos_tc2node, &node->hnode, classid); - - mlx5e_update_tx_netdev_queues(priv); - - return node; -} - -static struct mlx5e_qos_node *mlx5e_sw_node_create_root(struct mlx5e_priv *priv) -{ - struct mlx5e_qos_node *node; - - node = kzalloc(sizeof(*node), GFP_KERNEL); - if (!node) - return ERR_PTR(-ENOMEM); - - node->qid = MLX5E_QOS_QID_INNER; - node->classid = MLX5E_HTB_CLASSID_ROOT; - hash_add_rcu(priv->htb.qos_tc2node, &node->hnode, node->classid); - - return node; -} - -static struct mlx5e_qos_node *mlx5e_sw_node_find(struct mlx5e_priv *priv, u32 classid) -{ - struct mlx5e_qos_node *node = NULL; - - hash_for_each_possible(priv->htb.qos_tc2node, node, hnode, classid) { - if (node->classid == classid) - break; - } - - return node; -} - -static struct mlx5e_qos_node *mlx5e_sw_node_find_rcu(struct mlx5e_priv *priv, u32 classid) -{ - struct mlx5e_qos_node *node = NULL; - - hash_for_each_possible_rcu(priv->htb.qos_tc2node, node, hnode, classid) { - if (node->classid == classid) - break; - } - - return node; -} - -static void mlx5e_sw_node_delete(struct mlx5e_priv *priv, struct mlx5e_qos_node *node) -{ - hash_del_rcu(&node->hnode); - if (node->qid != MLX5E_QOS_QID_INNER) { - __clear_bit(node->qid, priv->htb.qos_used_qids); - mlx5e_update_tx_netdev_queues(priv); - } - /* Make sure this qid is no longer selected by mlx5e_select_queue, so - * that mlx5e_reactivate_qos_sq can safely restart the netdev TX queue. - */ - synchronize_net(); - kfree(node); -} - /* TX datapath API */ -static u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid) +u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid) { /* These channel params are safe to access from the datapath, because: - * 1. This function is called only after checking priv->htb.maj_id != 0, + * 1. This function is called only after checking selq->htb_maj_id != 0, * and the number of queues can't change while HTB offload is active. - * 2. When priv->htb.maj_id becomes 0, synchronize_rcu waits for + * 2. When selq->htb_maj_id becomes 0, synchronize_rcu waits for * mlx5e_select_queue to finish while holding priv->state_lock, * preventing other code from changing the number of queues. */ @@ -154,30 +49,7 @@ static u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid) return (chs->params.num_channels + is_ptp) * mlx5e_get_dcb_num_tc(&chs->params) + qid; } -int mlx5e_get_txq_by_classid(struct mlx5e_priv *priv, u16 classid) -{ - struct mlx5e_qos_node *node; - u16 qid; - int res; - - rcu_read_lock(); - - node = mlx5e_sw_node_find_rcu(priv, classid); - if (!node) { - res = -ENOENT; - goto out; - } - qid = READ_ONCE(node->qid); - if (qid == MLX5E_QOS_QID_INNER) { - res = -EINVAL; - goto out; - } - res = mlx5e_qid_from_qos(&priv->channels, qid); - -out: - rcu_read_unlock(); - return res; -} +/* SQ lifecycle */ static struct mlx5e_txqsq *mlx5e_get_qos_sq(struct mlx5e_priv *priv, int qid) { @@ -194,10 +66,8 @@ static struct mlx5e_txqsq *mlx5e_get_qos_sq(struct mlx5e_priv *priv, int qid) return mlx5e_state_dereference(priv, qos_sqs[qid]); } -/* SQ lifecycle */ - -static int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs, - struct mlx5e_qos_node *node) +int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs, + u16 node_qid, u32 hw_id) { struct mlx5e_create_cq_param ccp = {}; struct mlx5e_txqsq __rcu **qos_sqs; @@ -210,13 +80,13 @@ static int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs params = &chs->params; - txq_ix = mlx5e_qid_from_qos(chs, node->qid); + txq_ix = mlx5e_qid_from_qos(chs, node_qid); - WARN_ON(node->qid > priv->htb.max_qos_sqs); - if (node->qid == priv->htb.max_qos_sqs) { + WARN_ON(node_qid > priv->htb_max_qos_sqs); + if (node_qid == priv->htb_max_qos_sqs) { struct mlx5e_sq_stats *stats, **stats_list = NULL; - if (priv->htb.max_qos_sqs == 0) { + if (priv->htb_max_qos_sqs == 0) { stats_list = kvcalloc(mlx5e_qos_max_leaf_nodes(priv->mdev), sizeof(*stats_list), GFP_KERNEL); @@ -229,16 +99,16 @@ static int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs return -ENOMEM; } if (stats_list) - WRITE_ONCE(priv->htb.qos_sq_stats, stats_list); - WRITE_ONCE(priv->htb.qos_sq_stats[node->qid], stats); - /* Order max_qos_sqs increment after writing the array pointer. + WRITE_ONCE(priv->htb_qos_sq_stats, stats_list); + WRITE_ONCE(priv->htb_qos_sq_stats[node_qid], stats); + /* Order htb_max_qos_sqs increment after writing the array pointer. * Pairs with smp_load_acquire in en_stats.c. */ - smp_store_release(&priv->htb.max_qos_sqs, priv->htb.max_qos_sqs + 1); + smp_store_release(&priv->htb_max_qos_sqs, priv->htb_max_qos_sqs + 1); } - ix = node->qid % params->num_channels; - qid = node->qid / params->num_channels; + ix = node_qid % params->num_channels; + qid = node_qid / params->num_channels; c = chs->c[ix]; qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs); @@ -257,8 +127,8 @@ static int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs if (err) goto err_free_sq; err = mlx5e_open_txqsq(c, priv->tisn[c->lag_port][0], txq_ix, params, - ¶m_sq, sq, 0, node->hw_id, - priv->htb.qos_sq_stats[node->qid]); + ¶m_sq, sq, 0, hw_id, + priv->htb_qos_sq_stats[node_qid]); if (err) goto err_close_cq; @@ -273,14 +143,22 @@ err_free_sq: return err; } -static void mlx5e_activate_qos_sq(struct mlx5e_priv *priv, struct mlx5e_qos_node *node) +static int mlx5e_open_qos_sq_cb_wrapper(void *data, u16 node_qid, u32 hw_id) +{ + struct qos_sq_callback_params *cb_params = data; + + return mlx5e_open_qos_sq(cb_params->priv, cb_params->chs, node_qid, hw_id); +} + +int mlx5e_activate_qos_sq(void *data, u16 node_qid, u32 hw_id) { + struct mlx5e_priv *priv = data; struct mlx5e_txqsq *sq; u16 qid; - sq = mlx5e_get_qos_sq(priv, node->qid); + sq = mlx5e_get_qos_sq(priv, node_qid); - qid = mlx5e_qid_from_qos(&priv->channels, node->qid); + qid = mlx5e_qid_from_qos(&priv->channels, node_qid); /* If it's a new queue, it will be marked as started at this point. * Stop it before updating txq2sq. @@ -295,11 +173,13 @@ static void mlx5e_activate_qos_sq(struct mlx5e_priv *priv, struct mlx5e_qos_node */ smp_wmb(); - qos_dbg(priv->mdev, "Activate QoS SQ qid %u\n", node->qid); + qos_dbg(priv->mdev, "Activate QoS SQ qid %u\n", node_qid); mlx5e_activate_txqsq(sq); + + return 0; } -static void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid) +void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid) { struct mlx5e_txqsq *sq; @@ -319,7 +199,7 @@ static void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid) smp_wmb(); } -static void mlx5e_close_qos_sq(struct mlx5e_priv *priv, u16 qid) +void mlx5e_close_qos_sq(struct mlx5e_priv *priv, u16 qid) { struct mlx5e_txqsq __rcu **qos_sqs; struct mlx5e_params *params; @@ -369,7 +249,7 @@ void mlx5e_qos_close_queues(struct mlx5e_channel *c) kvfree(qos_sqs); } -static void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs) +void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs) { int i; @@ -377,7 +257,7 @@ static void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs) mlx5e_qos_close_queues(chs->c[i]); } -static int mlx5e_qos_alloc_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs) +int mlx5e_qos_alloc_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs) { u16 qos_sqs_size; int i; @@ -413,24 +293,20 @@ err_free: int mlx5e_qos_open_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs) { - struct mlx5e_qos_node *node = NULL; - int bkt, err; - - if (!priv->htb.maj_id) - return 0; + struct qos_sq_callback_params callback_params; + int err; err = mlx5e_qos_alloc_queues(priv, chs); if (err) return err; - hash_for_each(priv->htb.qos_tc2node, bkt, node, hnode) { - if (node->qid == MLX5E_QOS_QID_INNER) - continue; - err = mlx5e_open_qos_sq(priv, chs, node); - if (err) { - mlx5e_qos_close_all_queues(chs); - return err; - } + callback_params.priv = priv; + callback_params.chs = chs; + + err = mlx5e_htb_enumerate_leaves(priv->htb, mlx5e_open_qos_sq_cb_wrapper, &callback_params); + if (err) { + mlx5e_qos_close_all_queues(chs); + return err; } return 0; @@ -438,14 +314,7 @@ int mlx5e_qos_open_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs) void mlx5e_qos_activate_queues(struct mlx5e_priv *priv) { - struct mlx5e_qos_node *node = NULL; - int bkt; - - hash_for_each(priv->htb.qos_tc2node, bkt, node, hnode) { - if (node->qid == MLX5E_QOS_QID_INNER) - continue; - mlx5e_activate_qos_sq(priv, node); - } + mlx5e_htb_enumerate_leaves(priv->htb, mlx5e_activate_qos_sq, priv); } void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c) @@ -474,7 +343,7 @@ void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c) } } -static void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs) +void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs) { int i; @@ -482,293 +351,14 @@ static void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs) mlx5e_qos_deactivate_queues(chs->c[i]); } -/* HTB API */ - -int mlx5e_htb_root_add(struct mlx5e_priv *priv, u16 htb_maj_id, u16 htb_defcls, - struct netlink_ext_ack *extack) -{ - struct mlx5e_qos_node *root; - bool opened; - int err; - - qos_dbg(priv->mdev, "TC_HTB_CREATE handle %04x:, default :%04x\n", htb_maj_id, htb_defcls); - - if (!mlx5_qos_is_supported(priv->mdev)) { - NL_SET_ERR_MSG_MOD(extack, - "Missing QoS capabilities. Try disabling SRIOV or use a supported device."); - return -EOPNOTSUPP; - } - - opened = test_bit(MLX5E_STATE_OPENED, &priv->state); - if (opened) { - mlx5e_selq_prepare(&priv->selq, &priv->channels.params, true); - - err = mlx5e_qos_alloc_queues(priv, &priv->channels); - if (err) - goto err_cancel_selq; - } - - root = mlx5e_sw_node_create_root(priv); - if (IS_ERR(root)) { - err = PTR_ERR(root); - goto err_free_queues; - } - - err = mlx5_qos_create_root_node(priv->mdev, &root->hw_id); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Firmware error. Try upgrading firmware."); - goto err_sw_node_delete; - } - - WRITE_ONCE(priv->htb.defcls, htb_defcls); - /* Order maj_id after defcls - pairs with - * mlx5e_select_queue/mlx5e_select_htb_queues. - */ - smp_store_release(&priv->htb.maj_id, htb_maj_id); - - if (opened) - mlx5e_selq_apply(&priv->selq); - - return 0; - -err_sw_node_delete: - mlx5e_sw_node_delete(priv, root); - -err_free_queues: - if (opened) - mlx5e_qos_close_all_queues(&priv->channels); -err_cancel_selq: - mlx5e_selq_cancel(&priv->selq); - return err; -} - -int mlx5e_htb_root_del(struct mlx5e_priv *priv) -{ - struct mlx5e_qos_node *root; - int err; - - qos_dbg(priv->mdev, "TC_HTB_DESTROY\n"); - - /* Wait until real_num_tx_queues is updated for mlx5e_select_queue, - * so that we can safely switch to its non-HTB non-PTP fastpath. - */ - synchronize_net(); - - mlx5e_selq_prepare(&priv->selq, &priv->channels.params, false); - mlx5e_selq_apply(&priv->selq); - - WRITE_ONCE(priv->htb.maj_id, 0); - - root = mlx5e_sw_node_find(priv, MLX5E_HTB_CLASSID_ROOT); - if (!root) { - qos_err(priv->mdev, "Failed to find the root node in the QoS tree\n"); - return -ENOENT; - } - err = mlx5_qos_destroy_node(priv->mdev, root->hw_id); - if (err) - qos_err(priv->mdev, "Failed to destroy root node %u, err = %d\n", - root->hw_id, err); - mlx5e_sw_node_delete(priv, root); - - mlx5e_qos_deactivate_all_queues(&priv->channels); - mlx5e_qos_close_all_queues(&priv->channels); - - return err; -} - -static int mlx5e_htb_convert_rate(struct mlx5e_priv *priv, u64 rate, - struct mlx5e_qos_node *parent, u32 *bw_share) -{ - u64 share = 0; - - while (parent->classid != MLX5E_HTB_CLASSID_ROOT && !parent->max_average_bw) - parent = parent->parent; - - if (parent->max_average_bw) - share = div64_u64(div_u64(rate * 100, BYTES_IN_MBIT), - parent->max_average_bw); - else - share = 101; - - *bw_share = share == 0 ? 1 : share > 100 ? 0 : share; - - qos_dbg(priv->mdev, "Convert: rate %llu, parent ceil %llu -> bw_share %u\n", - rate, (u64)parent->max_average_bw * BYTES_IN_MBIT, *bw_share); - - return 0; -} - -static void mlx5e_htb_convert_ceil(struct mlx5e_priv *priv, u64 ceil, u32 *max_average_bw) -{ - /* Hardware treats 0 as "unlimited", set at least 1. */ - *max_average_bw = max_t(u32, div_u64(ceil, BYTES_IN_MBIT), 1); - - qos_dbg(priv->mdev, "Convert: ceil %llu -> max_average_bw %u\n", - ceil, *max_average_bw); -} - -int mlx5e_htb_leaf_alloc_queue(struct mlx5e_priv *priv, u16 classid, - u32 parent_classid, u64 rate, u64 ceil, - struct netlink_ext_ack *extack) -{ - struct mlx5e_qos_node *node, *parent; - int qid; - int err; - - qos_dbg(priv->mdev, "TC_HTB_LEAF_ALLOC_QUEUE classid %04x, parent %04x, rate %llu, ceil %llu\n", - classid, parent_classid, rate, ceil); - - qid = mlx5e_find_unused_qos_qid(priv); - if (qid < 0) { - NL_SET_ERR_MSG_MOD(extack, "Maximum amount of leaf classes is reached."); - return qid; - } - - parent = mlx5e_sw_node_find(priv, parent_classid); - if (!parent) - return -EINVAL; - - node = mlx5e_sw_node_create_leaf(priv, classid, qid, parent); - if (IS_ERR(node)) - return PTR_ERR(node); - - node->rate = rate; - mlx5e_htb_convert_rate(priv, rate, node->parent, &node->bw_share); - mlx5e_htb_convert_ceil(priv, ceil, &node->max_average_bw); - - err = mlx5_qos_create_leaf_node(priv->mdev, node->parent->hw_id, - node->bw_share, node->max_average_bw, - &node->hw_id); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); - qos_err(priv->mdev, "Failed to create a leaf node (class %04x), err = %d\n", - classid, err); - mlx5e_sw_node_delete(priv, node); - return err; - } - - if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { - err = mlx5e_open_qos_sq(priv, &priv->channels, node); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); - qos_warn(priv->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", - classid, err); - } else { - mlx5e_activate_qos_sq(priv, node); - } - } - - return mlx5e_qid_from_qos(&priv->channels, node->qid); -} - -int mlx5e_htb_leaf_to_inner(struct mlx5e_priv *priv, u16 classid, u16 child_classid, - u64 rate, u64 ceil, struct netlink_ext_ack *extack) -{ - struct mlx5e_qos_node *node, *child; - int err, tmp_err; - u32 new_hw_id; - u16 qid; - - qos_dbg(priv->mdev, "TC_HTB_LEAF_TO_INNER classid %04x, upcoming child %04x, rate %llu, ceil %llu\n", - classid, child_classid, rate, ceil); - - node = mlx5e_sw_node_find(priv, classid); - if (!node) - return -ENOENT; - - err = mlx5_qos_create_inner_node(priv->mdev, node->parent->hw_id, - node->bw_share, node->max_average_bw, - &new_hw_id); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating an inner node."); - qos_err(priv->mdev, "Failed to create an inner node (class %04x), err = %d\n", - classid, err); - return err; - } - - /* Intentionally reuse the qid for the upcoming first child. */ - child = mlx5e_sw_node_create_leaf(priv, child_classid, node->qid, node); - if (IS_ERR(child)) { - err = PTR_ERR(child); - goto err_destroy_hw_node; - } - - child->rate = rate; - mlx5e_htb_convert_rate(priv, rate, node, &child->bw_share); - mlx5e_htb_convert_ceil(priv, ceil, &child->max_average_bw); - - err = mlx5_qos_create_leaf_node(priv->mdev, new_hw_id, child->bw_share, - child->max_average_bw, &child->hw_id); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); - qos_err(priv->mdev, "Failed to create a leaf node (class %04x), err = %d\n", - classid, err); - goto err_delete_sw_node; - } - - /* No fail point. */ - - qid = node->qid; - /* Pairs with mlx5e_get_txq_by_classid. */ - WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); - - if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { - mlx5e_deactivate_qos_sq(priv, qid); - mlx5e_close_qos_sq(priv, qid); - } - - err = mlx5_qos_destroy_node(priv->mdev, node->hw_id); - if (err) /* Not fatal. */ - qos_warn(priv->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", - node->hw_id, classid, err); - - node->hw_id = new_hw_id; - - if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { - err = mlx5e_open_qos_sq(priv, &priv->channels, child); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); - qos_warn(priv->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", - classid, err); - } else { - mlx5e_activate_qos_sq(priv, child); - } - } - - return 0; - -err_delete_sw_node: - child->qid = MLX5E_QOS_QID_INNER; - mlx5e_sw_node_delete(priv, child); - -err_destroy_hw_node: - tmp_err = mlx5_qos_destroy_node(priv->mdev, new_hw_id); - if (tmp_err) /* Not fatal. */ - qos_warn(priv->mdev, "Failed to roll back creation of an inner node %u (class %04x), err = %d\n", - new_hw_id, classid, tmp_err); - return err; -} - -static struct mlx5e_qos_node *mlx5e_sw_node_find_by_qid(struct mlx5e_priv *priv, u16 qid) -{ - struct mlx5e_qos_node *node = NULL; - int bkt; - - hash_for_each(priv->htb.qos_tc2node, bkt, node, hnode) - if (node->qid == qid) - break; - - return node; -} - -static void mlx5e_reactivate_qos_sq(struct mlx5e_priv *priv, u16 qid, struct netdev_queue *txq) +void mlx5e_reactivate_qos_sq(struct mlx5e_priv *priv, u16 qid, struct netdev_queue *txq) { qos_dbg(priv->mdev, "Reactivate QoS SQ qid %u\n", qid); netdev_tx_reset_queue(txq); netif_tx_start_queue(txq); } -static void mlx5e_reset_qdisc(struct net_device *dev, u16 qid) +void mlx5e_reset_qdisc(struct net_device *dev, u16 qid) { struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, qid); struct Qdisc *qdisc = dev_queue->qdisc_sleeping; @@ -781,251 +371,65 @@ static void mlx5e_reset_qdisc(struct net_device *dev, u16 qid) spin_unlock_bh(qdisc_lock(qdisc)); } -int mlx5e_htb_leaf_del(struct mlx5e_priv *priv, u16 *classid, - struct netlink_ext_ack *extack) +int mlx5e_htb_setup_tc(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb_qopt) { - struct mlx5e_qos_node *node; - struct netdev_queue *txq; - u16 qid, moved_qid; - bool opened; - int err; - - qos_dbg(priv->mdev, "TC_HTB_LEAF_DEL classid %04x\n", *classid); - - node = mlx5e_sw_node_find(priv, *classid); - if (!node) - return -ENOENT; - - /* Store qid for reuse. */ - qid = node->qid; - - opened = test_bit(MLX5E_STATE_OPENED, &priv->state); - if (opened) { - txq = netdev_get_tx_queue(priv->netdev, - mlx5e_qid_from_qos(&priv->channels, qid)); - mlx5e_deactivate_qos_sq(priv, qid); - mlx5e_close_qos_sq(priv, qid); - } - - err = mlx5_qos_destroy_node(priv->mdev, node->hw_id); - if (err) /* Not fatal. */ - qos_warn(priv->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", - node->hw_id, *classid, err); - - mlx5e_sw_node_delete(priv, node); - - moved_qid = mlx5e_qos_cur_leaf_nodes(priv); - - if (moved_qid == 0) { - /* The last QoS SQ was just destroyed. */ - if (opened) - mlx5e_reactivate_qos_sq(priv, qid, txq); - return 0; - } - moved_qid--; - - if (moved_qid < qid) { - /* The highest QoS SQ was just destroyed. */ - WARN(moved_qid != qid - 1, "Gaps in queue numeration: destroyed queue %u, the highest queue is %u", - qid, moved_qid); - if (opened) - mlx5e_reactivate_qos_sq(priv, qid, txq); - return 0; - } - - WARN(moved_qid == qid, "Can't move node with qid %u to itself", qid); - qos_dbg(priv->mdev, "Moving QoS SQ %u to %u\n", moved_qid, qid); - - node = mlx5e_sw_node_find_by_qid(priv, moved_qid); - WARN(!node, "Could not find a node with qid %u to move to queue %u", - moved_qid, qid); - - /* Stop traffic to the old queue. */ - WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); - __clear_bit(moved_qid, priv->htb.qos_used_qids); - - if (opened) { - txq = netdev_get_tx_queue(priv->netdev, - mlx5e_qid_from_qos(&priv->channels, moved_qid)); - mlx5e_deactivate_qos_sq(priv, moved_qid); - mlx5e_close_qos_sq(priv, moved_qid); - } - - /* Prevent packets from the old class from getting into the new one. */ - mlx5e_reset_qdisc(priv->netdev, moved_qid); - - __set_bit(qid, priv->htb.qos_used_qids); - WRITE_ONCE(node->qid, qid); - - if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { - err = mlx5e_open_qos_sq(priv, &priv->channels, node); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); - qos_warn(priv->mdev, "Failed to create a QoS SQ (class %04x) while moving qid %u to %u, err = %d\n", - node->classid, moved_qid, qid, err); - } else { - mlx5e_activate_qos_sq(priv, node); - } - } - - mlx5e_update_tx_netdev_queues(priv); - if (opened) - mlx5e_reactivate_qos_sq(priv, moved_qid, txq); - - *classid = node->classid; - return 0; -} - -int mlx5e_htb_leaf_del_last(struct mlx5e_priv *priv, u16 classid, bool force, - struct netlink_ext_ack *extack) -{ - struct mlx5e_qos_node *node, *parent; - u32 old_hw_id, new_hw_id; - int err, saved_err = 0; - u16 qid; - - qos_dbg(priv->mdev, "TC_HTB_LEAF_DEL_LAST%s classid %04x\n", - force ? "_FORCE" : "", classid); - - node = mlx5e_sw_node_find(priv, classid); - if (!node) - return -ENOENT; - - err = mlx5_qos_create_leaf_node(priv->mdev, node->parent->parent->hw_id, - node->parent->bw_share, - node->parent->max_average_bw, - &new_hw_id); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); - qos_err(priv->mdev, "Failed to create a leaf node (class %04x), err = %d\n", - classid, err); - if (!force) - return err; - saved_err = err; - } - - /* Store qid for reuse and prevent clearing the bit. */ - qid = node->qid; - /* Pairs with mlx5e_get_txq_by_classid. */ - WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); - - if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { - mlx5e_deactivate_qos_sq(priv, qid); - mlx5e_close_qos_sq(priv, qid); - } - - /* Prevent packets from the old class from getting into the new one. */ - mlx5e_reset_qdisc(priv->netdev, qid); - - err = mlx5_qos_destroy_node(priv->mdev, node->hw_id); - if (err) /* Not fatal. */ - qos_warn(priv->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", - node->hw_id, classid, err); - - parent = node->parent; - mlx5e_sw_node_delete(priv, node); + struct mlx5e_htb *htb = priv->htb; + int res; - node = parent; - WRITE_ONCE(node->qid, qid); + if (!htb && htb_qopt->command != TC_HTB_CREATE) + return -EINVAL; - /* Early return on error in force mode. Parent will still be an inner - * node to be deleted by a following delete operation. - */ - if (saved_err) - return saved_err; - - old_hw_id = node->hw_id; - node->hw_id = new_hw_id; - - if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { - err = mlx5e_open_qos_sq(priv, &priv->channels, node); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); - qos_warn(priv->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", - classid, err); - } else { - mlx5e_activate_qos_sq(priv, node); + switch (htb_qopt->command) { + case TC_HTB_CREATE: + if (!mlx5_qos_is_supported(priv->mdev)) { + NL_SET_ERR_MSG_MOD(htb_qopt->extack, + "Missing QoS capabilities. Try disabling SRIOV or use a supported device."); + return -EOPNOTSUPP; } - } - - err = mlx5_qos_destroy_node(priv->mdev, old_hw_id); - if (err) /* Not fatal. */ - qos_warn(priv->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", - node->hw_id, classid, err); - - return 0; -} - -static int mlx5e_qos_update_children(struct mlx5e_priv *priv, struct mlx5e_qos_node *node, - struct netlink_ext_ack *extack) -{ - struct mlx5e_qos_node *child; - int err = 0; - int bkt; - - hash_for_each(priv->htb.qos_tc2node, bkt, child, hnode) { - u32 old_bw_share = child->bw_share; - int err_one; - - if (child->parent != node) - continue; - - mlx5e_htb_convert_rate(priv, child->rate, node, &child->bw_share); - if (child->bw_share == old_bw_share) - continue; - - err_one = mlx5_qos_update_node(priv->mdev, child->hw_id, child->bw_share, - child->max_average_bw, child->hw_id); - if (!err && err_one) { - err = err_one; - - NL_SET_ERR_MSG_MOD(extack, "Firmware error when modifying a child node."); - qos_err(priv->mdev, "Failed to modify a child node (class %04x), err = %d\n", - node->classid, err); + priv->htb = mlx5e_htb_alloc(); + htb = priv->htb; + if (!htb) + return -ENOMEM; + res = mlx5e_htb_init(htb, htb_qopt, priv->netdev, priv->mdev, &priv->selq, priv); + if (res) { + mlx5e_htb_free(htb); + priv->htb = NULL; } + return res; + case TC_HTB_DESTROY: + mlx5e_htb_cleanup(htb); + mlx5e_htb_free(htb); + priv->htb = NULL; + return 0; + case TC_HTB_LEAF_ALLOC_QUEUE: + res = mlx5e_htb_leaf_alloc_queue(htb, htb_qopt->classid, htb_qopt->parent_classid, + htb_qopt->rate, htb_qopt->ceil, htb_qopt->extack); + if (res < 0) + return res; + htb_qopt->qid = res; + return 0; + case TC_HTB_LEAF_TO_INNER: + return mlx5e_htb_leaf_to_inner(htb, htb_qopt->parent_classid, htb_qopt->classid, + htb_qopt->rate, htb_qopt->ceil, htb_qopt->extack); + case TC_HTB_LEAF_DEL: + return mlx5e_htb_leaf_del(htb, &htb_qopt->classid, htb_qopt->extack); + case TC_HTB_LEAF_DEL_LAST: + case TC_HTB_LEAF_DEL_LAST_FORCE: + return mlx5e_htb_leaf_del_last(htb, htb_qopt->classid, + htb_qopt->command == TC_HTB_LEAF_DEL_LAST_FORCE, + htb_qopt->extack); + case TC_HTB_NODE_MODIFY: + return mlx5e_htb_node_modify(htb, htb_qopt->classid, htb_qopt->rate, htb_qopt->ceil, + htb_qopt->extack); + case TC_HTB_LEAF_QUERY_QUEUE: + res = mlx5e_htb_get_txq_by_classid(htb, htb_qopt->classid); + if (res < 0) + return res; + htb_qopt->qid = res; + return 0; + default: + return -EOPNOTSUPP; } - - return err; -} - -int mlx5e_htb_node_modify(struct mlx5e_priv *priv, u16 classid, u64 rate, u64 ceil, - struct netlink_ext_ack *extack) -{ - u32 bw_share, max_average_bw; - struct mlx5e_qos_node *node; - bool ceil_changed = false; - int err; - - qos_dbg(priv->mdev, "TC_HTB_LEAF_MODIFY classid %04x, rate %llu, ceil %llu\n", - classid, rate, ceil); - - node = mlx5e_sw_node_find(priv, classid); - if (!node) - return -ENOENT; - - node->rate = rate; - mlx5e_htb_convert_rate(priv, rate, node->parent, &bw_share); - mlx5e_htb_convert_ceil(priv, ceil, &max_average_bw); - - err = mlx5_qos_update_node(priv->mdev, node->parent->hw_id, bw_share, - max_average_bw, node->hw_id); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Firmware error when modifying a node."); - qos_err(priv->mdev, "Failed to modify a node (class %04x), err = %d\n", - classid, err); - return err; - } - - if (max_average_bw != node->max_average_bw) - ceil_changed = true; - - node->bw_share = bw_share; - node->max_average_bw = max_average_bw; - - if (ceil_changed) - err = mlx5e_qos_update_children(priv, node, extack); - - return err; } struct mlx5e_mqprio_rl { @@ -1111,3 +515,4 @@ int mlx5e_mqprio_rl_get_node_hw_id(struct mlx5e_mqprio_rl *rl, int tc, u32 *hw_i *hw_id = rl->leaves_id[tc]; return 0; } + diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.h b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.h index 5d9bd91d86c2..4947afa23b73 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.h @@ -6,40 +6,39 @@ #include <linux/mlx5/driver.h> -#define MLX5E_QOS_MAX_LEAF_NODES 256 +#define BYTES_IN_MBIT 125000 struct mlx5e_priv; +struct mlx5e_htb; struct mlx5e_channels; struct mlx5e_channel; +struct tc_htb_qopt_offload; int mlx5e_qos_bytes_rate_check(struct mlx5_core_dev *mdev, u64 nbytes); int mlx5e_qos_max_leaf_nodes(struct mlx5_core_dev *mdev); -int mlx5e_qos_cur_leaf_nodes(struct mlx5e_priv *priv); - -/* TX datapath API */ -int mlx5e_get_txq_by_classid(struct mlx5e_priv *priv, u16 classid); /* SQ lifecycle */ +int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs, + u16 node_qid, u32 hw_id); +int mlx5e_activate_qos_sq(void *data, u16 node_qid, u32 hw_id); +void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid); +void mlx5e_close_qos_sq(struct mlx5e_priv *priv, u16 qid); +void mlx5e_reactivate_qos_sq(struct mlx5e_priv *priv, u16 qid, struct netdev_queue *txq); +void mlx5e_reset_qdisc(struct net_device *dev, u16 qid); + int mlx5e_qos_open_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs); void mlx5e_qos_activate_queues(struct mlx5e_priv *priv); void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c); +void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs); void mlx5e_qos_close_queues(struct mlx5e_channel *c); +void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs); +int mlx5e_qos_alloc_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs); + +/* TX datapath API */ +u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid); /* HTB API */ -int mlx5e_htb_root_add(struct mlx5e_priv *priv, u16 htb_maj_id, u16 htb_defcls, - struct netlink_ext_ack *extack); -int mlx5e_htb_root_del(struct mlx5e_priv *priv); -int mlx5e_htb_leaf_alloc_queue(struct mlx5e_priv *priv, u16 classid, - u32 parent_classid, u64 rate, u64 ceil, - struct netlink_ext_ack *extack); -int mlx5e_htb_leaf_to_inner(struct mlx5e_priv *priv, u16 classid, u16 child_classid, - u64 rate, u64 ceil, struct netlink_ext_ack *extack); -int mlx5e_htb_leaf_del(struct mlx5e_priv *priv, u16 *classid, - struct netlink_ext_ack *extack); -int mlx5e_htb_leaf_del_last(struct mlx5e_priv *priv, u16 classid, bool force, - struct netlink_ext_ack *extack); -int mlx5e_htb_node_modify(struct mlx5e_priv *priv, u16 classid, u64 rate, u64 ceil, - struct netlink_ext_ack *extack); +int mlx5e_htb_setup_tc(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb); /* MQPRIO TX rate limit */ struct mlx5e_mqprio_rl; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c index 48dc121b2cb4..39ef2a2561a3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c @@ -269,6 +269,12 @@ mlx5_esw_bridge_port_obj_attr_set(struct net_device *dev, err = mlx5_esw_bridge_vlan_filtering_set(vport_num, esw_owner_vhca_id, attr->u.vlan_filtering, br_offloads); break; + case SWITCHDEV_ATTR_ID_BRIDGE_VLAN_PROTOCOL: + err = mlx5_esw_bridge_vlan_proto_set(vport_num, + esw_owner_vhca_id, + attr->u.vlan_protocol, + br_offloads); + break; default: err = -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/selq.c b/drivers/net/ethernet/mellanox/mlx5/core/en/selq.c index d98a277eb7f8..f675b1926340 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/selq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/selq.c @@ -7,6 +7,7 @@ #include <linux/rcupdate.h> #include "en.h" #include "en/ptp.h" +#include "en/htb.h" struct mlx5e_selq_params { unsigned int num_regular_queues; @@ -19,6 +20,8 @@ struct mlx5e_selq_params { bool is_ptp : 1; }; }; + u16 htb_maj_id; + u16 htb_defcls; }; int mlx5e_selq_init(struct mlx5e_selq *selq, struct mutex *state_lock) @@ -44,6 +47,8 @@ int mlx5e_selq_init(struct mlx5e_selq *selq, struct mutex *state_lock) .num_tcs = 1, .is_htb = false, .is_ptp = false, + .htb_maj_id = 0, + .htb_defcls = 0, }; rcu_assign_pointer(selq->active, init_params); @@ -64,21 +69,50 @@ void mlx5e_selq_cleanup(struct mlx5e_selq *selq) selq->standby = NULL; } -void mlx5e_selq_prepare(struct mlx5e_selq *selq, struct mlx5e_params *params, bool htb) +void mlx5e_selq_prepare_params(struct mlx5e_selq *selq, struct mlx5e_params *params) { + struct mlx5e_selq_params *selq_active; + lockdep_assert_held(selq->state_lock); WARN_ON_ONCE(selq->is_prepared); selq->is_prepared = true; + selq_active = rcu_dereference_protected(selq->active, + lockdep_is_held(selq->state_lock)); + *selq->standby = *selq_active; selq->standby->num_channels = params->num_channels; selq->standby->num_tcs = mlx5e_get_dcb_num_tc(params); selq->standby->num_regular_queues = selq->standby->num_channels * selq->standby->num_tcs; - selq->standby->is_htb = htb; selq->standby->is_ptp = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_TX_PORT_TS); } +bool mlx5e_selq_is_htb_enabled(struct mlx5e_selq *selq) +{ + struct mlx5e_selq_params *selq_active = + rcu_dereference_protected(selq->active, lockdep_is_held(selq->state_lock)); + + return selq_active->htb_maj_id; +} + +void mlx5e_selq_prepare_htb(struct mlx5e_selq *selq, u16 htb_maj_id, u16 htb_defcls) +{ + struct mlx5e_selq_params *selq_active; + + lockdep_assert_held(selq->state_lock); + WARN_ON_ONCE(selq->is_prepared); + + selq->is_prepared = true; + + selq_active = rcu_dereference_protected(selq->active, + lockdep_is_held(selq->state_lock)); + *selq->standby = *selq_active; + selq->standby->is_htb = htb_maj_id; + selq->standby->htb_maj_id = htb_maj_id; + selq->standby->htb_defcls = htb_defcls; +} + void mlx5e_selq_apply(struct mlx5e_selq *selq) { struct mlx5e_selq_params *old_params; @@ -137,20 +171,21 @@ static u16 mlx5e_select_ptpsq(struct net_device *dev, struct sk_buff *skb, return selq->num_regular_queues + up; } -static int mlx5e_select_htb_queue(struct mlx5e_priv *priv, struct sk_buff *skb) +static int mlx5e_select_htb_queue(struct mlx5e_priv *priv, struct sk_buff *skb, + struct mlx5e_selq_params *selq) { u16 classid; /* Order maj_id before defcls - pairs with mlx5e_htb_root_add. */ - if ((TC_H_MAJ(skb->priority) >> 16) == smp_load_acquire(&priv->htb.maj_id)) + if ((TC_H_MAJ(skb->priority) >> 16) == selq->htb_maj_id) classid = TC_H_MIN(skb->priority); else - classid = READ_ONCE(priv->htb.defcls); + classid = selq->htb_defcls; if (!classid) return 0; - return mlx5e_get_txq_by_classid(priv, classid); + return mlx5e_htb_get_txq_by_classid(priv->htb, classid); } u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb, @@ -187,10 +222,10 @@ u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb, up * selq->num_channels; } - if (unlikely(selq->is_htb)) { + if (unlikely(selq->htb_maj_id)) { /* num_tcs == 1, shortcut for PTP */ - txq_ix = mlx5e_select_htb_queue(priv, skb); + txq_ix = mlx5e_select_htb_queue(priv, skb, selq); if (txq_ix > 0) return txq_ix; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/selq.h b/drivers/net/ethernet/mellanox/mlx5/core/en/selq.h index 6c070141d8f1..fd590f80e4d1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/selq.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/selq.h @@ -21,7 +21,9 @@ struct sk_buff; int mlx5e_selq_init(struct mlx5e_selq *selq, struct mutex *state_lock); void mlx5e_selq_cleanup(struct mlx5e_selq *selq); -void mlx5e_selq_prepare(struct mlx5e_selq *selq, struct mlx5e_params *params, bool htb); +void mlx5e_selq_prepare_params(struct mlx5e_selq *selq, struct mlx5e_params *params); +void mlx5e_selq_prepare_htb(struct mlx5e_selq *selq, u16 htb_maj_id, u16 htb_defcls); +bool mlx5e_selq_is_htb_enabled(struct mlx5e_selq *selq); void mlx5e_selq_apply(struct mlx5e_selq *selq); void mlx5e_selq_cancel(struct mlx5e_selq *selq); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.c index 2755c25ba324..305fde62a78d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.c @@ -30,7 +30,7 @@ static struct mlx5e_tc_act *tc_acts_fdb[NUM_FLOW_ACTIONS] = { NULL, /* FLOW_ACTION_WAKE, */ NULL, /* FLOW_ACTION_QUEUE, */ &mlx5e_tc_act_sample, - NULL, /* FLOW_ACTION_POLICE, */ + &mlx5e_tc_act_police, &mlx5e_tc_act_ct, NULL, /* FLOW_ACTION_CT_METADATA, */ &mlx5e_tc_act_mpls_push, @@ -106,8 +106,8 @@ mlx5e_tc_act_init_parse_state(struct mlx5e_tc_act_parse_state *parse_state, { memset(parse_state, 0, sizeof(*parse_state)); parse_state->flow = flow; - parse_state->num_actions = flow_action->num_entries; parse_state->extack = extack; + parse_state->flow_action = flow_action; } void diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.h index f34714c5ddd4..095ff8ef80e2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.h @@ -13,7 +13,7 @@ struct mlx5_flow_attr; struct mlx5e_tc_act_parse_state { - unsigned int num_actions; + struct flow_action *flow_action; struct mlx5e_tc_flow *flow; struct netlink_ext_ack *extack; u32 actions; @@ -76,6 +76,7 @@ extern struct mlx5e_tc_act mlx5e_tc_act_ct; extern struct mlx5e_tc_act mlx5e_tc_act_sample; extern struct mlx5e_tc_act mlx5e_tc_act_ptype; extern struct mlx5e_tc_act mlx5e_tc_act_redirect_ingress; +extern struct mlx5e_tc_act mlx5e_tc_act_police; struct mlx5e_tc_act * mlx5e_tc_act_get(enum flow_action_id act_id, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/police.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/police.c new file mode 100644 index 000000000000..4bd9c04a49e3 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/police.c @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +#include "act.h" +#include "en/tc_priv.h" + +static bool +tc_act_can_offload_police(struct mlx5e_tc_act_parse_state *parse_state, + const struct flow_action_entry *act, + int act_index, + struct mlx5_flow_attr *attr) +{ + if (act->police.notexceed.act_id != FLOW_ACTION_PIPE && + act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) { + NL_SET_ERR_MSG_MOD(parse_state->extack, + "Offload not supported when conform action is not pipe or ok"); + return false; + } + if (mlx5e_policer_validate(parse_state->flow_action, act, + parse_state->extack)) + return false; + + return !!mlx5e_get_flow_meters(parse_state->flow->priv->mdev); +} + +static int +tc_act_parse_police(struct mlx5e_tc_act_parse_state *parse_state, + const struct flow_action_entry *act, + struct mlx5e_priv *priv, + struct mlx5_flow_attr *attr) +{ + struct mlx5e_flow_meter_params *params; + + params = &attr->meter_attr.params; + params->index = act->hw_index; + if (act->police.rate_bytes_ps) { + params->mode = MLX5_RATE_LIMIT_BPS; + /* change rate to bits per second */ + params->rate = act->police.rate_bytes_ps << 3; + params->burst = act->police.burst; + } else if (act->police.rate_pkt_ps) { + params->mode = MLX5_RATE_LIMIT_PPS; + params->rate = act->police.rate_pkt_ps; + params->burst = act->police.burst_pkt; + } else { + return -EOPNOTSUPP; + } + + attr->action |= MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO; + attr->exe_aso_type = MLX5_EXE_ASO_FLOW_METER; + + return 0; +} + +static bool +tc_act_is_multi_table_act_police(struct mlx5e_priv *priv, + const struct flow_action_entry *act, + struct mlx5_flow_attr *attr) +{ + return true; +} + +struct mlx5e_tc_act mlx5e_tc_act_police = { + .can_offload = tc_act_can_offload_police, + .parse_action = tc_act_parse_police, + .is_multi_table_act = tc_act_is_multi_table_act_police, +}; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/trap.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/trap.c index a7d9eab19e4a..53b270f652b9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/trap.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/trap.c @@ -12,7 +12,7 @@ tc_act_can_offload_trap(struct mlx5e_tc_act_parse_state *parse_state, { struct netlink_ext_ack *extack = parse_state->extack; - if (parse_state->num_actions != 1) { + if (parse_state->flow_action->num_entries != 1) { NL_SET_ERR_MSG_MOD(extack, "action trap is supported as a sole action only"); return false; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c new file mode 100644 index 000000000000..ca33f673396f --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c @@ -0,0 +1,474 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +#include <linux/math64.h> +#include "lib/aso.h" +#include "en/tc/post_act.h" +#include "meter.h" +#include "en/tc_priv.h" +#include "post_meter.h" + +#define MLX5_START_COLOR_SHIFT 28 +#define MLX5_METER_MODE_SHIFT 24 +#define MLX5_CBS_EXP_SHIFT 24 +#define MLX5_CBS_MAN_SHIFT 16 +#define MLX5_CIR_EXP_SHIFT 8 + +/* cir = 8*(10^9)*cir_mantissa/(2^cir_exponent)) bits/s */ +#define MLX5_CONST_CIR 8000000000ULL +#define MLX5_CALC_CIR(m, e) ((MLX5_CONST_CIR * (m)) >> (e)) +#define MLX5_MAX_CIR ((MLX5_CONST_CIR * 0x100) - 1) + +/* cbs = cbs_mantissa*2^cbs_exponent */ +#define MLX5_CALC_CBS(m, e) ((m) << (e)) +#define MLX5_MAX_CBS ((0x100ULL << 0x1F) - 1) +#define MLX5_MAX_HW_CBS 0x7FFFFFFF + +struct mlx5e_flow_meter_aso_obj { + struct list_head entry; + int base_id; + int total_meters; + + unsigned long meters_map[0]; /* must be at the end of this struct */ +}; + +struct mlx5e_flow_meters { + enum mlx5_flow_namespace_type ns_type; + struct mlx5_aso *aso; + struct mutex aso_lock; /* Protects aso operations */ + int log_granularity; + u32 pdn; + + DECLARE_HASHTABLE(hashtbl, 8); + + struct mutex sync_lock; /* protect flow meter operations */ + struct list_head partial_list; + struct list_head full_list; + + struct mlx5_core_dev *mdev; + struct mlx5e_post_act *post_act; + + struct mlx5e_post_meter_priv *post_meter; +}; + +static void +mlx5e_flow_meter_cir_calc(u64 cir, u8 *man, u8 *exp) +{ + s64 _cir, _delta, delta = S64_MAX; + u8 e, _man = 0, _exp = 0; + u64 m; + + for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */ + m = cir << e; + if ((s64)m < 0) /* overflow */ + break; + m = div64_u64(m, MLX5_CONST_CIR); + if (m > 0xFF) /* man width 8 bit */ + continue; + _cir = MLX5_CALC_CIR(m, e); + _delta = cir - _cir; + if (_delta < delta) { + _man = m; + _exp = e; + if (!_delta) + goto found; + delta = _delta; + } + } + +found: + *man = _man; + *exp = _exp; +} + +static void +mlx5e_flow_meter_cbs_calc(u64 cbs, u8 *man, u8 *exp) +{ + s64 _cbs, _delta, delta = S64_MAX; + u8 e, _man = 0, _exp = 0; + u64 m; + + for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */ + m = cbs >> e; + if (m > 0xFF) /* man width 8 bit */ + continue; + _cbs = MLX5_CALC_CBS(m, e); + _delta = cbs - _cbs; + if (_delta < delta) { + _man = m; + _exp = e; + if (!_delta) + goto found; + delta = _delta; + } + } + +found: + *man = _man; + *exp = _exp; +} + +int +mlx5e_tc_meter_modify(struct mlx5_core_dev *mdev, + struct mlx5e_flow_meter_handle *meter, + struct mlx5e_flow_meter_params *meter_params) +{ + struct mlx5_wqe_aso_ctrl_seg *aso_ctrl; + struct mlx5_wqe_aso_data_seg *aso_data; + struct mlx5e_flow_meters *flow_meters; + u8 cir_man, cir_exp, cbs_man, cbs_exp; + struct mlx5_aso_wqe *aso_wqe; + struct mlx5_aso *aso; + u64 rate, burst; + u8 ds_cnt; + int err; + + rate = meter_params->rate; + burst = meter_params->burst; + + /* HW treats each packet as 128 bytes in PPS mode */ + if (meter_params->mode == MLX5_RATE_LIMIT_PPS) { + rate <<= 10; + burst <<= 7; + } + + if (!rate || rate > MLX5_MAX_CIR || !burst || burst > MLX5_MAX_CBS) + return -EINVAL; + + /* HW has limitation of total 31 bits for cbs */ + if (burst > MLX5_MAX_HW_CBS) { + mlx5_core_warn(mdev, + "burst(%lld) is too large, use HW allowed value(%d)\n", + burst, MLX5_MAX_HW_CBS); + burst = MLX5_MAX_HW_CBS; + } + + mlx5_core_dbg(mdev, "meter mode=%d\n", meter_params->mode); + mlx5e_flow_meter_cir_calc(rate, &cir_man, &cir_exp); + mlx5_core_dbg(mdev, "rate=%lld, cir=%lld, exp=%d, man=%d\n", + rate, MLX5_CALC_CIR(cir_man, cir_exp), cir_exp, cir_man); + mlx5e_flow_meter_cbs_calc(burst, &cbs_man, &cbs_exp); + mlx5_core_dbg(mdev, "burst=%lld, cbs=%lld, exp=%d, man=%d\n", + burst, MLX5_CALC_CBS((u64)cbs_man, cbs_exp), cbs_exp, cbs_man); + + if (!cir_man || !cbs_man) + return -EINVAL; + + flow_meters = meter->flow_meters; + aso = flow_meters->aso; + + mutex_lock(&flow_meters->aso_lock); + aso_wqe = mlx5_aso_get_wqe(aso); + ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe_data), MLX5_SEND_WQE_DS); + mlx5_aso_build_wqe(aso, ds_cnt, aso_wqe, meter->obj_id, + MLX5_ACCESS_ASO_OPC_MOD_FLOW_METER); + + aso_ctrl = &aso_wqe->aso_ctrl; + memset(aso_ctrl, 0, sizeof(*aso_ctrl)); + aso_ctrl->data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BYTEWISE_64BYTE << 6; + aso_ctrl->condition_1_0_operand = MLX5_ASO_ALWAYS_TRUE | + MLX5_ASO_ALWAYS_TRUE << 4; + aso_ctrl->data_offset_condition_operand = MLX5_ASO_LOGICAL_OR << 6; + aso_ctrl->data_mask = cpu_to_be64(0x80FFFFFFULL << (meter->idx ? 0 : 32)); + + aso_data = (struct mlx5_wqe_aso_data_seg *)(aso_wqe + 1); + memset(aso_data, 0, sizeof(*aso_data)); + aso_data->bytewise_data[meter->idx * 8] = cpu_to_be32((0x1 << 31) | /* valid */ + (MLX5_FLOW_METER_COLOR_GREEN << MLX5_START_COLOR_SHIFT)); + if (meter_params->mode == MLX5_RATE_LIMIT_PPS) + aso_data->bytewise_data[meter->idx * 8] |= + cpu_to_be32(MLX5_FLOW_METER_MODE_NUM_PACKETS << MLX5_METER_MODE_SHIFT); + else + aso_data->bytewise_data[meter->idx * 8] |= + cpu_to_be32(MLX5_FLOW_METER_MODE_BYTES_IP_LENGTH << MLX5_METER_MODE_SHIFT); + + aso_data->bytewise_data[meter->idx * 8 + 2] = cpu_to_be32((cbs_exp << MLX5_CBS_EXP_SHIFT) | + (cbs_man << MLX5_CBS_MAN_SHIFT) | + (cir_exp << MLX5_CIR_EXP_SHIFT) | + cir_man); + + mlx5_aso_post_wqe(aso, true, &aso_wqe->ctrl); + + /* With newer FW, the wait for the first ASO WQE is more than 2us, put the wait 10ms. */ + err = mlx5_aso_poll_cq(aso, true, 10); + mutex_unlock(&flow_meters->aso_lock); + + return err; +} + +static int +mlx5e_flow_meter_create_aso_obj(struct mlx5e_flow_meters *flow_meters, int *obj_id) +{ + u32 in[MLX5_ST_SZ_DW(create_flow_meter_aso_obj_in)] = {}; + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; + struct mlx5_core_dev *mdev = flow_meters->mdev; + void *obj; + int err; + + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, + MLX5_GENERAL_OBJECT_TYPES_FLOW_METER_ASO); + MLX5_SET(general_obj_in_cmd_hdr, in, log_obj_range, flow_meters->log_granularity); + + obj = MLX5_ADDR_OF(create_flow_meter_aso_obj_in, in, flow_meter_aso_obj); + MLX5_SET(flow_meter_aso_obj, obj, meter_aso_access_pd, flow_meters->pdn); + + err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); + if (!err) { + *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); + mlx5_core_dbg(mdev, "flow meter aso obj(0x%x) created\n", *obj_id); + } + + return err; +} + +static void +mlx5e_flow_meter_destroy_aso_obj(struct mlx5_core_dev *mdev, u32 obj_id) +{ + u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {}; + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; + + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, + MLX5_GENERAL_OBJECT_TYPES_FLOW_METER_ASO); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id); + + mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); + mlx5_core_dbg(mdev, "flow meter aso obj(0x%x) destroyed\n", obj_id); +} + +static struct mlx5e_flow_meter_handle * +__mlx5e_flow_meter_alloc(struct mlx5e_flow_meters *flow_meters) +{ + struct mlx5_core_dev *mdev = flow_meters->mdev; + struct mlx5e_flow_meter_aso_obj *meters_obj; + struct mlx5e_flow_meter_handle *meter; + int err, pos, total; + u32 id; + + meter = kzalloc(sizeof(*meter), GFP_KERNEL); + if (!meter) + return ERR_PTR(-ENOMEM); + + meters_obj = list_first_entry_or_null(&flow_meters->partial_list, + struct mlx5e_flow_meter_aso_obj, + entry); + /* 2 meters in one object */ + total = 1 << (flow_meters->log_granularity + 1); + if (!meters_obj) { + err = mlx5e_flow_meter_create_aso_obj(flow_meters, &id); + if (err) { + mlx5_core_err(mdev, "Failed to create flow meter ASO object\n"); + goto err_create; + } + + meters_obj = kzalloc(sizeof(*meters_obj) + BITS_TO_BYTES(total), + GFP_KERNEL); + if (!meters_obj) { + err = -ENOMEM; + goto err_mem; + } + + meters_obj->base_id = id; + meters_obj->total_meters = total; + list_add(&meters_obj->entry, &flow_meters->partial_list); + pos = 0; + } else { + pos = find_first_zero_bit(meters_obj->meters_map, total); + if (bitmap_weight(meters_obj->meters_map, total) == total - 1) { + list_del(&meters_obj->entry); + list_add(&meters_obj->entry, &flow_meters->full_list); + } + } + + bitmap_set(meters_obj->meters_map, pos, 1); + meter->flow_meters = flow_meters; + meter->meters_obj = meters_obj; + meter->obj_id = meters_obj->base_id + pos / 2; + meter->idx = pos % 2; + + mlx5_core_dbg(mdev, "flow meter allocated, obj_id=0x%x, index=%d\n", + meter->obj_id, meter->idx); + + return meter; + +err_mem: + mlx5e_flow_meter_destroy_aso_obj(mdev, id); +err_create: + kfree(meter); + return ERR_PTR(err); +} + +static void +__mlx5e_flow_meter_free(struct mlx5e_flow_meter_handle *meter) +{ + struct mlx5e_flow_meters *flow_meters = meter->flow_meters; + struct mlx5_core_dev *mdev = flow_meters->mdev; + struct mlx5e_flow_meter_aso_obj *meters_obj; + int n, pos; + + meters_obj = meter->meters_obj; + pos = (meter->obj_id - meters_obj->base_id) * 2 + meter->idx; + bitmap_clear(meters_obj->meters_map, pos, 1); + n = bitmap_weight(meters_obj->meters_map, meters_obj->total_meters); + if (n == 0) { + list_del(&meters_obj->entry); + mlx5e_flow_meter_destroy_aso_obj(mdev, meters_obj->base_id); + kfree(meters_obj); + } else if (n == meters_obj->total_meters - 1) { + list_del(&meters_obj->entry); + list_add(&meters_obj->entry, &flow_meters->partial_list); + } + + mlx5_core_dbg(mdev, "flow meter freed, obj_id=0x%x, index=%d\n", + meter->obj_id, meter->idx); + kfree(meter); +} + +struct mlx5e_flow_meter_handle * +mlx5e_tc_meter_get(struct mlx5_core_dev *mdev, struct mlx5e_flow_meter_params *params) +{ + struct mlx5e_flow_meters *flow_meters; + struct mlx5e_flow_meter_handle *meter; + int err; + + flow_meters = mlx5e_get_flow_meters(mdev); + if (!flow_meters) + return ERR_PTR(-EOPNOTSUPP); + + mutex_lock(&flow_meters->sync_lock); + hash_for_each_possible(flow_meters->hashtbl, meter, hlist, params->index) + if (meter->params.index == params->index) + goto add_ref; + + meter = __mlx5e_flow_meter_alloc(flow_meters); + if (IS_ERR(meter)) { + err = PTR_ERR(meter); + goto err_alloc; + } + + hash_add(flow_meters->hashtbl, &meter->hlist, params->index); + meter->params.index = params->index; + +add_ref: + meter->refcnt++; + + if (meter->params.mode != params->mode || meter->params.rate != params->rate || + meter->params.burst != params->burst) { + err = mlx5e_tc_meter_modify(mdev, meter, params); + if (err) + goto err_update; + + meter->params.mode = params->mode; + meter->params.rate = params->rate; + meter->params.burst = params->burst; + } + + mutex_unlock(&flow_meters->sync_lock); + return meter; + +err_update: + if (--meter->refcnt == 0) { + hash_del(&meter->hlist); + __mlx5e_flow_meter_free(meter); + } +err_alloc: + mutex_unlock(&flow_meters->sync_lock); + return ERR_PTR(err); +} + +void +mlx5e_tc_meter_put(struct mlx5e_flow_meter_handle *meter) +{ + struct mlx5e_flow_meters *flow_meters = meter->flow_meters; + + mutex_lock(&flow_meters->sync_lock); + if (--meter->refcnt == 0) { + hash_del(&meter->hlist); + __mlx5e_flow_meter_free(meter); + } + mutex_unlock(&flow_meters->sync_lock); +} + +struct mlx5_flow_table * +mlx5e_tc_meter_get_post_meter_ft(struct mlx5e_flow_meters *flow_meters) +{ + return mlx5e_post_meter_get_ft(flow_meters->post_meter); +} + +struct mlx5e_flow_meters * +mlx5e_flow_meters_init(struct mlx5e_priv *priv, + enum mlx5_flow_namespace_type ns_type, + struct mlx5e_post_act *post_act) +{ + struct mlx5_core_dev *mdev = priv->mdev; + struct mlx5e_flow_meters *flow_meters; + int err; + + if (!(MLX5_CAP_GEN_64(mdev, general_obj_types) & + MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_FLOW_METER_ASO)) + return ERR_PTR(-EOPNOTSUPP); + + if (IS_ERR_OR_NULL(post_act)) { + netdev_dbg(priv->netdev, + "flow meter offload is not supported, post action is missing\n"); + return ERR_PTR(-EOPNOTSUPP); + } + + flow_meters = kzalloc(sizeof(*flow_meters), GFP_KERNEL); + if (!flow_meters) + return ERR_PTR(-ENOMEM); + + err = mlx5_core_alloc_pd(mdev, &flow_meters->pdn); + if (err) { + mlx5_core_err(mdev, "Failed to alloc pd for flow meter aso, err=%d\n", err); + goto err_out; + } + + flow_meters->aso = mlx5_aso_create(mdev, flow_meters->pdn); + if (IS_ERR(flow_meters->aso)) { + mlx5_core_warn(mdev, "Failed to create aso wqe for flow meter\n"); + err = PTR_ERR(flow_meters->aso); + goto err_sq; + } + + flow_meters->post_meter = mlx5e_post_meter_init(priv, ns_type, post_act); + if (IS_ERR(flow_meters->post_meter)) { + err = PTR_ERR(flow_meters->post_meter); + goto err_post_meter; + } + + mutex_init(&flow_meters->sync_lock); + INIT_LIST_HEAD(&flow_meters->partial_list); + INIT_LIST_HEAD(&flow_meters->full_list); + + flow_meters->ns_type = ns_type; + flow_meters->mdev = mdev; + flow_meters->post_act = post_act; + mutex_init(&flow_meters->aso_lock); + flow_meters->log_granularity = min_t(int, 6, + MLX5_CAP_QOS(mdev, log_meter_aso_max_alloc)); + + return flow_meters; + +err_post_meter: + mlx5_aso_destroy(flow_meters->aso); +err_sq: + mlx5_core_dealloc_pd(mdev, flow_meters->pdn); +err_out: + kfree(flow_meters); + return ERR_PTR(err); +} + +void +mlx5e_flow_meters_cleanup(struct mlx5e_flow_meters *flow_meters) +{ + if (IS_ERR_OR_NULL(flow_meters)) + return; + + mlx5e_post_meter_cleanup(flow_meters->post_meter); + mlx5_aso_destroy(flow_meters->aso); + mlx5_core_dealloc_pd(flow_meters->mdev, flow_meters->pdn); + + kfree(flow_meters); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.h new file mode 100644 index 000000000000..78885db5dc7d --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5_EN_FLOW_METER_H__ +#define __MLX5_EN_FLOW_METER_H__ + +struct mlx5e_flow_meter_aso_obj; +struct mlx5e_flow_meters; +struct mlx5_flow_attr; + +enum mlx5e_flow_meter_mode { + MLX5_RATE_LIMIT_BPS, + MLX5_RATE_LIMIT_PPS, +}; + +struct mlx5e_flow_meter_params { + enum mlx5e_flow_meter_mode mode; + /* police action index */ + u32 index; + u64 rate; + u64 burst; +}; + +struct mlx5e_flow_meter_handle { + struct mlx5e_flow_meters *flow_meters; + struct mlx5e_flow_meter_aso_obj *meters_obj; + u32 obj_id; + u8 idx; + + int refcnt; + struct hlist_node hlist; + struct mlx5e_flow_meter_params params; +}; + +struct mlx5e_meter_attr { + struct mlx5e_flow_meter_params params; + struct mlx5e_flow_meter_handle *meter; +}; + +int +mlx5e_tc_meter_modify(struct mlx5_core_dev *mdev, + struct mlx5e_flow_meter_handle *meter, + struct mlx5e_flow_meter_params *meter_params); + +struct mlx5e_flow_meter_handle * +mlx5e_tc_meter_get(struct mlx5_core_dev *mdev, struct mlx5e_flow_meter_params *params); +void +mlx5e_tc_meter_put(struct mlx5e_flow_meter_handle *meter); + +struct mlx5_flow_table * +mlx5e_tc_meter_get_post_meter_ft(struct mlx5e_flow_meters *flow_meters); + +struct mlx5e_flow_meters * +mlx5e_flow_meters_init(struct mlx5e_priv *priv, + enum mlx5_flow_namespace_type ns_type, + struct mlx5e_post_act *post_action); +void +mlx5e_flow_meters_cleanup(struct mlx5e_flow_meters *flow_meters); + +#endif /* __MLX5_EN_FLOW_METER_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_act.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_act.c index dea137dd744b..33c1411ed8db 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_act.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_act.c @@ -22,9 +22,9 @@ struct mlx5e_post_act_handle { u32 id; }; -#define MLX5_POST_ACTION_BITS (mlx5e_tc_attr_to_reg_mappings[FTEID_TO_REG].mlen) -#define MLX5_POST_ACTION_MAX GENMASK(MLX5_POST_ACTION_BITS - 1, 0) -#define MLX5_POST_ACTION_MASK MLX5_POST_ACTION_MAX +#define MLX5_POST_ACTION_BITS MLX5_REG_MAPPING_MBITS(FTEID_TO_REG) +#define MLX5_POST_ACTION_MASK MLX5_REG_MAPPING_MASK(FTEID_TO_REG) +#define MLX5_POST_ACTION_MAX MLX5_POST_ACTION_MASK struct mlx5e_post_act * mlx5e_tc_post_act_init(struct mlx5e_priv *priv, struct mlx5_fs_chains *chains, @@ -36,7 +36,7 @@ mlx5e_tc_post_act_init(struct mlx5e_priv *priv, struct mlx5_fs_chains *chains, int err; if (!MLX5_CAP_FLOWTABLE_TYPE(priv->mdev, ignore_flow_level, table_type)) { - if (priv->mdev->coredev_type != MLX5_COREDEV_VF) + if (priv->mdev->coredev_type == MLX5_COREDEV_PF) mlx5_core_warn(priv->mdev, "firmware level support is missing\n"); err = -EOPNOTSUPP; goto err_check; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_meter.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_meter.c new file mode 100644 index 000000000000..efa20356764e --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_meter.c @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +#include "en/tc_priv.h" +#include "post_meter.h" +#include "en/tc/post_act.h" + +#define MLX5_PACKET_COLOR_BITS MLX5_REG_MAPPING_MBITS(PACKET_COLOR_TO_REG) +#define MLX5_PACKET_COLOR_MASK MLX5_REG_MAPPING_MASK(PACKET_COLOR_TO_REG) + +struct mlx5e_post_meter_priv { + struct mlx5_flow_table *ft; + struct mlx5_flow_group *fg; + struct mlx5_flow_handle *fwd_green_rule; + struct mlx5_flow_handle *drop_red_rule; +}; + +struct mlx5_flow_table * +mlx5e_post_meter_get_ft(struct mlx5e_post_meter_priv *post_meter) +{ + return post_meter->ft; +} + +static int +mlx5e_post_meter_table_create(struct mlx5e_priv *priv, + enum mlx5_flow_namespace_type ns_type, + struct mlx5e_post_meter_priv *post_meter) +{ + struct mlx5_flow_table_attr ft_attr = {}; + struct mlx5_flow_namespace *root_ns; + + root_ns = mlx5_get_flow_namespace(priv->mdev, ns_type); + if (!root_ns) { + mlx5_core_warn(priv->mdev, "Failed to get namespace for flow meter\n"); + return -EOPNOTSUPP; + } + + ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED; + ft_attr.prio = FDB_SLOW_PATH; + ft_attr.max_fte = 2; + ft_attr.level = 1; + + post_meter->ft = mlx5_create_flow_table(root_ns, &ft_attr); + if (IS_ERR(post_meter->ft)) { + mlx5_core_warn(priv->mdev, "Failed to create post_meter table\n"); + return PTR_ERR(post_meter->ft); + } + + return 0; +} + +static int +mlx5e_post_meter_fg_create(struct mlx5e_priv *priv, + struct mlx5e_post_meter_priv *post_meter) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + void *misc2, *match_criteria; + u32 *flow_group_in; + int err = 0; + + flow_group_in = kvzalloc(inlen, GFP_KERNEL); + if (!flow_group_in) + return -ENOMEM; + + MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, + MLX5_MATCH_MISC_PARAMETERS_2); + match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, + match_criteria); + misc2 = MLX5_ADDR_OF(fte_match_param, match_criteria, misc_parameters_2); + MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_5, MLX5_PACKET_COLOR_MASK); + MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); + MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1); + + post_meter->fg = mlx5_create_flow_group(post_meter->ft, flow_group_in); + if (IS_ERR(post_meter->fg)) { + mlx5_core_warn(priv->mdev, "Failed to create post_meter flow group\n"); + err = PTR_ERR(post_meter->fg); + } + + kvfree(flow_group_in); + return err; +} + +static int +mlx5e_post_meter_rules_create(struct mlx5e_priv *priv, + struct mlx5e_post_meter_priv *post_meter, + struct mlx5e_post_act *post_act) +{ + struct mlx5_flow_destination dest = {}; + struct mlx5_flow_act flow_act = {}; + struct mlx5_flow_handle *rule; + struct mlx5_flow_spec *spec; + int err; + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) + return -ENOMEM; + + mlx5e_tc_match_to_reg_match(spec, PACKET_COLOR_TO_REG, + MLX5_FLOW_METER_COLOR_RED, MLX5_PACKET_COLOR_MASK); + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; + flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; + + rule = mlx5_add_flow_rules(post_meter->ft, spec, &flow_act, NULL, 0); + if (IS_ERR(rule)) { + mlx5_core_warn(priv->mdev, "Failed to create post_meter flow drop rule\n"); + err = PTR_ERR(rule); + goto err_red; + } + post_meter->drop_red_rule = rule; + + mlx5e_tc_match_to_reg_match(spec, PACKET_COLOR_TO_REG, + MLX5_FLOW_METER_COLOR_GREEN, MLX5_PACKET_COLOR_MASK); + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest.ft = mlx5e_tc_post_act_get_ft(post_act); + + rule = mlx5_add_flow_rules(post_meter->ft, spec, &flow_act, &dest, 1); + if (IS_ERR(rule)) { + mlx5_core_warn(priv->mdev, "Failed to create post_meter flow fwd rule\n"); + err = PTR_ERR(rule); + goto err_green; + } + post_meter->fwd_green_rule = rule; + + kvfree(spec); + return 0; + +err_green: + mlx5_del_flow_rules(post_meter->drop_red_rule); +err_red: + kvfree(spec); + return err; +} + +static void +mlx5e_post_meter_rules_destroy(struct mlx5e_post_meter_priv *post_meter) +{ + mlx5_del_flow_rules(post_meter->drop_red_rule); + mlx5_del_flow_rules(post_meter->fwd_green_rule); +} + +static void +mlx5e_post_meter_fg_destroy(struct mlx5e_post_meter_priv *post_meter) +{ + mlx5_destroy_flow_group(post_meter->fg); +} + +static void +mlx5e_post_meter_table_destroy(struct mlx5e_post_meter_priv *post_meter) +{ + mlx5_destroy_flow_table(post_meter->ft); +} + +struct mlx5e_post_meter_priv * +mlx5e_post_meter_init(struct mlx5e_priv *priv, + enum mlx5_flow_namespace_type ns_type, + struct mlx5e_post_act *post_act) +{ + struct mlx5e_post_meter_priv *post_meter; + int err; + + post_meter = kzalloc(sizeof(*post_meter), GFP_KERNEL); + if (!post_meter) + return ERR_PTR(-ENOMEM); + + err = mlx5e_post_meter_table_create(priv, ns_type, post_meter); + if (err) + goto err_ft; + + err = mlx5e_post_meter_fg_create(priv, post_meter); + if (err) + goto err_fg; + + err = mlx5e_post_meter_rules_create(priv, post_meter, post_act); + if (err) + goto err_rules; + + return post_meter; + +err_rules: + mlx5e_post_meter_fg_destroy(post_meter); +err_fg: + mlx5e_post_meter_table_destroy(post_meter); +err_ft: + kfree(post_meter); + return ERR_PTR(err); +} + +void +mlx5e_post_meter_cleanup(struct mlx5e_post_meter_priv *post_meter) +{ + mlx5e_post_meter_rules_destroy(post_meter); + mlx5e_post_meter_fg_destroy(post_meter); + mlx5e_post_meter_table_destroy(post_meter); + kfree(post_meter); +} + diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_meter.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_meter.h new file mode 100644 index 000000000000..c74f3cbd810d --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_meter.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5_EN_POST_METER_H__ +#define __MLX5_EN_POST_METER_H__ + +#define packet_color_to_reg { \ + .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_5, \ + .moffset = 0, \ + .mlen = 8, \ + .soffset = MLX5_BYTE_OFF(fte_match_param, \ + misc_parameters_2.metadata_reg_c_5), \ +} + +struct mlx5e_post_meter_priv; + +struct mlx5_flow_table * +mlx5e_post_meter_get_ft(struct mlx5e_post_meter_priv *post_meter); + +struct mlx5e_post_meter_priv * +mlx5e_post_meter_init(struct mlx5e_priv *priv, + enum mlx5_flow_namespace_type ns_type, + struct mlx5e_post_act *post_act); +void +mlx5e_post_meter_cleanup(struct mlx5e_post_meter_priv *post_meter); + +#endif /* __MLX5_EN_POST_METER_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c index ba171c7f0a67..864ce0c393e6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c @@ -36,8 +36,8 @@ #define MLX5_CT_STATE_RELATED_BIT BIT(5) #define MLX5_CT_STATE_INVALID_BIT BIT(6) -#define MLX5_CT_LABELS_BITS (mlx5e_tc_attr_to_reg_mappings[LABELS_TO_REG].mlen) -#define MLX5_CT_LABELS_MASK GENMASK(MLX5_CT_LABELS_BITS - 1, 0) +#define MLX5_CT_LABELS_BITS MLX5_REG_MAPPING_MBITS(LABELS_TO_REG) +#define MLX5_CT_LABELS_MASK MLX5_REG_MAPPING_MASK(LABELS_TO_REG) /* Statically allocate modify actions for * ipv6 and port nat (5) + tuple fields (4) + nic mode zone restore (1) = 10. @@ -2062,7 +2062,7 @@ mlx5_tc_ct_init_check_support(struct mlx5e_priv *priv, /* Ignore_flow_level support isn't supported by default for VFs and so post_act * won't be supported. Skip showing error msg. */ - if (priv->mdev->coredev_type != MLX5_COREDEV_VF) + if (priv->mdev->coredev_type == MLX5_COREDEV_PF) err_msg = "post action is missing"; err = -EOPNOTSUPP; goto out_err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h index 00a3ba862afb..5bbd6b92840f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h @@ -62,10 +62,11 @@ struct mlx5_ct_attr { misc_parameters_2.metadata_reg_c_4),\ } +/* 8 LSB of metadata C5 are reserved for packet color */ #define fteid_to_reg_ct {\ .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_5,\ - .moffset = 0,\ - .mlen = 32,\ + .moffset = 8,\ + .mlen = 24,\ .soffset = MLX5_BYTE_OFF(fte_match_param,\ misc_parameters_2.metadata_reg_c_5),\ } @@ -84,10 +85,8 @@ struct mlx5_ct_attr { .mlen = ESW_ZONE_ID_BITS,\ } -#define REG_MAPPING_MLEN(reg) (mlx5e_tc_attr_to_reg_mappings[reg].mlen) -#define REG_MAPPING_MOFFSET(reg) (mlx5e_tc_attr_to_reg_mappings[reg].moffset) -#define MLX5_CT_ZONE_BITS (mlx5e_tc_attr_to_reg_mappings[ZONE_TO_REG].mlen) -#define MLX5_CT_ZONE_MASK GENMASK(MLX5_CT_ZONE_BITS - 1, 0) +#define MLX5_CT_ZONE_BITS MLX5_REG_MAPPING_MBITS(ZONE_TO_REG) +#define MLX5_CT_ZONE_MASK MLX5_REG_MAPPING_MASK(ZONE_TO_REG) #if IS_ENABLED(CONFIG_MLX5_TC_CT) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h index 3b74a6fd5c43..d2bdfd6872bc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h @@ -203,7 +203,13 @@ struct mlx5_fc *mlx5e_tc_get_counter(struct mlx5e_tc_flow *flow); struct mlx5e_tc_int_port_priv * mlx5e_get_int_port_priv(struct mlx5e_priv *priv); +struct mlx5e_flow_meters *mlx5e_get_flow_meters(struct mlx5_core_dev *dev); + void *mlx5e_get_match_headers_value(u32 flags, struct mlx5_flow_spec *spec); void *mlx5e_get_match_headers_criteria(u32 flags, struct mlx5_flow_spec *spec); +int mlx5e_policer_validate(const struct flow_action *action, + const struct flow_action_entry *act, + struct netlink_ext_ack *extack); + #endif /* __MLX5_EN_TC_PRIV_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.h deleted file mode 100644 index e4eeb2ba21c7..000000000000 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.h +++ /dev/null @@ -1,21 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ -/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ - -#ifndef __MLX5_IPSEC_STEERING_H__ -#define __MLX5_IPSEC_STEERING_H__ - -#include "en.h" -#include "ipsec.h" -#include "ipsec_offload.h" -#include "en/fs.h" - -void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec); -int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec); -int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_priv *priv, - struct mlx5_accel_esp_xfrm_attrs *attrs, - u32 ipsec_obj_id, - struct mlx5e_ipsec_rule *ipsec_rule); -void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_priv *priv, - struct mlx5_accel_esp_xfrm_attrs *attrs, - struct mlx5e_ipsec_rule *ipsec_rule); -#endif /* __MLX5_IPSEC_STEERING_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c index f239fb2e832f..fba21edf88d8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c @@ -457,7 +457,7 @@ bool mlx5e_ktls_handle_tx_skb(struct net_device *netdev, struct mlx5e_txqsq *sq, int datalen; u32 seq; - datalen = skb->len - (skb_transport_offset(skb) + tcp_hdrlen(skb)); + datalen = skb->len - skb_tcp_all_headers(skb); if (!datalen) return true; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index 6e80585d731f..b811207fe5ed 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -30,6 +30,8 @@ * SOFTWARE. */ +#include <linux/ethtool_netlink.h> + #include "en.h" #include "en/port.h" #include "en/params.h" @@ -305,12 +307,18 @@ static void mlx5e_get_ethtool_stats(struct net_device *dev, } void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param) { param->rx_max_pending = 1 << MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE; param->tx_max_pending = 1 << MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE; param->rx_pending = 1 << priv->channels.params.log_rq_mtu_frames; param->tx_pending = 1 << priv->channels.params.log_sq_size; + + kernel_param->tcp_data_split = + (priv->channels.params.packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) ? + ETHTOOL_TCP_DATA_SPLIT_ENABLED : + ETHTOOL_TCP_DATA_SPLIT_DISABLED; } static void mlx5e_get_ringparam(struct net_device *dev, @@ -320,7 +328,7 @@ static void mlx5e_get_ringparam(struct net_device *dev, { struct mlx5e_priv *priv = netdev_priv(dev); - mlx5e_ethtool_get_ringparam(priv, param); + mlx5e_ethtool_get_ringparam(priv, param, kernel_param); } int mlx5e_ethtool_set_ringparam(struct mlx5e_priv *priv, @@ -451,7 +459,7 @@ int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv, * because the numeration of the QoS SQs will change, while per-queue * qdiscs are attached. */ - if (priv->htb.maj_id) { + if (mlx5e_selq_is_htb_enabled(&priv->selq)) { err = -EINVAL; netdev_err(priv->netdev, "%s: HTB offload is active, cannot change the number of channels\n", __func__); @@ -2067,7 +2075,7 @@ static int set_pflag_tx_port_ts(struct net_device *netdev, bool enable) * the numeration of the QoS SQs will change, while per-queue qdiscs are * attached. */ - if (priv->htb.maj_id) { + if (mlx5e_selq_is_htb_enabled(&priv->selq)) { netdev_err(priv->netdev, "%s: HTB offload is active, cannot change the PTP state\n", __func__); return -EINVAL; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c index ad0d234632a3..9466202fd97b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c @@ -742,10 +742,7 @@ mlx5e_ethtool_flow_replace(struct mlx5e_priv *priv, eth_rule->flow_spec = *fs; eth_rule->eth_ft = eth_ft; - if (!eth_ft->ft) { - err = -EINVAL; - goto del_ethtool_rule; - } + rule = add_ethtool_flow_rule(priv, eth_rule, eth_ft->ft, fs, rss_context); if (IS_ERR(rule)) { err = PTR_ERR(rule); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 087952b84ccb..180b2f418339 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -31,7 +31,6 @@ */ #include <net/tc_act/tc_gact.h> -#include <net/pkt_cls.h> #include <linux/mlx5/fs.h> #include <net/vxlan.h> #include <net/geneve.h> @@ -64,6 +63,7 @@ #include "en/devlink.h" #include "lib/mlx5.h" #include "en/ptp.h" +#include "en/htb.h" #include "qos.h" #include "en/trap.h" @@ -1912,8 +1912,7 @@ static int mlx5e_txq_get_qos_node_hw_id(struct mlx5e_params *params, int txq_ix, { int tc; - if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL || - !params->mqprio.channel.rl) { + if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL) { *hw_id = 0; return 0; } @@ -1922,7 +1921,14 @@ static int mlx5e_txq_get_qos_node_hw_id(struct mlx5e_params *params, int txq_ix, if (tc < 0) return tc; - return mlx5e_mqprio_rl_get_node_hw_id(params->mqprio.channel.rl, tc, hw_id); + if (tc >= params->mqprio.num_tc) { + WARN(1, "Unexpected TCs configuration. tc %d is out of range of %u", + tc, params->mqprio.num_tc); + return -EINVAL; + } + + *hw_id = params->mqprio.channel.hw_id[tc]; + return 0; } static int mlx5e_open_sqs(struct mlx5e_channel *c, @@ -2383,9 +2389,11 @@ int mlx5e_open_channels(struct mlx5e_priv *priv, goto err_close_channels; } - err = mlx5e_qos_open_queues(priv, chs); - if (err) - goto err_close_ptp; + if (priv->htb) { + err = mlx5e_qos_open_queues(priv, chs); + if (err) + goto err_close_ptp; + } mlx5e_health_channels_update(priv); kvfree(cparam); @@ -2567,9 +2575,11 @@ static int mlx5e_netdev_set_tcs(struct net_device *netdev, u16 nch, u8 ntc, int mlx5e_update_tx_netdev_queues(struct mlx5e_priv *priv) { - int qos_queues, nch, ntc, num_txqs, err; + int nch, ntc, num_txqs, err; + int qos_queues = 0; - qos_queues = mlx5e_qos_cur_leaf_nodes(priv); + if (priv->htb) + qos_queues = mlx5e_htb_cur_leaf_nodes(priv->htb); nch = priv->channels.params.num_channels; ntc = mlx5e_get_dcb_num_tc(&priv->channels.params); @@ -2615,13 +2625,6 @@ static int mlx5e_update_netdev_queues(struct mlx5e_priv *priv) netdev_warn(netdev, "netif_set_real_num_rx_queues failed, %d\n", err); goto err_txqs; } - if (priv->mqprio_rl != priv->channels.params.mqprio.channel.rl) { - if (priv->mqprio_rl) { - mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); - mlx5e_mqprio_rl_free(priv->mqprio_rl); - } - priv->mqprio_rl = priv->channels.params.mqprio.channel.rl; - } return 0; @@ -2724,7 +2727,8 @@ void mlx5e_activate_priv_channels(struct mlx5e_priv *priv) { mlx5e_build_txq_maps(priv); mlx5e_activate_channels(&priv->channels); - mlx5e_qos_activate_queues(priv); + if (priv->htb) + mlx5e_qos_activate_queues(priv); mlx5e_xdp_tx_enable(priv); /* dev_watchdog() wants all TX queues to be started when the carrier is @@ -2841,7 +2845,7 @@ int mlx5e_safe_switch_params(struct mlx5e_priv *priv, new_chs.params = *params; - mlx5e_selq_prepare(&priv->selq, &new_chs.params, !!priv->htb.maj_id); + mlx5e_selq_prepare_params(&priv->selq, &new_chs.params); err = mlx5e_open_channels(priv, &new_chs); if (err) @@ -2897,7 +2901,7 @@ int mlx5e_open_locked(struct net_device *netdev) struct mlx5e_priv *priv = netdev_priv(netdev); int err; - mlx5e_selq_prepare(&priv->selq, &priv->channels.params, !!priv->htb.maj_id); + mlx5e_selq_prepare_params(&priv->selq, &priv->channels.params); set_bit(MLX5E_STATE_OPENED, &priv->state); @@ -3135,6 +3139,11 @@ err_close_tises: static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv) { + if (priv->mqprio_rl) { + mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); + mlx5e_mqprio_rl_free(priv->mqprio_rl); + priv->mqprio_rl = NULL; + } mlx5e_destroy_tises(priv); } @@ -3203,19 +3212,38 @@ static void mlx5e_params_mqprio_dcb_set(struct mlx5e_params *params, u8 num_tc) { params->mqprio.mode = TC_MQPRIO_MODE_DCB; params->mqprio.num_tc = num_tc; - params->mqprio.channel.rl = NULL; mlx5e_mqprio_build_default_tc_to_txq(params->mqprio.tc_to_txq, num_tc, params->num_channels); } +static void mlx5e_mqprio_rl_update_params(struct mlx5e_params *params, + struct mlx5e_mqprio_rl *rl) +{ + int tc; + + for (tc = 0; tc < TC_MAX_QUEUE; tc++) { + u32 hw_id = 0; + + if (rl) + mlx5e_mqprio_rl_get_node_hw_id(rl, tc, &hw_id); + params->mqprio.channel.hw_id[tc] = hw_id; + } +} + static void mlx5e_params_mqprio_channel_set(struct mlx5e_params *params, - struct tc_mqprio_qopt *qopt, + struct tc_mqprio_qopt_offload *mqprio, struct mlx5e_mqprio_rl *rl) { + int tc; + params->mqprio.mode = TC_MQPRIO_MODE_CHANNEL; - params->mqprio.num_tc = qopt->num_tc; - params->mqprio.channel.rl = rl; - mlx5e_mqprio_build_tc_to_txq(params->mqprio.tc_to_txq, qopt); + params->mqprio.num_tc = mqprio->qopt.num_tc; + + for (tc = 0; tc < TC_MAX_QUEUE; tc++) + params->mqprio.channel.max_rate[tc] = mqprio->max_rate[tc]; + + mlx5e_mqprio_rl_update_params(params, rl); + mlx5e_mqprio_build_tc_to_txq(params->mqprio.tc_to_txq, &mqprio->qopt); } static void mlx5e_params_mqprio_reset(struct mlx5e_params *params) @@ -3241,6 +3269,12 @@ static int mlx5e_setup_tc_mqprio_dcb(struct mlx5e_priv *priv, err = mlx5e_safe_switch_params(priv, &new_params, mlx5e_num_channels_changed_ctx, NULL, true); + if (!err && priv->mqprio_rl) { + mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); + mlx5e_mqprio_rl_free(priv->mqprio_rl); + priv->mqprio_rl = NULL; + } + priv->max_opened_tc = max_t(u8, priv->max_opened_tc, mlx5e_get_dcb_num_tc(&priv->channels.params)); return err; @@ -3299,16 +3333,38 @@ static int mlx5e_mqprio_channel_validate(struct mlx5e_priv *priv, return 0; } -static bool mlx5e_mqprio_rate_limit(struct tc_mqprio_qopt_offload *mqprio) +static bool mlx5e_mqprio_rate_limit(u8 num_tc, u64 max_rate[]) { int tc; - for (tc = 0; tc < mqprio->qopt.num_tc; tc++) - if (mqprio->max_rate[tc]) + for (tc = 0; tc < num_tc; tc++) + if (max_rate[tc]) return true; return false; } +static struct mlx5e_mqprio_rl *mlx5e_mqprio_rl_create(struct mlx5_core_dev *mdev, + u8 num_tc, u64 max_rate[]) +{ + struct mlx5e_mqprio_rl *rl; + int err; + + if (!mlx5e_mqprio_rate_limit(num_tc, max_rate)) + return NULL; + + rl = mlx5e_mqprio_rl_alloc(); + if (!rl) + return ERR_PTR(-ENOMEM); + + err = mlx5e_mqprio_rl_init(rl, mdev, num_tc, max_rate); + if (err) { + mlx5e_mqprio_rl_free(rl); + return ERR_PTR(err); + } + + return rl; +} + static int mlx5e_setup_tc_mqprio_channel(struct mlx5e_priv *priv, struct tc_mqprio_qopt_offload *mqprio) { @@ -3322,32 +3378,32 @@ static int mlx5e_setup_tc_mqprio_channel(struct mlx5e_priv *priv, if (err) return err; - rl = NULL; - if (mlx5e_mqprio_rate_limit(mqprio)) { - rl = mlx5e_mqprio_rl_alloc(); - if (!rl) - return -ENOMEM; - err = mlx5e_mqprio_rl_init(rl, priv->mdev, mqprio->qopt.num_tc, - mqprio->max_rate); - if (err) { - mlx5e_mqprio_rl_free(rl); - return err; - } - } + rl = mlx5e_mqprio_rl_create(priv->mdev, mqprio->qopt.num_tc, mqprio->max_rate); + if (IS_ERR(rl)) + return PTR_ERR(rl); new_params = priv->channels.params; - mlx5e_params_mqprio_channel_set(&new_params, &mqprio->qopt, rl); + mlx5e_params_mqprio_channel_set(&new_params, mqprio, rl); nch_changed = mlx5e_get_dcb_num_tc(&priv->channels.params) > 1; preactivate = nch_changed ? mlx5e_num_channels_changed_ctx : mlx5e_update_netdev_queues_ctx; err = mlx5e_safe_switch_params(priv, &new_params, preactivate, NULL, true); - if (err && rl) { - mlx5e_mqprio_rl_cleanup(rl); - mlx5e_mqprio_rl_free(rl); + if (err) { + if (rl) { + mlx5e_mqprio_rl_cleanup(rl); + mlx5e_mqprio_rl_free(rl); + } + return err; } - return err; + if (priv->mqprio_rl) { + mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); + mlx5e_mqprio_rl_free(priv->mqprio_rl); + } + priv->mqprio_rl = rl; + + return 0; } static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv, @@ -3356,7 +3412,7 @@ static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv, /* MQPRIO is another toplevel qdisc that can't be attached * simultaneously with the offloaded HTB. */ - if (WARN_ON(priv->htb.maj_id)) + if (WARN_ON(mlx5e_selq_is_htb_enabled(&priv->selq))) return -EINVAL; switch (mqprio->mode) { @@ -3369,47 +3425,6 @@ static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv, } } -static int mlx5e_setup_tc_htb(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb) -{ - int res; - - switch (htb->command) { - case TC_HTB_CREATE: - return mlx5e_htb_root_add(priv, htb->parent_classid, htb->classid, - htb->extack); - case TC_HTB_DESTROY: - return mlx5e_htb_root_del(priv); - case TC_HTB_LEAF_ALLOC_QUEUE: - res = mlx5e_htb_leaf_alloc_queue(priv, htb->classid, htb->parent_classid, - htb->rate, htb->ceil, htb->extack); - if (res < 0) - return res; - htb->qid = res; - return 0; - case TC_HTB_LEAF_TO_INNER: - return mlx5e_htb_leaf_to_inner(priv, htb->parent_classid, htb->classid, - htb->rate, htb->ceil, htb->extack); - case TC_HTB_LEAF_DEL: - return mlx5e_htb_leaf_del(priv, &htb->classid, htb->extack); - case TC_HTB_LEAF_DEL_LAST: - case TC_HTB_LEAF_DEL_LAST_FORCE: - return mlx5e_htb_leaf_del_last(priv, htb->classid, - htb->command == TC_HTB_LEAF_DEL_LAST_FORCE, - htb->extack); - case TC_HTB_NODE_MODIFY: - return mlx5e_htb_node_modify(priv, htb->classid, htb->rate, htb->ceil, - htb->extack); - case TC_HTB_LEAF_QUERY_QUEUE: - res = mlx5e_get_txq_by_classid(priv, htb->classid); - if (res < 0) - return res; - htb->qid = res; - return 0; - default: - return -EOPNOTSUPP; - } -} - static LIST_HEAD(mlx5e_block_cb_list); static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type, @@ -3443,7 +3458,7 @@ static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type, return err; case TC_SETUP_QDISC_HTB: mutex_lock(&priv->state_lock); - err = mlx5e_setup_tc_htb(priv, type_data); + err = mlx5e_htb_setup_tc(priv, type_data); mutex_unlock(&priv->state_lock); return err; default: @@ -3594,20 +3609,7 @@ static int set_feature_lro(struct net_device *netdev, bool enable) mutex_lock(&priv->state_lock); - if (enable && priv->xsk.refcnt) { - netdev_warn(netdev, "LRO is incompatible with AF_XDP (%u XSKs are active)\n", - priv->xsk.refcnt); - err = -EINVAL; - goto out; - } - cur_params = &priv->channels.params; - if (enable && !MLX5E_GET_PFLAG(cur_params, MLX5E_PFLAG_RX_STRIDING_RQ)) { - netdev_warn(netdev, "can't set LRO with legacy RQ\n"); - err = -EINVAL; - goto out; - } - new_params = *cur_params; if (enable) @@ -3676,6 +3678,7 @@ static int set_feature_cvlan_filter(struct net_device *netdev, bool enable) static int set_feature_hw_tc(struct net_device *netdev, bool enable) { struct mlx5e_priv *priv = netdev_priv(netdev); + int err = 0; #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) if (!enable && mlx5e_tc_num_filters(priv, MLX5_TC_FLAG(NIC_OFFLOAD))) { @@ -3685,12 +3688,14 @@ static int set_feature_hw_tc(struct net_device *netdev, bool enable) } #endif - if (!enable && priv->htb.maj_id) { + mutex_lock(&priv->state_lock); + if (!enable && mlx5e_selq_is_htb_enabled(&priv->selq)) { netdev_err(netdev, "Active HTB offload, can't turn hw_tc_offload off\n"); - return -EINVAL; + err = -EINVAL; } + mutex_unlock(&priv->state_lock); - return 0; + return err; } static int set_feature_rx_all(struct net_device *netdev, bool enable) @@ -3916,6 +3921,11 @@ static netdev_features_t mlx5e_fix_features(struct net_device *netdev, } if (priv->xsk.refcnt) { + if (features & NETIF_F_LRO) { + netdev_warn(netdev, "LRO is incompatible with AF_XDP (%u XSKs are active)\n", + priv->xsk.refcnt); + features &= ~NETIF_F_LRO; + } if (features & NETIF_F_GRO_HW) { netdev_warn(netdev, "HW GRO is incompatible with AF_XDP (%u XSKs are active)\n", priv->xsk.refcnt); @@ -5110,6 +5120,23 @@ static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv) priv->rx_res = NULL; } +static void mlx5e_set_mqprio_rl(struct mlx5e_priv *priv) +{ + struct mlx5e_params *params; + struct mlx5e_mqprio_rl *rl; + + params = &priv->channels.params; + if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL) + return; + + rl = mlx5e_mqprio_rl_create(priv->mdev, params->mqprio.num_tc, + params->mqprio.channel.max_rate); + if (IS_ERR(rl)) + rl = NULL; + priv->mqprio_rl = rl; + mlx5e_mqprio_rl_update_params(params, rl); +} + static int mlx5e_init_nic_tx(struct mlx5e_priv *priv) { int err; @@ -5120,6 +5147,7 @@ static int mlx5e_init_nic_tx(struct mlx5e_priv *priv) return err; } + mlx5e_set_mqprio_rl(priv); mlx5e_dcbnl_initialize(priv); return 0; } @@ -5293,7 +5321,6 @@ int mlx5e_priv_init(struct mlx5e_priv *priv, if (err) goto err_free_cpumask; - hash_init(priv->htb.qos_tc2node); INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work); INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work); INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work); @@ -5350,14 +5377,9 @@ void mlx5e_priv_cleanup(struct mlx5e_priv *priv) mutex_unlock(&priv->state_lock); free_cpumask_var(priv->scratchpad.cpumask); - for (i = 0; i < priv->htb.max_qos_sqs; i++) - kfree(priv->htb.qos_sq_stats[i]); - kvfree(priv->htb.qos_sq_stats); - - if (priv->mqprio_rl) { - mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); - mlx5e_mqprio_rl_free(priv->mqprio_rl); - } + for (i = 0; i < priv->htb_max_qos_sqs; i++) + kfree(priv->htb_qos_sq_stats[i]); + kvfree(priv->htb_qos_sq_stats); memset(priv, 0, sizeof(*priv)); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index f797fd97d305..ae90b06d21e2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -229,7 +229,7 @@ mlx5e_rep_get_ringparam(struct net_device *dev, { struct mlx5e_priv *priv = netdev_priv(dev); - mlx5e_ethtool_get_ringparam(priv, param); + mlx5e_ethtool_get_ringparam(priv, param, kernel_param); } static int diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h index adf5cc6a7b8c..dec183ccd4ac 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h @@ -62,6 +62,7 @@ struct mlx5_tc_int_port_priv; struct mlx5e_rep_bond; struct mlx5e_tc_tun_encap; struct mlx5e_post_act; +struct mlx5e_flow_meters; struct mlx5_rep_uplink_priv { /* indirect block callbacks are invoked on bind/unbind events @@ -97,6 +98,8 @@ struct mlx5_rep_uplink_priv { /* OVS internal port support */ struct mlx5e_tc_int_port_priv *int_port_priv; + + struct mlx5e_flow_meters *flow_meters; }; struct mlx5e_rep_priv { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index 1e87bb2b7541..7409829d1201 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -474,8 +474,8 @@ static void mlx5e_stats_grp_sw_update_stats_qos(struct mlx5e_priv *priv, int i; /* Pairs with smp_store_release in mlx5e_open_qos_sq. */ - max_qos_sqs = smp_load_acquire(&priv->htb.max_qos_sqs); - stats = READ_ONCE(priv->htb.qos_sq_stats); + max_qos_sqs = smp_load_acquire(&priv->htb_max_qos_sqs); + stats = READ_ONCE(priv->htb_qos_sq_stats); for (i = 0; i < max_qos_sqs; i++) { mlx5e_stats_grp_sw_update_stats_sq(s, READ_ONCE(stats[i])); @@ -2100,6 +2100,8 @@ static const struct counter_desc ptp_cq_stats_desc[] = { { MLX5E_DECLARE_PTP_CQ_STAT(struct mlx5e_ptp_cq_stats, err_cqe) }, { MLX5E_DECLARE_PTP_CQ_STAT(struct mlx5e_ptp_cq_stats, abort) }, { MLX5E_DECLARE_PTP_CQ_STAT(struct mlx5e_ptp_cq_stats, abort_abs_diff_ns) }, + { MLX5E_DECLARE_PTP_CQ_STAT(struct mlx5e_ptp_cq_stats, resync_cqe) }, + { MLX5E_DECLARE_PTP_CQ_STAT(struct mlx5e_ptp_cq_stats, resync_event) }, }; static const struct counter_desc ptp_rq_stats_desc[] = { @@ -2184,13 +2186,13 @@ static const struct counter_desc qos_sq_stats_desc[] = { static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(qos) { /* Pairs with smp_store_release in mlx5e_open_qos_sq. */ - return NUM_QOS_SQ_STATS * smp_load_acquire(&priv->htb.max_qos_sqs); + return NUM_QOS_SQ_STATS * smp_load_acquire(&priv->htb_max_qos_sqs); } static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(qos) { /* Pairs with smp_store_release in mlx5e_open_qos_sq. */ - u16 max_qos_sqs = smp_load_acquire(&priv->htb.max_qos_sqs); + u16 max_qos_sqs = smp_load_acquire(&priv->htb_max_qos_sqs); int i, qid; for (qid = 0; qid < max_qos_sqs; qid++) @@ -2208,8 +2210,8 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(qos) int i, qid; /* Pairs with smp_store_release in mlx5e_open_qos_sq. */ - max_qos_sqs = smp_load_acquire(&priv->htb.max_qos_sqs); - stats = READ_ONCE(priv->htb.qos_sq_stats); + max_qos_sqs = smp_load_acquire(&priv->htb_max_qos_sqs); + stats = READ_ONCE(priv->htb_qos_sq_stats); for (qid = 0; qid < max_qos_sqs; qid++) { struct mlx5e_sq_stats *s = READ_ONCE(stats[qid]); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h index e48b15b55b6f..ed4fc940e4ef 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h @@ -453,6 +453,8 @@ struct mlx5e_ptp_cq_stats { u64 err_cqe; u64 abort; u64 abort_abs_diff_ns; + u64 resync_cqe; + u64 resync_event; }; struct mlx5e_stats { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index 9ca2c8763237..2e12280a936f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -59,6 +59,7 @@ #include "en/tc_tun_encap.h" #include "en/tc/sample.h" #include "en/tc/act/act.h" +#include "en/tc/post_meter.h" #include "lib/devcom.h" #include "lib/geneve.h" #include "lib/fs_chains.h" @@ -104,6 +105,7 @@ struct mlx5e_tc_attr_to_reg_mapping mlx5e_tc_attr_to_reg_mappings[] = { .mlen = 16, }, [NIC_ZONE_RESTORE_TO_REG] = nic_zone_restore_to_reg_ct, + [PACKET_COLOR_TO_REG] = packet_color_to_reg, }; /* To avoid false lock dependency warning set the tc_ht lock @@ -240,6 +242,30 @@ mlx5e_get_int_port_priv(struct mlx5e_priv *priv) return NULL; } +struct mlx5e_flow_meters * +mlx5e_get_flow_meters(struct mlx5_core_dev *dev) +{ + struct mlx5_eswitch *esw = dev->priv.eswitch; + struct mlx5_rep_uplink_priv *uplink_priv; + struct mlx5e_rep_priv *uplink_rpriv; + struct mlx5e_priv *priv; + + if (is_mdev_switchdev_mode(dev)) { + uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH); + uplink_priv = &uplink_rpriv->uplink_priv; + priv = netdev_priv(uplink_rpriv->netdev); + if (!uplink_priv->flow_meters) + uplink_priv->flow_meters = + mlx5e_flow_meters_init(priv, + MLX5_FLOW_NAMESPACE_FDB, + uplink_priv->post_act); + if (!IS_ERR(uplink_priv->flow_meters)) + return uplink_priv->flow_meters; + } + + return NULL; +} + static struct mlx5_tc_ct_priv * get_ct_priv(struct mlx5e_priv *priv) { @@ -319,12 +345,39 @@ mlx5_tc_rule_delete(struct mlx5e_priv *priv, mlx5e_del_offloaded_nic_rule(priv, rule, attr); } +static bool +is_flow_meter_action(struct mlx5_flow_attr *attr) +{ + return ((attr->action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) && + (attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER)); +} + +static int +mlx5e_tc_add_flow_meter(struct mlx5e_priv *priv, + struct mlx5_flow_attr *attr) +{ + struct mlx5e_flow_meter_handle *meter; + + meter = mlx5e_tc_meter_get(priv->mdev, &attr->meter_attr.params); + if (IS_ERR(meter)) { + mlx5_core_err(priv->mdev, "Failed to get flow meter\n"); + return PTR_ERR(meter); + } + + attr->meter_attr.meter = meter; + attr->dest_ft = mlx5e_tc_meter_get_post_meter_ft(meter->flow_meters); + attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + + return 0; +} + struct mlx5_flow_handle * mlx5e_tc_rule_offload(struct mlx5e_priv *priv, struct mlx5_flow_spec *spec, struct mlx5_flow_attr *attr) { struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; + int err; if (attr->flags & MLX5_ATTR_FLAG_CT) { struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts = @@ -341,6 +394,12 @@ mlx5e_tc_rule_offload(struct mlx5e_priv *priv, if (attr->flags & MLX5_ATTR_FLAG_SAMPLE) return mlx5e_tc_sample_offload(get_sample_priv(priv), spec, attr); + if (is_flow_meter_action(attr)) { + err = mlx5e_tc_add_flow_meter(priv, attr); + if (err) + return ERR_PTR(err); + } + return mlx5_eswitch_add_offloaded_rule(esw, spec, attr); } @@ -367,6 +426,9 @@ mlx5e_tc_rule_unoffload(struct mlx5e_priv *priv, } mlx5_eswitch_del_offloaded_rule(esw, rule, attr); + + if (attr->meter_attr.meter) + mlx5e_tc_meter_put(attr->meter_attr.meter); } int @@ -4519,9 +4581,9 @@ static int apply_police_params(struct mlx5e_priv *priv, u64 rate, return err; } -static int mlx5e_policer_validate(const struct flow_action *action, - const struct flow_action_entry *act, - struct netlink_ext_ack *extack) +int mlx5e_policer_validate(const struct flow_action *action, + const struct flow_action_entry *act, + struct netlink_ext_ack *extack) { if (act->police.exceed.act_id != FLOW_ACTION_DROP) { NL_SET_ERR_MSG_MOD(extack, @@ -4955,6 +5017,7 @@ void mlx5e_tc_esw_cleanup(struct mlx5_rep_uplink_priv *uplink_priv) mlx5e_tc_sample_cleanup(uplink_priv->tc_psample); mlx5e_tc_int_port_cleanup(uplink_priv->int_port_priv); mlx5_tc_ct_clean(uplink_priv->ct_priv); + mlx5e_flow_meters_cleanup(uplink_priv->flow_meters); mlx5e_tc_post_act_destroy(uplink_priv->post_act); } @@ -5060,7 +5123,7 @@ bool mlx5e_tc_update_skb(struct mlx5_cqe64 *cqe, tc_skb_ext->chain = chain; - zone_restore_id = (reg_b >> REG_MAPPING_MOFFSET(NIC_ZONE_RESTORE_TO_REG)) & + zone_restore_id = (reg_b >> MLX5_REG_MAPPING_MOFFSET(NIC_ZONE_RESTORE_TO_REG)) & ESW_ZONE_ID_MASK; if (!mlx5e_tc_ct_restore_flow(tc->ct, skb, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h index e2a1250aeca1..517f2252b5ff 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h @@ -39,6 +39,7 @@ #include "en/tc_ct.h" #include "en/tc_tun.h" #include "en/tc/int_port.h" +#include "en/tc/meter.h" #include "en_rep.h" #define MLX5E_TC_FLOW_ID_MASK 0x0000ffff @@ -71,6 +72,7 @@ struct mlx5_flow_attr { struct mlx5_modify_hdr *modify_hdr; struct mlx5_ct_attr ct_attr; struct mlx5e_sample_attr sample_attr; + struct mlx5e_meter_attr meter_attr; struct mlx5e_tc_flow_parse_attr *parse_attr; u32 chain; u16 prio; @@ -83,6 +85,7 @@ struct mlx5_flow_attr { u8 tun_ip_version; int tunnel_id; /* mapped tunnel id */ u32 flags; + u32 exe_aso_type; struct list_head list; struct mlx5e_post_act_handle *post_act_handle; struct { @@ -229,6 +232,7 @@ enum mlx5e_tc_attr_to_reg { FTEID_TO_REG, NIC_CHAIN_TO_REG, NIC_ZONE_RESTORE_TO_REG, + PACKET_COLOR_TO_REG, }; struct mlx5e_tc_attr_to_reg_mapping { @@ -241,6 +245,10 @@ struct mlx5e_tc_attr_to_reg_mapping { extern struct mlx5e_tc_attr_to_reg_mapping mlx5e_tc_attr_to_reg_mappings[]; +#define MLX5_REG_MAPPING_MOFFSET(reg_id) (mlx5e_tc_attr_to_reg_mappings[reg_id].moffset) +#define MLX5_REG_MAPPING_MBITS(reg_id) (mlx5e_tc_attr_to_reg_mappings[reg_id].mlen) +#define MLX5_REG_MAPPING_MASK(reg_id) (GENMASK(mlx5e_tc_attr_to_reg_mappings[reg_id].mlen - 1, 0)) + bool mlx5e_is_valid_eswitch_fwd_dev(struct mlx5e_priv *priv, struct net_device *out_dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c index 9a7250be229f..dc1e01e93d5a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c @@ -152,14 +152,14 @@ mlx5e_tx_get_gso_ihs(struct mlx5e_txqsq *sq, struct sk_buff *skb, int *hopbyhop) *hopbyhop = 0; if (skb->encapsulation) { - ihs = skb_inner_transport_offset(skb) + inner_tcp_hdrlen(skb); + ihs = skb_tcp_all_headers(skb); stats->tso_inner_packets++; stats->tso_inner_bytes += skb->len - ihs; } else { if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { ihs = skb_transport_offset(skb) + sizeof(struct udphdr); } else { - ihs = skb_transport_offset(skb) + tcp_hdrlen(skb); + ihs = skb_tcp_all_headers(skb); if (ipv6_has_hopopt_jumbo(skb)) { *hopbyhop = sizeof(struct hop_jumbo_hdr); ihs -= sizeof(struct hop_jumbo_hdr); @@ -631,12 +631,22 @@ void mlx5e_tx_mpwqe_ensure_complete(struct mlx5e_txqsq *sq) mlx5e_tx_mpwqe_session_complete(sq); } +static void mlx5e_cqe_ts_id_eseg(struct mlx5e_ptpsq *ptpsq, struct sk_buff *skb, + struct mlx5_wqe_eth_seg *eseg) +{ + if (ptpsq->ts_cqe_ctr_mask && unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) + eseg->flow_table_metadata = cpu_to_be32(ptpsq->skb_fifo_pc & + ptpsq->ts_cqe_ctr_mask); +} + static void mlx5e_txwqe_build_eseg(struct mlx5e_priv *priv, struct mlx5e_txqsq *sq, struct sk_buff *skb, struct mlx5e_accel_tx_state *accel, struct mlx5_wqe_eth_seg *eseg, u16 ihs) { mlx5e_accel_tx_eseg(priv, skb, eseg, ihs); mlx5e_txwqe_build_eseg_csum(sq, skb, accel, eseg); + if (unlikely(sq->ptpsq)) + mlx5e_cqe_ts_id_eseg(sq->ptpsq, skb, eseg); } netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c index 05e08cec5a8c..4fbff7bcc155 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* Copyright (c) 2021 Mellanox Technologies. */ +#include <linux/build_bug.h> #include <linux/list.h> #include <linux/notifier.h> #include <net/netevent.h> @@ -12,26 +13,57 @@ #define CREATE_TRACE_POINTS #include "diag/bridge_tracepoint.h" -#define MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE 64000 +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_SIZE 12000 +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_UNTAGGED_GRP_SIZE 16000 #define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_FROM 0 -#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_TO (MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE / 4 - 1) -#define MLX5_ESW_BRIDGE_INGRESS_TABLE_FILTER_GRP_IDX_FROM \ +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_TO \ + (MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_SIZE - 1) +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_FROM \ (MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_TO + 1) -#define MLX5_ESW_BRIDGE_INGRESS_TABLE_FILTER_GRP_IDX_TO \ - (MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE / 2 - 1) -#define MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_FROM \ - (MLX5_ESW_BRIDGE_INGRESS_TABLE_FILTER_GRP_IDX_TO + 1) -#define MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_TO (MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE - 1) - -#define MLX5_ESW_BRIDGE_EGRESS_TABLE_SIZE 64000 +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_TO \ + (MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_FROM + \ + MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_SIZE - 1) +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_FROM \ + (MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_TO + 1) +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_TO \ + (MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_FROM + \ + MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_SIZE - 1) +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_FROM \ + (MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_TO + 1) +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_TO \ + (MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_FROM + \ + MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_SIZE - 1) +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_FROM \ + (MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_TO + 1) +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_TO \ + (MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_FROM + \ + MLX5_ESW_BRIDGE_INGRESS_TABLE_UNTAGGED_GRP_SIZE - 1) +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE \ + (MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_TO + 1) +static_assert(MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE == 64000); + +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_SIZE 16000 +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_SIZE (32000 - 1) #define MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_FROM 0 -#define MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_TO (MLX5_ESW_BRIDGE_EGRESS_TABLE_SIZE / 2 - 1) -#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_FROM \ +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_TO \ + (MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_SIZE - 1) +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_FROM \ (MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_TO + 1) -#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_TO (MLX5_ESW_BRIDGE_EGRESS_TABLE_SIZE - 2) +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_TO \ + (MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_FROM + \ + MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_SIZE - 1) +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_FROM \ + (MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_TO + 1) +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_TO \ + (MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_FROM + \ + MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_SIZE - 1) #define MLX5_ESW_BRIDGE_EGRESS_TABLE_MISS_GRP_IDX_FROM \ (MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_TO + 1) -#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MISS_GRP_IDX_TO (MLX5_ESW_BRIDGE_EGRESS_TABLE_SIZE - 1) +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MISS_GRP_IDX_TO \ + MLX5_ESW_BRIDGE_EGRESS_TABLE_MISS_GRP_IDX_FROM +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_SIZE \ + (MLX5_ESW_BRIDGE_EGRESS_TABLE_MISS_GRP_IDX_TO + 1) +static_assert(MLX5_ESW_BRIDGE_EGRESS_TABLE_SIZE == 64000); #define MLX5_ESW_BRIDGE_SKIP_TABLE_SIZE 0 @@ -63,12 +95,14 @@ struct mlx5_esw_bridge { struct mlx5_flow_table *egress_ft; struct mlx5_flow_group *egress_vlan_fg; + struct mlx5_flow_group *egress_qinq_fg; struct mlx5_flow_group *egress_mac_fg; struct mlx5_flow_group *egress_miss_fg; struct mlx5_pkt_reformat *egress_miss_pkt_reformat; struct mlx5_flow_handle *egress_miss_handle; unsigned long ageing_time; u32 flags; + u16 vlan_proto; }; static void @@ -138,7 +172,9 @@ mlx5_esw_bridge_table_create(int max_fte, u32 level, struct mlx5_eswitch *esw) } static struct mlx5_flow_group * -mlx5_esw_bridge_ingress_vlan_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *ingress_ft) +mlx5_esw_bridge_ingress_vlan_proto_fg_create(unsigned int from, unsigned int to, u16 vlan_proto, + struct mlx5_eswitch *esw, + struct mlx5_flow_table *ingress_ft) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); struct mlx5_flow_group *fg; @@ -154,30 +190,53 @@ mlx5_esw_bridge_ingress_vlan_fg_create(struct mlx5_eswitch *esw, struct mlx5_flo MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.smac_47_16); MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.smac_15_0); - MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.cvlan_tag); + if (vlan_proto == ETH_P_8021Q) + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.cvlan_tag); + else if (vlan_proto == ETH_P_8021AD) + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.svlan_tag); MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.first_vid); MLX5_SET(fte_match_param, match, misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask()); - MLX5_SET(create_flow_group_in, in, start_flow_index, - MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_FROM); - MLX5_SET(create_flow_group_in, in, end_flow_index, - MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_TO); + MLX5_SET(create_flow_group_in, in, start_flow_index, from); + MLX5_SET(create_flow_group_in, in, end_flow_index, to); fg = mlx5_create_flow_group(ingress_ft, in); kvfree(in); if (IS_ERR(fg)) esw_warn(esw->dev, - "Failed to create VLAN flow group for bridge ingress table (err=%ld)\n", - PTR_ERR(fg)); + "Failed to create VLAN(proto=%x) flow group for bridge ingress table (err=%ld)\n", + vlan_proto, PTR_ERR(fg)); return fg; } static struct mlx5_flow_group * -mlx5_esw_bridge_ingress_filter_fg_create(struct mlx5_eswitch *esw, - struct mlx5_flow_table *ingress_ft) +mlx5_esw_bridge_ingress_vlan_fg_create(struct mlx5_eswitch *esw, + struct mlx5_flow_table *ingress_ft) +{ + unsigned int from = MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_FROM; + unsigned int to = MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_TO; + + return mlx5_esw_bridge_ingress_vlan_proto_fg_create(from, to, ETH_P_8021Q, esw, ingress_ft); +} + +static struct mlx5_flow_group * +mlx5_esw_bridge_ingress_qinq_fg_create(struct mlx5_eswitch *esw, + struct mlx5_flow_table *ingress_ft) +{ + unsigned int from = MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_FROM; + unsigned int to = MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_TO; + + return mlx5_esw_bridge_ingress_vlan_proto_fg_create(from, to, ETH_P_8021AD, esw, + ingress_ft); +} + +static struct mlx5_flow_group * +mlx5_esw_bridge_ingress_vlan_proto_filter_fg_create(unsigned int from, unsigned int to, + u16 vlan_proto, struct mlx5_eswitch *esw, + struct mlx5_flow_table *ingress_ft) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); struct mlx5_flow_group *fg; @@ -193,27 +252,48 @@ mlx5_esw_bridge_ingress_filter_fg_create(struct mlx5_eswitch *esw, MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.smac_47_16); MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.smac_15_0); - MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.cvlan_tag); - + if (vlan_proto == ETH_P_8021Q) + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.cvlan_tag); + else if (vlan_proto == ETH_P_8021AD) + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.svlan_tag); MLX5_SET(fte_match_param, match, misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask()); - MLX5_SET(create_flow_group_in, in, start_flow_index, - MLX5_ESW_BRIDGE_INGRESS_TABLE_FILTER_GRP_IDX_FROM); - MLX5_SET(create_flow_group_in, in, end_flow_index, - MLX5_ESW_BRIDGE_INGRESS_TABLE_FILTER_GRP_IDX_TO); + MLX5_SET(create_flow_group_in, in, start_flow_index, from); + MLX5_SET(create_flow_group_in, in, end_flow_index, to); fg = mlx5_create_flow_group(ingress_ft, in); if (IS_ERR(fg)) esw_warn(esw->dev, "Failed to create bridge ingress table VLAN filter flow group (err=%ld)\n", PTR_ERR(fg)); - kvfree(in); return fg; } static struct mlx5_flow_group * +mlx5_esw_bridge_ingress_vlan_filter_fg_create(struct mlx5_eswitch *esw, + struct mlx5_flow_table *ingress_ft) +{ + unsigned int from = MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_FROM; + unsigned int to = MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_TO; + + return mlx5_esw_bridge_ingress_vlan_proto_filter_fg_create(from, to, ETH_P_8021Q, esw, + ingress_ft); +} + +static struct mlx5_flow_group * +mlx5_esw_bridge_ingress_qinq_filter_fg_create(struct mlx5_eswitch *esw, + struct mlx5_flow_table *ingress_ft) +{ + unsigned int from = MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_FROM; + unsigned int to = MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_TO; + + return mlx5_esw_bridge_ingress_vlan_proto_filter_fg_create(from, to, ETH_P_8021AD, esw, + ingress_ft); +} + +static struct mlx5_flow_group * mlx5_esw_bridge_ingress_mac_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *ingress_ft) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); @@ -250,7 +330,9 @@ mlx5_esw_bridge_ingress_mac_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow } static struct mlx5_flow_group * -mlx5_esw_bridge_egress_vlan_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *egress_ft) +mlx5_esw_bridge_egress_vlan_proto_fg_create(unsigned int from, unsigned int to, u16 vlan_proto, + struct mlx5_eswitch *esw, + struct mlx5_flow_table *egress_ft) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); struct mlx5_flow_group *fg; @@ -265,13 +347,14 @@ mlx5_esw_bridge_egress_vlan_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.dmac_47_16); MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.dmac_15_0); - MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.cvlan_tag); + if (vlan_proto == ETH_P_8021Q) + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.cvlan_tag); + else if (vlan_proto == ETH_P_8021AD) + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.svlan_tag); MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.first_vid); - MLX5_SET(create_flow_group_in, in, start_flow_index, - MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_FROM); - MLX5_SET(create_flow_group_in, in, end_flow_index, - MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_TO); + MLX5_SET(create_flow_group_in, in, start_flow_index, from); + MLX5_SET(create_flow_group_in, in, end_flow_index, to); fg = mlx5_create_flow_group(egress_ft, in); if (IS_ERR(fg)) @@ -283,6 +366,25 @@ mlx5_esw_bridge_egress_vlan_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow } static struct mlx5_flow_group * +mlx5_esw_bridge_egress_vlan_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *egress_ft) +{ + unsigned int from = MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_FROM; + unsigned int to = MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_TO; + + return mlx5_esw_bridge_egress_vlan_proto_fg_create(from, to, ETH_P_8021Q, esw, egress_ft); +} + +static struct mlx5_flow_group * +mlx5_esw_bridge_egress_qinq_fg_create(struct mlx5_eswitch *esw, + struct mlx5_flow_table *egress_ft) +{ + unsigned int from = MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_FROM; + unsigned int to = MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_TO; + + return mlx5_esw_bridge_egress_vlan_proto_fg_create(from, to, ETH_P_8021AD, esw, egress_ft); +} + +static struct mlx5_flow_group * mlx5_esw_bridge_egress_mac_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *egress_ft) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); @@ -346,7 +448,7 @@ mlx5_esw_bridge_egress_miss_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow static int mlx5_esw_bridge_ingress_table_init(struct mlx5_esw_bridge_offloads *br_offloads) { - struct mlx5_flow_group *mac_fg, *filter_fg, *vlan_fg; + struct mlx5_flow_group *mac_fg, *qinq_filter_fg, *qinq_fg, *vlan_filter_fg, *vlan_fg; struct mlx5_flow_table *ingress_ft, *skip_ft; struct mlx5_eswitch *esw = br_offloads->esw; int err; @@ -374,10 +476,22 @@ mlx5_esw_bridge_ingress_table_init(struct mlx5_esw_bridge_offloads *br_offloads) goto err_vlan_fg; } - filter_fg = mlx5_esw_bridge_ingress_filter_fg_create(esw, ingress_ft); - if (IS_ERR(filter_fg)) { - err = PTR_ERR(filter_fg); - goto err_filter_fg; + vlan_filter_fg = mlx5_esw_bridge_ingress_vlan_filter_fg_create(esw, ingress_ft); + if (IS_ERR(vlan_filter_fg)) { + err = PTR_ERR(vlan_filter_fg); + goto err_vlan_filter_fg; + } + + qinq_fg = mlx5_esw_bridge_ingress_qinq_fg_create(esw, ingress_ft); + if (IS_ERR(qinq_fg)) { + err = PTR_ERR(qinq_fg); + goto err_qinq_fg; + } + + qinq_filter_fg = mlx5_esw_bridge_ingress_qinq_filter_fg_create(esw, ingress_ft); + if (IS_ERR(qinq_filter_fg)) { + err = PTR_ERR(qinq_filter_fg); + goto err_qinq_filter_fg; } mac_fg = mlx5_esw_bridge_ingress_mac_fg_create(esw, ingress_ft); @@ -389,13 +503,19 @@ mlx5_esw_bridge_ingress_table_init(struct mlx5_esw_bridge_offloads *br_offloads) br_offloads->ingress_ft = ingress_ft; br_offloads->skip_ft = skip_ft; br_offloads->ingress_vlan_fg = vlan_fg; - br_offloads->ingress_filter_fg = filter_fg; + br_offloads->ingress_vlan_filter_fg = vlan_filter_fg; + br_offloads->ingress_qinq_fg = qinq_fg; + br_offloads->ingress_qinq_filter_fg = qinq_filter_fg; br_offloads->ingress_mac_fg = mac_fg; return 0; err_mac_fg: - mlx5_destroy_flow_group(filter_fg); -err_filter_fg: + mlx5_destroy_flow_group(qinq_filter_fg); +err_qinq_filter_fg: + mlx5_destroy_flow_group(qinq_fg); +err_qinq_fg: + mlx5_destroy_flow_group(vlan_filter_fg); +err_vlan_filter_fg: mlx5_destroy_flow_group(vlan_fg); err_vlan_fg: mlx5_destroy_flow_table(skip_ft); @@ -409,8 +529,12 @@ mlx5_esw_bridge_ingress_table_cleanup(struct mlx5_esw_bridge_offloads *br_offloa { mlx5_destroy_flow_group(br_offloads->ingress_mac_fg); br_offloads->ingress_mac_fg = NULL; - mlx5_destroy_flow_group(br_offloads->ingress_filter_fg); - br_offloads->ingress_filter_fg = NULL; + mlx5_destroy_flow_group(br_offloads->ingress_qinq_filter_fg); + br_offloads->ingress_qinq_filter_fg = NULL; + mlx5_destroy_flow_group(br_offloads->ingress_qinq_fg); + br_offloads->ingress_qinq_fg = NULL; + mlx5_destroy_flow_group(br_offloads->ingress_vlan_filter_fg); + br_offloads->ingress_vlan_filter_fg = NULL; mlx5_destroy_flow_group(br_offloads->ingress_vlan_fg); br_offloads->ingress_vlan_fg = NULL; mlx5_destroy_flow_table(br_offloads->skip_ft); @@ -428,7 +552,7 @@ static int mlx5_esw_bridge_egress_table_init(struct mlx5_esw_bridge_offloads *br_offloads, struct mlx5_esw_bridge *bridge) { - struct mlx5_flow_group *miss_fg = NULL, *mac_fg, *vlan_fg; + struct mlx5_flow_group *miss_fg = NULL, *mac_fg, *vlan_fg, *qinq_fg; struct mlx5_pkt_reformat *miss_pkt_reformat = NULL; struct mlx5_flow_handle *miss_handle = NULL; struct mlx5_eswitch *esw = br_offloads->esw; @@ -447,6 +571,12 @@ mlx5_esw_bridge_egress_table_init(struct mlx5_esw_bridge_offloads *br_offloads, goto err_vlan_fg; } + qinq_fg = mlx5_esw_bridge_egress_qinq_fg_create(esw, egress_ft); + if (IS_ERR(qinq_fg)) { + err = PTR_ERR(qinq_fg); + goto err_qinq_fg; + } + mac_fg = mlx5_esw_bridge_egress_mac_fg_create(esw, egress_ft); if (IS_ERR(mac_fg)) { err = PTR_ERR(mac_fg); @@ -491,6 +621,7 @@ skip_miss_flow: bridge->egress_ft = egress_ft; bridge->egress_vlan_fg = vlan_fg; + bridge->egress_qinq_fg = qinq_fg; bridge->egress_mac_fg = mac_fg; bridge->egress_miss_fg = miss_fg; bridge->egress_miss_pkt_reformat = miss_pkt_reformat; @@ -498,6 +629,8 @@ skip_miss_flow: return 0; err_mac_fg: + mlx5_destroy_flow_group(qinq_fg); +err_qinq_fg: mlx5_destroy_flow_group(vlan_fg); err_vlan_fg: mlx5_destroy_flow_table(egress_ft); @@ -515,6 +648,7 @@ mlx5_esw_bridge_egress_table_cleanup(struct mlx5_esw_bridge *bridge) if (bridge->egress_miss_fg) mlx5_destroy_flow_group(bridge->egress_miss_fg); mlx5_destroy_flow_group(bridge->egress_mac_fg); + mlx5_destroy_flow_group(bridge->egress_qinq_fg); mlx5_destroy_flow_group(bridge->egress_vlan_fg); mlx5_destroy_flow_table(bridge->egress_ft); } @@ -559,10 +693,17 @@ mlx5_esw_bridge_ingress_flow_with_esw_create(u16 vport_num, const unsigned char flow_act.pkt_reformat = vlan->pkt_reformat_push; flow_act.modify_hdr = vlan->pkt_mod_hdr_push_mark; } else if (vlan) { - MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, - outer_headers.cvlan_tag); - MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value, - outer_headers.cvlan_tag); + if (bridge->vlan_proto == ETH_P_8021Q) { + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, + outer_headers.cvlan_tag); + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value, + outer_headers.cvlan_tag); + } else if (bridge->vlan_proto == ETH_P_8021AD) { + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, + outer_headers.svlan_tag); + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value, + outer_headers.svlan_tag); + } MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, outer_headers.first_vid); MLX5_SET(fte_match_param, rule_spec->match_value, outer_headers.first_vid, @@ -645,10 +786,17 @@ mlx5_esw_bridge_ingress_filter_flow_create(u16 vport_num, const unsigned char *a MLX5_SET(fte_match_param, rule_spec->match_value, misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_for_match(br_offloads->esw, vport_num)); - MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, - outer_headers.cvlan_tag); - MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value, - outer_headers.cvlan_tag); + if (bridge->vlan_proto == ETH_P_8021Q) { + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, + outer_headers.cvlan_tag); + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value, + outer_headers.cvlan_tag); + } else if (bridge->vlan_proto == ETH_P_8021AD) { + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, + outer_headers.svlan_tag); + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value, + outer_headers.svlan_tag); + } handle = mlx5_add_flow_rules(br_offloads->ingress_ft, rule_spec, &flow_act, &dest, 1); @@ -696,10 +844,17 @@ mlx5_esw_bridge_egress_flow_create(u16 vport_num, u16 esw_owner_vhca_id, const u flow_act.pkt_reformat = vlan->pkt_reformat_pop; } - MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, - outer_headers.cvlan_tag); - MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value, - outer_headers.cvlan_tag); + if (bridge->vlan_proto == ETH_P_8021Q) { + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, + outer_headers.cvlan_tag); + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value, + outer_headers.cvlan_tag); + } else if (bridge->vlan_proto == ETH_P_8021AD) { + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, + outer_headers.svlan_tag); + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value, + outer_headers.svlan_tag); + } MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, outer_headers.first_vid); MLX5_SET(fte_match_param, rule_spec->match_value, outer_headers.first_vid, @@ -774,6 +929,7 @@ static struct mlx5_esw_bridge *mlx5_esw_bridge_create(int ifindex, bridge->ifindex = ifindex; bridge->refcnt = 1; bridge->ageing_time = clock_t_to_jiffies(BR_DEFAULT_AGEING_TIME); + bridge->vlan_proto = ETH_P_8021Q; list_add(&bridge->list, &br_offloads->bridges); return bridge; @@ -911,12 +1067,13 @@ mlx5_esw_bridge_vlan_lookup(u16 vid, struct mlx5_esw_bridge_port *port) } static int -mlx5_esw_bridge_vlan_push_create(struct mlx5_esw_bridge_vlan *vlan, struct mlx5_eswitch *esw) +mlx5_esw_bridge_vlan_push_create(u16 vlan_proto, struct mlx5_esw_bridge_vlan *vlan, + struct mlx5_eswitch *esw) { struct { __be16 h_vlan_proto; __be16 h_vlan_TCI; - } vlan_hdr = { htons(ETH_P_8021Q), htons(vlan->vid) }; + } vlan_hdr = { htons(vlan_proto), htons(vlan->vid) }; struct mlx5_pkt_reformat_params reformat_params = {}; struct mlx5_pkt_reformat *pkt_reformat; @@ -1008,36 +1165,58 @@ mlx5_esw_bridge_vlan_push_mark_cleanup(struct mlx5_esw_bridge_vlan *vlan, struct vlan->pkt_mod_hdr_push_mark = NULL; } -static struct mlx5_esw_bridge_vlan * -mlx5_esw_bridge_vlan_create(u16 vid, u16 flags, struct mlx5_esw_bridge_port *port, - struct mlx5_eswitch *esw) +static int +mlx5_esw_bridge_vlan_push_pop_create(u16 vlan_proto, u16 flags, struct mlx5_esw_bridge_vlan *vlan, + struct mlx5_eswitch *esw) { - struct mlx5_esw_bridge_vlan *vlan; int err; - vlan = kvzalloc(sizeof(*vlan), GFP_KERNEL); - if (!vlan) - return ERR_PTR(-ENOMEM); - - vlan->vid = vid; - vlan->flags = flags; - INIT_LIST_HEAD(&vlan->fdb_list); - if (flags & BRIDGE_VLAN_INFO_PVID) { - err = mlx5_esw_bridge_vlan_push_create(vlan, esw); + err = mlx5_esw_bridge_vlan_push_create(vlan_proto, vlan, esw); if (err) - goto err_vlan_push; + return err; err = mlx5_esw_bridge_vlan_push_mark_create(vlan, esw); if (err) goto err_vlan_push_mark; } + if (flags & BRIDGE_VLAN_INFO_UNTAGGED) { err = mlx5_esw_bridge_vlan_pop_create(vlan, esw); if (err) goto err_vlan_pop; } + return 0; + +err_vlan_pop: + if (vlan->pkt_mod_hdr_push_mark) + mlx5_esw_bridge_vlan_push_mark_cleanup(vlan, esw); +err_vlan_push_mark: + if (vlan->pkt_reformat_push) + mlx5_esw_bridge_vlan_push_cleanup(vlan, esw); + return err; +} + +static struct mlx5_esw_bridge_vlan * +mlx5_esw_bridge_vlan_create(u16 vlan_proto, u16 vid, u16 flags, struct mlx5_esw_bridge_port *port, + struct mlx5_eswitch *esw) +{ + struct mlx5_esw_bridge_vlan *vlan; + int err; + + vlan = kvzalloc(sizeof(*vlan), GFP_KERNEL); + if (!vlan) + return ERR_PTR(-ENOMEM); + + vlan->vid = vid; + vlan->flags = flags; + INIT_LIST_HEAD(&vlan->fdb_list); + + err = mlx5_esw_bridge_vlan_push_pop_create(vlan_proto, flags, vlan, esw); + if (err) + goto err_vlan_push_pop; + err = xa_insert(&port->vlans, vid, vlan, GFP_KERNEL); if (err) goto err_xa_insert; @@ -1048,13 +1227,11 @@ mlx5_esw_bridge_vlan_create(u16 vid, u16 flags, struct mlx5_esw_bridge_port *por err_xa_insert: if (vlan->pkt_reformat_pop) mlx5_esw_bridge_vlan_pop_cleanup(vlan, esw); -err_vlan_pop: if (vlan->pkt_mod_hdr_push_mark) mlx5_esw_bridge_vlan_push_mark_cleanup(vlan, esw); -err_vlan_push_mark: if (vlan->pkt_reformat_push) mlx5_esw_bridge_vlan_push_cleanup(vlan, esw); -err_vlan_push: +err_vlan_push_pop: kvfree(vlan); return ERR_PTR(err); } @@ -1102,6 +1279,50 @@ static void mlx5_esw_bridge_port_vlans_flush(struct mlx5_esw_bridge_port *port, mlx5_esw_bridge_vlan_cleanup(port, vlan, bridge); } +static int mlx5_esw_bridge_port_vlans_recreate(struct mlx5_esw_bridge_port *port, + struct mlx5_esw_bridge *bridge) +{ + struct mlx5_esw_bridge_offloads *br_offloads = bridge->br_offloads; + struct mlx5_esw_bridge_vlan *vlan; + unsigned long i; + int err; + + xa_for_each(&port->vlans, i, vlan) { + mlx5_esw_bridge_vlan_flush(vlan, bridge); + err = mlx5_esw_bridge_vlan_push_pop_create(bridge->vlan_proto, vlan->flags, vlan, + br_offloads->esw); + if (err) { + esw_warn(br_offloads->esw->dev, + "Failed to create VLAN=%u(proto=%x) push/pop actions (vport=%u,err=%d)\n", + vlan->vid, bridge->vlan_proto, port->vport_num, + err); + return err; + } + } + + return 0; +} + +static int +mlx5_esw_bridge_vlans_recreate(struct mlx5_esw_bridge *bridge) +{ + struct mlx5_esw_bridge_offloads *br_offloads = bridge->br_offloads; + struct mlx5_esw_bridge_port *port; + unsigned long i; + int err; + + xa_for_each(&br_offloads->ports, i, port) { + if (port->bridge != bridge) + continue; + + err = mlx5_esw_bridge_port_vlans_recreate(port, bridge); + if (err) + return err; + } + + return 0; +} + static struct mlx5_esw_bridge_vlan * mlx5_esw_bridge_port_vlan_lookup(u16 vid, u16 vport_num, u16 esw_owner_vhca_id, struct mlx5_esw_bridge *bridge, struct mlx5_eswitch *esw) @@ -1287,6 +1508,32 @@ int mlx5_esw_bridge_vlan_filtering_set(u16 vport_num, u16 esw_owner_vhca_id, boo return 0; } +int mlx5_esw_bridge_vlan_proto_set(u16 vport_num, u16 esw_owner_vhca_id, u16 proto, + struct mlx5_esw_bridge_offloads *br_offloads) +{ + struct mlx5_esw_bridge_port *port; + struct mlx5_esw_bridge *bridge; + + port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, + br_offloads); + if (!port) + return -EINVAL; + + bridge = port->bridge; + if (bridge->vlan_proto == proto) + return 0; + if (proto != ETH_P_8021Q && proto != ETH_P_8021AD) { + esw_warn(br_offloads->esw->dev, "Can't set unsupported VLAN protocol %x", proto); + return -EOPNOTSUPP; + } + + mlx5_esw_bridge_fdb_flush(bridge); + bridge->vlan_proto = proto; + mlx5_esw_bridge_vlans_recreate(bridge); + + return 0; +} + static int mlx5_esw_bridge_vport_init(u16 vport_num, u16 esw_owner_vhca_id, u16 flags, struct mlx5_esw_bridge_offloads *br_offloads, struct mlx5_esw_bridge *bridge) @@ -1434,7 +1681,8 @@ int mlx5_esw_bridge_port_vlan_add(u16 vport_num, u16 esw_owner_vhca_id, u16 vid, mlx5_esw_bridge_vlan_cleanup(port, vlan, port->bridge); } - vlan = mlx5_esw_bridge_vlan_create(vid, flags, port, br_offloads->esw); + vlan = mlx5_esw_bridge_vlan_create(port->bridge->vlan_proto, vid, flags, port, + br_offloads->esw); if (IS_ERR(vlan)) { NL_SET_ERR_MSG_MOD(extack, "Failed to create VLAN entry"); return PTR_ERR(vlan); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h index efc39975226e..10851a515bca 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h @@ -26,7 +26,9 @@ struct mlx5_esw_bridge_offloads { struct mlx5_flow_table *ingress_ft; struct mlx5_flow_group *ingress_vlan_fg; - struct mlx5_flow_group *ingress_filter_fg; + struct mlx5_flow_group *ingress_vlan_filter_fg; + struct mlx5_flow_group *ingress_qinq_fg; + struct mlx5_flow_group *ingress_qinq_filter_fg; struct mlx5_flow_group *ingress_mac_fg; struct mlx5_flow_table *skip_ft; @@ -60,6 +62,8 @@ int mlx5_esw_bridge_ageing_time_set(u16 vport_num, u16 esw_owner_vhca_id, unsign struct mlx5_esw_bridge_offloads *br_offloads); int mlx5_esw_bridge_vlan_filtering_set(u16 vport_num, u16 esw_owner_vhca_id, bool enable, struct mlx5_esw_bridge_offloads *br_offloads); +int mlx5_esw_bridge_vlan_proto_set(u16 vport_num, u16 esw_owner_vhca_id, u16 proto, + struct mlx5_esw_bridge_offloads *br_offloads); int mlx5_esw_bridge_port_vlan_add(u16 vport_num, u16 esw_owner_vhca_id, u16 vid, u16 flags, struct mlx5_esw_bridge_offloads *br_offloads, struct netlink_ext_ack *extack); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/debugfs.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/debugfs.c new file mode 100644 index 000000000000..2db13c71e88c --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/debugfs.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#include <linux/debugfs.h> +#include "eswitch.h" + +enum vnic_diag_counter { + MLX5_VNIC_DIAG_TOTAL_Q_UNDER_PROCESSOR_HANDLE, + MLX5_VNIC_DIAG_SEND_QUEUE_PRIORITY_UPDATE_FLOW, + MLX5_VNIC_DIAG_COMP_EQ_OVERRUN, + MLX5_VNIC_DIAG_ASYNC_EQ_OVERRUN, + MLX5_VNIC_DIAG_CQ_OVERRUN, + MLX5_VNIC_DIAG_INVALID_COMMAND, + MLX5_VNIC_DIAG_QOUTA_EXCEEDED_COMMAND, +}; + +static int mlx5_esw_query_vnic_diag(struct mlx5_vport *vport, enum vnic_diag_counter counter, + u32 *val) +{ + u32 out[MLX5_ST_SZ_DW(query_vnic_env_out)] = {}; + u32 in[MLX5_ST_SZ_DW(query_vnic_env_in)] = {}; + struct mlx5_core_dev *dev = vport->dev; + u16 vport_num = vport->vport; + void *vnic_diag_out; + int err; + + MLX5_SET(query_vnic_env_in, in, opcode, MLX5_CMD_OP_QUERY_VNIC_ENV); + MLX5_SET(query_vnic_env_in, in, vport_number, vport_num); + if (!mlx5_esw_is_manager_vport(dev->priv.eswitch, vport_num)) + MLX5_SET(query_vnic_env_in, in, other_vport, 1); + + err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + if (err) + return err; + + vnic_diag_out = MLX5_ADDR_OF(query_vnic_env_out, out, vport_env); + switch (counter) { + case MLX5_VNIC_DIAG_TOTAL_Q_UNDER_PROCESSOR_HANDLE: + *val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out, total_error_queues); + break; + case MLX5_VNIC_DIAG_SEND_QUEUE_PRIORITY_UPDATE_FLOW: + *val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out, + send_queue_priority_update_flow); + break; + case MLX5_VNIC_DIAG_COMP_EQ_OVERRUN: + *val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out, comp_eq_overrun); + break; + case MLX5_VNIC_DIAG_ASYNC_EQ_OVERRUN: + *val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out, async_eq_overrun); + break; + case MLX5_VNIC_DIAG_CQ_OVERRUN: + *val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out, cq_overrun); + break; + case MLX5_VNIC_DIAG_INVALID_COMMAND: + *val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out, invalid_command); + break; + case MLX5_VNIC_DIAG_QOUTA_EXCEEDED_COMMAND: + *val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out, quota_exceeded_command); + break; + } + + return 0; +} + +static int __show_vnic_diag(struct seq_file *file, struct mlx5_vport *vport, + enum vnic_diag_counter type) +{ + u32 val = 0; + int ret; + + ret = mlx5_esw_query_vnic_diag(vport, type, &val); + if (ret) + return ret; + + seq_printf(file, "%d\n", val); + return 0; +} + +static int total_q_under_processor_handle_show(struct seq_file *file, void *priv) +{ + return __show_vnic_diag(file, file->private, MLX5_VNIC_DIAG_TOTAL_Q_UNDER_PROCESSOR_HANDLE); +} + +static int send_queue_priority_update_flow_show(struct seq_file *file, void *priv) +{ + return __show_vnic_diag(file, file->private, + MLX5_VNIC_DIAG_SEND_QUEUE_PRIORITY_UPDATE_FLOW); +} + +static int comp_eq_overrun_show(struct seq_file *file, void *priv) +{ + return __show_vnic_diag(file, file->private, MLX5_VNIC_DIAG_COMP_EQ_OVERRUN); +} + +static int async_eq_overrun_show(struct seq_file *file, void *priv) +{ + return __show_vnic_diag(file, file->private, MLX5_VNIC_DIAG_ASYNC_EQ_OVERRUN); +} + +static int cq_overrun_show(struct seq_file *file, void *priv) +{ + return __show_vnic_diag(file, file->private, MLX5_VNIC_DIAG_CQ_OVERRUN); +} + +static int invalid_command_show(struct seq_file *file, void *priv) +{ + return __show_vnic_diag(file, file->private, MLX5_VNIC_DIAG_INVALID_COMMAND); +} + +static int quota_exceeded_command_show(struct seq_file *file, void *priv) +{ + return __show_vnic_diag(file, file->private, MLX5_VNIC_DIAG_QOUTA_EXCEEDED_COMMAND); +} + +DEFINE_SHOW_ATTRIBUTE(total_q_under_processor_handle); +DEFINE_SHOW_ATTRIBUTE(send_queue_priority_update_flow); +DEFINE_SHOW_ATTRIBUTE(comp_eq_overrun); +DEFINE_SHOW_ATTRIBUTE(async_eq_overrun); +DEFINE_SHOW_ATTRIBUTE(cq_overrun); +DEFINE_SHOW_ATTRIBUTE(invalid_command); +DEFINE_SHOW_ATTRIBUTE(quota_exceeded_command); + +void mlx5_esw_vport_debugfs_destroy(struct mlx5_eswitch *esw, u16 vport_num) +{ + struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); + + debugfs_remove_recursive(vport->dbgfs); + vport->dbgfs = NULL; +} + +/* vnic diag dir name is "pf", "ecpf" or "{vf/sf}_xxxx" */ +#define VNIC_DIAG_DIR_NAME_MAX_LEN 8 + +void mlx5_esw_vport_debugfs_create(struct mlx5_eswitch *esw, u16 vport_num, bool is_sf, u16 sf_num) +{ + struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); + struct dentry *vnic_diag; + char dir_name[VNIC_DIAG_DIR_NAME_MAX_LEN]; + int err; + + if (!MLX5_CAP_GEN(esw->dev, vport_group_manager)) + return; + + if (vport_num == MLX5_VPORT_PF) { + strcpy(dir_name, "pf"); + } else if (vport_num == MLX5_VPORT_ECPF) { + strcpy(dir_name, "ecpf"); + } else { + err = snprintf(dir_name, VNIC_DIAG_DIR_NAME_MAX_LEN, "%s_%d", is_sf ? "sf" : "vf", + is_sf ? sf_num : vport_num - MLX5_VPORT_FIRST_VF); + if (WARN_ON(err < 0)) + return; + } + + vport->dbgfs = debugfs_create_dir(dir_name, esw->dbgfs); + vnic_diag = debugfs_create_dir("vnic_diag", vport->dbgfs); + + if (MLX5_CAP_GEN(esw->dev, vnic_env_queue_counters)) { + debugfs_create_file("total_q_under_processor_handle", 0444, vnic_diag, vport, + &total_q_under_processor_handle_fops); + debugfs_create_file("send_queue_priority_update_flow", 0444, vnic_diag, vport, + &send_queue_priority_update_flow_fops); + } + + if (MLX5_CAP_GEN(esw->dev, eq_overrun_count)) { + debugfs_create_file("comp_eq_overrun", 0444, vnic_diag, vport, + &comp_eq_overrun_fops); + debugfs_create_file("async_eq_overrun", 0444, vnic_diag, vport, + &async_eq_overrun_fops); + } + + if (MLX5_CAP_GEN(esw->dev, vnic_env_cq_overrun)) + debugfs_create_file("cq_overrun", 0444, vnic_diag, vport, &cq_overrun_fops); + + if (MLX5_CAP_GEN(esw->dev, invalid_command_count)) + debugfs_create_file("invalid_command", 0444, vnic_diag, vport, + &invalid_command_fops); + + if (MLX5_CAP_GEN(esw->dev, quota_exceeded_count)) + debugfs_create_file("quota_exceeded_command", 0444, vnic_diag, vport, + "a_exceeded_command_fops); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c index 7f9b96d9537e..9bc7be95db54 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c @@ -87,11 +87,11 @@ int mlx5_esw_offloads_devlink_port_register(struct mlx5_eswitch *esw, u16 vport_ devlink = priv_to_devlink(dev); dl_port_index = mlx5_esw_vport_to_devlink_port_index(dev, vport_num); - err = devlink_port_register(devlink, dl_port, dl_port_index); + err = devl_port_register(devlink, dl_port, dl_port_index); if (err) goto reg_err; - err = devlink_rate_leaf_create(dl_port, vport); + err = devl_rate_leaf_create(dl_port, vport); if (err) goto rate_err; @@ -99,7 +99,7 @@ int mlx5_esw_offloads_devlink_port_register(struct mlx5_eswitch *esw, u16 vport_ return 0; rate_err: - devlink_port_unregister(dl_port); + devl_port_unregister(dl_port); reg_err: mlx5_esw_dl_port_free(dl_port); return err; @@ -118,10 +118,10 @@ void mlx5_esw_offloads_devlink_port_unregister(struct mlx5_eswitch *esw, u16 vpo if (vport->dl_port->devlink_rate) { mlx5_esw_qos_vport_update_group(esw, vport, NULL, NULL); - devlink_rate_leaf_destroy(vport->dl_port); + devl_rate_leaf_destroy(vport->dl_port); } - devlink_port_unregister(vport->dl_port); + devl_port_unregister(vport->dl_port); mlx5_esw_dl_port_free(vport->dl_port); vport->dl_port = NULL; } @@ -156,11 +156,11 @@ int mlx5_esw_devlink_sf_port_register(struct mlx5_eswitch *esw, struct devlink_p devlink_port_attrs_pci_sf_set(dl_port, controller, pfnum, sfnum, !!controller); devlink = priv_to_devlink(dev); dl_port_index = mlx5_esw_vport_to_devlink_port_index(dev, vport_num); - err = devlink_port_register(devlink, dl_port, dl_port_index); + err = devl_port_register(devlink, dl_port, dl_port_index); if (err) return err; - err = devlink_rate_leaf_create(dl_port, vport); + err = devl_rate_leaf_create(dl_port, vport); if (err) goto rate_err; @@ -168,7 +168,7 @@ int mlx5_esw_devlink_sf_port_register(struct mlx5_eswitch *esw, struct devlink_p return 0; rate_err: - devlink_port_unregister(dl_port); + devl_port_unregister(dl_port); return err; } @@ -182,9 +182,9 @@ void mlx5_esw_devlink_sf_port_unregister(struct mlx5_eswitch *esw, u16 vport_num if (vport->dl_port->devlink_rate) { mlx5_esw_qos_vport_update_group(esw, vport, NULL, NULL); - devlink_rate_leaf_destroy(vport->dl_port); + devl_rate_leaf_destroy(vport->dl_port); } - devlink_port_unregister(vport->dl_port); + devl_port_unregister(vport->dl_port); vport->dl_port = NULL; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index 719ef26d23c0..30a6c9fbf1b6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -36,6 +36,7 @@ #include <linux/mlx5/vport.h> #include <linux/mlx5/fs.h> #include <linux/mlx5/mpfs.h> +#include <linux/debugfs.h> #include "esw/acl/lgcy.h" #include "esw/legacy.h" #include "esw/qos.h" @@ -1002,6 +1003,7 @@ int mlx5_eswitch_load_vport(struct mlx5_eswitch *esw, u16 vport_num, if (err) return err; + mlx5_esw_vport_debugfs_create(esw, vport_num, false, 0); err = esw_offloads_load_rep(esw, vport_num); if (err) goto err_rep; @@ -1009,6 +1011,7 @@ int mlx5_eswitch_load_vport(struct mlx5_eswitch *esw, u16 vport_num, return err; err_rep: + mlx5_esw_vport_debugfs_destroy(esw, vport_num); mlx5_esw_vport_disable(esw, vport_num); return err; } @@ -1016,6 +1019,7 @@ err_rep: void mlx5_eswitch_unload_vport(struct mlx5_eswitch *esw, u16 vport_num) { esw_offloads_unload_rep(esw, vport_num); + mlx5_esw_vport_debugfs_destroy(esw, vport_num); mlx5_esw_vport_disable(esw, vport_num); } @@ -1152,8 +1156,6 @@ mlx5_eswitch_update_num_of_vfs(struct mlx5_eswitch *esw, int num_vfs) { const u32 *out; - WARN_ON_ONCE(esw->mode != MLX5_ESWITCH_NONE); - if (num_vfs < 0) return; @@ -1186,6 +1188,9 @@ static int mlx5_esw_acls_ns_init(struct mlx5_eswitch *esw) int total_vports; int err; + if (esw->flags & MLX5_ESWITCH_VPORT_ACL_NS_CREATED) + return 0; + total_vports = mlx5_eswitch_get_total_vports(dev); if (MLX5_CAP_ESW_EGRESS_ACL(dev, ft_support)) { @@ -1203,6 +1208,7 @@ static int mlx5_esw_acls_ns_init(struct mlx5_eswitch *esw) } else { esw_warn(dev, "ingress ACL is not supported by FW\n"); } + esw->flags |= MLX5_ESWITCH_VPORT_ACL_NS_CREATED; return 0; err: @@ -1215,6 +1221,7 @@ static void mlx5_esw_acls_ns_cleanup(struct mlx5_eswitch *esw) { struct mlx5_core_dev *dev = esw->dev; + esw->flags &= ~MLX5_ESWITCH_VPORT_ACL_NS_CREATED; if (MLX5_CAP_ESW_INGRESS_ACL(dev, ft_support)) mlx5_fs_ingress_acls_cleanup(dev); if (MLX5_CAP_ESW_EGRESS_ACL(dev, ft_support)) @@ -1224,7 +1231,6 @@ static void mlx5_esw_acls_ns_cleanup(struct mlx5_eswitch *esw) /** * mlx5_eswitch_enable_locked - Enable eswitch * @esw: Pointer to eswitch - * @mode: Eswitch mode to enable * @num_vfs: Enable eswitch for given number of VFs. This is optional. * Valid value are 0, > 0 and MLX5_ESWITCH_IGNORE_NUM_VFS. * Caller should pass num_vfs > 0 when enabling eswitch for @@ -1238,7 +1244,7 @@ static void mlx5_esw_acls_ns_cleanup(struct mlx5_eswitch *esw) * mode. If num_vfs >=0 is provided, it setup VF related eswitch vports. * It returns 0 on success or error code on failure. */ -int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int mode, int num_vfs) +int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int num_vfs) { int err; @@ -1257,9 +1263,7 @@ int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int mode, int num_vfs) mlx5_eswitch_update_num_of_vfs(esw, num_vfs); - esw->mode = mode; - - if (mode == MLX5_ESWITCH_LEGACY) { + if (esw->mode == MLX5_ESWITCH_LEGACY) { err = esw_legacy_enable(esw); } else { mlx5_rescan_drivers(esw->dev); @@ -1269,22 +1273,19 @@ int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int mode, int num_vfs) if (err) goto abort; + esw->fdb_table.flags |= MLX5_ESW_FDB_CREATED; + mlx5_eswitch_event_handlers_register(esw); esw_info(esw->dev, "Enable: mode(%s), nvfs(%d), active vports(%d)\n", - mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS", + esw->mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS", esw->esw_funcs.num_vfs, esw->enabled_vports); - mlx5_esw_mode_change_notify(esw, mode); + mlx5_esw_mode_change_notify(esw, esw->mode); return 0; abort: - esw->mode = MLX5_ESWITCH_NONE; - - if (mode == MLX5_ESWITCH_OFFLOADS) - mlx5_rescan_drivers(esw->dev); - mlx5_esw_acls_ns_cleanup(esw); return err; } @@ -1299,20 +1300,23 @@ abort: */ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs) { + struct devlink *devlink; bool toggle_lag; int ret; if (!mlx5_esw_allowed(esw)) return 0; - toggle_lag = esw->mode == MLX5_ESWITCH_NONE; + toggle_lag = !mlx5_esw_is_fdb_created(esw); if (toggle_lag) mlx5_lag_disable_change(esw->dev); + devlink = priv_to_devlink(esw->dev); + devl_lock(devlink); down_write(&esw->mode_lock); - if (esw->mode == MLX5_ESWITCH_NONE) { - ret = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY, num_vfs); + if (!mlx5_esw_is_fdb_created(esw)) { + ret = mlx5_eswitch_enable_locked(esw, num_vfs); } else { enum mlx5_eswitch_vport_event vport_events; @@ -1323,6 +1327,7 @@ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs) esw->esw_funcs.num_vfs = num_vfs; } up_write(&esw->mode_lock); + devl_unlock(devlink); if (toggle_lag) mlx5_lag_enable_change(esw->dev); @@ -1330,56 +1335,90 @@ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs) return ret; } -void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw, bool clear_vf) +/* When disabling sriov, free driver level resources. */ +void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf) { - struct devlink *devlink = priv_to_devlink(esw->dev); - int old_mode; + struct devlink *devlink; - lockdep_assert_held_write(&esw->mode_lock); - - if (esw->mode == MLX5_ESWITCH_NONE) + if (!mlx5_esw_allowed(esw)) return; - esw_info(esw->dev, "Disable: mode(%s), nvfs(%d), active vports(%d)\n", + devlink = priv_to_devlink(esw->dev); + devl_lock(devlink); + down_write(&esw->mode_lock); + /* If driver is unloaded, this function is called twice by remove_one() + * and mlx5_unload(). Prevent the second call. + */ + if (!esw->esw_funcs.num_vfs && !clear_vf) + goto unlock; + + esw_info(esw->dev, "Unload vfs: mode(%s), nvfs(%d), active vports(%d)\n", esw->mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS", esw->esw_funcs.num_vfs, esw->enabled_vports); - /* Notify eswitch users that it is exiting from current mode. - * So that it can do necessary cleanup before the eswitch is disabled. + mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); + if (clear_vf) + mlx5_eswitch_clear_vf_vports_info(esw); + /* If disabling sriov in switchdev mode, free meta rules here + * because it depends on num_vfs. */ - mlx5_esw_mode_change_notify(esw, MLX5_ESWITCH_NONE); + if (esw->mode == MLX5_ESWITCH_OFFLOADS) { + struct devlink *devlink = priv_to_devlink(esw->dev); - mlx5_eswitch_event_handlers_unregister(esw); + esw_offloads_del_send_to_vport_meta_rules(esw); + devl_rate_nodes_destroy(devlink); + } - if (esw->mode == MLX5_ESWITCH_LEGACY) - esw_legacy_disable(esw); - else if (esw->mode == MLX5_ESWITCH_OFFLOADS) - esw_offloads_disable(esw); + esw->esw_funcs.num_vfs = 0; - old_mode = esw->mode; - esw->mode = MLX5_ESWITCH_NONE; +unlock: + up_write(&esw->mode_lock); + devl_unlock(devlink); +} - if (old_mode == MLX5_ESWITCH_OFFLOADS) - mlx5_rescan_drivers(esw->dev); +/* Free resources for corresponding eswitch mode. It is called by devlink + * when changing eswitch mode or modprobe when unloading driver. + */ +void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw) +{ + struct devlink *devlink = priv_to_devlink(esw->dev); + + /* Notify eswitch users that it is exiting from current mode. + * So that it can do necessary cleanup before the eswitch is disabled. + */ + mlx5_esw_mode_change_notify(esw, MLX5_ESWITCH_LEGACY); - devlink_rate_nodes_destroy(devlink); + mlx5_eswitch_event_handlers_unregister(esw); + esw_info(esw->dev, "Disable: mode(%s), nvfs(%d), active vports(%d)\n", + esw->mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS", + esw->esw_funcs.num_vfs, esw->enabled_vports); + + esw->fdb_table.flags &= ~MLX5_ESW_FDB_CREATED; + if (esw->mode == MLX5_ESWITCH_OFFLOADS) + esw_offloads_disable(esw); + else if (esw->mode == MLX5_ESWITCH_LEGACY) + esw_legacy_disable(esw); mlx5_esw_acls_ns_cleanup(esw); - if (clear_vf) - mlx5_eswitch_clear_vf_vports_info(esw); + if (esw->mode == MLX5_ESWITCH_OFFLOADS) + devl_rate_nodes_destroy(devlink); } -void mlx5_eswitch_disable(struct mlx5_eswitch *esw, bool clear_vf) +void mlx5_eswitch_disable(struct mlx5_eswitch *esw) { + struct devlink *devlink; + if (!mlx5_esw_allowed(esw)) return; mlx5_lag_disable_change(esw->dev); + devlink = priv_to_devlink(esw->dev); + devl_lock(devlink); down_write(&esw->mode_lock); - mlx5_eswitch_disable_locked(esw, clear_vf); - esw->esw_funcs.num_vfs = 0; + mlx5_eswitch_disable_locked(esw); up_write(&esw->mode_lock); + devl_unlock(devlink); mlx5_lag_enable_change(esw->dev); } @@ -1573,7 +1612,7 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev) refcount_set(&esw->qos.refcnt, 0); esw->enabled_vports = 0; - esw->mode = MLX5_ESWITCH_NONE; + esw->mode = MLX5_ESWITCH_LEGACY; esw->offloads.inline_mode = MLX5_INLINE_MODE_NONE; if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) && MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap)) @@ -1587,6 +1626,7 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev) dev->priv.eswitch = esw; BLOCKING_INIT_NOTIFIER_HEAD(&esw->n_head); + esw->dbgfs = debugfs_create_dir("esw", mlx5_debugfs_get_dev_root(esw->dev)); esw_info(dev, "Total vports %d, per vport: max uc(%d) max mc(%d)\n", esw->total_vports, @@ -1610,6 +1650,7 @@ void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw) esw_info(esw->dev, "cleanup\n"); + debugfs_remove_recursive(esw->dbgfs); esw->dev->priv.eswitch = NULL; destroy_workqueue(esw->work_queue); WARN_ON(refcount_read(&esw->qos.refcnt)); @@ -1875,7 +1916,7 @@ u8 mlx5_eswitch_mode(const struct mlx5_core_dev *dev) { struct mlx5_eswitch *esw = dev->priv.eswitch; - return mlx5_esw_allowed(esw) ? esw->mode : MLX5_ESWITCH_NONE; + return mlx5_esw_allowed(esw) ? esw->mode : MLX5_ESWITCH_LEGACY; } EXPORT_SYMBOL_GPL(mlx5_eswitch_mode); @@ -1995,8 +2036,6 @@ int mlx5_esw_try_lock(struct mlx5_eswitch *esw) */ void mlx5_esw_unlock(struct mlx5_eswitch *esw) { - if (!mlx5_esw_allowed(esw)) - return; up_write(&esw->mode_lock); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index 2754a732914d..87ce5a208cb5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -191,6 +191,7 @@ struct mlx5_vport { enum mlx5_eswitch_vport_event enabled_events; int index; struct devlink_port *dl_port; + struct dentry *dbgfs; }; struct mlx5_esw_indir_table; @@ -282,10 +283,15 @@ struct mlx5_esw_functions { enum { MLX5_ESWITCH_VPORT_MATCH_METADATA = BIT(0), MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED = BIT(1), + MLX5_ESWITCH_VPORT_ACL_NS_CREATED = BIT(2), }; struct mlx5_esw_bridge_offloads; +enum { + MLX5_ESW_FDB_CREATED = BIT(0), +}; + struct mlx5_eswitch { struct mlx5_core_dev *dev; struct mlx5_nb nb; @@ -331,12 +337,14 @@ struct mlx5_eswitch { u32 large_group_num; } params; struct blocking_notifier_head n_head; + struct dentry *dbgfs; }; void esw_offloads_disable(struct mlx5_eswitch *esw); int esw_offloads_enable(struct mlx5_eswitch *esw); void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw); int esw_offloads_init_reps(struct mlx5_eswitch *esw); +void esw_offloads_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw); bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw); int mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch *esw, bool enable); @@ -350,10 +358,11 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev); void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw); #define MLX5_ESWITCH_IGNORE_NUM_VFS (-1) -int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int mode, int num_vfs); +int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int num_vfs); int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs); -void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw, bool clear_vf); -void mlx5_eswitch_disable(struct mlx5_eswitch *esw, bool clear_vf); +void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf); +void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw); +void mlx5_eswitch_disable(struct mlx5_eswitch *esw); int mlx5_eswitch_set_vport_mac(struct mlx5_eswitch *esw, u16 vport, const u8 *mac); int mlx5_eswitch_set_vport_state(struct mlx5_eswitch *esw, @@ -575,6 +584,11 @@ mlx5_esw_devlink_port_index_to_vport_num(unsigned int dl_port_index) return dl_port_index & 0xffff; } +static inline bool mlx5_esw_is_fdb_created(struct mlx5_eswitch *esw) +{ + return esw->fdb_table.flags & MLX5_ESW_FDB_CREATED; +} + /* TODO: This mlx5e_tc function shouldn't be called by eswitch */ void mlx5e_tc_clean_fdb_peer_flows(struct mlx5_eswitch *esw); @@ -672,6 +686,9 @@ int mlx5_esw_offloads_devlink_port_register(struct mlx5_eswitch *esw, u16 vport_ void mlx5_esw_offloads_devlink_port_unregister(struct mlx5_eswitch *esw, u16 vport_num); struct devlink_port *mlx5_esw_offloads_devlink_port(struct mlx5_eswitch *esw, u16 vport_num); +void mlx5_esw_vport_debugfs_create(struct mlx5_eswitch *esw, u16 vport_num, bool is_sf, u16 sf_num); +void mlx5_esw_vport_debugfs_destroy(struct mlx5_eswitch *esw, u16 vport_num); + int mlx5_esw_devlink_sf_port_register(struct mlx5_eswitch *esw, struct devlink_port *dl_port, u16 vport_num, u32 controller, u32 sfnum); void mlx5_esw_devlink_sf_port_unregister(struct mlx5_eswitch *esw, u16 vport_num); @@ -719,7 +736,8 @@ int mlx5_eswitch_reload_reps(struct mlx5_eswitch *esw); static inline int mlx5_eswitch_init(struct mlx5_core_dev *dev) { return 0; } static inline void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw) {} static inline int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs) { return 0; } -static inline void mlx5_eswitch_disable(struct mlx5_eswitch *esw, bool clear_vf) {} +static inline void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf) {} +static inline void mlx5_eswitch_disable(struct mlx5_eswitch *esw) {} static inline bool mlx5_eswitch_is_funcs_handler(struct mlx5_core_dev *dev) { return false; } static inline int mlx5_eswitch_set_vport_state(struct mlx5_eswitch *esw, u16 vport, int link_state) { return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 2ce3728576d1..da1959caae41 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -512,6 +512,20 @@ esw_cleanup_dests(struct mlx5_eswitch *esw, } } +static void +esw_setup_meter(struct mlx5_flow_attr *attr, struct mlx5_flow_act *flow_act) +{ + struct mlx5e_flow_meter_handle *meter; + + meter = attr->meter_attr.meter; + flow_act->exe_aso.type = attr->exe_aso_type; + flow_act->exe_aso.object_id = meter->obj_id; + flow_act->exe_aso.flow_meter.meter_idx = meter->idx; + flow_act->exe_aso.flow_meter.init_color = MLX5_FLOW_METER_COLOR_GREEN; + /* use metadata reg 5 for packet color */ + flow_act->exe_aso.return_reg_id = 5; +} + struct mlx5_flow_handle * mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec, @@ -579,6 +593,10 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) flow_act.modify_hdr = attr->modify_hdr; + if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) && + attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER) + esw_setup_meter(attr, &flow_act); + if (split) { fwd_attr.chain = attr->chain; fwd_attr.prio = attr->prio; @@ -1040,6 +1058,15 @@ static void mlx5_eswitch_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw) mlx5_del_flow_rules(flows[i]); kvfree(flows); + /* If changing eswitch mode from switchdev to legacy, but num_vfs is not 0, + * meta rules could be freed again. So set it to NULL. + */ + esw->fdb_table.offloads.send_to_vport_meta_rules = NULL; +} + +void esw_offloads_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw) +{ + mlx5_eswitch_del_send_to_vport_meta_rules(esw); } static int @@ -2034,7 +2061,7 @@ static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode) if (!MLX5_CAP_GEN(dev, vport_group_manager)) return -EOPNOTSUPP; - if (esw->mode == MLX5_ESWITCH_NONE) + if (!mlx5_esw_is_fdb_created(esw)) return -EOPNOTSUPP; switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { @@ -2170,18 +2197,18 @@ static int esw_offloads_start(struct mlx5_eswitch *esw, { int err, err1; - mlx5_eswitch_disable_locked(esw, false); - err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_OFFLOADS, - esw->dev->priv.sriov.num_vfs); + esw->mode = MLX5_ESWITCH_OFFLOADS; + err = mlx5_eswitch_enable_locked(esw, esw->dev->priv.sriov.num_vfs); if (err) { NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to offloads"); - err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY, - MLX5_ESWITCH_IGNORE_NUM_VFS); + esw->mode = MLX5_ESWITCH_LEGACY; + err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS); if (err1) { NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch back to legacy"); } + mlx5_rescan_drivers(esw->dev); } if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) { if (mlx5_eswitch_inline_mode_get(esw, @@ -2894,7 +2921,7 @@ int mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch *esw, bool enable) int err = 0; down_write(&esw->mode_lock); - if (esw->mode != MLX5_ESWITCH_NONE) { + if (mlx5_esw_is_fdb_created(esw)) { err = -EBUSY; goto done; } @@ -3055,6 +3082,7 @@ static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw) static void esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out) { + struct devlink *devlink; bool host_pf_disabled; u16 new_num_vfs; @@ -3066,6 +3094,8 @@ esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out) if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled) return; + devlink = priv_to_devlink(esw->dev); + devl_lock(devlink); /* Number of VFs can only change from "0 to x" or "x to 0". */ if (esw->esw_funcs.num_vfs > 0) { mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); @@ -3078,6 +3108,7 @@ esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out) return; } esw->esw_funcs.num_vfs = new_num_vfs; + devl_unlock(devlink); } static void esw_functions_changed_event_handler(struct work_struct *work) @@ -3229,13 +3260,12 @@ static int esw_offloads_stop(struct mlx5_eswitch *esw, { int err, err1; - mlx5_eswitch_disable_locked(esw, false); - err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY, - MLX5_ESWITCH_IGNORE_NUM_VFS); + esw->mode = MLX5_ESWITCH_LEGACY; + err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS); if (err) { NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy"); - err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_OFFLOADS, - MLX5_ESWITCH_IGNORE_NUM_VFS); + esw->mode = MLX5_ESWITCH_OFFLOADS; + err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS); if (err1) { NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch back to offloads"); @@ -3334,36 +3364,6 @@ static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode) return 0; } -static int eswitch_devlink_esw_mode_check(const struct mlx5_eswitch *esw) -{ - /* devlink commands in NONE eswitch mode are currently supported only - * on ECPF. - */ - return (esw->mode == MLX5_ESWITCH_NONE && - !mlx5_core_is_ecpf_esw_manager(esw->dev)) ? -EOPNOTSUPP : 0; -} - -/* FIXME: devl_unlock() followed by devl_lock() inside driver callback - * is never correct and prone to races. It's a transitional workaround, - * never repeat this pattern. - * - * This code MUST be fixed before removing devlink_mutex as it is safe - * to do only because of that mutex. - */ -static void mlx5_eswtich_mode_callback_enter(struct devlink *devlink, - struct mlx5_eswitch *esw) -{ - devl_unlock(devlink); - down_write(&esw->mode_lock); -} - -static void mlx5_eswtich_mode_callback_exit(struct devlink *devlink, - struct mlx5_eswitch *esw) -{ - up_write(&esw->mode_lock); - devl_lock(devlink); -} - int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, struct netlink_ext_ack *extack) { @@ -3378,15 +3378,6 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, if (esw_mode_from_devlink(mode, &mlx5_mode)) return -EINVAL; - /* FIXME: devl_unlock() followed by devl_lock() inside driver callback - * is never correct and prone to races. It's a transitional workaround, - * never repeat this pattern. - * - * This code MUST be fixed before removing devlink_mutex as it is safe - * to do only because of that mutex. - */ - devl_unlock(devlink); - mlx5_lag_disable_change(esw->dev); err = mlx5_esw_try_lock(esw); if (err < 0) { @@ -3399,6 +3390,7 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, if (cur_mlx5_mode == mlx5_mode) goto unlock; + mlx5_eswitch_disable_locked(esw); if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) { if (mlx5_devlink_trap_get_num_active(esw->dev)) { NL_SET_ERR_MSG_MOD(extack, @@ -3409,6 +3401,7 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, err = esw_offloads_start(esw, extack); } else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) { err = esw_offloads_stop(esw, extack); + mlx5_rescan_drivers(esw->dev); } else { err = -EINVAL; } @@ -3417,7 +3410,6 @@ unlock: mlx5_esw_unlock(esw); enable_lag: mlx5_lag_enable_change(esw->dev); - devl_lock(devlink); return err; } @@ -3430,14 +3422,9 @@ int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode) if (IS_ERR(esw)) return PTR_ERR(esw); - mlx5_eswtich_mode_callback_enter(devlink, esw); - err = eswitch_devlink_esw_mode_check(esw); - if (err) - goto unlock; - + down_write(&esw->mode_lock); err = esw_mode_to_devlink(esw->mode, mode); -unlock: - mlx5_eswtich_mode_callback_exit(devlink, esw); + up_write(&esw->mode_lock); return err; } @@ -3484,10 +3471,7 @@ int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode, if (IS_ERR(esw)) return PTR_ERR(esw); - mlx5_eswtich_mode_callback_enter(devlink, esw); - err = eswitch_devlink_esw_mode_check(esw); - if (err) - goto out; + down_write(&esw->mode_lock); switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: @@ -3521,11 +3505,11 @@ int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode, goto out; esw->offloads.inline_mode = mlx5_mode; - mlx5_eswtich_mode_callback_exit(devlink, esw); + up_write(&esw->mode_lock); return 0; out: - mlx5_eswtich_mode_callback_exit(devlink, esw); + up_write(&esw->mode_lock); return err; } @@ -3538,14 +3522,9 @@ int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode) if (IS_ERR(esw)) return PTR_ERR(esw); - mlx5_eswtich_mode_callback_enter(devlink, esw); - err = eswitch_devlink_esw_mode_check(esw); - if (err) - goto unlock; - + down_write(&esw->mode_lock); err = esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode); -unlock: - mlx5_eswtich_mode_callback_exit(devlink, esw); + up_write(&esw->mode_lock); return err; } @@ -3555,16 +3534,13 @@ int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, { struct mlx5_core_dev *dev = devlink_priv(devlink); struct mlx5_eswitch *esw; - int err; + int err = 0; esw = mlx5_devlink_eswitch_get(devlink); if (IS_ERR(esw)) return PTR_ERR(esw); - mlx5_eswtich_mode_callback_enter(devlink, esw); - err = eswitch_devlink_esw_mode_check(esw); - if (err) - goto unlock; + down_write(&esw->mode_lock); if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE && (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) || @@ -3607,7 +3583,7 @@ int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, } unlock: - mlx5_eswtich_mode_callback_exit(devlink, esw); + up_write(&esw->mode_lock); return err; } @@ -3615,21 +3591,15 @@ int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, enum devlink_eswitch_encap_mode *encap) { struct mlx5_eswitch *esw; - int err; esw = mlx5_devlink_eswitch_get(devlink); if (IS_ERR(esw)) return PTR_ERR(esw); - mlx5_eswtich_mode_callback_enter(devlink, esw); - err = eswitch_devlink_esw_mode_check(esw); - if (err) - goto unlock; - + down_write(&esw->mode_lock); *encap = esw->offloads.encap; -unlock: - mlx5_eswtich_mode_callback_exit(devlink, esw); - return err; + up_write(&esw->mode_lock); + return 0; } static bool @@ -3752,12 +3722,14 @@ int mlx5_esw_offloads_sf_vport_enable(struct mlx5_eswitch *esw, struct devlink_p if (err) goto devlink_err; + mlx5_esw_vport_debugfs_create(esw, vport_num, true, sfnum); err = mlx5_esw_offloads_rep_load(esw, vport_num); if (err) goto rep_err; return 0; rep_err: + mlx5_esw_vport_debugfs_destroy(esw, vport_num); mlx5_esw_devlink_sf_port_unregister(esw, vport_num); devlink_err: mlx5_esw_vport_disable(esw, vport_num); @@ -3767,6 +3739,7 @@ devlink_err: void mlx5_esw_offloads_sf_vport_disable(struct mlx5_eswitch *esw, u16 vport_num) { mlx5_esw_offloads_rep_unload(esw, vport_num); + mlx5_esw_vport_debugfs_destroy(esw, vport_num); mlx5_esw_devlink_sf_port_unregister(esw, vport_num); mlx5_esw_vport_disable(esw, vport_num); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c index 2ccf7bef9b05..735dc805dad7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c @@ -479,6 +479,30 @@ static int mlx5_set_extended_dest(struct mlx5_core_dev *dev, return 0; } + +static void +mlx5_cmd_set_fte_flow_meter(struct fs_fte *fte, void *in_flow_context) +{ + void *exe_aso_ctrl; + void *execute_aso; + + execute_aso = MLX5_ADDR_OF(flow_context, in_flow_context, + execute_aso[0]); + MLX5_SET(execute_aso, execute_aso, valid, 1); + MLX5_SET(execute_aso, execute_aso, aso_object_id, + fte->action.exe_aso.object_id); + + exe_aso_ctrl = MLX5_ADDR_OF(execute_aso, execute_aso, exe_aso_ctrl); + MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, return_reg_id, + fte->action.exe_aso.return_reg_id); + MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, aso_type, + fte->action.exe_aso.type); + MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, init_color, + fte->action.exe_aso.flow_meter.init_color); + MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, meter_id, + fte->action.exe_aso.flow_meter.meter_idx); +} + static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev, int opmod, int modify_mask, struct mlx5_flow_table *ft, @@ -663,6 +687,15 @@ static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev, list_size); } + if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) { + if (fte->action.exe_aso.type == MLX5_EXE_ASO_FLOW_METER) { + mlx5_cmd_set_fte_flow_meter(fte, in_flow_context); + } else { + err = -EOPNOTSUPP; + goto err_out; + } + } + err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); err_out: kvfree(in); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c index 21e5c709b2d3..f1b908d40fa6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c @@ -2895,6 +2895,14 @@ static int create_fdb_bypass(struct mlx5_flow_steering *steering) return 0; } +static void cleanup_fdb_root_ns(struct mlx5_flow_steering *steering) +{ + cleanup_root_ns(steering->fdb_root_ns); + steering->fdb_root_ns = NULL; + kfree(steering->fdb_sub_ns); + steering->fdb_sub_ns = NULL; +} + static int init_fdb_root_ns(struct mlx5_flow_steering *steering) { struct fs_prio *maj_prio; @@ -2945,10 +2953,7 @@ static int init_fdb_root_ns(struct mlx5_flow_steering *steering) return 0; out_err: - cleanup_root_ns(steering->fdb_root_ns); - kfree(steering->fdb_sub_ns); - steering->fdb_sub_ns = NULL; - steering->fdb_root_ns = NULL; + cleanup_fdb_root_ns(steering); return err; } @@ -3108,10 +3113,7 @@ void mlx5_fs_core_cleanup(struct mlx5_core_dev *dev) struct mlx5_flow_steering *steering = dev->priv.steering; cleanup_root_ns(steering->root_ns); - cleanup_root_ns(steering->fdb_root_ns); - steering->fdb_root_ns = NULL; - kfree(steering->fdb_sub_ns); - steering->fdb_sub_ns = NULL; + cleanup_fdb_root_ns(steering); cleanup_root_ns(steering->port_sel_root_ns); cleanup_root_ns(steering->sniffer_rx_root_ns); cleanup_root_ns(steering->sniffer_tx_root_ns); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c index cfb8bedba512..079fa44ada71 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c @@ -289,6 +289,10 @@ int mlx5_cmd_init_hca(struct mlx5_core_dev *dev, uint32_t *sw_owner_id) sw_owner_id[i]); } + if (MLX5_CAP_GEN_2_MAX(dev, sw_vhca_id_valid) && + dev->priv.sw_vhca_id > 0) + MLX5_SET(init_hca_in, in, sw_vhca_id, dev->priv.sw_vhca_id); + return mlx5_cmd_exec_in(dev, init_hca, in); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c index 8da73ef5680f..ac3757beaea2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c @@ -83,7 +83,7 @@ static void mlx5i_get_ringparam(struct net_device *dev, { struct mlx5e_priv *priv = mlx5i_epriv(dev); - mlx5e_ethtool_get_ringparam(priv, param); + mlx5e_ethtool_get_ringparam(priv, param, kernel_param); } static int mlx5i_set_channels(struct net_device *dev, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c index 5d41e19378e0..0f34e3c80d1f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c @@ -638,6 +638,7 @@ static int mlx5_deactivate_lag(struct mlx5_lag *ldev) static bool mlx5_lag_check_prereq(struct mlx5_lag *ldev) { #ifdef CONFIG_MLX5_ESWITCH + struct mlx5_core_dev *dev; u8 mode; #endif int i; @@ -647,11 +648,11 @@ static bool mlx5_lag_check_prereq(struct mlx5_lag *ldev) return false; #ifdef CONFIG_MLX5_ESWITCH - mode = mlx5_eswitch_mode(ldev->pf[MLX5_LAG_P1].dev); - - if (mode != MLX5_ESWITCH_NONE && mode != MLX5_ESWITCH_OFFLOADS) + dev = ldev->pf[MLX5_LAG_P1].dev; + if ((mlx5_sriov_is_enabled(dev)) && !is_mdev_switchdev_mode(dev)) return false; + mode = mlx5_eswitch_mode(dev); for (i = 0; i < ldev->ports; i++) if (mlx5_eswitch_mode(ldev->pf[i].dev) != mode) return false; @@ -766,8 +767,7 @@ static bool mlx5_lag_is_roce_lag(struct mlx5_lag *ldev) #ifdef CONFIG_MLX5_ESWITCH for (i = 0; i < ldev->ports; i++) - roce_lag = roce_lag && - ldev->pf[i].dev->priv.eswitch->mode == MLX5_ESWITCH_NONE; + roce_lag = roce_lag && is_mdev_legacy_mode(ldev->pf[i].dev); #endif return roce_lag; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c new file mode 100644 index 000000000000..21e14507ff5c --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c @@ -0,0 +1,433 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +#include <linux/mlx5/device.h> +#include <linux/mlx5/transobj.h> +#include "aso.h" +#include "wq.h" + +struct mlx5_aso_cq { + /* data path - accessed per cqe */ + struct mlx5_cqwq wq; + + /* data path - accessed per napi poll */ + struct mlx5_core_cq mcq; + + /* control */ + struct mlx5_core_dev *mdev; + struct mlx5_wq_ctrl wq_ctrl; +} ____cacheline_aligned_in_smp; + +struct mlx5_aso { + /* data path */ + u16 cc; + u16 pc; + + struct mlx5_wqe_ctrl_seg *doorbell_cseg; + struct mlx5_aso_cq cq; + + /* read only */ + struct mlx5_wq_cyc wq; + void __iomem *uar_map; + u32 sqn; + + /* control path */ + struct mlx5_wq_ctrl wq_ctrl; + +} ____cacheline_aligned_in_smp; + +static void mlx5_aso_free_cq(struct mlx5_aso_cq *cq) +{ + mlx5_wq_destroy(&cq->wq_ctrl); +} + +static int mlx5_aso_alloc_cq(struct mlx5_core_dev *mdev, int numa_node, + void *cqc_data, struct mlx5_aso_cq *cq) +{ + struct mlx5_core_cq *mcq = &cq->mcq; + struct mlx5_wq_param param; + int err; + u32 i; + + param.buf_numa_node = numa_node; + param.db_numa_node = numa_node; + + err = mlx5_cqwq_create(mdev, ¶m, cqc_data, &cq->wq, &cq->wq_ctrl); + if (err) + return err; + + mcq->cqe_sz = 64; + mcq->set_ci_db = cq->wq_ctrl.db.db; + mcq->arm_db = cq->wq_ctrl.db.db + 1; + + for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) { + struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i); + + cqe->op_own = 0xf1; + } + + cq->mdev = mdev; + + return 0; +} + +static int create_aso_cq(struct mlx5_aso_cq *cq, void *cqc_data) +{ + u32 out[MLX5_ST_SZ_DW(create_cq_out)]; + struct mlx5_core_dev *mdev = cq->mdev; + struct mlx5_core_cq *mcq = &cq->mcq; + void *in, *cqc; + int inlen, eqn; + int err; + + err = mlx5_vector2eqn(mdev, 0, &eqn); + if (err) + return err; + + inlen = MLX5_ST_SZ_BYTES(create_cq_in) + + sizeof(u64) * cq->wq_ctrl.buf.npages; + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) + return -ENOMEM; + + cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); + + memcpy(cqc, cqc_data, MLX5_ST_SZ_BYTES(cqc)); + + mlx5_fill_page_frag_array(&cq->wq_ctrl.buf, + (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas)); + + MLX5_SET(cqc, cqc, cq_period_mode, DIM_CQ_PERIOD_MODE_START_FROM_EQE); + MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); + MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index); + MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift - + MLX5_ADAPTER_PAGE_SHIFT); + MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma); + + err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out)); + + kvfree(in); + + return err; +} + +static void mlx5_aso_destroy_cq(struct mlx5_aso_cq *cq) +{ + mlx5_core_destroy_cq(cq->mdev, &cq->mcq); + mlx5_wq_destroy(&cq->wq_ctrl); +} + +static int mlx5_aso_create_cq(struct mlx5_core_dev *mdev, int numa_node, + struct mlx5_aso_cq *cq) +{ + void *cqc_data; + int err; + + cqc_data = kvzalloc(MLX5_ST_SZ_BYTES(cqc), GFP_KERNEL); + if (!cqc_data) + return -ENOMEM; + + MLX5_SET(cqc, cqc_data, log_cq_size, 1); + MLX5_SET(cqc, cqc_data, uar_page, mdev->priv.uar->index); + if (MLX5_CAP_GEN(mdev, cqe_128_always) && cache_line_size() >= 128) + MLX5_SET(cqc, cqc_data, cqe_sz, CQE_STRIDE_128_PAD); + + err = mlx5_aso_alloc_cq(mdev, numa_node, cqc_data, cq); + if (err) { + mlx5_core_err(mdev, "Failed to alloc aso wq cq, err=%d\n", err); + goto err_out; + } + + err = create_aso_cq(cq, cqc_data); + if (err) { + mlx5_core_err(mdev, "Failed to create aso wq cq, err=%d\n", err); + goto err_free_cq; + } + + kvfree(cqc_data); + return 0; + +err_free_cq: + mlx5_aso_free_cq(cq); +err_out: + kvfree(cqc_data); + return err; +} + +static int mlx5_aso_alloc_sq(struct mlx5_core_dev *mdev, int numa_node, + void *sqc_data, struct mlx5_aso *sq) +{ + void *sqc_wq = MLX5_ADDR_OF(sqc, sqc_data, wq); + struct mlx5_wq_cyc *wq = &sq->wq; + struct mlx5_wq_param param; + int err; + + sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map; + + param.db_numa_node = numa_node; + param.buf_numa_node = numa_node; + err = mlx5_wq_cyc_create(mdev, ¶m, sqc_wq, wq, &sq->wq_ctrl); + if (err) + return err; + wq->db = &wq->db[MLX5_SND_DBR]; + + return 0; +} + +static int create_aso_sq(struct mlx5_core_dev *mdev, int pdn, + void *sqc_data, struct mlx5_aso *sq) +{ + void *in, *sqc, *wq; + int inlen, err; + + inlen = MLX5_ST_SZ_BYTES(create_sq_in) + + sizeof(u64) * sq->wq_ctrl.buf.npages; + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) + return -ENOMEM; + + sqc = MLX5_ADDR_OF(create_sq_in, in, ctx); + wq = MLX5_ADDR_OF(sqc, sqc, wq); + + memcpy(sqc, sqc_data, MLX5_ST_SZ_BYTES(sqc)); + MLX5_SET(sqc, sqc, cqn, sq->cq.mcq.cqn); + + MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST); + MLX5_SET(sqc, sqc, flush_in_error_en, 1); + + MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); + MLX5_SET(wq, wq, uar_page, mdev->mlx5e_res.hw_objs.bfreg.index); + MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift - + MLX5_ADAPTER_PAGE_SHIFT); + MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma); + + mlx5_fill_page_frag_array(&sq->wq_ctrl.buf, + (__be64 *)MLX5_ADDR_OF(wq, wq, pas)); + + err = mlx5_core_create_sq(mdev, in, inlen, &sq->sqn); + + kvfree(in); + + return err; +} + +static int mlx5_aso_set_sq_rdy(struct mlx5_core_dev *mdev, u32 sqn) +{ + void *in, *sqc; + int inlen, err; + + inlen = MLX5_ST_SZ_BYTES(modify_sq_in); + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) + return -ENOMEM; + + MLX5_SET(modify_sq_in, in, sq_state, MLX5_SQC_STATE_RST); + sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx); + MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RDY); + + err = mlx5_core_modify_sq(mdev, sqn, in); + + kvfree(in); + + return err; +} + +static int mlx5_aso_create_sq_rdy(struct mlx5_core_dev *mdev, u32 pdn, + void *sqc_data, struct mlx5_aso *sq) +{ + int err; + + err = create_aso_sq(mdev, pdn, sqc_data, sq); + if (err) + return err; + + err = mlx5_aso_set_sq_rdy(mdev, sq->sqn); + if (err) + mlx5_core_destroy_sq(mdev, sq->sqn); + + return err; +} + +static void mlx5_aso_free_sq(struct mlx5_aso *sq) +{ + mlx5_wq_destroy(&sq->wq_ctrl); +} + +static void mlx5_aso_destroy_sq(struct mlx5_aso *sq) +{ + mlx5_core_destroy_sq(sq->cq.mdev, sq->sqn); + mlx5_aso_free_sq(sq); +} + +static int mlx5_aso_create_sq(struct mlx5_core_dev *mdev, int numa_node, + u32 pdn, struct mlx5_aso *sq) +{ + void *sqc_data, *wq; + int err; + + sqc_data = kvzalloc(MLX5_ST_SZ_BYTES(sqc), GFP_KERNEL); + if (!sqc_data) + return -ENOMEM; + + wq = MLX5_ADDR_OF(sqc, sqc_data, wq); + MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB)); + MLX5_SET(wq, wq, pd, pdn); + MLX5_SET(wq, wq, log_wq_sz, 1); + + err = mlx5_aso_alloc_sq(mdev, numa_node, sqc_data, sq); + if (err) { + mlx5_core_err(mdev, "Failed to alloc aso wq sq, err=%d\n", err); + goto err_out; + } + + err = mlx5_aso_create_sq_rdy(mdev, pdn, sqc_data, sq); + if (err) { + mlx5_core_err(mdev, "Failed to open aso wq sq, err=%d\n", err); + goto err_free_asosq; + } + + mlx5_core_dbg(mdev, "aso sq->sqn = 0x%x\n", sq->sqn); + + kvfree(sqc_data); + return 0; + +err_free_asosq: + mlx5_aso_free_sq(sq); +err_out: + kvfree(sqc_data); + return err; +} + +struct mlx5_aso *mlx5_aso_create(struct mlx5_core_dev *mdev, u32 pdn) +{ + int numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); + struct mlx5_aso *aso; + int err; + + aso = kzalloc(sizeof(*aso), GFP_KERNEL); + if (!aso) + return ERR_PTR(-ENOMEM); + + err = mlx5_aso_create_cq(mdev, numa_node, &aso->cq); + if (err) + goto err_cq; + + err = mlx5_aso_create_sq(mdev, numa_node, pdn, aso); + if (err) + goto err_sq; + + return aso; + +err_sq: + mlx5_aso_destroy_cq(&aso->cq); +err_cq: + kfree(aso); + return ERR_PTR(err); +} + +void mlx5_aso_destroy(struct mlx5_aso *aso) +{ + if (IS_ERR_OR_NULL(aso)) + return; + + mlx5_aso_destroy_sq(aso); + mlx5_aso_destroy_cq(&aso->cq); + kfree(aso); +} + +void mlx5_aso_build_wqe(struct mlx5_aso *aso, u8 ds_cnt, + struct mlx5_aso_wqe *aso_wqe, + u32 obj_id, u32 opc_mode) +{ + struct mlx5_wqe_ctrl_seg *cseg = &aso_wqe->ctrl; + + cseg->opmod_idx_opcode = cpu_to_be32((opc_mode << MLX5_WQE_CTRL_WQE_OPC_MOD_SHIFT) | + (aso->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) | + MLX5_OPCODE_ACCESS_ASO); + cseg->qpn_ds = cpu_to_be32((aso->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | ds_cnt); + cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; + cseg->general_id = cpu_to_be32(obj_id); +} + +void *mlx5_aso_get_wqe(struct mlx5_aso *aso) +{ + u16 pi; + + pi = mlx5_wq_cyc_ctr2ix(&aso->wq, aso->pc); + return mlx5_wq_cyc_get_wqe(&aso->wq, pi); +} + +void mlx5_aso_post_wqe(struct mlx5_aso *aso, bool with_data, + struct mlx5_wqe_ctrl_seg *doorbell_cseg) +{ + doorbell_cseg->fm_ce_se |= MLX5_WQE_CTRL_CQ_UPDATE; + /* ensure wqe is visible to device before updating doorbell record */ + dma_wmb(); + + if (with_data) + aso->pc += MLX5_ASO_WQEBBS_DATA; + else + aso->pc += MLX5_ASO_WQEBBS; + *aso->wq.db = cpu_to_be32(aso->pc); + + /* ensure doorbell record is visible to device before ringing the + * doorbell + */ + wmb(); + + mlx5_write64((__be32 *)doorbell_cseg, aso->uar_map); + + /* Ensure doorbell is written on uar_page before poll_cq */ + WRITE_ONCE(doorbell_cseg, NULL); +} + +int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data, u32 interval_ms) +{ + struct mlx5_aso_cq *cq = &aso->cq; + struct mlx5_cqe64 *cqe; + unsigned long expires; + + cqe = mlx5_cqwq_get_cqe(&cq->wq); + + expires = jiffies + msecs_to_jiffies(interval_ms); + while (!cqe && time_is_after_jiffies(expires)) { + usleep_range(2, 10); + cqe = mlx5_cqwq_get_cqe(&cq->wq); + } + + if (!cqe) + return -ETIMEDOUT; + + /* sq->cc must be updated only after mlx5_cqwq_update_db_record(), + * otherwise a cq overrun may occur + */ + mlx5_cqwq_pop(&cq->wq); + + if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) { + struct mlx5_err_cqe *err_cqe; + + mlx5_core_err(cq->mdev, "Bad OP in ASOSQ CQE: 0x%x\n", + get_cqe_opcode(cqe)); + + err_cqe = (struct mlx5_err_cqe *)cqe; + mlx5_core_err(cq->mdev, "vendor_err_synd=%x\n", + err_cqe->vendor_err_synd); + mlx5_core_err(cq->mdev, "syndrome=%x\n", + err_cqe->syndrome); + print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, + 16, 1, err_cqe, + sizeof(*err_cqe), false); + } + + mlx5_cqwq_update_db_record(&cq->wq); + + /* ensure cq space is freed before enabling more cqes */ + wmb(); + + if (with_data) + aso->cc += MLX5_ASO_WQEBBS_DATA; + else + aso->cc += MLX5_ASO_WQEBBS; + + return 0; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h new file mode 100644 index 000000000000..b3bbf284fe71 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5_LIB_ASO_H__ +#define __MLX5_LIB_ASO_H__ + +#include <linux/mlx5/qp.h> +#include "mlx5_core.h" + +#define MLX5_ASO_WQEBBS \ + (DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe), MLX5_SEND_WQE_BB)) +#define MLX5_ASO_WQEBBS_DATA \ + (DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe_data), MLX5_SEND_WQE_BB)) +#define MLX5_WQE_CTRL_WQE_OPC_MOD_SHIFT 24 + +struct mlx5_wqe_aso_ctrl_seg { + __be32 va_h; + __be32 va_l; /* include read_enable */ + __be32 l_key; + u8 data_mask_mode; + u8 condition_1_0_operand; + u8 condition_1_0_offset; + u8 data_offset_condition_operand; + __be32 condition_0_data; + __be32 condition_0_mask; + __be32 condition_1_data; + __be32 condition_1_mask; + __be64 bitwise_data; + __be64 data_mask; +}; + +struct mlx5_wqe_aso_data_seg { + __be32 bytewise_data[16]; +}; + +struct mlx5_aso_wqe { + struct mlx5_wqe_ctrl_seg ctrl; + struct mlx5_wqe_aso_ctrl_seg aso_ctrl; +}; + +struct mlx5_aso_wqe_data { + struct mlx5_wqe_ctrl_seg ctrl; + struct mlx5_wqe_aso_ctrl_seg aso_ctrl; + struct mlx5_wqe_aso_data_seg aso_data; +}; + +enum { + MLX5_ASO_LOGICAL_AND, + MLX5_ASO_LOGICAL_OR, +}; + +enum { + MLX5_ASO_ALWAYS_FALSE, + MLX5_ASO_ALWAYS_TRUE, + MLX5_ASO_EQUAL, + MLX5_ASO_NOT_EQUAL, + MLX5_ASO_GREATER_OR_EQUAL, + MLX5_ASO_LESSER_OR_EQUAL, + MLX5_ASO_LESSER, + MLX5_ASO_GREATER, + MLX5_ASO_CYCLIC_GREATER, + MLX5_ASO_CYCLIC_LESSER, +}; + +enum { + MLX5_ASO_DATA_MASK_MODE_BITWISE_64BIT, + MLX5_ASO_DATA_MASK_MODE_BYTEWISE_64BYTE, + MLX5_ASO_DATA_MASK_MODE_CALCULATED_64BYTE, +}; + +enum { + MLX5_ACCESS_ASO_OPC_MOD_FLOW_METER = 0x2, +}; + +struct mlx5_aso; + +void *mlx5_aso_get_wqe(struct mlx5_aso *aso); +void mlx5_aso_build_wqe(struct mlx5_aso *aso, u8 ds_cnt, + struct mlx5_aso_wqe *aso_wqe, + u32 obj_id, u32 opc_mode); +void mlx5_aso_post_wqe(struct mlx5_aso *aso, bool with_data, + struct mlx5_wqe_ctrl_seg *doorbell_cseg); +int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data, u32 interval_ms); + +struct mlx5_aso *mlx5_aso_create(struct mlx5_core_dev *mdev, u32 pdn); +void mlx5_aso_destroy(struct mlx5_aso *aso); +#endif /* __MLX5_LIB_ASO_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/dm.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/dm.c index 3d5e57ff558c..9482e51ac82a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/dm.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/dm.c @@ -12,13 +12,16 @@ struct mlx5_dm { spinlock_t lock; unsigned long *steering_sw_icm_alloc_blocks; unsigned long *header_modify_sw_icm_alloc_blocks; + unsigned long *header_modify_pattern_sw_icm_alloc_blocks; }; struct mlx5_dm *mlx5_dm_create(struct mlx5_core_dev *dev) { + u64 header_modify_pattern_icm_blocks = 0; u64 header_modify_icm_blocks = 0; u64 steering_icm_blocks = 0; struct mlx5_dm *dm; + bool support_v2; if (!(MLX5_CAP_GEN_64(dev, general_obj_types) & MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM)) return NULL; @@ -35,8 +38,7 @@ struct mlx5_dm *mlx5_dm_create(struct mlx5_core_dev *dev) MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)); dm->steering_sw_icm_alloc_blocks = - kcalloc(BITS_TO_LONGS(steering_icm_blocks), - sizeof(unsigned long), GFP_KERNEL); + bitmap_zalloc(steering_icm_blocks, GFP_KERNEL); if (!dm->steering_sw_icm_alloc_blocks) goto err_steering; } @@ -47,16 +49,33 @@ struct mlx5_dm *mlx5_dm_create(struct mlx5_core_dev *dev) MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)); dm->header_modify_sw_icm_alloc_blocks = - kcalloc(BITS_TO_LONGS(header_modify_icm_blocks), - sizeof(unsigned long), GFP_KERNEL); + bitmap_zalloc(header_modify_icm_blocks, GFP_KERNEL); if (!dm->header_modify_sw_icm_alloc_blocks) goto err_modify_hdr; } + support_v2 = MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner_v2) && + MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner_v2) && + MLX5_CAP64_DEV_MEM(dev, header_modify_pattern_sw_icm_start_address); + + if (support_v2) { + header_modify_pattern_icm_blocks = + BIT(MLX5_CAP_DEV_MEM(dev, log_header_modify_pattern_sw_icm_size) - + MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)); + + dm->header_modify_pattern_sw_icm_alloc_blocks = + bitmap_zalloc(header_modify_pattern_icm_blocks, GFP_KERNEL); + if (!dm->header_modify_pattern_sw_icm_alloc_blocks) + goto err_pattern; + } + return dm; +err_pattern: + bitmap_free(dm->header_modify_sw_icm_alloc_blocks); + err_modify_hdr: - kfree(dm->steering_sw_icm_alloc_blocks); + bitmap_free(dm->steering_sw_icm_alloc_blocks); err_steering: kfree(dm); @@ -75,7 +94,7 @@ void mlx5_dm_cleanup(struct mlx5_core_dev *dev) WARN_ON(!bitmap_empty(dm->steering_sw_icm_alloc_blocks, BIT(MLX5_CAP_DEV_MEM(dev, log_steering_sw_icm_size) - MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)))); - kfree(dm->steering_sw_icm_alloc_blocks); + bitmap_free(dm->steering_sw_icm_alloc_blocks); } if (dm->header_modify_sw_icm_alloc_blocks) { @@ -83,7 +102,15 @@ void mlx5_dm_cleanup(struct mlx5_core_dev *dev) BIT(MLX5_CAP_DEV_MEM(dev, log_header_modify_sw_icm_size) - MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)))); - kfree(dm->header_modify_sw_icm_alloc_blocks); + bitmap_free(dm->header_modify_sw_icm_alloc_blocks); + } + + if (dm->header_modify_pattern_sw_icm_alloc_blocks) { + WARN_ON(!bitmap_empty(dm->header_modify_pattern_sw_icm_alloc_blocks, + BIT(MLX5_CAP_DEV_MEM(dev, + log_header_modify_pattern_sw_icm_size) - + MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)))); + bitmap_free(dm->header_modify_pattern_sw_icm_alloc_blocks); } kfree(dm); @@ -130,6 +157,13 @@ int mlx5_dm_sw_icm_alloc(struct mlx5_core_dev *dev, enum mlx5_sw_icm_type type, log_header_modify_sw_icm_size); block_map = dm->header_modify_sw_icm_alloc_blocks; break; + case MLX5_SW_ICM_TYPE_HEADER_MODIFY_PATTERN: + icm_start_addr = MLX5_CAP64_DEV_MEM(dev, + header_modify_pattern_sw_icm_start_address); + log_icm_size = MLX5_CAP_DEV_MEM(dev, + log_header_modify_pattern_sw_icm_size); + block_map = dm->header_modify_pattern_sw_icm_alloc_blocks; + break; default: return -EINVAL; } @@ -203,6 +237,11 @@ int mlx5_dm_sw_icm_dealloc(struct mlx5_core_dev *dev, enum mlx5_sw_icm_type type icm_start_addr = MLX5_CAP64_DEV_MEM(dev, header_modify_sw_icm_start_address); block_map = dm->header_modify_sw_icm_alloc_blocks; break; + case MLX5_SW_ICM_TYPE_HEADER_MODIFY_PATTERN: + icm_start_addr = MLX5_CAP64_DEV_MEM(dev, + header_modify_pattern_sw_icm_start_address); + block_map = dm->header_modify_pattern_sw_icm_alloc_blocks; + break; default: return -EINVAL; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index c9b4e50a593e..8b621c1ddd14 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -90,6 +90,8 @@ module_param_named(prof_sel, prof_sel, uint, 0444); MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2"); static u32 sw_owner_id[4]; +#define MAX_SW_VHCA_ID (BIT(__mlx5_bit_sz(cmd_hca_cap_2, sw_vhca_id)) - 1) +static DEFINE_IDA(sw_vhca_ida); enum { MLX5_ATOMIC_REQ_MODE_BE = 0x0, @@ -314,13 +316,6 @@ struct mlx5_reg_host_endianness { u8 rsvd[15]; }; -#define CAP_MASK(pos, size) ((u64)((1 << (size)) - 1) << (pos)) - -enum { - MLX5_CAP_BITS_RW_MASK = CAP_MASK(MLX5_CAP_OFF_CMDIF_CSUM, 2) | - MLX5_DEV_CAP_FLAG_DCT, -}; - static u16 to_fw_pkey_sz(struct mlx5_core_dev *dev, u32 size) { switch (size) { @@ -499,6 +494,31 @@ static int max_uc_list_get_devlink_param(struct mlx5_core_dev *dev) return err; } +static int handle_hca_cap_2(struct mlx5_core_dev *dev, void *set_ctx) +{ + void *set_hca_cap; + int err; + + if (!MLX5_CAP_GEN_MAX(dev, hca_cap_2)) + return 0; + + err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL_2); + if (err) + return err; + + if (!MLX5_CAP_GEN_2_MAX(dev, sw_vhca_id_valid) || + !(dev->priv.sw_vhca_id > 0)) + return 0; + + set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, + capability); + memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_GENERAL_2]->cur, + MLX5_ST_SZ_BYTES(cmd_hca_cap_2)); + MLX5_SET(cmd_hca_cap_2, set_hca_cap, sw_vhca_id_valid, 1); + + return set_caps(dev, set_ctx, MLX5_CAP_GENERAL_2); +} + static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx) { struct mlx5_profile *prof = &dev->profile; @@ -669,6 +689,13 @@ static int set_hca_cap(struct mlx5_core_dev *dev) goto out; } + memset(set_ctx, 0, set_sz); + err = handle_hca_cap_2(dev, set_ctx); + if (err) { + mlx5_core_err(dev, "handle_hca_cap_2 failed\n"); + goto out; + } + out: kfree(set_ctx); return err; @@ -1257,6 +1284,7 @@ static void mlx5_unload(struct mlx5_core_dev *dev) { mlx5_sf_dev_table_destroy(dev); mlx5_sriov_detach(dev); + mlx5_eswitch_disable(dev->priv.eswitch); mlx5_lag_remove_mdev(dev); mlx5_ec_cleanup(dev); mlx5_sf_hw_table_destroy(dev); @@ -1512,6 +1540,18 @@ int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) if (err) goto err_hca_caps; + /* The conjunction of sw_vhca_id with sw_owner_id will be a global + * unique id per function which uses mlx5_core. + * Those values are supplied to FW as part of the init HCA command to + * be used by both driver and FW when it's applicable. + */ + dev->priv.sw_vhca_id = ida_alloc_range(&sw_vhca_ida, 1, + MAX_SW_VHCA_ID, + GFP_KERNEL); + if (dev->priv.sw_vhca_id < 0) + mlx5_core_err(dev, "failed to allocate sw_vhca_id, err=%d\n", + dev->priv.sw_vhca_id); + return 0; err_hca_caps: @@ -1536,6 +1576,9 @@ void mlx5_mdev_uninit(struct mlx5_core_dev *dev) { struct mlx5_priv *priv = &dev->priv; + if (priv->sw_vhca_id > 0) + ida_free(&sw_vhca_ida, dev->priv.sw_vhca_id); + mlx5_hca_caps_free(dev); mlx5_adev_cleanup(dev); mlx5_pagealloc_cleanup(dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c index 3be659cd91f1..7d955a4d9f14 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c @@ -501,7 +501,7 @@ static int mlx5_sf_esw_event(struct notifier_block *nb, unsigned long event, voi case MLX5_ESWITCH_OFFLOADS: mlx5_sf_table_enable(table); break; - case MLX5_ESWITCH_NONE: + case MLX5_ESWITCH_LEGACY: mlx5_sf_table_disable(table); break; default: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c index 2935614f6fa9..5757cd6e1819 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c @@ -145,8 +145,7 @@ mlx5_device_disable_sriov(struct mlx5_core_dev *dev, int num_vfs, bool clear_vf) sriov->vfs_ctx[vf].enabled = 0; } - if (MLX5_ESWITCH_MANAGER(dev)) - mlx5_eswitch_disable(dev->priv.eswitch, clear_vf); + mlx5_eswitch_disable_sriov(dev->priv.eswitch, clear_vf); if (mlx5_wait_for_pages(dev, &dev->priv.vfs_pages)) mlx5_core_warn(dev, "timeout reclaiming VFs pages\n"); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/vport.c b/drivers/net/ethernet/mellanox/mlx5/core/vport.c index ac020cb78072..d5c317325030 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/vport.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/vport.c @@ -1086,9 +1086,17 @@ int mlx5_nic_vport_affiliate_multiport(struct mlx5_core_dev *master_mdev, goto free; MLX5_SET(modify_nic_vport_context_in, in, field_select.affiliation, 1); - MLX5_SET(modify_nic_vport_context_in, in, - nic_vport_context.affiliated_vhca_id, - MLX5_CAP_GEN(master_mdev, vhca_id)); + if (MLX5_CAP_GEN_2(master_mdev, sw_vhca_id_valid)) { + MLX5_SET(modify_nic_vport_context_in, in, + nic_vport_context.vhca_id_type, VHCA_ID_TYPE_SW); + MLX5_SET(modify_nic_vport_context_in, in, + nic_vport_context.affiliated_vhca_id, + MLX5_CAP_GEN_2(master_mdev, sw_vhca_id)); + } else { + MLX5_SET(modify_nic_vport_context_in, in, + nic_vport_context.affiliated_vhca_id, + MLX5_CAP_GEN(master_mdev, vhca_id)); + } MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.affiliation_criteria, MLX5_CAP_GEN(port_mdev, affiliate_nic_vport_criteria)); |