diff options
| author | Jakub Kicinski <kuba@kernel.org> | 2025-09-30 17:21:17 -0700 |
|---|---|---|
| committer | Jakub Kicinski <kuba@kernel.org> | 2025-09-30 17:21:17 -0700 |
| commit | 3cfb33f92dfb118ccaec5613c677f444af12bcc7 (patch) | |
| tree | b2d4ae47b03df176b2d8bf2ce6637ed5fb55204c | |
| parent | fd8c8216648cd8c047bd3bcad65424ed44b5b450 (diff) | |
| parent | a833538d1d8db96b78bac04eec9be51b297f1d23 (diff) | |
Merge branch 'net-mlx5-misc-changes-2025-09-28'
Tariq Toukan says:
====================
net/mlx5: misc changes 2025-09-28
This series contains misc enhancements to the mlx5 driver.
v1: https://lore.kernel.org/1758531671-819655-1-git-send-email-tariqt@nvidia.com
====================
Link: https://patch.msgid.link/1759094723-843774-1-git-send-email-tariqt@nvidia.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
14 files changed, 995 insertions, 1275 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c index c68ba0e58fa6..c96cbc4b0dbf 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c @@ -75,15 +75,14 @@ struct mlx5e_rss { struct mlx5e_tir *inner_tir[MLX5E_NUM_INDIR_TIRS]; struct mlx5e_rqt rqt; struct mlx5_core_dev *mdev; /* primary */ - u32 drop_rqn; - bool inner_ft_support; + struct mlx5e_rss_params params; bool enabled; refcount_t refcnt; }; bool mlx5e_rss_get_inner_ft_support(struct mlx5e_rss *rss) { - return rss->inner_ft_support; + return rss->params.inner_ft_support; } void mlx5e_rss_params_indir_modify_actual_size(struct mlx5e_rss *rss, u32 num_channels) @@ -91,7 +90,7 @@ void mlx5e_rss_params_indir_modify_actual_size(struct mlx5e_rss *rss, u32 num_ch rss->indir.actual_table_size = mlx5e_rqt_size(rss->mdev, num_channels); } -int mlx5e_rss_params_indir_init(struct mlx5e_rss_params_indir *indir, struct mlx5_core_dev *mdev, +int mlx5e_rss_params_indir_init(struct mlx5e_rss_params_indir *indir, u32 actual_table_size, u32 max_table_size) { indir->table = kvmalloc_array(max_table_size, sizeof(*indir->table), GFP_KERNEL); @@ -139,7 +138,8 @@ static struct mlx5e_rss *mlx5e_rss_init_copy(const struct mlx5e_rss *from) if (!rss) return ERR_PTR(-ENOMEM); - err = mlx5e_rss_params_indir_init(&rss->indir, from->mdev, from->indir.actual_table_size, + err = mlx5e_rss_params_indir_init(&rss->indir, + from->indir.actual_table_size, from->indir.max_table_size); if (err) goto err_free_rss; @@ -192,11 +192,12 @@ mlx5e_rss_get_tt_config(struct mlx5e_rss *rss, enum mlx5_traffic_types tt) return rss_tt; } -static int mlx5e_rss_create_tir(struct mlx5e_rss *rss, - enum mlx5_traffic_types tt, - const struct mlx5e_packet_merge_param *init_pkt_merge_param, - bool inner) +static int +mlx5e_rss_create_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, + const struct mlx5e_packet_merge_param *pkt_merge_param, + bool inner) { + bool rss_inner = rss->params.inner_ft_support; struct mlx5e_rss_params_traffic_type rss_tt; struct mlx5e_tir_builder *builder; struct mlx5e_tir **tir_p; @@ -204,7 +205,7 @@ static int mlx5e_rss_create_tir(struct mlx5e_rss *rss, u32 rqtn; int err; - if (inner && !rss->inner_ft_support) { + if (inner && !rss_inner) { mlx5e_rss_warn(rss->mdev, "Cannot create inner indirect TIR[%d], RSS inner FT is not supported.\n", tt); @@ -227,8 +228,8 @@ static int mlx5e_rss_create_tir(struct mlx5e_rss *rss, rqtn = mlx5e_rqt_get_rqtn(&rss->rqt); mlx5e_tir_builder_build_rqt(builder, rss->mdev->mlx5e_res.hw_objs.td.tdn, - rqtn, rss->inner_ft_support); - mlx5e_tir_builder_build_packet_merge(builder, init_pkt_merge_param); + rqtn, rss_inner); + mlx5e_tir_builder_build_packet_merge(builder, pkt_merge_param); rss_tt = mlx5e_rss_get_tt_config(rss, tt); mlx5e_tir_builder_build_rss(builder, &rss->hash, &rss_tt, inner); @@ -264,15 +265,16 @@ static void mlx5e_rss_destroy_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types *tir_p = NULL; } -static int mlx5e_rss_create_tirs(struct mlx5e_rss *rss, - const struct mlx5e_packet_merge_param *init_pkt_merge_param, - bool inner) +static int +mlx5e_rss_create_tirs(struct mlx5e_rss *rss, + const struct mlx5e_packet_merge_param *pkt_merge_param, + bool inner) { enum mlx5_traffic_types tt, max_tt; int err; for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) { - err = mlx5e_rss_create_tir(rss, tt, init_pkt_merge_param, inner); + err = mlx5e_rss_create_tir(rss, tt, pkt_merge_param, inner); if (err) goto err_destroy_tirs; } @@ -335,7 +337,7 @@ static int mlx5e_rss_update_tirs(struct mlx5e_rss *rss) tt, err); } - if (!rss->inner_ft_support) + if (!rss->params.inner_ft_support) continue; err = mlx5e_rss_update_tir(rss, tt, true); @@ -355,14 +357,16 @@ static int mlx5e_rss_init_no_tirs(struct mlx5e_rss *rss) refcount_set(&rss->refcnt, 1); return mlx5e_rqt_init_direct(&rss->rqt, rss->mdev, true, - rss->drop_rqn, rss->indir.max_table_size); + rss->params.drop_rqn, + rss->indir.max_table_size); } -struct mlx5e_rss *mlx5e_rss_init(struct mlx5_core_dev *mdev, bool inner_ft_support, u32 drop_rqn, - const struct mlx5e_packet_merge_param *init_pkt_merge_param, - enum mlx5e_rss_init_type type, unsigned int nch, - unsigned int max_nch) +struct mlx5e_rss * +mlx5e_rss_init(struct mlx5_core_dev *mdev, + const struct mlx5e_rss_params *params, + const struct mlx5e_rss_init_params *init_params) { + u32 rqt_max_size, rqt_size; struct mlx5e_rss *rss; int err; @@ -370,29 +374,31 @@ struct mlx5e_rss *mlx5e_rss_init(struct mlx5_core_dev *mdev, bool inner_ft_suppo if (!rss) return ERR_PTR(-ENOMEM); - err = mlx5e_rss_params_indir_init(&rss->indir, mdev, - mlx5e_rqt_size(mdev, nch), - mlx5e_rqt_size(mdev, max_nch)); + rqt_size = mlx5e_rqt_size(mdev, init_params->nch); + rqt_max_size = mlx5e_rqt_size(mdev, init_params->max_nch); + err = mlx5e_rss_params_indir_init(&rss->indir, rqt_size, rqt_max_size); if (err) goto err_free_rss; rss->mdev = mdev; - rss->inner_ft_support = inner_ft_support; - rss->drop_rqn = drop_rqn; + rss->params = *params; err = mlx5e_rss_init_no_tirs(rss); if (err) goto err_free_indir; - if (type == MLX5E_RSS_INIT_NO_TIRS) + if (init_params->type == MLX5E_RSS_INIT_NO_TIRS) goto out; - err = mlx5e_rss_create_tirs(rss, init_pkt_merge_param, false); + err = mlx5e_rss_create_tirs(rss, init_params->pkt_merge_param, + false); if (err) goto err_destroy_rqt; - if (inner_ft_support) { - err = mlx5e_rss_create_tirs(rss, init_pkt_merge_param, true); + if (params->inner_ft_support) { + err = mlx5e_rss_create_tirs(rss, + init_params->pkt_merge_param, + true); if (err) goto err_destroy_tirs; } @@ -418,7 +424,7 @@ int mlx5e_rss_cleanup(struct mlx5e_rss *rss) mlx5e_rss_destroy_tirs(rss, false); - if (rss->inner_ft_support) + if (rss->params.inner_ft_support) mlx5e_rss_destroy_tirs(rss, true); mlx5e_rqt_destroy(&rss->rqt); @@ -448,7 +454,7 @@ u32 mlx5e_rss_get_tirn(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, { struct mlx5e_tir *tir; - WARN_ON(inner && !rss->inner_ft_support); + WARN_ON(inner && !rss->params.inner_ft_support); tir = rss_get_tir(rss, tt, inner); WARN_ON(!tir); @@ -468,10 +474,10 @@ bool mlx5e_rss_valid_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, bool /* Fill the "tirn" output parameter. * Create the requested TIR if it's its first usage. */ -int mlx5e_rss_obtain_tirn(struct mlx5e_rss *rss, - enum mlx5_traffic_types tt, - const struct mlx5e_packet_merge_param *init_pkt_merge_param, - bool inner, u32 *tirn) +int +mlx5e_rss_obtain_tirn(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, + const struct mlx5e_packet_merge_param *pkt_merge_param, + bool inner, u32 *tirn) { struct mlx5e_tir *tir; @@ -479,7 +485,7 @@ int mlx5e_rss_obtain_tirn(struct mlx5e_rss *rss, if (!tir) { /* TIR doesn't exist, create one */ int err; - err = mlx5e_rss_create_tir(rss, tt, init_pkt_merge_param, inner); + err = mlx5e_rss_create_tir(rss, tt, pkt_merge_param, inner); if (err) return err; tir = rss_get_tir(rss, tt, inner); @@ -512,10 +518,11 @@ void mlx5e_rss_disable(struct mlx5e_rss *rss) int err; rss->enabled = false; - err = mlx5e_rqt_redirect_direct(&rss->rqt, rss->drop_rqn, NULL); + err = mlx5e_rqt_redirect_direct(&rss->rqt, rss->params.drop_rqn, NULL); if (err) mlx5e_rss_warn(rss->mdev, "Failed to redirect RQT %#x to drop RQ %#x: err = %d\n", - mlx5e_rqt_get_rqtn(&rss->rqt), rss->drop_rqn, err); + mlx5e_rqt_get_rqtn(&rss->rqt), + rss->params.drop_rqn, err); } int mlx5e_rss_packet_merge_set_param(struct mlx5e_rss *rss, @@ -548,7 +555,7 @@ int mlx5e_rss_packet_merge_set_param(struct mlx5e_rss *rss, } inner_tir: - if (!rss->inner_ft_support) + if (!rss->params.inner_ft_support) continue; tir = rss_get_tir(rss, tt, true); @@ -681,7 +688,7 @@ int mlx5e_rss_set_hash_fields(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, return err; } - if (!(rss->inner_ft_support)) + if (!(rss->params.inner_ft_support)) return 0; err = mlx5e_rss_update_tir(rss, tt, true); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h index c6c1b2847cf5..5fb03cd0a411 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h @@ -13,19 +13,31 @@ enum mlx5e_rss_init_type { MLX5E_RSS_INIT_TIRS }; +struct mlx5e_rss_init_params { + enum mlx5e_rss_init_type type; + const struct mlx5e_packet_merge_param *pkt_merge_param; + unsigned int nch; + unsigned int max_nch; +}; + +struct mlx5e_rss_params { + bool inner_ft_support; + u32 drop_rqn; +}; + struct mlx5e_rss_params_traffic_type mlx5e_rss_get_default_tt_config(enum mlx5_traffic_types tt); struct mlx5e_rss; -int mlx5e_rss_params_indir_init(struct mlx5e_rss_params_indir *indir, struct mlx5_core_dev *mdev, +int mlx5e_rss_params_indir_init(struct mlx5e_rss_params_indir *indir, u32 actual_table_size, u32 max_table_size); void mlx5e_rss_params_indir_cleanup(struct mlx5e_rss_params_indir *indir); void mlx5e_rss_params_indir_modify_actual_size(struct mlx5e_rss *rss, u32 num_channels); -struct mlx5e_rss *mlx5e_rss_init(struct mlx5_core_dev *mdev, bool inner_ft_support, u32 drop_rqn, - const struct mlx5e_packet_merge_param *init_pkt_merge_param, - enum mlx5e_rss_init_type type, unsigned int nch, - unsigned int max_nch); +struct mlx5e_rss * +mlx5e_rss_init(struct mlx5_core_dev *mdev, + const struct mlx5e_rss_params *params, + const struct mlx5e_rss_init_params *init_params); int mlx5e_rss_cleanup(struct mlx5e_rss *rss); void mlx5e_rss_refcnt_inc(struct mlx5e_rss *rss); @@ -37,10 +49,10 @@ u32 mlx5e_rss_get_tirn(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, bool inner); bool mlx5e_rss_valid_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, bool inner); u32 mlx5e_rss_get_rqtn(struct mlx5e_rss *rss); -int mlx5e_rss_obtain_tirn(struct mlx5e_rss *rss, - enum mlx5_traffic_types tt, - const struct mlx5e_packet_merge_param *init_pkt_merge_param, - bool inner, u32 *tirn); +int +mlx5e_rss_obtain_tirn(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, + const struct mlx5e_packet_merge_param *pkt_merge_param, + bool inner, u32 *tirn); void mlx5e_rss_enable(struct mlx5e_rss *rss, u32 *rqns, u32 *vhca_ids, unsigned int num_rqns); void mlx5e_rss_disable(struct mlx5e_rss *rss); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c index a2acbfee2b77..ac26a32845d0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c @@ -54,17 +54,30 @@ static int mlx5e_rx_res_rss_init_def(struct mlx5e_rx_res *res, unsigned int init_nch) { bool inner_ft_support = res->features & MLX5E_RX_RES_FEATURE_INNER_FT; + struct mlx5e_rss_init_params init_params; + struct mlx5e_rss_params rss_params; struct mlx5e_rss *rss; if (WARN_ON(res->rss[0])) return -EINVAL; - rss = mlx5e_rss_init(res->mdev, inner_ft_support, res->drop_rqn, - &res->pkt_merge_param, MLX5E_RSS_INIT_TIRS, init_nch, res->max_nch); + init_params = (struct mlx5e_rss_init_params) { + .type = MLX5E_RSS_INIT_TIRS, + .pkt_merge_param = &res->pkt_merge_param, + .nch = init_nch, + .max_nch = res->max_nch, + }; + + rss_params = (struct mlx5e_rss_params) { + .inner_ft_support = inner_ft_support, + .drop_rqn = res->drop_rqn, + }; + + rss = mlx5e_rss_init(res->mdev, &rss_params, &init_params); if (IS_ERR(rss)) return PTR_ERR(rss); - mlx5e_rss_set_indir_uniform(rss, init_nch); + mlx5e_rss_set_indir_uniform(rss, init_params.nch); res->rss[0] = rss; @@ -74,18 +87,30 @@ static int mlx5e_rx_res_rss_init_def(struct mlx5e_rx_res *res, int mlx5e_rx_res_rss_init(struct mlx5e_rx_res *res, u32 rss_idx, unsigned int init_nch) { bool inner_ft_support = res->features & MLX5E_RX_RES_FEATURE_INNER_FT; + struct mlx5e_rss_init_params init_params; + struct mlx5e_rss_params rss_params; struct mlx5e_rss *rss; if (WARN_ON_ONCE(res->rss[rss_idx])) return -ENOSPC; - rss = mlx5e_rss_init(res->mdev, inner_ft_support, res->drop_rqn, - &res->pkt_merge_param, MLX5E_RSS_INIT_NO_TIRS, init_nch, - res->max_nch); + init_params = (struct mlx5e_rss_init_params) { + .type = MLX5E_RSS_INIT_NO_TIRS, + .pkt_merge_param = &res->pkt_merge_param, + .nch = init_nch, + .max_nch = res->max_nch, + }; + + rss_params = (struct mlx5e_rss_params) { + .inner_ft_support = inner_ft_support, + .drop_rqn = res->drop_rqn, + }; + + rss = mlx5e_rss_init(res->mdev, &rss_params, &init_params); if (IS_ERR(rss)) return PTR_ERR(rss); - mlx5e_rss_set_indir_uniform(rss, init_nch); + mlx5e_rss_set_indir_uniform(rss, init_params.nch); if (res->rss_active) { u32 *vhca_ids = get_vhca_ids(res, 0); @@ -438,7 +463,7 @@ static void mlx5e_rx_res_ptp_destroy(struct mlx5e_rx_res *res) struct mlx5e_rx_res * mlx5e_rx_res_create(struct mlx5_core_dev *mdev, enum mlx5e_rx_res_features features, unsigned int max_nch, u32 drop_rqn, - const struct mlx5e_packet_merge_param *init_pkt_merge_param, + const struct mlx5e_packet_merge_param *pkt_merge_param, unsigned int init_nch) { bool multi_vhca = features & MLX5E_RX_RES_FEATURE_MULTI_VHCA; @@ -454,7 +479,7 @@ mlx5e_rx_res_create(struct mlx5_core_dev *mdev, enum mlx5e_rx_res_features featu res->max_nch = max_nch; res->drop_rqn = drop_rqn; - res->pkt_merge_param = *init_pkt_merge_param; + res->pkt_merge_param = *pkt_merge_param; init_rwsem(&res->pkt_merge_param_sem); err = mlx5e_rx_res_rss_init_def(res, init_nch); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h index 1d049e2aa264..65a857c215e1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h @@ -27,7 +27,7 @@ enum mlx5e_rx_res_features { struct mlx5e_rx_res * mlx5e_rx_res_create(struct mlx5_core_dev *mdev, enum mlx5e_rx_res_features features, unsigned int max_nch, u32 drop_rqn, - const struct mlx5e_packet_merge_param *init_pkt_merge_param, + const struct mlx5e_packet_merge_param *pkt_merge_param, unsigned int init_nch); void mlx5e_rx_res_destroy(struct mlx5e_rx_res *res); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index fd45384a855b..53e5ae252eac 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -1494,7 +1494,8 @@ static int mlx5e_get_rxfh(struct net_device *netdev, struct ethtool_rxfh_param * } static int mlx5e_rxfh_hfunc_check(struct mlx5e_priv *priv, - const struct ethtool_rxfh_param *rxfh) + const struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { unsigned int count; @@ -1504,8 +1505,10 @@ static int mlx5e_rxfh_hfunc_check(struct mlx5e_priv *priv, unsigned int xor8_max_channels = mlx5e_rqt_max_num_channels_allowed_for_xor8(); if (count > xor8_max_channels) { - netdev_err(priv->netdev, "%s: Cannot set RSS hash function to XOR, current number of channels (%d) exceeds the maximum allowed for XOR8 RSS hfunc (%d)\n", - __func__, count, xor8_max_channels); + NL_SET_ERR_MSG_FMT_MOD( + extack, + "Number of channels (%u) exceeds the max for XOR8 RSS (%u)", + count, xor8_max_channels); return -EINVAL; } } @@ -1524,7 +1527,7 @@ static int mlx5e_set_rxfh(struct net_device *dev, mutex_lock(&priv->state_lock); - err = mlx5e_rxfh_hfunc_check(priv, rxfh); + err = mlx5e_rxfh_hfunc_check(priv, rxfh, extack); if (err) goto unlock; @@ -1550,7 +1553,7 @@ static int mlx5e_create_rxfh_context(struct net_device *dev, mutex_lock(&priv->state_lock); - err = mlx5e_rxfh_hfunc_check(priv, rxfh); + err = mlx5e_rxfh_hfunc_check(priv, rxfh, extack); if (err) goto unlock; @@ -1590,7 +1593,7 @@ static int mlx5e_modify_rxfh_context(struct net_device *dev, mutex_lock(&priv->state_lock); - err = mlx5e_rxfh_hfunc_check(priv, rxfh); + err = mlx5e_rxfh_hfunc_check(priv, rxfh, extack); if (err) goto unlock; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index b6d6584fc6fe..00c2763e57ca 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -758,11 +758,11 @@ static int mlx5e_hairpin_create_indirect_rqt(struct mlx5e_hairpin *hp) struct mlx5e_priv *priv = hp->func_priv; struct mlx5_core_dev *mdev = priv->mdev; struct mlx5e_rss_params_indir indir; + u32 rqt_size; int err; - err = mlx5e_rss_params_indir_init(&indir, mdev, - mlx5e_rqt_size(mdev, hp->num_channels), - mlx5e_rqt_size(mdev, hp->num_channels)); + rqt_size = mlx5e_rqt_size(mdev, hp->num_channels); + err = mlx5e_rss_params_indir_init(&indir, rqt_size, rqt_size); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c index 5f2d6c35f1ad..56e6f54b1e2e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c @@ -971,8 +971,9 @@ esw_qos_vport_tc_enable(struct mlx5_vport *vport, enum sched_node_type type, max_level = 1 << MLX5_CAP_QOS(vport_node->esw->dev, log_esw_max_sched_depth); if (new_level > max_level) { - NL_SET_ERR_MSG_MOD(extack, - "TC arbitration on leafs is not supported beyond max scheduling depth"); + NL_SET_ERR_MSG_FMT_MOD(extack, + "TC arbitration on leafs is not supported beyond max depth %d", + max_level); return -EOPNOTSUPP; } } @@ -1444,8 +1445,9 @@ static int esw_qos_node_enable_tc_arbitration(struct mlx5_esw_sched_node *node, new_level = node->level + 1; max_level = 1 << MLX5_CAP_QOS(node->esw->dev, log_esw_max_sched_depth); if (new_level > max_level) { - NL_SET_ERR_MSG_MOD(extack, - "TC arbitration on nodes is not supported beyond max scheduling depth"); + NL_SET_ERR_MSG_FMT_MOD(extack, + "TC arbitration on nodes is not supported beyond max depth %d", + max_level); return -EOPNOTSUPP; } @@ -1997,8 +1999,9 @@ mlx5_esw_qos_node_validate_set_parent(struct mlx5_esw_sched_node *node, max_level = 1 << MLX5_CAP_QOS(node->esw->dev, log_esw_max_sched_depth); if (new_level > max_level) { - NL_SET_ERR_MSG_MOD(extack, - "Node hierarchy depth exceeds the maximum supported level"); + NL_SET_ERR_MSG_FMT_MOD(extack, + "Node hierarchy depth %d exceeds the maximum supported level %d", + new_level, max_level); return -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index b8ec55929ab1..52c3de24bea3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -3774,6 +3774,29 @@ void mlx5_eswitch_unblock_mode(struct mlx5_core_dev *dev) up_write(&esw->mode_lock); } +/* Returns false only when uplink netdev exists and its netns is different from + * devlink's netns. True for all others so entering switchdev mode is allowed. + */ +static bool mlx5_devlink_netdev_netns_immutable_set(struct devlink *devlink, + bool immutable) +{ + struct mlx5_core_dev *mdev = devlink_priv(devlink); + struct net_device *netdev; + bool ret; + + netdev = mlx5_uplink_netdev_get(mdev); + if (!netdev) + return true; + + rtnl_lock(); + netdev->netns_immutable = immutable; + ret = net_eq(dev_net(netdev), devlink_net(devlink)); + rtnl_unlock(); + + mlx5_uplink_netdev_put(mdev, netdev); + return ret; +} + int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, struct netlink_ext_ack *extack) { @@ -3816,6 +3839,14 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, esw->eswitch_operation_in_progress = true; up_write(&esw->mode_lock); + if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && + !mlx5_devlink_netdev_netns_immutable_set(devlink, true)) { + NL_SET_ERR_MSG_MOD(extack, + "Can't change E-Switch mode to switchdev when netdev net namespace has diverged from the devlink's."); + err = -EINVAL; + goto skip; + } + if (mode == DEVLINK_ESWITCH_MODE_LEGACY) esw->dev->priv.flags |= MLX5_PRIV_FLAGS_SWITCH_LEGACY; mlx5_eswitch_disable_locked(esw); @@ -3834,6 +3865,8 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, } skip: + if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && err) + mlx5_devlink_netdev_netns_immutable_set(devlink, false); down_write(&esw->mode_lock); esw->eswitch_operation_in_progress = false; unlock: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c index adeccc588e5d..6ef0c4be27e1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c @@ -51,9 +51,6 @@ static void hws_bwc_matcher_init_attr(struct mlx5hws_bwc_matcher *bwc_matcher, u8 size_log_rx, u8 size_log_tx, struct mlx5hws_matcher_attr *attr) { - struct mlx5hws_bwc_matcher *first_matcher = - bwc_matcher->complex_first_bwc_matcher; - memset(attr, 0, sizeof(*attr)); attr->priority = priority; @@ -66,9 +63,6 @@ static void hws_bwc_matcher_init_attr(struct mlx5hws_bwc_matcher *bwc_matcher, attr->size[MLX5HWS_MATCHER_SIZE_TYPE_TX].rule.num_log = size_log_tx; attr->resizable = true; attr->max_num_of_at_attach = MLX5HWS_BWC_MATCHER_ATTACH_AT_NUM; - - attr->isolated_matcher_end_ft_id = - first_matcher ? first_matcher->matcher->end_ft_id : 0; } static int @@ -171,10 +165,16 @@ hws_bwc_matcher_move_all_simple(struct mlx5hws_bwc_matcher *bwc_matcher) static int hws_bwc_matcher_move_all(struct mlx5hws_bwc_matcher *bwc_matcher) { - if (!bwc_matcher->complex) + switch (bwc_matcher->matcher_type) { + case MLX5HWS_BWC_MATCHER_SIMPLE: return hws_bwc_matcher_move_all_simple(bwc_matcher); - - return mlx5hws_bwc_matcher_move_all_complex(bwc_matcher); + case MLX5HWS_BWC_MATCHER_COMPLEX_FIRST: + return mlx5hws_bwc_matcher_complex_move_first(bwc_matcher); + case MLX5HWS_BWC_MATCHER_COMPLEX_SUBMATCHER: + return mlx5hws_bwc_matcher_complex_move(bwc_matcher); + default: + return -EINVAL; + } } static int hws_bwc_matcher_move(struct mlx5hws_bwc_matcher *bwc_matcher) @@ -249,6 +249,7 @@ int mlx5hws_bwc_matcher_create_simple(struct mlx5hws_bwc_matcher *bwc_matcher, bwc_matcher->tx_size.size_log, &attr); + bwc_matcher->matcher_type = MLX5HWS_BWC_MATCHER_SIMPLE; bwc_matcher->priority = priority; bwc_matcher->size_of_at_array = MLX5HWS_BWC_MATCHER_ATTACH_AT_NUM; @@ -393,7 +394,7 @@ int mlx5hws_bwc_matcher_destroy(struct mlx5hws_bwc_matcher *bwc_matcher) "BWC matcher destroy: matcher still has %u RX and %u TX rules\n", rx_rules, tx_rules); - if (bwc_matcher->complex) + if (bwc_matcher->matcher_type == MLX5HWS_BWC_MATCHER_COMPLEX_FIRST) mlx5hws_bwc_matcher_destroy_complex(bwc_matcher); else mlx5hws_bwc_matcher_destroy_simple(bwc_matcher); @@ -651,7 +652,8 @@ int mlx5hws_bwc_rule_destroy_simple(struct mlx5hws_bwc_rule *bwc_rule) int mlx5hws_bwc_rule_destroy(struct mlx5hws_bwc_rule *bwc_rule) { - bool is_complex = !!bwc_rule->bwc_matcher->complex; + bool is_complex = bwc_rule->bwc_matcher->matcher_type == + MLX5HWS_BWC_MATCHER_COMPLEX_FIRST; int ret = 0; if (is_complex) @@ -1147,7 +1149,7 @@ mlx5hws_bwc_rule_create(struct mlx5hws_bwc_matcher *bwc_matcher, bwc_queue_idx = hws_bwc_gen_queue_idx(ctx); - if (bwc_matcher->complex) + if (bwc_matcher->matcher_type == MLX5HWS_BWC_MATCHER_COMPLEX_FIRST) ret = mlx5hws_bwc_rule_create_complex(bwc_rule, params, flow_source, @@ -1216,10 +1218,9 @@ int mlx5hws_bwc_rule_action_update(struct mlx5hws_bwc_rule *bwc_rule, return -EINVAL; } - /* For complex rule, the update should happen on the second matcher */ - if (bwc_rule->isolated_bwc_rule) - return hws_bwc_rule_action_update(bwc_rule->isolated_bwc_rule, - rule_actions); - else - return hws_bwc_rule_action_update(bwc_rule, rule_actions); + /* For complex rules, the update should happen on the last subrule. */ + while (bwc_rule->next_subrule) + bwc_rule = bwc_rule->next_subrule; + + return hws_bwc_rule_action_update(bwc_rule, rule_actions); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.h index af391d70c14f..b905511f5c53 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.h @@ -18,6 +18,21 @@ #define MLX5HWS_BWC_POLLING_TIMEOUT 60 +enum mlx5hws_bwc_matcher_type { + /* Standalone bwc matcher. */ + MLX5HWS_BWC_MATCHER_SIMPLE, + /* The first matcher of a complex matcher. When rules are inserted into + * a matcher of this type, they are split into subrules and inserted + * into their corresponding submatchers. + */ + MLX5HWS_BWC_MATCHER_COMPLEX_FIRST, + /* A submatcher that is part of a complex matcher. For most purposes + * these are treated as simple matchers, except when it comes to moving + * rules during resize. + */ + MLX5HWS_BWC_MATCHER_COMPLEX_SUBMATCHER, +}; + struct mlx5hws_bwc_matcher_complex_data; struct mlx5hws_bwc_matcher_size { @@ -31,9 +46,9 @@ struct mlx5hws_bwc_matcher { struct mlx5hws_match_template *mt; struct mlx5hws_action_template **at; struct mlx5hws_bwc_matcher_complex_data *complex; - struct mlx5hws_bwc_matcher *complex_first_bwc_matcher; u8 num_of_at; u8 size_of_at_array; + enum mlx5hws_bwc_matcher_type matcher_type; u32 priority; struct mlx5hws_bwc_matcher_size rx_size; struct mlx5hws_bwc_matcher_size tx_size; @@ -43,8 +58,8 @@ struct mlx5hws_bwc_matcher { struct mlx5hws_bwc_rule { struct mlx5hws_bwc_matcher *bwc_matcher; struct mlx5hws_rule *rule; - struct mlx5hws_bwc_rule *isolated_bwc_rule; - struct mlx5hws_bwc_complex_rule_hash_node *complex_hash_node; + struct mlx5hws_bwc_rule *next_subrule; + struct mlx5hws_bwc_complex_subrule_data *subrule_data; u32 flow_source; u16 bwc_queue_idx; bool skip_rx; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc_complex.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc_complex.c index 14e79579c719..660630f18ce9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc_complex.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc_complex.c @@ -3,25 +3,27 @@ #include "internal.h" -#define HWS_CLEAR_MATCH_PARAM(mask, field) \ - MLX5_SET(fte_match_param, (mask)->match_buf, field, 0) - -#define HWS_SZ_MATCH_PARAM (MLX5_ST_SZ_DW_MATCH_PARAM * 4) - -static const struct rhashtable_params hws_refcount_hash = { - .key_len = sizeof_field(struct mlx5hws_bwc_complex_rule_hash_node, - match_buf), - .key_offset = offsetof(struct mlx5hws_bwc_complex_rule_hash_node, - match_buf), - .head_offset = offsetof(struct mlx5hws_bwc_complex_rule_hash_node, - hash_node), - .automatic_shrinking = true, - .min_size = 1, +/* We chain submatchers by applying three rules on a subrule: modify header (to + * set register C6), jump to table (to the next submatcher) and the mandatory + * last rule. + */ +#define HWS_NUM_CHAIN_ACTIONS 3 + +static const struct rhashtable_params hws_rules_hash_params = { + .key_len = sizeof_field(struct mlx5hws_bwc_complex_subrule_data, + match_tag), + .key_offset = + offsetof(struct mlx5hws_bwc_complex_subrule_data, match_tag), + .head_offset = + offsetof(struct mlx5hws_bwc_complex_subrule_data, hash_node), + .automatic_shrinking = true, .min_size = 1, }; -bool mlx5hws_bwc_match_params_is_complex(struct mlx5hws_context *ctx, - u8 match_criteria_enable, - struct mlx5hws_match_parameters *mask) +static bool +hws_match_params_exceeds_definer(struct mlx5hws_context *ctx, + u8 match_criteria_enable, + struct mlx5hws_match_parameters *mask, + bool allow_jumbo) { struct mlx5hws_definer match_layout = {0}; struct mlx5hws_match_template *mt; @@ -36,11 +38,11 @@ bool mlx5hws_bwc_match_params_is_complex(struct mlx5hws_context *ctx, mask->match_sz, match_criteria_enable); if (!mt) { - mlx5hws_err(ctx, "BWC: failed creating match template\n"); + mlx5hws_err(ctx, "Complex matcher: failed creating match template\n"); return false; } - ret = mlx5hws_definer_calc_layout(ctx, mt, &match_layout); + ret = mlx5hws_definer_calc_layout(ctx, mt, &match_layout, allow_jumbo); if (ret) { /* The only case that we're interested in is E2BIG, * which means that the match parameters need to be @@ -64,825 +66,481 @@ bool mlx5hws_bwc_match_params_is_complex(struct mlx5hws_context *ctx, return is_complex; } -static void -hws_bwc_matcher_complex_params_clear_fld(struct mlx5hws_context *ctx, - enum mlx5hws_definer_fname fname, +bool mlx5hws_bwc_match_params_is_complex(struct mlx5hws_context *ctx, + u8 match_criteria_enable, struct mlx5hws_match_parameters *mask) { - struct mlx5hws_cmd_query_caps *caps = ctx->caps; - - switch (fname) { - case MLX5HWS_DEFINER_FNAME_ETH_TYPE_O: - case MLX5HWS_DEFINER_FNAME_ETH_TYPE_I: - case MLX5HWS_DEFINER_FNAME_ETH_L3_TYPE_O: - case MLX5HWS_DEFINER_FNAME_ETH_L3_TYPE_I: - case MLX5HWS_DEFINER_FNAME_IP_VERSION_O: - case MLX5HWS_DEFINER_FNAME_IP_VERSION_I: - /* Because of the strict requirements for IP address matching - * that require ethtype/ip_version matching as well, don't clear - * these fields - have them in both parts of the complex matcher - */ - break; - case MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.smac_47_16); - break; - case MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.smac_47_16); - break; - case MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.smac_15_0); - break; - case MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.smac_15_0); - break; - case MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.dmac_47_16); - break; - case MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.dmac_47_16); - break; - case MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.dmac_15_0); - break; - case MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.dmac_15_0); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_TYPE_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.cvlan_tag); - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.svlan_tag); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_TYPE_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.cvlan_tag); - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.svlan_tag); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_FIRST_PRIO_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.first_prio); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_FIRST_PRIO_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.first_prio); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_CFI_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.first_cfi); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_CFI_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.first_cfi); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_ID_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.first_vid); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_ID_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.first_vid); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_TYPE_O: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters.outer_second_cvlan_tag); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters.outer_second_svlan_tag); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_TYPE_I: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters.inner_second_cvlan_tag); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters.inner_second_svlan_tag); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_PRIO_O: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.outer_second_prio); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_PRIO_I: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.inner_second_prio); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_CFI_O: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.outer_second_cfi); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_CFI_I: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.inner_second_cfi); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_ID_O: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.outer_second_vid); - break; - case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_ID_I: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.inner_second_vid); - break; - case MLX5HWS_DEFINER_FNAME_IPV4_IHL_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.ipv4_ihl); - break; - case MLX5HWS_DEFINER_FNAME_IPV4_IHL_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.ipv4_ihl); - break; - case MLX5HWS_DEFINER_FNAME_IP_DSCP_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.ip_dscp); - break; - case MLX5HWS_DEFINER_FNAME_IP_DSCP_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.ip_dscp); - break; - case MLX5HWS_DEFINER_FNAME_IP_ECN_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.ip_ecn); - break; - case MLX5HWS_DEFINER_FNAME_IP_ECN_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.ip_ecn); - break; - case MLX5HWS_DEFINER_FNAME_IP_TTL_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.ttl_hoplimit); - break; - case MLX5HWS_DEFINER_FNAME_IP_TTL_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.ttl_hoplimit); - break; - case MLX5HWS_DEFINER_FNAME_IPV4_DST_O: - HWS_CLEAR_MATCH_PARAM(mask, - outer_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_31_0); - break; - case MLX5HWS_DEFINER_FNAME_IPV4_SRC_O: - HWS_CLEAR_MATCH_PARAM(mask, - outer_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_31_0); - break; - case MLX5HWS_DEFINER_FNAME_IPV4_DST_I: - HWS_CLEAR_MATCH_PARAM(mask, - inner_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_31_0); - break; - case MLX5HWS_DEFINER_FNAME_IPV4_SRC_I: - HWS_CLEAR_MATCH_PARAM(mask, - inner_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_31_0); - break; - case MLX5HWS_DEFINER_FNAME_IP_FRAG_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.frag); - break; - case MLX5HWS_DEFINER_FNAME_IP_FRAG_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.frag); - break; - case MLX5HWS_DEFINER_FNAME_IPV6_FLOW_LABEL_O: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters.outer_ipv6_flow_label); - break; - case MLX5HWS_DEFINER_FNAME_IPV6_FLOW_LABEL_I: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters.inner_ipv6_flow_label); - break; - case MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_O: - case MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_O: - case MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_O: - case MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_O: - HWS_CLEAR_MATCH_PARAM(mask, - outer_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_127_96); - HWS_CLEAR_MATCH_PARAM(mask, - outer_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_95_64); - HWS_CLEAR_MATCH_PARAM(mask, - outer_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_63_32); - HWS_CLEAR_MATCH_PARAM(mask, - outer_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_31_0); - break; - case MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_O: - case MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_O: - case MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_O: - case MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_O: - HWS_CLEAR_MATCH_PARAM(mask, - outer_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_127_96); - HWS_CLEAR_MATCH_PARAM(mask, - outer_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_95_64); - HWS_CLEAR_MATCH_PARAM(mask, - outer_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_63_32); - HWS_CLEAR_MATCH_PARAM(mask, - outer_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_31_0); - break; - case MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_I: - case MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_I: - case MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_I: - case MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_I: - HWS_CLEAR_MATCH_PARAM(mask, - inner_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_127_96); - HWS_CLEAR_MATCH_PARAM(mask, - inner_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_95_64); - HWS_CLEAR_MATCH_PARAM(mask, - inner_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_63_32); - HWS_CLEAR_MATCH_PARAM(mask, - inner_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_31_0); - break; - case MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_I: - case MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_I: - case MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_I: - case MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_I: - HWS_CLEAR_MATCH_PARAM(mask, - inner_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_127_96); - HWS_CLEAR_MATCH_PARAM(mask, - inner_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_95_64); - HWS_CLEAR_MATCH_PARAM(mask, - inner_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_63_32); - HWS_CLEAR_MATCH_PARAM(mask, - inner_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_31_0); - break; - case MLX5HWS_DEFINER_FNAME_IP_PROTOCOL_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.ip_protocol); - break; - case MLX5HWS_DEFINER_FNAME_IP_PROTOCOL_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.ip_protocol); - break; - case MLX5HWS_DEFINER_FNAME_L4_SPORT_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.tcp_sport); - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.udp_sport); - break; - case MLX5HWS_DEFINER_FNAME_L4_SPORT_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.tcp_dport); - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.udp_dport); - break; - case MLX5HWS_DEFINER_FNAME_L4_DPORT_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.tcp_dport); - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.udp_dport); - break; - case MLX5HWS_DEFINER_FNAME_L4_DPORT_I: - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.tcp_dport); - HWS_CLEAR_MATCH_PARAM(mask, inner_headers.udp_dport); - break; - case MLX5HWS_DEFINER_FNAME_TCP_FLAGS_O: - HWS_CLEAR_MATCH_PARAM(mask, outer_headers.tcp_flags); - break; - case MLX5HWS_DEFINER_FNAME_TCP_ACK_NUM: - case MLX5HWS_DEFINER_FNAME_TCP_SEQ_NUM: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_3.outer_tcp_seq_num); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_3.outer_tcp_ack_num); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_3.inner_tcp_seq_num); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_3.inner_tcp_ack_num); - break; - case MLX5HWS_DEFINER_FNAME_GTP_TEID: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.gtpu_teid); - break; - case MLX5HWS_DEFINER_FNAME_GTP_MSG_TYPE: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.gtpu_msg_type); - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.gtpu_msg_flags); - break; - case MLX5HWS_DEFINER_FNAME_GTPU_FIRST_EXT_DW0: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_3.gtpu_first_ext_dw_0); - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.gtpu_dw_0); - break; - case MLX5HWS_DEFINER_FNAME_GTPU_DW2: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.gtpu_dw_2); - break; - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_0: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_1: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_2: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_3: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_4: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_5: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_6: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_7: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_2.outer_first_mpls_over_gre); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_2.outer_first_mpls_over_udp); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_3.geneve_tlv_option_0_data); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_4.prog_sample_field_id_0); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_4.prog_sample_field_value_0); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_4.prog_sample_field_value_1); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_4.prog_sample_field_id_2); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_4.prog_sample_field_value_2); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_4.prog_sample_field_id_3); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_4.prog_sample_field_value_3); - break; - case MLX5HWS_DEFINER_FNAME_VXLAN_VNI: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.vxlan_vni); - break; - case MLX5HWS_DEFINER_FNAME_VXLAN_GPE_FLAGS: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_3.outer_vxlan_gpe_flags); - break; - case MLX5HWS_DEFINER_FNAME_VXLAN_GPE_RSVD0: - break; - case MLX5HWS_DEFINER_FNAME_VXLAN_GPE_PROTO: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_3.outer_vxlan_gpe_next_protocol); - break; - case MLX5HWS_DEFINER_FNAME_VXLAN_GPE_VNI: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_3.outer_vxlan_gpe_vni); - break; - case MLX5HWS_DEFINER_FNAME_GENEVE_OPT_LEN: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.geneve_opt_len); - break; - case MLX5HWS_DEFINER_FNAME_GENEVE_OAM: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.geneve_oam); - break; - case MLX5HWS_DEFINER_FNAME_GENEVE_PROTO: - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters.geneve_protocol_type); - break; - case MLX5HWS_DEFINER_FNAME_GENEVE_VNI: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.geneve_vni); - break; - case MLX5HWS_DEFINER_FNAME_SOURCE_QP: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.source_sqn); - break; - case MLX5HWS_DEFINER_FNAME_SOURCE_GVMI: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.source_port); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters.source_eswitch_owner_vhca_id); - break; - case MLX5HWS_DEFINER_FNAME_REG_0: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_0); - break; - case MLX5HWS_DEFINER_FNAME_REG_1: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_1); - break; - case MLX5HWS_DEFINER_FNAME_REG_2: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_2); - break; - case MLX5HWS_DEFINER_FNAME_REG_3: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_3); - break; - case MLX5HWS_DEFINER_FNAME_REG_4: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_4); - break; - case MLX5HWS_DEFINER_FNAME_REG_5: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_5); - break; - case MLX5HWS_DEFINER_FNAME_REG_7: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_7); - break; - case MLX5HWS_DEFINER_FNAME_REG_A: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_a); - break; - case MLX5HWS_DEFINER_FNAME_GRE_C: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.gre_c_present); - break; - case MLX5HWS_DEFINER_FNAME_GRE_K: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.gre_k_present); - break; - case MLX5HWS_DEFINER_FNAME_GRE_S: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.gre_s_present); - break; - case MLX5HWS_DEFINER_FNAME_GRE_PROTOCOL: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.gre_protocol); - break; - case MLX5HWS_DEFINER_FNAME_GRE_OPT_KEY: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.gre_key.key); - break; - case MLX5HWS_DEFINER_FNAME_ICMP_DW1: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.icmp_header_data); - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.icmp_type); - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.icmp_code); - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters_3.icmpv6_header_data); - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.icmpv6_type); - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.icmpv6_code); - break; - case MLX5HWS_DEFINER_FNAME_MPLS0_O: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.outer_first_mpls); - break; - case MLX5HWS_DEFINER_FNAME_MPLS0_I: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.inner_first_mpls); - break; - case MLX5HWS_DEFINER_FNAME_TNL_HDR_0: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_5.tunnel_header_0); - break; - case MLX5HWS_DEFINER_FNAME_TNL_HDR_1: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_5.tunnel_header_1); - break; - case MLX5HWS_DEFINER_FNAME_TNL_HDR_2: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_5.tunnel_header_2); - break; - case MLX5HWS_DEFINER_FNAME_TNL_HDR_3: - HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_5.tunnel_header_3); - break; - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER0_OK: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER1_OK: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER2_OK: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER3_OK: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER4_OK: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER5_OK: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER6_OK: - case MLX5HWS_DEFINER_FNAME_FLEX_PARSER7_OK: - /* assuming this is flex parser for geneve option */ - if ((fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER0_OK && - ctx->caps->flex_parser_id_geneve_tlv_option_0 != 0) || - (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER1_OK && - ctx->caps->flex_parser_id_geneve_tlv_option_0 != 1) || - (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER2_OK && - ctx->caps->flex_parser_id_geneve_tlv_option_0 != 2) || - (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER3_OK && - ctx->caps->flex_parser_id_geneve_tlv_option_0 != 3) || - (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER4_OK && - ctx->caps->flex_parser_id_geneve_tlv_option_0 != 4) || - (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER5_OK && - ctx->caps->flex_parser_id_geneve_tlv_option_0 != 5) || - (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER6_OK && - ctx->caps->flex_parser_id_geneve_tlv_option_0 != 6) || - (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER7_OK && - ctx->caps->flex_parser_id_geneve_tlv_option_0 != 7)) { - mlx5hws_err(ctx, - "Complex params: unsupported field %s (%d), flex parser ID for geneve is %d\n", - mlx5hws_definer_fname_to_str(fname), fname, - caps->flex_parser_id_geneve_tlv_option_0); - break; - } - HWS_CLEAR_MATCH_PARAM(mask, - misc_parameters.geneve_tlv_option_0_exist); - break; - case MLX5HWS_DEFINER_FNAME_REG_6: - default: - mlx5hws_err(ctx, "Complex params: unsupported field %s (%d)\n", - mlx5hws_definer_fname_to_str(fname), fname); - break; - } + return hws_match_params_exceeds_definer(ctx, match_criteria_enable, + mask, true); } -static bool -hws_bwc_matcher_complex_params_comb_is_valid(struct mlx5hws_definer_fc *fc, - int fc_sz, - u32 combination_num) +static int +hws_get_last_set_dword_idx(const struct mlx5hws_match_parameters *mask) { - bool m1[MLX5HWS_DEFINER_FNAME_MAX] = {0}; - bool m2[MLX5HWS_DEFINER_FNAME_MAX] = {0}; - bool is_first_matcher; int i; - for (i = 0; i < fc_sz; i++) { - is_first_matcher = !(combination_num & BIT(i)); - if (is_first_matcher) - m1[fc[i].fname] = true; - else - m2[fc[i].fname] = true; - } - - /* Not all the fields can be split into separate matchers. - * Some should be together on the same matcher. - * For example, IPv6 parts - the whole IPv6 address should be on the - * same matcher in order for us to deduce if it's IPv6 or IPv4 address. - */ - if (m1[MLX5HWS_DEFINER_FNAME_IP_FRAG_O] && - (m2[MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_O] || - m2[MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_O] || - m2[MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_O] || - m2[MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_O])) - return false; - - if (m2[MLX5HWS_DEFINER_FNAME_IP_FRAG_O] && - (m1[MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_O] || - m1[MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_O] || - m1[MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_O] || - m1[MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_O])) - return false; + for (i = mask->match_sz / 4 - 1; i >= 0; i--) + if (mask->match_buf[i]) + return i; - if (m1[MLX5HWS_DEFINER_FNAME_IP_FRAG_I] && - (m2[MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_I] || - m2[MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_I] || - m2[MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_I] || - m2[MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_I])) - return false; + return -1; +} - if (m2[MLX5HWS_DEFINER_FNAME_IP_FRAG_I] && - (m1[MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_I] || - m1[MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_I] || - m1[MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_I] || - m1[MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_I])) - return false; +static bool hws_match_mask_is_empty(const struct mlx5hws_match_parameters *mask) +{ + return hws_get_last_set_dword_idx(mask) == -1; +} - /* Don't split outer IPv6 dest address. */ - if ((m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_O] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_O] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_O] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_O]) && - (m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_O] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_O] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_O] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_O])) - return false; +static bool hws_dword_is_inner_ipaddr_off(int dword_off) +{ + /* IPv4 and IPv6 addresses share the same entry via a union, and the + * source and dest addresses are contiguous in the fte_match_param. So + * we need to check 8 words. + */ + static const int inner_ip_dword_off = + __mlx5_dw_off(fte_match_param, inner_headers.src_ipv4_src_ipv6); - /* Don't split outer IPv6 source address. */ - if ((m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_O] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_O] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_O] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_O]) && - (m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_O] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_O] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_O] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_O])) - return false; + return dword_off >= inner_ip_dword_off && + dword_off < inner_ip_dword_off + 8; +} - /* Don't split inner IPv6 dest address. */ - if ((m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_I] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_I] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_I] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_I]) && - (m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_I] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_I] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_I] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_I])) - return false; +static bool hws_dword_is_outer_ipaddr_off(int dword_off) +{ + static const int outer_ip_dword_off = + __mlx5_dw_off(fte_match_param, outer_headers.src_ipv4_src_ipv6); - /* Don't split inner IPv6 source address. */ - if ((m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_I] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_I] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_I] || - m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_I]) && - (m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_I] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_I] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_I] || - m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_I])) - return false; + return dword_off >= outer_ip_dword_off && + dword_off < outer_ip_dword_off + 8; +} - /* Don't split GRE parameters. */ - if ((m1[MLX5HWS_DEFINER_FNAME_GRE_C] || - m1[MLX5HWS_DEFINER_FNAME_GRE_K] || - m1[MLX5HWS_DEFINER_FNAME_GRE_S] || - m1[MLX5HWS_DEFINER_FNAME_GRE_PROTOCOL]) && - (m2[MLX5HWS_DEFINER_FNAME_GRE_C] || - m2[MLX5HWS_DEFINER_FNAME_GRE_K] || - m2[MLX5HWS_DEFINER_FNAME_GRE_S] || - m2[MLX5HWS_DEFINER_FNAME_GRE_PROTOCOL])) - return false; +static void hws_add_dword_to_mask(struct mlx5hws_match_parameters *mask, + const struct mlx5hws_match_parameters *orig, + int dword_idx, bool *added_inner_ipv, + bool *added_outer_ipv) +{ + mask->match_buf[dword_idx] |= orig->match_buf[dword_idx]; - /* Don't split TCP ack/seq numbers. */ - if ((m1[MLX5HWS_DEFINER_FNAME_TCP_ACK_NUM] || - m1[MLX5HWS_DEFINER_FNAME_TCP_SEQ_NUM]) && - (m2[MLX5HWS_DEFINER_FNAME_TCP_ACK_NUM] || - m2[MLX5HWS_DEFINER_FNAME_TCP_SEQ_NUM])) - return false; + *added_inner_ipv = false; + *added_outer_ipv = false; - /* Don't split flex parser. */ - if ((m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_0] || - m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_1] || - m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_2] || - m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_3] || - m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_4] || - m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_5] || - m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_6] || - m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_7]) && - (m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_0] || - m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_1] || - m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_2] || - m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_3] || - m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_4] || - m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_5] || - m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_6] || - m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_7])) - return false; + /* Any IP address fragment must be accompanied by a match on IP version. + * Use the `added_ipv` variables to keep track if we added IP versions + * specifically for this dword, so that we can roll them back if the + * match params become too large to fit into a definer. + */ + if (hws_dword_is_inner_ipaddr_off(dword_idx) && + !MLX5_GET(fte_match_param, mask->match_buf, + inner_headers.ip_version)) { + MLX5_SET(fte_match_param, mask->match_buf, + inner_headers.ip_version, 0xf); + *added_inner_ipv = true; + } + if (hws_dword_is_outer_ipaddr_off(dword_idx) && + !MLX5_GET(fte_match_param, mask->match_buf, + outer_headers.ip_version)) { + MLX5_SET(fte_match_param, mask->match_buf, + outer_headers.ip_version, 0xf); + *added_outer_ipv = true; + } +} - return true; +static void hws_remove_dword_from_mask(struct mlx5hws_match_parameters *mask, + int dword_idx, bool added_inner_ipv, + bool added_outer_ipv) +{ + mask->match_buf[dword_idx] = 0; + if (added_inner_ipv) + MLX5_SET(fte_match_param, mask->match_buf, + inner_headers.ip_version, 0); + if (added_outer_ipv) + MLX5_SET(fte_match_param, mask->match_buf, + outer_headers.ip_version, 0); } -static void -hws_bwc_matcher_complex_params_comb_create(struct mlx5hws_context *ctx, - struct mlx5hws_match_parameters *m, - struct mlx5hws_match_parameters *m1, - struct mlx5hws_match_parameters *m2, - struct mlx5hws_definer_fc *fc, - int fc_sz, - u32 combination_num) +/* Avoid leaving a single lower dword in `mask` if there are others present in + * `orig`. Splitting IPv6 addresses like this causes them to be interpreted as + * IPv4. + */ +static void hws_avoid_ipv6_split_of(struct mlx5hws_match_parameters *orig, + struct mlx5hws_match_parameters *mask, + int off) { - bool is_first_matcher; - int i; + /* Masks are allocated to a full fte_match_param, but it can't hurt to + * double check. + */ + if (orig->match_sz <= off + 3 || mask->match_sz <= off + 3) + return; - memcpy(m1->match_buf, m->match_buf, m->match_sz); - memcpy(m2->match_buf, m->match_buf, m->match_sz); + /* Lower dword is not set, nothing to do. */ + if (!mask->match_buf[off + 3]) + return; - for (i = 0; i < fc_sz; i++) { - is_first_matcher = !(combination_num & BIT(i)); - hws_bwc_matcher_complex_params_clear_fld(ctx, - fc[i].fname, - is_first_matcher ? - m2 : m1); - } + /* Higher dwords also present in `mask`, no ambiguity. */ + if (mask->match_buf[off] || mask->match_buf[off + 1] || + mask->match_buf[off + 2]) + return; + + /* There are no higher dwords in `orig`, i.e. we match on IPv4. */ + if (!orig->match_buf[off] && !orig->match_buf[off + 1] && + !orig->match_buf[off + 2]) + return; - MLX5_SET(fte_match_param, m2->match_buf, - misc_parameters_2.metadata_reg_c_6, -1); + /* Put the lower dword back in `orig`. It is always safe to do this, the + * dword will just be picked up in the next submask. + */ + orig->match_buf[off + 3] = mask->match_buf[off + 3]; + mask->match_buf[off + 3] = 0; } -static void -hws_bwc_matcher_complex_params_destroy(struct mlx5hws_match_parameters *mask_1, - struct mlx5hws_match_parameters *mask_2) +static void hws_avoid_ipv6_split(struct mlx5hws_match_parameters *orig, + struct mlx5hws_match_parameters *mask) { - kfree(mask_1->match_buf); - kfree(mask_2->match_buf); + hws_avoid_ipv6_split_of(orig, mask, + __mlx5_dw_off(fte_match_param, + outer_headers.src_ipv4_src_ipv6)); + hws_avoid_ipv6_split_of(orig, mask, + __mlx5_dw_off(fte_match_param, + outer_headers.dst_ipv4_dst_ipv6)); + hws_avoid_ipv6_split_of(orig, mask, + __mlx5_dw_off(fte_match_param, + inner_headers.src_ipv4_src_ipv6)); + hws_avoid_ipv6_split_of(orig, mask, + __mlx5_dw_off(fte_match_param, + inner_headers.dst_ipv4_dst_ipv6)); } -static int -hws_bwc_matcher_complex_params_create(struct mlx5hws_context *ctx, - u8 match_criteria, - struct mlx5hws_match_parameters *mask, - struct mlx5hws_match_parameters *mask_1, - struct mlx5hws_match_parameters *mask_2) +/* Build a subset of the `orig` match parameters into `mask`. This subset is + * guaranteed to fit in a single definer an as such is a candidate for being a + * part of a complex matcher. Upon successful execution, the match params that + * go into `mask` are cleared from `orig`. + */ +static int hws_get_simple_params(struct mlx5hws_context *ctx, u8 match_criteria, + struct mlx5hws_match_parameters *orig, + struct mlx5hws_match_parameters *mask) { - struct mlx5hws_definer_fc *fc; - u32 num_of_combinations; - int fc_sz = 0; - int res = 0; - u32 i; - - if (MLX5_GET(fte_match_param, mask->match_buf, - misc_parameters_2.metadata_reg_c_6)) { - mlx5hws_err(ctx, "Complex matcher: REG_C_6 matching is reserved\n"); - res = -EINVAL; - goto out; - } + bool added_inner_ipv, added_outer_ipv; + int dword_idx; + u32 *backup; + int ret; - mask_1->match_buf = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), - GFP_KERNEL); - mask_2->match_buf = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), - GFP_KERNEL); - if (!mask_1->match_buf || !mask_2->match_buf) { - mlx5hws_err(ctx, "Complex matcher: failed to allocate match_param\n"); - res = -ENOMEM; - goto free_params; - } + dword_idx = hws_get_last_set_dword_idx(orig); + /* Nothing to do, we consumed all of the match params before. */ + if (dword_idx == -1) + return 0; - mask_1->match_sz = mask->match_sz; - mask_2->match_sz = mask->match_sz; + backup = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL); + if (!backup) + return -ENOMEM; - fc = mlx5hws_definer_conv_match_params_to_compressed_fc(ctx, - match_criteria, - mask->match_buf, - &fc_sz); - if (!fc) { - res = -ENOMEM; - goto free_params; - } + while (1) { + dword_idx = hws_get_last_set_dword_idx(orig); + /* Nothing to do, we consumed all of the original match params + * into this subset, which still fits into a single matcher. + */ + if (dword_idx == -1) { + ret = 0; + goto free_backup; + } - if (fc_sz >= sizeof(num_of_combinations) * BITS_PER_BYTE) { - mlx5hws_err(ctx, - "Complex matcher: too many match parameters (%d)\n", - fc_sz); - res = -EINVAL; - goto free_fc; + memcpy(backup, mask->match_buf, mask->match_sz); + + /* Try to add this dword to the current subset. */ + hws_add_dword_to_mask(mask, orig, dword_idx, &added_inner_ipv, + &added_outer_ipv); + + if (hws_match_params_exceeds_definer(ctx, match_criteria, mask, + false)) { + /* We just added a match param that makes the definer + * too large. Revert and return what we had before. + * Note that we can't just zero out the affected fields, + * because it's possible that the dword we're looking at + * wasn't zero before (e.g. it included auto-added + * matches in IP version. This is why we employ the + * rather cumbersome memcpy for backing up. + */ + memcpy(mask->match_buf, backup, mask->match_sz); + /* Possible future improvement: We can't add any more + * dwords, but it may be possible to squeeze in + * individual bytes, as definers have special slots for + * those. + * + * For now, keep the code simple. This results in an + * extra submatcher in some cases, but it's good enough. + */ + ret = 0; + break; + } + + /* The current subset of match params still fits in a single + * definer. Remove the dword from the original mask. + * + * Also remove any explicit match on IP version if we just + * included one here. We will still automatically add it to + * accompany any IP address fragment, but do not need to + * consider it by itself. + */ + hws_remove_dword_from_mask(orig, dword_idx, added_inner_ipv, + added_outer_ipv); } - /* We have list of all the match fields from the match parameter. - * Now try all the possibilities of splitting them into two match - * buffers and look for the supported combination. + /* Make sure we have not picked up a single lower dword of an IPv6 + * address, as the firmware will erroneously treat it as an IPv4 + * address. */ - num_of_combinations = 1 << fc_sz; + hws_avoid_ipv6_split(orig, mask); - /* Start from combination at index 1 - we know that 0 is unsupported */ - for (i = 1; i < num_of_combinations; i++) { - if (!hws_bwc_matcher_complex_params_comb_is_valid(fc, fc_sz, i)) - continue; +free_backup: + kfree(backup); - hws_bwc_matcher_complex_params_comb_create(ctx, - mask, mask_1, mask_2, - fc, fc_sz, i); - /* We now have two separate sets of match params. - * Check if each of them can be used in its own matcher. + return ret; +} + +static int +hws_bwc_matcher_split_mask(struct mlx5hws_context *ctx, u8 match_criteria, + const struct mlx5hws_match_parameters *mask, + struct mlx5hws_match_parameters *submasks, + int *num_submasks) +{ + struct mlx5hws_match_parameters mask_copy; + int ret, i = 0; + + mask_copy.match_sz = MLX5_ST_SZ_BYTES(fte_match_param); + mask_copy.match_buf = kzalloc(mask_copy.match_sz, GFP_KERNEL); + if (!mask_copy.match_buf) + return -ENOMEM; + + memcpy(mask_copy.match_buf, mask->match_buf, mask->match_sz); + + while (!hws_match_mask_is_empty(&mask_copy)) { + if (i >= MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS) { + mlx5hws_err(ctx, + "Complex matcher: mask too large for %d matchers\n", + MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS); + ret = -E2BIG; + goto free_copy; + } + /* All but the first matcher need to match on register C6 to + * connect pieces of the complex rule together. */ - if (!mlx5hws_bwc_match_params_is_complex(ctx, - match_criteria, - mask_1) && - !mlx5hws_bwc_match_params_is_complex(ctx, - match_criteria, - mask_2)) - break; + if (i > 0) { + MLX5_SET(fte_match_param, submasks[i].match_buf, + misc_parameters_2.metadata_reg_c_6, -1); + match_criteria |= MLX5HWS_DEFINER_MATCH_CRITERIA_MISC2; + } + ret = hws_get_simple_params(ctx, match_criteria, &mask_copy, + &submasks[i]); + if (ret < 0) + goto free_copy; + i++; } - if (i == num_of_combinations) { - /* We've scanned all the combinations, but to no avail */ - mlx5hws_err(ctx, "Complex matcher: couldn't find match params combination\n"); - res = -EINVAL; - goto free_fc; - } + *num_submasks = i; + ret = 0; - kfree(fc); - return 0; +free_copy: + kfree(mask_copy.match_buf); -free_fc: - kfree(fc); -free_params: - hws_bwc_matcher_complex_params_destroy(mask_1, mask_2); -out: - return res; + return ret; } -static int -hws_bwc_isolated_table_create(struct mlx5hws_bwc_matcher *bwc_matcher, - struct mlx5hws_table *table) +static struct mlx5hws_table * +hws_isolated_table_create(const struct mlx5hws_bwc_matcher *cmatcher) { + struct mlx5hws_bwc_complex_submatcher *first_subm; struct mlx5hws_cmd_ft_modify_attr ft_attr = {0}; - struct mlx5hws_context *ctx = table->ctx; struct mlx5hws_table_attr tbl_attr = {0}; - struct mlx5hws_table *isolated_tbl; - int ret = 0; + struct mlx5hws_table *orig_tbl; + struct mlx5hws_context *ctx; + struct mlx5hws_table *tbl; + int ret; - tbl_attr.type = table->type; - tbl_attr.level = table->level; + first_subm = &cmatcher->complex->submatchers[0]; + orig_tbl = first_subm->tbl; + ctx = orig_tbl->ctx; - bwc_matcher->complex->isolated_tbl = - mlx5hws_table_create(ctx, &tbl_attr); - isolated_tbl = bwc_matcher->complex->isolated_tbl; - if (!isolated_tbl) - return -EINVAL; + tbl_attr.type = orig_tbl->type; + tbl_attr.level = orig_tbl->level; + tbl = mlx5hws_table_create(ctx, &tbl_attr); + if (!tbl) + return ERR_PTR(-EINVAL); - /* Set the default miss of the isolated table to - * point to the end anchor of the original matcher. + /* Set the default miss of the isolated table to point + * to the end anchor of the original matcher. */ - mlx5hws_cmd_set_attr_connect_miss_tbl(ctx, - isolated_tbl->fw_ft_type, - isolated_tbl->type, - &ft_attr); - ft_attr.table_miss_id = bwc_matcher->matcher->end_ft_id; - - ret = mlx5hws_cmd_flow_table_modify(ctx->mdev, - &ft_attr, - isolated_tbl->ft_id); + mlx5hws_cmd_set_attr_connect_miss_tbl(ctx, tbl->fw_ft_type, + tbl->type, &ft_attr); + ft_attr.table_miss_id = first_subm->bwc_matcher->matcher->end_ft_id; + + ret = mlx5hws_cmd_flow_table_modify(ctx->mdev, &ft_attr, tbl->ft_id); if (ret) { - mlx5hws_err(ctx, "Failed setting isolated tbl default miss\n"); + mlx5hws_err(ctx, "Complex matcher: failed to set isolated tbl default miss\n"); goto destroy_tbl; } - return 0; + return tbl; destroy_tbl: - mlx5hws_table_destroy(isolated_tbl); - return ret; + mlx5hws_table_destroy(tbl); + + return ERR_PTR(ret); } -static void hws_bwc_isolated_table_destroy(struct mlx5hws_table *isolated_tbl) +static int hws_submatcher_init_first(struct mlx5hws_bwc_matcher *cmatcher, + struct mlx5hws_table *table, u32 priority, + u8 match_criteria, + struct mlx5hws_match_parameters *mask) { - /* This table is isolated - no table is pointing to it, no need to - * disconnect it from anywhere, it won't affect any other table's miss. + enum mlx5hws_action_type action_types[HWS_NUM_CHAIN_ACTIONS]; + struct mlx5hws_bwc_complex_submatcher *subm; + int ret; + + subm = &cmatcher->complex->submatchers[0]; + + /* The first submatcher lives in the original table and does not have an + * associated jump to table action. It also points to the outer complex + * matcher. */ - mlx5hws_table_destroy(isolated_tbl); + subm->tbl = table; + subm->action_tbl = NULL; + subm->bwc_matcher = cmatcher; + + action_types[0] = MLX5HWS_ACTION_TYP_MODIFY_HDR; + action_types[1] = MLX5HWS_ACTION_TYP_TBL; + action_types[2] = MLX5HWS_ACTION_TYP_LAST; + + ret = mlx5hws_bwc_matcher_create_simple(subm->bwc_matcher, subm->tbl, + priority, match_criteria, mask, + action_types); + if (ret) + return ret; + + subm->bwc_matcher->matcher_type = MLX5HWS_BWC_MATCHER_COMPLEX_FIRST; + + ret = rhashtable_init(&subm->rules_hash, &hws_rules_hash_params); + if (ret) + goto destroy_matcher; + mutex_init(&subm->hash_lock); + ida_init(&subm->chain_ida); + + return 0; + +destroy_matcher: + mlx5hws_bwc_matcher_destroy_simple(subm->bwc_matcher); + + return ret; } -static int -hws_bwc_isolated_matcher_create(struct mlx5hws_bwc_matcher *bwc_matcher, - struct mlx5hws_table *table, - u8 match_criteria_enable, - struct mlx5hws_match_parameters *mask) +static int hws_submatcher_init(struct mlx5hws_bwc_matcher *cmatcher, int idx, + struct mlx5hws_table *table, u32 priority, + u8 match_criteria, + struct mlx5hws_match_parameters *mask) { - struct mlx5hws_table *isolated_tbl = bwc_matcher->complex->isolated_tbl; - struct mlx5hws_bwc_matcher *isolated_bwc_matcher; - struct mlx5hws_context *ctx = table->ctx; + enum mlx5hws_action_type action_types[HWS_NUM_CHAIN_ACTIONS]; + struct mlx5hws_bwc_complex_submatcher *subm; + bool is_last; int ret; - isolated_bwc_matcher = kzalloc(sizeof(*bwc_matcher), GFP_KERNEL); - if (!isolated_bwc_matcher) - return -ENOMEM; + if (!idx) + return hws_submatcher_init_first(cmatcher, table, priority, + match_criteria, mask); + + subm = &cmatcher->complex->submatchers[idx]; + is_last = idx == cmatcher->complex->num_submatchers - 1; + + subm->tbl = hws_isolated_table_create(cmatcher); + if (IS_ERR(subm->tbl)) + return PTR_ERR(subm->tbl); + + subm->action_tbl = + mlx5hws_action_create_dest_table(subm->tbl->ctx, subm->tbl, + MLX5HWS_ACTION_FLAG_HWS_FDB); + if (!subm->action_tbl) { + ret = -EINVAL; + goto destroy_tbl; + } + + subm->bwc_matcher = kzalloc(sizeof(*subm->bwc_matcher), GFP_KERNEL); + if (!subm->bwc_matcher) { + ret = -ENOMEM; + goto destroy_action; + } - bwc_matcher->complex->isolated_bwc_matcher = isolated_bwc_matcher; + /* Every matcher other than the first also matches of register C6 to + * bind subrules together in the complex rule using the chain ids. + */ + match_criteria |= MLX5HWS_DEFINER_MATCH_CRITERIA_MISC2; - /* Isolated BWC matcher needs access to the first BWC matcher */ - isolated_bwc_matcher->complex_first_bwc_matcher = bwc_matcher; + action_types[0] = MLX5HWS_ACTION_TYP_MODIFY_HDR; + action_types[1] = MLX5HWS_ACTION_TYP_TBL; + action_types[2] = MLX5HWS_ACTION_TYP_LAST; - /* Isolated matcher needs to match on REG_C_6, - * so make sure its criteria bit is on. + /* Every matcher other than the last sets register C6 and jumps to the + * next submatcher's table. The final submatcher will use the + * user-supplied actions and will attach an action template at rule + * insertion time. */ - match_criteria_enable |= MLX5HWS_DEFINER_MATCH_CRITERIA_MISC2; - - ret = mlx5hws_bwc_matcher_create_simple(isolated_bwc_matcher, - isolated_tbl, - 0, - match_criteria_enable, - mask, - NULL); - if (ret) { - mlx5hws_err(ctx, "Complex matcher: failed creating isolated BWC matcher\n"); + ret = mlx5hws_bwc_matcher_create_simple(subm->bwc_matcher, subm->tbl, + priority, match_criteria, mask, + is_last ? NULL : action_types); + if (ret) goto free_matcher; - } + + subm->bwc_matcher->matcher_type = + MLX5HWS_BWC_MATCHER_COMPLEX_SUBMATCHER; + + ret = rhashtable_init(&subm->rules_hash, &hws_rules_hash_params); + if (ret) + goto destroy_matcher; + mutex_init(&subm->hash_lock); + ida_init(&subm->chain_ida); return 0; +destroy_matcher: + mlx5hws_bwc_matcher_destroy_simple(subm->bwc_matcher); free_matcher: - kfree(bwc_matcher->complex->isolated_bwc_matcher); + kfree(subm->bwc_matcher); +destroy_action: + mlx5hws_action_destroy(subm->action_tbl); +destroy_tbl: + mlx5hws_table_destroy(subm->tbl); + return ret; } -static void -hws_bwc_isolated_matcher_destroy(struct mlx5hws_bwc_matcher *bwc_matcher) +static void hws_submatcher_destroy(struct mlx5hws_bwc_matcher *cmatcher, + int idx) { - mlx5hws_bwc_matcher_destroy_simple(bwc_matcher); - kfree(bwc_matcher); + struct mlx5hws_bwc_complex_submatcher *subm; + + subm = &cmatcher->complex->submatchers[idx]; + + ida_destroy(&subm->chain_ida); + mutex_destroy(&subm->hash_lock); + rhashtable_destroy(&subm->rules_hash); + + if (subm->bwc_matcher) { + mlx5hws_bwc_matcher_destroy_simple(subm->bwc_matcher); + if (idx) + kfree(subm->bwc_matcher); + } + + /* We own all of the isolated tables, but not the original one. */ + if (idx) { + mlx5hws_action_destroy(subm->action_tbl); + mlx5hws_table_destroy(subm->tbl); + } } static int -hws_bwc_isolated_actions_create(struct mlx5hws_bwc_matcher *bwc_matcher, - struct mlx5hws_table *table) +hws_complex_data_actions_init(struct mlx5hws_bwc_matcher_complex_data *cdata) { - struct mlx5hws_table *isolated_tbl = bwc_matcher->complex->isolated_tbl; + struct mlx5hws_context *ctx = cdata->submatchers[0].tbl->ctx; u8 modify_hdr_action[MLX5_ST_SZ_BYTES(set_action_in)] = {0}; - struct mlx5hws_context *ctx = table->ctx; struct mlx5hws_action_mh_pattern ptrn; int ret = 0; - /* Create action to jump to isolated table */ - - bwc_matcher->complex->action_go_to_tbl = - mlx5hws_action_create_dest_table(ctx, - isolated_tbl, - MLX5HWS_ACTION_FLAG_HWS_FDB); - if (!bwc_matcher->complex->action_go_to_tbl) { - mlx5hws_err(ctx, "Complex matcher: failed to create go-to-tbl action\n"); - return -EINVAL; - } - /* Create modify header action to set REG_C_6 */ - MLX5_SET(set_action_in, modify_hdr_action, action_type, MLX5_MODIFICATION_TYPE_SET); MLX5_SET(set_action_in, modify_hdr_action, @@ -895,19 +553,18 @@ hws_bwc_isolated_actions_create(struct mlx5hws_bwc_matcher *bwc_matcher, ptrn.data = (void *)modify_hdr_action; ptrn.sz = MLX5HWS_ACTION_DOUBLE_SIZE; - bwc_matcher->complex->action_metadata = + cdata->action_metadata = mlx5hws_action_create_modify_header(ctx, 1, &ptrn, 0, MLX5HWS_ACTION_FLAG_HWS_FDB); - if (!bwc_matcher->complex->action_metadata) { - ret = -EINVAL; - goto destroy_action_go_to_tbl; + if (!cdata->action_metadata) { + mlx5hws_err(ctx, "Complex matcher: failed to create set reg C6 action\n"); + return -EINVAL; } /* Create last action */ - - bwc_matcher->complex->action_last = + cdata->action_last = mlx5hws_action_create_last(ctx, MLX5HWS_ACTION_FLAG_HWS_FDB); - if (!bwc_matcher->complex->action_last) { + if (!cdata->action_last) { mlx5hws_err(ctx, "Complex matcher: failed to create last action\n"); ret = -EINVAL; goto destroy_action_metadata; @@ -916,196 +573,130 @@ hws_bwc_isolated_actions_create(struct mlx5hws_bwc_matcher *bwc_matcher, return 0; destroy_action_metadata: - mlx5hws_action_destroy(bwc_matcher->complex->action_metadata); -destroy_action_go_to_tbl: - mlx5hws_action_destroy(bwc_matcher->complex->action_go_to_tbl); + mlx5hws_action_destroy(cdata->action_metadata); + return ret; } static void -hws_bwc_isolated_actions_destroy(struct mlx5hws_bwc_matcher *bwc_matcher) +hws_complex_data_actions_destroy(struct mlx5hws_bwc_matcher_complex_data *cdata) { - mlx5hws_action_destroy(bwc_matcher->complex->action_last); - mlx5hws_action_destroy(bwc_matcher->complex->action_metadata); - mlx5hws_action_destroy(bwc_matcher->complex->action_go_to_tbl); + mlx5hws_action_destroy(cdata->action_last); + mlx5hws_action_destroy(cdata->action_metadata); } int mlx5hws_bwc_matcher_create_complex(struct mlx5hws_bwc_matcher *bwc_matcher, struct mlx5hws_table *table, - u32 priority, - u8 match_criteria_enable, + u32 priority, u8 match_criteria_enable, struct mlx5hws_match_parameters *mask) { - enum mlx5hws_action_type complex_init_action_types[3]; - struct mlx5hws_bwc_matcher *isolated_bwc_matcher; - struct mlx5hws_match_parameters mask_1 = {0}; - struct mlx5hws_match_parameters mask_2 = {0}; + struct mlx5hws_match_parameters + submasks[MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS] = {0}; + struct mlx5hws_bwc_matcher_complex_data *cdata; struct mlx5hws_context *ctx = table->ctx; - int ret; - - ret = hws_bwc_matcher_complex_params_create(table->ctx, - match_criteria_enable, - mask, &mask_1, &mask_2); - if (ret) - goto err; - - bwc_matcher->complex = - kzalloc(sizeof(*bwc_matcher->complex), GFP_KERNEL); - if (!bwc_matcher->complex) { - ret = -ENOMEM; - goto free_masks; - } + int num_submatchers; + int i, ret; - ret = rhashtable_init(&bwc_matcher->complex->refcount_hash, - &hws_refcount_hash); - if (ret) { - mlx5hws_err(ctx, "Complex matcher: failed to initialize rhashtable\n"); - goto free_complex; + for (i = 0; i < ARRAY_SIZE(submasks); i++) { + submasks[i].match_sz = MLX5_ST_SZ_BYTES(fte_match_param); + submasks[i].match_buf = kzalloc(submasks[i].match_sz, + GFP_KERNEL); + if (!submasks[i].match_buf) { + ret = -ENOMEM; + goto free_submasks; + } } - mutex_init(&bwc_matcher->complex->hash_lock); - ida_init(&bwc_matcher->complex->metadata_ida); - - /* Create initial action template for the first matcher. - * Usually the initial AT is just dummy, but in case of complex - * matcher we know exactly which actions should it have. - */ - - complex_init_action_types[0] = MLX5HWS_ACTION_TYP_MODIFY_HDR; - complex_init_action_types[1] = MLX5HWS_ACTION_TYP_TBL; - complex_init_action_types[2] = MLX5HWS_ACTION_TYP_LAST; - - /* Create the first matcher */ - - ret = mlx5hws_bwc_matcher_create_simple(bwc_matcher, - table, - priority, - match_criteria_enable, - &mask_1, - complex_init_action_types); + ret = hws_bwc_matcher_split_mask(ctx, match_criteria_enable, mask, + submasks, &num_submatchers); if (ret) - goto destroy_ida; - - /* Create isolated table to hold the second isolated matcher */ + goto free_submasks; - ret = hws_bwc_isolated_table_create(bwc_matcher, table); - if (ret) { - mlx5hws_err(ctx, "Complex matcher: failed creating isolated table\n"); - goto destroy_first_matcher; + cdata = kzalloc(sizeof(*cdata), GFP_KERNEL); + if (!cdata) { + ret = -ENOMEM; + goto free_submasks; } - /* Now create the second BWC matcher - the isolated one */ + bwc_matcher->complex = cdata; + cdata->num_submatchers = num_submatchers; - ret = hws_bwc_isolated_matcher_create(bwc_matcher, table, - match_criteria_enable, &mask_2); - if (ret) { - mlx5hws_err(ctx, "Complex matcher: failed creating isolated matcher\n"); - goto destroy_isolated_tbl; + for (i = 0; i < num_submatchers; i++) { + ret = hws_submatcher_init(bwc_matcher, i, table, priority, + match_criteria_enable, &submasks[i]); + if (ret) + goto destroy_submatchers; } - /* Create action for isolated matcher's rules */ - - ret = hws_bwc_isolated_actions_create(bwc_matcher, table); - if (ret) { - mlx5hws_err(ctx, "Complex matcher: failed creating isolated actions\n"); - goto destroy_isolated_matcher; - } + ret = hws_complex_data_actions_init(cdata); + if (ret) + goto destroy_submatchers; - hws_bwc_matcher_complex_params_destroy(&mask_1, &mask_2); - return 0; + ret = 0; + goto free_submasks; -destroy_isolated_matcher: - isolated_bwc_matcher = bwc_matcher->complex->isolated_bwc_matcher; - hws_bwc_isolated_matcher_destroy(isolated_bwc_matcher); -destroy_isolated_tbl: - hws_bwc_isolated_table_destroy(bwc_matcher->complex->isolated_tbl); -destroy_first_matcher: - mlx5hws_bwc_matcher_destroy_simple(bwc_matcher); -destroy_ida: - ida_destroy(&bwc_matcher->complex->metadata_ida); - mutex_destroy(&bwc_matcher->complex->hash_lock); - rhashtable_destroy(&bwc_matcher->complex->refcount_hash); -free_complex: - kfree(bwc_matcher->complex); +destroy_submatchers: + while (i--) + hws_submatcher_destroy(bwc_matcher, i); + kfree(cdata); bwc_matcher->complex = NULL; -free_masks: - hws_bwc_matcher_complex_params_destroy(&mask_1, &mask_2); -err: + +free_submasks: + for (i = 0; i < ARRAY_SIZE(submasks); i++) + kfree(submasks[i].match_buf); + return ret; } void mlx5hws_bwc_matcher_destroy_complex(struct mlx5hws_bwc_matcher *bwc_matcher) { - struct mlx5hws_bwc_matcher *isolated_bwc_matcher = - bwc_matcher->complex->isolated_bwc_matcher; - - hws_bwc_isolated_actions_destroy(bwc_matcher); - hws_bwc_isolated_matcher_destroy(isolated_bwc_matcher); - hws_bwc_isolated_table_destroy(bwc_matcher->complex->isolated_tbl); - mlx5hws_bwc_matcher_destroy_simple(bwc_matcher); - ida_destroy(&bwc_matcher->complex->metadata_ida); - mutex_destroy(&bwc_matcher->complex->hash_lock); - rhashtable_destroy(&bwc_matcher->complex->refcount_hash); + int i; + + hws_complex_data_actions_destroy(bwc_matcher->complex); + for (i = 0; i < bwc_matcher->complex->num_submatchers; i++) + hws_submatcher_destroy(bwc_matcher, i); kfree(bwc_matcher->complex); bwc_matcher->complex = NULL; } -static void -hws_bwc_matcher_complex_hash_lock(struct mlx5hws_bwc_matcher *bwc_matcher) -{ - mutex_lock(&bwc_matcher->complex->hash_lock); -} - -static void -hws_bwc_matcher_complex_hash_unlock(struct mlx5hws_bwc_matcher *bwc_matcher) -{ - mutex_unlock(&bwc_matcher->complex->hash_lock); -} - static int -hws_bwc_rule_complex_hash_node_get(struct mlx5hws_bwc_rule *bwc_rule, - struct mlx5hws_match_parameters *params) +hws_complex_get_subrule_data(struct mlx5hws_bwc_rule *bwc_rule, + struct mlx5hws_bwc_complex_submatcher *subm, + u32 *match_params) +__must_hold(&subm->hash_lock) { - struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher; - struct mlx5hws_bwc_complex_rule_hash_node *node, *old_node; - struct rhashtable *refcount_hash; - int ret, i; - - bwc_rule->complex_hash_node = NULL; + struct mlx5hws_bwc_matcher *bwc_matcher = subm->bwc_matcher; + struct mlx5hws_bwc_complex_subrule_data *sr_data, *old_data; + struct mlx5hws_match_template *mt; + int ret; - node = kzalloc(sizeof(*node), GFP_KERNEL); - if (unlikely(!node)) + sr_data = kzalloc(sizeof(*sr_data), GFP_KERNEL); + if (!sr_data) return -ENOMEM; - ret = ida_alloc(&bwc_matcher->complex->metadata_ida, GFP_KERNEL); + ret = ida_alloc(&subm->chain_ida, GFP_KERNEL); if (ret < 0) - goto err_free_node; - node->tag = ret; + goto free_sr_data; + sr_data->chain_id = ret; - refcount_set(&node->refcount, 1); + refcount_set(&sr_data->refcount, 1); - /* Clear match buffer - turn off all the unrelated fields - * in accordance with the match params mask for the first - * matcher out of the two parts of the complex matcher. - * The resulting mask is the key for the hash. - */ - for (i = 0; i < MLX5_ST_SZ_DW_MATCH_PARAM; i++) - node->match_buf[i] = params->match_buf[i] & - bwc_matcher->mt->match_param[i]; - - refcount_hash = &bwc_matcher->complex->refcount_hash; - old_node = rhashtable_lookup_get_insert_fast(refcount_hash, - &node->hash_node, - hws_refcount_hash); - if (IS_ERR(old_node)) { - ret = PTR_ERR(old_node); - goto err_free_ida; + mt = bwc_matcher->matcher->mt; + mlx5hws_definer_create_tag(match_params, mt->fc, mt->fc_sz, + (u8 *)&sr_data->match_tag); + + old_data = rhashtable_lookup_get_insert_fast(&subm->rules_hash, + &sr_data->hash_node, + hws_rules_hash_params); + if (IS_ERR(old_data)) { + ret = PTR_ERR(old_data); + goto free_ida; } - if (old_node) { + if (old_data) { /* Rule with the same tag already exists - update refcount */ - refcount_inc(&old_node->refcount); + refcount_inc(&old_data->refcount); /* Let the new rule use the same tag as the existing rule. * Note that we don't have any indication for the rule creation * process that a rule with similar matching params already @@ -1114,247 +705,281 @@ hws_bwc_rule_complex_hash_node_get(struct mlx5hws_bwc_rule *bwc_rule, * There's some performance advantage in skipping such cases, * so this is left for future optimizations. */ - ida_free(&bwc_matcher->complex->metadata_ida, node->tag); - kfree(node); - node = old_node; + bwc_rule->subrule_data = old_data; + ret = 0; + goto free_ida; } - bwc_rule->complex_hash_node = node; + bwc_rule->subrule_data = sr_data; return 0; -err_free_ida: - ida_free(&bwc_matcher->complex->metadata_ida, node->tag); -err_free_node: - kfree(node); +free_ida: + ida_free(&subm->chain_ida, sr_data->chain_id); +free_sr_data: + kfree(sr_data); + return ret; } static void -hws_bwc_rule_complex_hash_node_put(struct mlx5hws_bwc_rule *bwc_rule, - bool *is_last_rule) +hws_complex_put_subrule_data(struct mlx5hws_bwc_rule *bwc_rule, + struct mlx5hws_bwc_complex_submatcher *subm, + bool *is_last_rule) +__must_hold(&subm->hash_lock) { - struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher; - struct mlx5hws_bwc_complex_rule_hash_node *node; + struct mlx5hws_bwc_complex_subrule_data *sr_data; if (is_last_rule) *is_last_rule = false; - node = bwc_rule->complex_hash_node; - if (refcount_dec_and_test(&node->refcount)) { - rhashtable_remove_fast(&bwc_matcher->complex->refcount_hash, - &node->hash_node, - hws_refcount_hash); - ida_free(&bwc_matcher->complex->metadata_ida, node->tag); - kfree(node); + sr_data = bwc_rule->subrule_data; + if (refcount_dec_and_test(&sr_data->refcount)) { + rhashtable_remove_fast(&subm->rules_hash, + &sr_data->hash_node, + hws_rules_hash_params); + ida_free(&subm->chain_ida, sr_data->chain_id); + kfree(sr_data); if (is_last_rule) *is_last_rule = true; } - bwc_rule->complex_hash_node = NULL; + bwc_rule->subrule_data = NULL; } -int mlx5hws_bwc_rule_create_complex(struct mlx5hws_bwc_rule *bwc_rule, - struct mlx5hws_match_parameters *params, - u32 flow_source, - struct mlx5hws_rule_action rule_actions[], - u16 bwc_queue_idx) +static int hws_complex_subrule_create(struct mlx5hws_bwc_matcher *cmatcher, + struct mlx5hws_bwc_rule *subrule, + u32 *match_params, u32 flow_source, + int bwc_queue_idx, int subm_idx, + struct mlx5hws_rule_action *actions, + u32 *chain_id) { - struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher; - struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx; + struct mlx5hws_rule_action chain_actions[HWS_NUM_CHAIN_ACTIONS] = {0}; u8 modify_hdr_action[MLX5_ST_SZ_BYTES(set_action_in)] = {0}; - struct mlx5hws_rule_action rule_actions_1[3] = {0}; - struct mlx5hws_bwc_matcher *isolated_bwc_matcher; - u32 *match_buf_2; - u32 metadata_val; - int ret = 0; + struct mlx5hws_bwc_matcher_complex_data *cdata; + struct mlx5hws_bwc_complex_submatcher *subm; + int ret; - isolated_bwc_matcher = bwc_matcher->complex->isolated_bwc_matcher; - bwc_rule->isolated_bwc_rule = - mlx5hws_bwc_rule_alloc(isolated_bwc_matcher); - if (unlikely(!bwc_rule->isolated_bwc_rule)) - return -ENOMEM; + cdata = cmatcher->complex; + subm = &cdata->submatchers[subm_idx]; - hws_bwc_matcher_complex_hash_lock(bwc_matcher); + mutex_lock(&subm->hash_lock); - /* Get a new hash node for this complex rule. - * If this is a unique set of match params for the first matcher, - * we will get a new hash node with newly allocated IDA. - * Otherwise we will get an existing node with IDA and updated refcount. - */ - ret = hws_bwc_rule_complex_hash_node_get(bwc_rule, params); - if (unlikely(ret)) { - mlx5hws_err(ctx, "Complex rule: failed getting RHT node for this rule\n"); - goto free_isolated_rule; + ret = hws_complex_get_subrule_data(subrule, subm, match_params); + if (ret) + goto unlock; + + *chain_id = subrule->subrule_data->chain_id; + + if (!actions) { + MLX5_SET(set_action_in, modify_hdr_action, data, *chain_id); + chain_actions[0].action = cdata->action_metadata; + chain_actions[0].modify_header.data = modify_hdr_action; + chain_actions[1].action = + cdata->submatchers[subm_idx + 1].action_tbl; + chain_actions[2].action = cdata->action_last; + actions = chain_actions; } - /* No need to clear match buffer's fields in accordance to what - * will actually be matched on first and second matchers. - * Both matchers were created with the appropriate masks - * and each of them holds the appropriate field copy array, - * so rule creation will use only the fields that will be copied - * in accordance with setters in field copy array. - * We do, however, need to temporary allocate match buffer - * for the second (isolated) rule in order to not modify - * user's match params buffer. - */ - - match_buf_2 = kmemdup(params->match_buf, - MLX5_ST_SZ_BYTES(fte_match_param), - GFP_KERNEL); - if (unlikely(!match_buf_2)) { - mlx5hws_err(ctx, "Complex rule: failed allocating match_buf\n"); - ret = -ENOMEM; - goto hash_node_put; - } + ret = mlx5hws_bwc_rule_create_simple(subrule, match_params, actions, + flow_source, bwc_queue_idx); + if (ret) + goto put_subrule_data; - /* On 2nd matcher, use unique 32-bit ID as a matching tag */ - metadata_val = bwc_rule->complex_hash_node->tag; - MLX5_SET(fte_match_param, match_buf_2, - misc_parameters_2.metadata_reg_c_6, metadata_val); - - /* Isolated rule's rule_actions contain all the original actions */ - ret = mlx5hws_bwc_rule_create_simple(bwc_rule->isolated_bwc_rule, - match_buf_2, - rule_actions, - flow_source, - bwc_queue_idx); - kfree(match_buf_2); - if (unlikely(ret)) { - mlx5hws_err(ctx, - "Complex rule: failed creating isolated BWC rule (%d)\n", - ret); - goto hash_node_put; - } + ret = 0; + goto unlock; - /* First rule's rule_actions contain setting metadata and - * jump to isolated table that contains the second matcher. - * Set metadata value to a unique value for this rule. - */ +put_subrule_data: + hws_complex_put_subrule_data(subrule, subm, NULL); +unlock: + mutex_unlock(&subm->hash_lock); - MLX5_SET(set_action_in, modify_hdr_action, - action_type, MLX5_MODIFICATION_TYPE_SET); - MLX5_SET(set_action_in, modify_hdr_action, - field, MLX5_MODI_META_REG_C_6); - MLX5_SET(set_action_in, modify_hdr_action, - length, 0); /* zero means length of 32 */ - MLX5_SET(set_action_in, modify_hdr_action, - offset, 0); - MLX5_SET(set_action_in, modify_hdr_action, - data, metadata_val); + return ret; +} - rule_actions_1[0].action = bwc_matcher->complex->action_metadata; - rule_actions_1[0].modify_header.offset = 0; - rule_actions_1[0].modify_header.data = modify_hdr_action; +static int hws_complex_subrule_destroy(struct mlx5hws_bwc_rule *bwc_rule, + struct mlx5hws_bwc_matcher *cmatcher, + int subm_idx) +{ + struct mlx5hws_bwc_matcher_complex_data *cdata; + struct mlx5hws_bwc_complex_submatcher *subm; + struct mlx5hws_context *ctx; + bool is_last_rule; + int ret = 0; - rule_actions_1[1].action = bwc_matcher->complex->action_go_to_tbl; - rule_actions_1[2].action = bwc_matcher->complex->action_last; + cdata = cmatcher->complex; + subm = &cdata->submatchers[subm_idx]; + ctx = subm->tbl->ctx; - ret = mlx5hws_bwc_rule_create_simple(bwc_rule, - params->match_buf, - rule_actions_1, - flow_source, - bwc_queue_idx); + mutex_lock(&subm->hash_lock); - if (unlikely(ret)) { + hws_complex_put_subrule_data(bwc_rule, subm, &is_last_rule); + bwc_rule->rule->skip_delete = !is_last_rule; + ret = mlx5hws_bwc_rule_destroy_simple(bwc_rule); + if (unlikely(ret)) mlx5hws_err(ctx, - "Complex rule: failed creating first BWC rule (%d)\n", - ret); - goto destroy_isolated_rule; - } + "Complex rule: failed to delete subrule %d (%d)\n", + subm_idx, ret); - hws_bwc_matcher_complex_hash_unlock(bwc_matcher); + if (subm_idx) + mlx5hws_bwc_rule_free(bwc_rule); - return 0; + mutex_unlock(&subm->hash_lock); -destroy_isolated_rule: - mlx5hws_bwc_rule_destroy_simple(bwc_rule->isolated_bwc_rule); -hash_node_put: - hws_bwc_rule_complex_hash_node_put(bwc_rule, NULL); -free_isolated_rule: - hws_bwc_matcher_complex_hash_unlock(bwc_matcher); - mlx5hws_bwc_rule_free(bwc_rule->isolated_bwc_rule); return ret; } -int mlx5hws_bwc_rule_destroy_complex(struct mlx5hws_bwc_rule *bwc_rule) +int mlx5hws_bwc_rule_create_complex(struct mlx5hws_bwc_rule *bwc_rule, + struct mlx5hws_match_parameters *params, + u32 flow_source, + struct mlx5hws_rule_action rule_actions[], + u16 bwc_queue_idx) { - struct mlx5hws_context *ctx = bwc_rule->bwc_matcher->matcher->tbl->ctx; - struct mlx5hws_bwc_rule *isolated_bwc_rule; - int ret_isolated, ret; - bool is_last_rule; + struct mlx5hws_bwc_rule + *subrules[MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS] = {0}; + struct mlx5hws_bwc_matcher *cmatcher = bwc_rule->bwc_matcher; + struct mlx5hws_bwc_matcher_complex_data *cdata; + struct mlx5hws_rule_action *subrule_actions; + struct mlx5hws_bwc_complex_submatcher *subm; + struct mlx5hws_bwc_rule *subrule; + u32 *match_params; + u32 chain_id; + int i, ret; - hws_bwc_matcher_complex_hash_lock(bwc_rule->bwc_matcher); + cdata = cmatcher->complex; + if (!cdata) + return -EINVAL; - hws_bwc_rule_complex_hash_node_put(bwc_rule, &is_last_rule); - bwc_rule->rule->skip_delete = !is_last_rule; + /* Duplicate user data because we will modify it to set register C6 + * values. For the same reason, make sure that we allocate a full + * match_param even if the user gave us fewer bytes. We need to ensure + * there is space for the match on C6. + */ + match_params = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL); + if (!match_params) + return -ENOMEM; - ret = mlx5hws_bwc_rule_destroy_simple(bwc_rule); - if (unlikely(ret)) - mlx5hws_err(ctx, "BWC complex rule: failed destroying first rule\n"); + memcpy(match_params, params->match_buf, params->match_sz); + + ret = hws_complex_subrule_create(cmatcher, bwc_rule, match_params, + flow_source, bwc_queue_idx, 0, + NULL, &chain_id); + if (ret) + goto free_match_params; + subrules[0] = bwc_rule; + + for (i = 1; i < cdata->num_submatchers; i++) { + subm = &cdata->submatchers[i]; + subrule = mlx5hws_bwc_rule_alloc(subm->bwc_matcher); + if (!subrule) { + ret = -ENOMEM; + goto destroy_subrules; + } + + /* Match on the previous subrule's chain_id. This is how + * subrules are connected in steering. + */ + MLX5_SET(fte_match_param, match_params, + misc_parameters_2.metadata_reg_c_6, chain_id); + + /* The last subrule uses the complex rule's user-specified + * actions. Everything else uses the chaining rules based on the + * next table and chain_id. + */ + subrule_actions = + i == cdata->num_submatchers - 1 ? rule_actions : NULL; + + ret = hws_complex_subrule_create(cmatcher, subrule, + match_params, flow_source, + bwc_queue_idx, i, + subrule_actions, &chain_id); + if (ret) { + mlx5hws_bwc_rule_free(subrule); + goto destroy_subrules; + } + + subrules[i] = subrule; + } + + for (i = 0; i < cdata->num_submatchers - 1; i++) + subrules[i]->next_subrule = subrules[i + 1]; - isolated_bwc_rule = bwc_rule->isolated_bwc_rule; - ret_isolated = mlx5hws_bwc_rule_destroy_simple(isolated_bwc_rule); - if (unlikely(ret_isolated)) - mlx5hws_err(ctx, "BWC complex rule: failed destroying second (isolated) rule\n"); + kfree(match_params); - hws_bwc_matcher_complex_hash_unlock(bwc_rule->bwc_matcher); + return 0; - mlx5hws_bwc_rule_free(isolated_bwc_rule); +destroy_subrules: + while (i--) + hws_complex_subrule_destroy(subrules[i], cmatcher, i); +free_match_params: + kfree(match_params); - return ret || ret_isolated; + return ret; } -static void -hws_bwc_matcher_clear_hash_rtcs(struct mlx5hws_bwc_matcher *bwc_matcher) +int mlx5hws_bwc_rule_destroy_complex(struct mlx5hws_bwc_rule *bwc_rule) { - struct mlx5hws_bwc_complex_rule_hash_node *node; - struct rhashtable_iter iter; + struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher; + struct mlx5hws_bwc_rule + *subrules[MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS] = {0}; + struct mlx5hws_bwc_matcher_complex_data *cdata; + int i, err, ret_val; + + cdata = bwc_matcher->complex; + + /* Construct a list of all the subrules we need to destroy. */ + subrules[0] = bwc_rule; + for (i = 1; i < cdata->num_submatchers; i++) + subrules[i] = subrules[i - 1]->next_subrule; + + ret_val = 0; + for (i = 0; i < cdata->num_submatchers; i++) { + err = hws_complex_subrule_destroy(subrules[i], bwc_matcher, i); + /* If something goes wrong, plow along to destroy all of the + * subrules but return an error upstack. + */ + if (unlikely(err)) + ret_val = err; + } - rhashtable_walk_enter(&bwc_matcher->complex->refcount_hash, &iter); - rhashtable_walk_start(&iter); + return ret_val; +} - while ((node = rhashtable_walk_next(&iter)) != NULL) { - if (IS_ERR(node)) +static void +hws_bwc_matcher_init_move(struct mlx5hws_bwc_matcher *bwc_matcher) +{ + struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx; + u16 bwc_queues = mlx5hws_bwc_queues(ctx); + struct mlx5hws_bwc_rule *bwc_rule; + struct list_head *rules_list; + int i; + + for (i = 0; i < bwc_queues; i++) { + rules_list = &bwc_matcher->rules[i]; + if (list_empty(rules_list)) continue; - node->rtc_valid = false; - } - rhashtable_walk_stop(&iter); - rhashtable_walk_exit(&iter); + list_for_each_entry(bwc_rule, rules_list, list_node) { + if (!bwc_rule->subrule_data) + continue; + bwc_rule->subrule_data->was_moved = false; + } + } } -int -mlx5hws_bwc_matcher_move_all_complex(struct mlx5hws_bwc_matcher *bwc_matcher) +int mlx5hws_bwc_matcher_complex_move(struct mlx5hws_bwc_matcher *bwc_matcher) { struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx; struct mlx5hws_matcher *matcher = bwc_matcher->matcher; u16 bwc_queues = mlx5hws_bwc_queues(ctx); struct mlx5hws_bwc_rule *tmp_bwc_rule; struct mlx5hws_rule_attr rule_attr; - struct mlx5hws_table *isolated_tbl; int move_error = 0, poll_error = 0; struct mlx5hws_rule *tmp_rule; struct list_head *rules_list; u32 expected_completions = 1; - u32 end_ft_id; - int i, ret; + int i, ret = 0; - /* We are rehashing the matcher that is the first part of the complex - * matcher. Need to update the isolated matcher to point to the end_ft - * of this new matcher. This needs to be done before moving any rules - * to prevent possible steering loops. - */ - isolated_tbl = bwc_matcher->complex->isolated_tbl; - end_ft_id = bwc_matcher->matcher->resize_dst->end_ft_id; - ret = mlx5hws_matcher_update_end_ft_isolated(isolated_tbl, end_ft_id); - if (ret) { - mlx5hws_err(ctx, - "Failed updating end_ft of isolated matcher (%d)\n", - ret); - return ret; - } - - hws_bwc_matcher_clear_hash_rtcs(bwc_matcher); + hws_bwc_matcher_init_move(bwc_matcher); mlx5hws_bwc_rule_fill_attr(bwc_matcher, 0, 0, &rule_attr); @@ -1369,15 +994,15 @@ mlx5hws_bwc_matcher_move_all_complex(struct mlx5hws_bwc_matcher *bwc_matcher) /* Check if a rule with similar tag has already * been moved. */ - if (tmp_bwc_rule->complex_hash_node->rtc_valid) { - /* This rule is a duplicate of rule with similar - * tag that has already been moved earlier. - * Just update this rule's RTCs. + if (tmp_bwc_rule->subrule_data->was_moved) { + /* This rule is a duplicate of rule with + * identical tag that has already been moved + * earlier. Just update this rule's RTCs. */ tmp_bwc_rule->rule->rtc_0 = - tmp_bwc_rule->complex_hash_node->rtc_0; + tmp_bwc_rule->subrule_data->rtc_0; tmp_bwc_rule->rule->rtc_1 = - tmp_bwc_rule->complex_hash_node->rtc_1; + tmp_bwc_rule->subrule_data->rtc_1; tmp_bwc_rule->rule->matcher = tmp_bwc_rule->rule->matcher->resize_dst; continue; @@ -1425,12 +1050,12 @@ mlx5hws_bwc_matcher_move_all_complex(struct mlx5hws_bwc_matcher *bwc_matcher) /* Done moving the rule to the new matcher, * now update RTCs for all the duplicated rules. */ - tmp_bwc_rule->complex_hash_node->rtc_0 = + tmp_bwc_rule->subrule_data->rtc_0 = tmp_bwc_rule->rule->rtc_0; - tmp_bwc_rule->complex_hash_node->rtc_1 = + tmp_bwc_rule->subrule_data->rtc_1 = tmp_bwc_rule->rule->rtc_1; - tmp_bwc_rule->complex_hash_node->rtc_valid = true; + tmp_bwc_rule->subrule_data->was_moved = true; } } @@ -1442,3 +1067,35 @@ mlx5hws_bwc_matcher_move_all_complex(struct mlx5hws_bwc_matcher *bwc_matcher) return ret; } + +int +mlx5hws_bwc_matcher_complex_move_first(struct mlx5hws_bwc_matcher *bwc_matcher) +{ + struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx; + struct mlx5hws_bwc_matcher_complex_data *cdata; + struct mlx5hws_table *isolated_tbl; + u32 end_ft_id; + int i, ret; + + cdata = bwc_matcher->complex; + + /* We are rehashing the first submatcher. We need to update the + * subsequent submatchers to point to the end_ft of this new matcher. + * This needs to be done before moving any rules to prevent possible + * steering loops. + */ + end_ft_id = bwc_matcher->matcher->resize_dst->end_ft_id; + for (i = 1; i < cdata->num_submatchers; i++) { + isolated_tbl = cdata->submatchers[i].tbl; + ret = mlx5hws_matcher_update_end_ft_isolated(isolated_tbl, + end_ft_id); + if (ret) { + mlx5hws_err(ctx, + "Complex matcher: failed updating end_ft of isolated matcher (%d)\n", + ret); + return ret; + } + } + + return mlx5hws_bwc_matcher_complex_move(bwc_matcher); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc_complex.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc_complex.h index a6887c7e39d5..d07de631ce9f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc_complex.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc_complex.h @@ -4,25 +4,60 @@ #ifndef HWS_BWC_COMPLEX_H_ #define HWS_BWC_COMPLEX_H_ -struct mlx5hws_bwc_complex_rule_hash_node { - u32 match_buf[MLX5_ST_SZ_DW_MATCH_PARAM]; - u32 tag; +#define MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS 4 + +/* A matcher can't contain two rules with the same match tag, but it is possible + * that two different complex rules' subrules have the same match tag. In that + * case, those subrules correspond to a single rule, and we need to refcount. + */ +struct mlx5hws_bwc_complex_subrule_data { + struct mlx5hws_rule_match_tag match_tag; refcount_t refcount; - bool rtc_valid; + /* The chain_id is what glues individual subrules into larger complex + * rules. It is the value that this subrule writes to register C6, and + * that the next subrule matches against. + */ + u32 chain_id; u32 rtc_0; u32 rtc_1; + /* During rehash we iterate through all the subrules to move them. But + * two or more subrules can share the same physical rule in the + * submatcher, so we use `was_moved` to keep track if a given rule was + * already moved. + */ + bool was_moved; struct rhash_head hash_node; }; +struct mlx5hws_bwc_complex_submatcher { + /* Isolated table that the matcher lives in. Not set for the first + * matcher, which lives in the original table. + */ + struct mlx5hws_table *tbl; + /* Match a rule with this action to go to `tbl`. This is set in all + * submatchers but the first. + */ + struct mlx5hws_action *action_tbl; + /* This submatcher's simple matcher. The first submatcher points to the + * outer (complex) matcher. + */ + struct mlx5hws_bwc_matcher *bwc_matcher; + struct rhashtable rules_hash; + struct ida chain_ida; + struct mutex hash_lock; /* Protect the hash and ida. */ +}; + struct mlx5hws_bwc_matcher_complex_data { - struct mlx5hws_table *isolated_tbl; - struct mlx5hws_bwc_matcher *isolated_bwc_matcher; + struct mlx5hws_bwc_complex_submatcher + submatchers[MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS]; + int num_submatchers; + /* Actions used by all but the last submatcher to point to the next + * submatcher in the chain. The last submatcher uses the action template + * from the complex matcher, to perform the actions that the user + * originally requested. + */ struct mlx5hws_action *action_metadata; - struct mlx5hws_action *action_go_to_tbl; struct mlx5hws_action *action_last; - struct rhashtable refcount_hash; - struct mutex hash_lock; /* Protect the refcount rhashtable */ - struct ida metadata_ida; }; bool mlx5hws_bwc_match_params_is_complex(struct mlx5hws_context *ctx, @@ -37,7 +72,10 @@ int mlx5hws_bwc_matcher_create_complex(struct mlx5hws_bwc_matcher *bwc_matcher, void mlx5hws_bwc_matcher_destroy_complex(struct mlx5hws_bwc_matcher *bwc_matcher); -int mlx5hws_bwc_matcher_move_all_complex(struct mlx5hws_bwc_matcher *bwc_matcher); +int mlx5hws_bwc_matcher_complex_move(struct mlx5hws_bwc_matcher *bwc_matcher); + +int +mlx5hws_bwc_matcher_complex_move_first(struct mlx5hws_bwc_matcher *bwc_matcher); int mlx5hws_bwc_rule_create_complex(struct mlx5hws_bwc_rule *bwc_rule, struct mlx5hws_match_parameters *params, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/definer.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/definer.c index c4bb6967f74d..82fd122d4284 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/definer.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/definer.c @@ -1831,80 +1831,6 @@ err_free_fc: return ret; } -struct mlx5hws_definer_fc * -mlx5hws_definer_conv_match_params_to_compressed_fc(struct mlx5hws_context *ctx, - u8 match_criteria_enable, - u32 *match_param, - int *fc_sz) -{ - struct mlx5hws_definer_fc *compressed_fc = NULL; - struct mlx5hws_definer_conv_data cd = {0}; - struct mlx5hws_definer_fc *fc; - int ret; - - fc = hws_definer_alloc_fc(ctx, MLX5HWS_DEFINER_FNAME_MAX); - if (!fc) - return NULL; - - cd.fc = fc; - cd.ctx = ctx; - - if (match_criteria_enable & MLX5HWS_DEFINER_MATCH_CRITERIA_OUTER) { - ret = hws_definer_conv_outer(&cd, match_param); - if (ret) - goto err_free_fc; - } - - if (match_criteria_enable & MLX5HWS_DEFINER_MATCH_CRITERIA_INNER) { - ret = hws_definer_conv_inner(&cd, match_param); - if (ret) - goto err_free_fc; - } - - if (match_criteria_enable & MLX5HWS_DEFINER_MATCH_CRITERIA_MISC) { - ret = hws_definer_conv_misc(&cd, match_param); - if (ret) - goto err_free_fc; - } - - if (match_criteria_enable & MLX5HWS_DEFINER_MATCH_CRITERIA_MISC2) { - ret = hws_definer_conv_misc2(&cd, match_param); - if (ret) - goto err_free_fc; - } - - if (match_criteria_enable & MLX5HWS_DEFINER_MATCH_CRITERIA_MISC3) { - ret = hws_definer_conv_misc3(&cd, match_param); - if (ret) - goto err_free_fc; - } - - if (match_criteria_enable & MLX5HWS_DEFINER_MATCH_CRITERIA_MISC4) { - ret = hws_definer_conv_misc4(&cd, match_param); - if (ret) - goto err_free_fc; - } - - if (match_criteria_enable & MLX5HWS_DEFINER_MATCH_CRITERIA_MISC5) { - ret = hws_definer_conv_misc5(&cd, match_param); - if (ret) - goto err_free_fc; - } - - /* Allocate fc array on mt */ - compressed_fc = hws_definer_alloc_compressed_fc(fc); - if (!compressed_fc) { - mlx5hws_err(ctx, - "Convert to compressed fc: failed to set field copy to match template\n"); - goto err_free_fc; - } - *fc_sz = hws_definer_get_fc_size(fc); - -err_free_fc: - kfree(fc); - return compressed_fc; -} - static int hws_definer_find_byte_in_tag(struct mlx5hws_definer *definer, u32 hl_byte_off, @@ -2067,7 +1993,7 @@ hws_definer_copy_sel_ctrl(struct mlx5hws_definer_sel_ctrl *ctrl, static int hws_definer_find_best_match_fit(struct mlx5hws_context *ctx, struct mlx5hws_definer *definer, - u8 *hl) + u8 *hl, bool allow_jumbo) { struct mlx5hws_definer_sel_ctrl ctrl = {0}; bool found; @@ -2084,6 +2010,9 @@ hws_definer_find_best_match_fit(struct mlx5hws_context *ctx, return 0; } + if (!allow_jumbo) + return -E2BIG; + /* Try to create a full/limited jumbo definer */ ctrl.allowed_full_dw = ctx->caps->full_dw_jumbo_support ? DW_SELECTORS : DW_SELECTORS_MATCH; @@ -2160,7 +2089,8 @@ int mlx5hws_definer_compare(struct mlx5hws_definer *definer_a, int mlx5hws_definer_calc_layout(struct mlx5hws_context *ctx, struct mlx5hws_match_template *mt, - struct mlx5hws_definer *match_definer) + struct mlx5hws_definer *match_definer, + bool allow_jumbo) { u8 *match_hl; int ret; @@ -2182,7 +2112,8 @@ mlx5hws_definer_calc_layout(struct mlx5hws_context *ctx, } /* Find the match definer layout for header layout match union */ - ret = hws_definer_find_best_match_fit(ctx, match_definer, match_hl); + ret = hws_definer_find_best_match_fit(ctx, match_definer, match_hl, + allow_jumbo); if (ret) { if (ret == -E2BIG) mlx5hws_dbg(ctx, @@ -2370,7 +2301,7 @@ int mlx5hws_definer_mt_init(struct mlx5hws_context *ctx, struct mlx5hws_definer match_layout = {0}; int ret; - ret = mlx5hws_definer_calc_layout(ctx, mt, &match_layout); + ret = mlx5hws_definer_calc_layout(ctx, mt, &match_layout, true); if (ret) { mlx5hws_err(ctx, "Failed to calculate matcher definer layout\n"); return ret; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/definer.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/definer.h index 62da55389331..141f3eb2e307 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/definer.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/definer.h @@ -823,13 +823,8 @@ void mlx5hws_definer_free(struct mlx5hws_context *ctx, int mlx5hws_definer_calc_layout(struct mlx5hws_context *ctx, struct mlx5hws_match_template *mt, - struct mlx5hws_definer *match_definer); - -struct mlx5hws_definer_fc * -mlx5hws_definer_conv_match_params_to_compressed_fc(struct mlx5hws_context *ctx, - u8 match_criteria_enable, - u32 *match_param, - int *fc_sz); + struct mlx5hws_definer *match_definer, + bool allow_jumbo); const char *mlx5hws_definer_fname_to_str(enum mlx5hws_definer_fname fname); |