path: root/drivers/net/ethernet/mellanox/mlx5/core/lib/ipsec_fs_roce.c

// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */

#include "fs_core.h"
#include "fs_cmd.h"
#include "en.h"
#include "lib/ipsec_fs_roce.h"
#include "mlx5_core.h"
#include <linux/random.h>

struct mlx5_ipsec_miss {
	struct mlx5_flow_group *group;
	struct mlx5_flow_handle *rule;
};

struct mlx5_ipsec_rx_roce {
	struct mlx5_flow_group *g;
	struct mlx5_flow_table *ft;
	struct mlx5_flow_handle *rule;
	struct mlx5_ipsec_miss roce_miss;
	struct mlx5_flow_table *nic_master_ft;
	struct mlx5_flow_group *nic_master_group;
	struct mlx5_flow_handle *nic_master_rule;
	struct mlx5_flow_table *goto_alias_ft;
	u32 alias_id;

	struct mlx5_flow_table *ft_rdma;
	struct mlx5_flow_namespace *ns_rdma;
};

struct mlx5_ipsec_tx_roce {
	struct mlx5_flow_group *g;
	struct mlx5_flow_table *ft;
	struct mlx5_flow_handle *rule;
	struct mlx5_flow_table *goto_alias_ft;
	u32 alias_id;
	struct mlx5_flow_namespace *ns;
};

struct mlx5_ipsec_fs {
	struct mlx5_ipsec_rx_roce ipv4_rx;
	struct mlx5_ipsec_rx_roce ipv6_rx;
	struct mlx5_ipsec_tx_roce tx;
	struct mlx5_devcom_comp_dev **devcom;
};

static void ipsec_fs_roce_setup_udp_dport(struct mlx5_flow_spec *spec,
					  u16 dport)
{
	spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_UDP);
	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.udp_dport);
	MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, dport);
}

static bool ipsec_fs_create_alias_supported(struct mlx5_core_dev *mdev,
					    struct mlx5_core_dev *master_mdev)
{
	u64 obj_allowed_m = MLX5_CAP_GEN_2_64(master_mdev, allowed_object_for_other_vhca_access);
	u32 obj_supp_m = MLX5_CAP_GEN_2(master_mdev, cross_vhca_object_to_object_supported);
	u64 obj_allowed = MLX5_CAP_GEN_2_64(mdev, allowed_object_for_other_vhca_access);
	u32 obj_supp = MLX5_CAP_GEN_2(mdev, cross_vhca_object_to_object_supported);

	if (!(obj_supp &
	    MLX5_CROSS_VHCA_OBJ_TO_OBJ_SUPPORTED_LOCAL_FLOW_TABLE_TO_REMOTE_FLOW_TABLE_MISS) ||
	    !(obj_supp_m &
	    MLX5_CROSS_VHCA_OBJ_TO_OBJ_SUPPORTED_LOCAL_FLOW_TABLE_TO_REMOTE_FLOW_TABLE_MISS))
		return false;

	if (!(obj_allowed & MLX5_ALLOWED_OBJ_FOR_OTHER_VHCA_ACCESS_FLOW_TABLE) ||
	    !(obj_allowed_m & MLX5_ALLOWED_OBJ_FOR_OTHER_VHCA_ACCESS_FLOW_TABLE))
		return false;

	return true;
}

static int ipsec_fs_create_aliased_ft(struct mlx5_core_dev *ibv_owner,
				      struct mlx5_core_dev *ibv_allowed,
				      struct mlx5_flow_table *ft,
				      u32 *obj_id)
{
	u32 aliased_object_id = (ft->type << FT_ID_FT_TYPE_OFFSET) | ft->id;
	u16 vhca_id_to_be_accessed = MLX5_CAP_GEN(ibv_owner, vhca_id);
	struct mlx5_cmd_allow_other_vhca_access_attr allow_attr = {};
	struct mlx5_cmd_alias_obj_create_attr alias_attr = {};
	char key[ACCESS_KEY_LEN];
	int ret;
	int i;

	if (!ipsec_fs_create_alias_supported(ibv_owner, ibv_allowed))
		return -EOPNOTSUPP;

	for (i = 0; i < ACCESS_KEY_LEN; i++)
		key[i] = get_random_u64() & 0xFF;

	memcpy(allow_attr.access_key, key, ACCESS_KEY_LEN);
	allow_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS;
	allow_attr.obj_id = aliased_object_id;

	ret = mlx5_cmd_allow_other_vhca_access(ibv_owner, &allow_attr);
	if (ret) {
		mlx5_core_err(ibv_owner, "Failed to allow other vhca access err=%d\n",
			      ret);
		return ret;
	}

	memcpy(alias_attr.access_key, key, ACCESS_KEY_LEN);
	alias_attr.obj_id = aliased_object_id;
	alias_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS;
	alias_attr.vhca_id = vhca_id_to_be_accessed;
	ret = mlx5_cmd_alias_obj_create(ibv_allowed, &alias_attr, obj_id);
	if (ret) {
		mlx5_core_err(ibv_allowed, "Failed to create alias object err=%d\n",
			      ret);
		return ret;
	}

	return 0;
}

static int
ipsec_fs_roce_rx_rule_setup(struct mlx5_core_dev *mdev,
			    struct mlx5_flow_destination *default_dst,
			    struct mlx5_ipsec_rx_roce *roce)
{
	bool is_mpv_slave = mlx5_core_is_mp_slave(mdev);
	struct mlx5_flow_destination dst = {};
	MLX5_DECLARE_FLOW_ACT(flow_act);
	struct mlx5_flow_handle *rule;
	struct mlx5_flow_spec *spec;
	int err = 0;

	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
	if (!spec)
		return -ENOMEM;

	ipsec_fs_roce_setup_udp_dport(spec, ROCE_V2_UDP_DPORT);

	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
	if (is_mpv_slave) {
		dst.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
		dst.ft = roce->goto_alias_ft;
	} else {
		dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
		dst.ft = roce->ft_rdma;
	}
	rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, &dst, 1);
	if (IS_ERR(rule)) {
		err = PTR_ERR(rule);
		mlx5_core_err(mdev, "Fail to add RX RoCE IPsec rule err=%d\n",
			      err);
		goto out;
	}

	roce->rule = rule;

	memset(spec, 0, sizeof(*spec));
	rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, default_dst, 1);
	if (IS_ERR(rule)) {
		err = PTR_ERR(rule);
		mlx5_core_err(mdev, "Fail to add RX RoCE IPsec miss rule err=%d\n",
			      err);
		goto fail_add_default_rule;
	}

	roce->roce_miss.rule = rule;

	if (!is_mpv_slave)
		goto out;

	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
	dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
	dst.ft = roce->ft_rdma;
	rule = mlx5_add_flow_rules(roce->nic_master_ft, NULL, &flow_act, &dst,
				   1);
	if (IS_ERR(rule)) {
		err = PTR_ERR(rule);
		mlx5_core_err(mdev, "Fail to add RX RoCE IPsec rule for alias err=%d\n",
			      err);
		goto fail_add_nic_master_rule;
	}
	roce->nic_master_rule = rule;

	kvfree(spec);
	return 0;

fail_add_nic_master_rule:
	mlx5_del_flow_rules(roce->roce_miss.rule);
fail_add_default_rule:
	mlx5_del_flow_rules(roce->rule);
out:
	kvfree(spec);
	return err;
}

static int ipsec_fs_roce_tx_rule_setup(struct mlx5_core_dev *mdev,
				       struct mlx5_ipsec_tx_roce *roce,
				       struct mlx5_flow_table *pol_ft)
{
	struct mlx5_flow_destination dst = {};
	MLX5_DECLARE_FLOW_ACT(flow_act);
	struct mlx5_flow_handle *rule;
	int err = 0;

	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
	dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
	dst.ft = pol_ft;
	rule = mlx5_add_flow_rules(roce->ft, NULL, &flow_act, &dst,
				   1);
	if (IS_ERR(rule)) {
		err = PTR_ERR(rule);
		mlx5_core_err(mdev, "Fail to add TX RoCE IPsec rule err=%d\n",
			      err);
		goto out;
	}
	roce->rule = rule;

out:
	return err;
}

static int ipsec_fs_roce_tx_mpv_rule_setup(struct mlx5_core_dev *mdev,
					   struct mlx5_ipsec_tx_roce *roce,
					   struct mlx5_flow_table *pol_ft)
{
	struct mlx5_flow_destination dst = {};
	MLX5_DECLARE_FLOW_ACT(flow_act);
	struct mlx5_flow_handle *rule;
	struct mlx5_flow_spec *spec;
	int err = 0;

	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
	if (!spec)
		return -ENOMEM;

	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters.source_vhca_port);
	MLX5_SET(fte_match_param, spec->match_value, misc_parameters.source_vhca_port,
		 MLX5_CAP_GEN(mdev, native_port_num));

	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
	dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
	dst.ft = roce->goto_alias_ft;
	rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, &dst, 1);
	if (IS_ERR(rule)) {
		err = PTR_ERR(rule);
		mlx5_core_err(mdev, "Fail to add TX RoCE IPsec rule err=%d\n",
			      err);
		goto out;
	}
	roce->rule = rule;

	/* No need for miss rule, since on miss we go to next PRIO, in which
	 * if master is configured, he will catch the traffic to go to his
	 * encryption table.
	 */

out:
	kvfree(spec);
	return err;
}

#define MLX5_TX_ROCE_GROUP_SIZE BIT(0)
#define MLX5_IPSEC_RDMA_TX_FT_LEVEL 0
#define MLX5_IPSEC_NIC_GOTO_ALIAS_FT_LEVEL 3 /* Since last used level in NIC ipsec is 2 */

static int ipsec_fs_roce_tx_mpv_create_ft(struct mlx5_core_dev *mdev,
					  struct mlx5_ipsec_tx_roce *roce,
					  struct mlx5_flow_table *pol_ft,
					  struct mlx5e_priv *peer_priv)
{
	struct mlx5_flow_namespace *roce_ns, *nic_ns;
	struct mlx5_flow_table_attr ft_attr = {};
	struct mlx5_flow_table next_ft;
	struct mlx5_flow_table *ft;
	int err;

	roce_ns = mlx5_get_flow_namespace(peer_priv->mdev, MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC);
	if (!roce_ns)
		return -EOPNOTSUPP;

	nic_ns = mlx5_get_flow_namespace(peer_priv->mdev, MLX5_FLOW_NAMESPACE_EGRESS_IPSEC);
	if (!nic_ns)
		return -EOPNOTSUPP;

	err = ipsec_fs_create_aliased_ft(mdev, peer_priv->mdev, pol_ft, &roce->alias_id);
	if (err)
		return err;

	next_ft.id = roce->alias_id;
	ft_attr.max_fte = 1;
	ft_attr.next_ft = &next_ft;
	ft_attr.level = MLX5_IPSEC_NIC_GOTO_ALIAS_FT_LEVEL;
	ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED;
	ft = mlx5_create_flow_table(nic_ns, &ft_attr);
	if (IS_ERR(ft)) {
		err = PTR_ERR(ft);
		mlx5_core_err(mdev, "Fail to create RoCE IPsec goto alias ft err=%d\n", err);
		goto destroy_alias;
	}

	roce->goto_alias_ft = ft;

	memset(&ft_attr, 0, sizeof(ft_attr));
	ft_attr.max_fte = 1;
	ft_attr.level = MLX5_IPSEC_RDMA_TX_FT_LEVEL;
	ft = mlx5_create_flow_table(roce_ns, &ft_attr);
	if (IS_ERR(ft)) {
		err = PTR_ERR(ft);
		mlx5_core_err(mdev, "Fail to create RoCE IPsec tx ft err=%d\n", err);
		goto destroy_alias_ft;
	}

	roce->ft = ft;

	return 0;

destroy_alias_ft:
	mlx5_destroy_flow_table(roce->goto_alias_ft);
destroy_alias:
	mlx5_cmd_alias_obj_destroy(peer_priv->mdev, roce->alias_id,
				   MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS);
	return err;
}

static int ipsec_fs_roce_tx_mpv_create_group_rules(struct mlx5_core_dev *mdev,
						   struct mlx5_ipsec_tx_roce *roce,
						   struct mlx5_flow_table *pol_ft,
						   u32 *in)
{
	struct mlx5_flow_group *g;
	int ix = 0;
	int err;
	u8 *mc;

	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
	MLX5_SET_TO_ONES(fte_match_param, mc, misc_parameters.source_vhca_port);
	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_MISC_PARAMETERS);

	MLX5_SET_CFG(in, start_flow_index, ix);
	ix += MLX5_TX_ROCE_GROUP_SIZE;
	MLX5_SET_CFG(in, end_flow_index, ix - 1);
	g = mlx5_create_flow_group(roce->ft, in);
	if (IS_ERR(g)) {
		err = PTR_ERR(g);
		mlx5_core_err(mdev, "Fail to create RoCE IPsec tx group err=%d\n", err);
		return err;
	}
	roce->g = g;

	err = ipsec_fs_roce_tx_mpv_rule_setup(mdev, roce, pol_ft);
	if (err) {
		mlx5_core_err(mdev, "Fail to create RoCE IPsec tx rules err=%d\n", err);
		goto destroy_group;
	}

	return 0;

destroy_group:
	mlx5_destroy_flow_group(roce->g);
	return err;
}

static int ipsec_fs_roce_tx_mpv_create(struct mlx5_core_dev *mdev,
				       struct mlx5_ipsec_fs *ipsec_roce,
				       struct mlx5_flow_table *pol_ft,
				       u32 *in)
{
	struct mlx5_devcom_comp_dev *tmp = NULL;
	struct mlx5_ipsec_tx_roce *roce;
	struct mlx5e_priv *peer_priv;
	int err;

	if (!mlx5_devcom_for_each_peer_begin(*ipsec_roce->devcom))
		return -EOPNOTSUPP;

	peer_priv = mlx5_devcom_get_next_peer_data(*ipsec_roce->devcom, &tmp);
	if (!peer_priv) {
		err = -EOPNOTSUPP;
		goto release_peer;
	}

	roce = &ipsec_roce->tx;

	err = ipsec_fs_roce_tx_mpv_create_ft(mdev, roce, pol_ft, peer_priv);
	if (err) {
		mlx5_core_err(mdev, "Fail to create RoCE IPsec tables err=%d\n", err);
		goto release_peer;
	}

	err = ipsec_fs_roce_tx_mpv_create_group_rules(mdev, roce, pol_ft, in);
	if (err) {
		mlx5_core_err(mdev, "Fail to create RoCE IPsec tx group/rule err=%d\n", err);
		goto destroy_tables;
	}

	mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom);
	return 0;

destroy_tables:
	mlx5_destroy_flow_table(roce->ft);
	mlx5_destroy_flow_table(roce->goto_alias_ft);
	mlx5_cmd_alias_obj_destroy(peer_priv->mdev, roce->alias_id,
				   MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS);
release_peer:
	mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom);
	return err;
}

static void roce_rx_mpv_destroy_tables(struct mlx5_core_dev *mdev, struct mlx5_ipsec_rx_roce *roce)
{
	mlx5_destroy_flow_table(roce->goto_alias_ft);
	mlx5_cmd_alias_obj_destroy(mdev, roce->alias_id,
				   MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS);
	mlx5_destroy_flow_group(roce->nic_master_group);
	mlx5_destroy_flow_table(roce->nic_master_ft);
}

#define MLX5_RX_ROCE_GROUP_SIZE BIT(0)
#define MLX5_IPSEC_RX_IPV4_FT_LEVEL 3
#define MLX5_IPSEC_RX_IPV6_FT_LEVEL 2

static int ipsec_fs_roce_rx_mpv_create(struct mlx5_core_dev *mdev,
				       struct mlx5_ipsec_fs *ipsec_roce,
				       struct mlx5_flow_namespace *ns,
				       u32 family, u32 level, u32 prio)
{
	struct mlx5_flow_namespace *roce_ns, *nic_ns;
	struct mlx5_flow_table_attr ft_attr = {};
	struct mlx5_devcom_comp_dev *tmp = NULL;
	struct mlx5_ipsec_rx_roce *roce;
	struct mlx5_flow_table next_ft;
	struct mlx5_flow_table *ft;
	struct mlx5_flow_group *g;
	struct mlx5e_priv *peer_priv;
	int ix = 0;
	u32 *in;
	int err;

	roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx :
				     &ipsec_roce->ipv6_rx;

	if (!mlx5_devcom_for_each_peer_begin(*ipsec_roce->devcom))
		return -EOPNOTSUPP;

	peer_priv = mlx5_devcom_get_next_peer_data(*ipsec_roce->devcom, &tmp);
	if (!peer_priv) {
		err = -EOPNOTSUPP;
		goto release_peer;
	}

	roce_ns = mlx5_get_flow_namespace(peer_priv->mdev, MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC);
	if (!roce_ns) {
		err = -EOPNOTSUPP;
		goto release_peer;
	}

	nic_ns = mlx5_get_flow_namespace(peer_priv->mdev, MLX5_FLOW_NAMESPACE_KERNEL);
	if (!nic_ns) {
		err = -EOPNOTSUPP;
		goto release_peer;
	}

	in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL);
	if (!in) {
		err = -ENOMEM;
		goto release_peer;
	}

	ft_attr.level = (family == AF_INET) ? MLX5_IPSEC_RX_IPV4_FT_LEVEL :
					      MLX5_IPSEC_RX_IPV6_FT_LEVEL;
	ft_attr.max_fte = 1;
	ft = mlx5_create_flow_table(roce_ns, &ft_attr);
	if (IS_ERR(ft)) {
		err = PTR_ERR(ft);
		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx ft at rdma master err=%d\n", err);
		goto free_in;
	}

	roce->ft_rdma = ft;

	ft_attr.max_fte = 1;
	ft_attr.prio = prio;
	ft_attr.level = level + 2;
	ft = mlx5_create_flow_table(nic_ns, &ft_attr);
	if (IS_ERR(ft)) {
		err = PTR_ERR(ft);
		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx ft at NIC master err=%d\n", err);
		goto destroy_ft_rdma;
	}
	roce->nic_master_ft = ft;

	MLX5_SET_CFG(in, start_flow_index, ix);
	ix += 1;
	MLX5_SET_CFG(in, end_flow_index, ix - 1);
	g = mlx5_create_flow_group(roce->nic_master_ft, in);
	if (IS_ERR(g)) {
		err = PTR_ERR(g);
		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx group aliased err=%d\n", err);
		goto destroy_nic_master_ft;
	}
	roce->nic_master_group = g;

	err = ipsec_fs_create_aliased_ft(peer_priv->mdev, mdev, roce->nic_master_ft,
					 &roce->alias_id);
	if (err) {
		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx alias FT err=%d\n", err);
		goto destroy_group;
	}

	next_ft.id = roce->alias_id;
	ft_attr.max_fte = 1;
	ft_attr.prio = prio;
	ft_attr.level = roce->ft->level + 1;
	ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED;
	ft_attr.next_ft = &next_ft;
	ft = mlx5_create_flow_table(ns, &ft_attr);
	if (IS_ERR(ft)) {
		err = PTR_ERR(ft);
		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx ft at NIC slave err=%d\n", err);
		goto destroy_alias;
	}
	roce->goto_alias_ft = ft;

	kvfree(in);
	mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom);
	return 0;

destroy_alias:
	mlx5_cmd_alias_obj_destroy(mdev, roce->alias_id,
				   MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS);
destroy_group:
	mlx5_destroy_flow_group(roce->nic_master_group);
destroy_nic_master_ft:
	mlx5_destroy_flow_table(roce->nic_master_ft);
destroy_ft_rdma:
	mlx5_destroy_flow_table(roce->ft_rdma);
free_in:
	kvfree(in);
release_peer:
	mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom);
	return err;
}

void mlx5_ipsec_fs_roce_tx_destroy(struct mlx5_ipsec_fs *ipsec_roce,
				   struct mlx5_core_dev *mdev)
{
	struct mlx5_devcom_comp_dev *tmp = NULL;
	struct mlx5_ipsec_tx_roce *tx_roce;
	struct mlx5e_priv *peer_priv;

	if (!ipsec_roce)
		return;

	tx_roce = &ipsec_roce->tx;

	mlx5_del_flow_rules(tx_roce->rule);
	mlx5_destroy_flow_group(tx_roce->g);
	mlx5_destroy_flow_table(tx_roce->ft);

	if (!mlx5_core_is_mp_slave(mdev))
		return;

	if (!mlx5_devcom_for_each_peer_begin(*ipsec_roce->devcom))
		return;

	peer_priv = mlx5_devcom_get_next_peer_data(*ipsec_roce->devcom, &tmp);
	if (!peer_priv) {
		mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom);
		return;
	}

	mlx5_destroy_flow_table(tx_roce->goto_alias_ft);
	mlx5_cmd_alias_obj_destroy(peer_priv->mdev, tx_roce->alias_id,
				   MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS);
	mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom);
}

int mlx5_ipsec_fs_roce_tx_create(struct mlx5_core_dev *mdev,
				 struct mlx5_ipsec_fs *ipsec_roce,
				 struct mlx5_flow_table *pol_ft)
{
	struct mlx5_flow_table_attr ft_attr = {};
	struct mlx5_ipsec_tx_roce *roce;
	struct mlx5_flow_table *ft;
	struct mlx5_flow_group *g;
	int ix = 0;
	int err;
	u32 *in;

	if (!ipsec_roce)
		return 0;

	roce = &ipsec_roce->tx;

	in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL);
	if (!in)
		return -ENOMEM;

	if (mlx5_core_is_mp_slave(mdev)) {
		err = ipsec_fs_roce_tx_mpv_create(mdev, ipsec_roce, pol_ft, in);
		goto free_in;
	}

	ft_attr.max_fte = 1;
	ft_attr.prio = 1;
	ft_attr.level = MLX5_IPSEC_RDMA_TX_FT_LEVEL;
	ft = mlx5_create_flow_table(roce->ns, &ft_attr);
	if (IS_ERR(ft)) {
		err = PTR_ERR(ft);
		mlx5_core_err(mdev, "Fail to create RoCE IPsec tx ft err=%d\n", err);
		goto free_in;
	}

	roce->ft = ft;

	MLX5_SET_CFG(in, start_flow_index, ix);
	ix += MLX5_TX_ROCE_GROUP_SIZE;
	MLX5_SET_CFG(in, end_flow_index, ix - 1);
	g = mlx5_create_flow_group(ft, in);
	if (IS_ERR(g)) {
		err = PTR_ERR(g);
		mlx5_core_err(mdev, "Fail to create RoCE IPsec tx group err=%d\n", err);
		goto destroy_table;
	}
	roce->g = g;

	err = ipsec_fs_roce_tx_rule_setup(mdev, roce, pol_ft);
	if (err) {
		mlx5_core_err(mdev, "Fail to create RoCE IPsec tx rules err=%d\n", err);
		goto destroy_group;
	}

	kvfree(in);
	return 0;

destroy_group:
	mlx5_destroy_flow_group(roce->g);
destroy_table:
	mlx5_destroy_flow_table(ft);
free_in:
	kvfree(in);
	return err;
}

struct mlx5_flow_table *mlx5_ipsec_fs_roce_ft_get(struct mlx5_ipsec_fs *ipsec_roce, u32 family)
{
	struct mlx5_ipsec_rx_roce *rx_roce;

	if (!ipsec_roce)
		return NULL;

	rx_roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx :
					&ipsec_roce->ipv6_rx;

	return rx_roce->ft;
}

void mlx5_ipsec_fs_roce_rx_destroy(struct mlx5_ipsec_fs *ipsec_roce, u32 family,
				   struct mlx5_core_dev *mdev)
{
	bool is_mpv_slave = mlx5_core_is_mp_slave(mdev);
	struct mlx5_ipsec_rx_roce *rx_roce;

	if (!ipsec_roce)
		return;

	rx_roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx :
					&ipsec_roce->ipv6_rx;

	if (is_mpv_slave)
		mlx5_del_flow_rules(rx_roce->nic_master_rule);
	mlx5_del_flow_rules(rx_roce->roce_miss.rule);
	mlx5_del_flow_rules(rx_roce->rule);
	if (is_mpv_slave)
		roce_rx_mpv_destroy_tables(mdev, rx_roce);
	mlx5_destroy_flow_table(rx_roce->ft_rdma);
	mlx5_destroy_flow_group(rx_roce->roce_miss.group);
	mlx5_destroy_flow_group(rx_roce->g);
	mlx5_destroy_flow_table(rx_roce->ft);
}

int mlx5_ipsec_fs_roce_rx_create(struct mlx5_core_dev *mdev,
				 struct mlx5_ipsec_fs *ipsec_roce,
				 struct mlx5_flow_namespace *ns,
				 struct mlx5_flow_destination *default_dst,
				 u32 family, u32 level, u32 prio)
{
	bool is_mpv_slave = mlx5_core_is_mp_slave(mdev);
	struct mlx5_flow_table_attr ft_attr = {};
	struct mlx5_ipsec_rx_roce *roce;
	struct mlx5_flow_table *ft;
	struct mlx5_flow_group *g;
	void *outer_headers_c;
	int ix = 0;
	u32 *in;
	int err;
	u8 *mc;

	if (!ipsec_roce)
		return 0;

	roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx :
				     &ipsec_roce->ipv6_rx;

	ft_attr.max_fte = 2;
	ft_attr.level = level;
	ft_attr.prio = prio;
	ft = mlx5_create_flow_table(ns, &ft_attr);
	if (IS_ERR(ft)) {
		err = PTR_ERR(ft);
		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx ft at nic err=%d\n", err);
		return err;
	}

	roce->ft = ft;

	in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL);
	if (!in) {
		err = -ENOMEM;
		goto fail_nomem;
	}

	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
	outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
	MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol);
	MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport);

	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
	MLX5_SET_CFG(in, start_flow_index, ix);
	ix += MLX5_RX_ROCE_GROUP_SIZE;
	MLX5_SET_CFG(in, end_flow_index, ix - 1);
	g = mlx5_create_flow_group(ft, in);
	if (IS_ERR(g)) {
		err = PTR_ERR(g);
		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx group at nic err=%d\n", err);
		goto fail_group;
	}
	roce->g = g;

	memset(in, 0, MLX5_ST_SZ_BYTES(create_flow_group_in));
	MLX5_SET_CFG(in, start_flow_index, ix);
	ix += MLX5_RX_ROCE_GROUP_SIZE;
	MLX5_SET_CFG(in, end_flow_index, ix - 1);
	g = mlx5_create_flow_group(ft, in);
	if (IS_ERR(g)) {
		err = PTR_ERR(g);
		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx miss group at nic err=%d\n", err);
		goto fail_mgroup;
	}
	roce->roce_miss.group = g;

	if (is_mpv_slave) {
		err = ipsec_fs_roce_rx_mpv_create(mdev, ipsec_roce, ns, family, level, prio);
		if (err) {
			mlx5_core_err(mdev, "Fail to create RoCE IPsec rx alias err=%d\n", err);
			goto fail_mpv_create;
		}
	} else {
		memset(&ft_attr, 0, sizeof(ft_attr));
		if (family == AF_INET)
			ft_attr.level = 1;
		ft_attr.max_fte = 1;
		ft = mlx5_create_flow_table(roce->ns_rdma, &ft_attr);
		if (IS_ERR(ft)) {
			err = PTR_ERR(ft);
			mlx5_core_err(mdev,
				      "Fail to create RoCE IPsec rx ft at rdma err=%d\n", err);
			goto fail_rdma_table;
		}

		roce->ft_rdma = ft;
	}

	err = ipsec_fs_roce_rx_rule_setup(mdev, default_dst, roce);
	if (err) {
		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx rules err=%d\n", err);
		goto fail_setup_rule;
	}

	kvfree(in);
	return 0;

fail_setup_rule:
	if (is_mpv_slave)
		roce_rx_mpv_destroy_tables(mdev, roce);
	mlx5_destroy_flow_table(roce->ft_rdma);
fail_mpv_create:
fail_rdma_table:
	mlx5_destroy_flow_group(roce->roce_miss.group);
fail_mgroup:
	mlx5_destroy_flow_group(roce->g);
fail_group:
	kvfree(in);
fail_nomem:
	mlx5_destroy_flow_table(roce->ft);
	return err;
}

void mlx5_ipsec_fs_roce_cleanup(struct mlx5_ipsec_fs *ipsec_roce)
{
	kfree(ipsec_roce);
}

struct mlx5_ipsec_fs *mlx5_ipsec_fs_roce_init(struct mlx5_core_dev *mdev,
					      struct mlx5_devcom_comp_dev **devcom)
{
	struct mlx5_ipsec_fs *roce_ipsec;
	struct mlx5_flow_namespace *ns;

	ns = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC);
	if (!ns) {
		mlx5_core_err(mdev, "Failed to get RoCE rx ns\n");
		return NULL;
	}

	roce_ipsec = kzalloc(sizeof(*roce_ipsec), GFP_KERNEL);
	if (!roce_ipsec)
		return NULL;

	roce_ipsec->ipv4_rx.ns_rdma = ns;
	roce_ipsec->ipv6_rx.ns_rdma = ns;

	ns = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC);
	if (!ns) {
		mlx5_core_err(mdev, "Failed to get RoCE tx ns\n");
		goto err_tx;
	}

	roce_ipsec->tx.ns = ns;

	roce_ipsec->devcom = devcom;

	return roce_ipsec;

err_tx:
	kfree(roce_ipsec);
	return NULL;
}