diff options
22 files changed, 4792 insertions, 90 deletions
diff --git a/drivers/net/ethernet/intel/iavf/iavf.h b/drivers/net/ethernet/intel/iavf/iavf.h index 23a6557fc3db..48cd1d06761c 100644 --- a/drivers/net/ethernet/intel/iavf/iavf.h +++ b/drivers/net/ethernet/intel/iavf/iavf.h @@ -33,6 +33,7 @@ #include <net/udp.h> #include <net/tc_act/tc_gact.h> #include <net/tc_act/tc_mirred.h> +#include <net/tc_act/tc_skbedit.h> #include "iavf_type.h" #include <linux/avf/virtchnl.h> @@ -393,6 +394,8 @@ struct iavf_adapter { VIRTCHNL_VF_OFFLOAD_VLAN_V2) #define CRC_OFFLOAD_ALLOWED(_a) ((_a)->vf_res->vf_cap_flags & \ VIRTCHNL_VF_OFFLOAD_CRC) +#define TC_U32_SUPPORT(_a) ((_a)->vf_res->vf_cap_flags & \ + VIRTCHNL_VF_OFFLOAD_TC_U32) #define VLAN_V2_FILTERING_ALLOWED(_a) \ (VLAN_V2_ALLOWED((_a)) && \ ((_a)->vlan_v2_caps.filtering.filtering_support.outer || \ @@ -437,6 +440,7 @@ struct iavf_adapter { #define IAVF_MAX_FDIR_FILTERS 128 /* max allowed Flow Director filters */ u16 fdir_active_fltr; + u16 raw_fdir_active_fltr; struct list_head fdir_list_head; spinlock_t fdir_fltr_lock; /* protect the Flow Director filter list */ @@ -444,6 +448,32 @@ struct iavf_adapter { spinlock_t adv_rss_lock; /* protect the RSS management list */ }; +/* Must be called with fdir_fltr_lock lock held */ +static inline bool iavf_fdir_max_reached(struct iavf_adapter *adapter) +{ + return adapter->fdir_active_fltr + adapter->raw_fdir_active_fltr >= + IAVF_MAX_FDIR_FILTERS; +} + +static inline void +iavf_inc_fdir_active_fltr(struct iavf_adapter *adapter, + struct iavf_fdir_fltr *fltr) +{ + if (iavf_is_raw_fdir(fltr)) + adapter->raw_fdir_active_fltr++; + else + adapter->fdir_active_fltr++; +} + +static inline void +iavf_dec_fdir_active_fltr(struct iavf_adapter *adapter, + struct iavf_fdir_fltr *fltr) +{ + if (iavf_is_raw_fdir(fltr)) + adapter->raw_fdir_active_fltr--; + else + adapter->fdir_active_fltr--; +} /* Ethtool Private Flags */ diff --git a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c index 52273f7eab2c..74a1e9fe1821 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c +++ b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c @@ -927,7 +927,7 @@ iavf_get_ethtool_fdir_entry(struct iavf_adapter *adapter, spin_lock_bh(&adapter->fdir_fltr_lock); - rule = iavf_find_fdir_fltr_by_loc(adapter, fsp->location); + rule = iavf_find_fdir_fltr(adapter, false, fsp->location); if (!rule) { ret = -EINVAL; goto release_lock; @@ -1072,6 +1072,9 @@ iavf_get_fdir_fltr_ids(struct iavf_adapter *adapter, struct ethtool_rxnfc *cmd, spin_lock_bh(&adapter->fdir_fltr_lock); list_for_each_entry(fltr, &adapter->fdir_list_head, list) { + if (iavf_is_raw_fdir(fltr)) + continue; + if (cnt == cmd->rule_cnt) { val = -EMSGSIZE; goto release_lock; @@ -1263,15 +1266,7 @@ static int iavf_add_fdir_ethtool(struct iavf_adapter *adapter, struct ethtool_rx return -EINVAL; spin_lock_bh(&adapter->fdir_fltr_lock); - if (adapter->fdir_active_fltr >= IAVF_MAX_FDIR_FILTERS) { - spin_unlock_bh(&adapter->fdir_fltr_lock); - dev_err(&adapter->pdev->dev, - "Unable to add Flow Director filter because VF reached the limit of max allowed filters (%u)\n", - IAVF_MAX_FDIR_FILTERS); - return -ENOSPC; - } - - if (iavf_find_fdir_fltr_by_loc(adapter, fsp->location)) { + if (iavf_find_fdir_fltr(adapter, false, fsp->location)) { dev_err(&adapter->pdev->dev, "Failed to add Flow Director filter, it already exists\n"); spin_unlock_bh(&adapter->fdir_fltr_lock); return -EEXIST; @@ -1291,23 +1286,10 @@ static int iavf_add_fdir_ethtool(struct iavf_adapter *adapter, struct ethtool_rx } err = iavf_add_fdir_fltr_info(adapter, fsp, fltr); - if (err) - goto ret; - - spin_lock_bh(&adapter->fdir_fltr_lock); - iavf_fdir_list_add_fltr(adapter, fltr); - adapter->fdir_active_fltr++; - - if (adapter->link_up) - fltr->state = IAVF_FDIR_FLTR_ADD_REQUEST; - else - fltr->state = IAVF_FDIR_FLTR_INACTIVE; - spin_unlock_bh(&adapter->fdir_fltr_lock); + if (!err) + err = iavf_fdir_add_fltr(adapter, fltr); - if (adapter->link_up) - iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ADD_FDIR_FILTER); -ret: - if (err && fltr) + if (err) kfree(fltr); mutex_unlock(&adapter->crit_lock); @@ -1324,34 +1306,11 @@ ret: static int iavf_del_fdir_ethtool(struct iavf_adapter *adapter, struct ethtool_rxnfc *cmd) { struct ethtool_rx_flow_spec *fsp = (struct ethtool_rx_flow_spec *)&cmd->fs; - struct iavf_fdir_fltr *fltr = NULL; - int err = 0; if (!(adapter->flags & IAVF_FLAG_FDIR_ENABLED)) return -EOPNOTSUPP; - spin_lock_bh(&adapter->fdir_fltr_lock); - fltr = iavf_find_fdir_fltr_by_loc(adapter, fsp->location); - if (fltr) { - if (fltr->state == IAVF_FDIR_FLTR_ACTIVE) { - fltr->state = IAVF_FDIR_FLTR_DEL_REQUEST; - } else if (fltr->state == IAVF_FDIR_FLTR_INACTIVE) { - list_del(&fltr->list); - kfree(fltr); - adapter->fdir_active_fltr--; - fltr = NULL; - } else { - err = -EBUSY; - } - } else if (adapter->fdir_active_fltr) { - err = -EINVAL; - } - spin_unlock_bh(&adapter->fdir_fltr_lock); - - if (fltr && fltr->state == IAVF_FDIR_FLTR_DEL_REQUEST) - iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_DEL_FDIR_FILTER); - - return err; + return iavf_fdir_del_fltr(adapter, false, fsp->location); } /** diff --git a/drivers/net/ethernet/intel/iavf/iavf_fdir.c b/drivers/net/ethernet/intel/iavf/iavf_fdir.c index 2d47b0b4640e..a1b3b44cc14a 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_fdir.c +++ b/drivers/net/ethernet/intel/iavf/iavf_fdir.c @@ -796,6 +796,9 @@ bool iavf_fdir_is_dup_fltr(struct iavf_adapter *adapter, struct iavf_fdir_fltr * spin_lock_bh(&adapter->fdir_fltr_lock); list_for_each_entry(tmp, &adapter->fdir_list_head, list) { + if (iavf_is_raw_fdir(fltr)) + continue; + if (tmp->flow_type != fltr->flow_type) continue; @@ -815,33 +818,52 @@ bool iavf_fdir_is_dup_fltr(struct iavf_adapter *adapter, struct iavf_fdir_fltr * } /** - * iavf_find_fdir_fltr_by_loc - find filter with location + * iavf_find_fdir_fltr - find FDIR filter * @adapter: pointer to the VF adapter structure - * @loc: location to find. + * @is_raw: filter type, is raw (tc u32) or not (ethtool) + * @data: data to ID the filter, type dependent * - * Returns pointer to Flow Director filter if found or null + * Returns: pointer to Flow Director filter if found or NULL. Lock must be held. */ -struct iavf_fdir_fltr *iavf_find_fdir_fltr_by_loc(struct iavf_adapter *adapter, u32 loc) +struct iavf_fdir_fltr *iavf_find_fdir_fltr(struct iavf_adapter *adapter, + bool is_raw, u32 data) { struct iavf_fdir_fltr *rule; - list_for_each_entry(rule, &adapter->fdir_list_head, list) - if (rule->loc == loc) + list_for_each_entry(rule, &adapter->fdir_list_head, list) { + if ((is_raw && rule->cls_u32_handle == data) || + (!is_raw && rule->loc == data)) return rule; + } return NULL; } /** - * iavf_fdir_list_add_fltr - add a new node to the flow director filter list + * iavf_fdir_add_fltr - add a new node to the flow director filter list * @adapter: pointer to the VF adapter structure * @fltr: filter node to add to structure + * + * Return: 0 on success or negative errno on failure. */ -void iavf_fdir_list_add_fltr(struct iavf_adapter *adapter, struct iavf_fdir_fltr *fltr) +int iavf_fdir_add_fltr(struct iavf_adapter *adapter, + struct iavf_fdir_fltr *fltr) { struct iavf_fdir_fltr *rule, *parent = NULL; + spin_lock_bh(&adapter->fdir_fltr_lock); + if (iavf_fdir_max_reached(adapter)) { + spin_unlock_bh(&adapter->fdir_fltr_lock); + dev_err(&adapter->pdev->dev, + "Unable to add Flow Director filter (limit (%u) reached)\n", + IAVF_MAX_FDIR_FILTERS); + return -ENOSPC; + } + list_for_each_entry(rule, &adapter->fdir_list_head, list) { + if (iavf_is_raw_fdir(fltr)) + break; + if (rule->loc >= fltr->loc) break; parent = rule; @@ -851,4 +873,55 @@ void iavf_fdir_list_add_fltr(struct iavf_adapter *adapter, struct iavf_fdir_fltr list_add(&fltr->list, &parent->list); else list_add(&fltr->list, &adapter->fdir_list_head); + + iavf_inc_fdir_active_fltr(adapter, fltr); + + if (adapter->link_up) + fltr->state = IAVF_FDIR_FLTR_ADD_REQUEST; + else + fltr->state = IAVF_FDIR_FLTR_INACTIVE; + spin_unlock_bh(&adapter->fdir_fltr_lock); + + if (adapter->link_up) + iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ADD_FDIR_FILTER); + + return 0; +} + +/** + * iavf_fdir_del_fltr - delete a flow director filter from the list + * @adapter: pointer to the VF adapter structure + * @is_raw: filter type, is raw (tc u32) or not (ethtool) + * @data: data to ID the filter, type dependent + * + * Return: 0 on success or negative errno on failure. + */ +int iavf_fdir_del_fltr(struct iavf_adapter *adapter, bool is_raw, u32 data) +{ + struct iavf_fdir_fltr *fltr = NULL; + int err = 0; + + spin_lock_bh(&adapter->fdir_fltr_lock); + fltr = iavf_find_fdir_fltr(adapter, is_raw, data); + + if (fltr) { + if (fltr->state == IAVF_FDIR_FLTR_ACTIVE) { + fltr->state = IAVF_FDIR_FLTR_DEL_REQUEST; + } else if (fltr->state == IAVF_FDIR_FLTR_INACTIVE) { + list_del(&fltr->list); + iavf_dec_fdir_active_fltr(adapter, fltr); + kfree(fltr); + fltr = NULL; + } else { + err = -EBUSY; + } + } else if (adapter->fdir_active_fltr) { + err = -EINVAL; + } + + if (fltr && fltr->state == IAVF_FDIR_FLTR_DEL_REQUEST) + iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_DEL_FDIR_FILTER); + + spin_unlock_bh(&adapter->fdir_fltr_lock); + return err; } diff --git a/drivers/net/ethernet/intel/iavf/iavf_fdir.h b/drivers/net/ethernet/intel/iavf/iavf_fdir.h index d31bd923ba8c..e84a5351162f 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_fdir.h +++ b/drivers/net/ethernet/intel/iavf/iavf_fdir.h @@ -117,17 +117,26 @@ struct iavf_fdir_fltr { u32 flow_id; + u32 cls_u32_handle; /* for FDIR added via tc u32 */ u32 loc; /* Rule location inside the flow table */ u32 q_index; struct virtchnl_fdir_add vc_add_msg; }; +static inline bool iavf_is_raw_fdir(struct iavf_fdir_fltr *fltr) +{ + return !fltr->vc_add_msg.rule_cfg.proto_hdrs.count; +} + int iavf_validate_fdir_fltr_masks(struct iavf_adapter *adapter, struct iavf_fdir_fltr *fltr); int iavf_fill_fdir_add_msg(struct iavf_adapter *adapter, struct iavf_fdir_fltr *fltr); void iavf_print_fdir_fltr(struct iavf_adapter *adapter, struct iavf_fdir_fltr *fltr); bool iavf_fdir_is_dup_fltr(struct iavf_adapter *adapter, struct iavf_fdir_fltr *fltr); -void iavf_fdir_list_add_fltr(struct iavf_adapter *adapter, struct iavf_fdir_fltr *fltr); -struct iavf_fdir_fltr *iavf_find_fdir_fltr_by_loc(struct iavf_adapter *adapter, u32 loc); +int iavf_fdir_add_fltr(struct iavf_adapter *adapter, + struct iavf_fdir_fltr *fltr); +int iavf_fdir_del_fltr(struct iavf_adapter *adapter, bool is_raw, u32 data); +struct iavf_fdir_fltr *iavf_find_fdir_fltr(struct iavf_adapter *adapter, + bool is_raw, u32 data); #endif /* _IAVF_FDIR_H_ */ diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c index ff11bafb3b4f..f782402cd789 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_main.c +++ b/drivers/net/ethernet/intel/iavf/iavf_main.c @@ -4013,7 +4013,7 @@ static int iavf_delete_clsflower(struct iavf_adapter *adapter, /** * iavf_setup_tc_cls_flower - flower classifier offloads - * @adapter: board private structure + * @adapter: pointer to iavf adapter structure * @cls_flower: pointer to flow_cls_offload struct with flow info */ static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter, @@ -4032,6 +4032,154 @@ static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter, } /** + * iavf_add_cls_u32 - Add U32 classifier offloads + * @adapter: pointer to iavf adapter structure + * @cls_u32: pointer to tc_cls_u32_offload struct with flow info + * + * Return: 0 on success or negative errno on failure. + */ +static int iavf_add_cls_u32(struct iavf_adapter *adapter, + struct tc_cls_u32_offload *cls_u32) +{ + struct netlink_ext_ack *extack = cls_u32->common.extack; + struct virtchnl_fdir_rule *rule_cfg; + struct virtchnl_filter_action *vact; + struct virtchnl_proto_hdrs *hdrs; + struct ethhdr *spec_h, *mask_h; + const struct tc_action *act; + struct iavf_fdir_fltr *fltr; + struct tcf_exts *exts; + unsigned int q_index; + int i, status = 0; + int off_base = 0; + + if (cls_u32->knode.link_handle) { + NL_SET_ERR_MSG_MOD(extack, "Linking not supported"); + return -EOPNOTSUPP; + } + + fltr = kzalloc(sizeof(*fltr), GFP_KERNEL); + if (!fltr) + return -ENOMEM; + + rule_cfg = &fltr->vc_add_msg.rule_cfg; + hdrs = &rule_cfg->proto_hdrs; + hdrs->count = 0; + + /* The parser lib at the PF expects the packet starting with MAC hdr */ + switch (ntohs(cls_u32->common.protocol)) { + case ETH_P_802_3: + break; + case ETH_P_IP: + spec_h = (struct ethhdr *)hdrs->raw.spec; + mask_h = (struct ethhdr *)hdrs->raw.mask; + spec_h->h_proto = htons(ETH_P_IP); + mask_h->h_proto = htons(0xFFFF); + off_base += ETH_HLEN; + break; + default: + NL_SET_ERR_MSG_MOD(extack, "Only 802_3 and ip filter protocols are supported"); + status = -EOPNOTSUPP; + goto free_alloc; + } + + for (i = 0; i < cls_u32->knode.sel->nkeys; i++) { + __be32 val, mask; + int off; + + off = off_base + cls_u32->knode.sel->keys[i].off; + val = cls_u32->knode.sel->keys[i].val; + mask = cls_u32->knode.sel->keys[i].mask; + + if (off >= sizeof(hdrs->raw.spec)) { + NL_SET_ERR_MSG_MOD(extack, "Input exceeds maximum allowed."); + status = -EINVAL; + goto free_alloc; + } + + memcpy(&hdrs->raw.spec[off], &val, sizeof(val)); + memcpy(&hdrs->raw.mask[off], &mask, sizeof(mask)); + hdrs->raw.pkt_len = off + sizeof(val); + } + + /* Only one action is allowed */ + rule_cfg->action_set.count = 1; + vact = &rule_cfg->action_set.actions[0]; + exts = cls_u32->knode.exts; + + tcf_exts_for_each_action(i, act, exts) { + /* FDIR queue */ + if (is_tcf_skbedit_rx_queue_mapping(act)) { + q_index = tcf_skbedit_rx_queue_mapping(act); + if (q_index >= adapter->num_active_queues) { + status = -EINVAL; + goto free_alloc; + } + + vact->type = VIRTCHNL_ACTION_QUEUE; + vact->act_conf.queue.index = q_index; + break; + } + + /* Drop */ + if (is_tcf_gact_shot(act)) { + vact->type = VIRTCHNL_ACTION_DROP; + break; + } + + /* Unsupported action */ + NL_SET_ERR_MSG_MOD(extack, "Unsupported action."); + status = -EOPNOTSUPP; + goto free_alloc; + } + + fltr->vc_add_msg.vsi_id = adapter->vsi.id; + fltr->cls_u32_handle = cls_u32->knode.handle; + return iavf_fdir_add_fltr(adapter, fltr); + +free_alloc: + kfree(fltr); + return status; +} + +/** + * iavf_del_cls_u32 - Delete U32 classifier offloads + * @adapter: pointer to iavf adapter structure + * @cls_u32: pointer to tc_cls_u32_offload struct with flow info + * + * Return: 0 on success or negative errno on failure. + */ +static int iavf_del_cls_u32(struct iavf_adapter *adapter, + struct tc_cls_u32_offload *cls_u32) +{ + return iavf_fdir_del_fltr(adapter, true, cls_u32->knode.handle); +} + +/** + * iavf_setup_tc_cls_u32 - U32 filter offloads + * @adapter: pointer to iavf adapter structure + * @cls_u32: pointer to tc_cls_u32_offload struct with flow info + * + * Return: 0 on success or negative errno on failure. + */ +static int iavf_setup_tc_cls_u32(struct iavf_adapter *adapter, + struct tc_cls_u32_offload *cls_u32) +{ + if (!TC_U32_SUPPORT(adapter) || !FDIR_FLTR_SUPPORT(adapter)) + return -EOPNOTSUPP; + + switch (cls_u32->command) { + case TC_CLSU32_NEW_KNODE: + case TC_CLSU32_REPLACE_KNODE: + return iavf_add_cls_u32(adapter, cls_u32); + case TC_CLSU32_DELETE_KNODE: + return iavf_del_cls_u32(adapter, cls_u32); + default: + return -EOPNOTSUPP; + } +} + +/** * iavf_setup_tc_block_cb - block callback for tc * @type: type of offload * @type_data: offload data @@ -4050,6 +4198,8 @@ static int iavf_setup_tc_block_cb(enum tc_setup_type type, void *type_data, switch (type) { case TC_SETUP_CLSFLOWER: return iavf_setup_tc_cls_flower(cb_priv, type_data); + case TC_SETUP_CLSU32: + return iavf_setup_tc_cls_u32(cb_priv, type_data); default: return -EOPNOTSUPP; } @@ -4332,8 +4482,8 @@ static void iavf_disable_fdir(struct iavf_adapter *adapter) fdir->state == IAVF_FDIR_FLTR_INACTIVE) { /* Delete filters not registered in PF */ list_del(&fdir->list); + iavf_dec_fdir_active_fltr(adapter, fdir); kfree(fdir); - adapter->fdir_active_fltr--; } else if (fdir->state == IAVF_FDIR_FLTR_ADD_PENDING || fdir->state == IAVF_FDIR_FLTR_DIS_REQUEST || fdir->state == IAVF_FDIR_FLTR_ACTIVE) { @@ -4843,9 +4993,11 @@ int iavf_process_config(struct iavf_adapter *adapter) /* get HW VLAN features that can be toggled */ hw_vlan_features = iavf_get_netdev_vlan_hw_features(adapter); - /* Enable cloud filter if ADQ is supported */ - if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) + /* Enable HW TC offload if ADQ or tc U32 is supported */ + if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ || + TC_U32_SUPPORT(adapter)) hw_features |= NETIF_F_HW_TC; + if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_USO) hw_features |= NETIF_F_GSO_UDP_L4; diff --git a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c index 1e543f6a7c30..7e810b65380c 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c +++ b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c @@ -142,6 +142,7 @@ int iavf_send_vf_config_msg(struct iavf_adapter *adapter) VIRTCHNL_VF_OFFLOAD_WB_ON_ITR | VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2 | VIRTCHNL_VF_OFFLOAD_ENCAP | + VIRTCHNL_VF_OFFLOAD_TC_U32 | VIRTCHNL_VF_OFFLOAD_VLAN_V2 | VIRTCHNL_VF_OFFLOAD_CRC | VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM | @@ -1961,8 +1962,8 @@ static void iavf_activate_fdir_filters(struct iavf_adapter *adapter) * list on PF is already cleared after a reset */ list_del(&f->list); + iavf_dec_fdir_active_fltr(adapter, f); kfree(f); - adapter->fdir_active_fltr--; } } spin_unlock_bh(&adapter->fdir_fltr_lock); @@ -2135,8 +2136,8 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, dev_err(&adapter->pdev->dev, "%s\n", msg); list_del(&fdir->list); + iavf_dec_fdir_active_fltr(adapter, fdir); kfree(fdir); - adapter->fdir_active_fltr--; } } spin_unlock_bh(&adapter->fdir_fltr_lock); @@ -2451,8 +2452,12 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, list) { if (fdir->state == IAVF_FDIR_FLTR_ADD_PENDING) { if (add_fltr->status == VIRTCHNL_FDIR_SUCCESS) { - dev_info(&adapter->pdev->dev, "Flow Director filter with location %u is added\n", - fdir->loc); + if (!iavf_is_raw_fdir(fdir)) + dev_info(&adapter->pdev->dev, "Flow Director filter with location %u is added\n", + fdir->loc); + else + dev_info(&adapter->pdev->dev, "Flow Director filter (raw) for TC handle %x is added\n", + TC_U32_USERHTID(fdir->cls_u32_handle)); fdir->state = IAVF_FDIR_FLTR_ACTIVE; fdir->flow_id = add_fltr->flow_id; } else { @@ -2460,8 +2465,8 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, add_fltr->status); iavf_print_fdir_fltr(adapter, fdir); list_del(&fdir->list); + iavf_dec_fdir_active_fltr(adapter, fdir); kfree(fdir); - adapter->fdir_active_fltr--; } } } @@ -2479,11 +2484,15 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, if (del_fltr->status == VIRTCHNL_FDIR_SUCCESS || del_fltr->status == VIRTCHNL_FDIR_FAILURE_RULE_NONEXIST) { - dev_info(&adapter->pdev->dev, "Flow Director filter with location %u is deleted\n", - fdir->loc); + if (!iavf_is_raw_fdir(fdir)) + dev_info(&adapter->pdev->dev, "Flow Director filter with location %u is deleted\n", + fdir->loc); + else + dev_info(&adapter->pdev->dev, "Flow Director filter (raw) for TC handle %x is deleted\n", + TC_U32_USERHTID(fdir->cls_u32_handle)); list_del(&fdir->list); + iavf_dec_fdir_active_fltr(adapter, fdir); kfree(fdir); - adapter->fdir_active_fltr--; } else { fdir->state = IAVF_FDIR_FLTR_ACTIVE; dev_info(&adapter->pdev->dev, "Failed to delete Flow Director filter with status: %d\n", diff --git a/drivers/net/ethernet/intel/ice/Makefile b/drivers/net/ethernet/intel/ice/Makefile index 03500e28ac99..b4f6fa4ba13d 100644 --- a/drivers/net/ethernet/intel/ice/Makefile +++ b/drivers/net/ethernet/intel/ice/Makefile @@ -28,6 +28,8 @@ ice-y := ice_main.o \ ice_vlan_mode.o \ ice_flex_pipe.o \ ice_flow.o \ + ice_parser.o \ + ice_parser_rt.o \ ice_idc.o \ devlink/devlink.o \ devlink/devlink_port.o \ diff --git a/drivers/net/ethernet/intel/ice/ice_common.h b/drivers/net/ethernet/intel/ice/ice_common.h index 66f29bac783a..27208a60cece 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.h +++ b/drivers/net/ethernet/intel/ice/ice_common.h @@ -10,6 +10,7 @@ #include "ice_type.h" #include "ice_nvm.h" #include "ice_flex_pipe.h" +#include "ice_parser.h" #include <linux/avf/virtchnl.h> #include "ice_switch.h" #include "ice_fdir.h" diff --git a/drivers/net/ethernet/intel/ice/ice_ddp.c b/drivers/net/ethernet/intel/ice/ice_ddp.c index f182179529b7..953262b88a58 100644 --- a/drivers/net/ethernet/intel/ice/ice_ddp.c +++ b/drivers/net/ethernet/intel/ice/ice_ddp.c @@ -289,11 +289,11 @@ void *ice_pkg_enum_section(struct ice_seg *ice_seg, struct ice_pkg_enum *state, * indicates a base offset of 10, and the index for the entry is 2, then * section handler function should set the offset to 10 + 2 = 12. */ -static void *ice_pkg_enum_entry(struct ice_seg *ice_seg, - struct ice_pkg_enum *state, u32 sect_type, - u32 *offset, - void *(*handler)(u32 sect_type, void *section, - u32 index, u32 *offset)) +void *ice_pkg_enum_entry(struct ice_seg *ice_seg, + struct ice_pkg_enum *state, u32 sect_type, + u32 *offset, + void *(*handler)(u32 sect_type, void *section, + u32 index, u32 *offset)) { void *entry; diff --git a/drivers/net/ethernet/intel/ice/ice_ddp.h b/drivers/net/ethernet/intel/ice/ice_ddp.h index 622543f08b43..97f272317475 100644 --- a/drivers/net/ethernet/intel/ice/ice_ddp.h +++ b/drivers/net/ethernet/intel/ice/ice_ddp.h @@ -261,10 +261,17 @@ struct ice_meta_sect { #define ICE_SID_CDID_KEY_BUILDER_RSS 47 #define ICE_SID_CDID_REDIR_RSS 48 +#define ICE_SID_RXPARSER_CAM 50 +#define ICE_SID_RXPARSER_NOMATCH_CAM 51 +#define ICE_SID_RXPARSER_IMEM 52 #define ICE_SID_RXPARSER_MARKER_PTYPE 55 #define ICE_SID_RXPARSER_BOOST_TCAM 56 +#define ICE_SID_RXPARSER_PROTO_GRP 57 #define ICE_SID_RXPARSER_METADATA_INIT 58 #define ICE_SID_TXPARSER_BOOST_TCAM 66 +#define ICE_SID_RXPARSER_MARKER_GRP 72 +#define ICE_SID_RXPARSER_PG_SPILL 76 +#define ICE_SID_RXPARSER_NOMATCH_SPILL 78 #define ICE_SID_XLT0_PE 80 #define ICE_SID_XLT_KEY_BUILDER_PE 81 @@ -276,6 +283,7 @@ struct ice_meta_sect { #define ICE_SID_CDID_KEY_BUILDER_PE 87 #define ICE_SID_CDID_REDIR_PE 88 +#define ICE_SID_RXPARSER_FLAG_REDIR 97 /* Label Metadata section IDs */ #define ICE_SID_LBL_FIRST 0x80000010 #define ICE_SID_LBL_RXPARSER_TMEM 0x80000018 @@ -451,6 +459,11 @@ int ice_update_pkg(struct ice_hw *hw, struct ice_buf *bufs, u32 count); int ice_pkg_buf_reserve_section(struct ice_buf_build *bld, u16 count); u16 ice_pkg_buf_get_active_sections(struct ice_buf_build *bld); +void * +ice_pkg_enum_entry(struct ice_seg *ice_seg, struct ice_pkg_enum *state, + u32 sect_type, u32 *offset, + void *(*handler)(u32 sect_type, void *section, + u32 index, u32 *offset)); void *ice_pkg_enum_section(struct ice_seg *ice_seg, struct ice_pkg_enum *state, u32 sect_type); diff --git a/drivers/net/ethernet/intel/ice/ice_flex_pipe.c b/drivers/net/ethernet/intel/ice/ice_flex_pipe.c index 20d5db88c99f..ed95072ca6e3 100644 --- a/drivers/net/ethernet/intel/ice/ice_flex_pipe.c +++ b/drivers/net/ethernet/intel/ice/ice_flex_pipe.c @@ -2981,6 +2981,50 @@ ice_add_prof_attrib(struct ice_prof_map *prof, u8 ptg, u16 ptype, } /** + * ice_disable_fd_swap - set register appropriately to disable FD SWAP + * @hw: pointer to the HW struct + * @prof_id: profile ID + */ +static void +ice_disable_fd_swap(struct ice_hw *hw, u8 prof_id) +{ + u16 swap_val, fvw_num; + unsigned int i; + + swap_val = ICE_SWAP_VALID; + fvw_num = hw->blk[ICE_BLK_FD].es.fvw / ICE_FDIR_REG_SET_SIZE; + + /* Since the SWAP Flag in the Programming Desc doesn't work, + * here add method to disable the SWAP Option via setting + * certain SWAP and INSET register sets. + */ + for (i = 0; i < fvw_num ; i++) { + u32 raw_swap, raw_in; + unsigned int j; + + raw_swap = 0; + raw_in = 0; + + for (j = 0; j < ICE_FDIR_REG_SET_SIZE; j++) { + raw_swap |= (swap_val++) << (j * BITS_PER_BYTE); + raw_in |= ICE_INSET_DFLT << (j * BITS_PER_BYTE); + } + + /* write the FDIR swap register set */ + wr32(hw, GLQF_FDSWAP(prof_id, i), raw_swap); + + ice_debug(hw, ICE_DBG_INIT, "swap wr(%d, %d): 0x%x = 0x%08x\n", + prof_id, i, GLQF_FDSWAP(prof_id, i), raw_swap); + + /* write the FDIR inset register set */ + wr32(hw, GLQF_FDINSET(prof_id, i), raw_in); + + ice_debug(hw, ICE_DBG_INIT, "inset wr(%d, %d): 0x%x = 0x%08x\n", + prof_id, i, GLQF_FDINSET(prof_id, i), raw_in); + } +} + +/* * ice_add_prof - add profile * @hw: pointer to the HW struct * @blk: hardware block @@ -2991,6 +3035,7 @@ ice_add_prof_attrib(struct ice_prof_map *prof, u8 ptg, u16 ptype, * @es: extraction sequence (length of array is determined by the block) * @masks: mask for extraction sequence * @symm: symmetric setting for RSS profiles + * @fd_swap: enable/disable FDIR paired src/dst fields swap option * * This function registers a profile, which matches a set of PTYPES with a * particular extraction sequence. While the hardware profile is allocated @@ -3000,7 +3045,7 @@ ice_add_prof_attrib(struct ice_prof_map *prof, u8 ptg, u16 ptype, int ice_add_prof(struct ice_hw *hw, enum ice_block blk, u64 id, u8 ptypes[], const struct ice_ptype_attributes *attr, u16 attr_cnt, - struct ice_fv_word *es, u16 *masks, bool symm) + struct ice_fv_word *es, u16 *masks, bool symm, bool fd_swap) { u32 bytes = DIV_ROUND_UP(ICE_FLOW_PTYPE_MAX, BITS_PER_BYTE); DECLARE_BITMAP(ptgs_used, ICE_XLT1_CNT); @@ -3020,7 +3065,7 @@ ice_add_prof(struct ice_hw *hw, enum ice_block blk, u64 id, u8 ptypes[], status = ice_alloc_prof_id(hw, blk, &prof_id); if (status) goto err_ice_add_prof; - if (blk == ICE_BLK_FD) { + if (blk == ICE_BLK_FD && fd_swap) { /* For Flow Director block, the extraction sequence may * need to be altered in the case where there are paired * fields that have no match. This is necessary because @@ -3031,6 +3076,8 @@ ice_add_prof(struct ice_hw *hw, enum ice_block blk, u64 id, u8 ptypes[], status = ice_update_fd_swap(hw, prof_id, es); if (status) goto err_ice_add_prof; + } else if (blk == ICE_BLK_FD) { + ice_disable_fd_swap(hw, prof_id); } status = ice_update_prof_masking(hw, blk, prof_id, masks); if (status) @@ -4099,6 +4146,54 @@ err_ice_add_prof_id_flow: } /** + * ice_flow_assoc_fdir_prof - add an FDIR profile for main/ctrl VSI + * @hw: pointer to the HW struct + * @blk: HW block + * @dest_vsi: dest VSI + * @fdir_vsi: fdir programming VSI + * @hdl: profile handle + * + * Update the hardware tables to enable the FDIR profile indicated by @hdl for + * the VSI specified by @dest_vsi. On success, the flow will be enabled. + * + * Return: 0 on success or negative errno on failure. + */ +int +ice_flow_assoc_fdir_prof(struct ice_hw *hw, enum ice_block blk, + u16 dest_vsi, u16 fdir_vsi, u64 hdl) +{ + u16 vsi_num; + int status; + + if (blk != ICE_BLK_FD) + return -EINVAL; + + vsi_num = ice_get_hw_vsi_num(hw, dest_vsi); + status = ice_add_prof_id_flow(hw, blk, vsi_num, hdl); + if (status) { + ice_debug(hw, ICE_DBG_FLOW, "Adding HW profile failed for main VSI flow entry: %d\n", + status); + return status; + } + + vsi_num = ice_get_hw_vsi_num(hw, fdir_vsi); + status = ice_add_prof_id_flow(hw, blk, vsi_num, hdl); + if (status) { + ice_debug(hw, ICE_DBG_FLOW, "Adding HW profile failed for ctrl VSI flow entry: %d\n", + status); + goto err; + } + + return 0; + +err: + vsi_num = ice_get_hw_vsi_num(hw, dest_vsi); + ice_rem_prof_id_flow(hw, blk, vsi_num, hdl); + + return status; +} + +/** * ice_rem_prof_from_list - remove a profile from list * @hw: pointer to the HW struct * @lst: list to remove the profile from diff --git a/drivers/net/ethernet/intel/ice/ice_flex_pipe.h b/drivers/net/ethernet/intel/ice/ice_flex_pipe.h index b39d7cdc381f..90b9b0993122 100644 --- a/drivers/net/ethernet/intel/ice/ice_flex_pipe.h +++ b/drivers/net/ethernet/intel/ice/ice_flex_pipe.h @@ -6,6 +6,8 @@ #include "ice_type.h" +#define ICE_FDIR_REG_SET_SIZE 4 + int ice_acquire_change_lock(struct ice_hw *hw, enum ice_aq_res_access_type access); void ice_release_change_lock(struct ice_hw *hw); @@ -42,13 +44,16 @@ bool ice_hw_ptype_ena(struct ice_hw *hw, u16 ptype); int ice_add_prof(struct ice_hw *hw, enum ice_block blk, u64 id, u8 ptypes[], const struct ice_ptype_attributes *attr, u16 attr_cnt, - struct ice_fv_word *es, u16 *masks, bool symm); + struct ice_fv_word *es, u16 *masks, bool symm, bool fd_swap); struct ice_prof_map * ice_search_prof_id(struct ice_hw *hw, enum ice_block blk, u64 id); int ice_add_prof_id_flow(struct ice_hw *hw, enum ice_block blk, u16 vsi, u64 hdl); int ice_rem_prof_id_flow(struct ice_hw *hw, enum ice_block blk, u16 vsi, u64 hdl); +int +ice_flow_assoc_fdir_prof(struct ice_hw *hw, enum ice_block blk, + u16 dest_vsi, u16 fdir_vsi, u64 hdl); enum ice_ddp_state ice_init_pkg(struct ice_hw *hw, u8 *buff, u32 len); enum ice_ddp_state ice_copy_and_init_pkg(struct ice_hw *hw, const u8 *buf, u32 len); diff --git a/drivers/net/ethernet/intel/ice/ice_flow.c b/drivers/net/ethernet/intel/ice/ice_flow.c index fc2b58f56279..d97b751052f2 100644 --- a/drivers/net/ethernet/intel/ice/ice_flow.c +++ b/drivers/net/ethernet/intel/ice/ice_flow.c @@ -409,6 +409,29 @@ static const u32 ice_ptypes_gtpc_tid[] = { }; /* Packet types for GTPU */ +static const struct ice_ptype_attributes ice_attr_gtpu_session[] = { + { ICE_MAC_IPV4_GTPU_IPV4_FRAG, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV4_GTPU_IPV4_PAY, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV4_GTPU_IPV4_UDP_PAY, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV4_GTPU_IPV4_TCP, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV4_GTPU_IPV4_ICMP, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV6_GTPU_IPV4_FRAG, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV6_GTPU_IPV4_PAY, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV6_GTPU_IPV4_UDP_PAY, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV6_GTPU_IPV4_TCP, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV6_GTPU_IPV4_ICMP, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV4_GTPU_IPV6_FRAG, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV4_GTPU_IPV6_PAY, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV4_GTPU_IPV6_UDP_PAY, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV4_GTPU_IPV6_TCP, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV4_GTPU_IPV6_ICMPV6, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV6_GTPU_IPV6_FRAG, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV6_GTPU_IPV6_PAY, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV6_GTPU_IPV6_UDP_PAY, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV6_GTPU_IPV6_TCP, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV6_GTPU_IPV6_ICMPV6, ICE_PTYPE_ATTR_GTP_SESSION }, +}; + static const struct ice_ptype_attributes ice_attr_gtpu_eh[] = { { ICE_MAC_IPV4_GTPU_IPV4_FRAG, ICE_PTYPE_ATTR_GTP_PDU_EH }, { ICE_MAC_IPV4_GTPU_IPV4_PAY, ICE_PTYPE_ATTR_GTP_PDU_EH }, @@ -1400,7 +1423,7 @@ ice_flow_add_prof_sync(struct ice_hw *hw, enum ice_block blk, /* Add a HW profile for this flow profile */ status = ice_add_prof(hw, blk, prof_id, (u8 *)params->ptypes, params->attr, params->attr_cnt, params->es, - params->mask, symm); + params->mask, symm, true); if (status) { ice_debug(hw, ICE_DBG_FLOW, "Error adding a HW flow profile\n"); goto out; @@ -1523,6 +1546,90 @@ ice_flow_disassoc_prof(struct ice_hw *hw, enum ice_block blk, return status; } +#define FLAG_GTP_EH_PDU_LINK BIT_ULL(13) +#define FLAG_GTP_EH_PDU BIT_ULL(14) + +#define HI_BYTE_IN_WORD GENMASK(15, 8) +#define LO_BYTE_IN_WORD GENMASK(7, 0) + +#define FLAG_GTPU_MSK \ + (FLAG_GTP_EH_PDU | FLAG_GTP_EH_PDU_LINK) +#define FLAG_GTPU_UP \ + (FLAG_GTP_EH_PDU | FLAG_GTP_EH_PDU_LINK) +#define FLAG_GTPU_DW FLAG_GTP_EH_PDU + +/** + * ice_flow_set_parser_prof - Set flow profile based on the parsed profile info + * @hw: pointer to the HW struct + * @dest_vsi: dest VSI + * @fdir_vsi: fdir programming VSI + * @prof: stores parsed profile info from raw flow + * @blk: classification blk + * + * Return: 0 on success or negative errno on failure. + */ +int +ice_flow_set_parser_prof(struct ice_hw *hw, u16 dest_vsi, u16 fdir_vsi, + struct ice_parser_profile *prof, enum ice_block blk) +{ + u64 id = find_first_bit(prof->ptypes, ICE_FLOW_PTYPE_MAX); + struct ice_flow_prof_params *params __free(kfree); + u8 fv_words = hw->blk[blk].es.fvw; + int status; + int i, idx; + + params = kzalloc(sizeof(*params), GFP_KERNEL); + if (!params) + return -ENOMEM; + + for (i = 0; i < ICE_MAX_FV_WORDS; i++) { + params->es[i].prot_id = ICE_PROT_INVALID; + params->es[i].off = ICE_FV_OFFSET_INVAL; + } + + for (i = 0; i < prof->fv_num; i++) { + if (hw->blk[blk].es.reverse) + idx = fv_words - i - 1; + else + idx = i; + params->es[idx].prot_id = prof->fv[i].proto_id; + params->es[idx].off = prof->fv[i].offset; + params->mask[idx] = (((prof->fv[i].msk) << BITS_PER_BYTE) & + HI_BYTE_IN_WORD) | + (((prof->fv[i].msk) >> BITS_PER_BYTE) & + LO_BYTE_IN_WORD); + } + + switch (prof->flags) { + case FLAG_GTPU_DW: + params->attr = ice_attr_gtpu_down; + params->attr_cnt = ARRAY_SIZE(ice_attr_gtpu_down); + break; + case FLAG_GTPU_UP: + params->attr = ice_attr_gtpu_up; + params->attr_cnt = ARRAY_SIZE(ice_attr_gtpu_up); + break; + default: + if (prof->flags_msk & FLAG_GTPU_MSK) { + params->attr = ice_attr_gtpu_session; + params->attr_cnt = ARRAY_SIZE(ice_attr_gtpu_session); + } + break; + } + + status = ice_add_prof(hw, blk, id, (u8 *)prof->ptypes, + params->attr, params->attr_cnt, + params->es, params->mask, false, false); + if (status) + return status; + + status = ice_flow_assoc_fdir_prof(hw, blk, dest_vsi, fdir_vsi, id); + if (status) + ice_rem_prof(hw, blk, id); + + return status; +} + /** * ice_flow_add_prof - Add a flow profile for packet segments and matched fields * @hw: pointer to the HW struct diff --git a/drivers/net/ethernet/intel/ice/ice_flow.h b/drivers/net/ethernet/intel/ice/ice_flow.h index 2fd2e0cb483d..6cb7bb879c98 100644 --- a/drivers/net/ethernet/intel/ice/ice_flow.h +++ b/drivers/net/ethernet/intel/ice/ice_flow.h @@ -5,6 +5,7 @@ #define _ICE_FLOW_H_ #include "ice_flex_type.h" +#include "ice_parser.h" #define ICE_FLOW_ENTRY_HANDLE_INVAL 0 #define ICE_FLOW_FLD_OFF_INVAL 0xffff @@ -326,6 +327,7 @@ enum ice_rss_cfg_hdr_type { ICE_RSS_ANY_HEADERS }; +struct ice_vsi; struct ice_rss_hash_cfg { u32 addl_hdrs; /* protocol header fields */ u64 hash_flds; /* hash bit field (ICE_FLOW_HASH_*) to configure */ @@ -445,6 +447,9 @@ ice_flow_add_prof(struct ice_hw *hw, enum ice_block blk, enum ice_flow_dir dir, bool symm, struct ice_flow_prof **prof); int ice_flow_rem_prof(struct ice_hw *hw, enum ice_block blk, u64 prof_id); int +ice_flow_set_parser_prof(struct ice_hw *hw, u16 dest_vsi, u16 fdir_vsi, + struct ice_parser_profile *prof, enum ice_block blk); +int ice_flow_add_entry(struct ice_hw *hw, enum ice_block blk, u64 prof_id, u64 entry_id, u16 vsi, enum ice_flow_priority prio, void *data, u64 *entry_h); diff --git a/drivers/net/ethernet/intel/ice/ice_parser.c b/drivers/net/ethernet/intel/ice/ice_parser.c new file mode 100644 index 000000000000..664beb64f557 --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_parser.c @@ -0,0 +1,2430 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2024 Intel Corporation */ + +#include "ice_common.h" + +struct ice_pkg_sect_hdr { + __le16 count; + __le16 offset; +}; + +/** + * ice_parser_sect_item_get - parse an item from a section + * @sect_type: section type + * @section: section object + * @index: index of the item to get + * @offset: dummy as prototype of ice_pkg_enum_entry's last parameter + * + * Return: a pointer to the item or NULL. + */ +static void *ice_parser_sect_item_get(u32 sect_type, void *section, + u32 index, u32 __maybe_unused *offset) +{ + size_t data_off = ICE_SEC_DATA_OFFSET; + struct ice_pkg_sect_hdr *hdr; + size_t size; + + if (!section) + return NULL; + + switch (sect_type) { + case ICE_SID_RXPARSER_IMEM: + size = ICE_SID_RXPARSER_IMEM_ENTRY_SIZE; + break; + case ICE_SID_RXPARSER_METADATA_INIT: + size = ICE_SID_RXPARSER_METADATA_INIT_ENTRY_SIZE; + break; + case ICE_SID_RXPARSER_CAM: + size = ICE_SID_RXPARSER_CAM_ENTRY_SIZE; + break; + case ICE_SID_RXPARSER_PG_SPILL: + size = ICE_SID_RXPARSER_PG_SPILL_ENTRY_SIZE; + break; + case ICE_SID_RXPARSER_NOMATCH_CAM: + size = ICE_SID_RXPARSER_NOMATCH_CAM_ENTRY_SIZE; + break; + case ICE_SID_RXPARSER_NOMATCH_SPILL: + size = ICE_SID_RXPARSER_NOMATCH_SPILL_ENTRY_SIZE; + break; + case ICE_SID_RXPARSER_BOOST_TCAM: + size = ICE_SID_RXPARSER_BOOST_TCAM_ENTRY_SIZE; + break; + case ICE_SID_LBL_RXPARSER_TMEM: + data_off = ICE_SEC_LBL_DATA_OFFSET; + size = ICE_SID_LBL_ENTRY_SIZE; + break; + case ICE_SID_RXPARSER_MARKER_PTYPE: + size = ICE_SID_RXPARSER_MARKER_TYPE_ENTRY_SIZE; + break; + case ICE_SID_RXPARSER_MARKER_GRP: + size = ICE_SID_RXPARSER_MARKER_GRP_ENTRY_SIZE; + break; + case ICE_SID_RXPARSER_PROTO_GRP: + size = ICE_SID_RXPARSER_PROTO_GRP_ENTRY_SIZE; + break; + case ICE_SID_RXPARSER_FLAG_REDIR: + size = ICE_SID_RXPARSER_FLAG_REDIR_ENTRY_SIZE; + break; + default: + return NULL; + } + + hdr = section; + if (index >= le16_to_cpu(hdr->count)) + return NULL; + + return section + data_off + index * size; +} + +/** + * ice_parser_create_table - create an item table from a section + * @hw: pointer to the hardware structure + * @sect_type: section type + * @item_size: item size in bytes + * @length: number of items in the table to create + * @parse_item: the function to parse the item + * @no_offset: ignore header offset, calculate index from 0 + * + * Return: a pointer to the allocated table or ERR_PTR. + */ +static void * +ice_parser_create_table(struct ice_hw *hw, u32 sect_type, + u32 item_size, u32 length, + void (*parse_item)(struct ice_hw *hw, u16 idx, + void *item, void *data, + int size), bool no_offset) +{ + struct ice_pkg_enum state = {}; + struct ice_seg *seg = hw->seg; + void *table, *data, *item; + u16 idx = 0; + + if (!seg) + return ERR_PTR(-EINVAL); + + table = kzalloc(item_size * length, GFP_KERNEL); + if (!table) + return ERR_PTR(-ENOMEM); + + do { + data = ice_pkg_enum_entry(seg, &state, sect_type, NULL, + ice_parser_sect_item_get); + seg = NULL; + if (data) { + struct ice_pkg_sect_hdr *hdr = state.sect; + + if (!no_offset) + idx = le16_to_cpu(hdr->offset) + + state.entry_idx; + + item = (void *)((uintptr_t)table + idx * item_size); + parse_item(hw, idx, item, data, item_size); + + if (no_offset) + idx++; + } + } while (data); + + return table; +} + +/*** ICE_SID_RXPARSER_IMEM section ***/ +static void ice_imem_bst_bm_dump(struct ice_hw *hw, struct ice_bst_main *bm) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "boost main:\n"); + dev_info(dev, "\talu0 = %d\n", bm->alu0); + dev_info(dev, "\talu1 = %d\n", bm->alu1); + dev_info(dev, "\talu2 = %d\n", bm->alu2); + dev_info(dev, "\tpg = %d\n", bm->pg); +} + +static void ice_imem_bst_kb_dump(struct ice_hw *hw, + struct ice_bst_keybuilder *kb) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "boost key builder:\n"); + dev_info(dev, "\tpriority = %d\n", kb->prio); + dev_info(dev, "\ttsr_ctrl = %d\n", kb->tsr_ctrl); +} + +static void ice_imem_np_kb_dump(struct ice_hw *hw, + struct ice_np_keybuilder *kb) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "next proto key builder:\n"); + dev_info(dev, "\topc = %d\n", kb->opc); + dev_info(dev, "\tstart_or_reg0 = %d\n", kb->start_reg0); + dev_info(dev, "\tlen_or_reg1 = %d\n", kb->len_reg1); +} + +static void ice_imem_pg_kb_dump(struct ice_hw *hw, + struct ice_pg_keybuilder *kb) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "parse graph key builder:\n"); + dev_info(dev, "\tflag0_ena = %d\n", kb->flag0_ena); + dev_info(dev, "\tflag1_ena = %d\n", kb->flag1_ena); + dev_info(dev, "\tflag2_ena = %d\n", kb->flag2_ena); + dev_info(dev, "\tflag3_ena = %d\n", kb->flag3_ena); + dev_info(dev, "\tflag0_idx = %d\n", kb->flag0_idx); + dev_info(dev, "\tflag1_idx = %d\n", kb->flag1_idx); + dev_info(dev, "\tflag2_idx = %d\n", kb->flag2_idx); + dev_info(dev, "\tflag3_idx = %d\n", kb->flag3_idx); + dev_info(dev, "\talu_reg_idx = %d\n", kb->alu_reg_idx); +} + +static void ice_imem_alu_dump(struct ice_hw *hw, + struct ice_alu *alu, int index) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "alu%d:\n", index); + dev_info(dev, "\topc = %d\n", alu->opc); + dev_info(dev, "\tsrc_start = %d\n", alu->src_start); + dev_info(dev, "\tsrc_len = %d\n", alu->src_len); + dev_info(dev, "\tshift_xlate_sel = %d\n", alu->shift_xlate_sel); + dev_info(dev, "\tshift_xlate_key = %d\n", alu->shift_xlate_key); + dev_info(dev, "\tsrc_reg_id = %d\n", alu->src_reg_id); + dev_info(dev, "\tdst_reg_id = %d\n", alu->dst_reg_id); + dev_info(dev, "\tinc0 = %d\n", alu->inc0); + dev_info(dev, "\tinc1 = %d\n", alu->inc1); + dev_info(dev, "\tproto_offset_opc = %d\n", alu->proto_offset_opc); + dev_info(dev, "\tproto_offset = %d\n", alu->proto_offset); + dev_info(dev, "\tbranch_addr = %d\n", alu->branch_addr); + dev_info(dev, "\timm = %d\n", alu->imm); + dev_info(dev, "\tdst_start = %d\n", alu->dst_start); + dev_info(dev, "\tdst_len = %d\n", alu->dst_len); + dev_info(dev, "\tflags_extr_imm = %d\n", alu->flags_extr_imm); + dev_info(dev, "\tflags_start_imm= %d\n", alu->flags_start_imm); +} + +/** + * ice_imem_dump - dump an imem item info + * @hw: pointer to the hardware structure + * @item: imem item to dump + */ +static void ice_imem_dump(struct ice_hw *hw, struct ice_imem_item *item) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "index = %d\n", item->idx); + ice_imem_bst_bm_dump(hw, &item->b_m); + ice_imem_bst_kb_dump(hw, &item->b_kb); + dev_info(dev, "pg priority = %d\n", item->pg_prio); + ice_imem_np_kb_dump(hw, &item->np_kb); + ice_imem_pg_kb_dump(hw, &item->pg_kb); + ice_imem_alu_dump(hw, &item->alu0, 0); + ice_imem_alu_dump(hw, &item->alu1, 1); + ice_imem_alu_dump(hw, &item->alu2, 2); +} + +#define ICE_IM_BM_ALU0 BIT(0) +#define ICE_IM_BM_ALU1 BIT(1) +#define ICE_IM_BM_ALU2 BIT(2) +#define ICE_IM_BM_PG BIT(3) + +/** + * ice_imem_bm_init - parse 4 bits of Boost Main + * @bm: pointer to the Boost Main structure + * @data: Boost Main data to be parsed + */ +static void ice_imem_bm_init(struct ice_bst_main *bm, u8 data) +{ + bm->alu0 = FIELD_GET(ICE_IM_BM_ALU0, data); + bm->alu1 = FIELD_GET(ICE_IM_BM_ALU1, data); + bm->alu2 = FIELD_GET(ICE_IM_BM_ALU2, data); + bm->pg = FIELD_GET(ICE_IM_BM_PG, data); +} + +#define ICE_IM_BKB_PRIO GENMASK(7, 0) +#define ICE_IM_BKB_TSR_CTRL BIT(8) + +/** + * ice_imem_bkb_init - parse 10 bits of Boost Main Build + * @bkb: pointer to the Boost Main Build structure + * @data: Boost Main Build data to be parsed + */ +static void ice_imem_bkb_init(struct ice_bst_keybuilder *bkb, u16 data) +{ + bkb->prio = FIELD_GET(ICE_IM_BKB_PRIO, data); + bkb->tsr_ctrl = FIELD_GET(ICE_IM_BKB_TSR_CTRL, data); +} + +#define ICE_IM_NPKB_OPC GENMASK(1, 0) +#define ICE_IM_NPKB_S_R0 GENMASK(9, 2) +#define ICE_IM_NPKB_L_R1 GENMASK(17, 10) + +/** + * ice_imem_npkb_init - parse 18 bits of Next Protocol Key Build + * @kb: pointer to the Next Protocol Key Build structure + * @data: Next Protocol Key Build data to be parsed + */ +static void ice_imem_npkb_init(struct ice_np_keybuilder *kb, u32 data) +{ + kb->opc = FIELD_GET(ICE_IM_NPKB_OPC, data); + kb->start_reg0 = FIELD_GET(ICE_IM_NPKB_S_R0, data); + kb->len_reg1 = FIELD_GET(ICE_IM_NPKB_L_R1, data); +} + +#define ICE_IM_PGKB_F0_ENA BIT_ULL(0) +#define ICE_IM_PGKB_F0_IDX GENMASK_ULL(6, 1) +#define ICE_IM_PGKB_F1_ENA BIT_ULL(7) +#define ICE_IM_PGKB_F1_IDX GENMASK_ULL(13, 8) +#define ICE_IM_PGKB_F2_ENA BIT_ULL(14) +#define ICE_IM_PGKB_F2_IDX GENMASK_ULL(20, 15) +#define ICE_IM_PGKB_F3_ENA BIT_ULL(21) +#define ICE_IM_PGKB_F3_IDX GENMASK_ULL(27, 22) +#define ICE_IM_PGKB_AR_IDX GENMASK_ULL(34, 28) + +/** + * ice_imem_pgkb_init - parse 35 bits of Parse Graph Key Build + * @kb: pointer to the Parse Graph Key Build structure + * @data: Parse Graph Key Build data to be parsed + */ +static void ice_imem_pgkb_init(struct ice_pg_keybuilder *kb, u64 data) +{ + kb->flag0_ena = FIELD_GET(ICE_IM_PGKB_F0_ENA, data); + kb->flag0_idx = FIELD_GET(ICE_IM_PGKB_F0_IDX, data); + kb->flag1_ena = FIELD_GET(ICE_IM_PGKB_F1_ENA, data); + kb->flag1_idx = FIELD_GET(ICE_IM_PGKB_F1_IDX, data); + kb->flag2_ena = FIELD_GET(ICE_IM_PGKB_F2_ENA, data); + kb->flag2_idx = FIELD_GET(ICE_IM_PGKB_F2_IDX, data); + kb->flag3_ena = FIELD_GET(ICE_IM_PGKB_F3_ENA, data); + kb->flag3_idx = FIELD_GET(ICE_IM_PGKB_F3_IDX, data); + kb->alu_reg_idx = FIELD_GET(ICE_IM_PGKB_AR_IDX, data); +} + +#define ICE_IM_ALU_OPC GENMASK_ULL(5, 0) +#define ICE_IM_ALU_SS GENMASK_ULL(13, 6) +#define ICE_IM_ALU_SL GENMASK_ULL(18, 14) +#define ICE_IM_ALU_SXS BIT_ULL(19) +#define ICE_IM_ALU_SXK GENMASK_ULL(23, 20) +#define ICE_IM_ALU_SRID GENMASK_ULL(30, 24) +#define ICE_IM_ALU_DRID GENMASK_ULL(37, 31) +#define ICE_IM_ALU_INC0 BIT_ULL(38) +#define ICE_IM_ALU_INC1 BIT_ULL(39) +#define ICE_IM_ALU_POO GENMASK_ULL(41, 40) +#define ICE_IM_ALU_PO GENMASK_ULL(49, 42) +#define ICE_IM_ALU_BA_S 50 /* offset for the 2nd 64-bits field */ +#define ICE_IM_ALU_BA GENMASK_ULL(57 - ICE_IM_ALU_BA_S, \ + 50 - ICE_IM_ALU_BA_S) +#define ICE_IM_ALU_IMM GENMASK_ULL(73 - ICE_IM_ALU_BA_S, \ + 58 - ICE_IM_ALU_BA_S) +#define ICE_IM_ALU_DFE BIT_ULL(74 - ICE_IM_ALU_BA_S) +#define ICE_IM_ALU_DS GENMASK_ULL(80 - ICE_IM_ALU_BA_S, \ + 75 - ICE_IM_ALU_BA_S) +#define ICE_IM_ALU_DL GENMASK_ULL(86 - ICE_IM_ALU_BA_S, \ + 81 - ICE_IM_ALU_BA_S) +#define ICE_IM_ALU_FEI BIT_ULL(87 - ICE_IM_ALU_BA_S) +#define ICE_IM_ALU_FSI GENMASK_ULL(95 - ICE_IM_ALU_BA_S, \ + 88 - ICE_IM_ALU_BA_S) + +/** + * ice_imem_alu_init - parse 96 bits of ALU entry + * @alu: pointer to the ALU entry structure + * @data: ALU entry data to be parsed + * @off: offset of the ALU entry data + */ +static void ice_imem_alu_init(struct ice_alu *alu, u8 *data, u8 off) +{ + u64 d64; + u8 idd; + + d64 = *((u64 *)data) >> off; + + alu->opc = FIELD_GET(ICE_IM_ALU_OPC, d64); + alu->src_start = FIELD_GET(ICE_IM_ALU_SS, d64); + alu->src_len = FIELD_GET(ICE_IM_ALU_SL, d64); + alu->shift_xlate_sel = FIELD_GET(ICE_IM_ALU_SXS, d64); + alu->shift_xlate_key = FIELD_GET(ICE_IM_ALU_SXK, d64); + alu->src_reg_id = FIELD_GET(ICE_IM_ALU_SRID, d64); + alu->dst_reg_id = FIELD_GET(ICE_IM_ALU_DRID, d64); + alu->inc0 = FIELD_GET(ICE_IM_ALU_INC0, d64); + alu->inc1 = FIELD_GET(ICE_IM_ALU_INC1, d64); + alu->proto_offset_opc = FIELD_GET(ICE_IM_ALU_POO, d64); + alu->proto_offset = FIELD_GET(ICE_IM_ALU_PO, d64); + + idd = (ICE_IM_ALU_BA_S + off) / BITS_PER_BYTE; + off = (ICE_IM_ALU_BA_S + off) % BITS_PER_BYTE; + d64 = *((u64 *)(&data[idd])) >> off; + + alu->branch_addr = FIELD_GET(ICE_IM_ALU_BA, d64); + alu->imm = FIELD_GET(ICE_IM_ALU_IMM, d64); + alu->dedicate_flags_ena = FIELD_GET(ICE_IM_ALU_DFE, d64); + alu->dst_start = FIELD_GET(ICE_IM_ALU_DS, d64); + alu->dst_len = FIELD_GET(ICE_IM_ALU_DL, d64); + alu->flags_extr_imm = FIELD_GET(ICE_IM_ALU_FEI, d64); + alu->flags_start_imm = FIELD_GET(ICE_IM_ALU_FSI, d64); +} + +#define ICE_IMEM_BM_S 0 +#define ICE_IMEM_BKB_S 4 +#define ICE_IMEM_BKB_IDD (ICE_IMEM_BKB_S / BITS_PER_BYTE) +#define ICE_IMEM_BKB_OFF (ICE_IMEM_BKB_S % BITS_PER_BYTE) +#define ICE_IMEM_PGP GENMASK(15, 14) +#define ICE_IMEM_NPKB_S 16 +#define ICE_IMEM_NPKB_IDD (ICE_IMEM_NPKB_S / BITS_PER_BYTE) +#define ICE_IMEM_NPKB_OFF (ICE_IMEM_NPKB_S % BITS_PER_BYTE) +#define ICE_IMEM_PGKB_S 34 +#define ICE_IMEM_PGKB_IDD (ICE_IMEM_PGKB_S / BITS_PER_BYTE) +#define ICE_IMEM_PGKB_OFF (ICE_IMEM_PGKB_S % BITS_PER_BYTE) +#define ICE_IMEM_ALU0_S 69 +#define ICE_IMEM_ALU0_IDD (ICE_IMEM_ALU0_S / BITS_PER_BYTE) +#define ICE_IMEM_ALU0_OFF (ICE_IMEM_ALU0_S % BITS_PER_BYTE) +#define ICE_IMEM_ALU1_S 165 +#define ICE_IMEM_ALU1_IDD (ICE_IMEM_ALU1_S / BITS_PER_BYTE) +#define ICE_IMEM_ALU1_OFF (ICE_IMEM_ALU1_S % BITS_PER_BYTE) +#define ICE_IMEM_ALU2_S 357 +#define ICE_IMEM_ALU2_IDD (ICE_IMEM_ALU2_S / BITS_PER_BYTE) +#define ICE_IMEM_ALU2_OFF (ICE_IMEM_ALU2_S % BITS_PER_BYTE) + +/** + * ice_imem_parse_item - parse 384 bits of IMEM entry + * @hw: pointer to the hardware structure + * @idx: index of IMEM entry + * @item: item of IMEM entry + * @data: IMEM entry data to be parsed + * @size: size of IMEM entry + */ +static void ice_imem_parse_item(struct ice_hw *hw, u16 idx, void *item, + void *data, int __maybe_unused size) +{ + struct ice_imem_item *ii = item; + u8 *buf = data; + + ii->idx = idx; + + ice_imem_bm_init(&ii->b_m, *(u8 *)buf); + ice_imem_bkb_init(&ii->b_kb, + *((u16 *)(&buf[ICE_IMEM_BKB_IDD])) >> + ICE_IMEM_BKB_OFF); + + ii->pg_prio = FIELD_GET(ICE_IMEM_PGP, *(u16 *)buf); + + ice_imem_npkb_init(&ii->np_kb, + *((u32 *)(&buf[ICE_IMEM_NPKB_IDD])) >> + ICE_IMEM_NPKB_OFF); + ice_imem_pgkb_init(&ii->pg_kb, + *((u64 *)(&buf[ICE_IMEM_PGKB_IDD])) >> + ICE_IMEM_PGKB_OFF); + + ice_imem_alu_init(&ii->alu0, + &buf[ICE_IMEM_ALU0_IDD], + ICE_IMEM_ALU0_OFF); + ice_imem_alu_init(&ii->alu1, + &buf[ICE_IMEM_ALU1_IDD], + ICE_IMEM_ALU1_OFF); + ice_imem_alu_init(&ii->alu2, + &buf[ICE_IMEM_ALU2_IDD], + ICE_IMEM_ALU2_OFF); + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_imem_dump(hw, ii); +} + +/** + * ice_imem_table_get - create an imem table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated IMEM table. + */ +static struct ice_imem_item *ice_imem_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_IMEM, + sizeof(struct ice_imem_item), + ICE_IMEM_TABLE_SIZE, + ice_imem_parse_item, false); +} + +/*** ICE_SID_RXPARSER_METADATA_INIT section ***/ +/** + * ice_metainit_dump - dump an metainit item info + * @hw: pointer to the hardware structure + * @item: metainit item to dump + */ +static void ice_metainit_dump(struct ice_hw *hw, struct ice_metainit_item *item) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "index = %d\n", item->idx); + + dev_info(dev, "tsr = %d\n", item->tsr); + dev_info(dev, "ho = %d\n", item->ho); + dev_info(dev, "pc = %d\n", item->pc); + dev_info(dev, "pg_rn = %d\n", item->pg_rn); + dev_info(dev, "cd = %d\n", item->cd); + + dev_info(dev, "gpr_a_ctrl = %d\n", item->gpr_a_ctrl); + dev_info(dev, "gpr_a_data_mdid = %d\n", item->gpr_a_data_mdid); + dev_info(dev, "gpr_a_data_start = %d\n", item->gpr_a_data_start); + dev_info(dev, "gpr_a_data_len = %d\n", item->gpr_a_data_len); + dev_info(dev, "gpr_a_id = %d\n", item->gpr_a_id); + + dev_info(dev, "gpr_b_ctrl = %d\n", item->gpr_b_ctrl); + dev_info(dev, "gpr_b_data_mdid = %d\n", item->gpr_b_data_mdid); + dev_info(dev, "gpr_b_data_start = %d\n", item->gpr_b_data_start); + dev_info(dev, "gpr_b_data_len = %d\n", item->gpr_b_data_len); + dev_info(dev, "gpr_b_id = %d\n", item->gpr_b_id); + + dev_info(dev, "gpr_c_ctrl = %d\n", item->gpr_c_ctrl); + dev_info(dev, "gpr_c_data_mdid = %d\n", item->gpr_c_data_mdid); + dev_info(dev, "gpr_c_data_start = %d\n", item->gpr_c_data_start); + dev_info(dev, "gpr_c_data_len = %d\n", item->gpr_c_data_len); + dev_info(dev, "gpr_c_id = %d\n", item->gpr_c_id); + + dev_info(dev, "gpr_d_ctrl = %d\n", item->gpr_d_ctrl); + dev_info(dev, "gpr_d_data_mdid = %d\n", item->gpr_d_data_mdid); + dev_info(dev, "gpr_d_data_start = %d\n", item->gpr_d_data_start); + dev_info(dev, "gpr_d_data_len = %d\n", item->gpr_d_data_len); + dev_info(dev, "gpr_d_id = %d\n", item->gpr_d_id); + + dev_info(dev, "flags = 0x%llx\n", (unsigned long long)(item->flags)); +} + +#define ICE_MI_TSR GENMASK_ULL(7, 0) +#define ICE_MI_HO GENMASK_ULL(16, 8) +#define ICE_MI_PC GENMASK_ULL(24, 17) +#define ICE_MI_PGRN GENMASK_ULL(35, 25) +#define ICE_MI_CD GENMASK_ULL(38, 36) +#define ICE_MI_GAC BIT_ULL(39) +#define ICE_MI_GADM GENMASK_ULL(44, 40) +#define ICE_MI_GADS GENMASK_ULL(48, 45) +#define ICE_MI_GADL GENMASK_ULL(53, 49) +#define ICE_MI_GAI GENMASK_ULL(59, 56) +#define ICE_MI_GBC BIT_ULL(60) +#define ICE_MI_GBDM_S 61 /* offset for the 2nd 64-bits field */ +#define ICE_MI_GBDM_IDD (ICE_MI_GBDM_S / BITS_PER_BYTE) +#define ICE_MI_GBDM_OFF (ICE_MI_GBDM_S % BITS_PER_BYTE) + +#define ICE_MI_GBDM_GENMASK_ULL(high, low) \ + GENMASK_ULL((high) - ICE_MI_GBDM_S, (low) - ICE_MI_GBDM_S) +#define ICE_MI_GBDM ICE_MI_GBDM_GENMASK_ULL(65, 61) +#define ICE_MI_GBDS ICE_MI_GBDM_GENMASK_ULL(69, 66) +#define ICE_MI_GBDL ICE_MI_GBDM_GENMASK_ULL(74, 70) +#define ICE_MI_GBI ICE_MI_GBDM_GENMASK_ULL(80, 77) +#define ICE_MI_GCC BIT_ULL(81 - ICE_MI_GBDM_S) +#define ICE_MI_GCDM ICE_MI_GBDM_GENMASK_ULL(86, 82) +#define ICE_MI_GCDS ICE_MI_GBDM_GENMASK_ULL(90, 87) +#define ICE_MI_GCDL ICE_MI_GBDM_GENMASK_ULL(95, 91) +#define ICE_MI_GCI ICE_MI_GBDM_GENMASK_ULL(101, 98) +#define ICE_MI_GDC BIT_ULL(102 - ICE_MI_GBDM_S) +#define ICE_MI_GDDM ICE_MI_GBDM_GENMASK_ULL(107, 103) +#define ICE_MI_GDDS ICE_MI_GBDM_GENMASK_ULL(111, 108) +#define ICE_MI_GDDL ICE_MI_GBDM_GENMASK_ULL(116, 112) +#define ICE_MI_GDI ICE_MI_GBDM_GENMASK_ULL(122, 119) +#define ICE_MI_FLAG_S 123 /* offset for the 3rd 64-bits field */ +#define ICE_MI_FLAG_IDD (ICE_MI_FLAG_S / BITS_PER_BYTE) +#define ICE_MI_FLAG_OFF (ICE_MI_FLAG_S % BITS_PER_BYTE) +#define ICE_MI_FLAG GENMASK_ULL(186 - ICE_MI_FLAG_S, \ + 123 - ICE_MI_FLAG_S) + +/** + * ice_metainit_parse_item - parse 192 bits of Metadata Init entry + * @hw: pointer to the hardware structure + * @idx: index of Metadata Init entry + * @item: item of Metadata Init entry + * @data: Metadata Init entry data to be parsed + * @size: size of Metadata Init entry + */ +static void ice_metainit_parse_item(struct ice_hw *hw, u16 idx, void *item, + void *data, int __maybe_unused size) +{ + struct ice_metainit_item *mi = item; + u8 *buf = data; + u64 d64; + + mi->idx = idx; + + d64 = *(u64 *)buf; + + mi->tsr = FIELD_GET(ICE_MI_TSR, d64); + mi->ho = FIELD_GET(ICE_MI_HO, d64); + mi->pc = FIELD_GET(ICE_MI_PC, d64); + mi->pg_rn = FIELD_GET(ICE_MI_PGRN, d64); + mi->cd = FIELD_GET(ICE_MI_CD, d64); + + mi->gpr_a_ctrl = FIELD_GET(ICE_MI_GAC, d64); + mi->gpr_a_data_mdid = FIELD_GET(ICE_MI_GADM, d64); + mi->gpr_a_data_start = FIELD_GET(ICE_MI_GADS, d64); + mi->gpr_a_data_len = FIELD_GET(ICE_MI_GADL, d64); + mi->gpr_a_id = FIELD_GET(ICE_MI_GAI, d64); + + mi->gpr_b_ctrl = FIELD_GET(ICE_MI_GBC, d64); + + d64 = *((u64 *)&buf[ICE_MI_GBDM_IDD]) >> ICE_MI_GBDM_OFF; + + mi->gpr_b_data_mdid = FIELD_GET(ICE_MI_GBDM, d64); + mi->gpr_b_data_start = FIELD_GET(ICE_MI_GBDS, d64); + mi->gpr_b_data_len = FIELD_GET(ICE_MI_GBDL, d64); + mi->gpr_b_id = FIELD_GET(ICE_MI_GBI, d64); + + mi->gpr_c_ctrl = FIELD_GET(ICE_MI_GCC, d64); + mi->gpr_c_data_mdid = FIELD_GET(ICE_MI_GCDM, d64); + mi->gpr_c_data_start = FIELD_GET(ICE_MI_GCDS, d64); + mi->gpr_c_data_len = FIELD_GET(ICE_MI_GCDL, d64); + mi->gpr_c_id = FIELD_GET(ICE_MI_GCI, d64); + + mi->gpr_d_ctrl = FIELD_GET(ICE_MI_GDC, d64); + mi->gpr_d_data_mdid = FIELD_GET(ICE_MI_GDDM, d64); + mi->gpr_d_data_start = FIELD_GET(ICE_MI_GDDS, d64); + mi->gpr_d_data_len = FIELD_GET(ICE_MI_GDDL, d64); + mi->gpr_d_id = FIELD_GET(ICE_MI_GDI, d64); + + d64 = *((u64 *)&buf[ICE_MI_FLAG_IDD]) >> ICE_MI_FLAG_OFF; + + mi->flags = FIELD_GET(ICE_MI_FLAG, d64); + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_metainit_dump(hw, mi); +} + +/** + * ice_metainit_table_get - create a metainit table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Metadata initialization table. + */ +static struct ice_metainit_item *ice_metainit_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_METADATA_INIT, + sizeof(struct ice_metainit_item), + ICE_METAINIT_TABLE_SIZE, + ice_metainit_parse_item, false); +} + +/** + * ice_bst_tcam_search - find a TCAM item with specific type + * @tcam_table: the TCAM table + * @lbl_table: the lbl table to search + * @type: the type we need to match against + * @start: start searching from this index + * + * Return: a pointer to the matching BOOST TCAM item or NULL. + */ +struct ice_bst_tcam_item * +ice_bst_tcam_search(struct ice_bst_tcam_item *tcam_table, + struct ice_lbl_item *lbl_table, + enum ice_lbl_type type, u16 *start) +{ + u16 i = *start; + + for (; i < ICE_BST_TCAM_TABLE_SIZE; i++) { + if (lbl_table[i].type == type) { + *start = i; + return &tcam_table[lbl_table[i].idx]; + } + } + + return NULL; +} + +/*** ICE_SID_RXPARSER_CAM, ICE_SID_RXPARSER_PG_SPILL, + * ICE_SID_RXPARSER_NOMATCH_CAM and ICE_SID_RXPARSER_NOMATCH_CAM + * sections ***/ +static void ice_pg_cam_key_dump(struct ice_hw *hw, struct ice_pg_cam_key *key) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "key:\n"); + dev_info(dev, "\tvalid = %d\n", key->valid); + dev_info(dev, "\tnode_id = %d\n", key->node_id); + dev_info(dev, "\tflag0 = %d\n", key->flag0); + dev_info(dev, "\tflag1 = %d\n", key->flag1); + dev_info(dev, "\tflag2 = %d\n", key->flag2); + dev_info(dev, "\tflag3 = %d\n", key->flag3); + dev_info(dev, "\tboost_idx = %d\n", key->boost_idx); + dev_info(dev, "\talu_reg = 0x%04x\n", key->alu_reg); + dev_info(dev, "\tnext_proto = 0x%08x\n", key->next_proto); +} + +static void ice_pg_nm_cam_key_dump(struct ice_hw *hw, + struct ice_pg_nm_cam_key *key) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "key:\n"); + dev_info(dev, "\tvalid = %d\n", key->valid); + dev_info(dev, "\tnode_id = %d\n", key->node_id); + dev_info(dev, "\tflag0 = %d\n", key->flag0); + dev_info(dev, "\tflag1 = %d\n", key->flag1); + dev_info(dev, "\tflag2 = %d\n", key->flag2); + dev_info(dev, "\tflag3 = %d\n", key->flag3); + dev_info(dev, "\tboost_idx = %d\n", key->boost_idx); + dev_info(dev, "\talu_reg = 0x%04x\n", key->alu_reg); +} + +static void ice_pg_cam_action_dump(struct ice_hw *hw, + struct ice_pg_cam_action *action) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "action:\n"); + dev_info(dev, "\tnext_node = %d\n", action->next_node); + dev_info(dev, "\tnext_pc = %d\n", action->next_pc); + dev_info(dev, "\tis_pg = %d\n", action->is_pg); + dev_info(dev, "\tproto_id = %d\n", action->proto_id); + dev_info(dev, "\tis_mg = %d\n", action->is_mg); + dev_info(dev, "\tmarker_id = %d\n", action->marker_id); + dev_info(dev, "\tis_last_round = %d\n", action->is_last_round); + dev_info(dev, "\tho_polarity = %d\n", action->ho_polarity); + dev_info(dev, "\tho_inc = %d\n", action->ho_inc); +} + +/** + * ice_pg_cam_dump - dump an parse graph cam info + * @hw: pointer to the hardware structure + * @item: parse graph cam to dump + */ +static void ice_pg_cam_dump(struct ice_hw *hw, struct ice_pg_cam_item *item) +{ + dev_info(ice_hw_to_dev(hw), "index = %d\n", item->idx); + ice_pg_cam_key_dump(hw, &item->key); + ice_pg_cam_action_dump(hw, &item->action); +} + +/** + * ice_pg_nm_cam_dump - dump an parse graph no match cam info + * @hw: pointer to the hardware structure + * @item: parse graph no match cam to dump + */ +static void ice_pg_nm_cam_dump(struct ice_hw *hw, + struct ice_pg_nm_cam_item *item) +{ + dev_info(ice_hw_to_dev(hw), "index = %d\n", item->idx); + ice_pg_nm_cam_key_dump(hw, &item->key); + ice_pg_cam_action_dump(hw, &item->action); +} + +#define ICE_PGCA_NN GENMASK_ULL(10, 0) +#define ICE_PGCA_NPC GENMASK_ULL(18, 11) +#define ICE_PGCA_IPG BIT_ULL(19) +#define ICE_PGCA_PID GENMASK_ULL(30, 23) +#define ICE_PGCA_IMG BIT_ULL(31) +#define ICE_PGCA_MID GENMASK_ULL(39, 32) +#define ICE_PGCA_ILR BIT_ULL(40) +#define ICE_PGCA_HOP BIT_ULL(41) +#define ICE_PGCA_HOI GENMASK_ULL(50, 42) + +/** + * ice_pg_cam_action_init - parse 55 bits of Parse Graph CAM Action + * @action: pointer to the Parse Graph CAM Action structure + * @data: Parse Graph CAM Action data to be parsed + */ +static void ice_pg_cam_action_init(struct ice_pg_cam_action *action, u64 data) +{ + action->next_node = FIELD_GET(ICE_PGCA_NN, data); + action->next_pc = FIELD_GET(ICE_PGCA_NPC, data); + action->is_pg = FIELD_GET(ICE_PGCA_IPG, data); + action->proto_id = FIELD_GET(ICE_PGCA_PID, data); + action->is_mg = FIELD_GET(ICE_PGCA_IMG, data); + action->marker_id = FIELD_GET(ICE_PGCA_MID, data); + action->is_last_round = FIELD_GET(ICE_PGCA_ILR, data); + action->ho_polarity = FIELD_GET(ICE_PGCA_HOP, data); + action->ho_inc = FIELD_GET(ICE_PGCA_HOI, data); +} + +#define ICE_PGNCK_VLD BIT_ULL(0) +#define ICE_PGNCK_NID GENMASK_ULL(11, 1) +#define ICE_PGNCK_F0 BIT_ULL(12) +#define ICE_PGNCK_F1 BIT_ULL(13) +#define ICE_PGNCK_F2 BIT_ULL(14) +#define ICE_PGNCK_F3 BIT_ULL(15) +#define ICE_PGNCK_BH BIT_ULL(16) +#define ICE_PGNCK_BI GENMASK_ULL(24, 17) +#define ICE_PGNCK_AR GENMASK_ULL(40, 25) + +/** + * ice_pg_nm_cam_key_init - parse 41 bits of Parse Graph NoMatch CAM Key + * @key: pointer to the Parse Graph NoMatch CAM Key structure + * @data: Parse Graph NoMatch CAM Key data to be parsed + */ +static void ice_pg_nm_cam_key_init(struct ice_pg_nm_cam_key *key, u64 data) +{ + key->valid = FIELD_GET(ICE_PGNCK_VLD, data); + key->node_id = FIELD_GET(ICE_PGNCK_NID, data); + key->flag0 = FIELD_GET(ICE_PGNCK_F0, data); + key->flag1 = FIELD_GET(ICE_PGNCK_F1, data); + key->flag2 = FIELD_GET(ICE_PGNCK_F2, data); + key->flag3 = FIELD_GET(ICE_PGNCK_F3, data); + + if (FIELD_GET(ICE_PGNCK_BH, data)) + key->boost_idx = FIELD_GET(ICE_PGNCK_BI, data); + else + key->boost_idx = 0; + + key->alu_reg = FIELD_GET(ICE_PGNCK_AR, data); +} + +#define ICE_PGCK_VLD BIT_ULL(0) +#define ICE_PGCK_NID GENMASK_ULL(11, 1) +#define ICE_PGCK_F0 BIT_ULL(12) +#define ICE_PGCK_F1 BIT_ULL(13) +#define ICE_PGCK_F2 BIT_ULL(14) +#define ICE_PGCK_F3 BIT_ULL(15) +#define ICE_PGCK_BH BIT_ULL(16) +#define ICE_PGCK_BI GENMASK_ULL(24, 17) +#define ICE_PGCK_AR GENMASK_ULL(40, 25) +#define ICE_PGCK_NPK_S 41 /* offset for the 2nd 64-bits field */ +#define ICE_PGCK_NPK_IDD (ICE_PGCK_NPK_S / BITS_PER_BYTE) +#define ICE_PGCK_NPK_OFF (ICE_PGCK_NPK_S % BITS_PER_BYTE) +#define ICE_PGCK_NPK GENMASK_ULL(72 - ICE_PGCK_NPK_S, \ + 41 - ICE_PGCK_NPK_S) + +/** + * ice_pg_cam_key_init - parse 73 bits of Parse Graph CAM Key + * @key: pointer to the Parse Graph CAM Key structure + * @data: Parse Graph CAM Key data to be parsed + */ +static void ice_pg_cam_key_init(struct ice_pg_cam_key *key, u8 *data) +{ + u64 d64 = *(u64 *)data; + + key->valid = FIELD_GET(ICE_PGCK_VLD, d64); + key->node_id = FIELD_GET(ICE_PGCK_NID, d64); + key->flag0 = FIELD_GET(ICE_PGCK_F0, d64); + key->flag1 = FIELD_GET(ICE_PGCK_F1, d64); + key->flag2 = FIELD_GET(ICE_PGCK_F2, d64); + key->flag3 = FIELD_GET(ICE_PGCK_F3, d64); + + if (FIELD_GET(ICE_PGCK_BH, d64)) + key->boost_idx = FIELD_GET(ICE_PGCK_BI, d64); + else + key->boost_idx = 0; + + key->alu_reg = FIELD_GET(ICE_PGCK_AR, d64); + + d64 = *((u64 *)&data[ICE_PGCK_NPK_IDD]) >> ICE_PGCK_NPK_OFF; + + key->next_proto = FIELD_GET(ICE_PGCK_NPK, d64); +} + +#define ICE_PG_CAM_ACT_S 73 +#define ICE_PG_CAM_ACT_IDD (ICE_PG_CAM_ACT_S / BITS_PER_BYTE) +#define ICE_PG_CAM_ACT_OFF (ICE_PG_CAM_ACT_S % BITS_PER_BYTE) + +/** + * ice_pg_cam_parse_item - parse 128 bits of Parse Graph CAM Entry + * @hw: pointer to the hardware structure + * @idx: index of Parse Graph CAM Entry + * @item: item of Parse Graph CAM Entry + * @data: Parse Graph CAM Entry data to be parsed + * @size: size of Parse Graph CAM Entry + */ +static void ice_pg_cam_parse_item(struct ice_hw *hw, u16 idx, void *item, + void *data, int __maybe_unused size) +{ + struct ice_pg_cam_item *ci = item; + u8 *buf = data; + u64 d64; + + ci->idx = idx; + + ice_pg_cam_key_init(&ci->key, buf); + + d64 = *((u64 *)&buf[ICE_PG_CAM_ACT_IDD]) >> ICE_PG_CAM_ACT_OFF; + ice_pg_cam_action_init(&ci->action, d64); + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_pg_cam_dump(hw, ci); +} + +#define ICE_PG_SP_CAM_KEY_S 56 +#define ICE_PG_SP_CAM_KEY_IDD (ICE_PG_SP_CAM_KEY_S / BITS_PER_BYTE) + +/** + * ice_pg_sp_cam_parse_item - parse 136 bits of Parse Graph Spill CAM Entry + * @hw: pointer to the hardware structure + * @idx: index of Parse Graph Spill CAM Entry + * @item: item of Parse Graph Spill CAM Entry + * @data: Parse Graph Spill CAM Entry data to be parsed + * @size: size of Parse Graph Spill CAM Entry + */ +static void ice_pg_sp_cam_parse_item(struct ice_hw *hw, u16 idx, void *item, + void *data, int __maybe_unused size) +{ + struct ice_pg_cam_item *ci = item; + u8 *buf = data; + u64 d64; + + ci->idx = idx; + + d64 = *(u64 *)buf; + ice_pg_cam_action_init(&ci->action, d64); + + ice_pg_cam_key_init(&ci->key, &buf[ICE_PG_SP_CAM_KEY_IDD]); + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_pg_cam_dump(hw, ci); +} + +#define ICE_PG_NM_CAM_ACT_S 41 +#define ICE_PG_NM_CAM_ACT_IDD (ICE_PG_NM_CAM_ACT_S / BITS_PER_BYTE) +#define ICE_PG_NM_CAM_ACT_OFF (ICE_PG_NM_CAM_ACT_S % BITS_PER_BYTE) + +/** + * ice_pg_nm_cam_parse_item - parse 96 bits of Parse Graph NoMatch CAM Entry + * @hw: pointer to the hardware structure + * @idx: index of Parse Graph NoMatch CAM Entry + * @item: item of Parse Graph NoMatch CAM Entry + * @data: Parse Graph NoMatch CAM Entry data to be parsed + * @size: size of Parse Graph NoMatch CAM Entry + */ +static void ice_pg_nm_cam_parse_item(struct ice_hw *hw, u16 idx, void *item, + void *data, int __maybe_unused size) +{ + struct ice_pg_nm_cam_item *ci = item; + u8 *buf = data; + u64 d64; + + ci->idx = idx; + + d64 = *(u64 *)buf; + ice_pg_nm_cam_key_init(&ci->key, d64); + + d64 = *((u64 *)&buf[ICE_PG_NM_CAM_ACT_IDD]) >> ICE_PG_NM_CAM_ACT_OFF; + ice_pg_cam_action_init(&ci->action, d64); + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_pg_nm_cam_dump(hw, ci); +} + +#define ICE_PG_NM_SP_CAM_ACT_S 56 +#define ICE_PG_NM_SP_CAM_ACT_IDD (ICE_PG_NM_SP_CAM_ACT_S / BITS_PER_BYTE) +#define ICE_PG_NM_SP_CAM_ACT_OFF (ICE_PG_NM_SP_CAM_ACT_S % BITS_PER_BYTE) + +/** + * ice_pg_nm_sp_cam_parse_item - parse 104 bits of Parse Graph NoMatch Spill + * CAM Entry + * @hw: pointer to the hardware structure + * @idx: index of Parse Graph NoMatch Spill CAM Entry + * @item: item of Parse Graph NoMatch Spill CAM Entry + * @data: Parse Graph NoMatch Spill CAM Entry data to be parsed + * @size: size of Parse Graph NoMatch Spill CAM Entry + */ +static void ice_pg_nm_sp_cam_parse_item(struct ice_hw *hw, u16 idx, + void *item, void *data, + int __maybe_unused size) +{ + struct ice_pg_nm_cam_item *ci = item; + u8 *buf = data; + u64 d64; + + ci->idx = idx; + + d64 = *(u64 *)buf; + ice_pg_cam_action_init(&ci->action, d64); + + d64 = *((u64 *)&buf[ICE_PG_NM_SP_CAM_ACT_IDD]) >> + ICE_PG_NM_SP_CAM_ACT_OFF; + ice_pg_nm_cam_key_init(&ci->key, d64); + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_pg_nm_cam_dump(hw, ci); +} + +/** + * ice_pg_cam_table_get - create a parse graph cam table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Parse Graph CAM table. + */ +static struct ice_pg_cam_item *ice_pg_cam_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_CAM, + sizeof(struct ice_pg_cam_item), + ICE_PG_CAM_TABLE_SIZE, + ice_pg_cam_parse_item, false); +} + +/** + * ice_pg_sp_cam_table_get - create a parse graph spill cam table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Parse Graph Spill CAM table. + */ +static struct ice_pg_cam_item *ice_pg_sp_cam_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_PG_SPILL, + sizeof(struct ice_pg_cam_item), + ICE_PG_SP_CAM_TABLE_SIZE, + ice_pg_sp_cam_parse_item, false); +} + +/** + * ice_pg_nm_cam_table_get - create a parse graph no match cam table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Parse Graph No Match CAM table. + */ +static struct ice_pg_nm_cam_item *ice_pg_nm_cam_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_NOMATCH_CAM, + sizeof(struct ice_pg_nm_cam_item), + ICE_PG_NM_CAM_TABLE_SIZE, + ice_pg_nm_cam_parse_item, false); +} + +/** + * ice_pg_nm_sp_cam_table_get - create a parse graph no match spill cam table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Parse Graph No Match Spill CAM table. + */ +static struct ice_pg_nm_cam_item *ice_pg_nm_sp_cam_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_NOMATCH_SPILL, + sizeof(struct ice_pg_nm_cam_item), + ICE_PG_NM_SP_CAM_TABLE_SIZE, + ice_pg_nm_sp_cam_parse_item, false); +} + +static bool __ice_pg_cam_match(struct ice_pg_cam_item *item, + struct ice_pg_cam_key *key) +{ + return (item->key.valid && + !memcmp(&item->key.val, &key->val, sizeof(key->val))); +} + +static bool __ice_pg_nm_cam_match(struct ice_pg_nm_cam_item *item, + struct ice_pg_cam_key *key) +{ + return (item->key.valid && + !memcmp(&item->key.val, &key->val, sizeof(item->key.val))); +} + +/** + * ice_pg_cam_match - search parse graph cam table by key + * @table: parse graph cam table to search + * @size: cam table size + * @key: search key + * + * Return: a pointer to the matching PG CAM item or NULL. + */ +struct ice_pg_cam_item *ice_pg_cam_match(struct ice_pg_cam_item *table, + int size, struct ice_pg_cam_key *key) +{ + int i; + + for (i = 0; i < size; i++) { + struct ice_pg_cam_item *item = &table[i]; + + if (__ice_pg_cam_match(item, key)) + return item; + } + + return NULL; +} + +/** + * ice_pg_nm_cam_match - search parse graph no match cam table by key + * @table: parse graph no match cam table to search + * @size: cam table size + * @key: search key + * + * Return: a pointer to the matching PG No Match CAM item or NULL. + */ +struct ice_pg_nm_cam_item * +ice_pg_nm_cam_match(struct ice_pg_nm_cam_item *table, int size, + struct ice_pg_cam_key *key) +{ + int i; + + for (i = 0; i < size; i++) { + struct ice_pg_nm_cam_item *item = &table[i]; + + if (__ice_pg_nm_cam_match(item, key)) + return item; + } + + return NULL; +} + +/*** Ternary match ***/ +/* Perform a ternary match on a 1-byte pattern (@pat) given @key and @key_inv + * Rules (per bit): + * Key == 0 and Key_inv == 0 : Never match (Don't care) + * Key == 0 and Key_inv == 1 : Match on bit == 1 + * Key == 1 and Key_inv == 0 : Match on bit == 0 + * Key == 1 and Key_inv == 1 : Always match (Don't care) + * + * Return: true if all bits match, false otherwise. + */ +static bool ice_ternary_match_byte(u8 key, u8 key_inv, u8 pat) +{ + u8 bit_key, bit_key_inv, bit_pat; + int i; + + for (i = 0; i < BITS_PER_BYTE; i++) { + bit_key = key & BIT(i); + bit_key_inv = key_inv & BIT(i); + bit_pat = pat & BIT(i); + + if (bit_key != 0 && bit_key_inv != 0) + continue; + + if ((bit_key == 0 && bit_key_inv == 0) || bit_key == bit_pat) + return false; + } + + return true; +} + +static bool ice_ternary_match(const u8 *key, const u8 *key_inv, + const u8 *pat, int len) +{ + int i; + + for (i = 0; i < len; i++) + if (!ice_ternary_match_byte(key[i], key_inv[i], pat[i])) + return false; + + return true; +} + +/*** ICE_SID_RXPARSER_BOOST_TCAM and ICE_SID_LBL_RXPARSER_TMEM sections ***/ +static void ice_bst_np_kb_dump(struct ice_hw *hw, struct ice_np_keybuilder *kb) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "next proto key builder:\n"); + dev_info(dev, "\topc = %d\n", kb->opc); + dev_info(dev, "\tstart_reg0 = %d\n", kb->start_reg0); + dev_info(dev, "\tlen_reg1 = %d\n", kb->len_reg1); +} + +static void ice_bst_pg_kb_dump(struct ice_hw *hw, struct ice_pg_keybuilder *kb) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "parse graph key builder:\n"); + dev_info(dev, "\tflag0_ena = %d\n", kb->flag0_ena); + dev_info(dev, "\tflag1_ena = %d\n", kb->flag1_ena); + dev_info(dev, "\tflag2_ena = %d\n", kb->flag2_ena); + dev_info(dev, "\tflag3_ena = %d\n", kb->flag3_ena); + dev_info(dev, "\tflag0_idx = %d\n", kb->flag0_idx); + dev_info(dev, "\tflag1_idx = %d\n", kb->flag1_idx); + dev_info(dev, "\tflag2_idx = %d\n", kb->flag2_idx); + dev_info(dev, "\tflag3_idx = %d\n", kb->flag3_idx); + dev_info(dev, "\talu_reg_idx = %d\n", kb->alu_reg_idx); +} + +static void ice_bst_alu_dump(struct ice_hw *hw, struct ice_alu *alu, int idx) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "alu%d:\n", idx); + dev_info(dev, "\topc = %d\n", alu->opc); + dev_info(dev, "\tsrc_start = %d\n", alu->src_start); + dev_info(dev, "\tsrc_len = %d\n", alu->src_len); + dev_info(dev, "\tshift_xlate_sel = %d\n", alu->shift_xlate_sel); + dev_info(dev, "\tshift_xlate_key = %d\n", alu->shift_xlate_key); + dev_info(dev, "\tsrc_reg_id = %d\n", alu->src_reg_id); + dev_info(dev, "\tdst_reg_id = %d\n", alu->dst_reg_id); + dev_info(dev, "\tinc0 = %d\n", alu->inc0); + dev_info(dev, "\tinc1 = %d\n", alu->inc1); + dev_info(dev, "\tproto_offset_opc = %d\n", alu->proto_offset_opc); + dev_info(dev, "\tproto_offset = %d\n", alu->proto_offset); + dev_info(dev, "\tbranch_addr = %d\n", alu->branch_addr); + dev_info(dev, "\timm = %d\n", alu->imm); + dev_info(dev, "\tdst_start = %d\n", alu->dst_start); + dev_info(dev, "\tdst_len = %d\n", alu->dst_len); + dev_info(dev, "\tflags_extr_imm = %d\n", alu->flags_extr_imm); + dev_info(dev, "\tflags_start_imm= %d\n", alu->flags_start_imm); +} + +/** + * ice_bst_tcam_dump - dump a boost tcam info + * @hw: pointer to the hardware structure + * @item: boost tcam to dump + */ +static void ice_bst_tcam_dump(struct ice_hw *hw, struct ice_bst_tcam_item *item) +{ + struct device *dev = ice_hw_to_dev(hw); + int i; + + dev_info(dev, "addr = %d\n", item->addr); + + dev_info(dev, "key : "); + for (i = 0; i < ICE_BST_TCAM_KEY_SIZE; i++) + dev_info(dev, "%02x ", item->key[i]); + + dev_info(dev, "\n"); + + dev_info(dev, "key_inv: "); + for (i = 0; i < ICE_BST_TCAM_KEY_SIZE; i++) + dev_info(dev, "%02x ", item->key_inv[i]); + + dev_info(dev, "\n"); + + dev_info(dev, "hit_idx_grp = %d\n", item->hit_idx_grp); + dev_info(dev, "pg_prio = %d\n", item->pg_prio); + + ice_bst_np_kb_dump(hw, &item->np_kb); + ice_bst_pg_kb_dump(hw, &item->pg_kb); + + ice_bst_alu_dump(hw, &item->alu0, ICE_ALU0_IDX); + ice_bst_alu_dump(hw, &item->alu1, ICE_ALU1_IDX); + ice_bst_alu_dump(hw, &item->alu2, ICE_ALU2_IDX); +} + +static void ice_lbl_dump(struct ice_hw *hw, struct ice_lbl_item *item) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "index = %u\n", item->idx); + dev_info(dev, "type = %u\n", item->type); + dev_info(dev, "label = %s\n", item->label); +} + +#define ICE_BST_ALU_OPC GENMASK_ULL(5, 0) +#define ICE_BST_ALU_SS GENMASK_ULL(13, 6) +#define ICE_BST_ALU_SL GENMASK_ULL(18, 14) +#define ICE_BST_ALU_SXS BIT_ULL(19) +#define ICE_BST_ALU_SXK GENMASK_ULL(23, 20) +#define ICE_BST_ALU_SRID GENMASK_ULL(30, 24) +#define ICE_BST_ALU_DRID GENMASK_ULL(37, 31) +#define ICE_BST_ALU_INC0 BIT_ULL(38) +#define ICE_BST_ALU_INC1 BIT_ULL(39) +#define ICE_BST_ALU_POO GENMASK_ULL(41, 40) +#define ICE_BST_ALU_PO GENMASK_ULL(49, 42) +#define ICE_BST_ALU_BA_S 50 /* offset for the 2nd 64-bits field */ +#define ICE_BST_ALU_BA GENMASK_ULL(57 - ICE_BST_ALU_BA_S, \ + 50 - ICE_BST_ALU_BA_S) +#define ICE_BST_ALU_IMM GENMASK_ULL(73 - ICE_BST_ALU_BA_S, \ + 58 - ICE_BST_ALU_BA_S) +#define ICE_BST_ALU_DFE BIT_ULL(74 - ICE_BST_ALU_BA_S) +#define ICE_BST_ALU_DS GENMASK_ULL(80 - ICE_BST_ALU_BA_S, \ + 75 - ICE_BST_ALU_BA_S) +#define ICE_BST_ALU_DL GENMASK_ULL(86 - ICE_BST_ALU_BA_S, \ + 81 - ICE_BST_ALU_BA_S) +#define ICE_BST_ALU_FEI BIT_ULL(87 - ICE_BST_ALU_BA_S) +#define ICE_BST_ALU_FSI GENMASK_ULL(95 - ICE_BST_ALU_BA_S, \ + 88 - ICE_BST_ALU_BA_S) + +/** + * ice_bst_alu_init - parse 96 bits of ALU entry + * @alu: pointer to the ALU entry structure + * @data: ALU entry data to be parsed + * @off: offset of the ALU entry data + */ +static void ice_bst_alu_init(struct ice_alu *alu, u8 *data, u8 off) +{ + u64 d64; + u8 idd; + + d64 = *((u64 *)data) >> off; + + alu->opc = FIELD_GET(ICE_BST_ALU_OPC, d64); + alu->src_start = FIELD_GET(ICE_BST_ALU_SS, d64); + alu->src_len = FIELD_GET(ICE_BST_ALU_SL, d64); + alu->shift_xlate_sel = FIELD_GET(ICE_BST_ALU_SXS, d64); + alu->shift_xlate_key = FIELD_GET(ICE_BST_ALU_SXK, d64); + alu->src_reg_id = FIELD_GET(ICE_BST_ALU_SRID, d64); + alu->dst_reg_id = FIELD_GET(ICE_BST_ALU_DRID, d64); + alu->inc0 = FIELD_GET(ICE_BST_ALU_INC0, d64); + alu->inc1 = FIELD_GET(ICE_BST_ALU_INC1, d64); + alu->proto_offset_opc = FIELD_GET(ICE_BST_ALU_POO, d64); + alu->proto_offset = FIELD_GET(ICE_BST_ALU_PO, d64); + + idd = (ICE_BST_ALU_BA_S + off) / BITS_PER_BYTE; + off = (ICE_BST_ALU_BA_S + off) % BITS_PER_BYTE; + d64 = *((u64 *)(&data[idd])) >> off; + + alu->branch_addr = FIELD_GET(ICE_BST_ALU_BA, d64); + alu->imm = FIELD_GET(ICE_BST_ALU_IMM, d64); + alu->dedicate_flags_ena = FIELD_GET(ICE_BST_ALU_DFE, d64); + alu->dst_start = FIELD_GET(ICE_BST_ALU_DS, d64); + alu->dst_len = FIELD_GET(ICE_BST_ALU_DL, d64); + alu->flags_extr_imm = FIELD_GET(ICE_BST_ALU_FEI, d64); + alu->flags_start_imm = FIELD_GET(ICE_BST_ALU_FSI, d64); +} + +#define ICE_BST_PGKB_F0_ENA BIT_ULL(0) +#define ICE_BST_PGKB_F0_IDX GENMASK_ULL(6, 1) +#define ICE_BST_PGKB_F1_ENA BIT_ULL(7) +#define ICE_BST_PGKB_F1_IDX GENMASK_ULL(13, 8) +#define ICE_BST_PGKB_F2_ENA BIT_ULL(14) +#define ICE_BST_PGKB_F2_IDX GENMASK_ULL(20, 15) +#define ICE_BST_PGKB_F3_ENA BIT_ULL(21) +#define ICE_BST_PGKB_F3_IDX GENMASK_ULL(27, 22) +#define ICE_BST_PGKB_AR_IDX GENMASK_ULL(34, 28) + +/** + * ice_bst_pgkb_init - parse 35 bits of Parse Graph Key Build + * @kb: pointer to the Parse Graph Key Build structure + * @data: Parse Graph Key Build data to be parsed + */ +static void ice_bst_pgkb_init(struct ice_pg_keybuilder *kb, u64 data) +{ + kb->flag0_ena = FIELD_GET(ICE_BST_PGKB_F0_ENA, data); + kb->flag0_idx = FIELD_GET(ICE_BST_PGKB_F0_IDX, data); + kb->flag1_ena = FIELD_GET(ICE_BST_PGKB_F1_ENA, data); + kb->flag1_idx = FIELD_GET(ICE_BST_PGKB_F1_IDX, data); + kb->flag2_ena = FIELD_GET(ICE_BST_PGKB_F2_ENA, data); + kb->flag2_idx = FIELD_GET(ICE_BST_PGKB_F2_IDX, data); + kb->flag3_ena = FIELD_GET(ICE_BST_PGKB_F3_ENA, data); + kb->flag3_idx = FIELD_GET(ICE_BST_PGKB_F3_IDX, data); + kb->alu_reg_idx = FIELD_GET(ICE_BST_PGKB_AR_IDX, data); +} + +#define ICE_BST_NPKB_OPC GENMASK(1, 0) +#define ICE_BST_NPKB_S_R0 GENMASK(9, 2) +#define ICE_BST_NPKB_L_R1 GENMASK(17, 10) + +/** + * ice_bst_npkb_init - parse 18 bits of Next Protocol Key Build + * @kb: pointer to the Next Protocol Key Build structure + * @data: Next Protocol Key Build data to be parsed + */ +static void ice_bst_npkb_init(struct ice_np_keybuilder *kb, u32 data) +{ + kb->opc = FIELD_GET(ICE_BST_NPKB_OPC, data); + kb->start_reg0 = FIELD_GET(ICE_BST_NPKB_S_R0, data); + kb->len_reg1 = FIELD_GET(ICE_BST_NPKB_L_R1, data); +} + +#define ICE_BT_KEY_S 32 +#define ICE_BT_KEY_IDD (ICE_BT_KEY_S / BITS_PER_BYTE) +#define ICE_BT_KIV_S 192 +#define ICE_BT_KIV_IDD (ICE_BT_KIV_S / BITS_PER_BYTE) +#define ICE_BT_HIG_S 352 +#define ICE_BT_HIG_IDD (ICE_BT_HIG_S / BITS_PER_BYTE) +#define ICE_BT_PGP_S 360 +#define ICE_BT_PGP_IDD (ICE_BT_PGP_S / BITS_PER_BYTE) +#define ICE_BT_PGP_M GENMASK(361 - ICE_BT_PGP_S, 360 - ICE_BT_PGP_S) +#define ICE_BT_NPKB_S 362 +#define ICE_BT_NPKB_IDD (ICE_BT_NPKB_S / BITS_PER_BYTE) +#define ICE_BT_NPKB_OFF (ICE_BT_NPKB_S % BITS_PER_BYTE) +#define ICE_BT_PGKB_S 380 +#define ICE_BT_PGKB_IDD (ICE_BT_PGKB_S / BITS_PER_BYTE) +#define ICE_BT_PGKB_OFF (ICE_BT_PGKB_S % BITS_PER_BYTE) +#define ICE_BT_ALU0_S 415 +#define ICE_BT_ALU0_IDD (ICE_BT_ALU0_S / BITS_PER_BYTE) +#define ICE_BT_ALU0_OFF (ICE_BT_ALU0_S % BITS_PER_BYTE) +#define ICE_BT_ALU1_S 511 +#define ICE_BT_ALU1_IDD (ICE_BT_ALU1_S / BITS_PER_BYTE) +#define ICE_BT_ALU1_OFF (ICE_BT_ALU1_S % BITS_PER_BYTE) +#define ICE_BT_ALU2_S 607 +#define ICE_BT_ALU2_IDD (ICE_BT_ALU2_S / BITS_PER_BYTE) +#define ICE_BT_ALU2_OFF (ICE_BT_ALU2_S % BITS_PER_BYTE) + +/** + * ice_bst_parse_item - parse 704 bits of Boost TCAM entry + * @hw: pointer to the hardware structure + * @idx: index of Boost TCAM entry + * @item: item of Boost TCAM entry + * @data: Boost TCAM entry data to be parsed + * @size: size of Boost TCAM entry + */ +static void ice_bst_parse_item(struct ice_hw *hw, u16 idx, void *item, + void *data, int __maybe_unused size) +{ + struct ice_bst_tcam_item *ti = item; + u8 *buf = (u8 *)data; + int i; + + ti->addr = *(u16 *)buf; + + for (i = 0; i < ICE_BST_TCAM_KEY_SIZE; i++) { + ti->key[i] = buf[ICE_BT_KEY_IDD + i]; + ti->key_inv[i] = buf[ICE_BT_KIV_IDD + i]; + } + ti->hit_idx_grp = buf[ICE_BT_HIG_IDD]; + ti->pg_prio = buf[ICE_BT_PGP_IDD] & ICE_BT_PGP_M; + + ice_bst_npkb_init(&ti->np_kb, + *((u32 *)(&buf[ICE_BT_NPKB_IDD])) >> + ICE_BT_NPKB_OFF); + ice_bst_pgkb_init(&ti->pg_kb, + *((u64 *)(&buf[ICE_BT_PGKB_IDD])) >> + ICE_BT_PGKB_OFF); + + ice_bst_alu_init(&ti->alu0, &buf[ICE_BT_ALU0_IDD], ICE_BT_ALU0_OFF); + ice_bst_alu_init(&ti->alu1, &buf[ICE_BT_ALU1_IDD], ICE_BT_ALU1_OFF); + ice_bst_alu_init(&ti->alu2, &buf[ICE_BT_ALU2_IDD], ICE_BT_ALU2_OFF); + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_bst_tcam_dump(hw, ti); +} + +/** + * ice_bst_tcam_table_get - create a boost tcam table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Boost TCAM table. + */ +static struct ice_bst_tcam_item *ice_bst_tcam_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_BOOST_TCAM, + sizeof(struct ice_bst_tcam_item), + ICE_BST_TCAM_TABLE_SIZE, + ice_bst_parse_item, true); +} + +static void ice_parse_lbl_item(struct ice_hw *hw, u16 idx, void *item, + void *data, int __maybe_unused size) +{ + struct ice_lbl_item *lbl_item = item; + struct ice_lbl_item *lbl_data = data; + + lbl_item->idx = lbl_data->idx; + memcpy(lbl_item->label, lbl_data->label, sizeof(lbl_item->label)); + + if (strstarts(lbl_item->label, ICE_LBL_BST_DVM)) + lbl_item->type = ICE_LBL_BST_TYPE_DVM; + else if (strstarts(lbl_item->label, ICE_LBL_BST_SVM)) + lbl_item->type = ICE_LBL_BST_TYPE_SVM; + else if (strstarts(lbl_item->label, ICE_LBL_TNL_VXLAN)) + lbl_item->type = ICE_LBL_BST_TYPE_VXLAN; + else if (strstarts(lbl_item->label, ICE_LBL_TNL_GENEVE)) + lbl_item->type = ICE_LBL_BST_TYPE_GENEVE; + else if (strstarts(lbl_item->label, ICE_LBL_TNL_UDP_ECPRI)) + lbl_item->type = ICE_LBL_BST_TYPE_UDP_ECPRI; + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_lbl_dump(hw, lbl_item); +} + +/** + * ice_bst_lbl_table_get - create a boost label table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Boost label table. + */ +static struct ice_lbl_item *ice_bst_lbl_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_LBL_RXPARSER_TMEM, + sizeof(struct ice_lbl_item), + ICE_BST_TCAM_TABLE_SIZE, + ice_parse_lbl_item, true); +} + +/** + * ice_bst_tcam_match - match a pattern on the boost tcam table + * @tcam_table: boost tcam table to search + * @pat: pattern to match + * + * Return: a pointer to the matching Boost TCAM item or NULL. + */ +struct ice_bst_tcam_item * +ice_bst_tcam_match(struct ice_bst_tcam_item *tcam_table, u8 *pat) +{ + int i; + + for (i = 0; i < ICE_BST_TCAM_TABLE_SIZE; i++) { + struct ice_bst_tcam_item *item = &tcam_table[i]; + + if (item->hit_idx_grp == 0) + continue; + if (ice_ternary_match(item->key, item->key_inv, pat, + ICE_BST_TCAM_KEY_SIZE)) + return item; + } + + return NULL; +} + +/*** ICE_SID_RXPARSER_MARKER_PTYPE section ***/ +/** + * ice_ptype_mk_tcam_dump - dump an ptype marker tcam info + * @hw: pointer to the hardware structure + * @item: ptype marker tcam to dump + */ +static void ice_ptype_mk_tcam_dump(struct ice_hw *hw, + struct ice_ptype_mk_tcam_item *item) +{ + struct device *dev = ice_hw_to_dev(hw); + int i; + + dev_info(dev, "address = %d\n", item->address); + dev_info(dev, "ptype = %d\n", item->ptype); + + dev_info(dev, "key :"); + for (i = 0; i < ICE_PTYPE_MK_TCAM_KEY_SIZE; i++) + dev_info(dev, "%02x ", item->key[i]); + + dev_info(dev, "\n"); + + dev_info(dev, "key_inv:"); + for (i = 0; i < ICE_PTYPE_MK_TCAM_KEY_SIZE; i++) + dev_info(dev, "%02x ", item->key_inv[i]); + + dev_info(dev, "\n"); +} + +static void ice_parse_ptype_mk_tcam_item(struct ice_hw *hw, u16 idx, + void *item, void *data, int size) +{ + memcpy(item, data, size); + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_ptype_mk_tcam_dump(hw, + (struct ice_ptype_mk_tcam_item *)item); +} + +/** + * ice_ptype_mk_tcam_table_get - create a ptype marker tcam table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Marker PType TCAM table. + */ +static +struct ice_ptype_mk_tcam_item *ice_ptype_mk_tcam_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_MARKER_PTYPE, + sizeof(struct ice_ptype_mk_tcam_item), + ICE_PTYPE_MK_TCAM_TABLE_SIZE, + ice_parse_ptype_mk_tcam_item, true); +} + +/** + * ice_ptype_mk_tcam_match - match a pattern on a ptype marker tcam table + * @table: ptype marker tcam table to search + * @pat: pattern to match + * @len: length of the pattern + * + * Return: a pointer to the matching Marker PType item or NULL. + */ +struct ice_ptype_mk_tcam_item * +ice_ptype_mk_tcam_match(struct ice_ptype_mk_tcam_item *table, + u8 *pat, int len) +{ + int i; + + for (i = 0; i < ICE_PTYPE_MK_TCAM_TABLE_SIZE; i++) { + struct ice_ptype_mk_tcam_item *item = &table[i]; + + if (ice_ternary_match(item->key, item->key_inv, pat, len)) + return item; + } + + return NULL; +} + +/*** ICE_SID_RXPARSER_MARKER_GRP section ***/ +/** + * ice_mk_grp_dump - dump an marker group item info + * @hw: pointer to the hardware structure + * @item: marker group item to dump + */ +static void ice_mk_grp_dump(struct ice_hw *hw, struct ice_mk_grp_item *item) +{ + struct device *dev = ice_hw_to_dev(hw); + int i; + + dev_info(dev, "index = %d\n", item->idx); + + dev_info(dev, "markers: "); + for (i = 0; i < ICE_MK_COUNT_PER_GRP; i++) + dev_info(dev, "%d ", item->markers[i]); + + dev_info(dev, "\n"); +} + +static void ice_mk_grp_parse_item(struct ice_hw *hw, u16 idx, void *item, + void *data, int __maybe_unused size) +{ + struct ice_mk_grp_item *grp = item; + u8 *buf = data; + int i; + + grp->idx = idx; + + for (i = 0; i < ICE_MK_COUNT_PER_GRP; i++) + grp->markers[i] = buf[i]; + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_mk_grp_dump(hw, grp); +} + +/** + * ice_mk_grp_table_get - create a marker group table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Marker Group ID table. + */ +static struct ice_mk_grp_item *ice_mk_grp_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_MARKER_GRP, + sizeof(struct ice_mk_grp_item), + ICE_MK_GRP_TABLE_SIZE, + ice_mk_grp_parse_item, false); +} + +/*** ICE_SID_RXPARSER_PROTO_GRP section ***/ +static void ice_proto_off_dump(struct ice_hw *hw, + struct ice_proto_off *po, int idx) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "proto %d\n", idx); + dev_info(dev, "\tpolarity = %d\n", po->polarity); + dev_info(dev, "\tproto_id = %d\n", po->proto_id); + dev_info(dev, "\toffset = %d\n", po->offset); +} + +/** + * ice_proto_grp_dump - dump a proto group item info + * @hw: pointer to the hardware structure + * @item: proto group item to dump + */ +static void ice_proto_grp_dump(struct ice_hw *hw, + struct ice_proto_grp_item *item) +{ + int i; + + dev_info(ice_hw_to_dev(hw), "index = %d\n", item->idx); + + for (i = 0; i < ICE_PROTO_COUNT_PER_GRP; i++) + ice_proto_off_dump(hw, &item->po[i], i); +} + +#define ICE_PO_POL BIT(0) +#define ICE_PO_PID GENMASK(8, 1) +#define ICE_PO_OFF GENMASK(21, 12) + +/** + * ice_proto_off_parse - parse 22 bits of Protocol entry + * @po: pointer to the Protocol entry structure + * @data: Protocol entry data to be parsed + */ +static void ice_proto_off_parse(struct ice_proto_off *po, u32 data) +{ + po->polarity = FIELD_GET(ICE_PO_POL, data); + po->proto_id = FIELD_GET(ICE_PO_PID, data); + po->offset = FIELD_GET(ICE_PO_OFF, data); +} + +/** + * ice_proto_grp_parse_item - parse 192 bits of Protocol Group Table entry + * @hw: pointer to the hardware structure + * @idx: index of Protocol Group Table entry + * @item: item of Protocol Group Table entry + * @data: Protocol Group Table entry data to be parsed + * @size: size of Protocol Group Table entry + */ +static void ice_proto_grp_parse_item(struct ice_hw *hw, u16 idx, void *item, + void *data, int __maybe_unused size) +{ + struct ice_proto_grp_item *grp = item; + u8 *buf = (u8 *)data; + u8 idd, off; + u32 d32; + int i; + + grp->idx = idx; + + for (i = 0; i < ICE_PROTO_COUNT_PER_GRP; i++) { + idd = (ICE_PROTO_GRP_ITEM_SIZE * i) / BITS_PER_BYTE; + off = (ICE_PROTO_GRP_ITEM_SIZE * i) % BITS_PER_BYTE; + d32 = *((u32 *)&buf[idd]) >> off; + ice_proto_off_parse(&grp->po[i], d32); + } + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_proto_grp_dump(hw, grp); +} + +/** + * ice_proto_grp_table_get - create a proto group table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Protocol Group table. + */ +static struct ice_proto_grp_item *ice_proto_grp_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_PROTO_GRP, + sizeof(struct ice_proto_grp_item), + ICE_PROTO_GRP_TABLE_SIZE, + ice_proto_grp_parse_item, false); +} + +/*** ICE_SID_RXPARSER_FLAG_REDIR section ***/ +/** + * ice_flg_rd_dump - dump a flag redirect item info + * @hw: pointer to the hardware structure + * @item: flag redirect item to dump + */ +static void ice_flg_rd_dump(struct ice_hw *hw, struct ice_flg_rd_item *item) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "index = %d\n", item->idx); + dev_info(dev, "expose = %d\n", item->expose); + dev_info(dev, "intr_flg_id = %d\n", item->intr_flg_id); +} + +#define ICE_FRT_EXPO BIT(0) +#define ICE_FRT_IFID GENMASK(6, 1) + +/** + * ice_flg_rd_parse_item - parse 8 bits of Flag Redirect Table entry + * @hw: pointer to the hardware structure + * @idx: index of Flag Redirect Table entry + * @item: item of Flag Redirect Table entry + * @data: Flag Redirect Table entry data to be parsed + * @size: size of Flag Redirect Table entry + */ +static void ice_flg_rd_parse_item(struct ice_hw *hw, u16 idx, void *item, + void *data, int __maybe_unused size) +{ + struct ice_flg_rd_item *rdi = item; + u8 d8 = *(u8 *)data; + + rdi->idx = idx; + rdi->expose = FIELD_GET(ICE_FRT_EXPO, d8); + rdi->intr_flg_id = FIELD_GET(ICE_FRT_IFID, d8); + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_flg_rd_dump(hw, rdi); +} + +/** + * ice_flg_rd_table_get - create a flag redirect table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Flags Redirection table. + */ +static struct ice_flg_rd_item *ice_flg_rd_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_FLAG_REDIR, + sizeof(struct ice_flg_rd_item), + ICE_FLG_RD_TABLE_SIZE, + ice_flg_rd_parse_item, false); +} + +/** + * ice_flg_redirect - redirect a parser flag to packet flag + * @table: flag redirect table + * @psr_flg: parser flag to redirect + * + * Return: flag or 0 if @psr_flag = 0. + */ +u64 ice_flg_redirect(struct ice_flg_rd_item *table, u64 psr_flg) +{ + u64 flg = 0; + int i; + + for (i = 0; i < ICE_FLG_RDT_SIZE; i++) { + struct ice_flg_rd_item *item = &table[i]; + + if (!item->expose) + continue; + + if (psr_flg & BIT(item->intr_flg_id)) + flg |= BIT(i); + } + + return flg; +} + +/*** ICE_SID_XLT_KEY_BUILDER_SW, ICE_SID_XLT_KEY_BUILDER_ACL, + * ICE_SID_XLT_KEY_BUILDER_FD and ICE_SID_XLT_KEY_BUILDER_RSS + * sections ***/ +static void ice_xlt_kb_entry_dump(struct ice_hw *hw, + struct ice_xlt_kb_entry *entry, int idx) +{ + struct device *dev = ice_hw_to_dev(hw); + int i; + + dev_info(dev, "key builder entry %d\n", idx); + dev_info(dev, "\txlt1_ad_sel = %d\n", entry->xlt1_ad_sel); + dev_info(dev, "\txlt2_ad_sel = %d\n", entry->xlt2_ad_sel); + + for (i = 0; i < ICE_XLT_KB_FLAG0_14_CNT; i++) + dev_info(dev, "\tflg%d_sel = %d\n", i, entry->flg0_14_sel[i]); + + dev_info(dev, "\txlt1_md_sel = %d\n", entry->xlt1_md_sel); + dev_info(dev, "\txlt2_md_sel = %d\n", entry->xlt2_md_sel); +} + +/** + * ice_xlt_kb_dump - dump a xlt key build info + * @hw: pointer to the hardware structure + * @kb: key build to dump + */ +static void ice_xlt_kb_dump(struct ice_hw *hw, struct ice_xlt_kb *kb) +{ + struct device *dev = ice_hw_to_dev(hw); + int i; + + dev_info(dev, "xlt1_pm = %d\n", kb->xlt1_pm); + dev_info(dev, "xlt2_pm = %d\n", kb->xlt2_pm); + dev_info(dev, "prof_id_pm = %d\n", kb->prof_id_pm); + dev_info(dev, "flag15 lo = 0x%08x\n", (u32)kb->flag15); + dev_info(dev, "flag15 hi = 0x%08x\n", + (u32)(kb->flag15 >> (sizeof(u32) * BITS_PER_BYTE))); + + for (i = 0; i < ICE_XLT_KB_TBL_CNT; i++) + ice_xlt_kb_entry_dump(hw, &kb->entries[i], i); +} + +#define ICE_XLT_KB_X1AS_S 32 /* offset for the 1st 64-bits field */ +#define ICE_XLT_KB_X1AS_IDD (ICE_XLT_KB_X1AS_S / BITS_PER_BYTE) +#define ICE_XLT_KB_X1AS_OFF (ICE_XLT_KB_X1AS_S % BITS_PER_BYTE) +#define ICE_XLT_KB_X1AS GENMASK_ULL(34 - ICE_XLT_KB_X1AS_S, \ + 32 - ICE_XLT_KB_X1AS_S) +#define ICE_XLT_KB_X2AS GENMASK_ULL(37 - ICE_XLT_KB_X1AS_S, \ + 35 - ICE_XLT_KB_X1AS_S) +#define ICE_XLT_KB_FL00 GENMASK_ULL(46 - ICE_XLT_KB_X1AS_S, \ + 38 - ICE_XLT_KB_X1AS_S) +#define ICE_XLT_KB_FL01 GENMASK_ULL(55 - ICE_XLT_KB_X1AS_S, \ + 47 - ICE_XLT_KB_X1AS_S) +#define ICE_XLT_KB_FL02 GENMASK_ULL(64 - ICE_XLT_KB_X1AS_S, \ + 56 - ICE_XLT_KB_X1AS_S) +#define ICE_XLT_KB_FL03 GENMASK_ULL(73 - ICE_XLT_KB_X1AS_S, \ + 65 - ICE_XLT_KB_X1AS_S) +#define ICE_XLT_KB_FL04 GENMASK_ULL(82 - ICE_XLT_KB_X1AS_S, \ + 74 - ICE_XLT_KB_X1AS_S) +#define ICE_XLT_KB_FL05 GENMASK_ULL(91 - ICE_XLT_KB_X1AS_S, \ + 83 - ICE_XLT_KB_X1AS_S) +#define ICE_XLT_KB_FL06_S 92 /* offset for the 2nd 64-bits field */ +#define ICE_XLT_KB_FL06_IDD (ICE_XLT_KB_FL06_S / BITS_PER_BYTE) +#define ICE_XLT_KB_FL06_OFF (ICE_XLT_KB_FL06_S % BITS_PER_BYTE) +#define ICE_XLT_KB_FL06 GENMASK_ULL(100 - ICE_XLT_KB_FL06_S, \ + 92 - ICE_XLT_KB_FL06_S) +#define ICE_XLT_KB_FL07 GENMASK_ULL(109 - ICE_XLT_KB_FL06_S, \ + 101 - ICE_XLT_KB_FL06_S) +#define ICE_XLT_KB_FL08 GENMASK_ULL(118 - ICE_XLT_KB_FL06_S, \ + 110 - ICE_XLT_KB_FL06_S) +#define ICE_XLT_KB_FL09 GENMASK_ULL(127 - ICE_XLT_KB_FL06_S, \ + 119 - ICE_XLT_KB_FL06_S) +#define ICE_XLT_KB_FL10 GENMASK_ULL(136 - ICE_XLT_KB_FL06_S, \ + 128 - ICE_XLT_KB_FL06_S) +#define ICE_XLT_KB_FL11 GENMASK_ULL(145 - ICE_XLT_KB_FL06_S, \ + 137 - ICE_XLT_KB_FL06_S) +#define ICE_XLT_KB_FL12_S 146 /* offset for the 3rd 64-bits field */ +#define ICE_XLT_KB_FL12_IDD (ICE_XLT_KB_FL12_S / BITS_PER_BYTE) +#define ICE_XLT_KB_FL12_OFF (ICE_XLT_KB_FL12_S % BITS_PER_BYTE) +#define ICE_XLT_KB_FL12 GENMASK_ULL(154 - ICE_XLT_KB_FL12_S, \ + 146 - ICE_XLT_KB_FL12_S) +#define ICE_XLT_KB_FL13 GENMASK_ULL(163 - ICE_XLT_KB_FL12_S, \ + 155 - ICE_XLT_KB_FL12_S) +#define ICE_XLT_KB_FL14 GENMASK_ULL(181 - ICE_XLT_KB_FL12_S, \ + 164 - ICE_XLT_KB_FL12_S) +#define ICE_XLT_KB_X1MS GENMASK_ULL(186 - ICE_XLT_KB_FL12_S, \ + 182 - ICE_XLT_KB_FL12_S) +#define ICE_XLT_KB_X2MS GENMASK_ULL(191 - ICE_XLT_KB_FL12_S, \ + 187 - ICE_XLT_KB_FL12_S) + +/** + * ice_kb_entry_init - parse 192 bits of XLT Key Builder entry + * @entry: pointer to the XLT Key Builder entry structure + * @data: XLT Key Builder entry data to be parsed + */ +static void ice_kb_entry_init(struct ice_xlt_kb_entry *entry, u8 *data) +{ + u8 i = 0; + u64 d64; + + d64 = *((u64 *)&data[ICE_XLT_KB_X1AS_IDD]) >> ICE_XLT_KB_X1AS_OFF; + + entry->xlt1_ad_sel = FIELD_GET(ICE_XLT_KB_X1AS, d64); + entry->xlt2_ad_sel = FIELD_GET(ICE_XLT_KB_X2AS, d64); + + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL00, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL01, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL02, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL03, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL04, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL05, d64); + + d64 = *((u64 *)&data[ICE_XLT_KB_FL06_IDD]) >> ICE_XLT_KB_FL06_OFF; + + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL06, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL07, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL08, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL09, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL10, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL11, d64); + + d64 = *((u64 *)&data[ICE_XLT_KB_FL12_IDD]) >> ICE_XLT_KB_FL12_OFF; + + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL12, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL13, d64); + entry->flg0_14_sel[i] = FIELD_GET(ICE_XLT_KB_FL14, d64); + + entry->xlt1_md_sel = FIELD_GET(ICE_XLT_KB_X1MS, d64); + entry->xlt2_md_sel = FIELD_GET(ICE_XLT_KB_X2MS, d64); +} + +#define ICE_XLT_KB_X1PM_OFF 0 +#define ICE_XLT_KB_X2PM_OFF 1 +#define ICE_XLT_KB_PIPM_OFF 2 +#define ICE_XLT_KB_FL15_OFF 4 +#define ICE_XLT_KB_TBL_OFF 12 + +/** + * ice_parse_kb_data - parse 204 bits of XLT Key Build Table + * @hw: pointer to the hardware structure + * @kb: pointer to the XLT Key Build Table structure + * @data: XLT Key Build Table data to be parsed + */ +static void ice_parse_kb_data(struct ice_hw *hw, struct ice_xlt_kb *kb, + void *data) +{ + u8 *buf = data; + int i; + + kb->xlt1_pm = buf[ICE_XLT_KB_X1PM_OFF]; + kb->xlt2_pm = buf[ICE_XLT_KB_X2PM_OFF]; + kb->prof_id_pm = buf[ICE_XLT_KB_PIPM_OFF]; + + kb->flag15 = *(u64 *)&buf[ICE_XLT_KB_FL15_OFF]; + for (i = 0; i < ICE_XLT_KB_TBL_CNT; i++) + ice_kb_entry_init(&kb->entries[i], + &buf[ICE_XLT_KB_TBL_OFF + + i * ICE_XLT_KB_TBL_ENTRY_SIZE]); + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_xlt_kb_dump(hw, kb); +} + +static struct ice_xlt_kb *ice_xlt_kb_get(struct ice_hw *hw, u32 sect_type) +{ + struct ice_pkg_enum state = {}; + struct ice_seg *seg = hw->seg; + struct ice_xlt_kb *kb; + void *data; + + if (!seg) + return ERR_PTR(-EINVAL); + + kb = kzalloc(sizeof(*kb), GFP_KERNEL); + if (!kb) + return ERR_PTR(-ENOMEM); + + data = ice_pkg_enum_section(seg, &state, sect_type); + if (!data) { + ice_debug(hw, ICE_DBG_PARSER, "failed to find section type %d.\n", + sect_type); + kfree(kb); + return ERR_PTR(-EINVAL); + } + + ice_parse_kb_data(hw, kb, data); + + return kb; +} + +/** + * ice_xlt_kb_get_sw - create switch xlt key build + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Key Builder table for Switch. + */ +static struct ice_xlt_kb *ice_xlt_kb_get_sw(struct ice_hw *hw) +{ + return ice_xlt_kb_get(hw, ICE_SID_XLT_KEY_BUILDER_SW); +} + +/** + * ice_xlt_kb_get_acl - create acl xlt key build + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Key Builder table for ACL. + */ +static struct ice_xlt_kb *ice_xlt_kb_get_acl(struct ice_hw *hw) +{ + return ice_xlt_kb_get(hw, ICE_SID_XLT_KEY_BUILDER_ACL); +} + +/** + * ice_xlt_kb_get_fd - create fdir xlt key build + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Key Builder table for Flow Director. + */ +static struct ice_xlt_kb *ice_xlt_kb_get_fd(struct ice_hw *hw) +{ + return ice_xlt_kb_get(hw, ICE_SID_XLT_KEY_BUILDER_FD); +} + +/** + * ice_xlt_kb_get_rss - create rss xlt key build + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Key Builder table for RSS. + */ +static struct ice_xlt_kb *ice_xlt_kb_get_rss(struct ice_hw *hw) +{ + return ice_xlt_kb_get(hw, ICE_SID_XLT_KEY_BUILDER_RSS); +} + +#define ICE_XLT_KB_MASK GENMASK_ULL(5, 0) + +/** + * ice_xlt_kb_flag_get - aggregate 64 bits packet flag into 16 bits xlt flag + * @kb: xlt key build + * @pkt_flag: 64 bits packet flag + * + * Return: XLT flag or 0 if @pkt_flag = 0. + */ +u16 ice_xlt_kb_flag_get(struct ice_xlt_kb *kb, u64 pkt_flag) +{ + struct ice_xlt_kb_entry *entry = &kb->entries[0]; + u16 flag = 0; + int i; + + /* check flag 15 */ + if (kb->flag15 & pkt_flag) + flag = BIT(ICE_XLT_KB_FLAG0_14_CNT); + + /* check flag 0 - 14 */ + for (i = 0; i < ICE_XLT_KB_FLAG0_14_CNT; i++) { + /* only check first entry */ + u16 idx = entry->flg0_14_sel[i] & ICE_XLT_KB_MASK; + + if (pkt_flag & BIT(idx)) + flag |= (u16)BIT(i); + } + + return flag; +} + +/*** Parser API ***/ +/** + * ice_parser_create - create a parser instance + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated parser instance or ERR_PTR + * in case of error. + */ +struct ice_parser *ice_parser_create(struct ice_hw *hw) +{ + struct ice_parser *p; + void *err; + + p = kzalloc(sizeof(*p), GFP_KERNEL); + if (!p) + return ERR_PTR(-ENOMEM); + + p->hw = hw; + p->rt.psr = p; + + p->imem_table = ice_imem_table_get(hw); + if (IS_ERR(p->imem_table)) { + err = p->imem_table; + goto err; + } + + p->mi_table = ice_metainit_table_get(hw); + if (IS_ERR(p->mi_table)) { + err = p->mi_table; + goto err; + } + + p->pg_cam_table = ice_pg_cam_table_get(hw); + if (IS_ERR(p->pg_cam_table)) { + err = p->pg_cam_table; + goto err; + } + + p->pg_sp_cam_table = ice_pg_sp_cam_table_get(hw); + if (IS_ERR(p->pg_sp_cam_table)) { + err = p->pg_sp_cam_table; + goto err; + } + + p->pg_nm_cam_table = ice_pg_nm_cam_table_get(hw); + if (IS_ERR(p->pg_nm_cam_table)) { + err = p->pg_nm_cam_table; + goto err; + } + + p->pg_nm_sp_cam_table = ice_pg_nm_sp_cam_table_get(hw); + if (IS_ERR(p->pg_nm_sp_cam_table)) { + err = p->pg_nm_sp_cam_table; + goto err; + } + + p->bst_tcam_table = ice_bst_tcam_table_get(hw); + if (IS_ERR(p->bst_tcam_table)) { + err = p->bst_tcam_table; + goto err; + } + + p->bst_lbl_table = ice_bst_lbl_table_get(hw); + if (IS_ERR(p->bst_lbl_table)) { + err = p->bst_lbl_table; + goto err; + } + + p->ptype_mk_tcam_table = ice_ptype_mk_tcam_table_get(hw); + if (IS_ERR(p->ptype_mk_tcam_table)) { + err = p->ptype_mk_tcam_table; + goto err; + } + + p->mk_grp_table = ice_mk_grp_table_get(hw); + if (IS_ERR(p->mk_grp_table)) { + err = p->mk_grp_table; + goto err; + } + + p->proto_grp_table = ice_proto_grp_table_get(hw); + if (IS_ERR(p->proto_grp_table)) { + err = p->proto_grp_table; + goto err; + } + + p->flg_rd_table = ice_flg_rd_table_get(hw); + if (IS_ERR(p->flg_rd_table)) { + err = p->flg_rd_table; + goto err; + } + + p->xlt_kb_sw = ice_xlt_kb_get_sw(hw); + if (IS_ERR(p->xlt_kb_sw)) { + err = p->xlt_kb_sw; + goto err; + } + + p->xlt_kb_acl = ice_xlt_kb_get_acl(hw); + if (IS_ERR(p->xlt_kb_acl)) { + err = p->xlt_kb_acl; + goto err; + } + + p->xlt_kb_fd = ice_xlt_kb_get_fd(hw); + if (IS_ERR(p->xlt_kb_fd)) { + err = p->xlt_kb_fd; + goto err; + } + + p->xlt_kb_rss = ice_xlt_kb_get_rss(hw); + if (IS_ERR(p->xlt_kb_rss)) { + err = p->xlt_kb_rss; + goto err; + } + + return p; +err: + ice_parser_destroy(p); + return err; +} + +/** + * ice_parser_destroy - destroy a parser instance + * @psr: pointer to a parser instance + */ +void ice_parser_destroy(struct ice_parser *psr) +{ + kfree(psr->imem_table); + kfree(psr->mi_table); + kfree(psr->pg_cam_table); + kfree(psr->pg_sp_cam_table); + kfree(psr->pg_nm_cam_table); + kfree(psr->pg_nm_sp_cam_table); + kfree(psr->bst_tcam_table); + kfree(psr->bst_lbl_table); + kfree(psr->ptype_mk_tcam_table); + kfree(psr->mk_grp_table); + kfree(psr->proto_grp_table); + kfree(psr->flg_rd_table); + kfree(psr->xlt_kb_sw); + kfree(psr->xlt_kb_acl); + kfree(psr->xlt_kb_fd); + kfree(psr->xlt_kb_rss); + + kfree(psr); +} + +/** + * ice_parser_run - parse on a packet in binary and return the result + * @psr: pointer to a parser instance + * @pkt_buf: packet data + * @pkt_len: packet length + * @rslt: input/output parameter to save parser result. + * + * Return: 0 on success or errno. + */ +int ice_parser_run(struct ice_parser *psr, const u8 *pkt_buf, + int pkt_len, struct ice_parser_result *rslt) +{ + ice_parser_rt_reset(&psr->rt); + ice_parser_rt_pktbuf_set(&psr->rt, pkt_buf, pkt_len); + + return ice_parser_rt_execute(&psr->rt, rslt); +} + +/** + * ice_parser_result_dump - dump a parser result info + * @hw: pointer to the hardware structure + * @rslt: parser result info to dump + */ +void ice_parser_result_dump(struct ice_hw *hw, struct ice_parser_result *rslt) +{ + struct device *dev = ice_hw_to_dev(hw); + int i; + + dev_info(dev, "ptype = %d\n", rslt->ptype); + for (i = 0; i < rslt->po_num; i++) + dev_info(dev, "proto = %d, offset = %d\n", + rslt->po[i].proto_id, rslt->po[i].offset); + + dev_info(dev, "flags_psr = 0x%016llx\n", rslt->flags_psr); + dev_info(dev, "flags_pkt = 0x%016llx\n", rslt->flags_pkt); + dev_info(dev, "flags_sw = 0x%04x\n", rslt->flags_sw); + dev_info(dev, "flags_fd = 0x%04x\n", rslt->flags_fd); + dev_info(dev, "flags_rss = 0x%04x\n", rslt->flags_rss); +} + +#define ICE_BT_VLD_KEY 0xFF +#define ICE_BT_INV_KEY 0xFE + +static void ice_bst_dvm_set(struct ice_parser *psr, enum ice_lbl_type type, + bool on) +{ + u16 i = 0; + + while (true) { + struct ice_bst_tcam_item *item; + u8 key; + + item = ice_bst_tcam_search(psr->bst_tcam_table, + psr->bst_lbl_table, + type, &i); + if (!item) + break; + + key = on ? ICE_BT_VLD_KEY : ICE_BT_INV_KEY; + item->key[ICE_BT_VM_OFF] = key; + item->key_inv[ICE_BT_VM_OFF] = key; + i++; + } +} + +/** + * ice_parser_dvm_set - configure double vlan mode for parser + * @psr: pointer to a parser instance + * @on: true to turn on; false to turn off + */ +void ice_parser_dvm_set(struct ice_parser *psr, bool on) +{ + ice_bst_dvm_set(psr, ICE_LBL_BST_TYPE_DVM, on); + ice_bst_dvm_set(psr, ICE_LBL_BST_TYPE_SVM, !on); +} + +static int ice_tunnel_port_set(struct ice_parser *psr, enum ice_lbl_type type, + u16 udp_port, bool on) +{ + u8 *buf = (u8 *)&udp_port; + u16 i = 0; + + while (true) { + struct ice_bst_tcam_item *item; + + item = ice_bst_tcam_search(psr->bst_tcam_table, + psr->bst_lbl_table, + type, &i); + if (!item) + break; + + /* found empty slot to add */ + if (on && item->key[ICE_BT_TUN_PORT_OFF_H] == ICE_BT_INV_KEY && + item->key_inv[ICE_BT_TUN_PORT_OFF_H] == ICE_BT_INV_KEY) { + item->key_inv[ICE_BT_TUN_PORT_OFF_L] = + buf[ICE_UDP_PORT_OFF_L]; + item->key_inv[ICE_BT_TUN_PORT_OFF_H] = + buf[ICE_UDP_PORT_OFF_H]; + + item->key[ICE_BT_TUN_PORT_OFF_L] = + ICE_BT_VLD_KEY - buf[ICE_UDP_PORT_OFF_L]; + item->key[ICE_BT_TUN_PORT_OFF_H] = + ICE_BT_VLD_KEY - buf[ICE_UDP_PORT_OFF_H]; + + return 0; + /* found a matched slot to delete */ + } else if (!on && + (item->key_inv[ICE_BT_TUN_PORT_OFF_L] == + buf[ICE_UDP_PORT_OFF_L] || + item->key_inv[ICE_BT_TUN_PORT_OFF_H] == + buf[ICE_UDP_PORT_OFF_H])) { + item->key_inv[ICE_BT_TUN_PORT_OFF_L] = ICE_BT_VLD_KEY; + item->key_inv[ICE_BT_TUN_PORT_OFF_H] = ICE_BT_INV_KEY; + + item->key[ICE_BT_TUN_PORT_OFF_L] = ICE_BT_VLD_KEY; + item->key[ICE_BT_TUN_PORT_OFF_H] = ICE_BT_INV_KEY; + + return 0; + } + i++; + } + + return -EINVAL; +} + +/** + * ice_parser_vxlan_tunnel_set - configure vxlan tunnel for parser + * @psr: pointer to a parser instance + * @udp_port: vxlan tunnel port in UDP header + * @on: true to turn on; false to turn off + * + * Return: 0 on success or errno on failure. + */ +int ice_parser_vxlan_tunnel_set(struct ice_parser *psr, + u16 udp_port, bool on) +{ + return ice_tunnel_port_set(psr, ICE_LBL_BST_TYPE_VXLAN, udp_port, on); +} + +/** + * ice_parser_geneve_tunnel_set - configure geneve tunnel for parser + * @psr: pointer to a parser instance + * @udp_port: geneve tunnel port in UDP header + * @on: true to turn on; false to turn off + * + * Return: 0 on success or errno on failure. + */ +int ice_parser_geneve_tunnel_set(struct ice_parser *psr, + u16 udp_port, bool on) +{ + return ice_tunnel_port_set(psr, ICE_LBL_BST_TYPE_GENEVE, udp_port, on); +} + +/** + * ice_parser_ecpri_tunnel_set - configure ecpri tunnel for parser + * @psr: pointer to a parser instance + * @udp_port: ecpri tunnel port in UDP header + * @on: true to turn on; false to turn off + * + * Return: 0 on success or errno on failure. + */ +int ice_parser_ecpri_tunnel_set(struct ice_parser *psr, + u16 udp_port, bool on) +{ + return ice_tunnel_port_set(psr, ICE_LBL_BST_TYPE_UDP_ECPRI, + udp_port, on); +} + +/** + * ice_nearest_proto_id - find nearest protocol ID + * @rslt: pointer to a parser result instance + * @offset: a min value for the protocol offset + * @proto_id: the protocol ID (output) + * @proto_off: the protocol offset (output) + * + * From the protocols in @rslt, find the nearest protocol that has offset + * larger than @offset. + * + * Return: if true, the protocol's ID and offset + */ +static bool ice_nearest_proto_id(struct ice_parser_result *rslt, u16 offset, + u8 *proto_id, u16 *proto_off) +{ + u16 dist = U16_MAX; + u8 proto = 0; + int i; + + for (i = 0; i < rslt->po_num; i++) { + if (offset < rslt->po[i].offset) + continue; + if (offset - rslt->po[i].offset < dist) { + proto = rslt->po[i].proto_id; + dist = offset - rslt->po[i].offset; + } + } + + if (dist % 2) + return false; + + *proto_id = proto; + *proto_off = dist; + + return true; +} + +/* default flag mask to cover GTP_EH_PDU, GTP_EH_PDU_LINK and TUN2 + * In future, the flag masks should learn from DDP + */ +#define ICE_KEYBUILD_FLAG_MASK_DEFAULT_SW 0x4002 +#define ICE_KEYBUILD_FLAG_MASK_DEFAULT_ACL 0x0000 +#define ICE_KEYBUILD_FLAG_MASK_DEFAULT_FD 0x6080 +#define ICE_KEYBUILD_FLAG_MASK_DEFAULT_RSS 0x6010 + +/** + * ice_parser_profile_init - initialize a FXP profile based on parser result + * @rslt: a instance of a parser result + * @pkt_buf: packet data buffer + * @msk_buf: packet mask buffer + * @buf_len: packet length + * @blk: FXP pipeline stage + * @prof: input/output parameter to save the profile + * + * Return: 0 on success or errno on failure. + */ +int ice_parser_profile_init(struct ice_parser_result *rslt, + const u8 *pkt_buf, const u8 *msk_buf, + int buf_len, enum ice_block blk, + struct ice_parser_profile *prof) +{ + u8 proto_id = U8_MAX; + u16 proto_off = 0; + u16 off; + + memset(prof, 0, sizeof(*prof)); + set_bit(rslt->ptype, prof->ptypes); + if (blk == ICE_BLK_SW) { + prof->flags = rslt->flags_sw; + prof->flags_msk = ICE_KEYBUILD_FLAG_MASK_DEFAULT_SW; + } else if (blk == ICE_BLK_ACL) { + prof->flags = rslt->flags_acl; + prof->flags_msk = ICE_KEYBUILD_FLAG_MASK_DEFAULT_ACL; + } else if (blk == ICE_BLK_FD) { + prof->flags = rslt->flags_fd; + prof->flags_msk = ICE_KEYBUILD_FLAG_MASK_DEFAULT_FD; + } else if (blk == ICE_BLK_RSS) { + prof->flags = rslt->flags_rss; + prof->flags_msk = ICE_KEYBUILD_FLAG_MASK_DEFAULT_RSS; + } else { + return -EINVAL; + } + + for (off = 0; off < buf_len - 1; off++) { + if (msk_buf[off] == 0 && msk_buf[off + 1] == 0) + continue; + if (!ice_nearest_proto_id(rslt, off, &proto_id, &proto_off)) + continue; + if (prof->fv_num >= ICE_PARSER_FV_MAX) + return -EINVAL; + + prof->fv[prof->fv_num].proto_id = proto_id; + prof->fv[prof->fv_num].offset = proto_off; + prof->fv[prof->fv_num].spec = *(const u16 *)&pkt_buf[off]; + prof->fv[prof->fv_num].msk = *(const u16 *)&msk_buf[off]; + prof->fv_num++; + } + + return 0; +} + +/** + * ice_parser_profile_dump - dump an FXP profile info + * @hw: pointer to the hardware structure + * @prof: profile info to dump + */ +void ice_parser_profile_dump(struct ice_hw *hw, + struct ice_parser_profile *prof) +{ + struct device *dev = ice_hw_to_dev(hw); + u16 i; + + dev_info(dev, "ptypes:\n"); + for (i = 0; i < ICE_FLOW_PTYPE_MAX; i++) + if (test_bit(i, prof->ptypes)) + dev_info(dev, "\t%u\n", i); + + for (i = 0; i < prof->fv_num; i++) + dev_info(dev, "proto = %u, offset = %2u, spec = 0x%04x, mask = 0x%04x\n", + prof->fv[i].proto_id, prof->fv[i].offset, + prof->fv[i].spec, prof->fv[i].msk); + + dev_info(dev, "flags = 0x%04x\n", prof->flags); + dev_info(dev, "flags_msk = 0x%04x\n", prof->flags_msk); +} diff --git a/drivers/net/ethernet/intel/ice/ice_parser.h b/drivers/net/ethernet/intel/ice/ice_parser.h new file mode 100644 index 000000000000..6509d807627c --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_parser.h @@ -0,0 +1,540 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2024 Intel Corporation */ + +#ifndef _ICE_PARSER_H_ +#define _ICE_PARSER_H_ + +#define ICE_SEC_DATA_OFFSET 4 +#define ICE_SID_RXPARSER_IMEM_ENTRY_SIZE 48 +#define ICE_SID_RXPARSER_METADATA_INIT_ENTRY_SIZE 24 +#define ICE_SID_RXPARSER_CAM_ENTRY_SIZE 16 +#define ICE_SID_RXPARSER_PG_SPILL_ENTRY_SIZE 17 +#define ICE_SID_RXPARSER_NOMATCH_CAM_ENTRY_SIZE 12 +#define ICE_SID_RXPARSER_NOMATCH_SPILL_ENTRY_SIZE 13 +#define ICE_SID_RXPARSER_BOOST_TCAM_ENTRY_SIZE 88 +#define ICE_SID_RXPARSER_MARKER_TYPE_ENTRY_SIZE 24 +#define ICE_SID_RXPARSER_MARKER_GRP_ENTRY_SIZE 8 +#define ICE_SID_RXPARSER_PROTO_GRP_ENTRY_SIZE 24 +#define ICE_SID_RXPARSER_FLAG_REDIR_ENTRY_SIZE 1 + +#define ICE_SEC_LBL_DATA_OFFSET 2 +#define ICE_SID_LBL_ENTRY_SIZE 66 + +/*** ICE_SID_RXPARSER_IMEM section ***/ +#define ICE_IMEM_TABLE_SIZE 192 + +/* TCAM boost Master; if bit is set, and TCAM hit, TCAM output overrides iMEM + * output. + */ +struct ice_bst_main { + bool alu0; + bool alu1; + bool alu2; + bool pg; +}; + +struct ice_bst_keybuilder { + u8 prio; /* 0-3: PG precedence within ALUs (3 highest) */ + bool tsr_ctrl; /* TCAM Search Register control */ +}; + +/* Next protocol Key builder */ +struct ice_np_keybuilder { + u8 opc; + u8 start_reg0; + u8 len_reg1; +}; + +enum ice_np_keybuilder_opcode { + ICE_NPKB_OPC_EXTRACT = 0, + ICE_NPKB_OPC_BUILD = 1, + ICE_NPKB_OPC_BYPASS = 2, +}; + +/* Parse Graph Key builder */ +struct ice_pg_keybuilder { + bool flag0_ena; + bool flag1_ena; + bool flag2_ena; + bool flag3_ena; + u8 flag0_idx; + u8 flag1_idx; + u8 flag2_idx; + u8 flag3_idx; + u8 alu_reg_idx; +}; + +enum ice_alu_idx { + ICE_ALU0_IDX = 0, + ICE_ALU1_IDX = 1, + ICE_ALU2_IDX = 2, +}; + +enum ice_alu_opcode { + ICE_ALU_PARK = 0, + ICE_ALU_MOV_ADD = 1, + ICE_ALU_ADD = 2, + ICE_ALU_MOV_AND = 4, + ICE_ALU_AND = 5, + ICE_ALU_AND_IMM = 6, + ICE_ALU_MOV_OR = 7, + ICE_ALU_OR = 8, + ICE_ALU_MOV_XOR = 9, + ICE_ALU_XOR = 10, + ICE_ALU_NOP = 11, + ICE_ALU_BR = 12, + ICE_ALU_BREQ = 13, + ICE_ALU_BRNEQ = 14, + ICE_ALU_BRGT = 15, + ICE_ALU_BRLT = 16, + ICE_ALU_BRGEQ = 17, + ICE_ALU_BRLEG = 18, + ICE_ALU_SETEQ = 19, + ICE_ALU_ANDEQ = 20, + ICE_ALU_OREQ = 21, + ICE_ALU_SETNEQ = 22, + ICE_ALU_ANDNEQ = 23, + ICE_ALU_ORNEQ = 24, + ICE_ALU_SETGT = 25, + ICE_ALU_ANDGT = 26, + ICE_ALU_ORGT = 27, + ICE_ALU_SETLT = 28, + ICE_ALU_ANDLT = 29, + ICE_ALU_ORLT = 30, + ICE_ALU_MOV_SUB = 31, + ICE_ALU_SUB = 32, + ICE_ALU_INVALID = 64, +}; + +enum ice_proto_off_opcode { + ICE_PO_OFF_REMAIN = 0, + ICE_PO_OFF_HDR_ADD = 1, + ICE_PO_OFF_HDR_SUB = 2, +}; + +struct ice_alu { + enum ice_alu_opcode opc; + u8 src_start; + u8 src_len; + bool shift_xlate_sel; + u8 shift_xlate_key; + u8 src_reg_id; + u8 dst_reg_id; + bool inc0; + bool inc1; + u8 proto_offset_opc; + u8 proto_offset; + u8 branch_addr; + u16 imm; + bool dedicate_flags_ena; + u8 dst_start; + u8 dst_len; + bool flags_extr_imm; + u8 flags_start_imm; +}; + +/* Parser program code (iMEM) */ +struct ice_imem_item { + u16 idx; + struct ice_bst_main b_m; + struct ice_bst_keybuilder b_kb; + u8 pg_prio; + struct ice_np_keybuilder np_kb; + struct ice_pg_keybuilder pg_kb; + struct ice_alu alu0; + struct ice_alu alu1; + struct ice_alu alu2; +}; + +/*** ICE_SID_RXPARSER_METADATA_INIT section ***/ +#define ICE_METAINIT_TABLE_SIZE 16 + +/* Metadata Initialization item */ +struct ice_metainit_item { + u16 idx; + + u8 tsr; /* TCAM Search key Register */ + u16 ho; /* Header Offset register */ + u16 pc; /* Program Counter register */ + u16 pg_rn; /* Parse Graph Root Node */ + u8 cd; /* Control Domain ID */ + + /* General Purpose Registers */ + bool gpr_a_ctrl; + u8 gpr_a_data_mdid; + u8 gpr_a_data_start; + u8 gpr_a_data_len; + u8 gpr_a_id; + + bool gpr_b_ctrl; + u8 gpr_b_data_mdid; + u8 gpr_b_data_start; + u8 gpr_b_data_len; + u8 gpr_b_id; + + bool gpr_c_ctrl; + u8 gpr_c_data_mdid; + u8 gpr_c_data_start; + u8 gpr_c_data_len; + u8 gpr_c_id; + + bool gpr_d_ctrl; + u8 gpr_d_data_mdid; + u8 gpr_d_data_start; + u8 gpr_d_data_len; + u8 gpr_d_id; + + u64 flags; /* Initial value for all flags */ +}; + +/*** ICE_SID_RXPARSER_CAM, ICE_SID_RXPARSER_PG_SPILL, + * ICE_SID_RXPARSER_NOMATCH_CAM and ICE_SID_RXPARSER_NOMATCH_CAM + * sections ***/ +#define ICE_PG_CAM_TABLE_SIZE 2048 +#define ICE_PG_SP_CAM_TABLE_SIZE 128 +#define ICE_PG_NM_CAM_TABLE_SIZE 1024 +#define ICE_PG_NM_SP_CAM_TABLE_SIZE 64 + +struct ice_pg_cam_key { + bool valid; + struct_group_attr(val, __packed, + u16 node_id; /* Node ID of protocol in parse graph */ + bool flag0; + bool flag1; + bool flag2; + bool flag3; + u8 boost_idx; /* Boost TCAM match index */ + u16 alu_reg; + u32 next_proto; /* next Protocol value (must be last) */ + ); +}; + +struct ice_pg_nm_cam_key { + bool valid; + struct_group_attr(val, __packed, + u16 node_id; + bool flag0; + bool flag1; + bool flag2; + bool flag3; + u8 boost_idx; + u16 alu_reg; + ); +}; + +struct ice_pg_cam_action { + u16 next_node; /* Parser Node ID for the next round */ + u8 next_pc; /* next Program Counter */ + bool is_pg; /* is protocol group */ + u8 proto_id; /* protocol ID or proto group ID */ + bool is_mg; /* is marker group */ + u8 marker_id; /* marker ID or marker group ID */ + bool is_last_round; + bool ho_polarity; /* header offset polarity */ + u16 ho_inc; +}; + +/* Parse Graph item */ +struct ice_pg_cam_item { + u16 idx; + struct ice_pg_cam_key key; + struct ice_pg_cam_action action; +}; + +/* Parse Graph No Match item */ +struct ice_pg_nm_cam_item { + u16 idx; + struct ice_pg_nm_cam_key key; + struct ice_pg_cam_action action; +}; + +struct ice_pg_cam_item *ice_pg_cam_match(struct ice_pg_cam_item *table, + int size, struct ice_pg_cam_key *key); +struct ice_pg_nm_cam_item * +ice_pg_nm_cam_match(struct ice_pg_nm_cam_item *table, int size, + struct ice_pg_cam_key *key); + +/*** ICE_SID_RXPARSER_BOOST_TCAM and ICE_SID_LBL_RXPARSER_TMEM sections ***/ +#define ICE_BST_TCAM_TABLE_SIZE 256 +#define ICE_BST_TCAM_KEY_SIZE 20 +#define ICE_BST_KEY_TCAM_SIZE 19 + +/* Boost TCAM item */ +struct ice_bst_tcam_item { + u16 addr; + u8 key[ICE_BST_TCAM_KEY_SIZE]; + u8 key_inv[ICE_BST_TCAM_KEY_SIZE]; + u8 hit_idx_grp; + u8 pg_prio; + struct ice_np_keybuilder np_kb; + struct ice_pg_keybuilder pg_kb; + struct ice_alu alu0; + struct ice_alu alu1; + struct ice_alu alu2; +}; + +#define ICE_LBL_LEN 64 +#define ICE_LBL_BST_DVM "BOOST_MAC_VLAN_DVM" +#define ICE_LBL_BST_SVM "BOOST_MAC_VLAN_SVM" +#define ICE_LBL_TNL_VXLAN "TNL_VXLAN" +#define ICE_LBL_TNL_GENEVE "TNL_GENEVE" +#define ICE_LBL_TNL_UDP_ECPRI "TNL_UDP_ECPRI" + +enum ice_lbl_type { + ICE_LBL_BST_TYPE_UNKNOWN, + ICE_LBL_BST_TYPE_DVM, + ICE_LBL_BST_TYPE_SVM, + ICE_LBL_BST_TYPE_VXLAN, + ICE_LBL_BST_TYPE_GENEVE, + ICE_LBL_BST_TYPE_UDP_ECPRI, +}; + +struct ice_lbl_item { + u16 idx; + char label[ICE_LBL_LEN]; + + /* must be at the end, not part of the DDP section */ + enum ice_lbl_type type; +}; + +struct ice_bst_tcam_item * +ice_bst_tcam_match(struct ice_bst_tcam_item *tcam_table, u8 *pat); +struct ice_bst_tcam_item * +ice_bst_tcam_search(struct ice_bst_tcam_item *tcam_table, + struct ice_lbl_item *lbl_table, + enum ice_lbl_type type, u16 *start); + +/*** ICE_SID_RXPARSER_MARKER_PTYPE section ***/ +#define ICE_PTYPE_MK_TCAM_TABLE_SIZE 1024 +#define ICE_PTYPE_MK_TCAM_KEY_SIZE 10 + +struct ice_ptype_mk_tcam_item { + u16 address; + u16 ptype; + u8 key[ICE_PTYPE_MK_TCAM_KEY_SIZE]; + u8 key_inv[ICE_PTYPE_MK_TCAM_KEY_SIZE]; +} __packed; + +struct ice_ptype_mk_tcam_item * +ice_ptype_mk_tcam_match(struct ice_ptype_mk_tcam_item *table, + u8 *pat, int len); +/*** ICE_SID_RXPARSER_MARKER_GRP section ***/ +#define ICE_MK_GRP_TABLE_SIZE 128 +#define ICE_MK_COUNT_PER_GRP 8 + +/* Marker Group item */ +struct ice_mk_grp_item { + int idx; + u8 markers[ICE_MK_COUNT_PER_GRP]; +}; + +/*** ICE_SID_RXPARSER_PROTO_GRP section ***/ +#define ICE_PROTO_COUNT_PER_GRP 8 +#define ICE_PROTO_GRP_TABLE_SIZE 192 +#define ICE_PROTO_GRP_ITEM_SIZE 22 +struct ice_proto_off { + bool polarity; /* true: positive, false: negative */ + u8 proto_id; + u16 offset; /* 10 bit protocol offset */ +}; + +/* Protocol Group item */ +struct ice_proto_grp_item { + u16 idx; + struct ice_proto_off po[ICE_PROTO_COUNT_PER_GRP]; +}; + +/*** ICE_SID_RXPARSER_FLAG_REDIR section ***/ +#define ICE_FLG_RD_TABLE_SIZE 64 +#define ICE_FLG_RDT_SIZE 64 + +/* Flags Redirection item */ +struct ice_flg_rd_item { + u16 idx; + bool expose; + u8 intr_flg_id; /* Internal Flag ID */ +}; + +u64 ice_flg_redirect(struct ice_flg_rd_item *table, u64 psr_flg); + +/*** ICE_SID_XLT_KEY_BUILDER_SW, ICE_SID_XLT_KEY_BUILDER_ACL, + * ICE_SID_XLT_KEY_BUILDER_FD and ICE_SID_XLT_KEY_BUILDER_RSS + * sections ***/ +#define ICE_XLT_KB_FLAG0_14_CNT 15 +#define ICE_XLT_KB_TBL_CNT 8 +#define ICE_XLT_KB_TBL_ENTRY_SIZE 24 + +struct ice_xlt_kb_entry { + u8 xlt1_ad_sel; + u8 xlt2_ad_sel; + u16 flg0_14_sel[ICE_XLT_KB_FLAG0_14_CNT]; + u8 xlt1_md_sel; + u8 xlt2_md_sel; +}; + +/* XLT Key Builder */ +struct ice_xlt_kb { + u8 xlt1_pm; /* XLT1 Partition Mode */ + u8 xlt2_pm; /* XLT2 Partition Mode */ + u8 prof_id_pm; /* Profile ID Partition Mode */ + u64 flag15; + + struct ice_xlt_kb_entry entries[ICE_XLT_KB_TBL_CNT]; +}; + +u16 ice_xlt_kb_flag_get(struct ice_xlt_kb *kb, u64 pkt_flag); + +/*** Parser API ***/ +#define ICE_GPR_HV_IDX 64 +#define ICE_GPR_HV_SIZE 32 +#define ICE_GPR_ERR_IDX 84 +#define ICE_GPR_FLG_IDX 104 +#define ICE_GPR_FLG_SIZE 16 + +#define ICE_GPR_TSR_IDX 108 /* TSR: TCAM Search Register */ +#define ICE_GPR_NN_IDX 109 /* NN: Next Parsing Cycle Node ID */ +#define ICE_GPR_HO_IDX 110 /* HO: Next Parsing Cycle hdr Offset */ +#define ICE_GPR_NP_IDX 111 /* NP: Next Parsing Cycle */ + +#define ICE_PARSER_MAX_PKT_LEN 504 +#define ICE_PARSER_PKT_REV 32 +#define ICE_PARSER_GPR_NUM 128 +#define ICE_PARSER_FLG_NUM 64 +#define ICE_PARSER_ERR_NUM 16 +#define ICE_BST_KEY_SIZE 10 +#define ICE_MARKER_ID_SIZE 9 +#define ICE_MARKER_MAX_SIZE \ + (ICE_MARKER_ID_SIZE * BITS_PER_BYTE - 1) +#define ICE_MARKER_ID_NUM 8 +#define ICE_PO_PAIR_SIZE 256 + +struct ice_gpr_pu { + /* array of flags to indicate if GRP needs to be updated */ + bool gpr_val_upd[ICE_PARSER_GPR_NUM]; + u16 gpr_val[ICE_PARSER_GPR_NUM]; + u64 flg_msk; + u64 flg_val; + u16 err_msk; + u16 err_val; +}; + +enum ice_pg_prio { + ICE_PG_P0 = 0, + ICE_PG_P1 = 1, + ICE_PG_P2 = 2, + ICE_PG_P3 = 3, +}; + +struct ice_parser_rt { + struct ice_parser *psr; + u16 gpr[ICE_PARSER_GPR_NUM]; + u8 pkt_buf[ICE_PARSER_MAX_PKT_LEN + ICE_PARSER_PKT_REV]; + u16 pkt_len; + u16 po; + u8 bst_key[ICE_BST_KEY_SIZE]; + struct ice_pg_cam_key pg_key; + struct ice_alu *alu0; + struct ice_alu *alu1; + struct ice_alu *alu2; + struct ice_pg_cam_action *action; + u8 pg_prio; + struct ice_gpr_pu pu; + u8 markers[ICE_MARKER_ID_SIZE]; + bool protocols[ICE_PO_PAIR_SIZE]; + u16 offsets[ICE_PO_PAIR_SIZE]; +}; + +struct ice_parser_proto_off { + u8 proto_id; /* hardware protocol ID */ + u16 offset; /* offset from the start of the protocol header */ +}; + +#define ICE_PARSER_PROTO_OFF_PAIR_SIZE 16 +#define ICE_PARSER_FLAG_PSR_SIZE 8 +#define ICE_PARSER_FV_SIZE 48 +#define ICE_PARSER_FV_MAX 24 +#define ICE_BT_TUN_PORT_OFF_H 16 +#define ICE_BT_TUN_PORT_OFF_L 15 +#define ICE_BT_VM_OFF 0 +#define ICE_UDP_PORT_OFF_H 1 +#define ICE_UDP_PORT_OFF_L 0 + +struct ice_parser_result { + u16 ptype; /* 16 bits hardware PTYPE */ + /* array of protocol and header offset pairs */ + struct ice_parser_proto_off po[ICE_PARSER_PROTO_OFF_PAIR_SIZE]; + int po_num; /* # of protocol-offset pairs must <= 16 */ + u64 flags_psr; /* parser flags */ + u64 flags_pkt; /* packet flags */ + u16 flags_sw; /* key builder flags for SW */ + u16 flags_acl; /* key builder flags for ACL */ + u16 flags_fd; /* key builder flags for FD */ + u16 flags_rss; /* key builder flags for RSS */ +}; + +void ice_parser_rt_reset(struct ice_parser_rt *rt); +void ice_parser_rt_pktbuf_set(struct ice_parser_rt *rt, const u8 *pkt_buf, + int pkt_len); +int ice_parser_rt_execute(struct ice_parser_rt *rt, + struct ice_parser_result *rslt); + +struct ice_parser { + struct ice_hw *hw; /* pointer to the hardware structure */ + + struct ice_imem_item *imem_table; + struct ice_metainit_item *mi_table; + + struct ice_pg_cam_item *pg_cam_table; + struct ice_pg_cam_item *pg_sp_cam_table; + struct ice_pg_nm_cam_item *pg_nm_cam_table; + struct ice_pg_nm_cam_item *pg_nm_sp_cam_table; + + struct ice_bst_tcam_item *bst_tcam_table; + struct ice_lbl_item *bst_lbl_table; + struct ice_ptype_mk_tcam_item *ptype_mk_tcam_table; + struct ice_mk_grp_item *mk_grp_table; + struct ice_proto_grp_item *proto_grp_table; + struct ice_flg_rd_item *flg_rd_table; + + struct ice_xlt_kb *xlt_kb_sw; + struct ice_xlt_kb *xlt_kb_acl; + struct ice_xlt_kb *xlt_kb_fd; + struct ice_xlt_kb *xlt_kb_rss; + + struct ice_parser_rt rt; +}; + +struct ice_parser *ice_parser_create(struct ice_hw *hw); +void ice_parser_destroy(struct ice_parser *psr); +void ice_parser_dvm_set(struct ice_parser *psr, bool on); +int ice_parser_vxlan_tunnel_set(struct ice_parser *psr, u16 udp_port, bool on); +int ice_parser_geneve_tunnel_set(struct ice_parser *psr, u16 udp_port, bool on); +int ice_parser_ecpri_tunnel_set(struct ice_parser *psr, u16 udp_port, bool on); +int ice_parser_run(struct ice_parser *psr, const u8 *pkt_buf, + int pkt_len, struct ice_parser_result *rslt); +void ice_parser_result_dump(struct ice_hw *hw, struct ice_parser_result *rslt); + +struct ice_parser_fv { + u8 proto_id; /* hardware protocol ID */ + u16 offset; /* offset from the start of the protocol header */ + u16 spec; /* pattern to match */ + u16 msk; /* pattern mask */ +}; + +struct ice_parser_profile { + /* array of field vectors */ + struct ice_parser_fv fv[ICE_PARSER_FV_SIZE]; + int fv_num; /* # of field vectors must <= 48 */ + u16 flags; /* key builder flags */ + u16 flags_msk; /* key builder flag mask */ + + DECLARE_BITMAP(ptypes, ICE_FLOW_PTYPE_MAX); /* PTYPE bitmap */ +}; + +int ice_parser_profile_init(struct ice_parser_result *rslt, + const u8 *pkt_buf, const u8 *msk_buf, + int buf_len, enum ice_block blk, + struct ice_parser_profile *prof); +void ice_parser_profile_dump(struct ice_hw *hw, + struct ice_parser_profile *prof); +#endif /* _ICE_PARSER_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_parser_rt.c b/drivers/net/ethernet/intel/ice/ice_parser_rt.c new file mode 100644 index 000000000000..d5bcc266b01e --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_parser_rt.c @@ -0,0 +1,861 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2024 Intel Corporation */ + +#include "ice_common.h" + +static void ice_rt_tsr_set(struct ice_parser_rt *rt, u16 tsr) +{ + rt->gpr[ICE_GPR_TSR_IDX] = tsr; +} + +static void ice_rt_ho_set(struct ice_parser_rt *rt, u16 ho) +{ + rt->gpr[ICE_GPR_HO_IDX] = ho; + memcpy(&rt->gpr[ICE_GPR_HV_IDX], &rt->pkt_buf[ho], ICE_GPR_HV_SIZE); +} + +static void ice_rt_np_set(struct ice_parser_rt *rt, u16 pc) +{ + rt->gpr[ICE_GPR_NP_IDX] = pc; +} + +static void ice_rt_nn_set(struct ice_parser_rt *rt, u16 node) +{ + rt->gpr[ICE_GPR_NN_IDX] = node; +} + +static void +ice_rt_flag_set(struct ice_parser_rt *rt, unsigned int idx, bool set) +{ + struct ice_hw *hw = rt->psr->hw; + unsigned int word, id; + + word = idx / ICE_GPR_FLG_SIZE; + id = idx % ICE_GPR_FLG_SIZE; + + if (set) { + rt->gpr[ICE_GPR_FLG_IDX + word] |= (u16)BIT(id); + ice_debug(hw, ICE_DBG_PARSER, "Set parser flag %u\n", idx); + } else { + rt->gpr[ICE_GPR_FLG_IDX + word] &= ~(u16)BIT(id); + ice_debug(hw, ICE_DBG_PARSER, "Clear parser flag %u\n", idx); + } +} + +static void ice_rt_gpr_set(struct ice_parser_rt *rt, int idx, u16 val) +{ + struct ice_hw *hw = rt->psr->hw; + + if (idx == ICE_GPR_HO_IDX) + ice_rt_ho_set(rt, val); + else + rt->gpr[idx] = val; + + ice_debug(hw, ICE_DBG_PARSER, "Set GPR %d value %d\n", idx, val); +} + +static void ice_rt_err_set(struct ice_parser_rt *rt, unsigned int idx, bool set) +{ + struct ice_hw *hw = rt->psr->hw; + + if (set) { + rt->gpr[ICE_GPR_ERR_IDX] |= (u16)BIT(idx); + ice_debug(hw, ICE_DBG_PARSER, "Set parser error %u\n", idx); + } else { + rt->gpr[ICE_GPR_ERR_IDX] &= ~(u16)BIT(idx); + ice_debug(hw, ICE_DBG_PARSER, "Reset parser error %u\n", idx); + } +} + +/** + * ice_parser_rt_reset - reset the parser runtime + * @rt: pointer to the parser runtime + */ +void ice_parser_rt_reset(struct ice_parser_rt *rt) +{ + struct ice_parser *psr = rt->psr; + struct ice_metainit_item *mi; + unsigned int i; + + mi = &psr->mi_table[0]; + + memset(rt, 0, sizeof(*rt)); + rt->psr = psr; + + ice_rt_tsr_set(rt, mi->tsr); + ice_rt_ho_set(rt, mi->ho); + ice_rt_np_set(rt, mi->pc); + ice_rt_nn_set(rt, mi->pg_rn); + + for (i = 0; i < ICE_PARSER_FLG_NUM; i++) { + if (mi->flags & BIT(i)) + ice_rt_flag_set(rt, i, true); + } +} + +/** + * ice_parser_rt_pktbuf_set - set a packet into parser runtime + * @rt: pointer to the parser runtime + * @pkt_buf: buffer with packet data + * @pkt_len: packet buffer length + */ +void ice_parser_rt_pktbuf_set(struct ice_parser_rt *rt, const u8 *pkt_buf, + int pkt_len) +{ + int len = min(ICE_PARSER_MAX_PKT_LEN, pkt_len); + u16 ho = rt->gpr[ICE_GPR_HO_IDX]; + + memcpy(rt->pkt_buf, pkt_buf, len); + rt->pkt_len = pkt_len; + + memcpy(&rt->gpr[ICE_GPR_HV_IDX], &rt->pkt_buf[ho], ICE_GPR_HV_SIZE); +} + +static void ice_bst_key_init(struct ice_parser_rt *rt, + struct ice_imem_item *imem) +{ + u8 tsr = (u8)rt->gpr[ICE_GPR_TSR_IDX]; + u16 ho = rt->gpr[ICE_GPR_HO_IDX]; + u8 *key = rt->bst_key; + int idd, i; + + idd = ICE_BST_TCAM_KEY_SIZE - 1; + if (imem->b_kb.tsr_ctrl) + key[idd] = tsr; + else + key[idd] = imem->b_kb.prio; + + idd = ICE_BST_KEY_TCAM_SIZE - 1; + for (i = idd; i >= 0; i--) { + int j; + + j = ho + idd - i; + if (j < ICE_PARSER_MAX_PKT_LEN) + key[i] = rt->pkt_buf[ho + idd - i]; + else + key[i] = 0; + } + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Generated Boost TCAM Key:\n"); + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "%02X %02X %02X %02X %02X %02X %02X %02X %02X %02X\n", + key[0], key[1], key[2], key[3], key[4], + key[5], key[6], key[7], key[8], key[9]); + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "\n"); +} + +static u16 ice_bit_rev_u16(u16 v, int len) +{ + return bitrev16(v) >> (BITS_PER_TYPE(v) - len); +} + +static u32 ice_bit_rev_u32(u32 v, int len) +{ + return bitrev32(v) >> (BITS_PER_TYPE(v) - len); +} + +static u32 ice_hv_bit_sel(struct ice_parser_rt *rt, int start, int len) +{ + int offset; + u32 buf[2]; + u64 val; + + offset = ICE_GPR_HV_IDX + (start / BITS_PER_TYPE(u16)); + + memcpy(buf, &rt->gpr[offset], sizeof(buf)); + + buf[0] = bitrev8x4(buf[0]); + buf[1] = bitrev8x4(buf[1]); + + val = *(u64 *)buf; + val >>= start % BITS_PER_TYPE(u16); + + return ice_bit_rev_u32(val, len); +} + +static u32 ice_pk_build(struct ice_parser_rt *rt, + struct ice_np_keybuilder *kb) +{ + if (kb->opc == ICE_NPKB_OPC_EXTRACT) + return ice_hv_bit_sel(rt, kb->start_reg0, kb->len_reg1); + else if (kb->opc == ICE_NPKB_OPC_BUILD) + return rt->gpr[kb->start_reg0] | + ((u32)rt->gpr[kb->len_reg1] << BITS_PER_TYPE(u16)); + else if (kb->opc == ICE_NPKB_OPC_BYPASS) + return 0; + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Unsupported OP Code %u\n", + kb->opc); + return U32_MAX; +} + +static bool ice_flag_get(struct ice_parser_rt *rt, unsigned int index) +{ + int word = index / ICE_GPR_FLG_SIZE; + int id = index % ICE_GPR_FLG_SIZE; + + return !!(rt->gpr[ICE_GPR_FLG_IDX + word] & (u16)BIT(id)); +} + +static int ice_imem_pgk_init(struct ice_parser_rt *rt, + struct ice_imem_item *imem) +{ + memset(&rt->pg_key, 0, sizeof(rt->pg_key)); + rt->pg_key.next_proto = ice_pk_build(rt, &imem->np_kb); + if (rt->pg_key.next_proto == U32_MAX) + return -EINVAL; + + if (imem->pg_kb.flag0_ena) + rt->pg_key.flag0 = ice_flag_get(rt, imem->pg_kb.flag0_idx); + if (imem->pg_kb.flag1_ena) + rt->pg_key.flag1 = ice_flag_get(rt, imem->pg_kb.flag1_idx); + if (imem->pg_kb.flag2_ena) + rt->pg_key.flag2 = ice_flag_get(rt, imem->pg_kb.flag2_idx); + if (imem->pg_kb.flag3_ena) + rt->pg_key.flag3 = ice_flag_get(rt, imem->pg_kb.flag3_idx); + + rt->pg_key.alu_reg = rt->gpr[imem->pg_kb.alu_reg_idx]; + rt->pg_key.node_id = rt->gpr[ICE_GPR_NN_IDX]; + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Generate Parse Graph Key: node_id(%d), flag0-3(%d,%d,%d,%d), boost_idx(%d), alu_reg(0x%04x), next_proto(0x%08x)\n", + rt->pg_key.node_id, + rt->pg_key.flag0, + rt->pg_key.flag1, + rt->pg_key.flag2, + rt->pg_key.flag3, + rt->pg_key.boost_idx, + rt->pg_key.alu_reg, + rt->pg_key.next_proto); + + return 0; +} + +static void ice_imem_alu0_set(struct ice_parser_rt *rt, + struct ice_imem_item *imem) +{ + rt->alu0 = &imem->alu0; + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Load ALU0 from imem pc %d\n", + imem->idx); +} + +static void ice_imem_alu1_set(struct ice_parser_rt *rt, + struct ice_imem_item *imem) +{ + rt->alu1 = &imem->alu1; + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Load ALU1 from imem pc %d\n", + imem->idx); +} + +static void ice_imem_alu2_set(struct ice_parser_rt *rt, + struct ice_imem_item *imem) +{ + rt->alu2 = &imem->alu2; + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Load ALU2 from imem pc %d\n", + imem->idx); +} + +static void ice_imem_pgp_set(struct ice_parser_rt *rt, + struct ice_imem_item *imem) +{ + rt->pg_prio = imem->pg_prio; + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Load PG priority %d from imem pc %d\n", + rt->pg_prio, imem->idx); +} + +static int ice_bst_pgk_init(struct ice_parser_rt *rt, + struct ice_bst_tcam_item *bst) +{ + memset(&rt->pg_key, 0, sizeof(rt->pg_key)); + rt->pg_key.boost_idx = bst->hit_idx_grp; + rt->pg_key.next_proto = ice_pk_build(rt, &bst->np_kb); + if (rt->pg_key.next_proto == U32_MAX) + return -EINVAL; + + if (bst->pg_kb.flag0_ena) + rt->pg_key.flag0 = ice_flag_get(rt, bst->pg_kb.flag0_idx); + if (bst->pg_kb.flag1_ena) + rt->pg_key.flag1 = ice_flag_get(rt, bst->pg_kb.flag1_idx); + if (bst->pg_kb.flag2_ena) + rt->pg_key.flag2 = ice_flag_get(rt, bst->pg_kb.flag2_idx); + if (bst->pg_kb.flag3_ena) + rt->pg_key.flag3 = ice_flag_get(rt, bst->pg_kb.flag3_idx); + + rt->pg_key.alu_reg = rt->gpr[bst->pg_kb.alu_reg_idx]; + rt->pg_key.node_id = rt->gpr[ICE_GPR_NN_IDX]; + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Generate Parse Graph Key: node_id(%d), flag0-3(%d,%d,%d,%d), boost_idx(%d), alu_reg(0x%04x), next_proto(0x%08x)\n", + rt->pg_key.node_id, + rt->pg_key.flag0, + rt->pg_key.flag1, + rt->pg_key.flag2, + rt->pg_key.flag3, + rt->pg_key.boost_idx, + rt->pg_key.alu_reg, + rt->pg_key.next_proto); + + return 0; +} + +static void ice_bst_alu0_set(struct ice_parser_rt *rt, + struct ice_bst_tcam_item *bst) +{ + rt->alu0 = &bst->alu0; + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Load ALU0 from boost address %d\n", + bst->addr); +} + +static void ice_bst_alu1_set(struct ice_parser_rt *rt, + struct ice_bst_tcam_item *bst) +{ + rt->alu1 = &bst->alu1; + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Load ALU1 from boost address %d\n", + bst->addr); +} + +static void ice_bst_alu2_set(struct ice_parser_rt *rt, + struct ice_bst_tcam_item *bst) +{ + rt->alu2 = &bst->alu2; + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Load ALU2 from boost address %d\n", + bst->addr); +} + +static void ice_bst_pgp_set(struct ice_parser_rt *rt, + struct ice_bst_tcam_item *bst) +{ + rt->pg_prio = bst->pg_prio; + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Load PG priority %d from boost address %d\n", + rt->pg_prio, bst->addr); +} + +static struct ice_pg_cam_item *ice_rt_pg_cam_match(struct ice_parser_rt *rt) +{ + struct ice_parser *psr = rt->psr; + struct ice_pg_cam_item *item; + + item = ice_pg_cam_match(psr->pg_cam_table, ICE_PG_CAM_TABLE_SIZE, + &rt->pg_key); + if (!item) + item = ice_pg_cam_match(psr->pg_sp_cam_table, + ICE_PG_SP_CAM_TABLE_SIZE, &rt->pg_key); + return item; +} + +static +struct ice_pg_nm_cam_item *ice_rt_pg_nm_cam_match(struct ice_parser_rt *rt) +{ + struct ice_parser *psr = rt->psr; + struct ice_pg_nm_cam_item *item; + + item = ice_pg_nm_cam_match(psr->pg_nm_cam_table, + ICE_PG_NM_CAM_TABLE_SIZE, &rt->pg_key); + + if (!item) + item = ice_pg_nm_cam_match(psr->pg_nm_sp_cam_table, + ICE_PG_NM_SP_CAM_TABLE_SIZE, + &rt->pg_key); + return item; +} + +static void ice_gpr_add(struct ice_parser_rt *rt, int idx, u16 val) +{ + rt->pu.gpr_val_upd[idx] = true; + rt->pu.gpr_val[idx] = val; + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Pending update for register %d value %d\n", + idx, val); +} + +static void ice_pg_exe(struct ice_parser_rt *rt) +{ + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Executing ParseGraph action ...\n"); + + ice_gpr_add(rt, ICE_GPR_NP_IDX, rt->action->next_pc); + ice_gpr_add(rt, ICE_GPR_NN_IDX, rt->action->next_node); + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Executing ParseGraph action done.\n"); +} + +static void ice_flg_add(struct ice_parser_rt *rt, int idx, bool val) +{ + rt->pu.flg_msk |= BIT(idx); + if (val) + rt->pu.flg_val |= BIT(idx); + else + rt->pu.flg_val &= ~BIT(idx); + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Pending update for flag %d value %d\n", + idx, val); +} + +static void ice_flg_update(struct ice_parser_rt *rt, struct ice_alu *alu) +{ + u32 hv_bit_sel; + int i; + + if (!alu->dedicate_flags_ena) + return; + + if (alu->flags_extr_imm) { + for (i = 0; i < alu->dst_len; i++) + ice_flg_add(rt, alu->dst_start + i, + !!(alu->flags_start_imm & BIT(i))); + } else { + for (i = 0; i < alu->dst_len; i++) { + hv_bit_sel = ice_hv_bit_sel(rt, + alu->flags_start_imm + i, + 1); + ice_flg_add(rt, alu->dst_start + i, !!hv_bit_sel); + } + } +} + +static void ice_po_update(struct ice_parser_rt *rt, struct ice_alu *alu) +{ + if (alu->proto_offset_opc == ICE_PO_OFF_HDR_ADD) + rt->po = (u16)(rt->gpr[ICE_GPR_HO_IDX] + alu->proto_offset); + else if (alu->proto_offset_opc == ICE_PO_OFF_HDR_SUB) + rt->po = (u16)(rt->gpr[ICE_GPR_HO_IDX] - alu->proto_offset); + else if (alu->proto_offset_opc == ICE_PO_OFF_REMAIN) + rt->po = rt->gpr[ICE_GPR_HO_IDX]; + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Update Protocol Offset = %d\n", + rt->po); +} + +static u16 ice_reg_bit_sel(struct ice_parser_rt *rt, int reg_idx, + int start, int len) +{ + int offset; + u32 val; + + offset = ICE_GPR_HV_IDX + (start / BITS_PER_TYPE(u16)); + + memcpy(&val, &rt->gpr[offset], sizeof(val)); + + val = bitrev8x4(val); + val >>= start % BITS_PER_TYPE(u16); + + return ice_bit_rev_u16(val, len); +} + +static void ice_err_add(struct ice_parser_rt *rt, int idx, bool val) +{ + rt->pu.err_msk |= (u16)BIT(idx); + if (val) + rt->pu.flg_val |= (u64)BIT_ULL(idx); + else + rt->pu.flg_val &= ~(u64)BIT_ULL(idx); + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Pending update for error %d value %d\n", + idx, val); +} + +static void ice_dst_reg_bit_set(struct ice_parser_rt *rt, struct ice_alu *alu, + bool val) +{ + u16 flg_idx; + + if (alu->dedicate_flags_ena) { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "DedicatedFlagsEnable should not be enabled in opcode %d\n", + alu->opc); + return; + } + + if (alu->dst_reg_id == ICE_GPR_ERR_IDX) { + if (alu->dst_start >= ICE_PARSER_ERR_NUM) { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Invalid error %d\n", + alu->dst_start); + return; + } + ice_err_add(rt, alu->dst_start, val); + } else if (alu->dst_reg_id >= ICE_GPR_FLG_IDX) { + flg_idx = (u16)(((alu->dst_reg_id - ICE_GPR_FLG_IDX) << 4) + + alu->dst_start); + + if (flg_idx >= ICE_PARSER_FLG_NUM) { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Invalid flag %d\n", + flg_idx); + return; + } + ice_flg_add(rt, flg_idx, val); + } else { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Unexpected Dest Register Bit set, RegisterID %d Start %d\n", + alu->dst_reg_id, alu->dst_start); + } +} + +static void ice_alu_exe(struct ice_parser_rt *rt, struct ice_alu *alu) +{ + u16 dst, src, shift, imm; + + if (alu->shift_xlate_sel) { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "shift_xlate_sel != 0 is not expected\n"); + return; + } + + ice_po_update(rt, alu); + ice_flg_update(rt, alu); + + dst = rt->gpr[alu->dst_reg_id]; + src = ice_reg_bit_sel(rt, alu->src_reg_id, + alu->src_start, alu->src_len); + shift = alu->shift_xlate_key; + imm = alu->imm; + + switch (alu->opc) { + case ICE_ALU_PARK: + break; + case ICE_ALU_MOV_ADD: + dst = (src << shift) + imm; + ice_gpr_add(rt, alu->dst_reg_id, dst); + break; + case ICE_ALU_ADD: + dst += (src << shift) + imm; + ice_gpr_add(rt, alu->dst_reg_id, dst); + break; + case ICE_ALU_ORLT: + if (src < imm) + ice_dst_reg_bit_set(rt, alu, true); + ice_gpr_add(rt, ICE_GPR_NP_IDX, alu->branch_addr); + break; + case ICE_ALU_OREQ: + if (src == imm) + ice_dst_reg_bit_set(rt, alu, true); + ice_gpr_add(rt, ICE_GPR_NP_IDX, alu->branch_addr); + break; + case ICE_ALU_SETEQ: + ice_dst_reg_bit_set(rt, alu, src == imm); + ice_gpr_add(rt, ICE_GPR_NP_IDX, alu->branch_addr); + break; + case ICE_ALU_MOV_XOR: + dst = (src << shift) ^ imm; + ice_gpr_add(rt, alu->dst_reg_id, dst); + break; + default: + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Unsupported ALU instruction %d\n", + alu->opc); + break; + } +} + +static void ice_alu0_exe(struct ice_parser_rt *rt) +{ + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Executing ALU0 ...\n"); + ice_alu_exe(rt, rt->alu0); + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Executing ALU0 done.\n"); +} + +static void ice_alu1_exe(struct ice_parser_rt *rt) +{ + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Executing ALU1 ...\n"); + ice_alu_exe(rt, rt->alu1); + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Executing ALU1 done.\n"); +} + +static void ice_alu2_exe(struct ice_parser_rt *rt) +{ + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Executing ALU2 ...\n"); + ice_alu_exe(rt, rt->alu2); + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Executing ALU2 done.\n"); +} + +static void ice_pu_exe(struct ice_parser_rt *rt) +{ + struct ice_gpr_pu *pu = &rt->pu; + unsigned int i; + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Updating Registers ...\n"); + + for (i = 0; i < ICE_PARSER_GPR_NUM; i++) { + if (pu->gpr_val_upd[i]) + ice_rt_gpr_set(rt, i, pu->gpr_val[i]); + } + + for (i = 0; i < ICE_PARSER_FLG_NUM; i++) { + if (pu->flg_msk & BIT(i)) + ice_rt_flag_set(rt, i, pu->flg_val & BIT(i)); + } + + for (i = 0; i < ICE_PARSER_ERR_NUM; i++) { + if (pu->err_msk & BIT(i)) + ice_rt_err_set(rt, i, pu->err_val & BIT(i)); + } + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Updating Registers done.\n"); +} + +static void ice_alu_pg_exe(struct ice_parser_rt *rt) +{ + memset(&rt->pu, 0, sizeof(rt->pu)); + + switch (rt->pg_prio) { + case (ICE_PG_P0): + ice_pg_exe(rt); + ice_alu0_exe(rt); + ice_alu1_exe(rt); + ice_alu2_exe(rt); + break; + case (ICE_PG_P1): + ice_alu0_exe(rt); + ice_pg_exe(rt); + ice_alu1_exe(rt); + ice_alu2_exe(rt); + break; + case (ICE_PG_P2): + ice_alu0_exe(rt); + ice_alu1_exe(rt); + ice_pg_exe(rt); + ice_alu2_exe(rt); + break; + case (ICE_PG_P3): + ice_alu0_exe(rt); + ice_alu1_exe(rt); + ice_alu2_exe(rt); + ice_pg_exe(rt); + break; + } + + ice_pu_exe(rt); + + if (rt->action->ho_inc == 0) + return; + + if (rt->action->ho_polarity) + ice_rt_ho_set(rt, rt->gpr[ICE_GPR_HO_IDX] + rt->action->ho_inc); + else + ice_rt_ho_set(rt, rt->gpr[ICE_GPR_HO_IDX] - rt->action->ho_inc); +} + +static void ice_proto_off_update(struct ice_parser_rt *rt) +{ + struct ice_parser *psr = rt->psr; + + if (rt->action->is_pg) { + struct ice_proto_grp_item *proto_grp = + &psr->proto_grp_table[rt->action->proto_id]; + u16 po; + int i; + + for (i = 0; i < ICE_PROTO_COUNT_PER_GRP; i++) { + struct ice_proto_off *entry = &proto_grp->po[i]; + + if (entry->proto_id == U8_MAX) + break; + + if (!entry->polarity) + po = rt->po + entry->offset; + else + po = rt->po - entry->offset; + + rt->protocols[entry->proto_id] = true; + rt->offsets[entry->proto_id] = po; + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Set Protocol %d at offset %d\n", + entry->proto_id, po); + } + } else { + rt->protocols[rt->action->proto_id] = true; + rt->offsets[rt->action->proto_id] = rt->po; + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Set Protocol %d at offset %d\n", + rt->action->proto_id, rt->po); + } +} + +static void ice_marker_set(struct ice_parser_rt *rt, int idx) +{ + unsigned int byte = idx / BITS_PER_BYTE; + unsigned int bit = idx % BITS_PER_BYTE; + + rt->markers[byte] |= (u8)BIT(bit); +} + +static void ice_marker_update(struct ice_parser_rt *rt) +{ + struct ice_parser *psr = rt->psr; + + if (rt->action->is_mg) { + struct ice_mk_grp_item *mk_grp = + &psr->mk_grp_table[rt->action->marker_id]; + int i; + + for (i = 0; i < ICE_MARKER_ID_NUM; i++) { + u8 marker = mk_grp->markers[i]; + + if (marker == ICE_MARKER_MAX_SIZE) + break; + + ice_marker_set(rt, marker); + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Set Marker %d\n", + marker); + } + } else { + if (rt->action->marker_id != ICE_MARKER_MAX_SIZE) + ice_marker_set(rt, rt->action->marker_id); + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Set Marker %d\n", + rt->action->marker_id); + } +} + +static u16 ice_ptype_resolve(struct ice_parser_rt *rt) +{ + struct ice_ptype_mk_tcam_item *item; + struct ice_parser *psr = rt->psr; + + item = ice_ptype_mk_tcam_match(psr->ptype_mk_tcam_table, + rt->markers, ICE_MARKER_ID_SIZE); + if (item) + return item->ptype; + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Could not resolve PTYPE\n"); + return U16_MAX; +} + +static void ice_proto_off_resolve(struct ice_parser_rt *rt, + struct ice_parser_result *rslt) +{ + int i; + + for (i = 0; i < ICE_PO_PAIR_SIZE - 1; i++) { + if (rt->protocols[i]) { + rslt->po[rslt->po_num].proto_id = (u8)i; + rslt->po[rslt->po_num].offset = rt->offsets[i]; + rslt->po_num++; + } + } +} + +static void ice_result_resolve(struct ice_parser_rt *rt, + struct ice_parser_result *rslt) +{ + struct ice_parser *psr = rt->psr; + + memset(rslt, 0, sizeof(*rslt)); + + memcpy(&rslt->flags_psr, &rt->gpr[ICE_GPR_FLG_IDX], + ICE_PARSER_FLAG_PSR_SIZE); + rslt->flags_pkt = ice_flg_redirect(psr->flg_rd_table, rslt->flags_psr); + rslt->flags_sw = ice_xlt_kb_flag_get(psr->xlt_kb_sw, rslt->flags_pkt); + rslt->flags_fd = ice_xlt_kb_flag_get(psr->xlt_kb_fd, rslt->flags_pkt); + rslt->flags_rss = ice_xlt_kb_flag_get(psr->xlt_kb_rss, rslt->flags_pkt); + + ice_proto_off_resolve(rt, rslt); + rslt->ptype = ice_ptype_resolve(rt); +} + +/** + * ice_parser_rt_execute - parser execution routine + * @rt: pointer to the parser runtime + * @rslt: input/output parameter to save parser result + * + * Return: 0 on success or errno. + */ +int ice_parser_rt_execute(struct ice_parser_rt *rt, + struct ice_parser_result *rslt) +{ + struct ice_pg_nm_cam_item *pg_nm_cam; + struct ice_parser *psr = rt->psr; + struct ice_pg_cam_item *pg_cam; + int status = 0; + u16 node; + u16 pc; + + node = rt->gpr[ICE_GPR_NN_IDX]; + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Start with Node: %u\n", node); + + while (true) { + struct ice_bst_tcam_item *bst; + struct ice_imem_item *imem; + + pc = rt->gpr[ICE_GPR_NP_IDX]; + imem = &psr->imem_table[pc]; + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Load imem at pc: %u\n", + pc); + + ice_bst_key_init(rt, imem); + bst = ice_bst_tcam_match(psr->bst_tcam_table, rt->bst_key); + if (!bst) { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "No Boost TCAM Match\n"); + status = ice_imem_pgk_init(rt, imem); + if (status) + break; + ice_imem_alu0_set(rt, imem); + ice_imem_alu1_set(rt, imem); + ice_imem_alu2_set(rt, imem); + ice_imem_pgp_set(rt, imem); + } else { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Boost TCAM Match address: %u\n", + bst->addr); + if (imem->b_m.pg) { + status = ice_bst_pgk_init(rt, bst); + if (status) + break; + ice_bst_pgp_set(rt, bst); + } else { + status = ice_imem_pgk_init(rt, imem); + if (status) + break; + ice_imem_pgp_set(rt, imem); + } + + if (imem->b_m.alu0) + ice_bst_alu0_set(rt, bst); + else + ice_imem_alu0_set(rt, imem); + + if (imem->b_m.alu1) + ice_bst_alu1_set(rt, bst); + else + ice_imem_alu1_set(rt, imem); + + if (imem->b_m.alu2) + ice_bst_alu2_set(rt, bst); + else + ice_imem_alu2_set(rt, imem); + } + + rt->action = NULL; + pg_cam = ice_rt_pg_cam_match(rt); + if (!pg_cam) { + pg_nm_cam = ice_rt_pg_nm_cam_match(rt); + if (pg_nm_cam) { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Match ParseGraph Nomatch CAM Address %u\n", + pg_nm_cam->idx); + rt->action = &pg_nm_cam->action; + } + } else { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Match ParseGraph CAM Address %u\n", + pg_cam->idx); + rt->action = &pg_cam->action; + } + + if (!rt->action) { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Failed to match ParseGraph CAM, stop parsing.\n"); + status = -EINVAL; + break; + } + + ice_alu_pg_exe(rt); + ice_marker_update(rt); + ice_proto_off_update(rt); + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Go to node %u\n", + rt->action->next_node); + + if (rt->action->is_last_round) { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Last Round in ParseGraph Action, stop parsing.\n"); + break; + } + + if (rt->gpr[ICE_GPR_HO_IDX] >= rt->pkt_len) { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Header Offset (%u) is larger than packet len (%u), stop parsing\n", + rt->gpr[ICE_GPR_HO_IDX], rt->pkt_len); + break; + } + } + + ice_result_resolve(rt, rslt); + + return status; +} diff --git a/drivers/net/ethernet/intel/ice/ice_type.h b/drivers/net/ethernet/intel/ice/ice_type.h index 96037bef3e78..b9e443232335 100644 --- a/drivers/net/ethernet/intel/ice/ice_type.h +++ b/drivers/net/ethernet/intel/ice/ice_type.h @@ -61,6 +61,7 @@ static inline u32 ice_round_to_num(u32 N, u32 R) ICE_DBG_AQ_DESC | \ ICE_DBG_AQ_DESC_BUF | \ ICE_DBG_AQ_CMD) +#define ICE_DBG_PARSER BIT_ULL(28) #define ICE_DBG_USER BIT_ULL(31) diff --git a/drivers/net/ethernet/intel/ice/ice_vf_lib.h b/drivers/net/ethernet/intel/ice/ice_vf_lib.h index fec16919ec19..be4266899690 100644 --- a/drivers/net/ethernet/intel/ice/ice_vf_lib.h +++ b/drivers/net/ethernet/intel/ice/ice_vf_lib.h @@ -12,6 +12,7 @@ #include <net/devlink.h> #include <linux/avf/virtchnl.h> #include "ice_type.h" +#include "ice_flow.h" #include "ice_virtchnl_fdir.h" #include "ice_vsi_vlan_ops.h" @@ -52,6 +53,12 @@ struct ice_mdd_vf_events { u16 last_printed; }; +/* Structure to store fdir fv entry */ +struct ice_fdir_prof_info { + struct ice_parser_profile prof; + u64 fdir_active_cnt; +}; + /* VF operations */ struct ice_vf_ops { enum ice_disq_rst_src reset_type; @@ -91,6 +98,7 @@ struct ice_vf { u16 lan_vsi_idx; /* index into PF struct */ u16 ctrl_vsi_idx; struct ice_vf_fdir fdir; + struct ice_fdir_prof_info fdir_prof_info[ICE_MAX_PTGS]; /* first vector index of this VF in the PF space */ int first_vector_idx; struct ice_sw *vf_sw_id; /* switch ID the VF VSIs connect to */ diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl.c b/drivers/net/ethernet/intel/ice/ice_virtchnl.c index 1c6ce0c4ed4e..59f62306b9cb 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl.c +++ b/drivers/net/ethernet/intel/ice/ice_virtchnl.c @@ -461,6 +461,10 @@ static int ice_vc_get_vf_res_msg(struct ice_vf *vf, u8 *msg) if (vf->driver_caps & VIRTCHNL_VF_OFFLOAD_FDIR_PF) vfres->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_FDIR_PF; + if (vf->driver_caps & VIRTCHNL_VF_OFFLOAD_TC_U32 && + vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_FDIR_PF) + vfres->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_TC_U32; + if (vf->driver_caps & VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2) vfres->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2; diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_fdir.c b/drivers/net/ethernet/intel/ice/ice_virtchnl_fdir.c index b4feb0927687..14e3f0f89c78 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl_fdir.c +++ b/drivers/net/ethernet/intel/ice/ice_virtchnl_fdir.c @@ -26,6 +26,15 @@ enum ice_fdir_tunnel_type { ICE_FDIR_TUNNEL_TYPE_NONE = 0, ICE_FDIR_TUNNEL_TYPE_GTPU, ICE_FDIR_TUNNEL_TYPE_GTPU_EH, + ICE_FDIR_TUNNEL_TYPE_ECPRI, + ICE_FDIR_TUNNEL_TYPE_GTPU_INNER, + ICE_FDIR_TUNNEL_TYPE_GTPU_EH_INNER, + ICE_FDIR_TUNNEL_TYPE_GRE, + ICE_FDIR_TUNNEL_TYPE_GTPOGRE, + ICE_FDIR_TUNNEL_TYPE_GTPOGRE_INNER, + ICE_FDIR_TUNNEL_TYPE_GRE_INNER, + ICE_FDIR_TUNNEL_TYPE_L2TPV2, + ICE_FDIR_TUNNEL_TYPE_L2TPV2_INNER, }; struct virtchnl_fdir_fltr_conf { @@ -33,6 +42,11 @@ struct virtchnl_fdir_fltr_conf { enum ice_fdir_tunnel_type ttype; u64 inset_flag; u32 flow_id; + + struct ice_parser_profile *prof; + bool parser_ena; + u8 *pkt_buf; + u8 pkt_len; }; struct virtchnl_fdir_inset_map { @@ -787,6 +801,107 @@ err_exit: } /** + * ice_vc_fdir_is_raw_flow - check if FDIR flow is raw (binary) + * @proto: virtchnl protocol headers + * + * Check if the FDIR rule is raw flow (protocol agnostic flow) or not. Note + * that common FDIR rule must have non-zero proto->count. Thus, we choose the + * tunnel_level and count of proto as the indicators. If both tunnel_level and + * count of proto are zero, this FDIR rule will be regarded as raw flow. + * + * Returns: true if headers describe raw flow, false otherwise. + */ +static bool +ice_vc_fdir_is_raw_flow(struct virtchnl_proto_hdrs *proto) +{ + return (proto->tunnel_level == 0 && proto->count == 0); +} + +/** + * ice_vc_fdir_parse_raw - parse a virtchnl raw FDIR rule + * @vf: pointer to the VF info + * @proto: virtchnl protocol headers + * @conf: FDIR configuration for each filter + * + * Parse the virtual channel filter's raw flow and store it in @conf + * + * Return: 0 on success or negative errno on failure. + */ +static int +ice_vc_fdir_parse_raw(struct ice_vf *vf, + struct virtchnl_proto_hdrs *proto, + struct virtchnl_fdir_fltr_conf *conf) +{ + u8 *pkt_buf, *msk_buf __free(kfree); + struct ice_parser_result rslt; + struct ice_pf *pf = vf->pf; + struct ice_parser *psr; + int status = -ENOMEM; + struct ice_hw *hw; + u16 udp_port = 0; + + pkt_buf = kzalloc(proto->raw.pkt_len, GFP_KERNEL); + msk_buf = kzalloc(proto->raw.pkt_len, GFP_KERNEL); + if (!pkt_buf || !msk_buf) + goto err_mem_alloc; + + memcpy(pkt_buf, proto->raw.spec, proto->raw.pkt_len); + memcpy(msk_buf, proto->raw.mask, proto->raw.pkt_len); + + hw = &pf->hw; + + /* Get raw profile info via Parser Lib */ + psr = ice_parser_create(hw); + if (IS_ERR(psr)) { + status = PTR_ERR(psr); + goto err_mem_alloc; + } + + ice_parser_dvm_set(psr, ice_is_dvm_ena(hw)); + + if (ice_get_open_tunnel_port(hw, &udp_port, TNL_VXLAN)) + ice_parser_vxlan_tunnel_set(psr, udp_port, true); + + status = ice_parser_run(psr, pkt_buf, proto->raw.pkt_len, &rslt); + if (status) + goto err_parser_destroy; + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_parser_result_dump(hw, &rslt); + + conf->prof = kzalloc(sizeof(*conf->prof), GFP_KERNEL); + if (!conf->prof) { + status = -ENOMEM; + goto err_parser_destroy; + } + + status = ice_parser_profile_init(&rslt, pkt_buf, msk_buf, + proto->raw.pkt_len, ICE_BLK_FD, + conf->prof); + if (status) + goto err_parser_profile_init; + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_parser_profile_dump(hw, conf->prof); + + /* Store raw flow info into @conf */ + conf->pkt_len = proto->raw.pkt_len; + conf->pkt_buf = pkt_buf; + conf->parser_ena = true; + + ice_parser_destroy(psr); + return 0; + +err_parser_profile_init: + kfree(conf->prof); +err_parser_destroy: + ice_parser_destroy(psr); +err_mem_alloc: + kfree(pkt_buf); + return status; +} + +/** * ice_vc_fdir_parse_pattern * @vf: pointer to the VF info * @fltr: virtual channel add cmd buffer @@ -813,6 +928,10 @@ ice_vc_fdir_parse_pattern(struct ice_vf *vf, struct virtchnl_fdir_add *fltr, return -EINVAL; } + /* For raw FDIR filters created by the parser */ + if (ice_vc_fdir_is_raw_flow(proto)) + return ice_vc_fdir_parse_raw(vf, proto, conf); + for (i = 0; i < proto->count; i++) { struct virtchnl_proto_hdr *hdr = &proto->proto_hdr[i]; struct ip_esp_hdr *esph; @@ -1101,8 +1220,10 @@ ice_vc_validate_fdir_fltr(struct ice_vf *vf, struct virtchnl_fdir_add *fltr, struct virtchnl_proto_hdrs *proto = &fltr->rule_cfg.proto_hdrs; int ret; - if (!ice_vc_validate_pattern(vf, proto)) - return -EINVAL; + /* For raw FDIR filters created by the parser */ + if (!ice_vc_fdir_is_raw_flow(proto)) + if (!ice_vc_validate_pattern(vf, proto)) + return -EINVAL; ret = ice_vc_fdir_parse_pattern(vf, fltr, conf); if (ret) @@ -1295,11 +1416,15 @@ static int ice_vc_fdir_write_fltr(struct ice_vf *vf, return -ENOMEM; ice_fdir_get_prgm_desc(hw, input, &desc, add); - ret = ice_fdir_get_gen_prgm_pkt(hw, input, pkt, false, is_tun); - if (ret) { - dev_dbg(dev, "Gen training pkt for VF %d ptype %d failed\n", - vf->vf_id, input->flow_type); - goto err_free_pkt; + if (conf->parser_ena) { + memcpy(pkt, conf->pkt_buf, conf->pkt_len); + } else { + ret = ice_fdir_get_gen_prgm_pkt(hw, input, pkt, false, is_tun); + if (ret) { + dev_dbg(dev, "Gen training pkt for VF %d ptype %d failed\n", + vf->vf_id, input->flow_type); + goto err_free_pkt; + } } ret = ice_prgm_fdir_fltr(ctrl_vsi, &desc, pkt); @@ -1521,6 +1646,16 @@ err_exit: return ret; } +static int ice_fdir_is_tunnel(enum ice_fdir_tunnel_type ttype) +{ + return (ttype == ICE_FDIR_TUNNEL_TYPE_GRE_INNER || + ttype == ICE_FDIR_TUNNEL_TYPE_GTPU_INNER || + ttype == ICE_FDIR_TUNNEL_TYPE_GTPU_EH_INNER || + ttype == ICE_FDIR_TUNNEL_TYPE_GTPOGRE_INNER || + ttype == ICE_FDIR_TUNNEL_TYPE_ECPRI || + ttype == ICE_FDIR_TUNNEL_TYPE_L2TPV2_INNER); +} + /** * ice_vc_add_fdir_fltr_post * @vf: pointer to the VF structure @@ -1782,6 +1917,158 @@ static void ice_vc_fdir_clear_irq_ctx(struct ice_vf *vf) } /** + * ice_vc_parser_fv_check_diff - check two parsed FDIR profile fv context + * @fv_a: struct of parsed FDIR profile field vector + * @fv_b: struct of parsed FDIR profile field vector + * + * Check if the two parsed FDIR profile field vector context are different, + * including proto_id, offset and mask. + * + * Return: true on different, false on otherwise. + */ +static bool ice_vc_parser_fv_check_diff(struct ice_parser_fv *fv_a, + struct ice_parser_fv *fv_b) +{ + return (fv_a->proto_id != fv_b->proto_id || + fv_a->offset != fv_b->offset || + fv_a->msk != fv_b->msk); +} + +/** + * ice_vc_parser_fv_save - save parsed FDIR profile fv context + * @fv: struct of parsed FDIR profile field vector + * @fv_src: parsed FDIR profile field vector context to save + * + * Save the parsed FDIR profile field vector context, including proto_id, + * offset and mask. + * + * Return: Void. + */ +static void ice_vc_parser_fv_save(struct ice_parser_fv *fv, + struct ice_parser_fv *fv_src) +{ + fv->proto_id = fv_src->proto_id; + fv->offset = fv_src->offset; + fv->msk = fv_src->msk; + fv->spec = 0; +} + +/** + * ice_vc_add_fdir_raw - add a raw FDIR filter for VF + * @vf: pointer to the VF info + * @conf: FDIR configuration for each filter + * @v_ret: the final VIRTCHNL code + * @stat: pointer to the VIRTCHNL_OP_ADD_FDIR_FILTER + * @len: length of the stat + * + * Return: 0 on success or negative errno on failure. + */ +static int +ice_vc_add_fdir_raw(struct ice_vf *vf, + struct virtchnl_fdir_fltr_conf *conf, + enum virtchnl_status_code *v_ret, + struct virtchnl_fdir_add *stat, int len) +{ + struct ice_vsi *vf_vsi, *ctrl_vsi; + struct ice_fdir_prof_info *pi; + struct ice_pf *pf = vf->pf; + int ret, ptg, id, i; + struct device *dev; + struct ice_hw *hw; + bool fv_found; + + dev = ice_pf_to_dev(pf); + hw = &pf->hw; + *v_ret = VIRTCHNL_STATUS_ERR_PARAM; + stat->status = VIRTCHNL_FDIR_FAILURE_RULE_NORESOURCE; + + id = find_first_bit(conf->prof->ptypes, ICE_FLOW_PTYPE_MAX); + ptg = hw->blk[ICE_BLK_FD].xlt1.t[id]; + + vf_vsi = ice_get_vf_vsi(vf); + if (!vf_vsi) { + dev_err(dev, "Can not get FDIR vf_vsi for VF %d\n", vf->vf_id); + return -ENODEV; + } + + ctrl_vsi = pf->vsi[vf->ctrl_vsi_idx]; + if (!ctrl_vsi) { + dev_err(dev, "Can not get FDIR ctrl_vsi for VF %d\n", + vf->vf_id); + return -ENODEV; + } + + fv_found = false; + + /* Check if profile info already exists, then update the counter */ + pi = &vf->fdir_prof_info[ptg]; + if (pi->fdir_active_cnt != 0) { + for (i = 0; i < ICE_MAX_FV_WORDS; i++) + if (ice_vc_parser_fv_check_diff(&pi->prof.fv[i], + &conf->prof->fv[i])) + break; + if (i == ICE_MAX_FV_WORDS) { + fv_found = true; + pi->fdir_active_cnt++; + } + } + + /* HW profile setting is only required for the first time */ + if (!fv_found) { + ret = ice_flow_set_parser_prof(hw, vf_vsi->idx, + ctrl_vsi->idx, conf->prof, + ICE_BLK_FD); + + if (ret) { + *v_ret = VIRTCHNL_STATUS_ERR_NO_MEMORY; + dev_dbg(dev, "VF %d: insert hw prof failed\n", + vf->vf_id); + return ret; + } + } + + ret = ice_vc_fdir_insert_entry(vf, conf, &conf->flow_id); + if (ret) { + *v_ret = VIRTCHNL_STATUS_ERR_NO_MEMORY; + dev_dbg(dev, "VF %d: insert FDIR list failed\n", + vf->vf_id); + return ret; + } + + ret = ice_vc_fdir_set_irq_ctx(vf, conf, + VIRTCHNL_OP_ADD_FDIR_FILTER); + if (ret) { + dev_dbg(dev, "VF %d: set FDIR context failed\n", + vf->vf_id); + goto err_rem_entry; + } + + ret = ice_vc_fdir_write_fltr(vf, conf, true, false); + if (ret) { + dev_err(dev, "VF %d: adding FDIR raw flow rule failed, ret:%d\n", + vf->vf_id, ret); + goto err_clr_irq; + } + + /* Save parsed profile fv info of the FDIR rule for the first time */ + if (!fv_found) { + for (i = 0; i < conf->prof->fv_num; i++) + ice_vc_parser_fv_save(&pi->prof.fv[i], + &conf->prof->fv[i]); + pi->prof.fv_num = conf->prof->fv_num; + pi->fdir_active_cnt = 1; + } + + return 0; + +err_clr_irq: + ice_vc_fdir_clear_irq_ctx(vf); +err_rem_entry: + ice_vc_fdir_remove_entry(vf, conf, conf->flow_id); + return ret; +} + +/** * ice_vc_add_fdir_fltr - add a FDIR filter for VF by the msg buffer * @vf: pointer to the VF info * @msg: pointer to the msg buffer @@ -1846,7 +2133,7 @@ int ice_vc_add_fdir_fltr(struct ice_vf *vf, u8 *msg) len = sizeof(*stat); ret = ice_vc_validate_fdir_fltr(vf, fltr, conf); if (ret) { - v_ret = VIRTCHNL_STATUS_SUCCESS; + v_ret = VIRTCHNL_STATUS_ERR_PARAM; stat->status = VIRTCHNL_FDIR_FAILURE_RULE_INVALID; dev_dbg(dev, "Invalid FDIR filter from VF %d\n", vf->vf_id); goto err_free_conf; @@ -1861,6 +2148,15 @@ int ice_vc_add_fdir_fltr(struct ice_vf *vf, u8 *msg) goto exit; } + /* For raw FDIR filters created by the parser */ + if (conf->parser_ena) { + ret = ice_vc_add_fdir_raw(vf, conf, &v_ret, stat, len); + if (ret) + goto err_free_conf; + goto exit; + } + + is_tun = ice_fdir_is_tunnel(conf->ttype); ret = ice_vc_fdir_config_input_set(vf, fltr, conf, is_tun); if (ret) { v_ret = VIRTCHNL_STATUS_SUCCESS; @@ -1922,6 +2218,78 @@ err_exit: } /** + * ice_vc_del_fdir_raw - delete a raw FDIR filter for VF + * @vf: pointer to the VF info + * @conf: FDIR configuration for each filter + * @v_ret: the final VIRTCHNL code + * @stat: pointer to the VIRTCHNL_OP_DEL_FDIR_FILTER + * @len: length of the stat + * + * Return: 0 on success or negative errno on failure. + */ +static int +ice_vc_del_fdir_raw(struct ice_vf *vf, + struct virtchnl_fdir_fltr_conf *conf, + enum virtchnl_status_code *v_ret, + struct virtchnl_fdir_del *stat, int len) +{ + struct ice_vsi *vf_vsi, *ctrl_vsi; + enum ice_block blk = ICE_BLK_FD; + struct ice_fdir_prof_info *pi; + struct ice_pf *pf = vf->pf; + struct device *dev; + struct ice_hw *hw; + unsigned long id; + u16 vsi_num; + int ptg; + int ret; + + dev = ice_pf_to_dev(pf); + hw = &pf->hw; + *v_ret = VIRTCHNL_STATUS_ERR_PARAM; + stat->status = VIRTCHNL_FDIR_FAILURE_RULE_NORESOURCE; + + id = find_first_bit(conf->prof->ptypes, ICE_FLOW_PTYPE_MAX); + ptg = hw->blk[ICE_BLK_FD].xlt1.t[id]; + + ret = ice_vc_fdir_write_fltr(vf, conf, false, false); + if (ret) { + dev_err(dev, "VF %u: deleting FDIR raw flow rule failed: %d\n", + vf->vf_id, ret); + return ret; + } + + vf_vsi = ice_get_vf_vsi(vf); + if (!vf_vsi) { + dev_err(dev, "Can not get FDIR vf_vsi for VF %u\n", vf->vf_id); + return -ENODEV; + } + + ctrl_vsi = pf->vsi[vf->ctrl_vsi_idx]; + if (!ctrl_vsi) { + dev_err(dev, "Can not get FDIR ctrl_vsi for VF %u\n", + vf->vf_id); + return -ENODEV; + } + + pi = &vf->fdir_prof_info[ptg]; + if (pi->fdir_active_cnt != 0) { + pi->fdir_active_cnt--; + /* Remove the profile id flow if no active FDIR rule left */ + if (!pi->fdir_active_cnt) { + vsi_num = ice_get_hw_vsi_num(hw, ctrl_vsi->idx); + ice_rem_prof_id_flow(hw, blk, vsi_num, id); + + vsi_num = ice_get_hw_vsi_num(hw, vf_vsi->idx); + ice_rem_prof_id_flow(hw, blk, vsi_num, id); + } + } + + conf->parser_ena = false; + return 0; +} + +/** * ice_vc_del_fdir_fltr - delete a FDIR filter for VF by the msg buffer * @vf: pointer to the VF info * @msg: pointer to the msg buffer @@ -1933,7 +2301,10 @@ int ice_vc_del_fdir_fltr(struct ice_vf *vf, u8 *msg) struct virtchnl_fdir_del *fltr = (struct virtchnl_fdir_del *)msg; struct virtchnl_fdir_del *stat = NULL; struct virtchnl_fdir_fltr_conf *conf; + struct ice_vf_fdir *fdir = &vf->fdir; enum virtchnl_status_code v_ret; + struct ice_fdir_fltr *input; + enum ice_fltr_ptype flow; struct device *dev; struct ice_pf *pf; int is_tun = 0; @@ -1983,6 +2354,15 @@ int ice_vc_del_fdir_fltr(struct ice_vf *vf, u8 *msg) goto err_exit; } + /* For raw FDIR filters created by the parser */ + if (conf->parser_ena) { + ret = ice_vc_del_fdir_raw(vf, conf, &v_ret, stat, len); + if (ret) + goto err_del_tmr; + goto exit; + } + + is_tun = ice_fdir_is_tunnel(conf->ttype); ret = ice_vc_fdir_write_fltr(vf, conf, false, is_tun); if (ret) { v_ret = VIRTCHNL_STATUS_SUCCESS; @@ -1992,6 +2372,13 @@ int ice_vc_del_fdir_fltr(struct ice_vf *vf, u8 *msg) goto err_del_tmr; } + /* Remove unused profiles to avoid unexpected behaviors */ + input = &conf->input; + flow = input->flow_type; + if (fdir->fdir_fltr_cnt[flow][is_tun] == 1) + ice_vc_fdir_rem_prof(vf, flow, is_tun); + +exit: kfree(stat); return ret; diff --git a/include/linux/avf/virtchnl.h b/include/linux/avf/virtchnl.h index 8e177b67e82f..f41395264dca 100644 --- a/include/linux/avf/virtchnl.h +++ b/include/linux/avf/virtchnl.h @@ -247,6 +247,7 @@ VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_vsi_resource); /* used to negotiate communicating link speeds in Mbps */ #define VIRTCHNL_VF_CAP_ADV_LINK_SPEED BIT(7) #define VIRTCHNL_VF_OFFLOAD_CRC BIT(10) +#define VIRTCHNL_VF_OFFLOAD_TC_U32 BIT(11) #define VIRTCHNL_VF_OFFLOAD_VLAN_V2 BIT(15) #define VIRTCHNL_VF_OFFLOAD_VLAN BIT(16) #define VIRTCHNL_VF_OFFLOAD_RX_POLLING BIT(17) @@ -1121,6 +1122,7 @@ enum virtchnl_vfr_states { }; #define VIRTCHNL_MAX_NUM_PROTO_HDRS 32 +#define VIRTCHNL_MAX_SIZE_RAW_PACKET 1024 #define PROTO_HDR_SHIFT 5 #define PROTO_HDR_FIELD_START(proto_hdr_type) ((proto_hdr_type) << PROTO_HDR_SHIFT) #define PROTO_HDR_FIELD_MASK ((1UL << PROTO_HDR_SHIFT) - 1) @@ -1266,13 +1268,22 @@ struct virtchnl_proto_hdrs { u8 pad[3]; /** * specify where protocol header start from. + * must be 0 when sending a raw packet request. * 0 - from the outer layer * 1 - from the first inner layer * 2 - from the second inner layer * .... **/ int count; /* the proto layers must < VIRTCHNL_MAX_NUM_PROTO_HDRS */ - struct virtchnl_proto_hdr proto_hdr[VIRTCHNL_MAX_NUM_PROTO_HDRS]; + union { + struct virtchnl_proto_hdr + proto_hdr[VIRTCHNL_MAX_NUM_PROTO_HDRS]; + struct { + u16 pkt_len; + u8 spec[VIRTCHNL_MAX_SIZE_RAW_PACKET]; + u8 mask[VIRTCHNL_MAX_SIZE_RAW_PACKET]; + } raw; + }; }; VIRTCHNL_CHECK_STRUCT_LEN(2312, virtchnl_proto_hdrs); |